drive: add --drive-server-side-across-configs

In #2728 and 55b9a4e we decided to allow server side operations
between google drives with different configurations.

This works in some cases (eg between teamdrives) but does not work in
the general case, and this caused breakage in quite a number of
people's workflows.

This change makes the feature conditional on the
--drive-server-side-across-configs flag which defaults to off.

See: https://forum.rclone.org/t/gdrive-to-gdrive-error-404-file-not-found/9621/10

Fixes #3119

.golangci.yml

@@ -1,9 +1,5 @@
 # golangci-lint configuration options
 
-run:
-  build-tags:
-    - cmount
-
 linters:
   enable:
     - deadcode

.travis.yml

@@ -1,54 +1,100 @@
+---
 language: go
 sudo: required
-dist: trusty
+dist: xenial
 os:
-- linux
-go:
-- 1.8.x
-- 1.9.x
-- 1.10.x
-- 1.11.x
-- 1.12.x
-- tip
+  - linux
 go_import_path: github.com/ncw/rclone
 before_install:
-- if [[ $TRAVIS_OS_NAME == linux ]]; then sudo modprobe fuse ; sudo chmod 666 /dev/fuse ; sudo chown root:$USER /etc/fuse.conf ; fi
-- if [[ $TRAVIS_OS_NAME == osx ]]; then brew update && brew tap caskroom/cask && brew cask install osxfuse ; fi
+  - git fetch --unshallow --tags
+  - |
+    if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
+      sudo modprobe fuse
+      sudo chmod 666 /dev/fuse
+      sudo chown root:$USER /etc/fuse.conf
+    fi
+    if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
+      brew update
+      brew tap caskroom/cask
+      brew cask install osxfuse
+    fi
+    if [[ "$TRAVIS_OS_NAME" == "windows" ]]; then
+      choco install -y winfsp zip make
+      cd ../..                  # fix crlf in git checkout
+      mv $TRAVIS_REPO_SLUG _old
+      git config --global core.autocrlf false
+      git clone _old $TRAVIS_REPO_SLUG
+      cd $TRAVIS_REPO_SLUG
+    fi
 install:
-- git fetch --unshallow --tags
-- make vars
-- make build_dep
-script:
-- make check
-- make quicktest
-- make compile_all
+  - make vars
 env:
   global:
     - GOTAGS=cmount
     - GO111MODULE=off
+    - GITHUB_USER=ncw
     - secure: gU8gCV9R8Kv/Gn0SmCP37edpfIbPoSvsub48GK7qxJdTU628H0KOMiZW/T0gtV5d67XJZ4eKnhJYlxwwxgSgfejO32Rh5GlYEKT/FuVoH0BD72dM1GDFLSrUiUYOdoHvf/BKIFA3dJFT4lk2ASy4Zh7SEoXHG6goBlqUpYx8hVA=
-    - secure: AMjrMAksDy3QwqGqnvtUg8FL/GNVgNqTqhntLF9HSU0njHhX6YurGGnfKdD9vNHlajPQOewvmBjwNLcDWGn2WObdvmh9Ohep0EmOjZ63kliaRaSSQueSd8y0idfqMQAxep0SObOYbEDVmQh0RCAE9wOVKRaPgw98XvgqWGDq5Tw=
     - secure: Uaiveq+/rvQjO03GzvQZV2J6pZfedoFuhdXrLVhhHSeP4ZBca0olw7xaqkabUyP3LkVYXMDSX8EbyeuQT1jfEe5wp5sBdfaDtuYW6heFyjiHIIIbVyBfGXon6db4ETBjOaX/Xt8uktrgNge6qFlj+kpnmpFGxf0jmDLw1zgg7tk=
 addons:
   apt:
     packages:
-    - fuse
-    - libfuse-dev
-    - rpm
-    - pkg-config
+      - fuse
+      - libfuse-dev
+      - rpm
+      - pkg-config
 cache:
   directories:
     - $HOME/.cache/go-build
 matrix:
   allow_failures:
-  - go: tip
+    - go: tip
   include:
-  - os: osx
-    go: 1.12.x
-    env: GOTAGS=""
-    cache:
-      directories:
-        - $HOME/Library/Caches/go-build
+    - go: 1.9.x
+      script:
+        - make quicktest
+    - go: 1.10.x
+      script:
+        - make quicktest
+    - go: 1.11.x
+      script:
+        - make quicktest
+    - go: 1.12.x
+      env:
+        - GOTAGS=cmount
+      script:
+        - make build_dep
+        - make check
+        - make quicktest
+        - make racequicktest
+        - make compile_all
+    - os: osx
+      go: 1.12.x
+      env:
+        - GOTAGS=  # cmount doesn't work on osx travis for some reason
+      cache:
+        directories:
+          - $HOME/Library/Caches/go-build
+      script:
+        - make
+        - make quicktest
+        - make racequicktest
+    # - os: windows
+    #   go: 1.12.x
+    #   env:
+    #     - GOTAGS=cmount
+    #     - CPATH='C:\Program Files (x86)\WinFsp\inc\fuse'
+    #   #filter_secrets: false     # works around a problem with secrets under windows
+    #   cache:
+    #     directories:
+    #       - ${LocalAppData}/go-build
+    #   script:
+    #     - make
+    #     - make quicktest
+    #     - make racequicktest
+    - go: tip
+      script:
+        - make quicktest
+
 deploy:
   provider: script
   script: make travis_beta
@@ -57,4 +103,4 @@ deploy:
     repo: ncw/rclone
     all_branches: true
     go: 1.12.x
-    condition: $TRAVIS_PULL_REQUEST == false
+    condition: $TRAVIS_PULL_REQUEST == false && $TRAVIS_OS_NAME != "windows"

CONTRIBUTING.md

@@ -135,7 +135,7 @@ then change into the project root and run
 
     make test
 
-This command is run daily on the the integration test server. You can
+This command is run daily on the integration test server. You can
 find the results at https://pub.rclone.org/integration-tests/
 
 ## Code Organisation ##

Makefile

@@ -17,8 +17,6 @@ ifneq ($(TAG),$(LAST_TAG))
 endif
 GO_VERSION := $(shell go version)
 GO_FILES := $(shell go list ./... | grep -v /vendor/ )
-# Run full tests if go >= go1.12
-FULL_TESTS := $(shell go version | perl -lne 'print "go$$1.$$2" if /go(\d+)\.(\d+)/ && ($$1 > 1 || $$2 >= 12)')
 BETA_PATH := $(BRANCH_PATH)$(TAG)
 BETA_URL := https://beta.rclone.org/$(BETA_PATH)/
 BETA_UPLOAD_ROOT := memstore:beta-rclone-org
@@ -26,6 +24,7 @@ BETA_UPLOAD := $(BETA_UPLOAD_ROOT)/$(BETA_PATH)
 # Pass in GOTAGS=xyz on the make command line to set build tags
 ifdef GOTAGS
 BUILDTAGS=-tags "$(GOTAGS)"
+LINTTAGS=--build-tags "$(GOTAGS)"
 endif
 
 .PHONY: rclone vars version
@@ -42,7 +41,6 @@ vars:
 	@echo LAST_TAG="'$(LAST_TAG)'"
 	@echo NEW_TAG="'$(NEW_TAG)'"
 	@echo GO_VERSION="'$(GO_VERSION)'"
-	@echo FULL_TESTS="'$(FULL_TESTS)'"
 	@echo BETA_URL="'$(BETA_URL)'"
 
 version:
@@ -57,28 +55,19 @@ test:	rclone
 # Quick test
 quicktest:
 	RCLONE_CONFIG="/notfound" go test $(BUILDTAGS) $(GO_FILES)
-ifdef FULL_TESTS
+
+racequicktest:
 	RCLONE_CONFIG="/notfound" go test $(BUILDTAGS) -cpu=2 -race $(GO_FILES)
-endif
 
 # Do source code quality checks
 check:	rclone
-ifdef FULL_TESTS
-	@# we still run go vet for -printfuncs which golangci-lint doesn't do yet
-	@# see: https://github.com/golangci/golangci-lint/issues/204
 	@echo "-- START CODE QUALITY REPORT -------------------------------"
-	@go vet $(BUILDTAGS) -printfuncs Debugf,Infof,Logf,Errorf ./...
-	@golangci-lint run ./...
+	@golangci-lint run $(LINTTAGS) ./...
 	@echo "-- END CODE QUALITY REPORT ---------------------------------"
-else
-	@echo Skipping source quality tests as version of go too old
-endif
 
 # Get the build dependencies
 build_dep:
-ifdef FULL_TESTS
 	go run bin/get-github-release.go -extract golangci-lint golangci/golangci-lint 'golangci-lint-.*\.tar\.gz'
-endif
 
 # Get the release dependencies
 release_dep:
@@ -109,7 +98,7 @@ commanddocs: rclone
 	XDG_CACHE_HOME="" XDG_CONFIG_HOME="" HOME="\$$HOME" USER="\$$USER" rclone gendocs docs/content/commands/
 
 backenddocs: rclone bin/make_backend_docs.py
-	./bin/make_backend_docs.py
+	XDG_CACHE_HOME="" XDG_CONFIG_HOME="" HOME="\$$HOME" USER="\$$USER" ./bin/make_backend_docs.py
 
 rcdocs: rclone
 	bin/make_rc_docs.sh
@@ -162,11 +151,7 @@ log_since_last_release:
 	git log $(LAST_TAG)..
 
 compile_all:
-ifdef FULL_TESTS
 	go run bin/cross-compile.go -parallel 8 -compile-only $(BUILDTAGS) $(TAG)
-else
-	@echo Skipping compile all as version of go too old
-endif
 
 appveyor_upload:
 	rclone --config bin/travis.rclone.conf -v copy --exclude '*beta-latest*' build/ $(BETA_UPLOAD)
@@ -186,6 +171,11 @@ BUILD_FLAGS := -exclude "^(windows|darwin)/"
 ifeq ($(TRAVIS_OS_NAME),osx)
 	BUILD_FLAGS := -include "^darwin/" -cgo
 endif
+ifeq ($(TRAVIS_OS_NAME),windows)
+# BUILD_FLAGS := -include "^windows/" -cgo
+# 386 doesn't build yet
+	BUILD_FLAGS := -include "^windows/amd64" -cgo
+endif
 
 travis_beta:
 ifeq ($(TRAVIS_OS_NAME),linux)
@@ -201,7 +191,7 @@ endif
 
 # Fetch the binary builds from travis and appveyor
 fetch_binaries:
-	rclone -P sync $(BETA_UPLOAD) build/
+	rclone -P sync --exclude "/testbuilds/**" --delete-excluded $(BETA_UPLOAD) build/
 
 serve:	website
 	cd docs && hugo server -v -w

README.md

@@ -7,11 +7,11 @@
 [Changelog](https://rclone.org/changelog/) |
 [Installation](https://rclone.org/install/) |
 [Forum](https://forum.rclone.org/) | 
-[G+](https://google.com/+RcloneOrg)
 
 [![Build Status](https://travis-ci.org/ncw/rclone.svg?branch=master)](https://travis-ci.org/ncw/rclone)
 [![Windows Build Status](https://ci.appveyor.com/api/projects/status/github/ncw/rclone?branch=master&passingText=windows%20-%20ok&svg=true)](https://ci.appveyor.com/project/ncw/rclone)
 [![CircleCI](https://circleci.com/gh/ncw/rclone/tree/master.svg?style=svg)](https://circleci.com/gh/ncw/rclone/tree/master)
+[![Go Report Card](https://goreportcard.com/badge/github.com/ncw/rclone)](https://goreportcard.com/report/github.com/ncw/rclone)
 [![GoDoc](https://godoc.org/github.com/ncw/rclone?status.svg)](https://godoc.org/github.com/ncw/rclone) 
 
 # Rclone
@@ -73,6 +73,8 @@ Please see [the full list of all storage providers and their features](https://r
   * Optional encryption ([Crypt](https://rclone.org/crypt/))
   * Optional cache ([Cache](https://rclone.org/cache/))
   * Optional FUSE mount ([rclone mount](https://rclone.org/commands/rclone_mount/))
+  * Multi-threaded downloads to local disk
+  * Can [serve](https://rclone.org/commands/rclone_serve/) local or remote files over HTTP/WebDav/FTP/SFTP/dlna
 
 ## Installation & documentation
 

RELEASE.md

@@ -11,7 +11,7 @@ Making a release
   * edit docs/content/changelog.md
   * make doc
   * git status - to check for new man pages - git add them
-  * git commit -a -v -m "Version v1.XX"
+  * git commit -a -v -m "Version v1.XX.0"
   * make retag
   * git push --tags origin master
   * # Wait for the appveyor and travis builds to complete then...
@@ -27,6 +27,7 @@ Making a release
 
 Early in the next release cycle update the vendored dependencies
   * Review any pinned packages in go.mod and remove if possible
+      * GO111MODULE=on go get -u github.com/spf13/cobra@master
   * make update
   * git status
   * git add new files

backend/alias/alias.go

@@ -14,7 +14,7 @@ import (
 func init() {
 	fsi := &fs.RegInfo{
 		Name:        "alias",
-		Description: "Alias for a existing remote",
+		Description: "Alias for an existing remote",
 		NewFs:       NewFs,
 		Options: []fs.Option{{
 			Name:     "remote",

backend/amazonclouddrive/amazonclouddrive.go

@@ -32,7 +32,6 @@ import (
 	"github.com/ncw/rclone/lib/dircache"
 	"github.com/ncw/rclone/lib/oauthutil"
 	"github.com/ncw/rclone/lib/pacer"
-	"github.com/ncw/rclone/lib/rest"
 	"github.com/pkg/errors"
 	"golang.org/x/oauth2"
 )
@@ -1093,7 +1092,7 @@ func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
 		if !bigObject {
 			in, resp, err = file.OpenHeaders(headers)
 		} else {
-			in, resp, err = file.OpenTempURLHeaders(rest.ClientWithHeaderReset(o.fs.noAuthClient, headers), headers)
+			in, resp, err = file.OpenTempURLHeaders(o.fs.noAuthClient, headers)
 		}
 		return o.fs.shouldRetry(resp, err)
 	})

backend/azureblob/azureblob.go

@@ -1,6 +1,6 @@
 // Package azureblob provides an interface to the Microsoft Azure blob object storage system
 
-// +build !plan9,!solaris,go1.8
+// +build !plan9,!solaris
 
 package azureblob
 

backend/azureblob/azureblob_internal_test.go

@@ -1,4 +1,4 @@
-// +build !plan9,!solaris,go1.8
+// +build !plan9,!solaris
 
 package azureblob
 

backend/azureblob/azureblob_test.go

@@ -1,6 +1,6 @@
 // Test AzureBlob filesystem interface
 
-// +build !plan9,!solaris,go1.8
+// +build !plan9,!solaris
 
 package azureblob
 

backend/azureblob/azureblob_unsupported.go

@@ -1,6 +1,6 @@
 // Build for azureblob for unsupported platforms to stop go complaining
 // about "no buildable Go source files "
 
-// +build plan9 solaris !go1.8
+// +build plan9 solaris
 
 package azureblob

backend/b2/b2.go

@@ -952,6 +952,13 @@ func (f *Fs) hide(Name string) error {
 		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
+		if apiErr, ok := err.(*api.Error); ok {
+			if apiErr.Code == "already_hidden" {
+				// sometimes eventual consistency causes this, so
+				// ignore this error since it is harmless
+				return nil
+			}
+		}
 		return errors.Wrapf(err, "failed to hide %q", Name)
 	}
 	return nil
@@ -1206,7 +1213,7 @@ func (o *Object) parseTimeString(timeString string) (err error) {
 	unixMilliseconds, err := strconv.ParseInt(timeString, 10, 64)
 	if err != nil {
 		fs.Debugf(o, "Failed to parse mod time string %q: %v", timeString, err)
-		return err
+		return nil
 	}
 	o.modTime = time.Unix(unixMilliseconds/1E3, (unixMilliseconds%1E3)*1E6).UTC()
 	return nil

backend/cache/cache.go

@@ -20,6 +20,7 @@ import (
 
 	"github.com/ncw/rclone/backend/crypt"
 	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/cache"
 	"github.com/ncw/rclone/fs/config"
 	"github.com/ncw/rclone/fs/config/configmap"
 	"github.com/ncw/rclone/fs/config/configstruct"
@@ -481,7 +482,7 @@ func NewFs(name, rootPath string, m configmap.Mapper) (fs.Fs, error) {
 			return nil, errors.Wrapf(err, "failed to create cache directory %v", f.opt.TempWritePath)
 		}
 		f.opt.TempWritePath = filepath.ToSlash(f.opt.TempWritePath)
-		f.tempFs, err = fs.NewFs(f.opt.TempWritePath)
+		f.tempFs, err = cache.Get(f.opt.TempWritePath)
 		if err != nil {
 			return nil, errors.Wrapf(err, "failed to create temp fs: %v", err)
 		}
@@ -1191,7 +1192,7 @@ func (f *Fs) Rmdir(dir string) error {
 		}
 
 		var queuedEntries []*Object
-		err = walk.Walk(f.tempFs, dir, true, -1, func(path string, entries fs.DirEntries, err error) error {
+		err = walk.ListR(f.tempFs, dir, true, -1, walk.ListObjects, func(entries fs.DirEntries) error {
 			for _, o := range entries {
 				if oo, ok := o.(fs.Object); ok {
 					co := ObjectFromOriginal(f, oo)
@@ -1287,7 +1288,7 @@ func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error {
 		}
 
 		var queuedEntries []*Object
-		err := walk.Walk(f.tempFs, srcRemote, true, -1, func(path string, entries fs.DirEntries, err error) error {
+		err := walk.ListR(f.tempFs, srcRemote, true, -1, walk.ListObjects, func(entries fs.DirEntries) error {
 			for _, o := range entries {
 				if oo, ok := o.(fs.Object); ok {
 					co := ObjectFromOriginal(f, oo)

backend/cache/cache_test.go

@@ -15,7 +15,9 @@ import (
 // TestIntegration runs integration tests against the remote
 func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
-		RemoteName: "TestCache:",
-		NilObject:  (*cache.Object)(nil),
+		RemoteName:                   "TestCache:",
+		NilObject:                    (*cache.Object)(nil),
+		UnimplementableFsMethods:     []string{"PublicLink", "MergeDirs", "OpenWriterAt"},
+		UnimplementableObjectMethods: []string{"MimeType", "ID", "GetTier", "SetTier"},
 	})
 }

backend/cache/storage_persistent.go

@@ -1023,7 +1023,7 @@ func (b *Persistent) ReconcileTempUploads(cacheFs *Fs) error {
 		}
 
 		var queuedEntries []fs.Object
-		err = walk.Walk(cacheFs.tempFs, "", true, -1, func(path string, entries fs.DirEntries, err error) error {
+		err = walk.ListR(cacheFs.tempFs, "", true, -1, walk.ListObjects, func(entries fs.DirEntries) error {
 			for _, o := range entries {
 				if oo, ok := o.(fs.Object); ok {
 					queuedEntries = append(queuedEntries, oo)

backend/crypt/crypt.go

@@ -169,23 +169,10 @@ func NewFs(name, rpath string, m configmap.Mapper) (fs.Fs, error) {
 		WriteMimeType:           false,
 		BucketBased:             true,
 		CanHaveEmptyDirectories: true,
+		SetTier:                 true,
+		GetTier:                 true,
 	}).Fill(f).Mask(wrappedFs).WrapsFs(f, wrappedFs)
 
-	doChangeNotify := wrappedFs.Features().ChangeNotify
-	if doChangeNotify != nil {
-		f.features.ChangeNotify = func(notifyFunc func(string, fs.EntryType), pollInterval <-chan time.Duration) {
-			wrappedNotifyFunc := func(path string, entryType fs.EntryType) {
-				decrypted, err := f.DecryptFileName(path)
-				if err != nil {
-					fs.Logf(f, "ChangeNotify was unable to decrypt %q: %s", path, err)
-					return
-				}
-				notifyFunc(decrypted, entryType)
-			}
-			doChangeNotify(wrappedNotifyFunc, pollInterval)
-		}
-	}
-
 	return f, err
 }
 
@@ -202,6 +189,7 @@ type Options struct {
 // Fs represents a wrapped fs.Fs
 type Fs struct {
 	fs.Fs
+	wrapper  fs.Fs
 	name     string
 	root     string
 	opt      Options
@@ -544,6 +532,16 @@ func (f *Fs) UnWrap() fs.Fs {
 	return f.Fs
 }
 
+// WrapFs returns the Fs that is wrapping this Fs
+func (f *Fs) WrapFs() fs.Fs {
+	return f.wrapper
+}
+
+// SetWrapper sets the Fs that is wrapping this Fs
+func (f *Fs) SetWrapper(wrapper fs.Fs) {
+	f.wrapper = wrapper
+}
+
 // EncryptFileName returns an encrypted file name
 func (f *Fs) EncryptFileName(fileName string) string {
 	return f.cipher.EncryptFileName(fileName)
@@ -616,6 +614,75 @@ func (f *Fs) ComputeHash(o *Object, src fs.Object, hashType hash.Type) (hashStr
 	return m.Sums()[hashType], nil
 }
 
+// MergeDirs merges the contents of all the directories passed
+// in into the first one and rmdirs the other directories.
+func (f *Fs) MergeDirs(dirs []fs.Directory) error {
+	do := f.Fs.Features().MergeDirs
+	if do == nil {
+		return errors.New("MergeDirs not supported")
+	}
+	out := make([]fs.Directory, len(dirs))
+	for i, dir := range dirs {
+		out[i] = fs.NewDirCopy(dir).SetRemote(f.cipher.EncryptDirName(dir.Remote()))
+	}
+	return do(out)
+}
+
+// DirCacheFlush resets the directory cache - used in testing
+// as an optional interface
+func (f *Fs) DirCacheFlush() {
+	do := f.Fs.Features().DirCacheFlush
+	if do != nil {
+		do()
+	}
+}
+
+// PublicLink generates a public link to the remote path (usually readable by anyone)
+func (f *Fs) PublicLink(remote string) (string, error) {
+	do := f.Fs.Features().PublicLink
+	if do == nil {
+		return "", errors.New("PublicLink not supported")
+	}
+	o, err := f.NewObject(remote)
+	if err != nil {
+		// assume it is a directory
+		return do(f.cipher.EncryptDirName(remote))
+	}
+	return do(o.(*Object).Object.Remote())
+}
+
+// ChangeNotify calls the passed function with a path
+// that has had changes. If the implementation
+// uses polling, it should adhere to the given interval.
+func (f *Fs) ChangeNotify(notifyFunc func(string, fs.EntryType), pollIntervalChan <-chan time.Duration) {
+	do := f.Fs.Features().ChangeNotify
+	if do == nil {
+		return
+	}
+	wrappedNotifyFunc := func(path string, entryType fs.EntryType) {
+		// fs.Debugf(f, "ChangeNotify: path %q entryType %d", path, entryType)
+		var (
+			err       error
+			decrypted string
+		)
+		switch entryType {
+		case fs.EntryDirectory:
+			decrypted, err = f.cipher.DecryptDirName(path)
+		case fs.EntryObject:
+			decrypted, err = f.cipher.DecryptFileName(path)
+		default:
+			fs.Errorf(path, "crypt ChangeNotify: ignoring unknown EntryType %d", entryType)
+			return
+		}
+		if err != nil {
+			fs.Logf(f, "ChangeNotify was unable to decrypt %q: %s", path, err)
+			return
+		}
+		notifyFunc(decrypted, entryType)
+	}
+	do(wrappedNotifyFunc, pollIntervalChan)
+}
+
 // Object describes a wrapped for being read from the Fs
 //
 // This decrypts the remote name and decrypts the data
@@ -774,6 +841,34 @@ func (o *ObjectInfo) Hash(hash hash.Type) (string, error) {
 	return "", nil
 }
 
+// ID returns the ID of the Object if known, or "" if not
+func (o *Object) ID() string {
+	do, ok := o.Object.(fs.IDer)
+	if !ok {
+		return ""
+	}
+	return do.ID()
+}
+
+// SetTier performs changing storage tier of the Object if
+// multiple storage classes supported
+func (o *Object) SetTier(tier string) error {
+	do, ok := o.Object.(fs.SetTierer)
+	if !ok {
+		return errors.New("crypt: underlying remote does not support SetTier")
+	}
+	return do.SetTier(tier)
+}
+
+// GetTier returns storage tier or class of the Object
+func (o *Object) GetTier() string {
+	do, ok := o.Object.(fs.GetTierer)
+	if !ok {
+		return ""
+	}
+	return do.GetTier()
+}
+
 // Check the interfaces are satisfied
 var (
 	_ fs.Fs              = (*Fs)(nil)
@@ -787,7 +882,15 @@ var (
 	_ fs.UnWrapper       = (*Fs)(nil)
 	_ fs.ListRer         = (*Fs)(nil)
 	_ fs.Abouter         = (*Fs)(nil)
+	_ fs.Wrapper         = (*Fs)(nil)
+	_ fs.MergeDirser     = (*Fs)(nil)
+	_ fs.DirCacheFlusher = (*Fs)(nil)
+	_ fs.ChangeNotifier  = (*Fs)(nil)
+	_ fs.PublicLinker    = (*Fs)(nil)
 	_ fs.ObjectInfo      = (*ObjectInfo)(nil)
 	_ fs.Object          = (*Object)(nil)
 	_ fs.ObjectUnWrapper = (*Object)(nil)
+	_ fs.IDer            = (*Object)(nil)
+	_ fs.SetTierer       = (*Object)(nil)
+	_ fs.GetTierer       = (*Object)(nil)
 )

backend/crypt/crypt_test.go

@@ -21,8 +21,10 @@ func TestIntegration(t *testing.T) {
 		t.Skip("Skipping as -remote not set")
 	}
 	fstests.Run(t, &fstests.Opt{
-		RemoteName: *fstest.RemoteName,
-		NilObject:  (*crypt.Object)(nil),
+		RemoteName:                   *fstest.RemoteName,
+		NilObject:                    (*crypt.Object)(nil),
+		UnimplementableFsMethods:     []string{"OpenWriterAt"},
+		UnimplementableObjectMethods: []string{"MimeType"},
 	})
 }
 
@@ -42,6 +44,8 @@ func TestStandard(t *testing.T) {
 			{Name: name, Key: "password", Value: obscure.MustObscure("potato")},
 			{Name: name, Key: "filename_encryption", Value: "standard"},
 		},
+		UnimplementableFsMethods:     []string{"OpenWriterAt"},
+		UnimplementableObjectMethods: []string{"MimeType"},
 	})
 }
 
@@ -61,6 +65,8 @@ func TestOff(t *testing.T) {
 			{Name: name, Key: "password", Value: obscure.MustObscure("potato2")},
 			{Name: name, Key: "filename_encryption", Value: "off"},
 		},
+		UnimplementableFsMethods:     []string{"OpenWriterAt"},
+		UnimplementableObjectMethods: []string{"MimeType"},
 	})
 }
 
@@ -80,6 +86,8 @@ func TestObfuscate(t *testing.T) {
 			{Name: name, Key: "password", Value: obscure.MustObscure("potato2")},
 			{Name: name, Key: "filename_encryption", Value: "obfuscate"},
 		},
-		SkipBadWindowsCharacters: true,
+		SkipBadWindowsCharacters:     true,
+		UnimplementableFsMethods:     []string{"OpenWriterAt"},
+		UnimplementableObjectMethods: []string{"MimeType"},
 	})
 }

backend/drive/drive.go

@@ -1,7 +1,4 @@
 // Package drive interfaces with the Google Drive object storage system
-
-// +build go1.9
-
 package drive
 
 // FIXME need to deal with some corner cases
@@ -240,6 +237,22 @@ func init() {
 			Default:  false,
 			Help:     "Skip google documents in all listings.\nIf given, gdocs practically become invisible to rclone.",
 			Advanced: true,
+		}, {
+			Name:    "skip_checksum_gphotos",
+			Default: false,
+			Help: `Skip MD5 checksum on Google photos and videos only.
+
+Use this if you get checksum errors when transferring Google photos or
+videos.
+
+Setting this flag will cause Google photos and videos to return a
+blank MD5 checksum.
+
+Google photos are identifed by being in the "photos" space.
+
+Corrupted checksums are caused by Google modifying the image/video but
+not updating the checksum.`,
+			Advanced: true,
 		}, {
 			Name:    "shared_with_me",
 			Default: false,
@@ -369,6 +382,16 @@ will download it anyway.`,
 			Default:  defaultBurst,
 			Help:     "Number of API calls to allow without sleeping.",
 			Advanced: true,
+		}, {
+			Name:    "server_side_across_configs",
+			Default: false,
+			Help: `Allow server side operations (eg copy) to work across different drive configs.
+
+This can be useful if you wish to do a server side copy between two
+different Google drives.  Note that this isn't enabled by default
+because it isn't easy to tell if it will work beween any two
+configurations.`,
+			Advanced: true,
 		}},
 	})
 
@@ -396,6 +419,7 @@ type Options struct {
 	AuthOwnerOnly             bool          `config:"auth_owner_only"`
 	UseTrash                  bool          `config:"use_trash"`
 	SkipGdocs                 bool          `config:"skip_gdocs"`
+	SkipChecksumGphotos       bool          `config:"skip_checksum_gphotos"`
 	SharedWithMe              bool          `config:"shared_with_me"`
 	TrashedOnly               bool          `config:"trashed_only"`
 	Extensions                string        `config:"formats"`
@@ -413,6 +437,7 @@ type Options struct {
 	V2DownloadMinSize         fs.SizeSuffix `config:"v2_download_min_size"`
 	PacerMinSleep             fs.Duration   `config:"pacer_min_sleep"`
 	PacerBurst                int           `config:"pacer_burst"`
+	ServerSideAcrossConfigs   bool          `config:"server_side_across_configs"`
 }
 
 // Fs represents a remote drive server
@@ -615,6 +640,9 @@ func (f *Fs) list(dirIDs []string, title string, directoriesOnly, filesOnly, inc
 	if f.opt.AuthOwnerOnly {
 		fields += ",owners"
 	}
+	if f.opt.SkipChecksumGphotos {
+		fields += ",spaces"
+	}
 
 	fields = fmt.Sprintf("files(%s),nextPageToken", fields)
 
@@ -907,7 +935,7 @@ func NewFs(name, path string, m configmap.Mapper) (fs.Fs, error) {
 		ReadMimeType:            true,
 		WriteMimeType:           true,
 		CanHaveEmptyDirectories: true,
-		ServerSideAcrossConfigs: true,
+		ServerSideAcrossConfigs: opt.ServerSideAcrossConfigs,
 	}).Fill(f)
 
 	// Create a new authorized Drive client.
@@ -1002,6 +1030,15 @@ func (f *Fs) newBaseObject(remote string, info *drive.File) baseObject {
 
 // newRegularObject creates a fs.Object for a normal drive.File
 func (f *Fs) newRegularObject(remote string, info *drive.File) fs.Object {
+	// wipe checksum if SkipChecksumGphotos and file is type Photo or Video
+	if f.opt.SkipChecksumGphotos {
+		for _, space := range info.Spaces {
+			if space == "photos" {
+				info.Md5Checksum = ""
+				break
+			}
+		}
+	}
 	return &Object{
 		baseObject: f.newBaseObject(remote, info),
 		url:        fmt.Sprintf("%sfiles/%s?alt=media", f.svc.BasePath, info.Id),
@@ -1843,16 +1880,24 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
 		remote = remote[:len(remote)-len(ext)]
 	}
 
+	// Look to see if there is an existing object
+	existingObject, _ := f.NewObject(remote)
+
 	createInfo, err := f.createFileInfo(remote, src.ModTime())
 	if err != nil {
 		return nil, err
 	}
 
+	supportTeamDrives, err := f.ShouldSupportTeamDrives(src)
+	if err != nil {
+		return nil, err
+	}
+
 	var info *drive.File
 	err = f.pacer.Call(func() (bool, error) {
 		info, err = f.svc.Files.Copy(srcObj.id, createInfo).
 			Fields(partialFields).
-			SupportsTeamDrives(f.isTeamDrive).
+			SupportsTeamDrives(supportTeamDrives).
 			KeepRevisionForever(f.opt.KeepRevisionForever).
 			Do()
 		return shouldRetry(err)
@@ -1860,7 +1905,17 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
 	if err != nil {
 		return nil, err
 	}
-	return f.newObjectWithInfo(remote, info)
+	newObject, err := f.newObjectWithInfo(remote, info)
+	if err != nil {
+		return nil, err
+	}
+	if existingObject != nil {
+		err = existingObject.Remove()
+		if err != nil {
+			fs.Errorf(existingObject, "Failed to remove existing object after copy: %v", err)
+		}
+	}
+	return newObject, nil
 }
 
 // Purge deletes all the files and the container
@@ -1986,6 +2041,11 @@ func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
 	dstParents := strings.Join(dstInfo.Parents, ",")
 	dstInfo.Parents = nil
 
+	supportTeamDrives, err := f.ShouldSupportTeamDrives(src)
+	if err != nil {
+		return nil, err
+	}
+
 	// Do the move
 	var info *drive.File
 	err = f.pacer.Call(func() (bool, error) {
@@ -1993,7 +2053,7 @@ func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
 			RemoveParents(srcParentID).
 			AddParents(dstParents).
 			Fields(partialFields).
-			SupportsTeamDrives(f.isTeamDrive).
+			SupportsTeamDrives(supportTeamDrives).
 			Do()
 		return shouldRetry(err)
 	})
@@ -2004,6 +2064,20 @@ func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
 	return f.newObjectWithInfo(remote, info)
 }
 
+// ShouldSupportTeamDrives returns the request should support TeamDrives
+func (f *Fs) ShouldSupportTeamDrives(src fs.Object) (bool, error) {
+	srcIsTeamDrive := false
+	if srcFs, ok := src.Fs().(*Fs); ok {
+		srcIsTeamDrive = srcFs.isTeamDrive
+	}
+
+	if f.isTeamDrive {
+		return true, nil
+	}
+
+	return srcIsTeamDrive, nil
+}
+
 // PublicLink adds a "readable by anyone with link" permission on the given file or folder.
 func (f *Fs) PublicLink(remote string) (link string, err error) {
 	id, err := f.dirCache.FindDir(remote, false)

backend/drive/drive_internal_test.go

@@ -1,5 +1,3 @@
-// +build go1.9
-
 package drive
 
 import (

backend/drive/drive_test.go

@@ -1,7 +1,5 @@
 // Test Drive filesystem interface
 
-// +build go1.9
-
 package drive
 
 import (

backend/drive/drive_unsupported.go

@@ -1,6 +0,0 @@
-// Build for unsupported platforms to stop go complaining
-// about "no buildable Go source files "
-
-// +build !go1.9
-
-package drive

backend/drive/upload.go

@@ -8,8 +8,6 @@
 //
 // This contains code adapted from google.golang.org/api (C) the GO AUTHORS
 
-// +build go1.9
-
 package drive
 
 import (

backend/ftp/ftp.go

@@ -15,6 +15,7 @@ import (
 	"github.com/ncw/rclone/fs/config/configstruct"
 	"github.com/ncw/rclone/fs/config/obscure"
 	"github.com/ncw/rclone/fs/hash"
+	"github.com/ncw/rclone/lib/pacer"
 	"github.com/ncw/rclone/lib/readers"
 	"github.com/pkg/errors"
 )
@@ -45,6 +46,11 @@ func init() {
 				Help:       "FTP password",
 				IsPassword: true,
 				Required:   true,
+			}, {
+				Name:     "concurrency",
+				Help:     "Maximum number of FTP simultaneous connections, 0 for unlimited",
+				Default:  0,
+				Advanced: true,
 			},
 		},
 	})
@@ -52,10 +58,11 @@ func init() {
 
 // Options defines the configuration for this backend
 type Options struct {
-	Host string `config:"host"`
-	User string `config:"user"`
-	Pass string `config:"pass"`
-	Port string `config:"port"`
+	Host        string `config:"host"`
+	User        string `config:"user"`
+	Pass        string `config:"pass"`
+	Port        string `config:"port"`
+	Concurrency int    `config:"concurrency"`
 }
 
 // Fs represents a remote FTP server
@@ -70,6 +77,7 @@ type Fs struct {
 	dialAddr string
 	poolMu   sync.Mutex
 	pool     []*ftp.ServerConn
+	tokens   *pacer.TokenDispenser
 }
 
 // Object describes an FTP file
@@ -128,6 +136,9 @@ func (f *Fs) ftpConnection() (*ftp.ServerConn, error) {
 
 // Get an FTP connection from the pool, or open a new one
 func (f *Fs) getFtpConnection() (c *ftp.ServerConn, err error) {
+	if f.opt.Concurrency > 0 {
+		f.tokens.Get()
+	}
 	f.poolMu.Lock()
 	if len(f.pool) > 0 {
 		c = f.pool[0]
@@ -147,6 +158,9 @@ func (f *Fs) getFtpConnection() (c *ftp.ServerConn, err error) {
 // if err is not nil then it checks the connection is alive using a
 // NOOP request
 func (f *Fs) putFtpConnection(pc **ftp.ServerConn, err error) {
+	if f.opt.Concurrency > 0 {
+		defer f.tokens.Put()
+	}
 	c := *pc
 	*pc = nil
 	if err != nil {
@@ -198,6 +212,7 @@ func NewFs(name, root string, m configmap.Mapper) (ff fs.Fs, err error) {
 		user:     user,
 		pass:     pass,
 		dialAddr: dialAddr,
+		tokens:   pacer.NewTokenDispenser(opt.Concurrency),
 	}
 	f.features = (&fs.Features{
 		CanHaveEmptyDirectories: true,
@@ -330,11 +345,36 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "list")
 	}
-	files, err := c.List(path.Join(f.root, dir))
-	f.putFtpConnection(&c, err)
-	if err != nil {
-		return nil, translateErrorDir(err)
+
+	var listErr error
+	var files []*ftp.Entry
+
+	resultchan := make(chan []*ftp.Entry, 1)
+	errchan := make(chan error, 1)
+	go func() {
+		result, err := c.List(path.Join(f.root, dir))
+		f.putFtpConnection(&c, err)
+		if err != nil {
+			errchan <- err
+			return
+		}
+		resultchan <- result
+	}()
+
+	// Wait for List for up to Timeout seconds
+	timer := time.NewTimer(fs.Config.Timeout)
+	select {
+	case listErr = <-errchan:
+		timer.Stop()
+		return nil, translateErrorDir(listErr)
+	case files = <-resultchan:
+		timer.Stop()
+	case <-timer.C:
+		// if timer fired assume no error but connection dead
+		fs.Errorf(f, "Timeout when waiting for List")
+		return nil, errors.New("Timeout when waiting for List")
 	}
+
 	// Annoyingly FTP returns success for a directory which
 	// doesn't exist, so check it really doesn't exist if no
 	// entries found.

backend/googlecloudstorage/googlecloudstorage.go

@@ -1,7 +1,4 @@
 // Package googlecloudstorage provides an interface to Google Cloud Storage
-
-// +build go1.9
-
 package googlecloudstorage
 
 /*

backend/googlecloudstorage/googlecloudstorage_test.go

@@ -1,7 +1,5 @@
 // Test GoogleCloudStorage filesystem interface
 
-// +build go1.9
-
 package googlecloudstorage_test
 
 import (

backend/googlecloudstorage/googlecloudstorage_unsupported.go

@@ -1,6 +0,0 @@
-// Build for unsupported platforms to stop go complaining
-// about "no buildable Go source files "
-
-// +build !go1.9
-
-package googlecloudstorage

backend/http/http.go

@@ -6,6 +6,7 @@ package http
 
 import (
 	"io"
+	"mime"
 	"net/http"
 	"net/url"
 	"path"
@@ -44,6 +45,22 @@ func init() {
 				Value: "https://user:pass@example.com",
 				Help:  "Connect to example.com using a username and password",
 			}},
+		}, {
+			Name: "no_slash",
+			Help: `Set this if the site doesn't end directories with /
+
+Use this if your target website does not use / on the end of
+directories.
+
+A / on the end of a path is how rclone normally tells the difference
+between files and directories.  If this flag is set, then rclone will
+treat all files with Content-Type: text/html as directories and read
+URLs from them rather than downloading them.
+
+Note that this may cause rclone to confuse genuine HTML files with
+directories.`,
+			Default:  false,
+			Advanced: true,
 		}},
 	}
 	fs.Register(fsi)
@@ -52,6 +69,7 @@ func init() {
 // Options defines the configuration for this backend
 type Options struct {
 	Endpoint string `config:"url"`
+	NoSlash  bool   `config:"no_slash"`
 }
 
 // Fs stores the interface to the remote HTTP files
@@ -270,14 +288,20 @@ func parse(base *url.URL, in io.Reader) (names []string, err error) {
 	if err != nil {
 		return nil, err
 	}
-	var walk func(*html.Node)
+	var (
+		walk func(*html.Node)
+		seen = make(map[string]struct{})
+	)
 	walk = func(n *html.Node) {
 		if n.Type == html.ElementNode && n.Data == "a" {
 			for _, a := range n.Attr {
 				if a.Key == "href" {
 					name, err := parseName(base, a.Val)
 					if err == nil {
-						names = append(names, name)
+						if _, found := seen[name]; !found {
+							names = append(names, name)
+							seen[name] = struct{}{}
+						}
 					}
 					break
 				}
@@ -302,14 +326,16 @@ func (f *Fs) readDir(dir string) (names []string, err error) {
 		return nil, errors.Errorf("internal error: readDir URL %q didn't end in /", URL)
 	}
 	res, err := f.httpClient.Get(URL)
-	if err == nil && res.StatusCode == http.StatusNotFound {
-		return nil, fs.ErrorDirNotFound
+	if err == nil {
+		defer fs.CheckClose(res.Body, &err)
+		if res.StatusCode == http.StatusNotFound {
+			return nil, fs.ErrorDirNotFound
+		}
 	}
 	err = statusError(res, err)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to readDir")
 	}
-	defer fs.CheckClose(res.Body, &err)
 
 	contentType := strings.SplitN(res.Header.Get("Content-Type"), ";", 2)[0]
 	switch contentType {
@@ -353,11 +379,16 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 				fs:     f,
 				remote: remote,
 			}
-			if err = file.stat(); err != nil {
+			switch err = file.stat(); err {
+			case nil:
+				entries = append(entries, file)
+			case fs.ErrorNotAFile:
+				// ...found a directory not a file
+				dir := fs.NewDir(remote, timeUnset)
+				entries = append(entries, dir)
+			default:
 				fs.Debugf(remote, "skipping because of error: %v", err)
-				continue
 			}
-			entries = append(entries, file)
 		}
 	}
 	return entries, nil
@@ -433,6 +464,16 @@ func (o *Object) stat() error {
 	o.size = parseInt64(res.Header.Get("Content-Length"), -1)
 	o.modTime = t
 	o.contentType = res.Header.Get("Content-Type")
+	// If NoSlash is set then check ContentType to see if it is a directory
+	if o.fs.opt.NoSlash {
+		mediaType, _, err := mime.ParseMediaType(o.contentType)
+		if err != nil {
+			return errors.Wrapf(err, "failed to parse Content-Type: %q", o.contentType)
+		}
+		if mediaType == "text/html" {
+			return fs.ErrorNotAFile
+		}
+	}
 	return nil
 }
 

backend/http/http_internal_test.go

@@ -1,5 +1,3 @@
-// +build go1.8
-
 package http
 
 import (
@@ -65,7 +63,7 @@ func prepare(t *testing.T) (fs.Fs, func()) {
 	return f, tidy
 }
 
-func testListRoot(t *testing.T, f fs.Fs) {
+func testListRoot(t *testing.T, f fs.Fs, noSlash bool) {
 	entries, err := f.List("")
 	require.NoError(t, err)
 
@@ -93,15 +91,29 @@ func testListRoot(t *testing.T, f fs.Fs) {
 
 	e = entries[3]
 	assert.Equal(t, "two.html", e.Remote())
-	assert.Equal(t, int64(7), e.Size())
-	_, ok = e.(*Object)
-	assert.True(t, ok)
+	if noSlash {
+		assert.Equal(t, int64(-1), e.Size())
+		_, ok = e.(fs.Directory)
+		assert.True(t, ok)
+	} else {
+		assert.Equal(t, int64(41), e.Size())
+		_, ok = e.(*Object)
+		assert.True(t, ok)
+	}
 }
 
 func TestListRoot(t *testing.T) {
 	f, tidy := prepare(t)
 	defer tidy()
-	testListRoot(t, f)
+	testListRoot(t, f, false)
+}
+
+func TestListRootNoSlash(t *testing.T) {
+	f, tidy := prepare(t)
+	f.(*Fs).opt.NoSlash = true
+	defer tidy()
+
+	testListRoot(t, f, true)
 }
 
 func TestListSubDir(t *testing.T) {
@@ -194,7 +206,7 @@ func TestIsAFileRoot(t *testing.T) {
 	f, err := NewFs(remoteName, "one%.txt", m)
 	assert.Equal(t, err, fs.ErrorIsFile)
 
-	testListRoot(t, f)
+	testListRoot(t, f, false)
 }
 
 func TestIsAFileSubDir(t *testing.T) {

backend/http/test/files/two.html

@@ -1 +1 @@
-potato
+<a href="two.html/file.txt">file.txt</a>

backend/hubic/hubic_test.go

@@ -11,7 +11,9 @@ import (
 // TestIntegration runs integration tests against the remote
 func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
-		RemoteName: "TestHubic:",
-		NilObject:  (*hubic.Object)(nil),
+		RemoteName:          "TestHubic:",
+		NilObject:           (*hubic.Object)(nil),
+		SkipFsCheckWrap:     true,
+		SkipObjectCheckWrap: true,
 	})
 }

backend/jottacloud/api/types.go

@@ -314,3 +314,9 @@ type UploadResponse struct {
 	Deleted   interface{} `json:"deleted"`
 	Mime      string      `json:"mime"`
 }
+
+// DeviceRegistrationResponse is the response to registering a device
+type DeviceRegistrationResponse struct {
+	ClientID     string `json:"client_id"`
+	ClientSecret string `json:"client_secret"`
+}

backend/jottacloud/jottacloud.go

@@ -8,6 +8,7 @@ import (
 	"io"
 	"io/ioutil"
 	"log"
+	"math/rand"
 	"net/http"
 	"net/url"
 	"os"
@@ -40,15 +41,21 @@ const (
 	maxSleep                    = 2 * time.Second
 	decayConstant               = 2 // bigger for slower decay, exponential
 	defaultDevice               = "Jotta"
-	defaultMountpoint           = "Sync" // nolint
+	defaultMountpoint           = "Archive"
 	rootURL                     = "https://www.jottacloud.com/jfs/"
 	apiURL                      = "https://api.jottacloud.com/files/v1/"
 	baseURL                     = "https://www.jottacloud.com/"
 	tokenURL                    = "https://api.jottacloud.com/auth/v1/token"
+	registerURL                 = "https://api.jottacloud.com/auth/v1/register"
 	cachePrefix                 = "rclone-jcmd5-"
 	rcloneClientID              = "nibfk8biu12ju7hpqomr8b1e40"
 	rcloneEncryptedClientSecret = "Vp8eAv7eVElMnQwN-kgU9cbhgApNDaMqWdlDi5qFydlQoji4JBxrGMF2"
 	configUsername              = "user"
+	configClientID              = "client_id"
+	configClientSecret          = "client_secret"
+	configDevice                = "device"
+	configMountpoint            = "mountpoint"
+	charset                     = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
 )
 
 var (
@@ -58,14 +65,13 @@ var (
 			AuthURL:  tokenURL,
 			TokenURL: tokenURL,
 		},
-		ClientID:     rcloneClientID,
-		ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
-		RedirectURL:  oauthutil.RedirectLocalhostURL,
+		RedirectURL: oauthutil.RedirectLocalhostURL,
 	}
 )
 
 // Register with Fs
 func init() {
+	// needs to be done early so we can use oauth during config
 	fs.Register(&fs.RegInfo{
 		Name:        "jottacloud",
 		Description: "JottaCloud",
@@ -79,14 +85,62 @@ func init() {
 				}
 			}
 
+			srv := rest.NewClient(fshttp.NewClient(fs.Config))
+
+			// ask if we should create a device specifc token: https://github.com/ncw/rclone/issues/2995
+			fmt.Printf("\nDo you want to create a machine specific API key?\n\nRclone has it's own Jottacloud API KEY which works fine as long as one only uses rclone on a single machine. When you want to use rclone with this account on more than one machine it's recommended to create a machine specific API key. These keys can NOT be shared between machines.\n\n")
+			if config.Confirm() {
+				// random generator to generate random device names
+				seededRand := rand.New(rand.NewSource(time.Now().UnixNano()))
+				randonDeviceNamePartLength := 21
+				randomDeviceNamePart := make([]byte, randonDeviceNamePartLength)
+				for i := range randomDeviceNamePart {
+					randomDeviceNamePart[i] = charset[seededRand.Intn(len(charset))]
+				}
+				randomDeviceName := "rclone-" + string(randomDeviceNamePart)
+				fs.Debugf(nil, "Trying to register device '%s'", randomDeviceName)
+
+				values := url.Values{}
+				values.Set("device_id", randomDeviceName)
+
+				// all information comes from https://github.com/ttyridal/aiojotta/wiki/Jotta-protocol-3.-Authentication#token-authentication
+				opts := rest.Opts{
+					Method:       "POST",
+					RootURL:      registerURL,
+					ContentType:  "application/x-www-form-urlencoded",
+					ExtraHeaders: map[string]string{"Authorization": "Bearer c2xrZmpoYWRsZmFramhkc2xma2phaHNkbGZramhhc2xkZmtqaGFzZGxrZmpobGtq"},
+					Parameters:   values,
+				}
+
+				var deviceRegistration api.DeviceRegistrationResponse
+				_, err := srv.CallJSON(&opts, nil, &deviceRegistration)
+				if err != nil {
+					log.Fatalf("Failed to register device: %v", err)
+				}
+
+				m.Set(configClientID, deviceRegistration.ClientID)
+				m.Set(configClientSecret, obscure.MustObscure(deviceRegistration.ClientSecret))
+				fs.Debugf(nil, "Got clientID '%s' and clientSecret '%s'", deviceRegistration.ClientID, deviceRegistration.ClientSecret)
+			}
+
+			clientID, ok := m.Get(configClientID)
+			if !ok {
+				clientID = rcloneClientID
+			}
+			clientSecret, ok := m.Get(configClientSecret)
+			if !ok {
+				clientSecret = rcloneEncryptedClientSecret
+			}
+			oauthConfig.ClientID = clientID
+			oauthConfig.ClientSecret = obscure.MustReveal(clientSecret)
+
 			username, ok := m.Get(configUsername)
 			if !ok {
 				log.Fatalf("No username defined")
 			}
-			password := config.GetPassword("Your Jottacloud password is only required during config and will not be stored.")
+			password := config.GetPassword("Your Jottacloud password is only required during setup and will not be stored.")
 
 			// prepare out token request with username and password
-			srv := rest.NewClient(fshttp.NewClient(fs.Config))
 			values := url.Values{}
 			values.Set("grant_type", "PASSWORD")
 			values.Set("password", password)
@@ -106,7 +160,7 @@ func init() {
 				// if 2fa is enabled the first request is expected to fail. We will do another request with the 2fa code as an additional http header
 				if resp != nil {
 					if resp.Header.Get("X-JottaCloud-OTP") == "required; SMS" {
-						fmt.Printf("This account has 2 factor authentication enabled you will receive a verification code via SMS.\n")
+						fmt.Printf("This account uses 2 factor authentication you will receive a verification code via SMS.\n")
 						fmt.Printf("Enter verification code> ")
 						authCode := config.ReadLine()
 						authCode = strings.Replace(authCode, "-", "", -1) // the sms received contains a pair of 3 digit numbers seperated by '-' but wants a single 6 digit number
@@ -129,23 +183,49 @@ func init() {
 			// finally save them in the config
 			err = oauthutil.PutToken(name, m, &token, true)
 			if err != nil {
-				log.Fatalf("Error while setting token: %s", err)
+				log.Fatalf("Error while saving token: %s", err)
+			}
+
+			fmt.Printf("\nDo you want to use a non standard device/mountpoint e.g. for accessing files uploaded using the official Jottacloud client?\n\n")
+			if config.Confirm() {
+				oAuthClient, _, err := oauthutil.NewClient(name, m, oauthConfig)
+				if err != nil {
+					log.Fatalf("Failed to load oAuthClient: %s", err)
+				}
+
+				srv = rest.NewClient(oAuthClient).SetRoot(rootURL)
+
+				acc, err := getAccountInfo(srv, username)
+				if err != nil {
+					log.Fatalf("Error getting devices: %s", err)
+				}
+				fmt.Printf("Please select the device to use. Normally this will be Jotta\n")
+				var deviceNames []string
+				for i := range acc.Devices {
+					deviceNames = append(deviceNames, acc.Devices[i].Name)
+				}
+				result := config.Choose("Devices", deviceNames, nil, false)
+				m.Set(configDevice, result)
+
+				dev, err := getDeviceInfo(srv, path.Join(username, result))
+				if err != nil {
+					log.Fatalf("Error getting Mountpoint: %s", err)
+				}
+				if len(dev.MountPoints) == 0 {
+					log.Fatalf("No Mountpoints found for this device.")
+				}
+				fmt.Printf("Please select the mountpoint to user. Normally this will be Archive\n")
+				var mountpointNames []string
+				for i := range dev.MountPoints {
+					mountpointNames = append(mountpointNames, dev.MountPoints[i].Name)
+				}
+				result = config.Choose("Mountpoints", mountpointNames, nil, false)
+				m.Set(configMountpoint, result)
 			}
 		},
 		Options: []fs.Option{{
 			Name: configUsername,
 			Help: "User Name:",
-		}, {
-			Name:     "mountpoint",
-			Help:     "The mountpoint to use.",
-			Required: true,
-			Examples: []fs.OptionExample{{
-				Value: "Sync",
-				Help:  "Will be synced by the official client.",
-			}, {
-				Value: "Archive",
-				Help:  "Archive",
-			}},
 		}, {
 			Name:     "md5_memory_limit",
 			Help:     "Files bigger than this will be cached on disk to calculate the MD5 if required.",
@@ -173,6 +253,7 @@ func init() {
 // Options defines the configuration for this backend
 type Options struct {
 	User               string        `config:"user"`
+	Device             string        `config:"device"`
 	Mountpoint         string        `config:"mountpoint"`
 	MD5MemoryThreshold fs.SizeSuffix `config:"md5_memory_limit"`
 	HardDelete         bool          `config:"hard_delete"`
@@ -280,18 +361,31 @@ func (f *Fs) readMetaDataForPath(path string) (info *api.JottaFile, err error) {
 	return &result, nil
 }
 
-// getAccountInfo retrieves account information
-func (f *Fs) getAccountInfo() (info *api.AccountInfo, err error) {
+// getAccountInfo queries general information about the account.
+// Takes rest.Client and username as parameter to be easily usable
+// during config
+func getAccountInfo(srv *rest.Client, username string) (info *api.AccountInfo, err error) {
 	opts := rest.Opts{
 		Method: "GET",
-		Path:   urlPathEscape(f.user),
+		Path:   urlPathEscape(username),
 	}
 
-	var resp *http.Response
-	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.srv.CallXML(&opts, nil, &info)
-		return shouldRetry(resp, err)
-	})
+	_, err = srv.CallXML(&opts, nil, &info)
+	if err != nil {
+		return nil, err
+	}
+
+	return info, nil
+}
+
+// getDeviceInfo queries Information about a jottacloud device
+func getDeviceInfo(srv *rest.Client, path string) (info *api.JottaDevice, err error) {
+	opts := rest.Opts{
+		Method: "GET",
+		Path:   urlPathEscape(path),
+	}
+
+	_, err = srv.CallXML(&opts, nil, &info)
 	if err != nil {
 		return nil, err
 	}
@@ -300,12 +394,18 @@ func (f *Fs) getAccountInfo() (info *api.AccountInfo, err error) {
 }
 
 // setEndpointUrl reads the account id and generates the API endpoint URL
-func (f *Fs) setEndpointURL(mountpoint string) (err error) {
-	info, err := f.getAccountInfo()
+func (f *Fs) setEndpointURL() (err error) {
+	info, err := getAccountInfo(f.srv, f.user)
 	if err != nil {
 		return errors.Wrap(err, "failed to get endpoint url")
 	}
-	f.endpointURL = urlPathEscape(path.Join(info.Username, defaultDevice, mountpoint))
+	if f.opt.Device == "" {
+		f.opt.Device = defaultDevice
+	}
+	if f.opt.Mountpoint == "" {
+		f.opt.Mountpoint = defaultMountpoint
+	}
+	f.endpointURL = urlPathEscape(path.Join(info.Username, f.opt.Device, f.opt.Mountpoint))
 	return nil
 }
 
@@ -381,8 +481,16 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	rootIsDir := strings.HasSuffix(root, "/")
 	root = parsePath(root)
 
-	// add jottacloud to the long list of sites that don't follow the oauth spec correctly
-	oauth2.RegisterBrokenAuthHeaderProvider("https://www.jottacloud.com/")
+	clientID, ok := m.Get(configClientID)
+	if !ok {
+		clientID = rcloneClientID
+	}
+	clientSecret, ok := m.Get(configClientSecret)
+	if !ok {
+		clientSecret = rcloneEncryptedClientSecret
+	}
+	oauthConfig.ClientID = clientID
+	oauthConfig.ClientSecret = obscure.MustReveal(clientSecret)
 
 	// the oauth client for the api servers needs
 	// a filter to fix the grant_type issues (see above)
@@ -422,7 +530,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		return err
 	})
 
-	err = f.setEndpointURL(opt.Mountpoint)
+	err = f.setEndpointURL()
 	if err != nil {
 		return nil, errors.Wrap(err, "couldn't get account info")
 	}
@@ -680,6 +788,9 @@ func (f *Fs) createObject(remote string, modTime time.Time, size int64) (o *Obje
 //
 // The new object may have been created if an error is returned
 func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+	if f.opt.Device != "Jotta" {
+		return nil, errors.New("upload not supported for devices other than Jotta")
+	}
 	o := f.createObject(src.Remote(), src.ModTime(), src.Size())
 	return o, o.Update(in, src, options...)
 }
@@ -943,7 +1054,7 @@ func (f *Fs) PublicLink(remote string) (link string, err error) {
 
 // About gets quota information
 func (f *Fs) About() (*fs.Usage, error) {
-	info, err := f.getAccountInfo()
+	info, err := getAccountInfo(f.srv, f.user)
 	if err != nil {
 		return nil, err
 	}

backend/local/local.go

@@ -28,8 +28,9 @@ import (
 )
 
 // Constants
-const devUnset = 0xdeadbeefcafebabe // a device id meaning it is unset
-const linkSuffix = ".rclonelink"    // The suffix added to a translated symbolic link
+const devUnset = 0xdeadbeefcafebabe                                       // a device id meaning it is unset
+const linkSuffix = ".rclonelink"                                          // The suffix added to a translated symbolic link
+const useReadDir = (runtime.GOOS == "windows" || runtime.GOOS == "plan9") // these OSes read FileInfos directly
 
 // Register with Fs
 func init() {
@@ -327,7 +328,14 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 
 	fd, err := os.Open(fsDirPath)
 	if err != nil {
-		return nil, errors.Wrapf(err, "failed to open directory %q", dir)
+		isPerm := os.IsPermission(err)
+		err = errors.Wrapf(err, "failed to open directory %q", dir)
+		fs.Errorf(dir, "%v", err)
+		if isPerm {
+			accounting.Stats.Error(fserrors.NoRetryError(err))
+			err = nil // ignore error but fail sync
+		}
+		return nil, err
 	}
 	defer func() {
 		cerr := fd.Close()
@@ -337,12 +345,38 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 	}()
 
 	for {
-		fis, err := fd.Readdir(1024)
-		if err == io.EOF && len(fis) == 0 {
-			break
+		var fis []os.FileInfo
+		if useReadDir {
+			// Windows and Plan9 read the directory entries with the stat information in which
+			// shouldn't fail because of unreadable entries.
+			fis, err = fd.Readdir(1024)
+			if err == io.EOF && len(fis) == 0 {
+				break
+			}
+		} else {
+			// For other OSes we read the names only (which shouldn't fail) then stat the
+			// individual ourselves so we can log errors but not fail the directory read.
+			var names []string
+			names, err = fd.Readdirnames(1024)
+			if err == io.EOF && len(names) == 0 {
+				break
+			}
+			if err == nil {
+				for _, name := range names {
+					namepath := filepath.Join(fsDirPath, name)
+					fi, fierr := os.Lstat(namepath)
+					if fierr != nil {
+						err = errors.Wrapf(err, "failed to read directory %q", namepath)
+						fs.Errorf(dir, "%v", fierr)
+						accounting.Stats.Error(fserrors.NoRetryError(fierr)) // fail the sync
+						continue
+					}
+					fis = append(fis, fi)
+				}
+			}
 		}
 		if err != nil {
-			return nil, errors.Wrapf(err, "failed to read directory %q", dir)
+			return nil, errors.Wrap(err, "failed to read directory entry")
 		}
 
 		for _, fi := range fis {
@@ -709,9 +743,10 @@ func (o *Object) Hash(r hash.Type) (string, error) {
 
 	o.fs.objectHashesMu.Lock()
 	hashes := o.hashes
+	hashValue, hashFound := o.hashes[r]
 	o.fs.objectHashesMu.Unlock()
 
-	if !o.modTime.Equal(oldtime) || oldsize != o.size || hashes == nil {
+	if !o.modTime.Equal(oldtime) || oldsize != o.size || hashes == nil || !hashFound {
 		var in io.ReadCloser
 
 		if !o.translatedLink {
@@ -722,7 +757,7 @@ func (o *Object) Hash(r hash.Type) (string, error) {
 		if err != nil {
 			return "", errors.Wrap(err, "hash: failed to open")
 		}
-		hashes, err = hash.Stream(in)
+		hashes, err = hash.StreamTypes(in, hash.NewHashSet(r))
 		closeErr := in.Close()
 		if err != nil {
 			return "", errors.Wrap(err, "hash: failed to read")
@@ -730,11 +765,16 @@ func (o *Object) Hash(r hash.Type) (string, error) {
 		if closeErr != nil {
 			return "", errors.Wrap(closeErr, "hash: failed to close")
 		}
+		hashValue = hashes[r]
 		o.fs.objectHashesMu.Lock()
-		o.hashes = hashes
+		if o.hashes == nil {
+			o.hashes = hashes
+		} else {
+			o.hashes[r] = hashValue
+		}
 		o.fs.objectHashesMu.Unlock()
 	}
-	return hashes[r], nil
+	return hashValue, nil
 }
 
 // Size returns the size of an object in bytes
@@ -998,6 +1038,36 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 	return o.lstat()
 }
 
+// OpenWriterAt opens with a handle for random access writes
+//
+// Pass in the remote desired and the size if known.
+//
+// It truncates any existing object
+func (f *Fs) OpenWriterAt(remote string, size int64) (fs.WriterAtCloser, error) {
+	// Temporary Object under construction
+	o := f.newObject(remote, "")
+
+	err := o.mkdirAll()
+	if err != nil {
+		return nil, err
+	}
+
+	if o.translatedLink {
+		return nil, errors.New("can't open a symlink for random writing")
+	}
+
+	out, err := file.OpenFile(o.path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
+	if err != nil {
+		return nil, err
+	}
+	// Pre-allocate the file for performance reasons
+	err = preAllocate(size, out)
+	if err != nil {
+		fs.Debugf(o, "Failed to pre-allocate: %v", err)
+	}
+	return out, nil
+}
+
 // setMetadata sets the file info from the os.FileInfo passed in
 func (o *Object) setMetadata(info os.FileInfo) {
 	// Don't overwrite the info if we don't need to
@@ -1139,10 +1209,11 @@ func cleanWindowsName(f *Fs, name string) string {
 
 // Check the interfaces are satisfied
 var (
-	_ fs.Fs          = &Fs{}
-	_ fs.Purger      = &Fs{}
-	_ fs.PutStreamer = &Fs{}
-	_ fs.Mover       = &Fs{}
-	_ fs.DirMover    = &Fs{}
-	_ fs.Object      = &Object{}
+	_ fs.Fs             = &Fs{}
+	_ fs.Purger         = &Fs{}
+	_ fs.PutStreamer    = &Fs{}
+	_ fs.Mover          = &Fs{}
+	_ fs.DirMover       = &Fs{}
+	_ fs.OpenWriterAter = &Fs{}
+	_ fs.Object         = &Object{}
 )

backend/local/preallocate_unix.go

@@ -4,16 +4,40 @@ package local
 
 import (
 	"os"
+	"sync/atomic"
 
+	"github.com/ncw/rclone/fs"
 	"golang.org/x/sys/unix"
 )
 
+var (
+	fallocFlags = [...]uint32{
+		unix.FALLOC_FL_KEEP_SIZE,                             // Default
+		unix.FALLOC_FL_KEEP_SIZE | unix.FALLOC_FL_PUNCH_HOLE, // for ZFS #3066
+	}
+	fallocFlagsIndex int32
+)
+
 // preAllocate the file for performance reasons
 func preAllocate(size int64, out *os.File) error {
 	if size <= 0 {
 		return nil
 	}
-	err := unix.Fallocate(int(out.Fd()), unix.FALLOC_FL_KEEP_SIZE, 0, size)
+	index := atomic.LoadInt32(&fallocFlagsIndex)
+again:
+	if index >= int32(len(fallocFlags)) {
+		return nil // Fallocate is disabled
+	}
+	flags := fallocFlags[index]
+	err := unix.Fallocate(int(out.Fd()), flags, 0, size)
+	if err == unix.ENOTSUP {
+		// Try the next flags combination
+		index++
+		atomic.StoreInt32(&fallocFlagsIndex, index)
+		fs.Debugf(nil, "preAllocate: got error on fallocate, trying combination %d/%d: %v", index, len(fallocFlags), err)
+		goto again
+
+	}
 	// FIXME could be doing something here
 	// if err == unix.ENOSPC {
 	// 	log.Printf("No space")

backend/mega/mega.go

@@ -402,6 +402,35 @@ func (f *Fs) clearRoot() {
 	//log.Printf("cleared root directory")
 }
 
+// CleanUp deletes all files currently in trash
+func (f *Fs) CleanUp() (err error) {
+	trash := f.srv.FS.GetTrash()
+	items := []*mega.Node{}
+	_, err = f.list(trash, func(item *mega.Node) bool {
+		items = append(items, item)
+		return false
+	})
+	if err != nil {
+		return errors.Wrap(err, "CleanUp failed to list items in trash")
+	}
+	fs.Infof(f, "Deleting %d items from the trash", len(items))
+	errors := 0
+	// similar to f.deleteNode(trash) but with HardDelete as true
+	for _, item := range items {
+		fs.Debugf(f, "Deleting trash %q", item.GetName())
+		deleteErr := f.pacer.Call(func() (bool, error) {
+			err := f.srv.Delete(item, true)
+			return shouldRetry(err)
+		})
+		if deleteErr != nil {
+			err = deleteErr
+			errors++
+		}
+	}
+	fs.Infof(f, "Deleted %d items from the trash with %d errors", len(items), errors)
+	return err
+}
+
 // Return an Object from a path
 //
 // If it can't be found it returns the error fs.ErrorObjectNotFound.

backend/onedrive/onedrive.go

@@ -14,6 +14,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/ncw/rclone/lib/atexit"
+
 	"github.com/ncw/rclone/backend/onedrive/api"
 	"github.com/ncw/rclone/fs"
 	"github.com/ncw/rclone/fs/config"
@@ -335,8 +337,13 @@ func shouldRetry(resp *http.Response, err error) (bool, error) {
 
 // readMetaDataForPathRelativeToID reads the metadata for a path relative to an item that is addressed by its normalized ID.
 // if `relPath` == "", it reads the metadata for the item with that ID.
+//
+// We address items using the pattern `drives/driveID/items/itemID:/relativePath`
+// instead of simply using `drives/driveID/root:/itemPath` because it works for
+// "shared with me" folders in OneDrive Personal (See #2536, #2778)
+// This path pattern comes from https://github.com/OneDrive/onedrive-api-docs/issues/908#issuecomment-417488480
 func (f *Fs) readMetaDataForPathRelativeToID(normalizedID string, relPath string) (info *api.Item, resp *http.Response, err error) {
-	opts := newOptsCall(normalizedID, "GET", ":/"+rest.URLPathEscape(replaceReservedChars(relPath)))
+	opts := newOptsCall(normalizedID, "GET", ":/"+withTrailingColon(rest.URLPathEscape(replaceReservedChars(relPath))))
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(&opts, nil, &info)
 		return shouldRetry(resp, err)
@@ -378,9 +385,19 @@ func (f *Fs) readMetaDataForPath(path string) (info *api.Item, resp *http.Respon
 	var dirCacheFoundRoot bool
 	var rootNormalizedID string
 	if f.dirCache != nil {
-		var ok bool
-		if rootNormalizedID, ok = f.dirCache.Get(""); ok {
-			dirCacheFoundRoot = true
+		var dirCacheRootIDExists bool
+		rootNormalizedID, dirCacheRootIDExists = f.dirCache.Get("")
+		if f.root == "" {
+			// if f.root == "", it means f.root is the absolute root of the drive
+			// and its ID should have been found in NewFs
+			dirCacheFoundRoot = dirCacheRootIDExists
+		} else if _, err := f.dirCache.RootParentID(); err == nil {
+			// if root is in a folder, it must have a parent folder, and
+			// if dirCache has found root in NewFs, the parent folder's ID
+			// should be present.
+			// This RootParentID() check is a fix for #3164 which describes
+			// a possible case where the root is not found.
+			dirCacheFoundRoot = dirCacheRootIDExists
 		}
 	}
 
@@ -703,9 +720,7 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 			id := info.GetID()
 			f.dirCache.Put(remote, id)
 			d := fs.NewDir(remote, time.Time(info.GetLastModifiedDateTime())).SetID(id)
-			if folder != nil {
-				d.SetItems(folder.ChildCount)
-			}
+			d.SetItems(folder.ChildCount)
 			entries = append(entries, d)
 		} else {
 			o, err := f.newObjectWithInfo(remote, info)
@@ -819,9 +834,6 @@ func (f *Fs) purgeCheck(dir string, check bool) error {
 		return err
 	}
 	f.dirCache.FlushDir(dir)
-	if err != nil {
-		return err
-	}
 	return nil
 }
 
@@ -1340,12 +1352,12 @@ func (o *Object) setModTime(modTime time.Time) (*api.Item, error) {
 		opts = rest.Opts{
 			Method:  "PATCH",
 			RootURL: rootURL,
-			Path:    "/" + drive + "/items/" + trueDirID + ":/" + rest.URLPathEscape(leaf),
+			Path:    "/" + drive + "/items/" + trueDirID + ":/" + withTrailingColon(rest.URLPathEscape(leaf)),
 		}
 	} else {
 		opts = rest.Opts{
 			Method: "PATCH",
-			Path:   "/root:/" + rest.URLPathEscape(o.srvPath()),
+			Path:   "/root:/" + withTrailingColon(rest.URLPathEscape(o.srvPath())),
 		}
 	}
 	update := api.SetFileSystemInfo{
@@ -1491,23 +1503,40 @@ func (o *Object) uploadMultipart(in io.Reader, size int64, modTime time.Time) (i
 		return nil, errors.New("unknown-sized upload not supported")
 	}
 
+	uploadURLChan := make(chan string, 1)
+	gracefulCancel := func() {
+		uploadURL, ok := <-uploadURLChan
+		// Reading from uploadURLChan blocks the atexit process until
+		// we are able to use uploadURL to cancel the upload
+		if !ok { // createUploadSession failed - no need to cancel upload
+			return
+		}
+
+		fs.Debugf(o, "Cancelling multipart upload")
+		cancelErr := o.cancelUploadSession(uploadURL)
+		if cancelErr != nil {
+			fs.Logf(o, "Failed to cancel multipart upload: %v", cancelErr)
+		}
+	}
+	cancelFuncHandle := atexit.Register(gracefulCancel)
+
 	// Create upload session
 	fs.Debugf(o, "Starting multipart upload")
 	session, err := o.createUploadSession(modTime)
 	if err != nil {
+		close(uploadURLChan)
+		atexit.Unregister(cancelFuncHandle)
 		return nil, err
 	}
 	uploadURL := session.UploadURL
+	uploadURLChan <- uploadURL
 
-	// Cancel the session if something went wrong
 	defer func() {
 		if err != nil {
-			fs.Debugf(o, "Cancelling multipart upload: %v", err)
-			cancelErr := o.cancelUploadSession(uploadURL)
-			if cancelErr != nil {
-				fs.Logf(o, "Failed to cancel multipart upload: %v", err)
-			}
+			fs.Debugf(o, "Error encountered during upload: %v", err)
+			gracefulCancel()
 		}
+		atexit.Unregister(cancelFuncHandle)
 	}()
 
 	// Upload the chunks
@@ -1668,6 +1697,21 @@ func getRelativePathInsideBase(base, target string) (string, bool) {
 	return "", false
 }
 
+// Adds a ":" at the end of `remotePath` in a proper manner.
+// If `remotePath` already ends with "/", change it to ":/"
+// If `remotePath` is "", return "".
+// A workaround for #2720 and #3039
+func withTrailingColon(remotePath string) string {
+	if remotePath == "" {
+		return ""
+	}
+
+	if strings.HasSuffix(remotePath, "/") {
+		return remotePath[:len(remotePath)-1] + ":/"
+	}
+	return remotePath + ":"
+}
+
 // Check the interfaces are satisfied
 var (
 	_ fs.Fs              = (*Fs)(nil)

backend/opendrive/opendrive.go

@@ -287,9 +287,6 @@ func (f *Fs) purgeCheck(dir string, check bool) error {
 		return err
 	}
 	f.dirCache.FlushDir(dir)
-	if err != nil {
-		return err
-	}
 	return nil
 }
 

backend/s3/s3.go

@@ -369,6 +369,10 @@ func init() {
 				Value:    "s3.us-west-1.wasabisys.com",
 				Help:     "Wasabi US West endpoint",
 				Provider: "Wasabi",
+			}, {
+				Value:    "s3.eu-central-1.wasabisys.com",
+				Help:     "Wasabi EU Central endpoint",
+				Provider: "Wasabi",
 			}},
 		}, {
 			Name:     "location_constraint",
@@ -645,6 +649,9 @@ isn't set then "acl" is used instead.`,
 			}, {
 				Value: "GLACIER",
 				Help:  "Glacier storage class",
+			}, {
+				Value: "DEEP_ARCHIVE",
+				Help:  "Glacier Deep Archive storage class",
 			}},
 		}, {
 			// Mapping from here: https://www.alibabacloud.com/help/doc-detail/64919.htm
@@ -729,6 +736,14 @@ If it is set then rclone will use v2 authentication.
 Use this only if v4 signatures don't work, eg pre Jewel/v10 CEPH.`,
 			Default:  false,
 			Advanced: true,
+		}, {
+			Name:     "use_accelerate_endpoint",
+			Provider: "AWS",
+			Help: `If true use the AWS S3 accelerated endpoint.
+
+See: [AWS S3 Transfer acceleration](https://docs.aws.amazon.com/AmazonS3/latest/dev/transfer-acceleration-examples.html)`,
+			Default:  false,
+			Advanced: true,
 		}},
 	})
 }
@@ -749,25 +764,26 @@ const (
 
 // Options defines the configuration for this backend
 type Options struct {
-	Provider             string        `config:"provider"`
-	EnvAuth              bool          `config:"env_auth"`
-	AccessKeyID          string        `config:"access_key_id"`
-	SecretAccessKey      string        `config:"secret_access_key"`
-	Region               string        `config:"region"`
-	Endpoint             string        `config:"endpoint"`
-	LocationConstraint   string        `config:"location_constraint"`
-	ACL                  string        `config:"acl"`
-	BucketACL            string        `config:"bucket_acl"`
-	ServerSideEncryption string        `config:"server_side_encryption"`
-	SSEKMSKeyID          string        `config:"sse_kms_key_id"`
-	StorageClass         string        `config:"storage_class"`
-	UploadCutoff         fs.SizeSuffix `config:"upload_cutoff"`
-	ChunkSize            fs.SizeSuffix `config:"chunk_size"`
-	DisableChecksum      bool          `config:"disable_checksum"`
-	SessionToken         string        `config:"session_token"`
-	UploadConcurrency    int           `config:"upload_concurrency"`
-	ForcePathStyle       bool          `config:"force_path_style"`
-	V2Auth               bool          `config:"v2_auth"`
+	Provider              string        `config:"provider"`
+	EnvAuth               bool          `config:"env_auth"`
+	AccessKeyID           string        `config:"access_key_id"`
+	SecretAccessKey       string        `config:"secret_access_key"`
+	Region                string        `config:"region"`
+	Endpoint              string        `config:"endpoint"`
+	LocationConstraint    string        `config:"location_constraint"`
+	ACL                   string        `config:"acl"`
+	BucketACL             string        `config:"bucket_acl"`
+	ServerSideEncryption  string        `config:"server_side_encryption"`
+	SSEKMSKeyID           string        `config:"sse_kms_key_id"`
+	StorageClass          string        `config:"storage_class"`
+	UploadCutoff          fs.SizeSuffix `config:"upload_cutoff"`
+	ChunkSize             fs.SizeSuffix `config:"chunk_size"`
+	DisableChecksum       bool          `config:"disable_checksum"`
+	SessionToken          string        `config:"session_token"`
+	UploadConcurrency     int           `config:"upload_concurrency"`
+	ForcePathStyle        bool          `config:"force_path_style"`
+	V2Auth                bool          `config:"v2_auth"`
+	UseAccelerateEndpoint bool          `config:"use_accelerate_endpoint"`
 }
 
 // Fs represents a remote s3 server
@@ -941,14 +957,15 @@ func s3Connection(opt *Options) (*s3.S3, *session.Session, error) {
 	if opt.Region == "" {
 		opt.Region = "us-east-1"
 	}
-	if opt.Provider == "Alibaba" || opt.Provider == "Netease" {
+	if opt.Provider == "Alibaba" || opt.Provider == "Netease" || opt.UseAccelerateEndpoint {
 		opt.ForcePathStyle = false
 	}
 	awsConfig := aws.NewConfig().
 		WithMaxRetries(maxRetries).
 		WithCredentials(cred).
 		WithHTTPClient(fshttp.NewClient(fs.Config)).
-		WithS3ForcePathStyle(opt.ForcePathStyle)
+		WithS3ForcePathStyle(opt.ForcePathStyle).
+		WithS3UseAccelerate(opt.UseAccelerateEndpoint)
 	if opt.Region != "" {
 		awsConfig.WithRegion(opt.Region)
 	}
@@ -1719,6 +1736,9 @@ func (o *Object) SetModTime(modTime time.Time) error {
 	if o.fs.opt.SSEKMSKeyID != "" {
 		req.SSEKMSKeyId = &o.fs.opt.SSEKMSKeyID
 	}
+	if o.fs.opt.StorageClass == "GLACIER" || o.fs.opt.StorageClass == "DEEP_ARCHIVE" {
+		return fs.ErrorCantSetModTime
+	}
 	if o.fs.opt.StorageClass != "" {
 		req.StorageClass = &o.fs.opt.StorageClass
 	}

backend/sftp/sftp.go

@@ -1,6 +1,6 @@
 // Package sftp provides a filesystem interface using github.com/pkg/sftp
 
-// +build !plan9,go1.9
+// +build !plan9
 
 package sftp
 
@@ -14,6 +14,7 @@ import (
 	"os/user"
 	"path"
 	"regexp"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -25,6 +26,7 @@ import (
 	"github.com/ncw/rclone/fs/config/obscure"
 	"github.com/ncw/rclone/fs/fshttp"
 	"github.com/ncw/rclone/fs/hash"
+	"github.com/ncw/rclone/lib/env"
 	"github.com/ncw/rclone/lib/readers"
 	"github.com/pkg/errors"
 	"github.com/pkg/sftp"
@@ -186,10 +188,10 @@ func readCurrentUser() (userName string) {
 	return os.Getenv("LOGNAME")
 }
 
-// Dial starts a client connection to the given SSH server. It is a
+// dial starts a client connection to the given SSH server. It is a
 // convenience function that connects to the given network address,
 // initiates the SSH handshake, and then sets up a Client.
-func Dial(network, addr string, sshConfig *ssh.ClientConfig) (*ssh.Client, error) {
+func (f *Fs) dial(network, addr string, sshConfig *ssh.ClientConfig) (*ssh.Client, error) {
 	dialer := fshttp.NewDialer(fs.Config)
 	conn, err := dialer.Dial(network, addr)
 	if err != nil {
@@ -199,6 +201,7 @@ func Dial(network, addr string, sshConfig *ssh.ClientConfig) (*ssh.Client, error
 	if err != nil {
 		return nil, err
 	}
+	fs.Debugf(f, "New connection %s->%s to %q", c.LocalAddr(), c.RemoteAddr(), c.ServerVersion())
 	return ssh.NewClient(c, chans, reqs), nil
 }
 
@@ -244,7 +247,7 @@ func (f *Fs) sftpConnection() (c *conn, err error) {
 	c = &conn{
 		err: make(chan error, 1),
 	}
-	c.sshClient, err = Dial("tcp", f.opt.Host+":"+f.opt.Port, f.config)
+	c.sshClient, err = f.dial("tcp", f.opt.Host+":"+f.opt.Port, f.config)
 	if err != nil {
 		return nil, errors.Wrap(err, "couldn't connect SSH")
 	}
@@ -315,18 +318,6 @@ func (f *Fs) putSftpConnection(pc **conn, err error) {
 	f.poolMu.Unlock()
 }
 
-// shellExpand replaces a leading "~" with "${HOME}" and expands all environment
-// variables afterwards.
-func shellExpand(s string) string {
-	if s != "" {
-		if s[0] == '~' {
-			s = "${HOME}" + s[1:]
-		}
-		s = os.ExpandEnv(s)
-	}
-	return s
-}
-
 // NewFs creates a new Fs object from the name and root. It connects to
 // the host specified in the config file.
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
@@ -347,6 +338,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		Auth:            []ssh.AuthMethod{},
 		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
 		Timeout:         fs.Config.ConnectTimeout,
+		ClientVersion:   "SSH-2.0-" + fs.Config.UserAgent,
 	}
 
 	if opt.UseInsecureCipher {
@@ -354,7 +346,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		sshConfig.Config.Ciphers = append(sshConfig.Config.Ciphers, "aes128-cbc")
 	}
 
-	keyFile := shellExpand(opt.KeyFile)
+	keyFile := env.ShellExpand(opt.KeyFile)
 	// Add ssh agent-auth if no password or file specified
 	if (opt.Pass == "" && keyFile == "") || opt.KeyUseAgent {
 		sshAgentClient, _, err := sshagent.New()
@@ -813,6 +805,49 @@ func (f *Fs) Hashes() hash.Set {
 	return set
 }
 
+// About gets usage stats
+func (f *Fs) About() (*fs.Usage, error) {
+	c, err := f.getSftpConnection()
+	if err != nil {
+		return nil, errors.Wrap(err, "About get SFTP connection")
+	}
+	session, err := c.sshClient.NewSession()
+	f.putSftpConnection(&c, err)
+	if err != nil {
+		return nil, errors.Wrap(err, "About put SFTP connection")
+	}
+
+	var stdout, stderr bytes.Buffer
+	session.Stdout = &stdout
+	session.Stderr = &stderr
+	escapedPath := shellEscape(f.root)
+	if f.opt.PathOverride != "" {
+		escapedPath = shellEscape(path.Join(f.opt.PathOverride, f.root))
+	}
+	if len(escapedPath) == 0 {
+		escapedPath = "/"
+	}
+	err = session.Run("df -k " + escapedPath)
+	if err != nil {
+		_ = session.Close()
+		return nil, errors.Wrap(err, "About invocation of df failed. Your remote may not support about.")
+	}
+	_ = session.Close()
+
+	usageTotal, usageUsed, usageAvail := parseUsage(stdout.Bytes())
+	usage := &fs.Usage{}
+	if usageTotal >= 0 {
+		usage.Total = fs.NewUsageValue(usageTotal)
+	}
+	if usageUsed >= 0 {
+		usage.Used = fs.NewUsageValue(usageUsed)
+	}
+	if usageAvail >= 0 {
+		usage.Free = fs.NewUsageValue(usageAvail)
+	}
+	return usage, nil
+}
+
 // Fs is the filesystem this remote sftp file object is located within
 func (o *Object) Fs() fs.Info {
 	return o.fs
@@ -903,6 +938,35 @@ func parseHash(bytes []byte) string {
 	return strings.Split(string(bytes), " ")[0] // Split at hash / filename separator
 }
 
+// Parses the byte array output from the SSH session
+// returned by an invocation of df into
+// the disk size, used space, and avaliable space on the disk, in that order.
+// Only works when `df` has output info on only one disk
+func parseUsage(bytes []byte) (spaceTotal int64, spaceUsed int64, spaceAvail int64) {
+	spaceTotal, spaceUsed, spaceAvail = -1, -1, -1
+	lines := strings.Split(string(bytes), "\n")
+	if len(lines) < 2 {
+		return
+	}
+	split := strings.Fields(lines[1])
+	if len(split) < 6 {
+		return
+	}
+	spaceTotal, err := strconv.ParseInt(split[1], 10, 64)
+	if err != nil {
+		spaceTotal = -1
+	}
+	spaceUsed, err = strconv.ParseInt(split[2], 10, 64)
+	if err != nil {
+		spaceUsed = -1
+	}
+	spaceAvail, err = strconv.ParseInt(split[3], 10, 64)
+	if err != nil {
+		spaceAvail = -1
+	}
+	return spaceTotal * 1024, spaceUsed * 1024, spaceAvail * 1024
+}
+
 // Size returns the size in bytes of the remote sftp file
 func (o *Object) Size() int64 {
 	return o.size

backend/sftp/sftp_internal_test.go

@@ -1,4 +1,4 @@
-// +build !plan9,go1.9
+// +build !plan9
 
 package sftp
 
@@ -35,3 +35,17 @@ func TestParseHash(t *testing.T) {
 		assert.Equal(t, test.checksum, got, fmt.Sprintf("Test %d sshOutput = %q", i, test.sshOutput))
 	}
 }
+
+func TestParseUsage(t *testing.T) {
+	for i, test := range []struct {
+		sshOutput string
+		usage     [3]int64
+	}{
+		{"Filesystem     1K-blocks     Used Available Use% Mounted on\n/dev/root       91283092 81111888  10154820  89% /", [3]int64{93473886208, 83058573312, 10398535680}},
+		{"Filesystem     1K-blocks  Used Available Use% Mounted on\ntmpfs             818256  1636    816620   1% /run", [3]int64{837894144, 1675264, 836218880}},
+		{"Filesystem   1024-blocks     Used Available Capacity iused      ifree %iused  Mounted on\n/dev/disk0s2   244277768 94454848 149566920    39%  997820 4293969459    0%   /", [3]int64{250140434432, 96721764352, 153156526080}},
+	} {
+		gotSpaceTotal, gotSpaceUsed, gotSpaceAvail := parseUsage([]byte(test.sshOutput))
+		assert.Equal(t, test.usage, [3]int64{gotSpaceTotal, gotSpaceUsed, gotSpaceAvail}, fmt.Sprintf("Test %d sshOutput = %q", i, test.sshOutput))
+	}
+}

backend/sftp/sftp_test.go

@@ -1,6 +1,6 @@
 // Test Sftp filesystem interface
 
-// +build !plan9,go1.9
+// +build !plan9
 
 package sftp_test
 

backend/sftp/sftp_unsupported.go

@@ -1,6 +1,6 @@
 // Build for sftp for unsupported platforms to stop go complaining
 // about "no buildable Go source files "
 
-// +build plan9 !go1.9
+// +build plan9
 
 package sftp

backend/sftp/stringlock.go

@@ -1,4 +1,4 @@
-// +build !plan9,go1.9
+// +build !plan9
 
 package sftp
 

backend/sftp/stringlock_test.go

@@ -1,4 +1,4 @@
-// +build !plan9,go1.9
+// +build !plan9
 
 package sftp
 

backend/swift/auth.go

@@ -2,6 +2,7 @@ package swift
 
 import (
 	"net/http"
+	"time"
 
 	"github.com/ncw/swift"
 )
@@ -65,6 +66,14 @@ func (a *auth) Token() string {
 	return a.parentAuth.Token()
 }
 
+// Expires returns the time the token expires if known or Zero if not.
+func (a *auth) Expires() (t time.Time) {
+	if do, ok := a.parentAuth.(swift.Expireser); ok {
+		t = do.Expires()
+	}
+	return t
+}
+
 // The CDN url if available
 func (a *auth) CdnUrl() string { // nolint
 	if a.parentAuth == nil {
@@ -74,4 +83,7 @@ func (a *auth) CdnUrl() string { // nolint
 }
 
 // Check the interfaces are satisfied
-var _ swift.Authenticator = (*auth)(nil)
+var (
+	_ swift.Authenticator = (*auth)(nil)
+	_ swift.Expireser     = (*auth)(nil)
+)

backend/swift/swift.go

@@ -286,6 +286,31 @@ func shouldRetry(err error) (bool, error) {
 	return fserrors.ShouldRetry(err), err
 }
 
+// shouldRetryHeaders returns a boolean as to whether this err
+// deserves to be retried.  It reads the headers passed in looking for
+// `Retry-After`. It returns the err as a convenience
+func shouldRetryHeaders(headers swift.Headers, err error) (bool, error) {
+	if swiftError, ok := err.(*swift.Error); ok && swiftError.StatusCode == 429 {
+		if value := headers["Retry-After"]; value != "" {
+			retryAfter, parseErr := strconv.Atoi(value)
+			if parseErr != nil {
+				fs.Errorf(nil, "Failed to parse Retry-After: %q: %v", value, parseErr)
+			} else {
+				duration := time.Second * time.Duration(retryAfter)
+				if duration <= 60*time.Second {
+					// Do a short sleep immediately
+					fs.Debugf(nil, "Sleeping for %v to obey Retry-After", duration)
+					time.Sleep(duration)
+					return true, err
+				}
+				// Delay a long sleep for a retry
+				return false, fserrors.NewErrorRetryAfter(duration)
+			}
+		}
+	}
+	return shouldRetry(err)
+}
+
 // Pattern to match a swift path
 var matcher = regexp.MustCompile(`^/*([^/]*)(.*)$`)
 
@@ -413,8 +438,9 @@ func NewFsWithConnection(opt *Options, name, root string, c *swift.Connection, n
 		// Check to see if the object exists - ignoring directory markers
 		var info swift.Object
 		err = f.pacer.Call(func() (bool, error) {
-			info, _, err = f.c.Object(container, directory)
-			return shouldRetry(err)
+			var rxHeaders swift.Headers
+			info, rxHeaders, err = f.c.Object(container, directory)
+			return shouldRetryHeaders(rxHeaders, err)
 		})
 		if err == nil && info.ContentType != directoryMarkerContentType {
 			f.root = path.Dir(directory)
@@ -723,8 +749,9 @@ func (f *Fs) Mkdir(dir string) error {
 	var err error = swift.ContainerNotFound
 	if !f.noCheckContainer {
 		err = f.pacer.Call(func() (bool, error) {
-			_, _, err = f.c.Container(f.container)
-			return shouldRetry(err)
+			var rxHeaders swift.Headers
+			_, rxHeaders, err = f.c.Container(f.container)
+			return shouldRetryHeaders(rxHeaders, err)
 		})
 	}
 	if err == swift.ContainerNotFound {
@@ -816,8 +843,9 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
 	}
 	srcFs := srcObj.fs
 	err = f.pacer.Call(func() (bool, error) {
-		_, err = f.c.ObjectCopy(srcFs.container, srcFs.root+srcObj.remote, f.container, f.root+remote, nil)
-		return shouldRetry(err)
+		var rxHeaders swift.Headers
+		rxHeaders, err = f.c.ObjectCopy(srcFs.container, srcFs.root+srcObj.remote, f.container, f.root+remote, nil)
+		return shouldRetryHeaders(rxHeaders, err)
 	})
 	if err != nil {
 		return nil, err
@@ -927,7 +955,7 @@ func (o *Object) readMetaData() (err error) {
 	var h swift.Headers
 	err = o.fs.pacer.Call(func() (bool, error) {
 		info, h, err = o.fs.c.Object(o.fs.container, o.fs.root+o.remote)
-		return shouldRetry(err)
+		return shouldRetryHeaders(h, err)
 	})
 	if err != nil {
 		if err == swift.ObjectNotFound {
@@ -1002,8 +1030,9 @@ func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
 	headers := fs.OpenOptionHeaders(options)
 	_, isRanging := headers["Range"]
 	err = o.fs.pacer.Call(func() (bool, error) {
-		in, _, err = o.fs.c.ObjectOpen(o.fs.container, o.fs.root+o.remote, !isRanging, headers)
-		return shouldRetry(err)
+		var rxHeaders swift.Headers
+		in, rxHeaders, err = o.fs.c.ObjectOpen(o.fs.container, o.fs.root+o.remote, !isRanging, headers)
+		return shouldRetryHeaders(rxHeaders, err)
 	})
 	return
 }
@@ -1074,8 +1103,9 @@ func (o *Object) updateChunks(in0 io.Reader, headers swift.Headers, size int64,
 	// Create the segmentsContainer if it doesn't exist
 	var err error
 	err = o.fs.pacer.Call(func() (bool, error) {
-		_, _, err = o.fs.c.Container(o.fs.segmentsContainer)
-		return shouldRetry(err)
+		var rxHeaders swift.Headers
+		_, rxHeaders, err = o.fs.c.Container(o.fs.segmentsContainer)
+		return shouldRetryHeaders(rxHeaders, err)
 	})
 	if err == swift.ContainerNotFound {
 		headers := swift.Headers{}
@@ -1115,8 +1145,9 @@ func (o *Object) updateChunks(in0 io.Reader, headers swift.Headers, size int64,
 		segmentPath := fmt.Sprintf("%s/%08d", segmentsPath, i)
 		fs.Debugf(o, "Uploading segment file %q into %q", segmentPath, o.fs.segmentsContainer)
 		err = o.fs.pacer.CallNoRetry(func() (bool, error) {
-			_, err = o.fs.c.ObjectPut(o.fs.segmentsContainer, segmentPath, segmentReader, true, "", "", headers)
-			return shouldRetry(err)
+			var rxHeaders swift.Headers
+			rxHeaders, err = o.fs.c.ObjectPut(o.fs.segmentsContainer, segmentPath, segmentReader, true, "", "", headers)
+			return shouldRetryHeaders(rxHeaders, err)
 		})
 		if err != nil {
 			return "", err
@@ -1129,8 +1160,9 @@ func (o *Object) updateChunks(in0 io.Reader, headers swift.Headers, size int64,
 	emptyReader := bytes.NewReader(nil)
 	manifestName := o.fs.root + o.remote
 	err = o.fs.pacer.Call(func() (bool, error) {
-		_, err = o.fs.c.ObjectPut(o.fs.container, manifestName, emptyReader, true, "", contentType, headers)
-		return shouldRetry(err)
+		var rxHeaders swift.Headers
+		rxHeaders, err = o.fs.c.ObjectPut(o.fs.container, manifestName, emptyReader, true, "", contentType, headers)
+		return shouldRetryHeaders(rxHeaders, err)
 	})
 	return uniquePrefix + "/", err
 }
@@ -1174,7 +1206,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 		var rxHeaders swift.Headers
 		err = o.fs.pacer.CallNoRetry(func() (bool, error) {
 			rxHeaders, err = o.fs.c.ObjectPut(o.fs.container, o.fs.root+o.remote, in, true, "", contentType, headers)
-			return shouldRetry(err)
+			return shouldRetryHeaders(rxHeaders, err)
 		})
 		if err != nil {
 			return err

backend/swift/swift_internal_test.go

@@ -1,6 +1,13 @@
 package swift
 
-import "testing"
+import (
+	"testing"
+	"time"
+
+	"github.com/ncw/rclone/fs/fserrors"
+	"github.com/ncw/swift"
+	"github.com/stretchr/testify/assert"
+)
 
 func TestInternalUrlEncode(t *testing.T) {
 	for _, test := range []struct {
@@ -23,3 +30,37 @@ func TestInternalUrlEncode(t *testing.T) {
 		}
 	}
 }
+
+func TestInternalShouldRetryHeaders(t *testing.T) {
+	headers := swift.Headers{
+		"Content-Length": "64",
+		"Content-Type":   "text/html; charset=UTF-8",
+		"Date":           "Mon: 18 Mar 2019 12:11:23 GMT",
+		"Retry-After":    "1",
+	}
+	err := &swift.Error{
+		StatusCode: 429,
+		Text:       "Too Many Requests",
+	}
+
+	// Short sleep should just do the sleep
+	start := time.Now()
+	retry, gotErr := shouldRetryHeaders(headers, err)
+	dt := time.Since(start)
+	assert.True(t, retry)
+	assert.Equal(t, err, gotErr)
+	assert.True(t, dt > time.Second/2)
+
+	// Long sleep should return RetryError
+	headers["Retry-After"] = "3600"
+	start = time.Now()
+	retry, gotErr = shouldRetryHeaders(headers, err)
+	dt = time.Since(start)
+	assert.True(t, dt < time.Second)
+	assert.False(t, retry)
+	assert.Equal(t, true, fserrors.IsRetryAfterError(gotErr))
+	after := gotErr.(fserrors.RetryAfter).RetryAfter()
+	dt = after.Sub(start)
+	assert.True(t, dt >= time.Hour-time.Second && dt <= time.Hour+time.Second)
+
+}

backend/union/union.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/cache"
 	"github.com/ncw/rclone/fs/config/configmap"
 	"github.com/ncw/rclone/fs/config/configstruct"
 	"github.com/ncw/rclone/fs/hash"
@@ -342,7 +343,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		if configName != "local" {
 			rootString = configName + ":" + rootString
 		}
-		myFs, err := fs.NewFs(rootString)
+		myFs, err := cache.Get(rootString)
 		if err != nil {
 			if err == fs.ErrorIsFile {
 				return myFs, err

backend/webdav/webdav.go

@@ -644,10 +644,18 @@ func (f *Fs) _mkdir(dirPath string) error {
 		Path:       dirPath,
 		NoResponse: true,
 	}
-	return f.pacer.Call(func() (bool, error) {
+	err := f.pacer.Call(func() (bool, error) {
 		resp, err := f.srv.Call(&opts)
 		return shouldRetry(resp, err)
 	})
+	if apiErr, ok := err.(*api.Error); ok {
+		// already exists
+		// owncloud returns 423/StatusLocked if the create is already in progress
+		if apiErr.StatusCode == http.StatusMethodNotAllowed || apiErr.StatusCode == http.StatusNotAcceptable || apiErr.StatusCode == http.StatusLocked {
+			return nil
+		}
+	}
+	return err
 }
 
 // mkdir makes the directory and parents using native paths
@@ -655,12 +663,7 @@ func (f *Fs) mkdir(dirPath string) error {
 	// defer log.Trace(dirPath, "")("")
 	err := f._mkdir(dirPath)
 	if apiErr, ok := err.(*api.Error); ok {
-		// already exists
-		// owncloud returns 423/StatusLocked if the create is already in progress
-		if apiErr.StatusCode == http.StatusMethodNotAllowed || apiErr.StatusCode == http.StatusNotAcceptable || apiErr.StatusCode == http.StatusLocked {
-			return nil
-		}
-		// parent does not exist
+		// parent does not exist so create it first then try again
 		if apiErr.StatusCode == http.StatusConflict {
 			err = f.mkParentDir(dirPath)
 			if err == nil {
@@ -913,11 +916,13 @@ func (f *Fs) About() (*fs.Usage, error) {
 		return nil, errors.Wrap(err, "about call failed")
 	}
 	usage := &fs.Usage{}
-	if q.Available >= 0 && q.Used >= 0 {
-		usage.Total = fs.NewUsageValue(q.Available + q.Used)
-	}
-	if q.Used >= 0 {
-		usage.Used = fs.NewUsageValue(q.Used)
+	if q.Available != 0 || q.Used != 0 {
+		if q.Available >= 0 && q.Used >= 0 {
+			usage.Total = fs.NewUsageValue(q.Available + q.Used)
+		}
+		if q.Used >= 0 {
+			usage.Used = fs.NewUsageValue(q.Used)
+		}
 	}
 	return usage, nil
 }

bin/get-github-release.go

@@ -244,8 +244,10 @@ func getAssetFromReleasesPage(project string, matchName *regexp.Regexp) (assetUR
 				if a.Key == "href" {
 					if name := path.Base(a.Val); matchName.MatchString(name) && isOurOsArch(name) {
 						if u, err := rest.URLJoin(base, a.Val); err == nil {
-							assetName = name
-							assetURL = u.String()
+							if assetName == "" {
+								assetName = name
+								assetURL = u.String()
+							}
 						}
 					}
 					break

cmd/about/about.go

@@ -95,6 +95,9 @@ Use the --json flag for a computer readable output, eg
 			if err != nil {
 				return errors.Wrap(err, "About call failed")
 			}
+			if u == nil {
+				return errors.New("nil usage returned")
+			}
 			if jsonOutput {
 				out := json.NewEncoder(os.Stdout)
 				out.SetIndent("", "\t")

cmd/cmd.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/ncw/rclone/fs"
 	"github.com/ncw/rclone/fs/accounting"
+	"github.com/ncw/rclone/fs/cache"
 	"github.com/ncw/rclone/fs/config/configflags"
 	"github.com/ncw/rclone/fs/config/flags"
 	"github.com/ncw/rclone/fs/filter"
@@ -83,7 +84,7 @@ func NewFsFile(remote string) (fs.Fs, string) {
 		fs.CountError(err)
 		log.Fatalf("Failed to create file system for %q: %v", remote, err)
 	}
-	f, err := fs.NewFs(remote)
+	f, err := cache.Get(remote)
 	switch err {
 	case fs.ErrorIsFile:
 		return f, path.Base(fsPath)
@@ -131,7 +132,7 @@ func NewFsSrc(args []string) fs.Fs {
 //
 // This must point to a directory
 func newFsDir(remote string) fs.Fs {
-	f, err := fs.NewFs(remote)
+	f, err := cache.Get(remote)
 	if err != nil {
 		fs.CountError(err)
 		log.Fatalf("Failed to create file system for %q: %v", remote, err)
@@ -180,7 +181,7 @@ func NewFsSrcDstFiles(args []string) (fsrc fs.Fs, srcFileName string, fdst fs.Fs
 			log.Fatalf("%q is a directory", args[1])
 		}
 	}
-	fdst, err := fs.NewFs(dstRemote)
+	fdst, err := cache.Get(dstRemote)
 	switch err {
 	case fs.ErrorIsFile:
 		fs.CountError(err)
@@ -230,22 +231,34 @@ func Run(Retry bool, showStats bool, cmd *cobra.Command, f func() error) {
 	SigInfoHandler()
 	for try := 1; try <= *retries; try++ {
 		err = f()
-		if !Retry || (err == nil && !accounting.Stats.Errored()) {
+		fs.CountError(err)
+		lastErr := accounting.Stats.GetLastError()
+		if err == nil {
+			err = lastErr
+		}
+		if !Retry || !accounting.Stats.Errored() {
 			if try > 1 {
 				fs.Errorf(nil, "Attempt %d/%d succeeded", try, *retries)
 			}
 			break
 		}
-		if fserrors.IsFatalError(err) || accounting.Stats.HadFatalError() {
+		if accounting.Stats.HadFatalError() {
 			fs.Errorf(nil, "Fatal error received - not attempting retries")
 			break
 		}
-		if fserrors.IsNoRetryError(err) || (accounting.Stats.Errored() && !accounting.Stats.HadRetryError()) {
+		if accounting.Stats.Errored() && !accounting.Stats.HadRetryError() {
 			fs.Errorf(nil, "Can't retry this error - not attempting retries")
 			break
 		}
-		if err != nil {
-			fs.Errorf(nil, "Attempt %d/%d failed with %d errors and: %v", try, *retries, accounting.Stats.GetErrors(), err)
+		if retryAfter := accounting.Stats.RetryAfter(); !retryAfter.IsZero() {
+			d := retryAfter.Sub(time.Now())
+			if d > 0 {
+				fs.Logf(nil, "Received retry after error - sleeping until %s (%v)", retryAfter.Format(time.RFC3339Nano), d)
+				time.Sleep(d)
+			}
+		}
+		if lastErr != nil {
+			fs.Errorf(nil, "Attempt %d/%d failed with %d errors and: %v", try, *retries, accounting.Stats.GetErrors(), lastErr)
 		} else {
 			fs.Errorf(nil, "Attempt %d/%d failed with %d errors", try, *retries, accounting.Stats.GetErrors())
 		}
@@ -258,7 +271,12 @@ func Run(Retry bool, showStats bool, cmd *cobra.Command, f func() error) {
 	}
 	stopStats()
 	if err != nil {
-		log.Printf("Failed to %s: %v", cmd.Name(), err)
+		nerrs := accounting.Stats.GetErrors()
+		if nerrs <= 1 {
+			log.Printf("Failed to %s: %v", cmd.Name(), err)
+		} else {
+			log.Printf("Failed to %s with %d errors: last error was: %v", cmd.Name(), nerrs, err)
+		}
 		resolveExitCode(err)
 	}
 	if showStats && (accounting.Stats.Errored() || *statsInterval > 0) {

cmd/cmount/mount.go

@@ -127,7 +127,7 @@ func waitFor(fn func() bool) (ok bool) {
 func mount(f fs.Fs, mountpoint string) (*vfs.VFS, <-chan error, func() error, error) {
 	fs.Debugf(f, "Mounting on %q", mountpoint)
 
-	// Check the mountpoint - in Windows the mountpoint musn't exist before the mount
+	// Check the mountpoint - in Windows the mountpoint mustn't exist before the mount
 	if runtime.GOOS != "windows" {
 		fi, err := os.Stat(mountpoint)
 		if err != nil {

cmd/gendocs/gendocs.go

@@ -11,6 +11,7 @@ import (
 	"github.com/ncw/rclone/cmd"
 	"github.com/spf13/cobra"
 	"github.com/spf13/cobra/doc"
+	"github.com/spf13/pflag"
 )
 
 func init() {
@@ -50,6 +51,10 @@ rclone.org website.`,
 			base := strings.TrimSuffix(name, path.Ext(name))
 			return "/commands/" + strings.ToLower(base) + "/"
 		}
+		// Hide all of the root entries flags
+		cmd.Root.Flags().VisitAll(func(flag *pflag.Flag) {
+			flag.Hidden = true
+		})
 		return doc.GenMarkdownTreeCustom(cmd.Root, out, prepender, linkHandler)
 	},
 }

cmd/help.go

@@ -45,7 +45,7 @@ __rclone_custom_func() {
         else
             __rclone_init_completion -n : || return
         fi
-        if [[ $cur =~ ^[[:alnum:]_]*$ ]]; then
+        if [[ $cur != *:* ]]; then
             local remote
             while IFS= read -r remote; do
                 [[ $remote != $cur* ]] || COMPREPLY+=("$remote")
@@ -54,10 +54,10 @@ __rclone_custom_func() {
                 local paths=("$cur"*)
                 [[ ! -f ${paths[0]} ]] || COMPREPLY+=("${paths[@]}")
             fi
-        elif [[ $cur =~ ^[[:alnum:]_]+: ]]; then
+        else
             local path=${cur#*:}
             if [[ $path == */* ]]; then
-                local prefix=${path%/*}
+                local prefix=$(eval printf '%s' "${path%/*}")
             else
                 local prefix=
             fi
@@ -66,6 +66,7 @@ __rclone_custom_func() {
                 local reply=${prefix:+$prefix/}$line
                 [[ $reply != $path* ]] || COMPREPLY+=("$reply")
             done < <(rclone lsf "${cur%%:*}:$prefix" 2>/dev/null)
+	    [[ ! ${COMPREPLY[@]} ]] || compopt -o filenames
         fi
         [[ ! ${COMPREPLY[@]} ]] || compopt -o nospace
     fi

cmd/ls/lshelp/lshelp.go

@@ -2,7 +2,7 @@ package lshelp
 
 // Help describes the common help for all the list commands
 var Help = `
-Any of the filtering options can be applied to this commmand.
+Any of the filtering options can be applied to this command.
 
 There are several related list commands
 

cmd/lsf/lsf.go

@@ -70,6 +70,7 @@ output:
     o - Original ID of underlying object
     m - MimeType of object if known
     e - encrypted name
+    T - tier of storage if known, eg "Hot" or "Cool"
 
 So if you wanted the path, size and modification time, you would use
 --format "pst", or maybe --format "tsp" to put the path last.
@@ -164,6 +165,8 @@ func Lsf(fsrc fs.Fs, out io.Writer) error {
 	list.SetAbsolute(absolute)
 	var opt = operations.ListJSONOpt{
 		NoModTime: true,
+		DirsOnly:  dirsOnly,
+		FilesOnly: filesOnly,
 		Recurse:   recurse,
 	}
 
@@ -189,21 +192,14 @@ func Lsf(fsrc fs.Fs, out io.Writer) error {
 		case 'o':
 			list.AddOrigID()
 			opt.ShowOrigIDs = true
+		case 'T':
+			list.AddTier()
 		default:
 			return errors.Errorf("Unknown format character %q", char)
 		}
 	}
 
 	return operations.ListJSON(fsrc, "", &opt, func(item *operations.ListJSONItem) error {
-		if item.IsDir {
-			if filesOnly {
-				return nil
-			}
-		} else {
-			if dirsOnly {
-				return nil
-			}
-		}
 		_, _ = fmt.Fprintln(out, list.Format(item))
 		return nil
 	})

cmd/lsjson/lsjson.go

@@ -23,6 +23,8 @@ func init() {
 	commandDefintion.Flags().BoolVarP(&opt.NoModTime, "no-modtime", "", false, "Don't read the modification time (can speed things up).")
 	commandDefintion.Flags().BoolVarP(&opt.ShowEncrypted, "encrypted", "M", false, "Show the encrypted names.")
 	commandDefintion.Flags().BoolVarP(&opt.ShowOrigIDs, "original", "", false, "Show the ID of the underlying Object.")
+	commandDefintion.Flags().BoolVarP(&opt.FilesOnly, "files-only", "", false, "Show only files in the listing.")
+	commandDefintion.Flags().BoolVarP(&opt.DirsOnly, "dirs-only", "", false, "Show only directories in the listing.")
 }
 
 var commandDefintion = &cobra.Command{
@@ -55,6 +57,10 @@ If --no-modtime is specified then ModTime will be blank.
 
 If --encrypted is not specified the Encrypted won't be emitted.
 
+If --dirs-only is not specified files in addition to directories are returned
+
+If --files-only is not specified directories in addition to the files will be returned.
+
 The Path field will only show folders below the remote path being listed.
 If "remote:path" contains the file "subfolder/file.txt", the Path for "file.txt"
 will be "subfolder/file.txt", not "remote:path/subfolder/file.txt".

cmd/mount/dir.go

@@ -3,6 +3,7 @@
 package mount
 
 import (
+	"context"
 	"os"
 	"time"
 
@@ -12,7 +13,6 @@ import (
 	"github.com/ncw/rclone/fs/log"
 	"github.com/ncw/rclone/vfs"
 	"github.com/pkg/errors"
-	"golang.org/x/net/context" // switch to "context" when we stop supporting go1.8
 )
 
 // Dir represents a directory entry
@@ -20,7 +20,7 @@ type Dir struct {
 	*vfs.Dir
 }
 
-// Check interface satsified
+// Check interface satisfied
 var _ fusefs.Node = (*Dir)(nil)
 
 // Attr updates the attributes of a directory

cmd/mount/file.go

@@ -3,6 +3,7 @@
 package mount
 
 import (
+	"context"
 	"io"
 	"time"
 
@@ -11,7 +12,6 @@ import (
 	"github.com/ncw/rclone/cmd/mountlib"
 	"github.com/ncw/rclone/fs/log"
 	"github.com/ncw/rclone/vfs"
-	"golang.org/x/net/context" // switch to "context" when we stop supporting go1.8
 )
 
 // File represents a file

cmd/mount/fs.go

@@ -5,6 +5,7 @@
 package mount
 
 import (
+	"context"
 	"syscall"
 
 	"bazil.org/fuse"
@@ -15,7 +16,6 @@ import (
 	"github.com/ncw/rclone/vfs"
 	"github.com/ncw/rclone/vfs/vfsflags"
 	"github.com/pkg/errors"
-	"golang.org/x/net/context" // switch to "context" when we stop supporting go1.8
 )
 
 // FS represents the top level filing system
@@ -24,7 +24,7 @@ type FS struct {
 	f fs.Fs
 }
 
-// Check interface satistfied
+// Check interface satisfied
 var _ fusefs.FS = (*FS)(nil)
 
 // NewFS makes a new FS
@@ -46,7 +46,7 @@ func (f *FS) Root() (node fusefs.Node, err error) {
 	return &Dir{root}, nil
 }
 
-// Check interface satsified
+// Check interface satisfied
 var _ fusefs.FSStatfser = (*FS)(nil)
 
 // Statfs is called to obtain file system metadata.

cmd/mount/handle.go

@@ -3,13 +3,13 @@
 package mount
 
 import (
+	"context"
 	"io"
 
 	"bazil.org/fuse"
 	fusefs "bazil.org/fuse/fs"
 	"github.com/ncw/rclone/fs/log"
 	"github.com/ncw/rclone/vfs"
-	"golang.org/x/net/context" // switch to "context" when we stop supporting go1.8
 )
 
 // FileHandle is an open for read file handle on a File

cmd/moveto/moveto.go

@@ -19,7 +19,7 @@ If source:path is a file or directory then it moves it to a file or
 directory named dest:path.
 
 This can be used to rename files or upload single files to other than
-their existing name.  If the source is a directory then it acts exacty
+their existing name.  If the source is a directory then it acts exactly
 like the move command.
 
 So

cmd/ncdu/ncdu.go

@@ -361,7 +361,7 @@ func (u *UI) Draw() error {
 		Linef(0, h-1, w, termbox.ColorBlack, termbox.ColorWhite, ' ', "Total usage: %v, Objects: %d%s", fs.SizeSuffix(size), count, message)
 	}
 
-	// Show the box on top if requred
+	// Show the box on top if required
 	if u.showBox {
 		u.Box()
 	}

cmd/progress.go

@@ -82,7 +82,7 @@ var (
 	progressMu sync.Mutex
 )
 
-// printProgress prings the progress with an optional log
+// printProgress prints the progress with an optional log
 func printProgress(logMessage string) {
 	progressMu.Lock()
 	defer progressMu.Unlock()

cmd/serve/dlna/cd-service-desc.go

@@ -1,451 +0,0 @@
-package dlna
-
-const contentDirectoryServiceDescription = `<?xml version="1.0"?>
-<scpd xmlns="urn:schemas-upnp-org:service-1-0">
-  <specVersion>
-    <major>1</major>
-    <minor>0</minor>
-  </specVersion>
-  <actionList>
-    <action>
-      <name>GetSearchCapabilities</name>
-      <argumentList>
-        <argument>
-          <name>SearchCaps</name>
-          <direction>out</direction>
-          <relatedStateVariable>SearchCapabilities</relatedStateVariable>
-        </argument>
-      </argumentList>
-    </action>
-    <action>
-      <name>GetSortCapabilities</name>
-      <argumentList>
-        <argument>
-          <name>SortCaps</name>
-          <direction>out</direction>
-          <relatedStateVariable>SortCapabilities</relatedStateVariable>
-        </argument>
-      </argumentList>
-    </action>
-    <action>
-      <name>GetSortExtensionCapabilities</name>
-      <argumentList>
-        <argument>
-          <name>SortExtensionCaps</name>
-          <direction>out</direction>
-          <relatedStateVariable>SortExtensionCapabilities</relatedStateVariable>
-        </argument>
-      </argumentList>
-    </action>
-    <action>
-      <name>GetFeatureList</name>
-      <argumentList>
-        <argument>
-          <name>FeatureList</name>
-          <direction>out</direction>
-          <relatedStateVariable>FeatureList</relatedStateVariable>
-        </argument>
-      </argumentList>
-    </action>
-    <action>
-      <name>GetSystemUpdateID</name>
-      <argumentList>
-        <argument>
-          <name>Id</name>
-          <direction>out</direction>
-          <relatedStateVariable>SystemUpdateID</relatedStateVariable>
-        </argument>
-      </argumentList>
-    </action>
-    <action>
-      <name>Browse</name>
-      <argumentList>
-        <argument>
-          <name>ObjectID</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>BrowseFlag</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_BrowseFlag</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>Filter</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_Filter</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>StartingIndex</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_Index</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>RequestedCount</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_Count</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>SortCriteria</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_SortCriteria</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>Result</name>
-          <direction>out</direction>
-          <relatedStateVariable>A_ARG_TYPE_Result</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>NumberReturned</name>
-          <direction>out</direction>
-          <relatedStateVariable>A_ARG_TYPE_Count</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>TotalMatches</name>
-          <direction>out</direction>
-          <relatedStateVariable>A_ARG_TYPE_Count</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>UpdateID</name>
-          <direction>out</direction>
-          <relatedStateVariable>A_ARG_TYPE_UpdateID</relatedStateVariable>
-        </argument>
-      </argumentList>
-    </action>
-    <action>
-      <name>Search</name>
-      <argumentList>
-        <argument>
-          <name>ContainerID</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>SearchCriteria</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_SearchCriteria</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>Filter</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_Filter</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>StartingIndex</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_Index</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>RequestedCount</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_Count</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>SortCriteria</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_SortCriteria</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>Result</name>
-          <direction>out</direction>
-          <relatedStateVariable>A_ARG_TYPE_Result</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>NumberReturned</name>
-          <direction>out</direction>
-          <relatedStateVariable>A_ARG_TYPE_Count</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>TotalMatches</name>
-          <direction>out</direction>
-          <relatedStateVariable>A_ARG_TYPE_Count</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>UpdateID</name>
-          <direction>out</direction>
-          <relatedStateVariable>A_ARG_TYPE_UpdateID</relatedStateVariable>
-        </argument>
-      </argumentList>
-    </action>
-    <action>
-      <name>CreateObject</name>
-      <argumentList>
-        <argument>
-          <name>ContainerID</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>Elements</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_Result</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>ObjectID</name>
-          <direction>out</direction>
-          <relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>Result</name>
-          <direction>out</direction>
-          <relatedStateVariable>A_ARG_TYPE_Result</relatedStateVariable>
-        </argument>
-      </argumentList>
-    </action>
-    <action>
-      <name>DestroyObject</name>
-      <argumentList>
-        <argument>
-          <name>ObjectID</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
-        </argument>
-      </argumentList>
-    </action>
-    <action>
-      <name>UpdateObject</name>
-      <argumentList>
-        <argument>
-          <name>ObjectID</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>CurrentTagValue</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_TagValueList</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>NewTagValue</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_TagValueList</relatedStateVariable>
-        </argument>
-      </argumentList>
-    </action>
-    <action>
-      <name>MoveObject</name>
-      <argumentList>
-        <argument>
-          <name>ObjectID</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>NewParentID</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>NewObjectID</name>
-          <direction>out</direction>
-          <relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
-        </argument>
-      </argumentList>
-    </action>
-    <action>
-      <name>ImportResource</name>
-      <argumentList>
-        <argument>
-          <name>SourceURI</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_URI</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>DestinationURI</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_URI</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>TransferID</name>
-          <direction>out</direction>
-          <relatedStateVariable>A_ARG_TYPE_TransferID</relatedStateVariable>
-        </argument>
-      </argumentList>
-    </action>
-    <action>
-      <name>ExportResource</name>
-      <argumentList>
-        <argument>
-          <name>SourceURI</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_URI</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>DestinationURI</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_URI</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>TransferID</name>
-          <direction>out</direction>
-          <relatedStateVariable>A_ARG_TYPE_TransferID</relatedStateVariable>
-        </argument>
-      </argumentList>
-    </action>
-    <action>
-      <name>StopTransferResource</name>
-      <argumentList>
-        <argument>
-          <name>TransferID</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_TransferID</relatedStateVariable>
-        </argument>
-      </argumentList>
-    </action>
-    <action>
-      <name>DeleteResource</name>
-      <argumentList>
-        <argument>
-          <name>ResourceURI</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_URI</relatedStateVariable>
-        </argument>
-      </argumentList>
-    </action>
-    <action>
-      <name>GetTransferProgress</name>
-      <argumentList>
-        <argument>
-          <name>TransferID</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_TransferID</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>TransferStatus</name>
-          <direction>out</direction>
-          <relatedStateVariable>A_ARG_TYPE_TransferStatus</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>TransferLength</name>
-          <direction>out</direction>
-          <relatedStateVariable>A_ARG_TYPE_TransferLength</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>TransferTotal</name>
-          <direction>out</direction>
-          <relatedStateVariable>A_ARG_TYPE_TransferTotal</relatedStateVariable>
-        </argument>
-      </argumentList>
-    </action>
-    <action>
-      <name>CreateReference</name>
-      <argumentList>
-        <argument>
-          <name>ContainerID</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>ObjectID</name>
-          <direction>in</direction>
-          <relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
-        </argument>
-        <argument>
-          <name>NewID</name>
-          <direction>out</direction>
-          <relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
-        </argument>
-      </argumentList>
-    </action>
-  </actionList>
-  <serviceStateTable>
-    <stateVariable sendEvents="no">
-      <name>SearchCapabilities</name>
-      <dataType>string</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="no">
-      <name>SortCapabilities</name>
-      <dataType>string</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="no">
-      <name>SortExtensionCapabilities</name>
-      <dataType>string</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="yes">
-      <name>SystemUpdateID</name>
-      <dataType>ui4</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="yes">
-      <name>ContainerUpdateIDs</name>
-      <dataType>string</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="yes">
-      <name>TransferIDs</name>
-      <dataType>string</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="no">
-      <name>FeatureList</name>
-      <dataType>string</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="no">
-      <name>A_ARG_TYPE_ObjectID</name>
-      <dataType>string</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="no">
-      <name>A_ARG_TYPE_Result</name>
-      <dataType>string</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="no">
-      <name>A_ARG_TYPE_SearchCriteria</name>
-      <dataType>string</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="no">
-      <name>A_ARG_TYPE_BrowseFlag</name>
-      <dataType>string</dataType>
-      <allowedValueList>
-        <allowedValue>BrowseMetadata</allowedValue>
-        <allowedValue>BrowseDirectChildren</allowedValue>
-      </allowedValueList>
-    </stateVariable>
-    <stateVariable sendEvents="no">
-      <name>A_ARG_TYPE_Filter</name>
-      <dataType>string</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="no">
-      <name>A_ARG_TYPE_SortCriteria</name>
-      <dataType>string</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="no">
-      <name>A_ARG_TYPE_Index</name>
-      <dataType>ui4</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="no">
-      <name>A_ARG_TYPE_Count</name>
-      <dataType>ui4</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="no">
-      <name>A_ARG_TYPE_UpdateID</name>
-      <dataType>ui4</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="no">
-      <name>A_ARG_TYPE_TransferID</name>
-      <dataType>ui4</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="no">
-      <name>A_ARG_TYPE_TransferStatus</name>
-      <dataType>string</dataType>
-      <allowedValueList>
-        <allowedValue>COMPLETED</allowedValue>
-        <allowedValue>ERROR</allowedValue>
-        <allowedValue>IN_PROGRESS</allowedValue>
-        <allowedValue>STOPPED</allowedValue>
-      </allowedValueList>
-    </stateVariable>
-    <stateVariable sendEvents="no">
-      <name>A_ARG_TYPE_TransferLength</name>
-      <dataType>string</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="no">
-      <name>A_ARG_TYPE_TransferTotal</name>
-      <dataType>string</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="no">
-      <name>A_ARG_TYPE_TagValueList</name>
-      <dataType>string</dataType>
-    </stateVariable>
-    <stateVariable sendEvents="no">
-      <name>A_ARG_TYPE_URI</name>
-      <dataType>uri</dataType>
-    </stateVariable>
-  </serviceStateTable>
-</scpd>`

cmd/serve/dlna/cds.go

@@ -9,11 +9,13 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"regexp"
 	"sort"
 
 	"github.com/anacrolix/dms/dlna"
 	"github.com/anacrolix/dms/upnp"
 	"github.com/anacrolix/dms/upnpav"
+	"github.com/ncw/rclone/fs"
 	"github.com/ncw/rclone/vfs"
 	"github.com/pkg/errors"
 )
@@ -27,6 +29,8 @@ func (cds *contentDirectoryService) updateIDString() string {
 	return fmt.Sprintf("%d", uint32(os.Getpid()))
 }
 
+var mediaMimeTypeRegexp = regexp.MustCompile("^(video|audio|image)/")
+
 // Turns the given entry and DMS host into a UPnP object. A nil object is
 // returned if the entry is not of interest.
 func (cds *contentDirectoryService) cdsObjectToUpnpavObject(cdsObject object, fileInfo os.FileInfo, host string) (ret interface{}, err error) {
@@ -47,8 +51,13 @@ func (cds *contentDirectoryService) cdsObjectToUpnpavObject(cdsObject object, fi
 		return
 	}
 
-	// Hardcode "videoItem" so that files show up in VLC.
-	obj.Class = "object.item.videoItem"
+	mimeType := fs.MimeTypeFromName(fileInfo.Name())
+	mediaType := mediaMimeTypeRegexp.FindStringSubmatch(mimeType)
+	if mediaType == nil {
+		return
+	}
+
+	obj.Class = "object.item." + mediaType[1] + "Item"
 	obj.Title = fileInfo.Name()
 
 	item := upnpav.Item{
@@ -65,8 +74,7 @@ func (cds *contentDirectoryService) cdsObjectToUpnpavObject(cdsObject object, fi
 				"path": {cdsObject.Path},
 			}.Encode(),
 		}).String(),
-		// Hardcode "video/x-matroska" so that files show up in VLC.
-		ProtocolInfo: fmt.Sprintf("http-get:*:video/x-matroska:%s", dlna.ContentFeatures{
+		ProtocolInfo: fmt.Sprintf("http-get:*:%s:%s", mimeType, dlna.ContentFeatures{
 			SupportRange: true,
 		}.String()),
 		Bitrate:    0,
@@ -106,14 +114,14 @@ func (cds *contentDirectoryService) readContainer(o object, host string) (ret []
 		}
 		obj, err := cds.cdsObjectToUpnpavObject(child, de, host)
 		if err != nil {
-			log.Printf("error with %s: %s", child.FilePath(), err)
+			fs.Errorf(cds, "error with %s: %s", child.FilePath(), err)
 			continue
 		}
-		if obj != nil {
-			ret = append(ret, obj)
-		} else {
-			log.Printf("bad %s", de)
+		if obj == nil {
+			fs.Debugf(cds, "unrecognized file type: %s", de)
+			continue
 		}
+		ret = append(ret, obj)
 	}
 
 	return
@@ -192,6 +200,14 @@ func (cds *contentDirectoryService) Handle(action string, argsXML []byte, r *htt
 				"Result":         didlLite(string(result)),
 				"UpdateID":       cds.updateIDString(),
 			}, nil
+		case "BrowseMetadata":
+			result, err := xml.Marshal(obj)
+			if err != nil {
+				return nil, err
+			}
+			return map[string]string{
+				"Result": didlLite(string(result)),
+			}, nil
 		default:
 			return nil, upnp.Errorf(upnp.ArgumentValueInvalidErrorCode, "unhandled browse flag: %v", browse.BrowseFlag)
 		}
@@ -199,6 +215,19 @@ func (cds *contentDirectoryService) Handle(action string, argsXML []byte, r *htt
 		return map[string]string{
 			"SearchCaps": "",
 		}, nil
+	// Samsung Extensions
+	case "X_GetFeatureList":
+		return map[string]string{
+			"FeatureList": `<Features xmlns="urn:schemas-upnp-org:av:avs" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="urn:schemas-upnp-org:av:avs http://www.upnp.org/schemas/av/avs.xsd">
+	<Feature name="samsung.com_BASICVIEW" version="1">
+		<container id="/" type="object.item.imageItem"/>
+		<container id="/" type="object.item.audioItem"/>
+		<container id="/" type="object.item.videoItem"/>
+	</Feature>
+</Features>`}, nil
+	case "X_SetBookmark":
+		// just ignore
+		return map[string]string{}, nil
 	default:
 		return nil, upnp.InvalidActionError
 	}

cmd/serve/dlna/cms.go

@@ -0,0 +1,26 @@
+package dlna
+
+import (
+	"net/http"
+
+	"github.com/anacrolix/dms/upnp"
+)
+
+const defaultProtocolInfo = "http-get:*:video/mpeg:*,http-get:*:video/mp4:*,http-get:*:video/vnd.dlna.mpeg-tts:*,http-get:*:video/avi:*,http-get:*:video/x-matroska:*,http-get:*:video/x-ms-wmv:*,http-get:*:video/wtv:*,http-get:*:audio/mpeg:*,http-get:*:audio/mp3:*,http-get:*:audio/mp4:*,http-get:*:audio/x-ms-wma*,http-get:*:audio/wav:*,http-get:*:audio/L16:*,http-get:*image/jpeg:*,http-get:*image/png:*,http-get:*image/gif:*,http-get:*image/tiff:*"
+
+type connectionManagerService struct {
+	*server
+	upnp.Eventing
+}
+
+func (cms *connectionManagerService) Handle(action string, argsXML []byte, r *http.Request) (map[string]string, error) {
+	switch action {
+	case "GetProtocolInfo":
+		return map[string]string{
+			"Source": defaultProtocolInfo,
+			"Sink":   "",
+		}, nil
+	default:
+		return nil, upnp.InvalidActionError
+	}
+}

cmd/serve/dlna/data/assets_generate.go

@@ -0,0 +1,24 @@
+//go:generate go run assets_generate.go
+// The "go:generate" directive compiles static assets by running assets_generate.go
+// +build ignore
+
+package main
+
+import (
+	"log"
+	"net/http"
+
+	"github.com/shurcooL/vfsgen"
+)
+
+func main() {
+	var AssetDir http.FileSystem = http.Dir("./static")
+	err := vfsgen.Generate(AssetDir, vfsgen.Options{
+		PackageName:  "data",
+		BuildTags:    "!dev",
+		VariableName: "Assets",
+	})
+	if err != nil {
+		log.Fatalln(err)
+	}
+}

cmd/serve/dlna/data/data.go

@@ -0,0 +1,4 @@
+//go:generate go run assets_generate.go
+// The "go:generate" directive compiles static assets by running assets_generate.go
+
+package data

cmd/serve/dlna/data/static/ConnectionManager.xml

@@ -0,0 +1,182 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<scpd xmlns="urn:schemas-upnp-org:service-1-0">
+	<specVersion>
+		<major>1</major>
+		<minor>0</minor>
+	</specVersion>
+	<actionList>
+		<action>
+			<name>GetProtocolInfo</name>
+			<argumentList>
+				<argument>
+					<name>Source</name>
+					<direction>out</direction>
+					<relatedStateVariable>SourceProtocolInfo</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>Sink</name>
+					<direction>out</direction>
+					<relatedStateVariable>SinkProtocolInfo</relatedStateVariable>
+				</argument>
+			</argumentList>
+		</action>
+		<action>
+			<name>PrepareForConnection</name>
+			<argumentList>
+				<argument>
+					<name>RemoteProtocolInfo</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_ProtocolInfo</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>PeerConnectionManager</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_ConnectionManager</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>PeerConnectionID</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_ConnectionID</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>Direction</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_Direction</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>ConnectionID</name>
+					<direction>out</direction>
+					<relatedStateVariable>A_ARG_TYPE_ConnectionID</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>AVTransportID</name>
+					<direction>out</direction>
+					<relatedStateVariable>A_ARG_TYPE_AVTransportID</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>RcsID</name>
+					<direction>out</direction>
+					<relatedStateVariable>A_ARG_TYPE_RcsID</relatedStateVariable>
+				</argument>
+			</argumentList>
+		</action>
+		<action>
+			<name>ConnectionComplete</name>
+			<argumentList>
+				<argument>
+					<name>ConnectionID</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_ConnectionID</relatedStateVariable>
+				</argument>
+			</argumentList>
+		</action>
+		<action>
+			<name>GetCurrentConnectionIDs</name>
+			<argumentList>
+				<argument>
+					<name>ConnectionIDs</name>
+					<direction>out</direction>
+					<relatedStateVariable>CurrentConnectionIDs</relatedStateVariable>
+				</argument>
+			</argumentList>
+		</action>
+		<action>
+			<name>GetCurrentConnectionInfo</name>
+			<argumentList>
+				<argument>
+					<name>ConnectionID</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_ConnectionID</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>RcsID</name>
+					<direction>out</direction>
+					<relatedStateVariable>A_ARG_TYPE_RcsID</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>AVTransportID</name>
+					<direction>out</direction>
+					<relatedStateVariable>A_ARG_TYPE_AVTransportID</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>ProtocolInfo</name>
+					<direction>out</direction>
+					<relatedStateVariable>A_ARG_TYPE_ProtocolInfo</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>PeerConnectionManager</name>
+					<direction>out</direction>
+					<relatedStateVariable>A_ARG_TYPE_ConnectionManager</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>PeerConnectionID</name>
+					<direction>out</direction>
+					<relatedStateVariable>A_ARG_TYPE_ConnectionID</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>Direction</name>
+					<direction>out</direction>
+					<relatedStateVariable>A_ARG_TYPE_Direction</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>Status</name>
+					<direction>out</direction>
+					<relatedStateVariable>A_ARG_TYPE_ConnectionStatus</relatedStateVariable>
+				</argument>
+			</argumentList>
+		</action>
+	</actionList>
+	<serviceStateTable>
+		<stateVariable sendEvents="yes">
+			<name>SourceProtocolInfo</name>
+			<dataType>string</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="yes">
+			<name>SinkProtocolInfo</name>
+			<dataType>string</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="yes">
+			<name>CurrentConnectionIDs</name>
+			<dataType>string</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>A_ARG_TYPE_ConnectionStatus</name>
+			<dataType>string</dataType>
+			<allowedValueList>
+				<allowedValue>OK</allowedValue>
+				<allowedValue>ContentFormatMismatch</allowedValue>
+				<allowedValue>InsufficientBandwidth</allowedValue>
+				<allowedValue>UnreliableChannel</allowedValue>
+				<allowedValue>Unknown</allowedValue>
+			</allowedValueList>
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>A_ARG_TYPE_ConnectionManager</name>
+			<dataType>string</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>A_ARG_TYPE_Direction</name>
+			<dataType>string</dataType>
+			<allowedValueList>
+				<allowedValue>Input</allowedValue>
+				<allowedValue>Output</allowedValue>
+			</allowedValueList>
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>A_ARG_TYPE_ProtocolInfo</name>
+			<dataType>string</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>A_ARG_TYPE_ConnectionID</name>
+			<dataType>i4</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>A_ARG_TYPE_AVTransportID</name>
+			<dataType>i4</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>A_ARG_TYPE_RcsID</name>
+			<dataType>i4</dataType>
+		</stateVariable>
+	</serviceStateTable>
+</scpd>

cmd/serve/dlna/data/static/ContentDirectory.xml

@@ -0,0 +1,504 @@
+<?xml version="1.0"?>
+<scpd xmlns="urn:schemas-upnp-org:service-1-0">
+	<specVersion>
+		<major>1</major>
+		<minor>0</minor>
+	</specVersion>
+	<actionList>
+		<action>
+			<name>GetSearchCapabilities</name>
+			<argumentList>
+				<argument>
+					<name>SearchCaps</name>
+					<direction>out</direction>
+					<relatedStateVariable>SearchCapabilities</relatedStateVariable>
+				</argument>
+			</argumentList>
+		</action>
+		<action>
+			<name>GetSortCapabilities</name>
+			<argumentList>
+				<argument>
+					<name>SortCaps</name>
+					<direction>out</direction>
+					<relatedStateVariable>SortCapabilities</relatedStateVariable>
+				</argument>
+			</argumentList>
+		</action>
+		<action>
+			<name>GetSortExtensionCapabilities</name>
+			<argumentList>
+				<argument>
+					<name>SortExtensionCaps</name>
+					<direction>out</direction>
+					<relatedStateVariable>SortExtensionCapabilities</relatedStateVariable>
+				</argument>
+			</argumentList>
+		</action>
+		<action>
+			<name>GetFeatureList</name>
+			<argumentList>
+				<argument>
+					<name>FeatureList</name>
+					<direction>out</direction>
+					<relatedStateVariable>FeatureList</relatedStateVariable>
+				</argument>
+			</argumentList>
+		</action>
+		<action>
+			<name>GetSystemUpdateID</name>
+			<argumentList>
+				<argument>
+					<name>Id</name>
+					<direction>out</direction>
+					<relatedStateVariable>SystemUpdateID</relatedStateVariable>
+				</argument>
+			</argumentList>
+		</action>
+		<action>
+			<name>Browse</name>
+			<argumentList>
+				<argument>
+					<name>ObjectID</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>BrowseFlag</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_BrowseFlag</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>Filter</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_Filter</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>StartingIndex</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_Index</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>RequestedCount</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_Count</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>SortCriteria</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_SortCriteria</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>Result</name>
+					<direction>out</direction>
+					<relatedStateVariable>A_ARG_TYPE_Result</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>NumberReturned</name>
+					<direction>out</direction>
+					<relatedStateVariable>A_ARG_TYPE_Count</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>TotalMatches</name>
+					<direction>out</direction>
+					<relatedStateVariable>A_ARG_TYPE_Count</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>UpdateID</name>
+					<direction>out</direction>
+					<relatedStateVariable>A_ARG_TYPE_UpdateID</relatedStateVariable>
+				</argument>
+			</argumentList>
+		</action>
+		<action>
+			<name>Search</name>
+			<argumentList>
+				<argument>
+					<name>ContainerID</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>SearchCriteria</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_SearchCriteria</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>Filter</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_Filter</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>StartingIndex</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_Index</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>RequestedCount</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_Count</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>SortCriteria</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_SortCriteria</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>Result</name>
+					<direction>out</direction>
+					<relatedStateVariable>A_ARG_TYPE_Result</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>NumberReturned</name>
+					<direction>out</direction>
+					<relatedStateVariable>A_ARG_TYPE_Count</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>TotalMatches</name>
+					<direction>out</direction>
+					<relatedStateVariable>A_ARG_TYPE_Count</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>UpdateID</name>
+					<direction>out</direction>
+					<relatedStateVariable>A_ARG_TYPE_UpdateID</relatedStateVariable>
+				</argument>
+			</argumentList>
+		</action>
+		<action>
+			<name>CreateObject</name>
+			<argumentList>
+				<argument>
+					<name>ContainerID</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>Elements</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_Result</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>ObjectID</name>
+					<direction>out</direction>
+					<relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>Result</name>
+					<direction>out</direction>
+					<relatedStateVariable>A_ARG_TYPE_Result</relatedStateVariable>
+				</argument>
+			</argumentList>
+		</action>
+		<action>
+			<name>DestroyObject</name>
+			<argumentList>
+				<argument>
+					<name>ObjectID</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
+				</argument>
+			</argumentList>
+		</action>
+		<action>
+			<name>UpdateObject</name>
+			<argumentList>
+				<argument>
+					<name>ObjectID</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>CurrentTagValue</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_TagValueList</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>NewTagValue</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_TagValueList</relatedStateVariable>
+				</argument>
+			</argumentList>
+		</action>
+		<action>
+			<name>MoveObject</name>
+			<argumentList>
+				<argument>
+					<name>ObjectID</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>NewParentID</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>NewObjectID</name>
+					<direction>out</direction>
+					<relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
+				</argument>
+			</argumentList>
+		</action>
+		<action>
+			<name>ImportResource</name>
+			<argumentList>
+				<argument>
+					<name>SourceURI</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_URI</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>DestinationURI</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_URI</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>TransferID</name>
+					<direction>out</direction>
+					<relatedStateVariable>A_ARG_TYPE_TransferID</relatedStateVariable>
+				</argument>
+			</argumentList>
+		</action>
+		<action>
+			<name>ExportResource</name>
+			<argumentList>
+				<argument>
+					<name>SourceURI</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_URI</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>DestinationURI</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_URI</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>TransferID</name>
+					<direction>out</direction>
+					<relatedStateVariable>A_ARG_TYPE_TransferID</relatedStateVariable>
+				</argument>
+			</argumentList>
+		</action>
+		<action>
+			<name>StopTransferResource</name>
+			<argumentList>
+				<argument>
+					<name>TransferID</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_TransferID</relatedStateVariable>
+				</argument>
+			</argumentList>
+		</action>
+		<action>
+			<name>DeleteResource</name>
+			<argumentList>
+				<argument>
+					<name>ResourceURI</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_URI</relatedStateVariable>
+				</argument>
+			</argumentList>
+		</action>
+		<action>
+			<name>GetTransferProgress</name>
+			<argumentList>
+				<argument>
+					<name>TransferID</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_TransferID</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>TransferStatus</name>
+					<direction>out</direction>
+					<relatedStateVariable>A_ARG_TYPE_TransferStatus</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>TransferLength</name>
+					<direction>out</direction>
+					<relatedStateVariable>A_ARG_TYPE_TransferLength</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>TransferTotal</name>
+					<direction>out</direction>
+					<relatedStateVariable>A_ARG_TYPE_TransferTotal</relatedStateVariable>
+				</argument>
+			</argumentList>
+		</action>
+		<action>
+			<name>CreateReference</name>
+			<argumentList>
+				<argument>
+					<name>ContainerID</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>ObjectID</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>NewID</name>
+					<direction>out</direction>
+					<relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
+				</argument>
+			</argumentList>
+		</action>
+		<action>
+			<name>X_GetFeatureList</name>
+			<argumentList>
+				<argument>
+					<name>FeatureList</name>
+					<direction>out</direction>
+					<relatedStateVariable>A_ARG_TYPE_Featurelist</relatedStateVariable>
+				</argument>
+			</argumentList>
+		</action>
+		<action>
+			<name>X_SetBookmark</name>
+			<argumentList>
+				<argument>
+					<name>CategoryType</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_CategoryType</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>RID</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_RID</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>ObjectID</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_ObjectID</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>PosSecond</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_PosSec</relatedStateVariable>
+				</argument>
+			</argumentList>
+		</action>
+	</actionList>
+	<serviceStateTable>
+		<stateVariable sendEvents="no">
+			<name>SearchCapabilities</name>
+			<dataType>string</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>SortCapabilities</name>
+			<dataType>string</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>SortExtensionCapabilities</name>
+			<dataType>string</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="yes">
+			<name>SystemUpdateID</name>
+			<dataType>ui4</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="yes">
+			<name>ContainerUpdateIDs</name>
+			<dataType>string</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="yes">
+			<name>TransferIDs</name>
+			<dataType>string</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>FeatureList</name>
+			<dataType>string</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>A_ARG_TYPE_ObjectID</name>
+			<dataType>string</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>A_ARG_TYPE_Result</name>
+			<dataType>string</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>A_ARG_TYPE_SearchCriteria</name>
+			<dataType>string</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>A_ARG_TYPE_BrowseFlag</name>
+			<dataType>string</dataType>
+			<allowedValueList>
+				<allowedValue>BrowseMetadata</allowedValue>
+				<allowedValue>BrowseDirectChildren</allowedValue>
+			</allowedValueList>
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>A_ARG_TYPE_Filter</name>
+			<dataType>string</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>A_ARG_TYPE_SortCriteria</name>
+			<dataType>string</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>A_ARG_TYPE_Index</name>
+			<dataType>ui4</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>A_ARG_TYPE_Count</name>
+			<dataType>ui4</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>A_ARG_TYPE_UpdateID</name>
+			<dataType>ui4</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>A_ARG_TYPE_TransferID</name>
+			<dataType>ui4</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>A_ARG_TYPE_TransferStatus</name>
+			<dataType>string</dataType>
+			<allowedValueList>
+				<allowedValue>COMPLETED</allowedValue>
+				<allowedValue>ERROR</allowedValue>
+				<allowedValue>IN_PROGRESS</allowedValue>
+				<allowedValue>STOPPED</allowedValue>
+			</allowedValueList>
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>A_ARG_TYPE_TransferLength</name>
+			<dataType>string</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>A_ARG_TYPE_TransferTotal</name>
+			<dataType>string</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>A_ARG_TYPE_TagValueList</name>
+			<dataType>string</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>A_ARG_TYPE_URI</name>
+			<dataType>uri</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>A_ARG_TYPE_CategoryType</name>
+			<dataType>ui4</dataType>
+			<defaultValue />
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>A_ARG_TYPE_RID</name>
+			<dataType>ui4</dataType>
+			<defaultValue />
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>A_ARG_TYPE_PosSec</name>
+			<dataType>ui4</dataType>
+			<defaultValue />
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>A_ARG_TYPE_Featurelist</name>
+			<dataType>string</dataType>
+			<defaultValue />
+		</stateVariable>
+	</serviceStateTable>
+</scpd>

cmd/serve/dlna/data/static/X_MS_MediaReceiverRegistrar.xml

@@ -0,0 +1,88 @@
+<?xml version="1.0" ?>
+<scpd xmlns="urn:schemas-upnp-org:service-1-0">
+	<specVersion>
+		<major>1</major>
+		<minor>0</minor>
+	</specVersion>
+	<actionList>
+		<action>
+			<name>IsAuthorized</name>
+			<argumentList>
+				<argument>
+					<name>DeviceID</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_DeviceID</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>Result</name>
+					<direction>out</direction>
+					<relatedStateVariable>A_ARG_TYPE_Result</relatedStateVariable>
+				</argument>
+			</argumentList>
+		</action>
+		<action>
+			<name>RegisterDevice</name>
+			<argumentList>
+				<argument>
+					<name>RegistrationReqMsg</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_RegistrationReqMsg</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>RegistrationRespMsg</name>
+					<direction>out</direction>
+					<relatedStateVariable>A_ARG_TYPE_RegistrationRespMsg</relatedStateVariable>
+				</argument>
+			</argumentList>
+		</action>
+		<action>
+			<name>IsValidated</name>
+			<argumentList>
+				<argument>
+					<name>DeviceID</name>
+					<direction>in</direction>
+					<relatedStateVariable>A_ARG_TYPE_DeviceID</relatedStateVariable>
+				</argument>
+				<argument>
+					<name>Result</name>
+					<direction>out</direction>
+					<relatedStateVariable>A_ARG_TYPE_Result</relatedStateVariable>
+				</argument>
+			</argumentList>
+		</action>
+	</actionList>
+	<serviceStateTable>
+		<stateVariable sendEvents="no">
+			<name>A_ARG_TYPE_DeviceID</name>
+			<dataType>string</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>A_ARG_TYPE_Result</name>
+			<dataType>int</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>A_ARG_TYPE_RegistrationReqMsg</name>
+			<dataType>bin.base64</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="no">
+			<name>A_ARG_TYPE_RegistrationRespMsg</name>
+			<dataType>bin.base64</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="yes">
+			<name>AuthorizationGrantedUpdateID</name>
+			<dataType>ui4</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="yes">
+			<name>AuthorizationDeniedUpdateID</name>
+			<dataType>ui4</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="yes">
+			<name>ValidationSucceededUpdateID</name>
+			<dataType>ui4</dataType>
+		</stateVariable>
+		<stateVariable sendEvents="yes">
+			<name>ValidationRevokedUpdateID</name>
+			<dataType>ui4</dataType>
+		</stateVariable>
+	</serviceStateTable>
+</scpd>

cmd/serve/dlna/dlna.go

@@ -4,20 +4,21 @@ import (
 	"bytes"
 	"encoding/xml"
 	"fmt"
-	"log"
 	"net"
 	"net/http"
 	"net/url"
 	"os"
-	"path"
 	"strconv"
 	"strings"
+	"text/template"
 	"time"
 
+	dms_dlna "github.com/anacrolix/dms/dlna"
 	"github.com/anacrolix/dms/soap"
 	"github.com/anacrolix/dms/ssdp"
 	"github.com/anacrolix/dms/upnp"
 	"github.com/ncw/rclone/cmd"
+	"github.com/ncw/rclone/cmd/serve/dlna/data"
 	"github.com/ncw/rclone/cmd/serve/dlna/dlnaflags"
 	"github.com/ncw/rclone/fs"
 	"github.com/ncw/rclone/vfs"
@@ -51,7 +52,7 @@ players might show files that they are not able to play back correctly.
 		cmd.Run(false, false, command, func() error {
 			s := newServer(f, &dlnaflags.Opt)
 			if err := s.Serve(); err != nil {
-				log.Fatal(err)
+				return err
 			}
 			s.Wait()
 			return nil
@@ -60,45 +61,12 @@ players might show files that they are not able to play back correctly.
 }
 
 const (
-	serverField         = "Linux/3.4 DLNADOC/1.50 UPnP/1.0 DMS/1.0"
-	rootDeviceType      = "urn:schemas-upnp-org:device:MediaServer:1"
-	rootDeviceModelName = "rclone"
-	resPath             = "/res"
-	rootDescPath        = "/rootDesc.xml"
-	serviceControlURL   = "/ctl"
+	serverField       = "Linux/3.4 DLNADOC/1.50 UPnP/1.0 DMS/1.0"
+	rootDescPath      = "/rootDesc.xml"
+	resPath           = "/res"
+	serviceControlURL = "/ctl"
 )
 
-// Groups the service definition with its XML description.
-type service struct {
-	upnp.Service
-	SCPD string
-}
-
-// Exposed UPnP AV services.
-var services = []*service{
-	{
-		Service: upnp.Service{
-			ServiceType: "urn:schemas-upnp-org:service:ContentDirectory:1",
-			ServiceId:   "urn:upnp-org:serviceId:ContentDirectory",
-			ControlURL:  serviceControlURL,
-		},
-		SCPD: contentDirectoryServiceDescription,
-	},
-}
-
-func devices() []string {
-	return []string{
-		"urn:schemas-upnp-org:device:MediaServer:1",
-	}
-}
-
-func serviceTypes() (ret []string) {
-	for _, s := range services {
-		ret = append(ret, s.ServiceType)
-	}
-	return
-}
-
 type server struct {
 	// The service SOAP handler keyed by service URN.
 	services map[string]UPnPService
@@ -107,10 +75,9 @@ type server struct {
 
 	HTTPConn       net.Listener
 	httpListenAddr string
-	httpServeMux   *http.ServeMux
+	handler        http.Handler
 
-	rootDeviceUUID string
-	rootDescXML    []byte
+	RootDeviceUUID string
 
 	FriendlyName string
 
@@ -125,16 +92,16 @@ type server struct {
 }
 
 func newServer(f fs.Fs, opt *dlnaflags.Options) *server {
-	hostName, err := os.Hostname()
-	if err != nil {
-		hostName = ""
-	} else {
-		hostName = " (" + hostName + ")"
+	friendlyName := opt.FriendlyName
+	if friendlyName == "" {
+		friendlyName = makeDefaultFriendlyName()
 	}
 
 	s := &server{
 		AnnounceInterval: 10 * time.Second,
-		FriendlyName:     "rclone" + hostName,
+		FriendlyName:     friendlyName,
+		RootDeviceUUID:   makeDeviceUUID(friendlyName),
+		Interfaces:       listInterfaces(),
 
 		httpListenAddr: opt.ListenAddr,
 
@@ -142,35 +109,29 @@ func newServer(f fs.Fs, opt *dlnaflags.Options) *server {
 		vfs: vfs.New(f, &vfsflags.Opt),
 	}
 
-	s.initServicesMap()
-	s.listInterfaces()
-
-	s.httpServeMux = http.NewServeMux()
-	s.rootDeviceUUID = makeDeviceUUID(s.FriendlyName)
-	s.rootDescXML, err = xml.MarshalIndent(
-		upnp.DeviceDesc{
-			SpecVersion: upnp.SpecVersion{Major: 1, Minor: 0},
-			Device: upnp.Device{
-				DeviceType:   rootDeviceType,
-				FriendlyName: s.FriendlyName,
-				Manufacturer: "rclone (rclone.org)",
-				ModelName:    rootDeviceModelName,
-				UDN:          s.rootDeviceUUID,
-				ServiceList: func() (ss []upnp.Service) {
-					for _, s := range services {
-						ss = append(ss, s.Service)
-					}
-					return
-				}(),
-			},
+	s.services = map[string]UPnPService{
+		"ContentDirectory": &contentDirectoryService{
+			server: s,
+		},
+		"ConnectionManager": &connectionManagerService{
+			server: s,
 		},
-		" ", "  ")
-	if err != nil {
-		// Contents are hardcoded, so this will never happen in production.
-		log.Panicf("Marshal root descriptor XML: %v", err)
 	}
-	s.rootDescXML = append([]byte(`<?xml version="1.0"?>`), s.rootDescXML...)
-	s.initMux(s.httpServeMux)
+
+	// Setup the various http routes.
+	r := http.NewServeMux()
+	r.HandleFunc(resPath, s.resourceHandler)
+	if opt.LogTrace {
+		r.Handle(rootDescPath, traceLogging(http.HandlerFunc(s.rootDescHandler)))
+		r.Handle(serviceControlURL, traceLogging(http.HandlerFunc(s.serviceControlHandler)))
+	} else {
+		r.HandleFunc(rootDescPath, s.rootDescHandler)
+		r.HandleFunc(serviceControlURL, s.serviceControlHandler)
+	}
+	r.Handle("/static/", http.StripPrefix("/static/",
+		withHeader("Cache-Control", "public, max-age=86400",
+			http.FileServer(data.Assets))))
+	s.handler = logging(withHeader("Server", serverField, r))
 
 	return s
 }
@@ -182,117 +143,137 @@ type UPnPService interface {
 	Unsubscribe(sid string) error
 }
 
-// initServicesMap is called during initialization of the server to prepare some internal datastructures.
-func (s *server) initServicesMap() {
-	urn, err := upnp.ParseServiceType(services[0].ServiceType)
-	if err != nil {
-		// The service type is hardcoded, so this error should never happen.
-		log.Panicf("ParseServiceType: %v", err)
-	}
-	s.services = map[string]UPnPService{
-		urn.Type: &contentDirectoryService{
-			server: s,
-		},
-	}
-	return
+// Formats the server as a string (used for logging.)
+func (s *server) String() string {
+	return fmt.Sprintf("DLNA server on %v", s.httpListenAddr)
 }
 
-// listInterfaces is called during initialization of the server to list the network interfaces
-// on the machine.
-func (s *server) listInterfaces() {
-	ifs, err := net.Interfaces()
+// Returns rclone version number as the model number.
+func (s *server) ModelNumber() string {
+	return fs.Version
+}
+
+// Template used to generate the root device XML descriptor.
+//
+// Due to the use of namespaces and various subtleties with device compatibility,
+// it turns out to be easier to use a template than to marshal XML.
+//
+// For rendering, it is passed the server object for context.
+var rootDescTmpl = template.Must(template.New("rootDesc").Parse(`<?xml version="1.0"?>
+<root xmlns="urn:schemas-upnp-org:device-1-0"
+      xmlns:dlna="urn:schemas-dlna-org:device-1-0"
+      xmlns:sec="http://www.sec.co.kr/dlna">
+  <specVersion>
+    <major>1</major>
+    <minor>0</minor>
+  </specVersion>
+  <device>
+    <deviceType>urn:schemas-upnp-org:device:MediaServer:1</deviceType>
+    <friendlyName>{{.FriendlyName}}</friendlyName>
+    <manufacturer>rclone (rclone.org)</manufacturer>
+    <manufacturerURL>https://rclone.org/</manufacturerURL>
+    <modelDescription>rclone</modelDescription>
+    <modelName>rclone</modelName>
+    <modelNumber>{{.ModelNumber}}</modelNumber>
+    <modelURL>https://rclone.org/</modelURL>
+    <serialNumber>00000000</serialNumber>
+    <UDN>{{.RootDeviceUUID}}</UDN>
+    <dlna:X_DLNACAP/>
+    <dlna:X_DLNADOC>DMS-1.50</dlna:X_DLNADOC>
+    <dlna:X_DLNADOC>M-DMS-1.50</dlna:X_DLNADOC>
+    <sec:ProductCap>smi,DCM10,getMediaInfo.sec,getCaptionInfo.sec</sec:ProductCap>
+    <sec:X_ProductCap>smi,DCM10,getMediaInfo.sec,getCaptionInfo.sec</sec:X_ProductCap>
+    <iconList>
+      <icon>
+        <mimetype>image/png</mimetype>
+        <width>48</width>
+        <height>48</height>
+        <depth>8</depth>
+        <url>/static/rclone-48x48.png</url>
+      </icon>
+      <icon>
+        <mimetype>image/png</mimetype>
+        <width>120</width>
+        <height>120</height>
+        <depth>8</depth>
+        <url>/static/rclone-120x120.png</url>
+      </icon>
+    </iconList>
+    <serviceList>
+      <service>
+        <serviceType>urn:schemas-upnp-org:service:ContentDirectory:1</serviceType>
+        <serviceId>urn:upnp-org:serviceId:ContentDirectory</serviceId>
+        <SCPDURL>/static/ContentDirectory.xml</SCPDURL>
+        <controlURL>/ctl</controlURL>
+        <eventSubURL></eventSubURL>
+      </service>
+      <service>
+        <serviceType>urn:schemas-upnp-org:service:ConnectionManager:1</serviceType>
+        <serviceId>urn:upnp-org:serviceId:ConnectionManager</serviceId>
+        <SCPDURL>/static/ConnectionManager.xml</SCPDURL>
+        <controlURL>/ctl</controlURL>
+        <eventSubURL></eventSubURL>
+      </service>
+      <service>
+        <serviceType>urn:microsoft.com:service:X_MS_MediaReceiverRegistrar:1</serviceType>
+        <serviceId>urn:microsoft.com:serviceId:X_MS_MediaReceiverRegistrar</serviceId>
+        <SCPDURL>/static/X_MS_MediaReceiverRegistrar.xml</SCPDURL>
+        <controlURL>/ctl</controlURL>
+        <eventSubURL></eventSubURL>
+      </service>
+    </serviceList>
+    <presentationURL>/</presentationURL>
+  </device>
+</root>`))
+
+// Renders the root device descriptor.
+func (s *server) rootDescHandler(w http.ResponseWriter, r *http.Request) {
+	buffer := new(bytes.Buffer)
+	err := rootDescTmpl.Execute(buffer, s)
 	if err != nil {
-		fs.Errorf(s.f, "list network interfaces: %v", err)
+		serveError(s, w, "Failed to create root descriptor XML", err)
 		return
 	}
 
-	var tmp []net.Interface
-	for _, intf := range ifs {
-		if intf.Flags&net.FlagUp == 0 || intf.MTU <= 0 {
-			continue
-		}
-		s.Interfaces = append(s.Interfaces, intf)
-		tmp = append(tmp, intf)
+	w.Header().Set("content-type", `text/xml; charset="utf-8"`)
+	w.Header().Set("cache-control", "private, max-age=60")
+	w.Header().Set("content-length", strconv.FormatInt(int64(buffer.Len()), 10))
+	_, err = buffer.WriteTo(w)
+	if err != nil {
+		// Network error
+		fs.Debugf(s, "Error writing rootDesc: %v", err)
 	}
 }
 
-func (s *server) initMux(mux *http.ServeMux) {
-	mux.HandleFunc(resPath, func(w http.ResponseWriter, r *http.Request) {
-		remotePath := r.URL.Query().Get("path")
-		node, err := s.vfs.Stat(remotePath)
-		if err != nil {
-			http.Error(w, err.Error(), http.StatusInternalServerError)
-			return
-		}
-
-		w.Header().Set("Content-Length", strconv.FormatInt(node.Size(), 10))
-
-		file := node.(*vfs.File)
-		in, err := file.Open(os.O_RDONLY)
-		if err != nil {
-			http.Error(w, err.Error(), http.StatusInternalServerError)
-		}
-		defer fs.CheckClose(in, &err)
-
-		http.ServeContent(w, r, remotePath, node.ModTime(), in)
-		return
-	})
-
-	mux.HandleFunc(rootDescPath, func(w http.ResponseWriter, r *http.Request) {
-		w.Header().Set("content-type", `text/xml; charset="utf-8"`)
-		w.Header().Set("content-length", fmt.Sprint(len(s.rootDescXML)))
-		w.Header().Set("server", serverField)
-		_, err := w.Write(s.rootDescXML)
-		if err != nil {
-			fs.Errorf(s, "Failed to serve root descriptor XML: %v", err)
-		}
-	})
-
-	// Install handlers to serve SCPD for each UPnP service.
-	for _, s := range services {
-		p := path.Join("/scpd", s.ServiceId)
-		s.SCPDURL = p
-
-		mux.HandleFunc(s.SCPDURL, func(serviceDesc string) http.HandlerFunc {
-			return func(w http.ResponseWriter, r *http.Request) {
-				w.Header().Set("content-type", `text/xml; charset="utf-8"`)
-				http.ServeContent(w, r, ".xml", time.Time{}, bytes.NewReader([]byte(serviceDesc)))
-			}
-		}(s.SCPD))
-	}
-
-	mux.HandleFunc(serviceControlURL, s.serviceControlHandler)
-}
-
 // Handle a service control HTTP request.
 func (s *server) serviceControlHandler(w http.ResponseWriter, r *http.Request) {
 	soapActionString := r.Header.Get("SOAPACTION")
 	soapAction, err := upnp.ParseActionHTTPHeader(soapActionString)
 	if err != nil {
-		http.Error(w, err.Error(), http.StatusBadRequest)
+		serveError(s, w, "Could not parse SOAPACTION header", err)
 		return
 	}
 	var env soap.Envelope
 	if err := xml.NewDecoder(r.Body).Decode(&env); err != nil {
-		http.Error(w, err.Error(), http.StatusBadRequest)
+		serveError(s, w, "Could not parse SOAP request body", err)
 		return
 	}
 
 	w.Header().Set("Content-Type", `text/xml; charset="utf-8"`)
 	w.Header().Set("Ext", "")
-	w.Header().Set("server", serverField)
 	soapRespXML, code := func() ([]byte, int) {
 		respArgs, err := s.soapActionResponse(soapAction, env.Body.Action, r)
 		if err != nil {
+			fs.Errorf(s, "Error invoking %v: %v", soapAction, err)
 			upnpErr := upnp.ConvertError(err)
-			return mustMarshalXML(soap.NewFault("UPnPError", upnpErr)), 500
+			return mustMarshalXML(soap.NewFault("UPnPError", upnpErr)), http.StatusInternalServerError
 		}
-		return marshalSOAPResponse(soapAction, respArgs), 200
+		return marshalSOAPResponse(soapAction, respArgs), http.StatusOK
 	}()
 	bodyStr := fmt.Sprintf(`<?xml version="1.0" encoding="utf-8" standalone="yes"?><s:Envelope xmlns:s="http://schemas.xmlsoap.org/soap/envelope/" s:encodingStyle="http://schemas.xmlsoap.org/soap/encoding/"><s:Body>%s</s:Body></s:Envelope>`, soapRespXML)
 	w.WriteHeader(code)
 	if _, err := w.Write([]byte(bodyStr)); err != nil {
-		log.Print(err)
+		fs.Infof(s, "Error writing response: %v", err)
 	}
 }
 
@@ -306,6 +287,36 @@ func (s *server) soapActionResponse(sa upnp.SoapAction, actionRequestXML []byte,
 	return service.Handle(sa.Action, actionRequestXML, r)
 }
 
+// Serves actual resources (media files).
+func (s *server) resourceHandler(w http.ResponseWriter, r *http.Request) {
+	remotePath := r.URL.Query().Get("path")
+	node, err := s.vfs.Stat(remotePath)
+	if err != nil {
+		http.NotFound(w, r)
+		return
+	}
+
+	w.Header().Set("Content-Length", strconv.FormatInt(node.Size(), 10))
+
+	// add some DLNA specific headers
+	if r.Header.Get("getContentFeatures.dlna.org") != "" {
+		w.Header().Set("contentFeatures.dlna.org", dms_dlna.ContentFeatures{
+			SupportRange: true,
+		}.String())
+	}
+	w.Header().Set("transferMode.dlna.org", "Streaming")
+
+	file := node.(*vfs.File)
+	in, err := file.Open(os.O_RDONLY)
+	if err != nil {
+		serveError(node, w, "Could not open resource", err)
+		return
+	}
+	defer fs.CheckClose(in, &err)
+
+	http.ServeContent(w, r, remotePath, node.ModTime(), in)
+}
+
 // Serve runs the server - returns the error only if
 // the listener was not started; does not block, so
 // use s.Wait() to block on the listener indefinitely.
@@ -381,13 +392,19 @@ func (s *server) ssdpInterface(intf net.Interface) {
 		return url.String()
 	}
 
+	// Note that the devices and services advertised here via SSDP should be
+	// in agreement with the rootDesc XML descriptor that is defined above.
 	ssdpServer := ssdp.Server{
-		Interface:      intf,
-		Devices:        devices(),
-		Services:       serviceTypes(),
+		Interface: intf,
+		Devices: []string{
+			"urn:schemas-upnp-org:device:MediaServer:1"},
+		Services: []string{
+			"urn:schemas-upnp-org:service:ContentDirectory:1",
+			"urn:schemas-upnp-org:service:ConnectionManager:1",
+			"urn:microsoft.com:service:X_MS_MediaReceiverRegistrar:1"},
 		Location:       advertiseLocationFn,
 		Server:         serverField,
-		UUID:           s.rootDeviceUUID,
+		UUID:           s.RootDeviceUUID,
 		NotifyInterval: s.AnnounceInterval,
 	}
 
@@ -405,16 +422,16 @@ func (s *server) ssdpInterface(intf net.Interface) {
 			// good.
 			return
 		}
-		log.Printf("Error creating ssdp server on %s: %s", intf.Name, err)
+		fs.Errorf(s, "Error creating ssdp server on %s: %s", intf.Name, err)
 		return
 	}
 	defer ssdpServer.Close()
-	log.Println("Started SSDP on", intf.Name)
+	fs.Infof(s, "Started SSDP on %v", intf.Name)
 	stopped := make(chan struct{})
 	go func() {
 		defer close(stopped)
 		if err := ssdpServer.Serve(); err != nil {
-			log.Printf("%q: %q\n", intf.Name, err)
+			fs.Errorf(s, "%q: %q\n", intf.Name, err)
 		}
 	}()
 	select {
@@ -426,9 +443,7 @@ func (s *server) ssdpInterface(intf net.Interface) {
 
 func (s *server) serveHTTP() error {
 	srv := &http.Server{
-		Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			s.httpServeMux.ServeHTTP(w, r)
-		}),
+		Handler: s.handler,
 	}
 	err := srv.Serve(s.HTTPConn)
 	select {

cmd/serve/dlna/dlna_test.go

@@ -1,5 +1,3 @@
-// +build go1.8
-
 package dlna
 
 import (
@@ -22,11 +20,11 @@ import (
 
 var (
 	dlnaServer *server
+	testURL    string
 )
 
 const (
-	testBindAddress = "localhost:51777"
-	testURL         = "http://" + testBindAddress + "/"
+	testBindAddress = "localhost:0"
 )
 
 func startServer(t *testing.T, f fs.Fs) {
@@ -34,6 +32,7 @@ func startServer(t *testing.T, f fs.Fs) {
 	opt.ListenAddr = testBindAddress
 	dlnaServer = newServer(f, &opt)
 	assert.NoError(t, dlnaServer.Serve())
+	testURL = "http://" + dlnaServer.HTTPConn.Addr().String() + "/"
 }
 
 func TestInit(t *testing.T) {
@@ -59,6 +58,11 @@ func TestRootSCPD(t *testing.T) {
 	// Make sure that the SCPD contains a CDS service.
 	require.Contains(t, string(body),
 		"<serviceType>urn:schemas-upnp-org:service:ContentDirectory:1</serviceType>")
+	// Make sure that the SCPD contains a CM service.
+	require.Contains(t, string(body),
+		"<serviceType>urn:schemas-upnp-org:service:ConnectionManager:1</serviceType>")
+	// Ensure that the SCPD url is configured.
+	require.Regexp(t, "<SCPDURL>/.*</SCPDURL>", string(body))
 }
 
 // Make sure that it serves content from the remote.

cmd/serve/dlna/dlna_util.go

@@ -6,11 +6,28 @@ import (
 	"fmt"
 	"io"
 	"log"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"net/http/httputil"
+	"os"
 
 	"github.com/anacrolix/dms/soap"
 	"github.com/anacrolix/dms/upnp"
+	"github.com/ncw/rclone/fs"
 )
 
+// Return a default "friendly name" for the server.
+func makeDefaultFriendlyName() string {
+	hostName, err := os.Hostname()
+	if err != nil {
+		hostName = ""
+	} else {
+		hostName = " (" + hostName + ")"
+	}
+	return "rclone" + hostName
+}
+
 func makeDeviceUUID(unique string) string {
 	h := md5.New()
 	if _, err := io.WriteString(h, unique); err != nil {
@@ -20,6 +37,24 @@ func makeDeviceUUID(unique string) string {
 	return upnp.FormatUUID(buf)
 }
 
+// Get all available active network interfaces.
+func listInterfaces() []net.Interface {
+	ifs, err := net.Interfaces()
+	if err != nil {
+		log.Printf("list network interfaces: %v", err)
+		return []net.Interface{}
+	}
+
+	var active []net.Interface
+	for _, intf := range ifs {
+		if intf.Flags&net.FlagUp == 0 || intf.MTU <= 0 {
+			continue
+		}
+		active = append(active, intf)
+	}
+	return active
+}
+
 func didlLite(chardata string) string {
 	return `<DIDL-Lite` +
 		` xmlns:dc="http://purl.org/dc/elements/1.1/"` +
@@ -50,3 +85,99 @@ func marshalSOAPResponse(sa upnp.SoapAction, args map[string]string) []byte {
 	return []byte(fmt.Sprintf(`<u:%[1]sResponse xmlns:u="%[2]s">%[3]s</u:%[1]sResponse>`,
 		sa.Action, sa.ServiceURN.String(), mustMarshalXML(soapArgs)))
 }
+
+type loggingResponseWriter struct {
+	http.ResponseWriter
+	request   *http.Request
+	committed bool
+}
+
+func (lrw *loggingResponseWriter) logRequest(code int, err interface{}) {
+	// Choose appropriate log level based on response status code.
+	var level fs.LogLevel
+	if code < 400 && err == nil {
+		level = fs.LogLevelInfo
+	} else {
+		level = fs.LogLevelError
+	}
+
+	fs.LogPrintf(level, lrw.request.URL.Path, "%s %s %d %s %s",
+		lrw.request.RemoteAddr, lrw.request.Method, code,
+		lrw.request.Header.Get("SOAPACTION"), err)
+}
+
+func (lrw *loggingResponseWriter) WriteHeader(code int) {
+	lrw.committed = true
+	lrw.logRequest(code, nil)
+	lrw.ResponseWriter.WriteHeader(code)
+}
+
+// HTTP handler that logs requests and any errors or panics.
+func logging(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		lrw := &loggingResponseWriter{ResponseWriter: w, request: r}
+		defer func() {
+			err := recover()
+			if err != nil {
+				if !lrw.committed {
+					lrw.logRequest(http.StatusInternalServerError, err)
+					http.Error(w, fmt.Sprint(err), http.StatusInternalServerError)
+				} else {
+					// Too late to send the error to client, but at least log it.
+					fs.Errorf(r.URL.Path, "Recovered panic: %v", err)
+				}
+			}
+		}()
+		next.ServeHTTP(lrw, r)
+	})
+}
+
+// HTTP handler that logs complete request and response bodies for debugging.
+// Error recovery and general request logging are left to logging().
+func traceLogging(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		dump, err := httputil.DumpRequest(r, true)
+		if err != nil {
+			serveError(nil, w, "error dumping request", err)
+			return
+		}
+		fs.Debugf(nil, "%s", dump)
+
+		recorder := httptest.NewRecorder()
+		next.ServeHTTP(recorder, r)
+
+		dump, err = httputil.DumpResponse(recorder.Result(), true)
+		if err != nil {
+			// log the error but ignore it
+			fs.Errorf(nil, "error dumping response: %v", err)
+		} else {
+			fs.Debugf(nil, "%s", dump)
+		}
+
+		// copy from recorder to the real response writer
+		for k, v := range recorder.Header() {
+			w.Header()[k] = v
+		}
+		w.WriteHeader(recorder.Code)
+		_, err = recorder.Body.WriteTo(w)
+		if err != nil {
+			// Network error
+			fs.Debugf(nil, "Error writing response: %v", err)
+		}
+	})
+}
+
+// HTTP handler that sets headers.
+func withHeader(name string, value string, next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set(name, value)
+		next.ServeHTTP(w, r)
+	})
+}
+
+// serveError returns an http.StatusInternalServerError and logs the error
+func serveError(what interface{}, w http.ResponseWriter, text string, err error) {
+	fs.CountError(err)
+	fs.Errorf(what, "%s: %v", text, err)
+	http.Error(w, text+".", http.StatusInternalServerError)
+}

cmd/serve/dlna/dlnaflags/dlnaflags.go

@@ -14,16 +14,25 @@ Use --addr to specify which IP address and port the server should
 listen on, eg --addr 1.2.3.4:8000 or --addr :8080 to listen to all
 IPs.
 
+Use --name to choose the friendly server name, which is by
+default "rclone (hostname)".
+
+Use --log-trace in conjunction with -vv to enable additional debug
+logging of all UPNP traffic.
 `
 
 // Options is the type for DLNA serving options.
 type Options struct {
-	ListenAddr string
+	ListenAddr   string
+	FriendlyName string
+	LogTrace     bool
 }
 
 // DefaultOpt contains the defaults options for DLNA serving.
 var DefaultOpt = Options{
-	ListenAddr: ":7879",
+	ListenAddr:   ":7879",
+	FriendlyName: "",
+	LogTrace:     false,
 }
 
 // Opt contains the options for DLNA serving.
@@ -34,6 +43,8 @@ var (
 func addFlagsPrefix(flagSet *pflag.FlagSet, prefix string, Opt *Options) {
 	rc.AddOption("dlna", &Opt)
 	flags.StringVarP(flagSet, &Opt.ListenAddr, prefix+"addr", "", Opt.ListenAddr, "ip:port or :port to bind the DLNA http server to.")
+	flags.StringVarP(flagSet, &Opt.FriendlyName, prefix+"name", "", Opt.FriendlyName, "name of DLNA server")
+	flags.BoolVarP(flagSet, &Opt.LogTrace, prefix+"log-trace", "", Opt.LogTrace, "enable trace logging of SOAP traffic")
 }
 
 // AddFlags add the command line flags for DLNA serving.

cmd/serve/ftp/ftp.go

@@ -78,6 +78,7 @@ func newServer(f fs.Fs, opt *ftpopt.Options) (*server, error) {
 		},
 		Hostname:     host,
 		Port:         portNum,
+		PublicIp:     opt.PublicIP,
 		PassivePorts: opt.PassivePorts,
 		Auth: &Auth{
 			BasicUser: opt.BasicUser,
@@ -155,7 +156,7 @@ func (f *DriverFactory) NewDriver() (ftp.Driver, error) {
 	}, nil
 }
 
-//Driver impletation of ftp server
+//Driver implementation of ftp server
 type Driver struct {
 	vfs  *vfs.VFS
 	lock sync.Mutex
@@ -378,7 +379,7 @@ func (d *Driver) PutFile(path string, data io.Reader, appendData bool) (n int64,
 	return bytes, nil
 }
 
-//FileInfo  struct ot hold file infor for ftp server
+//FileInfo struct to hold file info for ftp server
 type FileInfo struct {
 	os.FileInfo
 
@@ -387,7 +388,7 @@ type FileInfo struct {
 	group uint32
 }
 
-//Mode return êrm mode of file.
+//Mode return mode of file.
 func (f *FileInfo) Mode() os.FileMode {
 	return f.mode
 }
@@ -407,7 +408,7 @@ func (f *FileInfo) Group() string {
 	str := fmt.Sprint(f.group)
 	g, err := user.LookupGroupId(str)
 	if err != nil {
-		return str //Group not found default to numrical value
+		return str //Group not found default to numerical value
 	}
 	return g.Name
 }

cmd/serve/ftp/ftpflags/ftpflags.go

@@ -16,6 +16,7 @@ var (
 func AddFlagsPrefix(flagSet *pflag.FlagSet, prefix string, Opt *ftpopt.Options) {
 	rc.AddOption("ftp", &Opt)
 	flags.StringVarP(flagSet, &Opt.ListenAddr, prefix+"addr", "", Opt.ListenAddr, "IPaddress:Port or :Port to bind server to.")
+	flags.StringVarP(flagSet, &Opt.PublicIP, prefix+"public-ip", "", Opt.PublicIP, "Public IP address to advertise for passive connections.")
 	flags.StringVarP(flagSet, &Opt.PassivePorts, prefix+"passive-port", "", Opt.PassivePorts, "Passive port range to use.")
 	flags.StringVarP(flagSet, &Opt.BasicUser, prefix+"user", "", Opt.BasicUser, "User name for authentication.")
 	flags.StringVarP(flagSet, &Opt.BasicPass, prefix+"pass", "", Opt.BasicPass, "Password for authentication. (empty value allow every password)")

cmd/serve/ftp/ftpopt/ftp_options.go

@@ -24,6 +24,7 @@ You can set a single username and password with the --user and --pass flags.
 type Options struct {
 	//TODO add more options
 	ListenAddr   string // Port to listen on
+	PublicIP     string // Passive ports range
 	PassivePorts string // Passive ports range
 	BasicUser    string // single username for basic auth if not using Htpasswd
 	BasicPass    string // password for BasicUser
@@ -32,6 +33,7 @@ type Options struct {
 // DefaultOpt is the default values used for Options
 var DefaultOpt = Options{
 	ListenAddr:   "localhost:2121",
+	PublicIP:     "",
 	PassivePorts: "30000-32000",
 	BasicUser:    "anonymous",
 	BasicPass:    "",

cmd/serve/http/http_test.go

@@ -1,11 +1,8 @@
-// +build go1.8
-
 package http
 
 import (
 	"flag"
 	"io/ioutil"
-	"net"
 	"net/http"
 	"strings"
 	"testing"
@@ -23,11 +20,11 @@ import (
 var (
 	updateGolden = flag.Bool("updategolden", false, "update golden files for regression test")
 	httpServer   *server
+	testURL      string
 )
 
 const (
-	testBindAddress = "localhost:51777"
-	testURL         = "http://" + testBindAddress + "/"
+	testBindAddress = "localhost:0"
 )
 
 func startServer(t *testing.T, f fs.Fs) {
@@ -35,13 +32,14 @@ func startServer(t *testing.T, f fs.Fs) {
 	opt.ListenAddr = testBindAddress
 	httpServer = newServer(f, &opt)
 	assert.NoError(t, httpServer.Serve())
+	testURL = httpServer.Server.URL()
 
 	// try to connect to the test server
 	pause := time.Millisecond
 	for i := 0; i < 10; i++ {
-		conn, err := net.Dial("tcp", testBindAddress)
+		resp, err := http.Head(testURL)
 		if err == nil {
-			_ = conn.Close()
+			_ = resp.Body.Close()
 			return
 		}
 		// t.Logf("couldn't connect, sleeping for %v: %v", pause, err)

cmd/serve/httplib/http_new.go

@@ -1,21 +0,0 @@
-// HTTP parts go1.8+
-
-//+build go1.8
-
-package httplib
-
-import (
-	"net/http"
-	"time"
-)
-
-// Initialise the http.Server for post go1.8
-func initServer(s *http.Server) {
-	s.ReadHeaderTimeout = 10 * time.Second // time to send the headers
-	s.IdleTimeout = 60 * time.Second       // time to keep idle connections open
-}
-
-// closeServer closes the server in a non graceful way
-func closeServer(s *http.Server) error {
-	return s.Close()
-}

cmd/serve/httplib/http_old.go

@@ -1,18 +0,0 @@
-// HTTP parts pre go1.8
-
-//+build !go1.8
-
-package httplib
-
-import (
-	"net/http"
-)
-
-// Initialise the http.Server for pre go1.8
-func initServer(s *http.Server) {
-}
-
-// closeServer closes the server in a non graceful way
-func closeServer(s *http.Server) error {
-	return nil
-}

cmd/serve/httplib/httplib.go

@@ -180,17 +180,17 @@ func NewServer(handler http.Handler, opt *Options) *Server {
 
 	// FIXME make a transport?
 	s.httpServer = &http.Server{
-		Addr:           s.Opt.ListenAddr,
-		Handler:        handler,
-		ReadTimeout:    s.Opt.ServerReadTimeout,
-		WriteTimeout:   s.Opt.ServerWriteTimeout,
-		MaxHeaderBytes: s.Opt.MaxHeaderBytes,
+		Addr:              s.Opt.ListenAddr,
+		Handler:           handler,
+		ReadTimeout:       s.Opt.ServerReadTimeout,
+		WriteTimeout:      s.Opt.ServerWriteTimeout,
+		MaxHeaderBytes:    s.Opt.MaxHeaderBytes,
+		ReadHeaderTimeout: 10 * time.Second, // time to send the headers
+		IdleTimeout:       60 * time.Second, // time to keep idle connections open
 		TLSConfig: &tls.Config{
 			MinVersion: tls.VersionTLS10, // disable SSL v3.0 and earlier
 		},
 	}
-	// go version specific initialisation
-	initServer(s.httpServer)
 
 	if s.Opt.ClientCA != "" {
 		if !s.useSSL {
@@ -267,7 +267,7 @@ func (s *Server) Wait() {
 
 // Close shuts the running server down
 func (s *Server) Close() {
-	err := closeServer(s.httpServer)
+	err := s.httpServer.Close()
 	if err != nil {
 		log.Printf("Error on closing HTTP server: %v", err)
 		return

cmd/serve/httplib/serve/data/data.go

@@ -11,7 +11,7 @@ import (
 	"github.com/pkg/errors"
 )
 
-// GetTemplate eturns the HTML template for serving directories via HTTP
+// GetTemplate returns the HTML template for serving directories via HTTP
 func GetTemplate() (tpl *template.Template, err error) {
 	templateFile, err := Assets.Open("index.html")
 	if err != nil {

cmd/serve/restic/restic.go

@@ -330,25 +330,12 @@ func (s *server) listObjects(w http.ResponseWriter, r *http.Request, remote stri
 	ls := listItems{}
 
 	// if remote supports ListR use that directly, otherwise use recursive Walk
-	var err error
-	if ListR := s.f.Features().ListR; ListR != nil {
-		err = ListR(remote, func(entries fs.DirEntries) error {
-			for _, entry := range entries {
-				ls.add(entry)
-			}
-			return nil
-		})
-	} else {
-		err = walk.Walk(s.f, remote, true, -1, func(path string, entries fs.DirEntries, err error) error {
-			if err == nil {
-				for _, entry := range entries {
-					ls.add(entry)
-				}
-			}
-			return err
-		})
-	}
-
+	err := walk.ListR(s.f, remote, true, -1, walk.ListObjects, func(entries fs.DirEntries) error {
+		for _, entry := range entries {
+			ls.add(entry)
+		}
+		return nil
+	})
 	if err != nil {
 		_, err = fserrors.Cause(err)
 		if err != fs.ErrorDirNotFound {

cmd/serve/restic/restic_test.go

@@ -17,8 +17,7 @@ import (
 )
 
 const (
-	testBindAddress = "localhost:51779"
-	testURL         = "http://" + testBindAddress + "/"
+	testBindAddress = "localhost:0"
 	resticSource    = "../../../../../restic/restic"
 )
 
@@ -62,7 +61,7 @@ func TestRestic(t *testing.T) {
 		}
 		cmd := exec.Command("go", args...)
 		cmd.Env = append(os.Environ(),
-			"RESTIC_TEST_REST_REPOSITORY=rest:"+testURL+path,
+			"RESTIC_TEST_REST_REPOSITORY=rest:"+w.Server.URL()+path,
 		)
 		out, err := cmd.CombinedOutput()
 		if len(out) != 0 {

cmd/serve/serve.go

@@ -3,12 +3,12 @@ package serve
 import (
 	"errors"
 
-	"github.com/ncw/rclone/cmd/serve/dlna"
-
 	"github.com/ncw/rclone/cmd"
+	"github.com/ncw/rclone/cmd/serve/dlna"
 	"github.com/ncw/rclone/cmd/serve/ftp"
 	"github.com/ncw/rclone/cmd/serve/http"
 	"github.com/ncw/rclone/cmd/serve/restic"
+	"github.com/ncw/rclone/cmd/serve/sftp"
 	"github.com/ncw/rclone/cmd/serve/webdav"
 	"github.com/spf13/cobra"
 )
@@ -27,6 +27,9 @@ func init() {
 	if ftp.Command != nil {
 		Command.AddCommand(ftp.Command)
 	}
+	if sftp.Command != nil {
+		Command.AddCommand(sftp.Command)
+	}
 	cmd.Root.AddCommand(Command)
 }
 

cmd/serve/sftp/connection.go

@@ -0,0 +1,254 @@
+// +build !plan9
+
+package sftp
+
+import (
+	"fmt"
+	"io"
+	"net"
+	"regexp"
+	"strings"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/hash"
+	"github.com/ncw/rclone/vfs"
+	"github.com/pkg/errors"
+	"github.com/pkg/sftp"
+	"golang.org/x/crypto/ssh"
+)
+
+func describeConn(c interface {
+	RemoteAddr() net.Addr
+	LocalAddr() net.Addr
+}) string {
+	return fmt.Sprintf("serve sftp %s->%s", c.RemoteAddr(), c.LocalAddr())
+}
+
+// Return the exit status of the command
+type exitStatus struct {
+	RC uint32
+}
+
+// The incoming exec command
+type execCommand struct {
+	Command string
+}
+
+var shellUnEscapeRegex = regexp.MustCompile(`\\(.)`)
+
+// Unescape a string that was escaped by rclone
+func shellUnEscape(str string) string {
+	str = strings.Replace(str, "'\n'", "\n", -1)
+	str = shellUnEscapeRegex.ReplaceAllString(str, `$1`)
+	return str
+}
+
+// Info about the current connection
+type conn struct {
+	vfs      *vfs.VFS
+	f        fs.Fs
+	handlers sftp.Handlers
+	what     string
+}
+
+// execCommand implements an extrememly limited number of commands to
+// interoperate with the rclone sftp backend
+func (c *conn) execCommand(out io.Writer, command string) (err error) {
+	binary, args := command, ""
+	space := strings.Index(command, " ")
+	if space >= 0 {
+		binary = command[:space]
+		args = strings.TrimLeft(command[space+1:], " ")
+	}
+	args = shellUnEscape(args)
+	fs.Debugf(c.what, "exec command: binary = %q, args = %q", binary, args)
+	switch binary {
+	case "df":
+		about := c.f.Features().About
+		if about == nil {
+			return errors.New("df not supported")
+		}
+		usage, err := about()
+		if err != nil {
+			return errors.Wrap(err, "About failed")
+		}
+		total, used, free := int64(-1), int64(-1), int64(-1)
+		if usage.Total != nil {
+			total = *usage.Total / 1024
+		}
+		if usage.Used != nil {
+			used = *usage.Used / 1024
+		}
+		if usage.Free != nil {
+			free = *usage.Free / 1024
+		}
+		perc := int64(0)
+		if total > 0 && used >= 0 {
+			perc = (100 * used) / total
+		}
+		_, err = fmt.Fprintf(out, `		Filesystem                   1K-blocks      Used Available Use%% Mounted on
+/dev/root %d %d  %d  %d%% /
+`, total, used, free, perc)
+		if err != nil {
+			return errors.Wrap(err, "send output failed")
+		}
+	case "md5sum", "sha1sum":
+		ht := hash.MD5
+		if binary == "sha1sum" {
+			ht = hash.SHA1
+		}
+		node, err := c.vfs.Stat(args)
+		if err != nil {
+			return errors.Wrapf(err, "hash failed finding file %q", args)
+		}
+		if node.IsDir() {
+			return errors.New("can't hash directory")
+		}
+		o, ok := node.DirEntry().(fs.ObjectInfo)
+		if !ok {
+			return errors.New("unexpected non file")
+		}
+		hash, err := o.Hash(ht)
+		if err != nil {
+			return errors.Wrap(err, "hash failed")
+		}
+		_, err = fmt.Fprintf(out, "%s  %s\n", hash, args)
+		if err != nil {
+			return errors.Wrap(err, "send output failed")
+		}
+	case "echo":
+		// special cases for rclone command detection
+		switch args {
+		case "'abc' | md5sum":
+			if c.f.Hashes().Contains(hash.MD5) {
+				_, err = fmt.Fprintf(out, "0bee89b07a248e27c83fc3d5951213c1  -\n")
+				if err != nil {
+					return errors.Wrap(err, "send output failed")
+				}
+			} else {
+				return errors.New("md5 hash not supported")
+			}
+		case "'abc' | sha1sum":
+			if c.f.Hashes().Contains(hash.SHA1) {
+				_, err = fmt.Fprintf(out, "03cfd743661f07975fa2f1220c5194cbaff48451  -\n")
+				if err != nil {
+					return errors.Wrap(err, "send output failed")
+				}
+			} else {
+				return errors.New("sha1 hash not supported")
+			}
+		default:
+			_, err = fmt.Fprintf(out, "%s\n", args)
+			if err != nil {
+				return errors.Wrap(err, "send output failed")
+			}
+		}
+	default:
+		return errors.Errorf("%q not implemented\n", command)
+	}
+	return nil
+}
+
+// handle a new incoming channel request
+func (c *conn) handleChannel(newChannel ssh.NewChannel) {
+	fs.Debugf(c.what, "Incoming channel: %s\n", newChannel.ChannelType())
+	if newChannel.ChannelType() != "session" {
+		err := newChannel.Reject(ssh.UnknownChannelType, "unknown channel type")
+		fs.Debugf(c.what, "Unknown channel type: %s\n", newChannel.ChannelType())
+		if err != nil {
+			fs.Errorf(c.what, "Failed to reject unknown channel: %v", err)
+		}
+		return
+	}
+	channel, requests, err := newChannel.Accept()
+	if err != nil {
+		fs.Errorf(c.what, "could not accept channel: %v", err)
+		return
+	}
+	defer func() {
+		err := channel.Close()
+		if err != nil {
+			fs.Debugf(c.what, "Failed to close channel: %v", err)
+		}
+	}()
+	fs.Debugf(c.what, "Channel accepted\n")
+
+	isSFTP := make(chan bool, 1)
+	var command execCommand
+
+	// Handle out-of-band requests
+	go func(in <-chan *ssh.Request) {
+		for req := range in {
+			fs.Debugf(c.what, "Request: %v\n", req.Type)
+			ok := false
+			var subSystemIsSFTP bool
+			var reply []byte
+			switch req.Type {
+			case "subsystem":
+				fs.Debugf(c.what, "Subsystem: %s\n", req.Payload[4:])
+				if string(req.Payload[4:]) == "sftp" {
+					ok = true
+					subSystemIsSFTP = true
+				}
+			case "exec":
+				err := ssh.Unmarshal(req.Payload, &command)
+				if err != nil {
+					fs.Errorf(c.what, "ignoring bad exec command: %v", err)
+				} else {
+					ok = true
+					subSystemIsSFTP = false
+				}
+			}
+			fs.Debugf(c.what, " - accepted: %v\n", ok)
+			err = req.Reply(ok, reply)
+			if err != nil {
+				fs.Errorf(c.what, "Failed to Reply to request: %v", err)
+				return
+			}
+			if ok {
+				// Wake up main routine after we have responded
+				isSFTP <- subSystemIsSFTP
+			}
+		}
+	}(requests)
+
+	// Wait for either subsystem "sftp" or "exec" request
+	if <-isSFTP {
+		fs.Debugf(c.what, "Starting SFTP server")
+		server := sftp.NewRequestServer(channel, c.handlers)
+		defer func() {
+			err := server.Close()
+			if err != nil {
+				fs.Debugf(c.what, "Failed to close server: %v", err)
+			}
+		}()
+		err = server.Serve()
+		if err == io.EOF || err == nil {
+			fs.Debugf(c.what, "exited session")
+		} else {
+			fs.Errorf(c.what, "completed with error: %v", err)
+		}
+	} else {
+		var rc = uint32(0)
+		err := c.execCommand(channel, command.Command)
+		if err != nil {
+			rc = 1
+			_, errPrint := fmt.Fprintf(channel.Stderr(), "%v\n", err)
+			if errPrint != nil {
+				fs.Errorf(c.what, "Failed to write to stderr: %v", errPrint)
+			}
+			fs.Debugf(c.what, "command %q failed with error: %v", command.Command, err)
+		}
+		_, err = channel.SendRequest("exit-status", false, ssh.Marshal(exitStatus{RC: rc}))
+		if err != nil {
+			fs.Errorf(c.what, "Failed to send exit status: %v", err)
+		}
+	}
+}
+
+// Service the incoming Channel channel in go routine
+func (c *conn) handleChannels(chans <-chan ssh.NewChannel) {
+	for newChannel := range chans {
+		go c.handleChannel(newChannel)
+	}
+}

cmd/serve/sftp/connection_test.go

@@ -0,0 +1,25 @@
+// +build !plan9
+
+package sftp
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestShellEscape(t *testing.T) {
+	for i, test := range []struct {
+		unescaped, escaped string
+	}{
+		{"", ""},
+		{"/this/is/harmless", "/this/is/harmless"},
+		{"$(rm -rf /)", "\\$\\(rm\\ -rf\\ /\\)"},
+		{"/test/\n", "/test/'\n'"},
+		{":\"'", ":\\\"\\'"},
+	} {
+		got := shellUnEscape(test.escaped)
+		assert.Equal(t, test.unescaped, got, fmt.Sprintf("Test %d unescaped = %q", i, test.unescaped))
+	}
+}

cmd/serve/sftp/handler.go

@@ -0,0 +1,154 @@
+// +build !plan9
+
+package sftp
+
+import (
+	"io"
+	"os"
+	"syscall"
+	"time"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/vfs"
+	"github.com/pkg/sftp"
+)
+
+// vfsHandler converts the VFS to be served by SFTP
+type vfsHandler struct {
+	*vfs.VFS
+}
+
+// vfsHandler returns a Handlers object with the test handlers.
+func newVFSHandler(vfs *vfs.VFS) (sftp.Handlers, error) {
+	v := vfsHandler{VFS: vfs}
+	return sftp.Handlers{
+		FileGet:  v,
+		FilePut:  v,
+		FileCmd:  v,
+		FileList: v,
+	}, nil
+}
+
+func (v vfsHandler) Fileread(r *sftp.Request) (io.ReaderAt, error) {
+	file, err := v.OpenFile(r.Filepath, os.O_RDONLY, 0777)
+	if err != nil {
+		return nil, err
+	}
+	return file, nil
+}
+
+func (v vfsHandler) Filewrite(r *sftp.Request) (io.WriterAt, error) {
+	file, err := v.OpenFile(r.Filepath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0777)
+	if err != nil {
+		return nil, err
+	}
+	return file, nil
+}
+
+func (v vfsHandler) Filecmd(r *sftp.Request) error {
+	switch r.Method {
+	case "Setstat":
+		node, err := v.Stat(r.Filepath)
+		if err != nil {
+			return err
+		}
+		attr := r.Attributes()
+		if attr.Mtime != 0 {
+			modTime := time.Unix(int64(attr.Mtime), 0)
+			err := node.SetModTime(modTime)
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	case "Rename":
+		err := v.Rename(r.Filepath, r.Target)
+		if err != nil {
+			return err
+		}
+	case "Rmdir", "Remove":
+		node, err := v.Stat(r.Filepath)
+		if err != nil {
+			return err
+		}
+		err = node.Remove()
+		if err != nil {
+			return err
+		}
+	case "Mkdir":
+		dir, leaf, err := v.StatParent(r.Filepath)
+		if err != nil {
+			return err
+		}
+		_, err = dir.Mkdir(leaf)
+		if err != nil {
+			return err
+		}
+	case "Symlink":
+		// FIXME
+		// _, err := v.fetch(r.Filepath)
+		// if err != nil {
+		// 	return err
+		// }
+		// link := newMemFile(r.Target, false)
+		// link.symlink = r.Filepath
+		// v.files[r.Target] = link
+	}
+	return nil
+}
+
+type listerat []os.FileInfo
+
+// Modeled after strings.Reader's ReadAt() implementation
+func (f listerat) ListAt(ls []os.FileInfo, offset int64) (int, error) {
+	var n int
+	if offset >= int64(len(f)) {
+		return 0, io.EOF
+	}
+	n = copy(ls, f[offset:])
+	if n < len(ls) {
+		return n, io.EOF
+	}
+	return n, nil
+}
+
+func (v vfsHandler) Filelist(r *sftp.Request) (l sftp.ListerAt, err error) {
+	var node vfs.Node
+	var handle vfs.Handle
+	switch r.Method {
+	case "List":
+		node, err = v.Stat(r.Filepath)
+		if err != nil {
+			return nil, err
+		}
+		if !node.IsDir() {
+			return nil, syscall.ENOTDIR
+		}
+		handle, err = node.Open(os.O_RDONLY)
+		if err != nil {
+			return nil, err
+		}
+		defer fs.CheckClose(handle, &err)
+		fis, err := handle.Readdir(-1)
+		if err != nil {
+			return nil, err
+		}
+		return listerat(fis), nil
+	case "Stat":
+		node, err = v.Stat(r.Filepath)
+		if err != nil {
+			return nil, err
+		}
+		return listerat([]os.FileInfo{node}), nil
+	case "Readlink":
+		// FIXME
+		// if file.symlink != "" {
+		// 	file, err = v.fetch(file.symlink)
+		// 	if err != nil {
+		// 		return nil, err
+		// 	}
+		// }
+		// return listerat([]os.FileInfo{file}), nil
+	}
+	return nil, nil
+}

cmd/serve/sftp/server.go

@@ -0,0 +1,282 @@
+// +build !plan9
+
+package sftp
+
+import (
+	"bytes"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/subtle"
+	"crypto/x509"
+	"encoding/pem"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/config"
+	"github.com/ncw/rclone/lib/env"
+	"github.com/ncw/rclone/vfs"
+	"github.com/ncw/rclone/vfs/vfsflags"
+	"github.com/pkg/errors"
+	"github.com/pkg/sftp"
+	"golang.org/x/crypto/ssh"
+)
+
+// server contains everything to run the server
+type server struct {
+	f        fs.Fs
+	opt      Options
+	vfs      *vfs.VFS
+	config   *ssh.ServerConfig
+	handlers sftp.Handlers
+	listener net.Listener
+	waitChan chan struct{} // for waiting on the listener to close
+}
+
+func newServer(f fs.Fs, opt *Options) *server {
+	s := &server{
+		f:        f,
+		vfs:      vfs.New(f, &vfsflags.Opt),
+		opt:      *opt,
+		waitChan: make(chan struct{}),
+	}
+	return s
+}
+
+func (s *server) acceptConnections() {
+	for {
+		nConn, err := s.listener.Accept()
+		if err != nil {
+			if strings.Contains(err.Error(), "use of closed network connection") {
+				return
+			}
+			fs.Errorf(nil, "Failed to accept incoming connection: %v", err)
+			continue
+		}
+		what := describeConn(nConn)
+
+		// Before use, a handshake must be performed on the incoming net.Conn.
+		sshConn, chans, reqs, err := ssh.NewServerConn(nConn, s.config)
+		if err != nil {
+			fs.Errorf(what, "SSH login failed: %v", err)
+			continue
+		}
+
+		fs.Infof(what, "SSH login from %s using %s", sshConn.User(), sshConn.ClientVersion())
+
+		// Discard all global out-of-band Requests
+		go ssh.DiscardRequests(reqs)
+
+		c := &conn{
+			vfs:      s.vfs,
+			f:        s.f,
+			handlers: s.handlers,
+			what:     what,
+		}
+
+		// Accept all channels
+		go c.handleChannels(chans)
+	}
+}
+
+// Based on example server code from golang.org/x/crypto/ssh and server_standalone
+func (s *server) serve() (err error) {
+	var authorizedKeysMap map[string]struct{}
+
+	// Load the authorized keys
+	if s.opt.AuthorizedKeys != "" {
+		authKeysFile := env.ShellExpand(s.opt.AuthorizedKeys)
+		authorizedKeysMap, err = loadAuthorizedKeys(authKeysFile)
+		// If user set the flag away from the default then report an error
+		if err != nil && s.opt.AuthorizedKeys != DefaultOpt.AuthorizedKeys {
+			return err
+		}
+		fs.Logf(nil, "Loaded %d authorized keys from %q", len(authorizedKeysMap), authKeysFile)
+	}
+
+	if !s.opt.NoAuth && len(authorizedKeysMap) == 0 && s.opt.User == "" && s.opt.Pass == "" {
+		return errors.New("no authorization found, use --user/--pass or --authorized-keys or --no-auth")
+	}
+
+	// An SSH server is represented by a ServerConfig, which holds
+	// certificate details and handles authentication of ServerConns.
+	s.config = &ssh.ServerConfig{
+		ServerVersion: "SSH-2.0-" + fs.Config.UserAgent,
+		PasswordCallback: func(c ssh.ConnMetadata, pass []byte) (*ssh.Permissions, error) {
+			fs.Debugf(describeConn(c), "Password login attempt for %s", c.User())
+			if s.opt.User != "" && s.opt.Pass != "" {
+				userOK := subtle.ConstantTimeCompare([]byte(c.User()), []byte(s.opt.User))
+				passOK := subtle.ConstantTimeCompare(pass, []byte(s.opt.Pass))
+				if (userOK & passOK) == 1 {
+					return nil, nil
+				}
+			}
+			return nil, fmt.Errorf("password rejected for %q", c.User())
+		},
+		PublicKeyCallback: func(c ssh.ConnMetadata, pubKey ssh.PublicKey) (*ssh.Permissions, error) {
+			fs.Debugf(describeConn(c), "Public key login attempt for %s", c.User())
+			if _, ok := authorizedKeysMap[string(pubKey.Marshal())]; ok {
+				return &ssh.Permissions{
+					// Record the public key used for authentication.
+					Extensions: map[string]string{
+						"pubkey-fp": ssh.FingerprintSHA256(pubKey),
+					},
+				}, nil
+			}
+			return nil, fmt.Errorf("unknown public key for %q", c.User())
+		},
+		AuthLogCallback: func(conn ssh.ConnMetadata, method string, err error) {
+			status := "OK"
+			if err != nil {
+				status = err.Error()
+			}
+			fs.Debugf(describeConn(conn), "ssh auth %q from %q: %s", method, conn.ClientVersion(), status)
+		},
+		NoClientAuth: s.opt.NoAuth,
+	}
+
+	// Load the private key, from the cache if not explicitly configured
+	keyPath := s.opt.Key
+	cachePath := filepath.Join(config.CacheDir, "serve-sftp")
+	if keyPath == "" {
+		keyPath = filepath.Join(cachePath, "id_rsa")
+	}
+	private, err := loadPrivateKey(keyPath)
+	if err != nil && s.opt.Key == "" {
+		fs.Debugf(nil, "Failed to load %q: %v", keyPath, err)
+		// If loading a cached key failed, make the keys and retry
+		err = os.MkdirAll(cachePath, 0700)
+		if err != nil {
+			return errors.Wrap(err, "failed to create cache path")
+		}
+		const bits = 2048
+		fs.Logf(nil, "Generating %d bit key pair at %q", bits, keyPath)
+		err = makeSSHKeyPair(bits, keyPath+".pub", keyPath)
+		if err != nil {
+			return errors.Wrap(err, "failed to create SSH key pair")
+		}
+		// reload the new keys
+		private, err = loadPrivateKey(keyPath)
+	}
+	if err != nil {
+		return err
+	}
+	fs.Debugf(nil, "Loaded private key from %q", keyPath)
+
+	s.config.AddHostKey(private)
+
+	// Once a ServerConfig has been configured, connections can be
+	// accepted.
+	s.listener, err = net.Listen("tcp", s.opt.ListenAddr)
+	if err != nil {
+		log.Fatal("failed to listen for connection", err)
+	}
+	fs.Logf(nil, "SFTP server listening on %v\n", s.listener.Addr())
+
+	s.handlers, err = newVFSHandler(s.vfs)
+	if err != nil {
+		return errors.Wrap(err, "serve sftp: failed to create fs")
+	}
+
+	go s.acceptConnections()
+
+	return nil
+}
+
+// Addr returns the address the server is listening on
+func (s *server) Addr() string {
+	return s.listener.Addr().String()
+}
+
+// Serve runs the sftp server in the background.
+//
+// Use s.Close() and s.Wait() to shutdown server
+func (s *server) Serve() error {
+	err := s.serve()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// Wait blocks while the listener is open.
+func (s *server) Wait() {
+	<-s.waitChan
+}
+
+// Close shuts the running server down
+func (s *server) Close() {
+	err := s.listener.Close()
+	if err != nil {
+		fs.Errorf(nil, "Error on closing SFTP server: %v", err)
+		return
+	}
+	close(s.waitChan)
+}
+
+func loadPrivateKey(keyPath string) (ssh.Signer, error) {
+	privateBytes, err := ioutil.ReadFile(keyPath)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to load private key")
+	}
+	private, err := ssh.ParsePrivateKey(privateBytes)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to parse private key")
+	}
+	return private, nil
+}
+
+// Public key authentication is done by comparing
+// the public key of a received connection
+// with the entries in the authorized_keys file.
+func loadAuthorizedKeys(authorizedKeysPath string) (authorizedKeysMap map[string]struct{}, err error) {
+	authorizedKeysBytes, err := ioutil.ReadFile(authorizedKeysPath)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to load authorized keys")
+	}
+	authorizedKeysMap = make(map[string]struct{})
+	for len(authorizedKeysBytes) > 0 {
+		pubKey, _, _, rest, err := ssh.ParseAuthorizedKey(authorizedKeysBytes)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to parse authorized keys")
+		}
+		authorizedKeysMap[string(pubKey.Marshal())] = struct{}{}
+		authorizedKeysBytes = bytes.TrimSpace(rest)
+	}
+	return authorizedKeysMap, nil
+}
+
+// makeSSHKeyPair make a pair of public and private keys for SSH access.
+// Public key is encoded in the format for inclusion in an OpenSSH authorized_keys file.
+// Private Key generated is PEM encoded
+//
+// Originally from: https://stackoverflow.com/a/34347463/164234
+func makeSSHKeyPair(bits int, pubKeyPath, privateKeyPath string) (err error) {
+	privateKey, err := rsa.GenerateKey(rand.Reader, bits)
+	if err != nil {
+		return err
+	}
+
+	// generate and write private key as PEM
+	privateKeyFile, err := os.OpenFile(privateKeyPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
+	if err != nil {
+		return err
+	}
+	defer fs.CheckClose(privateKeyFile, &err)
+	privateKeyPEM := &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(privateKey)}
+	if err := pem.Encode(privateKeyFile, privateKeyPEM); err != nil {
+		return err
+	}
+
+	// generate and write public key
+	pub, err := ssh.NewPublicKey(&privateKey.PublicKey)
+	if err != nil {
+		return err
+	}
+	return ioutil.WriteFile(pubKeyPath, ssh.MarshalAuthorizedKey(pub), 0644)
+}

cmd/serve/sftp/sftp.go

@@ -0,0 +1,101 @@
+// Package sftp implements an SFTP server to serve an rclone VFS
+
+// +build !plan9
+
+package sftp
+
+import (
+	"github.com/ncw/rclone/cmd"
+	"github.com/ncw/rclone/fs/config/flags"
+	"github.com/ncw/rclone/fs/rc"
+	"github.com/ncw/rclone/vfs"
+	"github.com/ncw/rclone/vfs/vfsflags"
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+)
+
+// Options contains options for the http Server
+type Options struct {
+	ListenAddr     string // Port to listen on
+	Key            string // Path to private key
+	AuthorizedKeys string // Path to authorized keys file
+	User           string // single username
+	Pass           string // password for user
+	NoAuth         bool   // allow no authentication on connections
+}
+
+// DefaultOpt is the default values used for Options
+var DefaultOpt = Options{
+	ListenAddr:     "localhost:2022",
+	AuthorizedKeys: "~/.ssh/authorized_keys",
+}
+
+// Opt is options set by command line flags
+var Opt = DefaultOpt
+
+// AddFlags adds flags for the sftp
+func AddFlags(flagSet *pflag.FlagSet, Opt *Options) {
+	rc.AddOption("sftp", &Opt)
+	flags.StringVarP(flagSet, &Opt.ListenAddr, "addr", "", Opt.ListenAddr, "IPaddress:Port or :Port to bind server to.")
+	flags.StringVarP(flagSet, &Opt.Key, "key", "", Opt.Key, "SSH private key file (leave blank to auto generate)")
+	flags.StringVarP(flagSet, &Opt.AuthorizedKeys, "authorized-keys", "", Opt.AuthorizedKeys, "Authorized keys file")
+	flags.StringVarP(flagSet, &Opt.User, "user", "", Opt.User, "User name for authentication.")
+	flags.StringVarP(flagSet, &Opt.Pass, "pass", "", Opt.Pass, "Password for authentication.")
+	flags.BoolVarP(flagSet, &Opt.NoAuth, "no-auth", "", Opt.NoAuth, "Allow connections with no authentication if set.")
+}
+
+func init() {
+	vfsflags.AddFlags(Command.Flags())
+	AddFlags(Command.Flags(), &Opt)
+}
+
+// Command definition for cobra
+var Command = &cobra.Command{
+	Use:   "sftp remote:path",
+	Short: `Serve the remote over SFTP.`,
+	Long: `rclone serve sftp implements an SFTP server to serve the remote
+over SFTP.  This can be used with an SFTP client or you can make a
+remote of type sftp to use with it.
+
+You can use the filter flags (eg --include, --exclude) to control what
+is served.
+
+The server will log errors.  Use -v to see access logs.
+
+--bwlimit will be respected for file transfers.  Use --stats to
+control the stats printing.
+
+You must provide some means of authentication, either with --user/--pass,
+an authorized keys file (specify location with --authorized-keys - the
+default is the same as ssh) or set the --no-auth flag for no
+authentication when logging in.
+
+Note that this also implements a small number of shell commands so
+that it can provide md5sum/sha1sum/df information for the rclone sftp
+backend.  This means that is can support SHA1SUMs, MD5SUMs and the
+about command when paired with the rclone sftp backend.
+
+If you don't supply a --key then rclone will generate one and cache it
+for later use.
+
+By default the server binds to localhost:2022 - if you want it to be
+reachable externally then supply "--addr :2022" for example.
+
+Note that the default of "--vfs-cache-mode off" is fine for the rclone
+sftp backend, but it may not be with other SFTP clients.
+
+` + vfs.Help,
+	Run: func(command *cobra.Command, args []string) {
+		cmd.CheckArgs(1, 1, command, args)
+		f := cmd.NewFsSrc(args)
+		cmd.Run(false, true, command, func() error {
+			s := newServer(f, &Opt)
+			err := s.Serve()
+			if err != nil {
+				return err
+			}
+			s.Wait()
+			return nil
+		})
+	},
+}