crypt: add --crypt-pass-corrupted-blocks flag

Before this change any attempt to access a google doc in an rclone
mount would give the error "partial downloads are not supported while
exporting Google Documents" as the mount uses ranged requests to read
data.

This implements ranged requests for a limited number of scenarios,
just enough so that Google docs can be cat-ed from an rclone mount.
When they are cat-ed then they receive their correct size also.

.circleci/config.yml

@@ -1,3 +1,4 @@
+---
 version: 2
 
 jobs:
@@ -41,7 +42,9 @@ jobs:
     - run:
         name: Upload artifacts
         command: |
-            make circleci_upload
+          if [[ $CIRCLE_PULL_REQUEST != "" ]]; then
+              make circleci_upload
+          fi
 
     - store_artifacts:
         path: /tmp/rclone.dist

Makefile

@@ -67,7 +67,7 @@ ifdef FULL_TESTS
 	go vet $(BUILDTAGS) -printfuncs Debugf,Infof,Logf,Errorf ./...
 	errcheck $(BUILDTAGS) ./...
 	find . -name \*.go | grep -v /vendor/ | xargs goimports -d | grep . ; test $$? -eq 1
-	go list ./... | xargs -n1 golint | grep -E -v '(StorageUrl|CdnUrl)' ; test $$? -eq 1
+	go list ./... | xargs -n1 golint | grep -E -v '(StorageUrl|CdnUrl|ApplicationCredentialId)' ; test $$? -eq 1
 else
 	@echo Skipping source quality tests as version of go too old
 endif

README.md

@@ -51,6 +51,7 @@ Rclone *("rsync for cloud storage")* is a command line program to sync files and
   * put.io [:page_facing_up:](https://rclone.org/webdav/#put-io)
   * QingStor [:page_facing_up:](https://rclone.org/qingstor/)
   * Rackspace Cloud Files [:page_facing_up:](https://rclone.org/swift/)
+  * Scaleway [:page_facing_up:](https://rclone.org/s3/#scaleway)
   * SFTP [:page_facing_up:](https://rclone.org/sftp/)
   * Wasabi [:page_facing_up:](https://rclone.org/s3/#wasabi)
   * WebDAV [:page_facing_up:](https://rclone.org/webdav/)

backend/azureblob/azureblob.go

@@ -754,6 +754,35 @@ func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.
 	return fs, fs.Update(in, src, options...)
 }
 
+// Check if the container exists
+//
+// NB this can return incorrect results if called immediately after container deletion
+func (f *Fs) dirExists() (bool, error) {
+	options := azblob.ListBlobsSegmentOptions{
+		Details: azblob.BlobListingDetails{
+			Copy:             false,
+			Metadata:         false,
+			Snapshots:        false,
+			UncommittedBlobs: false,
+			Deleted:          false,
+		},
+		MaxResults: 1,
+	}
+	err := f.pacer.Call(func() (bool, error) {
+		ctx := context.Background()
+		_, err := f.cntURL.ListBlobsHierarchySegment(ctx, azblob.Marker{}, "", options)
+		return f.shouldRetry(err)
+	})
+	if err == nil {
+		return true, nil
+	}
+	// Check http error code along with service code, current SDK doesn't populate service code correctly sometimes
+	if storageErr, ok := err.(azblob.StorageError); ok && (storageErr.ServiceCode() == azblob.ServiceCodeContainerNotFound || storageErr.Response().StatusCode == http.StatusNotFound) {
+		return false, nil
+	}
+	return false, err
+}
+
 // Mkdir creates the container if it doesn't exist
 func (f *Fs) Mkdir(dir string) error {
 	f.containerOKMu.Lock()
@@ -761,6 +790,15 @@ func (f *Fs) Mkdir(dir string) error {
 	if f.containerOK {
 		return nil
 	}
+	if !f.containerDeleted {
+		exists, err := f.dirExists()
+		if err == nil {
+			f.containerOK = exists
+		}
+		if err != nil || exists {
+			return err
+		}
+	}
 
 	// now try to create the container
 	err := f.pacer.Call(func() (bool, error) {

backend/b2/b2.go

@@ -120,20 +120,26 @@ these chunks are buffered in memory and there might a maximum of
 minimim size.`,
 			Default:  fs.SizeSuffix(defaultChunkSize),
 			Advanced: true,
+		}, {
+			Name:     "disable_checksum",
+			Help:     `Disable checksums for large (> upload cutoff) files`,
+			Default:  false,
+			Advanced: true,
 		}},
 	})
 }
 
 // Options defines the configuration for this backend
 type Options struct {
-	Account      string        `config:"account"`
-	Key          string        `config:"key"`
-	Endpoint     string        `config:"endpoint"`
-	TestMode     string        `config:"test_mode"`
-	Versions     bool          `config:"versions"`
-	HardDelete   bool          `config:"hard_delete"`
-	UploadCutoff fs.SizeSuffix `config:"upload_cutoff"`
-	ChunkSize    fs.SizeSuffix `config:"chunk_size"`
+	Account         string        `config:"account"`
+	Key             string        `config:"key"`
+	Endpoint        string        `config:"endpoint"`
+	TestMode        string        `config:"test_mode"`
+	Versions        bool          `config:"versions"`
+	HardDelete      bool          `config:"hard_delete"`
+	UploadCutoff    fs.SizeSuffix `config:"upload_cutoff"`
+	ChunkSize       fs.SizeSuffix `config:"chunk_size"`
+	DisableCheckSum bool          `config:"disable_checksum"`
 }
 
 // Fs represents a remote b2 server
@@ -1500,11 +1506,6 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 		},
 		ContentLength: &size,
 	}
-	// for go1.8 (see release notes) we must nil the Body if we want a
-	// "Content-Length: 0" header which b2 requires for all files.
-	if size == 0 {
-		opts.Body = nil
-	}
 	var response api.FileInfo
 	// Don't retry, return a retry error instead
 	err = o.fs.pacer.CallNoRetry(func() (bool, error) {

backend/b2/upload.go

@@ -116,8 +116,10 @@ func (f *Fs) newLargeUpload(o *Object, in io.Reader, src fs.ObjectInfo) (up *lar
 		},
 	}
 	// Set the SHA1 if known
-	if calculatedSha1, err := src.Hash(hash.SHA1); err == nil && calculatedSha1 != "" {
-		request.Info[sha1Key] = calculatedSha1
+	if !o.fs.opt.DisableCheckSum {
+		if calculatedSha1, err := src.Hash(hash.SHA1); err == nil && calculatedSha1 != "" {
+			request.Info[sha1Key] = calculatedSha1
+		}
 	}
 	var response api.StartLargeFileResponse
 	err = f.pacer.Call(func() (bool, error) {

backend/crypt/cipher.go

@@ -144,6 +144,7 @@ type cipher struct {
 	buffers        sync.Pool // encrypt/decrypt buffers
 	cryptoRand     io.Reader // read crypto random numbers from here
 	dirNameEncrypt bool
+	passCorrupted  bool
 }
 
 // newCipher initialises the cipher.  If salt is "" then it uses a built in salt val
@@ -163,6 +164,11 @@ func newCipher(mode NameEncryptionMode, password, salt string, dirNameEncrypt bo
 	return c, nil
 }
 
+// Set to pass corrupted blocks
+func (c *cipher) setPassCorrupted(passCorrupted bool) {
+	c.passCorrupted = passCorrupted
+}
+
 // Key creates all the internal keys from the password passed in using
 // scrypt.
 //
@@ -822,7 +828,10 @@ func (fh *decrypter) fillBuffer() (err error) {
 		if err != nil {
 			return err // return pending error as it is likely more accurate
 		}
-		return ErrorEncryptedBadBlock
+		if !fh.c.passCorrupted {
+			return ErrorEncryptedBadBlock
+		}
+		fs.Errorf(nil, "passing corrupted block")
 	}
 	fh.bufIndex = 0
 	fh.bufSize = n - blockHeaderSize

backend/crypt/crypt.go

@@ -17,7 +17,6 @@ import (
 	"github.com/pkg/errors"
 )
 
-// Globals
 // Register with Fs
 func init() {
 	fs.Register(&fs.RegInfo{
@@ -80,6 +79,15 @@ names, or for debugging purposes.`,
 			Default:  false,
 			Hide:     fs.OptionHideConfigurator,
 			Advanced: true,
+		}, {
+			Name: "pass_corrupted_blocks",
+			Help: `Pass through corrupted blocks to the output.
+
+This is for debugging corruption problems in crypt - it shouldn't be needed normally.
+`,
+			Default:  false,
+			Hide:     fs.OptionHideConfigurator,
+			Advanced: true,
 		}},
 	})
 }
@@ -108,6 +116,7 @@ func newCipherForConfig(opt *Options) (Cipher, error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to make cipher")
 	}
+	cipher.setPassCorrupted(opt.PassCorruptedBlocks)
 	return cipher, nil
 }
 
@@ -197,6 +206,7 @@ type Options struct {
 	Password                string `config:"password"`
 	Password2               string `config:"password2"`
 	ShowMapping             bool   `config:"show_mapping"`
+	PassCorruptedBlocks     bool   `config:"pass_corrupted_blocks"`
 }
 
 // Fs represents a wrapped fs.Fs

backend/drive/drive.go

@@ -39,6 +39,7 @@ import (
 	"github.com/ncw/rclone/lib/dircache"
 	"github.com/ncw/rclone/lib/oauthutil"
 	"github.com/ncw/rclone/lib/pacer"
+	"github.com/ncw/rclone/lib/readers"
 	"github.com/pkg/errors"
 	"golang.org/x/oauth2"
 	"golang.org/x/oauth2/google"
@@ -54,7 +55,8 @@ const (
 	driveFolderType             = "application/vnd.google-apps.folder"
 	timeFormatIn                = time.RFC3339
 	timeFormatOut               = "2006-01-02T15:04:05.000000000Z07:00"
-	minSleep                    = 10 * time.Millisecond
+	defaultMinSleep             = fs.Duration(100 * time.Millisecond)
+	defaultBurst                = 100
 	defaultExportExtensions     = "docx,xlsx,pptx,svg"
 	scopePrefix                 = "https://www.googleapis.com/auth/"
 	defaultScope                = "drive"
@@ -356,6 +358,16 @@ will download it anyway.`,
 			Default:  fs.SizeSuffix(-1),
 			Help:     "If Object's are greater, use drive v2 API to download.",
 			Advanced: true,
+		}, {
+			Name:     "pacer_min_sleep",
+			Default:  defaultMinSleep,
+			Help:     "Minimum time to sleep between API calls.",
+			Advanced: true,
+		}, {
+			Name:     "pacer_burst",
+			Default:  defaultBurst,
+			Help:     "Number of API calls to allow without sleeping.",
+			Advanced: true,
 		}},
 	})
 
@@ -398,6 +410,8 @@ type Options struct {
 	AcknowledgeAbuse          bool          `config:"acknowledge_abuse"`
 	KeepRevisionForever       bool          `config:"keep_revision_forever"`
 	V2DownloadMinSize         fs.SizeSuffix `config:"v2_download_min_size"`
+	PacerMinSleep             fs.Duration   `config:"pacer_min_sleep"`
+	PacerBurst                int           `config:"pacer_burst"`
 }
 
 // Fs represents a remote drive server
@@ -744,7 +758,7 @@ func configTeamDrive(opt *Options, m configmap.Mapper, name string) error {
 	listFailed := false
 	for {
 		var teamDrives *drive.TeamDriveList
-		err = newPacer().Call(func() (bool, error) {
+		err = newPacer(opt).Call(func() (bool, error) {
 			teamDrives, err = listTeamDrives.Do()
 			return shouldRetry(err)
 		})
@@ -774,8 +788,8 @@ func configTeamDrive(opt *Options, m configmap.Mapper, name string) error {
 }
 
 // newPacer makes a pacer configured for drive
-func newPacer() *pacer.Pacer {
-	return pacer.New().SetMinSleep(minSleep).SetPacer(pacer.GoogleDrivePacer)
+func newPacer(opt *Options) *pacer.Pacer {
+	return pacer.New().SetMinSleep(time.Duration(opt.PacerMinSleep)).SetBurst(opt.PacerBurst).SetPacer(pacer.GoogleDrivePacer)
 }
 
 func getServiceAccountClient(opt *Options, credentialsData []byte) (*http.Client, error) {
@@ -879,7 +893,7 @@ func NewFs(name, path string, m configmap.Mapper) (fs.Fs, error) {
 		name:  name,
 		root:  root,
 		opt:   *opt,
-		pacer: newPacer(),
+		pacer: newPacer(opt),
 	}
 	f.isTeamDrive = opt.TeamDriveID != ""
 	f.features = (&fs.Features{
@@ -2481,16 +2495,32 @@ func (o *documentObject) Open(options ...fs.OpenOption) (in io.ReadCloser, err e
 	// Update the size with what we are reading as it can change from
 	// the HEAD in the listing to this GET. This stops rclone marking
 	// the transfer as corrupted.
+	var offset, end int64 = 0, -1
+	var newOptions = options[:0]
 	for _, o := range options {
+		// Note that Range requests don't work on Google docs:
 		// https://developers.google.com/drive/v3/web/manage-downloads#partial_download
-		if _, ok := o.(*fs.RangeOption); ok {
-			return nil, errors.New("partial downloads are not supported while exporting Google Documents")
+		// So do a subset of them manually
+		switch x := o.(type) {
+		case *fs.RangeOption:
+			offset, end = x.Start, x.End
+		case *fs.SeekOption:
+			offset, end = x.Offset, -1
+		default:
+			newOptions = append(newOptions, o)
 		}
 	}
+	options = newOptions
+	if offset != 0 {
+		return nil, errors.New("partial downloads are not supported while exporting Google Documents")
+	}
 	in, err = o.baseObject.open(o.url, options...)
 	if in != nil {
 		in = &openDocumentFile{o: o, in: in}
 	}
+	if end >= 0 {
+		in = readers.NewLimitedReadCloser(in, end-offset+1)
+	}
 	return
 }
 func (o *linkObject) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {

backend/jottacloud/api/types.go

@@ -9,7 +9,10 @@ import (
 )
 
 const (
+	// default time format for almost all request and responses
 	timeFormat = "2006-01-02-T15:04:05Z0700"
+	// the API server seems to use a different format
+	apiTimeFormat = "2006-01-02T15:04:05Z07:00"
 )
 
 // Time represents time values in the Jottacloud API. It uses a custom RFC3339 like format.
@@ -40,6 +43,9 @@ func (t *Time) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
 // Return Time string in Jottacloud format
 func (t Time) String() string { return time.Time(t).Format(timeFormat) }
 
+// APIString returns Time string in Jottacloud API format
+func (t Time) APIString() string { return time.Time(t).Format(apiTimeFormat) }
+
 // Flag is a hacky type for checking if an attribute is present
 type Flag bool
 
@@ -58,6 +64,15 @@ func (f *Flag) MarshalXMLAttr(name xml.Name) (xml.Attr, error) {
 	return attr, errors.New("unimplemented")
 }
 
+// TokenJSON is the struct representing the HTTP response from OAuth2
+// providers returning a token in JSON form.
+type TokenJSON struct {
+	AccessToken  string `json:"access_token"`
+	TokenType    string `json:"token_type"`
+	RefreshToken string `json:"refresh_token"`
+	ExpiresIn    int32  `json:"expires_in"` // at least PayPal returns string, while most return number
+}
+
 /*
 GET http://www.jottacloud.com/JFS/<account>
 
@@ -265,3 +280,37 @@ func (e *Error) Error() string {
 	}
 	return out
 }
+
+// AllocateFileRequest to prepare an upload to Jottacloud
+type AllocateFileRequest struct {
+	Bytes    int64  `json:"bytes"`
+	Created  string `json:"created"`
+	Md5      string `json:"md5"`
+	Modified string `json:"modified"`
+	Path     string `json:"path"`
+}
+
+// AllocateFileResponse for upload requests
+type AllocateFileResponse struct {
+	Name      string `json:"name"`
+	Path      string `json:"path"`
+	State     string `json:"state"`
+	UploadID  string `json:"upload_id"`
+	UploadURL string `json:"upload_url"`
+	Bytes     int64  `json:"bytes"`
+	ResumePos int64  `json:"resume_pos"`
+}
+
+// UploadResponse after an upload
+type UploadResponse struct {
+	Name      string      `json:"name"`
+	Path      string      `json:"path"`
+	Kind      string      `json:"kind"`
+	ContentID string      `json:"content_id"`
+	Bytes     int64       `json:"bytes"`
+	Md5       string      `json:"md5"`
+	Created   int64       `json:"created"`
+	Modified  int64       `json:"modified"`
+	Deleted   interface{} `json:"deleted"`
+	Mime      string      `json:"mime"`
+}

backend/jottacloud/jottacloud.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"log"
 	"net/http"
 	"net/url"
 	"os"
@@ -26,22 +27,41 @@ import (
 	"github.com/ncw/rclone/fs/fshttp"
 	"github.com/ncw/rclone/fs/hash"
 	"github.com/ncw/rclone/fs/walk"
+	"github.com/ncw/rclone/lib/oauthutil"
 	"github.com/ncw/rclone/lib/pacer"
 	"github.com/ncw/rclone/lib/rest"
 	"github.com/pkg/errors"
+	"golang.org/x/oauth2"
 )
 
 // Globals
 const (
-	minSleep          = 10 * time.Millisecond
-	maxSleep          = 2 * time.Second
-	decayConstant     = 2 // bigger for slower decay, exponential
-	defaultDevice     = "Jotta"
-	defaultMountpoint = "Sync"
-	rootURL           = "https://www.jottacloud.com/jfs/"
-	apiURL            = "https://api.jottacloud.com"
-	shareURL          = "https://www.jottacloud.com/"
-	cachePrefix       = "rclone-jcmd5-"
+	minSleep                    = 10 * time.Millisecond
+	maxSleep                    = 2 * time.Second
+	decayConstant               = 2 // bigger for slower decay, exponential
+	defaultDevice               = "Jotta"
+	defaultMountpoint           = "Sync"
+	rootURL                     = "https://www.jottacloud.com/jfs/"
+	apiURL                      = "https://api.jottacloud.com/files/v1/"
+	baseURL                     = "https://www.jottacloud.com/"
+	tokenURL                    = "https://api.jottacloud.com/auth/v1/token"
+	cachePrefix                 = "rclone-jcmd5-"
+	rcloneClientID              = "nibfk8biu12ju7hpqomr8b1e40"
+	rcloneEncryptedClientSecret = "Vp8eAv7eVElMnQwN-kgU9cbhgApNDaMqWdlDi5qFydlQoji4JBxrGMF2"
+	configUsername              = "user"
+)
+
+var (
+	// Description of how to auth for this app for a personal account
+	oauthConfig = &oauth2.Config{
+		Endpoint: oauth2.Endpoint{
+			AuthURL:  tokenURL,
+			TokenURL: tokenURL,
+		},
+		ClientID:     rcloneClientID,
+		ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
+		RedirectURL:  oauthutil.RedirectLocalhostURL,
+	}
 )
 
 // Register with Fs
@@ -50,13 +70,71 @@ func init() {
 		Name:        "jottacloud",
 		Description: "JottaCloud",
 		NewFs:       NewFs,
+		Config: func(name string, m configmap.Mapper) {
+			tokenString, ok := m.Get("token")
+			if ok && tokenString != "" {
+				fmt.Printf("Already have a token - refresh?\n")
+				if !config.Confirm() {
+					return
+				}
+			}
+
+			username, ok := m.Get(configUsername)
+			if !ok {
+				log.Fatalf("No username defined")
+			}
+			password := config.GetPassword("Your Jottacloud password is only required during config and will not be stored.")
+
+			// prepare out token request with username and password
+			srv := rest.NewClient(fshttp.NewClient(fs.Config))
+			values := url.Values{}
+			values.Set("grant_type", "PASSWORD")
+			values.Set("password", password)
+			values.Set("username", username)
+			values.Set("client_id", oauthConfig.ClientID)
+			values.Set("client_secret", oauthConfig.ClientSecret)
+			opts := rest.Opts{
+				Method:      "POST",
+				RootURL:     oauthConfig.Endpoint.AuthURL,
+				ContentType: "application/x-www-form-urlencoded",
+				Parameters:  values,
+			}
+
+			var jsonToken api.TokenJSON
+			resp, err := srv.CallJSON(&opts, nil, &jsonToken)
+			if err != nil {
+				// if 2fa is enabled the first request is expected to fail. we'lls do another request with the 2fa code as an additional http header
+				if resp != nil {
+					if resp.Header.Get("X-JottaCloud-OTP") == "required; SMS" {
+						fmt.Printf("This account has 2 factor authentication enabled you will receive a verification code via SMS.\n")
+						fmt.Printf("Enter verification code> ")
+						authCode := config.ReadLine()
+						authCode = strings.Replace(authCode, "-", "", -1) // the sms received contains a pair of 3 digit numbers seperated by '-' but wants a single 6 digit number
+						opts.ExtraHeaders = make(map[string]string)
+						opts.ExtraHeaders["X-Jottacloud-Otp"] = authCode
+						resp, err = srv.CallJSON(&opts, nil, &jsonToken)
+					}
+				}
+				if err != nil {
+					log.Fatalf("Failed to get resource token: %v", err)
+				}
+			}
+
+			var token oauth2.Token
+			token.AccessToken = jsonToken.AccessToken
+			token.RefreshToken = jsonToken.RefreshToken
+			token.TokenType = jsonToken.TokenType
+			token.Expiry = time.Now().Add(time.Duration(jsonToken.ExpiresIn) * time.Second)
+
+			// finally save them in the config
+			err = oauthutil.PutToken(name, m, &token, true)
+			if err != nil {
+				log.Fatalf("Error while setting token: %s", err)
+			}
+		},
 		Options: []fs.Option{{
-			Name: "user",
-			Help: "User Name",
-		}, {
-			Name:       "pass",
-			Help:       "Password.",
-			IsPassword: true,
+			Name: configUsername,
+			Help: "User Name:",
 		}, {
 			Name:     "mountpoint",
 			Help:     "The mountpoint to use.",
@@ -83,6 +161,11 @@ func init() {
 			Help:     "Remove existing public link to file/folder with link command rather than creating.\nDefault is false, meaning link command will create or retrieve public link.",
 			Default:  false,
 			Advanced: true,
+		}, {
+			Name:     "upload_resume_limit",
+			Help:     "Files bigger than this can be resumed if the upload failes.",
+			Default:  fs.SizeSuffix(10 * 1024 * 1024),
+			Advanced: true,
 		}},
 	})
 }
@@ -90,23 +173,25 @@ func init() {
 // Options defines the configuration for this backend
 type Options struct {
 	User               string        `config:"user"`
-	Pass               string        `config:"pass"`
 	Mountpoint         string        `config:"mountpoint"`
 	MD5MemoryThreshold fs.SizeSuffix `config:"md5_memory_limit"`
 	HardDelete         bool          `config:"hard_delete"`
 	Unlink             bool          `config:"unlink"`
+	UploadThreshold    fs.SizeSuffix `config:"upload_resume_limit"`
 }
 
 // Fs represents a remote jottacloud
 type Fs struct {
-	name        string
-	root        string
-	user        string
-	opt         Options
-	features    *fs.Features
-	endpointURL string
-	srv         *rest.Client
-	pacer       *pacer.Pacer
+	name         string
+	root         string
+	user         string
+	opt          Options
+	features     *fs.Features
+	endpointURL  string
+	srv          *rest.Client
+	apiSrv       *rest.Client
+	pacer        *pacer.Pacer
+	tokenRenewer *oauthutil.Renew // renew the token on expiry
 }
 
 // Object describes a jottacloud object
@@ -261,6 +346,29 @@ func (o *Object) filePath() string {
 	return o.fs.filePath(o.remote)
 }
 
+// Jottacloud requires the grant_type 'refresh_token' string
+// to be uppercase and throws a 400 Bad Request if we use the
+// lower case used by the oauth2 module
+//
+// This filter catches all refresh requests, reads the body,
+// changes the case and then sends it on
+func grantTypeFilter(req *http.Request) {
+	if tokenURL == req.URL.String() {
+		// read the entire body
+		refreshBody, err := ioutil.ReadAll(req.Body)
+		if err != nil {
+			return
+		}
+		_ = req.Body.Close()
+
+		// make the refesh token upper case
+		refreshBody = []byte(strings.Replace(string(refreshBody), "grant_type=refresh_token", "grant_type=REFRESH_TOKEN", 1))
+
+		// set the new ReadCloser (with a dummy Close())
+		req.Body = ioutil.NopCloser(bytes.NewReader(refreshBody))
+	}
+}
+
 // NewFs constructs an Fs from the path, container:path
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
@@ -273,25 +381,29 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	rootIsDir := strings.HasSuffix(root, "/")
 	root = parsePath(root)
 
-	user := config.FileGet(name, "user")
-	pass := config.FileGet(name, "pass")
-
-	if opt.Pass != "" {
-		var err error
-		opt.Pass, err = obscure.Reveal(opt.Pass)
-		if err != nil {
-			return nil, errors.Wrap(err, "couldn't decrypt password")
-		}
+	// the oauth client for the api servers needs
+	// a filter to fix the grant_type issues (see above)
+	baseClient := fshttp.NewClient(fs.Config)
+	if do, ok := baseClient.Transport.(interface {
+		SetRequestFilter(f func(req *http.Request))
+	}); ok {
+		do.SetRequestFilter(grantTypeFilter)
+	} else {
+		fs.Debugf(name+":", "Couldn't add request filter - uploads will fail")
+	}
+	oAuthClient, ts, err := oauthutil.NewClientWithBaseClient(name, m, oauthConfig, baseClient)
+	if err != nil {
+		return nil, errors.Wrap(err, "Failed to configure Jottacloud oauth client")
 	}
 
 	f := &Fs{
-		name: name,
-		root: root,
-		user: opt.User,
-		opt:  *opt,
-		//endpointURL: rest.URLPathEscape(path.Join(user, defaultDevice, opt.Mountpoint)),
-		srv:   rest.NewClient(fshttp.NewClient(fs.Config)).SetRoot(rootURL),
-		pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
+		name:   name,
+		root:   root,
+		user:   opt.User,
+		opt:    *opt,
+		srv:    rest.NewClient(oAuthClient).SetRoot(rootURL),
+		apiSrv: rest.NewClient(oAuthClient).SetRoot(apiURL),
+		pacer:  pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
 	}
 	f.features = (&fs.Features{
 		CaseInsensitive:         true,
@@ -299,14 +411,14 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		ReadMimeType:            true,
 		WriteMimeType:           true,
 	}).Fill(f)
-
-	if user == "" || pass == "" {
-		return nil, errors.New("jottacloud needs user and password")
-	}
-
-	f.srv.SetUserPass(opt.User, opt.Pass)
 	f.srv.SetErrorHandler(errorHandler)
 
+	// Renew the token in the background
+	f.tokenRenewer = oauthutil.NewRenew(f.String(), ts, func() error {
+		_, err := f.readMetaDataForPath("")
+		return err
+	})
+
 	err = f.setEndpointURL(opt.Mountpoint)
 	if err != nil {
 		return nil, errors.Wrap(err, "couldn't get account info")
@@ -331,7 +443,6 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		// return an error with an fs which points to the parent
 		return f, fs.ErrorIsFile
 	}
-
 	return f, nil
 }
 
@@ -348,7 +459,7 @@ func (f *Fs) newObjectWithInfo(remote string, info *api.JottaFile) (fs.Object, e
 		// Set info
 		err = o.setMetaData(info)
 	} else {
-		err = o.readMetaData() // reads info and meta, returning an error
+		err = o.readMetaData(false) // reads info and meta, returning an error
 	}
 	if err != nil {
 		return nil, err
@@ -396,7 +507,7 @@ func (f *Fs) CreateDir(path string) (jf *api.JottaFolder, err error) {
 // This should return ErrDirNotFound if the directory isn't
 // found.
 func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
-	//fmt.Printf("List: %s\n", dir)
+	//fmt.Printf("List: %s\n", f.filePath(dir))
 	opts := rest.Opts{
 		Method: "GET",
 		Path:   f.filePath(dir),
@@ -676,7 +787,6 @@ func (f *Fs) copyOrMove(method, src, dest string) (info *api.JottaFile, err erro
 	if err != nil {
 		return nil, err
 	}
-
 	return info, nil
 }
 
@@ -824,7 +934,7 @@ func (f *Fs) PublicLink(remote string) (link string, err error) {
 	if result.PublicSharePath == "" {
 		return "", errors.New("couldn't create public link - no link path received")
 	}
-	link = path.Join(shareURL, result.PublicSharePath)
+	link = path.Join(baseURL, result.PublicSharePath)
 	return link, nil
 }
 
@@ -880,7 +990,7 @@ func (o *Object) Hash(t hash.Type) (string, error) {
 
 // Size returns the size of an object in bytes
 func (o *Object) Size() int64 {
-	err := o.readMetaData()
+	err := o.readMetaData(false)
 	if err != nil {
 		fs.Logf(o, "Failed to read metadata: %v", err)
 		return 0
@@ -903,14 +1013,17 @@ func (o *Object) setMetaData(info *api.JottaFile) (err error) {
 	return nil
 }
 
-func (o *Object) readMetaData() (err error) {
-	if o.hasMetaData {
+func (o *Object) readMetaData(force bool) (err error) {
+	if o.hasMetaData && !force {
 		return nil
 	}
 	info, err := o.fs.readMetaDataForPath(o.remote)
 	if err != nil {
 		return err
 	}
+	if info.Deleted {
+		return fs.ErrorObjectNotFound
+	}
 	return o.setMetaData(info)
 }
 
@@ -919,7 +1032,7 @@ func (o *Object) readMetaData() (err error) {
 // It attempts to read the objects mtime and if that isn't present the
 // LastModified returned in the http headers
 func (o *Object) ModTime() time.Time {
-	err := o.readMetaData()
+	err := o.readMetaData(false)
 	if err != nil {
 		fs.Logf(o, "Failed to read metadata: %v", err)
 		return time.Now()
@@ -1040,43 +1153,74 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 		in = wrap(in)
 	}
 
+	// use the api to allocate the file first and get resume / deduplication info
 	var resp *http.Response
-	var result api.JottaFile
 	opts := rest.Opts{
-		Method:        "POST",
-		Path:          o.filePath(),
-		Body:          in,
-		ContentType:   fs.MimeType(src),
-		ContentLength: &size,
-		ExtraHeaders:  make(map[string]string),
-		Parameters:    url.Values{},
+		Method:       "POST",
+		Path:         "allocate",
+		ExtraHeaders: make(map[string]string),
+	}
+	fileDate := api.Time(src.ModTime()).APIString()
+
+	// the allocate request
+	var request = api.AllocateFileRequest{
+		Bytes:    size,
+		Created:  fileDate,
+		Modified: fileDate,
+		Md5:      md5String,
+		Path:     path.Join(o.fs.opt.Mountpoint, replaceReservedChars(path.Join(o.fs.root, o.remote))),
 	}
 
-	opts.ExtraHeaders["JMd5"] = md5String
-	opts.Parameters.Set("cphash", md5String)
-	opts.ExtraHeaders["JSize"] = strconv.FormatInt(size, 10)
-	// opts.ExtraHeaders["JCreated"] = api.Time(src.ModTime()).String()
-	opts.ExtraHeaders["JModified"] = api.Time(src.ModTime()).String()
-
-	// Parameters observed in other implementations
-	//opts.ExtraHeaders["X-Jfs-DeviceName"] = "Jotta"
-	//opts.ExtraHeaders["X-Jfs-Devicename-Base64"] = ""
-	//opts.ExtraHeaders["X-Jftp-Version"] = "2.4" this appears to be the current version
-	//opts.ExtraHeaders["jx_csid"] = ""
-	//opts.ExtraHeaders["jx_lisence"] = ""
-
-	opts.Parameters.Set("umode", "nomultipart")
-
+	// send it
+	var response api.AllocateFileResponse
 	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
-		resp, err = o.fs.srv.CallXML(&opts, nil, &result)
+		resp, err = o.fs.apiSrv.CallJSON(&opts, &request, &response)
 		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		return err
 	}
 
-	// TODO: Check returned Metadata? Timeout on big uploads?
-	return o.setMetaData(&result)
+	// If the file state is INCOMPLETE and CORRPUT, try to upload a then
+	if response.State != "COMPLETED" {
+		// how much do we still have to upload?
+		remainingBytes := size - response.ResumePos
+		opts = rest.Opts{
+			Method:        "POST",
+			RootURL:       response.UploadURL,
+			ContentLength: &remainingBytes,
+			ContentType:   "application/octet-stream",
+			Body:          in,
+			ExtraHeaders:  make(map[string]string),
+		}
+		if response.ResumePos != 0 {
+			opts.ExtraHeaders["Range"] = "bytes=" + strconv.FormatInt(response.ResumePos, 10) + "-" + strconv.FormatInt(size-1, 10)
+		}
+
+		// copy the already uploaded bytes into the trash :)
+		var result api.UploadResponse
+		_, err = io.CopyN(ioutil.Discard, in, response.ResumePos)
+		if err != nil {
+			return err
+		}
+
+		// send the remaining bytes
+		resp, err = o.fs.apiSrv.CallJSON(&opts, nil, &result)
+		if err != nil {
+			return err
+		}
+
+		// finally update the meta data
+		o.hasMetaData = true
+		o.size = int64(result.Bytes)
+		o.md5 = result.Md5
+		o.modTime = time.Unix(result.Modified/1000, 0)
+	} else {
+		// If the file state is COMPLETE we don't need to upload it because the file was allready found but we still ned to update our metadata
+		return o.readMetaData(true)
+	}
+
+	return nil
 }
 
 // Remove an object

backend/local/lchtimes.go

@@ -0,0 +1,20 @@
+// +build windows plan9
+
+package local
+
+import (
+	"time"
+)
+
+const haveLChtimes = false
+
+// lChtimes changes the access and modification times of the named
+// link, similar to the Unix utime() or utimes() functions.
+//
+// The underlying filesystem may truncate or round the values to a
+// less precise time unit.
+// If there is an error, it will be of type *PathError.
+func lChtimes(name string, atime time.Time, mtime time.Time) error {
+	// Does nothing
+	return nil
+}

backend/local/lchtimes_unix.go

@@ -0,0 +1,28 @@
+// +build !windows,!plan9
+
+package local
+
+import (
+	"os"
+	"time"
+
+	"golang.org/x/sys/unix"
+)
+
+const haveLChtimes = true
+
+// lChtimes changes the access and modification times of the named
+// link, similar to the Unix utime() or utimes() functions.
+//
+// The underlying filesystem may truncate or round the values to a
+// less precise time unit.
+// If there is an error, it will be of type *PathError.
+func lChtimes(name string, atime time.Time, mtime time.Time) error {
+	var utimes [2]unix.Timespec
+	utimes[0] = unix.NsecToTimespec(atime.UnixNano())
+	utimes[1] = unix.NsecToTimespec(mtime.UnixNano())
+	if e := unix.UtimesNanoAt(unix.AT_FDCWD, name, utimes[0:], unix.AT_SYMLINK_NOFOLLOW); e != nil {
+		return &os.PathError{Op: "lchtimes", Path: name, Err: e}
+	}
+	return nil
+}

backend/local/local.go

@@ -2,10 +2,10 @@
 package local
 
 import (
+	"bytes"
 	"fmt"
 	"io"
 	"io/ioutil"
-	"log"
 	"os"
 	"path"
 	"path/filepath"
@@ -29,6 +29,7 @@ import (
 
 // Constants
 const devUnset = 0xdeadbeefcafebabe // a device id meaning it is unset
+const linkSuffix = ".rclonelink"    // The suffix added to a translated symbolic link
 
 // Register with Fs
 func init() {
@@ -50,6 +51,13 @@ func init() {
 			NoPrefix: true,
 			ShortOpt: "L",
 			Advanced: true,
+		}, {
+			Name:     "links",
+			Help:     "Translate symlinks to/from regular files with a '" + linkSuffix + "' extension",
+			Default:  false,
+			NoPrefix: true,
+			ShortOpt: "l",
+			Advanced: true,
 		}, {
 			Name: "skip_links",
 			Help: `Don't warn about skipped symlinks.
@@ -94,12 +102,13 @@ check can be disabled with this flag.`,
 
 // Options defines the configuration for this backend
 type Options struct {
-	FollowSymlinks bool `config:"copy_links"`
-	SkipSymlinks   bool `config:"skip_links"`
-	NoUTFNorm      bool `config:"no_unicode_normalization"`
-	NoCheckUpdated bool `config:"no_check_updated"`
-	NoUNC          bool `config:"nounc"`
-	OneFileSystem  bool `config:"one_file_system"`
+	FollowSymlinks    bool `config:"copy_links"`
+	TranslateSymlinks bool `config:"links"`
+	SkipSymlinks      bool `config:"skip_links"`
+	NoUTFNorm         bool `config:"no_unicode_normalization"`
+	NoCheckUpdated    bool `config:"no_check_updated"`
+	NoUNC             bool `config:"nounc"`
+	OneFileSystem     bool `config:"one_file_system"`
 }
 
 // Fs represents a local filesystem rooted at root
@@ -121,17 +130,20 @@ type Fs struct {
 
 // Object represents a local filesystem object
 type Object struct {
-	fs      *Fs    // The Fs this object is part of
-	remote  string // The remote path - properly UTF-8 encoded - for rclone
-	path    string // The local path - may not be properly UTF-8 encoded - for OS
-	size    int64  // file metadata - always present
-	mode    os.FileMode
-	modTime time.Time
-	hashes  map[hash.Type]string // Hashes
+	fs             *Fs    // The Fs this object is part of
+	remote         string // The remote path - properly UTF-8 encoded - for rclone
+	path           string // The local path - may not be properly UTF-8 encoded - for OS
+	size           int64  // file metadata - always present
+	mode           os.FileMode
+	modTime        time.Time
+	hashes         map[hash.Type]string // Hashes
+	translatedLink bool                 // Is this object a translated link
 }
 
 // ------------------------------------------------------------
 
+var errLinksAndCopyLinks = errors.New("can't use -l/--links with -L/--copy-links")
+
 // NewFs constructs an Fs from the path
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
@@ -140,6 +152,9 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	if err != nil {
 		return nil, err
 	}
+	if opt.TranslateSymlinks && opt.FollowSymlinks {
+		return nil, errLinksAndCopyLinks
+	}
 
 	if opt.NoUTFNorm {
 		fs.Errorf(nil, "The --local-no-unicode-normalization flag is deprecated and will be removed")
@@ -167,7 +182,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	if err == nil {
 		f.dev = readDevice(fi, f.opt.OneFileSystem)
 	}
-	if err == nil && fi.Mode().IsRegular() {
+	if err == nil && f.isRegular(fi.Mode()) {
 		// It is a file, so use the parent as the root
 		f.root = filepath.Dir(f.root)
 		// return an error with an fs which points to the parent
@@ -176,6 +191,20 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	return f, nil
 }
 
+// Determine whether a file is a 'regular' file,
+// Symlinks are regular files, only if the TranslateSymlink
+// option is in-effect
+func (f *Fs) isRegular(mode os.FileMode) bool {
+	if !f.opt.TranslateSymlinks {
+		return mode.IsRegular()
+	}
+
+	// fi.Mode().IsRegular() tests that all mode bits are zero
+	// Since symlinks are accepted, test that all other bits are zero,
+	// except the symlink bit
+	return mode&os.ModeType&^os.ModeSymlink == 0
+}
+
 // Name of the remote (as passed into NewFs)
 func (f *Fs) Name() string {
 	return f.name
@@ -206,18 +235,38 @@ func (f *Fs) caseInsensitive() bool {
 	return runtime.GOOS == "windows" || runtime.GOOS == "darwin"
 }
 
+// translateLink checks whether the remote is a translated link
+// and returns a new path, removing the suffix as needed,
+// It also returns whether this is a translated link at all
+//
+// for regular files, dstPath is returned unchanged
+func translateLink(remote, dstPath string) (newDstPath string, isTranslatedLink bool) {
+	isTranslatedLink = strings.HasSuffix(remote, linkSuffix)
+	newDstPath = strings.TrimSuffix(dstPath, linkSuffix)
+	return newDstPath, isTranslatedLink
+}
+
 // newObject makes a half completed Object
 //
 // if dstPath is empty then it is made from remote
 func (f *Fs) newObject(remote, dstPath string) *Object {
+	translatedLink := false
+
 	if dstPath == "" {
 		dstPath = f.cleanPath(filepath.Join(f.root, remote))
 	}
 	remote = f.cleanRemote(remote)
+
+	if f.opt.TranslateSymlinks {
+		// Possibly receive a new name for dstPath
+		dstPath, translatedLink = translateLink(remote, dstPath)
+	}
+
 	return &Object{
-		fs:     f,
-		remote: remote,
-		path:   dstPath,
+		fs:             f,
+		remote:         remote,
+		path:           dstPath,
+		translatedLink: translatedLink,
 	}
 }
 
@@ -239,6 +288,11 @@ func (f *Fs) newObjectWithInfo(remote, dstPath string, info os.FileInfo) (fs.Obj
 			}
 			return nil, err
 		}
+		// Handle the odd case, that a symlink was specfied by name without the link suffix
+		if o.fs.opt.TranslateSymlinks && o.mode&os.ModeSymlink != 0 && !o.translatedLink {
+			return nil, fs.ErrorObjectNotFound
+		}
+
 	}
 	if o.mode.IsDir() {
 		return nil, errors.Wrapf(fs.ErrorNotAFile, "%q", remote)
@@ -262,6 +316,7 @@ func (f *Fs) NewObject(remote string) (fs.Object, error) {
 // This should return ErrDirNotFound if the directory isn't
 // found.
 func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
+
 	dir = f.dirNames.Load(dir)
 	fsDirPath := f.cleanPath(filepath.Join(f.root, dir))
 	remote := f.cleanRemote(dir)
@@ -318,6 +373,10 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 					entries = append(entries, d)
 				}
 			} else {
+				// Check whether this link should be translated
+				if f.opt.TranslateSymlinks && fi.Mode()&os.ModeSymlink != 0 {
+					newRemote += linkSuffix
+				}
 				fso, err := f.newObjectWithInfo(newRemote, newPath, fi)
 				if err != nil {
 					return nil, err
@@ -531,7 +590,7 @@ func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
 		// OK
 	} else if err != nil {
 		return nil, err
-	} else if !dstObj.mode.IsRegular() {
+	} else if !dstObj.fs.isRegular(dstObj.mode) {
 		// It isn't a file
 		return nil, errors.New("can't move file onto non-file")
 	}
@@ -653,7 +712,13 @@ func (o *Object) Hash(r hash.Type) (string, error) {
 	o.fs.objectHashesMu.Unlock()
 
 	if !o.modTime.Equal(oldtime) || oldsize != o.size || hashes == nil {
-		in, err := file.Open(o.path)
+		var in io.ReadCloser
+
+		if !o.translatedLink {
+			in, err = file.Open(o.path)
+		} else {
+			in, err = o.openTranslatedLink(0, -1)
+		}
 		if err != nil {
 			return "", errors.Wrap(err, "hash: failed to open")
 		}
@@ -684,7 +749,12 @@ func (o *Object) ModTime() time.Time {
 
 // SetModTime sets the modification time of the local fs object
 func (o *Object) SetModTime(modTime time.Time) error {
-	err := os.Chtimes(o.path, modTime, modTime)
+	var err error
+	if o.translatedLink {
+		err = lChtimes(o.path, modTime, modTime)
+	} else {
+		err = os.Chtimes(o.path, modTime, modTime)
+	}
 	if err != nil {
 		return err
 	}
@@ -702,7 +772,7 @@ func (o *Object) Storable() bool {
 		}
 	}
 	mode := o.mode
-	if mode&os.ModeSymlink != 0 {
+	if mode&os.ModeSymlink != 0 && !o.fs.opt.TranslateSymlinks {
 		if !o.fs.opt.SkipSymlinks {
 			fs.Logf(o, "Can't follow symlink without -L/--copy-links")
 		}
@@ -743,7 +813,6 @@ func (file *localOpenFile) Read(p []byte) (n int, err error) {
 	}
 
 	n, err = file.in.Read(p)
-	log.Printf("*** Read result n=%d, err=%v, len(p)=%d", n, err, len(p))
 	if n > 0 {
 		// Hash routines never return an error
 		_, _ = file.hash.Write(p[:n])
@@ -764,6 +833,16 @@ func (file *localOpenFile) Close() (err error) {
 	return err
 }
 
+// Returns a ReadCloser() object that contains the contents of a symbolic link
+func (o *Object) openTranslatedLink(offset, limit int64) (lrc io.ReadCloser, err error) {
+	// Read the link and return the destination  it as the contents of the object
+	linkdst, err := os.Readlink(o.path)
+	if err != nil {
+		return nil, err
+	}
+	return readers.NewLimitedReadCloser(ioutil.NopCloser(strings.NewReader(linkdst[offset:])), limit), nil
+}
+
 // Open an object for read
 func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
 	var offset, limit int64 = 0, -1
@@ -783,6 +862,11 @@ func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
 		}
 	}
 
+	// Handle a translated link
+	if o.translatedLink {
+		return o.openTranslatedLink(offset, limit)
+	}
+
 	fd, err := file.Open(o.path)
 	if err != nil {
 		return
@@ -814,8 +898,19 @@ func (o *Object) mkdirAll() error {
 	return os.MkdirAll(dir, 0777)
 }
 
+type nopWriterCloser struct {
+	*bytes.Buffer
+}
+
+func (nwc nopWriterCloser) Close() error {
+	// noop
+	return nil
+}
+
 // Update the object from in with modTime and size
 func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
+	var out io.WriteCloser
+
 	hashes := hash.Supported
 	for _, option := range options {
 		switch x := option.(type) {
@@ -829,15 +924,23 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 		return err
 	}
 
-	out, err := file.OpenFile(o.path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
-	if err != nil {
-		return err
-	}
-
-	// Pre-allocate the file for performance reasons
-	err = preAllocate(src.Size(), out)
-	if err != nil {
-		fs.Debugf(o, "Failed to pre-allocate: %v", err)
+	var symlinkData bytes.Buffer
+	// If the object is a regular file, create it.
+	// If it is a translated link, just read in the contents, and
+	// then create a symlink
+	if !o.translatedLink {
+		f, err := file.OpenFile(o.path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
+		if err != nil {
+			return err
+		}
+		// Pre-allocate the file for performance reasons
+		err = preAllocate(src.Size(), f)
+		if err != nil {
+			fs.Debugf(o, "Failed to pre-allocate: %v", err)
+		}
+		out = f
+	} else {
+		out = nopWriterCloser{&symlinkData}
 	}
 
 	// Calculate the hash of the object we are reading as we go along
@@ -852,6 +955,26 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 	if err == nil {
 		err = closeErr
 	}
+
+	if o.translatedLink {
+		if err == nil {
+			// Remove any current symlink or file, if one exsits
+			if _, err := os.Lstat(o.path); err == nil {
+				if removeErr := os.Remove(o.path); removeErr != nil {
+					fs.Errorf(o, "Failed to remove previous file: %v", removeErr)
+					return removeErr
+				}
+			}
+			// Use the contents for the copied object to create a symlink
+			err = os.Symlink(symlinkData.String(), o.path)
+		}
+
+		// only continue if symlink creation succeeded
+		if err != nil {
+			return err
+		}
+	}
+
 	if err != nil {
 		fs.Logf(o, "Removing partially written file on error: %v", err)
 		if removeErr := os.Remove(o.path); removeErr != nil {

backend/local/local_internal_test.go

@@ -1,10 +1,16 @@
 package local
 
 import (
+	"io/ioutil"
+	"os"
 	"path"
+	"path/filepath"
+	"runtime"
 	"testing"
 	"time"
 
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/config/configmap"
 	"github.com/ncw/rclone/fs/hash"
 	"github.com/ncw/rclone/fstest"
 	"github.com/ncw/rclone/lib/file"
@@ -42,6 +48,9 @@ func TestUpdatingCheck(t *testing.T) {
 	if err != nil {
 		t.Fatalf("failed opening file %q: %v", filePath, err)
 	}
+	defer func() {
+		require.NoError(t, fd.Close())
+	}()
 
 	fi, err := fd.Stat()
 	require.NoError(t, err)
@@ -72,3 +81,108 @@ func TestUpdatingCheck(t *testing.T) {
 	require.NoError(t, err)
 
 }
+
+func TestSymlink(t *testing.T) {
+	r := fstest.NewRun(t)
+	defer r.Finalise()
+	f := r.Flocal.(*Fs)
+	dir := f.root
+
+	// Write a file
+	modTime1 := fstest.Time("2001-02-03T04:05:10.123123123Z")
+	file1 := r.WriteFile("file.txt", "hello", modTime1)
+
+	// Write a symlink
+	modTime2 := fstest.Time("2002-02-03T04:05:10.123123123Z")
+	symlinkPath := filepath.Join(dir, "symlink.txt")
+	require.NoError(t, os.Symlink("file.txt", symlinkPath))
+	require.NoError(t, lChtimes(symlinkPath, modTime2, modTime2))
+
+	// Object viewed as symlink
+	file2 := fstest.NewItem("symlink.txt"+linkSuffix, "file.txt", modTime2)
+	if runtime.GOOS == "windows" {
+		file2.Size = 0 // symlinks are 0 length under Windows
+	}
+
+	// Object viewed as destination
+	file2d := fstest.NewItem("symlink.txt", "hello", modTime1)
+
+	// Check with no symlink flags
+	fstest.CheckItems(t, r.Flocal, file1)
+	fstest.CheckItems(t, r.Fremote)
+
+	// Set fs into "-L" mode
+	f.opt.FollowSymlinks = true
+	f.opt.TranslateSymlinks = false
+	f.lstat = os.Stat
+
+	fstest.CheckItems(t, r.Flocal, file1, file2d)
+	fstest.CheckItems(t, r.Fremote)
+
+	// Set fs into "-l" mode
+	f.opt.FollowSymlinks = false
+	f.opt.TranslateSymlinks = true
+	f.lstat = os.Lstat
+
+	fstest.CheckListingWithPrecision(t, r.Flocal, []fstest.Item{file1, file2}, nil, fs.ModTimeNotSupported)
+	if haveLChtimes {
+		fstest.CheckItems(t, r.Flocal, file1, file2)
+	}
+
+	// Create a symlink
+	modTime3 := fstest.Time("2002-03-03T04:05:10.123123123Z")
+	file3 := r.WriteObjectTo(r.Flocal, "symlink2.txt"+linkSuffix, "file.txt", modTime3, false)
+	if runtime.GOOS == "windows" {
+		file3.Size = 0 // symlinks are 0 length under Windows
+	}
+	fstest.CheckListingWithPrecision(t, r.Flocal, []fstest.Item{file1, file2, file3}, nil, fs.ModTimeNotSupported)
+	if haveLChtimes {
+		fstest.CheckItems(t, r.Flocal, file1, file2, file3)
+	}
+
+	// Check it got the correct contents
+	symlinkPath = filepath.Join(dir, "symlink2.txt")
+	fi, err := os.Lstat(symlinkPath)
+	require.NoError(t, err)
+	assert.False(t, fi.Mode().IsRegular())
+	linkText, err := os.Readlink(symlinkPath)
+	require.NoError(t, err)
+	assert.Equal(t, "file.txt", linkText)
+
+	// Check that NewObject gets the correct object
+	o, err := r.Flocal.NewObject("symlink2.txt" + linkSuffix)
+	require.NoError(t, err)
+	assert.Equal(t, "symlink2.txt"+linkSuffix, o.Remote())
+	if runtime.GOOS != "windows" {
+		assert.Equal(t, int64(8), o.Size())
+	}
+
+	// Check that NewObject doesn't see the non suffixed version
+	_, err = r.Flocal.NewObject("symlink2.txt")
+	require.Equal(t, fs.ErrorObjectNotFound, err)
+
+	// Check reading the object
+	in, err := o.Open()
+	require.NoError(t, err)
+	contents, err := ioutil.ReadAll(in)
+	require.NoError(t, err)
+	require.Equal(t, "file.txt", string(contents))
+	require.NoError(t, in.Close())
+
+	// Check reading the object with range
+	in, err = o.Open(&fs.RangeOption{Start: 2, End: 5})
+	require.NoError(t, err)
+	contents, err = ioutil.ReadAll(in)
+	require.NoError(t, err)
+	require.Equal(t, "file.txt"[2:5+1], string(contents))
+	require.NoError(t, in.Close())
+}
+
+func TestSymlinkError(t *testing.T) {
+	m := configmap.Simple{
+		"links":      "true",
+		"copy_links": "true",
+	}
+	_, err := NewFs("local", "/", m)
+	assert.Equal(t, errLinksAndCopyLinks, err)
+}

backend/onedrive/onedrive.go

@@ -492,11 +492,11 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 
 	// Get rootID
 	rootInfo, _, err := f.readMetaDataForPath("")
-	if err != nil || rootInfo.ID == "" {
+	if err != nil || rootInfo.GetID() == "" {
 		return nil, errors.Wrap(err, "failed to get root")
 	}
 
-	f.dirCache = dircache.New(root, rootInfo.ID, f)
+	f.dirCache = dircache.New(root, rootInfo.GetID(), f)
 
 	// Find the current root
 	err = f.dirCache.FindRoot(false)
@@ -1560,10 +1560,6 @@ func (o *Object) uploadSinglepart(in io.Reader, size int64, modTime time.Time) (
 		}
 	}
 
-	if size == 0 {
-		opts.Body = nil
-	}
-
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.CallJSON(&opts, nil, &info)
 		if apiErr, ok := err.(*api.Error); ok {

backend/qingstor/qingstor.go

@@ -101,10 +101,13 @@ enough memory, then increasing this will speed up the transfers.`,
 This is the number of chunks of the same file that are uploaded
 concurrently.
 
+NB if you set this to > 1 then the checksums of multpart uploads
+become corrupted (the uploads themselves are not corrupted though).
+
 If you are uploading small numbers of large file over high speed link
 and these uploads do not fully utilize your bandwidth, then increasing
 this may help to speed up the transfers.`,
-			Default:  4,
+			Default:  1,
 			Advanced: true,
 		}},
 	})

backend/s3/s3.go

@@ -833,7 +833,7 @@ var retryErrorCodes = []int{
 //S3 is pretty resilient, and the built in retry handling is probably sufficient
 // as it should notice closed connections and timeouts which are the most likely
 // sort of failure modes
-func shouldRetry(err error) (bool, error) {
+func (f *Fs) shouldRetry(err error) (bool, error) {
 	// If this is an awserr object, try and extract more useful information to determine if we should retry
 	if awsError, ok := err.(awserr.Error); ok {
 		// Simple case, check the original embedded error in case it's generically retriable
@@ -842,6 +842,15 @@ func shouldRetry(err error) (bool, error) {
 		}
 		// Failing that, if it's a RequestFailure it's probably got an http status code we can check
 		if reqErr, ok := err.(awserr.RequestFailure); ok {
+			// 301 if wrong region for bucket
+			if reqErr.StatusCode() == http.StatusMovedPermanently {
+				urfbErr := f.updateRegionForBucket()
+				if urfbErr != nil {
+					fs.Errorf(f, "Failed to update region for bucket: %v", urfbErr)
+					return false, err
+				}
+				return true, err
+			}
 			for _, e := range retryErrorCodes {
 				if reqErr.StatusCode() == e {
 					return true, err
@@ -930,12 +939,17 @@ func s3Connection(opt *Options) (*s3.S3, *session.Session, error) {
 		opt.ForcePathStyle = false
 	}
 	awsConfig := aws.NewConfig().
-		WithRegion(opt.Region).
 		WithMaxRetries(maxRetries).
 		WithCredentials(cred).
-		WithEndpoint(opt.Endpoint).
 		WithHTTPClient(fshttp.NewClient(fs.Config)).
 		WithS3ForcePathStyle(opt.ForcePathStyle)
+	if opt.Region != "" {
+		awsConfig.WithRegion(opt.Region)
+	}
+	if opt.Endpoint != "" {
+		awsConfig.WithEndpoint(opt.Endpoint)
+	}
+
 	// awsConfig.WithLogLevel(aws.LogDebugWithSigning)
 	awsSessionOpts := session.Options{
 		Config: *awsConfig,
@@ -1052,7 +1066,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		}
 		err = f.pacer.Call(func() (bool, error) {
 			_, err = f.c.HeadObject(&req)
-			return shouldRetry(err)
+			return f.shouldRetry(err)
 		})
 		if err == nil {
 			f.root = path.Dir(directory)
@@ -1102,6 +1116,51 @@ func (f *Fs) NewObject(remote string) (fs.Object, error) {
 	return f.newObjectWithInfo(remote, nil)
 }
 
+// Gets the bucket location
+func (f *Fs) getBucketLocation() (string, error) {
+	req := s3.GetBucketLocationInput{
+		Bucket: &f.bucket,
+	}
+	var resp *s3.GetBucketLocationOutput
+	var err error
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err = f.c.GetBucketLocation(&req)
+		return f.shouldRetry(err)
+	})
+	if err != nil {
+		return "", err
+	}
+	return s3.NormalizeBucketLocation(aws.StringValue(resp.LocationConstraint)), nil
+}
+
+// Updates the region for the bucket by reading the region from the
+// bucket then updating the session.
+func (f *Fs) updateRegionForBucket() error {
+	region, err := f.getBucketLocation()
+	if err != nil {
+		return errors.Wrap(err, "reading bucket location failed")
+	}
+	if aws.StringValue(f.c.Config.Endpoint) != "" {
+		return errors.Errorf("can't set region to %q as endpoint is set", region)
+	}
+	if aws.StringValue(f.c.Config.Region) == region {
+		return errors.Errorf("region is already %q - not updating", region)
+	}
+
+	// Make a new session with the new region
+	oldRegion := f.opt.Region
+	f.opt.Region = region
+	c, ses, err := s3Connection(&f.opt)
+	if err != nil {
+		return errors.Wrap(err, "creating new session failed")
+	}
+	f.c = c
+	f.ses = ses
+
+	fs.Logf(f, "Switched region to %q from %q", region, oldRegion)
+	return nil
+}
+
 // listFn is called from list to handle an object.
 type listFn func(remote string, object *s3.Object, isDirectory bool) error
 
@@ -1134,7 +1193,7 @@ func (f *Fs) list(dir string, recurse bool, fn listFn) error {
 		var err error
 		err = f.pacer.Call(func() (bool, error) {
 			resp, err = f.c.ListObjects(&req)
-			return shouldRetry(err)
+			return f.shouldRetry(err)
 		})
 		if err != nil {
 			if awsErr, ok := err.(awserr.RequestFailure); ok {
@@ -1263,7 +1322,7 @@ func (f *Fs) listBuckets(dir string) (entries fs.DirEntries, err error) {
 	var resp *s3.ListBucketsOutput
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.c.ListBuckets(&req)
-		return shouldRetry(err)
+		return f.shouldRetry(err)
 	})
 	if err != nil {
 		return nil, err
@@ -1351,7 +1410,7 @@ func (f *Fs) dirExists() (bool, error) {
 	}
 	err := f.pacer.Call(func() (bool, error) {
 		_, err := f.c.HeadBucket(&req)
-		return shouldRetry(err)
+		return f.shouldRetry(err)
 	})
 	if err == nil {
 		return true, nil
@@ -1391,7 +1450,7 @@ func (f *Fs) Mkdir(dir string) error {
 	}
 	err := f.pacer.Call(func() (bool, error) {
 		_, err := f.c.CreateBucket(&req)
-		return shouldRetry(err)
+		return f.shouldRetry(err)
 	})
 	if err, ok := err.(awserr.Error); ok {
 		if err.Code() == "BucketAlreadyOwnedByYou" {
@@ -1420,7 +1479,7 @@ func (f *Fs) Rmdir(dir string) error {
 	}
 	err := f.pacer.Call(func() (bool, error) {
 		_, err := f.c.DeleteBucket(&req)
-		return shouldRetry(err)
+		return f.shouldRetry(err)
 	})
 	if err == nil {
 		f.bucketOK = false
@@ -1481,7 +1540,7 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
 	}
 	err = f.pacer.Call(func() (bool, error) {
 		_, err = f.c.CopyObject(&req)
-		return shouldRetry(err)
+		return f.shouldRetry(err)
 	})
 	if err != nil {
 		return nil, err
@@ -1563,7 +1622,7 @@ func (o *Object) readMetaData() (err error) {
 	err = o.fs.pacer.Call(func() (bool, error) {
 		var err error
 		resp, err = o.fs.c.HeadObject(&req)
-		return shouldRetry(err)
+		return o.fs.shouldRetry(err)
 	})
 	if err != nil {
 		if awsErr, ok := err.(awserr.RequestFailure); ok {
@@ -1659,7 +1718,7 @@ func (o *Object) SetModTime(modTime time.Time) error {
 	}
 	err = o.fs.pacer.Call(func() (bool, error) {
 		_, err := o.fs.c.CopyObject(&req)
-		return shouldRetry(err)
+		return o.fs.shouldRetry(err)
 	})
 	return err
 }
@@ -1691,7 +1750,7 @@ func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
 	err = o.fs.pacer.Call(func() (bool, error) {
 		var err error
 		resp, err = o.fs.c.GetObject(&req)
-		return shouldRetry(err)
+		return o.fs.shouldRetry(err)
 	})
 	if err, ok := err.(awserr.RequestFailure); ok {
 		if err.Code() == "InvalidObjectState" {
@@ -1782,7 +1841,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 		}
 		err = o.fs.pacer.CallNoRetry(func() (bool, error) {
 			_, err = uploader.Upload(&req)
-			return shouldRetry(err)
+			return o.fs.shouldRetry(err)
 		})
 		if err != nil {
 			return err
@@ -1838,11 +1897,11 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 		err = o.fs.pacer.CallNoRetry(func() (bool, error) {
 			resp, err := o.fs.srv.Do(httpReq)
 			if err != nil {
-				return shouldRetry(err)
+				return o.fs.shouldRetry(err)
 			}
 			body, err := rest.ReadBody(resp)
 			if err != nil {
-				return shouldRetry(err)
+				return o.fs.shouldRetry(err)
 			}
 			if resp.StatusCode >= 200 && resp.StatusCode < 299 {
 				return false, nil
@@ -1870,7 +1929,7 @@ func (o *Object) Remove() error {
 	}
 	err := o.fs.pacer.Call(func() (bool, error) {
 		_, err := o.fs.c.DeleteObject(&req)
-		return shouldRetry(err)
+		return o.fs.shouldRetry(err)
 	})
 	return err
 }

backend/sftp/sftp.go

@@ -565,9 +565,13 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 		// If file is a symlink (not a regular file is the best cross platform test we can do), do a stat to
 		// pick up the size and type of the destination, instead of the size and type of the symlink.
 		if !info.Mode().IsRegular() {
+			oldInfo := info
 			info, err = f.stat(remote)
 			if err != nil {
-				return nil, errors.Wrap(err, "stat of non-regular file/dir failed")
+				if !os.IsNotExist(err) {
+					fs.Errorf(remote, "stat of non-regular file/dir failed: %v", err)
+				}
+				info = oldInfo
 			}
 		}
 		if info.IsDir() {

backend/swift/swift.go

@@ -130,6 +130,15 @@ func init() {
 		}, {
 			Name: "auth_token",
 			Help: "Auth Token from alternate authentication - optional (OS_AUTH_TOKEN)",
+		}, {
+			Name: "application_credential_id",
+			Help: "Application Credential ID (OS_APPLICATION_CREDENTIAL_ID)",
+		}, {
+			Name: "application_credential_name",
+			Help: "Application Credential Name (OS_APPLICATION_CREDENTIAL_NAME)",
+		}, {
+			Name: "application_credential_secret",
+			Help: "Application Credential Secret (OS_APPLICATION_CREDENTIAL_SECRET)",
 		}, {
 			Name:    "auth_version",
 			Help:    "AuthVersion - optional - set to (1,2,3) if your auth URL has no version (ST_AUTH_VERSION)",
@@ -173,23 +182,26 @@ provider.`,
 
 // Options defines the configuration for this backend
 type Options struct {
-	EnvAuth       bool          `config:"env_auth"`
-	User          string        `config:"user"`
-	Key           string        `config:"key"`
-	Auth          string        `config:"auth"`
-	UserID        string        `config:"user_id"`
-	Domain        string        `config:"domain"`
-	Tenant        string        `config:"tenant"`
-	TenantID      string        `config:"tenant_id"`
-	TenantDomain  string        `config:"tenant_domain"`
-	Region        string        `config:"region"`
-	StorageURL    string        `config:"storage_url"`
-	AuthToken     string        `config:"auth_token"`
-	AuthVersion   int           `config:"auth_version"`
-	StoragePolicy string        `config:"storage_policy"`
-	EndpointType  string        `config:"endpoint_type"`
-	ChunkSize     fs.SizeSuffix `config:"chunk_size"`
-	NoChunk       bool          `config:"no_chunk"`
+	EnvAuth                     bool          `config:"env_auth"`
+	User                        string        `config:"user"`
+	Key                         string        `config:"key"`
+	Auth                        string        `config:"auth"`
+	UserID                      string        `config:"user_id"`
+	Domain                      string        `config:"domain"`
+	Tenant                      string        `config:"tenant"`
+	TenantID                    string        `config:"tenant_id"`
+	TenantDomain                string        `config:"tenant_domain"`
+	Region                      string        `config:"region"`
+	StorageURL                  string        `config:"storage_url"`
+	AuthToken                   string        `config:"auth_token"`
+	AuthVersion                 int           `config:"auth_version"`
+	ApplicationCredentialId     string        `config:"application_credential_id"`
+	ApplicationCredentialName   string        `config:"application_credential_name"`
+	ApplicationCredentialSecret string        `config:"application_credential_secret"`
+	StoragePolicy               string        `config:"storage_policy"`
+	EndpointType                string        `config:"endpoint_type"`
+	ChunkSize                   fs.SizeSuffix `config:"chunk_size"`
+	NoChunk                     bool          `config:"no_chunk"`
 }
 
 // Fs represents a remote swift server
@@ -293,22 +305,25 @@ func parsePath(path string) (container, directory string, err error) {
 func swiftConnection(opt *Options, name string) (*swift.Connection, error) {
 	c := &swift.Connection{
 		// Keep these in the same order as the Config for ease of checking
-		UserName:       opt.User,
-		ApiKey:         opt.Key,
-		AuthUrl:        opt.Auth,
-		UserId:         opt.UserID,
-		Domain:         opt.Domain,
-		Tenant:         opt.Tenant,
-		TenantId:       opt.TenantID,
-		TenantDomain:   opt.TenantDomain,
-		Region:         opt.Region,
-		StorageUrl:     opt.StorageURL,
-		AuthToken:      opt.AuthToken,
-		AuthVersion:    opt.AuthVersion,
-		EndpointType:   swift.EndpointType(opt.EndpointType),
-		ConnectTimeout: 10 * fs.Config.ConnectTimeout, // Use the timeouts in the transport
-		Timeout:        10 * fs.Config.Timeout,        // Use the timeouts in the transport
-		Transport:      fshttp.NewTransport(fs.Config),
+		UserName:                    opt.User,
+		ApiKey:                      opt.Key,
+		AuthUrl:                     opt.Auth,
+		UserId:                      opt.UserID,
+		Domain:                      opt.Domain,
+		Tenant:                      opt.Tenant,
+		TenantId:                    opt.TenantID,
+		TenantDomain:                opt.TenantDomain,
+		Region:                      opt.Region,
+		StorageUrl:                  opt.StorageURL,
+		AuthToken:                   opt.AuthToken,
+		AuthVersion:                 opt.AuthVersion,
+		ApplicationCredentialId:     opt.ApplicationCredentialId,
+		ApplicationCredentialName:   opt.ApplicationCredentialName,
+		ApplicationCredentialSecret: opt.ApplicationCredentialSecret,
+		EndpointType:                swift.EndpointType(opt.EndpointType),
+		ConnectTimeout:              10 * fs.Config.ConnectTimeout, // Use the timeouts in the transport
+		Timeout:                     10 * fs.Config.Timeout,        // Use the timeouts in the transport
+		Transport:                   fshttp.NewTransport(fs.Config),
 	}
 	if opt.EnvAuth {
 		err := c.ApplyEnvironment()
@@ -318,11 +333,13 @@ func swiftConnection(opt *Options, name string) (*swift.Connection, error) {
 	}
 	StorageUrl, AuthToken := c.StorageUrl, c.AuthToken // nolint
 	if !c.Authenticated() {
-		if c.UserName == "" && c.UserId == "" {
-			return nil, errors.New("user name or user id not found for authentication (and no storage_url+auth_token is provided)")
-		}
-		if c.ApiKey == "" {
-			return nil, errors.New("key not found")
+		if (c.ApplicationCredentialId != "" || c.ApplicationCredentialName != "") && c.ApplicationCredentialSecret == "" {
+			if c.UserName == "" && c.UserId == "" {
+				return nil, errors.New("user name or user id not found for authentication (and no storage_url+auth_token is provided)")
+			}
+			if c.ApiKey == "" {
+				return nil, errors.New("key not found")
+			}
 		}
 		if c.AuthUrl == "" {
 			return nil, errors.New("auth not found")

backend/union/union.go

@@ -376,6 +376,11 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	}).Fill(f)
 	features = features.Mask(f.wr) // mask the features just on the writable fs
 
+	// Really need the union of all remotes for these, so
+	// re-instate and calculate separately.
+	features.ChangeNotify = f.ChangeNotify
+	features.DirCacheFlush = f.DirCacheFlush
+
 	// FIXME maybe should be masking the bools here?
 
 	// Clear ChangeNotify and DirCacheFlush if all are nil

backend/webdav/api/types.go

@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/hash"
 )
 
 const (
@@ -65,11 +66,12 @@ type Response struct {
 // Note that status collects all the status values for which we just
 // check the first is OK.
 type Prop struct {
-	Status   []string  `xml:"DAV: status"`
-	Name     string    `xml:"DAV: prop>displayname,omitempty"`
-	Type     *xml.Name `xml:"DAV: prop>resourcetype>collection,omitempty"`
-	Size     int64     `xml:"DAV: prop>getcontentlength,omitempty"`
-	Modified Time      `xml:"DAV: prop>getlastmodified,omitempty"`
+	Status    []string  `xml:"DAV: status"`
+	Name      string    `xml:"DAV: prop>displayname,omitempty"`
+	Type      *xml.Name `xml:"DAV: prop>resourcetype>collection,omitempty"`
+	Size      int64     `xml:"DAV: prop>getcontentlength,omitempty"`
+	Modified  Time      `xml:"DAV: prop>getlastmodified,omitempty"`
+	Checksums []string  `xml:"prop>checksums>checksum,omitempty"`
 }
 
 // Parse a status of the form "HTTP/1.1 200 OK" or "HTTP/1.1 200"
@@ -95,6 +97,26 @@ func (p *Prop) StatusOK() bool {
 	return false
 }
 
+// Hashes returns a map of all checksums - may be nil
+func (p *Prop) Hashes() (hashes map[hash.Type]string) {
+	if len(p.Checksums) == 0 {
+		return nil
+	}
+	hashes = make(map[hash.Type]string)
+	for _, checksums := range p.Checksums {
+		checksums = strings.ToLower(checksums)
+		for _, checksum := range strings.Split(checksums, " ") {
+			switch {
+			case strings.HasPrefix(checksum, "sha1:"):
+				hashes[hash.SHA1] = checksum[5:]
+			case strings.HasPrefix(checksum, "md5:"):
+				hashes[hash.MD5] = checksum[4:]
+			}
+		}
+	}
+	return hashes
+}
+
 // PropValue is a tagged name and value
 type PropValue struct {
 	XMLName xml.Name `xml:""`
@@ -187,3 +209,22 @@ func (t *Time) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
 	}
 	return err
 }
+
+// Quota is used to read the bytes used and available
+//
+// <d:multistatus xmlns:d="DAV:" xmlns:s="http://sabredav.org/ns" xmlns:oc="http://owncloud.org/ns" xmlns:nc="http://nextcloud.org/ns">
+//  <d:response>
+//   <d:href>/remote.php/webdav/</d:href>
+//   <d:propstat>
+//    <d:prop>
+//     <d:quota-available-bytes>-3</d:quota-available-bytes>
+//     <d:quota-used-bytes>376461895</d:quota-used-bytes>
+//    </d:prop>
+//    <d:status>HTTP/1.1 200 OK</d:status>
+//   </d:propstat>
+//  </d:response>
+// </d:multistatus>
+type Quota struct {
+	Available int64 `xml:"DAV: response>propstat>prop>quota-available-bytes"`
+	Used      int64 `xml:"DAV: response>propstat>prop>quota-used-bytes"`
+}

backend/webdav/webdav.go

@@ -2,23 +2,13 @@
 // object storage system.
 package webdav
 
-// Owncloud: Getting Oc-Checksum:
-// SHA1:f572d396fae9206628714fb2ce00f72e94f2258f on HEAD but not on
-// nextcloud?
-
-// docs for file webdav
-// https://docs.nextcloud.com/server/12/developer_manual/client_apis/WebDAV/index.html
-
-// indicates checksums can be set as metadata here
-// https://github.com/nextcloud/server/issues/6129
-// owncloud seems to have checksums as metadata though - can read them
-
 // SetModTime might be possible
 // https://stackoverflow.com/questions/3579608/webdav-can-a-client-modify-the-mtime-of-a-file
 // ...support for a PROPSET to lastmodified (mind the missing get) which does the utime() call might be an option.
 // For example the ownCloud WebDAV server does it that way.
 
 import (
+	"bytes"
 	"encoding/xml"
 	"fmt"
 	"io"
@@ -116,6 +106,7 @@ type Fs struct {
 	canStream          bool          // set if can stream
 	useOCMtime         bool          // set if can use X-OC-Mtime
 	retryWithZeroDepth bool          // some vendors (sharepoint) won't list files when Depth is 1 (our default)
+	hasChecksums       bool          // set if can use owncloud style checksums
 }
 
 // Object describes a webdav object
@@ -127,7 +118,8 @@ type Object struct {
 	hasMetaData bool      // whether info below has been set
 	size        int64     // size of the object
 	modTime     time.Time // modification time of the object
-	sha1        string    // SHA-1 of the object content
+	sha1        string    // SHA-1 of the object content if known
+	md5         string    // MD5 of the object content if known
 }
 
 // ------------------------------------------------------------
@@ -194,6 +186,9 @@ func (f *Fs) readMetaDataForPath(path string, depth string) (info *api.Prop, err
 		},
 		NoRedirect: true,
 	}
+	if f.hasChecksums {
+		opts.Body = bytes.NewBuffer(owncloudProps)
+	}
 	var result api.Multistatus
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
@@ -357,9 +352,11 @@ func (f *Fs) setQuirks(vendor string) error {
 		f.canStream = true
 		f.precision = time.Second
 		f.useOCMtime = true
+		f.hasChecksums = true
 	case "nextcloud":
 		f.precision = time.Second
 		f.useOCMtime = true
+		f.hasChecksums = true
 	case "sharepoint":
 		// To mount sharepoint, two Cookies are required
 		// They have to be set instead of BasicAuth
@@ -426,6 +423,22 @@ func (f *Fs) NewObject(remote string) (fs.Object, error) {
 	return f.newObjectWithInfo(remote, nil)
 }
 
+// Read the normal props, plus the checksums
+//
+// <oc:checksums><oc:checksum>SHA1:f572d396fae9206628714fb2ce00f72e94f2258f MD5:b1946ac92492d2347c6235b4d2611184 ADLER32:084b021f</oc:checksum></oc:checksums>
+var owncloudProps = []byte(`<?xml version="1.0"?>
+<d:propfind  xmlns:d="DAV:" xmlns:oc="http://owncloud.org/ns" xmlns:nc="http://nextcloud.org/ns">
+ <d:prop>
+  <d:displayname />
+  <d:getlastmodified />
+  <d:getcontentlength />
+  <d:resourcetype />
+  <d:getcontenttype />
+  <oc:checksums />
+ </d:prop>
+</d:propfind>
+`)
+
 // list the objects into the function supplied
 //
 // If directories is set it only sends directories
@@ -445,6 +458,9 @@ func (f *Fs) listAll(dir string, directoriesOnly bool, filesOnly bool, depth str
 			"Depth": depth,
 		},
 	}
+	if f.hasChecksums {
+		opts.Body = bytes.NewBuffer(owncloudProps)
+	}
 	var result api.Multistatus
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
@@ -847,9 +863,52 @@ func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error {
 
 // Hashes returns the supported hash sets.
 func (f *Fs) Hashes() hash.Set {
+	if f.hasChecksums {
+		return hash.NewHashSet(hash.MD5, hash.SHA1)
+	}
 	return hash.Set(hash.None)
 }
 
+// About gets quota information
+func (f *Fs) About() (*fs.Usage, error) {
+	opts := rest.Opts{
+		Method: "PROPFIND",
+		Path:   "",
+		ExtraHeaders: map[string]string{
+			"Depth": "0",
+		},
+	}
+	opts.Body = bytes.NewBuffer([]byte(`<?xml version="1.0" ?>
+<D:propfind xmlns:D="DAV:">
+ <D:prop>
+  <D:quota-available-bytes/>
+  <D:quota-used-bytes/>
+ </D:prop>
+</D:propfind>
+`))
+	var q = api.Quota{
+		Available: -1,
+		Used:      -1,
+	}
+	var resp *http.Response
+	var err error
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err = f.srv.CallXML(&opts, nil, &q)
+		return shouldRetry(resp, err)
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "about call failed")
+	}
+	usage := &fs.Usage{}
+	if q.Available >= 0 && q.Used >= 0 {
+		usage.Total = fs.NewUsageValue(q.Available + q.Used)
+	}
+	if q.Used >= 0 {
+		usage.Used = fs.NewUsageValue(q.Used)
+	}
+	return usage, nil
+}
+
 // ------------------------------------------------------------
 
 // Fs returns the parent Fs
@@ -870,12 +929,17 @@ func (o *Object) Remote() string {
 	return o.remote
 }
 
-// Hash returns the SHA-1 of an object returning a lowercase hex string
+// Hash returns the SHA1 or MD5 of an object returning a lowercase hex string
 func (o *Object) Hash(t hash.Type) (string, error) {
-	if t != hash.SHA1 {
-		return "", hash.ErrUnsupported
+	if o.fs.hasChecksums {
+		switch t {
+		case hash.SHA1:
+			return o.sha1, nil
+		case hash.MD5:
+			return o.md5, nil
+		}
 	}
-	return o.sha1, nil
+	return "", hash.ErrUnsupported
 }
 
 // Size returns the size of an object in bytes
@@ -893,6 +957,11 @@ func (o *Object) setMetaData(info *api.Prop) (err error) {
 	o.hasMetaData = true
 	o.size = info.Size
 	o.modTime = time.Time(info.Modified)
+	if o.fs.hasChecksums {
+		hashes := info.Hashes()
+		o.sha1 = hashes[hash.SHA1]
+		o.md5 = hashes[hash.MD5]
+	}
 	return nil
 }
 
@@ -972,9 +1041,21 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 		ContentLength: &size, // FIXME this isn't necessary with owncloud - See https://github.com/nextcloud/nextcloud-snap/issues/365
 		ContentType:   fs.MimeType(src),
 	}
-	if o.fs.useOCMtime {
-		opts.ExtraHeaders = map[string]string{
-			"X-OC-Mtime": fmt.Sprintf("%f", float64(src.ModTime().UnixNano())/1E9),
+	if o.fs.useOCMtime || o.fs.hasChecksums {
+		opts.ExtraHeaders = map[string]string{}
+		if o.fs.useOCMtime {
+			opts.ExtraHeaders["X-OC-Mtime"] = fmt.Sprintf("%f", float64(src.ModTime().UnixNano())/1E9)
+		}
+		if o.fs.hasChecksums {
+			// Set an upload checksum - prefer SHA1
+			//
+			// This is used as an upload integrity test. If we set
+			// only SHA1 here, owncloud will calculate the MD5 too.
+			if sha1, _ := src.Hash(hash.SHA1); sha1 != "" {
+				opts.ExtraHeaders["OC-Checksum"] = "SHA1:" + sha1
+			} else if md5, _ := src.Hash(hash.MD5); md5 != "" {
+				opts.ExtraHeaders["OC-Checksum"] = "MD5:" + md5
+			}
 		}
 	}
 	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
@@ -1018,5 +1099,6 @@ var (
 	_ fs.Copier      = (*Fs)(nil)
 	_ fs.Mover       = (*Fs)(nil)
 	_ fs.DirMover    = (*Fs)(nil)
+	_ fs.Abouter     = (*Fs)(nil)
 	_ fs.Object      = (*Object)(nil)
 )

bin/tidy-beta

@@ -3,7 +3,7 @@
 
 version="$1"
 if [ "$version" = "" ]; then
-    echo "Syntax: $0 <version> [delete]"
+    echo "Syntax: $0 <version, eg v1.42> [delete]"
     exit 1
 fi
 dry_run="--dry-run"
@@ -14,4 +14,4 @@ else
     echo "Use '$0 $version delete' to actually delete files"
 fi
 
-rclone ${dry_run} --fast-list -P --checkers 16 --transfers 16 delete --include "**/${version}**" memstore:beta-rclone-org
+rclone ${dry_run} -P --fast-list --checkers 16 --transfers 16 delete --include "**${version}**" memstore:beta-rclone-org

cmd/mountlib/mount.go

@@ -326,7 +326,11 @@ func ClipBlocks(b *uint64) {
 	var max uint64
 	switch runtime.GOOS {
 	case "windows":
-		max = (1 << 43) - 1
+		if runtime.GOARCH == "386" {
+			max = (1 << 32) - 1
+		} else {
+			max = (1 << 43) - 1
+		}
 	case "darwin":
 		// OSX FUSE only supports 32 bit number of blocks
 		// https://github.com/osxfuse/osxfuse/issues/396

cmd/rcd/rcd.go

@@ -17,7 +17,7 @@ var commandDefintion = &cobra.Command{
 	Use:   "rcd <path to files to serve>*",
 	Short: `Run rclone listening to remote control commands only.`,
 	Long: `
-This runs rclone so that it only listents to remote control commands.
+This runs rclone so that it only listens to remote control commands.
 
 This is useful if you are controlling rclone via the rc API.
 

docs/content/about.md

@@ -44,6 +44,7 @@ Rclone is a command line program to sync files and directories to and from:
 * {{< provider name="put.io" home="https://put.io/" config="/webdav/#put-io" >}}
 * {{< provider name="QingStor" home="https://www.qingcloud.com/products/storage" config="/qingstor/" >}}
 * {{< provider name="Rackspace Cloud Files" home="https://www.rackspace.com/cloud/files" config="/swift/" >}}
+* {{< provider name="Scaleway" home="https://www.scaleway.com/object-storage/" config="/s3/#scaleway" >}}
 * {{< provider name="SFTP" home="https://en.wikipedia.org/wiki/SFTP" config="/sftp/" >}}
 * {{< provider name="Wasabi" home="https://wasabi.com/" config="/s3/#wasabi" >}}
 * {{< provider name="WebDAV" home="https://en.wikipedia.org/wiki/WebDAV" config="/webdav/" >}}

docs/content/authors.md

@@ -228,3 +228,8 @@ Contributors
   * nicolov <nicolov@users.noreply.github.com>
   * Dario Guzik <dario@guzik.com.ar>
   * qip <qip@users.noreply.github.com>
+  * yair@unicorn <yair@unicorn>
+  * Matt Robinson <brimstone@the.narro.ws>
+  * kayrus <kay.diam@gmail.com>
+  * Rémy Léone <remy.leone@gmail.com>
+  * Wojciech Smigielski <wojciech.hieronim.smigielski@gmail.com>

docs/content/faq.md

@@ -97,8 +97,6 @@ In general the variables are called `http_proxy` (for services reached
 over `http`) and `https_proxy` (for services reached over `https`).  Most
 public services will be using `https`, but you may wish to set both.
 
-If you ever use `FTP` then you would need to set `ftp_proxy`.
-
 The content of the variable is `protocol://server:port`.  The protocol
 value is the one used to talk to the proxy server, itself, and is commonly
 either `http` or `socks5`.
@@ -122,6 +120,8 @@ e.g.
     export no_proxy=localhost,127.0.0.0/8,my.host.name
     export NO_PROXY=$no_proxy
 
+Note that the ftp backend does not support `ftp_proxy` yet.
+
 ### Rclone gives x509: failed to load system roots and no roots provided error ###
 
 This means that `rclone` can't file the SSL root certificates.  Likely

docs/content/ftp.md

@@ -175,3 +175,6 @@ Note that `--timeout` isn't supported (but `--contimeout` is).
 Note that `--bind` isn't supported.
 
 FTP could support server side move but doesn't yet.
+
+Note that the ftp backend does not support the `ftp_proxy` environment
+variable yet.

docs/content/local.md

@@ -81,7 +81,8 @@ Normally rclone will ignore symlinks or junction points (which behave
 like symlinks under Windows).
 
 If you supply `--copy-links` or `-L` then rclone will follow the
-symlink and copy the pointed to file or directory.
+symlink and copy the pointed to file or directory.  Note that this
+flag is incompatible with `-links` / `-l`.
 
 This flag applies to all commands.
 
@@ -116,6 +117,75 @@ $ rclone -L ls /tmp/a
         6 b/one
 ```
 
+#### --links, -l 
+
+Normally rclone will ignore symlinks or junction points (which behave
+like symlinks under Windows).
+
+If you supply this flag then rclone will copy symbolic links from the local storage,
+and store them as text files, with a '.rclonelink' suffix in the remote storage.
+
+The text file will contain the target of the symbolic link (see example).
+
+This flag applies to all commands.
+
+For example, supposing you have a directory structure like this
+
+```
+$ tree /tmp/a
+/tmp/a
+├── file1 -> ./file4
+└── file2 -> /home/user/file3
+```
+
+Copying the entire directory with '-l'
+
+```
+$ rclone copyto -l /tmp/a/file1 remote:/tmp/a/
+```
+
+The remote files are created with a '.rclonelink' suffix
+
+```
+$ rclone ls remote:/tmp/a
+       5 file1.rclonelink
+      14 file2.rclonelink
+```
+
+The remote files will contain the target of the symbolic links
+
+```
+$ rclone cat remote:/tmp/a/file1.rclonelink
+./file4
+
+$ rclone cat remote:/tmp/a/file2.rclonelink
+/home/user/file3
+```
+
+Copying them back with '-l'
+
+```
+$ rclone copyto -l remote:/tmp/a/ /tmp/b/
+
+$ tree /tmp/b
+/tmp/b
+├── file1 -> ./file4
+└── file2 -> /home/user/file3
+```
+
+However, if copied back without '-l'
+
+```
+$ rclone copyto remote:/tmp/a/ /tmp/b/
+
+$ tree /tmp/b
+/tmp/b
+├── file1.rclonelink
+└── file2.rclonelink
+````
+
+Note that this flag is incompatible with `-copy-links` / `-L`.
+
 ### Restricting filesystems with --one-file-system
 
 Normally rclone will recurse through filesystems as mounted.

docs/content/overview.md

@@ -36,7 +36,7 @@ Here is an overview of the major features of each cloud storage system.
 | pCloud                       | MD5, SHA1   | Yes     | No               | No              | W         |
 | QingStor                     | MD5         | No      | No               | No              | R/W       |
 | SFTP                         | MD5, SHA1 ‡ | Yes     | Depends          | No              | -         |
-| WebDAV                       | -           | Yes ††  | Depends          | No              | -         |
+| WebDAV                       | MD5, SHA1 ††| Yes ††† | Depends          | No              | -         |
 | Yandex Disk                  | MD5         | Yes     | No               | No              | R/W       |
 | The local filesystem         | All         | Yes     | Depends          | No              | -         |
 
@@ -57,7 +57,9 @@ This is an SHA256 sum of all the 4MB block SHA256s.
 ‡ SFTP supports checksums if the same login has shell access and `md5sum`
 or `sha1sum` as well as `echo` are in the remote's PATH.
 
-†† WebDAV supports modtimes when used with Owncloud and Nextcloud only.
+†† WebDAV supports hashes when used with Owncloud and Nextcloud only.
+
+††† WebDAV supports modtimes when used with Owncloud and Nextcloud only.
 
 ‡‡ Microsoft OneDrive Personal supports SHA1 hashes, whereas OneDrive
 for business and SharePoint server support Microsoft's own
@@ -147,7 +149,7 @@ operations more efficient.
 | pCloud                       | Yes   | Yes  | Yes  | Yes     | Yes     | No    | No           | No [#2178](https://github.com/ncw/rclone/issues/2178) | Yes |
 | QingStor                     | No    | Yes  | No   | No      | No      | Yes   | No           | No [#2178](https://github.com/ncw/rclone/issues/2178) | No  |
 | SFTP                         | No    | No   | Yes  | Yes     | No      | No    | Yes          | No [#2178](https://github.com/ncw/rclone/issues/2178) | No  |
-| WebDAV                       | Yes   | Yes  | Yes  | Yes     | No      | No    | Yes ‡        | No [#2178](https://github.com/ncw/rclone/issues/2178) | No  |
+| WebDAV                       | Yes   | Yes  | Yes  | Yes     | No      | No    | Yes ‡        | No [#2178](https://github.com/ncw/rclone/issues/2178) | Yes  |
 | Yandex Disk                  | Yes   | Yes  | Yes  | Yes     | Yes     | No    | Yes          | Yes         | Yes |
 | The local filesystem         | Yes   | No   | Yes  | Yes     | No      | No    | Yes          | No          | Yes |
 
@@ -218,5 +220,7 @@ on the particular cloud provider.
 This is used to fetch quota information from the remote, like bytes
 used/free/quota and bytes used in the trash.
 
+This is also used to return the space used, available for `rclone mount`.
+
 If the server can't do `About` then `rclone about` will return an
 error.

docs/content/s3.md

@@ -1413,6 +1413,28 @@ So once set up, for example to copy files into a bucket
 rclone copy /path/to/files minio:bucket
 ```
 
+### Scaleway {#scaleway}
+
+[Scaleway](https://www.scaleway.com/object-storage/) The Object Storage platform allows you to store anything from backups, logs and web assets to documents and photos.
+Files can be dropped from the Scaleway console or transferred through our API and CLI or using any S3-compatible tool.
+
+Scaleway provides an S3 interface which can be configured for use with rclone like this:
+
+```
+[scaleway]
+type = s3
+env_auth = false
+endpoint = s3.nl-ams.scw.cloud
+access_key_id = SCWXXXXXXXXXXXXXX
+secret_access_key = 1111111-2222-3333-44444-55555555555555
+region = nl-ams
+location_constraint =
+acl = private
+force_path_style = false
+server_side_encryption =
+storage_class =
+```
+
 ### Wasabi ###
 
 [Wasabi](https://wasabi.com) is a cloud-based object storage service for a

docs/content/swift.md

@@ -329,6 +329,33 @@ User ID to log in - optional - most swift systems use user and leave this blank
 - Type:        string
 - Default:     ""
 
+#### --swift-application-credential-id
+
+Application Credential ID to log in - optional (v3 auth) (OS_APPLICATION_CREDENTIAL_ID).
+
+- Config:      application_credential_id
+- Env Var:     RCLONE_SWIFT_APPLICATION_CREDENTIAL_ID
+- Type:        string
+- Default:     ""
+
+#### --swift-application-credential-name
+
+Application Credential name to log in - optional (v3 auth) (OS_APPLICATION_CREDENTIAL_NAME).
+
+- Config:      application_credential_name
+- Env Var:     RCLONE_SWIFT_APPLICATION_CREDENTIAL_NAME
+- Type:        string
+- Default:     ""
+
+#### --swift-application-credential-secret
+
+Application Credential secret to log in - optional (v3 auth) (OS_APPLICATION_CREDENTIAL_SECRET).
+
+- Config:      application_credential_secret
+- Env Var:     RCLONE_SWIFT_APPLICATION_CREDENTIAL_SECRET
+- Type:        string
+- Default:     ""
+
 #### --swift-domain
 
 User domain - optional (v3 auth) (OS_USER_DOMAIN_NAME)

docs/content/webdav.md

@@ -99,7 +99,11 @@ To copy a local directory to an WebDAV directory called backup
 Plain WebDAV does not support modified times.  However when used with
 Owncloud or Nextcloud rclone will support modified times.
 
-Hashes are not supported.
+Likewise plain WebDAV does not support hashes, however when used with
+Owncloud or Nexcloud rclone will support SHA1 and MD5 hashes.
+Depending on the exact version of Owncloud or Nextcloud hashes may
+appear on all objects, or only on objects which had a hash uploaded
+with them.
 
 <!--- autogenerated options start - DO NOT EDIT, instead edit fs.RegInfo in backend/webdav/webdav.go then run make backenddocs -->
 ### Standard Options

fs/operations/operations.go

@@ -26,6 +26,7 @@ import (
 	"github.com/ncw/rclone/fs/walk"
 	"github.com/ncw/rclone/lib/readers"
 	"github.com/pkg/errors"
+	"golang.org/x/sync/errgroup"
 )
 
 // CheckHashes checks the two files to see if they have common
@@ -1585,3 +1586,81 @@ func (l *ListFormat) Format(entry fs.DirEntry) (result string) {
 	}
 	return result
 }
+
+// DirMove renames srcRemote to dstRemote
+//
+// It does this by loading the directory tree into memory (using ListR
+// if available) and doing renames in parallel.
+func DirMove(f fs.Fs, srcRemote, dstRemote string) (err error) {
+	// Use DirMove if possible
+	if doDirMove := f.Features().DirMove; doDirMove != nil {
+		return doDirMove(f, srcRemote, dstRemote)
+	}
+
+	// Load the directory tree into memory
+	tree, err := walk.NewDirTree(f, srcRemote, true, -1)
+	if err != nil {
+		return errors.Wrap(err, "RenameDir tree walk")
+	}
+
+	// Get the directories in sorted order
+	dirs := tree.Dirs()
+
+	// Make the destination directories - must be done in order not in parallel
+	for _, dir := range dirs {
+		dstPath := dstRemote + dir[len(srcRemote):]
+		err := f.Mkdir(dstPath)
+		if err != nil {
+			return errors.Wrap(err, "RenameDir mkdir")
+		}
+	}
+
+	// Rename the files in parallel
+	type rename struct {
+		o       fs.Object
+		newPath string
+	}
+	renames := make(chan rename, fs.Config.Transfers)
+	g, ctx := errgroup.WithContext(context.Background())
+	for i := 0; i < fs.Config.Transfers; i++ {
+		g.Go(func() error {
+			for job := range renames {
+				dstOverwritten, _ := f.NewObject(job.newPath)
+				_, err := Move(f, dstOverwritten, job.newPath, job.o)
+				if err != nil {
+					return err
+				}
+				select {
+				case <-ctx.Done():
+					return ctx.Err()
+				default:
+				}
+
+			}
+			return nil
+		})
+	}
+	for dir, entries := range tree {
+		dstPath := dstRemote + dir[len(srcRemote):]
+		for _, entry := range entries {
+			if o, ok := entry.(fs.Object); ok {
+				renames <- rename{o, path.Join(dstPath, path.Base(o.Remote()))}
+			}
+		}
+	}
+	close(renames)
+	err = g.Wait()
+	if err != nil {
+		return errors.Wrap(err, "RenameDir renames")
+	}
+
+	// Remove the source directories in reverse order
+	for i := len(dirs) - 1; i >= 0; i-- {
+		err := f.Rmdir(dirs[i])
+		if err != nil {
+			return errors.Wrap(err, "RenameDir rmdir")
+		}
+	}
+
+	return nil
+}

fs/operations/operations_test.go

@@ -956,3 +956,90 @@ func TestListFormat(t *testing.T) {
 	assert.Equal(t, fmt.Sprintf("%d", items[1].Size())+"|subdir/|"+items[1].ModTime().Local().Format("2006-01-02 15:04:05"), list.Format(items[1]))
 
 }
+
+func TestDirMove(t *testing.T) {
+	r := fstest.NewRun(t)
+	defer r.Finalise()
+
+	r.Mkdir(r.Fremote)
+
+	// Make some files and dirs
+	r.ForceMkdir(r.Fremote)
+	files := []fstest.Item{
+		r.WriteObject("A1/one", "one", t1),
+		r.WriteObject("A1/two", "two", t2),
+		r.WriteObject("A1/B1/three", "three", t3),
+		r.WriteObject("A1/B1/C1/four", "four", t1),
+		r.WriteObject("A1/B1/C2/five", "five", t2),
+	}
+	require.NoError(t, operations.Mkdir(r.Fremote, "A1/B2"))
+	require.NoError(t, operations.Mkdir(r.Fremote, "A1/B1/C3"))
+
+	fstest.CheckListingWithPrecision(
+		t,
+		r.Fremote,
+		files,
+		[]string{
+			"A1",
+			"A1/B1",
+			"A1/B2",
+			"A1/B1/C1",
+			"A1/B1/C2",
+			"A1/B1/C3",
+		},
+		fs.GetModifyWindow(r.Fremote),
+	)
+
+	require.NoError(t, operations.DirMove(r.Fremote, "A1", "A2"))
+
+	for i := range files {
+		files[i].Path = strings.Replace(files[i].Path, "A1/", "A2/", -1)
+		files[i].WinPath = ""
+	}
+
+	fstest.CheckListingWithPrecision(
+		t,
+		r.Fremote,
+		files,
+		[]string{
+			"A2",
+			"A2/B1",
+			"A2/B2",
+			"A2/B1/C1",
+			"A2/B1/C2",
+			"A2/B1/C3",
+		},
+		fs.GetModifyWindow(r.Fremote),
+	)
+
+	// Disable DirMove
+	features := r.Fremote.Features()
+	oldDirMove := features.DirMove
+	features.DirMove = nil
+	defer func() {
+		features.DirMove = oldDirMove
+	}()
+
+	require.NoError(t, operations.DirMove(r.Fremote, "A2", "A3"))
+
+	for i := range files {
+		files[i].Path = strings.Replace(files[i].Path, "A2/", "A3/", -1)
+		files[i].WinPath = ""
+	}
+
+	fstest.CheckListingWithPrecision(
+		t,
+		r.Fremote,
+		files,
+		[]string{
+			"A3",
+			"A3/B1",
+			"A3/B2",
+			"A3/B1/C1",
+			"A3/B1/C2",
+			"A3/B1/C3",
+		},
+		fs.GetModifyWindow(r.Fremote),
+	)
+
+}

go.mod

@@ -30,7 +30,7 @@ require (
 	github.com/kr/pretty v0.1.0 // indirect
 	github.com/mattn/go-runewidth v0.0.3 // indirect
 	github.com/ncw/go-acd v0.0.0-20171120105400-887eb06ab6a2
-	github.com/ncw/swift v1.0.43
+	github.com/ncw/swift v1.0.44
 	github.com/nsf/termbox-go v0.0.0-20181027232701-60ab7e3d12ed
 	github.com/okzk/sdnotify v0.0.0-20180710141335-d9becc38acbd
 	github.com/patrickmn/go-cache v2.1.0+incompatible

go.sum

@@ -74,6 +74,8 @@ github.com/ncw/swift v1.0.42 h1:ztvRb6hs52IHOcaYt73f9lXYLIeIuWgdooRDhdyllGI=
 github.com/ncw/swift v1.0.42/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM=
 github.com/ncw/swift v1.0.43 h1:TZn2l/bPV0CqG+/G5BFh/ROWnyX7dL2D0URaOjNQRsw=
 github.com/ncw/swift v1.0.43/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM=
+github.com/ncw/swift v1.0.44 h1:EKvOTvUxElbpDWqxsyVaVGvc2IfuOqQnRmjnR2AGhQ4=
+github.com/ncw/swift v1.0.44/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM=
 github.com/nsf/termbox-go v0.0.0-20181027232701-60ab7e3d12ed h1:bAVGG6B+R5qpSylrrA+BAMrzYkdAoiTaKPVxRB+4cyM=
 github.com/nsf/termbox-go v0.0.0-20181027232701-60ab7e3d12ed/go.mod h1:IuKpRQcYE1Tfu+oAQqaLisqDeXgjyyltCfsaoYN18NQ=
 github.com/okzk/sdnotify v0.0.0-20180710141335-d9becc38acbd h1:+iAPaTbi1gZpcpDwe/BW1fx7Xoesv69hLNGPheoyhBs=

lib/pacer/pacer.go

@@ -2,12 +2,14 @@
 package pacer
 
 import (
+	"context"
 	"math/rand"
 	"sync"
 	"time"
 
 	"github.com/ncw/rclone/fs"
 	"github.com/ncw/rclone/fs/fserrors"
+	"golang.org/x/time/rate"
 )
 
 // Pacer state
@@ -15,6 +17,8 @@ type Pacer struct {
 	mu                 sync.Mutex    // Protecting read/writes
 	minSleep           time.Duration // minimum sleep time
 	maxSleep           time.Duration // maximum sleep time
+	burst              int           // number of calls to send without rate limiting
+	limiter            *rate.Limiter // rate limiter for the minsleep
 	decayConstant      uint          // decay constant
 	attackConstant     uint          // attack constant
 	pacer              chan struct{} // To pace the operations
@@ -76,7 +80,6 @@ type Paced func() (bool, error)
 // New returns a Pacer with sensible defaults
 func New() *Pacer {
 	p := &Pacer{
-		minSleep:       10 * time.Millisecond,
 		maxSleep:       2 * time.Second,
 		decayConstant:  2,
 		attackConstant: 1,
@@ -86,6 +89,7 @@ func New() *Pacer {
 	p.sleepTime = p.minSleep
 	p.SetPacer(DefaultPacer)
 	p.SetMaxConnections(fs.Config.Checkers + fs.Config.Transfers)
+	p.SetMinSleep(10 * time.Millisecond)
 
 	// Put the first pacing token in
 	p.pacer <- struct{}{}
@@ -114,6 +118,16 @@ func (p *Pacer) SetMinSleep(t time.Duration) *Pacer {
 	defer p.mu.Unlock()
 	p.minSleep = t
 	p.sleepTime = p.minSleep
+	p.limiter = rate.NewLimiter(rate.Every(p.minSleep), p.burst)
+	return p
+}
+
+// SetBurst sets the burst with no limiting of the pacer
+func (p *Pacer) SetBurst(n int) *Pacer {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	p.burst = n
+	p.limiter = rate.NewLimiter(rate.Every(p.minSleep), p.burst)
 	return p
 }
 
@@ -216,11 +230,19 @@ func (p *Pacer) beginCall() {
 
 	p.mu.Lock()
 	// Restart the timer
-	go func(t time.Duration) {
+	go func(sleepTime, minSleep time.Duration) {
 		// fs.Debugf(f, "New sleep for %v at %v", t, time.Now())
-		time.Sleep(t)
+		// Sleep the minimum time with the rate limiter
+		if minSleep > 0 && sleepTime >= minSleep {
+			_ = p.limiter.Wait(context.Background())
+			sleepTime -= minSleep
+		}
+		// Then sleep the remaining time
+		if sleepTime > 0 {
+			time.Sleep(sleepTime)
+		}
 		p.pacer <- struct{}{}
-	}(p.sleepTime)
+	}(p.sleepTime, p.minSleep)
 	p.mu.Unlock()
 }
 

lib/rest/rest.go

@@ -198,7 +198,17 @@ func (api *Client) Call(opts *Opts) (resp *http.Response, err error) {
 	if opts.Parameters != nil && len(opts.Parameters) > 0 {
 		url += "?" + opts.Parameters.Encode()
 	}
-	req, err := http.NewRequest(opts.Method, url, opts.Body)
+	body := opts.Body
+	// If length is set and zero then nil out the body to stop use
+	// use of chunked encoding and insert a "Content-Length: 0"
+	// header.
+	//
+	// If we don't do this we get "Content-Length" headers for all
+	// files except 0 length files.
+	if opts.ContentLength != nil && *opts.ContentLength == 0 {
+		body = nil
+	}
+	req, err := http.NewRequest(opts.Method, url, body)
 	if err != nil {
 		return
 	}

vendor/github.com/ncw/swift/README.md

@@ -139,7 +139,7 @@ Contributors
 - Cezar Sa Espinola <cezarsa@gmail.com>
 - Sam Gunaratne <samgzeit@gmail.com>
 - Richard Scothern <richard.scothern@gmail.com>
-- Michel Couillard <couillard.michel@voxlog.ca>
+- Michel Couillard <!--<couillard.michel@voxlog.ca>--> <michel.couillard@gmail.com>
 - Christopher Waldon <ckwaldon@us.ibm.com>
 - dennis <dai.haojun@gmail.com>
 - hag <hannes.georg@xing.com>
@@ -152,3 +152,4 @@ Contributors
 - Charles Hsu <charles0126@gmail.com>
 - Omar Ali <omarali@users.noreply.github.com>
 - Andreas Andersen <andreas@softwaredesign.se>
+- kayrus <kay.diam@gmail.com>

vendor/github.com/ncw/swift/auth_v3.go

@@ -3,14 +3,16 @@ package swift
 import (
 	"bytes"
 	"encoding/json"
+	"fmt"
 	"net/http"
 	"strings"
 )
 
 const (
-	v3AuthMethodToken        = "token"
-	v3AuthMethodPassword     = "password"
-	v3CatalogTypeObjectStore = "object-store"
+	v3AuthMethodToken                 = "token"
+	v3AuthMethodPassword              = "password"
+	v3AuthMethodApplicationCredential = "application_credential"
+	v3CatalogTypeObjectStore          = "object-store"
 )
 
 // V3 Authentication request
@@ -19,9 +21,10 @@ const (
 type v3AuthRequest struct {
 	Auth struct {
 		Identity struct {
-			Methods  []string        `json:"methods"`
-			Password *v3AuthPassword `json:"password,omitempty"`
-			Token    *v3AuthToken    `json:"token,omitempty"`
+			Methods               []string                     `json:"methods"`
+			Password              *v3AuthPassword              `json:"password,omitempty"`
+			Token                 *v3AuthToken                 `json:"token,omitempty"`
+			ApplicationCredential *v3AuthApplicationCredential `json:"application_credential,omitempty"`
 		} `json:"identity"`
 		Scope *v3Scope `json:"scope,omitempty"`
 	} `json:"auth"`
@@ -63,6 +66,13 @@ type v3AuthPassword struct {
 	User v3User `json:"user"`
 }
 
+type v3AuthApplicationCredential struct {
+	Id     string  `json:"id,omitempty"`
+	Name   string  `json:"name,omitempty"`
+	Secret string  `json:"secret,omitempty"`
+	User   *v3User `json:"user,omitempty"`
+}
+
 // V3 Authentication response
 type v3AuthResponse struct {
 	Token struct {
@@ -117,7 +127,57 @@ func (auth *v3Auth) Request(c *Connection) (*http.Request, error) {
 
 	v3 := v3AuthRequest{}
 
-	if c.UserName == "" && c.UserId == "" {
+	if (c.ApplicationCredentialId != "" || c.ApplicationCredentialName != "") && c.ApplicationCredentialSecret != "" {
+		var user *v3User
+
+		if c.ApplicationCredentialId != "" {
+			c.ApplicationCredentialName = ""
+			user = &v3User{}
+		}
+
+		if user == nil && c.UserId != "" {
+			// UserID could be used without the domain information
+			user = &v3User{
+				Id: c.UserId,
+			}
+		}
+
+		if user == nil && c.UserName == "" {
+			// Make sure that Username or UserID are provided
+			return nil, fmt.Errorf("UserID or Name should be provided")
+		}
+
+		if user == nil && c.DomainId != "" {
+			user = &v3User{
+				Name: c.UserName,
+				Domain: &v3Domain{
+					Id: c.DomainId,
+				},
+			}
+		}
+
+		if user == nil && c.Domain != "" {
+			user = &v3User{
+				Name: c.UserName,
+				Domain: &v3Domain{
+					Name: c.Domain,
+				},
+			}
+		}
+
+		// Make sure that DomainID or DomainName are provided among Username
+		if user == nil {
+			return nil, fmt.Errorf("DomainID or Domain should be provided")
+		}
+
+		v3.Auth.Identity.Methods = []string{v3AuthMethodApplicationCredential}
+		v3.Auth.Identity.ApplicationCredential = &v3AuthApplicationCredential{
+			Id:     c.ApplicationCredentialId,
+			Name:   c.ApplicationCredentialName,
+			Secret: c.ApplicationCredentialSecret,
+			User:   user,
+		}
+	} else if c.UserName == "" && c.UserId == "" {
 		v3.Auth.Identity.Methods = []string{v3AuthMethodToken}
 		v3.Auth.Identity.Token = &v3AuthToken{Id: c.ApiKey}
 	} else {
@@ -140,27 +200,29 @@ func (auth *v3Auth) Request(c *Connection) (*http.Request, error) {
 		v3.Auth.Identity.Password.User.Domain = domain
 	}
 
-	if c.TrustId != "" {
-		v3.Auth.Scope = &v3Scope{Trust: &v3Trust{Id: c.TrustId}}
-	} else if c.TenantId != "" || c.Tenant != "" {
+	if v3.Auth.Identity.Methods[0] != v3AuthMethodApplicationCredential {
+		if c.TrustId != "" {
+			v3.Auth.Scope = &v3Scope{Trust: &v3Trust{Id: c.TrustId}}
+		} else if c.TenantId != "" || c.Tenant != "" {
 
-		v3.Auth.Scope = &v3Scope{Project: &v3Project{}}
+			v3.Auth.Scope = &v3Scope{Project: &v3Project{}}
 
-		if c.TenantId != "" {
-			v3.Auth.Scope.Project.Id = c.TenantId
-		} else if c.Tenant != "" {
-			v3.Auth.Scope.Project.Name = c.Tenant
-			switch {
-			case c.TenantDomain != "":
-				v3.Auth.Scope.Project.Domain = &v3Domain{Name: c.TenantDomain}
-			case c.TenantDomainId != "":
-				v3.Auth.Scope.Project.Domain = &v3Domain{Id: c.TenantDomainId}
-			case c.Domain != "":
-				v3.Auth.Scope.Project.Domain = &v3Domain{Name: c.Domain}
-			case c.DomainId != "":
-				v3.Auth.Scope.Project.Domain = &v3Domain{Id: c.DomainId}
-			default:
-				v3.Auth.Scope.Project.Domain = &v3Domain{Name: "Default"}
+			if c.TenantId != "" {
+				v3.Auth.Scope.Project.Id = c.TenantId
+			} else if c.Tenant != "" {
+				v3.Auth.Scope.Project.Name = c.Tenant
+				switch {
+				case c.TenantDomain != "":
+					v3.Auth.Scope.Project.Domain = &v3Domain{Name: c.TenantDomain}
+				case c.TenantDomainId != "":
+					v3.Auth.Scope.Project.Domain = &v3Domain{Id: c.TenantDomainId}
+				case c.Domain != "":
+					v3.Auth.Scope.Project.Domain = &v3Domain{Name: c.Domain}
+				case c.DomainId != "":
+					v3.Auth.Scope.Project.Domain = &v3Domain{Id: c.DomainId}
+				default:
+					v3.Auth.Scope.Project.Domain = &v3Domain{Name: "Default"}
+				}
 			}
 		}
 	}

vendor/github.com/ncw/swift/swift.go

@@ -96,26 +96,29 @@ const (
 type Connection struct {
 	// Parameters - fill these in before calling Authenticate
 	// They are all optional except UserName, ApiKey and AuthUrl
-	Domain         string            // User's domain name
-	DomainId       string            // User's domain Id
-	UserName       string            // UserName for api
-	UserId         string            // User Id
-	ApiKey         string            // Key for api access
-	AuthUrl        string            // Auth URL
-	Retries        int               // Retries on error (default is 3)
-	UserAgent      string            // Http User agent (default goswift/1.0)
-	ConnectTimeout time.Duration     // Connect channel timeout (default 10s)
-	Timeout        time.Duration     // Data channel timeout (default 60s)
-	Region         string            // Region to use eg "LON", "ORD" - default is use first region (v2,v3 auth only)
-	AuthVersion    int               // Set to 1, 2 or 3 or leave at 0 for autodetect
-	Internal       bool              // Set this to true to use the the internal / service network
-	Tenant         string            // Name of the tenant (v2,v3 auth only)
-	TenantId       string            // Id of the tenant (v2,v3 auth only)
-	EndpointType   EndpointType      // Endpoint type (v2,v3 auth only) (default is public URL unless Internal is set)
-	TenantDomain   string            // Name of the tenant's domain (v3 auth only), only needed if it differs from the user domain
-	TenantDomainId string            // Id of the tenant's domain (v3 auth only), only needed if it differs the from user domain
-	TrustId        string            // Id of the trust (v3 auth only)
-	Transport      http.RoundTripper `json:"-" xml:"-"` // Optional specialised http.Transport (eg. for Google Appengine)
+	Domain                      string            // User's domain name
+	DomainId                    string            // User's domain Id
+	UserName                    string            // UserName for api
+	UserId                      string            // User Id
+	ApiKey                      string            // Key for api access
+	ApplicationCredentialId     string            // Application Credential ID
+	ApplicationCredentialName   string            // Application Credential Name
+	ApplicationCredentialSecret string            // Application Credential Secret
+	AuthUrl                     string            // Auth URL
+	Retries                     int               // Retries on error (default is 3)
+	UserAgent                   string            // Http User agent (default goswift/1.0)
+	ConnectTimeout              time.Duration     // Connect channel timeout (default 10s)
+	Timeout                     time.Duration     // Data channel timeout (default 60s)
+	Region                      string            // Region to use eg "LON", "ORD" - default is use first region (v2,v3 auth only)
+	AuthVersion                 int               // Set to 1, 2 or 3 or leave at 0 for autodetect
+	Internal                    bool              // Set this to true to use the the internal / service network
+	Tenant                      string            // Name of the tenant (v2,v3 auth only)
+	TenantId                    string            // Id of the tenant (v2,v3 auth only)
+	EndpointType                EndpointType      // Endpoint type (v2,v3 auth only) (default is public URL unless Internal is set)
+	TenantDomain                string            // Name of the tenant's domain (v3 auth only), only needed if it differs from the user domain
+	TenantDomainId              string            // Id of the tenant's domain (v3 auth only), only needed if it differs the from user domain
+	TrustId                     string            // Id of the trust (v3 auth only)
+	Transport                   http.RoundTripper `json:"-" xml:"-"` // Optional specialised http.Transport (eg. for Google Appengine)
 	// These are filled in after Authenticate is called as are the defaults for above
 	StorageUrl string
 	AuthToken  string
@@ -194,6 +197,9 @@ func setFromEnv(param interface{}, name string) (err error) {
 //     OS_USERNAME - UserName for api
 //     OS_USER_ID - User Id
 //     OS_PASSWORD - Key for api access
+//     OS_APPLICATION_CREDENTIAL_ID - Application Credential ID
+//     OS_APPLICATION_CREDENTIAL_NAME - Application Credential Name
+//     OS_APPLICATION_CREDENTIAL_SECRET - Application Credential Secret
 //     OS_USER_DOMAIN_NAME - User's domain name
 //     OS_USER_DOMAIN_ID - User's domain Id
 //     OS_PROJECT_NAME - Name of the project
@@ -227,6 +233,9 @@ func (c *Connection) ApplyEnvironment() (err error) {
 		{&c.UserName, "OS_USERNAME"},
 		{&c.UserId, "OS_USER_ID"},
 		{&c.ApiKey, "OS_PASSWORD"},
+		{&c.ApplicationCredentialId, "OS_APPLICATION_CREDENTIAL_ID"},
+		{&c.ApplicationCredentialName, "OS_APPLICATION_CREDENTIAL_NAME"},
+		{&c.ApplicationCredentialSecret, "OS_APPLICATION_CREDENTIAL_SECRET"},
 		{&c.AuthUrl, "OS_AUTH_URL"},
 		{&c.Retries, "GOSWIFT_RETRIES"},
 		{&c.UserAgent, "GOSWIFT_USER_AGENT"},

vendor/modules.txt

@@ -103,7 +103,7 @@ github.com/kr/fs
 github.com/mattn/go-runewidth
 # github.com/ncw/go-acd v0.0.0-20171120105400-887eb06ab6a2
 github.com/ncw/go-acd
-# github.com/ncw/swift v1.0.43
+# github.com/ncw/swift v1.0.44
 github.com/ncw/swift
 # github.com/nsf/termbox-go v0.0.0-20181027232701-60ab7e3d12ed
 github.com/nsf/termbox-go

vfs/dir.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/ncw/rclone/fs"
 	"github.com/ncw/rclone/fs/list"
+	"github.com/ncw/rclone/fs/operations"
 	"github.com/ncw/rclone/fs/walk"
 	"github.com/pkg/errors"
 )
@@ -576,15 +577,15 @@ func (d *Dir) Rename(oldName, newName string, destDir *Dir) error {
 			return err
 		}
 	case fs.Directory:
-		doDirMove := d.f.Features().DirMove
-		if doDirMove == nil {
-			err := errors.Errorf("Fs %q can't rename directories (no DirMove)", d.f)
+		features := d.f.Features()
+		if features.DirMove == nil && features.Move == nil && features.Copy == nil {
+			err := errors.Errorf("Fs %q can't rename directories (no DirMove, Move or Copy)", d.f)
 			fs.Errorf(oldPath, "Dir.Rename error: %v", err)
 			return err
 		}
 		srcRemote := x.Remote()
 		dstRemote := newPath
-		err = doDirMove(d.f, srcRemote, dstRemote)
+		err = operations.DirMove(d.f, srcRemote, dstRemote)
 		if err != nil {
 			fs.Errorf(oldPath, "Dir.Rename error: %v", err)
 			return err