sftp: allow cert based auth via optional pubkey

Before this change we sorted transfers in the stats list solely on
time started. However if --check-first was in use then lots of
transfers could be started in the same millisecond. Because Windows
time resolution is only 1mS this caused the entries to sort equal and
bounce around in the list.

This change fixes the sort so that if the time is equal it uses the
name which should stabilize the order.

Fixes #4599

RELEASE.md

@@ -21,7 +21,7 @@ This file describes how to make the various kinds of releases
   * git status - to check for new man pages - git add them
   * git commit -a -v -m "Version v1.XX.0"
   * make retag
-  * git push --tags origin master
+  * git push --follow-tags origin
   * # Wait for the GitHub builds to complete then...
   * make fetch_binaries
   * make tarball
@@ -65,7 +65,7 @@ Now
   * git cherry-pick any fixes
   * Do the steps as above
   * make startstable
-  * NB this overwrites the current beta so we need to do this - FIXME is this true any more?
+  * NB this overwrites the current beta so we need to do this
   * git co master
   * # cherry pick the changes to the changelog
   * git checkout ${BASE_TAG}-stable docs/content/changelog.md

backend/chunker/chunker.go

@@ -958,6 +958,8 @@ func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, remote st
 		}
 		info := f.wrapInfo(src, chunkRemote, size)
 
+		// Refill chunkLimit and let basePut repeatedly call chunkingReader.Read()
+		c.chunkLimit = c.chunkSize
 		// TODO: handle range/limit options
 		chunk, errChunk := basePut(ctx, wrapIn, info, options...)
 		if errChunk != nil {
@@ -1166,10 +1168,14 @@ func (c *chunkingReader) updateHashes() {
 func (c *chunkingReader) Read(buf []byte) (bytesRead int, err error) {
 	if c.chunkLimit <= 0 {
 		// Chunk complete - switch to next one.
+		// Note #1:
 		// We might not get here because some remotes (eg. box multi-uploader)
 		// read the specified size exactly and skip the concluding EOF Read.
 		// Then a check in the put loop will kick in.
-		c.chunkLimit = c.chunkSize
+		// Note #2:
+		// The crypt backend after receiving EOF here will call Read again
+		// and we must insist on returning EOF, so we postpone refilling
+		// chunkLimit to the main loop.
 		return 0, io.EOF
 	}
 	if int64(len(buf)) > c.chunkLimit {

backend/drive/drive.go

@@ -2302,10 +2302,9 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 		// preserve the description on copy for docs
 		info, err := f.getFile(actualID(srcObj.id), "description")
 		if err != nil {
-			fs.Errorf(srcObj, "Failed to read description for Google Doc: %v", err)
-		} else {
-			createInfo.Description = info.Description
+			return nil, errors.Wrap(err, "failed to read description for Google Doc")
 		}
+		createInfo.Description = info.Description
 	} else {
 		// don't overwrite the description on copy for files
 		// this should work for docs but it doesn't - it is probably a bug in Google Drive

backend/mailru/api/m1.go

@@ -159,71 +159,6 @@ type FolderInfoResponse struct {
 	Email  string `json:"email"`
 }
 
-// ShardInfoResponse ...
-type ShardInfoResponse struct {
-	Email string `json:"email"`
-	Body  struct {
-		Video []struct {
-			Count string `json:"count"`
-			URL   string `json:"url"`
-		} `json:"video"`
-		ViewDirect []struct {
-			Count string `json:"count"`
-			URL   string `json:"url"`
-		} `json:"view_direct"`
-		WeblinkView []struct {
-			Count string `json:"count"`
-			URL   string `json:"url"`
-		} `json:"weblink_view"`
-		WeblinkVideo []struct {
-			Count string `json:"count"`
-			URL   string `json:"url"`
-		} `json:"weblink_video"`
-		WeblinkGet []struct {
-			Count int    `json:"count"`
-			URL   string `json:"url"`
-		} `json:"weblink_get"`
-		Stock []struct {
-			Count string `json:"count"`
-			URL   string `json:"url"`
-		} `json:"stock"`
-		WeblinkThumbnails []struct {
-			Count string `json:"count"`
-			URL   string `json:"url"`
-		} `json:"weblink_thumbnails"`
-		PublicUpload []struct {
-			Count string `json:"count"`
-			URL   string `json:"url"`
-		} `json:"public_upload"`
-		Auth []struct {
-			Count string `json:"count"`
-			URL   string `json:"url"`
-		} `json:"auth"`
-		Web []struct {
-			Count string `json:"count"`
-			URL   string `json:"url"`
-		} `json:"web"`
-		View []struct {
-			Count string `json:"count"`
-			URL   string `json:"url"`
-		} `json:"view"`
-		Upload []struct {
-			Count string `json:"count"`
-			URL   string `json:"url"`
-		} `json:"upload"`
-		Get []struct {
-			Count string `json:"count"`
-			URL   string `json:"url"`
-		} `json:"get"`
-		Thumbnails []struct {
-			Count string `json:"count"`
-			URL   string `json:"url"`
-		} `json:"thumbnails"`
-	} `json:"body"`
-	Time   int64 `json:"time"`
-	Status int   `json:"status"`
-}
-
 // CleanupResponse ...
 type CleanupResponse struct {
 	Email     string `json:"email"`

backend/mailru/mailru.go

@@ -37,6 +37,7 @@ import (
 	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/oauthutil"
 	"github.com/rclone/rclone/lib/pacer"
+	"github.com/rclone/rclone/lib/readers"
 	"github.com/rclone/rclone/lib/rest"
 
 	"github.com/pkg/errors"
@@ -1861,30 +1862,30 @@ func (f *Fs) uploadShard(ctx context.Context) (string, error) {
 		return f.shardURL, nil
 	}
 
-	token, err := f.accessToken()
-	if err != nil {
-		return "", err
-	}
-
 	opts := rest.Opts{
-		Method: "GET",
-		Path:   "/api/m1/dispatcher",
-		Parameters: url.Values{
-			"client_id":    {api.OAuthClientID},
-			"access_token": {token},
-		},
+		RootURL: api.DispatchServerURL,
+		Method:  "GET",
+		Path:    "/u",
 	}
 
-	var info api.ShardInfoResponse
+	var (
+		res *http.Response
+		url string
+		err error
+	)
 	err = f.pacer.Call(func() (bool, error) {
-		res, err := f.srv.CallJSON(ctx, &opts, nil, &info)
-		return shouldRetry(res, err, f, &opts)
+		res, err = f.srv.Call(ctx, &opts)
+		if err == nil {
+			url, err = readBodyWord(res)
+		}
+		return fserrors.ShouldRetry(err), err
 	})
 	if err != nil {
+		closeBody(res)
 		return "", err
 	}
 
-	f.shardURL = info.Body.Upload[0].URL
+	f.shardURL = url
 	f.shardExpiry = time.Now().Add(shardExpirySec * time.Second)
 	fs.Debugf(f, "new upload shard: %s", f.shardURL)
 
@@ -2116,7 +2117,18 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 		return nil, err
 	}
 
-	start, end, partial := getTransferRange(o.size, options...)
+	start, end, partialRequest := getTransferRange(o.size, options...)
+
+	headers := map[string]string{
+		"Accept":       "*/*",
+		"Content-Type": "application/octet-stream",
+	}
+	if partialRequest {
+		rangeStr := fmt.Sprintf("bytes=%d-%d", start, end-1)
+		headers["Range"] = rangeStr
+		// headers["Content-Range"] = rangeStr
+		headers["Accept-Ranges"] = "bytes"
+	}
 
 	// TODO: set custom timeouts
 	opts := rest.Opts{
@@ -2127,10 +2139,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 			"client_id": {api.OAuthClientID},
 			"token":     {token},
 		},
-		ExtraHeaders: map[string]string{
-			"Accept": "*/*",
-			"Range":  fmt.Sprintf("bytes=%d-%d", start, end-1),
-		},
+		ExtraHeaders: headers,
 	}
 
 	var res *http.Response
@@ -2151,18 +2160,36 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 		return nil, err
 	}
 
-	var hasher gohash.Hash
-	if !partial {
+	// Server should respond with Status 206 and Content-Range header to a range
+	// request. Status 200 (and no Content-Range) means a full-content response.
+	partialResponse := res.StatusCode == 206
+
+	var (
+		hasher     gohash.Hash
+		wrapStream io.ReadCloser
+	)
+	if !partialResponse {
 		// Cannot check hash of partial download
 		hasher = mrhash.New()
 	}
-	wrapStream := &endHandler{
+	wrapStream = &endHandler{
 		ctx:    ctx,
 		stream: res.Body,
 		hasher: hasher,
 		o:      o,
 		server: server,
 	}
+	if partialRequest && !partialResponse {
+		fs.Debugf(o, "Server returned full content instead of range")
+		if start > 0 {
+			// Discard the beginning of the data
+			_, err = io.CopyN(ioutil.Discard, wrapStream, start)
+			if err != nil {
+				return nil, err
+			}
+		}
+		wrapStream = readers.NewLimitedReadCloser(wrapStream, end-start)
+	}
 	return wrapStream, nil
 }
 

backend/s3/s3.go

@@ -169,7 +169,7 @@ func init() {
 				Help:  "Asia Pacific (Mumbai)\nNeeds location constraint ap-south-1.",
 			}, {
 				Value: "ap-east-1",
-				Help:  "Asia Patific (Hong Kong) Region\nNeeds location constraint ap-east-1.",
+				Help:  "Asia Pacific (Hong Kong) Region\nNeeds location constraint ap-east-1.",
 			}, {
 				Value: "sa-east-1",
 				Help:  "South America (Sao Paulo) Region\nNeeds location constraint sa-east-1.",

backend/seafile/api/types.go

@@ -46,7 +46,7 @@ type Library struct {
 	Encrypted bool   `json:"encrypted"`
 	Owner     string `json:"owner"`
 	ID        string `json:"id"`
-	Size      int    `json:"size"`
+	Size      int64  `json:"size"`
 	Name      string `json:"name"`
 	Modified  int64  `json:"mtime"`
 }

backend/seafile/seafile.go

@@ -1004,7 +1004,7 @@ func (f *Fs) listLibraries(ctx context.Context) (entries fs.DirEntries, err erro
 
 	for _, library := range libraries {
 		d := fs.NewDir(library.Name, time.Unix(library.Modified, 0))
-		d.SetSize(int64(library.Size))
+		d.SetSize(library.Size)
 		entries = append(entries, d)
 	}
 

backend/sftp/sftp.go

@@ -82,6 +82,9 @@ func init() {
 Only PEM encrypted key files (old OpenSSH format) are supported. Encrypted keys
 in the new OpenSSH format can't be used.`,
 			IsPassword: true,
+		}, {
+			Name: "pubkey_file",
+			Help: "Optional path to public key file; set this if you have a signed certificate you want to use for authentication." + env.ShellExpandHelp,
 		}, {
 			Name: "key_use_agent",
 			Help: `When set forces the usage of the ssh-agent.
@@ -190,6 +193,7 @@ type Options struct {
 	KeyPem            string `config:"key_pem"`
 	KeyFile           string `config:"key_file"`
 	KeyFilePass       string `config:"key_file_pass"`
+	PubKeyFile        string `config:"pubkey_file"`
 	KeyUseAgent       bool   `config:"key_use_agent"`
 	UseInsecureCipher bool   `config:"use_insecure_cipher"`
 	DisableHashCheck  bool   `config:"disable_hashcheck"`
@@ -438,6 +442,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	}
 
 	keyFile := env.ShellExpand(opt.KeyFile)
+	pubkeyFile := env.ShellExpand(opt.PubKeyFile)
 	//keyPem := env.ShellExpand(opt.KeyPem)
 	// Add ssh agent-auth if no password or file or key PEM specified
 	if (opt.Pass == "" && keyFile == "" && !opt.AskPassword && opt.KeyPem == "") || opt.KeyUseAgent {
@@ -507,7 +512,29 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		if err != nil {
 			return nil, errors.Wrap(err, "failed to parse private key file")
 		}
-		sshConfig.Auth = append(sshConfig.Auth, ssh.PublicKeys(signer))
+
+		// If a public key has been specified then use that
+		if pubkeyFile != "" {
+			certfile, err := ioutil.ReadFile(pubkeyFile)
+			if err != nil {
+				return nil, errors.Wrap(err, "unable to read cert file")
+			}
+
+			pk, _, _, _, err := ssh.ParseAuthorizedKey(certfile)
+			if err != nil {
+				return nil, errors.Wrap(err, "unable to parse cert file")
+			}
+
+			// And the signer for this, which includes the private key signer
+			// This is what we'll pass to the ssh client.
+			pubsigner, err := ssh.NewCertSigner(pk.(*ssh.Certificate), signer)
+			if err != nil {
+				return nil, errors.Wrap(err, "error generating cert signer")
+			}
+			sshConfig.Auth = append(sshConfig.Auth, ssh.PublicKeys(pubsigner))
+		} else {
+			sshConfig.Auth = append(sshConfig.Auth, ssh.PublicKeys(signer))
+		}
 	}
 
 	// Auth from password if specified
@@ -1087,7 +1114,7 @@ func shellEscape(str string) string {
 func parseHash(bytes []byte) string {
 	// For strings with backslash *sum writes a leading \
 	// https://unix.stackexchange.com/q/313733/94054
-	return strings.Split(strings.TrimLeft(string(bytes), "\\"), " ")[0] // Split at hash / filename separator
+	return strings.ToLower(strings.Split(strings.TrimLeft(string(bytes), "\\"), " ")[0]) // Split at hash / filename separator / all convert to lowercase
 }
 
 // Parses the byte array output from the SSH session

bin/not-in-stable.go

@@ -10,6 +10,8 @@ import (
 	"os"
 	"os/exec"
 	"regexp"
+
+	"github.com/coreos/go-semver/semver"
 )
 
 // version=$(sed <VERSION -e 's/\.[0-9]+*$//g')
@@ -28,7 +30,7 @@ func readCommits(from, to string) (logMap map[string]string, logs []string) {
 	cmd := exec.Command("git", "log", "--oneline", from+".."+to)
 	out, err := cmd.Output()
 	if err != nil {
-		log.Fatalf("failed to run git log: %v", err)
+		log.Fatalf("failed to run git log %s: %v", from+".."+to, err)
 	}
 	logMap = map[string]string{}
 	logs = []string{}
@@ -53,15 +55,20 @@ func main() {
 	if len(args) != 0 {
 		log.Fatalf("Syntax: %s", os.Args[0])
 	}
+	// v1.54.0
 	versionBytes, err := ioutil.ReadFile("VERSION")
 	if err != nil {
 		log.Fatalf("Failed to read version: %v", err)
 	}
-	i := bytes.LastIndexByte(versionBytes, '.')
-	version := string(versionBytes[:i])
-	log.Printf("Finding commits not in stable %s", version)
-	masterMap, masterLogs := readCommits(version+".0", "master")
-	stableMap, _ := readCommits(version+".0", version+"-stable")
+	if versionBytes[0] == 'v' {
+		versionBytes = versionBytes[1:]
+	}
+	versionBytes = bytes.TrimSpace(versionBytes)
+	semver := semver.New(string(versionBytes))
+	stable := fmt.Sprintf("v%d.%d", semver.Major, semver.Minor-1)
+	log.Printf("Finding commits in %v not in stable %s", semver, stable)
+	masterMap, masterLogs := readCommits(stable+".0", "master")
+	stableMap, _ := readCommits(stable+".0", stable+"-stable")
 	for _, logMessage := range masterLogs {
 		// Commit found in stable already
 		if _, found := stableMap[logMessage]; found {

cmd/mount2/fs.go

@@ -67,8 +67,8 @@ func setAttr(node vfs.Node, attr *fuse.Attr) {
 	modTime := node.ModTime()
 	// set attributes
 	vfs := node.VFS()
-	attr.Owner.Gid = vfs.Opt.UID
-	attr.Owner.Uid = vfs.Opt.GID
+	attr.Owner.Gid = vfs.Opt.GID
+	attr.Owner.Uid = vfs.Opt.UID
 	attr.Mode = getMode(node)
 	attr.Size = Size
 	attr.Nlink = 1

docs/content/authors.md

@@ -410,3 +410,6 @@ put them back in again.` >}}
   * WarpedPixel <WarpedPixel@users.noreply.github.com>
   * Sam Edwards <sam@samedwards.ca>
   * wjielai <gouki0123@gmail.com>
+  * Muffin King <jinxz_k@live.com>
+  * Christopher Stewart <6573710+1f47a@users.noreply.github.com>
+  * Russell Cattelan <cattelan@digitalelves.com>

docs/content/changelog.md

@@ -5,6 +5,36 @@ description: "Rclone Changelog"
 
 # Changelog
 
+## v1.53.1 - 2020-09-13
+
+[See commits](https://github.com/rclone/rclone/compare/v1.53.0...v1.53.1)
+
+* Bug Fixes
+    * accounting: Remove new line from end of --stats-one-line display (Nick Craig-Wood)
+    * check
+        * Add back missing --download flag (Nick Craig-Wood)
+        * Fix docs (Nick Craig-Wood)
+    * docs
+        * Note --log-file does append (Nick Craig-Wood)
+        * Add full stops for consistency in rclone --help (edwardxml)
+        * Add Tencent COS to s3 provider list (wjielai)
+        * Updated mount command to reflect that it requires Go 1.13 or newer (Evan Harris)
+        * jottacloud: Mention that uploads from local disk will not need to cache files to disk for md5 calculation (albertony)
+        * Fix formatting of rc docs page (Nick Craig-Wood)
+    * build
+        * Include vendor tar ball in release and fix startdev (Nick Craig-Wood)
+        * Fix "Illegal instruction" error for ARMv6 builds (Nick Craig-Wood)
+        * Fix architecture name in ARMv7 build (Nick Craig-Wood)
+* VFS
+    * Fix spurious error "vfs cache: failed to _ensure cache EOF" (Nick Craig-Wood)
+    * Log an ERROR if we fail to set the file to be sparse (Nick Craig-Wood)
+* Local
+    * Log an ERROR if we fail to set the file to be sparse (Nick Craig-Wood)
+* Drive
+    * Re-adds special oauth help text (Tim Gallant)
+* Opendrive
+    * Do not retry 400 errors (Evan Harris)
+
 ## v1.53.0 - 2020-09-02
 
 [See commits](https://github.com/rclone/rclone/compare/v1.52.0...v1.53.0)

docs/content/s3.md

@@ -139,7 +139,7 @@ Choose a number from below, or type in your own value
    / Asia Pacific (Mumbai)
 13 | Needs location constraint ap-south-1.
    \ "ap-south-1"
-   / Asia Patific (Hong Kong) Region
+   / Asia Pacific (Hong Kong) Region
 14 | Needs location constraint ap-east-1.
    \ "ap-east-1"
    / South America (Sao Paulo) Region

docs/content/sftp.md

@@ -102,7 +102,7 @@ excess files in the directory.
 The SFTP remote supports three authentication methods:
 
   * Password
-  * Key file
+  * Key file, including certificate signed keys
   * ssh-agent
 
 Key files should be PEM-encoded private key files. For instance `/home/$USER/.ssh/id_rsa`.
@@ -128,6 +128,17 @@ Using an ssh-agent is the only way to load encrypted OpenSSH keys at the moment.
 If you set the `--sftp-ask-password` option, rclone will prompt for a
 password when needed and no password has been configured.
 
+If you have a certificate then you can provide the path to the public key that contains the certificate.  For example:
+
+```
+[remote]
+type = sftp
+host = example.com
+user = sftpuser
+key_file = ~/id_rsa
+pubkey_file = ~/id_rsa-cert.pub
+````
+
 ### ssh-agent on macOS ###
 
 Note that there seem to be various problems with using an ssh-agent on
@@ -247,6 +258,18 @@ when the ssh-agent contains many keys.
 - Type:        bool
 - Default:     false
 
+#### --sftp-pubkey-file
+
+Path to public key file, set if you want to use certificate based authentication
+
+Leading `~` will be expanded in the file name as will environment variables such as `${RCLONE_CONFIG_DIR}`.
+
+
+- Config:      pubkey_file
+- Env Var:     RCLONE_SFTP_PUBKEY_FILE
+- Type:        string
+- Default:     ""
+
 #### --sftp-use-insecure-cipher
 
 Enable the use of insecure ciphers and key exchange methods. 

fs/accounting/stats_groups.go

@@ -366,6 +366,8 @@ func (sg *statsGroups) sum() *StatsInfo {
 				sum.lastError = stats.lastError
 			}
 			sum.startedTransfers = append(sum.startedTransfers, stats.startedTransfers...)
+			sum.oldDuration += stats.oldDuration
+			sum.oldTimeRanges = append(sum.oldTimeRanges, stats.oldTimeRanges...)
 		}
 		stats.mu.RUnlock()
 	}

fs/accounting/stats_groups_test.go

@@ -4,8 +4,10 @@ import (
 	"fmt"
 	"runtime"
 	"testing"
+	"time"
 
 	"github.com/rclone/rclone/fstest/testy"
+	"github.com/stretchr/testify/assert"
 )
 
 func TestStatsGroupOperations(t *testing.T) {
@@ -43,17 +45,26 @@ func TestStatsGroupOperations(t *testing.T) {
 		t.Parallel()
 		stats1 := NewStats()
 		stats1.bytes = 5
-		stats1.errors = 5
+		stats1.errors = 6
+		stats1.oldDuration = time.Second
+		stats1.oldTimeRanges = []timeRange{{time.Now(), time.Now().Add(time.Second)}}
 		stats2 := NewStats()
+		stats2.bytes = 10
+		stats2.errors = 12
+		stats2.oldDuration = 2 * time.Second
+		stats2.oldTimeRanges = []timeRange{{time.Now(), time.Now().Add(2 * time.Second)}}
 		sg := newStatsGroups()
 		sg.set("test1", stats1)
 		sg.set("test2", stats2)
 		sum := sg.sum()
-		if sum.bytes != stats1.bytes+stats2.bytes {
-			t.Fatalf("sum() => bytes %d, expected %d", sum.bytes, stats1.bytes+stats2.bytes)
-		}
-		if sum.errors != stats1.errors+stats2.errors {
-			t.Fatalf("sum() => errors %d, expected %d", sum.errors, stats1.errors+stats2.errors)
+		assert.Equal(t, stats1.bytes+stats2.bytes, sum.bytes)
+		assert.Equal(t, stats1.errors+stats2.errors, sum.errors)
+		assert.Equal(t, stats1.oldDuration+stats2.oldDuration, sum.oldDuration)
+		// dict can iterate in either order
+		a := timeRanges{stats1.oldTimeRanges[0], stats2.oldTimeRanges[0]}
+		b := timeRanges{stats2.oldTimeRanges[0], stats1.oldTimeRanges[0]}
+		if !assert.ObjectsAreEqual(a, sum.oldTimeRanges) {
+			assert.Equal(t, b, sum.oldTimeRanges)
 		}
 	})
 

fs/accounting/transfermap.go

@@ -72,8 +72,16 @@ func (tm *transferMap) _sortedSlice() []*Transfer {
 	for _, tr := range tm.items {
 		s = append(s, tr)
 	}
+	// sort by time first and if equal by name.  Note that the relatively
+	// low time resolution on Windows can cause equal times.
 	sort.Slice(s, func(i, j int) bool {
-		return s[i].startedAt.Before(s[j].startedAt)
+		a, b := s[i], s[j]
+		if a.startedAt.Before(b.startedAt) {
+			return true
+		} else if !a.startedAt.Equal(b.startedAt) {
+			return false
+		}
+		return a.remote < b.remote
 	})
 	return s
 }

fstest/test_all/config.yaml

@@ -41,15 +41,13 @@ backends:
    remote:   "TestChunkerChunk3bNometaLocal:"
    fastlist: true
    maxfile:  6k
- # Disable chunker with mailru tests until Mailru is fixed - see
- # https://github.com/rclone/rclone/issues/4376
- # - backend:  "chunker"
- #   remote:   "TestChunkerMailru:"
- #   fastlist: true
- # - backend:  "chunker"
- #   remote:   "TestChunkerChunk50bMailru:"
- #   fastlist: true
- #   maxfile:  10k
+ - backend:  "chunker"
+   remote:   "TestChunkerMailru:"
+   fastlist: true
+ - backend:  "chunker"
+   remote:   "TestChunkerChunk50bMailru:"
+   fastlist: true
+   maxfile:  10k
  - backend:  "chunker"
    remote:   "TestChunkerChunk50bYandex:"
    fastlist: true
@@ -73,6 +71,10 @@ backends:
    remote:   "TestChunkerChunk50bSHA1HashS3:"
    fastlist: true
    maxfile:  1k
+ - backend:  "chunker"
+   remote:   "TestChunkerOverCrypt:"
+   fastlist: true
+   maxfile:  6k
  - backend:  "chunker"
    remote:   "TestChunkerChunk50bMD5QuickS3:"
    fastlist: true

go.mod

@@ -72,5 +72,3 @@ require (
 	gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 // indirect
 	storj.io/uplink v1.2.0
 )
-
-replace github.com/jlaffaye/ftp => github.com/ncw/ftp v0.0.0-20200913100848-1d4a278dbc96

go.sum

@@ -238,8 +238,6 @@ github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M
 github.com/jlaffaye/ftp v0.0.0-20190624084859-c1312a7102bf/go.mod h1:lli8NYPQOFy3O++YmYbqVgOcQ1JPCwdOy+5zSjKJ9qY=
 github.com/jlaffaye/ftp v0.0.0-20200720194710-13949d38913e h1:itZyHiOkiB8mIGouegRNLM9LttGQ3yrgRmp/J/6H/0g=
 github.com/jlaffaye/ftp v0.0.0-20200720194710-13949d38913e/go.mod h1:2lmrmq866uF2tnje75wQHzmPXhmSWUt7Gyx2vgK1RCU=
-github.com/jlaffaye/ftp v0.0.0-20200812143550-39e3779af0db h1:e30IC+OuZIeMVK33/zE7wDvxDaRmGuRt/ps67pzcxAw=
-github.com/jlaffaye/ftp v0.0.0-20200812143550-39e3779af0db/go.mod h1:2lmrmq866uF2tnje75wQHzmPXhmSWUt7Gyx2vgK1RCU=
 github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc=
 github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
@@ -308,16 +306,6 @@ github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
-github.com/ncw/ftp v0.0.0-20200910202626-83bf7f3051fe h1:vJHusE940z05SKcdSy67YvPeZi9DkV7k1awGFKhCwXs=
-github.com/ncw/ftp v0.0.0-20200910202626-83bf7f3051fe/go.mod h1:2lmrmq866uF2tnje75wQHzmPXhmSWUt7Gyx2vgK1RCU=
-github.com/ncw/ftp v0.0.0-20200911161945-13c89676f4e4 h1:eLu30+/U5nLCkFegOUsGTFJ7wojikvdNESGEmA3ZnuM=
-github.com/ncw/ftp v0.0.0-20200911161945-13c89676f4e4/go.mod h1:2lmrmq866uF2tnje75wQHzmPXhmSWUt7Gyx2vgK1RCU=
-github.com/ncw/ftp v0.0.0-20200912103546-f95f53b8ef3e h1:UUzL92dVtNe6OQfRo+xxubN4Mj+chJro87Qa0bM+3Kg=
-github.com/ncw/ftp v0.0.0-20200912103546-f95f53b8ef3e/go.mod h1:2lmrmq866uF2tnje75wQHzmPXhmSWUt7Gyx2vgK1RCU=
-github.com/ncw/ftp v0.0.0-20200913095915-9bc640dd2108 h1:SBBlbVBRqSAb4KBfpQkevTEAmWOsX8exrX3XxNY/QOQ=
-github.com/ncw/ftp v0.0.0-20200913095915-9bc640dd2108/go.mod h1:2lmrmq866uF2tnje75wQHzmPXhmSWUt7Gyx2vgK1RCU=
-github.com/ncw/ftp v0.0.0-20200913100848-1d4a278dbc96 h1:DIvfe9Wqr8cuJulz8+guLCGyZRO22MDRjCk4fTvJ+7U=
-github.com/ncw/ftp v0.0.0-20200913100848-1d4a278dbc96/go.mod h1:2lmrmq866uF2tnje75wQHzmPXhmSWUt7Gyx2vgK1RCU=
 github.com/ncw/go-acd v0.0.0-20171120105400-887eb06ab6a2 h1:VlXvEx6JbFp7F9iz92zXP2Ew+9VupSpfybr+TxmjdH0=
 github.com/ncw/go-acd v0.0.0-20171120105400-887eb06ab6a2/go.mod h1:MLIrzg7gp/kzVBxRE1olT7CWYMCklcUWU+ekoxOD9x0=
 github.com/ncw/swift v1.0.52 h1:ACF3JufDGgeKp/9mrDgQlEgS8kRYC4XKcuzj/8EJjQU=

vfs/vfscache/item.go

@@ -239,8 +239,23 @@ func (item *Item) _truncate(size int64) (err error) {
 	// Use open handle if available
 	fd := item.fd
 	if fd == nil {
+		// If the metadata says we have some blockes cached then the
+		// file should exist, so open without O_CREATE
+		oFlags := os.O_WRONLY
+		if item.info.Rs.Size() == 0 {
+			oFlags |= os.O_CREATE
+		}
 		osPath := item.c.toOSPath(item.name) // No locking in Cache
-		fd, err = file.OpenFile(osPath, os.O_CREATE|os.O_WRONLY, 0600)
+		fd, err = file.OpenFile(osPath, oFlags, 0600)
+		if err != nil && os.IsNotExist(err) {
+			// If the metadata has info but the file doesn't
+			// not exist then it has been externally removed
+			fs.Errorf(item.name, "vfs cache: detected external removal of cache file")
+			item.info.Rs = nil      // show we have no blocks cached
+			item.info.Dirty = false // file can't be dirty if it doesn't exist
+			item._removeMeta("cache file externally deleted")
+			fd, err = file.OpenFile(osPath, os.O_CREATE|os.O_WRONLY, 0600)
+		}
 		if err != nil {
 			return errors.Wrap(err, "vfs cache: truncate: failed to open cache file")
 		}