mount: use the equivalent of kernel_cache by default #FIXME WIP

Before this copyto would parse windows paths incorrectly.

This change moves the parsing code into fspath and makes sure
fspath.Split calls fspath.Parse which does the parsing correctly for

This also renames fspath.RemoteParse to fspath.Parse for consistency

backend/googlecloudstorage/googlecloudstorage.go

@@ -480,7 +480,7 @@ func (f *Fs) list(dir string, recurse bool, fn listFn) (err error) {
 			remote := object.Name[rootLength:]
 			// is this a directory marker?
 			if (strings.HasSuffix(remote, "/") || remote == "") && object.Size == 0 {
-				if recurse {
+				if recurse && remote != "" {
 					// add a directory in if --fast-list since will have no prefixes
 					err = fn(remote[:len(remote)-1], object, true)
 					if err != nil {

backend/s3/s3.go

@@ -864,7 +864,7 @@ func (f *Fs) list(dir string, recurse bool, fn listFn) error {
 			remote := key[rootLength:]
 			// is this a directory marker?
 			if (strings.HasSuffix(remote, "/") || remote == "") && *object.Size == 0 {
-				if recurse {
+				if recurse && remote != "" {
 					// add a directory in if --fast-list since will have no prefixes
 					remote = remote[:len(remote)-1]
 					err = fn(remote, &s3.Object{Key: &remote}, true)

backend/sftp/sftp.go

@@ -76,7 +76,7 @@ func init() {
 			Optional: true,
 		}, {
 			Name:     "use_insecure_cipher",
-			Help:     "Enable the user of the aes128-cbc cipher. This cipher is insecure and may allow plaintext data to be recovered by an attacker.",
+			Help:     "Enable the use of the aes128-cbc cipher. This cipher is insecure and may allow plaintext data to be recovered by an attacker.",
 			Optional: true,
 			Examples: []fs.OptionExample{
 				{

cmd/cmd.go

@@ -245,7 +245,7 @@ func NewFsSrcDstFiles(args []string) (fsrc fs.Fs, srcFileName string, fdst fs.Fs
 	// If file exists then srcFileName != "", however if the file
 	// doesn't exist then we assume it is a directory...
 	if srcFileName != "" {
-		dstRemote, dstFileName = fspath.RemoteSplit(dstRemote)
+		dstRemote, dstFileName = fspath.Split(dstRemote)
 		if dstRemote == "" {
 			dstRemote = "."
 		}
@@ -268,7 +268,7 @@ func NewFsSrcDstFiles(args []string) (fsrc fs.Fs, srcFileName string, fdst fs.Fs
 
 // NewFsDstFile creates a new dst fs with a destination file name from the arguments
 func NewFsDstFile(args []string) (fdst fs.Fs, dstFileName string) {
-	dstRemote, dstFileName := fspath.RemoteSplit(args[0])
+	dstRemote, dstFileName := fspath.Split(args[0])
 	if dstRemote == "" {
 		dstRemote = "."
 	}

cmd/cmount/fs.go

@@ -192,7 +192,7 @@ func (fsys *FS) Destroy() {
 
 // Getattr reads the attributes for path
 func (fsys *FS) Getattr(path string, stat *fuse.Stat_t, fh uint64) (errc int) {
-	defer log.Trace(path, "fh=0x%X", fh)("stat=%+v, errc=%v", &stat, &errc)
+	defer log.Trace(path, "fh=0x%X", fh)("errc=%v", &errc)
 	node, _, errc := fsys.getNode(path, fh)
 	if errc == 0 {
 		errc = fsys.stat(node, stat)

cmd/mount/file.go

@@ -79,6 +79,9 @@ func (f *File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.OpenR
 		resp.Flags |= fuse.OpenNonSeekable
 	}
 
+	// Keep the file in the cache - equivalent of kernel_cache
+	resp.Flags |= fuse.OpenKeepCache
+
 	return &FileHandle{handle}, nil
 }
 

docs/content/authors.md

@@ -170,3 +170,4 @@ Contributors
   * Benjamin Joseph Dag <bjdag1234@users.noreply.github.com>
   * themylogin <themylogin@gmail.com>
   * Onno Zweers <onno.zweers@surfsara.nl>
+  * Jasper Lievisse Adriaanse <jasper@humppa.nl>

fs/config/config.go

@@ -30,6 +30,7 @@ import (
 	"github.com/ncw/rclone/fs/config/obscure"
 	"github.com/ncw/rclone/fs/driveletter"
 	"github.com/ncw/rclone/fs/fshttp"
+	"github.com/ncw/rclone/fs/fspath"
 	"github.com/pkg/errors"
 	"golang.org/x/crypto/nacl/secretbox"
 	"golang.org/x/text/unicode/norm"
@@ -865,12 +866,12 @@ func NewRemoteName() (name string) {
 	for {
 		fmt.Printf("name> ")
 		name = ReadLine()
-		parts := fs.Matcher.FindStringSubmatch(name + ":")
+		parts := fspath.Matcher.FindStringSubmatch(name + ":")
 		switch {
 		case name == "":
 			fmt.Printf("Can't use empty name.\n")
 		case driveletter.IsDriveLetter(name):
-			fmt.Printf("Can't use %q as it can be confused a drive letter.\n", name)
+			fmt.Printf("Can't use %q as it can be confused with a drive letter.\n", name)
 		case parts == nil:
 			fmt.Printf("Can't use %q as it has invalid characters in it.\n", name)
 		default:

fs/fs.go

@@ -9,12 +9,11 @@ import (
 	"os"
 	"path/filepath"
 	"reflect"
-	"regexp"
 	"sort"
 	"strings"
 	"time"
 
-	"github.com/ncw/rclone/fs/driveletter"
+	"github.com/ncw/rclone/fs/fspath"
 	"github.com/ncw/rclone/fs/hash"
 	"github.com/pkg/errors"
 )
@@ -786,24 +785,20 @@ func MustFind(name string) *RegInfo {
 	return fs
 }
 
-// Matcher is a pattern to match an rclone URL
-var Matcher = regexp.MustCompile(`^([\w_ -]+):(.*)$`)
-
 // ParseRemote deconstructs a path into configName, fsPath, looking up
 // the fsName in the config file (returning NotFoundInConfigFile if not found)
 func ParseRemote(path string) (fsInfo *RegInfo, configName, fsPath string, err error) {
-	parts := Matcher.FindStringSubmatch(path)
+	configName, fsPath = fspath.Parse(path)
 	var fsName string
-	fsName, configName, fsPath = "local", "local", path
-	if parts != nil && !driveletter.IsDriveLetter(parts[1]) {
-		configName, fsPath = parts[1], parts[2]
+	if configName != "" {
 		fsName = ConfigFileGet(configName, "type")
 		if fsName == "" {
 			return nil, "", "", ErrorNotFoundInConfigFile
 		}
+	} else {
+		fsName = "local"
+		configName = "local"
 	}
-	// change native directory separators to / if there are any
-	fsPath = filepath.ToSlash(fsPath)
 	fsInfo, err = Find(fsName)
 	return fsInfo, configName, fsPath, err
 }

fs/fspath/path.go

@@ -3,27 +3,46 @@ package fspath
 
 import (
 	"path"
-	"strings"
+	"path/filepath"
+	"regexp"
+
+	"github.com/ncw/rclone/fs/driveletter"
 )
 
-// RemoteSplit splits a remote into a parent and a leaf
+// Matcher is a pattern to match an rclone URL
+var Matcher = regexp.MustCompile(`^([\w_ -]+):(.*)$`)
+
+// Parse deconstructs a remote path into configName and fsPath
+//
+// If the path is a local path then configName will be returned as "".
+//
+// So "remote:path/to/dir" will return "remote", "path/to/dir"
+// and "/path/to/local" will return ("", "/path/to/local")
+//
+// Note that this will turn \ into / in the fsPath on Windows
+func Parse(path string) (configName, fsPath string) {
+	parts := Matcher.FindStringSubmatch(path)
+	configName, fsPath = "", path
+	if parts != nil && !driveletter.IsDriveLetter(parts[1]) {
+		configName, fsPath = parts[1], parts[2]
+	}
+	// change native directory separators to / if there are any
+	fsPath = filepath.ToSlash(fsPath)
+	return configName, fsPath
+}
+
+// Split splits a remote into a parent and a leaf
 //
 // if it returns leaf as an empty string then remote is a directory
 //
 // if it returns parent as an empty string then that means the current directory
 //
 // The returned values have the property that parent + leaf == remote
-func RemoteSplit(remote string) (parent string, leaf string) {
-	// Split remote on :
-	i := strings.Index(remote, ":")
-	remoteName := ""
-	remotePath := remote
-	if i >= 0 {
-		remoteName = remote[:i+1]
-		remotePath = remote[i+1:]
-	} else if strings.HasSuffix(remotePath, "/") {
-		// if no : and ends with / must be directory
-		return remotePath, ""
+// (except under Windows where \ will be translated into /)
+func Split(remote string) (parent string, leaf string) {
+	remoteName, remotePath := Parse(remote)
+	if remoteName != "" {
+		remoteName += ":"
 	}
 	// Construct new remote name without last segment
 	parent, leaf = path.Split(remotePath)

fs/fspath/path_test.go

@@ -7,8 +7,23 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-func TestRemoteSplit(t *testing.T) {
+func TestParse(t *testing.T) {
+	for _, test := range []struct {
+		in, wantConfigName, wantFsPath string
+	}{
+		{"", "", ""},
+		{"/path/to/file", "", "/path/to/file"},
+		{"path/to/file", "", "path/to/file"},
+		{"remote:path/to/file", "remote", "path/to/file"},
+		{"remote:/path/to/file", "remote", "/path/to/file"},
+	} {
+		gotConfigName, gotFsPath := Parse(test.in)
+		assert.Equal(t, test.wantConfigName, gotConfigName)
+		assert.Equal(t, test.wantFsPath, gotFsPath)
+	}
+}
 
+func TestSplit(t *testing.T) {
 	for _, test := range []struct {
 		remote, wantParent, wantLeaf string
 	}{
@@ -27,7 +42,7 @@ func TestRemoteSplit(t *testing.T) {
 		{"root/", "root/", ""},
 		{"a/b/", "a/b/", ""},
 	} {
-		gotParent, gotLeaf := RemoteSplit(test.remote)
+		gotParent, gotLeaf := Split(test.remote)
 		assert.Equal(t, test.wantParent, gotParent, test.remote)
 		assert.Equal(t, test.wantLeaf, gotLeaf, test.remote)
 		assert.Equal(t, test.remote, gotParent+gotLeaf, fmt.Sprintf("%s: %q + %q != %q", test.remote, gotParent, gotLeaf, test.remote))