mount: use the equivalent of kernel_cache by default #FIXME WIP

Before this copyto would parse windows paths incorrectly.

This change moves the parsing code into fspath and makes sure
fspath.Split calls fspath.Parse which does the parsing correctly for

This also renames fspath.RemoteParse to fspath.Parse for consistency

backend/googlecloudstorage/googlecloudstorage.go

@@ -480,7 +480,7 @@ func (f *Fs) list(dir string, recurse bool, fn listFn) (err error) {
 			remote := object.Name[rootLength:]
 			// is this a directory marker?
 			if (strings.HasSuffix(remote, "/") || remote == "") && object.Size == 0 {
-				if recurse {
+				if recurse && remote != "" {
 					// add a directory in if --fast-list since will have no prefixes
 					err = fn(remote[:len(remote)-1], object, true)
 					if err != nil {

backend/s3/s3.go

@@ -864,7 +864,7 @@ func (f *Fs) list(dir string, recurse bool, fn listFn) error {
 			remote := key[rootLength:]
 			// is this a directory marker?
 			if (strings.HasSuffix(remote, "/") || remote == "") && *object.Size == 0 {
-				if recurse {
+				if recurse && remote != "" {
 					// add a directory in if --fast-list since will have no prefixes
 					remote = remote[:len(remote)-1]
 					err = fn(remote, &s3.Object{Key: &remote}, true)

backend/sftp/sftp.go

@@ -76,7 +76,7 @@ func init() {
 			Optional: true,
 		}, {
 			Name:     "use_insecure_cipher",
-			Help:     "Enable the user of the aes128-cbc cipher. This cipher is insecure and may allow plaintext data to be recovered by an attacker.",
+			Help:     "Enable the use of the aes128-cbc cipher. This cipher is insecure and may allow plaintext data to be recovered by an attacker.",
 			Optional: true,
 			Examples: []fs.OptionExample{
 				{

cmd/cmd.go

@@ -245,7 +245,7 @@ func NewFsSrcDstFiles(args []string) (fsrc fs.Fs, srcFileName string, fdst fs.Fs
 	// If file exists then srcFileName != "", however if the file
 	// doesn't exist then we assume it is a directory...
 	if srcFileName != "" {
-		dstRemote, dstFileName = fspath.RemoteSplit(dstRemote)
+		dstRemote, dstFileName = fspath.Split(dstRemote)
 		if dstRemote == "" {
 			dstRemote = "."
 		}
@@ -268,7 +268,7 @@ func NewFsSrcDstFiles(args []string) (fsrc fs.Fs, srcFileName string, fdst fs.Fs
 
 // NewFsDstFile creates a new dst fs with a destination file name from the arguments
 func NewFsDstFile(args []string) (fdst fs.Fs, dstFileName string) {
-	dstRemote, dstFileName := fspath.RemoteSplit(args[0])
+	dstRemote, dstFileName := fspath.Split(args[0])
 	if dstRemote == "" {
 		dstRemote = "."
 	}

cmd/mount/file.go

@@ -79,6 +79,9 @@ func (f *File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.OpenR
 		resp.Flags |= fuse.OpenNonSeekable
 	}
 
+	// Keep the file in the cache - equivalent of kernel_cache
+	resp.Flags |= fuse.OpenKeepCache
+
 	return &FileHandle{handle}, nil
 }
 

docs/content/authors.md

@@ -169,3 +169,5 @@ Contributors
   * Kasper Byrdal Nielsen <byrdal76@gmail.com>
   * Benjamin Joseph Dag <bjdag1234@users.noreply.github.com>
   * themylogin <themylogin@gmail.com>
+  * Onno Zweers <onno.zweers@surfsara.nl>
+  * Jasper Lievisse Adriaanse <jasper@humppa.nl>

docs/content/azureblob.md

@@ -117,6 +117,39 @@ MD5 hashes are stored with blobs.  However blobs that were uploaded in
 chunks only have an MD5 if the source remote was capable of MD5
 hashes, eg the local disk.
 
+### Authenticating with Azure Blob Storage
+
+Rclone has 3 ways of authenticating with Azure Blob Storage:
+
+#### Account and Key
+
+This is the most straight forward and least flexible way.  Just fill in the `account` and `key` lines and leave the rest blank.
+
+#### Connection string
+
+This supports all the possible connection string variants.  Leave `account`, `key` and `sas_url` blank and put the connection string into the `connection_string` configuration parameter.
+
+Use this method if using an account level SAS; the Azure Portal shows connection strings you can cut and paste.
+
+#### SAS URL
+
+This only for a container level SAS URL - it does not work with an account level SAS URL. For account level SAS use the connection string method.
+
+To use it leave `account`, `key` and `connection_string` blank and fill in `sas_url`.
+
+To get a container level SAS URL right click on a container in the Azure Blob explorer in the Azure portal.
+
+You will only be able to use the container specified in the SAS URL with rclone, eg
+
+    rclone ls azureblob:container
+
+However these will not work
+
+    rclone lsd azureblob:
+    rclone ls azureblob:othercontainer
+
+This would be useful for temporarily allowing third parties access to a single container or putting credentials into an untrusted environment.
+
 ### Multipart uploads ###
 
 Rclone supports multipart uploads with Azure Blob storage.  Files

docs/content/webdav.md

@@ -189,6 +189,9 @@ pass = encryptedpassword
 
 ### dCache ###
 
+dCache is a storage system with WebDAV doors that support, beside basic and x509, 
+authentication with [Macaroons](https://www.dcache.org/manuals/workshop-2017-05-29-Umea/000-Final/anupam_macaroons_v02.pdf) (bearer tokens).
+
 Configure as normal using the `other` type.  Don't enter a username or
 password, instead enter your Macaroon as the `bearer_token`.
 
@@ -203,3 +206,6 @@ user =
 pass =
 bearer_token = your-macaroon
 ```
+
+There is a [script](https://github.com/onnozweers/dcache-scripts/blob/master/get-share-link) that
+obtains a Macaroon from a dCache WebDAV endpoint, and creates an rclone config file.

fs/config/config.go

@@ -30,6 +30,7 @@ import (
 	"github.com/ncw/rclone/fs/config/obscure"
 	"github.com/ncw/rclone/fs/driveletter"
 	"github.com/ncw/rclone/fs/fshttp"
+	"github.com/ncw/rclone/fs/fspath"
 	"github.com/pkg/errors"
 	"golang.org/x/crypto/nacl/secretbox"
 	"golang.org/x/text/unicode/norm"
@@ -865,12 +866,12 @@ func NewRemoteName() (name string) {
 	for {
 		fmt.Printf("name> ")
 		name = ReadLine()
-		parts := fs.Matcher.FindStringSubmatch(name + ":")
+		parts := fspath.Matcher.FindStringSubmatch(name + ":")
 		switch {
 		case name == "":
 			fmt.Printf("Can't use empty name.\n")
 		case driveletter.IsDriveLetter(name):
-			fmt.Printf("Can't use %q as it can be confused a drive letter.\n", name)
+			fmt.Printf("Can't use %q as it can be confused with a drive letter.\n", name)
 		case parts == nil:
 			fmt.Printf("Can't use %q as it has invalid characters in it.\n", name)
 		default:

fs/fs.go

@@ -9,12 +9,11 @@ import (
 	"os"
 	"path/filepath"
 	"reflect"
-	"regexp"
 	"sort"
 	"strings"
 	"time"
 
-	"github.com/ncw/rclone/fs/driveletter"
+	"github.com/ncw/rclone/fs/fspath"
 	"github.com/ncw/rclone/fs/hash"
 	"github.com/pkg/errors"
 )
@@ -786,24 +785,20 @@ func MustFind(name string) *RegInfo {
 	return fs
 }
 
-// Matcher is a pattern to match an rclone URL
-var Matcher = regexp.MustCompile(`^([\w_ -]+):(.*)$`)
-
 // ParseRemote deconstructs a path into configName, fsPath, looking up
 // the fsName in the config file (returning NotFoundInConfigFile if not found)
 func ParseRemote(path string) (fsInfo *RegInfo, configName, fsPath string, err error) {
-	parts := Matcher.FindStringSubmatch(path)
+	configName, fsPath = fspath.Parse(path)
 	var fsName string
-	fsName, configName, fsPath = "local", "local", path
-	if parts != nil && !driveletter.IsDriveLetter(parts[1]) {
-		configName, fsPath = parts[1], parts[2]
+	if configName != "" {
 		fsName = ConfigFileGet(configName, "type")
 		if fsName == "" {
 			return nil, "", "", ErrorNotFoundInConfigFile
 		}
+	} else {
+		fsName = "local"
+		configName = "local"
 	}
-	// change native directory separators to / if there are any
-	fsPath = filepath.ToSlash(fsPath)
 	fsInfo, err = Find(fsName)
 	return fsInfo, configName, fsPath, err
 }

fs/fspath/path.go

@@ -3,27 +3,46 @@ package fspath
 
 import (
 	"path"
-	"strings"
+	"path/filepath"
+	"regexp"
+
+	"github.com/ncw/rclone/fs/driveletter"
 )
 
-// RemoteSplit splits a remote into a parent and a leaf
+// Matcher is a pattern to match an rclone URL
+var Matcher = regexp.MustCompile(`^([\w_ -]+):(.*)$`)
+
+// Parse deconstructs a remote path into configName and fsPath
+//
+// If the path is a local path then configName will be returned as "".
+//
+// So "remote:path/to/dir" will return "remote", "path/to/dir"
+// and "/path/to/local" will return ("", "/path/to/local")
+//
+// Note that this will turn \ into / in the fsPath on Windows
+func Parse(path string) (configName, fsPath string) {
+	parts := Matcher.FindStringSubmatch(path)
+	configName, fsPath = "", path
+	if parts != nil && !driveletter.IsDriveLetter(parts[1]) {
+		configName, fsPath = parts[1], parts[2]
+	}
+	// change native directory separators to / if there are any
+	fsPath = filepath.ToSlash(fsPath)
+	return configName, fsPath
+}
+
+// Split splits a remote into a parent and a leaf
 //
 // if it returns leaf as an empty string then remote is a directory
 //
 // if it returns parent as an empty string then that means the current directory
 //
 // The returned values have the property that parent + leaf == remote
-func RemoteSplit(remote string) (parent string, leaf string) {
-	// Split remote on :
-	i := strings.Index(remote, ":")
-	remoteName := ""
-	remotePath := remote
-	if i >= 0 {
-		remoteName = remote[:i+1]
-		remotePath = remote[i+1:]
-	} else if strings.HasSuffix(remotePath, "/") {
-		// if no : and ends with / must be directory
-		return remotePath, ""
+// (except under Windows where \ will be translated into /)
+func Split(remote string) (parent string, leaf string) {
+	remoteName, remotePath := Parse(remote)
+	if remoteName != "" {
+		remoteName += ":"
 	}
 	// Construct new remote name without last segment
 	parent, leaf = path.Split(remotePath)

fs/fspath/path_test.go

@@ -7,8 +7,23 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-func TestRemoteSplit(t *testing.T) {
+func TestParse(t *testing.T) {
+	for _, test := range []struct {
+		in, wantConfigName, wantFsPath string
+	}{
+		{"", "", ""},
+		{"/path/to/file", "", "/path/to/file"},
+		{"path/to/file", "", "path/to/file"},
+		{"remote:path/to/file", "remote", "path/to/file"},
+		{"remote:/path/to/file", "remote", "/path/to/file"},
+	} {
+		gotConfigName, gotFsPath := Parse(test.in)
+		assert.Equal(t, test.wantConfigName, gotConfigName)
+		assert.Equal(t, test.wantFsPath, gotFsPath)
+	}
+}
 
+func TestSplit(t *testing.T) {
 	for _, test := range []struct {
 		remote, wantParent, wantLeaf string
 	}{
@@ -27,7 +42,7 @@ func TestRemoteSplit(t *testing.T) {
 		{"root/", "root/", ""},
 		{"a/b/", "a/b/", ""},
 	} {
-		gotParent, gotLeaf := RemoteSplit(test.remote)
+		gotParent, gotLeaf := Split(test.remote)
 		assert.Equal(t, test.wantParent, gotParent, test.remote)
 		assert.Equal(t, test.wantLeaf, gotLeaf, test.remote)
 		assert.Equal(t, test.remote, gotParent+gotLeaf, fmt.Sprintf("%s: %q + %q != %q", test.remote, gotParent, gotLeaf, test.remote))