jottacloud: avoid double url escaping of device/mountpoint - fixes #4697

.github/workflows/build.yml

@@ -124,8 +124,6 @@ jobs:
       - name: Install Libraries on macOS
         shell: bash
         run: |
-          brew untap local/homebrew-openssl  # workaround for https://github.com/actions/virtual-environments/issues/1811
-          brew untap local/homebrew-python2  # workaround for https://github.com/actions/virtual-environments/issues/1811
           brew update
           brew cask install osxfuse
         if: matrix.os == 'macOS-latest'

backend/s3/s3.go

@@ -1523,7 +1523,7 @@ func s3Connection(opt *Options) (*s3.S3, *session.Session, error) {
 			if req.Config.Credentials == credentials.AnonymousCredentials {
 				return
 			}
-			v2sign(opt, req.HTTPRequest)
+			sign(v.AccessKeyID, v.SecretAccessKey, req.HTTPRequest)
 		}
 		c.Handlers.Sign.Clear()
 		c.Handlers.Sign.PushBackNamed(corehandlers.BuildContentLengthHandler)
@@ -3202,12 +3202,6 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	// Read the metadata from the newly created object
 	o.meta = nil // wipe old metadata
 	err = o.readMetaData(ctx)
-	// Empty an Etag which is a valid md5sum for multipart
-	// uploads. This works around a bug in KS3 where the ETag is a
-	// correctly formed md5sum for multpart uploads
-	if multipart && matchMd5.MatchString(strings.Trim(strings.ToLower(o.etag), `"`)) {
-		o.etag = ""
-	}
 	return err
 }
 

backend/s3/v2sign.go

@@ -9,10 +9,7 @@ import (
 	"net/http"
 	"sort"
 	"strings"
-	"sync"
 	"time"
-
-	"github.com/rclone/rclone/fs"
 )
 
 // URL parameters that need to be added to the signature
@@ -36,20 +33,12 @@ var s3ParamsToSign = map[string]struct{}{
 	"response-cache-control":       {},
 	"response-content-disposition": {},
 	"response-content-encoding":    {},
-	"lifecycle":                    {},
-	"website":                      {},
-	"delete":                       {},
-	"cors":                         {},
-	"restore":                      {},
 }
 
-// Warn once about empty endpoint
-var warnEmptyEndpointOnce sync.Once
-
 // sign signs requests using v2 auth
 //
 // Cobbled together from goamz and aws-sdk-go
-func v2sign(opt *Options, req *http.Request) {
+func sign(AccessKey, SecretKey string, req *http.Request) {
 	// Set date
 	date := time.Now().UTC().Format(time.RFC1123)
 	req.Header.Set("Date", date)
@@ -59,26 +48,11 @@ func v2sign(opt *Options, req *http.Request) {
 	if uri == "" {
 		uri = "/"
 	}
-	// If not using path style then need to stick the bucket on
-	// the start of the requests if doing a bucket based query
-	if !opt.ForcePathStyle {
-		if opt.Endpoint == "" {
-			warnEmptyEndpointOnce.Do(func() {
-				fs.Logf(nil, "If using v2 auth with AWS and force_path_style=false, endpoint must be set in the config")
-			})
-		} else if req.URL.Host != opt.Endpoint {
-			// read the bucket off the start of the hostname
-			i := strings.IndexRune(req.URL.Host, '.')
-			if i >= 0 {
-				uri = "/" + req.URL.Host[:i] + uri
-			}
-		}
-	}
 
 	// Look through headers of interest
 	var md5 string
 	var contentType string
-	var headersToSign [][2]string // slice of key, value pairs
+	var headersToSign []string
 	for k, v := range req.Header {
 		k = strings.ToLower(k)
 		switch k {
@@ -89,26 +63,15 @@ func v2sign(opt *Options, req *http.Request) {
 		default:
 			if strings.HasPrefix(k, "x-amz-") {
 				vall := strings.Join(v, ",")
-				headersToSign = append(headersToSign, [2]string{k, vall})
+				headersToSign = append(headersToSign, k+":"+vall)
 			}
 		}
 	}
 	// Make headers of interest into canonical string
 	var joinedHeadersToSign string
 	if len(headersToSign) > 0 {
-		// sort by keys
-		sort.Slice(headersToSign, func(i, j int) bool {
-			return headersToSign[i][0] < headersToSign[j][0]
-		})
-		// join into key:value\n
-		var out strings.Builder
-		for _, kv := range headersToSign {
-			out.WriteString(kv[0])
-			out.WriteRune(':')
-			out.WriteString(kv[1])
-			out.WriteRune('\n')
-		}
-		joinedHeadersToSign = out.String()
+		sort.StringSlice(headersToSign).Sort()
+		joinedHeadersToSign = strings.Join(headersToSign, "\n") + "\n"
 	}
 
 	// Look for query parameters which need to be added to the signature
@@ -133,11 +96,11 @@ func v2sign(opt *Options, req *http.Request) {
 
 	// Make signature
 	payload := req.Method + "\n" + md5 + "\n" + contentType + "\n" + date + "\n" + joinedHeadersToSign + uri
-	hash := hmac.New(sha1.New, []byte(opt.SecretAccessKey))
+	hash := hmac.New(sha1.New, []byte(SecretKey))
 	_, _ = hash.Write([]byte(payload))
 	signature := make([]byte, base64.StdEncoding.EncodedLen(hash.Size()))
 	base64.StdEncoding.Encode(signature, hash.Sum(nil))
 
 	// Set signature in request
-	req.Header.Set("Authorization", "AWS "+opt.AccessKeyID+":"+string(signature))
+	req.Header.Set("Authorization", "AWS "+AccessKey+":"+string(signature))
 }

backend/union/union.go

@@ -145,16 +145,11 @@ func (f *Fs) Hashes() hash.Set {
 // Mkdir makes the root directory of the Fs object
 func (f *Fs) Mkdir(ctx context.Context, dir string) error {
 	upstreams, err := f.create(ctx, dir)
-	if err == fs.ErrorObjectNotFound {
-		if dir != parentDir(dir) {
-			if err := f.Mkdir(ctx, parentDir(dir)); err != nil {
-				return err
-			}
-			upstreams, err = f.create(ctx, dir)
-		} else if dir == "" {
-			// If root dirs not created then create them
-			upstreams, err = f.upstreams, nil
+	if err == fs.ErrorObjectNotFound && dir != parentDir(dir) {
+		if err := f.Mkdir(ctx, parentDir(dir)); err != nil {
+			return err
 		}
+		upstreams, err = f.create(ctx, dir)
 	}
 	if err != nil {
 		return err
@@ -823,7 +818,6 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	if err != nil {
 		return nil, err
 	}
-	fs.Debugf(f, "actionPolicy = %T, createPolicy = %T, searchPolicy = %T", f.actionPolicy, f.createPolicy, f.searchPolicy)
 	var features = (&fs.Features{
 		CaseInsensitive:         true,
 		DuplicateFiles:          false,

cmd/genautocomplete/genautocomplete_bash.go

@@ -2,7 +2,6 @@ package genautocomplete
 
 import (
 	"log"
-	"os"
 
 	"github.com/rclone/rclone/cmd"
 	"github.com/spf13/cobra"
@@ -30,20 +29,11 @@ them directly
 
 If you supply a command line argument the script will be written
 there.
-
-If output_file is "-", then the output will be written to stdout.
 `,
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(0, 1, command, args)
 		out := "/etc/bash_completion.d/rclone"
 		if len(args) > 0 {
-			if args[0] == "-" {
-				err := cmd.Root.GenBashCompletion(os.Stdout)
-				if err != nil {
-					log.Fatal(err)
-				}
-				return
-			}
 			out = args[0]
 		}
 		err := cmd.Root.GenBashCompletionFile(out)

cmd/genautocomplete/genautocomplete_fish.go

@@ -2,7 +2,6 @@ package genautocomplete
 
 import (
 	"log"
-	"os"
 
 	"github.com/rclone/rclone/cmd"
 	"github.com/spf13/cobra"
@@ -30,20 +29,11 @@ them directly
 
 If you supply a command line argument the script will be written
 there.
-
-If output_file is "-", then the output will be written to stdout.
 `,
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(0, 1, command, args)
 		out := "/etc/fish/completions/rclone.fish"
 		if len(args) > 0 {
-			if args[0] == "-" {
-				err := cmd.Root.GenFishCompletion(os.Stdout, true)
-				if err != nil {
-					log.Fatal(err)
-				}
-				return
-			}
 			out = args[0]
 		}
 		err := cmd.Root.GenFishCompletionFile(out, true)

cmd/genautocomplete/genautocomplete_test.go

@@ -11,10 +11,8 @@ import (
 func TestCompletionBash(t *testing.T) {
 	tempFile, err := ioutil.TempFile("", "completion_bash")
 	assert.NoError(t, err)
-	defer func() {
-		_ = tempFile.Close()
-		_ = os.Remove(tempFile.Name())
-	}()
+	defer func() { _ = tempFile.Close() }()
+	defer func() { _ = os.Remove(tempFile.Name()) }()
 
 	bashCommandDefinition.Run(bashCommandDefinition, []string{tempFile.Name()})
 
@@ -23,32 +21,11 @@ func TestCompletionBash(t *testing.T) {
 	assert.NotEmpty(t, string(bs))
 }
 
-func TestCompletionBashStdout(t *testing.T) {
-	originalStdout := os.Stdout
-	tempFile, err := ioutil.TempFile("", "completion_zsh")
-	assert.NoError(t, err)
-	defer func() {
-		_ = tempFile.Close()
-		_ = os.Remove(tempFile.Name())
-	}()
-
-	os.Stdout = tempFile
-	defer func() { os.Stdout = originalStdout }()
-
-	bashCommandDefinition.Run(bashCommandDefinition, []string{"-"})
-
-	output, err := ioutil.ReadFile(tempFile.Name())
-	assert.NoError(t, err)
-	assert.NotEmpty(t, string(output))
-}
-
 func TestCompletionZsh(t *testing.T) {
 	tempFile, err := ioutil.TempFile("", "completion_zsh")
 	assert.NoError(t, err)
-	defer func() {
-		_ = tempFile.Close()
-		_ = os.Remove(tempFile.Name())
-	}()
+	defer func() { _ = tempFile.Close() }()
+	defer func() { _ = os.Remove(tempFile.Name()) }()
 
 	zshCommandDefinition.Run(zshCommandDefinition, []string{tempFile.Name()})
 
@@ -57,31 +34,11 @@ func TestCompletionZsh(t *testing.T) {
 	assert.NotEmpty(t, string(bs))
 }
 
-func TestCompletionZshStdout(t *testing.T) {
-	originalStdout := os.Stdout
-	tempFile, err := ioutil.TempFile("", "completion_zsh")
-	assert.NoError(t, err)
-	defer func() {
-		_ = tempFile.Close()
-		_ = os.Remove(tempFile.Name())
-	}()
-
-	os.Stdout = tempFile
-	defer func() { os.Stdout = originalStdout }()
-
-	zshCommandDefinition.Run(zshCommandDefinition, []string{"-"})
-	output, err := ioutil.ReadFile(tempFile.Name())
-	assert.NoError(t, err)
-	assert.NotEmpty(t, string(output))
-}
-
 func TestCompletionFish(t *testing.T) {
 	tempFile, err := ioutil.TempFile("", "completion_fish")
 	assert.NoError(t, err)
-	defer func() {
-		_ = tempFile.Close()
-		_ = os.Remove(tempFile.Name())
-	}()
+	defer func() { _ = tempFile.Close() }()
+	defer func() { _ = os.Remove(tempFile.Name()) }()
 
 	fishCommandDefinition.Run(fishCommandDefinition, []string{tempFile.Name()})
 
@@ -89,22 +46,3 @@ func TestCompletionFish(t *testing.T) {
 	assert.NoError(t, err)
 	assert.NotEmpty(t, string(bs))
 }
-
-func TestCompletionFishStdout(t *testing.T) {
-	originalStdout := os.Stdout
-	tempFile, err := ioutil.TempFile("", "completion_zsh")
-	assert.NoError(t, err)
-	defer func() {
-		_ = tempFile.Close()
-		_ = os.Remove(tempFile.Name())
-	}()
-
-	os.Stdout = tempFile
-	defer func() { os.Stdout = originalStdout }()
-
-	fishCommandDefinition.Run(fishCommandDefinition, []string{"-"})
-
-	output, err := ioutil.ReadFile(tempFile.Name())
-	assert.NoError(t, err)
-	assert.NotEmpty(t, string(output))
-}

cmd/genautocomplete/genautocomplete_zsh.go

@@ -30,20 +30,11 @@ them directly
 
 If you supply a command line argument the script will be written
 there.
-
-If output_file is "-", then the output will be written to stdout.
 `,
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(0, 1, command, args)
 		out := "/usr/share/zsh/vendor-completions/_rclone"
 		if len(args) > 0 {
-			if args[0] == "-" {
-				err := cmd.Root.GenZshCompletion(os.Stdout)
-				if err != nil {
-					log.Fatal(err)
-				}
-				return
-			}
 			out = args[0]
 		}
 		outFile, err := os.Create(out)

docs/content/authors.md

@@ -421,4 +421,3 @@ put them back in again.` >}}
   * Dan Hipschman <dan.hipschman@opendoor.com>
   * Josh Soref <jsoref@users.noreply.github.com>
   * David <david@staron.nl>
-  * Ingo <ingo@hoffmann.cx>

docs/content/commands/rclone_genautocomplete_bash.md

@@ -27,7 +27,6 @@ them directly
 If you supply a command line argument the script will be written
 there.
 
-If output_file is `-`, then the output will be written to stdout.
 
 ```
 rclone genautocomplete bash [output_file] [flags]

docs/content/commands/rclone_genautocomplete_fish.md

@@ -27,7 +27,6 @@ them directly
 If you supply a command line argument the script will be written
 there.
 
-If output_file is `-`, then the output will be written to stdout.
 
 ```
 rclone genautocomplete fish [output_file] [flags]

docs/content/commands/rclone_genautocomplete_zsh.md

@@ -27,7 +27,6 @@ them directly
 If you supply a command line argument the script will be written
 there.
 
-If output_file is `-`, then the output will be written to stdout.
 
 ```
 rclone genautocomplete zsh [output_file] [flags]

fstest/test_all/config.go

@@ -27,16 +27,15 @@ type Test struct {
 //
 // FIXME make bucket based remotes set sub-dir automatically???
 type Backend struct {
-	Backend     string   // name of the backend directory
-	Remote      string   // name of the test remote
-	FastList    bool     // set to test with -fast-list
-	Short       bool     // set to test with -short
-	OneOnly     bool     // set to run only one backend test at once
-	MaxFile     string   // file size limit
-	CleanUp     bool     // when running clean, run cleanup first
-	Ignore      []string // test names to ignore the failure of
-	Tests       []string // paths of tests to run, blank for all
-	ListRetries int      // -list-retries if > 0
+	Backend  string   // name of the backend directory
+	Remote   string   // name of the test remote
+	FastList bool     // set to test with -fast-list
+	Short    bool     // set to test with -short
+	OneOnly  bool     // set to run only one backend test at once
+	MaxFile  string   // file size limit
+	CleanUp  bool     // when running clean, run cleanup first
+	Ignore   []string // test names to ignore the failure of
+	Tests    []string // paths of tests to run, blank for all
 }
 
 // includeTest returns true if this backend should be included in this
@@ -80,17 +79,16 @@ func (b *Backend) MakeRuns(t *Test) (runs []*Run) {
 			continue
 		}
 		run := &Run{
-			Remote:      b.Remote,
-			Backend:     b.Backend,
-			Path:        t.Path,
-			FastList:    fastlist,
-			Short:       (b.Short && t.Short),
-			NoRetries:   t.NoRetries,
-			OneOnly:     b.OneOnly,
-			NoBinary:    t.NoBinary,
-			SizeLimit:   int64(maxSize),
-			Ignore:      ignore,
-			ListRetries: b.ListRetries,
+			Remote:    b.Remote,
+			Backend:   b.Backend,
+			Path:      t.Path,
+			FastList:  fastlist,
+			Short:     (b.Short && t.Short),
+			NoRetries: t.NoRetries,
+			OneOnly:   b.OneOnly,
+			NoBinary:  t.NoBinary,
+			SizeLimit: int64(maxSize),
+			Ignore:    ignore,
 		}
 		if t.AddBackend {
 			run.Path = path.Join(run.Path, b.Backend)

fstest/test_all/config.yaml

@@ -20,7 +20,6 @@ backends:
  - backend:  "b2"
    remote:   "TestB2:"
    fastlist: true
-   listretries: 5
  - backend:  "crypt"
    remote:   "TestCryptDrive:"
    fastlist: true
@@ -147,12 +146,12 @@ backends:
  #   ignore:
  #     - TestIntegration/FsMkdir/FsPutFiles/FsCopy
  #     - TestIntegration/FsMkdir/FsPutFiles/SetTier
- # - backend:  "s3"
- #   remote:   "TestS3Ceph:"
- #   fastlist: true
- #   ignore:
- #     - TestIntegration/FsMkdir/FsPutFiles/FsCopy
- #     - TestIntegration/FsMkdir/FsPutFiles/SetTier
+ - backend:  "s3"
+   remote:   "TestS3Ceph:"
+   fastlist: true
+   ignore:
+     - TestIntegration/FsMkdir/FsPutFiles/FsCopy
+     - TestIntegration/FsMkdir/FsPutFiles/SetTier
  - backend:  "s3"
    remote:   "TestS3Alibaba:"
    fastlist: true
@@ -173,11 +172,11 @@ backends:
  - backend:  "swift"
    remote:   "TestSwift:"
    fastlist: true
- # - backend:  "swift"
- #   remote:   "TestSwiftCeph:"
- #   fastlist: true
- #   ignore:
- #     - TestIntegration/FsMkdir/FsPutFiles/FsCopy
+ - backend:  "swift"
+   remote:   "TestSwiftCeph:"
+   fastlist: true
+   ignore:
+     - TestIntegration/FsMkdir/FsPutFiles/FsCopy
  - backend:  "yandex"
    remote:   "TestYandex:"
    fastlist: false

fstest/test_all/run.go

@@ -35,17 +35,16 @@ var (
 // if retries are needed.
 type Run struct {
 	// Config
-	Remote      string // name of the test remote
-	Backend     string // name of the backend
-	Path        string // path to the source directory
-	FastList    bool   // add -fast-list to tests
-	Short       bool   // add -short
-	NoRetries   bool   // don't retry if set
-	OneOnly     bool   // only run test for this backend at once
-	NoBinary    bool   // set to not build a binary
-	SizeLimit   int64  // maximum test file size
-	Ignore      map[string]struct{}
-	ListRetries int // -list-retries if > 0
+	Remote    string // name of the test remote
+	Backend   string // name of the backend
+	Path      string // path to the source directory
+	FastList  bool   // add -fast-list to tests
+	Short     bool   // add -short
+	NoRetries bool   // don't retry if set
+	OneOnly   bool   // only run test for this backend at once
+	NoBinary  bool   // set to not build a binary
+	SizeLimit int64  // maximum test file size
+	Ignore    map[string]struct{}
 	// Internals
 	CmdLine     []string
 	CmdString   string
@@ -337,12 +336,8 @@ func (r *Run) Init() {
 		r.CmdLine = []string{"./" + r.BinaryName()}
 	}
 	r.CmdLine = append(r.CmdLine, prefix+"v", prefix+"timeout", timeout.String(), "-remote", r.Remote)
-	listRetries := *listRetries
-	if r.ListRetries > 0 {
-		listRetries = r.ListRetries
-	}
-	if listRetries > 0 {
-		r.CmdLine = append(r.CmdLine, "-list-retries", fmt.Sprint(listRetries))
+	if *listRetries > 0 {
+		r.CmdLine = append(r.CmdLine, "-list-retries", fmt.Sprint(*listRetries))
 	}
 	r.Try = 1
 	if *verbose {

go.sum

@@ -400,7 +400,6 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=