s3: fix v2 auth for multipart server side copy

Before this change the v2 signer sorted the headers for signing as
joined key:value pairs. However this put these two headers in the
wrong order.

    x-amz-copy-source-range:
    x-amz-copy-source:

This changes sorts on the keys before joining the values producing the
correct sort order.

    x-amz-copy-source:
    x-amz-copy-source-range:

This commit also adds some missing query parameters for signing that I
spotted in the s3cmd source.

.github/workflows/build.yml

@@ -124,6 +124,8 @@ jobs:
       - name: Install Libraries on macOS
         shell: bash
         run: |
+          brew untap local/homebrew-openssl  # workaround for https://github.com/actions/virtual-environments/issues/1811
+          brew untap local/homebrew-python2  # workaround for https://github.com/actions/virtual-environments/issues/1811
           brew update
           brew cask install osxfuse
         if: matrix.os == 'macOS-latest'

backend/jottacloud/jottacloud.go

@@ -553,7 +553,7 @@ func (f *Fs) setEndpointURL() {
 	if f.opt.Mountpoint == "" {
 		f.opt.Mountpoint = defaultMountpoint
 	}
-	f.endpointURL = urlPathEscape(path.Join(f.user, f.opt.Device, f.opt.Mountpoint))
+	f.endpointURL = path.Join(f.user, f.opt.Device, f.opt.Mountpoint)
 }
 
 // readMetaDataForPath reads the metadata from the path

backend/s3/s3.go

@@ -1523,7 +1523,7 @@ func s3Connection(opt *Options) (*s3.S3, *session.Session, error) {
 			if req.Config.Credentials == credentials.AnonymousCredentials {
 				return
 			}
-			sign(v.AccessKeyID, v.SecretAccessKey, req.HTTPRequest)
+			v2sign(opt, req.HTTPRequest)
 		}
 		c.Handlers.Sign.Clear()
 		c.Handlers.Sign.PushBackNamed(corehandlers.BuildContentLengthHandler)
@@ -3202,6 +3202,12 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	// Read the metadata from the newly created object
 	o.meta = nil // wipe old metadata
 	err = o.readMetaData(ctx)
+	// Empty an Etag which is a valid md5sum for multipart
+	// uploads. This works around a bug in KS3 where the ETag is a
+	// correctly formed md5sum for multpart uploads
+	if multipart && matchMd5.MatchString(strings.Trim(strings.ToLower(o.etag), `"`)) {
+		o.etag = ""
+	}
 	return err
 }
 

backend/s3/v2sign.go

@@ -9,7 +9,10 @@ import (
 	"net/http"
 	"sort"
 	"strings"
+	"sync"
 	"time"
+
+	"github.com/rclone/rclone/fs"
 )
 
 // URL parameters that need to be added to the signature
@@ -33,12 +36,20 @@ var s3ParamsToSign = map[string]struct{}{
 	"response-cache-control":       {},
 	"response-content-disposition": {},
 	"response-content-encoding":    {},
+	"lifecycle":                    {},
+	"website":                      {},
+	"delete":                       {},
+	"cors":                         {},
+	"restore":                      {},
 }
 
+// Warn once about empty endpoint
+var warnEmptyEndpointOnce sync.Once
+
 // sign signs requests using v2 auth
 //
 // Cobbled together from goamz and aws-sdk-go
-func sign(AccessKey, SecretKey string, req *http.Request) {
+func v2sign(opt *Options, req *http.Request) {
 	// Set date
 	date := time.Now().UTC().Format(time.RFC1123)
 	req.Header.Set("Date", date)
@@ -48,11 +59,26 @@ func sign(AccessKey, SecretKey string, req *http.Request) {
 	if uri == "" {
 		uri = "/"
 	}
+	// If not using path style then need to stick the bucket on
+	// the start of the requests if doing a bucket based query
+	if !opt.ForcePathStyle {
+		if opt.Endpoint == "" {
+			warnEmptyEndpointOnce.Do(func() {
+				fs.Logf(nil, "If using v2 auth with AWS and force_path_style=false, endpoint must be set in the config")
+			})
+		} else if req.URL.Host != opt.Endpoint {
+			// read the bucket off the start of the hostname
+			i := strings.IndexRune(req.URL.Host, '.')
+			if i >= 0 {
+				uri = "/" + req.URL.Host[:i] + uri
+			}
+		}
+	}
 
 	// Look through headers of interest
 	var md5 string
 	var contentType string
-	var headersToSign []string
+	var headersToSign [][2]string // slice of key, value pairs
 	for k, v := range req.Header {
 		k = strings.ToLower(k)
 		switch k {
@@ -63,15 +89,26 @@ func sign(AccessKey, SecretKey string, req *http.Request) {
 		default:
 			if strings.HasPrefix(k, "x-amz-") {
 				vall := strings.Join(v, ",")
-				headersToSign = append(headersToSign, k+":"+vall)
+				headersToSign = append(headersToSign, [2]string{k, vall})
 			}
 		}
 	}
 	// Make headers of interest into canonical string
 	var joinedHeadersToSign string
 	if len(headersToSign) > 0 {
-		sort.StringSlice(headersToSign).Sort()
-		joinedHeadersToSign = strings.Join(headersToSign, "\n") + "\n"
+		// sort by keys
+		sort.Slice(headersToSign, func(i, j int) bool {
+			return headersToSign[i][0] < headersToSign[j][0]
+		})
+		// join into key:value\n
+		var out strings.Builder
+		for _, kv := range headersToSign {
+			out.WriteString(kv[0])
+			out.WriteRune(':')
+			out.WriteString(kv[1])
+			out.WriteRune('\n')
+		}
+		joinedHeadersToSign = out.String()
 	}
 
 	// Look for query parameters which need to be added to the signature
@@ -96,11 +133,11 @@ func sign(AccessKey, SecretKey string, req *http.Request) {
 
 	// Make signature
 	payload := req.Method + "\n" + md5 + "\n" + contentType + "\n" + date + "\n" + joinedHeadersToSign + uri
-	hash := hmac.New(sha1.New, []byte(SecretKey))
+	hash := hmac.New(sha1.New, []byte(opt.SecretAccessKey))
 	_, _ = hash.Write([]byte(payload))
 	signature := make([]byte, base64.StdEncoding.EncodedLen(hash.Size()))
 	base64.StdEncoding.Encode(signature, hash.Sum(nil))
 
 	// Set signature in request
-	req.Header.Set("Authorization", "AWS "+AccessKey+":"+string(signature))
+	req.Header.Set("Authorization", "AWS "+opt.AccessKeyID+":"+string(signature))
 }

backend/union/union.go

@@ -145,11 +145,16 @@ func (f *Fs) Hashes() hash.Set {
 // Mkdir makes the root directory of the Fs object
 func (f *Fs) Mkdir(ctx context.Context, dir string) error {
 	upstreams, err := f.create(ctx, dir)
-	if err == fs.ErrorObjectNotFound && dir != parentDir(dir) {
-		if err := f.Mkdir(ctx, parentDir(dir)); err != nil {
-			return err
+	if err == fs.ErrorObjectNotFound {
+		if dir != parentDir(dir) {
+			if err := f.Mkdir(ctx, parentDir(dir)); err != nil {
+				return err
+			}
+			upstreams, err = f.create(ctx, dir)
+		} else if dir == "" {
+			// If root dirs not created then create them
+			upstreams, err = f.upstreams, nil
 		}
-		upstreams, err = f.create(ctx, dir)
 	}
 	if err != nil {
 		return err
@@ -818,6 +823,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	if err != nil {
 		return nil, err
 	}
+	fs.Debugf(f, "actionPolicy = %T, createPolicy = %T, searchPolicy = %T", f.actionPolicy, f.createPolicy, f.searchPolicy)
 	var features = (&fs.Features{
 		CaseInsensitive:         true,
 		DuplicateFiles:          false,

cmd/genautocomplete/genautocomplete_bash.go

@@ -2,6 +2,7 @@ package genautocomplete
 
 import (
 	"log"
+	"os"
 
 	"github.com/rclone/rclone/cmd"
 	"github.com/spf13/cobra"
@@ -29,11 +30,20 @@ them directly
 
 If you supply a command line argument the script will be written
 there.
+
+If output_file is "-", then the output will be written to stdout.
 `,
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(0, 1, command, args)
 		out := "/etc/bash_completion.d/rclone"
 		if len(args) > 0 {
+			if args[0] == "-" {
+				err := cmd.Root.GenBashCompletion(os.Stdout)
+				if err != nil {
+					log.Fatal(err)
+				}
+				return
+			}
 			out = args[0]
 		}
 		err := cmd.Root.GenBashCompletionFile(out)

cmd/genautocomplete/genautocomplete_fish.go

@@ -2,6 +2,7 @@ package genautocomplete
 
 import (
 	"log"
+	"os"
 
 	"github.com/rclone/rclone/cmd"
 	"github.com/spf13/cobra"
@@ -29,11 +30,20 @@ them directly
 
 If you supply a command line argument the script will be written
 there.
+
+If output_file is "-", then the output will be written to stdout.
 `,
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(0, 1, command, args)
 		out := "/etc/fish/completions/rclone.fish"
 		if len(args) > 0 {
+			if args[0] == "-" {
+				err := cmd.Root.GenFishCompletion(os.Stdout, true)
+				if err != nil {
+					log.Fatal(err)
+				}
+				return
+			}
 			out = args[0]
 		}
 		err := cmd.Root.GenFishCompletionFile(out, true)

cmd/genautocomplete/genautocomplete_test.go

@@ -11,8 +11,10 @@ import (
 func TestCompletionBash(t *testing.T) {
 	tempFile, err := ioutil.TempFile("", "completion_bash")
 	assert.NoError(t, err)
-	defer func() { _ = tempFile.Close() }()
-	defer func() { _ = os.Remove(tempFile.Name()) }()
+	defer func() {
+		_ = tempFile.Close()
+		_ = os.Remove(tempFile.Name())
+	}()
 
 	bashCommandDefinition.Run(bashCommandDefinition, []string{tempFile.Name()})
 
@@ -21,11 +23,32 @@ func TestCompletionBash(t *testing.T) {
 	assert.NotEmpty(t, string(bs))
 }
 
+func TestCompletionBashStdout(t *testing.T) {
+	originalStdout := os.Stdout
+	tempFile, err := ioutil.TempFile("", "completion_zsh")
+	assert.NoError(t, err)
+	defer func() {
+		_ = tempFile.Close()
+		_ = os.Remove(tempFile.Name())
+	}()
+
+	os.Stdout = tempFile
+	defer func() { os.Stdout = originalStdout }()
+
+	bashCommandDefinition.Run(bashCommandDefinition, []string{"-"})
+
+	output, err := ioutil.ReadFile(tempFile.Name())
+	assert.NoError(t, err)
+	assert.NotEmpty(t, string(output))
+}
+
 func TestCompletionZsh(t *testing.T) {
 	tempFile, err := ioutil.TempFile("", "completion_zsh")
 	assert.NoError(t, err)
-	defer func() { _ = tempFile.Close() }()
-	defer func() { _ = os.Remove(tempFile.Name()) }()
+	defer func() {
+		_ = tempFile.Close()
+		_ = os.Remove(tempFile.Name())
+	}()
 
 	zshCommandDefinition.Run(zshCommandDefinition, []string{tempFile.Name()})
 
@@ -34,11 +57,31 @@ func TestCompletionZsh(t *testing.T) {
 	assert.NotEmpty(t, string(bs))
 }
 
+func TestCompletionZshStdout(t *testing.T) {
+	originalStdout := os.Stdout
+	tempFile, err := ioutil.TempFile("", "completion_zsh")
+	assert.NoError(t, err)
+	defer func() {
+		_ = tempFile.Close()
+		_ = os.Remove(tempFile.Name())
+	}()
+
+	os.Stdout = tempFile
+	defer func() { os.Stdout = originalStdout }()
+
+	zshCommandDefinition.Run(zshCommandDefinition, []string{"-"})
+	output, err := ioutil.ReadFile(tempFile.Name())
+	assert.NoError(t, err)
+	assert.NotEmpty(t, string(output))
+}
+
 func TestCompletionFish(t *testing.T) {
 	tempFile, err := ioutil.TempFile("", "completion_fish")
 	assert.NoError(t, err)
-	defer func() { _ = tempFile.Close() }()
-	defer func() { _ = os.Remove(tempFile.Name()) }()
+	defer func() {
+		_ = tempFile.Close()
+		_ = os.Remove(tempFile.Name())
+	}()
 
 	fishCommandDefinition.Run(fishCommandDefinition, []string{tempFile.Name()})
 
@@ -46,3 +89,22 @@ func TestCompletionFish(t *testing.T) {
 	assert.NoError(t, err)
 	assert.NotEmpty(t, string(bs))
 }
+
+func TestCompletionFishStdout(t *testing.T) {
+	originalStdout := os.Stdout
+	tempFile, err := ioutil.TempFile("", "completion_zsh")
+	assert.NoError(t, err)
+	defer func() {
+		_ = tempFile.Close()
+		_ = os.Remove(tempFile.Name())
+	}()
+
+	os.Stdout = tempFile
+	defer func() { os.Stdout = originalStdout }()
+
+	fishCommandDefinition.Run(fishCommandDefinition, []string{"-"})
+
+	output, err := ioutil.ReadFile(tempFile.Name())
+	assert.NoError(t, err)
+	assert.NotEmpty(t, string(output))
+}

cmd/genautocomplete/genautocomplete_zsh.go

@@ -30,11 +30,20 @@ them directly
 
 If you supply a command line argument the script will be written
 there.
+
+If output_file is "-", then the output will be written to stdout.
 `,
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(0, 1, command, args)
 		out := "/usr/share/zsh/vendor-completions/_rclone"
 		if len(args) > 0 {
+			if args[0] == "-" {
+				err := cmd.Root.GenZshCompletion(os.Stdout)
+				if err != nil {
+					log.Fatal(err)
+				}
+				return
+			}
 			out = args[0]
 		}
 		outFile, err := os.Create(out)

docs/content/authors.md

@@ -421,3 +421,4 @@ put them back in again.` >}}
   * Dan Hipschman <dan.hipschman@opendoor.com>
   * Josh Soref <jsoref@users.noreply.github.com>
   * David <david@staron.nl>
+  * Ingo <ingo@hoffmann.cx>

docs/content/commands/rclone_genautocomplete_bash.md

@@ -27,6 +27,7 @@ them directly
 If you supply a command line argument the script will be written
 there.
 
+If output_file is `-`, then the output will be written to stdout.
 
 ```
 rclone genautocomplete bash [output_file] [flags]

docs/content/commands/rclone_genautocomplete_fish.md

@@ -27,6 +27,7 @@ them directly
 If you supply a command line argument the script will be written
 there.
 
+If output_file is `-`, then the output will be written to stdout.
 
 ```
 rclone genautocomplete fish [output_file] [flags]

docs/content/commands/rclone_genautocomplete_zsh.md

@@ -27,6 +27,7 @@ them directly
 If you supply a command line argument the script will be written
 there.
 
+If output_file is `-`, then the output will be written to stdout.
 
 ```
 rclone genautocomplete zsh [output_file] [flags]

fstest/test_all/config.go

@@ -27,15 +27,16 @@ type Test struct {
 //
 // FIXME make bucket based remotes set sub-dir automatically???
 type Backend struct {
-	Backend  string   // name of the backend directory
-	Remote   string   // name of the test remote
-	FastList bool     // set to test with -fast-list
-	Short    bool     // set to test with -short
-	OneOnly  bool     // set to run only one backend test at once
-	MaxFile  string   // file size limit
-	CleanUp  bool     // when running clean, run cleanup first
-	Ignore   []string // test names to ignore the failure of
-	Tests    []string // paths of tests to run, blank for all
+	Backend     string   // name of the backend directory
+	Remote      string   // name of the test remote
+	FastList    bool     // set to test with -fast-list
+	Short       bool     // set to test with -short
+	OneOnly     bool     // set to run only one backend test at once
+	MaxFile     string   // file size limit
+	CleanUp     bool     // when running clean, run cleanup first
+	Ignore      []string // test names to ignore the failure of
+	Tests       []string // paths of tests to run, blank for all
+	ListRetries int      // -list-retries if > 0
 }
 
 // includeTest returns true if this backend should be included in this
@@ -79,16 +80,17 @@ func (b *Backend) MakeRuns(t *Test) (runs []*Run) {
 			continue
 		}
 		run := &Run{
-			Remote:    b.Remote,
-			Backend:   b.Backend,
-			Path:      t.Path,
-			FastList:  fastlist,
-			Short:     (b.Short && t.Short),
-			NoRetries: t.NoRetries,
-			OneOnly:   b.OneOnly,
-			NoBinary:  t.NoBinary,
-			SizeLimit: int64(maxSize),
-			Ignore:    ignore,
+			Remote:      b.Remote,
+			Backend:     b.Backend,
+			Path:        t.Path,
+			FastList:    fastlist,
+			Short:       (b.Short && t.Short),
+			NoRetries:   t.NoRetries,
+			OneOnly:     b.OneOnly,
+			NoBinary:    t.NoBinary,
+			SizeLimit:   int64(maxSize),
+			Ignore:      ignore,
+			ListRetries: b.ListRetries,
 		}
 		if t.AddBackend {
 			run.Path = path.Join(run.Path, b.Backend)

fstest/test_all/config.yaml

@@ -20,6 +20,7 @@ backends:
  - backend:  "b2"
    remote:   "TestB2:"
    fastlist: true
+   listretries: 5
  - backend:  "crypt"
    remote:   "TestCryptDrive:"
    fastlist: true
@@ -146,12 +147,12 @@ backends:
  #   ignore:
  #     - TestIntegration/FsMkdir/FsPutFiles/FsCopy
  #     - TestIntegration/FsMkdir/FsPutFiles/SetTier
- - backend:  "s3"
-   remote:   "TestS3Ceph:"
-   fastlist: true
-   ignore:
-     - TestIntegration/FsMkdir/FsPutFiles/FsCopy
-     - TestIntegration/FsMkdir/FsPutFiles/SetTier
+ # - backend:  "s3"
+ #   remote:   "TestS3Ceph:"
+ #   fastlist: true
+ #   ignore:
+ #     - TestIntegration/FsMkdir/FsPutFiles/FsCopy
+ #     - TestIntegration/FsMkdir/FsPutFiles/SetTier
  - backend:  "s3"
    remote:   "TestS3Alibaba:"
    fastlist: true
@@ -172,11 +173,11 @@ backends:
  - backend:  "swift"
    remote:   "TestSwift:"
    fastlist: true
- - backend:  "swift"
-   remote:   "TestSwiftCeph:"
-   fastlist: true
-   ignore:
-     - TestIntegration/FsMkdir/FsPutFiles/FsCopy
+ # - backend:  "swift"
+ #   remote:   "TestSwiftCeph:"
+ #   fastlist: true
+ #   ignore:
+ #     - TestIntegration/FsMkdir/FsPutFiles/FsCopy
  - backend:  "yandex"
    remote:   "TestYandex:"
    fastlist: false

fstest/test_all/run.go

@@ -35,16 +35,17 @@ var (
 // if retries are needed.
 type Run struct {
 	// Config
-	Remote    string // name of the test remote
-	Backend   string // name of the backend
-	Path      string // path to the source directory
-	FastList  bool   // add -fast-list to tests
-	Short     bool   // add -short
-	NoRetries bool   // don't retry if set
-	OneOnly   bool   // only run test for this backend at once
-	NoBinary  bool   // set to not build a binary
-	SizeLimit int64  // maximum test file size
-	Ignore    map[string]struct{}
+	Remote      string // name of the test remote
+	Backend     string // name of the backend
+	Path        string // path to the source directory
+	FastList    bool   // add -fast-list to tests
+	Short       bool   // add -short
+	NoRetries   bool   // don't retry if set
+	OneOnly     bool   // only run test for this backend at once
+	NoBinary    bool   // set to not build a binary
+	SizeLimit   int64  // maximum test file size
+	Ignore      map[string]struct{}
+	ListRetries int // -list-retries if > 0
 	// Internals
 	CmdLine     []string
 	CmdString   string
@@ -336,8 +337,12 @@ func (r *Run) Init() {
 		r.CmdLine = []string{"./" + r.BinaryName()}
 	}
 	r.CmdLine = append(r.CmdLine, prefix+"v", prefix+"timeout", timeout.String(), "-remote", r.Remote)
-	if *listRetries > 0 {
-		r.CmdLine = append(r.CmdLine, "-list-retries", fmt.Sprint(*listRetries))
+	listRetries := *listRetries
+	if r.ListRetries > 0 {
+		listRetries = r.ListRetries
+	}
+	if listRetries > 0 {
+		r.CmdLine = append(r.CmdLine, "-list-retries", fmt.Sprint(listRetries))
 	}
 	r.Try = 1
 	if *verbose {

go.sum

@@ -400,6 +400,7 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=