log: update windows redirection code to use x/sys/windows and include dup

This modernises the Windows redirect code to use x/sys/windows and to
dup os.Stderr for use in the terminal.

.github/workflows/build.yml

@@ -97,12 +97,12 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 0
 
       - name: Install Go
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v3
         with:
           stable: 'false'
           go-version: ${{ matrix.go }}
@@ -162,7 +162,7 @@ jobs:
           env
 
       - name: Go module cache
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: ~/go/pkg/mod
           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
@@ -226,7 +226,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Code quality test
         uses: golangci/golangci-lint-action@v3
@@ -242,18 +242,18 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 0
 
       # Upgrade together with NDK version
       - name: Set up Go
-        uses: actions/setup-go@v1
+        uses: actions/setup-go@v3
         with:
           go-version: 1.19.x
 
       - name: Go module cache
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: ~/go/pkg/mod
           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}

.github/workflows/build_publish_docker_image.yml

@@ -12,7 +12,7 @@ jobs:
         name: Build image job
         steps:
             - name: Checkout master
-              uses: actions/checkout@v2
+              uses: actions/checkout@v3
               with:
                 fetch-depth: 0
             - name: Build and publish image

.github/workflows/build_publish_release_docker_image.yml

@@ -11,7 +11,7 @@ jobs:
         name: Build image job
         steps:
             - name: Checkout master
-              uses: actions/checkout@v2
+              uses: actions/checkout@v3
               with:
                 fetch-depth: 0
             - name: Get actual patch version
@@ -40,7 +40,7 @@ jobs:
         name: Build docker plugin job
         steps:
             - name: Checkout master
-              uses: actions/checkout@v2
+              uses: actions/checkout@v3
               with:
                 fetch-depth: 0
             - name: Build and publish docker plugin

backend/ftp/ftp.go

@@ -336,14 +336,44 @@ func (f *Fs) ftpConnection(ctx context.Context) (c *ftp.ServerConn, err error) {
 	fs.Debugf(f, "Connecting to FTP server")
 
 	// Make ftp library dial with fshttp dialer optionally using TLS
+	initialConnection := true
 	dial := func(network, address string) (conn net.Conn, err error) {
+		fs.Debugf(f, "dial(%q,%q)", network, address)
+		defer func() {
+			fs.Debugf(f, "> dial: conn=%T, err=%v", conn, err)
+		}()
 		conn, err = fshttp.NewDialer(ctx).Dial(network, address)
-		if f.tlsConf != nil && err == nil {
-			conn = tls.Client(conn, f.tlsConf)
+		if err != nil {
+			return nil, err
 		}
-		return
+		// Connect using cleartext only for non TLS
+		if f.tlsConf == nil {
+			return conn, nil
+		}
+		// Initial connection only needs to be cleartext for explicit TLS
+		if f.opt.ExplicitTLS && initialConnection {
+			initialConnection = false
+			return conn, nil
+		}
+		// Upgrade connection to TLS
+		tlsConn := tls.Client(conn, f.tlsConf)
+		// Do the initial handshake - tls.Client doesn't do it for us
+		// If we do this then connections to proftpd/pureftpd lock up
+		// See: https://github.com/rclone/rclone/issues/6426
+		// See: https://github.com/jlaffaye/ftp/issues/282
+		if false {
+			err = tlsConn.HandshakeContext(ctx)
+			if err != nil {
+				_ = conn.Close()
+				return nil, err
+			}
+		}
+		return tlsConn, nil
+	}
+	ftpConfig := []ftp.DialOption{
+		ftp.DialWithContext(ctx),
+		ftp.DialWithDialFunc(dial),
 	}
-	ftpConfig := []ftp.DialOption{ftp.DialWithDialFunc(dial)}
 
 	if f.opt.TLS {
 		// Our dialer takes care of TLS but ftp library also needs tlsConf
@@ -351,12 +381,6 @@ func (f *Fs) ftpConnection(ctx context.Context) (c *ftp.ServerConn, err error) {
 		ftpConfig = append(ftpConfig, ftp.DialWithTLS(f.tlsConf))
 	} else if f.opt.ExplicitTLS {
 		ftpConfig = append(ftpConfig, ftp.DialWithExplicitTLS(f.tlsConf))
-		// Initial connection needs to be cleartext for explicit TLS
-		conn, err := fshttp.NewDialer(ctx).Dial("tcp", f.dialAddr)
-		if err != nil {
-			return nil, err
-		}
-		ftpConfig = append(ftpConfig, ftp.DialWithNetConn(conn))
 	}
 	if f.opt.DisableEPSV {
 		ftpConfig = append(ftpConfig, ftp.DialWithDisabledEPSV(true))

backend/ftp/ftp_internal_test.go

@@ -34,9 +34,9 @@ func deriveFs(ctx context.Context, t *testing.T, f fs.Fs, opts settings) fs.Fs {
 // test that big file uploads do not cause network i/o timeout
 func (f *Fs) testUploadTimeout(t *testing.T) {
 	const (
-		fileSize    = 100000000             // 100 MiB
-		idleTimeout = 40 * time.Millisecond // small because test server is local
-		maxTime     = 10 * time.Second      // prevent test hangup
+		fileSize    = 100000000        // 100 MiB
+		idleTimeout = 1 * time.Second  // small because test server is local
+		maxTime     = 10 * time.Second // prevent test hangup
 	)
 
 	if testing.Short() {

backend/s3/s3.go

@@ -1566,8 +1566,21 @@ isn't set then "acl" is used instead.`,
 				Help:  "arn:aws:kms:*",
 			}},
 		}, {
-			Name:     "sse_customer_key",
-			Help:     "If using SSE-C you must provide the secret encryption key used to encrypt/decrypt your data.",
+			Name: "sse_customer_key",
+			Help: `To use SSE-C you may provide the secret encryption key used to encrypt/decrypt your data.
+
+Alternatively you can provide --sse-customer-key-base64.`,
+			Provider: "AWS,Ceph,ChinaMobile,Minio",
+			Advanced: true,
+			Examples: []fs.OptionExample{{
+				Value: "",
+				Help:  "None",
+			}},
+		}, {
+			Name: "sse_customer_key_base64",
+			Help: `If using SSE-C you must provide the secret encryption key encoded in base64 format to encrypt/decrypt your data.
+
+Alternatively you can provide --sse-customer-key.`,
 			Provider: "AWS,Ceph,ChinaMobile,Minio",
 			Advanced: true,
 			Examples: []fs.OptionExample{{
@@ -2142,6 +2155,7 @@ type Options struct {
 	SSEKMSKeyID           string               `config:"sse_kms_key_id"`
 	SSECustomerAlgorithm  string               `config:"sse_customer_algorithm"`
 	SSECustomerKey        string               `config:"sse_customer_key"`
+	SSECustomerKeyBase64  string               `config:"sse_customer_key_base64"`
 	SSECustomerKeyMD5     string               `config:"sse_customer_key_md5"`
 	StorageClass          string               `config:"storage_class"`
 	UploadCutoff          fs.SizeSuffix        `config:"upload_cutoff"`
@@ -2679,6 +2693,16 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	if opt.BucketACL == "" {
 		opt.BucketACL = opt.ACL
 	}
+	if opt.SSECustomerKeyBase64 != "" && opt.SSECustomerKey != "" {
+		return nil, errors.New("s3: can't use sse_customer_key and sse_customer_key_base64 at the same time")
+	} else if opt.SSECustomerKeyBase64 != "" {
+		// Decode the base64-encoded key and store it in the SSECustomerKey field
+		decoded, err := base64.StdEncoding.DecodeString(opt.SSECustomerKeyBase64)
+		if err != nil {
+			return nil, fmt.Errorf("s3: Could not decode sse_customer_key_base64: %w", err)
+		}
+		opt.SSECustomerKey = string(decoded)
+	}
 	if opt.SSECustomerKey != "" && opt.SSECustomerKeyMD5 == "" {
 		// calculate CustomerKeyMD5 if not supplied
 		md5sumBinary := md5.Sum([]byte(opt.SSECustomerKey))

backend/zoho/zoho.go

@@ -670,28 +670,27 @@ func isSimpleName(s string) bool {
 }
 
 func (f *Fs) upload(ctx context.Context, name string, parent string, size int64, in io.Reader, options ...fs.OpenOption) (*api.Item, error) {
-	uploadID := random.String(20) // random upload ID
-	opts := rest.Opts{
-		Method: "POST",
-		//RootURL: "https://upload.zoho.com/workdrive-api/v1",
-		RootURL:       "https://upload.zoho.eu/workdrive-api/v1",
-		Path:          "/stream/upload",
-		Body:          in,
-		ContentType:   fs.MimeTypeFromName(name), // FIXME should read mime type of original object
-		ContentLength: &size,
-		Options:       options,
-		ExtraHeaders: map[string]string{
-			"x-filename":   name,
-			"x-parent_id":  parent,
-			"upload-id":    uploadID,
-			"x-streammode": "1",
-		},
-	}
-	if size < 0 {
-		opts.ContentLength = nil
+	params := url.Values{}
+	params.Set("filename", name)
+	params.Set("parent_id", parent)
+	params.Set("override-name-exist", strconv.FormatBool(true))
+	formReader, contentType, overhead, err := rest.MultipartUpload(ctx, in, nil, "content", name)
+	if err != nil {
+		return nil, fmt.Errorf("failed to make multipart upload: %w", err)
+	}
+
+	contentLength := overhead + size
+	opts := rest.Opts{
+		Method:           "POST",
+		Path:             "/upload",
+		Body:             formReader,
+		ContentType:      contentType,
+		ContentLength:    &contentLength,
+		Options:          options,
+		Parameters:       params,
+		TransferEncoding: []string{"identity"},
 	}
 
-	var err error
 	var resp *http.Response
 	var uploadResponse *api.UploadResponse
 	err = f.pacer.CallNoRetry(func() (bool, error) {

cmd/progress.go

@@ -62,7 +62,7 @@ func startProgress() func() {
 				printProgress("")
 				fs.LogPrint = oldLogPrint
 				operations.SyncPrintf = oldSyncPrint
-				fmt.Println("")
+				fmt.Fprintln(terminal.Out, "")
 				return
 			}
 		}

docs/content/authors.md

@@ -640,3 +640,4 @@ put them back in again.` >}}
   * Øyvind Heddeland Instefjord <instefjord@outlook.com>
   * Dmitry Deniskin <110819396+ddeniskin@users.noreply.github.com>
   * Alexander Knorr <106825+opexxx@users.noreply.github.com>
+  * Richard Bateman <richard@batemansr.us>

docs/content/commands/rclone_mount.md

@@ -341,8 +341,6 @@ mount sftp1:subdir /mnt/data -t rclone -o vfs_cache_mode=writes,sftp_key_file=/p
 or create systemd mount units:
 ```
 # /etc/systemd/system/mnt-data.mount
-[Unit]
-After=network-online.target
 [Mount]
 Type=rclone
 What=sftp1:subdir
@@ -354,7 +352,6 @@ optionally accompanied by systemd automount unit
 ```
 # /etc/systemd/system/mnt-data.automount
 [Unit]
-After=network-online.target
 Before=remote-fs.target
 [Automount]
 Where=/mnt/data

docs/content/install.md

@@ -312,9 +312,10 @@ go build -trimpath -ldflags -s -tags cmount
 ```
 
 Instead of executing the `go build` command directly, you can run it via the
-Makefile, which also sets version information and copies the resulting rclone
-executable into your GOPATH bin folder (`$(go env GOPATH)/bin`, which
-corresponds to `~/go/bin/rclone` by default).
+Makefile. It changes the version number suffix from "-DEV" to "-beta" and
+appends commit details. It also copies the resulting rclone executable into
+your GOPATH bin folder (`$(go env GOPATH)/bin`, which corresponds to
+`~/go/bin/rclone` by default).
 
 ```
 make
@@ -326,7 +327,15 @@ To include mount command on macOS and Windows with Makefile build:
 make GOTAGS=cmount
 ```
 
-As an alternative you can download the source, build and install rclone in one
+There are other make targets that can be used for more advanced builds,
+such as cross-compiling for all supported os/architectures, embedding
+icon and version info resources into windows executable, and packaging
+results into release artifacts.
+See [Makefile](https://github.com/rclone/rclone/blob/master/Makefile)
+and [cross-compile.go](https://github.com/rclone/rclone/blob/master/bin/cross-compile.go)
+for details.
+
+Another alternative is to download the source, build and install rclone in one
 operation, as a regular Go package. The source will be stored it in the Go
 module cache, and the resulting executable will be in your GOPATH bin folder
 (`$(go env GOPATH)/bin`, which corresponds to `~/go/bin/rclone` by default).

fs/config/crypt.go

@@ -26,9 +26,6 @@ var (
 	// When nil, no encryption will be used for saving.
 	configKey []byte
 
-	// PasswordPromptOutput is output of prompt for password
-	PasswordPromptOutput = os.Stderr
-
 	// PassConfigKeyForDaemonization if set to true, the configKey
 	// is obscured with obscure.Obscure and saved to a temp file
 	// when it is calculated from the password. The path of that

fs/config/ui.go

@@ -716,9 +716,9 @@ func checkPassword(password string) (string, error) {
 
 // GetPassword asks the user for a password with the prompt given.
 func GetPassword(prompt string) string {
-	_, _ = fmt.Fprintln(PasswordPromptOutput, prompt)
+	_, _ = fmt.Fprintln(terminal.Out, prompt)
 	for {
-		_, _ = fmt.Fprint(PasswordPromptOutput, "password:")
+		_, _ = fmt.Fprint(terminal.Out, "password:")
 		password := ReadPassword()
 		password, err := checkPassword(password)
 		if err == nil {

fs/log/redirect_stderr_unix.go

@@ -9,17 +9,17 @@ import (
 	"log"
 	"os"
 
-	"github.com/rclone/rclone/fs/config"
+	"github.com/rclone/rclone/lib/terminal"
 	"golang.org/x/sys/unix"
 )
 
 // redirectStderr to the file passed in
 func redirectStderr(f *os.File) {
-	passPromptFd, err := unix.Dup(int(os.Stderr.Fd()))
+	termFd, err := unix.Dup(int(os.Stderr.Fd()))
 	if err != nil {
 		log.Fatalf("Failed to duplicate stderr: %v", err)
 	}
-	config.PasswordPromptOutput = os.NewFile(uintptr(passPromptFd), "passPrompt")
+	terminal.RawOut = os.NewFile(uintptr(termFd), "termOut")
 	err = unix.Dup2(int(f.Fd()), int(os.Stderr.Fd()))
 	if err != nil {
 		log.Fatalf("Failed to redirect stderr to file: %v", err)

fs/log/redirect_stderr_windows.go

@@ -12,29 +12,43 @@ package log
 import (
 	"log"
 	"os"
-	"syscall"
+
+	"github.com/rclone/rclone/lib/terminal"
+	"golang.org/x/sys/windows"
 )
 
-var (
-	kernel32         = syscall.MustLoadDLL("kernel32.dll")
-	procSetStdHandle = kernel32.MustFindProc("SetStdHandle")
-)
-
-func setStdHandle(stdhandle int32, handle syscall.Handle) error {
-	r0, _, e1 := syscall.Syscall(procSetStdHandle.Addr(), 2, uintptr(stdhandle), uintptr(handle), 0)
-	if r0 == 0 {
-		if e1 != 0 {
-			return error(e1)
-		}
-		return syscall.EINVAL
+// dup oldfd creating a functional copy as newfd
+// conceptually the same as the unix `dup()` function
+func dup(oldfd uintptr) (newfd uintptr, err error) {
+	var (
+		newfdHandle   windows.Handle
+		processHandle = windows.CurrentProcess()
+	)
+	err = windows.DuplicateHandle(
+		processHandle,                 // hSourceProcessHandle
+		windows.Handle(oldfd),         // hSourceHandle
+		processHandle,                 // hTargetProcessHandle
+		&newfdHandle,                  // lpTargetHandle
+		0,                             // dwDesiredAccess
+		true,                          // bInheritHandle
+		windows.DUPLICATE_SAME_ACCESS, // dwOptions
+	)
+	if err != nil {
+		return 0, err
 	}
-	return nil
+	return uintptr(newfdHandle), nil
 }
 
 // redirectStderr to the file passed in
 func redirectStderr(f *os.File) {
-	err := setStdHandle(syscall.STD_ERROR_HANDLE, syscall.Handle(f.Fd()))
+	termFd, err := dup(os.Stderr.Fd())
+	if err != nil {
+		log.Fatalf("Failed to duplicate stderr: %v", err)
+	}
+	terminal.RawOut = os.NewFile(termFd, "termOut")
+	err = windows.SetStdHandle(windows.STD_ERROR_HANDLE, windows.Handle(f.Fd()))
 	if err != nil {
 		log.Fatalf("Failed to redirect stderr to file: %v", err)
 	}
+	os.Stderr = f
 }

lib/terminal/terminal.go

@@ -68,17 +68,23 @@ const (
 var (
 	// make sure that start is only called once
 	once sync.Once
+
+	// RawOut is the underlying *os.File intended for terminal output
+	RawOut = os.Stderr
 )
 
 // Start the terminal - must be called before use
 func Start() {
 	once.Do(func() {
-		f := os.Stdout
+		f := RawOut
 		if !IsTerminal(int(f.Fd())) {
-			// If stdout not a tty then remove escape codes
+			// If output is not a tty then remove escape codes
 			Out = colorable.NewNonColorable(f)
 		} else if runtime.GOOS == "windows" && os.Getenv("TERM") != "" {
-			// If TERM is set just use stdout
+			// If TERM is set on Windows then we should just send output
+			// straight to the terminal for cygwin/git bash environments.
+			// We don't want to use NewColorable here because it will
+			// use Windows console calls which cygwin/git bash don't support.
 			Out = f
 		} else {
 			Out = colorable.NewColorable(f)

lib/terminal/terminal_normal.go

@@ -34,5 +34,5 @@ func ReadPassword(fd int) ([]byte, error) {
 
 // WriteTerminalTitle writes a string to the terminal title
 func WriteTerminalTitle(title string) {
-	fmt.Printf(ChangeTitle + title + BEL)
+	fmt.Fprintf(Out, ChangeTitle+title+BEL)
 }