zoho: attempt to fix large file upload - not working DO NOT MERGE

Before this fix, if an error ocurred reading the metadata, it could be
set as nil and then used, causing a crash.

This fix changes the readMetadata function so it returns an error, and
the error is always set if the metadata returned is nil.

.github/workflows/build.yml

@@ -97,12 +97,12 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v2
         with:
           fetch-depth: 0
 
       - name: Install Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v2
         with:
           stable: 'false'
           go-version: ${{ matrix.go }}
@@ -162,7 +162,7 @@ jobs:
           env
 
       - name: Go module cache
-        uses: actions/cache@v3
+        uses: actions/cache@v2
         with:
           path: ~/go/pkg/mod
           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
@@ -226,7 +226,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v2
 
       - name: Code quality test
         uses: golangci/golangci-lint-action@v3
@@ -242,18 +242,18 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v2
         with:
           fetch-depth: 0
 
       # Upgrade together with NDK version
       - name: Set up Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v1
         with:
           go-version: 1.19.x
 
       - name: Go module cache
-        uses: actions/cache@v3
+        uses: actions/cache@v2
         with:
           path: ~/go/pkg/mod
           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}

.github/workflows/build_publish_docker_image.yml

@@ -12,7 +12,7 @@ jobs:
         name: Build image job
         steps:
             - name: Checkout master
-              uses: actions/checkout@v3
+              uses: actions/checkout@v2
               with:
                 fetch-depth: 0
             - name: Build and publish image

.github/workflows/build_publish_release_docker_image.yml

@@ -11,7 +11,7 @@ jobs:
         name: Build image job
         steps:
             - name: Checkout master
-              uses: actions/checkout@v3
+              uses: actions/checkout@v2
               with:
                 fetch-depth: 0
             - name: Get actual patch version
@@ -40,7 +40,7 @@ jobs:
         name: Build docker plugin job
         steps:
             - name: Checkout master
-              uses: actions/checkout@v3
+              uses: actions/checkout@v2
               with:
                 fetch-depth: 0
             - name: Build and publish docker plugin

backend/ftp/ftp.go

@@ -336,44 +336,14 @@ func (f *Fs) ftpConnection(ctx context.Context) (c *ftp.ServerConn, err error) {
 	fs.Debugf(f, "Connecting to FTP server")
 
 	// Make ftp library dial with fshttp dialer optionally using TLS
-	initialConnection := true
 	dial := func(network, address string) (conn net.Conn, err error) {
-		fs.Debugf(f, "dial(%q,%q)", network, address)
-		defer func() {
-			fs.Debugf(f, "> dial: conn=%T, err=%v", conn, err)
-		}()
 		conn, err = fshttp.NewDialer(ctx).Dial(network, address)
-		if err != nil {
-			return nil, err
+		if f.tlsConf != nil && err == nil {
+			conn = tls.Client(conn, f.tlsConf)
 		}
-		// Connect using cleartext only for non TLS
-		if f.tlsConf == nil {
-			return conn, nil
-		}
-		// Initial connection only needs to be cleartext for explicit TLS
-		if f.opt.ExplicitTLS && initialConnection {
-			initialConnection = false
-			return conn, nil
-		}
-		// Upgrade connection to TLS
-		tlsConn := tls.Client(conn, f.tlsConf)
-		// Do the initial handshake - tls.Client doesn't do it for us
-		// If we do this then connections to proftpd/pureftpd lock up
-		// See: https://github.com/rclone/rclone/issues/6426
-		// See: https://github.com/jlaffaye/ftp/issues/282
-		if false {
-			err = tlsConn.HandshakeContext(ctx)
-			if err != nil {
-				_ = conn.Close()
-				return nil, err
-			}
-		}
-		return tlsConn, nil
-	}
-	ftpConfig := []ftp.DialOption{
-		ftp.DialWithContext(ctx),
-		ftp.DialWithDialFunc(dial),
+		return
 	}
+	ftpConfig := []ftp.DialOption{ftp.DialWithDialFunc(dial)}
 
 	if f.opt.TLS {
 		// Our dialer takes care of TLS but ftp library also needs tlsConf
@@ -381,6 +351,12 @@ func (f *Fs) ftpConnection(ctx context.Context) (c *ftp.ServerConn, err error) {
 		ftpConfig = append(ftpConfig, ftp.DialWithTLS(f.tlsConf))
 	} else if f.opt.ExplicitTLS {
 		ftpConfig = append(ftpConfig, ftp.DialWithExplicitTLS(f.tlsConf))
+		// Initial connection needs to be cleartext for explicit TLS
+		conn, err := fshttp.NewDialer(ctx).Dial("tcp", f.dialAddr)
+		if err != nil {
+			return nil, err
+		}
+		ftpConfig = append(ftpConfig, ftp.DialWithNetConn(conn))
 	}
 	if f.opt.DisableEPSV {
 		ftpConfig = append(ftpConfig, ftp.DialWithDisabledEPSV(true))

backend/ftp/ftp_internal_test.go

@@ -34,9 +34,9 @@ func deriveFs(ctx context.Context, t *testing.T, f fs.Fs, opts settings) fs.Fs {
 // test that big file uploads do not cause network i/o timeout
 func (f *Fs) testUploadTimeout(t *testing.T) {
 	const (
-		fileSize    = 100000000        // 100 MiB
-		idleTimeout = 1 * time.Second  // small because test server is local
-		maxTime     = 10 * time.Second // prevent test hangup
+		fileSize    = 100000000             // 100 MiB
+		idleTimeout = 40 * time.Millisecond // small because test server is local
+		maxTime     = 10 * time.Second      // prevent test hangup
 	)
 
 	if testing.Short() {

backend/s3/s3.go

@@ -1566,21 +1566,8 @@ isn't set then "acl" is used instead.`,
 				Help:  "arn:aws:kms:*",
 			}},
 		}, {
-			Name: "sse_customer_key",
-			Help: `To use SSE-C you may provide the secret encryption key used to encrypt/decrypt your data.
-
-Alternatively you can provide --sse-customer-key-base64.`,
-			Provider: "AWS,Ceph,ChinaMobile,Minio",
-			Advanced: true,
-			Examples: []fs.OptionExample{{
-				Value: "",
-				Help:  "None",
-			}},
-		}, {
-			Name: "sse_customer_key_base64",
-			Help: `If using SSE-C you must provide the secret encryption key encoded in base64 format to encrypt/decrypt your data.
-
-Alternatively you can provide --sse-customer-key.`,
+			Name:     "sse_customer_key",
+			Help:     "If using SSE-C you must provide the secret encryption key used to encrypt/decrypt your data.",
 			Provider: "AWS,Ceph,ChinaMobile,Minio",
 			Advanced: true,
 			Examples: []fs.OptionExample{{
@@ -2155,7 +2142,6 @@ type Options struct {
 	SSEKMSKeyID           string               `config:"sse_kms_key_id"`
 	SSECustomerAlgorithm  string               `config:"sse_customer_algorithm"`
 	SSECustomerKey        string               `config:"sse_customer_key"`
-	SSECustomerKeyBase64  string               `config:"sse_customer_key_base64"`
 	SSECustomerKeyMD5     string               `config:"sse_customer_key_md5"`
 	StorageClass          string               `config:"storage_class"`
 	UploadCutoff          fs.SizeSuffix        `config:"upload_cutoff"`
@@ -2693,16 +2679,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	if opt.BucketACL == "" {
 		opt.BucketACL = opt.ACL
 	}
-	if opt.SSECustomerKeyBase64 != "" && opt.SSECustomerKey != "" {
-		return nil, errors.New("s3: can't use sse_customer_key and sse_customer_key_base64 at the same time")
-	} else if opt.SSECustomerKeyBase64 != "" {
-		// Decode the base64-encoded key and store it in the SSECustomerKey field
-		decoded, err := base64.StdEncoding.DecodeString(opt.SSECustomerKeyBase64)
-		if err != nil {
-			return nil, fmt.Errorf("s3: Could not decode sse_customer_key_base64: %w", err)
-		}
-		opt.SSECustomerKey = string(decoded)
-	}
 	if opt.SSECustomerKey != "" && opt.SSECustomerKeyMD5 == "" {
 		// calculate CustomerKeyMD5 if not supplied
 		md5sumBinary := md5.Sum([]byte(opt.SSECustomerKey))

backend/zoho/zoho.go

@@ -670,27 +670,28 @@ func isSimpleName(s string) bool {
 }
 
 func (f *Fs) upload(ctx context.Context, name string, parent string, size int64, in io.Reader, options ...fs.OpenOption) (*api.Item, error) {
-	params := url.Values{}
-	params.Set("filename", name)
-	params.Set("parent_id", parent)
-	params.Set("override-name-exist", strconv.FormatBool(true))
-	formReader, contentType, overhead, err := rest.MultipartUpload(ctx, in, nil, "content", name)
-	if err != nil {
-		return nil, fmt.Errorf("failed to make multipart upload: %w", err)
-	}
-
-	contentLength := overhead + size
+	uploadID := random.String(20) // random upload ID
 	opts := rest.Opts{
-		Method:           "POST",
-		Path:             "/upload",
-		Body:             formReader,
-		ContentType:      contentType,
-		ContentLength:    &contentLength,
-		Options:          options,
-		Parameters:       params,
-		TransferEncoding: []string{"identity"},
+		Method: "POST",
+		//RootURL: "https://upload.zoho.com/workdrive-api/v1",
+		RootURL:       "https://upload.zoho.eu/workdrive-api/v1",
+		Path:          "/stream/upload",
+		Body:          in,
+		ContentType:   fs.MimeTypeFromName(name), // FIXME should read mime type of original object
+		ContentLength: &size,
+		Options:       options,
+		ExtraHeaders: map[string]string{
+			"x-filename":   name,
+			"x-parent_id":  parent,
+			"upload-id":    uploadID,
+			"x-streammode": "1",
+		},
+	}
+	if size < 0 {
+		opts.ContentLength = nil
 	}
 
+	var err error
 	var resp *http.Response
 	var uploadResponse *api.UploadResponse
 	err = f.pacer.CallNoRetry(func() (bool, error) {

cmd/progress.go

@@ -62,7 +62,7 @@ func startProgress() func() {
 				printProgress("")
 				fs.LogPrint = oldLogPrint
 				operations.SyncPrintf = oldSyncPrint
-				fmt.Fprintln(terminal.Out, "")
+				fmt.Println("")
 				return
 			}
 		}

docs/content/authors.md

@@ -640,4 +640,3 @@ put them back in again.` >}}
   * Øyvind Heddeland Instefjord <instefjord@outlook.com>
   * Dmitry Deniskin <110819396+ddeniskin@users.noreply.github.com>
   * Alexander Knorr <106825+opexxx@users.noreply.github.com>
-  * Richard Bateman <richard@batemansr.us>

docs/content/commands/rclone_mount.md

@@ -341,6 +341,8 @@ mount sftp1:subdir /mnt/data -t rclone -o vfs_cache_mode=writes,sftp_key_file=/p
 or create systemd mount units:
 ```
 # /etc/systemd/system/mnt-data.mount
+[Unit]
+After=network-online.target
 [Mount]
 Type=rclone
 What=sftp1:subdir
@@ -352,6 +354,7 @@ optionally accompanied by systemd automount unit
 ```
 # /etc/systemd/system/mnt-data.automount
 [Unit]
+After=network-online.target
 Before=remote-fs.target
 [Automount]
 Where=/mnt/data

docs/content/install.md

@@ -312,10 +312,9 @@ go build -trimpath -ldflags -s -tags cmount
 ```
 
 Instead of executing the `go build` command directly, you can run it via the
-Makefile. It changes the version number suffix from "-DEV" to "-beta" and
-appends commit details. It also copies the resulting rclone executable into
-your GOPATH bin folder (`$(go env GOPATH)/bin`, which corresponds to
-`~/go/bin/rclone` by default).
+Makefile, which also sets version information and copies the resulting rclone
+executable into your GOPATH bin folder (`$(go env GOPATH)/bin`, which
+corresponds to `~/go/bin/rclone` by default).
 
 ```
 make
@@ -327,15 +326,7 @@ To include mount command on macOS and Windows with Makefile build:
 make GOTAGS=cmount
 ```
 
-There are other make targets that can be used for more advanced builds,
-such as cross-compiling for all supported os/architectures, embedding
-icon and version info resources into windows executable, and packaging
-results into release artifacts.
-See [Makefile](https://github.com/rclone/rclone/blob/master/Makefile)
-and [cross-compile.go](https://github.com/rclone/rclone/blob/master/bin/cross-compile.go)
-for details.
-
-Another alternative is to download the source, build and install rclone in one
+As an alternative you can download the source, build and install rclone in one
 operation, as a regular Go package. The source will be stored it in the Go
 module cache, and the resulting executable will be in your GOPATH bin folder
 (`$(go env GOPATH)/bin`, which corresponds to `~/go/bin/rclone` by default).

fs/config/crypt.go

@@ -26,6 +26,9 @@ var (
 	// When nil, no encryption will be used for saving.
 	configKey []byte
 
+	// PasswordPromptOutput is output of prompt for password
+	PasswordPromptOutput = os.Stderr
+
 	// PassConfigKeyForDaemonization if set to true, the configKey
 	// is obscured with obscure.Obscure and saved to a temp file
 	// when it is calculated from the password. The path of that

fs/config/ui.go

@@ -716,9 +716,9 @@ func checkPassword(password string) (string, error) {
 
 // GetPassword asks the user for a password with the prompt given.
 func GetPassword(prompt string) string {
-	_, _ = fmt.Fprintln(terminal.Out, prompt)
+	_, _ = fmt.Fprintln(PasswordPromptOutput, prompt)
 	for {
-		_, _ = fmt.Fprint(terminal.Out, "password:")
+		_, _ = fmt.Fprint(PasswordPromptOutput, "password:")
 		password := ReadPassword()
 		password, err := checkPassword(password)
 		if err == nil {

fs/log/redirect_stderr_unix.go

@@ -9,17 +9,17 @@ import (
 	"log"
 	"os"
 
-	"github.com/rclone/rclone/lib/terminal"
+	"github.com/rclone/rclone/fs/config"
 	"golang.org/x/sys/unix"
 )
 
 // redirectStderr to the file passed in
 func redirectStderr(f *os.File) {
-	termFd, err := unix.Dup(int(os.Stderr.Fd()))
+	passPromptFd, err := unix.Dup(int(os.Stderr.Fd()))
 	if err != nil {
 		log.Fatalf("Failed to duplicate stderr: %v", err)
 	}
-	terminal.RawOut = os.NewFile(uintptr(termFd), "termOut")
+	config.PasswordPromptOutput = os.NewFile(uintptr(passPromptFd), "passPrompt")
 	err = unix.Dup2(int(f.Fd()), int(os.Stderr.Fd()))
 	if err != nil {
 		log.Fatalf("Failed to redirect stderr to file: %v", err)

fs/log/redirect_stderr_windows.go

@@ -12,43 +12,29 @@ package log
 import (
 	"log"
 	"os"
-
-	"github.com/rclone/rclone/lib/terminal"
-	"golang.org/x/sys/windows"
+	"syscall"
 )
 
-// dup oldfd creating a functional copy as newfd
-// conceptually the same as the unix `dup()` function
-func dup(oldfd uintptr) (newfd uintptr, err error) {
-	var (
-		newfdHandle   windows.Handle
-		processHandle = windows.CurrentProcess()
-	)
-	err = windows.DuplicateHandle(
-		processHandle,                 // hSourceProcessHandle
-		windows.Handle(oldfd),         // hSourceHandle
-		processHandle,                 // hTargetProcessHandle
-		&newfdHandle,                  // lpTargetHandle
-		0,                             // dwDesiredAccess
-		true,                          // bInheritHandle
-		windows.DUPLICATE_SAME_ACCESS, // dwOptions
-	)
-	if err != nil {
-		return 0, err
+var (
+	kernel32         = syscall.MustLoadDLL("kernel32.dll")
+	procSetStdHandle = kernel32.MustFindProc("SetStdHandle")
+)
+
+func setStdHandle(stdhandle int32, handle syscall.Handle) error {
+	r0, _, e1 := syscall.Syscall(procSetStdHandle.Addr(), 2, uintptr(stdhandle), uintptr(handle), 0)
+	if r0 == 0 {
+		if e1 != 0 {
+			return error(e1)
+		}
+		return syscall.EINVAL
 	}
-	return uintptr(newfdHandle), nil
+	return nil
 }
 
 // redirectStderr to the file passed in
 func redirectStderr(f *os.File) {
-	termFd, err := dup(os.Stderr.Fd())
-	if err != nil {
-		log.Fatalf("Failed to duplicate stderr: %v", err)
-	}
-	terminal.RawOut = os.NewFile(termFd, "termOut")
-	err = windows.SetStdHandle(windows.STD_ERROR_HANDLE, windows.Handle(f.Fd()))
+	err := setStdHandle(syscall.STD_ERROR_HANDLE, syscall.Handle(f.Fd()))
 	if err != nil {
 		log.Fatalf("Failed to redirect stderr to file: %v", err)
 	}
-	os.Stderr = f
 }

lib/terminal/terminal.go

@@ -68,23 +68,17 @@ const (
 var (
 	// make sure that start is only called once
 	once sync.Once
-
-	// RawOut is the underlying *os.File intended for terminal output
-	RawOut = os.Stderr
 )
 
 // Start the terminal - must be called before use
 func Start() {
 	once.Do(func() {
-		f := RawOut
+		f := os.Stdout
 		if !IsTerminal(int(f.Fd())) {
-			// If output is not a tty then remove escape codes
+			// If stdout not a tty then remove escape codes
 			Out = colorable.NewNonColorable(f)
 		} else if runtime.GOOS == "windows" && os.Getenv("TERM") != "" {
-			// If TERM is set on Windows then we should just send output
-			// straight to the terminal for cygwin/git bash environments.
-			// We don't want to use NewColorable here because it will
-			// use Windows console calls which cygwin/git bash don't support.
+			// If TERM is set just use stdout
 			Out = f
 		} else {
 			Out = colorable.NewColorable(f)

lib/terminal/terminal_normal.go

@@ -34,5 +34,5 @@ func ReadPassword(fd int) ([]byte, error) {
 
 // WriteTerminalTitle writes a string to the terminal title
 func WriteTerminalTitle(title string) {
-	fmt.Fprintf(Out, ChangeTitle+title+BEL)
+	fmt.Printf(ChangeTitle + title + BEL)
 }