ftp: attempt to fix zero length files with TLS - FIXME DO NOT MERGE DIRECTLY

In https://github.com/jlaffaye/ftp/commit/212daf295f the upstream FTP
library changed the way adding your own dialer works which meant that
connections when using explicit FTP were failing.

This patch reworks our connection code to bring it into the
expectations of the library.

backend/compress/compress.go

@@ -29,7 +29,6 @@ import (
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/fspath"
 	"github.com/rclone/rclone/fs/hash"
-	"github.com/rclone/rclone/fs/log"
 	"github.com/rclone/rclone/fs/object"
 	"github.com/rclone/rclone/fs/operations"
 )
@@ -368,16 +367,13 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 	if err != nil {
 		return nil, err
 	}
-	meta, err := readMetadata(ctx, mo)
-	if err != nil {
-		return nil, fmt.Errorf("error decoding metadata: %w", err)
+	meta := readMetadata(ctx, mo)
+	if meta == nil {
+		return nil, errors.New("error decoding metadata")
 	}
 	// Create our Object
 	o, err := f.Fs.NewObject(ctx, makeDataName(remote, meta.CompressionMetadata.Size, meta.Mode))
-	if err != nil {
-		return nil, err
-	}
-	return f.newObject(o, mo, meta), nil
+	return f.newObject(o, mo, meta), err
 }
 
 // checkCompressAndType checks if an object is compressible and determines it's mime type
@@ -681,7 +677,7 @@ func (f *Fs) putWithCustomFunctions(ctx context.Context, in io.Reader, src fs.Ob
 		}
 		return nil, err
 	}
-	return f.newObject(dataObject, mo, meta), nil
+	return f.newObject(dataObject, mo, meta), err
 }
 
 // Put in to the remote path with the modTime given of the given size
@@ -1044,19 +1040,24 @@ func newMetadata(size int64, mode int, cmeta sgzip.GzipMetadata, md5 string, mim
 }
 
 // This function will read the metadata from a metadata object.
-func readMetadata(ctx context.Context, mo fs.Object) (meta *ObjectMetadata, err error) {
+func readMetadata(ctx context.Context, mo fs.Object) (meta *ObjectMetadata) {
 	// Open our meradata object
 	rc, err := mo.Open(ctx)
 	if err != nil {
-		return nil, err
+		return nil
 	}
-	defer fs.CheckClose(rc, &err)
+	defer func() {
+		err := rc.Close()
+		if err != nil {
+			fs.Errorf(mo, "Error closing object: %v", err)
+		}
+	}()
 	jr := json.NewDecoder(rc)
 	meta = new(ObjectMetadata)
 	if err = jr.Decode(meta); err != nil {
-		return nil, err
+		return nil
 	}
-	return meta, nil
+	return meta
 }
 
 // Remove removes this object
@@ -1101,9 +1102,6 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	origName := o.Remote()
 	if o.meta.Mode != Uncompressed || compressible {
 		newObject, err = o.f.putWithCustomFunctions(ctx, in, o.f.wrapInfo(src, origName, src.Size()), options, o.f.Fs.Put, updateMeta, compressible, mimeType)
-		if err != nil {
-			return err
-		}
 		if newObject.Object.Remote() != o.Object.Remote() {
 			if removeErr := o.Object.Remove(ctx); removeErr != nil {
 				return removeErr
@@ -1117,9 +1115,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		}
 		// If we are, just update the object and metadata
 		newObject, err = o.f.putWithCustomFunctions(ctx, in, src, options, update, updateMeta, compressible, mimeType)
-		if err != nil {
-			return err
-		}
+	}
+	if err != nil {
+		return err
 	}
 	// Update object metadata and return
 	o.Object = newObject.Object
@@ -1130,9 +1128,6 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 // This will initialize the variables of a new press Object. The metadata object, mo, and metadata struct, meta, must be specified.
 func (f *Fs) newObject(o fs.Object, mo fs.Object, meta *ObjectMetadata) *Object {
-	if o == nil {
-		log.Trace(nil, "newObject(%#v, %#v, %#v) called with nil o", o, mo, meta)
-	}
 	return &Object{
 		Object: o,
 		f:      f,
@@ -1145,9 +1140,6 @@ func (f *Fs) newObject(o fs.Object, mo fs.Object, meta *ObjectMetadata) *Object
 
 // This initializes the variables of a press Object with only the size. The metadata will be loaded later on demand.
 func (f *Fs) newObjectSizeAndNameOnly(o fs.Object, moName string, size int64) *Object {
-	if o == nil {
-		log.Trace(nil, "newObjectSizeAndNameOnly(%#v, %#v, %#v) called with nil o", o, moName, size)
-	}
 	return &Object{
 		Object: o,
 		f:      f,
@@ -1175,7 +1167,7 @@ func (o *Object) loadMetadataIfNotLoaded(ctx context.Context) (err error) {
 		return err
 	}
 	if o.meta == nil {
-		o.meta, err = readMetadata(ctx, o.mo)
+		o.meta = readMetadata(ctx, o.mo)
 	}
 	return err
 }

backend/ftp/ftp.go

@@ -336,14 +336,44 @@ func (f *Fs) ftpConnection(ctx context.Context) (c *ftp.ServerConn, err error) {
 	fs.Debugf(f, "Connecting to FTP server")
 
 	// Make ftp library dial with fshttp dialer optionally using TLS
+	initialConnection := true
 	dial := func(network, address string) (conn net.Conn, err error) {
+		fs.Debugf(f, "dial(%q,%q)", network, address)
+		defer func() {
+			fs.Debugf(f, "> dial: conn=%T, err=%v", conn, err)
+		}()
 		conn, err = fshttp.NewDialer(ctx).Dial(network, address)
-		if f.tlsConf != nil && err == nil {
-			conn = tls.Client(conn, f.tlsConf)
+		if err != nil {
+			return nil, err
 		}
-		return
+		// Connect using cleartext only for non TLS
+		if f.tlsConf == nil {
+			return conn, nil
+		}
+		// Initial connection only needs to be cleartext for explicit TLS
+		if f.opt.ExplicitTLS && initialConnection {
+			initialConnection = false
+			return conn, nil
+		}
+		// Upgrade connection to TLS
+		tlsConn := tls.Client(conn, f.tlsConf)
+		// Do the initial handshake - tls.Client doesn't do it for us
+		// If we do this then connections to proftpd/pureftpd lock up
+		// See: https://github.com/rclone/rclone/issues/6426
+		// See: https://github.com/jlaffaye/ftp/issues/282
+		if false {
+			err = tlsConn.HandshakeContext(ctx)
+			if err != nil {
+				_ = conn.Close()
+				return nil, err
+			}
+		}
+		return tlsConn, nil
+	}
+	ftpConfig := []ftp.DialOption{
+		ftp.DialWithContext(ctx),
+		ftp.DialWithDialFunc(dial),
 	}
-	ftpConfig := []ftp.DialOption{ftp.DialWithDialFunc(dial)}
 
 	if f.opt.TLS {
 		// Our dialer takes care of TLS but ftp library also needs tlsConf
@@ -351,12 +381,6 @@ func (f *Fs) ftpConnection(ctx context.Context) (c *ftp.ServerConn, err error) {
 		ftpConfig = append(ftpConfig, ftp.DialWithTLS(f.tlsConf))
 	} else if f.opt.ExplicitTLS {
 		ftpConfig = append(ftpConfig, ftp.DialWithExplicitTLS(f.tlsConf))
-		// Initial connection needs to be cleartext for explicit TLS
-		conn, err := fshttp.NewDialer(ctx).Dial("tcp", f.dialAddr)
-		if err != nil {
-			return nil, err
-		}
-		ftpConfig = append(ftpConfig, ftp.DialWithNetConn(conn))
 	}
 	if f.opt.DisableEPSV {
 		ftpConfig = append(ftpConfig, ftp.DialWithDisabledEPSV(true))

backend/onedrive/onedrive.go

@@ -891,12 +891,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	}).Fill(ctx, f)
 	f.srv.SetErrorHandler(errorHandler)
 
-	// Disable change polling in China region
-	// See: https://github.com/rclone/rclone/issues/6444
-	if f.opt.Region == regionCN {
-		f.features.ChangeNotify = nil
-	}
-
 	// Renew the token in the background
 	f.tokenRenewer = oauthutil.NewRenew(f.String(), ts, func() error {
 		_, _, err := f.readMetaDataForPath(ctx, "")

backend/zoho/zoho.go

@@ -670,28 +670,27 @@ func isSimpleName(s string) bool {
 }
 
 func (f *Fs) upload(ctx context.Context, name string, parent string, size int64, in io.Reader, options ...fs.OpenOption) (*api.Item, error) {
-	uploadID := random.String(20) // random upload ID
-	opts := rest.Opts{
-		Method: "POST",
-		//RootURL: "https://upload.zoho.com/workdrive-api/v1",
-		RootURL:       "https://upload.zoho.eu/workdrive-api/v1",
-		Path:          "/stream/upload",
-		Body:          in,
-		ContentType:   fs.MimeTypeFromName(name), // FIXME should read mime type of original object
-		ContentLength: &size,
-		Options:       options,
-		ExtraHeaders: map[string]string{
-			"x-filename":   name,
-			"x-parent_id":  parent,
-			"upload-id":    uploadID,
-			"x-streammode": "1",
-		},
-	}
-	if size < 0 {
-		opts.ContentLength = nil
+	params := url.Values{}
+	params.Set("filename", name)
+	params.Set("parent_id", parent)
+	params.Set("override-name-exist", strconv.FormatBool(true))
+	formReader, contentType, overhead, err := rest.MultipartUpload(ctx, in, nil, "content", name)
+	if err != nil {
+		return nil, fmt.Errorf("failed to make multipart upload: %w", err)
+	}
+
+	contentLength := overhead + size
+	opts := rest.Opts{
+		Method:           "POST",
+		Path:             "/upload",
+		Body:             formReader,
+		ContentType:      contentType,
+		ContentLength:    &contentLength,
+		Options:          options,
+		Parameters:       params,
+		TransferEncoding: []string{"identity"},
 	}
 
-	var err error
 	var resp *http.Response
 	var uploadResponse *api.UploadResponse
 	err = f.pacer.CallNoRetry(func() (bool, error) {

docs/content/authors.md

@@ -637,6 +637,3 @@ put them back in again.` >}}
   * Ryan Morey <4590343+rmorey@users.noreply.github.com>
   * Simon Bos <simonbos9@gmail.com>
   * YFdyh000 <yfdyh000@gmail.com>  * Josh Soref <2119212+jsoref@users.noreply.github.com>
-  * Øyvind Heddeland Instefjord <instefjord@outlook.com>
-  * Dmitry Deniskin <110819396+ddeniskin@users.noreply.github.com>
-  * Alexander Knorr <106825+opexxx@users.noreply.github.com>

docs/content/install.md

@@ -14,7 +14,7 @@ Rclone is a Go program and comes as a single binary file.
   * Run `rclone config` to setup. See [rclone config docs](/docs/) for more details.
   * Optionally configure [automatic execution](#autostart).
 
-See below for some expanded Linux / macOS / Windows instructions.
+See below for some expanded Linux / macOS instructions.
 
 See the [usage](/docs/) docs for how to use rclone, or
 run `rclone -h`.
@@ -35,9 +35,7 @@ For beta installation, run:
 Note that this script checks the version of rclone installed first and
 won't re-download if not needed.
 
-## Linux installation {#linux}
-
-### Precompiled binary {#linux-precompiled}
+## Linux installation from precompiled binary
 
 Fetch and unpack
 
@@ -61,9 +59,7 @@ Run `rclone config` to setup. See [rclone config docs](/docs/) for more details.
 
     rclone config
 
-## macOS installation {#macos}
-
-### Installation with brew {#macos-brew}
+## macOS installation with brew
 
     brew install rclone
 
@@ -72,12 +68,7 @@ NOTE: This version of rclone will not support `mount` any more (see
 on macOS, either install a precompiled binary or enable the relevant option
 when [installing from source](#install-from-source).
 
-Note that this is a third party installer not controlled by the rclone
-developers so it may be out of date. Its current version is as below.
-
-[![Homebrew package](https://repology.org/badge/version-for-repo/homebrew/rclone.svg)](https://repology.org/project/rclone/versions)
-
-### Precompiled binary, using curl {#macos-precompiled}
+## macOS installation from precompiled binary, using curl
 
 To avoid problems with macOS gatekeeper enforcing the binary to be signed and
 notarized it is enough to download with `curl`.
@@ -105,7 +96,7 @@ Run `rclone config` to setup. See [rclone config docs](/docs/) for more details.
 
     rclone config
 
-### Precompiled binary, using a web browser  {#macos-precompiled-web}
+## macOS installation from precompiled binary, using a web browser
 
 When downloading a binary with a web browser, the browser will set the macOS
 gatekeeper quarantine attribute. Starting from Catalina, when attempting to run
@@ -118,73 +109,11 @@ The simplest fix is to run
 
     xattr -d com.apple.quarantine rclone
 
-## Windows installation {#windows}
+## Install with docker
 
-### Precompiled binary {#windows-precompiled}
-
-Fetch the correct binary for your processor type by clicking on these
-links. If not sure, use the first link.
-
-- [Intel/AMD - 64 Bit](https://downloads.rclone.org/rclone-current-linux-amd64.zip)
-- [Intel/AMD - 32 Bit](https://downloads.rclone.org/rclone-current-linux-386.zip)
-- [ARM - 64 Bit](https://downloads.rclone.org/rclone-current-linux-arm64.zip)
-
-Open this file in the Explorer and extract `rclone.exe`. Rclone is a
-portable executable so you can place it wherever is convenient.
-
-Open a CMD window (or powershell) and run the binary. Note that rclone
-does not launch a GUI by default, it runs in the CMD Window.
-
-- Run `rclone.exe config` to setup. See [rclone config docs](/docs/) for more details.
-- Optionally configure [automatic execution](#autostart).
-
-If you are planning to use the [rclone mount](/commands/rclone_mount/)
-feature then you will need to install the third party utility
-[WinFsp](https://winfsp.dev/) also.
-
-### Chocolatey package manager {#windows-chocolatey}
-
-Make sure you have [Choco](https://chocolatey.org/) installed
-
-```
-choco search rclone
-choco install rclone
-```
-
-This will install rclone on your Windows machine. If you are planning
-to use [rclone mount](/commands/rclone_mount/) then
-
-```
-choco install winfsp
-```
-
-will install that too.
-
-Note that this is a third party installer not controlled by the rclone
-developers so it may be out of date. Its current version is as below.
-
-[![Chocolatey package](https://repology.org/badge/version-for-repo/chocolatey/rclone.svg)](https://repology.org/project/rclone/versions)
-
-## Package manager installation {#package-manager}
-
-Many Linux, Windows, macOS and other OS distributions package and
-distribute rclone.
-
-The distributed versions of rclone are often quite out of date and for
-this reason we recommend one of the other installation methods if
-possible.
-
-You can get an idea of how up to date or not your OS distribution's
-package is here.
-
-[![Packaging status](https://repology.org/badge/vertical-allrepos/rclone.svg?columns=3)](https://repology.org/project/rclone/versions)
-
-## Docker installation {#docker}
-
-The rclone developers maintain a [docker image for rclone](https://hub.docker.com/r/rclone/rclone).
-
-These images are built as part of the release process based on a
-minimal Alpine Linux.
+The rclone maintains a [docker image for rclone](https://hub.docker.com/r/rclone/rclone).
+These images are autobuilt by docker hub from the rclone source based
+on a minimal Alpine linux image.
 
 The `:latest` tag will always point to the latest stable release.  You
 can use the `:beta` tag to get the latest build from master.  You can
@@ -259,7 +188,16 @@ ls ~/data/mount
 kill %1
 ```
 
-## Source installation {#source}
+## Install on Windows via Chocolateley Packet Manager
+
+Make sure you have [Choco](https://chocolatey.org/) installed
+```
+choco search rclone
+choco install rclone
+```
+This will install rclone on your windows machine
+
+## Install from source
 
 Make sure you have git and [Go](https://golang.org/) installed.
 Go version 1.17 or newer is required, latest release is recommended.
@@ -278,7 +216,7 @@ in the same folder. As an initial check you can now run `./rclone version`
 (`.\rclone version` on Windows).
 
 Note that on macOS and Windows the [mount](https://rclone.org/commands/rclone_mount/)
-command will not be available unless you specify an additional build tag `cmount`.
+command will not be available unless you specify additional build tag `cmount`.
 
 ```
 go build -tags cmount
@@ -297,7 +235,7 @@ distribution (make sure you install it in the classic mingw64 subsystem, the
 ucrt64 version is not compatible).
 
 Additionally, on Windows, you must install the third party utility
-[WinFsp](https://winfsp.dev/), with the "Developer" feature selected.
+[WinFsp](http://www.secfs.net/winfsp/), with the "Developer" feature selected.
 If building with cgo, you must also set environment variable CPATH pointing to
 the fuse include directory within the WinFsp installation
 (normally `C:\Program Files (x86)\WinFsp\inc\fuse`).
@@ -345,7 +283,7 @@ with the current version):
 go get github.com/rclone/rclone
 ```
 
-## Ansible installation {#ansible}
+## Installation with Ansible
 
 This can be done with [Stefan Weichinger's ansible
 role](https://github.com/stefangweichinger/ansible-rclone).
@@ -361,7 +299,7 @@ Instructions
           - rclone
 ```
 
-## Portable installation {#portable}
+## Portable installation
 
 As mentioned [above](https://rclone.org/install/#quickstart), rclone is single
 executable (`rclone`, or `rclone.exe` on Windows) that you can download as a

go.mod

@@ -2,6 +2,8 @@ module github.com/rclone/rclone
 
 go 1.17
 
+replace github.com/jlaffaye/ftp v0.0.0-20220904184306-99be0634ab9a => github.com/ncw/ftp v0.0.0-20220916150959-909597844e2a
+
 require (
 	bazil.org/fuse v0.0.0-20200524192727-fb710f7dfd05
 	github.com/Azure/azure-pipeline-go v0.2.3

go.sum

@@ -378,10 +378,6 @@ github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZ
 github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc=
 github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU=
 github.com/jlaffaye/ftp v0.0.0-20190624084859-c1312a7102bf/go.mod h1:lli8NYPQOFy3O++YmYbqVgOcQ1JPCwdOy+5zSjKJ9qY=
-github.com/jlaffaye/ftp v0.0.0-20220630165035-11536801d1ff h1:tN6UCYCBFNrPwvKf4RP9cIhGo6GcZ/IQTN8nqD7eCok=
-github.com/jlaffaye/ftp v0.0.0-20220630165035-11536801d1ff/go.mod h1:hhq4G4crv+nW2qXtNYcuzLeOudG92Ps37HEKeg2e3lE=
-github.com/jlaffaye/ftp v0.0.0-20220904184306-99be0634ab9a h1:s4ryRQyC5HKZh6qkjNAFcvmD7gImK5bZuj/YZkXy1vw=
-github.com/jlaffaye/ftp v0.0.0-20220904184306-99be0634ab9a/go.mod h1:hhq4G4crv+nW2qXtNYcuzLeOudG92Ps37HEKeg2e3lE=
 github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
@@ -460,6 +456,8 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY
 github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/ncw/ftp v0.0.0-20220916150959-909597844e2a h1:BglfrImWBkK0SyMLSsKmAgutB0rzkoLzt5j5Sjn+heg=
+github.com/ncw/ftp v0.0.0-20220916150959-909597844e2a/go.mod h1:hhq4G4crv+nW2qXtNYcuzLeOudG92Ps37HEKeg2e3lE=
 github.com/ncw/go-acd v0.0.0-20201019170801-fe55f33415b1 h1:nAjWYc03awJAjsozNehdGZsm5LP7AhLOvjgbS8zN1tk=
 github.com/ncw/go-acd v0.0.0-20201019170801-fe55f33415b1/go.mod h1:MLIrzg7gp/kzVBxRE1olT7CWYMCklcUWU+ekoxOD9x0=
 github.com/ncw/swift/v2 v2.0.1 h1:q1IN8hNViXEv8Zvg3Xdis4a3c4IlIGezkYz09zQL5J0=