log: update windows redirection code to use x/sys/windows and include dup

This modernises the Windows redirect code to use x/sys/windows and to
dup os.Stderr for use in the terminal.

.github/workflows/build.yml

@@ -97,12 +97,12 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 0
 
       - name: Install Go
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v3
         with:
           stable: 'false'
           go-version: ${{ matrix.go }}
@@ -162,7 +162,7 @@ jobs:
           env
 
       - name: Go module cache
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: ~/go/pkg/mod
           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
@@ -226,7 +226,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Code quality test
         uses: golangci/golangci-lint-action@v3
@@ -242,18 +242,18 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 0
 
       # Upgrade together with NDK version
       - name: Set up Go
-        uses: actions/setup-go@v1
+        uses: actions/setup-go@v3
         with:
           go-version: 1.19.x
 
       - name: Go module cache
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: ~/go/pkg/mod
           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}

.github/workflows/build_publish_docker_image.yml

@@ -12,7 +12,7 @@ jobs:
         name: Build image job
         steps:
             - name: Checkout master
-              uses: actions/checkout@v2
+              uses: actions/checkout@v3
               with:
                 fetch-depth: 0
             - name: Build and publish image

.github/workflows/build_publish_release_docker_image.yml

@@ -11,7 +11,7 @@ jobs:
         name: Build image job
         steps:
             - name: Checkout master
-              uses: actions/checkout@v2
+              uses: actions/checkout@v3
               with:
                 fetch-depth: 0
             - name: Get actual patch version
@@ -40,7 +40,7 @@ jobs:
         name: Build docker plugin job
         steps:
             - name: Checkout master
-              uses: actions/checkout@v2
+              uses: actions/checkout@v3
               with:
                 fetch-depth: 0
             - name: Build and publish docker plugin

.golangci.yml

@@ -20,7 +20,7 @@ issues:
   exclude-use-default: false
 
   # Maximum issues count per one linter. Set to 0 to disable. Default is 50.
-  max-per-linter: 0
+  max-issues-per-linter: 0
 
   # Maximum count of issues with the same text. Set to 0 to disable. Default is 3.
   max-same-issues: 0

README.md

@@ -49,6 +49,7 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
   * Internet Archive [:page_facing_up:](https://rclone.org/internetarchive/)
   * Jottacloud [:page_facing_up:](https://rclone.org/jottacloud/)
   * IBM COS S3 [:page_facing_up:](https://rclone.org/s3/#ibm-cos-s3)
+  * IONOS Cloud [:page_facing_up:](https://rclone.org/s3/#ionos)
   * Koofr [:page_facing_up:](https://rclone.org/koofr/)
   * Mail.ru Cloud [:page_facing_up:](https://rclone.org/mailru/)
   * Memset Memstore [:page_facing_up:](https://rclone.org/swift/)

RELEASE.md

@@ -53,6 +53,14 @@ doing that so it may be necessary to roll back dependencies to the
 version specified by `make updatedirect` in order to get rclone to
 build.
 
+## Tidy beta
+
+At some point after the release run
+
+    bin/tidy-beta v1.55
+
+where the version number is that of a couple ago to remove old beta binaries.
+
 ## Making a point release
 
 If rclone needs a point release due to some horrendous bug:

backend/compress/compress.go

@@ -29,6 +29,7 @@ import (
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/fspath"
 	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/fs/log"
 	"github.com/rclone/rclone/fs/object"
 	"github.com/rclone/rclone/fs/operations"
 )
@@ -367,13 +368,16 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 	if err != nil {
 		return nil, err
 	}
-	meta := readMetadata(ctx, mo)
-	if meta == nil {
-		return nil, errors.New("error decoding metadata")
+	meta, err := readMetadata(ctx, mo)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding metadata: %w", err)
 	}
 	// Create our Object
 	o, err := f.Fs.NewObject(ctx, makeDataName(remote, meta.CompressionMetadata.Size, meta.Mode))
-	return f.newObject(o, mo, meta), err
+	if err != nil {
+		return nil, err
+	}
+	return f.newObject(o, mo, meta), nil
 }
 
 // checkCompressAndType checks if an object is compressible and determines it's mime type
@@ -677,7 +681,7 @@ func (f *Fs) putWithCustomFunctions(ctx context.Context, in io.Reader, src fs.Ob
 		}
 		return nil, err
 	}
-	return f.newObject(dataObject, mo, meta), err
+	return f.newObject(dataObject, mo, meta), nil
 }
 
 // Put in to the remote path with the modTime given of the given size
@@ -1040,24 +1044,19 @@ func newMetadata(size int64, mode int, cmeta sgzip.GzipMetadata, md5 string, mim
 }
 
 // This function will read the metadata from a metadata object.
-func readMetadata(ctx context.Context, mo fs.Object) (meta *ObjectMetadata) {
+func readMetadata(ctx context.Context, mo fs.Object) (meta *ObjectMetadata, err error) {
 	// Open our meradata object
 	rc, err := mo.Open(ctx)
 	if err != nil {
-		return nil
+		return nil, err
 	}
-	defer func() {
-		err := rc.Close()
-		if err != nil {
-			fs.Errorf(mo, "Error closing object: %v", err)
-		}
-	}()
+	defer fs.CheckClose(rc, &err)
 	jr := json.NewDecoder(rc)
 	meta = new(ObjectMetadata)
 	if err = jr.Decode(meta); err != nil {
-		return nil
+		return nil, err
 	}
-	return meta
+	return meta, nil
 }
 
 // Remove removes this object
@@ -1102,6 +1101,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	origName := o.Remote()
 	if o.meta.Mode != Uncompressed || compressible {
 		newObject, err = o.f.putWithCustomFunctions(ctx, in, o.f.wrapInfo(src, origName, src.Size()), options, o.f.Fs.Put, updateMeta, compressible, mimeType)
+		if err != nil {
+			return err
+		}
 		if newObject.Object.Remote() != o.Object.Remote() {
 			if removeErr := o.Object.Remove(ctx); removeErr != nil {
 				return removeErr
@@ -1115,9 +1117,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		}
 		// If we are, just update the object and metadata
 		newObject, err = o.f.putWithCustomFunctions(ctx, in, src, options, update, updateMeta, compressible, mimeType)
-	}
-	if err != nil {
-		return err
+		if err != nil {
+			return err
+		}
 	}
 	// Update object metadata and return
 	o.Object = newObject.Object
@@ -1128,6 +1130,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 // This will initialize the variables of a new press Object. The metadata object, mo, and metadata struct, meta, must be specified.
 func (f *Fs) newObject(o fs.Object, mo fs.Object, meta *ObjectMetadata) *Object {
+	if o == nil {
+		log.Trace(nil, "newObject(%#v, %#v, %#v) called with nil o", o, mo, meta)
+	}
 	return &Object{
 		Object: o,
 		f:      f,
@@ -1140,6 +1145,9 @@ func (f *Fs) newObject(o fs.Object, mo fs.Object, meta *ObjectMetadata) *Object
 
 // This initializes the variables of a press Object with only the size. The metadata will be loaded later on demand.
 func (f *Fs) newObjectSizeAndNameOnly(o fs.Object, moName string, size int64) *Object {
+	if o == nil {
+		log.Trace(nil, "newObjectSizeAndNameOnly(%#v, %#v, %#v) called with nil o", o, moName, size)
+	}
 	return &Object{
 		Object: o,
 		f:      f,
@@ -1167,7 +1175,7 @@ func (o *Object) loadMetadataIfNotLoaded(ctx context.Context) (err error) {
 		return err
 	}
 	if o.meta == nil {
-		o.meta = readMetadata(ctx, o.mo)
+		o.meta, err = readMetadata(ctx, o.mo)
 	}
 	return err
 }

backend/drive/drive.go

@@ -1210,6 +1210,7 @@ func newFs(ctx context.Context, name, path string, m configmap.Mapper) (*Fs, err
 		WriteMimeType:           true,
 		CanHaveEmptyDirectories: true,
 		ServerSideAcrossConfigs: opt.ServerSideAcrossConfigs,
+		FilterAware:             true,
 	}).Fill(ctx, f)
 
 	// Create a new authorized Drive client.

backend/drive/drive_internal_test.go

@@ -518,6 +518,9 @@ func (f *Fs) InternalTestCopyID(t *testing.T) {
 
 // TestIntegration/FsMkdir/FsPutFiles/Internal/AgeQuery
 func (f *Fs) InternalTestAgeQuery(t *testing.T) {
+	// Check set up for filtering
+	assert.True(t, f.Features().FilterAware)
+
 	opt := &filter.Opt{}
 	err := opt.MaxAge.Set("1h")
 	assert.NoError(t, err)

backend/ftp/ftp.go

@@ -124,6 +124,11 @@ So for |concurrency 3| you'd use |--checkers 2 --transfers 2
 			Help:     "Use MDTM to set modification time (VsFtpd quirk)",
 			Default:  false,
 			Advanced: true,
+		}, {
+			Name:     "force_list_hidden",
+			Help:     "Use LIST -a to force listing of hidden files and folders. This will disable the use of MLSD.",
+			Default:  false,
+			Advanced: true,
 		}, {
 			Name:    "idle_timeout",
 			Default: fs.Duration(60 * time.Second),
@@ -205,6 +210,7 @@ type Options struct {
 	DisableMLSD       bool                 `config:"disable_mlsd"`
 	DisableUTF8       bool                 `config:"disable_utf8"`
 	WritingMDTM       bool                 `config:"writing_mdtm"`
+	ForceListHidden   bool                 `config:"force_list_hidden"`
 	IdleTimeout       fs.Duration          `config:"idle_timeout"`
 	CloseTimeout      fs.Duration          `config:"close_timeout"`
 	ShutTimeout       fs.Duration          `config:"shut_timeout"`
@@ -330,14 +336,44 @@ func (f *Fs) ftpConnection(ctx context.Context) (c *ftp.ServerConn, err error) {
 	fs.Debugf(f, "Connecting to FTP server")
 
 	// Make ftp library dial with fshttp dialer optionally using TLS
+	initialConnection := true
 	dial := func(network, address string) (conn net.Conn, err error) {
+		fs.Debugf(f, "dial(%q,%q)", network, address)
+		defer func() {
+			fs.Debugf(f, "> dial: conn=%T, err=%v", conn, err)
+		}()
 		conn, err = fshttp.NewDialer(ctx).Dial(network, address)
-		if f.tlsConf != nil && err == nil {
-			conn = tls.Client(conn, f.tlsConf)
+		if err != nil {
+			return nil, err
 		}
-		return
+		// Connect using cleartext only for non TLS
+		if f.tlsConf == nil {
+			return conn, nil
+		}
+		// Initial connection only needs to be cleartext for explicit TLS
+		if f.opt.ExplicitTLS && initialConnection {
+			initialConnection = false
+			return conn, nil
+		}
+		// Upgrade connection to TLS
+		tlsConn := tls.Client(conn, f.tlsConf)
+		// Do the initial handshake - tls.Client doesn't do it for us
+		// If we do this then connections to proftpd/pureftpd lock up
+		// See: https://github.com/rclone/rclone/issues/6426
+		// See: https://github.com/jlaffaye/ftp/issues/282
+		if false {
+			err = tlsConn.HandshakeContext(ctx)
+			if err != nil {
+				_ = conn.Close()
+				return nil, err
+			}
+		}
+		return tlsConn, nil
+	}
+	ftpConfig := []ftp.DialOption{
+		ftp.DialWithContext(ctx),
+		ftp.DialWithDialFunc(dial),
 	}
-	ftpConfig := []ftp.DialOption{ftp.DialWithDialFunc(dial)}
 
 	if f.opt.TLS {
 		// Our dialer takes care of TLS but ftp library also needs tlsConf
@@ -345,12 +381,6 @@ func (f *Fs) ftpConnection(ctx context.Context) (c *ftp.ServerConn, err error) {
 		ftpConfig = append(ftpConfig, ftp.DialWithTLS(f.tlsConf))
 	} else if f.opt.ExplicitTLS {
 		ftpConfig = append(ftpConfig, ftp.DialWithExplicitTLS(f.tlsConf))
-		// Initial connection needs to be cleartext for explicit TLS
-		conn, err := fshttp.NewDialer(ctx).Dial("tcp", f.dialAddr)
-		if err != nil {
-			return nil, err
-		}
-		ftpConfig = append(ftpConfig, ftp.DialWithNetConn(conn))
 	}
 	if f.opt.DisableEPSV {
 		ftpConfig = append(ftpConfig, ftp.DialWithDisabledEPSV(true))
@@ -367,6 +397,9 @@ func (f *Fs) ftpConnection(ctx context.Context) (c *ftp.ServerConn, err error) {
 	if f.opt.WritingMDTM {
 		ftpConfig = append(ftpConfig, ftp.DialWithWritingMDTM(true))
 	}
+	if f.opt.ForceListHidden {
+		ftpConfig = append(ftpConfig, ftp.DialWithForceListHidden(true))
+	}
 	if f.ci.Dump&(fs.DumpHeaders|fs.DumpBodies|fs.DumpRequests|fs.DumpResponses) != 0 {
 		ftpConfig = append(ftpConfig, ftp.DialWithDebugOutput(&debugLog{auth: f.ci.Dump&fs.DumpAuth != 0}))
 	}

backend/ftp/ftp_internal_test.go

@@ -34,9 +34,9 @@ func deriveFs(ctx context.Context, t *testing.T, f fs.Fs, opts settings) fs.Fs {
 // test that big file uploads do not cause network i/o timeout
 func (f *Fs) testUploadTimeout(t *testing.T) {
 	const (
-		fileSize    = 100000000             // 100 MiB
-		idleTimeout = 40 * time.Millisecond // small because test server is local
-		maxTime     = 10 * time.Second      // prevent test hangup
+		fileSize    = 100000000        // 100 MiB
+		idleTimeout = 1 * time.Second  // small because test server is local
+		maxTime     = 10 * time.Second // prevent test hangup
 	)
 
 	if testing.Short() {

backend/local/local.go

@@ -300,6 +300,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		ReadMetadata:            true,
 		WriteMetadata:           true,
 		UserMetadata:            xattrSupported, // can only R/W general purpose metadata if xattrs are supported
+		FilterAware:             true,
 	}).Fill(ctx, f)
 	if opt.FollowSymlinks {
 		f.lstat = os.Stat

backend/local/local_internal_test.go

@@ -378,6 +378,9 @@ func TestFilter(t *testing.T) {
 	r.WriteFile("excluded", "excluded file", when)
 	f := r.Flocal.(*Fs)
 
+	// Check set up for filtering
+	assert.True(t, f.Features().FilterAware)
+
 	// Add a filter
 	ctx, fi := filter.AddConfig(ctx)
 	require.NoError(t, fi.AddRule("+ included"))

backend/onedrive/onedrive.go

@@ -891,6 +891,12 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	}).Fill(ctx, f)
 	f.srv.SetErrorHandler(errorHandler)
 
+	// Disable change polling in China region
+	// See: https://github.com/rclone/rclone/issues/6444
+	if f.opt.Region == regionCN {
+		f.features.ChangeNotify = nil
+	}
+
 	// Renew the token in the background
 	f.tokenRenewer = oauthutil.NewRenew(f.String(), ts, func() error {
 		_, _, err := f.readMetaDataForPath(ctx, "")

backend/s3/s3.go

@@ -64,7 +64,7 @@ import (
 func init() {
 	fs.Register(&fs.RegInfo{
 		Name:        "s3",
-		Description: "Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi",
+		Description: "Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi",
 		NewFs:       NewFs,
 		CommandHelp: commandHelp,
 		Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
@@ -116,6 +116,9 @@ func init() {
 			}, {
 				Value: "IDrive",
 				Help:  "IDrive e2",
+			}, {
+				Value: "IONOS",
+				Help:  "IONOS Cloud",
 			}, {
 				Value: "LyveCloud",
 				Help:  "Seagate Lyve Cloud",
@@ -384,10 +387,24 @@ func init() {
 				Value: "auto",
 				Help:  "R2 buckets are automatically distributed across Cloudflare's data centers for low latency.",
 			}},
+		}, {
+			Name:     "region",
+			Help:     "Region where your bucket will be created and your data stored.\n",
+			Provider: "IONOS",
+			Examples: []fs.OptionExample{{
+				Value: "de",
+				Help:  "Frankfurt, Germany",
+			}, {
+				Value: "eu-central-2",
+				Help:  "Berlin, Germany",
+			}, {
+				Value: "eu-south-2",
+				Help:  "Logrono, Spain",
+			}},
 		}, {
 			Name:     "region",
 			Help:     "Region to connect to.\n\nLeave blank if you are using an S3 clone and you don't have a region.",
-			Provider: "!AWS,Alibaba,ChinaMobile,Cloudflare,ArvanCloud,RackCorp,Scaleway,Storj,TencentCOS,HuaweiOBS,IDrive",
+			Provider: "!AWS,Alibaba,ChinaMobile,Cloudflare,IONOS,ArvanCloud,RackCorp,Scaleway,Storj,TencentCOS,HuaweiOBS,IDrive",
 			Examples: []fs.OptionExample{{
 				Value: "",
 				Help:  "Use this if unsure.\nWill use v4 signatures and an empty region.",
@@ -698,6 +715,20 @@ func init() {
 				Value: "s3.private.sng01.cloud-object-storage.appdomain.cloud",
 				Help:  "Singapore Single Site Private Endpoint",
 			}},
+		}, {
+			Name:     "endpoint",
+			Help:     "Endpoint for IONOS S3 Object Storage.\n\nSpecify the endpoint from the same region.",
+			Provider: "IONOS",
+			Examples: []fs.OptionExample{{
+				Value: "s3-eu-central-1.ionoscloud.com",
+				Help:  "Frankfurt, Germany",
+			}, {
+				Value: "s3-eu-central-2.ionoscloud.com",
+				Help:  "Berlin, Germany",
+			}, {
+				Value: "s3-eu-south-2.ionoscloud.com",
+				Help:  "Logrono, Spain",
+			}},
 		}, {
 			// oss endpoints: https://help.aliyun.com/document_detail/31837.html
 			Name:     "endpoint",
@@ -1001,7 +1032,7 @@ func init() {
 		}, {
 			Name:     "endpoint",
 			Help:     "Endpoint for S3 API.\n\nRequired when using an S3 clone.",
-			Provider: "!AWS,IBMCOS,IDrive,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,ArvanCloud,Scaleway,StackPath,Storj,RackCorp",
+			Provider: "!AWS,IBMCOS,IDrive,IONOS,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,ArvanCloud,Scaleway,StackPath,Storj,RackCorp",
 			Examples: []fs.OptionExample{{
 				Value:    "objects-us-east-1.dream.io",
 				Help:     "Dream Objects endpoint",
@@ -1411,7 +1442,7 @@ func init() {
 		}, {
 			Name:     "location_constraint",
 			Help:     "Location constraint - must be set to match the Region.\n\nLeave blank if not sure. Used when creating buckets only.",
-			Provider: "!AWS,IBMCOS,IDrive,Alibaba,HuaweiOBS,ChinaMobile,Cloudflare,ArvanCloud,RackCorp,Scaleway,StackPath,Storj,TencentCOS",
+			Provider: "!AWS,Alibaba,HuaweiOBS,ChinaMobile,Cloudflare,IBMCOS,IDrive,IONOS,ArvanCloud,RackCorp,Scaleway,StackPath,Storj,TencentCOS",
 		}, {
 			Name: "acl",
 			Help: `Canned ACL used when creating buckets and storing or copying objects.
@@ -1535,8 +1566,21 @@ isn't set then "acl" is used instead.`,
 				Help:  "arn:aws:kms:*",
 			}},
 		}, {
-			Name:     "sse_customer_key",
-			Help:     "If using SSE-C you must provide the secret encryption key used to encrypt/decrypt your data.",
+			Name: "sse_customer_key",
+			Help: `To use SSE-C you may provide the secret encryption key used to encrypt/decrypt your data.
+
+Alternatively you can provide --sse-customer-key-base64.`,
+			Provider: "AWS,Ceph,ChinaMobile,Minio",
+			Advanced: true,
+			Examples: []fs.OptionExample{{
+				Value: "",
+				Help:  "None",
+			}},
+		}, {
+			Name: "sse_customer_key_base64",
+			Help: `If using SSE-C you must provide the secret encryption key encoded in base64 format to encrypt/decrypt your data.
+
+Alternatively you can provide --sse-customer-key.`,
 			Provider: "AWS,Ceph,ChinaMobile,Minio",
 			Advanced: true,
 			Examples: []fs.OptionExample{{
@@ -2111,6 +2155,7 @@ type Options struct {
 	SSEKMSKeyID           string               `config:"sse_kms_key_id"`
 	SSECustomerAlgorithm  string               `config:"sse_customer_algorithm"`
 	SSECustomerKey        string               `config:"sse_customer_key"`
+	SSECustomerKeyBase64  string               `config:"sse_customer_key_base64"`
 	SSECustomerKeyMD5     string               `config:"sse_customer_key_md5"`
 	StorageClass          string               `config:"storage_class"`
 	UploadCutoff          fs.SizeSuffix        `config:"upload_cutoff"`
@@ -2537,6 +2582,10 @@ func setQuirks(opt *Options) {
 		useMultipartEtag = false // untested
 	case "IDrive":
 		virtualHostStyle = false
+	case "IONOS":
+		// listObjectsV2 supported - https://api.ionos.com/docs/s3/#Basic-Operations-get-Bucket-list-type-2
+		virtualHostStyle = false
+		urlEncodeListings = false
 	case "LyveCloud":
 		useMultipartEtag = false // LyveCloud seems to calculate multipart Etags differently from AWS
 	case "Minio":
@@ -2644,6 +2693,16 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	if opt.BucketACL == "" {
 		opt.BucketACL = opt.ACL
 	}
+	if opt.SSECustomerKeyBase64 != "" && opt.SSECustomerKey != "" {
+		return nil, errors.New("s3: can't use sse_customer_key and sse_customer_key_base64 at the same time")
+	} else if opt.SSECustomerKeyBase64 != "" {
+		// Decode the base64-encoded key and store it in the SSECustomerKey field
+		decoded, err := base64.StdEncoding.DecodeString(opt.SSECustomerKeyBase64)
+		if err != nil {
+			return nil, fmt.Errorf("s3: Could not decode sse_customer_key_base64: %w", err)
+		}
+		opt.SSECustomerKey = string(decoded)
+	}
 	if opt.SSECustomerKey != "" && opt.SSECustomerKeyMD5 == "" {
 		// calculate CustomerKeyMD5 if not supplied
 		md5sumBinary := md5.Sum([]byte(opt.SSECustomerKey))
@@ -2758,8 +2817,7 @@ func (f *Fs) getMetaDataListing(ctx context.Context, wantRemote string) (info *s
 		if isDirectory {
 			return nil
 		}
-		// compare the base name only since the listing will have a prefix
-		if path.Base(wantRemote) != path.Base(gotRemote) {
+		if wantRemote != gotRemote {
 			return nil
 		}
 		info = object

backend/s3/s3_internal_test.go

@@ -6,16 +6,12 @@ import (
 	"context"
 	"crypto/md5"
 	"fmt"
-	"path"
-	"strings"
 	"testing"
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/cache"
-	"github.com/rclone/rclone/fs/fspath"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/fstest/fstests"
@@ -254,7 +250,7 @@ func (f *Fs) InternalTestVersions(t *testing.T) {
 	time.Sleep(2 * time.Second)
 
 	// Create an object
-	const fileName = "versions/test-versions.txt"
+	const fileName = "test-versions.txt"
 	contents := random.String(100)
 	item := fstest.NewItem(fileName, contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
 	obj := fstests.PutTestContents(ctx, t, f, &item, contents, true)
@@ -284,7 +280,7 @@ func (f *Fs) InternalTestVersions(t *testing.T) {
 		}()
 
 		// Read the contents
-		entries, err := f.List(ctx, "versions")
+		entries, err := f.List(ctx, "")
 		require.NoError(t, err)
 		tests := 0
 		var fileNameVersion string
@@ -299,24 +295,12 @@ func (f *Fs) InternalTestVersions(t *testing.T) {
 				t.Run("ReadVersion", func(t *testing.T) {
 					assert.Equal(t, contents, fstests.ReadObject(ctx, t, entry.(fs.Object), -1))
 				})
-				t.Run("NewFs", func(t *testing.T) {
-					// Check we can find the object with NewFs
-					fPath := fs.ConfigString(f)
-					fPath = strings.Replace(fPath, ":", ",versions=true:", 1)
-					subFPath := fspath.JoinRootPath(fPath, entry.Remote())
-					subF, err := cache.Get(ctx, subFPath)
-					require.Equal(t, fs.ErrorIsFile, err, "Remote %q didn't find a file", subFPath)
-					require.NotNil(t, subF)
-					o, err := subF.NewObject(ctx, path.Base(entry.Remote()))
-					require.NoError(t, err)
-					assert.Equal(t, contents, fstests.ReadObject(ctx, t, o, -1))
-				})
 				assert.WithinDuration(t, obj.(*Object).lastModified, versionTime, time.Second, "object time must be with 1 second of version time")
 				fileNameVersion = remote
 				tests++
 			}
 		}
-		assert.Equal(t, 2, tests, "object missing from listing: %v", entries)
+		assert.Equal(t, 2, tests, "object missing from listing")
 
 		// Check we can read the object with a version suffix
 		t.Run("NewObject", func(t *testing.T) {

backend/sftp/sftp.go

@@ -1171,6 +1171,10 @@ func (f *Fs) mkdir(ctx context.Context, dirPath string) error {
 	err = c.sftpClient.Mkdir(dirPath)
 	f.putSftpConnection(&c, err)
 	if err != nil {
+		if os.IsExist(err) {
+			fs.Debugf(f, "directory %q exists after Mkdir is attempted", dirPath)
+			return nil
+		}
 		return fmt.Errorf("mkdir %q failed: %w", dirPath, err)
 	}
 	return nil

bin/tidy-beta

@@ -15,6 +15,7 @@ else
 fi
 
 rclone ${dry_run} -vv -P --checkers 16 --transfers 16 delete \
+       --fast-list \
        --include "/${version}**" \
-       --include "/branch/${version}**" \
+       --include "/branch/*/${version}**" \
        memstore:beta-rclone-org

cmd/progress.go

@@ -62,7 +62,7 @@ func startProgress() func() {
 				printProgress("")
 				fs.LogPrint = oldLogPrint
 				operations.SyncPrintf = oldSyncPrint
-				fmt.Println("")
+				fmt.Fprintln(terminal.Out, "")
 				return
 			}
 		}

docs/content/_index.md

@@ -135,6 +135,7 @@ WebDAV or S3, that work out of the box.)
 {{< provider name="Jottacloud" home="https://www.jottacloud.com/en/" config="/jottacloud/" >}}
 {{< provider name="IBM COS S3" home="http://www.ibm.com/cloud/object-storage" config="/s3/#ibm-cos-s3" >}}
 {{< provider name="IDrive e2" home="https://www.idrive.com/e2/" config="/s3/#idrive-e2" >}}
+{{< provider name="IONOS Cloud" home="https://cloud.ionos.com/storage/object-storage" config="/s3/#ionos" >}}
 {{< provider name="Koofr" home="https://koofr.eu/" config="/koofr/" >}}
 {{< provider name="Mail.ru Cloud" home="https://cloud.mail.ru/" config="/mailru/" >}}
 {{< provider name="Memset Memstore" home="https://www.memset.com/cloud/storage/" config="/swift/" >}}

docs/content/authors.md

@@ -637,3 +637,7 @@ put them back in again.` >}}
   * Ryan Morey <4590343+rmorey@users.noreply.github.com>
   * Simon Bos <simonbos9@gmail.com>
   * YFdyh000 <yfdyh000@gmail.com>  * Josh Soref <2119212+jsoref@users.noreply.github.com>
+  * Øyvind Heddeland Instefjord <instefjord@outlook.com>
+  * Dmitry Deniskin <110819396+ddeniskin@users.noreply.github.com>
+  * Alexander Knorr <106825+opexxx@users.noreply.github.com>
+  * Richard Bateman <richard@batemansr.us>

docs/content/changelog.md

@@ -5,6 +5,21 @@ description: "Rclone Changelog"
 
 # Changelog
 
+## v1.59.2 - 2022-09-15
+
+[See commits](https://github.com/rclone/rclone/compare/v1.59.1...v1.59.2)
+
+* Bug Fixes
+    * config: Move locking to fix fatal error: concurrent map read and map write (Nick Craig-Wood)
+* Local
+    * Disable xattr support if the filesystems indicates it is not supported (Nick Craig-Wood)
+* Azure Blob
+    * Fix chunksize calculations producing too many parts (Nick Craig-Wood)
+* B2
+    * Fix chunksize calculations producing too many parts (Nick Craig-Wood)
+* S3
+    * Fix chunksize calculations producing too many parts (Nick Craig-Wood)
+
 ## v1.59.1 - 2022-08-08
 
 [See commits](https://github.com/rclone/rclone/compare/v1.59.0...v1.59.1)

docs/content/commands/rclone_mount.md

@@ -341,8 +341,6 @@ mount sftp1:subdir /mnt/data -t rclone -o vfs_cache_mode=writes,sftp_key_file=/p
 or create systemd mount units:
 ```
 # /etc/systemd/system/mnt-data.mount
-[Unit]
-After=network-online.target
 [Mount]
 Type=rclone
 What=sftp1:subdir
@@ -354,7 +352,6 @@ optionally accompanied by systemd automount unit
 ```
 # /etc/systemd/system/mnt-data.automount
 [Unit]
-After=network-online.target
 Before=remote-fs.target
 [Automount]
 Where=/mnt/data

docs/content/docs.md

@@ -1868,13 +1868,22 @@ By default, rclone doesn't keep track of renamed files, so if you
 rename a file locally then sync it to a remote, rclone will delete the
 old file on the remote and upload a new copy.
 
-If you use this flag, and the remote supports server-side copy or
-server-side move, and the source and destination have a compatible
-hash, then this will track renames during `sync`
-operations and perform renaming server-side.
+An rclone sync with `--track-renames` runs like a normal sync, but keeps
+track of objects which exist in the destination but not in the source
+(which would normally be deleted), and which objects exist in the
+source but not the destination (which would normally be transferred).
+These objects are then candidates for renaming.
 
-Files will be matched by size and hash - if both match then a rename
-will be considered.
+After the sync, rclone matches up the source only and destination only
+objects using the `--track-renames-strategy` specified and either
+renames the destination object or transfers the source and deletes the
+destination object. `--track-renames` is stateless like all of
+rclone's syncs.
+
+To use this flag the destination must support server-side copy or
+server-side move, and to use a hash based `--track-renames-strategy`
+(the default) the source and the destination must have a compatible
+hash.
 
 If the destination does not support server-side copy or move, rclone
 will fall back to the default behaviour and log an error level message
@@ -1892,7 +1901,7 @@ Note also that `--track-renames` is incompatible with
 
 ### --track-renames-strategy (hash,modtime,leaf,size) ###
 
-This option changes the matching criteria for `--track-renames`.
+This option changes the file matching criteria for `--track-renames`.
 
 The matching is controlled by a comma separated selection of these tokens:
 
@@ -1901,15 +1910,15 @@ The matching is controlled by a comma separated selection of these tokens:
 - `leaf` - the name of the file not including its directory name
 - `size` - the size of the file (this is always enabled)
 
-So using `--track-renames-strategy modtime,leaf` would match files
+The default option is `hash`.
+
+Using `--track-renames-strategy modtime,leaf` would match files
 based on modification time, the leaf of the file name and the size
 only.
 
 Using `--track-renames-strategy modtime` or `leaf` can enable
 `--track-renames` support for encrypted destinations.
 
-If nothing is specified, the default option is matching by `hash`es.
-
 Note that the `hash` strategy is not supported with encrypted destinations.
 
 ### --delete-(before,during,after) ###

docs/content/flags.md

@@ -357,6 +357,7 @@ and may be set in the config file.
       --ftp-tls-cache-size int                       Size of TLS session cache for all control and data connections (default 32)
       --ftp-user string                              FTP username (default "$USER")
       --ftp-writing-mdtm                             Use MDTM to set modification time (VsFtpd quirk)
+      --ftp-force-list-hidden                        Use LIST -a to force listing of hidden files and folders. This will disable the use of MLSD.
       --gcs-anonymous                                Access public buckets and objects without credentials
       --gcs-auth-url string                          Auth server URL
       --gcs-bucket-acl string                        Access Control List for new buckets

docs/content/ftp.md

@@ -310,6 +310,17 @@ Properties:
 - Type:        bool
 - Default:     false
 
+#### --ftp-force-list-hidden
+
+Use LIST -a to force listing of hidden files and folders. This will disable the use of MLSD.
+
+Properties:
+
+- Config:      force_list_hidden
+- Env Var:     RCLONE_FTP_FORCE_LIST_HIDDEN
+- Type:        bool
+- Default:     false
+
 #### --ftp-idle-timeout
 
 Max time before closing idle connections.

docs/content/install.md

@@ -14,7 +14,7 @@ Rclone is a Go program and comes as a single binary file.
   * Run `rclone config` to setup. See [rclone config docs](/docs/) for more details.
   * Optionally configure [automatic execution](#autostart).
 
-See below for some expanded Linux / macOS instructions.
+See below for some expanded Linux / macOS / Windows instructions.
 
 See the [usage](/docs/) docs for how to use rclone, or
 run `rclone -h`.
@@ -35,7 +35,9 @@ For beta installation, run:
 Note that this script checks the version of rclone installed first and
 won't re-download if not needed.
 
-## Linux installation from precompiled binary
+## Linux installation {#linux}
+
+### Precompiled binary {#linux-precompiled}
 
 Fetch and unpack
 
@@ -59,7 +61,9 @@ Run `rclone config` to setup. See [rclone config docs](/docs/) for more details.
 
     rclone config
 
-## macOS installation with brew
+## macOS installation {#macos}
+
+### Installation with brew {#macos-brew}
 
     brew install rclone
 
@@ -68,7 +72,12 @@ NOTE: This version of rclone will not support `mount` any more (see
 on macOS, either install a precompiled binary or enable the relevant option
 when [installing from source](#install-from-source).
 
-## macOS installation from precompiled binary, using curl
+Note that this is a third party installer not controlled by the rclone
+developers so it may be out of date. Its current version is as below.
+
+[![Homebrew package](https://repology.org/badge/version-for-repo/homebrew/rclone.svg)](https://repology.org/project/rclone/versions)
+
+### Precompiled binary, using curl {#macos-precompiled}
 
 To avoid problems with macOS gatekeeper enforcing the binary to be signed and
 notarized it is enough to download with `curl`.
@@ -96,7 +105,7 @@ Run `rclone config` to setup. See [rclone config docs](/docs/) for more details.
 
     rclone config
 
-## macOS installation from precompiled binary, using a web browser
+### Precompiled binary, using a web browser  {#macos-precompiled-web}
 
 When downloading a binary with a web browser, the browser will set the macOS
 gatekeeper quarantine attribute. Starting from Catalina, when attempting to run
@@ -109,11 +118,73 @@ The simplest fix is to run
 
     xattr -d com.apple.quarantine rclone
 
-## Install with docker
+## Windows installation {#windows}
 
-The rclone maintains a [docker image for rclone](https://hub.docker.com/r/rclone/rclone).
-These images are autobuilt by docker hub from the rclone source based
-on a minimal Alpine linux image.
+### Precompiled binary {#windows-precompiled}
+
+Fetch the correct binary for your processor type by clicking on these
+links. If not sure, use the first link.
+
+- [Intel/AMD - 64 Bit](https://downloads.rclone.org/rclone-current-linux-amd64.zip)
+- [Intel/AMD - 32 Bit](https://downloads.rclone.org/rclone-current-linux-386.zip)
+- [ARM - 64 Bit](https://downloads.rclone.org/rclone-current-linux-arm64.zip)
+
+Open this file in the Explorer and extract `rclone.exe`. Rclone is a
+portable executable so you can place it wherever is convenient.
+
+Open a CMD window (or powershell) and run the binary. Note that rclone
+does not launch a GUI by default, it runs in the CMD Window.
+
+- Run `rclone.exe config` to setup. See [rclone config docs](/docs/) for more details.
+- Optionally configure [automatic execution](#autostart).
+
+If you are planning to use the [rclone mount](/commands/rclone_mount/)
+feature then you will need to install the third party utility
+[WinFsp](https://winfsp.dev/) also.
+
+### Chocolatey package manager {#windows-chocolatey}
+
+Make sure you have [Choco](https://chocolatey.org/) installed
+
+```
+choco search rclone
+choco install rclone
+```
+
+This will install rclone on your Windows machine. If you are planning
+to use [rclone mount](/commands/rclone_mount/) then
+
+```
+choco install winfsp
+```
+
+will install that too.
+
+Note that this is a third party installer not controlled by the rclone
+developers so it may be out of date. Its current version is as below.
+
+[![Chocolatey package](https://repology.org/badge/version-for-repo/chocolatey/rclone.svg)](https://repology.org/project/rclone/versions)
+
+## Package manager installation {#package-manager}
+
+Many Linux, Windows, macOS and other OS distributions package and
+distribute rclone.
+
+The distributed versions of rclone are often quite out of date and for
+this reason we recommend one of the other installation methods if
+possible.
+
+You can get an idea of how up to date or not your OS distribution's
+package is here.
+
+[![Packaging status](https://repology.org/badge/vertical-allrepos/rclone.svg?columns=3)](https://repology.org/project/rclone/versions)
+
+## Docker installation {#docker}
+
+The rclone developers maintain a [docker image for rclone](https://hub.docker.com/r/rclone/rclone).
+
+These images are built as part of the release process based on a
+minimal Alpine Linux.
 
 The `:latest` tag will always point to the latest stable release.  You
 can use the `:beta` tag to get the latest build from master.  You can
@@ -188,7 +259,7 @@ ls ~/data/mount
 kill %1
 ```
 
-## Install from source
+## Source installation {#source}
 
 Make sure you have git and [Go](https://golang.org/) installed.
 Go version 1.17 or newer is required, latest release is recommended.
@@ -207,7 +278,7 @@ in the same folder. As an initial check you can now run `./rclone version`
 (`.\rclone version` on Windows).
 
 Note that on macOS and Windows the [mount](https://rclone.org/commands/rclone_mount/)
-command will not be available unless you specify additional build tag `cmount`.
+command will not be available unless you specify an additional build tag `cmount`.
 
 ```
 go build -tags cmount
@@ -226,7 +297,7 @@ distribution (make sure you install it in the classic mingw64 subsystem, the
 ucrt64 version is not compatible).
 
 Additionally, on Windows, you must install the third party utility
-[WinFsp](http://www.secfs.net/winfsp/), with the "Developer" feature selected.
+[WinFsp](https://winfsp.dev/), with the "Developer" feature selected.
 If building with cgo, you must also set environment variable CPATH pointing to
 the fuse include directory within the WinFsp installation
 (normally `C:\Program Files (x86)\WinFsp\inc\fuse`).
@@ -241,9 +312,10 @@ go build -trimpath -ldflags -s -tags cmount
 ```
 
 Instead of executing the `go build` command directly, you can run it via the
-Makefile, which also sets version information and copies the resulting rclone
-executable into your GOPATH bin folder (`$(go env GOPATH)/bin`, which
-corresponds to `~/go/bin/rclone` by default).
+Makefile. It changes the version number suffix from "-DEV" to "-beta" and
+appends commit details. It also copies the resulting rclone executable into
+your GOPATH bin folder (`$(go env GOPATH)/bin`, which corresponds to
+`~/go/bin/rclone` by default).
 
 ```
 make
@@ -255,7 +327,15 @@ To include mount command on macOS and Windows with Makefile build:
 make GOTAGS=cmount
 ```
 
-As an alternative you can download the source, build and install rclone in one
+There are other make targets that can be used for more advanced builds,
+such as cross-compiling for all supported os/architectures, embedding
+icon and version info resources into windows executable, and packaging
+results into release artifacts.
+See [Makefile](https://github.com/rclone/rclone/blob/master/Makefile)
+and [cross-compile.go](https://github.com/rclone/rclone/blob/master/bin/cross-compile.go)
+for details.
+
+Another alternative is to download the source, build and install rclone in one
 operation, as a regular Go package. The source will be stored it in the Go
 module cache, and the resulting executable will be in your GOPATH bin folder
 (`$(go env GOPATH)/bin`, which corresponds to `~/go/bin/rclone` by default).
@@ -274,7 +354,7 @@ with the current version):
 go get github.com/rclone/rclone
 ```
 
-## Installation with Ansible
+## Ansible installation {#ansible}
 
 This can be done with [Stefan Weichinger's ansible
 role](https://github.com/stefangweichinger/ansible-rclone).
@@ -290,7 +370,7 @@ Instructions
           - rclone
 ```
 
-## Portable installation
+## Portable installation {#portable}
 
 As mentioned [above](https://rclone.org/install/#quickstart), rclone is single
 executable (`rclone`, or `rclone.exe` on Windows) that you can download as a

docs/content/onedrive.md

@@ -152,7 +152,7 @@ A common error is that the publisher of the App is not verified.
 You may try to [verify you account](https://docs.microsoft.com/en-us/azure/active-directory/develop/publisher-verification-overview), or try to limit the App to your organization only, as shown below.
 
 1. Make sure to create the App with your business account.
-2. Follow the steps above to create an App. However, we need a different account type here: `Accounts in this organizational directory only (*** - Single tenant)`. Note that you can also change the account type aftering creating the App.
+2. Follow the steps above to create an App. However, we need a different account type here: `Accounts in this organizational directory only (*** - Single tenant)`. Note that you can also change the account type after creating the App.
 3. Find the [tenant ID](https://docs.microsoft.com/en-us/azure/active-directory/fundamentals/active-directory-how-to-find-tenant) of your organization.
 4. In the rclone config, set `auth_url` to `https://login.microsoftonline.com/YOUR_TENANT_ID/oauth2/v2.0/authorize`.
 5. In the rclone config, set `token_url` to `https://login.microsoftonline.com/YOUR_TENANT_ID/oauth2/v2.0/token`.

docs/content/s3.md

@@ -19,6 +19,7 @@ The S3 backend can be used with a number of different providers:
 {{< provider name="Huawei OBS" home="https://www.huaweicloud.com/intl/en-us/product/obs.html" config="/s3/#huawei-obs" >}}
 {{< provider name="IBM COS S3" home="http://www.ibm.com/cloud/object-storage" config="/s3/#ibm-cos-s3" >}}
 {{< provider name="IDrive e2" home="https://www.idrive.com/e2/" config="/s3/#idrive-e2" >}}
+{{< provider name="IONOS Cloud" home="https://cloud.ionos.com/storage/object-storage" config="/s3/#ionos" >}}
 {{< provider name="Minio" home="https://www.minio.io/" config="/s3/#minio" >}}
 {{< provider name="RackCorp Object Storage" home="https://www.rackcorp.com/" config="/s3/#RackCorp" >}}
 {{< provider name="Scaleway" home="https://www.scaleway.com/en/object-storage/" config="/s3/#scaleway" >}}
@@ -3533,6 +3534,169 @@ d) Delete this remote
 y/e/d> y
 ```
 
+### IONOS Cloud {#ionos}
+
+[IONOS S3 Object Storage](https://cloud.ionos.com/storage/object-storage) is a service offered by IONOS for storing and accessing unstructured data.
+To connect to the service, you will need an access key and a secret key. These can be found in the [Data Center Designer](https://dcd.ionos.com/), by selecting **Manager resources** > **Object Storage Key Manager**.
+
+
+Here is an example of a configuration. First, run `rclone config`. This will walk you through an interactive setup process. Type `n` to add the new remote, and then enter a name:
+
+```
+Enter name for new remote.
+name> ionos-fra
+```
+
+Type `s3` to choose the connection type:
+```
+Option Storage.
+Type of storage to configure.
+Choose a number from below, or type in your own value.
+[snip]
+XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi
+   \ (s3)
+[snip]
+Storage> s3
+```
+
+Type `IONOS`:
+```
+Option provider.
+Choose your S3 provider.
+Choose a number from below, or type in your own value.
+Press Enter to leave empty.
+[snip]
+XX / IONOS Cloud
+   \ (IONOS)
+[snip]
+provider> IONOS
+```
+
+Press Enter to choose the default option `Enter AWS credentials in the next step`:
+```
+Option env_auth.
+Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars).
+Only applies if access_key_id and secret_access_key is blank.
+Choose a number from below, or type in your own boolean value (true or false).
+Press Enter for the default (false).
+ 1 / Enter AWS credentials in the next step.
+   \ (false)
+ 2 / Get AWS credentials from the environment (env vars or IAM).
+   \ (true)
+env_auth>
+```
+
+Enter your Access Key and Secret key. These can be retrieved in the [Data Center Designer](https://dcd.ionos.com/), click on the menu “Manager resources”  / "Object Storage Key Manager".
+```
+Option access_key_id.
+AWS Access Key ID.
+Leave blank for anonymous access or runtime credentials.
+Enter a value. Press Enter to leave empty.
+access_key_id> YOUR_ACCESS_KEY
+
+Option secret_access_key.
+AWS Secret Access Key (password).
+Leave blank for anonymous access or runtime credentials.
+Enter a value. Press Enter to leave empty.
+secret_access_key> YOUR_SECRET_KEY
+```
+
+Choose the region where your bucket is located:
+```
+Option region.
+Region where your bucket will be created and your data stored.
+Choose a number from below, or type in your own value.
+Press Enter to leave empty.
+ 1 / Frankfurt, Germany
+   \ (de)
+ 2 / Berlin, Germany
+   \ (eu-central-2)
+ 3 / Logrono, Spain
+   \ (eu-south-2)
+region> 2
+```
+
+Choose the endpoint from the same region:
+```
+Option endpoint.
+Endpoint for IONOS S3 Object Storage.
+Specify the endpoint from the same region.
+Choose a number from below, or type in your own value.
+Press Enter to leave empty.
+ 1 / Frankfurt, Germany
+   \ (s3-eu-central-1.ionoscloud.com)
+ 2 / Berlin, Germany
+   \ (s3-eu-central-2.ionoscloud.com)
+ 3 / Logrono, Spain
+   \ (s3-eu-south-2.ionoscloud.com)
+endpoint> 1
+```
+
+Press Enter to choose the default option or choose the desired ACL setting:
+```
+Option acl.
+Canned ACL used when creating buckets and storing or copying objects.
+This ACL is used for creating objects and if bucket_acl isn't set, for creating buckets too.
+For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
+Note that this ACL is applied when server-side copying objects as S3
+doesn't copy the ACL from the source but rather writes a fresh one.
+Choose a number from below, or type in your own value.
+Press Enter to leave empty.
+   / Owner gets FULL_CONTROL.
+ 1 | No one else has access rights (default).
+   \ (private)
+   / Owner gets FULL_CONTROL.
+[snip]
+acl>
+```
+
+Press Enter to skip the advanced config:
+```
+Edit advanced config?
+y) Yes
+n) No (default)
+y/n>
+```
+
+Press Enter to save the configuration, and then `q` to quit the configuration process:
+```
+Configuration complete.
+Options:
+- type: s3
+- provider: IONOS
+- access_key_id: YOUR_ACCESS_KEY
+- secret_access_key: YOUR_SECRET_KEY
+- endpoint: s3-eu-central-1.ionoscloud.com
+Keep this "ionos-fra" remote?
+y) Yes this is OK (default)
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
+```
+
+Done! Now you can try some commands (for macOS, use `./rclone` instead of `rclone`).
+
+1)  Create a bucket (the name must be unique within the whole IONOS S3)
+```
+rclone mkdir ionos-fra:my-bucket
+```
+2)  List available buckets
+```
+rclone lsd ionos-fra:
+```
+4) Copy a file from local to remote
+```
+rclone copy /Users/file.txt ionos-fra:my-bucket
+```
+3)  List contents of a bucket
+```
+rclone ls ionos-fra:my-bucket
+```
+5)  Copy a file from remote to local
+```
+rclone copy ionos-fra:my-bucket/file.txt
+```
+
 ### Minio
 
 [Minio](https://minio.io/) is an object storage server built for cloud application developers and devops.

docs/layouts/chrome/menu.html

@@ -17,9 +17,9 @@
   <div class="card-body">
     <p class="menu">
       <!-- Non tracking sharing links from: https://sharingbuttons.io/ -->
-      <i class="fab fa-twitter" aria-hidden="true"></i> <a href="https://twitter.com/intent/tweet/?text=rclone%20-%20rsync%20for%20cloud%20storage%20from%20%40njcw&amp;url=https%3A%2F%2Frclone.org" target="_blank" rel="noopener" aria-label="Share on Twitter">Twitter</a><br />
-      <i class="fab fa-facebook" aria-hidden="true"></i> <a href="https://facebook.com/sharer/sharer.php?u=https%3A%2F%2Frclone.org" target="_blank" rel="noopener" aria-label="Share on Facebook">Facebook</a><br />
-      <i class="fab fa-reddit" aria-hidden="true"></i> <a  href="https://reddit.com/submit/?url=https%3A%2F%2Frclone.org&amp;resubmit=true&amp;title=rclone%20-%20rsync%20for%20cloud%20storage" target="_blank" rel="noopener" aria-label="Share on Reddit">Reddit</a><br />
+      <i class="fab fa-twitter fa-fw" aria-hidden="true"></i> <a href="https://twitter.com/intent/tweet/?text=rclone%20-%20rsync%20for%20cloud%20storage%20from%20%40njcw&amp;url=https%3A%2F%2Frclone.org" target="_blank" rel="noopener" aria-label="Share on Twitter">Twitter</a><br />
+      <i class="fab fa-facebook fa-fw" aria-hidden="true"></i> <a href="https://facebook.com/sharer/sharer.php?u=https%3A%2F%2Frclone.org" target="_blank" rel="noopener" aria-label="Share on Facebook">Facebook</a><br />
+      <i class="fab fa-reddit fa-fw" aria-hidden="true"></i> <a  href="https://reddit.com/submit/?url=https%3A%2F%2Frclone.org&amp;resubmit=true&amp;title=rclone%20-%20rsync%20for%20cloud%20storage" target="_blank" rel="noopener" aria-label="Share on Reddit">Reddit</a><br />
       <iframe src="//ghbtns.com/github-btn.html?user=rclone&amp;repo=rclone&amp;type=star&amp;count=true" allowtransparency="true" frameborder="0" scrolling="no" width="120" height="20"></iframe>
     </p>
   </div>
@@ -31,12 +31,12 @@
   </div>
   <div class="card-body">
     <p class="menu">
-      <i class="fa fa-comments" aria-hidden="true"></i> <a href="https://forum.rclone.org">Rclone forum</a><br />
-      <i class="fab fa-github" aria-hidden="true"></i> <a href="https://github.com/rclone/rclone">GitHub project</a><br />
-      <i class="fab fa-slack" aria-hidden="true"></i> <a href="https://slack-invite.rclone.org/">Rclone slack</a><br />
-      <i class="fa fa-book" aria-hidden="true"></i> <a href="https://github.com/rclone/rclone/wiki">Rclone Wiki</a><br />
-      <i class="fa fa-heart heart" aria-hidden="true"></i> <a href="/donate/">Donate</a><br />
-      <i class="fab fa-twitter" aria-hidden="true"></i> <a href="https://twitter.com/njcw">@njcw</a>
+      <i class="fa fa-comments fa-fw" aria-hidden="true"></i> <a href="https://forum.rclone.org">Rclone forum</a><br />
+      <i class="fab fa-github fa-fw" aria-hidden="true"></i> <a href="https://github.com/rclone/rclone">GitHub project</a><br />
+      <i class="fab fa-slack fa-fw" aria-hidden="true"></i> <a href="https://slack-invite.rclone.org/">Rclone slack</a><br />
+      <i class="fa fa-book fa-fw" aria-hidden="true"></i> <a href="https://github.com/rclone/rclone/wiki">Rclone Wiki</a><br />
+      <i class="fa fa-heart heart fa-fw" aria-hidden="true"></i> <a href="/donate/">Donate</a><br />
+      <i class="fab fa-twitter fa-fw" aria-hidden="true"></i> <a href="https://twitter.com/njcw">@njcw</a>
     </p>
   </div>
 </div>

docs/layouts/chrome/navbar.html

@@ -14,18 +14,18 @@
           Docs
         </a>
         <div class="dropdown-menu" aria-labelledby="navbarDropdown">
-          <a class="dropdown-item" href="/install/"><i class="fa fa-book"></i> Installation</a>
-          <a class="dropdown-item" href="/docs/"><i class="fa fa-book"></i> Usage</a>
-          <a class="dropdown-item" href="/filtering/"><i class="fa fa-book"></i> Filtering</a>
-          <a class="dropdown-item" href="/gui/"><i class="fa fa-book"></i> GUI</a>
-          <a class="dropdown-item" href="/rc/"><i class="fa fa-book"></i> Remote Control</a>
-          <a class="dropdown-item" href="/changelog/"><i class="fa fa-book"></i> Changelog</a>
-          <a class="dropdown-item" href="/bugs/"><i class="fa fa-book"></i> Bugs</a>
-          <a class="dropdown-item" href="/faq/"><i class="fa fa-book"></i> FAQ</a>
-          <a class="dropdown-item" href="/flags/"><i class="fa fa-book"></i> Flags</a>
-          <a class="dropdown-item" href="/licence/"><i class="fa fa-book"></i> Licence</a>
-          <a class="dropdown-item" href="/authors/"><i class="fa fa-book"></i> Authors</a>
-          <a class="dropdown-item" href="/privacy/"><i class="fa fa-book"></i> Privacy Policy</a>
+          <a class="dropdown-item" href="/install/"><i class="fa fa-book fa-fw"></i> Installation</a>
+          <a class="dropdown-item" href="/docs/"><i class="fa fa-book fa-fw"></i> Usage</a>
+          <a class="dropdown-item" href="/filtering/"><i class="fa fa-book fa-fw"></i> Filtering</a>
+          <a class="dropdown-item" href="/gui/"><i class="fa fa-book fa-fw"></i> GUI</a>
+          <a class="dropdown-item" href="/rc/"><i class="fa fa-book fa-fw"></i> Remote Control</a>
+          <a class="dropdown-item" href="/changelog/"><i class="fa fa-book fa-fw"></i> Changelog</a>
+          <a class="dropdown-item" href="/bugs/"><i class="fa fa-book fa-fw"></i> Bugs</a>
+          <a class="dropdown-item" href="/faq/"><i class="fa fa-book fa-fw"></i> FAQ</a>
+          <a class="dropdown-item" href="/flags/"><i class="fa fa-book fa-fw"></i> Flags</a>
+          <a class="dropdown-item" href="/licence/"><i class="fa fa-book fa-fw"></i> Licence</a>
+          <a class="dropdown-item" href="/authors/"><i class="fa fa-book fa-fw"></i> Authors</a>
+          <a class="dropdown-item" href="/privacy/"><i class="fa fa-book fa-fw"></i> Privacy Policy</a>
         </div>
       </li>
       <li class="nav-item active dropdown">
@@ -33,12 +33,12 @@
           Commands
         </a>
         <div class="dropdown-menu pre-scrollable" aria-labelledby="navbarDropdown">
-          <a class="dropdown-item" href="/commands/"><i class="fas fa-map"></i> Overview</a>
+          <a class="dropdown-item" href="/commands/"><i class="fas fa-map fa-fw"></i> Overview</a>
           <div class="dropdown-divider"></div>
           {{ with .Site.GetPage "/commands" }}
             {{ range .Data.Pages }}
               {{ if lt (countwords .Title) 3 }}
-                <a class="dropdown-item" href="{{ .RelPermalink }}"><i class="fa fa-book"></i> {{ .Title | markdownify }}</a>
+                <a class="dropdown-item" href="{{ .RelPermalink }}"><i class="fa fa-book fa-fw"></i> {{ .Title | markdownify }}</a>
               {{ end }}
             {{ end }}
           {{ end }}
@@ -49,65 +49,65 @@
           Storage Systems
         </a>
         <div class="dropdown-menu pre-scrollable" aria-labelledby="navbarDropdown">
-          <a class="dropdown-item" href="/overview/"><i class="fas fa-map"></i> Overview</a>
+          <a class="dropdown-item" href="/overview/"><i class="fas fa-map fa-fw"></i> Overview</a>
           <div class="dropdown-divider"></div>
-          <a class="dropdown-item" href="/fichier/"><i class="fa fa-archive"></i> 1Fichier</a>
-          <a class="dropdown-item" href="/netstorage/"><i class="fas fa-database"></i> Akamai NetStorage</a>
-          <a class="dropdown-item" href="/alias/"><i class="fa fa-link"></i> Alias</a>
-          <a class="dropdown-item" href="/amazonclouddrive/"><i class="fab fa-amazon"></i> Amazon Drive</a>
-          <a class="dropdown-item" href="/s3/"><i class="fab fa-amazon"></i> Amazon S3</a>
-          <a class="dropdown-item" href="/b2/"><i class="fa fa-fire"></i> Backblaze B2</a>
-          <a class="dropdown-item" href="/box/"><i class="fa fa-archive"></i> Box</a>
-          <a class="dropdown-item" href="/chunker/"><i class="fa fa-cut"></i> Chunker (splits large files)</a>
-          <a class="dropdown-item" href="/compress/"><i class="fas fa-compress"></i> Compress (transparent gzip compression)</a>
-          <a class="dropdown-item" href="/combine/"><i class="fa fa-folder-plus"></i> Combine (remotes into a directory tree)</a>
-          <a class="dropdown-item" href="/sharefile/"><i class="fas fa-share-square"></i> Citrix ShareFile</a>
-          <a class="dropdown-item" href="/crypt/"><i class="fa fa-lock"></i> Crypt (encrypts the others)</a>
-          <a class="dropdown-item" href="/koofr/#digi-storage"><i class="fa fa-cloud"></i> Digi Storage</a>
-          <a class="dropdown-item" href="/dropbox/"><i class="fab fa-dropbox"></i> Dropbox</a>
-          <a class="dropdown-item" href="/filefabric/"><i class="fa fa-cloud"></i> Enterprise File Fabric</a>
-          <a class="dropdown-item" href="/ftp/"><i class="fa fa-file"></i> FTP</a>
-          <a class="dropdown-item" href="/googlecloudstorage/"><i class="fab fa-google"></i> Google Cloud Storage</a>
-          <a class="dropdown-item" href="/drive/"><i class="fab fa-google"></i> Google Drive</a>
-          <a class="dropdown-item" href="/googlephotos/"><i class="fas fa-images"></i> Google Photos</a>
-          <a class="dropdown-item" href="/hasher/"><i class="fa fa-check-double"></i> Hasher (better checksums for others)</a>
-          <a class="dropdown-item" href="/hdfs/"><i class="fa fa-globe"></i> HDFS (Hadoop Distributed Filesystem)</a>
-          <a class="dropdown-item" href="/hidrive/"><i class="fa fa-cloud"></i> HiDrive</a>
-          <a class="dropdown-item" href="/http/"><i class="fa fa-globe"></i> HTTP</a>
-          <a class="dropdown-item" href="/hubic/"><i class="fa fa-space-shuttle"></i> Hubic</a>
-          <a class="dropdown-item" href="/internetarchive/"><i class="fa fa-archive"></i> Internet Archive</a>
-          <a class="dropdown-item" href="/jottacloud/"><i class="fa fa-cloud"></i> Jottacloud</a>
-          <a class="dropdown-item" href="/koofr/"><i class="fa fa-suitcase"></i> Koofr</a>
-          <a class="dropdown-item" href="/mailru/"><i class="fa fa-at"></i> Mail.ru Cloud</a>
-          <a class="dropdown-item" href="/mega/"><i class="fa fa-archive"></i> Mega</a>
-          <a class="dropdown-item" href="/memory/"><i class="fas fa-memory"></i> Memory</a>
-          <a class="dropdown-item" href="/azureblob/"><i class="fab fa-windows"></i> Microsoft Azure Blob Storage</a>
-          <a class="dropdown-item" href="/onedrive/"><i class="fab fa-windows"></i> Microsoft OneDrive</a>
-          <a class="dropdown-item" href="/opendrive/"><i class="fa fa-space-shuttle"></i> OpenDrive</a>
-          <a class="dropdown-item" href="/qingstor/"><i class="fas fa-hdd"></i> QingStor</a>
-          <a class="dropdown-item" href="/swift/"><i class="fa fa-space-shuttle"></i> Openstack Swift</a>
-          <a class="dropdown-item" href="/pcloud/"><i class="fa fa-cloud"></i> pCloud</a>
-          <a class="dropdown-item" href="/premiumizeme/"><i class="fa fa-user"></i> premiumize.me</a>
-          <a class="dropdown-item" href="/putio/"><i class="fas fa-parking"></i> put.io</a>
-          <a class="dropdown-item" href="/seafile/"><i class="fa fa-server"></i> Seafile</a>
-          <a class="dropdown-item" href="/sftp/"><i class="fa fa-server"></i> SFTP</a>
-          <a class="dropdown-item" href="/sia/"><i class="fa fa-globe"></i> Sia</a>
-          <a class="dropdown-item" href="/storj/"><i class="fas fa-dove"></i> Storj</a>
-          <a class="dropdown-item" href="/sugarsync/"><i class="fas fa-dove"></i> SugarSync</a>
-          <a class="dropdown-item" href="/uptobox/"><i class="fa fa-archive"></i> Uptobox</a>
-          <a class="dropdown-item" href="/union/"><i class="fa fa-link"></i> Union (merge backends)</a>
-          <a class="dropdown-item" href="/webdav/"><i class="fa fa-server"></i> WebDAV</a>
-          <a class="dropdown-item" href="/yandex/"><i class="fa fa-space-shuttle"></i> Yandex Disk</a>
-          <a class="dropdown-item" href="/zoho/"><i class="fas fa-folder"></i> Zoho WorkDrive</a>
+          <a class="dropdown-item" href="/fichier/"><i class="fa fa-archive fa-fw"></i> 1Fichier</a>
+          <a class="dropdown-item" href="/netstorage/"><i class="fas fa-database fa-fw"></i> Akamai NetStorage</a>
+          <a class="dropdown-item" href="/alias/"><i class="fa fa-link fa-fw"></i> Alias</a>
+          <a class="dropdown-item" href="/amazonclouddrive/"><i class="fab fa-amazon fa-fw"></i> Amazon Drive</a>
+          <a class="dropdown-item" href="/s3/"><i class="fab fa-amazon fa-fw"></i> Amazon S3</a>
+          <a class="dropdown-item" href="/b2/"><i class="fa fa-fire fa-fw"></i> Backblaze B2</a>
+          <a class="dropdown-item" href="/box/"><i class="fa fa-archive fa-fw"></i> Box</a>
+          <a class="dropdown-item" href="/chunker/"><i class="fa fa-cut fa-fw"></i> Chunker (splits large files)</a>
+          <a class="dropdown-item" href="/compress/"><i class="fas fa-compress fa-fw"></i> Compress (transparent gzip compression)</a>
+          <a class="dropdown-item" href="/combine/"><i class="fa fa-folder-plus fa-fw"></i> Combine (remotes into a directory tree)</a>
+          <a class="dropdown-item" href="/sharefile/"><i class="fas fa-share-square fa-fw"></i> Citrix ShareFile</a>
+          <a class="dropdown-item" href="/crypt/"><i class="fa fa-lock fa-fw"></i> Crypt (encrypts the others)</a>
+          <a class="dropdown-item" href="/koofr/#digi-storage"><i class="fa fa-cloud fa-fw"></i> Digi Storage</a>
+          <a class="dropdown-item" href="/dropbox/"><i class="fab fa-dropbox fa-fw"></i> Dropbox</a>
+          <a class="dropdown-item" href="/filefabric/"><i class="fa fa-cloud fa-fw"></i> Enterprise File Fabric</a>
+          <a class="dropdown-item" href="/ftp/"><i class="fa fa-file fa-fw"></i> FTP</a>
+          <a class="dropdown-item" href="/googlecloudstorage/"><i class="fab fa-google fa-fw"></i> Google Cloud Storage</a>
+          <a class="dropdown-item" href="/drive/"><i class="fab fa-google fa-fw"></i> Google Drive</a>
+          <a class="dropdown-item" href="/googlephotos/"><i class="fas fa-images fa-fw"></i> Google Photos</a>
+          <a class="dropdown-item" href="/hasher/"><i class="fa fa-check-double fa-fw"></i> Hasher (better checksums for others)</a>
+          <a class="dropdown-item" href="/hdfs/"><i class="fa fa-globe fa-fw"></i> HDFS (Hadoop Distributed Filesystem)</a>
+          <a class="dropdown-item" href="/hidrive/"><i class="fa fa-cloud fa-fw"></i> HiDrive</a>
+          <a class="dropdown-item" href="/http/"><i class="fa fa-globe fa-fw"></i> HTTP</a>
+          <a class="dropdown-item" href="/hubic/"><i class="fa fa-space-shuttle fa-fw"></i> Hubic</a>
+          <a class="dropdown-item" href="/internetarchive/"><i class="fa fa-archive fa-fw"></i> Internet Archive</a>
+          <a class="dropdown-item" href="/jottacloud/"><i class="fa fa-cloud fa-fw"></i> Jottacloud</a>
+          <a class="dropdown-item" href="/koofr/"><i class="fa fa-suitcase fa-fw"></i> Koofr</a>
+          <a class="dropdown-item" href="/mailru/"><i class="fa fa-at fa-fw"></i> Mail.ru Cloud</a>
+          <a class="dropdown-item" href="/mega/"><i class="fa fa-archive fa-fw"></i> Mega</a>
+          <a class="dropdown-item" href="/memory/"><i class="fas fa-memory fa-fw"></i> Memory</a>
+          <a class="dropdown-item" href="/azureblob/"><i class="fab fa-windows fa-fw"></i> Microsoft Azure Blob Storage</a>
+          <a class="dropdown-item" href="/onedrive/"><i class="fab fa-windows fa-fw"></i> Microsoft OneDrive</a>
+          <a class="dropdown-item" href="/opendrive/"><i class="fa fa-space-shuttle fa-fw"></i> OpenDrive</a>
+          <a class="dropdown-item" href="/qingstor/"><i class="fas fa-hdd fa-fw"></i> QingStor</a>
+          <a class="dropdown-item" href="/swift/"><i class="fa fa-space-shuttle fa-fw"></i> Openstack Swift</a>
+          <a class="dropdown-item" href="/pcloud/"><i class="fa fa-cloud fa-fw"></i> pCloud</a>
+          <a class="dropdown-item" href="/premiumizeme/"><i class="fa fa-user fa-fw"></i> premiumize.me</a>
+          <a class="dropdown-item" href="/putio/"><i class="fas fa-parking fa-fw"></i> put.io</a>
+          <a class="dropdown-item" href="/seafile/"><i class="fa fa-server fa-fw"></i> Seafile</a>
+          <a class="dropdown-item" href="/sftp/"><i class="fa fa-server fa-fw"></i> SFTP</a>
+          <a class="dropdown-item" href="/sia/"><i class="fa fa-globe fa-fw"></i> Sia</a>
+          <a class="dropdown-item" href="/storj/"><i class="fas fa-dove fa-fw"></i> Storj</a>
+          <a class="dropdown-item" href="/sugarsync/"><i class="fas fa-dove fa-fw"></i> SugarSync</a>
+          <a class="dropdown-item" href="/uptobox/"><i class="fa fa-archive fa-fw"></i> Uptobox</a>
+          <a class="dropdown-item" href="/union/"><i class="fa fa-link fa-fw"></i> Union (merge backends)</a>
+          <a class="dropdown-item" href="/webdav/"><i class="fa fa-server fa-fw"></i> WebDAV</a>
+          <a class="dropdown-item" href="/yandex/"><i class="fa fa-space-shuttle fa-fw"></i> Yandex Disk</a>
+          <a class="dropdown-item" href="/zoho/"><i class="fas fa-folder fa-fw"></i> Zoho WorkDrive</a>
           <div class="dropdown-divider"></div>
-          <a class="dropdown-item" href="/local/"><i class="fas fa-hdd"></i> The local filesystem</a>
+          <a class="dropdown-item" href="/local/"><i class="fas fa-hdd fa-fw"></i> The local filesystem</a>
         </div>
       </li>
       <li class="nav-item active">
-        <a class="nav-link" href="/contact/"><i class="fa fa-envelope"></i> Contact</a>
+        <a class="nav-link" href="/contact/"><i class="fa fa-envelope fa-fw"></i> Contact</a>
       </li>
       <li class="nav-item active">
-        <a class="nav-link" href="/donate/"><i class="fa fa-heart heart"></i> Donate</a>
+        <a class="nav-link" href="/donate/"><i class="fa fa-heart heart fa-fw"></i> Donate</a>
       </li>
     </ul>
     <form class="form-inline" name="search_form" action="https://google.com/search" target="_blank" onsubmit="on_search();">

docs/static/css/custom.css

@@ -142,18 +142,6 @@ h5 {
     font-size: 95%;
 }
 
-/* Align dropdown items when icons have different sizes */
-.dropdown-item .fa, .fab, .fad, .fal, .far, .fas {
-    width: 20px;
-    text-align: center;
-}
-
-/* Align menu items when icons have different sizes */
-.menu .fa, .fab, .fad, .fal, .far, .fas {
-    width: 18px;
-    text-align: center;
-}
-
 /* Make primary buttons rclone colours. Should learn sass and do this the proper way! */
 .btn-primary {
     background-color: #3f79ad;

fs/config.go

@@ -29,6 +29,12 @@ var (
 		return errors.New("no config file set handler")
 	}
 
+	// Check if the config file has the named section
+	//
+	// This is a function pointer to decouple the config
+	// implementation from the fs
+	ConfigFileHasSection = func(section string) bool { return false }
+
 	// CountError counts an error.  If any errors have been
 	// counted then rclone will exit with a non zero error code.
 	//

fs/config/config.go

@@ -117,6 +117,9 @@ func init() {
 	// Set the function pointers up in fs
 	fs.ConfigFileGet = FileGetFlag
 	fs.ConfigFileSet = SetValueAndSave
+	fs.ConfigFileHasSection = func(section string) bool {
+		return LoadedData().HasSection(section)
+	}
 	configPath = makeConfigPath()
 	cacheDir = makeCacheDir() // Has fallback to tempDir, so set that first
 	data = newDefaultStorage()

fs/config/crypt.go

@@ -26,9 +26,6 @@ var (
 	// When nil, no encryption will be used for saving.
 	configKey []byte
 
-	// PasswordPromptOutput is output of prompt for password
-	PasswordPromptOutput = os.Stderr
-
 	// PassConfigKeyForDaemonization if set to true, the configKey
 	// is obscured with obscure.Obscure and saved to a temp file
 	// when it is calculated from the password. The path of that

fs/config/ui.go

@@ -716,9 +716,9 @@ func checkPassword(password string) (string, error) {
 
 // GetPassword asks the user for a password with the prompt given.
 func GetPassword(prompt string) string {
-	_, _ = fmt.Fprintln(PasswordPromptOutput, prompt)
+	_, _ = fmt.Fprintln(terminal.Out, prompt)
 	for {
-		_, _ = fmt.Fprint(PasswordPromptOutput, "password:")
+		_, _ = fmt.Fprint(terminal.Out, "password:")
 		password := ReadPassword()
 		password, err := checkPassword(password)
 		if err == nil {

fs/features.go

@@ -29,6 +29,7 @@ type Features struct {
 	ReadMetadata            bool // can read metadata from objects
 	WriteMetadata           bool // can write metadata to objects
 	UserMetadata            bool // can read/write general purpose metadata
+	FilterAware             bool // can make use of filters if provided for listing
 
 	// Purge all files in the directory specified
 	//
@@ -320,6 +321,7 @@ func (ft *Features) Mask(ctx context.Context, f Fs) *Features {
 	// ft.IsLocal = ft.IsLocal && mask.IsLocal Don't propagate IsLocal
 	ft.SlowModTime = ft.SlowModTime && mask.SlowModTime
 	ft.SlowHash = ft.SlowHash && mask.SlowHash
+	ft.FilterAware = ft.FilterAware && mask.FilterAware
 
 	if mask.Purge == nil {
 		ft.Purge = nil

fs/log/redirect_stderr_unix.go

@@ -9,17 +9,17 @@ import (
 	"log"
 	"os"
 
-	"github.com/rclone/rclone/fs/config"
+	"github.com/rclone/rclone/lib/terminal"
 	"golang.org/x/sys/unix"
 )
 
 // redirectStderr to the file passed in
 func redirectStderr(f *os.File) {
-	passPromptFd, err := unix.Dup(int(os.Stderr.Fd()))
+	termFd, err := unix.Dup(int(os.Stderr.Fd()))
 	if err != nil {
 		log.Fatalf("Failed to duplicate stderr: %v", err)
 	}
-	config.PasswordPromptOutput = os.NewFile(uintptr(passPromptFd), "passPrompt")
+	terminal.RawOut = os.NewFile(uintptr(termFd), "termOut")
 	err = unix.Dup2(int(f.Fd()), int(os.Stderr.Fd()))
 	if err != nil {
 		log.Fatalf("Failed to redirect stderr to file: %v", err)

fs/log/redirect_stderr_windows.go

@@ -12,29 +12,43 @@ package log
 import (
 	"log"
 	"os"
-	"syscall"
+
+	"github.com/rclone/rclone/lib/terminal"
+	"golang.org/x/sys/windows"
 )
 
-var (
-	kernel32         = syscall.MustLoadDLL("kernel32.dll")
-	procSetStdHandle = kernel32.MustFindProc("SetStdHandle")
-)
-
-func setStdHandle(stdhandle int32, handle syscall.Handle) error {
-	r0, _, e1 := syscall.Syscall(procSetStdHandle.Addr(), 2, uintptr(stdhandle), uintptr(handle), 0)
-	if r0 == 0 {
-		if e1 != 0 {
-			return error(e1)
-		}
-		return syscall.EINVAL
+// dup oldfd creating a functional copy as newfd
+// conceptually the same as the unix `dup()` function
+func dup(oldfd uintptr) (newfd uintptr, err error) {
+	var (
+		newfdHandle   windows.Handle
+		processHandle = windows.CurrentProcess()
+	)
+	err = windows.DuplicateHandle(
+		processHandle,                 // hSourceProcessHandle
+		windows.Handle(oldfd),         // hSourceHandle
+		processHandle,                 // hTargetProcessHandle
+		&newfdHandle,                  // lpTargetHandle
+		0,                             // dwDesiredAccess
+		true,                          // bInheritHandle
+		windows.DUPLICATE_SAME_ACCESS, // dwOptions
+	)
+	if err != nil {
+		return 0, err
 	}
-	return nil
+	return uintptr(newfdHandle), nil
 }
 
 // redirectStderr to the file passed in
 func redirectStderr(f *os.File) {
-	err := setStdHandle(syscall.STD_ERROR_HANDLE, syscall.Handle(f.Fd()))
+	termFd, err := dup(os.Stderr.Fd())
+	if err != nil {
+		log.Fatalf("Failed to duplicate stderr: %v", err)
+	}
+	terminal.RawOut = os.NewFile(termFd, "termOut")
+	err = windows.SetStdHandle(windows.STD_ERROR_HANDLE, windows.Handle(f.Fd()))
 	if err != nil {
 		log.Fatalf("Failed to redirect stderr to file: %v", err)
 	}
+	os.Stderr = f
 }

fs/march/march.go

@@ -83,7 +83,7 @@ func (m *March) makeListDir(ctx context.Context, f fs.Fs, includeAll bool) listD
 	if !(ci.UseListR && f.Features().ListR != nil) && // !--fast-list active and
 		!(ci.NoTraverse && fi.HaveFilesFrom()) { // !(--files-from and --no-traverse)
 		return func(dir string) (entries fs.DirEntries, err error) {
-			dirCtx := filter.SetUseFilter(m.Ctx, !includeAll) // make filter-aware backends constrain List
+			dirCtx := filter.SetUseFilter(m.Ctx, f.Features().FilterAware && !includeAll) // make filter-aware backends constrain List
 			return list.DirSorted(dirCtx, f, includeAll, dir)
 		}
 	}
@@ -100,7 +100,7 @@ func (m *March) makeListDir(ctx context.Context, f fs.Fs, includeAll bool) listD
 		mu.Lock()
 		defer mu.Unlock()
 		if !started {
-			dirCtx := filter.SetUseFilter(m.Ctx, !includeAll) // make filter-aware backends constrain List
+			dirCtx := filter.SetUseFilter(m.Ctx, f.Features().FilterAware && !includeAll) // make filter-aware backends constrain List
 			dirs, dirsErr = walk.NewDirTree(dirCtx, f, m.Dir, includeAll, ci.MaxDepth)
 			started = true
 		}

fs/newfs.go

@@ -26,6 +26,9 @@ import (
 // up with drive letters.
 func NewFs(ctx context.Context, path string) (Fs, error) {
 	Debugf(nil, "Creating backend with remote %q", path)
+	if ConfigFileHasSection(path) {
+		Logf(nil, "%q refers to a local folder, use %q to refer to your remote or %q to hide this warning", path, path+":", "./"+path)
+	}
 	fsInfo, configName, fsPath, config, err := ConfigFs(path)
 	if err != nil {
 		return nil, err

fs/walk/walk.go

@@ -64,7 +64,7 @@ type Func func(path string, entries fs.DirEntries, err error) error
 func Walk(ctx context.Context, f fs.Fs, path string, includeAll bool, maxLevel int, fn Func) error {
 	ci := fs.GetConfig(ctx)
 	fi := filter.GetConfig(ctx)
-	ctx = filter.SetUseFilter(ctx, !includeAll) // make filter-aware backends constrain List
+	ctx = filter.SetUseFilter(ctx, f.Features().FilterAware && !includeAll) // make filter-aware backends constrain List
 	if ci.NoTraverse && fi.HaveFilesFrom() {
 		return walkR(ctx, f, path, includeAll, maxLevel, fn, fi.MakeListR(ctx, f.NewObject))
 	}
@@ -158,7 +158,7 @@ func ListR(ctx context.Context, f fs.Fs, path string, includeAll bool, maxLevel
 		fi.UsesDirectoryFilters() { // ...using any directory filters
 		return listRwalk(ctx, f, path, includeAll, maxLevel, listType, fn)
 	}
-	ctx = filter.SetUseFilter(ctx, !includeAll) // make filter-aware backends constrain List
+	ctx = filter.SetUseFilter(ctx, f.Features().FilterAware && !includeAll) // make filter-aware backends constrain List
 	return listR(ctx, f, path, includeAll, listType, fn, doListR, listType.Dirs() && f.Features().BucketBased)
 }
 

go.mod

@@ -135,7 +135,7 @@ require (
 	github.com/golang-jwt/jwt/v4 v4.1.0 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
-	github.com/jlaffaye/ftp v0.0.0-20220630165035-11536801d1ff
+	github.com/jlaffaye/ftp v0.0.0-20220904184306-99be0634ab9a
 	github.com/pkg/xattr v0.4.7
 	golang.org/x/mobile v0.0.0-20220722155234-aaac322e2105
 	golang.org/x/term v0.0.0-20220722155259-a9ba230a4035

go.sum

@@ -380,6 +380,8 @@ github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0
 github.com/jlaffaye/ftp v0.0.0-20190624084859-c1312a7102bf/go.mod h1:lli8NYPQOFy3O++YmYbqVgOcQ1JPCwdOy+5zSjKJ9qY=
 github.com/jlaffaye/ftp v0.0.0-20220630165035-11536801d1ff h1:tN6UCYCBFNrPwvKf4RP9cIhGo6GcZ/IQTN8nqD7eCok=
 github.com/jlaffaye/ftp v0.0.0-20220630165035-11536801d1ff/go.mod h1:hhq4G4crv+nW2qXtNYcuzLeOudG92Ps37HEKeg2e3lE=
+github.com/jlaffaye/ftp v0.0.0-20220904184306-99be0634ab9a h1:s4ryRQyC5HKZh6qkjNAFcvmD7gImK5bZuj/YZkXy1vw=
+github.com/jlaffaye/ftp v0.0.0-20220904184306-99be0634ab9a/go.mod h1:hhq4G4crv+nW2qXtNYcuzLeOudG92Ps37HEKeg2e3lE=
 github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=

lib/terminal/terminal.go

@@ -68,17 +68,23 @@ const (
 var (
 	// make sure that start is only called once
 	once sync.Once
+
+	// RawOut is the underlying *os.File intended for terminal output
+	RawOut = os.Stderr
 )
 
 // Start the terminal - must be called before use
 func Start() {
 	once.Do(func() {
-		f := os.Stdout
+		f := RawOut
 		if !IsTerminal(int(f.Fd())) {
-			// If stdout not a tty then remove escape codes
+			// If output is not a tty then remove escape codes
 			Out = colorable.NewNonColorable(f)
 		} else if runtime.GOOS == "windows" && os.Getenv("TERM") != "" {
-			// If TERM is set just use stdout
+			// If TERM is set on Windows then we should just send output
+			// straight to the terminal for cygwin/git bash environments.
+			// We don't want to use NewColorable here because it will
+			// use Windows console calls which cygwin/git bash don't support.
 			Out = f
 		} else {
 			Out = colorable.NewColorable(f)

lib/terminal/terminal_normal.go

@@ -34,5 +34,5 @@ func ReadPassword(fd int) ([]byte, error) {
 
 // WriteTerminalTitle writes a string to the terminal title
 func WriteTerminalTitle(title string) {
-	fmt.Printf(ChangeTitle + title + BEL)
+	fmt.Fprintf(Out, ChangeTitle+title+BEL)
 }