dropbox: make across config server side Move/Copy/DirMove fallback to copy

If we are doing a cross remote transfer then attempting a
Move/Copy/DirMove where we don't have permission gives
`from_lookup/not_found` errors.

This patch notices that error and only if we are doing a cross remote
transfer it engages the fallback where the file is streamed.

See: https://forum.rclone.org/t/dropbox-io-error-movedir-failed-from-lookup-not-found-when-try-to-move-copy-works/34088

.github/workflows/build.yml

@@ -25,12 +25,12 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        job_name: ['linux', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.16', 'go1.17']
+        job_name: ['linux', 'linux_386', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.17', 'go1.18']
 
         include:
           - job_name: linux
             os: ubuntu-latest
-            go: '1.18.x'
+            go: '1.19'
             gotags: cmount
             build_flags: '-include "^linux/"'
             check: true
@@ -39,9 +39,16 @@ jobs:
             librclonetest: true
             deploy: true
 
+          - job_name: linux_386
+            os: ubuntu-latest
+            go: '1.19'
+            goarch: 386
+            gotags: cmount
+            quicktest: true
+
           - job_name: mac_amd64
             os: macos-11
-            go: '1.18.x'
+            go: '1.19'
             gotags: 'cmount'
             build_flags: '-include "^darwin/amd64" -cgo'
             quicktest: true
@@ -50,14 +57,14 @@ jobs:
 
           - job_name: mac_arm64
             os: macos-11
-            go: '1.18.x'
+            go: '1.19'
             gotags: 'cmount'
             build_flags: '-include "^darwin/arm64" -cgo -macos-arch arm64 -cgo-cflags=-I/usr/local/include -cgo-ldflags=-L/usr/local/lib'
             deploy: true
 
           - job_name: windows
             os: windows-latest
-            go: '1.18.x'
+            go: '1.19'
             gotags: cmount
             cgo: '0'
             build_flags: '-include "^windows/"'
@@ -67,20 +74,20 @@ jobs:
 
           - job_name: other_os
             os: ubuntu-latest
-            go: '1.18.x'
+            go: '1.19'
             build_flags: '-exclude "^(windows/|darwin/|linux/)"'
             compile_all: true
             deploy: true
 
-          - job_name: go1.16
+          - job_name: go1.17
             os: ubuntu-latest
-            go: '1.16.x'
+            go: '1.17'
             quicktest: true
             racequicktest: true
 
-          - job_name: go1.17
+          - job_name: go1.18
             os: ubuntu-latest
-            go: '1.17.x'
+            go: '1.18'
             quicktest: true
             racequicktest: true
 
@@ -90,14 +97,13 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 0
 
       - name: Install Go
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v3
         with:
-          stable: 'false'
           go-version: ${{ matrix.go }}
           check-latest: true
 
@@ -155,7 +161,7 @@ jobs:
           env
 
       - name: Go module cache
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: ~/go/pkg/mod
           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
@@ -219,14 +225,27 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Code quality test
-        uses: golangci/golangci-lint-action@v2
+        uses: golangci/golangci-lint-action@v3
         with:
           # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version
           version: latest
 
+      # Run govulncheck on the latest go version, the one we build binaries with
+      - name: Install Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.19
+          check-latest: true
+
+      - name: Install govulncheck
+        run: go install golang.org/x/vuln/cmd/govulncheck@latest
+
+      - name: Scan for vulnerabilities
+        run: govulncheck ./...
+
   android:
     if: ${{ github.repository == 'rclone/rclone' || github.event.inputs.manual }}
     timeout-minutes: 30
@@ -235,18 +254,18 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 0
 
       # Upgrade together with NDK version
       - name: Set up Go
-        uses: actions/setup-go@v1
+        uses: actions/setup-go@v3
         with:
-          go-version: 1.18.x
+          go-version: 1.19
 
       - name: Go module cache
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: ~/go/pkg/mod
           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}

.github/workflows/build_publish_docker_image.yml

@@ -12,7 +12,7 @@ jobs:
         name: Build image job
         steps:
             - name: Checkout master
-              uses: actions/checkout@v2
+              uses: actions/checkout@v3
               with:
                 fetch-depth: 0
             - name: Build and publish image

.github/workflows/build_publish_release_docker_image.yml

@@ -11,7 +11,7 @@ jobs:
         name: Build image job
         steps:
             - name: Checkout master
-              uses: actions/checkout@v2
+              uses: actions/checkout@v3
               with:
                 fetch-depth: 0
             - name: Get actual patch version
@@ -40,7 +40,7 @@ jobs:
         name: Build docker plugin job
         steps:
             - name: Checkout master
-              uses: actions/checkout@v2
+              uses: actions/checkout@v3
               with:
                 fetch-depth: 0
             - name: Build and publish docker plugin

.golangci.yml

@@ -4,8 +4,8 @@ linters:
   enable:
     - deadcode
     - errcheck
-    #- goimports
-    #- revive
+    - goimports
+    - revive
     - ineffassign
     - structcheck
     - varcheck
@@ -20,7 +20,7 @@ issues:
   exclude-use-default: false
 
   # Maximum issues count per one linter. Set to 0 to disable. Default is 50.
-  max-per-linter: 0
+  max-issues-per-linter: 0
 
   # Maximum count of issues with the same text. Set to 0 to disable. Default is 3.
   max-same-issues: 0

CONTRIBUTING.md

@@ -77,7 +77,7 @@ Make sure you
   * Add [documentation](#writing-documentation) for a new feature.
   * [Commit your changes](#committing-your-changes) using the [message guideline](#commit-messages).
 
-When you are done with that push your changes to Github:
+When you are done with that push your changes to GitHub:
 
     git push -u origin my-new-feature
 
@@ -88,7 +88,7 @@ Your changes will then get reviewed and you might get asked to fix some stuff. I
 
 You may sometimes be asked to [base your changes on the latest master](#basing-your-changes-on-the-latest-master) or [squash your commits](#squashing-your-commits).
 
-## Using Git and Github ##
+## Using Git and GitHub ##
 
 ### Committing your changes ###
 

README.md

@@ -45,10 +45,10 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
   * HiDrive [:page_facing_up:](https://rclone.org/hidrive/)
   * HTTP [:page_facing_up:](https://rclone.org/http/)
   * Huawei Cloud Object Storage Service(OBS) [:page_facing_up:](https://rclone.org/s3/#huawei-obs)
-  * Hubic [:page_facing_up:](https://rclone.org/hubic/)
   * Internet Archive [:page_facing_up:](https://rclone.org/internetarchive/)
   * Jottacloud [:page_facing_up:](https://rclone.org/jottacloud/)
   * IBM COS S3 [:page_facing_up:](https://rclone.org/s3/#ibm-cos-s3)
+  * IONOS Cloud [:page_facing_up:](https://rclone.org/s3/#ionos)
   * Koofr [:page_facing_up:](https://rclone.org/koofr/)
   * Mail.ru Cloud [:page_facing_up:](https://rclone.org/mailru/)
   * Memset Memstore [:page_facing_up:](https://rclone.org/swift/)
@@ -62,17 +62,20 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
   * OpenDrive [:page_facing_up:](https://rclone.org/opendrive/)
   * OpenStack Swift [:page_facing_up:](https://rclone.org/swift/)
   * Oracle Cloud Storage [:page_facing_up:](https://rclone.org/swift/)
+  * Oracle Object Storage [:page_facing_up:](https://rclone.org/oracleobjectstorage/)
   * ownCloud [:page_facing_up:](https://rclone.org/webdav/#owncloud)
   * pCloud [:page_facing_up:](https://rclone.org/pcloud/)
   * premiumize.me [:page_facing_up:](https://rclone.org/premiumizeme/)
   * put.io [:page_facing_up:](https://rclone.org/putio/)
   * QingStor [:page_facing_up:](https://rclone.org/qingstor/)
+  * Qiniu Cloud Object Storage (Kodo) [:page_facing_up:](https://rclone.org/s3/#qiniu)
   * Rackspace Cloud Files [:page_facing_up:](https://rclone.org/swift/)
   * RackCorp Object Storage [:page_facing_up:](https://rclone.org/s3/#RackCorp)
   * Scaleway [:page_facing_up:](https://rclone.org/s3/#scaleway)
   * Seafile [:page_facing_up:](https://rclone.org/seafile/)
   * SeaweedFS [:page_facing_up:](https://rclone.org/s3/#seaweedfs)
   * SFTP [:page_facing_up:](https://rclone.org/sftp/)
+  * SMB / CIFS [:page_facing_up:](https://rclone.org/smb/)
   * StackPath [:page_facing_up:](https://rclone.org/s3/#stackpath)
   * Storj [:page_facing_up:](https://rclone.org/storj/)
   * SugarSync [:page_facing_up:](https://rclone.org/sugarsync/)

RELEASE.md

@@ -53,6 +53,14 @@ doing that so it may be necessary to roll back dependencies to the
 version specified by `make updatedirect` in order to get rclone to
 build.
 
+## Tidy beta
+
+At some point after the release run
+
+    bin/tidy-beta v1.55
+
+where the version number is that of a couple ago to remove old beta binaries.
+
 ## Making a point release
 
 If rclone needs a point release due to some horrendous bug:

VERSION

@@ -1 +1 @@
-v1.59.1
+v1.61.0

backend/alias/alias.go

@@ -1,3 +1,4 @@
+// Package alias implements a virtual provider to rename existing remotes.
 package alias
 
 import (

backend/all/all.go

@@ -1,3 +1,4 @@
+// Package all imports all the backends
 package all
 
 import (
@@ -23,7 +24,6 @@ import (
 	_ "github.com/rclone/rclone/backend/hdfs"
 	_ "github.com/rclone/rclone/backend/hidrive"
 	_ "github.com/rclone/rclone/backend/http"
-	_ "github.com/rclone/rclone/backend/hubic"
 	_ "github.com/rclone/rclone/backend/internetarchive"
 	_ "github.com/rclone/rclone/backend/jottacloud"
 	_ "github.com/rclone/rclone/backend/koofr"
@@ -34,6 +34,7 @@ import (
 	_ "github.com/rclone/rclone/backend/netstorage"
 	_ "github.com/rclone/rclone/backend/onedrive"
 	_ "github.com/rclone/rclone/backend/opendrive"
+	_ "github.com/rclone/rclone/backend/oracleobjectstorage"
 	_ "github.com/rclone/rclone/backend/pcloud"
 	_ "github.com/rclone/rclone/backend/premiumizeme"
 	_ "github.com/rclone/rclone/backend/putio"
@@ -43,6 +44,7 @@ import (
 	_ "github.com/rclone/rclone/backend/sftp"
 	_ "github.com/rclone/rclone/backend/sharefile"
 	_ "github.com/rclone/rclone/backend/sia"
+	_ "github.com/rclone/rclone/backend/smb"
 	_ "github.com/rclone/rclone/backend/storj"
 	_ "github.com/rclone/rclone/backend/sugarsync"
 	_ "github.com/rclone/rclone/backend/swift"

backend/amazonclouddrive/amazonclouddrive.go

@@ -556,9 +556,9 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 //
 // This is a workaround for Amazon sometimes returning
 //
-//  * 408 REQUEST_TIMEOUT
-//  * 504 GATEWAY_TIMEOUT
-//  * 500 Internal server error
+//   - 408 REQUEST_TIMEOUT
+//   - 504 GATEWAY_TIMEOUT
+//   - 500 Internal server error
 //
 // At the end of large uploads.  The speculation is that the timeout
 // is waiting for the sha1 hashing to complete and the file may well
@@ -626,7 +626,7 @@ func (f *Fs) checkUpload(ctx context.Context, resp *http.Response, in io.Reader,
 
 // Put the object into the container
 //
-// Copy the reader in to the new object which is returned
+// Copy the reader in to the new object which is returned.
 //
 // The new object may have been created if an error is returned
 func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
@@ -685,9 +685,9 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
 
 // Move src to this remote using server-side move operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -1002,7 +1002,6 @@ func (o *Object) readMetaData(ctx context.Context) (err error) {
 
 // ModTime returns the modification time of the object
 //
-//
 // It attempts to read the objects mtime and if that isn't present the
 // LastModified returned in the http headers
 func (o *Object) ModTime(ctx context.Context) time.Time {

backend/azureblob/azureblob.go

@@ -1,8 +1,7 @@
-// Package azureblob provides an interface to the Microsoft Azure blob object storage system
-
 //go:build !plan9 && !solaris && !js
 // +build !plan9,!solaris,!js
 
+// Package azureblob provides an interface to the Microsoft Azure blob object storage system
 package azureblob
 
 import (
@@ -13,9 +12,9 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"net/url"
+	"os"
 	"path"
 	"strconv"
 	"strings"
@@ -596,7 +595,15 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	)
 	switch {
 	case opt.UseEmulator:
-		credential, err := azblob.NewSharedKeyCredential(emulatorAccount, emulatorAccountKey)
+		var actualEmulatorAccount = emulatorAccount
+		if opt.Account != "" {
+			actualEmulatorAccount = opt.Account
+		}
+		var actualEmulatorKey = emulatorAccountKey
+		if opt.Key != "" {
+			actualEmulatorKey = opt.Key
+		}
+		credential, err := azblob.NewSharedKeyCredential(actualEmulatorAccount, actualEmulatorKey)
 		if err != nil {
 			return nil, fmt.Errorf("failed to parse credentials: %w", err)
 		}
@@ -718,7 +725,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 			return nil, fmt.Errorf("failed to make azure storage url from account and endpoint: %w", err)
 		}
 		// Try loading service principal credentials from file.
-		loadedCreds, err := ioutil.ReadFile(env.ShellExpand(opt.ServicePrincipalFile))
+		loadedCreds, err := os.ReadFile(env.ShellExpand(opt.ServicePrincipalFile))
 		if err != nil {
 			return nil, fmt.Errorf("error opening service principal credentials file: %w", err)
 		}
@@ -1115,7 +1122,7 @@ func (f *Fs) listContainersToFn(fn listContainerFn) error {
 
 // Put the object into the container
 //
-// Copy the reader in to the new object which is returned
+// Copy the reader in to the new object which is returned.
 //
 // The new object may have been created if an error is returned
 func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
@@ -1247,9 +1254,9 @@ func (f *Fs) Purge(ctx context.Context, dir string) error {
 
 // Copy src to this remote using server-side copy operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -1356,11 +1363,12 @@ func (o *Object) setMetadata(metadata azblob.Metadata) {
 // decodeMetaDataFromPropertiesResponse sets the metadata from the data passed in
 //
 // Sets
-//  o.id
-//  o.modTime
-//  o.size
-//  o.md5
-//  o.meta
+//
+//	o.id
+//	o.modTime
+//	o.size
+//	o.md5
+//	o.meta
 func (o *Object) decodeMetaDataFromPropertiesResponse(info *azblob.BlobGetPropertiesResponse) (err error) {
 	metadata := info.NewMetadata()
 	size := info.ContentLength()
@@ -1443,10 +1451,11 @@ func (o *Object) clearMetaData() {
 // readMetaData gets the metadata if it hasn't already been fetched
 //
 // Sets
-//  o.id
-//  o.modTime
-//  o.size
-//  o.md5
+//
+//	o.id
+//	o.modTime
+//	o.size
+//	o.md5
 func (o *Object) readMetaData() (err error) {
 	container, _ := o.split()
 	if !o.fs.containerOK(container) {
@@ -1676,14 +1685,34 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		}
 	}
 
-	uploadParts := int64(maxUploadParts)
+	// Apply upload options (also allows one to overwrite content-type)
+	for _, option := range options {
+		key, value := option.Header()
+		lowerKey := strings.ToLower(key)
+		switch lowerKey {
+		case "":
+			// ignore
+		case "cache-control":
+			httpHeaders.CacheControl = value
+		case "content-disposition":
+			httpHeaders.ContentDisposition = value
+		case "content-encoding":
+			httpHeaders.ContentEncoding = value
+		case "content-language":
+			httpHeaders.ContentLanguage = value
+		case "content-type":
+			httpHeaders.ContentType = value
+		}
+	}
+
+	uploadParts := maxUploadParts
 	if uploadParts < 1 {
 		uploadParts = 1
 	} else if uploadParts > maxUploadParts {
 		uploadParts = maxUploadParts
 	}
 	// calculate size of parts/blocks
-	partSize := chunksize.Calculator(o, int(uploadParts), o.fs.opt.ChunkSize)
+	partSize := chunksize.Calculator(o, src.Size(), uploadParts, o.fs.opt.ChunkSize)
 
 	putBlobOptions := azblob.UploadStreamToBlockBlobOptions{
 		BufferSize:      int(partSize),

backend/azureblob/imds.go

@@ -9,7 +9,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 
 	"github.com/Azure/go-autorest/autorest/adal"
@@ -97,7 +96,7 @@ func GetMSIToken(ctx context.Context, identity *userMSI) (adal.Token, error) {
 		return result, fmt.Errorf("MSI is not enabled on this VM: %w", err)
 	}
 	defer func() { // resp and Body should not be nil
-		_, err = io.Copy(ioutil.Discard, resp.Body)
+		_, err = io.Copy(io.Discard, resp.Body)
 		if err != nil {
 			fs.Debugf(nil, "Unable to drain IMDS response: %v", err)
 		}
@@ -112,12 +111,12 @@ func GetMSIToken(ctx context.Context, identity *userMSI) (adal.Token, error) {
 	case 200, 201, 202:
 		break
 	default:
-		body, _ := ioutil.ReadAll(resp.Body)
+		body, _ := io.ReadAll(resp.Body)
 		fs.Errorf(nil, "Couldn't obtain OAuth token from IMDS; server returned status code %d and body: %v", resp.StatusCode, string(body))
 		return result, httpError{Response: resp}
 	}
 
-	b, err := ioutil.ReadAll(resp.Body)
+	b, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return result, fmt.Errorf("couldn't read IMDS response: %w", err)
 	}

backend/b2/api/types.go

@@ -1,3 +1,4 @@
+// Package api provides types used by the Backblaze B2 API.
 package api
 
 import (
@@ -238,7 +239,7 @@ type GetFileInfoRequest struct {
 // If the original source of the file being uploaded has a last
 // modified time concept, Backblaze recommends using
 // src_last_modified_millis as the name, and a string holding the base
-// 10 number number of milliseconds since midnight, January 1, 1970
+// 10 number of milliseconds since midnight, January 1, 1970
 // UTC. This fits in a 64 bit integer such as the type "long" in the
 // programming language Java. It is intended to be compatible with
 // Java's time long. For example, it can be passed directly into the

backend/b2/b2.go

@@ -1,4 +1,4 @@
-// Package b2 provides an interface to the Backblaze B2 object storage system
+// Package b2 provides an interface to the Backblaze B2 object storage system.
 package b2
 
 // FIXME should we remove sha1 checks from here as rclone now supports
@@ -656,15 +656,15 @@ var errEndList = errors.New("end list")
 //
 // (bucket, directory) is the starting directory
 //
-// If prefix is set then it is removed from all file names
+// If prefix is set then it is removed from all file names.
 //
 // If addBucket is set then it adds the bucket to the start of the
-// remotes generated
+// remotes generated.
 //
-// If recurse is set the function will recursively list
+// If recurse is set the function will recursively list.
 //
 // If limit is > 0 then it limits to that many files (must be less
-// than 1000)
+// than 1000).
 //
 // If hidden is set then it will list the hidden (deleted) files too.
 //
@@ -1025,7 +1025,7 @@ func (f *Fs) clearBucketID(bucket string) {
 
 // Put the object into the bucket
 //
-// Copy the reader in to the new object which is returned
+// Copy the reader in to the new object which is returned.
 //
 // The new object may have been created if an error is returned
 func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
@@ -1334,9 +1334,9 @@ func (f *Fs) copy(ctx context.Context, dstObj *Object, srcObj *Object, newInfo *
 
 // Copy src to this remote using server-side copy operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -1478,7 +1478,7 @@ func (o *Object) Size() int64 {
 
 // Clean the SHA1
 //
-// Make sure it is lower case
+// Make sure it is lower case.
 //
 // Remove unverified prefix - see https://www.backblaze.com/b2/docs/uploading.html
 // Some tools (e.g. Cyberduck) use this
@@ -1490,10 +1490,11 @@ func cleanSHA1(sha1 string) string {
 // decodeMetaDataRaw sets the metadata from the data passed in
 //
 // Sets
-//  o.id
-//  o.modTime
-//  o.size
-//  o.sha1
+//
+//	o.id
+//	o.modTime
+//	o.size
+//	o.sha1
 func (o *Object) decodeMetaDataRaw(ID, SHA1 string, Size int64, UploadTimestamp api.Timestamp, Info map[string]string, mimeType string) (err error) {
 	o.id = ID
 	o.sha1 = SHA1
@@ -1512,10 +1513,11 @@ func (o *Object) decodeMetaDataRaw(ID, SHA1 string, Size int64, UploadTimestamp
 // decodeMetaData sets the metadata in the object from an api.File
 //
 // Sets
-//  o.id
-//  o.modTime
-//  o.size
-//  o.sha1
+//
+//	o.id
+//	o.modTime
+//	o.size
+//	o.sha1
 func (o *Object) decodeMetaData(info *api.File) (err error) {
 	return o.decodeMetaDataRaw(info.ID, info.SHA1, info.Size, info.UploadTimestamp, info.Info, info.ContentType)
 }
@@ -1523,10 +1525,11 @@ func (o *Object) decodeMetaData(info *api.File) (err error) {
 // decodeMetaDataFileInfo sets the metadata in the object from an api.FileInfo
 //
 // Sets
-//  o.id
-//  o.modTime
-//  o.size
-//  o.sha1
+//
+//	o.id
+//	o.modTime
+//	o.size
+//	o.sha1
 func (o *Object) decodeMetaDataFileInfo(info *api.FileInfo) (err error) {
 	return o.decodeMetaDataRaw(info.ID, info.SHA1, info.Size, info.UploadTimestamp, info.Info, info.ContentType)
 }
@@ -1584,10 +1587,11 @@ func (o *Object) getMetaData(ctx context.Context) (info *api.File, err error) {
 // readMetaData gets the metadata if it hasn't already been fetched
 //
 // Sets
-//  o.id
-//  o.modTime
-//  o.size
-//  o.sha1
+//
+//	o.id
+//	o.modTime
+//	o.size
+//	o.sha1
 func (o *Object) readMetaData(ctx context.Context) (err error) {
 	if o.id != "" {
 		return nil

backend/b2/upload.go

@@ -97,7 +97,7 @@ func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs
 	if size == -1 {
 		fs.Debugf(o, "Streaming upload with --b2-chunk-size %s allows uploads of up to %s and will fail only when that limit is reached.", f.opt.ChunkSize, maxParts*f.opt.ChunkSize)
 	} else {
-		chunkSize = chunksize.Calculator(src, maxParts, defaultChunkSize)
+		chunkSize = chunksize.Calculator(o, size, maxParts, defaultChunkSize)
 		parts = size / int64(chunkSize)
 		if size%int64(chunkSize) != 0 {
 			parts++

backend/box/api/types.go

@@ -14,7 +14,7 @@ const (
 	timeFormat = `"` + time.RFC3339 + `"`
 )
 
-// Time represents represents date and time information for the
+// Time represents date and time information for the
 // box API, by using RFC3339
 type Time time.Time
 

backend/box/box.go

@@ -17,9 +17,9 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"net/url"
+	"os"
 	"path"
 	"strconv"
 	"strings"
@@ -183,7 +183,7 @@ func refreshJWTToken(ctx context.Context, jsonFile string, boxSubType string, na
 }
 
 func getBoxConfig(configFile string) (boxConfig *api.ConfigJSON, err error) {
-	file, err := ioutil.ReadFile(configFile)
+	file, err := os.ReadFile(configFile)
 	if err != nil {
 		return nil, fmt.Errorf("box: failed to read Box config: %w", err)
 	}
@@ -266,7 +266,7 @@ type Fs struct {
 	root         string                // the path we are working on
 	opt          Options               // parsed options
 	features     *fs.Features          // optional features
-	srv          *rest.Client          // the connection to the one drive server
+	srv          *rest.Client          // the connection to the server
 	dirCache     *dircache.DirCache    // Map of directory path to directory id
 	pacer        *fs.Pacer             // pacer for API calls
 	tokenRenewer *oauthutil.Renew      // renew the token on expiry
@@ -692,7 +692,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 // Creates from the parameters passed in a half finished Object which
 // must have setMetaData called on it
 //
-// Returns the object, leaf, directoryID and error
+// Returns the object, leaf, directoryID and error.
 //
 // Used to create new objects
 func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, size int64) (o *Object, leaf string, directoryID string, err error) {
@@ -752,7 +752,7 @@ func (f *Fs) preUploadCheck(ctx context.Context, leaf, directoryID string, size
 
 // Put the object
 //
-// Copy the reader in to the new object which is returned
+// Copy the reader in to the new object which is returned.
 //
 // The new object may have been created if an error is returned
 func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
@@ -792,9 +792,9 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt
 
 // PutUnchecked the object into the container
 //
-// This will produce an error if the object already exists
+// This will produce an error if the object already exists.
 //
-// Copy the reader in to the new object which is returned
+// Copy the reader in to the new object which is returned.
 //
 // The new object may have been created if an error is returned
 func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
@@ -877,9 +877,9 @@ func (f *Fs) Precision() time.Duration {
 
 // Copy src to this remote using server-side copy operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -995,9 +995,9 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
 
 // Move src to this remote using server-side move operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -1235,7 +1235,6 @@ func (o *Object) readMetaData(ctx context.Context) (err error) {
 
 // ModTime returns the modification time of the object
 //
-//
 // It attempts to read the objects mtime and if that isn't present the
 // LastModified returned in the http headers
 func (o *Object) ModTime(ctx context.Context) time.Time {
@@ -1346,9 +1345,9 @@ func (o *Object) upload(ctx context.Context, in io.Reader, leaf, directoryID str
 
 // Update the object with the contents of the io.Reader, modTime and size
 //
-// If existing is set then it updates the object rather than creating a new one
+// If existing is set then it updates the object rather than creating a new one.
 //
-// The new object may have been created if an error is returned
+// The new object may have been created if an error is returned.
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
 	if o.fs.tokenRenewer != nil {
 		o.fs.tokenRenewer.Start()

backend/cache/cache.go

@@ -1,6 +1,7 @@
 //go:build !plan9 && !js
 // +build !plan9,!js
 
+// Package cache implements a virtual provider to cache existing remotes.
 package cache
 
 import (

backend/cache/cache_internal_test.go

@@ -11,7 +11,6 @@ import (
 	goflag "flag"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"log"
 	"math/rand"
 	"os"
@@ -167,7 +166,7 @@ func TestInternalVfsCache(t *testing.T) {
 			li2 := [2]string{path.Join("test", "one"), path.Join("test", "second")}
 			for _, r := range li2 {
 				var err error
-				ci, err := ioutil.ReadDir(path.Join(runInstance.chunkPath, runInstance.encryptRemoteIfNeeded(t, path.Join(id, r))))
+				ci, err := os.ReadDir(path.Join(runInstance.chunkPath, runInstance.encryptRemoteIfNeeded(t, path.Join(id, r))))
 				if err != nil || len(ci) == 0 {
 					log.Printf("========== '%v' not in cache", r)
 				} else {
@@ -841,7 +840,7 @@ func newRun() *run {
 	}
 
 	if uploadDir == "" {
-		r.tmpUploadDir, err = ioutil.TempDir("", "rclonecache-tmp")
+		r.tmpUploadDir, err = os.MkdirTemp("", "rclonecache-tmp")
 		if err != nil {
 			panic(fmt.Sprintf("Failed to create temp dir: %v", err))
 		}
@@ -984,7 +983,7 @@ func (r *run) randomReader(t *testing.T, size int64) io.ReadCloser {
 	chunk := int64(1024)
 	cnt := size / chunk
 	left := size % chunk
-	f, err := ioutil.TempFile("", "rclonecache-tempfile")
+	f, err := os.CreateTemp("", "rclonecache-tempfile")
 	require.NoError(t, err)
 
 	for i := 0; i < int(cnt); i++ {

backend/cache/plex.go

@@ -8,7 +8,7 @@ import (
 	"crypto/tls"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"net/url"
 	"strings"
@@ -167,7 +167,7 @@ func (p *plexConnector) listenWebsocket() {
 								continue
 							}
 							var data []byte
-							data, err = ioutil.ReadAll(resp.Body)
+							data, err = io.ReadAll(resp.Body)
 							if err != nil {
 								continue
 							}

backend/cache/storage_persistent.go

@@ -9,7 +9,6 @@ import (
 	"encoding/binary"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path"
 	"strconv"
@@ -473,7 +472,7 @@ func (b *Persistent) GetChunk(cachedObject *Object, offset int64) ([]byte, error
 	var data []byte
 
 	fp := path.Join(b.dataPath, cachedObject.abs(), strconv.FormatInt(offset, 10))
-	data, err := ioutil.ReadFile(fp)
+	data, err := os.ReadFile(fp)
 	if err != nil {
 		return nil, err
 	}
@@ -486,7 +485,7 @@ func (b *Persistent) AddChunk(fp string, data []byte, offset int64) error {
 	_ = os.MkdirAll(path.Join(b.dataPath, fp), os.ModePerm)
 
 	filePath := path.Join(b.dataPath, fp, strconv.FormatInt(offset, 10))
-	err := ioutil.WriteFile(filePath, data, os.ModePerm)
+	err := os.WriteFile(filePath, data, os.ModePerm)
 	if err != nil {
 		return err
 	}

backend/chunker/chunker.go

@@ -12,7 +12,6 @@ import (
 	"fmt"
 	gohash "hash"
 	"io"
-	"io/ioutil"
 	"math/rand"
 	"path"
 	"regexp"
@@ -32,7 +31,6 @@ import (
 	"github.com/rclone/rclone/fs/operations"
 )
 
-//
 // Chunker's composite files have one or more chunks
 // and optional metadata object. If it's present,
 // meta object is named after the original file.
@@ -65,7 +63,7 @@ import (
 // length of 13 decimals it makes a 7-digit base-36 number.
 //
 // When transactions is set to the norename style, data chunks will
-// keep their temporary chunk names (with the transacion identifier
+// keep their temporary chunk names (with the transaction identifier
 // suffix). To distinguish them from temporary chunks, the txn field
 // of the metadata file is set to match the transaction identifier of
 // the data chunks.
@@ -79,7 +77,6 @@ import (
 // Metadata format v1 does not define any control chunk types,
 // they are currently ignored aka reserved.
 // In future they can be used to implement resumable uploads etc.
-//
 const (
 	ctrlTypeRegStr   = `[a-z][a-z0-9]{2,6}`
 	tempSuffixFormat = `_%04s`
@@ -542,7 +539,6 @@ func (f *Fs) setChunkNameFormat(pattern string) error {
 //
 // xactID is a transaction identifier. Empty xactID denotes active chunk,
 // otherwise temporary chunk name is produced.
-//
 func (f *Fs) makeChunkName(filePath string, chunkNo int, ctrlType, xactID string) string {
 	dir, parentName := path.Split(filePath)
 	var name, tempSuffix string
@@ -708,7 +704,6 @@ func (f *Fs) newXactID(ctx context.Context, filePath string) (xactID string, err
 // directory together with dead chunks.
 // In future a flag named like `--chunker-list-hidden` may be added to
 // rclone that will tell List to reveal hidden chunks.
-//
 func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
 	entries, err = f.base.List(ctx, dir)
 	if err != nil {
@@ -868,7 +863,6 @@ func (f *Fs) processEntries(ctx context.Context, origEntries fs.DirEntries, dirP
 // Note that chunker prefers analyzing file names rather than reading
 // the content of meta object assuming that directory scans are fast
 // but opening even a small file can be slow on some backends.
-//
 func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 	return f.scanObject(ctx, remote, false)
 }
@@ -1043,7 +1037,7 @@ func (o *Object) readMetadata(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
-	metadata, err := ioutil.ReadAll(reader)
+	metadata, err := io.ReadAll(reader)
 	_ = reader.Close() // ensure file handle is freed on windows
 	if err != nil {
 		return err
@@ -1084,7 +1078,7 @@ func (o *Object) readMetadata(ctx context.Context) error {
 
 // readXactID returns the transaction ID stored in the passed metadata object
 func (o *Object) readXactID(ctx context.Context) (xactID string, err error) {
-	// if xactID has already been read and cahced return it now
+	// if xactID has already been read and cached return it now
 	if o.xIDCached {
 		return o.xactID, nil
 	}
@@ -1102,7 +1096,7 @@ func (o *Object) readXactID(ctx context.Context) (xactID string, err error) {
 	if err != nil {
 		return "", err
 	}
-	data, err := ioutil.ReadAll(reader)
+	data, err := io.ReadAll(reader)
 	_ = reader.Close() // ensure file handle is freed on windows
 	if err != nil {
 		return "", err
@@ -1586,7 +1580,6 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
 // This command will chain to `purge` from wrapped remote.
 // As a result it removes not only composite chunker files with their
 // active chunks but also all hidden temporary chunks in the directory.
-//
 func (f *Fs) Purge(ctx context.Context, dir string) error {
 	do := f.base.Features().Purge
 	if do == nil {
@@ -1628,7 +1621,6 @@ func (f *Fs) Purge(ctx context.Context, dir string) error {
 // Unsupported control chunks will get re-picked by a more recent
 // rclone version with unexpected results. This can be helped by
 // the `delete hidden` flag above or at least the user has been warned.
-//
 func (o *Object) Remove(ctx context.Context) (err error) {
 	if err := o.f.forbidChunk(o, o.Remote()); err != nil {
 		// operations.Move can still call Remove if chunker's Move refuses
@@ -1804,9 +1796,9 @@ func (f *Fs) okForServerSide(ctx context.Context, src fs.Object, opName string)
 
 // Copy src to this remote using server-side copy operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -1825,9 +1817,9 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 
 // Move src to this remote using server-side move operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -2125,7 +2117,6 @@ func (o *Object) SetModTime(ctx context.Context, mtime time.Time) error {
 // file, then tries to read it from metadata. This in theory
 // handles the unusual case when a small file has been tampered
 // on the level of wrapped remote but chunker is unaware of that.
-//
 func (o *Object) Hash(ctx context.Context, hashType hash.Type) (string, error) {
 	if err := o.readMetadata(ctx); err != nil {
 		return "", err // valid metadata is required to get hash, abort
@@ -2414,7 +2405,6 @@ type metaSimpleJSON struct {
 // - for files larger than chunk size
 // - if file contents can be mistaken as meta object
 // - if consistent hashing is On but wrapped remote can't provide given hash
-//
 func marshalSimpleJSON(ctx context.Context, size int64, nChunks int, md5, sha1, xactID string) ([]byte, error) {
 	version := metadataVersion
 	if xactID == "" && version == 2 {
@@ -2447,7 +2437,6 @@ func marshalSimpleJSON(ctx context.Context, size int64, nChunks int, md5, sha1,
 // New format will have a higher version number and cannot be correctly
 // handled by current implementation.
 // The version check below will then explicitly ask user to upgrade rclone.
-//
 func unmarshalSimpleJSON(ctx context.Context, metaObject fs.Object, data []byte) (info *ObjectInfo, madeByChunker bool, err error) {
 	// Be strict about JSON format
 	// to reduce possibility that a random small file resembles metadata.

backend/chunker/chunker_internal_test.go

@@ -5,7 +5,7 @@ import (
 	"context"
 	"flag"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"path"
 	"regexp"
 	"strings"
@@ -413,7 +413,7 @@ func testSmallFileInternals(t *testing.T, f *Fs) {
 		if r == nil {
 			return
 		}
-		data, err := ioutil.ReadAll(r)
+		data, err := io.ReadAll(r)
 		assert.NoError(t, err)
 		assert.Equal(t, contents, string(data))
 		_ = r.Close()
@@ -538,7 +538,7 @@ func testPreventCorruption(t *testing.T, f *Fs) {
 	assert.NoError(t, err)
 	var chunkContents []byte
 	assert.NotPanics(t, func() {
-		chunkContents, err = ioutil.ReadAll(r)
+		chunkContents, err = io.ReadAll(r)
 		_ = r.Close()
 	})
 	assert.NoError(t, err)
@@ -573,7 +573,7 @@ func testPreventCorruption(t *testing.T, f *Fs) {
 	r, err = willyChunk.Open(ctx)
 	assert.NoError(t, err)
 	assert.NotPanics(t, func() {
-		_, err = ioutil.ReadAll(r)
+		_, err = io.ReadAll(r)
 		_ = r.Close()
 	})
 	assert.NoError(t, err)
@@ -672,7 +672,7 @@ func testMetadataInput(t *testing.T, f *Fs) {
 		assert.NoError(t, err, "open "+description)
 		assert.NotNil(t, r, "open stream of "+description)
 		if err == nil && r != nil {
-			data, err := ioutil.ReadAll(r)
+			data, err := io.ReadAll(r)
 			assert.NoError(t, err, "read all of "+description)
 			assert.Equal(t, contents, string(data), description+" contents is ok")
 			_ = r.Close()
@@ -758,8 +758,8 @@ func testFutureProof(t *testing.T, f *Fs) {
 	assert.Error(t, err)
 
 	// Rcat must fail
-	in := ioutil.NopCloser(bytes.NewBufferString("abc"))
-	robj, err := operations.Rcat(ctx, f, file, in, modTime)
+	in := io.NopCloser(bytes.NewBufferString("abc"))
+	robj, err := operations.Rcat(ctx, f, file, in, modTime, nil)
 	assert.Nil(t, robj)
 	assert.NotNil(t, err)
 	if err != nil {
@@ -854,7 +854,7 @@ func testChunkerServerSideMove(t *testing.T, f *Fs) {
 	r, err := dstFile.Open(ctx)
 	assert.NoError(t, err)
 	assert.NotNil(t, r)
-	data, err := ioutil.ReadAll(r)
+	data, err := io.ReadAll(r)
 	assert.NoError(t, err)
 	assert.Equal(t, contents, string(data))
 	_ = r.Close()

backend/combine/combine.go

@@ -1,4 +1,4 @@
-// Package combine implents a backend to combine multipe remotes in a directory tree
+// Package combine implents a backend to combine multiple remotes in a directory tree
 package combine
 
 /*
@@ -457,9 +457,9 @@ func (f *Fs) Purge(ctx context.Context, dir string) error {
 
 // Copy src to this remote using server-side copy operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -491,9 +491,9 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 
 // Move src to this remote using server-side move operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //

backend/compress/compress.go

@@ -13,7 +13,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"regexp"
 	"strings"
@@ -29,6 +28,7 @@ import (
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/fspath"
 	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/fs/log"
 	"github.com/rclone/rclone/fs/object"
 	"github.com/rclone/rclone/fs/operations"
 )
@@ -90,7 +90,7 @@ Generally -1 (default, equivalent to 5) is recommended.
 Levels 1 to 9 increase compression at the cost of speed. Going past 6 
 generally offers very little return.
 
-Level -2 uses Huffmann encoding only. Only use if you know what you
+Level -2 uses Huffman encoding only. Only use if you know what you
 are doing.
 Level 0 turns off compression.`,
 			Default:  sgzip.DefaultCompression,
@@ -130,7 +130,7 @@ type Fs struct {
 	features *fs.Features // optional features
 }
 
-// NewFs contstructs an Fs from the path, container:path
+// NewFs constructs an Fs from the path, container:path
 func NewFs(ctx context.Context, name, rpath string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
 	opt := new(Options)
@@ -367,13 +367,16 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 	if err != nil {
 		return nil, err
 	}
-	meta := readMetadata(ctx, mo)
-	if meta == nil {
-		return nil, errors.New("error decoding metadata")
+	meta, err := readMetadata(ctx, mo)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding metadata: %w", err)
 	}
 	// Create our Object
 	o, err := f.Fs.NewObject(ctx, makeDataName(remote, meta.CompressionMetadata.Size, meta.Mode))
-	return f.newObject(o, mo, meta), err
+	if err != nil {
+		return nil, err
+	}
+	return f.newObject(o, mo, meta), nil
 }
 
 // checkCompressAndType checks if an object is compressible and determines it's mime type
@@ -451,7 +454,7 @@ func (f *Fs) rcat(ctx context.Context, dstFileName string, in io.ReadCloser, mod
 		return f.Fs.Put(ctx, bytes.NewBuffer(buf[:n]), src, options...)
 	}
 
-	// Need to include what we allready read
+	// Need to include what we already read
 	in = &ReadCloserWrapper{
 		Reader: io.MultiReader(bytes.NewReader(buf), in),
 		Closer: in,
@@ -464,7 +467,7 @@ func (f *Fs) rcat(ctx context.Context, dstFileName string, in io.ReadCloser, mod
 	}
 
 	fs.Debugf(f, "Target remote doesn't support streaming uploads, creating temporary local file")
-	tempFile, err := ioutil.TempFile("", "rclone-press-")
+	tempFile, err := os.CreateTemp("", "rclone-press-")
 	defer func() {
 		// these errors should be relatively uncritical and the upload should've succeeded so it's okay-ish
 		// to ignore them
@@ -542,8 +545,8 @@ func (f *Fs) putCompress(ctx context.Context, in io.Reader, src fs.ObjectInfo, o
 	}
 
 	// Transfer the data
-	o, err := f.rcat(ctx, makeDataName(src.Remote(), src.Size(), f.mode), ioutil.NopCloser(wrappedIn), src.ModTime(ctx), options)
-	//o, err := operations.Rcat(ctx, f.Fs, makeDataName(src.Remote(), src.Size(), f.mode), ioutil.NopCloser(wrappedIn), src.ModTime(ctx))
+	o, err := f.rcat(ctx, makeDataName(src.Remote(), src.Size(), f.mode), io.NopCloser(wrappedIn), src.ModTime(ctx), options)
+	//o, err := operations.Rcat(ctx, f.Fs, makeDataName(src.Remote(), src.Size(), f.mode), io.NopCloser(wrappedIn), src.ModTime(ctx))
 	if err != nil {
 		if o != nil {
 			removeErr := o.Remove(ctx)
@@ -677,7 +680,7 @@ func (f *Fs) putWithCustomFunctions(ctx context.Context, in io.Reader, src fs.Ob
 		}
 		return nil, err
 	}
-	return f.newObject(dataObject, mo, meta), err
+	return f.newObject(dataObject, mo, meta), nil
 }
 
 // Put in to the remote path with the modTime given of the given size
@@ -731,7 +734,7 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt
 	}
 
 	// If our new object is compressed we have to rename it with the correct size.
-	// Uncompressed objects don't store the size in the name so we they'll allready have the correct name.
+	// Uncompressed objects don't store the size in the name so we they'll already have the correct name.
 	if compressible {
 		wrapObj, err := operations.Move(ctx, f.Fs, nil, f.dataName(src.Remote(), newObj.size, compressible), newObj.Object)
 		if err != nil {
@@ -742,7 +745,7 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt
 	return newObj, nil
 }
 
-// Temporarely disabled. There might be a way to implement this correctly but with the current handling metadata duplicate objects
+// Temporarily disabled. There might be a way to implement this correctly but with the current handling metadata duplicate objects
 // will break stuff. Right no I can't think of a way to make this work.
 
 // PutUnchecked uploads the object
@@ -785,9 +788,9 @@ func (f *Fs) Purge(ctx context.Context, dir string) error {
 
 // Copy src to this remote using server side copy operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -835,9 +838,9 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 
 // Move src to this remote using server side move operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -1040,24 +1043,19 @@ func newMetadata(size int64, mode int, cmeta sgzip.GzipMetadata, md5 string, mim
 }
 
 // This function will read the metadata from a metadata object.
-func readMetadata(ctx context.Context, mo fs.Object) (meta *ObjectMetadata) {
+func readMetadata(ctx context.Context, mo fs.Object) (meta *ObjectMetadata, err error) {
 	// Open our meradata object
 	rc, err := mo.Open(ctx)
 	if err != nil {
-		return nil
+		return nil, err
 	}
-	defer func() {
-		err := rc.Close()
-		if err != nil {
-			fs.Errorf(mo, "Error closing object: %v", err)
-		}
-	}()
+	defer fs.CheckClose(rc, &err)
 	jr := json.NewDecoder(rc)
 	meta = new(ObjectMetadata)
 	if err = jr.Decode(meta); err != nil {
-		return nil
+		return nil, err
 	}
-	return meta
+	return meta, nil
 }
 
 // Remove removes this object
@@ -1102,6 +1100,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	origName := o.Remote()
 	if o.meta.Mode != Uncompressed || compressible {
 		newObject, err = o.f.putWithCustomFunctions(ctx, in, o.f.wrapInfo(src, origName, src.Size()), options, o.f.Fs.Put, updateMeta, compressible, mimeType)
+		if err != nil {
+			return err
+		}
 		if newObject.Object.Remote() != o.Object.Remote() {
 			if removeErr := o.Object.Remove(ctx); removeErr != nil {
 				return removeErr
@@ -1115,9 +1116,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		}
 		// If we are, just update the object and metadata
 		newObject, err = o.f.putWithCustomFunctions(ctx, in, src, options, update, updateMeta, compressible, mimeType)
-	}
-	if err != nil {
-		return err
+		if err != nil {
+			return err
+		}
 	}
 	// Update object metadata and return
 	o.Object = newObject.Object
@@ -1128,6 +1129,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 // This will initialize the variables of a new press Object. The metadata object, mo, and metadata struct, meta, must be specified.
 func (f *Fs) newObject(o fs.Object, mo fs.Object, meta *ObjectMetadata) *Object {
+	if o == nil {
+		log.Trace(nil, "newObject(%#v, %#v, %#v) called with nil o", o, mo, meta)
+	}
 	return &Object{
 		Object: o,
 		f:      f,
@@ -1140,6 +1144,9 @@ func (f *Fs) newObject(o fs.Object, mo fs.Object, meta *ObjectMetadata) *Object
 
 // This initializes the variables of a press Object with only the size. The metadata will be loaded later on demand.
 func (f *Fs) newObjectSizeAndNameOnly(o fs.Object, moName string, size int64) *Object {
+	if o == nil {
+		log.Trace(nil, "newObjectSizeAndNameOnly(%#v, %#v, %#v) called with nil o", o, moName, size)
+	}
 	return &Object{
 		Object: o,
 		f:      f,
@@ -1167,7 +1174,7 @@ func (o *Object) loadMetadataIfNotLoaded(ctx context.Context) (err error) {
 		return err
 	}
 	if o.meta == nil {
-		o.meta = readMetadata(ctx, o.mo)
+		o.meta, err = readMetadata(ctx, o.mo)
 	}
 	return err
 }

backend/crypt/cipher.go

@@ -127,11 +127,11 @@ type fileNameEncoding interface {
 // RFC4648
 //
 // The standard encoding is modified in two ways
-//  * it becomes lower case (no-one likes upper case filenames!)
-//  * we strip the padding character `=`
+//   - it becomes lower case (no-one likes upper case filenames!)
+//   - we strip the padding character `=`
 type caseInsensitiveBase32Encoding struct{}
 
-// EncodeToString encodes a strign using the modified version of
+// EncodeToString encodes a string using the modified version of
 // base32 encoding.
 func (caseInsensitiveBase32Encoding) EncodeToString(src []byte) string {
 	encoded := base32.HexEncoding.EncodeToString(src)
@@ -244,7 +244,7 @@ func (c *Cipher) putBlock(buf []byte) {
 
 // encryptSegment encrypts a path segment
 //
-// This uses EME with AES
+// This uses EME with AES.
 //
 // EME (ECB-Mix-ECB) is a wide-block encryption mode presented in the
 // 2003 paper "A Parallelizable Enciphering Mode" by Halevi and
@@ -254,8 +254,8 @@ func (c *Cipher) putBlock(buf []byte) {
 // same filename must encrypt to the same thing.
 //
 // This means that
-//  * filenames with the same name will encrypt the same
-//  * filenames which start the same won't have a common prefix
+//   - filenames with the same name will encrypt the same
+//   - filenames which start the same won't have a common prefix
 func (c *Cipher) encryptSegment(plaintext string) string {
 	if plaintext == "" {
 		return ""
@@ -1085,7 +1085,7 @@ func (c *Cipher) DecryptData(rc io.ReadCloser) (io.ReadCloser, error) {
 
 // DecryptDataSeek decrypts the data stream from offset
 //
-// The open function must return a ReadCloser opened to the offset supplied
+// The open function must return a ReadCloser opened to the offset supplied.
 //
 // You must use this form of DecryptData if you might want to Seek the file handle
 func (c *Cipher) DecryptDataSeek(ctx context.Context, open OpenRangeSeek, offset, limit int64) (ReadSeekCloser, error) {

backend/crypt/cipher_test.go

@@ -8,7 +8,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"strings"
 	"testing"
 
@@ -1073,7 +1072,7 @@ func testEncryptDecrypt(t *testing.T, bufSize int, copySize int64) {
 	source := newRandomSource(copySize)
 	encrypted, err := c.newEncrypter(source, nil)
 	assert.NoError(t, err)
-	decrypted, err := c.newDecrypter(ioutil.NopCloser(encrypted))
+	decrypted, err := c.newDecrypter(io.NopCloser(encrypted))
 	assert.NoError(t, err)
 	sink := newRandomSource(copySize)
 	n, err := io.CopyBuffer(sink, decrypted, buf)
@@ -1144,15 +1143,15 @@ func TestEncryptData(t *testing.T) {
 		buf := bytes.NewBuffer(test.in)
 		encrypted, err := c.EncryptData(buf)
 		assert.NoError(t, err)
-		out, err := ioutil.ReadAll(encrypted)
+		out, err := io.ReadAll(encrypted)
 		assert.NoError(t, err)
 		assert.Equal(t, test.expected, out)
 
 		// Check we can decode the data properly too...
 		buf = bytes.NewBuffer(out)
-		decrypted, err := c.DecryptData(ioutil.NopCloser(buf))
+		decrypted, err := c.DecryptData(io.NopCloser(buf))
 		assert.NoError(t, err)
-		out, err = ioutil.ReadAll(decrypted)
+		out, err = io.ReadAll(decrypted)
 		assert.NoError(t, err)
 		assert.Equal(t, test.in, out)
 	}
@@ -1187,7 +1186,7 @@ func TestNewEncrypterErrUnexpectedEOF(t *testing.T) {
 	fh, err := c.newEncrypter(in, nil)
 	assert.NoError(t, err)
 
-	n, err := io.CopyN(ioutil.Discard, fh, 1e6)
+	n, err := io.CopyN(io.Discard, fh, 1e6)
 	assert.Equal(t, io.ErrUnexpectedEOF, err)
 	assert.Equal(t, int64(32), n)
 }
@@ -1257,12 +1256,12 @@ func TestNewDecrypterErrUnexpectedEOF(t *testing.T) {
 
 	in2 := &readers.ErrorReader{Err: io.ErrUnexpectedEOF}
 	in1 := bytes.NewBuffer(file16)
-	in := ioutil.NopCloser(io.MultiReader(in1, in2))
+	in := io.NopCloser(io.MultiReader(in1, in2))
 
 	fh, err := c.newDecrypter(in)
 	assert.NoError(t, err)
 
-	n, err := io.CopyN(ioutil.Discard, fh, 1e6)
+	n, err := io.CopyN(io.Discard, fh, 1e6)
 	assert.Equal(t, io.ErrUnexpectedEOF, err)
 	assert.Equal(t, int64(16), n)
 }
@@ -1274,14 +1273,14 @@ func TestNewDecrypterSeekLimit(t *testing.T) {
 
 	// Make random data
 	const dataSize = 150000
-	plaintext, err := ioutil.ReadAll(newRandomSource(dataSize))
+	plaintext, err := io.ReadAll(newRandomSource(dataSize))
 	assert.NoError(t, err)
 
 	// Encrypt the data
 	buf := bytes.NewBuffer(plaintext)
 	encrypted, err := c.EncryptData(buf)
 	assert.NoError(t, err)
-	ciphertext, err := ioutil.ReadAll(encrypted)
+	ciphertext, err := io.ReadAll(encrypted)
 	assert.NoError(t, err)
 
 	trials := []int{0, 1, 2, 3, 4, 5, 7, 8, 9, 15, 16, 17, 31, 32, 33, 63, 64, 65,
@@ -1300,7 +1299,7 @@ func TestNewDecrypterSeekLimit(t *testing.T) {
 				end = len(ciphertext)
 			}
 		}
-		reader = ioutil.NopCloser(bytes.NewBuffer(ciphertext[int(underlyingOffset):end]))
+		reader = io.NopCloser(bytes.NewBuffer(ciphertext[int(underlyingOffset):end]))
 		return reader, nil
 	}
 
@@ -1490,7 +1489,7 @@ func TestDecrypterRead(t *testing.T) {
 			assert.NoError(t, err, what)
 			continue
 		}
-		_, err = ioutil.ReadAll(fh)
+		_, err = io.ReadAll(fh)
 		var expectedErr error
 		switch {
 		case i == fileHeaderSize:
@@ -1514,7 +1513,7 @@ func TestDecrypterRead(t *testing.T) {
 	cd := newCloseDetector(in)
 	fh, err := c.newDecrypter(cd)
 	assert.NoError(t, err)
-	_, err = ioutil.ReadAll(fh)
+	_, err = io.ReadAll(fh)
 	assert.Error(t, err, "potato")
 	assert.Equal(t, 0, cd.closed)
 
@@ -1524,13 +1523,13 @@ func TestDecrypterRead(t *testing.T) {
 	copy(file16copy, file16)
 	for i := range file16copy {
 		file16copy[i] ^= 0xFF
-		fh, err := c.newDecrypter(ioutil.NopCloser(bytes.NewBuffer(file16copy)))
+		fh, err := c.newDecrypter(io.NopCloser(bytes.NewBuffer(file16copy)))
 		if i < fileMagicSize {
 			assert.Error(t, err, ErrorEncryptedBadMagic.Error())
 			assert.Nil(t, fh)
 		} else {
 			assert.NoError(t, err)
-			_, err = ioutil.ReadAll(fh)
+			_, err = io.ReadAll(fh)
 			assert.Error(t, err, ErrorEncryptedFileBadHeader.Error())
 		}
 		file16copy[i] ^= 0xFF
@@ -1565,7 +1564,7 @@ func TestDecrypterClose(t *testing.T) {
 	assert.Equal(t, 0, cd.closed)
 
 	// close after reading
-	out, err := ioutil.ReadAll(fh)
+	out, err := io.ReadAll(fh)
 	assert.NoError(t, err)
 	assert.Equal(t, []byte{1}, out)
 	assert.Equal(t, io.EOF, fh.err)

backend/crypt/crypt.go

@@ -125,7 +125,7 @@ names, or for debugging purposes.`,
 
 This option could help with shortening the encrypted filename. The 
 suitable option would depend on the way your remote count the filename
-length and if it's case sensitve.`,
+length and if it's case sensitive.`,
 			Default: "base32",
 			Examples: []fs.OptionExample{
 				{
@@ -507,9 +507,9 @@ func (f *Fs) Purge(ctx context.Context, dir string) error {
 
 // Copy src to this remote using server-side copy operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -532,9 +532,9 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 
 // Move src to this remote using server-side move operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //

backend/drive/drive.go

@@ -14,9 +14,9 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"mime"
 	"net/http"
+	"os"
 	"path"
 	"regexp"
 	"sort"
@@ -758,6 +758,9 @@ func (f *Fs) shouldRetry(ctx context.Context, err error) (bool, error) {
 			} else if f.opt.StopOnDownloadLimit && reason == "downloadQuotaExceeded" {
 				fs.Errorf(f, "Received download limit error: %v", err)
 				return false, fserrors.FatalError(err)
+			} else if f.opt.StopOnUploadLimit && reason == "quotaExceeded" {
+				fs.Errorf(f, "Received upload limit error: %v", err)
+				return false, fserrors.FatalError(err)
 			} else if f.opt.StopOnUploadLimit && reason == "teamDriveFileLimitExceeded" {
 				fs.Errorf(f, "Received Shared Drive file limit error: %v", err)
 				return false, fserrors.FatalError(err)
@@ -1105,7 +1108,7 @@ func createOAuthClient(ctx context.Context, opt *Options, name string, m configm
 
 	// try loading service account credentials from env variable, then from a file
 	if len(opt.ServiceAccountCredentials) == 0 && opt.ServiceAccountFile != "" {
-		loadedCreds, err := ioutil.ReadFile(env.ShellExpand(opt.ServiceAccountFile))
+		loadedCreds, err := os.ReadFile(env.ShellExpand(opt.ServiceAccountFile))
 		if err != nil {
 			return nil, fmt.Errorf("error opening service account credentials file: %w", err)
 		}
@@ -1207,6 +1210,7 @@ func newFs(ctx context.Context, name, path string, m configmap.Mapper) (*Fs, err
 		WriteMimeType:           true,
 		CanHaveEmptyDirectories: true,
 		ServerSideAcrossConfigs: opt.ServerSideAcrossConfigs,
+		FilterAware:             true,
 	}).Fill(ctx, f)
 
 	// Create a new authorized Drive client.
@@ -2177,7 +2181,7 @@ func (f *Fs) createFileInfo(ctx context.Context, remote string, modTime time.Tim
 
 // Put the object
 //
-// Copy the reader in to the new object which is returned
+// Copy the reader in to the new object which is returned.
 //
 // The new object may have been created if an error is returned
 func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
@@ -2411,9 +2415,9 @@ func (f *Fs) Precision() time.Duration {
 
 // Copy src to this remote using server-side copy operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -2646,9 +2650,9 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
 
 // Move src to this remote using server-side move operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -3302,7 +3306,7 @@ drives found and a combined drive.
     upstreams = "My Drive=My Drive:" "Test Drive=Test Drive:"
 
 Adding this to the rclone config file will cause those team drives to
-be accessible with the aliases shown. Any illegal charactes will be
+be accessible with the aliases shown. Any illegal characters will be
 substituted with "_" and duplicate names will have numbers suffixed.
 It will also add a remote called AllDrives which shows all the shared
 drives combined into one directory tree.
@@ -3566,7 +3570,6 @@ func (f *Fs) getRemoteInfoWithExport(ctx context.Context, remote string) (
 
 // ModTime returns the modification time of the object
 //
-//
 // It attempts to read the objects mtime and if that isn't present the
 // LastModified returned in the http headers
 func (o *baseObject) ModTime(ctx context.Context) time.Time {
@@ -3797,7 +3800,7 @@ func (o *linkObject) Open(ctx context.Context, options ...fs.OpenOption) (in io.
 		data = data[:limit]
 	}
 
-	return ioutil.NopCloser(bytes.NewReader(data)), nil
+	return io.NopCloser(bytes.NewReader(data)), nil
 }
 
 func (o *baseObject) update(ctx context.Context, updateInfo *drive.File, uploadMimeType string, in io.Reader,
@@ -3823,7 +3826,7 @@ func (o *baseObject) update(ctx context.Context, updateInfo *drive.File, uploadM
 
 // Update the already existing object
 //
-// Copy the reader into the object updating modTime and size
+// Copy the reader into the object updating modTime and size.
 //
 // The new object may have been created if an error is returned
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {

backend/drive/drive_internal_test.go

@@ -7,7 +7,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"mime"
 	"os"
 	"path"
@@ -19,6 +18,7 @@ import (
 	_ "github.com/rclone/rclone/backend/local"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/filter"
+	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/fs/operations"
 	"github.com/rclone/rclone/fs/sync"
@@ -28,6 +28,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"google.golang.org/api/drive/v3"
+	"google.golang.org/api/googleapi"
 )
 
 func TestDriveScopes(t *testing.T) {
@@ -76,7 +77,7 @@ var additionalMimeTypes = map[string]string{
 // Load the example export formats into exportFormats for testing
 func TestInternalLoadExampleFormats(t *testing.T) {
 	fetchFormatsOnce.Do(func() {})
-	buf, err := ioutil.ReadFile(filepath.FromSlash("test/about.json"))
+	buf, err := os.ReadFile(filepath.FromSlash("test/about.json"))
 	var about struct {
 		ExportFormats map[string][]string `json:"exportFormats,omitempty"`
 		ImportFormats map[string][]string `json:"importFormats,omitempty"`
@@ -190,6 +191,60 @@ func TestExtensionsForImportFormats(t *testing.T) {
 	}
 }
 
+func (f *Fs) InternalTestShouldRetry(t *testing.T) {
+	ctx := context.Background()
+	gatewayTimeout := googleapi.Error{
+		Code: 503,
+	}
+	timeoutRetry, timeoutError := f.shouldRetry(ctx, &gatewayTimeout)
+	assert.True(t, timeoutRetry)
+	assert.Equal(t, &gatewayTimeout, timeoutError)
+	generic403 := googleapi.Error{
+		Code: 403,
+	}
+	rLEItem := googleapi.ErrorItem{
+		Reason:  "rateLimitExceeded",
+		Message: "User rate limit exceeded.",
+	}
+	generic403.Errors = append(generic403.Errors, rLEItem)
+	oldStopUpload := f.opt.StopOnUploadLimit
+	oldStopDownload := f.opt.StopOnDownloadLimit
+	f.opt.StopOnUploadLimit = true
+	f.opt.StopOnDownloadLimit = true
+	defer func() {
+		f.opt.StopOnUploadLimit = oldStopUpload
+		f.opt.StopOnDownloadLimit = oldStopDownload
+	}()
+	expectedRLError := fserrors.FatalError(&generic403)
+	rateLimitRetry, rateLimitErr := f.shouldRetry(ctx, &generic403)
+	assert.False(t, rateLimitRetry)
+	assert.Equal(t, rateLimitErr, expectedRLError)
+	dQEItem := googleapi.ErrorItem{
+		Reason: "downloadQuotaExceeded",
+	}
+	generic403.Errors[0] = dQEItem
+	expectedDQError := fserrors.FatalError(&generic403)
+	downloadQuotaRetry, downloadQuotaError := f.shouldRetry(ctx, &generic403)
+	assert.False(t, downloadQuotaRetry)
+	assert.Equal(t, downloadQuotaError, expectedDQError)
+	tDFLEItem := googleapi.ErrorItem{
+		Reason: "teamDriveFileLimitExceeded",
+	}
+	generic403.Errors[0] = tDFLEItem
+	expectedTDFLError := fserrors.FatalError(&generic403)
+	teamDriveFileLimitRetry, teamDriveFileLimitError := f.shouldRetry(ctx, &generic403)
+	assert.False(t, teamDriveFileLimitRetry)
+	assert.Equal(t, teamDriveFileLimitError, expectedTDFLError)
+	qEItem := googleapi.ErrorItem{
+		Reason: "quotaExceeded",
+	}
+	generic403.Errors[0] = qEItem
+	expectedQuotaError := fserrors.FatalError(&generic403)
+	quotaExceededRetry, quotaExceededError := f.shouldRetry(ctx, &generic403)
+	assert.False(t, quotaExceededRetry)
+	assert.Equal(t, quotaExceededError, expectedQuotaError)
+}
+
 func (f *Fs) InternalTestDocumentImport(t *testing.T) {
 	oldAllow := f.opt.AllowImportNameChange
 	f.opt.AllowImportNameChange = true
@@ -462,6 +517,9 @@ func (f *Fs) InternalTestCopyID(t *testing.T) {
 
 // TestIntegration/FsMkdir/FsPutFiles/Internal/AgeQuery
 func (f *Fs) InternalTestAgeQuery(t *testing.T) {
+	// Check set up for filtering
+	assert.True(t, f.Features().FilterAware)
+
 	opt := &filter.Opt{}
 	err := opt.MaxAge.Set("1h")
 	assert.NoError(t, err)
@@ -545,6 +603,7 @@ func (f *Fs) InternalTest(t *testing.T) {
 	t.Run("UnTrash", f.InternalTestUnTrash)
 	t.Run("CopyID", f.InternalTestCopyID)
 	t.Run("AgeQuery", f.InternalTestAgeQuery)
+	t.Run("ShouldRetry", f.InternalTestShouldRetry)
 }
 
 var _ fstests.InternalTester = (*Fs)(nil)

backend/dropbox/batcher.go

@@ -309,7 +309,7 @@ func (b *batcher) Shutdown() {
 	}
 	b.shutOnce.Do(func() {
 		atexit.Unregister(b.atexit)
-		fs.Infof(b.f, "Commiting uploads - please wait...")
+		fs.Infof(b.f, "Committing uploads - please wait...")
 		// show that batcher is shutting down
 		close(b.closed)
 		// quit the commitLoop by sending a quitRequest message

backend/dropbox/dropbox.go

@@ -268,7 +268,7 @@ default based on the batch_mode in use.
 			Advanced: true,
 		}, {
 			Name:     "batch_commit_timeout",
-			Help:     `Max time to wait for a batch to finish comitting`,
+			Help:     `Max time to wait for a batch to finish committing`,
 			Default:  fs.Duration(10 * time.Minute),
 			Advanced: true,
 		}, {
@@ -925,7 +925,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 
 // Put the object
 //
-// Copy the reader in to the new object which is returned
+// Copy the reader in to the new object which is returned.
 //
 // The new object may have been created if an error is returned
 func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
@@ -1044,9 +1044,9 @@ func (f *Fs) Precision() time.Duration {
 
 // Copy src to this remote using server-side copy operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -1078,6 +1078,15 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 		return shouldRetry(ctx, err)
 	})
 	if err != nil {
+		switch e := err.(type) {
+		case files.CopyV2APIError:
+			// If we are doing a cross remote transfer then from_lookup/not_found indicates we
+			// don't have permission to read the source, so engage the slow path
+			if srcObj.fs != f && e.EndpointError != nil && e.EndpointError.FromLookup != nil && e.EndpointError.FromLookup.Tag == files.LookupErrorNotFound {
+				fs.Debugf(srcObj, "Copy failed attempting fallback: %v", err)
+				return nil, fs.ErrorCantCopy
+			}
+		}
 		return nil, fmt.Errorf("copy failed: %w", err)
 	}
 
@@ -1105,9 +1114,9 @@ func (f *Fs) Purge(ctx context.Context, dir string) (err error) {
 
 // Move src to this remote using server-side move operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -1139,6 +1148,15 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 		return shouldRetry(ctx, err)
 	})
 	if err != nil {
+		switch e := err.(type) {
+		case files.MoveV2APIError:
+			// If we are doing a cross remote transfer then from_lookup/not_found indicates we
+			// don't have permission to read the source, so engage the slow path
+			if srcObj.fs != f && e.EndpointError != nil && e.EndpointError.FromLookup != nil && e.EndpointError.FromLookup.Tag == files.LookupErrorNotFound {
+				fs.Debugf(srcObj, "Move failed attempting fallback: %v", err)
+				return nil, fs.ErrorCantMove
+			}
+		}
 		return nil, fmt.Errorf("move failed: %w", err)
 	}
 
@@ -1257,6 +1275,15 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 		return shouldRetry(ctx, err)
 	})
 	if err != nil {
+		switch e := err.(type) {
+		case files.MoveV2APIError:
+			// If we are doing a cross remote transfer then from_lookup/not_found indicates we
+			// don't have permission to read the source, so engage the slow path
+			if srcFs != f && e.EndpointError != nil && e.EndpointError.FromLookup != nil && e.EndpointError.FromLookup.Tag == files.LookupErrorNotFound {
+				fs.Debugf(srcFs, "DirMove failed attempting fallback: %v", err)
+				return fs.ErrorCantDirMove
+			}
+		}
 		return fmt.Errorf("MoveDir failed: %w", err)
 	}
 
@@ -1669,7 +1696,7 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
 						correctOffset := uErr.EndpointError.IncorrectOffset.CorrectOffset
 						delta := int64(correctOffset) - int64(cursor.Offset)
 						skip += delta
-						what := fmt.Sprintf("incorrect offset error receved: sent %d, need %d, skip %d", cursor.Offset, correctOffset, skip)
+						what := fmt.Sprintf("incorrect offset error received: sent %d, need %d, skip %d", cursor.Offset, correctOffset, skip)
 						if skip < 0 {
 							return false, fmt.Errorf("can't seek backwards to correct offset: %s", what)
 						} else if skip == chunkSize {
@@ -1763,7 +1790,7 @@ func checkPathLength(name string) (err error) {
 
 // Update the already existing object
 //
-// Copy the reader into the object updating modTime and size
+// Copy the reader into the object updating modTime and size.
 //
 // The new object may have been created if an error is returned
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {

backend/fichier/fichier.go

@@ -1,3 +1,4 @@
+// Package fichier provides an interface to the 1Fichier storage system.
 package fichier
 
 import (

backend/fichier/structs.go

@@ -84,7 +84,7 @@ type CopyFileResponse struct {
 	URLs    []FileCopy `json:"urls"`
 }
 
-// FileCopy is used in the the CopyFileResponse
+// FileCopy is used in the CopyFileResponse
 type FileCopy struct {
 	FromURL string `json:"from_url"`
 	ToURL   string `json:"to_url"`

backend/filefabric/api/types.go

@@ -19,7 +19,7 @@ const (
 	timeFormatJSON = `"` + timeFormatParameters + `"`
 )
 
-// Time represents represents date and time information for the
+// Time represents date and time information for the
 // filefabric API
 type Time time.Time
 
@@ -95,7 +95,7 @@ type Status struct {
 	// Warning string `json:"warning"` // obsolete
 }
 
-// Status statisfies the error interface
+// Status satisfies the error interface
 func (e *Status) Error() string {
 	return fmt.Sprintf("%s (%s)", e.Message, e.Code)
 }

backend/filefabric/filefabric.go

@@ -20,7 +20,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"net/url"
 	"path"
@@ -150,7 +149,7 @@ type Fs struct {
 	opt             Options            // parsed options
 	features        *fs.Features       // optional features
 	m               configmap.Mapper   // to save config
-	srv             *rest.Client       // the connection to the one drive server
+	srv             *rest.Client       // the connection to the server
 	dirCache        *dircache.DirCache // Map of directory path to directory id
 	pacer           *fs.Pacer          // pacer for API calls
 	tokenMu         sync.Mutex         // hold when reading the token
@@ -373,7 +372,7 @@ type params map[string]interface{}
 
 // rpc calls the rpc.php method of the SME file fabric
 //
-// This is an entry point to all the method calls
+// This is an entry point to all the method calls.
 //
 // If result is nil then resp.Body will need closing
 func (f *Fs) rpc(ctx context.Context, function string, p params, result api.OKError, options []fs.OpenOption) (resp *http.Response, err error) {
@@ -678,7 +677,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 // Creates from the parameters passed in a half finished Object which
 // must have setMetaData called on it
 //
-// Returns the object, leaf, directoryID and error
+// Returns the object, leaf, directoryID and error.
 //
 // Used to create new objects
 func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, size int64) (o *Object, leaf string, directoryID string, err error) {
@@ -697,7 +696,7 @@ func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time,
 
 // Put the object
 //
-// Copy the reader in to the new object which is returned
+// Copy the reader in to the new object which is returned.
 //
 // The new object may have been created if an error is returned
 func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
@@ -783,9 +782,9 @@ func (f *Fs) Precision() time.Duration {
 
 // Copy src to this remote using server side copy operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -843,7 +842,7 @@ func (f *Fs) Purge(ctx context.Context, dir string) error {
 	return f.purgeCheck(ctx, dir, false)
 }
 
-// Wait for the the background task to complete if necessary
+// Wait for the background task to complete if necessary
 func (f *Fs) waitForBackgroundTask(ctx context.Context, taskID api.String) (err error) {
 	if taskID == "" || taskID == "0" {
 		// No task to wait for
@@ -956,9 +955,9 @@ func (f *Fs) move(ctx context.Context, isDir bool, id, oldLeaf, newLeaf, oldDire
 
 // Move src to this remote using server side move operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -1135,7 +1134,6 @@ func (o *Object) readMetaData(ctx context.Context) (err error) {
 
 // ModTime returns the modification time of the object
 //
-//
 // It attempts to read the objects mtime and if that isn't present the
 // LastModified returned in the http headers
 func (o *Object) ModTime(ctx context.Context) time.Time {
@@ -1187,7 +1185,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 		return nil, errors.New("can't download - no id")
 	}
 	if o.contentType == emptyMimeType {
-		return ioutil.NopCloser(bytes.NewReader([]byte{})), nil
+		return io.NopCloser(bytes.NewReader([]byte{})), nil
 	}
 	fs.FixRangeOption(options, o.size)
 	resp, err := o.fs.rpc(ctx, "getFile", params{
@@ -1201,7 +1199,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 
 // Update the object with the contents of the io.Reader, modTime and size
 //
-// If existing is set then it updates the object rather than creating a new one
+// If existing is set then it updates the object rather than creating a new one.
 //
 // The new object may have been created if an error is returned
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {

backend/ftp/ftp.go

@@ -15,7 +15,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/jlaffaye/ftp"
+	"github.com/rclone/ftp"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
 	"github.com/rclone/rclone/fs/config"
@@ -81,8 +81,22 @@ security from the server in order to upgrade a plain text connection
 to an encrypted one. Cannot be used in combination with implicit FTP.`,
 			Default: false,
 		}, {
-			Name:     "concurrency",
-			Help:     "Maximum number of FTP simultaneous connections, 0 for unlimited.",
+			Name: "concurrency",
+			Help: strings.Replace(`Maximum number of FTP simultaneous connections, 0 for unlimited.
+
+Note that setting this is very likely to cause deadlocks so it should
+be used with care.
+
+If you are doing a sync or copy then make sure concurrency is one more
+than the sum of |--transfers| and |--checkers|.
+
+If you use |--check-first| then it just needs to be one more than the
+maximum of |--checkers| and |--transfers|.
+
+So for |concurrency 3| you'd use |--checkers 2 --transfers 2
+--check-first| or |--checkers 1 --transfers 1|.
+
+`, "|", "`", -1),
 			Default:  0,
 			Advanced: true,
 		}, {
@@ -110,6 +124,11 @@ to an encrypted one. Cannot be used in combination with implicit FTP.`,
 			Help:     "Use MDTM to set modification time (VsFtpd quirk)",
 			Default:  false,
 			Advanced: true,
+		}, {
+			Name:     "force_list_hidden",
+			Help:     "Use LIST -a to force listing of hidden files and folders. This will disable the use of MLSD.",
+			Default:  false,
+			Advanced: true,
 		}, {
 			Name:    "idle_timeout",
 			Default: fs.Duration(60 * time.Second),
@@ -191,6 +210,7 @@ type Options struct {
 	DisableMLSD       bool                 `config:"disable_mlsd"`
 	DisableUTF8       bool                 `config:"disable_utf8"`
 	WritingMDTM       bool                 `config:"writing_mdtm"`
+	ForceListHidden   bool                 `config:"force_list_hidden"`
 	IdleTimeout       fs.Duration          `config:"idle_timeout"`
 	CloseTimeout      fs.Duration          `config:"close_timeout"`
 	ShutTimeout       fs.Duration          `config:"shut_timeout"`
@@ -316,14 +336,44 @@ func (f *Fs) ftpConnection(ctx context.Context) (c *ftp.ServerConn, err error) {
 	fs.Debugf(f, "Connecting to FTP server")
 
 	// Make ftp library dial with fshttp dialer optionally using TLS
+	initialConnection := true
 	dial := func(network, address string) (conn net.Conn, err error) {
+		fs.Debugf(f, "dial(%q,%q)", network, address)
+		defer func() {
+			fs.Debugf(f, "> dial: conn=%T, err=%v", conn, err)
+		}()
 		conn, err = fshttp.NewDialer(ctx).Dial(network, address)
-		if f.tlsConf != nil && err == nil {
-			conn = tls.Client(conn, f.tlsConf)
+		if err != nil {
+			return nil, err
 		}
-		return
+		// Connect using cleartext only for non TLS
+		if f.tlsConf == nil {
+			return conn, nil
+		}
+		// Initial connection only needs to be cleartext for explicit TLS
+		if f.opt.ExplicitTLS && initialConnection {
+			initialConnection = false
+			return conn, nil
+		}
+		// Upgrade connection to TLS
+		tlsConn := tls.Client(conn, f.tlsConf)
+		// Do the initial handshake - tls.Client doesn't do it for us
+		// If we do this then connections to proftpd/pureftpd lock up
+		// See: https://github.com/rclone/rclone/issues/6426
+		// See: https://github.com/jlaffaye/ftp/issues/282
+		if false {
+			err = tlsConn.HandshakeContext(ctx)
+			if err != nil {
+				_ = conn.Close()
+				return nil, err
+			}
+		}
+		return tlsConn, nil
+	}
+	ftpConfig := []ftp.DialOption{
+		ftp.DialWithContext(ctx),
+		ftp.DialWithDialFunc(dial),
 	}
-	ftpConfig := []ftp.DialOption{ftp.DialWithDialFunc(dial)}
 
 	if f.opt.TLS {
 		// Our dialer takes care of TLS but ftp library also needs tlsConf
@@ -331,12 +381,6 @@ func (f *Fs) ftpConnection(ctx context.Context) (c *ftp.ServerConn, err error) {
 		ftpConfig = append(ftpConfig, ftp.DialWithTLS(f.tlsConf))
 	} else if f.opt.ExplicitTLS {
 		ftpConfig = append(ftpConfig, ftp.DialWithExplicitTLS(f.tlsConf))
-		// Initial connection needs to be cleartext for explicit TLS
-		conn, err := fshttp.NewDialer(ctx).Dial("tcp", f.dialAddr)
-		if err != nil {
-			return nil, err
-		}
-		ftpConfig = append(ftpConfig, ftp.DialWithNetConn(conn))
 	}
 	if f.opt.DisableEPSV {
 		ftpConfig = append(ftpConfig, ftp.DialWithDisabledEPSV(true))
@@ -353,6 +397,9 @@ func (f *Fs) ftpConnection(ctx context.Context) (c *ftp.ServerConn, err error) {
 	if f.opt.WritingMDTM {
 		ftpConfig = append(ftpConfig, ftp.DialWithWritingMDTM(true))
 	}
+	if f.opt.ForceListHidden {
+		ftpConfig = append(ftpConfig, ftp.DialWithForceListHidden(true))
+	}
 	if f.ci.Dump&(fs.DumpHeaders|fs.DumpBodies|fs.DumpRequests|fs.DumpResponses) != 0 {
 		ftpConfig = append(ftpConfig, ftp.DialWithDebugOutput(&debugLog{auth: f.ci.Dump&fs.DumpAuth != 0}))
 	}
@@ -769,11 +816,12 @@ func (f *Fs) Hashes() hash.Set {
 
 // Precision shows whether modified time is supported or not depending on the
 // FTP server capabilities, namely whether FTP server:
-// - accepts the MDTM command to get file time (fGetTime)
-//   or supports MLSD returning precise file time in the list (fLstTime)
-// - accepts the MFMT command to set file time (fSetTime)
-//   or non-standard form of the MDTM command (fSetTime, too)
-//   used by VsFtpd for the same purpose (WritingMDTM)
+//   - accepts the MDTM command to get file time (fGetTime)
+//     or supports MLSD returning precise file time in the list (fLstTime)
+//   - accepts the MFMT command to set file time (fSetTime)
+//     or non-standard form of the MDTM command (fSetTime, too)
+//     used by VsFtpd for the same purpose (WritingMDTM)
+//
 // See "mdtm_write" in https://security.appspot.com/vsftpd/vsftpd_conf.html
 func (f *Fs) Precision() time.Duration {
 	if (f.fGetTime || f.fLstTime) && f.fSetTime {
@@ -1149,7 +1197,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.Read
 
 // Update the already existing object
 //
-// Copy the reader into the object updating modTime and size
+// Copy the reader into the object updating modTime and size.
 //
 // The new object may have been created if an error is returned
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {

backend/ftp/ftp_internal_test.go

@@ -34,9 +34,9 @@ func deriveFs(ctx context.Context, t *testing.T, f fs.Fs, opts settings) fs.Fs {
 // test that big file uploads do not cause network i/o timeout
 func (f *Fs) testUploadTimeout(t *testing.T) {
 	const (
-		fileSize    = 100000000             // 100 MiB
-		idleTimeout = 40 * time.Millisecond // small because test server is local
-		maxTime     = 10 * time.Second      // prevent test hangup
+		fileSize    = 100000000        // 100 MiB
+		idleTimeout = 1 * time.Second  // small because test server is local
+		maxTime     = 10 * time.Second // prevent test hangup
 	)
 
 	if testing.Short() {

backend/googlecloudstorage/googlecloudstorage.go

@@ -19,8 +19,8 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
+	"os"
 	"path"
 	"strconv"
 	"strings"
@@ -311,7 +311,7 @@ rclone does if you know the bucket exists already.
 			Help: `If set this will decompress gzip encoded objects.
 
 It is possible to upload objects to GCS with "Content-Encoding: gzip"
-set. Normally rclone will download these files files as compressed objects.
+set. Normally rclone will download these files as compressed objects.
 
 If this flag is set then rclone will decompress these files with
 "Content-Encoding: gzip" as they are received. This means that rclone
@@ -319,6 +319,10 @@ can't check the size and hash but the file contents will be decompressed.
 `,
 			Advanced: true,
 			Default:  false,
+		}, {
+			Name:     "endpoint",
+			Help:     "Endpoint for the service.\n\nLeave blank normally.",
+			Advanced: true,
 		}, {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
@@ -343,6 +347,7 @@ type Options struct {
 	StorageClass              string               `config:"storage_class"`
 	NoCheckBucket             bool                 `config:"no_check_bucket"`
 	Decompress                bool                 `config:"decompress"`
+	Endpoint                  string               `config:"endpoint"`
 	Enc                       encoder.MultiEncoder `config:"encoding"`
 }
 
@@ -482,7 +487,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 
 	// try loading service account credentials from env variable, then from a file
 	if opt.ServiceAccountCredentials == "" && opt.ServiceAccountFile != "" {
-		loadedCreds, err := ioutil.ReadFile(env.ShellExpand(opt.ServiceAccountFile))
+		loadedCreds, err := os.ReadFile(env.ShellExpand(opt.ServiceAccountFile))
 		if err != nil {
 			return nil, fmt.Errorf("error opening service account credentials file: %w", err)
 		}
@@ -523,7 +528,11 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 
 	// Create a new authorized Drive client.
 	f.client = oAuthClient
-	f.svc, err = storage.NewService(context.Background(), option.WithHTTPClient(f.client))
+	gcsOpts := []option.ClientOption{option.WithHTTPClient(f.client)}
+	if opt.Endpoint != "" {
+		gcsOpts = append(gcsOpts, option.WithEndpoint(opt.Endpoint))
+	}
+	f.svc, err = storage.NewService(context.Background(), gcsOpts...)
 	if err != nil {
 		return nil, fmt.Errorf("couldn't create Google Cloud Storage client: %w", err)
 	}
@@ -580,7 +589,7 @@ type listFn func(remote string, object *storage.Object, isDirectory bool) error
 //
 // dir is the starting directory, "" for root
 //
-// Set recurse to read sub directories
+// Set recurse to read sub directories.
 //
 // The remote has prefix removed from it and if addBucket is set
 // then it adds the bucket to the start.
@@ -798,7 +807,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
 
 // Put the object into the bucket
 //
-// Copy the reader in to the new object which is returned
+// Copy the reader in to the new object which is returned.
 //
 // The new object may have been created if an error is returned
 func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
@@ -900,9 +909,9 @@ func (f *Fs) Precision() time.Duration {
 
 // Copy src to this remote using server-side copy operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //

backend/googlephotos/api/types.go

@@ -1,3 +1,4 @@
+// Package api provides types used by the Google Photos API.
 package api
 
 import (

backend/googlephotos/googlephotos.go

@@ -178,7 +178,7 @@ type Fs struct {
 	opt        Options                // parsed options
 	features   *fs.Features           // optional features
 	unAuth     *rest.Client           // unauthenticated http client
-	srv        *rest.Client           // the connection to the one drive server
+	srv        *rest.Client           // the connection to the server
 	ts         *oauthutil.TokenSource // token source for oauth2
 	pacer      *fs.Pacer              // To pace the API calls
 	startTime  time.Time              // time Fs was started - used for datestamps
@@ -661,7 +661,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 
 // Put the object into the bucket
 //
-// Copy the reader in to the new object which is returned
+// Copy the reader in to the new object which is returned.
 //
 // The new object may have been created if an error is returned
 func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {

backend/googlephotos/googlephotos_test.go

@@ -3,7 +3,7 @@ package googlephotos
 import (
 	"context"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"path"
 	"testing"
@@ -99,7 +99,7 @@ func TestIntegration(t *testing.T) {
 			t.Run("ObjectOpen", func(t *testing.T) {
 				in, err := dstObj.Open(ctx)
 				require.NoError(t, err)
-				buf, err := ioutil.ReadAll(in)
+				buf, err := io.ReadAll(in)
 				require.NoError(t, err)
 				require.NoError(t, in.Close())
 				assert.True(t, len(buf) > 1000)

backend/googlephotos/pattern.go

@@ -315,7 +315,7 @@ func yearMonthDayFilter(ctx context.Context, f lister, match []string) (sf api.S
 
 // featureFilter creates a filter for the Feature enum
 //
-// The API only supports one feature, FAVORITES, so hardcode that feature
+// The API only supports one feature, FAVORITES, so hardcode that feature.
 //
 // https://developers.google.com/photos/library/reference/rest/v1/mediaItems/search#FeatureFilter
 func featureFilter(ctx context.Context, f lister, match []string) (sf api.SearchFilter) {

backend/hasher/object.go

@@ -5,7 +5,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"path"
 	"time"
 
@@ -118,7 +117,7 @@ func (o *Object) updateHashes(ctx context.Context) error {
 	defer func() {
 		_ = r.Close()
 	}()
-	if _, err = io.Copy(ioutil.Discard, r); err != nil {
+	if _, err = io.Copy(io.Discard, r); err != nil {
 		fs.Infof(o, "update failed (copy): %v", err)
 		return err
 	}

backend/hdfs/fs.go

@@ -265,9 +265,9 @@ func (f *Fs) Purge(ctx context.Context, dir string) error {
 
 // Move src to this remote using server-side move operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //

backend/hdfs/hdfs.go

@@ -1,6 +1,7 @@
 //go:build !plan9
 // +build !plan9
 
+// Package hdfs provides an interface to the HDFS storage system.
 package hdfs
 
 import (

backend/hidrive/hidrive.go

@@ -330,7 +330,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		f.tokenRenewer = oauthutil.NewRenew(f.String(), ts, transaction)
 	}
 
-	// Do not allow the root-prefix to be non-existent nor a directory,
+	// Do not allow the root-prefix to be nonexistent nor a directory,
 	// but it can be empty.
 	if f.opt.RootPrefix != "" {
 		item, err := f.fetchMetadataForPath(ctx, f.opt.RootPrefix, api.HiDriveObjectNoMetadataFields)
@@ -623,7 +623,7 @@ func (f *Fs) Purge(ctx context.Context, dir string) error {
 // should be retried after the parent-directories of the destination have been created.
 // If so, it will create the parent-directories.
 //
-// If any errors arrise while finding the source or
+// If any errors arise while finding the source or
 // creating the parent-directory those will be returned.
 // Otherwise returns the originalError.
 func (f *Fs) shouldRetryAndCreateParents(ctx context.Context, destinationPath string, sourcePath string, originalError error) (bool, error) {
@@ -961,7 +961,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		} else {
 			_, _, err = o.fs.uploadFileChunked(ctx, resolvedPath, in, modTime, int(o.fs.opt.UploadChunkSize), o.fs.opt.UploadConcurrency)
 		}
-		// Try to check if object was updated, eitherway.
+		// Try to check if object was updated, either way.
 		// Metadata should be updated even if the upload fails.
 		info, metaErr = o.fs.fetchMetadataForPath(ctx, resolvedPath, api.HiDriveObjectWithMetadataFields)
 	} else {

backend/hidrive/hidrivehash/hidrivehash_test.go

@@ -138,7 +138,7 @@ var testTable = []struct {
 	// pattern describes how to use data to construct the hash-input.
 	// For every entry n at even indices this repeats the data n times.
 	// For every entry m at odd indices this repeats a null-byte m times.
-	// The input-data is constructed by concatinating the results in order.
+	// The input-data is constructed by concatenating the results in order.
 	pattern []int64
 	out     []byte
 	name    string

backend/hidrive/hidrivehash/internal/internal.go

@@ -1,3 +1,4 @@
+// Package internal provides utilities for HiDrive.
 package internal
 
 import (

backend/http/http.go

@@ -13,7 +13,6 @@ import (
 	"net/http"
 	"net/url"
 	"path"
-	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -305,7 +304,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 		fs:     f,
 		remote: remote,
 	}
-	err := o.stat(ctx)
+	err := o.head(ctx)
 	if err != nil {
 		return nil, err
 	}
@@ -317,15 +316,6 @@ func (f *Fs) url(remote string) string {
 	return f.endpointURL + rest.URLPathEscape(remote)
 }
 
-// parse s into an int64, on failure return def
-func parseInt64(s string, def int64) int64 {
-	n, e := strconv.ParseInt(s, 10, 64)
-	if e != nil {
-		return def
-	}
-	return n
-}
-
 // Errors returned by parseName
 var (
 	errURLJoinFailed     = errors.New("URLJoin failed")
@@ -500,7 +490,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 					fs:     f,
 					remote: remote,
 				}
-				switch err := file.stat(ctx); err {
+				switch err := file.head(ctx); err {
 				case nil:
 					add(file)
 				case fs.ErrorNotAFile:
@@ -579,8 +569,8 @@ func (o *Object) url() string {
 	return o.fs.url(o.remote)
 }
 
-// stat updates the info field in the Object
-func (o *Object) stat(ctx context.Context) error {
+// head sends a HEAD request to update info fields in the Object
+func (o *Object) head(ctx context.Context) error {
 	if o.fs.opt.NoHead {
 		o.size = -1
 		o.modTime = timeUnset
@@ -601,13 +591,19 @@ func (o *Object) stat(ctx context.Context) error {
 	if err != nil {
 		return fmt.Errorf("failed to stat: %w", err)
 	}
+	return o.decodeMetadata(ctx, res)
+}
+
+// decodeMetadata updates info fields in the Object according to HTTP response headers
+func (o *Object) decodeMetadata(ctx context.Context, res *http.Response) error {
 	t, err := http.ParseTime(res.Header.Get("Last-Modified"))
 	if err != nil {
 		t = timeUnset
 	}
-	o.size = parseInt64(res.Header.Get("Content-Length"), -1)
 	o.modTime = t
 	o.contentType = res.Header.Get("Content-Type")
+	o.size = rest.ParseSizeFromHeaders(res.Header)
+
 	// If NoSlash is set then check ContentType to see if it is a directory
 	if o.fs.opt.NoSlash {
 		mediaType, _, err := mime.ParseMediaType(o.contentType)
@@ -653,6 +649,9 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	if err != nil {
 		return nil, fmt.Errorf("Open failed: %w", err)
 	}
+	if err = o.decodeMetadata(ctx, res); err != nil {
+		return nil, fmt.Errorf("decodeMetadata failed: %w", err)
+	}
 	return res.Body, nil
 }
 

backend/http/http_internal_test.go

@@ -3,7 +3,7 @@ package http
 import (
 	"context"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
@@ -41,12 +41,12 @@ func prepareServer(t *testing.T) (configmap.Simple, func()) {
 	// verify the file path is correct, and also check which line endings
 	// are used to get sizes right ("\n" except on Windows, but even there
 	// we may have "\n" or "\r\n" depending on git crlf setting)
-	fileList, err := ioutil.ReadDir(filesPath)
+	fileList, err := os.ReadDir(filesPath)
 	require.NoError(t, err)
 	require.Greater(t, len(fileList), 0)
 	for _, file := range fileList {
 		if !file.IsDir() {
-			data, _ := ioutil.ReadFile(filepath.Join(filesPath, file.Name()))
+			data, _ := os.ReadFile(filepath.Join(filesPath, file.Name()))
 			if strings.HasSuffix(string(data), "\r\n") {
 				lineEndSize = 2
 			}
@@ -194,31 +194,66 @@ func TestNewObject(t *testing.T) {
 }
 
 func TestOpen(t *testing.T) {
-	f, tidy := prepare(t)
+	m, tidy := prepareServer(t)
 	defer tidy()
 
-	o, err := f.NewObject(context.Background(), "four/under four.txt")
-	require.NoError(t, err)
+	for _, head := range []bool{false, true} {
+		if !head {
+			m.Set("no_head", "true")
+		}
+		f, err := NewFs(context.Background(), remoteName, "", m)
+		require.NoError(t, err)
 
-	// Test normal read
-	fd, err := o.Open(context.Background())
-	require.NoError(t, err)
-	data, err := ioutil.ReadAll(fd)
-	require.NoError(t, err)
-	require.NoError(t, fd.Close())
-	if lineEndSize == 2 {
-		assert.Equal(t, "beetroot\r\n", string(data))
-	} else {
-		assert.Equal(t, "beetroot\n", string(data))
+		for _, rangeRead := range []bool{false, true} {
+			o, err := f.NewObject(context.Background(), "four/under four.txt")
+			require.NoError(t, err)
+
+			if !head {
+				// Test mod time is still indeterminate
+				tObj := o.ModTime(context.Background())
+				assert.Equal(t, time.Duration(0), time.Unix(0, 0).Sub(tObj))
+
+				// Test file size is still indeterminate
+				assert.Equal(t, int64(-1), o.Size())
+			}
+
+			var data []byte
+			if !rangeRead {
+				// Test normal read
+				fd, err := o.Open(context.Background())
+				require.NoError(t, err)
+				data, err = io.ReadAll(fd)
+				require.NoError(t, err)
+				require.NoError(t, fd.Close())
+				if lineEndSize == 2 {
+					assert.Equal(t, "beetroot\r\n", string(data))
+				} else {
+					assert.Equal(t, "beetroot\n", string(data))
+				}
+			} else {
+				// Test with range request
+				fd, err := o.Open(context.Background(), &fs.RangeOption{Start: 1, End: 5})
+				require.NoError(t, err)
+				data, err = io.ReadAll(fd)
+				require.NoError(t, err)
+				require.NoError(t, fd.Close())
+				assert.Equal(t, "eetro", string(data))
+			}
+
+			fi, err := os.Stat(filepath.Join(filesPath, "four", "under four.txt"))
+			require.NoError(t, err)
+			tFile := fi.ModTime()
+
+			// Test the time is always correct on the object after file open
+			tObj := o.ModTime(context.Background())
+			fstest.AssertTimeEqualWithPrecision(t, o.Remote(), tFile, tObj, time.Second)
+
+			if !rangeRead {
+				// Test the file size
+				assert.Equal(t, int64(len(data)), o.Size())
+			}
+		}
 	}
-
-	// Test with range request
-	fd, err = o.Open(context.Background(), &fs.RangeOption{Start: 1, End: 5})
-	require.NoError(t, err)
-	data, err = ioutil.ReadAll(fd)
-	require.NoError(t, err)
-	require.NoError(t, fd.Close())
-	assert.Equal(t, "eetro", string(data))
 }
 
 func TestMimeType(t *testing.T) {

backend/hubic/auth.go

@@ -1,62 +0,0 @@
-package hubic
-
-import (
-	"context"
-	"net/http"
-	"time"
-
-	"github.com/ncw/swift/v2"
-	"github.com/rclone/rclone/fs"
-)
-
-// auth is an authenticator for swift
-type auth struct {
-	f *Fs
-}
-
-// newAuth creates a swift authenticator
-func newAuth(f *Fs) *auth {
-	return &auth{
-		f: f,
-	}
-}
-
-// Request constructs an http.Request for authentication
-//
-// returns nil for not needed
-func (a *auth) Request(ctx context.Context, c *swift.Connection) (r *http.Request, err error) {
-	const retries = 10
-	for try := 1; try <= retries; try++ {
-		err = a.f.getCredentials(context.TODO())
-		if err == nil {
-			break
-		}
-		time.Sleep(100 * time.Millisecond)
-		fs.Debugf(a.f, "retrying auth request %d/%d: %v", try, retries, err)
-	}
-	return nil, err
-}
-
-// Response parses the result of an http request
-func (a *auth) Response(ctx context.Context, resp *http.Response) error {
-	return nil
-}
-
-// The public storage URL - set Internal to true to read
-// internal/service net URL
-func (a *auth) StorageUrl(Internal bool) string { // nolint
-	return a.f.credentials.Endpoint
-}
-
-// The access token
-func (a *auth) Token() string {
-	return a.f.credentials.Token
-}
-
-// The CDN url if available
-func (a *auth) CdnUrl() string { // nolint
-	return ""
-}
-
-// Check the interfaces are satisfied
-var _ swift.Authenticator = (*auth)(nil)

backend/hubic/hubic.go

@@ -1,200 +0,0 @@
-// Package hubic provides an interface to the Hubic object storage
-// system.
-package hubic
-
-// This uses the normal swift mechanism to update the credentials and
-// ignores the expires field returned by the Hubic API.  This may need
-// to be revisited after some actual experience.
-
-import (
-	"context"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"io/ioutil"
-	"net/http"
-	"strings"
-	"time"
-
-	swiftLib "github.com/ncw/swift/v2"
-	"github.com/rclone/rclone/backend/swift"
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/config/configmap"
-	"github.com/rclone/rclone/fs/config/configstruct"
-	"github.com/rclone/rclone/fs/config/obscure"
-	"github.com/rclone/rclone/fs/fshttp"
-	"github.com/rclone/rclone/lib/oauthutil"
-	"golang.org/x/oauth2"
-)
-
-const (
-	rcloneClientID              = "api_hubic_svWP970PvSWbw5G3PzrAqZ6X2uHeZBPI"
-	rcloneEncryptedClientSecret = "leZKCcqy9movLhDWLVXX8cSLp_FzoiAPeEJOIOMRw1A5RuC4iLEPDYPWVF46adC_MVonnLdVEOTHVstfBOZ_lY4WNp8CK_YWlpRZ9diT5YI"
-)
-
-// Globals
-var (
-	// Description of how to auth for this app
-	oauthConfig = &oauth2.Config{
-		Scopes: []string{
-			"credentials.r", // Read OpenStack credentials
-		},
-		Endpoint: oauth2.Endpoint{
-			AuthURL:  "https://api.hubic.com/oauth/auth/",
-			TokenURL: "https://api.hubic.com/oauth/token/",
-		},
-		ClientID:     rcloneClientID,
-		ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
-		RedirectURL:  oauthutil.RedirectLocalhostURL,
-	}
-)
-
-// Register with Fs
-func init() {
-	fs.Register(&fs.RegInfo{
-		Name:        "hubic",
-		Description: "Hubic",
-		NewFs:       NewFs,
-		Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
-			return oauthutil.ConfigOut("", &oauthutil.Options{
-				OAuth2Config: oauthConfig,
-			})
-		},
-		Options: append(oauthutil.SharedOptions, swift.SharedOptions...),
-	})
-}
-
-// credentials is the JSON returned from the Hubic API to read the
-// OpenStack credentials
-type credentials struct {
-	Token    string `json:"token"`    // OpenStack token
-	Endpoint string `json:"endpoint"` // OpenStack endpoint
-	Expires  string `json:"expires"`  // Expires date - e.g. "2015-11-09T14:24:56+01:00"
-}
-
-// Fs represents a remote hubic
-type Fs struct {
-	fs.Fs                    // wrapped Fs
-	features    *fs.Features // optional features
-	client      *http.Client // client for oauth api
-	credentials credentials  // returned from the Hubic API
-	expires     time.Time    // time credentials expire
-}
-
-// Object describes a swift object
-type Object struct {
-	*swift.Object
-}
-
-// Return a string version
-func (o *Object) String() string {
-	if o == nil {
-		return "<nil>"
-	}
-	return o.Object.String()
-}
-
-// ------------------------------------------------------------
-
-// String converts this Fs to a string
-func (f *Fs) String() string {
-	if f.Fs == nil {
-		return "Hubic"
-	}
-	return fmt.Sprintf("Hubic %s", f.Fs.String())
-}
-
-// getCredentials reads the OpenStack Credentials using the Hubic API
-//
-// The credentials are read into the Fs
-func (f *Fs) getCredentials(ctx context.Context) (err error) {
-	req, err := http.NewRequestWithContext(ctx, "GET", "https://api.hubic.com/1.0/account/credentials", nil)
-	if err != nil {
-		return err
-	}
-	resp, err := f.client.Do(req)
-	if err != nil {
-		return err
-	}
-	defer fs.CheckClose(resp.Body, &err)
-	if resp.StatusCode < 200 || resp.StatusCode > 299 {
-		body, _ := ioutil.ReadAll(resp.Body)
-		bodyStr := strings.TrimSpace(strings.ReplaceAll(string(body), "\n", " "))
-		return fmt.Errorf("failed to get credentials: %s: %s", resp.Status, bodyStr)
-	}
-	decoder := json.NewDecoder(resp.Body)
-	var result credentials
-	err = decoder.Decode(&result)
-	if err != nil {
-		return err
-	}
-	// fs.Debugf(f, "Got credentials %+v", result)
-	if result.Token == "" || result.Endpoint == "" || result.Expires == "" {
-		return errors.New("couldn't read token, result and expired from credentials")
-	}
-	f.credentials = result
-	expires, err := time.Parse(time.RFC3339, result.Expires)
-	if err != nil {
-		return err
-	}
-	f.expires = expires
-	fs.Debugf(f, "Got swift credentials (expiry %v in %v)", f.expires, time.Until(f.expires))
-	return nil
-}
-
-// NewFs constructs an Fs from the path, container:path
-func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
-	client, _, err := oauthutil.NewClient(ctx, name, m, oauthConfig)
-	if err != nil {
-		return nil, fmt.Errorf("failed to configure Hubic: %w", err)
-	}
-
-	f := &Fs{
-		client: client,
-	}
-
-	// Make the swift Connection
-	ci := fs.GetConfig(ctx)
-	c := &swiftLib.Connection{
-		Auth:           newAuth(f),
-		ConnectTimeout: 10 * ci.ConnectTimeout, // Use the timeouts in the transport
-		Timeout:        10 * ci.Timeout,        // Use the timeouts in the transport
-		Transport:      fshttp.NewTransport(ctx),
-	}
-	err = c.Authenticate(ctx)
-	if err != nil {
-		return nil, fmt.Errorf("error authenticating swift connection: %w", err)
-	}
-
-	// Parse config into swift.Options struct
-	opt := new(swift.Options)
-	err = configstruct.Set(m, opt)
-	if err != nil {
-		return nil, err
-	}
-
-	// Make inner swift Fs from the connection
-	swiftFs, err := swift.NewFsWithConnection(ctx, opt, name, root, c, true)
-	if err != nil && err != fs.ErrorIsFile {
-		return nil, err
-	}
-	f.Fs = swiftFs
-	f.features = f.Fs.Features().Wrap(f)
-	return f, err
-}
-
-// Features returns the optional features of this Fs
-func (f *Fs) Features() *fs.Features {
-	return f.features
-}
-
-// UnWrap returns the Fs that this Fs is wrapping
-func (f *Fs) UnWrap() fs.Fs {
-	return f.Fs
-}
-
-// Check the interfaces are satisfied
-var (
-	_ fs.Fs        = (*Fs)(nil)
-	_ fs.UnWrapper = (*Fs)(nil)
-)

backend/hubic/hubic_test.go

@@ -1,19 +0,0 @@
-// Test Hubic filesystem interface
-package hubic_test
-
-import (
-	"testing"
-
-	"github.com/rclone/rclone/backend/hubic"
-	"github.com/rclone/rclone/fstest/fstests"
-)
-
-// TestIntegration runs integration tests against the remote
-func TestIntegration(t *testing.T) {
-	fstests.Run(t, &fstests.Opt{
-		RemoteName:          "TestHubic:",
-		NilObject:           (*hubic.Object)(nil),
-		SkipFsCheckWrap:     true,
-		SkipObjectCheckWrap: true,
-	})
-}

backend/internetarchive/internetarchive.go

@@ -227,7 +227,7 @@ type Object struct {
 	rawData json.RawMessage
 }
 
-// IAFile reprensents a subset of object in MetadataResponse.Files
+// IAFile represents a subset of object in MetadataResponse.Files
 type IAFile struct {
 	Name string `json:"name"`
 	// Source     string `json:"source"`
@@ -243,7 +243,7 @@ type IAFile struct {
 	rawData json.RawMessage
 }
 
-// MetadataResponse reprensents subset of the JSON object returned by (frontend)/metadata/
+// MetadataResponse represents subset of the JSON object returned by (frontend)/metadata/
 type MetadataResponse struct {
 	Files    []IAFile `json:"files"`
 	ItemSize int64    `json:"item_size"`
@@ -577,9 +577,9 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
 
 // Copy src to this remote using server-side copy operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -1273,7 +1273,7 @@ func trimPathPrefix(s, prefix string, enc encoder.MultiEncoder) string {
 	return enc.ToStandardPath(strings.TrimPrefix(s, prefix+"/"))
 }
 
-// mimicks urllib.parse.quote() on Python; exclude / from url.PathEscape
+// mimics urllib.parse.quote() on Python; exclude / from url.PathEscape
 func quotePath(s string) string {
 	seg := strings.Split(s, "/")
 	newValues := []string{}

backend/jottacloud/api/types.go

@@ -1,3 +1,4 @@
+// Package api provides types used by the Jottacloud API.
 package api
 
 import (

backend/jottacloud/jottacloud.go

@@ -1,3 +1,4 @@
+// Package jottacloud provides an interface to the Jottacloud storage system.
 package jottacloud
 
 import (
@@ -11,7 +12,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"math/rand"
 	"net/http"
 	"net/url"
@@ -821,7 +821,7 @@ func (f *Fs) allocatePathRaw(file string, absolute bool) string {
 func grantTypeFilter(req *http.Request) {
 	if legacyTokenURL == req.URL.String() {
 		// read the entire body
-		refreshBody, err := ioutil.ReadAll(req.Body)
+		refreshBody, err := io.ReadAll(req.Body)
 		if err != nil {
 			return
 		}
@@ -831,7 +831,7 @@ func grantTypeFilter(req *http.Request) {
 		refreshBody = []byte(strings.Replace(string(refreshBody), "grant_type=refresh_token", "grant_type=REFRESH_TOKEN", 1))
 
 		// set the new ReadCloser (with a dummy Close())
-		req.Body = ioutil.NopCloser(bytes.NewReader(refreshBody))
+		req.Body = io.NopCloser(bytes.NewReader(refreshBody))
 	}
 }
 
@@ -1247,7 +1247,7 @@ func (f *Fs) createObject(remote string, modTime time.Time, size int64) (o *Obje
 
 // Put the object
 //
-// Copy the reader in to the new object which is returned
+// Copy the reader in to the new object which is returned.
 //
 // The new object may have been created if an error is returned
 func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
@@ -1397,9 +1397,9 @@ func (f *Fs) copyOrMove(ctx context.Context, method, src, dest string) (info *ap
 
 // Copy src to this remote using server-side copy operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -1417,7 +1417,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	}
 	info, err := f.copyOrMove(ctx, "cp", srcObj.filePath(), remote)
 
-	// if destination was a trashed file then after a successfull copy the copied file is still in trash (bug in api?)
+	// if destination was a trashed file then after a successful copy the copied file is still in trash (bug in api?)
 	if err == nil && bool(info.Deleted) && !f.opt.TrashedOnly && info.State == "COMPLETED" {
 		fs.Debugf(src, "Server-side copied to trashed destination, restoring")
 		info, err = f.createOrUpdate(ctx, remote, srcObj.modTime, srcObj.size, srcObj.md5)
@@ -1433,9 +1433,9 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 
 // Move src to this remote using server-side move operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -1788,7 +1788,7 @@ func readMD5(in io.Reader, size, threshold int64) (md5sum string, out io.Reader,
 		var tempFile *os.File
 
 		// create the cache file
-		tempFile, err = ioutil.TempFile("", cachePrefix)
+		tempFile, err = os.CreateTemp("", cachePrefix)
 		if err != nil {
 			return
 		}
@@ -1816,7 +1816,7 @@ func readMD5(in io.Reader, size, threshold int64) (md5sum string, out io.Reader,
 	} else {
 		// that's a small file, just read it into memory
 		var inData []byte
-		inData, err = ioutil.ReadAll(teeReader)
+		inData, err = io.ReadAll(teeReader)
 		if err != nil {
 			return
 		}
@@ -1829,7 +1829,7 @@ func readMD5(in io.Reader, size, threshold int64) (md5sum string, out io.Reader,
 
 // Update the object with the contents of the io.Reader, modTime and size
 //
-// If existing is set then it updates the object rather than creating a new one
+// If existing is set then it updates the object rather than creating a new one.
 //
 // The new object may have been created if an error is returned
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
@@ -1913,7 +1913,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 		// copy the already uploaded bytes into the trash :)
 		var result api.UploadResponse
-		_, err = io.CopyN(ioutil.Discard, in, response.ResumePos)
+		_, err = io.CopyN(io.Discard, in, response.ResumePos)
 		if err != nil {
 			return err
 		}

backend/koofr/koofr.go

@@ -1,3 +1,4 @@
+// Package koofr provides an interface to the Koofr storage system.
 package koofr
 
 import (
@@ -667,7 +668,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
 	//
 	// https://app.koofr.net/content/links/39a6cc01-3b23-477a-8059-c0fb3b0f15de/files/get?path=%2F
 	//
-	// I am not sure about meaning of "path" parameter; in my expriments
+	// I am not sure about meaning of "path" parameter; in my experiments
 	// it is always "%2F", and omitting it or putting any other value
 	// results in 404.
 	//

backend/local/local.go

@@ -7,7 +7,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"path"
 	"path/filepath"
@@ -22,6 +21,7 @@ import (
 	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
+	"github.com/rclone/rclone/fs/filter"
 	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/lib/encoder"
@@ -123,8 +123,8 @@ routine so this flag shouldn't normally be used.`,
 			Help: `Don't check to see if the files change during upload.
 
 Normally rclone checks the size and modification time of files as they
-are being uploaded and aborts with a message which starts "can't copy
-- source file is being updated" if the file changes during upload.
+are being uploaded and aborts with a message which starts "can't copy -
+source file is being updated" if the file changes during upload.
 
 However on some file systems this modification time check may fail (e.g.
 [Glusterfs #2206](https://github.com/rclone/rclone/issues/2206)) so this
@@ -234,15 +234,16 @@ type Options struct {
 
 // Fs represents a local filesystem rooted at root
 type Fs struct {
-	name        string              // the name of the remote
-	root        string              // The root directory (OS path)
-	opt         Options             // parsed config options
-	features    *fs.Features        // optional features
-	dev         uint64              // device number of root node
-	precisionOk sync.Once           // Whether we need to read the precision
-	precision   time.Duration       // precision of local filesystem
-	warnedMu    sync.Mutex          // used for locking access to 'warned'.
-	warned      map[string]struct{} // whether we have warned about this string
+	name           string              // the name of the remote
+	root           string              // The root directory (OS path)
+	opt            Options             // parsed config options
+	features       *fs.Features        // optional features
+	dev            uint64              // device number of root node
+	precisionOk    sync.Once           // Whether we need to read the precision
+	precision      time.Duration       // precision of local filesystem
+	warnedMu       sync.Mutex          // used for locking access to 'warned'.
+	warned         map[string]struct{} // whether we have warned about this string
+	xattrSupported int32               // whether xattrs are supported (atomic access)
 
 	// do os.Lstat or os.Stat
 	lstat        func(name string) (os.FileInfo, error)
@@ -286,6 +287,9 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		dev:    devUnset,
 		lstat:  os.Lstat,
 	}
+	if xattrSupported {
+		f.xattrSupported = 1
+	}
 	f.root = cleanRootPath(root, f.opt.NoUNC, f.opt.Enc)
 	f.features = (&fs.Features{
 		CaseInsensitive:         f.caseInsensitive(),
@@ -295,6 +299,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		ReadMetadata:            true,
 		WriteMetadata:           true,
 		UserMetadata:            xattrSupported, // can only R/W general purpose metadata if xattrs are supported
+		FilterAware:             true,
 	}).Fill(ctx, f)
 	if opt.FollowSymlinks {
 		f.lstat = os.Stat
@@ -439,6 +444,8 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 // This should return ErrDirNotFound if the directory isn't
 // found.
 func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
+	filter, useFilter := filter.GetConfig(ctx), filter.GetUseFilter(ctx)
+
 	fsDirPath := f.localPath(dir)
 	_, err = os.Stat(fsDirPath)
 	if err != nil {
@@ -489,6 +496,13 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 						continue
 					}
 					if fierr != nil {
+						// Don't report errors on any file names that are excluded
+						if useFilter {
+							newRemote := f.cleanRemote(dir, name)
+							if !filter.IncludeRemote(newRemote) {
+								continue
+							}
+						}
 						err = fmt.Errorf("failed to read directory %q: %w", namepath, err)
 						fs.Errorf(dir, "%v", fierr)
 						_ = accounting.Stats(ctx).Error(fserrors.NoRetryError(fierr)) // fail the sync
@@ -522,6 +536,11 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 				}
 				mode = fi.Mode()
 			}
+			// Don't include non directory if not included
+			// we leave directory filtering to the layer above
+			if useFilter && !fi.IsDir() && !filter.IncludeRemote(newRemote) {
+				continue
+			}
 			if fi.IsDir() {
 				// Ignore directories which are symlinks.  These are junction points under windows which
 				// are kind of a souped up symlink. Unix doesn't have directories which are symlinks.
@@ -626,7 +645,7 @@ func (f *Fs) readPrecision() (precision time.Duration) {
 	precision = time.Second
 
 	// Create temporary file and test it
-	fd, err := ioutil.TempFile("", "rclone")
+	fd, err := os.CreateTemp("", "rclone")
 	if err != nil {
 		// If failed return 1s
 		// fmt.Println("Failed to create temp file", err)
@@ -695,9 +714,9 @@ func (f *Fs) Purge(ctx context.Context, dir string) error {
 
 // Move src to this remote using server-side move operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -1053,7 +1072,7 @@ func (o *Object) openTranslatedLink(offset, limit int64) (lrc io.ReadCloser, err
 	if err != nil {
 		return nil, err
 	}
-	return readers.NewLimitedReadCloser(ioutil.NopCloser(strings.NewReader(linkdst[offset:])), limit), nil
+	return readers.NewLimitedReadCloser(io.NopCloser(strings.NewReader(linkdst[offset:])), limit), nil
 }
 
 // Open an object for read
@@ -1380,30 +1399,27 @@ func (o *Object) writeMetadata(metadata fs.Metadata) (err error) {
 }
 
 func cleanRootPath(s string, noUNC bool, enc encoder.MultiEncoder) string {
-	if runtime.GOOS == "windows" {
-		if !filepath.IsAbs(s) && !strings.HasPrefix(s, "\\") {
+	if runtime.GOOS != "windows" || !strings.HasPrefix(s, "\\") {
+		if !filepath.IsAbs(s) {
 			s2, err := filepath.Abs(s)
 			if err == nil {
 				s = s2
 			}
+		} else {
+			s = filepath.Clean(s)
 		}
+	}
+	if runtime.GOOS == "windows" {
 		s = filepath.ToSlash(s)
 		vol := filepath.VolumeName(s)
 		s = vol + enc.FromStandardPath(s[len(vol):])
 		s = filepath.FromSlash(s)
-
 		if !noUNC {
 			// Convert to UNC
 			s = file.UNCPath(s)
 		}
 		return s
 	}
-	if !filepath.IsAbs(s) {
-		s2, err := filepath.Abs(s)
-		if err == nil {
-			s = s2
-		}
-	}
 	s = enc.FromStandardPath(s)
 	return s
 }

backend/local/local_internal_test.go

@@ -4,16 +4,18 @@ import (
 	"bytes"
 	"context"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"os"
 	"path"
 	"path/filepath"
 	"runtime"
+	"sort"
 	"testing"
 	"time"
 
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/config/configmap"
+	"github.com/rclone/rclone/fs/filter"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/fs/object"
 	"github.com/rclone/rclone/fstest"
@@ -148,7 +150,7 @@ func TestSymlink(t *testing.T) {
 	// Check reading the object
 	in, err := o.Open(ctx)
 	require.NoError(t, err)
-	contents, err := ioutil.ReadAll(in)
+	contents, err := io.ReadAll(in)
 	require.NoError(t, err)
 	require.Equal(t, "file.txt", string(contents))
 	require.NoError(t, in.Close())
@@ -156,7 +158,7 @@ func TestSymlink(t *testing.T) {
 	// Check reading the object with range
 	in, err = o.Open(ctx, &fs.RangeOption{Start: 2, End: 5})
 	require.NoError(t, err)
-	contents, err = ioutil.ReadAll(in)
+	contents, err = io.ReadAll(in)
 	require.NoError(t, err)
 	require.Equal(t, "file.txt"[2:5+1], string(contents))
 	require.NoError(t, in.Close())
@@ -190,7 +192,7 @@ func TestHashOnUpdate(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, "9a0364b9e99bb480dd25e1f0284c8555", md5)
 
-	// Reupload it with diferent contents but same size and timestamp
+	// Reupload it with different contents but same size and timestamp
 	var b = bytes.NewBufferString("CONTENT")
 	src := object.NewStaticObjectInfo(filePath, when, int64(b.Len()), true, nil, f)
 	err = o.Update(ctx, b, src)
@@ -366,3 +368,36 @@ func TestMetadata(t *testing.T) {
 	})
 
 }
+
+func TestFilter(t *testing.T) {
+	ctx := context.Background()
+	r := fstest.NewRun(t)
+	defer r.Finalise()
+	when := time.Now()
+	r.WriteFile("included", "included file", when)
+	r.WriteFile("excluded", "excluded file", when)
+	f := r.Flocal.(*Fs)
+
+	// Check set up for filtering
+	assert.True(t, f.Features().FilterAware)
+
+	// Add a filter
+	ctx, fi := filter.AddConfig(ctx)
+	require.NoError(t, fi.AddRule("+ included"))
+	require.NoError(t, fi.AddRule("- *"))
+
+	// Check listing without use filter flag
+	entries, err := f.List(ctx, "")
+	require.NoError(t, err)
+	sort.Sort(entries)
+	require.Equal(t, "[excluded included]", fmt.Sprint(entries))
+
+	// Add user filter flag
+	ctx = filter.SetUseFilter(ctx, true)
+
+	// Check listing with use filter flag
+	entries, err = f.List(ctx, "")
+	require.NoError(t, err)
+	sort.Sort(entries)
+	require.Equal(t, "[included]", fmt.Sprint(entries))
+}

backend/local/metadata_linux.go

@@ -5,19 +5,41 @@ package local
 
 import (
 	"fmt"
+	"sync"
 	"time"
 
 	"github.com/rclone/rclone/fs"
 	"golang.org/x/sys/unix"
 )
 
+var (
+	statxCheckOnce         sync.Once
+	readMetadataFromFileFn func(o *Object, m *fs.Metadata) (err error)
+)
+
 // Read the metadata from the file into metadata where possible
 func (o *Object) readMetadataFromFile(m *fs.Metadata) (err error) {
+	statxCheckOnce.Do(func() {
+		// Check statx() is available as it was only introduced in kernel 4.11
+		// If not, fall back to fstatat() which was introduced in 2.6.16 which is guaranteed for all Go versions
+		var stat unix.Statx_t
+		if unix.Statx(unix.AT_FDCWD, ".", 0, unix.STATX_ALL, &stat) != unix.ENOSYS {
+			readMetadataFromFileFn = readMetadataFromFileStatx
+		} else {
+			readMetadataFromFileFn = readMetadataFromFileFstatat
+		}
+	})
+	return readMetadataFromFileFn(o, m)
+}
+
+// Read the metadata from the file into metadata where possible
+func readMetadataFromFileStatx(o *Object, m *fs.Metadata) (err error) {
 	flags := unix.AT_SYMLINK_NOFOLLOW
 	if o.fs.opt.FollowSymlinks {
 		flags = 0
 	}
 	var stat unix.Statx_t
+	//  statx() was added to Linux in kernel 4.11
 	err = unix.Statx(unix.AT_FDCWD, o.path, flags, (0 |
 		unix.STATX_TYPE | // Want stx_mode & S_IFMT
 		unix.STATX_MODE | // Want stx_mode & ~S_IFMT
@@ -45,3 +67,36 @@ func (o *Object) readMetadataFromFile(m *fs.Metadata) (err error) {
 	setTime("btime", stat.Btime)
 	return nil
 }
+
+// Read the metadata from the file into metadata where possible
+func readMetadataFromFileFstatat(o *Object, m *fs.Metadata) (err error) {
+	flags := unix.AT_SYMLINK_NOFOLLOW
+	if o.fs.opt.FollowSymlinks {
+		flags = 0
+	}
+	var stat unix.Stat_t
+	// fstatat() was added to Linux in  kernel  2.6.16
+	// Go only supports 2.6.32 or later
+	err = unix.Fstatat(unix.AT_FDCWD, o.path, &stat, flags)
+	if err != nil {
+		return err
+	}
+	m.Set("mode", fmt.Sprintf("%0o", stat.Mode))
+	m.Set("uid", fmt.Sprintf("%d", stat.Uid))
+	m.Set("gid", fmt.Sprintf("%d", stat.Gid))
+	if stat.Rdev != 0 {
+		m.Set("rdev", fmt.Sprintf("%x", stat.Rdev))
+	}
+	setTime := func(key string, t unix.Timespec) {
+		// The types of t.Sec and t.Nsec vary from int32 to int64 on
+		// different Linux architectures so we need to cast them to
+		// int64 here and hence need to quiet the linter about
+		// unecessary casts.
+		//
+		// nolint: unconvert
+		m.Set(key, time.Unix(int64(t.Sec), int64(t.Nsec)).Format(metadataTimeFormat))
+	}
+	setTime("atime", stat.Atim)
+	setTime("mtime", stat.Mtim)
+	return nil
+}

backend/local/remove_test.go

@@ -1,7 +1,6 @@
 package local
 
 import (
-	"io/ioutil"
 	"os"
 	"sync"
 	"testing"
@@ -13,7 +12,7 @@ import (
 
 // Check we can remove an open file
 func TestRemove(t *testing.T) {
-	fd, err := ioutil.TempFile("", "rclone-remove-test")
+	fd, err := os.CreateTemp("", "rclone-remove-test")
 	require.NoError(t, err)
 	name := fd.Name()
 	defer func() {

backend/local/setbtime.go

@@ -9,7 +9,7 @@ import (
 
 const haveSetBTime = false
 
-// setBTime changes the the birth time of the file passed in
+// setBTime changes the birth time of the file passed in
 func setBTime(name string, btime time.Time) error {
 	// Does nothing
 	return nil

backend/local/setbtime_windows.go

@@ -5,13 +5,13 @@ package local
 
 import (
 	"os"
-	"time"
 	"syscall"
+	"time"
 )
 
 const haveSetBTime = true
 
-// setBTime sets the the birth time of the file passed in
+// setBTime sets the birth time of the file passed in
 func setBTime(name string, btime time.Time) (err error) {
 	h, err := syscall.Open(name, os.O_RDWR, 0755)
 	if err != nil {

backend/local/xattr.go

@@ -6,6 +6,8 @@ package local
 import (
 	"fmt"
 	"strings"
+	"sync/atomic"
+	"syscall"
 
 	"github.com/pkg/xattr"
 	"github.com/rclone/rclone/fs"
@@ -16,12 +18,30 @@ const (
 	xattrSupported = xattr.XATTR_SUPPORTED
 )
 
+// Check to see if the error supplied is a not supported error, and if
+// so, disable xattrs
+func (f *Fs) xattrIsNotSupported(err error) bool {
+	xattrErr, ok := err.(*xattr.Error)
+	if !ok {
+		return false
+	}
+	// Xattrs not supported can be ENOTSUP or ENOATTR or EINVAL (on Solaris)
+	if xattrErr.Err == syscall.EINVAL || xattrErr.Err == syscall.ENOTSUP || xattrErr.Err == xattr.ENOATTR {
+		// Show xattrs not supported
+		if atomic.CompareAndSwapInt32(&f.xattrSupported, 1, 0) {
+			fs.Errorf(f, "xattrs not supported - disabling: %v", err)
+		}
+		return true
+	}
+	return false
+}
+
 // getXattr returns the extended attributes for an object
 //
 // It doesn't return any attributes owned by this backend in
 // metadataKeys
 func (o *Object) getXattr() (metadata fs.Metadata, err error) {
-	if !xattrSupported {
+	if !xattrSupported || atomic.LoadInt32(&o.fs.xattrSupported) == 0 {
 		return nil, nil
 	}
 	var list []string
@@ -31,6 +51,9 @@ func (o *Object) getXattr() (metadata fs.Metadata, err error) {
 		list, err = xattr.LList(o.path)
 	}
 	if err != nil {
+		if o.fs.xattrIsNotSupported(err) {
+			return nil, nil
+		}
 		return nil, fmt.Errorf("failed to read xattr: %w", err)
 	}
 	if len(list) == 0 {
@@ -45,6 +68,9 @@ func (o *Object) getXattr() (metadata fs.Metadata, err error) {
 			v, err = xattr.LGet(o.path, k)
 		}
 		if err != nil {
+			if o.fs.xattrIsNotSupported(err) {
+				return nil, nil
+			}
 			return nil, fmt.Errorf("failed to read xattr key %q: %w", k, err)
 		}
 		k = strings.ToLower(k)
@@ -64,7 +90,7 @@ func (o *Object) getXattr() (metadata fs.Metadata, err error) {
 //
 // It doesn't set any attributes owned by this backend in metadataKeys
 func (o *Object) setXattr(metadata fs.Metadata) (err error) {
-	if !xattrSupported {
+	if !xattrSupported || atomic.LoadInt32(&o.fs.xattrSupported) == 0 {
 		return nil
 	}
 	for k, value := range metadata {
@@ -80,6 +106,9 @@ func (o *Object) setXattr(metadata fs.Metadata) (err error) {
 			err = xattr.LSet(o.path, k, v)
 		}
 		if err != nil {
+			if o.fs.xattrIsNotSupported(err) {
+				return nil
+			}
 			return fmt.Errorf("failed to set xattr key %q: %w", k, err)
 		}
 	}

backend/mailru/api/helpers.go

@@ -69,6 +69,11 @@ func (w *BinWriter) WritePu64(val int64) {
 	w.b.Write(w.a[:binary.PutUvarint(w.a, uint64(val))])
 }
 
+// WriteP64 writes an signed long as unsigned varint
+func (w *BinWriter) WriteP64(val int64) {
+	w.b.Write(w.a[:binary.PutUvarint(w.a, uint64(val))])
+}
+
 // WriteString writes a zero-terminated string
 func (w *BinWriter) WriteString(str string) {
 	buf := []byte(str)

backend/mailru/api/m1.go

@@ -1,3 +1,4 @@
+// Package api provides types used by the Mail.ru API.
 package api
 
 import (

backend/mailru/mailru.go

@@ -1,3 +1,4 @@
+// Package mailru provides an interface to the Mail.ru Cloud storage system.
 package mailru
 
 import (
@@ -17,7 +18,6 @@ import (
 
 	"encoding/hex"
 	"encoding/json"
-	"io/ioutil"
 	"net/http"
 	"net/url"
 
@@ -90,8 +90,13 @@ func init() {
 			Help:     "User name (usually email).",
 			Required: true,
 		}, {
-			Name:       "pass",
-			Help:       "Password.",
+			Name: "pass",
+			Help: `Password.
+
+This must be an app password - rclone will not work with your normal
+password. See the Configuration section in the docs for how to make an
+app password.
+`,
 			Required:   true,
 			IsPassword: true,
 		}, {
@@ -630,21 +635,17 @@ func (f *Fs) readItemMetaData(ctx context.Context, path string) (entry fs.DirEnt
 
 // itemToEntry converts API item to rclone directory entry
 // The dirSize return value is:
-//   <0 - for a file or in case of error
-//   =0 - for an empty directory
-//   >0 - for a non-empty directory
+//
+//	<0 - for a file or in case of error
+//	=0 - for an empty directory
+//	>0 - for a non-empty directory
 func (f *Fs) itemToDirEntry(ctx context.Context, item *api.ListItem) (entry fs.DirEntry, dirSize int, err error) {
 	remote, err := f.relPath(f.opt.Enc.ToStandardPath(item.Home))
 	if err != nil {
 		return nil, -1, err
 	}
 
-	mTime := int64(item.Mtime)
-	if mTime < 0 {
-		fs.Debugf(f, "Fixing invalid timestamp %d on mailru file %q", mTime, remote)
-		mTime = 0
-	}
-	modTime := time.Unix(mTime, 0)
+	modTime := time.Unix(int64(item.Mtime), 0)
 
 	isDir, err := f.isDir(item.Kind, remote)
 	if err != nil {
@@ -1658,7 +1659,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 	// Attempt to put by calculating hash in memory
 	if trySpeedup && size <= int64(o.fs.opt.SpeedupMaxMem) {
-		fileBuf, err = ioutil.ReadAll(in)
+		fileBuf, err = io.ReadAll(in)
 		if err != nil {
 			return err
 		}
@@ -1701,7 +1702,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	if size <= mrhash.Size {
 		// Optimize upload: skip extra request if data fits in the hash buffer.
 		if fileBuf == nil {
-			fileBuf, err = ioutil.ReadAll(wrapIn)
+			fileBuf, err = io.ReadAll(wrapIn)
 		}
 		if fileHash == nil && err == nil {
 			fileHash = mrhash.Sum(fileBuf)
@@ -2056,7 +2057,7 @@ func (o *Object) addFileMetaData(ctx context.Context, overwrite bool) error {
 	req.WritePu16(0) // revision
 	req.WriteString(o.fs.opt.Enc.FromStandardPath(o.absPath()))
 	req.WritePu64(o.size)
-	req.WritePu64(o.modTime.Unix())
+	req.WriteP64(o.modTime.Unix())
 	req.WritePu32(0)
 	req.Write(o.mrHash)
 
@@ -2212,7 +2213,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 		fs.Debugf(o, "Server returned full content instead of range")
 		if start > 0 {
 			// Discard the beginning of the data
-			_, err = io.CopyN(ioutil.Discard, wrapStream, start)
+			_, err = io.CopyN(io.Discard, wrapStream, start)
 			if err != nil {
 				closeBody(res)
 				return nil, err

backend/mega/mega.go

@@ -118,7 +118,7 @@ type Fs struct {
 
 // Object describes a mega object
 //
-// Will definitely have info but maybe not meta
+// Will definitely have info but maybe not meta.
 //
 // Normally rclone would just store an ID here but go-mega and mega.nz
 // expect you to build an entire tree of all the objects in memory.
@@ -347,7 +347,7 @@ func (f *Fs) mkdir(ctx context.Context, rootNode *mega.Node, dir string) (node *
 		}
 	}
 	if err != nil {
-		return nil, fmt.Errorf("internal error: mkdir called with non-existent root node: %w", err)
+		return nil, fmt.Errorf("internal error: mkdir called with nonexistent root node: %w", err)
 	}
 	// i is number of directories to create (may be 0)
 	// node is directory to create them from
@@ -387,7 +387,7 @@ func (f *Fs) findRoot(ctx context.Context, create bool) (*mega.Node, error) {
 		return f._rootNode, nil
 	}
 
-	// Check for pre-existing root
+	// Check for preexisting root
 	absRoot := f.srv.FS.GetRoot()
 	node, err := f.findDir(absRoot, f.root)
 	//log.Printf("findRoot findDir %p %v", node, err)
@@ -536,7 +536,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 // Creates from the parameters passed in a half finished Object which
 // must have setMetaData called on it
 //
-// Returns the dirNode, object, leaf and error
+// Returns the dirNode, object, leaf and error.
 //
 // Used to create new objects
 func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, size int64) (o *Object, dirNode *mega.Node, leaf string, err error) {
@@ -554,7 +554,7 @@ func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time,
 
 // Put the object
 //
-// Copy the reader in to the new object which is returned
+// Copy the reader in to the new object which is returned.
 //
 // The new object may have been created if an error is returned
 // PutUnchecked uploads the object
@@ -576,7 +576,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
 
 // PutUnchecked the object
 //
-// Copy the reader in to the new object which is returned
+// Copy the reader in to the new object which is returned.
 //
 // The new object may have been created if an error is returned
 // PutUnchecked uploads the object
@@ -749,9 +749,9 @@ func (f *Fs) move(ctx context.Context, dstRemote string, srcFs *Fs, srcRemote st
 
 // Move src to this remote using server-side move operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -979,7 +979,6 @@ func (o *Object) readMetaData(ctx context.Context) (err error) {
 
 // ModTime returns the modification time of the object
 //
-//
 // It attempts to read the objects mtime and if that isn't present the
 // LastModified returned in the http headers
 func (o *Object) ModTime(ctx context.Context) time.Time {
@@ -1115,7 +1114,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 
 // Update the object with the contents of the io.Reader, modTime and size
 //
-// If existing is set then it updates the object rather than creating a new one
+// If existing is set then it updates the object rather than creating a new one.
 //
 // The new object may have been created if an error is returned
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {

backend/memory/memory.go

@@ -8,7 +8,6 @@ import (
 	"encoding/hex"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"path"
 	"strings"
 	"sync"
@@ -418,7 +417,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
 
 // Put the object into the bucket
 //
-// Copy the reader in to the new object which is returned
+// Copy the reader in to the new object which is returned.
 //
 // The new object may have been created if an error is returned
 func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
@@ -463,9 +462,9 @@ func (f *Fs) Precision() time.Duration {
 
 // Copy src to this remote using server-side copy operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -575,7 +574,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 		}
 		data = data[:limit]
 	}
-	return ioutil.NopCloser(bytes.NewBuffer(data)), nil
+	return io.NopCloser(bytes.NewBuffer(data)), nil
 }
 
 // Update the object with the contents of the io.Reader, modTime and size
@@ -583,7 +582,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 // The new object may have been created if an error is returned
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
 	bucket, bucketPath := o.split()
-	data, err := ioutil.ReadAll(in)
+	data, err := io.ReadAll(in)
 	if err != nil {
 		return fmt.Errorf("failed to update memory object: %w", err)
 	}

backend/netstorage/netstorage.go

@@ -12,7 +12,6 @@ import (
 	"fmt"
 	gohash "hash"
 	"io"
-	"io/ioutil"
 	"math/rand"
 	"net/http"
 	"net/url"
@@ -118,7 +117,7 @@ type Fs struct {
 	filetype         string            // dir, file or symlink
 	dirscreated      map[string]bool   // if implicit dir has been created already
 	dirscreatedMutex sync.Mutex        // mutex to protect dirscreated
-	statcache        map[string][]File // cache successfull stat requests
+	statcache        map[string][]File // cache successful stat requests
 	statcacheMutex   sync.RWMutex      // RWMutex to protect statcache
 }
 
@@ -424,7 +423,7 @@ func (f *Fs) getFileName(file *File) string {
 func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
 	if f.filetype == "" {
 		// This happens in two scenarios.
-		// 1. NewFs is done on a non-existent object, then later rclone attempts to List/ListR this NewFs.
+		// 1. NewFs is done on a nonexistent object, then later rclone attempts to List/ListR this NewFs.
 		// 2. List/ListR is called from the context of test_all and not the regular rclone binary.
 		err := f.initFs(ctx, dir)
 		if err != nil {
@@ -488,7 +487,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {
 	if f.filetype == "" {
 		// This happens in two scenarios.
-		// 1. NewFs is done on a non-existent object, then later rclone attempts to List/ListR this NewFs.
+		// 1. NewFs is done on a nonexistent object, then later rclone attempts to List/ListR this NewFs.
 		// 2. List/ListR is called from the context of test_all and not the regular rclone binary.
 		err := f.initFs(ctx, dir)
 		if err != nil {
@@ -972,7 +971,7 @@ func (o *Object) netStorageUploadRequest(ctx context.Context, in io.Reader, src
 		URL = o.fs.url(src.Remote())
 	}
 	if strings.HasSuffix(URL, ".rclonelink") {
-		bits, err := ioutil.ReadAll(in)
+		bits, err := io.ReadAll(in)
 		if err != nil {
 			return err
 		}
@@ -1058,7 +1057,7 @@ func (o *Object) netStorageDownloadRequest(ctx context.Context, options []fs.Ope
 	if strings.HasSuffix(URL, ".rclonelink") && o.target != "" {
 		fs.Infof(nil, "Converting a symlink to the rclonelink file on download %q", URL)
 		reader := strings.NewReader(o.target)
-		readcloser := ioutil.NopCloser(reader)
+		readcloser := io.NopCloser(reader)
 		return readcloser, nil
 	}
 

backend/onedrive/api/types.go

@@ -1,5 +1,4 @@
-// Types passed and returned to and from the API
-
+// Package api provides types used by the OneDrive API.
 package api
 
 import (
@@ -14,7 +13,7 @@ const (
 	PackageTypeOneNote = "oneNote"
 )
 
-// Error is returned from one drive when things go wrong
+// Error is returned from OneDrive when things go wrong
 type Error struct {
 	ErrorInfo struct {
 		Code       string `json:"code"`
@@ -71,7 +70,7 @@ type Drive struct {
 	Quota     Quota       `json:"quota"`
 }
 
-// Timestamp represents represents date and time information for the
+// Timestamp represents date and time information for the
 // OneDrive API, by using ISO 8601 and is always in UTC time.
 type Timestamp time.Time
 
@@ -250,8 +249,8 @@ type MoveItemRequest struct {
 	FileSystemInfo  *FileSystemInfoFacet `json:"fileSystemInfo,omitempty"`  // File system information on client. Read-write.
 }
 
-//CreateShareLinkRequest is the request to create a sharing link
-//Always Type:view and Scope:anonymous for public sharing
+// CreateShareLinkRequest is the request to create a sharing link
+// Always Type:view and Scope:anonymous for public sharing
 type CreateShareLinkRequest struct {
 	Type     string     `json:"type"`                         // Link type in View, Edit or Embed
 	Scope    string     `json:"scope,omitempty"`              // Scope in anonymous, organization
@@ -259,7 +258,7 @@ type CreateShareLinkRequest struct {
 	Expiry   *time.Time `json:"expirationDateTime,omitempty"` // A String with format of yyyy-MM-ddTHH:mm:ssZ of DateTime indicates the expiration time of the permission.
 }
 
-//CreateShareLinkResponse is the response from CreateShareLinkRequest
+// CreateShareLinkResponse is the response from CreateShareLinkRequest
 type CreateShareLinkResponse struct {
 	ID    string   `json:"id"`
 	Roles []string `json:"roles"`

backend/onedrive/onedrive.go

@@ -118,7 +118,7 @@ func init() {
 					Help:  "Microsoft Cloud Germany",
 				}, {
 					Value: regionCN,
-					Help:  "Azure and Office 365 operated by 21Vianet in China",
+					Help:  "Azure and Office 365 operated by Vnet Group in China",
 				},
 			},
 		}, {
@@ -600,14 +600,14 @@ type Options struct {
 	Enc                     encoder.MultiEncoder `config:"encoding"`
 }
 
-// Fs represents a remote one drive
+// Fs represents a remote OneDrive
 type Fs struct {
 	name         string             // name of this remote
 	root         string             // the path we are working on
 	opt          Options            // parsed options
 	ci           *fs.ConfigInfo     // global config
 	features     *fs.Features       // optional features
-	srv          *rest.Client       // the connection to the one drive server
+	srv          *rest.Client       // the connection to the OneDrive server
 	dirCache     *dircache.DirCache // Map of directory path to directory id
 	pacer        *fs.Pacer          // pacer for API calls
 	tokenRenewer *oauthutil.Renew   // renew the token on expiry
@@ -615,7 +615,7 @@ type Fs struct {
 	driveType    string             // https://developer.microsoft.com/en-us/graph/docs/api-reference/v1.0/resources/drive
 }
 
-// Object describes a one drive object
+// Object describes a OneDrive object
 //
 // Will definitely have info but maybe not meta
 type Object struct {
@@ -645,7 +645,7 @@ func (f *Fs) Root() string {
 
 // String converts this Fs to a string
 func (f *Fs) String() string {
-	return fmt.Sprintf("One drive root '%s'", f.root)
+	return fmt.Sprintf("OneDrive root '%s'", f.root)
 }
 
 // Features returns the optional features of this Fs
@@ -653,7 +653,7 @@ func (f *Fs) Features() *fs.Features {
 	return f.features
 }
 
-// parsePath parses a one drive 'url'
+// parsePath parses a OneDrive 'url'
 func parsePath(path string) (root string) {
 	root = strings.Trim(path, "/")
 	return
@@ -727,7 +727,7 @@ func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, err
 // "shared with me" folders in OneDrive Personal (See #2536, #2778)
 // This path pattern comes from https://github.com/OneDrive/onedrive-api-docs/issues/908#issuecomment-417488480
 //
-// If `relPath` == '', do not append the slash (See #3664)
+// If `relPath` == ”, do not append the slash (See #3664)
 func (f *Fs) readMetaDataForPathRelativeToID(ctx context.Context, normalizedID string, relPath string) (info *api.Item, resp *http.Response, err error) {
 	opts, _ := f.newOptsCallWithIDPath(normalizedID, relPath, true, "GET", "")
 
@@ -891,6 +891,12 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	}).Fill(ctx, f)
 	f.srv.SetErrorHandler(errorHandler)
 
+	// Disable change polling in China region
+	// See: https://github.com/rclone/rclone/issues/6444
+	if f.opt.Region == regionCN {
+		f.features.ChangeNotify = nil
+	}
+
 	// Renew the token in the background
 	f.tokenRenewer = oauthutil.NewRenew(f.String(), ts, func() error {
 		_, _, err := f.readMetaDataForPath(ctx, "")
@@ -1137,7 +1143,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 // Creates from the parameters passed in a half finished Object which
 // must have setMetaData called on it
 //
-// Returns the object, leaf, directoryID and error
+// Returns the object, leaf, directoryID and error.
 //
 // Used to create new objects
 func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, size int64) (o *Object, leaf string, directoryID string, err error) {
@@ -1156,7 +1162,7 @@ func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time,
 
 // Put the object into the container
 //
-// Copy the reader in to the new object which is returned
+// Copy the reader in to the new object which is returned.
 //
 // The new object may have been created if an error is returned
 func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
@@ -1280,9 +1286,9 @@ func (f *Fs) waitForJob(ctx context.Context, location string, o *Object) error {
 
 // Copy src to this remote using server-side copy operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -1387,9 +1393,9 @@ func (f *Fs) Purge(ctx context.Context, dir string) error {
 
 // Move src to this remote using server-side move operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -1843,7 +1849,6 @@ func (o *Object) readMetaData(ctx context.Context) (err error) {
 
 // ModTime returns the modification time of the object
 //
-//
 // It attempts to read the objects mtime and if that isn't present the
 // LastModified returned in the http headers
 func (o *Object) ModTime(ctx context.Context) time.Time {
@@ -2184,7 +2189,7 @@ func (o *Object) ID() string {
  *       3. To avoid region-related issues, please don't manually build rest.Opts from scratch.
  *          Instead, use these helper function, and customize the URL afterwards if needed.
  *
- *       currently, the 21ViaNet's API differs in the following places:
+ *       currently, the Vnet Group's API differs in the following places:
  *       - https://{Endpoint}/drives/{driveID}/items/{leaf}:/{route}
  *           - this API doesn't work (gives invalid request)
  *           - can be replaced with the following API:
@@ -2233,7 +2238,7 @@ func escapeSingleQuote(str string) string {
 // newOptsCallWithIDPath build the rest.Opts structure with *a normalizedID (driveID#fileID, or simply fileID) and leaf*
 // using url template https://{Endpoint}/drives/{driveID}/items/{leaf}:/{route} (for international OneDrive)
 // or https://{Endpoint}/drives/{driveID}/items/children('{leaf}')/{route}
-// and https://{Endpoint}/drives/{driveID}/items/children('@a1')/{route}?@a1=URLEncode("'{leaf}'") (for 21ViaNet)
+// and https://{Endpoint}/drives/{driveID}/items/children('@a1')/{route}?@a1=URLEncode("'{leaf}'") (for Vnet Group)
 // if isPath is false, this function will only work when the leaf is "" or a child name (i.e. it doesn't accept multi-level leaf)
 // if isPath is true, multi-level leaf like a/b/c can be passed
 func (f *Fs) newOptsCallWithIDPath(normalizedID string, leaf string, isPath bool, method string, route string) (opts rest.Opts, ok bool) {

backend/opendrive/opendrive.go

@@ -1,3 +1,4 @@
+// Package opendrive provides an interface to the OpenDrive storage system.
 package opendrive
 
 import (
@@ -340,9 +341,9 @@ func (f *Fs) Precision() time.Duration {
 
 // Copy src to this remote using server-side copy operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -404,9 +405,9 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 
 // Move src to this remote using server-side move operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -562,7 +563,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 // Creates from the parameters passed in a half finished Object which
 // must have setMetaData called on it
 //
-// Returns the object, leaf, directoryID and error
+// Returns the object, leaf, directoryID and error.
 //
 // Used to create new objects
 func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, size int64) (o *Object, leaf string, directoryID string, err error) {
@@ -599,7 +600,7 @@ func (f *Fs) readMetaDataForFolderID(ctx context.Context, id string) (info *Fold
 
 // Put the object into the bucket
 //
-// Copy the reader in to the new object which is returned
+// Copy the reader in to the new object which is returned.
 //
 // The new object may have been created if an error is returned
 func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
@@ -829,7 +830,6 @@ func (o *Object) Size() int64 {
 
 // ModTime returns the modification time of the object
 //
-//
 // It attempts to read the objects mtime and if that isn't present the
 // LastModified returned in the http headers
 func (o *Object) ModTime(ctx context.Context) time.Time {

backend/oracleobjectstorage/client.go

@@ -0,0 +1,158 @@
+//go:build !plan9 && !solaris && !js
+// +build !plan9,!solaris,!js
+
+package oracleobjectstorage
+
+import (
+	"context"
+	"crypto/rsa"
+	"errors"
+	"net/http"
+	"os"
+
+	"github.com/oracle/oci-go-sdk/v65/common"
+	"github.com/oracle/oci-go-sdk/v65/common/auth"
+	"github.com/oracle/oci-go-sdk/v65/objectstorage"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/fserrors"
+	"github.com/rclone/rclone/fs/fshttp"
+)
+
+func getConfigurationProvider(opt *Options) (common.ConfigurationProvider, error) {
+	switch opt.Provider {
+	case instancePrincipal:
+		return auth.InstancePrincipalConfigurationProvider()
+	case userPrincipal:
+		if opt.ConfigFile != "" && !fileExists(opt.ConfigFile) {
+			fs.Errorf(userPrincipal, "oci config file doesn't exist at %v", opt.ConfigFile)
+		}
+		return common.CustomProfileConfigProvider(opt.ConfigFile, opt.ConfigProfile), nil
+	case resourcePrincipal:
+		return auth.ResourcePrincipalConfigurationProvider()
+	case noAuth:
+		fs.Infof("client", "using no auth provider")
+		return getNoAuthConfiguration()
+	default:
+	}
+	return common.DefaultConfigProvider(), nil
+}
+
+func newObjectStorageClient(ctx context.Context, opt *Options) (*objectstorage.ObjectStorageClient, error) {
+	p, err := getConfigurationProvider(opt)
+	if err != nil {
+		return nil, err
+	}
+	client, err := objectstorage.NewObjectStorageClientWithConfigurationProvider(p)
+	if err != nil {
+		fs.Errorf(opt.Provider, "failed to create object storage client, %v", err)
+		return nil, err
+	}
+	if opt.Region != "" {
+		client.SetRegion(opt.Region)
+	}
+	modifyClient(ctx, opt, &client.BaseClient)
+	return &client, err
+}
+
+func fileExists(filePath string) bool {
+	if _, err := os.Stat(filePath); errors.Is(err, os.ErrNotExist) {
+		return false
+	}
+	return true
+}
+
+func modifyClient(ctx context.Context, opt *Options, client *common.BaseClient) {
+	client.HTTPClient = getHTTPClient(ctx)
+	if opt.Provider == noAuth {
+		client.Signer = getNoAuthSigner()
+	}
+}
+
+// getClient makes http client according to the global options
+// this has rclone specific options support like dump headers, body etc.
+func getHTTPClient(ctx context.Context) *http.Client {
+	return fshttp.NewClient(ctx)
+}
+
+var retryErrorCodes = []int{
+	408, // Request Timeout
+	429, // Rate exceeded.
+	500, // Get occasional 500 Internal Server Error
+	503, // Service Unavailable
+	504, // Gateway Time-out
+}
+
+func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
+	if fserrors.ContextError(ctx, &err) {
+		return false, err
+	}
+	// If this is an ocierr object, try and extract more useful information to determine if we should retry
+	if ociError, ok := err.(common.ServiceError); ok {
+		// Simple case, check the original embedded error in case it's generically retryable
+		if fserrors.ShouldRetry(err) {
+			return true, err
+		}
+		// If it is a timeout then we want to retry that
+		if ociError.GetCode() == "RequestTimeout" {
+			return true, err
+		}
+	}
+	// Ok, not an oci error, check for generic failure conditions
+	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
+}
+
+func getNoAuthConfiguration() (common.ConfigurationProvider, error) {
+	return &noAuthConfigurator{}, nil
+}
+
+func getNoAuthSigner() common.HTTPRequestSigner {
+	return &noAuthSigner{}
+}
+
+type noAuthConfigurator struct {
+}
+
+type noAuthSigner struct {
+}
+
+func (n *noAuthSigner) Sign(*http.Request) error {
+	return nil
+}
+
+func (n *noAuthConfigurator) PrivateRSAKey() (*rsa.PrivateKey, error) {
+	return nil, nil
+}
+
+func (n *noAuthConfigurator) KeyID() (string, error) {
+	return "", nil
+}
+
+func (n *noAuthConfigurator) TenancyOCID() (string, error) {
+	return "", nil
+}
+
+func (n *noAuthConfigurator) UserOCID() (string, error) {
+	return "", nil
+}
+
+func (n *noAuthConfigurator) KeyFingerprint() (string, error) {
+	return "", nil
+}
+
+func (n *noAuthConfigurator) Region() (string, error) {
+	return "", nil
+}
+
+func (n *noAuthConfigurator) AuthType() (common.AuthConfig, error) {
+	return common.AuthConfig{
+		AuthType:         common.UnknownAuthenticationType,
+		IsFromConfigFile: false,
+		OboToken:         nil,
+	}, nil
+}
+
+// Check the interfaces are satisfied
+var (
+	_ common.ConfigurationProvider = &noAuthConfigurator{}
+	_ common.HTTPRequestSigner     = &noAuthSigner{}
+)

backend/oracleobjectstorage/command.go

@@ -0,0 +1,228 @@
+//go:build !plan9 && !solaris && !js
+// +build !plan9,!solaris,!js
+
+package oracleobjectstorage
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/oracle/oci-go-sdk/v65/common"
+	"github.com/oracle/oci-go-sdk/v65/objectstorage"
+	"github.com/rclone/rclone/fs"
+)
+
+// ------------------------------------------------------------
+// Command Interface Implementation
+// ------------------------------------------------------------
+
+const (
+	operationRename        = "rename"
+	operationListMultiPart = "list-multipart-uploads"
+	operationCleanup       = "cleanup"
+)
+
+var commandHelp = []fs.CommandHelp{{
+	Name:  operationRename,
+	Short: "change the name of an object",
+	Long: `This command can be used to rename a object.
+
+Usage Examples:
+
+    rclone backend rename oos:bucket relative-object-path-under-bucket object-new-name
+`,
+	Opts: nil,
+}, {
+	Name:  operationListMultiPart,
+	Short: "List the unfinished multipart uploads",
+	Long: `This command lists the unfinished multipart uploads in JSON format.
+
+    rclone backend list-multipart-uploads oos:bucket/path/to/object
+
+It returns a dictionary of buckets with values as lists of unfinished
+multipart uploads.
+
+You can call it with no bucket in which case it lists all bucket, with
+a bucket or with a bucket and path.
+
+    {
+      "test-bucket": [
+                {
+                        "namespace": "test-namespace",
+                        "bucket": "test-bucket",
+                        "object": "600m.bin",
+                        "uploadId": "51dd8114-52a4-b2f2-c42f-5291f05eb3c8",
+                        "timeCreated": "2022-07-29T06:21:16.595Z",
+                        "storageTier": "Standard"
+                }
+        ]
+`,
+}, {
+	Name:  operationCleanup,
+	Short: "Remove unfinished multipart uploads.",
+	Long: `This command removes unfinished multipart uploads of age greater than
+max-age which defaults to 24 hours.
+
+Note that you can use -i/--dry-run with this command to see what it
+would do.
+
+    rclone backend cleanup oos:bucket/path/to/object
+    rclone backend cleanup -o max-age=7w oos:bucket/path/to/object
+
+Durations are parsed as per the rest of rclone, 2h, 7d, 7w etc.
+`,
+	Opts: map[string]string{
+		"max-age": "Max age of upload to delete",
+	},
+},
+}
+
+/*
+Command the backend to run a named command
+
+The command run is name
+args may be used to read arguments from
+opts may be used to read optional arguments from
+
+The result should be capable of being JSON encoded
+If it is a string or a []string it will be shown to the user
+otherwise it will be JSON encoded and shown to the user like that
+*/
+func (f *Fs) Command(ctx context.Context, commandName string, args []string,
+	opt map[string]string) (result interface{}, err error) {
+	// fs.Debugf(f, "command %v, args: %v, opts:%v", commandName, args, opt)
+	switch commandName {
+	case operationRename:
+		if len(args) < 2 {
+			return nil, fmt.Errorf("path to object or its new name to rename is empty")
+		}
+		remote := args[0]
+		newName := args[1]
+		return f.rename(ctx, remote, newName)
+	case operationListMultiPart:
+		return f.listMultipartUploadsAll(ctx)
+	case operationCleanup:
+		maxAge := 24 * time.Hour
+		if opt["max-age"] != "" {
+			maxAge, err = fs.ParseDuration(opt["max-age"])
+			if err != nil {
+				return nil, fmt.Errorf("bad max-age: %w", err)
+			}
+		}
+		return nil, f.cleanUp(ctx, maxAge)
+	default:
+		return nil, fs.ErrorCommandNotFound
+	}
+}
+
+func (f *Fs) rename(ctx context.Context, remote, newName string) (interface{}, error) {
+	if remote == "" {
+		return nil, fmt.Errorf("path to object file cannot be empty")
+	}
+	if newName == "" {
+		return nil, fmt.Errorf("the object's new name cannot be empty")
+	}
+	o := &Object{
+		fs:     f,
+		remote: remote,
+	}
+	bucketName, objectPath := o.split()
+	err := o.readMetaData(ctx)
+	if err != nil {
+		fs.Errorf(f, "failed to read object:%v %v ", objectPath, err)
+		if strings.HasPrefix(objectPath, bucketName) {
+			fs.Errorf(f, "warn: ensure object path: %v is relative to bucket:%v and doesn't include the bucket name",
+				objectPath, bucketName)
+		}
+		return nil, fs.ErrorNotAFile
+	}
+	details := objectstorage.RenameObjectDetails{
+		SourceName: common.String(objectPath),
+		NewName:    common.String(newName),
+	}
+	request := objectstorage.RenameObjectRequest{
+		NamespaceName:       common.String(f.opt.Namespace),
+		BucketName:          common.String(bucketName),
+		RenameObjectDetails: details,
+		OpcClientRequestId:  nil,
+		RequestMetadata:     common.RequestMetadata{},
+	}
+	var response objectstorage.RenameObjectResponse
+	err = f.pacer.Call(func() (bool, error) {
+		response, err = f.srv.RenameObject(ctx, request)
+		return shouldRetry(ctx, response.HTTPResponse(), err)
+	})
+	if err != nil {
+		return nil, err
+	}
+	fs.Infof(f, "success: renamed object-path: %v to %v", objectPath, newName)
+	return "renamed successfully", nil
+}
+
+func (f *Fs) listMultipartUploadsAll(ctx context.Context) (uploadsMap map[string][]*objectstorage.MultipartUpload,
+	err error) {
+	uploadsMap = make(map[string][]*objectstorage.MultipartUpload)
+	bucket, directory := f.split("")
+	if bucket != "" {
+		uploads, err := f.listMultipartUploads(ctx, bucket, directory)
+		if err != nil {
+			return uploadsMap, err
+		}
+		uploadsMap[bucket] = uploads
+		return uploadsMap, nil
+	}
+	entries, err := f.listBuckets(ctx)
+	if err != nil {
+		return uploadsMap, err
+	}
+	for _, entry := range entries {
+		bucket := entry.Remote()
+		uploads, listErr := f.listMultipartUploads(ctx, bucket, "")
+		if listErr != nil {
+			err = listErr
+			fs.Errorf(f, "%v", err)
+		}
+		uploadsMap[bucket] = uploads
+	}
+	return uploadsMap, err
+}
+
+// listMultipartUploads lists all outstanding multipart uploads for (bucket, key)
+//
+// Note that rather lazily we treat key as a prefix, so it matches
+// directories and objects. This could surprise the user if they ask
+// for "dir" and it returns "dirKey"
+func (f *Fs) listMultipartUploads(ctx context.Context, bucketName, directory string) (
+	uploads []*objectstorage.MultipartUpload, err error) {
+
+	uploads = []*objectstorage.MultipartUpload{}
+	req := objectstorage.ListMultipartUploadsRequest{
+		NamespaceName: common.String(f.opt.Namespace),
+		BucketName:    common.String(bucketName),
+	}
+
+	var response objectstorage.ListMultipartUploadsResponse
+	for {
+		err = f.pacer.Call(func() (bool, error) {
+			response, err = f.srv.ListMultipartUploads(ctx, req)
+			return shouldRetry(ctx, response.HTTPResponse(), err)
+		})
+		if err != nil {
+			// fs.Debugf(f, "failed to list multi part uploads %v", err)
+			return uploads, err
+		}
+		for index, item := range response.Items {
+			if directory != "" && item.Object != nil && !strings.HasPrefix(*item.Object, directory) {
+				continue
+			}
+			uploads = append(uploads, &response.Items[index])
+		}
+		if response.OpcNextPage == nil {
+			break
+		}
+		req.Page = response.OpcNextPage
+	}
+	return uploads, nil
+}

backend/oracleobjectstorage/copy.go

@@ -0,0 +1,155 @@
+//go:build !plan9 && !solaris && !js
+// +build !plan9,!solaris,!js
+
+package oracleobjectstorage
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/oracle/oci-go-sdk/v65/common"
+	"github.com/oracle/oci-go-sdk/v65/objectstorage"
+	"github.com/rclone/rclone/fs"
+)
+
+// ------------------------------------------------------------
+// Implement Copier is an optional interfaces for Fs
+//------------------------------------------------------------
+
+// Copy src to this remote using server-side copy operations.
+// This is stored with the remote path given
+// It returns the destination Object and a possible error
+// Will only be called if src.Fs().Name() == f.Name()
+// If it isn't possible then return fs.ErrorCantCopy
+func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
+	// fs.Debugf(f, "copying %v to %v", src.Remote(), remote)
+	srcObj, ok := src.(*Object)
+	if !ok {
+		// fs.Debugf(src, "Can't copy - not same remote type")
+		return nil, fs.ErrorCantCopy
+	}
+	// Temporary Object under construction
+	dstObj := &Object{
+		fs:     f,
+		remote: remote,
+	}
+	err := f.copy(ctx, dstObj, srcObj)
+	if err != nil {
+		return nil, err
+	}
+	return f.NewObject(ctx, remote)
+}
+
+// copy does a server-side copy from dstObj <- srcObj
+//
+// If newInfo is nil then the metadata will be copied otherwise it
+// will be replaced with newInfo
+func (f *Fs) copy(ctx context.Context, dstObj *Object, srcObj *Object) (err error) {
+	srcBucket, srcPath := srcObj.split()
+	dstBucket, dstPath := dstObj.split()
+	if dstBucket != srcBucket {
+		exists, err := f.bucketExists(ctx, dstBucket)
+		if err != nil {
+			return err
+		}
+		if !exists {
+			err = f.makeBucket(ctx, dstBucket)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	copyObjectDetails := objectstorage.CopyObjectDetails{
+		SourceObjectName:          common.String(srcPath),
+		DestinationRegion:         common.String(dstObj.fs.opt.Region),
+		DestinationNamespace:      common.String(dstObj.fs.opt.Namespace),
+		DestinationBucket:         common.String(dstBucket),
+		DestinationObjectName:     common.String(dstPath),
+		DestinationObjectMetadata: metadataWithOpcPrefix(srcObj.meta),
+	}
+	req := objectstorage.CopyObjectRequest{
+		NamespaceName:     common.String(srcObj.fs.opt.Namespace),
+		BucketName:        common.String(srcBucket),
+		CopyObjectDetails: copyObjectDetails,
+	}
+	var resp objectstorage.CopyObjectResponse
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err = f.srv.CopyObject(ctx, req)
+		return shouldRetry(ctx, resp.HTTPResponse(), err)
+	})
+	if err != nil {
+		return err
+	}
+	workRequestID := resp.OpcWorkRequestId
+	timeout := time.Duration(f.opt.CopyTimeout)
+	dstName := dstObj.String()
+	// https://docs.oracle.com/en-us/iaas/Content/Object/Tasks/copyingobjects.htm
+	// To enable server side copy object, customers will have to
+	// grant policy to objectstorage service to manage object-family
+	// Allow service objectstorage-<region_identifier> to manage object-family in tenancy
+	// Another option to avoid the policy is to download and reupload the file.
+	// This download upload will work for maximum file size limit of 5GB
+	err = copyObjectWaitForWorkRequest(ctx, workRequestID, dstName, timeout, f.srv)
+	if err != nil {
+		return err
+	}
+	return err
+}
+
+func copyObjectWaitForWorkRequest(ctx context.Context, wID *string, entityType string, timeout time.Duration,
+	client *objectstorage.ObjectStorageClient) error {
+
+	stateConf := &StateChangeConf{
+		Pending: []string{
+			string(objectstorage.WorkRequestStatusAccepted),
+			string(objectstorage.WorkRequestStatusInProgress),
+			string(objectstorage.WorkRequestStatusCanceling),
+		},
+		Target: []string{
+			string(objectstorage.WorkRequestSummaryStatusCompleted),
+			string(objectstorage.WorkRequestSummaryStatusCanceled),
+			string(objectstorage.WorkRequestStatusFailed),
+		},
+		Refresh: func() (interface{}, string, error) {
+			getWorkRequestRequest := objectstorage.GetWorkRequestRequest{}
+			getWorkRequestRequest.WorkRequestId = wID
+			workRequestResponse, err := client.GetWorkRequest(context.Background(), getWorkRequestRequest)
+			wr := &workRequestResponse.WorkRequest
+			return workRequestResponse, string(wr.Status), err
+		},
+		Timeout: timeout,
+	}
+
+	wrr, e := stateConf.WaitForStateContext(ctx, entityType)
+	if e != nil {
+		return fmt.Errorf("work request did not succeed, workId: %s, entity: %s. Message: %s", *wID, entityType, e)
+	}
+
+	wr := wrr.(objectstorage.GetWorkRequestResponse).WorkRequest
+	if wr.Status == objectstorage.WorkRequestStatusFailed {
+		errorMessage, _ := getObjectStorageErrorFromWorkRequest(ctx, wID, client)
+		return fmt.Errorf("work request did not succeed, workId: %s, entity: %s. Message: %s", *wID, entityType, errorMessage)
+	}
+
+	return nil
+}
+
+func getObjectStorageErrorFromWorkRequest(ctx context.Context, workRequestID *string, client *objectstorage.ObjectStorageClient) (string, error) {
+	req := objectstorage.ListWorkRequestErrorsRequest{}
+	req.WorkRequestId = workRequestID
+	res, err := client.ListWorkRequestErrors(ctx, req)
+
+	if err != nil {
+		return "", err
+	}
+
+	allErrs := make([]string, 0)
+	for _, errs := range res.Items {
+		allErrs = append(allErrs, *errs.Message)
+	}
+
+	errorMessage := strings.Join(allErrs, "\n")
+	return errorMessage, nil
+}

backend/oracleobjectstorage/object.go

@@ -0,0 +1,621 @@
+//go:build !plan9 && !solaris && !js
+// +build !plan9,!solaris,!js
+
+package oracleobjectstorage
+
+import (
+	"context"
+	"encoding/base64"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"net/http"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/ncw/swift/v2"
+	"github.com/oracle/oci-go-sdk/v65/common"
+	"github.com/oracle/oci-go-sdk/v65/objectstorage"
+	"github.com/oracle/oci-go-sdk/v65/objectstorage/transfer"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/lib/atexit"
+)
+
+// ------------------------------------------------------------
+// Object Interface Implementation
+// ------------------------------------------------------------
+
+const (
+	metaMtime   = "mtime"     // the meta key to store mtime in - e.g. X-Amz-Meta-Mtime
+	metaMD5Hash = "md5chksum" // the meta key to store md5hash in
+	// StandardTier object storage tier
+	ociMetaPrefix = "opc-meta-"
+)
+
+var archive = "archive"
+var infrequentAccess = "infrequentaccess"
+var standard = "standard"
+
+var storageTierMap = map[string]*string{
+	archive:          &archive,
+	infrequentAccess: &infrequentAccess,
+	standard:         &standard,
+}
+
+var matchMd5 = regexp.MustCompile(`^[0-9a-f]{32}$`)
+
+// Object describes a oci bucket object
+type Object struct {
+	fs           *Fs               // what this object is part of
+	remote       string            // The remote path
+	md5          string            // MD5 hash if known
+	bytes        int64             // Size of the object
+	lastModified time.Time         // The modified time of the object if known
+	meta         map[string]string // The object metadata if known - may be nil
+	mimeType     string            // Content-Type of the object
+
+	// Metadata as pointers to strings as they often won't be present
+	storageTier *string // e.g. Standard
+}
+
+// split returns bucket and bucketPath from the object
+func (o *Object) split() (bucket, bucketPath string) {
+	return o.fs.split(o.remote)
+}
+
+// readMetaData gets the metadata if it hasn't already been fetched
+func (o *Object) readMetaData(ctx context.Context) (err error) {
+	fs.Debugf(o, "trying to read metadata %v", o.remote)
+	if o.meta != nil {
+		return nil
+	}
+	info, err := o.headObject(ctx)
+	if err != nil {
+		return err
+	}
+	return o.decodeMetaDataHead(info)
+}
+
+// headObject gets the metadata from the object unconditionally
+func (o *Object) headObject(ctx context.Context) (info *objectstorage.HeadObjectResponse, err error) {
+	bucketName, objectPath := o.split()
+	req := objectstorage.HeadObjectRequest{
+		NamespaceName: common.String(o.fs.opt.Namespace),
+		BucketName:    common.String(bucketName),
+		ObjectName:    common.String(objectPath),
+	}
+	var response objectstorage.HeadObjectResponse
+	err = o.fs.pacer.Call(func() (bool, error) {
+		var err error
+		response, err = o.fs.srv.HeadObject(ctx, req)
+		return shouldRetry(ctx, response.HTTPResponse(), err)
+	})
+	if err != nil {
+		if svcErr, ok := err.(common.ServiceError); ok {
+			if svcErr.GetHTTPStatusCode() == http.StatusNotFound {
+				return nil, fs.ErrorObjectNotFound
+			}
+		}
+		return nil, err
+	}
+	o.fs.cache.MarkOK(bucketName)
+	return &response, err
+}
+
+func (o *Object) decodeMetaDataHead(info *objectstorage.HeadObjectResponse) (err error) {
+	return o.setMetaData(
+		info.ContentLength,
+		info.ContentMd5,
+		info.ContentType,
+		info.LastModified,
+		info.StorageTier,
+		info.OpcMeta)
+}
+
+func (o *Object) decodeMetaDataObject(info *objectstorage.GetObjectResponse) (err error) {
+	return o.setMetaData(
+		info.ContentLength,
+		info.ContentMd5,
+		info.ContentType,
+		info.LastModified,
+		info.StorageTier,
+		info.OpcMeta)
+}
+
+func (o *Object) setMetaData(
+	contentLength *int64,
+	contentMd5 *string,
+	contentType *string,
+	lastModified *common.SDKTime,
+	storageTier interface{},
+	meta map[string]string) error {
+
+	if contentLength != nil {
+		o.bytes = *contentLength
+	}
+	if contentMd5 != nil {
+		md5, err := o.base64ToMd5(*contentMd5)
+		if err == nil {
+			o.md5 = md5
+		}
+	}
+	o.meta = meta
+	if o.meta == nil {
+		o.meta = map[string]string{}
+	}
+	// Read MD5 from metadata if present
+	if md5sumBase64, ok := o.meta[metaMD5Hash]; ok {
+		md5, err := o.base64ToMd5(md5sumBase64)
+		if err != nil {
+			o.md5 = md5
+		}
+	}
+	if lastModified == nil {
+		o.lastModified = time.Now()
+		fs.Logf(o, "Failed to read last modified")
+	} else {
+		o.lastModified = lastModified.Time
+	}
+	if contentType != nil {
+		o.mimeType = *contentType
+	}
+	if storageTier == nil || storageTier == "" {
+		o.storageTier = storageTierMap[standard]
+	} else {
+		tier := strings.ToLower(fmt.Sprintf("%v", storageTier))
+		o.storageTier = storageTierMap[tier]
+	}
+	return nil
+}
+
+func (o *Object) base64ToMd5(md5sumBase64 string) (md5 string, err error) {
+	md5sumBytes, err := base64.StdEncoding.DecodeString(md5sumBase64)
+	if err != nil {
+		fs.Debugf(o, "Failed to read md5sum from metadata %q: %v", md5sumBase64, err)
+		return "", err
+	} else if len(md5sumBytes) != 16 {
+		fs.Debugf(o, "failed to read md5sum from metadata %q: wrong length", md5sumBase64)
+		return "", fmt.Errorf("failed to read md5sum from metadata %q: wrong length", md5sumBase64)
+	}
+	return hex.EncodeToString(md5sumBytes), nil
+}
+
+// Fs returns the parent Fs
+func (o *Object) Fs() fs.Info {
+	return o.fs
+}
+
+// Remote returns the remote path
+func (o *Object) Remote() string {
+	return o.remote
+}
+
+// Return a string version
+func (o *Object) String() string {
+	if o == nil {
+		return "<nil>"
+	}
+	return o.remote
+}
+
+// Size returns the size of an object in bytes
+func (o *Object) Size() int64 {
+	return o.bytes
+}
+
+// GetTier returns storage class as string
+func (o *Object) GetTier() string {
+	if o.storageTier == nil || *o.storageTier == "" {
+		return standard
+	}
+	return *o.storageTier
+}
+
+// SetTier performs changing storage class
+func (o *Object) SetTier(tier string) (err error) {
+	ctx := context.TODO()
+	tier = strings.ToLower(tier)
+	bucketName, bucketPath := o.split()
+	tierEnum, ok := objectstorage.GetMappingStorageTierEnum(tier)
+	if !ok {
+		return fmt.Errorf("not a valid storage tier %v ", tier)
+	}
+
+	req := objectstorage.UpdateObjectStorageTierRequest{
+		NamespaceName: common.String(o.fs.opt.Namespace),
+		BucketName:    common.String(bucketName),
+		UpdateObjectStorageTierDetails: objectstorage.UpdateObjectStorageTierDetails{
+			ObjectName:  common.String(bucketPath),
+			StorageTier: tierEnum,
+		},
+	}
+	_, err = o.fs.srv.UpdateObjectStorageTier(ctx, req)
+	if err != nil {
+		return err
+	}
+	o.storageTier = storageTierMap[tier]
+	return err
+}
+
+// MimeType of an Object if known, "" otherwise
+func (o *Object) MimeType(ctx context.Context) string {
+	err := o.readMetaData(ctx)
+	if err != nil {
+		fs.Logf(o, "Failed to read metadata: %v", err)
+		return ""
+	}
+	return o.mimeType
+}
+
+// Hash returns the MD5 of an object returning a lowercase hex string
+func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
+	if t != hash.MD5 {
+		return "", hash.ErrUnsupported
+	}
+	// Convert base64 encoded md5 into lower case hex
+	if o.md5 == "" {
+		err := o.readMetaData(ctx)
+		if err != nil {
+			return "", err
+		}
+	}
+	return o.md5, nil
+}
+
+// ModTime returns the modification time of the object
+//
+// It attempts to read the objects mtime and if that isn't present the
+// LastModified returned to the http headers
+func (o *Object) ModTime(ctx context.Context) (result time.Time) {
+	if o.fs.ci.UseServerModTime {
+		return o.lastModified
+	}
+	err := o.readMetaData(ctx)
+	if err != nil {
+		fs.Logf(o, "Failed to read metadata: %v", err)
+		return time.Now()
+	}
+	// read mtime out of metadata if available
+	d, ok := o.meta[metaMtime]
+	if !ok || d == "" {
+		return o.lastModified
+	}
+	modTime, err := swift.FloatStringToTime(d)
+	if err != nil {
+		fs.Logf(o, "Failed to read mtime from object: %v", err)
+		return o.lastModified
+	}
+	return modTime
+}
+
+// SetModTime sets the modification time of the local fs object
+func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
+	err := o.readMetaData(ctx)
+	if err != nil {
+		return err
+	}
+	o.meta[metaMtime] = swift.TimeToFloatString(modTime)
+	_, err = o.fs.Copy(ctx, o, o.remote)
+	return err
+}
+
+// Storable returns if this object is storable
+func (o *Object) Storable() bool {
+	return true
+}
+
+// Remove an object
+func (o *Object) Remove(ctx context.Context) error {
+	bucketName, bucketPath := o.split()
+	req := objectstorage.DeleteObjectRequest{
+		NamespaceName: common.String(o.fs.opt.Namespace),
+		BucketName:    common.String(bucketName),
+		ObjectName:    common.String(bucketPath),
+	}
+	err := o.fs.pacer.Call(func() (bool, error) {
+		resp, err := o.fs.srv.DeleteObject(ctx, req)
+		return shouldRetry(ctx, resp.HTTPResponse(), err)
+	})
+	return err
+}
+
+// Open object file
+func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) {
+	bucketName, bucketPath := o.split()
+	req := objectstorage.GetObjectRequest{
+		NamespaceName: common.String(o.fs.opt.Namespace),
+		BucketName:    common.String(bucketName),
+		ObjectName:    common.String(bucketPath),
+	}
+	o.applyGetObjectOptions(&req, options...)
+
+	var resp objectstorage.GetObjectResponse
+	err := o.fs.pacer.Call(func() (bool, error) {
+		var err error
+		resp, err = o.fs.srv.GetObject(ctx, req)
+		return shouldRetry(ctx, resp.HTTPResponse(), err)
+	})
+	if err != nil {
+		return nil, err
+	}
+	// read size from ContentLength or ContentRange
+	bytes := resp.ContentLength
+	if resp.ContentRange != nil {
+		var contentRange = *resp.ContentRange
+		slash := strings.IndexRune(contentRange, '/')
+		if slash >= 0 {
+			i, err := strconv.ParseInt(contentRange[slash+1:], 10, 64)
+			if err == nil {
+				bytes = &i
+			} else {
+				fs.Debugf(o, "Failed to find parse integer from in %q: %v", contentRange, err)
+			}
+		} else {
+			fs.Debugf(o, "Failed to find length in %q", contentRange)
+		}
+	}
+	err = o.decodeMetaDataObject(&resp)
+	if err != nil {
+		return nil, err
+	}
+	o.bytes = *bytes
+	return resp.HTTPResponse().Body, nil
+}
+
+// Update an object if it has changed
+func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
+	bucketName, bucketPath := o.split()
+	err = o.fs.makeBucket(ctx, bucketName)
+	if err != nil {
+		return err
+	}
+
+	// determine if we like upload single or multipart.
+	size := src.Size()
+	multipart := size >= int64(o.fs.opt.UploadCutoff)
+
+	// Set the mtime in the metadata
+	modTime := src.ModTime(ctx)
+	metadata := map[string]string{
+		metaMtime: swift.TimeToFloatString(modTime),
+	}
+
+	// read the md5sum if available
+	// - for non-multipart
+	//    - so we can add a ContentMD5
+	//    - so we can add the md5sum in the metadata as metaMD5Hash if using SSE/SSE-C
+	// - for multipart provided checksums aren't disabled
+	//    - so we can add the md5sum in the metadata as metaMD5Hash
+	var md5sumBase64 string
+	var md5sumHex string
+	if !multipart || !o.fs.opt.DisableChecksum {
+		md5sumHex, err = src.Hash(ctx, hash.MD5)
+		if err == nil && matchMd5.MatchString(md5sumHex) {
+			hashBytes, err := hex.DecodeString(md5sumHex)
+			if err == nil {
+				md5sumBase64 = base64.StdEncoding.EncodeToString(hashBytes)
+				if multipart && !o.fs.opt.DisableChecksum {
+					// Set the md5sum as metadata on the object if
+					// - a multipart upload
+					// - the ETag is not an MD5, e.g. when using SSE/SSE-C
+					// provided checksums aren't disabled
+					metadata[metaMD5Hash] = md5sumBase64
+				}
+			}
+		}
+	}
+	// Guess the content type
+	mimeType := fs.MimeType(ctx, src)
+
+	if multipart {
+		chunkSize := int64(o.fs.opt.ChunkSize)
+		uploadRequest := transfer.UploadRequest{
+			NamespaceName:                       common.String(o.fs.opt.Namespace),
+			BucketName:                          common.String(bucketName),
+			ObjectName:                          common.String(bucketPath),
+			ContentType:                         common.String(mimeType),
+			PartSize:                            common.Int64(chunkSize),
+			AllowMultipartUploads:               common.Bool(true),
+			AllowParrallelUploads:               common.Bool(true),
+			ObjectStorageClient:                 o.fs.srv,
+			EnableMultipartChecksumVerification: common.Bool(!o.fs.opt.DisableChecksum),
+			NumberOfGoroutines:                  common.Int(o.fs.opt.UploadConcurrency),
+			Metadata:                            metadataWithOpcPrefix(metadata),
+		}
+		if o.fs.opt.StorageTier != "" {
+			storageTier, ok := objectstorage.GetMappingPutObjectStorageTierEnum(o.fs.opt.StorageTier)
+			if !ok {
+				return fmt.Errorf("not a valid storage tier: %v", o.fs.opt.StorageTier)
+			}
+			uploadRequest.StorageTier = storageTier
+		}
+		o.applyMultiPutOptions(&uploadRequest, options...)
+		uploadStreamRequest := transfer.UploadStreamRequest{
+			UploadRequest: uploadRequest,
+			StreamReader:  in,
+		}
+		uploadMgr := transfer.NewUploadManager()
+		var uploadID = ""
+
+		defer atexit.OnError(&err, func() {
+			if uploadID == "" {
+				return
+			}
+			if o.fs.opt.LeavePartsOnError {
+				return
+			}
+			fs.Debugf(o, "Cancelling multipart upload")
+			errCancel := o.fs.abortMultiPartUpload(
+				context.Background(),
+				bucketName,
+				bucketPath,
+				uploadID)
+			if errCancel != nil {
+				fs.Debugf(o, "Failed to cancel multipart upload: %v", errCancel)
+			}
+		})()
+
+		err = o.fs.pacer.Call(func() (bool, error) {
+			uploadResponse, err := uploadMgr.UploadStream(ctx, uploadStreamRequest)
+			var httpResponse *http.Response
+			if err == nil {
+				if uploadResponse.Type == transfer.MultipartUpload {
+					if uploadResponse.MultipartUploadResponse != nil {
+						httpResponse = uploadResponse.MultipartUploadResponse.HTTPResponse()
+					}
+				} else {
+					if uploadResponse.SinglepartUploadResponse != nil {
+						httpResponse = uploadResponse.SinglepartUploadResponse.HTTPResponse()
+					}
+				}
+			}
+			if err != nil {
+				uploadID := ""
+				if uploadResponse.MultipartUploadResponse != nil && uploadResponse.MultipartUploadResponse.UploadID != nil {
+					uploadID = *uploadResponse.MultipartUploadResponse.UploadID
+					fs.Debugf(o, "multipart streaming upload failed, aborting uploadID: %v, may retry", uploadID)
+					_ = o.fs.abortMultiPartUpload(ctx, bucketName, bucketPath, uploadID)
+				}
+			}
+			return shouldRetry(ctx, httpResponse, err)
+		})
+		if err != nil {
+			fs.Errorf(o, "multipart streaming upload failed %v", err)
+			return err
+		}
+	} else {
+		req := objectstorage.PutObjectRequest{
+			NamespaceName: common.String(o.fs.opt.Namespace),
+			BucketName:    common.String(bucketName),
+			ObjectName:    common.String(bucketPath),
+			ContentType:   common.String(mimeType),
+			PutObjectBody: io.NopCloser(in),
+			OpcMeta:       metadata,
+		}
+		if size >= 0 {
+			req.ContentLength = common.Int64(size)
+		}
+		if o.fs.opt.StorageTier != "" {
+			storageTier, ok := objectstorage.GetMappingPutObjectStorageTierEnum(o.fs.opt.StorageTier)
+			if !ok {
+				return fmt.Errorf("not a valid storage tier: %v", o.fs.opt.StorageTier)
+			}
+			req.StorageTier = storageTier
+		}
+		o.applyPutOptions(&req, options...)
+		err = o.fs.pacer.Call(func() (bool, error) {
+			resp, err := o.fs.srv.PutObject(ctx, req)
+			return shouldRetry(ctx, resp.HTTPResponse(), err)
+		})
+		if err != nil {
+			fs.Errorf(o, "put object failed %v", err)
+			return err
+		}
+	}
+	// Read the metadata from the newly created object
+	o.meta = nil // wipe old metadata
+	return o.readMetaData(ctx)
+}
+
+func (o *Object) applyPutOptions(req *objectstorage.PutObjectRequest, options ...fs.OpenOption) {
+	// Apply upload options
+	for _, option := range options {
+		key, value := option.Header()
+		lowerKey := strings.ToLower(key)
+		switch lowerKey {
+		case "":
+			// ignore
+		case "cache-control":
+			req.CacheControl = common.String(value)
+		case "content-disposition":
+			req.ContentDisposition = common.String(value)
+		case "content-encoding":
+			req.ContentEncoding = common.String(value)
+		case "content-language":
+			req.ContentLanguage = common.String(value)
+		case "content-type":
+			req.ContentType = common.String(value)
+		default:
+			if strings.HasPrefix(lowerKey, ociMetaPrefix) {
+				req.OpcMeta[lowerKey] = value
+			} else {
+				fs.Errorf(o, "Don't know how to set key %q on upload", key)
+			}
+		}
+	}
+}
+
+func (o *Object) applyGetObjectOptions(req *objectstorage.GetObjectRequest, options ...fs.OpenOption) {
+	fs.FixRangeOption(options, o.bytes)
+	for _, option := range options {
+		switch option.(type) {
+		case *fs.RangeOption, *fs.SeekOption:
+			_, value := option.Header()
+			req.Range = &value
+		default:
+			if option.Mandatory() {
+				fs.Logf(o, "Unsupported mandatory option: %v", option)
+			}
+		}
+	}
+	// Apply upload options
+	for _, option := range options {
+		key, value := option.Header()
+		lowerKey := strings.ToLower(key)
+		switch lowerKey {
+		case "":
+			// ignore
+		case "cache-control":
+			req.HttpResponseCacheControl = common.String(value)
+		case "content-disposition":
+			req.HttpResponseContentDisposition = common.String(value)
+		case "content-encoding":
+			req.HttpResponseContentEncoding = common.String(value)
+		case "content-language":
+			req.HttpResponseContentLanguage = common.String(value)
+		case "content-type":
+			req.HttpResponseContentType = common.String(value)
+		case "range":
+			// do nothing
+		default:
+			fs.Errorf(o, "Don't know how to set key %q on upload", key)
+		}
+	}
+}
+
+func (o *Object) applyMultiPutOptions(req *transfer.UploadRequest, options ...fs.OpenOption) {
+	// Apply upload options
+	for _, option := range options {
+		key, value := option.Header()
+		lowerKey := strings.ToLower(key)
+		switch lowerKey {
+		case "":
+			// ignore
+		case "content-encoding":
+			req.ContentEncoding = common.String(value)
+		case "content-language":
+			req.ContentLanguage = common.String(value)
+		case "content-type":
+			req.ContentType = common.String(value)
+		default:
+			if strings.HasPrefix(lowerKey, ociMetaPrefix) {
+				req.Metadata[lowerKey] = value
+			} else {
+				fs.Errorf(o, "Don't know how to set key %q on upload", key)
+			}
+		}
+	}
+}
+
+func metadataWithOpcPrefix(src map[string]string) map[string]string {
+	dst := make(map[string]string)
+	for lowerKey, value := range src {
+		if !strings.HasPrefix(lowerKey, ociMetaPrefix) {
+			dst[ociMetaPrefix+lowerKey] = value
+		}
+	}
+	return dst
+}

backend/oracleobjectstorage/options.go

@@ -0,0 +1,242 @@
+//go:build !plan9 && !solaris && !js
+// +build !plan9,!solaris,!js
+
+package oracleobjectstorage
+
+import (
+	"time"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config"
+	"github.com/rclone/rclone/lib/encoder"
+)
+
+const (
+	maxSizeForCopy             = 4768 * 1024 * 1024
+	minChunkSize               = fs.SizeSuffix(1024 * 1024 * 5)
+	defaultUploadCutoff        = fs.SizeSuffix(200 * 1024 * 1024)
+	defaultUploadConcurrency   = 10
+	maxUploadCutoff            = fs.SizeSuffix(5 * 1024 * 1024 * 1024)
+	minSleep                   = 100 * time.Millisecond
+	maxSleep                   = 5 * time.Minute
+	decayConstant              = 1 // bigger for slower decay, exponential
+	defaultCopyTimeoutDuration = fs.Duration(time.Minute)
+)
+
+const (
+	userPrincipal     = "user_principal_auth"
+	instancePrincipal = "instance_principal_auth"
+	resourcePrincipal = "resource_principal_auth"
+	environmentAuth   = "env_auth"
+	noAuth            = "no_auth"
+
+	userPrincipalHelpText = `use an OCI user and an API key for authentication.
+you’ll need to put in a config file your tenancy OCID, user OCID, region, the path, fingerprint to an API key.
+https://docs.oracle.com/en-us/iaas/Content/API/Concepts/sdkconfig.htm`
+
+	instancePrincipalHelpText = `use instance principals to authorize an instance to make API calls. 
+each instance has its own identity, and authenticates using the certificates that are read from instance metadata. 
+https://docs.oracle.com/en-us/iaas/Content/Identity/Tasks/callingservicesfrominstances.htm`
+
+	resourcePrincipalHelpText = `use resource principals to make API calls`
+
+	environmentAuthHelpText = `automatically pickup the credentials from runtime(env), first one to provide auth wins`
+
+	noAuthHelpText = `no credentials needed, this is typically for reading public buckets`
+)
+
+// Options defines the configuration for this backend
+type Options struct {
+	Provider          string               `config:"provider"`
+	Compartment       string               `config:"compartment"`
+	Namespace         string               `config:"namespace"`
+	Region            string               `config:"region"`
+	Endpoint          string               `config:"endpoint"`
+	Enc               encoder.MultiEncoder `config:"encoding"`
+	ConfigFile        string               `config:"config_file"`
+	ConfigProfile     string               `config:"config_profile"`
+	UploadCutoff      fs.SizeSuffix        `config:"upload_cutoff"`
+	ChunkSize         fs.SizeSuffix        `config:"chunk_size"`
+	UploadConcurrency int                  `config:"upload_concurrency"`
+	DisableChecksum   bool                 `config:"disable_checksum"`
+	CopyCutoff        fs.SizeSuffix        `config:"copy_cutoff"`
+	CopyTimeout       fs.Duration          `config:"copy_timeout"`
+	StorageTier       string               `config:"storage_tier"`
+	LeavePartsOnError bool                 `config:"leave_parts_on_error"`
+	NoCheckBucket     bool                 `config:"no_check_bucket"`
+}
+
+func newOptions() []fs.Option {
+	return []fs.Option{{
+		Name:     fs.ConfigProvider,
+		Help:     "Choose your Auth Provider",
+		Required: true,
+		Default:  environmentAuth,
+		Examples: []fs.OptionExample{{
+			Value: environmentAuth,
+			Help:  environmentAuthHelpText,
+		}, {
+			Value: userPrincipal,
+			Help:  userPrincipalHelpText,
+		}, {
+			Value: instancePrincipal,
+			Help:  instancePrincipalHelpText,
+		}, {
+			Value: resourcePrincipal,
+			Help:  resourcePrincipalHelpText,
+		}, {
+			Value: noAuth,
+			Help:  noAuthHelpText,
+		}},
+	}, {
+		Name:     "namespace",
+		Help:     "Object storage namespace",
+		Required: true,
+	}, {
+		Name:     "compartment",
+		Help:     "Object storage compartment OCID",
+		Provider: "!no_auth",
+		Required: true,
+	}, {
+		Name:     "region",
+		Help:     "Object storage Region",
+		Required: true,
+	}, {
+		Name:     "endpoint",
+		Help:     "Endpoint for Object storage API.\n\nLeave blank to use the default endpoint for the region.",
+		Required: false,
+	}, {
+		Name:     "config_file",
+		Help:     "Path to OCI config file",
+		Provider: userPrincipal,
+		Default:  "~/.oci/config",
+		Examples: []fs.OptionExample{{
+			Value: "~/.oci/config",
+			Help:  "oci configuration file location",
+		}},
+	}, {
+		Name:     "config_profile",
+		Help:     "Profile name inside the oci config file",
+		Provider: userPrincipal,
+		Default:  "Default",
+		Examples: []fs.OptionExample{{
+			Value: "Default",
+			Help:  "Use the default profile",
+		}},
+	}, {
+		Name: "upload_cutoff",
+		Help: `Cutoff for switching to chunked upload.
+
+Any files larger than this will be uploaded in chunks of chunk_size.
+The minimum is 0 and the maximum is 5 GiB.`,
+		Default:  defaultUploadCutoff,
+		Advanced: true,
+	}, {
+		Name: "chunk_size",
+		Help: `Chunk size to use for uploading.
+
+When uploading files larger than upload_cutoff or files with unknown
+size (e.g. from "rclone rcat" or uploaded with "rclone mount" or google
+photos or google docs) they will be uploaded as multipart uploads
+using this chunk size.
+
+Note that "upload_concurrency" chunks of this size are buffered
+in memory per transfer.
+
+If you are transferring large files over high-speed links and you have
+enough memory, then increasing this will speed up the transfers.
+
+Rclone will automatically increase the chunk size when uploading a
+large file of known size to stay below the 10,000 chunks limit.
+
+Files of unknown size are uploaded with the configured
+chunk_size. Since the default chunk size is 5 MiB and there can be at
+most 10,000 chunks, this means that by default the maximum size of
+a file you can stream upload is 48 GiB.  If you wish to stream upload
+larger files then you will need to increase chunk_size.
+
+Increasing the chunk size decreases the accuracy of the progress
+statistics displayed with "-P" flag.
+`,
+		Default:  minChunkSize,
+		Advanced: true,
+	}, {
+		Name: "upload_concurrency",
+		Help: `Concurrency for multipart uploads.
+
+This is the number of chunks of the same file that are uploaded
+concurrently.
+
+If you are uploading small numbers of large files over high-speed links
+and these uploads do not fully utilize your bandwidth, then increasing
+this may help to speed up the transfers.`,
+		Default:  defaultUploadConcurrency,
+		Advanced: true,
+	}, {
+		Name: "copy_cutoff",
+		Help: `Cutoff for switching to multipart copy.
+
+Any files larger than this that need to be server-side copied will be
+copied in chunks of this size.
+
+The minimum is 0 and the maximum is 5 GiB.`,
+		Default:  fs.SizeSuffix(maxSizeForCopy),
+		Advanced: true,
+	}, {
+		Name: "copy_timeout",
+		Help: `Timeout for copy.
+
+Copy is an asynchronous operation, specify timeout to wait for copy to succeed
+`,
+		Default:  defaultCopyTimeoutDuration,
+		Advanced: true,
+	}, {
+		Name: "disable_checksum",
+		Help: `Don't store MD5 checksum with object metadata.
+
+Normally rclone will calculate the MD5 checksum of the input before
+uploading it so it can add it to metadata on the object. This is great
+for data integrity checking but can cause long delays for large files
+to start uploading.`,
+		Default:  false,
+		Advanced: true,
+	}, {
+		Name:     config.ConfigEncoding,
+		Help:     config.ConfigEncodingHelp,
+		Advanced: true,
+		// Any UTF-8 character is valid in a key, however it can't handle
+		// invalid UTF-8 and / have a special meaning.
+		//
+		// The SDK can't seem to handle uploading files called '.
+		// - initial / encoding
+		// - doubled / encoding
+		// - trailing / encoding
+		// so that OSS keys are always valid file names
+		Default: encoder.EncodeInvalidUtf8 |
+			encoder.EncodeSlash |
+			encoder.EncodeDot,
+	}, {
+		Name: "leave_parts_on_error",
+		Help: `If true avoid calling abort upload on a failure, leaving all successfully uploaded parts on S3 for manual recovery.
+
+It should be set to true for resuming uploads across different sessions.
+
+WARNING: Storing parts of an incomplete multipart upload counts towards space usage on object storage and will add
+additional costs if not cleaned up.
+`,
+		Default:  false,
+		Advanced: true,
+	}, {
+		Name: "no_check_bucket",
+		Help: `If set, don't attempt to check the bucket exists or create it.
+
+This can be useful when trying to minimise the number of transactions
+rclone does if you know the bucket exists already.
+
+It can also be needed if the user you are using does not have bucket
+creation permissions.
+`,
+		Default:  false,
+		Advanced: true,
+	}}
+}

backend/oracleobjectstorage/oracleobjectstorage.go

@@ -0,0 +1,695 @@
+//go:build !plan9 && !solaris && !js
+// +build !plan9,!solaris,!js
+
+// Package oracleobjectstorage provides an interface to the OCI object storage system.
+package oracleobjectstorage
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"path"
+	"strings"
+	"time"
+
+	"github.com/oracle/oci-go-sdk/v65/common"
+	"github.com/oracle/oci-go-sdk/v65/objectstorage"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config/configmap"
+	"github.com/rclone/rclone/fs/config/configstruct"
+	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/fs/operations"
+	"github.com/rclone/rclone/fs/walk"
+	"github.com/rclone/rclone/lib/bucket"
+	"github.com/rclone/rclone/lib/pacer"
+)
+
+// Register with Fs
+func init() {
+	fs.Register(&fs.RegInfo{
+		Name:        "oracleobjectstorage",
+		Description: "Oracle Cloud Infrastructure Object Storage",
+		Prefix:      "oos",
+		NewFs:       NewFs,
+		CommandHelp: commandHelp,
+		Options:     newOptions(),
+	})
+}
+
+// Fs represents a remote object storage server
+type Fs struct {
+	name          string                             // name of this remote
+	root          string                             // the path we are working on if any
+	opt           Options                            // parsed config options
+	ci            *fs.ConfigInfo                     // global config
+	features      *fs.Features                       // optional features
+	srv           *objectstorage.ObjectStorageClient // the connection to the object storage
+	rootBucket    string                             // bucket part of root (if any)
+	rootDirectory string                             // directory part of root (if any)
+	cache         *bucket.Cache                      // cache for bucket creation status
+	pacer         *fs.Pacer                          // To pace the API calls
+}
+
+// NewFs Initialize backend
+func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
+	// Parse config into Options struct
+	opt := new(Options)
+	err := configstruct.Set(m, opt)
+	if err != nil {
+		return nil, err
+	}
+	ci := fs.GetConfig(ctx)
+	objectStorageClient, err := newObjectStorageClient(ctx, opt)
+	if err != nil {
+		return nil, err
+	}
+	p := pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))
+	f := &Fs{
+		name:  name,
+		opt:   *opt,
+		ci:    ci,
+		srv:   objectStorageClient,
+		cache: bucket.NewCache(),
+		pacer: fs.NewPacer(ctx, p),
+	}
+	f.setRoot(root)
+	f.features = (&fs.Features{
+		ReadMimeType:      true,
+		WriteMimeType:     true,
+		BucketBased:       true,
+		BucketBasedRootOK: true,
+		SetTier:           true,
+		GetTier:           true,
+		SlowModTime:       true,
+	}).Fill(ctx, f)
+	if f.rootBucket != "" && f.rootDirectory != "" && !strings.HasSuffix(root, "/") {
+		// Check to see if the (bucket,directory) is actually an existing file
+		oldRoot := f.root
+		newRoot, leaf := path.Split(oldRoot)
+		f.setRoot(newRoot)
+		_, err := f.NewObject(ctx, leaf)
+		if err != nil {
+			// File doesn't exist or is a directory so return old f
+			f.setRoot(oldRoot)
+			return f, nil
+		}
+		// return an error with fs which points to the parent
+		return f, fs.ErrorIsFile
+	}
+	return f, err
+}
+
+func checkUploadChunkSize(cs fs.SizeSuffix) error {
+	if cs < minChunkSize {
+		return fmt.Errorf("%s is less than %s", cs, minChunkSize)
+	}
+	return nil
+}
+
+func (f *Fs) setUploadChunkSize(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
+	err = checkUploadChunkSize(cs)
+	if err == nil {
+		old, f.opt.ChunkSize = f.opt.ChunkSize, cs
+	}
+	return
+}
+
+func checkUploadCutoff(cs fs.SizeSuffix) error {
+	if cs > maxUploadCutoff {
+		return fmt.Errorf("%s is greater than %s", cs, maxUploadCutoff)
+	}
+	return nil
+}
+
+func (f *Fs) setUploadCutoff(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
+	err = checkUploadCutoff(cs)
+	if err == nil {
+		old, f.opt.UploadCutoff = f.opt.UploadCutoff, cs
+	}
+	return
+}
+
+// ------------------------------------------------------------
+// Implement backed that represents a remote object storage server
+// Fs is the interface a cloud storage system must provide
+// ------------------------------------------------------------
+
+// Name of the remote (as passed into NewFs)
+func (f *Fs) Name() string {
+	return f.name
+}
+
+// Root of the remote (as passed into NewFs)
+func (f *Fs) Root() string {
+	return f.root
+}
+
+// String converts this Fs to a string
+func (f *Fs) String() string {
+	if f.rootBucket == "" {
+		return "oos:root"
+	}
+	if f.rootDirectory == "" {
+		return fmt.Sprintf("oos:bucket %s", f.rootBucket)
+	}
+	return fmt.Sprintf("oos:bucket %s, path %s", f.rootBucket, f.rootDirectory)
+}
+
+// Features returns the optional features of this Fs
+func (f *Fs) Features() *fs.Features {
+	return f.features
+}
+
+// Precision of the remote
+func (f *Fs) Precision() time.Duration {
+	return time.Millisecond
+}
+
+// Hashes returns the supported hash sets.
+func (f *Fs) Hashes() hash.Set {
+	return hash.Set(hash.MD5)
+}
+
+// setRoot changes the root of the Fs
+func (f *Fs) setRoot(root string) {
+	f.root = parsePath(root)
+	f.rootBucket, f.rootDirectory = bucket.Split(f.root)
+}
+
+// parsePath parses a remote 'url'
+func parsePath(path string) (root string) {
+	root = strings.Trim(path, "/")
+	return
+}
+
+// split returns bucket and bucketPath from the rootRelativePath
+// relative to f.root
+func (f *Fs) split(rootRelativePath string) (bucketName, bucketPath string) {
+	bucketName, bucketPath = bucket.Split(path.Join(f.root, rootRelativePath))
+	return f.opt.Enc.FromStandardName(bucketName), f.opt.Enc.FromStandardPath(bucketPath)
+}
+
+// List the objects and directories in dir into entries.  The
+// entries can be returned in any order but should be for a
+// complete directory.
+//
+// dir should be "" to list the root, and should not have
+// trailing slashes.
+//
+// This should return ErrDirNotFound if the directory isn't
+// found.
+func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
+	bucketName, directory := f.split(dir)
+	fs.Debugf(f, "listing: bucket : %v, directory: %v", bucketName, dir)
+	if bucketName == "" {
+		if directory != "" {
+			return nil, fs.ErrorListBucketRequired
+		}
+		return f.listBuckets(ctx)
+	}
+	return f.listDir(ctx, bucketName, directory, f.rootDirectory, f.rootBucket == "")
+}
+
+// listFn is called from list to handle an object.
+type listFn func(remote string, object *objectstorage.ObjectSummary, isDirectory bool) error
+
+// list the objects into the function supplied from
+// the bucket and root supplied
+// (bucket, directory) is the starting directory
+// If prefix is set then it is removed from all file names
+// If addBucket is set then it adds the bucket to the start of the remotes generated
+// If recurse is set the function will recursively list
+// If limit is > 0 then it limits to that many files (must be less than 1000)
+// If hidden is set then it will list the hidden (deleted) files too.
+// if findFile is set it will look for files called (bucket, directory)
+func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBucket bool, recurse bool, limit int,
+	fn listFn) (err error) {
+	if prefix != "" {
+		prefix += "/"
+	}
+	if directory != "" {
+		directory += "/"
+	}
+
+	delimiter := ""
+	if !recurse {
+		delimiter = "/"
+	}
+	chunkSize := 1000
+	if limit > 0 {
+		chunkSize = limit
+	}
+	var request = objectstorage.ListObjectsRequest{
+		NamespaceName: common.String(f.opt.Namespace),
+		BucketName:    common.String(bucket),
+		Prefix:        common.String(directory),
+		Limit:         common.Int(chunkSize),
+		Fields:        common.String("name,size,etag,timeCreated,md5,timeModified,storageTier,archivalState"),
+	}
+	if delimiter != "" {
+		request.Delimiter = common.String(delimiter)
+	}
+
+	for {
+		var resp objectstorage.ListObjectsResponse
+		err = f.pacer.Call(func() (bool, error) {
+			var err error
+			resp, err = f.srv.ListObjects(ctx, request)
+			return shouldRetry(ctx, resp.HTTPResponse(), err)
+		})
+		if err != nil {
+			if ociError, ok := err.(common.ServiceError); ok {
+				// If it is a timeout then we want to retry that
+				if ociError.GetHTTPStatusCode() == http.StatusNotFound {
+					err = fs.ErrorDirNotFound
+				}
+			}
+			if f.rootBucket == "" {
+				// if listing from the root ignore wrong region requests returning
+				// empty directory
+				if reqErr, ok := err.(common.ServiceError); ok {
+					// 301 if wrong region for bucket
+					if reqErr.GetHTTPStatusCode() == http.StatusMovedPermanently {
+						fs.Errorf(f, "Can't change region for bucket %q with no bucket specified", bucket)
+						return nil
+					}
+				}
+			}
+			return err
+		}
+		if !recurse {
+			for _, commonPrefix := range resp.ListObjects.Prefixes {
+				if commonPrefix == "" {
+					fs.Logf(f, "Nil common prefix received")
+					continue
+				}
+				remote := commonPrefix
+				remote = f.opt.Enc.ToStandardPath(remote)
+				if !strings.HasPrefix(remote, prefix) {
+					fs.Logf(f, "Odd name received %q", remote)
+					continue
+				}
+				remote = remote[len(prefix):]
+				if addBucket {
+					remote = path.Join(bucket, remote)
+				}
+				remote = strings.TrimSuffix(remote, "/")
+				err = fn(remote, &objectstorage.ObjectSummary{Name: &remote}, true)
+				if err != nil {
+					return err
+				}
+			}
+		}
+		for i := range resp.Objects {
+			object := &resp.Objects[i]
+			// Finish if file name no longer has prefix
+			//if prefix != "" && !strings.HasPrefix(file.Name, prefix) {
+			//	return nil
+			//}
+			remote := *object.Name
+			remote = f.opt.Enc.ToStandardPath(remote)
+			if !strings.HasPrefix(remote, prefix) {
+				// fs.Debugf(f, "Odd name received %v", object.Name)
+				continue
+			}
+			remote = remote[len(prefix):]
+			// Check for directory
+			isDirectory := remote == "" || strings.HasSuffix(remote, "/")
+			if addBucket {
+				remote = path.Join(bucket, remote)
+			}
+			// is this a directory marker?
+			if isDirectory && object.Size != nil && *object.Size == 0 {
+				continue // skip directory marker
+			}
+			if isDirectory && len(remote) > 1 {
+				remote = remote[:len(remote)-1]
+			}
+			err = fn(remote, object, isDirectory)
+			if err != nil {
+				return err
+			}
+		}
+		// end if no NextFileName
+		if resp.NextStartWith == nil {
+			break
+		}
+		request.Start = resp.NextStartWith
+	}
+	return nil
+}
+
+// Convert a list item into a DirEntry
+func (f *Fs) itemToDirEntry(ctx context.Context, remote string, object *objectstorage.ObjectSummary, isDirectory bool) (fs.DirEntry, error) {
+	if isDirectory {
+		size := int64(0)
+		if object.Size != nil {
+			size = *object.Size
+		}
+		d := fs.NewDir(remote, time.Time{}).SetSize(size)
+		return d, nil
+	}
+	o, err := f.newObjectWithInfo(ctx, remote, object)
+	if err != nil {
+		return nil, err
+	}
+	return o, nil
+}
+
+// listDir lists a single directory
+func (f *Fs) listDir(ctx context.Context, bucket, directory, prefix string, addBucket bool) (entries fs.DirEntries, err error) {
+	fn := func(remote string, object *objectstorage.ObjectSummary, isDirectory bool) error {
+		entry, err := f.itemToDirEntry(ctx, remote, object, isDirectory)
+		if err != nil {
+			return err
+		}
+		if entry != nil {
+			entries = append(entries, entry)
+		}
+		return nil
+	}
+	err = f.list(ctx, bucket, directory, prefix, addBucket, false, 0, fn)
+	if err != nil {
+		return nil, err
+	}
+	// bucket must be present if listing succeeded
+	f.cache.MarkOK(bucket)
+	return entries, nil
+}
+
+// listBuckets returns all the buckets to out
+func (f *Fs) listBuckets(ctx context.Context) (entries fs.DirEntries, err error) {
+	if f.opt.Provider == noAuth {
+		return nil, fmt.Errorf("can't list buckets with %v provider, use a valid auth provider in config file", noAuth)
+	}
+	var request = objectstorage.ListBucketsRequest{
+		NamespaceName: common.String(f.opt.Namespace),
+		CompartmentId: common.String(f.opt.Compartment),
+	}
+	var resp objectstorage.ListBucketsResponse
+	for {
+		err = f.pacer.Call(func() (bool, error) {
+			resp, err = f.srv.ListBuckets(ctx, request)
+			return shouldRetry(ctx, resp.HTTPResponse(), err)
+		})
+		if err != nil {
+			return nil, err
+		}
+		for _, item := range resp.Items {
+			bucketName := f.opt.Enc.ToStandardName(*item.Name)
+			f.cache.MarkOK(bucketName)
+			d := fs.NewDir(bucketName, item.TimeCreated.Time)
+			entries = append(entries, d)
+		}
+		if resp.OpcNextPage == nil {
+			break
+		}
+		request.Page = resp.OpcNextPage
+	}
+	return entries, nil
+}
+
+// Return an Object from a path
+// If it can't be found it returns the error fs.ErrorObjectNotFound.
+func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *objectstorage.ObjectSummary) (fs.Object, error) {
+	o := &Object{
+		fs:     f,
+		remote: remote,
+	}
+	if info != nil {
+		// Set info but not meta
+		if info.TimeModified == nil {
+			fs.Logf(o, "Failed to read last modified")
+			o.lastModified = time.Now()
+		} else {
+			o.lastModified = info.TimeModified.Time
+		}
+		if info.Md5 != nil {
+			md5, err := o.base64ToMd5(*info.Md5)
+			if err != nil {
+				o.md5 = md5
+			}
+		}
+		o.bytes = *info.Size
+		o.storageTier = storageTierMap[strings.ToLower(string(info.StorageTier))]
+	} else {
+		err := o.readMetaData(ctx) // reads info and headers, returning an error
+		if err != nil {
+			return nil, err
+		}
+	}
+	return o, nil
+}
+
+// NewObject finds the Object at remote.  If it can't be found
+// it returns the error fs.ErrorObjectNotFound.
+func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
+	return f.newObjectWithInfo(ctx, remote, nil)
+}
+
+// Put the object into the bucket
+// Copy the reader in to the new object which is returned
+// The new object may have been created if an error is returned
+func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+	// Temporary Object under construction
+	o := &Object{
+		fs:     f,
+		remote: src.Remote(),
+	}
+	return o, o.Update(ctx, in, src, options...)
+}
+
+// PutStream uploads to the remote path with the modTime given of indeterminate size
+func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+	return f.Put(ctx, in, src, options...)
+}
+
+// Mkdir creates the bucket if it doesn't exist
+func (f *Fs) Mkdir(ctx context.Context, dir string) error {
+	bucketName, _ := f.split(dir)
+	return f.makeBucket(ctx, bucketName)
+}
+
+// makeBucket creates the bucket if it doesn't exist
+func (f *Fs) makeBucket(ctx context.Context, bucketName string) error {
+	if f.opt.NoCheckBucket {
+		return nil
+	}
+	return f.cache.Create(bucketName, func() error {
+		details := objectstorage.CreateBucketDetails{
+			Name:             common.String(bucketName),
+			CompartmentId:    common.String(f.opt.Compartment),
+			PublicAccessType: objectstorage.CreateBucketDetailsPublicAccessTypeNopublicaccess,
+		}
+		req := objectstorage.CreateBucketRequest{
+			NamespaceName:       common.String(f.opt.Namespace),
+			CreateBucketDetails: details,
+		}
+		err := f.pacer.Call(func() (bool, error) {
+			resp, err := f.srv.CreateBucket(ctx, req)
+			return shouldRetry(ctx, resp.HTTPResponse(), err)
+		})
+		if err == nil {
+			fs.Infof(f, "Bucket %q created with accessType %q", bucketName,
+				objectstorage.CreateBucketDetailsPublicAccessTypeNopublicaccess)
+		}
+		if svcErr, ok := err.(common.ServiceError); ok {
+			if code := svcErr.GetCode(); code == "BucketAlreadyOwnedByYou" || code == "BucketAlreadyExists" {
+				err = nil
+			}
+		}
+		return err
+	}, func() (bool, error) {
+		return f.bucketExists(ctx, bucketName)
+	})
+}
+
+// Check if the bucket exists
+//
+// NB this can return incorrect results if called immediately after bucket deletion
+func (f *Fs) bucketExists(ctx context.Context, bucketName string) (bool, error) {
+	req := objectstorage.HeadBucketRequest{
+		NamespaceName: common.String(f.opt.Namespace),
+		BucketName:    common.String(bucketName),
+	}
+	err := f.pacer.Call(func() (bool, error) {
+		resp, err := f.srv.HeadBucket(ctx, req)
+		return shouldRetry(ctx, resp.HTTPResponse(), err)
+	})
+	if err == nil {
+		return true, nil
+	}
+	if err, ok := err.(common.ServiceError); ok {
+		if err.GetHTTPStatusCode() == http.StatusNotFound {
+			return false, nil
+		}
+	}
+	return false, err
+}
+
+// Rmdir delete an empty bucket. if bucket is not empty this is will fail with appropriate error
+func (f *Fs) Rmdir(ctx context.Context, dir string) error {
+	bucketName, directory := f.split(dir)
+	if bucketName == "" || directory != "" {
+		return nil
+	}
+	return f.cache.Remove(bucketName, func() error {
+		req := objectstorage.DeleteBucketRequest{
+			NamespaceName: common.String(f.opt.Namespace),
+			BucketName:    common.String(bucketName),
+		}
+		err := f.pacer.Call(func() (bool, error) {
+			resp, err := f.srv.DeleteBucket(ctx, req)
+			return shouldRetry(ctx, resp.HTTPResponse(), err)
+		})
+		if err == nil {
+			fs.Infof(f, "Bucket %q deleted", bucketName)
+		}
+		return err
+	})
+}
+
+func (f *Fs) abortMultiPartUpload(ctx context.Context, bucketName, bucketPath, uploadID string) (err error) {
+	if uploadID == "" {
+		return nil
+	}
+	request := objectstorage.AbortMultipartUploadRequest{
+		NamespaceName: common.String(f.opt.Namespace),
+		BucketName:    common.String(bucketName),
+		ObjectName:    common.String(bucketPath),
+		UploadId:      common.String(uploadID),
+	}
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err := f.srv.AbortMultipartUpload(ctx, request)
+		return shouldRetry(ctx, resp.HTTPResponse(), err)
+	})
+	return err
+}
+
+// cleanUpBucket removes all pending multipart uploads for a given bucket over the age of maxAge
+func (f *Fs) cleanUpBucket(ctx context.Context, bucket string, maxAge time.Duration,
+	uploads []*objectstorage.MultipartUpload) (err error) {
+	fs.Infof(f, "cleaning bucket %q of pending multipart uploads older than %v", bucket, maxAge)
+	for _, upload := range uploads {
+		if upload.TimeCreated != nil && upload.Object != nil && upload.UploadId != nil {
+			age := time.Since(upload.TimeCreated.Time)
+			what := fmt.Sprintf("pending multipart upload for bucket %q key %q dated %v (%v ago)", bucket, *upload.Object,
+				upload.TimeCreated, age)
+			if age > maxAge {
+				fs.Infof(f, "removing %s", what)
+				if operations.SkipDestructive(ctx, what, "remove pending upload") {
+					continue
+				}
+				ignoreErr := f.abortMultiPartUpload(ctx, *upload.Bucket, *upload.Object, *upload.UploadId)
+				if ignoreErr != nil {
+					// fs.Debugf(f, "ignoring error %s", ignoreErr)
+				}
+			} else {
+				// fs.Debugf(f, "ignoring %s", what)
+			}
+		} else {
+			fs.Infof(f, "MultipartUpload doesn't have sufficient details to abort.")
+		}
+	}
+	return err
+}
+
+// CleanUp removes all pending multipart uploads
+func (f *Fs) cleanUp(ctx context.Context, maxAge time.Duration) (err error) {
+	uploadsMap, err := f.listMultipartUploadsAll(ctx)
+	if err != nil {
+		return err
+	}
+	for bucketName, uploads := range uploadsMap {
+		cleanErr := f.cleanUpBucket(ctx, bucketName, maxAge, uploads)
+		if err != nil {
+			fs.Errorf(f, "Failed to cleanup bucket %q: %v", bucketName, cleanErr)
+			err = cleanErr
+		}
+	}
+	return err
+}
+
+// CleanUp removes all pending multipart uploads older than 24 hours
+func (f *Fs) CleanUp(ctx context.Context) (err error) {
+	return f.cleanUp(ctx, 24*time.Hour)
+}
+
+// ------------------------------------------------------------
+// Implement ListRer is an optional interfaces for Fs
+//------------------------------------------------------------
+
+/*
+ListR lists the objects and directories of the Fs starting
+from dir recursively into out.
+
+dir should be "" to start from the root, and should not
+have trailing slashes.
+
+This should return ErrDirNotFound if the directory isn't
+found.
+
+It should call callback for each tranche of entries read.
+These need not be returned in any particular order.  If
+callback returns an error then the listing will stop
+immediately.
+
+Don't implement this unless you have a more efficient way
+of listing recursively that doing a directory traversal.
+*/
+func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {
+	bucketName, directory := f.split(dir)
+	list := walk.NewListRHelper(callback)
+	listR := func(bucket, directory, prefix string, addBucket bool) error {
+		return f.list(ctx, bucket, directory, prefix, addBucket, true, 0, func(remote string, object *objectstorage.ObjectSummary, isDirectory bool) error {
+			entry, err := f.itemToDirEntry(ctx, remote, object, isDirectory)
+			if err != nil {
+				return err
+			}
+			return list.Add(entry)
+		})
+	}
+	if bucketName == "" {
+		entries, err := f.listBuckets(ctx)
+		if err != nil {
+			return err
+		}
+		for _, entry := range entries {
+			err = list.Add(entry)
+			if err != nil {
+				return err
+			}
+			bucketName := entry.Remote()
+			err = listR(bucketName, "", f.rootDirectory, true)
+			if err != nil {
+				return err
+			}
+			// bucket must be present if listing succeeded
+			f.cache.MarkOK(bucketName)
+		}
+	} else {
+		err = listR(bucketName, directory, f.rootDirectory, f.rootBucket == "")
+		if err != nil {
+			return err
+		}
+		// bucket must be present if listing succeeded
+		f.cache.MarkOK(bucketName)
+	}
+	return list.Flush()
+}
+
+// Check the interfaces are satisfied
+var (
+	_ fs.Fs          = &Fs{}
+	_ fs.Copier      = &Fs{}
+	_ fs.PutStreamer = &Fs{}
+	_ fs.ListRer     = &Fs{}
+	_ fs.Commander   = &Fs{}
+	_ fs.CleanUpper  = &Fs{}
+
+	_ fs.Object    = &Object{}
+	_ fs.MimeTyper = &Object{}
+	_ fs.GetTierer = &Object{}
+	_ fs.SetTierer = &Object{}
+)

backend/oracleobjectstorage/oracleobjectstorage_test.go

@@ -0,0 +1,33 @@
+//go:build !plan9 && !solaris && !js
+// +build !plan9,!solaris,!js
+
+package oracleobjectstorage
+
+import (
+	"testing"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fstest/fstests"
+)
+
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+	fstests.Run(t, &fstests.Opt{
+		RemoteName:  "TestOracleObjectStorage:",
+		TiersToTest: []string{"standard", "archive"},
+		NilObject:   (*Object)(nil),
+		ChunkedUpload: fstests.ChunkedUploadConfig{
+			MinChunkSize: minChunkSize,
+		},
+	})
+}
+
+func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
+	return f.setUploadChunkSize(cs)
+}
+
+func (f *Fs) SetUploadCutoff(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
+	return f.setUploadCutoff(cs)
+}
+
+var _ fstests.SetUploadChunkSizer = (*Fs)(nil)

backend/oracleobjectstorage/oracleobjectstorage_unsupported.go

@@ -0,0 +1,7 @@
+// Build for oracleobjectstorage for unsupported platforms to stop go complaining
+// about "no buildable Go source files "
+
+//go:build plan9 || solaris || js
+// +build plan9 solaris js
+
+package oracleobjectstorage

backend/oracleobjectstorage/waiter.go

@@ -0,0 +1,362 @@
+//go:build !plan9 && !solaris && !js
+// +build !plan9,!solaris,!js
+
+package oracleobjectstorage
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/rclone/rclone/fs"
+)
+
+var refreshGracePeriod = 30 * time.Second
+
+// StateRefreshFunc is a function type used for StateChangeConf that is
+// responsible for refreshing the item being watched for a state change.
+//
+// It returns three results. `result` is any object that will be returned
+// as the final object after waiting for state change. This allows you to
+// return the final updated object, for example an EC2 instance after refreshing
+// it. A nil result represents not found.
+//
+// `state` is the latest state of that object. And `err` is any error that
+// may have happened while refreshing the state.
+type StateRefreshFunc func() (result interface{}, state string, err error)
+
+// StateChangeConf is the configuration struct used for `WaitForState`.
+type StateChangeConf struct {
+	Delay          time.Duration    // Wait this time before starting checks
+	Pending        []string         // States that are "allowed" and will continue trying
+	Refresh        StateRefreshFunc // Refreshes the current state
+	Target         []string         // Target state
+	Timeout        time.Duration    // The amount of time to wait before timeout
+	MinTimeout     time.Duration    // Smallest time to wait before refreshes
+	PollInterval   time.Duration    // Override MinTimeout/backoff and only poll this often
+	NotFoundChecks int              // Number of times to allow not found (nil result from Refresh)
+
+	// This is to work around inconsistent APIs
+	ContinuousTargetOccurrence int // Number of times the Target state has to occur continuously
+}
+
+// WaitForStateContext watches an object and waits for it to achieve the state
+// specified in the configuration using the specified Refresh() func,
+// waiting the number of seconds specified in the timeout configuration.
+//
+// If the Refresh function returns an error, exit immediately with that error.
+//
+// If the Refresh function returns a state other than the Target state or one
+// listed in Pending, return immediately with an error.
+//
+// If the Timeout is exceeded before reaching the Target state, return an
+// error.
+//
+// Otherwise, the result is the result of the first call to the Refresh function to
+// reach the target state.
+//
+// Cancellation from the passed in context will cancel the refresh loop
+func (conf *StateChangeConf) WaitForStateContext(ctx context.Context, entityType string) (interface{}, error) {
+	// fs.Debugf(entityType, "Waiting for state to become: %s", conf.Target)
+
+	notfoundTick := 0
+	targetOccurrence := 0
+
+	// Set a default for times to check for not found
+	if conf.NotFoundChecks == 0 {
+		conf.NotFoundChecks = 20
+	}
+
+	if conf.ContinuousTargetOccurrence == 0 {
+		conf.ContinuousTargetOccurrence = 1
+	}
+
+	type Result struct {
+		Result interface{}
+		State  string
+		Error  error
+		Done   bool
+	}
+
+	// Read every result from the refresh loop, waiting for a positive result.Done.
+	resCh := make(chan Result, 1)
+	// cancellation channel for the refresh loop
+	cancelCh := make(chan struct{})
+
+	result := Result{}
+
+	go func() {
+		defer close(resCh)
+
+		select {
+		case <-time.After(conf.Delay):
+		case <-cancelCh:
+			return
+		}
+
+		// start with 0 delay for the first loop
+		var wait time.Duration
+
+		for {
+			// store the last result
+			resCh <- result
+
+			// wait and watch for cancellation
+			select {
+			case <-cancelCh:
+				return
+			case <-time.After(wait):
+				// first round had no wait
+				if wait == 0 {
+					wait = 100 * time.Millisecond
+				}
+			}
+
+			res, currentState, err := conf.Refresh()
+			result = Result{
+				Result: res,
+				State:  currentState,
+				Error:  err,
+			}
+
+			if err != nil {
+				resCh <- result
+				return
+			}
+
+			// If we're waiting for the absence of a thing, then return
+			if res == nil && len(conf.Target) == 0 {
+				targetOccurrence++
+				if conf.ContinuousTargetOccurrence == targetOccurrence {
+					result.Done = true
+					resCh <- result
+					return
+				}
+				continue
+			}
+
+			if res == nil {
+				// If we didn't find the resource, check if we have been
+				// not finding it for a while, and if so, report an error.
+				notfoundTick++
+				if notfoundTick > conf.NotFoundChecks {
+					result.Error = &NotFoundError{
+						LastError: err,
+						Retries:   notfoundTick,
+					}
+					resCh <- result
+					return
+				}
+			} else {
+				// Reset the counter for when a resource isn't found
+				notfoundTick = 0
+				found := false
+
+				for _, allowed := range conf.Target {
+					if currentState == allowed {
+						found = true
+						targetOccurrence++
+						if conf.ContinuousTargetOccurrence == targetOccurrence {
+							result.Done = true
+							resCh <- result
+							return
+						}
+						continue
+					}
+				}
+
+				for _, allowed := range conf.Pending {
+					if currentState == allowed {
+						found = true
+						targetOccurrence = 0
+						break
+					}
+				}
+
+				if !found && len(conf.Pending) > 0 {
+					result.Error = &UnexpectedStateError{
+						LastError:     err,
+						State:         result.State,
+						ExpectedState: conf.Target,
+					}
+					resCh <- result
+					return
+				}
+			}
+
+			// Wait between refreshes using exponential backoff, except when
+			// waiting for the target state to reoccur.
+			if targetOccurrence == 0 {
+				wait *= 2
+			}
+
+			// If a poll interval has been specified, choose that interval.
+			// Otherwise, bound the default value.
+			if conf.PollInterval > 0 && conf.PollInterval < 180*time.Second {
+				wait = conf.PollInterval
+			} else {
+				if wait < conf.MinTimeout {
+					wait = conf.MinTimeout
+				} else if wait > 10*time.Second {
+					wait = 10 * time.Second
+				}
+			}
+
+			// fs.Debugf(entityType, "[TRACE] Waiting %s before next try", wait)
+		}
+	}()
+
+	// store the last value result from the refresh loop
+	lastResult := Result{}
+
+	timeout := time.After(conf.Timeout)
+	for {
+		select {
+		case r, ok := <-resCh:
+			// channel closed, so return the last result
+			if !ok {
+				return lastResult.Result, lastResult.Error
+			}
+
+			// we reached the intended state
+			if r.Done {
+				return r.Result, r.Error
+			}
+
+			// still waiting, store the last result
+			lastResult = r
+		case <-ctx.Done():
+			close(cancelCh)
+			return nil, ctx.Err()
+		case <-timeout:
+			// fs.Debugf(entityType, "[WARN] WaitForState timeout after %s", conf.Timeout)
+			// fs.Debugf(entityType, "[WARN] WaitForState starting %s refresh grace period", refreshGracePeriod)
+
+			// cancel the goroutine and start our grace period timer
+			close(cancelCh)
+			timeout := time.After(refreshGracePeriod)
+
+			// we need a for loop and a label to break on, because we may have
+			// an extra response value to read, but still want to wait for the
+			// channel to close.
+		forSelect:
+			for {
+				select {
+				case r, ok := <-resCh:
+					if r.Done {
+						// the last refresh loop reached the desired state
+						return r.Result, r.Error
+					}
+
+					if !ok {
+						// the goroutine returned
+						break forSelect
+					}
+
+					// target state not reached, save the result for the
+					// TimeoutError and wait for the channel to close
+					lastResult = r
+				case <-ctx.Done():
+					fs.Errorf(entityType, "Context cancellation detected, abandoning grace period")
+					break forSelect
+				case <-timeout:
+					fs.Errorf(entityType, "WaitForState exceeded refresh grace period")
+					break forSelect
+				}
+			}
+
+			return nil, &TimeoutError{
+				LastError:     lastResult.Error,
+				LastState:     lastResult.State,
+				Timeout:       conf.Timeout,
+				ExpectedState: conf.Target,
+			}
+		}
+	}
+}
+
+// NotFoundError resource not found error
+type NotFoundError struct {
+	LastError    error
+	LastRequest  interface{}
+	LastResponse interface{}
+	Message      string
+	Retries      int
+}
+
+func (e *NotFoundError) Error() string {
+	if e.Message != "" {
+		return e.Message
+	}
+
+	if e.Retries > 0 {
+		return fmt.Sprintf("couldn't find resource (%d retries)", e.Retries)
+	}
+
+	return "couldn't find resource"
+}
+
+func (e *NotFoundError) Unwrap() error {
+	return e.LastError
+}
+
+// UnexpectedStateError is returned when Refresh returns a state that's neither in Target nor Pending
+type UnexpectedStateError struct {
+	LastError     error
+	State         string
+	ExpectedState []string
+}
+
+func (e *UnexpectedStateError) Error() string {
+	return fmt.Sprintf(
+		"unexpected state '%s', wanted target '%s'. last error: %s",
+		e.State,
+		strings.Join(e.ExpectedState, ", "),
+		e.LastError,
+	)
+}
+
+func (e *UnexpectedStateError) Unwrap() error {
+	return e.LastError
+}
+
+// TimeoutError is returned when WaitForState times out
+type TimeoutError struct {
+	LastError     error
+	LastState     string
+	Timeout       time.Duration
+	ExpectedState []string
+}
+
+func (e *TimeoutError) Error() string {
+	expectedState := "resource to be gone"
+	if len(e.ExpectedState) > 0 {
+		expectedState = fmt.Sprintf("state to become '%s'", strings.Join(e.ExpectedState, ", "))
+	}
+
+	extraInfo := make([]string, 0)
+	if e.LastState != "" {
+		extraInfo = append(extraInfo, fmt.Sprintf("last state: '%s'", e.LastState))
+	}
+	if e.Timeout > 0 {
+		extraInfo = append(extraInfo, fmt.Sprintf("timeout: %s", e.Timeout.String()))
+	}
+
+	suffix := ""
+	if len(extraInfo) > 0 {
+		suffix = fmt.Sprintf(" (%s)", strings.Join(extraInfo, ", "))
+	}
+
+	if e.LastError != nil {
+		return fmt.Sprintf("timeout while waiting for %s%s: %s",
+			expectedState, suffix, e.LastError)
+	}
+
+	return fmt.Sprintf("timeout while waiting for %s%s",
+		expectedState, suffix)
+}
+
+func (e *TimeoutError) Unwrap() error {
+	return e.LastError
+}

backend/pcloud/api/types.go

@@ -13,7 +13,7 @@ const (
 	timeFormat = `"` + time.RFC1123Z + `"`
 )
 
-// Time represents represents date and time information for the
+// Time represents date and time information for the
 // pcloud API, by using RFC1123Z
 type Time time.Time
 

backend/pcloud/pcloud.go

@@ -588,7 +588,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
 // Creates from the parameters passed in a half finished Object which
 // must have setMetaData called on it
 //
-// Returns the object, leaf, directoryID and error
+// Returns the object, leaf, directoryID and error.
 //
 // Used to create new objects
 func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, size int64) (o *Object, leaf string, directoryID string, err error) {
@@ -607,7 +607,7 @@ func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time,
 
 // Put the object into the container
 //
-// Copy the reader in to the new object which is returned
+// Copy the reader in to the new object which is returned.
 //
 // The new object may have been created if an error is returned
 func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
@@ -681,9 +681,9 @@ func (f *Fs) Precision() time.Duration {
 
 // Copy src to this remote using server-side copy operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -766,9 +766,9 @@ func (f *Fs) CleanUp(ctx context.Context) error {
 
 // Move src to this remote using server-side move operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -1073,7 +1073,6 @@ func (o *Object) readMetaData(ctx context.Context) (err error) {
 
 // ModTime returns the modification time of the object
 //
-//
 // It attempts to read the objects mtime and if that isn't present the
 // LastModified returned in the http headers
 func (o *Object) ModTime(ctx context.Context) time.Time {
@@ -1152,7 +1151,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 
 // Update the object with the contents of the io.Reader, modTime and size
 //
-// If existing is set then it updates the object rather than creating a new one
+// If existing is set then it updates the object rather than creating a new one.
 //
 // The new object may have been created if an error is returned
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {

backend/premiumizeme/premiumizeme.go

@@ -493,7 +493,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 // Creates from the parameters passed in a half finished Object which
 // must have setMetaData called on it
 //
-// Returns the object, leaf, directoryID and error
+// Returns the object, leaf, directoryID and error.
 //
 // Used to create new objects
 func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, size int64) (o *Object, leaf string, directoryID string, err error) {
@@ -512,7 +512,7 @@ func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time,
 
 // Put the object
 //
-// Copy the reader in to the new object which is returned
+// Copy the reader in to the new object which is returned.
 //
 // The new object may have been created if an error is returned
 func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
@@ -530,9 +530,9 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
 
 // PutUnchecked the object into the container
 //
-// This will produce an error if the object already exists
+// This will produce an error if the object already exists.
 //
-// Copy the reader in to the new object which is returned
+// Copy the reader in to the new object which is returned.
 //
 // The new object may have been created if an error is returned
 func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
@@ -694,9 +694,9 @@ func (f *Fs) move(ctx context.Context, isFile bool, id, oldLeaf, newLeaf, oldDir
 
 // Move src to this remote using server-side move operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -870,7 +870,6 @@ func (o *Object) readMetaData(ctx context.Context) (err error) {
 
 // ModTime returns the modification time of the object
 //
-//
 // It attempts to read the objects mtime and if that isn't present the
 // LastModified returned in the http headers
 func (o *Object) ModTime(ctx context.Context) time.Time {
@@ -917,7 +916,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 
 // Update the object with the contents of the io.Reader, modTime and size
 //
-// If existing is set then it updates the object rather than creating a new one
+// If existing is set then it updates the object rather than creating a new one.
 //
 // The new object may have been created if an error is returned
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {

backend/putio/fs.go

@@ -230,7 +230,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 
 // Put the object
 //
-// Copy the reader in to the new object which is returned
+// Copy the reader in to the new object which is returned.
 //
 // The new object may have been created if an error is returned
 func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (o fs.Object, err error) {
@@ -523,9 +523,9 @@ func (f *Fs) Purge(ctx context.Context, dir string) (err error) {
 
 // Copy src to this remote using server-side copy operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -562,9 +562,9 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (o fs.Objec
 
 // Move src to this remote using server-side move operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //

backend/putio/object.go

@@ -261,7 +261,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 
 // Update the already existing object
 //
-// Copy the reader into the object updating modTime and size
+// Copy the reader into the object updating modTime and size.
 //
 // The new object may have been created if an error is returned
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {

backend/putio/putio.go

@@ -1,3 +1,4 @@
+// Package putio provides an interface to the put.io storage system.
 package putio
 
 import (

backend/qingstor/qingstor.go

@@ -1,9 +1,8 @@
-// Package qingstor provides an interface to QingStor object storage
-// Home: https://www.qingcloud.com/
-
 //go:build !plan9 && !js
 // +build !plan9,!js
 
+// Package qingstor provides an interface to QingStor object storage
+// Home: https://www.qingcloud.com/
 package qingstor
 
 import (
@@ -431,9 +430,9 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
 
 // Copy src to this remote using server-side copy operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // Will only be called if src.Fs().Name() == f.Name()
 //
@@ -477,7 +476,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 
 // Return an Object from a path
 //
-//If it can't be found it returns the error ErrorObjectNotFound.
+// If it can't be found it returns the error ErrorObjectNotFound.
 func (f *Fs) newObjectWithInfo(remote string, info *qs.KeyType) (fs.Object, error) {
 	o := &Object{
 		fs:     f,

backend/s3/gen_setfrom.go

@@ -0,0 +1,101 @@
+// Generate boilerplate code for setting similar structs from each other
+
+//go:build ignore
+// +build ignore
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"reflect"
+	"strings"
+
+	"github.com/aws/aws-sdk-go/service/s3"
+)
+
+// flags
+var (
+	outputFile = flag.String("o", "", "Output file name, stdout if unset")
+)
+
+// globals
+var (
+	out io.Writer = os.Stdout
+)
+
+// genSetFrom generates code to set the public members of a from b
+//
+// a and b should be pointers to structs
+//
+// a can be a different type from b
+//
+// Only the Fields which have the same name and assignable type on a
+// and b will be set.
+//
+// This is useful for copying between almost identical structures that
+// are frequently present in auto-generated code for cloud storage
+// interfaces.
+func genSetFrom(a, b interface{}) {
+	name := fmt.Sprintf("setFrom_%T_%T", a, b)
+	name = strings.Replace(name, ".", "", -1)
+	name = strings.Replace(name, "*", "", -1)
+	fmt.Fprintf(out, "\n// %s copies matching elements from a to b\n", name)
+	fmt.Fprintf(out, "func %s(a %T, b %T) {\n", name, a, b)
+	ta := reflect.TypeOf(a).Elem()
+	tb := reflect.TypeOf(b).Elem()
+	va := reflect.ValueOf(a).Elem()
+	vb := reflect.ValueOf(b).Elem()
+	for i := 0; i < tb.NumField(); i++ {
+		bField := vb.Field(i)
+		tbField := tb.Field(i)
+		name := tbField.Name
+		aField := va.FieldByName(name)
+		taField, found := ta.FieldByName(name)
+		if found && aField.IsValid() && bField.IsValid() && aField.CanSet() && tbField.Type.AssignableTo(taField.Type) {
+			fmt.Fprintf(out, "\ta.%s = b.%s\n", name, name)
+		}
+	}
+	fmt.Fprintf(out, "}\n")
+}
+
+func main() {
+	flag.Parse()
+
+	if *outputFile != "" {
+		fd, err := os.Create(*outputFile)
+		if err != nil {
+			log.Fatal(err)
+		}
+		defer func() {
+			err := fd.Close()
+			if err != nil {
+				log.Fatal(err)
+			}
+		}()
+		out = fd
+	}
+
+	fmt.Fprintf(out, `// Code generated by "go run gen_setfrom.go"; DO NOT EDIT.
+
+package s3
+
+import "github.com/aws/aws-sdk-go/service/s3"
+`)
+
+	genSetFrom(new(s3.ListObjectsInput), new(s3.ListObjectsV2Input))
+	genSetFrom(new(s3.ListObjectsV2Output), new(s3.ListObjectsOutput))
+	genSetFrom(new(s3.ListObjectVersionsInput), new(s3.ListObjectsV2Input))
+	genSetFrom(new(s3.ObjectVersion), new(s3.DeleteMarkerEntry))
+	genSetFrom(new(s3.ListObjectsV2Output), new(s3.ListObjectVersionsOutput))
+	genSetFrom(new(s3.Object), new(s3.ObjectVersion))
+	genSetFrom(new(s3.CreateMultipartUploadInput), new(s3.HeadObjectOutput))
+	genSetFrom(new(s3.CreateMultipartUploadInput), new(s3.CopyObjectInput))
+	genSetFrom(new(s3.UploadPartCopyInput), new(s3.CopyObjectInput))
+	genSetFrom(new(s3.HeadObjectOutput), new(s3.GetObjectOutput))
+	genSetFrom(new(s3.CreateMultipartUploadInput), new(s3.PutObjectInput))
+	genSetFrom(new(s3.HeadObjectOutput), new(s3.PutObjectInput))
+}

backend/s3/s3_internal_test.go

@@ -4,14 +4,19 @@ import (
 	"bytes"
 	"compress/gzip"
 	"context"
+	"crypto/md5"
 	"fmt"
 	"testing"
 	"time"
 
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/fstest/fstests"
 	"github.com/rclone/rclone/lib/random"
+	"github.com/rclone/rclone/lib/version"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -26,9 +31,15 @@ func gz(t *testing.T, s string) string {
 	return buf.String()
 }
 
+func md5sum(t *testing.T, s string) string {
+	hash := md5.Sum([]byte(s))
+	return fmt.Sprintf("%x", hash)
+}
+
 func (f *Fs) InternalTestMetadata(t *testing.T) {
 	ctx := context.Background()
-	contents := gz(t, random.String(1000))
+	original := random.String(1000)
+	contents := gz(t, original)
 
 	item := fstest.NewItem("test-metadata", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
 	btime := time.Now()
@@ -65,6 +76,31 @@ func (f *Fs) InternalTestMetadata(t *testing.T) {
 			assert.Equal(t, v, got, k)
 		}
 	}
+
+	t.Run("GzipEncoding", func(t *testing.T) {
+		// Test that the gzipped file we uploaded can be
+		// downloaded with and without decompression
+		checkDownload := func(wantContents string, wantSize int64, wantHash string) {
+			gotContents := fstests.ReadObject(ctx, t, o, -1)
+			assert.Equal(t, wantContents, gotContents)
+			assert.Equal(t, wantSize, o.Size())
+			gotHash, err := o.Hash(ctx, hash.MD5)
+			require.NoError(t, err)
+			assert.Equal(t, wantHash, gotHash)
+		}
+
+		t.Run("NoDecompress", func(t *testing.T) {
+			checkDownload(contents, int64(len(contents)), md5sum(t, contents))
+		})
+		t.Run("Decompress", func(t *testing.T) {
+			f.opt.Decompress = true
+			defer func() {
+				f.opt.Decompress = false
+			}()
+			checkDownload(original, -1, "")
+		})
+
+	})
 }
 
 func (f *Fs) InternalTestNoHead(t *testing.T) {
@@ -80,13 +116,279 @@ func (f *Fs) InternalTestNoHead(t *testing.T) {
 	defer func() {
 		assert.NoError(t, obj.Remove(ctx))
 	}()
-	// PutTestcontests checks the received object
+	// PutTestcontents checks the received object
 
 }
 
+func TestVersionLess(t *testing.T) {
+	key1 := "key1"
+	key2 := "key2"
+	t1 := fstest.Time("2022-01-21T12:00:00+01:00")
+	t2 := fstest.Time("2022-01-21T12:00:01+01:00")
+	for n, test := range []struct {
+		a, b *s3.ObjectVersion
+		want bool
+	}{
+		{a: nil, b: nil, want: true},
+		{a: &s3.ObjectVersion{Key: &key1, LastModified: &t1}, b: nil, want: false},
+		{a: nil, b: &s3.ObjectVersion{Key: &key1, LastModified: &t1}, want: true},
+		{a: &s3.ObjectVersion{Key: &key1, LastModified: &t1}, b: &s3.ObjectVersion{Key: &key1, LastModified: &t1}, want: false},
+		{a: &s3.ObjectVersion{Key: &key1, LastModified: &t1}, b: &s3.ObjectVersion{Key: &key1, LastModified: &t2}, want: false},
+		{a: &s3.ObjectVersion{Key: &key1, LastModified: &t2}, b: &s3.ObjectVersion{Key: &key1, LastModified: &t1}, want: true},
+		{a: &s3.ObjectVersion{Key: &key1, LastModified: &t1}, b: &s3.ObjectVersion{Key: &key2, LastModified: &t1}, want: true},
+		{a: &s3.ObjectVersion{Key: &key2, LastModified: &t1}, b: &s3.ObjectVersion{Key: &key1, LastModified: &t1}, want: false},
+		{a: &s3.ObjectVersion{Key: &key1, LastModified: &t1, IsLatest: aws.Bool(false)}, b: &s3.ObjectVersion{Key: &key1, LastModified: &t1}, want: false},
+		{a: &s3.ObjectVersion{Key: &key1, LastModified: &t1, IsLatest: aws.Bool(true)}, b: &s3.ObjectVersion{Key: &key1, LastModified: &t1}, want: true},
+		{a: &s3.ObjectVersion{Key: &key1, LastModified: &t1, IsLatest: aws.Bool(false)}, b: &s3.ObjectVersion{Key: &key1, LastModified: &t1, IsLatest: aws.Bool(true)}, want: false},
+	} {
+		got := versionLess(test.a, test.b)
+		assert.Equal(t, test.want, got, fmt.Sprintf("%d: %+v", n, test))
+	}
+}
+
+func TestMergeDeleteMarkers(t *testing.T) {
+	key1 := "key1"
+	key2 := "key2"
+	t1 := fstest.Time("2022-01-21T12:00:00+01:00")
+	t2 := fstest.Time("2022-01-21T12:00:01+01:00")
+	for n, test := range []struct {
+		versions []*s3.ObjectVersion
+		markers  []*s3.DeleteMarkerEntry
+		want     []*s3.ObjectVersion
+	}{
+		{
+			versions: []*s3.ObjectVersion{},
+			markers:  []*s3.DeleteMarkerEntry{},
+			want:     []*s3.ObjectVersion{},
+		},
+		{
+			versions: []*s3.ObjectVersion{
+				{
+					Key:          &key1,
+					LastModified: &t1,
+				},
+			},
+			markers: []*s3.DeleteMarkerEntry{},
+			want: []*s3.ObjectVersion{
+				{
+					Key:          &key1,
+					LastModified: &t1,
+				},
+			},
+		},
+		{
+			versions: []*s3.ObjectVersion{},
+			markers: []*s3.DeleteMarkerEntry{
+				{
+					Key:          &key1,
+					LastModified: &t1,
+				},
+			},
+			want: []*s3.ObjectVersion{
+				{
+					Key:          &key1,
+					LastModified: &t1,
+					Size:         isDeleteMarker,
+				},
+			},
+		},
+		{
+			versions: []*s3.ObjectVersion{
+				{
+					Key:          &key1,
+					LastModified: &t2,
+				},
+				{
+					Key:          &key2,
+					LastModified: &t2,
+				},
+			},
+			markers: []*s3.DeleteMarkerEntry{
+				{
+					Key:          &key1,
+					LastModified: &t1,
+				},
+			},
+			want: []*s3.ObjectVersion{
+				{
+					Key:          &key1,
+					LastModified: &t2,
+				},
+				{
+					Key:          &key1,
+					LastModified: &t1,
+					Size:         isDeleteMarker,
+				},
+				{
+					Key:          &key2,
+					LastModified: &t2,
+				},
+			},
+		},
+	} {
+		got := mergeDeleteMarkers(test.versions, test.markers)
+		assert.Equal(t, test.want, got, fmt.Sprintf("%d: %+v", n, test))
+	}
+}
+
+func (f *Fs) InternalTestVersions(t *testing.T) {
+	ctx := context.Background()
+
+	// Enable versioning for this bucket during this test
+	_, err := f.setGetVersioning(ctx, "Enabled")
+	if err != nil {
+		t.Skipf("Couldn't enable versioning: %v", err)
+	}
+	defer func() {
+		// Disable versioning for this bucket
+		_, err := f.setGetVersioning(ctx, "Suspended")
+		assert.NoError(t, err)
+	}()
+
+	// Small pause to make the LastModified different since AWS
+	// only seems to track them to 1 second granularity
+	time.Sleep(2 * time.Second)
+
+	// Create an object
+	const fileName = "test-versions.txt"
+	contents := random.String(100)
+	item := fstest.NewItem(fileName, contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
+	obj := fstests.PutTestContents(ctx, t, f, &item, contents, true)
+	defer func() {
+		assert.NoError(t, obj.Remove(ctx))
+	}()
+
+	// Small pause
+	time.Sleep(2 * time.Second)
+
+	// Remove it
+	assert.NoError(t, obj.Remove(ctx))
+
+	// Small pause to make the LastModified different since AWS only seems to track them to 1 second granularity
+	time.Sleep(2 * time.Second)
+
+	// And create it with different size and contents
+	newContents := random.String(101)
+	newItem := fstest.NewItem(fileName, newContents, fstest.Time("2002-05-06T04:05:06.499999999Z"))
+	newObj := fstests.PutTestContents(ctx, t, f, &newItem, newContents, true)
+
+	t.Run("Versions", func(t *testing.T) {
+		// Set --s3-versions for this test
+		f.opt.Versions = true
+		defer func() {
+			f.opt.Versions = false
+		}()
+
+		// Read the contents
+		entries, err := f.List(ctx, "")
+		require.NoError(t, err)
+		tests := 0
+		var fileNameVersion string
+		for _, entry := range entries {
+			remote := entry.Remote()
+			if remote == fileName {
+				t.Run("ReadCurrent", func(t *testing.T) {
+					assert.Equal(t, newContents, fstests.ReadObject(ctx, t, entry.(fs.Object), -1))
+				})
+				tests++
+			} else if versionTime, p := version.Remove(remote); !versionTime.IsZero() && p == fileName {
+				t.Run("ReadVersion", func(t *testing.T) {
+					assert.Equal(t, contents, fstests.ReadObject(ctx, t, entry.(fs.Object), -1))
+				})
+				assert.WithinDuration(t, obj.(*Object).lastModified, versionTime, time.Second, "object time must be with 1 second of version time")
+				fileNameVersion = remote
+				tests++
+			}
+		}
+		assert.Equal(t, 2, tests, "object missing from listing")
+
+		// Check we can read the object with a version suffix
+		t.Run("NewObject", func(t *testing.T) {
+			o, err := f.NewObject(ctx, fileNameVersion)
+			require.NoError(t, err)
+			require.NotNil(t, o)
+			assert.Equal(t, int64(100), o.Size(), o.Remote())
+		})
+	})
+
+	t.Run("VersionAt", func(t *testing.T) {
+		// We set --s3-version-at for this test so make sure we reset it at the end
+		defer func() {
+			f.opt.VersionAt = fs.Time{}
+		}()
+
+		var (
+			firstObjectTime  = obj.(*Object).lastModified
+			secondObjectTime = newObj.(*Object).lastModified
+		)
+
+		for _, test := range []struct {
+			what     string
+			at       time.Time
+			want     []fstest.Item
+			wantErr  error
+			wantSize int64
+		}{
+			{
+				what:    "Before",
+				at:      firstObjectTime.Add(-time.Second),
+				want:    fstests.InternalTestFiles,
+				wantErr: fs.ErrorObjectNotFound,
+			},
+			{
+				what:     "AfterOne",
+				at:       firstObjectTime.Add(time.Second),
+				want:     append([]fstest.Item{item}, fstests.InternalTestFiles...),
+				wantSize: 100,
+			},
+			{
+				what:    "AfterDelete",
+				at:      secondObjectTime.Add(-time.Second),
+				want:    fstests.InternalTestFiles,
+				wantErr: fs.ErrorObjectNotFound,
+			},
+			{
+				what:     "AfterTwo",
+				at:       secondObjectTime.Add(time.Second),
+				want:     append([]fstest.Item{newItem}, fstests.InternalTestFiles...),
+				wantSize: 101,
+			},
+		} {
+			t.Run(test.what, func(t *testing.T) {
+				f.opt.VersionAt = fs.Time(test.at)
+				t.Run("List", func(t *testing.T) {
+					fstest.CheckListing(t, f, test.want)
+				})
+				t.Run("NewObject", func(t *testing.T) {
+					gotObj, gotErr := f.NewObject(ctx, fileName)
+					assert.Equal(t, test.wantErr, gotErr)
+					if gotErr == nil {
+						assert.Equal(t, test.wantSize, gotObj.Size())
+					}
+				})
+			})
+		}
+	})
+
+	t.Run("Cleanup", func(t *testing.T) {
+		require.NoError(t, f.CleanUpHidden(ctx))
+		items := append([]fstest.Item{newItem}, fstests.InternalTestFiles...)
+		fstest.CheckListing(t, f, items)
+		// Set --s3-versions for this test
+		f.opt.Versions = true
+		defer func() {
+			f.opt.Versions = false
+		}()
+		fstest.CheckListing(t, f, items)
+	})
+
+	// Purge gets tested later
+}
+
 func (f *Fs) InternalTest(t *testing.T) {
 	t.Run("Metadata", f.InternalTestMetadata)
 	t.Run("NoHead", f.InternalTestNoHead)
+	t.Run("Versions", f.InternalTestVersions)
 }
 
 var _ fstests.InternalTester = (*Fs)(nil)

backend/s3/setfrom.go

@@ -0,0 +1,239 @@
+// Code generated by "go run gen_setfrom.go"; DO NOT EDIT.
+
+package s3
+
+import "github.com/aws/aws-sdk-go/service/s3"
+
+// setFrom_s3ListObjectsInput_s3ListObjectsV2Input copies matching elements from a to b
+func setFrom_s3ListObjectsInput_s3ListObjectsV2Input(a *s3.ListObjectsInput, b *s3.ListObjectsV2Input) {
+	a.Bucket = b.Bucket
+	a.Delimiter = b.Delimiter
+	a.EncodingType = b.EncodingType
+	a.ExpectedBucketOwner = b.ExpectedBucketOwner
+	a.MaxKeys = b.MaxKeys
+	a.Prefix = b.Prefix
+	a.RequestPayer = b.RequestPayer
+}
+
+// setFrom_s3ListObjectsV2Output_s3ListObjectsOutput copies matching elements from a to b
+func setFrom_s3ListObjectsV2Output_s3ListObjectsOutput(a *s3.ListObjectsV2Output, b *s3.ListObjectsOutput) {
+	a.CommonPrefixes = b.CommonPrefixes
+	a.Contents = b.Contents
+	a.Delimiter = b.Delimiter
+	a.EncodingType = b.EncodingType
+	a.IsTruncated = b.IsTruncated
+	a.MaxKeys = b.MaxKeys
+	a.Name = b.Name
+	a.Prefix = b.Prefix
+}
+
+// setFrom_s3ListObjectVersionsInput_s3ListObjectsV2Input copies matching elements from a to b
+func setFrom_s3ListObjectVersionsInput_s3ListObjectsV2Input(a *s3.ListObjectVersionsInput, b *s3.ListObjectsV2Input) {
+	a.Bucket = b.Bucket
+	a.Delimiter = b.Delimiter
+	a.EncodingType = b.EncodingType
+	a.ExpectedBucketOwner = b.ExpectedBucketOwner
+	a.MaxKeys = b.MaxKeys
+	a.Prefix = b.Prefix
+}
+
+// setFrom_s3ObjectVersion_s3DeleteMarkerEntry copies matching elements from a to b
+func setFrom_s3ObjectVersion_s3DeleteMarkerEntry(a *s3.ObjectVersion, b *s3.DeleteMarkerEntry) {
+	a.IsLatest = b.IsLatest
+	a.Key = b.Key
+	a.LastModified = b.LastModified
+	a.Owner = b.Owner
+	a.VersionId = b.VersionId
+}
+
+// setFrom_s3ListObjectsV2Output_s3ListObjectVersionsOutput copies matching elements from a to b
+func setFrom_s3ListObjectsV2Output_s3ListObjectVersionsOutput(a *s3.ListObjectsV2Output, b *s3.ListObjectVersionsOutput) {
+	a.CommonPrefixes = b.CommonPrefixes
+	a.Delimiter = b.Delimiter
+	a.EncodingType = b.EncodingType
+	a.IsTruncated = b.IsTruncated
+	a.MaxKeys = b.MaxKeys
+	a.Name = b.Name
+	a.Prefix = b.Prefix
+}
+
+// setFrom_s3Object_s3ObjectVersion copies matching elements from a to b
+func setFrom_s3Object_s3ObjectVersion(a *s3.Object, b *s3.ObjectVersion) {
+	a.ChecksumAlgorithm = b.ChecksumAlgorithm
+	a.ETag = b.ETag
+	a.Key = b.Key
+	a.LastModified = b.LastModified
+	a.Owner = b.Owner
+	a.Size = b.Size
+	a.StorageClass = b.StorageClass
+}
+
+// setFrom_s3CreateMultipartUploadInput_s3HeadObjectOutput copies matching elements from a to b
+func setFrom_s3CreateMultipartUploadInput_s3HeadObjectOutput(a *s3.CreateMultipartUploadInput, b *s3.HeadObjectOutput) {
+	a.BucketKeyEnabled = b.BucketKeyEnabled
+	a.CacheControl = b.CacheControl
+	a.ContentDisposition = b.ContentDisposition
+	a.ContentEncoding = b.ContentEncoding
+	a.ContentLanguage = b.ContentLanguage
+	a.ContentType = b.ContentType
+	a.Metadata = b.Metadata
+	a.ObjectLockLegalHoldStatus = b.ObjectLockLegalHoldStatus
+	a.ObjectLockMode = b.ObjectLockMode
+	a.ObjectLockRetainUntilDate = b.ObjectLockRetainUntilDate
+	a.SSECustomerAlgorithm = b.SSECustomerAlgorithm
+	a.SSECustomerKeyMD5 = b.SSECustomerKeyMD5
+	a.SSEKMSKeyId = b.SSEKMSKeyId
+	a.ServerSideEncryption = b.ServerSideEncryption
+	a.StorageClass = b.StorageClass
+	a.WebsiteRedirectLocation = b.WebsiteRedirectLocation
+}
+
+// setFrom_s3CreateMultipartUploadInput_s3CopyObjectInput copies matching elements from a to b
+func setFrom_s3CreateMultipartUploadInput_s3CopyObjectInput(a *s3.CreateMultipartUploadInput, b *s3.CopyObjectInput) {
+	a.ACL = b.ACL
+	a.Bucket = b.Bucket
+	a.BucketKeyEnabled = b.BucketKeyEnabled
+	a.CacheControl = b.CacheControl
+	a.ChecksumAlgorithm = b.ChecksumAlgorithm
+	a.ContentDisposition = b.ContentDisposition
+	a.ContentEncoding = b.ContentEncoding
+	a.ContentLanguage = b.ContentLanguage
+	a.ContentType = b.ContentType
+	a.ExpectedBucketOwner = b.ExpectedBucketOwner
+	a.Expires = b.Expires
+	a.GrantFullControl = b.GrantFullControl
+	a.GrantRead = b.GrantRead
+	a.GrantReadACP = b.GrantReadACP
+	a.GrantWriteACP = b.GrantWriteACP
+	a.Key = b.Key
+	a.Metadata = b.Metadata
+	a.ObjectLockLegalHoldStatus = b.ObjectLockLegalHoldStatus
+	a.ObjectLockMode = b.ObjectLockMode
+	a.ObjectLockRetainUntilDate = b.ObjectLockRetainUntilDate
+	a.RequestPayer = b.RequestPayer
+	a.SSECustomerAlgorithm = b.SSECustomerAlgorithm
+	a.SSECustomerKey = b.SSECustomerKey
+	a.SSECustomerKeyMD5 = b.SSECustomerKeyMD5
+	a.SSEKMSEncryptionContext = b.SSEKMSEncryptionContext
+	a.SSEKMSKeyId = b.SSEKMSKeyId
+	a.ServerSideEncryption = b.ServerSideEncryption
+	a.StorageClass = b.StorageClass
+	a.Tagging = b.Tagging
+	a.WebsiteRedirectLocation = b.WebsiteRedirectLocation
+}
+
+// setFrom_s3UploadPartCopyInput_s3CopyObjectInput copies matching elements from a to b
+func setFrom_s3UploadPartCopyInput_s3CopyObjectInput(a *s3.UploadPartCopyInput, b *s3.CopyObjectInput) {
+	a.Bucket = b.Bucket
+	a.CopySource = b.CopySource
+	a.CopySourceIfMatch = b.CopySourceIfMatch
+	a.CopySourceIfModifiedSince = b.CopySourceIfModifiedSince
+	a.CopySourceIfNoneMatch = b.CopySourceIfNoneMatch
+	a.CopySourceIfUnmodifiedSince = b.CopySourceIfUnmodifiedSince
+	a.CopySourceSSECustomerAlgorithm = b.CopySourceSSECustomerAlgorithm
+	a.CopySourceSSECustomerKey = b.CopySourceSSECustomerKey
+	a.CopySourceSSECustomerKeyMD5 = b.CopySourceSSECustomerKeyMD5
+	a.ExpectedBucketOwner = b.ExpectedBucketOwner
+	a.ExpectedSourceBucketOwner = b.ExpectedSourceBucketOwner
+	a.Key = b.Key
+	a.RequestPayer = b.RequestPayer
+	a.SSECustomerAlgorithm = b.SSECustomerAlgorithm
+	a.SSECustomerKey = b.SSECustomerKey
+	a.SSECustomerKeyMD5 = b.SSECustomerKeyMD5
+}
+
+// setFrom_s3HeadObjectOutput_s3GetObjectOutput copies matching elements from a to b
+func setFrom_s3HeadObjectOutput_s3GetObjectOutput(a *s3.HeadObjectOutput, b *s3.GetObjectOutput) {
+	a.AcceptRanges = b.AcceptRanges
+	a.BucketKeyEnabled = b.BucketKeyEnabled
+	a.CacheControl = b.CacheControl
+	a.ChecksumCRC32 = b.ChecksumCRC32
+	a.ChecksumCRC32C = b.ChecksumCRC32C
+	a.ChecksumSHA1 = b.ChecksumSHA1
+	a.ChecksumSHA256 = b.ChecksumSHA256
+	a.ContentDisposition = b.ContentDisposition
+	a.ContentEncoding = b.ContentEncoding
+	a.ContentLanguage = b.ContentLanguage
+	a.ContentLength = b.ContentLength
+	a.ContentType = b.ContentType
+	a.DeleteMarker = b.DeleteMarker
+	a.ETag = b.ETag
+	a.Expiration = b.Expiration
+	a.Expires = b.Expires
+	a.LastModified = b.LastModified
+	a.Metadata = b.Metadata
+	a.MissingMeta = b.MissingMeta
+	a.ObjectLockLegalHoldStatus = b.ObjectLockLegalHoldStatus
+	a.ObjectLockMode = b.ObjectLockMode
+	a.ObjectLockRetainUntilDate = b.ObjectLockRetainUntilDate
+	a.PartsCount = b.PartsCount
+	a.ReplicationStatus = b.ReplicationStatus
+	a.RequestCharged = b.RequestCharged
+	a.Restore = b.Restore
+	a.SSECustomerAlgorithm = b.SSECustomerAlgorithm
+	a.SSECustomerKeyMD5 = b.SSECustomerKeyMD5
+	a.SSEKMSKeyId = b.SSEKMSKeyId
+	a.ServerSideEncryption = b.ServerSideEncryption
+	a.StorageClass = b.StorageClass
+	a.VersionId = b.VersionId
+	a.WebsiteRedirectLocation = b.WebsiteRedirectLocation
+}
+
+// setFrom_s3CreateMultipartUploadInput_s3PutObjectInput copies matching elements from a to b
+func setFrom_s3CreateMultipartUploadInput_s3PutObjectInput(a *s3.CreateMultipartUploadInput, b *s3.PutObjectInput) {
+	a.ACL = b.ACL
+	a.Bucket = b.Bucket
+	a.BucketKeyEnabled = b.BucketKeyEnabled
+	a.CacheControl = b.CacheControl
+	a.ChecksumAlgorithm = b.ChecksumAlgorithm
+	a.ContentDisposition = b.ContentDisposition
+	a.ContentEncoding = b.ContentEncoding
+	a.ContentLanguage = b.ContentLanguage
+	a.ContentType = b.ContentType
+	a.ExpectedBucketOwner = b.ExpectedBucketOwner
+	a.Expires = b.Expires
+	a.GrantFullControl = b.GrantFullControl
+	a.GrantRead = b.GrantRead
+	a.GrantReadACP = b.GrantReadACP
+	a.GrantWriteACP = b.GrantWriteACP
+	a.Key = b.Key
+	a.Metadata = b.Metadata
+	a.ObjectLockLegalHoldStatus = b.ObjectLockLegalHoldStatus
+	a.ObjectLockMode = b.ObjectLockMode
+	a.ObjectLockRetainUntilDate = b.ObjectLockRetainUntilDate
+	a.RequestPayer = b.RequestPayer
+	a.SSECustomerAlgorithm = b.SSECustomerAlgorithm
+	a.SSECustomerKey = b.SSECustomerKey
+	a.SSECustomerKeyMD5 = b.SSECustomerKeyMD5
+	a.SSEKMSEncryptionContext = b.SSEKMSEncryptionContext
+	a.SSEKMSKeyId = b.SSEKMSKeyId
+	a.ServerSideEncryption = b.ServerSideEncryption
+	a.StorageClass = b.StorageClass
+	a.Tagging = b.Tagging
+	a.WebsiteRedirectLocation = b.WebsiteRedirectLocation
+}
+
+// setFrom_s3HeadObjectOutput_s3PutObjectInput copies matching elements from a to b
+func setFrom_s3HeadObjectOutput_s3PutObjectInput(a *s3.HeadObjectOutput, b *s3.PutObjectInput) {
+	a.BucketKeyEnabled = b.BucketKeyEnabled
+	a.CacheControl = b.CacheControl
+	a.ChecksumCRC32 = b.ChecksumCRC32
+	a.ChecksumCRC32C = b.ChecksumCRC32C
+	a.ChecksumSHA1 = b.ChecksumSHA1
+	a.ChecksumSHA256 = b.ChecksumSHA256
+	a.ContentDisposition = b.ContentDisposition
+	a.ContentEncoding = b.ContentEncoding
+	a.ContentLanguage = b.ContentLanguage
+	a.ContentLength = b.ContentLength
+	a.ContentType = b.ContentType
+	a.Metadata = b.Metadata
+	a.ObjectLockLegalHoldStatus = b.ObjectLockLegalHoldStatus
+	a.ObjectLockMode = b.ObjectLockMode
+	a.ObjectLockRetainUntilDate = b.ObjectLockRetainUntilDate
+	a.SSECustomerAlgorithm = b.SSECustomerAlgorithm
+	a.SSECustomerKeyMD5 = b.SSECustomerKeyMD5
+	a.SSEKMSKeyId = b.SSEKMSKeyId
+	a.ServerSideEncryption = b.ServerSideEncryption
+	a.StorageClass = b.StorageClass
+	a.WebsiteRedirectLocation = b.WebsiteRedirectLocation
+}

backend/seafile/api/types.go

@@ -1,3 +1,4 @@
+// Package api provides types used by the Seafile API.
 package api
 
 // Some api objects are duplicated with only small differences,

backend/seafile/seafile.go

@@ -1,3 +1,4 @@
+// Package seafile provides an interface to the Seafile storage system.
 package seafile
 
 import (
@@ -136,7 +137,7 @@ type Fs struct {
 	features            *fs.Features // optional features
 	endpoint            *url.URL     // URL of the host
 	endpointURL         string       // endpoint as a string
-	srv                 *rest.Client // the connection to the one drive server
+	srv                 *rest.Client // the connection to the server
 	pacer               *fs.Pacer    // pacer for API calls
 	authMu              sync.Mutex   // Mutex to protect library decryption
 	createDirMutex      sync.Mutex   // Protect creation of directories
@@ -671,9 +672,9 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) e
 
 // Copy src to this remote using server-side copy operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // If it isn't possible then return fs.ErrorCantCopy
 func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
@@ -722,9 +723,9 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 
 // Move src to this remote using server-side move operations.
 //
-// This is stored with the remote path given
+// This is stored with the remote path given.
 //
-// It returns the destination Object and a possible error
+// It returns the destination Object and a possible error.
 //
 // If it isn't possible then return fs.ErrorCantMove
 func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {