Version v1.60.1

Before this change, when using -server-side-across-configs rclone
would direct Move/Copy/DirMove to the destination server.

However this should be directed to the source server. This is a little
unclear in the RFC, but the name of the parameter "Destination:" seems
clear and this is how dCache and Rucio have implemented it.

See: https://forum.rclone.org/t/webdav-copy-request-implemented-incorrectly/34072/

.github/dependabot.yml

@@ -1,10 +0,0 @@
-version: 2
-updates:
-  - package-ecosystem: "github-actions"
-    directory: "/"
-    schedule:
-      interval: "daily"
-  - package-ecosystem: "gomod"
-    directory: "/"
-    schedule:
-      interval: "daily"

.github/workflows/build.yml

@@ -15,24 +15,22 @@ on:
   workflow_dispatch:
     inputs:
       manual:
-        description: Manual run (bypass default conditions)
-        type: boolean
         required: true
         default: true
 
 jobs:
   build:
-    if: ${{ github.event.inputs.manual == 'true' || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name)) }}
+    if: ${{ github.repository == 'rclone/rclone' || github.event.inputs.manual }}
     timeout-minutes: 60
     strategy:
       fail-fast: false
       matrix:
-        job_name: ['linux', 'linux_386', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.18', 'go1.19']
+        job_name: ['linux', 'linux_386', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.17', 'go1.18']
 
         include:
           - job_name: linux
             os: ubuntu-latest
-            go: '1.20'
+            go: '1.19.x'
             gotags: cmount
             build_flags: '-include "^linux/"'
             check: true
@@ -43,14 +41,14 @@ jobs:
 
           - job_name: linux_386
             os: ubuntu-latest
-            go: '1.20'
+            go: '1.19.x'
             goarch: 386
             gotags: cmount
             quicktest: true
 
           - job_name: mac_amd64
             os: macos-11
-            go: '1.20'
+            go: '1.19.x'
             gotags: 'cmount'
             build_flags: '-include "^darwin/amd64" -cgo'
             quicktest: true
@@ -59,14 +57,14 @@ jobs:
 
           - job_name: mac_arm64
             os: macos-11
-            go: '1.20'
+            go: '1.19.x'
             gotags: 'cmount'
             build_flags: '-include "^darwin/arm64" -cgo -macos-arch arm64 -cgo-cflags=-I/usr/local/include -cgo-ldflags=-L/usr/local/lib'
             deploy: true
 
           - job_name: windows
             os: windows-latest
-            go: '1.20'
+            go: '1.19.x'
             gotags: cmount
             cgo: '0'
             build_flags: '-include "^windows/"'
@@ -76,20 +74,20 @@ jobs:
 
           - job_name: other_os
             os: ubuntu-latest
-            go: '1.20'
+            go: '1.19.x'
             build_flags: '-exclude "^(windows/|darwin/|linux/)"'
             compile_all: true
             deploy: true
 
-          - job_name: go1.18
+          - job_name: go1.17
             os: ubuntu-latest
-            go: '1.18'
+            go: '1.17.x'
             quicktest: true
             racequicktest: true
 
-          - job_name: go1.19
+          - job_name: go1.18
             os: ubuntu-latest
-            go: '1.19'
+            go: '1.18.x'
             quicktest: true
             racequicktest: true
 
@@ -106,6 +104,7 @@ jobs:
       - name: Install Go
         uses: actions/setup-go@v3
         with:
+          stable: 'false'
           go-version: ${{ matrix.go }}
           check-latest: true
 
@@ -124,7 +123,7 @@ jobs:
           sudo modprobe fuse
           sudo chmod 666 /dev/fuse
           sudo chown root:$USER /etc/fuse.conf
-          sudo apt-get install fuse3 libfuse-dev rpm pkg-config
+          sudo apt-get install fuse libfuse-dev rpm pkg-config
         if: matrix.os == 'ubuntu-latest'
 
       - name: Install Libraries on macOS
@@ -220,7 +219,7 @@ jobs:
         if: matrix.deploy && github.head_ref == '' && github.repository == 'rclone/rclone'
 
   lint:
-    if: ${{ github.event.inputs.manual == 'true' || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name)) }}
+    if: ${{ github.repository == 'rclone/rclone' || github.event.inputs.manual }}
     timeout-minutes: 30
     name: "lint"
     runs-on: ubuntu-latest
@@ -235,21 +234,8 @@ jobs:
           # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version
           version: latest
 
-      # Run govulncheck on the latest go version, the one we build binaries with
-      - name: Install Go
-        uses: actions/setup-go@v3
-        with:
-          go-version: '1.20'
-          check-latest: true
-
-      - name: Install govulncheck
-        run: go install golang.org/x/vuln/cmd/govulncheck@latest
-
-      - name: Scan for vulnerabilities
-        run: govulncheck ./...
-
   android:
-    if: ${{ github.event.inputs.manual == 'true' || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name)) }}
+    if: ${{ github.repository == 'rclone/rclone' || github.event.inputs.manual }}
     timeout-minutes: 30
     name: "android-all"
     runs-on: ubuntu-latest
@@ -264,7 +250,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v3
         with:
-          go-version: '1.20'
+          go-version: 1.19.x
 
       - name: Go module cache
         uses: actions/cache@v3

.github/workflows/winget.yml

@@ -1,14 +0,0 @@
-name: Publish to Winget
-on:
-  release:
-    types: [released]
-
-jobs:
-  publish:
-    runs-on: windows-latest # Action can only run on Windows
-    steps:
-      - uses: vedantmgoyal2009/winget-releaser@v2
-        with:
-          identifier: Rclone.Rclone
-          installers-regex: '-windows-\w+\.zip$'
-          token: ${{ secrets.WINGET_TOKEN }}

Dockerfile

@@ -11,7 +11,7 @@ RUN ./rclone version
 # Begin final image
 FROM alpine:latest
 
-RUN apk --no-cache add ca-certificates fuse3 tzdata && \
+RUN apk --no-cache add ca-certificates fuse tzdata && \
   echo "user_allow_other" >> /etc/fuse.conf
 
 COPY --from=builder /go/src/github.com/rclone/rclone/rclone /usr/local/bin/

Makefile

@@ -81,9 +81,6 @@ quicktest:
 racequicktest:
 	RCLONE_CONFIG="/notfound" go test $(BUILDTAGS) -cpu=2 -race ./...
 
-compiletest:
-	RCLONE_CONFIG="/notfound" go test $(BUILDTAGS) -run XXX ./...
-
 # Do source code quality checks
 check:	rclone
 	@echo "-- START CODE QUALITY REPORT -------------------------------"

README.md

@@ -50,7 +50,6 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
   * IBM COS S3 [:page_facing_up:](https://rclone.org/s3/#ibm-cos-s3)
   * IONOS Cloud [:page_facing_up:](https://rclone.org/s3/#ionos)
   * Koofr [:page_facing_up:](https://rclone.org/koofr/)
-  * Liara Object Storage [:page_facing_up:](https://rclone.org/s3/#liara-object-storage)
   * Mail.ru Cloud [:page_facing_up:](https://rclone.org/mailru/)
   * Memset Memstore [:page_facing_up:](https://rclone.org/swift/)
   * Mega [:page_facing_up:](https://rclone.org/mega/)

RELEASE.md

@@ -10,7 +10,7 @@ This file describes how to make the various kinds of releases
 ## Making a release
 
   * git checkout master # see below for stable branch
-  * git pull # IMPORTANT
+  * git pull
   * git status - make sure everything is checked in
   * Check GitHub actions build for master is Green
   * make test # see integration test server or run locally
@@ -21,7 +21,6 @@ This file describes how to make the various kinds of releases
   * git status - to check for new man pages - git add them
   * git commit -a -v -m "Version v1.XX.0"
   * make retag
-  * git push origin # without --follow-tags so it doesn't push the tag if it fails
   * git push --follow-tags origin
   * # Wait for the GitHub builds to complete then...
   * make fetch_binaries
@@ -75,7 +74,8 @@ Set vars
 First make the release branch.  If this is a second point release then
 this will be done already.
 
-  * git co -b ${BASE_TAG}-stable ${BASE_TAG}.0
+  * git branch ${BASE_TAG} ${BASE_TAG}-stable
+  * git co ${BASE_TAG}-stable
   * make startstable
 
 Now

VERSION

@@ -1 +1 @@
-v1.62.2
+v1.60.1

backend/azureblob/azureblob_internal_test.go

@@ -1,5 +1,5 @@
-//go:build !plan9 && !solaris && !js && go1.18
-// +build !plan9,!solaris,!js,go1.18
+//go:build !plan9 && !solaris && !js
+// +build !plan9,!solaris,!js
 
 package azureblob
 

backend/azureblob/azureblob_test.go

@@ -1,11 +1,12 @@
 // Test AzureBlob filesystem interface
 
-//go:build !plan9 && !solaris && !js && go1.18
-// +build !plan9,!solaris,!js,go1.18
+//go:build !plan9 && !solaris && !js
+// +build !plan9,!solaris,!js
 
 package azureblob
 
 import (
+	"context"
 	"testing"
 
 	"github.com/rclone/rclone/fs"
@@ -16,12 +17,10 @@ import (
 // TestIntegration runs integration tests against the remote
 func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
-		RemoteName:  "TestAzureBlob:",
-		NilObject:   (*Object)(nil),
-		TiersToTest: []string{"Hot", "Cool"},
-		ChunkedUpload: fstests.ChunkedUploadConfig{
-			MinChunkSize: defaultChunkSize,
-		},
+		RemoteName:    "TestAzureBlob:",
+		NilObject:     (*Object)(nil),
+		TiersToTest:   []string{"Hot", "Cool"},
+		ChunkedUpload: fstests.ChunkedUploadConfig{},
 	})
 }
 
@@ -33,6 +32,36 @@ var (
 	_ fstests.SetUploadChunkSizer = (*Fs)(nil)
 )
 
+// TestServicePrincipalFileSuccess checks that, given a proper JSON file, we can create a token.
+func TestServicePrincipalFileSuccess(t *testing.T) {
+	ctx := context.TODO()
+	credentials := `
+{
+    "appId": "my application (client) ID",
+    "password": "my secret",
+    "tenant": "my active directory tenant ID"
+}
+`
+	tokenRefresher, err := newServicePrincipalTokenRefresher(ctx, []byte(credentials))
+	if assert.NoError(t, err) {
+		assert.NotNil(t, tokenRefresher)
+	}
+}
+
+// TestServicePrincipalFileFailure checks that, given a JSON file with a missing secret, it returns an error.
+func TestServicePrincipalFileFailure(t *testing.T) {
+	ctx := context.TODO()
+	credentials := `
+{
+    "appId": "my application (client) ID",
+    "tenant": "my active directory tenant ID"
+}
+`
+	_, err := newServicePrincipalTokenRefresher(ctx, []byte(credentials))
+	assert.Error(t, err)
+	assert.EqualError(t, err, "error creating service principal token: parameter 'secret' cannot be empty")
+}
+
 func TestValidateAccessTier(t *testing.T) {
 	tests := map[string]struct {
 		accessTier string

backend/azureblob/azureblob_unsupported.go

@@ -1,7 +1,7 @@
 // Build for azureblob for unsupported platforms to stop go complaining
 // about "no buildable Go source files "
 
-//go:build plan9 || solaris || js || !go1.18
-// +build plan9 solaris js !go1.18
+//go:build plan9 || solaris || js
+// +build plan9 solaris js
 
 package azureblob

backend/azureblob/imds.go

@@ -0,0 +1,137 @@
+//go:build !plan9 && !solaris && !js
+// +build !plan9,!solaris,!js
+
+package azureblob
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+
+	"github.com/Azure/go-autorest/autorest/adal"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/fshttp"
+)
+
+const (
+	azureResource      = "https://storage.azure.com"
+	imdsAPIVersion     = "2018-02-01"
+	msiEndpointDefault = "http://169.254.169.254/metadata/identity/oauth2/token"
+)
+
+// This custom type is used to add the port the test server has bound to
+// to the request context.
+type testPortKey string
+
+type msiIdentifierType int
+
+const (
+	msiClientID msiIdentifierType = iota
+	msiObjectID
+	msiResourceID
+)
+
+type userMSI struct {
+	Type  msiIdentifierType
+	Value string
+}
+
+type httpError struct {
+	Response *http.Response
+}
+
+func (e httpError) Error() string {
+	return fmt.Sprintf("HTTP error %v (%v)", e.Response.StatusCode, e.Response.Status)
+}
+
+// GetMSIToken attempts to obtain an MSI token from the Azure Instance
+// Metadata Service.
+func GetMSIToken(ctx context.Context, identity *userMSI) (adal.Token, error) {
+	// Attempt to get an MSI token; silently continue if unsuccessful.
+	// This code has been lovingly stolen from azcopy's OAuthTokenManager.
+	result := adal.Token{}
+	req, err := http.NewRequestWithContext(ctx, "GET", msiEndpointDefault, nil)
+	if err != nil {
+		fs.Debugf(nil, "Failed to create request: %v", err)
+		return result, err
+	}
+	params := req.URL.Query()
+	params.Set("resource", azureResource)
+	params.Set("api-version", imdsAPIVersion)
+
+	// Specify user-assigned identity if requested.
+	if identity != nil {
+		switch identity.Type {
+		case msiClientID:
+			params.Set("client_id", identity.Value)
+		case msiObjectID:
+			params.Set("object_id", identity.Value)
+		case msiResourceID:
+			params.Set("mi_res_id", identity.Value)
+		default:
+			// If this happens, the calling function and this one don't agree on
+			// what valid ID types exist.
+			return result, fmt.Errorf("unknown MSI identity type specified")
+		}
+	}
+	req.URL.RawQuery = params.Encode()
+
+	// The Metadata header is required by all calls to IMDS.
+	req.Header.Set("Metadata", "true")
+
+	// If this function is run in a test, query the test server instead of IMDS.
+	testPort, isTest := ctx.Value(testPortKey("testPort")).(int)
+	if isTest {
+		req.URL.Host = fmt.Sprintf("localhost:%d", testPort)
+		req.Host = req.URL.Host
+	}
+
+	// Send request
+	httpClient := fshttp.NewClient(ctx)
+	resp, err := httpClient.Do(req)
+	if err != nil {
+		return result, fmt.Errorf("MSI is not enabled on this VM: %w", err)
+	}
+	defer func() { // resp and Body should not be nil
+		_, err = io.Copy(ioutil.Discard, resp.Body)
+		if err != nil {
+			fs.Debugf(nil, "Unable to drain IMDS response: %v", err)
+		}
+		err = resp.Body.Close()
+		if err != nil {
+			fs.Debugf(nil, "Unable to close IMDS response: %v", err)
+		}
+	}()
+	// Check if the status code indicates success
+	// The request returns 200 currently, add 201 and 202 as well for possible extension.
+	switch resp.StatusCode {
+	case 200, 201, 202:
+		break
+	default:
+		body, _ := ioutil.ReadAll(resp.Body)
+		fs.Errorf(nil, "Couldn't obtain OAuth token from IMDS; server returned status code %d and body: %v", resp.StatusCode, string(body))
+		return result, httpError{Response: resp}
+	}
+
+	b, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return result, fmt.Errorf("couldn't read IMDS response: %w", err)
+	}
+	// Remove BOM, if any. azcopy does this so I'm following along.
+	b = bytes.TrimPrefix(b, []byte("\xef\xbb\xbf"))
+
+	// This would be a good place to persist the token if a large number of rclone
+	// invocations are being made in a short amount of time. If the token is
+	// persisted, the azureblob code will need to check for expiry before every
+	// storage API call.
+	err = json.Unmarshal(b, &result)
+	if err != nil {
+		return result, fmt.Errorf("couldn't unmarshal IMDS response: %w", err)
+	}
+
+	return result, nil
+}

backend/azureblob/imds_test.go

@@ -0,0 +1,118 @@
+//go:build !plan9 && !solaris && !js
+// +build !plan9,!solaris,!js
+
+package azureblob
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+	"strings"
+	"testing"
+
+	"github.com/Azure/go-autorest/autorest/adal"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func handler(t *testing.T, actual *map[string]string) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		err := r.ParseForm()
+		require.NoError(t, err)
+		parameters := r.URL.Query()
+		(*actual)["path"] = r.URL.Path
+		(*actual)["Metadata"] = r.Header.Get("Metadata")
+		(*actual)["method"] = r.Method
+		for paramName := range parameters {
+			(*actual)[paramName] = parameters.Get(paramName)
+		}
+		// Make response.
+		response := adal.Token{}
+		responseBytes, err := json.Marshal(response)
+		require.NoError(t, err)
+		_, err = w.Write(responseBytes)
+		require.NoError(t, err)
+	}
+}
+
+func TestManagedIdentity(t *testing.T) {
+	// test user-assigned identity specifiers to use
+	testMSIClientID := "d859b29f-5c9c-42f8-a327-ec1bc6408d79"
+	testMSIObjectID := "9ffeb650-3ca0-4278-962b-5a38d520591a"
+	testMSIResourceID := "/subscriptions/fe714c49-b8a4-4d49-9388-96a20daa318f/resourceGroups/somerg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/someidentity"
+	tests := []struct {
+		identity              *userMSI
+		identityParameterName string
+		expectedAbsent        []string
+	}{
+		{&userMSI{msiClientID, testMSIClientID}, "client_id", []string{"object_id", "mi_res_id"}},
+		{&userMSI{msiObjectID, testMSIObjectID}, "object_id", []string{"client_id", "mi_res_id"}},
+		{&userMSI{msiResourceID, testMSIResourceID}, "mi_res_id", []string{"object_id", "client_id"}},
+		{nil, "(default)", []string{"object_id", "client_id", "mi_res_id"}},
+	}
+	alwaysExpected := map[string]string{
+		"path":        "/metadata/identity/oauth2/token",
+		"resource":    "https://storage.azure.com",
+		"Metadata":    "true",
+		"api-version": "2018-02-01",
+		"method":      "GET",
+	}
+	for _, test := range tests {
+		actual := make(map[string]string, 10)
+		testServer := httptest.NewServer(handler(t, &actual))
+		defer testServer.Close()
+		testServerPort, err := strconv.Atoi(strings.Split(testServer.URL, ":")[2])
+		require.NoError(t, err)
+		ctx := context.WithValue(context.TODO(), testPortKey("testPort"), testServerPort)
+		_, err = GetMSIToken(ctx, test.identity)
+		require.NoError(t, err)
+
+		// Validate expected query parameters present
+		expected := make(map[string]string)
+		for k, v := range alwaysExpected {
+			expected[k] = v
+		}
+		if test.identity != nil {
+			expected[test.identityParameterName] = test.identity.Value
+		}
+
+		for key := range expected {
+			value, exists := actual[key]
+			if assert.Truef(t, exists, "test of %s: query parameter %s was not passed",
+				test.identityParameterName, key) {
+				assert.Equalf(t, expected[key], value,
+					"test of %s: parameter %s has incorrect value", test.identityParameterName, key)
+			}
+		}
+
+		// Validate unexpected query parameters absent
+		for _, key := range test.expectedAbsent {
+			_, exists := actual[key]
+			assert.Falsef(t, exists, "query parameter %s was unexpectedly passed")
+		}
+	}
+}
+
+func errorHandler(resultCode int) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		http.Error(w, "Test error generated", resultCode)
+	}
+}
+
+func TestIMDSErrors(t *testing.T) {
+	errorCodes := []int{404, 429, 500}
+	for _, code := range errorCodes {
+		testServer := httptest.NewServer(errorHandler(code))
+		defer testServer.Close()
+		testServerPort, err := strconv.Atoi(strings.Split(testServer.URL, ":")[2])
+		require.NoError(t, err)
+		ctx := context.WithValue(context.TODO(), testPortKey("testPort"), testServerPort)
+		_, err = GetMSIToken(ctx, nil)
+		require.Error(t, err)
+		httpErr, ok := err.(httpError)
+		require.Truef(t, ok, "HTTP error %d did not result in an httpError object", code)
+		assert.Equalf(t, httpErr.Response.StatusCode, code, "desired error %d but didn't get it", code)
+	}
+}

backend/b2/b2.go

@@ -1221,7 +1221,7 @@ func (f *Fs) purge(ctx context.Context, dir string, oldOnly bool) error {
 					fs.Errorf(object.Name, "Can't create object %v", err)
 					continue
 				}
-				tr := accounting.Stats(ctx).NewCheckingTransfer(oi, "deleting")
+				tr := accounting.Stats(ctx).NewCheckingTransfer(oi)
 				err = f.deleteByID(ctx, object.ID, object.Name)
 				checkErr(err)
 				tr.Done(ctx, err)
@@ -1235,7 +1235,7 @@ func (f *Fs) purge(ctx context.Context, dir string, oldOnly bool) error {
 			if err != nil {
 				fs.Errorf(object, "Can't create object %+v", err)
 			}
-			tr := accounting.Stats(ctx).NewCheckingTransfer(oi, "checking")
+			tr := accounting.Stats(ctx).NewCheckingTransfer(oi)
 			if oldOnly && last != remote {
 				// Check current version of the file
 				if object.Action == "hide" {

backend/b2/upload.go

@@ -14,7 +14,6 @@ import (
 	"io"
 	"strings"
 	"sync"
-	"time"
 
 	"github.com/rclone/rclone/backend/b2/api"
 	"github.com/rclone/rclone/fs"
@@ -22,7 +21,6 @@ import (
 	"github.com/rclone/rclone/fs/chunksize"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/lib/atexit"
-	"github.com/rclone/rclone/lib/pool"
 	"github.com/rclone/rclone/lib/rest"
 	"golang.org/x/sync/errgroup"
 )
@@ -430,47 +428,18 @@ func (up *largeUpload) Upload(ctx context.Context) (err error) {
 	defer atexit.OnError(&err, func() { _ = up.cancel(ctx) })()
 	fs.Debugf(up.o, "Starting %s of large file in %d chunks (id %q)", up.what, up.parts, up.id)
 	var (
-		g, gCtx    = errgroup.WithContext(ctx)
-		remaining  = up.size
-		uploadPool *pool.Pool
-		ci         = fs.GetConfig(ctx)
+		g, gCtx   = errgroup.WithContext(ctx)
+		remaining = up.size
 	)
-	// If using large chunk size then make a temporary pool
-	if up.chunkSize <= int64(up.f.opt.ChunkSize) {
-		uploadPool = up.f.pool
-	} else {
-		uploadPool = pool.New(
-			time.Duration(up.f.opt.MemoryPoolFlushTime),
-			int(up.chunkSize),
-			ci.Transfers,
-			up.f.opt.MemoryPoolUseMmap,
-		)
-		defer uploadPool.Flush()
-	}
-	// Get an upload token and a buffer
-	getBuf := func() (buf []byte) {
-		up.f.getBuf(true)
-		if !up.doCopy {
-			buf = uploadPool.Get()
-		}
-		return buf
-	}
-	// Put an upload token and a buffer
-	putBuf := func(buf []byte) {
-		if !up.doCopy {
-			uploadPool.Put(buf)
-		}
-		up.f.putBuf(nil, true)
-	}
 	g.Go(func() error {
 		for part := int64(1); part <= up.parts; part++ {
 			// Get a block of memory from the pool and token which limits concurrency.
-			buf := getBuf()
+			buf := up.f.getBuf(up.doCopy)
 
 			// Fail fast, in case an errgroup managed function returns an error
 			// gCtx is cancelled. There is no point in uploading all the other parts.
 			if gCtx.Err() != nil {
-				putBuf(buf)
+				up.f.putBuf(buf, up.doCopy)
 				return nil
 			}
 
@@ -484,14 +453,14 @@ func (up *largeUpload) Upload(ctx context.Context) (err error) {
 				buf = buf[:reqSize]
 				_, err = io.ReadFull(up.in, buf)
 				if err != nil {
-					putBuf(buf)
+					up.f.putBuf(buf, up.doCopy)
 					return err
 				}
 			}
 
 			part := part // for the closure
 			g.Go(func() (err error) {
-				defer putBuf(buf)
+				defer up.f.putBuf(buf, up.doCopy)
 				if !up.doCopy {
 					err = up.transferChunk(gCtx, part, buf)
 				} else {

backend/box/box.go

@@ -17,9 +17,9 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"net/http"
 	"net/url"
-	"os"
 	"path"
 	"strconv"
 	"strings"
@@ -183,7 +183,7 @@ func refreshJWTToken(ctx context.Context, jsonFile string, boxSubType string, na
 }
 
 func getBoxConfig(configFile string) (boxConfig *api.ConfigJSON, err error) {
-	file, err := os.ReadFile(configFile)
+	file, err := ioutil.ReadFile(configFile)
 	if err != nil {
 		return nil, fmt.Errorf("box: failed to read Box config: %w", err)
 	}

backend/cache/cache.go

@@ -1038,7 +1038,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 		}
 		fs.Debugf(dir, "list: remove entry: %v", entryRemote)
 	}
-	entries = nil //nolint:ineffassign
+	entries = nil
 
 	// and then iterate over the ones from source (temp Objects will override source ones)
 	var batchDirectories []*Directory

backend/cache/cache_internal_test.go

@@ -11,6 +11,7 @@ import (
 	goflag "flag"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"log"
 	"math/rand"
 	"os"
@@ -101,12 +102,14 @@ func TestMain(m *testing.M) {
 
 func TestInternalListRootAndInnerRemotes(t *testing.T) {
 	id := fmt.Sprintf("tilrair%v", time.Now().Unix())
-	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, true, true, nil)
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, nil, nil)
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	// Instantiate inner fs
 	innerFolder := "inner"
 	runInstance.mkdir(t, rootFs, innerFolder)
-	rootFs2, _ := runInstance.newCacheFs(t, remoteName, id+"/"+innerFolder, true, true, nil)
+	rootFs2, boltDb2 := runInstance.newCacheFs(t, remoteName, id+"/"+innerFolder, true, true, nil, nil)
+	defer runInstance.cleanupFs(t, rootFs2, boltDb2)
 
 	runInstance.writeObjectString(t, rootFs2, "one", "content")
 	listRoot, err := runInstance.list(t, rootFs, "")
@@ -164,7 +167,7 @@ func TestInternalVfsCache(t *testing.T) {
 			li2 := [2]string{path.Join("test", "one"), path.Join("test", "second")}
 			for _, r := range li2 {
 				var err error
-				ci, err := os.ReadDir(path.Join(runInstance.chunkPath, runInstance.encryptRemoteIfNeeded(t, path.Join(id, r))))
+				ci, err := ioutil.ReadDir(path.Join(runInstance.chunkPath, runInstance.encryptRemoteIfNeeded(t, path.Join(id, r))))
 				if err != nil || len(ci) == 0 {
 					log.Printf("========== '%v' not in cache", r)
 				} else {
@@ -223,7 +226,8 @@ func TestInternalVfsCache(t *testing.T) {
 
 func TestInternalObjWrapFsFound(t *testing.T) {
 	id := fmt.Sprintf("tiowff%v", time.Now().Unix())
-	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, true, true, nil)
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, nil, nil)
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	cfs, err := runInstance.getCacheFs(rootFs)
 	require.NoError(t, err)
@@ -255,7 +259,8 @@ func TestInternalObjWrapFsFound(t *testing.T) {
 
 func TestInternalObjNotFound(t *testing.T) {
 	id := fmt.Sprintf("tionf%v", time.Now().Unix())
-	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, false, true, nil)
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	obj, err := rootFs.NewObject(context.Background(), "404")
 	require.Error(t, err)
@@ -265,7 +270,8 @@ func TestInternalObjNotFound(t *testing.T) {
 func TestInternalCachedWrittenContentMatches(t *testing.T) {
 	testy.SkipUnreliable(t)
 	id := fmt.Sprintf("ticwcm%v", time.Now().Unix())
-	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, false, true, nil)
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	cfs, err := runInstance.getCacheFs(rootFs)
 	require.NoError(t, err)
@@ -292,7 +298,8 @@ func TestInternalDoubleWrittenContentMatches(t *testing.T) {
 		t.Skip("Skip test on windows/386")
 	}
 	id := fmt.Sprintf("tidwcm%v", time.Now().Unix())
-	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, false, true, nil)
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	// write the object
 	runInstance.writeRemoteString(t, rootFs, "one", "one content")
@@ -310,7 +317,8 @@ func TestInternalDoubleWrittenContentMatches(t *testing.T) {
 func TestInternalCachedUpdatedContentMatches(t *testing.T) {
 	testy.SkipUnreliable(t)
 	id := fmt.Sprintf("ticucm%v", time.Now().Unix())
-	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, false, true, nil)
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
 	var err error
 
 	// create some rand test data
@@ -339,7 +347,8 @@ func TestInternalCachedUpdatedContentMatches(t *testing.T) {
 func TestInternalWrappedWrittenContentMatches(t *testing.T) {
 	id := fmt.Sprintf("tiwwcm%v", time.Now().Unix())
 	vfsflags.Opt.DirCacheTime = time.Second
-	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, true, true, nil)
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, nil, nil)
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
 	if runInstance.rootIsCrypt {
 		t.Skip("test skipped with crypt remote")
 	}
@@ -369,7 +378,8 @@ func TestInternalWrappedWrittenContentMatches(t *testing.T) {
 func TestInternalLargeWrittenContentMatches(t *testing.T) {
 	id := fmt.Sprintf("tilwcm%v", time.Now().Unix())
 	vfsflags.Opt.DirCacheTime = time.Second
-	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, true, true, nil)
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, nil, nil)
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
 	if runInstance.rootIsCrypt {
 		t.Skip("test skipped with crypt remote")
 	}
@@ -395,7 +405,8 @@ func TestInternalLargeWrittenContentMatches(t *testing.T) {
 
 func TestInternalWrappedFsChangeNotSeen(t *testing.T) {
 	id := fmt.Sprintf("tiwfcns%v", time.Now().Unix())
-	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, false, true, nil)
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	cfs, err := runInstance.getCacheFs(rootFs)
 	require.NoError(t, err)
@@ -449,7 +460,8 @@ func TestInternalWrappedFsChangeNotSeen(t *testing.T) {
 
 func TestInternalMoveWithNotify(t *testing.T) {
 	id := fmt.Sprintf("timwn%v", time.Now().Unix())
-	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, false, true, nil)
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
 	if !runInstance.wrappedIsExternal {
 		t.Skipf("Not external")
 	}
@@ -535,7 +547,8 @@ func TestInternalMoveWithNotify(t *testing.T) {
 
 func TestInternalNotifyCreatesEmptyParts(t *testing.T) {
 	id := fmt.Sprintf("tincep%v", time.Now().Unix())
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil)
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
 	if !runInstance.wrappedIsExternal {
 		t.Skipf("Not external")
 	}
@@ -621,7 +634,8 @@ func TestInternalNotifyCreatesEmptyParts(t *testing.T) {
 
 func TestInternalChangeSeenAfterDirCacheFlush(t *testing.T) {
 	id := fmt.Sprintf("ticsadcf%v", time.Now().Unix())
-	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, false, true, nil)
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	cfs, err := runInstance.getCacheFs(rootFs)
 	require.NoError(t, err)
@@ -653,7 +667,8 @@ func TestInternalChangeSeenAfterDirCacheFlush(t *testing.T) {
 
 func TestInternalCacheWrites(t *testing.T) {
 	id := "ticw"
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, map[string]string{"writes": "true"})
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, map[string]string{"writes": "true"})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	cfs, err := runInstance.getCacheFs(rootFs)
 	require.NoError(t, err)
@@ -674,7 +689,8 @@ func TestInternalMaxChunkSizeRespected(t *testing.T) {
 		t.Skip("Skip test on windows/386")
 	}
 	id := fmt.Sprintf("timcsr%v", time.Now().Unix())
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, map[string]string{"workers": "1"})
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, map[string]string{"workers": "1"})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	cfs, err := runInstance.getCacheFs(rootFs)
 	require.NoError(t, err)
@@ -709,7 +725,8 @@ func TestInternalMaxChunkSizeRespected(t *testing.T) {
 func TestInternalExpiredEntriesRemoved(t *testing.T) {
 	id := fmt.Sprintf("tieer%v", time.Now().Unix())
 	vfsflags.Opt.DirCacheTime = time.Second * 4 // needs to be lower than the defined
-	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, true, true, nil)
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, map[string]string{"info_age": "5s"}, nil)
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
 	cfs, err := runInstance.getCacheFs(rootFs)
 	require.NoError(t, err)
 
@@ -746,7 +763,9 @@ func TestInternalBug2117(t *testing.T) {
 	vfsflags.Opt.DirCacheTime = time.Second * 10
 
 	id := fmt.Sprintf("tib2117%v", time.Now().Unix())
-	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, false, true, map[string]string{"info_age": "72h", "chunk_clean_interval": "15m"})
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil,
+		map[string]string{"info_age": "72h", "chunk_clean_interval": "15m"})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	if runInstance.rootIsCrypt {
 		t.Skipf("skipping crypt")
@@ -822,7 +841,7 @@ func newRun() *run {
 	}
 
 	if uploadDir == "" {
-		r.tmpUploadDir, err = os.MkdirTemp("", "rclonecache-tmp")
+		r.tmpUploadDir, err = ioutil.TempDir("", "rclonecache-tmp")
 		if err != nil {
 			panic(fmt.Sprintf("Failed to create temp dir: %v", err))
 		}
@@ -847,7 +866,7 @@ func (r *run) encryptRemoteIfNeeded(t *testing.T, remote string) string {
 	return enc
 }
 
-func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool, flags map[string]string) (fs.Fs, *cache.Persistent) {
+func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool, cfg map[string]string, flags map[string]string) (fs.Fs, *cache.Persistent) {
 	fstest.Initialise()
 	remoteExists := false
 	for _, s := range config.FileSections() {
@@ -940,15 +959,10 @@ func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool
 	}
 	err = f.Mkdir(context.Background(), "")
 	require.NoError(t, err)
-
-	t.Cleanup(func() {
-		runInstance.cleanupFs(t, f)
-	})
-
 	return f, boltDb
 }
 
-func (r *run) cleanupFs(t *testing.T, f fs.Fs) {
+func (r *run) cleanupFs(t *testing.T, f fs.Fs, b *cache.Persistent) {
 	err := f.Features().Purge(context.Background(), "")
 	require.NoError(t, err)
 	cfs, err := r.getCacheFs(f)
@@ -970,7 +984,7 @@ func (r *run) randomReader(t *testing.T, size int64) io.ReadCloser {
 	chunk := int64(1024)
 	cnt := size / chunk
 	left := size % chunk
-	f, err := os.CreateTemp("", "rclonecache-tempfile")
+	f, err := ioutil.TempFile("", "rclonecache-tempfile")
 	require.NoError(t, err)
 
 	for i := 0; i < int(cnt); i++ {

backend/cache/cache_upload_test.go

@@ -21,8 +21,10 @@ import (
 
 func TestInternalUploadTempDirCreated(t *testing.T) {
 	id := fmt.Sprintf("tiutdc%v", time.Now().Unix())
-	runInstance.newCacheFs(t, remoteName, id, false, true,
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true,
+		nil,
 		map[string]string{"tmp_upload_path": path.Join(runInstance.tmpUploadDir, id)})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	_, err := os.Stat(path.Join(runInstance.tmpUploadDir, id))
 	require.NoError(t, err)
@@ -61,7 +63,9 @@ func testInternalUploadQueueOneFile(t *testing.T, id string, rootFs fs.Fs, boltD
 func TestInternalUploadQueueOneFileNoRest(t *testing.T) {
 	id := fmt.Sprintf("tiuqofnr%v", time.Now().Unix())
 	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
+		nil,
 		map[string]string{"tmp_upload_path": path.Join(runInstance.tmpUploadDir, id), "tmp_wait_time": "0s"})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	testInternalUploadQueueOneFile(t, id, rootFs, boltDb)
 }
@@ -69,15 +73,19 @@ func TestInternalUploadQueueOneFileNoRest(t *testing.T) {
 func TestInternalUploadQueueOneFileWithRest(t *testing.T) {
 	id := fmt.Sprintf("tiuqofwr%v", time.Now().Unix())
 	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
+		nil,
 		map[string]string{"tmp_upload_path": path.Join(runInstance.tmpUploadDir, id), "tmp_wait_time": "1m"})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	testInternalUploadQueueOneFile(t, id, rootFs, boltDb)
 }
 
 func TestInternalUploadMoveExistingFile(t *testing.T) {
 	id := fmt.Sprintf("tiumef%v", time.Now().Unix())
-	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, true, true,
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
+		nil,
 		map[string]string{"tmp_upload_path": path.Join(runInstance.tmpUploadDir, id), "tmp_wait_time": "3s"})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	err := rootFs.Mkdir(context.Background(), "one")
 	require.NoError(t, err)
@@ -111,8 +119,10 @@ func TestInternalUploadMoveExistingFile(t *testing.T) {
 
 func TestInternalUploadTempPathCleaned(t *testing.T) {
 	id := fmt.Sprintf("tiutpc%v", time.Now().Unix())
-	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, true, true,
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
+		nil,
 		map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "5s"})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	err := rootFs.Mkdir(context.Background(), "one")
 	require.NoError(t, err)
@@ -152,8 +162,10 @@ func TestInternalUploadTempPathCleaned(t *testing.T) {
 
 func TestInternalUploadQueueMoreFiles(t *testing.T) {
 	id := fmt.Sprintf("tiuqmf%v", time.Now().Unix())
-	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, true, true,
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
+		nil,
 		map[string]string{"tmp_upload_path": path.Join(runInstance.tmpUploadDir, id), "tmp_wait_time": "1s"})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	err := rootFs.Mkdir(context.Background(), "test")
 	require.NoError(t, err)
@@ -201,7 +213,9 @@ func TestInternalUploadQueueMoreFiles(t *testing.T) {
 func TestInternalUploadTempFileOperations(t *testing.T) {
 	id := "tiutfo"
 	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
+		nil,
 		map[string]string{"tmp_upload_path": path.Join(runInstance.tmpUploadDir, id), "tmp_wait_time": "1h"})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	boltDb.PurgeTempUploads()
 
@@ -329,7 +343,9 @@ func TestInternalUploadTempFileOperations(t *testing.T) {
 func TestInternalUploadUploadingFileOperations(t *testing.T) {
 	id := "tiuufo"
 	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
+		nil,
 		map[string]string{"tmp_upload_path": path.Join(runInstance.tmpUploadDir, id), "tmp_wait_time": "1h"})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
 
 	boltDb.PurgeTempUploads()
 

backend/cache/plex.go

@@ -8,7 +8,7 @@ import (
 	"crypto/tls"
 	"encoding/json"
 	"fmt"
-	"io"
+	"io/ioutil"
 	"net/http"
 	"net/url"
 	"strings"
@@ -167,7 +167,7 @@ func (p *plexConnector) listenWebsocket() {
 								continue
 							}
 							var data []byte
-							data, err = io.ReadAll(resp.Body)
+							data, err = ioutil.ReadAll(resp.Body)
 							if err != nil {
 								continue
 							}

backend/cache/storage_persistent.go

@@ -9,6 +9,7 @@ import (
 	"encoding/binary"
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"path"
 	"strconv"
@@ -472,7 +473,7 @@ func (b *Persistent) GetChunk(cachedObject *Object, offset int64) ([]byte, error
 	var data []byte
 
 	fp := path.Join(b.dataPath, cachedObject.abs(), strconv.FormatInt(offset, 10))
-	data, err := os.ReadFile(fp)
+	data, err := ioutil.ReadFile(fp)
 	if err != nil {
 		return nil, err
 	}
@@ -485,7 +486,7 @@ func (b *Persistent) AddChunk(fp string, data []byte, offset int64) error {
 	_ = os.MkdirAll(path.Join(b.dataPath, fp), os.ModePerm)
 
 	filePath := path.Join(b.dataPath, fp, strconv.FormatInt(offset, 10))
-	err := os.WriteFile(filePath, data, os.ModePerm)
+	err := ioutil.WriteFile(filePath, data, os.ModePerm)
 	if err != nil {
 		return err
 	}

backend/chunker/chunker.go

@@ -12,6 +12,7 @@ import (
 	"fmt"
 	gohash "hash"
 	"io"
+	"io/ioutil"
 	"math/rand"
 	"path"
 	"regexp"
@@ -1037,7 +1038,7 @@ func (o *Object) readMetadata(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
-	metadata, err := io.ReadAll(reader)
+	metadata, err := ioutil.ReadAll(reader)
 	_ = reader.Close() // ensure file handle is freed on windows
 	if err != nil {
 		return err
@@ -1096,7 +1097,7 @@ func (o *Object) readXactID(ctx context.Context) (xactID string, err error) {
 	if err != nil {
 		return "", err
 	}
-	data, err := io.ReadAll(reader)
+	data, err := ioutil.ReadAll(reader)
 	_ = reader.Close() // ensure file handle is freed on windows
 	if err != nil {
 		return "", err

backend/chunker/chunker_internal_test.go

@@ -5,7 +5,7 @@ import (
 	"context"
 	"flag"
 	"fmt"
-	"io"
+	"io/ioutil"
 	"path"
 	"regexp"
 	"strings"
@@ -413,7 +413,7 @@ func testSmallFileInternals(t *testing.T, f *Fs) {
 		if r == nil {
 			return
 		}
-		data, err := io.ReadAll(r)
+		data, err := ioutil.ReadAll(r)
 		assert.NoError(t, err)
 		assert.Equal(t, contents, string(data))
 		_ = r.Close()
@@ -538,7 +538,7 @@ func testPreventCorruption(t *testing.T, f *Fs) {
 	assert.NoError(t, err)
 	var chunkContents []byte
 	assert.NotPanics(t, func() {
-		chunkContents, err = io.ReadAll(r)
+		chunkContents, err = ioutil.ReadAll(r)
 		_ = r.Close()
 	})
 	assert.NoError(t, err)
@@ -573,7 +573,7 @@ func testPreventCorruption(t *testing.T, f *Fs) {
 	r, err = willyChunk.Open(ctx)
 	assert.NoError(t, err)
 	assert.NotPanics(t, func() {
-		_, err = io.ReadAll(r)
+		_, err = ioutil.ReadAll(r)
 		_ = r.Close()
 	})
 	assert.NoError(t, err)
@@ -672,7 +672,7 @@ func testMetadataInput(t *testing.T, f *Fs) {
 		assert.NoError(t, err, "open "+description)
 		assert.NotNil(t, r, "open stream of "+description)
 		if err == nil && r != nil {
-			data, err := io.ReadAll(r)
+			data, err := ioutil.ReadAll(r)
 			assert.NoError(t, err, "read all of "+description)
 			assert.Equal(t, contents, string(data), description+" contents is ok")
 			_ = r.Close()
@@ -758,8 +758,8 @@ func testFutureProof(t *testing.T, f *Fs) {
 	assert.Error(t, err)
 
 	// Rcat must fail
-	in := io.NopCloser(bytes.NewBufferString("abc"))
-	robj, err := operations.Rcat(ctx, f, file, in, modTime, nil)
+	in := ioutil.NopCloser(bytes.NewBufferString("abc"))
+	robj, err := operations.Rcat(ctx, f, file, in, modTime)
 	assert.Nil(t, robj)
 	assert.NotNil(t, err)
 	if err != nil {
@@ -854,7 +854,7 @@ func testChunkerServerSideMove(t *testing.T, f *Fs) {
 	r, err := dstFile.Open(ctx)
 	assert.NoError(t, err)
 	assert.NotNil(t, r)
-	data, err := io.ReadAll(r)
+	data, err := ioutil.ReadAll(r)
 	assert.NoError(t, err)
 	assert.Equal(t, contents, string(data))
 	_ = r.Close()

backend/combine/combine.go

@@ -631,7 +631,7 @@ func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, stream bo
 	if err != nil {
 		return nil, err
 	}
-	uSrc := fs.NewOverrideRemote(src, uRemote)
+	uSrc := operations.NewOverrideRemote(src, uRemote)
 	var o fs.Object
 	if stream {
 		o, err = u.f.Features().PutStream(ctx, in, uSrc, options...)

backend/compress/compress.go

@@ -13,6 +13,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"os"
 	"regexp"
 	"strings"
@@ -467,7 +468,7 @@ func (f *Fs) rcat(ctx context.Context, dstFileName string, in io.ReadCloser, mod
 	}
 
 	fs.Debugf(f, "Target remote doesn't support streaming uploads, creating temporary local file")
-	tempFile, err := os.CreateTemp("", "rclone-press-")
+	tempFile, err := ioutil.TempFile("", "rclone-press-")
 	defer func() {
 		// these errors should be relatively uncritical and the upload should've succeeded so it's okay-ish
 		// to ignore them
@@ -545,8 +546,8 @@ func (f *Fs) putCompress(ctx context.Context, in io.Reader, src fs.ObjectInfo, o
 	}
 
 	// Transfer the data
-	o, err := f.rcat(ctx, makeDataName(src.Remote(), src.Size(), f.mode), io.NopCloser(wrappedIn), src.ModTime(ctx), options)
-	//o, err := operations.Rcat(ctx, f.Fs, makeDataName(src.Remote(), src.Size(), f.mode), io.NopCloser(wrappedIn), src.ModTime(ctx))
+	o, err := f.rcat(ctx, makeDataName(src.Remote(), src.Size(), f.mode), ioutil.NopCloser(wrappedIn), src.ModTime(ctx), options)
+	//o, err := operations.Rcat(ctx, f.Fs, makeDataName(src.Remote(), src.Size(), f.mode), ioutil.NopCloser(wrappedIn), src.ModTime(ctx))
 	if err != nil {
 		if o != nil {
 			removeErr := o.Remove(ctx)

backend/crypt/cipher_test.go

@@ -8,6 +8,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"strings"
 	"testing"
 
@@ -1072,7 +1073,7 @@ func testEncryptDecrypt(t *testing.T, bufSize int, copySize int64) {
 	source := newRandomSource(copySize)
 	encrypted, err := c.newEncrypter(source, nil)
 	assert.NoError(t, err)
-	decrypted, err := c.newDecrypter(io.NopCloser(encrypted))
+	decrypted, err := c.newDecrypter(ioutil.NopCloser(encrypted))
 	assert.NoError(t, err)
 	sink := newRandomSource(copySize)
 	n, err := io.CopyBuffer(sink, decrypted, buf)
@@ -1143,15 +1144,15 @@ func TestEncryptData(t *testing.T) {
 		buf := bytes.NewBuffer(test.in)
 		encrypted, err := c.EncryptData(buf)
 		assert.NoError(t, err)
-		out, err := io.ReadAll(encrypted)
+		out, err := ioutil.ReadAll(encrypted)
 		assert.NoError(t, err)
 		assert.Equal(t, test.expected, out)
 
 		// Check we can decode the data properly too...
 		buf = bytes.NewBuffer(out)
-		decrypted, err := c.DecryptData(io.NopCloser(buf))
+		decrypted, err := c.DecryptData(ioutil.NopCloser(buf))
 		assert.NoError(t, err)
-		out, err = io.ReadAll(decrypted)
+		out, err = ioutil.ReadAll(decrypted)
 		assert.NoError(t, err)
 		assert.Equal(t, test.in, out)
 	}
@@ -1186,7 +1187,7 @@ func TestNewEncrypterErrUnexpectedEOF(t *testing.T) {
 	fh, err := c.newEncrypter(in, nil)
 	assert.NoError(t, err)
 
-	n, err := io.CopyN(io.Discard, fh, 1e6)
+	n, err := io.CopyN(ioutil.Discard, fh, 1e6)
 	assert.Equal(t, io.ErrUnexpectedEOF, err)
 	assert.Equal(t, int64(32), n)
 }
@@ -1256,12 +1257,12 @@ func TestNewDecrypterErrUnexpectedEOF(t *testing.T) {
 
 	in2 := &readers.ErrorReader{Err: io.ErrUnexpectedEOF}
 	in1 := bytes.NewBuffer(file16)
-	in := io.NopCloser(io.MultiReader(in1, in2))
+	in := ioutil.NopCloser(io.MultiReader(in1, in2))
 
 	fh, err := c.newDecrypter(in)
 	assert.NoError(t, err)
 
-	n, err := io.CopyN(io.Discard, fh, 1e6)
+	n, err := io.CopyN(ioutil.Discard, fh, 1e6)
 	assert.Equal(t, io.ErrUnexpectedEOF, err)
 	assert.Equal(t, int64(16), n)
 }
@@ -1273,14 +1274,14 @@ func TestNewDecrypterSeekLimit(t *testing.T) {
 
 	// Make random data
 	const dataSize = 150000
-	plaintext, err := io.ReadAll(newRandomSource(dataSize))
+	plaintext, err := ioutil.ReadAll(newRandomSource(dataSize))
 	assert.NoError(t, err)
 
 	// Encrypt the data
 	buf := bytes.NewBuffer(plaintext)
 	encrypted, err := c.EncryptData(buf)
 	assert.NoError(t, err)
-	ciphertext, err := io.ReadAll(encrypted)
+	ciphertext, err := ioutil.ReadAll(encrypted)
 	assert.NoError(t, err)
 
 	trials := []int{0, 1, 2, 3, 4, 5, 7, 8, 9, 15, 16, 17, 31, 32, 33, 63, 64, 65,
@@ -1299,7 +1300,7 @@ func TestNewDecrypterSeekLimit(t *testing.T) {
 				end = len(ciphertext)
 			}
 		}
-		reader = io.NopCloser(bytes.NewBuffer(ciphertext[int(underlyingOffset):end]))
+		reader = ioutil.NopCloser(bytes.NewBuffer(ciphertext[int(underlyingOffset):end]))
 		return reader, nil
 	}
 
@@ -1489,7 +1490,7 @@ func TestDecrypterRead(t *testing.T) {
 			assert.NoError(t, err, what)
 			continue
 		}
-		_, err = io.ReadAll(fh)
+		_, err = ioutil.ReadAll(fh)
 		var expectedErr error
 		switch {
 		case i == fileHeaderSize:
@@ -1513,7 +1514,7 @@ func TestDecrypterRead(t *testing.T) {
 	cd := newCloseDetector(in)
 	fh, err := c.newDecrypter(cd)
 	assert.NoError(t, err)
-	_, err = io.ReadAll(fh)
+	_, err = ioutil.ReadAll(fh)
 	assert.Error(t, err, "potato")
 	assert.Equal(t, 0, cd.closed)
 
@@ -1523,13 +1524,13 @@ func TestDecrypterRead(t *testing.T) {
 	copy(file16copy, file16)
 	for i := range file16copy {
 		file16copy[i] ^= 0xFF
-		fh, err := c.newDecrypter(io.NopCloser(bytes.NewBuffer(file16copy)))
+		fh, err := c.newDecrypter(ioutil.NopCloser(bytes.NewBuffer(file16copy)))
 		if i < fileMagicSize {
 			assert.Error(t, err, ErrorEncryptedBadMagic.Error())
 			assert.Nil(t, fh)
 		} else {
 			assert.NoError(t, err)
-			_, err = io.ReadAll(fh)
+			_, err = ioutil.ReadAll(fh)
 			assert.Error(t, err, ErrorEncryptedFileBadHeader.Error())
 		}
 		file16copy[i] ^= 0xFF
@@ -1564,7 +1565,7 @@ func TestDecrypterClose(t *testing.T) {
 	assert.Equal(t, 0, cd.closed)
 
 	// close after reading
-	out, err := io.ReadAll(fh)
+	out, err := ioutil.ReadAll(fh)
 	assert.NoError(t, err)
 	assert.Equal(t, []byte{1}, out)
 	assert.Equal(t, io.EOF, fh.err)

backend/crypt/crypt.go

@@ -235,7 +235,7 @@ func NewFs(ctx context.Context, name, rpath string, m configmap.Mapper) (fs.Fs,
 	// the features here are ones we could support, and they are
 	// ANDed with the ones from wrappedFs
 	f.features = (&fs.Features{
-		CaseInsensitive:         !cipher.dirNameEncrypt || cipher.NameEncryptionMode() == NameEncryptionOff,
+		CaseInsensitive:         cipher.NameEncryptionMode() == NameEncryptionOff,
 		DuplicateFiles:          true,
 		ReadMimeType:            false, // MimeTypes not supported with crypt
 		WriteMimeType:           false,
@@ -396,8 +396,6 @@ type putFn func(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ..
 
 // put implements Put or PutStream
 func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options []fs.OpenOption, put putFn) (fs.Object, error) {
-	ci := fs.GetConfig(ctx)
-
 	if f.opt.NoDataEncryption {
 		o, err := put(ctx, in, f.newObjectInfo(src, nonce{}), options...)
 		if err == nil && o != nil {
@@ -415,9 +413,6 @@ func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options [
 	// Find a hash the destination supports to compute a hash of
 	// the encrypted data
 	ht := f.Fs.Hashes().GetOne()
-	if ci.IgnoreChecksum {
-		ht = hash.None
-	}
 	var hasher *hash.MultiHasher
 	if ht != hash.None {
 		hasher, err = hash.NewMultiHasherTypes(hash.NewHashSet(ht))
@@ -1052,11 +1047,10 @@ func (o *ObjectInfo) Hash(ctx context.Context, hash hash.Type) (string, error) {
 	// Get the underlying object if there is one
 	if srcObj, ok = o.ObjectInfo.(fs.Object); ok {
 		// Prefer direct interface assertion
-	} else if do, ok := o.ObjectInfo.(*fs.OverrideRemote); ok {
-		// Unwrap if it is an operations.OverrideRemote
+	} else if do, ok := o.ObjectInfo.(fs.ObjectUnWrapper); ok {
+		// Otherwise likely is an operations.OverrideRemote
 		srcObj = do.UnWrap()
 	} else {
-		// Otherwise don't unwrap any further
 		return "", nil
 	}
 	// if this is wrapping a local object then we work out the hash

backend/crypt/crypt_internal_test.go

@@ -17,28 +17,41 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+type testWrapper struct {
+	fs.ObjectInfo
+}
+
+// UnWrap returns the Object that this Object is wrapping or nil if it
+// isn't wrapping anything
+func (o testWrapper) UnWrap() fs.Object {
+	if o, ok := o.ObjectInfo.(fs.Object); ok {
+		return o
+	}
+	return nil
+}
+
 // Create a temporary local fs to upload things from
 
-func makeTempLocalFs(t *testing.T) (localFs fs.Fs) {
+func makeTempLocalFs(t *testing.T) (localFs fs.Fs, cleanup func()) {
 	localFs, err := fs.TemporaryLocalFs(context.Background())
 	require.NoError(t, err)
-	t.Cleanup(func() {
+	cleanup = func() {
 		require.NoError(t, localFs.Rmdir(context.Background(), ""))
-	})
-	return localFs
+	}
+	return localFs, cleanup
 }
 
 // Upload a file to a remote
-func uploadFile(t *testing.T, f fs.Fs, remote, contents string) (obj fs.Object) {
+func uploadFile(t *testing.T, f fs.Fs, remote, contents string) (obj fs.Object, cleanup func()) {
 	inBuf := bytes.NewBufferString(contents)
 	t1 := time.Date(2012, time.December, 17, 18, 32, 31, 0, time.UTC)
 	upSrc := object.NewStaticObjectInfo(remote, t1, int64(len(contents)), true, nil, nil)
 	obj, err := f.Put(context.Background(), inBuf, upSrc)
 	require.NoError(t, err)
-	t.Cleanup(func() {
+	cleanup = func() {
 		require.NoError(t, obj.Remove(context.Background()))
-	})
-	return obj
+	}
+	return obj, cleanup
 }
 
 // Test the ObjectInfo
@@ -52,9 +65,11 @@ func testObjectInfo(t *testing.T, f *Fs, wrap bool) {
 		path = "_wrap"
 	}
 
-	localFs := makeTempLocalFs(t)
+	localFs, cleanupLocalFs := makeTempLocalFs(t)
+	defer cleanupLocalFs()
 
-	obj := uploadFile(t, localFs, path, contents)
+	obj, cleanupObj := uploadFile(t, localFs, path, contents)
+	defer cleanupObj()
 
 	// encrypt the data
 	inBuf := bytes.NewBufferString(contents)
@@ -68,7 +83,7 @@ func testObjectInfo(t *testing.T, f *Fs, wrap bool) {
 	var oi fs.ObjectInfo = obj
 	if wrap {
 		// wrap the object in an fs.ObjectUnwrapper if required
-		oi = fs.NewOverrideRemote(oi, "new_remote")
+		oi = testWrapper{oi}
 	}
 
 	// wrap the object in a crypt for upload using the nonce we
@@ -101,13 +116,16 @@ func testComputeHash(t *testing.T, f *Fs) {
 		t.Skipf("%v: does not support hashes", f.Fs)
 	}
 
-	localFs := makeTempLocalFs(t)
+	localFs, cleanupLocalFs := makeTempLocalFs(t)
+	defer cleanupLocalFs()
 
 	// Upload a file to localFs as a test object
-	localObj := uploadFile(t, localFs, path, contents)
+	localObj, cleanupLocalObj := uploadFile(t, localFs, path, contents)
+	defer cleanupLocalObj()
 
 	// Upload the same data to the remote Fs also
-	remoteObj := uploadFile(t, f, path, contents)
+	remoteObj, cleanupRemoteObj := uploadFile(t, f, path, contents)
+	defer cleanupRemoteObj()
 
 	// Calculate the expected Hash of the remote object
 	computedHash, err := f.ComputeHash(ctx, remoteObj.(*Object), localObj, hashType)

backend/drive/drive.go

@@ -14,10 +14,11 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"mime"
 	"net/http"
-	"os"
 	"path"
+	"regexp"
 	"sort"
 	"strconv"
 	"strings"
@@ -451,11 +452,7 @@ If downloading a file returns the error "This file has been identified
 as malware or spam and cannot be downloaded" with the error code
 "cannotDownloadAbusiveFile" then supply this flag to rclone to
 indicate you acknowledge the risks of downloading the file and rclone
-will download it anyway.
-
-Note that if you are using service account it will need Manager
-permission (not Content Manager) to for this flag to work. If the SA
-does not have the right permission, Google will just ignore the flag.`,
+will download it anyway.`,
 			Advanced: true,
 		}, {
 			Name:     "keep_revision_forever",
@@ -761,7 +758,7 @@ func (f *Fs) shouldRetry(ctx context.Context, err error) (bool, error) {
 			} else if f.opt.StopOnDownloadLimit && reason == "downloadQuotaExceeded" {
 				fs.Errorf(f, "Received download limit error: %v", err)
 				return false, fserrors.FatalError(err)
-			} else if f.opt.StopOnUploadLimit && (reason == "quotaExceeded" || reason == "storageQuotaExceeded") {
+			} else if f.opt.StopOnUploadLimit && reason == "quotaExceeded" {
 				fs.Errorf(f, "Received upload limit error: %v", err)
 				return false, fserrors.FatalError(err)
 			} else if f.opt.StopOnUploadLimit && reason == "teamDriveFileLimitExceeded" {
@@ -1111,7 +1108,7 @@ func createOAuthClient(ctx context.Context, opt *Options, name string, m configm
 
 	// try loading service account credentials from env variable, then from a file
 	if len(opt.ServiceAccountCredentials) == 0 && opt.ServiceAccountFile != "" {
-		loadedCreds, err := os.ReadFile(env.ShellExpand(opt.ServiceAccountFile))
+		loadedCreds, err := ioutil.ReadFile(env.ShellExpand(opt.ServiceAccountFile))
 		if err != nil {
 			return nil, fmt.Errorf("error opening service account credentials file: %w", err)
 		}
@@ -3326,9 +3323,9 @@ This takes an optional directory to trash which make this easier to
 use via the API.
 
     rclone backend untrash drive:directory
-    rclone backend --interactive untrash drive:directory subdir
+    rclone backend -i untrash drive:directory subdir
 
-Use the --interactive/-i or --dry-run flag to see what would be restored before restoring it.
+Use the -i flag to see what would be restored before restoring it.
 
 Result:
 
@@ -3358,7 +3355,7 @@ component will be used as the file name.
 If the destination is a drive backend then server-side copying will be
 attempted if possible.
 
-Use the --interactive/-i or --dry-run flag to see what would be copied before copying.
+Use the -i flag to see what would be copied before copying.
 `,
 }, {
 	Name:  "exportformats",
@@ -3434,12 +3431,13 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str
 		if err != nil {
 			return nil, err
 		}
+		re := regexp.MustCompile(`[^\w_. -]+`)
 		if _, ok := opt["config"]; ok {
 			lines := []string{}
 			upstreams := []string{}
 			names := make(map[string]struct{}, len(drives))
 			for i, drive := range drives {
-				name := fspath.MakeConfigName(drive.Name)
+				name := re.ReplaceAllString(drive.Name, "_")
 				for {
 					if _, found := names[name]; !found {
 						break
@@ -3802,7 +3800,7 @@ func (o *linkObject) Open(ctx context.Context, options ...fs.OpenOption) (in io.
 		data = data[:limit]
 	}
 
-	return io.NopCloser(bytes.NewReader(data)), nil
+	return ioutil.NopCloser(bytes.NewReader(data)), nil
 }
 
 func (o *baseObject) update(ctx context.Context, updateInfo *drive.File, uploadMimeType string, in io.Reader,

backend/drive/drive_internal_test.go

@@ -7,6 +7,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"mime"
 	"os"
 	"path"
@@ -77,7 +78,7 @@ var additionalMimeTypes = map[string]string{
 // Load the example export formats into exportFormats for testing
 func TestInternalLoadExampleFormats(t *testing.T) {
 	fetchFormatsOnce.Do(func() {})
-	buf, err := os.ReadFile(filepath.FromSlash("test/about.json"))
+	buf, err := ioutil.ReadFile(filepath.FromSlash("test/about.json"))
 	var about struct {
 		ExportFormats map[string][]string `json:"exportFormats,omitempty"`
 		ImportFormats map[string][]string `json:"importFormats,omitempty"`
@@ -243,15 +244,6 @@ func (f *Fs) InternalTestShouldRetry(t *testing.T) {
 	quotaExceededRetry, quotaExceededError := f.shouldRetry(ctx, &generic403)
 	assert.False(t, quotaExceededRetry)
 	assert.Equal(t, quotaExceededError, expectedQuotaError)
-
-	sqEItem := googleapi.ErrorItem{
-		Reason: "storageQuotaExceeded",
-	}
-	generic403.Errors[0] = sqEItem
-	expectedStorageQuotaError := fserrors.FatalError(&generic403)
-	storageQuotaExceededRetry, storageQuotaExceededError := f.shouldRetry(ctx, &generic403)
-	assert.False(t, storageQuotaExceededRetry)
-	assert.Equal(t, storageQuotaExceededError, expectedStorageQuotaError)
 }
 
 func (f *Fs) InternalTestDocumentImport(t *testing.T) {

backend/filefabric/filefabric.go

@@ -20,6 +20,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"net/http"
 	"net/url"
 	"path"
@@ -1185,7 +1186,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 		return nil, errors.New("can't download - no id")
 	}
 	if o.contentType == emptyMimeType {
-		return io.NopCloser(bytes.NewReader([]byte{})), nil
+		return ioutil.NopCloser(bytes.NewReader([]byte{})), nil
 	}
 	fs.FixRangeOption(options, o.size)
 	resp, err := o.fs.rpc(ctx, "getFile", params{

backend/ftp/ftp.go

@@ -15,7 +15,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/jlaffaye/ftp"
+	"github.com/rclone/ftp"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
 	"github.com/rclone/rclone/fs/config"
@@ -70,7 +70,7 @@ func init() {
 When using implicit FTP over TLS the client connects using TLS
 right from the start which breaks compatibility with
 non-TLS-aware servers. This is usually served over port 990 rather
-than port 21. Cannot be used in combination with explicit FTPS.`,
+than port 21. Cannot be used in combination with explicit FTP.`,
 			Default: false,
 		}, {
 			Name: "explicit_tls",
@@ -78,7 +78,7 @@ than port 21. Cannot be used in combination with explicit FTPS.`,
 
 When using explicit FTP over TLS the client explicitly requests
 security from the server in order to upgrade a plain text connection
-to an encrypted one. Cannot be used in combination with implicit FTPS.`,
+to an encrypted one. Cannot be used in combination with implicit FTP.`,
 			Default: false,
 		}, {
 			Name: "concurrency",
@@ -315,33 +315,18 @@ func (dl *debugLog) Write(p []byte) (n int, err error) {
 	return len(p), nil
 }
 
-// Return a *textproto.Error if err contains one or nil otherwise
-func textprotoError(err error) (errX *textproto.Error) {
-	if errors.As(err, &errX) {
-		return errX
-	}
-	return nil
-}
-
-// returns true if this FTP error should be retried
-func isRetriableFtpError(err error) bool {
-	if errX := textprotoError(err); errX != nil {
-		switch errX.Code {
-		case ftp.StatusNotAvailable, ftp.StatusTransfertAborted:
-			return true
-		}
-	}
-	return false
-}
-
 // shouldRetry returns a boolean as to whether this err deserve to be
 // retried.  It returns the err as a convenience
 func shouldRetry(ctx context.Context, err error) (bool, error) {
 	if fserrors.ContextError(ctx, &err) {
 		return false, err
 	}
-	if isRetriableFtpError(err) {
-		return true, err
+	switch errX := err.(type) {
+	case *textproto.Error:
+		switch errX.Code {
+		case ftp.StatusNotAvailable:
+			return true, err
+		}
 	}
 	return fserrors.ShouldRetry(err), err
 }
@@ -478,7 +463,8 @@ func (f *Fs) putFtpConnection(pc **ftp.ServerConn, err error) {
 	*pc = nil
 	if err != nil {
 		// If not a regular FTP error code then check the connection
-		if tpErr := textprotoError(err); tpErr != nil {
+		var tpErr *textproto.Error
+		if !errors.As(err, &tpErr) {
 			nopErr := c.NoOp()
 			if nopErr != nil {
 				fs.Debugf(f, "Connection failed, closing: %v", nopErr)
@@ -627,7 +613,8 @@ func (f *Fs) Shutdown(ctx context.Context) error {
 
 // translateErrorFile turns FTP errors into rclone errors if possible for a file
 func translateErrorFile(err error) error {
-	if errX := textprotoError(err); errX != nil {
+	switch errX := err.(type) {
+	case *textproto.Error:
 		switch errX.Code {
 		case ftp.StatusFileUnavailable, ftp.StatusFileActionIgnored:
 			err = fs.ErrorObjectNotFound
@@ -638,7 +625,8 @@ func translateErrorFile(err error) error {
 
 // translateErrorDir turns FTP errors into rclone errors if possible for a directory
 func translateErrorDir(err error) error {
-	if errX := textprotoError(err); errX != nil {
+	switch errX := err.(type) {
+	case *textproto.Error:
 		switch errX.Code {
 		case ftp.StatusFileUnavailable, ftp.StatusFileActionIgnored:
 			err = fs.ErrorDirNotFound
@@ -669,7 +657,8 @@ func (f *Fs) dirFromStandardPath(dir string) string {
 // findItem finds a directory entry for the name in its parent directory
 func (f *Fs) findItem(ctx context.Context, remote string) (entry *ftp.Entry, err error) {
 	// defer fs.Trace(remote, "")("o=%v, err=%v", &o, &err)
-	if remote == "" || remote == "." || remote == "/" {
+	fullPath := path.Join(f.root, remote)
+	if fullPath == "" || fullPath == "." || fullPath == "/" {
 		// if root, assume exists and synthesize an entry
 		return &ftp.Entry{
 			Name: "",
@@ -677,32 +666,13 @@ func (f *Fs) findItem(ctx context.Context, remote string) (entry *ftp.Entry, err
 			Time: time.Now(),
 		}, nil
 	}
+	dir := path.Dir(fullPath)
+	base := path.Base(fullPath)
 
 	c, err := f.getFtpConnection(ctx)
 	if err != nil {
 		return nil, fmt.Errorf("findItem: %w", err)
 	}
-
-	// returns TRUE if MLST is supported which is required to call GetEntry
-	if c.IsTimePreciseInList() {
-		entry, err := c.GetEntry(f.opt.Enc.FromStandardPath(remote))
-		f.putFtpConnection(&c, err)
-		if err != nil {
-			err = translateErrorFile(err)
-			if err == fs.ErrorObjectNotFound {
-				return nil, nil
-			}
-			return nil, err
-		}
-		if entry != nil {
-			f.entryToStandard(entry)
-		}
-		return entry, nil
-	}
-
-	dir := path.Dir(remote)
-	base := path.Base(remote)
-
 	files, err := c.List(f.dirFromStandardPath(dir))
 	f.putFtpConnection(&c, err)
 	if err != nil {
@@ -721,7 +691,7 @@ func (f *Fs) findItem(ctx context.Context, remote string) (entry *ftp.Entry, err
 // it returns the error fs.ErrorObjectNotFound.
 func (f *Fs) NewObject(ctx context.Context, remote string) (o fs.Object, err error) {
 	// defer fs.Trace(remote, "")("o=%v, err=%v", &o, &err)
-	entry, err := f.findItem(ctx, path.Join(f.root, remote))
+	entry, err := f.findItem(ctx, remote)
 	if err != nil {
 		return nil, err
 	}
@@ -743,7 +713,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (o fs.Object, err err
 
 // dirExists checks the directory pointed to by remote exists or not
 func (f *Fs) dirExists(ctx context.Context, remote string) (exists bool, err error) {
-	entry, err := f.findItem(ctx, path.Join(f.root, remote))
+	entry, err := f.findItem(ctx, remote)
 	if err != nil {
 		return false, fmt.Errorf("dirExists: %w", err)
 	}
@@ -887,18 +857,32 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt
 // getInfo reads the FileInfo for a path
 func (f *Fs) getInfo(ctx context.Context, remote string) (fi *FileInfo, err error) {
 	// defer fs.Trace(remote, "")("fi=%v, err=%v", &fi, &err)
-	file, err := f.findItem(ctx, remote)
+	dir := path.Dir(remote)
+	base := path.Base(remote)
+
+	c, err := f.getFtpConnection(ctx)
 	if err != nil {
-		return nil, err
-	} else if file != nil {
-		info := &FileInfo{
-			Name:    remote,
-			Size:    file.Size,
-			ModTime: file.Time,
-			precise: f.fLstTime,
-			IsDir:   file.Type == ftp.EntryTypeFolder,
+		return nil, fmt.Errorf("getInfo: %w", err)
+	}
+	files, err := c.List(f.dirFromStandardPath(dir))
+	f.putFtpConnection(&c, err)
+	if err != nil {
+		return nil, translateErrorFile(err)
+	}
+
+	for i := range files {
+		file := files[i]
+		f.entryToStandard(file)
+		if file.Name == base {
+			info := &FileInfo{
+				Name:    remote,
+				Size:    file.Size,
+				ModTime: file.Time,
+				precise: f.fLstTime,
+				IsDir:   file.Type == ftp.EntryTypeFolder,
+			}
+			return info, nil
 		}
-		return info, nil
 	}
 	return nil, fs.ErrorObjectNotFound
 }
@@ -929,7 +913,8 @@ func (f *Fs) mkdir(ctx context.Context, abspath string) error {
 	}
 	err = c.MakeDir(f.dirFromStandardPath(abspath))
 	f.putFtpConnection(&c, err)
-	if errX := textprotoError(err); errX != nil {
+	switch errX := err.(type) {
+	case *textproto.Error:
 		switch errX.Code {
 		case ftp.StatusFileUnavailable: // dir already exists: see issue #2181
 			err = nil
@@ -1170,7 +1155,8 @@ func (f *ftpReadCloser) Close() error {
 	// mask the error if it was caused by a premature close
 	// NB StatusAboutToSend is to work around a bug in pureftpd
 	// See: https://github.com/rclone/rclone/issues/3445#issuecomment-521654257
-	if errX := textprotoError(err); errX != nil {
+	switch errX := err.(type) {
+	case *textproto.Error:
 		switch errX.Code {
 		case ftp.StatusTransfertAborted, ftp.StatusFileUnavailable, ftp.StatusAboutToSend:
 			err = nil
@@ -1196,26 +1182,15 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.Read
 			}
 		}
 	}
-
-	var (
-		fd *ftp.Response
-		c  *ftp.ServerConn
-	)
-	err = o.fs.pacer.Call(func() (bool, error) {
-		c, err = o.fs.getFtpConnection(ctx)
-		if err != nil {
-			return false, err // getFtpConnection has retries already
-		}
-		fd, err = c.RetrFrom(o.fs.opt.Enc.FromStandardPath(path), uint64(offset))
-		if err != nil {
-			o.fs.putFtpConnection(&c, err)
-		}
-		return shouldRetry(ctx, err)
-	})
+	c, err := o.fs.getFtpConnection(ctx)
 	if err != nil {
 		return nil, fmt.Errorf("open: %w", err)
 	}
-
+	fd, err := c.RetrFrom(o.fs.opt.Enc.FromStandardPath(path), uint64(offset))
+	if err != nil {
+		o.fs.putFtpConnection(&c, err)
+		return nil, fmt.Errorf("open: %w", err)
+	}
 	rc = &ftpReadCloser{rc: readers.NewLimitedReadCloser(fd, limit), c: c, f: o.fs}
 	return rc, nil
 }
@@ -1248,10 +1223,13 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	}
 	err = c.Stor(o.fs.opt.Enc.FromStandardPath(path), in)
 	// Ignore error 250 here - send by some servers
-	if errX := textprotoError(err); errX != nil {
-		switch errX.Code {
-		case ftp.StatusRequestedFileActionOK:
-			err = nil
+	if err != nil {
+		switch errX := err.(type) {
+		case *textproto.Error:
+			switch errX.Code {
+			case ftp.StatusRequestedFileActionOK:
+				err = nil
+			}
 		}
 	}
 	if err != nil {

backend/googlecloudstorage/googlecloudstorage.go

@@ -19,8 +19,8 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"net/http"
-	"os"
 	"path"
 	"strconv"
 	"strings"
@@ -82,8 +82,7 @@ func init() {
 			saFile, _ := m.Get("service_account_file")
 			saCreds, _ := m.Get("service_account_credentials")
 			anonymous, _ := m.Get("anonymous")
-			envAuth, _ := m.Get("env_auth")
-			if saFile != "" || saCreds != "" || anonymous == "true" || envAuth == "true" {
+			if saFile != "" || saCreds != "" || anonymous == "true" {
 				return nil, nil
 			}
 			return oauthutil.ConfigOut("", &oauthutil.Options{
@@ -331,17 +330,6 @@ can't check the size and hash but the file contents will be decompressed.
 			Default: (encoder.Base |
 				encoder.EncodeCrLf |
 				encoder.EncodeInvalidUtf8),
-		}, {
-			Name:    "env_auth",
-			Help:    "Get GCP IAM credentials from runtime (environment variables or instance meta data if no env vars).\n\nOnly applies if service_account_file and service_account_credentials is blank.",
-			Default: false,
-			Examples: []fs.OptionExample{{
-				Value: "false",
-				Help:  "Enter credentials in the next step.",
-			}, {
-				Value: "true",
-				Help:  "Get GCP IAM credentials from the environment (env vars or IAM).",
-			}},
 		}}...),
 	})
 }
@@ -361,7 +349,6 @@ type Options struct {
 	Decompress                bool                 `config:"decompress"`
 	Endpoint                  string               `config:"endpoint"`
 	Enc                       encoder.MultiEncoder `config:"encoding"`
-	EnvAuth                   bool                 `config:"env_auth"`
 }
 
 // Fs represents a remote storage server
@@ -500,7 +487,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 
 	// try loading service account credentials from env variable, then from a file
 	if opt.ServiceAccountCredentials == "" && opt.ServiceAccountFile != "" {
-		loadedCreds, err := os.ReadFile(env.ShellExpand(opt.ServiceAccountFile))
+		loadedCreds, err := ioutil.ReadFile(env.ShellExpand(opt.ServiceAccountFile))
 		if err != nil {
 			return nil, fmt.Errorf("error opening service account credentials file: %w", err)
 		}
@@ -513,11 +500,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		if err != nil {
 			return nil, fmt.Errorf("failed configuring Google Cloud Storage Service Account: %w", err)
 		}
-	} else if opt.EnvAuth {
-		oAuthClient, err = google.DefaultClient(ctx, storage.DevstorageFullControlScope)
-		if err != nil {
-			return nil, fmt.Errorf("failed to configure Google Cloud Storage: %w", err)
-		}
 	} else {
 		oAuthClient, _, err = oauthutil.NewClient(ctx, name, m, storageConfig)
 		if err != nil {

backend/googlephotos/googlephotos_test.go

@@ -3,7 +3,7 @@ package googlephotos
 import (
 	"context"
 	"fmt"
-	"io"
+	"io/ioutil"
 	"net/http"
 	"path"
 	"testing"
@@ -12,6 +12,7 @@ import (
 	_ "github.com/rclone/rclone/backend/local"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/fs/operations"
 	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/lib/random"
 	"github.com/stretchr/testify/assert"
@@ -55,7 +56,7 @@ func TestIntegration(t *testing.T) {
 			require.NoError(t, err)
 			in, err := srcObj.Open(ctx)
 			require.NoError(t, err)
-			dstObj, err := f.Put(ctx, in, fs.NewOverrideRemote(srcObj, remote))
+			dstObj, err := f.Put(ctx, in, operations.NewOverrideRemote(srcObj, remote))
 			require.NoError(t, err)
 			assert.Equal(t, remote, dstObj.Remote())
 			_ = in.Close()
@@ -98,7 +99,7 @@ func TestIntegration(t *testing.T) {
 			t.Run("ObjectOpen", func(t *testing.T) {
 				in, err := dstObj.Open(ctx)
 				require.NoError(t, err)
-				buf, err := io.ReadAll(in)
+				buf, err := ioutil.ReadAll(in)
 				require.NoError(t, err)
 				require.NoError(t, in.Close())
 				assert.True(t, len(buf) > 1000)
@@ -220,7 +221,7 @@ func TestIntegration(t *testing.T) {
 		require.NoError(t, err)
 		in, err := srcObj.Open(ctx)
 		require.NoError(t, err)
-		dstObj, err := f.Put(ctx, in, fs.NewOverrideRemote(srcObj, remote))
+		dstObj, err := f.Put(ctx, in, operations.NewOverrideRemote(srcObj, remote))
 		require.NoError(t, err)
 		assert.Equal(t, remote, dstObj.Remote())
 		_ = in.Close()

backend/hasher/commands.go

@@ -161,7 +161,7 @@ func (f *Fs) dbImport(ctx context.Context, hashName, sumRemote string, sticky bo
 			if err := o.putHashes(ctx, hashMap{hashType: hash}); err != nil {
 				fs.Errorf(nil, "%s: failed to import: %v", remote, err)
 			}
-			accounting.Stats(ctx).NewCheckingTransfer(obj, "importing").Done(ctx, err)
+			accounting.Stats(ctx).NewCheckingTransfer(obj).Done(ctx, err)
 			doneCount++
 		}
 	})

backend/hasher/object.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"path"
 	"time"
 
@@ -117,7 +118,7 @@ func (o *Object) updateHashes(ctx context.Context) error {
 	defer func() {
 		_ = r.Close()
 	}()
-	if _, err = io.Copy(io.Discard, r); err != nil {
+	if _, err = io.Copy(ioutil.Discard, r); err != nil {
 		fs.Infof(o, "update failed (copy): %v", err)
 		return err
 	}

backend/http/http.go

@@ -13,6 +13,7 @@ import (
 	"net/http"
 	"net/url"
 	"path"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -304,7 +305,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 		fs:     f,
 		remote: remote,
 	}
-	err := o.head(ctx)
+	err := o.stat(ctx)
 	if err != nil {
 		return nil, err
 	}
@@ -316,6 +317,15 @@ func (f *Fs) url(remote string) string {
 	return f.endpointURL + rest.URLPathEscape(remote)
 }
 
+// parse s into an int64, on failure return def
+func parseInt64(s string, def int64) int64 {
+	n, e := strconv.ParseInt(s, 10, 64)
+	if e != nil {
+		return def
+	}
+	return n
+}
+
 // Errors returned by parseName
 var (
 	errURLJoinFailed     = errors.New("URLJoin failed")
@@ -490,7 +500,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 					fs:     f,
 					remote: remote,
 				}
-				switch err := file.head(ctx); err {
+				switch err := file.stat(ctx); err {
 				case nil:
 					add(file)
 				case fs.ErrorNotAFile:
@@ -569,8 +579,8 @@ func (o *Object) url() string {
 	return o.fs.url(o.remote)
 }
 
-// head sends a HEAD request to update info fields in the Object
-func (o *Object) head(ctx context.Context) error {
+// stat updates the info field in the Object
+func (o *Object) stat(ctx context.Context) error {
 	if o.fs.opt.NoHead {
 		o.size = -1
 		o.modTime = timeUnset
@@ -591,19 +601,13 @@ func (o *Object) head(ctx context.Context) error {
 	if err != nil {
 		return fmt.Errorf("failed to stat: %w", err)
 	}
-	return o.decodeMetadata(ctx, res)
-}
-
-// decodeMetadata updates info fields in the Object according to HTTP response headers
-func (o *Object) decodeMetadata(ctx context.Context, res *http.Response) error {
 	t, err := http.ParseTime(res.Header.Get("Last-Modified"))
 	if err != nil {
 		t = timeUnset
 	}
+	o.size = parseInt64(res.Header.Get("Content-Length"), -1)
 	o.modTime = t
 	o.contentType = res.Header.Get("Content-Type")
-	o.size = rest.ParseSizeFromHeaders(res.Header)
-
 	// If NoSlash is set then check ContentType to see if it is a directory
 	if o.fs.opt.NoSlash {
 		mediaType, _, err := mime.ParseMediaType(o.contentType)
@@ -649,9 +653,6 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	if err != nil {
 		return nil, fmt.Errorf("Open failed: %w", err)
 	}
-	if err = o.decodeMetadata(ctx, res); err != nil {
-		return nil, fmt.Errorf("decodeMetadata failed: %w", err)
-	}
 	return res.Body, nil
 }
 

backend/http/http_internal_test.go

@@ -3,7 +3,7 @@ package http
 import (
 	"context"
 	"fmt"
-	"io"
+	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
@@ -33,21 +33,20 @@ var (
 	lineEndSize = 1
 )
 
-// prepareServer prepares the test server and shuts it down automatically
-// when the test completes.
-func prepareServer(t *testing.T) configmap.Simple {
+// prepareServer the test server and return a function to tidy it up afterwards
+func prepareServer(t *testing.T) (configmap.Simple, func()) {
 	// file server for test/files
 	fileServer := http.FileServer(http.Dir(filesPath))
 
 	// verify the file path is correct, and also check which line endings
 	// are used to get sizes right ("\n" except on Windows, but even there
 	// we may have "\n" or "\r\n" depending on git crlf setting)
-	fileList, err := os.ReadDir(filesPath)
+	fileList, err := ioutil.ReadDir(filesPath)
 	require.NoError(t, err)
 	require.Greater(t, len(fileList), 0)
 	for _, file := range fileList {
 		if !file.IsDir() {
-			data, _ := os.ReadFile(filepath.Join(filesPath, file.Name()))
+			data, _ := ioutil.ReadFile(filepath.Join(filesPath, file.Name()))
 			if strings.HasSuffix(string(data), "\r\n") {
 				lineEndSize = 2
 			}
@@ -79,21 +78,20 @@ func prepareServer(t *testing.T) configmap.Simple {
 		"url":     ts.URL,
 		"headers": strings.Join(headers, ","),
 	}
-	t.Cleanup(ts.Close)
 
-	return m
+	// return a function to tidy up
+	return m, ts.Close
 }
 
-// prepare prepares the test server and shuts it down automatically
-// when the test completes.
-func prepare(t *testing.T) fs.Fs {
-	m := prepareServer(t)
+// prepare the test server and return a function to tidy it up afterwards
+func prepare(t *testing.T) (fs.Fs, func()) {
+	m, tidy := prepareServer(t)
 
 	// Instantiate it
 	f, err := NewFs(context.Background(), remoteName, "", m)
 	require.NoError(t, err)
 
-	return f
+	return f, tidy
 }
 
 func testListRoot(t *testing.T, f fs.Fs, noSlash bool) {
@@ -136,19 +134,22 @@ func testListRoot(t *testing.T, f fs.Fs, noSlash bool) {
 }
 
 func TestListRoot(t *testing.T) {
-	f := prepare(t)
+	f, tidy := prepare(t)
+	defer tidy()
 	testListRoot(t, f, false)
 }
 
 func TestListRootNoSlash(t *testing.T) {
-	f := prepare(t)
+	f, tidy := prepare(t)
 	f.(*Fs).opt.NoSlash = true
+	defer tidy()
 
 	testListRoot(t, f, true)
 }
 
 func TestListSubDir(t *testing.T) {
-	f := prepare(t)
+	f, tidy := prepare(t)
+	defer tidy()
 
 	entries, err := f.List(context.Background(), "three")
 	require.NoError(t, err)
@@ -165,7 +166,8 @@ func TestListSubDir(t *testing.T) {
 }
 
 func TestNewObject(t *testing.T) {
-	f := prepare(t)
+	f, tidy := prepare(t)
+	defer tidy()
 
 	o, err := f.NewObject(context.Background(), "four/under four.txt")
 	require.NoError(t, err)
@@ -192,69 +194,36 @@ func TestNewObject(t *testing.T) {
 }
 
 func TestOpen(t *testing.T) {
-	m := prepareServer(t)
+	f, tidy := prepare(t)
+	defer tidy()
 
-	for _, head := range []bool{false, true} {
-		if !head {
-			m.Set("no_head", "true")
-		}
-		f, err := NewFs(context.Background(), remoteName, "", m)
-		require.NoError(t, err)
+	o, err := f.NewObject(context.Background(), "four/under four.txt")
+	require.NoError(t, err)
 
-		for _, rangeRead := range []bool{false, true} {
-			o, err := f.NewObject(context.Background(), "four/under four.txt")
-			require.NoError(t, err)
-
-			if !head {
-				// Test mod time is still indeterminate
-				tObj := o.ModTime(context.Background())
-				assert.Equal(t, time.Duration(0), time.Unix(0, 0).Sub(tObj))
-
-				// Test file size is still indeterminate
-				assert.Equal(t, int64(-1), o.Size())
-			}
-
-			var data []byte
-			if !rangeRead {
-				// Test normal read
-				fd, err := o.Open(context.Background())
-				require.NoError(t, err)
-				data, err = io.ReadAll(fd)
-				require.NoError(t, err)
-				require.NoError(t, fd.Close())
-				if lineEndSize == 2 {
-					assert.Equal(t, "beetroot\r\n", string(data))
-				} else {
-					assert.Equal(t, "beetroot\n", string(data))
-				}
-			} else {
-				// Test with range request
-				fd, err := o.Open(context.Background(), &fs.RangeOption{Start: 1, End: 5})
-				require.NoError(t, err)
-				data, err = io.ReadAll(fd)
-				require.NoError(t, err)
-				require.NoError(t, fd.Close())
-				assert.Equal(t, "eetro", string(data))
-			}
-
-			fi, err := os.Stat(filepath.Join(filesPath, "four", "under four.txt"))
-			require.NoError(t, err)
-			tFile := fi.ModTime()
-
-			// Test the time is always correct on the object after file open
-			tObj := o.ModTime(context.Background())
-			fstest.AssertTimeEqualWithPrecision(t, o.Remote(), tFile, tObj, time.Second)
-
-			if !rangeRead {
-				// Test the file size
-				assert.Equal(t, int64(len(data)), o.Size())
-			}
-		}
+	// Test normal read
+	fd, err := o.Open(context.Background())
+	require.NoError(t, err)
+	data, err := ioutil.ReadAll(fd)
+	require.NoError(t, err)
+	require.NoError(t, fd.Close())
+	if lineEndSize == 2 {
+		assert.Equal(t, "beetroot\r\n", string(data))
+	} else {
+		assert.Equal(t, "beetroot\n", string(data))
 	}
+
+	// Test with range request
+	fd, err = o.Open(context.Background(), &fs.RangeOption{Start: 1, End: 5})
+	require.NoError(t, err)
+	data, err = ioutil.ReadAll(fd)
+	require.NoError(t, err)
+	require.NoError(t, fd.Close())
+	assert.Equal(t, "eetro", string(data))
 }
 
 func TestMimeType(t *testing.T) {
-	f := prepare(t)
+	f, tidy := prepare(t)
+	defer tidy()
 
 	o, err := f.NewObject(context.Background(), "four/under four.txt")
 	require.NoError(t, err)
@@ -265,7 +234,8 @@ func TestMimeType(t *testing.T) {
 }
 
 func TestIsAFileRoot(t *testing.T) {
-	m := prepareServer(t)
+	m, tidy := prepareServer(t)
+	defer tidy()
 
 	f, err := NewFs(context.Background(), remoteName, "one%.txt", m)
 	assert.Equal(t, err, fs.ErrorIsFile)
@@ -274,7 +244,8 @@ func TestIsAFileRoot(t *testing.T) {
 }
 
 func TestIsAFileSubDir(t *testing.T) {
-	m := prepareServer(t)
+	m, tidy := prepareServer(t)
+	defer tidy()
 
 	f, err := NewFs(context.Background(), remoteName, "three/underthree.txt", m)
 	assert.Equal(t, err, fs.ErrorIsFile)

backend/jottacloud/jottacloud.go

@@ -12,6 +12,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"math/rand"
 	"net/http"
 	"net/url"
@@ -821,7 +822,7 @@ func (f *Fs) allocatePathRaw(file string, absolute bool) string {
 func grantTypeFilter(req *http.Request) {
 	if legacyTokenURL == req.URL.String() {
 		// read the entire body
-		refreshBody, err := io.ReadAll(req.Body)
+		refreshBody, err := ioutil.ReadAll(req.Body)
 		if err != nil {
 			return
 		}
@@ -831,7 +832,7 @@ func grantTypeFilter(req *http.Request) {
 		refreshBody = []byte(strings.Replace(string(refreshBody), "grant_type=refresh_token", "grant_type=REFRESH_TOKEN", 1))
 
 		// set the new ReadCloser (with a dummy Close())
-		req.Body = io.NopCloser(bytes.NewReader(refreshBody))
+		req.Body = ioutil.NopCloser(bytes.NewReader(refreshBody))
 	}
 }
 
@@ -1788,7 +1789,7 @@ func readMD5(in io.Reader, size, threshold int64) (md5sum string, out io.Reader,
 		var tempFile *os.File
 
 		// create the cache file
-		tempFile, err = os.CreateTemp("", cachePrefix)
+		tempFile, err = ioutil.TempFile("", cachePrefix)
 		if err != nil {
 			return
 		}
@@ -1816,7 +1817,7 @@ func readMD5(in io.Reader, size, threshold int64) (md5sum string, out io.Reader,
 	} else {
 		// that's a small file, just read it into memory
 		var inData []byte
-		inData, err = io.ReadAll(teeReader)
+		inData, err = ioutil.ReadAll(teeReader)
 		if err != nil {
 			return
 		}
@@ -1913,7 +1914,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 		// copy the already uploaded bytes into the trash :)
 		var result api.UploadResponse
-		_, err = io.CopyN(io.Discard, in, response.ResumePos)
+		_, err = io.CopyN(ioutil.Discard, in, response.ResumePos)
 		if err != nil {
 			return err
 		}

backend/local/local.go

@@ -7,6 +7,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"os"
 	"path"
 	"path/filepath"
@@ -123,8 +124,8 @@ routine so this flag shouldn't normally be used.`,
 			Help: `Don't check to see if the files change during upload.
 
 Normally rclone checks the size and modification time of files as they
-are being uploaded and aborts with a message which starts "can't copy -
-source file is being updated" if the file changes during upload.
+are being uploaded and aborts with a message which starts "can't copy
+- source file is being updated" if the file changes during upload.
 
 However on some file systems this modification time check may fail (e.g.
 [Glusterfs #2206](https://github.com/rclone/rclone/issues/2206)) so this
@@ -503,7 +504,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 								continue
 							}
 						}
-						err = fmt.Errorf("failed to read directory %q: %w", namepath, fierr)
+						err = fmt.Errorf("failed to read directory %q: %w", namepath, err)
 						fs.Errorf(dir, "%v", fierr)
 						_ = accounting.Stats(ctx).Error(fserrors.NoRetryError(fierr)) // fail the sync
 						continue
@@ -524,10 +525,6 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 			if f.opt.FollowSymlinks && (mode&os.ModeSymlink) != 0 {
 				localPath := filepath.Join(fsDirPath, name)
 				fi, err = os.Stat(localPath)
-				// Quietly skip errors on excluded files and directories
-				if err != nil && useFilter && !filter.IncludeRemote(newRemote) {
-					continue
-				}
 				if os.IsNotExist(err) || isCircularSymlinkError(err) {
 					// Skip bad symlinks and circular symlinks
 					err = fserrors.NoRetryError(fmt.Errorf("symlink: %w", err))
@@ -649,7 +646,7 @@ func (f *Fs) readPrecision() (precision time.Duration) {
 	precision = time.Second
 
 	// Create temporary file and test it
-	fd, err := os.CreateTemp("", "rclone")
+	fd, err := ioutil.TempFile("", "rclone")
 	if err != nil {
 		// If failed return 1s
 		// fmt.Println("Failed to create temp file", err)
@@ -1076,7 +1073,7 @@ func (o *Object) openTranslatedLink(offset, limit int64) (lrc io.ReadCloser, err
 	if err != nil {
 		return nil, err
 	}
-	return readers.NewLimitedReadCloser(io.NopCloser(strings.NewReader(linkdst[offset:])), limit), nil
+	return readers.NewLimitedReadCloser(ioutil.NopCloser(strings.NewReader(linkdst[offset:])), limit), nil
 }
 
 // Open an object for read

backend/local/local_internal_test.go

@@ -4,7 +4,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
-	"io"
+	"io/ioutil"
 	"os"
 	"path"
 	"path/filepath"
@@ -14,7 +14,6 @@ import (
 	"time"
 
 	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/accounting"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/filter"
 	"github.com/rclone/rclone/fs/hash"
@@ -34,6 +33,7 @@ func TestMain(m *testing.M) {
 // Test copy with source file that's updating
 func TestUpdatingCheck(t *testing.T) {
 	r := fstest.NewRun(t)
+	defer r.Finalise()
 	filePath := "sub dir/local test"
 	r.WriteFile(filePath, "content", time.Now())
 
@@ -78,6 +78,7 @@ func TestUpdatingCheck(t *testing.T) {
 func TestSymlink(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
+	defer r.Finalise()
 	f := r.Flocal.(*Fs)
 	dir := f.root
 
@@ -149,7 +150,7 @@ func TestSymlink(t *testing.T) {
 	// Check reading the object
 	in, err := o.Open(ctx)
 	require.NoError(t, err)
-	contents, err := io.ReadAll(in)
+	contents, err := ioutil.ReadAll(in)
 	require.NoError(t, err)
 	require.Equal(t, "file.txt", string(contents))
 	require.NoError(t, in.Close())
@@ -157,7 +158,7 @@ func TestSymlink(t *testing.T) {
 	// Check reading the object with range
 	in, err = o.Open(ctx, &fs.RangeOption{Start: 2, End: 5})
 	require.NoError(t, err)
-	contents, err = io.ReadAll(in)
+	contents, err = ioutil.ReadAll(in)
 	require.NoError(t, err)
 	require.Equal(t, "file.txt"[2:5+1], string(contents))
 	require.NoError(t, in.Close())
@@ -176,6 +177,7 @@ func TestSymlinkError(t *testing.T) {
 func TestHashOnUpdate(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
+	defer r.Finalise()
 	const filePath = "file.txt"
 	when := time.Now()
 	r.WriteFile(filePath, "content", when)
@@ -206,6 +208,7 @@ func TestHashOnUpdate(t *testing.T) {
 func TestHashOnDelete(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
+	defer r.Finalise()
 	const filePath = "file.txt"
 	when := time.Now()
 	r.WriteFile(filePath, "content", when)
@@ -234,6 +237,7 @@ func TestHashOnDelete(t *testing.T) {
 func TestMetadata(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
+	defer r.Finalise()
 	const filePath = "metafile.txt"
 	when := time.Now()
 	const dayLength = len("2001-01-01")
@@ -368,6 +372,7 @@ func TestMetadata(t *testing.T) {
 func TestFilter(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
+	defer r.Finalise()
 	when := time.Now()
 	r.WriteFile("included", "included file", when)
 	r.WriteFile("excluded", "excluded file", when)
@@ -396,73 +401,3 @@ func TestFilter(t *testing.T) {
 	sort.Sort(entries)
 	require.Equal(t, "[included]", fmt.Sprint(entries))
 }
-
-func TestFilterSymlink(t *testing.T) {
-	ctx := context.Background()
-	r := fstest.NewRun(t)
-	defer r.Finalise()
-	when := time.Now()
-	f := r.Flocal.(*Fs)
-
-	// Create a file, a directory, a symlink to a file, a symlink to a directory and a dangling symlink
-	r.WriteFile("included.file", "included file", when)
-	r.WriteFile("included.dir/included.sub.file", "included sub file", when)
-	require.NoError(t, os.Symlink("included.file", filepath.Join(r.LocalName, "included.file.link")))
-	require.NoError(t, os.Symlink("included.dir", filepath.Join(r.LocalName, "included.dir.link")))
-	require.NoError(t, os.Symlink("dangling", filepath.Join(r.LocalName, "dangling.link")))
-
-	// Set fs into "-L" mode
-	f.opt.FollowSymlinks = true
-	f.opt.TranslateSymlinks = false
-	f.lstat = os.Stat
-
-	// Set fs into "-l" mode
-	// f.opt.FollowSymlinks = false
-	// f.opt.TranslateSymlinks = true
-	// f.lstat = os.Lstat
-
-	// Check set up for filtering
-	assert.True(t, f.Features().FilterAware)
-
-	// Reset global error count
-	accounting.Stats(ctx).ResetErrors()
-	assert.Equal(t, int64(0), accounting.Stats(ctx).GetErrors(), "global errors found")
-
-	// Add a filter
-	ctx, fi := filter.AddConfig(ctx)
-	require.NoError(t, fi.AddRule("+ included.file"))
-	require.NoError(t, fi.AddRule("+ included.file.link"))
-	require.NoError(t, fi.AddRule("+ included.dir/**"))
-	require.NoError(t, fi.AddRule("+ included.dir.link/**"))
-	require.NoError(t, fi.AddRule("- *"))
-
-	// Check listing without use filter flag
-	entries, err := f.List(ctx, "")
-	require.NoError(t, err)
-
-	// Check 1 global errors one for each dangling symlink
-	assert.Equal(t, int64(1), accounting.Stats(ctx).GetErrors(), "global errors found")
-	accounting.Stats(ctx).ResetErrors()
-
-	sort.Sort(entries)
-	require.Equal(t, "[included.dir included.dir.link included.file included.file.link]", fmt.Sprint(entries))
-
-	// Add user filter flag
-	ctx = filter.SetUseFilter(ctx, true)
-
-	// Check listing with use filter flag
-	entries, err = f.List(ctx, "")
-	require.NoError(t, err)
-	assert.Equal(t, int64(0), accounting.Stats(ctx).GetErrors(), "global errors found")
-
-	sort.Sort(entries)
-	require.Equal(t, "[included.dir included.dir.link included.file included.file.link]", fmt.Sprint(entries))
-
-	// Check listing through a symlink still works
-	entries, err = f.List(ctx, "included.dir")
-	require.NoError(t, err)
-	assert.Equal(t, int64(0), accounting.Stats(ctx).GetErrors(), "global errors found")
-
-	sort.Sort(entries)
-	require.Equal(t, "[included.dir/included.sub.file]", fmt.Sprint(entries))
-}

backend/local/remove_test.go

@@ -1,6 +1,7 @@
 package local
 
 import (
+	"io/ioutil"
 	"os"
 	"sync"
 	"testing"
@@ -12,7 +13,7 @@ import (
 
 // Check we can remove an open file
 func TestRemove(t *testing.T) {
-	fd, err := os.CreateTemp("", "rclone-remove-test")
+	fd, err := ioutil.TempFile("", "rclone-remove-test")
 	require.NoError(t, err)
 	name := fd.Name()
 	defer func() {

backend/mailru/mailru.go

@@ -18,6 +18,7 @@ import (
 
 	"encoding/hex"
 	"encoding/json"
+	"io/ioutil"
 	"net/http"
 	"net/url"
 
@@ -1659,7 +1660,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 	// Attempt to put by calculating hash in memory
 	if trySpeedup && size <= int64(o.fs.opt.SpeedupMaxMem) {
-		fileBuf, err = io.ReadAll(in)
+		fileBuf, err = ioutil.ReadAll(in)
 		if err != nil {
 			return err
 		}
@@ -1702,7 +1703,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	if size <= mrhash.Size {
 		// Optimize upload: skip extra request if data fits in the hash buffer.
 		if fileBuf == nil {
-			fileBuf, err = io.ReadAll(wrapIn)
+			fileBuf, err = ioutil.ReadAll(wrapIn)
 		}
 		if fileHash == nil && err == nil {
 			fileHash = mrhash.Sum(fileBuf)
@@ -2213,7 +2214,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 		fs.Debugf(o, "Server returned full content instead of range")
 		if start > 0 {
 			// Discard the beginning of the data
-			_, err = io.CopyN(io.Discard, wrapStream, start)
+			_, err = io.CopyN(ioutil.Discard, wrapStream, start)
 			if err != nil {
 				closeBody(res)
 				return nil, err

backend/mega/mega.go

@@ -83,17 +83,6 @@ than permanently deleting them.  If you specify this then rclone will
 permanently delete objects instead.`,
 			Default:  false,
 			Advanced: true,
-		}, {
-			Name: "use_https",
-			Help: `Use HTTPS for transfers.
-
-MEGA uses plain text HTTP connections by default.
-Some ISPs throttle HTTP connections, this causes transfers to become very slow.
-Enabling this will force MEGA to use HTTPS for all transfers.
-HTTPS is normally not necesary since all data is already encrypted anyway.
-Enabling it will increase CPU usage and add network overhead.`,
-			Default:  false,
-			Advanced: true,
 		}, {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
@@ -111,7 +100,6 @@ type Options struct {
 	Pass       string               `config:"pass"`
 	Debug      bool                 `config:"debug"`
 	HardDelete bool                 `config:"hard_delete"`
-	UseHTTPS   bool                 `config:"use_https"`
 	Enc        encoder.MultiEncoder `config:"encoding"`
 }
 
@@ -216,7 +204,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	if srv == nil {
 		srv = mega.New().SetClient(fshttp.NewClient(ctx))
 		srv.SetRetries(ci.LowLevelRetries) // let mega do the low level retries
-		srv.SetHTTPS(opt.UseHTTPS)
 		srv.SetLogger(func(format string, v ...interface{}) {
 			fs.Infof("*go-mega*", format, v...)
 		})

backend/memory/memory.go

@@ -8,6 +8,7 @@ import (
 	"encoding/hex"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"path"
 	"strings"
 	"sync"
@@ -574,7 +575,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 		}
 		data = data[:limit]
 	}
-	return io.NopCloser(bytes.NewBuffer(data)), nil
+	return ioutil.NopCloser(bytes.NewBuffer(data)), nil
 }
 
 // Update the object with the contents of the io.Reader, modTime and size
@@ -582,7 +583,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 // The new object may have been created if an error is returned
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
 	bucket, bucketPath := o.split()
-	data, err := io.ReadAll(in)
+	data, err := ioutil.ReadAll(in)
 	if err != nil {
 		return fmt.Errorf("failed to update memory object: %w", err)
 	}

backend/netstorage/netstorage.go

@@ -12,6 +12,7 @@ import (
 	"fmt"
 	gohash "hash"
 	"io"
+	"io/ioutil"
 	"math/rand"
 	"net/http"
 	"net/url"
@@ -971,7 +972,7 @@ func (o *Object) netStorageUploadRequest(ctx context.Context, in io.Reader, src
 		URL = o.fs.url(src.Remote())
 	}
 	if strings.HasSuffix(URL, ".rclonelink") {
-		bits, err := io.ReadAll(in)
+		bits, err := ioutil.ReadAll(in)
 		if err != nil {
 			return err
 		}
@@ -1057,7 +1058,7 @@ func (o *Object) netStorageDownloadRequest(ctx context.Context, options []fs.Ope
 	if strings.HasSuffix(URL, ".rclonelink") && o.target != "" {
 		fs.Infof(nil, "Converting a symlink to the rclonelink file on download %q", URL)
 		reader := strings.NewReader(o.target)
-		readcloser := io.NopCloser(reader)
+		readcloser := ioutil.NopCloser(reader)
 		return readcloser, nil
 	}
 

backend/onedrive/api/types.go

@@ -126,7 +126,6 @@ type HashesType struct {
 	Sha1Hash     string `json:"sha1Hash"`     // hex encoded SHA1 hash for the contents of the file (if available)
 	Crc32Hash    string `json:"crc32Hash"`    // hex encoded CRC32 value of the file (if available)
 	QuickXorHash string `json:"quickXorHash"` // base64 encoded QuickXorHash value of the file (if available)
-	Sha256Hash   string `json:"sha256Hash"`   // hex encoded SHA256 value of the file (if available)
 }
 
 // FileFacet groups file-related data on OneDrive into a single structure.

backend/onedrive/onedrive.go

@@ -259,48 +259,6 @@ this flag there.
 At the time of writing this only works with OneDrive personal paid accounts.
 `,
 			Advanced: true,
-		}, {
-			Name:    "hash_type",
-			Default: "auto",
-			Help: `Specify the hash in use for the backend.
-
-This specifies the hash type in use. If set to "auto" it will use the
-default hash which is is QuickXorHash.
-
-Before rclone 1.62 an SHA1 hash was used by default for Onedrive
-Personal. For 1.62 and later the default is to use a QuickXorHash for
-all onedrive types. If an SHA1 hash is desired then set this option
-accordingly.
-
-From July 2023 QuickXorHash will be the only available hash for
-both OneDrive for Business and OneDriver Personal.
-
-This can be set to "none" to not use any hashes.
-
-If the hash requested does not exist on the object, it will be
-returned as an empty string which is treated as a missing hash by
-rclone.
-`,
-			Examples: []fs.OptionExample{{
-				Value: "auto",
-				Help:  "Rclone chooses the best hash",
-			}, {
-				Value: "quickxor",
-				Help:  "QuickXor",
-			}, {
-				Value: "sha1",
-				Help:  "SHA1",
-			}, {
-				Value: "sha256",
-				Help:  "SHA256",
-			}, {
-				Value: "crc32",
-				Help:  "CRC32",
-			}, {
-				Value: "none",
-				Help:  "None - don't use any hashes",
-			}},
-			Advanced: true,
 		}, {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
@@ -553,7 +511,7 @@ Example: "https://contoso.sharepoint.com/sites/mysite" or "mysite"
 `)
 	case "url_end":
 		siteURL := config.Result
-		re := regexp.MustCompile(`https://.*\.sharepoint\.com/sites/(.*)`)
+		re := regexp.MustCompile(`https://.*\.sharepoint.com/sites/(.*)`)
 		match := re.FindStringSubmatch(siteURL)
 		if len(match) == 2 {
 			return chooseDrive(ctx, name, m, srv, chooseDriveOpt{
@@ -639,7 +597,6 @@ type Options struct {
 	LinkScope               string               `config:"link_scope"`
 	LinkType                string               `config:"link_type"`
 	LinkPassword            string               `config:"link_password"`
-	HashType                string               `config:"hash_type"`
 	Enc                     encoder.MultiEncoder `config:"encoding"`
 }
 
@@ -656,7 +613,6 @@ type Fs struct {
 	tokenRenewer *oauthutil.Renew   // renew the token on expiry
 	driveID      string             // ID to use for querying Microsoft Graph
 	driveType    string             // https://developer.microsoft.com/en-us/graph/docs/api-reference/v1.0/resources/drive
-	hashType     hash.Type          // type of the hash we are using
 }
 
 // Object describes a OneDrive object
@@ -670,7 +626,8 @@ type Object struct {
 	size          int64     // size of the object
 	modTime       time.Time // modification time of the object
 	id            string    // ID of the object
-	hash          string    // Hash of the content, usually QuickXorHash but set as hash_type
+	sha1          string    // SHA-1 of the object content
+	quickxorhash  string    // QuickXorHash of the object content
 	mimeType      string    // Content-Type of object from server (may not be as uploaded)
 }
 
@@ -925,7 +882,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		driveType: opt.DriveType,
 		srv:       rest.NewClient(oAuthClient).SetRoot(rootURL),
 		pacer:     fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
-		hashType:  QuickXorHashType,
 	}
 	f.features = (&fs.Features{
 		CaseInsensitive:         true,
@@ -935,15 +891,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	}).Fill(ctx, f)
 	f.srv.SetErrorHandler(errorHandler)
 
-	// Set the user defined hash
-	if opt.HashType == "auto" || opt.HashType == "" {
-		opt.HashType = QuickXorHashType.String()
-	}
-	err = f.hashType.Set(opt.HashType)
-	if err != nil {
-		return nil, err
-	}
-
 	// Disable change polling in China region
 	// See: https://github.com/rclone/rclone/issues/6444
 	if f.opt.Region == regionCN {
@@ -1609,7 +1556,10 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
 
 // Hashes returns the supported hash sets.
 func (f *Fs) Hashes() hash.Set {
-	return hash.Set(f.hashType)
+	if f.driveType == driveTypePersonal {
+		return hash.Set(hash.SHA1)
+	}
+	return hash.Set(QuickXorHashType)
 }
 
 // PublicLink returns a link for downloading without account.
@@ -1818,8 +1768,14 @@ func (o *Object) rootPath() string {
 
 // Hash returns the SHA-1 of an object returning a lowercase hex string
 func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
-	if t == o.fs.hashType {
-		return o.hash, nil
+	if o.fs.driveType == driveTypePersonal {
+		if t == hash.SHA1 {
+			return o.sha1, nil
+		}
+	} else {
+		if t == QuickXorHashType {
+			return o.quickxorhash, nil
+		}
 	}
 	return "", hash.ErrUnsupported
 }
@@ -1850,23 +1806,16 @@ func (o *Object) setMetaData(info *api.Item) (err error) {
 	file := info.GetFile()
 	if file != nil {
 		o.mimeType = file.MimeType
-		o.hash = ""
-		switch o.fs.hashType {
-		case QuickXorHashType:
-			if file.Hashes.QuickXorHash != "" {
-				h, err := base64.StdEncoding.DecodeString(file.Hashes.QuickXorHash)
-				if err != nil {
-					fs.Errorf(o, "Failed to decode QuickXorHash %q: %v", file.Hashes.QuickXorHash, err)
-				} else {
-					o.hash = hex.EncodeToString(h)
-				}
+		if file.Hashes.Sha1Hash != "" {
+			o.sha1 = strings.ToLower(file.Hashes.Sha1Hash)
+		}
+		if file.Hashes.QuickXorHash != "" {
+			h, err := base64.StdEncoding.DecodeString(file.Hashes.QuickXorHash)
+			if err != nil {
+				fs.Errorf(o, "Failed to decode QuickXorHash %q: %v", file.Hashes.QuickXorHash, err)
+			} else {
+				o.quickxorhash = hex.EncodeToString(h)
 			}
-		case hash.SHA1:
-			o.hash = strings.ToLower(file.Hashes.Sha1Hash)
-		case hash.SHA256:
-			o.hash = strings.ToLower(file.Hashes.Sha256Hash)
-		case hash.CRC32:
-			o.hash = strings.ToLower(file.Hashes.Crc32Hash)
 		}
 	}
 	fileSystemInfo := info.GetFileSystemInfo()

backend/onedrive/quickxorhash/quickxorhash.go

@@ -7,40 +7,51 @@
 // See: https://docs.microsoft.com/en-us/onedrive/developer/code-snippets/quickxorhash
 package quickxorhash
 
-// This code was ported from a fast C-implementation from
-// https://github.com/namazso/QuickXorHash
-// which has licenced as BSD Zero Clause License
-//
-// BSD Zero Clause License
-//
-// Copyright (c) 2022 namazso <admin@namazso.eu>
-//
-// Permission to use, copy, modify, and/or distribute this software for any
-// purpose with or without fee is hereby granted.
-//
-// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
-// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
-// AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
-// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
-// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
-// OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
-// PERFORMANCE OF THIS SOFTWARE.
+// This code was ported from the code snippet linked from
+// https://docs.microsoft.com/en-us/onedrive/developer/code-snippets/quickxorhash
+// Which has the copyright
 
-import "hash"
+// ------------------------------------------------------------------------------
+//  Copyright (c) 2016 Microsoft Corporation
+//
+//  Permission is hereby granted, free of charge, to any person obtaining a copy
+//  of this software and associated documentation files (the "Software"), to deal
+//  in the Software without restriction, including without limitation the rights
+//  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+//  copies of the Software, and to permit persons to whom the Software is
+//  furnished to do so, subject to the following conditions:
+//
+//  The above copyright notice and this permission notice shall be included in
+//  all copies or substantial portions of the Software.
+//
+//  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+//  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+//  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+//  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+//  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+//  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+//  THE SOFTWARE.
+// ------------------------------------------------------------------------------
+
+import (
+	"hash"
+)
 
 const (
 	// BlockSize is the preferred size for hashing
 	BlockSize = 64
 	// Size of the output checksum
-	Size        = 20
-	shift       = 11
-	widthInBits = 8 * Size
-	dataSize    = shift * widthInBits
+	Size           = 20
+	bitsInLastCell = 32
+	shift          = 11
+	widthInBits    = 8 * Size
+	dataSize       = (widthInBits-1)/64 + 1
 )
 
 type quickXorHash struct {
-	data [dataSize]byte
-	size uint64
+	data        [dataSize]uint64
+	lengthSoFar uint64
+	shiftSoFar  int
 }
 
 // New returns a new hash.Hash computing the quickXorHash checksum.
@@ -59,37 +70,94 @@ func New() hash.Hash {
 //
 // Implementations must not retain p.
 func (q *quickXorHash) Write(p []byte) (n int, err error) {
-	var i int
-	// fill last remain
-	lastRemain := int(q.size) % dataSize
-	if lastRemain != 0 {
-		i += xorBytes(q.data[lastRemain:], p)
+	currentshift := q.shiftSoFar
+
+	// The bitvector where we'll start xoring
+	vectorArrayIndex := currentshift / 64
+
+	// The position within the bit vector at which we begin xoring
+	vectorOffset := currentshift % 64
+	iterations := len(p)
+	if iterations > widthInBits {
+		iterations = widthInBits
 	}
 
-	if i != len(p) {
-		for len(p)-i >= dataSize {
-			i += xorBytes(q.data[:], p[i:])
+	for i := 0; i < iterations; i++ {
+		isLastCell := vectorArrayIndex == len(q.data)-1
+		var bitsInVectorCell int
+		if isLastCell {
+			bitsInVectorCell = bitsInLastCell
+		} else {
+			bitsInVectorCell = 64
+		}
+
+		// There's at least 2 bitvectors before we reach the end of the array
+		if vectorOffset <= bitsInVectorCell-8 {
+			for j := i; j < len(p); j += widthInBits {
+				q.data[vectorArrayIndex] ^= uint64(p[j]) << uint(vectorOffset)
+			}
+		} else {
+			index1 := vectorArrayIndex
+			var index2 int
+			if isLastCell {
+				index2 = 0
+			} else {
+				index2 = vectorArrayIndex + 1
+			}
+			low := byte(bitsInVectorCell - vectorOffset)
+
+			xoredByte := byte(0)
+			for j := i; j < len(p); j += widthInBits {
+				xoredByte ^= p[j]
+			}
+			q.data[index1] ^= uint64(xoredByte) << uint(vectorOffset)
+			q.data[index2] ^= uint64(xoredByte) >> low
+		}
+		vectorOffset += shift
+		for vectorOffset >= bitsInVectorCell {
+			if isLastCell {
+				vectorArrayIndex = 0
+			} else {
+				vectorArrayIndex = vectorArrayIndex + 1
+			}
+			vectorOffset -= bitsInVectorCell
 		}
-		xorBytes(q.data[:], p[i:])
 	}
-	q.size += uint64(len(p))
+
+	// Update the starting position in a circular shift pattern
+	q.shiftSoFar = (q.shiftSoFar + shift*(len(p)%widthInBits)) % widthInBits
+
+	q.lengthSoFar += uint64(len(p))
+
 	return len(p), nil
 }
 
 // Calculate the current checksum
-func (q *quickXorHash) checkSum() (h [Size + 1]byte) {
-	for i := 0; i < dataSize; i++ {
-		shift := (i * 11) % 160
-		shiftBytes := shift / 8
-		shiftBits := shift % 8
-		shifted := int(q.data[i]) << shiftBits
-		h[shiftBytes] ^= byte(shifted)
-		h[shiftBytes+1] ^= byte(shifted >> 8)
+func (q *quickXorHash) checkSum() (h [Size]byte) {
+	// Output the data as little endian bytes
+	ph := 0
+	for i := 0; i < len(q.data)-1; i++ {
+		d := q.data[i]
+		_ = h[ph+7] // bounds check
+		h[ph+0] = byte(d >> (8 * 0))
+		h[ph+1] = byte(d >> (8 * 1))
+		h[ph+2] = byte(d >> (8 * 2))
+		h[ph+3] = byte(d >> (8 * 3))
+		h[ph+4] = byte(d >> (8 * 4))
+		h[ph+5] = byte(d >> (8 * 5))
+		h[ph+6] = byte(d >> (8 * 6))
+		h[ph+7] = byte(d >> (8 * 7))
+		ph += 8
 	}
-	h[0] ^= h[20]
+	// remaining 32 bits
+	d := q.data[len(q.data)-1]
+	h[Size-4] = byte(d >> (8 * 0))
+	h[Size-3] = byte(d >> (8 * 1))
+	h[Size-2] = byte(d >> (8 * 2))
+	h[Size-1] = byte(d >> (8 * 3))
 
 	// XOR the file length with the least significant bits in little endian format
-	d := q.size
+	d = q.lengthSoFar
 	h[Size-8] ^= byte(d >> (8 * 0))
 	h[Size-7] ^= byte(d >> (8 * 1))
 	h[Size-6] ^= byte(d >> (8 * 2))
@@ -106,7 +174,7 @@ func (q *quickXorHash) checkSum() (h [Size + 1]byte) {
 // It does not change the underlying hash state.
 func (q *quickXorHash) Sum(b []byte) []byte {
 	hash := q.checkSum()
-	return append(b, hash[:Size]...)
+	return append(b, hash[:]...)
 }
 
 // Reset resets the Hash to its initial state.
@@ -128,10 +196,8 @@ func (q *quickXorHash) BlockSize() int {
 }
 
 // Sum returns the quickXorHash checksum of the data.
-func Sum(data []byte) (h [Size]byte) {
+func Sum(data []byte) [Size]byte {
 	var d quickXorHash
 	_, _ = d.Write(data)
-	s := d.checkSum()
-	copy(h[:], s[:])
-	return h
+	return d.checkSum()
 }

backend/onedrive/quickxorhash/quickxorhash_test.go

@@ -4,7 +4,6 @@ import (
 	"encoding/base64"
 	"fmt"
 	"hash"
-	"math/rand"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -167,16 +166,3 @@ func TestReset(t *testing.T) {
 
 // check interface
 var _ hash.Hash = (*quickXorHash)(nil)
-
-func BenchmarkQuickXorHash(b *testing.B) {
-	b.SetBytes(1 << 20)
-	buf := make([]byte, 1<<20)
-	rand.Read(buf)
-	h := New()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		h.Reset()
-		h.Write(buf)
-		h.Sum(nil)
-	}
-}

backend/onedrive/quickxorhash/xor.go

@@ -1,20 +0,0 @@
-//go:build !go1.20
-
-package quickxorhash
-
-func xorBytes(dst, src []byte) int {
-	n := len(dst)
-	if len(src) < n {
-		n = len(src)
-	}
-	if n == 0 {
-		return 0
-	}
-	dst = dst[:n]
-	//src = src[:n]
-	src = src[:len(dst)] // remove bounds check in loop
-	for i := range dst {
-		dst[i] ^= src[i]
-	}
-	return n
-}

backend/onedrive/quickxorhash/xor_1.20.go

@@ -1,9 +0,0 @@
-//go:build go1.20
-
-package quickxorhash
-
-import "crypto/subtle"
-
-func xorBytes(dst, src []byte) int {
-	return subtle.XORBytes(dst, src, dst)
-}

backend/oracleobjectstorage/byok.go

@@ -1,145 +0,0 @@
-//go:build !plan9 && !solaris && !js
-// +build !plan9,!solaris,!js
-
-package oracleobjectstorage
-
-import (
-	"crypto/sha256"
-	"encoding/base64"
-	"errors"
-	"fmt"
-	"os"
-	"strings"
-
-	"github.com/oracle/oci-go-sdk/v65/common"
-	"github.com/oracle/oci-go-sdk/v65/objectstorage"
-	"github.com/oracle/oci-go-sdk/v65/objectstorage/transfer"
-)
-
-const (
-	sseDefaultAlgorithm = "AES256"
-)
-
-func getSha256(p []byte) []byte {
-	h := sha256.New()
-	h.Write(p)
-	return h.Sum(nil)
-}
-
-func validateSSECustomerKeyOptions(opt *Options) error {
-	if opt.SSEKMSKeyID != "" && (opt.SSECustomerKeyFile != "" || opt.SSECustomerKey != "") {
-		return errors.New("oos: can't use vault sse_kms_key_id and local sse_customer_key at the same time")
-	}
-	if opt.SSECustomerKey != "" && opt.SSECustomerKeyFile != "" {
-		return errors.New("oos: can't use sse_customer_key and sse_customer_key_file at the same time")
-	}
-	if opt.SSEKMSKeyID != "" {
-		return nil
-	}
-	err := populateSSECustomerKeys(opt)
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
-func populateSSECustomerKeys(opt *Options) error {
-	if opt.SSECustomerKeyFile != "" {
-		// Reads the base64-encoded AES key data from the specified file and computes its SHA256 checksum
-		data, err := os.ReadFile(expandPath(opt.SSECustomerKeyFile))
-		if err != nil {
-			return fmt.Errorf("oos: error reading sse_customer_key_file: %v", err)
-		}
-		opt.SSECustomerKey = strings.TrimSpace(string(data))
-	}
-	if opt.SSECustomerKey != "" {
-		decoded, err := base64.StdEncoding.DecodeString(opt.SSECustomerKey)
-		if err != nil {
-			return fmt.Errorf("oos: Could not decode sse_customer_key_file: %w", err)
-		}
-		sha256Checksum := base64.StdEncoding.EncodeToString(getSha256(decoded))
-		if opt.SSECustomerKeySha256 == "" {
-			opt.SSECustomerKeySha256 = sha256Checksum
-		} else {
-			if opt.SSECustomerKeySha256 != sha256Checksum {
-				return fmt.Errorf("the computed SHA256 checksum "+
-					"(%v) of the key doesn't match the config entry sse_customer_key_sha256=(%v)",
-					sha256Checksum, opt.SSECustomerKeySha256)
-			}
-		}
-		if opt.SSECustomerAlgorithm == "" {
-			opt.SSECustomerAlgorithm = sseDefaultAlgorithm
-		}
-	}
-	return nil
-}
-
-// https://docs.oracle.com/en-us/iaas/Content/Object/Tasks/usingyourencryptionkeys.htm
-func useBYOKPutObject(fs *Fs, request *objectstorage.PutObjectRequest) {
-	if fs.opt.SSEKMSKeyID != "" {
-		request.OpcSseKmsKeyId = common.String(fs.opt.SSEKMSKeyID)
-	}
-	if fs.opt.SSECustomerAlgorithm != "" {
-		request.OpcSseCustomerAlgorithm = common.String(fs.opt.SSECustomerAlgorithm)
-	}
-	if fs.opt.SSECustomerKey != "" {
-		request.OpcSseCustomerKey = common.String(fs.opt.SSECustomerKey)
-	}
-	if fs.opt.SSECustomerKeySha256 != "" {
-		request.OpcSseCustomerKeySha256 = common.String(fs.opt.SSECustomerKeySha256)
-	}
-}
-
-func useBYOKHeadObject(fs *Fs, request *objectstorage.HeadObjectRequest) {
-	if fs.opt.SSECustomerAlgorithm != "" {
-		request.OpcSseCustomerAlgorithm = common.String(fs.opt.SSECustomerAlgorithm)
-	}
-	if fs.opt.SSECustomerKey != "" {
-		request.OpcSseCustomerKey = common.String(fs.opt.SSECustomerKey)
-	}
-	if fs.opt.SSECustomerKeySha256 != "" {
-		request.OpcSseCustomerKeySha256 = common.String(fs.opt.SSECustomerKeySha256)
-	}
-}
-
-func useBYOKGetObject(fs *Fs, request *objectstorage.GetObjectRequest) {
-	if fs.opt.SSECustomerAlgorithm != "" {
-		request.OpcSseCustomerAlgorithm = common.String(fs.opt.SSECustomerAlgorithm)
-	}
-	if fs.opt.SSECustomerKey != "" {
-		request.OpcSseCustomerKey = common.String(fs.opt.SSECustomerKey)
-	}
-	if fs.opt.SSECustomerKeySha256 != "" {
-		request.OpcSseCustomerKeySha256 = common.String(fs.opt.SSECustomerKeySha256)
-	}
-}
-
-func useBYOKCopyObject(fs *Fs, request *objectstorage.CopyObjectRequest) {
-	if fs.opt.SSEKMSKeyID != "" {
-		request.OpcSseKmsKeyId = common.String(fs.opt.SSEKMSKeyID)
-	}
-	if fs.opt.SSECustomerAlgorithm != "" {
-		request.OpcSseCustomerAlgorithm = common.String(fs.opt.SSECustomerAlgorithm)
-	}
-	if fs.opt.SSECustomerKey != "" {
-		request.OpcSseCustomerKey = common.String(fs.opt.SSECustomerKey)
-	}
-	if fs.opt.SSECustomerKeySha256 != "" {
-		request.OpcSseCustomerKeySha256 = common.String(fs.opt.SSECustomerKeySha256)
-	}
-}
-
-func useBYOKUpload(fs *Fs, request *transfer.UploadRequest) {
-	if fs.opt.SSEKMSKeyID != "" {
-		request.OpcSseKmsKeyId = common.String(fs.opt.SSEKMSKeyID)
-	}
-	if fs.opt.SSECustomerAlgorithm != "" {
-		request.OpcSseCustomerAlgorithm = common.String(fs.opt.SSECustomerAlgorithm)
-	}
-	if fs.opt.SSECustomerKey != "" {
-		request.OpcSseCustomerKey = common.String(fs.opt.SSECustomerKey)
-	}
-	if fs.opt.SSECustomerKeySha256 != "" {
-		request.OpcSseCustomerKeySha256 = common.String(fs.opt.SSECustomerKeySha256)
-	}
-}

backend/oracleobjectstorage/client.go

@@ -9,8 +9,6 @@ import (
 	"errors"
 	"net/http"
 	"os"
-	"path"
-	"strings"
 
 	"github.com/oracle/oci-go-sdk/v65/common"
 	"github.com/oracle/oci-go-sdk/v65/common/auth"
@@ -20,33 +18,15 @@ import (
 	"github.com/rclone/rclone/fs/fshttp"
 )
 
-func expandPath(filepath string) (expandedPath string) {
-	if filepath == "" {
-		return filepath
-	}
-	cleanedPath := path.Clean(filepath)
-	expandedPath = cleanedPath
-	if strings.HasPrefix(cleanedPath, "~") {
-		rest := cleanedPath[2:]
-		home, err := os.UserHomeDir()
-		if err != nil {
-			return expandedPath
-		}
-		expandedPath = path.Join(home, rest)
-	}
-	return
-}
-
 func getConfigurationProvider(opt *Options) (common.ConfigurationProvider, error) {
 	switch opt.Provider {
 	case instancePrincipal:
 		return auth.InstancePrincipalConfigurationProvider()
 	case userPrincipal:
-		expandConfigFilePath := expandPath(opt.ConfigFile)
-		if expandConfigFilePath != "" && !fileExists(expandConfigFilePath) {
-			fs.Errorf(userPrincipal, "oci config file doesn't exist at %v", expandConfigFilePath)
+		if opt.ConfigFile != "" && !fileExists(opt.ConfigFile) {
+			fs.Errorf(userPrincipal, "oci config file doesn't exist at %v", opt.ConfigFile)
 		}
-		return common.CustomProfileConfigProvider(expandConfigFilePath, opt.ConfigProfile), nil
+		return common.CustomProfileConfigProvider(opt.ConfigFile, opt.ConfigProfile), nil
 	case resourcePrincipal:
 		return auth.ResourcePrincipalConfigurationProvider()
 	case noAuth:

backend/oracleobjectstorage/command.go

@@ -65,8 +65,8 @@ a bucket or with a bucket and path.
 	Long: `This command removes unfinished multipart uploads of age greater than
 max-age which defaults to 24 hours.
 
-Note that you can use --interactive/-i or --dry-run with this command to see what
-it would do.
+Note that you can use -i/--dry-run with this command to see what it
+would do.
 
     rclone backend cleanup oos:bucket/path/to/object
     rclone backend cleanup -o max-age=7w oos:bucket/path/to/object

backend/oracleobjectstorage/copy.go

@@ -74,7 +74,6 @@ func (f *Fs) copy(ctx context.Context, dstObj *Object, srcObj *Object) (err erro
 		BucketName:        common.String(srcBucket),
 		CopyObjectDetails: copyObjectDetails,
 	}
-	useBYOKCopyObject(f, &req)
 	var resp objectstorage.CopyObjectResponse
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CopyObject(ctx, req)

backend/oracleobjectstorage/object.go

@@ -87,7 +87,6 @@ func (o *Object) headObject(ctx context.Context) (info *objectstorage.HeadObject
 		BucketName:    common.String(bucketName),
 		ObjectName:    common.String(objectPath),
 	}
-	useBYOKHeadObject(o.fs, &req)
 	var response objectstorage.HeadObjectResponse
 	err = o.fs.pacer.Call(func() (bool, error) {
 		var err error
@@ -100,7 +99,6 @@ func (o *Object) headObject(ctx context.Context) (info *objectstorage.HeadObject
 				return nil, fs.ErrorObjectNotFound
 			}
 		}
-		fs.Errorf(o, "Failed to head object: %v", err)
 		return nil, err
 	}
 	o.fs.cache.MarkOK(bucketName)
@@ -333,7 +331,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo
 		ObjectName:    common.String(bucketPath),
 	}
 	o.applyGetObjectOptions(&req, options...)
-	useBYOKGetObject(o.fs, &req)
+
 	var resp objectstorage.GetObjectResponse
 	err := o.fs.pacer.Call(func() (bool, error) {
 		var err error
@@ -435,7 +433,6 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 			uploadRequest.StorageTier = storageTier
 		}
 		o.applyMultiPutOptions(&uploadRequest, options...)
-		useBYOKUpload(o.fs, &uploadRequest)
 		uploadStreamRequest := transfer.UploadStreamRequest{
 			UploadRequest: uploadRequest,
 			StreamReader:  in,
@@ -509,10 +506,8 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 			req.StorageTier = storageTier
 		}
 		o.applyPutOptions(&req, options...)
-		useBYOKPutObject(o.fs, &req)
-		var resp objectstorage.PutObjectResponse
 		err = o.fs.pacer.Call(func() (bool, error) {
-			resp, err = o.fs.srv.PutObject(ctx, req)
+			resp, err := o.fs.srv.PutObject(ctx, req)
 			return shouldRetry(ctx, resp.HTTPResponse(), err)
 		})
 		if err != nil {

backend/oracleobjectstorage/options.go

@@ -17,7 +17,9 @@ const (
 	defaultUploadCutoff        = fs.SizeSuffix(200 * 1024 * 1024)
 	defaultUploadConcurrency   = 10
 	maxUploadCutoff            = fs.SizeSuffix(5 * 1024 * 1024 * 1024)
-	minSleep                   = 10 * time.Millisecond
+	minSleep                   = 100 * time.Millisecond
+	maxSleep                   = 5 * time.Minute
+	decayConstant              = 1 // bigger for slower decay, exponential
 	defaultCopyTimeoutDuration = fs.Duration(time.Minute)
 )
 
@@ -45,28 +47,23 @@ https://docs.oracle.com/en-us/iaas/Content/Identity/Tasks/callingservicesfromins
 
 // Options defines the configuration for this backend
 type Options struct {
-	Provider             string               `config:"provider"`
-	Compartment          string               `config:"compartment"`
-	Namespace            string               `config:"namespace"`
-	Region               string               `config:"region"`
-	Endpoint             string               `config:"endpoint"`
-	Enc                  encoder.MultiEncoder `config:"encoding"`
-	ConfigFile           string               `config:"config_file"`
-	ConfigProfile        string               `config:"config_profile"`
-	UploadCutoff         fs.SizeSuffix        `config:"upload_cutoff"`
-	ChunkSize            fs.SizeSuffix        `config:"chunk_size"`
-	UploadConcurrency    int                  `config:"upload_concurrency"`
-	DisableChecksum      bool                 `config:"disable_checksum"`
-	CopyCutoff           fs.SizeSuffix        `config:"copy_cutoff"`
-	CopyTimeout          fs.Duration          `config:"copy_timeout"`
-	StorageTier          string               `config:"storage_tier"`
-	LeavePartsOnError    bool                 `config:"leave_parts_on_error"`
-	NoCheckBucket        bool                 `config:"no_check_bucket"`
-	SSEKMSKeyID          string               `config:"sse_kms_key_id"`
-	SSECustomerAlgorithm string               `config:"sse_customer_algorithm"`
-	SSECustomerKey       string               `config:"sse_customer_key"`
-	SSECustomerKeyFile   string               `config:"sse_customer_key_file"`
-	SSECustomerKeySha256 string               `config:"sse_customer_key_sha256"`
+	Provider          string               `config:"provider"`
+	Compartment       string               `config:"compartment"`
+	Namespace         string               `config:"namespace"`
+	Region            string               `config:"region"`
+	Endpoint          string               `config:"endpoint"`
+	Enc               encoder.MultiEncoder `config:"encoding"`
+	ConfigFile        string               `config:"config_file"`
+	ConfigProfile     string               `config:"config_profile"`
+	UploadCutoff      fs.SizeSuffix        `config:"upload_cutoff"`
+	ChunkSize         fs.SizeSuffix        `config:"chunk_size"`
+	UploadConcurrency int                  `config:"upload_concurrency"`
+	DisableChecksum   bool                 `config:"disable_checksum"`
+	CopyCutoff        fs.SizeSuffix        `config:"copy_cutoff"`
+	CopyTimeout       fs.Duration          `config:"copy_timeout"`
+	StorageTier       string               `config:"storage_tier"`
+	LeavePartsOnError bool                 `config:"leave_parts_on_error"`
+	NoCheckBucket     bool                 `config:"no_check_bucket"`
 }
 
 func newOptions() []fs.Option {
@@ -126,22 +123,6 @@ func newOptions() []fs.Option {
 			Value: "Default",
 			Help:  "Use the default profile",
 		}},
-	}, {
-		// Mapping from here: https://github.com/oracle/oci-go-sdk/blob/master/objectstorage/storage_tier.go
-		Name:     "storage_tier",
-		Help:     "The storage class to use when storing new objects in storage. https://docs.oracle.com/en-us/iaas/Content/Object/Concepts/understandingstoragetiers.htm",
-		Default:  "Standard",
-		Advanced: true,
-		Examples: []fs.OptionExample{{
-			Value: "Standard",
-			Help:  "Standard storage tier, this is the default tier",
-		}, {
-			Value: "InfrequentAccess",
-			Help:  "InfrequentAccess storage tier",
-		}, {
-			Value: "Archive",
-			Help:  "Archive storage tier",
-		}},
 	}, {
 		Name: "upload_cutoff",
 		Help: `Cutoff for switching to chunked upload.
@@ -257,59 +238,5 @@ creation permissions.
 `,
 		Default:  false,
 		Advanced: true,
-	}, {
-		Name: "sse_customer_key_file",
-		Help: `To use SSE-C, a file containing the base64-encoded string of the AES-256 encryption key associated
-with the object. Please note only one of sse_customer_key_file|sse_customer_key|sse_kms_key_id is needed.'`,
-		Advanced: true,
-		Examples: []fs.OptionExample{{
-			Value: "",
-			Help:  "None",
-		}},
-	}, {
-		Name: "sse_customer_key",
-		Help: `To use SSE-C, the optional header that specifies the base64-encoded 256-bit encryption key to use to
-encrypt or  decrypt the data. Please note only one of sse_customer_key_file|sse_customer_key|sse_kms_key_id is
-needed. For more information, see Using Your Own Keys for Server-Side Encryption 
-(https://docs.cloud.oracle.com/Content/Object/Tasks/usingyourencryptionkeys.htm)`,
-		Advanced: true,
-		Examples: []fs.OptionExample{{
-			Value: "",
-			Help:  "None",
-		}},
-	}, {
-		Name: "sse_customer_key_sha256",
-		Help: `If using SSE-C, The optional header that specifies the base64-encoded SHA256 hash of the encryption
-key. This value is used to check the integrity of the encryption key. see Using Your Own Keys for 
-Server-Side Encryption (https://docs.cloud.oracle.com/Content/Object/Tasks/usingyourencryptionkeys.htm).`,
-		Advanced: true,
-		Examples: []fs.OptionExample{{
-			Value: "",
-			Help:  "None",
-		}},
-	}, {
-		Name: "sse_kms_key_id",
-		Help: `if using using your own master key in vault, this header specifies the 
-OCID (https://docs.cloud.oracle.com/Content/General/Concepts/identifiers.htm) of a master encryption key used to call
-the Key Management service to generate a data encryption key or to encrypt or decrypt a data encryption key.
-Please note only one of sse_customer_key_file|sse_customer_key|sse_kms_key_id is needed.`,
-		Advanced: true,
-		Examples: []fs.OptionExample{{
-			Value: "",
-			Help:  "None",
-		}},
-	}, {
-		Name: "sse_customer_algorithm",
-		Help: `If using SSE-C, the optional header that specifies "AES256" as the encryption algorithm.
-Object Storage supports "AES256" as the encryption algorithm. For more information, see
-Using Your Own Keys for Server-Side Encryption (https://docs.cloud.oracle.com/Content/Object/Tasks/usingyourencryptionkeys.htm).`,
-		Advanced: true,
-		Examples: []fs.OptionExample{{
-			Value: "",
-			Help:  "None",
-		}, {
-			Value: sseDefaultAlgorithm,
-			Help:  sseDefaultAlgorithm,
-		}},
 	}}
 }

backend/oracleobjectstorage/oracleobjectstorage.go

@@ -59,27 +59,19 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	if err != nil {
 		return nil, err
 	}
-	err = validateSSECustomerKeyOptions(opt)
-	if err != nil {
-		return nil, err
-	}
 	ci := fs.GetConfig(ctx)
 	objectStorageClient, err := newObjectStorageClient(ctx, opt)
 	if err != nil {
 		return nil, err
 	}
-	pc := fs.NewPacer(ctx, pacer.NewS3(pacer.MinSleep(minSleep)))
-	// Set pacer retries to 2 (1 try and 1 retry) because we are
-	// relying on SDK retry mechanism, but we allow 2 attempts to
-	// retry directory listings after XMLSyntaxError
-	pc.SetRetries(2)
+	p := pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))
 	f := &Fs{
 		name:  name,
 		opt:   *opt,
 		ci:    ci,
 		srv:   objectStorageClient,
 		cache: bucket.NewCache(),
-		pacer: pc,
+		pacer: fs.NewPacer(ctx, p),
 	}
 	f.setRoot(root)
 	f.features = (&fs.Features{

backend/s3/s3.go

@@ -15,6 +15,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"net/http"
 	"net/url"
 	"path"
@@ -25,8 +26,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/aws/aws-sdk-go/service/s3/s3manager"
-
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/aws/corehandlers"
@@ -66,7 +65,7 @@ import (
 func init() {
 	fs.Register(&fs.RegInfo{
 		Name:        "s3",
-		Description: "Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Liara, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi",
+		Description: "Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi",
 		NewFs:       NewFs,
 		CommandHelp: commandHelp,
 		Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
@@ -105,7 +104,7 @@ func init() {
 				Help:  "Arvan Cloud Object Storage (AOS)",
 			}, {
 				Value: "DigitalOcean",
-				Help:  "DigitalOcean Spaces",
+				Help:  "Digital Ocean Spaces",
 			}, {
 				Value: "Dreamhost",
 				Help:  "Dreamhost DreamObjects",
@@ -124,9 +123,6 @@ func init() {
 			}, {
 				Value: "LyveCloud",
 				Help:  "Seagate Lyve Cloud",
-			}, {
-				Value: "Liara",
-				Help:  "Liara Object Storage",
 			}, {
 				Value: "Minio",
 				Help:  "Minio Object Storage",
@@ -440,7 +436,7 @@ func init() {
 		}, {
 			Name:     "region",
 			Help:     "Region to connect to.\n\nLeave blank if you are using an S3 clone and you don't have a region.",
-			Provider: "!AWS,Alibaba,ChinaMobile,Cloudflare,IONOS,ArvanCloud,Liara,Qiniu,RackCorp,Scaleway,Storj,TencentCOS,HuaweiOBS,IDrive",
+			Provider: "!AWS,Alibaba,ChinaMobile,Cloudflare,IONOS,ArvanCloud,Qiniu,RackCorp,Scaleway,Storj,TencentCOS,HuaweiOBS,IDrive",
 			Examples: []fs.OptionExample{{
 				Value: "",
 				Help:  "Use this if unsure.\nWill use v4 signatures and an empty region.",
@@ -765,15 +761,6 @@ func init() {
 				Value: "s3-eu-south-2.ionoscloud.com",
 				Help:  "Logrono, Spain",
 			}},
-		}, {
-			// Liara endpoints: https://liara.ir/landing/object-storage
-			Name:     "endpoint",
-			Help:     "Endpoint for Liara Object Storage API.",
-			Provider: "Liara",
-			Examples: []fs.OptionExample{{
-				Value: "storage.iran.liara.space",
-				Help:  "The default endpoint\nIran",
-			}},
 		}, {
 			// oss endpoints: https://help.aliyun.com/document_detail/31837.html
 			Name:     "endpoint",
@@ -936,11 +923,17 @@ func init() {
 			}},
 		}, {
 			Name:     "endpoint",
-			Help:     "Endpoint for Storj Gateway.",
+			Help:     "Endpoint of the Shared Gateway.",
 			Provider: "Storj",
 			Examples: []fs.OptionExample{{
-				Value: "gateway.storjshare.io",
-				Help:  "Global Hosted Gateway",
+				Value: "gateway.eu1.storjshare.io",
+				Help:  "EU1 Shared Gateway",
+			}, {
+				Value: "gateway.us1.storjshare.io",
+				Help:  "US1 Shared Gateway",
+			}, {
+				Value: "gateway.ap1.storjshare.io",
+				Help:  "Asia-Pacific Shared Gateway",
 			}},
 		}, {
 			// cos endpoints: https://intl.cloud.tencent.com/document/product/436/6224
@@ -1098,34 +1091,22 @@ func init() {
 		}, {
 			Name:     "endpoint",
 			Help:     "Endpoint for S3 API.\n\nRequired when using an S3 clone.",
-			Provider: "!AWS,IBMCOS,IDrive,IONOS,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,Liara,ArvanCloud,Scaleway,StackPath,Storj,RackCorp,Qiniu",
+			Provider: "!AWS,IBMCOS,IDrive,IONOS,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,ArvanCloud,Scaleway,StackPath,Storj,RackCorp,Qiniu",
 			Examples: []fs.OptionExample{{
 				Value:    "objects-us-east-1.dream.io",
 				Help:     "Dream Objects endpoint",
 				Provider: "Dreamhost",
-			}, {
-				Value:    "syd1.digitaloceanspaces.com",
-				Help:     "DigitalOcean Spaces Sydney 1",
-				Provider: "DigitalOcean",
-			}, {
-				Value:    "sfo3.digitaloceanspaces.com",
-				Help:     "DigitalOcean Spaces San Francisco 3",
-				Provider: "DigitalOcean",
-			}, {
-				Value:    "fra1.digitaloceanspaces.com",
-				Help:     "DigitalOcean Spaces Frankfurt 1",
-				Provider: "DigitalOcean",
 			}, {
 				Value:    "nyc3.digitaloceanspaces.com",
-				Help:     "DigitalOcean Spaces New York 3",
+				Help:     "Digital Ocean Spaces New York 3",
 				Provider: "DigitalOcean",
 			}, {
 				Value:    "ams3.digitaloceanspaces.com",
-				Help:     "DigitalOcean Spaces Amsterdam 3",
+				Help:     "Digital Ocean Spaces Amsterdam 3",
 				Provider: "DigitalOcean",
 			}, {
 				Value:    "sgp1.digitaloceanspaces.com",
-				Help:     "DigitalOcean Spaces Singapore 1",
+				Help:     "Digital Ocean Spaces Singapore 1",
 				Provider: "DigitalOcean",
 			}, {
 				Value:    "localhost:8333",
@@ -1195,10 +1176,6 @@ func init() {
 				Value:    "s3.ap-southeast-2.wasabisys.com",
 				Help:     "Wasabi AP Southeast 2 (Sydney)",
 				Provider: "Wasabi",
-			}, {
-				Value:    "storage.iran.liara.space",
-				Help:     "Liara Iran endpoint",
-				Provider: "Liara",
 			}, {
 				Value:    "s3.ir-thr-at1.arvanstorage.com",
 				Help:     "ArvanCloud Tehran Iran (Asiatech) endpoint",
@@ -1582,7 +1559,7 @@ func init() {
 		}, {
 			Name:     "location_constraint",
 			Help:     "Location constraint - must be set to match the Region.\n\nLeave blank if not sure. Used when creating buckets only.",
-			Provider: "!AWS,Alibaba,HuaweiOBS,ChinaMobile,Cloudflare,IBMCOS,IDrive,IONOS,Liara,ArvanCloud,Qiniu,RackCorp,Scaleway,StackPath,Storj,TencentCOS",
+			Provider: "!AWS,Alibaba,HuaweiOBS,ChinaMobile,Cloudflare,IBMCOS,IDrive,IONOS,ArvanCloud,Qiniu,RackCorp,Scaleway,StackPath,Storj,TencentCOS",
 		}, {
 			Name: "acl",
 			Help: `Canned ACL used when creating buckets and storing or copying objects.
@@ -1592,11 +1569,7 @@ This ACL is used for creating objects and if bucket_acl isn't set, for creating
 For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
 
 Note that this ACL is applied when server-side copying objects as S3
-doesn't copy the ACL from the source but rather writes a fresh one.
-
-If the acl is an empty string then no X-Amz-Acl: header is added and
-the default (private) will be used.
-`,
+doesn't copy the ACL from the source but rather writes a fresh one.`,
 			Provider: "!Storj,Cloudflare",
 			Examples: []fs.OptionExample{{
 				Value:    "default",
@@ -1650,11 +1623,7 @@ the default (private) will be used.
 For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
 
 Note that this ACL is applied when only when creating buckets.  If it
-isn't set then "acl" is used instead.
-
-If the "acl" and "bucket_acl" are empty strings then no X-Amz-Acl:
-header is added and the default (private) will be used.
-`,
+isn't set then "acl" is used instead.`,
 			Advanced: true,
 			Examples: []fs.OptionExample{{
 				Value: "private",
@@ -1815,15 +1784,6 @@ If you leave it blank, this is calculated automatically from the sse_customer_ke
 				Value: "STANDARD_IA",
 				Help:  "Infrequent access storage mode",
 			}},
-		}, {
-			// Mapping from here: https://liara.ir/landing/object-storage
-			Name:     "storage_class",
-			Help:     "The storage class to use when storing new objects in Liara",
-			Provider: "Liara",
-			Examples: []fs.OptionExample{{
-				Value: "STANDARD",
-				Help:  "Standard storage class",
-			}},
 		}, {
 			// Mapping from here: https://www.arvancloud.com/en/products/cloud-storage
 			Name:     "storage_class",
@@ -2266,11 +2226,6 @@ rclone's choice here.
 			Help:     `Suppress setting and reading of system metadata`,
 			Advanced: true,
 			Default:  false,
-		}, {
-			Name:     "sts_endpoint",
-			Help:     "Endpoint for STS.\n\nLeave blank if using AWS to use the default endpoint for the region.",
-			Provider: "AWS",
-			Advanced: true,
 		},
 		}})
 }
@@ -2357,7 +2312,6 @@ type Options struct {
 	SecretAccessKey       string               `config:"secret_access_key"`
 	Region                string               `config:"region"`
 	Endpoint              string               `config:"endpoint"`
-	STSEndpoint           string               `config:"sts_endpoint"`
 	LocationConstraint    string               `config:"location_constraint"`
 	ACL                   string               `config:"acl"`
 	BucketACL             string               `config:"bucket_acl"`
@@ -2534,7 +2488,7 @@ func parsePath(path string) (root string) {
 // split returns bucket and bucketPath from the rootRelativePath
 // relative to f.root
 func (f *Fs) split(rootRelativePath string) (bucketName, bucketPath string) {
-	bucketName, bucketPath = bucket.Split(bucket.Join(f.root, rootRelativePath))
+	bucketName, bucketPath = bucket.Split(path.Join(f.root, rootRelativePath))
 	return f.opt.Enc.FromStandardName(bucketName), f.opt.Enc.FromStandardPath(bucketPath)
 }
 
@@ -2566,38 +2520,6 @@ func getClient(ctx context.Context, opt *Options) *http.Client {
 	}
 }
 
-// Default name resolver
-var defaultResolver = endpoints.DefaultResolver()
-
-// resolve (service, region) to endpoint
-//
-// Used to set endpoint for s3 services and not for other services
-type resolver map[string]string
-
-// Add a service to the resolver, ignoring empty urls
-func (r resolver) addService(service, url string) {
-	if url == "" {
-		return
-	}
-	if !strings.HasPrefix(url, "http") {
-		url = "https://" + url
-	}
-	r[service] = url
-}
-
-// EndpointFor return the endpoint for s3 if set or the default if not
-func (r resolver) EndpointFor(service, region string, opts ...func(*endpoints.Options)) (endpoints.ResolvedEndpoint, error) {
-	fs.Debugf(nil, "Resolving service %q region %q", service, region)
-	url, ok := r[service]
-	if ok {
-		return endpoints.ResolvedEndpoint{
-			URL:           url,
-			SigningRegion: region,
-		}, nil
-	}
-	return defaultResolver.EndpointFor(service, region, opts...)
-}
-
 // s3Connection makes a connection to s3
 func s3Connection(ctx context.Context, opt *Options, client *http.Client) (*s3.S3, *session.Session, error) {
 	ci := fs.GetConfig(ctx)
@@ -2676,12 +2598,8 @@ func s3Connection(ctx context.Context, opt *Options, client *http.Client) (*s3.S
 	if opt.Region != "" {
 		awsConfig.WithRegion(opt.Region)
 	}
-	if opt.Endpoint != "" || opt.STSEndpoint != "" {
-		// If endpoints are set, override the relevant services only
-		r := make(resolver)
-		r.addService("s3", opt.Endpoint)
-		r.addService("sts", opt.STSEndpoint)
-		awsConfig.WithEndpointResolver(r)
+	if opt.Endpoint != "" {
+		awsConfig.WithEndpoint(opt.Endpoint)
 	}
 
 	// awsConfig.WithLogLevel(aws.LogDebugWithSigning)
@@ -2837,10 +2755,6 @@ func setQuirks(opt *Options) {
 		// listObjectsV2 supported - https://api.ionos.com/docs/s3/#Basic-Operations-get-Bucket-list-type-2
 		virtualHostStyle = false
 		urlEncodeListings = false
-	case "Liara":
-		virtualHostStyle = false
-		urlEncodeListings = false
-		useMultipartEtag = false
 	case "LyveCloud":
 		useMultipartEtag = false // LyveCloud seems to calculate multipart Etags differently from AWS
 	case "Minio":
@@ -2932,14 +2846,6 @@ func (f *Fs) setRoot(root string) {
 	f.rootBucket, f.rootDirectory = bucket.Split(f.root)
 }
 
-// return a pointer to the string if non empty or nil if it is empty
-func stringPointerOrNil(s string) *string {
-	if s == "" {
-		return nil
-	}
-	return &s
-}
-
 // NewFs constructs an Fs from the path, bucket:path
 func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
@@ -2959,6 +2865,9 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	if opt.Versions && opt.VersionAt.IsSet() {
 		return nil, errors.New("s3: cant use --s3-versions and --s3-version-at at the same time")
 	}
+	if opt.ACL == "" {
+		opt.ACL = "private"
+	}
 	if opt.BucketACL == "" {
 		opt.BucketACL = opt.ACL
 	}
@@ -3031,15 +2940,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		GetTier:           true,
 		SlowModTime:       true,
 	}).Fill(ctx, f)
-	if opt.Provider == "Storj" {
-		f.features.SetTier = false
-		f.features.GetTier = false
-	}
-	if opt.Provider == "IDrive" {
-		f.features.SetTier = false
-	}
-	// f.listMultipartUploads()
-
 	if f.rootBucket != "" && f.rootDirectory != "" && !opt.NoHeadObject && !strings.HasSuffix(root, "/") {
 		// Check to see if the (bucket,directory) is actually an existing file
 		oldRoot := f.root
@@ -3054,6 +2954,14 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		// return an error with an fs which points to the parent
 		return f, fs.ErrorIsFile
 	}
+	if opt.Provider == "Storj" {
+		f.features.SetTier = false
+		f.features.GetTier = false
+	}
+	if opt.Provider == "IDrive" {
+		f.features.SetTier = false
+	}
+	// f.listMultipartUploads()
 	return f, nil
 }
 
@@ -3105,17 +3013,6 @@ func (f *Fs) getMetaDataListing(ctx context.Context, wantRemote string) (info *s
 	return info, versionID, nil
 }
 
-// stringClonePointer clones the string pointed to by sp into new
-// memory. This is useful to stop us keeping references to small
-// strings carved out of large XML responses.
-func stringClonePointer(sp *string) *string {
-	if sp == nil {
-		return nil
-	}
-	var s = *sp
-	return &s
-}
-
 // Return an Object from a path
 //
 // If it can't be found it returns the error ErrorObjectNotFound.
@@ -3141,8 +3038,8 @@ func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *s3.Obje
 		}
 		o.setMD5FromEtag(aws.StringValue(info.ETag))
 		o.bytes = aws.Int64Value(info.Size)
-		o.storageClass = stringClonePointer(info.StorageClass)
-		o.versionID = stringClonePointer(versionID)
+		o.storageClass = info.StorageClass
+		o.versionID = versionID
 	} else if !o.fs.opt.NoHeadObject {
 		err := o.readMetaData(ctx) // reads info and meta, returning an error
 		if err != nil {
@@ -3160,13 +3057,19 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 
 // Gets the bucket location
 func (f *Fs) getBucketLocation(ctx context.Context, bucket string) (string, error) {
-	region, err := s3manager.GetBucketRegion(ctx, f.ses, bucket, "", func(r *request.Request) {
-		r.Config.S3ForcePathStyle = aws.Bool(f.opt.ForcePathStyle)
+	req := s3.GetBucketLocationInput{
+		Bucket: &bucket,
+	}
+	var resp *s3.GetBucketLocationOutput
+	var err error
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err = f.c.GetBucketLocation(&req)
+		return f.shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return "", err
 	}
-	return region, nil
+	return s3.NormalizeBucketLocation(aws.StringValue(resp.LocationConstraint)), nil
 }
 
 // Updates the region for the bucket by reading the region from the
@@ -3283,9 +3186,6 @@ func (ls *v2List) List(ctx context.Context) (resp *s3.ListObjectsV2Output, versi
 	if err != nil {
 		return nil, nil, err
 	}
-	if aws.BoolValue(resp.IsTruncated) && (resp.NextContinuationToken == nil || *resp.NextContinuationToken == "") {
-		return nil, nil, errors.New("s3 protocol error: received listing v2 with IsTruncated set and no NextContinuationToken. Should you be using `--s3-list-version 1`?")
-	}
 	ls.req.ContinuationToken = resp.NextContinuationToken
 	return resp, nil, nil
 }
@@ -3468,16 +3368,15 @@ var errEndList = errors.New("end list")
 
 // list options
 type listOpt struct {
-	bucket        string  // bucket to list
-	directory     string  // directory with bucket
-	prefix        string  // prefix to remove from listing
-	addBucket     bool    // if set, the bucket is added to the start of the remote
-	recurse       bool    // if set, recurse to read sub directories
-	withVersions  bool    // if set, versions are produced
-	hidden        bool    // if set, return delete markers as objects with size == isDeleteMarker
-	findFile      bool    // if set, it will look for files called (bucket, directory)
-	versionAt     fs.Time // if set only show versions <= this time
-	noSkipMarkers bool    // if set return dir marker objects
+	bucket       string  // bucket to list
+	directory    string  // directory with bucket
+	prefix       string  // prefix to remove from listing
+	addBucket    bool    // if set, the bucket is added to the start of the remote
+	recurse      bool    // if set, recurse to read sub directories
+	withVersions bool    // if set, versions are produced
+	hidden       bool    // if set, return delete markers as objects with size == isDeleteMarker
+	findFile     bool    // if set, it will look for files called (bucket, directory)
+	versionAt    fs.Time // if set only show versions <= this time
 }
 
 // list lists the objects into the function supplied with the opt
@@ -3590,7 +3489,7 @@ func (f *Fs) list(ctx context.Context, opt listOpt, fn listFn) error {
 				}
 				remote = remote[len(opt.prefix):]
 				if opt.addBucket {
-					remote = bucket.Join(opt.bucket, remote)
+					remote = path.Join(opt.bucket, remote)
 				}
 				remote = strings.TrimSuffix(remote, "/")
 				err = fn(remote, &s3.Object{Key: &remote}, nil, true)
@@ -3619,10 +3518,10 @@ func (f *Fs) list(ctx context.Context, opt listOpt, fn listFn) error {
 			remote = remote[len(opt.prefix):]
 			isDirectory := remote == "" || strings.HasSuffix(remote, "/")
 			if opt.addBucket {
-				remote = bucket.Join(opt.bucket, remote)
+				remote = path.Join(opt.bucket, remote)
 			}
 			// is this a directory marker?
-			if isDirectory && object.Size != nil && *object.Size == 0 && !opt.noSkipMarkers {
+			if isDirectory && object.Size != nil && *object.Size == 0 {
 				continue // skip directory marker
 			}
 			if versionIDs != nil {
@@ -3845,7 +3744,7 @@ func (f *Fs) makeBucket(ctx context.Context, bucket string) error {
 	return f.cache.Create(bucket, func() error {
 		req := s3.CreateBucketInput{
 			Bucket: &bucket,
-			ACL:    stringPointerOrNil(f.opt.BucketACL),
+			ACL:    &f.opt.BucketACL,
 		}
 		if f.opt.LocationConstraint != "" {
 			req.CreateBucketConfiguration = &s3.CreateBucketConfiguration{
@@ -3910,9 +3809,9 @@ func pathEscape(s string) string {
 // method
 func (f *Fs) copy(ctx context.Context, req *s3.CopyObjectInput, dstBucket, dstPath, srcBucket, srcPath string, src *Object) error {
 	req.Bucket = &dstBucket
-	req.ACL = stringPointerOrNil(f.opt.ACL)
+	req.ACL = &f.opt.ACL
 	req.Key = &dstPath
-	source := pathEscape(bucket.Join(srcBucket, srcPath))
+	source := pathEscape(path.Join(srcBucket, srcPath))
 	if src.versionID != nil {
 		source += fmt.Sprintf("?versionId=%s", *src.versionID)
 	}
@@ -4147,9 +4046,9 @@ Usage Examples:
     rclone backend restore s3:bucket/path/to/directory [-o priority=PRIORITY] [-o lifetime=DAYS]
     rclone backend restore s3:bucket [-o priority=PRIORITY] [-o lifetime=DAYS]
 
-This flag also obeys the filters. Test first with --interactive/-i or --dry-run flags
+This flag also obeys the filters. Test first with -i/--interactive or --dry-run flags
 
-    rclone --interactive backend restore --include "*.txt" s3:bucket/path -o priority=Standard
+    rclone -i backend restore --include "*.txt" s3:bucket/path -o priority=Standard
 
 All the objects shown will be marked for restore, then
 
@@ -4217,8 +4116,8 @@ a bucket or with a bucket and path.
 	Long: `This command removes unfinished multipart uploads of age greater than
 max-age which defaults to 24 hours.
 
-Note that you can use --interactive/-i or --dry-run with this command to see what
-it would do.
+Note that you can use -i/--dry-run with this command to see what it
+would do.
 
     rclone backend cleanup s3:bucket/path/to/object
     rclone backend cleanup -o max-age=7w s3:bucket/path/to/object
@@ -4234,8 +4133,8 @@ Durations are parsed as per the rest of rclone, 2h, 7d, 7w etc.
 	Long: `This command removes any old hidden versions of files
 on a versions enabled bucket.
 
-Note that you can use --interactive/-i or --dry-run with this command to see what
-it would do.
+Note that you can use -i/--dry-run with this command to see what it
+would do.
 
     rclone backend cleanup-hidden s3:bucket/path/to/dir
 `,
@@ -4569,14 +4468,13 @@ func (f *Fs) purge(ctx context.Context, dir string, oldOnly bool) error {
 		delErr <- operations.DeleteFiles(ctx, delChan)
 	}()
 	checkErr(f.list(ctx, listOpt{
-		bucket:        bucket,
-		directory:     directory,
-		prefix:        f.rootDirectory,
-		addBucket:     f.rootBucket == "",
-		recurse:       true,
-		withVersions:  versioned,
-		hidden:        true,
-		noSkipMarkers: true,
+		bucket:       bucket,
+		directory:    directory,
+		prefix:       f.rootDirectory,
+		addBucket:    f.rootBucket == "",
+		recurse:      true,
+		withVersions: versioned,
+		hidden:       true,
 	}, func(remote string, object *s3.Object, versionID *string, isDirectory bool) error {
 		if isDirectory {
 			return nil
@@ -4586,7 +4484,7 @@ func (f *Fs) purge(ctx context.Context, dir string, oldOnly bool) error {
 			fs.Errorf(object, "Can't create object %+v", err)
 			return nil
 		}
-		tr := accounting.Stats(ctx).NewCheckingTransfer(oi, "checking")
+		tr := accounting.Stats(ctx).NewCheckingTransfer(oi)
 		// Work out whether the file is the current version or not
 		isCurrentVersion := !versioned || !version.Match(remote)
 		fs.Debugf(nil, "%q version %v", remote, version.Match(remote))
@@ -4889,12 +4787,23 @@ func (o *Object) downloadFromURL(ctx context.Context, bucketPath string, options
 		return nil, err
 	}
 
-	contentLength := rest.ParseSizeFromHeaders(resp.Header)
-	if contentLength < 0 {
-		fs.Debugf(o, "Failed to parse file size from headers")
+	contentLength := &resp.ContentLength
+	if resp.Header.Get("Content-Range") != "" {
+		var contentRange = resp.Header.Get("Content-Range")
+		slash := strings.IndexRune(contentRange, '/')
+		if slash >= 0 {
+			i, err := strconv.ParseInt(contentRange[slash+1:], 10, 64)
+			if err == nil {
+				contentLength = &i
+			} else {
+				fs.Debugf(o, "Failed to find parse integer from in %q: %v", contentRange, err)
+			}
+		} else {
+			fs.Debugf(o, "Failed to find length in %q", contentRange)
+		}
 	}
 
-	lastModified, err := http.ParseTime(resp.Header.Get("Last-Modified"))
+	lastModified, err := time.Parse(time.RFC1123, resp.Header.Get("Last-Modified"))
 	if err != nil {
 		fs.Debugf(o, "Failed to parse last modified from string %s, %v", resp.Header.Get("Last-Modified"), err)
 	}
@@ -4918,7 +4827,7 @@ func (o *Object) downloadFromURL(ctx context.Context, bucketPath string, options
 
 	var head = s3.HeadObjectOutput{
 		ETag:               header("Etag"),
-		ContentLength:      &contentLength,
+		ContentLength:      contentLength,
 		LastModified:       &lastModified,
 		Metadata:           metaData,
 		CacheControl:       header("Cache-Control"),
@@ -5030,7 +4939,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 
 var warnStreamUpload sync.Once
 
-func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, size int64, in io.Reader) (wantETag, gotETag string, versionID *string, err error) {
+func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, size int64, in io.Reader) (etag string, versionID *string, err error) {
 	f := o.fs
 
 	// make concurrency machinery
@@ -5074,7 +4983,7 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
 		return f.shouldRetry(ctx, err)
 	})
 	if err != nil {
-		return wantETag, gotETag, nil, fmt.Errorf("multipart upload failed to initialise: %w", err)
+		return etag, nil, fmt.Errorf("multipart upload failed to initialise: %w", err)
 	}
 	uid := cout.UploadId
 
@@ -5147,7 +5056,7 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
 			finished = true
 		} else if err != nil {
 			free()
-			return wantETag, gotETag, nil, fmt.Errorf("multipart upload failed to read source: %w", err)
+			return etag, nil, fmt.Errorf("multipart upload failed to read source: %w", err)
 		}
 		buf = buf[:n]
 
@@ -5202,7 +5111,7 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
 	}
 	err = g.Wait()
 	if err != nil {
-		return wantETag, gotETag, nil, err
+		return etag, nil, err
 	}
 
 	// sort the completed parts by part number
@@ -5224,17 +5133,14 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
 		return f.shouldRetry(ctx, err)
 	})
 	if err != nil {
-		return wantETag, gotETag, nil, fmt.Errorf("multipart upload failed to finalise: %w", err)
+		return etag, nil, fmt.Errorf("multipart upload failed to finalise: %w", err)
 	}
 	hashOfHashes := md5.Sum(md5s)
-	wantETag = fmt.Sprintf("%s-%d", hex.EncodeToString(hashOfHashes[:]), len(parts))
+	etag = fmt.Sprintf("%s-%d", hex.EncodeToString(hashOfHashes[:]), len(parts))
 	if resp != nil {
-		if resp.ETag != nil {
-			gotETag = *resp.ETag
-		}
 		versionID = resp.VersionId
 	}
-	return wantETag, gotETag, versionID, nil
+	return etag, versionID, nil
 }
 
 // unWrapAwsError unwraps AWS errors, looking for a non AWS error
@@ -5267,7 +5173,7 @@ func (o *Object) uploadSinglepartPutObject(ctx context.Context, req *s3.PutObjec
 		// Can't upload zero length files like this for some reason
 		r.Body = bytes.NewReader([]byte{})
 	} else {
-		r.SetStreamingBody(io.NopCloser(in))
+		r.SetStreamingBody(ioutil.NopCloser(in))
 	}
 	r.SetContext(ctx)
 	r.HTTPRequest.Header.Set("X-Amz-Content-Sha256", "UNSIGNED-PAYLOAD")
@@ -5381,7 +5287,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 	req := s3.PutObjectInput{
 		Bucket: &bucket,
-		ACL:    stringPointerOrNil(o.fs.opt.ACL),
+		ACL:    &o.fs.opt.ACL,
 		Key:    &bucketPath,
 	}
 
@@ -5534,58 +5440,52 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	}
 
 	var wantETag string        // Multipart upload Etag to check
-	var gotETag string         // Etag we got from the upload
+	var gotEtag string         // Etag we got from the upload
 	var lastModified time.Time // Time we got from the upload
 	var versionID *string      // versionID we got from the upload
 	if multipart {
-		wantETag, gotETag, versionID, err = o.uploadMultipart(ctx, &req, size, in)
+		wantETag, versionID, err = o.uploadMultipart(ctx, &req, size, in)
 	} else {
 		if o.fs.opt.UsePresignedRequest {
-			gotETag, lastModified, versionID, err = o.uploadSinglepartPresignedRequest(ctx, &req, size, in)
+			gotEtag, lastModified, versionID, err = o.uploadSinglepartPresignedRequest(ctx, &req, size, in)
 		} else {
-			gotETag, lastModified, versionID, err = o.uploadSinglepartPutObject(ctx, &req, size, in)
+			gotEtag, lastModified, versionID, err = o.uploadSinglepartPutObject(ctx, &req, size, in)
 		}
 	}
 	if err != nil {
 		return err
 	}
-	// Only record versionID if we are using --s3-versions or --s3-version-at
-	if o.fs.opt.Versions || o.fs.opt.VersionAt.IsSet() {
-		o.versionID = versionID
-	} else {
-		o.versionID = nil
-	}
+	o.versionID = versionID
 
 	// User requested we don't HEAD the object after uploading it
 	// so make up the object as best we can assuming it got
 	// uploaded properly. If size < 0 then we need to do the HEAD.
-	var head *s3.HeadObjectOutput
 	if o.fs.opt.NoHead && size >= 0 {
-		head = new(s3.HeadObjectOutput)
-		//structs.SetFrom(head, &req)
-		setFrom_s3HeadObjectOutput_s3PutObjectInput(head, &req)
+		var head s3.HeadObjectOutput
+		//structs.SetFrom(&head, &req)
+		setFrom_s3HeadObjectOutput_s3PutObjectInput(&head, &req)
 		head.ETag = &md5sumHex // doesn't matter quotes are missing
 		head.ContentLength = &size
-		// We get etag back from single and multipart upload so fill it in here
-		if gotETag != "" {
-			head.ETag = &gotETag
+		// If we have done a single part PUT request then we can read these
+		if gotEtag != "" {
+			head.ETag = &gotEtag
 		}
 		if lastModified.IsZero() {
 			lastModified = time.Now()
 		}
 		head.LastModified = &lastModified
 		head.VersionId = versionID
-	} else {
-		// Read the metadata from the newly created object
-		o.meta = nil // wipe old metadata
-		head, err = o.headObject(ctx)
-		if err != nil {
-			return err
-		}
+		o.setMetaData(&head)
+		return nil
+	}
+
+	// Read the metadata from the newly created object
+	o.meta = nil // wipe old metadata
+	head, err := o.headObject(ctx)
+	if err != nil {
+		return err
 	}
 	o.setMetaData(head)
-
-	// Check multipart upload ETag if required
 	if o.fs.opt.UseMultipartEtag.Value && !o.fs.etagIsNotMD5 && wantETag != "" && head.ETag != nil && *head.ETag != "" {
 		gotETag := strings.Trim(strings.ToLower(*head.ETag), `"`)
 		if wantETag != gotETag {

backend/seafile/renew.go

@@ -1,54 +0,0 @@
-package seafile
-
-import (
-	"sync"
-	"time"
-
-	"github.com/rclone/rclone/fs"
-)
-
-// Renew allows tokens to be renewed on expiry.
-type Renew struct {
-	ts       *time.Ticker     // timer indicating when it's time to renew the token
-	run      func() error     // the callback to do the renewal
-	done     chan interface{} // channel to end the go routine
-	shutdown *sync.Once
-}
-
-// NewRenew creates a new Renew struct and starts a background process
-// which renews the token whenever it expires.  It uses the run() call
-// to do the renewal.
-func NewRenew(every time.Duration, run func() error) *Renew {
-	r := &Renew{
-		ts:       time.NewTicker(every),
-		run:      run,
-		done:     make(chan interface{}),
-		shutdown: &sync.Once{},
-	}
-	go r.renewOnExpiry()
-	return r
-}
-
-func (r *Renew) renewOnExpiry() {
-	for {
-		select {
-		case <-r.ts.C:
-			err := r.run()
-			if err != nil {
-				fs.Errorf(nil, "error while refreshing decryption token: %s", err)
-			}
-
-		case <-r.done:
-			return
-		}
-	}
-}
-
-// Shutdown stops the ticker and no more renewal will take place.
-func (r *Renew) Shutdown() {
-	// closing a channel can only be done once
-	r.shutdown.Do(func() {
-		r.ts.Stop()
-		close(r.done)
-	})
-}

backend/seafile/renew_test.go

@@ -1,34 +0,0 @@
-package seafile
-
-import (
-	"sync/atomic"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestShouldAllowShutdownTwice(t *testing.T) {
-	renew := NewRenew(time.Hour, func() error {
-		return nil
-	})
-	renew.Shutdown()
-	renew.Shutdown()
-}
-
-func TestRenewalInTimeLimit(t *testing.T) {
-	var count int64
-
-	renew := NewRenew(100*time.Millisecond, func() error {
-		atomic.AddInt64(&count, 1)
-		return nil
-	})
-	time.Sleep(time.Second)
-	renew.Shutdown()
-
-	// there's no guarantee the CI agent can handle a simple goroutine
-	renewCount := atomic.LoadInt64(&count)
-	t.Logf("renew count = %d", renewCount)
-	assert.Greater(t, renewCount, int64(0))
-	assert.Less(t, renewCount, int64(11))
-}

backend/seafile/seafile.go

@@ -143,7 +143,6 @@ type Fs struct {
 	createDirMutex      sync.Mutex   // Protect creation of directories
 	useOldDirectoryAPI  bool         // Use the old API v2 if seafile < 7
 	moveDirNotAvailable bool         // Version < 7.0 don't have an API to move a directory
-	renew               *Renew       // Renew an encrypted library token
 }
 
 // ------------------------------------------------------------
@@ -269,11 +268,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 			}
 			// And remove the public link feature
 			f.features.PublicLink = nil
-
-			// renew the library password every 45 minutes
-			f.renew = NewRenew(45*time.Minute, func() error {
-				return f.authorizeLibrary(context.Background(), libraryID)
-			})
 		}
 	} else {
 		// Deactivate the cleaner feature since there's no library selected
@@ -389,15 +383,6 @@ func Config(ctx context.Context, name string, m configmap.Mapper, config fs.Conf
 	return nil, fmt.Errorf("unknown state %q", config.State)
 }
 
-// Shutdown the Fs
-func (f *Fs) Shutdown(ctx context.Context) error {
-	if f.renew == nil {
-		return nil
-	}
-	f.renew.Shutdown()
-	return nil
-}
-
 // sets the AuthorizationToken up
 func (f *Fs) setAuthorizationToken(token string) {
 	f.srv.SetHeader("Authorization", "Token "+token)
@@ -1346,7 +1331,6 @@ var (
 	_ fs.PutStreamer  = &Fs{}
 	_ fs.PublicLinker = &Fs{}
 	_ fs.UserInfoer   = &Fs{}
-	_ fs.Shutdowner   = &Fs{}
 	_ fs.Object       = &Object{}
 	_ fs.IDer         = &Object{}
 )

backend/seafile/webapi.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"net/http"
 	"net/url"
 	"path"
@@ -632,7 +633,7 @@ func (f *Fs) download(ctx context.Context, url string, size int64, options ...fs
 		})
 		if start > 0 {
 			// We need to read and discard the beginning of the data...
-			_, err = io.CopyN(io.Discard, resp.Body, start)
+			_, err = io.CopyN(ioutil.Discard, resp.Body, start)
 			if err != nil {
 				return nil, err
 			}

backend/sftp/sftp.go

@@ -10,6 +10,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"os"
 	"path"
 	"regexp"
@@ -122,10 +123,7 @@ This enables the use of the following insecure ciphers and key exchange methods:
 - diffie-hellman-group-exchange-sha256
 - diffie-hellman-group-exchange-sha1
 
-Those algorithms are insecure and may allow plaintext data to be recovered by an attacker.
-
-This must be false if you use either ciphers or key_exchange advanced options.
-`,
+Those algorithms are insecure and may allow plaintext data to be recovered by an attacker.`,
 			Default: false,
 			Examples: []fs.OptionExample{
 				{
@@ -327,46 +325,6 @@ and pass variables with spaces in in quotes, eg
 
     "VAR3=value with space" "VAR4=value with space" VAR5=nospacehere
 
-`,
-			Advanced: true,
-		}, {
-			Name:    "ciphers",
-			Default: fs.SpaceSepList{},
-			Help: `Space separated list of ciphers to be used for session encryption, ordered by preference.
-
-At least one must match with server configuration. This can be checked for example using ssh -Q cipher.
-
-This must not be set if use_insecure_cipher is true.
-
-Example:
-
-    aes128-ctr aes192-ctr aes256-ctr aes128-gcm@openssh.com aes256-gcm@openssh.com
-`,
-			Advanced: true,
-		}, {
-			Name:    "key_exchange",
-			Default: fs.SpaceSepList{},
-			Help: `Space separated list of key exchange algorithms, ordered by preference.
-
-At least one must match with server configuration. This can be checked for example using ssh -Q kex.
-
-This must not be set if use_insecure_cipher is true.
-
-Example:
-
-    sntrup761x25519-sha512@openssh.com curve25519-sha256 curve25519-sha256@libssh.org ecdh-sha2-nistp256
-`,
-			Advanced: true,
-		}, {
-			Name:    "macs",
-			Default: fs.SpaceSepList{},
-			Help: `Space separated list of MACs (message authentication code) algorithms, ordered by preference.
-
-At least one must match with server configuration. This can be checked for example using ssh -Q mac.
-
-Example:
-
-    umac-64-etm@openssh.com umac-128-etm@openssh.com hmac-sha2-256-etm@openssh.com
 `,
 			Advanced: true,
 		}},
@@ -404,9 +362,6 @@ type Options struct {
 	ChunkSize               fs.SizeSuffix   `config:"chunk_size"`
 	Concurrency             int             `config:"concurrency"`
 	SetEnv                  fs.SpaceSepList `config:"set_env"`
-	Ciphers                 fs.SpaceSepList `config:"ciphers"`
-	KeyExchange             fs.SpaceSepList `config:"key_exchange"`
-	MACs                    fs.SpaceSepList `config:"macs"`
 }
 
 // Fs stores the interface to the remote SFTP files
@@ -747,25 +702,10 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		sshConfig.HostKeyCallback = hostcallback
 	}
 
-	if opt.UseInsecureCipher && (opt.Ciphers != nil || opt.KeyExchange != nil) {
-		return nil, fmt.Errorf("use_insecure_cipher must be false if ciphers or key_exchange are set in advanced configuration")
-	}
-
-	sshConfig.Config.SetDefaults()
 	if opt.UseInsecureCipher {
+		sshConfig.Config.SetDefaults()
 		sshConfig.Config.Ciphers = append(sshConfig.Config.Ciphers, "aes128-cbc", "aes192-cbc", "aes256-cbc", "3des-cbc")
 		sshConfig.Config.KeyExchanges = append(sshConfig.Config.KeyExchanges, "diffie-hellman-group-exchange-sha1", "diffie-hellman-group-exchange-sha256")
-	} else {
-		if opt.Ciphers != nil {
-			sshConfig.Config.Ciphers = opt.Ciphers
-		}
-		if opt.KeyExchange != nil {
-			sshConfig.Config.KeyExchanges = opt.KeyExchange
-		}
-	}
-
-	if opt.MACs != nil {
-		sshConfig.Config.MACs = opt.MACs
 	}
 
 	keyFile := env.ShellExpand(opt.KeyFile)
@@ -782,7 +722,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 			return nil, fmt.Errorf("couldn't read ssh agent signers: %w", err)
 		}
 		if keyFile != "" {
-			pubBytes, err := os.ReadFile(keyFile + ".pub")
+			pubBytes, err := ioutil.ReadFile(keyFile + ".pub")
 			if err != nil {
 				return nil, fmt.Errorf("failed to read public key file: %w", err)
 			}
@@ -811,7 +751,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	if keyFile != "" || opt.KeyPem != "" {
 		var key []byte
 		if opt.KeyPem == "" {
-			key, err = os.ReadFile(keyFile)
+			key, err = ioutil.ReadFile(keyFile)
 			if err != nil {
 				return nil, fmt.Errorf("failed to read private key file: %w", err)
 			}
@@ -842,7 +782,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 
 		// If a public key has been specified then use that
 		if pubkeyFile != "" {
-			certfile, err := os.ReadFile(pubkeyFile)
+			certfile, err := ioutil.ReadFile(pubkeyFile)
 			if err != nil {
 				return nil, fmt.Errorf("unable to read cert file: %w", err)
 			}
@@ -975,24 +915,20 @@ func NewFsWithConnection(ctx context.Context, f *Fs, name string, root string, m
 			fs.Debugf(f, "Running shell type detection remote command: %s", shellCmd)
 			err = session.Run(shellCmd)
 			_ = session.Close()
-			f.shellType = defaultShellType
 			if err != nil {
+				f.shellType = defaultShellType
 				fs.Debugf(f, "Remote command failed: %v (stdout=%v) (stderr=%v)", err, bytes.TrimSpace(stdout.Bytes()), bytes.TrimSpace(stderr.Bytes()))
 			} else {
 				outBytes := stdout.Bytes()
 				fs.Debugf(f, "Remote command result: %s", outBytes)
 				outString := string(bytes.TrimSpace(stdout.Bytes()))
-				if outString != "" {
-					if strings.HasPrefix(outString, "Microsoft.PowerShell") { // PowerShell: "Microsoft.PowerShell%ComSpec%"
-						f.shellType = "powershell"
-					} else if !strings.HasSuffix(outString, "%ComSpec%") { // Command Prompt: "${ShellId}C:\WINDOWS\system32\cmd.exe"
-						// Additional positive test, to avoid misdetection on unpredicted Unix shell variants
-						s := strings.ToLower(outString)
-						if strings.Contains(s, ".exe") || strings.Contains(s, ".com") {
-							f.shellType = "cmd"
-						}
-					} // POSIX-based Unix shell: "%ComSpec%"
-				} // fish Unix shell: ""
+				if strings.HasPrefix(outString, "Microsoft.PowerShell") { // If PowerShell: "Microsoft.PowerShell%ComSpec%"
+					f.shellType = "powershell"
+				} else if !strings.HasSuffix(outString, "%ComSpec%") { // If Command Prompt: "${ShellId}C:\WINDOWS\system32\cmd.exe"
+					f.shellType = "cmd"
+				} else { // If Unix: "%ComSpec%"
+					f.shellType = "unix"
+				}
 			}
 		}
 		// Save permanently in config to avoid the extra work next time
@@ -1775,14 +1711,11 @@ func (o *Object) setMetadata(info os.FileInfo) {
 
 // statRemote stats the file or directory at the remote given
 func (f *Fs) stat(ctx context.Context, remote string) (info os.FileInfo, err error) {
-	absPath := remote
-	if !strings.HasPrefix(remote, "/") {
-		absPath = path.Join(f.absRoot, remote)
-	}
 	c, err := f.getSftpConnection(ctx)
 	if err != nil {
 		return nil, fmt.Errorf("stat: %w", err)
 	}
+	absPath := path.Join(f.absRoot, remote)
 	info, err = c.sftpClient.Stat(absPath)
 	f.putSftpConnection(&c, err)
 	return info, err

backend/sharefile/sharefile.go

@@ -77,6 +77,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"net/http"
 	"net/url"
 	"path"
@@ -478,7 +479,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	if err != nil {
 		return nil, fmt.Errorf("failed to open timezone db: %w", err)
 	}
-	tzdata, err := io.ReadAll(timezone)
+	tzdata, err := ioutil.ReadAll(timezone)
 	if err != nil {
 		return nil, fmt.Errorf("failed to read timezone: %w", err)
 	}

backend/sharefile/tzdata_vfsdata.go

@@ -10,6 +10,7 @@ import (
 	"compress/gzip"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"net/http"
 	"os"
 	pathpkg "path"
@@ -118,7 +119,7 @@ func (f *vfsgen۰CompressedFile) Read(p []byte) (n int, err error) {
 	}
 	if f.grPos < f.seekPos {
 		// Fast-forward.
-		_, err = io.CopyN(io.Discard, f.gr, f.seekPos-f.grPos)
+		_, err = io.CopyN(ioutil.Discard, f.gr, f.seekPos-f.grPos)
 		if err != nil {
 			return 0, err
 		}

backend/smb/connpool.go

@@ -34,10 +34,9 @@ func (f *Fs) dial(ctx context.Context, network, addr string) (*conn, error) {
 
 	d := &smb2.Dialer{
 		Initiator: &smb2.NTLMInitiator{
-			User:      f.opt.User,
-			Password:  pass,
-			Domain:    f.opt.Domain,
-			TargetSPN: f.opt.SPN,
+			User:     f.opt.User,
+			Password: pass,
+			Domain:   f.opt.Domain,
 		},
 	}
 
@@ -82,7 +81,7 @@ func (c *conn) closed() bool {
 		// list the shares
 		_, nopErr = c.smbSession.ListSharenames()
 	}
-	return nopErr != nil
+	return nopErr == nil
 }
 
 // Show that we are using a SMB session

backend/smb/smb.go

@@ -60,17 +60,6 @@ func init() {
 			Name:    "domain",
 			Help:    "Domain name for NTLM authentication.",
 			Default: "WORKGROUP",
-		}, {
-			Name: "spn",
-			Help: `Service principal name.
-
-Rclone presents this name to the server. Some servers use this as further
-authentication, and it often needs to be set for clusters. For example:
-
-    cifs/remotehost:1020
-
-Leave blank if not sure.
-`,
 		}, {
 			Name:    "idle_timeout",
 			Default: fs.Duration(60 * time.Second),
@@ -120,7 +109,6 @@ type Options struct {
 	User            string      `config:"user"`
 	Pass            string      `config:"pass"`
 	Domain          string      `config:"domain"`
-	SPN             string      `config:"spn"`
 	HideSpecial     bool        `config:"hide_special_share"`
 	CaseInsensitive bool        `config:"case_insensitive"`
 	IdleTimeout     fs.Duration `config:"idle_timeout"`
@@ -490,15 +478,11 @@ func (f *Fs) makeEntryRelative(share, _path, relative string, stat os.FileInfo)
 }
 
 func (f *Fs) ensureDirectory(ctx context.Context, share, _path string) error {
-	dir := path.Dir(_path)
-	if dir == "." {
-		return nil
-	}
 	cn, err := f.getConnection(ctx, share)
 	if err != nil {
 		return err
 	}
-	err = cn.smbShare.MkdirAll(f.toSambaPath(dir), 0o755)
+	err = cn.smbShare.MkdirAll(f.toSambaPath(path.Dir(_path)), 0o755)
 	f.putConnection(&cn)
 	return err
 }

backend/storj/fs.go

@@ -23,7 +23,6 @@ import (
 	"golang.org/x/text/unicode/norm"
 
 	"storj.io/uplink"
-	"storj.io/uplink/edge"
 )
 
 const (
@@ -32,9 +31,9 @@ const (
 )
 
 var satMap = map[string]string{
-	"us1.storj.io": "12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S@us1.storj.io:7777",
-	"eu1.storj.io": "12L9ZFwhzVpuEKMUNUqkaTLGzwY9G24tbiigLiXpmZWKwmcNDDs@eu1.storj.io:7777",
-	"ap1.storj.io": "121RTSDpyNZVcEU84Ticf2L1ntiuUimbWgfATz21tuvgk3vzoA6@ap1.storj.io:7777",
+	"us-central-1.storj.io":  "12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S@us-central-1.tardigrade.io:7777",
+	"europe-west-1.storj.io": "12L9ZFwhzVpuEKMUNUqkaTLGzwY9G24tbiigLiXpmZWKwmcNDDs@europe-west-1.tardigrade.io:7777",
+	"asia-east-1.storj.io":   "121RTSDpyNZVcEU84Ticf2L1ntiuUimbWgfATz21tuvgk3vzoA6@asia-east-1.tardigrade.io:7777",
 }
 
 // Register with Fs
@@ -106,16 +105,16 @@ func init() {
 				Name:     "satellite_address",
 				Help:     "Satellite address.\n\nCustom satellite address should match the format: `<nodeid>@<address>:<port>`.",
 				Provider: newProvider,
-				Default:  "us1.storj.io",
+				Default:  "us-central-1.storj.io",
 				Examples: []fs.OptionExample{{
-					Value: "us1.storj.io",
-					Help:  "US1",
+					Value: "us-central-1.storj.io",
+					Help:  "US Central 1",
 				}, {
-					Value: "eu1.storj.io",
-					Help:  "EU1",
+					Value: "europe-west-1.storj.io",
+					Help:  "Europe West 1",
 				}, {
-					Value: "ap1.storj.io",
-					Help:  "AP1",
+					Value: "asia-east-1.storj.io",
+					Help:  "Asia East 1",
 				},
 				},
 			},
@@ -157,13 +156,10 @@ type Fs struct {
 
 // Check the interfaces are satisfied.
 var (
-	_ fs.Fs           = &Fs{}
-	_ fs.ListRer      = &Fs{}
-	_ fs.PutStreamer  = &Fs{}
-	_ fs.Mover        = &Fs{}
-	_ fs.Copier       = &Fs{}
-	_ fs.Purger       = &Fs{}
-	_ fs.PublicLinker = &Fs{}
+	_ fs.Fs          = &Fs{}
+	_ fs.ListRer     = &Fs{}
+	_ fs.PutStreamer = &Fs{}
+	_ fs.Mover       = &Fs{}
 )
 
 // NewFs creates a filesystem backed by Storj.
@@ -548,7 +544,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
 	defer func() {
 		if err != nil {
 			aerr := upload.Abort()
-			if aerr != nil && !errors.Is(aerr, uplink.ErrUploadDone) {
+			if aerr != nil {
 				fs.Errorf(f, "cp input ./%s %+v: %+v", src.Remote(), options, aerr)
 			}
 		}
@@ -563,16 +559,6 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
 
 	_, err = io.Copy(upload, in)
 	if err != nil {
-		if errors.Is(err, uplink.ErrBucketNotFound) {
-			// Rclone assumes the backend will create the bucket if not existing yet.
-			// Here we create the bucket and return a retry error for rclone to retry the upload.
-			_, err = f.project.EnsureBucket(ctx, bucketName)
-			if err != nil {
-				return nil, err
-			}
-			return nil, fserrors.RetryError(errors.New("bucket was not available, now created, the upload must be retried"))
-		}
-
 		err = fserrors.RetryError(err)
 		fs.Errorf(f, "cp input ./%s %+v: %+v\n", src.Remote(), options, err)
 
@@ -734,143 +720,3 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	// Read the new object
 	return f.NewObject(ctx, remote)
 }
-
-// Copy src to this remote using server-side copy operations.
-//
-// This is stored with the remote path given.
-//
-// It returns the destination Object and a possible error.
-//
-// Will only be called if src.Fs().Name() == f.Name()
-//
-// If it isn't possible then return fs.ErrorCantCopy
-func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
-	srcObj, ok := src.(*Object)
-	if !ok {
-		fs.Debugf(src, "Can't copy - not same remote type")
-		return nil, fs.ErrorCantCopy
-	}
-
-	// Copy parameters
-	srcBucket, srcKey := bucket.Split(srcObj.absolute)
-	dstBucket, dstKey := f.absolute(remote)
-	options := uplink.CopyObjectOptions{}
-
-	// Do the copy
-	newObject, err := f.project.CopyObject(ctx, srcBucket, srcKey, dstBucket, dstKey, &options)
-	if err != nil {
-		// Make sure destination bucket exists
-		_, err := f.project.EnsureBucket(ctx, dstBucket)
-		if err != nil {
-			return nil, fmt.Errorf("copy object failed to create destination bucket: %w", err)
-		}
-		// And try again
-		newObject, err = f.project.CopyObject(ctx, srcBucket, srcKey, dstBucket, dstKey, &options)
-		if err != nil {
-			return nil, fmt.Errorf("copy object failed: %w", err)
-		}
-	}
-
-	// Return the new object
-	return newObjectFromUplink(f, remote, newObject), nil
-}
-
-// Purge all files in the directory specified
-//
-// Implement this if you have a way of deleting all the files
-// quicker than just running Remove() on the result of List()
-//
-// Return an error if it doesn't exist
-func (f *Fs) Purge(ctx context.Context, dir string) error {
-	bucket, directory := f.absolute(dir)
-	if bucket == "" {
-		return errors.New("can't purge from root")
-	}
-
-	if directory == "" {
-		_, err := f.project.DeleteBucketWithObjects(ctx, bucket)
-		if errors.Is(err, uplink.ErrBucketNotFound) {
-			return fs.ErrorDirNotFound
-		}
-		return err
-	}
-
-	fs.Infof(directory, "Quick delete is available only for entire bucket. Falling back to list and delete.")
-	objects := f.project.ListObjects(ctx, bucket,
-		&uplink.ListObjectsOptions{
-			Prefix:    directory + "/",
-			Recursive: true,
-		},
-	)
-	if err := objects.Err(); err != nil {
-		return err
-	}
-
-	empty := true
-	for objects.Next() {
-		empty = false
-		_, err := f.project.DeleteObject(ctx, bucket, objects.Item().Key)
-		if err != nil {
-			return err
-		}
-		fs.Infof(objects.Item().Key, "Deleted")
-	}
-
-	if empty {
-		return fs.ErrorDirNotFound
-	}
-
-	return nil
-}
-
-// PublicLink generates a public link to the remote path (usually readable by anyone)
-func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) {
-	bucket, key := f.absolute(remote)
-	if bucket == "" {
-		return "", errors.New("path must be specified")
-	}
-
-	// Rclone requires that a link is only generated if the remote path exists
-	if key == "" {
-		_, err := f.project.StatBucket(ctx, bucket)
-		if err != nil {
-			return "", err
-		}
-	} else {
-		_, err := f.project.StatObject(ctx, bucket, key)
-		if err != nil {
-			if !errors.Is(err, uplink.ErrObjectNotFound) {
-				return "", err
-			}
-			// No object found, check if there is such a prefix
-			iter := f.project.ListObjects(ctx, bucket, &uplink.ListObjectsOptions{Prefix: key + "/"})
-			if iter.Err() != nil {
-				return "", iter.Err()
-			}
-			if !iter.Next() {
-				return "", err
-			}
-		}
-	}
-
-	sharedPrefix := uplink.SharePrefix{Bucket: bucket, Prefix: key}
-
-	permission := uplink.ReadOnlyPermission()
-	if expire.IsSet() {
-		permission.NotAfter = time.Now().Add(time.Duration(expire))
-	}
-
-	sharedAccess, err := f.access.Share(permission, sharedPrefix)
-	if err != nil {
-		return "", fmt.Errorf("sharing access to object failed: %w", err)
-	}
-
-	creds, err := (&edge.Config{
-		AuthServiceAddress: "auth.storjshare.io:7777",
-	}).RegisterAccess(ctx, sharedAccess, &edge.RegisterAccessOptions{Public: true})
-	if err != nil {
-		return "", fmt.Errorf("creating public link failed: %w", err)
-	}
-
-	return edge.JoinShareURL("https://link.storjshare.io", creds.AccessKeyID, bucket, key, nil)
-}

backend/sugarsync/sugarsync_internal_test.go

@@ -2,7 +2,7 @@ package sugarsync
 
 import (
 	"bytes"
-	"io"
+	"io/ioutil"
 	"net/http"
 	"testing"
 
@@ -48,7 +48,7 @@ func TestErrorHandler(t *testing.T) {
 	} {
 		t.Run(test.name, func(t *testing.T) {
 			resp := http.Response{
-				Body:       io.NopCloser(bytes.NewBufferString(test.body)),
+				Body:       ioutil.NopCloser(bytes.NewBufferString(test.body)),
 				StatusCode: test.code,
 				Status:     test.status,
 			}

backend/swift/swift_test.go

@@ -6,6 +6,7 @@ import (
 	"context"
 	"errors"
 	"io"
+	"io/ioutil"
 	"testing"
 
 	"github.com/ncw/swift/v2"
@@ -135,7 +136,7 @@ func (f *Fs) testWithChunkFail(t *testing.T) {
 	buf := bytes.NewBufferString(contents[:errPosition])
 	errMessage := "potato"
 	er := &readers.ErrorReader{Err: errors.New(errMessage)}
-	in := io.NopCloser(io.MultiReader(buf, er))
+	in := ioutil.NopCloser(io.MultiReader(buf, er))
 
 	file.Size = contentSize
 	obji := object.NewStaticObjectInfo(file.Path, file.ModTime, file.Size, true, nil, nil)

backend/union/entry.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"sync"
 	"time"
 
@@ -86,7 +87,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 				errs[i] = fmt.Errorf("%s: %w", o.UpstreamFs().Name(), err)
 				if len(entries) > 1 {
 					// Drain the input buffer to allow other uploads to continue
-					_, _ = io.Copy(io.Discard, readers[i])
+					_, _ = io.Copy(ioutil.Discard, readers[i])
 				}
 			}
 		} else {

backend/union/union.go

@@ -7,6 +7,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"path"
 	"path/filepath"
 	"strings"
@@ -500,7 +501,7 @@ func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, stream bo
 			errs[i] = fmt.Errorf("%s: %w", u.Name(), err)
 			if len(upstreams) > 1 {
 				// Drain the input buffer to allow other uploads to continue
-				_, _ = io.Copy(io.Discard, readers[i])
+				_, _ = io.Copy(ioutil.Discard, readers[i])
 			}
 			return
 		}

backend/uptobox/uptobox.go

@@ -7,6 +7,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"net/http"
 	"net/url"
 	"path"
@@ -214,7 +215,7 @@ func NewFs(ctx context.Context, name string, root string, config configmap.Mappe
 
 	client := fshttp.NewClient(ctx)
 	f.srv = rest.NewClient(client).SetRoot(apiBaseURL)
-	f.IDRegexp = regexp.MustCompile(`https://uptobox\.com/([a-zA-Z0-9]+)`)
+	f.IDRegexp = regexp.MustCompile("https://uptobox.com/([a-zA-Z0-9]+)")
 
 	_, err = f.readMetaDataForPath(ctx, f.dirPath(""), &api.MetadataRequestOptions{Limit: 10})
 	if err != nil {
@@ -238,7 +239,7 @@ func NewFs(ctx context.Context, name string, root string, config configmap.Mappe
 func (f *Fs) decodeError(resp *http.Response, response interface{}) (err error) {
 	defer fs.CheckClose(resp.Body, &err)
 
-	body, err := io.ReadAll(resp.Body)
+	body, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
 		return err
 	}

backend/webdav/webdav.go

@@ -712,7 +712,6 @@ func (f *Fs) listAll(ctx context.Context, dir string, directoriesOnly bool, file
 			continue
 		}
 		subPath := u.Path[len(baseURL.Path):]
-		subPath = strings.TrimPrefix(subPath, "/") // ignore leading / here for davrods
 		if f.opt.Enc != encoder.EncodeZero {
 			subPath = f.opt.Enc.ToStandardPath(subPath)
 		}

backend/webdav/webdav_internal_test.go

@@ -24,23 +24,15 @@ var (
 // prepareServer the test server and return a function to tidy it up afterwards
 // with each request the headers option tests are executed
 func prepareServer(t *testing.T) (configmap.Simple, func()) {
-	// test the headers are there send send a dummy response to About
+	// file server
+	fileServer := http.FileServer(http.Dir(""))
+
+	// test the headers are there then pass on to fileServer
 	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		what := fmt.Sprintf("%s %s: Header ", r.Method, r.URL.Path)
 		assert.Equal(t, headers[1], r.Header.Get(headers[0]), what+headers[0])
 		assert.Equal(t, headers[3], r.Header.Get(headers[2]), what+headers[2])
-		fmt.Fprintf(w, `<d:multistatus xmlns:d="DAV:" xmlns:s="http://sabredav.org/ns" xmlns:oc="http://owncloud.org/ns" xmlns:nc="http://nextcloud.org/ns">
-<d:response>
- <d:href>/remote.php/webdav/</d:href>
- <d:propstat>
-  <d:prop>
-   <d:quota-available-bytes>-3</d:quota-available-bytes>
-   <d:quota-used-bytes>376461895</d:quota-used-bytes>
-  </d:prop>
-  <d:status>HTTP/1.1 200 OK</d:status>
- </d:propstat>
-</d:response>
-</d:multistatus>`)
+		fileServer.ServeHTTP(w, r)
 	})
 
 	// Make the test server
@@ -76,7 +68,7 @@ func TestHeaders(t *testing.T) {
 	f, tidy := prepare(t)
 	defer tidy()
 
-	// send an About response since that is all the dummy server can return
+	// any request will do
 	_, err := f.Features().About(context.Background())
 	require.NoError(t, err)
 }

backend/zoho/zoho.go

@@ -7,6 +7,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"net/http"
 	"net/url"
 	"path"
@@ -1218,7 +1219,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	if partialContent && resp.StatusCode == 200 {
 		if start > 0 {
 			// We need to read and discard the beginning of the data...
-			_, err = io.CopyN(io.Discard, resp.Body, start)
+			_, err = io.CopyN(ioutil.Discard, resp.Body, start)
 			if err != nil {
 				if resp != nil {
 					_ = resp.Body.Close()

bin/backend-versions.sh

@@ -1,17 +0,0 @@
-#!/bin/bash
-# This adds the version each backend was released to its docs page
-set -e
-for backend in $( find backend -maxdepth 1 -type d ); do
-    backend=$(basename $backend)
-    if [[ "$backend" == "backend" || "$backend" == "vfs" || "$backend" == "all" || "$backend" == "azurefile" ]]; then
-        continue
-    fi
-    
-    commit=$(git log --oneline -- $backend | tail -1 | cut -d' ' -f1)
-    if [ "$commit" == "" ]; then
-        commit=$(git log --oneline -- backend/$backend | tail -1 | cut -d' ' -f1)
-    fi
-    version=$(git tag --contains $commit | grep ^v | sort -n | head -1)
-    echo $backend $version
-    sed -i~ "4i versionIntroduced: \"$version\"" docs/content/${backend}.md
-done

bin/cross-compile.go

@@ -9,6 +9,7 @@ import (
 	"encoding/json"
 	"flag"
 	"fmt"
+	"io/ioutil"
 	"log"
 	"os"
 	"os/exec"
@@ -57,7 +58,6 @@ var osarches = []string{
 	"linux/386",
 	"linux/amd64",
 	"linux/arm",
-	"linux/arm-v6",
 	"linux/arm-v7",
 	"linux/arm64",
 	"linux/mips",
@@ -65,12 +65,10 @@ var osarches = []string{
 	"freebsd/386",
 	"freebsd/amd64",
 	"freebsd/arm",
-	"freebsd/arm-v6",
 	"freebsd/arm-v7",
 	"netbsd/386",
 	"netbsd/amd64",
 	"netbsd/arm",
-	"netbsd/arm-v6",
 	"netbsd/arm-v7",
 	"openbsd/386",
 	"openbsd/amd64",
@@ -85,16 +83,13 @@ var archFlags = map[string][]string{
 	"386":    {"GO386=softfloat"},
 	"mips":   {"GOMIPS=softfloat"},
 	"mipsle": {"GOMIPS=softfloat"},
-	"arm":    {"GOARM=5"},
-	"arm-v6": {"GOARM=6"},
 	"arm-v7": {"GOARM=7"},
 }
 
 // Map Go architectures to NFPM architectures
 // Any missing are passed straight through
 var goarchToNfpm = map[string]string{
-	"arm":    "arm5",
-	"arm-v6": "arm6",
+	"arm":    "arm6",
 	"arm-v7": "arm7",
 }
 
@@ -245,7 +240,7 @@ func buildWindowsResourceSyso(goarch string, versionTag string) string {
 		log.Printf("Failed to resolve path: %v", err)
 		return ""
 	}
-	err = os.WriteFile(jsonPath, bs, 0644)
+	err = ioutil.WriteFile(jsonPath, bs, 0644)
 	if err != nil {
 		log.Printf("Failed to write %s: %v", jsonPath, err)
 		return ""
@@ -481,7 +476,7 @@ func main() {
 		run("mkdir", "build")
 	}
 	chdir("build")
-	err := os.WriteFile("version.txt", []byte(fmt.Sprintf("rclone %s\n", version)), 0666)
+	err := ioutil.WriteFile("version.txt", []byte(fmt.Sprintf("rclone %s\n", version)), 0666)
 	if err != nil {
 		log.Fatalf("Couldn't write version.txt: %v", err)
 	}

bin/get-github-release.go

@@ -16,6 +16,7 @@ import (
 	"flag"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"log"
 	"net/http"
 	"net/url"
@@ -167,7 +168,7 @@ func defaultBinDir() string {
 
 // read the body or an error message
 func readBody(in io.Reader) string {
-	data, err := io.ReadAll(in)
+	data, err := ioutil.ReadAll(in)
 	if err != nil {
 		return fmt.Sprintf("Error reading body: %v", err.Error())
 	}

bin/not-in-stable.go

@@ -5,6 +5,7 @@ import (
 	"bytes"
 	"flag"
 	"fmt"
+	"io/ioutil"
 	"log"
 	"os"
 	"os/exec"
@@ -55,7 +56,7 @@ func main() {
 		log.Fatalf("Syntax: %s", os.Args[0])
 	}
 	// v1.54.0
-	versionBytes, err := os.ReadFile("VERSION")
+	versionBytes, err := ioutil.ReadFile("VERSION")
 	if err != nil {
 		log.Fatalf("Failed to read version: %v", err)
 	}

bin/upload-github

@@ -26,8 +26,7 @@ echo "Making release ${VERSION} anchor ${ANCHOR} to repo ${REPO}"
 gh release create "${VERSION}" \
     --repo ${REPO} \
     --title "rclone ${VERSION}" \
-    --notes-file "/tmp/${VERSION}-release-notes" \
-    --draft=true
+    --notes-file "/tmp/${VERSION}-release-notes"
 
 for build in build/*; do
     case $build in
@@ -41,10 +40,6 @@ for build in build/*; do
         "${build}"
 done
 
-gh release edit "${VERSION}" \
-    --repo ${REPO} \
-    --draft=false
-
 gh release view "${VERSION}" \
     --repo ${REPO}
 

cmd/about/about.go

@@ -93,9 +93,6 @@ provided by a backend. Where the value is unlimited it is omitted.
 Some backends does not support the ` + "`rclone about`" + ` command at all,
 see complete list in [documentation](https://rclone.org/overview/#optional-features).
 `,
-	Annotations: map[string]string{
-		"versionIntroduced": "v1.41",
-	},
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(1, 1, command, args)
 		f := cmd.NewFsSrc(args)

cmd/authorize/authorize.go

@@ -12,14 +12,12 @@ import (
 
 var (
 	noAutoBrowser bool
-	template      string
 )
 
 func init() {
 	cmd.Root.AddCommand(commandDefinition)
 	cmdFlags := commandDefinition.Flags()
 	flags.BoolVarP(cmdFlags, &noAutoBrowser, "auth-no-open-browser", "", false, "Do not automatically open auth link in default browser")
-	flags.StringVarP(cmdFlags, &template, "template", "", "", "The path to a custom Go template for generating HTML responses")
 }
 
 var commandDefinition = &cobra.Command{
@@ -30,15 +28,10 @@ Remote authorization. Used to authorize a remote or headless
 rclone from a machine with a browser - use as instructed by
 rclone config.
 
-Use --auth-no-open-browser to prevent rclone to open auth
-link in default browser automatically.
-
-Use --template to generate HTML output via a custom Go template. If a blank string is provided as an argument to this flag, the default template is used.`,
-	Annotations: map[string]string{
-		"versionIntroduced": "v1.27",
-	},
+Use the --auth-no-open-browser to prevent rclone to open auth
+link in default browser automatically.`,
 	RunE: func(command *cobra.Command, args []string) error {
 		cmd.CheckArgs(1, 3, command, args)
-		return config.Authorize(context.Background(), args, noAutoBrowser, template)
+		return config.Authorize(context.Background(), args, noAutoBrowser)
 	},
 }

cmd/backend/backend.go

@@ -58,9 +58,6 @@ Pass arguments to the backend by placing them on the end of the line
 Note to run these commands on a running backend then see
 [backend/command](/rc/#backend-command) in the rc docs.
 `,
-	Annotations: map[string]string{
-		"versionIntroduced": "v1.52",
-	},
 	RunE: func(command *cobra.Command, args []string) error {
 		cmd.CheckArgs(2, 1e6, command, args)
 		name, remote := args[0], args[1]

cmd/bisync/bilib/files.go

@@ -5,6 +5,7 @@ package bilib
 import (
 	"fmt"
 	"io"
+	"io/ioutil"
 	"os"
 	"path/filepath"
 	"regexp"
@@ -105,7 +106,7 @@ func CopyDir(src string, dst string) (err error) {
 		return
 	}
 
-	entries, err := os.ReadDir(src)
+	entries, err := ioutil.ReadDir(src)
 	if err != nil {
 		return
 	}
@@ -121,7 +122,7 @@ func CopyDir(src string, dst string) (err error) {
 			}
 		} else {
 			// Skip symlinks.
-			if entry.Type()&os.ModeSymlink != 0 {
+			if entry.Mode()&os.ModeSymlink != 0 {
 				continue
 			}
 

cmd/bisync/bilib/names.go

@@ -2,7 +2,7 @@ package bilib
 
 import (
 	"bytes"
-	"os"
+	"io/ioutil"
 	"sort"
 	"strconv"
 )
@@ -57,5 +57,5 @@ func SaveList(list []string, path string) error {
 		_, _ = buf.WriteString(strconv.Quote(s))
 		_ = buf.WriteByte('\n')
 	}
-	return os.WriteFile(path, buf.Bytes(), PermSecure)
+	return ioutil.WriteFile(path, buf.Bytes(), PermSecure)
 }

cmd/bisync/bisync_test.go

@@ -10,6 +10,7 @@ import (
 	"errors"
 	"flag"
 	"fmt"
+	"io/ioutil"
 	"log"
 	"os"
 	"path"
@@ -302,7 +303,7 @@ func (b *bisyncTest) runTestCase(ctx context.Context, t *testing.T, testCase str
 
 	// Execute test scenario
 	scenFile := filepath.Join(b.testDir, "scenario.txt")
-	scenBuf, err := os.ReadFile(scenFile)
+	scenBuf, err := ioutil.ReadFile(scenFile)
 	scenReplacer := b.newReplacer(false)
 	require.NoError(b.t, err)
 	b.step = 0
@@ -902,8 +903,8 @@ func (b *bisyncTest) compareResults() int {
 			// save mangled logs so difference is easier on eyes
 			goldenFile := filepath.Join(b.logDir, "mangled.golden.log")
 			resultFile := filepath.Join(b.logDir, "mangled.result.log")
-			require.NoError(b.t, os.WriteFile(goldenFile, []byte(goldenText), bilib.PermSecure))
-			require.NoError(b.t, os.WriteFile(resultFile, []byte(resultText), bilib.PermSecure))
+			require.NoError(b.t, ioutil.WriteFile(goldenFile, []byte(goldenText), bilib.PermSecure))
+			require.NoError(b.t, ioutil.WriteFile(resultFile, []byte(resultText), bilib.PermSecure))
 		}
 
 		if goldenText == resultText {
@@ -973,7 +974,7 @@ func (b *bisyncTest) storeGolden() {
 
 		goldName := b.toGolden(fileName)
 		goldPath := filepath.Join(b.goldenDir, goldName)
-		err := os.WriteFile(goldPath, []byte(text), bilib.PermSecure)
+		err := ioutil.WriteFile(goldPath, []byte(text), bilib.PermSecure)
 		assert.NoError(b.t, err, "writing golden file %s", goldName)
 
 		if goldName != fileName {
@@ -985,7 +986,7 @@ func (b *bisyncTest) storeGolden() {
 
 // mangleResult prepares test logs or listings for comparison
 func (b *bisyncTest) mangleResult(dir, file string, golden bool) string {
-	buf, err := os.ReadFile(filepath.Join(dir, file))
+	buf, err := ioutil.ReadFile(filepath.Join(dir, file))
 	require.NoError(b.t, err)
 	text := string(buf)
 
@@ -1204,7 +1205,7 @@ func (b *bisyncTest) ensureDir(parent, dir string, optional bool) string {
 }
 
 func (b *bisyncTest) listDir(dir string) (names []string) {
-	files, err := os.ReadDir(dir)
+	files, err := ioutil.ReadDir(dir)
 	require.NoError(b.t, err)
 	for _, file := range files {
 		names = append(names, filepath.Base(file.Name()))

cmd/bisync/cmd.go

@@ -9,6 +9,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
@@ -115,9 +116,6 @@ var commandDefinition = &cobra.Command{
 	Use:   "bisync remote1:path1 remote2:path2",
 	Short: shortHelp,
 	Long:  longHelp,
-	Annotations: map[string]string{
-		"versionIntroduced": "v1.58",
-	},
 	RunE: func(command *cobra.Command, args []string) error {
 		cmd.CheckArgs(2, 2, command, args)
 		fs1, file1, fs2, file2 := cmd.NewFsSrcDstFiles(args)
@@ -200,7 +198,7 @@ func (opt *Options) applyFilters(ctx context.Context) (context.Context, error) {
 	_ = f.Close()
 
 	hashFile := filtersFile + ".md5"
-	wantHash, err := os.ReadFile(hashFile)
+	wantHash, err := ioutil.ReadFile(hashFile)
 	if err != nil && !opt.Resync {
 		return ctx, fmt.Errorf("filters file md5 hash not found (must run --resync): %s", filtersFile)
 	}
@@ -211,7 +209,7 @@ func (opt *Options) applyFilters(ctx context.Context) (context.Context, error) {
 
 	if opt.Resync {
 		fs.Infof(nil, "Storing filters file hash to %s", hashFile)
-		if err := os.WriteFile(hashFile, []byte(gotHash), bilib.PermSecure); err != nil {
+		if err := ioutil.WriteFile(hashFile, []byte(gotHash), bilib.PermSecure); err != nil {
 			return ctx, err
 		}
 	}

cmd/bisync/operations.go

@@ -7,6 +7,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strconv"
@@ -80,7 +81,7 @@ func Bisync(ctx context.Context, fs1, fs2 fs.Fs, optArg *Options) (err error) {
 		}
 
 		pidStr := []byte(strconv.Itoa(os.Getpid()))
-		if err = os.WriteFile(lockFile, pidStr, bilib.PermSecure); err != nil {
+		if err = ioutil.WriteFile(lockFile, pidStr, bilib.PermSecure); err != nil {
 			return fmt.Errorf("cannot create lock file: %s: %w", lockFile, err)
 		}
 		fs.Debugf(nil, "Lock file created: %s", lockFile)

cmd/cachestats/cachestats.go

@@ -25,10 +25,6 @@ var commandDefinition = &cobra.Command{
 Print cache stats for a remote in JSON format
 `,
 	Hidden: true,
-	Annotations: map[string]string{
-		"versionIntroduced": "v1.39",
-		"status":            "Deprecated",
-	},
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(1, 1, command, args)
 		fs.Logf(nil, `"rclone cachestats" is deprecated, use "rclone backend stats %s" instead`, args[0])

cmd/cat/cat.go

@@ -4,6 +4,7 @@ package cat
 import (
 	"context"
 	"io"
+	"io/ioutil"
 	"log"
 	"os"
 	"strings"
@@ -57,9 +58,6 @@ the end and |--offset| and |--count| to print a section in the middle.
 Note that if offset is negative it will count from the end, so
 |--offset -1 --count 1| is equivalent to |--tail 1|.
 `, "|", "`"),
-	Annotations: map[string]string{
-		"versionIntroduced": "v1.33",
-	},
 	Run: func(command *cobra.Command, args []string) {
 		usedOffset := offset != 0 || count >= 0
 		usedHead := head > 0
@@ -79,7 +77,7 @@ Note that if offset is negative it will count from the end, so
 		fsrc := cmd.NewFsSrc(args)
 		var w io.Writer = os.Stdout
 		if discard {
-			w = io.Discard
+			w = ioutil.Discard
 		}
 		cmd.Run(false, false, command, func() error {
 			return operations.Cat(context.Background(), fsrc, w, offset, count)

cmd/checksum/checksum.go

@@ -37,9 +37,6 @@ that don't support hashes or if you really want to check all the data.
 
 Note that hash values in the SUM file are treated as case insensitive.
 `, "|", "`") + check.FlagsHelp,
-	Annotations: map[string]string{
-		"versionIntroduced": "v1.56",
-	},
 	RunE: func(command *cobra.Command, args []string) error {
 		cmd.CheckArgs(3, 3, command, args)
 		var hashType hash.Type

cmd/cleanup/cleanup.go

@@ -20,9 +20,6 @@ var commandDefinition = &cobra.Command{
 Clean up the remote if possible.  Empty the trash or delete old file
 versions. Not supported by all remotes.
 `,
-	Annotations: map[string]string{
-		"versionIntroduced": "v1.31",
-	},
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(1, 1, command, args)
 		fsrc := cmd.NewFsSrc(args)

cmd/cmd.go

@@ -73,13 +73,11 @@ func ShowVersion() {
 
 	linking, tagString := buildinfo.GetLinkingAndTags()
 
-	arch := buildinfo.GetArch()
-
 	fmt.Printf("rclone %s\n", fs.Version)
 	fmt.Printf("- os/version: %s\n", osVersion)
 	fmt.Printf("- os/kernel: %s\n", osKernel)
 	fmt.Printf("- os/type: %s\n", runtime.GOOS)
-	fmt.Printf("- os/arch: %s\n", arch)
+	fmt.Printf("- os/arch: %s\n", runtime.GOARCH)
 	fmt.Printf("- go/version: %s\n", runtime.Version())
 	fmt.Printf("- go/linking: %s\n", linking)
 	fmt.Printf("- go/tags: %s\n", tagString)
@@ -401,15 +399,9 @@ func initConfig() {
 	// Start accounting
 	accounting.Start(ctx)
 
-	// Configure console
+	// Hide console window
 	if ci.NoConsole {
-		// Hide the console window
 		terminal.HideConsole()
-	} else {
-		// Enable color support on stdout if possible.
-		// This enables virtual terminal processing on Windows 10,
-		// adding native support for ANSI/VT100 escape sequences.
-		terminal.EnableColorsStdout()
 	}
 
 	// Load filters