Replace sync with lib/sync for deadlock detection with deadlock build tag

Before this fix the SFTP sessions could timeout when doing hashes if
they took longer than the --timeout parameter.

This patch sends keepalive packets every minute while a shell command
is running to keep the connection open.

See: https://forum.rclone.org/t/rclone-check-over-sftp-failure-to-calculate-md5-hash-for-large-files/27487

backend/azureblob/azureblob.go

@@ -19,7 +19,7 @@ import (
 	"path"
 	"strconv"
 	"strings"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 	"time"
 
 	"github.com/Azure/azure-pipeline-go/pipeline"
@@ -50,8 +50,6 @@ const (
 	timeFormatOut         = "2006-01-02T15:04:05.000000000Z07:00"
 	storageDefaultBaseURL = "blob.core.windows.net"
 	defaultChunkSize      = 4 * fs.Mebi
-	maxChunkSize          = 100 * fs.Mebi
-	uploadConcurrency     = 4
 	defaultAccessTier     = azblob.AccessTierNone
 	maxTryTimeout         = time.Hour * 24 * 365 //max time of an azure web request response window (whether or not data is flowing)
 	// Default storage account, key and blob endpoint for emulator support,
@@ -134,12 +132,33 @@ msi_client_id, or msi_mi_res_id parameters.`,
 			Advanced: true,
 		}, {
 			Name: "chunk_size",
-			Help: `Upload chunk size (<= 100 MiB).
+			Help: `Upload chunk size.
 
 Note that this is stored in memory and there may be up to
-"--transfers" chunks stored at once in memory.`,
+"--transfers" * "--azureblob-upload-concurrency" chunks stored at once
+in memory.`,
 			Default:  defaultChunkSize,
 			Advanced: true,
+		}, {
+			Name: "upload_concurrency",
+			Help: `Concurrency for multipart uploads.
+
+This is the number of chunks of the same file that are uploaded
+concurrently.
+
+If you are uploading small numbers of large files over high-speed
+links and these uploads do not fully utilize your bandwidth, then
+increasing this may help to speed up the transfers.
+
+In tests, upload speed increases almost linearly with upload
+concurrency. For example to fill a gigabit pipe it may be necessary to
+raise this to 64. Note that this will use more memory.
+
+Note that chunks are stored in memory and there may be up to
+"--transfers" * "--azureblob-upload-concurrency" chunks stored at once
+in memory.`,
+			Default:  16,
+			Advanced: true,
 		}, {
 			Name: "list_chunk",
 			Help: `Size of blob list.
@@ -257,6 +276,7 @@ type Options struct {
 	Endpoint             string               `config:"endpoint"`
 	SASURL               string               `config:"sas_url"`
 	ChunkSize            fs.SizeSuffix        `config:"chunk_size"`
+	UploadConcurrency    int                  `config:"upload_concurrency"`
 	ListChunkSize        uint                 `config:"list_chunk"`
 	AccessTier           string               `config:"access_tier"`
 	ArchiveTierDelete    bool                 `config:"archive_tier_delete"`
@@ -416,9 +436,6 @@ func checkUploadChunkSize(cs fs.SizeSuffix) error {
 	if cs < minChunkSize {
 		return fmt.Errorf("%s is less than %s", cs, minChunkSize)
 	}
-	if cs > maxChunkSize {
-		return fmt.Errorf("%s is greater than %s", cs, maxChunkSize)
-	}
 	return nil
 }
 
@@ -1667,10 +1684,10 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 	putBlobOptions := azblob.UploadStreamToBlockBlobOptions{
 		BufferSize:      int(o.fs.opt.ChunkSize),
-		MaxBuffers:      uploadConcurrency,
+		MaxBuffers:      o.fs.opt.UploadConcurrency,
 		Metadata:        o.meta,
 		BlobHTTPHeaders: httpHeaders,
-		TransferManager: o.fs.newPoolWrapper(uploadConcurrency),
+		TransferManager: o.fs.newPoolWrapper(o.fs.opt.UploadConcurrency),
 	}
 
 	// Don't retry, return a retry error instead

backend/azureblob/azureblob_test.go

@@ -17,12 +17,10 @@ import (
 // TestIntegration runs integration tests against the remote
 func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
-		RemoteName:  "TestAzureBlob:",
+		RemoteName:    "TestAzureBlob:",
-		NilObject:   (*Object)(nil),
+		NilObject:     (*Object)(nil),
-		TiersToTest: []string{"Hot", "Cool"},
+		TiersToTest:   []string{"Hot", "Cool"},
-		ChunkedUpload: fstests.ChunkedUploadConfig{
+		ChunkedUpload: fstests.ChunkedUploadConfig{},
-			MaxChunkSize: maxChunkSize,
-		},
 	})
 }
 

backend/b2/b2.go

@@ -17,7 +17,7 @@ import (
 	"path"
 	"strconv"
 	"strings"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 	"time"
 
 	"github.com/rclone/rclone/backend/b2/api"

backend/b2/upload.go

@@ -13,7 +13,7 @@ import (
 	gohash "hash"
 	"io"
 	"strings"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 
 	"github.com/rclone/rclone/backend/b2/api"
 	"github.com/rclone/rclone/fs"

backend/box/box.go

@@ -23,7 +23,7 @@ import (
 	"path"
 	"strconv"
 	"strings"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 	"sync/atomic"
 	"time"
 

backend/box/upload.go

@@ -13,7 +13,7 @@ import (
 	"io"
 	"net/http"
 	"strconv"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 	"time"
 
 	"github.com/rclone/rclone/backend/box/api"

backend/cache/cache.go

@@ -16,7 +16,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 	"syscall"
 	"time"
 
@@ -489,7 +489,7 @@ func NewFs(ctx context.Context, name, rootPath string, m configmap.Mapper) (fs.F
 		f.opt.TempWritePath = filepath.ToSlash(f.opt.TempWritePath)
 		f.tempFs, err = cache.Get(ctx, f.opt.TempWritePath)
 		if err != nil {
-			return nil, fmt.Errorf("failed to create temp fs: %v: %w", err, err)
+			return nil, fmt.Errorf("failed to create temp fs: %w", err)
 		}
 		fs.Infof(name, "Upload Temp Rest Time: %v", f.opt.TempWaitTime)
 		fs.Infof(name, "Upload Temp FS: %v", f.opt.TempWritePath)

backend/cache/handle.go

@@ -11,7 +11,7 @@ import (
 	"path"
 	"runtime"
 	"strings"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 	"time"
 
 	"github.com/rclone/rclone/fs"

backend/cache/object.go

@@ -8,7 +8,7 @@ import (
 	"fmt"
 	"io"
 	"path"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 	"time"
 
 	"github.com/rclone/rclone/fs"

backend/cache/plex.go

@@ -12,7 +12,7 @@ import (
 	"net/http"
 	"net/url"
 	"strings"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 	"time"
 
 	cache "github.com/patrickmn/go-cache"

backend/cache/storage_persistent.go

@@ -14,7 +14,7 @@ import (
 	"path"
 	"strconv"
 	"strings"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 	"time"
 
 	"github.com/rclone/rclone/fs"

backend/chunker/chunker.go

@@ -19,7 +19,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 	"time"
 
 	"github.com/rclone/rclone/fs"

backend/crypt/cipher.go

@@ -7,15 +7,17 @@ import (
 	gocipher "crypto/cipher"
 	"crypto/rand"
 	"encoding/base32"
+	"encoding/base64"
 	"errors"
 	"fmt"
 	"io"
 	"strconv"
 	"strings"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 	"time"
 	"unicode/utf8"
 
+	"github.com/Max-Sum/base32768"
 	"github.com/rclone/rclone/backend/crypt/pkcs7"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
@@ -114,6 +116,57 @@ func (mode NameEncryptionMode) String() (out string) {
 	return out
 }
 
+// fileNameEncoding are the encoding methods dealing with encrypted file names
+type fileNameEncoding interface {
+	EncodeToString(src []byte) string
+	DecodeString(s string) ([]byte, error)
+}
+
+// caseInsensitiveBase32Encoding defines a file name encoding
+// using a modified version of standard base32 as described in
+// RFC4648
+//
+// The standard encoding is modified in two ways
+//  * it becomes lower case (no-one likes upper case filenames!)
+//  * we strip the padding character `=`
+type caseInsensitiveBase32Encoding struct{}
+
+// EncodeToString encodes a strign using the modified version of
+// base32 encoding.
+func (caseInsensitiveBase32Encoding) EncodeToString(src []byte) string {
+	encoded := base32.HexEncoding.EncodeToString(src)
+	encoded = strings.TrimRight(encoded, "=")
+	return strings.ToLower(encoded)
+}
+
+// DecodeString decodes a string as encoded by EncodeToString
+func (caseInsensitiveBase32Encoding) DecodeString(s string) ([]byte, error) {
+	if strings.HasSuffix(s, "=") {
+		return nil, ErrorBadBase32Encoding
+	}
+	// First figure out how many padding characters to add
+	roundUpToMultipleOf8 := (len(s) + 7) &^ 7
+	equals := roundUpToMultipleOf8 - len(s)
+	s = strings.ToUpper(s) + "========"[:equals]
+	return base32.HexEncoding.DecodeString(s)
+}
+
+// NewNameEncoding creates a NameEncoding from a string
+func NewNameEncoding(s string) (enc fileNameEncoding, err error) {
+	s = strings.ToLower(s)
+	switch s {
+	case "base32":
+		enc = caseInsensitiveBase32Encoding{}
+	case "base64":
+		enc = base64.RawURLEncoding
+	case "base32768":
+		enc = base32768.SafeEncoding
+	default:
+		err = fmt.Errorf("Unknown file name encoding mode %q", s)
+	}
+	return enc, err
+}
+
 // Cipher defines an encoding and decoding cipher for the crypt backend
 type Cipher struct {
 	dataKey        [32]byte                  // Key for secretbox
@@ -121,15 +174,17 @@ type Cipher struct {
 	nameTweak      [nameCipherBlockSize]byte // used to tweak the name crypto
 	block          gocipher.Block
 	mode           NameEncryptionMode
+	fileNameEnc    fileNameEncoding
 	buffers        sync.Pool // encrypt/decrypt buffers
 	cryptoRand     io.Reader // read crypto random numbers from here
 	dirNameEncrypt bool
 }
 
 // newCipher initialises the cipher.  If salt is "" then it uses a built in salt val
-func newCipher(mode NameEncryptionMode, password, salt string, dirNameEncrypt bool) (*Cipher, error) {
+func newCipher(mode NameEncryptionMode, password, salt string, dirNameEncrypt bool, enc fileNameEncoding) (*Cipher, error) {
 	c := &Cipher{
 		mode:           mode,
+		fileNameEnc:    enc,
 		cryptoRand:     rand.Reader,
 		dirNameEncrypt: dirNameEncrypt,
 	}
@@ -187,30 +242,6 @@ func (c *Cipher) putBlock(buf []byte) {
 	c.buffers.Put(buf)
 }
 
-// encodeFileName encodes a filename using a modified version of
-// standard base32 as described in RFC4648
-//
-// The standard encoding is modified in two ways
-//  * it becomes lower case (no-one likes upper case filenames!)
-//  * we strip the padding character `=`
-func encodeFileName(in []byte) string {
-	encoded := base32.HexEncoding.EncodeToString(in)
-	encoded = strings.TrimRight(encoded, "=")
-	return strings.ToLower(encoded)
-}
-
-// decodeFileName decodes a filename as encoded by encodeFileName
-func decodeFileName(in string) ([]byte, error) {
-	if strings.HasSuffix(in, "=") {
-		return nil, ErrorBadBase32Encoding
-	}
-	// First figure out how many padding characters to add
-	roundUpToMultipleOf8 := (len(in) + 7) &^ 7
-	equals := roundUpToMultipleOf8 - len(in)
-	in = strings.ToUpper(in) + "========"[:equals]
-	return base32.HexEncoding.DecodeString(in)
-}
-
 // encryptSegment encrypts a path segment
 //
 // This uses EME with AES
@@ -231,7 +262,7 @@ func (c *Cipher) encryptSegment(plaintext string) string {
 	}
 	paddedPlaintext := pkcs7.Pad(nameCipherBlockSize, []byte(plaintext))
 	ciphertext := eme.Transform(c.block, c.nameTweak[:], paddedPlaintext, eme.DirectionEncrypt)
-	return encodeFileName(ciphertext)
+	return c.fileNameEnc.EncodeToString(ciphertext)
 }
 
 // decryptSegment decrypts a path segment
@@ -239,7 +270,7 @@ func (c *Cipher) decryptSegment(ciphertext string) (string, error) {
 	if ciphertext == "" {
 		return "", nil
 	}
-	rawCiphertext, err := decodeFileName(ciphertext)
+	rawCiphertext, err := c.fileNameEnc.DecodeString(ciphertext)
 	if err != nil {
 		return "", err
 	}

backend/crypt/cipher_test.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/base32"
+	"encoding/base64"
 	"errors"
 	"fmt"
 	"io"
@@ -11,6 +12,7 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/Max-Sum/base32768"
 	"github.com/rclone/rclone/backend/crypt/pkcs7"
 	"github.com/rclone/rclone/lib/readers"
 	"github.com/stretchr/testify/assert"
@@ -45,11 +47,31 @@ func TestNewNameEncryptionModeString(t *testing.T) {
 	assert.Equal(t, NameEncryptionMode(3).String(), "Unknown mode #3")
 }
 
-func TestEncodeFileName(t *testing.T) {
+type EncodingTestCase struct {
-	for _, test := range []struct {
+	in       string
-		in       string
+	expected string
-		expected string
+}
-	}{
+
+func testEncodeFileName(t *testing.T, encoding string, testCases []EncodingTestCase, caseInsensitive bool) {
+	for _, test := range testCases {
+		enc, err := NewNameEncoding(encoding)
+		assert.NoError(t, err, "There should be no error creating name encoder for base32.")
+		actual := enc.EncodeToString([]byte(test.in))
+		assert.Equal(t, actual, test.expected, fmt.Sprintf("in=%q", test.in))
+		recovered, err := enc.DecodeString(test.expected)
+		assert.NoError(t, err)
+		assert.Equal(t, string(recovered), test.in, fmt.Sprintf("reverse=%q", test.expected))
+		if caseInsensitive {
+			in := strings.ToUpper(test.expected)
+			recovered, err = enc.DecodeString(in)
+			assert.NoError(t, err)
+			assert.Equal(t, string(recovered), test.in, fmt.Sprintf("reverse=%q", in))
+		}
+	}
+}
+
+func TestEncodeFileNameBase32(t *testing.T) {
+	testEncodeFileName(t, "base32", []EncodingTestCase{
 		{"", ""},
 		{"1", "64"},
 		{"12", "64p0"},
@@ -67,20 +89,56 @@ func TestEncodeFileName(t *testing.T) {
 		{"12345678901234", "64p36d1l6orjge9g64p36d0"},
 		{"123456789012345", "64p36d1l6orjge9g64p36d1l"},
 		{"1234567890123456", "64p36d1l6orjge9g64p36d1l6o"},
-	} {
+	}, true)
-		actual := encodeFileName([]byte(test.in))
-		assert.Equal(t, actual, test.expected, fmt.Sprintf("in=%q", test.in))
-		recovered, err := decodeFileName(test.expected)
-		assert.NoError(t, err)
-		assert.Equal(t, string(recovered), test.in, fmt.Sprintf("reverse=%q", test.expected))
-		in := strings.ToUpper(test.expected)
-		recovered, err = decodeFileName(in)
-		assert.NoError(t, err)
-		assert.Equal(t, string(recovered), test.in, fmt.Sprintf("reverse=%q", in))
-	}
 }
 
-func TestDecodeFileName(t *testing.T) {
+func TestEncodeFileNameBase64(t *testing.T) {
+	testEncodeFileName(t, "base64", []EncodingTestCase{
+		{"", ""},
+		{"1", "MQ"},
+		{"12", "MTI"},
+		{"123", "MTIz"},
+		{"1234", "MTIzNA"},
+		{"12345", "MTIzNDU"},
+		{"123456", "MTIzNDU2"},
+		{"1234567", "MTIzNDU2Nw"},
+		{"12345678", "MTIzNDU2Nzg"},
+		{"123456789", "MTIzNDU2Nzg5"},
+		{"1234567890", "MTIzNDU2Nzg5MA"},
+		{"12345678901", "MTIzNDU2Nzg5MDE"},
+		{"123456789012", "MTIzNDU2Nzg5MDEy"},
+		{"1234567890123", "MTIzNDU2Nzg5MDEyMw"},
+		{"12345678901234", "MTIzNDU2Nzg5MDEyMzQ"},
+		{"123456789012345", "MTIzNDU2Nzg5MDEyMzQ1"},
+		{"1234567890123456", "MTIzNDU2Nzg5MDEyMzQ1Ng"},
+	}, false)
+}
+
+func TestEncodeFileNameBase32768(t *testing.T) {
+	testEncodeFileName(t, "base32768", []EncodingTestCase{
+		{"", ""},
+		{"1", "㼿"},
+		{"12", "㻙ɟ"},
+		{"123", "㻙ⲿ"},
+		{"1234", "㻙ⲍƟ"},
+		{"12345", "㻙ⲍ⍟"},
+		{"123456", "㻙ⲍ⍆ʏ"},
+		{"1234567", "㻙ⲍ⍆觟"},
+		{"12345678", "㻙ⲍ⍆觓ɧ"},
+		{"123456789", "㻙ⲍ⍆觓栯"},
+		{"1234567890", "㻙ⲍ⍆觓栩ɣ"},
+		{"12345678901", "㻙ⲍ⍆觓栩朧"},
+		{"123456789012", "㻙ⲍ⍆觓栩朤ʅ"},
+		{"1234567890123", "㻙ⲍ⍆觓栩朤談"},
+		{"12345678901234", "㻙ⲍ⍆觓栩朤諆ɔ"},
+		{"123456789012345", "㻙ⲍ⍆觓栩朤諆媕"},
+		{"1234567890123456", "㻙ⲍ⍆觓栩朤諆媕䆿"},
+	}, false)
+}
+
+func TestDecodeFileNameBase32(t *testing.T) {
+	enc, err := NewNameEncoding("base32")
+	assert.NoError(t, err, "There should be no error creating name encoder for base32.")
 	// We've tested decoding the valid ones above, now concentrate on the invalid ones
 	for _, test := range []struct {
 		in          string
@@ -90,17 +148,65 @@ func TestDecodeFileName(t *testing.T) {
 		{"!", base32.CorruptInputError(0)},
 		{"hello=hello", base32.CorruptInputError(5)},
 	} {
-		actual, actualErr := decodeFileName(test.in)
+		actual, actualErr := enc.DecodeString(test.in)
 		assert.Equal(t, test.expectedErr, actualErr, fmt.Sprintf("in=%q got actual=%q, err = %v %T", test.in, actual, actualErr, actualErr))
 	}
 }
 
-func TestEncryptSegment(t *testing.T) {
+func TestDecodeFileNameBase64(t *testing.T) {
-	c, _ := newCipher(NameEncryptionStandard, "", "", true)
+	enc, err := NewNameEncoding("base64")
+	assert.NoError(t, err, "There should be no error creating name encoder for base32.")
+	// We've tested decoding the valid ones above, now concentrate on the invalid ones
 	for _, test := range []struct {
-		in       string
+		in          string
-		expected string
+		expectedErr error
 	}{
+		{"64=", base64.CorruptInputError(2)},
+		{"!", base64.CorruptInputError(0)},
+		{"Hello=Hello", base64.CorruptInputError(5)},
+	} {
+		actual, actualErr := enc.DecodeString(test.in)
+		assert.Equal(t, test.expectedErr, actualErr, fmt.Sprintf("in=%q got actual=%q, err = %v %T", test.in, actual, actualErr, actualErr))
+	}
+}
+
+func TestDecodeFileNameBase32768(t *testing.T) {
+	enc, err := NewNameEncoding("base32768")
+	assert.NoError(t, err, "There should be no error creating name encoder for base32.")
+	// We've tested decoding the valid ones above, now concentrate on the invalid ones
+	for _, test := range []struct {
+		in          string
+		expectedErr error
+	}{
+		{"㼿c", base32768.CorruptInputError(1)},
+		{"!", base32768.CorruptInputError(0)},
+		{"㻙ⲿ=㻙ⲿ", base32768.CorruptInputError(2)},
+	} {
+		actual, actualErr := enc.DecodeString(test.in)
+		assert.Equal(t, test.expectedErr, actualErr, fmt.Sprintf("in=%q got actual=%q, err = %v %T", test.in, actual, actualErr, actualErr))
+	}
+}
+
+func testEncryptSegment(t *testing.T, encoding string, testCases []EncodingTestCase, caseInsensitive bool) {
+	enc, _ := NewNameEncoding(encoding)
+	c, _ := newCipher(NameEncryptionStandard, "", "", true, enc)
+	for _, test := range testCases {
+		actual := c.encryptSegment(test.in)
+		assert.Equal(t, test.expected, actual, fmt.Sprintf("Testing %q", test.in))
+		recovered, err := c.decryptSegment(test.expected)
+		assert.NoError(t, err, fmt.Sprintf("Testing reverse %q", test.expected))
+		assert.Equal(t, test.in, recovered, fmt.Sprintf("Testing reverse %q", test.expected))
+		if caseInsensitive {
+			in := strings.ToUpper(test.expected)
+			recovered, err = c.decryptSegment(in)
+			assert.NoError(t, err, fmt.Sprintf("Testing reverse %q", in))
+			assert.Equal(t, test.in, recovered, fmt.Sprintf("Testing reverse %q", in))
+		}
+	}
+}
+
+func TestEncryptSegmentBase32(t *testing.T) {
+	testEncryptSegment(t, "base32", []EncodingTestCase{
 		{"", ""},
 		{"1", "p0e52nreeaj0a5ea7s64m4j72s"},
 		{"12", "l42g6771hnv3an9cgc8cr2n1ng"},
@@ -118,26 +224,61 @@ func TestEncryptSegment(t *testing.T) {
 		{"12345678901234", "moq0uqdlqrblrc5pa5u5c7hq9g"},
 		{"123456789012345", "eeam3li4rnommi3a762h5n7meg"},
 		{"1234567890123456", "mijbj0frqf6ms7frcr6bd9h0env53jv96pjaaoirk7forcgpt70g"},
-	} {
+	}, true)
-		actual := c.encryptSegment(test.in)
-		assert.Equal(t, test.expected, actual, fmt.Sprintf("Testing %q", test.in))
-		recovered, err := c.decryptSegment(test.expected)
-		assert.NoError(t, err, fmt.Sprintf("Testing reverse %q", test.expected))
-		assert.Equal(t, test.in, recovered, fmt.Sprintf("Testing reverse %q", test.expected))
-		in := strings.ToUpper(test.expected)
-		recovered, err = c.decryptSegment(in)
-		assert.NoError(t, err, fmt.Sprintf("Testing reverse %q", in))
-		assert.Equal(t, test.in, recovered, fmt.Sprintf("Testing reverse %q", in))
-	}
 }
 
-func TestDecryptSegment(t *testing.T) {
+func TestEncryptSegmentBase64(t *testing.T) {
+	testEncryptSegment(t, "base64", []EncodingTestCase{
+		{"", ""},
+		{"1", "yBxRX25ypgUVyj8MSxJnFw"},
+		{"12", "qQUDHOGN_jVdLIMQzYrhvA"},
+		{"123", "1CxFf2Mti1xIPYlGruDh-A"},
+		{"1234", "RL-xOTmsxsG7kuTy2XJUxw"},
+		{"12345", "3FP_GHoeBJdq0yLgaED8IQ"},
+		{"123456", "Xc4T1Gqrs3OVYnrE6dpEWQ"},
+		{"1234567", "uZeEzssOnDWHEOzLqjwpog"},
+		{"12345678", "8noiTP5WkkbEuijsPhOpxQ"},
+		{"123456789", "GeNxgLA0wiaGAKU3U7qL4Q"},
+		{"1234567890", "x1DUhdmqoVWYVBLD3dha-A"},
+		{"12345678901", "iEyP_3BZR6vvv_2WM6NbZw"},
+		{"123456789012", "4OPGvS4SZdjvS568APUaFw"},
+		{"1234567890123", "Y8c5Wr8OhYYUo7fPwdojdg"},
+		{"12345678901234", "tjQPabXW112wuVF8Vh46TA"},
+		{"123456789012345", "c5Vh1kTd8WtIajmFEtz2dA"},
+		{"1234567890123456", "tKa5gfvTzW4d-2bMtqYgdf5Rz-k2ZqViW6HfjbIZ6cE"},
+	}, false)
+}
+
+func TestEncryptSegmentBase32768(t *testing.T) {
+	testEncryptSegment(t, "base32768", []EncodingTestCase{
+		{"", ""},
+		{"1", "詮㪗鐮僀伎作㻖㢧⪟"},
+		{"12", "竢朧䉱虃光塬䟛⣡蓟"},
+		{"123", "遶㞟鋅缕袡鲅ⵝ蝁ꌟ"},
+		{"1234", "䢟銮䵵狌㐜燳谒颴詟"},
+		{"12345", "钉Ꞇ㖃蚩憶狫朰杜㜿"},
+		{"123456", "啇ᚵⵕ憗䋫➫➓肤卟"},
+		{"1234567", "茫螓翁連劘樓㶔抉矟"},
+		{"12345678", "龝☳䘊辄岅較络㧩襟"},
+		{"123456789", "ⲱ苀㱆犂媐Ꮤ锇惫靟"},
+		{"1234567890", "計宁憕偵匢皫╛纺ꌟ"},
+		{"12345678901", "檆䨿鑫㪺藝ꡖ勇䦛婟"},
+		{"123456789012", "雑頏䰂䲝淚哚鹡魺⪟"},
+		{"1234567890123", "塃璶繁躸圅㔟䗃肃懟"},
+		{"12345678901234", "腺ᕚ崚鏕鏥讥鼌䑺䲿"},
+		{"123456789012345", "怪绕滻蕶肣但⠥荖惟"},
+		{"1234567890123456", "肳哀旚挶靏鏻㾭䱠慟㪳ꏆ賊兲铧敻塹魀ʟ"},
+	}, false)
+}
+
+func TestDecryptSegmentBase32(t *testing.T) {
 	// We've tested the forwards above, now concentrate on the errors
 	longName := make([]byte, 3328)
 	for i := range longName {
 		longName[i] = 'a'
 	}
-	c, _ := newCipher(NameEncryptionStandard, "", "", true)
+	enc, _ := NewNameEncoding("base32")
+	c, _ := newCipher(NameEncryptionStandard, "", "", true, enc)
 	for _, test := range []struct {
 		in          string
 		expectedErr error
@@ -145,118 +286,371 @@ func TestDecryptSegment(t *testing.T) {
 		{"64=", ErrorBadBase32Encoding},
 		{"!", base32.CorruptInputError(0)},
 		{string(longName), ErrorTooLongAfterDecode},
-		{encodeFileName([]byte("a")), ErrorNotAMultipleOfBlocksize},
+		{enc.EncodeToString([]byte("a")), ErrorNotAMultipleOfBlocksize},
-		{encodeFileName([]byte("123456789abcdef")), ErrorNotAMultipleOfBlocksize},
+		{enc.EncodeToString([]byte("123456789abcdef")), ErrorNotAMultipleOfBlocksize},
-		{encodeFileName([]byte("123456789abcdef0")), pkcs7.ErrorPaddingTooLong},
+		{enc.EncodeToString([]byte("123456789abcdef0")), pkcs7.ErrorPaddingTooLong},
 	} {
 		actual, actualErr := c.decryptSegment(test.in)
 		assert.Equal(t, test.expectedErr, actualErr, fmt.Sprintf("in=%q got actual=%q, err = %v %T", test.in, actual, actualErr, actualErr))
 	}
 }
 
-func TestEncryptFileName(t *testing.T) {
+func TestDecryptSegmentBase64(t *testing.T) {
+	// We've tested the forwards above, now concentrate on the errors
+	longName := make([]byte, 2816)
+	for i := range longName {
+		longName[i] = 'a'
+	}
+	enc, _ := NewNameEncoding("base64")
+	c, _ := newCipher(NameEncryptionStandard, "", "", true, enc)
+	for _, test := range []struct {
+		in          string
+		expectedErr error
+	}{
+		{"6H=", base64.CorruptInputError(2)},
+		{"!", base64.CorruptInputError(0)},
+		{string(longName), ErrorTooLongAfterDecode},
+		{enc.EncodeToString([]byte("a")), ErrorNotAMultipleOfBlocksize},
+		{enc.EncodeToString([]byte("123456789abcdef")), ErrorNotAMultipleOfBlocksize},
+		{enc.EncodeToString([]byte("123456789abcdef0")), pkcs7.ErrorPaddingTooLong},
+	} {
+		actual, actualErr := c.decryptSegment(test.in)
+		assert.Equal(t, test.expectedErr, actualErr, fmt.Sprintf("in=%q got actual=%q, err = %v %T", test.in, actual, actualErr, actualErr))
+	}
+}
+
+func TestDecryptSegmentBase32768(t *testing.T) {
+	// We've tested the forwards above, now concentrate on the errors
+	longName := strings.Repeat("怪", 1280)
+	enc, _ := NewNameEncoding("base32768")
+	c, _ := newCipher(NameEncryptionStandard, "", "", true, enc)
+	for _, test := range []struct {
+		in          string
+		expectedErr error
+	}{
+		{"怪=", base32768.CorruptInputError(1)},
+		{"!", base32768.CorruptInputError(0)},
+		{longName, ErrorTooLongAfterDecode},
+		{enc.EncodeToString([]byte("a")), ErrorNotAMultipleOfBlocksize},
+		{enc.EncodeToString([]byte("123456789abcdef")), ErrorNotAMultipleOfBlocksize},
+		{enc.EncodeToString([]byte("123456789abcdef0")), pkcs7.ErrorPaddingTooLong},
+	} {
+		actual, actualErr := c.decryptSegment(test.in)
+		assert.Equal(t, test.expectedErr, actualErr, fmt.Sprintf("in=%q got actual=%q, err = %v %T", test.in, actual, actualErr, actualErr))
+	}
+}
+
+func testStandardEncryptFileName(t *testing.T, encoding string, testCasesEncryptDir []EncodingTestCase, testCasesNoEncryptDir []EncodingTestCase) {
 	// First standard mode
-	c, _ := newCipher(NameEncryptionStandard, "", "", true)
+	enc, _ := NewNameEncoding(encoding)
-	assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s", c.EncryptFileName("1"))
+	c, _ := newCipher(NameEncryptionStandard, "", "", true, enc)
-	assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng", c.EncryptFileName("1/12"))
+	for _, test := range testCasesEncryptDir {
-	assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng/qgm4avr35m5loi1th53ato71v0", c.EncryptFileName("1/12/123"))
+		assert.Equal(t, test.expected, c.EncryptFileName(test.in))
-	assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s-v2001-02-03-040506-123", c.EncryptFileName("1-v2001-02-03-040506-123"))
+	}
-	assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng-v2001-02-03-040506-123", c.EncryptFileName("1/12-v2001-02-03-040506-123"))
 	// Standard mode with directory name encryption off
-	c, _ = newCipher(NameEncryptionStandard, "", "", false)
+	c, _ = newCipher(NameEncryptionStandard, "", "", false, enc)
-	assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s", c.EncryptFileName("1"))
+	for _, test := range testCasesNoEncryptDir {
-	assert.Equal(t, "1/l42g6771hnv3an9cgc8cr2n1ng", c.EncryptFileName("1/12"))
+		assert.Equal(t, test.expected, c.EncryptFileName(test.in))
-	assert.Equal(t, "1/12/qgm4avr35m5loi1th53ato71v0", c.EncryptFileName("1/12/123"))
+	}
-	assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s-v2001-02-03-040506-123", c.EncryptFileName("1-v2001-02-03-040506-123"))
+}
-	assert.Equal(t, "1/l42g6771hnv3an9cgc8cr2n1ng-v2001-02-03-040506-123", c.EncryptFileName("1/12-v2001-02-03-040506-123"))
+
-	// Now off mode
+func TestStandardEncryptFileNameBase32(t *testing.T) {
-	c, _ = newCipher(NameEncryptionOff, "", "", true)
+	testStandardEncryptFileName(t, "base32", []EncodingTestCase{
+		{"1", "p0e52nreeaj0a5ea7s64m4j72s"},
+		{"1/12", "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng"},
+		{"1/12/123", "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng/qgm4avr35m5loi1th53ato71v0"},
+		{"1-v2001-02-03-040506-123", "p0e52nreeaj0a5ea7s64m4j72s-v2001-02-03-040506-123"},
+		{"1/12-v2001-02-03-040506-123", "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng-v2001-02-03-040506-123"},
+	}, []EncodingTestCase{
+		{"1", "p0e52nreeaj0a5ea7s64m4j72s"},
+		{"1/12", "1/l42g6771hnv3an9cgc8cr2n1ng"},
+		{"1/12/123", "1/12/qgm4avr35m5loi1th53ato71v0"},
+		{"1-v2001-02-03-040506-123", "p0e52nreeaj0a5ea7s64m4j72s-v2001-02-03-040506-123"},
+		{"1/12-v2001-02-03-040506-123", "1/l42g6771hnv3an9cgc8cr2n1ng-v2001-02-03-040506-123"},
+	})
+}
+
+func TestStandardEncryptFileNameBase64(t *testing.T) {
+	testStandardEncryptFileName(t, "base64", []EncodingTestCase{
+		{"1", "yBxRX25ypgUVyj8MSxJnFw"},
+		{"1/12", "yBxRX25ypgUVyj8MSxJnFw/qQUDHOGN_jVdLIMQzYrhvA"},
+		{"1/12/123", "yBxRX25ypgUVyj8MSxJnFw/qQUDHOGN_jVdLIMQzYrhvA/1CxFf2Mti1xIPYlGruDh-A"},
+		{"1-v2001-02-03-040506-123", "yBxRX25ypgUVyj8MSxJnFw-v2001-02-03-040506-123"},
+		{"1/12-v2001-02-03-040506-123", "yBxRX25ypgUVyj8MSxJnFw/qQUDHOGN_jVdLIMQzYrhvA-v2001-02-03-040506-123"},
+	}, []EncodingTestCase{
+		{"1", "yBxRX25ypgUVyj8MSxJnFw"},
+		{"1/12", "1/qQUDHOGN_jVdLIMQzYrhvA"},
+		{"1/12/123", "1/12/1CxFf2Mti1xIPYlGruDh-A"},
+		{"1-v2001-02-03-040506-123", "yBxRX25ypgUVyj8MSxJnFw-v2001-02-03-040506-123"},
+		{"1/12-v2001-02-03-040506-123", "1/qQUDHOGN_jVdLIMQzYrhvA-v2001-02-03-040506-123"},
+	})
+}
+
+func TestStandardEncryptFileNameBase32768(t *testing.T) {
+	testStandardEncryptFileName(t, "base32768", []EncodingTestCase{
+		{"1", "詮㪗鐮僀伎作㻖㢧⪟"},
+		{"1/12", "詮㪗鐮僀伎作㻖㢧⪟/竢朧䉱虃光塬䟛⣡蓟"},
+		{"1/12/123", "詮㪗鐮僀伎作㻖㢧⪟/竢朧䉱虃光塬䟛⣡蓟/遶㞟鋅缕袡鲅ⵝ蝁ꌟ"},
+		{"1-v2001-02-03-040506-123", "詮㪗鐮僀伎作㻖㢧⪟-v2001-02-03-040506-123"},
+		{"1/12-v2001-02-03-040506-123", "詮㪗鐮僀伎作㻖㢧⪟/竢朧䉱虃光塬䟛⣡蓟-v2001-02-03-040506-123"},
+	}, []EncodingTestCase{
+		{"1", "詮㪗鐮僀伎作㻖㢧⪟"},
+		{"1/12", "1/竢朧䉱虃光塬䟛⣡蓟"},
+		{"1/12/123", "1/12/遶㞟鋅缕袡鲅ⵝ蝁ꌟ"},
+		{"1-v2001-02-03-040506-123", "詮㪗鐮僀伎作㻖㢧⪟-v2001-02-03-040506-123"},
+		{"1/12-v2001-02-03-040506-123", "1/竢朧䉱虃光塬䟛⣡蓟-v2001-02-03-040506-123"},
+	})
+}
+
+func TestNonStandardEncryptFileName(t *testing.T) {
+	// Off mode
+	c, _ := newCipher(NameEncryptionOff, "", "", true, nil)
 	assert.Equal(t, "1/12/123.bin", c.EncryptFileName("1/12/123"))
 	// Obfuscation mode
-	c, _ = newCipher(NameEncryptionObfuscated, "", "", true)
+	c, _ = newCipher(NameEncryptionObfuscated, "", "", true, nil)
 	assert.Equal(t, "49.6/99.23/150.890/53.!!lipps", c.EncryptFileName("1/12/123/!hello"))
 	assert.Equal(t, "49.6/99.23/150.890/53-v2001-02-03-040506-123.!!lipps", c.EncryptFileName("1/12/123/!hello-v2001-02-03-040506-123"))
 	assert.Equal(t, "49.6/99.23/150.890/162.uryyB-v2001-02-03-040506-123.GKG", c.EncryptFileName("1/12/123/hello-v2001-02-03-040506-123.txt"))
 	assert.Equal(t, "161.\u00e4", c.EncryptFileName("\u00a1"))
 	assert.Equal(t, "160.\u03c2", c.EncryptFileName("\u03a0"))
 	// Obfuscation mode with directory name encryption off
-	c, _ = newCipher(NameEncryptionObfuscated, "", "", false)
+	c, _ = newCipher(NameEncryptionObfuscated, "", "", false, nil)
 	assert.Equal(t, "1/12/123/53.!!lipps", c.EncryptFileName("1/12/123/!hello"))
 	assert.Equal(t, "1/12/123/53-v2001-02-03-040506-123.!!lipps", c.EncryptFileName("1/12/123/!hello-v2001-02-03-040506-123"))
 	assert.Equal(t, "161.\u00e4", c.EncryptFileName("\u00a1"))
 	assert.Equal(t, "160.\u03c2", c.EncryptFileName("\u03a0"))
 }
 
-func TestDecryptFileName(t *testing.T) {
+func testStandardDecryptFileName(t *testing.T, encoding string, testCases []EncodingTestCase, caseInsensitive bool) {
-	for _, test := range []struct {
+	enc, _ := NewNameEncoding(encoding)
-		mode           NameEncryptionMode
+	for _, test := range testCases {
-		dirNameEncrypt bool
+		// Test when dirNameEncrypt=true
-		in             string
+		c, _ := newCipher(NameEncryptionStandard, "", "", true, enc)
-		expected       string
-		expectedErr    error
-	}{
-		{NameEncryptionStandard, true, "p0e52nreeaj0a5ea7s64m4j72s", "1", nil},
-		{NameEncryptionStandard, true, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng", "1/12", nil},
-		{NameEncryptionStandard, true, "p0e52nreeAJ0A5EA7S64M4J72S/L42G6771HNv3an9cgc8cr2n1ng", "1/12", nil},
-		{NameEncryptionStandard, true, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng/qgm4avr35m5loi1th53ato71v0", "1/12/123", nil},
-		{NameEncryptionStandard, true, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1/qgm4avr35m5loi1th53ato71v0", "", ErrorNotAMultipleOfBlocksize},
-		{NameEncryptionStandard, false, "1/12/qgm4avr35m5loi1th53ato71v0", "1/12/123", nil},
-		{NameEncryptionStandard, true, "p0e52nreeaj0a5ea7s64m4j72s-v2001-02-03-040506-123", "1-v2001-02-03-040506-123", nil},
-		{NameEncryptionOff, true, "1/12/123.bin", "1/12/123", nil},
-		{NameEncryptionOff, true, "1/12/123.bix", "", ErrorNotAnEncryptedFile},
-		{NameEncryptionOff, true, ".bin", "", ErrorNotAnEncryptedFile},
-		{NameEncryptionOff, true, "1/12/123-v2001-02-03-040506-123.bin", "1/12/123-v2001-02-03-040506-123", nil},
-		{NameEncryptionOff, true, "1/12/123-v1970-01-01-010101-123-v2001-02-03-040506-123.bin", "1/12/123-v1970-01-01-010101-123-v2001-02-03-040506-123", nil},
-		{NameEncryptionOff, true, "1/12/123-v1970-01-01-010101-123-v2001-02-03-040506-123.txt.bin", "1/12/123-v1970-01-01-010101-123-v2001-02-03-040506-123.txt", nil},
-		{NameEncryptionObfuscated, true, "!.hello", "hello", nil},
-		{NameEncryptionObfuscated, true, "hello", "", ErrorNotAnEncryptedFile},
-		{NameEncryptionObfuscated, true, "161.\u00e4", "\u00a1", nil},
-		{NameEncryptionObfuscated, true, "160.\u03c2", "\u03a0", nil},
-		{NameEncryptionObfuscated, false, "1/12/123/53.!!lipps", "1/12/123/!hello", nil},
-		{NameEncryptionObfuscated, false, "1/12/123/53-v2001-02-03-040506-123.!!lipps", "1/12/123/!hello-v2001-02-03-040506-123", nil},
-	} {
-		c, _ := newCipher(test.mode, "", "", test.dirNameEncrypt)
 		actual, actualErr := c.DecryptFileName(test.in)
-		what := fmt.Sprintf("Testing %q (mode=%v)", test.in, test.mode)
+		assert.NoError(t, actualErr)
-		assert.Equal(t, test.expected, actual, what)
+		assert.Equal(t, test.expected, actual)
-		assert.Equal(t, test.expectedErr, actualErr, what)
+		if caseInsensitive {
+			c, _ := newCipher(NameEncryptionStandard, "", "", true, enc)
+			actual, actualErr := c.DecryptFileName(strings.ToUpper(test.in))
+			assert.NoError(t, actualErr)
+			assert.Equal(t, test.expected, actual)
+		}
+		// Add a character should raise ErrorNotAMultipleOfBlocksize
+		actual, actualErr = c.DecryptFileName(enc.EncodeToString([]byte("1")) + test.in)
+		assert.Equal(t, ErrorNotAMultipleOfBlocksize, actualErr)
+		assert.Equal(t, "", actual)
+		// Test when dirNameEncrypt=false
+		noDirEncryptIn := test.in
+		if strings.LastIndex(test.expected, "/") != -1 {
+			noDirEncryptIn = test.expected[:strings.LastIndex(test.expected, "/")] + test.in[strings.LastIndex(test.in, "/"):]
+		}
+		c, _ = newCipher(NameEncryptionStandard, "", "", false, enc)
+		actual, actualErr = c.DecryptFileName(noDirEncryptIn)
+		assert.NoError(t, actualErr)
+		assert.Equal(t, test.expected, actual)
+	}
+}
+
+func TestStandardDecryptFileNameBase32(t *testing.T) {
+	testStandardDecryptFileName(t, "base32", []EncodingTestCase{
+		{"p0e52nreeaj0a5ea7s64m4j72s", "1"},
+		{"p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng", "1/12"},
+		{"p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng/qgm4avr35m5loi1th53ato71v0", "1/12/123"},
+	}, true)
+}
+
+func TestStandardDecryptFileNameBase64(t *testing.T) {
+	testStandardDecryptFileName(t, "base64", []EncodingTestCase{
+		{"yBxRX25ypgUVyj8MSxJnFw", "1"},
+		{"yBxRX25ypgUVyj8MSxJnFw/qQUDHOGN_jVdLIMQzYrhvA", "1/12"},
+		{"yBxRX25ypgUVyj8MSxJnFw/qQUDHOGN_jVdLIMQzYrhvA/1CxFf2Mti1xIPYlGruDh-A", "1/12/123"},
+	}, false)
+}
+
+func TestStandardDecryptFileNameBase32768(t *testing.T) {
+	testStandardDecryptFileName(t, "base32768", []EncodingTestCase{
+		{"詮㪗鐮僀伎作㻖㢧⪟", "1"},
+		{"詮㪗鐮僀伎作㻖㢧⪟/竢朧䉱虃光塬䟛⣡蓟", "1/12"},
+		{"詮㪗鐮僀伎作㻖㢧⪟/竢朧䉱虃光塬䟛⣡蓟/遶㞟鋅缕袡鲅ⵝ蝁ꌟ", "1/12/123"},
+	}, false)
+}
+
+func TestNonStandardDecryptFileName(t *testing.T) {
+	for _, encoding := range []string{"base32", "base64", "base32768"} {
+		enc, _ := NewNameEncoding(encoding)
+		for _, test := range []struct {
+			mode           NameEncryptionMode
+			dirNameEncrypt bool
+			in             string
+			expected       string
+			expectedErr    error
+		}{
+			{NameEncryptionOff, true, "1/12/123.bin", "1/12/123", nil},
+			{NameEncryptionOff, true, "1/12/123.bix", "", ErrorNotAnEncryptedFile},
+			{NameEncryptionOff, true, ".bin", "", ErrorNotAnEncryptedFile},
+			{NameEncryptionOff, true, "1/12/123-v2001-02-03-040506-123.bin", "1/12/123-v2001-02-03-040506-123", nil},
+			{NameEncryptionOff, true, "1/12/123-v1970-01-01-010101-123-v2001-02-03-040506-123.bin", "1/12/123-v1970-01-01-010101-123-v2001-02-03-040506-123", nil},
+			{NameEncryptionOff, true, "1/12/123-v1970-01-01-010101-123-v2001-02-03-040506-123.txt.bin", "1/12/123-v1970-01-01-010101-123-v2001-02-03-040506-123.txt", nil},
+			{NameEncryptionObfuscated, true, "!.hello", "hello", nil},
+			{NameEncryptionObfuscated, true, "hello", "", ErrorNotAnEncryptedFile},
+			{NameEncryptionObfuscated, true, "161.\u00e4", "\u00a1", nil},
+			{NameEncryptionObfuscated, true, "160.\u03c2", "\u03a0", nil},
+			{NameEncryptionObfuscated, false, "1/12/123/53.!!lipps", "1/12/123/!hello", nil},
+			{NameEncryptionObfuscated, false, "1/12/123/53-v2001-02-03-040506-123.!!lipps", "1/12/123/!hello-v2001-02-03-040506-123", nil},
+		} {
+			c, _ := newCipher(test.mode, "", "", test.dirNameEncrypt, enc)
+			actual, actualErr := c.DecryptFileName(test.in)
+			what := fmt.Sprintf("Testing %q (mode=%v)", test.in, test.mode)
+			assert.Equal(t, test.expected, actual, what)
+			assert.Equal(t, test.expectedErr, actualErr, what)
+		}
 	}
 }
 
 func TestEncDecMatches(t *testing.T) {
-	for _, test := range []struct {
+	for _, encoding := range []string{"base32", "base64", "base32768"} {
-		mode NameEncryptionMode
+		enc, _ := NewNameEncoding(encoding)
-		in   string
+		for _, test := range []struct {
-	}{
+			mode NameEncryptionMode
-		{NameEncryptionStandard, "1/2/3/4"},
+			in   string
-		{NameEncryptionOff, "1/2/3/4"},
+		}{
-		{NameEncryptionObfuscated, "1/2/3/4/!hello\u03a0"},
+			{NameEncryptionStandard, "1/2/3/4"},
-		{NameEncryptionObfuscated, "Avatar The Last Airbender"},
+			{NameEncryptionOff, "1/2/3/4"},
-	} {
+			{NameEncryptionObfuscated, "1/2/3/4/!hello\u03a0"},
-		c, _ := newCipher(test.mode, "", "", true)
+			{NameEncryptionObfuscated, "Avatar The Last Airbender"},
-		out, err := c.DecryptFileName(c.EncryptFileName(test.in))
+		} {
-		what := fmt.Sprintf("Testing %q (mode=%v)", test.in, test.mode)
+			c, _ := newCipher(test.mode, "", "", true, enc)
-		assert.Equal(t, out, test.in, what)
+			out, err := c.DecryptFileName(c.EncryptFileName(test.in))
-		assert.Equal(t, err, nil, what)
+			what := fmt.Sprintf("Testing %q (mode=%v)", test.in, test.mode)
+			assert.Equal(t, out, test.in, what)
+			assert.Equal(t, err, nil, what)
+		}
 	}
 }
 
-func TestEncryptDirName(t *testing.T) {
+func testStandardEncryptDirName(t *testing.T, encoding string, testCases []EncodingTestCase) {
+	enc, _ := NewNameEncoding(encoding)
+	c, _ := newCipher(NameEncryptionStandard, "", "", true, enc)
 	// First standard mode
-	c, _ := newCipher(NameEncryptionStandard, "", "", true)
+	for _, test := range testCases {
-	assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s", c.EncryptDirName("1"))
+		assert.Equal(t, test.expected, c.EncryptDirName(test.in))
-	assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng", c.EncryptDirName("1/12"))
+	}
-	assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng/qgm4avr35m5loi1th53ato71v0", c.EncryptDirName("1/12/123"))
-	// Standard mode with dir name encryption off
-	c, _ = newCipher(NameEncryptionStandard, "", "", false)
-	assert.Equal(t, "1/12", c.EncryptDirName("1/12"))
-	assert.Equal(t, "1/12/123", c.EncryptDirName("1/12/123"))
-	// Now off mode
-	c, _ = newCipher(NameEncryptionOff, "", "", true)
-	assert.Equal(t, "1/12/123", c.EncryptDirName("1/12/123"))
 }
 
-func TestDecryptDirName(t *testing.T) {
+func TestStandardEncryptDirNameBase32(t *testing.T) {
+	testStandardEncryptDirName(t, "base32", []EncodingTestCase{
+		{"1", "p0e52nreeaj0a5ea7s64m4j72s"},
+		{"1/12", "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng"},
+		{"1/12/123", "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng/qgm4avr35m5loi1th53ato71v0"},
+	})
+}
+
+func TestStandardEncryptDirNameBase64(t *testing.T) {
+	testStandardEncryptDirName(t, "base64", []EncodingTestCase{
+		{"1", "yBxRX25ypgUVyj8MSxJnFw"},
+		{"1/12", "yBxRX25ypgUVyj8MSxJnFw/qQUDHOGN_jVdLIMQzYrhvA"},
+		{"1/12/123", "yBxRX25ypgUVyj8MSxJnFw/qQUDHOGN_jVdLIMQzYrhvA/1CxFf2Mti1xIPYlGruDh-A"},
+	})
+}
+
+func TestStandardEncryptDirNameBase32768(t *testing.T) {
+	testStandardEncryptDirName(t, "base32768", []EncodingTestCase{
+		{"1", "詮㪗鐮僀伎作㻖㢧⪟"},
+		{"1/12", "詮㪗鐮僀伎作㻖㢧⪟/竢朧䉱虃光塬䟛⣡蓟"},
+		{"1/12/123", "詮㪗鐮僀伎作㻖㢧⪟/竢朧䉱虃光塬䟛⣡蓟/遶㞟鋅缕袡鲅ⵝ蝁ꌟ"},
+	})
+}
+
+func TestNonStandardEncryptDirName(t *testing.T) {
+	for _, encoding := range []string{"base32", "base64", "base32768"} {
+		enc, _ := NewNameEncoding(encoding)
+		c, _ := newCipher(NameEncryptionStandard, "", "", false, enc)
+		assert.Equal(t, "1/12", c.EncryptDirName("1/12"))
+		assert.Equal(t, "1/12/123", c.EncryptDirName("1/12/123"))
+		// Now off mode
+		c, _ = newCipher(NameEncryptionOff, "", "", true, enc)
+		assert.Equal(t, "1/12/123", c.EncryptDirName("1/12/123"))
+	}
+}
+
+func testStandardDecryptDirName(t *testing.T, encoding string, testCases []EncodingTestCase, caseInsensitive bool) {
+	enc, _ := NewNameEncoding(encoding)
+	for _, test := range testCases {
+		// Test dirNameEncrypt=true
+		c, _ := newCipher(NameEncryptionStandard, "", "", true, enc)
+		actual, actualErr := c.DecryptDirName(test.in)
+		assert.Equal(t, test.expected, actual)
+		assert.NoError(t, actualErr)
+		if caseInsensitive {
+			actual, actualErr := c.DecryptDirName(strings.ToUpper(test.in))
+			assert.Equal(t, actual, test.expected)
+			assert.NoError(t, actualErr)
+		}
+		actual, actualErr = c.DecryptDirName(enc.EncodeToString([]byte("1")) + test.in)
+		assert.Equal(t, "", actual)
+		assert.Equal(t, ErrorNotAMultipleOfBlocksize, actualErr)
+		// Test dirNameEncrypt=false
+		c, _ = newCipher(NameEncryptionStandard, "", "", false, enc)
+		actual, actualErr = c.DecryptDirName(test.in)
+		assert.Equal(t, test.in, actual)
+		assert.NoError(t, actualErr)
+		actual, actualErr = c.DecryptDirName(test.expected)
+		assert.Equal(t, test.expected, actual)
+		assert.NoError(t, actualErr)
+		// Test dirNameEncrypt=false
+	}
+}
+
+/*
+enc, _ := NewNameEncoding(encoding)
+for _, test := range []struct {
+	mode           NameEncryptionMode
+	dirNameEncrypt bool
+	in             string
+	expected       string
+	expectedErr    error
+}{
+	{NameEncryptionStandard, true, "p0e52nreeaj0a5ea7s64m4j72s", "1", nil},
+	{NameEncryptionStandard, true, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng", "1/12", nil},
+	{NameEncryptionStandard, true, "p0e52nreeAJ0A5EA7S64M4J72S/L42G6771HNv3an9cgc8cr2n1ng", "1/12", nil},
+	{NameEncryptionStandard, true, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng/qgm4avr35m5loi1th53ato71v0", "1/12/123", nil},
+	{NameEncryptionStandard, true, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1/qgm4avr35m5loi1th53ato71v0", "", ErrorNotAMultipleOfBlocksize},
+	{NameEncryptionStandard, false, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng", "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng", nil},
+	{NameEncryptionStandard, false, "1/12/123", "1/12/123", nil},
+} {
+	c, _ := newCipher(test.mode, "", "", test.dirNameEncrypt, enc)
+	actual, actualErr := c.DecryptDirName(test.in)
+	what := fmt.Sprintf("Testing %q (mode=%v)", test.in, test.mode)
+	assert.Equal(t, test.expected, actual, what)
+	assert.Equal(t, test.expectedErr, actualErr, what)
+}
+*/
+
+func TestStandardDecryptDirNameBase32(t *testing.T) {
+	testStandardDecryptDirName(t, "base32", []EncodingTestCase{
+		{"p0e52nreeaj0a5ea7s64m4j72s", "1"},
+		{"p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng", "1/12"},
+		{"p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng/qgm4avr35m5loi1th53ato71v0", "1/12/123"},
+	}, true)
+}
+
+func TestStandardDecryptDirNameBase64(t *testing.T) {
+	testStandardDecryptDirName(t, "base64", []EncodingTestCase{
+		{"yBxRX25ypgUVyj8MSxJnFw", "1"},
+		{"yBxRX25ypgUVyj8MSxJnFw/qQUDHOGN_jVdLIMQzYrhvA", "1/12"},
+		{"yBxRX25ypgUVyj8MSxJnFw/qQUDHOGN_jVdLIMQzYrhvA/1CxFf2Mti1xIPYlGruDh-A", "1/12/123"},
+	}, false)
+}
+
+func TestStandardDecryptDirNameBase32768(t *testing.T) {
+	testStandardDecryptDirName(t, "base32768", []EncodingTestCase{
+		{"詮㪗鐮僀伎作㻖㢧⪟", "1"},
+		{"詮㪗鐮僀伎作㻖㢧⪟/竢朧䉱虃光塬䟛⣡蓟", "1/12"},
+		{"詮㪗鐮僀伎作㻖㢧⪟/竢朧䉱虃光塬䟛⣡蓟/遶㞟鋅缕袡鲅ⵝ蝁ꌟ", "1/12/123"},
+	}, false)
+}
+
+func TestNonStandardDecryptDirName(t *testing.T) {
 	for _, test := range []struct {
 		mode           NameEncryptionMode
 		dirNameEncrypt bool
@@ -264,18 +658,11 @@ func TestDecryptDirName(t *testing.T) {
 		expected       string
 		expectedErr    error
 	}{
-		{NameEncryptionStandard, true, "p0e52nreeaj0a5ea7s64m4j72s", "1", nil},
-		{NameEncryptionStandard, true, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng", "1/12", nil},
-		{NameEncryptionStandard, true, "p0e52nreeAJ0A5EA7S64M4J72S/L42G6771HNv3an9cgc8cr2n1ng", "1/12", nil},
-		{NameEncryptionStandard, true, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng/qgm4avr35m5loi1th53ato71v0", "1/12/123", nil},
-		{NameEncryptionStandard, true, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1/qgm4avr35m5loi1th53ato71v0", "", ErrorNotAMultipleOfBlocksize},
-		{NameEncryptionStandard, false, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng", "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng", nil},
-		{NameEncryptionStandard, false, "1/12/123", "1/12/123", nil},
 		{NameEncryptionOff, true, "1/12/123.bin", "1/12/123.bin", nil},
 		{NameEncryptionOff, true, "1/12/123", "1/12/123", nil},
 		{NameEncryptionOff, true, ".bin", ".bin", nil},
 	} {
-		c, _ := newCipher(test.mode, "", "", test.dirNameEncrypt)
+		c, _ := newCipher(test.mode, "", "", test.dirNameEncrypt, nil)
 		actual, actualErr := c.DecryptDirName(test.in)
 		what := fmt.Sprintf("Testing %q (mode=%v)", test.in, test.mode)
 		assert.Equal(t, test.expected, actual, what)
@@ -284,7 +671,7 @@ func TestDecryptDirName(t *testing.T) {
 }
 
 func TestEncryptedSize(t *testing.T) {
-	c, _ := newCipher(NameEncryptionStandard, "", "", true)
+	c, _ := newCipher(NameEncryptionStandard, "", "", true, nil)
 	for _, test := range []struct {
 		in       int64
 		expected int64
@@ -308,7 +695,7 @@ func TestEncryptedSize(t *testing.T) {
 
 func TestDecryptedSize(t *testing.T) {
 	// Test the errors since we tested the reverse above
-	c, _ := newCipher(NameEncryptionStandard, "", "", true)
+	c, _ := newCipher(NameEncryptionStandard, "", "", true, nil)
 	for _, test := range []struct {
 		in          int64
 		expectedErr error
@@ -679,7 +1066,7 @@ func (z *zeroes) Read(p []byte) (n int, err error) {
 
 // Test encrypt decrypt with different buffer sizes
 func testEncryptDecrypt(t *testing.T, bufSize int, copySize int64) {
-	c, err := newCipher(NameEncryptionStandard, "", "", true)
+	c, err := newCipher(NameEncryptionStandard, "", "", true, nil)
 	assert.NoError(t, err)
 	c.cryptoRand = &zeroes{} // zero out the nonce
 	buf := make([]byte, bufSize)
@@ -749,7 +1136,7 @@ func TestEncryptData(t *testing.T) {
 		{[]byte{1}, file1},
 		{[]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}, file16},
 	} {
-		c, err := newCipher(NameEncryptionStandard, "", "", true)
+		c, err := newCipher(NameEncryptionStandard, "", "", true, nil)
 		assert.NoError(t, err)
 		c.cryptoRand = newRandomSource(1e8) // nodge the crypto rand generator
 
@@ -772,7 +1159,7 @@ func TestEncryptData(t *testing.T) {
 }
 
 func TestNewEncrypter(t *testing.T) {
-	c, err := newCipher(NameEncryptionStandard, "", "", true)
+	c, err := newCipher(NameEncryptionStandard, "", "", true, nil)
 	assert.NoError(t, err)
 	c.cryptoRand = newRandomSource(1e8) // nodge the crypto rand generator
 
@@ -788,13 +1175,12 @@ func TestNewEncrypter(t *testing.T) {
 	fh, err = c.newEncrypter(z, nil)
 	assert.Nil(t, fh)
 	assert.Error(t, err, "short read of nonce")
-
 }
 
 // Test the stream returning 0, io.ErrUnexpectedEOF - this used to
 // cause a fatal loop
 func TestNewEncrypterErrUnexpectedEOF(t *testing.T) {
-	c, err := newCipher(NameEncryptionStandard, "", "", true)
+	c, err := newCipher(NameEncryptionStandard, "", "", true, nil)
 	assert.NoError(t, err)
 
 	in := &readers.ErrorReader{Err: io.ErrUnexpectedEOF}
@@ -823,7 +1209,7 @@ func (c *closeDetector) Close() error {
 }
 
 func TestNewDecrypter(t *testing.T) {
-	c, err := newCipher(NameEncryptionStandard, "", "", true)
+	c, err := newCipher(NameEncryptionStandard, "", "", true, nil)
 	assert.NoError(t, err)
 	c.cryptoRand = newRandomSource(1e8) // nodge the crypto rand generator
 
@@ -866,7 +1252,7 @@ func TestNewDecrypter(t *testing.T) {
 
 // Test the stream returning 0, io.ErrUnexpectedEOF
 func TestNewDecrypterErrUnexpectedEOF(t *testing.T) {
-	c, err := newCipher(NameEncryptionStandard, "", "", true)
+	c, err := newCipher(NameEncryptionStandard, "", "", true, nil)
 	assert.NoError(t, err)
 
 	in2 := &readers.ErrorReader{Err: io.ErrUnexpectedEOF}
@@ -882,7 +1268,7 @@ func TestNewDecrypterErrUnexpectedEOF(t *testing.T) {
 }
 
 func TestNewDecrypterSeekLimit(t *testing.T) {
-	c, err := newCipher(NameEncryptionStandard, "", "", true)
+	c, err := newCipher(NameEncryptionStandard, "", "", true, nil)
 	assert.NoError(t, err)
 	c.cryptoRand = &zeroes{} // nodge the crypto rand generator
 
@@ -1088,7 +1474,7 @@ func TestDecrypterCalculateUnderlying(t *testing.T) {
 }
 
 func TestDecrypterRead(t *testing.T) {
-	c, err := newCipher(NameEncryptionStandard, "", "", true)
+	c, err := newCipher(NameEncryptionStandard, "", "", true, nil)
 	assert.NoError(t, err)
 
 	// Test truncating the file at each possible point
@@ -1152,7 +1538,7 @@ func TestDecrypterRead(t *testing.T) {
 }
 
 func TestDecrypterClose(t *testing.T) {
-	c, err := newCipher(NameEncryptionStandard, "", "", true)
+	c, err := newCipher(NameEncryptionStandard, "", "", true, nil)
 	assert.NoError(t, err)
 
 	cd := newCloseDetector(bytes.NewBuffer(file16))
@@ -1190,7 +1576,7 @@ func TestDecrypterClose(t *testing.T) {
 }
 
 func TestPutGetBlock(t *testing.T) {
-	c, err := newCipher(NameEncryptionStandard, "", "", true)
+	c, err := newCipher(NameEncryptionStandard, "", "", true, nil)
 	assert.NoError(t, err)
 
 	block := c.getBlock()
@@ -1201,7 +1587,7 @@ func TestPutGetBlock(t *testing.T) {
 }
 
 func TestKey(t *testing.T) {
-	c, err := newCipher(NameEncryptionStandard, "", "", true)
+	c, err := newCipher(NameEncryptionStandard, "", "", true, nil)
 	assert.NoError(t, err)
 
 	// Check zero keys OK

backend/crypt/crypt.go

@@ -116,6 +116,29 @@ names, or for debugging purposes.`,
 					Help:  "Encrypt file data.",
 				},
 			},
+		}, {
+			Name: "filename_encoding",
+			Help: `How to encode the encrypted filename to text string.
+
+This option could help with shortening the encrypted filename. The 
+suitable option would depend on the way your remote count the filename
+length and if it's case sensitve.`,
+			Default: "base32",
+			Examples: []fs.OptionExample{
+				{
+					Value: "base32",
+					Help:  "Encode using base32. Suitable for all remote.",
+				},
+				{
+					Value: "base64",
+					Help:  "Encode using base64. Suitable for case sensitive remote.",
+				},
+				{
+					Value: "base32768",
+					Help:  "Encode using base32768. Suitable if your remote counts UTF-16 or\nUnicode codepoint instead of UTF-8 byte length. (Eg. Onedrive)",
+				},
+			},
+			Advanced: true,
 		}},
 	})
 }
@@ -140,7 +163,11 @@ func newCipherForConfig(opt *Options) (*Cipher, error) {
 			return nil, fmt.Errorf("failed to decrypt password2: %w", err)
 		}
 	}
-	cipher, err := newCipher(mode, password, salt, opt.DirectoryNameEncryption)
+	enc, err := NewNameEncoding(opt.FilenameEncoding)
+	if err != nil {
+		return nil, err
+	}
+	cipher, err := newCipher(mode, password, salt, opt.DirectoryNameEncryption, enc)
 	if err != nil {
 		return nil, fmt.Errorf("failed to make cipher: %w", err)
 	}
@@ -229,6 +256,7 @@ type Options struct {
 	Password2               string `config:"password2"`
 	ServerSideAcrossConfigs bool   `config:"server_side_across_configs"`
 	ShowMapping             bool   `config:"show_mapping"`
+	FilenameEncoding        string `config:"filename_encoding"`
 }
 
 // Fs represents a wrapped fs.Fs

backend/crypt/crypt_test.go

@@ -29,7 +29,7 @@ func TestIntegration(t *testing.T) {
 }
 
 // TestStandard runs integration tests against the remote
-func TestStandard(t *testing.T) {
+func TestStandardBase32(t *testing.T) {
 	if *fstest.RemoteName != "" {
 		t.Skip("Skipping as -remote set")
 	}
@@ -49,6 +49,48 @@ func TestStandard(t *testing.T) {
 	})
 }
 
+func TestStandardBase64(t *testing.T) {
+	if *fstest.RemoteName != "" {
+		t.Skip("Skipping as -remote set")
+	}
+	tempdir := filepath.Join(os.TempDir(), "rclone-crypt-test-standard")
+	name := "TestCrypt"
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: name + ":",
+		NilObject:  (*crypt.Object)(nil),
+		ExtraConfig: []fstests.ExtraConfigItem{
+			{Name: name, Key: "type", Value: "crypt"},
+			{Name: name, Key: "remote", Value: tempdir},
+			{Name: name, Key: "password", Value: obscure.MustObscure("potato")},
+			{Name: name, Key: "filename_encryption", Value: "standard"},
+			{Name: name, Key: "filename_encoding", Value: "base64"},
+		},
+		UnimplementableFsMethods:     []string{"OpenWriterAt"},
+		UnimplementableObjectMethods: []string{"MimeType"},
+	})
+}
+
+func TestStandardBase32768(t *testing.T) {
+	if *fstest.RemoteName != "" {
+		t.Skip("Skipping as -remote set")
+	}
+	tempdir := filepath.Join(os.TempDir(), "rclone-crypt-test-standard")
+	name := "TestCrypt"
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: name + ":",
+		NilObject:  (*crypt.Object)(nil),
+		ExtraConfig: []fstests.ExtraConfigItem{
+			{Name: name, Key: "type", Value: "crypt"},
+			{Name: name, Key: "remote", Value: tempdir},
+			{Name: name, Key: "password", Value: obscure.MustObscure("potato")},
+			{Name: name, Key: "filename_encryption", Value: "standard"},
+			{Name: name, Key: "filename_encoding", Value: "base32768"},
+		},
+		UnimplementableFsMethods:     []string{"OpenWriterAt"},
+		UnimplementableObjectMethods: []string{"MimeType"},
+	})
+}
+
 // TestOff runs integration tests against the remote
 func TestOff(t *testing.T) {
 	if *fstest.RemoteName != "" {

backend/drive/drive.go

@@ -21,7 +21,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 	"sync/atomic"
 	"text/template"
 	"time"

backend/dropbox/batcher.go

@@ -10,7 +10,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 	"time"
 
 	"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/async"

backend/filefabric/filefabric.go

@@ -25,7 +25,7 @@ import (
 	"net/url"
 	"path"
 	"strings"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 	"sync/atomic"
 	"time"
 

backend/ftp/ftp.go

@@ -12,7 +12,7 @@ import (
 	"path"
 	"runtime"
 	"strings"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 	"time"
 
 	"github.com/jlaffaye/ftp"

backend/googlephotos/albums.go

@@ -5,7 +5,7 @@ package googlephotos
 import (
 	"path"
 	"strings"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 
 	"github.com/rclone/rclone/backend/googlephotos/api"
 )

backend/googlephotos/googlephotos.go

@@ -15,7 +15,7 @@ import (
 	"regexp"
 	"strconv"
 	"strings"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 	"time"
 
 	"github.com/rclone/rclone/backend/googlephotos/api"

backend/hasher/hasher.go

@@ -9,7 +9,7 @@ import (
 	"io"
 	"path"
 	"strings"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 	"time"
 
 	"github.com/rclone/rclone/fs"

backend/hasher/kv.go

@@ -294,7 +294,7 @@ func (f *Fs) dumpLine(r *hashRecord, path string, include bool, err error) strin
 		if hashVal == "" || err != nil {
 			hashVal = "-"
 		}
-		hashVal = fmt.Sprintf("%-*s", hash.Width(hashType), hashVal)
+		hashVal = fmt.Sprintf("%-*s", hash.Width(hashType, false), hashVal)
 		hashes = append(hashes, hashName+":"+hashVal)
 	}
 	hashesStr := strings.Join(hashes, " ")

backend/hdfs/fs.go

@@ -263,6 +263,98 @@ func (f *Fs) Purge(ctx context.Context, dir string) error {
 	return f.client.RemoveAll(realpath)
 }
 
+// Move src to this remote using server-side move operations.
+//
+// This is stored with the remote path given
+//
+// It returns the destination Object and a possible error
+//
+// Will only be called if src.Fs().Name() == f.Name()
+//
+// If it isn't possible then return fs.ErrorCantMove
+func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
+	srcObj, ok := src.(*Object)
+	if !ok {
+		fs.Debugf(src, "Can't move - not same remote type")
+		return nil, fs.ErrorCantMove
+	}
+
+	// Get the real paths from the remote specs:
+	sourcePath := srcObj.fs.realpath(srcObj.remote)
+	targetPath := f.realpath(remote)
+	fs.Debugf(f, "rename [%s] to [%s]", sourcePath, targetPath)
+
+	// Make sure the target folder exists:
+	dirname := path.Dir(targetPath)
+	err := f.client.MkdirAll(dirname, 0755)
+	if err != nil {
+		return nil, err
+	}
+
+	// Do the move
+	// Note that the underlying HDFS library hard-codes Overwrite=True, but this is expected rclone behaviour.
+	err = f.client.Rename(sourcePath, targetPath)
+	if err != nil {
+		return nil, err
+	}
+
+	// Look up the resulting object
+	info, err := f.client.Stat(targetPath)
+	if err != nil {
+		return nil, err
+	}
+
+	// And return it:
+	return &Object{
+		fs:      f,
+		remote:  remote,
+		size:    info.Size(),
+		modTime: info.ModTime(),
+	}, nil
+}
+
+// DirMove moves src, srcRemote to this remote at dstRemote
+// using server-side move operations.
+//
+// Will only be called if src.Fs().Name() == f.Name()
+//
+// If it isn't possible then return fs.ErrorCantDirMove
+//
+// If destination exists then return fs.ErrorDirExists
+func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) (err error) {
+	srcFs, ok := src.(*Fs)
+	if !ok {
+		return fs.ErrorCantDirMove
+	}
+
+	// Get the real paths from the remote specs:
+	sourcePath := srcFs.realpath(srcRemote)
+	targetPath := f.realpath(dstRemote)
+	fs.Debugf(f, "rename [%s] to [%s]", sourcePath, targetPath)
+
+	// Check if the destination exists:
+	info, err := f.client.Stat(targetPath)
+	if err == nil {
+		fs.Debugf(f, "target directory already exits, IsDir = [%t]", info.IsDir())
+		return fs.ErrorDirExists
+	}
+
+	// Make sure the targets parent folder exists:
+	dirname := path.Dir(targetPath)
+	err = f.client.MkdirAll(dirname, 0755)
+	if err != nil {
+		return err
+	}
+
+	// Do the move
+	err = f.client.Rename(sourcePath, targetPath)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 // About gets quota information from the Fs
 func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
 	info, err := f.client.StatFs()
@@ -318,4 +410,6 @@ var (
 	_ fs.Purger      = (*Fs)(nil)
 	_ fs.PutStreamer = (*Fs)(nil)
 	_ fs.Abouter     = (*Fs)(nil)
+	_ fs.Mover       = (*Fs)(nil)
+	_ fs.DirMover    = (*Fs)(nil)
 )

backend/http/http.go

@@ -15,7 +15,7 @@ import (
 	"path"
 	"strconv"
 	"strings"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 	"time"
 
 	"github.com/rclone/rclone/fs"

backend/jottacloud/jottacloud.go

@@ -69,6 +69,10 @@ const (
 	teliaCloudTokenURL = "https://cloud-auth.telia.se/auth/realms/telia_se/protocol/openid-connect/token"
 	teliaCloudAuthURL  = "https://cloud-auth.telia.se/auth/realms/telia_se/protocol/openid-connect/auth"
 	teliaCloudClientID = "desktop"
+
+	tele2CloudTokenURL = "https://mittcloud-auth.tele2.se/auth/realms/comhem/protocol/openid-connect/token"
+	tele2CloudAuthURL  = "https://mittcloud-auth.tele2.se/auth/realms/comhem/protocol/openid-connect/auth"
+	tele2CloudClientID = "desktop"
 )
 
 // Register with Fs
@@ -131,6 +135,9 @@ func Config(ctx context.Context, name string, m configmap.Mapper, config fs.Conf
 		}, {
 			Value: "telia",
 			Help:  "Telia Cloud authentication.\nUse this if you are using Telia Cloud.",
+		}, {
+			Value: "tele2",
+			Help:  "Tele2 Cloud authentication.\nUse this if you are using Tele2 Cloud.",
 		}})
 	case "auth_type_done":
 		// Jump to next state according to config chosen
@@ -238,6 +245,21 @@ machines.`)
 				RedirectURL: oauthutil.RedirectLocalhostURL,
 			},
 		})
+	case "tele2": // tele2 cloud config
+		m.Set("configVersion", fmt.Sprint(configVersion))
+		m.Set(configClientID, tele2CloudClientID)
+		m.Set(configTokenURL, tele2CloudTokenURL)
+		return oauthutil.ConfigOut("choose_device", &oauthutil.Options{
+			OAuth2Config: &oauth2.Config{
+				Endpoint: oauth2.Endpoint{
+					AuthURL:  tele2CloudAuthURL,
+					TokenURL: tele2CloudTokenURL,
+				},
+				ClientID:    tele2CloudClientID,
+				Scopes:      []string{"openid", "jotta-default", "offline_access"},
+				RedirectURL: oauthutil.RedirectLocalhostURL,
+			},
+		})
 	case "choose_device":
 		return fs.ConfigConfirm("choose_device_query", false, "config_non_standard", "Use a non standard device/mountpoint e.g. for accessing files uploaded using the official Jottacloud client?")
 	case "choose_device_query":

backend/local/local.go

@@ -13,7 +13,7 @@ import (
 	"path/filepath"
 	"runtime"
 	"strings"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 	"time"
 	"unicode/utf8"
 

backend/local/remove_test.go

@@ -3,7 +3,7 @@ package local
 import (
 	"io/ioutil"
 	"os"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 	"testing"
 	"time"
 

backend/mailru/mailru.go

@@ -12,7 +12,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 	"time"
 
 	"encoding/hex"

backend/mega/mega.go

@@ -22,7 +22,7 @@ import (
 	"io"
 	"path"
 	"strings"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 	"time"
 
 	"github.com/rclone/rclone/fs"

backend/memory/memory.go

@@ -11,7 +11,7 @@ import (
 	"io/ioutil"
 	"path"
 	"strings"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 	"time"
 
 	"github.com/rclone/rclone/fs"

backend/onedrive/onedrive.go

@@ -16,7 +16,7 @@ import (
 	"regexp"
 	"strconv"
 	"strings"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 	"time"
 
 	"github.com/rclone/rclone/backend/onedrive/api"

backend/qingstor/upload.go

@@ -13,7 +13,7 @@ import (
 	"hash"
 	"io"
 	"sort"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/lib/atexit"

backend/s3/s3.go

@@ -19,7 +19,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"

backend/seafile/pacer.go

@@ -4,7 +4,7 @@ import (
 	"context"
 	"fmt"
 	"net/url"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 	"time"
 
 	"github.com/rclone/rclone/fs"

backend/seafile/seafile.go

@@ -10,7 +10,7 @@ import (
 	"path"
 	"strconv"
 	"strings"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 	"time"
 
 	"github.com/coreos/go-semver/semver"

backend/sftp/sftp.go

@@ -17,7 +17,7 @@ import (
 	"regexp"
 	"strconv"
 	"strings"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 	"sync/atomic"
 	"time"
 
@@ -42,7 +42,8 @@ const (
 	hashCommandNotSupported = "none"
 	minSleep                = 100 * time.Millisecond
 	maxSleep                = 2 * time.Second
-	decayConstant           = 2 // bigger for slower decay, exponential
+	decayConstant           = 2           // bigger for slower decay, exponential
+	keepAliveInterval       = time.Minute // send keepalives every this long while running commands
 )
 
 var (
@@ -152,11 +153,11 @@ different. This issue affects among others Synology NAS boxes.
 
 Shared folders can be found in directories representing volumes
 
-    rclone sync /home/local/directory remote:/directory --ssh-path-override /volume2/directory
+    rclone sync /home/local/directory remote:/directory --sftp-path-override /volume2/directory
 
 Home directory can be found in a shared folder called "home"
 
-    rclone sync /home/local/directory remote:/home/directory --ssh-path-override /volume1/homes/USER/directory`,
+    rclone sync /home/local/directory remote:/home/directory --sftp-path-override /volume1/homes/USER/directory`,
 			Advanced: true,
 		}, {
 			Name:     "set_modtime",
@@ -339,6 +340,32 @@ func (c *conn) wait() {
 	c.err <- c.sshClient.Conn.Wait()
 }
 
+// Send a keepalive over the ssh connection
+func (c *conn) sendKeepAlive() {
+	_, _, err := c.sshClient.SendRequest("keepalive@openssh.com", true, nil)
+	if err != nil {
+		fs.Debugf(nil, "Failed to send keep alive: %v", err)
+	}
+}
+
+// Send keepalives every interval over the ssh connection until done is closed
+func (c *conn) sendKeepAlives(interval time.Duration) (done chan struct{}) {
+	done = make(chan struct{})
+	go func() {
+		t := time.NewTicker(interval)
+		defer t.Stop()
+		for {
+			select {
+			case <-t.C:
+				c.sendKeepAlive()
+			case <-done:
+				return
+			}
+		}
+	}()
+	return done
+}
+
 // Closes the connection
 func (c *conn) close() error {
 	sftpErr := c.sftpClient.Close()
@@ -1098,6 +1125,9 @@ func (f *Fs) run(ctx context.Context, cmd string) ([]byte, error) {
 	}
 	defer f.putSftpConnection(&c, err)
 
+	// Send keepalives while the connection is open
+	defer close(c.sendKeepAlives(keepAliveInterval))
+
 	session, err := c.sshClient.NewSession()
 	if err != nil {
 		return nil, fmt.Errorf("run: get SFTP session: %w", err)
@@ -1110,10 +1140,12 @@ func (f *Fs) run(ctx context.Context, cmd string) ([]byte, error) {
 	session.Stdout = &stdout
 	session.Stderr = &stderr
 
+	fs.Debugf(f, "Running remote command: %s", cmd)
 	err = session.Run(cmd)
 	if err != nil {
-		return nil, fmt.Errorf("failed to run %q: %s: %w", cmd, stderr.Bytes(), err)
+		return nil, fmt.Errorf("failed to run %q: %s: %w", cmd, bytes.TrimSpace(stderr.Bytes()), err)
 	}
+	fs.Debugf(f, "Remote command result: %s", bytes.TrimSpace(stdout.Bytes()))
 
 	return stdout.Bytes(), nil
 }
@@ -1155,8 +1187,8 @@ func (f *Fs) Hashes() hash.Set {
 	}
 
 	changed := false
-	md5Works := checkHash([]string{"md5sum", "md5 -r"}, "d41d8cd98f00b204e9800998ecf8427e", &f.opt.Md5sumCommand, &changed)
+	md5Works := checkHash([]string{"md5sum", "md5 -r", "rclone md5sum"}, "d41d8cd98f00b204e9800998ecf8427e", &f.opt.Md5sumCommand, &changed)
-	sha1Works := checkHash([]string{"sha1sum", "sha1 -r"}, "da39a3ee5e6b4b0d3255bfef95601890afd80709", &f.opt.Sha1sumCommand, &changed)
+	sha1Works := checkHash([]string{"sha1sum", "sha1 -r", "rclone sha1sum"}, "da39a3ee5e6b4b0d3255bfef95601890afd80709", &f.opt.Sha1sumCommand, &changed)
 
 	if changed {
 		f.m.Set("md5sum_command", f.opt.Md5sumCommand)
@@ -1230,8 +1262,6 @@ func (o *Object) Remote() string {
 // Hash returns the selected checksum of the file
 // If no checksum is available it returns ""
 func (o *Object) Hash(ctx context.Context, r hash.Type) (string, error) {
-	o.fs.addSession() // Show session in use
-	defer o.fs.removeSession()
 	if o.fs.opt.DisableHashCheck {
 		return "", nil
 	}
@@ -1255,36 +1285,16 @@ func (o *Object) Hash(ctx context.Context, r hash.Type) (string, error) {
 		return "", hash.ErrUnsupported
 	}
 
-	c, err := o.fs.getSftpConnection(ctx)
-	if err != nil {
-		return "", fmt.Errorf("Hash get SFTP connection: %w", err)
-	}
-	session, err := c.sshClient.NewSession()
-	o.fs.putSftpConnection(&c, err)
-	if err != nil {
-		return "", fmt.Errorf("Hash put SFTP connection: %w", err)
-	}
-
-	var stdout, stderr bytes.Buffer
-	session.Stdout = &stdout
-	session.Stderr = &stderr
 	escapedPath := shellEscape(o.path())
 	if o.fs.opt.PathOverride != "" {
 		escapedPath = shellEscape(path.Join(o.fs.opt.PathOverride, o.remote))
 	}
-	err = session.Run(hashCmd + " " + escapedPath)
+	b, err := o.fs.run(ctx, hashCmd+" "+escapedPath)
-	fs.Debugf(nil, "sftp cmd = %s", escapedPath)
 	if err != nil {
-		_ = session.Close()
+		return "", fmt.Errorf("failed to calculate %v hash: %w", r, err)
-		fs.Debugf(o, "Failed to calculate %v hash: %v (%s)", r, err, bytes.TrimSpace(stderr.Bytes()))
-		return "", nil
 	}
 
-	_ = session.Close()
-	b := stdout.Bytes()
-	fs.Debugf(nil, "sftp output = %q", b)
 	str := parseHash(b)
-	fs.Debugf(nil, "sftp hash = %q", str)
 	if r == hash.MD5 {
 		o.md5sum = &str
 	} else if r == hash.SHA1 {

backend/sftp/stringlock.go

@@ -3,7 +3,7 @@
 
 package sftp
 
-import "sync"
+import "github.com/rclone/rclone/lib/sync"
 
 // stringLock locks for string IDs passed in
 type stringLock struct {

backend/sftp/stringlock_test.go

@@ -5,7 +5,7 @@ package sftp
 
 import (
 	"fmt"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 	"testing"
 	"time"
 

backend/sharefile/upload.go

@@ -13,7 +13,7 @@ import (
 	"fmt"
 	"io"
 	"strings"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 
 	"github.com/rclone/rclone/backend/sharefile/api"
 	"github.com/rclone/rclone/fs"

backend/sugarsync/sugarsync.go

@@ -23,7 +23,7 @@ import (
 	"regexp"
 	"strconv"
 	"strings"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 	"time"
 
 	"github.com/rclone/rclone/backend/sugarsync/api"

backend/union/entry.go

@@ -4,7 +4,7 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 	"time"
 
 	"github.com/rclone/rclone/backend/union/upstream"

backend/union/policy/epall.go

@@ -3,7 +3,7 @@ package policy
 import (
 	"context"
 	"path"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 
 	"github.com/rclone/rclone/backend/union/upstream"
 	"github.com/rclone/rclone/fs"

backend/union/policy/newest.go

@@ -3,7 +3,7 @@ package policy
 import (
 	"context"
 	"path"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 	"time"
 
 	"github.com/rclone/rclone/backend/union/upstream"

backend/union/union.go

@@ -9,7 +9,7 @@ import (
 	"path"
 	"path/filepath"
 	"strings"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 	"time"
 
 	"github.com/rclone/rclone/backend/union/policy"

backend/union/upstream/upstream.go

@@ -9,7 +9,7 @@ import (
 	"path"
 	"path/filepath"
 	"strings"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 	"sync/atomic"
 	"time"
 

backend/webdav/api/types.go

@@ -6,7 +6,7 @@ import (
 	"regexp"
 	"strconv"
 	"strings"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 	"time"
 
 	"github.com/rclone/rclone/fs"

backend/webdav/webdav.go

@@ -21,7 +21,7 @@ import (
 	"path"
 	"strconv"
 	"strings"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 	"time"
 
 	"github.com/rclone/rclone/backend/webdav/api"

backend/yandex/yandex.go

@@ -66,6 +66,11 @@ func init() {
 			})
 		},
 		Options: append(oauthutil.SharedOptions, []fs.Option{{
+			Name:     "hard_delete",
+			Help:     "Delete files permanently rather than putting them into the trash.",
+			Default:  false,
+			Advanced: true,
+		}, {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
 			Advanced: true,
@@ -79,8 +84,9 @@ func init() {
 
 // Options defines the configuration for this backend
 type Options struct {
-	Token string               `config:"token"`
+	Token      string               `config:"token"`
-	Enc   encoder.MultiEncoder `config:"encoding"`
+	HardDelete bool                 `config:"hard_delete"`
+	Enc        encoder.MultiEncoder `config:"encoding"`
 }
 
 // Fs represents a remote yandex
@@ -630,7 +636,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
 		}
 	}
 	//delete directory
-	return f.delete(ctx, root, false)
+	return f.delete(ctx, root, f.opt.HardDelete)
 }
 
 // Rmdir deletes the container
@@ -1141,7 +1147,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 // Remove an object
 func (o *Object) Remove(ctx context.Context) error {
-	return o.fs.delete(ctx, o.filePath(), false)
+	return o.fs.delete(ctx, o.filePath(), o.fs.opt.HardDelete)
 }
 
 // MimeType of an Object if known, "" otherwise

bin/cross-compile.go

@@ -19,7 +19,7 @@ import (
 	"runtime"
 	"sort"
 	"strings"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 	"text/template"
 	"time"
 

cmd/cmd.go

@@ -19,7 +19,7 @@ import (
 	"runtime/pprof"
 	"strconv"
 	"strings"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 	"time"
 
 	"github.com/rclone/rclone/fs"

cmd/cmount/fs.go

@@ -9,7 +9,7 @@ import (
 	"io"
 	"os"
 	"path"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 	"sync/atomic"
 	"time"
 

cmd/hashsum/hashsum.go

@@ -2,7 +2,6 @@ package hashsum
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"os"
 
@@ -26,11 +25,11 @@ var (
 func init() {
 	cmd.Root.AddCommand(commandDefinition)
 	cmdFlags := commandDefinition.Flags()
-	AddHashFlags(cmdFlags)
+	AddHashsumFlags(cmdFlags)
 }
 
-// AddHashFlags is a convenience function to add the command flags OutputBase64 and DownloadFlag to hashsum, md5sum, sha1sum
+// AddHashsumFlags is a convenience function to add the command flags OutputBase64 and DownloadFlag to hashsum, md5sum, sha1sum
-func AddHashFlags(cmdFlags *pflag.FlagSet) {
+func AddHashsumFlags(cmdFlags *pflag.FlagSet) {
 	flags.BoolVarP(cmdFlags, &OutputBase64, "base64", "", OutputBase64, "Output base64 encoded hashsum")
 	flags.StringVarP(cmdFlags, &HashsumOutfile, "output-file", "", HashsumOutfile, "Output hashsums to a file rather than the terminal")
 	flags.StringVarP(cmdFlags, &ChecksumFile, "checkfile", "C", ChecksumFile, "Validate hashes against a given SUM file instead of printing them")
@@ -41,7 +40,7 @@ func AddHashFlags(cmdFlags *pflag.FlagSet) {
 func GetHashsumOutput(filename string) (out *os.File, close func(), err error) {
 	out, err = os.Create(filename)
 	if err != nil {
-		err = fmt.Errorf("Failed to open output file %v: %w", filename, err)
+		err = fmt.Errorf("failed to open output file %v: %w", filename, err)
 		return nil, nil, err
 	}
 
@@ -55,6 +54,32 @@ func GetHashsumOutput(filename string) (out *os.File, close func(), err error) {
 	return out, close, nil
 }
 
+// CreateFromStdinArg checks args and produces hashsum from standard input if it is requested
+func CreateFromStdinArg(ht hash.Type, args []string, startArg int) (bool, error) {
+	var stdinArg bool
+	if len(args) == startArg {
+		// Missing arg: Always read from stdin
+		stdinArg = true
+	} else if len(args) > startArg && args[startArg] == "-" {
+		// Special arg: Read from stdin only if there is data available
+		if fi, _ := os.Stdin.Stat(); fi.Mode()&os.ModeCharDevice == 0 {
+			stdinArg = true
+		}
+	}
+	if !stdinArg {
+		return false, nil
+	}
+	if HashsumOutfile == "" {
+		return true, operations.HashSumStream(ht, OutputBase64, os.Stdin, nil)
+	}
+	output, close, err := GetHashsumOutput(HashsumOutfile)
+	if err != nil {
+		return true, err
+	}
+	defer close()
+	return true, operations.HashSumStream(ht, OutputBase64, os.Stdin, output)
+}
+
 var commandDefinition = &cobra.Command{
 	Use:   "hashsum <hash> remote:path",
 	Short: `Produces a hashsum file for all the objects in the path.`,
@@ -68,6 +93,11 @@ not supported by the remote, no hash will be returned.  With the
 download flag, the file will be downloaded from the remote and
 hashed locally enabling any hash for any remote.
 
+This command can also hash data received on standard input (stdin),
+by not passing a remote:path, or by passing a hyphen as remote:path
+when there is data to read (if not, the hypen will be treated literaly,
+as a relative path).
+
 Run without a hash to see the list of all supported hashes, e.g.
 
     $ rclone hashsum
@@ -83,8 +113,6 @@ Note that hash names are case insensitive and values are output in lower case.
 		if len(args) == 0 {
 			fmt.Print(hash.HelpString(0))
 			return nil
-		} else if len(args) == 1 {
-			return errors.New("need hash type and remote")
 		}
 		var ht hash.Type
 		err := ht.Set(args[0])
@@ -92,8 +120,10 @@ Note that hash names are case insensitive and values are output in lower case.
 			fmt.Println(hash.HelpString(0))
 			return err
 		}
+		if found, err := CreateFromStdinArg(ht, args, 1); found {
+			return err
+		}
 		fsrc := cmd.NewFsSrc(args[1:])
-
 		cmd.Run(false, false, command, func() error {
 			if ChecksumFile != "" {
 				fsum, sumFile := cmd.NewFsFile(ChecksumFile)

cmd/md5sum/md5sum.go

@@ -13,7 +13,7 @@ import (
 func init() {
 	cmd.Root.AddCommand(commandDefinition)
 	cmdFlags := commandDefinition.Flags()
-	hashsum.AddHashFlags(cmdFlags)
+	hashsum.AddHashsumFlags(cmdFlags)
 }
 
 var commandDefinition = &cobra.Command{
@@ -27,9 +27,17 @@ By default, the hash is requested from the remote.  If MD5 is
 not supported by the remote, no hash will be returned.  With the
 download flag, the file will be downloaded from the remote and
 hashed locally enabling MD5 for any remote.
+
+This command can also hash data received on standard input (stdin),
+by not passing a remote:path, or by passing a hyphen as remote:path
+when there is data to read (if not, the hypen will be treated literaly,
+as a relative path).
 `,
-	Run: func(command *cobra.Command, args []string) {
+	RunE: func(command *cobra.Command, args []string) error {
-		cmd.CheckArgs(1, 1, command, args)
+		cmd.CheckArgs(0, 1, command, args)
+		if found, err := hashsum.CreateFromStdinArg(hash.MD5, args, 0); found {
+			return err
+		}
 		fsrc := cmd.NewFsSrc(args)
 		cmd.Run(false, false, command, func() error {
 			if hashsum.ChecksumFile != "" {
@@ -46,5 +54,6 @@ hashed locally enabling MD5 for any remote.
 			defer close()
 			return operations.HashLister(context.Background(), hash.MD5, hashsum.OutputBase64, hashsum.DownloadFlag, fsrc, output)
 		})
+		return nil
 	},
 }

cmd/mount/test/seekers.go

@@ -12,7 +12,7 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 	"time"
 )
 

cmd/mountlib/mount.go

@@ -7,7 +7,7 @@ import (
 	"os"
 	"runtime"
 	"strings"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 	"time"
 
 	"github.com/rclone/rclone/cmd"

cmd/mountlib/rc.go

@@ -5,7 +5,7 @@ import (
 	"errors"
 	"log"
 	"sort"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 	"time"
 
 	"github.com/rclone/rclone/fs"

cmd/ncdu/scan/scan.go

@@ -5,7 +5,7 @@ import (
 	"context"
 	"fmt"
 	"path"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/walk"

cmd/progress.go

@@ -6,7 +6,7 @@ import (
 	"bytes"
 	"fmt"
 	"strings"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 	"time"
 
 	"github.com/rclone/rclone/fs"

cmd/rcd/rcd.go

@@ -3,7 +3,7 @@ package rcd
 import (
 	"context"
 	"log"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 
 	sysdnotify "github.com/iguanesolutions/go-systemd/v5/notify"
 	"github.com/rclone/rclone/cmd"

cmd/serve/docker/driver.go

@@ -10,7 +10,7 @@ import (
 	"path/filepath"
 	"reflect"
 	"sort"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 	"time"
 
 	sysdnotify "github.com/iguanesolutions/go-systemd/v5/notify"

cmd/serve/ftp/ftp.go

@@ -14,7 +14,7 @@ import (
 	"os"
 	"os/user"
 	"strconv"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 	"time"
 
 	"github.com/rclone/rclone/cmd"

cmd/serve/restic/cache.go

@@ -5,7 +5,7 @@ package restic
 
 import (
 	"strings"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 
 	"github.com/rclone/rclone/fs"
 )

cmd/serve/sftp/sftp.go

@@ -100,6 +100,17 @@ be used with sshd via ~/.ssh/authorized_keys, for example:
 
     restrict,command="rclone serve sftp --stdio ./photos" ssh-rsa ...
 
+On the client you need to set "--transfers 1" when using --stdio.
+Otherwise multiple instances of the rclone server are started by OpenSSH
+which can lead to "corrupted on transfer" errors. This is the case because
+the client chooses indiscriminately which server to send commands to while
+the servers all have different views of the state of the filing system.
+
+The "restrict" in authorized_keys prevents SHA1SUMs and MD5SUMs from beeing
+used. Omitting "restrict" and using --sftp-path-override to enable
+checksumming is possible but less secure and you could use the SFTP server
+provided by OpenSSH in this case.
+
 ` + vfs.Help + proxy.Help,
 	Run: func(command *cobra.Command, args []string) {
 		var f fs.Fs

cmd/sha1sum/sha1sum.go

@@ -13,7 +13,7 @@ import (
 func init() {
 	cmd.Root.AddCommand(commandDefinition)
 	cmdFlags := commandDefinition.Flags()
-	hashsum.AddHashFlags(cmdFlags)
+	hashsum.AddHashsumFlags(cmdFlags)
 }
 
 var commandDefinition = &cobra.Command{
@@ -27,9 +27,20 @@ By default, the hash is requested from the remote.  If SHA-1 is
 not supported by the remote, no hash will be returned.  With the
 download flag, the file will be downloaded from the remote and
 hashed locally enabling SHA-1 for any remote.
+
+This command can also hash data received on standard input (stdin),
+by not passing a remote:path, or by passing a hyphen as remote:path
+when there is data to read (if not, the hypen will be treated literaly,
+as a relative path).
+
+This command can also hash data received on STDIN, if not passing
+a remote:path.
 `,
-	Run: func(command *cobra.Command, args []string) {
+	RunE: func(command *cobra.Command, args []string) error {
-		cmd.CheckArgs(1, 1, command, args)
+		cmd.CheckArgs(0, 1, command, args)
+		if found, err := hashsum.CreateFromStdinArg(hash.SHA1, args, 0); found {
+			return err
+		}
 		fsrc := cmd.NewFsSrc(args)
 		cmd.Run(false, false, command, func() error {
 			if hashsum.ChecksumFile != "" {
@@ -46,5 +57,6 @@ hashed locally enabling SHA-1 for any remote.
 			defer close()
 			return operations.HashLister(context.Background(), hash.SHA1, hashsum.OutputBase64, hashsum.DownloadFlag, fsrc, output)
 		})
+		return nil
 	},
 }

cmd/test/info/info.go

@@ -16,7 +16,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 	"time"
 
 	"github.com/rclone/rclone/cmd"

cmd/test/memory/memory.go

@@ -3,7 +3,7 @@ package memory
 import (
 	"context"
 	"runtime"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 
 	"github.com/rclone/rclone/cmd"
 	"github.com/rclone/rclone/cmd/test"

docs/content/authors.md

@@ -547,3 +547,9 @@ put them back in again.` >}}
   * bbabich <bbabich@datamossa.com>
   * David <dp.davide.palma@gmail.com>
   * Borna Butkovic <borna@favicode.net>
+  * Fredric Arklid <fredric.arklid@consid.se>
+  * Andy Jackson <Andrew.Jackson@bl.uk>
+  * Sinan Tan <i@tinytangent.com>
+  * deinferno <14363193+deinferno@users.noreply.github.com>
+  * rsapkf <rsapkfff@pm.me>
+  * Will Holtz <wholtz@gmail.com>

docs/content/azureblob.md

@@ -81,6 +81,14 @@ key.  It is stored using RFC3339 Format time with nanosecond
 precision.  The metadata is supplied during directory listings so
 there is no overhead to using it.
 
+### Performance
+
+When uploading large files, increasing the value of
+`--azureblob-upload-concurrency` will increase performance at the cost
+of using more memory. The default of 16 is set quite conservatively to
+use less memory. It maybe be necessary raise it to 64 or higher to
+fully utilize a 1 GBit/s link with a single file transfer.
+
 ### Restricted filename characters
 
 In addition to the [default restricted characters set](/overview/#restricted-characters)

docs/content/commands/rclone_config_create.md

@@ -107,8 +107,9 @@ At the end of the non interactive process, rclone will return a result
 with `State` as empty string.
 
 If `--all` is passed then rclone will ask all the config questions,
-not just the post config questions. Any parameters are used as
+not just the post config questions. Parameters that are supplied on
-defaults for questions as usual.
+the command line or from environment variables are used as defaults
+for questions as usual.
 
 Note that `bin/config.py` in the rclone source implements this protocol
 as a readable demonstration.

docs/content/crypt.md

@@ -373,6 +373,14 @@ total path length which rclone is more likely to breach using
 characters in length issues should not be encountered, irrespective of
 cloud storage provider.
 
+An experimental advanced option `filename_encoding` is now provided to
+address this problem to a certain degree.
+For cloud storage systems with case sensitive file names (e.g. Google Drive),
+`base64` can be used to reduce file name length. 
+For cloud storage systems using UTF-16 to store file names internally
+(e.g. OneDrive), `base32768` can be used to drastically reduce
+file name length. 
+
 An alternative, future rclone file name encryption mode may tolerate
 backend provider path length limits.
 

docs/content/jottacloud.md

@@ -41,8 +41,8 @@ Note that the web interface may refer to this token as a JottaCli token.
 
 ### Legacy authentication
 
-If you are using one of the whitelabel versions (e.g. from Elkjøp or Tele2) you may not have the option
+If you are using one of the whitelabel versions (e.g. from Elkjøp) you may not have the option
-to generate a CLI token. In this case you'll have to use the legacy authentication. To to this select
+to generate a CLI token. In this case you'll have to use the legacy authentication. To do this select
 yes when the setup asks for legacy authentication and enter your username and password.
 The rest of the setup is identical to the default setup.
 
@@ -53,6 +53,13 @@ additionally uses a separate authentication flow where the username is generated
 rclone to use Telia Cloud, choose Telia Cloud authentication in the setup. The rest of the setup is
 identical to the default setup.
 
+### Tele2 Cloud authentication
+
+As Tele2-Com Hem merger was completed this authentication can be used for former Com Hem Cloud and
+Tele2 Cloud customers as no support for creating a CLI token exists, and additionally uses a separate
+authentication flow where the username is generated internally. To setup rclone to use Tele2 Cloud,
+choose Tele2 Cloud authentication in the setup. The rest of the setup is identical to the default setup.
+
 ## Configuration
 
 Here is an example of how to make a remote called `remote` with the default setup.  First run:

docs/content/overview.md

@@ -422,7 +422,7 @@ upon backend-specific capabilities.
 | Google Cloud Storage         | Yes   | Yes  | No   | No      | No      | Yes   | Yes          | No           | No    | No       |
 | Google Drive                 | Yes   | Yes  | Yes  | Yes     | Yes     | Yes   | Yes          | Yes          | Yes   | Yes      |
 | Google Photos                | No    | No   | No   | No      | No      | No    | No           | No           | No    | No       |
-| HDFS                         | Yes   | No   | No   | No      | No      | No    | Yes          | No           | Yes   | Yes      |
+| HDFS                         | Yes   | No   | Yes  | Yes     | No      | No    | Yes          | No           | Yes   | Yes      |
 | HTTP                         | No    | No   | No   | No      | No      | No    | No           | No           | No    | Yes      |
 | Hubic                        | Yes † | Yes  | No   | No      | No      | Yes   | Yes          | No           | Yes   | No       |
 | Jottacloud                   | Yes   | Yes  | Yes  | Yes     | Yes     | Yes   | No           | Yes          | Yes   | Yes      |

docs/content/pcloud.md

@@ -80,7 +80,7 @@ List all the files in your pCloud
 
     rclone ls remote:
 
-To copy a local directory to an pCloud directory called backup
+To copy a local directory to a pCloud directory called backup
 
     rclone copy /home/source remote:backup
 

docs/content/sftp.md

@@ -620,7 +620,7 @@ issue](https://github.com/pkg/sftp/issues/156) is fixed.
 Note that since SFTP isn't HTTP based the following flags don't work
 with it: `--dump-headers`, `--dump-bodies`, `--dump-auth`
 
-Note that `--timeout` isn't supported (but `--contimeout` is).
+Note that `--timeout` and `--contimeout` are both supported.
 
 
 ## C14 {#c14}

docs/content/yandex.md

@@ -175,6 +175,15 @@ Leave blank to use the provider defaults.
 - Type:        string
 - Default:     ""
 
+#### --yandex-hard-delete
+
+Delete files permanently rather than putting them into the trash.
+
+- Config:      hard_delete
+- Env Var:     RCLONE_YANDEX_HARD_DELETE
+- Type:        bool
+- Default:     false
+
 #### --yandex-encoding
 
 This sets the encoding for the backend.

fs/accounting/accounting.go

@@ -6,7 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 	"time"
 	"unicode/utf8"
 

fs/accounting/inprogress.go

@@ -2,7 +2,7 @@ package accounting
 
 import (
 	"context"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 
 	"github.com/rclone/rclone/fs"
 )

fs/accounting/stats.go

@@ -6,7 +6,7 @@ import (
 	"fmt"
 	"sort"
 	"strings"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 	"time"
 
 	"github.com/rclone/rclone/fs"

fs/accounting/stats_groups.go

@@ -2,7 +2,7 @@ package accounting
 
 import (
 	"context"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 
 	"github.com/rclone/rclone/fs/rc"
 

fs/accounting/token_bucket.go

@@ -4,11 +4,11 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"sync"
 	"time"
 
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/rc"
+	"github.com/rclone/rclone/lib/sync"
 	"golang.org/x/time/rate"
 )
 

fs/accounting/transfer.go

@@ -4,7 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"io"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 	"time"
 
 	"github.com/rclone/rclone/fs"

fs/accounting/transfermap.go

@@ -5,7 +5,7 @@ import (
 	"fmt"
 	"sort"
 	"strings"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/rc"

fs/asyncreader/asyncreader.go

@@ -6,7 +6,7 @@ import (
 	"context"
 	"errors"
 	"io"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 	"time"
 
 	"github.com/rclone/rclone/fs"

fs/asyncreader/asyncreader_test.go

@@ -9,7 +9,7 @@ import (
 	"io/ioutil"
 	"math/rand"
 	"strings"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 	"testing"
 	"testing/iotest"
 	"time"

fs/cache/cache.go

@@ -4,7 +4,7 @@ package cache
 import (
 	"context"
 	"runtime"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/filter"

fs/chunkedreader/chunkedreader.go

@@ -4,7 +4,7 @@ import (
 	"context"
 	"errors"
 	"io"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/hash"

fs/config/configfile/configfile.go

@@ -8,7 +8,7 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 
 	"github.com/Unknwon/goconfig"
 	"github.com/rclone/rclone/fs"

fs/config/default_storage.go

@@ -2,7 +2,7 @@ package config
 
 import (
 	"encoding/json"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 )
 
 // defaultStorage implements config.Storage, providing in-memory config.

fs/filter/filter_test.go

@@ -6,7 +6,7 @@ import (
 	"io/ioutil"
 	"os"
 	"strings"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 	"testing"
 	"time"
 

fs/fs_test.go

@@ -7,7 +7,7 @@ import (
 	"fmt"
 	"os"
 	"strings"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 	"testing"
 	"time"
 

fs/fshttp/dialer.go

@@ -5,7 +5,7 @@ import (
 	"net"
 	"runtime"
 	"strings"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 	"time"
 
 	"github.com/rclone/rclone/fs"

fs/fshttp/http.go

@@ -12,7 +12,7 @@ import (
 	"net/http"
 	"net/http/cookiejar"
 	"net/http/httputil"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 	"time"
 
 	"github.com/rclone/rclone/fs"
@@ -136,12 +136,14 @@ func NewClient(ctx context.Context) *http.Client {
 // Transport is our http Transport which wraps an http.Transport
 // * Sets the User Agent
 // * Does logging
+// * Updates metrics
 type Transport struct {
 	*http.Transport
 	dump          fs.DumpFlags
 	filterRequest func(req *http.Request)
 	userAgent     string
 	headers       []*fs.HTTPOption
+	metrics       *Metrics
 }
 
 // newTransport wraps the http.Transport passed in and logs all
@@ -152,6 +154,7 @@ func newTransport(ci *fs.ConfigInfo, transport *http.Transport) *Transport {
 		dump:      ci.Dump,
 		userAgent: ci.UserAgent,
 		headers:   ci.Headers,
+		metrics:   DefaultMetrics,
 	}
 }
 
@@ -283,6 +286,9 @@ func (t *Transport) RoundTrip(req *http.Request) (resp *http.Response, err error
 		fs.Debugf(nil, "%s", separatorResp)
 		logMutex.Unlock()
 	}
+	// Update metrics
+	t.metrics.onResponse(req, resp)
+
 	if err == nil {
 		checkServerTime(req, resp)
 	}

fs/fshttp/prometheus.go

@@ -0,0 +1,51 @@
+package fshttp
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// Metrics provide Transport HTTP level metrics.
+type Metrics struct {
+	StatusCode *prometheus.CounterVec
+}
+
+// NewMetrics creates a new metrics instance, the instance shall be assigned to
+// DefaultMetrics before any processing takes place.
+func NewMetrics(namespace string) *Metrics {
+	return &Metrics{
+		StatusCode: prometheus.NewCounterVec(prometheus.CounterOpts{
+			Namespace: namespace,
+			Subsystem: "http",
+			Name:      "status_code",
+		}, []string{"host", "method", "code"}),
+	}
+}
+
+// DefaultMetrics specifies metrics used for new Transports.
+var DefaultMetrics = (*Metrics)(nil)
+
+// Collectors returns all prometheus metrics as collectors for registration.
+func (m *Metrics) Collectors() []prometheus.Collector {
+	if m == nil {
+		return nil
+	}
+	return []prometheus.Collector{
+		m.StatusCode,
+	}
+}
+
+func (m *Metrics) onResponse(req *http.Request, resp *http.Response) {
+	if m == nil {
+		return
+	}
+
+	var statusCode = 0
+	if resp != nil {
+		statusCode = resp.StatusCode
+	}
+
+	m.StatusCode.WithLabelValues(req.Host, req.Method, fmt.Sprint(statusCode)).Inc()
+}

fs/hash/hash.go

@@ -4,6 +4,7 @@ import (
 	"crypto/md5"
 	"crypto/sha1"
 	"crypto/sha256"
+	"encoding/base64"
 	"encoding/hex"
 	"errors"
 	"fmt"
@@ -92,8 +93,11 @@ func Supported() Set {
 }
 
 // Width returns the width in characters for any HashType
-func Width(hashType Type) int {
+func Width(hashType Type, base64Encoded bool) int {
 	if hash := type2hash[hashType]; hash != nil {
+		if base64Encoded {
+			return base64.URLEncoding.EncodedLen(hash.width / 2)
+		}
 		return hash.width
 	}
 	return 0
@@ -243,6 +247,18 @@ func (m *MultiHasher) Sum(hashType Type) ([]byte, error) {
 	return h.Sum(nil), nil
 }
 
+// SumString returns the specified hash from the multihasher as a hex or base64 encoded string
+func (m *MultiHasher) SumString(hashType Type, base64Encoded bool) (string, error) {
+	sum, err := m.Sum(hashType)
+	if err != nil {
+		return "", err
+	}
+	if base64Encoded {
+		return base64.URLEncoding.EncodeToString(sum), nil
+	}
+	return hex.EncodeToString(sum), nil
+}
+
 // Size returns the number of bytes written
 func (m *MultiHasher) Size() int64 {
 	return m.size

fs/march/march.go

@@ -7,7 +7,7 @@ import (
 	"path"
 	"sort"
 	"strings"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/dirtree"

fs/march/march_test.go

@@ -7,7 +7,7 @@ import (
 	"errors"
 	"fmt"
 	"strings"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 	"testing"
 
 	_ "github.com/rclone/rclone/backend/local"

fs/operations/check.go

@@ -10,7 +10,7 @@ import (
 	"os"
 	"regexp"
 	"strings"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 	"sync/atomic"
 
 	"github.com/rclone/rclone/fs"

fs/operations/operations.go

@@ -18,7 +18,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
-	"sync"
+	"github.com/rclone/rclone/lib/sync"
 	"sync/atomic"
 	"time"
 
@@ -405,7 +405,7 @@ func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Obj
 			if err == nil {
 				dst = newDst
 				in.ServerSideCopyEnd(dst.Size()) // account the bytes for the server-side transfer
-				err = in.Close()
+				_ = in.Close()
 			} else {
 				_ = in.Close()
 			}
@@ -598,6 +598,8 @@ func Move(ctx context.Context, fdst fs.Fs, dst fs.Object, remote string, src fs.
 			}
 		}
 		// Move dst <- src
+		in := tr.Account(ctx, nil) // account the transfer
+		in.ServerSideCopyStart()
 		newDst, err = doMove(ctx, src, remote)
 		switch err {
 		case nil:
@@ -606,13 +608,16 @@ func Move(ctx context.Context, fdst fs.Fs, dst fs.Object, remote string, src fs.
 			} else {
 				fs.Infof(src, "Moved (server-side)")
 			}
-
+			in.ServerSideCopyEnd(newDst.Size()) // account the bytes for the server-side transfer
+			_ = in.Close()
 			return newDst, nil
 		case fs.ErrorCantMove:
 			fs.Debugf(src, "Can't move, switching to copy")
+			_ = in.Close()
 		default:
 			err = fs.CountError(err)
 			fs.Errorf(src, "Couldn't move: %v", err)
+			_ = in.Close()
 			return newDst, err
 		}
 	}
@@ -946,7 +951,7 @@ func ListLong(ctx context.Context, f fs.Fs, w io.Writer) error {
 // hashSum returns the human-readable hash for ht passed in.  This may
 // be UNSUPPORTED or ERROR. If it isn't returning a valid hash it will
 // return an error.
-func hashSum(ctx context.Context, ht hash.Type, downloadFlag bool, o fs.Object) (string, error) {
+func hashSum(ctx context.Context, ht hash.Type, base64Encoded bool, downloadFlag bool, o fs.Object) (string, error) {
 	var sum string
 	var err error
 
@@ -968,7 +973,7 @@ func hashSum(ctx context.Context, ht hash.Type, downloadFlag bool, o fs.Object)
 		}
 		in, err := NewReOpen(ctx, o, fs.GetConfig(ctx).LowLevelRetries, options...)
 		if err != nil {
-			return "ERROR", fmt.Errorf("Failed to open file %v: %w", o, err)
+			return "ERROR", fmt.Errorf("failed to open file %v: %w", o, err)
 		}
 
 		// Account and buffer the transfer
@@ -977,21 +982,20 @@ func hashSum(ctx context.Context, ht hash.Type, downloadFlag bool, o fs.Object)
 		// Setup hasher
 		hasher, err := hash.NewMultiHasherTypes(hash.NewHashSet(ht))
 		if err != nil {
-			return "UNSUPPORTED", fmt.Errorf("Hash unsupported: %w", err)
+			return "UNSUPPORTED", fmt.Errorf("hash unsupported: %w", err)
 		}
 
 		// Copy to hasher, downloading the file and passing directly to hash
 		_, err = io.Copy(hasher, in)
 		if err != nil {
-			return "ERROR", fmt.Errorf("Failed to copy file to hasher: %w", err)
+			return "ERROR", fmt.Errorf("failed to copy file to hasher: %w", err)
 		}
 
-		// Get hash and encode as hex
+		// Get hash as hex or base64 encoded string
-		byteSum, err := hasher.Sum(ht)
+		sum, err = hasher.SumString(ht, base64Encoded)
 		if err != nil {
-			return "ERROR", fmt.Errorf("Hasher returned an error: %w", err)
+			return "ERROR", fmt.Errorf("hasher returned an error: %w", err)
 		}
-		sum = hex.EncodeToString(byteSum)
 	} else {
 		tr := accounting.Stats(ctx).NewCheckingTransfer(o)
 		defer func() {
@@ -999,11 +1003,15 @@ func hashSum(ctx context.Context, ht hash.Type, downloadFlag bool, o fs.Object)
 		}()
 
 		sum, err = o.Hash(ctx, ht)
+		if base64Encoded {
+			hexBytes, _ := hex.DecodeString(sum)
+			sum = base64.URLEncoding.EncodeToString(hexBytes)
+		}
 		if err == hash.ErrUnsupported {
-			return "", fmt.Errorf("Hash unsupported: %w", err)
+			return "", fmt.Errorf("hash unsupported: %w", err)
 		}
 		if err != nil {
-			return "", fmt.Errorf("Failed to get hash %v from backend: %v: %w", ht, err, err)
+			return "", fmt.Errorf("failed to get hash %v from backend: %w", ht, err)
 		}
 	}
 
@@ -1014,10 +1022,7 @@ func hashSum(ctx context.Context, ht hash.Type, downloadFlag bool, o fs.Object)
 // Updated to handle both standard hex encoding and base64
 // Updated to perform multiple hashes concurrently
 func HashLister(ctx context.Context, ht hash.Type, outputBase64 bool, downloadFlag bool, f fs.Fs, w io.Writer) error {
-	width := hash.Width(ht)
+	width := hash.Width(ht, outputBase64)
-	if outputBase64 {
-		width = base64.URLEncoding.EncodedLen(width / 2)
-	}
 	concurrencyControl := make(chan struct{}, fs.GetConfig(ctx).Transfers)
 	var wg sync.WaitGroup
 	err := ListFn(ctx, f, func(o fs.Object) {
@@ -1028,15 +1033,11 @@ func HashLister(ctx context.Context, ht hash.Type, outputBase64 bool, downloadFl
 				<-concurrencyControl
 				wg.Done()
 			}()
-			sum, err := hashSum(ctx, ht, downloadFlag, o)
+			sum, err := hashSum(ctx, ht, outputBase64, downloadFlag, o)
 			if err != nil {
 				fs.Errorf(o, "%v", fs.CountError(err))
 				return
 			}
-			if outputBase64 {
-				hexBytes, _ := hex.DecodeString(sum)
-				sum = base64.URLEncoding.EncodeToString(hexBytes)
-			}
 			syncFprintf(w, "%*s  %s\n", width, sum, o.Remote())
 		}()
 	})
@@ -1044,6 +1045,28 @@ func HashLister(ctx context.Context, ht hash.Type, outputBase64 bool, downloadFl
 	return err
 }
 
+// HashSumStream outputs a line compatible with md5sum to w based on the
+// input stream in and the hash type ht passed in. If outputBase64 is
+// set then the hash will be base64 instead of hexadecimal.
+func HashSumStream(ht hash.Type, outputBase64 bool, in io.ReadCloser, w io.Writer) error {
+	hasher, err := hash.NewMultiHasherTypes(hash.NewHashSet(ht))
+	if err != nil {
+		return fmt.Errorf("hash unsupported: %w", err)
+	}
+	written, err := io.Copy(hasher, in)
+	fs.Debugf(nil, "Creating %s hash of %d bytes read from input stream", ht, written)
+	if err != nil {
+		return fmt.Errorf("failed to copy input to hasher: %w", err)
+	}
+	sum, err := hasher.SumString(ht, outputBase64)
+	if err != nil {
+		return fmt.Errorf("hasher returned an error: %w", err)
+	}
+	width := hash.Width(ht, outputBase64)
+	syncFprintf(w, "%*s  -\n", width, sum)
+	return nil
+}
+
 // Count counts the objects and their sizes in the Fs
 //
 // Obeys includes and excludes