drive: prevent retries when query filter returned no entries

https://forum.rclone.org/t/performance-degradation-between-v1-56-2-and-v1-57-0-when-copying-to-google-drive-using-max-age-min-age-and-fast-list/27580/2

backend/cache/cache.go

@@ -489,7 +489,7 @@ func NewFs(ctx context.Context, name, rootPath string, m configmap.Mapper) (fs.F
 		f.opt.TempWritePath = filepath.ToSlash(f.opt.TempWritePath)
 		f.tempFs, err = cache.Get(ctx, f.opt.TempWritePath)
 		if err != nil {
-			return nil, fmt.Errorf("failed to create temp fs: %v: %w", err, err)
+			return nil, fmt.Errorf("failed to create temp fs: %w", err)
 		}
 		fs.Infof(name, "Upload Temp Rest Time: %v", f.opt.TempWaitTime)
 		fs.Infof(name, "Upload Temp FS: %v", f.opt.TempWritePath)

backend/crypt/cipher.go

@@ -7,6 +7,7 @@ import (
 	gocipher "crypto/cipher"
 	"crypto/rand"
 	"encoding/base32"
+	"encoding/base64"
 	"errors"
 	"fmt"
 	"io"
@@ -16,6 +17,7 @@ import (
 	"time"
 	"unicode/utf8"
 
+	"github.com/Max-Sum/base32768"
 	"github.com/rclone/rclone/backend/crypt/pkcs7"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
@@ -114,6 +116,57 @@ func (mode NameEncryptionMode) String() (out string) {
 	return out
 }
 
+// fileNameEncoding are the encoding methods dealing with encrypted file names
+type fileNameEncoding interface {
+	EncodeToString(src []byte) string
+	DecodeString(s string) ([]byte, error)
+}
+
+// caseInsensitiveBase32Encoding defines a file name encoding
+// using a modified version of standard base32 as described in
+// RFC4648
+//
+// The standard encoding is modified in two ways
+//  * it becomes lower case (no-one likes upper case filenames!)
+//  * we strip the padding character `=`
+type caseInsensitiveBase32Encoding struct{}
+
+// EncodeToString encodes a strign using the modified version of
+// base32 encoding.
+func (caseInsensitiveBase32Encoding) EncodeToString(src []byte) string {
+	encoded := base32.HexEncoding.EncodeToString(src)
+	encoded = strings.TrimRight(encoded, "=")
+	return strings.ToLower(encoded)
+}
+
+// DecodeString decodes a string as encoded by EncodeToString
+func (caseInsensitiveBase32Encoding) DecodeString(s string) ([]byte, error) {
+	if strings.HasSuffix(s, "=") {
+		return nil, ErrorBadBase32Encoding
+	}
+	// First figure out how many padding characters to add
+	roundUpToMultipleOf8 := (len(s) + 7) &^ 7
+	equals := roundUpToMultipleOf8 - len(s)
+	s = strings.ToUpper(s) + "========"[:equals]
+	return base32.HexEncoding.DecodeString(s)
+}
+
+// NewNameEncoding creates a NameEncoding from a string
+func NewNameEncoding(s string) (enc fileNameEncoding, err error) {
+	s = strings.ToLower(s)
+	switch s {
+	case "base32":
+		enc = caseInsensitiveBase32Encoding{}
+	case "base64":
+		enc = base64.RawURLEncoding
+	case "base32768":
+		enc = base32768.SafeEncoding
+	default:
+		err = fmt.Errorf("Unknown file name encoding mode %q", s)
+	}
+	return enc, err
+}
+
 // Cipher defines an encoding and decoding cipher for the crypt backend
 type Cipher struct {
 	dataKey        [32]byte                  // Key for secretbox
@@ -121,15 +174,17 @@ type Cipher struct {
 	nameTweak      [nameCipherBlockSize]byte // used to tweak the name crypto
 	block          gocipher.Block
 	mode           NameEncryptionMode
+	fileNameEnc    fileNameEncoding
 	buffers        sync.Pool // encrypt/decrypt buffers
 	cryptoRand     io.Reader // read crypto random numbers from here
 	dirNameEncrypt bool
 }
 
 // newCipher initialises the cipher.  If salt is "" then it uses a built in salt val
-func newCipher(mode NameEncryptionMode, password, salt string, dirNameEncrypt bool) (*Cipher, error) {
+func newCipher(mode NameEncryptionMode, password, salt string, dirNameEncrypt bool, enc fileNameEncoding) (*Cipher, error) {
 	c := &Cipher{
 		mode:           mode,
+		fileNameEnc:    enc,
 		cryptoRand:     rand.Reader,
 		dirNameEncrypt: dirNameEncrypt,
 	}
@@ -187,30 +242,6 @@ func (c *Cipher) putBlock(buf []byte) {
 	c.buffers.Put(buf)
 }
 
-// encodeFileName encodes a filename using a modified version of
-// standard base32 as described in RFC4648
-//
-// The standard encoding is modified in two ways
-//  * it becomes lower case (no-one likes upper case filenames!)
-//  * we strip the padding character `=`
-func encodeFileName(in []byte) string {
-	encoded := base32.HexEncoding.EncodeToString(in)
-	encoded = strings.TrimRight(encoded, "=")
-	return strings.ToLower(encoded)
-}
-
-// decodeFileName decodes a filename as encoded by encodeFileName
-func decodeFileName(in string) ([]byte, error) {
-	if strings.HasSuffix(in, "=") {
-		return nil, ErrorBadBase32Encoding
-	}
-	// First figure out how many padding characters to add
-	roundUpToMultipleOf8 := (len(in) + 7) &^ 7
-	equals := roundUpToMultipleOf8 - len(in)
-	in = strings.ToUpper(in) + "========"[:equals]
-	return base32.HexEncoding.DecodeString(in)
-}
-
 // encryptSegment encrypts a path segment
 //
 // This uses EME with AES
@@ -231,7 +262,7 @@ func (c *Cipher) encryptSegment(plaintext string) string {
 	}
 	paddedPlaintext := pkcs7.Pad(nameCipherBlockSize, []byte(plaintext))
 	ciphertext := eme.Transform(c.block, c.nameTweak[:], paddedPlaintext, eme.DirectionEncrypt)
-	return encodeFileName(ciphertext)
+	return c.fileNameEnc.EncodeToString(ciphertext)
 }
 
 // decryptSegment decrypts a path segment
@@ -239,7 +270,7 @@ func (c *Cipher) decryptSegment(ciphertext string) (string, error) {
 	if ciphertext == "" {
 		return "", nil
 	}
-	rawCiphertext, err := decodeFileName(ciphertext)
+	rawCiphertext, err := c.fileNameEnc.DecodeString(ciphertext)
 	if err != nil {
 		return "", err
 	}

backend/crypt/cipher_test.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/base32"
+	"encoding/base64"
 	"errors"
 	"fmt"
 	"io"
@@ -11,6 +12,7 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/Max-Sum/base32768"
 	"github.com/rclone/rclone/backend/crypt/pkcs7"
 	"github.com/rclone/rclone/lib/readers"
 	"github.com/stretchr/testify/assert"
@@ -45,11 +47,31 @@ func TestNewNameEncryptionModeString(t *testing.T) {
 	assert.Equal(t, NameEncryptionMode(3).String(), "Unknown mode #3")
 }
 
-func TestEncodeFileName(t *testing.T) {
-	for _, test := range []struct {
-		in       string
-		expected string
-	}{
+type EncodingTestCase struct {
+	in       string
+	expected string
+}
+
+func testEncodeFileName(t *testing.T, encoding string, testCases []EncodingTestCase, caseInsensitive bool) {
+	for _, test := range testCases {
+		enc, err := NewNameEncoding(encoding)
+		assert.NoError(t, err, "There should be no error creating name encoder for base32.")
+		actual := enc.EncodeToString([]byte(test.in))
+		assert.Equal(t, actual, test.expected, fmt.Sprintf("in=%q", test.in))
+		recovered, err := enc.DecodeString(test.expected)
+		assert.NoError(t, err)
+		assert.Equal(t, string(recovered), test.in, fmt.Sprintf("reverse=%q", test.expected))
+		if caseInsensitive {
+			in := strings.ToUpper(test.expected)
+			recovered, err = enc.DecodeString(in)
+			assert.NoError(t, err)
+			assert.Equal(t, string(recovered), test.in, fmt.Sprintf("reverse=%q", in))
+		}
+	}
+}
+
+func TestEncodeFileNameBase32(t *testing.T) {
+	testEncodeFileName(t, "base32", []EncodingTestCase{
 		{"", ""},
 		{"1", "64"},
 		{"12", "64p0"},
@@ -67,20 +89,56 @@ func TestEncodeFileName(t *testing.T) {
 		{"12345678901234", "64p36d1l6orjge9g64p36d0"},
 		{"123456789012345", "64p36d1l6orjge9g64p36d1l"},
 		{"1234567890123456", "64p36d1l6orjge9g64p36d1l6o"},
-	} {
-		actual := encodeFileName([]byte(test.in))
-		assert.Equal(t, actual, test.expected, fmt.Sprintf("in=%q", test.in))
-		recovered, err := decodeFileName(test.expected)
-		assert.NoError(t, err)
-		assert.Equal(t, string(recovered), test.in, fmt.Sprintf("reverse=%q", test.expected))
-		in := strings.ToUpper(test.expected)
-		recovered, err = decodeFileName(in)
-		assert.NoError(t, err)
-		assert.Equal(t, string(recovered), test.in, fmt.Sprintf("reverse=%q", in))
-	}
+	}, true)
 }
 
-func TestDecodeFileName(t *testing.T) {
+func TestEncodeFileNameBase64(t *testing.T) {
+	testEncodeFileName(t, "base64", []EncodingTestCase{
+		{"", ""},
+		{"1", "MQ"},
+		{"12", "MTI"},
+		{"123", "MTIz"},
+		{"1234", "MTIzNA"},
+		{"12345", "MTIzNDU"},
+		{"123456", "MTIzNDU2"},
+		{"1234567", "MTIzNDU2Nw"},
+		{"12345678", "MTIzNDU2Nzg"},
+		{"123456789", "MTIzNDU2Nzg5"},
+		{"1234567890", "MTIzNDU2Nzg5MA"},
+		{"12345678901", "MTIzNDU2Nzg5MDE"},
+		{"123456789012", "MTIzNDU2Nzg5MDEy"},
+		{"1234567890123", "MTIzNDU2Nzg5MDEyMw"},
+		{"12345678901234", "MTIzNDU2Nzg5MDEyMzQ"},
+		{"123456789012345", "MTIzNDU2Nzg5MDEyMzQ1"},
+		{"1234567890123456", "MTIzNDU2Nzg5MDEyMzQ1Ng"},
+	}, false)
+}
+
+func TestEncodeFileNameBase32768(t *testing.T) {
+	testEncodeFileName(t, "base32768", []EncodingTestCase{
+		{"", ""},
+		{"1", "㼿"},
+		{"12", "㻙ɟ"},
+		{"123", "㻙ⲿ"},
+		{"1234", "㻙ⲍƟ"},
+		{"12345", "㻙ⲍ⍟"},
+		{"123456", "㻙ⲍ⍆ʏ"},
+		{"1234567", "㻙ⲍ⍆觟"},
+		{"12345678", "㻙ⲍ⍆觓ɧ"},
+		{"123456789", "㻙ⲍ⍆觓栯"},
+		{"1234567890", "㻙ⲍ⍆觓栩ɣ"},
+		{"12345678901", "㻙ⲍ⍆觓栩朧"},
+		{"123456789012", "㻙ⲍ⍆觓栩朤ʅ"},
+		{"1234567890123", "㻙ⲍ⍆觓栩朤談"},
+		{"12345678901234", "㻙ⲍ⍆觓栩朤諆ɔ"},
+		{"123456789012345", "㻙ⲍ⍆觓栩朤諆媕"},
+		{"1234567890123456", "㻙ⲍ⍆觓栩朤諆媕䆿"},
+	}, false)
+}
+
+func TestDecodeFileNameBase32(t *testing.T) {
+	enc, err := NewNameEncoding("base32")
+	assert.NoError(t, err, "There should be no error creating name encoder for base32.")
 	// We've tested decoding the valid ones above, now concentrate on the invalid ones
 	for _, test := range []struct {
 		in          string
@@ -90,17 +148,65 @@ func TestDecodeFileName(t *testing.T) {
 		{"!", base32.CorruptInputError(0)},
 		{"hello=hello", base32.CorruptInputError(5)},
 	} {
-		actual, actualErr := decodeFileName(test.in)
+		actual, actualErr := enc.DecodeString(test.in)
 		assert.Equal(t, test.expectedErr, actualErr, fmt.Sprintf("in=%q got actual=%q, err = %v %T", test.in, actual, actualErr, actualErr))
 	}
 }
 
-func TestEncryptSegment(t *testing.T) {
-	c, _ := newCipher(NameEncryptionStandard, "", "", true)
+func TestDecodeFileNameBase64(t *testing.T) {
+	enc, err := NewNameEncoding("base64")
+	assert.NoError(t, err, "There should be no error creating name encoder for base32.")
+	// We've tested decoding the valid ones above, now concentrate on the invalid ones
 	for _, test := range []struct {
-		in       string
-		expected string
+		in          string
+		expectedErr error
 	}{
+		{"64=", base64.CorruptInputError(2)},
+		{"!", base64.CorruptInputError(0)},
+		{"Hello=Hello", base64.CorruptInputError(5)},
+	} {
+		actual, actualErr := enc.DecodeString(test.in)
+		assert.Equal(t, test.expectedErr, actualErr, fmt.Sprintf("in=%q got actual=%q, err = %v %T", test.in, actual, actualErr, actualErr))
+	}
+}
+
+func TestDecodeFileNameBase32768(t *testing.T) {
+	enc, err := NewNameEncoding("base32768")
+	assert.NoError(t, err, "There should be no error creating name encoder for base32.")
+	// We've tested decoding the valid ones above, now concentrate on the invalid ones
+	for _, test := range []struct {
+		in          string
+		expectedErr error
+	}{
+		{"㼿c", base32768.CorruptInputError(1)},
+		{"!", base32768.CorruptInputError(0)},
+		{"㻙ⲿ=㻙ⲿ", base32768.CorruptInputError(2)},
+	} {
+		actual, actualErr := enc.DecodeString(test.in)
+		assert.Equal(t, test.expectedErr, actualErr, fmt.Sprintf("in=%q got actual=%q, err = %v %T", test.in, actual, actualErr, actualErr))
+	}
+}
+
+func testEncryptSegment(t *testing.T, encoding string, testCases []EncodingTestCase, caseInsensitive bool) {
+	enc, _ := NewNameEncoding(encoding)
+	c, _ := newCipher(NameEncryptionStandard, "", "", true, enc)
+	for _, test := range testCases {
+		actual := c.encryptSegment(test.in)
+		assert.Equal(t, test.expected, actual, fmt.Sprintf("Testing %q", test.in))
+		recovered, err := c.decryptSegment(test.expected)
+		assert.NoError(t, err, fmt.Sprintf("Testing reverse %q", test.expected))
+		assert.Equal(t, test.in, recovered, fmt.Sprintf("Testing reverse %q", test.expected))
+		if caseInsensitive {
+			in := strings.ToUpper(test.expected)
+			recovered, err = c.decryptSegment(in)
+			assert.NoError(t, err, fmt.Sprintf("Testing reverse %q", in))
+			assert.Equal(t, test.in, recovered, fmt.Sprintf("Testing reverse %q", in))
+		}
+	}
+}
+
+func TestEncryptSegmentBase32(t *testing.T) {
+	testEncryptSegment(t, "base32", []EncodingTestCase{
 		{"", ""},
 		{"1", "p0e52nreeaj0a5ea7s64m4j72s"},
 		{"12", "l42g6771hnv3an9cgc8cr2n1ng"},
@@ -118,26 +224,61 @@ func TestEncryptSegment(t *testing.T) {
 		{"12345678901234", "moq0uqdlqrblrc5pa5u5c7hq9g"},
 		{"123456789012345", "eeam3li4rnommi3a762h5n7meg"},
 		{"1234567890123456", "mijbj0frqf6ms7frcr6bd9h0env53jv96pjaaoirk7forcgpt70g"},
-	} {
-		actual := c.encryptSegment(test.in)
-		assert.Equal(t, test.expected, actual, fmt.Sprintf("Testing %q", test.in))
-		recovered, err := c.decryptSegment(test.expected)
-		assert.NoError(t, err, fmt.Sprintf("Testing reverse %q", test.expected))
-		assert.Equal(t, test.in, recovered, fmt.Sprintf("Testing reverse %q", test.expected))
-		in := strings.ToUpper(test.expected)
-		recovered, err = c.decryptSegment(in)
-		assert.NoError(t, err, fmt.Sprintf("Testing reverse %q", in))
-		assert.Equal(t, test.in, recovered, fmt.Sprintf("Testing reverse %q", in))
-	}
+	}, true)
 }
 
-func TestDecryptSegment(t *testing.T) {
+func TestEncryptSegmentBase64(t *testing.T) {
+	testEncryptSegment(t, "base64", []EncodingTestCase{
+		{"", ""},
+		{"1", "yBxRX25ypgUVyj8MSxJnFw"},
+		{"12", "qQUDHOGN_jVdLIMQzYrhvA"},
+		{"123", "1CxFf2Mti1xIPYlGruDh-A"},
+		{"1234", "RL-xOTmsxsG7kuTy2XJUxw"},
+		{"12345", "3FP_GHoeBJdq0yLgaED8IQ"},
+		{"123456", "Xc4T1Gqrs3OVYnrE6dpEWQ"},
+		{"1234567", "uZeEzssOnDWHEOzLqjwpog"},
+		{"12345678", "8noiTP5WkkbEuijsPhOpxQ"},
+		{"123456789", "GeNxgLA0wiaGAKU3U7qL4Q"},
+		{"1234567890", "x1DUhdmqoVWYVBLD3dha-A"},
+		{"12345678901", "iEyP_3BZR6vvv_2WM6NbZw"},
+		{"123456789012", "4OPGvS4SZdjvS568APUaFw"},
+		{"1234567890123", "Y8c5Wr8OhYYUo7fPwdojdg"},
+		{"12345678901234", "tjQPabXW112wuVF8Vh46TA"},
+		{"123456789012345", "c5Vh1kTd8WtIajmFEtz2dA"},
+		{"1234567890123456", "tKa5gfvTzW4d-2bMtqYgdf5Rz-k2ZqViW6HfjbIZ6cE"},
+	}, false)
+}
+
+func TestEncryptSegmentBase32768(t *testing.T) {
+	testEncryptSegment(t, "base32768", []EncodingTestCase{
+		{"", ""},
+		{"1", "詮㪗鐮僀伎作㻖㢧⪟"},
+		{"12", "竢朧䉱虃光塬䟛⣡蓟"},
+		{"123", "遶㞟鋅缕袡鲅ⵝ蝁ꌟ"},
+		{"1234", "䢟銮䵵狌㐜燳谒颴詟"},
+		{"12345", "钉Ꞇ㖃蚩憶狫朰杜㜿"},
+		{"123456", "啇ᚵⵕ憗䋫➫➓肤卟"},
+		{"1234567", "茫螓翁連劘樓㶔抉矟"},
+		{"12345678", "龝☳䘊辄岅較络㧩襟"},
+		{"123456789", "ⲱ苀㱆犂媐Ꮤ锇惫靟"},
+		{"1234567890", "計宁憕偵匢皫╛纺ꌟ"},
+		{"12345678901", "檆䨿鑫㪺藝ꡖ勇䦛婟"},
+		{"123456789012", "雑頏䰂䲝淚哚鹡魺⪟"},
+		{"1234567890123", "塃璶繁躸圅㔟䗃肃懟"},
+		{"12345678901234", "腺ᕚ崚鏕鏥讥鼌䑺䲿"},
+		{"123456789012345", "怪绕滻蕶肣但⠥荖惟"},
+		{"1234567890123456", "肳哀旚挶靏鏻㾭䱠慟㪳ꏆ賊兲铧敻塹魀ʟ"},
+	}, false)
+}
+
+func TestDecryptSegmentBase32(t *testing.T) {
 	// We've tested the forwards above, now concentrate on the errors
 	longName := make([]byte, 3328)
 	for i := range longName {
 		longName[i] = 'a'
 	}
-	c, _ := newCipher(NameEncryptionStandard, "", "", true)
+	enc, _ := NewNameEncoding("base32")
+	c, _ := newCipher(NameEncryptionStandard, "", "", true, enc)
 	for _, test := range []struct {
 		in          string
 		expectedErr error
@@ -145,118 +286,371 @@ func TestDecryptSegment(t *testing.T) {
 		{"64=", ErrorBadBase32Encoding},
 		{"!", base32.CorruptInputError(0)},
 		{string(longName), ErrorTooLongAfterDecode},
-		{encodeFileName([]byte("a")), ErrorNotAMultipleOfBlocksize},
-		{encodeFileName([]byte("123456789abcdef")), ErrorNotAMultipleOfBlocksize},
-		{encodeFileName([]byte("123456789abcdef0")), pkcs7.ErrorPaddingTooLong},
+		{enc.EncodeToString([]byte("a")), ErrorNotAMultipleOfBlocksize},
+		{enc.EncodeToString([]byte("123456789abcdef")), ErrorNotAMultipleOfBlocksize},
+		{enc.EncodeToString([]byte("123456789abcdef0")), pkcs7.ErrorPaddingTooLong},
 	} {
 		actual, actualErr := c.decryptSegment(test.in)
 		assert.Equal(t, test.expectedErr, actualErr, fmt.Sprintf("in=%q got actual=%q, err = %v %T", test.in, actual, actualErr, actualErr))
 	}
 }
 
-func TestEncryptFileName(t *testing.T) {
+func TestDecryptSegmentBase64(t *testing.T) {
+	// We've tested the forwards above, now concentrate on the errors
+	longName := make([]byte, 2816)
+	for i := range longName {
+		longName[i] = 'a'
+	}
+	enc, _ := NewNameEncoding("base64")
+	c, _ := newCipher(NameEncryptionStandard, "", "", true, enc)
+	for _, test := range []struct {
+		in          string
+		expectedErr error
+	}{
+		{"6H=", base64.CorruptInputError(2)},
+		{"!", base64.CorruptInputError(0)},
+		{string(longName), ErrorTooLongAfterDecode},
+		{enc.EncodeToString([]byte("a")), ErrorNotAMultipleOfBlocksize},
+		{enc.EncodeToString([]byte("123456789abcdef")), ErrorNotAMultipleOfBlocksize},
+		{enc.EncodeToString([]byte("123456789abcdef0")), pkcs7.ErrorPaddingTooLong},
+	} {
+		actual, actualErr := c.decryptSegment(test.in)
+		assert.Equal(t, test.expectedErr, actualErr, fmt.Sprintf("in=%q got actual=%q, err = %v %T", test.in, actual, actualErr, actualErr))
+	}
+}
+
+func TestDecryptSegmentBase32768(t *testing.T) {
+	// We've tested the forwards above, now concentrate on the errors
+	longName := strings.Repeat("怪", 1280)
+	enc, _ := NewNameEncoding("base32768")
+	c, _ := newCipher(NameEncryptionStandard, "", "", true, enc)
+	for _, test := range []struct {
+		in          string
+		expectedErr error
+	}{
+		{"怪=", base32768.CorruptInputError(1)},
+		{"!", base32768.CorruptInputError(0)},
+		{longName, ErrorTooLongAfterDecode},
+		{enc.EncodeToString([]byte("a")), ErrorNotAMultipleOfBlocksize},
+		{enc.EncodeToString([]byte("123456789abcdef")), ErrorNotAMultipleOfBlocksize},
+		{enc.EncodeToString([]byte("123456789abcdef0")), pkcs7.ErrorPaddingTooLong},
+	} {
+		actual, actualErr := c.decryptSegment(test.in)
+		assert.Equal(t, test.expectedErr, actualErr, fmt.Sprintf("in=%q got actual=%q, err = %v %T", test.in, actual, actualErr, actualErr))
+	}
+}
+
+func testStandardEncryptFileName(t *testing.T, encoding string, testCasesEncryptDir []EncodingTestCase, testCasesNoEncryptDir []EncodingTestCase) {
 	// First standard mode
-	c, _ := newCipher(NameEncryptionStandard, "", "", true)
-	assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s", c.EncryptFileName("1"))
-	assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng", c.EncryptFileName("1/12"))
-	assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng/qgm4avr35m5loi1th53ato71v0", c.EncryptFileName("1/12/123"))
-	assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s-v2001-02-03-040506-123", c.EncryptFileName("1-v2001-02-03-040506-123"))
-	assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng-v2001-02-03-040506-123", c.EncryptFileName("1/12-v2001-02-03-040506-123"))
+	enc, _ := NewNameEncoding(encoding)
+	c, _ := newCipher(NameEncryptionStandard, "", "", true, enc)
+	for _, test := range testCasesEncryptDir {
+		assert.Equal(t, test.expected, c.EncryptFileName(test.in))
+	}
 	// Standard mode with directory name encryption off
-	c, _ = newCipher(NameEncryptionStandard, "", "", false)
-	assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s", c.EncryptFileName("1"))
-	assert.Equal(t, "1/l42g6771hnv3an9cgc8cr2n1ng", c.EncryptFileName("1/12"))
-	assert.Equal(t, "1/12/qgm4avr35m5loi1th53ato71v0", c.EncryptFileName("1/12/123"))
-	assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s-v2001-02-03-040506-123", c.EncryptFileName("1-v2001-02-03-040506-123"))
-	assert.Equal(t, "1/l42g6771hnv3an9cgc8cr2n1ng-v2001-02-03-040506-123", c.EncryptFileName("1/12-v2001-02-03-040506-123"))
-	// Now off mode
-	c, _ = newCipher(NameEncryptionOff, "", "", true)
+	c, _ = newCipher(NameEncryptionStandard, "", "", false, enc)
+	for _, test := range testCasesNoEncryptDir {
+		assert.Equal(t, test.expected, c.EncryptFileName(test.in))
+	}
+}
+
+func TestStandardEncryptFileNameBase32(t *testing.T) {
+	testStandardEncryptFileName(t, "base32", []EncodingTestCase{
+		{"1", "p0e52nreeaj0a5ea7s64m4j72s"},
+		{"1/12", "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng"},
+		{"1/12/123", "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng/qgm4avr35m5loi1th53ato71v0"},
+		{"1-v2001-02-03-040506-123", "p0e52nreeaj0a5ea7s64m4j72s-v2001-02-03-040506-123"},
+		{"1/12-v2001-02-03-040506-123", "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng-v2001-02-03-040506-123"},
+	}, []EncodingTestCase{
+		{"1", "p0e52nreeaj0a5ea7s64m4j72s"},
+		{"1/12", "1/l42g6771hnv3an9cgc8cr2n1ng"},
+		{"1/12/123", "1/12/qgm4avr35m5loi1th53ato71v0"},
+		{"1-v2001-02-03-040506-123", "p0e52nreeaj0a5ea7s64m4j72s-v2001-02-03-040506-123"},
+		{"1/12-v2001-02-03-040506-123", "1/l42g6771hnv3an9cgc8cr2n1ng-v2001-02-03-040506-123"},
+	})
+}
+
+func TestStandardEncryptFileNameBase64(t *testing.T) {
+	testStandardEncryptFileName(t, "base64", []EncodingTestCase{
+		{"1", "yBxRX25ypgUVyj8MSxJnFw"},
+		{"1/12", "yBxRX25ypgUVyj8MSxJnFw/qQUDHOGN_jVdLIMQzYrhvA"},
+		{"1/12/123", "yBxRX25ypgUVyj8MSxJnFw/qQUDHOGN_jVdLIMQzYrhvA/1CxFf2Mti1xIPYlGruDh-A"},
+		{"1-v2001-02-03-040506-123", "yBxRX25ypgUVyj8MSxJnFw-v2001-02-03-040506-123"},
+		{"1/12-v2001-02-03-040506-123", "yBxRX25ypgUVyj8MSxJnFw/qQUDHOGN_jVdLIMQzYrhvA-v2001-02-03-040506-123"},
+	}, []EncodingTestCase{
+		{"1", "yBxRX25ypgUVyj8MSxJnFw"},
+		{"1/12", "1/qQUDHOGN_jVdLIMQzYrhvA"},
+		{"1/12/123", "1/12/1CxFf2Mti1xIPYlGruDh-A"},
+		{"1-v2001-02-03-040506-123", "yBxRX25ypgUVyj8MSxJnFw-v2001-02-03-040506-123"},
+		{"1/12-v2001-02-03-040506-123", "1/qQUDHOGN_jVdLIMQzYrhvA-v2001-02-03-040506-123"},
+	})
+}
+
+func TestStandardEncryptFileNameBase32768(t *testing.T) {
+	testStandardEncryptFileName(t, "base32768", []EncodingTestCase{
+		{"1", "詮㪗鐮僀伎作㻖㢧⪟"},
+		{"1/12", "詮㪗鐮僀伎作㻖㢧⪟/竢朧䉱虃光塬䟛⣡蓟"},
+		{"1/12/123", "詮㪗鐮僀伎作㻖㢧⪟/竢朧䉱虃光塬䟛⣡蓟/遶㞟鋅缕袡鲅ⵝ蝁ꌟ"},
+		{"1-v2001-02-03-040506-123", "詮㪗鐮僀伎作㻖㢧⪟-v2001-02-03-040506-123"},
+		{"1/12-v2001-02-03-040506-123", "詮㪗鐮僀伎作㻖㢧⪟/竢朧䉱虃光塬䟛⣡蓟-v2001-02-03-040506-123"},
+	}, []EncodingTestCase{
+		{"1", "詮㪗鐮僀伎作㻖㢧⪟"},
+		{"1/12", "1/竢朧䉱虃光塬䟛⣡蓟"},
+		{"1/12/123", "1/12/遶㞟鋅缕袡鲅ⵝ蝁ꌟ"},
+		{"1-v2001-02-03-040506-123", "詮㪗鐮僀伎作㻖㢧⪟-v2001-02-03-040506-123"},
+		{"1/12-v2001-02-03-040506-123", "1/竢朧䉱虃光塬䟛⣡蓟-v2001-02-03-040506-123"},
+	})
+}
+
+func TestNonStandardEncryptFileName(t *testing.T) {
+	// Off mode
+	c, _ := newCipher(NameEncryptionOff, "", "", true, nil)
 	assert.Equal(t, "1/12/123.bin", c.EncryptFileName("1/12/123"))
 	// Obfuscation mode
-	c, _ = newCipher(NameEncryptionObfuscated, "", "", true)
+	c, _ = newCipher(NameEncryptionObfuscated, "", "", true, nil)
 	assert.Equal(t, "49.6/99.23/150.890/53.!!lipps", c.EncryptFileName("1/12/123/!hello"))
 	assert.Equal(t, "49.6/99.23/150.890/53-v2001-02-03-040506-123.!!lipps", c.EncryptFileName("1/12/123/!hello-v2001-02-03-040506-123"))
 	assert.Equal(t, "49.6/99.23/150.890/162.uryyB-v2001-02-03-040506-123.GKG", c.EncryptFileName("1/12/123/hello-v2001-02-03-040506-123.txt"))
 	assert.Equal(t, "161.\u00e4", c.EncryptFileName("\u00a1"))
 	assert.Equal(t, "160.\u03c2", c.EncryptFileName("\u03a0"))
 	// Obfuscation mode with directory name encryption off
-	c, _ = newCipher(NameEncryptionObfuscated, "", "", false)
+	c, _ = newCipher(NameEncryptionObfuscated, "", "", false, nil)
 	assert.Equal(t, "1/12/123/53.!!lipps", c.EncryptFileName("1/12/123/!hello"))
 	assert.Equal(t, "1/12/123/53-v2001-02-03-040506-123.!!lipps", c.EncryptFileName("1/12/123/!hello-v2001-02-03-040506-123"))
 	assert.Equal(t, "161.\u00e4", c.EncryptFileName("\u00a1"))
 	assert.Equal(t, "160.\u03c2", c.EncryptFileName("\u03a0"))
 }
 
-func TestDecryptFileName(t *testing.T) {
-	for _, test := range []struct {
-		mode           NameEncryptionMode
-		dirNameEncrypt bool
-		in             string
-		expected       string
-		expectedErr    error
-	}{
-		{NameEncryptionStandard, true, "p0e52nreeaj0a5ea7s64m4j72s", "1", nil},
-		{NameEncryptionStandard, true, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng", "1/12", nil},
-		{NameEncryptionStandard, true, "p0e52nreeAJ0A5EA7S64M4J72S/L42G6771HNv3an9cgc8cr2n1ng", "1/12", nil},
-		{NameEncryptionStandard, true, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng/qgm4avr35m5loi1th53ato71v0", "1/12/123", nil},
-		{NameEncryptionStandard, true, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1/qgm4avr35m5loi1th53ato71v0", "", ErrorNotAMultipleOfBlocksize},
-		{NameEncryptionStandard, false, "1/12/qgm4avr35m5loi1th53ato71v0", "1/12/123", nil},
-		{NameEncryptionStandard, true, "p0e52nreeaj0a5ea7s64m4j72s-v2001-02-03-040506-123", "1-v2001-02-03-040506-123", nil},
-		{NameEncryptionOff, true, "1/12/123.bin", "1/12/123", nil},
-		{NameEncryptionOff, true, "1/12/123.bix", "", ErrorNotAnEncryptedFile},
-		{NameEncryptionOff, true, ".bin", "", ErrorNotAnEncryptedFile},
-		{NameEncryptionOff, true, "1/12/123-v2001-02-03-040506-123.bin", "1/12/123-v2001-02-03-040506-123", nil},
-		{NameEncryptionOff, true, "1/12/123-v1970-01-01-010101-123-v2001-02-03-040506-123.bin", "1/12/123-v1970-01-01-010101-123-v2001-02-03-040506-123", nil},
-		{NameEncryptionOff, true, "1/12/123-v1970-01-01-010101-123-v2001-02-03-040506-123.txt.bin", "1/12/123-v1970-01-01-010101-123-v2001-02-03-040506-123.txt", nil},
-		{NameEncryptionObfuscated, true, "!.hello", "hello", nil},
-		{NameEncryptionObfuscated, true, "hello", "", ErrorNotAnEncryptedFile},
-		{NameEncryptionObfuscated, true, "161.\u00e4", "\u00a1", nil},
-		{NameEncryptionObfuscated, true, "160.\u03c2", "\u03a0", nil},
-		{NameEncryptionObfuscated, false, "1/12/123/53.!!lipps", "1/12/123/!hello", nil},
-		{NameEncryptionObfuscated, false, "1/12/123/53-v2001-02-03-040506-123.!!lipps", "1/12/123/!hello-v2001-02-03-040506-123", nil},
-	} {
-		c, _ := newCipher(test.mode, "", "", test.dirNameEncrypt)
+func testStandardDecryptFileName(t *testing.T, encoding string, testCases []EncodingTestCase, caseInsensitive bool) {
+	enc, _ := NewNameEncoding(encoding)
+	for _, test := range testCases {
+		// Test when dirNameEncrypt=true
+		c, _ := newCipher(NameEncryptionStandard, "", "", true, enc)
 		actual, actualErr := c.DecryptFileName(test.in)
-		what := fmt.Sprintf("Testing %q (mode=%v)", test.in, test.mode)
-		assert.Equal(t, test.expected, actual, what)
-		assert.Equal(t, test.expectedErr, actualErr, what)
+		assert.NoError(t, actualErr)
+		assert.Equal(t, test.expected, actual)
+		if caseInsensitive {
+			c, _ := newCipher(NameEncryptionStandard, "", "", true, enc)
+			actual, actualErr := c.DecryptFileName(strings.ToUpper(test.in))
+			assert.NoError(t, actualErr)
+			assert.Equal(t, test.expected, actual)
+		}
+		// Add a character should raise ErrorNotAMultipleOfBlocksize
+		actual, actualErr = c.DecryptFileName(enc.EncodeToString([]byte("1")) + test.in)
+		assert.Equal(t, ErrorNotAMultipleOfBlocksize, actualErr)
+		assert.Equal(t, "", actual)
+		// Test when dirNameEncrypt=false
+		noDirEncryptIn := test.in
+		if strings.LastIndex(test.expected, "/") != -1 {
+			noDirEncryptIn = test.expected[:strings.LastIndex(test.expected, "/")] + test.in[strings.LastIndex(test.in, "/"):]
+		}
+		c, _ = newCipher(NameEncryptionStandard, "", "", false, enc)
+		actual, actualErr = c.DecryptFileName(noDirEncryptIn)
+		assert.NoError(t, actualErr)
+		assert.Equal(t, test.expected, actual)
+	}
+}
+
+func TestStandardDecryptFileNameBase32(t *testing.T) {
+	testStandardDecryptFileName(t, "base32", []EncodingTestCase{
+		{"p0e52nreeaj0a5ea7s64m4j72s", "1"},
+		{"p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng", "1/12"},
+		{"p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng/qgm4avr35m5loi1th53ato71v0", "1/12/123"},
+	}, true)
+}
+
+func TestStandardDecryptFileNameBase64(t *testing.T) {
+	testStandardDecryptFileName(t, "base64", []EncodingTestCase{
+		{"yBxRX25ypgUVyj8MSxJnFw", "1"},
+		{"yBxRX25ypgUVyj8MSxJnFw/qQUDHOGN_jVdLIMQzYrhvA", "1/12"},
+		{"yBxRX25ypgUVyj8MSxJnFw/qQUDHOGN_jVdLIMQzYrhvA/1CxFf2Mti1xIPYlGruDh-A", "1/12/123"},
+	}, false)
+}
+
+func TestStandardDecryptFileNameBase32768(t *testing.T) {
+	testStandardDecryptFileName(t, "base32768", []EncodingTestCase{
+		{"詮㪗鐮僀伎作㻖㢧⪟", "1"},
+		{"詮㪗鐮僀伎作㻖㢧⪟/竢朧䉱虃光塬䟛⣡蓟", "1/12"},
+		{"詮㪗鐮僀伎作㻖㢧⪟/竢朧䉱虃光塬䟛⣡蓟/遶㞟鋅缕袡鲅ⵝ蝁ꌟ", "1/12/123"},
+	}, false)
+}
+
+func TestNonStandardDecryptFileName(t *testing.T) {
+	for _, encoding := range []string{"base32", "base64", "base32768"} {
+		enc, _ := NewNameEncoding(encoding)
+		for _, test := range []struct {
+			mode           NameEncryptionMode
+			dirNameEncrypt bool
+			in             string
+			expected       string
+			expectedErr    error
+		}{
+			{NameEncryptionOff, true, "1/12/123.bin", "1/12/123", nil},
+			{NameEncryptionOff, true, "1/12/123.bix", "", ErrorNotAnEncryptedFile},
+			{NameEncryptionOff, true, ".bin", "", ErrorNotAnEncryptedFile},
+			{NameEncryptionOff, true, "1/12/123-v2001-02-03-040506-123.bin", "1/12/123-v2001-02-03-040506-123", nil},
+			{NameEncryptionOff, true, "1/12/123-v1970-01-01-010101-123-v2001-02-03-040506-123.bin", "1/12/123-v1970-01-01-010101-123-v2001-02-03-040506-123", nil},
+			{NameEncryptionOff, true, "1/12/123-v1970-01-01-010101-123-v2001-02-03-040506-123.txt.bin", "1/12/123-v1970-01-01-010101-123-v2001-02-03-040506-123.txt", nil},
+			{NameEncryptionObfuscated, true, "!.hello", "hello", nil},
+			{NameEncryptionObfuscated, true, "hello", "", ErrorNotAnEncryptedFile},
+			{NameEncryptionObfuscated, true, "161.\u00e4", "\u00a1", nil},
+			{NameEncryptionObfuscated, true, "160.\u03c2", "\u03a0", nil},
+			{NameEncryptionObfuscated, false, "1/12/123/53.!!lipps", "1/12/123/!hello", nil},
+			{NameEncryptionObfuscated, false, "1/12/123/53-v2001-02-03-040506-123.!!lipps", "1/12/123/!hello-v2001-02-03-040506-123", nil},
+		} {
+			c, _ := newCipher(test.mode, "", "", test.dirNameEncrypt, enc)
+			actual, actualErr := c.DecryptFileName(test.in)
+			what := fmt.Sprintf("Testing %q (mode=%v)", test.in, test.mode)
+			assert.Equal(t, test.expected, actual, what)
+			assert.Equal(t, test.expectedErr, actualErr, what)
+		}
 	}
 }
 
 func TestEncDecMatches(t *testing.T) {
-	for _, test := range []struct {
-		mode NameEncryptionMode
-		in   string
-	}{
-		{NameEncryptionStandard, "1/2/3/4"},
-		{NameEncryptionOff, "1/2/3/4"},
-		{NameEncryptionObfuscated, "1/2/3/4/!hello\u03a0"},
-		{NameEncryptionObfuscated, "Avatar The Last Airbender"},
-	} {
-		c, _ := newCipher(test.mode, "", "", true)
-		out, err := c.DecryptFileName(c.EncryptFileName(test.in))
-		what := fmt.Sprintf("Testing %q (mode=%v)", test.in, test.mode)
-		assert.Equal(t, out, test.in, what)
-		assert.Equal(t, err, nil, what)
+	for _, encoding := range []string{"base32", "base64", "base32768"} {
+		enc, _ := NewNameEncoding(encoding)
+		for _, test := range []struct {
+			mode NameEncryptionMode
+			in   string
+		}{
+			{NameEncryptionStandard, "1/2/3/4"},
+			{NameEncryptionOff, "1/2/3/4"},
+			{NameEncryptionObfuscated, "1/2/3/4/!hello\u03a0"},
+			{NameEncryptionObfuscated, "Avatar The Last Airbender"},
+		} {
+			c, _ := newCipher(test.mode, "", "", true, enc)
+			out, err := c.DecryptFileName(c.EncryptFileName(test.in))
+			what := fmt.Sprintf("Testing %q (mode=%v)", test.in, test.mode)
+			assert.Equal(t, out, test.in, what)
+			assert.Equal(t, err, nil, what)
+		}
 	}
 }
 
-func TestEncryptDirName(t *testing.T) {
+func testStandardEncryptDirName(t *testing.T, encoding string, testCases []EncodingTestCase) {
+	enc, _ := NewNameEncoding(encoding)
+	c, _ := newCipher(NameEncryptionStandard, "", "", true, enc)
 	// First standard mode
-	c, _ := newCipher(NameEncryptionStandard, "", "", true)
-	assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s", c.EncryptDirName("1"))
-	assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng", c.EncryptDirName("1/12"))
-	assert.Equal(t, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng/qgm4avr35m5loi1th53ato71v0", c.EncryptDirName("1/12/123"))
-	// Standard mode with dir name encryption off
-	c, _ = newCipher(NameEncryptionStandard, "", "", false)
-	assert.Equal(t, "1/12", c.EncryptDirName("1/12"))
-	assert.Equal(t, "1/12/123", c.EncryptDirName("1/12/123"))
-	// Now off mode
-	c, _ = newCipher(NameEncryptionOff, "", "", true)
-	assert.Equal(t, "1/12/123", c.EncryptDirName("1/12/123"))
+	for _, test := range testCases {
+		assert.Equal(t, test.expected, c.EncryptDirName(test.in))
+	}
 }
 
-func TestDecryptDirName(t *testing.T) {
+func TestStandardEncryptDirNameBase32(t *testing.T) {
+	testStandardEncryptDirName(t, "base32", []EncodingTestCase{
+		{"1", "p0e52nreeaj0a5ea7s64m4j72s"},
+		{"1/12", "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng"},
+		{"1/12/123", "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng/qgm4avr35m5loi1th53ato71v0"},
+	})
+}
+
+func TestStandardEncryptDirNameBase64(t *testing.T) {
+	testStandardEncryptDirName(t, "base64", []EncodingTestCase{
+		{"1", "yBxRX25ypgUVyj8MSxJnFw"},
+		{"1/12", "yBxRX25ypgUVyj8MSxJnFw/qQUDHOGN_jVdLIMQzYrhvA"},
+		{"1/12/123", "yBxRX25ypgUVyj8MSxJnFw/qQUDHOGN_jVdLIMQzYrhvA/1CxFf2Mti1xIPYlGruDh-A"},
+	})
+}
+
+func TestStandardEncryptDirNameBase32768(t *testing.T) {
+	testStandardEncryptDirName(t, "base32768", []EncodingTestCase{
+		{"1", "詮㪗鐮僀伎作㻖㢧⪟"},
+		{"1/12", "詮㪗鐮僀伎作㻖㢧⪟/竢朧䉱虃光塬䟛⣡蓟"},
+		{"1/12/123", "詮㪗鐮僀伎作㻖㢧⪟/竢朧䉱虃光塬䟛⣡蓟/遶㞟鋅缕袡鲅ⵝ蝁ꌟ"},
+	})
+}
+
+func TestNonStandardEncryptDirName(t *testing.T) {
+	for _, encoding := range []string{"base32", "base64", "base32768"} {
+		enc, _ := NewNameEncoding(encoding)
+		c, _ := newCipher(NameEncryptionStandard, "", "", false, enc)
+		assert.Equal(t, "1/12", c.EncryptDirName("1/12"))
+		assert.Equal(t, "1/12/123", c.EncryptDirName("1/12/123"))
+		// Now off mode
+		c, _ = newCipher(NameEncryptionOff, "", "", true, enc)
+		assert.Equal(t, "1/12/123", c.EncryptDirName("1/12/123"))
+	}
+}
+
+func testStandardDecryptDirName(t *testing.T, encoding string, testCases []EncodingTestCase, caseInsensitive bool) {
+	enc, _ := NewNameEncoding(encoding)
+	for _, test := range testCases {
+		// Test dirNameEncrypt=true
+		c, _ := newCipher(NameEncryptionStandard, "", "", true, enc)
+		actual, actualErr := c.DecryptDirName(test.in)
+		assert.Equal(t, test.expected, actual)
+		assert.NoError(t, actualErr)
+		if caseInsensitive {
+			actual, actualErr := c.DecryptDirName(strings.ToUpper(test.in))
+			assert.Equal(t, actual, test.expected)
+			assert.NoError(t, actualErr)
+		}
+		actual, actualErr = c.DecryptDirName(enc.EncodeToString([]byte("1")) + test.in)
+		assert.Equal(t, "", actual)
+		assert.Equal(t, ErrorNotAMultipleOfBlocksize, actualErr)
+		// Test dirNameEncrypt=false
+		c, _ = newCipher(NameEncryptionStandard, "", "", false, enc)
+		actual, actualErr = c.DecryptDirName(test.in)
+		assert.Equal(t, test.in, actual)
+		assert.NoError(t, actualErr)
+		actual, actualErr = c.DecryptDirName(test.expected)
+		assert.Equal(t, test.expected, actual)
+		assert.NoError(t, actualErr)
+		// Test dirNameEncrypt=false
+	}
+}
+
+/*
+enc, _ := NewNameEncoding(encoding)
+for _, test := range []struct {
+	mode           NameEncryptionMode
+	dirNameEncrypt bool
+	in             string
+	expected       string
+	expectedErr    error
+}{
+	{NameEncryptionStandard, true, "p0e52nreeaj0a5ea7s64m4j72s", "1", nil},
+	{NameEncryptionStandard, true, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng", "1/12", nil},
+	{NameEncryptionStandard, true, "p0e52nreeAJ0A5EA7S64M4J72S/L42G6771HNv3an9cgc8cr2n1ng", "1/12", nil},
+	{NameEncryptionStandard, true, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng/qgm4avr35m5loi1th53ato71v0", "1/12/123", nil},
+	{NameEncryptionStandard, true, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1/qgm4avr35m5loi1th53ato71v0", "", ErrorNotAMultipleOfBlocksize},
+	{NameEncryptionStandard, false, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng", "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng", nil},
+	{NameEncryptionStandard, false, "1/12/123", "1/12/123", nil},
+} {
+	c, _ := newCipher(test.mode, "", "", test.dirNameEncrypt, enc)
+	actual, actualErr := c.DecryptDirName(test.in)
+	what := fmt.Sprintf("Testing %q (mode=%v)", test.in, test.mode)
+	assert.Equal(t, test.expected, actual, what)
+	assert.Equal(t, test.expectedErr, actualErr, what)
+}
+*/
+
+func TestStandardDecryptDirNameBase32(t *testing.T) {
+	testStandardDecryptDirName(t, "base32", []EncodingTestCase{
+		{"p0e52nreeaj0a5ea7s64m4j72s", "1"},
+		{"p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng", "1/12"},
+		{"p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng/qgm4avr35m5loi1th53ato71v0", "1/12/123"},
+	}, true)
+}
+
+func TestStandardDecryptDirNameBase64(t *testing.T) {
+	testStandardDecryptDirName(t, "base64", []EncodingTestCase{
+		{"yBxRX25ypgUVyj8MSxJnFw", "1"},
+		{"yBxRX25ypgUVyj8MSxJnFw/qQUDHOGN_jVdLIMQzYrhvA", "1/12"},
+		{"yBxRX25ypgUVyj8MSxJnFw/qQUDHOGN_jVdLIMQzYrhvA/1CxFf2Mti1xIPYlGruDh-A", "1/12/123"},
+	}, false)
+}
+
+func TestStandardDecryptDirNameBase32768(t *testing.T) {
+	testStandardDecryptDirName(t, "base32768", []EncodingTestCase{
+		{"詮㪗鐮僀伎作㻖㢧⪟", "1"},
+		{"詮㪗鐮僀伎作㻖㢧⪟/竢朧䉱虃光塬䟛⣡蓟", "1/12"},
+		{"詮㪗鐮僀伎作㻖㢧⪟/竢朧䉱虃光塬䟛⣡蓟/遶㞟鋅缕袡鲅ⵝ蝁ꌟ", "1/12/123"},
+	}, false)
+}
+
+func TestNonStandardDecryptDirName(t *testing.T) {
 	for _, test := range []struct {
 		mode           NameEncryptionMode
 		dirNameEncrypt bool
@@ -264,18 +658,11 @@ func TestDecryptDirName(t *testing.T) {
 		expected       string
 		expectedErr    error
 	}{
-		{NameEncryptionStandard, true, "p0e52nreeaj0a5ea7s64m4j72s", "1", nil},
-		{NameEncryptionStandard, true, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng", "1/12", nil},
-		{NameEncryptionStandard, true, "p0e52nreeAJ0A5EA7S64M4J72S/L42G6771HNv3an9cgc8cr2n1ng", "1/12", nil},
-		{NameEncryptionStandard, true, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng/qgm4avr35m5loi1th53ato71v0", "1/12/123", nil},
-		{NameEncryptionStandard, true, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1/qgm4avr35m5loi1th53ato71v0", "", ErrorNotAMultipleOfBlocksize},
-		{NameEncryptionStandard, false, "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng", "p0e52nreeaj0a5ea7s64m4j72s/l42g6771hnv3an9cgc8cr2n1ng", nil},
-		{NameEncryptionStandard, false, "1/12/123", "1/12/123", nil},
 		{NameEncryptionOff, true, "1/12/123.bin", "1/12/123.bin", nil},
 		{NameEncryptionOff, true, "1/12/123", "1/12/123", nil},
 		{NameEncryptionOff, true, ".bin", ".bin", nil},
 	} {
-		c, _ := newCipher(test.mode, "", "", test.dirNameEncrypt)
+		c, _ := newCipher(test.mode, "", "", test.dirNameEncrypt, nil)
 		actual, actualErr := c.DecryptDirName(test.in)
 		what := fmt.Sprintf("Testing %q (mode=%v)", test.in, test.mode)
 		assert.Equal(t, test.expected, actual, what)
@@ -284,7 +671,7 @@ func TestDecryptDirName(t *testing.T) {
 }
 
 func TestEncryptedSize(t *testing.T) {
-	c, _ := newCipher(NameEncryptionStandard, "", "", true)
+	c, _ := newCipher(NameEncryptionStandard, "", "", true, nil)
 	for _, test := range []struct {
 		in       int64
 		expected int64
@@ -308,7 +695,7 @@ func TestEncryptedSize(t *testing.T) {
 
 func TestDecryptedSize(t *testing.T) {
 	// Test the errors since we tested the reverse above
-	c, _ := newCipher(NameEncryptionStandard, "", "", true)
+	c, _ := newCipher(NameEncryptionStandard, "", "", true, nil)
 	for _, test := range []struct {
 		in          int64
 		expectedErr error
@@ -679,7 +1066,7 @@ func (z *zeroes) Read(p []byte) (n int, err error) {
 
 // Test encrypt decrypt with different buffer sizes
 func testEncryptDecrypt(t *testing.T, bufSize int, copySize int64) {
-	c, err := newCipher(NameEncryptionStandard, "", "", true)
+	c, err := newCipher(NameEncryptionStandard, "", "", true, nil)
 	assert.NoError(t, err)
 	c.cryptoRand = &zeroes{} // zero out the nonce
 	buf := make([]byte, bufSize)
@@ -749,7 +1136,7 @@ func TestEncryptData(t *testing.T) {
 		{[]byte{1}, file1},
 		{[]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}, file16},
 	} {
-		c, err := newCipher(NameEncryptionStandard, "", "", true)
+		c, err := newCipher(NameEncryptionStandard, "", "", true, nil)
 		assert.NoError(t, err)
 		c.cryptoRand = newRandomSource(1e8) // nodge the crypto rand generator
 
@@ -772,7 +1159,7 @@ func TestEncryptData(t *testing.T) {
 }
 
 func TestNewEncrypter(t *testing.T) {
-	c, err := newCipher(NameEncryptionStandard, "", "", true)
+	c, err := newCipher(NameEncryptionStandard, "", "", true, nil)
 	assert.NoError(t, err)
 	c.cryptoRand = newRandomSource(1e8) // nodge the crypto rand generator
 
@@ -788,13 +1175,12 @@ func TestNewEncrypter(t *testing.T) {
 	fh, err = c.newEncrypter(z, nil)
 	assert.Nil(t, fh)
 	assert.Error(t, err, "short read of nonce")
-
 }
 
 // Test the stream returning 0, io.ErrUnexpectedEOF - this used to
 // cause a fatal loop
 func TestNewEncrypterErrUnexpectedEOF(t *testing.T) {
-	c, err := newCipher(NameEncryptionStandard, "", "", true)
+	c, err := newCipher(NameEncryptionStandard, "", "", true, nil)
 	assert.NoError(t, err)
 
 	in := &readers.ErrorReader{Err: io.ErrUnexpectedEOF}
@@ -823,7 +1209,7 @@ func (c *closeDetector) Close() error {
 }
 
 func TestNewDecrypter(t *testing.T) {
-	c, err := newCipher(NameEncryptionStandard, "", "", true)
+	c, err := newCipher(NameEncryptionStandard, "", "", true, nil)
 	assert.NoError(t, err)
 	c.cryptoRand = newRandomSource(1e8) // nodge the crypto rand generator
 
@@ -866,7 +1252,7 @@ func TestNewDecrypter(t *testing.T) {
 
 // Test the stream returning 0, io.ErrUnexpectedEOF
 func TestNewDecrypterErrUnexpectedEOF(t *testing.T) {
-	c, err := newCipher(NameEncryptionStandard, "", "", true)
+	c, err := newCipher(NameEncryptionStandard, "", "", true, nil)
 	assert.NoError(t, err)
 
 	in2 := &readers.ErrorReader{Err: io.ErrUnexpectedEOF}
@@ -882,7 +1268,7 @@ func TestNewDecrypterErrUnexpectedEOF(t *testing.T) {
 }
 
 func TestNewDecrypterSeekLimit(t *testing.T) {
-	c, err := newCipher(NameEncryptionStandard, "", "", true)
+	c, err := newCipher(NameEncryptionStandard, "", "", true, nil)
 	assert.NoError(t, err)
 	c.cryptoRand = &zeroes{} // nodge the crypto rand generator
 
@@ -1088,7 +1474,7 @@ func TestDecrypterCalculateUnderlying(t *testing.T) {
 }
 
 func TestDecrypterRead(t *testing.T) {
-	c, err := newCipher(NameEncryptionStandard, "", "", true)
+	c, err := newCipher(NameEncryptionStandard, "", "", true, nil)
 	assert.NoError(t, err)
 
 	// Test truncating the file at each possible point
@@ -1152,7 +1538,7 @@ func TestDecrypterRead(t *testing.T) {
 }
 
 func TestDecrypterClose(t *testing.T) {
-	c, err := newCipher(NameEncryptionStandard, "", "", true)
+	c, err := newCipher(NameEncryptionStandard, "", "", true, nil)
 	assert.NoError(t, err)
 
 	cd := newCloseDetector(bytes.NewBuffer(file16))
@@ -1190,7 +1576,7 @@ func TestDecrypterClose(t *testing.T) {
 }
 
 func TestPutGetBlock(t *testing.T) {
-	c, err := newCipher(NameEncryptionStandard, "", "", true)
+	c, err := newCipher(NameEncryptionStandard, "", "", true, nil)
 	assert.NoError(t, err)
 
 	block := c.getBlock()
@@ -1201,7 +1587,7 @@ func TestPutGetBlock(t *testing.T) {
 }
 
 func TestKey(t *testing.T) {
-	c, err := newCipher(NameEncryptionStandard, "", "", true)
+	c, err := newCipher(NameEncryptionStandard, "", "", true, nil)
 	assert.NoError(t, err)
 
 	// Check zero keys OK

backend/crypt/crypt.go

@@ -116,6 +116,29 @@ names, or for debugging purposes.`,
 					Help:  "Encrypt file data.",
 				},
 			},
+		}, {
+			Name: "filename_encoding",
+			Help: `How to encode the encrypted filename to text string.
+
+This option could help with shortening the encrypted filename. The 
+suitable option would depend on the way your remote count the filename
+length and if it's case sensitve.`,
+			Default: "base32",
+			Examples: []fs.OptionExample{
+				{
+					Value: "base32",
+					Help:  "Encode using base32. Suitable for all remote.",
+				},
+				{
+					Value: "base64",
+					Help:  "Encode using base64. Suitable for case sensitive remote.",
+				},
+				{
+					Value: "base32768",
+					Help:  "Encode using base32768. Suitable if your remote counts UTF-16 or\nUnicode codepoint instead of UTF-8 byte length. (Eg. Onedrive)",
+				},
+			},
+			Advanced: true,
 		}},
 	})
 }
@@ -140,7 +163,11 @@ func newCipherForConfig(opt *Options) (*Cipher, error) {
 			return nil, fmt.Errorf("failed to decrypt password2: %w", err)
 		}
 	}
-	cipher, err := newCipher(mode, password, salt, opt.DirectoryNameEncryption)
+	enc, err := NewNameEncoding(opt.FilenameEncoding)
+	if err != nil {
+		return nil, err
+	}
+	cipher, err := newCipher(mode, password, salt, opt.DirectoryNameEncryption, enc)
 	if err != nil {
 		return nil, fmt.Errorf("failed to make cipher: %w", err)
 	}
@@ -229,6 +256,7 @@ type Options struct {
 	Password2               string `config:"password2"`
 	ServerSideAcrossConfigs bool   `config:"server_side_across_configs"`
 	ShowMapping             bool   `config:"show_mapping"`
+	FilenameEncoding        string `config:"filename_encoding"`
 }
 
 // Fs represents a wrapped fs.Fs

backend/crypt/crypt_test.go

@@ -29,7 +29,7 @@ func TestIntegration(t *testing.T) {
 }
 
 // TestStandard runs integration tests against the remote
-func TestStandard(t *testing.T) {
+func TestStandardBase32(t *testing.T) {
 	if *fstest.RemoteName != "" {
 		t.Skip("Skipping as -remote set")
 	}
@@ -49,6 +49,48 @@ func TestStandard(t *testing.T) {
 	})
 }
 
+func TestStandardBase64(t *testing.T) {
+	if *fstest.RemoteName != "" {
+		t.Skip("Skipping as -remote set")
+	}
+	tempdir := filepath.Join(os.TempDir(), "rclone-crypt-test-standard")
+	name := "TestCrypt"
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: name + ":",
+		NilObject:  (*crypt.Object)(nil),
+		ExtraConfig: []fstests.ExtraConfigItem{
+			{Name: name, Key: "type", Value: "crypt"},
+			{Name: name, Key: "remote", Value: tempdir},
+			{Name: name, Key: "password", Value: obscure.MustObscure("potato")},
+			{Name: name, Key: "filename_encryption", Value: "standard"},
+			{Name: name, Key: "filename_encoding", Value: "base64"},
+		},
+		UnimplementableFsMethods:     []string{"OpenWriterAt"},
+		UnimplementableObjectMethods: []string{"MimeType"},
+	})
+}
+
+func TestStandardBase32768(t *testing.T) {
+	if *fstest.RemoteName != "" {
+		t.Skip("Skipping as -remote set")
+	}
+	tempdir := filepath.Join(os.TempDir(), "rclone-crypt-test-standard")
+	name := "TestCrypt"
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: name + ":",
+		NilObject:  (*crypt.Object)(nil),
+		ExtraConfig: []fstests.ExtraConfigItem{
+			{Name: name, Key: "type", Value: "crypt"},
+			{Name: name, Key: "remote", Value: tempdir},
+			{Name: name, Key: "password", Value: obscure.MustObscure("potato")},
+			{Name: name, Key: "filename_encryption", Value: "standard"},
+			{Name: name, Key: "filename_encoding", Value: "base32768"},
+		},
+		UnimplementableFsMethods:     []string{"OpenWriterAt"},
+		UnimplementableObjectMethods: []string{"MimeType"},
+	})
+}
+
 // TestOff runs integration tests against the remote
 func TestOff(t *testing.T) {
 	if *fstest.RemoteName != "" {

backend/drive/drive.go

@@ -1693,6 +1693,11 @@ func (f *Fs) listRRunner(ctx context.Context, wg *sync.WaitGroup, in chan listRE
 	var paths []string
 	var grouping int32
 
+	usingQueryFilter := false
+	if fi, use := filter.GetConfig(ctx), filter.GetUseFilter(ctx); fi != nil && use {
+		usingQueryFilter = true
+	}
+
 	for dir := range in {
 		dirs = append(dirs[:0], dir.id)
 		paths = append(paths[:0], dir.path)
@@ -1765,7 +1770,8 @@ func (f *Fs) listRRunner(ctx context.Context, wg *sync.WaitGroup, in chan listRE
 		// drive where (A in parents) or (B in parents) returns nothing
 		// sometimes. See #3114, #4289 and
 		// https://issuetracker.google.com/issues/149522397
-		if len(dirs) > 1 && !foundItems {
+		// However, empty result is legitimate if query filter was applied.
+		if len(dirs) > 1 && !foundItems && !usingQueryFilter {
 			if atomic.SwapInt32(&f.grouping, 1) != 1 {
 				fs.Debugf(f, "Disabling ListR to work around bug in drive as multi listing (%d) returned no entries", len(dirs))
 			}
@@ -1783,7 +1789,8 @@ func (f *Fs) listRRunner(ctx context.Context, wg *sync.WaitGroup, in chan listRE
 		}
 		// If using a grouping of 1 and dir was empty then check to see if it
 		// is part of the group that caused grouping to be disabled.
-		if grouping == 1 && len(dirs) == 1 && !foundItems {
+		// However, empty result is legitimate if query filter was applied.
+		if grouping == 1 && len(dirs) == 1 && !foundItems && !usingQueryFilter {
 			f.listRmu.Lock()
 			if _, found := f.listRempties[dirs[0]]; found {
 				// Remove the ID

backend/hasher/kv.go

@@ -294,7 +294,7 @@ func (f *Fs) dumpLine(r *hashRecord, path string, include bool, err error) strin
 		if hashVal == "" || err != nil {
 			hashVal = "-"
 		}
-		hashVal = fmt.Sprintf("%-*s", hash.Width(hashType), hashVal)
+		hashVal = fmt.Sprintf("%-*s", hash.Width(hashType, false), hashVal)
 		hashes = append(hashes, hashName+":"+hashVal)
 	}
 	hashesStr := strings.Join(hashes, " ")

backend/hdfs/fs.go

@@ -263,6 +263,98 @@ func (f *Fs) Purge(ctx context.Context, dir string) error {
 	return f.client.RemoveAll(realpath)
 }
 
+// Move src to this remote using server-side move operations.
+//
+// This is stored with the remote path given
+//
+// It returns the destination Object and a possible error
+//
+// Will only be called if src.Fs().Name() == f.Name()
+//
+// If it isn't possible then return fs.ErrorCantMove
+func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
+	srcObj, ok := src.(*Object)
+	if !ok {
+		fs.Debugf(src, "Can't move - not same remote type")
+		return nil, fs.ErrorCantMove
+	}
+
+	// Get the real paths from the remote specs:
+	sourcePath := srcObj.fs.realpath(srcObj.remote)
+	targetPath := f.realpath(remote)
+	fs.Debugf(f, "rename [%s] to [%s]", sourcePath, targetPath)
+
+	// Make sure the target folder exists:
+	dirname := path.Dir(targetPath)
+	err := f.client.MkdirAll(dirname, 0755)
+	if err != nil {
+		return nil, err
+	}
+
+	// Do the move
+	// Note that the underlying HDFS library hard-codes Overwrite=True, but this is expected rclone behaviour.
+	err = f.client.Rename(sourcePath, targetPath)
+	if err != nil {
+		return nil, err
+	}
+
+	// Look up the resulting object
+	info, err := f.client.Stat(targetPath)
+	if err != nil {
+		return nil, err
+	}
+
+	// And return it:
+	return &Object{
+		fs:      f,
+		remote:  remote,
+		size:    info.Size(),
+		modTime: info.ModTime(),
+	}, nil
+}
+
+// DirMove moves src, srcRemote to this remote at dstRemote
+// using server-side move operations.
+//
+// Will only be called if src.Fs().Name() == f.Name()
+//
+// If it isn't possible then return fs.ErrorCantDirMove
+//
+// If destination exists then return fs.ErrorDirExists
+func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) (err error) {
+	srcFs, ok := src.(*Fs)
+	if !ok {
+		return fs.ErrorCantDirMove
+	}
+
+	// Get the real paths from the remote specs:
+	sourcePath := srcFs.realpath(srcRemote)
+	targetPath := f.realpath(dstRemote)
+	fs.Debugf(f, "rename [%s] to [%s]", sourcePath, targetPath)
+
+	// Check if the destination exists:
+	info, err := f.client.Stat(targetPath)
+	if err == nil {
+		fs.Debugf(f, "target directory already exits, IsDir = [%t]", info.IsDir())
+		return fs.ErrorDirExists
+	}
+
+	// Make sure the targets parent folder exists:
+	dirname := path.Dir(targetPath)
+	err = f.client.MkdirAll(dirname, 0755)
+	if err != nil {
+		return err
+	}
+
+	// Do the move
+	err = f.client.Rename(sourcePath, targetPath)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 // About gets quota information from the Fs
 func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
 	info, err := f.client.StatFs()
@@ -318,4 +410,6 @@ var (
 	_ fs.Purger      = (*Fs)(nil)
 	_ fs.PutStreamer = (*Fs)(nil)
 	_ fs.Abouter     = (*Fs)(nil)
+	_ fs.Mover       = (*Fs)(nil)
+	_ fs.DirMover    = (*Fs)(nil)
 )

backend/jottacloud/jottacloud.go

@@ -69,6 +69,10 @@ const (
 	teliaCloudTokenURL = "https://cloud-auth.telia.se/auth/realms/telia_se/protocol/openid-connect/token"
 	teliaCloudAuthURL  = "https://cloud-auth.telia.se/auth/realms/telia_se/protocol/openid-connect/auth"
 	teliaCloudClientID = "desktop"
+
+	tele2CloudTokenURL = "https://mittcloud-auth.tele2.se/auth/realms/comhem/protocol/openid-connect/token"
+	tele2CloudAuthURL  = "https://mittcloud-auth.tele2.se/auth/realms/comhem/protocol/openid-connect/auth"
+	tele2CloudClientID = "desktop"
 )
 
 // Register with Fs
@@ -131,6 +135,9 @@ func Config(ctx context.Context, name string, m configmap.Mapper, config fs.Conf
 		}, {
 			Value: "telia",
 			Help:  "Telia Cloud authentication.\nUse this if you are using Telia Cloud.",
+		}, {
+			Value: "tele2",
+			Help:  "Tele2 Cloud authentication.\nUse this if you are using Tele2 Cloud.",
 		}})
 	case "auth_type_done":
 		// Jump to next state according to config chosen
@@ -238,6 +245,21 @@ machines.`)
 				RedirectURL: oauthutil.RedirectLocalhostURL,
 			},
 		})
+	case "tele2": // tele2 cloud config
+		m.Set("configVersion", fmt.Sprint(configVersion))
+		m.Set(configClientID, tele2CloudClientID)
+		m.Set(configTokenURL, tele2CloudTokenURL)
+		return oauthutil.ConfigOut("choose_device", &oauthutil.Options{
+			OAuth2Config: &oauth2.Config{
+				Endpoint: oauth2.Endpoint{
+					AuthURL:  tele2CloudAuthURL,
+					TokenURL: tele2CloudTokenURL,
+				},
+				ClientID:    tele2CloudClientID,
+				Scopes:      []string{"openid", "jotta-default", "offline_access"},
+				RedirectURL: oauthutil.RedirectLocalhostURL,
+			},
+		})
 	case "choose_device":
 		return fs.ConfigConfirm("choose_device_query", false, "config_non_standard", "Use a non standard device/mountpoint e.g. for accessing files uploaded using the official Jottacloud client?")
 	case "choose_device_query":

backend/sftp/sftp.go

@@ -152,11 +152,11 @@ different. This issue affects among others Synology NAS boxes.
 
 Shared folders can be found in directories representing volumes
 
-    rclone sync /home/local/directory remote:/directory --ssh-path-override /volume2/directory
+    rclone sync /home/local/directory remote:/directory --sftp-path-override /volume2/directory
 
 Home directory can be found in a shared folder called "home"
 
-    rclone sync /home/local/directory remote:/home/directory --ssh-path-override /volume1/homes/USER/directory`,
+    rclone sync /home/local/directory remote:/home/directory --sftp-path-override /volume1/homes/USER/directory`,
 			Advanced: true,
 		}, {
 			Name:     "set_modtime",
@@ -1155,8 +1155,8 @@ func (f *Fs) Hashes() hash.Set {
 	}
 
 	changed := false
-	md5Works := checkHash([]string{"md5sum", "md5 -r"}, "d41d8cd98f00b204e9800998ecf8427e", &f.opt.Md5sumCommand, &changed)
-	sha1Works := checkHash([]string{"sha1sum", "sha1 -r"}, "da39a3ee5e6b4b0d3255bfef95601890afd80709", &f.opt.Sha1sumCommand, &changed)
+	md5Works := checkHash([]string{"md5sum", "md5 -r", "rclone md5sum"}, "d41d8cd98f00b204e9800998ecf8427e", &f.opt.Md5sumCommand, &changed)
+	sha1Works := checkHash([]string{"sha1sum", "sha1 -r", "rclone sha1sum"}, "da39a3ee5e6b4b0d3255bfef95601890afd80709", &f.opt.Sha1sumCommand, &changed)
 
 	if changed {
 		f.m.Set("md5sum_command", f.opt.Md5sumCommand)

cmd/hashsum/hashsum.go

@@ -2,7 +2,6 @@ package hashsum
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"os"
 
@@ -26,11 +25,11 @@ var (
 func init() {
 	cmd.Root.AddCommand(commandDefinition)
 	cmdFlags := commandDefinition.Flags()
-	AddHashFlags(cmdFlags)
+	AddHashsumFlags(cmdFlags)
 }
 
-// AddHashFlags is a convenience function to add the command flags OutputBase64 and DownloadFlag to hashsum, md5sum, sha1sum
-func AddHashFlags(cmdFlags *pflag.FlagSet) {
+// AddHashsumFlags is a convenience function to add the command flags OutputBase64 and DownloadFlag to hashsum, md5sum, sha1sum
+func AddHashsumFlags(cmdFlags *pflag.FlagSet) {
 	flags.BoolVarP(cmdFlags, &OutputBase64, "base64", "", OutputBase64, "Output base64 encoded hashsum")
 	flags.StringVarP(cmdFlags, &HashsumOutfile, "output-file", "", HashsumOutfile, "Output hashsums to a file rather than the terminal")
 	flags.StringVarP(cmdFlags, &ChecksumFile, "checkfile", "C", ChecksumFile, "Validate hashes against a given SUM file instead of printing them")
@@ -41,7 +40,7 @@ func AddHashFlags(cmdFlags *pflag.FlagSet) {
 func GetHashsumOutput(filename string) (out *os.File, close func(), err error) {
 	out, err = os.Create(filename)
 	if err != nil {
-		err = fmt.Errorf("Failed to open output file %v: %w", filename, err)
+		err = fmt.Errorf("failed to open output file %v: %w", filename, err)
 		return nil, nil, err
 	}
 
@@ -55,6 +54,32 @@ func GetHashsumOutput(filename string) (out *os.File, close func(), err error) {
 	return out, close, nil
 }
 
+// CreateFromStdinArg checks args and produces hashsum from standard input if it is requested
+func CreateFromStdinArg(ht hash.Type, args []string, startArg int) (bool, error) {
+	var stdinArg bool
+	if len(args) == startArg {
+		// Missing arg: Always read from stdin
+		stdinArg = true
+	} else if len(args) > startArg && args[startArg] == "-" {
+		// Special arg: Read from stdin only if there is data available
+		if fi, _ := os.Stdin.Stat(); fi.Mode()&os.ModeCharDevice == 0 {
+			stdinArg = true
+		}
+	}
+	if !stdinArg {
+		return false, nil
+	}
+	if HashsumOutfile == "" {
+		return true, operations.HashSumStream(ht, OutputBase64, os.Stdin, nil)
+	}
+	output, close, err := GetHashsumOutput(HashsumOutfile)
+	if err != nil {
+		return true, err
+	}
+	defer close()
+	return true, operations.HashSumStream(ht, OutputBase64, os.Stdin, output)
+}
+
 var commandDefinition = &cobra.Command{
 	Use:   "hashsum <hash> remote:path",
 	Short: `Produces a hashsum file for all the objects in the path.`,
@@ -68,6 +93,11 @@ not supported by the remote, no hash will be returned.  With the
 download flag, the file will be downloaded from the remote and
 hashed locally enabling any hash for any remote.
 
+This command can also hash data received on standard input (stdin),
+by not passing a remote:path, or by passing a hyphen as remote:path
+when there is data to read (if not, the hypen will be treated literaly,
+as a relative path).
+
 Run without a hash to see the list of all supported hashes, e.g.
 
     $ rclone hashsum
@@ -83,8 +113,6 @@ Note that hash names are case insensitive and values are output in lower case.
 		if len(args) == 0 {
 			fmt.Print(hash.HelpString(0))
 			return nil
-		} else if len(args) == 1 {
-			return errors.New("need hash type and remote")
 		}
 		var ht hash.Type
 		err := ht.Set(args[0])
@@ -92,8 +120,10 @@ Note that hash names are case insensitive and values are output in lower case.
 			fmt.Println(hash.HelpString(0))
 			return err
 		}
+		if found, err := CreateFromStdinArg(ht, args, 1); found {
+			return err
+		}
 		fsrc := cmd.NewFsSrc(args[1:])
-
 		cmd.Run(false, false, command, func() error {
 			if ChecksumFile != "" {
 				fsum, sumFile := cmd.NewFsFile(ChecksumFile)

cmd/md5sum/md5sum.go

@@ -13,7 +13,7 @@ import (
 func init() {
 	cmd.Root.AddCommand(commandDefinition)
 	cmdFlags := commandDefinition.Flags()
-	hashsum.AddHashFlags(cmdFlags)
+	hashsum.AddHashsumFlags(cmdFlags)
 }
 
 var commandDefinition = &cobra.Command{
@@ -27,9 +27,17 @@ By default, the hash is requested from the remote.  If MD5 is
 not supported by the remote, no hash will be returned.  With the
 download flag, the file will be downloaded from the remote and
 hashed locally enabling MD5 for any remote.
+
+This command can also hash data received on standard input (stdin),
+by not passing a remote:path, or by passing a hyphen as remote:path
+when there is data to read (if not, the hypen will be treated literaly,
+as a relative path).
 `,
-	Run: func(command *cobra.Command, args []string) {
-		cmd.CheckArgs(1, 1, command, args)
+	RunE: func(command *cobra.Command, args []string) error {
+		cmd.CheckArgs(0, 1, command, args)
+		if found, err := hashsum.CreateFromStdinArg(hash.MD5, args, 0); found {
+			return err
+		}
 		fsrc := cmd.NewFsSrc(args)
 		cmd.Run(false, false, command, func() error {
 			if hashsum.ChecksumFile != "" {
@@ -46,5 +54,6 @@ hashed locally enabling MD5 for any remote.
 			defer close()
 			return operations.HashLister(context.Background(), hash.MD5, hashsum.OutputBase64, hashsum.DownloadFlag, fsrc, output)
 		})
+		return nil
 	},
 }

cmd/serve/sftp/sftp.go

@@ -100,6 +100,17 @@ be used with sshd via ~/.ssh/authorized_keys, for example:
 
     restrict,command="rclone serve sftp --stdio ./photos" ssh-rsa ...
 
+On the client you need to set "--transfers 1" when using --stdio.
+Otherwise multiple instances of the rclone server are started by OpenSSH
+which can lead to "corrupted on transfer" errors. This is the case because
+the client chooses indiscriminately which server to send commands to while
+the servers all have different views of the state of the filing system.
+
+The "restrict" in authorized_keys prevents SHA1SUMs and MD5SUMs from beeing
+used. Omitting "restrict" and using --sftp-path-override to enable
+checksumming is possible but less secure and you could use the SFTP server
+provided by OpenSSH in this case.
+
 ` + vfs.Help + proxy.Help,
 	Run: func(command *cobra.Command, args []string) {
 		var f fs.Fs

cmd/sha1sum/sha1sum.go

@@ -13,7 +13,7 @@ import (
 func init() {
 	cmd.Root.AddCommand(commandDefinition)
 	cmdFlags := commandDefinition.Flags()
-	hashsum.AddHashFlags(cmdFlags)
+	hashsum.AddHashsumFlags(cmdFlags)
 }
 
 var commandDefinition = &cobra.Command{
@@ -27,9 +27,20 @@ By default, the hash is requested from the remote.  If SHA-1 is
 not supported by the remote, no hash will be returned.  With the
 download flag, the file will be downloaded from the remote and
 hashed locally enabling SHA-1 for any remote.
+
+This command can also hash data received on standard input (stdin),
+by not passing a remote:path, or by passing a hyphen as remote:path
+when there is data to read (if not, the hypen will be treated literaly,
+as a relative path).
+
+This command can also hash data received on STDIN, if not passing
+a remote:path.
 `,
-	Run: func(command *cobra.Command, args []string) {
-		cmd.CheckArgs(1, 1, command, args)
+	RunE: func(command *cobra.Command, args []string) error {
+		cmd.CheckArgs(0, 1, command, args)
+		if found, err := hashsum.CreateFromStdinArg(hash.SHA1, args, 0); found {
+			return err
+		}
 		fsrc := cmd.NewFsSrc(args)
 		cmd.Run(false, false, command, func() error {
 			if hashsum.ChecksumFile != "" {
@@ -46,5 +57,6 @@ hashed locally enabling SHA-1 for any remote.
 			defer close()
 			return operations.HashLister(context.Background(), hash.SHA1, hashsum.OutputBase64, hashsum.DownloadFlag, fsrc, output)
 		})
+		return nil
 	},
 }

docs/content/authors.md

@@ -547,3 +547,6 @@ put them back in again.` >}}
   * bbabich <bbabich@datamossa.com>
   * David <dp.davide.palma@gmail.com>
   * Borna Butkovic <borna@favicode.net>
+  * Fredric Arklid <fredric.arklid@consid.se>
+  * Andy Jackson <Andrew.Jackson@bl.uk>
+  * Sinan Tan <i@tinytangent.com>

docs/content/crypt.md

@@ -373,6 +373,14 @@ total path length which rclone is more likely to breach using
 characters in length issues should not be encountered, irrespective of
 cloud storage provider.
 
+An experimental advanced option `filename_encoding` is now provided to
+address this problem to a certain degree.
+For cloud storage systems with case sensitive file names (e.g. Google Drive),
+`base64` can be used to reduce file name length. 
+For cloud storage systems using UTF-16 to store file names internally
+(e.g. OneDrive), `base32768` can be used to drastically reduce
+file name length. 
+
 An alternative, future rclone file name encryption mode may tolerate
 backend provider path length limits.
 

docs/content/jottacloud.md

@@ -41,8 +41,8 @@ Note that the web interface may refer to this token as a JottaCli token.
 
 ### Legacy authentication
 
-If you are using one of the whitelabel versions (e.g. from Elkjøp or Tele2) you may not have the option
-to generate a CLI token. In this case you'll have to use the legacy authentication. To to this select
+If you are using one of the whitelabel versions (e.g. from Elkjøp) you may not have the option
+to generate a CLI token. In this case you'll have to use the legacy authentication. To do this select
 yes when the setup asks for legacy authentication and enter your username and password.
 The rest of the setup is identical to the default setup.
 
@@ -53,6 +53,13 @@ additionally uses a separate authentication flow where the username is generated
 rclone to use Telia Cloud, choose Telia Cloud authentication in the setup. The rest of the setup is
 identical to the default setup.
 
+### Tele2 Cloud authentication
+
+As Tele2-Com Hem merger was completed this authentication can be used for former Com Hem Cloud and
+Tele2 Cloud customers as no support for creating a CLI token exists, and additionally uses a separate
+authentication flow where the username is generated internally. To setup rclone to use Tele2 Cloud,
+choose Tele2 Cloud authentication in the setup. The rest of the setup is identical to the default setup.
+
 ## Configuration
 
 Here is an example of how to make a remote called `remote` with the default setup.  First run:

docs/content/overview.md

@@ -422,7 +422,7 @@ upon backend-specific capabilities.
 | Google Cloud Storage         | Yes   | Yes  | No   | No      | No      | Yes   | Yes          | No           | No    | No       |
 | Google Drive                 | Yes   | Yes  | Yes  | Yes     | Yes     | Yes   | Yes          | Yes          | Yes   | Yes      |
 | Google Photos                | No    | No   | No   | No      | No      | No    | No           | No           | No    | No       |
-| HDFS                         | Yes   | No   | No   | No      | No      | No    | Yes          | No           | Yes   | Yes      |
+| HDFS                         | Yes   | No   | Yes  | Yes     | No      | No    | Yes          | No           | Yes   | Yes      |
 | HTTP                         | No    | No   | No   | No      | No      | No    | No           | No           | No    | Yes      |
 | Hubic                        | Yes † | Yes  | No   | No      | No      | Yes   | Yes          | No           | Yes   | No       |
 | Jottacloud                   | Yes   | Yes  | Yes  | Yes     | Yes     | Yes   | No           | Yes          | Yes   | Yes      |

fs/fshttp/http.go

@@ -136,12 +136,14 @@ func NewClient(ctx context.Context) *http.Client {
 // Transport is our http Transport which wraps an http.Transport
 // * Sets the User Agent
 // * Does logging
+// * Updates metrics
 type Transport struct {
 	*http.Transport
 	dump          fs.DumpFlags
 	filterRequest func(req *http.Request)
 	userAgent     string
 	headers       []*fs.HTTPOption
+	metrics       *Metrics
 }
 
 // newTransport wraps the http.Transport passed in and logs all
@@ -152,6 +154,7 @@ func newTransport(ci *fs.ConfigInfo, transport *http.Transport) *Transport {
 		dump:      ci.Dump,
 		userAgent: ci.UserAgent,
 		headers:   ci.Headers,
+		metrics:   DefaultMetrics,
 	}
 }
 
@@ -283,6 +286,9 @@ func (t *Transport) RoundTrip(req *http.Request) (resp *http.Response, err error
 		fs.Debugf(nil, "%s", separatorResp)
 		logMutex.Unlock()
 	}
+	// Update metrics
+	t.metrics.onResponse(req, resp)
+
 	if err == nil {
 		checkServerTime(req, resp)
 	}

fs/fshttp/prometheus.go

@@ -0,0 +1,51 @@
+package fshttp
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// Metrics provide Transport HTTP level metrics.
+type Metrics struct {
+	StatusCode *prometheus.CounterVec
+}
+
+// NewMetrics creates a new metrics instance, the instance shall be assigned to
+// DefaultMetrics before any processing takes place.
+func NewMetrics(namespace string) *Metrics {
+	return &Metrics{
+		StatusCode: prometheus.NewCounterVec(prometheus.CounterOpts{
+			Namespace: namespace,
+			Subsystem: "http",
+			Name:      "status_code",
+		}, []string{"host", "method", "code"}),
+	}
+}
+
+// DefaultMetrics specifies metrics used for new Transports.
+var DefaultMetrics = (*Metrics)(nil)
+
+// Collectors returns all prometheus metrics as collectors for registration.
+func (m *Metrics) Collectors() []prometheus.Collector {
+	if m == nil {
+		return nil
+	}
+	return []prometheus.Collector{
+		m.StatusCode,
+	}
+}
+
+func (m *Metrics) onResponse(req *http.Request, resp *http.Response) {
+	if m == nil {
+		return
+	}
+
+	var statusCode = 0
+	if resp != nil {
+		statusCode = resp.StatusCode
+	}
+
+	m.StatusCode.WithLabelValues(req.Host, req.Method, fmt.Sprint(statusCode)).Inc()
+}

fs/hash/hash.go

@@ -4,6 +4,7 @@ import (
 	"crypto/md5"
 	"crypto/sha1"
 	"crypto/sha256"
+	"encoding/base64"
 	"encoding/hex"
 	"errors"
 	"fmt"
@@ -92,8 +93,11 @@ func Supported() Set {
 }
 
 // Width returns the width in characters for any HashType
-func Width(hashType Type) int {
+func Width(hashType Type, base64Encoded bool) int {
 	if hash := type2hash[hashType]; hash != nil {
+		if base64Encoded {
+			return base64.URLEncoding.EncodedLen(hash.width / 2)
+		}
 		return hash.width
 	}
 	return 0
@@ -243,6 +247,18 @@ func (m *MultiHasher) Sum(hashType Type) ([]byte, error) {
 	return h.Sum(nil), nil
 }
 
+// SumString returns the specified hash from the multihasher as a hex or base64 encoded string
+func (m *MultiHasher) SumString(hashType Type, base64Encoded bool) (string, error) {
+	sum, err := m.Sum(hashType)
+	if err != nil {
+		return "", err
+	}
+	if base64Encoded {
+		return base64.URLEncoding.EncodeToString(sum), nil
+	}
+	return hex.EncodeToString(sum), nil
+}
+
 // Size returns the number of bytes written
 func (m *MultiHasher) Size() int64 {
 	return m.size

fs/operations/operations.go

@@ -946,7 +946,7 @@ func ListLong(ctx context.Context, f fs.Fs, w io.Writer) error {
 // hashSum returns the human-readable hash for ht passed in.  This may
 // be UNSUPPORTED or ERROR. If it isn't returning a valid hash it will
 // return an error.
-func hashSum(ctx context.Context, ht hash.Type, downloadFlag bool, o fs.Object) (string, error) {
+func hashSum(ctx context.Context, ht hash.Type, base64Encoded bool, downloadFlag bool, o fs.Object) (string, error) {
 	var sum string
 	var err error
 
@@ -968,7 +968,7 @@ func hashSum(ctx context.Context, ht hash.Type, downloadFlag bool, o fs.Object)
 		}
 		in, err := NewReOpen(ctx, o, fs.GetConfig(ctx).LowLevelRetries, options...)
 		if err != nil {
-			return "ERROR", fmt.Errorf("Failed to open file %v: %w", o, err)
+			return "ERROR", fmt.Errorf("failed to open file %v: %w", o, err)
 		}
 
 		// Account and buffer the transfer
@@ -977,21 +977,20 @@ func hashSum(ctx context.Context, ht hash.Type, downloadFlag bool, o fs.Object)
 		// Setup hasher
 		hasher, err := hash.NewMultiHasherTypes(hash.NewHashSet(ht))
 		if err != nil {
-			return "UNSUPPORTED", fmt.Errorf("Hash unsupported: %w", err)
+			return "UNSUPPORTED", fmt.Errorf("hash unsupported: %w", err)
 		}
 
 		// Copy to hasher, downloading the file and passing directly to hash
 		_, err = io.Copy(hasher, in)
 		if err != nil {
-			return "ERROR", fmt.Errorf("Failed to copy file to hasher: %w", err)
+			return "ERROR", fmt.Errorf("failed to copy file to hasher: %w", err)
 		}
 
-		// Get hash and encode as hex
-		byteSum, err := hasher.Sum(ht)
+		// Get hash as hex or base64 encoded string
+		sum, err = hasher.SumString(ht, base64Encoded)
 		if err != nil {
-			return "ERROR", fmt.Errorf("Hasher returned an error: %w", err)
+			return "ERROR", fmt.Errorf("hasher returned an error: %w", err)
 		}
-		sum = hex.EncodeToString(byteSum)
 	} else {
 		tr := accounting.Stats(ctx).NewCheckingTransfer(o)
 		defer func() {
@@ -999,11 +998,15 @@ func hashSum(ctx context.Context, ht hash.Type, downloadFlag bool, o fs.Object)
 		}()
 
 		sum, err = o.Hash(ctx, ht)
+		if base64Encoded {
+			hexBytes, _ := hex.DecodeString(sum)
+			sum = base64.URLEncoding.EncodeToString(hexBytes)
+		}
 		if err == hash.ErrUnsupported {
-			return "", fmt.Errorf("Hash unsupported: %w", err)
+			return "", fmt.Errorf("hash unsupported: %w", err)
 		}
 		if err != nil {
-			return "", fmt.Errorf("Failed to get hash %v from backend: %v: %w", ht, err, err)
+			return "", fmt.Errorf("failed to get hash %v from backend: %w", ht, err)
 		}
 	}
 
@@ -1014,10 +1017,7 @@ func hashSum(ctx context.Context, ht hash.Type, downloadFlag bool, o fs.Object)
 // Updated to handle both standard hex encoding and base64
 // Updated to perform multiple hashes concurrently
 func HashLister(ctx context.Context, ht hash.Type, outputBase64 bool, downloadFlag bool, f fs.Fs, w io.Writer) error {
-	width := hash.Width(ht)
-	if outputBase64 {
-		width = base64.URLEncoding.EncodedLen(width / 2)
-	}
+	width := hash.Width(ht, outputBase64)
 	concurrencyControl := make(chan struct{}, fs.GetConfig(ctx).Transfers)
 	var wg sync.WaitGroup
 	err := ListFn(ctx, f, func(o fs.Object) {
@@ -1028,15 +1028,11 @@ func HashLister(ctx context.Context, ht hash.Type, outputBase64 bool, downloadFl
 				<-concurrencyControl
 				wg.Done()
 			}()
-			sum, err := hashSum(ctx, ht, downloadFlag, o)
+			sum, err := hashSum(ctx, ht, outputBase64, downloadFlag, o)
 			if err != nil {
 				fs.Errorf(o, "%v", fs.CountError(err))
 				return
 			}
-			if outputBase64 {
-				hexBytes, _ := hex.DecodeString(sum)
-				sum = base64.URLEncoding.EncodeToString(hexBytes)
-			}
 			syncFprintf(w, "%*s  %s\n", width, sum, o.Remote())
 		}()
 	})
@@ -1044,6 +1040,28 @@ func HashLister(ctx context.Context, ht hash.Type, outputBase64 bool, downloadFl
 	return err
 }
 
+// HashSumStream outputs a line compatible with md5sum to w based on the
+// input stream in and the hash type ht passed in. If outputBase64 is
+// set then the hash will be base64 instead of hexadecimal.
+func HashSumStream(ht hash.Type, outputBase64 bool, in io.ReadCloser, w io.Writer) error {
+	hasher, err := hash.NewMultiHasherTypes(hash.NewHashSet(ht))
+	if err != nil {
+		return fmt.Errorf("hash unsupported: %w", err)
+	}
+	written, err := io.Copy(hasher, in)
+	fs.Debugf(nil, "Creating %s hash of %d bytes read from input stream", ht, written)
+	if err != nil {
+		return fmt.Errorf("failed to copy input to hasher: %w", err)
+	}
+	sum, err := hasher.SumString(ht, outputBase64)
+	if err != nil {
+		return fmt.Errorf("hasher returned an error: %w", err)
+	}
+	width := hash.Width(ht, outputBase64)
+	syncFprintf(w, "%*s  -\n", width, sum)
+	return nil
+}
+
 // Count counts the objects and their sizes in the Fs
 //
 // Obeys includes and excludes

fs/operations/operations_test.go

@@ -316,6 +316,52 @@ func TestHashSumsWithErrors(t *testing.T) {
 	// TODO mock an unreadable file
 }
 
+func TestHashStream(t *testing.T) {
+	reader := strings.NewReader("")
+	in := ioutil.NopCloser(reader)
+	out := &bytes.Buffer{}
+	for _, test := range []struct {
+		input      string
+		ht         hash.Type
+		wantHex    string
+		wantBase64 string
+	}{
+		{
+			input:      "",
+			ht:         hash.MD5,
+			wantHex:    "d41d8cd98f00b204e9800998ecf8427e  -\n",
+			wantBase64: "1B2M2Y8AsgTpgAmY7PhCfg==  -\n",
+		},
+		{
+			input:      "",
+			ht:         hash.SHA1,
+			wantHex:    "da39a3ee5e6b4b0d3255bfef95601890afd80709  -\n",
+			wantBase64: "2jmj7l5rSw0yVb_vlWAYkK_YBwk=  -\n",
+		},
+		{
+			input:      "Hello world!",
+			ht:         hash.MD5,
+			wantHex:    "86fb269d190d2c85f6e0468ceca42a20  -\n",
+			wantBase64: "hvsmnRkNLIX24EaM7KQqIA==  -\n",
+		},
+		{
+			input:      "Hello world!",
+			ht:         hash.SHA1,
+			wantHex:    "d3486ae9136e7856bc42212385ea797094475802  -\n",
+			wantBase64: "00hq6RNueFa8QiEjhep5cJRHWAI=  -\n",
+		},
+	} {
+		reader.Reset(test.input)
+		require.NoError(t, operations.HashSumStream(test.ht, false, in, out))
+		assert.Equal(t, test.wantHex, out.String())
+		_, _ = reader.Seek(0, io.SeekStart)
+		out.Reset()
+		require.NoError(t, operations.HashSumStream(test.ht, true, in, out))
+		assert.Equal(t, test.wantBase64, out.String())
+		out.Reset()
+	}
+}
+
 func TestSuffixName(t *testing.T) {
 	ctx := context.Background()
 	ctx, ci := fs.AddConfig(ctx)

fs/rc/rcserver/rcserver.go

@@ -18,23 +18,22 @@ import (
 	"sync"
 	"time"
 
-	"github.com/rclone/rclone/fs/rc/webgui"
-
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
-	"github.com/skratchdot/open-golang/open"
-
 	"github.com/rclone/rclone/cmd/serve/httplib"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
 	"github.com/rclone/rclone/fs/cache"
 	"github.com/rclone/rclone/fs/config"
+	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/list"
 	"github.com/rclone/rclone/fs/rc"
 	"github.com/rclone/rclone/fs/rc/jobs"
 	"github.com/rclone/rclone/fs/rc/rcflags"
+	"github.com/rclone/rclone/fs/rc/webgui"
 	"github.com/rclone/rclone/lib/http/serve"
 	"github.com/rclone/rclone/lib/random"
+	"github.com/skratchdot/open-golang/open"
 )
 
 var promHandler http.Handler
@@ -43,6 +42,13 @@ var onlyOnceWarningAllowOrigin sync.Once
 func init() {
 	rcloneCollector := accounting.NewRcloneCollector(context.Background())
 	prometheus.MustRegister(rcloneCollector)
+
+	m := fshttp.NewMetrics("rclone")
+	for _, c := range m.Collectors() {
+		prometheus.MustRegister(c)
+	}
+	fshttp.DefaultMetrics = m
+
 	promHandler = promhttp.Handler()
 }
 

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/Azure/azure-storage-blob-go v0.14.0
 	github.com/Azure/go-autorest/autorest/adal v0.9.17
 	github.com/Azure/go-ntlmssp v0.0.0-20200615164410-66371956d46c
+	github.com/Max-Sum/base32768 v0.0.0-20191205131208-7937843c71d5 // indirect
 	github.com/Unknwon/goconfig v0.0.0-20200908083735-df7de6a44db8
 	github.com/a8m/tree v0.0.0-20210414114729-ce3525c5c2ef
 	github.com/aalpar/deheap v0.0.0-20210914013432-0cc84d79dec3

go.sum

@@ -76,6 +76,8 @@ github.com/Azure/go-ntlmssp v0.0.0-20200615164410-66371956d46c/go.mod h1:chxPXzS
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/Julusian/godocdown v0.0.0-20170816220326-6d19f8ff2df8/go.mod h1:INZr5t32rG59/5xeltqoCJoNY7e5x/3xoY9WSWVWg74=
+github.com/Max-Sum/base32768 v0.0.0-20191205131208-7937843c71d5 h1:w/vNc+SQRYKGWBHeDrzvvNttHwZEbSAP0kmTdORl4OI=
+github.com/Max-Sum/base32768 v0.0.0-20191205131208-7937843c71d5/go.mod h1:C8yoIfvESpM3GD07OCHU7fqI7lhwyZ2Td1rbNbTAhnc=
 github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
 github.com/Microsoft/go-winio v0.5.1 h1:aPJp2QD7OOrhO5tQXqQoGSJc+DjDtWTGLOmNyAm6FgY=
 github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=