mirror of
https://github.com/rclone/rclone.git
synced 2025-12-06 00:03:32 +00:00
lib/transform
lib/transform adds the transform library, supporting advanced path name transformations for converting and renaming files and directories by applying prefixes, suffixes, and other alterations. It also adds the --name-transform flag for use with sync, copy, and move. Multiple transformations can be used in sequence, applied in the order they are specified on the command line. By default --name-transform will only apply to file names. The means only the leaf file name will be transformed. However some of the transforms would be better applied to the whole path or just directories. To choose which which part of the file path is affected some tags can be added to the --name-transform: file Only transform the leaf name of files (DEFAULT) dir Only transform name of directories - these may appear anywhere in the path all Transform the entire path for files and directories Example syntax: --name-transform file,prefix=ABC --name-transform dir,prefix=DEF
This commit is contained in:
@@ -4,8 +4,6 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math"
|
||||
"mime"
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -13,6 +11,7 @@ import (
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/rclone/rclone/lib/terminal"
|
||||
"github.com/rclone/rclone/lib/transform"
|
||||
)
|
||||
|
||||
// Prefer describes strategies for resolving sync conflicts
|
||||
@@ -130,6 +129,7 @@ type (
|
||||
path2 namePair
|
||||
}
|
||||
)
|
||||
|
||||
type namePair struct {
|
||||
oldName string
|
||||
newName string
|
||||
@@ -240,24 +240,7 @@ func SuffixName(ctx context.Context, remote, suffix string) string {
|
||||
}
|
||||
ci := fs.GetConfig(ctx)
|
||||
if ci.SuffixKeepExtension {
|
||||
var (
|
||||
base = remote
|
||||
exts = ""
|
||||
first = true
|
||||
ext = path.Ext(remote)
|
||||
)
|
||||
for ext != "" {
|
||||
// Look second and subsequent extensions in mime types.
|
||||
// If they aren't found then don't keep it as an extension.
|
||||
if !first && mime.TypeByExtension(ext) == "" {
|
||||
break
|
||||
}
|
||||
base = base[:len(base)-len(ext)]
|
||||
exts = ext + exts
|
||||
first = false
|
||||
ext = path.Ext(base)
|
||||
}
|
||||
return base + suffix + exts
|
||||
return transform.SuffixKeepExtension(remote, suffix)
|
||||
}
|
||||
return remote + suffix
|
||||
}
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
"github.com/rclone/rclone/fs/log/logflags"
|
||||
"github.com/rclone/rclone/fs/rc/rcflags"
|
||||
"github.com/rclone/rclone/lib/atexit"
|
||||
"github.com/rclone/rclone/lib/transform/transformflags"
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/pflag"
|
||||
"golang.org/x/text/cases"
|
||||
@@ -137,6 +138,7 @@ func setupRootCommand(rootCmd *cobra.Command) {
|
||||
// Add global flags
|
||||
configflags.AddFlags(ci, pflag.CommandLine)
|
||||
filterflags.AddFlags(pflag.CommandLine)
|
||||
transformflags.AddFlags(pflag.CommandLine)
|
||||
rcflags.AddFlags(pflag.CommandLine)
|
||||
logflags.AddFlags(pflag.CommandLine)
|
||||
|
||||
@@ -191,7 +193,6 @@ func setupRootCommand(rootCmd *cobra.Command) {
|
||||
})
|
||||
|
||||
cobra.OnInitialize(initConfig)
|
||||
|
||||
}
|
||||
|
||||
// Traverse the tree of commands running fn on each
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"github.com/rclone/rclone/fs/filter"
|
||||
"github.com/rclone/rclone/fs/list"
|
||||
"github.com/rclone/rclone/fs/walk"
|
||||
"github.com/rclone/rclone/lib/transform"
|
||||
"golang.org/x/sync/errgroup"
|
||||
"golang.org/x/text/unicode/norm"
|
||||
)
|
||||
@@ -86,6 +87,7 @@ func (m *March) srcKey(entry fs.DirEntry) string {
|
||||
return ""
|
||||
}
|
||||
name := path.Base(entry.Remote())
|
||||
name = transform.Path(name, fs.DirEntryType(entry) == "directory")
|
||||
for _, transform := range m.transforms {
|
||||
name = transform(name)
|
||||
}
|
||||
@@ -94,7 +96,14 @@ func (m *March) srcKey(entry fs.DirEntry) string {
|
||||
|
||||
// dstKey turns a directory entry into a sort key using the defined transforms.
|
||||
func (m *March) dstKey(entry fs.DirEntry) string {
|
||||
return m.srcKey(entry) // FIXME actually do something different
|
||||
if entry == nil {
|
||||
return ""
|
||||
}
|
||||
name := path.Base(entry.Remote())
|
||||
for _, transform := range m.transforms {
|
||||
name = transform(name)
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
// makeListDir makes constructs a listing function for the given fs
|
||||
@@ -454,7 +463,6 @@ func (m *March) processJob(job listDirJob) ([]listDirJob, error) {
|
||||
noDst: true,
|
||||
})
|
||||
}
|
||||
|
||||
}, func(dst fs.DirEntry) {
|
||||
recurse := m.Callback.DstOnly(dst)
|
||||
if recurse && job.dstDepth > 0 {
|
||||
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/lib/atexit"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/transform"
|
||||
)
|
||||
|
||||
// State of the copy
|
||||
@@ -390,7 +391,7 @@ func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Obj
|
||||
f: f,
|
||||
dstFeatures: f.Features(),
|
||||
dst: dst,
|
||||
remote: remote,
|
||||
remote: transform.Path(remote, false),
|
||||
src: src,
|
||||
ci: ci,
|
||||
tr: tr,
|
||||
@@ -399,7 +400,7 @@ func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Obj
|
||||
}
|
||||
c.hashType, c.hashOption = CommonHash(ctx, f, src.Fs())
|
||||
if c.dst != nil {
|
||||
c.remote = c.dst.Remote()
|
||||
c.remote = transform.Path(c.dst.Remote(), false)
|
||||
}
|
||||
// Are we using partials?
|
||||
//
|
||||
|
||||
@@ -39,6 +39,7 @@ import (
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/random"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
"github.com/rclone/rclone/lib/transform"
|
||||
"golang.org/x/sync/errgroup"
|
||||
"golang.org/x/text/unicode/norm"
|
||||
)
|
||||
@@ -424,6 +425,8 @@ func MoveTransfer(ctx context.Context, fdst fs.Fs, dst fs.Object, remote string,
|
||||
|
||||
// move - see Move for help
|
||||
func move(ctx context.Context, fdst fs.Fs, dst fs.Object, remote string, src fs.Object, isTransfer bool) (newDst fs.Object, err error) {
|
||||
origRemote := remote // avoid double-transform on fallback to copy
|
||||
remote = transform.Path(remote, false)
|
||||
ci := fs.GetConfig(ctx)
|
||||
var tr *accounting.Transfer
|
||||
if isTransfer {
|
||||
@@ -447,7 +450,7 @@ func move(ctx context.Context, fdst fs.Fs, dst fs.Object, remote string, src fs.
|
||||
if doMove := fdst.Features().Move; doMove != nil && (SameConfig(src.Fs(), fdst) || (SameRemoteType(src.Fs(), fdst) && (fdst.Features().ServerSideAcrossConfigs || ci.ServerSideAcrossConfigs))) {
|
||||
// Delete destination if it exists and is not the same file as src (could be same file while seemingly different if the remote is case insensitive)
|
||||
if dst != nil {
|
||||
remote = dst.Remote()
|
||||
remote = transform.Path(dst.Remote(), false)
|
||||
if !SameObject(src, dst) {
|
||||
err = DeleteFile(ctx, dst)
|
||||
if err != nil {
|
||||
@@ -488,7 +491,7 @@ func move(ctx context.Context, fdst fs.Fs, dst fs.Object, remote string, src fs.
|
||||
}
|
||||
}
|
||||
// Move not found or didn't work so copy dst <- src
|
||||
newDst, err = Copy(ctx, fdst, dst, remote, src)
|
||||
newDst, err = Copy(ctx, fdst, dst, origRemote, src)
|
||||
if err != nil {
|
||||
fs.Errorf(src, "Not deleting source as copy failed: %v", err)
|
||||
return newDst, err
|
||||
@@ -516,24 +519,7 @@ func SuffixName(ctx context.Context, remote string) string {
|
||||
return remote
|
||||
}
|
||||
if ci.SuffixKeepExtension {
|
||||
var (
|
||||
base = remote
|
||||
exts = ""
|
||||
first = true
|
||||
ext = path.Ext(remote)
|
||||
)
|
||||
for ext != "" {
|
||||
// Look second and subsequent extensions in mime types.
|
||||
// If they aren't found then don't keep it as an extension.
|
||||
if !first && mime.TypeByExtension(ext) == "" {
|
||||
break
|
||||
}
|
||||
base = base[:len(base)-len(ext)]
|
||||
exts = ext + exts
|
||||
first = false
|
||||
ext = path.Ext(base)
|
||||
}
|
||||
return base + ci.Suffix + exts
|
||||
return transform.SuffixKeepExtension(remote, ci.Suffix)
|
||||
}
|
||||
return remote + ci.Suffix
|
||||
}
|
||||
|
||||
@@ -20,6 +20,7 @@ import (
|
||||
"github.com/rclone/rclone/fs/march"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/rclone/rclone/lib/errcount"
|
||||
"github.com/rclone/rclone/lib/transform"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
@@ -1254,8 +1255,8 @@ func (s *syncCopyMove) SrcOnly(src fs.DirEntry) (recurse bool) {
|
||||
s.logger(s.ctx, operations.MissingOnDst, src, nil, fs.ErrorIsDir)
|
||||
|
||||
// Create the directory and make sure the Metadata/ModTime is correct
|
||||
s.copyDirMetadata(s.ctx, s.fdst, nil, x.Remote(), x)
|
||||
s.markDirModified(x.Remote())
|
||||
s.copyDirMetadata(s.ctx, s.fdst, nil, transform.Path(x.Remote(), true), x)
|
||||
s.markDirModified(transform.Path(x.Remote(), true))
|
||||
return true
|
||||
default:
|
||||
panic("Bad object in DirEntries")
|
||||
@@ -1288,6 +1289,8 @@ func (s *syncCopyMove) Match(ctx context.Context, dst, src fs.DirEntry) (recurse
|
||||
}
|
||||
case fs.Directory:
|
||||
// Do the same thing to the entire contents of the directory
|
||||
srcX = fs.NewOverrideDirectory(srcX, transform.Path(src.Remote(), true))
|
||||
src = srcX
|
||||
s.markParentNotEmpty(src)
|
||||
dstX, ok := dst.(fs.Directory)
|
||||
if ok {
|
||||
@@ -1372,6 +1375,11 @@ func moveDir(ctx context.Context, fdst, fsrc fs.Fs, deleteEmptySrcDirs bool, cop
|
||||
return runSyncCopyMove(ctx, fdst, fsrc, fs.DeleteModeOff, true, deleteEmptySrcDirs, copyEmptySrcDirs)
|
||||
}
|
||||
|
||||
// Transform renames fdst in place
|
||||
func Transform(ctx context.Context, fdst fs.Fs, deleteEmptySrcDirs bool, copyEmptySrcDirs bool) error {
|
||||
return runSyncCopyMove(ctx, fdst, fdst, fs.DeleteModeOff, true, deleteEmptySrcDirs, copyEmptySrcDirs)
|
||||
}
|
||||
|
||||
// MoveDir moves fsrc into fdst
|
||||
func MoveDir(ctx context.Context, fdst, fsrc fs.Fs, deleteEmptySrcDirs bool, copyEmptySrcDirs bool) error {
|
||||
fi := filter.GetConfig(ctx)
|
||||
|
||||
@@ -27,6 +27,7 @@ import (
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/rclone/rclone/fstest"
|
||||
"github.com/rclone/rclone/lib/transform"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"golang.org/x/text/unicode/norm"
|
||||
@@ -2980,7 +2981,7 @@ func predictDstFromLogger(ctx context.Context) context.Context {
|
||||
if winner.Err != nil {
|
||||
errMsg = ";" + winner.Err.Error()
|
||||
}
|
||||
operations.SyncFprintf(opt.JSON, "%s;%s;%v;%s%s\n", file.ModTime(ctx).Local().Format(timeFormat), checksum, file.Size(), file.Remote(), errMsg)
|
||||
operations.SyncFprintf(opt.JSON, "%s;%s;%v;%s%s\n", file.ModTime(ctx).Local().Format(timeFormat), checksum, file.Size(), transform.Path(file.Remote(), false), errMsg) // TODO: should the transform be handled in the sync instead of here?
|
||||
}
|
||||
}
|
||||
return operations.WithSyncLogger(ctx, opt)
|
||||
|
||||
457
fs/sync/sync_transform_test.go
Normal file
457
fs/sync/sync_transform_test.go
Normal file
@@ -0,0 +1,457 @@
|
||||
// Test transform
|
||||
|
||||
package sync
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"context"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
_ "github.com/rclone/rclone/backend/all"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/filter"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/rclone/rclone/fs/walk"
|
||||
"github.com/rclone/rclone/fstest"
|
||||
"github.com/rclone/rclone/lib/transform"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"golang.org/x/text/unicode/norm"
|
||||
)
|
||||
|
||||
var debug = ``
|
||||
|
||||
func TestTransform(t *testing.T) {
|
||||
type args struct {
|
||||
TransformOpt []string
|
||||
TransformBackOpt []string
|
||||
Lossless bool // whether the TransformBackAlgo is always losslessly invertible
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
args args
|
||||
}{
|
||||
{name: "NFC", args: args{
|
||||
TransformOpt: []string{"nfc"},
|
||||
TransformBackOpt: []string{"nfd"},
|
||||
Lossless: false,
|
||||
}},
|
||||
{name: "NFD", args: args{
|
||||
TransformOpt: []string{"nfd"},
|
||||
TransformBackOpt: []string{"nfc"},
|
||||
Lossless: false,
|
||||
}},
|
||||
{name: "base64", args: args{
|
||||
TransformOpt: []string{"base64encode"},
|
||||
TransformBackOpt: []string{"base64encode"},
|
||||
Lossless: false,
|
||||
}},
|
||||
{name: "prefix", args: args{
|
||||
TransformOpt: []string{"prefix=PREFIX"},
|
||||
TransformBackOpt: []string{"trimprefix=PREFIX"},
|
||||
Lossless: true,
|
||||
}},
|
||||
{name: "suffix", args: args{
|
||||
TransformOpt: []string{"suffix=SUFFIX"},
|
||||
TransformBackOpt: []string{"trimsuffix=SUFFIX"},
|
||||
Lossless: true,
|
||||
}},
|
||||
{name: "truncate", args: args{
|
||||
TransformOpt: []string{"truncate=10"},
|
||||
TransformBackOpt: []string{"truncate=10"},
|
||||
Lossless: false,
|
||||
}},
|
||||
{name: "encoder", args: args{
|
||||
TransformOpt: []string{"encoder=Colon,SquareBracket"},
|
||||
TransformBackOpt: []string{"decoder=Colon,SquareBracket"},
|
||||
Lossless: true,
|
||||
}},
|
||||
{name: "ISO-8859-1", args: args{
|
||||
TransformOpt: []string{"ISO-8859-1"},
|
||||
TransformBackOpt: []string{"ISO-8859-1"},
|
||||
Lossless: false,
|
||||
}},
|
||||
{name: "charmap", args: args{
|
||||
TransformOpt: []string{"all,charmap=ISO-8859-7"},
|
||||
TransformBackOpt: []string{"all,charmap=ISO-8859-7"},
|
||||
Lossless: false,
|
||||
}},
|
||||
{name: "lowercase", args: args{
|
||||
TransformOpt: []string{"all,lowercase"},
|
||||
TransformBackOpt: []string{"all,lowercase"},
|
||||
Lossless: false,
|
||||
}},
|
||||
{name: "ascii", args: args{
|
||||
TransformOpt: []string{"all,ascii"},
|
||||
TransformBackOpt: []string{"all,ascii"},
|
||||
Lossless: false,
|
||||
}},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
r := fstest.NewRun(t)
|
||||
defer r.Finalise()
|
||||
|
||||
r.Mkdir(context.Background(), r.Flocal)
|
||||
r.Mkdir(context.Background(), r.Fremote)
|
||||
items := makeTestFiles(t, r, "dir1")
|
||||
deleteDSStore(t, r)
|
||||
r.CheckRemoteListing(t, items, nil)
|
||||
r.CheckLocalListing(t, items, nil)
|
||||
|
||||
err := transform.SetOptions(context.Background(), tt.args.TransformOpt...)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = Sync(context.Background(), r.Fremote, r.Flocal, true)
|
||||
assert.NoError(t, err)
|
||||
compareNames(t, r, items)
|
||||
|
||||
err = transform.SetOptions(context.Background(), tt.args.TransformBackOpt...)
|
||||
require.NoError(t, err)
|
||||
err = Sync(context.Background(), r.Fremote, r.Flocal, true)
|
||||
assert.NoError(t, err)
|
||||
compareNames(t, r, items)
|
||||
|
||||
if tt.args.Lossless {
|
||||
deleteDSStore(t, r)
|
||||
r.CheckRemoteItems(t, items...)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
const alphabet = "abcdefg123456789"
|
||||
|
||||
var extras = []string{"apple", "banana", "appleappleapplebanana", "splitbananasplit"}
|
||||
|
||||
func makeTestFiles(t *testing.T, r *fstest.Run, dir string) []fstest.Item {
|
||||
t.Helper()
|
||||
n := 0
|
||||
// Create test files
|
||||
items := []fstest.Item{}
|
||||
for _, c := range alphabet {
|
||||
var out strings.Builder
|
||||
for i := rune(0); i < 7; i++ {
|
||||
out.WriteRune(c + i)
|
||||
}
|
||||
fileName := filepath.Join(dir, fmt.Sprintf("%04d-%s.txt", n, out.String()))
|
||||
fileName = strings.ToValidUTF8(fileName, "")
|
||||
|
||||
if debug != "" {
|
||||
fileName = debug
|
||||
}
|
||||
|
||||
item := r.WriteObject(context.Background(), fileName, fileName, t1)
|
||||
r.WriteFile(fileName, fileName, t1)
|
||||
items = append(items, item)
|
||||
n++
|
||||
|
||||
if debug != "" {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
for _, extra := range extras {
|
||||
item := r.WriteObject(context.Background(), extra, extra, t1)
|
||||
r.WriteFile(extra, extra, t1)
|
||||
items = append(items, item)
|
||||
}
|
||||
|
||||
return items
|
||||
}
|
||||
|
||||
func deleteDSStore(t *testing.T, r *fstest.Run) {
|
||||
ctxDSStore, fi := filter.AddConfig(context.Background())
|
||||
err := fi.AddRule(`+ *.DS_Store`)
|
||||
assert.NoError(t, err)
|
||||
err = fi.AddRule(`- **`)
|
||||
assert.NoError(t, err)
|
||||
err = operations.Delete(ctxDSStore, r.Fremote)
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
func compareNames(t *testing.T, r *fstest.Run, items []fstest.Item) {
|
||||
var entries fs.DirEntries
|
||||
|
||||
deleteDSStore(t, r)
|
||||
err := walk.ListR(context.Background(), r.Fremote, "", true, -1, walk.ListObjects, func(e fs.DirEntries) error {
|
||||
entries = append(entries, e...)
|
||||
return nil
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
entries = slices.DeleteFunc(entries, func(E fs.DirEntry) bool { // remove those pesky .DS_Store files
|
||||
if strings.Contains(E.Remote(), ".DS_Store") {
|
||||
err := operations.DeleteFile(context.Background(), E.(fs.Object))
|
||||
assert.NoError(t, err)
|
||||
return true
|
||||
}
|
||||
return false
|
||||
})
|
||||
require.Equal(t, len(items), entries.Len())
|
||||
|
||||
// sort by CONVERTED name
|
||||
slices.SortStableFunc(items, func(a, b fstest.Item) int {
|
||||
aConv := transform.Path(a.Path, false)
|
||||
bConv := transform.Path(b.Path, false)
|
||||
return cmp.Compare(aConv, bConv)
|
||||
})
|
||||
slices.SortStableFunc(entries, func(a, b fs.DirEntry) int {
|
||||
return cmp.Compare(a.Remote(), b.Remote())
|
||||
})
|
||||
|
||||
for i, e := range entries {
|
||||
expect := transform.Path(items[i].Path, false)
|
||||
msg := fmt.Sprintf("expected %v, got %v", detectEncoding(expect), detectEncoding(e.Remote()))
|
||||
assert.Equal(t, expect, e.Remote(), msg)
|
||||
}
|
||||
}
|
||||
|
||||
func detectEncoding(s string) string {
|
||||
if norm.NFC.IsNormalString(s) && norm.NFD.IsNormalString(s) {
|
||||
return "BOTH"
|
||||
}
|
||||
if !norm.NFC.IsNormalString(s) && norm.NFD.IsNormalString(s) {
|
||||
return "NFD"
|
||||
}
|
||||
if norm.NFC.IsNormalString(s) && !norm.NFD.IsNormalString(s) {
|
||||
return "NFC"
|
||||
}
|
||||
return "OTHER"
|
||||
}
|
||||
|
||||
func TestTransformCopy(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
r := fstest.NewRun(t)
|
||||
err := transform.SetOptions(ctx, "all,suffix_keep_extension=_somesuffix")
|
||||
require.NoError(t, err)
|
||||
file1 := r.WriteFile("sub dir/hello world.txt", "hello world", t1)
|
||||
|
||||
r.Mkdir(ctx, r.Fremote)
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err = Sync(ctx, r.Fremote, r.Flocal, true)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
require.NoError(t, err)
|
||||
|
||||
r.CheckLocalItems(t, file1)
|
||||
r.CheckRemoteItems(t, fstest.NewItem("sub dir_somesuffix/hello world_somesuffix.txt", "hello world", t1))
|
||||
}
|
||||
|
||||
func TestDoubleTransform(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
r := fstest.NewRun(t)
|
||||
err := transform.SetOptions(ctx, "all,prefix=tac", "all,prefix=tic")
|
||||
require.NoError(t, err)
|
||||
file1 := r.WriteFile("toe/toe", "hello world", t1)
|
||||
|
||||
r.Mkdir(ctx, r.Fremote)
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err = Sync(ctx, r.Fremote, r.Flocal, true)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
require.NoError(t, err)
|
||||
|
||||
r.CheckLocalItems(t, file1)
|
||||
r.CheckRemoteItems(t, fstest.NewItem("tictactoe/tictactoe", "hello world", t1))
|
||||
}
|
||||
|
||||
func TestFileTag(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
r := fstest.NewRun(t)
|
||||
err := transform.SetOptions(ctx, "file,prefix=tac", "file,prefix=tic")
|
||||
require.NoError(t, err)
|
||||
file1 := r.WriteFile("toe/toe/toe", "hello world", t1)
|
||||
|
||||
r.Mkdir(ctx, r.Fremote)
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err = Sync(ctx, r.Fremote, r.Flocal, true)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
require.NoError(t, err)
|
||||
|
||||
r.CheckLocalItems(t, file1)
|
||||
r.CheckRemoteItems(t, fstest.NewItem("toe/toe/tictactoe", "hello world", t1))
|
||||
}
|
||||
|
||||
func TestNoTag(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
r := fstest.NewRun(t)
|
||||
err := transform.SetOptions(ctx, "prefix=tac", "prefix=tic")
|
||||
require.NoError(t, err)
|
||||
file1 := r.WriteFile("toe/toe/toe", "hello world", t1)
|
||||
|
||||
r.Mkdir(ctx, r.Fremote)
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err = Sync(ctx, r.Fremote, r.Flocal, true)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
require.NoError(t, err)
|
||||
|
||||
r.CheckLocalItems(t, file1)
|
||||
r.CheckRemoteItems(t, fstest.NewItem("toe/toe/tictactoe", "hello world", t1))
|
||||
}
|
||||
|
||||
func TestDirTag(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
r := fstest.NewRun(t)
|
||||
err := transform.SetOptions(ctx, "dir,prefix=tac", "dir,prefix=tic")
|
||||
require.NoError(t, err)
|
||||
r.WriteFile("toe/toe/toe.txt", "hello world", t1)
|
||||
_, err = operations.MkdirModTime(ctx, r.Flocal, "empty_dir", t1)
|
||||
require.NoError(t, err)
|
||||
|
||||
r.Mkdir(ctx, r.Fremote)
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err = Sync(ctx, r.Fremote, r.Flocal, true)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
require.NoError(t, err)
|
||||
|
||||
r.CheckLocalListing(t, []fstest.Item{fstest.NewItem("toe/toe/toe.txt", "hello world", t1)}, []string{"empty_dir", "toe", "toe/toe"})
|
||||
r.CheckRemoteListing(t, []fstest.Item{fstest.NewItem("tictactoe/tictactoe/toe.txt", "hello world", t1)}, []string{"tictacempty_dir", "tictactoe", "tictactoe/tictactoe"})
|
||||
}
|
||||
|
||||
func TestAllTag(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
r := fstest.NewRun(t)
|
||||
err := transform.SetOptions(ctx, "all,prefix=tac", "all,prefix=tic")
|
||||
require.NoError(t, err)
|
||||
r.WriteFile("toe/toe/toe.txt", "hello world", t1)
|
||||
_, err = operations.MkdirModTime(ctx, r.Flocal, "empty_dir", t1)
|
||||
require.NoError(t, err)
|
||||
|
||||
r.Mkdir(ctx, r.Fremote)
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err = Sync(ctx, r.Fremote, r.Flocal, true)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
require.NoError(t, err)
|
||||
|
||||
r.CheckLocalListing(t, []fstest.Item{fstest.NewItem("toe/toe/toe.txt", "hello world", t1)}, []string{"empty_dir", "toe", "toe/toe"})
|
||||
r.CheckRemoteListing(t, []fstest.Item{fstest.NewItem("tictactoe/tictactoe/tictactoe.txt", "hello world", t1)}, []string{"tictacempty_dir", "tictactoe", "tictactoe/tictactoe"})
|
||||
err = operations.Check(ctx, &operations.CheckOpt{Fsrc: r.Flocal, Fdst: r.Fremote}) // should not error even though dst has transformed names
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestRunTwice(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
r := fstest.NewRun(t)
|
||||
err := transform.SetOptions(ctx, "dir,prefix=tac", "dir,prefix=tic")
|
||||
require.NoError(t, err)
|
||||
file1 := r.WriteFile("toe/toe/toe.txt", "hello world", t1)
|
||||
|
||||
r.Mkdir(ctx, r.Fremote)
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err = Sync(ctx, r.Fremote, r.Flocal, true)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
require.NoError(t, err)
|
||||
|
||||
r.CheckLocalItems(t, file1)
|
||||
r.CheckRemoteItems(t, fstest.NewItem("tictactoe/tictactoe/toe.txt", "hello world", t1))
|
||||
|
||||
// result should not change second time, since src is unchanged
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err = Sync(ctx, r.Fremote, r.Flocal, true)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
require.NoError(t, err)
|
||||
|
||||
r.CheckLocalItems(t, file1)
|
||||
r.CheckRemoteItems(t, fstest.NewItem("tictactoe/tictactoe/toe.txt", "hello world", t1))
|
||||
}
|
||||
|
||||
func TestSyntax(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
err := transform.SetOptions(ctx, "prefix")
|
||||
assert.Error(t, err) // should error as required value is missing
|
||||
|
||||
err = transform.SetOptions(ctx, "banana")
|
||||
assert.Error(t, err) // should error as unrecognized option
|
||||
|
||||
err = transform.SetOptions(ctx, "=123")
|
||||
assert.Error(t, err) // should error as required key is missing
|
||||
|
||||
err = transform.SetOptions(ctx, "prefix=123")
|
||||
assert.NoError(t, err) // should not error
|
||||
}
|
||||
|
||||
func TestConflicting(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
r := fstest.NewRun(t)
|
||||
err := transform.SetOptions(ctx, "prefix=tac", "trimprefix=tac")
|
||||
require.NoError(t, err)
|
||||
file1 := r.WriteFile("toe/toe/toe", "hello world", t1)
|
||||
|
||||
r.Mkdir(ctx, r.Fremote)
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err = Sync(ctx, r.Fremote, r.Flocal, true)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
require.NoError(t, err)
|
||||
|
||||
// should result in no change as prefix and trimprefix cancel out
|
||||
r.CheckLocalItems(t, file1)
|
||||
r.CheckRemoteItems(t, fstest.NewItem("toe/toe/toe", "hello world", t1))
|
||||
}
|
||||
|
||||
func TestMove(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
r := fstest.NewRun(t)
|
||||
err := transform.SetOptions(ctx, "all,prefix=tac", "all,prefix=tic")
|
||||
require.NoError(t, err)
|
||||
r.WriteFile("toe/toe/toe.txt", "hello world", t1)
|
||||
_, err = operations.MkdirModTime(ctx, r.Flocal, "empty_dir", t1)
|
||||
require.NoError(t, err)
|
||||
|
||||
r.Mkdir(ctx, r.Fremote)
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err = MoveDir(ctx, r.Fremote, r.Flocal, true, true)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
require.NoError(t, err)
|
||||
|
||||
r.CheckLocalListing(t, []fstest.Item{}, []string{})
|
||||
r.CheckRemoteListing(t, []fstest.Item{fstest.NewItem("tictactoe/tictactoe/tictactoe.txt", "hello world", t1)}, []string{"tictacempty_dir", "tictactoe", "tictactoe/tictactoe"})
|
||||
}
|
||||
|
||||
func TestBase64(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
r := fstest.NewRun(t)
|
||||
err := transform.SetOptions(ctx, "all,base64encode")
|
||||
require.NoError(t, err)
|
||||
file1 := r.WriteFile("toe/toe/toe.txt", "hello world", t1)
|
||||
|
||||
r.Mkdir(ctx, r.Fremote)
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err = Sync(ctx, r.Fremote, r.Flocal, true)
|
||||
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
require.NoError(t, err)
|
||||
|
||||
r.CheckLocalItems(t, file1)
|
||||
r.CheckRemoteItems(t, fstest.NewItem("dG9l/dG9l/dG9lLnR4dA==", "hello world", t1))
|
||||
|
||||
// round trip
|
||||
err = transform.SetOptions(ctx, "all,base64decode")
|
||||
require.NoError(t, err)
|
||||
ctx = predictDstFromLogger(ctx)
|
||||
err = Sync(ctx, r.Flocal, r.Fremote, true)
|
||||
testLoggerVsLsf(ctx, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
require.NoError(t, err)
|
||||
|
||||
r.CheckLocalItems(t, file1)
|
||||
r.CheckRemoteItems(t, fstest.NewItem("dG9l/dG9l/dG9lLnR4dA==", "hello world", t1))
|
||||
}
|
||||
|
||||
func TestError(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
r := fstest.NewRun(t)
|
||||
err := transform.SetOptions(ctx, "all,prefix=ta/c") // has illegal character
|
||||
require.NoError(t, err)
|
||||
file1 := r.WriteFile("toe/toe/toe", "hello world", t1)
|
||||
|
||||
r.Mkdir(ctx, r.Fremote)
|
||||
// ctx = predictDstFromLogger(ctx)
|
||||
err = Sync(ctx, r.Fremote, r.Flocal, true)
|
||||
// testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
|
||||
assert.Error(t, err)
|
||||
|
||||
r.CheckLocalListing(t, []fstest.Item{file1}, []string{"toe", "toe/toe"})
|
||||
r.CheckRemoteListing(t, []fstest.Item{}, []string{})
|
||||
err = transform.SetOptions(ctx, "") // has illegal character
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
@@ -151,8 +151,8 @@ func init() {
|
||||
alias("Dot", EncodeDot)
|
||||
}
|
||||
|
||||
// validStrings returns all the valid MultiEncoder strings
|
||||
func validStrings() string {
|
||||
// ValidStrings returns all the valid MultiEncoder strings
|
||||
func ValidStrings() string {
|
||||
var out []string
|
||||
for k := range nameToEncoding {
|
||||
out = append(out, k)
|
||||
@@ -192,7 +192,7 @@ func (mask *MultiEncoder) Set(in string) error {
|
||||
} else {
|
||||
i, err := strconv.ParseUint(part, 0, 0)
|
||||
if err != nil {
|
||||
return fmt.Errorf("bad encoding %q: possible values are: %s", part, validStrings())
|
||||
return fmt.Errorf("bad encoding %q: possible values are: %s", part, ValidStrings())
|
||||
}
|
||||
out |= MultiEncoder(i)
|
||||
}
|
||||
@@ -313,8 +313,7 @@ func (mask MultiEncoder) Encode(in string) string {
|
||||
}
|
||||
if mask.Has(EncodeAsterisk) { // *
|
||||
switch r {
|
||||
case '*',
|
||||
'*':
|
||||
case '*', '*':
|
||||
return true
|
||||
}
|
||||
}
|
||||
@@ -346,64 +345,55 @@ func (mask MultiEncoder) Encode(in string) string {
|
||||
}
|
||||
if mask.Has(EncodeQuestion) { // ?
|
||||
switch r {
|
||||
case '?',
|
||||
'?':
|
||||
case '?', '?':
|
||||
return true
|
||||
}
|
||||
}
|
||||
if mask.Has(EncodeColon) { // :
|
||||
switch r {
|
||||
case ':',
|
||||
':':
|
||||
case ':', ':':
|
||||
return true
|
||||
}
|
||||
}
|
||||
if mask.Has(EncodePipe) { // |
|
||||
switch r {
|
||||
case '|',
|
||||
'|':
|
||||
case '|', '|':
|
||||
return true
|
||||
}
|
||||
}
|
||||
if mask.Has(EncodeDoubleQuote) { // "
|
||||
switch r {
|
||||
case '"',
|
||||
'"':
|
||||
case '"', '"':
|
||||
return true
|
||||
}
|
||||
}
|
||||
if mask.Has(EncodeSingleQuote) { // '
|
||||
switch r {
|
||||
case '\'',
|
||||
''':
|
||||
case '\'', ''':
|
||||
return true
|
||||
}
|
||||
}
|
||||
if mask.Has(EncodeBackQuote) { // `
|
||||
switch r {
|
||||
case '`',
|
||||
'`':
|
||||
case '`', '`':
|
||||
return true
|
||||
}
|
||||
}
|
||||
if mask.Has(EncodeDollar) { // $
|
||||
switch r {
|
||||
case '$',
|
||||
'$':
|
||||
case '$', '$':
|
||||
return true
|
||||
}
|
||||
}
|
||||
if mask.Has(EncodeSlash) { // /
|
||||
switch r {
|
||||
case '/',
|
||||
'/':
|
||||
case '/', '/':
|
||||
return true
|
||||
}
|
||||
}
|
||||
if mask.Has(EncodeBackSlash) { // \
|
||||
switch r {
|
||||
case '\\',
|
||||
'\':
|
||||
case '\\', '\':
|
||||
return true
|
||||
}
|
||||
}
|
||||
@@ -416,15 +406,13 @@ func (mask MultiEncoder) Encode(in string) string {
|
||||
}
|
||||
if mask.Has(EncodeHash) { // #
|
||||
switch r {
|
||||
case '#',
|
||||
'#':
|
||||
case '#', '#':
|
||||
return true
|
||||
}
|
||||
}
|
||||
if mask.Has(EncodePercent) { // %
|
||||
switch r {
|
||||
case '%',
|
||||
'%':
|
||||
case '%', '%':
|
||||
return true
|
||||
}
|
||||
}
|
||||
@@ -1182,6 +1170,7 @@ func appendQuotedBytes(w io.Writer, s string) {
|
||||
_, _ = fmt.Fprintf(w, string(QuoteRune)+"%02X", b)
|
||||
}
|
||||
}
|
||||
|
||||
func appendUnquotedByte(w io.Writer, s string) bool {
|
||||
if len(s) < 2 {
|
||||
return false
|
||||
@@ -1202,12 +1191,15 @@ func (identity) Decode(in string) string { return in }
|
||||
func (i identity) FromStandardPath(s string) string {
|
||||
return FromStandardPath(i, s)
|
||||
}
|
||||
|
||||
func (i identity) FromStandardName(s string) string {
|
||||
return FromStandardName(i, s)
|
||||
}
|
||||
|
||||
func (i identity) ToStandardPath(s string) string {
|
||||
return ToStandardPath(i, s)
|
||||
}
|
||||
|
||||
func (i identity) ToStandardName(s string) string {
|
||||
return ToStandardName(i, s)
|
||||
}
|
||||
|
||||
71
lib/transform/cmap.go
Normal file
71
lib/transform/cmap.go
Normal file
@@ -0,0 +1,71 @@
|
||||
package transform
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"golang.org/x/text/encoding/charmap"
|
||||
)
|
||||
|
||||
var (
|
||||
cmaps = map[int]*charmap.Charmap{}
|
||||
lock sync.Mutex
|
||||
)
|
||||
|
||||
type cmapChoices struct{}
|
||||
|
||||
func (cmapChoices) Choices() []string {
|
||||
choices := make([]string, 1)
|
||||
i := 0
|
||||
for _, enc := range charmap.All {
|
||||
c, ok := enc.(*charmap.Charmap)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
name := strings.ReplaceAll(c.String(), " ", "-")
|
||||
if name == "" {
|
||||
name = fmt.Sprintf("unknown-%d", i)
|
||||
}
|
||||
lock.Lock()
|
||||
cmaps[i] = c
|
||||
lock.Unlock()
|
||||
choices = append(choices, name)
|
||||
i++
|
||||
}
|
||||
return choices
|
||||
}
|
||||
|
||||
func (cmapChoices) Type() string {
|
||||
return "string"
|
||||
}
|
||||
|
||||
func charmapByID(cm fs.Enum[cmapChoices]) *charmap.Charmap {
|
||||
lock.Lock()
|
||||
c, ok := cmaps[int(cm)]
|
||||
lock.Unlock()
|
||||
if ok {
|
||||
return c
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func encodeWithReplacement(s string, cmap *charmap.Charmap) string {
|
||||
return strings.Map(func(r rune) rune {
|
||||
b, ok := cmap.EncodeRune(r)
|
||||
if !ok {
|
||||
return '_'
|
||||
}
|
||||
return cmap.DecodeByte(b)
|
||||
}, s)
|
||||
}
|
||||
|
||||
func toASCII(s string) string {
|
||||
return strings.Map(func(r rune) rune {
|
||||
if r <= 127 {
|
||||
return r
|
||||
}
|
||||
return -1
|
||||
}, s)
|
||||
}
|
||||
240
lib/transform/options.go
Normal file
240
lib/transform/options.go
Normal file
@@ -0,0 +1,240 @@
|
||||
package transform
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
)
|
||||
|
||||
func init() {
|
||||
fs.RegisterGlobalOptions(fs.OptionsInfo{Name: "name_transform", Opt: &Opt.Flags, Options: OptionsInfo, Reload: Reload})
|
||||
}
|
||||
|
||||
type transform struct {
|
||||
key transformAlgo // for example, "prefix"
|
||||
value string // for example, "some_prefix_"
|
||||
tag tag // file, dir, or all
|
||||
}
|
||||
|
||||
// Options stores the parsed and unparsed transform options.
|
||||
// their order must never be changed or sorted.
|
||||
type Options struct {
|
||||
Flags Flags // unparsed flag value like "file,prefix=ABC"
|
||||
transforms []transform // parsed from NameTransform
|
||||
}
|
||||
|
||||
// Flags is a slice of unparsed values set from command line flags or env vars
|
||||
type Flags struct {
|
||||
NameTransform []string `config:"name_transform"`
|
||||
}
|
||||
|
||||
// Opt is the default options modified by the environment variables and command line flags
|
||||
var Opt Options
|
||||
|
||||
// tag controls which part of the file path is affected (file, dir, all)
|
||||
type tag int
|
||||
|
||||
// tag modes
|
||||
const (
|
||||
file tag = iota // Only transform the leaf name of files (default)
|
||||
dir // Only transform name of directories - these may appear anywhere in the path
|
||||
all // Transform the entire path for files and directories
|
||||
)
|
||||
|
||||
// OptionsInfo describes the Options in use
|
||||
var OptionsInfo = fs.Options{{
|
||||
Name: "name_transform",
|
||||
Default: []string{},
|
||||
Help: "TODO",
|
||||
Groups: "Filter",
|
||||
}}
|
||||
|
||||
// Reload the transform options from the flags
|
||||
func Reload(ctx context.Context) (err error) {
|
||||
return newOpt(Opt)
|
||||
}
|
||||
|
||||
// SetOptions sets the options from flags passed in.
|
||||
// Any existing flags will be overwritten.
|
||||
// s should be in the same format as cmd line flags, i.e. "all,prefix=XXX"
|
||||
func SetOptions(ctx context.Context, s ...string) (err error) {
|
||||
Opt = Options{Flags: Flags{NameTransform: s}}
|
||||
return Reload(ctx)
|
||||
}
|
||||
|
||||
// overwite Opt.transforms with values from Opt.Flags
|
||||
func newOpt(opt Options) (err error) {
|
||||
Opt.transforms = []transform{}
|
||||
|
||||
for _, transform := range opt.Flags.NameTransform {
|
||||
t, err := parse(transform)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
Opt.transforms = append(Opt.transforms, t)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// parse a single instance of --name-transform
|
||||
func parse(s string) (t transform, err error) {
|
||||
if s == "" {
|
||||
return t, nil
|
||||
}
|
||||
s = t.parseTag(s)
|
||||
err = t.parseKeyVal(s)
|
||||
return t, err
|
||||
}
|
||||
|
||||
// parse the tag (file/dir/all), set the option accordingly, and return the trimmed string
|
||||
//
|
||||
// we don't worry about errors here because it will error anyway as an invalid key
|
||||
func (t *transform) parseTag(s string) string {
|
||||
if strings.HasPrefix(s, "file,") {
|
||||
t.tag = file
|
||||
return strings.TrimPrefix(s, "file,")
|
||||
}
|
||||
if strings.HasPrefix(s, "dir,") {
|
||||
t.tag = dir
|
||||
return strings.TrimPrefix(s, "dir,")
|
||||
}
|
||||
if strings.HasPrefix(s, "all,") {
|
||||
t.tag = all
|
||||
return strings.TrimPrefix(s, "all,")
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// parse key and value (if any) by splitting on '=' sign
|
||||
// (file/dir/all tag has already been trimmed)
|
||||
func (t *transform) parseKeyVal(s string) (err error) {
|
||||
if !strings.ContainsRune(s, '=') {
|
||||
err = t.key.Set(s)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if t.requiresValue() {
|
||||
fs.Debugf(nil, "received %v", s)
|
||||
return errors.New("value is required for " + t.key.String())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
split := strings.Split(s, "=")
|
||||
if len(split) != 2 {
|
||||
return errors.New("too many values")
|
||||
}
|
||||
if split[0] == "" {
|
||||
return errors.New("key cannot be blank")
|
||||
}
|
||||
err = t.key.Set(split[0])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
t.value = split[1]
|
||||
return nil
|
||||
}
|
||||
|
||||
// returns true if this particular algorithm requires a value
|
||||
func (t *transform) requiresValue() bool {
|
||||
switch t.key {
|
||||
case ConvFindReplace:
|
||||
return true
|
||||
case ConvPrefix:
|
||||
return true
|
||||
case ConvSuffix:
|
||||
return true
|
||||
case ConvSuffixKeepExtension:
|
||||
return true
|
||||
case ConvTrimPrefix:
|
||||
return true
|
||||
case ConvTrimSuffix:
|
||||
return true
|
||||
case ConvIndex:
|
||||
return true
|
||||
case ConvDate:
|
||||
return true
|
||||
case ConvTruncate:
|
||||
return true
|
||||
case ConvEncoder:
|
||||
return true
|
||||
case ConvDecoder:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// transformAlgo describes conversion setting
|
||||
type transformAlgo = fs.Enum[transformChoices]
|
||||
|
||||
// Supported transform options
|
||||
const (
|
||||
ConvNone transformAlgo = iota
|
||||
ConvToNFC
|
||||
ConvToNFD
|
||||
ConvToNFKC
|
||||
ConvToNFKD
|
||||
ConvFindReplace
|
||||
ConvPrefix
|
||||
ConvSuffix
|
||||
ConvSuffixKeepExtension
|
||||
ConvTrimPrefix
|
||||
ConvTrimSuffix
|
||||
ConvIndex
|
||||
ConvDate
|
||||
ConvTruncate
|
||||
ConvBase64Encode
|
||||
ConvBase64Decode
|
||||
ConvEncoder
|
||||
ConvDecoder
|
||||
ConvISO8859_1
|
||||
ConvWindows1252
|
||||
ConvMacintosh
|
||||
ConvCharmap
|
||||
ConvLowercase
|
||||
ConvUppercase
|
||||
ConvTitlecase
|
||||
ConvASCII
|
||||
ConvURL
|
||||
ConvMapper
|
||||
)
|
||||
|
||||
type transformChoices struct{}
|
||||
|
||||
func (transformChoices) Choices() []string {
|
||||
return []string{
|
||||
ConvNone: "none",
|
||||
ConvToNFC: "nfc",
|
||||
ConvToNFD: "nfd",
|
||||
ConvToNFKC: "nfkc",
|
||||
ConvToNFKD: "nfkd",
|
||||
ConvFindReplace: "replace",
|
||||
ConvPrefix: "prefix",
|
||||
ConvSuffix: "suffix",
|
||||
ConvSuffixKeepExtension: "suffix_keep_extension",
|
||||
ConvTrimPrefix: "trimprefix",
|
||||
ConvTrimSuffix: "trimsuffix",
|
||||
ConvIndex: "index",
|
||||
ConvDate: "date",
|
||||
ConvTruncate: "truncate",
|
||||
ConvBase64Encode: "base64encode",
|
||||
ConvBase64Decode: "base64decode",
|
||||
ConvEncoder: "encoder",
|
||||
ConvDecoder: "decoder",
|
||||
ConvISO8859_1: "ISO-8859-1",
|
||||
ConvWindows1252: "Windows-1252",
|
||||
ConvMacintosh: "Macintosh",
|
||||
ConvCharmap: "charmap",
|
||||
ConvLowercase: "lowercase",
|
||||
ConvUppercase: "uppercase",
|
||||
ConvTitlecase: "titlecase",
|
||||
ConvASCII: "ascii",
|
||||
ConvURL: "url",
|
||||
ConvMapper: "mapper",
|
||||
}
|
||||
}
|
||||
|
||||
func (transformChoices) Type() string {
|
||||
return "string"
|
||||
}
|
||||
226
lib/transform/transform.go
Normal file
226
lib/transform/transform.go
Normal file
@@ -0,0 +1,226 @@
|
||||
// Package transform holds functions for path name transformations
|
||||
package transform
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"mime"
|
||||
"os"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/lib/encoder"
|
||||
"golang.org/x/text/encoding/charmap"
|
||||
"golang.org/x/text/unicode/norm"
|
||||
)
|
||||
|
||||
// Path transforms a path s according to the --name-transform options in use
|
||||
//
|
||||
// If no transforms are in use, s is returned unchanged
|
||||
func Path(s string, isDir bool) string {
|
||||
if !Transforming() {
|
||||
return s
|
||||
}
|
||||
|
||||
var err error
|
||||
old := s
|
||||
for _, t := range Opt.transforms {
|
||||
if isDir && t.tag == file {
|
||||
continue
|
||||
}
|
||||
baseOnly := !isDir && t.tag == file
|
||||
if t.tag == dir && !isDir {
|
||||
s, err = transformDir(s, t)
|
||||
} else {
|
||||
s, err = transformPath(s, t, baseOnly)
|
||||
}
|
||||
if err != nil {
|
||||
fs.Error(s, err.Error()) // TODO: return err instead of logging it?
|
||||
}
|
||||
}
|
||||
if old != s {
|
||||
fs.Debugf(old, "transformed to: %v", s)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// Transforming returns true when transforms are in use
|
||||
func Transforming() bool {
|
||||
return len(Opt.transforms) > 0
|
||||
}
|
||||
|
||||
// transformPath transforms a path string according to the chosen TransformAlgo.
|
||||
// Each path segment is transformed separately, to preserve path separators.
|
||||
// If baseOnly is true, only the base will be transformed (useful for renaming while walking a dir tree recursively.)
|
||||
// for example, "some/nested/path" -> "some/nested/CONVERTEDPATH"
|
||||
// otherwise, the entire is path is transformed.
|
||||
func transformPath(s string, t transform, baseOnly bool) (string, error) {
|
||||
if s == "" || s == "/" || s == "\\" || s == "." {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
if baseOnly {
|
||||
transformedBase, err := transformPathSegment(path.Base(s), t)
|
||||
if err := validateSegment(transformedBase); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return path.Join(path.Dir(s), transformedBase), err
|
||||
}
|
||||
|
||||
segments := strings.Split(s, string(os.PathSeparator))
|
||||
transformedSegments := make([]string, len(segments))
|
||||
for _, seg := range segments {
|
||||
convSeg, err := transformPathSegment(seg, t)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := validateSegment(convSeg); err != nil {
|
||||
return "", err
|
||||
}
|
||||
transformedSegments = append(transformedSegments, convSeg)
|
||||
}
|
||||
return path.Join(transformedSegments...), nil
|
||||
}
|
||||
|
||||
// transform all but the last path segment
|
||||
func transformDir(s string, t transform) (string, error) {
|
||||
dirPath, err := transformPath(path.Dir(s), t, false)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return path.Join(dirPath, path.Base(s)), nil
|
||||
}
|
||||
|
||||
// transformPathSegment transforms one path segment (or really any string) according to the chosen TransformAlgo.
|
||||
// It assumes path separators have already been trimmed.
|
||||
func transformPathSegment(s string, t transform) (string, error) {
|
||||
switch t.key {
|
||||
case ConvNone:
|
||||
return s, nil
|
||||
case ConvToNFC:
|
||||
return norm.NFC.String(s), nil
|
||||
case ConvToNFD:
|
||||
return norm.NFD.String(s), nil
|
||||
case ConvToNFKC:
|
||||
return norm.NFKC.String(s), nil
|
||||
case ConvToNFKD:
|
||||
return norm.NFKD.String(s), nil
|
||||
case ConvBase64Encode:
|
||||
return base64.URLEncoding.EncodeToString([]byte(s)), nil // URLEncoding to avoid slashes
|
||||
case ConvBase64Decode:
|
||||
if s == ".DS_Store" {
|
||||
return s, nil
|
||||
}
|
||||
b, err := base64.URLEncoding.DecodeString(s)
|
||||
if err != nil {
|
||||
fs.Errorf(s, "base64 error")
|
||||
}
|
||||
return string(b), err
|
||||
case ConvFindReplace:
|
||||
split := strings.Split(t.value, ":")
|
||||
if len(split) != 2 {
|
||||
return s, fmt.Errorf("wrong number of values: %v", t.value)
|
||||
}
|
||||
return strings.ReplaceAll(s, split[0], split[1]), nil
|
||||
case ConvPrefix:
|
||||
return t.value + s, nil
|
||||
case ConvSuffix:
|
||||
return s + t.value, nil
|
||||
case ConvSuffixKeepExtension:
|
||||
return SuffixKeepExtension(s, t.value), nil
|
||||
case ConvTrimPrefix:
|
||||
return strings.TrimPrefix(s, t.value), nil
|
||||
case ConvTrimSuffix:
|
||||
return strings.TrimSuffix(s, t.value), nil
|
||||
case ConvTruncate:
|
||||
max, err := strconv.Atoi(t.value)
|
||||
if err != nil {
|
||||
return s, err
|
||||
}
|
||||
if max <= 0 {
|
||||
return s, nil
|
||||
}
|
||||
if utf8.RuneCountInString(s) <= max {
|
||||
return s, nil
|
||||
}
|
||||
runes := []rune(s)
|
||||
return string(runes[:max]), nil
|
||||
case ConvEncoder:
|
||||
var enc encoder.MultiEncoder
|
||||
err := enc.Set(t.value)
|
||||
if err != nil {
|
||||
return s, err
|
||||
}
|
||||
return enc.Encode(s), nil
|
||||
case ConvDecoder:
|
||||
var enc encoder.MultiEncoder
|
||||
err := enc.Set(t.value)
|
||||
if err != nil {
|
||||
return s, err
|
||||
}
|
||||
return enc.Decode(s), nil
|
||||
case ConvISO8859_1:
|
||||
return encodeWithReplacement(s, charmap.ISO8859_1), nil
|
||||
case ConvWindows1252:
|
||||
return encodeWithReplacement(s, charmap.Windows1252), nil
|
||||
case ConvMacintosh:
|
||||
return encodeWithReplacement(s, charmap.Macintosh), nil
|
||||
case ConvCharmap:
|
||||
var cmapType fs.Enum[cmapChoices]
|
||||
err := cmapType.Set(t.value)
|
||||
if err != nil {
|
||||
return s, err
|
||||
}
|
||||
c := charmapByID(cmapType)
|
||||
return encodeWithReplacement(s, c), nil
|
||||
case ConvLowercase:
|
||||
return strings.ToLower(s), nil
|
||||
case ConvUppercase:
|
||||
return strings.ToUpper(s), nil
|
||||
case ConvTitlecase:
|
||||
return strings.ToTitle(s), nil
|
||||
case ConvASCII:
|
||||
return toASCII(s), nil
|
||||
default:
|
||||
return "", errors.New("this option is not yet implemented")
|
||||
}
|
||||
}
|
||||
|
||||
// SuffixKeepExtension adds a suffix while keeping extension
|
||||
//
|
||||
// i.e. file.txt becomes file_somesuffix.txt not file.txt_somesuffix
|
||||
func SuffixKeepExtension(remote string, suffix string) string {
|
||||
var (
|
||||
base = remote
|
||||
exts = ""
|
||||
first = true
|
||||
ext = path.Ext(remote)
|
||||
)
|
||||
for ext != "" {
|
||||
// Look second and subsequent extensions in mime types.
|
||||
// If they aren't found then don't keep it as an extension.
|
||||
if !first && mime.TypeByExtension(ext) == "" {
|
||||
break
|
||||
}
|
||||
base = base[:len(base)-len(ext)]
|
||||
exts = ext + exts
|
||||
first = false
|
||||
ext = path.Ext(base)
|
||||
}
|
||||
return base + suffix + exts
|
||||
}
|
||||
|
||||
// forbid transformations that add/remove path separators
|
||||
func validateSegment(s string) error {
|
||||
if s == "" {
|
||||
return errors.New("transform cannot render path segments empty")
|
||||
}
|
||||
if strings.ContainsRune(s, '/') {
|
||||
return fmt.Errorf("transform cannot add path separators: %v", s)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
130
lib/transform/transform_test.go
Normal file
130
lib/transform/transform_test.go
Normal file
@@ -0,0 +1,130 @@
|
||||
package transform
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// sync tests are in fs/sync/sync_transform_test.go to avoid import cycle issues
|
||||
|
||||
func TestPath(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
path string
|
||||
want string
|
||||
}{
|
||||
{"", ""},
|
||||
{"toe/toe/toe", "tictactoe/tictactoe/tictactoe"},
|
||||
{"a/b/c", "tictaca/tictacb/tictacc"},
|
||||
} {
|
||||
err := SetOptions(context.Background(), "all,prefix=tac", "all,prefix=tic")
|
||||
require.NoError(t, err)
|
||||
|
||||
got := Path(test.path, false)
|
||||
assert.Equal(t, test.want, got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileTagOnFile(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
path string
|
||||
want string
|
||||
}{
|
||||
{"a/b/c.txt", "a/b/1c.txt"},
|
||||
} {
|
||||
err := SetOptions(context.Background(), "file,prefix=1")
|
||||
require.NoError(t, err)
|
||||
|
||||
got := Path(test.path, false)
|
||||
assert.Equal(t, test.want, got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDirTagOnFile(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
path string
|
||||
want string
|
||||
}{
|
||||
{"a/b/c.txt", "1a/1b/c.txt"},
|
||||
} {
|
||||
err := SetOptions(context.Background(), "dir,prefix=1")
|
||||
require.NoError(t, err)
|
||||
|
||||
got := Path(test.path, false)
|
||||
assert.Equal(t, test.want, got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAllTag(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
path string
|
||||
want string
|
||||
}{
|
||||
{"a/b/c.txt", "1a/1b/1c.txt"},
|
||||
} {
|
||||
err := SetOptions(context.Background(), "all,prefix=1")
|
||||
require.NoError(t, err)
|
||||
|
||||
got := Path(test.path, false)
|
||||
assert.Equal(t, test.want, got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileTagOnDir(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
path string
|
||||
want string
|
||||
}{
|
||||
{"a/b", "a/b"},
|
||||
} {
|
||||
err := SetOptions(context.Background(), "file,prefix=1")
|
||||
require.NoError(t, err)
|
||||
|
||||
got := Path(test.path, true)
|
||||
assert.Equal(t, test.want, got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDirTagOnDir(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
path string
|
||||
want string
|
||||
}{
|
||||
{"a/b", "1a/1b"},
|
||||
} {
|
||||
err := SetOptions(context.Background(), "dir,prefix=1")
|
||||
require.NoError(t, err)
|
||||
|
||||
got := Path(test.path, true)
|
||||
assert.Equal(t, test.want, got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVarious(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
path string
|
||||
want string
|
||||
flags []string
|
||||
}{
|
||||
{"stories/The Quick Brown Fox!.txt", "STORIES/THE QUICK BROWN FOX!.TXT", []string{"all,uppercase"}},
|
||||
{"stories/The Quick Brown Fox!.txt", "stories/The Slow Brown Turtle!.txt", []string{"all,replace=Fox:Turtle", "all,replace=Quick:Slow"}},
|
||||
{"stories/The Quick Brown Fox!.txt", "c3Rvcmllcw==/VGhlIFF1aWNrIEJyb3duIEZveCEudHh0", []string{"all,base64encode"}},
|
||||
{"c3Rvcmllcw==/VGhlIFF1aWNrIEJyb3duIEZveCEudHh0", "stories/The Quick Brown Fox!.txt", []string{"all,base64decode"}},
|
||||
{"stories/The Quick Brown 🦊 Fox Went to the Café!.txt", "stories/The Quick Brown 🦊 Fox Went to the Café!.txt", []string{"all,nfc"}},
|
||||
{"stories/The Quick Brown 🦊 Fox Went to the Café!.txt", "stories/The Quick Brown 🦊 Fox Went to the Café!.txt", []string{"all,nfd"}},
|
||||
{"stories/The Quick Brown 🦊 Fox!.txt", "stories/The Quick Brown Fox!.txt", []string{"all,ascii"}},
|
||||
{"stories/The Quick Brown Fox!.txt", "stories/The Quick Brown Fox!", []string{"all,trimsuffix=.txt"}},
|
||||
{"stories/The Quick Brown Fox!.txt", "OLD_stories/OLD_The Quick Brown Fox!.txt", []string{"all,prefix=OLD_"}},
|
||||
{"stories/The Quick Brown 🦊 Fox Went to the Café!.txt", "stories/The Quick Brown _ Fox Went to the Caf_!.txt", []string{"all,charmap=ISO-8859-7"}},
|
||||
{"stories/The Quick Brown Fox: A Memoir [draft].txt", "stories/The Quick Brown Fox: A Memoir [draft].txt", []string{"all,encoder=Colon,SquareBracket"}},
|
||||
{"stories/The Quick Brown 🦊 Fox Went to the Café!.txt", "stories/The Quick Brown 🦊 Fox", []string{"all,truncate=21"}},
|
||||
} {
|
||||
err := SetOptions(context.Background(), test.flags...)
|
||||
require.NoError(t, err)
|
||||
|
||||
got := Path(test.path, false)
|
||||
assert.Equal(t, test.want, got)
|
||||
}
|
||||
}
|
||||
14
lib/transform/transformflags/transformflags.go
Normal file
14
lib/transform/transformflags/transformflags.go
Normal file
@@ -0,0 +1,14 @@
|
||||
// Package transformflags implements command line flags to set up a transform
|
||||
package transformflags
|
||||
|
||||
import (
|
||||
"github.com/rclone/rclone/fs/config/flags"
|
||||
"github.com/rclone/rclone/lib/transform"
|
||||
|
||||
"github.com/spf13/pflag"
|
||||
)
|
||||
|
||||
// AddFlags adds the transform flags to the command
|
||||
func AddFlags(flagSet *pflag.FlagSet) {
|
||||
flags.AddFlagsFromOptions(flagSet, "", transform.OptionsInfo)
|
||||
}
|
||||
Reference in New Issue
Block a user