From 34a20555ca2ecadf9d008e7d3210a34057ea0f5c Mon Sep 17 00:00:00 2001 From: nielash Date: Sun, 4 May 2025 03:10:48 -0400 Subject: [PATCH] lib/transform lib/transform adds the transform library, supporting advanced path name transformations for converting and renaming files and directories by applying prefixes, suffixes, and other alterations. It also adds the --name-transform flag for use with sync, copy, and move. Multiple transformations can be used in sequence, applied in the order they are specified on the command line. By default --name-transform will only apply to file names. The means only the leaf file name will be transformed. However some of the transforms would be better applied to the whole path or just directories. To choose which which part of the file path is affected some tags can be added to the --name-transform: file Only transform the leaf name of files (DEFAULT) dir Only transform name of directories - these may appear anywhere in the path all Transform the entire path for files and directories Example syntax: --name-transform file,prefix=ABC --name-transform dir,prefix=DEF --- cmd/bisync/resolve.go | 23 +- cmd/help.go | 3 +- fs/march/march.go | 12 +- fs/operations/copy.go | 5 +- fs/operations/operations.go | 26 +- fs/sync/sync.go | 12 +- fs/sync/sync_test.go | 3 +- fs/sync/sync_transform_test.go | 457 ++++++++++++++++++ lib/encoder/encoder.go | 46 +- lib/transform/cmap.go | 71 +++ lib/transform/options.go | 240 +++++++++ lib/transform/transform.go | 226 +++++++++ lib/transform/transform_test.go | 130 +++++ .../transformflags/transformflags.go | 14 + 14 files changed, 1193 insertions(+), 75 deletions(-) create mode 100644 fs/sync/sync_transform_test.go create mode 100644 lib/transform/cmap.go create mode 100644 lib/transform/options.go create mode 100644 lib/transform/transform.go create mode 100644 lib/transform/transform_test.go create mode 100644 lib/transform/transformflags/transformflags.go diff --git a/cmd/bisync/resolve.go b/cmd/bisync/resolve.go index d058978f2..1a3d4e716 100644 --- a/cmd/bisync/resolve.go +++ b/cmd/bisync/resolve.go @@ -4,8 +4,6 @@ import ( "context" "fmt" "math" - "mime" - "path" "strings" "time" @@ -13,6 +11,7 @@ import ( "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/operations" "github.com/rclone/rclone/lib/terminal" + "github.com/rclone/rclone/lib/transform" ) // Prefer describes strategies for resolving sync conflicts @@ -130,6 +129,7 @@ type ( path2 namePair } ) + type namePair struct { oldName string newName string @@ -240,24 +240,7 @@ func SuffixName(ctx context.Context, remote, suffix string) string { } ci := fs.GetConfig(ctx) if ci.SuffixKeepExtension { - var ( - base = remote - exts = "" - first = true - ext = path.Ext(remote) - ) - for ext != "" { - // Look second and subsequent extensions in mime types. - // If they aren't found then don't keep it as an extension. - if !first && mime.TypeByExtension(ext) == "" { - break - } - base = base[:len(base)-len(ext)] - exts = ext + exts - first = false - ext = path.Ext(base) - } - return base + suffix + exts + return transform.SuffixKeepExtension(remote, suffix) } return remote + suffix } diff --git a/cmd/help.go b/cmd/help.go index f65ff6d53..c2f7c2b37 100644 --- a/cmd/help.go +++ b/cmd/help.go @@ -16,6 +16,7 @@ import ( "github.com/rclone/rclone/fs/log/logflags" "github.com/rclone/rclone/fs/rc/rcflags" "github.com/rclone/rclone/lib/atexit" + "github.com/rclone/rclone/lib/transform/transformflags" "github.com/spf13/cobra" "github.com/spf13/pflag" "golang.org/x/text/cases" @@ -137,6 +138,7 @@ func setupRootCommand(rootCmd *cobra.Command) { // Add global flags configflags.AddFlags(ci, pflag.CommandLine) filterflags.AddFlags(pflag.CommandLine) + transformflags.AddFlags(pflag.CommandLine) rcflags.AddFlags(pflag.CommandLine) logflags.AddFlags(pflag.CommandLine) @@ -191,7 +193,6 @@ func setupRootCommand(rootCmd *cobra.Command) { }) cobra.OnInitialize(initConfig) - } // Traverse the tree of commands running fn on each diff --git a/fs/march/march.go b/fs/march/march.go index b6da6c3fc..819f35cf3 100644 --- a/fs/march/march.go +++ b/fs/march/march.go @@ -15,6 +15,7 @@ import ( "github.com/rclone/rclone/fs/filter" "github.com/rclone/rclone/fs/list" "github.com/rclone/rclone/fs/walk" + "github.com/rclone/rclone/lib/transform" "golang.org/x/sync/errgroup" "golang.org/x/text/unicode/norm" ) @@ -86,6 +87,7 @@ func (m *March) srcKey(entry fs.DirEntry) string { return "" } name := path.Base(entry.Remote()) + name = transform.Path(name, fs.DirEntryType(entry) == "directory") for _, transform := range m.transforms { name = transform(name) } @@ -94,7 +96,14 @@ func (m *March) srcKey(entry fs.DirEntry) string { // dstKey turns a directory entry into a sort key using the defined transforms. func (m *March) dstKey(entry fs.DirEntry) string { - return m.srcKey(entry) // FIXME actually do something different + if entry == nil { + return "" + } + name := path.Base(entry.Remote()) + for _, transform := range m.transforms { + name = transform(name) + } + return name } // makeListDir makes constructs a listing function for the given fs @@ -454,7 +463,6 @@ func (m *March) processJob(job listDirJob) ([]listDirJob, error) { noDst: true, }) } - }, func(dst fs.DirEntry) { recurse := m.Callback.DstOnly(dst) if recurse && job.dstDepth > 0 { diff --git a/fs/operations/copy.go b/fs/operations/copy.go index 8228a14c7..967fd69f5 100644 --- a/fs/operations/copy.go +++ b/fs/operations/copy.go @@ -21,6 +21,7 @@ import ( "github.com/rclone/rclone/fs/hash" "github.com/rclone/rclone/lib/atexit" "github.com/rclone/rclone/lib/pacer" + "github.com/rclone/rclone/lib/transform" ) // State of the copy @@ -390,7 +391,7 @@ func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Obj f: f, dstFeatures: f.Features(), dst: dst, - remote: remote, + remote: transform.Path(remote, false), src: src, ci: ci, tr: tr, @@ -399,7 +400,7 @@ func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Obj } c.hashType, c.hashOption = CommonHash(ctx, f, src.Fs()) if c.dst != nil { - c.remote = c.dst.Remote() + c.remote = transform.Path(c.dst.Remote(), false) } // Are we using partials? // diff --git a/fs/operations/operations.go b/fs/operations/operations.go index 6d3ca4f76..ca1e8ee0a 100644 --- a/fs/operations/operations.go +++ b/fs/operations/operations.go @@ -39,6 +39,7 @@ import ( "github.com/rclone/rclone/lib/pacer" "github.com/rclone/rclone/lib/random" "github.com/rclone/rclone/lib/readers" + "github.com/rclone/rclone/lib/transform" "golang.org/x/sync/errgroup" "golang.org/x/text/unicode/norm" ) @@ -424,6 +425,8 @@ func MoveTransfer(ctx context.Context, fdst fs.Fs, dst fs.Object, remote string, // move - see Move for help func move(ctx context.Context, fdst fs.Fs, dst fs.Object, remote string, src fs.Object, isTransfer bool) (newDst fs.Object, err error) { + origRemote := remote // avoid double-transform on fallback to copy + remote = transform.Path(remote, false) ci := fs.GetConfig(ctx) var tr *accounting.Transfer if isTransfer { @@ -447,7 +450,7 @@ func move(ctx context.Context, fdst fs.Fs, dst fs.Object, remote string, src fs. if doMove := fdst.Features().Move; doMove != nil && (SameConfig(src.Fs(), fdst) || (SameRemoteType(src.Fs(), fdst) && (fdst.Features().ServerSideAcrossConfigs || ci.ServerSideAcrossConfigs))) { // Delete destination if it exists and is not the same file as src (could be same file while seemingly different if the remote is case insensitive) if dst != nil { - remote = dst.Remote() + remote = transform.Path(dst.Remote(), false) if !SameObject(src, dst) { err = DeleteFile(ctx, dst) if err != nil { @@ -488,7 +491,7 @@ func move(ctx context.Context, fdst fs.Fs, dst fs.Object, remote string, src fs. } } // Move not found or didn't work so copy dst <- src - newDst, err = Copy(ctx, fdst, dst, remote, src) + newDst, err = Copy(ctx, fdst, dst, origRemote, src) if err != nil { fs.Errorf(src, "Not deleting source as copy failed: %v", err) return newDst, err @@ -516,24 +519,7 @@ func SuffixName(ctx context.Context, remote string) string { return remote } if ci.SuffixKeepExtension { - var ( - base = remote - exts = "" - first = true - ext = path.Ext(remote) - ) - for ext != "" { - // Look second and subsequent extensions in mime types. - // If they aren't found then don't keep it as an extension. - if !first && mime.TypeByExtension(ext) == "" { - break - } - base = base[:len(base)-len(ext)] - exts = ext + exts - first = false - ext = path.Ext(base) - } - return base + ci.Suffix + exts + return transform.SuffixKeepExtension(remote, ci.Suffix) } return remote + ci.Suffix } diff --git a/fs/sync/sync.go b/fs/sync/sync.go index 0e015769e..008a25ed0 100644 --- a/fs/sync/sync.go +++ b/fs/sync/sync.go @@ -20,6 +20,7 @@ import ( "github.com/rclone/rclone/fs/march" "github.com/rclone/rclone/fs/operations" "github.com/rclone/rclone/lib/errcount" + "github.com/rclone/rclone/lib/transform" "golang.org/x/sync/errgroup" ) @@ -1254,8 +1255,8 @@ func (s *syncCopyMove) SrcOnly(src fs.DirEntry) (recurse bool) { s.logger(s.ctx, operations.MissingOnDst, src, nil, fs.ErrorIsDir) // Create the directory and make sure the Metadata/ModTime is correct - s.copyDirMetadata(s.ctx, s.fdst, nil, x.Remote(), x) - s.markDirModified(x.Remote()) + s.copyDirMetadata(s.ctx, s.fdst, nil, transform.Path(x.Remote(), true), x) + s.markDirModified(transform.Path(x.Remote(), true)) return true default: panic("Bad object in DirEntries") @@ -1288,6 +1289,8 @@ func (s *syncCopyMove) Match(ctx context.Context, dst, src fs.DirEntry) (recurse } case fs.Directory: // Do the same thing to the entire contents of the directory + srcX = fs.NewOverrideDirectory(srcX, transform.Path(src.Remote(), true)) + src = srcX s.markParentNotEmpty(src) dstX, ok := dst.(fs.Directory) if ok { @@ -1372,6 +1375,11 @@ func moveDir(ctx context.Context, fdst, fsrc fs.Fs, deleteEmptySrcDirs bool, cop return runSyncCopyMove(ctx, fdst, fsrc, fs.DeleteModeOff, true, deleteEmptySrcDirs, copyEmptySrcDirs) } +// Transform renames fdst in place +func Transform(ctx context.Context, fdst fs.Fs, deleteEmptySrcDirs bool, copyEmptySrcDirs bool) error { + return runSyncCopyMove(ctx, fdst, fdst, fs.DeleteModeOff, true, deleteEmptySrcDirs, copyEmptySrcDirs) +} + // MoveDir moves fsrc into fdst func MoveDir(ctx context.Context, fdst, fsrc fs.Fs, deleteEmptySrcDirs bool, copyEmptySrcDirs bool) error { fi := filter.GetConfig(ctx) diff --git a/fs/sync/sync_test.go b/fs/sync/sync_test.go index 4c6407190..a921baf35 100644 --- a/fs/sync/sync_test.go +++ b/fs/sync/sync_test.go @@ -27,6 +27,7 @@ import ( "github.com/rclone/rclone/fs/hash" "github.com/rclone/rclone/fs/operations" "github.com/rclone/rclone/fstest" + "github.com/rclone/rclone/lib/transform" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "golang.org/x/text/unicode/norm" @@ -2980,7 +2981,7 @@ func predictDstFromLogger(ctx context.Context) context.Context { if winner.Err != nil { errMsg = ";" + winner.Err.Error() } - operations.SyncFprintf(opt.JSON, "%s;%s;%v;%s%s\n", file.ModTime(ctx).Local().Format(timeFormat), checksum, file.Size(), file.Remote(), errMsg) + operations.SyncFprintf(opt.JSON, "%s;%s;%v;%s%s\n", file.ModTime(ctx).Local().Format(timeFormat), checksum, file.Size(), transform.Path(file.Remote(), false), errMsg) // TODO: should the transform be handled in the sync instead of here? } } return operations.WithSyncLogger(ctx, opt) diff --git a/fs/sync/sync_transform_test.go b/fs/sync/sync_transform_test.go new file mode 100644 index 000000000..892b040b1 --- /dev/null +++ b/fs/sync/sync_transform_test.go @@ -0,0 +1,457 @@ +// Test transform + +package sync + +import ( + "cmp" + "context" + "fmt" + "path/filepath" + "slices" + "strings" + "testing" + + _ "github.com/rclone/rclone/backend/all" + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/filter" + "github.com/rclone/rclone/fs/operations" + "github.com/rclone/rclone/fs/walk" + "github.com/rclone/rclone/fstest" + "github.com/rclone/rclone/lib/transform" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/text/unicode/norm" +) + +var debug = `` + +func TestTransform(t *testing.T) { + type args struct { + TransformOpt []string + TransformBackOpt []string + Lossless bool // whether the TransformBackAlgo is always losslessly invertible + } + tests := []struct { + name string + args args + }{ + {name: "NFC", args: args{ + TransformOpt: []string{"nfc"}, + TransformBackOpt: []string{"nfd"}, + Lossless: false, + }}, + {name: "NFD", args: args{ + TransformOpt: []string{"nfd"}, + TransformBackOpt: []string{"nfc"}, + Lossless: false, + }}, + {name: "base64", args: args{ + TransformOpt: []string{"base64encode"}, + TransformBackOpt: []string{"base64encode"}, + Lossless: false, + }}, + {name: "prefix", args: args{ + TransformOpt: []string{"prefix=PREFIX"}, + TransformBackOpt: []string{"trimprefix=PREFIX"}, + Lossless: true, + }}, + {name: "suffix", args: args{ + TransformOpt: []string{"suffix=SUFFIX"}, + TransformBackOpt: []string{"trimsuffix=SUFFIX"}, + Lossless: true, + }}, + {name: "truncate", args: args{ + TransformOpt: []string{"truncate=10"}, + TransformBackOpt: []string{"truncate=10"}, + Lossless: false, + }}, + {name: "encoder", args: args{ + TransformOpt: []string{"encoder=Colon,SquareBracket"}, + TransformBackOpt: []string{"decoder=Colon,SquareBracket"}, + Lossless: true, + }}, + {name: "ISO-8859-1", args: args{ + TransformOpt: []string{"ISO-8859-1"}, + TransformBackOpt: []string{"ISO-8859-1"}, + Lossless: false, + }}, + {name: "charmap", args: args{ + TransformOpt: []string{"all,charmap=ISO-8859-7"}, + TransformBackOpt: []string{"all,charmap=ISO-8859-7"}, + Lossless: false, + }}, + {name: "lowercase", args: args{ + TransformOpt: []string{"all,lowercase"}, + TransformBackOpt: []string{"all,lowercase"}, + Lossless: false, + }}, + {name: "ascii", args: args{ + TransformOpt: []string{"all,ascii"}, + TransformBackOpt: []string{"all,ascii"}, + Lossless: false, + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := fstest.NewRun(t) + defer r.Finalise() + + r.Mkdir(context.Background(), r.Flocal) + r.Mkdir(context.Background(), r.Fremote) + items := makeTestFiles(t, r, "dir1") + deleteDSStore(t, r) + r.CheckRemoteListing(t, items, nil) + r.CheckLocalListing(t, items, nil) + + err := transform.SetOptions(context.Background(), tt.args.TransformOpt...) + require.NoError(t, err) + + err = Sync(context.Background(), r.Fremote, r.Flocal, true) + assert.NoError(t, err) + compareNames(t, r, items) + + err = transform.SetOptions(context.Background(), tt.args.TransformBackOpt...) + require.NoError(t, err) + err = Sync(context.Background(), r.Fremote, r.Flocal, true) + assert.NoError(t, err) + compareNames(t, r, items) + + if tt.args.Lossless { + deleteDSStore(t, r) + r.CheckRemoteItems(t, items...) + } + }) + } +} + +const alphabet = "abcdefg123456789" + +var extras = []string{"apple", "banana", "appleappleapplebanana", "splitbananasplit"} + +func makeTestFiles(t *testing.T, r *fstest.Run, dir string) []fstest.Item { + t.Helper() + n := 0 + // Create test files + items := []fstest.Item{} + for _, c := range alphabet { + var out strings.Builder + for i := rune(0); i < 7; i++ { + out.WriteRune(c + i) + } + fileName := filepath.Join(dir, fmt.Sprintf("%04d-%s.txt", n, out.String())) + fileName = strings.ToValidUTF8(fileName, "") + + if debug != "" { + fileName = debug + } + + item := r.WriteObject(context.Background(), fileName, fileName, t1) + r.WriteFile(fileName, fileName, t1) + items = append(items, item) + n++ + + if debug != "" { + break + } + } + + for _, extra := range extras { + item := r.WriteObject(context.Background(), extra, extra, t1) + r.WriteFile(extra, extra, t1) + items = append(items, item) + } + + return items +} + +func deleteDSStore(t *testing.T, r *fstest.Run) { + ctxDSStore, fi := filter.AddConfig(context.Background()) + err := fi.AddRule(`+ *.DS_Store`) + assert.NoError(t, err) + err = fi.AddRule(`- **`) + assert.NoError(t, err) + err = operations.Delete(ctxDSStore, r.Fremote) + assert.NoError(t, err) +} + +func compareNames(t *testing.T, r *fstest.Run, items []fstest.Item) { + var entries fs.DirEntries + + deleteDSStore(t, r) + err := walk.ListR(context.Background(), r.Fremote, "", true, -1, walk.ListObjects, func(e fs.DirEntries) error { + entries = append(entries, e...) + return nil + }) + assert.NoError(t, err) + entries = slices.DeleteFunc(entries, func(E fs.DirEntry) bool { // remove those pesky .DS_Store files + if strings.Contains(E.Remote(), ".DS_Store") { + err := operations.DeleteFile(context.Background(), E.(fs.Object)) + assert.NoError(t, err) + return true + } + return false + }) + require.Equal(t, len(items), entries.Len()) + + // sort by CONVERTED name + slices.SortStableFunc(items, func(a, b fstest.Item) int { + aConv := transform.Path(a.Path, false) + bConv := transform.Path(b.Path, false) + return cmp.Compare(aConv, bConv) + }) + slices.SortStableFunc(entries, func(a, b fs.DirEntry) int { + return cmp.Compare(a.Remote(), b.Remote()) + }) + + for i, e := range entries { + expect := transform.Path(items[i].Path, false) + msg := fmt.Sprintf("expected %v, got %v", detectEncoding(expect), detectEncoding(e.Remote())) + assert.Equal(t, expect, e.Remote(), msg) + } +} + +func detectEncoding(s string) string { + if norm.NFC.IsNormalString(s) && norm.NFD.IsNormalString(s) { + return "BOTH" + } + if !norm.NFC.IsNormalString(s) && norm.NFD.IsNormalString(s) { + return "NFD" + } + if norm.NFC.IsNormalString(s) && !norm.NFD.IsNormalString(s) { + return "NFC" + } + return "OTHER" +} + +func TestTransformCopy(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + err := transform.SetOptions(ctx, "all,suffix_keep_extension=_somesuffix") + require.NoError(t, err) + file1 := r.WriteFile("sub dir/hello world.txt", "hello world", t1) + + r.Mkdir(ctx, r.Fremote) + ctx = predictDstFromLogger(ctx) + err = Sync(ctx, r.Fremote, r.Flocal, true) + testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t) + require.NoError(t, err) + + r.CheckLocalItems(t, file1) + r.CheckRemoteItems(t, fstest.NewItem("sub dir_somesuffix/hello world_somesuffix.txt", "hello world", t1)) +} + +func TestDoubleTransform(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + err := transform.SetOptions(ctx, "all,prefix=tac", "all,prefix=tic") + require.NoError(t, err) + file1 := r.WriteFile("toe/toe", "hello world", t1) + + r.Mkdir(ctx, r.Fremote) + ctx = predictDstFromLogger(ctx) + err = Sync(ctx, r.Fremote, r.Flocal, true) + testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t) + require.NoError(t, err) + + r.CheckLocalItems(t, file1) + r.CheckRemoteItems(t, fstest.NewItem("tictactoe/tictactoe", "hello world", t1)) +} + +func TestFileTag(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + err := transform.SetOptions(ctx, "file,prefix=tac", "file,prefix=tic") + require.NoError(t, err) + file1 := r.WriteFile("toe/toe/toe", "hello world", t1) + + r.Mkdir(ctx, r.Fremote) + ctx = predictDstFromLogger(ctx) + err = Sync(ctx, r.Fremote, r.Flocal, true) + testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t) + require.NoError(t, err) + + r.CheckLocalItems(t, file1) + r.CheckRemoteItems(t, fstest.NewItem("toe/toe/tictactoe", "hello world", t1)) +} + +func TestNoTag(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + err := transform.SetOptions(ctx, "prefix=tac", "prefix=tic") + require.NoError(t, err) + file1 := r.WriteFile("toe/toe/toe", "hello world", t1) + + r.Mkdir(ctx, r.Fremote) + ctx = predictDstFromLogger(ctx) + err = Sync(ctx, r.Fremote, r.Flocal, true) + testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t) + require.NoError(t, err) + + r.CheckLocalItems(t, file1) + r.CheckRemoteItems(t, fstest.NewItem("toe/toe/tictactoe", "hello world", t1)) +} + +func TestDirTag(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + err := transform.SetOptions(ctx, "dir,prefix=tac", "dir,prefix=tic") + require.NoError(t, err) + r.WriteFile("toe/toe/toe.txt", "hello world", t1) + _, err = operations.MkdirModTime(ctx, r.Flocal, "empty_dir", t1) + require.NoError(t, err) + + r.Mkdir(ctx, r.Fremote) + ctx = predictDstFromLogger(ctx) + err = Sync(ctx, r.Fremote, r.Flocal, true) + testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t) + require.NoError(t, err) + + r.CheckLocalListing(t, []fstest.Item{fstest.NewItem("toe/toe/toe.txt", "hello world", t1)}, []string{"empty_dir", "toe", "toe/toe"}) + r.CheckRemoteListing(t, []fstest.Item{fstest.NewItem("tictactoe/tictactoe/toe.txt", "hello world", t1)}, []string{"tictacempty_dir", "tictactoe", "tictactoe/tictactoe"}) +} + +func TestAllTag(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + err := transform.SetOptions(ctx, "all,prefix=tac", "all,prefix=tic") + require.NoError(t, err) + r.WriteFile("toe/toe/toe.txt", "hello world", t1) + _, err = operations.MkdirModTime(ctx, r.Flocal, "empty_dir", t1) + require.NoError(t, err) + + r.Mkdir(ctx, r.Fremote) + ctx = predictDstFromLogger(ctx) + err = Sync(ctx, r.Fremote, r.Flocal, true) + testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t) + require.NoError(t, err) + + r.CheckLocalListing(t, []fstest.Item{fstest.NewItem("toe/toe/toe.txt", "hello world", t1)}, []string{"empty_dir", "toe", "toe/toe"}) + r.CheckRemoteListing(t, []fstest.Item{fstest.NewItem("tictactoe/tictactoe/tictactoe.txt", "hello world", t1)}, []string{"tictacempty_dir", "tictactoe", "tictactoe/tictactoe"}) + err = operations.Check(ctx, &operations.CheckOpt{Fsrc: r.Flocal, Fdst: r.Fremote}) // should not error even though dst has transformed names + assert.NoError(t, err) +} + +func TestRunTwice(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + err := transform.SetOptions(ctx, "dir,prefix=tac", "dir,prefix=tic") + require.NoError(t, err) + file1 := r.WriteFile("toe/toe/toe.txt", "hello world", t1) + + r.Mkdir(ctx, r.Fremote) + ctx = predictDstFromLogger(ctx) + err = Sync(ctx, r.Fremote, r.Flocal, true) + testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t) + require.NoError(t, err) + + r.CheckLocalItems(t, file1) + r.CheckRemoteItems(t, fstest.NewItem("tictactoe/tictactoe/toe.txt", "hello world", t1)) + + // result should not change second time, since src is unchanged + ctx = predictDstFromLogger(ctx) + err = Sync(ctx, r.Fremote, r.Flocal, true) + testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t) + require.NoError(t, err) + + r.CheckLocalItems(t, file1) + r.CheckRemoteItems(t, fstest.NewItem("tictactoe/tictactoe/toe.txt", "hello world", t1)) +} + +func TestSyntax(t *testing.T) { + ctx := context.Background() + err := transform.SetOptions(ctx, "prefix") + assert.Error(t, err) // should error as required value is missing + + err = transform.SetOptions(ctx, "banana") + assert.Error(t, err) // should error as unrecognized option + + err = transform.SetOptions(ctx, "=123") + assert.Error(t, err) // should error as required key is missing + + err = transform.SetOptions(ctx, "prefix=123") + assert.NoError(t, err) // should not error +} + +func TestConflicting(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + err := transform.SetOptions(ctx, "prefix=tac", "trimprefix=tac") + require.NoError(t, err) + file1 := r.WriteFile("toe/toe/toe", "hello world", t1) + + r.Mkdir(ctx, r.Fremote) + ctx = predictDstFromLogger(ctx) + err = Sync(ctx, r.Fremote, r.Flocal, true) + testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t) + require.NoError(t, err) + + // should result in no change as prefix and trimprefix cancel out + r.CheckLocalItems(t, file1) + r.CheckRemoteItems(t, fstest.NewItem("toe/toe/toe", "hello world", t1)) +} + +func TestMove(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + err := transform.SetOptions(ctx, "all,prefix=tac", "all,prefix=tic") + require.NoError(t, err) + r.WriteFile("toe/toe/toe.txt", "hello world", t1) + _, err = operations.MkdirModTime(ctx, r.Flocal, "empty_dir", t1) + require.NoError(t, err) + + r.Mkdir(ctx, r.Fremote) + ctx = predictDstFromLogger(ctx) + err = MoveDir(ctx, r.Fremote, r.Flocal, true, true) + testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t) + require.NoError(t, err) + + r.CheckLocalListing(t, []fstest.Item{}, []string{}) + r.CheckRemoteListing(t, []fstest.Item{fstest.NewItem("tictactoe/tictactoe/tictactoe.txt", "hello world", t1)}, []string{"tictacempty_dir", "tictactoe", "tictactoe/tictactoe"}) +} + +func TestBase64(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + err := transform.SetOptions(ctx, "all,base64encode") + require.NoError(t, err) + file1 := r.WriteFile("toe/toe/toe.txt", "hello world", t1) + + r.Mkdir(ctx, r.Fremote) + ctx = predictDstFromLogger(ctx) + err = Sync(ctx, r.Fremote, r.Flocal, true) + testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t) + require.NoError(t, err) + + r.CheckLocalItems(t, file1) + r.CheckRemoteItems(t, fstest.NewItem("dG9l/dG9l/dG9lLnR4dA==", "hello world", t1)) + + // round trip + err = transform.SetOptions(ctx, "all,base64decode") + require.NoError(t, err) + ctx = predictDstFromLogger(ctx) + err = Sync(ctx, r.Flocal, r.Fremote, true) + testLoggerVsLsf(ctx, r.Flocal, operations.GetLoggerOpt(ctx).JSON, t) + require.NoError(t, err) + + r.CheckLocalItems(t, file1) + r.CheckRemoteItems(t, fstest.NewItem("dG9l/dG9l/dG9lLnR4dA==", "hello world", t1)) +} + +func TestError(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + err := transform.SetOptions(ctx, "all,prefix=ta/c") // has illegal character + require.NoError(t, err) + file1 := r.WriteFile("toe/toe/toe", "hello world", t1) + + r.Mkdir(ctx, r.Fremote) + // ctx = predictDstFromLogger(ctx) + err = Sync(ctx, r.Fremote, r.Flocal, true) + // testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t) + assert.Error(t, err) + + r.CheckLocalListing(t, []fstest.Item{file1}, []string{"toe", "toe/toe"}) + r.CheckRemoteListing(t, []fstest.Item{}, []string{}) + err = transform.SetOptions(ctx, "") // has illegal character + assert.NoError(t, err) +} diff --git a/lib/encoder/encoder.go b/lib/encoder/encoder.go index b1ff541a3..c8f41d76c 100644 --- a/lib/encoder/encoder.go +++ b/lib/encoder/encoder.go @@ -151,8 +151,8 @@ func init() { alias("Dot", EncodeDot) } -// validStrings returns all the valid MultiEncoder strings -func validStrings() string { +// ValidStrings returns all the valid MultiEncoder strings +func ValidStrings() string { var out []string for k := range nameToEncoding { out = append(out, k) @@ -192,7 +192,7 @@ func (mask *MultiEncoder) Set(in string) error { } else { i, err := strconv.ParseUint(part, 0, 0) if err != nil { - return fmt.Errorf("bad encoding %q: possible values are: %s", part, validStrings()) + return fmt.Errorf("bad encoding %q: possible values are: %s", part, ValidStrings()) } out |= MultiEncoder(i) } @@ -313,8 +313,7 @@ func (mask MultiEncoder) Encode(in string) string { } if mask.Has(EncodeAsterisk) { // * switch r { - case '*', - '＊': + case '*', '＊': return true } } @@ -346,64 +345,55 @@ func (mask MultiEncoder) Encode(in string) string { } if mask.Has(EncodeQuestion) { // ? switch r { - case '?', - '？': + case '?', '？': return true } } if mask.Has(EncodeColon) { // : switch r { - case ':', - '：': + case ':', '：': return true } } if mask.Has(EncodePipe) { // | switch r { - case '|', - '｜': + case '|', '｜': return true } } if mask.Has(EncodeDoubleQuote) { // " switch r { - case '"', - '＂': + case '"', '＂': return true } } if mask.Has(EncodeSingleQuote) { // ' switch r { - case '\'', - '＇': + case '\'', '＇': return true } } if mask.Has(EncodeBackQuote) { // ` switch r { - case '`', - '｀': + case '`', '｀': return true } } if mask.Has(EncodeDollar) { // $ switch r { - case '$', - '＄': + case '$', '＄': return true } } if mask.Has(EncodeSlash) { // / switch r { - case '/', - '／': + case '/', '／': return true } } if mask.Has(EncodeBackSlash) { // \ switch r { - case '\\', - '＼': + case '\\', '＼': return true } } @@ -416,15 +406,13 @@ func (mask MultiEncoder) Encode(in string) string { } if mask.Has(EncodeHash) { // # switch r { - case '#', - '＃': + case '#', '＃': return true } } if mask.Has(EncodePercent) { // % switch r { - case '%', - '％': + case '%', '％': return true } } @@ -1182,6 +1170,7 @@ func appendQuotedBytes(w io.Writer, s string) { _, _ = fmt.Fprintf(w, string(QuoteRune)+"%02X", b) } } + func appendUnquotedByte(w io.Writer, s string) bool { if len(s) < 2 { return false @@ -1202,12 +1191,15 @@ func (identity) Decode(in string) string { return in } func (i identity) FromStandardPath(s string) string { return FromStandardPath(i, s) } + func (i identity) FromStandardName(s string) string { return FromStandardName(i, s) } + func (i identity) ToStandardPath(s string) string { return ToStandardPath(i, s) } + func (i identity) ToStandardName(s string) string { return ToStandardName(i, s) } diff --git a/lib/transform/cmap.go b/lib/transform/cmap.go new file mode 100644 index 000000000..7f676e792 --- /dev/null +++ b/lib/transform/cmap.go @@ -0,0 +1,71 @@ +package transform + +import ( + "fmt" + "strings" + "sync" + + "github.com/rclone/rclone/fs" + "golang.org/x/text/encoding/charmap" +) + +var ( + cmaps = map[int]*charmap.Charmap{} + lock sync.Mutex +) + +type cmapChoices struct{} + +func (cmapChoices) Choices() []string { + choices := make([]string, 1) + i := 0 + for _, enc := range charmap.All { + c, ok := enc.(*charmap.Charmap) + if !ok { + continue + } + name := strings.ReplaceAll(c.String(), " ", "-") + if name == "" { + name = fmt.Sprintf("unknown-%d", i) + } + lock.Lock() + cmaps[i] = c + lock.Unlock() + choices = append(choices, name) + i++ + } + return choices +} + +func (cmapChoices) Type() string { + return "string" +} + +func charmapByID(cm fs.Enum[cmapChoices]) *charmap.Charmap { + lock.Lock() + c, ok := cmaps[int(cm)] + lock.Unlock() + if ok { + return c + } + return nil +} + +func encodeWithReplacement(s string, cmap *charmap.Charmap) string { + return strings.Map(func(r rune) rune { + b, ok := cmap.EncodeRune(r) + if !ok { + return '_' + } + return cmap.DecodeByte(b) + }, s) +} + +func toASCII(s string) string { + return strings.Map(func(r rune) rune { + if r <= 127 { + return r + } + return -1 + }, s) +} diff --git a/lib/transform/options.go b/lib/transform/options.go new file mode 100644 index 000000000..30243bcc7 --- /dev/null +++ b/lib/transform/options.go @@ -0,0 +1,240 @@ +package transform + +import ( + "context" + "errors" + "strings" + + "github.com/rclone/rclone/fs" +) + +func init() { + fs.RegisterGlobalOptions(fs.OptionsInfo{Name: "name_transform", Opt: &Opt.Flags, Options: OptionsInfo, Reload: Reload}) +} + +type transform struct { + key transformAlgo // for example, "prefix" + value string // for example, "some_prefix_" + tag tag // file, dir, or all +} + +// Options stores the parsed and unparsed transform options. +// their order must never be changed or sorted. +type Options struct { + Flags Flags // unparsed flag value like "file,prefix=ABC" + transforms []transform // parsed from NameTransform +} + +// Flags is a slice of unparsed values set from command line flags or env vars +type Flags struct { + NameTransform []string `config:"name_transform"` +} + +// Opt is the default options modified by the environment variables and command line flags +var Opt Options + +// tag controls which part of the file path is affected (file, dir, all) +type tag int + +// tag modes +const ( + file tag = iota // Only transform the leaf name of files (default) + dir // Only transform name of directories - these may appear anywhere in the path + all // Transform the entire path for files and directories +) + +// OptionsInfo describes the Options in use +var OptionsInfo = fs.Options{{ + Name: "name_transform", + Default: []string{}, + Help: "TODO", + Groups: "Filter", +}} + +// Reload the transform options from the flags +func Reload(ctx context.Context) (err error) { + return newOpt(Opt) +} + +// SetOptions sets the options from flags passed in. +// Any existing flags will be overwritten. +// s should be in the same format as cmd line flags, i.e. "all,prefix=XXX" +func SetOptions(ctx context.Context, s ...string) (err error) { + Opt = Options{Flags: Flags{NameTransform: s}} + return Reload(ctx) +} + +// overwite Opt.transforms with values from Opt.Flags +func newOpt(opt Options) (err error) { + Opt.transforms = []transform{} + + for _, transform := range opt.Flags.NameTransform { + t, err := parse(transform) + if err != nil { + return err + } + Opt.transforms = append(Opt.transforms, t) + } + return nil +} + +// parse a single instance of --name-transform +func parse(s string) (t transform, err error) { + if s == "" { + return t, nil + } + s = t.parseTag(s) + err = t.parseKeyVal(s) + return t, err +} + +// parse the tag (file/dir/all), set the option accordingly, and return the trimmed string +// +// we don't worry about errors here because it will error anyway as an invalid key +func (t *transform) parseTag(s string) string { + if strings.HasPrefix(s, "file,") { + t.tag = file + return strings.TrimPrefix(s, "file,") + } + if strings.HasPrefix(s, "dir,") { + t.tag = dir + return strings.TrimPrefix(s, "dir,") + } + if strings.HasPrefix(s, "all,") { + t.tag = all + return strings.TrimPrefix(s, "all,") + } + return s +} + +// parse key and value (if any) by splitting on '=' sign +// (file/dir/all tag has already been trimmed) +func (t *transform) parseKeyVal(s string) (err error) { + if !strings.ContainsRune(s, '=') { + err = t.key.Set(s) + if err != nil { + return err + } + if t.requiresValue() { + fs.Debugf(nil, "received %v", s) + return errors.New("value is required for " + t.key.String()) + } + return nil + } + split := strings.Split(s, "=") + if len(split) != 2 { + return errors.New("too many values") + } + if split[0] == "" { + return errors.New("key cannot be blank") + } + err = t.key.Set(split[0]) + if err != nil { + return err + } + t.value = split[1] + return nil +} + +// returns true if this particular algorithm requires a value +func (t *transform) requiresValue() bool { + switch t.key { + case ConvFindReplace: + return true + case ConvPrefix: + return true + case ConvSuffix: + return true + case ConvSuffixKeepExtension: + return true + case ConvTrimPrefix: + return true + case ConvTrimSuffix: + return true + case ConvIndex: + return true + case ConvDate: + return true + case ConvTruncate: + return true + case ConvEncoder: + return true + case ConvDecoder: + return true + } + return false +} + +// transformAlgo describes conversion setting +type transformAlgo = fs.Enum[transformChoices] + +// Supported transform options +const ( + ConvNone transformAlgo = iota + ConvToNFC + ConvToNFD + ConvToNFKC + ConvToNFKD + ConvFindReplace + ConvPrefix + ConvSuffix + ConvSuffixKeepExtension + ConvTrimPrefix + ConvTrimSuffix + ConvIndex + ConvDate + ConvTruncate + ConvBase64Encode + ConvBase64Decode + ConvEncoder + ConvDecoder + ConvISO8859_1 + ConvWindows1252 + ConvMacintosh + ConvCharmap + ConvLowercase + ConvUppercase + ConvTitlecase + ConvASCII + ConvURL + ConvMapper +) + +type transformChoices struct{} + +func (transformChoices) Choices() []string { + return []string{ + ConvNone: "none", + ConvToNFC: "nfc", + ConvToNFD: "nfd", + ConvToNFKC: "nfkc", + ConvToNFKD: "nfkd", + ConvFindReplace: "replace", + ConvPrefix: "prefix", + ConvSuffix: "suffix", + ConvSuffixKeepExtension: "suffix_keep_extension", + ConvTrimPrefix: "trimprefix", + ConvTrimSuffix: "trimsuffix", + ConvIndex: "index", + ConvDate: "date", + ConvTruncate: "truncate", + ConvBase64Encode: "base64encode", + ConvBase64Decode: "base64decode", + ConvEncoder: "encoder", + ConvDecoder: "decoder", + ConvISO8859_1: "ISO-8859-1", + ConvWindows1252: "Windows-1252", + ConvMacintosh: "Macintosh", + ConvCharmap: "charmap", + ConvLowercase: "lowercase", + ConvUppercase: "uppercase", + ConvTitlecase: "titlecase", + ConvASCII: "ascii", + ConvURL: "url", + ConvMapper: "mapper", + } +} + +func (transformChoices) Type() string { + return "string" +} diff --git a/lib/transform/transform.go b/lib/transform/transform.go new file mode 100644 index 000000000..ef41449e5 --- /dev/null +++ b/lib/transform/transform.go @@ -0,0 +1,226 @@ +// Package transform holds functions for path name transformations +package transform + +import ( + "encoding/base64" + "errors" + "fmt" + "mime" + "os" + "path" + "strconv" + "strings" + "unicode/utf8" + + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/lib/encoder" + "golang.org/x/text/encoding/charmap" + "golang.org/x/text/unicode/norm" +) + +// Path transforms a path s according to the --name-transform options in use +// +// If no transforms are in use, s is returned unchanged +func Path(s string, isDir bool) string { + if !Transforming() { + return s + } + + var err error + old := s + for _, t := range Opt.transforms { + if isDir && t.tag == file { + continue + } + baseOnly := !isDir && t.tag == file + if t.tag == dir && !isDir { + s, err = transformDir(s, t) + } else { + s, err = transformPath(s, t, baseOnly) + } + if err != nil { + fs.Error(s, err.Error()) // TODO: return err instead of logging it? + } + } + if old != s { + fs.Debugf(old, "transformed to: %v", s) + } + return s +} + +// Transforming returns true when transforms are in use +func Transforming() bool { + return len(Opt.transforms) > 0 +} + +// transformPath transforms a path string according to the chosen TransformAlgo. +// Each path segment is transformed separately, to preserve path separators. +// If baseOnly is true, only the base will be transformed (useful for renaming while walking a dir tree recursively.) +// for example, "some/nested/path" -> "some/nested/CONVERTEDPATH" +// otherwise, the entire is path is transformed. +func transformPath(s string, t transform, baseOnly bool) (string, error) { + if s == "" || s == "/" || s == "\\" || s == "." { + return "", nil + } + + if baseOnly { + transformedBase, err := transformPathSegment(path.Base(s), t) + if err := validateSegment(transformedBase); err != nil { + return "", err + } + return path.Join(path.Dir(s), transformedBase), err + } + + segments := strings.Split(s, string(os.PathSeparator)) + transformedSegments := make([]string, len(segments)) + for _, seg := range segments { + convSeg, err := transformPathSegment(seg, t) + if err != nil { + return "", err + } + if err := validateSegment(convSeg); err != nil { + return "", err + } + transformedSegments = append(transformedSegments, convSeg) + } + return path.Join(transformedSegments...), nil +} + +// transform all but the last path segment +func transformDir(s string, t transform) (string, error) { + dirPath, err := transformPath(path.Dir(s), t, false) + if err != nil { + return "", err + } + return path.Join(dirPath, path.Base(s)), nil +} + +// transformPathSegment transforms one path segment (or really any string) according to the chosen TransformAlgo. +// It assumes path separators have already been trimmed. +func transformPathSegment(s string, t transform) (string, error) { + switch t.key { + case ConvNone: + return s, nil + case ConvToNFC: + return norm.NFC.String(s), nil + case ConvToNFD: + return norm.NFD.String(s), nil + case ConvToNFKC: + return norm.NFKC.String(s), nil + case ConvToNFKD: + return norm.NFKD.String(s), nil + case ConvBase64Encode: + return base64.URLEncoding.EncodeToString([]byte(s)), nil // URLEncoding to avoid slashes + case ConvBase64Decode: + if s == ".DS_Store" { + return s, nil + } + b, err := base64.URLEncoding.DecodeString(s) + if err != nil { + fs.Errorf(s, "base64 error") + } + return string(b), err + case ConvFindReplace: + split := strings.Split(t.value, ":") + if len(split) != 2 { + return s, fmt.Errorf("wrong number of values: %v", t.value) + } + return strings.ReplaceAll(s, split[0], split[1]), nil + case ConvPrefix: + return t.value + s, nil + case ConvSuffix: + return s + t.value, nil + case ConvSuffixKeepExtension: + return SuffixKeepExtension(s, t.value), nil + case ConvTrimPrefix: + return strings.TrimPrefix(s, t.value), nil + case ConvTrimSuffix: + return strings.TrimSuffix(s, t.value), nil + case ConvTruncate: + max, err := strconv.Atoi(t.value) + if err != nil { + return s, err + } + if max <= 0 { + return s, nil + } + if utf8.RuneCountInString(s) <= max { + return s, nil + } + runes := []rune(s) + return string(runes[:max]), nil + case ConvEncoder: + var enc encoder.MultiEncoder + err := enc.Set(t.value) + if err != nil { + return s, err + } + return enc.Encode(s), nil + case ConvDecoder: + var enc encoder.MultiEncoder + err := enc.Set(t.value) + if err != nil { + return s, err + } + return enc.Decode(s), nil + case ConvISO8859_1: + return encodeWithReplacement(s, charmap.ISO8859_1), nil + case ConvWindows1252: + return encodeWithReplacement(s, charmap.Windows1252), nil + case ConvMacintosh: + return encodeWithReplacement(s, charmap.Macintosh), nil + case ConvCharmap: + var cmapType fs.Enum[cmapChoices] + err := cmapType.Set(t.value) + if err != nil { + return s, err + } + c := charmapByID(cmapType) + return encodeWithReplacement(s, c), nil + case ConvLowercase: + return strings.ToLower(s), nil + case ConvUppercase: + return strings.ToUpper(s), nil + case ConvTitlecase: + return strings.ToTitle(s), nil + case ConvASCII: + return toASCII(s), nil + default: + return "", errors.New("this option is not yet implemented") + } +} + +// SuffixKeepExtension adds a suffix while keeping extension +// +// i.e. file.txt becomes file_somesuffix.txt not file.txt_somesuffix +func SuffixKeepExtension(remote string, suffix string) string { + var ( + base = remote + exts = "" + first = true + ext = path.Ext(remote) + ) + for ext != "" { + // Look second and subsequent extensions in mime types. + // If they aren't found then don't keep it as an extension. + if !first && mime.TypeByExtension(ext) == "" { + break + } + base = base[:len(base)-len(ext)] + exts = ext + exts + first = false + ext = path.Ext(base) + } + return base + suffix + exts +} + +// forbid transformations that add/remove path separators +func validateSegment(s string) error { + if s == "" { + return errors.New("transform cannot render path segments empty") + } + if strings.ContainsRune(s, '/') { + return fmt.Errorf("transform cannot add path separators: %v", s) + } + return nil +} diff --git a/lib/transform/transform_test.go b/lib/transform/transform_test.go new file mode 100644 index 000000000..67c7d77ac --- /dev/null +++ b/lib/transform/transform_test.go @@ -0,0 +1,130 @@ +package transform + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// sync tests are in fs/sync/sync_transform_test.go to avoid import cycle issues + +func TestPath(t *testing.T) { + for _, test := range []struct { + path string + want string + }{ + {"", ""}, + {"toe/toe/toe", "tictactoe/tictactoe/tictactoe"}, + {"a/b/c", "tictaca/tictacb/tictacc"}, + } { + err := SetOptions(context.Background(), "all,prefix=tac", "all,prefix=tic") + require.NoError(t, err) + + got := Path(test.path, false) + assert.Equal(t, test.want, got) + } +} + +func TestFileTagOnFile(t *testing.T) { + for _, test := range []struct { + path string + want string + }{ + {"a/b/c.txt", "a/b/1c.txt"}, + } { + err := SetOptions(context.Background(), "file,prefix=1") + require.NoError(t, err) + + got := Path(test.path, false) + assert.Equal(t, test.want, got) + } +} + +func TestDirTagOnFile(t *testing.T) { + for _, test := range []struct { + path string + want string + }{ + {"a/b/c.txt", "1a/1b/c.txt"}, + } { + err := SetOptions(context.Background(), "dir,prefix=1") + require.NoError(t, err) + + got := Path(test.path, false) + assert.Equal(t, test.want, got) + } +} + +func TestAllTag(t *testing.T) { + for _, test := range []struct { + path string + want string + }{ + {"a/b/c.txt", "1a/1b/1c.txt"}, + } { + err := SetOptions(context.Background(), "all,prefix=1") + require.NoError(t, err) + + got := Path(test.path, false) + assert.Equal(t, test.want, got) + } +} + +func TestFileTagOnDir(t *testing.T) { + for _, test := range []struct { + path string + want string + }{ + {"a/b", "a/b"}, + } { + err := SetOptions(context.Background(), "file,prefix=1") + require.NoError(t, err) + + got := Path(test.path, true) + assert.Equal(t, test.want, got) + } +} + +func TestDirTagOnDir(t *testing.T) { + for _, test := range []struct { + path string + want string + }{ + {"a/b", "1a/1b"}, + } { + err := SetOptions(context.Background(), "dir,prefix=1") + require.NoError(t, err) + + got := Path(test.path, true) + assert.Equal(t, test.want, got) + } +} + +func TestVarious(t *testing.T) { + for _, test := range []struct { + path string + want string + flags []string + }{ + {"stories/The Quick Brown Fox!.txt", "STORIES/THE QUICK BROWN FOX!.TXT", []string{"all,uppercase"}}, + {"stories/The Quick Brown Fox!.txt", "stories/The Slow Brown Turtle!.txt", []string{"all,replace=Fox:Turtle", "all,replace=Quick:Slow"}}, + {"stories/The Quick Brown Fox!.txt", "c3Rvcmllcw==/VGhlIFF1aWNrIEJyb3duIEZveCEudHh0", []string{"all,base64encode"}}, + {"c3Rvcmllcw==/VGhlIFF1aWNrIEJyb3duIEZveCEudHh0", "stories/The Quick Brown Fox!.txt", []string{"all,base64decode"}}, + {"stories/The Quick Brown 🦊 Fox Went to the Café!.txt", "stories/The Quick Brown 🦊 Fox Went to the Café!.txt", []string{"all,nfc"}}, + {"stories/The Quick Brown 🦊 Fox Went to the Café!.txt", "stories/The Quick Brown 🦊 Fox Went to the Café!.txt", []string{"all,nfd"}}, + {"stories/The Quick Brown 🦊 Fox!.txt", "stories/The Quick Brown Fox!.txt", []string{"all,ascii"}}, + {"stories/The Quick Brown Fox!.txt", "stories/The Quick Brown Fox!", []string{"all,trimsuffix=.txt"}}, + {"stories/The Quick Brown Fox!.txt", "OLD_stories/OLD_The Quick Brown Fox!.txt", []string{"all,prefix=OLD_"}}, + {"stories/The Quick Brown 🦊 Fox Went to the Café!.txt", "stories/The Quick Brown _ Fox Went to the Caf_!.txt", []string{"all,charmap=ISO-8859-7"}}, + {"stories/The Quick Brown Fox: A Memoir [draft].txt", "stories/The Quick Brown Fox： A Memoir ［draft］.txt", []string{"all,encoder=Colon,SquareBracket"}}, + {"stories/The Quick Brown 🦊 Fox Went to the Café!.txt", "stories/The Quick Brown 🦊 Fox", []string{"all,truncate=21"}}, + } { + err := SetOptions(context.Background(), test.flags...) + require.NoError(t, err) + + got := Path(test.path, false) + assert.Equal(t, test.want, got) + } +} diff --git a/lib/transform/transformflags/transformflags.go b/lib/transform/transformflags/transformflags.go new file mode 100644 index 000000000..c03c2159d --- /dev/null +++ b/lib/transform/transformflags/transformflags.go @@ -0,0 +1,14 @@ +// Package transformflags implements command line flags to set up a transform +package transformflags + +import ( + "github.com/rclone/rclone/fs/config/flags" + "github.com/rclone/rclone/lib/transform" + + "github.com/spf13/pflag" +) + +// AddFlags adds the transform flags to the command +func AddFlags(flagSet *pflag.FlagSet) { + flags.AddFlagsFromOptions(flagSet, "", transform.OptionsInfo) +}