From 7b9f8eca0059613410921b978790db14903ec674 Mon Sep 17 00:00:00 2001 From: nielash Date: Sun, 4 May 2025 05:48:07 -0400 Subject: [PATCH] lib/transform: refactor and add TimeFormat support --- cmd/bisync/bilib/names.go | 80 ----------- cmd/bisync/resolve.go | 4 +- cmd/convmv/convmv.go | 2 +- cmd/convmv/convmv_test.go | 37 ++--- cmd/help.go | 2 - fs/config.go | 6 + fs/march/march.go | 14 +- fs/operations/copy.go | 4 +- fs/operations/operations.go | 50 +------ fs/sync/sync.go | 10 +- fs/sync/sync_test.go | 2 +- fs/sync/sync_transform_test.go | 34 ++--- lib/transform/help.go | 12 +- lib/transform/options.go | 70 ++++----- lib/transform/transform.go | 135 ++++++++++++++++-- lib/transform/transform_test.go | 40 ++++-- .../transformflags/transformflags.go | 14 -- 17 files changed, 249 insertions(+), 267 deletions(-) delete mode 100644 lib/transform/transformflags/transformflags.go diff --git a/cmd/bisync/bilib/names.go b/cmd/bisync/bilib/names.go index d8951a0b5..cb266eb63 100644 --- a/cmd/bisync/bilib/names.go +++ b/cmd/bisync/bilib/names.go @@ -5,8 +5,6 @@ import ( "os" "sort" "strconv" - "strings" - "time" ) // Names comprises a set of file names @@ -85,81 +83,3 @@ func (am AliasMap) Alias(name1 string) string { } return name1 } - -// ParseGlobs determines whether a string contains {brackets} -// and returns the substring (including both brackets) for replacing -// substring is first opening bracket to last closing bracket -- -// good for {{this}} but not {this}{this} -func ParseGlobs(s string) (hasGlobs bool, substring string) { - open := strings.Index(s, "{") - close := strings.LastIndex(s, "}") - if open >= 0 && close > open { - return true, s[open : close+1] - } - return false, "" -} - -// TrimBrackets converts {{this}} to this -func TrimBrackets(s string) string { - return strings.Trim(s, "{}") -} - -// TimeFormat converts a user-supplied string to a Go time constant, if possible -func TimeFormat(timeFormat string) string { - switch timeFormat { - case "Layout": - timeFormat = time.Layout - case "ANSIC": - timeFormat = time.ANSIC - case "UnixDate": - timeFormat = time.UnixDate - case "RubyDate": - timeFormat = time.RubyDate - case "RFC822": - timeFormat = time.RFC822 - case "RFC822Z": - timeFormat = time.RFC822Z - case "RFC850": - timeFormat = time.RFC850 - case "RFC1123": - timeFormat = time.RFC1123 - case "RFC1123Z": - timeFormat = time.RFC1123Z - case "RFC3339": - timeFormat = time.RFC3339 - case "RFC3339Nano": - timeFormat = time.RFC3339Nano - case "Kitchen": - timeFormat = time.Kitchen - case "Stamp": - timeFormat = time.Stamp - case "StampMilli": - timeFormat = time.StampMilli - case "StampMicro": - timeFormat = time.StampMicro - case "StampNano": - timeFormat = time.StampNano - case "DateTime": - // timeFormat = time.DateTime // missing in go1.19 - timeFormat = "2006-01-02 15:04:05" - case "DateOnly": - // timeFormat = time.DateOnly // missing in go1.19 - timeFormat = "2006-01-02" - case "TimeOnly": - // timeFormat = time.TimeOnly // missing in go1.19 - timeFormat = "15:04:05" - case "MacFriendlyTime", "macfriendlytime", "mac": - timeFormat = "2006-01-02 0304PM" // not actually a Go constant -- but useful as macOS filenames can't have colons - } - return timeFormat -} - -// AppyTimeGlobs converts "myfile-{DateOnly}.txt" to "myfile-2006-01-02.txt" -func AppyTimeGlobs(s string, t time.Time) string { - hasGlobs, substring := ParseGlobs(s) - if !hasGlobs { - return s - } - timeString := t.Local().Format(TimeFormat(TrimBrackets(substring))) - return strings.ReplaceAll(s, substring, timeString) -} diff --git a/cmd/bisync/resolve.go b/cmd/bisync/resolve.go index 1a3d4e716..b3e5d8048 100644 --- a/cmd/bisync/resolve.go +++ b/cmd/bisync/resolve.go @@ -96,8 +96,8 @@ func (b *bisyncRun) setResolveDefaults(ctx context.Context) error { } // replace glob variables, if any t := time.Now() // capture static time here so it is the same for all files throughout this run - b.opt.ConflictSuffix1 = bilib.AppyTimeGlobs(b.opt.ConflictSuffix1, t) - b.opt.ConflictSuffix2 = bilib.AppyTimeGlobs(b.opt.ConflictSuffix2, t) + b.opt.ConflictSuffix1 = transform.AppyTimeGlobs(b.opt.ConflictSuffix1, t) + b.opt.ConflictSuffix2 = transform.AppyTimeGlobs(b.opt.ConflictSuffix2, t) // append dot (intentionally allow more than one) b.opt.ConflictSuffix1 = "." + b.opt.ConflictSuffix1 diff --git a/cmd/convmv/convmv.go b/cmd/convmv/convmv.go index a54c8206e..833470b25 100644 --- a/cmd/convmv/convmv.go +++ b/cmd/convmv/convmv.go @@ -96,7 +96,7 @@ This can lead to race conditions when performing concurrent transfers. It is up cmd.CheckArgs(1, 1, command, args) fdst, srcFileName := cmd.NewFsFile(args[0]) cmd.Run(false, true, command, func() error { - if !transform.Transforming() { + if !transform.Transforming(context.Background()) { return errors.New("--name-transform must be set") } if srcFileName == "" { diff --git a/cmd/convmv/convmv_test.go b/cmd/convmv/convmv_test.go index 6147710b9..9276bbb84 100644 --- a/cmd/convmv/convmv_test.go +++ b/cmd/convmv/convmv_test.go @@ -105,31 +105,32 @@ func TestTransform(t *testing.T) { r := fstest.NewRun(t) defer r.Finalise() - r.Mkdir(context.Background(), r.Flocal) - r.Mkdir(context.Background(), r.Fremote) + ctx := context.Background() + r.Mkdir(ctx, r.Flocal) + r.Mkdir(ctx, r.Fremote) items := makeTestFiles(t, r, "dir1") - err := r.Fremote.Mkdir(context.Background(), "empty/empty") + err := r.Fremote.Mkdir(ctx, "empty/empty") require.NoError(t, err) - err = r.Flocal.Mkdir(context.Background(), "empty/empty") + err = r.Flocal.Mkdir(ctx, "empty/empty") require.NoError(t, err) deleteDSStore(t, r) r.CheckRemoteListing(t, items, []string{"dir1", "empty", "empty/empty"}) r.CheckLocalListing(t, items, []string{"dir1", "empty", "empty/empty"}) - err = transform.SetOptions(context.Background(), tt.args.TransformOpt...) + err = transform.SetOptions(ctx, tt.args.TransformOpt...) require.NoError(t, err) - err = sync.Transform(context.Background(), r.Fremote, true, true) + err = sync.Transform(ctx, r.Fremote, true, true) assert.NoError(t, err) - compareNames(t, r, items) + compareNames(ctx, t, r, items) - transformedItems := transformItems(t, items) - r.CheckRemoteListing(t, transformedItems, []string{transform.Path("dir1", true), transform.Path("empty", true), transform.Path("empty/empty", true)}) - err = transform.SetOptions(context.Background(), tt.args.TransformBackOpt...) + transformedItems := transformItems(ctx, t, items) + r.CheckRemoteListing(t, transformedItems, []string{transform.Path(ctx, "dir1", true), transform.Path(ctx, "empty", true), transform.Path(ctx, "empty/empty", true)}) + err = transform.SetOptions(ctx, tt.args.TransformBackOpt...) require.NoError(t, err) - err = sync.Transform(context.Background(), r.Fremote, true, true) + err = sync.Transform(ctx, r.Fremote, true, true) assert.NoError(t, err) - compareNames(t, r, transformedItems) + compareNames(ctx, t, r, transformedItems) if tt.args.Lossless { deleteDSStore(t, r) @@ -191,7 +192,7 @@ func deleteDSStore(t *testing.T, r *fstest.Run) { assert.NoError(t, err) } -func compareNames(t *testing.T, r *fstest.Run, items []fstest.Item) { +func compareNames(ctx context.Context, t *testing.T, r *fstest.Run, items []fstest.Item) { var entries fs.DirEntries deleteDSStore(t, r) @@ -212,8 +213,8 @@ func compareNames(t *testing.T, r *fstest.Run, items []fstest.Item) { // sort by CONVERTED name slices.SortStableFunc(items, func(a, b fstest.Item) int { - aConv := transform.Path(a.Path, false) - bConv := transform.Path(b.Path, false) + aConv := transform.Path(ctx, a.Path, false) + bConv := transform.Path(ctx, b.Path, false) return cmp.Compare(aConv, bConv) }) slices.SortStableFunc(entries, func(a, b fs.DirEntry) int { @@ -221,16 +222,16 @@ func compareNames(t *testing.T, r *fstest.Run, items []fstest.Item) { }) for i, e := range entries { - expect := transform.Path(items[i].Path, false) + expect := transform.Path(ctx, items[i].Path, false) msg := fmt.Sprintf("expected %v, got %v", detectEncoding(expect), detectEncoding(e.Remote())) assert.Equal(t, expect, e.Remote(), msg) } } -func transformItems(t *testing.T, items []fstest.Item) []fstest.Item { +func transformItems(ctx context.Context, t *testing.T, items []fstest.Item) []fstest.Item { transformedItems := []fstest.Item{} for _, item := range items { - newPath := transform.Path(item.Path, false) + newPath := transform.Path(ctx, item.Path, false) newItem := item newItem.Path = newPath transformedItems = append(transformedItems, newItem) diff --git a/cmd/help.go b/cmd/help.go index c2f7c2b37..1b985daca 100644 --- a/cmd/help.go +++ b/cmd/help.go @@ -16,7 +16,6 @@ import ( "github.com/rclone/rclone/fs/log/logflags" "github.com/rclone/rclone/fs/rc/rcflags" "github.com/rclone/rclone/lib/atexit" - "github.com/rclone/rclone/lib/transform/transformflags" "github.com/spf13/cobra" "github.com/spf13/pflag" "golang.org/x/text/cases" @@ -138,7 +137,6 @@ func setupRootCommand(rootCmd *cobra.Command) { // Add global flags configflags.AddFlags(ci, pflag.CommandLine) filterflags.AddFlags(pflag.CommandLine) - transformflags.AddFlags(pflag.CommandLine) rcflags.AddFlags(pflag.CommandLine) logflags.AddFlags(pflag.CommandLine) diff --git a/fs/config.go b/fs/config.go index 4f6c4dad2..950ae27e0 100644 --- a/fs/config.go +++ b/fs/config.go @@ -570,6 +570,11 @@ uploads or downloads to limit the number of total connections. `, "|", "`"), Default: 0, Advanced: true, +}, { + Name: "name_transform", + Default: []string{}, + Help: "`--name-transform` introduces path name transformations for `rclone copy`, `rclone sync`, and `rclone move`. These transformations enable modifications to source and destination file names by applying prefixes, suffixes, and other alterations during transfer operations. For detailed docs and examples, see [`convmv`](/commands/rclone_convmv/).", + Groups: "Filter", }} // ConfigInfo is filesystem config options @@ -681,6 +686,7 @@ type ConfigInfo struct { PartialSuffix string `config:"partial_suffix"` MetadataMapper SpaceSepList `config:"metadata_mapper"` MaxConnections int `config:"max_connections"` + NameTransform []string `config:"name_transform"` } func init() { diff --git a/fs/march/march.go b/fs/march/march.go index 819f35cf3..7f264635f 100644 --- a/fs/march/march.go +++ b/fs/march/march.go @@ -87,11 +87,8 @@ func (m *March) srcKey(entry fs.DirEntry) string { return "" } name := path.Base(entry.Remote()) - name = transform.Path(name, fs.DirEntryType(entry) == "directory") - for _, transform := range m.transforms { - name = transform(name) - } - return name + name = transform.Path(m.Ctx, name, fs.DirEntryType(entry) == "directory") + return transforms(name, m.transforms) } // dstKey turns a directory entry into a sort key using the defined transforms. @@ -99,8 +96,11 @@ func (m *March) dstKey(entry fs.DirEntry) string { if entry == nil { return "" } - name := path.Base(entry.Remote()) - for _, transform := range m.transforms { + return transforms(path.Base(entry.Remote()), m.transforms) +} + +func transforms(name string, transforms []matchTransformFn) string { + for _, transform := range transforms { name = transform(name) } return name diff --git a/fs/operations/copy.go b/fs/operations/copy.go index bfc58ead9..2d42fac07 100644 --- a/fs/operations/copy.go +++ b/fs/operations/copy.go @@ -391,7 +391,7 @@ func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Obj f: f, dstFeatures: f.Features(), dst: dst, - remote: transform.Path(remote, false), + remote: transform.Path(ctx, remote, false), src: src, ci: ci, tr: tr, @@ -400,7 +400,7 @@ func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Obj } c.hashType, c.hashOption = CommonHash(ctx, f, src.Fs()) if c.dst != nil { - c.remote = transform.Path(c.dst.Remote(), false) + c.remote = transform.Path(ctx, c.dst.Remote(), false) } // Are we using partials? // diff --git a/fs/operations/operations.go b/fs/operations/operations.go index 7c4087588..d8366c252 100644 --- a/fs/operations/operations.go +++ b/fs/operations/operations.go @@ -426,7 +426,7 @@ func MoveTransfer(ctx context.Context, fdst fs.Fs, dst fs.Object, remote string, // move - see Move for help func move(ctx context.Context, fdst fs.Fs, dst fs.Object, remote string, src fs.Object, isTransfer bool) (newDst fs.Object, err error) { origRemote := remote // avoid double-transform on fallback to copy - remote = transform.Path(remote, false) + remote = transform.Path(ctx, remote, false) ci := fs.GetConfig(ctx) var tr *accounting.Transfer if isTransfer { @@ -450,7 +450,7 @@ func move(ctx context.Context, fdst fs.Fs, dst fs.Object, remote string, src fs. if doMove := fdst.Features().Move; doMove != nil && (SameConfig(src.Fs(), fdst) || (SameRemoteType(src.Fs(), fdst) && (fdst.Features().ServerSideAcrossConfigs || ci.ServerSideAcrossConfigs))) { // Delete destination if it exists and is not the same file as src (could be same file while seemingly different if the remote is case insensitive) if dst != nil { - remote = transform.Path(dst.Remote(), false) + remote = transform.Path(ctx, dst.Remote(), false) if !SameObject(src, dst) { err = DeleteFile(ctx, dst) if err != nil { @@ -2206,50 +2206,10 @@ func (l *ListFormat) SetOutput(output []func(entry *ListJSONItem) string) { // AddModTime adds file's Mod Time to output func (l *ListFormat) AddModTime(timeFormat string) { - switch timeFormat { - case "": + if timeFormat == "" { timeFormat = "2006-01-02 15:04:05" - case "Layout": - timeFormat = time.Layout - case "ANSIC": - timeFormat = time.ANSIC - case "UnixDate": - timeFormat = time.UnixDate - case "RubyDate": - timeFormat = time.RubyDate - case "RFC822": - timeFormat = time.RFC822 - case "RFC822Z": - timeFormat = time.RFC822Z - case "RFC850": - timeFormat = time.RFC850 - case "RFC1123": - timeFormat = time.RFC1123 - case "RFC1123Z": - timeFormat = time.RFC1123Z - case "RFC3339": - timeFormat = time.RFC3339 - case "RFC3339Nano": - timeFormat = time.RFC3339Nano - case "Kitchen": - timeFormat = time.Kitchen - case "Stamp": - timeFormat = time.Stamp - case "StampMilli": - timeFormat = time.StampMilli - case "StampMicro": - timeFormat = time.StampMicro - case "StampNano": - timeFormat = time.StampNano - case "DateTime": - // timeFormat = time.DateTime // missing in go1.19 - timeFormat = "2006-01-02 15:04:05" - case "DateOnly": - // timeFormat = time.DateOnly // missing in go1.19 - timeFormat = "2006-01-02" - case "TimeOnly": - // timeFormat = time.TimeOnly // missing in go1.19 - timeFormat = "15:04:05" + } else { + timeFormat = transform.TimeFormat(timeFormat) } l.AppendOutput(func(entry *ListJSONItem) string { return entry.ModTime.When.Local().Format(timeFormat) diff --git a/fs/sync/sync.go b/fs/sync/sync.go index 5bb2bef5c..c7593f673 100644 --- a/fs/sync/sync.go +++ b/fs/sync/sync.go @@ -1125,7 +1125,7 @@ func (s *syncCopyMove) copyDirMetadata(ctx context.Context, f fs.Fs, dst fs.Dire newDst, err = operations.SetDirModTime(ctx, f, dst, dir, src.ModTime(ctx)) } } - if transform.Transforming() && newDst != nil && src.Remote() != newDst.Remote() { + if transform.Transforming(ctx) && newDst != nil && src.Remote() != newDst.Remote() { s.markParentNotEmpty(src) } // If we need to set modtime after and we created a dir, then save it for later @@ -1260,8 +1260,8 @@ func (s *syncCopyMove) SrcOnly(src fs.DirEntry) (recurse bool) { s.logger(s.ctx, operations.MissingOnDst, src, nil, fs.ErrorIsDir) // Create the directory and make sure the Metadata/ModTime is correct - s.copyDirMetadata(s.ctx, s.fdst, nil, transform.Path(x.Remote(), true), x) - s.markDirModified(transform.Path(x.Remote(), true)) + s.copyDirMetadata(s.ctx, s.fdst, nil, transform.Path(s.ctx, x.Remote(), true), x) + s.markDirModified(transform.Path(s.ctx, x.Remote(), true)) return true default: panic("Bad object in DirEntries") @@ -1294,9 +1294,9 @@ func (s *syncCopyMove) Match(ctx context.Context, dst, src fs.DirEntry) (recurse } case fs.Directory: // Do the same thing to the entire contents of the directory - srcX = fs.NewOverrideDirectory(srcX, transform.Path(src.Remote(), true)) + srcX = fs.NewOverrideDirectory(srcX, transform.Path(ctx, src.Remote(), true)) src = srcX - if !transform.Transforming() || src.Remote() != dst.Remote() { + if !transform.Transforming(ctx) || src.Remote() != dst.Remote() { s.markParentNotEmpty(src) } dstX, ok := dst.(fs.Directory) diff --git a/fs/sync/sync_test.go b/fs/sync/sync_test.go index a921baf35..3f765b0c0 100644 --- a/fs/sync/sync_test.go +++ b/fs/sync/sync_test.go @@ -2981,7 +2981,7 @@ func predictDstFromLogger(ctx context.Context) context.Context { if winner.Err != nil { errMsg = ";" + winner.Err.Error() } - operations.SyncFprintf(opt.JSON, "%s;%s;%v;%s%s\n", file.ModTime(ctx).Local().Format(timeFormat), checksum, file.Size(), transform.Path(file.Remote(), false), errMsg) // TODO: should the transform be handled in the sync instead of here? + operations.SyncFprintf(opt.JSON, "%s;%s;%v;%s%s\n", file.ModTime(ctx).Local().Format(timeFormat), checksum, file.Size(), transform.Path(ctx, file.Remote(), false), errMsg) // TODO: should the transform be handled in the sync instead of here? } } return operations.WithSyncLogger(ctx, opt) diff --git a/fs/sync/sync_transform_test.go b/fs/sync/sync_transform_test.go index a5c87a47b..d45a80983 100644 --- a/fs/sync/sync_transform_test.go +++ b/fs/sync/sync_transform_test.go @@ -6,7 +6,7 @@ import ( "cmp" "context" "fmt" - "path/filepath" + "path" "slices" "strings" "testing" @@ -96,25 +96,26 @@ func TestTransform(t *testing.T) { r := fstest.NewRun(t) defer r.Finalise() - r.Mkdir(context.Background(), r.Flocal) - r.Mkdir(context.Background(), r.Fremote) + ctx := context.Background() + r.Mkdir(ctx, r.Flocal) + r.Mkdir(ctx, r.Fremote) items := makeTestFiles(t, r, "dir1") deleteDSStore(t, r) r.CheckRemoteListing(t, items, nil) r.CheckLocalListing(t, items, nil) - err := transform.SetOptions(context.Background(), tt.args.TransformOpt...) + err := transform.SetOptions(ctx, tt.args.TransformOpt...) require.NoError(t, err) - err = Sync(context.Background(), r.Fremote, r.Flocal, true) + err = Sync(ctx, r.Fremote, r.Flocal, true) assert.NoError(t, err) - compareNames(t, r, items) + compareNames(ctx, t, r, items) - err = transform.SetOptions(context.Background(), tt.args.TransformBackOpt...) + err = transform.SetOptions(ctx, tt.args.TransformBackOpt...) require.NoError(t, err) - err = Sync(context.Background(), r.Fremote, r.Flocal, true) + err = Sync(ctx, r.Fremote, r.Flocal, true) assert.NoError(t, err) - compareNames(t, r, items) + compareNames(ctx, t, r, items) if tt.args.Lossless { deleteDSStore(t, r) @@ -138,8 +139,9 @@ func makeTestFiles(t *testing.T, r *fstest.Run, dir string) []fstest.Item { for i := rune(0); i < 7; i++ { out.WriteRune(c + i) } - fileName := filepath.Join(dir, fmt.Sprintf("%04d-%s.txt", n, out.String())) + fileName := path.Join(dir, fmt.Sprintf("%04d-%s.txt", n, out.String())) fileName = strings.ToValidUTF8(fileName, "") + fileName = strings.NewReplacer(":", "", "<", "", ">", "", "?", "").Replace(fileName) // remove characters illegal on windows if debug != "" { fileName = debug @@ -174,7 +176,7 @@ func deleteDSStore(t *testing.T, r *fstest.Run) { assert.NoError(t, err) } -func compareNames(t *testing.T, r *fstest.Run, items []fstest.Item) { +func compareNames(ctx context.Context, t *testing.T, r *fstest.Run, items []fstest.Item) { var entries fs.DirEntries deleteDSStore(t, r) @@ -195,8 +197,8 @@ func compareNames(t *testing.T, r *fstest.Run, items []fstest.Item) { // sort by CONVERTED name slices.SortStableFunc(items, func(a, b fstest.Item) int { - aConv := transform.Path(a.Path, false) - bConv := transform.Path(b.Path, false) + aConv := transform.Path(ctx, a.Path, false) + bConv := transform.Path(ctx, b.Path, false) return cmp.Compare(aConv, bConv) }) slices.SortStableFunc(entries, func(a, b fs.DirEntry) int { @@ -204,7 +206,7 @@ func compareNames(t *testing.T, r *fstest.Run, items []fstest.Item) { }) for i, e := range entries { - expect := transform.Path(items[i].Path, false) + expect := transform.Path(ctx, items[i].Path, false) msg := fmt.Sprintf("expected %v, got %v", detectEncoding(expect), detectEncoding(e.Remote())) assert.Equal(t, expect, e.Remote(), msg) } @@ -477,7 +479,5 @@ func TestError(t *testing.T) { assert.Error(t, err) r.CheckLocalListing(t, []fstest.Item{file1}, []string{"toe", "toe/toe"}) - r.CheckRemoteListing(t, []fstest.Item{}, []string{}) - err = transform.SetOptions(ctx, "") // has illegal character - assert.NoError(t, err) + r.CheckRemoteListing(t, []fstest.Item{file1}, []string{"toe", "toe/toe"}) } diff --git a/lib/transform/help.go b/lib/transform/help.go index a60e98f4e..dff8cf2d3 100644 --- a/lib/transform/help.go +++ b/lib/transform/help.go @@ -27,7 +27,7 @@ var commandList = []commands{ {command: "--name-transform trimsuffix=XXXX", description: "Removes XXXX if it appears at the end of the file name."}, {command: "--name-transform regex=/pattern/replacement/", description: "Applies a regex-based transformation."}, {command: "--name-transform replace=old:new", description: "Replaces occurrences of old with new in the file name."}, - {command: "--name-transform date=YYYYMMDD", description: "Appends or prefixes the specified date format."}, + {command: "--name-transform date={YYYYMMDD}", description: "Appends or prefixes the specified date format."}, {command: "--name-transform truncate=N", description: "Truncates the file name to a maximum of N characters."}, {command: "--name-transform base64encode", description: "Encodes the file name in Base64."}, {command: "--name-transform base64decode", description: "Decodes a Base64-encoded file name."}, @@ -59,6 +59,10 @@ var examples = []example{ {"stories/The Quick Brown 🦊 Fox Went to the Café!.txt", []string{"all,charmap=ISO-8859-7"}}, {"stories/The Quick Brown Fox: A Memoir [draft].txt", []string{"all,encoder=Colon,SquareBracket"}}, {"stories/The Quick Brown 🦊 Fox Went to the Café!.txt", []string{"all,truncate=21"}}, + {"stories/The Quick Brown Fox!.txt", []string{"all,command=echo"}}, + {"stories/The Quick Brown Fox!", []string{"date=-{YYYYMMDD}"}}, + {"stories/The Quick Brown Fox!", []string{"date=-{macfriendlytime}"}}, + {"stories/The Quick Brown Fox!.txt", []string{"all,regex=[\\.\\w]/ab"}}, } func (e example) command() string { @@ -70,11 +74,12 @@ func (e example) command() string { } func (e example) output() string { - err := SetOptions(context.Background(), e.flags...) + ctx := context.Background() + err := SetOptions(ctx, e.flags...) if err != nil { fs.Errorf(nil, "error generating help text: %v", err) } - return Path(e.path, false) + return Path(ctx, e.path, false) } // go run ./ convmv --help @@ -84,7 +89,6 @@ func sprintExamples() string { s += fmt.Sprintf("```\n%s\n", e.command()) s += fmt.Sprintf("// Output: %s\n```\n\n", e.output()) } - Opt = Options{} // reset return s } diff --git a/lib/transform/options.go b/lib/transform/options.go index 0d7d5c906..6227aa1b4 100644 --- a/lib/transform/options.go +++ b/lib/transform/options.go @@ -8,31 +8,12 @@ import ( "github.com/rclone/rclone/fs" ) -func init() { - fs.RegisterGlobalOptions(fs.OptionsInfo{Name: "name_transform", Opt: &Opt.Flags, Options: OptionsInfo, Reload: Reload}) -} - type transform struct { key transformAlgo // for example, "prefix" value string // for example, "some_prefix_" tag tag // file, dir, or all } -// Options stores the parsed and unparsed transform options. -// their order must never be changed or sorted. -type Options struct { - Flags Flags // unparsed flag value like "file,prefix=ABC" - transforms []transform // parsed from NameTransform -} - -// Flags is a slice of unparsed values set from command line flags or env vars -type Flags struct { - NameTransform []string `config:"name_transform"` -} - -// Opt is the default options modified by the environment variables and command line flags -var Opt Options - // tag controls which part of the file path is affected (file, dir, all) type tag int @@ -43,39 +24,38 @@ const ( all // Transform the entire path for files and directories ) -// OptionsInfo describes the Options in use -var OptionsInfo = fs.Options{{ - Name: "name_transform", - Default: []string{}, - Help: "`--name-transform` introduces path name transformations for `rclone copy`, `rclone sync`, and `rclone move`. These transformations enable modifications to source and destination file names by applying prefixes, suffixes, and other alterations during transfer operations. For detailed docs and examples, see [`convmv`](/commands/rclone_convmv/).", - Groups: "Filter", -}} - -// Reload the transform options from the flags -func Reload(ctx context.Context) (err error) { - return newOpt(Opt) +// Transforming returns true when transforms are in use +func Transforming(ctx context.Context) bool { + ci := fs.GetConfig(ctx) + return len(ci.NameTransform) > 0 } -// SetOptions sets the options from flags passed in. +// SetOptions sets the options in ctx from flags passed in. // Any existing flags will be overwritten. // s should be in the same format as cmd line flags, i.e. "all,prefix=XXX" func SetOptions(ctx context.Context, s ...string) (err error) { - Opt = Options{Flags: Flags{NameTransform: s}} - return Reload(ctx) + ci := fs.GetConfig(ctx) + ci.NameTransform = s + _, err = getOptions(ctx) + return err } -// overwite Opt.transforms with values from Opt.Flags -func newOpt(opt Options) (err error) { - Opt.transforms = []transform{} +// getOptions sets the options from flags passed in. +func getOptions(ctx context.Context) (opt []transform, err error) { + if !Transforming(ctx) { + return opt, nil + } - for _, transform := range opt.Flags.NameTransform { + ci := fs.GetConfig(ctx) + for _, transform := range ci.NameTransform { t, err := parse(transform) if err != nil { - return err + return opt, err } - Opt.transforms = append(Opt.transforms, t) + opt = append(opt, t) } - return nil + // TODO: should we store opt in ci and skip re-parsing when present, for efficiency? + return opt, nil } // parse a single instance of --name-transform @@ -161,6 +141,10 @@ func (t *transform) requiresValue() bool { return true case ConvDecoder: return true + case ConvRegex: + return true + case ConvCommand: + return true } return false } @@ -197,7 +181,8 @@ const ( ConvTitlecase ConvASCII ConvURL - ConvMapper + ConvRegex + ConvCommand ) type transformChoices struct{} @@ -231,7 +216,8 @@ func (transformChoices) Choices() []string { ConvTitlecase: "titlecase", ConvASCII: "ascii", ConvURL: "url", - ConvMapper: "mapper", + ConvRegex: "regex", + ConvCommand: "command", } } diff --git a/lib/transform/transform.go b/lib/transform/transform.go index ef41449e5..036fbcee0 100644 --- a/lib/transform/transform.go +++ b/lib/transform/transform.go @@ -2,14 +2,19 @@ package transform import ( + "bytes" + "context" "encoding/base64" "errors" "fmt" "mime" - "os" + "net/url" + "os/exec" "path" + "regexp" "strconv" "strings" + "time" "unicode/utf8" "github.com/rclone/rclone/fs" @@ -21,14 +26,18 @@ import ( // Path transforms a path s according to the --name-transform options in use // // If no transforms are in use, s is returned unchanged -func Path(s string, isDir bool) string { - if !Transforming() { +func Path(ctx context.Context, s string, isDir bool) string { + if !Transforming(ctx) { return s } - var err error old := s - for _, t := range Opt.transforms { + opt, err := getOptions(ctx) + if err != nil { + err = fs.CountError(ctx, err) + fs.Errorf(s, "Failed to parse transform flags: %v", err) + } + for _, t := range opt { if isDir && t.tag == file { continue } @@ -39,20 +48,21 @@ func Path(s string, isDir bool) string { s, err = transformPath(s, t, baseOnly) } if err != nil { - fs.Error(s, err.Error()) // TODO: return err instead of logging it? + err = fs.CountError(ctx, err) + fs.Errorf(s, "Failed to transform: %v", err) } } if old != s { fs.Debugf(old, "transformed to: %v", s) } + if strings.Count(old, "/") != strings.Count(s, "/") { + err = fs.CountError(ctx, fmt.Errorf("number of path segments must match: %v (%v), %v (%v)", old, strings.Count(old, "/"), s, strings.Count(s, "/"))) + fs.Errorf(old, "%v", err) + return old + } return s } -// Transforming returns true when transforms are in use -func Transforming() bool { - return len(Opt.transforms) > 0 -} - // transformPath transforms a path string according to the chosen TransformAlgo. // Each path segment is transformed separately, to preserve path separators. // If baseOnly is true, only the base will be transformed (useful for renaming while walking a dir tree recursively.) @@ -71,7 +81,7 @@ func transformPath(s string, t transform, baseOnly bool) (string, error) { return path.Join(path.Dir(s), transformedBase), err } - segments := strings.Split(s, string(os.PathSeparator)) + segments := strings.Split(s, "/") transformedSegments := make([]string, len(segments)) for _, seg := range segments { convSeg, err := transformPathSegment(seg, t) @@ -185,6 +195,19 @@ func transformPathSegment(s string, t transform) (string, error) { return strings.ToTitle(s), nil case ConvASCII: return toASCII(s), nil + case ConvURL: + return url.QueryEscape(s), nil + case ConvDate: + return s + AppyTimeGlobs(t.value, time.Now()), nil + case ConvRegex: + split := strings.Split(t.value, "/") + if len(split) != 2 { + return s, fmt.Errorf("regex syntax error: %v", t.value) + } + re := regexp.MustCompile(split[0]) + return re.ReplaceAllString(s, split[1]), nil + case ConvCommand: + return mapper(s, t.value) default: return "", errors.New("this option is not yet implemented") } @@ -216,7 +239,7 @@ func SuffixKeepExtension(remote string, suffix string) string { // forbid transformations that add/remove path separators func validateSegment(s string) error { - if s == "" { + if strings.TrimSpace(s) == "" { return errors.New("transform cannot render path segments empty") } if strings.ContainsRune(s, '/') { @@ -224,3 +247,89 @@ func validateSegment(s string) error { } return nil } + +// ParseGlobs determines whether a string contains {brackets} +// and returns the substring (including both brackets) for replacing +// substring is first opening bracket to last closing bracket -- +// good for {{this}} but not {this}{this} +func ParseGlobs(s string) (hasGlobs bool, substring string) { + open := strings.Index(s, "{") + close := strings.LastIndex(s, "}") + if open >= 0 && close > open { + return true, s[open : close+1] + } + return false, "" +} + +// TrimBrackets converts {{this}} to this +func TrimBrackets(s string) string { + return strings.Trim(s, "{}") +} + +// TimeFormat converts a user-supplied string to a Go time constant, if possible +func TimeFormat(timeFormat string) string { + switch timeFormat { + case "Layout": + timeFormat = time.Layout + case "ANSIC": + timeFormat = time.ANSIC + case "UnixDate": + timeFormat = time.UnixDate + case "RubyDate": + timeFormat = time.RubyDate + case "RFC822": + timeFormat = time.RFC822 + case "RFC822Z": + timeFormat = time.RFC822Z + case "RFC850": + timeFormat = time.RFC850 + case "RFC1123": + timeFormat = time.RFC1123 + case "RFC1123Z": + timeFormat = time.RFC1123Z + case "RFC3339": + timeFormat = time.RFC3339 + case "RFC3339Nano": + timeFormat = time.RFC3339Nano + case "Kitchen": + timeFormat = time.Kitchen + case "Stamp": + timeFormat = time.Stamp + case "StampMilli": + timeFormat = time.StampMilli + case "StampMicro": + timeFormat = time.StampMicro + case "StampNano": + timeFormat = time.StampNano + case "DateTime": + timeFormat = time.DateTime + case "DateOnly": + timeFormat = time.DateOnly + case "TimeOnly": + timeFormat = time.TimeOnly + case "MacFriendlyTime", "macfriendlytime", "mac": + timeFormat = "2006-01-02 0304PM" // not actually a Go constant -- but useful as macOS filenames can't have colons + case "YYYYMMDD": + timeFormat = "20060102" + } + return timeFormat +} + +// AppyTimeGlobs converts "myfile-{DateOnly}.txt" to "myfile-2006-01-02.txt" +func AppyTimeGlobs(s string, t time.Time) string { + hasGlobs, substring := ParseGlobs(s) + if !hasGlobs { + return s + } + timeString := t.Local().Format(TimeFormat(TrimBrackets(substring))) + return strings.ReplaceAll(s, substring, timeString) +} + +func mapper(s string, command string) (string, error) { + out, err := exec.Command(command, s).CombinedOutput() + if err != nil { + out = bytes.TrimSpace(out) + return s, fmt.Errorf("%s: error running command %q: %v", out, command+" "+s, err) + } + return string(bytes.TrimSpace(out)), nil +} diff --git a/lib/transform/transform_test.go b/lib/transform/transform_test.go index 67c7d77ac..bd5e0c9a7 100644 --- a/lib/transform/transform_test.go +++ b/lib/transform/transform_test.go @@ -3,6 +3,7 @@ package transform import ( "context" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -10,6 +11,12 @@ import ( // sync tests are in fs/sync/sync_transform_test.go to avoid import cycle issues +func newOptions(s ...string) (context.Context, error) { + ctx := context.Background() + err := SetOptions(ctx, s...) + return ctx, err +} + func TestPath(t *testing.T) { for _, test := range []struct { path string @@ -19,10 +26,10 @@ func TestPath(t *testing.T) { {"toe/toe/toe", "tictactoe/tictactoe/tictactoe"}, {"a/b/c", "tictaca/tictacb/tictacc"}, } { - err := SetOptions(context.Background(), "all,prefix=tac", "all,prefix=tic") + ctx, err := newOptions("all,prefix=tac", "all,prefix=tic") require.NoError(t, err) - got := Path(test.path, false) + got := Path(ctx, test.path, false) assert.Equal(t, test.want, got) } } @@ -34,10 +41,10 @@ func TestFileTagOnFile(t *testing.T) { }{ {"a/b/c.txt", "a/b/1c.txt"}, } { - err := SetOptions(context.Background(), "file,prefix=1") + ctx, err := newOptions("file,prefix=1") require.NoError(t, err) - got := Path(test.path, false) + got := Path(ctx, test.path, false) assert.Equal(t, test.want, got) } } @@ -49,10 +56,10 @@ func TestDirTagOnFile(t *testing.T) { }{ {"a/b/c.txt", "1a/1b/c.txt"}, } { - err := SetOptions(context.Background(), "dir,prefix=1") + ctx, err := newOptions("dir,prefix=1") require.NoError(t, err) - got := Path(test.path, false) + got := Path(ctx, test.path, false) assert.Equal(t, test.want, got) } } @@ -64,10 +71,10 @@ func TestAllTag(t *testing.T) { }{ {"a/b/c.txt", "1a/1b/1c.txt"}, } { - err := SetOptions(context.Background(), "all,prefix=1") + ctx, err := newOptions("all,prefix=1") require.NoError(t, err) - got := Path(test.path, false) + got := Path(ctx, test.path, false) assert.Equal(t, test.want, got) } } @@ -79,10 +86,10 @@ func TestFileTagOnDir(t *testing.T) { }{ {"a/b", "a/b"}, } { - err := SetOptions(context.Background(), "file,prefix=1") + ctx, err := newOptions("file,prefix=1") require.NoError(t, err) - got := Path(test.path, true) + got := Path(ctx, test.path, true) assert.Equal(t, test.want, got) } } @@ -94,10 +101,10 @@ func TestDirTagOnDir(t *testing.T) { }{ {"a/b", "1a/1b"}, } { - err := SetOptions(context.Background(), "dir,prefix=1") + ctx, err := newOptions("dir,prefix=1") require.NoError(t, err) - got := Path(test.path, true) + got := Path(ctx, test.path, true) assert.Equal(t, test.want, got) } } @@ -115,16 +122,21 @@ func TestVarious(t *testing.T) { {"stories/The Quick Brown 🦊 Fox Went to the Café!.txt", "stories/The Quick Brown 🦊 Fox Went to the Café!.txt", []string{"all,nfc"}}, {"stories/The Quick Brown 🦊 Fox Went to the Café!.txt", "stories/The Quick Brown 🦊 Fox Went to the Café!.txt", []string{"all,nfd"}}, {"stories/The Quick Brown 🦊 Fox!.txt", "stories/The Quick Brown Fox!.txt", []string{"all,ascii"}}, + {"stories/The Quick Brown 🦊 Fox!.txt", "stories/The+Quick+Brown+%F0%9F%A6%8A+Fox%21.txt", []string{"all,url"}}, {"stories/The Quick Brown Fox!.txt", "stories/The Quick Brown Fox!", []string{"all,trimsuffix=.txt"}}, {"stories/The Quick Brown Fox!.txt", "OLD_stories/OLD_The Quick Brown Fox!.txt", []string{"all,prefix=OLD_"}}, {"stories/The Quick Brown 🦊 Fox Went to the Café!.txt", "stories/The Quick Brown _ Fox Went to the Caf_!.txt", []string{"all,charmap=ISO-8859-7"}}, {"stories/The Quick Brown Fox: A Memoir [draft].txt", "stories/The Quick Brown Fox： A Memoir ［draft］.txt", []string{"all,encoder=Colon,SquareBracket"}}, {"stories/The Quick Brown 🦊 Fox Went to the Café!.txt", "stories/The Quick Brown 🦊 Fox", []string{"all,truncate=21"}}, + {"stories/The Quick Brown Fox!.txt", "stories/The Quick Brown Fox!.txt", []string{"all,command=echo"}}, + {"stories/The Quick Brown Fox!.txt", "stories/The Quick Brown Fox!.txt-" + time.Now().Local().Format("20060102"), []string{"date=-{YYYYMMDD}"}}, + {"stories/The Quick Brown Fox!.txt", "stories/The Quick Brown Fox!.txt-" + time.Now().Local().Format("2006-01-02 0304PM"), []string{"date=-{macfriendlytime}"}}, + {"stories/The Quick Brown Fox!.txt", "ababababababab/ababab ababababab ababababab ababab!abababab", []string{"all,regex=[\\.\\w]/ab"}}, } { - err := SetOptions(context.Background(), test.flags...) + ctx, err := newOptions(test.flags...) require.NoError(t, err) - got := Path(test.path, false) + got := Path(ctx, test.path, false) assert.Equal(t, test.want, got) } } diff --git a/lib/transform/transformflags/transformflags.go b/lib/transform/transformflags/transformflags.go deleted file mode 100644 index c03c2159d..000000000 --- a/lib/transform/transformflags/transformflags.go +++ /dev/null @@ -1,14 +0,0 @@ -// Package transformflags implements command line flags to set up a transform -package transformflags - -import ( - "github.com/rclone/rclone/fs/config/flags" - "github.com/rclone/rclone/lib/transform" - - "github.com/spf13/pflag" -) - -// AddFlags adds the transform flags to the command -func AddFlags(flagSet *pflag.FlagSet) { - flags.AddFlagsFromOptions(flagSet, "", transform.OptionsInfo) -}