lib/transform: refactor and add TimeFormat support

cmd/bisync/bilib/names.go

@@ -5,8 +5,6 @@ import (
 	"os"
 	"sort"
 	"strconv"
-	"strings"
-	"time"
 )
 
 // Names comprises a set of file names
@@ -85,81 +83,3 @@ func (am AliasMap) Alias(name1 string) string {
 	}
 	return name1
 }
-
-// ParseGlobs determines whether a string contains {brackets}
-// and returns the substring (including both brackets) for replacing
-// substring is first opening bracket to last closing bracket --
-// good for {{this}} but not {this}{this}
-func ParseGlobs(s string) (hasGlobs bool, substring string) {
-	open := strings.Index(s, "{")
-	close := strings.LastIndex(s, "}")
-	if open >= 0 && close > open {
-		return true, s[open : close+1]
-	}
-	return false, ""
-}
-
-// TrimBrackets converts {{this}} to this
-func TrimBrackets(s string) string {
-	return strings.Trim(s, "{}")
-}
-
-// TimeFormat converts a user-supplied string to a Go time constant, if possible
-func TimeFormat(timeFormat string) string {
-	switch timeFormat {
-	case "Layout":
-		timeFormat = time.Layout
-	case "ANSIC":
-		timeFormat = time.ANSIC
-	case "UnixDate":
-		timeFormat = time.UnixDate
-	case "RubyDate":
-		timeFormat = time.RubyDate
-	case "RFC822":
-		timeFormat = time.RFC822
-	case "RFC822Z":
-		timeFormat = time.RFC822Z
-	case "RFC850":
-		timeFormat = time.RFC850
-	case "RFC1123":
-		timeFormat = time.RFC1123
-	case "RFC1123Z":
-		timeFormat = time.RFC1123Z
-	case "RFC3339":
-		timeFormat = time.RFC3339
-	case "RFC3339Nano":
-		timeFormat = time.RFC3339Nano
-	case "Kitchen":
-		timeFormat = time.Kitchen
-	case "Stamp":
-		timeFormat = time.Stamp
-	case "StampMilli":
-		timeFormat = time.StampMilli
-	case "StampMicro":
-		timeFormat = time.StampMicro
-	case "StampNano":
-		timeFormat = time.StampNano
-	case "DateTime":
-		// timeFormat = time.DateTime // missing in go1.19
-		timeFormat = "2006-01-02 15:04:05"
-	case "DateOnly":
-		// timeFormat = time.DateOnly // missing in go1.19
-		timeFormat = "2006-01-02"
-	case "TimeOnly":
-		// timeFormat = time.TimeOnly // missing in go1.19
-		timeFormat = "15:04:05"
-	case "MacFriendlyTime", "macfriendlytime", "mac":
-		timeFormat = "2006-01-02 0304PM" // not actually a Go constant -- but useful as macOS filenames can't have colons
-	}
-	return timeFormat
-}
-
-// AppyTimeGlobs converts "myfile-{DateOnly}.txt" to "myfile-2006-01-02.txt"
-func AppyTimeGlobs(s string, t time.Time) string {
-	hasGlobs, substring := ParseGlobs(s)
-	if !hasGlobs {
-		return s
-	}
-	timeString := t.Local().Format(TimeFormat(TrimBrackets(substring)))
-	return strings.ReplaceAll(s, substring, timeString)
-}

cmd/bisync/resolve.go

@@ -96,8 +96,8 @@ func (b *bisyncRun) setResolveDefaults(ctx context.Context) error {
 	}
 	// replace glob variables, if any
 	t := time.Now() // capture static time here so it is the same for all files throughout this run
-	b.opt.ConflictSuffix1 = bilib.AppyTimeGlobs(b.opt.ConflictSuffix1, t)
+	b.opt.ConflictSuffix1 = transform.AppyTimeGlobs(b.opt.ConflictSuffix1, t)
-	b.opt.ConflictSuffix2 = bilib.AppyTimeGlobs(b.opt.ConflictSuffix2, t)
+	b.opt.ConflictSuffix2 = transform.AppyTimeGlobs(b.opt.ConflictSuffix2, t)
 
 	// append dot (intentionally allow more than one)
 	b.opt.ConflictSuffix1 = "." + b.opt.ConflictSuffix1

cmd/convmv/convmv.go

@@ -96,7 +96,7 @@ This can lead to race conditions when performing concurrent transfers. It is up
 		cmd.CheckArgs(1, 1, command, args)
 		fdst, srcFileName := cmd.NewFsFile(args[0])
 		cmd.Run(false, true, command, func() error {
-			if !transform.Transforming() {
+			if !transform.Transforming(context.Background()) {
 				return errors.New("--name-transform must be set")
 			}
 			if srcFileName == "" {

cmd/convmv/convmv_test.go

@@ -105,31 +105,32 @@ func TestTransform(t *testing.T) {
 			r := fstest.NewRun(t)
 			defer r.Finalise()
 
-			r.Mkdir(context.Background(), r.Flocal)
+			ctx := context.Background()
-			r.Mkdir(context.Background(), r.Fremote)
+			r.Mkdir(ctx, r.Flocal)
+			r.Mkdir(ctx, r.Fremote)
 			items := makeTestFiles(t, r, "dir1")
-			err := r.Fremote.Mkdir(context.Background(), "empty/empty")
+			err := r.Fremote.Mkdir(ctx, "empty/empty")
 			require.NoError(t, err)
-			err = r.Flocal.Mkdir(context.Background(), "empty/empty")
+			err = r.Flocal.Mkdir(ctx, "empty/empty")
 			require.NoError(t, err)
 			deleteDSStore(t, r)
 			r.CheckRemoteListing(t, items, []string{"dir1", "empty", "empty/empty"})
 			r.CheckLocalListing(t, items, []string{"dir1", "empty", "empty/empty"})
 
-			err = transform.SetOptions(context.Background(), tt.args.TransformOpt...)
+			err = transform.SetOptions(ctx, tt.args.TransformOpt...)
 			require.NoError(t, err)
 
-			err = sync.Transform(context.Background(), r.Fremote, true, true)
+			err = sync.Transform(ctx, r.Fremote, true, true)
 			assert.NoError(t, err)
-			compareNames(t, r, items)
+			compareNames(ctx, t, r, items)
 
-			transformedItems := transformItems(t, items)
+			transformedItems := transformItems(ctx, t, items)
-			r.CheckRemoteListing(t, transformedItems, []string{transform.Path("dir1", true), transform.Path("empty", true), transform.Path("empty/empty", true)})
+			r.CheckRemoteListing(t, transformedItems, []string{transform.Path(ctx, "dir1", true), transform.Path(ctx, "empty", true), transform.Path(ctx, "empty/empty", true)})
-			err = transform.SetOptions(context.Background(), tt.args.TransformBackOpt...)
+			err = transform.SetOptions(ctx, tt.args.TransformBackOpt...)
 			require.NoError(t, err)
-			err = sync.Transform(context.Background(), r.Fremote, true, true)
+			err = sync.Transform(ctx, r.Fremote, true, true)
 			assert.NoError(t, err)
-			compareNames(t, r, transformedItems)
+			compareNames(ctx, t, r, transformedItems)
 
 			if tt.args.Lossless {
 				deleteDSStore(t, r)
@@ -191,7 +192,7 @@ func deleteDSStore(t *testing.T, r *fstest.Run) {
 	assert.NoError(t, err)
 }
 
-func compareNames(t *testing.T, r *fstest.Run, items []fstest.Item) {
+func compareNames(ctx context.Context, t *testing.T, r *fstest.Run, items []fstest.Item) {
 	var entries fs.DirEntries
 
 	deleteDSStore(t, r)
@@ -212,8 +213,8 @@ func compareNames(t *testing.T, r *fstest.Run, items []fstest.Item) {
 
 	// sort by CONVERTED name
 	slices.SortStableFunc(items, func(a, b fstest.Item) int {
-		aConv := transform.Path(a.Path, false)
+		aConv := transform.Path(ctx, a.Path, false)
-		bConv := transform.Path(b.Path, false)
+		bConv := transform.Path(ctx, b.Path, false)
 		return cmp.Compare(aConv, bConv)
 	})
 	slices.SortStableFunc(entries, func(a, b fs.DirEntry) int {
@@ -221,16 +222,16 @@ func compareNames(t *testing.T, r *fstest.Run, items []fstest.Item) {
 	})
 
 	for i, e := range entries {
-		expect := transform.Path(items[i].Path, false)
+		expect := transform.Path(ctx, items[i].Path, false)
 		msg := fmt.Sprintf("expected %v, got %v", detectEncoding(expect), detectEncoding(e.Remote()))
 		assert.Equal(t, expect, e.Remote(), msg)
 	}
 }
 
-func transformItems(t *testing.T, items []fstest.Item) []fstest.Item {
+func transformItems(ctx context.Context, t *testing.T, items []fstest.Item) []fstest.Item {
 	transformedItems := []fstest.Item{}
 	for _, item := range items {
-		newPath := transform.Path(item.Path, false)
+		newPath := transform.Path(ctx, item.Path, false)
 		newItem := item
 		newItem.Path = newPath
 		transformedItems = append(transformedItems, newItem)

cmd/help.go

@@ -16,7 +16,6 @@ import (
 	"github.com/rclone/rclone/fs/log/logflags"
 	"github.com/rclone/rclone/fs/rc/rcflags"
 	"github.com/rclone/rclone/lib/atexit"
-	"github.com/rclone/rclone/lib/transform/transformflags"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 	"golang.org/x/text/cases"
@@ -138,7 +137,6 @@ func setupRootCommand(rootCmd *cobra.Command) {
 	// Add global flags
 	configflags.AddFlags(ci, pflag.CommandLine)
 	filterflags.AddFlags(pflag.CommandLine)
-	transformflags.AddFlags(pflag.CommandLine)
 	rcflags.AddFlags(pflag.CommandLine)
 	logflags.AddFlags(pflag.CommandLine)
 

fs/config.go

@@ -570,6 +570,11 @@ uploads or downloads to limit the number of total connections.
 `, "|", "`"),
 	Default:  0,
 	Advanced: true,
+}, {
+	Name:    "name_transform",
+	Default: []string{},
+	Help:    "`--name-transform` introduces path name transformations for `rclone copy`, `rclone sync`, and `rclone move`. These transformations enable modifications to source and destination file names by applying prefixes, suffixes, and other alterations during transfer operations. For detailed docs and examples, see [`convmv`](/commands/rclone_convmv/).",
+	Groups:  "Filter",
 }}
 
 // ConfigInfo is filesystem config options
@@ -681,6 +686,7 @@ type ConfigInfo struct {
 	PartialSuffix              string            `config:"partial_suffix"`
 	MetadataMapper             SpaceSepList      `config:"metadata_mapper"`
 	MaxConnections             int               `config:"max_connections"`
+	NameTransform              []string          `config:"name_transform"`
 }
 
 func init() {

fs/march/march.go

@@ -87,11 +87,8 @@ func (m *March) srcKey(entry fs.DirEntry) string {
 		return ""
 	}
 	name := path.Base(entry.Remote())
-	name = transform.Path(name, fs.DirEntryType(entry) == "directory")
+	name = transform.Path(m.Ctx, name, fs.DirEntryType(entry) == "directory")
-	for _, transform := range m.transforms {
+	return transforms(name, m.transforms)
-		name = transform(name)
-	}
-	return name
 }
 
 // dstKey turns a directory entry into a sort key using the defined transforms.
@@ -99,8 +96,11 @@ func (m *March) dstKey(entry fs.DirEntry) string {
 	if entry == nil {
 		return ""
 	}
-	name := path.Base(entry.Remote())
+	return transforms(path.Base(entry.Remote()), m.transforms)
-	for _, transform := range m.transforms {
+}
+
+func transforms(name string, transforms []matchTransformFn) string {
+	for _, transform := range transforms {
 		name = transform(name)
 	}
 	return name

fs/operations/copy.go

@@ -391,7 +391,7 @@ func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Obj
 		f:           f,
 		dstFeatures: f.Features(),
 		dst:         dst,
-		remote:      transform.Path(remote, false),
+		remote:      transform.Path(ctx, remote, false),
 		src:         src,
 		ci:          ci,
 		tr:          tr,
@@ -400,7 +400,7 @@ func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Obj
 	}
 	c.hashType, c.hashOption = CommonHash(ctx, f, src.Fs())
 	if c.dst != nil {
-		c.remote = transform.Path(c.dst.Remote(), false)
+		c.remote = transform.Path(ctx, c.dst.Remote(), false)
 	}
 	// Are we using partials?
 	//

fs/operations/operations.go

@@ -426,7 +426,7 @@ func MoveTransfer(ctx context.Context, fdst fs.Fs, dst fs.Object, remote string,
 // move - see Move for help
 func move(ctx context.Context, fdst fs.Fs, dst fs.Object, remote string, src fs.Object, isTransfer bool) (newDst fs.Object, err error) {
 	origRemote := remote // avoid double-transform on fallback to copy
-	remote = transform.Path(remote, false)
+	remote = transform.Path(ctx, remote, false)
 	ci := fs.GetConfig(ctx)
 	var tr *accounting.Transfer
 	if isTransfer {
@@ -450,7 +450,7 @@ func move(ctx context.Context, fdst fs.Fs, dst fs.Object, remote string, src fs.
 	if doMove := fdst.Features().Move; doMove != nil && (SameConfig(src.Fs(), fdst) || (SameRemoteType(src.Fs(), fdst) && (fdst.Features().ServerSideAcrossConfigs || ci.ServerSideAcrossConfigs))) {
 		// Delete destination if it exists and is not the same file as src (could be same file while seemingly different if the remote is case insensitive)
 		if dst != nil {
-			remote = transform.Path(dst.Remote(), false)
+			remote = transform.Path(ctx, dst.Remote(), false)
 			if !SameObject(src, dst) {
 				err = DeleteFile(ctx, dst)
 				if err != nil {
@@ -2206,50 +2206,10 @@ func (l *ListFormat) SetOutput(output []func(entry *ListJSONItem) string) {
 
 // AddModTime adds file's Mod Time to output
 func (l *ListFormat) AddModTime(timeFormat string) {
-	switch timeFormat {
+	if timeFormat == "" {
-	case "":
 		timeFormat = "2006-01-02 15:04:05"
-	case "Layout":
+	} else {
-		timeFormat = time.Layout
+		timeFormat = transform.TimeFormat(timeFormat)
-	case "ANSIC":
-		timeFormat = time.ANSIC
-	case "UnixDate":
-		timeFormat = time.UnixDate
-	case "RubyDate":
-		timeFormat = time.RubyDate
-	case "RFC822":
-		timeFormat = time.RFC822
-	case "RFC822Z":
-		timeFormat = time.RFC822Z
-	case "RFC850":
-		timeFormat = time.RFC850
-	case "RFC1123":
-		timeFormat = time.RFC1123
-	case "RFC1123Z":
-		timeFormat = time.RFC1123Z
-	case "RFC3339":
-		timeFormat = time.RFC3339
-	case "RFC3339Nano":
-		timeFormat = time.RFC3339Nano
-	case "Kitchen":
-		timeFormat = time.Kitchen
-	case "Stamp":
-		timeFormat = time.Stamp
-	case "StampMilli":
-		timeFormat = time.StampMilli
-	case "StampMicro":
-		timeFormat = time.StampMicro
-	case "StampNano":
-		timeFormat = time.StampNano
-	case "DateTime":
-		// timeFormat = time.DateTime // missing in go1.19
-		timeFormat = "2006-01-02 15:04:05"
-	case "DateOnly":
-		// timeFormat = time.DateOnly // missing in go1.19
-		timeFormat = "2006-01-02"
-	case "TimeOnly":
-		// timeFormat = time.TimeOnly // missing in go1.19
-		timeFormat = "15:04:05"
 	}
 	l.AppendOutput(func(entry *ListJSONItem) string {
 		return entry.ModTime.When.Local().Format(timeFormat)

fs/sync/sync.go

@@ -1125,7 +1125,7 @@ func (s *syncCopyMove) copyDirMetadata(ctx context.Context, f fs.Fs, dst fs.Dire
 			newDst, err = operations.SetDirModTime(ctx, f, dst, dir, src.ModTime(ctx))
 		}
 	}
-	if transform.Transforming() && newDst != nil && src.Remote() != newDst.Remote() {
+	if transform.Transforming(ctx) && newDst != nil && src.Remote() != newDst.Remote() {
 		s.markParentNotEmpty(src)
 	}
 	// If we need to set modtime after and we created a dir, then save it for later
@@ -1260,8 +1260,8 @@ func (s *syncCopyMove) SrcOnly(src fs.DirEntry) (recurse bool) {
 		s.logger(s.ctx, operations.MissingOnDst, src, nil, fs.ErrorIsDir)
 
 		// Create the directory and make sure the Metadata/ModTime is correct
-		s.copyDirMetadata(s.ctx, s.fdst, nil, transform.Path(x.Remote(), true), x)
+		s.copyDirMetadata(s.ctx, s.fdst, nil, transform.Path(s.ctx, x.Remote(), true), x)
-		s.markDirModified(transform.Path(x.Remote(), true))
+		s.markDirModified(transform.Path(s.ctx, x.Remote(), true))
 		return true
 	default:
 		panic("Bad object in DirEntries")
@@ -1294,9 +1294,9 @@ func (s *syncCopyMove) Match(ctx context.Context, dst, src fs.DirEntry) (recurse
 		}
 	case fs.Directory:
 		// Do the same thing to the entire contents of the directory
-		srcX = fs.NewOverrideDirectory(srcX, transform.Path(src.Remote(), true))
+		srcX = fs.NewOverrideDirectory(srcX, transform.Path(ctx, src.Remote(), true))
 		src = srcX
-		if !transform.Transforming() || src.Remote() != dst.Remote() {
+		if !transform.Transforming(ctx) || src.Remote() != dst.Remote() {
 			s.markParentNotEmpty(src)
 		}
 		dstX, ok := dst.(fs.Directory)

fs/sync/sync_test.go

@@ -2981,7 +2981,7 @@ func predictDstFromLogger(ctx context.Context) context.Context {
 			if winner.Err != nil {
 				errMsg = ";" + winner.Err.Error()
 			}
-			operations.SyncFprintf(opt.JSON, "%s;%s;%v;%s%s\n", file.ModTime(ctx).Local().Format(timeFormat), checksum, file.Size(), transform.Path(file.Remote(), false), errMsg) // TODO: should the transform be handled in the sync instead of here?
+			operations.SyncFprintf(opt.JSON, "%s;%s;%v;%s%s\n", file.ModTime(ctx).Local().Format(timeFormat), checksum, file.Size(), transform.Path(ctx, file.Remote(), false), errMsg) // TODO: should the transform be handled in the sync instead of here?
 		}
 	}
 	return operations.WithSyncLogger(ctx, opt)

fs/sync/sync_transform_test.go

@@ -6,7 +6,7 @@ import (
 	"cmp"
 	"context"
 	"fmt"
-	"path/filepath"
+	"path"
 	"slices"
 	"strings"
 	"testing"
@@ -96,25 +96,26 @@ func TestTransform(t *testing.T) {
 			r := fstest.NewRun(t)
 			defer r.Finalise()
 
-			r.Mkdir(context.Background(), r.Flocal)
+			ctx := context.Background()
-			r.Mkdir(context.Background(), r.Fremote)
+			r.Mkdir(ctx, r.Flocal)
+			r.Mkdir(ctx, r.Fremote)
 			items := makeTestFiles(t, r, "dir1")
 			deleteDSStore(t, r)
 			r.CheckRemoteListing(t, items, nil)
 			r.CheckLocalListing(t, items, nil)
 
-			err := transform.SetOptions(context.Background(), tt.args.TransformOpt...)
+			err := transform.SetOptions(ctx, tt.args.TransformOpt...)
 			require.NoError(t, err)
 
-			err = Sync(context.Background(), r.Fremote, r.Flocal, true)
+			err = Sync(ctx, r.Fremote, r.Flocal, true)
 			assert.NoError(t, err)
-			compareNames(t, r, items)
+			compareNames(ctx, t, r, items)
 
-			err = transform.SetOptions(context.Background(), tt.args.TransformBackOpt...)
+			err = transform.SetOptions(ctx, tt.args.TransformBackOpt...)
 			require.NoError(t, err)
-			err = Sync(context.Background(), r.Fremote, r.Flocal, true)
+			err = Sync(ctx, r.Fremote, r.Flocal, true)
 			assert.NoError(t, err)
-			compareNames(t, r, items)
+			compareNames(ctx, t, r, items)
 
 			if tt.args.Lossless {
 				deleteDSStore(t, r)
@@ -138,8 +139,9 @@ func makeTestFiles(t *testing.T, r *fstest.Run, dir string) []fstest.Item {
 		for i := rune(0); i < 7; i++ {
 			out.WriteRune(c + i)
 		}
-		fileName := filepath.Join(dir, fmt.Sprintf("%04d-%s.txt", n, out.String()))
+		fileName := path.Join(dir, fmt.Sprintf("%04d-%s.txt", n, out.String()))
 		fileName = strings.ToValidUTF8(fileName, "")
+		fileName = strings.NewReplacer(":", "", "<", "", ">", "", "?", "").Replace(fileName) // remove characters illegal on windows
 
 		if debug != "" {
 			fileName = debug
@@ -174,7 +176,7 @@ func deleteDSStore(t *testing.T, r *fstest.Run) {
 	assert.NoError(t, err)
 }
 
-func compareNames(t *testing.T, r *fstest.Run, items []fstest.Item) {
+func compareNames(ctx context.Context, t *testing.T, r *fstest.Run, items []fstest.Item) {
 	var entries fs.DirEntries
 
 	deleteDSStore(t, r)
@@ -195,8 +197,8 @@ func compareNames(t *testing.T, r *fstest.Run, items []fstest.Item) {
 
 	// sort by CONVERTED name
 	slices.SortStableFunc(items, func(a, b fstest.Item) int {
-		aConv := transform.Path(a.Path, false)
+		aConv := transform.Path(ctx, a.Path, false)
-		bConv := transform.Path(b.Path, false)
+		bConv := transform.Path(ctx, b.Path, false)
 		return cmp.Compare(aConv, bConv)
 	})
 	slices.SortStableFunc(entries, func(a, b fs.DirEntry) int {
@@ -204,7 +206,7 @@ func compareNames(t *testing.T, r *fstest.Run, items []fstest.Item) {
 	})
 
 	for i, e := range entries {
-		expect := transform.Path(items[i].Path, false)
+		expect := transform.Path(ctx, items[i].Path, false)
 		msg := fmt.Sprintf("expected %v, got %v", detectEncoding(expect), detectEncoding(e.Remote()))
 		assert.Equal(t, expect, e.Remote(), msg)
 	}
@@ -477,7 +479,5 @@ func TestError(t *testing.T) {
 	assert.Error(t, err)
 
 	r.CheckLocalListing(t, []fstest.Item{file1}, []string{"toe", "toe/toe"})
-	r.CheckRemoteListing(t, []fstest.Item{}, []string{})
+	r.CheckRemoteListing(t, []fstest.Item{file1}, []string{"toe", "toe/toe"})
-	err = transform.SetOptions(ctx, "") // has illegal character
-	assert.NoError(t, err)
 }

lib/transform/help.go

@@ -27,7 +27,7 @@ var commandList = []commands{
 	{command: "--name-transform trimsuffix=XXXX", description: "Removes XXXX if it appears at the end of the file name."},
 	{command: "--name-transform regex=/pattern/replacement/", description: "Applies a regex-based transformation."},
 	{command: "--name-transform replace=old:new", description: "Replaces occurrences of old with new in the file name."},
-	{command: "--name-transform date=YYYYMMDD", description: "Appends or prefixes the specified date format."},
+	{command: "--name-transform date={YYYYMMDD}", description: "Appends or prefixes the specified date format."},
 	{command: "--name-transform truncate=N", description: "Truncates the file name to a maximum of N characters."},
 	{command: "--name-transform base64encode", description: "Encodes the file name in Base64."},
 	{command: "--name-transform base64decode", description: "Decodes a Base64-encoded file name."},
@@ -59,6 +59,10 @@ var examples = []example{
 	{"stories/The Quick Brown 🦊 Fox Went to the Café!.txt", []string{"all,charmap=ISO-8859-7"}},
 	{"stories/The Quick Brown Fox: A Memoir [draft].txt", []string{"all,encoder=Colon,SquareBracket"}},
 	{"stories/The Quick Brown 🦊 Fox Went to the Café!.txt", []string{"all,truncate=21"}},
+	{"stories/The Quick Brown Fox!.txt", []string{"all,command=echo"}},
+	{"stories/The Quick Brown Fox!", []string{"date=-{YYYYMMDD}"}},
+	{"stories/The Quick Brown Fox!", []string{"date=-{macfriendlytime}"}},
+	{"stories/The Quick Brown Fox!.txt", []string{"all,regex=[\\.\\w]/ab"}},
 }
 
 func (e example) command() string {
@@ -70,11 +74,12 @@ func (e example) command() string {
 }
 
 func (e example) output() string {
-	err := SetOptions(context.Background(), e.flags...)
+	ctx := context.Background()
+	err := SetOptions(ctx, e.flags...)
 	if err != nil {
 		fs.Errorf(nil, "error generating help text: %v", err)
 	}
-	return Path(e.path, false)
+	return Path(ctx, e.path, false)
 }
 
 // go run ./ convmv --help
@@ -84,7 +89,6 @@ func sprintExamples() string {
 		s += fmt.Sprintf("```\n%s\n", e.command())
 		s += fmt.Sprintf("// Output: %s\n```\n\n", e.output())
 	}
-	Opt = Options{} // reset
 	return s
 }
 

lib/transform/options.go

@@ -8,31 +8,12 @@ import (
 	"github.com/rclone/rclone/fs"
 )
 
-func init() {
-	fs.RegisterGlobalOptions(fs.OptionsInfo{Name: "name_transform", Opt: &Opt.Flags, Options: OptionsInfo, Reload: Reload})
-}
-
 type transform struct {
 	key   transformAlgo // for example, "prefix"
 	value string        // for example, "some_prefix_"
 	tag   tag           // file, dir, or all
 }
 
-// Options stores the parsed and unparsed transform options.
-// their order must never be changed or sorted.
-type Options struct {
-	Flags      Flags       // unparsed flag value like "file,prefix=ABC"
-	transforms []transform // parsed from NameTransform
-}
-
-// Flags is a slice of unparsed values set from command line flags or env vars
-type Flags struct {
-	NameTransform []string `config:"name_transform"`
-}
-
-// Opt is the default options modified by the environment variables and command line flags
-var Opt Options
-
 // tag controls which part of the file path is affected (file, dir, all)
 type tag int
 
@@ -43,39 +24,38 @@ const (
 	all             // Transform the entire path for files and directories
 )
 
-// OptionsInfo describes the Options in use
+// Transforming returns true when transforms are in use
-var OptionsInfo = fs.Options{{
+func Transforming(ctx context.Context) bool {
-	Name:    "name_transform",
+	ci := fs.GetConfig(ctx)
-	Default: []string{},
+	return len(ci.NameTransform) > 0
-	Help:    "`--name-transform` introduces path name transformations for `rclone copy`, `rclone sync`, and `rclone move`. These transformations enable modifications to source and destination file names by applying prefixes, suffixes, and other alterations during transfer operations. For detailed docs and examples, see [`convmv`](/commands/rclone_convmv/).",
-	Groups:  "Filter",
-}}
-
-// Reload the transform options from the flags
-func Reload(ctx context.Context) (err error) {
-	return newOpt(Opt)
 }
 
-// SetOptions sets the options from flags passed in.
+// SetOptions sets the options in ctx from flags passed in.
 // Any existing flags will be overwritten.
 // s should be in the same format as cmd line flags, i.e. "all,prefix=XXX"
 func SetOptions(ctx context.Context, s ...string) (err error) {
-	Opt = Options{Flags: Flags{NameTransform: s}}
+	ci := fs.GetConfig(ctx)
-	return Reload(ctx)
+	ci.NameTransform = s
+	_, err = getOptions(ctx)
+	return err
 }
 
-// overwite Opt.transforms with values from Opt.Flags
+// getOptions sets the options from flags passed in.
-func newOpt(opt Options) (err error) {
+func getOptions(ctx context.Context) (opt []transform, err error) {
-	Opt.transforms = []transform{}
+	if !Transforming(ctx) {
+		return opt, nil
+	}
 
-	for _, transform := range opt.Flags.NameTransform {
+	ci := fs.GetConfig(ctx)
+	for _, transform := range ci.NameTransform {
 		t, err := parse(transform)
 		if err != nil {
-			return err
+			return opt, err
 		}
-		Opt.transforms = append(Opt.transforms, t)
+		opt = append(opt, t)
 	}
-	return nil
+	// TODO: should we store opt in ci and skip re-parsing when present, for efficiency?
+	return opt, nil
 }
 
 // parse a single instance of --name-transform
@@ -161,6 +141,10 @@ func (t *transform) requiresValue() bool {
 		return true
 	case ConvDecoder:
 		return true
+	case ConvRegex:
+		return true
+	case ConvCommand:
+		return true
 	}
 	return false
 }
@@ -197,7 +181,8 @@ const (
 	ConvTitlecase
 	ConvASCII
 	ConvURL
-	ConvMapper
+	ConvRegex
+	ConvCommand
 )
 
 type transformChoices struct{}
@@ -231,7 +216,8 @@ func (transformChoices) Choices() []string {
 		ConvTitlecase:           "titlecase",
 		ConvASCII:               "ascii",
 		ConvURL:                 "url",
-		ConvMapper:              "mapper",
+		ConvRegex:               "regex",
+		ConvCommand:             "command",
 	}
 }
 

lib/transform/transform.go

@@ -2,14 +2,19 @@
 package transform
 
 import (
+	"bytes"
+	"context"
 	"encoding/base64"
 	"errors"
 	"fmt"
 	"mime"
-	"os"
+	"net/url"
+	"os/exec"
 	"path"
+	"regexp"
 	"strconv"
 	"strings"
+	"time"
 	"unicode/utf8"
 
 	"github.com/rclone/rclone/fs"
@@ -21,14 +26,18 @@ import (
 // Path transforms a path s according to the --name-transform options in use
 //
 // If no transforms are in use, s is returned unchanged
-func Path(s string, isDir bool) string {
+func Path(ctx context.Context, s string, isDir bool) string {
-	if !Transforming() {
+	if !Transforming(ctx) {
 		return s
 	}
 
-	var err error
 	old := s
-	for _, t := range Opt.transforms {
+	opt, err := getOptions(ctx)
+	if err != nil {
+		err = fs.CountError(ctx, err)
+		fs.Errorf(s, "Failed to parse transform flags: %v", err)
+	}
+	for _, t := range opt {
 		if isDir && t.tag == file {
 			continue
 		}
@@ -39,20 +48,21 @@ func Path(s string, isDir bool) string {
 			s, err = transformPath(s, t, baseOnly)
 		}
 		if err != nil {
-			fs.Error(s, err.Error()) // TODO: return err instead of logging it?
+			err = fs.CountError(ctx, err)
+			fs.Errorf(s, "Failed to transform: %v", err)
 		}
 	}
 	if old != s {
 		fs.Debugf(old, "transformed to: %v", s)
 	}
+	if strings.Count(old, "/") != strings.Count(s, "/") {
+		err = fs.CountError(ctx, fmt.Errorf("number of path segments must match: %v (%v), %v (%v)", old, strings.Count(old, "/"), s, strings.Count(s, "/")))
+		fs.Errorf(old, "%v", err)
+		return old
+	}
 	return s
 }
 
-// Transforming returns true when transforms are in use
-func Transforming() bool {
-	return len(Opt.transforms) > 0
-}
-
 // transformPath transforms a path string according to the chosen TransformAlgo.
 // Each path segment is transformed separately, to preserve path separators.
 // If baseOnly is true, only the base will be transformed (useful for renaming while walking a dir tree recursively.)
@@ -71,7 +81,7 @@ func transformPath(s string, t transform, baseOnly bool) (string, error) {
 		return path.Join(path.Dir(s), transformedBase), err
 	}
 
-	segments := strings.Split(s, string(os.PathSeparator))
+	segments := strings.Split(s, "/")
 	transformedSegments := make([]string, len(segments))
 	for _, seg := range segments {
 		convSeg, err := transformPathSegment(seg, t)
@@ -185,6 +195,19 @@ func transformPathSegment(s string, t transform) (string, error) {
 		return strings.ToTitle(s), nil
 	case ConvASCII:
 		return toASCII(s), nil
+	case ConvURL:
+		return url.QueryEscape(s), nil
+	case ConvDate:
+		return s + AppyTimeGlobs(t.value, time.Now()), nil
+	case ConvRegex:
+		split := strings.Split(t.value, "/")
+		if len(split) != 2 {
+			return s, fmt.Errorf("regex syntax error: %v", t.value)
+		}
+		re := regexp.MustCompile(split[0])
+		return re.ReplaceAllString(s, split[1]), nil
+	case ConvCommand:
+		return mapper(s, t.value)
 	default:
 		return "", errors.New("this option is not yet implemented")
 	}
@@ -216,7 +239,7 @@ func SuffixKeepExtension(remote string, suffix string) string {
 
 // forbid transformations that add/remove path separators
 func validateSegment(s string) error {
-	if s == "" {
+	if strings.TrimSpace(s) == "" {
 		return errors.New("transform cannot render path segments empty")
 	}
 	if strings.ContainsRune(s, '/') {
@@ -224,3 +247,89 @@ func validateSegment(s string) error {
 	}
 	return nil
 }
+
+// ParseGlobs determines whether a string contains {brackets}
+// and returns the substring (including both brackets) for replacing
+// substring is first opening bracket to last closing bracket --
+// good for {{this}} but not {this}{this}
+func ParseGlobs(s string) (hasGlobs bool, substring string) {
+	open := strings.Index(s, "{")
+	close := strings.LastIndex(s, "}")
+	if open >= 0 && close > open {
+		return true, s[open : close+1]
+	}
+	return false, ""
+}
+
+// TrimBrackets converts {{this}} to this
+func TrimBrackets(s string) string {
+	return strings.Trim(s, "{}")
+}
+
+// TimeFormat converts a user-supplied string to a Go time constant, if possible
+func TimeFormat(timeFormat string) string {
+	switch timeFormat {
+	case "Layout":
+		timeFormat = time.Layout
+	case "ANSIC":
+		timeFormat = time.ANSIC
+	case "UnixDate":
+		timeFormat = time.UnixDate
+	case "RubyDate":
+		timeFormat = time.RubyDate
+	case "RFC822":
+		timeFormat = time.RFC822
+	case "RFC822Z":
+		timeFormat = time.RFC822Z
+	case "RFC850":
+		timeFormat = time.RFC850
+	case "RFC1123":
+		timeFormat = time.RFC1123
+	case "RFC1123Z":
+		timeFormat = time.RFC1123Z
+	case "RFC3339":
+		timeFormat = time.RFC3339
+	case "RFC3339Nano":
+		timeFormat = time.RFC3339Nano
+	case "Kitchen":
+		timeFormat = time.Kitchen
+	case "Stamp":
+		timeFormat = time.Stamp
+	case "StampMilli":
+		timeFormat = time.StampMilli
+	case "StampMicro":
+		timeFormat = time.StampMicro
+	case "StampNano":
+		timeFormat = time.StampNano
+	case "DateTime":
+		timeFormat = time.DateTime
+	case "DateOnly":
+		timeFormat = time.DateOnly
+	case "TimeOnly":
+		timeFormat = time.TimeOnly
+	case "MacFriendlyTime", "macfriendlytime", "mac":
+		timeFormat = "2006-01-02 0304PM" // not actually a Go constant -- but useful as macOS filenames can't have colons
+	case "YYYYMMDD":
+		timeFormat = "20060102"
+	}
+	return timeFormat
+}
+
+// AppyTimeGlobs converts "myfile-{DateOnly}.txt" to "myfile-2006-01-02.txt"
+func AppyTimeGlobs(s string, t time.Time) string {
+	hasGlobs, substring := ParseGlobs(s)
+	if !hasGlobs {
+		return s
+	}
+	timeString := t.Local().Format(TimeFormat(TrimBrackets(substring)))
+	return strings.ReplaceAll(s, substring, timeString)
+}
+
+func mapper(s string, command string) (string, error) {
+	out, err := exec.Command(command, s).CombinedOutput()
+	if err != nil {
+		out = bytes.TrimSpace(out)
+		return s, fmt.Errorf("%s: error running command %q: %v", out, command+" "+s, err)
+	}
+	return string(bytes.TrimSpace(out)), nil
+}

lib/transform/transform_test.go

@@ -3,6 +3,7 @@ package transform
 import (
 	"context"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -10,6 +11,12 @@ import (
 
 // sync tests are in fs/sync/sync_transform_test.go to avoid import cycle issues
 
+func newOptions(s ...string) (context.Context, error) {
+	ctx := context.Background()
+	err := SetOptions(ctx, s...)
+	return ctx, err
+}
+
 func TestPath(t *testing.T) {
 	for _, test := range []struct {
 		path string
@@ -19,10 +26,10 @@ func TestPath(t *testing.T) {
 		{"toe/toe/toe", "tictactoe/tictactoe/tictactoe"},
 		{"a/b/c", "tictaca/tictacb/tictacc"},
 	} {
-		err := SetOptions(context.Background(), "all,prefix=tac", "all,prefix=tic")
+		ctx, err := newOptions("all,prefix=tac", "all,prefix=tic")
 		require.NoError(t, err)
 
-		got := Path(test.path, false)
+		got := Path(ctx, test.path, false)
 		assert.Equal(t, test.want, got)
 	}
 }
@@ -34,10 +41,10 @@ func TestFileTagOnFile(t *testing.T) {
 	}{
 		{"a/b/c.txt", "a/b/1c.txt"},
 	} {
-		err := SetOptions(context.Background(), "file,prefix=1")
+		ctx, err := newOptions("file,prefix=1")
 		require.NoError(t, err)
 
-		got := Path(test.path, false)
+		got := Path(ctx, test.path, false)
 		assert.Equal(t, test.want, got)
 	}
 }
@@ -49,10 +56,10 @@ func TestDirTagOnFile(t *testing.T) {
 	}{
 		{"a/b/c.txt", "1a/1b/c.txt"},
 	} {
-		err := SetOptions(context.Background(), "dir,prefix=1")
+		ctx, err := newOptions("dir,prefix=1")
 		require.NoError(t, err)
 
-		got := Path(test.path, false)
+		got := Path(ctx, test.path, false)
 		assert.Equal(t, test.want, got)
 	}
 }
@@ -64,10 +71,10 @@ func TestAllTag(t *testing.T) {
 	}{
 		{"a/b/c.txt", "1a/1b/1c.txt"},
 	} {
-		err := SetOptions(context.Background(), "all,prefix=1")
+		ctx, err := newOptions("all,prefix=1")
 		require.NoError(t, err)
 
-		got := Path(test.path, false)
+		got := Path(ctx, test.path, false)
 		assert.Equal(t, test.want, got)
 	}
 }
@@ -79,10 +86,10 @@ func TestFileTagOnDir(t *testing.T) {
 	}{
 		{"a/b", "a/b"},
 	} {
-		err := SetOptions(context.Background(), "file,prefix=1")
+		ctx, err := newOptions("file,prefix=1")
 		require.NoError(t, err)
 
-		got := Path(test.path, true)
+		got := Path(ctx, test.path, true)
 		assert.Equal(t, test.want, got)
 	}
 }
@@ -94,10 +101,10 @@ func TestDirTagOnDir(t *testing.T) {
 	}{
 		{"a/b", "1a/1b"},
 	} {
-		err := SetOptions(context.Background(), "dir,prefix=1")
+		ctx, err := newOptions("dir,prefix=1")
 		require.NoError(t, err)
 
-		got := Path(test.path, true)
+		got := Path(ctx, test.path, true)
 		assert.Equal(t, test.want, got)
 	}
 }
@@ -115,16 +122,21 @@ func TestVarious(t *testing.T) {
 		{"stories/The Quick Brown 🦊 Fox Went to the Café!.txt", "stories/The Quick Brown 🦊 Fox Went to the Café!.txt", []string{"all,nfc"}},
 		{"stories/The Quick Brown 🦊 Fox Went to the Café!.txt", "stories/The Quick Brown 🦊 Fox Went to the Café!.txt", []string{"all,nfd"}},
 		{"stories/The Quick Brown 🦊 Fox!.txt", "stories/The Quick Brown  Fox!.txt", []string{"all,ascii"}},
+		{"stories/The Quick Brown 🦊 Fox!.txt", "stories/The+Quick+Brown+%F0%9F%A6%8A+Fox%21.txt", []string{"all,url"}},
 		{"stories/The Quick Brown Fox!.txt", "stories/The Quick Brown Fox!", []string{"all,trimsuffix=.txt"}},
 		{"stories/The Quick Brown Fox!.txt", "OLD_stories/OLD_The Quick Brown Fox!.txt", []string{"all,prefix=OLD_"}},
 		{"stories/The Quick Brown 🦊 Fox Went to the Café!.txt", "stories/The Quick Brown _ Fox Went to the Caf_!.txt", []string{"all,charmap=ISO-8859-7"}},
 		{"stories/The Quick Brown Fox: A Memoir [draft].txt", "stories/The Quick Brown Fox： A Memoir ［draft］.txt", []string{"all,encoder=Colon,SquareBracket"}},
 		{"stories/The Quick Brown 🦊 Fox Went to the Café!.txt", "stories/The Quick Brown 🦊 Fox", []string{"all,truncate=21"}},
+		{"stories/The Quick Brown Fox!.txt", "stories/The Quick Brown Fox!.txt", []string{"all,command=echo"}},
+		{"stories/The Quick Brown Fox!.txt", "stories/The Quick Brown Fox!.txt-" + time.Now().Local().Format("20060102"), []string{"date=-{YYYYMMDD}"}},
+		{"stories/The Quick Brown Fox!.txt", "stories/The Quick Brown Fox!.txt-" + time.Now().Local().Format("2006-01-02 0304PM"), []string{"date=-{macfriendlytime}"}},
+		{"stories/The Quick Brown Fox!.txt", "ababababababab/ababab ababababab ababababab ababab!abababab", []string{"all,regex=[\\.\\w]/ab"}},
 	} {
-		err := SetOptions(context.Background(), test.flags...)
+		ctx, err := newOptions(test.flags...)
 		require.NoError(t, err)
 
-		got := Path(test.path, false)
+		got := Path(ctx, test.path, false)
 		assert.Equal(t, test.want, got)
 	}
 }

lib/transform/transformflags/transformflags.go

@@ -1,14 +0,0 @@
-// Package transformflags implements command line flags to set up a transform
-package transformflags
-
-import (
-	"github.com/rclone/rclone/fs/config/flags"
-	"github.com/rclone/rclone/lib/transform"
-
-	"github.com/spf13/pflag"
-)
-
-// AddFlags adds the transform flags to the command
-func AddFlags(flagSet *pflag.FlagSet) {
-	flags.AddFlagsFromOptions(flagSet, "", transform.OptionsInfo)
-}