diff --git a/cmd/all/all.go b/cmd/all/all.go index 18c04eaf7..00bdbfa8b 100644 --- a/cmd/all/all.go +++ b/cmd/all/all.go @@ -15,6 +15,7 @@ import ( _ "github.com/rclone/rclone/cmd/cleanup" _ "github.com/rclone/rclone/cmd/cmount" _ "github.com/rclone/rclone/cmd/config" + _ "github.com/rclone/rclone/cmd/convmv" _ "github.com/rclone/rclone/cmd/copy" _ "github.com/rclone/rclone/cmd/copyto" _ "github.com/rclone/rclone/cmd/copyurl" diff --git a/cmd/convmv/convmv.go b/cmd/convmv/convmv.go new file mode 100644 index 000000000..a54c8206e --- /dev/null +++ b/cmd/convmv/convmv.go @@ -0,0 +1,108 @@ +// Package convmv provides the convmv command. +package convmv + +import ( + "context" + "errors" + "strings" + + "github.com/rclone/rclone/cmd" + "github.com/rclone/rclone/fs/config/flags" + "github.com/rclone/rclone/fs/operations" + "github.com/rclone/rclone/fs/sync" + "github.com/rclone/rclone/lib/transform" + "github.com/spf13/cobra" +) + +// Globals +var ( + deleteEmptySrcDirs = false + createEmptySrcDirs = false +) + +func init() { + cmd.Root.AddCommand(commandDefinition) + cmdFlags := commandDefinition.Flags() + flags.BoolVarP(cmdFlags, &deleteEmptySrcDirs, "delete-empty-src-dirs", "", deleteEmptySrcDirs, "Delete empty source dirs after move", "") + flags.BoolVarP(cmdFlags, &createEmptySrcDirs, "create-empty-src-dirs", "", createEmptySrcDirs, "Create empty source dirs on destination after move", "") +} + +var commandDefinition = &cobra.Command{ + Use: "convmv dest:path --name-transform XXX", + Short: `Convert file and directory names in place.`, + // Warning¡ "¡" will be replaced by backticks below + Long: strings.ReplaceAll(` +convmv supports advanced path name transformations for converting and renaming files and directories by applying prefixes, suffixes, and other alterations. + +`+transform.SprintList()+` + +Multiple transformations can be used in sequence, applied in the order they are specified on the command line. + +The ¡--name-transform¡ flag is also available in ¡sync¡, ¡copy¡, and ¡move¡. + +## Files vs Directories ## + +By default ¡--name-transform¡ will only apply to file names. The means only the leaf file name will be transformed. +However some of the transforms would be better applied to the whole path or just directories. +To choose which which part of the file path is affected some tags can be added to the ¡--name-transform¡ + +| Tag | Effect | +|------|------| +| ¡file¡ | Only transform the leaf name of files (DEFAULT) | +| ¡dir¡ | Only transform name of directories - these may appear anywhere in the path | +| ¡all¡ | Transform the entire path for files and directories | + +This is used by adding the tag into the transform name like this: ¡--name-transform file,prefix=ABC¡ or ¡--name-transform dir,prefix=DEF¡. + +For some conversions using all is more likely to be useful, for example ¡--name-transform all,nfc¡ + +Note that ¡--name-transform¡ may not add path separators ¡/¡ to the name. This will cause an error. + +## Ordering and Conflicts ## + +* Transformations will be applied in the order specified by the user. + * If the ¡file¡ tag is in use (the default) then only the leaf name of files will be transformed. + * If the ¡dir¡ tag is in use then directories anywhere in the path will be transformed + * If the ¡all¡ tag is in use then directories and files anywhere in the path will be transformed + * Each transformation will be run one path segment at a time. + * If a transformation adds a ¡/¡ or ends up with an empty path segment then that will be an error. +* It is up to the user to put the transformations in a sensible order. + * Conflicting transformations, such as ¡prefix¡ followed by ¡trimprefix¡ or ¡nfc¡ followed by ¡nfd¡, are possible. + * Instead of enforcing mutual exclusivity, transformations are applied in sequence as specified by the +user, allowing for intentional use cases (e.g., trimming one prefix before adding another). + * Users should be aware that certain combinations may lead to unexpected results and should verify +transformations using ¡--dry-run¡ before execution. + +## Race Conditions and Non-Deterministic Behavior ## + +Some transformations, such as ¡replace=old:new¡, may introduce conflicts where multiple source files map to the same destination name. +This can lead to race conditions when performing concurrent transfers. It is up to the user to anticipate these. +* If two files from the source are transformed into the same name at the destination, the final state may be non-deterministic. +* Running rclone check after a sync using such transformations may erroneously report missing or differing files due to overwritten results. + +* To minimize risks, users should: + * Carefully review transformations that may introduce conflicts. + * Use ¡--dry-run¡ to inspect changes before executing a sync (but keep in mind that it won't show the effect of non-deterministic transformations). + * Avoid transformations that cause multiple distinct source files to map to the same destination name. + * Consider disabling concurrency with ¡--transfers=1¡ if necessary. + * Certain transformations (e.g. ¡prefix¡) will have a multiplying effect every time they are used. Avoid these when using ¡bisync¡. + + `, "¡", "`"), + Annotations: map[string]string{ + "versionIntroduced": "v1.70", + "groups": "Filter,Listing,Important,Copy", + }, + Run: func(command *cobra.Command, args []string) { + cmd.CheckArgs(1, 1, command, args) + fdst, srcFileName := cmd.NewFsFile(args[0]) + cmd.Run(false, true, command, func() error { + if !transform.Transforming() { + return errors.New("--name-transform must be set") + } + if srcFileName == "" { + return sync.Transform(context.Background(), fdst, deleteEmptySrcDirs, createEmptySrcDirs) + } + return operations.TransformFile(context.Background(), fdst, srcFileName) + }) + }, +} diff --git a/cmd/convmv/convmv_test.go b/cmd/convmv/convmv_test.go new file mode 100644 index 000000000..6147710b9 --- /dev/null +++ b/cmd/convmv/convmv_test.go @@ -0,0 +1,252 @@ +// Package convmv provides the convmv command. +package convmv + +import ( + "cmp" + "context" + "fmt" + "path" + "slices" + "strings" + "testing" + + _ "github.com/rclone/rclone/backend/all" // import all backends + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/filter" + "github.com/rclone/rclone/fs/operations" + "github.com/rclone/rclone/fs/sync" + "github.com/rclone/rclone/fs/walk" + "github.com/rclone/rclone/fstest" + "github.com/rclone/rclone/lib/transform" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/text/unicode/norm" +) + +// Some times used in the tests +var ( + t1 = fstest.Time("2001-02-03T04:05:06.499999999Z") + debug = `` +) + +// TestMain drives the tests +func TestMain(m *testing.M) { + fstest.TestMain(m) +} + +func TestTransform(t *testing.T) { + type args struct { + TransformOpt []string + TransformBackOpt []string + Lossless bool // whether the TransformBackAlgo is always losslessly invertible + } + tests := []struct { + name string + args args + }{ + {name: "NFC", args: args{ + TransformOpt: []string{"all,nfc"}, + TransformBackOpt: []string{"all,nfd"}, + Lossless: false, + }}, + {name: "NFD", args: args{ + TransformOpt: []string{"all,nfd"}, + TransformBackOpt: []string{"all,nfc"}, + Lossless: false, + }}, + {name: "base64", args: args{ + TransformOpt: []string{"all,base64encode"}, + TransformBackOpt: []string{"all,base64encode"}, + Lossless: false, + }}, + {name: "prefix", args: args{ + TransformOpt: []string{"all,prefix=PREFIX"}, + TransformBackOpt: []string{"all,trimprefix=PREFIX"}, + Lossless: true, + }}, + {name: "suffix", args: args{ + TransformOpt: []string{"all,suffix=SUFFIX"}, + TransformBackOpt: []string{"all,trimsuffix=SUFFIX"}, + Lossless: true, + }}, + {name: "truncate", args: args{ + TransformOpt: []string{"all,truncate=10"}, + TransformBackOpt: []string{"all,truncate=10"}, + Lossless: false, + }}, + {name: "encoder", args: args{ + TransformOpt: []string{"all,encoder=Colon,SquareBracket"}, + TransformBackOpt: []string{"all,decoder=Colon,SquareBracket"}, + Lossless: true, + }}, + {name: "ISO-8859-1", args: args{ + TransformOpt: []string{"all,ISO-8859-1"}, + TransformBackOpt: []string{"all,ISO-8859-1"}, + Lossless: false, + }}, + {name: "charmap", args: args{ + TransformOpt: []string{"all,charmap=ISO-8859-7"}, + TransformBackOpt: []string{"all,charmap=ISO-8859-7"}, + Lossless: false, + }}, + {name: "lowercase", args: args{ + TransformOpt: []string{"all,lowercase"}, + TransformBackOpt: []string{"all,lowercase"}, + Lossless: false, + }}, + {name: "ascii", args: args{ + TransformOpt: []string{"all,ascii"}, + TransformBackOpt: []string{"all,ascii"}, + Lossless: false, + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := fstest.NewRun(t) + defer r.Finalise() + + r.Mkdir(context.Background(), r.Flocal) + r.Mkdir(context.Background(), r.Fremote) + items := makeTestFiles(t, r, "dir1") + err := r.Fremote.Mkdir(context.Background(), "empty/empty") + require.NoError(t, err) + err = r.Flocal.Mkdir(context.Background(), "empty/empty") + require.NoError(t, err) + deleteDSStore(t, r) + r.CheckRemoteListing(t, items, []string{"dir1", "empty", "empty/empty"}) + r.CheckLocalListing(t, items, []string{"dir1", "empty", "empty/empty"}) + + err = transform.SetOptions(context.Background(), tt.args.TransformOpt...) + require.NoError(t, err) + + err = sync.Transform(context.Background(), r.Fremote, true, true) + assert.NoError(t, err) + compareNames(t, r, items) + + transformedItems := transformItems(t, items) + r.CheckRemoteListing(t, transformedItems, []string{transform.Path("dir1", true), transform.Path("empty", true), transform.Path("empty/empty", true)}) + err = transform.SetOptions(context.Background(), tt.args.TransformBackOpt...) + require.NoError(t, err) + err = sync.Transform(context.Background(), r.Fremote, true, true) + assert.NoError(t, err) + compareNames(t, r, transformedItems) + + if tt.args.Lossless { + deleteDSStore(t, r) + r.CheckRemoteListing(t, items, []string{"dir1", "empty", "empty/empty"}) + } + }) + } +} + +// const alphabet = "ƀɀɠʀҠԀڀڠݠހ߀ကႠᄀᄠᅀᆀᇠሀሠበዠጠᎠᏀᐠᑀᑠᒀᒠᓀᓠᔀᔠᕀᕠᖀᖠᗀᗠᘀᘠᙀᚠᛀកᠠᡀᣀᦀ᧠ᨠᯀᰀᴀ⇠⋀⍀⍠⎀⎠⏀␀─┠╀╠▀■◀◠☀☠♀♠⚀⚠⛀⛠✀✠❀➀➠⠀⠠⡀⡠⢀⢠⣀⣠⤀⤠⥀⥠⦠⨠⩀⪀⪠⫠⬀⬠⭀ⰀⲀⲠⳀⴀⵀ⺠⻀㇀㐀㐠㑀㑠㒀㒠㓀㓠㔀㔠㕀㕠㖀㖠㗀㗠㘀㘠㙀㙠㚀㚠㛀㛠㜀㜠㝀㝠㞀㞠㟀㟠㠀㠠㡀㡠㢀㢠㣀㣠㤀㤠㥀㥠㦀㦠㧀㧠㨀㨠㩀㩠㪀㪠㫀㫠㬀㬠㭀㭠㮀㮠㯀㯠㰀㰠㱀㱠㲀㲠㳀㳠㴀㴠㵀㵠㶀㶠㷀㷠㸀㸠㹀㹠㺀㺠㻀㻠㼀㼠㽀㽠㾀㾠㿀㿠䀀䀠䁀䁠䂀䂠䃀䃠䄀䄠䅀䅠䆀䆠䇀䇠䈀䈠䉀䉠䊀䊠䋀䋠䌀䌠䍀䍠䎀䎠䏀䏠䐀䐠䑀䑠䒀䒠䓀䓠䔀䔠䕀䕠䖀䖠䗀䗠䘀䘠䙀䙠䚀䚠䛀䛠䜀䜠䝀䝠䞀䞠䟀䟠䠀䠠䡀䡠䢀䢠䣀䣠䤀䤠䥀䥠䦀䦠䧀䧠䨀䨠䩀䩠䪀䪠䫀䫠䬀䬠䭀䭠䮀䮠䯀䯠䰀䰠䱀䱠䲀䲠䳀䳠䴀䴠䵀䵠䶀䷀䷠一丠乀习亀亠什仠伀传佀你侀侠俀俠倀倠偀偠傀傠僀僠儀儠兀兠冀冠净几刀删剀剠劀加勀勠匀匠區占厀厠叀叠吀吠呀呠咀咠哀哠唀唠啀啠喀喠嗀嗠嘀嘠噀噠嚀嚠囀因圀圠址坠垀垠埀埠堀堠塀塠墀墠壀壠夀夠奀奠妀妠姀姠娀娠婀婠媀媠嫀嫠嬀嬠孀孠宀宠寀寠尀尠局屠岀岠峀峠崀崠嵀嵠嶀嶠巀巠帀帠幀幠庀庠廀廠开张彀彠往徠忀忠怀怠恀恠悀悠惀惠愀愠慀慠憀憠懀懠戀戠所扠技抠拀拠挀挠捀捠掀掠揀揠搀搠摀摠撀撠擀擠攀攠敀敠斀斠旀无昀映晀晠暀暠曀曠最朠杀杠枀枠柀柠栀栠桀桠梀梠检棠椀椠楀楠榀榠槀槠樀樠橀橠檀檠櫀櫠欀欠歀歠殀殠毀毠氀氠汀池沀沠泀泠洀洠浀浠涀涠淀淠渀渠湀湠満溠滀滠漀漠潀潠澀澠激濠瀀瀠灀灠炀炠烀烠焀焠煀煠熀熠燀燠爀爠牀牠犀犠狀狠猀猠獀獠玀玠珀珠琀琠瑀瑠璀璠瓀瓠甀甠畀畠疀疠痀痠瘀瘠癀癠皀皠盀盠眀眠着睠瞀瞠矀矠砀砠础硠碀碠磀磠礀礠祀祠禀禠秀秠稀稠穀穠窀窠竀章笀笠筀筠简箠節篠簀簠籀籠粀粠糀糠紀素絀絠綀綠緀締縀縠繀繠纀纠绀绠缀缠罀罠羀羠翀翠耀耠聀聠肀肠胀胠脀脠腀腠膀膠臀臠舀舠艀艠芀芠苀苠茀茠荀荠莀莠菀菠萀萠葀葠蒀蒠蓀蓠蔀蔠蕀蕠薀薠藀藠蘀蘠虀虠蚀蚠蛀蛠蜀蜠蝀蝠螀螠蟀蟠蠀蠠血衠袀袠裀裠褀褠襀襠覀覠觀觠言訠詀詠誀誠諀諠謀謠譀譠讀讠诀诠谀谠豀豠貀負賀賠贀贠赀赠趀趠跀跠踀踠蹀蹠躀躠軀軠輀輠轀轠辀辠迀迠退造遀遠邀邠郀郠鄀鄠酀酠醀醠釀釠鈀鈠鉀鉠銀銠鋀鋠錀錠鍀鍠鎀鎠鏀鏠鐀鐠鑀鑠钀钠铀铠销锠镀镠門閠闀闠阀阠陀陠隀隠雀雠需霠靀靠鞀鞠韀韠頀頠顀顠颀颠飀飠餀餠饀饠馀馠駀駠騀騠驀驠骀骠髀髠鬀鬠魀魠鮀鮠鯀鯠鰀鰠鱀鱠鲀鲠鳀鳠鴀鴠鵀鵠鶀鶠鷀鷠鸀鸠鹀鹠麀麠黀黠鼀鼠齀齠龀龠ꀀꀠꁀꁠꂀꂠꃀꃠꄀꄠꅀꅠꆀꆠꇀꇠꈀꈠꉀꉠꊀꊠꋀꋠꌀꌠꍀꍠꎀꎠꏀꏠꐀꐠꑀꑠ꒠ꔀꔠꕀꕠꖀꖠꗀꗠꙀꚠꛀ꜀꜠ꝀꞀꡀ測試_Русский___ě_áñ" +const alphabet = "abcdefg123456789" + +var extras = []string{"apple", "banana", "appleappleapplebanana", "splitbananasplit"} + +func makeTestFiles(t *testing.T, r *fstest.Run, dir string) []fstest.Item { + t.Helper() + n := 0 + // Create test files + items := []fstest.Item{} + for _, c := range alphabet { + var out strings.Builder + for i := rune(0); i < 7; i++ { + out.WriteRune(c + i) + } + fileName := path.Join(dir, fmt.Sprintf("%04d-%s.txt", n, out.String())) + fileName = strings.ToValidUTF8(fileName, "") + fileName = strings.NewReplacer(":", "", "<", "", ">", "", "?", "").Replace(fileName) // remove characters illegal on windows + + if debug != "" { + fileName = debug + } + + item := r.WriteObject(context.Background(), fileName, fileName, t1) + r.WriteFile(fileName, fileName, t1) + items = append(items, item) + n++ + + if debug != "" { + break + } + } + + for _, extra := range extras { + item := r.WriteObject(context.Background(), extra, extra, t1) + r.WriteFile(extra, extra, t1) + items = append(items, item) + } + + return items +} + +func deleteDSStore(t *testing.T, r *fstest.Run) { + ctxDSStore, fi := filter.AddConfig(context.Background()) + err := fi.AddRule(`+ *.DS_Store`) + assert.NoError(t, err) + err = fi.AddRule(`- **`) + assert.NoError(t, err) + err = operations.Delete(ctxDSStore, r.Fremote) + assert.NoError(t, err) +} + +func compareNames(t *testing.T, r *fstest.Run, items []fstest.Item) { + var entries fs.DirEntries + + deleteDSStore(t, r) + err := walk.ListR(context.Background(), r.Fremote, "", true, -1, walk.ListObjects, func(e fs.DirEntries) error { + entries = append(entries, e...) + return nil + }) + assert.NoError(t, err) + entries = slices.DeleteFunc(entries, func(E fs.DirEntry) bool { // remove those pesky .DS_Store files + if strings.Contains(E.Remote(), ".DS_Store") { + err := operations.DeleteFile(context.Background(), E.(fs.Object)) + assert.NoError(t, err) + return true + } + return false + }) + require.Equal(t, len(items), entries.Len()) + + // sort by CONVERTED name + slices.SortStableFunc(items, func(a, b fstest.Item) int { + aConv := transform.Path(a.Path, false) + bConv := transform.Path(b.Path, false) + return cmp.Compare(aConv, bConv) + }) + slices.SortStableFunc(entries, func(a, b fs.DirEntry) int { + return cmp.Compare(a.Remote(), b.Remote()) + }) + + for i, e := range entries { + expect := transform.Path(items[i].Path, false) + msg := fmt.Sprintf("expected %v, got %v", detectEncoding(expect), detectEncoding(e.Remote())) + assert.Equal(t, expect, e.Remote(), msg) + } +} + +func transformItems(t *testing.T, items []fstest.Item) []fstest.Item { + transformedItems := []fstest.Item{} + for _, item := range items { + newPath := transform.Path(item.Path, false) + newItem := item + newItem.Path = newPath + transformedItems = append(transformedItems, newItem) + } + return transformedItems +} + +func detectEncoding(s string) string { + if norm.NFC.IsNormalString(s) && norm.NFD.IsNormalString(s) { + return "BOTH" + } + if !norm.NFC.IsNormalString(s) && norm.NFD.IsNormalString(s) { + return "NFD" + } + if norm.NFC.IsNormalString(s) && !norm.NFD.IsNormalString(s) { + return "NFC" + } + return "OTHER" +} diff --git a/docs/content/docs.md b/docs/content/docs.md index 483c748a2..93880b93c 100644 --- a/docs/content/docs.md +++ b/docs/content/docs.md @@ -1848,6 +1848,13 @@ If the backend has a `--backend-upload-concurrency` setting (eg number of transfers instead if it is larger than the value of `--multi-thread-streams` or `--multi-thread-streams` isn't set. +### --name-transform COMMAND[=XXXX] ### +`--name-transform` introduces path name transformations for +`rclone copy`, `rclone sync`, and `rclone move`. These transformations +enable modifications to source and destination file names by applying +prefixes, suffixes, and other alterations during transfer operations. +For detailed docs and examples, see [`convmv`](/commands/rclone_convmv/). + ### --no-check-dest ### The `--no-check-dest` can be used with `move` or `copy` and it causes diff --git a/fs/operations/copy.go b/fs/operations/copy.go index 967fd69f5..bfc58ead9 100644 --- a/fs/operations/copy.go +++ b/fs/operations/copy.go @@ -415,5 +415,5 @@ func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Obj // CopyFile moves a single file possibly to a new name func CopyFile(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName string, srcFileName string) (err error) { - return moveOrCopyFile(ctx, fdst, fsrc, dstFileName, srcFileName, true) + return moveOrCopyFile(ctx, fdst, fsrc, dstFileName, srcFileName, true, false) } diff --git a/fs/operations/operations.go b/fs/operations/operations.go index ca1e8ee0a..7c4087588 100644 --- a/fs/operations/operations.go +++ b/fs/operations/operations.go @@ -456,6 +456,8 @@ func move(ctx context.Context, fdst fs.Fs, dst fs.Object, remote string, src fs. if err != nil { return newDst, err } + } else if src.Remote() == remote { + return newDst, nil } else if needsMoveCaseInsensitive(fdst, fdst, remote, src.Remote(), false) { doMove = func(ctx context.Context, src fs.Object, remote string) (fs.Object, error) { return MoveCaseInsensitive(ctx, fdst, fdst, remote, src.Remote(), false, src) @@ -1980,12 +1982,12 @@ func MoveCaseInsensitive(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileNam } // moveOrCopyFile moves or copies a single file possibly to a new name -func moveOrCopyFile(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName string, srcFileName string, cp bool) (err error) { +func moveOrCopyFile(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName string, srcFileName string, cp bool, allowOverlap bool) (err error) { ci := fs.GetConfig(ctx) logger, usingLogger := GetLogger(ctx) dstFilePath := path.Join(fdst.Root(), dstFileName) srcFilePath := path.Join(fsrc.Root(), srcFileName) - if fdst.Name() == fsrc.Name() && dstFilePath == srcFilePath { + if fdst.Name() == fsrc.Name() && dstFilePath == srcFilePath && !allowOverlap { fs.Debugf(fdst, "don't need to copy/move %s, it is already at target location", dstFileName) if usingLogger { srcObj, _ := fsrc.NewObject(ctx, srcFileName) @@ -2092,7 +2094,14 @@ func moveOrCopyFile(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName str // // This is treated as a transfer. func MoveFile(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName string, srcFileName string) (err error) { - return moveOrCopyFile(ctx, fdst, fsrc, dstFileName, srcFileName, false) + return moveOrCopyFile(ctx, fdst, fsrc, dstFileName, srcFileName, false, false) +} + +// TransformFile transforms a file in place using --name-transform +// +// This is treated as a transfer. +func TransformFile(ctx context.Context, fdst fs.Fs, srcFileName string) (err error) { + return moveOrCopyFile(ctx, fdst, fdst, srcFileName, srcFileName, false, true) } // SetTier changes tier of object in remote diff --git a/fs/operations/rc.go b/fs/operations/rc.go index 46bf41770..18119cc82 100644 --- a/fs/operations/rc.go +++ b/fs/operations/rc.go @@ -62,7 +62,7 @@ func rcList(ctx context.Context, in rc.Params) (out rc.Params, err error) { if rc.NotErrParamNotFound(err) { return nil, err } - var list = []*ListJSONItem{} + list := []*ListJSONItem{} err = ListJSON(ctx, f, remote, &opt, func(item *ListJSONItem) error { list = append(list, item) return nil @@ -193,7 +193,7 @@ func rcMoveOrCopyFile(ctx context.Context, in rc.Params, cp bool) (out rc.Params if err != nil { return nil, err } - return nil, moveOrCopyFile(ctx, dstFs, srcFs, dstRemote, srcRemote, cp) + return nil, moveOrCopyFile(ctx, dstFs, srcFs, dstRemote, srcRemote, cp, false) } func init() { @@ -289,7 +289,6 @@ func rcSingleCommand(ctx context.Context, in rc.Params, name string, noRemote bo var request *http.Request request, err := in.GetHTTPRequest() - if err != nil { return nil, err } @@ -629,12 +628,12 @@ func rcBackend(ctx context.Context, in rc.Params) (out rc.Params, err error) { if err != nil { return nil, err } - var opt = map[string]string{} + opt := map[string]string{} err = in.GetStructMissingOK("opt", &opt) if err != nil { return nil, err } - var arg = []string{} + arg := []string{} err = in.GetStructMissingOK("arg", &arg) if err != nil { return nil, err @@ -642,7 +641,6 @@ func rcBackend(ctx context.Context, in rc.Params) (out rc.Params, err error) { result, err := doCommand(ctx, command, arg, opt) if err != nil { return nil, fmt.Errorf("command %q failed: %w", command, err) - } out = make(rc.Params) out["result"] = result @@ -685,7 +683,6 @@ func rcDu(ctx context.Context, in rc.Params) (out rc.Params, err error) { dir, err := in.GetString("dir") if rc.IsErrParamNotFound(err) { dir = config.GetCacheDir() - } else if err != nil { return nil, err } diff --git a/fs/sync/sync.go b/fs/sync/sync.go index 008a25ed0..5bb2bef5c 100644 --- a/fs/sync/sync.go +++ b/fs/sync/sync.go @@ -96,6 +96,7 @@ type syncCopyMove struct { setDirModTimes []setDirModTime // directories that need their modtime set setDirModTimesMaxLevel int // max level of the directories to set modifiedDirs map[string]struct{} // dirs with changed contents (if s.setDirModTimeAfter) + allowOverlap bool // whether we allow src and dst to overlap (i.e. for convmv) } // For keeping track of delayed modtime sets @@ -127,8 +128,8 @@ func (strategy trackRenamesStrategy) leaf() bool { return (strategy & trackRenamesStrategyLeaf) != 0 } -func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.DeleteMode, DoMove bool, deleteEmptySrcDirs bool, copyEmptySrcDirs bool) (*syncCopyMove, error) { - if (deleteMode != fs.DeleteModeOff || DoMove) && operations.OverlappingFilterCheck(ctx, fdst, fsrc) { +func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.DeleteMode, DoMove bool, deleteEmptySrcDirs bool, copyEmptySrcDirs bool, allowOverlap bool) (*syncCopyMove, error) { + if (deleteMode != fs.DeleteModeOff || DoMove) && operations.OverlappingFilterCheck(ctx, fdst, fsrc) && !allowOverlap { return nil, fserrors.FatalError(fs.ErrorOverlapping) } ci := fs.GetConfig(ctx) @@ -162,6 +163,7 @@ func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.Delete setDirModTime: (!ci.NoUpdateDirModTime && fsrc.Features().CanHaveEmptyDirectories) && (fdst.Features().WriteDirSetModTime || fdst.Features().MkdirMetadata != nil || fdst.Features().DirSetModTime != nil), setDirModTimeAfter: !ci.NoUpdateDirModTime && (!copyEmptySrcDirs || fsrc.Features().CanHaveEmptyDirectories && fdst.Features().DirModTimeUpdatesOnWrite), modifiedDirs: make(map[string]struct{}), + allowOverlap: allowOverlap, } s.logger, s.usingLogger = operations.GetLogger(ctx) @@ -923,7 +925,7 @@ func (s *syncCopyMove) tryRename(src fs.Object) bool { // // dir is the start directory, "" for root func (s *syncCopyMove) run() error { - if operations.Same(s.fdst, s.fsrc) { + if operations.Same(s.fdst, s.fsrc) && !s.allowOverlap { fs.Errorf(s.fdst, "Nothing to do as source and destination are the same") return nil } @@ -1123,6 +1125,9 @@ func (s *syncCopyMove) copyDirMetadata(ctx context.Context, f fs.Fs, dst fs.Dire newDst, err = operations.SetDirModTime(ctx, f, dst, dir, src.ModTime(ctx)) } } + if transform.Transforming() && newDst != nil && src.Remote() != newDst.Remote() { + s.markParentNotEmpty(src) + } // If we need to set modtime after and we created a dir, then save it for later if s.setDirModTime && s.setDirModTimeAfter && err == nil { if newDst != nil { @@ -1291,7 +1296,9 @@ func (s *syncCopyMove) Match(ctx context.Context, dst, src fs.DirEntry) (recurse // Do the same thing to the entire contents of the directory srcX = fs.NewOverrideDirectory(srcX, transform.Path(src.Remote(), true)) src = srcX - s.markParentNotEmpty(src) + if !transform.Transforming() || src.Remote() != dst.Remote() { + s.markParentNotEmpty(src) + } dstX, ok := dst.(fs.Directory) if ok { s.logger(s.ctx, operations.Match, src, dst, fs.ErrorIsDir) @@ -1330,7 +1337,7 @@ func (s *syncCopyMove) Match(ctx context.Context, dst, src fs.DirEntry) (recurse // If DoMove is true then files will be moved instead of copied. // // dir is the start directory, "" for root -func runSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.DeleteMode, DoMove bool, deleteEmptySrcDirs bool, copyEmptySrcDirs bool) error { +func runSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.DeleteMode, DoMove bool, deleteEmptySrcDirs bool, copyEmptySrcDirs bool, allowOverlap bool) error { ci := fs.GetConfig(ctx) if deleteMode != fs.DeleteModeOff && DoMove { return fserrors.FatalError(errors.New("can't delete and move at the same time")) @@ -1341,7 +1348,7 @@ func runSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.Delete return fserrors.FatalError(errors.New("can't use --delete-before with --track-renames")) } // only delete stuff during in this pass - do, err := newSyncCopyMove(ctx, fdst, fsrc, fs.DeleteModeOnly, false, deleteEmptySrcDirs, copyEmptySrcDirs) + do, err := newSyncCopyMove(ctx, fdst, fsrc, fs.DeleteModeOnly, false, deleteEmptySrcDirs, copyEmptySrcDirs, allowOverlap) if err != nil { return err } @@ -1352,7 +1359,7 @@ func runSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.Delete // Next pass does a copy only deleteMode = fs.DeleteModeOff } - do, err := newSyncCopyMove(ctx, fdst, fsrc, deleteMode, DoMove, deleteEmptySrcDirs, copyEmptySrcDirs) + do, err := newSyncCopyMove(ctx, fdst, fsrc, deleteMode, DoMove, deleteEmptySrcDirs, copyEmptySrcDirs, allowOverlap) if err != nil { return err } @@ -1362,22 +1369,22 @@ func runSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.Delete // Sync fsrc into fdst func Sync(ctx context.Context, fdst, fsrc fs.Fs, copyEmptySrcDirs bool) error { ci := fs.GetConfig(ctx) - return runSyncCopyMove(ctx, fdst, fsrc, ci.DeleteMode, false, false, copyEmptySrcDirs) + return runSyncCopyMove(ctx, fdst, fsrc, ci.DeleteMode, false, false, copyEmptySrcDirs, false) } // CopyDir copies fsrc into fdst func CopyDir(ctx context.Context, fdst, fsrc fs.Fs, copyEmptySrcDirs bool) error { - return runSyncCopyMove(ctx, fdst, fsrc, fs.DeleteModeOff, false, false, copyEmptySrcDirs) + return runSyncCopyMove(ctx, fdst, fsrc, fs.DeleteModeOff, false, false, copyEmptySrcDirs, false) } // moveDir moves fsrc into fdst func moveDir(ctx context.Context, fdst, fsrc fs.Fs, deleteEmptySrcDirs bool, copyEmptySrcDirs bool) error { - return runSyncCopyMove(ctx, fdst, fsrc, fs.DeleteModeOff, true, deleteEmptySrcDirs, copyEmptySrcDirs) + return runSyncCopyMove(ctx, fdst, fsrc, fs.DeleteModeOff, true, deleteEmptySrcDirs, copyEmptySrcDirs, false) } // Transform renames fdst in place func Transform(ctx context.Context, fdst fs.Fs, deleteEmptySrcDirs bool, copyEmptySrcDirs bool) error { - return runSyncCopyMove(ctx, fdst, fdst, fs.DeleteModeOff, true, deleteEmptySrcDirs, copyEmptySrcDirs) + return runSyncCopyMove(ctx, fdst, fdst, fs.DeleteModeOff, true, deleteEmptySrcDirs, copyEmptySrcDirs, true) } // MoveDir moves fsrc into fdst diff --git a/fs/sync/sync_transform_test.go b/fs/sync/sync_transform_test.go index 892b040b1..a5c87a47b 100644 --- a/fs/sync/sync_transform_test.go +++ b/fs/sync/sync_transform_test.go @@ -409,6 +409,32 @@ func TestMove(t *testing.T) { r.CheckRemoteListing(t, []fstest.Item{fstest.NewItem("tictactoe/tictactoe/tictactoe.txt", "hello world", t1)}, []string{"tictacempty_dir", "tictactoe", "tictactoe/tictactoe"}) } +func TestTransformFile(t *testing.T) { + ctx := context.Background() + r := fstest.NewRun(t) + err := transform.SetOptions(ctx, "all,prefix=tac", "all,prefix=tic") + require.NoError(t, err) + r.WriteFile("toe/toe/toe.txt", "hello world", t1) + _, err = operations.MkdirModTime(ctx, r.Flocal, "empty_dir", t1) + require.NoError(t, err) + + r.Mkdir(ctx, r.Fremote) + ctx = predictDstFromLogger(ctx) + err = MoveDir(ctx, r.Fremote, r.Flocal, true, true) + testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t) + require.NoError(t, err) + + r.CheckLocalListing(t, []fstest.Item{}, []string{}) + r.CheckRemoteListing(t, []fstest.Item{fstest.NewItem("tictactoe/tictactoe/tictactoe.txt", "hello world", t1)}, []string{"tictacempty_dir", "tictactoe", "tictactoe/tictactoe"}) + + err = transform.SetOptions(ctx, "all,trimprefix=tic", "all,trimprefix=tac") + require.NoError(t, err) + err = operations.TransformFile(ctx, r.Fremote, "tictactoe/tictactoe/tictactoe.txt") + require.NoError(t, err) + r.CheckLocalListing(t, []fstest.Item{}, []string{}) + r.CheckRemoteListing(t, []fstest.Item{fstest.NewItem("toe/toe/toe.txt", "hello world", t1)}, []string{"tictacempty_dir", "tictactoe", "tictactoe/tictactoe", "toe", "toe/toe"}) +} + func TestBase64(t *testing.T) { ctx := context.Background() r := fstest.NewRun(t) diff --git a/lib/transform/help.go b/lib/transform/help.go new file mode 100644 index 000000000..a60e98f4e --- /dev/null +++ b/lib/transform/help.go @@ -0,0 +1,132 @@ +package transform + +import ( + "context" + "fmt" + "strings" + + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/lib/encoder" +) + +type commands struct { + command string + description string +} + +type example struct { + path string + flags []string +} + +var commandList = []commands{ + {command: "--name-transform prefix=XXXX", description: "Prepends XXXX to the file name."}, + {command: "--name-transform suffix=XXXX", description: "Appends XXXX to the file name after the extension."}, + {command: "--name-transform suffix_keep_extension=XXXX", description: "Appends XXXX to the file name while preserving the original file extension."}, + {command: "--name-transform trimprefix=XXXX", description: "Removes XXXX if it appears at the start of the file name."}, + {command: "--name-transform trimsuffix=XXXX", description: "Removes XXXX if it appears at the end of the file name."}, + {command: "--name-transform regex=/pattern/replacement/", description: "Applies a regex-based transformation."}, + {command: "--name-transform replace=old:new", description: "Replaces occurrences of old with new in the file name."}, + {command: "--name-transform date=YYYYMMDD", description: "Appends or prefixes the specified date format."}, + {command: "--name-transform truncate=N", description: "Truncates the file name to a maximum of N characters."}, + {command: "--name-transform base64encode", description: "Encodes the file name in Base64."}, + {command: "--name-transform base64decode", description: "Decodes a Base64-encoded file name."}, + {command: "--name-transform encoder=ENCODING", description: "Converts the file name to the specified encoding (e.g., ISO-8859-1, Windows-1252, Macintosh)."}, + {command: "--name-transform decoder=ENCODING", description: "Decodes the file name from the specified encoding."}, + {command: "--name-transform charmap=MAP", description: "Applies a character mapping transformation."}, + {command: "--name-transform lowercase", description: "Converts the file name to lowercase."}, + {command: "--name-transform uppercase", description: "Converts the file name to UPPERCASE."}, + {command: "--name-transform titlecase", description: "Converts the file name to Title Case."}, + {command: "--name-transform ascii", description: "Strips non-ASCII characters."}, + {command: "--name-transform url", description: "URL-encodes the file name."}, + {command: "--name-transform nfc", description: "Converts the file name to NFC Unicode normalization form."}, + {command: "--name-transform nfd", description: "Converts the file name to NFD Unicode normalization form."}, + {command: "--name-transform nfkc", description: "Converts the file name to NFKC Unicode normalization form."}, + {command: "--name-transform nfkd", description: "Converts the file name to NFKD Unicode normalization form."}, + {command: "--name-transform command=/path/to/my/programfile names.", description: "Executes an external program to transform"}, +} + +var examples = []example{ + {"stories/The Quick Brown Fox!.txt", []string{"all,uppercase"}}, + {"stories/The Quick Brown Fox!.txt", []string{"all,replace=Fox:Turtle", "all,replace=Quick:Slow"}}, + {"stories/The Quick Brown Fox!.txt", []string{"all,base64encode"}}, + {"c3Rvcmllcw==/VGhlIFF1aWNrIEJyb3duIEZveCEudHh0", []string{"all,base64decode"}}, + {"stories/The Quick Brown 🦊 Fox Went to the Café!.txt", []string{"all,nfc"}}, + {"stories/The Quick Brown 🦊 Fox Went to the Café!.txt", []string{"all,nfd"}}, + {"stories/The Quick Brown 🦊 Fox!.txt", []string{"all,ascii"}}, + {"stories/The Quick Brown Fox!.txt", []string{"all,trimsuffix=.txt"}}, + {"stories/The Quick Brown Fox!.txt", []string{"all,prefix=OLD_"}}, + {"stories/The Quick Brown 🦊 Fox Went to the Café!.txt", []string{"all,charmap=ISO-8859-7"}}, + {"stories/The Quick Brown Fox: A Memoir [draft].txt", []string{"all,encoder=Colon,SquareBracket"}}, + {"stories/The Quick Brown 🦊 Fox Went to the Café!.txt", []string{"all,truncate=21"}}, +} + +func (e example) command() string { + s := fmt.Sprintf(`rclone convmv %q`, e.path) + for _, f := range e.flags { + s += fmt.Sprintf(" --name-transform %q", f) + } + return s +} + +func (e example) output() string { + err := SetOptions(context.Background(), e.flags...) + if err != nil { + fs.Errorf(nil, "error generating help text: %v", err) + } + return Path(e.path, false) +} + +// go run ./ convmv --help +func sprintExamples() string { + s := "Examples: \n\n" + for _, e := range examples { + s += fmt.Sprintf("```\n%s\n", e.command()) + s += fmt.Sprintf("// Output: %s\n```\n\n", e.output()) + } + Opt = Options{} // reset + return s +} + +func commandTable() string { + s := `| Command | Description | +|------|------|` + for _, c := range commandList { + s += fmt.Sprintf("\n| `%s` | %s |", c.command, c.description) + } + s += "\n\n\n" + return s +} + +// SprintList returns the example help text as a string +func SprintList() string { + var algos transformAlgo + var charmaps fs.Enum[cmapChoices] + s := commandTable() + s += fmt.Sprintln("Conversion modes: \n```") + for _, v := range algos.Choices() { + s += fmt.Sprintln(v + " ") + } + s += fmt.Sprintln("```") + + s += fmt.Sprintln("Char maps: \n```") + for _, v := range charmaps.Choices() { + s += fmt.Sprintln(v + " ") + } + s += fmt.Sprintln("```") + + s += fmt.Sprintln("Encoding masks: \n```") + for _, v := range strings.Split(encoder.ValidStrings(), ",") { + s += fmt.Sprintln(v + " ") + } + s += fmt.Sprintln("```") + + s += sprintExamples() + + return s +} + +// PrintList prints the example help text to stdout +func PrintList() { + fmt.Println(SprintList()) +} diff --git a/lib/transform/options.go b/lib/transform/options.go index 30243bcc7..0d7d5c906 100644 --- a/lib/transform/options.go +++ b/lib/transform/options.go @@ -47,7 +47,7 @@ const ( var OptionsInfo = fs.Options{{ Name: "name_transform", Default: []string{}, - Help: "TODO", + Help: "`--name-transform` introduces path name transformations for `rclone copy`, `rclone sync`, and `rclone move`. These transformations enable modifications to source and destination file names by applying prefixes, suffixes, and other alterations during transfer operations. For detailed docs and examples, see [`convmv`](/commands/rclone_convmv/).", Groups: "Filter", }}