1
0
mirror of https://github.com/rclone/rclone.git synced 2025-12-10 13:23:21 +00:00

lib/transform

lib/transform adds the transform library, supporting advanced path name
transformations for converting and renaming files and directories by applying
prefixes, suffixes, and other alterations.

It also adds the --name-transform flag for use with sync, copy, and move.

Multiple transformations can be used in sequence, applied in the order they are
specified on the command line.

By default --name-transform will only apply to file names. The means only the leaf
file name will be transformed. However some of the transforms would be better
applied to the whole path or just directories. To choose which which part of the
file path is affected some tags can be added to the --name-transform:

file	Only transform the leaf name of files (DEFAULT)
dir	Only transform name of directories - these may appear anywhere in the path
all	Transform the entire path for files and directories

Example syntax:
--name-transform file,prefix=ABC
--name-transform dir,prefix=DEF
This commit is contained in:
nielash
2025-05-04 03:10:48 -04:00
parent f20ee1488b
commit 34a20555ca
14 changed files with 1193 additions and 75 deletions

71
lib/transform/cmap.go Normal file
View File

@@ -0,0 +1,71 @@
package transform
import (
"fmt"
"strings"
"sync"
"github.com/rclone/rclone/fs"
"golang.org/x/text/encoding/charmap"
)
var (
cmaps = map[int]*charmap.Charmap{}
lock sync.Mutex
)
type cmapChoices struct{}
func (cmapChoices) Choices() []string {
choices := make([]string, 1)
i := 0
for _, enc := range charmap.All {
c, ok := enc.(*charmap.Charmap)
if !ok {
continue
}
name := strings.ReplaceAll(c.String(), " ", "-")
if name == "" {
name = fmt.Sprintf("unknown-%d", i)
}
lock.Lock()
cmaps[i] = c
lock.Unlock()
choices = append(choices, name)
i++
}
return choices
}
func (cmapChoices) Type() string {
return "string"
}
func charmapByID(cm fs.Enum[cmapChoices]) *charmap.Charmap {
lock.Lock()
c, ok := cmaps[int(cm)]
lock.Unlock()
if ok {
return c
}
return nil
}
func encodeWithReplacement(s string, cmap *charmap.Charmap) string {
return strings.Map(func(r rune) rune {
b, ok := cmap.EncodeRune(r)
if !ok {
return '_'
}
return cmap.DecodeByte(b)
}, s)
}
func toASCII(s string) string {
return strings.Map(func(r rune) rune {
if r <= 127 {
return r
}
return -1
}, s)
}

240
lib/transform/options.go Normal file
View File

@@ -0,0 +1,240 @@
package transform
import (
"context"
"errors"
"strings"
"github.com/rclone/rclone/fs"
)
func init() {
fs.RegisterGlobalOptions(fs.OptionsInfo{Name: "name_transform", Opt: &Opt.Flags, Options: OptionsInfo, Reload: Reload})
}
type transform struct {
key transformAlgo // for example, "prefix"
value string // for example, "some_prefix_"
tag tag // file, dir, or all
}
// Options stores the parsed and unparsed transform options.
// their order must never be changed or sorted.
type Options struct {
Flags Flags // unparsed flag value like "file,prefix=ABC"
transforms []transform // parsed from NameTransform
}
// Flags is a slice of unparsed values set from command line flags or env vars
type Flags struct {
NameTransform []string `config:"name_transform"`
}
// Opt is the default options modified by the environment variables and command line flags
var Opt Options
// tag controls which part of the file path is affected (file, dir, all)
type tag int
// tag modes
const (
file tag = iota // Only transform the leaf name of files (default)
dir // Only transform name of directories - these may appear anywhere in the path
all // Transform the entire path for files and directories
)
// OptionsInfo describes the Options in use
var OptionsInfo = fs.Options{{
Name: "name_transform",
Default: []string{},
Help: "TODO",
Groups: "Filter",
}}
// Reload the transform options from the flags
func Reload(ctx context.Context) (err error) {
return newOpt(Opt)
}
// SetOptions sets the options from flags passed in.
// Any existing flags will be overwritten.
// s should be in the same format as cmd line flags, i.e. "all,prefix=XXX"
func SetOptions(ctx context.Context, s ...string) (err error) {
Opt = Options{Flags: Flags{NameTransform: s}}
return Reload(ctx)
}
// overwite Opt.transforms with values from Opt.Flags
func newOpt(opt Options) (err error) {
Opt.transforms = []transform{}
for _, transform := range opt.Flags.NameTransform {
t, err := parse(transform)
if err != nil {
return err
}
Opt.transforms = append(Opt.transforms, t)
}
return nil
}
// parse a single instance of --name-transform
func parse(s string) (t transform, err error) {
if s == "" {
return t, nil
}
s = t.parseTag(s)
err = t.parseKeyVal(s)
return t, err
}
// parse the tag (file/dir/all), set the option accordingly, and return the trimmed string
//
// we don't worry about errors here because it will error anyway as an invalid key
func (t *transform) parseTag(s string) string {
if strings.HasPrefix(s, "file,") {
t.tag = file
return strings.TrimPrefix(s, "file,")
}
if strings.HasPrefix(s, "dir,") {
t.tag = dir
return strings.TrimPrefix(s, "dir,")
}
if strings.HasPrefix(s, "all,") {
t.tag = all
return strings.TrimPrefix(s, "all,")
}
return s
}
// parse key and value (if any) by splitting on '=' sign
// (file/dir/all tag has already been trimmed)
func (t *transform) parseKeyVal(s string) (err error) {
if !strings.ContainsRune(s, '=') {
err = t.key.Set(s)
if err != nil {
return err
}
if t.requiresValue() {
fs.Debugf(nil, "received %v", s)
return errors.New("value is required for " + t.key.String())
}
return nil
}
split := strings.Split(s, "=")
if len(split) != 2 {
return errors.New("too many values")
}
if split[0] == "" {
return errors.New("key cannot be blank")
}
err = t.key.Set(split[0])
if err != nil {
return err
}
t.value = split[1]
return nil
}
// returns true if this particular algorithm requires a value
func (t *transform) requiresValue() bool {
switch t.key {
case ConvFindReplace:
return true
case ConvPrefix:
return true
case ConvSuffix:
return true
case ConvSuffixKeepExtension:
return true
case ConvTrimPrefix:
return true
case ConvTrimSuffix:
return true
case ConvIndex:
return true
case ConvDate:
return true
case ConvTruncate:
return true
case ConvEncoder:
return true
case ConvDecoder:
return true
}
return false
}
// transformAlgo describes conversion setting
type transformAlgo = fs.Enum[transformChoices]
// Supported transform options
const (
ConvNone transformAlgo = iota
ConvToNFC
ConvToNFD
ConvToNFKC
ConvToNFKD
ConvFindReplace
ConvPrefix
ConvSuffix
ConvSuffixKeepExtension
ConvTrimPrefix
ConvTrimSuffix
ConvIndex
ConvDate
ConvTruncate
ConvBase64Encode
ConvBase64Decode
ConvEncoder
ConvDecoder
ConvISO8859_1
ConvWindows1252
ConvMacintosh
ConvCharmap
ConvLowercase
ConvUppercase
ConvTitlecase
ConvASCII
ConvURL
ConvMapper
)
type transformChoices struct{}
func (transformChoices) Choices() []string {
return []string{
ConvNone: "none",
ConvToNFC: "nfc",
ConvToNFD: "nfd",
ConvToNFKC: "nfkc",
ConvToNFKD: "nfkd",
ConvFindReplace: "replace",
ConvPrefix: "prefix",
ConvSuffix: "suffix",
ConvSuffixKeepExtension: "suffix_keep_extension",
ConvTrimPrefix: "trimprefix",
ConvTrimSuffix: "trimsuffix",
ConvIndex: "index",
ConvDate: "date",
ConvTruncate: "truncate",
ConvBase64Encode: "base64encode",
ConvBase64Decode: "base64decode",
ConvEncoder: "encoder",
ConvDecoder: "decoder",
ConvISO8859_1: "ISO-8859-1",
ConvWindows1252: "Windows-1252",
ConvMacintosh: "Macintosh",
ConvCharmap: "charmap",
ConvLowercase: "lowercase",
ConvUppercase: "uppercase",
ConvTitlecase: "titlecase",
ConvASCII: "ascii",
ConvURL: "url",
ConvMapper: "mapper",
}
}
func (transformChoices) Type() string {
return "string"
}

226
lib/transform/transform.go Normal file
View File

@@ -0,0 +1,226 @@
// Package transform holds functions for path name transformations
package transform
import (
"encoding/base64"
"errors"
"fmt"
"mime"
"os"
"path"
"strconv"
"strings"
"unicode/utf8"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/lib/encoder"
"golang.org/x/text/encoding/charmap"
"golang.org/x/text/unicode/norm"
)
// Path transforms a path s according to the --name-transform options in use
//
// If no transforms are in use, s is returned unchanged
func Path(s string, isDir bool) string {
if !Transforming() {
return s
}
var err error
old := s
for _, t := range Opt.transforms {
if isDir && t.tag == file {
continue
}
baseOnly := !isDir && t.tag == file
if t.tag == dir && !isDir {
s, err = transformDir(s, t)
} else {
s, err = transformPath(s, t, baseOnly)
}
if err != nil {
fs.Error(s, err.Error()) // TODO: return err instead of logging it?
}
}
if old != s {
fs.Debugf(old, "transformed to: %v", s)
}
return s
}
// Transforming returns true when transforms are in use
func Transforming() bool {
return len(Opt.transforms) > 0
}
// transformPath transforms a path string according to the chosen TransformAlgo.
// Each path segment is transformed separately, to preserve path separators.
// If baseOnly is true, only the base will be transformed (useful for renaming while walking a dir tree recursively.)
// for example, "some/nested/path" -> "some/nested/CONVERTEDPATH"
// otherwise, the entire is path is transformed.
func transformPath(s string, t transform, baseOnly bool) (string, error) {
if s == "" || s == "/" || s == "\\" || s == "." {
return "", nil
}
if baseOnly {
transformedBase, err := transformPathSegment(path.Base(s), t)
if err := validateSegment(transformedBase); err != nil {
return "", err
}
return path.Join(path.Dir(s), transformedBase), err
}
segments := strings.Split(s, string(os.PathSeparator))
transformedSegments := make([]string, len(segments))
for _, seg := range segments {
convSeg, err := transformPathSegment(seg, t)
if err != nil {
return "", err
}
if err := validateSegment(convSeg); err != nil {
return "", err
}
transformedSegments = append(transformedSegments, convSeg)
}
return path.Join(transformedSegments...), nil
}
// transform all but the last path segment
func transformDir(s string, t transform) (string, error) {
dirPath, err := transformPath(path.Dir(s), t, false)
if err != nil {
return "", err
}
return path.Join(dirPath, path.Base(s)), nil
}
// transformPathSegment transforms one path segment (or really any string) according to the chosen TransformAlgo.
// It assumes path separators have already been trimmed.
func transformPathSegment(s string, t transform) (string, error) {
switch t.key {
case ConvNone:
return s, nil
case ConvToNFC:
return norm.NFC.String(s), nil
case ConvToNFD:
return norm.NFD.String(s), nil
case ConvToNFKC:
return norm.NFKC.String(s), nil
case ConvToNFKD:
return norm.NFKD.String(s), nil
case ConvBase64Encode:
return base64.URLEncoding.EncodeToString([]byte(s)), nil // URLEncoding to avoid slashes
case ConvBase64Decode:
if s == ".DS_Store" {
return s, nil
}
b, err := base64.URLEncoding.DecodeString(s)
if err != nil {
fs.Errorf(s, "base64 error")
}
return string(b), err
case ConvFindReplace:
split := strings.Split(t.value, ":")
if len(split) != 2 {
return s, fmt.Errorf("wrong number of values: %v", t.value)
}
return strings.ReplaceAll(s, split[0], split[1]), nil
case ConvPrefix:
return t.value + s, nil
case ConvSuffix:
return s + t.value, nil
case ConvSuffixKeepExtension:
return SuffixKeepExtension(s, t.value), nil
case ConvTrimPrefix:
return strings.TrimPrefix(s, t.value), nil
case ConvTrimSuffix:
return strings.TrimSuffix(s, t.value), nil
case ConvTruncate:
max, err := strconv.Atoi(t.value)
if err != nil {
return s, err
}
if max <= 0 {
return s, nil
}
if utf8.RuneCountInString(s) <= max {
return s, nil
}
runes := []rune(s)
return string(runes[:max]), nil
case ConvEncoder:
var enc encoder.MultiEncoder
err := enc.Set(t.value)
if err != nil {
return s, err
}
return enc.Encode(s), nil
case ConvDecoder:
var enc encoder.MultiEncoder
err := enc.Set(t.value)
if err != nil {
return s, err
}
return enc.Decode(s), nil
case ConvISO8859_1:
return encodeWithReplacement(s, charmap.ISO8859_1), nil
case ConvWindows1252:
return encodeWithReplacement(s, charmap.Windows1252), nil
case ConvMacintosh:
return encodeWithReplacement(s, charmap.Macintosh), nil
case ConvCharmap:
var cmapType fs.Enum[cmapChoices]
err := cmapType.Set(t.value)
if err != nil {
return s, err
}
c := charmapByID(cmapType)
return encodeWithReplacement(s, c), nil
case ConvLowercase:
return strings.ToLower(s), nil
case ConvUppercase:
return strings.ToUpper(s), nil
case ConvTitlecase:
return strings.ToTitle(s), nil
case ConvASCII:
return toASCII(s), nil
default:
return "", errors.New("this option is not yet implemented")
}
}
// SuffixKeepExtension adds a suffix while keeping extension
//
// i.e. file.txt becomes file_somesuffix.txt not file.txt_somesuffix
func SuffixKeepExtension(remote string, suffix string) string {
var (
base = remote
exts = ""
first = true
ext = path.Ext(remote)
)
for ext != "" {
// Look second and subsequent extensions in mime types.
// If they aren't found then don't keep it as an extension.
if !first && mime.TypeByExtension(ext) == "" {
break
}
base = base[:len(base)-len(ext)]
exts = ext + exts
first = false
ext = path.Ext(base)
}
return base + suffix + exts
}
// forbid transformations that add/remove path separators
func validateSegment(s string) error {
if s == "" {
return errors.New("transform cannot render path segments empty")
}
if strings.ContainsRune(s, '/') {
return fmt.Errorf("transform cannot add path separators: %v", s)
}
return nil
}

View File

@@ -0,0 +1,130 @@
package transform
import (
"context"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// sync tests are in fs/sync/sync_transform_test.go to avoid import cycle issues
func TestPath(t *testing.T) {
for _, test := range []struct {
path string
want string
}{
{"", ""},
{"toe/toe/toe", "tictactoe/tictactoe/tictactoe"},
{"a/b/c", "tictaca/tictacb/tictacc"},
} {
err := SetOptions(context.Background(), "all,prefix=tac", "all,prefix=tic")
require.NoError(t, err)
got := Path(test.path, false)
assert.Equal(t, test.want, got)
}
}
func TestFileTagOnFile(t *testing.T) {
for _, test := range []struct {
path string
want string
}{
{"a/b/c.txt", "a/b/1c.txt"},
} {
err := SetOptions(context.Background(), "file,prefix=1")
require.NoError(t, err)
got := Path(test.path, false)
assert.Equal(t, test.want, got)
}
}
func TestDirTagOnFile(t *testing.T) {
for _, test := range []struct {
path string
want string
}{
{"a/b/c.txt", "1a/1b/c.txt"},
} {
err := SetOptions(context.Background(), "dir,prefix=1")
require.NoError(t, err)
got := Path(test.path, false)
assert.Equal(t, test.want, got)
}
}
func TestAllTag(t *testing.T) {
for _, test := range []struct {
path string
want string
}{
{"a/b/c.txt", "1a/1b/1c.txt"},
} {
err := SetOptions(context.Background(), "all,prefix=1")
require.NoError(t, err)
got := Path(test.path, false)
assert.Equal(t, test.want, got)
}
}
func TestFileTagOnDir(t *testing.T) {
for _, test := range []struct {
path string
want string
}{
{"a/b", "a/b"},
} {
err := SetOptions(context.Background(), "file,prefix=1")
require.NoError(t, err)
got := Path(test.path, true)
assert.Equal(t, test.want, got)
}
}
func TestDirTagOnDir(t *testing.T) {
for _, test := range []struct {
path string
want string
}{
{"a/b", "1a/1b"},
} {
err := SetOptions(context.Background(), "dir,prefix=1")
require.NoError(t, err)
got := Path(test.path, true)
assert.Equal(t, test.want, got)
}
}
func TestVarious(t *testing.T) {
for _, test := range []struct {
path string
want string
flags []string
}{
{"stories/The Quick Brown Fox!.txt", "STORIES/THE QUICK BROWN FOX!.TXT", []string{"all,uppercase"}},
{"stories/The Quick Brown Fox!.txt", "stories/The Slow Brown Turtle!.txt", []string{"all,replace=Fox:Turtle", "all,replace=Quick:Slow"}},
{"stories/The Quick Brown Fox!.txt", "c3Rvcmllcw==/VGhlIFF1aWNrIEJyb3duIEZveCEudHh0", []string{"all,base64encode"}},
{"c3Rvcmllcw==/VGhlIFF1aWNrIEJyb3duIEZveCEudHh0", "stories/The Quick Brown Fox!.txt", []string{"all,base64decode"}},
{"stories/The Quick Brown 🦊 Fox Went to the Café!.txt", "stories/The Quick Brown 🦊 Fox Went to the Café!.txt", []string{"all,nfc"}},
{"stories/The Quick Brown 🦊 Fox Went to the Café!.txt", "stories/The Quick Brown 🦊 Fox Went to the Café!.txt", []string{"all,nfd"}},
{"stories/The Quick Brown 🦊 Fox!.txt", "stories/The Quick Brown Fox!.txt", []string{"all,ascii"}},
{"stories/The Quick Brown Fox!.txt", "stories/The Quick Brown Fox!", []string{"all,trimsuffix=.txt"}},
{"stories/The Quick Brown Fox!.txt", "OLD_stories/OLD_The Quick Brown Fox!.txt", []string{"all,prefix=OLD_"}},
{"stories/The Quick Brown 🦊 Fox Went to the Café!.txt", "stories/The Quick Brown _ Fox Went to the Caf_!.txt", []string{"all,charmap=ISO-8859-7"}},
{"stories/The Quick Brown Fox: A Memoir [draft].txt", "stories/The Quick Brown Fox A Memoir draft.txt", []string{"all,encoder=Colon,SquareBracket"}},
{"stories/The Quick Brown 🦊 Fox Went to the Café!.txt", "stories/The Quick Brown 🦊 Fox", []string{"all,truncate=21"}},
} {
err := SetOptions(context.Background(), test.flags...)
require.NoError(t, err)
got := Path(test.path, false)
assert.Equal(t, test.want, got)
}
}

View File

@@ -0,0 +1,14 @@
// Package transformflags implements command line flags to set up a transform
package transformflags
import (
"github.com/rclone/rclone/fs/config/flags"
"github.com/rclone/rclone/lib/transform"
"github.com/spf13/pflag"
)
// AddFlags adds the transform flags to the command
func AddFlags(flagSet *pflag.FlagSet) {
flags.AddFlagsFromOptions(flagSet, "", transform.OptionsInfo)
}