mirror of
https://github.com/rclone/rclone.git
synced 2026-01-06 18:43:50 +00:00
lib/transform: add transform library and --name-transform flag
lib/transform adds the transform library, supporting advanced path name transformations for converting and renaming files and directories by applying prefixes, suffixes, and other alterations. It also adds the --name-transform flag for use with sync, copy, and move. Multiple transformations can be used in sequence, applied in the order they are specified on the command line. By default --name-transform will only apply to file names. The means only the leaf file name will be transformed. However some of the transforms would be better applied to the whole path or just directories. To choose which which part of the file path is affected some tags can be added to the --name-transform: file Only transform the leaf name of files (DEFAULT) dir Only transform name of directories - these may appear anywhere in the path all Transform the entire path for files and directories Example syntax: --name-transform file,prefix=ABC --name-transform dir,prefix=DEF
This commit is contained in:
nielash
Nick Craig-Wood
71
lib/transform/cmap.go
Normal file
71
lib/transform/cmap.go
Normal file
@@ -0,0 +1,71 @@
|
||||
package transform
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"golang.org/x/text/encoding/charmap"
|
||||
)
|
||||
|
||||
var (
|
||||
cmaps = map[int]*charmap.Charmap{}
|
||||
lock sync.Mutex
|
||||
)
|
||||
|
||||
type cmapChoices struct{}
|
||||
|
||||
func (cmapChoices) Choices() []string {
|
||||
choices := make([]string, 1)
|
||||
i := 0
|
||||
for _, enc := range charmap.All {
|
||||
c, ok := enc.(*charmap.Charmap)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
name := strings.ReplaceAll(c.String(), " ", "-")
|
||||
if name == "" {
|
||||
name = fmt.Sprintf("unknown-%d", i)
|
||||
}
|
||||
lock.Lock()
|
||||
cmaps[i] = c
|
||||
lock.Unlock()
|
||||
choices = append(choices, name)
|
||||
i++
|
||||
}
|
||||
return choices
|
||||
}
|
||||
|
||||
func (cmapChoices) Type() string {
|
||||
return "string"
|
||||
}
|
||||
|
||||
func charmapByID(cm fs.Enum[cmapChoices]) *charmap.Charmap {
|
||||
lock.Lock()
|
||||
c, ok := cmaps[int(cm)]
|
||||
lock.Unlock()
|
||||
if ok {
|
||||
return c
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func encodeWithReplacement(s string, cmap *charmap.Charmap) string {
|
||||
return strings.Map(func(r rune) rune {
|
||||
b, ok := cmap.EncodeRune(r)
|
||||
if !ok {
|
||||
return '_'
|
||||
}
|
||||
return cmap.DecodeByte(b)
|
||||
}, s)
|
||||
}
|
||||
|
||||
func toASCII(s string) string {
|
||||
return strings.Map(func(r rune) rune {
|
||||
if r <= 127 {
|
||||
return r
|
||||
}
|
||||
return -1
|
||||
}, s)
|
||||
}
|
||||
140
lib/transform/help.go
Normal file
140
lib/transform/help.go
Normal file
@@ -0,0 +1,140 @@
|
||||
package transform
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/lib/encoder"
|
||||
)
|
||||
|
||||
type commands struct {
|
||||
command string
|
||||
description string
|
||||
}
|
||||
|
||||
type example struct {
|
||||
path string
|
||||
flags []string
|
||||
}
|
||||
|
||||
var commandList = []commands{
|
||||
{command: "--name-transform prefix=XXXX", description: "Prepends XXXX to the file name."},
|
||||
{command: "--name-transform suffix=XXXX", description: "Appends XXXX to the file name after the extension."},
|
||||
{command: "--name-transform suffix_keep_extension=XXXX", description: "Appends XXXX to the file name while preserving the original file extension."},
|
||||
{command: "--name-transform trimprefix=XXXX", description: "Removes XXXX if it appears at the start of the file name."},
|
||||
{command: "--name-transform trimsuffix=XXXX", description: "Removes XXXX if it appears at the end of the file name."},
|
||||
{command: "--name-transform regex=/pattern/replacement/", description: "Applies a regex-based transformation."},
|
||||
{command: "--name-transform replace=old:new", description: "Replaces occurrences of old with new in the file name."},
|
||||
{command: "--name-transform date={YYYYMMDD}", description: "Appends or prefixes the specified date format."},
|
||||
{command: "--name-transform truncate=N", description: "Truncates the file name to a maximum of N characters."},
|
||||
{command: "--name-transform base64encode", description: "Encodes the file name in Base64."},
|
||||
{command: "--name-transform base64decode", description: "Decodes a Base64-encoded file name."},
|
||||
{command: "--name-transform encoder=ENCODING", description: "Converts the file name to the specified encoding (e.g., ISO-8859-1, Windows-1252, Macintosh)."},
|
||||
{command: "--name-transform decoder=ENCODING", description: "Decodes the file name from the specified encoding."},
|
||||
{command: "--name-transform charmap=MAP", description: "Applies a character mapping transformation."},
|
||||
{command: "--name-transform lowercase", description: "Converts the file name to lowercase."},
|
||||
{command: "--name-transform uppercase", description: "Converts the file name to UPPERCASE."},
|
||||
{command: "--name-transform titlecase", description: "Converts the file name to Title Case."},
|
||||
{command: "--name-transform ascii", description: "Strips non-ASCII characters."},
|
||||
{command: "--name-transform url", description: "URL-encodes the file name."},
|
||||
{command: "--name-transform nfc", description: "Converts the file name to NFC Unicode normalization form."},
|
||||
{command: "--name-transform nfd", description: "Converts the file name to NFD Unicode normalization form."},
|
||||
{command: "--name-transform nfkc", description: "Converts the file name to NFKC Unicode normalization form."},
|
||||
{command: "--name-transform nfkd", description: "Converts the file name to NFKD Unicode normalization form."},
|
||||
{command: "--name-transform command=/path/to/my/programfile names.", description: "Executes an external program to transform"},
|
||||
}
|
||||
|
||||
var examples = []example{
|
||||
{"stories/The Quick Brown Fox!.txt", []string{"all,uppercase"}},
|
||||
{"stories/The Quick Brown Fox!.txt", []string{"all,replace=Fox:Turtle", "all,replace=Quick:Slow"}},
|
||||
{"stories/The Quick Brown Fox!.txt", []string{"all,base64encode"}},
|
||||
{"c3Rvcmllcw==/VGhlIFF1aWNrIEJyb3duIEZveCEudHh0", []string{"all,base64decode"}},
|
||||
{"stories/The Quick Brown 🦊 Fox Went to the Café!.txt", []string{"all,nfc"}},
|
||||
{"stories/The Quick Brown 🦊 Fox Went to the Café!.txt", []string{"all,nfd"}},
|
||||
{"stories/The Quick Brown 🦊 Fox!.txt", []string{"all,ascii"}},
|
||||
{"stories/The Quick Brown Fox!.txt", []string{"all,trimsuffix=.txt"}},
|
||||
{"stories/The Quick Brown Fox!.txt", []string{"all,prefix=OLD_"}},
|
||||
{"stories/The Quick Brown 🦊 Fox Went to the Café!.txt", []string{"all,charmap=ISO-8859-7"}},
|
||||
{"stories/The Quick Brown Fox: A Memoir [draft].txt", []string{"all,encoder=Colon,SquareBracket"}},
|
||||
{"stories/The Quick Brown 🦊 Fox Went to the Café!.txt", []string{"all,truncate=21"}},
|
||||
{"stories/The Quick Brown Fox!.txt", []string{"all,command=echo"}},
|
||||
{"stories/The Quick Brown Fox!", []string{"date=-{YYYYMMDD}"}},
|
||||
{"stories/The Quick Brown Fox!", []string{"date=-{macfriendlytime}"}},
|
||||
{"stories/The Quick Brown Fox!.txt", []string{"all,regex=[\\.\\w]/ab"}},
|
||||
}
|
||||
|
||||
func (e example) command() string {
|
||||
s := fmt.Sprintf(`rclone convmv %q`, e.path)
|
||||
for _, f := range e.flags {
|
||||
s += fmt.Sprintf(" --name-transform %q", f)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func (e example) output() string {
|
||||
ctx := context.Background()
|
||||
err := SetOptions(ctx, e.flags...)
|
||||
if err != nil {
|
||||
fs.Errorf(nil, "error generating help text: %v", err)
|
||||
}
|
||||
return Path(ctx, e.path, false)
|
||||
}
|
||||
|
||||
// go run ./ convmv --help
|
||||
func sprintExamples() string {
|
||||
s := "Examples: \n\n"
|
||||
for _, e := range examples {
|
||||
s += fmt.Sprintf("```\n%s\n", e.command())
|
||||
s += fmt.Sprintf("// Output: %s\n```\n\n", e.output())
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func commandTable() string {
|
||||
s := `| Command | Description |
|
||||
|------|------|`
|
||||
for _, c := range commandList {
|
||||
s += fmt.Sprintf("\n| `%s` | %s |", c.command, c.description)
|
||||
}
|
||||
s += "\n\n\n"
|
||||
return s
|
||||
}
|
||||
|
||||
var generatingHelpText bool
|
||||
|
||||
// SprintList returns the example help text as a string
|
||||
func SprintList() string {
|
||||
var algos transformAlgo
|
||||
var charmaps fs.Enum[cmapChoices]
|
||||
generatingHelpText = true
|
||||
s := commandTable()
|
||||
s += fmt.Sprintln("Conversion modes: \n```")
|
||||
for _, v := range algos.Choices() {
|
||||
s += fmt.Sprintln(v + " ")
|
||||
}
|
||||
s += fmt.Sprintln("```")
|
||||
|
||||
s += fmt.Sprintln("Char maps: \n```")
|
||||
for _, v := range charmaps.Choices() {
|
||||
s += fmt.Sprintln(v + " ")
|
||||
}
|
||||
s += fmt.Sprintln("```")
|
||||
|
||||
s += fmt.Sprintln("Encoding masks: \n```")
|
||||
for _, v := range strings.Split(encoder.ValidStrings(), ",") {
|
||||
s += fmt.Sprintln(v + " ")
|
||||
}
|
||||
s += fmt.Sprintln("```")
|
||||
|
||||
s += sprintExamples()
|
||||
|
||||
generatingHelpText = false
|
||||
return s
|
||||
}
|
||||
|
||||
// PrintList prints the example help text to stdout
|
||||
func PrintList() {
|
||||
fmt.Println(SprintList())
|
||||
}
|
||||
248
lib/transform/options.go
Normal file
248
lib/transform/options.go
Normal file
@@ -0,0 +1,248 @@
|
||||
package transform
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
)
|
||||
|
||||
type transform struct {
|
||||
key transformAlgo // for example, "prefix"
|
||||
value string // for example, "some_prefix_"
|
||||
tag tag // file, dir, or all
|
||||
}
|
||||
|
||||
// tag controls which part of the file path is affected (file, dir, all)
|
||||
type tag int
|
||||
|
||||
// tag modes
|
||||
const (
|
||||
file tag = iota // Only transform the leaf name of files (default)
|
||||
dir // Only transform name of directories - these may appear anywhere in the path
|
||||
all // Transform the entire path for files and directories
|
||||
)
|
||||
|
||||
// Transforming returns true when transforms are in use
|
||||
func Transforming(ctx context.Context) bool {
|
||||
ci := fs.GetConfig(ctx)
|
||||
return len(ci.NameTransform) > 0
|
||||
}
|
||||
|
||||
// SetOptions sets the options in ctx from flags passed in.
|
||||
// Any existing flags will be overwritten.
|
||||
// s should be in the same format as cmd line flags, i.e. "all,prefix=XXX"
|
||||
func SetOptions(ctx context.Context, s ...string) (err error) {
|
||||
ci := fs.GetConfig(ctx)
|
||||
ci.NameTransform = s
|
||||
_, err = getOptions(ctx)
|
||||
return err
|
||||
}
|
||||
|
||||
// cache to minimize re-parsing
|
||||
var (
|
||||
cachedNameTransform []string
|
||||
cachedOpt []transform
|
||||
cacheLock sync.Mutex
|
||||
)
|
||||
|
||||
// getOptions sets the options from flags passed in.
|
||||
func getOptions(ctx context.Context) (opt []transform, err error) {
|
||||
if !Transforming(ctx) {
|
||||
return opt, nil
|
||||
}
|
||||
|
||||
ci := fs.GetConfig(ctx)
|
||||
|
||||
// return cached opt if available
|
||||
if cachedNameTransform != nil && slices.Equal(ci.NameTransform, cachedNameTransform) {
|
||||
return cachedOpt, nil
|
||||
}
|
||||
|
||||
for _, transform := range ci.NameTransform {
|
||||
t, err := parse(transform)
|
||||
if err != nil {
|
||||
return opt, err
|
||||
}
|
||||
opt = append(opt, t)
|
||||
}
|
||||
updateCache(ci.NameTransform, opt)
|
||||
return opt, nil
|
||||
}
|
||||
|
||||
func updateCache(nt []string, o []transform) {
|
||||
cacheLock.Lock()
|
||||
cachedNameTransform = slices.Clone(nt)
|
||||
cachedOpt = o
|
||||
cacheLock.Unlock()
|
||||
}
|
||||
|
||||
// parse a single instance of --name-transform
|
||||
func parse(s string) (t transform, err error) {
|
||||
if s == "" {
|
||||
return t, nil
|
||||
}
|
||||
s = t.parseTag(s)
|
||||
err = t.parseKeyVal(s)
|
||||
return t, err
|
||||
}
|
||||
|
||||
// parse the tag (file/dir/all), set the option accordingly, and return the trimmed string
|
||||
//
|
||||
// we don't worry about errors here because it will error anyway as an invalid key
|
||||
func (t *transform) parseTag(s string) string {
|
||||
if strings.HasPrefix(s, "file,") {
|
||||
t.tag = file
|
||||
return strings.TrimPrefix(s, "file,")
|
||||
}
|
||||
if strings.HasPrefix(s, "dir,") {
|
||||
t.tag = dir
|
||||
return strings.TrimPrefix(s, "dir,")
|
||||
}
|
||||
if strings.HasPrefix(s, "all,") {
|
||||
t.tag = all
|
||||
return strings.TrimPrefix(s, "all,")
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// parse key and value (if any) by splitting on '=' sign
|
||||
// (file/dir/all tag has already been trimmed)
|
||||
func (t *transform) parseKeyVal(s string) (err error) {
|
||||
if !strings.ContainsRune(s, '=') {
|
||||
err = t.key.Set(s)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if t.requiresValue() {
|
||||
fs.Debugf(nil, "received %v", s)
|
||||
return errors.New("value is required for " + t.key.String())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
split := strings.Split(s, "=")
|
||||
if len(split) != 2 {
|
||||
return errors.New("too many values")
|
||||
}
|
||||
if split[0] == "" {
|
||||
return errors.New("key cannot be blank")
|
||||
}
|
||||
err = t.key.Set(split[0])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
t.value = split[1]
|
||||
return nil
|
||||
}
|
||||
|
||||
// returns true if this particular algorithm requires a value
|
||||
func (t *transform) requiresValue() bool {
|
||||
switch t.key {
|
||||
case ConvFindReplace:
|
||||
return true
|
||||
case ConvPrefix:
|
||||
return true
|
||||
case ConvSuffix:
|
||||
return true
|
||||
case ConvSuffixKeepExtension:
|
||||
return true
|
||||
case ConvTrimPrefix:
|
||||
return true
|
||||
case ConvTrimSuffix:
|
||||
return true
|
||||
case ConvIndex:
|
||||
return true
|
||||
case ConvDate:
|
||||
return true
|
||||
case ConvTruncate:
|
||||
return true
|
||||
case ConvEncoder:
|
||||
return true
|
||||
case ConvDecoder:
|
||||
return true
|
||||
case ConvRegex:
|
||||
return true
|
||||
case ConvCommand:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// transformAlgo describes conversion setting
|
||||
type transformAlgo = fs.Enum[transformChoices]
|
||||
|
||||
// Supported transform options
|
||||
const (
|
||||
ConvNone transformAlgo = iota
|
||||
ConvToNFC
|
||||
ConvToNFD
|
||||
ConvToNFKC
|
||||
ConvToNFKD
|
||||
ConvFindReplace
|
||||
ConvPrefix
|
||||
ConvSuffix
|
||||
ConvSuffixKeepExtension
|
||||
ConvTrimPrefix
|
||||
ConvTrimSuffix
|
||||
ConvIndex
|
||||
ConvDate
|
||||
ConvTruncate
|
||||
ConvBase64Encode
|
||||
ConvBase64Decode
|
||||
ConvEncoder
|
||||
ConvDecoder
|
||||
ConvISO8859_1
|
||||
ConvWindows1252
|
||||
ConvMacintosh
|
||||
ConvCharmap
|
||||
ConvLowercase
|
||||
ConvUppercase
|
||||
ConvTitlecase
|
||||
ConvASCII
|
||||
ConvURL
|
||||
ConvRegex
|
||||
ConvCommand
|
||||
)
|
||||
|
||||
type transformChoices struct{}
|
||||
|
||||
func (transformChoices) Choices() []string {
|
||||
return []string{
|
||||
ConvNone: "none",
|
||||
ConvToNFC: "nfc",
|
||||
ConvToNFD: "nfd",
|
||||
ConvToNFKC: "nfkc",
|
||||
ConvToNFKD: "nfkd",
|
||||
ConvFindReplace: "replace",
|
||||
ConvPrefix: "prefix",
|
||||
ConvSuffix: "suffix",
|
||||
ConvSuffixKeepExtension: "suffix_keep_extension",
|
||||
ConvTrimPrefix: "trimprefix",
|
||||
ConvTrimSuffix: "trimsuffix",
|
||||
ConvIndex: "index",
|
||||
ConvDate: "date",
|
||||
ConvTruncate: "truncate",
|
||||
ConvBase64Encode: "base64encode",
|
||||
ConvBase64Decode: "base64decode",
|
||||
ConvEncoder: "encoder",
|
||||
ConvDecoder: "decoder",
|
||||
ConvISO8859_1: "ISO-8859-1",
|
||||
ConvWindows1252: "Windows-1252",
|
||||
ConvMacintosh: "Macintosh",
|
||||
ConvCharmap: "charmap",
|
||||
ConvLowercase: "lowercase",
|
||||
ConvUppercase: "uppercase",
|
||||
ConvTitlecase: "titlecase",
|
||||
ConvASCII: "ascii",
|
||||
ConvURL: "url",
|
||||
ConvRegex: "regex",
|
||||
ConvCommand: "command",
|
||||
}
|
||||
}
|
||||
|
||||
func (transformChoices) Type() string {
|
||||
return "string"
|
||||
}
|
||||
336
lib/transform/transform.go
Normal file
336
lib/transform/transform.go
Normal file
@@ -0,0 +1,336 @@
|
||||
// Package transform holds functions for path name transformations
|
||||
package transform
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"mime"
|
||||
"net/url"
|
||||
"os/exec"
|
||||
"path"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/fserrors"
|
||||
"github.com/rclone/rclone/lib/encoder"
|
||||
"golang.org/x/text/encoding/charmap"
|
||||
"golang.org/x/text/unicode/norm"
|
||||
)
|
||||
|
||||
// Path transforms a path s according to the --name-transform options in use
|
||||
//
|
||||
// If no transforms are in use, s is returned unchanged
|
||||
func Path(ctx context.Context, s string, isDir bool) string {
|
||||
if !Transforming(ctx) {
|
||||
return s
|
||||
}
|
||||
|
||||
old := s
|
||||
opt, err := getOptions(ctx)
|
||||
if err != nil {
|
||||
err = fs.CountError(ctx, err)
|
||||
fs.Errorf(s, "Failed to parse transform flags: %v", err)
|
||||
}
|
||||
for _, t := range opt {
|
||||
if isDir && t.tag == file {
|
||||
continue
|
||||
}
|
||||
baseOnly := !isDir && t.tag == file
|
||||
if t.tag == dir && !isDir {
|
||||
s, err = transformDir(s, t)
|
||||
} else {
|
||||
s, err = transformPath(s, t, baseOnly)
|
||||
}
|
||||
if err != nil {
|
||||
err = fs.CountError(ctx, fserrors.NoRetryError(err))
|
||||
fs.Errorf(s, "Failed to transform: %v", err)
|
||||
}
|
||||
}
|
||||
if old != s && !generatingHelpText {
|
||||
fs.Debugf(old, "transformed to: %v", s)
|
||||
}
|
||||
if strings.Count(old, "/") != strings.Count(s, "/") {
|
||||
err = fs.CountError(ctx, fserrors.NoRetryError(fmt.Errorf("number of path segments must match: %v (%v), %v (%v)", old, strings.Count(old, "/"), s, strings.Count(s, "/"))))
|
||||
fs.Errorf(old, "%v", err)
|
||||
return old
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// transformPath transforms a path string according to the chosen TransformAlgo.
|
||||
// Each path segment is transformed separately, to preserve path separators.
|
||||
// If baseOnly is true, only the base will be transformed (useful for renaming while walking a dir tree recursively.)
|
||||
// for example, "some/nested/path" -> "some/nested/CONVERTEDPATH"
|
||||
// otherwise, the entire is path is transformed.
|
||||
func transformPath(s string, t transform, baseOnly bool) (string, error) {
|
||||
if s == "" || s == "/" || s == "\\" || s == "." {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
if baseOnly {
|
||||
transformedBase, err := transformPathSegment(path.Base(s), t)
|
||||
if err := validateSegment(transformedBase); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return path.Join(path.Dir(s), transformedBase), err
|
||||
}
|
||||
|
||||
segments := strings.Split(s, "/")
|
||||
transformedSegments := make([]string, len(segments))
|
||||
for _, seg := range segments {
|
||||
convSeg, err := transformPathSegment(seg, t)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := validateSegment(convSeg); err != nil {
|
||||
return "", err
|
||||
}
|
||||
transformedSegments = append(transformedSegments, convSeg)
|
||||
}
|
||||
return path.Join(transformedSegments...), nil
|
||||
}
|
||||
|
||||
// transform all but the last path segment
|
||||
func transformDir(s string, t transform) (string, error) {
|
||||
dirPath, err := transformPath(path.Dir(s), t, false)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return path.Join(dirPath, path.Base(s)), nil
|
||||
}
|
||||
|
||||
// transformPathSegment transforms one path segment (or really any string) according to the chosen TransformAlgo.
|
||||
// It assumes path separators have already been trimmed.
|
||||
func transformPathSegment(s string, t transform) (string, error) {
|
||||
switch t.key {
|
||||
case ConvNone:
|
||||
return s, nil
|
||||
case ConvToNFC:
|
||||
return norm.NFC.String(s), nil
|
||||
case ConvToNFD:
|
||||
return norm.NFD.String(s), nil
|
||||
case ConvToNFKC:
|
||||
return norm.NFKC.String(s), nil
|
||||
case ConvToNFKD:
|
||||
return norm.NFKD.String(s), nil
|
||||
case ConvBase64Encode:
|
||||
return base64.URLEncoding.EncodeToString([]byte(s)), nil // URLEncoding to avoid slashes
|
||||
case ConvBase64Decode:
|
||||
if s == ".DS_Store" {
|
||||
return s, nil
|
||||
}
|
||||
b, err := base64.URLEncoding.DecodeString(s)
|
||||
if err != nil {
|
||||
fs.Errorf(s, "base64 error")
|
||||
}
|
||||
return string(b), err
|
||||
case ConvFindReplace:
|
||||
split := strings.Split(t.value, ":")
|
||||
if len(split) != 2 {
|
||||
return s, fmt.Errorf("wrong number of values: %v", t.value)
|
||||
}
|
||||
return strings.ReplaceAll(s, split[0], split[1]), nil
|
||||
case ConvPrefix:
|
||||
return t.value + s, nil
|
||||
case ConvSuffix:
|
||||
return s + t.value, nil
|
||||
case ConvSuffixKeepExtension:
|
||||
return SuffixKeepExtension(s, t.value), nil
|
||||
case ConvTrimPrefix:
|
||||
return strings.TrimPrefix(s, t.value), nil
|
||||
case ConvTrimSuffix:
|
||||
return strings.TrimSuffix(s, t.value), nil
|
||||
case ConvTruncate:
|
||||
max, err := strconv.Atoi(t.value)
|
||||
if err != nil {
|
||||
return s, err
|
||||
}
|
||||
if max <= 0 {
|
||||
return s, nil
|
||||
}
|
||||
if utf8.RuneCountInString(s) <= max {
|
||||
return s, nil
|
||||
}
|
||||
runes := []rune(s)
|
||||
return string(runes[:max]), nil
|
||||
case ConvEncoder:
|
||||
var enc encoder.MultiEncoder
|
||||
err := enc.Set(t.value)
|
||||
if err != nil {
|
||||
return s, err
|
||||
}
|
||||
return enc.Encode(s), nil
|
||||
case ConvDecoder:
|
||||
var enc encoder.MultiEncoder
|
||||
err := enc.Set(t.value)
|
||||
if err != nil {
|
||||
return s, err
|
||||
}
|
||||
return enc.Decode(s), nil
|
||||
case ConvISO8859_1:
|
||||
return encodeWithReplacement(s, charmap.ISO8859_1), nil
|
||||
case ConvWindows1252:
|
||||
return encodeWithReplacement(s, charmap.Windows1252), nil
|
||||
case ConvMacintosh:
|
||||
return encodeWithReplacement(s, charmap.Macintosh), nil
|
||||
case ConvCharmap:
|
||||
var cmapType fs.Enum[cmapChoices]
|
||||
err := cmapType.Set(t.value)
|
||||
if err != nil {
|
||||
return s, err
|
||||
}
|
||||
c := charmapByID(cmapType)
|
||||
return encodeWithReplacement(s, c), nil
|
||||
case ConvLowercase:
|
||||
return strings.ToLower(s), nil
|
||||
case ConvUppercase:
|
||||
return strings.ToUpper(s), nil
|
||||
case ConvTitlecase:
|
||||
return strings.ToTitle(s), nil
|
||||
case ConvASCII:
|
||||
return toASCII(s), nil
|
||||
case ConvURL:
|
||||
return url.QueryEscape(s), nil
|
||||
case ConvDate:
|
||||
return s + AppyTimeGlobs(t.value, time.Now()), nil
|
||||
case ConvRegex:
|
||||
split := strings.Split(t.value, "/")
|
||||
if len(split) != 2 {
|
||||
return s, fmt.Errorf("regex syntax error: %v", t.value)
|
||||
}
|
||||
re := regexp.MustCompile(split[0])
|
||||
return re.ReplaceAllString(s, split[1]), nil
|
||||
case ConvCommand:
|
||||
return mapper(s, t.value)
|
||||
default:
|
||||
return "", errors.New("this option is not yet implemented")
|
||||
}
|
||||
}
|
||||
|
||||
// SuffixKeepExtension adds a suffix while keeping extension
|
||||
//
|
||||
// i.e. file.txt becomes file_somesuffix.txt not file.txt_somesuffix
|
||||
func SuffixKeepExtension(remote string, suffix string) string {
|
||||
var (
|
||||
base = remote
|
||||
exts = ""
|
||||
first = true
|
||||
ext = path.Ext(remote)
|
||||
)
|
||||
for ext != "" {
|
||||
// Look second and subsequent extensions in mime types.
|
||||
// If they aren't found then don't keep it as an extension.
|
||||
if !first && mime.TypeByExtension(ext) == "" {
|
||||
break
|
||||
}
|
||||
base = base[:len(base)-len(ext)]
|
||||
exts = ext + exts
|
||||
first = false
|
||||
ext = path.Ext(base)
|
||||
}
|
||||
return base + suffix + exts
|
||||
}
|
||||
|
||||
// forbid transformations that add/remove path separators
|
||||
func validateSegment(s string) error {
|
||||
if strings.TrimSpace(s) == "" {
|
||||
return errors.New("transform cannot render path segments empty")
|
||||
}
|
||||
if strings.ContainsRune(s, '/') {
|
||||
return fmt.Errorf("transform cannot add path separators: %v", s)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ParseGlobs determines whether a string contains {brackets}
|
||||
// and returns the substring (including both brackets) for replacing
|
||||
// substring is first opening bracket to last closing bracket --
|
||||
// good for {{this}} but not {this}{this}
|
||||
func ParseGlobs(s string) (hasGlobs bool, substring string) {
|
||||
open := strings.Index(s, "{")
|
||||
close := strings.LastIndex(s, "}")
|
||||
if open >= 0 && close > open {
|
||||
return true, s[open : close+1]
|
||||
}
|
||||
return false, ""
|
||||
}
|
||||
|
||||
// TrimBrackets converts {{this}} to this
|
||||
func TrimBrackets(s string) string {
|
||||
return strings.Trim(s, "{}")
|
||||
}
|
||||
|
||||
// TimeFormat converts a user-supplied string to a Go time constant, if possible
|
||||
func TimeFormat(timeFormat string) string {
|
||||
switch timeFormat {
|
||||
case "Layout":
|
||||
timeFormat = time.Layout
|
||||
case "ANSIC":
|
||||
timeFormat = time.ANSIC
|
||||
case "UnixDate":
|
||||
timeFormat = time.UnixDate
|
||||
case "RubyDate":
|
||||
timeFormat = time.RubyDate
|
||||
case "RFC822":
|
||||
timeFormat = time.RFC822
|
||||
case "RFC822Z":
|
||||
timeFormat = time.RFC822Z
|
||||
case "RFC850":
|
||||
timeFormat = time.RFC850
|
||||
case "RFC1123":
|
||||
timeFormat = time.RFC1123
|
||||
case "RFC1123Z":
|
||||
timeFormat = time.RFC1123Z
|
||||
case "RFC3339":
|
||||
timeFormat = time.RFC3339
|
||||
case "RFC3339Nano":
|
||||
timeFormat = time.RFC3339Nano
|
||||
case "Kitchen":
|
||||
timeFormat = time.Kitchen
|
||||
case "Stamp":
|
||||
timeFormat = time.Stamp
|
||||
case "StampMilli":
|
||||
timeFormat = time.StampMilli
|
||||
case "StampMicro":
|
||||
timeFormat = time.StampMicro
|
||||
case "StampNano":
|
||||
timeFormat = time.StampNano
|
||||
case "DateTime":
|
||||
timeFormat = time.DateTime
|
||||
case "DateOnly":
|
||||
timeFormat = time.DateOnly
|
||||
case "TimeOnly":
|
||||
timeFormat = time.TimeOnly
|
||||
case "MacFriendlyTime", "macfriendlytime", "mac":
|
||||
timeFormat = "2006-01-02 0304PM" // not actually a Go constant -- but useful as macOS filenames can't have colons
|
||||
case "YYYYMMDD":
|
||||
timeFormat = "20060102"
|
||||
}
|
||||
return timeFormat
|
||||
}
|
||||
|
||||
// AppyTimeGlobs converts "myfile-{DateOnly}.txt" to "myfile-2006-01-02.txt"
|
||||
func AppyTimeGlobs(s string, t time.Time) string {
|
||||
hasGlobs, substring := ParseGlobs(s)
|
||||
if !hasGlobs {
|
||||
return s
|
||||
}
|
||||
timeString := t.Local().Format(TimeFormat(TrimBrackets(substring)))
|
||||
return strings.ReplaceAll(s, substring, timeString)
|
||||
}
|
||||
|
||||
func mapper(s string, command string) (string, error) {
|
||||
out, err := exec.Command(command, s).CombinedOutput()
|
||||
if err != nil {
|
||||
out = bytes.TrimSpace(out)
|
||||
return s, fmt.Errorf("%s: error running command %q: %v", out, command+" "+s, err)
|
||||
}
|
||||
return string(bytes.TrimSpace(out)), nil
|
||||
}
|
||||
142
lib/transform/transform_test.go
Normal file
142
lib/transform/transform_test.go
Normal file
@@ -0,0 +1,142 @@
|
||||
package transform
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// sync tests are in fs/sync/sync_transform_test.go to avoid import cycle issues
|
||||
|
||||
func newOptions(s ...string) (context.Context, error) {
|
||||
ctx := context.Background()
|
||||
err := SetOptions(ctx, s...)
|
||||
return ctx, err
|
||||
}
|
||||
|
||||
func TestPath(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
path string
|
||||
want string
|
||||
}{
|
||||
{"", ""},
|
||||
{"toe/toe/toe", "tictactoe/tictactoe/tictactoe"},
|
||||
{"a/b/c", "tictaca/tictacb/tictacc"},
|
||||
} {
|
||||
ctx, err := newOptions("all,prefix=tac", "all,prefix=tic")
|
||||
require.NoError(t, err)
|
||||
|
||||
got := Path(ctx, test.path, false)
|
||||
assert.Equal(t, test.want, got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileTagOnFile(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
path string
|
||||
want string
|
||||
}{
|
||||
{"a/b/c.txt", "a/b/1c.txt"},
|
||||
} {
|
||||
ctx, err := newOptions("file,prefix=1")
|
||||
require.NoError(t, err)
|
||||
|
||||
got := Path(ctx, test.path, false)
|
||||
assert.Equal(t, test.want, got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDirTagOnFile(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
path string
|
||||
want string
|
||||
}{
|
||||
{"a/b/c.txt", "1a/1b/c.txt"},
|
||||
} {
|
||||
ctx, err := newOptions("dir,prefix=1")
|
||||
require.NoError(t, err)
|
||||
|
||||
got := Path(ctx, test.path, false)
|
||||
assert.Equal(t, test.want, got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAllTag(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
path string
|
||||
want string
|
||||
}{
|
||||
{"a/b/c.txt", "1a/1b/1c.txt"},
|
||||
} {
|
||||
ctx, err := newOptions("all,prefix=1")
|
||||
require.NoError(t, err)
|
||||
|
||||
got := Path(ctx, test.path, false)
|
||||
assert.Equal(t, test.want, got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileTagOnDir(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
path string
|
||||
want string
|
||||
}{
|
||||
{"a/b", "a/b"},
|
||||
} {
|
||||
ctx, err := newOptions("file,prefix=1")
|
||||
require.NoError(t, err)
|
||||
|
||||
got := Path(ctx, test.path, true)
|
||||
assert.Equal(t, test.want, got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDirTagOnDir(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
path string
|
||||
want string
|
||||
}{
|
||||
{"a/b", "1a/1b"},
|
||||
} {
|
||||
ctx, err := newOptions("dir,prefix=1")
|
||||
require.NoError(t, err)
|
||||
|
||||
got := Path(ctx, test.path, true)
|
||||
assert.Equal(t, test.want, got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVarious(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
path string
|
||||
want string
|
||||
flags []string
|
||||
}{
|
||||
{"stories/The Quick Brown Fox!.txt", "STORIES/THE QUICK BROWN FOX!.TXT", []string{"all,uppercase"}},
|
||||
{"stories/The Quick Brown Fox!.txt", "stories/The Slow Brown Turtle!.txt", []string{"all,replace=Fox:Turtle", "all,replace=Quick:Slow"}},
|
||||
{"stories/The Quick Brown Fox!.txt", "c3Rvcmllcw==/VGhlIFF1aWNrIEJyb3duIEZveCEudHh0", []string{"all,base64encode"}},
|
||||
{"c3Rvcmllcw==/VGhlIFF1aWNrIEJyb3duIEZveCEudHh0", "stories/The Quick Brown Fox!.txt", []string{"all,base64decode"}},
|
||||
{"stories/The Quick Brown 🦊 Fox Went to the Café!.txt", "stories/The Quick Brown 🦊 Fox Went to the Café!.txt", []string{"all,nfc"}},
|
||||
{"stories/The Quick Brown 🦊 Fox Went to the Café!.txt", "stories/The Quick Brown 🦊 Fox Went to the Café!.txt", []string{"all,nfd"}},
|
||||
{"stories/The Quick Brown 🦊 Fox!.txt", "stories/The Quick Brown Fox!.txt", []string{"all,ascii"}},
|
||||
{"stories/The Quick Brown 🦊 Fox!.txt", "stories/The+Quick+Brown+%F0%9F%A6%8A+Fox%21.txt", []string{"all,url"}},
|
||||
{"stories/The Quick Brown Fox!.txt", "stories/The Quick Brown Fox!", []string{"all,trimsuffix=.txt"}},
|
||||
{"stories/The Quick Brown Fox!.txt", "OLD_stories/OLD_The Quick Brown Fox!.txt", []string{"all,prefix=OLD_"}},
|
||||
{"stories/The Quick Brown 🦊 Fox Went to the Café!.txt", "stories/The Quick Brown _ Fox Went to the Caf_!.txt", []string{"all,charmap=ISO-8859-7"}},
|
||||
{"stories/The Quick Brown Fox: A Memoir [draft].txt", "stories/The Quick Brown Fox： A Memoir ［draft］.txt", []string{"all,encoder=Colon,SquareBracket"}},
|
||||
{"stories/The Quick Brown 🦊 Fox Went to the Café!.txt", "stories/The Quick Brown 🦊 Fox", []string{"all,truncate=21"}},
|
||||
{"stories/The Quick Brown Fox!.txt", "stories/The Quick Brown Fox!.txt", []string{"all,command=echo"}},
|
||||
{"stories/The Quick Brown Fox!.txt", "stories/The Quick Brown Fox!.txt-" + time.Now().Local().Format("20060102"), []string{"date=-{YYYYMMDD}"}},
|
||||
{"stories/The Quick Brown Fox!.txt", "stories/The Quick Brown Fox!.txt-" + time.Now().Local().Format("2006-01-02 0304PM"), []string{"date=-{macfriendlytime}"}},
|
||||
{"stories/The Quick Brown Fox!.txt", "ababababababab/ababab ababababab ababababab ababab!abababab", []string{"all,regex=[\\.\\w]/ab"}},
|
||||
} {
|
||||
ctx, err := newOptions(test.flags...)
|
||||
require.NoError(t, err)
|
||||
|
||||
got := Path(ctx, test.path, false)
|
||||
assert.Equal(t, test.want, got)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user