jgaunt/rclone
1
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2025-12-15 15:53:41 +00:00
Code Issues Projects Releases Wiki Activity

bisync: refactor normalization code, fix deltas - fixes #7270

Browse Source 
Refactored the case / unicode normalization logic to be much more efficient,
 and fix the last outstanding issue from #7270. Before this change, we were
 doing lots of for loops and re-normalizing strings we had already normalized
 earlier. Now, we leave the normalizing entirely to March and avoid
 re-transforming later, which seems to make a large difference in terms of
 performance.
This commit is contained in:
nielash
2023-11-09 05:04:33 -05:00
parent 58fd6d7b94
commit 98f539de8f
11 changed files with 242 additions and 153 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
cmd/bisync/queue.go
View File

@@ -22,6 +22,7 @@ type Results struct {
Src string
Dst string
Name string
AltName string
Size int64
Modtime time.Time
Hash string
@@ -58,6 +59,21 @@ func resultName(result Results, side, src, dst fs.DirEntry) string {
return ""
}
// returns the opposite side's name, only if different
func altName(name string, src, dst fs.DirEntry) string {
if src != nil && dst != nil {
if src.Remote() != dst.Remote() {
switch name {
case src.Remote():
return dst.Remote()
case dst.Remote():
return src.Remote()
}
}
}
return ""
}
// WriteResults is Bisync's LoggerFn
func WriteResults(ctx context.Context, sigil operations.Sigil, src, dst fs.DirEntry, err error) {
lock.Lock()
@@ -77,6 +93,7 @@ func WriteResults(ctx context.Context, sigil operations.Sigil, src, dst fs.DirEn
for i, side := range fss {
result.Name = resultName(result, side, src, dst)
result.AltName = altName(result.Name, src, dst)
result.IsSrc = i == 0
result.IsDst = i == 1
result.Flags = "-"
@@ -129,7 +146,7 @@ func ReadResults(results io.Reader) []Results {
return slice
}
func (b *bisyncRun) fastCopy(ctx context.Context, fsrc, fdst fs.Fs, files bilib.Names, queueName string, altNames bilib.Names) ([]Results, error) {
func (b *bisyncRun) fastCopy(ctx context.Context, fsrc, fdst fs.Fs, files bilib.Names, queueName string) ([]Results, error) {
if err := b.saveQueue(files, queueName); err != nil {
return nil, err
}
@@ -139,10 +156,9 @@ func (b *bisyncRun) fastCopy(ctx context.Context, fsrc, fdst fs.Fs, files bilib.
if err := filterCopy.AddFile(file); err != nil {
return nil, err
}
}
if altNames.NotEmpty() {
for _, file := range altNames.ToList() {
if err := filterCopy.AddFile(file); err != nil {
alias := b.aliases.Alias(file)
if alias != file {
if err := filterCopy.AddFile(alias); err != nil {
return nil, err
}
}
@@ -249,22 +265,3 @@ func (b *bisyncRun) saveQueue(files bilib.Names, jobName string) error {
queueFile := fmt.Sprintf("%s.%s.que", b.basePath, jobName)
return files.Save(queueFile)
}
func (b *bisyncRun) findAltNames(ctx context.Context, dst fs.Fs, queue bilib.Names, newListing string, altNames bilib.Names) {
ci := fs.GetConfig(ctx)
if queue.NotEmpty() && (!ci.NoUnicodeNormalization || ci.IgnoreCaseSync || b.fs1.Features().CaseInsensitive || b.fs2.Features().CaseInsensitive) {
// search list for existing file that matches queueFile when normalized
for _, queueFile := range queue.ToList() {
normalizedName := ApplyTransforms(ctx, dst, queueFile)
candidates, err := b.loadListing(newListing)
if err != nil {
fs.Errorf(candidates, "cannot read new listing: %v", err)
}
for _, filename := range candidates.list {
if ApplyTransforms(ctx, dst, filename) == normalizedName && filename != queueFile {
altNames.Add(filename) // original, not normalized
}
}
}
}
}