jgaunt/rclone
1
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2026-01-04 17:43:50 +00:00
Code Issues Projects Releases Wiki Activity

bisync: handle unicode and case normalization consistently - mostly-fixes #7270

Browse Source 
Before this change, Bisync sometimes normalized NFD to NFC and sometimes
did not, causing errors in some scenarios (particularly for users of macOS).
It was similarly inconsistent in its handling of case-insensitivity.

There were three main places where Bisync should have normalized, but didn't:

1. When building the list of files that need to be transferred during --resync
2. When building the list of deltas during a non-resync
3. When comparing Path1 to Path2 during --check-sync

After this change, 1 and 3 are resolved, and bisync supports
--no-unicode-normalization and --ignore-case-sync in the same way as sync.
2 will be addressed in a future update.
This commit is contained in:
nielash
2023-10-08 23:16:23 -04:00
parent 11afc3dde0
commit f7f4651828
32 changed files with 441 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path1.lst vendored
View File

@@ -1,10 +1,9 @@
# bisync listing v1 from test
- 6148 md5:23b446fda9938c607142c5133cf90689 - 2000-01-01T00:00:00.000000000+0000 ".DS_Store"
- 109 md5:294d25b294ff26a5243dba914ac3fbf7 - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy1.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy2.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy3.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy4.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy5.txt"
- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-02T00:00:00.000000000+0000 "file1.txt"
- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-02T00:00:00.000000000+0000 "subdir/file20.txt"
- 109 - - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy1.txt"
- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy2.txt"
- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy3.txt"
- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy4.txt"
- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy5.txt"
- 19 - - 2001-01-02T00:00:00.000000000+0000 "file1.txt"
- 19 - - 2001-01-02T00:00:00.000000000+0000 "subdir/file20.txt"

1
cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path1.lst-new vendored
View File

@@ -1,5 +1,4 @@
# bisync listing v1 from test
- 6148 md5:23b446fda9938c607142c5133cf90689 - 2000-01-01T00:00:00.000000000+0000 ".DS_Store"
- 109 md5:294d25b294ff26a5243dba914ac3fbf7 - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy1.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy2.txt"

17
cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path1.lst-old vendored
View File

@@ -1,10 +1,9 @@
# bisync listing v1 from test
- 6148 md5:23b446fda9938c607142c5133cf90689 - 2000-01-01T00:00:00.000000000+0000 ".DS_Store"
- 109 md5:294d25b294ff26a5243dba914ac3fbf7 - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy1.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy2.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy3.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy4.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy5.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "subdir/file20.txt"
- 109 - - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy1.txt"
- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy2.txt"
- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy3.txt"
- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy4.txt"
- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy5.txt"
- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.txt"
- 0 - - 2000-01-01T00:00:00.000000000+0000 "subdir/file20.txt"

17
cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path2.lst vendored
View File

@@ -1,10 +1,9 @@
# bisync listing v1 from test
- 6148 md5:23b446fda9938c607142c5133cf90689 - 2000-01-01T00:00:00.000000000+0000 ".DS_Store"
- 109 md5:294d25b294ff26a5243dba914ac3fbf7 - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy1.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy2.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy3.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy4.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy5.txt"
- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-02T00:00:00.000000000+0000 "file1.txt"
- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-02T00:00:00.000000000+0000 "subdir/file20.txt"
- 109 - - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy1.txt"
- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy2.txt"
- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy3.txt"
- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy4.txt"
- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy5.txt"
- 19 - - 2001-01-02T00:00:00.000000000+0000 "file1.txt"
- 19 - - 2001-01-02T00:00:00.000000000+0000 "subdir/file20.txt"

1
cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path2.lst-new vendored
View File

@@ -1,5 +1,4 @@
# bisync listing v1 from test
- 6148 md5:23b446fda9938c607142c5133cf90689 - 2000-01-01T00:00:00.000000000+0000 ".DS_Store"
- 109 md5:294d25b294ff26a5243dba914ac3fbf7 - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy1.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy2.txt"

17
cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path2.lst-old vendored
View File

@@ -1,10 +1,9 @@
# bisync listing v1 from test
- 6148 md5:23b446fda9938c607142c5133cf90689 - 2000-01-01T00:00:00.000000000+0000 ".DS_Store"
- 109 md5:294d25b294ff26a5243dba914ac3fbf7 - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy1.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy2.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy3.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy4.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy5.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "subdir/file20.txt"
- 109 - - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy1.txt"
- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy2.txt"
- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy3.txt"
- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy4.txt"
- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy5.txt"
- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.txt"
- 0 - - 2000-01-01T00:00:00.000000000+0000 "subdir/file20.txt"