From 86c89f43a0651f0341dfb949694e46a9415aa667 Mon Sep 17 00:00:00 2001 From: a-s-z-home Date: Sat, 3 Nov 2018 20:13:43 +0100 Subject: [PATCH 1/6] Automatically exclude .duplicacy directory only, if nobackup_file is not set. --- src/duplicacy_entry.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/duplicacy_entry.go b/src/duplicacy_entry.go index 1f1b795..cf9db7e 100644 --- a/src/duplicacy_entry.go +++ b/src/duplicacy_entry.go @@ -481,7 +481,7 @@ func ListEntries(top string, path string, fileList *[]*Entry, patterns []string, entries := make([]*Entry, 0, 4) for _, f := range files { - if f.Name() == DUPLICACY_DIRECTORY { + if (f.Name() == DUPLICACY_DIRECTORY) && (nobackupFile == "") { continue } entry := CreateEntryFromFileInfo(f, normalizedPath) From 166f6e6266a733e49ecc28181c909cbf1eefbc1e Mon Sep 17 00:00:00 2001 From: a-s-z-home Date: Sat, 3 Nov 2018 20:20:00 +0100 Subject: [PATCH 2/6] Added string array helper functions Contains and Find. --- src/duplicacy_utils.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/duplicacy_utils.go b/src/duplicacy_utils.go index 1bfa728..de18c90 100644 --- a/src/duplicacy_utils.go +++ b/src/duplicacy_utils.go @@ -474,3 +474,24 @@ func MinInt(x, y int) int { } return y } + +// Contains tells whether a contains x. +func Contains(a []string, x string) bool { + for _, n := range a { + if x == n { + return true + } + } + return false +} + +// Find returns the smallest index i at which x == a[i], +// or len(a) if there is no such index. +func Find(a []string, x string) int { + for i, n := range a { + if x == n { + return i + } + } + return len(a) +} \ No newline at end of file From 96dd28995b6ad9d3775eb26af9f1330c1fe5cb82 Mon Sep 17 00:00:00 2001 From: a-s-z-home Date: Sat, 3 Nov 2018 20:39:03 +0100 Subject: [PATCH 3/6] Added an include mechanism for filter file. - Using @, you can now include other files. Relative paths are supported. This is useful, if you have several repositories with some different filters and a common filter base set. --- src/duplicacy_snapshot.go | 95 ++++++++++++++++++++++++--------------- 1 file changed, 60 insertions(+), 35 deletions(-) diff --git a/src/duplicacy_snapshot.go b/src/duplicacy_snapshot.go index 3355dc4..fd2e928 100644 --- a/src/duplicacy_snapshot.go +++ b/src/duplicacy_snapshot.go @@ -11,6 +11,7 @@ import ( "io/ioutil" "os" "path" + "path/filepath" "strconv" "strings" "time" @@ -68,44 +69,15 @@ func CreateSnapshotFromDirectory(id string, top string, nobackupFile string) (sn var patterns []string - patternFile, err := ioutil.ReadFile(path.Join(GetDuplicacyPreferencePath(), "filters")) - if err == nil { - for _, pattern := range strings.Split(string(patternFile), "\n") { - pattern = strings.TrimSpace(pattern) - if len(pattern) == 0 { - continue - } + patterns = ProcessFilterFile(path.Join(GetDuplicacyPreferencePath(), "filters"), make([]string, 0)) - if pattern[0] == '#' { - continue - } + LOG_DEBUG("REGEX_DEBUG", "There are %d compiled regular expressions stored", len(RegexMap)) - if IsUnspecifiedFilter(pattern) { - pattern = "+" + pattern - } + LOG_INFO("SNAPSHOT_FILTER", "Loaded %d include/exclude pattern(s)", len(patterns)) - if IsEmptyFilter(pattern) { - continue - } - - if strings.HasPrefix(pattern, "i:") || strings.HasPrefix(pattern, "e:") { - valid, err := IsValidRegex(pattern[2:]) - if !valid || err != nil { - LOG_ERROR("SNAPSHOT_FILTER", "Invalid regular expression encountered for filter: \"%s\", error: %v", pattern, err) - } - } - - patterns = append(patterns, pattern) - } - - LOG_DEBUG("REGEX_DEBUG", "There are %d compiled regular expressions stored", len(RegexMap)) - - LOG_INFO("SNAPSHOT_FILTER", "Loaded %d include/exclude pattern(s)", len(patterns)) - - if IsTracing() { - for _, pattern := range patterns { - LOG_TRACE("SNAPSHOT_PATTERN", "Pattern: %s", pattern) - } + if IsTracing() { + for _, pattern := range patterns { + LOG_TRACE("SNAPSHOT_PATTERN", "Pattern: %s", pattern) } } @@ -150,6 +122,59 @@ func CreateSnapshotFromDirectory(id string, top string, nobackupFile string) (sn return snapshot, skippedDirectories, skippedFiles, nil } +func ProcessFilterFile(patternFile string, includedFiles []string) (patterns []string) { + if Contains(includedFiles, patternFile) { + // cycle in include mechanism discovered. + LOG_WARN("SNAPSHOT_FILTER", "Cycle in filter includes: %s", strings.Join(includedFiles, " => ")) + return patterns + } + includedFiles = append(includedFiles, patternFile) + LOG_INFO("SNAPSHOT_FILTER", "Parsing filter file %s ...", patternFile) + patternFileContent, err := ioutil.ReadFile(patternFile) + if err == nil { + for _, pattern := range strings.Split(string(patternFileContent), "\n") { + pattern = strings.TrimSpace(pattern) + if len(pattern) == 0 { + continue + } + + if strings.HasPrefix(pattern, "@") { + patternIncludeFile := strings.TrimSpace(pattern[1:]) + if ! filepath.IsAbs(patternIncludeFile) { + patternIncludeFile = joinPath(filepath.Dir(patternFile), patternIncludeFile) + } + for _, pattern := range ProcessFilterFile(patternIncludeFile, includedFiles) { + patterns = append(patterns, pattern) + } + continue + } + + if pattern[0] == '#' { + continue + } + + if IsUnspecifiedFilter(pattern) { + pattern = "+" + pattern + } + + if IsEmptyFilter(pattern) { + continue + } + + if strings.HasPrefix(pattern, "i:") || strings.HasPrefix(pattern, "e:") { + valid, err := IsValidRegex(pattern[2:]) + if !valid || err != nil { + LOG_ERROR("SNAPSHOT_FILTER", "Invalid regular expression encountered for filter: \"%s\", error: %v", pattern, err) + } + } + + patterns = append(patterns, pattern) + } + } + + return patterns +} + // This is the struct used to save/load incomplete snapshots type IncompleteSnapshot struct { Files []*Entry From aaebf4510c776c5e84af020e0ab25e17ae2a444d Mon Sep 17 00:00:00 2001 From: a-s-z-home Date: Mon, 5 Nov 2018 00:32:12 +0100 Subject: [PATCH 4/6] - Replaced static check for .duplicacy directory with usage of predefined filters. Do not "misuse" property nobackupFile to trigger this feature. - Restructured ProcessFilterFile function and splitted it in smaller parts. - Prepare usage of new filter syntax for arguments of restore command. --- src/duplicacy_entry.go | 3 - src/duplicacy_snapshot.go | 138 +++++++++++++++++++++++++------------- 2 files changed, 90 insertions(+), 51 deletions(-) diff --git a/src/duplicacy_entry.go b/src/duplicacy_entry.go index cf9db7e..c640865 100644 --- a/src/duplicacy_entry.go +++ b/src/duplicacy_entry.go @@ -481,9 +481,6 @@ func ListEntries(top string, path string, fileList *[]*Entry, patterns []string, entries := make([]*Entry, 0, 4) for _, f := range files { - if (f.Name() == DUPLICACY_DIRECTORY) && (nobackupFile == "") { - continue - } entry := CreateEntryFromFileInfo(f, normalizedPath) if len(patterns) > 0 && !MatchPath(entry.Path, patterns) { continue diff --git a/src/duplicacy_snapshot.go b/src/duplicacy_snapshot.go index fd2e928..956c250 100644 --- a/src/duplicacy_snapshot.go +++ b/src/duplicacy_snapshot.go @@ -12,6 +12,7 @@ import ( "os" "path" "path/filepath" + "regexp" "strconv" "strings" "time" @@ -69,18 +70,7 @@ func CreateSnapshotFromDirectory(id string, top string, nobackupFile string) (sn var patterns []string - patterns = ProcessFilterFile(path.Join(GetDuplicacyPreferencePath(), "filters"), make([]string, 0)) - - LOG_DEBUG("REGEX_DEBUG", "There are %d compiled regular expressions stored", len(RegexMap)) - - LOG_INFO("SNAPSHOT_FILTER", "Loaded %d include/exclude pattern(s)", len(patterns)) - - if IsTracing() { - for _, pattern := range patterns { - LOG_TRACE("SNAPSHOT_PATTERN", "Pattern: %s", pattern) - } - - } + patterns = ProcessFilters() directories := make([]*Entry, 0, 256) directories = append(directories, CreateEntry("", 0, 0, 0)) @@ -122,6 +112,43 @@ func CreateSnapshotFromDirectory(id string, top string, nobackupFile string) (sn return snapshot, skippedDirectories, skippedFiles, nil } +func AppendPattern(patterns []string, new_pattern string) (new_patterns []string) { + for _, pattern := range patterns { + if pattern == new_pattern { + LOG_INFO("SNAPSHOT_FILTER", "Ignoring duplicate pattern: %s ...", new_pattern) + return patterns + } + } + new_patterns = append(patterns, new_pattern) + return new_patterns +} +func ProcessFilters() (patterns []string) { + patternFileLines := []string{ + `# ============================ exclude the internal files and directories in all ".duplicacy" subfolders`, + `e:(?i)(^|/)` + regexp.QuoteMeta(DUPLICACY_DIRECTORY) + `/cache/`, + `e:(?i)(^|/)` + regexp.QuoteMeta(DUPLICACY_DIRECTORY) + `/temporary$`, + `# ============================ exclude the internal files and directories in toplevel ".duplicacy" subfolder`, + `e:(?i)^` + regexp.QuoteMeta(DUPLICACY_DIRECTORY) + `/incomplete$`, + `e:(?i)^` + regexp.QuoteMeta(DUPLICACY_DIRECTORY) + `/logs/`, + "@" + joinPath(GetDuplicacyPreferencePath(), "filters"), + } + LOG_DEBUG("SNAPSHOT_FILTER", "Adding standard filters ...") + patterns = ProcessFilterLines(patternFileLines, make([]string, 0)) + + LOG_DEBUG("REGEX_DEBUG", "There are %d compiled regular expressions stored", len(RegexMap)) + + LOG_INFO("SNAPSHOT_FILTER", "Loaded %d include/exclude pattern(s)", len(patterns)) + + if IsTracing() { + for _, pattern := range patterns { + LOG_TRACE("SNAPSHOT_PATTERN", "Pattern: %s", pattern) + } + + } + + return patterns +} + func ProcessFilterFile(patternFile string, includedFiles []string) (patterns []string) { if Contains(includedFiles, patternFile) { // cycle in include mechanism discovered. @@ -132,44 +159,59 @@ func ProcessFilterFile(patternFile string, includedFiles []string) (patterns []s LOG_INFO("SNAPSHOT_FILTER", "Parsing filter file %s ...", patternFile) patternFileContent, err := ioutil.ReadFile(patternFile) if err == nil { - for _, pattern := range strings.Split(string(patternFileContent), "\n") { - pattern = strings.TrimSpace(pattern) - if len(pattern) == 0 { - continue - } + patternFileLines := strings.Split(string(patternFileContent), "\n") + patterns = ProcessFilterLines(patternFileLines, includedFiles) + } + return patterns +} - if strings.HasPrefix(pattern, "@") { - patternIncludeFile := strings.TrimSpace(pattern[1:]) - if ! filepath.IsAbs(patternIncludeFile) { - patternIncludeFile = joinPath(filepath.Dir(patternFile), patternIncludeFile) - } - for _, pattern := range ProcessFilterFile(patternIncludeFile, includedFiles) { - patterns = append(patterns, pattern) - } - continue - } - - if pattern[0] == '#' { - continue - } - - if IsUnspecifiedFilter(pattern) { - pattern = "+" + pattern - } - - if IsEmptyFilter(pattern) { - continue - } - - if strings.HasPrefix(pattern, "i:") || strings.HasPrefix(pattern, "e:") { - valid, err := IsValidRegex(pattern[2:]) - if !valid || err != nil { - LOG_ERROR("SNAPSHOT_FILTER", "Invalid regular expression encountered for filter: \"%s\", error: %v", pattern, err) - } - } - - patterns = append(patterns, pattern) +func ProcessFilterLines(patternFileLines []string, includedFiles []string) (patterns []string) { + for _, pattern := range patternFileLines { + pattern = strings.TrimSpace(pattern) + if len(pattern) == 0 { + continue } + + if strings.HasPrefix(pattern, "@") { + patternIncludeFile := strings.TrimSpace(pattern[1:]) + if patternIncludeFile == "" { + continue + } + if ! filepath.IsAbs(patternIncludeFile) { + basePath := "" + if len(includedFiles) == 0 { + basePath, _ = os.Getwd() + } else { + basePath = filepath.Dir(includedFiles[len(includedFiles)-1]) + } + patternIncludeFile = joinPath(basePath, patternIncludeFile) + } + for _, pattern := range ProcessFilterFile(patternIncludeFile, includedFiles) { + patterns = AppendPattern(patterns, pattern) + } + continue + } + + if pattern[0] == '#' { + continue + } + + if IsUnspecifiedFilter(pattern) { + pattern = "+" + pattern + } + + if IsEmptyFilter(pattern) { + continue + } + + if strings.HasPrefix(pattern, "i:") || strings.HasPrefix(pattern, "e:") { + valid, err := IsValidRegex(pattern[2:]) + if !valid || err != nil { + LOG_ERROR("SNAPSHOT_FILTER", "Invalid regular expression encountered for filter: \"%s\", error: %v", pattern, err) + } + } + + patterns = AppendPattern(patterns, pattern) } return patterns From e1fa39008dc95dea42ce40d7b6170c4e7ee1f66b Mon Sep 17 00:00:00 2001 From: a-s-z-home Date: Mon, 5 Nov 2018 00:59:39 +0100 Subject: [PATCH 5/6] Use new filter processing function for restore command. - You can now include a filter file by using "@". --- duplicacy/duplicacy_main.go | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/duplicacy/duplicacy_main.go b/duplicacy/duplicacy_main.go index ca0c4a4..8c0cb0e 100644 --- a/duplicacy/duplicacy_main.go +++ b/duplicacy/duplicacy_main.go @@ -784,27 +784,17 @@ func restoreRepository(context *cli.Context) { pattern = pattern[1:] } - if duplicacy.IsUnspecifiedFilter(pattern) { - pattern = "+" + pattern - } - - if duplicacy.IsEmptyFilter(pattern) { - continue - } - - if strings.HasPrefix(pattern, "i:") || strings.HasPrefix(pattern, "e:") { - valid, err := duplicacy.IsValidRegex(pattern[2:]) - if !valid || err != nil { - duplicacy.LOG_ERROR("SNAPSHOT_FILTER", "Invalid regular expression encountered for filter: \"%s\", error: %v", pattern, err) - } - } - patterns = append(patterns, pattern) + + } + patterns = duplicacy.ProcessFilterLines(patterns, make([]string, 0)) duplicacy.LOG_DEBUG("REGEX_DEBUG", "There are %d compiled regular expressions stored", len(duplicacy.RegexMap)) + duplicacy.LOG_INFO("SNAPSHOT_FILTER", "Loaded %d include/exclude pattern(s)", len(patterns)) + storage.SetRateLimits(context.Int("limit-rate"), 0) backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, preference.NobackupFile) duplicacy.SavePassword(*preference, "password", password) From 5e8baab4ecf273599a53b794d3babfc7b5c3de81 Mon Sep 17 00:00:00 2001 From: a-s-z-home Date: Mon, 5 Nov 2018 22:39:11 +0100 Subject: [PATCH 6/6] - Reverted changes to exclude mechanism of .duplicacy directory. --- src/duplicacy_entry.go | 3 +++ src/duplicacy_snapshot.go | 13 +------------ 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/src/duplicacy_entry.go b/src/duplicacy_entry.go index c640865..1f1b795 100644 --- a/src/duplicacy_entry.go +++ b/src/duplicacy_entry.go @@ -481,6 +481,9 @@ func ListEntries(top string, path string, fileList *[]*Entry, patterns []string, entries := make([]*Entry, 0, 4) for _, f := range files { + if f.Name() == DUPLICACY_DIRECTORY { + continue + } entry := CreateEntryFromFileInfo(f, normalizedPath) if len(patterns) > 0 && !MatchPath(entry.Path, patterns) { continue diff --git a/src/duplicacy_snapshot.go b/src/duplicacy_snapshot.go index 956c250..49b8cbc 100644 --- a/src/duplicacy_snapshot.go +++ b/src/duplicacy_snapshot.go @@ -12,7 +12,6 @@ import ( "os" "path" "path/filepath" - "regexp" "strconv" "strings" "time" @@ -123,17 +122,7 @@ func AppendPattern(patterns []string, new_pattern string) (new_patterns []string return new_patterns } func ProcessFilters() (patterns []string) { - patternFileLines := []string{ - `# ============================ exclude the internal files and directories in all ".duplicacy" subfolders`, - `e:(?i)(^|/)` + regexp.QuoteMeta(DUPLICACY_DIRECTORY) + `/cache/`, - `e:(?i)(^|/)` + regexp.QuoteMeta(DUPLICACY_DIRECTORY) + `/temporary$`, - `# ============================ exclude the internal files and directories in toplevel ".duplicacy" subfolder`, - `e:(?i)^` + regexp.QuoteMeta(DUPLICACY_DIRECTORY) + `/incomplete$`, - `e:(?i)^` + regexp.QuoteMeta(DUPLICACY_DIRECTORY) + `/logs/`, - "@" + joinPath(GetDuplicacyPreferencePath(), "filters"), - } - LOG_DEBUG("SNAPSHOT_FILTER", "Adding standard filters ...") - patterns = ProcessFilterLines(patternFileLines, make([]string, 0)) + patterns = ProcessFilterFile(joinPath(GetDuplicacyPreferencePath(), "filters"), make([]string, 0)) LOG_DEBUG("REGEX_DEBUG", "There are %d compiled regular expressions stored", len(RegexMap))