diff --git a/duplicacy/duplicacy_main.go b/duplicacy/duplicacy_main.go index da789e0..964f046 100644 --- a/duplicacy/duplicacy_main.go +++ b/duplicacy/duplicacy_main.go @@ -782,27 +782,17 @@ func restoreRepository(context *cli.Context) { pattern = pattern[1:] } - if duplicacy.IsUnspecifiedFilter(pattern) { - pattern = "+" + pattern - } - - if duplicacy.IsEmptyFilter(pattern) { - continue - } - - if strings.HasPrefix(pattern, "i:") || strings.HasPrefix(pattern, "e:") { - valid, err := duplicacy.IsValidRegex(pattern[2:]) - if !valid || err != nil { - duplicacy.LOG_ERROR("SNAPSHOT_FILTER", "Invalid regular expression encountered for filter: \"%s\", error: %v", pattern, err) - } - } - patterns = append(patterns, pattern) + + } + patterns = duplicacy.ProcessFilterLines(patterns, make([]string, 0)) duplicacy.LOG_DEBUG("REGEX_DEBUG", "There are %d compiled regular expressions stored", len(duplicacy.RegexMap)) + duplicacy.LOG_INFO("SNAPSHOT_FILTER", "Loaded %d include/exclude pattern(s)", len(patterns)) + storage.SetRateLimits(context.Int("limit-rate"), 0) backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, preference.NobackupFile) duplicacy.SavePassword(*preference, "password", password) diff --git a/src/duplicacy_snapshot.go b/src/duplicacy_snapshot.go index 3355dc4..49b8cbc 100644 --- a/src/duplicacy_snapshot.go +++ b/src/duplicacy_snapshot.go @@ -11,6 +11,7 @@ import ( "io/ioutil" "os" "path" + "path/filepath" "strconv" "strings" "time" @@ -68,47 +69,7 @@ func CreateSnapshotFromDirectory(id string, top string, nobackupFile string) (sn var patterns []string - patternFile, err := ioutil.ReadFile(path.Join(GetDuplicacyPreferencePath(), "filters")) - if err == nil { - for _, pattern := range strings.Split(string(patternFile), "\n") { - pattern = strings.TrimSpace(pattern) - if len(pattern) == 0 { - continue - } - - if pattern[0] == '#' { - continue - } - - if IsUnspecifiedFilter(pattern) { - pattern = "+" + pattern - } - - if IsEmptyFilter(pattern) { - continue - } - - if strings.HasPrefix(pattern, "i:") || strings.HasPrefix(pattern, "e:") { - valid, err := IsValidRegex(pattern[2:]) - if !valid || err != nil { - LOG_ERROR("SNAPSHOT_FILTER", "Invalid regular expression encountered for filter: \"%s\", error: %v", pattern, err) - } - } - - patterns = append(patterns, pattern) - } - - LOG_DEBUG("REGEX_DEBUG", "There are %d compiled regular expressions stored", len(RegexMap)) - - LOG_INFO("SNAPSHOT_FILTER", "Loaded %d include/exclude pattern(s)", len(patterns)) - - if IsTracing() { - for _, pattern := range patterns { - LOG_TRACE("SNAPSHOT_PATTERN", "Pattern: %s", pattern) - } - } - - } + patterns = ProcessFilters() directories := make([]*Entry, 0, 256) directories = append(directories, CreateEntry("", 0, 0, 0)) @@ -150,6 +111,101 @@ func CreateSnapshotFromDirectory(id string, top string, nobackupFile string) (sn return snapshot, skippedDirectories, skippedFiles, nil } +func AppendPattern(patterns []string, new_pattern string) (new_patterns []string) { + for _, pattern := range patterns { + if pattern == new_pattern { + LOG_INFO("SNAPSHOT_FILTER", "Ignoring duplicate pattern: %s ...", new_pattern) + return patterns + } + } + new_patterns = append(patterns, new_pattern) + return new_patterns +} +func ProcessFilters() (patterns []string) { + patterns = ProcessFilterFile(joinPath(GetDuplicacyPreferencePath(), "filters"), make([]string, 0)) + + LOG_DEBUG("REGEX_DEBUG", "There are %d compiled regular expressions stored", len(RegexMap)) + + LOG_INFO("SNAPSHOT_FILTER", "Loaded %d include/exclude pattern(s)", len(patterns)) + + if IsTracing() { + for _, pattern := range patterns { + LOG_TRACE("SNAPSHOT_PATTERN", "Pattern: %s", pattern) + } + + } + + return patterns +} + +func ProcessFilterFile(patternFile string, includedFiles []string) (patterns []string) { + if Contains(includedFiles, patternFile) { + // cycle in include mechanism discovered. + LOG_WARN("SNAPSHOT_FILTER", "Cycle in filter includes: %s", strings.Join(includedFiles, " => ")) + return patterns + } + includedFiles = append(includedFiles, patternFile) + LOG_INFO("SNAPSHOT_FILTER", "Parsing filter file %s ...", patternFile) + patternFileContent, err := ioutil.ReadFile(patternFile) + if err == nil { + patternFileLines := strings.Split(string(patternFileContent), "\n") + patterns = ProcessFilterLines(patternFileLines, includedFiles) + } + return patterns +} + +func ProcessFilterLines(patternFileLines []string, includedFiles []string) (patterns []string) { + for _, pattern := range patternFileLines { + pattern = strings.TrimSpace(pattern) + if len(pattern) == 0 { + continue + } + + if strings.HasPrefix(pattern, "@") { + patternIncludeFile := strings.TrimSpace(pattern[1:]) + if patternIncludeFile == "" { + continue + } + if ! filepath.IsAbs(patternIncludeFile) { + basePath := "" + if len(includedFiles) == 0 { + basePath, _ = os.Getwd() + } else { + basePath = filepath.Dir(includedFiles[len(includedFiles)-1]) + } + patternIncludeFile = joinPath(basePath, patternIncludeFile) + } + for _, pattern := range ProcessFilterFile(patternIncludeFile, includedFiles) { + patterns = AppendPattern(patterns, pattern) + } + continue + } + + if pattern[0] == '#' { + continue + } + + if IsUnspecifiedFilter(pattern) { + pattern = "+" + pattern + } + + if IsEmptyFilter(pattern) { + continue + } + + if strings.HasPrefix(pattern, "i:") || strings.HasPrefix(pattern, "e:") { + valid, err := IsValidRegex(pattern[2:]) + if !valid || err != nil { + LOG_ERROR("SNAPSHOT_FILTER", "Invalid regular expression encountered for filter: \"%s\", error: %v", pattern, err) + } + } + + patterns = AppendPattern(patterns, pattern) + } + + return patterns +} + // This is the struct used to save/load incomplete snapshots type IncompleteSnapshot struct { Files []*Entry diff --git a/src/duplicacy_utils.go b/src/duplicacy_utils.go index 5dda266..6d74808 100644 --- a/src/duplicacy_utils.go +++ b/src/duplicacy_utils.go @@ -467,3 +467,24 @@ func MinInt(x, y int) int { } return y } + +// Contains tells whether a contains x. +func Contains(a []string, x string) bool { + for _, n := range a { + if x == n { + return true + } + } + return false +} + +// Find returns the smallest index i at which x == a[i], +// or len(a) if there is no such index. +func Find(a []string, x string) int { + for i, n := range a { + if x == n { + return i + } + } + return len(a) +} \ No newline at end of file