diff --git a/duplicacy/duplicacy_main.go b/duplicacy/duplicacy_main.go index 4a7d8c6..4bcf6f0 100644 --- a/duplicacy/duplicacy_main.go +++ b/duplicacy/duplicacy_main.go @@ -536,6 +536,11 @@ func setPreference(context *cli.Context) { newPreference.NobackupFile = context.String("nobackup-file") + triBool = context.Generic("exclude-by-attribute").(*TriBool) + if triBool.IsSet() { + newPreference.ExcludeByAttribute = triBool.IsTrue() + } + key := context.String("key") value := context.String("value") @@ -717,7 +722,7 @@ func backupRepository(context *cli.Context) { uploadRateLimit := context.Int("limit-rate") enumOnly := context.Bool("enum-only") storage.SetRateLimits(0, uploadRateLimit) - backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, preference.NobackupFile) + backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, preference.NobackupFile, preference.ExcludeByAttribute) duplicacy.SavePassword(*preference, "password", password) backupManager.SetupSnapshotCache(preference.Name) @@ -806,7 +811,7 @@ func restoreRepository(context *cli.Context) { duplicacy.LOG_DEBUG("REGEX_DEBUG", "There are %d compiled regular expressions stored", len(duplicacy.RegexMap)) storage.SetRateLimits(context.Int("limit-rate"), 0) - backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, preference.NobackupFile) + backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, preference.NobackupFile, preference.ExcludeByAttribute) duplicacy.SavePassword(*preference, "password", password) backupManager.SetupSnapshotCache(preference.Name) @@ -846,7 +851,7 @@ func listSnapshots(context *cli.Context) { tag := context.String("t") revisions := getRevisions(context) - backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, preference.NobackupFile) + backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, preference.NobackupFile, preference.ExcludeByAttribute) duplicacy.SavePassword(*preference, "password", password) id := preference.SnapshotID @@ -894,7 +899,7 @@ func checkSnapshots(context *cli.Context) { tag := context.String("t") revisions := getRevisions(context) - backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, preference.NobackupFile) + backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, preference.NobackupFile, preference.ExcludeByAttribute) duplicacy.SavePassword(*preference, "password", password) id := preference.SnapshotID @@ -949,7 +954,7 @@ func printFile(context *cli.Context) { snapshotID = context.String("id") } - backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, preference.NobackupFile) + backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, preference.NobackupFile, preference.ExcludeByAttribute) duplicacy.SavePassword(*preference, "password", password) backupManager.SetupSnapshotCache(preference.Name) @@ -1005,11 +1010,11 @@ func diff(context *cli.Context) { } compareByHash := context.Bool("hash") - backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, preference.NobackupFile) + backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, preference.NobackupFile, preference.ExcludeByAttribute) duplicacy.SavePassword(*preference, "password", password) backupManager.SetupSnapshotCache(preference.Name) - backupManager.SnapshotManager.Diff(repository, snapshotID, revisions, path, compareByHash, preference.NobackupFile) + backupManager.SnapshotManager.Diff(repository, snapshotID, revisions, path, compareByHash, preference.NobackupFile, preference.ExcludeByAttribute) runScript(context, preference.Name, "post") } @@ -1048,7 +1053,7 @@ func showHistory(context *cli.Context) { revisions := getRevisions(context) showLocalHash := context.Bool("hash") - backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, preference.NobackupFile) + backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, preference.NobackupFile, preference.ExcludeByAttribute) duplicacy.SavePassword(*preference, "password", password) backupManager.SetupSnapshotCache(preference.Name) @@ -1111,7 +1116,7 @@ func pruneSnapshots(context *cli.Context) { os.Exit(ArgumentExitCode) } - backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, preference.NobackupFile) + backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, preference.NobackupFile, preference.ExcludeByAttribute) duplicacy.SavePassword(*preference, "password", password) backupManager.SetupSnapshotCache(preference.Name) @@ -1151,7 +1156,7 @@ func copySnapshots(context *cli.Context) { sourcePassword = duplicacy.GetPassword(*source, "password", "Enter source storage password:", false, false) } - sourceManager := duplicacy.CreateBackupManager(source.SnapshotID, sourceStorage, repository, sourcePassword, source.NobackupFile) + sourceManager := duplicacy.CreateBackupManager(source.SnapshotID, sourceStorage, repository, sourcePassword, source.NobackupFile, source.ExcludeByAttribute) sourceManager.SetupSnapshotCache(source.Name) duplicacy.SavePassword(*source, "password", sourcePassword) @@ -1184,7 +1189,7 @@ func copySnapshots(context *cli.Context) { destinationStorage.SetRateLimits(0, context.Int("upload-limit-rate")) destinationManager := duplicacy.CreateBackupManager(destination.SnapshotID, destinationStorage, repository, - destinationPassword, destination.NobackupFile) + destinationPassword, destination.NobackupFile, destination.ExcludeByAttribute) duplicacy.SavePassword(*destination, "password", destinationPassword) destinationManager.SetupSnapshotCache(destination.Name) @@ -1818,7 +1823,12 @@ func main() { Name: "nobackup-file", Usage: "Directories containing a file with this name will not be backed up", Argument: "", - Value: "", + }, + cli.GenericFlag{ + Name: "exclude-by-attribute", + Usage: "Exclude files based on file attributes. (macOS only, com_apple_backup_excludeItem)", + Value: &TriBool{}, + Arg: "true", }, cli.StringFlag{ Name: "key", diff --git a/src/duplicacy_backupmanager.go b/src/duplicacy_backupmanager.go index 0b908dd..8671c3c 100644 --- a/src/duplicacy_backupmanager.go +++ b/src/duplicacy_backupmanager.go @@ -35,6 +35,8 @@ type BackupManager struct { config *Config // contains a number of options nobackupFile string // don't backup directory when this file name is found + + excludeByAttribute bool // don't backup file based on file attribute } func (manager *BackupManager) SetDryRun(dryRun bool) { @@ -44,7 +46,7 @@ func (manager *BackupManager) SetDryRun(dryRun bool) { // CreateBackupManager creates a backup manager using the specified 'storage'. 'snapshotID' is a unique id to // identify snapshots created for this repository. 'top' is the top directory of the repository. 'password' is the // master key which can be nil if encryption is not enabled. -func CreateBackupManager(snapshotID string, storage Storage, top string, password string, nobackupFile string) *BackupManager { +func CreateBackupManager(snapshotID string, storage Storage, top string, password string, nobackupFile string, excludeByAttribute bool) *BackupManager { config, _, err := DownloadConfig(storage, password) if err != nil { @@ -67,6 +69,8 @@ func CreateBackupManager(snapshotID string, storage Storage, top string, passwor config: config, nobackupFile: nobackupFile, + + excludeByAttribute: excludeByAttribute, } if IsDebugging() { @@ -188,7 +192,7 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta defer DeleteShadowCopy() LOG_INFO("BACKUP_INDEXING", "Indexing %s", top) - localSnapshot, skippedDirectories, skippedFiles, err := CreateSnapshotFromDirectory(manager.snapshotID, shadowTop, manager.nobackupFile) + localSnapshot, skippedDirectories, skippedFiles, err := CreateSnapshotFromDirectory(manager.snapshotID, shadowTop, manager.nobackupFile, manager.excludeByAttribute) if err != nil { LOG_ERROR("SNAPSHOT_LIST", "Failed to list the directory %s: %v", top, err) return false @@ -760,7 +764,7 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu remoteSnapshot := manager.SnapshotManager.DownloadSnapshot(manager.snapshotID, revision) manager.SnapshotManager.DownloadSnapshotContents(remoteSnapshot, patterns, true) - localSnapshot, _, _, err := CreateSnapshotFromDirectory(manager.snapshotID, top, manager.nobackupFile) + localSnapshot, _, _, err := CreateSnapshotFromDirectory(manager.snapshotID, top, manager.nobackupFile, manager.excludeByAttribute) if err != nil { LOG_ERROR("SNAPSHOT_LIST", "Failed to list the repository: %v", err) return false diff --git a/src/duplicacy_backupmanager_test.go b/src/duplicacy_backupmanager_test.go index dce5f6c..4312cf4 100644 --- a/src/duplicacy_backupmanager_test.go +++ b/src/duplicacy_backupmanager_test.go @@ -239,7 +239,7 @@ func TestBackupManager(t *testing.T) { time.Sleep(time.Duration(delay) * time.Second) SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy") - backupManager := CreateBackupManager("host1", storage, testDir, password, "") + backupManager := CreateBackupManager("host1", storage, testDir, password, "", false) backupManager.SetupSnapshotCache("default") SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy") diff --git a/src/duplicacy_entry.go b/src/duplicacy_entry.go index 1f1b795..40fa66f 100644 --- a/src/duplicacy_entry.go +++ b/src/duplicacy_entry.go @@ -28,6 +28,29 @@ var fileModeMask = os.ModePerm | os.ModeSetuid | os.ModeSetgid | os.ModeSticky // Regex for matching 'StartChunk:StartOffset:EndChunk:EndOffset' var contentRegex = regexp.MustCompile(`^([0-9]+):([0-9]+):([0-9]+):([0-9]+)`) +// AttributeExcludeName attribute name to determine file exclusion +var AttributeExcludeName = getDefaultAttributeExcludeName() + +// AttributeExcludeValue attribute value to determine file exclusion +var AttributeExcludeValue = getDefaultAttributeExcludeValue() + +func getDefaultAttributeExcludeName() string { + if runtime.GOOS == "darwin" { + return "com.apple.metadata:com_apple_backup_excludeItem" + } + if runtime.GOOS == "linux" { + return "duplicacy_exclude" + } + return "" +} + +func getDefaultAttributeExcludeValue() string { + if runtime.GOOS == "darwin" { + return "com.apple.backupd" + } + return "" +} + // Entry encapsulates information about a file or directory. type Entry struct { Path string @@ -443,7 +466,7 @@ func (files FileInfoCompare) Less(i, j int) bool { // ListEntries returns a list of entries representing file and subdirectories under the directory 'path'. Entry paths // are normalized as relative to 'top'. 'patterns' are used to exclude or include certain files. -func ListEntries(top string, path string, fileList *[]*Entry, patterns []string, nobackupFile string, discardAttributes bool) (directoryList []*Entry, +func ListEntries(top string, path string, fileList *[]*Entry, patterns []string, nobackupFile string, discardAttributes bool, excludeByAttribute bool) (directoryList []*Entry, skippedFiles []string, err error) { LOG_DEBUG("LIST_ENTRIES", "Listing %s", path) @@ -521,6 +544,17 @@ func ListEntries(top string, path string, fileList *[]*Entry, patterns []string, entry.ReadAttributes(top) } + if excludeByAttribute && (runtime.GOOS == "darwin" || runtime.GOOS == "linux") { + attrValue, ok := entry.Attributes[AttributeExcludeName] + if ok { + attrValueString := string(attrValue) + if strings.Contains(attrValueString, AttributeExcludeValue) { + LOG_WARN("LIST_NOBACKUPXATTR", "%s is excluded due to extended attribute: %s", entry.Path, AttributeExcludeName) + continue + } + } + } + if f.Mode()&(os.ModeNamedPipe|os.ModeSocket|os.ModeDevice) != 0 { LOG_WARN("LIST_SKIP", "Skipped non-regular file %s", entry.Path) skippedFiles = append(skippedFiles, entry.Path) diff --git a/src/duplicacy_entry_test.go b/src/duplicacy_entry_test.go index d809412..246d17d 100644 --- a/src/duplicacy_entry_test.go +++ b/src/duplicacy_entry_test.go @@ -9,8 +9,12 @@ import ( "math/rand" "os" "path/filepath" + "runtime" "sort" + "strings" "testing" + + "github.com/gilbertchen/xattr" ) func TestEntrySort(t *testing.T) { @@ -173,7 +177,7 @@ func TestEntryList(t *testing.T) { directory := directories[len(directories)-1] directories = directories[:len(directories)-1] entries = append(entries, directory) - subdirectories, _, err := ListEntries(testDir, directory.Path, &entries, nil, "", false) + subdirectories, _, err := ListEntries(testDir, directory.Path, &entries, nil, "", false, false) if err != nil { t.Errorf("ListEntries(%s, %s) returned an error: %s", testDir, directory.Path, err) } @@ -216,3 +220,110 @@ func TestEntryList(t *testing.T) { } } + +// TestEntryExcludeByAttribute tests the excludeByAttribute parameter to the ListEntries function +func TestEntryExcludeByAttribute(t *testing.T) { + + if !(runtime.GOOS == "darwin" || runtime.GOOS == "linux") { + t.Skip("skipping test not darwin or linux") + } + + testDir := filepath.Join(os.TempDir(), "duplicacy_test") + + os.RemoveAll(testDir) + os.MkdirAll(testDir, 0700) + + // Files or folders named with "exclude" below will have the exclusion attribute set on them + // When ListEntries is called with excludeByAttribute true, they should be excluded. + DATA := [...]string{ + "excludefile", + "includefile", + "excludedir/", + "excludedir/file", + "includedir/", + "includedir/includefile", + "includedir/excludefile", + } + + for _, file := range DATA { + fullPath := filepath.Join(testDir, file) + if file[len(file)-1] == '/' { + err := os.Mkdir(fullPath, 0700) + if err != nil { + t.Errorf("Mkdir(%s) returned an error: %s", fullPath, err) + } + continue + } + + err := ioutil.WriteFile(fullPath, []byte(file), 0700) + if err != nil { + t.Errorf("WriteFile(%s) returned an error: %s", fullPath, err) + } + } + + for _, file := range DATA { + fullPath := filepath.Join(testDir, file) + if strings.Contains(file, "exclude") { + xattr.Setxattr(fullPath, AttributeExcludeName, []byte(AttributeExcludeValue)) + } + } + + for _, excludeByAttribute := range [2]bool{true, false} { + t.Logf("testing excludeByAttribute: %t", excludeByAttribute) + directories := make([]*Entry, 0, 4) + directories = append(directories, CreateEntry("", 0, 0, 0)) + + entries := make([]*Entry, 0, 4) + + for len(directories) > 0 { + directory := directories[len(directories)-1] + directories = directories[:len(directories)-1] + entries = append(entries, directory) + subdirectories, _, err := ListEntries(testDir, directory.Path, &entries, nil, "", false, excludeByAttribute) + if err != nil { + t.Errorf("ListEntries(%s, %s) returned an error: %s", testDir, directory.Path, err) + } + directories = append(directories, subdirectories...) + } + + entries = entries[1:] + + for _, entry := range entries { + t.Logf("entry: %s", entry.Path) + } + + i := 0 + for _, file := range DATA { + entryFound := false + var entry *Entry + for _, entry = range entries { + if entry.Path == file { + entryFound = true + break + } + } + + if excludeByAttribute && strings.Contains(file, "exclude") { + if entryFound { + t.Errorf("file: %s, expected to be excluded but wasn't. attributes: %v", file, entry.Attributes) + i++ + } else { + t.Logf("file: %s, excluded", file) + } + } else { + if entryFound { + t.Logf("file: %s, included. attributes: %v", file, entry.Attributes) + i++ + } else { + t.Errorf("file: %s, expected to be included but wasn't", file) + } + } + } + + } + + if !t.Failed() { + os.RemoveAll(testDir) + } + +} diff --git a/src/duplicacy_preference.go b/src/duplicacy_preference.go index 3ae35e9..57f613c 100644 --- a/src/duplicacy_preference.go +++ b/src/duplicacy_preference.go @@ -15,16 +15,17 @@ import ( // Preference stores options for each storage. type Preference struct { - Name string `json:"name"` - SnapshotID string `json:"id"` - RepositoryPath string `json:"repository"` - StorageURL string `json:"storage"` - Encrypted bool `json:"encrypted"` - BackupProhibited bool `json:"no_backup"` - RestoreProhibited bool `json:"no_restore"` - DoNotSavePassword bool `json:"no_save_password"` - NobackupFile string `json:"nobackup_file"` - Keys map[string]string `json:"keys"` + Name string `json:"name"` + SnapshotID string `json:"id"` + RepositoryPath string `json:"repository"` + StorageURL string `json:"storage"` + Encrypted bool `json:"encrypted"` + BackupProhibited bool `json:"no_backup"` + RestoreProhibited bool `json:"no_restore"` + DoNotSavePassword bool `json:"no_save_password"` + NobackupFile string `json:"nobackup_file"` + ExcludeByAttribute bool `json:"exclude_by_attribute"` + Keys map[string]string `json:"keys"` } var preferencePath string diff --git a/src/duplicacy_snapshot.go b/src/duplicacy_snapshot.go index 3355dc4..ae249bf 100644 --- a/src/duplicacy_snapshot.go +++ b/src/duplicacy_snapshot.go @@ -57,7 +57,7 @@ func CreateEmptySnapshot(id string) (snapshto *Snapshot) { // CreateSnapshotFromDirectory creates a snapshot from the local directory 'top'. Only 'Files' // will be constructed, while 'ChunkHashes' and 'ChunkLengths' can only be populated after uploading. -func CreateSnapshotFromDirectory(id string, top string, nobackupFile string) (snapshot *Snapshot, skippedDirectories []string, +func CreateSnapshotFromDirectory(id string, top string, nobackupFile string, excludeByAttribute bool) (snapshot *Snapshot, skippedDirectories []string, skippedFiles []string, err error) { snapshot = &Snapshot{ @@ -125,7 +125,7 @@ func CreateSnapshotFromDirectory(id string, top string, nobackupFile string) (sn directory := directories[len(directories)-1] directories = directories[:len(directories)-1] snapshot.Files = append(snapshot.Files, directory) - subdirectories, skipped, err := ListEntries(top, directory.Path, &snapshot.Files, patterns, nobackupFile, snapshot.discardAttributes) + subdirectories, skipped, err := ListEntries(top, directory.Path, &snapshot.Files, patterns, nobackupFile, snapshot.discardAttributes, excludeByAttribute) if err != nil { LOG_WARN("LIST_FAILURE", "Failed to list subdirectory: %v", err) skippedDirectories = append(skippedDirectories, directory.Path) diff --git a/src/duplicacy_snapshotmanager.go b/src/duplicacy_snapshotmanager.go index b40c385..138ddcb 100644 --- a/src/duplicacy_snapshotmanager.go +++ b/src/duplicacy_snapshotmanager.go @@ -1262,7 +1262,7 @@ func (manager *SnapshotManager) PrintFile(snapshotID string, revision int, path // Diff compares two snapshots, or two revision of a file if the file argument is given. func (manager *SnapshotManager) Diff(top string, snapshotID string, revisions []int, - filePath string, compareByHash bool, nobackupFile string) bool { + filePath string, compareByHash bool, nobackupFile string, excludeByAttribute bool) bool { LOG_DEBUG("DIFF_PARAMETERS", "top: %s, id: %s, revision: %v, path: %s, compareByHash: %t", top, snapshotID, revisions, filePath, compareByHash) @@ -1275,7 +1275,7 @@ func (manager *SnapshotManager) Diff(top string, snapshotID string, revisions [] if len(revisions) <= 1 { // Only scan the repository if filePath is not provided if len(filePath) == 0 { - rightSnapshot, _, _, err = CreateSnapshotFromDirectory(snapshotID, top, nobackupFile) + rightSnapshot, _, _, err = CreateSnapshotFromDirectory(snapshotID, top, nobackupFile, excludeByAttribute) if err != nil { LOG_ERROR("SNAPSHOT_LIST", "Failed to list the directory %s: %v", top, err) return false