From c88e148d59173d11c8608112aa3fc0d1a1233213 Mon Sep 17 00:00:00 2001 From: Etienne Charlier Date: Mon, 5 Jun 2017 23:16:11 +0200 Subject: [PATCH 1/6] First steps -pref-dir --- .gitignore | 3 ++ duplicacy_backupmanager.go | 11 +++-- duplicacy_entry.go | 1 + duplicacy_preference.go | 53 ++++++++++++++++++++-- duplicacy_shadowcopy_windows.go | 7 ++- duplicacy_snapshot.go | 4 +- duplicacy_snapshotmanager.go | 5 ++- duplicacy_storage.go | 2 +- integration_tests/test.sh | 80 +++++++++++++++++++++++++++++++++ main/duplicacy_main.go | 59 +++++++++++++++++------- 10 files changed, 194 insertions(+), 31 deletions(-) create mode 100644 .gitignore create mode 100755 integration_tests/test.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..648c6fb --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +.idea +duplicacy_main + diff --git a/duplicacy_backupmanager.go b/duplicacy_backupmanager.go index 1f27205..d7ce67d 100644 --- a/duplicacy_backupmanager.go +++ b/duplicacy_backupmanager.go @@ -71,8 +71,9 @@ func CreateBackupManager(snapshotID string, storage Storage, top string, passwor // SetupSnapshotCache creates the snapshot cache, which is merely a local storage under the default .duplicacy // directory func (manager *BackupManager) SetupSnapshotCache(top string, storageName string) bool { - - cacheDir := path.Join(top, DUPLICACY_DIRECTORY, "cache", storageName) + + duplicacyDirectory := GetDotDuplicacyPathName(top) + cacheDir := path.Join(duplicacyDirectory, "cache", storageName) storage, err := CreateFileStorage(cacheDir, 1) if err != nil { @@ -600,6 +601,7 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu } } + // How will behave restore when repo created using -repo-dir ,?? err = os.Mkdir(path.Join(top, DUPLICACY_DIRECTORY), 0744) if err != nil && !os.IsExist(err) { LOG_ERROR("RESTORE_MKDIR", "Failed to create the preference directory: %v", err) @@ -978,8 +980,9 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun var existingFile, newFile *os.File var err error - - temporaryPath := path.Join(top, DUPLICACY_DIRECTORY, "temporary") + + duplicacyDirectory := GetDotDuplicacyPathName(top) + temporaryPath := path.Join(duplicacyDirectory, "temporary") fullPath := joinPath(top, entry.Path) defer func() { diff --git a/duplicacy_entry.go b/duplicacy_entry.go index fe0ceee..9237aae 100644 --- a/duplicacy_entry.go +++ b/duplicacy_entry.go @@ -22,6 +22,7 @@ import ( // This is the hidden directory in the repository for storing various files. var DUPLICACY_DIRECTORY = ".duplicacy" +var DUPLICACY_FILE = ".duplicacy" // Regex for matching 'StartChunk:StartOffset:EndChunk:EndOffset' var contentRegex = regexp.MustCompile(`^([0-9]+):([0-9]+):([0-9]+):([0-9]+)`) diff --git a/duplicacy_preference.go b/duplicacy_preference.go index 139e80e..6934bde 100644 --- a/duplicacy_preference.go +++ b/duplicacy_preference.go @@ -9,6 +9,7 @@ import ( "path" "io/ioutil" "reflect" + "os" ) // Preference stores options for each storage. @@ -25,9 +26,52 @@ type Preference struct { var Preferences [] Preference -func LoadPreferences(repository string) (bool) { +// Compute .duplicacy directory path name: +// - if .duplicacy is a directory -> compute absolute path name and return it +// - if .duplicacy is a file -> assumed this file contains the real path name of .duplicacy +// - if pointed directory does not exits... return error +func GetDotDuplicacyPathName( repository string) (duplicacyDirectory string){ + + dotDuplicacy := path.Join(repository, DUPLICACY_DIRECTORY) //TOKEEP + + stat, err := os.Stat(dotDuplicacy) + if err != nil && !os.IsNotExist(err) { + LOG_ERROR("DOT_DUPLICACY_PATH", "Failed to retrieve the information about the directory %s: %v", + repository, err) + return "" + } + + if stat != nil && stat.IsDir() { + // $repository/.duplicacy exists and is a directory --> we found the .duplicacy directory + return path.Clean(dotDuplicacy) + } + + if stat != nil && stat.Mode().IsRegular() { + b, err := ioutil.ReadFile(dotDuplicacy) // just pass the file name + if err != nil { + LOG_ERROR("DOT_DUPLICACY_PATH", "Failed to read file %s: %v", + dotDuplicacy, err) + return "" + } + dot_duplicacy := string(b) // convert content to a 'string' + stat, err := os.Stat(dot_duplicacy) + if err != nil && !os.IsNotExist(err) { + LOG_ERROR("DOT_DUPLICACY_PATH", "Failed to retrieve the information about the directory %s: %v", + repository, err) + return "" + } + if stat != nil && stat.IsDir() { + // If expression read from .duplicacy file is a directory --> we found the .duplicacy directory + return path.Clean( dot_duplicacy) + } + } + return "" +} - description, err := ioutil.ReadFile(path.Join(repository, DUPLICACY_DIRECTORY, "preferences")) +func LoadPreferences(repository string) (bool) { + + duplicacyDirectory := GetDotDuplicacyPathName(repository) + description, err := ioutil.ReadFile(path.Join(duplicacyDirectory, "preferences")) if err != nil { LOG_ERROR("PREFERENCE_OPEN", "Failed to read the preference file from repository %s: %v", repository, err) return false @@ -53,8 +97,9 @@ func SavePreferences(repository string) (bool) { LOG_ERROR("PREFERENCE_MARSHAL", "Failed to marshal the repository preferences: %v", err) return false } - - preferenceFile := path.Join(repository, DUPLICACY_DIRECTORY, "/preferences") + duplicacyDirectory := GetDotDuplicacyPathName(repository) + preferenceFile := path.Join(duplicacyDirectory, "/preferences") + err = ioutil.WriteFile(preferenceFile, description, 0644) if err != nil { LOG_ERROR("PREFERENCE_WRITE", "Failed to save the preference file %s: %v", preferenceFile, err) diff --git a/duplicacy_shadowcopy_windows.go b/duplicacy_shadowcopy_windows.go index 1f1749c..03d9f21 100644 --- a/duplicacy_shadowcopy_windows.go +++ b/duplicacy_shadowcopy_windows.go @@ -509,8 +509,11 @@ func CreateShadowCopy(top string, shadowCopy bool) (shadowTop string) { LOG_INFO("VSS_DONE", "Shadow copy %s created", SnapshotIDString) snapshotPath := uint16ArrayToString(properties.SnapshotDeviceObject) - - shadowLink = path.Join(top, DUPLICACY_DIRECTORY) + "\\shadow" + + duplicacyDirectory := GetDotDuplicacyPathName(top) + shadowLink = path.Join(duplicacyDirectory, "shadow") + // FIXME: Not using path.Join : is this intentional ? + //shadowLink = path.Join(top, DUPLICACY_DIRECTORY) + "\\shadow" os.Remove(shadowLink) err = os.Symlink(snapshotPath + "\\", shadowLink) if err != nil { diff --git a/duplicacy_snapshot.go b/duplicacy_snapshot.go index 75699ed..b1b00ee 100644 --- a/duplicacy_snapshot.go +++ b/duplicacy_snapshot.go @@ -67,7 +67,9 @@ func CreateSnapshotFromDirectory(id string, top string) (snapshot *Snapshot, ski } var patterns []string - patternFile, err := ioutil.ReadFile(path.Join(top, DUPLICACY_DIRECTORY, "filters")) + + duplicacyDirectory := GetDotDuplicacyPathName(top) + patternFile, err := ioutil.ReadFile(path.Join(duplicacyDirectory, "filters")) if err == nil { for _, pattern := range strings.Split(string(patternFile), "\n") { pattern = strings.TrimSpace(pattern) diff --git a/duplicacy_snapshotmanager.go b/duplicacy_snapshotmanager.go index 1091a0e..592bdc7 100644 --- a/duplicacy_snapshotmanager.go +++ b/duplicacy_snapshotmanager.go @@ -1510,8 +1510,9 @@ func (manager *SnapshotManager) PruneSnapshots(top string, selfID string, snapsh if len(revisionsToBeDeleted) > 0 && (len(tags) > 0 || len(retentions) > 0) { LOG_WARN("DELETE_OPTIONS", "Tags or retention policy will be ignored if at least one revision is specified") } - - logDir := path.Join(top, DUPLICACY_DIRECTORY, "logs") + + duplicacyDirectory := GetDotDuplicacyPathName(top) + logDir := path.Join(duplicacyDirectory, "logs") os.Mkdir(logDir, 0700) logFileName := path.Join(logDir, time.Now().Format("prune-log-20060102-150405")) logFile, err := os.OpenFile(logFileName, os.O_WRONLY | os.O_CREATE | os.O_TRUNC, 0600) diff --git a/duplicacy_storage.go b/duplicacy_storage.go index 1ea632a..006839f 100644 --- a/duplicacy_storage.go +++ b/duplicacy_storage.go @@ -81,7 +81,7 @@ func checkHostKey(repository string, hostname string, remote net.Addr, key ssh.P return nil } - duplicacyDirectory := path.Join(repository, DUPLICACY_DIRECTORY) + duplicacyDirectory := GetDotDuplicacyPathName(repository) hostFile := path.Join(duplicacyDirectory, "knowns_hosts") file, err := os.OpenFile(hostFile, os.O_RDWR | os.O_CREATE, 0600) if err != nil { diff --git a/integration_tests/test.sh b/integration_tests/test.sh new file mode 100755 index 0000000..de945b2 --- /dev/null +++ b/integration_tests/test.sh @@ -0,0 +1,80 @@ +#!/bin/bash +set -x +get_abs_filename() { + # $1 : relative filename + echo "$(cd "$(dirname "$1")" && pwd)/$(basename "$1")" +} +DUPLICACY=$(get_abs_filename ../duplicacy_main) + +TEST_ZONE=$HOME/DUPLICACY_TEST_ZONE +TEST_REPO=$TEST_ZONE/TEST_REPO +TEST_STORAGE=$TEST_ZONE/TEST_STORAGE +TEST_DOT_DUPLICACY=$TEST_ZONE/TEST_DOT_DUPLICACY +TEST_RESTORE_POINT=$TEST_ZONE/RESTORE_POINT +function fixture() +{ + # clean TEST_RESTORE_POINT + rm -rf $TEST_RESTORE_POINT + mkdir -p $TEST_RESTORE_POINT + + # clean TEST_STORAGE + rm -rf $TEST_STORAGE + mkdir -p $TEST_STORAGE + + # clean TEST_DOT_DUPLICACY + rm -rf $TEST_DOT_DUPLICACY + mkdir -p $TEST_DOT_DUPLICACY + + # Create test repository + rm -rf $TEST_REPO + mkdir -p $TEST_REPO + pushd $TEST_REPO + echo "file1" > file1 + mkdir dir1 + echo "file2 >dir1/file2" + popd +} + +function init_repo() +{ + pushd $TEST_REPO + $DUPLICACY init integration-tests $TEST_STORAGE + $DUPLICACY backup + popd + +} + +function init_repo_pref_dir() +{ + pushd $TEST_REPO + $DUPLICACY init -pref-dir "$TEST_ZONE/TEST_DOT_DUPLICACY" integration-tests $TEST_STORAGE + $DUPLICACY backup + popd + +} + +function backup() +{ + pushd $TEST_REPO + NOW=`date` + echo $NOW > "file-$NOW" + $DUPLICACY backup + popd +} + + +function restore() +{ + pushd $TEST_REPO + $DUPLICACY restore -r 2 -delete + popd +} + +fixture +init_repo_pref_dir + +backup +backup +backup +restore + diff --git a/main/duplicacy_main.go b/main/duplicacy_main.go index cc87143..8be84f9 100644 --- a/main/duplicacy_main.go +++ b/main/duplicacy_main.go @@ -18,6 +18,8 @@ import ( "github.com/gilbertchen/cli" "github.com/gilbertchen/duplicacy" + + "io/ioutil" ) const ( @@ -26,6 +28,7 @@ const ( var ScriptEnabled bool + func getRepositoryPreference(context *cli.Context, storageName string) (repository string, preference *duplicacy.Preference) { @@ -34,16 +37,15 @@ func getRepositoryPreference(context *cli.Context, storageName string) (reposito duplicacy.LOG_ERROR("REPOSITORY_PATH", "Failed to retrieve the current working directory: %v", err) return "", nil } - for { - stat, err := os.Stat(path.Join(repository, duplicacy.DUPLICACY_DIRECTORY)) + stat, err := os.Stat(path.Join(repository, duplicacy.DUPLICACY_DIRECTORY)) //TOKEEP if err != nil && !os.IsNotExist(err) { duplicacy.LOG_ERROR("REPOSITORY_PATH", "Failed to retrieve the information about the directory %s: %v", repository, err) return "", nil } - if stat != nil && stat.IsDir() { + if stat != nil && (stat.IsDir() || stat.Mode().IsRegular()) { break } @@ -54,10 +56,10 @@ func getRepositoryPreference(context *cli.Context, storageName string) (reposito } repository = parent } - duplicacy.LoadPreferences(repository) - - duplicacy.SetKeyringFile(path.Join(repository, duplicacy.DUPLICACY_DIRECTORY, "keyring")) + + duplicacyDirectory := duplicacy.GetDotDuplicacyPathName(repository) + duplicacy.SetKeyringFile(path.Join(duplicacyDirectory, "keyring")) if storageName == "" { storageName = context.String("storage") @@ -142,8 +144,9 @@ func runScript(context *cli.Context, repository string, storageName string, phas if !ScriptEnabled { return false } - - scriptDir, _ := filepath.Abs(path.Join(repository, duplicacy.DUPLICACY_DIRECTORY, "scripts")) + + duplicacyDirectory := duplicacy.GetDotDuplicacyPathName(repository) + scriptDir, _ := filepath.Abs(path.Join(duplicacyDirectory, "scripts")) scriptName := phase + "-" + context.Command.Name script := path.Join(scriptDir, scriptName) @@ -174,14 +177,14 @@ func runScript(context *cli.Context, repository string, storageName string, phas } func initRepository(context *cli.Context) { - configRespository(context, true) + configRepository(context, true) } func addStorage(context *cli.Context) { - configRespository(context, false) + configRepository(context, false) } -func configRespository(context *cli.Context, init bool) { +func configRepository(context *cli.Context, init bool) { setGlobalOptions(context) defer duplicacy.CatchLogException() @@ -220,8 +223,15 @@ func configRespository(context *cli.Context, init bool) { duplicacy.LOG_ERROR("REPOSITORY_PATH", "Failed to retrieve the current working directory: %v", err) return } - - duplicacyDirectory := path.Join(repository, duplicacy.DUPLICACY_DIRECTORY) + + duplicacyDirectory := context.String("pref-dir") + if duplicacyDirectory == "" { + + duplicacyDirectory = path.Join(repository, duplicacy.DUPLICACY_DIRECTORY) // TOKEEP + } + duplicacy.LOG_INFO("PREF_PATH", "-pref-dir value: --|%s|-- ", duplicacyDirectory) + + if stat, _ := os.Stat(path.Join(duplicacyDirectory, "preferences")); stat != nil { duplicacy.LOG_ERROR("REPOSITORY_INIT", "The repository %s has already been initialized", repository) return @@ -230,10 +240,20 @@ func configRespository(context *cli.Context, init bool) { err = os.Mkdir(duplicacyDirectory, 0744) if err != nil && !os.IsExist(err) { duplicacy.LOG_ERROR("REPOSITORY_INIT", "Failed to create the directory %s: %v", - duplicacy.DUPLICACY_DIRECTORY, err) + duplicacyDirectory, err) return } - + if context.String("pref-dir") != "" { + // out of tree preference file + // write real path into .duplicacy file inside repository + duplicacyFileName := path.Join(repository, duplicacy.DUPLICACY_FILE) + d1 := []byte(duplicacyDirectory) + err = ioutil.WriteFile(duplicacyFileName, d1, 0644) + if err != nil { + duplicacy.LOG_ERROR("REPOSITORY_PATH", "Failed to write %s file inside repository %v", duplicacyFileName, err) + return + } + } duplicacy.SetKeyringFile(path.Join(duplicacyDirectory, "keyring")) } else { @@ -547,7 +567,6 @@ func changePassword(context *cli.Context) { duplicacy.LOG_INFO("STORAGE_SET", "The password for storage %s has been changed", preference.StorageURL) } - func backupRepository(context *cli.Context) { setGlobalOptions(context) defer duplicacy.CatchLogException() @@ -1071,7 +1090,8 @@ func infoStorage(context *cli.Context) { repository := context.String("repository") if repository != "" { - duplicacy.SetKeyringFile(path.Join(repository, duplicacy.DUPLICACY_DIRECTORY, "keyring")) + duplicacyDirectory := duplicacy.GetDotDuplicacyPathName(repository) + duplicacy.SetKeyringFile(path.Join(duplicacyDirectory, "keyring")) } isEncrypted := context.Bool("e") @@ -1132,6 +1152,11 @@ func main() { Usage: "the minimum size of chunks (defaults to chunk-size / 4)", Argument: "1M", }, + cli.StringFlag{ + Name: "pref-dir", + Usage: "Specify alternate location for .duplicacy preferences directory (absolute or relative to current directory)", + Argument: "", + }, }, Usage: "Initialize the storage if necessary and the current directory as the repository", ArgsUsage: " ", From c688c501d339edd08351aa54cdb12af699f49286 Mon Sep 17 00:00:00 2001 From: Etienne Charlier Date: Wed, 7 Jun 2017 21:01:50 +0200 Subject: [PATCH 2/6] Refactor variable names and revert shadow copy path computation --- duplicacy_backupmanager.go | 8 ++++---- duplicacy_preference.go | 26 +++++++++++++------------- duplicacy_shadowcopy_windows.go | 6 ++---- duplicacy_snapshot.go | 4 ++-- duplicacy_snapshotmanager.go | 4 ++-- duplicacy_storage.go | 6 +++--- main/duplicacy_main.go | 30 +++++++++++++++--------------- 7 files changed, 41 insertions(+), 43 deletions(-) diff --git a/duplicacy_backupmanager.go b/duplicacy_backupmanager.go index d7ce67d..2fd3474 100644 --- a/duplicacy_backupmanager.go +++ b/duplicacy_backupmanager.go @@ -72,8 +72,8 @@ func CreateBackupManager(snapshotID string, storage Storage, top string, passwor // directory func (manager *BackupManager) SetupSnapshotCache(top string, storageName string) bool { - duplicacyDirectory := GetDotDuplicacyPathName(top) - cacheDir := path.Join(duplicacyDirectory, "cache", storageName) + preferencePath := GetDuplicacyPreferencePath(top) + cacheDir := path.Join(preferencePath, "cache", storageName) storage, err := CreateFileStorage(cacheDir, 1) if err != nil { @@ -981,8 +981,8 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun var existingFile, newFile *os.File var err error - duplicacyDirectory := GetDotDuplicacyPathName(top) - temporaryPath := path.Join(duplicacyDirectory, "temporary") + preferencePath := GetDuplicacyPreferencePath(top) + temporaryPath := path.Join(preferencePath, "temporary") fullPath := joinPath(top, entry.Path) defer func() { diff --git a/duplicacy_preference.go b/duplicacy_preference.go index 6934bde..fe05f95 100644 --- a/duplicacy_preference.go +++ b/duplicacy_preference.go @@ -30,11 +30,11 @@ var Preferences [] Preference // - if .duplicacy is a directory -> compute absolute path name and return it // - if .duplicacy is a file -> assumed this file contains the real path name of .duplicacy // - if pointed directory does not exits... return error -func GetDotDuplicacyPathName( repository string) (duplicacyDirectory string){ +func GetDuplicacyPreferencePath( repository string) (preferencePath string){ - dotDuplicacy := path.Join(repository, DUPLICACY_DIRECTORY) //TOKEEP + preferencePath = path.Join(repository, DUPLICACY_DIRECTORY) //TOKEEP - stat, err := os.Stat(dotDuplicacy) + stat, err := os.Stat(preferencePath) if err != nil && !os.IsNotExist(err) { LOG_ERROR("DOT_DUPLICACY_PATH", "Failed to retrieve the information about the directory %s: %v", repository, err) @@ -43,18 +43,18 @@ func GetDotDuplicacyPathName( repository string) (duplicacyDirectory string){ if stat != nil && stat.IsDir() { // $repository/.duplicacy exists and is a directory --> we found the .duplicacy directory - return path.Clean(dotDuplicacy) + return path.Clean(preferencePath) } if stat != nil && stat.Mode().IsRegular() { - b, err := ioutil.ReadFile(dotDuplicacy) // just pass the file name + b, err := ioutil.ReadFile(preferencePath) // just pass the file name if err != nil { LOG_ERROR("DOT_DUPLICACY_PATH", "Failed to read file %s: %v", - dotDuplicacy, err) + preferencePath, err) return "" } - dot_duplicacy := string(b) // convert content to a 'string' - stat, err := os.Stat(dot_duplicacy) + dotDuplicacyContent := string(b) // convert content to a 'string' + stat, err := os.Stat(dotDuplicacyContent) if err != nil && !os.IsNotExist(err) { LOG_ERROR("DOT_DUPLICACY_PATH", "Failed to retrieve the information about the directory %s: %v", repository, err) @@ -62,7 +62,7 @@ func GetDotDuplicacyPathName( repository string) (duplicacyDirectory string){ } if stat != nil && stat.IsDir() { // If expression read from .duplicacy file is a directory --> we found the .duplicacy directory - return path.Clean( dot_duplicacy) + return path.Clean(dotDuplicacyContent) } } return "" @@ -70,8 +70,8 @@ func GetDotDuplicacyPathName( repository string) (duplicacyDirectory string){ func LoadPreferences(repository string) (bool) { - duplicacyDirectory := GetDotDuplicacyPathName(repository) - description, err := ioutil.ReadFile(path.Join(duplicacyDirectory, "preferences")) + preferencePath := GetDuplicacyPreferencePath(repository) + description, err := ioutil.ReadFile(path.Join(preferencePath, "preferences")) if err != nil { LOG_ERROR("PREFERENCE_OPEN", "Failed to read the preference file from repository %s: %v", repository, err) return false @@ -97,8 +97,8 @@ func SavePreferences(repository string) (bool) { LOG_ERROR("PREFERENCE_MARSHAL", "Failed to marshal the repository preferences: %v", err) return false } - duplicacyDirectory := GetDotDuplicacyPathName(repository) - preferenceFile := path.Join(duplicacyDirectory, "/preferences") + preferencePath := GetDuplicacyPreferencePath(repository) + preferenceFile := path.Join(preferencePath, "/preferences") err = ioutil.WriteFile(preferenceFile, description, 0644) if err != nil { diff --git a/duplicacy_shadowcopy_windows.go b/duplicacy_shadowcopy_windows.go index 03d9f21..12f8f57 100644 --- a/duplicacy_shadowcopy_windows.go +++ b/duplicacy_shadowcopy_windows.go @@ -510,10 +510,8 @@ func CreateShadowCopy(top string, shadowCopy bool) (shadowTop string) { snapshotPath := uint16ArrayToString(properties.SnapshotDeviceObject) - duplicacyDirectory := GetDotDuplicacyPathName(top) - shadowLink = path.Join(duplicacyDirectory, "shadow") - // FIXME: Not using path.Join : is this intentional ? - //shadowLink = path.Join(top, DUPLICACY_DIRECTORY) + "\\shadow" + preferencePath := GetDuplicacyPreferencePath(top) + shadowLink = preferencePath + "\\shadow" os.Remove(shadowLink) err = os.Symlink(snapshotPath + "\\", shadowLink) if err != nil { diff --git a/duplicacy_snapshot.go b/duplicacy_snapshot.go index b1b00ee..69938b7 100644 --- a/duplicacy_snapshot.go +++ b/duplicacy_snapshot.go @@ -68,8 +68,8 @@ func CreateSnapshotFromDirectory(id string, top string) (snapshot *Snapshot, ski var patterns []string - duplicacyDirectory := GetDotDuplicacyPathName(top) - patternFile, err := ioutil.ReadFile(path.Join(duplicacyDirectory, "filters")) + preferencePath := GetDuplicacyPreferencePath(top) + patternFile, err := ioutil.ReadFile(path.Join(preferencePath, "filters")) if err == nil { for _, pattern := range strings.Split(string(patternFile), "\n") { pattern = strings.TrimSpace(pattern) diff --git a/duplicacy_snapshotmanager.go b/duplicacy_snapshotmanager.go index 592bdc7..41d3e53 100644 --- a/duplicacy_snapshotmanager.go +++ b/duplicacy_snapshotmanager.go @@ -1511,8 +1511,8 @@ func (manager *SnapshotManager) PruneSnapshots(top string, selfID string, snapsh LOG_WARN("DELETE_OPTIONS", "Tags or retention policy will be ignored if at least one revision is specified") } - duplicacyDirectory := GetDotDuplicacyPathName(top) - logDir := path.Join(duplicacyDirectory, "logs") + preferencePath := GetDuplicacyPreferencePath(top) + logDir := path.Join(preferencePath, "logs") os.Mkdir(logDir, 0700) logFileName := path.Join(logDir, time.Now().Format("prune-log-20060102-150405")) logFile, err := os.OpenFile(logFileName, os.O_WRONLY | os.O_CREATE | os.O_TRUNC, 0600) diff --git a/duplicacy_storage.go b/duplicacy_storage.go index 006839f..1c415e2 100644 --- a/duplicacy_storage.go +++ b/duplicacy_storage.go @@ -80,9 +80,9 @@ func checkHostKey(repository string, hostname string, remote net.Addr, key ssh.P if len(repository) == 0 { return nil } - - duplicacyDirectory := GetDotDuplicacyPathName(repository) - hostFile := path.Join(duplicacyDirectory, "knowns_hosts") + + preferencePath := GetDuplicacyPreferencePath(repository) + hostFile := path.Join(preferencePath, "knowns_hosts") file, err := os.OpenFile(hostFile, os.O_RDWR | os.O_CREATE, 0600) if err != nil { return err diff --git a/main/duplicacy_main.go b/main/duplicacy_main.go index 8be84f9..8b005b8 100644 --- a/main/duplicacy_main.go +++ b/main/duplicacy_main.go @@ -58,8 +58,8 @@ func getRepositoryPreference(context *cli.Context, storageName string) (reposito } duplicacy.LoadPreferences(repository) - duplicacyDirectory := duplicacy.GetDotDuplicacyPathName(repository) - duplicacy.SetKeyringFile(path.Join(duplicacyDirectory, "keyring")) + preferencePath := duplicacy.GetDuplicacyPreferencePath(repository) + duplicacy.SetKeyringFile(path.Join(preferencePath, "keyring")) if storageName == "" { storageName = context.String("storage") @@ -145,8 +145,8 @@ func runScript(context *cli.Context, repository string, storageName string, phas return false } - duplicacyDirectory := duplicacy.GetDotDuplicacyPathName(repository) - scriptDir, _ := filepath.Abs(path.Join(duplicacyDirectory, "scripts")) + preferencePath := duplicacy.GetDuplicacyPreferencePath(repository) + scriptDir, _ := filepath.Abs(path.Join(preferencePath, "scripts")) scriptName := phase + "-" + context.Command.Name script := path.Join(scriptDir, scriptName) @@ -224,37 +224,37 @@ func configRepository(context *cli.Context, init bool) { return } - duplicacyDirectory := context.String("pref-dir") - if duplicacyDirectory == "" { + preferencePath := context.String("pref-dir") + if preferencePath == "" { - duplicacyDirectory = path.Join(repository, duplicacy.DUPLICACY_DIRECTORY) // TOKEEP + preferencePath = path.Join(repository, duplicacy.DUPLICACY_DIRECTORY) // TOKEEP } - duplicacy.LOG_INFO("PREF_PATH", "-pref-dir value: --|%s|-- ", duplicacyDirectory) + duplicacy.LOG_INFO("PREF_PATH", "-pref-dir value: --|%s|-- ", preferencePath) - if stat, _ := os.Stat(path.Join(duplicacyDirectory, "preferences")); stat != nil { + if stat, _ := os.Stat(path.Join(preferencePath, "preferences")); stat != nil { duplicacy.LOG_ERROR("REPOSITORY_INIT", "The repository %s has already been initialized", repository) return } - err = os.Mkdir(duplicacyDirectory, 0744) + err = os.Mkdir(preferencePath, 0744) if err != nil && !os.IsExist(err) { duplicacy.LOG_ERROR("REPOSITORY_INIT", "Failed to create the directory %s: %v", - duplicacyDirectory, err) + preferencePath, err) return } if context.String("pref-dir") != "" { // out of tree preference file // write real path into .duplicacy file inside repository duplicacyFileName := path.Join(repository, duplicacy.DUPLICACY_FILE) - d1 := []byte(duplicacyDirectory) + d1 := []byte(preferencePath) err = ioutil.WriteFile(duplicacyFileName, d1, 0644) if err != nil { duplicacy.LOG_ERROR("REPOSITORY_PATH", "Failed to write %s file inside repository %v", duplicacyFileName, err) return } } - duplicacy.SetKeyringFile(path.Join(duplicacyDirectory, "keyring")) + duplicacy.SetKeyringFile(path.Join(preferencePath, "keyring")) } else { repository, _ = getRepositoryPreference(context, "") @@ -1090,8 +1090,8 @@ func infoStorage(context *cli.Context) { repository := context.String("repository") if repository != "" { - duplicacyDirectory := duplicacy.GetDotDuplicacyPathName(repository) - duplicacy.SetKeyringFile(path.Join(duplicacyDirectory, "keyring")) + preferencePath := duplicacy.GetDuplicacyPreferencePath(repository) + duplicacy.SetKeyringFile(path.Join(preferencePath, "keyring")) } isEncrypted := context.Bool("e") From 1361b553ac561bf7435c7405882eda53321e9457 Mon Sep 17 00:00:00 2001 From: Etienne Charlier Date: Thu, 8 Jun 2017 22:21:57 +0200 Subject: [PATCH 3/6] Remove logging statement; refactor test scripts --- integration_tests/test.sh | 73 ++------------------ integration_tests/test_functions.sh | 103 ++++++++++++++++++++++++++++ main/duplicacy_main.go | 1 - 3 files changed, 108 insertions(+), 69 deletions(-) create mode 100644 integration_tests/test_functions.sh diff --git a/integration_tests/test.sh b/integration_tests/test.sh index de945b2..4cfee56 100755 --- a/integration_tests/test.sh +++ b/integration_tests/test.sh @@ -1,80 +1,17 @@ #!/bin/bash -set -x -get_abs_filename() { - # $1 : relative filename - echo "$(cd "$(dirname "$1")" && pwd)/$(basename "$1")" -} -DUPLICACY=$(get_abs_filename ../duplicacy_main) - -TEST_ZONE=$HOME/DUPLICACY_TEST_ZONE -TEST_REPO=$TEST_ZONE/TEST_REPO -TEST_STORAGE=$TEST_ZONE/TEST_STORAGE -TEST_DOT_DUPLICACY=$TEST_ZONE/TEST_DOT_DUPLICACY -TEST_RESTORE_POINT=$TEST_ZONE/RESTORE_POINT -function fixture() -{ - # clean TEST_RESTORE_POINT - rm -rf $TEST_RESTORE_POINT - mkdir -p $TEST_RESTORE_POINT - - # clean TEST_STORAGE - rm -rf $TEST_STORAGE - mkdir -p $TEST_STORAGE - - # clean TEST_DOT_DUPLICACY - rm -rf $TEST_DOT_DUPLICACY - mkdir -p $TEST_DOT_DUPLICACY - - # Create test repository - rm -rf $TEST_REPO - mkdir -p $TEST_REPO - pushd $TEST_REPO - echo "file1" > file1 - mkdir dir1 - echo "file2 >dir1/file2" - popd -} - -function init_repo() -{ - pushd $TEST_REPO - $DUPLICACY init integration-tests $TEST_STORAGE - $DUPLICACY backup - popd - -} - -function init_repo_pref_dir() -{ - pushd $TEST_REPO - $DUPLICACY init -pref-dir "$TEST_ZONE/TEST_DOT_DUPLICACY" integration-tests $TEST_STORAGE - $DUPLICACY backup - popd - -} - -function backup() -{ - pushd $TEST_REPO - NOW=`date` - echo $NOW > "file-$NOW" - $DUPLICACY backup - popd -} -function restore() -{ - pushd $TEST_REPO - $DUPLICACY restore -r 2 -delete - popd -} +. ./test_functions.sh fixture init_repo_pref_dir backup +add_file file3 backup +add_file file4 backup +add_file file5 restore + diff --git a/integration_tests/test_functions.sh b/integration_tests/test_functions.sh new file mode 100644 index 0000000..0739411 --- /dev/null +++ b/integration_tests/test_functions.sh @@ -0,0 +1,103 @@ +#!/bin/bash + +get_abs_filename() { + # $1 : relative filename + echo "$(cd "$(dirname "$1")" && pwd)/$(basename "$1")" +} + +pushd () { + command pushd "$@" > /dev/null +} + +popd () { + command popd "$@" > /dev/null +} + + +# Functions used to create integration tests suite + +DUPLICACY=$(get_abs_filename ../duplicacy_main) + +# Base directory where test repositories will be created +TEST_ZONE=$HOME/DUPLICACY_TEST_ZONE +# Test Repository +TEST_REPO=$TEST_ZONE/TEST_REPO + +# Storage for test ( For now, only local path storage is supported by test suite) +TEST_STORAGE=$TEST_ZONE/TEST_STORAGE + +# Preference directory ( for testing the -pref-dir option) +DUPLICACY_PREF_DIR=$TEST_ZONE/TEST_DUPLICACY_PREF_DIR + +# Scratch pad for testing restore +TEST_RESTORE_POINT=$TEST_ZONE/RESTORE_POINT + +# Make sure $TEST_ZONE is in know state + + + +function fixture() +{ + # clean TEST_RESTORE_POINT + rm -rf $TEST_RESTORE_POINT + mkdir -p $TEST_RESTORE_POINT + + # clean TEST_STORAGE + rm -rf $TEST_STORAGE + mkdir -p $TEST_STORAGE + + # clean TEST_DOT_DUPLICACY + rm -rf $DUPLICACY_PREF_DIR + mkdir -p $DUPLICACY_PREF_DIR + + # Create test repository + rm -rf ${TEST_REPO} + mkdir -p ${TEST_REPO} + pushd ${TEST_REPO} + echo "file1" > file1 + mkdir dir1 + echo "file2" > dir1/file2 + popd +} + +function init_repo() +{ + pushd ${TEST_REPO} + ${DUPLICACY} init integration-tests $TEST_STORAGE + ${DUPLICACY} backup + popd + +} + +function init_repo_pref_dir() +{ + pushd ${TEST_REPO} + ${DUPLICACY} init -pref-dir "${DUPLICACY_PREF_DIR}" integration-tests ${TEST_STORAGE} + ${DUPLICACY} backup + popd + +} + +function add_file() +{ + FILE_NAME=$1 + pushd ${TEST_REPO} + echo ${FILE_NAME} > "${FILE_NAME}" + popd +} + + +function backup() +{ + pushd ${TEST_REPO} + ${DUPLICACY} backup + popd +} + + +function restore() +{ + pushd ${TEST_REPO} + ${DUPLICACY} restore -r 2 -delete + popd +} \ No newline at end of file diff --git a/main/duplicacy_main.go b/main/duplicacy_main.go index 8b005b8..fb7f430 100644 --- a/main/duplicacy_main.go +++ b/main/duplicacy_main.go @@ -229,7 +229,6 @@ func configRepository(context *cli.Context, init bool) { preferencePath = path.Join(repository, duplicacy.DUPLICACY_DIRECTORY) // TOKEEP } - duplicacy.LOG_INFO("PREF_PATH", "-pref-dir value: --|%s|-- ", preferencePath) if stat, _ := os.Stat(path.Join(preferencePath, "preferences")); stat != nil { From 9e1740c1d63246c9c99ecb0137a555f75aa013cb Mon Sep 17 00:00:00 2001 From: Etienne Charlier Date: Sat, 10 Jun 2017 17:14:58 +0200 Subject: [PATCH 4/6] Fix merge error --- main/duplicacy_main.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/main/duplicacy_main.go b/main/duplicacy_main.go index 936aa93..1fa7ba3 100644 --- a/main/duplicacy_main.go +++ b/main/duplicacy_main.go @@ -17,8 +17,7 @@ import ( "github.com/gilbertchen/cli" - "github.com/gilbertchen/duplicacy" - + "github.com/gilbertchen/duplicacy/src" "io/ioutil" ) From aa909c0c151502260e29e964850ac49470ed4ada Mon Sep 17 00:00:00 2001 From: Etienne Charlier Date: Sun, 11 Jun 2017 13:48:11 +0200 Subject: [PATCH 5/6] Update documentation --- GUIDE.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/GUIDE.md b/GUIDE.md index face86f..a847773 100644 --- a/GUIDE.md +++ b/GUIDE.md @@ -16,6 +16,7 @@ OPTIONS: -chunk-size, -c 4M the average size of chunks -max-chunk-size, -max 16M the maximum size of chunks (defaults to chunk-size * 4) -min-chunk-size, -min 1M the minimum size of chunks (defaults to chunk-size / 4) + -pref-dir Specify alternate location for .duplicacy preferences directory ``` The *init* command first connects to the storage specified by the storage URL. If the storage has been already been @@ -33,6 +34,8 @@ The -e option controls whether or not encryption will be enabled for the storage The three chunk size parameters are passed to the variable-size chunking algorithm. Their values are important to the overall performance, especially for cloud storages. If the chunk size is too small, a lot of overhead will be in sending requests and receiving responses. If the chunk size is too large, the effect of deduplication will be less obvious as more data will need to be transferred with each chunk. +The -pref-dir controls the location of the preferences directory. If not specified, a directory named .duplicacy is created in the repository. If specified, it must point to a non-existing directory. The directory is created and a .duplicacy file is created in the repository. The .duplicacy file contains the absolute path name to the preferences directory. + Once a storage has been initialized with these parameters, these parameters cannot be modified any more. #### Backup From 7aa0eca47c562fd6ca861c5e0922b3d5fd4a1ff7 Mon Sep 17 00:00:00 2001 From: Etienne Charlier Date: Sun, 11 Jun 2017 14:10:14 +0200 Subject: [PATCH 6/6] Fix typo --- main/duplicacy_main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main/duplicacy_main.go b/main/duplicacy_main.go index 1fa7ba3..6f6bfcb 100644 --- a/main/duplicacy_main.go +++ b/main/duplicacy_main.go @@ -1153,7 +1153,7 @@ func main() { cli.StringFlag{ Name: "pref-dir", Usage: "Specify alternate location for .duplicacy preferences directory (absolute or relative to current directory)", - Argument: "", + Argument: "", }, }, Usage: "Initialize the storage if necessary and the current directory as the repository",