diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..648c6fb --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +.idea +duplicacy_main + diff --git a/GUIDE.md b/GUIDE.md index face86f..a847773 100644 --- a/GUIDE.md +++ b/GUIDE.md @@ -16,6 +16,7 @@ OPTIONS: -chunk-size, -c 4M the average size of chunks -max-chunk-size, -max 16M the maximum size of chunks (defaults to chunk-size * 4) -min-chunk-size, -min 1M the minimum size of chunks (defaults to chunk-size / 4) + -pref-dir Specify alternate location for .duplicacy preferences directory ``` The *init* command first connects to the storage specified by the storage URL. If the storage has been already been @@ -33,6 +34,8 @@ The -e option controls whether or not encryption will be enabled for the storage The three chunk size parameters are passed to the variable-size chunking algorithm. Their values are important to the overall performance, especially for cloud storages. If the chunk size is too small, a lot of overhead will be in sending requests and receiving responses. If the chunk size is too large, the effect of deduplication will be less obvious as more data will need to be transferred with each chunk. +The -pref-dir controls the location of the preferences directory. If not specified, a directory named .duplicacy is created in the repository. If specified, it must point to a non-existing directory. The directory is created and a .duplicacy file is created in the repository. The .duplicacy file contains the absolute path name to the preferences directory. + Once a storage has been initialized with these parameters, these parameters cannot be modified any more. #### Backup diff --git a/duplicacy/duplicacy_main.go b/duplicacy/duplicacy_main.go index 1e85596..6f6bfcb 100644 --- a/duplicacy/duplicacy_main.go +++ b/duplicacy/duplicacy_main.go @@ -18,6 +18,7 @@ import ( "github.com/gilbertchen/cli" "github.com/gilbertchen/duplicacy/src" + "io/ioutil" ) const ( @@ -36,14 +37,14 @@ func getRepositoryPreference(context *cli.Context, storageName string) (reposito } for { - stat, err := os.Stat(path.Join(repository, duplicacy.DUPLICACY_DIRECTORY)) + stat, err := os.Stat(path.Join(repository, duplicacy.DUPLICACY_DIRECTORY)) //TOKEEP if err != nil && !os.IsNotExist(err) { duplicacy.LOG_ERROR("REPOSITORY_PATH", "Failed to retrieve the information about the directory %s: %v", repository, err) return "", nil } - if stat != nil && stat.IsDir() { + if stat != nil && (stat.IsDir() || stat.Mode().IsRegular()) { break } @@ -54,10 +55,10 @@ func getRepositoryPreference(context *cli.Context, storageName string) (reposito } repository = parent } - duplicacy.LoadPreferences(repository) - - duplicacy.SetKeyringFile(path.Join(repository, duplicacy.DUPLICACY_DIRECTORY, "keyring")) + + preferencePath := duplicacy.GetDuplicacyPreferencePath(repository) + duplicacy.SetKeyringFile(path.Join(preferencePath, "keyring")) if storageName == "" { storageName = context.String("storage") @@ -142,8 +143,9 @@ func runScript(context *cli.Context, repository string, storageName string, phas if !ScriptEnabled { return false } - - scriptDir, _ := filepath.Abs(path.Join(repository, duplicacy.DUPLICACY_DIRECTORY, "scripts")) + + preferencePath := duplicacy.GetDuplicacyPreferencePath(repository) + scriptDir, _ := filepath.Abs(path.Join(preferencePath, "scripts")) scriptName := phase + "-" + context.Command.Name script := path.Join(scriptDir, scriptName) @@ -174,14 +176,14 @@ func runScript(context *cli.Context, repository string, storageName string, phas } func initRepository(context *cli.Context) { - configRespository(context, true) + configRepository(context, true) } func addStorage(context *cli.Context) { - configRespository(context, false) + configRepository(context, false) } -func configRespository(context *cli.Context, init bool) { +func configRepository(context *cli.Context, init bool) { setGlobalOptions(context) defer duplicacy.CatchLogException() @@ -220,21 +222,37 @@ func configRespository(context *cli.Context, init bool) { duplicacy.LOG_ERROR("REPOSITORY_PATH", "Failed to retrieve the current working directory: %v", err) return } - - duplicacyDirectory := path.Join(repository, duplicacy.DUPLICACY_DIRECTORY) - if stat, _ := os.Stat(path.Join(duplicacyDirectory, "preferences")); stat != nil { + + preferencePath := context.String("pref-dir") + if preferencePath == "" { + + preferencePath = path.Join(repository, duplicacy.DUPLICACY_DIRECTORY) // TOKEEP + } + + + if stat, _ := os.Stat(path.Join(preferencePath, "preferences")); stat != nil { duplicacy.LOG_ERROR("REPOSITORY_INIT", "The repository %s has already been initialized", repository) return } - err = os.Mkdir(duplicacyDirectory, 0744) + err = os.Mkdir(preferencePath, 0744) if err != nil && !os.IsExist(err) { duplicacy.LOG_ERROR("REPOSITORY_INIT", "Failed to create the directory %s: %v", - duplicacy.DUPLICACY_DIRECTORY, err) + preferencePath, err) return } - - duplicacy.SetKeyringFile(path.Join(duplicacyDirectory, "keyring")) + if context.String("pref-dir") != "" { + // out of tree preference file + // write real path into .duplicacy file inside repository + duplicacyFileName := path.Join(repository, duplicacy.DUPLICACY_FILE) + d1 := []byte(preferencePath) + err = ioutil.WriteFile(duplicacyFileName, d1, 0644) + if err != nil { + duplicacy.LOG_ERROR("REPOSITORY_PATH", "Failed to write %s file inside repository %v", duplicacyFileName, err) + return + } + } + duplicacy.SetKeyringFile(path.Join(preferencePath, "keyring")) } else { repository, _ = getRepositoryPreference(context, "") @@ -547,7 +565,6 @@ func changePassword(context *cli.Context) { duplicacy.LOG_INFO("STORAGE_SET", "The password for storage %s has been changed", preference.StorageURL) } - func backupRepository(context *cli.Context) { setGlobalOptions(context) defer duplicacy.CatchLogException() @@ -1071,7 +1088,8 @@ func infoStorage(context *cli.Context) { repository := context.String("repository") if repository != "" { - duplicacy.SetKeyringFile(path.Join(repository, duplicacy.DUPLICACY_DIRECTORY, "keyring")) + preferencePath := duplicacy.GetDuplicacyPreferencePath(repository) + duplicacy.SetKeyringFile(path.Join(preferencePath, "keyring")) } isEncrypted := context.Bool("e") @@ -1132,6 +1150,11 @@ func main() { Usage: "the minimum size of chunks (defaults to chunk-size / 4)", Argument: "1M", }, + cli.StringFlag{ + Name: "pref-dir", + Usage: "Specify alternate location for .duplicacy preferences directory (absolute or relative to current directory)", + Argument: "", + }, }, Usage: "Initialize the storage if necessary and the current directory as the repository", ArgsUsage: " ", diff --git a/integration_tests/test.sh b/integration_tests/test.sh new file mode 100755 index 0000000..4cfee56 --- /dev/null +++ b/integration_tests/test.sh @@ -0,0 +1,17 @@ +#!/bin/bash + + +. ./test_functions.sh + +fixture +init_repo_pref_dir + +backup +add_file file3 +backup +add_file file4 +backup +add_file file5 +restore + + diff --git a/integration_tests/test_functions.sh b/integration_tests/test_functions.sh new file mode 100644 index 0000000..0739411 --- /dev/null +++ b/integration_tests/test_functions.sh @@ -0,0 +1,103 @@ +#!/bin/bash + +get_abs_filename() { + # $1 : relative filename + echo "$(cd "$(dirname "$1")" && pwd)/$(basename "$1")" +} + +pushd () { + command pushd "$@" > /dev/null +} + +popd () { + command popd "$@" > /dev/null +} + + +# Functions used to create integration tests suite + +DUPLICACY=$(get_abs_filename ../duplicacy_main) + +# Base directory where test repositories will be created +TEST_ZONE=$HOME/DUPLICACY_TEST_ZONE +# Test Repository +TEST_REPO=$TEST_ZONE/TEST_REPO + +# Storage for test ( For now, only local path storage is supported by test suite) +TEST_STORAGE=$TEST_ZONE/TEST_STORAGE + +# Preference directory ( for testing the -pref-dir option) +DUPLICACY_PREF_DIR=$TEST_ZONE/TEST_DUPLICACY_PREF_DIR + +# Scratch pad for testing restore +TEST_RESTORE_POINT=$TEST_ZONE/RESTORE_POINT + +# Make sure $TEST_ZONE is in know state + + + +function fixture() +{ + # clean TEST_RESTORE_POINT + rm -rf $TEST_RESTORE_POINT + mkdir -p $TEST_RESTORE_POINT + + # clean TEST_STORAGE + rm -rf $TEST_STORAGE + mkdir -p $TEST_STORAGE + + # clean TEST_DOT_DUPLICACY + rm -rf $DUPLICACY_PREF_DIR + mkdir -p $DUPLICACY_PREF_DIR + + # Create test repository + rm -rf ${TEST_REPO} + mkdir -p ${TEST_REPO} + pushd ${TEST_REPO} + echo "file1" > file1 + mkdir dir1 + echo "file2" > dir1/file2 + popd +} + +function init_repo() +{ + pushd ${TEST_REPO} + ${DUPLICACY} init integration-tests $TEST_STORAGE + ${DUPLICACY} backup + popd + +} + +function init_repo_pref_dir() +{ + pushd ${TEST_REPO} + ${DUPLICACY} init -pref-dir "${DUPLICACY_PREF_DIR}" integration-tests ${TEST_STORAGE} + ${DUPLICACY} backup + popd + +} + +function add_file() +{ + FILE_NAME=$1 + pushd ${TEST_REPO} + echo ${FILE_NAME} > "${FILE_NAME}" + popd +} + + +function backup() +{ + pushd ${TEST_REPO} + ${DUPLICACY} backup + popd +} + + +function restore() +{ + pushd ${TEST_REPO} + ${DUPLICACY} restore -r 2 -delete + popd +} \ No newline at end of file diff --git a/src/duplicacy_backupmanager.go b/src/duplicacy_backupmanager.go index 1f27205..2fd3474 100644 --- a/src/duplicacy_backupmanager.go +++ b/src/duplicacy_backupmanager.go @@ -71,8 +71,9 @@ func CreateBackupManager(snapshotID string, storage Storage, top string, passwor // SetupSnapshotCache creates the snapshot cache, which is merely a local storage under the default .duplicacy // directory func (manager *BackupManager) SetupSnapshotCache(top string, storageName string) bool { - - cacheDir := path.Join(top, DUPLICACY_DIRECTORY, "cache", storageName) + + preferencePath := GetDuplicacyPreferencePath(top) + cacheDir := path.Join(preferencePath, "cache", storageName) storage, err := CreateFileStorage(cacheDir, 1) if err != nil { @@ -600,6 +601,7 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu } } + // How will behave restore when repo created using -repo-dir ,?? err = os.Mkdir(path.Join(top, DUPLICACY_DIRECTORY), 0744) if err != nil && !os.IsExist(err) { LOG_ERROR("RESTORE_MKDIR", "Failed to create the preference directory: %v", err) @@ -978,8 +980,9 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun var existingFile, newFile *os.File var err error - - temporaryPath := path.Join(top, DUPLICACY_DIRECTORY, "temporary") + + preferencePath := GetDuplicacyPreferencePath(top) + temporaryPath := path.Join(preferencePath, "temporary") fullPath := joinPath(top, entry.Path) defer func() { diff --git a/src/duplicacy_entry.go b/src/duplicacy_entry.go index fe0ceee..9237aae 100644 --- a/src/duplicacy_entry.go +++ b/src/duplicacy_entry.go @@ -22,6 +22,7 @@ import ( // This is the hidden directory in the repository for storing various files. var DUPLICACY_DIRECTORY = ".duplicacy" +var DUPLICACY_FILE = ".duplicacy" // Regex for matching 'StartChunk:StartOffset:EndChunk:EndOffset' var contentRegex = regexp.MustCompile(`^([0-9]+):([0-9]+):([0-9]+):([0-9]+)`) diff --git a/src/duplicacy_preference.go b/src/duplicacy_preference.go index 139e80e..fe05f95 100644 --- a/src/duplicacy_preference.go +++ b/src/duplicacy_preference.go @@ -9,6 +9,7 @@ import ( "path" "io/ioutil" "reflect" + "os" ) // Preference stores options for each storage. @@ -25,9 +26,52 @@ type Preference struct { var Preferences [] Preference -func LoadPreferences(repository string) (bool) { +// Compute .duplicacy directory path name: +// - if .duplicacy is a directory -> compute absolute path name and return it +// - if .duplicacy is a file -> assumed this file contains the real path name of .duplicacy +// - if pointed directory does not exits... return error +func GetDuplicacyPreferencePath( repository string) (preferencePath string){ + + preferencePath = path.Join(repository, DUPLICACY_DIRECTORY) //TOKEEP + + stat, err := os.Stat(preferencePath) + if err != nil && !os.IsNotExist(err) { + LOG_ERROR("DOT_DUPLICACY_PATH", "Failed to retrieve the information about the directory %s: %v", + repository, err) + return "" + } + + if stat != nil && stat.IsDir() { + // $repository/.duplicacy exists and is a directory --> we found the .duplicacy directory + return path.Clean(preferencePath) + } + + if stat != nil && stat.Mode().IsRegular() { + b, err := ioutil.ReadFile(preferencePath) // just pass the file name + if err != nil { + LOG_ERROR("DOT_DUPLICACY_PATH", "Failed to read file %s: %v", + preferencePath, err) + return "" + } + dotDuplicacyContent := string(b) // convert content to a 'string' + stat, err := os.Stat(dotDuplicacyContent) + if err != nil && !os.IsNotExist(err) { + LOG_ERROR("DOT_DUPLICACY_PATH", "Failed to retrieve the information about the directory %s: %v", + repository, err) + return "" + } + if stat != nil && stat.IsDir() { + // If expression read from .duplicacy file is a directory --> we found the .duplicacy directory + return path.Clean(dotDuplicacyContent) + } + } + return "" +} - description, err := ioutil.ReadFile(path.Join(repository, DUPLICACY_DIRECTORY, "preferences")) +func LoadPreferences(repository string) (bool) { + + preferencePath := GetDuplicacyPreferencePath(repository) + description, err := ioutil.ReadFile(path.Join(preferencePath, "preferences")) if err != nil { LOG_ERROR("PREFERENCE_OPEN", "Failed to read the preference file from repository %s: %v", repository, err) return false @@ -53,8 +97,9 @@ func SavePreferences(repository string) (bool) { LOG_ERROR("PREFERENCE_MARSHAL", "Failed to marshal the repository preferences: %v", err) return false } - - preferenceFile := path.Join(repository, DUPLICACY_DIRECTORY, "/preferences") + preferencePath := GetDuplicacyPreferencePath(repository) + preferenceFile := path.Join(preferencePath, "/preferences") + err = ioutil.WriteFile(preferenceFile, description, 0644) if err != nil { LOG_ERROR("PREFERENCE_WRITE", "Failed to save the preference file %s: %v", preferenceFile, err) diff --git a/src/duplicacy_shadowcopy_windows.go b/src/duplicacy_shadowcopy_windows.go index 1f1749c..12f8f57 100644 --- a/src/duplicacy_shadowcopy_windows.go +++ b/src/duplicacy_shadowcopy_windows.go @@ -509,8 +509,9 @@ func CreateShadowCopy(top string, shadowCopy bool) (shadowTop string) { LOG_INFO("VSS_DONE", "Shadow copy %s created", SnapshotIDString) snapshotPath := uint16ArrayToString(properties.SnapshotDeviceObject) - - shadowLink = path.Join(top, DUPLICACY_DIRECTORY) + "\\shadow" + + preferencePath := GetDuplicacyPreferencePath(top) + shadowLink = preferencePath + "\\shadow" os.Remove(shadowLink) err = os.Symlink(snapshotPath + "\\", shadowLink) if err != nil { diff --git a/src/duplicacy_snapshot.go b/src/duplicacy_snapshot.go index 75699ed..69938b7 100644 --- a/src/duplicacy_snapshot.go +++ b/src/duplicacy_snapshot.go @@ -67,7 +67,9 @@ func CreateSnapshotFromDirectory(id string, top string) (snapshot *Snapshot, ski } var patterns []string - patternFile, err := ioutil.ReadFile(path.Join(top, DUPLICACY_DIRECTORY, "filters")) + + preferencePath := GetDuplicacyPreferencePath(top) + patternFile, err := ioutil.ReadFile(path.Join(preferencePath, "filters")) if err == nil { for _, pattern := range strings.Split(string(patternFile), "\n") { pattern = strings.TrimSpace(pattern) diff --git a/src/duplicacy_snapshotmanager.go b/src/duplicacy_snapshotmanager.go index 1091a0e..41d3e53 100644 --- a/src/duplicacy_snapshotmanager.go +++ b/src/duplicacy_snapshotmanager.go @@ -1510,8 +1510,9 @@ func (manager *SnapshotManager) PruneSnapshots(top string, selfID string, snapsh if len(revisionsToBeDeleted) > 0 && (len(tags) > 0 || len(retentions) > 0) { LOG_WARN("DELETE_OPTIONS", "Tags or retention policy will be ignored if at least one revision is specified") } - - logDir := path.Join(top, DUPLICACY_DIRECTORY, "logs") + + preferencePath := GetDuplicacyPreferencePath(top) + logDir := path.Join(preferencePath, "logs") os.Mkdir(logDir, 0700) logFileName := path.Join(logDir, time.Now().Format("prune-log-20060102-150405")) logFile, err := os.OpenFile(logFileName, os.O_WRONLY | os.O_CREATE | os.O_TRUNC, 0600) diff --git a/src/duplicacy_storage.go b/src/duplicacy_storage.go index d7dc7be..8d434de 100644 --- a/src/duplicacy_storage.go +++ b/src/duplicacy_storage.go @@ -80,9 +80,9 @@ func checkHostKey(repository string, hostname string, remote net.Addr, key ssh.P if len(repository) == 0 { return nil } - - duplicacyDirectory := path.Join(repository, DUPLICACY_DIRECTORY) - hostFile := path.Join(duplicacyDirectory, "knowns_hosts") + + preferencePath := GetDuplicacyPreferencePath(repository) + hostFile := path.Join(preferencePath, "knowns_hosts") file, err := os.OpenFile(hostFile, os.O_RDWR | os.O_CREATE, 0600) if err != nil { return err