Merge pull request #71 from ech1965/pref-dir

add -pref-dir command line option for init subcommand

.gitignore

@@ -0,0 +1,3 @@
+.idea
+duplicacy_main
+

GUIDE.md

@@ -16,6 +16,7 @@ OPTIONS:
    -chunk-size, -c 4M              the average size of chunks
    -max-chunk-size, -max 16M       the maximum size of chunks (defaults to chunk-size * 4)
    -min-chunk-size, -min 1M        the minimum size of chunks (defaults to chunk-size / 4)
+   -pref-dir <preference directory path> 	Specify alternate location for .duplicacy preferences directory
 ```
 
 The *init* command first connects to the storage specified by the storage URL.  If the storage has been already been
@@ -33,6 +34,8 @@ The -e option controls whether or not encryption will be enabled for the storage
 
 The three chunk size parameters are passed to the variable-size chunking algorithm.  Their values are important to the overall performance, especially for cloud storages.  If the chunk size is too small, a lot of overhead will be in sending requests and receiving responses.  If the chunk size is too large, the effect of deduplication will be less obvious as more data will need to be transferred with each chunk.
 
+The -pref-dir controls the location of the preferences directory. If not specified, a directory named .duplicacy is created in the repository. If specified, it must point to a non-existing directory. The directory is created and a .duplicacy file is created in the repository. The .duplicacy file contains the absolute path name to the preferences directory.
+
 Once a storage has been initialized with these parameters, these parameters cannot be modified any more.
 
 #### Backup

duplicacy/duplicacy_main.go

@@ -18,6 +18,7 @@ import (
     "github.com/gilbertchen/cli"
 
     "github.com/gilbertchen/duplicacy/src"
+    "io/ioutil"
 )
 
 const (
@@ -36,14 +37,14 @@ func getRepositoryPreference(context *cli.Context, storageName string) (reposito
     }
 
     for {
-        stat, err := os.Stat(path.Join(repository, duplicacy.DUPLICACY_DIRECTORY))
+        stat, err := os.Stat(path.Join(repository, duplicacy.DUPLICACY_DIRECTORY)) //TOKEEP
         if err != nil && !os.IsNotExist(err) {
             duplicacy.LOG_ERROR("REPOSITORY_PATH", "Failed to retrieve the information about the directory %s: %v",
                                 repository, err)
             return "", nil
         }
 
-        if stat != nil && stat.IsDir() {
+        if stat != nil && (stat.IsDir() || stat.Mode().IsRegular()) {
             break
         }
 
@@ -54,10 +55,10 @@ func getRepositoryPreference(context *cli.Context, storageName string) (reposito
         }
         repository = parent
     }
-
     duplicacy.LoadPreferences(repository)
-
+    
-    duplicacy.SetKeyringFile(path.Join(repository, duplicacy.DUPLICACY_DIRECTORY, "keyring"))
+    preferencePath := duplicacy.GetDuplicacyPreferencePath(repository)
+    duplicacy.SetKeyringFile(path.Join(preferencePath, "keyring"))
 
     if storageName == "" {
         storageName = context.String("storage")
@@ -142,8 +143,9 @@ func runScript(context *cli.Context, repository string, storageName string, phas
     if !ScriptEnabled {
         return false
     }
-
+    
-    scriptDir, _ := filepath.Abs(path.Join(repository, duplicacy.DUPLICACY_DIRECTORY, "scripts"))
+    preferencePath := duplicacy.GetDuplicacyPreferencePath(repository)
+    scriptDir, _ := filepath.Abs(path.Join(preferencePath, "scripts"))
     scriptName := phase + "-" + context.Command.Name
 
     script := path.Join(scriptDir, scriptName)
@@ -174,14 +176,14 @@ func runScript(context *cli.Context, repository string, storageName string, phas
 }
 
 func initRepository(context *cli.Context) {
-    configRespository(context, true)
+    configRepository(context, true)
 }
 
 func addStorage(context *cli.Context) {
-    configRespository(context, false)
+    configRepository(context, false)
 }
 
-func configRespository(context *cli.Context, init bool) {
+func configRepository(context *cli.Context, init bool) {
 
     setGlobalOptions(context)
     defer duplicacy.CatchLogException()
@@ -220,21 +222,37 @@ func configRespository(context *cli.Context, init bool) {
             duplicacy.LOG_ERROR("REPOSITORY_PATH", "Failed to retrieve the current working directory: %v", err)
             return
         }
-
+    
-        duplicacyDirectory := path.Join(repository, duplicacy.DUPLICACY_DIRECTORY)
+        preferencePath :=  context.String("pref-dir")
-        if stat, _ := os.Stat(path.Join(duplicacyDirectory, "preferences")); stat != nil {
+        if preferencePath == "" {
+            
+            preferencePath = path.Join(repository, duplicacy.DUPLICACY_DIRECTORY) // TOKEEP
+        }
+        
+        
+        if stat, _ := os.Stat(path.Join(preferencePath, "preferences")); stat != nil {
             duplicacy.LOG_ERROR("REPOSITORY_INIT", "The repository %s has already been initialized", repository)
             return
         }
 
-        err = os.Mkdir(duplicacyDirectory, 0744)
+        err = os.Mkdir(preferencePath, 0744)
         if err != nil && !os.IsExist(err) {
             duplicacy.LOG_ERROR("REPOSITORY_INIT", "Failed to create the directory %s: %v",
-                                duplicacy.DUPLICACY_DIRECTORY, err)
+                preferencePath, err)
             return
         }
-
+        if context.String("pref-dir") != "" {
-        duplicacy.SetKeyringFile(path.Join(duplicacyDirectory, "keyring"))
+            // out of tree preference file
+            // write real path into .duplicacy file inside repository
+            duplicacyFileName := path.Join(repository, duplicacy.DUPLICACY_FILE)
+            d1 := []byte(preferencePath)
+            err = ioutil.WriteFile(duplicacyFileName, d1, 0644)
+            if err != nil {
+                duplicacy.LOG_ERROR("REPOSITORY_PATH", "Failed to write %s file inside repository  %v", duplicacyFileName, err)
+                return
+            }
+        }
+        duplicacy.SetKeyringFile(path.Join(preferencePath, "keyring"))
 
     } else {
         repository, _ = getRepositoryPreference(context, "")
@@ -547,7 +565,6 @@ func changePassword(context *cli.Context) {
     duplicacy.LOG_INFO("STORAGE_SET", "The password for storage %s has been changed", preference.StorageURL)
 }
 
-
 func backupRepository(context *cli.Context) {
     setGlobalOptions(context)
     defer duplicacy.CatchLogException()
@@ -1071,7 +1088,8 @@ func infoStorage(context *cli.Context) {
 
     repository := context.String("repository")
     if repository != "" {
-        duplicacy.SetKeyringFile(path.Join(repository, duplicacy.DUPLICACY_DIRECTORY, "keyring"))
+        preferencePath := duplicacy.GetDuplicacyPreferencePath(repository)
+        duplicacy.SetKeyringFile(path.Join(preferencePath, "keyring"))
     }
 
     isEncrypted := context.Bool("e")
@@ -1132,6 +1150,11 @@ func main() {
                     Usage: "the minimum size of chunks (defaults to chunk-size / 4)",
                     Argument: "1M",
                 },
+                cli.StringFlag{
+                    Name: "pref-dir",
+                    Usage: "Specify alternate location for .duplicacy preferences directory (absolute or relative to current directory)",
+                    Argument: "<preferences directory path>",
+                },
             },
             Usage:  "Initialize the storage if necessary and the current directory as the repository",
             ArgsUsage: "<snapshot id> <storage url>",

integration_tests/test.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+
+. ./test_functions.sh
+
+fixture
+init_repo_pref_dir
+
+backup
+add_file file3
+backup
+add_file file4
+backup
+add_file file5
+restore
+
+

integration_tests/test_functions.sh

@@ -0,0 +1,103 @@
+#!/bin/bash
+
+get_abs_filename() {
+  # $1 : relative filename
+  echo "$(cd "$(dirname "$1")" && pwd)/$(basename "$1")"
+}
+
+pushd () {
+    command pushd "$@" > /dev/null
+}
+
+popd () {
+    command popd "$@" > /dev/null
+}
+
+
+# Functions used to create integration tests suite
+
+DUPLICACY=$(get_abs_filename ../duplicacy_main)
+
+# Base directory where test repositories will be created
+TEST_ZONE=$HOME/DUPLICACY_TEST_ZONE
+# Test Repository
+TEST_REPO=$TEST_ZONE/TEST_REPO
+
+# Storage for test ( For now, only local path storage is supported by test suite)
+TEST_STORAGE=$TEST_ZONE/TEST_STORAGE
+
+# Preference directory ( for testing the -pref-dir option)
+DUPLICACY_PREF_DIR=$TEST_ZONE/TEST_DUPLICACY_PREF_DIR
+
+# Scratch pad for testing restore
+TEST_RESTORE_POINT=$TEST_ZONE/RESTORE_POINT
+
+# Make sure $TEST_ZONE is in know state
+
+
+
+function fixture()
+{
+  # clean TEST_RESTORE_POINT
+  rm -rf $TEST_RESTORE_POINT
+  mkdir -p $TEST_RESTORE_POINT
+
+  # clean TEST_STORAGE
+  rm -rf $TEST_STORAGE
+  mkdir -p $TEST_STORAGE
+
+  # clean TEST_DOT_DUPLICACY
+  rm -rf $DUPLICACY_PREF_DIR
+  mkdir -p $DUPLICACY_PREF_DIR
+
+  # Create test repository
+  rm -rf ${TEST_REPO}
+  mkdir -p ${TEST_REPO}
+  pushd ${TEST_REPO}
+    echo "file1" > file1
+    mkdir dir1
+    echo "file2" > dir1/file2
+  popd
+}
+
+function init_repo()
+{
+  pushd ${TEST_REPO}
+  ${DUPLICACY} init integration-tests $TEST_STORAGE
+  ${DUPLICACY} backup
+  popd
+
+}
+
+function init_repo_pref_dir()
+{
+  pushd ${TEST_REPO}
+  ${DUPLICACY} init -pref-dir "${DUPLICACY_PREF_DIR}"  integration-tests ${TEST_STORAGE}
+  ${DUPLICACY} backup
+  popd
+
+}
+
+function add_file()
+{
+  FILE_NAME=$1
+  pushd ${TEST_REPO}
+    echo ${FILE_NAME} > "${FILE_NAME}"
+  popd
+}
+
+
+function backup()
+{
+  pushd ${TEST_REPO}
+  ${DUPLICACY} backup
+  popd
+}
+
+
+function restore()
+{
+  pushd ${TEST_REPO}
+  ${DUPLICACY} restore -r 2 -delete
+  popd
+}

src/duplicacy_backupmanager.go

@@ -71,8 +71,9 @@ func CreateBackupManager(snapshotID string, storage Storage, top string, passwor
 // SetupSnapshotCache creates the snapshot cache, which is merely a local storage under the default .duplicacy
 // directory
 func (manager *BackupManager) SetupSnapshotCache(top string, storageName string) bool {
-
+    
-    cacheDir := path.Join(top, DUPLICACY_DIRECTORY, "cache", storageName)
+    preferencePath := GetDuplicacyPreferencePath(top)
+    cacheDir := path.Join(preferencePath, "cache", storageName)
 
     storage, err := CreateFileStorage(cacheDir, 1)
     if err != nil {
@@ -600,6 +601,7 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
         }
     }
 
+    // How will behave restore when repo created using -repo-dir ,??
     err = os.Mkdir(path.Join(top, DUPLICACY_DIRECTORY), 0744)
     if err != nil && !os.IsExist(err) {
         LOG_ERROR("RESTORE_MKDIR", "Failed to create the preference directory: %v", err)
@@ -978,8 +980,9 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
 
     var existingFile, newFile *os.File
     var err error
-
+    
-    temporaryPath := path.Join(top, DUPLICACY_DIRECTORY, "temporary")
+    preferencePath := GetDuplicacyPreferencePath(top)
+    temporaryPath := path.Join(preferencePath, "temporary")
     fullPath := joinPath(top, entry.Path)
 
     defer func() {

src/duplicacy_entry.go

@@ -22,6 +22,7 @@ import (
 
 // This is the hidden directory in the repository for storing various files.
 var DUPLICACY_DIRECTORY = ".duplicacy"
+var DUPLICACY_FILE = ".duplicacy"
 
 // Regex for matching 'StartChunk:StartOffset:EndChunk:EndOffset'
 var contentRegex = regexp.MustCompile(`^([0-9]+):([0-9]+):([0-9]+):([0-9]+)`)

src/duplicacy_preference.go

@@ -9,6 +9,7 @@ import (
     "path"
     "io/ioutil"
     "reflect"
+    "os"
 )
 
 // Preference stores options for each storage.
@@ -25,9 +26,52 @@ type Preference struct {
 
 var Preferences [] Preference
 
-func LoadPreferences(repository string) (bool) {
+// Compute .duplicacy directory path name:
+//  - if .duplicacy is a directory -> compute absolute path name and return it
+//  - if .duplicacy is a file -> assumed this file contains the real path name of .duplicacy
+//  - if pointed directory does not exits... return error
+func GetDuplicacyPreferencePath( repository string) (preferencePath string){
+    
+    preferencePath = path.Join(repository, DUPLICACY_DIRECTORY) //TOKEEP
+    
+    stat, err := os.Stat(preferencePath)
+    if err != nil && !os.IsNotExist(err) {
+        LOG_ERROR("DOT_DUPLICACY_PATH", "Failed to retrieve the information about the directory %s: %v",
+            repository, err)
+        return ""
+    }
+    
+    if stat != nil && stat.IsDir() {
+        // $repository/.duplicacy exists and is a directory --> we found the .duplicacy directory
+        return path.Clean(preferencePath)
+    }
+    
+    if stat != nil && stat.Mode().IsRegular() {
+        b, err := ioutil.ReadFile(preferencePath) // just pass the file name
+        if err != nil {
+            LOG_ERROR("DOT_DUPLICACY_PATH", "Failed to read file %s: %v",
+                preferencePath, err)
+            return ""
+        }
+        dotDuplicacyContent := string(b) // convert content to a 'string'
+        stat, err := os.Stat(dotDuplicacyContent)
+        if err != nil && !os.IsNotExist(err) {
+            LOG_ERROR("DOT_DUPLICACY_PATH", "Failed to retrieve the information about the directory %s: %v",
+                repository, err)
+            return ""
+        }
+        if stat != nil && stat.IsDir() {
+            // If expression read from .duplicacy file is a directory --> we found the .duplicacy directory
+            return path.Clean(dotDuplicacyContent)
+        }
+    }
+    return ""
+}
 
-    description, err := ioutil.ReadFile(path.Join(repository, DUPLICACY_DIRECTORY, "preferences"))
+func LoadPreferences(repository string) (bool) {
+    
+    preferencePath := GetDuplicacyPreferencePath(repository)
+    description, err := ioutil.ReadFile(path.Join(preferencePath, "preferences"))
     if err != nil {
         LOG_ERROR("PREFERENCE_OPEN", "Failed to read the preference file from repository %s: %v", repository, err)
         return false
@@ -53,8 +97,9 @@ func SavePreferences(repository string) (bool) {
         LOG_ERROR("PREFERENCE_MARSHAL", "Failed to marshal the repository preferences: %v", err)
         return false
     }
-
+    preferencePath := GetDuplicacyPreferencePath(repository)
-    preferenceFile := path.Join(repository, DUPLICACY_DIRECTORY, "/preferences")
+    preferenceFile := path.Join(preferencePath, "/preferences")
+    
     err = ioutil.WriteFile(preferenceFile, description, 0644)
     if err != nil {
         LOG_ERROR("PREFERENCE_WRITE", "Failed to save the preference file %s: %v", preferenceFile, err)

src/duplicacy_shadowcopy_windows.go

@@ -509,8 +509,9 @@ func CreateShadowCopy(top string, shadowCopy bool) (shadowTop string) {
     LOG_INFO("VSS_DONE", "Shadow copy %s created", SnapshotIDString)
 
     snapshotPath := uint16ArrayToString(properties.SnapshotDeviceObject)
-
+    
-    shadowLink = path.Join(top, DUPLICACY_DIRECTORY) + "\\shadow"
+    preferencePath := GetDuplicacyPreferencePath(top)
+    shadowLink =  preferencePath + "\\shadow"
     os.Remove(shadowLink)
     err = os.Symlink(snapshotPath + "\\", shadowLink)
     if err != nil {

src/duplicacy_snapshot.go

@@ -67,7 +67,9 @@ func CreateSnapshotFromDirectory(id string, top string) (snapshot *Snapshot, ski
     }
 
     var patterns []string
-    patternFile, err := ioutil.ReadFile(path.Join(top, DUPLICACY_DIRECTORY, "filters"))
+    
+    preferencePath := GetDuplicacyPreferencePath(top)
+    patternFile, err := ioutil.ReadFile(path.Join(preferencePath, "filters"))
     if err == nil {
         for _, pattern := range strings.Split(string(patternFile), "\n") {
             pattern = strings.TrimSpace(pattern)

src/duplicacy_snapshotmanager.go

@@ -1510,8 +1510,9 @@ func (manager *SnapshotManager) PruneSnapshots(top string, selfID string, snapsh
     if len(revisionsToBeDeleted) > 0 && (len(tags) > 0 || len(retentions) > 0) {
         LOG_WARN("DELETE_OPTIONS", "Tags or retention policy will be ignored if at least one revision is specified")
     }
-
+    
-    logDir := path.Join(top, DUPLICACY_DIRECTORY, "logs")
+    preferencePath := GetDuplicacyPreferencePath(top)
+    logDir := path.Join(preferencePath, "logs")
     os.Mkdir(logDir, 0700)
     logFileName := path.Join(logDir, time.Now().Format("prune-log-20060102-150405"))
     logFile, err := os.OpenFile(logFileName, os.O_WRONLY | os.O_CREATE | os.O_TRUNC, 0600)

src/duplicacy_storage.go

@@ -80,9 +80,9 @@ func checkHostKey(repository string, hostname string, remote net.Addr, key ssh.P
     if len(repository) == 0 {
         return nil
     }
-
+    
-    duplicacyDirectory := path.Join(repository, DUPLICACY_DIRECTORY)
+    preferencePath := GetDuplicacyPreferencePath(repository)
-    hostFile := path.Join(duplicacyDirectory, "knowns_hosts")
+    hostFile := path.Join(preferencePath, "knowns_hosts")
     file, err := os.OpenFile(hostFile, os.O_RDWR | os.O_CREATE, 0600)
     if err != nil {
         return err