Implement fast resume; refactor GetDuplicacyPreferencePath()

duplicacy/duplicacy_main.go

@@ -13,6 +13,7 @@ import (
     "strings"
     "strconv"
     "os/exec"
+    "os/signal"
     "encoding/json"
 
     "github.com/gilbertchen/cli"
@@ -57,7 +58,7 @@ func getRepositoryPreference(context *cli.Context, storageName string) (reposito
     }
     duplicacy.LoadPreferences(repository)
     
-    preferencePath := duplicacy.GetDuplicacyPreferencePath(repository)
+    preferencePath := duplicacy.GetDuplicacyPreferencePath()
     duplicacy.SetKeyringFile(path.Join(preferencePath, "keyring"))
 
     if storageName == "" {
@@ -138,13 +139,13 @@ func setGlobalOptions(context *cli.Context) {
     duplicacy.RunInBackground = context.GlobalBool("background")
 }
 
-func runScript(context *cli.Context, repository string, storageName string, phase string) bool {
+func runScript(context *cli.Context, storageName string, phase string) bool {
 
     if !ScriptEnabled {
         return false
     }
     
-    preferencePath := duplicacy.GetDuplicacyPreferencePath(repository)
+    preferencePath := duplicacy.GetDuplicacyPreferencePath()
     scriptDir, _ := filepath.Abs(path.Join(preferencePath, "scripts"))
     scriptName := phase + "-" + context.Command.Name
 
@@ -225,7 +226,6 @@ func configRepository(context *cli.Context, init bool) {
     
         preferencePath :=  context.String("pref-dir")
         if preferencePath == "" {
-            
             preferencePath = path.Join(repository, duplicacy.DUPLICACY_DIRECTORY) // TOKEEP
         }
         
@@ -252,6 +252,7 @@ func configRepository(context *cli.Context, init bool) {
                 return
             }
         }
+        duplicacy.SetDuplicacyPreferencePath(preferencePath)
         duplicacy.SetKeyringFile(path.Join(preferencePath, "keyring"))
 
     } else {
@@ -269,7 +270,7 @@ func configRepository(context *cli.Context, init bool) {
         Encrypted: context.Bool("encrypt"),
     }
 
-    storage := duplicacy.CreateStorage(repository, preference, true, 1)
+    storage := duplicacy.CreateStorage(preference, true, 1)
     storagePassword := ""
     if preference.Encrypted {
         prompt := fmt.Sprintf("Enter storage password for %s:", preference.StorageURL)
@@ -359,7 +360,7 @@ func configRepository(context *cli.Context, init bool) {
 
             }
 
-            otherStorage := duplicacy.CreateStorage(repository, *otherPreference, false, 1)
+            otherStorage := duplicacy.CreateStorage(*otherPreference, false, 1)
 
             otherPassword := ""
             if otherPreference.Encrypted {
@@ -386,7 +387,7 @@ func configRepository(context *cli.Context, init bool) {
 
     duplicacy.Preferences = append(duplicacy.Preferences, preference)
 
-    duplicacy.SavePreferences(repository)
+    duplicacy.SavePreferences()
 
     duplicacy.LOG_INFO("REPOSITORY_INIT", "%s will be backed up to %s with id %s",
                        repository, preference.StorageURL, preference.SnapshotID)
@@ -507,7 +508,7 @@ func setPreference(context *cli.Context) {
                            oldPreference.StorageURL)
     } else {
         *oldPreference = newPreference
-        duplicacy.SavePreferences(repository)
+        duplicacy.SavePreferences()
         duplicacy.LOG_INFO("STORAGE_SET", "New options for storage %s have been saved", oldPreference.StorageURL)
     }
 }
@@ -524,9 +525,9 @@ func changePassword(context *cli.Context) {
         os.Exit(ArgumentExitCode)
     }
 
-    repository, preference := getRepositoryPreference(context, "")
+    _, preference := getRepositoryPreference(context, "")
 
-    storage := duplicacy.CreateStorage(repository, *preference, false, 1)
+    storage := duplicacy.CreateStorage(*preference, false, 1)
     if storage == nil {
         return
     }
@@ -583,7 +584,7 @@ func backupRepository(context *cli.Context) {
         return
     }
 
-    runScript(context, repository,  preference.Name, "pre")
+    runScript(context, preference.Name, "pre")
 
     threads := context.Int("threads")
     if threads < 1 {
@@ -591,7 +592,7 @@ func backupRepository(context *cli.Context) {
     }
 
     duplicacy.LOG_INFO("STORAGE_SET", "Storage set to %s", preference.StorageURL)
-    storage := duplicacy.CreateStorage(repository, *preference, false, threads)
+    storage := duplicacy.CreateStorage(*preference, false, threads)
     if storage == nil {
         return
     }
@@ -615,10 +616,10 @@ func backupRepository(context *cli.Context) {
     backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password)
     duplicacy.SavePassword(*preference, "password", password)
 
-    backupManager.SetupSnapshotCache(repository, preference.Name)
+    backupManager.SetupSnapshotCache(preference.Name)
     backupManager.Backup(repository, quickMode, threads, context.String("t"), showStatistics, enableVSS)
 
-    runScript(context, repository,  preference.Name, "post")
+    runScript(context, preference.Name, "post")
 }
 
 func restoreRepository(context *cli.Context) {
@@ -640,7 +641,7 @@ func restoreRepository(context *cli.Context) {
         return
     }
 
-    runScript(context, repository,  preference.Name, "pre")
+    runScript(context, preference.Name, "pre")
 
     threads := context.Int("threads")
     if threads < 1 {
@@ -648,7 +649,7 @@ func restoreRepository(context *cli.Context) {
     }
 
     duplicacy.LOG_INFO("STORAGE_SET", "Storage set to %s", preference.StorageURL)
-    storage := duplicacy.CreateStorage(repository, *preference, false, threads)
+    storage := duplicacy.CreateStorage(*preference, false, threads)
     if storage == nil {
         return
     }
@@ -690,10 +691,10 @@ func restoreRepository(context *cli.Context) {
     backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password)
     duplicacy.SavePassword(*preference, "password", password)
 
-    backupManager.SetupSnapshotCache(repository, preference.Name)
+    backupManager.SetupSnapshotCache(preference.Name)
     backupManager.Restore(repository, revision, true, quickMode, threads, overwrite, deleteMode, showStatistics, patterns)
 
-    runScript(context, repository,  preference.Name, "post")
+    runScript(context, preference.Name, "post")
 }
 
 func listSnapshots(context *cli.Context) {
@@ -710,10 +711,10 @@ func listSnapshots(context *cli.Context) {
 
     duplicacy.LOG_INFO("STORAGE_SET", "Storage set to %s", preference.StorageURL)
 
-    runScript(context, repository,  preference.Name, "pre")
+    runScript(context, preference.Name, "pre")
 
     resetPassword := context.Bool("reset-passwords")
-    storage := duplicacy.CreateStorage(repository, *preference, resetPassword, 1)
+    storage := duplicacy.CreateStorage(*preference, resetPassword, 1)
     if storage == nil {
         return
     }
@@ -740,10 +741,10 @@ func listSnapshots(context *cli.Context) {
     showFiles := context.Bool("files")
     showChunks := context.Bool("chunks")
 
-    backupManager.SetupSnapshotCache(repository, preference.Name)
+    backupManager.SetupSnapshotCache(preference.Name)
     backupManager.SnapshotManager.ListSnapshots(id, revisions, tag, showFiles, showChunks)
 
-    runScript(context, repository,  preference.Name, "post")
+    runScript(context, preference.Name, "post")
 }
 
 func checkSnapshots(context *cli.Context) {
@@ -760,9 +761,9 @@ func checkSnapshots(context *cli.Context) {
 
     duplicacy.LOG_INFO("STORAGE_SET", "Storage set to %s", preference.StorageURL)
 
-    runScript(context, repository,  preference.Name, "pre")
+    runScript(context, preference.Name, "pre")
 
-    storage := duplicacy.CreateStorage(repository, *preference, false, 1)
+    storage := duplicacy.CreateStorage(*preference, false, 1)
     if storage == nil {
         return
     }
@@ -790,10 +791,10 @@ func checkSnapshots(context *cli.Context) {
     searchFossils := context.Bool("fossils")
     resurrect := context.Bool("resurrect")
 
-    backupManager.SetupSnapshotCache(repository, preference.Name)
+    backupManager.SetupSnapshotCache(preference.Name)
     backupManager.SnapshotManager.CheckSnapshots(id, revisions, tag, showStatistics, checkFiles, searchFossils, resurrect)
 
-    runScript(context, repository,  preference.Name, "post")
+    runScript(context, preference.Name, "post")
 }
 
 func printFile(context *cli.Context) {
@@ -808,11 +809,11 @@ func printFile(context *cli.Context) {
 
     repository, preference := getRepositoryPreference(context, "")
 
-    runScript(context, repository,  preference.Name, "pre")
+    runScript(context, preference.Name, "pre")
 
     // Do not print out storage for this command
     //duplicacy.LOG_INFO("STORAGE_SET", "Storage set to %s", preference.StorageURL)
-    storage := duplicacy.CreateStorage(repository, *preference, false, 1)
+    storage := duplicacy.CreateStorage(*preference, false, 1)
     if storage == nil {
         return
     }
@@ -832,7 +833,7 @@ func printFile(context *cli.Context) {
     backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password)
     duplicacy.SavePassword(*preference, "password", password)
 
-    backupManager.SetupSnapshotCache(repository, preference.Name)
+    backupManager.SetupSnapshotCache(preference.Name)
 
     file := ""
     if len(context.Args()) > 0 {
@@ -840,7 +841,7 @@ func printFile(context *cli.Context) {
     }
     backupManager.SnapshotManager.PrintFile(snapshotID, revision, file)
 
-    runScript(context, repository,  preference.Name, "post")
+    runScript(context, preference.Name, "post")
 }
 
 func diff(context *cli.Context) {
@@ -855,10 +856,10 @@ func diff(context *cli.Context) {
 
     repository, preference := getRepositoryPreference(context, "")
 
-    runScript(context, repository,  preference.Name, "pre")
+    runScript(context, preference.Name, "pre")
 
     duplicacy.LOG_INFO("STORAGE_SET", "Storage set to %s", preference.StorageURL)
-    storage := duplicacy.CreateStorage(repository, *preference, false, 1)
+    storage := duplicacy.CreateStorage(*preference, false, 1)
     if storage == nil {
         return
     }
@@ -889,10 +890,10 @@ func diff(context *cli.Context) {
     backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password)
     duplicacy.SavePassword(*preference, "password", password)
 
-    backupManager.SetupSnapshotCache(repository, preference.Name)
+    backupManager.SetupSnapshotCache(preference.Name)
     backupManager.SnapshotManager.Diff(repository, snapshotID, revisions, path, compareByHash)
 
-    runScript(context, repository,  preference.Name, "post")
+    runScript(context, preference.Name, "post")
 }
 
 func showHistory(context *cli.Context) {
@@ -907,10 +908,10 @@ func showHistory(context *cli.Context) {
 
     repository, preference := getRepositoryPreference(context, "")
 
-    runScript(context, repository,  preference.Name, "pre")
+    runScript(context, preference.Name, "pre")
 
     duplicacy.LOG_INFO("STORAGE_SET", "Storage set to %s", preference.StorageURL)
-    storage := duplicacy.CreateStorage(repository, *preference, false, 1)
+    storage := duplicacy.CreateStorage(*preference, false, 1)
     if storage == nil {
         return
     }
@@ -932,10 +933,10 @@ func showHistory(context *cli.Context) {
     backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password)
     duplicacy.SavePassword(*preference, "password", password)
 
-    backupManager.SetupSnapshotCache(repository, preference.Name)
+    backupManager.SetupSnapshotCache(preference.Name)
     backupManager.SnapshotManager.ShowHistory(repository, snapshotID, revisions, path, showLocalHash)
 
-    runScript(context, repository,  preference.Name, "post")
+    runScript(context, preference.Name, "post")
 }
 
 func pruneSnapshots(context *cli.Context) {
@@ -950,10 +951,10 @@ func pruneSnapshots(context *cli.Context) {
 
     repository, preference := getRepositoryPreference(context, "")
 
-    runScript(context, repository,  preference.Name, "pre")
+    runScript(context, preference.Name, "pre")
 
     duplicacy.LOG_INFO("STORAGE_SET", "Storage set to %s", preference.StorageURL)
-    storage := duplicacy.CreateStorage(repository, *preference, false, 1)
+    storage := duplicacy.CreateStorage(*preference, false, 1)
     if storage == nil {
         return
     }
@@ -990,11 +991,11 @@ func pruneSnapshots(context *cli.Context) {
     backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password)
     duplicacy.SavePassword(*preference, "password", password)
 
-    backupManager.SetupSnapshotCache(repository, preference.Name)
+    backupManager.SetupSnapshotCache(preference.Name)
-    backupManager.SnapshotManager.PruneSnapshots(repository, selfID, snapshotID, revisions, tags, retentions,
+    backupManager.SnapshotManager.PruneSnapshots(selfID, snapshotID, revisions, tags, retentions,
                                                  exhaustive, exclusive, ignoredIDs, dryRun, deleteOnly, collectOnly)
 
-    runScript(context, repository,  preference.Name, "post")
+    runScript(context, preference.Name, "post")
 }
 
 func copySnapshots(context *cli.Context) {
@@ -1009,10 +1010,10 @@ func copySnapshots(context *cli.Context) {
 
     repository, source := getRepositoryPreference(context, context.String("from"))
 
-    runScript(context, repository,  source.Name, "pre")
+    runScript(context, source.Name, "pre")
 
     duplicacy.LOG_INFO("STORAGE_SET", "Source storage set to %s", source.StorageURL)
-    sourceStorage := duplicacy.CreateStorage(repository, *source, false, 1)
+    sourceStorage := duplicacy.CreateStorage(*source, false, 1)
     if sourceStorage == nil {
         return
     }
@@ -1023,7 +1024,7 @@ func copySnapshots(context *cli.Context) {
     }
 
     sourceManager := duplicacy.CreateBackupManager(source.SnapshotID, sourceStorage, repository, sourcePassword)
-    sourceManager.SetupSnapshotCache(repository, source.Name)
+    sourceManager.SetupSnapshotCache(source.Name)
     duplicacy.SavePassword(*source, "password", sourcePassword)
 
 
@@ -1042,7 +1043,7 @@ func copySnapshots(context *cli.Context) {
 
 
     duplicacy.LOG_INFO("STORAGE_SET", "Destination storage set to %s", destination.StorageURL)
-    destinationStorage := duplicacy.CreateStorage(repository, *destination, false, 1)
+    destinationStorage := duplicacy.CreateStorage(*destination, false, 1)
     if destinationStorage == nil {
         return
     }
@@ -1059,7 +1060,7 @@ func copySnapshots(context *cli.Context) {
     destinationManager := duplicacy.CreateBackupManager(destination.SnapshotID, destinationStorage, repository,
                                                         destinationPassword)
     duplicacy.SavePassword(*destination, "password", destinationPassword)
-    destinationManager.SetupSnapshotCache(repository, destination.Name)
+    destinationManager.SetupSnapshotCache(destination.Name)
 
     revisions := getRevisions(context)
     snapshotID := ""
@@ -1073,7 +1074,7 @@ func copySnapshots(context *cli.Context) {
     }
 
     sourceManager.CopySnapshots(destinationManager, snapshotID, revisions, threads)
-    runScript(context, repository,  source.Name, "post")
+    runScript(context, source.Name, "post")
 }
 
 func infoStorage(context *cli.Context) {
@@ -1088,7 +1089,8 @@ func infoStorage(context *cli.Context) {
 
     repository := context.String("repository")
     if repository != "" {
-        preferencePath := duplicacy.GetDuplicacyPreferencePath(repository)
+        preferencePath := path.Join(repository, duplicacy.DUPLICACY_DIRECTORY)
+        duplicacy.SetDuplicacyPreferencePath(preferencePath)
         duplicacy.SetKeyringFile(path.Join(preferencePath, "keyring"))
     }
 
@@ -1106,7 +1108,7 @@ func infoStorage(context *cli.Context) {
         password = duplicacy.GetPassword(preference, "password", "Enter the storage password:", false, false)
     }
 
-    storage := duplicacy.CreateStorage("", preference, context.Bool("reset-passwords"), 1)
+    storage := duplicacy.CreateStorage(preference, context.Bool("reset-passwords"), 1)
     config, isStorageEncrypted, err := duplicacy.DownloadConfig(storage, password)
 
     if isStorageEncrypted {
@@ -1682,6 +1684,17 @@ func main() {
     app.HelpName = "duplicacy"
     app.Usage = "A new generation cloud backup tool based on lock-free deduplication"
     app.Version = "2.0.3"
+
+    // If the program is interrupted, call the RunAtError function.
+    c := make(chan os.Signal, 1)                                       
+    signal.Notify(c, os.Interrupt)                                     
+    go func() {                                                        
+        for _ = range c {
+            duplicacy.RunAtError()                                             
+            os.Exit(1)                                                     
+        }                                                                
+    }()        
+
     err := app.Run(os.Args)
     if err != nil {
         os.Exit(2)

integration_tests/resume_test.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+
+. ./test_functions.sh
+
+fixture
+
+pushd ${TEST_REPO}
+${DUPLICACY} init integration-tests $TEST_STORAGE -c 4k
+
+# Create 10 20k files
+add_file file1 20000
+add_file file2 20000
+add_file file3 20000
+add_file file4 20000
+add_file file5 20000
+add_file file6 20000
+add_file file7 20000
+add_file file8 20000
+add_file file9 20000
+add_file file10 20000
+
+# Limit the rate to 10k/s so the backup will take about 10 seconds
+${DUPLICACY} backup -limit-rate 10 -threads 4 &
+# Kill the backup after 3 seconds
+DUPLICACY_PID=$!
+sleep 3
+kill -2 ${DUPLICACY_PID}
+
+# Try it again to test the multiple-resume case
+${DUPLICACY} backup -limit-rate 10  -threads 4&
+DUPLICACY_PID=$!
+sleep 3
+kill -2 ${DUPLICACY_PID}
+
+# Fail the backup before uploading the snapshot
+env DUPLICACY_FAIL_SNAPSHOT=true ${DUPLICACY} backup
+
+# Now complete the backup
+${DUPLICACY} backup
+${DUPLICACY} check --files
+popd

integration_tests/test_functions.sh

@@ -91,8 +91,9 @@ function init_repo_pref_dir()
 function add_file()
 {
   FILE_NAME=$1
+  FILE_SIZE=${2:-20000000}
   pushd ${TEST_REPO}
-  dd if=/dev/urandom of=${FILE_NAME} bs=1000 count=20000
+  dd if=/dev/urandom of=${FILE_NAME} bs=1 count=$(($RANDOM % ${FILE_SIZE})) &> /dev/null
   popd
 }
 

src/duplicacy_backupmanager.go

@@ -13,6 +13,7 @@ import (
     "path"
     "time"
     "sort"
+    "sync"
     "sync/atomic"
     "strings"
     "strconv"
@@ -70,9 +71,9 @@ func CreateBackupManager(snapshotID string, storage Storage, top string, passwor
 
 // SetupSnapshotCache creates the snapshot cache, which is merely a local storage under the default .duplicacy
 // directory
-func (manager *BackupManager) SetupSnapshotCache(top string, storageName string) bool {
+func (manager *BackupManager) SetupSnapshotCache(storageName string) bool {
     
-    preferencePath := GetDuplicacyPreferencePath(top)
+    preferencePath := GetDuplicacyPreferencePath()
     cacheDir := path.Join(preferencePath, "cache", storageName)
 
     storage, err := CreateFileStorage(cacheDir, 1)
@@ -94,11 +95,19 @@ func (manager *BackupManager) SetupSnapshotCache(top string, storageName string)
     return true
 }
 
+// setEntryContent sets the 4 content pointers for each entry in 'entries'.  'offset' indicates the value
+// to be added to the StartChunk and EndChunk points, used when intending to append 'entries' to the
+// original unchanged entry list.
+//
+// This function assumes the Size field of each entry is equal to the length of the chunk content that belong
+// to the file.
 func setEntryContent(entries[] *Entry, chunkLengths[]int, offset int) {
     if len(entries) == 0 {
         return
     }
 
+    // The following code works by iterating over 'entries' and 'chunkLength' and keeping track of the
+    // accumulated total file size and the accumulated total chunk size.
     i := 0
     totalChunkSize := int64(0)
     totalFileSize := entries[i].Size
@@ -115,6 +124,8 @@ func setEntryContent(entries[] *Entry, chunkLengths[]int, offset int) {
                 break
             }
 
+            // If the current file ends at the end of the current chunk, the next file will
+            // start at the next chunk
             if totalChunkSize + int64(length) == totalFileSize {
                 entries[i].StartChunk = j + 1 + offset
                 entries[i].StartOffset = 0
@@ -126,6 +137,9 @@ func setEntryContent(entries[] *Entry, chunkLengths[]int, offset int) {
             totalFileSize += entries[i].Size
         }
 
+        if i >= len(entries) {
+            break
+        }
         totalChunkSize += int64(length)
     }
 }
@@ -150,7 +164,6 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
 
     remoteSnapshot := manager.SnapshotManager.downloadLatestSnapshot(manager.snapshotID)
     if remoteSnapshot == nil {
-        quickMode = false
         remoteSnapshot = CreateEmptySnapshot(manager.snapshotID)
         LOG_INFO("BACKUP_START", "No previous backup found")
     } else {
@@ -171,13 +184,24 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
     // UploadChunk.
     chunkCache := make(map[string]bool)
 
+    var incompleteSnapshot *Snapshot
+
+    // A revision number of 0 means this is the initial backup
     if remoteSnapshot.Revision > 0 {
-        // Add all chunks in the last snapshot to the
+        // Add all chunks in the last snapshot to the cache
         for _, chunkID := range manager.SnapshotManager.GetSnapshotChunks(remoteSnapshot) {
             chunkCache[chunkID] = true
         }
-    } else if manager.storage.IsFastListing() {
+    } else {
-        // If the listing operation is fast, list all chunks and put them in the cache.
+        
+        // In quick mode, attempt to load the incomplete snapshot from last incomplete backup if there is one.
+        if quickMode {
+            incompleteSnapshot = LoadIncompleteSnapshot()
+        }
+
+        // If the listing operation is fast or there is an incomplete snapshot, list all chunks and
+        // put them in the cache.
+        if manager.storage.IsFastListing() || incompleteSnapshot != nil {
             LOG_INFO("BACKUP_LIST", "Listing all chunks")
             allChunks, _ := manager.SnapshotManager.ListAllFiles(manager.storage, "chunks/")
 
@@ -195,6 +219,39 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
             }
         }
 
+
+        if incompleteSnapshot != nil {
+
+            // This is the last chunk from the incomplete snapshot that can be found in the cache
+            lastCompleteChunk := -1
+            for i, chunkHash := range incompleteSnapshot.ChunkHashes {
+                chunkID := manager.config.GetChunkIDFromHash(chunkHash)
+                if _, ok := chunkCache[chunkID]; ok {
+                    lastCompleteChunk = i
+                } else {
+                    break
+                }
+            }
+
+            // Only keep those files whose chunks exist in the cache
+            var files []*Entry
+            for _, file := range incompleteSnapshot.Files {
+                if file.StartChunk <= lastCompleteChunk && file.EndChunk <= lastCompleteChunk {
+                    files = append(files, file)
+                } else {
+                    break
+                }
+            }
+            incompleteSnapshot.Files = files
+
+            // Remove incomplete chunks (they may not have been uploaded)
+            incompleteSnapshot.ChunkHashes = incompleteSnapshot.ChunkHashes[:lastCompleteChunk + 1]
+            incompleteSnapshot.ChunkLengths = incompleteSnapshot.ChunkLengths[:lastCompleteChunk + 1]            
+            remoteSnapshot = incompleteSnapshot
+        }
+
+    }
+
     var numberOfNewFileChunks int                 // number of new file chunks
     var totalUploadedFileChunkLength int64        // total length of uploaded file chunks
     var totalUploadedFileChunkBytes int64         // how many actual bytes have been uploaded
@@ -211,10 +268,11 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
     var modifiedEntries [] *Entry    // Files that has been modified or newly created
     var preservedEntries [] *Entry   // Files unchanges
 
-    // If the quick mode is enabled, we simply treat all files as if they were new, and break them into chunks.
+    // If the quick mode is disable and there isn't an incomplete snapshot from last (failed) backup, 
+    // we simply treat all files as if they were new, and break them into chunks.
     // Otherwise, we need to find those that are new or recently modified
 
-    if !quickMode {
+    if remoteSnapshot.Revision == 0 && incompleteSnapshot == nil {
         modifiedEntries = localSnapshot.Files
         for _, entry := range modifiedEntries {
             totalModifiedFileSize += entry.Size
@@ -268,7 +326,7 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
     var preservedChunkHashes []string
     var preservedChunkLengths []int
 
-    // For each preserved file, adjust the indices StartChunk and EndChunk.  This is done by finding gaps
+    // For each preserved file, adjust the StartChunk and EndChunk pointers.  This is done by finding gaps
     // between these indices and subtracting the number of deleted chunks.
     last := -1
     deletedChunks := 0
@@ -295,6 +353,7 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
     var uploadedEntries [] *Entry
     var uploadedChunkHashes []string
     var uploadedChunkLengths []int
+    var uploadedChunkLock = &sync.Mutex{}
 
     // the file reader implements the Reader interface. When an EOF is encounter, it opens the next file unless it
     // is the last file.
@@ -318,6 +377,37 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
     chunkMaker := CreateChunkMaker(manager.config, false)
     chunkUploader := CreateChunkUploader(manager.config, manager.storage, nil, threads, nil)
 
+    localSnapshotReady := false
+    var once sync.Once
+
+    if remoteSnapshot.Revision == 0 {
+        // In case an error occurs during the initial backup, save the incomplete snapshot
+        RunAtError = func() {
+            once.Do(
+                func() {
+                    if !localSnapshotReady {
+                        // Lock it to gain exclusive access to uploadedChunkHashes and uploadedChunkLengths
+                        uploadedChunkLock.Lock()
+                        for _, entry := range uploadedEntries {
+                            entry.EndChunk = -1
+                        }
+                        setEntryContent(uploadedEntries, uploadedChunkLengths, len(preservedChunkHashes))
+                        if len(preservedChunkHashes) > 0 {
+                            localSnapshot.ChunkHashes = preservedChunkHashes
+                            localSnapshot.ChunkHashes = append(localSnapshot.ChunkHashes, uploadedChunkHashes...)
+                            localSnapshot.ChunkLengths = preservedChunkLengths
+                            localSnapshot.ChunkLengths = append(localSnapshot.ChunkLengths, uploadedChunkLengths...)
+                        } else {
+                            localSnapshot.ChunkHashes = uploadedChunkHashes
+                            localSnapshot.ChunkLengths = uploadedChunkLengths
+                        }
+                        uploadedChunkLock.Unlock()
+                    }
+                    SaveIncompleteSnapshot(localSnapshot)
+                })
+        }
+    }
+
     if fileReader.CurrentFile != nil {
 
         LOG_TRACE("PACK_START", "Packing %s", fileReader.CurrentEntry.Path)
@@ -398,8 +488,11 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
                     chunkUploader.StartChunk(chunk, chunkIndex)
                 }
 
+                // Must lock it because the RunAtError function called by other threads may access these two slices
+                uploadedChunkLock.Lock()
                 uploadedChunkHashes = append(uploadedChunkHashes, hash)
                 uploadedChunkLengths = append(uploadedChunkLengths, chunkSize)
+                uploadedChunkLock.Unlock()
 
             },
             func (fileSize int64, hash string) (io.Reader, bool) {
@@ -445,6 +538,8 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
         localSnapshot.ChunkLengths = uploadedChunkLengths
     }
 
+    localSnapshotReady = true
+
     localSnapshot.EndTime = time.Now().Unix()
 
     err = manager.SnapshotManager.CheckSnapshot(localSnapshot)
@@ -455,10 +550,15 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
 
     localSnapshot.Tag = tag
     localSnapshot.Options = ""
-    if !quickMode {
+    if !quickMode || remoteSnapshot.Revision == 0 {
         localSnapshot.Options = "-hash"
     }
 
+    if _, found := os.LookupEnv("DUPLICACY_FAIL_SNAPSHOT"); found {
+        LOG_ERROR("SNAPSHOT_FAIL", "Artificially fail the backup for testing purposes")
+        return false
+    }
+
     if shadowCopy {
         if localSnapshot.Options == "" {
             localSnapshot.Options = "-vss"
@@ -505,6 +605,8 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
     manager.SnapshotManager.CleanSnapshotCache(localSnapshot, nil)
     LOG_INFO("BACKUP_END", "Backup for %s at revision %d completed", top, localSnapshot.Revision)
 
+    RunAtError = func() {}
+    RemoveIncompleteSnapshot()
 
     totalSnapshotChunks := len(localSnapshot.FileSequence) + len(localSnapshot.ChunkSequence) +
                         len(localSnapshot.LengthSequence)
@@ -981,7 +1083,7 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
     var existingFile, newFile *os.File
     var err error
     
-    preferencePath := GetDuplicacyPreferencePath(top)
+    preferencePath := GetDuplicacyPreferencePath()
     temporaryPath := path.Join(preferencePath, "temporary")
     fullPath := joinPath(top, entry.Path)
 

src/duplicacy_backupmanager_test.go

@@ -215,8 +215,9 @@ func TestBackupManager(t *testing.T) {
 
     time.Sleep(time.Duration(delay) * time.Second)
 
+    SetDuplicacyPreferencePath(testDir + "/repository1")
     backupManager := CreateBackupManager("host1", storage, testDir, password)
-    backupManager.SetupSnapshotCache(testDir + "/repository1", "default")
+    backupManager.SetupSnapshotCache("default")
 
     backupManager.Backup(testDir + "/repository1", /*quickMode=*/true, threads, "first", false, false)
     time.Sleep(time.Duration(delay) * time.Second)

src/duplicacy_log.go

@@ -160,6 +160,9 @@ const (
     otherExitCode     = 101
 )
 
+// This is the function to be called before exiting when an error occurs.
+var RunAtError func() = func() {}
+
 func CatchLogException() {
     if r := recover(); r != nil {
         switch e := r.(type) {
@@ -167,10 +170,12 @@ func CatchLogException() {
             if printStackTrace {
                 debug.PrintStack()
             }
+            RunAtError()
             os.Exit(duplicacyExitCode)
         default:
             fmt.Fprintf(os.Stderr, "%v\n", e)
             debug.PrintStack()
+            RunAtError()
             os.Exit(otherExitCode)
         }
     }

src/duplicacy_preference.go

@@ -24,53 +24,38 @@ type Preference struct {
     Keys map[string]string    `json:"keys"`
 }
 
+var preferencePath string
 var Preferences [] Preference
 
-// Compute .duplicacy directory path name:
+func LoadPreferences(repository string) bool {
-//  - if .duplicacy is a directory -> compute absolute path name and return it
-//  - if .duplicacy is a file -> assumed this file contains the real path name of .duplicacy
-//  - if pointed directory does not exits... return error
-func GetDuplicacyPreferencePath( repository string) (preferencePath string){
     
-    preferencePath = path.Join(repository, DUPLICACY_DIRECTORY) //TOKEEP
+    preferencePath = path.Join(repository, DUPLICACY_DIRECTORY)
     
     stat, err := os.Stat(preferencePath)
-    if err != nil && !os.IsNotExist(err) {
-        LOG_ERROR("DOT_DUPLICACY_PATH", "Failed to retrieve the information about the directory %s: %v",
-            repository, err)
-        return ""
-    }
-    
-    if stat != nil && stat.IsDir() {
-        // $repository/.duplicacy exists and is a directory --> we found the .duplicacy directory
-        return path.Clean(preferencePath)
-    }
-    
-    if stat != nil && stat.Mode().IsRegular() {
-        b, err := ioutil.ReadFile(preferencePath) // just pass the file name
     if err != nil {
-            LOG_ERROR("DOT_DUPLICACY_PATH", "Failed to read file %s: %v",
+        LOG_ERROR("PREFERENCE_PATH", "Failed to retrieve the information about the directory %s: %v", repository, err)
-                preferencePath, err)
+        return false
-            return ""
-        }
-        dotDuplicacyContent := string(b) // convert content to a 'string'
-        stat, err := os.Stat(dotDuplicacyContent)
-        if err != nil && !os.IsNotExist(err) {
-            LOG_ERROR("DOT_DUPLICACY_PATH", "Failed to retrieve the information about the directory %s: %v",
-                repository, err)
-            return ""
-        }
-        if stat != nil && stat.IsDir() {
-            // If expression read from .duplicacy file is a directory --> we found the .duplicacy directory
-            return path.Clean(dotDuplicacyContent)
-        }
-    }
-    return ""
     }
     
-func LoadPreferences(repository string) (bool) {
+    if !stat.IsDir() {
+        content, err := ioutil.ReadFile(preferencePath)
+        if err != nil {
+            LOG_ERROR("DOT_DUPLICACY_PATH", "Failed to locate the preference path: %v", err)
+            return false
+        }
+        realPreferencePath := string(content)
+        stat, err := os.Stat(realPreferencePath)
+        if err != nil {
+            LOG_ERROR("PREFERENCE_PATH", "Failed to retrieve the information about the directory %s: %v", content, err)
+            return false
+        }
+        if !stat.IsDir() {
+            LOG_ERROR("PREFERENCE_PATH", "The preference path %s is not a directory",  realPreferencePath)
+        }
+
+        preferencePath = realPreferencePath
+    }
     
-    preferencePath := GetDuplicacyPreferencePath(repository)
     description, err := ioutil.ReadFile(path.Join(preferencePath, "preferences"))
     if err != nil {
         LOG_ERROR("PREFERENCE_OPEN", "Failed to read the preference file from repository %s: %v", repository, err)
@@ -91,14 +76,27 @@ func LoadPreferences(repository string) (bool) {
     return true
 }
 
-func SavePreferences(repository string) (bool) {
+func GetDuplicacyPreferencePath() string {
+    if preferencePath == "" {
+        LOG_ERROR("PREFERENCE_PATH", "The preference path has not been set")
+        return ""
+    }
+    return preferencePath
+}
+
+// Normally 'preferencePath' is set in LoadPreferences; however, if LoadPreferences is not called, this function
+// provide another change to set 'preferencePath'
+func SetDuplicacyPreferencePath(p string) {
+    preferencePath = p
+}
+
+func SavePreferences() (bool) {
     description, err := json.MarshalIndent(Preferences, "", "    ")
     if err != nil {
         LOG_ERROR("PREFERENCE_MARSHAL", "Failed to marshal the repository preferences: %v", err)
         return false
     }
-    preferencePath := GetDuplicacyPreferencePath(repository)
+    preferenceFile := path.Join(GetDuplicacyPreferencePath(), "preferences")
-    preferenceFile := path.Join(preferencePath, "/preferences")
     
     err = ioutil.WriteFile(preferenceFile, description, 0644)
     if err != nil {

src/duplicacy_shadowcopy_windows.go

@@ -509,7 +509,7 @@ func CreateShadowCopy(top string, shadowCopy bool) (shadowTop string) {
 
     snapshotPath := uint16ArrayToString(properties.SnapshotDeviceObject)
     
-    preferencePath := GetDuplicacyPreferencePath(top)
+    preferencePath := GetDuplicacyPreferencePath()
     shadowLink =  preferencePath + "\\shadow"
     os.Remove(shadowLink)
     err = os.Symlink(snapshotPath + "\\", shadowLink)

src/duplicacy_snapshot.go

@@ -68,8 +68,7 @@ func CreateSnapshotFromDirectory(id string, top string) (snapshot *Snapshot, ski
 
     var patterns []string
     
-    preferencePath := GetDuplicacyPreferencePath(top)
+    patternFile, err := ioutil.ReadFile(path.Join(GetDuplicacyPreferencePath(), "filters"))
-    patternFile, err := ioutil.ReadFile(path.Join(preferencePath, "filters"))
     if err == nil {
         for _, pattern := range strings.Split(string(patternFile), "\n") {
             pattern = strings.TrimSpace(pattern)
@@ -138,6 +137,96 @@ func CreateSnapshotFromDirectory(id string, top string) (snapshot *Snapshot, ski
     return snapshot, skippedDirectories, skippedFiles, nil
 }
 
+// This is the struct used to save/load incomplete snapshots
+type IncompleteSnapshot struct {
+    Files [] *Entry
+    ChunkHashes []string
+    ChunkLengths [] int
+}
+
+// LoadIncompleteSnapshot loads the incomplete snapshot if it exists
+func LoadIncompleteSnapshot() (snapshot *Snapshot) {
+    snapshotFile := path.Join(GetDuplicacyPreferencePath(), "incomplete")
+    description, err := ioutil.ReadFile(snapshotFile)    
+    if err != nil {
+        return nil
+    }
+
+    var incompleteSnapshot IncompleteSnapshot
+
+    err = json.Unmarshal(description, &incompleteSnapshot)
+    if err != nil {
+        return nil
+    }
+
+    var chunkHashes []string
+    for _, chunkHash := range incompleteSnapshot.ChunkHashes {
+        hash, err := hex.DecodeString(chunkHash)
+        if err != nil {
+            return nil
+        }
+        chunkHashes = append(chunkHashes, string(hash))
+    }
+
+    snapshot = &Snapshot {
+        Files: incompleteSnapshot.Files,
+        ChunkHashes: chunkHashes,
+        ChunkLengths: incompleteSnapshot.ChunkLengths,
+    }      
+    LOG_INFO("INCOMPLETE_LOAD", "Incomplete snpashot loaded from %s", snapshotFile)
+    return snapshot
+}
+
+// SaveIncompleteSnapshot saves the incomplete snapshot under the preference directory
+func SaveIncompleteSnapshot(snapshot *Snapshot) {
+    var files []*Entry
+    for _, file := range snapshot.Files {
+        if file.EndChunk >= 0 {
+            file.Attributes = nil
+            files = append(files, file)
+        } else {
+            break
+        }
+    }
+    var chunkHashes []string
+    for _, chunkHash := range snapshot.ChunkHashes {
+        chunkHashes = append(chunkHashes, hex.EncodeToString([]byte(chunkHash)))
+    }
+
+    incompleteSnapshot := IncompleteSnapshot {
+        Files: files,
+        ChunkHashes: chunkHashes,
+        ChunkLengths: snapshot.ChunkLengths,
+    }
+
+    description, err := json.Marshal(incompleteSnapshot)
+    if err != nil {
+        LOG_WARN("INCOMPLETE_ENCODE", "Failed to encode the incomplete snapshot: %v", err)
+        return
+    }
+
+    snapshotFile := path.Join(GetDuplicacyPreferencePath(), "incomplete")    
+    err = ioutil.WriteFile(snapshotFile, description, 0644)
+    if err != nil {
+        LOG_WARN("INCOMPLETE_WRITE", "Failed to save the incomplete snapshot: %v", err)        
+        return
+    }
+
+    LOG_INFO("INCOMPLETE_SAVE", "Incomplete snapshot saved to %s", snapshotFile)
+}
+
+func RemoveIncompleteSnapshot() {
+    snapshotFile := path.Join(GetDuplicacyPreferencePath(), "incomplete")    
+    if stat, err := os.Stat(snapshotFile); err == nil && !stat.IsDir() {
+        err = os.Remove(snapshotFile)
+        if err != nil {
+            LOG_INFO("INCOMPLETE_SAVE", "Failed to remove ncomplete snapshot: %v", err)
+        } else {
+            LOG_INFO("INCOMPLETE_SAVE", "Removed incomplete snapshot %s", snapshotFile)    
+        }
+    }
+}
+
 // CreateSnapshotFromDescription creates a snapshot from json decription.
 func CreateSnapshotFromDescription(description []byte) (snapshot *Snapshot, err error) {
 

src/duplicacy_snapshotmanager.go

@@ -1496,7 +1496,7 @@ func (manager *SnapshotManager) resurrectChunk(fossilPath string, chunkID string
 // Note that a snapshot being created when step 2 is in progress may reference a fossil.  To avoid this
 // problem, never remove the lastest revision (unless exclusive is true), and only cache chunks referenced
 // by the lastest revision.
-func (manager *SnapshotManager) PruneSnapshots(top string, selfID string, snapshotID string, revisionsToBeDeleted []int,
+func (manager *SnapshotManager) PruneSnapshots(selfID string, snapshotID string, revisionsToBeDeleted []int,
                                                tags []string, retentions []string,
                                                exhaustive bool, exclusive bool, ignoredIDs []string,
                                                dryRun bool, deleteOnly bool, collectOnly bool) bool {
@@ -1511,7 +1511,7 @@ func (manager *SnapshotManager) PruneSnapshots(top string, selfID string, snapsh
         LOG_WARN("DELETE_OPTIONS", "Tags or retention policy will be ignored if at least one revision is specified")
     }
     
-    preferencePath := GetDuplicacyPreferencePath(top)
+    preferencePath := GetDuplicacyPreferencePath()
     logDir := path.Join(preferencePath, "logs")
     os.Mkdir(logDir, 0700)
     logFileName := path.Join(logDir, time.Now().Format("prune-log-20060102-150405"))

src/duplicacy_snapshotmanager_test.go

@@ -248,11 +248,11 @@ func TestSingleRepositoryPrune(t *testing.T) {
     checkTestSnapshots(snapshotManager, 3, 0)
 
     t.Logf("Removing snapshot repository1 revision 1 with --exclusive")
-    snapshotManager.PruneSnapshots(testDir, "repository1", "repository1", []int{1}, []string{}, []string{}, false, true, []string{}, false, false, false)
+    snapshotManager.PruneSnapshots("repository1", "repository1", []int{1}, []string{}, []string{}, false, true, []string{}, false, false, false)
     checkTestSnapshots(snapshotManager, 2, 0)
 
     t.Logf("Removing snapshot repository1 revision 2 without --exclusive")
-    snapshotManager.PruneSnapshots(testDir, "repository1", "repository1", []int{2}, []string{}, []string{}, false, false, []string{}, false, false, false)
+    snapshotManager.PruneSnapshots("repository1", "repository1", []int{2}, []string{}, []string{}, false, false, []string{}, false, false, false)
     checkTestSnapshots(snapshotManager, 1, 2)
 
     t.Logf("Creating 1 snapshot")
@@ -261,7 +261,7 @@ func TestSingleRepositoryPrune(t *testing.T) {
     checkTestSnapshots(snapshotManager, 2, 2)
 
     t.Logf("Prune without removing any snapshots -- fossils will be deleted")
-    snapshotManager.PruneSnapshots(testDir, "repository1", "repository1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false)
+    snapshotManager.PruneSnapshots("repository1", "repository1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false)
     checkTestSnapshots(snapshotManager, 2, 0)
 }
 
@@ -288,11 +288,11 @@ func TestSingleHostPrune(t *testing.T) {
     checkTestSnapshots(snapshotManager, 3, 0)
 
     t.Logf("Removing snapshot vm1@host1 revision 1 without --exclusive")
-    snapshotManager.PruneSnapshots(testDir, "vm1@host1", "vm1@host1", []int{1}, []string{}, []string{}, false, false, []string{}, false, false, false)
+    snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{1}, []string{}, []string{}, false, false, []string{}, false, false, false)
     checkTestSnapshots(snapshotManager, 2, 2)
 
     t.Logf("Prune without removing any snapshots -- no fossils will be deleted")
-    snapshotManager.PruneSnapshots(testDir, "vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false)
+    snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false)
     checkTestSnapshots(snapshotManager, 2, 2)
 
     t.Logf("Creating 1 snapshot")
@@ -301,7 +301,7 @@ func TestSingleHostPrune(t *testing.T) {
     checkTestSnapshots(snapshotManager, 3, 2)
 
     t.Logf("Prune without removing any snapshots -- fossils will be deleted")
-    snapshotManager.PruneSnapshots(testDir, "vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false)
+    snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false)
     checkTestSnapshots(snapshotManager, 3, 0)
 
 }
@@ -329,11 +329,11 @@ func TestMultipleHostPrune(t *testing.T) {
     checkTestSnapshots(snapshotManager, 3, 0)
 
     t.Logf("Removing snapshot vm1@host1 revision 1 without --exclusive")
-    snapshotManager.PruneSnapshots(testDir, "vm1@host1", "vm1@host1", []int{1}, []string{}, []string{}, false, false, []string{}, false, false, false)
+    snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{1}, []string{}, []string{}, false, false, []string{}, false, false, false)
     checkTestSnapshots(snapshotManager, 2, 2)
 
     t.Logf("Prune without removing any snapshots -- no fossils will be deleted")
-    snapshotManager.PruneSnapshots(testDir, "vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false)
+    snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false)
     checkTestSnapshots(snapshotManager, 2, 2)
 
     t.Logf("Creating 1 snapshot")
@@ -342,7 +342,7 @@ func TestMultipleHostPrune(t *testing.T) {
     checkTestSnapshots(snapshotManager, 3, 2)
 
     t.Logf("Prune without removing any snapshots -- no fossils will be deleted")
-    snapshotManager.PruneSnapshots(testDir, "vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false)
+    snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false)
     checkTestSnapshots(snapshotManager, 3, 2)
 
     t.Logf("Creating 1 snapshot")
@@ -351,7 +351,7 @@ func TestMultipleHostPrune(t *testing.T) {
     checkTestSnapshots(snapshotManager, 4, 2)
 
     t.Logf("Prune without removing any snapshots -- fossils will be deleted")
-    snapshotManager.PruneSnapshots(testDir, "vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false)
+    snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false)
     checkTestSnapshots(snapshotManager, 4, 0)
 }
 
@@ -376,7 +376,7 @@ func TestPruneAndResurrect(t *testing.T) {
     checkTestSnapshots(snapshotManager, 2, 0)
 
     t.Logf("Removing snapshot vm1@host1 revision 1 without --exclusive")
-    snapshotManager.PruneSnapshots(testDir, "vm1@host1", "vm1@host1", []int{1}, []string{}, []string{}, false, false, []string{}, false, false, false)
+    snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{1}, []string{}, []string{}, false, false, []string{}, false, false, false)
     checkTestSnapshots(snapshotManager, 1, 2)
 
     t.Logf("Creating 1 snapshot")
@@ -385,7 +385,7 @@ func TestPruneAndResurrect(t *testing.T) {
     checkTestSnapshots(snapshotManager, 2, 2)
 
     t.Logf("Prune without removing any snapshots -- one fossil will be resurrected")
-    snapshotManager.PruneSnapshots(testDir, "vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false)
+    snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false)
     checkTestSnapshots(snapshotManager, 2, 0)
 }
 
@@ -413,11 +413,11 @@ func TestInactiveHostPrune(t *testing.T) {
     checkTestSnapshots(snapshotManager, 3, 0)
 
     t.Logf("Removing snapshot vm1@host1 revision 1")
-    snapshotManager.PruneSnapshots(testDir, "vm1@host1", "vm1@host1", []int{1}, []string{}, []string{}, false, false, []string{}, false, false, false)
+    snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{1}, []string{}, []string{}, false, false, []string{}, false, false, false)
     checkTestSnapshots(snapshotManager, 2, 2)
 
     t.Logf("Prune without removing any snapshots -- no fossils will be deleted")
-    snapshotManager.PruneSnapshots(testDir, "vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false)
+    snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false)
     checkTestSnapshots(snapshotManager, 2, 2)
 
     t.Logf("Creating 1 snapshot")
@@ -426,7 +426,7 @@ func TestInactiveHostPrune(t *testing.T) {
     checkTestSnapshots(snapshotManager, 3, 2)
 
     t.Logf("Prune without removing any snapshots -- fossils will be deleted")
-    snapshotManager.PruneSnapshots(testDir, "vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false)
+    snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false)
     checkTestSnapshots(snapshotManager, 3, 0)
 }
 
@@ -454,14 +454,14 @@ func TestRetentionPolicy(t *testing.T) {
     checkTestSnapshots(snapshotManager, 30, 0)
 
     t.Logf("Removing snapshot vm1@host1 0:20 with --exclusive")
-    snapshotManager.PruneSnapshots(testDir, "vm1@host1", "vm1@host1", []int{}, []string{}, []string{"0:20"}, false, true, []string{}, false, false, false)
+    snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{"0:20"}, false, true, []string{}, false, false, false)
     checkTestSnapshots(snapshotManager, 19, 0)
 
     t.Logf("Removing snapshot vm1@host1 -k 0:20 with --exclusive")
-    snapshotManager.PruneSnapshots(testDir, "vm1@host1", "vm1@host1", []int{}, []string{}, []string{"0:20"}, false, true, []string{}, false, false, false)
+    snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{"0:20"}, false, true, []string{}, false, false, false)
     checkTestSnapshots(snapshotManager, 19, 0)
 
     t.Logf("Removing snapshot vm1@host1 -k 3:14 -k 2:7 with --exclusive")
-    snapshotManager.PruneSnapshots(testDir, "vm1@host1", "vm1@host1", []int{}, []string{}, []string{"3:14", "2:7"}, false, true, []string{}, false, false, false)
+    snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{"3:14", "2:7"}, false, true, []string{}, false, false, false)
     checkTestSnapshots(snapshotManager, 12, 0)
 }

src/duplicacy_storage.go

@@ -75,13 +75,9 @@ func (storage *RateLimitedStorage) SetRateLimits(downloadRateLimit int, uploadRa
     storage.UploadRateLimit = uploadRateLimit
 }
 
-func checkHostKey(repository string, hostname string, remote net.Addr, key ssh.PublicKey) error {
+func checkHostKey(hostname string, remote net.Addr, key ssh.PublicKey) error {
     
-    if len(repository) == 0 {
+    preferencePath := GetDuplicacyPreferencePath()
-        return nil
-    }
-    
-    preferencePath := GetDuplicacyPreferencePath(repository)
     hostFile := path.Join(preferencePath, "known_hosts")
     file, err := os.OpenFile(hostFile, os.O_RDWR | os.O_CREATE, 0600)
     if err != nil {
@@ -126,7 +122,7 @@ func checkHostKey(repository string, hostname string, remote net.Addr, key ssh.P
 }
 
 // CreateStorage creates a storage object based on the provide storage URL.
-func CreateStorage(repository string, preference Preference, resetPassword bool, threads int) (storage Storage) {
+func CreateStorage(preference Preference, resetPassword bool, threads int) (storage Storage) {
 
     storageURL := preference.StorageURL
 
@@ -282,7 +278,7 @@ func CreateStorage(repository string, preference Preference, resetPassword bool,
         }
 
         hostKeyChecker := func(hostname string, remote net.Addr, key ssh.PublicKey) error {
-            return checkHostKey(repository, hostname, remote, key)
+            return checkHostKey(hostname, remote, key)
         }
 
         sftpStorage, err := CreateSFTPStorage(server, port, username, storageDir, authMethods, hostKeyChecker, threads)