diff --git a/duplicacy/duplicacy_main.go b/duplicacy/duplicacy_main.go index 5ec25b7..41dcb57 100644 --- a/duplicacy/duplicacy_main.go +++ b/duplicacy/duplicacy_main.go @@ -794,6 +794,7 @@ func restoreRepository(context *cli.Context) { setOwner := !context.Bool("ignore-owner") showStatistics := context.Bool("stats") + persist := context.Bool("persist") var patterns []string for _, pattern := range context.Args() { @@ -824,7 +825,7 @@ func restoreRepository(context *cli.Context) { loadRSAPrivateKey(context.String("key"), preference, backupManager, false) backupManager.SetupSnapshotCache(preference.Name) - backupManager.Restore(repository, revision, true, quickMode, threads, overwrite, deleteMode, setOwner, showStatistics, patterns) + backupManager.Restore(repository, revision, true, quickMode, threads, overwrite, deleteMode, setOwner, showStatistics, patterns, persist) runScript(context, preference.Name, "post") } @@ -934,9 +935,10 @@ func checkSnapshots(context *cli.Context) { checkChunks := context.Bool("chunks") searchFossils := context.Bool("fossils") resurrect := context.Bool("resurrect") + persist := context.Bool("persist") backupManager.SetupSnapshotCache(preference.Name) - backupManager.SnapshotManager.CheckSnapshots(id, revisions, tag, showStatistics, showTabular, checkFiles, checkChunks, searchFossils, resurrect, threads) + backupManager.SnapshotManager.CheckSnapshots(id, revisions, tag, showStatistics, showTabular, checkFiles, checkChunks, searchFossils, resurrect, threads, persist) runScript(context, preference.Name, "post") } @@ -1510,6 +1512,10 @@ func main() { Usage: "the RSA private key to decrypt file chunks", Argument: "", }, + cli.BoolFlag{ + Name: "persist", + Usage: "continue processing despite chunk errors or existing files (without -overwrite), reporting any affected files", + }, }, Usage: "Restore the repository to a previously saved snapshot", ArgsUsage: "[--] [pattern] ...", @@ -1627,6 +1633,10 @@ func main() { Usage: "number of threads used to verify chunks", Argument: "", }, + cli.BoolFlag{ + Name: "persist", + Usage: "continue processing despite chunk errors, reporting any affected (corrupted) files", + }, }, Usage: "Check the integrity of snapshots", ArgsUsage: " ", diff --git a/src/duplicacy_backupmanager.go b/src/duplicacy_backupmanager.go index 2e5a908..97623d5 100644 --- a/src/duplicacy_backupmanager.go +++ b/src/duplicacy_backupmanager.go @@ -8,6 +8,7 @@ import ( "bytes" "encoding/hex" "encoding/json" + "errors" "fmt" "io" "os" @@ -741,7 +742,7 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta // the same as 'top'. 'quickMode' will bypass files with unchanged sizes and timestamps. 'deleteMode' will // remove local files that don't exist in the snapshot. 'patterns' is used to include/exclude certain files. func (manager *BackupManager) Restore(top string, revision int, inPlace bool, quickMode bool, threads int, overwrite bool, - deleteMode bool, setOwner bool, showStatistics bool, patterns []string) bool { + deleteMode bool, setOwner bool, showStatistics bool, patterns []string, allowFailures bool) bool { startTime := time.Now().Unix() @@ -809,6 +810,18 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu var totalFileSize int64 var downloadedFileSize int64 + var failedFileSize int64 + var skippedFileSize int64 + + var downloadedFiles []*Entry + var skippedFiles []*Entry + + // for storing failed files and reason for failure + type FailedEntry struct { + entry *Entry + failReason string + } + var failedFiles []*FailedEntry i := 0 for _, entry := range remoteSnapshot.Files { @@ -827,6 +840,8 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu i++ if quickMode && local.IsSameAs(entry) { LOG_TRACE("RESTORE_SKIP", "File %s unchanged (by size and timestamp)", local.Path) + skippedFileSize += entry.Size + skippedFiles = append(skippedFiles, entry) skipped = true } } @@ -892,14 +907,13 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu // Sort entries by their starting chunks in order to linearize the access to the chunk chain. sort.Sort(ByChunk(fileEntries)) - chunkDownloader := CreateChunkDownloader(manager.config, manager.storage, nil, showStatistics, threads) + chunkDownloader := CreateChunkDownloader(manager.config, manager.storage, nil, showStatistics, threads, allowFailures) chunkDownloader.AddFiles(remoteSnapshot, fileEntries) chunkMaker := CreateChunkMaker(manager.config, true) startDownloadingTime := time.Now().Unix() - var downloadedFiles []*Entry // Now download files one by one for _, file := range fileEntries { @@ -909,12 +923,16 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu if quickMode { if file.IsSameAsFileInfo(stat) { LOG_TRACE("RESTORE_SKIP", "File %s unchanged (by size and timestamp)", file.Path) + skippedFileSize += file.Size + skippedFiles = append(skippedFiles, file) continue } } if file.Size == 0 && file.IsSameAsFileInfo(stat) { LOG_TRACE("RESTORE_SKIP", "File %s unchanged (size 0)", file.Path) + skippedFileSize += file.Size + skippedFiles = append(skippedFiles, file) continue } } else { @@ -937,17 +955,33 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu file.RestoreMetadata(fullPath, nil, setOwner) if !showStatistics { LOG_INFO("DOWNLOAD_DONE", "Downloaded %s (0)", file.Path) + downloadedFileSize += file.Size + downloadedFiles = append(downloadedFiles, file) } continue } - if manager.RestoreFile(chunkDownloader, chunkMaker, file, top, inPlace, overwrite, showStatistics, - totalFileSize, downloadedFileSize, startDownloadingTime) { + downloaded, err := manager.RestoreFile(chunkDownloader, chunkMaker, file, top, inPlace, overwrite, showStatistics, + totalFileSize, downloadedFileSize, startDownloadingTime, allowFailures) + if err != nil { + // RestoreFile produced an error + failedFileSize += file.Size + failedFiles = append(failedFiles, &FailedEntry{file, err.Error()}) + continue + } + + // No error + if downloaded { + // No error, file was restored downloadedFileSize += file.Size downloadedFiles = append(downloadedFiles, file) + file.RestoreMetadata(fullPath, nil, setOwner) + } else { + // No error, file was skipped + skippedFileSize += file.Size + skippedFiles = append(skippedFiles, file) } - file.RestoreMetadata(fullPath, nil, setOwner) } if deleteMode && len(patterns) == 0 { @@ -971,6 +1005,16 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu for _, file := range downloadedFiles { LOG_INFO("DOWNLOAD_DONE", "Downloaded %s (%d)", file.Path, file.Size) } + for _, file := range skippedFiles { + LOG_INFO("DOWNLOAD_SKIP", "Skipped %s (%d)", file.Path, file.Size) + } + } + + if len(failedFiles) > 0 { + for _, failed := range failedFiles { + file := failed.entry + LOG_WARN("RESTORE_STATS", "Restore failed %s (%d): %s", file.Path, file.Size, failed.failReason) + } } LOG_INFO("RESTORE_END", "Restored %s to revision %d", top, revision) @@ -978,6 +1022,13 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu LOG_INFO("RESTORE_STATS", "Files: %d total, %s bytes", len(fileEntries), PrettySize(totalFileSize)) LOG_INFO("RESTORE_STATS", "Downloaded %d file, %s bytes, %d chunks", len(downloadedFiles), PrettySize(downloadedFileSize), chunkDownloader.numberOfDownloadedChunks) + LOG_INFO("RESTORE_STATS", "Skipped %d file, %s bytes", + len(skippedFiles), PrettySize(skippedFileSize)) + LOG_INFO("RESTORE_STATS", "Failed %d file, %s bytes", + len(failedFiles), PrettySize(failedFileSize)) + } + if len(failedFiles) > 0 { + LOG_WARN("RESTORE_STATS", "Some files could not be restored") } runningTime := time.Now().Unix() - startTime @@ -1149,8 +1200,11 @@ func (manager *BackupManager) UploadSnapshot(chunkMaker *ChunkMaker, uploader *C // Restore downloads a file from the storage. If 'inPlace' is false, the download file is saved first to a temporary // file under the .duplicacy directory and then replaces the existing one. Otherwise, the existing file will be // overwritten directly. +// Return: true, nil: Restored file; +// false, nil: Skipped file; +// false, error: Failure to restore file (only if allowFailures == true) func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chunkMaker *ChunkMaker, entry *Entry, top string, inPlace bool, overwrite bool, - showStatistics bool, totalFileSize int64, downloadedFileSize int64, startTime int64) bool { + showStatistics bool, totalFileSize int64, downloadedFileSize int64, startTime int64, allowFailures bool) (bool, error) { LOG_TRACE("DOWNLOAD_START", "Downloading %s", entry.Path) @@ -1161,6 +1215,14 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun temporaryPath := path.Join(preferencePath, "temporary") fullPath := joinPath(top, entry.Path) + // Function to call on errors ignored by allowFailures + var onFailure LogFunc + if allowFailures { + onFailure = LOG_WARN + } else { + onFailure = LOG_ERROR + } + defer func() { if existingFile != nil { existingFile.Close() @@ -1195,7 +1257,7 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun existingFile, err = os.OpenFile(fullPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) if err != nil { LOG_ERROR("DOWNLOAD_CREATE", "Failed to create the file %s for in-place writing: %v", fullPath, err) - return false + return false, nil } n := int64(1) @@ -1207,18 +1269,18 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun _, err = existingFile.Seek(entry.Size-n, 0) if err != nil { LOG_ERROR("DOWNLOAD_CREATE", "Failed to resize the initial file %s for in-place writing: %v", fullPath, err) - return false + return false, nil } _, err = existingFile.Write([]byte("\x00\x00")[:n]) if err != nil { LOG_ERROR("DOWNLOAD_CREATE", "Failed to initialize the sparse file %s for in-place writing: %v", fullPath, err) - return false + return false, nil } existingFile.Close() existingFile, err = os.Open(fullPath) if err != nil { LOG_ERROR("DOWNLOAD_OPEN", "Can't reopen the initial file just created: %v", err) - return false + return false, nil } isNewFile = true } @@ -1226,10 +1288,43 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun LOG_TRACE("DOWNLOAD_OPEN", "Can't open the existing file: %v", err) } } else { + // File already exists. Read file and hash entire contents. If fileHash == entry.Hash, skip file. + // This is done before additional processing so that any identical files can be skipped regardless of the + // -overwrite option + fileHasher := manager.config.NewFileHasher() + buffer := make([]byte, 64*1024) + + for { + n, err := existingFile.Read(buffer) + if err == io.EOF { + break + } + if err != nil { + LOG_ERROR("DOWNLOAD_OPEN", "Failed to read existing file: %v", err) + return false, nil + } + if n > 0 { + fileHasher.Write(buffer[:n]) + } + } + + fileHash := hex.EncodeToString(fileHasher.Sum(nil)) + + if fileHash == entry.Hash && fileHash != "" { + LOG_TRACE("DOWNLOAD_SKIP", "File %s unchanged (by hash)", entry.Path) + return false, nil + } + + // fileHash != entry.Hash, warn/error depending on -overwrite option if !overwrite { - LOG_ERROR("DOWNLOAD_OVERWRITE", - "File %s already exists. Please specify the -overwrite option to continue", entry.Path) - return false + var msg string + if allowFailures { + msg = "File %s already exists. Please specify the -overwrite option to overwrite" + } else { + msg = "File %s already exists. Please specify the -overwrite option to continue" + } + onFailure("DOWNLOAD_OVERWRITE", msg, entry.Path) // will exit program here if allowFailures = false + return false, errors.New("file exists") } } @@ -1299,7 +1394,7 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun } if err != nil { LOG_ERROR("DOWNLOAD_SPLIT", "Failed to read existing file: %v", err) - return false + return false, nil } } if count > 0 { @@ -1339,9 +1434,11 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun return nil, false }) } + + // This is an additional check comparing fileHash to entry.Hash above, so this should no longer occur if fileHash == entry.Hash && fileHash != "" { LOG_TRACE("DOWNLOAD_SKIP", "File %s unchanged (by hash)", entry.Path) - return false + return false, nil } } @@ -1369,7 +1466,7 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun existingFile, err = os.OpenFile(fullPath, os.O_RDWR, 0) if err != nil { LOG_ERROR("DOWNLOAD_OPEN", "Failed to open the file %s for in-place writing", fullPath) - return false + return false, nil } } @@ -1401,7 +1498,7 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun _, err = existingFile.Seek(offset, 0) if err != nil { LOG_ERROR("DOWNLOAD_SEEK", "Failed to set the offset to %d for file %s: %v", offset, fullPath, err) - return false + return false, nil } // Check if the chunk is available in the existing file @@ -1411,19 +1508,21 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun _, err := io.CopyN(hasher, existingFile, int64(existingLengths[j])) if err != nil { LOG_ERROR("DOWNLOAD_READ", "Failed to read the existing chunk %s: %v", hash, err) - return false + return false, nil } if IsDebugging() { LOG_DEBUG("DOWNLOAD_UNCHANGED", "Chunk %s is unchanged", manager.config.GetChunkIDFromHash(hash)) } } else { chunk := chunkDownloader.WaitForChunk(i) - _, err = existingFile.Write(chunk.GetBytes()[start:end]) - if err != nil { - LOG_ERROR("DOWNLOAD_WRITE", "Failed to write to the file: %v", err) - return false + if !chunk.isBroken { // only write if chunk downloaded correctly + _, err = existingFile.Write(chunk.GetBytes()[start:end]) + if err != nil { + LOG_ERROR("DOWNLOAD_WRITE", "Failed to write to the file: %v", err) + return false, nil + } + hasher.Write(chunk.GetBytes()[start:end]) } - hasher.Write(chunk.GetBytes()[start:end]) } offset += int64(end - start) @@ -1432,15 +1531,15 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun // Must truncate the file if the new size is smaller if err = existingFile.Truncate(offset); err != nil { LOG_ERROR("DOWNLOAD_TRUNCATE", "Failed to truncate the file at %d: %v", offset, err) - return false + return false, nil } // Verify the download by hash hash := hex.EncodeToString(hasher.Sum(nil)) if hash != entry.Hash && hash != "" && entry.Hash != "" && !strings.HasPrefix(entry.Hash, "#") { - LOG_ERROR("DOWNLOAD_HASH", "File %s has a mismatched hash: %s instead of %s (in-place)", + onFailure("DOWNLOAD_HASH", "File %s has a mismatched hash: %s instead of %s (in-place)", fullPath, "", entry.Hash) - return false + return false, errors.New("file corrupt (hash mismatch)") } } else { @@ -1449,7 +1548,7 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun newFile, err = os.OpenFile(temporaryPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) if err != nil { LOG_ERROR("DOWNLOAD_OPEN", "Failed to open file for writing: %v", err) - return false + return false, nil } hasher := manager.config.NewFileHasher() @@ -1488,21 +1587,23 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun if !hasLocalCopy { chunk := chunkDownloader.WaitForChunk(i) // If the chunk was downloaded from the storage, we may still need a portion of it. - start := 0 - if i == entry.StartChunk { - start = entry.StartOffset + if !chunk.isBroken { // only get data if chunk downloaded correctly + start := 0 + if i == entry.StartChunk { + start = entry.StartOffset + } + end := chunk.GetLength() + if i == entry.EndChunk { + end = entry.EndOffset + } + data = chunk.GetBytes()[start:end] } - end := chunk.GetLength() - if i == entry.EndChunk { - end = entry.EndOffset - } - data = chunk.GetBytes()[start:end] } _, err = newFile.Write(data) if err != nil { LOG_ERROR("DOWNLOAD_WRITE", "Failed to write file: %v", err) - return false + return false, nil } hasher.Write(data) @@ -1511,9 +1612,9 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun hash := hex.EncodeToString(hasher.Sum(nil)) if hash != entry.Hash && hash != "" && entry.Hash != "" && !strings.HasPrefix(entry.Hash, "#") { - LOG_ERROR("DOWNLOAD_HASH", "File %s has a mismatched hash: %s instead of %s", + onFailure("DOWNLOAD_HASH", "File %s has a mismatched hash: %s instead of %s", entry.Path, hash, entry.Hash) - return false + return false, errors.New("file corrupt (hash mismatch)") } if existingFile != nil { @@ -1527,20 +1628,20 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun err = os.Remove(fullPath) if err != nil && !os.IsNotExist(err) { LOG_ERROR("DOWNLOAD_REMOVE", "Failed to remove the old file: %v", err) - return false + return false, nil } err = os.Rename(temporaryPath, fullPath) if err != nil { LOG_ERROR("DOWNLOAD_RENAME", "Failed to rename the file %s to %s: %v", temporaryPath, fullPath, err) - return false + return false, nil } } if !showStatistics { LOG_INFO("DOWNLOAD_DONE", "Downloaded %s (%d)", entry.Path, entry.Size) } - return true + return true, nil } // CopySnapshots copies the specified snapshots from one storage to the other. @@ -1705,7 +1806,7 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho LOG_DEBUG("SNAPSHOT_COPY", "Chunks to copy = %d, to skip = %d, total = %d", chunksToCopy, chunksToSkip, chunksToCopy+chunksToSkip) LOG_DEBUG("SNAPSHOT_COPY", "Total chunks in source snapshot revisions = %d\n", len(chunks)) - chunkDownloader := CreateChunkDownloader(manager.config, manager.storage, nil, false, threads) + chunkDownloader := CreateChunkDownloader(manager.config, manager.storage, nil, false, threads, false) chunkUploader := CreateChunkUploader(otherManager.config, otherManager.storage, nil, threads, func(chunk *Chunk, chunkIndex int, skipped bool, chunkSize int, uploadSize int) { diff --git a/src/duplicacy_backupmanager_test.go b/src/duplicacy_backupmanager_test.go index eab2d82..12afbc1 100644 --- a/src/duplicacy_backupmanager_test.go +++ b/src/duplicacy_backupmanager_test.go @@ -247,7 +247,7 @@ func TestBackupManager(t *testing.T) { time.Sleep(time.Duration(delay) * time.Second) SetDuplicacyPreferencePath(testDir + "/repository2/.duplicacy") backupManager.Restore(testDir+"/repository2", threads /*inPlace=*/, false /*quickMode=*/, false, threads /*overwrite=*/, true, - /*deleteMode=*/ false /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, nil) + /*deleteMode=*/ false /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, nil /*allowFailures=*/, false) for _, f := range []string{"file1", "file2", "dir1/file3"} { if _, err := os.Stat(testDir + "/repository2/" + f); os.IsNotExist(err) { @@ -271,7 +271,7 @@ func TestBackupManager(t *testing.T) { time.Sleep(time.Duration(delay) * time.Second) SetDuplicacyPreferencePath(testDir + "/repository2/.duplicacy") backupManager.Restore(testDir+"/repository2", 2 /*inPlace=*/, true /*quickMode=*/, true, threads /*overwrite=*/, true, - /*deleteMode=*/ false /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, nil) + /*deleteMode=*/ false /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, nil /*allowFailures=*/, false) for _, f := range []string{"file1", "file2", "dir1/file3"} { hash1 := getFileHash(testDir + "/repository1/" + f) @@ -299,7 +299,7 @@ func TestBackupManager(t *testing.T) { SetDuplicacyPreferencePath(testDir + "/repository2/.duplicacy") backupManager.Restore(testDir+"/repository2", 3 /*inPlace=*/, true /*quickMode=*/, false, threads /*overwrite=*/, true, - /*deleteMode=*/ true /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, nil) + /*deleteMode=*/ true /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, nil /*allowFailures=*/, false) for _, f := range []string{"file1", "file2", "dir1/file3"} { hash1 := getFileHash(testDir + "/repository1/" + f) @@ -326,7 +326,7 @@ func TestBackupManager(t *testing.T) { os.Remove(testDir + "/repository1/dir1/file3") SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy") backupManager.Restore(testDir+"/repository1", 3 /*inPlace=*/, true /*quickMode=*/, false, threads /*overwrite=*/, true, - /*deleteMode=*/ false /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, []string{"+file2", "+dir1/file3", "-*"}) + /*deleteMode=*/ false /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, []string{"+file2", "+dir1/file3", "-*"} /*allowFailures=*/, false) for _, f := range []string{"file1", "file2", "dir1/file3"} { hash1 := getFileHash(testDir + "/repository1/" + f) @@ -341,7 +341,7 @@ func TestBackupManager(t *testing.T) { t.Errorf("Expected 3 snapshots but got %d", numberOfSnapshots) } backupManager.SnapshotManager.CheckSnapshots( /*snapshotID*/ "host1" /*revisions*/, []int{1, 2, 3} /*tag*/, "", - /*showStatistics*/ false /*showTabular*/, false /*checkFiles*/, false /*checkChunks*/, false /*searchFossils*/, false /*resurrect*/, false, 1) + /*showStatistics*/ false /*showTabular*/, false /*checkFiles*/, false /*checkChunks*/, false /*searchFossils*/, false /*resurrect*/, false, 1 /*allowFailures*/, false) backupManager.SnapshotManager.PruneSnapshots("host1", "host1" /*revisions*/, []int{1} /*tags*/, nil /*retentions*/, nil, /*exhaustive*/ false /*exclusive=*/, false /*ignoredIDs*/, nil /*dryRun*/, false /*deleteOnly*/, false /*collectOnly*/, false, 1) numberOfSnapshots = backupManager.SnapshotManager.ListSnapshots( /*snapshotID*/ "host1" /*revisionsToList*/, nil /*tag*/, "" /*showFiles*/, false /*showChunks*/, false) @@ -349,7 +349,7 @@ func TestBackupManager(t *testing.T) { t.Errorf("Expected 2 snapshots but got %d", numberOfSnapshots) } backupManager.SnapshotManager.CheckSnapshots( /*snapshotID*/ "host1" /*revisions*/, []int{2, 3} /*tag*/, "", - /*showStatistics*/ false /*showTabular*/, false /*checkFiles*/, false /*checkChunks*/, false /*searchFossils*/, false /*resurrect*/, false, 1) + /*showStatistics*/ false /*showTabular*/, false /*checkFiles*/, false /*checkChunks*/, false /*searchFossils*/, false /*resurrect*/, false, 1 /*allowFailures*/, false) backupManager.Backup(testDir+"/repository1" /*quickMode=*/, false, threads, "fourth", false, false, 0, false) backupManager.SnapshotManager.PruneSnapshots("host1", "host1" /*revisions*/, nil /*tags*/, nil /*retentions*/, nil, /*exhaustive*/ false /*exclusive=*/, true /*ignoredIDs*/, nil /*dryRun*/, false /*deleteOnly*/, false /*collectOnly*/, false, 1) @@ -358,9 +358,338 @@ func TestBackupManager(t *testing.T) { t.Errorf("Expected 3 snapshots but got %d", numberOfSnapshots) } backupManager.SnapshotManager.CheckSnapshots( /*snapshotID*/ "host1" /*revisions*/, []int{2, 3, 4} /*tag*/, "", - /*showStatistics*/ false /*showTabular*/, false /*checkFiles*/, false /*checkChunks*/, false /*searchFossils*/, false /*resurrect*/, false, 1) + /*showStatistics*/ false /*showTabular*/, false /*checkFiles*/, false /*checkChunks*/, false /*searchFossils*/, false /*resurrect*/, false, 1 /*allowFailures*/, false) /*buf := make([]byte, 1<<16) runtime.Stack(buf, true) fmt.Printf("%s", buf)*/ } + +// Create file with random file with certain seed +func createRandomFileSeeded(path string, maxSize int, seed int64) { + rand.Seed(seed) + file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) + if err != nil { + LOG_ERROR("RANDOM_FILE", "Can't open %s for writing: %v", path, err) + return + } + + defer file.Close() + + size := maxSize/2 + rand.Int()%(maxSize/2) + + buffer := make([]byte, 32*1024) + for size > 0 { + bytes := size + if bytes > cap(buffer) { + bytes = cap(buffer) + } + rand.Read(buffer[:bytes]) + bytes, err = file.Write(buffer[:bytes]) + if err != nil { + LOG_ERROR("RANDOM_FILE", "Failed to write to %s: %v", path, err) + return + } + size -= bytes + } +} + +func corruptFile(path string, start int, length int, seed int64) { + rand.Seed(seed) + + file, err := os.OpenFile(path, os.O_WRONLY, 0644) + if err != nil { + LOG_ERROR("CORRUPT_FILE", "Can't open %s for writing: %v", path, err) + return + } + + defer func() { + if file != nil { + file.Close() + } + }() + + _, err = file.Seek(int64(start), 0) + if err != nil { + LOG_ERROR("CORRUPT_FILE", "Can't seek to the offset %d: %v", start, err) + return + } + + buffer := make([]byte, length) + rand.Read(buffer) + + _, err = file.Write(buffer) + if err != nil { + LOG_ERROR("CORRUPT_FILE", "Failed to write to %s: %v", path, err) + return + } +} + +func TestBackupManagerPersist(t *testing.T) { + // We want deterministic output here so we can test the expected files are corrupted by missing or corrupt chunks + // There use rand functions with fixed seed, and known keys + + setTestingT(t) + SetLoggingLevel(INFO) + + defer func() { + if r := recover(); r != nil { + switch e := r.(type) { + case Exception: + t.Errorf("%s %s", e.LogID, e.Message) + debug.PrintStack() + default: + t.Errorf("%v", e) + debug.PrintStack() + } + } + }() + + testDir := path.Join(os.TempDir(), "duplicacy_test") + os.RemoveAll(testDir) + os.MkdirAll(testDir, 0700) + os.Mkdir(testDir+"/repository1", 0700) + os.Mkdir(testDir+"/repository1/dir1", 0700) + os.Mkdir(testDir+"/repository1/.duplicacy", 0700) + os.Mkdir(testDir+"/repository2", 0700) + os.Mkdir(testDir+"/repository2/.duplicacy", 0700) + os.Mkdir(testDir+"/repository3", 0700) + os.Mkdir(testDir+"/repository3/.duplicacy", 0700) + + maxFileSize := 1000000 + //maxFileSize := 200000 + + createRandomFileSeeded(testDir+"/repository1/file1", maxFileSize,1) + createRandomFileSeeded(testDir+"/repository1/file2", maxFileSize,2) + createRandomFileSeeded(testDir+"/repository1/dir1/file3", maxFileSize,3) + + threads := 1 + + password := "duplicacy" + + // We want deterministic output, plus ability to test encrypted storage + // So make unencrypted storage with default keys, and encrypted as bit-identical copy of this but with password + unencStorage, err := loadStorage(testDir+"/unenc_storage", threads) + if err != nil { + t.Errorf("Failed to create storage: %v", err) + return + } + delay := 0 + if _, ok := unencStorage.(*ACDStorage); ok { + delay = 1 + } + if _, ok := unencStorage.(*OneDriveStorage); ok { + delay = 5 + } + + time.Sleep(time.Duration(delay) * time.Second) + cleanStorage(unencStorage) + + if !ConfigStorage(unencStorage, 16384, 100, 64*1024, 256*1024, 16*1024, "", nil, false, "") { + t.Errorf("Failed to initialize the unencrypted storage") + } + time.Sleep(time.Duration(delay) * time.Second) + unencConfig, _, err := DownloadConfig(unencStorage, "") + if err != nil { + t.Errorf("Failed to download storage config: %v", err) + return + } + + // Make encrypted storage + storage, err := loadStorage(testDir+"/enc_storage", threads) + if err != nil { + t.Errorf("Failed to create encrypted storage: %v", err) + return + } + time.Sleep(time.Duration(delay) * time.Second) + cleanStorage(storage) + + if !ConfigStorage(storage, 16384, 100, 64*1024, 256*1024, 16*1024, password, unencConfig, true, "") { + t.Errorf("Failed to initialize the encrypted storage") + } + time.Sleep(time.Duration(delay) * time.Second) + + // do unencrypted backup + SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy") + unencBackupManager := CreateBackupManager("host1", unencStorage, testDir, "", "", "") + unencBackupManager.SetupSnapshotCache("default") + + SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy") + unencBackupManager.Backup(testDir+"/repository1" /*quickMode=*/, true, threads, "first", false, false, 0, false) + time.Sleep(time.Duration(delay) * time.Second) + + + // do encrypted backup + SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy") + encBackupManager := CreateBackupManager("host1", storage, testDir, password, "", "") + encBackupManager.SetupSnapshotCache("default") + + SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy") + encBackupManager.Backup(testDir+"/repository1" /*quickMode=*/, true, threads, "first", false, false, 0, false) + time.Sleep(time.Duration(delay) * time.Second) + + + // check snapshots + unencBackupManager.SnapshotManager.CheckSnapshots( /*snapshotID*/ "host1" /*revisions*/, []int{1} /*tag*/, "", + /*showStatistics*/ true /*showTabular*/, false /*checkFiles*/, true /*checkChunks*/, false, + /*searchFossils*/ false /*resurrect*/, false, 1 /*allowFailures*/, false) + + encBackupManager.SnapshotManager.CheckSnapshots( /*snapshotID*/ "host1" /*revisions*/, []int{1} /*tag*/, "", + /*showStatistics*/ true /*showTabular*/, false /*checkFiles*/, true /*checkChunks*/, false, + /*searchFossils*/ false /*resurrect*/, false, 1 /*allowFailures*/, false) + + // check functions + checkAllUncorrupted := func(cmpRepository string) { + for _, f := range []string{"file1", "file2", "dir1/file3"} { + if _, err := os.Stat(testDir + cmpRepository + "/" + f); os.IsNotExist(err) { + t.Errorf("File %s does not exist", f) + continue + } + + hash1 := getFileHash(testDir + "/repository1/" + f) + hash2 := getFileHash(testDir + cmpRepository + "/" + f) + if hash1 != hash2 { + t.Errorf("File %s has different hashes: %s vs %s", f, hash1, hash2) + } + } + } + checkMissingFile := func(cmpRepository string, expectMissing string) { + for _, f := range []string{"file1", "file2", "dir1/file3"} { + _, err := os.Stat(testDir + cmpRepository + "/" + f) + if err==nil { + if f==expectMissing { + t.Errorf("File %s exists, expected to be missing", f) + } + continue + } + if os.IsNotExist(err) { + if f!=expectMissing { + t.Errorf("File %s does not exist", f) + } + continue + } + + hash1 := getFileHash(testDir + "/repository1/" + f) + hash2 := getFileHash(testDir + cmpRepository + "/" + f) + if hash1 != hash2 { + t.Errorf("File %s has different hashes: %s vs %s", f, hash1, hash2) + } + } + } + checkCorruptedFile := func(cmpRepository string, expectCorrupted string) { + for _, f := range []string{"file1", "file2", "dir1/file3"} { + if _, err := os.Stat(testDir + cmpRepository + "/" + f); os.IsNotExist(err) { + t.Errorf("File %s does not exist", f) + continue + } + + hash1 := getFileHash(testDir + "/repository1/" + f) + hash2 := getFileHash(testDir + cmpRepository + "/" + f) + if (f==expectCorrupted) { + if hash1 == hash2 { + t.Errorf("File %s has same hashes, expected to be corrupted: %s vs %s", f, hash1, hash2) + } + + } else { + if hash1 != hash2 { + t.Errorf("File %s has different hashes: %s vs %s", f, hash1, hash2) + } + } + } + } + + // test restore all uncorrupted to repository3 + SetDuplicacyPreferencePath(testDir + "/repository3/.duplicacy") + unencBackupManager.Restore(testDir+"/repository3", threads /*inPlace=*/, true /*quickMode=*/, false, threads /*overwrite=*/, false, + /*deleteMode=*/ false /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, nil /*allowFailures=*/, false) + checkAllUncorrupted("/repository3") + + // test for corrupt files and -persist + // corrupt a chunk + chunkToCorrupt1 := "/4d/538e5dfd2b08e782bfeb56d1360fb5d7eb9d8c4b2531cc2fca79efbaec910c" + // this should affect file1 + chunkToCorrupt2 := "/2b/f953a766d0196ce026ae259e76e3c186a0e4bcd3ce10f1571d17f86f0a5497" + // this should affect dir1/file3 + + for i := 0; i < 2; i++ { + if i==0 { + // test corrupt chunks + corruptFile(testDir+"/unenc_storage"+"/chunks"+chunkToCorrupt1, 128, 128, 4) + corruptFile(testDir+"/enc_storage"+"/chunks"+chunkToCorrupt2, 128, 128, 4) + } else { + // test missing chunks + os.Remove(testDir+"/unenc_storage"+"/chunks"+chunkToCorrupt1) + os.Remove(testDir+"/enc_storage"+"/chunks"+chunkToCorrupt2) + } + + // check snapshots with --persist (allowFailures == true) + // this would cause a panic and os.Exit from duplicacy_log if allowFailures == false + unencBackupManager.SnapshotManager.CheckSnapshots( /*snapshotID*/ "host1" /*revisions*/, []int{1} /*tag*/, "", + /*showStatistics*/ true /*showTabular*/, false /*checkFiles*/, true /*checkChunks*/, false, + /*searchFossils*/ false /*resurrect*/, false, 1 /*allowFailures*/, true) + + encBackupManager.SnapshotManager.CheckSnapshots( /*snapshotID*/ "host1" /*revisions*/, []int{1} /*tag*/, "", + /*showStatistics*/ true /*showTabular*/, false /*checkFiles*/, true /*checkChunks*/, false, + /*searchFossils*/ false /*resurrect*/, false, 1 /*allowFailures*/, true) + + + // test restore corrupted, inPlace = true, corrupted files will have hash failures + os.RemoveAll(testDir+"/repository2") + SetDuplicacyPreferencePath(testDir + "/repository2/.duplicacy") + unencBackupManager.Restore(testDir+"/repository2", threads /*inPlace=*/, true /*quickMode=*/, false, threads /*overwrite=*/, false, + /*deleteMode=*/ false /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, nil /*allowFailures=*/, true) + + // check restore, expect file1 to be corrupted + checkCorruptedFile("/repository2", "file1") + + + os.RemoveAll(testDir+"/repository2") + SetDuplicacyPreferencePath(testDir + "/repository2/.duplicacy") + encBackupManager.Restore(testDir+"/repository2", threads /*inPlace=*/, true /*quickMode=*/, false, threads /*overwrite=*/, false, + /*deleteMode=*/ false /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, nil /*allowFailures=*/, true) + + // check restore, expect file3 to be corrupted + checkCorruptedFile("/repository2", "dir1/file3") + + //SetLoggingLevel(DEBUG) + // test restore corrupted, inPlace = false, corrupted files will be missing + os.RemoveAll(testDir+"/repository2") + SetDuplicacyPreferencePath(testDir + "/repository2/.duplicacy") + unencBackupManager.Restore(testDir+"/repository2", threads /*inPlace=*/, false /*quickMode=*/, false, threads /*overwrite=*/, false, + /*deleteMode=*/ false /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, nil /*allowFailures=*/, true) + + // check restore, expect file1 to be corrupted + checkMissingFile("/repository2", "file1") + + + os.RemoveAll(testDir+"/repository2") + SetDuplicacyPreferencePath(testDir + "/repository2/.duplicacy") + encBackupManager.Restore(testDir+"/repository2", threads /*inPlace=*/, false /*quickMode=*/, false, threads /*overwrite=*/, false, + /*deleteMode=*/ false /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, nil /*allowFailures=*/, true) + + // check restore, expect file3 to be corrupted + checkMissingFile("/repository2", "dir1/file3") + + // test restore corrupted files from different backups, inPlace = true + // with overwrite=true, corrupted file1 from unenc will be restored correctly from enc + // the latter will not touch the existing file3 with correct hash + os.RemoveAll(testDir+"/repository2") + unencBackupManager.Restore(testDir+"/repository2", threads /*inPlace=*/, true /*quickMode=*/, false, threads /*overwrite=*/, false, + /*deleteMode=*/ false /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, nil /*allowFailures=*/, true) + encBackupManager.Restore(testDir+"/repository2", threads /*inPlace=*/, true /*quickMode=*/, false, threads /*overwrite=*/, true, + /*deleteMode=*/ false /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, nil /*allowFailures=*/, true) + checkAllUncorrupted("/repository2") + + // restore to repository3, with overwrite and allowFailures (true/false), quickMode = false (use hashes) + // should always succeed as uncorrupted files already exist with correct hash, so these will be ignored + SetDuplicacyPreferencePath(testDir + "/repository3/.duplicacy") + unencBackupManager.Restore(testDir+"/repository3", threads /*inPlace=*/, true /*quickMode=*/, false, threads /*overwrite=*/, true, + /*deleteMode=*/ false /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, nil /*allowFailures=*/, false) + checkAllUncorrupted("/repository3") + + unencBackupManager.Restore(testDir+"/repository3", threads /*inPlace=*/, true /*quickMode=*/, false, threads /*overwrite=*/, true, + /*deleteMode=*/ false /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, nil /*allowFailures=*/, true) + checkAllUncorrupted("/repository3") + } + +} \ No newline at end of file diff --git a/src/duplicacy_chunk.go b/src/duplicacy_chunk.go index 932f305..ab6f5a1 100644 --- a/src/duplicacy_chunk.go +++ b/src/duplicacy_chunk.go @@ -65,6 +65,8 @@ type Chunk struct { isSnapshot bool // Indicates if the chunk is a snapshot chunk (instead of a file chunk). This is only used by RSA // encryption, where a snapshot chunk is not encrypted by RSA + + isBroken bool // Indicates the chunk did not download correctly. This is only used for -persist (allowFailures) mode } // Magic word to identify a duplicacy format encrypted file, plus a version number. @@ -122,6 +124,7 @@ func (chunk *Chunk) Reset(hashNeeded bool) { chunk.id = "" chunk.size = 0 chunk.isSnapshot = false + chunk.isBroken = false } // Write implements the Writer interface. diff --git a/src/duplicacy_chunkdownloader.go b/src/duplicacy_chunkdownloader.go index 468b2cf..7769c77 100644 --- a/src/duplicacy_chunkdownloader.go +++ b/src/duplicacy_chunkdownloader.go @@ -36,6 +36,7 @@ type ChunkDownloader struct { snapshotCache *FileStorage // Used as cache if not nil; usually for downloading snapshot chunks showStatistics bool // Show a stats log for each chunk if true threads int // Number of threads + allowFailures bool // Whether to failfast on download error, or continue taskList []ChunkDownloadTask // The list of chunks to be downloaded completedTasks map[int]bool // Store downloaded chunks @@ -53,13 +54,14 @@ type ChunkDownloader struct { numberOfActiveChunks int // The number of chunks that is being downloaded or has been downloaded but not reclaimed } -func CreateChunkDownloader(config *Config, storage Storage, snapshotCache *FileStorage, showStatistics bool, threads int) *ChunkDownloader { +func CreateChunkDownloader(config *Config, storage Storage, snapshotCache *FileStorage, showStatistics bool, threads int, allowFailures bool) *ChunkDownloader { downloader := &ChunkDownloader{ config: config, storage: storage, snapshotCache: snapshotCache, showStatistics: showStatistics, threads: threads, + allowFailures: allowFailures, taskList: nil, completedTasks: make(map[int]bool), @@ -357,13 +359,27 @@ func (downloader *ChunkDownloader) Download(threadIndex int, task ChunkDownloadT // will be set up before the encryption chunk.Reset(false) + // This lambda function allows different handling of failures depending on allowFailures state + onFailure := func(failureFn LogFunc, logID string, format string, v ...interface{}) { + if downloader.allowFailures { + // Allowing failures: Convert message to warning, mark chunk isBroken = true and complete goroutine + LOG_WARN(logID, format, v...) + chunk.isBroken = true + downloader.completionChannel <- ChunkDownloadCompletion{chunk: chunk, chunkIndex: task.chunkIndex} + + } else { + // Process failure as normal + failureFn(logID, format, v...) + } + } + const MaxDownloadAttempts = 3 for downloadAttempt := 0; ; downloadAttempt++ { // Find the chunk by ID first. chunkPath, exist, _, err := downloader.storage.FindChunk(threadIndex, chunkID, false) if err != nil { - LOG_ERROR("DOWNLOAD_CHUNK", "Failed to find the chunk %s: %v", chunkID, err) + onFailure(LOG_ERROR, "DOWNLOAD_CHUNK", "Failed to find the chunk %s: %v", chunkID, err) return false } @@ -371,7 +387,7 @@ func (downloader *ChunkDownloader) Download(threadIndex int, task ChunkDownloadT // No chunk is found. Have to find it in the fossil pool again. fossilPath, exist, _, err := downloader.storage.FindChunk(threadIndex, chunkID, true) if err != nil { - LOG_ERROR("DOWNLOAD_CHUNK", "Failed to find the chunk %s: %v", chunkID, err) + onFailure(LOG_ERROR, "DOWNLOAD_CHUNK", "Failed to find the chunk %s: %v", chunkID, err) return false } @@ -395,9 +411,9 @@ func (downloader *ChunkDownloader) Download(threadIndex int, task ChunkDownloadT // A chunk is not found. This is a serious error and hopefully it will never happen. if err != nil { - LOG_FATAL("DOWNLOAD_CHUNK", "Chunk %s can't be found: %v", chunkID, err) + onFailure(LOG_FATAL, "DOWNLOAD_CHUNK", "Chunk %s can't be found: %v", chunkID, err) } else { - LOG_FATAL("DOWNLOAD_CHUNK", "Chunk %s can't be found", chunkID) + onFailure(LOG_FATAL, "DOWNLOAD_CHUNK", "Chunk %s can't be found", chunkID) } return false } @@ -406,7 +422,7 @@ func (downloader *ChunkDownloader) Download(threadIndex int, task ChunkDownloadT // downloading again. err = downloader.storage.MoveFile(threadIndex, fossilPath, chunkPath) if err != nil { - LOG_FATAL("DOWNLOAD_CHUNK", "Failed to resurrect chunk %s: %v", chunkID, err) + onFailure(LOG_FATAL, "DOWNLOAD_CHUNK", "Failed to resurrect chunk %s: %v", chunkID, err) return false } @@ -423,7 +439,7 @@ func (downloader *ChunkDownloader) Download(threadIndex int, task ChunkDownloadT chunk.Reset(false) continue } else { - LOG_ERROR("DOWNLOAD_CHUNK", "Failed to download the chunk %s: %v", chunkID, err) + onFailure(LOG_ERROR, "DOWNLOAD_CHUNK", "Failed to download the chunk %s: %v", chunkID, err) return false } } @@ -435,7 +451,7 @@ func (downloader *ChunkDownloader) Download(threadIndex int, task ChunkDownloadT chunk.Reset(false) continue } else { - LOG_ERROR("DOWNLOAD_DECRYPT", "Failed to decrypt the chunk %s: %v", chunkID, err) + onFailure(LOG_ERROR, "DOWNLOAD_DECRYPT", "Failed to decrypt the chunk %s: %v", chunkID, err) return false } } @@ -447,7 +463,7 @@ func (downloader *ChunkDownloader) Download(threadIndex int, task ChunkDownloadT chunk.Reset(false) continue } else { - LOG_FATAL("DOWNLOAD_CORRUPTED", "The chunk %s has a hash id of %s", chunkID, actualChunkID) + onFailure(LOG_FATAL, "DOWNLOAD_CORRUPTED", "The chunk %s has a hash id of %s", chunkID, actualChunkID) return false } } diff --git a/src/duplicacy_chunkuploader_test.go b/src/duplicacy_chunkuploader_test.go index f8312ec..c31c1c1 100644 --- a/src/duplicacy_chunkuploader_test.go +++ b/src/duplicacy_chunkuploader_test.go @@ -101,7 +101,7 @@ func TestUploaderAndDownloader(t *testing.T) { chunkUploader.Stop() - chunkDownloader := CreateChunkDownloader(config, storage, nil, true, testThreads) + chunkDownloader := CreateChunkDownloader(config, storage, nil, true, testThreads, false) chunkDownloader.totalChunkSize = int64(totalFileSize) for _, chunk := range chunks { diff --git a/src/duplicacy_log.go b/src/duplicacy_log.go index d71852d..9666e4f 100644 --- a/src/duplicacy_log.go +++ b/src/duplicacy_log.go @@ -87,6 +87,9 @@ func SetLoggingLevel(level int) { loggingLevel = level } +// log function type for passing as a parameter to functions +type LogFunc func(logID string, format string, v ...interface{}) + func LOG_DEBUG(logID string, format string, v ...interface{}) { logf(DEBUG, logID, format, v...) } diff --git a/src/duplicacy_snapshotmanager.go b/src/duplicacy_snapshotmanager.go index be3c635..58bd552 100644 --- a/src/duplicacy_snapshotmanager.go +++ b/src/duplicacy_snapshotmanager.go @@ -270,7 +270,7 @@ func (reader *sequenceReader) Read(data []byte) (n int, err error) { func (manager *SnapshotManager) CreateChunkDownloader() { if manager.chunkDownloader == nil { - manager.chunkDownloader = CreateChunkDownloader(manager.config, manager.storage, manager.snapshotCache, false, 1) + manager.chunkDownloader = CreateChunkDownloader(manager.config, manager.storage, manager.snapshotCache, false, 1, false) } } @@ -802,9 +802,9 @@ func (manager *SnapshotManager) ListSnapshots(snapshotID string, revisionsToList // ListSnapshots shows the information about a snapshot. func (manager *SnapshotManager) CheckSnapshots(snapshotID string, revisionsToCheck []int, tag string, showStatistics bool, showTabular bool, - checkFiles bool, checkChunks, searchFossils bool, resurrect bool, threads int) bool { + checkFiles bool, checkChunks, searchFossils bool, resurrect bool, threads int, allowFailures bool) bool { - manager.chunkDownloader = CreateChunkDownloader(manager.config, manager.storage, manager.snapshotCache, false, threads) + manager.chunkDownloader = CreateChunkDownloader(manager.config, manager.storage, manager.snapshotCache, false, threads, allowFailures) LOG_DEBUG("LIST_PARAMETERS", "id: %s, revisions: %v, tag: %s, showStatistics: %t, showTabular: %t, checkFiles: %t, searchFossils: %t, resurrect: %t", snapshotID, revisionsToCheck, tag, showStatistics, showTabular, checkFiles, searchFossils, resurrect) @@ -1003,7 +1003,11 @@ func (manager *SnapshotManager) CheckSnapshots(snapshotID string, revisionsToChe manager.chunkDownloader.AddChunk(chunkHash) } manager.chunkDownloader.WaitForCompletion() - LOG_INFO("SNAPSHOT_VERIFY", "All %d chunks have been successfully verified", len(*allChunkHashes)) + if allowFailures { + LOG_INFO("SNAPSHOT_VERIFY", "All %d chunks have been verified, see above for any errors", len(*allChunkHashes)) + } else { + LOG_INFO("SNAPSHOT_VERIFY", "All %d chunks have been successfully verified", len(*allChunkHashes)) + } } return true } diff --git a/src/duplicacy_snapshotmanager_test.go b/src/duplicacy_snapshotmanager_test.go index 695a738..0bb9c2d 100644 --- a/src/duplicacy_snapshotmanager_test.go +++ b/src/duplicacy_snapshotmanager_test.go @@ -620,7 +620,7 @@ func TestPruneNewSnapshots(t *testing.T) { // Now chunkHash1 wil be resurrected snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false, 1) checkTestSnapshots(snapshotManager, 4, 0) - snapshotManager.CheckSnapshots("vm1@host1", []int{2, 3}, "", false, false, false, false, false, false, 1) + snapshotManager.CheckSnapshots("vm1@host1", []int{2, 3}, "", false, false, false, false, false, false, 1, false) } // A fossil collection left by an aborted prune should be ignored if any supposedly deleted snapshot exists @@ -669,7 +669,7 @@ func TestPruneGhostSnapshots(t *testing.T) { // Run the prune again but the fossil collection should be igored, since revision 1 still exists snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false, 1) checkTestSnapshots(snapshotManager, 3, 2) - snapshotManager.CheckSnapshots("vm1@host1", []int{1, 2, 3}, "", false, false, false, false, true /*searchFossils*/, false, 1) + snapshotManager.CheckSnapshots("vm1@host1", []int{1, 2, 3}, "", false, false, false, false, true /*searchFossils*/, false, 1, false) // Prune snapshot 1 again snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{1}, []string{}, []string{}, false, false, []string{}, false, false, false, 1) @@ -683,5 +683,5 @@ func TestPruneGhostSnapshots(t *testing.T) { // Run the prune again and this time the fossil collection will be processed and the fossils removed snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false, 1) checkTestSnapshots(snapshotManager, 3, 0) - snapshotManager.CheckSnapshots("vm1@host1", []int{2, 3, 4}, "", false, false, false, false, false, false, 1) + snapshotManager.CheckSnapshots("vm1@host1", []int{2, 3, 4}, "", false, false, false, false, false, false, 1, false) }