From 457e5181514441563a4118fca02583ccdba17f8e Mon Sep 17 00:00:00 2001 From: niknah Date: Wed, 6 Sep 2017 19:19:36 +1000 Subject: [PATCH 1/8] Added backup --dry-run option. --- duplicacy/duplicacy_main.go | 6 ++++++ src/duplicacy_backupmanager.go | 19 ++++++++++++++----- src/duplicacy_chunkuploader.go | 10 ++++++---- src/duplicacy_config.go | 1 + src/duplicacy_snapshotmanager.go | 11 ++++++++--- 5 files changed, 35 insertions(+), 12 deletions(-) diff --git a/duplicacy/duplicacy_main.go b/duplicacy/duplicacy_main.go index b00686b..0bc931d 100644 --- a/duplicacy/duplicacy_main.go +++ b/duplicacy/duplicacy_main.go @@ -623,12 +623,14 @@ func backupRepository(context *cli.Context) { enableVSS := context.Bool("vss") + dryRun := context.Bool("dry-run") uploadRateLimit := context.Int("limit-rate") storage.SetRateLimits(0, uploadRateLimit) backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password) duplicacy.SavePassword(*preference, "password", password) backupManager.SetupSnapshotCache(preference.Name) + backupManager.SetDryRun(dryRun) backupManager.Backup(repository, quickMode, threads, context.String("t"), showStatistics, enableVSS) runScript(context, preference.Name, "post") @@ -1202,6 +1204,10 @@ func main() { Usage: "the maximum upload rate (in kilobytes/sec)", Argument: "", }, + cli.BoolFlag { + Name: "dry-run", + Usage: "Dry run for testing, don't backup anything. Use with -stats and -d", + }, cli.BoolFlag { Name: "vss", Usage: "enable the Volume Shadow Copy service (Windows only)", diff --git a/src/duplicacy_backupmanager.go b/src/duplicacy_backupmanager.go index 9090653..9fa9a16 100644 --- a/src/duplicacy_backupmanager.go +++ b/src/duplicacy_backupmanager.go @@ -36,6 +36,10 @@ type BackupManager struct { } +func (manager *BackupManager) SetDryRun(dryRun bool) { + manager.config.dryRun = dryRun +} + // CreateBackupManager creates a backup manager using the specified 'storage'. 'snapshotID' is a unique id to // identify snapshots created for this repository. 'top' is the top directory of the repository. 'password' is the @@ -630,7 +634,9 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta } skippedFiles = append(skippedFiles, fileReader.SkippedFiles...) - manager.SnapshotManager.CleanSnapshotCache(localSnapshot, nil) + if !manager.config.dryRun { + manager.SnapshotManager.CleanSnapshotCache(localSnapshot, nil) + } LOG_INFO("BACKUP_END", "Backup for %s at revision %d completed", top, localSnapshot.Revision) RunAtError = func() {} @@ -1048,7 +1054,7 @@ func (manager *BackupManager) UploadSnapshot(chunkMaker *ChunkMaker, uploader *C if _, found := chunkCache[chunkID]; found { completionFunc(chunk, 0, true, chunk.GetLength(), 0) } else { - uploader.StartChunk(chunk, len(sequence)) + uploader.StartChunk(chunk, len(sequence)) } sequence = append(sequence, chunk.GetHash()) }, @@ -1108,8 +1114,9 @@ func (manager *BackupManager) UploadSnapshot(chunkMaker *ChunkMaker, uploader *C } path := fmt.Sprintf("snapshots/%s/%d", manager.snapshotID, snapshot.Revision) - manager.SnapshotManager.UploadFile(path, path, description) - + if !manager.config.dryRun { + manager.SnapshotManager.UploadFile(path, path, description) + } return totalSnapshotChunkSize, numberOfNewSnapshotChunks, totalUploadedSnapshotChunkSize, totalUploadedSnapshotChunkBytes } @@ -1598,7 +1605,9 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho chunkUploader.Stop() for _, snapshot := range snapshots { - otherManager.storage.CreateDirectory(0, fmt.Sprintf("snapshots/%s", manager.snapshotID)) + if !manager.config.dryRun { + otherManager.storage.CreateDirectory(0, fmt.Sprintf("snapshots/%s", manager.snapshotID)) + } description, _ := snapshot.MarshalJSON() path := fmt.Sprintf("snapshots/%s/%d", manager.snapshotID, snapshot.Revision) otherManager.SnapshotManager.UploadFile(path, path, description) diff --git a/src/duplicacy_chunkuploader.go b/src/duplicacy_chunkuploader.go index a9f2777..88c409e 100644 --- a/src/duplicacy_chunkuploader.go +++ b/src/duplicacy_chunkuploader.go @@ -134,10 +134,12 @@ func (uploader *ChunkUploader) Upload(threadIndex int, task ChunkUploadTask) boo return false } - err = uploader.storage.UploadFile(threadIndex, chunkPath, chunk.GetBytes()) - if err != nil { - LOG_ERROR("UPLOAD_CHUNK", "Failed to upload the chunk %s: %v", chunkID, err) - return false + if !uploader.config.dryRun { + err = uploader.storage.UploadFile(threadIndex, chunkPath, chunk.GetBytes()) + if err != nil { + LOG_ERROR("UPLOAD_CHUNK", "Failed to upload the chunk %s: %v", chunkID, err) + return false + } } LOG_DEBUG("CHUNK_UPLOAD", "Chunk %s has been uploaded", chunkID) diff --git a/src/duplicacy_config.go b/src/duplicacy_config.go index 8c3fb8e..e92f2f2 100644 --- a/src/duplicacy_config.go +++ b/src/duplicacy_config.go @@ -55,6 +55,7 @@ type Config struct { chunkPool chan *Chunk `json:"-"` numberOfChunks int32 + dryRun bool } // Create an alias to avoid recursive calls on Config.MarshalJSON diff --git a/src/duplicacy_snapshotmanager.go b/src/duplicacy_snapshotmanager.go index 16150ad..8b2df9a 100644 --- a/src/duplicacy_snapshotmanager.go +++ b/src/duplicacy_snapshotmanager.go @@ -2215,9 +2215,11 @@ func (manager *SnapshotManager) DownloadFile(path string, derivationKey string) return nil } - err = manager.snapshotCache.UploadFile(0, path, manager.fileChunk.GetBytes()) - if err != nil { - LOG_WARN("DOWNLOAD_FILE_CACHE", "Failed to add the file %s to the snapshot cache: %v", path, err) + if !manager.config.dryRun { + err = manager.snapshotCache.UploadFile(0, path, manager.fileChunk.GetBytes()) + if err != nil { + LOG_WARN("DOWNLOAD_FILE_CACHE", "Failed to add the file %s to the snapshot cache: %v", path, err) + } } LOG_DEBUG("DOWNLOAD_FILE", "Downloaded file %s", path) @@ -2227,6 +2229,9 @@ func (manager *SnapshotManager) DownloadFile(path string, derivationKey string) // UploadFile uploads a non-chunk file from the storage. func (manager *SnapshotManager) UploadFile(path string, derivationKey string, content []byte) bool { + if manager.config.dryRun { + return true + } manager.fileChunk.Reset(false) manager.fileChunk.Write(content) From eb4c875fd034ad9bc77c149abb8eae3e70dd9a62 Mon Sep 17 00:00:00 2001 From: niknah Date: Tue, 12 Sep 2017 06:48:14 +1000 Subject: [PATCH 2/8] Added -dry-run to doc. --- GUIDE.md | 1 + 1 file changed, 1 insertion(+) diff --git a/GUIDE.md b/GUIDE.md index 285f169..e352e13 100644 --- a/GUIDE.md +++ b/GUIDE.md @@ -50,6 +50,7 @@ OPTIONS: -stats show statistics during and after backup -threads number of uploading threads -limit-rate the maximum upload rate (in kilobytes/sec) + -dry-run dry run for testing, don't backup anything. Use with -stats and -d -vss enable the Volume Shadow Copy service (Windows only) -storage backup to the specified storage instead of the default one ``` From 491252e3e4373a3fdbd9e78024691847c9cb1ff1 Mon Sep 17 00:00:00 2001 From: niknah Date: Thu, 14 Sep 2017 14:12:32 +1000 Subject: [PATCH 3/8] Didn't merge --dry-run with the latest updates properly. --- src/duplicacy_backupmanager.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/duplicacy_backupmanager.go b/src/duplicacy_backupmanager.go index aa0519b..eb85064 100644 --- a/src/duplicacy_backupmanager.go +++ b/src/duplicacy_backupmanager.go @@ -1695,15 +1695,15 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho LOG_INFO("SNAPSHOT_COPY", "Total chunks copied = %d, skipped = %d.", totalCopied, totalSkipped) for _, snapshot := range snapshots { - if !manager.config.dryRun { - if revisionMap[snapshot.ID][snapshot.Revision] == false { - continue - } + if revisionMap[snapshot.ID][snapshot.Revision] == false { + continue + } + if !manager.config.dryRun { + otherManager.storage.CreateDirectory(0, fmt.Sprintf("snapshots/%s", snapshot.ID)) + description, _ := snapshot.MarshalJSON() + path := fmt.Sprintf("snapshots/%s/%d", snapshot.ID, snapshot.Revision) + otherManager.SnapshotManager.UploadFile(path, path, description) } - otherManager.storage.CreateDirectory(0, fmt.Sprintf("snapshots/%s", snapshot.ID)) - description, _ := snapshot.MarshalJSON() - path := fmt.Sprintf("snapshots/%s/%d", snapshot.ID, snapshot.Revision) - otherManager.SnapshotManager.UploadFile(path, path, description) LOG_INFO("SNAPSHOT_COPY", "Copied snapshot %s at revision %d", snapshot.ID, snapshot.Revision) } From 03c2a190ee36151c54eae7fee2064434459ec4e5 Mon Sep 17 00:00:00 2001 From: niknah Date: Thu, 14 Sep 2017 14:20:58 +1000 Subject: [PATCH 4/8] Don't create extra directories on --dry-run --- src/duplicacy_snapshotmanager.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/duplicacy_snapshotmanager.go b/src/duplicacy_snapshotmanager.go index 8b2df9a..6662da9 100644 --- a/src/duplicacy_snapshotmanager.go +++ b/src/duplicacy_snapshotmanager.go @@ -600,7 +600,9 @@ func (manager *SnapshotManager) ListAllFiles(storage Storage, top string) (allFi if len(files) > 1024 && !storage.IsFastListing() { for i := 0; i < 256; i++ { subdir := dir + fmt.Sprintf("%02x\n", i) - manager.storage.CreateDirectory(0, subdir) + if !manager.config.dryRun { + manager.storage.CreateDirectory(0, subdir) + } } } } else { From f74ea0368e8b7316413203f092748a2cb8474881 Mon Sep 17 00:00:00 2001 From: niknah Date: Fri, 15 Sep 2017 02:21:07 +1000 Subject: [PATCH 5/8] ListFiles Don't delete or create directories in dryRun --- src/duplicacy_snapshotmanager.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/duplicacy_snapshotmanager.go b/src/duplicacy_snapshotmanager.go index 6662da9..bde81d0 100644 --- a/src/duplicacy_snapshotmanager.go +++ b/src/duplicacy_snapshotmanager.go @@ -594,21 +594,21 @@ func (manager *SnapshotManager) ListAllFiles(storage Storage, top string) (allFi } } - if top == "chunks/" { - // We're listing all chunks so this is the perfect place to detect if a directory contains too many - // chunks. Create sub-directories if necessary - if len(files) > 1024 && !storage.IsFastListing() { - for i := 0; i < 256; i++ { - subdir := dir + fmt.Sprintf("%02x\n", i) - if !manager.config.dryRun { + if !manager.config.dryRun { + if top == "chunks/" { + // We're listing all chunks so this is the perfect place to detect if a directory contains too many + // chunks. Create sub-directories if necessary + if len(files) > 1024 && !storage.IsFastListing() { + for i := 0; i < 256; i++ { + subdir := dir + fmt.Sprintf("%02x\n", i) manager.storage.CreateDirectory(0, subdir) } } - } - } else { - // Remove chunk sub-directories that are empty - if len(files) == 0 && strings.HasPrefix(dir, "chunks/") && dir != "chunks/" { - storage.DeleteFile(0, dir) + } else { + // Remove chunk sub-directories that are empty + if len(files) == 0 && strings.HasPrefix(dir, "chunks/") && dir != "chunks/" { + storage.DeleteFile(0, dir) + } } } } From f25783d59d1c58ada32ee36ca6b53e8eeced005d Mon Sep 17 00:00:00 2001 From: niknah Date: Mon, 18 Sep 2017 01:38:51 +1000 Subject: [PATCH 6/8] Comments from @gilbertchen, thanks. --- src/duplicacy_backupmanager.go | 10 ++++------ src/duplicacy_snapshotmanager.go | 11 +++-------- 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/src/duplicacy_backupmanager.go b/src/duplicacy_backupmanager.go index eb85064..5cdc81f 100644 --- a/src/duplicacy_backupmanager.go +++ b/src/duplicacy_backupmanager.go @@ -1698,12 +1698,10 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho if revisionMap[snapshot.ID][snapshot.Revision] == false { continue } - if !manager.config.dryRun { - otherManager.storage.CreateDirectory(0, fmt.Sprintf("snapshots/%s", snapshot.ID)) - description, _ := snapshot.MarshalJSON() - path := fmt.Sprintf("snapshots/%s/%d", snapshot.ID, snapshot.Revision) - otherManager.SnapshotManager.UploadFile(path, path, description) - } + otherManager.storage.CreateDirectory(0, fmt.Sprintf("snapshots/%s", snapshot.ID)) + description, _ := snapshot.MarshalJSON() + path := fmt.Sprintf("snapshots/%s/%d", snapshot.ID, snapshot.Revision) + otherManager.SnapshotManager.UploadFile(path, path, description) LOG_INFO("SNAPSHOT_COPY", "Copied snapshot %s at revision %d", snapshot.ID, snapshot.Revision) } diff --git a/src/duplicacy_snapshotmanager.go b/src/duplicacy_snapshotmanager.go index bde81d0..e94b16a 100644 --- a/src/duplicacy_snapshotmanager.go +++ b/src/duplicacy_snapshotmanager.go @@ -2217,11 +2217,9 @@ func (manager *SnapshotManager) DownloadFile(path string, derivationKey string) return nil } - if !manager.config.dryRun { - err = manager.snapshotCache.UploadFile(0, path, manager.fileChunk.GetBytes()) - if err != nil { - LOG_WARN("DOWNLOAD_FILE_CACHE", "Failed to add the file %s to the snapshot cache: %v", path, err) - } + err = manager.snapshotCache.UploadFile(0, path, manager.fileChunk.GetBytes()) + if err != nil { + LOG_WARN("DOWNLOAD_FILE_CACHE", "Failed to add the file %s to the snapshot cache: %v", path, err) } LOG_DEBUG("DOWNLOAD_FILE", "Downloaded file %s", path) @@ -2231,9 +2229,6 @@ func (manager *SnapshotManager) DownloadFile(path string, derivationKey string) // UploadFile uploads a non-chunk file from the storage. func (manager *SnapshotManager) UploadFile(path string, derivationKey string, content []byte) bool { - if manager.config.dryRun { - return true - } manager.fileChunk.Reset(false) manager.fileChunk.Write(content) From f20e823119dadd2bbce2d8868ad24cdf586078ca Mon Sep 17 00:00:00 2001 From: niknah Date: Mon, 18 Sep 2017 18:43:22 +1000 Subject: [PATCH 7/8] Different upload message for dryRun. --- src/duplicacy_chunkuploader.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/duplicacy_chunkuploader.go b/src/duplicacy_chunkuploader.go index 88c409e..99cb900 100644 --- a/src/duplicacy_chunkuploader.go +++ b/src/duplicacy_chunkuploader.go @@ -140,9 +140,11 @@ func (uploader *ChunkUploader) Upload(threadIndex int, task ChunkUploadTask) boo LOG_ERROR("UPLOAD_CHUNK", "Failed to upload the chunk %s: %v", chunkID, err) return false } + LOG_DEBUG("CHUNK_UPLOAD", "Chunk %s has been uploaded", chunkID) + } else { + LOG_DEBUG("CHUNK_UPLOAD", "Uploading was skipped for chunk %s", chunkID) } - LOG_DEBUG("CHUNK_UPLOAD", "Chunk %s has been uploaded", chunkID) uploader.completionFunc(chunk, task.chunkIndex, false, chunkSize, chunk.GetLength()) atomic.AddInt32(&uploader.numberOfUploadingTasks, -1) return true From c63621cb8cee344ac036c651038058ffd5c3bfd5 Mon Sep 17 00:00:00 2001 From: niknah Date: Mon, 18 Sep 2017 18:43:44 +1000 Subject: [PATCH 8/8] Tab spacing fixes --- src/duplicacy_backupmanager.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/duplicacy_backupmanager.go b/src/duplicacy_backupmanager.go index 5cdc81f..aa13137 100644 --- a/src/duplicacy_backupmanager.go +++ b/src/duplicacy_backupmanager.go @@ -1054,7 +1054,7 @@ func (manager *BackupManager) UploadSnapshot(chunkMaker *ChunkMaker, uploader *C if _, found := chunkCache[chunkID]; found { completionFunc(chunk, 0, true, chunk.GetLength(), 0) } else { - uploader.StartChunk(chunk, len(sequence)) + uploader.StartChunk(chunk, len(sequence)) } sequence = append(sequence, chunk.GetHash()) }, @@ -1696,7 +1696,7 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho for _, snapshot := range snapshots { if revisionMap[snapshot.ID][snapshot.Revision] == false { - continue + continue } otherManager.storage.CreateDirectory(0, fmt.Sprintf("snapshots/%s", snapshot.ID)) description, _ := snapshot.MarshalJSON()