diff --git a/integration_tests/copy_test.sh b/integration_tests/copy_test.sh index 329a95c..d260494 100755 --- a/integration_tests/copy_test.sh +++ b/integration_tests/copy_test.sh @@ -11,11 +11,21 @@ ${DUPLICACY} add -copy default secondary integration-tests $SECONDARY_STORAGE add_file file1 add_file file2 ${DUPLICACY} backup -${DUPLICACY} copy -from default -to secondary +${DUPLICACY} copy -from default -to secondary add_file file3 add_file file4 ${DUPLICACY} backup -${DUPLICACY} copy -from default -to secondary -${DUPLICACY} check --files -stats +${DUPLICACY} copy -from default -to secondary +${DUPLICACY} check --files -stats -storage default ${DUPLICACY} check --files -stats -storage secondary +# Prune revisions from default storage +${DUPLICACY} -d -v -log prune -r 1-2 -exclusive -exhaustive -storage default +# Copy snapshot revisions from secondary back to default +${DUPLICACY} copy -from secondary -to default +# Check snapshot revisions again to make sure we're ok! +${DUPLICACY} check --files -stats -storage default +${DUPLICACY} check --files -stats -storage secondary +# Check for orphaned or missing chunks +${DUPLICACY} prune -exhaustive -exclusive -storage default +${DUPLICACY} prune -exhaustive -exclusive -storage secondary popd diff --git a/src/duplicacy_backupmanager.go b/src/duplicacy_backupmanager.go index 411fcbe..1d52079 100644 --- a/src/duplicacy_backupmanager.go +++ b/src/duplicacy_backupmanager.go @@ -1501,7 +1501,6 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho } var snapshots [] *Snapshot - var otherSnapshots [] *Snapshot var snapshotIDs [] string var err error @@ -1557,17 +1556,6 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho snapshots = append(snapshots, snapshot) } - otherRevisions, err := otherManager.SnapshotManager.ListSnapshotRevisions(id) - if err != nil { - LOG_ERROR("SNAPSHOT_LIST", "Failed to list all revisions at the destination for snapshot %s: %v", id, err) - return false - } - - for _, otherRevision := range otherRevisions { - otherSnapshot := otherManager.SnapshotManager.DownloadSnapshot(id, otherRevision) - otherSnapshots = append(otherSnapshots, otherSnapshot) - } - } if len(snapshots) == 0 { @@ -1576,6 +1564,7 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho } chunks := make(map[string]bool) + otherChunks := make(map[string]bool) for _, snapshot := range snapshots { @@ -1586,15 +1575,21 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho LOG_TRACE("SNAPSHOT_COPY", "Copying snapshot %s at revision %d", snapshot.ID, snapshot.Revision) for _, chunkHash := range snapshot.FileSequence { - chunks[chunkHash] = true + if _, found := chunks[chunkHash]; !found { + chunks[chunkHash] = true + } } for _, chunkHash := range snapshot.ChunkSequence { - chunks[chunkHash] = true + if _, found := chunks[chunkHash]; !found { + chunks[chunkHash] = true + } } for _, chunkHash := range snapshot.LengthSequence { - chunks[chunkHash] = true + if _, found := chunks[chunkHash]; !found { + chunks[chunkHash] = true + } } description := manager.SnapshotManager.DownloadSequence(snapshot.ChunkSequence) @@ -1606,45 +1601,43 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho } for _, chunkHash := range snapshot.ChunkHashes { - chunks[chunkHash] = true - } - } - - for _, otherSnapshot := range otherSnapshots { - - for _, chunkHash := range otherSnapshot.FileSequence { - if _, found := chunks[chunkHash]; found { - chunks[chunkHash] = false - } - } - - for _, chunkHash := range otherSnapshot.ChunkSequence { - if _, found := chunks[chunkHash]; found { - chunks[chunkHash] = false - } - } - - for _, chunkHash := range otherSnapshot.LengthSequence { - if _, found := chunks[chunkHash]; found { - chunks[chunkHash] = false - } - } - - description := otherManager.SnapshotManager.DownloadSequence(otherSnapshot.ChunkSequence) - err := otherSnapshot.LoadChunks(description) - if err != nil { - LOG_ERROR("SNAPSHOT_CHUNK", "Failed to load chunks for destination snapshot %s at revision %d: %v", - otherSnapshot.ID, otherSnapshot.Revision, err) - return false - } - - for _, chunkHash := range otherSnapshot.ChunkHashes { - if _, found := chunks[chunkHash]; found { - chunks[chunkHash] = false + if _, found := chunks[chunkHash]; !found { + chunks[chunkHash] = true } } } + otherChunkFiles, otherChunkSizes := otherManager.SnapshotManager.ListAllFiles(otherManager.storage, "chunks/") + + for i, otherChunkID := range otherChunkFiles { + otherChunkID = strings.Replace(otherChunkID, "/", "", -1) + if len(otherChunkID) != 64 { + continue + } + if otherChunkSizes[i] == 0 { + LOG_DEBUG("SNAPSHOT_COPY", "Chunk %s has length = 0", otherChunkID) + continue + } + otherChunks[otherChunkID] = false + } + + LOG_DEBUG("SNAPSHOT_COPY", "Found %d chunks on destination storage", len(otherChunks)) + + chunksToCopy := 0 + chunksToSkip := 0 + + for chunkHash, _ := range chunks { + otherChunkID := otherManager.config.GetChunkIDFromHash(chunkHash) + if _, found := otherChunks[otherChunkID]; found { + chunksToSkip++ + } else { + chunksToCopy++ + } + } + + LOG_DEBUG("SNAPSHOT_COPY", "Chunks to copy = %d, to skip = %d, total = %d", chunksToCopy, chunksToSkip, chunksToCopy + chunksToSkip) + LOG_DEBUG("SNAPSHOT_COPY", "Total chunks in source snapshot revisions = %d\n", len(chunks)) + chunkDownloader := CreateChunkDownloader(manager.config, manager.storage, nil, false, threads) chunkUploader := CreateChunkUploader(otherManager.config, otherManager.storage, nil, threads, @@ -1663,11 +1656,11 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho totalSkipped := 0 chunkIndex := 0 - for chunkHash, needsCopy := range chunks { + for chunkHash, _ := range chunks { chunkIndex++ chunkID := manager.config.GetChunkIDFromHash(chunkHash) - if needsCopy { - newChunkID := otherManager.config.GetChunkIDFromHash(chunkHash) + newChunkID := otherManager.config.GetChunkIDFromHash(chunkHash) + if _, found := otherChunks[newChunkID]; !found { LOG_DEBUG("SNAPSHOT_COPY", "Copying chunk %s to %s", chunkID, newChunkID) i := chunkDownloader.AddChunk(chunkHash) chunk := chunkDownloader.WaitForChunk(i) @@ -1685,7 +1678,7 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho chunkDownloader.Stop() chunkUploader.Stop() - LOG_INFO("SNAPSHOT_COPY", "Total chunks copied = %d, skipped = %d.", totalCopied, totalSkipped) + LOG_INFO("SNAPSHOT_COPY", "Copy complete, %d total chunks, %d chunks copied, %d skipped", totalCopied + totalSkipped, totalCopied, totalSkipped) for _, snapshot := range snapshots { if revisionMap[snapshot.ID][snapshot.Revision] == false {