diff --git a/src/duplicacy_chunkoperator.go b/src/duplicacy_chunkoperator.go index 6407938..1585e8e 100644 --- a/src/duplicacy_chunkoperator.go +++ b/src/duplicacy_chunkoperator.go @@ -124,7 +124,20 @@ func (operator *ChunkOperator) Run(threadIndex int, task ChunkOperatorTask) { LOG_ERROR("CHUNK_FIND", "Failed to locate the path for the chunk %s: %v", task.chunkID, err) return } else if !exist { - LOG_ERROR("CHUNK_FIND", "Chunk %s does not exist in the storage", task.chunkID) + if task.operation == ChunkOperationDelete { + LOG_WARN("CHUNK_FIND", "Chunk %s does not exist in the storage", task.chunkID) + return + } + + fossilPath, exist, _, _ := operator.storage.FindChunk(threadIndex, task.chunkID, true) + if exist { + LOG_WARN("CHUNK_FOSSILIZE", "Chunk %s is already a fossil", task.chunkID) + operator.fossilsLock.Lock() + operator.fossils = append(operator.fossils, fossilPath) + operator.fossilsLock.Unlock() + } else { + LOG_ERROR("CHUNK_FIND", "Chunk %s does not exist in the storage", task.chunkID) + } return } task.filePath = filePath @@ -162,6 +175,9 @@ func (operator *ChunkOperator) Run(threadIndex int, task ChunkOperatorTask) { if err == nil { LOG_TRACE("CHUNK_DELETE", "Deleted chunk file %s as the fossil already exists", task.chunkID) } + operator.fossilsLock.Lock() + operator.fossils = append(operator.fossils, fossilPath) + operator.fossilsLock.Unlock() } else { LOG_ERROR("CHUNK_DELETE", "Failed to fossilize the chunk %s: %v", task.chunkID, err) } diff --git a/src/duplicacy_log.go b/src/duplicacy_log.go index a18b4fc..b6326ee 100644 --- a/src/duplicacy_log.go +++ b/src/duplicacy_log.go @@ -129,7 +129,7 @@ func logf(level int, logID string, format string, v ...interface{}) { // fmt.Printf("%s %s %s %s\n", now.Format("2006-01-02 15:04:05.000"), getLevelName(level), logID, message) if testingT != nil { - if level < WARN { + if level <= WARN { if level >= loggingLevel { testingT.Logf("%s %s %s %s\n", now.Format("2006-01-02 15:04:05.000"), getLevelName(level), logID, message) diff --git a/src/duplicacy_snapshotmanager.go b/src/duplicacy_snapshotmanager.go index 19774c2..6905c23 100644 --- a/src/duplicacy_snapshotmanager.go +++ b/src/duplicacy_snapshotmanager.go @@ -38,6 +38,9 @@ type FossilCollection struct { // The lastest revision for each snapshot id when the fossil collection was created. LastRevisions map[string]int `json:"last_revisions"` + // Record the set of snapshots that have been removed by the prune command that created this fossil collection + DeletedRevisions map[string][]int `json:"deleted_revisions"` + // Fossils (i.e., chunks not referenced by any snapshots) Fossils []string `json:"fossils"` @@ -55,6 +58,7 @@ func CreateFossilCollection(allSnapshots map[string][]*Snapshot) *FossilCollecti return &FossilCollection{ LastRevisions: lastRevisions, + DeletedRevisions: make(map[string][]int), } } @@ -1742,6 +1746,29 @@ func (manager *SnapshotManager) PruneSnapshots(selfID string, snapshotID string, return false } + // Determine if any deleted revisions exist + exist := false + for snapshotID, revisionList := range collection.DeletedRevisions { + for _, revision := range revisionList { + for _, snapshot := range allSnapshots[snapshotID] { + if revision == snapshot.Revision { + LOG_INFO("FOSSIL_GHOSTSNAPSHOT", "Snapshot %s revision %d should have been deleted already", snapshotID, revision) + exist = true + } + } + } + } + + if exist { + err = manager.snapshotCache.DeleteFile(0, collectionFile) + if err != nil { + LOG_WARN("FOSSIL_FILE", "Failed to remove the fossil collection file %s: %v", collectionFile, err) + } else { + LOG_INFO("FOSSIL_IGNORE", "The fossil collection file %s has been ignored due to ghost snapshots", collectionFile) + } + continue + } + for _, fossil := range collection.Fossils { referencedFossils[fossil] = true } @@ -1936,6 +1963,15 @@ func (manager *SnapshotManager) PruneSnapshots(selfID string, snapshotID string, collection.AddFossil(fossil) } + // Save the deleted revision in the fossil collection + for _, snapshots := range allSnapshots { + for _, snapshot := range snapshots { + if snapshot.Flag { + collection.DeletedRevisions[snapshot.ID] = append(collection.DeletedRevisions[snapshot.ID], snapshot.Revision) + } + } + } + // Save the fossil collection if it is not empty. if !collection.IsEmpty() && !dryRun && !exclusive { collection.EndTime = time.Now().Unix() diff --git a/src/duplicacy_snapshotmanager_test.go b/src/duplicacy_snapshotmanager_test.go index e3e0373..333f5b7 100644 --- a/src/duplicacy_snapshotmanager_test.go +++ b/src/duplicacy_snapshotmanager_test.go @@ -14,6 +14,7 @@ import ( "strings" "testing" "time" + "io/ioutil" ) func createDummySnapshot(snapshotID string, revision int, endTime int64) *Snapshot { @@ -620,4 +621,67 @@ func TestPruneNewSnapshots(t *testing.T) { snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false, 1) checkTestSnapshots(snapshotManager, 4, 0) snapshotManager.CheckSnapshots("vm1@host1", []int{2, 3}, "", false, false, false, false, false); -} \ No newline at end of file +} + +// A fossil collection left by an aborted prune should be ignored if any supposedly deleted snapshot exists +func TestPruneGhostSnapshots(t *testing.T) { + setTestingT(t) + + EnableStackTrace() + + testDir := path.Join(os.TempDir(), "duplicacy_test", "snapshot_test") + + snapshotManager := createTestSnapshotManager(testDir) + + chunkSize := 1024 + chunkHash1 := uploadRandomChunk(snapshotManager, chunkSize) + chunkHash2 := uploadRandomChunk(snapshotManager, chunkSize) + chunkHash3 := uploadRandomChunk(snapshotManager, chunkSize) + + now := time.Now().Unix() + day := int64(24 * 3600) + t.Logf("Creating 2 snapshots") + createTestSnapshot(snapshotManager, "vm1@host1", 1, now-3*day-3600, now-3*day-60, []string{chunkHash1, chunkHash2}, "tag") + createTestSnapshot(snapshotManager, "vm1@host1", 2, now-2*day-3600, now-2*day-60, []string{chunkHash2, chunkHash3}, "tag") + checkTestSnapshots(snapshotManager, 2, 0) + + snapshot1, err := ioutil.ReadFile(path.Join(testDir, "snapshots", "vm1@host1", "1")) + if err != nil { + t.Errorf("Failed to read snapshot file: %v", err) + } + + t.Logf("Prune snapshot 1") + // chunkHash1 should be marked as fossil + snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{1}, []string{}, []string{}, false, false, []string{}, false, false, false, 1) + checkTestSnapshots(snapshotManager, 1, 2) + + // Recover the snapshot file for revision 1; this is to simulate a scenario where prune may encounter a network error after + // leaving the fossil collection but before deleting any snapshots. + err = ioutil.WriteFile(path.Join(testDir, "snapshots", "vm1@host1", "1"), snapshot1, 0644) + if err != nil { + t.Errorf("Failed to write snapshot file: %v", err) + } + + // Create another snapshot of vm1 so the fossil collection becomes eligible for processing. + chunkHash4 := uploadRandomChunk(snapshotManager, chunkSize) + createTestSnapshot(snapshotManager, "vm1@host1", 3, now - day - 3600, now - day - 60, []string{chunkHash3, chunkHash4}, "tag") + + // Run the prune again but the fossil collection should be igored, since revision 1 still exists + snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false, 1) + checkTestSnapshots(snapshotManager, 3, 2) + snapshotManager.CheckSnapshots("vm1@host1", []int{1, 2, 3}, "", false, false, false, true /*searchFossils*/, false); + + // Prune snapshot 1 again + snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{1}, []string{}, []string{}, false, false, []string{}, false, false, false, 1) + checkTestSnapshots(snapshotManager, 2, 2) + + // Create another snapshot + chunkHash5 := uploadRandomChunk(snapshotManager, chunkSize) + createTestSnapshot(snapshotManager, "vm1@host1", 4, now + 3600, now + 3600 * 2, []string{chunkHash5, chunkHash5}, "tag") + checkTestSnapshots(snapshotManager, 3, 2) + + // Run the prune again and this time the fossil collection will be processed and the fossils removed + snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false, 1) + checkTestSnapshots(snapshotManager, 3, 0) + snapshotManager.CheckSnapshots("vm1@host1", []int{2, 3, 4}, "", false, false, false, false, false); +}