Record deleted snapshots in the fossil collection and if any deleted snapshot still exist nude the fossil collection

src/duplicacy_chunkoperator.go

@@ -124,7 +124,20 @@ func (operator *ChunkOperator) Run(threadIndex int, task ChunkOperatorTask) {
 				LOG_ERROR("CHUNK_FIND", "Failed to locate the path for the chunk %s: %v", task.chunkID, err)
 				return
 			} else if !exist {
-				LOG_ERROR("CHUNK_FIND", "Chunk %s does not exist in the storage", task.chunkID)
+				if task.operation == ChunkOperationDelete {
+					LOG_WARN("CHUNK_FIND", "Chunk %s does not exist in the storage", task.chunkID)
+					return
+				}
+
+				fossilPath, exist, _, _ := operator.storage.FindChunk(threadIndex, task.chunkID, true)
+				if exist {
+					LOG_WARN("CHUNK_FOSSILIZE", "Chunk %s is already a fossil", task.chunkID)
+					operator.fossilsLock.Lock()
+					operator.fossils = append(operator.fossils, fossilPath)
+					operator.fossilsLock.Unlock()
+				} else {
+					LOG_ERROR("CHUNK_FIND", "Chunk %s does not exist in the storage", task.chunkID)
+				}
 				return
 			}
 			task.filePath = filePath
@@ -162,6 +175,9 @@ func (operator *ChunkOperator) Run(threadIndex int, task ChunkOperatorTask) {
 				if err == nil {
 					LOG_TRACE("CHUNK_DELETE", "Deleted chunk file %s as the fossil already exists", task.chunkID)
 				}
+				operator.fossilsLock.Lock()
+				operator.fossils = append(operator.fossils, fossilPath)
+				operator.fossilsLock.Unlock()
 			} else {
 				LOG_ERROR("CHUNK_DELETE", "Failed to fossilize the chunk %s: %v", task.chunkID, err)
 			}

src/duplicacy_log.go

@@ -129,7 +129,7 @@ func logf(level int, logID string, format string, v ...interface{}) {
 	// fmt.Printf("%s %s %s %s\n", now.Format("2006-01-02 15:04:05.000"), getLevelName(level), logID, message)
 
 	if testingT != nil {
-		if level < WARN {
+		if level <= WARN {
 			if level >= loggingLevel {
 				testingT.Logf("%s %s %s %s\n",
 					now.Format("2006-01-02 15:04:05.000"), getLevelName(level), logID, message)

src/duplicacy_snapshotmanager.go

@@ -38,6 +38,9 @@ type FossilCollection struct {
 	// The lastest revision for each snapshot id when the fossil collection was created.
 	LastRevisions map[string]int `json:"last_revisions"`
 
+	// Record the set of snapshots that have been removed by the prune command that created this fossil collection
+	DeletedRevisions map[string][]int `json:"deleted_revisions"`
+
 	// Fossils (i.e., chunks not referenced by any snapshots)
 	Fossils []string `json:"fossils"`
 
@@ -55,6 +58,7 @@ func CreateFossilCollection(allSnapshots map[string][]*Snapshot) *FossilCollecti
 
 	return &FossilCollection{
 		LastRevisions: lastRevisions,
+		DeletedRevisions: make(map[string][]int),
 	}
 }
 
@@ -1742,6 +1746,29 @@ func (manager *SnapshotManager) PruneSnapshots(selfID string, snapshotID string,
 			return false
 		}
 
+		// Determine if any deleted revisions exist
+		exist := false
+		for snapshotID, revisionList := range collection.DeletedRevisions {
+			for _, revision := range revisionList {
+				for _, snapshot := range allSnapshots[snapshotID] {
+					if revision == snapshot.Revision {
+						LOG_INFO("FOSSIL_GHOSTSNAPSHOT", "Snapshot %s revision %d should have been deleted already", snapshotID, revision)
+						exist = true
+					}
+				}
+			}
+		}
+
+		if exist {
+			err = manager.snapshotCache.DeleteFile(0, collectionFile)
+			if err != nil {
+				LOG_WARN("FOSSIL_FILE", "Failed to remove the fossil collection file %s: %v", collectionFile, err)
+			} else {
+				LOG_INFO("FOSSIL_IGNORE", "The fossil collection file %s has been ignored due to ghost snapshots", collectionFile)
+			}
+			continue
+		}
+
 		for _, fossil := range collection.Fossils {
 			referencedFossils[fossil] = true
 		}
@@ -1936,6 +1963,15 @@ func (manager *SnapshotManager) PruneSnapshots(selfID string, snapshotID string,
 		collection.AddFossil(fossil)
 	}
 
+	// Save the deleted revision in the fossil collection
+	for _, snapshots := range allSnapshots {
+		for _, snapshot := range snapshots {
+			if snapshot.Flag {
+				collection.DeletedRevisions[snapshot.ID] = append(collection.DeletedRevisions[snapshot.ID], snapshot.Revision)
+			}
+		}
+	}
+
 	// Save the fossil collection if it is not empty.
 	if !collection.IsEmpty() && !dryRun && !exclusive {
 		collection.EndTime = time.Now().Unix()

src/duplicacy_snapshotmanager_test.go

@@ -14,6 +14,7 @@ import (
 	"strings"
 	"testing"
 	"time"
+	"io/ioutil"
 )
 
 func createDummySnapshot(snapshotID string, revision int, endTime int64) *Snapshot {
@@ -620,4 +621,67 @@ func TestPruneNewSnapshots(t *testing.T) {
 	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false, 1)
 	checkTestSnapshots(snapshotManager, 4, 0)
 	snapshotManager.CheckSnapshots("vm1@host1", []int{2, 3}, "", false, false, false, false, false);
-}
+}
+
+// A fossil collection left by an aborted prune should be ignored if any supposedly deleted snapshot exists
+func TestPruneGhostSnapshots(t *testing.T) {
+	setTestingT(t)
+
+	EnableStackTrace()
+
+	testDir := path.Join(os.TempDir(), "duplicacy_test", "snapshot_test")
+
+	snapshotManager := createTestSnapshotManager(testDir)
+
+	chunkSize := 1024
+	chunkHash1 := uploadRandomChunk(snapshotManager, chunkSize)
+	chunkHash2 := uploadRandomChunk(snapshotManager, chunkSize)
+	chunkHash3 := uploadRandomChunk(snapshotManager, chunkSize)
+
+	now := time.Now().Unix()
+	day := int64(24 * 3600)
+	t.Logf("Creating 2 snapshots")
+	createTestSnapshot(snapshotManager, "vm1@host1", 1, now-3*day-3600, now-3*day-60, []string{chunkHash1, chunkHash2}, "tag")
+	createTestSnapshot(snapshotManager, "vm1@host1", 2, now-2*day-3600, now-2*day-60, []string{chunkHash2, chunkHash3}, "tag")
+	checkTestSnapshots(snapshotManager, 2, 0)
+
+	snapshot1, err := ioutil.ReadFile(path.Join(testDir, "snapshots", "vm1@host1", "1"))
+	if err != nil {
+		t.Errorf("Failed to read snapshot file: %v", err)
+	}
+
+	t.Logf("Prune snapshot 1")
+	// chunkHash1 should be marked as fossil
+	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{1}, []string{}, []string{}, false, false, []string{}, false, false, false, 1)
+	checkTestSnapshots(snapshotManager, 1, 2)
+
+	// Recover the snapshot file for revision 1; this is to simulate a scenario where prune may encounter a network error after
+	// leaving the fossil collection but before deleting any snapshots.
+	err = ioutil.WriteFile(path.Join(testDir, "snapshots", "vm1@host1", "1"), snapshot1, 0644)
+	if err != nil {
+		t.Errorf("Failed to write snapshot file: %v", err)
+	}
+
+	// Create another snapshot of vm1 so the fossil collection becomes eligible for processing.
+	chunkHash4 := uploadRandomChunk(snapshotManager, chunkSize)
+	createTestSnapshot(snapshotManager, "vm1@host1", 3, now - day - 3600, now - day - 60, []string{chunkHash3, chunkHash4}, "tag")
+
+	// Run the prune again but the fossil collection should be igored, since revision 1 still exists
+	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false, 1)
+	checkTestSnapshots(snapshotManager, 3, 2)
+	snapshotManager.CheckSnapshots("vm1@host1", []int{1, 2, 3}, "", false, false, false, true /*searchFossils*/, false);
+
+	// Prune snapshot 1 again
+	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{1}, []string{}, []string{}, false, false, []string{}, false, false, false, 1)
+	checkTestSnapshots(snapshotManager, 2, 2)
+
+	// Create another snapshot
+	chunkHash5 := uploadRandomChunk(snapshotManager, chunkSize)
+	createTestSnapshot(snapshotManager, "vm1@host1", 4, now + 3600, now + 3600 * 2, []string{chunkHash5, chunkHash5}, "tag")
+	checkTestSnapshots(snapshotManager, 3, 2)
+
+	// Run the prune again and this time the fossil collection will be processed and the fossils removed
+	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false, 1)
+	checkTestSnapshots(snapshotManager, 3, 0)
+	snapshotManager.CheckSnapshots("vm1@host1", []int{2, 3, 4}, "", false, false, false, false, false);
+}