Add keepChunkHashes flag to GetSnapshotChunks, allowing reduced memory

src/duplicacy_backupmanager.go

@@ -199,7 +199,7 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
 	// A revision number of 0 means this is the initial backup
 	if remoteSnapshot.Revision > 0 {
 		// Add all chunks in the last snapshot to the cache
-		for _, chunkID := range manager.SnapshotManager.GetSnapshotChunks(remoteSnapshot) {
+		for _, chunkID := range manager.SnapshotManager.GetSnapshotChunks(remoteSnapshot, true) {
 			chunkCache[chunkID] = true
 		}
 	} else {

src/duplicacy_snapshot.go

@@ -357,6 +357,11 @@ func (snapshot *Snapshot) LoadChunks(description []byte) (err error) {
 	return err
 }
 
+// ClearChunks removes loaded chunks from memory
+func (snapshot *Snapshot) ClearChunks() {
+	snapshot.ChunkHashes = nil
+}
+
 // LoadLengths construct 'ChunkLengths' from the json description.
 func (snapshot *Snapshot) LoadLengths(description []byte) (err error) {
 	return json.Unmarshal(description, &snapshot.ChunkLengths)

src/duplicacy_snapshotmanager.go

@@ -387,7 +387,7 @@ func (manager *SnapshotManager) CleanSnapshotCache(latestSnapshot *Snapshot, all
 	chunks := make(map[string]bool)
 
 	if latestSnapshot != nil {
-		for _, chunkID := range manager.GetSnapshotChunks(latestSnapshot) {
+		for _, chunkID := range manager.GetSnapshotChunks(latestSnapshot, false) {
 			chunks[chunkID] = true
 		}
 	}
@@ -597,8 +597,9 @@ func (manager *SnapshotManager) ListAllFiles(storage Storage, top string) (allFi
 	return allFiles, allSizes
 }
 
-// GetSnapshotChunks returns all chunks referenced by a given snapshot.
-func (manager *SnapshotManager) GetSnapshotChunks(snapshot *Snapshot) (chunks []string) {
+// GetSnapshotChunks returns all chunks referenced by a given snapshot. If
+// keepChunkHashes is true, snapshot.ChunkHashes will be populated.
+func (manager *SnapshotManager) GetSnapshotChunks(snapshot *Snapshot, keepChunkHashes bool) (chunks []string) {
 
 	for _, chunkHash := range snapshot.FileSequence {
 		chunks = append(chunks, manager.config.GetChunkIDFromHash(chunkHash))
@@ -627,6 +628,10 @@ func (manager *SnapshotManager) GetSnapshotChunks(snapshot *Snapshot) (chunks []
 		chunks = append(chunks, manager.config.GetChunkIDFromHash(chunkHash))
 	}
 
+	if !keepChunkHashes {
+		snapshot.ClearChunks()
+	}
+
 	return chunks
 }
 
@@ -715,7 +720,7 @@ func (manager *SnapshotManager) ListSnapshots(snapshotID string, revisionsToList
 			}
 
 			if showChunks {
-				for _, chunkID := range manager.GetSnapshotChunks(snapshot) {
+				for _, chunkID := range manager.GetSnapshotChunks(snapshot, false) {
 					LOG_INFO("SNAPSHOT_CHUNKS", "chunk: %s", chunkID)
 				}
 			}
@@ -805,7 +810,7 @@ func (manager *SnapshotManager) CheckSnapshots(snapshotID string, revisionsToChe
 			}
 
 			chunks := make(map[string]bool)
-			for _, chunkID := range manager.GetSnapshotChunks(snapshot) {
+			for _, chunkID := range manager.GetSnapshotChunks(snapshot, false) {
 				chunks[chunkID] = true
 			}
 
@@ -895,7 +900,7 @@ func (manager *SnapshotManager) ShowStatistics(snapshotMap map[string][]*Snapsho
 		for _, snapshot := range snapshotList {
 
 			chunks := make(map[string]bool)
-			for _, chunkID := range manager.GetSnapshotChunks(snapshot) {
+			for _, chunkID := range manager.GetSnapshotChunks(snapshot, false) {
 				chunks[chunkID] = true
 				snapshotChunks[chunkID] = true
 			}
@@ -948,7 +953,7 @@ func (manager *SnapshotManager) ShowStatisticsTabular(snapshotMap map[string][]*
 		earliestSeenChunks := make(map[string]int)
 
 		for _, snapshot := range snapshotList {
-			for _, chunkID := range manager.GetSnapshotChunks(snapshot) {
+			for _, chunkID := range manager.GetSnapshotChunks(snapshot, true) {
 				if earliestSeenChunks[chunkID] == 0 {
 					earliestSeenChunks[chunkID] = math.MaxInt32
 				}
@@ -959,7 +964,7 @@ func (manager *SnapshotManager) ShowStatisticsTabular(snapshotMap map[string][]*
 		for _, snapshot := range snapshotList {
 
 			chunks := make(map[string]bool)
-			for _, chunkID := range manager.GetSnapshotChunks(snapshot) {
+			for _, chunkID := range manager.GetSnapshotChunks(snapshot, true) {
 				chunks[chunkID] = true
 				snapshotChunks[chunkID] = true
 			}
@@ -1754,7 +1759,7 @@ func (manager *SnapshotManager) PruneSnapshots(selfID string, snapshotID string,
 			newChunks := make(map[string]bool)
 
 			for _, newSnapshot := range newSnapshots {
-				for _, chunk := range manager.GetSnapshotChunks(newSnapshot) {
+				for _, chunk := range manager.GetSnapshotChunks(newSnapshot, false) {
 					newChunks[chunk] = true
 				}
 			}
@@ -1938,7 +1943,7 @@ func (manager *SnapshotManager) PruneSnapshots(selfID string, snapshotID string,
 				continue
 			}
 
-			chunks := manager.GetSnapshotChunks(snapshot)
+			chunks := manager.GetSnapshotChunks(snapshot, false)
 
 			for _, chunk := range chunks {
 				// The initial value is 'false'.  When a referenced chunk is found it will change the value to 'true'.
@@ -2064,7 +2069,7 @@ func (manager *SnapshotManager) PruneSnapshots(selfID string, snapshotID string,
 					continue
 				}
 
-				chunks := manager.GetSnapshotChunks(snapshot)
+				chunks := manager.GetSnapshotChunks(snapshot, false)
 
 				for _, chunk := range chunks {
 

src/duplicacy_snapshotmanager_test.go

@@ -199,7 +199,7 @@ func checkTestSnapshots(manager *SnapshotManager, expectedSnapshots int, expecte
 			snapshot := manager.DownloadSnapshot(snapshotID, revision)
 			numberOfSnapshots++
 
-			for _, chunk := range manager.GetSnapshotChunks(snapshot) {
+			for _, chunk := range manager.GetSnapshotChunks(snapshot, false) {
 				chunks[chunk] = true
 			}
 		}