From 67a3103467afc5481b9e861979dfc8ef1a4497b6 Mon Sep 17 00:00:00 2001 From: Gilbert Chen Date: Tue, 22 Sep 2020 08:27:09 -0400 Subject: [PATCH] Add a -max-list-rate option to backup to slow down the listing This option sets the maximum number of files that can be listed in one second. --- duplicacy/duplicacy_main.go | 10 +++++++++- src/duplicacy_backupmanager.go | 6 +++--- src/duplicacy_backupmanager_test.go | 12 ++++++------ src/duplicacy_snapshot.go | 13 ++++++++++++- src/duplicacy_snapshotmanager.go | 2 +- 5 files changed, 31 insertions(+), 12 deletions(-) diff --git a/duplicacy/duplicacy_main.go b/duplicacy/duplicacy_main.go index 5f3736c..93b8274 100644 --- a/duplicacy/duplicacy_main.go +++ b/duplicacy/duplicacy_main.go @@ -745,13 +745,14 @@ func backupRepository(context *cli.Context) { dryRun := context.Bool("dry-run") uploadRateLimit := context.Int("limit-rate") enumOnly := context.Bool("enum-only") + listRateLimit := context.Int("max-list-rate") storage.SetRateLimits(0, uploadRateLimit) backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, preference.NobackupFile, preference.FiltersFile) duplicacy.SavePassword(*preference, "password", password) backupManager.SetupSnapshotCache(preference.Name) backupManager.SetDryRun(dryRun) - backupManager.Backup(repository, quickMode, threads, context.String("t"), showStatistics, enableVSS, vssTimeout, enumOnly) + backupManager.Backup(repository, quickMode, threads, context.String("t"), showStatistics, enableVSS, vssTimeout, enumOnly, listRateLimit) runScript(context, preference.Name, "post") } @@ -1465,6 +1466,13 @@ func main() { Name: "enum-only", Usage: "enumerate the repository recursively and then exit", }, + cli.IntFlag{ + Name: "max-list-rate", + Value: 0, + Usage: "the maximum number of files to list in one second", + Argument: "", + }, + }, Usage: "Save a snapshot of the repository to the storage", ArgsUsage: " ", diff --git a/src/duplicacy_backupmanager.go b/src/duplicacy_backupmanager.go index dd04ba3..6d62a5e 100644 --- a/src/duplicacy_backupmanager.go +++ b/src/duplicacy_backupmanager.go @@ -171,7 +171,7 @@ func setEntryContent(entries []*Entry, chunkLengths []int, offset int) { // unmodified files with last backup). Otherwise (or if this is the first backup), the entire repository will // be scanned to create the snapshot. 'tag' is the tag assigned to the new snapshot. func (manager *BackupManager) Backup(top string, quickMode bool, threads int, tag string, - showStatistics bool, shadowCopy bool, shadowCopyTimeout int, enumOnly bool) bool { + showStatistics bool, shadowCopy bool, shadowCopyTimeout int, enumOnly bool, listRateLimit int) bool { var err error top, err = filepath.Abs(top) @@ -201,7 +201,7 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta LOG_INFO("BACKUP_INDEXING", "Indexing %s", top) localSnapshot, skippedDirectories, skippedFiles, err := CreateSnapshotFromDirectory(manager.snapshotID, shadowTop, - manager.nobackupFile, manager.filtersFile) + manager.nobackupFile, manager.filtersFile, listRateLimit) if err != nil { LOG_ERROR("SNAPSHOT_LIST", "Failed to list the directory %s: %v", top, err) return false @@ -784,7 +784,7 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu manager.SnapshotManager.DownloadSnapshotContents(remoteSnapshot, patterns, true) localSnapshot, _, _, err := CreateSnapshotFromDirectory(manager.snapshotID, top, manager.nobackupFile, - manager.filtersFile) + manager.filtersFile, 0) if err != nil { LOG_ERROR("SNAPSHOT_LIST", "Failed to list the repository: %v", err) return 0 diff --git a/src/duplicacy_backupmanager_test.go b/src/duplicacy_backupmanager_test.go index 085c905..653c69b 100644 --- a/src/duplicacy_backupmanager_test.go +++ b/src/duplicacy_backupmanager_test.go @@ -249,7 +249,7 @@ func TestBackupManager(t *testing.T) { backupManager.SetupSnapshotCache("default") SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy") - backupManager.Backup(testDir+"/repository1" /*quickMode=*/, true, threads, "first", false, false, 0, false) + backupManager.Backup(testDir+"/repository1" /*quickMode=*/, true, threads, "first", false, false, 0, false, 0) time.Sleep(time.Duration(delay) * time.Second) SetDuplicacyPreferencePath(testDir + "/repository2/.duplicacy") failedFiles := backupManager.Restore(testDir+"/repository2", threads /*inPlace=*/, false /*quickMode=*/, false, threads /*overwrite=*/, true, @@ -274,7 +274,7 @@ func TestBackupManager(t *testing.T) { modifyFile(testDir+"/repository1/dir1/file3", 0.3) SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy") - backupManager.Backup(testDir+"/repository1" /*quickMode=*/, true, threads, "second", false, false, 0, false) + backupManager.Backup(testDir+"/repository1" /*quickMode=*/, true, threads, "second", false, false, 0, false, 0) time.Sleep(time.Duration(delay) * time.Second) SetDuplicacyPreferencePath(testDir + "/repository2/.duplicacy") failedFiles = backupManager.Restore(testDir+"/repository2", 2 /*inPlace=*/, true /*quickMode=*/, true, threads /*overwrite=*/, true, @@ -295,7 +295,7 @@ func TestBackupManager(t *testing.T) { os.Mkdir(testDir+"/repository1/dir2/dir3", 0700) os.Mkdir(testDir+"/repository1/dir4", 0700) SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy") - backupManager.Backup(testDir+"/repository1" /*quickMode=*/, false, threads, "third", false, false, 0, false) + backupManager.Backup(testDir+"/repository1" /*quickMode=*/, false, threads, "third", false, false, 0, false, 0) time.Sleep(time.Duration(delay) * time.Second) // Create some directories and files under repository2 that will be deleted during restore @@ -360,7 +360,7 @@ func TestBackupManager(t *testing.T) { } backupManager.SnapshotManager.CheckSnapshots( /*snapshotID*/ "host1" /*revisions*/, []int{2, 3} /*tag*/, "", /*showStatistics*/ false /*showTabular*/, false /*checkFiles*/, false /*checkChunks*/, false /*searchFossils*/, false /*resurrect*/, false, 1 /*allowFailures*/, false) - backupManager.Backup(testDir+"/repository1" /*quickMode=*/, false, threads, "fourth", false, false, 0, false) + backupManager.Backup(testDir+"/repository1" /*quickMode=*/, false, threads, "fourth", false, false, 0, false, 0) backupManager.SnapshotManager.PruneSnapshots("host1", "host1" /*revisions*/, nil /*tags*/, nil /*retentions*/, nil, /*exhaustive*/ false /*exclusive=*/, true /*ignoredIDs*/, nil /*dryRun*/, false /*deleteOnly*/, false /*collectOnly*/, false, 1) numberOfSnapshots = backupManager.SnapshotManager.ListSnapshots( /*snapshotID*/ "host1" /*revisionsToList*/, nil /*tag*/, "" /*showFiles*/, false /*showChunks*/, false) @@ -525,7 +525,7 @@ func TestPersistRestore(t *testing.T) { unencBackupManager.SetupSnapshotCache("default") SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy") - unencBackupManager.Backup(testDir+"/repository1" /*quickMode=*/, true, threads, "first", false, false, 0, false) + unencBackupManager.Backup(testDir+"/repository1" /*quickMode=*/, true, threads, "first", false, false, 0, false, 0) time.Sleep(time.Duration(delay) * time.Second) @@ -535,7 +535,7 @@ func TestPersistRestore(t *testing.T) { encBackupManager.SetupSnapshotCache("default") SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy") - encBackupManager.Backup(testDir+"/repository1" /*quickMode=*/, true, threads, "first", false, false, 0, false) + encBackupManager.Backup(testDir+"/repository1" /*quickMode=*/, true, threads, "first", false, false, 0, false, 0) time.Sleep(time.Duration(delay) * time.Second) diff --git a/src/duplicacy_snapshot.go b/src/duplicacy_snapshot.go index 45180f9..46d3f2f 100644 --- a/src/duplicacy_snapshot.go +++ b/src/duplicacy_snapshot.go @@ -58,7 +58,7 @@ func CreateEmptySnapshot(id string) (snapshto *Snapshot) { // CreateSnapshotFromDirectory creates a snapshot from the local directory 'top'. Only 'Files' // will be constructed, while 'ChunkHashes' and 'ChunkLengths' can only be populated after uploading. -func CreateSnapshotFromDirectory(id string, top string, nobackupFile string, filtersFile string) (snapshot *Snapshot, skippedDirectories []string, +func CreateSnapshotFromDirectory(id string, top string, nobackupFile string, filtersFile string, listRateLimit int) (snapshot *Snapshot, skippedDirectories []string, skippedFiles []string, err error) { snapshot = &Snapshot{ @@ -84,11 +84,22 @@ func CreateSnapshotFromDirectory(id string, top string, nobackupFile string, fil attributeThreshold, _ = strconv.Atoi(attributeThresholdValue) } + startTime := time.Now() + for len(directories) > 0 { directory := directories[len(directories)-1] directories = directories[:len(directories)-1] snapshot.Files = append(snapshot.Files, directory) + + if listRateLimit > 0 { + maxFiles := int(time.Now().Sub(startTime).Seconds() * float64(listRateLimit)) + if len(snapshot.Files) > maxFiles { + delay := float64(len(snapshot.Files) - maxFiles) / float64(listRateLimit) + time.Sleep(time.Duration(delay * 1000.0) * time.Millisecond) + } + } + subdirectories, skipped, err := ListEntries(top, directory.Path, &snapshot.Files, patterns, nobackupFile, snapshot.discardAttributes) if err != nil { if directory.Path == "" { diff --git a/src/duplicacy_snapshotmanager.go b/src/duplicacy_snapshotmanager.go index da3b08e..a8742ac 100644 --- a/src/duplicacy_snapshotmanager.go +++ b/src/duplicacy_snapshotmanager.go @@ -1396,7 +1396,7 @@ func (manager *SnapshotManager) Diff(top string, snapshotID string, revisions [] if len(revisions) <= 1 { // Only scan the repository if filePath is not provided if len(filePath) == 0 { - rightSnapshot, _, _, err = CreateSnapshotFromDirectory(snapshotID, top, nobackupFile, filtersFile) + rightSnapshot, _, _, err = CreateSnapshotFromDirectory(snapshotID, top, nobackupFile, filtersFile, 0) if err != nil { LOG_ERROR("SNAPSHOT_LIST", "Failed to list the directory %s: %v", top, err) return false