Add a -max-list-rate option to backup to slow down the listing

This option sets the maximum number of files that can be listed in one
second.

duplicacy/duplicacy_main.go

@@ -745,13 +745,14 @@ func backupRepository(context *cli.Context) {
 	dryRun := context.Bool("dry-run")
 	uploadRateLimit := context.Int("limit-rate")
 	enumOnly := context.Bool("enum-only")
+	listRateLimit := context.Int("max-list-rate")
 	storage.SetRateLimits(0, uploadRateLimit)
 	backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, preference.NobackupFile, preference.FiltersFile)
 	duplicacy.SavePassword(*preference, "password", password)
 
 	backupManager.SetupSnapshotCache(preference.Name)
 	backupManager.SetDryRun(dryRun)
-	backupManager.Backup(repository, quickMode, threads, context.String("t"), showStatistics, enableVSS, vssTimeout, enumOnly)
+	backupManager.Backup(repository, quickMode, threads, context.String("t"), showStatistics, enableVSS, vssTimeout, enumOnly, listRateLimit)
 
 	runScript(context, preference.Name, "post")
 }
@@ -1465,6 +1466,13 @@ func main() {
 					Name:  "enum-only",
 					Usage: "enumerate the repository recursively and then exit",
 				},
+				cli.IntFlag{
+					Name:     "max-list-rate",
+					Value:    0,
+					Usage:    "the maximum number of files to list in one second",
+					Argument: "<number>",
+				},
+
 			},
 			Usage:     "Save a snapshot of the repository to the storage",
 			ArgsUsage: " ",

src/duplicacy_backupmanager.go

@@ -171,7 +171,7 @@ func setEntryContent(entries []*Entry, chunkLengths []int, offset int) {
 // unmodified files with last backup).  Otherwise (or if this is the first backup), the entire repository will
 // be scanned to create the snapshot.  'tag' is the tag assigned to the new snapshot.
 func (manager *BackupManager) Backup(top string, quickMode bool, threads int, tag string,
-	showStatistics bool, shadowCopy bool, shadowCopyTimeout int, enumOnly bool) bool {
+	showStatistics bool, shadowCopy bool, shadowCopyTimeout int, enumOnly bool, listRateLimit int) bool {
 
 	var err error
 	top, err = filepath.Abs(top)
@@ -201,7 +201,7 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
 
 	LOG_INFO("BACKUP_INDEXING", "Indexing %s", top)
 	localSnapshot, skippedDirectories, skippedFiles, err := CreateSnapshotFromDirectory(manager.snapshotID, shadowTop,
-		                                                                                manager.nobackupFile, manager.filtersFile)
+		                                                                                manager.nobackupFile, manager.filtersFile, listRateLimit)
 	if err != nil {
 		LOG_ERROR("SNAPSHOT_LIST", "Failed to list the directory %s: %v", top, err)
 		return false
@@ -784,7 +784,7 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
 	manager.SnapshotManager.DownloadSnapshotContents(remoteSnapshot, patterns, true)
 
 	localSnapshot, _, _, err := CreateSnapshotFromDirectory(manager.snapshotID, top, manager.nobackupFile,
-		                                                    manager.filtersFile)
+		                                                    manager.filtersFile, 0)
 	if err != nil {
 		LOG_ERROR("SNAPSHOT_LIST", "Failed to list the repository: %v", err)
 		return 0

src/duplicacy_backupmanager_test.go

@@ -249,7 +249,7 @@ func TestBackupManager(t *testing.T) {
 	backupManager.SetupSnapshotCache("default")
 
 	SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy")
-	backupManager.Backup(testDir+"/repository1" /*quickMode=*/, true, threads, "first", false, false, 0, false)
+	backupManager.Backup(testDir+"/repository1" /*quickMode=*/, true, threads, "first", false, false, 0, false, 0)
 	time.Sleep(time.Duration(delay) * time.Second)
 	SetDuplicacyPreferencePath(testDir + "/repository2/.duplicacy")
 	failedFiles := backupManager.Restore(testDir+"/repository2", threads /*inPlace=*/, false /*quickMode=*/, false, threads /*overwrite=*/, true,
@@ -274,7 +274,7 @@ func TestBackupManager(t *testing.T) {
 	modifyFile(testDir+"/repository1/dir1/file3", 0.3)
 
 	SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy")
-	backupManager.Backup(testDir+"/repository1" /*quickMode=*/, true, threads, "second", false, false, 0, false)
+	backupManager.Backup(testDir+"/repository1" /*quickMode=*/, true, threads, "second", false, false, 0, false, 0)
 	time.Sleep(time.Duration(delay) * time.Second)
 	SetDuplicacyPreferencePath(testDir + "/repository2/.duplicacy")
 	failedFiles = backupManager.Restore(testDir+"/repository2", 2 /*inPlace=*/, true /*quickMode=*/, true, threads /*overwrite=*/, true,
@@ -295,7 +295,7 @@ func TestBackupManager(t *testing.T) {
 	os.Mkdir(testDir+"/repository1/dir2/dir3", 0700)
 	os.Mkdir(testDir+"/repository1/dir4", 0700)
 	SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy")
-	backupManager.Backup(testDir+"/repository1" /*quickMode=*/, false, threads, "third", false, false, 0, false)
+	backupManager.Backup(testDir+"/repository1" /*quickMode=*/, false, threads, "third", false, false, 0, false, 0)
 	time.Sleep(time.Duration(delay) * time.Second)
 
 	// Create some directories and files under repository2 that will be deleted during restore
@@ -360,7 +360,7 @@ func TestBackupManager(t *testing.T) {
 	}
 	backupManager.SnapshotManager.CheckSnapshots( /*snapshotID*/ "host1" /*revisions*/, []int{2, 3} /*tag*/, "",
 		/*showStatistics*/ false /*showTabular*/, false /*checkFiles*/, false /*checkChunks*/, false /*searchFossils*/, false /*resurrect*/, false, 1 /*allowFailures*/, false)
-	backupManager.Backup(testDir+"/repository1" /*quickMode=*/, false, threads, "fourth", false, false, 0, false)
+	backupManager.Backup(testDir+"/repository1" /*quickMode=*/, false, threads, "fourth", false, false, 0, false, 0)
 	backupManager.SnapshotManager.PruneSnapshots("host1", "host1" /*revisions*/, nil /*tags*/, nil /*retentions*/, nil,
 		/*exhaustive*/ false /*exclusive=*/, true /*ignoredIDs*/, nil /*dryRun*/, false /*deleteOnly*/, false /*collectOnly*/, false, 1)
 	numberOfSnapshots = backupManager.SnapshotManager.ListSnapshots( /*snapshotID*/ "host1" /*revisionsToList*/, nil /*tag*/, "" /*showFiles*/, false /*showChunks*/, false)
@@ -525,7 +525,7 @@ func TestPersistRestore(t *testing.T) {
 	unencBackupManager.SetupSnapshotCache("default")
 
 	SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy")
-	unencBackupManager.Backup(testDir+"/repository1" /*quickMode=*/, true, threads, "first", false, false, 0, false)
+	unencBackupManager.Backup(testDir+"/repository1" /*quickMode=*/, true, threads, "first", false, false, 0, false, 0)
 	time.Sleep(time.Duration(delay) * time.Second)
 
 
@@ -535,7 +535,7 @@ func TestPersistRestore(t *testing.T) {
 	encBackupManager.SetupSnapshotCache("default")
 
 	SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy")
-	encBackupManager.Backup(testDir+"/repository1" /*quickMode=*/, true, threads, "first", false, false, 0, false)
+	encBackupManager.Backup(testDir+"/repository1" /*quickMode=*/, true, threads, "first", false, false, 0, false, 0)
 	time.Sleep(time.Duration(delay) * time.Second)
 
 

src/duplicacy_snapshot.go

@@ -58,7 +58,7 @@ func CreateEmptySnapshot(id string) (snapshto *Snapshot) {
 
 // CreateSnapshotFromDirectory creates a snapshot from the local directory 'top'.  Only 'Files'
 // will be constructed, while 'ChunkHashes' and 'ChunkLengths' can only be populated after uploading.
-func CreateSnapshotFromDirectory(id string, top string, nobackupFile string, filtersFile string) (snapshot *Snapshot, skippedDirectories []string,
+func CreateSnapshotFromDirectory(id string, top string, nobackupFile string, filtersFile string, listRateLimit int) (snapshot *Snapshot, skippedDirectories []string,
 	skippedFiles []string, err error) {
 
 	snapshot = &Snapshot{
@@ -84,11 +84,22 @@ func CreateSnapshotFromDirectory(id string, top string, nobackupFile string, fil
 		attributeThreshold, _ = strconv.Atoi(attributeThresholdValue)
 	}
 
+	startTime := time.Now()
+
 	for len(directories) > 0 {
 
 		directory := directories[len(directories)-1]
 		directories = directories[:len(directories)-1]
 		snapshot.Files = append(snapshot.Files, directory)
+
+		if listRateLimit > 0 {
+			maxFiles := int(time.Now().Sub(startTime).Seconds() * float64(listRateLimit))
+			if len(snapshot.Files) > maxFiles  {
+				delay := float64(len(snapshot.Files) - maxFiles) / float64(listRateLimit)
+				time.Sleep(time.Duration(delay * 1000.0) * time.Millisecond)
+			}
+		}
+
 		subdirectories, skipped, err := ListEntries(top, directory.Path, &snapshot.Files, patterns, nobackupFile, snapshot.discardAttributes)
 		if err != nil {
 			if directory.Path == "" {

src/duplicacy_snapshotmanager.go

@@ -1396,7 +1396,7 @@ func (manager *SnapshotManager) Diff(top string, snapshotID string, revisions []
 	if len(revisions) <= 1 {
 		// Only scan the repository if filePath is not provided
 		if len(filePath) == 0 {
-			rightSnapshot, _, _, err = CreateSnapshotFromDirectory(snapshotID, top, nobackupFile, filtersFile)
+			rightSnapshot, _, _, err = CreateSnapshotFromDirectory(snapshotID, top, nobackupFile, filtersFile, 0)
 			if err != nil {
 				LOG_ERROR("SNAPSHOT_LIST", "Failed to list the directory %s: %v", top, err)
 				return false