diff --git a/duplicacy/duplicacy_main.go b/duplicacy/duplicacy_main.go index 93b8274..5f3736c 100644 --- a/duplicacy/duplicacy_main.go +++ b/duplicacy/duplicacy_main.go @@ -745,14 +745,13 @@ func backupRepository(context *cli.Context) { dryRun := context.Bool("dry-run") uploadRateLimit := context.Int("limit-rate") enumOnly := context.Bool("enum-only") - listRateLimit := context.Int("max-list-rate") storage.SetRateLimits(0, uploadRateLimit) backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, preference.NobackupFile, preference.FiltersFile) duplicacy.SavePassword(*preference, "password", password) backupManager.SetupSnapshotCache(preference.Name) backupManager.SetDryRun(dryRun) - backupManager.Backup(repository, quickMode, threads, context.String("t"), showStatistics, enableVSS, vssTimeout, enumOnly, listRateLimit) + backupManager.Backup(repository, quickMode, threads, context.String("t"), showStatistics, enableVSS, vssTimeout, enumOnly) runScript(context, preference.Name, "post") } @@ -1466,13 +1465,6 @@ func main() { Name: "enum-only", Usage: "enumerate the repository recursively and then exit", }, - cli.IntFlag{ - Name: "max-list-rate", - Value: 0, - Usage: "the maximum number of files to list in one second", - Argument: "", - }, - }, Usage: "Save a snapshot of the repository to the storage", ArgsUsage: " ", diff --git a/src/duplicacy_backupmanager.go b/src/duplicacy_backupmanager.go index 6d62a5e..dd04ba3 100644 --- a/src/duplicacy_backupmanager.go +++ b/src/duplicacy_backupmanager.go @@ -171,7 +171,7 @@ func setEntryContent(entries []*Entry, chunkLengths []int, offset int) { // unmodified files with last backup). Otherwise (or if this is the first backup), the entire repository will // be scanned to create the snapshot. 'tag' is the tag assigned to the new snapshot. func (manager *BackupManager) Backup(top string, quickMode bool, threads int, tag string, - showStatistics bool, shadowCopy bool, shadowCopyTimeout int, enumOnly bool, listRateLimit int) bool { + showStatistics bool, shadowCopy bool, shadowCopyTimeout int, enumOnly bool) bool { var err error top, err = filepath.Abs(top) @@ -201,7 +201,7 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta LOG_INFO("BACKUP_INDEXING", "Indexing %s", top) localSnapshot, skippedDirectories, skippedFiles, err := CreateSnapshotFromDirectory(manager.snapshotID, shadowTop, - manager.nobackupFile, manager.filtersFile, listRateLimit) + manager.nobackupFile, manager.filtersFile) if err != nil { LOG_ERROR("SNAPSHOT_LIST", "Failed to list the directory %s: %v", top, err) return false @@ -784,7 +784,7 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu manager.SnapshotManager.DownloadSnapshotContents(remoteSnapshot, patterns, true) localSnapshot, _, _, err := CreateSnapshotFromDirectory(manager.snapshotID, top, manager.nobackupFile, - manager.filtersFile, 0) + manager.filtersFile) if err != nil { LOG_ERROR("SNAPSHOT_LIST", "Failed to list the repository: %v", err) return 0 diff --git a/src/duplicacy_backupmanager_test.go b/src/duplicacy_backupmanager_test.go index 653c69b..085c905 100644 --- a/src/duplicacy_backupmanager_test.go +++ b/src/duplicacy_backupmanager_test.go @@ -249,7 +249,7 @@ func TestBackupManager(t *testing.T) { backupManager.SetupSnapshotCache("default") SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy") - backupManager.Backup(testDir+"/repository1" /*quickMode=*/, true, threads, "first", false, false, 0, false, 0) + backupManager.Backup(testDir+"/repository1" /*quickMode=*/, true, threads, "first", false, false, 0, false) time.Sleep(time.Duration(delay) * time.Second) SetDuplicacyPreferencePath(testDir + "/repository2/.duplicacy") failedFiles := backupManager.Restore(testDir+"/repository2", threads /*inPlace=*/, false /*quickMode=*/, false, threads /*overwrite=*/, true, @@ -274,7 +274,7 @@ func TestBackupManager(t *testing.T) { modifyFile(testDir+"/repository1/dir1/file3", 0.3) SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy") - backupManager.Backup(testDir+"/repository1" /*quickMode=*/, true, threads, "second", false, false, 0, false, 0) + backupManager.Backup(testDir+"/repository1" /*quickMode=*/, true, threads, "second", false, false, 0, false) time.Sleep(time.Duration(delay) * time.Second) SetDuplicacyPreferencePath(testDir + "/repository2/.duplicacy") failedFiles = backupManager.Restore(testDir+"/repository2", 2 /*inPlace=*/, true /*quickMode=*/, true, threads /*overwrite=*/, true, @@ -295,7 +295,7 @@ func TestBackupManager(t *testing.T) { os.Mkdir(testDir+"/repository1/dir2/dir3", 0700) os.Mkdir(testDir+"/repository1/dir4", 0700) SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy") - backupManager.Backup(testDir+"/repository1" /*quickMode=*/, false, threads, "third", false, false, 0, false, 0) + backupManager.Backup(testDir+"/repository1" /*quickMode=*/, false, threads, "third", false, false, 0, false) time.Sleep(time.Duration(delay) * time.Second) // Create some directories and files under repository2 that will be deleted during restore @@ -360,7 +360,7 @@ func TestBackupManager(t *testing.T) { } backupManager.SnapshotManager.CheckSnapshots( /*snapshotID*/ "host1" /*revisions*/, []int{2, 3} /*tag*/, "", /*showStatistics*/ false /*showTabular*/, false /*checkFiles*/, false /*checkChunks*/, false /*searchFossils*/, false /*resurrect*/, false, 1 /*allowFailures*/, false) - backupManager.Backup(testDir+"/repository1" /*quickMode=*/, false, threads, "fourth", false, false, 0, false, 0) + backupManager.Backup(testDir+"/repository1" /*quickMode=*/, false, threads, "fourth", false, false, 0, false) backupManager.SnapshotManager.PruneSnapshots("host1", "host1" /*revisions*/, nil /*tags*/, nil /*retentions*/, nil, /*exhaustive*/ false /*exclusive=*/, true /*ignoredIDs*/, nil /*dryRun*/, false /*deleteOnly*/, false /*collectOnly*/, false, 1) numberOfSnapshots = backupManager.SnapshotManager.ListSnapshots( /*snapshotID*/ "host1" /*revisionsToList*/, nil /*tag*/, "" /*showFiles*/, false /*showChunks*/, false) @@ -525,7 +525,7 @@ func TestPersistRestore(t *testing.T) { unencBackupManager.SetupSnapshotCache("default") SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy") - unencBackupManager.Backup(testDir+"/repository1" /*quickMode=*/, true, threads, "first", false, false, 0, false, 0) + unencBackupManager.Backup(testDir+"/repository1" /*quickMode=*/, true, threads, "first", false, false, 0, false) time.Sleep(time.Duration(delay) * time.Second) @@ -535,7 +535,7 @@ func TestPersistRestore(t *testing.T) { encBackupManager.SetupSnapshotCache("default") SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy") - encBackupManager.Backup(testDir+"/repository1" /*quickMode=*/, true, threads, "first", false, false, 0, false, 0) + encBackupManager.Backup(testDir+"/repository1" /*quickMode=*/, true, threads, "first", false, false, 0, false) time.Sleep(time.Duration(delay) * time.Second) diff --git a/src/duplicacy_snapshot.go b/src/duplicacy_snapshot.go index 46d3f2f..45180f9 100644 --- a/src/duplicacy_snapshot.go +++ b/src/duplicacy_snapshot.go @@ -58,7 +58,7 @@ func CreateEmptySnapshot(id string) (snapshto *Snapshot) { // CreateSnapshotFromDirectory creates a snapshot from the local directory 'top'. Only 'Files' // will be constructed, while 'ChunkHashes' and 'ChunkLengths' can only be populated after uploading. -func CreateSnapshotFromDirectory(id string, top string, nobackupFile string, filtersFile string, listRateLimit int) (snapshot *Snapshot, skippedDirectories []string, +func CreateSnapshotFromDirectory(id string, top string, nobackupFile string, filtersFile string) (snapshot *Snapshot, skippedDirectories []string, skippedFiles []string, err error) { snapshot = &Snapshot{ @@ -84,22 +84,11 @@ func CreateSnapshotFromDirectory(id string, top string, nobackupFile string, fil attributeThreshold, _ = strconv.Atoi(attributeThresholdValue) } - startTime := time.Now() - for len(directories) > 0 { directory := directories[len(directories)-1] directories = directories[:len(directories)-1] snapshot.Files = append(snapshot.Files, directory) - - if listRateLimit > 0 { - maxFiles := int(time.Now().Sub(startTime).Seconds() * float64(listRateLimit)) - if len(snapshot.Files) > maxFiles { - delay := float64(len(snapshot.Files) - maxFiles) / float64(listRateLimit) - time.Sleep(time.Duration(delay * 1000.0) * time.Millisecond) - } - } - subdirectories, skipped, err := ListEntries(top, directory.Path, &snapshot.Files, patterns, nobackupFile, snapshot.discardAttributes) if err != nil { if directory.Path == "" { diff --git a/src/duplicacy_snapshotmanager.go b/src/duplicacy_snapshotmanager.go index a8742ac..da3b08e 100644 --- a/src/duplicacy_snapshotmanager.go +++ b/src/duplicacy_snapshotmanager.go @@ -1396,7 +1396,7 @@ func (manager *SnapshotManager) Diff(top string, snapshotID string, revisions [] if len(revisions) <= 1 { // Only scan the repository if filePath is not provided if len(filePath) == 0 { - rightSnapshot, _, _, err = CreateSnapshotFromDirectory(snapshotID, top, nobackupFile, filtersFile, 0) + rightSnapshot, _, _, err = CreateSnapshotFromDirectory(snapshotID, top, nobackupFile, filtersFile) if err != nil { LOG_ERROR("SNAPSHOT_LIST", "Failed to list the directory %s: %v", top, err) return false