From b41e8a24a998690836075b6a78b9446c3b4d180b Mon Sep 17 00:00:00 2001 From: Jeff Thompson Date: Sat, 2 Sep 2017 22:36:26 -0500 Subject: [PATCH] Skip chunks to copy if already on destination for issue #134 --- GUIDE.md | 4 +- duplicacy/duplicacy_main.go | 14 ++-- src/duplicacy_backupmanager.go | 131 ++++++++++++++++++++++++++++----- src/duplicacy_storage.go | 11 ++- src/duplicacy_utils.go | 19 +++-- 5 files changed, 145 insertions(+), 34 deletions(-) diff --git a/GUIDE.md b/GUIDE.md index 94029a8..e33b05c 100644 --- a/GUIDE.md +++ b/GUIDE.md @@ -440,10 +440,10 @@ For the *restore* command, the include/exclude patterns are specified as the com Duplicacy will attempt to retrieve in three ways the storage password and the storage-specific access tokens/keys. * If a secret vault service is available, Duplicacy will store passwords/keys entered by the user in such a secret vault and later retrieve them when needed. On Mac OS X it is Keychain, and on Linux it is gnome-keyring. On Windows the passwords/keys are encrypted and decrypted by the Data Protection API, and encrypted passwords/keys are stored in the file *.duplicacy/keyring*. However, if the -no-save-password option is specified for the storage, then Duplicacy will not save passwords this way. -* If an environment variable for a password is provided, Duplicacy will always take it. The table below shows the name of the environment variable for each kind of password. Note that if the storage is not the default one, the storage name will be included in the name of the environment variable. +* If an environment variable for a password is provided, Duplicacy will always take it. The table below shows the name of the environment variable for each kind of password. Note that if the storage is not the default one, the storage name will be included in the name of the environment variable (in uppercase). For example, if your storage name is b2, then the environment variable should be named DUPLICACY_B2_PASSWORD. * If a matching key and its value are saved to the preference file (.duplicacy/preferences) by the *set* command, the value will be used as the password. The last column in the table below lists the name of the preference key for each type of password. -| password type | environment variable (default storage) | environment variable (non-default storage) | key in preferences | +| password type | environment variable (default storage) | environment variable (non-default storage in uppercase) | key in preferences | |:----------------:|:----------------:|:----------------:|:----------------:| | storage password | DUPLICACY_PASSWORD | DUPLICACY_<STORAGENAME>_PASSWORD | password | | sftp password | DUPLICACY_SSH_PASSWORD | DUPLICACY_<STORAGENAME>_SSH_PASSWORD | ssh_password | diff --git a/duplicacy/duplicacy_main.go b/duplicacy/duplicacy_main.go index b00686b..3174cff 100644 --- a/duplicacy/duplicacy_main.go +++ b/duplicacy/duplicacy_main.go @@ -1020,12 +1020,17 @@ func copySnapshots(context *cli.Context) { os.Exit(ArgumentExitCode) } + threads := context.Int("threads") + if threads < 1 { + threads = 1 + } + repository, source := getRepositoryPreference(context, context.String("from")) runScript(context, source.Name, "pre") duplicacy.LOG_INFO("STORAGE_SET", "Source storage set to %s", source.StorageURL) - sourceStorage := duplicacy.CreateStorage(*source, false, 1) + sourceStorage := duplicacy.CreateStorage(*source, false, threads) if sourceStorage == nil { return } @@ -1055,7 +1060,7 @@ func copySnapshots(context *cli.Context) { duplicacy.LOG_INFO("STORAGE_SET", "Destination storage set to %s", destination.StorageURL) - destinationStorage := duplicacy.CreateStorage(*destination, false, 1) + destinationStorage := duplicacy.CreateStorage(*destination, false, threads) if destinationStorage == nil { return } @@ -1080,11 +1085,6 @@ func copySnapshots(context *cli.Context) { snapshotID = context.String("id") } - threads := context.Int("threads") - if threads < 1 { - threads = 1 - } - sourceManager.CopySnapshots(destinationManager, snapshotID, revisions, threads) runScript(context, source.Name, "post") } diff --git a/src/duplicacy_backupmanager.go b/src/duplicacy_backupmanager.go index 9090653..e23f8bc 100644 --- a/src/duplicacy_backupmanager.go +++ b/src/duplicacy_backupmanager.go @@ -1484,14 +1484,27 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho return false } - revisionMap := make(map[int]bool) + if snapshotID == "" && len(revisionsToBeCopied) > 0 { + LOG_ERROR("SNAPSHOT_ERROR", "You must specify the snapshot id when one or more revisions are specified.") + return false + } + + revisionMap := make(map[string]map[int]bool) + + _, found := revisionMap[snapshotID] + if !found { + revisionMap[snapshotID] = make(map[int]bool) + } + for _, revision := range revisionsToBeCopied { - revisionMap[revision] = true + revisionMap[snapshotID][revision] = true } var snapshots [] *Snapshot + var otherSnapshots [] *Snapshot var snapshotIDs [] string var err error + if snapshotID == "" { snapshotIDs, err = manager.SnapshotManager.ListSnapshotIDs() if err != nil { @@ -1503,6 +1516,10 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho } for _, id := range snapshotIDs { + _, found := revisionMap[id] + if !found { + revisionMap[id] = make(map[int]bool) + } revisions, err := manager.SnapshotManager.ListSnapshotRevisions(id) if err != nil { LOG_ERROR("SNAPSHOT_LIST", "Failed to list all revisions for snapshot %s: %v", id, err) @@ -1511,9 +1528,14 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho for _, revision := range revisions { if len(revisionsToBeCopied) > 0 { - if _, found := revisionMap[revision]; !found { + if _, found := revisionMap[id][revision]; found { + revisionMap[id][revision] = true + } else { + revisionMap[id][revision] = false continue } + } else { + revisionMap[id][revision] = true } snapshotPath := fmt.Sprintf("snapshots/%s/%d", id, revision) @@ -1525,21 +1547,44 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho } if exist { - LOG_INFO("SNAPSHOT_EXIST", "Snapshot %s at revision %d already exists in the destination storage", + LOG_INFO("SNAPSHOT_EXIST", "Snapshot %s at revision %d already exists at the destination storage", id, revision) + revisionMap[id][revision] = false continue } snapshot := manager.SnapshotManager.DownloadSnapshot(id, revision) snapshots = append(snapshots, snapshot) } + + otherRevisions, err := otherManager.SnapshotManager.ListSnapshotRevisions(id) + if err != nil { + LOG_ERROR("SNAPSHOT_LIST", "Failed to list all revisions at the destination for snapshot %s: %v", id, err) + return false + } + + for _, otherRevision := range otherRevisions { + otherSnapshot := otherManager.SnapshotManager.DownloadSnapshot(id, otherRevision) + otherSnapshots = append(otherSnapshots, otherSnapshot) + } + + } + + if len(snapshots) == 0 { + LOG_INFO("SNAPSHOT_COPY", "Nothing to copy, all snapshot revisions exist at the destination.") + return true } chunks := make(map[string]bool) for _, snapshot := range snapshots { + if revisionMap[snapshot.ID][snapshot.Revision] == false { + continue + } + LOG_TRACE("SNAPSHOT_COPY", "Copying snapshot %s at revision %d", snapshot.ID, snapshot.Revision) + for _, chunkHash := range snapshot.FileSequence { chunks[chunkHash] = true } @@ -1565,42 +1610,90 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho } } + for _, otherSnapshot := range otherSnapshots { + + for _, chunkHash := range otherSnapshot.FileSequence { + if _, found := chunks[chunkHash]; found { + chunks[chunkHash] = false + } + } + + for _, chunkHash := range otherSnapshot.ChunkSequence { + if _, found := chunks[chunkHash]; found { + chunks[chunkHash] = false + } + } + + for _, chunkHash := range otherSnapshot.LengthSequence { + if _, found := chunks[chunkHash]; found { + chunks[chunkHash] = false + } + } + + description := otherManager.SnapshotManager.DownloadSequence(otherSnapshot.ChunkSequence) + err := otherSnapshot.LoadChunks(description) + if err != nil { + LOG_ERROR("SNAPSHOT_CHUNK", "Failed to load chunks for destination snapshot %s at revision %d: %v", + otherSnapshot.ID, otherSnapshot.Revision, err) + return false + } + + for _, chunkHash := range otherSnapshot.ChunkHashes { + if _, found := chunks[chunkHash]; found { + chunks[chunkHash] = false + } + } + } + chunkDownloader := CreateChunkDownloader(manager.config, manager.storage, nil, false, threads) chunkUploader := CreateChunkUploader(otherManager.config, otherManager.storage, nil, threads, func(chunk *Chunk, chunkIndex int, skipped bool, chunkSize int, uploadSize int) { if skipped { - LOG_INFO("SNAPSHOT_COPY", "Chunk %s (%d/%d) exists in the destination", chunk.GetID(), chunkIndex, len(chunks)) + LOG_INFO("SNAPSHOT_COPY", "Chunk %s (%d/%d) exists at the destination", chunk.GetID(), chunkIndex, len(chunks)) } else { - LOG_INFO("SNAPSHOT_COPY", "Copied chunk %s (%d/%d)", chunk.GetID(), chunkIndex, len(chunks)) + LOG_INFO("SNAPSHOT_COPY", "Chunk %s (%d/%d) copied to the destination", chunk.GetID(), chunkIndex, len(chunks)) } otherManager.config.PutChunk(chunk) }) + chunkUploader.Start() + totalCopied := 0 + totalSkipped := 0 chunkIndex := 0 - for chunkHash, _ := range chunks { + + for chunkHash, needsCopy := range chunks { chunkIndex++ chunkID := manager.config.GetChunkIDFromHash(chunkHash) - newChunkID := otherManager.config.GetChunkIDFromHash(chunkHash) - - LOG_DEBUG("SNAPSHOT_COPY", "Copying chunk %s to %s", chunkID, newChunkID) - - i := chunkDownloader.AddChunk(chunkHash) - chunk := chunkDownloader.WaitForChunk(i) - newChunk := otherManager.config.GetChunk() - newChunk.Reset(true) - newChunk.Write(chunk.GetBytes()) - chunkUploader.StartChunk(newChunk, chunkIndex) + if needsCopy { + newChunkID := otherManager.config.GetChunkIDFromHash(chunkHash) + LOG_DEBUG("SNAPSHOT_COPY", "Copying chunk %s to %s", chunkID, newChunkID) + i := chunkDownloader.AddChunk(chunkHash) + chunk := chunkDownloader.WaitForChunk(i) + newChunk := otherManager.config.GetChunk() + newChunk.Reset(true) + newChunk.Write(chunk.GetBytes()) + chunkUploader.StartChunk(newChunk, chunkIndex) + totalCopied++ + } else { + LOG_INFO("SNAPSHOT_COPY", "Chunk %s (%d/%d) exists at the destination.", chunkID, chunkIndex, len(chunks)) + totalSkipped++ + } } chunkDownloader.Stop() chunkUploader.Stop() + LOG_INFO("SNAPSHOT_COPY", "Total chunks copied = %d, skipped = %d.", totalCopied, totalSkipped) + for _, snapshot := range snapshots { - otherManager.storage.CreateDirectory(0, fmt.Sprintf("snapshots/%s", manager.snapshotID)) + if revisionMap[snapshot.ID][snapshot.Revision] == false { + continue + } + otherManager.storage.CreateDirectory(0, fmt.Sprintf("snapshots/%s", snapshot.ID)) description, _ := snapshot.MarshalJSON() - path := fmt.Sprintf("snapshots/%s/%d", manager.snapshotID, snapshot.Revision) + path := fmt.Sprintf("snapshots/%s/%d", snapshot.ID, snapshot.Revision) otherManager.SnapshotManager.UploadFile(path, path, description) LOG_INFO("SNAPSHOT_COPY", "Copied snapshot %s at revision %d", snapshot.ID, snapshot.Revision) } diff --git a/src/duplicacy_storage.go b/src/duplicacy_storage.go index 819462b..d0e3638 100644 --- a/src/duplicacy_storage.go +++ b/src/duplicacy_storage.go @@ -199,6 +199,7 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor if username != "" { username = username[:len(username) - 1] } + keyFile := GetPasswordFromPreference(preference, "ssh_key_file") password := "" passwordCallback := func() (string, error) { @@ -219,7 +220,6 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor } } - keyFile := "" publicKeysCallback := func() ([]ssh.Signer, error) { LOG_DEBUG("SSH_PUBLICKEY", "Attempting public key authentication") @@ -273,10 +273,19 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor } authMethods := [] ssh.AuthMethod { + } + passwordAuthMethods := [] ssh.AuthMethod { ssh.PasswordCallback(passwordCallback), ssh.KeyboardInteractive(keyboardInteractive), + } + keyFileAuthMethods := [] ssh.AuthMethod { ssh.PublicKeysCallback(publicKeysCallback), } + if keyFile!="" { + authMethods = append(keyFileAuthMethods,passwordAuthMethods...) + } else { + authMethods = append(passwordAuthMethods,keyFileAuthMethods...) + } if RunInBackground { diff --git a/src/duplicacy_utils.go b/src/duplicacy_utils.go index feda7ac..757a756 100644 --- a/src/duplicacy_utils.go +++ b/src/duplicacy_utils.go @@ -118,10 +118,8 @@ func GenerateKeyFromPassword(password string) []byte { return pbkdf2.Key([]byte(password), DEFAULT_KEY, 16384, 32, sha256.New) } -// GetPassword attempts to get the password from KeyChain/KeyRing, environment variables, or keyboard input. -func GetPassword(preference Preference, passwordType string, prompt string, - showPassword bool, resetPassword bool) (string) { - +// Get password from preference, env, but don't start any keyring request +func GetPasswordFromPreference(preference Preference, passwordType string) (string) { passwordID := passwordType if preference.Name != "default" { passwordID = preference.Name + "_" + passwordID @@ -139,6 +137,17 @@ func GetPassword(preference Preference, passwordType string, prompt string, LOG_DEBUG("PASSWORD_KEYCHAIN", "Reading %s from preferences", passwordID) return preference.Keys[passwordID] } + return "" +} + +// GetPassword attempts to get the password from KeyChain/KeyRing, environment variables, or keyboard input. +func GetPassword(preference Preference, passwordType string, prompt string, + showPassword bool, resetPassword bool) (string) { + passwordID := passwordType + password := GetPasswordFromPreference(preference,passwordType) + if password != "" { + return password + } if resetPassword && !RunInBackground { keyringSet(passwordID, "") @@ -155,7 +164,7 @@ func GetPassword(preference Preference, passwordType string, prompt string, } - password := "" + password = "" fmt.Printf("%s", prompt) if showPassword { scanner := bufio.NewScanner(os.Stdin)