From d0771be2dd12252724b264b64208a824b953c7d9 Mon Sep 17 00:00:00 2001 From: Mark Feit Date: Tue, 16 Jan 2018 22:06:11 -0500 Subject: [PATCH 1/7] Added semi-dedicated Wasabi storage module. #322 --- src/duplicacy_storage.go | 32 ++++++ src/duplicacy_storage_test.go | 6 +- src/duplicacy_wasabistorage.go | 189 +++++++++++++++++++++++++++++++++ 3 files changed, 225 insertions(+), 2 deletions(-) create mode 100644 src/duplicacy_wasabistorage.go diff --git a/src/duplicacy_storage.go b/src/duplicacy_storage.go index 6aa64a4..941ed65 100644 --- a/src/duplicacy_storage.go +++ b/src/duplicacy_storage.go @@ -461,6 +461,38 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor SavePassword(preference, "s3_secret", secretKey) return storage + + } else if matched[1] == "wasabi" { + + region := matched[2] + endpoint := matched[3] + bucket := matched[5] + + key := GetPassword(preference, "wasabi_key", + "Enter Wasabi key:", true, resetPassword) + secret := GetPassword(preference, "wasabi_secret", + "Enter Wasabi secret:", true, resetPassword) + + storageDir := "" + if strings.Contains(bucket, "/") { + firstSlash := strings.Index(bucket, "/") + storageDir = bucket[firstSlash+1:] + bucket = bucket[:firstSlash] + } + + storage, err := CreateWasabiStorage(region, endpoint, + bucket, storageDir, key, secret, threads) + + if err != nil { + LOG_ERROR("STORAGE_CREATE", "Failed to load the S3 storage at %s: %v", storageURL, err) + return nil + } + + SavePassword(preference, "key", key) + SavePassword(preference, "secret", secret) + + return storage + } else if matched[1] == "dropbox" { storageDir := matched[3] + matched[5] token := GetPassword(preference, "dropbox_token", "Enter Dropbox access token:", true, resetPassword) diff --git a/src/duplicacy_storage_test.go b/src/duplicacy_storage_test.go index d804360..d3955cb 100644 --- a/src/duplicacy_storage_test.go +++ b/src/duplicacy_storage_test.go @@ -78,10 +78,12 @@ func loadStorage(localStoragePath string, threads int) (Storage, error) { storage, err := CreateSFTPStorageWithPassword(config["server"], port, config["username"], config["directory"], 2, config["password"], threads) storage.SetDefaultNestingLevels([]int{2, 3}, 2) return storage, err - } else if testStorageName == "s3" || testStorageName == "wasabi" { + } else if testStorageName == "s3" { storage, err := CreateS3Storage(config["region"], config["endpoint"], config["bucket"], config["directory"], config["access_key"], config["secret_key"], threads, true, false) storage.SetDefaultNestingLevels([]int{2, 3}, 2) - return storage, err + } else if testStorageName == "wasabi" { + storage, err := CreateWasabiStorage(config["region"], config["endpoint"], config["bucket"], config["directory"], config["access_key"], config["secret_key"], threads) + storage.SetDefaultNestingLevels([]int{2, 3}, 2) } else if testStorageName == "s3c" { storage, err := CreateS3CStorage(config["region"], config["endpoint"], config["bucket"], config["directory"], config["access_key"], config["secret_key"], threads) storage.SetDefaultNestingLevels([]int{2, 3}, 2) diff --git a/src/duplicacy_wasabistorage.go b/src/duplicacy_wasabistorage.go new file mode 100644 index 0000000..9c6ce7c --- /dev/null +++ b/src/duplicacy_wasabistorage.go @@ -0,0 +1,189 @@ +// +// Storage module for Wasabi (https://www.wasabi.com) +// + +// Wasabi is nominally compatible with AWS S3, but the copy-and-delete +// method used for renaming objects creates additional expense under +// Wasabi's billing system. This module is a pass-through to the +// existing S3 module for everything other than that one operation. +// +// This module copyright 2017 Mark Feit (https://github.com/markfeit) +// and may be distributed under the same terms as Duplicacy. + +package duplicacy + +import ( + "crypto/hmac" + "crypto/sha1" + "encoding/base64" + "errors" + "fmt" + "net/http" + "time" +) + +type WasabiStorage struct { + StorageBase + + s3 *S3Storage + region string + endpoint string + bucket string + storageDir string + key string + secret string +} + +// See the Storage interface in duplicacy_storage.go for function +// descriptions. + +func CreateWasabiStorage( + regionName string, endpoint string, + bucketName string, storageDir string, + accessKey string, secretKey string, + threads int, +) (storage *WasabiStorage, err error) { + + s3storage, error := CreateS3Storage(regionName, endpoint, bucketName, + storageDir, accessKey, secretKey, threads, + true, // isSSLSupported + false, // isMinioCompatible + ) + + if err != nil { + return nil, error + } + + wasabi := &WasabiStorage{ + + // Pass-through to existing S3 module + s3: s3storage, + + // Local copies required for renaming + region: regionName, + endpoint: endpoint, + bucket: bucketName, + storageDir: storageDir, + key: accessKey, + secret: secretKey, + } + + wasabi.DerivedStorage = wasabi + wasabi.SetDefaultNestingLevels([]int{0}, 0) + + return wasabi, nil +} + +func (storage *WasabiStorage) ListFiles( + threadIndex int, dir string, +) (files []string, sizes []int64, err error) { + return storage.s3.ListFiles(threadIndex, dir) +} + +func (storage *WasabiStorage) DeleteFile( + threadIndex int, filePath string, +) (err error) { + return storage.s3.DeleteFile(threadIndex, filePath) + +} + +// This is a lightweight implementation of a call to Wasabi for a +// rename. It's designed to get the job done with as few dependencies +// on other packages as possible rather than being somethng +// general-purpose and reusable. +func (storage *WasabiStorage) MoveFile( + threadIndex int, from string, to string, +) (err error) { + + // The from path includes the bucket + from_path := fmt.Sprintf("/%s/%s/%s", storage.bucket, storage.storageDir, from) + + // The to path does not. + // to_path := fmt.Sprintf("%s/%s", storage.storageDir, from) + + // We get the region with an @ on the end, so don't add another. + object := fmt.Sprintf("https://%s%s%s", + storage.region, storage.endpoint, from_path) + + // The object's new name is relative to the top of the bucket. + new_name := fmt.Sprintf("%s/%s", storage.storageDir, to) + + timestamp := time.Now().Format(time.RFC1123Z) + + signing_string := fmt.Sprintf("MOVE\n\n\n%s\n%s", timestamp, from_path) + + signer := hmac.New(sha1.New, []byte(storage.secret)) + signer.Write([]byte(signing_string)) + + signature := base64.StdEncoding.EncodeToString(signer.Sum(nil)) + + authorization := fmt.Sprintf("AWS %s:%s", storage.key, signature) + + request, error := http.NewRequest("MOVE", object, nil) + if error != nil { + return error + } + request.Header.Add("Authorization", authorization) + request.Header.Add("Date", timestamp) + request.Header.Add("Destination", new_name) + request.Header.Add("Host", storage.endpoint) + request.Header.Add("Overwrite", "true") + + client := &http.Client{} + + response, error := client.Do(request) + if error != nil { + return error + } + + if response.StatusCode != 200 { + return errors.New(response.Status) + } + + return nil +} + +func (storage *WasabiStorage) CreateDirectory( + threadIndex int, dir string, +) (err error) { + return storage.s3.CreateDirectory(threadIndex, dir) +} + +func (storage *WasabiStorage) GetFileInfo( + threadIndex int, filePath string, +) (exist bool, isDir bool, size int64, err error) { + return storage.s3.GetFileInfo(threadIndex, filePath) +} + +func (storage *WasabiStorage) DownloadFile( + threadIndex int, filePath string, chunk *Chunk, +) (err error) { + return storage.s3.DownloadFile(threadIndex, filePath, chunk) +} + +func (storage *WasabiStorage) UploadFile( + threadIndex int, filePath string, content []byte, +) (err error) { + return storage.s3.UploadFile(threadIndex, filePath, content) +} + +func (storage *WasabiStorage) IsCacheNeeded() bool { + return storage.s3.IsCacheNeeded() +} + +func (storage *WasabiStorage) IsMoveFileImplemented() bool { + // This is implemented locally since S3 does a copy and delete + return true +} + +func (storage *WasabiStorage) IsStrongConsistent() bool { + // Wasabi has it, S3 doesn't. + return true +} + +func (storage *WasabiStorage) IsFastListing() bool { + return true +} + +func (storage *WasabiStorage) EnableTestMode() { +} From 30f753e49985e4870aea005228b2e65c48a4d574 Mon Sep 17 00:00:00 2001 From: Mark Feit Date: Tue, 16 Jan 2018 22:19:41 -0500 Subject: [PATCH 2/7] Cosmetic and key name fixes. #322 --- src/duplicacy_storage.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/duplicacy_storage.go b/src/duplicacy_storage.go index 941ed65..df13c90 100644 --- a/src/duplicacy_storage.go +++ b/src/duplicacy_storage.go @@ -484,12 +484,12 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor bucket, storageDir, key, secret, threads) if err != nil { - LOG_ERROR("STORAGE_CREATE", "Failed to load the S3 storage at %s: %v", storageURL, err) + LOG_ERROR("STORAGE_CREATE", "Failed to load the Wasabi storage at %s: %v", storageURL, err) return nil } - SavePassword(preference, "key", key) - SavePassword(preference, "secret", secret) + SavePassword(preference, "wasabi_key", key) + SavePassword(preference, "wasabi_secret", secret) return storage From 8aaca37a2bed21599d99cefcfcc076eec8cdc3b2 Mon Sep 17 00:00:00 2001 From: Mark Feit Date: Thu, 18 Jan 2018 08:30:00 -0500 Subject: [PATCH 3/7] Added note about Wasabi dependency. --- src/duplicacy_s3storage.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/duplicacy_s3storage.go b/src/duplicacy_s3storage.go index 01d5e9f..e29af95 100644 --- a/src/duplicacy_s3storage.go +++ b/src/duplicacy_s3storage.go @@ -2,6 +2,11 @@ // Free for personal use and commercial trial // Commercial use requires per-user licenses available from https://duplicacy.com +// NOTE: The code in the Wasabi storage module relies on all functions +// in this one except MoveFile(), IsMoveFileImplemented() and +// IsStrongConsistent(). Changes to the API here will need to be +// reflected there. + package duplicacy import ( From b52d6b3f7f65d8bed577b0660e1a4ac701cfbcce Mon Sep 17 00:00:00 2001 From: Mark Feit Date: Thu, 18 Jan 2018 08:34:33 -0500 Subject: [PATCH 4/7] Incorporated PR feedback; call S3 for IsFastListing() #322 --- src/duplicacy_wasabistorage.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/duplicacy_wasabistorage.go b/src/duplicacy_wasabistorage.go index 9c6ce7c..3045ef2 100644 --- a/src/duplicacy_wasabistorage.go +++ b/src/duplicacy_wasabistorage.go @@ -32,6 +32,7 @@ type WasabiStorage struct { storageDir string key string secret string + client *http.Client } // See the Storage interface in duplicacy_storage.go for function @@ -66,6 +67,7 @@ func CreateWasabiStorage( storageDir: storageDir, key: accessKey, secret: secretKey, + client: &http.Client{}, } wasabi.DerivedStorage = wasabi @@ -129,12 +131,11 @@ func (storage *WasabiStorage) MoveFile( request.Header.Add("Host", storage.endpoint) request.Header.Add("Overwrite", "true") - client := &http.Client{} - - response, error := client.Do(request) + response, error := storage.client.Do(request) if error != nil { return error } + defer response.Body.Close() if response.StatusCode != 200 { return errors.New(response.Status) @@ -182,7 +183,7 @@ func (storage *WasabiStorage) IsStrongConsistent() bool { } func (storage *WasabiStorage) IsFastListing() bool { - return true + return storage.s3.IsFastListing() } func (storage *WasabiStorage) EnableTestMode() { From cc6e96527e24661e42569fab127ebdbcaabb6937 Mon Sep 17 00:00:00 2001 From: Mark Feit Date: Sat, 10 Feb 2018 10:15:40 -0500 Subject: [PATCH 5/7] Add/rearrange returns to keep the compiler from complaining. --- src/duplicacy_storage_test.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/duplicacy_storage_test.go b/src/duplicacy_storage_test.go index d3955cb..5fe29f3 100644 --- a/src/duplicacy_storage_test.go +++ b/src/duplicacy_storage_test.go @@ -80,9 +80,11 @@ func loadStorage(localStoragePath string, threads int) (Storage, error) { return storage, err } else if testStorageName == "s3" { storage, err := CreateS3Storage(config["region"], config["endpoint"], config["bucket"], config["directory"], config["access_key"], config["secret_key"], threads, true, false) + return storage, err storage.SetDefaultNestingLevels([]int{2, 3}, 2) } else if testStorageName == "wasabi" { storage, err := CreateWasabiStorage(config["region"], config["endpoint"], config["bucket"], config["directory"], config["access_key"], config["secret_key"], threads) + return storage, err storage.SetDefaultNestingLevels([]int{2, 3}, 2) } else if testStorageName == "s3c" { storage, err := CreateS3CStorage(config["region"], config["endpoint"], config["bucket"], config["directory"], config["access_key"], config["secret_key"], threads) @@ -140,9 +142,9 @@ func loadStorage(localStoragePath string, threads int) (Storage, error) { storage, err := CreateHubicStorage(config["token_file"], config["storage_path"], threads) storage.SetDefaultNestingLevels([]int{2, 3}, 2) return storage, err - } else { - return nil, fmt.Errorf("Invalid storage named: %s", testStorageName) } + + return nil, fmt.Errorf("Invalid storage named: %s", testStorageName) } func cleanStorage(storage Storage) { @@ -585,12 +587,11 @@ func TestCleanStorage(t *testing.T) { storage.DeleteFile(0, "config") LOG_INFO("DELETE_FILE", "Deleted config") - files, _, err := storage.ListFiles(0, "chunks/") for _, file := range files { - if len(file) > 0 && file[len(file)-1] != '/' { - LOG_DEBUG("FILE_EXIST", "File %s exists after deletion", file) - } + if len(file) > 0 && file[len(file)-1] != '/' { + LOG_DEBUG("FILE_EXIST", "File %s exists after deletion", file) + } } } From 9d632c04346ee1c8861f72311665b86e725d78a6 Mon Sep 17 00:00:00 2001 From: Mark Feit Date: Sat, 10 Feb 2018 10:16:04 -0500 Subject: [PATCH 6/7] Handle application/testing region string inconsistency. --- src/duplicacy_wasabistorage.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/duplicacy_wasabistorage.go b/src/duplicacy_wasabistorage.go index 3045ef2..e0d432f 100644 --- a/src/duplicacy_wasabistorage.go +++ b/src/duplicacy_wasabistorage.go @@ -19,6 +19,7 @@ import ( "errors" "fmt" "net/http" + "strings" "time" ) @@ -45,6 +46,15 @@ func CreateWasabiStorage( threads int, ) (storage *WasabiStorage, err error) { + // The application code that parses the URL from the + // configuration leaves an @ on the end, while the code that + // does the testing doesn't. Force the incoming value to + // behave like the application. + + if regionName != "" && !strings.HasSuffix(regionName, "@") { + regionName = regionName + "@" + } + s3storage, error := CreateS3Storage(regionName, endpoint, bucketName, storageDir, accessKey, secretKey, threads, true, // isSSLSupported @@ -100,9 +110,6 @@ func (storage *WasabiStorage) MoveFile( // The from path includes the bucket from_path := fmt.Sprintf("/%s/%s/%s", storage.bucket, storage.storageDir, from) - // The to path does not. - // to_path := fmt.Sprintf("%s/%s", storage.storageDir, from) - // We get the region with an @ on the end, so don't add another. object := fmt.Sprintf("https://%s%s%s", storage.region, storage.endpoint, from_path) From 8fdb399e1b7d6a2118224613c1f688b2c7b8ed03 Mon Sep 17 00:00:00 2001 From: Mark Feit Date: Sun, 11 Feb 2018 07:51:34 -0500 Subject: [PATCH 7/7] Correct handling of @ in region to be consistent with everythng else. --- src/duplicacy_storage.go | 4 ++++ src/duplicacy_wasabistorage.go | 13 +------------ 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/src/duplicacy_storage.go b/src/duplicacy_storage.go index df13c90..db61c7b 100644 --- a/src/duplicacy_storage.go +++ b/src/duplicacy_storage.go @@ -468,6 +468,10 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor endpoint := matched[3] bucket := matched[5] + if region != "" { + region = region[:len(region)-1] + } + key := GetPassword(preference, "wasabi_key", "Enter Wasabi key:", true, resetPassword) secret := GetPassword(preference, "wasabi_secret", diff --git a/src/duplicacy_wasabistorage.go b/src/duplicacy_wasabistorage.go index e0d432f..b4011c4 100644 --- a/src/duplicacy_wasabistorage.go +++ b/src/duplicacy_wasabistorage.go @@ -19,7 +19,6 @@ import ( "errors" "fmt" "net/http" - "strings" "time" ) @@ -46,15 +45,6 @@ func CreateWasabiStorage( threads int, ) (storage *WasabiStorage, err error) { - // The application code that parses the URL from the - // configuration leaves an @ on the end, while the code that - // does the testing doesn't. Force the incoming value to - // behave like the application. - - if regionName != "" && !strings.HasSuffix(regionName, "@") { - regionName = regionName + "@" - } - s3storage, error := CreateS3Storage(regionName, endpoint, bucketName, storageDir, accessKey, secretKey, threads, true, // isSSLSupported @@ -110,8 +100,7 @@ func (storage *WasabiStorage) MoveFile( // The from path includes the bucket from_path := fmt.Sprintf("/%s/%s/%s", storage.bucket, storage.storageDir, from) - // We get the region with an @ on the end, so don't add another. - object := fmt.Sprintf("https://%s%s%s", + object := fmt.Sprintf("https://%s@%s%s", storage.region, storage.endpoint, from_path) // The object's new name is relative to the top of the bucket.