diff --git a/src/duplicacy_s3storage.go b/src/duplicacy_s3storage.go index 01d5e9f..e29af95 100644 --- a/src/duplicacy_s3storage.go +++ b/src/duplicacy_s3storage.go @@ -2,6 +2,11 @@ // Free for personal use and commercial trial // Commercial use requires per-user licenses available from https://duplicacy.com +// NOTE: The code in the Wasabi storage module relies on all functions +// in this one except MoveFile(), IsMoveFileImplemented() and +// IsStrongConsistent(). Changes to the API here will need to be +// reflected there. + package duplicacy import ( diff --git a/src/duplicacy_storage.go b/src/duplicacy_storage.go index d0ac913..f72dbe9 100644 --- a/src/duplicacy_storage.go +++ b/src/duplicacy_storage.go @@ -461,6 +461,42 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor SavePassword(preference, "s3_secret", secretKey) return storage + + } else if matched[1] == "wasabi" { + + region := matched[2] + endpoint := matched[3] + bucket := matched[5] + + if region != "" { + region = region[:len(region)-1] + } + + key := GetPassword(preference, "wasabi_key", + "Enter Wasabi key:", true, resetPassword) + secret := GetPassword(preference, "wasabi_secret", + "Enter Wasabi secret:", true, resetPassword) + + storageDir := "" + if strings.Contains(bucket, "/") { + firstSlash := strings.Index(bucket, "/") + storageDir = bucket[firstSlash+1:] + bucket = bucket[:firstSlash] + } + + storage, err := CreateWasabiStorage(region, endpoint, + bucket, storageDir, key, secret, threads) + + if err != nil { + LOG_ERROR("STORAGE_CREATE", "Failed to load the Wasabi storage at %s: %v", storageURL, err) + return nil + } + + SavePassword(preference, "wasabi_key", key) + SavePassword(preference, "wasabi_secret", secret) + + return storage + } else if matched[1] == "dropbox" { storageDir := matched[3] + matched[5] token := GetPassword(preference, "dropbox_token", "Enter Dropbox access token:", true, resetPassword) diff --git a/src/duplicacy_storage_test.go b/src/duplicacy_storage_test.go index 462fe04..36075cc 100644 --- a/src/duplicacy_storage_test.go +++ b/src/duplicacy_storage_test.go @@ -78,10 +78,14 @@ func loadStorage(localStoragePath string, threads int) (Storage, error) { storage, err := CreateSFTPStorageWithPassword(config["server"], port, config["username"], config["directory"], 2, config["password"], threads) storage.SetDefaultNestingLevels([]int{2, 3}, 2) return storage, err - } else if testStorageName == "s3" || testStorageName == "wasabi" { + } else if testStorageName == "s3" { storage, err := CreateS3Storage(config["region"], config["endpoint"], config["bucket"], config["directory"], config["access_key"], config["secret_key"], threads, true, false) - storage.SetDefaultNestingLevels([]int{2, 3}, 2) return storage, err + storage.SetDefaultNestingLevels([]int{2, 3}, 2) + } else if testStorageName == "wasabi" { + storage, err := CreateWasabiStorage(config["region"], config["endpoint"], config["bucket"], config["directory"], config["access_key"], config["secret_key"], threads) + return storage, err + storage.SetDefaultNestingLevels([]int{2, 3}, 2) } else if testStorageName == "s3c" { storage, err := CreateS3CStorage(config["region"], config["endpoint"], config["bucket"], config["directory"], config["access_key"], config["secret_key"], threads) storage.SetDefaultNestingLevels([]int{2, 3}, 2) @@ -145,6 +149,8 @@ func loadStorage(localStoragePath string, threads int) (Storage, error) { } else { return nil, fmt.Errorf("Invalid storage named: %s", testStorageName) } + + return nil, fmt.Errorf("Invalid storage named: %s", testStorageName) } func cleanStorage(storage Storage) { @@ -587,12 +593,11 @@ func TestCleanStorage(t *testing.T) { storage.DeleteFile(0, "config") LOG_INFO("DELETE_FILE", "Deleted config") - files, _, err := storage.ListFiles(0, "chunks/") for _, file := range files { - if len(file) > 0 && file[len(file)-1] != '/' { - LOG_DEBUG("FILE_EXIST", "File %s exists after deletion", file) - } + if len(file) > 0 && file[len(file)-1] != '/' { + LOG_DEBUG("FILE_EXIST", "File %s exists after deletion", file) + } } } diff --git a/src/duplicacy_wasabistorage.go b/src/duplicacy_wasabistorage.go new file mode 100644 index 0000000..b4011c4 --- /dev/null +++ b/src/duplicacy_wasabistorage.go @@ -0,0 +1,186 @@ +// +// Storage module for Wasabi (https://www.wasabi.com) +// + +// Wasabi is nominally compatible with AWS S3, but the copy-and-delete +// method used for renaming objects creates additional expense under +// Wasabi's billing system. This module is a pass-through to the +// existing S3 module for everything other than that one operation. +// +// This module copyright 2017 Mark Feit (https://github.com/markfeit) +// and may be distributed under the same terms as Duplicacy. + +package duplicacy + +import ( + "crypto/hmac" + "crypto/sha1" + "encoding/base64" + "errors" + "fmt" + "net/http" + "time" +) + +type WasabiStorage struct { + StorageBase + + s3 *S3Storage + region string + endpoint string + bucket string + storageDir string + key string + secret string + client *http.Client +} + +// See the Storage interface in duplicacy_storage.go for function +// descriptions. + +func CreateWasabiStorage( + regionName string, endpoint string, + bucketName string, storageDir string, + accessKey string, secretKey string, + threads int, +) (storage *WasabiStorage, err error) { + + s3storage, error := CreateS3Storage(regionName, endpoint, bucketName, + storageDir, accessKey, secretKey, threads, + true, // isSSLSupported + false, // isMinioCompatible + ) + + if err != nil { + return nil, error + } + + wasabi := &WasabiStorage{ + + // Pass-through to existing S3 module + s3: s3storage, + + // Local copies required for renaming + region: regionName, + endpoint: endpoint, + bucket: bucketName, + storageDir: storageDir, + key: accessKey, + secret: secretKey, + client: &http.Client{}, + } + + wasabi.DerivedStorage = wasabi + wasabi.SetDefaultNestingLevels([]int{0}, 0) + + return wasabi, nil +} + +func (storage *WasabiStorage) ListFiles( + threadIndex int, dir string, +) (files []string, sizes []int64, err error) { + return storage.s3.ListFiles(threadIndex, dir) +} + +func (storage *WasabiStorage) DeleteFile( + threadIndex int, filePath string, +) (err error) { + return storage.s3.DeleteFile(threadIndex, filePath) + +} + +// This is a lightweight implementation of a call to Wasabi for a +// rename. It's designed to get the job done with as few dependencies +// on other packages as possible rather than being somethng +// general-purpose and reusable. +func (storage *WasabiStorage) MoveFile( + threadIndex int, from string, to string, +) (err error) { + + // The from path includes the bucket + from_path := fmt.Sprintf("/%s/%s/%s", storage.bucket, storage.storageDir, from) + + object := fmt.Sprintf("https://%s@%s%s", + storage.region, storage.endpoint, from_path) + + // The object's new name is relative to the top of the bucket. + new_name := fmt.Sprintf("%s/%s", storage.storageDir, to) + + timestamp := time.Now().Format(time.RFC1123Z) + + signing_string := fmt.Sprintf("MOVE\n\n\n%s\n%s", timestamp, from_path) + + signer := hmac.New(sha1.New, []byte(storage.secret)) + signer.Write([]byte(signing_string)) + + signature := base64.StdEncoding.EncodeToString(signer.Sum(nil)) + + authorization := fmt.Sprintf("AWS %s:%s", storage.key, signature) + + request, error := http.NewRequest("MOVE", object, nil) + if error != nil { + return error + } + request.Header.Add("Authorization", authorization) + request.Header.Add("Date", timestamp) + request.Header.Add("Destination", new_name) + request.Header.Add("Host", storage.endpoint) + request.Header.Add("Overwrite", "true") + + response, error := storage.client.Do(request) + if error != nil { + return error + } + defer response.Body.Close() + + if response.StatusCode != 200 { + return errors.New(response.Status) + } + + return nil +} + +func (storage *WasabiStorage) CreateDirectory( + threadIndex int, dir string, +) (err error) { + return storage.s3.CreateDirectory(threadIndex, dir) +} + +func (storage *WasabiStorage) GetFileInfo( + threadIndex int, filePath string, +) (exist bool, isDir bool, size int64, err error) { + return storage.s3.GetFileInfo(threadIndex, filePath) +} + +func (storage *WasabiStorage) DownloadFile( + threadIndex int, filePath string, chunk *Chunk, +) (err error) { + return storage.s3.DownloadFile(threadIndex, filePath, chunk) +} + +func (storage *WasabiStorage) UploadFile( + threadIndex int, filePath string, content []byte, +) (err error) { + return storage.s3.UploadFile(threadIndex, filePath, content) +} + +func (storage *WasabiStorage) IsCacheNeeded() bool { + return storage.s3.IsCacheNeeded() +} + +func (storage *WasabiStorage) IsMoveFileImplemented() bool { + // This is implemented locally since S3 does a copy and delete + return true +} + +func (storage *WasabiStorage) IsStrongConsistent() bool { + // Wasabi has it, S3 doesn't. + return true +} + +func (storage *WasabiStorage) IsFastListing() bool { + return storage.s3.IsFastListing() +} + +func (storage *WasabiStorage) EnableTestMode() { +}