diff --git a/src/duplicacy_storage.go b/src/duplicacy_storage.go index 6aa64a4..941ed65 100644 --- a/src/duplicacy_storage.go +++ b/src/duplicacy_storage.go @@ -461,6 +461,38 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor SavePassword(preference, "s3_secret", secretKey) return storage + + } else if matched[1] == "wasabi" { + + region := matched[2] + endpoint := matched[3] + bucket := matched[5] + + key := GetPassword(preference, "wasabi_key", + "Enter Wasabi key:", true, resetPassword) + secret := GetPassword(preference, "wasabi_secret", + "Enter Wasabi secret:", true, resetPassword) + + storageDir := "" + if strings.Contains(bucket, "/") { + firstSlash := strings.Index(bucket, "/") + storageDir = bucket[firstSlash+1:] + bucket = bucket[:firstSlash] + } + + storage, err := CreateWasabiStorage(region, endpoint, + bucket, storageDir, key, secret, threads) + + if err != nil { + LOG_ERROR("STORAGE_CREATE", "Failed to load the S3 storage at %s: %v", storageURL, err) + return nil + } + + SavePassword(preference, "key", key) + SavePassword(preference, "secret", secret) + + return storage + } else if matched[1] == "dropbox" { storageDir := matched[3] + matched[5] token := GetPassword(preference, "dropbox_token", "Enter Dropbox access token:", true, resetPassword) diff --git a/src/duplicacy_storage_test.go b/src/duplicacy_storage_test.go index d804360..d3955cb 100644 --- a/src/duplicacy_storage_test.go +++ b/src/duplicacy_storage_test.go @@ -78,10 +78,12 @@ func loadStorage(localStoragePath string, threads int) (Storage, error) { storage, err := CreateSFTPStorageWithPassword(config["server"], port, config["username"], config["directory"], 2, config["password"], threads) storage.SetDefaultNestingLevels([]int{2, 3}, 2) return storage, err - } else if testStorageName == "s3" || testStorageName == "wasabi" { + } else if testStorageName == "s3" { storage, err := CreateS3Storage(config["region"], config["endpoint"], config["bucket"], config["directory"], config["access_key"], config["secret_key"], threads, true, false) storage.SetDefaultNestingLevels([]int{2, 3}, 2) - return storage, err + } else if testStorageName == "wasabi" { + storage, err := CreateWasabiStorage(config["region"], config["endpoint"], config["bucket"], config["directory"], config["access_key"], config["secret_key"], threads) + storage.SetDefaultNestingLevels([]int{2, 3}, 2) } else if testStorageName == "s3c" { storage, err := CreateS3CStorage(config["region"], config["endpoint"], config["bucket"], config["directory"], config["access_key"], config["secret_key"], threads) storage.SetDefaultNestingLevels([]int{2, 3}, 2) diff --git a/src/duplicacy_wasabistorage.go b/src/duplicacy_wasabistorage.go new file mode 100644 index 0000000..9c6ce7c --- /dev/null +++ b/src/duplicacy_wasabistorage.go @@ -0,0 +1,189 @@ +// +// Storage module for Wasabi (https://www.wasabi.com) +// + +// Wasabi is nominally compatible with AWS S3, but the copy-and-delete +// method used for renaming objects creates additional expense under +// Wasabi's billing system. This module is a pass-through to the +// existing S3 module for everything other than that one operation. +// +// This module copyright 2017 Mark Feit (https://github.com/markfeit) +// and may be distributed under the same terms as Duplicacy. + +package duplicacy + +import ( + "crypto/hmac" + "crypto/sha1" + "encoding/base64" + "errors" + "fmt" + "net/http" + "time" +) + +type WasabiStorage struct { + StorageBase + + s3 *S3Storage + region string + endpoint string + bucket string + storageDir string + key string + secret string +} + +// See the Storage interface in duplicacy_storage.go for function +// descriptions. + +func CreateWasabiStorage( + regionName string, endpoint string, + bucketName string, storageDir string, + accessKey string, secretKey string, + threads int, +) (storage *WasabiStorage, err error) { + + s3storage, error := CreateS3Storage(regionName, endpoint, bucketName, + storageDir, accessKey, secretKey, threads, + true, // isSSLSupported + false, // isMinioCompatible + ) + + if err != nil { + return nil, error + } + + wasabi := &WasabiStorage{ + + // Pass-through to existing S3 module + s3: s3storage, + + // Local copies required for renaming + region: regionName, + endpoint: endpoint, + bucket: bucketName, + storageDir: storageDir, + key: accessKey, + secret: secretKey, + } + + wasabi.DerivedStorage = wasabi + wasabi.SetDefaultNestingLevels([]int{0}, 0) + + return wasabi, nil +} + +func (storage *WasabiStorage) ListFiles( + threadIndex int, dir string, +) (files []string, sizes []int64, err error) { + return storage.s3.ListFiles(threadIndex, dir) +} + +func (storage *WasabiStorage) DeleteFile( + threadIndex int, filePath string, +) (err error) { + return storage.s3.DeleteFile(threadIndex, filePath) + +} + +// This is a lightweight implementation of a call to Wasabi for a +// rename. It's designed to get the job done with as few dependencies +// on other packages as possible rather than being somethng +// general-purpose and reusable. +func (storage *WasabiStorage) MoveFile( + threadIndex int, from string, to string, +) (err error) { + + // The from path includes the bucket + from_path := fmt.Sprintf("/%s/%s/%s", storage.bucket, storage.storageDir, from) + + // The to path does not. + // to_path := fmt.Sprintf("%s/%s", storage.storageDir, from) + + // We get the region with an @ on the end, so don't add another. + object := fmt.Sprintf("https://%s%s%s", + storage.region, storage.endpoint, from_path) + + // The object's new name is relative to the top of the bucket. + new_name := fmt.Sprintf("%s/%s", storage.storageDir, to) + + timestamp := time.Now().Format(time.RFC1123Z) + + signing_string := fmt.Sprintf("MOVE\n\n\n%s\n%s", timestamp, from_path) + + signer := hmac.New(sha1.New, []byte(storage.secret)) + signer.Write([]byte(signing_string)) + + signature := base64.StdEncoding.EncodeToString(signer.Sum(nil)) + + authorization := fmt.Sprintf("AWS %s:%s", storage.key, signature) + + request, error := http.NewRequest("MOVE", object, nil) + if error != nil { + return error + } + request.Header.Add("Authorization", authorization) + request.Header.Add("Date", timestamp) + request.Header.Add("Destination", new_name) + request.Header.Add("Host", storage.endpoint) + request.Header.Add("Overwrite", "true") + + client := &http.Client{} + + response, error := client.Do(request) + if error != nil { + return error + } + + if response.StatusCode != 200 { + return errors.New(response.Status) + } + + return nil +} + +func (storage *WasabiStorage) CreateDirectory( + threadIndex int, dir string, +) (err error) { + return storage.s3.CreateDirectory(threadIndex, dir) +} + +func (storage *WasabiStorage) GetFileInfo( + threadIndex int, filePath string, +) (exist bool, isDir bool, size int64, err error) { + return storage.s3.GetFileInfo(threadIndex, filePath) +} + +func (storage *WasabiStorage) DownloadFile( + threadIndex int, filePath string, chunk *Chunk, +) (err error) { + return storage.s3.DownloadFile(threadIndex, filePath, chunk) +} + +func (storage *WasabiStorage) UploadFile( + threadIndex int, filePath string, content []byte, +) (err error) { + return storage.s3.UploadFile(threadIndex, filePath, content) +} + +func (storage *WasabiStorage) IsCacheNeeded() bool { + return storage.s3.IsCacheNeeded() +} + +func (storage *WasabiStorage) IsMoveFileImplemented() bool { + // This is implemented locally since S3 does a copy and delete + return true +} + +func (storage *WasabiStorage) IsStrongConsistent() bool { + // Wasabi has it, S3 doesn't. + return true +} + +func (storage *WasabiStorage) IsFastListing() bool { + return true +} + +func (storage *WasabiStorage) EnableTestMode() { +}