mirror of
https://github.com/gilbertchen/duplicacy
synced 2026-01-01 16:13:31 +00:00
Release source code under the Fair Source 5 License
This commit is contained in:
Gilbert Chen
516
duplicacy_b2client.go
Normal file
516
duplicacy_b2client.go
Normal file
@@ -0,0 +1,516 @@
|
||||
// Copyright (c) Acrosync LLC. All rights reserved.
|
||||
// Licensed under the Fair Source License 0.9 (https://fair.io/)
|
||||
// User Limitation: 5 users
|
||||
|
||||
package duplicacy
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
"bytes"
|
||||
"strconv"
|
||||
"io/ioutil"
|
||||
"encoding/json"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"crypto/sha1"
|
||||
"math/rand"
|
||||
)
|
||||
|
||||
type B2Error struct {
|
||||
Status int
|
||||
Code string
|
||||
Message string
|
||||
}
|
||||
|
||||
func (err *B2Error) Error() string {
|
||||
return fmt.Sprintf("%d %s", err.Status, err.Message)
|
||||
}
|
||||
|
||||
type B2UploadArgument struct {
|
||||
URL string
|
||||
Token string
|
||||
}
|
||||
|
||||
var B2AuthorizationURL = "https://api.backblazeb2.com/b2api/v1/b2_authorize_account"
|
||||
|
||||
type B2Client struct {
|
||||
HTTPClient *http.Client
|
||||
AccountID string
|
||||
ApplicationKey string
|
||||
AuthorizationToken string
|
||||
APIURL string
|
||||
DownloadURL string
|
||||
BucketName string
|
||||
BucketID string
|
||||
|
||||
UploadURL string
|
||||
UploadToken string
|
||||
|
||||
TestMode bool
|
||||
|
||||
}
|
||||
|
||||
func NewB2Client(accountID string, applicationKey string) *B2Client {
|
||||
client := &B2Client{
|
||||
HTTPClient: http.DefaultClient,
|
||||
AccountID: accountID,
|
||||
ApplicationKey: applicationKey,
|
||||
}
|
||||
return client
|
||||
}
|
||||
|
||||
func (client *B2Client) retry(backoff int, response *http.Response) int {
|
||||
if response != nil {
|
||||
if backoffList, found := response.Header["Retry-After"]; found && len(backoffList) > 0 {
|
||||
retryAfter, _ := strconv.Atoi(backoffList[0])
|
||||
if retryAfter >= 1 {
|
||||
time.Sleep(time.Duration(retryAfter) * time.Second)
|
||||
return 0
|
||||
}
|
||||
}
|
||||
}
|
||||
if backoff == 0 {
|
||||
backoff = 1
|
||||
} else {
|
||||
backoff *= 2
|
||||
}
|
||||
time.Sleep(time.Duration(backoff) * time.Second)
|
||||
return backoff
|
||||
}
|
||||
|
||||
func (client *B2Client) call(url string, input interface{}) (io.ReadCloser, int64, error) {
|
||||
|
||||
var response *http.Response
|
||||
|
||||
backoff := 0
|
||||
for i := 0; i < 8; i++ {
|
||||
var inputReader *bytes.Reader
|
||||
method := "POST"
|
||||
|
||||
switch input.(type) {
|
||||
default:
|
||||
jsonInput, err := json.Marshal(input)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
inputReader = bytes.NewReader(jsonInput)
|
||||
case []byte:
|
||||
inputReader = bytes.NewReader(input.([]byte))
|
||||
case int:
|
||||
method = "GET"
|
||||
inputReader = bytes.NewReader([]byte(""))
|
||||
}
|
||||
|
||||
request, err := http.NewRequest(method, url, inputReader)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
if url == B2AuthorizationURL {
|
||||
request.Header.Set("Authorization", "Basic " + base64.StdEncoding.EncodeToString([]byte(client.AccountID + ":" + client.ApplicationKey)))
|
||||
} else {
|
||||
request.Header.Set("Authorization", client.AuthorizationToken)
|
||||
}
|
||||
|
||||
if client.TestMode {
|
||||
r := rand.Float32()
|
||||
if r < 0.5 {
|
||||
request.Header.Set("X-Bz-Test-Mode", "expire_some_account_authorization_tokens")
|
||||
} else {
|
||||
request.Header.Set("X-Bz-Test-Mode", "force_cap_exceeded")
|
||||
}
|
||||
}
|
||||
|
||||
response, err = client.HTTPClient.Do(request)
|
||||
if err != nil {
|
||||
if url != B2AuthorizationURL {
|
||||
LOG_DEBUG("BACKBLAZE_CALL", "URL request '%s' returned an error: %v", url, err)
|
||||
backoff = client.retry(backoff, response)
|
||||
continue
|
||||
}
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
if response.StatusCode < 400 {
|
||||
return response.Body, response.ContentLength, nil
|
||||
}
|
||||
|
||||
LOG_DEBUG("BACKBLAZE_CALL", "URL request '%s' returned status code %d", url, response.StatusCode)
|
||||
|
||||
io.Copy(ioutil.Discard, response.Body)
|
||||
response.Body.Close()
|
||||
if response.StatusCode == 401 {
|
||||
if url == B2AuthorizationURL {
|
||||
return nil, 0, fmt.Errorf("Authorization failure")
|
||||
}
|
||||
client.AuthorizeAccount()
|
||||
continue
|
||||
} else if response.StatusCode == 403 {
|
||||
if !client.TestMode {
|
||||
return nil, 0, fmt.Errorf("B2 cap exceeded")
|
||||
}
|
||||
continue
|
||||
} else if response.StatusCode == 429 || response.StatusCode == 408 {
|
||||
backoff = client.retry(backoff, response)
|
||||
continue
|
||||
} else if response.StatusCode >= 500 && response.StatusCode <= 599 {
|
||||
backoff = client.retry(backoff, response)
|
||||
continue
|
||||
}
|
||||
|
||||
defer response.Body.Close()
|
||||
|
||||
e := &B2Error {
|
||||
}
|
||||
|
||||
if err := json.NewDecoder(response.Body).Decode(e); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
return nil, 0, e
|
||||
}
|
||||
|
||||
return nil, 0, fmt.Errorf("Maximum backoff reached")
|
||||
}
|
||||
|
||||
type B2AuthorizeAccountOutput struct {
|
||||
AccountID string
|
||||
AuthorizationToken string
|
||||
APIURL string
|
||||
DownloadURL string
|
||||
}
|
||||
|
||||
func (client *B2Client) AuthorizeAccount() (err error) {
|
||||
|
||||
readCloser, _, err := client.call(B2AuthorizationURL, make(map[string]string))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer readCloser.Close()
|
||||
|
||||
output := &B2AuthorizeAccountOutput {}
|
||||
|
||||
if err = json.NewDecoder(readCloser).Decode(&output); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
client.AuthorizationToken = output.AuthorizationToken
|
||||
client.APIURL = output.APIURL
|
||||
client.DownloadURL = output.DownloadURL
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type ListBucketOutput struct {
|
||||
AccoundID string
|
||||
BucketID string
|
||||
BucketName string
|
||||
BucketType string
|
||||
}
|
||||
|
||||
func (client *B2Client) FindBucket(bucketName string) (err error) {
|
||||
|
||||
input := make(map[string]string)
|
||||
input["accountId"] = client.AccountID
|
||||
|
||||
url := client.APIURL + "/b2api/v1/b2_list_buckets"
|
||||
|
||||
readCloser, _, err := client.call(url, input)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer readCloser.Close()
|
||||
|
||||
output := make(map[string][]ListBucketOutput, 0)
|
||||
|
||||
if err = json.NewDecoder(readCloser).Decode(&output); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, bucket := range output["buckets"] {
|
||||
if bucket.BucketName == bucketName {
|
||||
client.BucketName = bucket.BucketName
|
||||
client.BucketID = bucket.BucketID
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if client.BucketID == "" {
|
||||
return fmt.Errorf("Bucket %s not found", bucketName)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type B2Entry struct {
|
||||
FileID string
|
||||
FileName string
|
||||
Action string
|
||||
Size int64
|
||||
UploadTimestamp int64
|
||||
}
|
||||
|
||||
type B2ListFileNamesOutput struct {
|
||||
Files []*B2Entry
|
||||
NextFileName string
|
||||
NextFileId string
|
||||
}
|
||||
|
||||
func (client *B2Client) ListFileNames(startFileName string, singleFile bool, includeVersions bool) (files []*B2Entry, err error) {
|
||||
|
||||
maxFileCount := 1000
|
||||
if singleFile {
|
||||
if includeVersions {
|
||||
maxFileCount = 4
|
||||
if client.TestMode {
|
||||
maxFileCount = 1
|
||||
}
|
||||
} else {
|
||||
maxFileCount = 1
|
||||
}
|
||||
} else if client.TestMode {
|
||||
maxFileCount = 10
|
||||
}
|
||||
|
||||
input := make(map[string]interface{})
|
||||
input["bucketId"] = client.BucketID
|
||||
input["startFileName"] = startFileName
|
||||
input["maxFileCount"] = maxFileCount
|
||||
|
||||
for {
|
||||
url := client.APIURL + "/b2api/v1/b2_list_file_names"
|
||||
if includeVersions {
|
||||
url = client.APIURL + "/b2api/v1/b2_list_file_versions"
|
||||
}
|
||||
readCloser, _, err := client.call(url, input)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
defer readCloser.Close()
|
||||
|
||||
output := B2ListFileNamesOutput {
|
||||
}
|
||||
|
||||
if err = json.NewDecoder(readCloser).Decode(&output); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
ioutil.ReadAll(readCloser)
|
||||
|
||||
if startFileName == "" {
|
||||
files = append(files, output.Files...)
|
||||
} else {
|
||||
for _, file := range output.Files {
|
||||
if singleFile {
|
||||
if file.FileName == startFileName {
|
||||
files = append(files, file)
|
||||
if !includeVersions {
|
||||
output.NextFileName = ""
|
||||
break
|
||||
}
|
||||
} else {
|
||||
output.NextFileName = ""
|
||||
break
|
||||
}
|
||||
} else {
|
||||
if strings.HasPrefix(file.FileName, startFileName) {
|
||||
files = append(files, file)
|
||||
} else {
|
||||
output.NextFileName = ""
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if len(output.NextFileName) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
input["startFileName"] = output.NextFileName
|
||||
if includeVersions {
|
||||
input["startFileId"] = output.NextFileId
|
||||
}
|
||||
}
|
||||
|
||||
return files, nil
|
||||
}
|
||||
|
||||
func (client *B2Client) DeleteFile(fileName string, fileID string) (err error) {
|
||||
|
||||
input := make(map[string]string)
|
||||
input["fileName"] = fileName
|
||||
input["fileId"] = fileID
|
||||
|
||||
url := client.APIURL + "/b2api/v1/b2_delete_file_version"
|
||||
readCloser, _, err := client.call(url, input)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
readCloser.Close()
|
||||
return nil
|
||||
}
|
||||
|
||||
type B2HideFileOutput struct {
|
||||
FileID string
|
||||
}
|
||||
|
||||
func (client *B2Client) HideFile(fileName string) (fileID string, err error) {
|
||||
|
||||
input := make(map[string]string)
|
||||
input["bucketId"] = client.BucketID
|
||||
input["fileName"] = fileName
|
||||
|
||||
url := client.APIURL + "/b2api/v1/b2_hide_file"
|
||||
readCloser, _, err := client.call(url, input)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
defer readCloser.Close()
|
||||
|
||||
output := & B2HideFileOutput {}
|
||||
|
||||
if err = json.NewDecoder(readCloser).Decode(&output); err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
readCloser.Close()
|
||||
return output.FileID, nil
|
||||
}
|
||||
|
||||
func (client *B2Client) DownloadFile(filePath string) (io.ReadCloser, int64, error) {
|
||||
|
||||
url := client.DownloadURL + "/file/" + client.BucketName + "/" + filePath
|
||||
|
||||
return client.call(url, 0)
|
||||
}
|
||||
|
||||
type B2GetUploadArgumentOutput struct {
|
||||
BucketID string
|
||||
UploadURL string
|
||||
AuthorizationToken string
|
||||
}
|
||||
|
||||
func (client *B2Client) getUploadURL() (error) {
|
||||
input := make(map[string]string)
|
||||
input["bucketId"] = client.BucketID
|
||||
|
||||
url := client.APIURL + "/b2api/v1/b2_get_upload_url"
|
||||
readCloser, _, err := client.call(url, input)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer readCloser.Close()
|
||||
|
||||
output := & B2GetUploadArgumentOutput {}
|
||||
|
||||
if err = json.NewDecoder(readCloser).Decode(&output); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
client.UploadURL = output.UploadURL
|
||||
client.UploadToken = output.AuthorizationToken
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (client *B2Client) UploadFile(filePath string, content []byte, rateLimit int) (err error) {
|
||||
|
||||
|
||||
hasher := sha1.New()
|
||||
hasher.Write(content)
|
||||
hash := hex.EncodeToString(hasher.Sum(nil))
|
||||
|
||||
headers := make(map[string]string)
|
||||
headers["X-Bz-File-Name"] = filePath
|
||||
headers["Content-Type"] = "application/octet-stream"
|
||||
headers["X-Bz-Content-Sha1"] = hash
|
||||
|
||||
var response *http.Response
|
||||
|
||||
backoff := 0
|
||||
for i := 0; i < 8; i++ {
|
||||
|
||||
if client.UploadURL == "" || client.UploadToken == "" {
|
||||
err = client.getUploadURL()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
request, err := http.NewRequest("POST", client.UploadURL, CreateRateLimitedReader(content, rateLimit))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
request.ContentLength = int64(len(content))
|
||||
|
||||
request.Header.Set("Authorization", client.UploadToken)
|
||||
request.Header.Set("X-Bz-File-Name", filePath)
|
||||
request.Header.Set("Content-Type", "application/octet-stream")
|
||||
request.Header.Set("X-Bz-Content-Sha1", hash)
|
||||
|
||||
for key, value := range headers {
|
||||
request.Header.Set(key, value)
|
||||
}
|
||||
|
||||
if client.TestMode {
|
||||
r := rand.Float32()
|
||||
if r < 0.8 {
|
||||
request.Header.Set("X-Bz-Test-Mode", "fail_some_uploads")
|
||||
} else if r < 0.9 {
|
||||
request.Header.Set("X-Bz-Test-Mode", "expire_some_account_authorization_tokens")
|
||||
} else {
|
||||
request.Header.Set("X-Bz-Test-Mode", "force_cap_exceeded")
|
||||
}
|
||||
}
|
||||
|
||||
response, err = client.HTTPClient.Do(request)
|
||||
if err != nil {
|
||||
LOG_DEBUG("BACKBLAZE_UPLOAD", "URL request '%s' returned an error: %v", client.UploadURL, err)
|
||||
backoff = client.retry(backoff, response)
|
||||
client.UploadURL = ""
|
||||
client.UploadToken = ""
|
||||
continue
|
||||
}
|
||||
|
||||
io.Copy(ioutil.Discard, response.Body)
|
||||
response.Body.Close()
|
||||
|
||||
if response.StatusCode < 400 {
|
||||
return nil
|
||||
}
|
||||
|
||||
LOG_DEBUG("BACKBLAZE_UPLOAD", "URL request '%s' returned status code %d", client.UploadURL, response.StatusCode)
|
||||
|
||||
if response.StatusCode == 401 {
|
||||
LOG_INFO("BACKBLAZE_UPLOAD", "Re-authorizatoin required")
|
||||
client.UploadURL = ""
|
||||
client.UploadToken = ""
|
||||
continue
|
||||
} else if response.StatusCode == 403 {
|
||||
if !client.TestMode {
|
||||
return fmt.Errorf("B2 cap exceeded")
|
||||
}
|
||||
continue
|
||||
} else {
|
||||
LOG_INFO("BACKBLAZE_UPLOAD", "URL request '%s' returned status code %d", client.UploadURL, response.StatusCode)
|
||||
backoff = client.retry(backoff, response)
|
||||
client.UploadURL = ""
|
||||
client.UploadToken = ""
|
||||
}
|
||||
}
|
||||
|
||||
return fmt.Errorf("Maximum backoff reached")
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user