mirror of
https://github.com/gilbertchen/duplicacy
synced 2025-12-06 00:03:38 +00:00
517 lines
14 KiB
Go
517 lines
14 KiB
Go
// Copyright (c) Acrosync LLC. All rights reserved.
|
|
// Licensed under the Fair Source License 0.9 (https://fair.io/)
|
|
// User Limitation: 5 users
|
|
|
|
package duplicacy
|
|
|
|
import (
|
|
"fmt"
|
|
"time"
|
|
"bytes"
|
|
"strconv"
|
|
"io/ioutil"
|
|
"encoding/json"
|
|
"encoding/base64"
|
|
"encoding/hex"
|
|
"io"
|
|
"net/http"
|
|
"strings"
|
|
"crypto/sha1"
|
|
"math/rand"
|
|
)
|
|
|
|
type B2Error struct {
|
|
Status int
|
|
Code string
|
|
Message string
|
|
}
|
|
|
|
func (err *B2Error) Error() string {
|
|
return fmt.Sprintf("%d %s", err.Status, err.Message)
|
|
}
|
|
|
|
type B2UploadArgument struct {
|
|
URL string
|
|
Token string
|
|
}
|
|
|
|
var B2AuthorizationURL = "https://api.backblazeb2.com/b2api/v1/b2_authorize_account"
|
|
|
|
type B2Client struct {
|
|
HTTPClient *http.Client
|
|
AccountID string
|
|
ApplicationKey string
|
|
AuthorizationToken string
|
|
APIURL string
|
|
DownloadURL string
|
|
BucketName string
|
|
BucketID string
|
|
|
|
UploadURL string
|
|
UploadToken string
|
|
|
|
TestMode bool
|
|
|
|
}
|
|
|
|
func NewB2Client(accountID string, applicationKey string) *B2Client {
|
|
client := &B2Client{
|
|
HTTPClient: http.DefaultClient,
|
|
AccountID: accountID,
|
|
ApplicationKey: applicationKey,
|
|
}
|
|
return client
|
|
}
|
|
|
|
func (client *B2Client) retry(backoff int, response *http.Response) int {
|
|
if response != nil {
|
|
if backoffList, found := response.Header["Retry-After"]; found && len(backoffList) > 0 {
|
|
retryAfter, _ := strconv.Atoi(backoffList[0])
|
|
if retryAfter >= 1 {
|
|
time.Sleep(time.Duration(retryAfter) * time.Second)
|
|
return 0
|
|
}
|
|
}
|
|
}
|
|
if backoff == 0 {
|
|
backoff = 1
|
|
} else {
|
|
backoff *= 2
|
|
}
|
|
time.Sleep(time.Duration(backoff) * time.Second)
|
|
return backoff
|
|
}
|
|
|
|
func (client *B2Client) call(url string, input interface{}) (io.ReadCloser, int64, error) {
|
|
|
|
var response *http.Response
|
|
|
|
backoff := 0
|
|
for i := 0; i < 8; i++ {
|
|
var inputReader *bytes.Reader
|
|
method := "POST"
|
|
|
|
switch input.(type) {
|
|
default:
|
|
jsonInput, err := json.Marshal(input)
|
|
if err != nil {
|
|
return nil, 0, err
|
|
}
|
|
inputReader = bytes.NewReader(jsonInput)
|
|
case []byte:
|
|
inputReader = bytes.NewReader(input.([]byte))
|
|
case int:
|
|
method = "GET"
|
|
inputReader = bytes.NewReader([]byte(""))
|
|
}
|
|
|
|
request, err := http.NewRequest(method, url, inputReader)
|
|
if err != nil {
|
|
return nil, 0, err
|
|
}
|
|
|
|
if url == B2AuthorizationURL {
|
|
request.Header.Set("Authorization", "Basic " + base64.StdEncoding.EncodeToString([]byte(client.AccountID + ":" + client.ApplicationKey)))
|
|
} else {
|
|
request.Header.Set("Authorization", client.AuthorizationToken)
|
|
}
|
|
|
|
if client.TestMode {
|
|
r := rand.Float32()
|
|
if r < 0.5 {
|
|
request.Header.Set("X-Bz-Test-Mode", "expire_some_account_authorization_tokens")
|
|
} else {
|
|
request.Header.Set("X-Bz-Test-Mode", "force_cap_exceeded")
|
|
}
|
|
}
|
|
|
|
response, err = client.HTTPClient.Do(request)
|
|
if err != nil {
|
|
if url != B2AuthorizationURL {
|
|
LOG_DEBUG("BACKBLAZE_CALL", "URL request '%s' returned an error: %v", url, err)
|
|
backoff = client.retry(backoff, response)
|
|
continue
|
|
}
|
|
return nil, 0, err
|
|
}
|
|
|
|
if response.StatusCode < 400 {
|
|
return response.Body, response.ContentLength, nil
|
|
}
|
|
|
|
LOG_DEBUG("BACKBLAZE_CALL", "URL request '%s' returned status code %d", url, response.StatusCode)
|
|
|
|
io.Copy(ioutil.Discard, response.Body)
|
|
response.Body.Close()
|
|
if response.StatusCode == 401 {
|
|
if url == B2AuthorizationURL {
|
|
return nil, 0, fmt.Errorf("Authorization failure")
|
|
}
|
|
client.AuthorizeAccount()
|
|
continue
|
|
} else if response.StatusCode == 403 {
|
|
if !client.TestMode {
|
|
return nil, 0, fmt.Errorf("B2 cap exceeded")
|
|
}
|
|
continue
|
|
} else if response.StatusCode == 429 || response.StatusCode == 408 {
|
|
backoff = client.retry(backoff, response)
|
|
continue
|
|
} else if response.StatusCode >= 500 && response.StatusCode <= 599 {
|
|
backoff = client.retry(backoff, response)
|
|
continue
|
|
}
|
|
|
|
defer response.Body.Close()
|
|
|
|
e := &B2Error {
|
|
}
|
|
|
|
if err := json.NewDecoder(response.Body).Decode(e); err != nil {
|
|
return nil, 0, err
|
|
}
|
|
|
|
return nil, 0, e
|
|
}
|
|
|
|
return nil, 0, fmt.Errorf("Maximum backoff reached")
|
|
}
|
|
|
|
type B2AuthorizeAccountOutput struct {
|
|
AccountID string
|
|
AuthorizationToken string
|
|
APIURL string
|
|
DownloadURL string
|
|
}
|
|
|
|
func (client *B2Client) AuthorizeAccount() (err error) {
|
|
|
|
readCloser, _, err := client.call(B2AuthorizationURL, make(map[string]string))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
defer readCloser.Close()
|
|
|
|
output := &B2AuthorizeAccountOutput {}
|
|
|
|
if err = json.NewDecoder(readCloser).Decode(&output); err != nil {
|
|
return err
|
|
}
|
|
|
|
client.AuthorizationToken = output.AuthorizationToken
|
|
client.APIURL = output.APIURL
|
|
client.DownloadURL = output.DownloadURL
|
|
|
|
return nil
|
|
}
|
|
|
|
type ListBucketOutput struct {
|
|
AccoundID string
|
|
BucketID string
|
|
BucketName string
|
|
BucketType string
|
|
}
|
|
|
|
func (client *B2Client) FindBucket(bucketName string) (err error) {
|
|
|
|
input := make(map[string]string)
|
|
input["accountId"] = client.AccountID
|
|
|
|
url := client.APIURL + "/b2api/v1/b2_list_buckets"
|
|
|
|
readCloser, _, err := client.call(url, input)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
defer readCloser.Close()
|
|
|
|
output := make(map[string][]ListBucketOutput, 0)
|
|
|
|
if err = json.NewDecoder(readCloser).Decode(&output); err != nil {
|
|
return err
|
|
}
|
|
|
|
for _, bucket := range output["buckets"] {
|
|
if bucket.BucketName == bucketName {
|
|
client.BucketName = bucket.BucketName
|
|
client.BucketID = bucket.BucketID
|
|
break
|
|
}
|
|
}
|
|
|
|
if client.BucketID == "" {
|
|
return fmt.Errorf("Bucket %s not found", bucketName)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
type B2Entry struct {
|
|
FileID string
|
|
FileName string
|
|
Action string
|
|
Size int64
|
|
UploadTimestamp int64
|
|
}
|
|
|
|
type B2ListFileNamesOutput struct {
|
|
Files []*B2Entry
|
|
NextFileName string
|
|
NextFileId string
|
|
}
|
|
|
|
func (client *B2Client) ListFileNames(startFileName string, singleFile bool, includeVersions bool) (files []*B2Entry, err error) {
|
|
|
|
maxFileCount := 1000
|
|
if singleFile {
|
|
if includeVersions {
|
|
maxFileCount = 4
|
|
if client.TestMode {
|
|
maxFileCount = 1
|
|
}
|
|
} else {
|
|
maxFileCount = 1
|
|
}
|
|
} else if client.TestMode {
|
|
maxFileCount = 10
|
|
}
|
|
|
|
input := make(map[string]interface{})
|
|
input["bucketId"] = client.BucketID
|
|
input["startFileName"] = startFileName
|
|
input["maxFileCount"] = maxFileCount
|
|
|
|
for {
|
|
url := client.APIURL + "/b2api/v1/b2_list_file_names"
|
|
if includeVersions {
|
|
url = client.APIURL + "/b2api/v1/b2_list_file_versions"
|
|
}
|
|
readCloser, _, err := client.call(url, input)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
defer readCloser.Close()
|
|
|
|
output := B2ListFileNamesOutput {
|
|
}
|
|
|
|
if err = json.NewDecoder(readCloser).Decode(&output); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
ioutil.ReadAll(readCloser)
|
|
|
|
if startFileName == "" {
|
|
files = append(files, output.Files...)
|
|
} else {
|
|
for _, file := range output.Files {
|
|
if singleFile {
|
|
if file.FileName == startFileName {
|
|
files = append(files, file)
|
|
if !includeVersions {
|
|
output.NextFileName = ""
|
|
break
|
|
}
|
|
} else {
|
|
output.NextFileName = ""
|
|
break
|
|
}
|
|
} else {
|
|
if strings.HasPrefix(file.FileName, startFileName) {
|
|
files = append(files, file)
|
|
} else {
|
|
output.NextFileName = ""
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
if len(output.NextFileName) == 0 {
|
|
break
|
|
}
|
|
|
|
input["startFileName"] = output.NextFileName
|
|
if includeVersions {
|
|
input["startFileId"] = output.NextFileId
|
|
}
|
|
}
|
|
|
|
return files, nil
|
|
}
|
|
|
|
func (client *B2Client) DeleteFile(fileName string, fileID string) (err error) {
|
|
|
|
input := make(map[string]string)
|
|
input["fileName"] = fileName
|
|
input["fileId"] = fileID
|
|
|
|
url := client.APIURL + "/b2api/v1/b2_delete_file_version"
|
|
readCloser, _, err := client.call(url, input)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
readCloser.Close()
|
|
return nil
|
|
}
|
|
|
|
type B2HideFileOutput struct {
|
|
FileID string
|
|
}
|
|
|
|
func (client *B2Client) HideFile(fileName string) (fileID string, err error) {
|
|
|
|
input := make(map[string]string)
|
|
input["bucketId"] = client.BucketID
|
|
input["fileName"] = fileName
|
|
|
|
url := client.APIURL + "/b2api/v1/b2_hide_file"
|
|
readCloser, _, err := client.call(url, input)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
defer readCloser.Close()
|
|
|
|
output := & B2HideFileOutput {}
|
|
|
|
if err = json.NewDecoder(readCloser).Decode(&output); err != nil {
|
|
return "", err
|
|
}
|
|
|
|
readCloser.Close()
|
|
return output.FileID, nil
|
|
}
|
|
|
|
func (client *B2Client) DownloadFile(filePath string) (io.ReadCloser, int64, error) {
|
|
|
|
url := client.DownloadURL + "/file/" + client.BucketName + "/" + filePath
|
|
|
|
return client.call(url, 0)
|
|
}
|
|
|
|
type B2GetUploadArgumentOutput struct {
|
|
BucketID string
|
|
UploadURL string
|
|
AuthorizationToken string
|
|
}
|
|
|
|
func (client *B2Client) getUploadURL() (error) {
|
|
input := make(map[string]string)
|
|
input["bucketId"] = client.BucketID
|
|
|
|
url := client.APIURL + "/b2api/v1/b2_get_upload_url"
|
|
readCloser, _, err := client.call(url, input)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
defer readCloser.Close()
|
|
|
|
output := & B2GetUploadArgumentOutput {}
|
|
|
|
if err = json.NewDecoder(readCloser).Decode(&output); err != nil {
|
|
return err
|
|
}
|
|
|
|
client.UploadURL = output.UploadURL
|
|
client.UploadToken = output.AuthorizationToken
|
|
|
|
return nil
|
|
}
|
|
|
|
func (client *B2Client) UploadFile(filePath string, content []byte, rateLimit int) (err error) {
|
|
|
|
|
|
hasher := sha1.New()
|
|
hasher.Write(content)
|
|
hash := hex.EncodeToString(hasher.Sum(nil))
|
|
|
|
headers := make(map[string]string)
|
|
headers["X-Bz-File-Name"] = filePath
|
|
headers["Content-Type"] = "application/octet-stream"
|
|
headers["X-Bz-Content-Sha1"] = hash
|
|
|
|
var response *http.Response
|
|
|
|
backoff := 0
|
|
for i := 0; i < 8; i++ {
|
|
|
|
if client.UploadURL == "" || client.UploadToken == "" {
|
|
err = client.getUploadURL()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
request, err := http.NewRequest("POST", client.UploadURL, CreateRateLimitedReader(content, rateLimit))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
request.ContentLength = int64(len(content))
|
|
|
|
request.Header.Set("Authorization", client.UploadToken)
|
|
request.Header.Set("X-Bz-File-Name", filePath)
|
|
request.Header.Set("Content-Type", "application/octet-stream")
|
|
request.Header.Set("X-Bz-Content-Sha1", hash)
|
|
|
|
for key, value := range headers {
|
|
request.Header.Set(key, value)
|
|
}
|
|
|
|
if client.TestMode {
|
|
r := rand.Float32()
|
|
if r < 0.8 {
|
|
request.Header.Set("X-Bz-Test-Mode", "fail_some_uploads")
|
|
} else if r < 0.9 {
|
|
request.Header.Set("X-Bz-Test-Mode", "expire_some_account_authorization_tokens")
|
|
} else {
|
|
request.Header.Set("X-Bz-Test-Mode", "force_cap_exceeded")
|
|
}
|
|
}
|
|
|
|
response, err = client.HTTPClient.Do(request)
|
|
if err != nil {
|
|
LOG_DEBUG("BACKBLAZE_UPLOAD", "URL request '%s' returned an error: %v", client.UploadURL, err)
|
|
backoff = client.retry(backoff, response)
|
|
client.UploadURL = ""
|
|
client.UploadToken = ""
|
|
continue
|
|
}
|
|
|
|
io.Copy(ioutil.Discard, response.Body)
|
|
response.Body.Close()
|
|
|
|
if response.StatusCode < 400 {
|
|
return nil
|
|
}
|
|
|
|
LOG_DEBUG("BACKBLAZE_UPLOAD", "URL request '%s' returned status code %d", client.UploadURL, response.StatusCode)
|
|
|
|
if response.StatusCode == 401 {
|
|
LOG_INFO("BACKBLAZE_UPLOAD", "Re-authorizatoin required")
|
|
client.UploadURL = ""
|
|
client.UploadToken = ""
|
|
continue
|
|
} else if response.StatusCode == 403 {
|
|
if !client.TestMode {
|
|
return fmt.Errorf("B2 cap exceeded")
|
|
}
|
|
continue
|
|
} else {
|
|
LOG_INFO("BACKBLAZE_UPLOAD", "URL request '%s' returned status code %d", client.UploadURL, response.StatusCode)
|
|
backoff = client.retry(backoff, response)
|
|
client.UploadURL = ""
|
|
client.UploadToken = ""
|
|
}
|
|
}
|
|
|
|
return fmt.Errorf("Maximum backoff reached")
|
|
}
|
|
|