mirror of
https://github.com/gilbertchen/duplicacy
synced 2025-12-15 15:53:26 +00:00
601 lines
15 KiB
Go
601 lines
15 KiB
Go
// Copyright (c) Acrosync LLC. All rights reserved.
|
|
// Free for personal use and commercial trial
|
|
// Commercial use requires per-user licenses available from https://duplicacy.com
|
|
|
|
package duplicacy
|
|
|
|
import (
|
|
"bytes"
|
|
"crypto/sha1"
|
|
"encoding/base64"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"io/ioutil"
|
|
"math/rand"
|
|
"net/http"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
type B2Error struct {
|
|
Status int
|
|
Code string
|
|
Message string
|
|
}
|
|
|
|
func (err *B2Error) Error() string {
|
|
return fmt.Sprintf("%d %s", err.Status, err.Message)
|
|
}
|
|
|
|
type B2UploadArgument struct {
|
|
URL string
|
|
Token string
|
|
}
|
|
|
|
var B2AuthorizationURL = "https://api.backblazeb2.com/b2api/v1/b2_authorize_account"
|
|
|
|
type B2Client struct {
|
|
HTTPClient *http.Client
|
|
AccountID string
|
|
ApplicationKey string
|
|
AuthorizationToken string
|
|
APIURL string
|
|
DownloadURL string
|
|
BucketName string
|
|
BucketID string
|
|
|
|
UploadURL string
|
|
UploadToken string
|
|
|
|
TestMode bool
|
|
}
|
|
|
|
func NewB2Client(accountID string, applicationKey string) *B2Client {
|
|
client := &B2Client{
|
|
HTTPClient: http.DefaultClient,
|
|
AccountID: accountID,
|
|
ApplicationKey: applicationKey,
|
|
}
|
|
return client
|
|
}
|
|
|
|
func (client *B2Client) retry(backoff int, response *http.Response) int {
|
|
if response != nil {
|
|
if backoffList, found := response.Header["Retry-After"]; found && len(backoffList) > 0 {
|
|
retryAfter, _ := strconv.Atoi(backoffList[0])
|
|
if retryAfter >= 1 {
|
|
time.Sleep(time.Duration(retryAfter) * time.Second)
|
|
return 0
|
|
}
|
|
}
|
|
}
|
|
if backoff == 0 {
|
|
backoff = 1
|
|
} else {
|
|
backoff *= 2
|
|
}
|
|
time.Sleep(time.Duration(backoff) * time.Second)
|
|
return backoff
|
|
}
|
|
|
|
func (client *B2Client) call(url string, method string, requestHeaders map[string]string, input interface{}) (io.ReadCloser, http.Header, int64, error) {
|
|
|
|
switch method {
|
|
case http.MethodGet:
|
|
break
|
|
case http.MethodHead:
|
|
break
|
|
case http.MethodPost:
|
|
break
|
|
default:
|
|
return nil, nil, 0, fmt.Errorf("unhandled http request method: " + method)
|
|
}
|
|
|
|
var response *http.Response
|
|
|
|
backoff := 0
|
|
for i := 0; i < 8; i++ {
|
|
var inputReader *bytes.Reader
|
|
|
|
switch input.(type) {
|
|
default:
|
|
jsonInput, err := json.Marshal(input)
|
|
if err != nil {
|
|
return nil, nil, 0, err
|
|
}
|
|
inputReader = bytes.NewReader(jsonInput)
|
|
case []byte:
|
|
inputReader = bytes.NewReader(input.([]byte))
|
|
case int:
|
|
inputReader = bytes.NewReader([]byte(""))
|
|
}
|
|
|
|
request, err := http.NewRequest(method, url, inputReader)
|
|
if err != nil {
|
|
return nil, nil, 0, err
|
|
}
|
|
|
|
if url == B2AuthorizationURL {
|
|
request.Header.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(client.AccountID+":"+client.ApplicationKey)))
|
|
} else {
|
|
request.Header.Set("Authorization", client.AuthorizationToken)
|
|
}
|
|
|
|
if requestHeaders != nil {
|
|
for key, value := range requestHeaders {
|
|
request.Header.Set(key, value)
|
|
}
|
|
}
|
|
|
|
if client.TestMode {
|
|
r := rand.Float32()
|
|
if r < 0.5 {
|
|
request.Header.Set("X-Bz-Test-Mode", "expire_some_account_authorization_tokens")
|
|
} else {
|
|
request.Header.Set("X-Bz-Test-Mode", "force_cap_exceeded")
|
|
}
|
|
}
|
|
|
|
response, err = client.HTTPClient.Do(request)
|
|
if err != nil {
|
|
if url != B2AuthorizationURL {
|
|
LOG_DEBUG("BACKBLAZE_CALL", "URL request '%s' returned an error: %v", url, err)
|
|
backoff = client.retry(backoff, response)
|
|
continue
|
|
}
|
|
return nil, nil, 0, err
|
|
}
|
|
|
|
if response.StatusCode < 300 {
|
|
return response.Body, response.Header, response.ContentLength, nil
|
|
}
|
|
|
|
LOG_DEBUG("BACKBLAZE_CALL", "URL request '%s %s' returned status code %d", method, url, response.StatusCode)
|
|
|
|
io.Copy(ioutil.Discard, response.Body)
|
|
response.Body.Close()
|
|
if response.StatusCode == 401 {
|
|
if url == B2AuthorizationURL {
|
|
return nil, nil, 0, fmt.Errorf("Authorization failure")
|
|
}
|
|
client.AuthorizeAccount()
|
|
continue
|
|
} else if response.StatusCode == 403 {
|
|
if !client.TestMode {
|
|
return nil, nil, 0, fmt.Errorf("B2 cap exceeded")
|
|
}
|
|
continue
|
|
} else if response.StatusCode == 404 {
|
|
if http.MethodHead == method {
|
|
return nil, nil, 0, nil
|
|
}
|
|
} else if response.StatusCode == 416 {
|
|
if http.MethodHead == method {
|
|
// 416 Requested Range Not Satisfiable
|
|
return nil, nil, 0, fmt.Errorf("URL request '%s' returned status code %d", url, response.StatusCode)
|
|
}
|
|
} else if response.StatusCode == 429 || response.StatusCode == 408 {
|
|
backoff = client.retry(backoff, response)
|
|
continue
|
|
} else if response.StatusCode >= 500 && response.StatusCode <= 599 {
|
|
backoff = client.retry(backoff, response)
|
|
continue
|
|
} else {
|
|
LOG_INFO("BACKBLAZE_CALL", "URL request '%s' returned status code %d", url, response.StatusCode)
|
|
backoff = client.retry(backoff, response)
|
|
continue
|
|
}
|
|
|
|
defer response.Body.Close()
|
|
|
|
e := &B2Error{}
|
|
|
|
if err := json.NewDecoder(response.Body).Decode(e); err != nil {
|
|
return nil, nil, 0, err
|
|
}
|
|
|
|
return nil, nil, 0, e
|
|
}
|
|
|
|
return nil, nil, 0, fmt.Errorf("Maximum backoff reached")
|
|
}
|
|
|
|
type B2AuthorizeAccountOutput struct {
|
|
AccountID string
|
|
AuthorizationToken string
|
|
APIURL string
|
|
DownloadURL string
|
|
}
|
|
|
|
func (client *B2Client) AuthorizeAccount() (err error) {
|
|
|
|
readCloser, _, _, err := client.call(B2AuthorizationURL, http.MethodPost, nil, make(map[string]string))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
defer readCloser.Close()
|
|
|
|
output := &B2AuthorizeAccountOutput{}
|
|
|
|
if err = json.NewDecoder(readCloser).Decode(&output); err != nil {
|
|
return err
|
|
}
|
|
|
|
client.AuthorizationToken = output.AuthorizationToken
|
|
client.APIURL = output.APIURL
|
|
client.DownloadURL = output.DownloadURL
|
|
|
|
return nil
|
|
}
|
|
|
|
type ListBucketOutput struct {
|
|
AccoundID string
|
|
BucketID string
|
|
BucketName string
|
|
BucketType string
|
|
}
|
|
|
|
func (client *B2Client) FindBucket(bucketName string) (err error) {
|
|
|
|
input := make(map[string]string)
|
|
input["accountId"] = client.AccountID
|
|
|
|
url := client.APIURL + "/b2api/v1/b2_list_buckets"
|
|
|
|
readCloser, _, _, err := client.call(url, http.MethodPost, nil, input)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
defer readCloser.Close()
|
|
|
|
output := make(map[string][]ListBucketOutput, 0)
|
|
|
|
if err = json.NewDecoder(readCloser).Decode(&output); err != nil {
|
|
return err
|
|
}
|
|
|
|
for _, bucket := range output["buckets"] {
|
|
if bucket.BucketName == bucketName {
|
|
client.BucketName = bucket.BucketName
|
|
client.BucketID = bucket.BucketID
|
|
break
|
|
}
|
|
}
|
|
|
|
if client.BucketID == "" {
|
|
return fmt.Errorf("Bucket %s not found", bucketName)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
type B2Entry struct {
|
|
FileID string
|
|
FileName string
|
|
Action string
|
|
Size int64
|
|
UploadTimestamp int64
|
|
}
|
|
|
|
type B2ListFileNamesOutput struct {
|
|
Files []*B2Entry
|
|
NextFileName string
|
|
NextFileId string
|
|
}
|
|
|
|
func (client *B2Client) ListFileNames(startFileName string, singleFile bool, includeVersions bool) (files []*B2Entry, err error) {
|
|
|
|
maxFileCount := 1000
|
|
if singleFile {
|
|
if includeVersions {
|
|
maxFileCount = 4
|
|
if client.TestMode {
|
|
maxFileCount = 1
|
|
}
|
|
} else {
|
|
maxFileCount = 1
|
|
}
|
|
} else if client.TestMode {
|
|
maxFileCount = 10
|
|
}
|
|
|
|
input := make(map[string]interface{})
|
|
input["bucketId"] = client.BucketID
|
|
input["startFileName"] = startFileName
|
|
input["maxFileCount"] = maxFileCount
|
|
|
|
for {
|
|
url := client.APIURL + "/b2api/v1/b2_list_file_names"
|
|
requestHeaders := map[string]string{}
|
|
requestMethod := http.MethodPost
|
|
var requestInput interface{}
|
|
requestInput = input
|
|
if includeVersions {
|
|
url = client.APIURL + "/b2api/v1/b2_list_file_versions"
|
|
} else if singleFile {
|
|
// handle a single file with no versions as a special case to download the last byte of the file
|
|
url = client.DownloadURL + "/file/" + client.BucketName + "/" + startFileName
|
|
// requesting byte -1 works for empty files where 0-0 fails with a 416 error
|
|
requestHeaders["Range"] = "bytes=-1"
|
|
// HEAD request
|
|
requestMethod = http.MethodHead
|
|
requestInput = 0
|
|
}
|
|
var readCloser io.ReadCloser
|
|
var responseHeader http.Header
|
|
var err error
|
|
readCloser, responseHeader, _, err = client.call(url, requestMethod, requestHeaders, requestInput)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if readCloser != nil {
|
|
defer readCloser.Close()
|
|
}
|
|
|
|
output := B2ListFileNamesOutput{}
|
|
|
|
if singleFile && !includeVersions {
|
|
if responseHeader == nil {
|
|
LOG_DEBUG("BACKBLAZE_LIST", "b2_download_file_by_name did not return headers")
|
|
return []*B2Entry{}, nil
|
|
}
|
|
requiredHeaders := []string{
|
|
"x-bz-file-id",
|
|
"x-bz-file-name",
|
|
}
|
|
missingKeys := []string{}
|
|
for _, headerKey := range requiredHeaders {
|
|
if "" == responseHeader.Get(headerKey) {
|
|
missingKeys = append(missingKeys, headerKey)
|
|
}
|
|
}
|
|
if len(missingKeys) > 0 {
|
|
return nil, fmt.Errorf("b2_download_file_by_name missing headers: %s", missingKeys)
|
|
}
|
|
// construct the B2Entry from the response headers of the download request
|
|
fileID := responseHeader.Get("x-bz-file-id")
|
|
fileName := responseHeader.Get("x-bz-file-name")
|
|
fileAction := "upload"
|
|
// byte range that is returned: "bytes #-#/#
|
|
rangeString := responseHeader.Get("Content-Range")
|
|
// total file size; 1 if file has content, 0 if it's empty
|
|
lengthString := responseHeader.Get("Content-Length")
|
|
var fileSize int64
|
|
if "" != rangeString {
|
|
fileSize, _ = strconv.ParseInt(rangeString[strings.Index(rangeString, "/")+1:], 0, 64)
|
|
} else if "" != lengthString {
|
|
// this should only execute if the requested file is empty and the range request didn't result in a Content-Range header
|
|
fileSize, _ = strconv.ParseInt(lengthString, 0, 64)
|
|
if fileSize != 0 {
|
|
return nil, fmt.Errorf("b2_download_file_by_name returned non-zero file length")
|
|
}
|
|
} else {
|
|
return nil, fmt.Errorf("could not parse b2_download_file_by_name headers")
|
|
}
|
|
fileUploadTimestamp, _ := strconv.ParseInt(responseHeader.Get("X-Bz-Upload-Timestamp"), 0, 64)
|
|
|
|
return []*B2Entry{&B2Entry{fileID, fileName, fileAction, fileSize, fileUploadTimestamp}}, nil
|
|
}
|
|
|
|
if err = json.NewDecoder(readCloser).Decode(&output); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
ioutil.ReadAll(readCloser)
|
|
|
|
if startFileName == "" {
|
|
files = append(files, output.Files...)
|
|
} else {
|
|
for _, file := range output.Files {
|
|
if singleFile {
|
|
if file.FileName == startFileName {
|
|
files = append(files, file)
|
|
if !includeVersions {
|
|
output.NextFileName = ""
|
|
break
|
|
}
|
|
} else {
|
|
output.NextFileName = ""
|
|
break
|
|
}
|
|
} else {
|
|
if strings.HasPrefix(file.FileName, startFileName) {
|
|
files = append(files, file)
|
|
} else {
|
|
output.NextFileName = ""
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
if len(output.NextFileName) == 0 {
|
|
break
|
|
}
|
|
|
|
input["startFileName"] = output.NextFileName
|
|
if includeVersions {
|
|
input["startFileId"] = output.NextFileId
|
|
}
|
|
}
|
|
|
|
return files, nil
|
|
}
|
|
|
|
func (client *B2Client) DeleteFile(fileName string, fileID string) (err error) {
|
|
|
|
input := make(map[string]string)
|
|
input["fileName"] = fileName
|
|
input["fileId"] = fileID
|
|
|
|
url := client.APIURL + "/b2api/v1/b2_delete_file_version"
|
|
readCloser, _, _, err := client.call(url, http.MethodPost, make(map[string]string), input)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
readCloser.Close()
|
|
return nil
|
|
}
|
|
|
|
type B2HideFileOutput struct {
|
|
FileID string
|
|
}
|
|
|
|
func (client *B2Client) HideFile(fileName string) (fileID string, err error) {
|
|
|
|
input := make(map[string]string)
|
|
input["bucketId"] = client.BucketID
|
|
input["fileName"] = fileName
|
|
|
|
url := client.APIURL + "/b2api/v1/b2_hide_file"
|
|
readCloser, _, _, err := client.call(url, http.MethodPost, make(map[string]string), input)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
defer readCloser.Close()
|
|
|
|
output := &B2HideFileOutput{}
|
|
|
|
if err = json.NewDecoder(readCloser).Decode(&output); err != nil {
|
|
return "", err
|
|
}
|
|
|
|
readCloser.Close()
|
|
return output.FileID, nil
|
|
}
|
|
|
|
func (client *B2Client) DownloadFile(filePath string) (io.ReadCloser, int64, error) {
|
|
|
|
url := client.DownloadURL + "/file/" + client.BucketName + "/" + filePath
|
|
|
|
readCloser, _, len, err := client.call(url, http.MethodGet, make(map[string]string), 0)
|
|
return readCloser, len, err
|
|
}
|
|
|
|
type B2GetUploadArgumentOutput struct {
|
|
BucketID string
|
|
UploadURL string
|
|
AuthorizationToken string
|
|
}
|
|
|
|
func (client *B2Client) getUploadURL() error {
|
|
input := make(map[string]string)
|
|
input["bucketId"] = client.BucketID
|
|
|
|
url := client.APIURL + "/b2api/v1/b2_get_upload_url"
|
|
readCloser, _, _, err := client.call(url, http.MethodPost, make(map[string]string), input)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
defer readCloser.Close()
|
|
|
|
output := &B2GetUploadArgumentOutput{}
|
|
|
|
if err = json.NewDecoder(readCloser).Decode(&output); err != nil {
|
|
return err
|
|
}
|
|
|
|
client.UploadURL = output.UploadURL
|
|
client.UploadToken = output.AuthorizationToken
|
|
|
|
return nil
|
|
}
|
|
|
|
func (client *B2Client) UploadFile(filePath string, content []byte, rateLimit int) (err error) {
|
|
|
|
hasher := sha1.New()
|
|
hasher.Write(content)
|
|
hash := hex.EncodeToString(hasher.Sum(nil))
|
|
|
|
headers := make(map[string]string)
|
|
headers["X-Bz-File-Name"] = filePath
|
|
headers["Content-Type"] = "application/octet-stream"
|
|
headers["X-Bz-Content-Sha1"] = hash
|
|
|
|
var response *http.Response
|
|
|
|
backoff := 0
|
|
for i := 0; i < 8; i++ {
|
|
|
|
if client.UploadURL == "" || client.UploadToken == "" {
|
|
err = client.getUploadURL()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
request, err := http.NewRequest("POST", client.UploadURL, CreateRateLimitedReader(content, rateLimit))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
request.ContentLength = int64(len(content))
|
|
|
|
request.Header.Set("Authorization", client.UploadToken)
|
|
request.Header.Set("X-Bz-File-Name", filePath)
|
|
request.Header.Set("Content-Type", "application/octet-stream")
|
|
request.Header.Set("X-Bz-Content-Sha1", hash)
|
|
|
|
for key, value := range headers {
|
|
request.Header.Set(key, value)
|
|
}
|
|
|
|
if client.TestMode {
|
|
r := rand.Float32()
|
|
if r < 0.8 {
|
|
request.Header.Set("X-Bz-Test-Mode", "fail_some_uploads")
|
|
} else if r < 0.9 {
|
|
request.Header.Set("X-Bz-Test-Mode", "expire_some_account_authorization_tokens")
|
|
} else {
|
|
request.Header.Set("X-Bz-Test-Mode", "force_cap_exceeded")
|
|
}
|
|
}
|
|
|
|
response, err = client.HTTPClient.Do(request)
|
|
if err != nil {
|
|
LOG_DEBUG("BACKBLAZE_UPLOAD", "URL request '%s' returned an error: %v", client.UploadURL, err)
|
|
backoff = client.retry(backoff, response)
|
|
client.UploadURL = ""
|
|
client.UploadToken = ""
|
|
continue
|
|
}
|
|
|
|
io.Copy(ioutil.Discard, response.Body)
|
|
response.Body.Close()
|
|
|
|
if response.StatusCode < 300 {
|
|
return nil
|
|
}
|
|
|
|
LOG_DEBUG("BACKBLAZE_UPLOAD", "URL request '%s' returned status code %d", client.UploadURL, response.StatusCode)
|
|
|
|
if response.StatusCode == 401 {
|
|
LOG_INFO("BACKBLAZE_UPLOAD", "Re-authorization required")
|
|
client.UploadURL = ""
|
|
client.UploadToken = ""
|
|
continue
|
|
} else if response.StatusCode == 403 {
|
|
if !client.TestMode {
|
|
return fmt.Errorf("B2 cap exceeded")
|
|
}
|
|
continue
|
|
} else {
|
|
LOG_INFO("BACKBLAZE_UPLOAD", "URL request '%s' returned status code %d", client.UploadURL, response.StatusCode)
|
|
backoff = client.retry(backoff, response)
|
|
client.UploadURL = ""
|
|
client.UploadToken = ""
|
|
}
|
|
}
|
|
|
|
return fmt.Errorf("Maximum backoff reached")
|
|
}
|