mirror of
https://github.com/gilbertchen/duplicacy
synced 2025-12-19 09:43:17 +00:00
Run goimports on all source files
This commit is contained in:
Gilbert Chen
@@ -5,208 +5,209 @@
|
||||
package duplicacy
|
||||
|
||||
import (
|
||||
"time"
|
||||
"github.com/gilbertchen/goamz/aws"
|
||||
"github.com/gilbertchen/goamz/s3"
|
||||
"time"
|
||||
|
||||
"github.com/gilbertchen/goamz/aws"
|
||||
"github.com/gilbertchen/goamz/s3"
|
||||
)
|
||||
|
||||
// S3CStorage is a storage backend for s3 compatible storages that require V2 Signing.
|
||||
type S3CStorage struct {
|
||||
RateLimitedStorage
|
||||
RateLimitedStorage
|
||||
|
||||
buckets []*s3.Bucket
|
||||
storageDir string
|
||||
buckets []*s3.Bucket
|
||||
storageDir string
|
||||
}
|
||||
|
||||
// CreateS3CStorage creates a amazon s3 storage object.
|
||||
func CreateS3CStorage(regionName string, endpoint string, bucketName string, storageDir string,
|
||||
accessKey string, secretKey string, threads int) (storage *S3CStorage, err error) {
|
||||
accessKey string, secretKey string, threads int) (storage *S3CStorage, err error) {
|
||||
|
||||
var region aws.Region
|
||||
var region aws.Region
|
||||
|
||||
if endpoint == "" {
|
||||
if regionName == "" {
|
||||
regionName = "us-east-1"
|
||||
}
|
||||
region = aws.Regions[regionName]
|
||||
} else {
|
||||
region = aws.Region { Name: regionName, S3Endpoint:"https://" + endpoint }
|
||||
}
|
||||
if endpoint == "" {
|
||||
if regionName == "" {
|
||||
regionName = "us-east-1"
|
||||
}
|
||||
region = aws.Regions[regionName]
|
||||
} else {
|
||||
region = aws.Region{Name: regionName, S3Endpoint: "https://" + endpoint}
|
||||
}
|
||||
|
||||
auth := aws.Auth{ AccessKey: accessKey, SecretKey: secretKey }
|
||||
auth := aws.Auth{AccessKey: accessKey, SecretKey: secretKey}
|
||||
|
||||
var buckets []*s3.Bucket
|
||||
for i := 0; i < threads; i++ {
|
||||
s3Client := s3.New(auth, region)
|
||||
s3Client.AttemptStrategy = aws.AttemptStrategy{
|
||||
Min: 8,
|
||||
Total: 300 * time.Second,
|
||||
Delay: 1000 * time.Millisecond,
|
||||
}
|
||||
var buckets []*s3.Bucket
|
||||
for i := 0; i < threads; i++ {
|
||||
s3Client := s3.New(auth, region)
|
||||
s3Client.AttemptStrategy = aws.AttemptStrategy{
|
||||
Min: 8,
|
||||
Total: 300 * time.Second,
|
||||
Delay: 1000 * time.Millisecond,
|
||||
}
|
||||
|
||||
bucket := s3Client.Bucket(bucketName)
|
||||
buckets = append(buckets, bucket)
|
||||
}
|
||||
bucket := s3Client.Bucket(bucketName)
|
||||
buckets = append(buckets, bucket)
|
||||
}
|
||||
|
||||
if len(storageDir) > 0 && storageDir[len(storageDir) - 1] != '/' {
|
||||
storageDir += "/"
|
||||
}
|
||||
if len(storageDir) > 0 && storageDir[len(storageDir)-1] != '/' {
|
||||
storageDir += "/"
|
||||
}
|
||||
|
||||
storage = &S3CStorage {
|
||||
buckets: buckets,
|
||||
storageDir: storageDir,
|
||||
}
|
||||
storage = &S3CStorage{
|
||||
buckets: buckets,
|
||||
storageDir: storageDir,
|
||||
}
|
||||
|
||||
return storage, nil
|
||||
return storage, nil
|
||||
}
|
||||
|
||||
// ListFiles return the list of files and subdirectories under 'dir' (non-recursively)
|
||||
func (storage *S3CStorage) ListFiles(threadIndex int, dir string) (files []string, sizes []int64, err error) {
|
||||
if len(dir) > 0 && dir[len(dir) - 1] != '/' {
|
||||
dir += "/"
|
||||
}
|
||||
if len(dir) > 0 && dir[len(dir)-1] != '/' {
|
||||
dir += "/"
|
||||
}
|
||||
|
||||
dirLength := len(storage.storageDir + dir)
|
||||
if dir == "snapshots/" {
|
||||
results, err := storage.buckets[threadIndex].List(storage.storageDir + dir, "/", "", 100)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
dirLength := len(storage.storageDir + dir)
|
||||
if dir == "snapshots/" {
|
||||
results, err := storage.buckets[threadIndex].List(storage.storageDir+dir, "/", "", 100)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
for _, subDir := range results.CommonPrefixes {
|
||||
files = append(files, subDir[dirLength:])
|
||||
}
|
||||
return files, nil, nil
|
||||
} else if dir == "chunks/" {
|
||||
marker := ""
|
||||
for {
|
||||
results, err := storage.buckets[threadIndex].List(storage.storageDir + dir, "", marker, 1000)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
for _, subDir := range results.CommonPrefixes {
|
||||
files = append(files, subDir[dirLength:])
|
||||
}
|
||||
return files, nil, nil
|
||||
} else if dir == "chunks/" {
|
||||
marker := ""
|
||||
for {
|
||||
results, err := storage.buckets[threadIndex].List(storage.storageDir+dir, "", marker, 1000)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
for _, object := range results.Contents {
|
||||
files = append(files, object.Key[dirLength:])
|
||||
sizes = append(sizes, object.Size)
|
||||
}
|
||||
for _, object := range results.Contents {
|
||||
files = append(files, object.Key[dirLength:])
|
||||
sizes = append(sizes, object.Size)
|
||||
}
|
||||
|
||||
if !results.IsTruncated {
|
||||
break
|
||||
}
|
||||
if !results.IsTruncated {
|
||||
break
|
||||
}
|
||||
|
||||
marker = results.Contents[len(results.Contents) - 1].Key
|
||||
}
|
||||
return files, sizes, nil
|
||||
marker = results.Contents[len(results.Contents)-1].Key
|
||||
}
|
||||
return files, sizes, nil
|
||||
|
||||
} else {
|
||||
} else {
|
||||
|
||||
results, err := storage.buckets[threadIndex].List(storage.storageDir + dir, "", "", 1000)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
results, err := storage.buckets[threadIndex].List(storage.storageDir+dir, "", "", 1000)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
for _, object := range results.Contents {
|
||||
files = append(files, object.Key[dirLength:])
|
||||
}
|
||||
return files, nil, nil
|
||||
}
|
||||
for _, object := range results.Contents {
|
||||
files = append(files, object.Key[dirLength:])
|
||||
}
|
||||
return files, nil, nil
|
||||
}
|
||||
}
|
||||
|
||||
// DeleteFile deletes the file or directory at 'filePath'.
|
||||
func (storage *S3CStorage) DeleteFile(threadIndex int, filePath string) (err error) {
|
||||
return storage.buckets[threadIndex].Del(storage.storageDir + filePath)
|
||||
return storage.buckets[threadIndex].Del(storage.storageDir + filePath)
|
||||
}
|
||||
|
||||
// MoveFile renames the file.
|
||||
func (storage *S3CStorage) MoveFile(threadIndex int, from string, to string) (err error) {
|
||||
|
||||
options := s3.CopyOptions { ContentType: "application/duplicacy" }
|
||||
_, err = storage.buckets[threadIndex].PutCopy(storage.storageDir + to, s3.Private, options, storage.buckets[threadIndex].Name + "/" + storage.storageDir + from)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
options := s3.CopyOptions{ContentType: "application/duplicacy"}
|
||||
_, err = storage.buckets[threadIndex].PutCopy(storage.storageDir+to, s3.Private, options, storage.buckets[threadIndex].Name+"/"+storage.storageDir+from)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
return storage.DeleteFile(threadIndex, from)
|
||||
return storage.DeleteFile(threadIndex, from)
|
||||
}
|
||||
|
||||
// CreateDirectory creates a new directory.
|
||||
func (storage *S3CStorage) CreateDirectory(threadIndex int, dir string) (err error) {
|
||||
return nil
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetFileInfo returns the information about the file or directory at 'filePath'.
|
||||
func (storage *S3CStorage) GetFileInfo(threadIndex int, filePath string) (exist bool, isDir bool, size int64, err error) {
|
||||
|
||||
response, err := storage.buckets[threadIndex].Head(storage.storageDir + filePath, nil)
|
||||
if err != nil {
|
||||
if e, ok := err.(*s3.Error); ok && (e.StatusCode == 403 || e.StatusCode == 404) {
|
||||
return false, false, 0, nil
|
||||
} else {
|
||||
return false, false, 0, err
|
||||
}
|
||||
}
|
||||
response, err := storage.buckets[threadIndex].Head(storage.storageDir+filePath, nil)
|
||||
if err != nil {
|
||||
if e, ok := err.(*s3.Error); ok && (e.StatusCode == 403 || e.StatusCode == 404) {
|
||||
return false, false, 0, nil
|
||||
} else {
|
||||
return false, false, 0, err
|
||||
}
|
||||
}
|
||||
|
||||
if response.StatusCode == 403 || response.StatusCode == 404 {
|
||||
return false, false, 0, nil
|
||||
} else {
|
||||
return true, false, response.ContentLength, nil
|
||||
}
|
||||
if response.StatusCode == 403 || response.StatusCode == 404 {
|
||||
return false, false, 0, nil
|
||||
} else {
|
||||
return true, false, response.ContentLength, nil
|
||||
}
|
||||
}
|
||||
|
||||
// FindChunk finds the chunk with the specified id. If 'isFossil' is true, it will search for chunk files with
|
||||
// the suffix '.fsl'.
|
||||
func (storage *S3CStorage) FindChunk(threadIndex int, chunkID string, isFossil bool) (filePath string, exist bool, size int64, err error) {
|
||||
|
||||
filePath = "chunks/" + chunkID
|
||||
if isFossil {
|
||||
filePath += ".fsl"
|
||||
}
|
||||
filePath = "chunks/" + chunkID
|
||||
if isFossil {
|
||||
filePath += ".fsl"
|
||||
}
|
||||
|
||||
exist, _, size, err = storage.GetFileInfo(threadIndex, filePath)
|
||||
exist, _, size, err = storage.GetFileInfo(threadIndex, filePath)
|
||||
|
||||
if err != nil {
|
||||
return "", false, 0, err
|
||||
} else {
|
||||
return filePath, exist, size, err
|
||||
}
|
||||
if err != nil {
|
||||
return "", false, 0, err
|
||||
} else {
|
||||
return filePath, exist, size, err
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// DownloadFile reads the file at 'filePath' into the chunk.
|
||||
func (storage *S3CStorage) DownloadFile(threadIndex int, filePath string, chunk *Chunk) (err error) {
|
||||
|
||||
readCloser, err := storage.buckets[threadIndex].GetReader(storage.storageDir + filePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
readCloser, err := storage.buckets[threadIndex].GetReader(storage.storageDir + filePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer readCloser.Close()
|
||||
defer readCloser.Close()
|
||||
|
||||
_, err = RateLimitedCopy(chunk, readCloser, storage.DownloadRateLimit / len(storage.buckets))
|
||||
return err
|
||||
_, err = RateLimitedCopy(chunk, readCloser, storage.DownloadRateLimit/len(storage.buckets))
|
||||
return err
|
||||
|
||||
}
|
||||
|
||||
// UploadFile writes 'content' to the file at 'filePath'.
|
||||
func (storage *S3CStorage) UploadFile(threadIndex int, filePath string, content []byte) (err error) {
|
||||
|
||||
options := s3.Options { }
|
||||
reader := CreateRateLimitedReader(content, storage.UploadRateLimit / len(storage.buckets))
|
||||
return storage.buckets[threadIndex].PutReader(storage.storageDir + filePath, reader, int64(len(content)), "application/duplicacy", s3.Private, options)
|
||||
options := s3.Options{}
|
||||
reader := CreateRateLimitedReader(content, storage.UploadRateLimit/len(storage.buckets))
|
||||
return storage.buckets[threadIndex].PutReader(storage.storageDir+filePath, reader, int64(len(content)), "application/duplicacy", s3.Private, options)
|
||||
}
|
||||
|
||||
// If a local snapshot cache is needed for the storage to avoid downloading/uploading chunks too often when
|
||||
// managing snapshots.
|
||||
func (storage *S3CStorage) IsCacheNeeded () (bool) { return true }
|
||||
func (storage *S3CStorage) IsCacheNeeded() bool { return true }
|
||||
|
||||
// If the 'MoveFile' method is implemented.
|
||||
func (storage *S3CStorage) IsMoveFileImplemented() (bool) { return true }
|
||||
func (storage *S3CStorage) IsMoveFileImplemented() bool { return true }
|
||||
|
||||
// If the storage can guarantee strong consistency.
|
||||
func (storage *S3CStorage) IsStrongConsistent() (bool) { return false }
|
||||
func (storage *S3CStorage) IsStrongConsistent() bool { return false }
|
||||
|
||||
// If the storage supports fast listing of files names.
|
||||
func (storage *S3CStorage) IsFastListing() (bool) { return true }
|
||||
func (storage *S3CStorage) IsFastListing() bool { return true }
|
||||
|
||||
// Enable the test mode.
|
||||
func (storage *S3CStorage) EnableTestMode() {}
|
||||
|
||||
Reference in New Issue
Block a user