duplicacy/src/duplicacy_s3cstorage.go

// Copyright (c) Acrosync LLC. All rights reserved.
// Free for personal use and commercial trial
// Commercial use requires per-user licenses available from https://duplicacy.com

package duplicacy

import (
	"time"

	"github.com/gilbertchen/goamz/aws"
	"github.com/gilbertchen/goamz/s3"
)

// S3CStorage is a storage backend for s3 compatible storages that require V2 Signing.
type S3CStorage struct {
	StorageBase

	buckets    []*s3.Bucket
	storageDir string
}

// CreateS3CStorage creates a amazon s3 storage object.
func CreateS3CStorage(regionName string, endpoint string, bucketName string, storageDir string,
	accessKey string, secretKey string, threads int) (storage *S3CStorage, err error) {

	var region aws.Region

	if endpoint == "" {
		if regionName == "" {
			regionName = "us-east-1"
		}
		region = aws.Regions[regionName]
	} else {
		region = aws.Region{Name: regionName, S3Endpoint: "https://" + endpoint}
	}

	auth := aws.Auth{AccessKey: accessKey, SecretKey: secretKey}

	var buckets []*s3.Bucket
	for i := 0; i < threads; i++ {
		s3Client := s3.New(auth, region)
		s3Client.AttemptStrategy = aws.AttemptStrategy{
			Min:   8,
			Total: 300 * time.Second,
			Delay: 1000 * time.Millisecond,
		}

		bucket := s3Client.Bucket(bucketName)
		buckets = append(buckets, bucket)
	}

	if len(storageDir) > 0 && storageDir[len(storageDir)-1] != '/' {
		storageDir += "/"
	}

	storage = &S3CStorage{
		buckets:    buckets,
		storageDir: storageDir,
	}

	storage.DerivedStorage = storage
	storage.SetDefaultNestingLevels([]int{0}, 0)
	return storage, nil
}

// ListFiles return the list of files and subdirectories under 'dir' (non-recursively)
func (storage *S3CStorage) ListFiles(threadIndex int, dir string) (files []string, sizes []int64, err error) {
	if len(dir) > 0 && dir[len(dir)-1] != '/' {
		dir += "/"
	}

	dirLength := len(storage.storageDir + dir)
	if dir == "snapshots/" {
		results, err := storage.buckets[threadIndex].List(storage.storageDir+dir, "/", "", 100)
		if err != nil {
			return nil, nil, err
		}

		for _, subDir := range results.CommonPrefixes {
			files = append(files, subDir[dirLength:])
		}
		return files, nil, nil
	} else if dir == "chunks/" {
		marker := ""
		for {
			results, err := storage.buckets[threadIndex].List(storage.storageDir+dir, "", marker, 1000)
			if err != nil {
				return nil, nil, err
			}

			for _, object := range results.Contents {
				files = append(files, object.Key[dirLength:])
				sizes = append(sizes, object.Size)
			}

			if !results.IsTruncated {
				break
			}

			marker = results.Contents[len(results.Contents)-1].Key
		}
		return files, sizes, nil

	} else {

		results, err := storage.buckets[threadIndex].List(storage.storageDir+dir, "", "", 1000)
		if err != nil {
			return nil, nil, err
		}

		for _, object := range results.Contents {
			files = append(files, object.Key[dirLength:])
		}
		return files, nil, nil
	}
}

// DeleteFile deletes the file or directory at 'filePath'.
func (storage *S3CStorage) DeleteFile(threadIndex int, filePath string) (err error) {
	return storage.buckets[threadIndex].Del(storage.storageDir + filePath)
}

// MoveFile renames the file.
func (storage *S3CStorage) MoveFile(threadIndex int, from string, to string) (err error) {

	options := s3.CopyOptions{ContentType: "application/duplicacy"}
	_, err = storage.buckets[threadIndex].PutCopy(storage.storageDir+to, s3.Private, options, storage.buckets[threadIndex].Name+"/"+storage.storageDir+from)
	if err != nil {
		return nil
	}

	return storage.DeleteFile(threadIndex, from)
}

// CreateDirectory creates a new directory.
func (storage *S3CStorage) CreateDirectory(threadIndex int, dir string) (err error) {
	return nil
}

// GetFileInfo returns the information about the file or directory at 'filePath'.
func (storage *S3CStorage) GetFileInfo(threadIndex int, filePath string) (exist bool, isDir bool, size int64, err error) {

	response, err := storage.buckets[threadIndex].Head(storage.storageDir+filePath, nil)
	if err != nil {
		if e, ok := err.(*s3.Error); ok && (e.StatusCode == 403 || e.StatusCode == 404) {
			return false, false, 0, nil
		} else {
			return false, false, 0, err
		}
	}

	if response.StatusCode == 403 || response.StatusCode == 404 {
		return false, false, 0, nil
	} else {
		return true, false, response.ContentLength, nil
	}
}

// DownloadFile reads the file at 'filePath' into the chunk.
func (storage *S3CStorage) DownloadFile(threadIndex int, filePath string, chunk *Chunk) (err error) {

	readCloser, err := storage.buckets[threadIndex].GetReader(storage.storageDir + filePath)
	if err != nil {
		return err
	}

	defer readCloser.Close()

	_, err = RateLimitedCopy(chunk, readCloser, storage.DownloadRateLimit/len(storage.buckets))
	return err

}

// UploadFile writes 'content' to the file at 'filePath'.
func (storage *S3CStorage) UploadFile(threadIndex int, filePath string, content []byte) (err error) {

	options := s3.Options{}
	reader := CreateRateLimitedReader(content, storage.UploadRateLimit/len(storage.buckets))
	return storage.buckets[threadIndex].PutReader(storage.storageDir+filePath, reader, int64(len(content)), "application/duplicacy", s3.Private, options)
}

// If a local snapshot cache is needed for the storage to avoid downloading/uploading chunks too often when
// managing snapshots.
func (storage *S3CStorage) IsCacheNeeded() bool { return true }

// If the 'MoveFile' method is implemented.
func (storage *S3CStorage) IsMoveFileImplemented() bool { return true }

// If the storage can guarantee strong consistency.
func (storage *S3CStorage) IsStrongConsistent() bool { return false }

// If the storage supports fast listing of files names.
func (storage *S3CStorage) IsFastListing() bool { return true }

// Enable the test mode.
func (storage *S3CStorage) EnableTestMode() {}