mirror of
https://github.com/gilbertchen/duplicacy
synced 2025-12-06 00:03:38 +00:00
306 lines
9.5 KiB
Go
306 lines
9.5 KiB
Go
// Copyright (c) Acrosync LLC. All rights reserved.
|
|
// Licensed under the Fair Source License 0.9 (https://fair.io/)
|
|
// User Limitation: 5 users
|
|
|
|
package duplicacy
|
|
|
|
import (
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"net"
|
|
"path"
|
|
"time"
|
|
"runtime"
|
|
"math/rand"
|
|
|
|
"golang.org/x/crypto/ssh"
|
|
"github.com/pkg/sftp"
|
|
)
|
|
|
|
type SFTPStorage struct {
|
|
RateLimitedStorage
|
|
|
|
client *sftp.Client
|
|
storageDir string
|
|
numberOfThreads int
|
|
}
|
|
|
|
func CreateSFTPStorageWithPassword(server string, port int, username string, storageDir string,
|
|
password string, threads int) (storage *SFTPStorage, err error) {
|
|
|
|
authMethods := [] ssh.AuthMethod { ssh.Password(password) }
|
|
|
|
|
|
hostKeyCallback := func(hostname string, remote net.Addr,
|
|
key ssh.PublicKey) error {
|
|
return nil
|
|
}
|
|
|
|
return CreateSFTPStorage(server, port, username, storageDir, authMethods, hostKeyCallback, threads)
|
|
}
|
|
|
|
func CreateSFTPStorage(server string, port int, username string, storageDir string,
|
|
authMethods [] ssh.AuthMethod,
|
|
hostKeyCallback func(hostname string, remote net.Addr,
|
|
key ssh.PublicKey) error, threads int) (storage *SFTPStorage, err error) {
|
|
|
|
config := &ssh.ClientConfig{
|
|
User: username,
|
|
Auth: authMethods,
|
|
HostKeyCallback: hostKeyCallback,
|
|
}
|
|
|
|
if server == "sftp.hidrive.strato.com" {
|
|
config.Ciphers = []string {"aes128-cbc", "aes128-ctr", "aes256-ctr"}
|
|
}
|
|
|
|
serverAddress := fmt.Sprintf("%s:%d", server, port)
|
|
connection, err := ssh.Dial("tcp", serverAddress, config)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
client, err := sftp.NewClient(connection)
|
|
if err != nil {
|
|
connection.Close()
|
|
return nil, err
|
|
}
|
|
|
|
for storageDir[len(storageDir) - 1] == '/' {
|
|
storageDir = storageDir[:len(storageDir) - 1]
|
|
}
|
|
|
|
fileInfo, err := client.Stat(storageDir)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("Can't access the storage path %s: %v", storageDir, err)
|
|
}
|
|
|
|
if !fileInfo.IsDir() {
|
|
return nil, fmt.Errorf("The storage path %s is not a directory", storageDir)
|
|
}
|
|
|
|
storage = &SFTPStorage {
|
|
client: client,
|
|
storageDir: storageDir,
|
|
numberOfThreads: threads,
|
|
}
|
|
|
|
// Random number fo generating the temporary chunk file suffix.
|
|
rand.Seed(time.Now().UnixNano())
|
|
|
|
runtime.SetFinalizer(storage, CloseSFTPStorage)
|
|
|
|
return storage, nil
|
|
}
|
|
|
|
func CloseSFTPStorage(storage *SFTPStorage) {
|
|
storage.client.Close()
|
|
}
|
|
|
|
// ListFiles return the list of files and subdirectories under 'file' (non-recursively)
|
|
func (storage *SFTPStorage) ListFiles(threadIndex int, dirPath string) (files []string, sizes []int64, err error) {
|
|
|
|
entries, err := storage.client.ReadDir(path.Join(storage.storageDir, dirPath))
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
|
|
for _, entry := range entries {
|
|
name := entry.Name()
|
|
if entry.IsDir() && name[len(name) - 1] != '/' {
|
|
name += "/"
|
|
}
|
|
|
|
files = append(files, name)
|
|
sizes = append(sizes, entry.Size())
|
|
}
|
|
|
|
return files, sizes, nil
|
|
}
|
|
|
|
// DeleteFile deletes the file or directory at 'filePath'.
|
|
func (storage *SFTPStorage) DeleteFile(threadIndex int, filePath string) (err error) {
|
|
fullPath := path.Join(storage.storageDir, filePath)
|
|
fileInfo, err := storage.client.Stat(fullPath)
|
|
if err != nil {
|
|
if os.IsNotExist(err) {
|
|
LOG_TRACE("SFTP_STORAGE", "File %s has disappeared before deletion", filePath)
|
|
return nil
|
|
}
|
|
return err
|
|
}
|
|
if fileInfo == nil {
|
|
return nil
|
|
}
|
|
return storage.client.Remove(path.Join(storage.storageDir, filePath))
|
|
}
|
|
|
|
// MoveFile renames the file.
|
|
func (storage *SFTPStorage) MoveFile(threadIndex int, from string, to string) (err error) {
|
|
toPath := path.Join(storage.storageDir, to)
|
|
fileInfo, err := storage.client.Stat(toPath)
|
|
if fileInfo != nil {
|
|
return fmt.Errorf("The destination file %s already exists", toPath)
|
|
}
|
|
return storage.client.Rename(path.Join(storage.storageDir, from),
|
|
path.Join(storage.storageDir, to))
|
|
}
|
|
|
|
// CreateDirectory creates a new directory.
|
|
func (storage *SFTPStorage) CreateDirectory(threadIndex int, dirPath string) (err error) {
|
|
fullPath := path.Join(storage.storageDir, dirPath)
|
|
fileInfo, err := storage.client.Stat(fullPath)
|
|
if fileInfo != nil && fileInfo.IsDir() {
|
|
return nil
|
|
}
|
|
return storage.client.Mkdir(path.Join(storage.storageDir, dirPath))
|
|
}
|
|
|
|
// GetFileInfo returns the information about the file or directory at 'filePath'.
|
|
func (storage *SFTPStorage) GetFileInfo(threadIndex int, filePath string) (exist bool, isDir bool, size int64, err error) {
|
|
fileInfo, err := storage.client.Stat(path.Join(storage.storageDir, filePath))
|
|
if err != nil {
|
|
if os.IsNotExist(err) {
|
|
return false, false, 0, nil
|
|
} else {
|
|
return false, false, 0, err
|
|
}
|
|
}
|
|
|
|
if fileInfo == nil {
|
|
return false, false, 0, nil
|
|
}
|
|
|
|
return true, fileInfo.IsDir(), fileInfo.Size(), nil
|
|
}
|
|
|
|
// FindChunk finds the chunk with the specified id. If 'isFossil' is true, it will search for chunk files with
|
|
// the suffix '.fsl'.
|
|
func (storage *SFTPStorage) FindChunk(threadIndex int, chunkID string, isFossil bool) (filePath string, exist bool, size int64, err error) {
|
|
dir := path.Join(storage.storageDir, "chunks")
|
|
|
|
suffix := ""
|
|
if isFossil {
|
|
suffix = ".fsl"
|
|
}
|
|
|
|
// The minimum level of directories to dive into before searching for the chunk file.
|
|
minimumLevel := 2
|
|
|
|
for level := 0; level * 2 < len(chunkID); level ++ {
|
|
if level >= minimumLevel {
|
|
filePath = path.Join(dir, chunkID[2 * level:]) + suffix
|
|
if stat, err := storage.client.Stat(filePath); err == nil && !stat.IsDir() {
|
|
return filePath[len(storage.storageDir) + 1:], true, stat.Size(), nil
|
|
} else if err == nil && stat.IsDir() {
|
|
return filePath[len(storage.storageDir) + 1:], true, 0, fmt.Errorf("The path %s is a directory", filePath)
|
|
}
|
|
}
|
|
|
|
// Find the subdirectory the chunk file may reside.
|
|
subDir := path.Join(dir, chunkID[2 * level: 2 * level + 2])
|
|
stat, err := storage.client.Stat(subDir)
|
|
if err == nil && stat.IsDir() {
|
|
dir = subDir
|
|
continue
|
|
}
|
|
|
|
if level < minimumLevel {
|
|
// Create the subdirectory if is doesn't exist.
|
|
|
|
if err == nil && !stat.IsDir() {
|
|
return "", false, 0, fmt.Errorf("The path %s is not a directory", subDir)
|
|
}
|
|
|
|
err = storage.client.Mkdir(subDir)
|
|
if err != nil {
|
|
return "", false, 0, fmt.Errorf("Failed to create the directory %s: %v", subDir, err)
|
|
}
|
|
|
|
dir = subDir
|
|
continue
|
|
}
|
|
|
|
// Teh chunk must be under this subdirectory but it doesn't exist.
|
|
return path.Join(dir, chunkID[2 * level:])[len(storage.storageDir) + 1:] + suffix, false, 0, nil
|
|
|
|
}
|
|
|
|
LOG_FATAL("CHUNK_FIND", "Chunk %s is still not found after having searched a maximum level of directories",
|
|
chunkID)
|
|
return "", false, 0, nil
|
|
}
|
|
|
|
// DownloadFile reads the file at 'filePath' into the chunk.
|
|
func (storage *SFTPStorage) DownloadFile(threadIndex int, filePath string, chunk *Chunk) (err error) {
|
|
file, err := storage.client.Open(path.Join(storage.storageDir, filePath))
|
|
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
defer file.Close()
|
|
if _, err = RateLimitedCopy(chunk, file, storage.DownloadRateLimit / storage.numberOfThreads); err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// UploadFile writes 'content' to the file at 'filePath'.
|
|
func (storage *SFTPStorage) UploadFile(threadIndex int, filePath string, content []byte) (err error) {
|
|
|
|
fullPath := path.Join(storage.storageDir, filePath)
|
|
|
|
letters := "abcdefghijklmnopqrstuvwxyz"
|
|
suffix := make([]byte, 8)
|
|
for i := range suffix {
|
|
suffix[i] = letters[rand.Intn(len(letters))]
|
|
}
|
|
|
|
temporaryFile := fullPath + "." + string(suffix) + ".tmp"
|
|
|
|
file, err := storage.client.OpenFile(temporaryFile, os.O_WRONLY | os.O_CREATE | os.O_TRUNC)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
reader := CreateRateLimitedReader(content, storage.UploadRateLimit / storage.numberOfThreads)
|
|
_, err = io.Copy(file, reader)
|
|
if err != nil {
|
|
file.Close()
|
|
return err
|
|
}
|
|
file.Close()
|
|
|
|
err = storage.client.Rename(temporaryFile, fullPath)
|
|
if err != nil {
|
|
|
|
if _, err = storage.client.Stat(fullPath); err == nil {
|
|
storage.client.Remove(temporaryFile)
|
|
return nil
|
|
} else {
|
|
return fmt.Errorf("Uploaded file but failed to store it at %s", fullPath)
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// If a local snapshot cache is needed for the storage to avoid downloading/uploading chunks too often when
|
|
// managing snapshots.
|
|
func (storage *SFTPStorage) IsCacheNeeded () (bool) { return true }
|
|
|
|
// If the 'MoveFile' method is implemented.
|
|
func (storage *SFTPStorage) IsMoveFileImplemented() (bool) { return true }
|
|
|
|
// If the storage can guarantee strong consistency.
|
|
func (storage *SFTPStorage) IsStrongConsistent() (bool) { return true }
|
|
|
|
// If the storage supports fast listing of files names.
|
|
func (storage *SFTPStorage) IsFastListing() (bool) { return false }
|
|
|
|
// Enable the test mode.
|
|
func (storage *SFTPStorage) EnableTestMode() {}
|