jgaunt/duplicacy
1
0
Fork 0
mirror of https://github.com/gilbertchen/duplicacy synced 2025-12-17 00:33:17 +00:00
Code Issues Releases Wiki Activity

Merge pull request #211 from lowne/single-nesting

Browse Source 
Nesting level parameterization for local and SFTP storages
This commit is contained in:
gilbertchen
2017-10-05 22:27:16 -04:00
committed by GitHub
parent 2f6287a45d 6e3c1657fa
commit 746c1656a8
5 changed files with 26 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
src/duplicacy_dropboxstorage.go
View File

@@ -15,12 +15,13 @@ import (
type DropboxStorage struct { type DropboxStorage struct {
RateLimitedStorage RateLimitedStorage
clients []*dropbox.Files clients []*dropbox.Files
storageDir string minimumNesting int // The minimum level of directories to dive into before searching for the chunk file.
storageDir string
} }
// CreateDropboxStorage creates a dropbox storage object. // CreateDropboxStorage creates a dropbox storage object.
func CreateDropboxStorage(accessToken string, storageDir string, threads int) (storage *DropboxStorage, err error) { func CreateDropboxStorage(accessToken string, storageDir string, minimumNesting int, threads int) (storage *DropboxStorage, err error) {
var clients []*dropbox.Files var clients []*dropbox.Files
for i := 0; i < threads; i++ { for i := 0; i < threads; i++ {
@@ -37,8 +38,9 @@ func CreateDropboxStorage(accessToken string, storageDir string, threads int) (s
} }
storage = &DropboxStorage{ storage = &DropboxStorage{
clients: clients, clients: clients,
storageDir: storageDir, storageDir: storageDir,
minimumNesting: minimumNesting,
} }
err = storage.CreateDirectory(0, "") err = storage.CreateDirectory(0, "")
@@ -189,11 +191,8 @@ func (storage *DropboxStorage) FindChunk(threadIndex int, chunkID string, isFoss
suffix = ".fsl" suffix = ".fsl"
} }
// The minimum level of directories to dive into before searching for the chunk file.
minimumLevel := 1
for level := 0; level*2 < len(chunkID); level++ { for level := 0; level*2 < len(chunkID); level++ {
if level >= minimumLevel { if level >= storage.minimumNesting {
filePath = path.Join(dir, chunkID[2*level:]) + suffix filePath = path.Join(dir, chunkID[2*level:]) + suffix
var size int64 var size int64
exist, _, size, err = storage.GetFileInfo(threadIndex, filePath) exist, _, size, err = storage.GetFileInfo(threadIndex, filePath)
@@ -217,7 +216,7 @@ func (storage *DropboxStorage) FindChunk(threadIndex int, chunkID string, isFoss
continue continue
} }
if level < minimumLevel { if level < storage.minimumNesting {
// Create the subdirectory if it doesn't exist. // Create the subdirectory if it doesn't exist.
err = storage.CreateDirectory(threadIndex, subDir) err = storage.CreateDirectory(threadIndex, subDir)
if err != nil { if err != nil {

10
src/duplicacy_filestorage.go
View File

@@ -18,14 +18,14 @@ import (
type FileStorage struct { type FileStorage struct {
RateLimitedStorage RateLimitedStorage
minimumLevel int // The minimum level of directories to dive into before searching for the chunk file. minimumNesting int // The minimum level of directories to dive into before searching for the chunk file.
isCacheNeeded bool // Network storages require caching isCacheNeeded bool // Network storages require caching
storageDir string storageDir string
numberOfThreads int numberOfThreads int
} }
// CreateFileStorage creates a file storage. // CreateFileStorage creates a file storage.
func CreateFileStorage(storageDir string, minimumLevel int, isCacheNeeded bool, threads int) (storage *FileStorage, err error) { func CreateFileStorage(storageDir string, minimumNesting int, isCacheNeeded bool, threads int) (storage *FileStorage, err error) {
var stat os.FileInfo var stat os.FileInfo
@@ -51,7 +51,7 @@ func CreateFileStorage(storageDir string, minimumLevel int, isCacheNeeded bool,
storage = &FileStorage{ storage = &FileStorage{
storageDir: storageDir, storageDir: storageDir,
minimumLevel: minimumLevel, minimumNesting: minimumNesting,
isCacheNeeded: isCacheNeeded, isCacheNeeded: isCacheNeeded,
numberOfThreads: threads, numberOfThreads: threads,
} }
@@ -137,7 +137,7 @@ func (storage *FileStorage) FindChunk(threadIndex int, chunkID string, isFossil
} }
for level := 0; level*2 < len(chunkID); level++ { for level := 0; level*2 < len(chunkID); level++ {
if level >= storage.minimumLevel { if level >= storage.minimumNesting {
filePath = path.Join(dir, chunkID[2*level:]) + suffix filePath = path.Join(dir, chunkID[2*level:]) + suffix
// Use Lstat() instead of Stat() since 1) Stat() doesn't work for deduplicated disks on Windows and 2) there isn't // Use Lstat() instead of Stat() since 1) Stat() doesn't work for deduplicated disks on Windows and 2) there isn't
// really a need to follow the link if filePath is a link. // really a need to follow the link if filePath is a link.
@@ -159,7 +159,7 @@ func (storage *FileStorage) FindChunk(threadIndex int, chunkID string, isFossil
continue continue
} }
if level < storage.minimumLevel { if level < storage.minimumNesting {
// Create the subdirectory if it doesn't exist. // Create the subdirectory if it doesn't exist.
if err == nil && !stat.IsDir() { if err == nil && !stat.IsDir() {

17
src/duplicacy_sftpstorage.go
View File

@@ -22,12 +22,13 @@ type SFTPStorage struct {
RateLimitedStorage RateLimitedStorage
client *sftp.Client client *sftp.Client
minimumNesting int // The minimum level of directories to dive into before searching for the chunk file.
storageDir string storageDir string
numberOfThreads int numberOfThreads int
} }
func CreateSFTPStorageWithPassword(server string, port int, username string, storageDir string, func CreateSFTPStorageWithPassword(server string, port int, username string, storageDir string,
password string, threads int) (storage *SFTPStorage, err error) { minimumNesting int, password string, threads int) (storage *SFTPStorage, err error) {
authMethods := []ssh.AuthMethod{ssh.Password(password)} authMethods := []ssh.AuthMethod{ssh.Password(password)}
@@ -36,10 +37,10 @@ func CreateSFTPStorageWithPassword(server string, port int, username string, sto
return nil return nil
} }
return CreateSFTPStorage(server, port, username, storageDir, authMethods, hostKeyCallback, threads) return CreateSFTPStorage(server, port, username, storageDir, minimumNesting, authMethods, hostKeyCallback, threads)
} }
func CreateSFTPStorage(server string, port int, username string, storageDir string, func CreateSFTPStorage(server string, port int, username string, storageDir string, minimumNesting int,
authMethods []ssh.AuthMethod, authMethods []ssh.AuthMethod,
hostKeyCallback func(hostname string, remote net.Addr, hostKeyCallback func(hostname string, remote net.Addr,
key ssh.PublicKey) error, threads int) (storage *SFTPStorage, err error) { key ssh.PublicKey) error, threads int) (storage *SFTPStorage, err error) {
@@ -82,6 +83,7 @@ func CreateSFTPStorage(server string, port int, username string, storageDir stri
storage = &SFTPStorage{ storage = &SFTPStorage{
client: client, client: client,
storageDir: storageDir, storageDir: storageDir,
minimumNesting: minimumNesting,
numberOfThreads: threads, numberOfThreads: threads,
} }
@@ -184,11 +186,8 @@ func (storage *SFTPStorage) FindChunk(threadIndex int, chunkID string, isFossil
suffix = ".fsl" suffix = ".fsl"
} }
// The minimum level of directories to dive into before searching for the chunk file.
minimumLevel := 2
for level := 0; level*2 < len(chunkID); level++ { for level := 0; level*2 < len(chunkID); level++ {
if level >= minimumLevel { if level >= storage.minimumNesting {
filePath = path.Join(dir, chunkID[2*level:]) + suffix filePath = path.Join(dir, chunkID[2*level:]) + suffix
if stat, err := storage.client.Stat(filePath); err == nil && !stat.IsDir() { if stat, err := storage.client.Stat(filePath); err == nil && !stat.IsDir() {
return filePath[len(storage.storageDir)+1:], true, stat.Size(), nil return filePath[len(storage.storageDir)+1:], true, stat.Size(), nil
@@ -205,7 +204,7 @@ func (storage *SFTPStorage) FindChunk(threadIndex int, chunkID string, isFossil
continue continue
} }
if level < minimumLevel { if level < storage.minimumNesting {
// Create the subdirectory if is doesn't exist. // Create the subdirectory if is doesn't exist.
if err == nil && !stat.IsDir() { if err == nil && !stat.IsDir() {
@@ -225,7 +224,7 @@ func (storage *SFTPStorage) FindChunk(threadIndex int, chunkID string, isFossil
continue continue
} }
// Teh chunk must be under this subdirectory but it doesn't exist. // The chunk must be under this subdirectory but it doesn't exist.
return path.Join(dir, chunkID[2*level:])[len(storage.storageDir)+1:] + suffix, false, 0, nil return path.Join(dir, chunkID[2*level:])[len(storage.storageDir)+1:] + suffix, false, 0, nil
} }

4
src/duplicacy_storage.go
View File

@@ -313,7 +313,7 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
return checkHostKey(hostname, remote, key) return checkHostKey(hostname, remote, key)
} }
sftpStorage, err := CreateSFTPStorage(server, port, username, storageDir, authMethods, hostKeyChecker, threads) sftpStorage, err := CreateSFTPStorage(server, port, username, storageDir, 2, authMethods, hostKeyChecker, threads)
if err != nil { if err != nil {
LOG_ERROR("STORAGE_CREATE", "Failed to load the SFTP storage at %s: %v", storageURL, err) LOG_ERROR("STORAGE_CREATE", "Failed to load the SFTP storage at %s: %v", storageURL, err)
return nil return nil
@@ -375,7 +375,7 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
} else if matched[1] == "dropbox" { } else if matched[1] == "dropbox" {
storageDir := matched[3] + matched[5] storageDir := matched[3] + matched[5]
token := GetPassword(preference, "dropbox_token", "Enter Dropbox access token:", true, resetPassword) token := GetPassword(preference, "dropbox_token", "Enter Dropbox access token:", true, resetPassword)
dropboxStorage, err := CreateDropboxStorage(token, storageDir, threads) dropboxStorage, err := CreateDropboxStorage(token, storageDir, 1, threads)
if err != nil { if err != nil {
LOG_ERROR("STORAGE_CREATE", "Failed to load the dropbox storage: %v", err) LOG_ERROR("STORAGE_CREATE", "Failed to load the dropbox storage: %v", err)
return nil return nil

4
src/duplicacy_storage_test.go
View File

@@ -67,7 +67,7 @@ func loadStorage(localStoragePath string, threads int) (Storage, error) {
return CreateFileStorage(localStoragePath, 2, true, threads) return CreateFileStorage(localStoragePath, 2, true, threads)
} else if testStorageName == "sftp" { } else if testStorageName == "sftp" {
port, _ := strconv.Atoi(storage["port"]) port, _ := strconv.Atoi(storage["port"])
return CreateSFTPStorageWithPassword(storage["server"], port, storage["username"], storage["directory"], storage["password"], threads) return CreateSFTPStorageWithPassword(storage["server"], port, storage["username"], storage["directory"], 2, storage["password"], threads)
} else if testStorageName == "s3" || testStorageName == "wasabi" { } else if testStorageName == "s3" || testStorageName == "wasabi" {
return CreateS3Storage(storage["region"], storage["endpoint"], storage["bucket"], storage["directory"], storage["access_key"], storage["secret_key"], threads, true, false) return CreateS3Storage(storage["region"], storage["endpoint"], storage["bucket"], storage["directory"], storage["access_key"], storage["secret_key"], threads, true, false)
} else if testStorageName == "s3c" { } else if testStorageName == "s3c" {
@@ -77,7 +77,7 @@ func loadStorage(localStoragePath string, threads int) (Storage, error) {
} else if testStorageName == "minios" { } else if testStorageName == "minios" {
return CreateS3Storage(storage["region"], storage["endpoint"], storage["bucket"], storage["directory"], storage["access_key"], storage["secret_key"], threads, true, true) return CreateS3Storage(storage["region"], storage["endpoint"], storage["bucket"], storage["directory"], storage["access_key"], storage["secret_key"], threads, true, true)
} else if testStorageName == "dropbox" { } else if testStorageName == "dropbox" {
return CreateDropboxStorage(storage["token"], storage["directory"], threads) return CreateDropboxStorage(storage["token"], storage["directory"], 1, threads)
} else if testStorageName == "b2" { } else if testStorageName == "b2" {
return CreateB2Storage(storage["account"], storage["key"], storage["bucket"], threads) return CreateB2Storage(storage["account"], storage["key"], storage["bucket"], threads)
} else if testStorageName == "gcs-s3" { } else if testStorageName == "gcs-s3" {