jgaunt/duplicacy
1
0
Fork 0
mirror of https://github.com/gilbertchen/duplicacy synced 2025-12-17 00:33:17 +00:00
Code Issues Releases Wiki Activity

Merge pull request #211 from lowne/single-nesting

Browse Source 
Nesting level parameterization for local and SFTP storages
This commit is contained in:
gilbertchen
2017-10-05 22:27:16 -04:00
committed by GitHub
parent 2f6287a45d 6e3c1657fa
commit 746c1656a8
5 changed files with 26 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
src/duplicacy_dropboxstorage.go
View File

@@ -15,12 +15,13 @@ import (
type DropboxStorage struct {
RateLimitedStorage
clients []*dropbox.Files
storageDir string
clients []*dropbox.Files
minimumNesting int // The minimum level of directories to dive into before searching for the chunk file.
storageDir string
}
// CreateDropboxStorage creates a dropbox storage object.
func CreateDropboxStorage(accessToken string, storageDir string, threads int) (storage *DropboxStorage, err error) {
func CreateDropboxStorage(accessToken string, storageDir string, minimumNesting int, threads int) (storage *DropboxStorage, err error) {
var clients []*dropbox.Files
for i := 0; i < threads; i++ {
@@ -37,8 +38,9 @@ func CreateDropboxStorage(accessToken string, storageDir string, threads int) (s
}
storage = &DropboxStorage{
clients: clients,
storageDir: storageDir,
clients: clients,
storageDir: storageDir,
minimumNesting: minimumNesting,
}
err = storage.CreateDirectory(0, "")
@@ -189,11 +191,8 @@ func (storage *DropboxStorage) FindChunk(threadIndex int, chunkID string, isFoss
suffix = ".fsl"
}
// The minimum level of directories to dive into before searching for the chunk file.
minimumLevel := 1
for level := 0; level*2 < len(chunkID); level++ {
if level >= minimumLevel {
if level >= storage.minimumNesting {
filePath = path.Join(dir, chunkID[2*level:]) + suffix
var size int64
exist, _, size, err = storage.GetFileInfo(threadIndex, filePath)
@@ -217,7 +216,7 @@ func (storage *DropboxStorage) FindChunk(threadIndex int, chunkID string, isFoss
continue
}
if level < minimumLevel {
if level < storage.minimumNesting {
// Create the subdirectory if it doesn't exist.
err = storage.CreateDirectory(threadIndex, subDir)
if err != nil {

10
src/duplicacy_filestorage.go
View File

@@ -18,14 +18,14 @@ import (
type FileStorage struct {
RateLimitedStorage
minimumLevel int // The minimum level of directories to dive into before searching for the chunk file.
minimumNesting int // The minimum level of directories to dive into before searching for the chunk file.
isCacheNeeded bool // Network storages require caching
storageDir string
numberOfThreads int
}
// CreateFileStorage creates a file storage.
func CreateFileStorage(storageDir string, minimumLevel int, isCacheNeeded bool, threads int) (storage *FileStorage, err error) {
func CreateFileStorage(storageDir string, minimumNesting int, isCacheNeeded bool, threads int) (storage *FileStorage, err error) {
var stat os.FileInfo
@@ -51,7 +51,7 @@ func CreateFileStorage(storageDir string, minimumLevel int, isCacheNeeded bool,
storage = &FileStorage{
storageDir: storageDir,
minimumLevel: minimumLevel,
minimumNesting: minimumNesting,
isCacheNeeded: isCacheNeeded,
numberOfThreads: threads,
}
@@ -137,7 +137,7 @@ func (storage *FileStorage) FindChunk(threadIndex int, chunkID string, isFossil
}
for level := 0; level*2 < len(chunkID); level++ {
if level >= storage.minimumLevel {
if level >= storage.minimumNesting {
filePath = path.Join(dir, chunkID[2*level:]) + suffix
// Use Lstat() instead of Stat() since 1) Stat() doesn't work for deduplicated disks on Windows and 2) there isn't
// really a need to follow the link if filePath is a link.
@@ -159,7 +159,7 @@ func (storage *FileStorage) FindChunk(threadIndex int, chunkID string, isFossil
continue
}
if level < storage.minimumLevel {
if level < storage.minimumNesting {
// Create the subdirectory if it doesn't exist.
if err == nil && !stat.IsDir() {

17
src/duplicacy_sftpstorage.go
View File

@@ -22,12 +22,13 @@ type SFTPStorage struct {
RateLimitedStorage
client *sftp.Client
minimumNesting int // The minimum level of directories to dive into before searching for the chunk file.
storageDir string
numberOfThreads int
}
func CreateSFTPStorageWithPassword(server string, port int, username string, storageDir string,
password string, threads int) (storage *SFTPStorage, err error) {
minimumNesting int, password string, threads int) (storage *SFTPStorage, err error) {
authMethods := []ssh.AuthMethod{ssh.Password(password)}
@@ -36,10 +37,10 @@ func CreateSFTPStorageWithPassword(server string, port int, username string, sto
return nil
}
return CreateSFTPStorage(server, port, username, storageDir, authMethods, hostKeyCallback, threads)
return CreateSFTPStorage(server, port, username, storageDir, minimumNesting, authMethods, hostKeyCallback, threads)
}
func CreateSFTPStorage(server string, port int, username string, storageDir string,
func CreateSFTPStorage(server string, port int, username string, storageDir string, minimumNesting int,
authMethods []ssh.AuthMethod,
hostKeyCallback func(hostname string, remote net.Addr,
key ssh.PublicKey) error, threads int) (storage *SFTPStorage, err error) {
@@ -82,6 +83,7 @@ func CreateSFTPStorage(server string, port int, username string, storageDir stri
storage = &SFTPStorage{
client: client,
storageDir: storageDir,
minimumNesting: minimumNesting,
numberOfThreads: threads,
}
@@ -184,11 +186,8 @@ func (storage *SFTPStorage) FindChunk(threadIndex int, chunkID string, isFossil
suffix = ".fsl"
}
// The minimum level of directories to dive into before searching for the chunk file.
minimumLevel := 2
for level := 0; level*2 < len(chunkID); level++ {
if level >= minimumLevel {
if level >= storage.minimumNesting {
filePath = path.Join(dir, chunkID[2*level:]) + suffix
if stat, err := storage.client.Stat(filePath); err == nil && !stat.IsDir() {
return filePath[len(storage.storageDir)+1:], true, stat.Size(), nil
@@ -205,7 +204,7 @@ func (storage *SFTPStorage) FindChunk(threadIndex int, chunkID string, isFossil
continue
}
if level < minimumLevel {
if level < storage.minimumNesting {
// Create the subdirectory if is doesn't exist.
if err == nil && !stat.IsDir() {
@@ -225,7 +224,7 @@ func (storage *SFTPStorage) FindChunk(threadIndex int, chunkID string, isFossil
continue
}
// Teh chunk must be under this subdirectory but it doesn't exist.
// The chunk must be under this subdirectory but it doesn't exist.
return path.Join(dir, chunkID[2*level:])[len(storage.storageDir)+1:] + suffix, false, 0, nil
}

4
src/duplicacy_storage.go
View File

@@ -313,7 +313,7 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
return checkHostKey(hostname, remote, key)
}
sftpStorage, err := CreateSFTPStorage(server, port, username, storageDir, authMethods, hostKeyChecker, threads)
sftpStorage, err := CreateSFTPStorage(server, port, username, storageDir, 2, authMethods, hostKeyChecker, threads)
if err != nil {
LOG_ERROR("STORAGE_CREATE", "Failed to load the SFTP storage at %s: %v", storageURL, err)
return nil
@@ -375,7 +375,7 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
} else if matched[1] == "dropbox" {
storageDir := matched[3] + matched[5]
token := GetPassword(preference, "dropbox_token", "Enter Dropbox access token:", true, resetPassword)
dropboxStorage, err := CreateDropboxStorage(token, storageDir, threads)
dropboxStorage, err := CreateDropboxStorage(token, storageDir, 1, threads)
if err != nil {
LOG_ERROR("STORAGE_CREATE", "Failed to load the dropbox storage: %v", err)
return nil

4
src/duplicacy_storage_test.go
View File

@@ -67,7 +67,7 @@ func loadStorage(localStoragePath string, threads int) (Storage, error) {
return CreateFileStorage(localStoragePath, 2, true, threads)
} else if testStorageName == "sftp" {
port, _ := strconv.Atoi(storage["port"])
return CreateSFTPStorageWithPassword(storage["server"], port, storage["username"], storage["directory"], storage["password"], threads)
return CreateSFTPStorageWithPassword(storage["server"], port, storage["username"], storage["directory"], 2, storage["password"], threads)
} else if testStorageName == "s3" || testStorageName == "wasabi" {
return CreateS3Storage(storage["region"], storage["endpoint"], storage["bucket"], storage["directory"], storage["access_key"], storage["secret_key"], threads, true, false)
} else if testStorageName == "s3c" {
@@ -77,7 +77,7 @@ func loadStorage(localStoragePath string, threads int) (Storage, error) {
} else if testStorageName == "minios" {
return CreateS3Storage(storage["region"], storage["endpoint"], storage["bucket"], storage["directory"], storage["access_key"], storage["secret_key"], threads, true, true)
} else if testStorageName == "dropbox" {
return CreateDropboxStorage(storage["token"], storage["directory"], threads)
return CreateDropboxStorage(storage["token"], storage["directory"], 1, threads)
} else if testStorageName == "b2" {
return CreateB2Storage(storage["account"], storage["key"], storage["bucket"], threads)
} else if testStorageName == "gcs-s3" {