jgaunt/duplicacy
1
0
Fork 0
mirror of https://github.com/gilbertchen/duplicacy synced 2025-12-17 16:53:19 +00:00
Code Issues Releases Wiki Activity

default to single-dir-nesting for local and SFTP storages

Browse Source
This commit is contained in:
Mark Lowne
2017-09-25 15:05:09 +02:00
parent 923cd0aa63
commit 3dad87f13a
6 changed files with 32 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
src/duplicacy_dropboxstorage.go
View File

@@ -16,11 +16,12 @@ type DropboxStorage struct {
RateLimitedStorage
clients []*dropbox.Files
minimumNesting int // The minimum level of directories to dive into before searching for the chunk file.
storageDir string
}
// CreateDropboxStorage creates a dropbox storage object.
func CreateDropboxStorage(accessToken string, storageDir string, threads int) (storage *DropboxStorage, err error) {
func CreateDropboxStorage(accessToken string, storageDir string, minimumNesting int, threads int) (storage *DropboxStorage, err error) {
var clients []*dropbox.Files
for i := 0; i < threads; i++ {
@@ -39,6 +40,7 @@ func CreateDropboxStorage(accessToken string, storageDir string, threads int) (s
storage = &DropboxStorage{
clients: clients,
storageDir: storageDir,
minimumNesting: minimumNesting,
}
err = storage.CreateDirectory(0, "")
@@ -189,11 +191,8 @@ func (storage *DropboxStorage) FindChunk(threadIndex int, chunkID string, isFoss
suffix = ".fsl"
}
// The minimum level of directories to dive into before searching for the chunk file.
minimumLevel := 1
for level := 0; level*2 < len(chunkID); level++ {
if level >= minimumLevel {
if level >= storage.minimumNesting {
filePath = path.Join(dir, chunkID[2*level:]) + suffix
var size int64
exist, _, size, err = storage.GetFileInfo(threadIndex, filePath)
@@ -217,7 +216,7 @@ func (storage *DropboxStorage) FindChunk(threadIndex int, chunkID string, isFoss
continue
}
if level < minimumLevel {
if level < storage.minimumNesting {
// Create the subdirectory if it doesn't exist.
err = storage.CreateDirectory(threadIndex, subDir)
if err != nil {

10
src/duplicacy_filestorage.go
View File

@@ -18,14 +18,14 @@ import (
type FileStorage struct {
RateLimitedStorage
minimumLevel int // The minimum level of directories to dive into before searching for the chunk file.
minimumNesting int // The minimum level of directories to dive into before searching for the chunk file.
isCacheNeeded bool // Network storages require caching
storageDir string
numberOfThreads int
}
// CreateFileStorage creates a file storage.
func CreateFileStorage(storageDir string, minimumLevel int, isCacheNeeded bool, threads int) (storage *FileStorage, err error) {
func CreateFileStorage(storageDir string, minimumNesting int, isCacheNeeded bool, threads int) (storage *FileStorage, err error) {
var stat os.FileInfo
@@ -51,7 +51,7 @@ func CreateFileStorage(storageDir string, minimumLevel int, isCacheNeeded bool,
storage = &FileStorage{
storageDir: storageDir,
minimumLevel: minimumLevel,
minimumNesting: minimumNesting,
isCacheNeeded: isCacheNeeded,
numberOfThreads: threads,
}
@@ -137,7 +137,7 @@ func (storage *FileStorage) FindChunk(threadIndex int, chunkID string, isFossil
}
for level := 0; level*2 < len(chunkID); level++ {
if level >= storage.minimumLevel {
if level >= storage.minimumNesting {
filePath = path.Join(dir, chunkID[2*level:]) + suffix
// Use Lstat() instead of Stat() since 1) Stat() doesn't work for deduplicated disks on Windows and 2) there isn't
// really a need to follow the link if filePath is a link.
@@ -159,7 +159,7 @@ func (storage *FileStorage) FindChunk(threadIndex int, chunkID string, isFossil
continue
}
if level < storage.minimumLevel {
if level < storage.minimumNesting {
// Create the subdirectory if it doesn't exist.
if err == nil && !stat.IsDir() {

17
src/duplicacy_sftpstorage.go
View File

@@ -22,12 +22,13 @@ type SFTPStorage struct {
RateLimitedStorage
client *sftp.Client
minimumNesting int // The minimum level of directories to dive into before searching for the chunk file.
storageDir string
numberOfThreads int
}
func CreateSFTPStorageWithPassword(server string, port int, username string, storageDir string,
password string, threads int) (storage *SFTPStorage, err error) {
minimumNesting int, password string, threads int) (storage *SFTPStorage, err error) {
authMethods := []ssh.AuthMethod{ssh.Password(password)}
@@ -36,10 +37,10 @@ func CreateSFTPStorageWithPassword(server string, port int, username string, sto
return nil
}
return CreateSFTPStorage(server, port, username, storageDir, authMethods, hostKeyCallback, threads)
return CreateSFTPStorage(server, port, username, storageDir, minimumNesting, authMethods, hostKeyCallback, threads)
}
func CreateSFTPStorage(server string, port int, username string, storageDir string,
func CreateSFTPStorage(server string, port int, username string, storageDir string, minimumNesting int,
authMethods []ssh.AuthMethod,
hostKeyCallback func(hostname string, remote net.Addr,
key ssh.PublicKey) error, threads int) (storage *SFTPStorage, err error) {
@@ -82,6 +83,7 @@ func CreateSFTPStorage(server string, port int, username string, storageDir stri
storage = &SFTPStorage{
client: client,
storageDir: storageDir,
minimumNesting: minimumNesting,
numberOfThreads: threads,
}
@@ -184,11 +186,8 @@ func (storage *SFTPStorage) FindChunk(threadIndex int, chunkID string, isFossil
suffix = ".fsl"
}
// The minimum level of directories to dive into before searching for the chunk file.
minimumLevel := 2
for level := 0; level*2 < len(chunkID); level++ {
if level >= minimumLevel {
if level >= storage.minimumNesting {
filePath = path.Join(dir, chunkID[2*level:]) + suffix
if stat, err := storage.client.Stat(filePath); err == nil && !stat.IsDir() {
return filePath[len(storage.storageDir)+1:], true, stat.Size(), nil
@@ -205,7 +204,7 @@ func (storage *SFTPStorage) FindChunk(threadIndex int, chunkID string, isFossil
continue
}
if level < minimumLevel {
if level < storage.minimumNesting {
// Create the subdirectory if is doesn't exist.
if err == nil && !stat.IsDir() {
@@ -225,7 +224,7 @@ func (storage *SFTPStorage) FindChunk(threadIndex int, chunkID string, isFossil
continue
}
// Teh chunk must be under this subdirectory but it doesn't exist.
// The chunk must be under this subdirectory but it doesn't exist.
return path.Join(dir, chunkID[2*level:])[len(storage.storageDir)+1:] + suffix, false, 0, nil
}

4
src/duplicacy_snapshotmanager_test.go
View File

@@ -95,14 +95,14 @@ func createTestSnapshotManager(testDir string) *SnapshotManager {
os.RemoveAll(testDir)
os.MkdirAll(testDir, 0700)
storage, _ := CreateFileStorage(testDir, 2, false, 1)
storage, _ := CreateFileStorage(testDir, 1, false, 1)
storage.CreateDirectory(0, "chunks")
storage.CreateDirectory(0, "snapshots")
config := CreateConfig()
snapshotManager := CreateSnapshotManager(config, storage)
cacheDir := path.Join(testDir, "cache")
snapshotCache, _ := CreateFileStorage(cacheDir, 2, false, 1)
snapshotCache, _ := CreateFileStorage(cacheDir, 1, false, 1)
snapshotCache.CreateDirectory(0, "chunks")
snapshotCache.CreateDirectory(0, "snapshots")

8
src/duplicacy_storage.go
View File

@@ -146,7 +146,7 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
}
if isFileStorage {
fileStorage, err := CreateFileStorage(storageURL, 2, isCacheNeeded, threads)
fileStorage, err := CreateFileStorage(storageURL, 1, isCacheNeeded, threads)
if err != nil {
LOG_ERROR("STORAGE_CREATE", "Failed to load the file storage at %s: %v", storageURL, err)
return nil
@@ -164,7 +164,7 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
}
if strings.HasPrefix(storageURL, "samba://") {
fileStorage, err := CreateFileStorage(storageURL[8:], 2, true, threads)
fileStorage, err := CreateFileStorage(storageURL[8:], 1, true, threads)
if err != nil {
LOG_ERROR("STORAGE_CREATE", "Failed to load the file storage at %s: %v", storageURL, err)
return nil
@@ -311,7 +311,7 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
return checkHostKey(hostname, remote, key)
}
sftpStorage, err := CreateSFTPStorage(server, port, username, storageDir, authMethods, hostKeyChecker, threads)
sftpStorage, err := CreateSFTPStorage(server, port, username, storageDir, 1, authMethods, hostKeyChecker, threads)
if err != nil {
LOG_ERROR("STORAGE_CREATE", "Failed to load the SFTP storage at %s: %v", storageURL, err)
return nil
@@ -373,7 +373,7 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
} else if matched[1] == "dropbox" {
storageDir := matched[3] + matched[5]
token := GetPassword(preference, "dropbox_token", "Enter Dropbox access token:", true, resetPassword)
dropboxStorage, err := CreateDropboxStorage(token, storageDir, threads)
dropboxStorage, err := CreateDropboxStorage(token, storageDir, 1, threads)
if err != nil {
LOG_ERROR("STORAGE_CREATE", "Failed to load the dropbox storage: %v", err)
return nil

8
src/duplicacy_storage_test.go
View File

@@ -41,7 +41,7 @@ func init() {
func loadStorage(localStoragePath string, threads int) (Storage, error) {
if testStorageName == "" || testStorageName == "file" {
return CreateFileStorage(localStoragePath, 2, false, threads)
return CreateFileStorage(localStoragePath, 1, false, threads)
}
config, err := ioutil.ReadFile("test_storage.conf")
@@ -64,10 +64,10 @@ func loadStorage(localStoragePath string, threads int) (Storage, error) {
if testStorageName == "flat" {
return CreateFileStorage(localStoragePath, 0, false, threads)
} else if testStorageName == "samba" {
return CreateFileStorage(localStoragePath, 2, true, threads)
return CreateFileStorage(localStoragePath, 1, true, threads)
} else if testStorageName == "sftp" {
port, _ := strconv.Atoi(storage["port"])
return CreateSFTPStorageWithPassword(storage["server"], port, storage["username"], storage["directory"], storage["password"], threads)
return CreateSFTPStorageWithPassword(storage["server"], port, storage["username"], storage["directory"], 1, storage["password"], threads)
} else if testStorageName == "s3" || testStorageName == "wasabi" {
return CreateS3Storage(storage["region"], storage["endpoint"], storage["bucket"], storage["directory"], storage["access_key"], storage["secret_key"], threads, true, false)
} else if testStorageName == "s3c" {
@@ -77,7 +77,7 @@ func loadStorage(localStoragePath string, threads int) (Storage, error) {
} else if testStorageName == "minios" {
return CreateS3Storage(storage["region"], storage["endpoint"], storage["bucket"], storage["directory"], storage["access_key"], storage["secret_key"], threads, true, true)
} else if testStorageName == "dropbox" {
return CreateDropboxStorage(storage["token"], storage["directory"], threads)
return CreateDropboxStorage(storage["token"], storage["directory"], 1, threads)
} else if testStorageName == "b2" {
return CreateB2Storage(storage["account"], storage["key"], storage["bucket"], threads)
} else if testStorageName == "gcs-s3" {