jgaunt/duplicacy
1
0
Fork 0
mirror of https://github.com/gilbertchen/duplicacy synced 2025-12-17 16:53:19 +00:00
Code Issues Releases Wiki Activity

default to single-dir-nesting for local and SFTP storages

Browse Source
This commit is contained in:
Mark Lowne
2017-09-25 15:05:09 +02:00
parent 923cd0aa63
commit 3dad87f13a
6 changed files with 32 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
src/duplicacy_dropboxstorage.go
View File

@@ -15,12 +15,13 @@ import (
type DropboxStorage struct { type DropboxStorage struct {
RateLimitedStorage RateLimitedStorage
clients []*dropbox.Files clients []*dropbox.Files
storageDir string minimumNesting int // The minimum level of directories to dive into before searching for the chunk file.
storageDir string
} }
// CreateDropboxStorage creates a dropbox storage object. // CreateDropboxStorage creates a dropbox storage object.
func CreateDropboxStorage(accessToken string, storageDir string, threads int) (storage *DropboxStorage, err error) { func CreateDropboxStorage(accessToken string, storageDir string, minimumNesting int, threads int) (storage *DropboxStorage, err error) {
var clients []*dropbox.Files var clients []*dropbox.Files
for i := 0; i < threads; i++ { for i := 0; i < threads; i++ {
@@ -37,8 +38,9 @@ func CreateDropboxStorage(accessToken string, storageDir string, threads int) (s
} }
storage = &DropboxStorage{ storage = &DropboxStorage{
clients: clients, clients: clients,
storageDir: storageDir, storageDir: storageDir,
minimumNesting: minimumNesting,
} }
err = storage.CreateDirectory(0, "") err = storage.CreateDirectory(0, "")
@@ -189,11 +191,8 @@ func (storage *DropboxStorage) FindChunk(threadIndex int, chunkID string, isFoss
suffix = ".fsl" suffix = ".fsl"
} }
// The minimum level of directories to dive into before searching for the chunk file.
minimumLevel := 1
for level := 0; level*2 < len(chunkID); level++ { for level := 0; level*2 < len(chunkID); level++ {
if level >= minimumLevel { if level >= storage.minimumNesting {
filePath = path.Join(dir, chunkID[2*level:]) + suffix filePath = path.Join(dir, chunkID[2*level:]) + suffix
var size int64 var size int64
exist, _, size, err = storage.GetFileInfo(threadIndex, filePath) exist, _, size, err = storage.GetFileInfo(threadIndex, filePath)
@@ -217,7 +216,7 @@ func (storage *DropboxStorage) FindChunk(threadIndex int, chunkID string, isFoss
continue continue
} }
if level < minimumLevel { if level < storage.minimumNesting {
// Create the subdirectory if it doesn't exist. // Create the subdirectory if it doesn't exist.
err = storage.CreateDirectory(threadIndex, subDir) err = storage.CreateDirectory(threadIndex, subDir)
if err != nil { if err != nil {

10
src/duplicacy_filestorage.go
View File

@@ -18,14 +18,14 @@ import (
type FileStorage struct { type FileStorage struct {
RateLimitedStorage RateLimitedStorage
minimumLevel int // The minimum level of directories to dive into before searching for the chunk file. minimumNesting int // The minimum level of directories to dive into before searching for the chunk file.
isCacheNeeded bool // Network storages require caching isCacheNeeded bool // Network storages require caching
storageDir string storageDir string
numberOfThreads int numberOfThreads int
} }
// CreateFileStorage creates a file storage. // CreateFileStorage creates a file storage.
func CreateFileStorage(storageDir string, minimumLevel int, isCacheNeeded bool, threads int) (storage *FileStorage, err error) { func CreateFileStorage(storageDir string, minimumNesting int, isCacheNeeded bool, threads int) (storage *FileStorage, err error) {
var stat os.FileInfo var stat os.FileInfo
@@ -51,7 +51,7 @@ func CreateFileStorage(storageDir string, minimumLevel int, isCacheNeeded bool,
storage = &FileStorage{ storage = &FileStorage{
storageDir: storageDir, storageDir: storageDir,
minimumLevel: minimumLevel, minimumNesting: minimumNesting,
isCacheNeeded: isCacheNeeded, isCacheNeeded: isCacheNeeded,
numberOfThreads: threads, numberOfThreads: threads,
} }
@@ -137,7 +137,7 @@ func (storage *FileStorage) FindChunk(threadIndex int, chunkID string, isFossil
} }
for level := 0; level*2 < len(chunkID); level++ { for level := 0; level*2 < len(chunkID); level++ {
if level >= storage.minimumLevel { if level >= storage.minimumNesting {
filePath = path.Join(dir, chunkID[2*level:]) + suffix filePath = path.Join(dir, chunkID[2*level:]) + suffix
// Use Lstat() instead of Stat() since 1) Stat() doesn't work for deduplicated disks on Windows and 2) there isn't // Use Lstat() instead of Stat() since 1) Stat() doesn't work for deduplicated disks on Windows and 2) there isn't
// really a need to follow the link if filePath is a link. // really a need to follow the link if filePath is a link.
@@ -159,7 +159,7 @@ func (storage *FileStorage) FindChunk(threadIndex int, chunkID string, isFossil
continue continue
} }
if level < storage.minimumLevel { if level < storage.minimumNesting {
// Create the subdirectory if it doesn't exist. // Create the subdirectory if it doesn't exist.
if err == nil && !stat.IsDir() { if err == nil && !stat.IsDir() {

17
src/duplicacy_sftpstorage.go
View File

@@ -22,12 +22,13 @@ type SFTPStorage struct {
RateLimitedStorage RateLimitedStorage
client *sftp.Client client *sftp.Client
minimumNesting int // The minimum level of directories to dive into before searching for the chunk file.
storageDir string storageDir string
numberOfThreads int numberOfThreads int
} }
func CreateSFTPStorageWithPassword(server string, port int, username string, storageDir string, func CreateSFTPStorageWithPassword(server string, port int, username string, storageDir string,
password string, threads int) (storage *SFTPStorage, err error) { minimumNesting int, password string, threads int) (storage *SFTPStorage, err error) {
authMethods := []ssh.AuthMethod{ssh.Password(password)} authMethods := []ssh.AuthMethod{ssh.Password(password)}
@@ -36,10 +37,10 @@ func CreateSFTPStorageWithPassword(server string, port int, username string, sto
return nil return nil
} }
return CreateSFTPStorage(server, port, username, storageDir, authMethods, hostKeyCallback, threads) return CreateSFTPStorage(server, port, username, storageDir, minimumNesting, authMethods, hostKeyCallback, threads)
} }
func CreateSFTPStorage(server string, port int, username string, storageDir string, func CreateSFTPStorage(server string, port int, username string, storageDir string, minimumNesting int,
authMethods []ssh.AuthMethod, authMethods []ssh.AuthMethod,
hostKeyCallback func(hostname string, remote net.Addr, hostKeyCallback func(hostname string, remote net.Addr,
key ssh.PublicKey) error, threads int) (storage *SFTPStorage, err error) { key ssh.PublicKey) error, threads int) (storage *SFTPStorage, err error) {
@@ -82,6 +83,7 @@ func CreateSFTPStorage(server string, port int, username string, storageDir stri
storage = &SFTPStorage{ storage = &SFTPStorage{
client: client, client: client,
storageDir: storageDir, storageDir: storageDir,
minimumNesting: minimumNesting,
numberOfThreads: threads, numberOfThreads: threads,
} }
@@ -184,11 +186,8 @@ func (storage *SFTPStorage) FindChunk(threadIndex int, chunkID string, isFossil
suffix = ".fsl" suffix = ".fsl"
} }
// The minimum level of directories to dive into before searching for the chunk file.
minimumLevel := 2
for level := 0; level*2 < len(chunkID); level++ { for level := 0; level*2 < len(chunkID); level++ {
if level >= minimumLevel { if level >= storage.minimumNesting {
filePath = path.Join(dir, chunkID[2*level:]) + suffix filePath = path.Join(dir, chunkID[2*level:]) + suffix
if stat, err := storage.client.Stat(filePath); err == nil && !stat.IsDir() { if stat, err := storage.client.Stat(filePath); err == nil && !stat.IsDir() {
return filePath[len(storage.storageDir)+1:], true, stat.Size(), nil return filePath[len(storage.storageDir)+1:], true, stat.Size(), nil
@@ -205,7 +204,7 @@ func (storage *SFTPStorage) FindChunk(threadIndex int, chunkID string, isFossil
continue continue
} }
if level < minimumLevel { if level < storage.minimumNesting {
// Create the subdirectory if is doesn't exist. // Create the subdirectory if is doesn't exist.
if err == nil && !stat.IsDir() { if err == nil && !stat.IsDir() {
@@ -225,7 +224,7 @@ func (storage *SFTPStorage) FindChunk(threadIndex int, chunkID string, isFossil
continue continue
} }
// Teh chunk must be under this subdirectory but it doesn't exist. // The chunk must be under this subdirectory but it doesn't exist.
return path.Join(dir, chunkID[2*level:])[len(storage.storageDir)+1:] + suffix, false, 0, nil return path.Join(dir, chunkID[2*level:])[len(storage.storageDir)+1:] + suffix, false, 0, nil
} }

4
src/duplicacy_snapshotmanager_test.go
View File

@@ -95,14 +95,14 @@ func createTestSnapshotManager(testDir string) *SnapshotManager {
os.RemoveAll(testDir) os.RemoveAll(testDir)
os.MkdirAll(testDir, 0700) os.MkdirAll(testDir, 0700)
storage, _ := CreateFileStorage(testDir, 2, false, 1) storage, _ := CreateFileStorage(testDir, 1, false, 1)
storage.CreateDirectory(0, "chunks") storage.CreateDirectory(0, "chunks")
storage.CreateDirectory(0, "snapshots") storage.CreateDirectory(0, "snapshots")
config := CreateConfig() config := CreateConfig()
snapshotManager := CreateSnapshotManager(config, storage) snapshotManager := CreateSnapshotManager(config, storage)
cacheDir := path.Join(testDir, "cache") cacheDir := path.Join(testDir, "cache")
snapshotCache, _ := CreateFileStorage(cacheDir, 2, false, 1) snapshotCache, _ := CreateFileStorage(cacheDir, 1, false, 1)
snapshotCache.CreateDirectory(0, "chunks") snapshotCache.CreateDirectory(0, "chunks")
snapshotCache.CreateDirectory(0, "snapshots") snapshotCache.CreateDirectory(0, "snapshots")

8
src/duplicacy_storage.go
View File

@@ -146,7 +146,7 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
} }
if isFileStorage { if isFileStorage {
fileStorage, err := CreateFileStorage(storageURL, 2, isCacheNeeded, threads) fileStorage, err := CreateFileStorage(storageURL, 1, isCacheNeeded, threads)
if err != nil { if err != nil {
LOG_ERROR("STORAGE_CREATE", "Failed to load the file storage at %s: %v", storageURL, err) LOG_ERROR("STORAGE_CREATE", "Failed to load the file storage at %s: %v", storageURL, err)
return nil return nil
@@ -164,7 +164,7 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
} }
if strings.HasPrefix(storageURL, "samba://") { if strings.HasPrefix(storageURL, "samba://") {
fileStorage, err := CreateFileStorage(storageURL[8:], 2, true, threads) fileStorage, err := CreateFileStorage(storageURL[8:], 1, true, threads)
if err != nil { if err != nil {
LOG_ERROR("STORAGE_CREATE", "Failed to load the file storage at %s: %v", storageURL, err) LOG_ERROR("STORAGE_CREATE", "Failed to load the file storage at %s: %v", storageURL, err)
return nil return nil
@@ -311,7 +311,7 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
return checkHostKey(hostname, remote, key) return checkHostKey(hostname, remote, key)
} }
sftpStorage, err := CreateSFTPStorage(server, port, username, storageDir, authMethods, hostKeyChecker, threads) sftpStorage, err := CreateSFTPStorage(server, port, username, storageDir, 1, authMethods, hostKeyChecker, threads)
if err != nil { if err != nil {
LOG_ERROR("STORAGE_CREATE", "Failed to load the SFTP storage at %s: %v", storageURL, err) LOG_ERROR("STORAGE_CREATE", "Failed to load the SFTP storage at %s: %v", storageURL, err)
return nil return nil
@@ -373,7 +373,7 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
} else if matched[1] == "dropbox" { } else if matched[1] == "dropbox" {
storageDir := matched[3] + matched[5] storageDir := matched[3] + matched[5]
token := GetPassword(preference, "dropbox_token", "Enter Dropbox access token:", true, resetPassword) token := GetPassword(preference, "dropbox_token", "Enter Dropbox access token:", true, resetPassword)
dropboxStorage, err := CreateDropboxStorage(token, storageDir, threads) dropboxStorage, err := CreateDropboxStorage(token, storageDir, 1, threads)
if err != nil { if err != nil {
LOG_ERROR("STORAGE_CREATE", "Failed to load the dropbox storage: %v", err) LOG_ERROR("STORAGE_CREATE", "Failed to load the dropbox storage: %v", err)
return nil return nil

8
src/duplicacy_storage_test.go
View File

@@ -41,7 +41,7 @@ func init() {
func loadStorage(localStoragePath string, threads int) (Storage, error) { func loadStorage(localStoragePath string, threads int) (Storage, error) {
if testStorageName == "" || testStorageName == "file" { if testStorageName == "" || testStorageName == "file" {
return CreateFileStorage(localStoragePath, 2, false, threads) return CreateFileStorage(localStoragePath, 1, false, threads)
} }
config, err := ioutil.ReadFile("test_storage.conf") config, err := ioutil.ReadFile("test_storage.conf")
@@ -64,10 +64,10 @@ func loadStorage(localStoragePath string, threads int) (Storage, error) {
if testStorageName == "flat" { if testStorageName == "flat" {
return CreateFileStorage(localStoragePath, 0, false, threads) return CreateFileStorage(localStoragePath, 0, false, threads)
} else if testStorageName == "samba" { } else if testStorageName == "samba" {
return CreateFileStorage(localStoragePath, 2, true, threads) return CreateFileStorage(localStoragePath, 1, true, threads)
} else if testStorageName == "sftp" { } else if testStorageName == "sftp" {
port, _ := strconv.Atoi(storage["port"]) port, _ := strconv.Atoi(storage["port"])
return CreateSFTPStorageWithPassword(storage["server"], port, storage["username"], storage["directory"], storage["password"], threads) return CreateSFTPStorageWithPassword(storage["server"], port, storage["username"], storage["directory"], 1, storage["password"], threads)
} else if testStorageName == "s3" || testStorageName == "wasabi" { } else if testStorageName == "s3" || testStorageName == "wasabi" {
return CreateS3Storage(storage["region"], storage["endpoint"], storage["bucket"], storage["directory"], storage["access_key"], storage["secret_key"], threads, true, false) return CreateS3Storage(storage["region"], storage["endpoint"], storage["bucket"], storage["directory"], storage["access_key"], storage["secret_key"], threads, true, false)
} else if testStorageName == "s3c" { } else if testStorageName == "s3c" {
@@ -77,7 +77,7 @@ func loadStorage(localStoragePath string, threads int) (Storage, error) {
} else if testStorageName == "minios" { } else if testStorageName == "minios" {
return CreateS3Storage(storage["region"], storage["endpoint"], storage["bucket"], storage["directory"], storage["access_key"], storage["secret_key"], threads, true, true) return CreateS3Storage(storage["region"], storage["endpoint"], storage["bucket"], storage["directory"], storage["access_key"], storage["secret_key"], threads, true, true)
} else if testStorageName == "dropbox" { } else if testStorageName == "dropbox" {
return CreateDropboxStorage(storage["token"], storage["directory"], threads) return CreateDropboxStorage(storage["token"], storage["directory"], 1, threads)
} else if testStorageName == "b2" { } else if testStorageName == "b2" {
return CreateB2Storage(storage["account"], storage["key"], storage["bucket"], threads) return CreateB2Storage(storage["account"], storage["key"], storage["bucket"], threads)
} else if testStorageName == "gcs-s3" { } else if testStorageName == "gcs-s3" {