diff --git a/src/duplicacy_dropboxstorage.go b/src/duplicacy_dropboxstorage.go index 5cbe592..b9c76fa 100644 --- a/src/duplicacy_dropboxstorage.go +++ b/src/duplicacy_dropboxstorage.go @@ -15,12 +15,13 @@ import ( type DropboxStorage struct { RateLimitedStorage - clients []*dropbox.Files - storageDir string + clients []*dropbox.Files + minimumNesting int // The minimum level of directories to dive into before searching for the chunk file. + storageDir string } // CreateDropboxStorage creates a dropbox storage object. -func CreateDropboxStorage(accessToken string, storageDir string, threads int) (storage *DropboxStorage, err error) { +func CreateDropboxStorage(accessToken string, storageDir string, minimumNesting int, threads int) (storage *DropboxStorage, err error) { var clients []*dropbox.Files for i := 0; i < threads; i++ { @@ -37,8 +38,9 @@ func CreateDropboxStorage(accessToken string, storageDir string, threads int) (s } storage = &DropboxStorage{ - clients: clients, - storageDir: storageDir, + clients: clients, + storageDir: storageDir, + minimumNesting: minimumNesting, } err = storage.CreateDirectory(0, "") @@ -189,11 +191,8 @@ func (storage *DropboxStorage) FindChunk(threadIndex int, chunkID string, isFoss suffix = ".fsl" } - // The minimum level of directories to dive into before searching for the chunk file. - minimumLevel := 1 - for level := 0; level*2 < len(chunkID); level++ { - if level >= minimumLevel { + if level >= storage.minimumNesting { filePath = path.Join(dir, chunkID[2*level:]) + suffix var size int64 exist, _, size, err = storage.GetFileInfo(threadIndex, filePath) @@ -217,7 +216,7 @@ func (storage *DropboxStorage) FindChunk(threadIndex int, chunkID string, isFoss continue } - if level < minimumLevel { + if level < storage.minimumNesting { // Create the subdirectory if it doesn't exist. err = storage.CreateDirectory(threadIndex, subDir) if err != nil { diff --git a/src/duplicacy_filestorage.go b/src/duplicacy_filestorage.go index dc82ce2..76289ce 100644 --- a/src/duplicacy_filestorage.go +++ b/src/duplicacy_filestorage.go @@ -18,14 +18,14 @@ import ( type FileStorage struct { RateLimitedStorage - minimumLevel int // The minimum level of directories to dive into before searching for the chunk file. + minimumNesting int // The minimum level of directories to dive into before searching for the chunk file. isCacheNeeded bool // Network storages require caching storageDir string numberOfThreads int } // CreateFileStorage creates a file storage. -func CreateFileStorage(storageDir string, minimumLevel int, isCacheNeeded bool, threads int) (storage *FileStorage, err error) { +func CreateFileStorage(storageDir string, minimumNesting int, isCacheNeeded bool, threads int) (storage *FileStorage, err error) { var stat os.FileInfo @@ -51,7 +51,7 @@ func CreateFileStorage(storageDir string, minimumLevel int, isCacheNeeded bool, storage = &FileStorage{ storageDir: storageDir, - minimumLevel: minimumLevel, + minimumNesting: minimumNesting, isCacheNeeded: isCacheNeeded, numberOfThreads: threads, } @@ -137,7 +137,7 @@ func (storage *FileStorage) FindChunk(threadIndex int, chunkID string, isFossil } for level := 0; level*2 < len(chunkID); level++ { - if level >= storage.minimumLevel { + if level >= storage.minimumNesting { filePath = path.Join(dir, chunkID[2*level:]) + suffix // Use Lstat() instead of Stat() since 1) Stat() doesn't work for deduplicated disks on Windows and 2) there isn't // really a need to follow the link if filePath is a link. @@ -159,7 +159,7 @@ func (storage *FileStorage) FindChunk(threadIndex int, chunkID string, isFossil continue } - if level < storage.minimumLevel { + if level < storage.minimumNesting { // Create the subdirectory if it doesn't exist. if err == nil && !stat.IsDir() { diff --git a/src/duplicacy_sftpstorage.go b/src/duplicacy_sftpstorage.go index 6503f09..bf457d9 100644 --- a/src/duplicacy_sftpstorage.go +++ b/src/duplicacy_sftpstorage.go @@ -22,12 +22,13 @@ type SFTPStorage struct { RateLimitedStorage client *sftp.Client + minimumNesting int // The minimum level of directories to dive into before searching for the chunk file. storageDir string numberOfThreads int } func CreateSFTPStorageWithPassword(server string, port int, username string, storageDir string, - password string, threads int) (storage *SFTPStorage, err error) { + minimumNesting int, password string, threads int) (storage *SFTPStorage, err error) { authMethods := []ssh.AuthMethod{ssh.Password(password)} @@ -36,10 +37,10 @@ func CreateSFTPStorageWithPassword(server string, port int, username string, sto return nil } - return CreateSFTPStorage(server, port, username, storageDir, authMethods, hostKeyCallback, threads) + return CreateSFTPStorage(server, port, username, storageDir, minimumNesting, authMethods, hostKeyCallback, threads) } -func CreateSFTPStorage(server string, port int, username string, storageDir string, +func CreateSFTPStorage(server string, port int, username string, storageDir string, minimumNesting int, authMethods []ssh.AuthMethod, hostKeyCallback func(hostname string, remote net.Addr, key ssh.PublicKey) error, threads int) (storage *SFTPStorage, err error) { @@ -82,6 +83,7 @@ func CreateSFTPStorage(server string, port int, username string, storageDir stri storage = &SFTPStorage{ client: client, storageDir: storageDir, + minimumNesting: minimumNesting, numberOfThreads: threads, } @@ -184,11 +186,8 @@ func (storage *SFTPStorage) FindChunk(threadIndex int, chunkID string, isFossil suffix = ".fsl" } - // The minimum level of directories to dive into before searching for the chunk file. - minimumLevel := 2 - for level := 0; level*2 < len(chunkID); level++ { - if level >= minimumLevel { + if level >= storage.minimumNesting { filePath = path.Join(dir, chunkID[2*level:]) + suffix if stat, err := storage.client.Stat(filePath); err == nil && !stat.IsDir() { return filePath[len(storage.storageDir)+1:], true, stat.Size(), nil @@ -205,7 +204,7 @@ func (storage *SFTPStorage) FindChunk(threadIndex int, chunkID string, isFossil continue } - if level < minimumLevel { + if level < storage.minimumNesting { // Create the subdirectory if is doesn't exist. if err == nil && !stat.IsDir() { @@ -225,7 +224,7 @@ func (storage *SFTPStorage) FindChunk(threadIndex int, chunkID string, isFossil continue } - // Teh chunk must be under this subdirectory but it doesn't exist. + // The chunk must be under this subdirectory but it doesn't exist. return path.Join(dir, chunkID[2*level:])[len(storage.storageDir)+1:] + suffix, false, 0, nil } diff --git a/src/duplicacy_storage.go b/src/duplicacy_storage.go index c147a6b..9cd6fe4 100644 --- a/src/duplicacy_storage.go +++ b/src/duplicacy_storage.go @@ -313,7 +313,7 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor return checkHostKey(hostname, remote, key) } - sftpStorage, err := CreateSFTPStorage(server, port, username, storageDir, authMethods, hostKeyChecker, threads) + sftpStorage, err := CreateSFTPStorage(server, port, username, storageDir, 2, authMethods, hostKeyChecker, threads) if err != nil { LOG_ERROR("STORAGE_CREATE", "Failed to load the SFTP storage at %s: %v", storageURL, err) return nil @@ -375,7 +375,7 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor } else if matched[1] == "dropbox" { storageDir := matched[3] + matched[5] token := GetPassword(preference, "dropbox_token", "Enter Dropbox access token:", true, resetPassword) - dropboxStorage, err := CreateDropboxStorage(token, storageDir, threads) + dropboxStorage, err := CreateDropboxStorage(token, storageDir, 1, threads) if err != nil { LOG_ERROR("STORAGE_CREATE", "Failed to load the dropbox storage: %v", err) return nil diff --git a/src/duplicacy_storage_test.go b/src/duplicacy_storage_test.go index 4b3629b..174c6cf 100644 --- a/src/duplicacy_storage_test.go +++ b/src/duplicacy_storage_test.go @@ -67,7 +67,7 @@ func loadStorage(localStoragePath string, threads int) (Storage, error) { return CreateFileStorage(localStoragePath, 2, true, threads) } else if testStorageName == "sftp" { port, _ := strconv.Atoi(storage["port"]) - return CreateSFTPStorageWithPassword(storage["server"], port, storage["username"], storage["directory"], storage["password"], threads) + return CreateSFTPStorageWithPassword(storage["server"], port, storage["username"], storage["directory"], 2, storage["password"], threads) } else if testStorageName == "s3" || testStorageName == "wasabi" { return CreateS3Storage(storage["region"], storage["endpoint"], storage["bucket"], storage["directory"], storage["access_key"], storage["secret_key"], threads, true, false) } else if testStorageName == "s3c" { @@ -77,7 +77,7 @@ func loadStorage(localStoragePath string, threads int) (Storage, error) { } else if testStorageName == "minios" { return CreateS3Storage(storage["region"], storage["endpoint"], storage["bucket"], storage["directory"], storage["access_key"], storage["secret_key"], threads, true, true) } else if testStorageName == "dropbox" { - return CreateDropboxStorage(storage["token"], storage["directory"], threads) + return CreateDropboxStorage(storage["token"], storage["directory"], 1, threads) } else if testStorageName == "b2" { return CreateB2Storage(storage["account"], storage["key"], storage["bucket"], threads) } else if testStorageName == "gcs-s3" {