diff --git a/src/duplicacy_acdclient.go b/src/duplicacy_acdclient.go index 99923f9..95901f8 100644 --- a/src/duplicacy_acdclient.go +++ b/src/duplicacy_acdclient.go @@ -69,7 +69,7 @@ func NewACDClient(tokenFile string) (*ACDClient, error) { func (client *ACDClient) call(url string, method string, input interface{}, contentType string) (io.ReadCloser, int64, error) { - LOG_DEBUG("ACD_CALL", "Calling %s", url) + //LOG_DEBUG("ACD_CALL", "%s %s", method, url) var response *http.Response @@ -256,7 +256,7 @@ type ACDListEntriesOutput struct { Entries []ACDEntry `json:"data"` } -func (client *ACDClient) ListEntries(parentID string, listFiles bool) ([]ACDEntry, error) { +func (client *ACDClient) ListEntries(parentID string, listFiles bool, listDirectories bool) ([]ACDEntry, error) { startToken := "" @@ -264,20 +264,22 @@ func (client *ACDClient) ListEntries(parentID string, listFiles bool) ([]ACDEntr for { - url := client.MetadataURL + "nodes/" + parentID + "/children?filters=" + url := client.MetadataURL + "nodes/" + parentID + "/children?" - if listFiles { - url += "kind:FILE" - } else { - url += "kind:FOLDER" + if listFiles && !listDirectories { + url += "filters=kind:FILE&" + } else if !listFiles && listDirectories { + url += "filters=kind:FOLDER&" } if startToken != "" { - url += "&startToken=" + startToken + url += "startToken=" + startToken + "&" } if client.TestMode { - url += "&limit=8" + url += "limit=8" + } else { + url += "limit=200" } readCloser, _, err := client.call(url, "GET", 0, "") diff --git a/src/duplicacy_acdclient_test.go b/src/duplicacy_acdclient_test.go index 7b66832..54f75e2 100644 --- a/src/duplicacy_acdclient_test.go +++ b/src/duplicacy_acdclient_test.go @@ -103,7 +103,7 @@ func TestACDClient(t *testing.T) { } } - entries, err := acdClient.ListEntries(test1ID, true) + entries, err := acdClient.ListEntries(test1ID, true, false) if err != nil { t.Errorf("Error list randomly generated files: %v", err) return @@ -117,7 +117,7 @@ func TestACDClient(t *testing.T) { } } - entries, err = acdClient.ListEntries(test2ID, true) + entries, err = acdClient.ListEntries(test2ID, true, false) if err != nil { t.Errorf("Error list randomly generated files: %v", err) return diff --git a/src/duplicacy_acdstorage.go b/src/duplicacy_acdstorage.go index 36717ef..ecbd092 100644 --- a/src/duplicacy_acdstorage.go +++ b/src/duplicacy_acdstorage.go @@ -9,10 +9,11 @@ import ( "path" "strings" "sync" + "time" ) type ACDStorage struct { - RateLimitedStorage + StorageBase client *ACDClient idCache map[string]string @@ -35,11 +36,13 @@ func CreateACDStorage(tokenFile string, storagePath string, threads int) (storag numberOfThreads: threads, } - storagePathID, _, _, err := storage.getIDFromPath(0, storagePath) + storagePathID, err := storage.getIDFromPath(0, storagePath, false) if err != nil { return nil, err } + // Set 'storagePath' as the root of the storage and clean up the id cache accordingly + storage.idCache = make(map[string]string) storage.idCache[""] = storagePathID for _, dir := range []string{"chunks", "fossils", "snapshots"} { @@ -48,7 +51,6 @@ func CreateACDStorage(tokenFile string, storagePath string, threads int) (storag return nil, err } if dirID == "" { - dirID, err = client.CreateDirectory(storagePathID, dir) if err != nil { return nil, err } @@ -58,8 +60,9 @@ func CreateACDStorage(tokenFile string, storagePath string, threads int) (storag storage.idCache[dir] = dirID } + storage.DerivedStorage = storage + storage.SetDefaultNestingLevels([]int{0}, 0) return storage, nil - } func (storage *ACDStorage) getPathID(path string) string { @@ -88,6 +91,9 @@ func (storage *ACDStorage) deletePathID(path string) { storage.idCacheLock.Unlock() } +// convertFilePath converts the path for a fossil in the form of 'chunks/id.fsl' to 'fossils/id'. This is because +// ACD doesn't support file renaming. Instead, it only allows one file to be moved from one directory to another. +// By adding a layer of path conversion we're pretending that we can rename between 'chunks/id' and 'chunks/id.fsl' func (storage *ACDStorage) convertFilePath(filePath string) string { if strings.HasPrefix(filePath, "chunks/") && strings.HasSuffix(filePath, ".fsl") { return "fossils/" + filePath[len("chunks/"):len(filePath)-len(".fsl")] @@ -95,35 +101,80 @@ func (storage *ACDStorage) convertFilePath(filePath string) string { return filePath } -func (storage *ACDStorage) getIDFromPath(threadIndex int, path string) (fileID string, isDir bool, size int64, err error) { +// getIDFromPath returns the id of the given path. If 'createDirectories' is true, create the given path and all its +// parent directories if they don't exist. Note that if 'createDirectories' is false, it may return an empty 'fileID' +// if the file doesn't exist. +func (storage *ACDStorage) getIDFromPath(threadIndex int, filePath string, createDirectories bool) (fileID string, err error) { + + if fileID, ok := storage.findPathID(filePath); ok { + return fileID, nil + } parentID, ok := storage.findPathID("") if !ok { - parentID, isDir, size, err = storage.client.ListByName("", "") + parentID, _, _, err = storage.client.ListByName("", "") if err != nil { - return "", false, 0, err + return "", err } + storage.savePathID("", parentID) } - names := strings.Split(path, "/") + names := strings.Split(filePath, "/") + current := "" for i, name := range names { - parentID, isDir, _, err = storage.client.ListByName(parentID, name) - if err != nil { - return "", false, 0, err + + current = path.Join(current, name) + fileID, ok := storage.findPathID(current) + if ok { + parentID = fileID + continue } - if parentID == "" { - if i == len(names)-1 { - return "", false, 0, nil - } else { - return "", false, 0, fmt.Errorf("File path '%s' does not exist", path) + isDir := false + fileID, isDir, _, err = storage.client.ListByName(parentID, name) + if err != nil { + return "", err + } + if fileID == "" { + if !createDirectories { + return "", nil } + // Create the current directory + fileID, err = storage.client.CreateDirectory(parentID, name) + if err != nil { + // Check if the directory has been created by another thread + if e, ok := err.(ACDError); !ok || e.Status != 409 { + return "", fmt.Errorf("Failed to create directory '%s': %v", current, err) + } + // A 409 means the directory may have already created by another thread. Wait 10 seconds + // until we seed the directory. + for i := 0; i < 10; i++ { + var createErr error + fileID, isDir, _, createErr = storage.client.ListByName(parentID, name) + if createErr != nil { + return "", createErr + } + if fileID == "" { + time.Sleep(time.Second) + } else { + break + } + } + if fileID == "" { + return "", fmt.Errorf("All attempts to create directory '%s' failed: %v", current, err) + } + } else { + isDir = true + } + } else { + storage.savePathID(current, fileID) } if i != len(names)-1 && !isDir { - return "", false, 0, fmt.Errorf("Invalid path %s", path) + return "", fmt.Errorf("Path '%s' is not a directory", current) } + parentID = fileID } - return parentID, isDir, size, err + return parentID, nil } // ListFiles return the list of files and subdirectories under 'dir' (non-recursively) @@ -136,7 +187,7 @@ func (storage *ACDStorage) ListFiles(threadIndex int, dir string) ([]string, []i if dir == "snapshots" { - entries, err := storage.client.ListEntries(storage.getPathID(dir), false) + entries, err := storage.client.ListEntries(storage.getPathID(dir), false, true) if err != nil { return nil, nil, err } @@ -159,9 +210,10 @@ func (storage *ACDStorage) ListFiles(threadIndex int, dir string) ([]string, []i if pathID == "" { return nil, nil, nil } + storage.savePathID(dir, pathID) } - entries, err := storage.client.ListEntries(pathID, true) + entries, err := storage.client.ListEntries(pathID, true, false) if err != nil { return nil, nil, err } @@ -176,21 +228,33 @@ func (storage *ACDStorage) ListFiles(threadIndex int, dir string) ([]string, []i } else { files := []string{} sizes := []int64{} - for _, parent := range []string{"chunks", "fossils"} { - entries, err := storage.client.ListEntries(storage.getPathID(parent), true) + parents := []string{"chunks", "fossils"} + for i := 0; i < len(parents); i++ { + parent := parents[i] + pathID, ok := storage.findPathID(parent) + if !ok { + continue + } + entries, err := storage.client.ListEntries(pathID, true, true) if err != nil { return nil, nil, err } - for _, entry := range entries { - name := entry.Name - if parent == "fossils" { - name += ".fsl" + if entry.Kind != "FOLDER" { + name := entry.Name + if strings.HasPrefix(parent, "fossils") { + name = parent + "/" + name + ".fsl" + name = name[len("fossils/"):] + } else { + name = parent + "/" + name + name = name[len("chunks/"):] + } + files = append(files, name) + sizes = append(sizes, entry.Size) + } else { + parents = append(parents, parent+"/"+entry.Name) } - storage.savePathID(parent+"/"+entry.Name, entry.ID) - files = append(files, name) - sizes = append(sizes, entry.Size) } } return files, sizes, nil @@ -201,17 +265,13 @@ func (storage *ACDStorage) ListFiles(threadIndex int, dir string) ([]string, []i // DeleteFile deletes the file or directory at 'filePath'. func (storage *ACDStorage) DeleteFile(threadIndex int, filePath string) (err error) { filePath = storage.convertFilePath(filePath) - fileID, ok := storage.findPathID(filePath) - if !ok { - fileID, _, _, err = storage.getIDFromPath(threadIndex, filePath) - if err != nil { - return err - } - if fileID == "" { - LOG_TRACE("ACD_STORAGE", "File %s has disappeared before deletion", filePath) - return nil - } - storage.savePathID(filePath, fileID) + fileID, err := storage.getIDFromPath(threadIndex, filePath, false) + if err != nil { + return err + } + if fileID == "" { + LOG_TRACE("ACD_STORAGE", "File '%s' to be deleted does not exist", filePath) + return nil } err = storage.client.DeleteFile(fileID) @@ -232,11 +292,19 @@ func (storage *ACDStorage) MoveFile(threadIndex int, from string, to string) (er return fmt.Errorf("Attempting to rename file %s with unknown id", from) } - fromParentID := storage.getPathID("chunks") - toParentID := storage.getPathID("fossils") + fromParent := path.Dir(from) + fromParentID, err := storage.getIDFromPath(threadIndex, fromParent, false) + if err != nil { + return fmt.Errorf("Failed to retrieve the id of the parent directory '%s': %v", fromParent, err) + } + if fromParentID == "" { + return fmt.Errorf("The parent directory '%s' does not exist", fromParent) + } - if strings.HasPrefix(from, "fossils") { - fromParentID, toParentID = toParentID, fromParentID + toParent := path.Dir(to) + toParentID, err := storage.getIDFromPath(threadIndex, toParent, true) + if err != nil { + return fmt.Errorf("Failed to retrieve the id of the parent directory '%s': %v", toParent, err) } err = storage.client.MoveFile(fileID, fromParentID, toParentID) @@ -261,24 +329,25 @@ func (storage *ACDStorage) CreateDirectory(threadIndex int, dir string) (err err dir = dir[:len(dir)-1] } - if dir == "chunks" || dir == "snapshots" { - return nil + parentPath := path.Dir(dir) + if parentPath == "." { + parentPath = "" + } + parentID, ok := storage.findPathID(parentPath) + if !ok { + return fmt.Errorf("Path directory '%s' has unknown id", parentPath) } - if strings.HasPrefix(dir, "snapshots/") { - name := dir[len("snapshots/"):] - dirID, err := storage.client.CreateDirectory(storage.getPathID("snapshots"), name) - if err != nil { - if e, ok := err.(ACDError); ok && e.Status == 409 { - return nil - } else { - return err - } + name := path.Base(dir) + dirID, err := storage.client.CreateDirectory(parentID, name) + if err != nil { + if e, ok := err.(ACDError); ok && e.Status == 409 { + return nil + } else { + return err } - storage.savePathID(dir, dirID) - return nil - } + storage.savePathID(dir, dirID) return nil } @@ -291,8 +360,21 @@ func (storage *ACDStorage) GetFileInfo(threadIndex int, filePath string) (exist } filePath = storage.convertFilePath(filePath) - fileID := "" - fileID, isDir, size, err = storage.getIDFromPath(threadIndex, filePath) + + parentPath := path.Dir(filePath) + if parentPath == "." { + parentPath = "" + } + parentID, err := storage.getIDFromPath(threadIndex, parentPath, false) + if err != nil { + return false, false, 0, err + } + if parentID == "" { + return false, false, 0, nil + } + + name := path.Base(filePath) + fileID, isDir, size, err := storage.client.ListByName(parentID, name) if err != nil { return false, false, 0, err } @@ -300,43 +382,18 @@ func (storage *ACDStorage) GetFileInfo(threadIndex int, filePath string) (exist return false, false, 0, nil } + storage.savePathID(filePath, fileID) return true, isDir, size, nil } -// FindChunk finds the chunk with the specified id. If 'isFossil' is true, it will search for chunk files with -// the suffix '.fsl'. -func (storage *ACDStorage) FindChunk(threadIndex int, chunkID string, isFossil bool) (filePath string, exist bool, size int64, err error) { - parentID := "" - filePath = "chunks/" + chunkID - realPath := filePath - if isFossil { - parentID = storage.getPathID("fossils") - filePath += ".fsl" - realPath = "fossils/" + chunkID + ".fsl" - } else { - parentID = storage.getPathID("chunks") - } - - fileID := "" - fileID, _, size, err = storage.client.ListByName(parentID, chunkID) - if fileID != "" { - storage.savePathID(realPath, fileID) - } - return filePath, fileID != "", size, err -} - // DownloadFile reads the file at 'filePath' into the chunk. func (storage *ACDStorage) DownloadFile(threadIndex int, filePath string, chunk *Chunk) (err error) { - fileID, ok := storage.findPathID(filePath) - if !ok { - fileID, _, _, err = storage.getIDFromPath(threadIndex, filePath) - if err != nil { - return err - } - if fileID == "" { - return fmt.Errorf("File path '%s' does not exist", filePath) - } - storage.savePathID(filePath, fileID) + fileID, err := storage.getIDFromPath(threadIndex, filePath, false) + if err != nil { + return err + } + if fileID == "" { + return fmt.Errorf("File path '%s' does not exist", filePath) } readCloser, _, err := storage.client.DownloadFile(fileID) @@ -353,22 +410,16 @@ func (storage *ACDStorage) DownloadFile(threadIndex int, filePath string, chunk // UploadFile writes 'content' to the file at 'filePath'. func (storage *ACDStorage) UploadFile(threadIndex int, filePath string, content []byte) (err error) { parent := path.Dir(filePath) - if parent == "." { parent = "" } - parentID, ok := storage.findPathID(parent) - - if !ok { - parentID, _, _, err = storage.getIDFromPath(threadIndex, parent) - if err != nil { - return err - } - if parentID == "" { - return fmt.Errorf("File path '%s' does not exist", parent) - } - storage.savePathID(parent, parentID) + parentID, err := storage.getIDFromPath(threadIndex, parent, true) + if err != nil { + return err + } + if parentID == "" { + return fmt.Errorf("File path '%s' does not exist", parent) } fileID, err := storage.client.UploadFile(parentID, path.Base(filePath), content, storage.UploadRateLimit/storage.numberOfThreads) diff --git a/src/duplicacy_azurestorage.go b/src/duplicacy_azurestorage.go index fec4337..d21ecbc 100644 --- a/src/duplicacy_azurestorage.go +++ b/src/duplicacy_azurestorage.go @@ -12,7 +12,7 @@ import ( ) type AzureStorage struct { - RateLimitedStorage + StorageBase containers []*storage.Container } @@ -47,6 +47,8 @@ func CreateAzureStorage(accountName string, accountKey string, containers: containers, } + azureStorage.DerivedStorage = azureStorage + azureStorage.SetDefaultNestingLevels([]int{0}, 0) return } @@ -149,23 +151,6 @@ func (storage *AzureStorage) GetFileInfo(threadIndex int, filePath string) (exis return true, false, blob.Properties.ContentLength, nil } -// FindChunk finds the chunk with the specified id. If 'isFossil' is true, it will search for chunk files with -// the suffix '.fsl'. -func (storage *AzureStorage) FindChunk(threadIndex int, chunkID string, isFossil bool) (filePath string, exist bool, size int64, err error) { - filePath = "chunks/" + chunkID - if isFossil { - filePath += ".fsl" - } - - exist, _, size, err = storage.GetFileInfo(threadIndex, filePath) - - if err != nil { - return "", false, 0, err - } else { - return filePath, exist, size, err - } -} - // DownloadFile reads the file at 'filePath' into the chunk. func (storage *AzureStorage) DownloadFile(threadIndex int, filePath string, chunk *Chunk) (err error) { readCloser, err := storage.containers[threadIndex].GetBlobReference(filePath).Get(nil) diff --git a/src/duplicacy_b2storage.go b/src/duplicacy_b2storage.go index f3bf859..48b9042 100644 --- a/src/duplicacy_b2storage.go +++ b/src/duplicacy_b2storage.go @@ -9,7 +9,7 @@ import ( ) type B2Storage struct { - RateLimitedStorage + StorageBase clients []*B2Client } @@ -38,6 +38,9 @@ func CreateB2Storage(accountID string, applicationKey string, bucket string, thr storage = &B2Storage{ clients: clients, } + + storage.DerivedStorage = storage + storage.SetDefaultNestingLevels([]int{0}, 0) return storage, nil } @@ -204,17 +207,6 @@ func (storage *B2Storage) GetFileInfo(threadIndex int, filePath string) (exist b return true, false, entries[0].Size, nil } -// FindChunk finds the chunk with the specified id. If 'isFossil' is true, it will search for chunk files with -// the suffix '.fsl'. -func (storage *B2Storage) FindChunk(threadIndex int, chunkID string, isFossil bool) (filePath string, exist bool, size int64, err error) { - filePath = "chunks/" + chunkID - if isFossil { - filePath += ".fsl" - } - exist, _, size, err = storage.GetFileInfo(threadIndex, filePath) - return filePath, exist, size, err -} - // DownloadFile reads the file at 'filePath' into the chunk. func (storage *B2Storage) DownloadFile(threadIndex int, filePath string, chunk *Chunk) (err error) { diff --git a/src/duplicacy_backupmanager.go b/src/duplicacy_backupmanager.go index ccc21af..82d6f44 100644 --- a/src/duplicacy_backupmanager.go +++ b/src/duplicacy_backupmanager.go @@ -79,7 +79,7 @@ func (manager *BackupManager) SetupSnapshotCache(storageName string) bool { preferencePath := GetDuplicacyPreferencePath() cacheDir := path.Join(preferencePath, "cache", storageName) - storage, err := CreateFileStorage(cacheDir, 2, false, 1) + storage, err := CreateFileStorage(cacheDir, false, 1) if err != nil { LOG_ERROR("BACKUP_CACHE", "Failed to create the snapshot cache dir: %v", err) return false @@ -93,6 +93,7 @@ func (manager *BackupManager) SetupSnapshotCache(storageName string) bool { } } + storage.SetDefaultNestingLevels([]int{1}, 1) manager.snapshotCache = storage manager.SnapshotManager.snapshotCache = storage return true diff --git a/src/duplicacy_backupmanager_test.go b/src/duplicacy_backupmanager_test.go index 7808799..a71a6d4 100644 --- a/src/duplicacy_backupmanager_test.go +++ b/src/duplicacy_backupmanager_test.go @@ -336,6 +336,30 @@ func TestBackupManager(t *testing.T) { } } + numberOfSnapshots := backupManager.SnapshotManager.ListSnapshots( /*snapshotID*/ "host1" /*revisionsToList*/, nil /*tag*/, "" /*showFiles*/, false /*showChunks*/, false) + if numberOfSnapshots != 3 { + t.Errorf("Expected 3 snapshots but got %d", numberOfSnapshots) + } + backupManager.SnapshotManager.CheckSnapshots( /*snapshotID*/ "host1" /*revisions*/, []int{1, 2, 3} /*tag*/, "", + /*showStatistics*/ false /*showTabular*/, false /*checkFiles*/, false /*searchFossils*/, false /*resurrect*/, false) + backupManager.SnapshotManager.PruneSnapshots("host1", "host1" /*revisions*/, []int{1} /*tags*/, nil /*retentions*/, nil, + /*exhaustive*/ false /*exclusive=*/, false /*ignoredIDs*/, nil /*dryRun*/, false /*deleteOnly*/, false /*collectOnly*/, false) + numberOfSnapshots = backupManager.SnapshotManager.ListSnapshots( /*snapshotID*/ "host1" /*revisionsToList*/, nil /*tag*/, "" /*showFiles*/, false /*showChunks*/, false) + if numberOfSnapshots != 2 { + t.Errorf("Expected 2 snapshots but got %d", numberOfSnapshots) + } + backupManager.SnapshotManager.CheckSnapshots( /*snapshotID*/ "host1" /*revisions*/, []int{2, 3} /*tag*/, "", + /*showStatistics*/ false /*showTabular*/, false /*checkFiles*/, false /*searchFossils*/, false /*resurrect*/, false) + backupManager.Backup(testDir+"/repository1" /*quickMode=*/, false, threads, "fourth", false, false) + backupManager.SnapshotManager.PruneSnapshots("host1", "host1" /*revisions*/, nil /*tags*/, nil /*retentions*/, nil, + /*exhaustive*/ false /*exclusive=*/, true /*ignoredIDs*/, nil /*dryRun*/, false /*deleteOnly*/, false /*collectOnly*/, false) + numberOfSnapshots = backupManager.SnapshotManager.ListSnapshots( /*snapshotID*/ "host1" /*revisionsToList*/, nil /*tag*/, "" /*showFiles*/, false /*showChunks*/, false) + if numberOfSnapshots != 3 { + t.Errorf("Expected 3 snapshots but got %d", numberOfSnapshots) + } + backupManager.SnapshotManager.CheckSnapshots( /*snapshotID*/ "host1" /*revisions*/, []int{2, 3, 4} /*tag*/, "", + /*showStatistics*/ false /*showTabular*/, false /*checkFiles*/, false /*searchFossils*/, false /*resurrect*/, false) + /*buf := make([]byte, 1<<16) runtime.Stack(buf, true) fmt.Printf("%s", buf)*/ diff --git a/src/duplicacy_chunkdownloader.go b/src/duplicacy_chunkdownloader.go index eae28b4..ded4dee 100644 --- a/src/duplicacy_chunkdownloader.go +++ b/src/duplicacy_chunkdownloader.go @@ -326,7 +326,7 @@ func (downloader *ChunkDownloader) Download(threadIndex int, task ChunkDownloadT } const MaxDownloadAttempts = 3 - for downloadAttempt := 0;; downloadAttempt++ { + for downloadAttempt := 0; ; downloadAttempt++ { err = downloader.storage.DownloadFile(threadIndex, chunkPath, chunk) if err != nil { if err == io.ErrUnexpectedEOF && downloadAttempt < MaxDownloadAttempts { diff --git a/src/duplicacy_config.go b/src/duplicacy_config.go index 1af340d..5635562 100644 --- a/src/duplicacy_config.go +++ b/src/duplicacy_config.go @@ -46,6 +46,8 @@ type Config struct { ChunkSeed []byte `json:"chunk-seed"` + FixedNesting bool `json:"fixed-nesting"` + // Use HMAC-SHA256(hashKey, plaintext) as the chunk hash. // Use HMAC-SHA256(idKey, chunk hash) as the file name of the chunk // For chunks, use HMAC-SHA256(chunkKey, chunk hash) as the encryption key @@ -63,7 +65,7 @@ type Config struct { // for encrypting a non-chunk file FileKey []byte `json:"-"` - chunkPool chan *Chunk `json:"-"` + chunkPool chan *Chunk numberOfChunks int32 dryRun bool } @@ -148,6 +150,7 @@ func CreateConfigFromParameters(compressionLevel int, averageChunkSize int, maxi AverageChunkSize: averageChunkSize, MaximumChunkSize: maximumChunkSize, MinimumChunkSize: mininumChunkSize, + FixedNesting: true, } if isEncrypted { @@ -380,6 +383,8 @@ func DownloadConfig(storage Storage, password string) (config *Config, isEncrypt return nil, false, fmt.Errorf("Failed to parse the config file: %v", err) } + storage.SetNestingLevels(config) + return config, false, nil } diff --git a/src/duplicacy_dropboxstorage.go b/src/duplicacy_dropboxstorage.go index b9c76fa..842b52b 100644 --- a/src/duplicacy_dropboxstorage.go +++ b/src/duplicacy_dropboxstorage.go @@ -6,18 +6,17 @@ package duplicacy import ( "fmt" - "path" "strings" "github.com/gilbertchen/go-dropbox" ) type DropboxStorage struct { - RateLimitedStorage + StorageBase - clients []*dropbox.Files - minimumNesting int // The minimum level of directories to dive into before searching for the chunk file. - storageDir string + clients []*dropbox.Files + minimumNesting int // The minimum level of directories to dive into before searching for the chunk file. + storageDir string } // CreateDropboxStorage creates a dropbox storage object. @@ -38,9 +37,9 @@ func CreateDropboxStorage(accessToken string, storageDir string, minimumNesting } storage = &DropboxStorage{ - clients: clients, - storageDir: storageDir, - minimumNesting: minimumNesting, + clients: clients, + storageDir: storageDir, + minimumNesting: minimumNesting, } err = storage.CreateDirectory(0, "") @@ -48,6 +47,8 @@ func CreateDropboxStorage(accessToken string, storageDir string, minimumNesting return nil, fmt.Errorf("Can't create storage directory: %v", err) } + storage.DerivedStorage = storage + storage.SetDefaultNestingLevels([]int{1}, 1) return storage, nil } @@ -181,63 +182,6 @@ func (storage *DropboxStorage) GetFileInfo(threadIndex int, filePath string) (ex return true, output.Tag == "folder", int64(output.Size), nil } -// FindChunk finds the chunk with the specified id. If 'isFossil' is true, it will search for chunk files with -// the suffix '.fsl'. -func (storage *DropboxStorage) FindChunk(threadIndex int, chunkID string, isFossil bool) (filePath string, exist bool, size int64, err error) { - dir := "/chunks" - - suffix := "" - if isFossil { - suffix = ".fsl" - } - - for level := 0; level*2 < len(chunkID); level++ { - if level >= storage.minimumNesting { - filePath = path.Join(dir, chunkID[2*level:]) + suffix - var size int64 - exist, _, size, err = storage.GetFileInfo(threadIndex, filePath) - if err != nil { - return "", false, 0, err - } - if exist { - return filePath, exist, size, nil - } - } - - // Find the subdirectory the chunk file may reside. - subDir := path.Join(dir, chunkID[2*level:2*level+2]) - exist, _, _, err = storage.GetFileInfo(threadIndex, subDir) - if err != nil { - return "", false, 0, err - } - - if exist { - dir = subDir - continue - } - - if level < storage.minimumNesting { - // Create the subdirectory if it doesn't exist. - err = storage.CreateDirectory(threadIndex, subDir) - if err != nil { - return "", false, 0, err - } - - dir = subDir - continue - } - - // Teh chunk must be under this subdirectory but it doesn't exist. - return path.Join(dir, chunkID[2*level:])[1:] + suffix, false, 0, nil - - } - - LOG_FATAL("CHUNK_FIND", "Chunk %s is still not found after having searched a maximum level of directories", - chunkID) - return "", false, 0, nil - -} - // DownloadFile reads the file at 'filePath' into the chunk. func (storage *DropboxStorage) DownloadFile(threadIndex int, filePath string, chunk *Chunk) (err error) { diff --git a/src/duplicacy_filestorage.go b/src/duplicacy_filestorage.go index 76289ce..1fbd090 100644 --- a/src/duplicacy_filestorage.go +++ b/src/duplicacy_filestorage.go @@ -11,21 +11,21 @@ import ( "math/rand" "os" "path" + "strings" "time" ) // FileStorage is a local on-disk file storage implementing the Storage interface. type FileStorage struct { - RateLimitedStorage + StorageBase - minimumNesting int // The minimum level of directories to dive into before searching for the chunk file. isCacheNeeded bool // Network storages require caching storageDir string numberOfThreads int } // CreateFileStorage creates a file storage. -func CreateFileStorage(storageDir string, minimumNesting int, isCacheNeeded bool, threads int) (storage *FileStorage, err error) { +func CreateFileStorage(storageDir string, isCacheNeeded bool, threads int) (storage *FileStorage, err error) { var stat os.FileInfo @@ -51,7 +51,6 @@ func CreateFileStorage(storageDir string, minimumNesting int, isCacheNeeded bool storage = &FileStorage{ storageDir: storageDir, - minimumNesting: minimumNesting, isCacheNeeded: isCacheNeeded, numberOfThreads: threads, } @@ -59,6 +58,8 @@ func CreateFileStorage(storageDir string, minimumNesting int, isCacheNeeded bool // Random number fo generating the temporary chunk file suffix. rand.Seed(time.Now().UnixNano()) + storage.DerivedStorage = storage + storage.SetDefaultNestingLevels([]int{2, 3}, 2) return storage, nil } @@ -126,67 +127,6 @@ func (storage *FileStorage) GetFileInfo(threadIndex int, filePath string) (exist return true, stat.IsDir(), stat.Size(), nil } -// FindChunk finds the chunk with the specified id. If 'isFossil' is true, it will search for chunk files with the -// suffix '.fsl'. -func (storage *FileStorage) FindChunk(threadIndex int, chunkID string, isFossil bool) (filePath string, exist bool, size int64, err error) { - dir := path.Join(storage.storageDir, "chunks") - - suffix := "" - if isFossil { - suffix = ".fsl" - } - - for level := 0; level*2 < len(chunkID); level++ { - if level >= storage.minimumNesting { - filePath = path.Join(dir, chunkID[2*level:]) + suffix - // Use Lstat() instead of Stat() since 1) Stat() doesn't work for deduplicated disks on Windows and 2) there isn't - // really a need to follow the link if filePath is a link. - stat, err := os.Lstat(filePath) - if err != nil { - LOG_DEBUG("FS_FIND", "File %s can't be found: %v", filePath, err) - } else if stat.IsDir() { - return filePath[len(storage.storageDir)+1:], false, 0, fmt.Errorf("The path %s is a directory", filePath) - } else { - return filePath[len(storage.storageDir)+1:], true, stat.Size(), nil - } - } - - // Find the subdirectory the chunk file may reside. - subDir := path.Join(dir, chunkID[2*level:2*level+2]) - stat, err := os.Stat(subDir) - if err == nil && stat.IsDir() { - dir = subDir - continue - } - - if level < storage.minimumNesting { - // Create the subdirectory if it doesn't exist. - - if err == nil && !stat.IsDir() { - return "", false, 0, fmt.Errorf("The path %s is not a directory", subDir) - } - - err = os.Mkdir(subDir, 0744) - if err != nil { - // The directory may have been created by other threads so check it again. - stat, _ := os.Stat(subDir) - if stat == nil || !stat.IsDir() { - return "", false, 0, err - } - } - dir = subDir - continue - } - - // The chunk must be under this subdirectory but it doesn't exist. - return path.Join(dir, chunkID[2*level:])[len(storage.storageDir)+1:] + suffix, false, 0, nil - - } - - return "", false, 0, fmt.Errorf("The maximum level of directories searched") - -} - // DownloadFile reads the file at 'filePath' into the chunk. func (storage *FileStorage) DownloadFile(threadIndex int, filePath string, chunk *Chunk) (err error) { @@ -210,6 +150,26 @@ func (storage *FileStorage) UploadFile(threadIndex int, filePath string, content fullPath := path.Join(storage.storageDir, filePath) + if len(strings.Split(filePath, "/")) > 2 { + dir := path.Dir(fullPath) + // Use Lstat() instead of Stat() since 1) Stat() doesn't work for deduplicated disks on Windows and 2) there isn't + // really a need to follow the link if filePath is a link. + stat, err := os.Lstat(dir) + if err != nil { + if !os.IsNotExist(err) { + return err + } + err = os.MkdirAll(dir, 0744) + if err != nil { + return err + } + } else { + if !stat.IsDir() { + fmt.Errorf("The path %s is not a directory", dir) + } + } + } + letters := "abcdefghijklmnopqrstuvwxyz" suffix := make([]byte, 8) for i := range suffix { diff --git a/src/duplicacy_gcdstorage.go b/src/duplicacy_gcdstorage.go index 233104e..13c43e0 100644 --- a/src/duplicacy_gcdstorage.go +++ b/src/duplicacy_gcdstorage.go @@ -25,17 +25,18 @@ import ( ) type GCDStorage struct { - RateLimitedStorage + StorageBase service *drive.Service idCache map[string]string - idCacheLock *sync.Mutex + idCacheLock sync.Mutex backoffs []int // desired backoff time in seconds for each thread attempts []int // number of failed attempts since last success for each thread - isConnected bool - numberOfThreads int - TestMode bool + createDirectoryLock sync.Mutex + isConnected bool + numberOfThreads int + TestMode bool } type GCDConfig struct { @@ -91,7 +92,11 @@ func (storage *GCDStorage) shouldRetry(threadIndex int, err error) (bool, error) retry = err.Temporary() } - if !retry || storage.attempts[threadIndex] >= MAX_ATTEMPTS { + if !retry { + return false, err + } + + if storage.attempts[threadIndex] >= MAX_ATTEMPTS { LOG_INFO("GCD_RETRY", "[%d] Maximum number of retries reached (backoff: %d, attempts: %d)", threadIndex, storage.backoffs[threadIndex], storage.attempts[threadIndex]) storage.backoffs[threadIndex] = 1 @@ -114,6 +119,9 @@ func (storage *GCDStorage) shouldRetry(threadIndex int, err error) (bool, error) return true, nil } +// convertFilePath converts the path for a fossil in the form of 'chunks/id.fsl' to 'fossils/id'. This is because +// ACD doesn't support file renaming. Instead, it only allows one file to be moved from one directory to another. +// By adding a layer of path conversion we're pretending that we can rename between 'chunks/id' and 'chunks/id.fsl' func (storage *GCDStorage) convertFilePath(filePath string) string { if strings.HasPrefix(filePath, "chunks/") && strings.HasSuffix(filePath, ".fsl") { return "fossils/" + filePath[len("chunks/"):len(filePath)-len(".fsl")] @@ -147,7 +155,7 @@ func (storage *GCDStorage) deletePathID(path string) { storage.idCacheLock.Unlock() } -func (storage *GCDStorage) listFiles(threadIndex int, parentID string, listFiles bool) ([]*drive.File, error) { +func (storage *GCDStorage) listFiles(threadIndex int, parentID string, listFiles bool, listDirectories bool) ([]*drive.File, error) { if parentID == "" { return nil, fmt.Errorf("No parent ID provided") @@ -157,11 +165,11 @@ func (storage *GCDStorage) listFiles(threadIndex int, parentID string, listFiles startToken := "" - query := "'" + parentID + "' in parents and " - if listFiles { - query += "mimeType != 'application/vnd.google-apps.folder'" - } else { - query += "mimeType = 'application/vnd.google-apps.folder'" + query := "'" + parentID + "' in parents " + if listFiles && !listDirectories { + query += "and mimeType != 'application/vnd.google-apps.folder'" + } else if !listFiles && !listDirectories { + query += "and mimeType = 'application/vnd.google-apps.folder'" } maxCount := int64(1000) @@ -222,7 +230,14 @@ func (storage *GCDStorage) listByName(threadIndex int, parentID string, name str return file.Id, file.MimeType == "application/vnd.google-apps.folder", file.Size, nil } -func (storage *GCDStorage) getIDFromPath(threadIndex int, path string) (string, error) { +// getIDFromPath returns the id of the given path. If 'createDirectories' is true, create the given path and all its +// parent directories if they don't exist. Note that if 'createDirectories' is false, it may return an empty 'fileID' +// if the file doesn't exist. +func (storage *GCDStorage) getIDFromPath(threadIndex int, filePath string, createDirectories bool) (string, error) { + + if fileID, ok := storage.findPathID(filePath); ok { + return fileID, nil + } fileID := "root" @@ -230,22 +245,18 @@ func (storage *GCDStorage) getIDFromPath(threadIndex int, path string) (string, fileID = rootID } - names := strings.Split(path, "/") + names := strings.Split(filePath, "/") current := "" for i, name := range names { - - if len(current) == 0 { - current = name - } else { - current = current + "/" + name - } - + // Find the intermediate directory in the cache first. + current = path.Join(current, name) currentID, ok := storage.findPathID(current) if ok { fileID = currentID continue } + // Check if the directory exists. var err error var isDir bool fileID, isDir, _, err = storage.listByName(threadIndex, fileID, name) @@ -253,10 +264,30 @@ func (storage *GCDStorage) getIDFromPath(threadIndex int, path string) (string, return "", err } if fileID == "" { - return "", fmt.Errorf("Path %s doesn't exist", path) + if !createDirectories { + return "", nil + } + + // Only one thread can create the directory at a time -- GCD allows multiple directories + // to have the same name but different ids. + storage.createDirectoryLock.Lock() + err = storage.CreateDirectory(threadIndex, current) + storage.createDirectoryLock.Unlock() + + if err != nil { + return "", fmt.Errorf("Failed to create directory '%s': %v", current, err) + } + currentID, ok = storage.findPathID(current) + if !ok { + return "", fmt.Errorf("Directory '%s' created by id not found", current) + } + fileID = currentID + continue + } else { + storage.savePathID(current, fileID) } if i != len(names)-1 && !isDir { - return "", fmt.Errorf("Invalid path %s", path) + return "", fmt.Errorf("Path '%s' is not a directory", current) } } return fileID, nil @@ -275,13 +306,13 @@ func CreateGCDStorage(tokenFile string, storagePath string, threads int) (storag return nil, err } - config := oauth2.Config{ + oauth2Config := oauth2.Config{ ClientID: gcdConfig.ClientID, ClientSecret: gcdConfig.ClientSecret, Endpoint: gcdConfig.Endpoint, } - authClient := config.Client(context.Background(), &gcdConfig.Token) + authClient := oauth2Config.Client(context.Background(), &gcdConfig.Token) service, err := drive.New(authClient) if err != nil { @@ -292,7 +323,6 @@ func CreateGCDStorage(tokenFile string, storagePath string, threads int) (storag service: service, numberOfThreads: threads, idCache: make(map[string]string), - idCacheLock: &sync.Mutex{}, backoffs: make([]int, threads), attempts: make([]int, threads), } @@ -302,7 +332,7 @@ func CreateGCDStorage(tokenFile string, storagePath string, threads int) (storag storage.attempts[i] = 0 } - storagePathID, err := storage.getIDFromPath(0, storagePath) + storagePathID, err := storage.getIDFromPath(0, storagePath, false) if err != nil { return nil, err } @@ -328,8 +358,9 @@ func CreateGCDStorage(tokenFile string, storagePath string, threads int) (storag storage.isConnected = true + storage.DerivedStorage = storage + storage.SetDefaultNestingLevels([]int{0}, 0) return storage, nil - } // ListFiles return the list of files and subdirectories under 'dir' (non-recursively) @@ -340,7 +371,7 @@ func (storage *GCDStorage) ListFiles(threadIndex int, dir string) ([]string, []i if dir == "snapshots" { - files, err := storage.listFiles(threadIndex, storage.getPathID(dir), false) + files, err := storage.listFiles(threadIndex, storage.getPathID(dir), false, true) if err != nil { return nil, nil, err } @@ -353,12 +384,15 @@ func (storage *GCDStorage) ListFiles(threadIndex int, dir string) ([]string, []i } return subDirs, nil, nil } else if strings.HasPrefix(dir, "snapshots/") { - pathID, err := storage.getIDFromPath(threadIndex, dir) + pathID, err := storage.getIDFromPath(threadIndex, dir, false) if err != nil { return nil, nil, err } + if pathID == "" { + return nil, nil, fmt.Errorf("Path '%s' does not exist", dir) + } - entries, err := storage.listFiles(threadIndex, pathID, true) + entries, err := storage.listFiles(threadIndex, pathID, true, false) if err != nil { return nil, nil, err } @@ -374,20 +408,33 @@ func (storage *GCDStorage) ListFiles(threadIndex int, dir string) ([]string, []i files := []string{} sizes := []int64{} - for _, parent := range []string{"chunks", "fossils"} { - entries, err := storage.listFiles(threadIndex, storage.getPathID(parent), true) + parents := []string{"chunks", "fossils"} + for i := 0; i < len(parents); i++ { + parent := parents[i] + pathID, ok := storage.findPathID(parent) + if !ok { + continue + } + entries, err := storage.listFiles(threadIndex, pathID, true, true) if err != nil { return nil, nil, err } - for _, entry := range entries { - name := entry.Name - if parent == "fossils" { - name += ".fsl" + if entry.MimeType != "application/vnd.google-apps.folder" { + name := entry.Name + if strings.HasPrefix(parent, "fossils") { + name = parent + "/" + name + ".fsl" + name = name[len("fossils/"):] + } else { + name = parent + "/" + name + name = name[len("chunks/"):] + } + files = append(files, name) + sizes = append(sizes, entry.Size) + } else { + parents = append(parents, parent+"/"+entry.Name) } storage.savePathID(parent+"/"+entry.Name, entry.Id) - files = append(files, name) - sizes = append(sizes, entry.Size) } } return files, sizes, nil @@ -398,13 +445,10 @@ func (storage *GCDStorage) ListFiles(threadIndex int, dir string) ([]string, []i // DeleteFile deletes the file or directory at 'filePath'. func (storage *GCDStorage) DeleteFile(threadIndex int, filePath string) (err error) { filePath = storage.convertFilePath(filePath) - fileID, ok := storage.findPathID(filePath) - if !ok { - fileID, err = storage.getIDFromPath(threadIndex, filePath) - if err != nil { - LOG_TRACE("GCD_STORAGE", "Ignored file deletion error: %v", err) - return nil - } + fileID, err := storage.getIDFromPath(threadIndex, filePath, false) + if err != nil { + LOG_TRACE("GCD_STORAGE", "Ignored file deletion error: %v", err) + return nil } for { @@ -432,14 +476,22 @@ func (storage *GCDStorage) MoveFile(threadIndex int, from string, to string) (er fileID, ok := storage.findPathID(from) if !ok { - return fmt.Errorf("Attempting to rename file %s with unknown id", to) + return fmt.Errorf("Attempting to rename file %s with unknown id", from) } - fromParentID := storage.getPathID("chunks") - toParentID := storage.getPathID("fossils") + fromParent := path.Dir(from) + fromParentID, err := storage.getIDFromPath(threadIndex, fromParent, false) + if err != nil { + return fmt.Errorf("Failed to retrieve the id of the parent directory '%s': %v", fromParent, err) + } + if fromParentID == "" { + return fmt.Errorf("The parent directory '%s' does not exist", fromParent) + } - if strings.HasPrefix(from, "fossils") { - fromParentID, toParentID = toParentID, fromParentID + toParent := path.Dir(to) + toParentID, err := storage.getIDFromPath(threadIndex, toParent, true) + if err != nil { + return fmt.Errorf("Failed to retrieve the id of the parent directory '%s': %v", toParent, err) } for { @@ -458,7 +510,7 @@ func (storage *GCDStorage) MoveFile(threadIndex int, from string, to string) (er return nil } -// CreateDirectory creates a new directory. +// createDirectory creates a new directory. func (storage *GCDStorage) CreateDirectory(threadIndex int, dir string) (err error) { for len(dir) > 0 && dir[len(dir)-1] == '/' { @@ -477,13 +529,15 @@ func (storage *GCDStorage) CreateDirectory(threadIndex int, dir string) (err err return nil } - parentID := storage.getPathID("") - name := dir - - if strings.HasPrefix(dir, "snapshots/") { - parentID = storage.getPathID("snapshots") - name = dir[len("snapshots/"):] + parentDir := path.Dir(dir) + if parentDir == "." { + parentDir = "" } + parentID := storage.getPathID(parentDir) + if parentID == "" { + return fmt.Errorf("Parent directory '%s' does not exist", parentDir) + } + name := path.Base(dir) file := &drive.File{ Name: name, @@ -495,10 +549,19 @@ func (storage *GCDStorage) CreateDirectory(threadIndex int, dir string) (err err file, err = storage.service.Files.Create(file).Fields("id").Do() if retry, err := storage.shouldRetry(threadIndex, err); err == nil && !retry { break - } else if retry { - continue } else { - return err + + // Check if the directory has already been created by other thread + exist, _, _, newErr := storage.GetFileInfo(threadIndex, dir) + if newErr == nil && exist { + return nil + } + + if retry { + continue + } else { + return err + } } } @@ -511,18 +574,21 @@ func (storage *GCDStorage) GetFileInfo(threadIndex int, filePath string) (exist for len(filePath) > 0 && filePath[len(filePath)-1] == '/' { filePath = filePath[:len(filePath)-1] } + filePath = storage.convertFilePath(filePath) - // GetFileInfo is never called on a fossil fileID, ok := storage.findPathID(filePath) if !ok { dir := path.Dir(filePath) if dir == "." { dir = "" } - dirID, err := storage.getIDFromPath(threadIndex, dir) + dirID, err := storage.getIDFromPath(threadIndex, dir, false) if err != nil { return false, false, 0, err } + if dirID == "" { + return false, false, 0, nil + } fileID, isDir, size, err = storage.listByName(threadIndex, dirID, path.Base(filePath)) if fileID != "" { @@ -543,37 +609,15 @@ func (storage *GCDStorage) GetFileInfo(threadIndex int, filePath string) (exist } } -// FindChunk finds the chunk with the specified id. If 'isFossil' is true, it will search for chunk files with -// the suffix '.fsl'. -func (storage *GCDStorage) FindChunk(threadIndex int, chunkID string, isFossil bool) (filePath string, exist bool, size int64, err error) { - parentID := "" - filePath = "chunks/" + chunkID - realPath := storage.convertFilePath(filePath) - if isFossil { - parentID = storage.getPathID("fossils") - filePath += ".fsl" - } else { - parentID = storage.getPathID("chunks") - } - - fileID := "" - fileID, _, size, err = storage.listByName(threadIndex, parentID, chunkID) - if fileID != "" { - storage.savePathID(realPath, fileID) - } - return filePath, fileID != "", size, err -} - // DownloadFile reads the file at 'filePath' into the chunk. func (storage *GCDStorage) DownloadFile(threadIndex int, filePath string, chunk *Chunk) (err error) { // We never download the fossil so there is no need to convert the path - fileID, ok := storage.findPathID(filePath) - if !ok { - fileID, err = storage.getIDFromPath(threadIndex, filePath) - if err != nil { - return err - } - storage.savePathID(filePath, fileID) + fileID, err := storage.getIDFromPath(threadIndex, filePath, false) + if err != nil { + return err + } + if fileID == "" { + return fmt.Errorf("%s does not exist", filePath) } var response *http.Response @@ -605,13 +649,9 @@ func (storage *GCDStorage) UploadFile(threadIndex int, filePath string, content parent = "" } - parentID, ok := storage.findPathID(parent) - if !ok { - parentID, err = storage.getIDFromPath(threadIndex, parent) - if err != nil { - return err - } - storage.savePathID(parent, parentID) + parentID, err := storage.getIDFromPath(threadIndex, parent, true) + if err != nil { + return err } file := &drive.File{ diff --git a/src/duplicacy_gcsstorage.go b/src/duplicacy_gcsstorage.go index 5b79b9a..fc28092 100644 --- a/src/duplicacy_gcsstorage.go +++ b/src/duplicacy_gcsstorage.go @@ -24,7 +24,7 @@ import ( ) type GCSStorage struct { - RateLimitedStorage + StorageBase bucket *gcs.BucketHandle storageDir string @@ -101,8 +101,9 @@ func CreateGCSStorage(tokenFile string, bucketName string, storageDir string, th numberOfThreads: threads, } + storage.DerivedStorage = storage + storage.SetDefaultNestingLevels([]int{0}, 0) return storage, nil - } func (storage *GCSStorage) shouldRetry(backoff *int, err error) (bool, error) { @@ -238,19 +239,6 @@ func (storage *GCSStorage) GetFileInfo(threadIndex int, filePath string) (exist return true, false, attributes.Size, nil } -// FindChunk finds the chunk with the specified id. If 'isFossil' is true, it will search for chunk files with -// the suffix '.fsl'. -func (storage *GCSStorage) FindChunk(threadIndex int, chunkID string, isFossil bool) (filePath string, exist bool, size int64, err error) { - filePath = "chunks/" + chunkID - if isFossil { - filePath += ".fsl" - } - - exist, _, size, err = storage.GetFileInfo(threadIndex, filePath) - - return filePath, exist, size, err -} - // DownloadFile reads the file at 'filePath' into the chunk. func (storage *GCSStorage) DownloadFile(threadIndex int, filePath string, chunk *Chunk) (err error) { readCloser, err := storage.bucket.Object(storage.storageDir + filePath).NewReader(context.Background()) diff --git a/src/duplicacy_hubicstorage.go b/src/duplicacy_hubicstorage.go index 2420466..5773fb3 100644 --- a/src/duplicacy_hubicstorage.go +++ b/src/duplicacy_hubicstorage.go @@ -10,7 +10,7 @@ import ( ) type HubicStorage struct { - RateLimitedStorage + StorageBase client *HubicClient storageDir string @@ -64,8 +64,9 @@ func CreateHubicStorage(tokenFile string, storagePath string, threads int) (stor } } + storage.DerivedStorage = storage + storage.SetDefaultNestingLevels([]int{0}, 0) return storage, nil - } // ListFiles return the list of files and subdirectories under 'dir' (non-recursively) @@ -158,18 +159,6 @@ func (storage *HubicStorage) GetFileInfo(threadIndex int, filePath string) (exis return storage.client.GetFileInfo(storage.storageDir + "/" + filePath) } -// FindChunk finds the chunk with the specified id. If 'isFossil' is true, it will search for chunk files with -// the suffix '.fsl'. -func (storage *HubicStorage) FindChunk(threadIndex int, chunkID string, isFossil bool) (filePath string, exist bool, size int64, err error) { - filePath = "chunks/" + chunkID - if isFossil { - filePath += ".fsl" - } - - exist, _, size, err = storage.client.GetFileInfo(storage.storageDir + "/" + filePath) - return filePath, exist, size, err -} - // DownloadFile reads the file at 'filePath' into the chunk. func (storage *HubicStorage) DownloadFile(threadIndex int, filePath string, chunk *Chunk) (err error) { readCloser, _, err := storage.client.DownloadFile(storage.storageDir + "/" + filePath) diff --git a/src/duplicacy_onestorage.go b/src/duplicacy_onestorage.go index 5d4baac..769ef24 100644 --- a/src/duplicacy_onestorage.go +++ b/src/duplicacy_onestorage.go @@ -11,7 +11,7 @@ import ( ) type OneDriveStorage struct { - RateLimitedStorage + StorageBase client *OneDriveClient storageDir string @@ -65,10 +65,19 @@ func CreateOneDriveStorage(tokenFile string, storagePath string, threads int) (s } } + storage.DerivedStorage = storage + storage.SetDefaultNestingLevels([]int{0}, 0) return storage, nil } +func (storage *OneDriveStorage) convertFilePath(filePath string) string { + if strings.HasPrefix(filePath, "chunks/") && strings.HasSuffix(filePath, ".fsl") { + return "fossils/" + filePath[len("chunks/"):len(filePath)-len(".fsl")] + } + return filePath +} + // ListFiles return the list of files and subdirectories under 'dir' (non-recursively) func (storage *OneDriveStorage) ListFiles(threadIndex int, dir string) ([]string, []int64, error) { for len(dir) > 0 && dir[len(dir)-1] == '/' { @@ -105,19 +114,29 @@ func (storage *OneDriveStorage) ListFiles(threadIndex int, dir string) ([]string } else { files := []string{} sizes := []int64{} - for _, parent := range []string{"chunks", "fossils"} { + parents := []string{"chunks", "fossils"} + for i := 0; i < len(parents); i++ { + parent := parents[i] entries, err := storage.client.ListEntries(storage.storageDir + "/" + parent) if err != nil { return nil, nil, err } for _, entry := range entries { - name := entry.Name - if parent == "fossils" { - name += ".fsl" + if len(entry.Folder) == 0 { + name := entry.Name + if strings.HasPrefix(parent, "fossils") { + name = parent + "/" + name + ".fsl" + name = name[len("fossils/"):] + } else { + name = parent + "/" + name + name = name[len("chunks/"):] + } + files = append(files, name) + sizes = append(sizes, entry.Size) + } else { + parents = append(parents, parent+"/"+entry.Name) } - files = append(files, name) - sizes = append(sizes, entry.Size) } } return files, sizes, nil @@ -127,9 +146,7 @@ func (storage *OneDriveStorage) ListFiles(threadIndex int, dir string) ([]string // DeleteFile deletes the file or directory at 'filePath'. func (storage *OneDriveStorage) DeleteFile(threadIndex int, filePath string) (err error) { - if strings.HasSuffix(filePath, ".fsl") && strings.HasPrefix(filePath, "chunks/") { - filePath = "fossils/" + filePath[len("chunks/"):len(filePath)-len(".fsl")] - } + filePath = storage.convertFilePath(filePath) err = storage.client.DeleteFile(storage.storageDir + "/" + filePath) if e, ok := err.(OneDriveError); ok && e.Status == 404 { @@ -141,14 +158,11 @@ func (storage *OneDriveStorage) DeleteFile(threadIndex int, filePath string) (er // MoveFile renames the file. func (storage *OneDriveStorage) MoveFile(threadIndex int, from string, to string) (err error) { - fromPath := storage.storageDir + "/" + from - toParent := storage.storageDir + "/fossils" - if strings.HasSuffix(from, ".fsl") { - fromPath = storage.storageDir + "/fossils/" + from[len("chunks/"):len(from)-len(".fsl")] - toParent = storage.storageDir + "/chunks" - } - err = storage.client.MoveFile(fromPath, toParent) + fromPath := storage.storageDir + "/" + storage.convertFilePath(from) + toPath := storage.storageDir + "/" + storage.convertFilePath(to) + + err = storage.client.MoveFile(fromPath, path.Dir(toPath)) if err != nil { if e, ok := err.(OneDriveError); ok && e.Status == 409 { LOG_DEBUG("ONEDRIVE_MOVE", "Ignore 409 conflict error") @@ -180,24 +194,13 @@ func (storage *OneDriveStorage) GetFileInfo(threadIndex int, filePath string) (e for len(filePath) > 0 && filePath[len(filePath)-1] == '/' { filePath = filePath[:len(filePath)-1] } + + filePath = storage.convertFilePath(filePath) + fileID, isDir, size, err := storage.client.GetFileInfo(storage.storageDir + "/" + filePath) return fileID != "", isDir, size, err } -// FindChunk finds the chunk with the specified id. If 'isFossil' is true, it will search for chunk files with -// the suffix '.fsl'. -func (storage *OneDriveStorage) FindChunk(threadIndex int, chunkID string, isFossil bool) (filePath string, exist bool, size int64, err error) { - filePath = "chunks/" + chunkID - realPath := storage.storageDir + "/" + filePath - if isFossil { - filePath += ".fsl" - realPath = storage.storageDir + "/fossils/" + chunkID - } - - fileID, _, size, err := storage.client.GetFileInfo(realPath) - return filePath, fileID != "", size, err -} - // DownloadFile reads the file at 'filePath' into the chunk. func (storage *OneDriveStorage) DownloadFile(threadIndex int, filePath string, chunk *Chunk) (err error) { readCloser, _, err := storage.client.DownloadFile(storage.storageDir + "/" + filePath) diff --git a/src/duplicacy_s3cstorage.go b/src/duplicacy_s3cstorage.go index 9f44801..76014cc 100644 --- a/src/duplicacy_s3cstorage.go +++ b/src/duplicacy_s3cstorage.go @@ -13,7 +13,7 @@ import ( // S3CStorage is a storage backend for s3 compatible storages that require V2 Signing. type S3CStorage struct { - RateLimitedStorage + StorageBase buckets []*s3.Bucket storageDir string @@ -58,6 +58,8 @@ func CreateS3CStorage(regionName string, endpoint string, bucketName string, sto storageDir: storageDir, } + storage.DerivedStorage = storage + storage.SetDefaultNestingLevels([]int{0}, 0) return storage, nil } @@ -154,25 +156,6 @@ func (storage *S3CStorage) GetFileInfo(threadIndex int, filePath string) (exist } } -// FindChunk finds the chunk with the specified id. If 'isFossil' is true, it will search for chunk files with -// the suffix '.fsl'. -func (storage *S3CStorage) FindChunk(threadIndex int, chunkID string, isFossil bool) (filePath string, exist bool, size int64, err error) { - - filePath = "chunks/" + chunkID - if isFossil { - filePath += ".fsl" - } - - exist, _, size, err = storage.GetFileInfo(threadIndex, filePath) - - if err != nil { - return "", false, 0, err - } else { - return filePath, exist, size, err - } - -} - // DownloadFile reads the file at 'filePath' into the chunk. func (storage *S3CStorage) DownloadFile(threadIndex int, filePath string, chunk *Chunk) (err error) { diff --git a/src/duplicacy_s3storage.go b/src/duplicacy_s3storage.go index 1b01d32..01d5e9f 100644 --- a/src/duplicacy_s3storage.go +++ b/src/duplicacy_s3storage.go @@ -16,7 +16,7 @@ import ( ) type S3Storage struct { - RateLimitedStorage + StorageBase client *s3.S3 bucket string @@ -53,7 +53,7 @@ func CreateS3Storage(regionName string, endpoint string, bucketName string, stor } } - config := &aws.Config{ + s3Config := &aws.Config{ Region: aws.String(regionName), Credentials: auth, Endpoint: aws.String(endpoint), @@ -66,12 +66,14 @@ func CreateS3Storage(regionName string, endpoint string, bucketName string, stor } storage = &S3Storage{ - client: s3.New(session.New(config)), + client: s3.New(session.New(s3Config)), bucket: bucketName, storageDir: storageDir, numberOfThreads: threads, } + storage.DerivedStorage = storage + storage.SetDefaultNestingLevels([]int{0}, 0) return storage, nil } @@ -188,25 +190,6 @@ func (storage *S3Storage) GetFileInfo(threadIndex int, filePath string) (exist b } } -// FindChunk finds the chunk with the specified id. If 'isFossil' is true, it will search for chunk files with -// the suffix '.fsl'. -func (storage *S3Storage) FindChunk(threadIndex int, chunkID string, isFossil bool) (filePath string, exist bool, size int64, err error) { - - filePath = "chunks/" + chunkID - if isFossil { - filePath += ".fsl" - } - - exist, _, size, err = storage.GetFileInfo(threadIndex, filePath) - - if err != nil { - return "", false, 0, err - } else { - return filePath, exist, size, err - } - -} - // DownloadFile reads the file at 'filePath' into the chunk. func (storage *S3Storage) DownloadFile(threadIndex int, filePath string, chunk *Chunk) (err error) { diff --git a/src/duplicacy_sftpstorage.go b/src/duplicacy_sftpstorage.go index bf457d9..1a75cec 100644 --- a/src/duplicacy_sftpstorage.go +++ b/src/duplicacy_sftpstorage.go @@ -12,6 +12,7 @@ import ( "os" "path" "runtime" + "strings" "time" "github.com/pkg/sftp" @@ -19,10 +20,10 @@ import ( ) type SFTPStorage struct { - RateLimitedStorage + StorageBase client *sftp.Client - minimumNesting int // The minimum level of directories to dive into before searching for the chunk file. + minimumNesting int // The minimum level of directories to dive into before searching for the chunk file. storageDir string numberOfThreads int } @@ -45,18 +46,18 @@ func CreateSFTPStorage(server string, port int, username string, storageDir stri hostKeyCallback func(hostname string, remote net.Addr, key ssh.PublicKey) error, threads int) (storage *SFTPStorage, err error) { - config := &ssh.ClientConfig{ + sftpConfig := &ssh.ClientConfig{ User: username, Auth: authMethods, HostKeyCallback: hostKeyCallback, } if server == "sftp.hidrive.strato.com" { - config.Ciphers = []string{"aes128-cbc", "aes128-ctr", "aes256-ctr"} + sftpConfig.Ciphers = []string{"aes128-cbc", "aes128-ctr", "aes256-ctr"} } serverAddress := fmt.Sprintf("%s:%d", server, port) - connection, err := ssh.Dial("tcp", serverAddress, config) + connection, err := ssh.Dial("tcp", serverAddress, sftpConfig) if err != nil { return nil, err } @@ -92,6 +93,8 @@ func CreateSFTPStorage(server string, port int, username string, storageDir stri runtime.SetFinalizer(storage, CloseSFTPStorage) + storage.DerivedStorage = storage + storage.SetDefaultNestingLevels([]int{2, 3}, 2) return storage, nil } @@ -176,64 +179,6 @@ func (storage *SFTPStorage) GetFileInfo(threadIndex int, filePath string) (exist return true, fileInfo.IsDir(), fileInfo.Size(), nil } -// FindChunk finds the chunk with the specified id. If 'isFossil' is true, it will search for chunk files with -// the suffix '.fsl'. -func (storage *SFTPStorage) FindChunk(threadIndex int, chunkID string, isFossil bool) (filePath string, exist bool, size int64, err error) { - dir := path.Join(storage.storageDir, "chunks") - - suffix := "" - if isFossil { - suffix = ".fsl" - } - - for level := 0; level*2 < len(chunkID); level++ { - if level >= storage.minimumNesting { - filePath = path.Join(dir, chunkID[2*level:]) + suffix - if stat, err := storage.client.Stat(filePath); err == nil && !stat.IsDir() { - return filePath[len(storage.storageDir)+1:], true, stat.Size(), nil - } else if err == nil && stat.IsDir() { - return filePath[len(storage.storageDir)+1:], true, 0, fmt.Errorf("The path %s is a directory", filePath) - } - } - - // Find the subdirectory the chunk file may reside. - subDir := path.Join(dir, chunkID[2*level:2*level+2]) - stat, err := storage.client.Stat(subDir) - if err == nil && stat.IsDir() { - dir = subDir - continue - } - - if level < storage.minimumNesting { - // Create the subdirectory if is doesn't exist. - - if err == nil && !stat.IsDir() { - return "", false, 0, fmt.Errorf("The path %s is not a directory", subDir) - } - - err = storage.client.Mkdir(subDir) - if err != nil { - // The directory may have been created by other threads so check it again. - stat, _ := storage.client.Stat(subDir) - if stat == nil || !stat.IsDir() { - return "", false, 0, fmt.Errorf("Failed to create the directory %s: %v", subDir, err) - } - } - - dir = subDir - continue - } - - // The chunk must be under this subdirectory but it doesn't exist. - return path.Join(dir, chunkID[2*level:])[len(storage.storageDir)+1:] + suffix, false, 0, nil - - } - - LOG_FATAL("CHUNK_FIND", "Chunk %s is still not found after having searched a maximum level of directories", - chunkID) - return "", false, 0, nil -} - // DownloadFile reads the file at 'filePath' into the chunk. func (storage *SFTPStorage) DownloadFile(threadIndex int, filePath string, chunk *Chunk) (err error) { file, err := storage.client.Open(path.Join(storage.storageDir, filePath)) @@ -255,6 +200,30 @@ func (storage *SFTPStorage) UploadFile(threadIndex int, filePath string, content fullPath := path.Join(storage.storageDir, filePath) + dirs := strings.Split(filePath, "/") + if len(dirs) > 1 { + fullDir := path.Dir(fullPath) + _, err := storage.client.Stat(fullDir) + if err != nil { + // The error may be caused by a non-existent fullDir, or a broken connection. In either case, + // we just assume it is the former because there isn't a way to tell which is the case. + for i, _ := range dirs[1 : len(dirs)-1] { + subDir := path.Join(storage.storageDir, path.Join(dirs[0:i+2]...)) + // We don't check the error; just keep going blindly but always store the last err + err = storage.client.Mkdir(subDir) + } + + // If there is an error creating the dirs, we check fullDir one more time, because another thread + // may happen to create the same fullDir ahead of this thread + if err != nil { + _, err := storage.client.Stat(fullDir) + if err != nil { + return err + } + } + } + } + letters := "abcdefghijklmnopqrstuvwxyz" suffix := make([]byte, 8) for i := range suffix { @@ -301,7 +270,14 @@ func (storage *SFTPStorage) IsMoveFileImplemented() bool { return true } func (storage *SFTPStorage) IsStrongConsistent() bool { return true } // If the storage supports fast listing of files names. -func (storage *SFTPStorage) IsFastListing() bool { return false } +func (storage *SFTPStorage) IsFastListing() bool { + for _, level := range storage.readLevels { + if level > 1 { + return false + } + } + return true +} // Enable the test mode. func (storage *SFTPStorage) EnableTestMode() {} diff --git a/src/duplicacy_snapshotmanager.go b/src/duplicacy_snapshotmanager.go index f1fc279..82db18d 100644 --- a/src/duplicacy_snapshotmanager.go +++ b/src/duplicacy_snapshotmanager.go @@ -592,24 +592,6 @@ func (manager *SnapshotManager) ListAllFiles(storage Storage, top string) (allFi allSizes = append(allSizes, sizes[i]) } } - - if !manager.config.dryRun { - if top == "chunks/" { - // We're listing all chunks so this is the perfect place to detect if a directory contains too many - // chunks. Create sub-directories if necessary - if len(files) > 1024 && !storage.IsFastListing() { - for i := 0; i < 256; i++ { - subdir := dir + fmt.Sprintf("%02x\n", i) - manager.storage.CreateDirectory(0, subdir) - } - } - } else { - // Remove chunk sub-directories that are empty - if len(files) == 0 && strings.HasPrefix(dir, "chunks/") && dir != "chunks/" { - storage.DeleteFile(0, dir) - } - } - } } return allFiles, allSizes diff --git a/src/duplicacy_snapshotmanager_test.go b/src/duplicacy_snapshotmanager_test.go index 6324150..b9e95a6 100644 --- a/src/duplicacy_snapshotmanager_test.go +++ b/src/duplicacy_snapshotmanager_test.go @@ -95,14 +95,14 @@ func createTestSnapshotManager(testDir string) *SnapshotManager { os.RemoveAll(testDir) os.MkdirAll(testDir, 0700) - storage, _ := CreateFileStorage(testDir, 2, false, 1) + storage, _ := CreateFileStorage(testDir, false, 1) storage.CreateDirectory(0, "chunks") storage.CreateDirectory(0, "snapshots") config := CreateConfig() snapshotManager := CreateSnapshotManager(config, storage) cacheDir := path.Join(testDir, "cache") - snapshotCache, _ := CreateFileStorage(cacheDir, 2, false, 1) + snapshotCache, _ := CreateFileStorage(cacheDir, false, 1) snapshotCache.CreateDirectory(0, "chunks") snapshotCache.CreateDirectory(0, "snapshots") diff --git a/src/duplicacy_storage.go b/src/duplicacy_storage.go index 63da10c..017f7df 100644 --- a/src/duplicacy_storage.go +++ b/src/duplicacy_storage.go @@ -5,6 +5,7 @@ package duplicacy import ( + "encoding/json" "fmt" "io/ioutil" "net" @@ -20,8 +21,10 @@ import ( ) type Storage interface { - // ListFiles return the list of files and subdirectories under 'dir' (non-recursively) - ListFiles(threadIndex int, dir string) (files []string, size []int64, err error) + // ListFiles return the list of files and subdirectories under 'dir'. A subdirectories returned must have a trailing '/', with + // a size of 0. If 'dir' is 'snapshots', only subdirectories will be returned. If 'dir' is 'snapshots/repository_id', then only + // files will be returned. If 'dir' is 'chunks', the implementation can return the list either recusively or non-recusively. + ListFiles(threadIndex int, dir string) (files []string, sizes []int64, err error) // DeleteFile deletes the file or directory at 'filePath'. DeleteFile(threadIndex int, filePath string) (err error) @@ -45,6 +48,9 @@ type Storage interface { // UploadFile writes 'content' to the file at 'filePath'. UploadFile(threadIndex int, filePath string, content []byte) (err error) + // SetNestingLevels sets up the chunk nesting structure. + SetNestingLevels(config *Config) + // If a local snapshot cache is needed for the storage to avoid downloading/uploading chunks too often when // managing snapshots. IsCacheNeeded() bool @@ -65,16 +71,97 @@ type Storage interface { SetRateLimits(downloadRateLimit int, uploadRateLimit int) } -type RateLimitedStorage struct { - DownloadRateLimit int - UploadRateLimit int +// StorageBase is the base struct from which all storages are derived from +type StorageBase struct { + DownloadRateLimit int // Maximum download rate (bytes/seconds) + UploadRateLimit int // Maximum upload reate (bytes/seconds) + + DerivedStorage Storage // Used as the pointer to the derived storage class + + readLevels []int // At which nesting level to find the chunk with the given id + writeLevel int // Store the uploaded chunk to this level } -func (storage *RateLimitedStorage) SetRateLimits(downloadRateLimit int, uploadRateLimit int) { +// SetRateLimits sets the maximum download and upload rates +func (storage *StorageBase) SetRateLimits(downloadRateLimit int, uploadRateLimit int) { storage.DownloadRateLimit = downloadRateLimit storage.UploadRateLimit = uploadRateLimit } +// SetDefaultNestingLevels sets the default read and write levels. This is usually called by +// derived storages to set the levels with old values so that storages initialied by ealier versions +// will continue to work. +func (storage *StorageBase) SetDefaultNestingLevels(readLevels []int, writeLevel int) { + storage.readLevels = readLevels + storage.writeLevel = writeLevel +} + +// SetNestingLevels sets the new read and write levels (normally both at 1) if the 'config' file has +// the 'fixed-nesting' key, or if a file named 'nesting' exists on the storage. +func (storage *StorageBase) SetNestingLevels(config *Config) { + + // 'FixedNesting' is true only for the 'config' file with the new format (2.0.10+) + if config.FixedNesting { + + storage.readLevels = nil + + // Check if the 'nesting' file exist + exist, _, _, err := storage.DerivedStorage.GetFileInfo(0, "nesting") + if err == nil && exist { + nestingFile := CreateChunk(CreateConfig(), true) + if storage.DerivedStorage.DownloadFile(0, "config", nestingFile) == nil { + var nesting struct { + ReadLevels []int `json:"read-levels"` + WriteLevel int `json:"write-level"` + } + if json.Unmarshal(nestingFile.GetBytes(), &nesting) == nil { + storage.readLevels = nesting.ReadLevels + storage.writeLevel = nesting.WriteLevel + } + } + } + + if len(storage.readLevels) == 0 { + storage.readLevels = []int{1} + storage.writeLevel = 1 + } + } + + LOG_DEBUG("STORAGE_NESTING", "Chunk read levels: %v, write level: %d", storage.readLevels, storage.writeLevel) + for _, level := range storage.readLevels { + if storage.writeLevel == level { + return + } + } + LOG_ERROR("STORAGE_NESTING", "The write level %d isn't in the read levels %v", storage.readLevels, storage.writeLevel) +} + +// FindChunk finds the chunk with the specified id at the levels one by one as specified by 'readLevels'. +func (storage *StorageBase) FindChunk(threadIndex int, chunkID string, isFossil bool) (filePath string, exist bool, size int64, err error) { + chunkPaths := make([]string, 0) + for _, level := range storage.readLevels { + chunkPath := "chunks/" + for i := 0; i < level; i++ { + chunkPath += chunkID[2*i:2*i+2] + "/" + } + chunkPath += chunkID[2*level:] + if isFossil { + chunkPath += ".fsl" + } + exist, _, size, err = storage.DerivedStorage.GetFileInfo(threadIndex, chunkPath) + if err == nil && exist { + return chunkPath, exist, size, err + } + chunkPaths = append(chunkPaths, chunkPath) + } + for i, level := range storage.readLevels { + if storage.writeLevel == level { + return chunkPaths[i], false, 0, nil + } + } + return "", false, 0, fmt.Errorf("Invalid chunk nesting setup") +} + func checkHostKey(hostname string, remote net.Addr, key ssh.PublicKey) error { if preferencePath == "" { @@ -148,7 +235,7 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor } if isFileStorage { - fileStorage, err := CreateFileStorage(storageURL, 2, isCacheNeeded, threads) + fileStorage, err := CreateFileStorage(storageURL, isCacheNeeded, threads) if err != nil { LOG_ERROR("STORAGE_CREATE", "Failed to load the file storage at %s: %v", storageURL, err) return nil @@ -157,7 +244,7 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor } if strings.HasPrefix(storageURL, "flat://") { - fileStorage, err := CreateFileStorage(storageURL[7:], 0, false, threads) + fileStorage, err := CreateFileStorage(storageURL[7:], false, threads) if err != nil { LOG_ERROR("STORAGE_CREATE", "Failed to load the file storage at %s: %v", storageURL, err) return nil @@ -166,7 +253,7 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor } if strings.HasPrefix(storageURL, "samba://") { - fileStorage, err := CreateFileStorage(storageURL[8:], 2, true, threads) + fileStorage, err := CreateFileStorage(storageURL[8:], true, threads) if err != nil { LOG_ERROR("STORAGE_CREATE", "Failed to load the file storage at %s: %v", storageURL, err) return nil diff --git a/src/duplicacy_storage_test.go b/src/duplicacy_storage_test.go index 174c6cf..44bf7ed 100644 --- a/src/duplicacy_storage_test.go +++ b/src/duplicacy_storage_test.go @@ -15,7 +15,6 @@ import ( "path" "runtime/debug" "strconv" - "strings" "testing" "time" @@ -41,61 +40,100 @@ func init() { func loadStorage(localStoragePath string, threads int) (Storage, error) { if testStorageName == "" || testStorageName == "file" { - return CreateFileStorage(localStoragePath, 2, false, threads) + storage, err := CreateFileStorage(localStoragePath, false, threads) + if storage != nil { + // Use a read level of at least 2 because this will catch more errors than a read level of 1. + storage.SetDefaultNestingLevels([]int{2, 3}, 2) + } + return storage, err } - config, err := ioutil.ReadFile("test_storage.conf") + description, err := ioutil.ReadFile("test_storage.conf") if err != nil { return nil, err } - storages := make(map[string]map[string]string) + configs := make(map[string]map[string]string) - err = json.Unmarshal(config, &storages) + err = json.Unmarshal(description, &configs) if err != nil { return nil, err } - storage, found := storages[testStorageName] + config, found := configs[testStorageName] if !found { return nil, fmt.Errorf("No storage named '%s' found", testStorageName) } if testStorageName == "flat" { - return CreateFileStorage(localStoragePath, 0, false, threads) + storage, err := CreateFileStorage(localStoragePath, false, threads) + storage.SetDefaultNestingLevels([]int{2, 3}, 2) + return storage, err } else if testStorageName == "samba" { - return CreateFileStorage(localStoragePath, 2, true, threads) + storage, err := CreateFileStorage(localStoragePath, true, threads) + storage.SetDefaultNestingLevels([]int{2, 3}, 2) + return storage, err } else if testStorageName == "sftp" { - port, _ := strconv.Atoi(storage["port"]) - return CreateSFTPStorageWithPassword(storage["server"], port, storage["username"], storage["directory"], 2, storage["password"], threads) + port, _ := strconv.Atoi(config["port"]) + storage, err := CreateSFTPStorageWithPassword(config["server"], port, config["username"], config["directory"], 2, config["password"], threads) + storage.SetDefaultNestingLevels([]int{2, 3}, 2) + return storage, err } else if testStorageName == "s3" || testStorageName == "wasabi" { - return CreateS3Storage(storage["region"], storage["endpoint"], storage["bucket"], storage["directory"], storage["access_key"], storage["secret_key"], threads, true, false) + storage, err := CreateS3Storage(config["region"], config["endpoint"], config["bucket"], config["directory"], config["access_key"], config["secret_key"], threads, true, false) + storage.SetDefaultNestingLevels([]int{2, 3}, 2) + return storage, err } else if testStorageName == "s3c" { - return CreateS3CStorage(storage["region"], storage["endpoint"], storage["bucket"], storage["directory"], storage["access_key"], storage["secret_key"], threads) + storage, err := CreateS3CStorage(config["region"], config["endpoint"], config["bucket"], config["directory"], config["access_key"], config["secret_key"], threads) + storage.SetDefaultNestingLevels([]int{2, 3}, 2) + return storage, err } else if testStorageName == "minio" { - return CreateS3Storage(storage["region"], storage["endpoint"], storage["bucket"], storage["directory"], storage["access_key"], storage["secret_key"], threads, false, true) + storage, err := CreateS3Storage(config["region"], config["endpoint"], config["bucket"], config["directory"], config["access_key"], config["secret_key"], threads, false, true) + storage.SetDefaultNestingLevels([]int{2, 3}, 2) + return storage, err } else if testStorageName == "minios" { - return CreateS3Storage(storage["region"], storage["endpoint"], storage["bucket"], storage["directory"], storage["access_key"], storage["secret_key"], threads, true, true) + storage, err := CreateS3Storage(config["region"], config["endpoint"], config["bucket"], config["directory"], config["access_key"], config["secret_key"], threads, true, true) + storage.SetDefaultNestingLevels([]int{2, 3}, 2) + return storage, err } else if testStorageName == "dropbox" { - return CreateDropboxStorage(storage["token"], storage["directory"], 1, threads) + storage, err := CreateDropboxStorage(config["token"], config["directory"], 1, threads) + storage.SetDefaultNestingLevels([]int{2, 3}, 2) + return storage, err } else if testStorageName == "b2" { - return CreateB2Storage(storage["account"], storage["key"], storage["bucket"], threads) + storage, err := CreateB2Storage(config["account"], config["key"], config["bucket"], threads) + storage.SetDefaultNestingLevels([]int{2, 3}, 2) + return storage, err } else if testStorageName == "gcs-s3" { - return CreateS3Storage(storage["region"], storage["endpoint"], storage["bucket"], storage["directory"], storage["access_key"], storage["secret_key"], threads, true, false) + storage, err := CreateS3Storage(config["region"], config["endpoint"], config["bucket"], config["directory"], config["access_key"], config["secret_key"], threads, true, false) + storage.SetDefaultNestingLevels([]int{2, 3}, 2) + return storage, err } else if testStorageName == "gcs" { - return CreateGCSStorage(storage["token_file"], storage["bucket"], storage["directory"], threads) + storage, err := CreateGCSStorage(config["token_file"], config["bucket"], config["directory"], threads) + storage.SetDefaultNestingLevels([]int{2, 3}, 2) + return storage, err } else if testStorageName == "gcs-sa" { - return CreateGCSStorage(storage["token_file"], storage["bucket"], storage["directory"], threads) + storage, err := CreateGCSStorage(config["token_file"], config["bucket"], config["directory"], threads) + storage.SetDefaultNestingLevels([]int{2, 3}, 2) + return storage, err } else if testStorageName == "azure" { - return CreateAzureStorage(storage["account"], storage["key"], storage["container"], threads) + storage, err := CreateAzureStorage(config["account"], config["key"], config["container"], threads) + storage.SetDefaultNestingLevels([]int{2, 3}, 2) + return storage, err } else if testStorageName == "acd" { - return CreateACDStorage(storage["token_file"], storage["storage_path"], threads) + storage, err := CreateACDStorage(config["token_file"], config["storage_path"], threads) + storage.SetDefaultNestingLevels([]int{2, 3}, 2) + return storage, err } else if testStorageName == "gcd" { - return CreateGCDStorage(storage["token_file"], storage["storage_path"], threads) + storage, err := CreateGCDStorage(config["token_file"], config["storage_path"], threads) + storage.SetDefaultNestingLevels([]int{2, 3}, 2) + return storage, err } else if testStorageName == "one" { - return CreateOneDriveStorage(storage["token_file"], storage["storage_path"], threads) + storage, err := CreateOneDriveStorage(config["token_file"], config["storage_path"], threads) + storage.SetDefaultNestingLevels([]int{2, 3}, 2) + return storage, err } else if testStorageName == "hubic" { - return CreateHubicStorage(storage["token_file"], storage["storage_path"], threads) + storage, err := CreateHubicStorage(config["token_file"], config["storage_path"], threads) + storage.SetDefaultNestingLevels([]int{2, 3}, 2) + return storage, err } else { return nil, fmt.Errorf("Invalid storage named: %s", testStorageName) } @@ -266,6 +304,33 @@ func TestStorage(t *testing.T) { storage.CreateDirectory(0, "snapshots/repository1") storage.CreateDirectory(0, "snapshots/repository2") + + storage.CreateDirectory(0, "shared") + + // Upload to the same directory by multiple goroutines + count := 8 + finished := make(chan int, count) + for i := 0; i < count; i++ { + go func(name string) { + err := storage.UploadFile(0, name, []byte("this is a test file")) + if err != nil { + t.Errorf("Error to upload '%s': %v", name, err) + } + finished <- 0 + }(fmt.Sprintf("shared/a/b/c/%d", i)) + } + + for i := 0; i < count; i++ { + <-finished + } + + for i := 0; i < count; i++ { + storage.DeleteFile(0, fmt.Sprintf("shared/a/b/c/%d", i)) + } + storage.DeleteFile(0, "shared/a/b/c") + storage.DeleteFile(0, "shared/a/b") + storage.DeleteFile(0, "shared/a") + time.Sleep(time.Duration(delay) * time.Second) { @@ -338,7 +403,7 @@ func TestStorage(t *testing.T) { } } - numberOfFiles := 20 + numberOfFiles := 10 maxFileSize := 64 * 1024 if testQuickMode { @@ -374,15 +439,7 @@ func TestStorage(t *testing.T) { t.Errorf("Failed to upload the file %s: %v", filePath, err) return } - LOG_INFO("STORAGE_CHUNK", "Uploaded chunk: %s, size: %d", chunkID, len(content)) - } - - allChunks := []string{} - for _, file := range listChunks(storage) { - file = strings.Replace(file, "/", "", -1) - if len(file) == 64 { - allChunks = append(allChunks, file) - } + LOG_INFO("STORAGE_CHUNK", "Uploaded chunk: %s, size: %d", filePath, len(content)) } LOG_INFO("STORAGE_FOSSIL", "Making %s a fossil", chunks[0]) @@ -412,7 +469,7 @@ func TestStorage(t *testing.T) { t.Errorf("Error downloading file %s: %v", filePath, err) continue } - LOG_INFO("STORAGE_CHUNK", "Downloaded chunk: %s, size: %d", chunkID, chunk.GetLength()) + LOG_INFO("STORAGE_CHUNK", "Downloaded chunk: %s, size: %d", filePath, chunk.GetLength()) } hasher := sha256.New() @@ -447,6 +504,11 @@ func TestStorage(t *testing.T) { } } + allChunks := []string{} + for _, file := range listChunks(storage) { + allChunks = append(allChunks, file) + } + for _, file := range allChunks { err = storage.DeleteFile(0, "chunks/"+file)