Implement new chunk directory structure

src/duplicacy_acdclient.go

@@ -69,7 +69,7 @@ func NewACDClient(tokenFile string) (*ACDClient, error) {
 
 func (client *ACDClient) call(url string, method string, input interface{}, contentType string) (io.ReadCloser, int64, error) {
 
-	LOG_DEBUG("ACD_CALL", "Calling %s", url)
+	//LOG_DEBUG("ACD_CALL", "%s %s", method, url)
 
 	var response *http.Response
 
@@ -256,7 +256,7 @@ type ACDListEntriesOutput struct {
 	Entries   []ACDEntry `json:"data"`
 }
 
-func (client *ACDClient) ListEntries(parentID string, listFiles bool) ([]ACDEntry, error) {
+func (client *ACDClient) ListEntries(parentID string, listFiles bool, listDirectories bool) ([]ACDEntry, error) {
 
 	startToken := ""
 
@@ -264,20 +264,22 @@ func (client *ACDClient) ListEntries(parentID string, listFiles bool) ([]ACDEntr
 
 	for {
 
-		url := client.MetadataURL + "nodes/" + parentID + "/children?filters="
+		url := client.MetadataURL + "nodes/" + parentID + "/children?"
 
-		if listFiles {
-			url += "kind:FILE"
-		} else {
-			url += "kind:FOLDER"
+		if listFiles && !listDirectories {
+			url += "filters=kind:FILE&"
+		} else if !listFiles && listDirectories {
+			url += "filters=kind:FOLDER&"
 		}
 
 		if startToken != "" {
-			url += "&startToken=" + startToken
+			url += "startToken=" + startToken + "&"
 		}
 
 		if client.TestMode {
-			url += "&limit=8"
+			url += "limit=8"
+		} else {
+			url += "limit=200"
 		}
 
 		readCloser, _, err := client.call(url, "GET", 0, "")

src/duplicacy_acdclient_test.go

@@ -103,7 +103,7 @@ func TestACDClient(t *testing.T) {
 		}
 	}
 
-	entries, err := acdClient.ListEntries(test1ID, true)
+	entries, err := acdClient.ListEntries(test1ID, true, false)
 	if err != nil {
 		t.Errorf("Error list randomly generated files: %v", err)
 		return
@@ -117,7 +117,7 @@ func TestACDClient(t *testing.T) {
 		}
 	}
 
-	entries, err = acdClient.ListEntries(test2ID, true)
+	entries, err = acdClient.ListEntries(test2ID, true, false)
 	if err != nil {
 		t.Errorf("Error list randomly generated files: %v", err)
 		return

src/duplicacy_acdstorage.go

@@ -9,10 +9,11 @@ import (
 	"path"
 	"strings"
 	"sync"
+	"time"
 )
 
 type ACDStorage struct {
-	RateLimitedStorage
+	StorageBase
 
 	client          *ACDClient
 	idCache         map[string]string
@@ -35,11 +36,13 @@ func CreateACDStorage(tokenFile string, storagePath string, threads int) (storag
 		numberOfThreads: threads,
 	}
 
-	storagePathID, _, _, err := storage.getIDFromPath(0, storagePath)
+	storagePathID, err := storage.getIDFromPath(0, storagePath, false)
 	if err != nil {
 		return nil, err
 	}
 
+	// Set 'storagePath' as the root of the storage and clean up the id cache accordingly
+	storage.idCache = make(map[string]string)
 	storage.idCache[""] = storagePathID
 
 	for _, dir := range []string{"chunks", "fossils", "snapshots"} {
@@ -48,7 +51,6 @@ func CreateACDStorage(tokenFile string, storagePath string, threads int) (storag
 			return nil, err
 		}
 		if dirID == "" {
-			dirID, err = client.CreateDirectory(storagePathID, dir)
 			if err != nil {
 				return nil, err
 			}
@@ -58,8 +60,9 @@ func CreateACDStorage(tokenFile string, storagePath string, threads int) (storag
 		storage.idCache[dir] = dirID
 	}
 
+	storage.DerivedStorage = storage
+	storage.SetDefaultNestingLevels([]int{0}, 0)
 	return storage, nil
-
 }
 
 func (storage *ACDStorage) getPathID(path string) string {
@@ -88,6 +91,9 @@ func (storage *ACDStorage) deletePathID(path string) {
 	storage.idCacheLock.Unlock()
 }
 
+// convertFilePath converts the path for a fossil in the form of 'chunks/id.fsl' to 'fossils/id'.  This is because
+// ACD doesn't support file renaming. Instead, it only allows one file to be moved from one directory to another.
+// By adding a layer of path conversion we're pretending that we can rename between 'chunks/id' and 'chunks/id.fsl'
 func (storage *ACDStorage) convertFilePath(filePath string) string {
 	if strings.HasPrefix(filePath, "chunks/") && strings.HasSuffix(filePath, ".fsl") {
 		return "fossils/" + filePath[len("chunks/"):len(filePath)-len(".fsl")]
@@ -95,35 +101,80 @@ func (storage *ACDStorage) convertFilePath(filePath string) string {
 	return filePath
 }
 
-func (storage *ACDStorage) getIDFromPath(threadIndex int, path string) (fileID string, isDir bool, size int64, err error) {
+// getIDFromPath returns the id of the given path.  If 'createDirectories' is true, create the given path and all its
+// parent directories if they don't exist.  Note that if 'createDirectories' is false, it may return an empty 'fileID'
+// if the file doesn't exist.
+func (storage *ACDStorage) getIDFromPath(threadIndex int, filePath string, createDirectories bool) (fileID string, err error) {
+
+	if fileID, ok := storage.findPathID(filePath); ok {
+		return fileID, nil
+	}
 
 	parentID, ok := storage.findPathID("")
 	if !ok {
-		parentID, isDir, size, err = storage.client.ListByName("", "")
+		parentID, _, _, err = storage.client.ListByName("", "")
 		if err != nil {
-			return "", false, 0, err
+			return "", err
 		}
+		storage.savePathID("", parentID)
 	}
 
-	names := strings.Split(path, "/")
+	names := strings.Split(filePath, "/")
+	current := ""
 	for i, name := range names {
-		parentID, isDir, _, err = storage.client.ListByName(parentID, name)
-		if err != nil {
-			return "", false, 0, err
+
+		current = path.Join(current, name)
+		fileID, ok := storage.findPathID(current)
+		if ok {
+			parentID = fileID
+			continue
 		}
-		if parentID == "" {
-			if i == len(names)-1 {
-				return "", false, 0, nil
-			} else {
-				return "", false, 0, fmt.Errorf("File path '%s' does not exist", path)
+		isDir := false
+		fileID, isDir, _, err = storage.client.ListByName(parentID, name)
+		if err != nil {
+			return "", err
+		}
+		if fileID == "" {
+			if !createDirectories {
+				return "", nil
 			}
+			// Create the current directory
+			fileID, err = storage.client.CreateDirectory(parentID, name)
+			if err != nil {
+				// Check if the directory has been created by another thread
+				if e, ok := err.(ACDError); !ok || e.Status != 409 {
+					return "", fmt.Errorf("Failed to create directory '%s': %v", current, err)
+				}
+				// A 409 means the directory may have already created by another thread.  Wait 10 seconds
+				// until we seed the directory.
+				for i := 0; i < 10; i++ {
+					var createErr error
+					fileID, isDir, _, createErr = storage.client.ListByName(parentID, name)
+					if createErr != nil {
+						return "", createErr
+					}
+					if fileID == "" {
+						time.Sleep(time.Second)
+					} else {
+						break
+					}
+				}
+				if fileID == "" {
+					return "", fmt.Errorf("All attempts to create directory '%s' failed: %v", current, err)
+				}
+			} else {
+				isDir = true
+			}
+		} else {
+			storage.savePathID(current, fileID)
 		}
 		if i != len(names)-1 && !isDir {
-			return "", false, 0, fmt.Errorf("Invalid path %s", path)
+			return "", fmt.Errorf("Path '%s' is not a directory", current)
 		}
+		parentID = fileID
 	}
 
-	return parentID, isDir, size, err
+	return parentID, nil
 }
 
 // ListFiles return the list of files and subdirectories under 'dir' (non-recursively)
@@ -136,7 +187,7 @@ func (storage *ACDStorage) ListFiles(threadIndex int, dir string) ([]string, []i
 
 	if dir == "snapshots" {
 
-		entries, err := storage.client.ListEntries(storage.getPathID(dir), false)
+		entries, err := storage.client.ListEntries(storage.getPathID(dir), false, true)
 		if err != nil {
 			return nil, nil, err
 		}
@@ -159,9 +210,10 @@ func (storage *ACDStorage) ListFiles(threadIndex int, dir string) ([]string, []i
 			if pathID == "" {
 				return nil, nil, nil
 			}
+			storage.savePathID(dir, pathID)
 		}
 
-		entries, err := storage.client.ListEntries(pathID, true)
+		entries, err := storage.client.ListEntries(pathID, true, false)
 		if err != nil {
 			return nil, nil, err
 		}
@@ -176,21 +228,33 @@ func (storage *ACDStorage) ListFiles(threadIndex int, dir string) ([]string, []i
 	} else {
 		files := []string{}
 		sizes := []int64{}
-		for _, parent := range []string{"chunks", "fossils"} {
-			entries, err := storage.client.ListEntries(storage.getPathID(parent), true)
+		parents := []string{"chunks", "fossils"}
+		for i := 0; i < len(parents); i++ {
+			parent := parents[i]
+			pathID, ok := storage.findPathID(parent)
+			if !ok {
+				continue
+			}
+			entries, err := storage.client.ListEntries(pathID, true, true)
 			if err != nil {
 				return nil, nil, err
 			}
-
 			for _, entry := range entries {
-				name := entry.Name
-				if parent == "fossils" {
-					name += ".fsl"
+				if entry.Kind != "FOLDER" {
+					name := entry.Name
+					if strings.HasPrefix(parent, "fossils") {
+						name = parent + "/" + name + ".fsl"
+						name = name[len("fossils/"):]
+					} else {
+						name = parent + "/" + name
+						name = name[len("chunks/"):]
+					}
+					files = append(files, name)
+					sizes = append(sizes, entry.Size)
+				} else {
+					parents = append(parents, parent+"/"+entry.Name)
 				}
-
 				storage.savePathID(parent+"/"+entry.Name, entry.ID)
-				files = append(files, name)
-				sizes = append(sizes, entry.Size)
 			}
 		}
 		return files, sizes, nil
@@ -201,17 +265,13 @@ func (storage *ACDStorage) ListFiles(threadIndex int, dir string) ([]string, []i
 // DeleteFile deletes the file or directory at 'filePath'.
 func (storage *ACDStorage) DeleteFile(threadIndex int, filePath string) (err error) {
 	filePath = storage.convertFilePath(filePath)
-	fileID, ok := storage.findPathID(filePath)
-	if !ok {
-		fileID, _, _, err = storage.getIDFromPath(threadIndex, filePath)
-		if err != nil {
-			return err
-		}
-		if fileID == "" {
-			LOG_TRACE("ACD_STORAGE", "File %s has disappeared before deletion", filePath)
-			return nil
-		}
-		storage.savePathID(filePath, fileID)
+	fileID, err := storage.getIDFromPath(threadIndex, filePath, false)
+	if err != nil {
+		return err
+	}
+	if fileID == "" {
+		LOG_TRACE("ACD_STORAGE", "File '%s' to be deleted does not exist", filePath)
+		return nil
 	}
 
 	err = storage.client.DeleteFile(fileID)
@@ -232,11 +292,19 @@ func (storage *ACDStorage) MoveFile(threadIndex int, from string, to string) (er
 		return fmt.Errorf("Attempting to rename file %s with unknown id", from)
 	}
 
-	fromParentID := storage.getPathID("chunks")
-	toParentID := storage.getPathID("fossils")
+	fromParent := path.Dir(from)
+	fromParentID, err := storage.getIDFromPath(threadIndex, fromParent, false)
+	if err != nil {
+		return fmt.Errorf("Failed to retrieve the id of the parent directory '%s': %v", fromParent, err)
+	}
+	if fromParentID == "" {
+		return fmt.Errorf("The parent directory '%s' does not exist", fromParent)
+	}
 
-	if strings.HasPrefix(from, "fossils") {
-		fromParentID, toParentID = toParentID, fromParentID
+	toParent := path.Dir(to)
+	toParentID, err := storage.getIDFromPath(threadIndex, toParent, true)
+	if err != nil {
+		return fmt.Errorf("Failed to retrieve the id of the parent directory '%s': %v", toParent, err)
 	}
 
 	err = storage.client.MoveFile(fileID, fromParentID, toParentID)
@@ -261,24 +329,25 @@ func (storage *ACDStorage) CreateDirectory(threadIndex int, dir string) (err err
 		dir = dir[:len(dir)-1]
 	}
 
-	if dir == "chunks" || dir == "snapshots" {
-		return nil
+	parentPath := path.Dir(dir)
+	if parentPath == "." {
+		parentPath = ""
+	}
+	parentID, ok := storage.findPathID(parentPath)
+	if !ok {
+		return fmt.Errorf("Path directory '%s' has unknown id", parentPath)
 	}
 
-	if strings.HasPrefix(dir, "snapshots/") {
-		name := dir[len("snapshots/"):]
-		dirID, err := storage.client.CreateDirectory(storage.getPathID("snapshots"), name)
-		if err != nil {
-			if e, ok := err.(ACDError); ok && e.Status == 409 {
-				return nil
-			} else {
-				return err
-			}
+	name := path.Base(dir)
+	dirID, err := storage.client.CreateDirectory(parentID, name)
+	if err != nil {
+		if e, ok := err.(ACDError); ok && e.Status == 409 {
+			return nil
+		} else {
+			return err
 		}
-		storage.savePathID(dir, dirID)
-		return nil
-
 	}
+	storage.savePathID(dir, dirID)
 
 	return nil
 }
@@ -291,8 +360,21 @@ func (storage *ACDStorage) GetFileInfo(threadIndex int, filePath string) (exist
 	}
 
 	filePath = storage.convertFilePath(filePath)
-	fileID := ""
-	fileID, isDir, size, err = storage.getIDFromPath(threadIndex, filePath)
+
+	parentPath := path.Dir(filePath)
+	if parentPath == "." {
+		parentPath = ""
+	}
+	parentID, err := storage.getIDFromPath(threadIndex, parentPath, false)
+	if err != nil {
+		return false, false, 0, err
+	}
+	if parentID == "" {
+		return false, false, 0, nil
+	}
+
+	name := path.Base(filePath)
+	fileID, isDir, size, err := storage.client.ListByName(parentID, name)
 	if err != nil {
 		return false, false, 0, err
 	}
@@ -300,43 +382,18 @@ func (storage *ACDStorage) GetFileInfo(threadIndex int, filePath string) (exist
 		return false, false, 0, nil
 	}
 
+	storage.savePathID(filePath, fileID)
 	return true, isDir, size, nil
 }
 
-// FindChunk finds the chunk with the specified id.  If 'isFossil' is true, it will search for chunk files with
-// the suffix '.fsl'.
-func (storage *ACDStorage) FindChunk(threadIndex int, chunkID string, isFossil bool) (filePath string, exist bool, size int64, err error) {
-	parentID := ""
-	filePath = "chunks/" + chunkID
-	realPath := filePath
-	if isFossil {
-		parentID = storage.getPathID("fossils")
-		filePath += ".fsl"
-		realPath = "fossils/" + chunkID + ".fsl"
-	} else {
-		parentID = storage.getPathID("chunks")
-	}
-
-	fileID := ""
-	fileID, _, size, err = storage.client.ListByName(parentID, chunkID)
-	if fileID != "" {
-		storage.savePathID(realPath, fileID)
-	}
-	return filePath, fileID != "", size, err
-}
-
 // DownloadFile reads the file at 'filePath' into the chunk.
 func (storage *ACDStorage) DownloadFile(threadIndex int, filePath string, chunk *Chunk) (err error) {
-	fileID, ok := storage.findPathID(filePath)
-	if !ok {
-		fileID, _, _, err = storage.getIDFromPath(threadIndex, filePath)
-		if err != nil {
-			return err
-		}
-		if fileID == "" {
-			return fmt.Errorf("File path '%s' does not exist", filePath)
-		}
-		storage.savePathID(filePath, fileID)
+	fileID, err := storage.getIDFromPath(threadIndex, filePath, false)
+	if err != nil {
+		return err
+	}
+	if fileID == "" {
+		return fmt.Errorf("File path '%s' does not exist", filePath)
 	}
 
 	readCloser, _, err := storage.client.DownloadFile(fileID)
@@ -353,22 +410,16 @@ func (storage *ACDStorage) DownloadFile(threadIndex int, filePath string, chunk
 // UploadFile writes 'content' to the file at 'filePath'.
 func (storage *ACDStorage) UploadFile(threadIndex int, filePath string, content []byte) (err error) {
 	parent := path.Dir(filePath)
-
 	if parent == "." {
 		parent = ""
 	}
 
-	parentID, ok := storage.findPathID(parent)
-
-	if !ok {
-		parentID, _, _, err = storage.getIDFromPath(threadIndex, parent)
-		if err != nil {
-			return err
-		}
-		if parentID == "" {
-			return fmt.Errorf("File path '%s' does not exist", parent)
-		}
-		storage.savePathID(parent, parentID)
+	parentID, err := storage.getIDFromPath(threadIndex, parent, true)
+	if err != nil {
+		return err
+	}
+	if parentID == "" {
+		return fmt.Errorf("File path '%s' does not exist", parent)
 	}
 
 	fileID, err := storage.client.UploadFile(parentID, path.Base(filePath), content, storage.UploadRateLimit/storage.numberOfThreads)

src/duplicacy_azurestorage.go

@@ -12,7 +12,7 @@ import (
 )
 
 type AzureStorage struct {
-	RateLimitedStorage
+	StorageBase
 
 	containers []*storage.Container
 }
@@ -47,6 +47,8 @@ func CreateAzureStorage(accountName string, accountKey string,
 		containers: containers,
 	}
 
+	azureStorage.DerivedStorage = azureStorage
+	azureStorage.SetDefaultNestingLevels([]int{0}, 0)
 	return
 }
 
@@ -149,23 +151,6 @@ func (storage *AzureStorage) GetFileInfo(threadIndex int, filePath string) (exis
 	return true, false, blob.Properties.ContentLength, nil
 }
 
-// FindChunk finds the chunk with the specified id.  If 'isFossil' is true, it will search for chunk files with
-// the suffix '.fsl'.
-func (storage *AzureStorage) FindChunk(threadIndex int, chunkID string, isFossil bool) (filePath string, exist bool, size int64, err error) {
-	filePath = "chunks/" + chunkID
-	if isFossil {
-		filePath += ".fsl"
-	}
-
-	exist, _, size, err = storage.GetFileInfo(threadIndex, filePath)
-
-	if err != nil {
-		return "", false, 0, err
-	} else {
-		return filePath, exist, size, err
-	}
-}
-
 // DownloadFile reads the file at 'filePath' into the chunk.
 func (storage *AzureStorage) DownloadFile(threadIndex int, filePath string, chunk *Chunk) (err error) {
 	readCloser, err := storage.containers[threadIndex].GetBlobReference(filePath).Get(nil)

src/duplicacy_b2storage.go

@@ -9,7 +9,7 @@ import (
 )
 
 type B2Storage struct {
-	RateLimitedStorage
+	StorageBase
 
 	clients []*B2Client
 }
@@ -38,6 +38,9 @@ func CreateB2Storage(accountID string, applicationKey string, bucket string, thr
 	storage = &B2Storage{
 		clients: clients,
 	}
+
+	storage.DerivedStorage = storage
+	storage.SetDefaultNestingLevels([]int{0}, 0)
 	return storage, nil
 }
 
@@ -204,17 +207,6 @@ func (storage *B2Storage) GetFileInfo(threadIndex int, filePath string) (exist b
 	return true, false, entries[0].Size, nil
 }
 
-// FindChunk finds the chunk with the specified id.  If 'isFossil' is true, it will search for chunk files with
-// the suffix '.fsl'.
-func (storage *B2Storage) FindChunk(threadIndex int, chunkID string, isFossil bool) (filePath string, exist bool, size int64, err error) {
-	filePath = "chunks/" + chunkID
-	if isFossil {
-		filePath += ".fsl"
-	}
-	exist, _, size, err = storage.GetFileInfo(threadIndex, filePath)
-	return filePath, exist, size, err
-}
-
 // DownloadFile reads the file at 'filePath' into the chunk.
 func (storage *B2Storage) DownloadFile(threadIndex int, filePath string, chunk *Chunk) (err error) {
 

src/duplicacy_backupmanager.go

@@ -79,7 +79,7 @@ func (manager *BackupManager) SetupSnapshotCache(storageName string) bool {
 	preferencePath := GetDuplicacyPreferencePath()
 	cacheDir := path.Join(preferencePath, "cache", storageName)
 
-	storage, err := CreateFileStorage(cacheDir, 2, false, 1)
+	storage, err := CreateFileStorage(cacheDir, false, 1)
 	if err != nil {
 		LOG_ERROR("BACKUP_CACHE", "Failed to create the snapshot cache dir: %v", err)
 		return false
@@ -93,6 +93,7 @@ func (manager *BackupManager) SetupSnapshotCache(storageName string) bool {
 		}
 	}
 
+	storage.SetDefaultNestingLevels([]int{1}, 1)
 	manager.snapshotCache = storage
 	manager.SnapshotManager.snapshotCache = storage
 	return true

src/duplicacy_backupmanager_test.go

@@ -336,6 +336,30 @@ func TestBackupManager(t *testing.T) {
 		}
 	}
 
+	numberOfSnapshots := backupManager.SnapshotManager.ListSnapshots( /*snapshotID*/ "host1" /*revisionsToList*/, nil /*tag*/, "" /*showFiles*/, false /*showChunks*/, false)
+	if numberOfSnapshots != 3 {
+		t.Errorf("Expected 3 snapshots but got %d", numberOfSnapshots)
+	}
+	backupManager.SnapshotManager.CheckSnapshots( /*snapshotID*/ "host1" /*revisions*/, []int{1, 2, 3} /*tag*/, "",
+		/*showStatistics*/ false /*showTabular*/, false /*checkFiles*/, false /*searchFossils*/, false /*resurrect*/, false)
+	backupManager.SnapshotManager.PruneSnapshots("host1", "host1" /*revisions*/, []int{1} /*tags*/, nil /*retentions*/, nil,
+		/*exhaustive*/ false /*exclusive=*/, false /*ignoredIDs*/, nil /*dryRun*/, false /*deleteOnly*/, false /*collectOnly*/, false)
+	numberOfSnapshots = backupManager.SnapshotManager.ListSnapshots( /*snapshotID*/ "host1" /*revisionsToList*/, nil /*tag*/, "" /*showFiles*/, false /*showChunks*/, false)
+	if numberOfSnapshots != 2 {
+		t.Errorf("Expected 2 snapshots but got %d", numberOfSnapshots)
+	}
+	backupManager.SnapshotManager.CheckSnapshots( /*snapshotID*/ "host1" /*revisions*/, []int{2, 3} /*tag*/, "",
+		/*showStatistics*/ false /*showTabular*/, false /*checkFiles*/, false /*searchFossils*/, false /*resurrect*/, false)
+	backupManager.Backup(testDir+"/repository1" /*quickMode=*/, false, threads, "fourth", false, false)
+	backupManager.SnapshotManager.PruneSnapshots("host1", "host1" /*revisions*/, nil /*tags*/, nil /*retentions*/, nil,
+		/*exhaustive*/ false /*exclusive=*/, true /*ignoredIDs*/, nil /*dryRun*/, false /*deleteOnly*/, false /*collectOnly*/, false)
+	numberOfSnapshots = backupManager.SnapshotManager.ListSnapshots( /*snapshotID*/ "host1" /*revisionsToList*/, nil /*tag*/, "" /*showFiles*/, false /*showChunks*/, false)
+	if numberOfSnapshots != 3 {
+		t.Errorf("Expected 3 snapshots but got %d", numberOfSnapshots)
+	}
+	backupManager.SnapshotManager.CheckSnapshots( /*snapshotID*/ "host1" /*revisions*/, []int{2, 3, 4} /*tag*/, "",
+		/*showStatistics*/ false /*showTabular*/, false /*checkFiles*/, false /*searchFossils*/, false /*resurrect*/, false)
+
 	/*buf := make([]byte, 1<<16)
 	  runtime.Stack(buf, true)
 	  fmt.Printf("%s", buf)*/

src/duplicacy_chunkdownloader.go

@@ -326,7 +326,7 @@ func (downloader *ChunkDownloader) Download(threadIndex int, task ChunkDownloadT
 	}
 
 	const MaxDownloadAttempts = 3
-	for downloadAttempt := 0;; downloadAttempt++ {
+	for downloadAttempt := 0; ; downloadAttempt++ {
 		err = downloader.storage.DownloadFile(threadIndex, chunkPath, chunk)
 		if err != nil {
 			if err == io.ErrUnexpectedEOF && downloadAttempt < MaxDownloadAttempts {

src/duplicacy_config.go

@@ -46,6 +46,8 @@ type Config struct {
 
 	ChunkSeed []byte `json:"chunk-seed"`
 
+	FixedNesting bool `json:"fixed-nesting"`
+
 	// Use HMAC-SHA256(hashKey, plaintext) as the chunk hash.
 	// Use HMAC-SHA256(idKey, chunk hash) as the file name of the chunk
 	// For chunks, use HMAC-SHA256(chunkKey, chunk hash) as the encryption key
@@ -63,7 +65,7 @@ type Config struct {
 	// for encrypting a non-chunk file
 	FileKey []byte `json:"-"`
 
-	chunkPool      chan *Chunk `json:"-"`
+	chunkPool      chan *Chunk
 	numberOfChunks int32
 	dryRun         bool
 }
@@ -148,6 +150,7 @@ func CreateConfigFromParameters(compressionLevel int, averageChunkSize int, maxi
 		AverageChunkSize: averageChunkSize,
 		MaximumChunkSize: maximumChunkSize,
 		MinimumChunkSize: mininumChunkSize,
+		FixedNesting:     true,
 	}
 
 	if isEncrypted {
@@ -380,6 +383,8 @@ func DownloadConfig(storage Storage, password string) (config *Config, isEncrypt
 		return nil, false, fmt.Errorf("Failed to parse the config file: %v", err)
 	}
 
+	storage.SetNestingLevels(config)
+
 	return config, false, nil
 
 }

src/duplicacy_dropboxstorage.go

@@ -6,18 +6,17 @@ package duplicacy
 
 import (
 	"fmt"
-	"path"
 	"strings"
 
 	"github.com/gilbertchen/go-dropbox"
 )
 
 type DropboxStorage struct {
-	RateLimitedStorage
+	StorageBase
 
-	clients         []*dropbox.Files
-	minimumNesting  int  // The minimum level of directories to dive into before searching for the chunk file.
-	storageDir      string
+	clients        []*dropbox.Files
+	minimumNesting int // The minimum level of directories to dive into before searching for the chunk file.
+	storageDir     string
 }
 
 // CreateDropboxStorage creates a dropbox storage object.
@@ -38,9 +37,9 @@ func CreateDropboxStorage(accessToken string, storageDir string, minimumNesting
 	}
 
 	storage = &DropboxStorage{
-		clients:         clients,
-		storageDir:      storageDir,
-		minimumNesting:  minimumNesting,
+		clients:        clients,
+		storageDir:     storageDir,
+		minimumNesting: minimumNesting,
 	}
 
 	err = storage.CreateDirectory(0, "")
@@ -48,6 +47,8 @@ func CreateDropboxStorage(accessToken string, storageDir string, minimumNesting
 		return nil, fmt.Errorf("Can't create storage directory: %v", err)
 	}
 
+	storage.DerivedStorage = storage
+	storage.SetDefaultNestingLevels([]int{1}, 1)
 	return storage, nil
 }
 
@@ -181,63 +182,6 @@ func (storage *DropboxStorage) GetFileInfo(threadIndex int, filePath string) (ex
 	return true, output.Tag == "folder", int64(output.Size), nil
 }
 
-// FindChunk finds the chunk with the specified id.  If 'isFossil' is true, it will search for chunk files with
-// the suffix '.fsl'.
-func (storage *DropboxStorage) FindChunk(threadIndex int, chunkID string, isFossil bool) (filePath string, exist bool, size int64, err error) {
-	dir := "/chunks"
-
-	suffix := ""
-	if isFossil {
-		suffix = ".fsl"
-	}
-
-	for level := 0; level*2 < len(chunkID); level++ {
-		if level >= storage.minimumNesting {
-			filePath = path.Join(dir, chunkID[2*level:]) + suffix
-			var size int64
-			exist, _, size, err = storage.GetFileInfo(threadIndex, filePath)
-			if err != nil {
-				return "", false, 0, err
-			}
-			if exist {
-				return filePath, exist, size, nil
-			}
-		}
-
-		// Find the subdirectory the chunk file may reside.
-		subDir := path.Join(dir, chunkID[2*level:2*level+2])
-		exist, _, _, err = storage.GetFileInfo(threadIndex, subDir)
-		if err != nil {
-			return "", false, 0, err
-		}
-
-		if exist {
-			dir = subDir
-			continue
-		}
-
-		if level < storage.minimumNesting {
-			// Create the subdirectory if it doesn't exist.
-			err = storage.CreateDirectory(threadIndex, subDir)
-			if err != nil {
-				return "", false, 0, err
-			}
-
-			dir = subDir
-			continue
-		}
-
-		// Teh chunk must be under this subdirectory but it doesn't exist.
-		return path.Join(dir, chunkID[2*level:])[1:] + suffix, false, 0, nil
-
-	}
-
-	LOG_FATAL("CHUNK_FIND", "Chunk %s is still not found after having searched a maximum level of directories",
-		chunkID)
-	return "", false, 0, nil
-
-}
-
 // DownloadFile reads the file at 'filePath' into the chunk.
 func (storage *DropboxStorage) DownloadFile(threadIndex int, filePath string, chunk *Chunk) (err error) {
 

src/duplicacy_filestorage.go

@@ -11,21 +11,21 @@ import (
 	"math/rand"
 	"os"
 	"path"
+	"strings"
 	"time"
 )
 
 // FileStorage is a local on-disk file storage implementing the Storage interface.
 type FileStorage struct {
-	RateLimitedStorage
+	StorageBase
 
-	minimumNesting  int  // The minimum level of directories to dive into before searching for the chunk file.
 	isCacheNeeded   bool // Network storages require caching
 	storageDir      string
 	numberOfThreads int
 }
 
 // CreateFileStorage creates a file storage.
-func CreateFileStorage(storageDir string, minimumNesting int, isCacheNeeded bool, threads int) (storage *FileStorage, err error) {
+func CreateFileStorage(storageDir string, isCacheNeeded bool, threads int) (storage *FileStorage, err error) {
 
 	var stat os.FileInfo
 
@@ -51,7 +51,6 @@ func CreateFileStorage(storageDir string, minimumNesting int, isCacheNeeded bool
 
 	storage = &FileStorage{
 		storageDir:      storageDir,
-		minimumNesting:  minimumNesting,
 		isCacheNeeded:   isCacheNeeded,
 		numberOfThreads: threads,
 	}
@@ -59,6 +58,8 @@ func CreateFileStorage(storageDir string, minimumNesting int, isCacheNeeded bool
 	// Random number fo generating the temporary chunk file suffix.
 	rand.Seed(time.Now().UnixNano())
 
+	storage.DerivedStorage = storage
+	storage.SetDefaultNestingLevels([]int{2, 3}, 2)
 	return storage, nil
 }
 
@@ -126,67 +127,6 @@ func (storage *FileStorage) GetFileInfo(threadIndex int, filePath string) (exist
 	return true, stat.IsDir(), stat.Size(), nil
 }
 
-// FindChunk finds the chunk with the specified id.  If 'isFossil' is true, it will search for chunk files with the
-// suffix '.fsl'.
-func (storage *FileStorage) FindChunk(threadIndex int, chunkID string, isFossil bool) (filePath string, exist bool, size int64, err error) {
-	dir := path.Join(storage.storageDir, "chunks")
-
-	suffix := ""
-	if isFossil {
-		suffix = ".fsl"
-	}
-
-	for level := 0; level*2 < len(chunkID); level++ {
-		if level >= storage.minimumNesting {
-			filePath = path.Join(dir, chunkID[2*level:]) + suffix
-			// Use Lstat() instead of Stat() since 1) Stat() doesn't work for deduplicated disks on Windows and 2) there isn't
-			// really a need to follow the link if filePath is a link.
-			stat, err := os.Lstat(filePath)
-			if err != nil {
-				LOG_DEBUG("FS_FIND", "File %s can't be found: %v", filePath, err)
-			} else if stat.IsDir() {
-				return filePath[len(storage.storageDir)+1:], false, 0, fmt.Errorf("The path %s is a directory", filePath)
-			} else {
-				return filePath[len(storage.storageDir)+1:], true, stat.Size(), nil
-			}
-		}
-
-		// Find the subdirectory the chunk file may reside.
-		subDir := path.Join(dir, chunkID[2*level:2*level+2])
-		stat, err := os.Stat(subDir)
-		if err == nil && stat.IsDir() {
-			dir = subDir
-			continue
-		}
-
-		if level < storage.minimumNesting {
-			// Create the subdirectory if it doesn't exist.
-
-			if err == nil && !stat.IsDir() {
-				return "", false, 0, fmt.Errorf("The path %s is not a directory", subDir)
-			}
-
-			err = os.Mkdir(subDir, 0744)
-			if err != nil {
-				// The directory may have been created by other threads so check it again.
-				stat, _ := os.Stat(subDir)
-				if stat == nil || !stat.IsDir() {
-					return "", false, 0, err
-				}
-			}
-			dir = subDir
-			continue
-		}
-
-		// The chunk must be under this subdirectory but it doesn't exist.
-		return path.Join(dir, chunkID[2*level:])[len(storage.storageDir)+1:] + suffix, false, 0, nil
-
-	}
-
-	return "", false, 0, fmt.Errorf("The maximum level of directories searched")
-
-}
-
 // DownloadFile reads the file at 'filePath' into the chunk.
 func (storage *FileStorage) DownloadFile(threadIndex int, filePath string, chunk *Chunk) (err error) {
 
@@ -210,6 +150,26 @@ func (storage *FileStorage) UploadFile(threadIndex int, filePath string, content
 
 	fullPath := path.Join(storage.storageDir, filePath)
 
+	if len(strings.Split(filePath, "/")) > 2 {
+		dir := path.Dir(fullPath)
+		// Use Lstat() instead of Stat() since 1) Stat() doesn't work for deduplicated disks on Windows and 2) there isn't
+		// really a need to follow the link if filePath is a link.
+		stat, err := os.Lstat(dir)
+		if err != nil {
+			if !os.IsNotExist(err) {
+				return err
+			}
+			err = os.MkdirAll(dir, 0744)
+			if err != nil {
+				return err
+			}
+		} else {
+			if !stat.IsDir() {
+				fmt.Errorf("The path %s is not a directory", dir)
+			}
+		}
+	}
+
 	letters := "abcdefghijklmnopqrstuvwxyz"
 	suffix := make([]byte, 8)
 	for i := range suffix {

src/duplicacy_gcdstorage.go

@@ -25,17 +25,18 @@ import (
 )
 
 type GCDStorage struct {
-	RateLimitedStorage
+	StorageBase
 
 	service     *drive.Service
 	idCache     map[string]string
-	idCacheLock *sync.Mutex
+	idCacheLock sync.Mutex
 	backoffs    []int // desired backoff time in seconds for each thread
 	attempts    []int // number of failed attempts since last success for each thread
 
-	isConnected     bool
-	numberOfThreads int
-	TestMode        bool
+	createDirectoryLock sync.Mutex
+	isConnected         bool
+	numberOfThreads     int
+	TestMode            bool
 }
 
 type GCDConfig struct {
@@ -91,7 +92,11 @@ func (storage *GCDStorage) shouldRetry(threadIndex int, err error) (bool, error)
 		retry = err.Temporary()
 	}
 
-	if !retry || storage.attempts[threadIndex] >= MAX_ATTEMPTS {
+	if !retry {
+		return false, err
+	}
+
+	if storage.attempts[threadIndex] >= MAX_ATTEMPTS {
 		LOG_INFO("GCD_RETRY", "[%d] Maximum number of retries reached (backoff: %d, attempts: %d)",
 			threadIndex, storage.backoffs[threadIndex], storage.attempts[threadIndex])
 		storage.backoffs[threadIndex] = 1
@@ -114,6 +119,9 @@ func (storage *GCDStorage) shouldRetry(threadIndex int, err error) (bool, error)
 	return true, nil
 }
 
+// convertFilePath converts the path for a fossil in the form of 'chunks/id.fsl' to 'fossils/id'.  This is because
+// ACD doesn't support file renaming. Instead, it only allows one file to be moved from one directory to another.
+// By adding a layer of path conversion we're pretending that we can rename between 'chunks/id' and 'chunks/id.fsl'
 func (storage *GCDStorage) convertFilePath(filePath string) string {
 	if strings.HasPrefix(filePath, "chunks/") && strings.HasSuffix(filePath, ".fsl") {
 		return "fossils/" + filePath[len("chunks/"):len(filePath)-len(".fsl")]
@@ -147,7 +155,7 @@ func (storage *GCDStorage) deletePathID(path string) {
 	storage.idCacheLock.Unlock()
 }
 
-func (storage *GCDStorage) listFiles(threadIndex int, parentID string, listFiles bool) ([]*drive.File, error) {
+func (storage *GCDStorage) listFiles(threadIndex int, parentID string, listFiles bool, listDirectories bool) ([]*drive.File, error) {
 
 	if parentID == "" {
 		return nil, fmt.Errorf("No parent ID provided")
@@ -157,11 +165,11 @@ func (storage *GCDStorage) listFiles(threadIndex int, parentID string, listFiles
 
 	startToken := ""
 
-	query := "'" + parentID + "' in parents and "
-	if listFiles {
-		query += "mimeType != 'application/vnd.google-apps.folder'"
-	} else {
-		query += "mimeType = 'application/vnd.google-apps.folder'"
+	query := "'" + parentID + "' in parents "
+	if listFiles && !listDirectories {
+		query += "and mimeType != 'application/vnd.google-apps.folder'"
+	} else if !listFiles && !listDirectories {
+		query += "and mimeType = 'application/vnd.google-apps.folder'"
 	}
 
 	maxCount := int64(1000)
@@ -222,7 +230,14 @@ func (storage *GCDStorage) listByName(threadIndex int, parentID string, name str
 	return file.Id, file.MimeType == "application/vnd.google-apps.folder", file.Size, nil
 }
 
-func (storage *GCDStorage) getIDFromPath(threadIndex int, path string) (string, error) {
+// getIDFromPath returns the id of the given path.  If 'createDirectories' is true, create the given path and all its
+// parent directories if they don't exist.  Note that if 'createDirectories' is false, it may return an empty 'fileID'
+// if the file doesn't exist.
+func (storage *GCDStorage) getIDFromPath(threadIndex int, filePath string, createDirectories bool) (string, error) {
+
+	if fileID, ok := storage.findPathID(filePath); ok {
+		return fileID, nil
+	}
 
 	fileID := "root"
 
@@ -230,22 +245,18 @@ func (storage *GCDStorage) getIDFromPath(threadIndex int, path string) (string,
 		fileID = rootID
 	}
 
-	names := strings.Split(path, "/")
+	names := strings.Split(filePath, "/")
 	current := ""
 	for i, name := range names {
-
-		if len(current) == 0 {
-			current = name
-		} else {
-			current = current + "/" + name
-		}
-
+		// Find the intermediate directory in the cache first.
+		current = path.Join(current, name)
 		currentID, ok := storage.findPathID(current)
 		if ok {
 			fileID = currentID
 			continue
 		}
 
+		// Check if the directory exists.
 		var err error
 		var isDir bool
 		fileID, isDir, _, err = storage.listByName(threadIndex, fileID, name)
@@ -253,10 +264,30 @@ func (storage *GCDStorage) getIDFromPath(threadIndex int, path string) (string,
 			return "", err
 		}
 		if fileID == "" {
-			return "", fmt.Errorf("Path %s doesn't exist", path)
+			if !createDirectories {
+				return "", nil
+			}
+
+			// Only one thread can create the directory at a time -- GCD allows multiple directories
+			// to have the same name but different ids.
+			storage.createDirectoryLock.Lock()
+			err = storage.CreateDirectory(threadIndex, current)
+			storage.createDirectoryLock.Unlock()
+
+			if err != nil {
+				return "", fmt.Errorf("Failed to create directory '%s': %v", current, err)
+			}
+			currentID, ok = storage.findPathID(current)
+			if !ok {
+				return "", fmt.Errorf("Directory '%s' created by id not found", current)
+			}
+			fileID = currentID
+			continue
+		} else {
+			storage.savePathID(current, fileID)
 		}
 		if i != len(names)-1 && !isDir {
-			return "", fmt.Errorf("Invalid path %s", path)
+			return "", fmt.Errorf("Path '%s' is not a directory", current)
 		}
 	}
 	return fileID, nil
@@ -275,13 +306,13 @@ func CreateGCDStorage(tokenFile string, storagePath string, threads int) (storag
 		return nil, err
 	}
 
-	config := oauth2.Config{
+	oauth2Config := oauth2.Config{
 		ClientID:     gcdConfig.ClientID,
 		ClientSecret: gcdConfig.ClientSecret,
 		Endpoint:     gcdConfig.Endpoint,
 	}
 
-	authClient := config.Client(context.Background(), &gcdConfig.Token)
+	authClient := oauth2Config.Client(context.Background(), &gcdConfig.Token)
 
 	service, err := drive.New(authClient)
 	if err != nil {
@@ -292,7 +323,6 @@ func CreateGCDStorage(tokenFile string, storagePath string, threads int) (storag
 		service:         service,
 		numberOfThreads: threads,
 		idCache:         make(map[string]string),
-		idCacheLock:     &sync.Mutex{},
 		backoffs:        make([]int, threads),
 		attempts:        make([]int, threads),
 	}
@@ -302,7 +332,7 @@ func CreateGCDStorage(tokenFile string, storagePath string, threads int) (storag
 		storage.attempts[i] = 0
 	}
 
-	storagePathID, err := storage.getIDFromPath(0, storagePath)
+	storagePathID, err := storage.getIDFromPath(0, storagePath, false)
 	if err != nil {
 		return nil, err
 	}
@@ -328,8 +358,9 @@ func CreateGCDStorage(tokenFile string, storagePath string, threads int) (storag
 
 	storage.isConnected = true
 
+	storage.DerivedStorage = storage
+	storage.SetDefaultNestingLevels([]int{0}, 0)
 	return storage, nil
-
 }
 
 // ListFiles return the list of files and subdirectories under 'dir' (non-recursively)
@@ -340,7 +371,7 @@ func (storage *GCDStorage) ListFiles(threadIndex int, dir string) ([]string, []i
 
 	if dir == "snapshots" {
 
-		files, err := storage.listFiles(threadIndex, storage.getPathID(dir), false)
+		files, err := storage.listFiles(threadIndex, storage.getPathID(dir), false, true)
 		if err != nil {
 			return nil, nil, err
 		}
@@ -353,12 +384,15 @@ func (storage *GCDStorage) ListFiles(threadIndex int, dir string) ([]string, []i
 		}
 		return subDirs, nil, nil
 	} else if strings.HasPrefix(dir, "snapshots/") {
-		pathID, err := storage.getIDFromPath(threadIndex, dir)
+		pathID, err := storage.getIDFromPath(threadIndex, dir, false)
 		if err != nil {
 			return nil, nil, err
 		}
+		if pathID == "" {
+			return nil, nil, fmt.Errorf("Path '%s' does not exist", dir)
+		}
 
-		entries, err := storage.listFiles(threadIndex, pathID, true)
+		entries, err := storage.listFiles(threadIndex, pathID, true, false)
 		if err != nil {
 			return nil, nil, err
 		}
@@ -374,20 +408,33 @@ func (storage *GCDStorage) ListFiles(threadIndex int, dir string) ([]string, []i
 		files := []string{}
 		sizes := []int64{}
 
-		for _, parent := range []string{"chunks", "fossils"} {
-			entries, err := storage.listFiles(threadIndex, storage.getPathID(parent), true)
+		parents := []string{"chunks", "fossils"}
+		for i := 0; i < len(parents); i++ {
+			parent := parents[i]
+			pathID, ok := storage.findPathID(parent)
+			if !ok {
+				continue
+			}
+			entries, err := storage.listFiles(threadIndex, pathID, true, true)
 			if err != nil {
 				return nil, nil, err
 			}
-
 			for _, entry := range entries {
-				name := entry.Name
-				if parent == "fossils" {
-					name += ".fsl"
+				if entry.MimeType != "application/vnd.google-apps.folder" {
+					name := entry.Name
+					if strings.HasPrefix(parent, "fossils") {
+						name = parent + "/" + name + ".fsl"
+						name = name[len("fossils/"):]
+					} else {
+						name = parent + "/" + name
+						name = name[len("chunks/"):]
+					}
+					files = append(files, name)
+					sizes = append(sizes, entry.Size)
+				} else {
+					parents = append(parents, parent+"/"+entry.Name)
 				}
 				storage.savePathID(parent+"/"+entry.Name, entry.Id)
-				files = append(files, name)
-				sizes = append(sizes, entry.Size)
 			}
 		}
 		return files, sizes, nil
@@ -398,13 +445,10 @@ func (storage *GCDStorage) ListFiles(threadIndex int, dir string) ([]string, []i
 // DeleteFile deletes the file or directory at 'filePath'.
 func (storage *GCDStorage) DeleteFile(threadIndex int, filePath string) (err error) {
 	filePath = storage.convertFilePath(filePath)
-	fileID, ok := storage.findPathID(filePath)
-	if !ok {
-		fileID, err = storage.getIDFromPath(threadIndex, filePath)
-		if err != nil {
-			LOG_TRACE("GCD_STORAGE", "Ignored file deletion error: %v", err)
-			return nil
-		}
+	fileID, err := storage.getIDFromPath(threadIndex, filePath, false)
+	if err != nil {
+		LOG_TRACE("GCD_STORAGE", "Ignored file deletion error: %v", err)
+		return nil
 	}
 
 	for {
@@ -432,14 +476,22 @@ func (storage *GCDStorage) MoveFile(threadIndex int, from string, to string) (er
 
 	fileID, ok := storage.findPathID(from)
 	if !ok {
-		return fmt.Errorf("Attempting to rename file %s with unknown id", to)
+		return fmt.Errorf("Attempting to rename file %s with unknown id", from)
 	}
 
-	fromParentID := storage.getPathID("chunks")
-	toParentID := storage.getPathID("fossils")
+	fromParent := path.Dir(from)
+	fromParentID, err := storage.getIDFromPath(threadIndex, fromParent, false)
+	if err != nil {
+		return fmt.Errorf("Failed to retrieve the id of the parent directory '%s': %v", fromParent, err)
+	}
+	if fromParentID == "" {
+		return fmt.Errorf("The parent directory '%s' does not exist", fromParent)
+	}
 
-	if strings.HasPrefix(from, "fossils") {
-		fromParentID, toParentID = toParentID, fromParentID
+	toParent := path.Dir(to)
+	toParentID, err := storage.getIDFromPath(threadIndex, toParent, true)
+	if err != nil {
+		return fmt.Errorf("Failed to retrieve the id of the parent directory '%s': %v", toParent, err)
 	}
 
 	for {
@@ -458,7 +510,7 @@ func (storage *GCDStorage) MoveFile(threadIndex int, from string, to string) (er
 	return nil
 }
 
-// CreateDirectory creates a new directory.
+// createDirectory creates a new directory.
 func (storage *GCDStorage) CreateDirectory(threadIndex int, dir string) (err error) {
 
 	for len(dir) > 0 && dir[len(dir)-1] == '/' {
@@ -477,13 +529,15 @@ func (storage *GCDStorage) CreateDirectory(threadIndex int, dir string) (err err
 		return nil
 	}
 
-	parentID := storage.getPathID("")
-	name := dir
-
-	if strings.HasPrefix(dir, "snapshots/") {
-		parentID = storage.getPathID("snapshots")
-		name = dir[len("snapshots/"):]
+	parentDir := path.Dir(dir)
+	if parentDir == "." {
+		parentDir = ""
 	}
+	parentID := storage.getPathID(parentDir)
+	if parentID == "" {
+		return fmt.Errorf("Parent directory '%s' does not exist", parentDir)
+	}
+	name := path.Base(dir)
 
 	file := &drive.File{
 		Name:     name,
@@ -495,10 +549,19 @@ func (storage *GCDStorage) CreateDirectory(threadIndex int, dir string) (err err
 		file, err = storage.service.Files.Create(file).Fields("id").Do()
 		if retry, err := storage.shouldRetry(threadIndex, err); err == nil && !retry {
 			break
-		} else if retry {
-			continue
 		} else {
-			return err
+
+			// Check if the directory has already been created by other thread
+			exist, _, _, newErr := storage.GetFileInfo(threadIndex, dir)
+			if newErr == nil && exist {
+				return nil
+			}
+
+			if retry {
+				continue
+			} else {
+				return err
+			}
 		}
 	}
 
@@ -511,18 +574,21 @@ func (storage *GCDStorage) GetFileInfo(threadIndex int, filePath string) (exist
 	for len(filePath) > 0 && filePath[len(filePath)-1] == '/' {
 		filePath = filePath[:len(filePath)-1]
 	}
+	filePath = storage.convertFilePath(filePath)
 
-	// GetFileInfo is never called on a fossil
 	fileID, ok := storage.findPathID(filePath)
 	if !ok {
 		dir := path.Dir(filePath)
 		if dir == "." {
 			dir = ""
 		}
-		dirID, err := storage.getIDFromPath(threadIndex, dir)
+		dirID, err := storage.getIDFromPath(threadIndex, dir, false)
 		if err != nil {
 			return false, false, 0, err
 		}
+		if dirID == "" {
+			return false, false, 0, nil
+		}
 
 		fileID, isDir, size, err = storage.listByName(threadIndex, dirID, path.Base(filePath))
 		if fileID != "" {
@@ -543,37 +609,15 @@ func (storage *GCDStorage) GetFileInfo(threadIndex int, filePath string) (exist
 	}
 }
 
-// FindChunk finds the chunk with the specified id.  If 'isFossil' is true, it will search for chunk files with
-// the suffix '.fsl'.
-func (storage *GCDStorage) FindChunk(threadIndex int, chunkID string, isFossil bool) (filePath string, exist bool, size int64, err error) {
-	parentID := ""
-	filePath = "chunks/" + chunkID
-	realPath := storage.convertFilePath(filePath)
-	if isFossil {
-		parentID = storage.getPathID("fossils")
-		filePath += ".fsl"
-	} else {
-		parentID = storage.getPathID("chunks")
-	}
-
-	fileID := ""
-	fileID, _, size, err = storage.listByName(threadIndex, parentID, chunkID)
-	if fileID != "" {
-		storage.savePathID(realPath, fileID)
-	}
-	return filePath, fileID != "", size, err
-}
-
 // DownloadFile reads the file at 'filePath' into the chunk.
 func (storage *GCDStorage) DownloadFile(threadIndex int, filePath string, chunk *Chunk) (err error) {
 	// We never download the fossil so there is no need to convert the path
-	fileID, ok := storage.findPathID(filePath)
-	if !ok {
-		fileID, err = storage.getIDFromPath(threadIndex, filePath)
-		if err != nil {
-			return err
-		}
-		storage.savePathID(filePath, fileID)
+	fileID, err := storage.getIDFromPath(threadIndex, filePath, false)
+	if err != nil {
+		return err
+	}
+	if fileID == "" {
+		return fmt.Errorf("%s does not exist", filePath)
 	}
 
 	var response *http.Response
@@ -605,13 +649,9 @@ func (storage *GCDStorage) UploadFile(threadIndex int, filePath string, content
 		parent = ""
 	}
 
-	parentID, ok := storage.findPathID(parent)
-	if !ok {
-		parentID, err = storage.getIDFromPath(threadIndex, parent)
-		if err != nil {
-			return err
-		}
-		storage.savePathID(parent, parentID)
+	parentID, err := storage.getIDFromPath(threadIndex, parent, true)
+	if err != nil {
+		return err
 	}
 
 	file := &drive.File{

src/duplicacy_gcsstorage.go

@@ -24,7 +24,7 @@ import (
 )
 
 type GCSStorage struct {
-	RateLimitedStorage
+	StorageBase
 
 	bucket     *gcs.BucketHandle
 	storageDir string
@@ -101,8 +101,9 @@ func CreateGCSStorage(tokenFile string, bucketName string, storageDir string, th
 		numberOfThreads: threads,
 	}
 
+	storage.DerivedStorage = storage
+	storage.SetDefaultNestingLevels([]int{0}, 0)
 	return storage, nil
-
 }
 
 func (storage *GCSStorage) shouldRetry(backoff *int, err error) (bool, error) {
@@ -238,19 +239,6 @@ func (storage *GCSStorage) GetFileInfo(threadIndex int, filePath string) (exist
 	return true, false, attributes.Size, nil
 }
 
-// FindChunk finds the chunk with the specified id.  If 'isFossil' is true, it will search for chunk files with
-// the suffix '.fsl'.
-func (storage *GCSStorage) FindChunk(threadIndex int, chunkID string, isFossil bool) (filePath string, exist bool, size int64, err error) {
-	filePath = "chunks/" + chunkID
-	if isFossil {
-		filePath += ".fsl"
-	}
-
-	exist, _, size, err = storage.GetFileInfo(threadIndex, filePath)
-
-	return filePath, exist, size, err
-}
-
 // DownloadFile reads the file at 'filePath' into the chunk.
 func (storage *GCSStorage) DownloadFile(threadIndex int, filePath string, chunk *Chunk) (err error) {
 	readCloser, err := storage.bucket.Object(storage.storageDir + filePath).NewReader(context.Background())

src/duplicacy_hubicstorage.go

@@ -10,7 +10,7 @@ import (
 )
 
 type HubicStorage struct {
-	RateLimitedStorage
+	StorageBase
 
 	client          *HubicClient
 	storageDir      string
@@ -64,8 +64,9 @@ func CreateHubicStorage(tokenFile string, storagePath string, threads int) (stor
 		}
 	}
 
+	storage.DerivedStorage = storage
+	storage.SetDefaultNestingLevels([]int{0}, 0)
 	return storage, nil
-
 }
 
 // ListFiles return the list of files and subdirectories under 'dir' (non-recursively)
@@ -158,18 +159,6 @@ func (storage *HubicStorage) GetFileInfo(threadIndex int, filePath string) (exis
 	return storage.client.GetFileInfo(storage.storageDir + "/" + filePath)
 }
 
-// FindChunk finds the chunk with the specified id.  If 'isFossil' is true, it will search for chunk files with
-// the suffix '.fsl'.
-func (storage *HubicStorage) FindChunk(threadIndex int, chunkID string, isFossil bool) (filePath string, exist bool, size int64, err error) {
-	filePath = "chunks/" + chunkID
-	if isFossil {
-		filePath += ".fsl"
-	}
-
-	exist, _, size, err = storage.client.GetFileInfo(storage.storageDir + "/" + filePath)
-	return filePath, exist, size, err
-}
-
 // DownloadFile reads the file at 'filePath' into the chunk.
 func (storage *HubicStorage) DownloadFile(threadIndex int, filePath string, chunk *Chunk) (err error) {
 	readCloser, _, err := storage.client.DownloadFile(storage.storageDir + "/" + filePath)

src/duplicacy_onestorage.go

@@ -11,7 +11,7 @@ import (
 )
 
 type OneDriveStorage struct {
-	RateLimitedStorage
+	StorageBase
 
 	client         *OneDriveClient
 	storageDir     string
@@ -65,10 +65,19 @@ func CreateOneDriveStorage(tokenFile string, storagePath string, threads int) (s
 		}
 	}
 
+	storage.DerivedStorage = storage
+	storage.SetDefaultNestingLevels([]int{0}, 0)
 	return storage, nil
 
 }
 
+func (storage *OneDriveStorage) convertFilePath(filePath string) string {
+	if strings.HasPrefix(filePath, "chunks/") && strings.HasSuffix(filePath, ".fsl") {
+		return "fossils/" + filePath[len("chunks/"):len(filePath)-len(".fsl")]
+	}
+	return filePath
+}
+
 // ListFiles return the list of files and subdirectories under 'dir' (non-recursively)
 func (storage *OneDriveStorage) ListFiles(threadIndex int, dir string) ([]string, []int64, error) {
 	for len(dir) > 0 && dir[len(dir)-1] == '/' {
@@ -105,19 +114,29 @@ func (storage *OneDriveStorage) ListFiles(threadIndex int, dir string) ([]string
 	} else {
 		files := []string{}
 		sizes := []int64{}
-		for _, parent := range []string{"chunks", "fossils"} {
+		parents := []string{"chunks", "fossils"}
+		for i := 0; i < len(parents); i++ {
+			parent := parents[i]
 			entries, err := storage.client.ListEntries(storage.storageDir + "/" + parent)
 			if err != nil {
 				return nil, nil, err
 			}
 
 			for _, entry := range entries {
-				name := entry.Name
-				if parent == "fossils" {
-					name += ".fsl"
+				if len(entry.Folder) == 0 {
+					name := entry.Name
+					if strings.HasPrefix(parent, "fossils") {
+						name = parent + "/" + name + ".fsl"
+						name = name[len("fossils/"):]
+					} else {
+						name = parent + "/" + name
+						name = name[len("chunks/"):]
+					}
+					files = append(files, name)
+					sizes = append(sizes, entry.Size)
+				} else {
+					parents = append(parents, parent+"/"+entry.Name)
 				}
-				files = append(files, name)
-				sizes = append(sizes, entry.Size)
 			}
 		}
 		return files, sizes, nil
@@ -127,9 +146,7 @@ func (storage *OneDriveStorage) ListFiles(threadIndex int, dir string) ([]string
 
 // DeleteFile deletes the file or directory at 'filePath'.
 func (storage *OneDriveStorage) DeleteFile(threadIndex int, filePath string) (err error) {
-	if strings.HasSuffix(filePath, ".fsl") && strings.HasPrefix(filePath, "chunks/") {
-		filePath = "fossils/" + filePath[len("chunks/"):len(filePath)-len(".fsl")]
-	}
+	filePath = storage.convertFilePath(filePath)
 
 	err = storage.client.DeleteFile(storage.storageDir + "/" + filePath)
 	if e, ok := err.(OneDriveError); ok && e.Status == 404 {
@@ -141,14 +158,11 @@ func (storage *OneDriveStorage) DeleteFile(threadIndex int, filePath string) (er
 
 // MoveFile renames the file.
 func (storage *OneDriveStorage) MoveFile(threadIndex int, from string, to string) (err error) {
-	fromPath := storage.storageDir + "/" + from
-	toParent := storage.storageDir + "/fossils"
-	if strings.HasSuffix(from, ".fsl") {
-		fromPath = storage.storageDir + "/fossils/" + from[len("chunks/"):len(from)-len(".fsl")]
-		toParent = storage.storageDir + "/chunks"
-	}
 
-	err = storage.client.MoveFile(fromPath, toParent)
+	fromPath := storage.storageDir + "/" + storage.convertFilePath(from)
+	toPath := storage.storageDir + "/" + storage.convertFilePath(to)
+
+	err = storage.client.MoveFile(fromPath, path.Dir(toPath))
 	if err != nil {
 		if e, ok := err.(OneDriveError); ok && e.Status == 409 {
 			LOG_DEBUG("ONEDRIVE_MOVE", "Ignore 409 conflict error")
@@ -180,24 +194,13 @@ func (storage *OneDriveStorage) GetFileInfo(threadIndex int, filePath string) (e
 	for len(filePath) > 0 && filePath[len(filePath)-1] == '/' {
 		filePath = filePath[:len(filePath)-1]
 	}
+
+	filePath = storage.convertFilePath(filePath)
+
 	fileID, isDir, size, err := storage.client.GetFileInfo(storage.storageDir + "/" + filePath)
 	return fileID != "", isDir, size, err
 }
 
-// FindChunk finds the chunk with the specified id.  If 'isFossil' is true, it will search for chunk files with
-// the suffix '.fsl'.
-func (storage *OneDriveStorage) FindChunk(threadIndex int, chunkID string, isFossil bool) (filePath string, exist bool, size int64, err error) {
-	filePath = "chunks/" + chunkID
-	realPath := storage.storageDir + "/" + filePath
-	if isFossil {
-		filePath += ".fsl"
-		realPath = storage.storageDir + "/fossils/" + chunkID
-	}
-
-	fileID, _, size, err := storage.client.GetFileInfo(realPath)
-	return filePath, fileID != "", size, err
-}
-
 // DownloadFile reads the file at 'filePath' into the chunk.
 func (storage *OneDriveStorage) DownloadFile(threadIndex int, filePath string, chunk *Chunk) (err error) {
 	readCloser, _, err := storage.client.DownloadFile(storage.storageDir + "/" + filePath)

src/duplicacy_s3cstorage.go

@@ -13,7 +13,7 @@ import (
 
 // S3CStorage is a storage backend for s3 compatible storages that require V2 Signing.
 type S3CStorage struct {
-	RateLimitedStorage
+	StorageBase
 
 	buckets    []*s3.Bucket
 	storageDir string
@@ -58,6 +58,8 @@ func CreateS3CStorage(regionName string, endpoint string, bucketName string, sto
 		storageDir: storageDir,
 	}
 
+	storage.DerivedStorage = storage
+	storage.SetDefaultNestingLevels([]int{0}, 0)
 	return storage, nil
 }
 
@@ -154,25 +156,6 @@ func (storage *S3CStorage) GetFileInfo(threadIndex int, filePath string) (exist
 	}
 }
 
-// FindChunk finds the chunk with the specified id.  If 'isFossil' is true, it will search for chunk files with
-// the suffix '.fsl'.
-func (storage *S3CStorage) FindChunk(threadIndex int, chunkID string, isFossil bool) (filePath string, exist bool, size int64, err error) {
-
-	filePath = "chunks/" + chunkID
-	if isFossil {
-		filePath += ".fsl"
-	}
-
-	exist, _, size, err = storage.GetFileInfo(threadIndex, filePath)
-
-	if err != nil {
-		return "", false, 0, err
-	} else {
-		return filePath, exist, size, err
-	}
-
-}
-
 // DownloadFile reads the file at 'filePath' into the chunk.
 func (storage *S3CStorage) DownloadFile(threadIndex int, filePath string, chunk *Chunk) (err error) {
 

src/duplicacy_s3storage.go

@@ -16,7 +16,7 @@ import (
 )
 
 type S3Storage struct {
-	RateLimitedStorage
+	StorageBase
 
 	client          *s3.S3
 	bucket          string
@@ -53,7 +53,7 @@ func CreateS3Storage(regionName string, endpoint string, bucketName string, stor
 		}
 	}
 
-	config := &aws.Config{
+	s3Config := &aws.Config{
 		Region:           aws.String(regionName),
 		Credentials:      auth,
 		Endpoint:         aws.String(endpoint),
@@ -66,12 +66,14 @@ func CreateS3Storage(regionName string, endpoint string, bucketName string, stor
 	}
 
 	storage = &S3Storage{
-		client:          s3.New(session.New(config)),
+		client:          s3.New(session.New(s3Config)),
 		bucket:          bucketName,
 		storageDir:      storageDir,
 		numberOfThreads: threads,
 	}
 
+	storage.DerivedStorage = storage
+	storage.SetDefaultNestingLevels([]int{0}, 0)
 	return storage, nil
 }
 
@@ -188,25 +190,6 @@ func (storage *S3Storage) GetFileInfo(threadIndex int, filePath string) (exist b
 	}
 }
 
-// FindChunk finds the chunk with the specified id.  If 'isFossil' is true, it will search for chunk files with
-// the suffix '.fsl'.
-func (storage *S3Storage) FindChunk(threadIndex int, chunkID string, isFossil bool) (filePath string, exist bool, size int64, err error) {
-
-	filePath = "chunks/" + chunkID
-	if isFossil {
-		filePath += ".fsl"
-	}
-
-	exist, _, size, err = storage.GetFileInfo(threadIndex, filePath)
-
-	if err != nil {
-		return "", false, 0, err
-	} else {
-		return filePath, exist, size, err
-	}
-
-}
-
 // DownloadFile reads the file at 'filePath' into the chunk.
 func (storage *S3Storage) DownloadFile(threadIndex int, filePath string, chunk *Chunk) (err error) {
 

src/duplicacy_sftpstorage.go

@@ -12,6 +12,7 @@ import (
 	"os"
 	"path"
 	"runtime"
+	"strings"
 	"time"
 
 	"github.com/pkg/sftp"
@@ -19,10 +20,10 @@ import (
 )
 
 type SFTPStorage struct {
-	RateLimitedStorage
+	StorageBase
 
 	client          *sftp.Client
-	minimumNesting  int  // The minimum level of directories to dive into before searching for the chunk file.
+	minimumNesting  int // The minimum level of directories to dive into before searching for the chunk file.
 	storageDir      string
 	numberOfThreads int
 }
@@ -45,18 +46,18 @@ func CreateSFTPStorage(server string, port int, username string, storageDir stri
 	hostKeyCallback func(hostname string, remote net.Addr,
 		key ssh.PublicKey) error, threads int) (storage *SFTPStorage, err error) {
 
-	config := &ssh.ClientConfig{
+	sftpConfig := &ssh.ClientConfig{
 		User:            username,
 		Auth:            authMethods,
 		HostKeyCallback: hostKeyCallback,
 	}
 
 	if server == "sftp.hidrive.strato.com" {
-		config.Ciphers = []string{"aes128-cbc", "aes128-ctr", "aes256-ctr"}
+		sftpConfig.Ciphers = []string{"aes128-cbc", "aes128-ctr", "aes256-ctr"}
 	}
 
 	serverAddress := fmt.Sprintf("%s:%d", server, port)
-	connection, err := ssh.Dial("tcp", serverAddress, config)
+	connection, err := ssh.Dial("tcp", serverAddress, sftpConfig)
 	if err != nil {
 		return nil, err
 	}
@@ -92,6 +93,8 @@ func CreateSFTPStorage(server string, port int, username string, storageDir stri
 
 	runtime.SetFinalizer(storage, CloseSFTPStorage)
 
+	storage.DerivedStorage = storage
+	storage.SetDefaultNestingLevels([]int{2, 3}, 2)
 	return storage, nil
 }
 
@@ -176,64 +179,6 @@ func (storage *SFTPStorage) GetFileInfo(threadIndex int, filePath string) (exist
 	return true, fileInfo.IsDir(), fileInfo.Size(), nil
 }
 
-// FindChunk finds the chunk with the specified id.  If 'isFossil' is true, it will search for chunk files with
-// the suffix '.fsl'.
-func (storage *SFTPStorage) FindChunk(threadIndex int, chunkID string, isFossil bool) (filePath string, exist bool, size int64, err error) {
-	dir := path.Join(storage.storageDir, "chunks")
-
-	suffix := ""
-	if isFossil {
-		suffix = ".fsl"
-	}
-
-	for level := 0; level*2 < len(chunkID); level++ {
-		if level >= storage.minimumNesting {
-			filePath = path.Join(dir, chunkID[2*level:]) + suffix
-			if stat, err := storage.client.Stat(filePath); err == nil && !stat.IsDir() {
-				return filePath[len(storage.storageDir)+1:], true, stat.Size(), nil
-			} else if err == nil && stat.IsDir() {
-				return filePath[len(storage.storageDir)+1:], true, 0, fmt.Errorf("The path %s is a directory", filePath)
-			}
-		}
-
-		// Find the subdirectory the chunk file may reside.
-		subDir := path.Join(dir, chunkID[2*level:2*level+2])
-		stat, err := storage.client.Stat(subDir)
-		if err == nil && stat.IsDir() {
-			dir = subDir
-			continue
-		}
-
-		if level < storage.minimumNesting {
-			// Create the subdirectory if is doesn't exist.
-
-			if err == nil && !stat.IsDir() {
-				return "", false, 0, fmt.Errorf("The path %s is not a directory", subDir)
-			}
-
-			err = storage.client.Mkdir(subDir)
-			if err != nil {
-				// The directory may have been created by other threads so check it again.
-				stat, _ := storage.client.Stat(subDir)
-				if stat == nil || !stat.IsDir() {
-					return "", false, 0, fmt.Errorf("Failed to create the directory %s: %v", subDir, err)
-				}
-			}
-
-			dir = subDir
-			continue
-		}
-
-		// The chunk must be under this subdirectory but it doesn't exist.
-		return path.Join(dir, chunkID[2*level:])[len(storage.storageDir)+1:] + suffix, false, 0, nil
-
-	}
-
-	LOG_FATAL("CHUNK_FIND", "Chunk %s is still not found after having searched a maximum level of directories",
-		chunkID)
-	return "", false, 0, nil
-}
-
 // DownloadFile reads the file at 'filePath' into the chunk.
 func (storage *SFTPStorage) DownloadFile(threadIndex int, filePath string, chunk *Chunk) (err error) {
 	file, err := storage.client.Open(path.Join(storage.storageDir, filePath))
@@ -255,6 +200,30 @@ func (storage *SFTPStorage) UploadFile(threadIndex int, filePath string, content
 
 	fullPath := path.Join(storage.storageDir, filePath)
 
+	dirs := strings.Split(filePath, "/")
+	if len(dirs) > 1 {
+		fullDir := path.Dir(fullPath)
+		_, err := storage.client.Stat(fullDir)
+		if err != nil {
+			// The error may be caused by a non-existent fullDir, or a broken connection.  In either case,
+			// we just assume it is the former because there isn't a way to tell which is the case.
+			for i, _ := range dirs[1 : len(dirs)-1] {
+				subDir := path.Join(storage.storageDir, path.Join(dirs[0:i+2]...))
+				// We don't check the error; just keep going blindly but always store the last err
+				err = storage.client.Mkdir(subDir)
+			}
+
+			// If there is an error creating the dirs, we check fullDir one more time, because another thread
+			// may happen to create the same fullDir ahead of this thread
+			if err != nil {
+				_, err := storage.client.Stat(fullDir)
+				if err != nil {
+					return err
+				}
+			}
+		}
+	}
+
 	letters := "abcdefghijklmnopqrstuvwxyz"
 	suffix := make([]byte, 8)
 	for i := range suffix {
@@ -301,7 +270,14 @@ func (storage *SFTPStorage) IsMoveFileImplemented() bool { return true }
 func (storage *SFTPStorage) IsStrongConsistent() bool { return true }
 
 // If the storage supports fast listing of files names.
-func (storage *SFTPStorage) IsFastListing() bool { return false }
+func (storage *SFTPStorage) IsFastListing() bool {
+	for _, level := range storage.readLevels {
+		if level > 1 {
+			return false
+		}
+	}
+	return true
+}
 
 // Enable the test mode.
 func (storage *SFTPStorage) EnableTestMode() {}

src/duplicacy_snapshotmanager.go

@@ -592,24 +592,6 @@ func (manager *SnapshotManager) ListAllFiles(storage Storage, top string) (allFi
 				allSizes = append(allSizes, sizes[i])
 			}
 		}
-
-		if !manager.config.dryRun {
-			if top == "chunks/" {
-				// We're listing all chunks so this is the perfect place to detect if a directory contains too many
-				// chunks. Create sub-directories if necessary
-				if len(files) > 1024 && !storage.IsFastListing() {
-					for i := 0; i < 256; i++ {
-						subdir := dir + fmt.Sprintf("%02x\n", i)
-						manager.storage.CreateDirectory(0, subdir)
-					}
-				}
-			} else {
-				// Remove chunk sub-directories that are empty
-				if len(files) == 0 && strings.HasPrefix(dir, "chunks/") && dir != "chunks/" {
-					storage.DeleteFile(0, dir)
-				}
-			}
-		}
 	}
 
 	return allFiles, allSizes

src/duplicacy_snapshotmanager_test.go

@@ -95,14 +95,14 @@ func createTestSnapshotManager(testDir string) *SnapshotManager {
 	os.RemoveAll(testDir)
 	os.MkdirAll(testDir, 0700)
 
-	storage, _ := CreateFileStorage(testDir, 2, false, 1)
+	storage, _ := CreateFileStorage(testDir, false, 1)
 	storage.CreateDirectory(0, "chunks")
 	storage.CreateDirectory(0, "snapshots")
 	config := CreateConfig()
 	snapshotManager := CreateSnapshotManager(config, storage)
 
 	cacheDir := path.Join(testDir, "cache")
-	snapshotCache, _ := CreateFileStorage(cacheDir, 2, false, 1)
+	snapshotCache, _ := CreateFileStorage(cacheDir, false, 1)
 	snapshotCache.CreateDirectory(0, "chunks")
 	snapshotCache.CreateDirectory(0, "snapshots")
 

src/duplicacy_storage.go

@@ -5,6 +5,7 @@
 package duplicacy
 
 import (
+	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"net"
@@ -20,8 +21,10 @@ import (
 )
 
 type Storage interface {
-	// ListFiles return the list of files and subdirectories under 'dir' (non-recursively)
-	ListFiles(threadIndex int, dir string) (files []string, size []int64, err error)
+	// ListFiles return the list of files and subdirectories under 'dir'.  A subdirectories returned must have a trailing '/', with
+	// a size of 0.  If 'dir' is 'snapshots', only subdirectories will be returned.  If 'dir' is 'snapshots/repository_id', then only
+	// files will be returned.  If 'dir' is 'chunks', the implementation can return the list either recusively or non-recusively.
+	ListFiles(threadIndex int, dir string) (files []string, sizes []int64, err error)
 
 	// DeleteFile deletes the file or directory at 'filePath'.
 	DeleteFile(threadIndex int, filePath string) (err error)
@@ -45,6 +48,9 @@ type Storage interface {
 	// UploadFile writes 'content' to the file at 'filePath'.
 	UploadFile(threadIndex int, filePath string, content []byte) (err error)
 
+	// SetNestingLevels sets up the chunk nesting structure.
+	SetNestingLevels(config *Config)
+
 	// If a local snapshot cache is needed for the storage to avoid downloading/uploading chunks too often when
 	// managing snapshots.
 	IsCacheNeeded() bool
@@ -65,16 +71,97 @@ type Storage interface {
 	SetRateLimits(downloadRateLimit int, uploadRateLimit int)
 }
 
-type RateLimitedStorage struct {
-	DownloadRateLimit int
-	UploadRateLimit   int
+// StorageBase is the base struct from which all storages are derived from
+type StorageBase struct {
+	DownloadRateLimit int // Maximum download rate (bytes/seconds)
+	UploadRateLimit   int // Maximum upload reate (bytes/seconds)
+
+	DerivedStorage Storage // Used as the pointer to the derived storage class
+
+	readLevels []int // At which nesting level to find the chunk with the given id
+	writeLevel int   // Store the uploaded chunk to this level
 }
 
-func (storage *RateLimitedStorage) SetRateLimits(downloadRateLimit int, uploadRateLimit int) {
+// SetRateLimits sets the maximum download and upload rates
+func (storage *StorageBase) SetRateLimits(downloadRateLimit int, uploadRateLimit int) {
 	storage.DownloadRateLimit = downloadRateLimit
 	storage.UploadRateLimit = uploadRateLimit
 }
 
+// SetDefaultNestingLevels sets the default read and write levels.  This is usually called by
+// derived storages to set the levels with old values so that storages initialied by ealier versions
+// will continue to work.
+func (storage *StorageBase) SetDefaultNestingLevels(readLevels []int, writeLevel int) {
+	storage.readLevels = readLevels
+	storage.writeLevel = writeLevel
+}
+
+// SetNestingLevels sets the new read and write levels (normally both at 1) if the 'config' file has
+// the 'fixed-nesting' key, or if a file named 'nesting' exists on the storage.
+func (storage *StorageBase) SetNestingLevels(config *Config) {
+
+	// 'FixedNesting' is true only for the 'config' file with the new format (2.0.10+)
+	if config.FixedNesting {
+
+		storage.readLevels = nil
+
+		// Check if the 'nesting' file exist
+		exist, _, _, err := storage.DerivedStorage.GetFileInfo(0, "nesting")
+		if err == nil && exist {
+			nestingFile := CreateChunk(CreateConfig(), true)
+			if storage.DerivedStorage.DownloadFile(0, "config", nestingFile) == nil {
+				var nesting struct {
+					ReadLevels []int `json:"read-levels"`
+					WriteLevel int   `json:"write-level"`
+				}
+				if json.Unmarshal(nestingFile.GetBytes(), &nesting) == nil {
+					storage.readLevels = nesting.ReadLevels
+					storage.writeLevel = nesting.WriteLevel
+				}
+			}
+		}
+
+		if len(storage.readLevels) == 0 {
+			storage.readLevels = []int{1}
+			storage.writeLevel = 1
+		}
+	}
+
+	LOG_DEBUG("STORAGE_NESTING", "Chunk read levels: %v, write level: %d", storage.readLevels, storage.writeLevel)
+	for _, level := range storage.readLevels {
+		if storage.writeLevel == level {
+			return
+		}
+	}
+	LOG_ERROR("STORAGE_NESTING", "The write level %d isn't in the read levels %v", storage.readLevels, storage.writeLevel)
+}
+
+// FindChunk finds the chunk with the specified id at the levels one by one as specified by 'readLevels'.
+func (storage *StorageBase) FindChunk(threadIndex int, chunkID string, isFossil bool) (filePath string, exist bool, size int64, err error) {
+	chunkPaths := make([]string, 0)
+	for _, level := range storage.readLevels {
+		chunkPath := "chunks/"
+		for i := 0; i < level; i++ {
+			chunkPath += chunkID[2*i:2*i+2] + "/"
+		}
+		chunkPath += chunkID[2*level:]
+		if isFossil {
+			chunkPath += ".fsl"
+		}
+		exist, _, size, err = storage.DerivedStorage.GetFileInfo(threadIndex, chunkPath)
+		if err == nil && exist {
+			return chunkPath, exist, size, err
+		}
+		chunkPaths = append(chunkPaths, chunkPath)
+	}
+	for i, level := range storage.readLevels {
+		if storage.writeLevel == level {
+			return chunkPaths[i], false, 0, nil
+		}
+	}
+	return "", false, 0, fmt.Errorf("Invalid chunk nesting setup")
+}
+
 func checkHostKey(hostname string, remote net.Addr, key ssh.PublicKey) error {
 
 	if preferencePath == "" {
@@ -148,7 +235,7 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
 	}
 
 	if isFileStorage {
-		fileStorage, err := CreateFileStorage(storageURL, 2, isCacheNeeded, threads)
+		fileStorage, err := CreateFileStorage(storageURL, isCacheNeeded, threads)
 		if err != nil {
 			LOG_ERROR("STORAGE_CREATE", "Failed to load the file storage at %s: %v", storageURL, err)
 			return nil
@@ -157,7 +244,7 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
 	}
 
 	if strings.HasPrefix(storageURL, "flat://") {
-		fileStorage, err := CreateFileStorage(storageURL[7:], 0, false, threads)
+		fileStorage, err := CreateFileStorage(storageURL[7:], false, threads)
 		if err != nil {
 			LOG_ERROR("STORAGE_CREATE", "Failed to load the file storage at %s: %v", storageURL, err)
 			return nil
@@ -166,7 +253,7 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
 	}
 
 	if strings.HasPrefix(storageURL, "samba://") {
-		fileStorage, err := CreateFileStorage(storageURL[8:], 2, true, threads)
+		fileStorage, err := CreateFileStorage(storageURL[8:], true, threads)
 		if err != nil {
 			LOG_ERROR("STORAGE_CREATE", "Failed to load the file storage at %s: %v", storageURL, err)
 			return nil

src/duplicacy_storage_test.go

@@ -15,7 +15,6 @@ import (
 	"path"
 	"runtime/debug"
 	"strconv"
-	"strings"
 	"testing"
 	"time"
 
@@ -41,61 +40,100 @@ func init() {
 func loadStorage(localStoragePath string, threads int) (Storage, error) {
 
 	if testStorageName == "" || testStorageName == "file" {
-		return CreateFileStorage(localStoragePath, 2, false, threads)
+		storage, err := CreateFileStorage(localStoragePath, false, threads)
+		if storage != nil {
+			// Use a read level of at least 2 because this will catch more errors than a read level of 1.
+			storage.SetDefaultNestingLevels([]int{2, 3}, 2)
+		}
+		return storage, err
 	}
 
-	config, err := ioutil.ReadFile("test_storage.conf")
+	description, err := ioutil.ReadFile("test_storage.conf")
 	if err != nil {
 		return nil, err
 	}
 
-	storages := make(map[string]map[string]string)
+	configs := make(map[string]map[string]string)
 
-	err = json.Unmarshal(config, &storages)
+	err = json.Unmarshal(description, &configs)
 	if err != nil {
 		return nil, err
 	}
 
-	storage, found := storages[testStorageName]
+	config, found := configs[testStorageName]
 	if !found {
 		return nil, fmt.Errorf("No storage named '%s' found", testStorageName)
 	}
 
 	if testStorageName == "flat" {
-		return CreateFileStorage(localStoragePath, 0, false, threads)
+		storage, err := CreateFileStorage(localStoragePath, false, threads)
+		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
+		return storage, err
 	} else if testStorageName == "samba" {
-		return CreateFileStorage(localStoragePath, 2, true, threads)
+		storage, err := CreateFileStorage(localStoragePath, true, threads)
+		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
+		return storage, err
 	} else if testStorageName == "sftp" {
-		port, _ := strconv.Atoi(storage["port"])
-		return CreateSFTPStorageWithPassword(storage["server"], port, storage["username"], storage["directory"], 2, storage["password"], threads)
+		port, _ := strconv.Atoi(config["port"])
+		storage, err := CreateSFTPStorageWithPassword(config["server"], port, config["username"], config["directory"], 2, config["password"], threads)
+		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
+		return storage, err
 	} else if testStorageName == "s3" || testStorageName == "wasabi" {
-		return CreateS3Storage(storage["region"], storage["endpoint"], storage["bucket"], storage["directory"], storage["access_key"], storage["secret_key"], threads, true, false)
+		storage, err := CreateS3Storage(config["region"], config["endpoint"], config["bucket"], config["directory"], config["access_key"], config["secret_key"], threads, true, false)
+		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
+		return storage, err
 	} else if testStorageName == "s3c" {
-		return CreateS3CStorage(storage["region"], storage["endpoint"], storage["bucket"], storage["directory"], storage["access_key"], storage["secret_key"], threads)
+		storage, err := CreateS3CStorage(config["region"], config["endpoint"], config["bucket"], config["directory"], config["access_key"], config["secret_key"], threads)
+		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
+		return storage, err
 	} else if testStorageName == "minio" {
-		return CreateS3Storage(storage["region"], storage["endpoint"], storage["bucket"], storage["directory"], storage["access_key"], storage["secret_key"], threads, false, true)
+		storage, err := CreateS3Storage(config["region"], config["endpoint"], config["bucket"], config["directory"], config["access_key"], config["secret_key"], threads, false, true)
+		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
+		return storage, err
 	} else if testStorageName == "minios" {
-		return CreateS3Storage(storage["region"], storage["endpoint"], storage["bucket"], storage["directory"], storage["access_key"], storage["secret_key"], threads, true, true)
+		storage, err := CreateS3Storage(config["region"], config["endpoint"], config["bucket"], config["directory"], config["access_key"], config["secret_key"], threads, true, true)
+		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
+		return storage, err
 	} else if testStorageName == "dropbox" {
-		return CreateDropboxStorage(storage["token"], storage["directory"], 1, threads)
+		storage, err := CreateDropboxStorage(config["token"], config["directory"], 1, threads)
+		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
+		return storage, err
 	} else if testStorageName == "b2" {
-		return CreateB2Storage(storage["account"], storage["key"], storage["bucket"], threads)
+		storage, err := CreateB2Storage(config["account"], config["key"], config["bucket"], threads)
+		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
+		return storage, err
 	} else if testStorageName == "gcs-s3" {
-		return CreateS3Storage(storage["region"], storage["endpoint"], storage["bucket"], storage["directory"], storage["access_key"], storage["secret_key"], threads, true, false)
+		storage, err := CreateS3Storage(config["region"], config["endpoint"], config["bucket"], config["directory"], config["access_key"], config["secret_key"], threads, true, false)
+		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
+		return storage, err
 	} else if testStorageName == "gcs" {
-		return CreateGCSStorage(storage["token_file"], storage["bucket"], storage["directory"], threads)
+		storage, err := CreateGCSStorage(config["token_file"], config["bucket"], config["directory"], threads)
+		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
+		return storage, err
 	} else if testStorageName == "gcs-sa" {
-		return CreateGCSStorage(storage["token_file"], storage["bucket"], storage["directory"], threads)
+		storage, err := CreateGCSStorage(config["token_file"], config["bucket"], config["directory"], threads)
+		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
+		return storage, err
 	} else if testStorageName == "azure" {
-		return CreateAzureStorage(storage["account"], storage["key"], storage["container"], threads)
+		storage, err := CreateAzureStorage(config["account"], config["key"], config["container"], threads)
+		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
+		return storage, err
 	} else if testStorageName == "acd" {
-		return CreateACDStorage(storage["token_file"], storage["storage_path"], threads)
+		storage, err := CreateACDStorage(config["token_file"], config["storage_path"], threads)
+		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
+		return storage, err
 	} else if testStorageName == "gcd" {
-		return CreateGCDStorage(storage["token_file"], storage["storage_path"], threads)
+		storage, err := CreateGCDStorage(config["token_file"], config["storage_path"], threads)
+		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
+		return storage, err
 	} else if testStorageName == "one" {
-		return CreateOneDriveStorage(storage["token_file"], storage["storage_path"], threads)
+		storage, err := CreateOneDriveStorage(config["token_file"], config["storage_path"], threads)
+		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
+		return storage, err
 	} else if testStorageName == "hubic" {
-		return CreateHubicStorage(storage["token_file"], storage["storage_path"], threads)
+		storage, err := CreateHubicStorage(config["token_file"], config["storage_path"], threads)
+		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
+		return storage, err
 	} else {
 		return nil, fmt.Errorf("Invalid storage named: %s", testStorageName)
 	}
@@ -266,6 +304,33 @@ func TestStorage(t *testing.T) {
 
 	storage.CreateDirectory(0, "snapshots/repository1")
 	storage.CreateDirectory(0, "snapshots/repository2")
+
+	storage.CreateDirectory(0, "shared")
+
+	// Upload to the same directory by multiple goroutines
+	count := 8
+	finished := make(chan int, count)
+	for i := 0; i < count; i++ {
+		go func(name string) {
+			err := storage.UploadFile(0, name, []byte("this is a test file"))
+			if err != nil {
+				t.Errorf("Error to upload '%s': %v", name, err)
+			}
+			finished <- 0
+		}(fmt.Sprintf("shared/a/b/c/%d", i))
+	}
+
+	for i := 0; i < count; i++ {
+		<-finished
+	}
+
+	for i := 0; i < count; i++ {
+		storage.DeleteFile(0, fmt.Sprintf("shared/a/b/c/%d", i))
+	}
+	storage.DeleteFile(0, "shared/a/b/c")
+	storage.DeleteFile(0, "shared/a/b")
+	storage.DeleteFile(0, "shared/a")
+
 	time.Sleep(time.Duration(delay) * time.Second)
 	{
 
@@ -338,7 +403,7 @@ func TestStorage(t *testing.T) {
 		}
 	}
 
-	numberOfFiles := 20
+	numberOfFiles := 10
 	maxFileSize := 64 * 1024
 
 	if testQuickMode {
@@ -374,15 +439,7 @@ func TestStorage(t *testing.T) {
 			t.Errorf("Failed to upload the file %s: %v", filePath, err)
 			return
 		}
-		LOG_INFO("STORAGE_CHUNK", "Uploaded chunk: %s, size: %d", chunkID, len(content))
-	}
-
-	allChunks := []string{}
-	for _, file := range listChunks(storage) {
-		file = strings.Replace(file, "/", "", -1)
-		if len(file) == 64 {
-			allChunks = append(allChunks, file)
-		}
+		LOG_INFO("STORAGE_CHUNK", "Uploaded chunk: %s, size: %d", filePath, len(content))
 	}
 
 	LOG_INFO("STORAGE_FOSSIL", "Making %s a fossil", chunks[0])
@@ -412,7 +469,7 @@ func TestStorage(t *testing.T) {
 				t.Errorf("Error downloading file %s: %v", filePath, err)
 				continue
 			}
-			LOG_INFO("STORAGE_CHUNK", "Downloaded chunk: %s, size: %d", chunkID, chunk.GetLength())
+			LOG_INFO("STORAGE_CHUNK", "Downloaded chunk: %s, size: %d", filePath, chunk.GetLength())
 		}
 
 		hasher := sha256.New()
@@ -447,6 +504,11 @@ func TestStorage(t *testing.T) {
 		}
 	}
 
+	allChunks := []string{}
+	for _, file := range listChunks(storage) {
+		allChunks = append(allChunks, file)
+	}
+
 	for _, file := range allChunks {
 
 		err = storage.DeleteFile(0, "chunks/"+file)