Revert "Fixed a bug in creating directories in Google Drive storage backend"

This reverts commit 801433340a. That fix puts everything in the cache which leads to a memory explosion problem. The correct fix is to only put directories in the cache.
2025-12-06 00:03:38 +00:00 · 2017-12-11 08:26:32 -05:00
parent 801433340a
commit 61e4329522
1 changed files with 89 additions and 102 deletions
--- a/src/duplicacy_gcdstorage.go
+++ b/src/duplicacy_gcdstorage.go
@@ -24,24 +24,14 @@ import (
 	"google.golang.org/api/googleapi"
 )

-var (
-	GCDFileMimeType = "application/octet-stream"
-	GCDDirectoryMimeType = "application/vnd.google-apps.folder"
-)
-type GCDFileInfo struct {
-	ID string
-	IsDir bool
-	Size int64
-}
-
 type GCDStorage struct {
 	StorageBase

-	service           *drive.Service
-	fileInfoCache     map[string]GCDFileInfo
-	fileInfoLock      sync.Mutex
-	backoffs          []int // desired backoff time in seconds for each thread
-	attempts          []int // number of failed attempts since last success for each thread
+	service     *drive.Service
+	idCache     map[string]string
+	idCacheLock sync.Mutex
+	backoffs    []int // desired backoff time in seconds for each thread
+	attempts    []int // number of failed attempts since last success for each thread

 	createDirectoryLock sync.Mutex
 	isConnected         bool
@@ -139,32 +129,30 @@ func (storage *GCDStorage) convertFilePath(filePath string) string {
 	return filePath
 }

-// getPathIDFromCache assumes that the path to be found exists in the cache
-func (storage *GCDStorage) getPathIDFromCache(path string) (pathID string) {
-	storage.fileInfoLock.Lock()
-	fileInfo := storage.fileInfoCache[path]
-	storage.fileInfoLock.Unlock()
-	return fileInfo.ID
+func (storage *GCDStorage) getPathID(path string) string {
+	storage.idCacheLock.Lock()
+	pathID := storage.idCache[path]
+	storage.idCacheLock.Unlock()
+	return pathID
 }

-// findFileInfoFromCache doesn't assume that the path to be found exists in the cache
-func (storage *GCDStorage) findFileInfoFromCache(path string) (pathID string, exist bool, isDir bool, size int64) {
-	storage.fileInfoLock.Lock()
-	fileInfo, ok := storage.fileInfoCache[path]
-	storage.fileInfoLock.Unlock()
-	return fileInfo.ID, ok, fileInfo.IsDir, fileInfo.Size
+func (storage *GCDStorage) findPathID(path string) (string, bool) {
+	storage.idCacheLock.Lock()
+	pathID, ok := storage.idCache[path]
+	storage.idCacheLock.Unlock()
+	return pathID, ok
 }

-func (storage *GCDStorage) saveFileInfoToCache(path string, pathID string, isDir bool, size int64) {
-	storage.fileInfoLock.Lock()
-	storage.fileInfoCache[path] = GCDFileInfo { ID: pathID, IsDir: isDir, Size: size }
-	storage.fileInfoLock.Unlock()
+func (storage *GCDStorage) savePathID(path string, pathID string) {
+	storage.idCacheLock.Lock()
+	storage.idCache[path] = pathID
+	storage.idCacheLock.Unlock()
 }

-func (storage *GCDStorage) deleteFileInfoFromCache(path string) {
-	storage.fileInfoLock.Lock()
-	delete(storage.fileInfoCache, path)
-	storage.fileInfoLock.Unlock()
+func (storage *GCDStorage) deletePathID(path string) {
+	storage.idCacheLock.Lock()
+	delete(storage.idCache, path)
+	storage.idCacheLock.Unlock()
 }

 func (storage *GCDStorage) listFiles(threadIndex int, parentID string, listFiles bool, listDirectories bool) ([]*drive.File, error) {
@@ -177,7 +165,7 @@ func (storage *GCDStorage) listFiles(threadIndex int, parentID string, listFiles

 	startToken := ""

-	query := "'" + parentID + "' in parents and trashed != true "
+	query := "'" + parentID + "' in parents "
 	if listFiles && !listDirectories {
 		query += "and mimeType != 'application/vnd.google-apps.folder'"
 	} else if !listFiles && !listDirectories {
@@ -221,7 +209,7 @@ func (storage *GCDStorage) listByName(threadIndex int, parentID string, name str
 	var err error

 	for {
-		query := "name = '" + name + "' and '" + parentID + "' in parents and trashed != true"
+		query := "name = '" + name + "' and '" + parentID + "' in parents"
 		fileList, err = storage.service.Files.List().Q(query).Fields("files(name, mimeType, id, size)").Do()

 		if retry, e := storage.shouldRetry(threadIndex, err); e == nil && !retry {
@@ -239,7 +227,7 @@ func (storage *GCDStorage) listByName(threadIndex int, parentID string, name str

 	file := fileList.Files[0]

-	return file.Id, file.MimeType == GCDDirectoryMimeType, file.Size, nil
+	return file.Id, file.MimeType == "application/vnd.google-apps.folder", file.Size, nil
 }

 // getIDFromPath returns the id of the given path.  If 'createDirectories' is true, create the given path and all its
@@ -247,13 +235,13 @@ func (storage *GCDStorage) listByName(threadIndex int, parentID string, name str
 // if the file doesn't exist.
 func (storage *GCDStorage) getIDFromPath(threadIndex int, filePath string, createDirectories bool) (string, error) {

-	if fileID, ok, _, _ := storage.findFileInfoFromCache(filePath); ok {
+	if fileID, ok := storage.findPathID(filePath); ok {
 		return fileID, nil
 	}

 	fileID := "root"

-	if rootID, ok, _, _ := storage.findFileInfoFromCache(""); ok {
+	if rootID, ok := storage.findPathID(""); ok {
 		fileID = rootID
 	}

@@ -262,7 +250,7 @@ func (storage *GCDStorage) getIDFromPath(threadIndex int, filePath string, creat
 	for i, name := range names {
 		// Find the intermediate directory in the cache first.
 		current = path.Join(current, name)
-		currentID, ok, _, _ := storage.findFileInfoFromCache(current)
+		currentID, ok := storage.findPathID(current)
 		if ok {
 			fileID = currentID
 			continue
@@ -271,8 +259,7 @@ func (storage *GCDStorage) getIDFromPath(threadIndex int, filePath string, creat
 		// Check if the directory exists.
 		var err error
 		var isDir bool
-		var size int64
-		fileID, isDir, size, err = storage.listByName(threadIndex, fileID, name)
+		fileID, isDir, _, err = storage.listByName(threadIndex, fileID, name)
 		if err != nil {
 			return "", err
 		}
@@ -290,14 +277,14 @@ func (storage *GCDStorage) getIDFromPath(threadIndex int, filePath string, creat
 			if err != nil {
 				return "", fmt.Errorf("Failed to create directory '%s': %v", current, err)
 			}
-			currentID, ok, _, _= storage.findFileInfoFromCache(current)
+			currentID, ok = storage.findPathID(current)
 			if !ok {
 				return "", fmt.Errorf("Directory '%s' created by id not found", current)
 			}
 			fileID = currentID
 			continue
 		} else {
-			storage.saveFileInfoToCache(current, fileID, isDir, size)
+			storage.savePathID(current, fileID)
 		}
 		if i != len(names)-1 && !isDir {
 			return "", fmt.Errorf("Path '%s' is not a directory", current)
@@ -335,7 +322,7 @@ func CreateGCDStorage(tokenFile string, storagePath string, threads int) (storag
 	storage = &GCDStorage{
 		service:         service,
 		numberOfThreads: threads,
-		fileInfoCache:   make(map[string]GCDFileInfo),
+		idCache:         make(map[string]string),
 		backoffs:        make([]int, threads),
 		attempts:        make([]int, threads),
 	}
@@ -350,9 +337,7 @@ func CreateGCDStorage(tokenFile string, storagePath string, threads int) (storag
 		return nil, err
 	}

-	// Clear the file info cache and set storagePathID as the root id
-	storage.fileInfoCache = make(map[string]GCDFileInfo)
-	storage.saveFileInfoToCache("", storagePathID, true, 0)
+	storage.idCache[""] = storagePathID

 	for _, dir := range []string{"chunks", "snapshots", "fossils"} {
 		dirID, isDir, _, err := storage.listByName(0, storagePathID, dir)
@@ -367,7 +352,7 @@ func CreateGCDStorage(tokenFile string, storagePath string, threads int) (storag
 		} else if !isDir {
 			return nil, fmt.Errorf("%s/%s is not a directory", storagePath, dir)
 		} else {
-			storage.saveFileInfoToCache(dir, dirID, true, 0)
+			storage.idCache[dir] = dirID
 		}
 	}

@@ -386,7 +371,7 @@ func (storage *GCDStorage) ListFiles(threadIndex int, dir string) ([]string, []i

 	if dir == "snapshots" {

-		files, err := storage.listFiles(threadIndex, storage.getPathIDFromCache(dir), false, true)
+		files, err := storage.listFiles(threadIndex, storage.getPathID(dir), false, true)
 		if err != nil {
 			return nil, nil, err
 		}
@@ -394,8 +379,8 @@ func (storage *GCDStorage) ListFiles(threadIndex int, dir string) ([]string, []i
 		subDirs := []string{}

 		for _, file := range files {
-			storage.saveFileInfoToCache("snapshots/" + file.Name, file.Id, file.MimeType == GCDDirectoryMimeType, file.Size)
-			subDirs = append(subDirs, file.Name + "/")
+			storage.savePathID("snapshots/"+file.Name, file.Id)
+			subDirs = append(subDirs, file.Name+"/")
 		}
 		return subDirs, nil, nil
 	} else if strings.HasPrefix(dir, "snapshots/") {
@@ -415,7 +400,7 @@ func (storage *GCDStorage) ListFiles(threadIndex int, dir string) ([]string, []i
 		files := []string{}

 		for _, entry := range entries {
-			storage.saveFileInfoToCache(dir + "/" + entry.Name, entry.Id, entry.MimeType == GCDDirectoryMimeType, entry.Size)
+			storage.savePathID(dir+"/"+entry.Name, entry.Id)
 			files = append(files, entry.Name)
 		}
 		return files, nil, nil
@@ -426,7 +411,7 @@ func (storage *GCDStorage) ListFiles(threadIndex int, dir string) ([]string, []i
 		parents := []string{"chunks", "fossils"}
 		for i := 0; i < len(parents); i++ {
 			parent := parents[i]
-			pathID, ok, _, _ := storage.findFileInfoFromCache(parent)
+			pathID, ok := storage.findPathID(parent)
 			if !ok {
 				continue
 			}
@@ -435,7 +420,7 @@ func (storage *GCDStorage) ListFiles(threadIndex int, dir string) ([]string, []i
 				return nil, nil, err
 			}
 			for _, entry := range entries {
-				if entry.MimeType != GCDDirectoryMimeType {
+				if entry.MimeType != "application/vnd.google-apps.folder" {
 					name := entry.Name
 					if strings.HasPrefix(parent, "fossils") {
 						name = parent + "/" + name + ".fsl"
@@ -447,9 +432,9 @@ func (storage *GCDStorage) ListFiles(threadIndex int, dir string) ([]string, []i
 					files = append(files, name)
 					sizes = append(sizes, entry.Size)
 				} else {
-					parents = append(parents, parent + "/" + entry.Name)
+					parents = append(parents, parent+"/"+entry.Name)
 				}
-				storage.saveFileInfoToCache(parent + "/" + entry.Name, entry.Id, entry.MimeType == GCDDirectoryMimeType, entry.Size)
+				storage.savePathID(parent+"/"+entry.Name, entry.Id)
 			}
 		}
 		return files, sizes, nil
@@ -469,7 +454,7 @@ func (storage *GCDStorage) DeleteFile(threadIndex int, filePath string) (err err
 	for {
 		err = storage.service.Files.Delete(fileID).Fields("id").Do()
 		if retry, err := storage.shouldRetry(threadIndex, err); err == nil && !retry {
-			storage.deleteFileInfoFromCache(filePath)
+			storage.deletePathID(filePath)
 			return nil
 		} else if retry {
 			continue
@@ -489,7 +474,7 @@ func (storage *GCDStorage) MoveFile(threadIndex int, from string, to string) (er
 	from = storage.convertFilePath(from)
 	to = storage.convertFilePath(to)

-	fileID, ok, isDir, size:= storage.findFileInfoFromCache(from)
+	fileID, ok := storage.findPathID(from)
 	if !ok {
 		return fmt.Errorf("Attempting to rename file %s with unknown id", from)
 	}
@@ -520,8 +505,8 @@ func (storage *GCDStorage) MoveFile(threadIndex int, from string, to string) (er
 		}
 	}

-	storage.saveFileInfoToCache(to, fileID, isDir, size)
-	storage.deleteFileInfoFromCache(from)
+	storage.savePathID(to, storage.getPathID(from))
+	storage.deletePathID(from)
 	return nil
 }

@@ -548,21 +533,19 @@ func (storage *GCDStorage) CreateDirectory(threadIndex int, dir string) (err err
 	if parentDir == "." {
 		parentDir = ""
 	}
-	parentID := storage.getPathIDFromCache(parentDir)
+	parentID := storage.getPathID(parentDir)
 	if parentID == "" {
 		return fmt.Errorf("Parent directory '%s' does not exist", parentDir)
 	}
 	name := path.Base(dir)

-	var file *drive.File
+	file := &drive.File{
+		Name:     name,
+		MimeType: "application/vnd.google-apps.folder",
+		Parents:  []string{parentID},
+	}
+
 	for {
-
-		file = &drive.File{
-			Name:     name,
-			MimeType: GCDDirectoryMimeType,
-			Parents:  []string{parentID},
-		}
-
 		file, err = storage.service.Files.Create(file).Fields("id").Do()
 		if retry, err := storage.shouldRetry(threadIndex, err); err == nil && !retry {
 			break
@@ -582,7 +565,7 @@ func (storage *GCDStorage) CreateDirectory(threadIndex int, dir string) (err err
 		}
 	}

-	storage.saveFileInfoToCache(dir, file.Id, true, 0)
+	storage.savePathID(dir, file.Id)
 	return nil
 }

@@ -593,28 +576,37 @@ func (storage *GCDStorage) GetFileInfo(threadIndex int, filePath string) (exist
 	}
 	filePath = storage.convertFilePath(filePath)

-	fileID, ok, isDir, size := storage.findFileInfoFromCache(filePath)
-	if ok {
-		return true, isDir, size, nil
+	fileID, ok := storage.findPathID(filePath)
+	if !ok {
+		dir := path.Dir(filePath)
+		if dir == "." {
+			dir = ""
+		}
+		dirID, err := storage.getIDFromPath(threadIndex, dir, false)
+		if err != nil {
+			return false, false, 0, err
+		}
+		if dirID == "" {
+			return false, false, 0, nil
+		}
+
+		fileID, isDir, size, err = storage.listByName(threadIndex, dirID, path.Base(filePath))
+		if fileID != "" {
+			storage.savePathID(filePath, fileID)
+		}
+		return fileID != "", isDir, size, err
 	}

-	dir := path.Dir(filePath)
-	if dir == "." {
-		dir = ""
+	for {
+		file, err := storage.service.Files.Get(fileID).Fields("id, mimeType").Do()
+		if retry, err := storage.shouldRetry(threadIndex, err); err == nil && !retry {
+			return true, file.MimeType == "application/vnd.google-apps.folder", file.Size, nil
+		} else if retry {
+			continue
+		} else {
+			return false, false, 0, err
+		}
 	}
-	dirID, err := storage.getIDFromPath(threadIndex, dir, false)
-	if err != nil {
-		return false, false, 0, err
-	}
-	if dirID == "" {
-		return false, false, 0, nil
-	}
-
-	fileID, isDir, size, err = storage.listByName(threadIndex, dirID, path.Base(filePath))
-	if err != nil {
-		return false, false, 0, err
-	}
-	return fileID != "", isDir, size, err
 }

 // DownloadFile reads the file at 'filePath' into the chunk.
@@ -662,18 +654,15 @@ func (storage *GCDStorage) UploadFile(threadIndex int, filePath string, content
 		return err
 	}

-	var file *drive.File
+	file := &drive.File{
+		Name:     path.Base(filePath),
+		MimeType: "application/octet-stream",
+		Parents:  []string{parentID},
+	}

 	for {
-
-		file = &drive.File{
-			Name:     path.Base(filePath),
-			MimeType: GCDFileMimeType,
-			Parents:  []string{parentID},
-		}
-
 		reader := CreateRateLimitedReader(content, storage.UploadRateLimit/storage.numberOfThreads)
-		file, err = storage.service.Files.Create(file).Media(reader).Fields("id").Do()
+		_, err = storage.service.Files.Create(file).Media(reader).Fields("id").Do()
 		if retry, err := storage.shouldRetry(threadIndex, err); err == nil && !retry {
 			break
 		} else if retry {
@@ -683,8 +672,6 @@ func (storage *GCDStorage) UploadFile(threadIndex int, filePath string, content
 		}
 	}

-	storage.saveFileInfoToCache(filePath, file.Id, false, int64(len(content)))
-
 	return err
 }