diff --git a/src/duplicacy_gcdstorage.go b/src/duplicacy_gcdstorage.go index d01e020..275a8f9 100644 --- a/src/duplicacy_gcdstorage.go +++ b/src/duplicacy_gcdstorage.go @@ -24,24 +24,14 @@ import ( "google.golang.org/api/googleapi" ) -var ( - GCDFileMimeType = "application/octet-stream" - GCDDirectoryMimeType = "application/vnd.google-apps.folder" -) -type GCDFileInfo struct { - ID string - IsDir bool - Size int64 -} - type GCDStorage struct { StorageBase - service *drive.Service - fileInfoCache map[string]GCDFileInfo - fileInfoLock sync.Mutex - backoffs []int // desired backoff time in seconds for each thread - attempts []int // number of failed attempts since last success for each thread + service *drive.Service + idCache map[string]string + idCacheLock sync.Mutex + backoffs []int // desired backoff time in seconds for each thread + attempts []int // number of failed attempts since last success for each thread createDirectoryLock sync.Mutex isConnected bool @@ -139,32 +129,30 @@ func (storage *GCDStorage) convertFilePath(filePath string) string { return filePath } -// getPathIDFromCache assumes that the path to be found exists in the cache -func (storage *GCDStorage) getPathIDFromCache(path string) (pathID string) { - storage.fileInfoLock.Lock() - fileInfo := storage.fileInfoCache[path] - storage.fileInfoLock.Unlock() - return fileInfo.ID +func (storage *GCDStorage) getPathID(path string) string { + storage.idCacheLock.Lock() + pathID := storage.idCache[path] + storage.idCacheLock.Unlock() + return pathID } -// findFileInfoFromCache doesn't assume that the path to be found exists in the cache -func (storage *GCDStorage) findFileInfoFromCache(path string) (pathID string, exist bool, isDir bool, size int64) { - storage.fileInfoLock.Lock() - fileInfo, ok := storage.fileInfoCache[path] - storage.fileInfoLock.Unlock() - return fileInfo.ID, ok, fileInfo.IsDir, fileInfo.Size +func (storage *GCDStorage) findPathID(path string) (string, bool) { + storage.idCacheLock.Lock() + pathID, ok := storage.idCache[path] + storage.idCacheLock.Unlock() + return pathID, ok } -func (storage *GCDStorage) saveFileInfoToCache(path string, pathID string, isDir bool, size int64) { - storage.fileInfoLock.Lock() - storage.fileInfoCache[path] = GCDFileInfo { ID: pathID, IsDir: isDir, Size: size } - storage.fileInfoLock.Unlock() +func (storage *GCDStorage) savePathID(path string, pathID string) { + storage.idCacheLock.Lock() + storage.idCache[path] = pathID + storage.idCacheLock.Unlock() } -func (storage *GCDStorage) deleteFileInfoFromCache(path string) { - storage.fileInfoLock.Lock() - delete(storage.fileInfoCache, path) - storage.fileInfoLock.Unlock() +func (storage *GCDStorage) deletePathID(path string) { + storage.idCacheLock.Lock() + delete(storage.idCache, path) + storage.idCacheLock.Unlock() } func (storage *GCDStorage) listFiles(threadIndex int, parentID string, listFiles bool, listDirectories bool) ([]*drive.File, error) { @@ -177,7 +165,7 @@ func (storage *GCDStorage) listFiles(threadIndex int, parentID string, listFiles startToken := "" - query := "'" + parentID + "' in parents and trashed != true " + query := "'" + parentID + "' in parents " if listFiles && !listDirectories { query += "and mimeType != 'application/vnd.google-apps.folder'" } else if !listFiles && !listDirectories { @@ -221,7 +209,7 @@ func (storage *GCDStorage) listByName(threadIndex int, parentID string, name str var err error for { - query := "name = '" + name + "' and '" + parentID + "' in parents and trashed != true" + query := "name = '" + name + "' and '" + parentID + "' in parents" fileList, err = storage.service.Files.List().Q(query).Fields("files(name, mimeType, id, size)").Do() if retry, e := storage.shouldRetry(threadIndex, err); e == nil && !retry { @@ -239,7 +227,7 @@ func (storage *GCDStorage) listByName(threadIndex int, parentID string, name str file := fileList.Files[0] - return file.Id, file.MimeType == GCDDirectoryMimeType, file.Size, nil + return file.Id, file.MimeType == "application/vnd.google-apps.folder", file.Size, nil } // getIDFromPath returns the id of the given path. If 'createDirectories' is true, create the given path and all its @@ -247,13 +235,13 @@ func (storage *GCDStorage) listByName(threadIndex int, parentID string, name str // if the file doesn't exist. func (storage *GCDStorage) getIDFromPath(threadIndex int, filePath string, createDirectories bool) (string, error) { - if fileID, ok, _, _ := storage.findFileInfoFromCache(filePath); ok { + if fileID, ok := storage.findPathID(filePath); ok { return fileID, nil } fileID := "root" - if rootID, ok, _, _ := storage.findFileInfoFromCache(""); ok { + if rootID, ok := storage.findPathID(""); ok { fileID = rootID } @@ -262,7 +250,7 @@ func (storage *GCDStorage) getIDFromPath(threadIndex int, filePath string, creat for i, name := range names { // Find the intermediate directory in the cache first. current = path.Join(current, name) - currentID, ok, _, _ := storage.findFileInfoFromCache(current) + currentID, ok := storage.findPathID(current) if ok { fileID = currentID continue @@ -271,8 +259,7 @@ func (storage *GCDStorage) getIDFromPath(threadIndex int, filePath string, creat // Check if the directory exists. var err error var isDir bool - var size int64 - fileID, isDir, size, err = storage.listByName(threadIndex, fileID, name) + fileID, isDir, _, err = storage.listByName(threadIndex, fileID, name) if err != nil { return "", err } @@ -290,14 +277,14 @@ func (storage *GCDStorage) getIDFromPath(threadIndex int, filePath string, creat if err != nil { return "", fmt.Errorf("Failed to create directory '%s': %v", current, err) } - currentID, ok, _, _= storage.findFileInfoFromCache(current) + currentID, ok = storage.findPathID(current) if !ok { return "", fmt.Errorf("Directory '%s' created by id not found", current) } fileID = currentID continue } else { - storage.saveFileInfoToCache(current, fileID, isDir, size) + storage.savePathID(current, fileID) } if i != len(names)-1 && !isDir { return "", fmt.Errorf("Path '%s' is not a directory", current) @@ -335,7 +322,7 @@ func CreateGCDStorage(tokenFile string, storagePath string, threads int) (storag storage = &GCDStorage{ service: service, numberOfThreads: threads, - fileInfoCache: make(map[string]GCDFileInfo), + idCache: make(map[string]string), backoffs: make([]int, threads), attempts: make([]int, threads), } @@ -350,9 +337,7 @@ func CreateGCDStorage(tokenFile string, storagePath string, threads int) (storag return nil, err } - // Clear the file info cache and set storagePathID as the root id - storage.fileInfoCache = make(map[string]GCDFileInfo) - storage.saveFileInfoToCache("", storagePathID, true, 0) + storage.idCache[""] = storagePathID for _, dir := range []string{"chunks", "snapshots", "fossils"} { dirID, isDir, _, err := storage.listByName(0, storagePathID, dir) @@ -367,7 +352,7 @@ func CreateGCDStorage(tokenFile string, storagePath string, threads int) (storag } else if !isDir { return nil, fmt.Errorf("%s/%s is not a directory", storagePath, dir) } else { - storage.saveFileInfoToCache(dir, dirID, true, 0) + storage.idCache[dir] = dirID } } @@ -386,7 +371,7 @@ func (storage *GCDStorage) ListFiles(threadIndex int, dir string) ([]string, []i if dir == "snapshots" { - files, err := storage.listFiles(threadIndex, storage.getPathIDFromCache(dir), false, true) + files, err := storage.listFiles(threadIndex, storage.getPathID(dir), false, true) if err != nil { return nil, nil, err } @@ -394,8 +379,8 @@ func (storage *GCDStorage) ListFiles(threadIndex int, dir string) ([]string, []i subDirs := []string{} for _, file := range files { - storage.saveFileInfoToCache("snapshots/" + file.Name, file.Id, file.MimeType == GCDDirectoryMimeType, file.Size) - subDirs = append(subDirs, file.Name + "/") + storage.savePathID("snapshots/"+file.Name, file.Id) + subDirs = append(subDirs, file.Name+"/") } return subDirs, nil, nil } else if strings.HasPrefix(dir, "snapshots/") { @@ -415,7 +400,7 @@ func (storage *GCDStorage) ListFiles(threadIndex int, dir string) ([]string, []i files := []string{} for _, entry := range entries { - storage.saveFileInfoToCache(dir + "/" + entry.Name, entry.Id, entry.MimeType == GCDDirectoryMimeType, entry.Size) + storage.savePathID(dir+"/"+entry.Name, entry.Id) files = append(files, entry.Name) } return files, nil, nil @@ -426,7 +411,7 @@ func (storage *GCDStorage) ListFiles(threadIndex int, dir string) ([]string, []i parents := []string{"chunks", "fossils"} for i := 0; i < len(parents); i++ { parent := parents[i] - pathID, ok, _, _ := storage.findFileInfoFromCache(parent) + pathID, ok := storage.findPathID(parent) if !ok { continue } @@ -435,7 +420,7 @@ func (storage *GCDStorage) ListFiles(threadIndex int, dir string) ([]string, []i return nil, nil, err } for _, entry := range entries { - if entry.MimeType != GCDDirectoryMimeType { + if entry.MimeType != "application/vnd.google-apps.folder" { name := entry.Name if strings.HasPrefix(parent, "fossils") { name = parent + "/" + name + ".fsl" @@ -447,9 +432,9 @@ func (storage *GCDStorage) ListFiles(threadIndex int, dir string) ([]string, []i files = append(files, name) sizes = append(sizes, entry.Size) } else { - parents = append(parents, parent + "/" + entry.Name) + parents = append(parents, parent+"/"+entry.Name) } - storage.saveFileInfoToCache(parent + "/" + entry.Name, entry.Id, entry.MimeType == GCDDirectoryMimeType, entry.Size) + storage.savePathID(parent+"/"+entry.Name, entry.Id) } } return files, sizes, nil @@ -469,7 +454,7 @@ func (storage *GCDStorage) DeleteFile(threadIndex int, filePath string) (err err for { err = storage.service.Files.Delete(fileID).Fields("id").Do() if retry, err := storage.shouldRetry(threadIndex, err); err == nil && !retry { - storage.deleteFileInfoFromCache(filePath) + storage.deletePathID(filePath) return nil } else if retry { continue @@ -489,7 +474,7 @@ func (storage *GCDStorage) MoveFile(threadIndex int, from string, to string) (er from = storage.convertFilePath(from) to = storage.convertFilePath(to) - fileID, ok, isDir, size:= storage.findFileInfoFromCache(from) + fileID, ok := storage.findPathID(from) if !ok { return fmt.Errorf("Attempting to rename file %s with unknown id", from) } @@ -520,8 +505,8 @@ func (storage *GCDStorage) MoveFile(threadIndex int, from string, to string) (er } } - storage.saveFileInfoToCache(to, fileID, isDir, size) - storage.deleteFileInfoFromCache(from) + storage.savePathID(to, storage.getPathID(from)) + storage.deletePathID(from) return nil } @@ -548,21 +533,19 @@ func (storage *GCDStorage) CreateDirectory(threadIndex int, dir string) (err err if parentDir == "." { parentDir = "" } - parentID := storage.getPathIDFromCache(parentDir) + parentID := storage.getPathID(parentDir) if parentID == "" { return fmt.Errorf("Parent directory '%s' does not exist", parentDir) } name := path.Base(dir) - var file *drive.File + file := &drive.File{ + Name: name, + MimeType: "application/vnd.google-apps.folder", + Parents: []string{parentID}, + } + for { - - file = &drive.File{ - Name: name, - MimeType: GCDDirectoryMimeType, - Parents: []string{parentID}, - } - file, err = storage.service.Files.Create(file).Fields("id").Do() if retry, err := storage.shouldRetry(threadIndex, err); err == nil && !retry { break @@ -582,7 +565,7 @@ func (storage *GCDStorage) CreateDirectory(threadIndex int, dir string) (err err } } - storage.saveFileInfoToCache(dir, file.Id, true, 0) + storage.savePathID(dir, file.Id) return nil } @@ -593,28 +576,37 @@ func (storage *GCDStorage) GetFileInfo(threadIndex int, filePath string) (exist } filePath = storage.convertFilePath(filePath) - fileID, ok, isDir, size := storage.findFileInfoFromCache(filePath) - if ok { - return true, isDir, size, nil + fileID, ok := storage.findPathID(filePath) + if !ok { + dir := path.Dir(filePath) + if dir == "." { + dir = "" + } + dirID, err := storage.getIDFromPath(threadIndex, dir, false) + if err != nil { + return false, false, 0, err + } + if dirID == "" { + return false, false, 0, nil + } + + fileID, isDir, size, err = storage.listByName(threadIndex, dirID, path.Base(filePath)) + if fileID != "" { + storage.savePathID(filePath, fileID) + } + return fileID != "", isDir, size, err } - dir := path.Dir(filePath) - if dir == "." { - dir = "" + for { + file, err := storage.service.Files.Get(fileID).Fields("id, mimeType").Do() + if retry, err := storage.shouldRetry(threadIndex, err); err == nil && !retry { + return true, file.MimeType == "application/vnd.google-apps.folder", file.Size, nil + } else if retry { + continue + } else { + return false, false, 0, err + } } - dirID, err := storage.getIDFromPath(threadIndex, dir, false) - if err != nil { - return false, false, 0, err - } - if dirID == "" { - return false, false, 0, nil - } - - fileID, isDir, size, err = storage.listByName(threadIndex, dirID, path.Base(filePath)) - if err != nil { - return false, false, 0, err - } - return fileID != "", isDir, size, err } // DownloadFile reads the file at 'filePath' into the chunk. @@ -662,18 +654,15 @@ func (storage *GCDStorage) UploadFile(threadIndex int, filePath string, content return err } - var file *drive.File + file := &drive.File{ + Name: path.Base(filePath), + MimeType: "application/octet-stream", + Parents: []string{parentID}, + } for { - - file = &drive.File{ - Name: path.Base(filePath), - MimeType: GCDFileMimeType, - Parents: []string{parentID}, - } - reader := CreateRateLimitedReader(content, storage.UploadRateLimit/storage.numberOfThreads) - file, err = storage.service.Files.Create(file).Media(reader).Fields("id").Do() + _, err = storage.service.Files.Create(file).Media(reader).Fields("id").Do() if retry, err := storage.shouldRetry(threadIndex, err); err == nil && !retry { break } else if retry { @@ -683,8 +672,6 @@ func (storage *GCDStorage) UploadFile(threadIndex int, filePath string, content } } - storage.saveFileInfoToCache(filePath, file.Id, false, int64(len(content))) - return err }