diff --git a/src/duplicacy_gcdstorage.go b/src/duplicacy_gcdstorage.go index 275a8f9..c3efc0a 100644 --- a/src/duplicacy_gcdstorage.go +++ b/src/duplicacy_gcdstorage.go @@ -24,11 +24,16 @@ import ( "google.golang.org/api/googleapi" ) +var ( + GCDFileMimeType = "application/octet-stream" + GCDDirectoryMimeType = "application/vnd.google-apps.folder" +) + type GCDStorage struct { StorageBase service *drive.Service - idCache map[string]string + idCache map[string]string // only directories are saved in this cache idCacheLock sync.Mutex backoffs []int // desired backoff time in seconds for each thread attempts []int // number of failed attempts since last success for each thread @@ -165,7 +170,7 @@ func (storage *GCDStorage) listFiles(threadIndex int, parentID string, listFiles startToken := "" - query := "'" + parentID + "' in parents " + query := "'" + parentID + "' in parents and trashed = false " if listFiles && !listDirectories { query += "and mimeType != 'application/vnd.google-apps.folder'" } else if !listFiles && !listDirectories { @@ -209,7 +214,7 @@ func (storage *GCDStorage) listByName(threadIndex int, parentID string, name str var err error for { - query := "name = '" + name + "' and '" + parentID + "' in parents" + query := "name = '" + name + "' and '" + parentID + "' in parents and trashed = false " fileList, err = storage.service.Files.List().Q(query).Fields("files(name, mimeType, id, size)").Do() if retry, e := storage.shouldRetry(threadIndex, err); e == nil && !retry { @@ -227,7 +232,7 @@ func (storage *GCDStorage) listByName(threadIndex int, parentID string, name str file := fileList.Files[0] - return file.Id, file.MimeType == "application/vnd.google-apps.folder", file.Size, nil + return file.Id, file.MimeType == GCDDirectoryMimeType, file.Size, nil } // getIDFromPath returns the id of the given path. If 'createDirectories' is true, create the given path and all its @@ -283,10 +288,10 @@ func (storage *GCDStorage) getIDFromPath(threadIndex int, filePath string, creat } fileID = currentID continue - } else { + } else if isDir { storage.savePathID(current, fileID) } - if i != len(names)-1 && !isDir { + if i != len(names) - 1 && !isDir { return "", fmt.Errorf("Path '%s' is not a directory", current) } } @@ -337,6 +342,8 @@ func CreateGCDStorage(tokenFile string, storagePath string, threads int) (storag return nil, err } + // Reset the id cache and start with 'storagePathID' as the root + storage.idCache = make(map[string]string) storage.idCache[""] = storagePathID for _, dir := range []string{"chunks", "snapshots", "fossils"} { @@ -379,8 +386,8 @@ func (storage *GCDStorage) ListFiles(threadIndex int, dir string) ([]string, []i subDirs := []string{} for _, file := range files { - storage.savePathID("snapshots/"+file.Name, file.Id) - subDirs = append(subDirs, file.Name+"/") + storage.savePathID("snapshots/" + file.Name, file.Id) + subDirs = append(subDirs, file.Name + "/") } return subDirs, nil, nil } else if strings.HasPrefix(dir, "snapshots/") { @@ -400,7 +407,6 @@ func (storage *GCDStorage) ListFiles(threadIndex int, dir string) ([]string, []i files := []string{} for _, entry := range entries { - storage.savePathID(dir+"/"+entry.Name, entry.Id) files = append(files, entry.Name) } return files, nil, nil @@ -420,7 +426,7 @@ func (storage *GCDStorage) ListFiles(threadIndex int, dir string) ([]string, []i return nil, nil, err } for _, entry := range entries { - if entry.MimeType != "application/vnd.google-apps.folder" { + if entry.MimeType != GCDDirectoryMimeType { name := entry.Name if strings.HasPrefix(parent, "fossils") { name = parent + "/" + name + ".fsl" @@ -432,9 +438,9 @@ func (storage *GCDStorage) ListFiles(threadIndex int, dir string) ([]string, []i files = append(files, name) sizes = append(sizes, entry.Size) } else { - parents = append(parents, parent+"/"+entry.Name) + parents = append(parents, parent+ "/" + entry.Name) + storage.savePathID(parent + "/" + entry.Name, entry.Id) } - storage.savePathID(parent+"/"+entry.Name, entry.Id) } } return files, sizes, nil @@ -474,9 +480,12 @@ func (storage *GCDStorage) MoveFile(threadIndex int, from string, to string) (er from = storage.convertFilePath(from) to = storage.convertFilePath(to) - fileID, ok := storage.findPathID(from) - if !ok { - return fmt.Errorf("Attempting to rename file %s with unknown id", from) + fileID, err := storage.getIDFromPath(threadIndex, from, false) + if err != nil { + return fmt.Errorf("Failed to retrieve the id of '%s': %v", from, err) + } + if fileID == "" { + return fmt.Errorf("The file '%s' to be moved does not exist", from) } fromParent := path.Dir(from) @@ -505,8 +514,6 @@ func (storage *GCDStorage) MoveFile(threadIndex int, from string, to string) (er } } - storage.savePathID(to, storage.getPathID(from)) - storage.deletePathID(from) return nil } @@ -539,21 +546,22 @@ func (storage *GCDStorage) CreateDirectory(threadIndex int, dir string) (err err } name := path.Base(dir) - file := &drive.File{ - Name: name, - MimeType: "application/vnd.google-apps.folder", - Parents: []string{parentID}, - } + var file *drive.File for { + file = &drive.File{ + Name: name, + MimeType: GCDDirectoryMimeType, + Parents: []string{parentID}, + } + file, err = storage.service.Files.Create(file).Fields("id").Do() if retry, err := storage.shouldRetry(threadIndex, err); err == nil && !retry { break } else { // Check if the directory has already been created by other thread - exist, _, _, newErr := storage.GetFileInfo(threadIndex, dir) - if newErr == nil && exist { + if _, ok := storage.findPathID(dir); ok { return nil } @@ -577,36 +585,29 @@ func (storage *GCDStorage) GetFileInfo(threadIndex int, filePath string) (exist filePath = storage.convertFilePath(filePath) fileID, ok := storage.findPathID(filePath) - if !ok { - dir := path.Dir(filePath) - if dir == "." { - dir = "" - } - dirID, err := storage.getIDFromPath(threadIndex, dir, false) - if err != nil { - return false, false, 0, err - } - if dirID == "" { - return false, false, 0, nil - } - - fileID, isDir, size, err = storage.listByName(threadIndex, dirID, path.Base(filePath)) - if fileID != "" { - storage.savePathID(filePath, fileID) - } - return fileID != "", isDir, size, err + if ok { + // Only directories are saved in the case so this must be a directory + return true, true, 0, nil } - for { - file, err := storage.service.Files.Get(fileID).Fields("id, mimeType").Do() - if retry, err := storage.shouldRetry(threadIndex, err); err == nil && !retry { - return true, file.MimeType == "application/vnd.google-apps.folder", file.Size, nil - } else if retry { - continue - } else { - return false, false, 0, err - } + dir := path.Dir(filePath) + if dir == "." { + dir = "" } + dirID, err := storage.getIDFromPath(threadIndex, dir, false) + if err != nil { + return false, false, 0, err + } + if dirID == "" { + return false, false, 0, nil + } + + fileID, isDir, size, err = storage.listByName(threadIndex, dirID, path.Base(filePath)) + if fileID != "" && isDir { + storage.savePathID(filePath, fileID) + } + return fileID != "", isDir, size, err + } // DownloadFile reads the file at 'filePath' into the chunk. @@ -656,7 +657,7 @@ func (storage *GCDStorage) UploadFile(threadIndex int, filePath string, content file := &drive.File{ Name: path.Base(filePath), - MimeType: "application/octet-stream", + MimeType: GCDFileMimeType, Parents: []string{parentID}, }