Fix the GCD directory creating bug; only save directories in the id cache

src/duplicacy_gcdstorage.go

@@ -24,11 +24,16 @@ import (
 	"google.golang.org/api/googleapi"
 )
 
+var (
+	GCDFileMimeType = "application/octet-stream"
+	GCDDirectoryMimeType = "application/vnd.google-apps.folder"
+)
+
 type GCDStorage struct {
 	StorageBase
 
 	service     *drive.Service
-	idCache     map[string]string
+	idCache     map[string]string  // only directories are saved in this cache
 	idCacheLock sync.Mutex
 	backoffs    []int // desired backoff time in seconds for each thread
 	attempts    []int // number of failed attempts since last success for each thread
@@ -165,7 +170,7 @@ func (storage *GCDStorage) listFiles(threadIndex int, parentID string, listFiles
 
 	startToken := ""
 
-	query := "'" + parentID + "' in parents "
+	query := "'" + parentID + "' in parents and trashed = false "
 	if listFiles && !listDirectories {
 		query += "and mimeType != 'application/vnd.google-apps.folder'"
 	} else if !listFiles && !listDirectories {
@@ -209,7 +214,7 @@ func (storage *GCDStorage) listByName(threadIndex int, parentID string, name str
 	var err error
 
 	for {
-		query := "name = '" + name + "' and '" + parentID + "' in parents"
+		query := "name = '" + name + "' and '" + parentID + "' in parents and trashed = false "
 		fileList, err = storage.service.Files.List().Q(query).Fields("files(name, mimeType, id, size)").Do()
 
 		if retry, e := storage.shouldRetry(threadIndex, err); e == nil && !retry {
@@ -227,7 +232,7 @@ func (storage *GCDStorage) listByName(threadIndex int, parentID string, name str
 
 	file := fileList.Files[0]
 
-	return file.Id, file.MimeType == "application/vnd.google-apps.folder", file.Size, nil
+	return file.Id, file.MimeType == GCDDirectoryMimeType, file.Size, nil
 }
 
 // getIDFromPath returns the id of the given path.  If 'createDirectories' is true, create the given path and all its
@@ -283,10 +288,10 @@ func (storage *GCDStorage) getIDFromPath(threadIndex int, filePath string, creat
 			}
 			fileID = currentID
 			continue
-		} else {
+		} else if isDir {
 			storage.savePathID(current, fileID)
 		}
-		if i != len(names)-1 && !isDir {
+		if i != len(names) - 1 && !isDir {
 			return "", fmt.Errorf("Path '%s' is not a directory", current)
 		}
 	}
@@ -337,6 +342,8 @@ func CreateGCDStorage(tokenFile string, storagePath string, threads int) (storag
 		return nil, err
 	}
 
+	// Reset the id cache and start with 'storagePathID' as the root
+	storage.idCache = make(map[string]string)
 	storage.idCache[""] = storagePathID
 
 	for _, dir := range []string{"chunks", "snapshots", "fossils"} {
@@ -379,8 +386,8 @@ func (storage *GCDStorage) ListFiles(threadIndex int, dir string) ([]string, []i
 		subDirs := []string{}
 
 		for _, file := range files {
-			storage.savePathID("snapshots/"+file.Name, file.Id)
+			storage.savePathID("snapshots/" + file.Name, file.Id)
-			subDirs = append(subDirs, file.Name+"/")
+			subDirs = append(subDirs, file.Name + "/")
 		}
 		return subDirs, nil, nil
 	} else if strings.HasPrefix(dir, "snapshots/") {
@@ -400,7 +407,6 @@ func (storage *GCDStorage) ListFiles(threadIndex int, dir string) ([]string, []i
 		files := []string{}
 
 		for _, entry := range entries {
-			storage.savePathID(dir+"/"+entry.Name, entry.Id)
 			files = append(files, entry.Name)
 		}
 		return files, nil, nil
@@ -420,7 +426,7 @@ func (storage *GCDStorage) ListFiles(threadIndex int, dir string) ([]string, []i
 				return nil, nil, err
 			}
 			for _, entry := range entries {
-				if entry.MimeType != "application/vnd.google-apps.folder" {
+				if entry.MimeType != GCDDirectoryMimeType {
 					name := entry.Name
 					if strings.HasPrefix(parent, "fossils") {
 						name = parent + "/" + name + ".fsl"
@@ -432,9 +438,9 @@ func (storage *GCDStorage) ListFiles(threadIndex int, dir string) ([]string, []i
 					files = append(files, name)
 					sizes = append(sizes, entry.Size)
 				} else {
-					parents = append(parents, parent+"/"+entry.Name)
+					parents = append(parents, parent+ "/" + entry.Name)
+					storage.savePathID(parent + "/" + entry.Name, entry.Id)
 				}
-				storage.savePathID(parent+"/"+entry.Name, entry.Id)
 			}
 		}
 		return files, sizes, nil
@@ -474,9 +480,12 @@ func (storage *GCDStorage) MoveFile(threadIndex int, from string, to string) (er
 	from = storage.convertFilePath(from)
 	to = storage.convertFilePath(to)
 
-	fileID, ok := storage.findPathID(from)
+	fileID, err := storage.getIDFromPath(threadIndex, from, false)
-	if !ok {
+	if err != nil {
-		return fmt.Errorf("Attempting to rename file %s with unknown id", from)
+		return fmt.Errorf("Failed to retrieve the id of '%s': %v", from, err)
+	}
+	if fileID == "" {
+		return fmt.Errorf("The file '%s' to be moved does not exist", from)
 	}
 
 	fromParent := path.Dir(from)
@@ -505,8 +514,6 @@ func (storage *GCDStorage) MoveFile(threadIndex int, from string, to string) (er
 		}
 	}
 
-	storage.savePathID(to, storage.getPathID(from))
-	storage.deletePathID(from)
 	return nil
 }
 
@@ -539,21 +546,22 @@ func (storage *GCDStorage) CreateDirectory(threadIndex int, dir string) (err err
 	}
 	name := path.Base(dir)
 
-	file := &drive.File{
+	var file *drive.File
-		Name:     name,
-		MimeType: "application/vnd.google-apps.folder",
-		Parents:  []string{parentID},
-	}
 
 	for {
+		file = &drive.File{
+			Name:     name,
+			MimeType: GCDDirectoryMimeType,
+			Parents:  []string{parentID},
+		}
+
 		file, err = storage.service.Files.Create(file).Fields("id").Do()
 		if retry, err := storage.shouldRetry(threadIndex, err); err == nil && !retry {
 			break
 		} else {
 
 			// Check if the directory has already been created by other thread
-			exist, _, _, newErr := storage.GetFileInfo(threadIndex, dir)
+			if _, ok := storage.findPathID(dir); ok {
-			if newErr == nil && exist {
 				return nil
 			}
 
@@ -577,36 +585,29 @@ func (storage *GCDStorage) GetFileInfo(threadIndex int, filePath string) (exist
 	filePath = storage.convertFilePath(filePath)
 
 	fileID, ok := storage.findPathID(filePath)
-	if !ok {
+	if ok {
-		dir := path.Dir(filePath)
+		// Only directories are saved in the case so this must be a directory
-		if dir == "." {
+		return true, true, 0, nil
-			dir = ""
-		}
-		dirID, err := storage.getIDFromPath(threadIndex, dir, false)
-		if err != nil {
-			return false, false, 0, err
-		}
-		if dirID == "" {
-			return false, false, 0, nil
-		}
-
-		fileID, isDir, size, err = storage.listByName(threadIndex, dirID, path.Base(filePath))
-		if fileID != "" {
-			storage.savePathID(filePath, fileID)
-		}
-		return fileID != "", isDir, size, err
 	}
 
-	for {
+	dir := path.Dir(filePath)
-		file, err := storage.service.Files.Get(fileID).Fields("id, mimeType").Do()
+	if dir == "." {
-		if retry, err := storage.shouldRetry(threadIndex, err); err == nil && !retry {
+		dir = ""
-			return true, file.MimeType == "application/vnd.google-apps.folder", file.Size, nil
-		} else if retry {
-			continue
-		} else {
-			return false, false, 0, err
-		}
 	}
+	dirID, err := storage.getIDFromPath(threadIndex, dir, false)
+	if err != nil {
+		return false, false, 0, err
+	}
+	if dirID == "" {
+		return false, false, 0, nil
+	}
+
+	fileID, isDir, size, err = storage.listByName(threadIndex, dirID, path.Base(filePath))
+	if fileID != "" && isDir {
+		storage.savePathID(filePath, fileID)
+	}
+	return fileID != "", isDir, size, err
+
 }
 
 // DownloadFile reads the file at 'filePath' into the chunk.
@@ -656,7 +657,7 @@ func (storage *GCDStorage) UploadFile(threadIndex int, filePath string, content
 
 	file := &drive.File{
 		Name:     path.Base(filePath),
-		MimeType: "application/octet-stream",
+		MimeType: GCDFileMimeType,
 		Parents:  []string{parentID},
 	}