Merge branch 'master' into memory_optimization

src/duplicacy_chunkdownloader.go

@@ -255,5 +255,4 @@ func (downloader *ChunkDownloader) WaitForCompletion() {
 			downloader.lastChunkIndex++
 		}
 	}
-}
-
+}

src/duplicacy_gcdstorage.go

@@ -41,6 +41,7 @@ type GCDStorage struct {
 	backoffs    []int // desired backoff time in seconds for each thread
 	attempts    []int // number of failed attempts since last success for each thread
 	driveID     string // the ID of the shared drive or 'root' (GCDUserDrive) if the user's drive
+	spaces      string // 'appDataFolder' if scope is drive.appdata; 'drive' otherwise
 
 	createDirectoryLock sync.Mutex
 	isConnected         bool
@@ -199,7 +200,7 @@ func (storage *GCDStorage) listFiles(threadIndex int, parentID string, listFiles
 		var err error
 
 		for {
-			q := storage.service.Files.List().Q(query).Fields("nextPageToken", "files(name, mimeType, id, size)").PageToken(startToken).PageSize(maxCount)
+			q := storage.service.Files.List().Q(query).Fields("nextPageToken", "files(name, mimeType, id, size)").PageToken(startToken).PageSize(maxCount).Spaces(storage.spaces)
 			if storage.driveID != GCDUserDrive {
 				q = q.DriveId(storage.driveID).IncludeItemsFromAllDrives(true).Corpora("drive").SupportsAllDrives(true)
 			}
@@ -231,7 +232,7 @@ func (storage *GCDStorage) listByName(threadIndex int, parentID string, name str
 
 	for {
 		query := "name = '" + name + "' and '" + parentID + "' in parents and trashed = false "
-		q := storage.service.Files.List().Q(query).Fields("files(name, mimeType, id, size)")
+		q := storage.service.Files.List().Q(query).Fields("files(name, mimeType, id, size)").Spaces(storage.spaces)
 		if storage.driveID != GCDUserDrive {
 			q = q.DriveId(storage.driveID).IncludeItemsFromAllDrives(true).Corpora("drive").SupportsAllDrives(true)
 		}
@@ -255,6 +256,29 @@ func (storage *GCDStorage) listByName(threadIndex int, parentID string, name str
 	return file.Id, file.MimeType == GCDDirectoryMimeType, file.Size, nil
 }
 
+// Returns the id of the shared folder with the given name if it exists
+func (storage *GCDStorage) findSharedFolder(threadIndex int, name string) (string, error) {
+
+	query := "name = '" + name + "' and sharedWithMe and trashed = false and mimeType = 'application/vnd.google-apps.folder'"
+	q := storage.service.Files.List().Q(query).Fields("files(name, mimeType, id, size)").Spaces(storage.spaces)
+	if storage.driveID != GCDUserDrive {
+		q = q.DriveId(storage.driveID).IncludeItemsFromAllDrives(true).Corpora("drive").SupportsAllDrives(true)
+	}
+
+	fileList, err := q.Do()
+	if err != nil {
+		return "", err
+	}
+
+	if len(fileList.Files) == 0 {
+		return "", nil
+	}
+
+	file := fileList.Files[0]
+
+	return file.Id, nil
+}
+
 // getIDFromPath returns the id of the given path.  If 'createDirectories' is true, create the given path and all its
 // parent directories if they don't exist.  Note that if 'createDirectories' is false, it may return an empty 'fileID'
 // if the file doesn't exist.
@@ -344,11 +368,23 @@ func CreateGCDStorage(tokenFile string, driveID string, storagePath string, thre
 
 	var tokenSource oauth2.TokenSource
 
+	scope := drive.DriveScope
+
 	if isServiceAccount {
-		config, err := google.JWTConfigFromJSON(description, drive.DriveScope)
+
+		if newScope, ok := object["scope"]; ok {
+			scope = newScope.(string)
+		}
+
+		config, err := google.JWTConfigFromJSON(description, scope)
 		if err != nil {
 			return nil, err
 		}
+
+		if subject, ok := object["subject"]; ok {
+		    config.Subject = subject.(string)
+		}
+
 		tokenSource = config.TokenSource(ctx)
 	} else {
 		gcdConfig := &GCDConfig{}
@@ -398,6 +434,7 @@ func CreateGCDStorage(tokenFile string, driveID string, storagePath string, thre
 		backoffs:        make([]int, threads),
 		attempts:        make([]int, threads),
 		driveID:         driveID,
+		spaces:          "drive",
 	}
 
 	for i := range storage.backoffs {
@@ -405,10 +442,29 @@ func CreateGCDStorage(tokenFile string, driveID string, storagePath string, thre
 		storage.attempts[i] = 0
 	}
 
-	storage.savePathID("", driveID)
-	storagePathID, err := storage.getIDFromPath(0, storagePath, true)
-	if err != nil {
-		return nil, err
+
+	if scope == drive.DriveAppdataScope {
+		storage.spaces = "appDataFolder"
+		storage.savePathID("", "appDataFolder")
+	} else {
+		storage.savePathID("", driveID)
+	}
+
+	storagePathID := ""
+
+	// When using service acount, check if storagePath is a shared folder which takes priority over regular folders.
+	if isServiceAccount && !strings.Contains(storagePath, "/") {
+		storagePathID, err = storage.findSharedFolder(0, storagePath)
+		if err != nil {
+			LOG_WARN("GCD_STORAGE", "Failed to check if %s is a shared folder: %v", storagePath, err)
+		}
+	}
+
+	if storagePathID == "" {
+		storagePathID, err = storage.getIDFromPath(0, storagePath, true)
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	// Reset the id cache and start with 'storagePathID' as the root

src/duplicacy_storage_test.go

@@ -142,6 +142,10 @@ func loadStorage(localStoragePath string, threads int) (Storage, error) {
 		storage, err := CreateGCDStorage(config["token_file"], config["drive"], config["storage_path"], threads)
 		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
 		return storage, err
+	} else if testStorageName == "gcd-impersonate" {
+		storage, err := CreateGCDStorage(config["token_file"], config["drive"], config["storage_path"], threads)
+		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
+		return storage, err
 	} else if testStorageName == "one" {
 		storage, err := CreateOneDriveStorage(config["token_file"], false, config["storage_path"], threads)
 		storage.SetDefaultNestingLevels([]int{2, 3}, 2)