From 474f07e5cc0166a95aeca5779c3344c4ad5a7060 Mon Sep 17 00:00:00 2001 From: Gilbert Chen Date: Mon, 23 Nov 2020 09:44:10 -0500 Subject: [PATCH 1/4] Support GCD impersonation via modified service account file --- src/duplicacy_gcdstorage.go | 29 +++++++++++++++++++++++++---- src/duplicacy_storage_test.go | 4 ++++ 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/src/duplicacy_gcdstorage.go b/src/duplicacy_gcdstorage.go index 85c4c93..0130020 100644 --- a/src/duplicacy_gcdstorage.go +++ b/src/duplicacy_gcdstorage.go @@ -41,6 +41,7 @@ type GCDStorage struct { backoffs []int // desired backoff time in seconds for each thread attempts []int // number of failed attempts since last success for each thread driveID string // the ID of the shared drive or 'root' (GCDUserDrive) if the user's drive + spaces string // 'appDataFolder' if scope is drive.appdata; 'drive' otherwise createDirectoryLock sync.Mutex isConnected bool @@ -199,7 +200,7 @@ func (storage *GCDStorage) listFiles(threadIndex int, parentID string, listFiles var err error for { - q := storage.service.Files.List().Q(query).Fields("nextPageToken", "files(name, mimeType, id, size)").PageToken(startToken).PageSize(maxCount) + q := storage.service.Files.List().Q(query).Fields("nextPageToken", "files(name, mimeType, id, size)").PageToken(startToken).PageSize(maxCount).Spaces(storage.spaces) if storage.driveID != GCDUserDrive { q = q.DriveId(storage.driveID).IncludeItemsFromAllDrives(true).Corpora("drive").SupportsAllDrives(true) } @@ -231,7 +232,7 @@ func (storage *GCDStorage) listByName(threadIndex int, parentID string, name str for { query := "name = '" + name + "' and '" + parentID + "' in parents and trashed = false " - q := storage.service.Files.List().Q(query).Fields("files(name, mimeType, id, size)") + q := storage.service.Files.List().Q(query).Fields("files(name, mimeType, id, size)").Spaces(storage.spaces) if storage.driveID != GCDUserDrive { q = q.DriveId(storage.driveID).IncludeItemsFromAllDrives(true).Corpora("drive").SupportsAllDrives(true) } @@ -344,11 +345,23 @@ func CreateGCDStorage(tokenFile string, driveID string, storagePath string, thre var tokenSource oauth2.TokenSource + scope := drive.DriveScope + if isServiceAccount { - config, err := google.JWTConfigFromJSON(description, drive.DriveScope) + + if newScope, ok := object["scope"]; ok { + scope = newScope.(string) + } + + config, err := google.JWTConfigFromJSON(description, scope) if err != nil { return nil, err } + + if subject, ok := object["subject"]; ok { + config.Subject = subject.(string) + } + tokenSource = config.TokenSource(ctx) } else { gcdConfig := &GCDConfig{} @@ -398,6 +411,7 @@ func CreateGCDStorage(tokenFile string, driveID string, storagePath string, thre backoffs: make([]int, threads), attempts: make([]int, threads), driveID: driveID, + spaces: "drive", } for i := range storage.backoffs { @@ -405,7 +419,14 @@ func CreateGCDStorage(tokenFile string, driveID string, storagePath string, thre storage.attempts[i] = 0 } - storage.savePathID("", driveID) + + if scope == drive.DriveAppdataScope { + storage.spaces = "appDataFolder" + storage.savePathID("", "appDataFolder") + } else { + storage.savePathID("", driveID) + } + storagePathID, err := storage.getIDFromPath(0, storagePath, true) if err != nil { return nil, err diff --git a/src/duplicacy_storage_test.go b/src/duplicacy_storage_test.go index 43c1c2a..e7fe5f2 100644 --- a/src/duplicacy_storage_test.go +++ b/src/duplicacy_storage_test.go @@ -142,6 +142,10 @@ func loadStorage(localStoragePath string, threads int) (Storage, error) { storage, err := CreateGCDStorage(config["token_file"], config["drive"], config["storage_path"], threads) storage.SetDefaultNestingLevels([]int{2, 3}, 2) return storage, err + } else if testStorageName == "gcd-impersonate" { + storage, err := CreateGCDStorage(config["token_file"], config["drive"], config["storage_path"], threads) + storage.SetDefaultNestingLevels([]int{2, 3}, 2) + return storage, err } else if testStorageName == "one" { storage, err := CreateOneDriveStorage(config["token_file"], false, config["storage_path"], threads) storage.SetDefaultNestingLevels([]int{2, 3}, 2) From e43e848d47176144049b93938f09820c61effc72 Mon Sep 17 00:00:00 2001 From: Gilbert Chen Date: Tue, 9 Mar 2021 22:46:23 -0500 Subject: [PATCH 2/4] Find the storage path in shared folders first when connecting to Google Drive When connecting to Google Drive with a service account key, only files in the service account's own hidden drive space are listable. This change finds the given storage path among shared folders first so that folders from the user space can be made accessible via service account. --- src/duplicacy_gcdstorage.go | 41 ++++++++++++++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/src/duplicacy_gcdstorage.go b/src/duplicacy_gcdstorage.go index 0130020..af86a6e 100644 --- a/src/duplicacy_gcdstorage.go +++ b/src/duplicacy_gcdstorage.go @@ -256,6 +256,29 @@ func (storage *GCDStorage) listByName(threadIndex int, parentID string, name str return file.Id, file.MimeType == GCDDirectoryMimeType, file.Size, nil } +// Returns the id of the shared folder with the given name if it exists +func (storage *GCDStorage) findSharedFolder(threadIndex int, name string) (string, error) { + + query := "name = '" + name + "' and sharedWithMe and trashed = false and mimeType = 'application/vnd.google-apps.folder'" + q := storage.service.Files.List().Q(query).Fields("files(name, mimeType, id, size)").Spaces(storage.spaces) + if storage.driveID != GCDUserDrive { + q = q.DriveId(storage.driveID).IncludeItemsFromAllDrives(true).Corpora("drive").SupportsAllDrives(true) + } + + fileList, err := q.Do() + if err != nil { + return "", err + } + + if len(fileList.Files) == 0 { + return "", nil + } + + file := fileList.Files[0] + + return file.Id, nil +} + // getIDFromPath returns the id of the given path. If 'createDirectories' is true, create the given path and all its // parent directories if they don't exist. Note that if 'createDirectories' is false, it may return an empty 'fileID' // if the file doesn't exist. @@ -427,9 +450,21 @@ func CreateGCDStorage(tokenFile string, driveID string, storagePath string, thre storage.savePathID("", driveID) } - storagePathID, err := storage.getIDFromPath(0, storagePath, true) - if err != nil { - return nil, err + storagePathID := "" + + // When using service acount, check if storagePath is a shared folder which takes priority over regular folders. + if isServiceAccount && !strings.Contains(storagePath, "/") { + storagePathID, err = storage.findSharedFolder(0, storagePath) + if err != nil { + LOG_WARN("GCD_STORAGE", "Failed to check if %s is a shared folder: %v", storagePath, err) + } + } + + if storagePathID == "" { + storagePathID, err = storage.getIDFromPath(0, storagePath, true) + if err != nil { + return nil, err + } } // Reset the id cache and start with 'storagePathID' as the root From cacf6618d21f12039e4edd8758ab3e2c4aac04a6 Mon Sep 17 00:00:00 2001 From: Gilbert Chen Date: Fri, 8 Oct 2021 14:04:56 -0400 Subject: [PATCH 3/4] Download a fossil directly instead of turning it back to a chunk first This is to avoid the read-after-rename consistency issue where the effect of renaming may not be observed by the subsequent attempt to download the just renamed chunk. --- src/duplicacy_chunkdownloader.go | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/src/duplicacy_chunkdownloader.go b/src/duplicacy_chunkdownloader.go index f48cd0f..b4c0579 100644 --- a/src/duplicacy_chunkdownloader.go +++ b/src/duplicacy_chunkdownloader.go @@ -427,17 +427,10 @@ func (downloader *ChunkDownloader) Download(threadIndex int, task ChunkDownloadT return false } - // We can't download the fossil directly. We have to turn it back into a regular chunk and try - // downloading again. - err = downloader.storage.MoveFile(threadIndex, fossilPath, chunkPath) - if err != nil { - completeFailedChunk(chunk) - LOG_WERROR(downloader.allowFailures, "DOWNLOAD_CHUNK", "Failed to resurrect chunk %s: %v", chunkID, err) - return false - } - - LOG_WARN("DOWNLOAD_RESURRECT", "Fossil %s has been resurrected", chunkID) - continue + // Don't try to resurrect the fossil as we did before. This is to avoid the potential read-after-rename + // consistency issue. Instead, download the fossil directly; resurrection should be taken care of later. + chunkPath = fossilPath + LOG_WARN("DOWNLOAD_FOSSIL", "Chunk %s is a fossil", chunkID) } err = downloader.storage.DownloadFile(threadIndex, chunkPath, chunk) From 68b60499d7dca73dd94fe6f01a4f2e0fd3481282 Mon Sep 17 00:00:00 2001 From: Gilbert Chen Date: Fri, 15 Oct 2021 20:45:53 -0400 Subject: [PATCH 4/4] Add a global option to print memory usage This option, -print-memory-usage, will print memory usage every second while the program is running. --- duplicacy/duplicacy_main.go | 8 ++++++++ src/duplicacy_utils.go | 14 ++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/duplicacy/duplicacy_main.go b/duplicacy/duplicacy_main.go index 9fe4ce0..88cad5d 100644 --- a/duplicacy/duplicacy_main.go +++ b/duplicacy/duplicacy_main.go @@ -147,6 +147,10 @@ func setGlobalOptions(context *cli.Context) { duplicacy.SetLoggingLevel(duplicacy.DEBUG) } + if context.GlobalBool("print-memory-usage") { + go duplicacy.PrintMemoryUsage() + } + ScriptEnabled = true if context.GlobalBool("no-script") { ScriptEnabled = false @@ -2180,6 +2184,10 @@ func main() { Usage: "suppress logs with the specified id", Argument: "", }, + cli.BoolFlag{ + Name: "print-memory-usage", + Usage: "print memory usage every second", + }, } app.HideVersion = true diff --git a/src/duplicacy_utils.go b/src/duplicacy_utils.go index 3b3cc5a..7f6754b 100644 --- a/src/duplicacy_utils.go +++ b/src/duplicacy_utils.go @@ -14,6 +14,7 @@ import ( "strconv" "strings" "time" + "runtime" "github.com/gilbertchen/gopass" "golang.org/x/crypto/pbkdf2" @@ -460,3 +461,16 @@ func AtoSize(sizeString string) int { return size } + +func PrintMemoryUsage() { + + for { + var m runtime.MemStats + runtime.ReadMemStats(&m) + + LOG_INFO("MEMORY_STATS", "Currently allocated: %s, total allocated: %s, system memory: %s, number of GCs: %d", + PrettySize(int64(m.Alloc)), PrettySize(int64(m.TotalAlloc)), PrettySize(int64(m.Sys)), m.NumGC) + + time.Sleep(time.Second) + } +}