From 426110e961b443a49eb6d61fd4d17347513528c7 Mon Sep 17 00:00:00 2001 From: Richard Sanger Date: Wed, 6 Nov 2019 00:37:50 +1300 Subject: [PATCH 1/3] Adds support for GDrive Shared Drives A shared drive can be accessed via gcd://sharedDriveId@path/to/storage sharedDriveId is optional and if omitted duplicacy stores to the user's drive. This remains backwards compatible with existing drives. E.g. gcd://path/to/storage Note: Shared Drives were previously named Team Drives. --- src/duplicacy_gcdstorage.go | 33 ++++++++++++++++++++++++--------- src/duplicacy_storage.go | 6 +++++- src/duplicacy_storage_test.go | 2 +- 3 files changed, 30 insertions(+), 11 deletions(-) diff --git a/src/duplicacy_gcdstorage.go b/src/duplicacy_gcdstorage.go index 61fdb32..c29270d 100644 --- a/src/duplicacy_gcdstorage.go +++ b/src/duplicacy_gcdstorage.go @@ -27,6 +27,7 @@ import ( var ( GCDFileMimeType = "application/octet-stream" GCDDirectoryMimeType = "application/vnd.google-apps.folder" + GCDUserDrive = "root" ) type GCDStorage struct { @@ -37,6 +38,7 @@ type GCDStorage struct { idCacheLock sync.Mutex backoffs []int // desired backoff time in seconds for each thread attempts []int // number of failed attempts since last success for each thread + driveID string // the ID of the shared drive or 'root' (GCDUserDrive) if the user's drive createDirectoryLock sync.Mutex isConnected bool @@ -191,7 +193,11 @@ func (storage *GCDStorage) listFiles(threadIndex int, parentID string, listFiles var err error for { - fileList, err = storage.service.Files.List().Q(query).Fields("nextPageToken", "files(name, mimeType, id, size)").PageToken(startToken).PageSize(maxCount).Do() + q := storage.service.Files.List().Q(query).Fields("nextPageToken", "files(name, mimeType, id, size)").PageToken(startToken).PageSize(maxCount) + if storage.driveID != GCDUserDrive { + q = q.DriveId(storage.driveID).IncludeItemsFromAllDrives(true).Corpora("drive").SupportsAllDrives(true) + } + fileList, err = q.Do() if retry, e := storage.shouldRetry(threadIndex, err); e == nil && !retry { break } else if retry { @@ -219,7 +225,11 @@ func (storage *GCDStorage) listByName(threadIndex int, parentID string, name str for { query := "name = '" + name + "' and '" + parentID + "' in parents and trashed = false " - fileList, err = storage.service.Files.List().Q(query).Fields("files(name, mimeType, id, size)").Do() + q := storage.service.Files.List().Q(query).Fields("files(name, mimeType, id, size)") + if storage.driveID != GCDUserDrive { + q = q.DriveId(storage.driveID).IncludeItemsFromAllDrives(true).Corpora("drive").SupportsAllDrives(true) + } + fileList, err = q.Do() if retry, e := storage.shouldRetry(threadIndex, err); e == nil && !retry { break @@ -248,7 +258,7 @@ func (storage *GCDStorage) getIDFromPath(threadIndex int, filePath string, creat return fileID, nil } - fileID := "root" + fileID := storage.driveID if rootID, ok := storage.findPathID(""); ok { fileID = rootID @@ -303,7 +313,7 @@ func (storage *GCDStorage) getIDFromPath(threadIndex int, filePath string, creat } // CreateGCDStorage creates a GCD storage object. -func CreateGCDStorage(tokenFile string, storagePath string, threads int) (storage *GCDStorage, err error) { +func CreateGCDStorage(tokenFile string, driveID string, storagePath string, threads int) (storage *GCDStorage, err error) { description, err := ioutil.ReadFile(tokenFile) if err != nil { @@ -328,12 +338,17 @@ func CreateGCDStorage(tokenFile string, storagePath string, threads int) (storag return nil, err } + if len(driveID) == 0 { + driveID = GCDUserDrive + } + storage = &GCDStorage{ service: service, numberOfThreads: threads, idCache: make(map[string]string), backoffs: make([]int, threads), attempts: make([]int, threads), + driveID: driveID, } for i := range storage.backoffs { @@ -462,7 +477,7 @@ func (storage *GCDStorage) DeleteFile(threadIndex int, filePath string) (err err } for { - err = storage.service.Files.Delete(fileID).Fields("id").Do() + err = storage.service.Files.Delete(fileID).SupportsAllDrives(true).Fields("id").Do() if retry, err := storage.shouldRetry(threadIndex, err); err == nil && !retry { storage.deletePathID(filePath) return nil @@ -508,7 +523,7 @@ func (storage *GCDStorage) MoveFile(threadIndex int, from string, to string) (er } for { - _, err = storage.service.Files.Update(fileID, nil).AddParents(toParentID).RemoveParents(fromParentID).Do() + _, err = storage.service.Files.Update(fileID, nil).SupportsAllDrives(true).AddParents(toParentID).RemoveParents(fromParentID).Do() if retry, err := storage.shouldRetry(threadIndex, err); err == nil && !retry { break } else if retry { @@ -559,7 +574,7 @@ func (storage *GCDStorage) CreateDirectory(threadIndex int, dir string) (err err Parents: []string{parentID}, } - file, err = storage.service.Files.Create(file).Fields("id").Do() + file, err = storage.service.Files.Create(file).SupportsAllDrives(true).Fields("id").Do() if retry, err := storage.shouldRetry(threadIndex, err); err == nil && !retry { break } else { @@ -630,7 +645,7 @@ func (storage *GCDStorage) DownloadFile(threadIndex int, filePath string, chunk for { // AcknowledgeAbuse(true) lets the download proceed even if GCD thinks that it contains malware. // TODO: Should this prompt the user or log a warning? - req := storage.service.Files.Get(fileID) + req := storage.service.Files.Get(fileID).SupportsAllDrives(true) if e, ok := err.(*googleapi.Error); ok { if strings.Contains(err.Error(), "cannotDownloadAbusiveFile") || len(e.Errors) > 0 && e.Errors[0].Reason == "cannotDownloadAbusiveFile" { LOG_WARN("GCD_STORAGE", "%s is marked as abusive, will download anyway.", filePath) @@ -676,7 +691,7 @@ func (storage *GCDStorage) UploadFile(threadIndex int, filePath string, content for { reader := CreateRateLimitedReader(content, storage.UploadRateLimit/storage.numberOfThreads) - _, err = storage.service.Files.Create(file).Media(reader).Fields("id").Do() + _, err = storage.service.Files.Create(file).SupportsAllDrives(true).Media(reader).Fields("id").Do() if retry, err := storage.shouldRetry(threadIndex, err); err == nil && !retry { break } else if retry { diff --git a/src/duplicacy_storage.go b/src/duplicacy_storage.go index 7cf85cf..9655da3 100644 --- a/src/duplicacy_storage.go +++ b/src/duplicacy_storage.go @@ -582,10 +582,14 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor SavePassword(preference, "gcs_token", tokenFile) return gcsStorage } else if matched[1] == "gcd" { + driveID := matched[2] + if len(driveID) != 0 { + driveID = driveID[:len(driveID)-1] + } storagePath := matched[3] + matched[4] prompt := fmt.Sprintf("Enter the path of the Google Drive token file (downloadable from https://duplicacy.com/gcd_start):") tokenFile := GetPassword(preference, "gcd_token", prompt, true, resetPassword) - gcdStorage, err := CreateGCDStorage(tokenFile, storagePath, threads) + gcdStorage, err := CreateGCDStorage(tokenFile, driveID, storagePath, threads) if err != nil { LOG_ERROR("STORAGE_CREATE", "Failed to load the Google Drive storage at %s: %v", storageURL, err) return nil diff --git a/src/duplicacy_storage_test.go b/src/duplicacy_storage_test.go index 0d1f1b3..627722b 100644 --- a/src/duplicacy_storage_test.go +++ b/src/duplicacy_storage_test.go @@ -131,7 +131,7 @@ func loadStorage(localStoragePath string, threads int) (Storage, error) { storage.SetDefaultNestingLevels([]int{2, 3}, 2) return storage, err } else if testStorageName == "gcd" { - storage, err := CreateGCDStorage(config["token_file"], config["storage_path"], threads) + storage, err := CreateGCDStorage(config["token_file"], "", config["storage_path"], threads) storage.SetDefaultNestingLevels([]int{2, 3}, 2) return storage, err } else if testStorageName == "one" { From 7719bb9f297660c6bad5129dfd5a60da708b54d6 Mon Sep 17 00:00:00 2001 From: Richard Sanger Date: Wed, 27 Nov 2019 00:00:40 +1300 Subject: [PATCH 2/3] Fix: backup to shared drive root Allows writing to the drive root using: gcd://driveid@ or gcd://driveid@/ To write to the root of the default user's drive use the special shared drive named 'root': gcd://root@/ --- src/duplicacy_gcdstorage.go | 5 +++++ src/duplicacy_storage.go | 7 ++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/duplicacy_gcdstorage.go b/src/duplicacy_gcdstorage.go index c29270d..c04aff2 100644 --- a/src/duplicacy_gcdstorage.go +++ b/src/duplicacy_gcdstorage.go @@ -264,6 +264,11 @@ func (storage *GCDStorage) getIDFromPath(threadIndex int, filePath string, creat fileID = rootID } + // Write directly to the root of the drive + if filePath == "" { + return fileID, nil + } + names := strings.Split(filePath, "/") current := "" for i, name := range names { diff --git a/src/duplicacy_storage.go b/src/duplicacy_storage.go index 9655da3..8c32032 100644 --- a/src/duplicacy_storage.go +++ b/src/duplicacy_storage.go @@ -582,8 +582,13 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor SavePassword(preference, "gcs_token", tokenFile) return gcsStorage } else if matched[1] == "gcd" { + // Handle writing directly to the root of the drive + // For gcd://driveid@/, driveid@ is match[3] not match[2] + if matched[2] == "" && strings.HasSuffix(matched[3], "@") { + matched[2], matched[3] = matched[3], matched[2] + } driveID := matched[2] - if len(driveID) != 0 { + if driveID != "" { driveID = driveID[:len(driveID)-1] } storagePath := matched[3] + matched[4] From aa07feeac017d48286815e11bdb0c62d4f29c1a0 Mon Sep 17 00:00:00 2001 From: Richard Sanger Date: Sat, 11 Jan 2020 17:25:21 +1300 Subject: [PATCH 3/3] Fix bug in gcd: init fails to create directories init would not create directories in the root of a drive as it did not know the root drive's ID. --- src/duplicacy_gcdstorage.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/duplicacy_gcdstorage.go b/src/duplicacy_gcdstorage.go index c04aff2..4eb0215 100644 --- a/src/duplicacy_gcdstorage.go +++ b/src/duplicacy_gcdstorage.go @@ -264,11 +264,6 @@ func (storage *GCDStorage) getIDFromPath(threadIndex int, filePath string, creat fileID = rootID } - // Write directly to the root of the drive - if filePath == "" { - return fileID, nil - } - names := strings.Split(filePath, "/") current := "" for i, name := range names { @@ -361,6 +356,7 @@ func CreateGCDStorage(tokenFile string, driveID string, storagePath string, thre storage.attempts[i] = 0 } + storage.savePathID("", driveID) storagePathID, err := storage.getIDFromPath(0, storagePath, true) if err != nil { return nil, err