From d26ffe2cffba2064c3c76b0cb9a93001b2892190 Mon Sep 17 00:00:00 2001 From: Gilbert Chen Date: Thu, 19 Mar 2020 14:59:26 -0400 Subject: [PATCH] Add support for OneDrive for Business The new storage prefix for OneDrive for Business is odb:// The token file can be downloaded from https://duplicacy.com/odb_start OneDrive for Business requires basically the same set of API calls with different endpoints. However, one major difference is that for files larger than 4MB, an upload session must be created first which is then used to upload the file content. Other than that, there are a few minor differences such as creating an existing directory, or moving files to a non-existent directory. --- src/duplicacy_oneclient.go | 145 +++++++++++++++++++++++++++----- src/duplicacy_oneclient_test.go | 2 +- src/duplicacy_onestorage.go | 5 +- src/duplicacy_storage.go | 6 +- src/duplicacy_storage_test.go | 10 ++- 5 files changed, 141 insertions(+), 27 deletions(-) diff --git a/src/duplicacy_oneclient.go b/src/duplicacy_oneclient.go index 91e87c2..b22c317 100644 --- a/src/duplicacy_oneclient.go +++ b/src/duplicacy_oneclient.go @@ -15,6 +15,7 @@ import ( "strings" "sync" "time" + "path/filepath" "golang.org/x/oauth2" ) @@ -32,9 +33,6 @@ type OneDriveErrorResponse struct { Error OneDriveError `json:"error"` } -var OneDriveRefreshTokenURL = "https://duplicacy.com/one_refresh" -var OneDriveAPIURL = "https://api.onedrive.com/v1.0" - type OneDriveClient struct { HTTPClient *http.Client @@ -44,9 +42,13 @@ type OneDriveClient struct { IsConnected bool TestMode bool + + IsBusiness bool + RefreshTokenURL string + APIURL string } -func NewOneDriveClient(tokenFile string) (*OneDriveClient, error) { +func NewOneDriveClient(tokenFile string, isBusiness bool) (*OneDriveClient, error) { description, err := ioutil.ReadFile(tokenFile) if err != nil { @@ -63,6 +65,15 @@ func NewOneDriveClient(tokenFile string) (*OneDriveClient, error) { TokenFile: tokenFile, Token: token, TokenLock: &sync.Mutex{}, + IsBusiness: isBusiness, + } + + if isBusiness { + client.RefreshTokenURL = "https://duplicacy.com/odb_refresh" + client.APIURL = "https://graph.microsoft.com/v1.0/me" + } else { + client.RefreshTokenURL = "https://duplicacy.com/one_refresh" + client.APIURL = "https://api.onedrive.com/v1.0" } client.RefreshToken(false) @@ -106,9 +117,10 @@ func (client *OneDriveClient) call(url string, method string, input interface{}, if reader, ok := inputReader.(*RateLimitedReader); ok { request.ContentLength = reader.Length() + request.Header.Set("Content-Range", fmt.Sprintf("bytes 0-%d/%d", reader.Length() - 1, reader.Length())) } - if url != OneDriveRefreshTokenURL { + if url != client.RefreshTokenURL { client.TokenLock.Lock() request.Header.Set("Authorization", "Bearer "+client.Token.AccessToken) client.TokenLock.Unlock() @@ -152,7 +164,7 @@ func (client *OneDriveClient) call(url string, method string, input interface{}, if response.StatusCode == 401 { - if url == OneDriveRefreshTokenURL { + if url == client.RefreshTokenURL { return nil, 0, OneDriveError{Status: response.StatusCode, Message: "Authorization error when refreshing token"} } @@ -161,6 +173,8 @@ func (client *OneDriveClient) call(url string, method string, input interface{}, return nil, 0, err } continue + } else if response.StatusCode == 409 { + return nil, 0, OneDriveError{Status: response.StatusCode, Message: "Conflict"} } else if response.StatusCode > 401 && response.StatusCode != 404 { retryAfter := time.Duration(rand.Float32() * 1000.0 * float32(backoff)) LOG_INFO("ONEDRIVE_RETRY", "Response code: %d; retry after %d milliseconds", response.StatusCode, retryAfter) @@ -188,7 +202,7 @@ func (client *OneDriveClient) RefreshToken(force bool) (err error) { return nil } - readCloser, _, err := client.call(OneDriveRefreshTokenURL, "POST", client.Token, "") + readCloser, _, err := client.call(client.RefreshTokenURL, "POST", client.Token, "") if err != nil { return fmt.Errorf("failed to refresh the access token: %v", err) } @@ -228,9 +242,9 @@ func (client *OneDriveClient) ListEntries(path string) ([]OneDriveEntry, error) entries := []OneDriveEntry{} - url := OneDriveAPIURL + "/drive/root:/" + path + ":/children" + url := client.APIURL + "/drive/root:/" + path + ":/children" if path == "" { - url = OneDriveAPIURL + "/drive/root/children" + url = client.APIURL + "/drive/root/children" } if client.TestMode { url += "?top=8" @@ -266,7 +280,7 @@ func (client *OneDriveClient) ListEntries(path string) ([]OneDriveEntry, error) func (client *OneDriveClient) GetFileInfo(path string) (string, bool, int64, error) { - url := OneDriveAPIURL + "/drive/root:/" + path + url := client.APIURL + "/drive/root:/" + path url += "?select=id,name,size,folder" readCloser, _, err := client.call(url, "GET", 0, "") @@ -291,28 +305,95 @@ func (client *OneDriveClient) GetFileInfo(path string) (string, bool, int64, err func (client *OneDriveClient) DownloadFile(path string) (io.ReadCloser, int64, error) { - url := OneDriveAPIURL + "/drive/items/root:/" + path + ":/content" + url := client.APIURL + "/drive/items/root:/" + path + ":/content" return client.call(url, "GET", 0, "") } func (client *OneDriveClient) UploadFile(path string, content []byte, rateLimit int) (err error) { - url := OneDriveAPIURL + "/drive/root:/" + path + ":/content" + // Upload file using the simple method; this is only possible for OneDrive Personal or if the file + // is smaller than 4MB for OneDrive Business + if !client.IsBusiness || len(content) < 4 * 1024 * 1024 || (client.TestMode && rand.Int() % 2 == 0) { + url := client.APIURL + "/drive/root:/" + path + ":/content" - readCloser, _, err := client.call(url, "PUT", CreateRateLimitedReader(content, rateLimit), "application/octet-stream") + readCloser, _, err := client.call(url, "PUT", CreateRateLimitedReader(content, rateLimit), "application/octet-stream") + if err != nil { + return err + } + readCloser.Close() + return nil + } + + // For large files, create an upload session first + uploadURL, err := client.CreateUploadSession(path) if err != nil { return err } + return client.UploadFileSession(uploadURL, content, rateLimit) +} + +func (client *OneDriveClient) CreateUploadSession(path string) (uploadURL string, err error) { + + type CreateUploadSessionItem struct { + ConflictBehavior string `json:"@microsoft.graph.conflictBehavior"` + Name string `json:"name"` + } + + input := map[string]interface{} { + "item": CreateUploadSessionItem { + ConflictBehavior: "replace", + Name: filepath.Base(path), + }, + } + + readCloser, _, err := client.call(client.APIURL + "/drive/root:/" + path + ":/createUploadSession", "POST", input, "application/json") + if err != nil { + return "", err + } + + type CreateUploadSessionOutput struct { + UploadURL string `json:"uploadUrl"` + } + + output := &CreateUploadSessionOutput{} + + if err = json.NewDecoder(readCloser).Decode(&output); err != nil { + return "", err + } + + readCloser.Close() + return output.UploadURL, nil +} + +func (client *OneDriveClient) UploadFileSession(uploadURL string, content []byte, rateLimit int) (err error) { + + readCloser, _, err := client.call(uploadURL, "PUT", CreateRateLimitedReader(content, rateLimit), "") + if err != nil { + return err + } + type UploadFileSessionOutput struct { + Size int `json:"size"` + } + output := &UploadFileSessionOutput{} + + if err = json.NewDecoder(readCloser).Decode(&output); err != nil { + return fmt.Errorf("Failed to complete the file upload session: %v", err) + } + + if output.Size != len(content) { + return fmt.Errorf("Uploaded %d bytes out of %d bytes", output.Size, len(content)) + } + readCloser.Close() return nil } func (client *OneDriveClient) DeleteFile(path string) error { - url := OneDriveAPIURL + "/drive/root:/" + path + url := client.APIURL + "/drive/root:/" + path readCloser, _, err := client.call(url, "DELETE", 0, "") if err != nil { @@ -325,7 +406,7 @@ func (client *OneDriveClient) DeleteFile(path string) error { func (client *OneDriveClient) MoveFile(path string, parent string) error { - url := OneDriveAPIURL + "/drive/root:/" + path + url := client.APIURL + "/drive/root:/" + path parentReference := make(map[string]string) parentReference["path"] = "/drive/root:/" + parent @@ -335,6 +416,20 @@ func (client *OneDriveClient) MoveFile(path string, parent string) error { readCloser, _, err := client.call(url, "PATCH", parameters, "application/json") if err != nil { + if e, ok := err.(OneDriveError); ok && e.Status == 400 { + // The destination directory doesn't exist; trying to create it... + dir := filepath.Dir(parent) + if dir == "." { + dir = "" + } + client.CreateDirectory(dir, filepath.Base(parent)) + readCloser, _, err = client.call(url, "PATCH", parameters, "application/json") + if err != nil { + return nil + } + + } + return err } @@ -344,24 +439,29 @@ func (client *OneDriveClient) MoveFile(path string, parent string) error { func (client *OneDriveClient) CreateDirectory(path string, name string) error { - url := OneDriveAPIURL + "/root/children" + url := client.APIURL + "/root/children" if path != "" { - parentID, isDir, _, err := client.GetFileInfo(path) + pathID, isDir, _, err := client.GetFileInfo(path) if err != nil { return err } - if parentID == "" { - return fmt.Errorf("The path '%s' does not exist", path) + if pathID == "" { + dir := filepath.Dir(path) + if dir != "." { + // The parent directory doesn't exist; trying to create it... + client.CreateDirectory(dir, filepath.Base(path)) + isDir = true + } } if !isDir { return fmt.Errorf("The path '%s' is not a directory", path) } - url = OneDriveAPIURL + "/drive/items/" + parentID + "/children" + url = client.APIURL + "/drive/root:/" + path + ":/children" } parameters := make(map[string]interface{}) @@ -370,6 +470,11 @@ func (client *OneDriveClient) CreateDirectory(path string, name string) error { readCloser, _, err := client.call(url, "POST", parameters, "application/json") if err != nil { + if e, ok := err.(OneDriveError); ok && e.Status == 409 { + // This error usually means the directory already exists + LOG_TRACE("ONEDRIVE_MKDIR", "The directory '%s/%s' already exists", path, name) + return nil + } return err } diff --git a/src/duplicacy_oneclient_test.go b/src/duplicacy_oneclient_test.go index 998ac34..b124bfe 100644 --- a/src/duplicacy_oneclient_test.go +++ b/src/duplicacy_oneclient_test.go @@ -17,7 +17,7 @@ import ( func TestOneDriveClient(t *testing.T) { - oneDriveClient, err := NewOneDriveClient("one-token.json") + oneDriveClient, err := NewOneDriveClient("one-token.json", false) if err != nil { t.Errorf("Failed to create the OneDrive client: %v", err) return diff --git a/src/duplicacy_onestorage.go b/src/duplicacy_onestorage.go index dfc8d7b..e6cb9c2 100644 --- a/src/duplicacy_onestorage.go +++ b/src/duplicacy_onestorage.go @@ -19,13 +19,13 @@ type OneDriveStorage struct { } // CreateOneDriveStorage creates an OneDrive storage object. -func CreateOneDriveStorage(tokenFile string, storagePath string, threads int) (storage *OneDriveStorage, err error) { +func CreateOneDriveStorage(tokenFile string, isBusiness bool, storagePath string, threads int) (storage *OneDriveStorage, err error) { for len(storagePath) > 0 && storagePath[len(storagePath)-1] == '/' { storagePath = storagePath[:len(storagePath)-1] } - client, err := NewOneDriveClient(tokenFile) + client, err := NewOneDriveClient(tokenFile, isBusiness) if err != nil { return nil, err } @@ -80,6 +80,7 @@ func (storage *OneDriveStorage) convertFilePath(filePath string) string { // ListFiles return the list of files and subdirectories under 'dir' (non-recursively) func (storage *OneDriveStorage) ListFiles(threadIndex int, dir string) ([]string, []int64, error) { + for len(dir) > 0 && dir[len(dir)-1] == '/' { dir = dir[:len(dir)-1] } diff --git a/src/duplicacy_storage.go b/src/duplicacy_storage.go index c9ab70d..252730d 100644 --- a/src/duplicacy_storage.go +++ b/src/duplicacy_storage.go @@ -610,11 +610,11 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor } SavePassword(preference, "gcd_token", tokenFile) return gcdStorage - } else if matched[1] == "one" { + } else if matched[1] == "one" || matched[1] == "odb" { storagePath := matched[3] + matched[4] prompt := fmt.Sprintf("Enter the path of the OneDrive token file (downloadable from https://duplicacy.com/one_start):") - tokenFile := GetPassword(preference, "one_token", prompt, true, resetPassword) - oneDriveStorage, err := CreateOneDriveStorage(tokenFile, storagePath, threads) + tokenFile := GetPassword(preference, matched[1] + "_token", prompt, true, resetPassword) + oneDriveStorage, err := CreateOneDriveStorage(tokenFile, matched[1] == "odb", storagePath, threads) if err != nil { LOG_ERROR("STORAGE_CREATE", "Failed to load the OneDrive storage at %s: %v", storageURL, err) return nil diff --git a/src/duplicacy_storage_test.go b/src/duplicacy_storage_test.go index ce4c0e8..76bc45f 100644 --- a/src/duplicacy_storage_test.go +++ b/src/duplicacy_storage_test.go @@ -137,7 +137,15 @@ func loadStorage(localStoragePath string, threads int) (Storage, error) { storage.SetDefaultNestingLevels([]int{2, 3}, 2) return storage, err } else if testStorageName == "one" { - storage, err := CreateOneDriveStorage(config["token_file"], config["storage_path"], threads) + storage, err := CreateOneDriveStorage(config["token_file"], false, config["storage_path"], threads) + storage.SetDefaultNestingLevels([]int{2, 3}, 2) + return storage, err + } else if testStorageName == "odb" { + storage, err := CreateOneDriveStorage(config["token_file"], true, config["storage_path"], threads) + storage.SetDefaultNestingLevels([]int{2, 3}, 2) + return storage, err + } else if testStorageName == "one" { + storage, err := CreateOneDriveStorage(config["token_file"], false, config["storage_path"], threads) storage.SetDefaultNestingLevels([]int{2, 3}, 2) return storage, err } else if testStorageName == "hubic" {