From 0db8b9831b41e06955eeaae020d63ecdb9eff990 Mon Sep 17 00:00:00 2001 From: Gilbert Chen Date: Mon, 2 Apr 2018 20:03:50 -0400 Subject: [PATCH 1/2] Implement the WebDAV backend --- src/duplicacy_storage.go | 24 +- src/duplicacy_storage_test.go | 7 + src/duplicacy_webdavstorage.go | 399 +++++++++++++++++++++++++++++++++ 3 files changed, 429 insertions(+), 1 deletion(-) create mode 100644 src/duplicacy_webdavstorage.go diff --git a/src/duplicacy_storage.go b/src/duplicacy_storage.go index f72dbe9..456260d 100644 --- a/src/duplicacy_storage.go +++ b/src/duplicacy_storage.go @@ -261,7 +261,7 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor return fileStorage } - urlRegex := regexp.MustCompile(`^(\w+)://([\w\-]+@)?([^/]+)(/(.+))?`) + urlRegex := regexp.MustCompile(`^(\w+)://([\w\-@\.]+@)?([^/]+)(/(.+))?`) matched := urlRegex.FindStringSubmatch(storageURL) @@ -606,6 +606,28 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor } SavePassword(preference, "swift_key", key) return swiftStorage + } else if matched[1] == "webdav" { + server := matched[3] + username := matched[2] + username = username[:len(username) - 1] + storageDir := matched[5] + port := 0 + + if strings.Contains(server, ":") { + index := strings.Index(server, ":") + port, _ = strconv.Atoi(server[index+1:]) + server = server[:index] + } + + prompt := fmt.Sprintf("Enter the WebDAV password:") + password := GetPassword(preference, "webdav_password", prompt, true, resetPassword) + webDAVStorage, err := CreateWebDAVStorage(server, port, username, password, storageDir, threads) + if err != nil { + LOG_ERROR("STORAGE_CREATE", "Failed to load the OpenStack Swift storage at %s: %v", storageURL, err) + return nil + } + SavePassword(preference, "webdav_password", password) + return webDAVStorage } else { LOG_ERROR("STORAGE_CREATE", "The storage type '%s' is not supported", matched[1]) return nil diff --git a/src/duplicacy_storage_test.go b/src/duplicacy_storage_test.go index 36075cc..398b16d 100644 --- a/src/duplicacy_storage_test.go +++ b/src/duplicacy_storage_test.go @@ -146,6 +146,13 @@ func loadStorage(localStoragePath string, threads int) (Storage, error) { storage, err := CreateSwiftStorage(config["storage_url"], config["key"], threads) storage.SetDefaultNestingLevels([]int{2, 3}, 2) return storage, err + } else if testStorageName == "pcloud" { + storage, err := CreateWebDAVStorage(config["host"], 0, config["username"], config["password"], config["storage_path"], threads) + if err != nil { + return nil, err + } + storage.SetDefaultNestingLevels([]int{2, 3}, 2) + return storage, err } else { return nil, fmt.Errorf("Invalid storage named: %s", testStorageName) } diff --git a/src/duplicacy_webdavstorage.go b/src/duplicacy_webdavstorage.go new file mode 100644 index 0000000..aade4c1 --- /dev/null +++ b/src/duplicacy_webdavstorage.go @@ -0,0 +1,399 @@ +// Copyright (c) Acrosync LLC. All rights reserved. +// Free for personal use and commercial trial +// Commercial use requires per-user licenses available from https://duplicacy.com +// +// +// This storage backend is based on the work by Yuri Karamani from https://github.com/karamani/webdavclnt, +// released under the MIT license. +// +package duplicacy + +import ( + "bytes" + "encoding/xml" + "errors" + "fmt" + "io" + "io/ioutil" + "math/rand" + "net/http" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "time" +) + +type WebDAVStorage struct { + StorageBase + + host string + port int + username string + password string + storageDir string + + client *http.Client + threads int + directoryCache map[string]int // stores directories known to exist by this backend + directoryCacheLock sync.Mutex // lock for accessing directoryCache +} + +func CreateWebDAVStorage(host string, port int, username string, password string, storageDir string, threads int) (storage *WebDAVStorage, err error) { + if storageDir[len(storageDir)-1] != '/' { + storageDir += "/" + } + + storage = &WebDAVStorage{ + host: host, + port: port, + username: username, + password: password, + storageDir: "", + + client: http.DefaultClient, + threads: threads, + directoryCache: make(map[string]int), + } + + exist, isDir, _, err := storage.GetFileInfo(0, storageDir) + if err != nil { + return nil, err + } + if !exist { + return nil, fmt.Errorf("Storage path %s does not exist", storageDir) + } + if !isDir { + return nil, fmt.Errorf("Storage path %s is not a directory", storageDir) + } + storage.storageDir = storageDir + + for _, dir := range []string{"snapshots", "chunks"} { + storage.CreateDirectory(0, dir) + } + + storage.DerivedStorage = storage + storage.SetDefaultNestingLevels([]int{0}, 0) + return storage, nil +} + +func (storage *WebDAVStorage) createConnectionString(uri string) string { + + url := storage.host + + if !strings.HasPrefix(url, "http://") && !strings.HasPrefix(url, "https://") { + url = "https://" + url + } + if storage.port > 0 { + url += fmt.Sprintf(":%d", storage.port) + } + return url + "/" + storage.storageDir + uri +} + +func (storage *WebDAVStorage) retry(backoff int) int { + delay := rand.Intn(backoff*500) + backoff*500 + time.Sleep(time.Duration(delay) * time.Millisecond) + backoff *= 2 + return backoff +} + +func (storage *WebDAVStorage) sendRequest(method string, uri string, data []byte) (io.ReadCloser, http.Header, error) { + + backoff := 1 + for i := 0; i < 8; i++ { + + var dataReader io.Reader + headers := make(map[string]string) + if method == "PROPFIND" { + headers["Content-Type"] = "application/xml" + headers["Depth"] = "1" + dataReader = bytes.NewReader(data) + } else if method == "PUT" { + headers["Content-Type"] = "application/octet-stream" + dataReader = CreateRateLimitedReader(data, storage.UploadRateLimit/storage.threads) + } else if method == "MOVE" { + headers["Destination"] = storage.createConnectionString(string(data)) + headers["Content-Type"] = "application/octet-stream" + dataReader = bytes.NewReader([]byte("")) + } else { + headers["Content-Type"] = "application/octet-stream" + dataReader = bytes.NewReader(data) + } + + request, err := http.NewRequest(method, storage.createConnectionString(uri), dataReader) + if err != nil { + return nil, nil, err + } + + if len(storage.username) > 0 { + request.SetBasicAuth(storage.username, storage.password) + } + + for key, value := range headers { + request.Header.Set(key, value) + } + + response, err := storage.client.Do(request) + if err != nil { + LOG_TRACE("WEBDAV_RETRY", "URL request '%s %s' returned an error (%v)", method, uri, err) + backoff = storage.retry(backoff) + continue + } + + if response.StatusCode < 300 { + return response.Body, response.Header, nil + } + response.Body.Close() + if response.StatusCode == 401 { + return nil, nil, errors.New("Login failed") + } else if response.StatusCode == 404 { + return nil, nil, os.ErrNotExist + } + LOG_INFO("WEBDAV_RETRY", "URL request '%s %s' returned status code %d", method, uri, response.StatusCode) + backoff = storage.retry(backoff) + } + return nil, nil, errors.New("Maximum backoff reached") +} + +type WebDAVProperties map[string]string + +type WebDAVPropValue struct { + XMLName xml.Name `xml:""` + Value string `xml:",chardata"` +} + +type WebDAVProp struct { + PropList []WebDAVPropValue `xml:",any"` +} + +type WebDAVPropStat struct { + Prop *WebDAVProp `xml:"prop"` +} + +type WebDAVResponse struct { + Href string `xml:"href"` + PropStat *WebDAVPropStat `xml:"propstat"` +} + +type WebDAVMultiStatus struct { + Responses []WebDAVResponse `xml:"response"` +} + +func (storage *WebDAVStorage) getProperties(uri string, properties ...string) (map[string]WebDAVProperties, error) { + + propfind := "" + for _, p := range properties { + propfind += fmt.Sprintf("<%s/>", p) + } + propfind += "" + + body := fmt.Sprintf(`%s`, propfind) + + readCloser, _, err := storage.sendRequest("PROPFIND", uri, []byte(body)) + if err != nil { + return nil, err + } + defer readCloser.Close() + content, err := ioutil.ReadAll(readCloser) + if err != nil { + return nil, err + } + + object := WebDAVMultiStatus{} + err = xml.Unmarshal(content, &object) + if err != nil { + return nil, err + } + + if object.Responses == nil || len(object.Responses) == 0 { + return nil, errors.New("no WebDAV responses") + } + + responses := make(map[string]WebDAVProperties) + + for _, responseTag := range object.Responses { + if responseTag.PropStat == nil || responseTag.PropStat.Prop == nil || responseTag.PropStat.Prop.PropList == nil { + return nil, errors.New("no WebDAV properties") + } + + properties := make(WebDAVProperties) + for _, prop := range responseTag.PropStat.Prop.PropList { + properties[prop.XMLName.Local] = prop.Value + } + + responseKey := responseTag.Href + responses[responseKey] = properties + } + + return responses, nil +} + +// ListFiles return the list of files and subdirectories under 'dir'. A subdirectories returned must have a trailing '/', with +// a size of 0. If 'dir' is 'snapshots', only subdirectories will be returned. If 'dir' is 'snapshots/repository_id', then only +// files will be returned. If 'dir' is 'chunks', the implementation can return the list either recusively or non-recusively. +func (storage *WebDAVStorage) ListFiles(threadIndex int, dir string) (files []string, sizes []int64, err error) { + if dir[len(dir)-1] != '/' { + dir += "/" + } + properties, err := storage.getProperties(dir, "getcontentlength") + if err != nil { + return nil, nil, err + } + + prefixLength := len(storage.storageDir) + len(dir) + 1 + + for file, m := range properties { + if len(file) <= prefixLength { + continue + } + + if length, exist := m["getcontentlength"]; exist && length != "" { + // This is a file; don't need it when listing "snapshots/" + if dir != "snapshots/" { + size, _ := strconv.Atoi(length) + files = append(files, file[prefixLength:]) + sizes = append(sizes, int64(size)) + } + } else { + // This is a dir + file := file[prefixLength:] + if file[len(file)-1] != '/' { + file += "/" + } + files = append(files, file) + sizes = append(sizes, int64(0)) + } + } + + return files, sizes, nil +} + +// GetFileInfo returns the information about the file or directory at 'filePath'. +func (storage *WebDAVStorage) GetFileInfo(threadIndex int, filePath string) (exist bool, isDir bool, size int64, err error) { + readCloser, header, err := storage.sendRequest("HEAD", filePath, []byte("")) + if err != nil { + if err == os.ErrNotExist { + return false, false, 0, nil + } else { + return false, false, 0, err + } + } + readCloser.Close() + + contentLength := header.Get("Content-Length") + if contentLength == "" { + return true, true, 0, nil + } else { + value, _ := strconv.Atoi(contentLength) + return true, false, int64(value), nil + } + +} + +// DeleteFile deletes the file or directory at 'filePath'. +func (storage *WebDAVStorage) DeleteFile(threadIndex int, filePath string) (err error) { + readCloser, _, err := storage.sendRequest("DELETE", filePath, []byte("")) + if err != nil { + return err + } + readCloser.Close() + return nil +} + +// MoveFile renames the file. +func (storage *WebDAVStorage) MoveFile(threadIndex int, from string, to string) (err error) { + readCloser, _, err := storage.sendRequest("MOVE", from, []byte(to)) + if err != nil { + return err + } + readCloser.Close() + return nil +} + +// createParentDirectory creates the parent directory if it doesn't exist in the cache +func (storage *WebDAVStorage) createParentDirectory(threadIndex int, dir string) (err error) { + parent := filepath.Dir(dir) + + if parent == "." { + return nil + } + + storage.directoryCacheLock.Lock() + _, exist := storage.directoryCache[parent] + storage.directoryCacheLock.Unlock() + + if exist { + return nil + } + + err = storage.CreateDirectory(threadIndex, parent) + if err == nil { + storage.directoryCacheLock.Lock() + storage.directoryCache[parent] = 1 + storage.directoryCacheLock.Unlock() + } + return err +} + +// CreateDirectory creates a new directory. +func (storage *WebDAVStorage) CreateDirectory(threadIndex int, dir string) (err error) { + for dir != "" && dir[len(dir)-1] == '/' { + dir = dir[:len(dir)-1] + } + + if dir == "" { + return nil + } + + // If there is an error in creating the parent directory, proceed anyway + storage.createParentDirectory(threadIndex, dir) + + readCloser, _, err := storage.sendRequest("MKCOL", dir, []byte("")) + if err != nil { + return err + } + readCloser.Close() + return nil +} + +// DownloadFile reads the file at 'filePath' into the chunk. +func (storage *WebDAVStorage) DownloadFile(threadIndex int, filePath string, chunk *Chunk) (err error) { + readCloser, _, err := storage.sendRequest("GET", filePath, nil) + if err != nil { + return err + } + + _, err = RateLimitedCopy(chunk, readCloser, storage.DownloadRateLimit/storage.threads) + return err +} + +// UploadFile writes 'content' to the file at 'filePath'. +func (storage *WebDAVStorage) UploadFile(threadIndex int, filePath string, content []byte) (err error) { + + // If there is an error in creating the parent directory, proceed anyway + storage.createParentDirectory(threadIndex, filePath) + + readCloser, _, err := storage.sendRequest("PUT", filePath, content) + if err != nil { + return err + } + readCloser.Close() + return nil +} + +// If a local snapshot cache is needed for the storage to avoid downloading/uploading chunks too often when +// managing snapshots. +func (storage *WebDAVStorage) IsCacheNeeded() bool { return true } + +// If the 'MoveFile' method is implemented. +func (storage *WebDAVStorage) IsMoveFileImplemented() bool { return true } + +// If the storage can guarantee strong consistency. +func (storage *WebDAVStorage) IsStrongConsistent() bool { return false } + +// If the storage supports fast listing of files names. +func (storage *WebDAVStorage) IsFastListing() bool { return false } + +// Enable the test mode. +func (storage *WebDAVStorage) EnableTestMode() {} From 02cd41f4d0491e6c4490c2c7999ccc177c851972 Mon Sep 17 00:00:00 2001 From: Gilbert Chen Date: Thu, 5 Apr 2018 15:29:41 -0400 Subject: [PATCH 2/2] A few improvements to make WebDAV work better with pcloud and box.com --- src/duplicacy_chunkdownloader.go | 15 +++- src/duplicacy_storage.go | 9 +-- src/duplicacy_storage_test.go | 5 +- src/duplicacy_webdavstorage.go | 117 ++++++++++++++++++++++--------- 4 files changed, 105 insertions(+), 41 deletions(-) diff --git a/src/duplicacy_chunkdownloader.go b/src/duplicacy_chunkdownloader.go index 1647157..e0ecb1f 100644 --- a/src/duplicacy_chunkdownloader.go +++ b/src/duplicacy_chunkdownloader.go @@ -317,8 +317,19 @@ func (downloader *ChunkDownloader) Download(threadIndex int, task ChunkDownloadT } if !exist { - // Retry for the Hubic backend as it may return 404 even when the chunk exists - if _, ok := downloader.storage.(*HubicStorage); ok && downloadAttempt < MaxDownloadAttempts { + + retry := false + + // Retry for Hubic or WebDAV as it may return 404 even when the chunk exists + if _, ok := downloader.storage.(*HubicStorage); ok { + retry = true + } + + if _, ok := downloader.storage.(*WebDAVStorage); ok { + retry = true + } + + if retry && downloadAttempt < MaxDownloadAttempts { LOG_WARN("DOWNLOAD_RETRY", "Failed to find the chunk %s; retrying", chunkID) continue } diff --git a/src/duplicacy_storage.go b/src/duplicacy_storage.go index 456260d..1cc89a6 100644 --- a/src/duplicacy_storage.go +++ b/src/duplicacy_storage.go @@ -261,7 +261,7 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor return fileStorage } - urlRegex := regexp.MustCompile(`^(\w+)://([\w\-@\.]+@)?([^/]+)(/(.+))?`) + urlRegex := regexp.MustCompile(`^([\w-]+)://([\w\-@\.]+@)?([^/]+)(/(.+))?`) matched := urlRegex.FindStringSubmatch(storageURL) @@ -606,12 +606,13 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor } SavePassword(preference, "swift_key", key) return swiftStorage - } else if matched[1] == "webdav" { + } else if matched[1] == "webdav" || matched[1] == "webdav-http" { server := matched[3] username := matched[2] username = username[:len(username) - 1] storageDir := matched[5] port := 0 + useHTTP := matched[1] == "webdav-http" if strings.Contains(server, ":") { index := strings.Index(server, ":") @@ -621,9 +622,9 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor prompt := fmt.Sprintf("Enter the WebDAV password:") password := GetPassword(preference, "webdav_password", prompt, true, resetPassword) - webDAVStorage, err := CreateWebDAVStorage(server, port, username, password, storageDir, threads) + webDAVStorage, err := CreateWebDAVStorage(server, port, username, password, storageDir, useHTTP, threads) if err != nil { - LOG_ERROR("STORAGE_CREATE", "Failed to load the OpenStack Swift storage at %s: %v", storageURL, err) + LOG_ERROR("STORAGE_CREATE", "Failed to load the WebDAV storage at %s: %v", storageURL, err) return nil } SavePassword(preference, "webdav_password", password) diff --git a/src/duplicacy_storage_test.go b/src/duplicacy_storage_test.go index 398b16d..c0fdf40 100644 --- a/src/duplicacy_storage_test.go +++ b/src/duplicacy_storage_test.go @@ -146,8 +146,8 @@ func loadStorage(localStoragePath string, threads int) (Storage, error) { storage, err := CreateSwiftStorage(config["storage_url"], config["key"], threads) storage.SetDefaultNestingLevels([]int{2, 3}, 2) return storage, err - } else if testStorageName == "pcloud" { - storage, err := CreateWebDAVStorage(config["host"], 0, config["username"], config["password"], config["storage_path"], threads) + } else if testStorageName == "pcloud" || testStorageName == "box" { + storage, err := CreateWebDAVStorage(config["host"], 0, config["username"], config["password"], config["storage_path"], false, threads) if err != nil { return nil, err } @@ -387,6 +387,7 @@ func TestStorage(t *testing.T) { snapshotIDs := []string{} for _, snapshotDir := range snapshotDirs { + LOG_INFO("debug", "snapshot dir: %s", snapshotDir) if len(snapshotDir) > 0 && snapshotDir[len(snapshotDir)-1] == '/' { snapshotIDs = append(snapshotIDs, snapshotDir[:len(snapshotDir)-1]) } diff --git a/src/duplicacy_webdavstorage.go b/src/duplicacy_webdavstorage.go index aade4c1..598efbe 100644 --- a/src/duplicacy_webdavstorage.go +++ b/src/duplicacy_webdavstorage.go @@ -17,12 +17,12 @@ import ( "io/ioutil" "math/rand" "net/http" - "os" + //"net/http/httputil" "path/filepath" "strconv" - "strings" "sync" "time" + "strings" ) type WebDAVStorage struct { @@ -33,6 +33,7 @@ type WebDAVStorage struct { username string password string storageDir string + useHTTP bool client *http.Client threads int @@ -40,7 +41,15 @@ type WebDAVStorage struct { directoryCacheLock sync.Mutex // lock for accessing directoryCache } -func CreateWebDAVStorage(host string, port int, username string, password string, storageDir string, threads int) (storage *WebDAVStorage, err error) { +var ( + errWebDAVAuthorizationFailure = errors.New("Authentication failed") + errWebDAVMovedPermanently = errors.New("Moved permanently") + errWebDAVNotExist = errors.New("Path does not exist") + errWebDAVMaximumBackoff = errors.New("Maximum backoff reached") + errWebDAVMethodNotAllowed = errors.New("Method not allowed") +) + +func CreateWebDAVStorage(host string, port int, username string, password string, storageDir string, useHTTP bool, threads int) (storage *WebDAVStorage, err error) { if storageDir[len(storageDir)-1] != '/' { storageDir += "/" } @@ -51,12 +60,18 @@ func CreateWebDAVStorage(host string, port int, username string, password string username: username, password: password, storageDir: "", + useHTTP: false, client: http.DefaultClient, threads: threads, directoryCache: make(map[string]int), } + // Make sure it doesn't follow redirect + storage.client.CheckRedirect = func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + } + exist, isDir, _, err := storage.GetFileInfo(0, storageDir) if err != nil { return nil, err @@ -82,9 +97,12 @@ func (storage *WebDAVStorage) createConnectionString(uri string) string { url := storage.host - if !strings.HasPrefix(url, "http://") && !strings.HasPrefix(url, "https://") { + if storage.useHTTP { + url = "http://" + url + } else { url = "https://" + url } + if storage.port > 0 { url += fmt.Sprintf(":%d", storage.port) } @@ -98,7 +116,7 @@ func (storage *WebDAVStorage) retry(backoff int) int { return backoff } -func (storage *WebDAVStorage) sendRequest(method string, uri string, data []byte) (io.ReadCloser, http.Header, error) { +func (storage *WebDAVStorage) sendRequest(method string, uri string, depth int, data []byte) (io.ReadCloser, http.Header, error) { backoff := 1 for i := 0; i < 8; i++ { @@ -107,7 +125,7 @@ func (storage *WebDAVStorage) sendRequest(method string, uri string, data []byte headers := make(map[string]string) if method == "PROPFIND" { headers["Content-Type"] = "application/xml" - headers["Depth"] = "1" + headers["Depth"] = fmt.Sprintf("%d", depth) dataReader = bytes.NewReader(data) } else if method == "PUT" { headers["Content-Type"] = "application/octet-stream" @@ -134,6 +152,9 @@ func (storage *WebDAVStorage) sendRequest(method string, uri string, data []byte request.Header.Set(key, value) } + //requestDump, err := httputil.DumpRequest(request, true) + //LOG_INFO("debug", "Request: %s", requestDump) + response, err := storage.client.Do(request) if err != nil { LOG_TRACE("WEBDAV_RETRY", "URL request '%s %s' returned an error (%v)", method, uri, err) @@ -144,23 +165,31 @@ func (storage *WebDAVStorage) sendRequest(method string, uri string, data []byte if response.StatusCode < 300 { return response.Body, response.Header, nil } + + if response.StatusCode == 301 { + return nil, nil, errWebDAVMovedPermanently + } + response.Body.Close() - if response.StatusCode == 401 { - return nil, nil, errors.New("Login failed") - } else if response.StatusCode == 404 { - return nil, nil, os.ErrNotExist + if response.StatusCode == 404 { + // Retry if it is UPLOAD, otherwise return immediately + if method != "PUT" { + return nil, nil, errWebDAVNotExist + } + } else if response.StatusCode == 405 { + return nil, nil, errWebDAVMethodNotAllowed } LOG_INFO("WEBDAV_RETRY", "URL request '%s %s' returned status code %d", method, uri, response.StatusCode) backoff = storage.retry(backoff) } - return nil, nil, errors.New("Maximum backoff reached") + return nil, nil, errWebDAVMaximumBackoff } type WebDAVProperties map[string]string type WebDAVPropValue struct { XMLName xml.Name `xml:""` - Value string `xml:",chardata"` + Value string `xml:",innerxml"` } type WebDAVProp struct { @@ -180,7 +209,7 @@ type WebDAVMultiStatus struct { Responses []WebDAVResponse `xml:"response"` } -func (storage *WebDAVStorage) getProperties(uri string, properties ...string) (map[string]WebDAVProperties, error) { +func (storage *WebDAVStorage) getProperties(uri string, depth int, properties ...string) (map[string]WebDAVProperties, error) { propfind := "" for _, p := range properties { @@ -190,7 +219,7 @@ func (storage *WebDAVStorage) getProperties(uri string, properties ...string) (m body := fmt.Sprintf(`%s`, propfind) - readCloser, _, err := storage.sendRequest("PROPFIND", uri, []byte(body)) + readCloser, _, err := storage.sendRequest("PROPFIND", uri, depth, []byte(body)) if err != nil { return nil, err } @@ -213,6 +242,7 @@ func (storage *WebDAVStorage) getProperties(uri string, properties ...string) (m responses := make(map[string]WebDAVProperties) for _, responseTag := range object.Responses { + if responseTag.PropStat == nil || responseTag.PropStat.Prop == nil || responseTag.PropStat.Prop.PropList == nil { return nil, errors.New("no WebDAV properties") } @@ -224,6 +254,7 @@ func (storage *WebDAVStorage) getProperties(uri string, properties ...string) (m responseKey := responseTag.Href responses[responseKey] = properties + } return responses, nil @@ -236,7 +267,7 @@ func (storage *WebDAVStorage) ListFiles(threadIndex int, dir string) (files []st if dir[len(dir)-1] != '/' { dir += "/" } - properties, err := storage.getProperties(dir, "getcontentlength") + properties, err := storage.getProperties(dir, 1, "getcontentlength", "resourcetype") if err != nil { return nil, nil, err } @@ -248,10 +279,22 @@ func (storage *WebDAVStorage) ListFiles(threadIndex int, dir string) (files []st continue } - if length, exist := m["getcontentlength"]; exist && length != "" { - // This is a file; don't need it when listing "snapshots/" + isDir := false + size := 0 + if resourceType, exist := m["resourcetype"]; exist && strings.Contains(resourceType, "collection") { + isDir = true + } else if length, exist := m["getcontentlength"]; exist { + if length == "" { + isDir = true + } else { + size, _ = strconv.Atoi(length) + } + } else { + continue + } + + if !isDir { if dir != "snapshots/" { - size, _ := strconv.Atoi(length) files = append(files, file[prefixLength:]) sizes = append(sizes, int64(size)) } @@ -271,29 +314,33 @@ func (storage *WebDAVStorage) ListFiles(threadIndex int, dir string) (files []st // GetFileInfo returns the information about the file or directory at 'filePath'. func (storage *WebDAVStorage) GetFileInfo(threadIndex int, filePath string) (exist bool, isDir bool, size int64, err error) { - readCloser, header, err := storage.sendRequest("HEAD", filePath, []byte("")) + properties, err := storage.getProperties(filePath, 0, "getcontentlength", "resourcetype") if err != nil { - if err == os.ErrNotExist { + if err == errWebDAVNotExist { return false, false, 0, nil - } else { - return false, false, 0, err } + if err == errWebDAVMovedPermanently { + // This must be a directory + return true, true, 0, nil + } + return false, false, 0, err } - readCloser.Close() - contentLength := header.Get("Content-Length") - if contentLength == "" { + if m, exist := properties["/" + storage.storageDir + filePath]; !exist { + return false, false, 0, nil + } else if resourceType, exist := m["resourcetype"]; exist && strings.Contains(resourceType, "collection") { return true, true, 0, nil - } else { - value, _ := strconv.Atoi(contentLength) + } else if length, exist := m["getcontentlength"]; exist && length != ""{ + value, _ := strconv.Atoi(length) return true, false, int64(value), nil + } else { + return true, true, 0, nil } - } // DeleteFile deletes the file or directory at 'filePath'. func (storage *WebDAVStorage) DeleteFile(threadIndex int, filePath string) (err error) { - readCloser, _, err := storage.sendRequest("DELETE", filePath, []byte("")) + readCloser, _, err := storage.sendRequest("DELETE", filePath, 0, []byte("")) if err != nil { return err } @@ -303,7 +350,7 @@ func (storage *WebDAVStorage) DeleteFile(threadIndex int, filePath string) (err // MoveFile renames the file. func (storage *WebDAVStorage) MoveFile(threadIndex int, from string, to string) (err error) { - readCloser, _, err := storage.sendRequest("MOVE", from, []byte(to)) + readCloser, _, err := storage.sendRequest("MOVE", from, 0, []byte(to)) if err != nil { return err } @@ -349,8 +396,12 @@ func (storage *WebDAVStorage) CreateDirectory(threadIndex int, dir string) (err // If there is an error in creating the parent directory, proceed anyway storage.createParentDirectory(threadIndex, dir) - readCloser, _, err := storage.sendRequest("MKCOL", dir, []byte("")) + readCloser, _, err := storage.sendRequest("MKCOL", dir, 0, []byte("")) if err != nil { + if err == errWebDAVMethodNotAllowed || err == errWebDAVMovedPermanently { + // We simply ignore these errors and assume that the directory already exists + return nil + } return err } readCloser.Close() @@ -359,7 +410,7 @@ func (storage *WebDAVStorage) CreateDirectory(threadIndex int, dir string) (err // DownloadFile reads the file at 'filePath' into the chunk. func (storage *WebDAVStorage) DownloadFile(threadIndex int, filePath string, chunk *Chunk) (err error) { - readCloser, _, err := storage.sendRequest("GET", filePath, nil) + readCloser, _, err := storage.sendRequest("GET", filePath, 0, nil) if err != nil { return err } @@ -374,7 +425,7 @@ func (storage *WebDAVStorage) UploadFile(threadIndex int, filePath string, conte // If there is an error in creating the parent directory, proceed anyway storage.createParentDirectory(threadIndex, filePath) - readCloser, _, err := storage.sendRequest("PUT", filePath, content) + readCloser, _, err := storage.sendRequest("PUT", filePath, 0, content) if err != nil { return err }