diff --git a/src/duplicacy_storage.go b/src/duplicacy_storage.go index f72dbe9..456260d 100644 --- a/src/duplicacy_storage.go +++ b/src/duplicacy_storage.go @@ -261,7 +261,7 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor return fileStorage } - urlRegex := regexp.MustCompile(`^(\w+)://([\w\-]+@)?([^/]+)(/(.+))?`) + urlRegex := regexp.MustCompile(`^(\w+)://([\w\-@\.]+@)?([^/]+)(/(.+))?`) matched := urlRegex.FindStringSubmatch(storageURL) @@ -606,6 +606,28 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor } SavePassword(preference, "swift_key", key) return swiftStorage + } else if matched[1] == "webdav" { + server := matched[3] + username := matched[2] + username = username[:len(username) - 1] + storageDir := matched[5] + port := 0 + + if strings.Contains(server, ":") { + index := strings.Index(server, ":") + port, _ = strconv.Atoi(server[index+1:]) + server = server[:index] + } + + prompt := fmt.Sprintf("Enter the WebDAV password:") + password := GetPassword(preference, "webdav_password", prompt, true, resetPassword) + webDAVStorage, err := CreateWebDAVStorage(server, port, username, password, storageDir, threads) + if err != nil { + LOG_ERROR("STORAGE_CREATE", "Failed to load the OpenStack Swift storage at %s: %v", storageURL, err) + return nil + } + SavePassword(preference, "webdav_password", password) + return webDAVStorage } else { LOG_ERROR("STORAGE_CREATE", "The storage type '%s' is not supported", matched[1]) return nil diff --git a/src/duplicacy_storage_test.go b/src/duplicacy_storage_test.go index 36075cc..398b16d 100644 --- a/src/duplicacy_storage_test.go +++ b/src/duplicacy_storage_test.go @@ -146,6 +146,13 @@ func loadStorage(localStoragePath string, threads int) (Storage, error) { storage, err := CreateSwiftStorage(config["storage_url"], config["key"], threads) storage.SetDefaultNestingLevels([]int{2, 3}, 2) return storage, err + } else if testStorageName == "pcloud" { + storage, err := CreateWebDAVStorage(config["host"], 0, config["username"], config["password"], config["storage_path"], threads) + if err != nil { + return nil, err + } + storage.SetDefaultNestingLevels([]int{2, 3}, 2) + return storage, err } else { return nil, fmt.Errorf("Invalid storage named: %s", testStorageName) } diff --git a/src/duplicacy_webdavstorage.go b/src/duplicacy_webdavstorage.go new file mode 100644 index 0000000..aade4c1 --- /dev/null +++ b/src/duplicacy_webdavstorage.go @@ -0,0 +1,399 @@ +// Copyright (c) Acrosync LLC. All rights reserved. +// Free for personal use and commercial trial +// Commercial use requires per-user licenses available from https://duplicacy.com +// +// +// This storage backend is based on the work by Yuri Karamani from https://github.com/karamani/webdavclnt, +// released under the MIT license. +// +package duplicacy + +import ( + "bytes" + "encoding/xml" + "errors" + "fmt" + "io" + "io/ioutil" + "math/rand" + "net/http" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "time" +) + +type WebDAVStorage struct { + StorageBase + + host string + port int + username string + password string + storageDir string + + client *http.Client + threads int + directoryCache map[string]int // stores directories known to exist by this backend + directoryCacheLock sync.Mutex // lock for accessing directoryCache +} + +func CreateWebDAVStorage(host string, port int, username string, password string, storageDir string, threads int) (storage *WebDAVStorage, err error) { + if storageDir[len(storageDir)-1] != '/' { + storageDir += "/" + } + + storage = &WebDAVStorage{ + host: host, + port: port, + username: username, + password: password, + storageDir: "", + + client: http.DefaultClient, + threads: threads, + directoryCache: make(map[string]int), + } + + exist, isDir, _, err := storage.GetFileInfo(0, storageDir) + if err != nil { + return nil, err + } + if !exist { + return nil, fmt.Errorf("Storage path %s does not exist", storageDir) + } + if !isDir { + return nil, fmt.Errorf("Storage path %s is not a directory", storageDir) + } + storage.storageDir = storageDir + + for _, dir := range []string{"snapshots", "chunks"} { + storage.CreateDirectory(0, dir) + } + + storage.DerivedStorage = storage + storage.SetDefaultNestingLevels([]int{0}, 0) + return storage, nil +} + +func (storage *WebDAVStorage) createConnectionString(uri string) string { + + url := storage.host + + if !strings.HasPrefix(url, "http://") && !strings.HasPrefix(url, "https://") { + url = "https://" + url + } + if storage.port > 0 { + url += fmt.Sprintf(":%d", storage.port) + } + return url + "/" + storage.storageDir + uri +} + +func (storage *WebDAVStorage) retry(backoff int) int { + delay := rand.Intn(backoff*500) + backoff*500 + time.Sleep(time.Duration(delay) * time.Millisecond) + backoff *= 2 + return backoff +} + +func (storage *WebDAVStorage) sendRequest(method string, uri string, data []byte) (io.ReadCloser, http.Header, error) { + + backoff := 1 + for i := 0; i < 8; i++ { + + var dataReader io.Reader + headers := make(map[string]string) + if method == "PROPFIND" { + headers["Content-Type"] = "application/xml" + headers["Depth"] = "1" + dataReader = bytes.NewReader(data) + } else if method == "PUT" { + headers["Content-Type"] = "application/octet-stream" + dataReader = CreateRateLimitedReader(data, storage.UploadRateLimit/storage.threads) + } else if method == "MOVE" { + headers["Destination"] = storage.createConnectionString(string(data)) + headers["Content-Type"] = "application/octet-stream" + dataReader = bytes.NewReader([]byte("")) + } else { + headers["Content-Type"] = "application/octet-stream" + dataReader = bytes.NewReader(data) + } + + request, err := http.NewRequest(method, storage.createConnectionString(uri), dataReader) + if err != nil { + return nil, nil, err + } + + if len(storage.username) > 0 { + request.SetBasicAuth(storage.username, storage.password) + } + + for key, value := range headers { + request.Header.Set(key, value) + } + + response, err := storage.client.Do(request) + if err != nil { + LOG_TRACE("WEBDAV_RETRY", "URL request '%s %s' returned an error (%v)", method, uri, err) + backoff = storage.retry(backoff) + continue + } + + if response.StatusCode < 300 { + return response.Body, response.Header, nil + } + response.Body.Close() + if response.StatusCode == 401 { + return nil, nil, errors.New("Login failed") + } else if response.StatusCode == 404 { + return nil, nil, os.ErrNotExist + } + LOG_INFO("WEBDAV_RETRY", "URL request '%s %s' returned status code %d", method, uri, response.StatusCode) + backoff = storage.retry(backoff) + } + return nil, nil, errors.New("Maximum backoff reached") +} + +type WebDAVProperties map[string]string + +type WebDAVPropValue struct { + XMLName xml.Name `xml:""` + Value string `xml:",chardata"` +} + +type WebDAVProp struct { + PropList []WebDAVPropValue `xml:",any"` +} + +type WebDAVPropStat struct { + Prop *WebDAVProp `xml:"prop"` +} + +type WebDAVResponse struct { + Href string `xml:"href"` + PropStat *WebDAVPropStat `xml:"propstat"` +} + +type WebDAVMultiStatus struct { + Responses []WebDAVResponse `xml:"response"` +} + +func (storage *WebDAVStorage) getProperties(uri string, properties ...string) (map[string]WebDAVProperties, error) { + + propfind := "" + for _, p := range properties { + propfind += fmt.Sprintf("<%s/>", p) + } + propfind += "" + + body := fmt.Sprintf(`%s`, propfind) + + readCloser, _, err := storage.sendRequest("PROPFIND", uri, []byte(body)) + if err != nil { + return nil, err + } + defer readCloser.Close() + content, err := ioutil.ReadAll(readCloser) + if err != nil { + return nil, err + } + + object := WebDAVMultiStatus{} + err = xml.Unmarshal(content, &object) + if err != nil { + return nil, err + } + + if object.Responses == nil || len(object.Responses) == 0 { + return nil, errors.New("no WebDAV responses") + } + + responses := make(map[string]WebDAVProperties) + + for _, responseTag := range object.Responses { + if responseTag.PropStat == nil || responseTag.PropStat.Prop == nil || responseTag.PropStat.Prop.PropList == nil { + return nil, errors.New("no WebDAV properties") + } + + properties := make(WebDAVProperties) + for _, prop := range responseTag.PropStat.Prop.PropList { + properties[prop.XMLName.Local] = prop.Value + } + + responseKey := responseTag.Href + responses[responseKey] = properties + } + + return responses, nil +} + +// ListFiles return the list of files and subdirectories under 'dir'. A subdirectories returned must have a trailing '/', with +// a size of 0. If 'dir' is 'snapshots', only subdirectories will be returned. If 'dir' is 'snapshots/repository_id', then only +// files will be returned. If 'dir' is 'chunks', the implementation can return the list either recusively or non-recusively. +func (storage *WebDAVStorage) ListFiles(threadIndex int, dir string) (files []string, sizes []int64, err error) { + if dir[len(dir)-1] != '/' { + dir += "/" + } + properties, err := storage.getProperties(dir, "getcontentlength") + if err != nil { + return nil, nil, err + } + + prefixLength := len(storage.storageDir) + len(dir) + 1 + + for file, m := range properties { + if len(file) <= prefixLength { + continue + } + + if length, exist := m["getcontentlength"]; exist && length != "" { + // This is a file; don't need it when listing "snapshots/" + if dir != "snapshots/" { + size, _ := strconv.Atoi(length) + files = append(files, file[prefixLength:]) + sizes = append(sizes, int64(size)) + } + } else { + // This is a dir + file := file[prefixLength:] + if file[len(file)-1] != '/' { + file += "/" + } + files = append(files, file) + sizes = append(sizes, int64(0)) + } + } + + return files, sizes, nil +} + +// GetFileInfo returns the information about the file or directory at 'filePath'. +func (storage *WebDAVStorage) GetFileInfo(threadIndex int, filePath string) (exist bool, isDir bool, size int64, err error) { + readCloser, header, err := storage.sendRequest("HEAD", filePath, []byte("")) + if err != nil { + if err == os.ErrNotExist { + return false, false, 0, nil + } else { + return false, false, 0, err + } + } + readCloser.Close() + + contentLength := header.Get("Content-Length") + if contentLength == "" { + return true, true, 0, nil + } else { + value, _ := strconv.Atoi(contentLength) + return true, false, int64(value), nil + } + +} + +// DeleteFile deletes the file or directory at 'filePath'. +func (storage *WebDAVStorage) DeleteFile(threadIndex int, filePath string) (err error) { + readCloser, _, err := storage.sendRequest("DELETE", filePath, []byte("")) + if err != nil { + return err + } + readCloser.Close() + return nil +} + +// MoveFile renames the file. +func (storage *WebDAVStorage) MoveFile(threadIndex int, from string, to string) (err error) { + readCloser, _, err := storage.sendRequest("MOVE", from, []byte(to)) + if err != nil { + return err + } + readCloser.Close() + return nil +} + +// createParentDirectory creates the parent directory if it doesn't exist in the cache +func (storage *WebDAVStorage) createParentDirectory(threadIndex int, dir string) (err error) { + parent := filepath.Dir(dir) + + if parent == "." { + return nil + } + + storage.directoryCacheLock.Lock() + _, exist := storage.directoryCache[parent] + storage.directoryCacheLock.Unlock() + + if exist { + return nil + } + + err = storage.CreateDirectory(threadIndex, parent) + if err == nil { + storage.directoryCacheLock.Lock() + storage.directoryCache[parent] = 1 + storage.directoryCacheLock.Unlock() + } + return err +} + +// CreateDirectory creates a new directory. +func (storage *WebDAVStorage) CreateDirectory(threadIndex int, dir string) (err error) { + for dir != "" && dir[len(dir)-1] == '/' { + dir = dir[:len(dir)-1] + } + + if dir == "" { + return nil + } + + // If there is an error in creating the parent directory, proceed anyway + storage.createParentDirectory(threadIndex, dir) + + readCloser, _, err := storage.sendRequest("MKCOL", dir, []byte("")) + if err != nil { + return err + } + readCloser.Close() + return nil +} + +// DownloadFile reads the file at 'filePath' into the chunk. +func (storage *WebDAVStorage) DownloadFile(threadIndex int, filePath string, chunk *Chunk) (err error) { + readCloser, _, err := storage.sendRequest("GET", filePath, nil) + if err != nil { + return err + } + + _, err = RateLimitedCopy(chunk, readCloser, storage.DownloadRateLimit/storage.threads) + return err +} + +// UploadFile writes 'content' to the file at 'filePath'. +func (storage *WebDAVStorage) UploadFile(threadIndex int, filePath string, content []byte) (err error) { + + // If there is an error in creating the parent directory, proceed anyway + storage.createParentDirectory(threadIndex, filePath) + + readCloser, _, err := storage.sendRequest("PUT", filePath, content) + if err != nil { + return err + } + readCloser.Close() + return nil +} + +// If a local snapshot cache is needed for the storage to avoid downloading/uploading chunks too often when +// managing snapshots. +func (storage *WebDAVStorage) IsCacheNeeded() bool { return true } + +// If the 'MoveFile' method is implemented. +func (storage *WebDAVStorage) IsMoveFileImplemented() bool { return true } + +// If the storage can guarantee strong consistency. +func (storage *WebDAVStorage) IsStrongConsistent() bool { return false } + +// If the storage supports fast listing of files names. +func (storage *WebDAVStorage) IsFastListing() bool { return false } + +// Enable the test mode. +func (storage *WebDAVStorage) EnableTestMode() {}