jgaunt/duplicacy
1
0
Fork 0
mirror of https://github.com/gilbertchen/duplicacy synced 2025-12-06 00:03:38 +00:00
Code Issues Releases Wiki Activity

A few improvements to make WebDAV work better with pcloud and box.com

Browse Source
This commit is contained in:
Gilbert Chen
2018-04-05 15:29:41 -04:00
parent 0db8b9831b
commit 02cd41f4d0
4 changed files with 105 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
src/duplicacy_chunkdownloader.go
View File

@@ -317,8 +317,19 @@ func (downloader *ChunkDownloader) Download(threadIndex int, task ChunkDownloadT
} }
if !exist { if !exist {
// Retry for the Hubic backend as it may return 404 even when the chunk exists
if _, ok := downloader.storage.(*HubicStorage); ok && downloadAttempt < MaxDownloadAttempts { retry := false
// Retry for Hubic or WebDAV as it may return 404 even when the chunk exists
if _, ok := downloader.storage.(*HubicStorage); ok {
retry = true
}
if _, ok := downloader.storage.(*WebDAVStorage); ok {
retry = true
}
if retry && downloadAttempt < MaxDownloadAttempts {
LOG_WARN("DOWNLOAD_RETRY", "Failed to find the chunk %s; retrying", chunkID) LOG_WARN("DOWNLOAD_RETRY", "Failed to find the chunk %s; retrying", chunkID)
continue continue
} }

9
src/duplicacy_storage.go
View File

@@ -261,7 +261,7 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
return fileStorage return fileStorage
} }
urlRegex := regexp.MustCompile(`^(\w+)://([\w\-@\.]+@)?([^/]+)(/(.+))?`) urlRegex := regexp.MustCompile(`^([\w-]+)://([\w\-@\.]+@)?([^/]+)(/(.+))?`)
matched := urlRegex.FindStringSubmatch(storageURL) matched := urlRegex.FindStringSubmatch(storageURL)
@@ -606,12 +606,13 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
} }
SavePassword(preference, "swift_key", key) SavePassword(preference, "swift_key", key)
return swiftStorage return swiftStorage
} else if matched[1] == "webdav" { } else if matched[1] == "webdav" || matched[1] == "webdav-http" {
server := matched[3] server := matched[3]
username := matched[2] username := matched[2]
username = username[:len(username) - 1] username = username[:len(username) - 1]
storageDir := matched[5] storageDir := matched[5]
port := 0 port := 0
useHTTP := matched[1] == "webdav-http"
if strings.Contains(server, ":") { if strings.Contains(server, ":") {
index := strings.Index(server, ":") index := strings.Index(server, ":")
@@ -621,9 +622,9 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
prompt := fmt.Sprintf("Enter the WebDAV password:") prompt := fmt.Sprintf("Enter the WebDAV password:")
password := GetPassword(preference, "webdav_password", prompt, true, resetPassword) password := GetPassword(preference, "webdav_password", prompt, true, resetPassword)
webDAVStorage, err := CreateWebDAVStorage(server, port, username, password, storageDir, threads) webDAVStorage, err := CreateWebDAVStorage(server, port, username, password, storageDir, useHTTP, threads)
if err != nil { if err != nil {
LOG_ERROR("STORAGE_CREATE", "Failed to load the OpenStack Swift storage at %s: %v", storageURL, err) LOG_ERROR("STORAGE_CREATE", "Failed to load the WebDAV storage at %s: %v", storageURL, err)
return nil return nil
} }
SavePassword(preference, "webdav_password", password) SavePassword(preference, "webdav_password", password)

5
src/duplicacy_storage_test.go
View File

@@ -146,8 +146,8 @@ func loadStorage(localStoragePath string, threads int) (Storage, error) {
storage, err := CreateSwiftStorage(config["storage_url"], config["key"], threads) storage, err := CreateSwiftStorage(config["storage_url"], config["key"], threads)
storage.SetDefaultNestingLevels([]int{2, 3}, 2) storage.SetDefaultNestingLevels([]int{2, 3}, 2)
return storage, err return storage, err
} else if testStorageName == "pcloud" { } else if testStorageName == "pcloud" || testStorageName == "box" {
storage, err := CreateWebDAVStorage(config["host"], 0, config["username"], config["password"], config["storage_path"], threads) storage, err := CreateWebDAVStorage(config["host"], 0, config["username"], config["password"], config["storage_path"], false, threads)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -387,6 +387,7 @@ func TestStorage(t *testing.T) {
snapshotIDs := []string{} snapshotIDs := []string{}
for _, snapshotDir := range snapshotDirs { for _, snapshotDir := range snapshotDirs {
LOG_INFO("debug", "snapshot dir: %s", snapshotDir)
if len(snapshotDir) > 0 && snapshotDir[len(snapshotDir)-1] == '/' { if len(snapshotDir) > 0 && snapshotDir[len(snapshotDir)-1] == '/' {
snapshotIDs = append(snapshotIDs, snapshotDir[:len(snapshotDir)-1]) snapshotIDs = append(snapshotIDs, snapshotDir[:len(snapshotDir)-1])
} }

117
src/duplicacy_webdavstorage.go
View File

@@ -17,12 +17,12 @@ import (
"io/ioutil" "io/ioutil"
"math/rand" "math/rand"
"net/http" "net/http"
"os" //"net/http/httputil"
"path/filepath" "path/filepath"
"strconv" "strconv"
"strings"
"sync" "sync"
"time" "time"
"strings"
) )
type WebDAVStorage struct { type WebDAVStorage struct {
@@ -33,6 +33,7 @@ type WebDAVStorage struct {
username string username string
password string password string
storageDir string storageDir string
useHTTP bool
client *http.Client client *http.Client
threads int threads int
@@ -40,7 +41,15 @@ type WebDAVStorage struct {
directoryCacheLock sync.Mutex // lock for accessing directoryCache directoryCacheLock sync.Mutex // lock for accessing directoryCache
} }
func CreateWebDAVStorage(host string, port int, username string, password string, storageDir string, threads int) (storage *WebDAVStorage, err error) { var (
errWebDAVAuthorizationFailure = errors.New("Authentication failed")
errWebDAVMovedPermanently = errors.New("Moved permanently")
errWebDAVNotExist = errors.New("Path does not exist")
errWebDAVMaximumBackoff = errors.New("Maximum backoff reached")
errWebDAVMethodNotAllowed = errors.New("Method not allowed")
)
func CreateWebDAVStorage(host string, port int, username string, password string, storageDir string, useHTTP bool, threads int) (storage *WebDAVStorage, err error) {
if storageDir[len(storageDir)-1] != '/' { if storageDir[len(storageDir)-1] != '/' {
storageDir += "/" storageDir += "/"
} }
@@ -51,12 +60,18 @@ func CreateWebDAVStorage(host string, port int, username string, password string
username: username, username: username,
password: password, password: password,
storageDir: "", storageDir: "",
useHTTP: false,
client: http.DefaultClient, client: http.DefaultClient,
threads: threads, threads: threads,
directoryCache: make(map[string]int), directoryCache: make(map[string]int),
} }
// Make sure it doesn't follow redirect
storage.client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
return http.ErrUseLastResponse
}
exist, isDir, _, err := storage.GetFileInfo(0, storageDir) exist, isDir, _, err := storage.GetFileInfo(0, storageDir)
if err != nil { if err != nil {
return nil, err return nil, err
@@ -82,9 +97,12 @@ func (storage *WebDAVStorage) createConnectionString(uri string) string {
url := storage.host url := storage.host
if !strings.HasPrefix(url, "http://") && !strings.HasPrefix(url, "https://") { if storage.useHTTP {
url = "http://" + url
} else {
url = "https://" + url url = "https://" + url
} }
if storage.port > 0 { if storage.port > 0 {
url += fmt.Sprintf(":%d", storage.port) url += fmt.Sprintf(":%d", storage.port)
} }
@@ -98,7 +116,7 @@ func (storage *WebDAVStorage) retry(backoff int) int {
return backoff return backoff
} }
func (storage *WebDAVStorage) sendRequest(method string, uri string, data []byte) (io.ReadCloser, http.Header, error) { func (storage *WebDAVStorage) sendRequest(method string, uri string, depth int, data []byte) (io.ReadCloser, http.Header, error) {
backoff := 1 backoff := 1
for i := 0; i < 8; i++ { for i := 0; i < 8; i++ {
@@ -107,7 +125,7 @@ func (storage *WebDAVStorage) sendRequest(method string, uri string, data []byte
headers := make(map[string]string) headers := make(map[string]string)
if method == "PROPFIND" { if method == "PROPFIND" {
headers["Content-Type"] = "application/xml" headers["Content-Type"] = "application/xml"
headers["Depth"] = "1" headers["Depth"] = fmt.Sprintf("%d", depth)
dataReader = bytes.NewReader(data) dataReader = bytes.NewReader(data)
} else if method == "PUT" { } else if method == "PUT" {
headers["Content-Type"] = "application/octet-stream" headers["Content-Type"] = "application/octet-stream"
@@ -134,6 +152,9 @@ func (storage *WebDAVStorage) sendRequest(method string, uri string, data []byte
request.Header.Set(key, value) request.Header.Set(key, value)
} }
//requestDump, err := httputil.DumpRequest(request, true)
//LOG_INFO("debug", "Request: %s", requestDump)
response, err := storage.client.Do(request) response, err := storage.client.Do(request)
if err != nil { if err != nil {
LOG_TRACE("WEBDAV_RETRY", "URL request '%s %s' returned an error (%v)", method, uri, err) LOG_TRACE("WEBDAV_RETRY", "URL request '%s %s' returned an error (%v)", method, uri, err)
@@ -144,23 +165,31 @@ func (storage *WebDAVStorage) sendRequest(method string, uri string, data []byte
if response.StatusCode < 300 { if response.StatusCode < 300 {
return response.Body, response.Header, nil return response.Body, response.Header, nil
} }
if response.StatusCode == 301 {
return nil, nil, errWebDAVMovedPermanently
}
response.Body.Close() response.Body.Close()
if response.StatusCode == 401 { if response.StatusCode == 404 {
return nil, nil, errors.New("Login failed") // Retry if it is UPLOAD, otherwise return immediately
} else if response.StatusCode == 404 { if method != "PUT" {
return nil, nil, os.ErrNotExist return nil, nil, errWebDAVNotExist
}
} else if response.StatusCode == 405 {
return nil, nil, errWebDAVMethodNotAllowed
} }
LOG_INFO("WEBDAV_RETRY", "URL request '%s %s' returned status code %d", method, uri, response.StatusCode) LOG_INFO("WEBDAV_RETRY", "URL request '%s %s' returned status code %d", method, uri, response.StatusCode)
backoff = storage.retry(backoff) backoff = storage.retry(backoff)
} }
return nil, nil, errors.New("Maximum backoff reached") return nil, nil, errWebDAVMaximumBackoff
} }
type WebDAVProperties map[string]string type WebDAVProperties map[string]string
type WebDAVPropValue struct { type WebDAVPropValue struct {
XMLName xml.Name `xml:""` XMLName xml.Name `xml:""`
Value string `xml:",chardata"` Value string `xml:",innerxml"`
} }
type WebDAVProp struct { type WebDAVProp struct {
@@ -180,7 +209,7 @@ type WebDAVMultiStatus struct {
Responses []WebDAVResponse `xml:"response"` Responses []WebDAVResponse `xml:"response"`
} }
func (storage *WebDAVStorage) getProperties(uri string, properties ...string) (map[string]WebDAVProperties, error) { func (storage *WebDAVStorage) getProperties(uri string, depth int, properties ...string) (map[string]WebDAVProperties, error) {
propfind := "<prop>" propfind := "<prop>"
for _, p := range properties { for _, p := range properties {
@@ -190,7 +219,7 @@ func (storage *WebDAVStorage) getProperties(uri string, properties ...string) (m
body := fmt.Sprintf(`<?xml version="1.0" encoding="utf-8" ?><propfind xmlns="DAV:">%s</propfind>`, propfind) body := fmt.Sprintf(`<?xml version="1.0" encoding="utf-8" ?><propfind xmlns="DAV:">%s</propfind>`, propfind)
readCloser, _, err := storage.sendRequest("PROPFIND", uri, []byte(body)) readCloser, _, err := storage.sendRequest("PROPFIND", uri, depth, []byte(body))
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -213,6 +242,7 @@ func (storage *WebDAVStorage) getProperties(uri string, properties ...string) (m
responses := make(map[string]WebDAVProperties) responses := make(map[string]WebDAVProperties)
for _, responseTag := range object.Responses { for _, responseTag := range object.Responses {
if responseTag.PropStat == nil || responseTag.PropStat.Prop == nil || responseTag.PropStat.Prop.PropList == nil { if responseTag.PropStat == nil || responseTag.PropStat.Prop == nil || responseTag.PropStat.Prop.PropList == nil {
return nil, errors.New("no WebDAV properties") return nil, errors.New("no WebDAV properties")
} }
@@ -224,6 +254,7 @@ func (storage *WebDAVStorage) getProperties(uri string, properties ...string) (m
responseKey := responseTag.Href responseKey := responseTag.Href
responses[responseKey] = properties responses[responseKey] = properties
} }
return responses, nil return responses, nil
@@ -236,7 +267,7 @@ func (storage *WebDAVStorage) ListFiles(threadIndex int, dir string) (files []st
if dir[len(dir)-1] != '/' { if dir[len(dir)-1] != '/' {
dir += "/" dir += "/"
} }
properties, err := storage.getProperties(dir, "getcontentlength") properties, err := storage.getProperties(dir, 1, "getcontentlength", "resourcetype")
if err != nil { if err != nil {
return nil, nil, err return nil, nil, err
} }
@@ -248,10 +279,22 @@ func (storage *WebDAVStorage) ListFiles(threadIndex int, dir string) (files []st
continue continue
} }
if length, exist := m["getcontentlength"]; exist && length != "" { isDir := false
// This is a file; don't need it when listing "snapshots/" size := 0
if resourceType, exist := m["resourcetype"]; exist && strings.Contains(resourceType, "collection") {
isDir = true
} else if length, exist := m["getcontentlength"]; exist {
if length == "" {
isDir = true
} else {
size, _ = strconv.Atoi(length)
}
} else {
continue
}
if !isDir {
if dir != "snapshots/" { if dir != "snapshots/" {
size, _ := strconv.Atoi(length)
files = append(files, file[prefixLength:]) files = append(files, file[prefixLength:])
sizes = append(sizes, int64(size)) sizes = append(sizes, int64(size))
} }
@@ -271,29 +314,33 @@ func (storage *WebDAVStorage) ListFiles(threadIndex int, dir string) (files []st
// GetFileInfo returns the information about the file or directory at 'filePath'. // GetFileInfo returns the information about the file or directory at 'filePath'.
func (storage *WebDAVStorage) GetFileInfo(threadIndex int, filePath string) (exist bool, isDir bool, size int64, err error) { func (storage *WebDAVStorage) GetFileInfo(threadIndex int, filePath string) (exist bool, isDir bool, size int64, err error) {
readCloser, header, err := storage.sendRequest("HEAD", filePath, []byte("")) properties, err := storage.getProperties(filePath, 0, "getcontentlength", "resourcetype")
if err != nil { if err != nil {
if err == os.ErrNotExist { if err == errWebDAVNotExist {
return false, false, 0, nil return false, false, 0, nil
} else {
return false, false, 0, err
} }
if err == errWebDAVMovedPermanently {
// This must be a directory
return true, true, 0, nil
}
return false, false, 0, err
} }
readCloser.Close()
contentLength := header.Get("Content-Length") if m, exist := properties["/" + storage.storageDir + filePath]; !exist {
if contentLength == "" { return false, false, 0, nil
} else if resourceType, exist := m["resourcetype"]; exist && strings.Contains(resourceType, "collection") {
return true, true, 0, nil return true, true, 0, nil
} else { } else if length, exist := m["getcontentlength"]; exist && length != ""{
value, _ := strconv.Atoi(contentLength) value, _ := strconv.Atoi(length)
return true, false, int64(value), nil return true, false, int64(value), nil
} else {
return true, true, 0, nil
} }
} }
// DeleteFile deletes the file or directory at 'filePath'. // DeleteFile deletes the file or directory at 'filePath'.
func (storage *WebDAVStorage) DeleteFile(threadIndex int, filePath string) (err error) { func (storage *WebDAVStorage) DeleteFile(threadIndex int, filePath string) (err error) {
readCloser, _, err := storage.sendRequest("DELETE", filePath, []byte("")) readCloser, _, err := storage.sendRequest("DELETE", filePath, 0, []byte(""))
if err != nil { if err != nil {
return err return err
} }
@@ -303,7 +350,7 @@ func (storage *WebDAVStorage) DeleteFile(threadIndex int, filePath string) (err
// MoveFile renames the file. // MoveFile renames the file.
func (storage *WebDAVStorage) MoveFile(threadIndex int, from string, to string) (err error) { func (storage *WebDAVStorage) MoveFile(threadIndex int, from string, to string) (err error) {
readCloser, _, err := storage.sendRequest("MOVE", from, []byte(to)) readCloser, _, err := storage.sendRequest("MOVE", from, 0, []byte(to))
if err != nil { if err != nil {
return err return err
} }
@@ -349,8 +396,12 @@ func (storage *WebDAVStorage) CreateDirectory(threadIndex int, dir string) (err
// If there is an error in creating the parent directory, proceed anyway // If there is an error in creating the parent directory, proceed anyway
storage.createParentDirectory(threadIndex, dir) storage.createParentDirectory(threadIndex, dir)
readCloser, _, err := storage.sendRequest("MKCOL", dir, []byte("")) readCloser, _, err := storage.sendRequest("MKCOL", dir, 0, []byte(""))
if err != nil { if err != nil {
if err == errWebDAVMethodNotAllowed || err == errWebDAVMovedPermanently {
// We simply ignore these errors and assume that the directory already exists
return nil
}
return err return err
} }
readCloser.Close() readCloser.Close()
@@ -359,7 +410,7 @@ func (storage *WebDAVStorage) CreateDirectory(threadIndex int, dir string) (err
// DownloadFile reads the file at 'filePath' into the chunk. // DownloadFile reads the file at 'filePath' into the chunk.
func (storage *WebDAVStorage) DownloadFile(threadIndex int, filePath string, chunk *Chunk) (err error) { func (storage *WebDAVStorage) DownloadFile(threadIndex int, filePath string, chunk *Chunk) (err error) {
readCloser, _, err := storage.sendRequest("GET", filePath, nil) readCloser, _, err := storage.sendRequest("GET", filePath, 0, nil)
if err != nil { if err != nil {
return err return err
} }
@@ -374,7 +425,7 @@ func (storage *WebDAVStorage) UploadFile(threadIndex int, filePath string, conte
// If there is an error in creating the parent directory, proceed anyway // If there is an error in creating the parent directory, proceed anyway
storage.createParentDirectory(threadIndex, filePath) storage.createParentDirectory(threadIndex, filePath)
readCloser, _, err := storage.sendRequest("PUT", filePath, content) readCloser, _, err := storage.sendRequest("PUT", filePath, 0, content)
if err != nil { if err != nil {
return err return err
} }