Bump version to 2.2.1

The root cause was path.Dir can't handle Windows paths that use \ as the
separator.

.github/ISSUE_TEMPLATE.md

@@ -0,0 +1,5 @@
+Please submit an issue for bug reports or feature requests.  If you have any questions please post them on https://forum.duplicacy.com.
+
+When you're reporting a bug, please specify the OS, version, command line arguments, or any info that you think is helpful for the diagnosis.  If Duplicacy reports an error, please post the program output here.
+
+Note that this repository hosts the CLI version of Duplicacy only.  If you're reporting anything related to the GUI version, please visit https://forum.duplicacy.com.

LICENSE.md

@@ -1,8 +1,7 @@
 Copyright © 2017 Acrosync LLC
 
 * Free for personal use or commercial trial
-* Non-trial commercial use requires per-user CLI licenses available from [duplicacy.com](https://duplicacy.com/buy) at a cost of $20 per year
-* A user is defined as the computer account that creates or edits the files to be backed up; if a backup contains files created or edited by multiple users for commercial purposes, one CLI license is required for each user
+* Non-trial commercial use requires per-computer CLI licenses available from [duplicacy.com](https://duplicacy.com/buy.html) at a cost of $50 per year
 * The computer with a valid commercial license for the GUI version may run the CLI version without a CLI license
 * CLI licenses are not required to restore or manage backups; only the backup command requires valid CLI licenses
 * Modification and redistribution are permitted, but commercial use of derivative works is subject to the same requirements of this license

README.md

@@ -90,8 +90,7 @@ The following table compares the feature lists of all these backup tools:
 ## License
 
 * Free for personal use or commercial trial
-* Non-trial commercial use requires per-user CLI licenses available from [duplicacy.com](https://duplicacy.com/buy) at a cost of $20 per year
-* A user is defined as the computer account that creates or edits the files to be backed up; if a backup contains files created or edited by multiple users for commercial purposes, one CLI license is required for each user
+* Non-trial commercial use requires per-computer CLI licenses available from [duplicacy.com](https://duplicacy.com/buy.html) at a cost of $50 per year
 * The computer with a valid commercial license for the GUI version may run the CLI version without a CLI license
 * CLI licenses are not required to restore or manage backups; only the backup command requires valid CLI licenses
 * Modification and redistribution are permitted, but commercial use of derivative works is subject to the same requirements of this license

duplicacy/duplicacy_main.go

@@ -7,6 +7,7 @@ package main
 import (
 	"encoding/json"
 	"fmt"
+	"net/http"
 	"os"
 	"os/exec"
 	"os/signal"
@@ -16,7 +17,6 @@ import (
 	"runtime"
 	"strconv"
 	"strings"
-	"net/http"
 
 	_ "net/http/pprof"
 
@@ -32,6 +32,7 @@ const (
 )
 
 var ScriptEnabled bool
+var GitCommit = "unofficial"
 
 func getRepositoryPreference(context *cli.Context, storageName string) (repository string,
 	preference *duplicacy.Preference) {
@@ -158,8 +159,6 @@ func setGlobalOptions(context *cli.Context) {
 		}()
 	}
 
-
-
 	duplicacy.RunInBackground = context.GlobalBool("background")
 }
 
@@ -308,11 +307,11 @@ func configRepository(context *cli.Context, init bool) {
 		repositoryPath = context.String("repository")
 	}
 	preference := duplicacy.Preference{
-		Name:       storageName,
-		SnapshotID: snapshotID,
+		Name:           storageName,
+		SnapshotID:     snapshotID,
 		RepositoryPath: repositoryPath,
-		StorageURL: storageURL,
-		Encrypted:  context.Bool("encrypt"),
+		StorageURL:     storageURL,
+		Encrypted:      context.Bool("encrypt"),
 	}
 
 	storage := duplicacy.CreateStorage(preference, true, 1)
@@ -532,7 +531,7 @@ func setPreference(context *cli.Context) {
 	if triBool.IsSet() {
 		newPreference.DoNotSavePassword = triBool.IsTrue()
 	}
-	
+
 	newPreference.NobackupFile = context.String("nobackup-file")
 
 	key := context.String("key")
@@ -649,7 +648,7 @@ func changePassword(context *cli.Context) {
 				duplicacy.LOG_INFO("CONFIG_CLEAN", "The local copy of the old config has been removed")
 			}
 		}
-	} ()
+	}()
 
 	err = storage.DeleteFile(0, "config")
 	if err != nil {
@@ -783,27 +782,17 @@ func restoreRepository(context *cli.Context) {
 			pattern = pattern[1:]
 		}
 
-		if duplicacy.IsUnspecifiedFilter(pattern) {
-			pattern = "+" + pattern
-		}
-
-		if duplicacy.IsEmptyFilter(pattern) {
-			continue
-		}
-
-		if strings.HasPrefix(pattern, "i:") || strings.HasPrefix(pattern, "e:") {
-			valid, err := duplicacy.IsValidRegex(pattern[2:])
-			if !valid || err != nil {
-				duplicacy.LOG_ERROR("SNAPSHOT_FILTER", "Invalid regular expression encountered for filter: \"%s\", error: %v", pattern, err)
-			}
-		}
-
 		patterns = append(patterns, pattern)
 
+	
+
 	}
+	patterns = duplicacy.ProcessFilterLines(patterns, make([]string, 0))
 
 	duplicacy.LOG_DEBUG("REGEX_DEBUG", "There are %d compiled regular expressions stored", len(duplicacy.RegexMap))
 
+	duplicacy.LOG_INFO("SNAPSHOT_FILTER", "Loaded %d include/exclude pattern(s)", len(patterns))
+
 	storage.SetRateLimits(context.Int("limit-rate"), 0)
 	backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, preference.NobackupFile)
 	duplicacy.SavePassword(*preference, "password", password)
@@ -1066,12 +1055,17 @@ func pruneSnapshots(context *cli.Context) {
 		os.Exit(ArgumentExitCode)
 	}
 
+	threads := context.Int("threads")
+	if threads < 1 {
+		threads = 1
+	}
+
 	repository, preference := getRepositoryPreference(context, "")
 
 	runScript(context, preference.Name, "pre")
 
 	duplicacy.LOG_INFO("STORAGE_SET", "Storage set to %s", preference.StorageURL)
-	storage := duplicacy.CreateStorage(*preference, false, 1)
+	storage := duplicacy.CreateStorage(*preference, false, threads)
 	if storage == nil {
 		return
 	}
@@ -1110,7 +1104,7 @@ func pruneSnapshots(context *cli.Context) {
 
 	backupManager.SetupSnapshotCache(preference.Name)
 	backupManager.SnapshotManager.PruneSnapshots(selfID, snapshotID, revisions, tags, retentions,
-		exhaustive, exclusive, ignoredIDs, dryRun, deleteOnly, collectOnly)
+		exhaustive, exclusive, ignoredIDs, dryRun, deleteOnly, collectOnly, threads)
 
 	runScript(context, preference.Name, "post")
 }
@@ -1256,7 +1250,7 @@ func infoStorage(context *cli.Context) {
 
 	for _, dir := range dirs {
 		if len(dir) > 0 && dir[len(dir)-1] == '/' {
-			duplicacy.LOG_INFO("STORAGE_SNAPSHOT", "%s", dir[0:len(dir) - 1])
+			duplicacy.LOG_INFO("STORAGE_SNAPSHOT", "%s", dir[0:len(dir)-1])
 		}
 	}
 
@@ -1292,7 +1286,7 @@ func benchmark(context *cli.Context) {
 	}
 
 	threads := downloadThreads
-	if (threads < uploadThreads) {
+	if threads < uploadThreads {
 		threads = uploadThreads
 	}
 
@@ -1303,7 +1297,7 @@ func benchmark(context *cli.Context) {
 	if storage == nil {
 		return
 	}
-	duplicacy.Benchmark(repository, storage, int64(fileSize) * 1000000, chunkSize * 1024 * 1024, chunkCount, uploadThreads, downloadThreads)
+	duplicacy.Benchmark(repository, storage, int64(fileSize) * 1024 * 1024, chunkSize * 1024 * 1024, chunkCount, uploadThreads, downloadThreads)
 }
 
 func main() {
@@ -1395,7 +1389,7 @@ func main() {
 				},
 				cli.BoolFlag{
 					Name:  "vss",
-					Usage: "enable the Volume Shadow Copy service (Windows only)",
+					Usage: "enable the Volume Shadow Copy service (Windows and macOS using APFS only)",
 				},
 				cli.IntFlag{
 					Name:     "vss-timeout",
@@ -1702,6 +1696,12 @@ func main() {
 					Usage:    "prune snapshots from the specified storage",
 					Argument: "<storage name>",
 				},
+				cli.IntFlag{
+					Name:     "threads",
+					Value:    1,
+					Usage:    "number of threads used to prune unreferenced chunks",
+					Argument: "<n>",
+				},
 			},
 			Usage:     "Prune snapshots by revision, tag, or retention policy",
 			ArgsUsage: " ",
@@ -1761,8 +1761,8 @@ func main() {
 					Argument: "<storage name>",
 				},
 				cli.BoolFlag{
-					Name:     "bit-identical",
-					Usage:    "(when using -copy) make the new storage bit-identical to also allow rsync etc.",
+					Name:  "bit-identical",
+					Usage: "(when using -copy) make the new storage bit-identical to also allow rsync etc.",
 				},
 				cli.StringFlag{
 					Name:     "repository",
@@ -1803,10 +1803,10 @@ func main() {
 					Arg:   "true",
 				},
 				cli.StringFlag{
-					Name:  "nobackup-file",
-					Usage: "Directories containing a file with this name will not be backed up",
+					Name:     "nobackup-file",
+					Usage:    "Directories containing a file with this name will not be backed up",
 					Argument: "<file name>",
-					Value:   "",
+					Value:    "",
 				},
 				cli.StringFlag{
 					Name:  "key",
@@ -1928,6 +1928,11 @@ func main() {
 					Usage:    "the number of download threads (default to 1)",
 					Argument: "<n>",
 				},
+				cli.StringFlag{
+					Name:     "storage",
+					Usage:    "run the download/upload test agaist the specified storage",
+					Argument: "<storage name>",
+				},
 			},
 			Usage:     "Run a set of benchmarks to test download and upload speeds",
 			ArgsUsage: " ",
@@ -1967,8 +1972,8 @@ func main() {
 			Argument: "<address:port>",
 		},
 		cli.StringFlag{
-			Name:	"comment",
-			Usage:	"add a comment to identify the process",
+			Name:  "comment",
+			Usage: "add a comment to identify the process",
 		},
 	}
 
@@ -1976,13 +1981,13 @@ func main() {
 	app.Name = "duplicacy"
 	app.HelpName = "duplicacy"
 	app.Usage = "A new generation cloud backup tool based on lock-free deduplication"
-	app.Version = "2.1.0"
+	app.Version = "2.2.1" + " (" + GitCommit + ")"
 
 	// If the program is interrupted, call the RunAtError function.
 	c := make(chan os.Signal, 1)
 	signal.Notify(c, os.Interrupt)
 	go func() {
-		for _ = range c {
+		for range c {
 			duplicacy.RunAtError()
 			os.Exit(1)
 		}

src/duplicacy_azurestorage.go

@@ -104,7 +104,7 @@ func (azureStorage *AzureStorage) ListFiles(threadIndex int, dir string) (files
 
 	if dir == "snapshots/" {
 
-		for subDir, _ := range subDirs {
+		for subDir := range subDirs {
 			files = append(files, subDir)
 		}
 

src/duplicacy_b2client.go

@@ -5,19 +5,22 @@
 package duplicacy
 
 import (
-	"bytes"
-	"crypto/sha1"
-	"encoding/base64"
-	"encoding/hex"
-	"encoding/json"
-	"fmt"
 	"io"
-	"io/ioutil"
-	"math/rand"
-	"net/http"
+	"os"
+	"fmt"
+	"bytes"
+	"time"
+	"sync"
 	"strconv"
 	"strings"
-	"time"
+	"net/url"
+	"net/http"
+	"math/rand"
+	"io/ioutil"
+	"crypto/sha1"
+	"encoding/hex"
+	"encoding/json"
+	"encoding/base64"
 )
 
 type B2Error struct {
@@ -39,66 +42,112 @@ var B2AuthorizationURL = "https://api.backblazeb2.com/b2api/v1/b2_authorize_acco
 
 type B2Client struct {
 	HTTPClient         *http.Client
+
 	AccountID          string
+	ApplicationKeyID   string
 	ApplicationKey     string
+	BucketName         string
+	BucketID           string
+	StorageDir         string
+
+	Lock               sync.Mutex
 	AuthorizationToken string
 	APIURL             string
 	DownloadURL        string
-	BucketName         string
-	BucketID           string
+	IsAuthorized       bool
 
-	UploadURL   string
-	UploadToken string
+	UploadURLs         []string
+	UploadTokens       []string
 
-	TestMode bool
+	Threads            int
+	MaximumRetries     int
+	TestMode           bool
 }
 
-func NewB2Client(accountID string, applicationKey string) *B2Client {
+// URL encode the given path but keep the slashes intact
+func B2Escape(path string) string {
+	var components []string
+	for _, c := range strings.Split(path, "/") {
+		components = append(components, url.QueryEscape(c))
+	}
+	return strings.Join(components, "/")
+}
+
+func NewB2Client(applicationKeyID string, applicationKey string, storageDir string, threads int) *B2Client {
+
+	for storageDir != "" && storageDir[0] == '/' {
+		storageDir = storageDir[1:]
+	}
+
+	if storageDir != "" && storageDir[len(storageDir) - 1] != '/' {
+		storageDir += "/"
+	}
+
+	maximumRetries := 10
+	if value, found := os.LookupEnv("DUPLICACY_B2_RETRIES"); found && value != "" {
+		maximumRetries, _ = strconv.Atoi(value)
+		LOG_INFO("B2_RETRIES", "Setting maximum retries for B2 to %d", maximumRetries)
+	}
+
 	client := &B2Client{
-		HTTPClient:     http.DefaultClient,
-		AccountID:      accountID,
-		ApplicationKey: applicationKey,
+		HTTPClient:       http.DefaultClient,
+		ApplicationKeyID: applicationKeyID,
+		ApplicationKey:   applicationKey,
+		StorageDir:       storageDir,
+		UploadURLs:       make([]string, threads),
+		UploadTokens:     make([]string, threads),
+		Threads:          threads,
+		MaximumRetries:   maximumRetries,
 	}
 	return client
 }
 
-func (client *B2Client) retry(backoff int, response *http.Response) int {
+func (client *B2Client) getAPIURL() string {
+	client.Lock.Lock()
+	defer client.Lock.Unlock()
+	return client.APIURL
+}
+
+func (client *B2Client) getDownloadURL() string {
+	client.Lock.Lock()
+	defer client.Lock.Unlock()
+	return client.DownloadURL
+}
+
+func (client *B2Client) retry(retries int, response *http.Response) int {
 	if response != nil {
 		if backoffList, found := response.Header["Retry-After"]; found && len(backoffList) > 0 {
 			retryAfter, _ := strconv.Atoi(backoffList[0])
 			if retryAfter >= 1 {
 				time.Sleep(time.Duration(retryAfter) * time.Second)
-				return 0
+				return 1
 			}
 		}
 	}
-	if backoff == 0 {
-		backoff = 1
-	} else {
-		backoff *= 2
+
+	if retries >= client.MaximumRetries + 1 {
+		return 0
 	}
-	time.Sleep(time.Duration(backoff) * time.Second)
-	return backoff
+	retries++
+	delay := 1 << uint(retries)
+	if delay > 64 {
+		delay = 64
+	}
+	delayInSeconds := (rand.Float32() + 1.0) * float32(delay) / 2.0
+
+	time.Sleep(time.Duration(delayInSeconds) * time.Second)
+	return retries
 }
 
-func (client *B2Client) call(url string, method string, requestHeaders map[string]string, input interface{}) (io.ReadCloser, http.Header, int64, error) {
-
-	switch method {
-	case http.MethodGet:
-		break
-	case http.MethodHead:
-		break
-	case http.MethodPost:
-		break
-	default:
-		return nil, nil, 0, fmt.Errorf("unhandled http request method: " + method)
-	}
+func (client *B2Client) call(threadIndex int, requestURL string, method string, requestHeaders map[string]string, input interface{}) (
+	                         io.ReadCloser, http.Header, int64, error) {
 
 	var response *http.Response
 
-	backoff := 0
-	for i := 0; i < 8; i++ {
-		var inputReader *bytes.Reader
+	retries := 0
+	for {
+		var inputReader io.Reader
+		isUpload := false
 
 		switch input.(type) {
 		default:
@@ -107,21 +156,43 @@ func (client *B2Client) call(url string, method string, requestHeaders map[strin
 				return nil, nil, 0, err
 			}
 			inputReader = bytes.NewReader(jsonInput)
-		case []byte:
-			inputReader = bytes.NewReader(input.([]byte))
 		case int:
 			inputReader = bytes.NewReader([]byte(""))
+		case []byte:
+			isUpload = true
+			inputReader = bytes.NewReader(input.([]byte))
+		case *RateLimitedReader:
+			isUpload = true
+			rateLimitedReader := input.(*RateLimitedReader)
+			rateLimitedReader.Reset()
+			inputReader = rateLimitedReader
 		}
 
-		request, err := http.NewRequest(method, url, inputReader)
+
+		if isUpload {
+			if client.UploadURLs[threadIndex] == "" || client.UploadTokens[threadIndex] == "" {
+				err := client.getUploadURL(threadIndex)
+				if err != nil {
+					return nil, nil, 0, err
+				}
+			}
+			requestURL = client.UploadURLs[threadIndex]
+		}
+
+		request, err := http.NewRequest(method, requestURL, inputReader)
 		if err != nil {
 			return nil, nil, 0, err
 		}
 
-		if url == B2AuthorizationURL {
-			request.Header.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(client.AccountID+":"+client.ApplicationKey)))
+		if requestURL == B2AuthorizationURL {
+			request.Header.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(client.ApplicationKeyID+":"+client.ApplicationKey)))
+		} else if isUpload {
+			request.ContentLength, _ = strconv.ParseInt(requestHeaders["Content-Length"], 10, 64)
+			request.Header.Set("Authorization", client.UploadTokens[threadIndex])
 		} else {
+			client.Lock.Lock()
 			request.Header.Set("Authorization", client.AuthorizationToken)
+			client.Lock.Unlock()
 		}
 
 		if requestHeaders != nil {
@@ -132,7 +203,9 @@ func (client *B2Client) call(url string, method string, requestHeaders map[strin
 
 		if client.TestMode {
 			r := rand.Float32()
-			if r < 0.5 {
+			if r < 0.5 && isUpload {
+				request.Header.Set("X-Bz-Test-Mode", "fail_some_uploads")
+			} else if r < 0.75 {
 				request.Header.Set("X-Bz-Test-Mode", "expire_some_account_authorization_tokens")
 			} else {
 				request.Header.Set("X-Bz-Test-Mode", "force_cap_exceeded")
@@ -141,27 +214,46 @@ func (client *B2Client) call(url string, method string, requestHeaders map[strin
 
 		response, err = client.HTTPClient.Do(request)
 		if err != nil {
-			if url != B2AuthorizationURL {
-				LOG_DEBUG("BACKBLAZE_CALL", "URL request '%s' returned an error: %v", url, err)
-				backoff = client.retry(backoff, response)
-				continue
+
+			// Don't retry when the first authorization request fails
+			if requestURL == B2AuthorizationURL && !client.IsAuthorized {
+				return nil, nil, 0, err
 			}
-			return nil, nil, 0, err
+
+			LOG_TRACE("BACKBLAZE_CALL", "[%d] URL request '%s' returned an error: %v", threadIndex, requestURL, err)
+
+			retries = client.retry(retries, response)
+			if retries <= 0 {
+				return nil, nil, 0, err
+			}
+
+			// Clear the upload url to requrest a new one on retry
+			if isUpload {
+				client.UploadURLs[threadIndex] = ""
+				client.UploadTokens[threadIndex] = ""
+			}
+			continue
+
 		}
 
 		if response.StatusCode < 300 {
 			return response.Body, response.Header, response.ContentLength, nil
 		}
 
-		LOG_DEBUG("BACKBLAZE_CALL", "URL request '%s %s' returned status code %d", method, url, response.StatusCode)
+		e := &B2Error{}
+		if err := json.NewDecoder(response.Body).Decode(e); err != nil {
+			LOG_TRACE("BACKBLAZE_CALL", "[%d] URL request '%s %s' returned status code %d", threadIndex, method, requestURL, response.StatusCode)
+		} else {
+			LOG_TRACE("BACKBLAZE_CALL", "[%d] URL request '%s %s' returned %d %s", threadIndex, method, requestURL, response.StatusCode, e.Message)
+		}
 
-		io.Copy(ioutil.Discard, response.Body)
 		response.Body.Close()
+
 		if response.StatusCode == 401 {
-			if url == B2AuthorizationURL {
+			if requestURL == B2AuthorizationURL {
 				return nil, nil, 0, fmt.Errorf("Authorization failure")
 			}
-			client.AuthorizeAccount()
+			client.AuthorizeAccount(threadIndex)
 			continue
 		} else if response.StatusCode == 403 {
 			if !client.TestMode {
@@ -175,32 +267,21 @@ func (client *B2Client) call(url string, method string, requestHeaders map[strin
 		} else if response.StatusCode == 416 {
 			if http.MethodHead == method {
 				// 416 Requested Range Not Satisfiable
-				return nil, nil, 0, fmt.Errorf("URL request '%s' returned status code %d", url, response.StatusCode)
+				return nil, nil, 0, fmt.Errorf("URL request '%s' returned %d %s", requestURL, response.StatusCode, e.Message)
 			}
-		} else if response.StatusCode == 429 || response.StatusCode == 408 {
-			backoff = client.retry(backoff, response)
-			continue
-		} else if response.StatusCode >= 500 && response.StatusCode <= 599 {
-			backoff = client.retry(backoff, response)
-			continue
-		} else {
-			LOG_INFO("BACKBLAZE_CALL", "URL request '%s' returned status code %d", url, response.StatusCode)
-			backoff = client.retry(backoff, response)
-			continue
 		}
 
-		defer response.Body.Close()
-
-		e := &B2Error{}
-
-		if err := json.NewDecoder(response.Body).Decode(e); err != nil {
-			return nil, nil, 0, err
+		retries = client.retry(retries, response)
+		if retries <= 0 {
+			return nil, nil, 0, fmt.Errorf("URL request '%s' returned %d %s", requestURL, response.StatusCode, e.Message)
 		}
 
-		return nil, nil, 0, e
+		if isUpload {
+			client.UploadURLs[threadIndex] = ""
+			client.UploadTokens[threadIndex] = ""
+		}
 	}
 
-	return nil, nil, 0, fmt.Errorf("Maximum backoff reached")
 }
 
 type B2AuthorizeAccountOutput struct {
@@ -210,9 +291,11 @@ type B2AuthorizeAccountOutput struct {
 	DownloadURL        string
 }
 
-func (client *B2Client) AuthorizeAccount() (err error) {
+func (client *B2Client) AuthorizeAccount(threadIndex int) (err error) {
+	client.Lock.Lock()
+	defer client.Lock.Unlock()
 
-	readCloser, _, _, err := client.call(B2AuthorizationURL, http.MethodPost, nil, make(map[string]string))
+	readCloser, _, _, err := client.call(threadIndex, B2AuthorizationURL, http.MethodPost, nil, make(map[string]string))
 	if err != nil {
 		return err
 	}
@@ -225,15 +308,20 @@ func (client *B2Client) AuthorizeAccount() (err error) {
 		return err
 	}
 
+	// The account id may be different from the application key id so we're getting the account id from the returned
+	// json object here, which is needed by the b2_list_buckets call.
+	client.AccountID = output.AccountID
+
 	client.AuthorizationToken = output.AuthorizationToken
 	client.APIURL = output.APIURL
 	client.DownloadURL = output.DownloadURL
+	client.IsAuthorized = true
 
 	return nil
 }
 
 type ListBucketOutput struct {
-	AccoundID  string
+	AccountID  string
 	BucketID   string
 	BucketName string
 	BucketType string
@@ -244,9 +332,9 @@ func (client *B2Client) FindBucket(bucketName string) (err error) {
 	input := make(map[string]string)
 	input["accountId"] = client.AccountID
 
-	url := client.APIURL + "/b2api/v1/b2_list_buckets"
+	url := client.getAPIURL() + "/b2api/v1/b2_list_buckets"
 
-	readCloser, _, _, err := client.call(url, http.MethodPost, nil, input)
+	readCloser, _, _, err := client.call(0, url, http.MethodPost, nil, input)
 	if err != nil {
 		return err
 	}
@@ -288,7 +376,7 @@ type B2ListFileNamesOutput struct {
 	NextFileId   string
 }
 
-func (client *B2Client) ListFileNames(startFileName string, singleFile bool, includeVersions bool) (files []*B2Entry, err error) {
+func (client *B2Client) ListFileNames(threadIndex int, startFileName string, singleFile bool, includeVersions bool) (files []*B2Entry, err error) {
 
 	maxFileCount := 1000
 	if singleFile {
@@ -306,20 +394,21 @@ func (client *B2Client) ListFileNames(startFileName string, singleFile bool, inc
 
 	input := make(map[string]interface{})
 	input["bucketId"] = client.BucketID
-	input["startFileName"] = startFileName
+	input["startFileName"] = client.StorageDir + startFileName
 	input["maxFileCount"] = maxFileCount
+	input["prefix"] = client.StorageDir
 
 	for {
-		url := client.APIURL + "/b2api/v1/b2_list_file_names"
+		url := client.getAPIURL() + "/b2api/v1/b2_list_file_names"
 		requestHeaders := map[string]string{}
 		requestMethod := http.MethodPost
 		var requestInput interface{}
 		requestInput = input
 		if includeVersions {
-			url = client.APIURL + "/b2api/v1/b2_list_file_versions"
+			url = client.getAPIURL() + "/b2api/v1/b2_list_file_versions"
 		} else if singleFile {
 			// handle a single file with no versions as a special case to download the last byte of the file
-			url = client.DownloadURL + "/file/" + client.BucketName + "/" + startFileName
+			url = client.getDownloadURL() + "/file/" + client.BucketName + "/" + B2Escape(client.StorageDir + startFileName)
 			// requesting byte -1 works for empty files where 0-0 fails with a 416 error
 			requestHeaders["Range"] = "bytes=-1"
 			// HEAD request
@@ -329,7 +418,7 @@ func (client *B2Client) ListFileNames(startFileName string, singleFile bool, inc
 		var readCloser io.ReadCloser
 		var responseHeader http.Header
 		var err error
-		readCloser, responseHeader, _, err = client.call(url, requestMethod, requestHeaders, requestInput)
+		readCloser, responseHeader, _, err = client.call(threadIndex, url, requestMethod, requestHeaders, requestInput)
 		if err != nil {
 			return nil, err
 		}
@@ -342,7 +431,7 @@ func (client *B2Client) ListFileNames(startFileName string, singleFile bool, inc
 
 		if singleFile && !includeVersions {
 			if responseHeader == nil {
-				LOG_DEBUG("BACKBLAZE_LIST", "b2_download_file_by_name did not return headers")
+				LOG_DEBUG("BACKBLAZE_LIST", "%s did not return headers", url)
 				return []*B2Entry{}, nil
 			}
 			requiredHeaders := []string{
@@ -356,7 +445,7 @@ func (client *B2Client) ListFileNames(startFileName string, singleFile bool, inc
 				}
 			}
 			if len(missingKeys) > 0 {
-				return nil, fmt.Errorf("b2_download_file_by_name missing headers: %s", missingKeys)
+				return nil, fmt.Errorf("%s missing headers: %s", url, missingKeys)
 			}
 			// construct the B2Entry from the response headers of the download request
 			fileID := responseHeader.Get("x-bz-file-id")
@@ -373,14 +462,14 @@ func (client *B2Client) ListFileNames(startFileName string, singleFile bool, inc
 				// this should only execute if the requested file is empty and the range request didn't result in a Content-Range header
 				fileSize, _ = strconv.ParseInt(lengthString, 0, 64)
 				if fileSize != 0 {
-					return nil, fmt.Errorf("b2_download_file_by_name returned non-zero file length")
+					return nil, fmt.Errorf("%s returned non-zero file length", url)
 				}
 			} else {
-				return nil, fmt.Errorf("could not parse b2_download_file_by_name headers")
+				return nil, fmt.Errorf("could not parse headers returned by %s", url)
 			}
 			fileUploadTimestamp, _ := strconv.ParseInt(responseHeader.Get("X-Bz-Upload-Timestamp"), 0, 64)
 
-			return []*B2Entry{&B2Entry{fileID, fileName, fileAction, fileSize, fileUploadTimestamp}}, nil
+			return []*B2Entry{{fileID, fileName[len(client.StorageDir):], fileAction, fileSize, fileUploadTimestamp}}, nil
 		}
 
 		if err = json.NewDecoder(readCloser).Decode(&output); err != nil {
@@ -389,31 +478,27 @@ func (client *B2Client) ListFileNames(startFileName string, singleFile bool, inc
 
 		ioutil.ReadAll(readCloser)
 
-		if startFileName == "" {
-			files = append(files, output.Files...)
-		} else {
-			for _, file := range output.Files {
-				if singleFile {
-					if file.FileName == startFileName {
-						files = append(files, file)
-						if !includeVersions {
-							output.NextFileName = ""
-							break
-						}
-					} else {
+		for _, file := range output.Files {
+			file.FileName = file.FileName[len(client.StorageDir):]
+			if singleFile {
+				if file.FileName == startFileName {
+					files = append(files, file)
+					if !includeVersions {
 						output.NextFileName = ""
 						break
 					}
 				} else {
-					if strings.HasPrefix(file.FileName, startFileName) {
-						files = append(files, file)
-					} else {
-						output.NextFileName = ""
-						break
-					}
+					output.NextFileName = ""
+					break
+				}
+			} else {
+				if strings.HasPrefix(file.FileName, startFileName) {
+					files = append(files, file)
+				} else {
+					output.NextFileName = ""
+					break
 				}
 			}
-
 		}
 
 		if len(output.NextFileName) == 0 {
@@ -429,14 +514,14 @@ func (client *B2Client) ListFileNames(startFileName string, singleFile bool, inc
 	return files, nil
 }
 
-func (client *B2Client) DeleteFile(fileName string, fileID string) (err error) {
+func (client *B2Client) DeleteFile(threadIndex int, fileName string, fileID string) (err error) {
 
 	input := make(map[string]string)
-	input["fileName"] = fileName
+	input["fileName"] = client.StorageDir + fileName
 	input["fileId"] = fileID
 
-	url := client.APIURL + "/b2api/v1/b2_delete_file_version"
-	readCloser, _, _, err := client.call(url, http.MethodPost, make(map[string]string), input)
+	url := client.getAPIURL() + "/b2api/v1/b2_delete_file_version"
+	readCloser, _, _, err := client.call(threadIndex, url, http.MethodPost, make(map[string]string), input)
 	if err != nil {
 		return err
 	}
@@ -449,14 +534,14 @@ type B2HideFileOutput struct {
 	FileID string
 }
 
-func (client *B2Client) HideFile(fileName string) (fileID string, err error) {
+func (client *B2Client) HideFile(threadIndex int, fileName string) (fileID string, err error) {
 
 	input := make(map[string]string)
 	input["bucketId"] = client.BucketID
-	input["fileName"] = fileName
+	input["fileName"] = client.StorageDir + fileName
 
-	url := client.APIURL + "/b2api/v1/b2_hide_file"
-	readCloser, _, _, err := client.call(url, http.MethodPost, make(map[string]string), input)
+	url := client.getAPIURL() + "/b2api/v1/b2_hide_file"
+	readCloser, _, _, err := client.call(threadIndex, url, http.MethodPost, make(map[string]string), input)
 	if err != nil {
 		return "", err
 	}
@@ -473,11 +558,11 @@ func (client *B2Client) HideFile(fileName string) (fileID string, err error) {
 	return output.FileID, nil
 }
 
-func (client *B2Client) DownloadFile(filePath string) (io.ReadCloser, int64, error) {
+func (client *B2Client) DownloadFile(threadIndex int, filePath string) (io.ReadCloser, int64, error) {
 
-	url := client.DownloadURL + "/file/" + client.BucketName + "/" + filePath
+	url := client.getDownloadURL() + "/file/" + client.BucketName + "/" + B2Escape(client.StorageDir + filePath)
 
-	readCloser, _, len, err := client.call(url, http.MethodGet, make(map[string]string), 0)
+	readCloser, _, len, err := client.call(threadIndex, url, http.MethodGet, make(map[string]string), 0)
 	return readCloser, len, err
 }
 
@@ -487,12 +572,12 @@ type B2GetUploadArgumentOutput struct {
 	AuthorizationToken string
 }
 
-func (client *B2Client) getUploadURL() error {
+func (client *B2Client) getUploadURL(threadIndex int) error {
 	input := make(map[string]string)
 	input["bucketId"] = client.BucketID
 
-	url := client.APIURL + "/b2api/v1/b2_get_upload_url"
-	readCloser, _, _, err := client.call(url, http.MethodPost, make(map[string]string), input)
+	url := client.getAPIURL() + "/b2api/v1/b2_get_upload_url"
+	readCloser, _, _, err := client.call(threadIndex, url, http.MethodPost, make(map[string]string), input)
 	if err != nil {
 		return err
 	}
@@ -505,96 +590,29 @@ func (client *B2Client) getUploadURL() error {
 		return err
 	}
 
-	client.UploadURL = output.UploadURL
-	client.UploadToken = output.AuthorizationToken
+	client.UploadURLs[threadIndex] = output.UploadURL
+	client.UploadTokens[threadIndex] = output.AuthorizationToken
 
 	return nil
 }
 
-func (client *B2Client) UploadFile(filePath string, content []byte, rateLimit int) (err error) {
+func (client *B2Client) UploadFile(threadIndex int, filePath string, content []byte, rateLimit int) (err error) {
 
 	hasher := sha1.New()
 	hasher.Write(content)
 	hash := hex.EncodeToString(hasher.Sum(nil))
 
 	headers := make(map[string]string)
-	headers["X-Bz-File-Name"] = filePath
+	headers["X-Bz-File-Name"] = B2Escape(client.StorageDir + filePath)
+	headers["Content-Length"] = fmt.Sprintf("%d", len(content))
 	headers["Content-Type"] = "application/octet-stream"
 	headers["X-Bz-Content-Sha1"] = hash
 
-	var response *http.Response
-
-	backoff := 0
-	for i := 0; i < 8; i++ {
-
-		if client.UploadURL == "" || client.UploadToken == "" {
-			err = client.getUploadURL()
-			if err != nil {
-				return err
-			}
-		}
-
-		request, err := http.NewRequest("POST", client.UploadURL, CreateRateLimitedReader(content, rateLimit))
-		if err != nil {
-			return err
-		}
-		request.ContentLength = int64(len(content))
-
-		request.Header.Set("Authorization", client.UploadToken)
-		request.Header.Set("X-Bz-File-Name", filePath)
-		request.Header.Set("Content-Type", "application/octet-stream")
-		request.Header.Set("X-Bz-Content-Sha1", hash)
-
-		for key, value := range headers {
-			request.Header.Set(key, value)
-		}
-
-		if client.TestMode {
-			r := rand.Float32()
-			if r < 0.8 {
-				request.Header.Set("X-Bz-Test-Mode", "fail_some_uploads")
-			} else if r < 0.9 {
-				request.Header.Set("X-Bz-Test-Mode", "expire_some_account_authorization_tokens")
-			} else {
-				request.Header.Set("X-Bz-Test-Mode", "force_cap_exceeded")
-			}
-		}
-
-		response, err = client.HTTPClient.Do(request)
-		if err != nil {
-			LOG_DEBUG("BACKBLAZE_UPLOAD", "URL request '%s' returned an error: %v", client.UploadURL, err)
-			backoff = client.retry(backoff, response)
-			client.UploadURL = ""
-			client.UploadToken = ""
-			continue
-		}
-
-		io.Copy(ioutil.Discard, response.Body)
-		response.Body.Close()
-
-		if response.StatusCode < 300 {
-			return nil
-		}
-
-		LOG_DEBUG("BACKBLAZE_UPLOAD", "URL request '%s' returned status code %d", client.UploadURL, response.StatusCode)
-
-		if response.StatusCode == 401 {
-			LOG_INFO("BACKBLAZE_UPLOAD", "Re-authorization required")
-			client.UploadURL = ""
-			client.UploadToken = ""
-			continue
-		} else if response.StatusCode == 403 {
-			if !client.TestMode {
-				return fmt.Errorf("B2 cap exceeded")
-			}
-			continue
-		} else {
-			LOG_INFO("BACKBLAZE_UPLOAD", "URL request '%s' returned status code %d", client.UploadURL, response.StatusCode)
-			backoff = client.retry(backoff, response)
-			client.UploadURL = ""
-			client.UploadToken = ""
-		}
+	readCloser, _, _, err := client.call(threadIndex, "", http.MethodPost, headers, CreateRateLimitedReader(content, rateLimit))
+	if err != nil {
+		return err
 	}
 
-	return fmt.Errorf("Maximum backoff reached")
+	readCloser.Close()
+	return nil
 }

src/duplicacy_b2client_test.go

@@ -37,7 +37,7 @@ func createB2ClientForTest(t *testing.T) (*B2Client, string) {
 		return nil, ""
 	}
 
-	return NewB2Client(b2["account"], b2["key"]), b2["bucket"]
+	return NewB2Client(b2["account"], b2["key"], b2["directory"], 1), b2["bucket"]
 
 }
 
@@ -50,7 +50,7 @@ func TestB2Client(t *testing.T) {
 
 	b2Client.TestMode = true
 
-	err := b2Client.AuthorizeAccount()
+	err := b2Client.AuthorizeAccount(0)
 	if err != nil {
 		t.Errorf("Failed to authorize the b2 account: %v", err)
 		return
@@ -64,14 +64,14 @@ func TestB2Client(t *testing.T) {
 
 	testDirectory := "b2client_test/"
 
-	files, err := b2Client.ListFileNames(testDirectory, false, false)
+	files, err := b2Client.ListFileNames(0, testDirectory, false, false)
 	if err != nil {
 		t.Errorf("Failed to list files: %v", err)
 		return
 	}
 
 	for _, file := range files {
-		err = b2Client.DeleteFile(file.FileName, file.FileID)
+		err = b2Client.DeleteFile(0, file.FileName, file.FileID)
 		if err != nil {
 			t.Errorf("Failed to delete file '%s': %v", file.FileName, err)
 		}
@@ -90,14 +90,14 @@ func TestB2Client(t *testing.T) {
 		hash := sha256.Sum256(content)
 		name := hex.EncodeToString(hash[:])
 
-		err = b2Client.UploadFile(testDirectory+name, content, 100)
+		err = b2Client.UploadFile(0, testDirectory+name, content, 100)
 		if err != nil {
 			t.Errorf("Error uploading file '%s': %v", name, err)
 			return
 		}
 	}
 
-	files, err = b2Client.ListFileNames(testDirectory, false, false)
+	files, err = b2Client.ListFileNames(0, testDirectory, false, false)
 	if err != nil {
 		t.Errorf("Failed to list files: %v", err)
 		return
@@ -105,7 +105,7 @@ func TestB2Client(t *testing.T) {
 
 	for _, file := range files {
 
-		readCloser, _, err := b2Client.DownloadFile(file.FileName)
+		readCloser, _, err := b2Client.DownloadFile(0, file.FileName)
 		if err != nil {
 			t.Errorf("Error downloading file '%s': %v", file.FileName, err)
 			return
@@ -125,7 +125,7 @@ func TestB2Client(t *testing.T) {
 	}
 
 	for _, file := range files {
-		err = b2Client.DeleteFile(file.FileName, file.FileID)
+		err = b2Client.DeleteFile(0, file.FileName, file.FileID)
 		if err != nil {
 			t.Errorf("Failed to delete file '%s': %v", file.FileName, err)
 		}

src/duplicacy_b2storage.go

@@ -11,32 +11,26 @@ import (
 type B2Storage struct {
 	StorageBase
 
-	clients []*B2Client
+	client *B2Client
 }
 
 // CreateB2Storage creates a B2 storage object.
-func CreateB2Storage(accountID string, applicationKey string, bucket string, threads int) (storage *B2Storage, err error) {
+func CreateB2Storage(accountID string, applicationKey string, bucket string, storageDir string, threads int) (storage *B2Storage, err error) {
 
-	var clients []*B2Client
+	client := NewB2Client(accountID, applicationKey, storageDir, threads)
 
-	for i := 0; i < threads; i++ {
-		client := NewB2Client(accountID, applicationKey)
+	err = client.AuthorizeAccount(0)
+	if err != nil {
+		return nil, err
+	}
 
-		err = client.AuthorizeAccount()
-		if err != nil {
-			return nil, err
-		}
-
-		err = client.FindBucket(bucket)
-		if err != nil {
-			return nil, err
-		}
-
-		clients = append(clients, client)
+	err = client.FindBucket(bucket)
+	if err != nil {
+		return nil, err
 	}
 
 	storage = &B2Storage{
-		clients: clients,
+		client: client,
 	}
 
 	storage.DerivedStorage = storage
@@ -56,7 +50,7 @@ func (storage *B2Storage) ListFiles(threadIndex int, dir string) (files []string
 		includeVersions = true
 	}
 
-	entries, err := storage.clients[threadIndex].ListFileNames(dir, false, includeVersions)
+	entries, err := storage.client.ListFileNames(threadIndex, dir, false, includeVersions)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -71,7 +65,7 @@ func (storage *B2Storage) ListFiles(threadIndex int, dir string) (files []string
 			subDirs[subDir+"/"] = true
 		}
 
-		for subDir, _ := range subDirs {
+		for subDir := range subDirs {
 			files = append(files, subDir)
 		}
 	} else if dir == "chunks" {
@@ -102,7 +96,7 @@ func (storage *B2Storage) DeleteFile(threadIndex int, filePath string) (err erro
 
 	if strings.HasSuffix(filePath, ".fsl") {
 		filePath = filePath[:len(filePath)-len(".fsl")]
-		entries, err := storage.clients[threadIndex].ListFileNames(filePath, true, true)
+		entries, err := storage.client.ListFileNames(threadIndex, filePath, true, true)
 		if err != nil {
 			return err
 		}
@@ -116,7 +110,7 @@ func (storage *B2Storage) DeleteFile(threadIndex int, filePath string) (err erro
 
 			toBeDeleted = true
 
-			err = storage.clients[threadIndex].DeleteFile(filePath, entry.FileID)
+			err = storage.client.DeleteFile(threadIndex, filePath, entry.FileID)
 			if err != nil {
 				return err
 			}
@@ -125,7 +119,7 @@ func (storage *B2Storage) DeleteFile(threadIndex int, filePath string) (err erro
 		return nil
 
 	} else {
-		entries, err := storage.clients[threadIndex].ListFileNames(filePath, true, false)
+		entries, err := storage.client.ListFileNames(threadIndex, filePath, true, false)
 		if err != nil {
 			return err
 		}
@@ -133,7 +127,7 @@ func (storage *B2Storage) DeleteFile(threadIndex int, filePath string) (err erro
 		if len(entries) == 0 {
 			return nil
 		}
-		return storage.clients[threadIndex].DeleteFile(filePath, entries[0].FileID)
+		return storage.client.DeleteFile(threadIndex, filePath, entries[0].FileID)
 	}
 }
 
@@ -160,10 +154,10 @@ func (storage *B2Storage) MoveFile(threadIndex int, from string, to string) (err
 	}
 
 	if filePath == from {
-		_, err = storage.clients[threadIndex].HideFile(from)
+		_, err = storage.client.HideFile(threadIndex, from)
 		return err
 	} else {
-		entries, err := storage.clients[threadIndex].ListFileNames(filePath, true, true)
+		entries, err := storage.client.ListFileNames(threadIndex, filePath, true, true)
 		if err != nil {
 			return err
 		}
@@ -171,7 +165,7 @@ func (storage *B2Storage) MoveFile(threadIndex int, from string, to string) (err
 			return nil
 		}
 
-		return storage.clients[threadIndex].DeleteFile(filePath, entries[0].FileID)
+		return storage.client.DeleteFile(threadIndex, filePath, entries[0].FileID)
 	}
 }
 
@@ -188,7 +182,7 @@ func (storage *B2Storage) GetFileInfo(threadIndex int, filePath string) (exist b
 		filePath = filePath[:len(filePath)-len(".fsl")]
 	}
 
-	entries, err := storage.clients[threadIndex].ListFileNames(filePath, true, isFossil)
+	entries, err := storage.client.ListFileNames(threadIndex, filePath, true, isFossil)
 	if err != nil {
 		return false, false, 0, err
 	}
@@ -211,21 +205,21 @@ func (storage *B2Storage) GetFileInfo(threadIndex int, filePath string) (exist b
 func (storage *B2Storage) DownloadFile(threadIndex int, filePath string, chunk *Chunk) (err error) {
 
 	filePath = strings.Replace(filePath, " ", "%20", -1)
-	readCloser, _, err := storage.clients[threadIndex].DownloadFile(filePath)
+	readCloser, _, err := storage.client.DownloadFile(threadIndex, filePath)
 	if err != nil {
 		return err
 	}
 
 	defer readCloser.Close()
 
-	_, err = RateLimitedCopy(chunk, readCloser, storage.DownloadRateLimit/len(storage.clients))
+	_, err = RateLimitedCopy(chunk, readCloser, storage.DownloadRateLimit/storage.client.Threads)
 	return err
 }
 
 // UploadFile writes 'content' to the file at 'filePath'.
 func (storage *B2Storage) UploadFile(threadIndex int, filePath string, content []byte) (err error) {
 	filePath = strings.Replace(filePath, " ", "%20", -1)
-	return storage.clients[threadIndex].UploadFile(filePath, content, storage.UploadRateLimit/len(storage.clients))
+	return storage.client.UploadFile(threadIndex, filePath, content, storage.UploadRateLimit/storage.client.Threads)
 }
 
 // If a local snapshot cache is needed for the storage to avoid downloading/uploading chunks too often when
@@ -243,7 +237,5 @@ func (storage *B2Storage) IsFastListing() bool { return true }
 
 // Enable the test mode.
 func (storage *B2Storage) EnableTestMode() {
-	for _, client := range storage.clients {
-		client.TestMode = true
-	}
+	storage.client.TestMode = true
 }

src/duplicacy_backupmanager.go

@@ -33,7 +33,7 @@ type BackupManager struct {
 	snapshotCache   *FileStorage     // for copies of chunks needed by snapshots
 
 	config *Config // contains a number of options
-	
+
 	nobackupFile string // don't backup directory when this file name is found
 }
 
@@ -65,7 +65,7 @@ func CreateBackupManager(snapshotID string, storage Storage, top string, passwor
 		SnapshotManager: snapshotManager,
 
 		config: config,
-		
+
 		nobackupFile: nobackupFile,
 	}
 
@@ -472,7 +472,7 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
 
 			uploadedModifiedFileSize := atomic.AddInt64(&uploadedModifiedFileSize, int64(chunkSize))
 
-			if IsTracing() || showStatistics {
+			if (IsTracing() || showStatistics) && totalModifiedFileSize > 0 {
 				now := time.Now().Unix()
 				if now <= startUploadingTime {
 					now = startUploadingTime + 1
@@ -807,6 +807,7 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
 				if compare == 0 {
 					i++
 					if quickMode && local.IsSameAs(entry) {
+						LOG_TRACE("RESTORE_SKIP", "File %s unchanged (by size and timestamp)", local.Path)
 						skipped = true
 					}
 				}
@@ -825,6 +826,7 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
 				if stat.Mode()&os.ModeSymlink != 0 {
 					isRegular, link, err := Readlink(fullPath)
 					if err == nil && link == entry.Link && !isRegular {
+						entry.RestoreMetadata(fullPath, nil, setOwner)
 						continue
 					}
 				}
@@ -837,6 +839,7 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
 				LOG_ERROR("RESTORE_SYMLINK", "Can't create symlink %s: %v", entry.Path, err)
 				return false
 			}
+			entry.RestoreMetadata(fullPath, nil, setOwner)
 			LOG_TRACE("DOWNLOAD_DONE", "Symlink %s updated", entry.Path)
 		} else if entry.IsDir() {
 			stat, err := os.Stat(fullPath)
@@ -896,7 +899,8 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
 				continue
 			}
 		} else {
-			err = os.MkdirAll(path.Dir(fullPath), 0744)
+			parent, _ := SplitDir(fullPath)
+			err = os.MkdirAll(parent, 0744)
 			if err != nil {
 				LOG_ERROR("DOWNLOAD_MKDIR", "Failed to create directory: %v", err)
 			}
@@ -979,12 +983,12 @@ type fileEncoder struct {
 	buffer         *bytes.Buffer
 }
 
-// Read reads data from the embeded buffer
+// Read reads data from the embedded buffer
 func (encoder fileEncoder) Read(data []byte) (n int, err error) {
 	return encoder.buffer.Read(data)
 }
 
-// NextFile switchs to the next file and generates its json description in the buffer.  It also takes care of
+// NextFile switches to the next file and generates its json description in the buffer.  It also takes care of
 // the ending ']' and the commas between files.
 func (encoder *fileEncoder) NextFile() (io.Reader, bool) {
 	if encoder.currentIndex == len(encoder.files) {
@@ -1124,7 +1128,7 @@ func (manager *BackupManager) UploadSnapshot(chunkMaker *ChunkMaker, uploader *C
 }
 
 // Restore downloads a file from the storage.  If 'inPlace' is false, the download file is saved first to a temporary
-// file under the .duplicacy directory and then replaces the existing one.  Otherwise, the exising file will be
+// file under the .duplicacy directory and then replaces the existing one.  Otherwise, the existing file will be
 // overwritten directly.
 func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chunkMaker *ChunkMaker, entry *Entry, top string, inPlace bool, overwrite bool,
 	showStatistics bool, totalFileSize int64, downloadedFileSize int64, startTime int64) bool {
@@ -1160,6 +1164,9 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
 	lengthMap := make(map[string]int)
 	var offset int64
 
+	// If the file is newly created (needed by sparse file optimization)
+	isNewFile := false
+
 	existingFile, err = os.Open(fullPath)
 	if err != nil {
 		if os.IsNotExist(err) {
@@ -1194,6 +1201,7 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
 					LOG_ERROR("DOWNLOAD_OPEN", "Can't reopen the initial file just created: %v", err)
 					return false
 				}
+				isNewFile = true
 			}
 		} else {
 			LOG_TRACE("DOWNLOAD_OPEN", "Can't open the existing file: %v", err)
@@ -1206,6 +1214,9 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
 		}
 	}
 
+	// The key in this map is the number of zeroes.  The value is the corresponding hash.
+	knownHashes := make(map[int]string)
+
 	fileHash := ""
 	if existingFile != nil {
 
@@ -1215,6 +1226,7 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
 			fileHasher := manager.config.NewFileHasher()
 			buffer := make([]byte, 64*1024)
 			err = nil
+			isSkipped := false
 			// We set to read one more byte so the file hash will be different if the file to be restored is a
 			// truncated portion of the existing file
 			for i := entry.StartChunk; i <= entry.EndChunk+1; i++ {
@@ -1230,6 +1242,28 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
 					chunkSize = 1 // the size of extra chunk beyond EndChunk
 				}
 				count := 0
+
+				if isNewFile {
+					if hash, found := knownHashes[chunkSize]; found {
+						// We have read the same number of zeros before, so we just retrieve the hash from the map
+						existingChunks = append(existingChunks, hash)
+						existingLengths = append(existingLengths, chunkSize)
+						offsetMap[hash] = offset
+						lengthMap[hash] = chunkSize
+						offset += int64(chunkSize)
+						isSkipped = true
+						continue
+					}
+				}
+
+				if isSkipped {
+					_, err := existingFile.Seek(offset, 0)
+					if err != nil {
+						LOG_ERROR("DOWNLOAD_SEEK", "Failed to seek to offset %d: %v", offset, err)
+					}
+					isSkipped = false
+				}
+
 				for count < chunkSize {
 					n := chunkSize - count
 					if n > cap(buffer) {
@@ -1256,12 +1290,16 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
 					offsetMap[hash] = offset
 					lengthMap[hash] = chunkSize
 					offset += int64(chunkSize)
+					if isNewFile {
+						knownHashes[chunkSize] = hash
+					}
 				}
 
 				if err == io.EOF {
 					break
 				}
 			}
+
 			fileHash = hex.EncodeToString(fileHasher.Sum(nil))
 		} else {
 			// If it is not inplace, we want to reuse any chunks in the existing file regardless their offets, so
@@ -1491,7 +1529,7 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho
 	revisionsToBeCopied []int, threads int) bool {
 
 	if !manager.config.IsCompatiableWith(otherManager.config) {
-		LOG_ERROR("CONFIG_INCOMPATIABLE", "Two storages are not compatiable for the copy operation")
+		LOG_ERROR("CONFIG_INCOMPATIBLE", "Two storages are not compatible for the copy operation")
 		return false
 	}
 
@@ -1637,7 +1675,7 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho
 	chunksToCopy := 0
 	chunksToSkip := 0
 
-	for chunkHash, _ := range chunks {
+	for chunkHash := range chunks {
 		otherChunkID := otherManager.config.GetChunkIDFromHash(chunkHash)
 		if _, found := otherChunks[otherChunkID]; found {
 			chunksToSkip++
@@ -1667,7 +1705,7 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho
 	totalSkipped := 0
 	chunkIndex := 0
 
-	for chunkHash, _ := range chunks {
+	for chunkHash := range chunks {
 		chunkIndex++
 		chunkID := manager.config.GetChunkIDFromHash(chunkHash)
 		newChunkID := otherManager.config.GetChunkIDFromHash(chunkHash)

src/duplicacy_backupmanager_test.go

@@ -246,8 +246,8 @@ func TestBackupManager(t *testing.T) {
 	backupManager.Backup(testDir+"/repository1" /*quickMode=*/, true, threads, "first", false, false, 0, false)
 	time.Sleep(time.Duration(delay) * time.Second)
 	SetDuplicacyPreferencePath(testDir + "/repository2/.duplicacy")
-	backupManager.Restore(testDir+"/repository2", threads, /*inPlace=*/false, /*quickMode=*/false, threads, /*overwrite=*/true,
-		/*deleteMode=*/false, /*setowner=*/false, /*showStatistics=*/false, /*patterns=*/ nil)
+	backupManager.Restore(testDir+"/repository2", threads /*inPlace=*/, false /*quickMode=*/, false, threads /*overwrite=*/, true,
+		/*deleteMode=*/ false /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, nil)
 
 	for _, f := range []string{"file1", "file2", "dir1/file3"} {
 		if _, err := os.Stat(testDir + "/repository2/" + f); os.IsNotExist(err) {
@@ -270,8 +270,8 @@ func TestBackupManager(t *testing.T) {
 	backupManager.Backup(testDir+"/repository1" /*quickMode=*/, true, threads, "second", false, false, 0, false)
 	time.Sleep(time.Duration(delay) * time.Second)
 	SetDuplicacyPreferencePath(testDir + "/repository2/.duplicacy")
-	backupManager.Restore(testDir+"/repository2", 2, /*inPlace=*/true, /*quickMode=*/true, threads, /*overwrite=*/true,
-		/*deleteMode=*/false, /*setowner=*/false, /*showStatistics=*/false, /*patterns=*/nil)
+	backupManager.Restore(testDir+"/repository2", 2 /*inPlace=*/, true /*quickMode=*/, true, threads /*overwrite=*/, true,
+		/*deleteMode=*/ false /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, nil)
 
 	for _, f := range []string{"file1", "file2", "dir1/file3"} {
 		hash1 := getFileHash(testDir + "/repository1/" + f)
@@ -298,8 +298,8 @@ func TestBackupManager(t *testing.T) {
 	createRandomFile(testDir+"/repository2/dir5/file5", 100)
 
 	SetDuplicacyPreferencePath(testDir + "/repository2/.duplicacy")
-	backupManager.Restore(testDir+"/repository2", 3, /*inPlace=*/true, /*quickMode=*/false, threads, /*overwrite=*/true,
-		/*deleteMode=*/true, /*setowner=*/false, /*showStatistics=*/false, /*patterns=*/nil)
+	backupManager.Restore(testDir+"/repository2", 3 /*inPlace=*/, true /*quickMode=*/, false, threads /*overwrite=*/, true,
+		/*deleteMode=*/ true /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, nil)
 
 	for _, f := range []string{"file1", "file2", "dir1/file3"} {
 		hash1 := getFileHash(testDir + "/repository1/" + f)
@@ -325,8 +325,8 @@ func TestBackupManager(t *testing.T) {
 	os.Remove(testDir + "/repository1/file2")
 	os.Remove(testDir + "/repository1/dir1/file3")
 	SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy")
-	backupManager.Restore(testDir+"/repository1", 3, /*inPlace=*/true, /*quickMode=*/false, threads, /*overwrite=*/true,
-		/*deleteMode=*/false,  /*setowner=*/false, /*showStatistics=*/false, /*patterns=*/[]string{"+file2", "+dir1/file3", "-*"})
+	backupManager.Restore(testDir+"/repository1", 3 /*inPlace=*/, true /*quickMode=*/, false, threads /*overwrite=*/, true,
+		/*deleteMode=*/ false /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, []string{"+file2", "+dir1/file3", "-*"})
 
 	for _, f := range []string{"file1", "file2", "dir1/file3"} {
 		hash1 := getFileHash(testDir + "/repository1/" + f)
@@ -343,7 +343,7 @@ func TestBackupManager(t *testing.T) {
 	backupManager.SnapshotManager.CheckSnapshots( /*snapshotID*/ "host1" /*revisions*/, []int{1, 2, 3} /*tag*/, "",
 		/*showStatistics*/ false /*showTabular*/, false /*checkFiles*/, false /*searchFossils*/, false /*resurrect*/, false)
 	backupManager.SnapshotManager.PruneSnapshots("host1", "host1" /*revisions*/, []int{1} /*tags*/, nil /*retentions*/, nil,
-		/*exhaustive*/ false /*exclusive=*/, false /*ignoredIDs*/, nil /*dryRun*/, false /*deleteOnly*/, false /*collectOnly*/, false)
+		/*exhaustive*/ false /*exclusive=*/, false /*ignoredIDs*/, nil /*dryRun*/, false /*deleteOnly*/, false /*collectOnly*/, false, 1)
 	numberOfSnapshots = backupManager.SnapshotManager.ListSnapshots( /*snapshotID*/ "host1" /*revisionsToList*/, nil /*tag*/, "" /*showFiles*/, false /*showChunks*/, false)
 	if numberOfSnapshots != 2 {
 		t.Errorf("Expected 2 snapshots but got %d", numberOfSnapshots)
@@ -352,7 +352,7 @@ func TestBackupManager(t *testing.T) {
 		/*showStatistics*/ false /*showTabular*/, false /*checkFiles*/, false /*searchFossils*/, false /*resurrect*/, false)
 	backupManager.Backup(testDir+"/repository1" /*quickMode=*/, false, threads, "fourth", false, false, 0, false)
 	backupManager.SnapshotManager.PruneSnapshots("host1", "host1" /*revisions*/, nil /*tags*/, nil /*retentions*/, nil,
-		/*exhaustive*/ false /*exclusive=*/, true /*ignoredIDs*/, nil /*dryRun*/, false /*deleteOnly*/, false /*collectOnly*/, false)
+		/*exhaustive*/ false /*exclusive=*/, true /*ignoredIDs*/, nil /*dryRun*/, false /*deleteOnly*/, false /*collectOnly*/, false, 1)
 	numberOfSnapshots = backupManager.SnapshotManager.ListSnapshots( /*snapshotID*/ "host1" /*revisionsToList*/, nil /*tag*/, "" /*showFiles*/, false /*showChunks*/, false)
 	if numberOfSnapshots != 3 {
 		t.Errorf("Expected 3 snapshots but got %d", numberOfSnapshots)

src/duplicacy_chunk.go

@@ -5,18 +5,18 @@
 package duplicacy
 
 import (
-	"os"
 	"bytes"
 	"compress/zlib"
 	"crypto/aes"
 	"crypto/cipher"
-	"crypto/rand"
 	"crypto/hmac"
+	"crypto/rand"
 	"crypto/sha256"
 	"encoding/hex"
 	"fmt"
 	"hash"
 	"io"
+	"os"
 	"runtime"
 
 	"github.com/bkaradzic/go-lz4"
@@ -250,10 +250,7 @@ func (chunk *Chunk) Encrypt(encryptionKey []byte, derivationKey string) (err err
 	// PKCS7 is used.  Compressed chunk sizes leaks information about the original chunks so we want the padding sizes
 	// to be the maximum allowed by PKCS7
 	dataLength := encryptedBuffer.Len() - offset
-	paddingLength := dataLength % 256
-	if paddingLength == 0 {
-		paddingLength = 256
-	}
+	paddingLength := 256 - dataLength%256
 
 	encryptedBuffer.Write(bytes.Repeat([]byte{byte(paddingLength)}, paddingLength))
 	encryptedBuffer.Write(bytes.Repeat([]byte{0}, gcm.Overhead()))
@@ -270,7 +267,7 @@ func (chunk *Chunk) Encrypt(encryptionKey []byte, derivationKey string) (err err
 
 }
 
-// This is to ensure compability with Vertical Backup, which still uses HMAC-SHA256 (instead of HMAC-BLAKE2) to
+// This is to ensure compatibility with Vertical Backup, which still uses HMAC-SHA256 (instead of HMAC-BLAKE2) to
 // derive the key used to encrypt/decrypt files and chunks.
 
 var DecryptWithHMACSHA256 = false
@@ -345,7 +342,6 @@ func (chunk *Chunk) Decrypt(encryptionKey []byte, derivationKey string) (err err
 			return err
 		}
 
-
 		paddingLength := int(decryptedBytes[len(decryptedBytes)-1])
 		if paddingLength == 0 {
 			paddingLength = 256

src/duplicacy_chunk_test.go

@@ -22,6 +22,8 @@ func TestChunk(t *testing.T) {
 	config.CompressionLevel = DEFAULT_COMPRESSION_LEVEL
 	maxSize := 1000000
 
+	remainderLength := -1
+
 	for i := 0; i < 500; i++ {
 
 		size := rand.Int() % maxSize
@@ -44,6 +46,12 @@ func TestChunk(t *testing.T) {
 		encryptedData := make([]byte, chunk.GetLength())
 		copy(encryptedData, chunk.GetBytes())
 
+		if remainderLength == -1 {
+			remainderLength = len(encryptedData) % 256
+		} else if len(encryptedData)%256 != remainderLength {
+			t.Errorf("Incorrect padding size")
+		}
+
 		chunk.Reset(false)
 		chunk.Write(encryptedData)
 		err = chunk.Decrypt(key, "")
@@ -63,7 +71,7 @@ func TestChunk(t *testing.T) {
 		}
 
 		if bytes.Compare(plainData, decryptedData) != 0 {
-			t.Logf("orginal length: %d, decrypted length: %d", len(plainData), len(decryptedData))
+			t.Logf("Original length: %d, decrypted length: %d", len(plainData), len(decryptedData))
 			t.Errorf("Original data:\n%x\nDecrypted data:\n%x\n", plainData, decryptedData)
 		}
 

src/duplicacy_chunkdownloader.go

@@ -178,7 +178,7 @@ func (downloader *ChunkDownloader) Reclaim(chunkIndex int) {
 		return
 	}
 
-	for i, _ := range downloader.completedTasks {
+	for i := range downloader.completedTasks {
 		if i < chunkIndex && downloader.taskList[i].chunk != nil {
 			downloader.config.PutChunk(downloader.taskList[i].chunk)
 			downloader.taskList[i].chunk = nil
@@ -253,7 +253,7 @@ func (downloader *ChunkDownloader) Stop() {
 		downloader.numberOfDownloadingChunks--
 	}
 
-	for i, _ := range downloader.completedTasks {
+	for i := range downloader.completedTasks {
 		downloader.config.PutChunk(downloader.taskList[i].chunk)
 		downloader.taskList[i].chunk = nil
 		downloader.numberOfActiveChunks--

src/duplicacy_chunkoperator.go

@@ -0,0 +1,209 @@
+// Copyright (c) Acrosync LLC. All rights reserved.
+// Free for personal use and commercial trial
+// Commercial use requires per-user licenses available from https://duplicacy.com
+
+package duplicacy
+
+import (
+	"sync"
+	"sync/atomic"
+	"time"
+)
+
+// These are operations that ChunkOperator will perform.
+const (
+	ChunkOperationFind      = 0
+	ChunkOperationDelete    = 1
+	ChunkOperationFossilize = 2
+	ChunkOperationResurrect = 3
+)
+
+// ChunkOperatorTask is used to pass parameters for different kinds of chunk operations.
+type ChunkOperatorTask struct {
+	operation int    // The type of operation
+	chunkID   string // The chunk id
+	filePath  string // The path of the chunk file; it may be empty
+}
+
+// ChunkOperator is capable of performing multi-threaded operations on chunks.
+type ChunkOperator struct {
+	numberOfActiveTasks int64                  // The number of chunks that are being operated on
+	storage             Storage                // This storage
+	threads             int                    // Number of threads
+	taskQueue           chan ChunkOperatorTask // Operating goroutines are waiting on this channel for input
+	stopChannel         chan bool              // Used to stop all the goroutines
+
+	fossils     []string    // For fossilize operation, the paths of the fossils are stored in this slice
+	fossilsLock *sync.Mutex // The lock for 'fossils'
+}
+
+// CreateChunkOperator creates a new ChunkOperator.
+func CreateChunkOperator(storage Storage, threads int) *ChunkOperator {
+	operator := &ChunkOperator{
+		storage: storage,
+		threads: threads,
+
+		taskQueue:   make(chan ChunkOperatorTask, threads*4),
+		stopChannel: make(chan bool),
+
+		fossils:     make([]string, 0),
+		fossilsLock: &sync.Mutex{},
+	}
+
+	// Start the operator goroutines
+	for i := 0; i < operator.threads; i++ {
+		go func(threadIndex int) {
+			defer CatchLogException()
+			for {
+				select {
+				case task := <-operator.taskQueue:
+					operator.Run(threadIndex, task)
+				case <-operator.stopChannel:
+					return
+				}
+			}
+		}(i)
+	}
+
+	return operator
+}
+
+func (operator *ChunkOperator) Stop() {
+	if atomic.LoadInt64(&operator.numberOfActiveTasks) < 0 {
+		return
+	}
+
+	for atomic.LoadInt64(&operator.numberOfActiveTasks) > 0 {
+		time.Sleep(100 * time.Millisecond)
+	}
+	for i := 0; i < operator.threads; i++ {
+		operator.stopChannel <- false
+	}
+
+	// Assign -1 to numberOfActiveTasks so Stop() can be called multiple times
+	atomic.AddInt64(&operator.numberOfActiveTasks, int64(-1))
+}
+
+func (operator *ChunkOperator) AddTask(operation int, chunkID string, filePath string) {
+
+	task := ChunkOperatorTask{
+		operation: operation,
+		chunkID:   chunkID,
+		filePath:  filePath,
+	}
+	operator.taskQueue <- task
+	atomic.AddInt64(&operator.numberOfActiveTasks, int64(1))
+}
+
+func (operator *ChunkOperator) Find(chunkID string) {
+	operator.AddTask(ChunkOperationFind, chunkID, "")
+}
+
+func (operator *ChunkOperator) Delete(chunkID string, filePath string) {
+	operator.AddTask(ChunkOperationDelete, chunkID, filePath)
+}
+
+func (operator *ChunkOperator) Fossilize(chunkID string, filePath string) {
+	operator.AddTask(ChunkOperationFossilize, chunkID, filePath)
+}
+
+func (operator *ChunkOperator) Resurrect(chunkID string, filePath string) {
+	operator.AddTask(ChunkOperationResurrect, chunkID, filePath)
+}
+
+func (operator *ChunkOperator) Run(threadIndex int, task ChunkOperatorTask) {
+	defer func() {
+		atomic.AddInt64(&operator.numberOfActiveTasks, int64(-1))
+	}()
+
+	// task.filePath may be empty.  If so, find the chunk first.
+	if task.operation == ChunkOperationDelete || task.operation == ChunkOperationFossilize {
+		if task.filePath == "" {
+			filePath, exist, _, err := operator.storage.FindChunk(threadIndex, task.chunkID, false)
+			if err != nil {
+				LOG_ERROR("CHUNK_FIND", "Failed to locate the path for the chunk %s: %v", task.chunkID, err)
+				return
+			} else if !exist {
+				if task.operation == ChunkOperationDelete {
+					LOG_WARN("CHUNK_FIND", "Chunk %s does not exist in the storage", task.chunkID)
+					return
+				}
+
+				fossilPath, exist, _, _ := operator.storage.FindChunk(threadIndex, task.chunkID, true)
+				if exist {
+					LOG_WARN("CHUNK_FOSSILIZE", "Chunk %s is already a fossil", task.chunkID)
+					operator.fossilsLock.Lock()
+					operator.fossils = append(operator.fossils, fossilPath)
+					operator.fossilsLock.Unlock()
+				} else {
+					LOG_ERROR("CHUNK_FIND", "Chunk %s does not exist in the storage", task.chunkID)
+				}
+				return
+			}
+			task.filePath = filePath
+		}
+	}
+
+	if task.operation == ChunkOperationFind {
+		_, exist, _, err := operator.storage.FindChunk(threadIndex, task.chunkID, false)
+		if err != nil {
+			LOG_ERROR("CHUNK_FIND", "Failed to locate the path for the chunk %s: %v", task.chunkID, err)
+		} else if !exist {
+			LOG_ERROR("CHUNK_FIND", "Chunk %s does not exist in the storage", task.chunkID)
+		} else {
+			LOG_DEBUG("CHUNK_FIND", "Chunk %s exists in the storage", task.chunkID)
+		}
+	} else if task.operation == ChunkOperationDelete {
+		err := operator.storage.DeleteFile(threadIndex, task.filePath)
+		if err != nil {
+			LOG_WARN("CHUNK_DELETE", "Failed to remove the file %s: %v", task.filePath, err)
+		} else {
+			if task.chunkID != "" {
+				LOG_INFO("CHUNK_DELETE", "The chunk %s has been permanently removed", task.chunkID)
+			} else {
+				LOG_INFO("CHUNK_DELETE", "Deleted file %s from the storage", task.filePath)
+			}
+		}
+	} else if task.operation == ChunkOperationFossilize {
+
+		fossilPath := task.filePath + ".fsl"
+
+		err := operator.storage.MoveFile(threadIndex, task.filePath, fossilPath)
+		if err != nil {
+			if _, exist, _, _ := operator.storage.FindChunk(threadIndex, task.chunkID, true); exist {
+				err := operator.storage.DeleteFile(threadIndex, task.filePath)
+				if err == nil {
+					LOG_TRACE("CHUNK_DELETE", "Deleted chunk file %s as the fossil already exists", task.chunkID)
+				}
+				operator.fossilsLock.Lock()
+				operator.fossils = append(operator.fossils, fossilPath)
+				operator.fossilsLock.Unlock()
+			} else {
+				LOG_ERROR("CHUNK_DELETE", "Failed to fossilize the chunk %s: %v", task.chunkID, err)
+			}
+		} else {
+			LOG_TRACE("CHUNK_FOSSILIZE", "The chunk %s has been marked as a fossil", task.chunkID)
+			operator.fossilsLock.Lock()
+			operator.fossils = append(operator.fossils, fossilPath)
+			operator.fossilsLock.Unlock()
+		}
+	} else if task.operation == ChunkOperationResurrect {
+		chunkPath, exist, _, err := operator.storage.FindChunk(threadIndex, task.chunkID, false)
+		if err != nil {
+			LOG_ERROR("CHUNK_FIND", "Failed to locate the path for the chunk %s: %v", task.chunkID, err)
+		}
+
+		if exist {
+			operator.storage.DeleteFile(threadIndex, task.filePath)
+			LOG_INFO("FOSSIL_RESURRECT", "The chunk %s already exists", task.chunkID)
+		} else {
+			err := operator.storage.MoveFile(threadIndex, task.filePath, chunkPath)
+			if err != nil {
+				LOG_ERROR("FOSSIL_RESURRECT", "Failed to resurrect the chunk %s from the fossil %s: %v",
+					task.chunkID, task.filePath, err)
+			} else {
+				LOG_INFO("FOSSIL_RESURRECT", "The chunk %s has been resurrected", task.filePath)
+			}
+		}
+	}
+}

src/duplicacy_entry.go

@@ -22,6 +22,9 @@ import (
 var DUPLICACY_DIRECTORY = ".duplicacy"
 var DUPLICACY_FILE = ".duplicacy"
 
+// Mask for file permission bits
+var fileModeMask = os.ModePerm | os.ModeSetuid | os.ModeSetgid | os.ModeSticky
+
 // Regex for matching 'StartChunk:StartOffset:EndChunk:EndOffset'
 var contentRegex = regexp.MustCompile(`^([0-9]+):([0-9]+):([0-9]+):([0-9]+)`)
 
@@ -269,7 +272,7 @@ func (entry *Entry) IsLink() bool {
 }
 
 func (entry *Entry) GetPermissions() os.FileMode {
-	return os.FileMode(entry.Mode) & os.ModePerm
+	return os.FileMode(entry.Mode) & fileModeMask
 }
 
 func (entry *Entry) IsSameAs(other *Entry) bool {
@@ -289,7 +292,7 @@ func (entry *Entry) String(maxSizeDigits int) string {
 func (entry *Entry) RestoreMetadata(fullPath string, fileInfo *os.FileInfo, setOwner bool) bool {
 
 	if fileInfo == nil {
-		stat, err := os.Stat(fullPath)
+		stat, err := os.Lstat(fullPath)
 		fileInfo = &stat
 		if err != nil {
 			LOG_ERROR("RESTORE_STAT", "Failed to retrieve the file info: %v", err)
@@ -297,7 +300,15 @@ func (entry *Entry) RestoreMetadata(fullPath string, fileInfo *os.FileInfo, setO
 		}
 	}
 
-	if (*fileInfo).Mode()&os.ModePerm != entry.GetPermissions() {
+	// Note that chown can remove setuid/setgid bits so should be called before chmod
+	if setOwner {
+		if !SetOwner(fullPath, entry, fileInfo) {
+			return false
+		}
+	}
+
+	// Only set the permission if the file is not a symlink
+	if !entry.IsLink() && (*fileInfo).Mode()&fileModeMask != entry.GetPermissions() {
 		err := os.Chmod(fullPath, entry.GetPermissions())
 		if err != nil {
 			LOG_ERROR("RESTORE_CHMOD", "Failed to set the file permissions: %v", err)
@@ -305,7 +316,8 @@ func (entry *Entry) RestoreMetadata(fullPath string, fileInfo *os.FileInfo, setO
 		}
 	}
 
-	if (*fileInfo).ModTime().Unix() != entry.Time {
+	// Only set the time if the file is not a symlink
+	if !entry.IsLink() && (*fileInfo).ModTime().Unix() != entry.Time {
 		modifiedTime := time.Unix(entry.Time, 0)
 		err := os.Chtimes(fullPath, modifiedTime, modifiedTime)
 		if err != nil {
@@ -318,11 +330,7 @@ func (entry *Entry) RestoreMetadata(fullPath string, fileInfo *os.FileInfo, setO
 		entry.SetAttributesToFile(fullPath)
 	}
 
-	if setOwner {
-		return SetOwner(fullPath, entry, fileInfo)
-	} else {
-		return true
-	}
+	return true
 }
 
 // Return -1 if 'left' should appear before 'right', 1 if opposite, and 0 if they are the same.
@@ -448,10 +456,10 @@ func ListEntries(top string, path string, fileList *[]*Entry, patterns []string,
 	if err != nil {
 		return directoryList, nil, err
 	}
-	
+
 	// This binary search works because ioutil.ReadDir returns files sorted by Name() by default
 	if nobackupFile != "" {
-		ii := sort.Search(len(files), func(ii int) bool { return strings.Compare(files[ii].Name(), nobackupFile) >= 0})
+		ii := sort.Search(len(files), func(ii int) bool { return strings.Compare(files[ii].Name(), nobackupFile) >= 0 })
 		if ii < len(files) && files[ii].Name() == nobackupFile {
 			LOG_DEBUG("LIST_NOBACKUP", "%s is excluded due to nobackup file", path)
 			return directoryList, skippedFiles, nil

src/duplicacy_filereader.go

@@ -34,7 +34,7 @@ func CreateFileReader(top string, files []*Entry) *FileReader {
 	return reader
 }
 
-// NextFile switchs to the next file in the file reader.
+// NextFile switches to the next file in the file reader.
 func (reader *FileReader) NextFile() bool {
 
 	if reader.CurrentFile != nil {

src/duplicacy_filestorage.go

@@ -165,7 +165,7 @@ func (storage *FileStorage) UploadFile(threadIndex int, filePath string, content
 			}
 		} else {
 			if !stat.IsDir() {
-				fmt.Errorf("The path %s is not a directory", dir)
+				return fmt.Errorf("The path %s is not a directory", dir)
 			}
 		}
 	}
@@ -190,7 +190,16 @@ func (storage *FileStorage) UploadFile(threadIndex int, filePath string, content
 		return err
 	}
 
-	file.Close()
+	err = file.Sync()
+	if err != nil {
+		file.Close()
+		return err
+	}
+
+	err = file.Close()
+	if err != nil {
+		return err
+	}
 
 	err = os.Rename(temporaryFile, fullPath)
 	if err != nil {

src/duplicacy_gcdstorage.go

@@ -25,7 +25,7 @@ import (
 )
 
 var (
-	GCDFileMimeType = "application/octet-stream"
+	GCDFileMimeType      = "application/octet-stream"
 	GCDDirectoryMimeType = "application/vnd.google-apps.folder"
 )
 
@@ -33,7 +33,7 @@ type GCDStorage struct {
 	StorageBase
 
 	service     *drive.Service
-	idCache     map[string]string  // only directories are saved in this cache
+	idCache     map[string]string // only directories are saved in this cache
 	idCacheLock sync.Mutex
 	backoffs    []int // desired backoff time in seconds for each thread
 	attempts    []int // number of failed attempts since last success for each thread
@@ -78,6 +78,10 @@ func (storage *GCDStorage) shouldRetry(threadIndex int, err error) (bool, error)
 			// User Rate Limit Exceeded
 			message = e.Message
 			retry = true
+		} else if e.Code == 408 {
+			// Request timeout
+			message = e.Message
+			retry = true
 		} else if e.Code == 401 {
 			// Only retry on authorization error when storage has been connected before
 			if storage.isConnected {
@@ -291,7 +295,7 @@ func (storage *GCDStorage) getIDFromPath(threadIndex int, filePath string, creat
 		} else if isDir {
 			storage.savePathID(current, fileID)
 		}
-		if i != len(names) - 1 && !isDir {
+		if i != len(names)-1 && !isDir {
 			return "", fmt.Errorf("Path '%s' is not a directory", current)
 		}
 	}
@@ -386,11 +390,11 @@ func (storage *GCDStorage) ListFiles(threadIndex int, dir string) ([]string, []i
 		subDirs := []string{}
 
 		for _, file := range files {
-			storage.savePathID("snapshots/" + file.Name, file.Id)
-			subDirs = append(subDirs, file.Name + "/")
+			storage.savePathID("snapshots/"+file.Name, file.Id)
+			subDirs = append(subDirs, file.Name+"/")
 		}
 		return subDirs, nil, nil
-	} else if strings.HasPrefix(dir, "snapshots/") {
+	} else if strings.HasPrefix(dir, "snapshots/") || strings.HasPrefix(dir, "benchmark") {
 		pathID, err := storage.getIDFromPath(threadIndex, dir, false)
 		if err != nil {
 			return nil, nil, err
@@ -438,8 +442,8 @@ func (storage *GCDStorage) ListFiles(threadIndex int, dir string) ([]string, []i
 					files = append(files, name)
 					sizes = append(sizes, entry.Size)
 				} else {
-					parents = append(parents, parent+ "/" + entry.Name)
-					storage.savePathID(parent + "/" + entry.Name, entry.Id)
+					parents = append(parents, parent+"/"+entry.Name)
+					storage.savePathID(parent+"/"+entry.Name, entry.Id)
 				}
 			}
 		}
@@ -624,13 +628,22 @@ func (storage *GCDStorage) DownloadFile(threadIndex int, filePath string, chunk
 	var response *http.Response
 
 	for {
-		response, err = storage.service.Files.Get(fileID).Download()
-		if retry, err := storage.shouldRetry(threadIndex, err); err == nil && !retry {
+		// AcknowledgeAbuse(true) lets the download proceed even if GCD thinks that it contains malware.
+		// TODO: Should this prompt the user or log a warning?
+		req := storage.service.Files.Get(fileID)
+		if e, ok := err.(*googleapi.Error); ok {
+			if strings.Contains(err.Error(), "cannotDownloadAbusiveFile") || len(e.Errors) > 0 && e.Errors[0].Reason == "cannotDownloadAbusiveFile" {
+				LOG_WARN("GCD_STORAGE", "%s is marked as abusive, will download anyway.", filePath)
+				req = req.AcknowledgeAbuse(true)
+			}
+		}
+		response, err = req.Download()
+		if retry, retry_err := storage.shouldRetry(threadIndex, err); retry_err == nil && !retry {
 			break
 		} else if retry {
 			continue
 		} else {
-			return err
+			return retry_err
 		}
 	}
 

src/duplicacy_hubicstorage.go

@@ -113,7 +113,7 @@ func (storage *HubicStorage) ListFiles(threadIndex int, dir string) ([]string, [
 
 		for _, entry := range entries {
 			if entry.Type == "application/directory" {
-				files = append(files,  entry.Name + "/")
+				files = append(files, entry.Name+"/")
 				sizes = append(sizes, 0)
 			} else {
 				files = append(files, entry.Name)

src/duplicacy_keyring_windows.go

@@ -136,6 +136,16 @@ func keyringSet(key string, value string) bool {
 	if value == "" {
 		keyring[key] = nil
 	} else {
+
+		// Check if the value to be set is the same as the existing one
+		existingEncryptedValue := keyring[key]
+		if len(existingEncryptedValue) > 0 {
+			existingValue, err := keyringDecrypt(existingEncryptedValue)
+			if err == nil && string(existingValue) == value {
+				return true
+			}
+		}
+
 		encryptedValue, err := keyringEncrypt([]byte(value))
 		if err != nil {
 			LOG_DEBUG("KEYRING_ENCRYPT", "Failed to encrypt the value: %v", err)

src/duplicacy_log.go

@@ -129,7 +129,7 @@ func logf(level int, logID string, format string, v ...interface{}) {
 	// fmt.Printf("%s %s %s %s\n", now.Format("2006-01-02 15:04:05.000"), getLevelName(level), logID, message)
 
 	if testingT != nil {
-		if level < WARN {
+		if level <= WARN {
 			if level >= loggingLevel {
 				testingT.Logf("%s %s %s %s\n",
 					now.Format("2006-01-02 15:04:05.000"), getLevelName(level), logID, message)

src/duplicacy_onestorage.go

@@ -97,7 +97,7 @@ func (storage *OneDriveStorage) ListFiles(threadIndex int, dir string) ([]string
 			}
 		}
 		return subDirs, nil, nil
-	} else if strings.HasPrefix(dir, "snapshots/") {
+	} else if strings.HasPrefix(dir, "snapshots/") || strings.HasPrefix(dir, "benchmark") {
 		entries, err := storage.client.ListEntries(storage.storageDir + "/" + dir)
 		if err != nil {
 			return nil, nil, err

src/duplicacy_s3storage.go

@@ -210,7 +210,7 @@ func (storage *S3Storage) DownloadFile(threadIndex int, filePath string, chunk *
 
 	defer output.Body.Close()
 
-	_, err = RateLimitedCopy(chunk, output.Body, storage.DownloadRateLimit/len(storage.bucket))
+	_, err = RateLimitedCopy(chunk, output.Body, storage.DownloadRateLimit/storage.numberOfThreads)
 	return err
 
 }
@@ -225,7 +225,7 @@ func (storage *S3Storage) UploadFile(threadIndex int, filePath string, content [
 			Bucket:      aws.String(storage.bucket),
 			Key:         aws.String(storage.storageDir + filePath),
 			ACL:         aws.String(s3.ObjectCannedACLPrivate),
-			Body:        CreateRateLimitedReader(content, storage.UploadRateLimit/len(storage.bucket)),
+			Body:        CreateRateLimitedReader(content, storage.UploadRateLimit/storage.numberOfThreads),
 			ContentType: aws.String("application/duplicacy"),
 		}
 
@@ -237,8 +237,6 @@ func (storage *S3Storage) UploadFile(threadIndex int, filePath string, content [
 		LOG_INFO("S3_RETRY", "Retrying on %s: %v", reflect.TypeOf(err), err)
 		attempts += 1
 	}
-
-	return err
 }
 
 // If a local snapshot cache is needed for the storage to avoid downloading/uploading chunks too often when

src/duplicacy_sftpstorage.go

@@ -14,6 +14,7 @@ import (
 	"runtime"
 	"strings"
 	"time"
+	"sync"
 
 	"github.com/pkg/sftp"
 	"golang.org/x/crypto/ssh"
@@ -23,9 +24,13 @@ type SFTPStorage struct {
 	StorageBase
 
 	client          *sftp.Client
+	clientLock      sync.Mutex
 	minimumNesting  int // The minimum level of directories to dive into before searching for the chunk file.
 	storageDir      string
 	numberOfThreads int
+	numberOfTries   int
+	serverAddress   string
+	sftpConfig      *ssh.ClientConfig
 }
 
 func CreateSFTPStorageWithPassword(server string, port int, username string, storageDir string,
@@ -86,6 +91,9 @@ func CreateSFTPStorage(server string, port int, username string, storageDir stri
 		storageDir:      storageDir,
 		minimumNesting:  minimumNesting,
 		numberOfThreads: threads,
+		numberOfTries:   6,
+		serverAddress:   serverAddress,
+		sftpConfig:      sftpConfig,
 	}
 
 	// Random number fo generating the temporary chunk file suffix.
@@ -99,13 +107,60 @@ func CreateSFTPStorage(server string, port int, username string, storageDir stri
 }
 
 func CloseSFTPStorage(storage *SFTPStorage) {
-	storage.client.Close()
+	if storage.client != nil {
+		storage.client.Close()
+		storage.client = nil
+	}
 }
 
+func (storage *SFTPStorage) getSFTPClient() *sftp.Client {
+	storage.clientLock.Lock()
+	defer storage.clientLock.Unlock()
+	return storage.client
+}
+
+func (storage *SFTPStorage) retry(f func () error) error {
+	delay := time.Second
+	for i := 0;; i++ {
+		err := f()
+		if err != nil && strings.Contains(err.Error(), "EOF") && i < storage.numberOfTries {
+			LOG_WARN("SFTP_RETRY", "Encountered an error (%v); retry after %d second(s)", err, delay/time.Second)
+			time.Sleep(delay)
+			delay *= 2
+
+			storage.clientLock.Lock()
+			if storage.client != nil {
+				storage.client.Close()
+				storage.client = nil
+			}
+
+			connection, err := ssh.Dial("tcp", storage.serverAddress, storage.sftpConfig)
+			if err != nil {
+				storage.clientLock.Unlock()
+				return err
+			}
+
+			client, err := sftp.NewClient(connection)
+			if err != nil {
+				connection.Close()
+				storage.clientLock.Unlock()
+				return err
+			}
+			storage.client = client
+			storage.clientLock.Unlock()
+			continue
+		}
+		return err
+	}
+}
 // ListFiles return the list of files and subdirectories under 'file' (non-recursively)
 func (storage *SFTPStorage) ListFiles(threadIndex int, dirPath string) (files []string, sizes []int64, err error) {
 
-	entries, err := storage.client.ReadDir(path.Join(storage.storageDir, dirPath))
+	var entries []os.FileInfo
+	err = storage.retry(func() error {
+		entries, err = storage.getSFTPClient().ReadDir(path.Join(storage.storageDir, dirPath))
+		return err
+	})
 	if err != nil {
 		return nil, nil, err
 	}
@@ -126,7 +181,11 @@ func (storage *SFTPStorage) ListFiles(threadIndex int, dirPath string) (files []
 // DeleteFile deletes the file or directory at 'filePath'.
 func (storage *SFTPStorage) DeleteFile(threadIndex int, filePath string) (err error) {
 	fullPath := path.Join(storage.storageDir, filePath)
-	fileInfo, err := storage.client.Stat(fullPath)
+	var fileInfo os.FileInfo
+	err = storage.retry(func() error {
+		fileInfo, err = storage.getSFTPClient().Stat(fullPath)
+		return err
+	})
 	if err != nil {
 		if os.IsNotExist(err) {
 			LOG_TRACE("SFTP_STORAGE", "File %s has disappeared before deletion", filePath)
@@ -137,33 +196,47 @@ func (storage *SFTPStorage) DeleteFile(threadIndex int, filePath string) (err er
 	if fileInfo == nil {
 		return nil
 	}
-	return storage.client.Remove(path.Join(storage.storageDir, filePath))
+	return storage.retry(func() error { return storage.getSFTPClient().Remove(path.Join(storage.storageDir, filePath)) })
 }
 
 // MoveFile renames the file.
 func (storage *SFTPStorage) MoveFile(threadIndex int, from string, to string) (err error) {
 	toPath := path.Join(storage.storageDir, to)
-	fileInfo, err := storage.client.Stat(toPath)
+	var fileInfo os.FileInfo
+	err = storage.retry(func() error {
+		fileInfo, err = storage.getSFTPClient().Stat(toPath)
+		return err
+	})
 	if fileInfo != nil {
 		return fmt.Errorf("The destination file %s already exists", toPath)
 	}
-	return storage.client.Rename(path.Join(storage.storageDir, from),
-		path.Join(storage.storageDir, to))
+	err = storage.retry(func() error { return storage.getSFTPClient().Rename(path.Join(storage.storageDir, from),
+		path.Join(storage.storageDir, to)) })
+	return err
 }
 
 // CreateDirectory creates a new directory.
 func (storage *SFTPStorage) CreateDirectory(threadIndex int, dirPath string) (err error) {
 	fullPath := path.Join(storage.storageDir, dirPath)
-	fileInfo, err := storage.client.Stat(fullPath)
+	var fileInfo os.FileInfo
+	err = storage.retry(func() error {
+		fileInfo, err = storage.getSFTPClient().Stat(fullPath)
+		return err
+	})
 	if fileInfo != nil && fileInfo.IsDir() {
 		return nil
 	}
-	return storage.client.Mkdir(path.Join(storage.storageDir, dirPath))
+	return storage.retry(func() error { return storage.getSFTPClient().Mkdir(path.Join(storage.storageDir, dirPath)) })
 }
 
 // GetFileInfo returns the information about the file or directory at 'filePath'.
 func (storage *SFTPStorage) GetFileInfo(threadIndex int, filePath string) (exist bool, isDir bool, size int64, err error) {
-	fileInfo, err := storage.client.Stat(path.Join(storage.storageDir, filePath))
+	var fileInfo os.FileInfo
+	err = storage.retry(func() error {
+		fileInfo, err = storage.getSFTPClient().Stat(path.Join(storage.storageDir, filePath))
+		return err
+	})
+
 	if err != nil {
 		if os.IsNotExist(err) {
 			return false, false, 0, nil
@@ -181,18 +254,19 @@ func (storage *SFTPStorage) GetFileInfo(threadIndex int, filePath string) (exist
 
 // DownloadFile reads the file at 'filePath' into the chunk.
 func (storage *SFTPStorage) DownloadFile(threadIndex int, filePath string, chunk *Chunk) (err error) {
-	file, err := storage.client.Open(path.Join(storage.storageDir, filePath))
+	return storage.retry(func() error {
+		file, err := storage.getSFTPClient().Open(path.Join(storage.storageDir, filePath))
 
-	if err != nil {
-		return err
-	}
+		if err != nil {
+			return err
+		}
 
-	defer file.Close()
-	if _, err = RateLimitedCopy(chunk, file, storage.DownloadRateLimit/storage.numberOfThreads); err != nil {
-		return err
-	}
-
-	return nil
+		defer file.Close()
+		if _, err = RateLimitedCopy(chunk, file, storage.DownloadRateLimit/storage.numberOfThreads); err != nil {
+			return err
+		}
+		return nil
+	})
 }
 
 // UploadFile writes 'content' to the file at 'filePath'.
@@ -203,20 +277,26 @@ func (storage *SFTPStorage) UploadFile(threadIndex int, filePath string, content
 	dirs := strings.Split(filePath, "/")
 	if len(dirs) > 1 {
 		fullDir := path.Dir(fullPath)
-		_, err := storage.client.Stat(fullDir)
+		err = storage.retry(func() error {
+			_, err := storage.getSFTPClient().Stat(fullDir)
+			return err
+		})
 		if err != nil {
 			// The error may be caused by a non-existent fullDir, or a broken connection.  In either case,
 			// we just assume it is the former because there isn't a way to tell which is the case.
-			for i, _ := range dirs[1 : len(dirs)-1] {
+			for i := range dirs[1 : len(dirs)-1] {
 				subDir := path.Join(storage.storageDir, path.Join(dirs[0:i+2]...))
 				// We don't check the error; just keep going blindly but always store the last err
-				err = storage.client.Mkdir(subDir)
+				err = storage.getSFTPClient().Mkdir(subDir)
 			}
 
 			// If there is an error creating the dirs, we check fullDir one more time, because another thread
 			// may happen to create the same fullDir ahead of this thread
 			if err != nil {
-				_, err := storage.client.Stat(fullDir)
+				err = storage.retry(func() error {
+					_, err := storage.getSFTPClient().Stat(fullDir)
+					return err
+				})
 				if err != nil {
 					return err
 				}
@@ -224,39 +304,41 @@ func (storage *SFTPStorage) UploadFile(threadIndex int, filePath string, content
 		}
 	}
 
-	letters := "abcdefghijklmnopqrstuvwxyz"
-	suffix := make([]byte, 8)
-	for i := range suffix {
-		suffix[i] = letters[rand.Intn(len(letters))]
-	}
+	return storage.retry(func() error {
 
-	temporaryFile := fullPath + "." + string(suffix) + ".tmp"
-
-	file, err := storage.client.OpenFile(temporaryFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC)
-	if err != nil {
-		return err
-	}
-
-	reader := CreateRateLimitedReader(content, storage.UploadRateLimit/storage.numberOfThreads)
-	_, err = io.Copy(file, reader)
-	if err != nil {
-		file.Close()
-		return err
-	}
-	file.Close()
-
-	err = storage.client.Rename(temporaryFile, fullPath)
-	if err != nil {
-
-		if _, err = storage.client.Stat(fullPath); err == nil {
-			storage.client.Remove(temporaryFile)
-			return nil
-		} else {
-			return fmt.Errorf("Uploaded file but failed to store it at %s: %v", fullPath, err)
+		letters := "abcdefghijklmnopqrstuvwxyz"
+		suffix := make([]byte, 8)
+		for i := range suffix {
+			suffix[i] = letters[rand.Intn(len(letters))]
 		}
-	}
 
-	return nil
+		temporaryFile := fullPath + "." + string(suffix) + ".tmp"
+
+		file, err := storage.getSFTPClient().OpenFile(temporaryFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC)
+		if err != nil {
+			return err
+		}
+
+		reader := CreateRateLimitedReader(content, storage.UploadRateLimit/storage.numberOfThreads)
+		_, err = io.Copy(file, reader)
+		if err != nil {
+			file.Close()
+			return err
+		}
+		file.Close()
+
+		err = storage.getSFTPClient().Rename(temporaryFile, fullPath)
+		if err != nil {
+			if _, err = storage.getSFTPClient().Stat(fullPath); err == nil {
+				storage.getSFTPClient().Remove(temporaryFile)
+				return nil
+			} else {
+				return fmt.Errorf("Uploaded file but failed to store it at %s: %v", fullPath, err)
+			}
+		}
+
+		return nil
+	})
 }
 
 // If a local snapshot cache is needed for the storage to avoid downloading/uploading chunks too often when

src/duplicacy_shadowcopy.go

@@ -3,6 +3,7 @@
 // Commercial use requires per-user licenses available from https://duplicacy.com
 
 // +build !windows
+// +build !darwin
 
 package duplicacy
 

src/duplicacy_shadowcopy_darwin.go

@@ -0,0 +1,170 @@
+//
+// Shadow copy module for Mac OSX using APFS snapshot
+//
+//
+// This module copyright 2018 Adam Marcus (https://github.com/amarcu5)
+// and may be distributed under the same terms as Duplicacy.
+
+package duplicacy
+
+import (
+	"context"
+	"errors"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"strings"
+	"syscall"
+	"time"
+)
+
+var snapshotPath string
+var snapshotDate string
+
+// Converts char array to string
+func CharsToString(ca []int8) string {
+
+	len := len(ca)
+	ba := make([]byte, len)
+
+	for i, v := range ca {
+		ba[i] = byte(v)
+		if ba[i] == 0 {
+			len = i
+			break
+		}
+	}
+
+	return string(ba[:len])
+}
+
+// Get ID of device containing path
+func GetPathDeviceId(path string) (deviceId int32, err error) {
+
+	stat := syscall.Stat_t{}
+
+	err = syscall.Stat(path, &stat)
+	if err != nil {
+		return 0, err
+	}
+
+	return stat.Dev, nil
+}
+
+// Executes shell command with timeout and returns stdout
+func CommandWithTimeout(timeoutInSeconds int, name string, arg ...string) (output string, err error) {
+
+	ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeoutInSeconds)*time.Second)
+	defer cancel()
+
+	cmd := exec.CommandContext(ctx, name, arg...)
+	out, err := cmd.Output()
+
+	if ctx.Err() == context.DeadlineExceeded {
+		err = errors.New("Command '" + name + "' timed out")
+	}
+
+	output = string(out)
+	return output, err
+}
+
+func DeleteShadowCopy() {
+
+	if snapshotPath == "" {
+		return
+	}
+
+	err := exec.Command("/sbin/umount", "-f", snapshotPath).Run()
+	if err != nil {
+		LOG_ERROR("VSS_DELETE", "Error while unmounting snapshot")
+		return
+	}
+
+	err = exec.Command("tmutil", "deletelocalsnapshots", snapshotDate).Run()
+	if err != nil {
+		LOG_ERROR("VSS_DELETE", "Error while deleting local snapshot")
+		return
+	}
+
+	err = os.RemoveAll(snapshotPath)
+	if err != nil {
+		LOG_ERROR("VSS_DELETE", "Error while deleting temporary mount directory")
+		return
+	}
+
+	LOG_INFO("VSS_DELETE", "Shadow copy unmounted and deleted at %s", snapshotPath)
+
+	snapshotPath = ""
+}
+
+func CreateShadowCopy(top string, shadowCopy bool, timeoutInSeconds int) (shadowTop string) {
+
+	if !shadowCopy {
+		return top
+	}
+
+	// Check repository filesystem is APFS
+	stat := syscall.Statfs_t{}
+	err := syscall.Statfs(top, &stat)
+	if err != nil {
+		LOG_ERROR("VSS_INIT", "Unable to determine filesystem of repository path")
+		return top
+	}
+	if CharsToString(stat.Fstypename[:]) != "apfs" {
+		LOG_WARN("VSS_INIT", "VSS requires APFS filesystem")
+		return top
+	}
+
+	// Check path is local as tmutil snapshots will not support APFS formatted external drives
+	deviceIdLocal, err := GetPathDeviceId("/")
+	if err != nil {
+		LOG_ERROR("VSS_INIT", "Unable to get device ID of path: /")
+		return top
+	}
+	deviceIdRepository, err := GetPathDeviceId(top)
+	if err != nil {
+		LOG_ERROR("VSS_INIT", "Unable to get device ID of path: ", top)
+		return top
+	}
+	if deviceIdLocal != deviceIdRepository {
+		LOG_WARN("VSS_PATH", "VSS not supported for non-local repository path: ", top)
+		return top
+	}
+
+	if timeoutInSeconds <= 60 {
+		timeoutInSeconds = 60
+	}
+
+	// Create mount point
+	snapshotPath, err = ioutil.TempDir("/tmp/", "snp_")
+	if err != nil {
+		LOG_ERROR("VSS_CREATE", "Failed to create temporary mount directory")
+		return top
+	}
+
+	// Use tmutil to create snapshot
+	tmutilOutput, err := CommandWithTimeout(timeoutInSeconds, "tmutil", "snapshot")
+	if err != nil {
+		LOG_ERROR("VSS_CREATE", "Error while calling tmutil: ", err)
+		return top
+	}
+
+	colonPos := strings.IndexByte(tmutilOutput, ':')
+	if colonPos < 0 {
+		LOG_ERROR("VSS_CREATE", "Snapshot creation failed: ", tmutilOutput)
+		return top
+	}
+	snapshotDate = strings.TrimSpace(tmutilOutput[colonPos+1:])
+
+	// Mount snapshot as readonly and hide from GUI i.e. Finder
+	_, err = CommandWithTimeout(timeoutInSeconds,
+		"/sbin/mount", "-t", "apfs", "-o", "nobrowse,-r,-s=com.apple.TimeMachine."+snapshotDate, "/", snapshotPath)
+	if err != nil {
+		LOG_ERROR("VSS_CREATE", "Error while mounting snapshot: ", err)
+		return top
+	}
+
+	LOG_INFO("VSS_DONE", "Shadow copy created and mounted at %s", snapshotPath)
+
+	return snapshotPath + top
+}

src/duplicacy_snapshot.go

@@ -11,6 +11,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"time"
@@ -68,47 +69,7 @@ func CreateSnapshotFromDirectory(id string, top string, nobackupFile string) (sn
 
 	var patterns []string
 
-	patternFile, err := ioutil.ReadFile(path.Join(GetDuplicacyPreferencePath(), "filters"))
-	if err == nil {
-		for _, pattern := range strings.Split(string(patternFile), "\n") {
-			pattern = strings.TrimSpace(pattern)
-			if len(pattern) == 0 {
-				continue
-			}
-
-			if pattern[0] == '#' {
-				continue
-			}
-
-			if IsUnspecifiedFilter(pattern) {
-				pattern = "+" + pattern
-			}
-
-			if IsEmptyFilter(pattern) {
-				continue
-			}
-
-			if strings.HasPrefix(pattern, "i:") || strings.HasPrefix(pattern, "e:") {
-				valid, err := IsValidRegex(pattern[2:])
-				if !valid || err != nil {
-					LOG_ERROR("SNAPSHOT_FILTER", "Invalid regular expression encountered for filter: \"%s\", error: %v", pattern, err)
-				}
-			}
-
-			patterns = append(patterns, pattern)
-		}
-
-		LOG_DEBUG("REGEX_DEBUG", "There are %d compiled regular expressions stored", len(RegexMap))
-
-		LOG_INFO("SNAPSHOT_FILTER", "Loaded %d include/exclude pattern(s)", len(patterns))
-
-		if IsTracing() {
-			for _, pattern := range patterns {
-				LOG_TRACE("SNAPSHOT_PATTERN", "Pattern: %s", pattern)
-			}
-		}
-
-	}
+	patterns = ProcessFilters()
 
 	directories := make([]*Entry, 0, 256)
 	directories = append(directories, CreateEntry("", 0, 0, 0))
@@ -150,6 +111,103 @@ func CreateSnapshotFromDirectory(id string, top string, nobackupFile string) (sn
 	return snapshot, skippedDirectories, skippedFiles, nil
 }
 
+func AppendPattern(patterns []string, new_pattern string) (new_patterns []string) {
+	for _, pattern := range patterns {
+		if pattern == new_pattern {
+			LOG_INFO("SNAPSHOT_FILTER", "Ignoring duplicate pattern: %s ...", new_pattern)
+			return patterns
+		}
+	}
+	new_patterns = append(patterns, new_pattern)
+	return new_patterns
+}
+func ProcessFilters() (patterns []string) {
+	patterns = ProcessFilterFile(joinPath(GetDuplicacyPreferencePath(), "filters"), make([]string, 0))
+
+	LOG_DEBUG("REGEX_DEBUG", "There are %d compiled regular expressions stored", len(RegexMap))
+
+	LOG_INFO("SNAPSHOT_FILTER", "Loaded %d include/exclude pattern(s)", len(patterns))
+
+	if IsTracing() {
+		for _, pattern := range patterns {
+			LOG_TRACE("SNAPSHOT_PATTERN", "Pattern: %s", pattern)
+		}
+
+	}
+
+	return patterns
+}
+
+func ProcessFilterFile(patternFile string, includedFiles []string) (patterns []string) {
+	for _, file := range includedFiles {
+		if file == patternFile {
+			// cycle in include mechanism discovered.
+			LOG_ERROR("SNAPSHOT_FILTER", "The filter file %s has already been included", patternFile)
+			return patterns
+		}
+	}
+	includedFiles = append(includedFiles, patternFile)
+	LOG_INFO("SNAPSHOT_FILTER", "Parsing filter file %s", patternFile)
+	patternFileContent, err := ioutil.ReadFile(patternFile)
+	if err == nil {
+		patternFileLines := strings.Split(string(patternFileContent), "\n")
+		patterns = ProcessFilterLines(patternFileLines, includedFiles)
+	}
+	return patterns
+}
+
+func ProcessFilterLines(patternFileLines []string, includedFiles []string) (patterns []string) {
+	for _, pattern := range patternFileLines {
+		pattern = strings.TrimSpace(pattern)
+		if len(pattern) == 0 {
+			continue
+		}
+
+		if strings.HasPrefix(pattern, "@") {
+			patternIncludeFile := strings.TrimSpace(pattern[1:])
+			if patternIncludeFile == "" {
+				continue
+			}
+			if ! filepath.IsAbs(patternIncludeFile) {
+				basePath := ""
+				if len(includedFiles) == 0 {
+					basePath, _ = os.Getwd()
+				} else {
+					basePath = filepath.Dir(includedFiles[len(includedFiles)-1])
+				}
+				patternIncludeFile = joinPath(basePath, patternIncludeFile)
+			}
+			for _, pattern := range ProcessFilterFile(patternIncludeFile, includedFiles) {
+				patterns = AppendPattern(patterns, pattern)
+			}
+			continue
+		}
+
+		if pattern[0] == '#' {
+			continue
+		}
+
+		if IsUnspecifiedFilter(pattern) {
+			pattern = "+" + pattern
+		}
+
+		if IsEmptyFilter(pattern) {
+			continue
+		}
+
+		if strings.HasPrefix(pattern, "i:") || strings.HasPrefix(pattern, "e:") {
+			valid, err := IsValidRegex(pattern[2:])
+			if !valid || err != nil {
+				LOG_ERROR("SNAPSHOT_FILTER", "Invalid regular expression encountered for filter: \"%s\", error: %v", pattern, err)
+			}
+		}
+
+		patterns = AppendPattern(patterns, pattern)
+	}
+
+	return patterns
+}
+
 // This is the struct used to save/load incomplete snapshots
 type IncompleteSnapshot struct {
 	Files        []*Entry

src/duplicacy_snapshotmanager.go

@@ -38,6 +38,9 @@ type FossilCollection struct {
 	// The lastest revision for each snapshot id when the fossil collection was created.
 	LastRevisions map[string]int `json:"last_revisions"`
 
+	// Record the set of snapshots that have been removed by the prune command that created this fossil collection
+	DeletedRevisions map[string][]int `json:"deleted_revisions"`
+
 	// Fossils (i.e., chunks not referenced by any snapshots)
 	Fossils []string `json:"fossils"`
 
@@ -54,7 +57,8 @@ func CreateFossilCollection(allSnapshots map[string][]*Snapshot) *FossilCollecti
 	}
 
 	return &FossilCollection{
-		LastRevisions: lastRevisions,
+		LastRevisions:    lastRevisions,
+		DeletedRevisions: make(map[string][]int),
 	}
 }
 
@@ -105,6 +109,8 @@ func (collection *FossilCollection) IsDeletable(isStrongConsistent bool, ignored
 				continue
 			}
 
+			newSnapshots = append(newSnapshots, snapshot)
+
 			extraTime := 0
 			if !isStrongConsistent {
 				extraTime = secondsInDay / 2
@@ -114,7 +120,6 @@ func (collection *FossilCollection) IsDeletable(isStrongConsistent bool, ignored
 			// might be in progress (although very unlikely).  So we only deem it deletable if that is not the case.
 			if snapshot.EndTime > collection.EndTime+int64(extraTime) {
 				hasNewSnapshot[hostID] = true
-				newSnapshots = append(newSnapshots, snapshot)
 				break
 			} else {
 				LOG_TRACE("SNAPSHOT_UNDELETABLE",
@@ -165,6 +170,16 @@ func (collection *FossilCollection) IsEmpty() bool {
 	return len(collection.Fossils) == 0 && len(collection.Temporaries) == 0
 }
 
+// Calculates the number of days between two times ignoring the hours, minutes and seconds.
+func getDaysBetween(start int64, end int64) int {
+	startTime := time.Unix(start, 0).In(time.Now().Location())
+	endTime := time.Unix(end, 0).In(time.Now().Location())
+	startDate := time.Date(startTime.Year(), startTime.Month(), startTime.Day(), 0, 0, 0, 0, startTime.Location())
+	endDate := time.Date(endTime.Year(), endTime.Month(), endTime.Day(), 0, 0, 0, 0, endTime.Location())
+	hours := int(endDate.Sub(startDate).Hours())
+	return (hours + 1) / 24
+}
+
 // SnapshotManager is mainly responsible for downloading, and deleting snapshots.
 type SnapshotManager struct {
 
@@ -175,6 +190,7 @@ type SnapshotManager struct {
 	snapshotCache *FileStorage
 
 	chunkDownloader *ChunkDownloader
+	chunkOperator   *ChunkOperator
 }
 
 // CreateSnapshotManager creates a snapshot manager
@@ -370,7 +386,7 @@ func (manager *SnapshotManager) CleanSnapshotCache(latestSnapshot *Snapshot, all
 
 	if allSnapshots == nil {
 		// If the 'fossils' directory exists then don't clean the cache as all snapshots will be needed later
-		// during the fossil collection phase.  The deletion procedure creates this direcotry.
+		// during the fossil collection phase.  The deletion procedure creates this directory.
 		// We only check this condition when allSnapshots is nil because
 		// in thise case it is the deletion procedure that is trying to clean the snapshot cache.
 		exist, _, _, err := manager.snapshotCache.GetFileInfo(0, "fossils")
@@ -673,6 +689,9 @@ func (manager *SnapshotManager) ListSnapshots(snapshotID string, revisionsToList
 		for _, revision := range revisions {
 
 			snapshot := manager.DownloadSnapshot(snapshotID, revision)
+			if tag != "" && snapshot.Tag != tag {
+				continue
+			}
 			creationTime := time.Unix(snapshot.StartTime, 0).Format("2006-01-02 15:04")
 			tagWithSpace := ""
 			if len(snapshot.Tag) > 0 {
@@ -681,15 +700,16 @@ func (manager *SnapshotManager) ListSnapshots(snapshotID string, revisionsToList
 			LOG_INFO("SNAPSHOT_INFO", "Snapshot %s revision %d created at %s %s%s",
 				snapshotID, revision, creationTime, tagWithSpace, snapshot.Options)
 
-			if tag != "" && snapshot.Tag != tag {
-				continue
-			}
-
 			if showFiles {
 				manager.DownloadSnapshotFileSequence(snapshot, nil, false)
 			}
 
 			if showFiles {
+
+				if snapshot.NumberOfFiles > 0 {
+					LOG_INFO("SNAPSHOT_STATS", "Files: %d", snapshot.NumberOfFiles)
+				}
+
 				maxSize := int64(9)
 				maxSizeDigits := 1
 				totalFiles := 0
@@ -786,7 +806,8 @@ func (manager *SnapshotManager) CheckSnapshots(snapshotID string, revisionsToChe
 	}
 
 	snapshotIDIndex := 0
-	for snapshotID, _ = range snapshotMap {
+	totalMissingChunks := 0
+	for snapshotID = range snapshotMap {
 
 		revisions := revisionsToCheck
 		if len(revisions) == 0 || showStatistics {
@@ -799,11 +820,28 @@ func (manager *SnapshotManager) CheckSnapshots(snapshotID string, revisionsToChe
 
 		for _, revision := range revisions {
 			snapshot := manager.DownloadSnapshot(snapshotID, revision)
-			snapshotMap[snapshotID] = append(snapshotMap[snapshotID], snapshot)
-
 			if tag != "" && snapshot.Tag != tag {
 				continue
 			}
+			snapshotMap[snapshotID] = append(snapshotMap[snapshotID], snapshot)
+		}
+	}
+
+	totalRevisions := 0
+	for _, snapshotList := range snapshotMap {
+		totalRevisions += len(snapshotList)
+	}
+	LOG_INFO("SNAPSHOT_CHECK", "%d snapshots and %d revisions", len(snapshotMap), totalRevisions)
+
+	var totalChunkSize int64
+	for _, size := range chunkSizeMap {
+		totalChunkSize += size
+	}
+	LOG_INFO("SNAPSHOT_CHECK", "Total chunk size is %s in %d chunks", PrettyNumber(totalChunkSize), len(chunkSizeMap))
+
+	for snapshotID = range snapshotMap {
+
+		for _, snapshot := range snapshotMap[snapshotID] {
 
 			if checkFiles {
 				manager.DownloadSnapshotContents(snapshot, nil, false)
@@ -817,22 +855,36 @@ func (manager *SnapshotManager) CheckSnapshots(snapshotID string, revisionsToChe
 			}
 
 			missingChunks := 0
-			for chunkID, _ := range chunks {
+			for chunkID := range chunks {
 
 				_, found := chunkSizeMap[chunkID]
 
 				if !found {
+
+					// Look up the chunk again in case it actually exists, but only if there aren't
+					// too many missing chunks.
+					if missingChunks < 100 {
+						_, exist, _, err := manager.storage.FindChunk(0, chunkID, false)
+						if err != nil {
+							LOG_WARN("SNAPSHOT_VALIDATE", "Failed to check the existence of chunk %s: %v",
+							         chunkID, err)
+						} else if exist {
+							LOG_INFO("SNAPSHOT_VALIDATE", "Chunk %s is confirmed to exist", chunkID)
+							continue
+						}
+					}
+
 					if !searchFossils {
 						missingChunks += 1
 						LOG_WARN("SNAPSHOT_VALIDATE",
 							"Chunk %s referenced by snapshot %s at revision %d does not exist",
-							chunkID, snapshotID, revision)
+							chunkID, snapshotID, snapshot.Revision)
 						continue
 					}
 
 					chunkPath, exist, size, err := manager.storage.FindChunk(0, chunkID, true)
 					if err != nil {
-						LOG_ERROR("SNAPSHOT_VALIDATE", "Failed to check the existence of chunk %s: %v",
+						LOG_ERROR("SNAPSHOT_VALIDATE", "Failed to check the existence of fossil %s: %v",
 							chunkID, err)
 						return false
 					}
@@ -841,7 +893,7 @@ func (manager *SnapshotManager) CheckSnapshots(snapshotID string, revisionsToChe
 						missingChunks += 1
 						LOG_WARN("SNAPSHOT_VALIDATE",
 							"Chunk %s referenced by snapshot %s at revision %d does not exist",
-							chunkID, snapshotID, revision)
+							chunkID, snapshotID, snapshot.Revision)
 						continue
 					}
 
@@ -849,7 +901,7 @@ func (manager *SnapshotManager) CheckSnapshots(snapshotID string, revisionsToChe
 						manager.resurrectChunk(chunkPath, chunkID)
 					} else {
 						LOG_WARN("SNAPSHOT_FOSSIL", "Chunk %s referenced by snapshot %s at revision %d "+
-							"has been marked as a fossil", chunkID, snapshotID, revision)
+							"has been marked as a fossil", chunkID, snapshotID, snapshot.Revision)
 					}
 
 					chunkSizeMap[chunkID] = size
@@ -871,18 +923,23 @@ func (manager *SnapshotManager) CheckSnapshots(snapshotID string, revisionsToChe
 			}
 
 			if missingChunks > 0 {
-				LOG_ERROR("SNAPSHOT_CHECK", "Some chunks referenced by snapshot %s at revision %d are missing",
-					snapshotID, revision)
-				return false
+				LOG_WARN("SNAPSHOT_CHECK", "Some chunks referenced by snapshot %s at revision %d are missing",
+					snapshotID, snapshot.Revision)
+				totalMissingChunks += missingChunks
+			} else {
+				LOG_INFO("SNAPSHOT_CHECK", "All chunks referenced by snapshot %s at revision %d exist",
+					snapshotID, snapshot.Revision)
 			}
-
-			LOG_INFO("SNAPSHOT_CHECK", "All chunks referenced by snapshot %s at revision %d exist",
-				snapshotID, revision)
 		}
 
 		snapshotIDIndex += 1
 	}
 
+	if totalMissingChunks > 0 {
+		LOG_ERROR("SNAPSHOT_CHECK", "Some chunks referenced by some snapshots do not exist in the storage")
+		return false
+	}
+
 	if showTabular {
 		manager.ShowStatisticsTabular(snapshotMap, chunkSizeMap, chunkUniqueMap, chunkSnapshotMap)
 	} else if showStatistics {
@@ -910,7 +967,7 @@ func (manager *SnapshotManager) ShowStatistics(snapshotMap map[string][]*Snapsho
 			var totalChunkSize int64
 			var uniqueChunkSize int64
 
-			for chunkID, _ := range chunks {
+			for chunkID := range chunks {
 				chunkSize := chunkSizeMap[chunkID]
 				totalChunkSize += chunkSize
 				if chunkUniqueMap[chunkID] {
@@ -928,7 +985,7 @@ func (manager *SnapshotManager) ShowStatistics(snapshotMap map[string][]*Snapsho
 
 		var totalChunkSize int64
 		var uniqueChunkSize int64
-		for chunkID, _ := range snapshotChunks {
+		for chunkID := range snapshotChunks {
 			chunkSize := chunkSizeMap[chunkID]
 			totalChunkSize += chunkSize
 
@@ -955,18 +1012,20 @@ func (manager *SnapshotManager) ShowStatisticsTabular(snapshotMap map[string][]*
 		earliestSeenChunks := make(map[string]int)
 
 		for _, snapshot := range snapshotList {
-			for _, chunkID := range manager.GetSnapshotChunks(snapshot, true) {
+			for _, chunkID := range manager.GetSnapshotChunks(snapshot, false) {
 				if earliestSeenChunks[chunkID] == 0 {
 					earliestSeenChunks[chunkID] = math.MaxInt32
 				}
-				earliestSeenChunks[chunkID] = MinInt(earliestSeenChunks[chunkID], snapshot.Revision)
+				if earliestSeenChunks[chunkID] > snapshot.Revision {
+					earliestSeenChunks[chunkID] = snapshot.Revision
+				}
 			}
 		}
 
 		for _, snapshot := range snapshotList {
 
 			chunks := make(map[string]bool)
-			for _, chunkID := range manager.GetSnapshotChunks(snapshot, true) {
+			for _, chunkID := range manager.GetSnapshotChunks(snapshot, false) {
 				chunks[chunkID] = true
 				snapshotChunks[chunkID] = true
 			}
@@ -978,7 +1037,7 @@ func (manager *SnapshotManager) ShowStatisticsTabular(snapshotMap map[string][]*
 			var newChunkCount int64
 			var newChunkSize int64
 
-			for chunkID, _ := range chunks {
+			for chunkID := range chunks {
 				chunkSize := chunkSizeMap[chunkID]
 				totalChunkSize += chunkSize
 				totalChunkCount += 1
@@ -1006,7 +1065,7 @@ func (manager *SnapshotManager) ShowStatisticsTabular(snapshotMap map[string][]*
 		var uniqueChunkSize int64
 		var totalChunkCount int64
 		var uniqueChunkCount int64
-		for chunkID, _ := range snapshotChunks {
+		for chunkID := range snapshotChunks {
 			chunkSize := chunkSizeMap[chunkID]
 			totalChunkSize += chunkSize
 			totalChunkCount += 1
@@ -1111,7 +1170,7 @@ func (manager *SnapshotManager) VerifySnapshot(snapshot *Snapshot) bool {
 	}
 }
 
-// RetrieveFile retrieve the file in the specifed snapshot.
+// RetrieveFile retrieves the file in the specified snapshot.
 func (manager *SnapshotManager) RetrieveFile(snapshot *Snapshot, file *Entry, output func([]byte)) bool {
 
 	if file.Size == 0 {
@@ -1226,15 +1285,14 @@ func (manager *SnapshotManager) PrintFile(snapshotID string, revision int, path
 	}
 
 	file := manager.FindFile(snapshot, path, false)
-	var content []byte
-	if !manager.RetrieveFile(snapshot, file, func(chunk []byte) { content = append(content, chunk...) }) {
+	if !manager.RetrieveFile(snapshot, file, func(chunk []byte) {
+			fmt.Printf("%s", chunk)
+		}) {
 		LOG_ERROR("SNAPSHOT_RETRIEVE", "File %s is corrupted in snapshot %s at revision %d",
 			path, snapshot.ID, snapshot.Revision)
 		return false
 	}
 
-	fmt.Printf("%s", string(content))
-
 	return true
 }
 
@@ -1503,36 +1561,11 @@ func (manager *SnapshotManager) ShowHistory(top string, snapshotID string, revis
 }
 
 // fossilizeChunk turns the chunk into a fossil.
-func (manager *SnapshotManager) fossilizeChunk(chunkID string, filePath string,
-	exclusive bool, collection *FossilCollection) bool {
+func (manager *SnapshotManager) fossilizeChunk(chunkID string, filePath string, exclusive bool) bool {
 	if exclusive {
-		err := manager.storage.DeleteFile(0, filePath)
-		if err != nil {
-			LOG_ERROR("CHUNK_DELETE", "Failed to remove the chunk %s: %v", chunkID, err)
-			return false
-		} else {
-			LOG_TRACE("CHUNK_DELETE", "Deleted chunk file %s", chunkID)
-		}
-
+		manager.chunkOperator.Delete(chunkID, filePath)
 	} else {
-		fossilPath := filePath + ".fsl"
-
-		err := manager.storage.MoveFile(0, filePath, fossilPath)
-		if err != nil {
-			if _, exist, _, _ := manager.storage.FindChunk(0, chunkID, true); exist {
-				err := manager.storage.DeleteFile(0, filePath)
-				if err == nil {
-					LOG_TRACE("CHUNK_DELETE", "Deleted chunk file %s as the fossil already exists", chunkID)
-				}
-			} else {
-				LOG_ERROR("CHUNK_DELETE", "Failed to fossilize the chunk %s: %v", chunkID, err)
-				return false
-			}
-		} else {
-			LOG_TRACE("CHUNK_FOSSILIZE", "Fossilized chunk %s", chunkID)
-		}
-
-		collection.AddFossil(fossilPath)
+		manager.chunkOperator.Fossilize(chunkID, filePath)
 	}
 
 	return true
@@ -1581,7 +1614,7 @@ func (manager *SnapshotManager) resurrectChunk(fossilPath string, chunkID string
 func (manager *SnapshotManager) PruneSnapshots(selfID string, snapshotID string, revisionsToBeDeleted []int,
 	tags []string, retentions []string,
 	exhaustive bool, exclusive bool, ignoredIDs []string,
-	dryRun bool, deleteOnly bool, collectOnly bool) bool {
+	dryRun bool, deleteOnly bool, collectOnly bool, threads int) bool {
 
 	LOG_DEBUG("DELETE_PARAMETERS",
 		"id: %s, revisions: %v, tags: %v, retentions: %v, exhaustive: %t, exclusive: %t, "+
@@ -1593,6 +1626,9 @@ func (manager *SnapshotManager) PruneSnapshots(selfID string, snapshotID string,
 		LOG_WARN("DELETE_OPTIONS", "Tags or retention policy will be ignored if at least one revision is specified")
 	}
 
+	manager.chunkOperator = CreateChunkOperator(manager.storage, threads)
+	defer manager.chunkOperator.Stop()
+
 	prefPath := GetDuplicacyPreferencePath()
 	logDir := path.Join(prefPath, "logs")
 	err := os.MkdirAll(logDir, 0700)
@@ -1762,6 +1798,29 @@ func (manager *SnapshotManager) PruneSnapshots(selfID string, snapshotID string,
 			return false
 		}
 
+		// Determine if any deleted revisions exist
+		exist := false
+		for snapshotID, revisionList := range collection.DeletedRevisions {
+			for _, revision := range revisionList {
+				for _, snapshot := range allSnapshots[snapshotID] {
+					if revision == snapshot.Revision {
+						LOG_INFO("FOSSIL_GHOSTSNAPSHOT", "Snapshot %s revision %d should have been deleted already", snapshotID, revision)
+						exist = true
+					}
+				}
+			}
+		}
+
+		if exist {
+			err = manager.snapshotCache.DeleteFile(0, collectionFile)
+			if err != nil {
+				LOG_WARN("FOSSIL_FILE", "Failed to remove the fossil collection file %s: %v", collectionFile, err)
+			} else {
+				LOG_INFO("FOSSIL_IGNORE", "The fossil collection file %s has been ignored due to ghost snapshots", collectionFile)
+			}
+			continue
+		}
+
 		for _, fossil := range collection.Fossils {
 			referencedFossils[fossil] = true
 		}
@@ -1778,6 +1837,8 @@ func (manager *SnapshotManager) PruneSnapshots(selfID string, snapshotID string,
 			newChunks := make(map[string]bool)
 
 			for _, newSnapshot := range newSnapshots {
+				fmt.Fprintf(logFile, "Snapshot %s revision %d was created after collection %s\n", newSnapshot.ID, newSnapshot.Revision, collectionName)
+				LOG_INFO("PRUNE_NEWSNAPSHOT", "Snapshot %s revision %d was created after collection %s", newSnapshot.ID, newSnapshot.Revision, collectionName)
 				for _, chunk := range manager.GetSnapshotChunks(newSnapshot, false) {
 					newChunks[chunk] = true
 				}
@@ -1796,20 +1857,15 @@ func (manager *SnapshotManager) PruneSnapshots(selfID string, snapshotID string,
 						continue
 					}
 
-					manager.resurrectChunk(fossil, chunk)
+					manager.chunkOperator.Resurrect(chunk, fossil)
 					fmt.Fprintf(logFile, "Resurrected fossil %s (collection %s)\n", chunk, collectionName)
 
 				} else {
 					if dryRun {
 						LOG_INFO("FOSSIL_DELETE", "The chunk %s would be permanently removed", chunk)
 					} else {
-						err = manager.storage.DeleteFile(0, fossil)
-						if err != nil {
-							LOG_WARN("FOSSIL_DELETE", "The chunk %s could not be removed: %v", chunk, err)
-						} else {
-							LOG_INFO("FOSSIL_DELETE", "The chunk %s has been permanently removed", chunk)
-							fmt.Fprintf(logFile, "Deleted fossil %s (collection %s)\n", chunk, collectionName)
-						}
+						manager.chunkOperator.Delete(chunk, fossil)
+						fmt.Fprintf(logFile, "Deleted fossil %s (collection %s)\n", chunk, collectionName)
 					}
 				}
 			}
@@ -1820,8 +1876,7 @@ func (manager *SnapshotManager) PruneSnapshots(selfID string, snapshotID string,
 					LOG_INFO("TEMPORARY_DELETE", "The temporary file %s would be deleted", temporary)
 				} else {
 					// Fail silently, since temporary files are supposed to be renamed or deleted after upload is done
-					_ = manager.storage.DeleteFile(0, temporary)
-					LOG_INFO("TEMPORARY_DELETE", "The temporary file %s has been deleted", temporary)
+					manager.chunkOperator.Delete("", temporary)
 					fmt.Fprintf(logFile, "Deleted temporary %s (collection %s)\n", temporary, collectionName)
 				}
 			}
@@ -1898,7 +1953,7 @@ func (manager *SnapshotManager) PruneSnapshots(selfID string, snapshotID string,
 
 				// Find out which retent policy applies based on the age.
 				for i < len(retentionPolicies) &&
-					int(now-snapshot.StartTime) < retentionPolicies[i].Age*secondsInDay {
+					getDaysBetween(snapshot.StartTime, now) < retentionPolicies[i].Age {
 					i++
 					lastSnapshotTime = 0
 				}
@@ -1911,9 +1966,8 @@ func (manager *SnapshotManager) PruneSnapshots(selfID string, snapshotID string,
 						snapshot.Flag = true
 						toBeDeleted++
 					} else if lastSnapshotTime != 0 &&
-						int(snapshot.StartTime-lastSnapshotTime) < retentionPolicies[i].Interval*secondsInDay-600 {
-						// Delete the snapshot if it is too close to the last kept one.  Note that a tolerance of 10
-						// minutes was subtracted from the interval.
+						getDaysBetween(lastSnapshotTime, snapshot.StartTime) < retentionPolicies[i].Interval {
+						// Delete the snapshot if it is too close to the last kept one.
 						LOG_DEBUG("SNAPSHOT_DELETE", "Snapshot %s at revision %d to be deleted - older than %d days, less than %d days from previous",
 							snapshot.ID, snapshot.Revision, retentionPolicies[i].Age, retentionPolicies[i].Interval)
 						snapshot.Flag = true
@@ -1949,14 +2003,28 @@ func (manager *SnapshotManager) PruneSnapshots(selfID string, snapshotID string,
 	if exhaustive {
 		success = manager.pruneSnapshotsExhaustive(referencedFossils, allSnapshots, collection, logFile, dryRun, exclusive)
 	} else {
-		success = manager.pruneSnapshots(allSnapshots, collection, logFile, dryRun, exclusive)
+		success = manager.pruneSnapshotsNonExhaustive(allSnapshots, collection, logFile, dryRun, exclusive)
 	}
 	if !success {
 		return false
 	}
 
+	manager.chunkOperator.Stop()
+	for _, fossil := range manager.chunkOperator.fossils {
+		collection.AddFossil(fossil)
+	}
+
+	// Save the deleted revision in the fossil collection
+	for _, snapshots := range allSnapshots {
+		for _, snapshot := range snapshots {
+			if snapshot.Flag {
+				collection.DeletedRevisions[snapshot.ID] = append(collection.DeletedRevisions[snapshot.ID], snapshot.Revision)
+			}
+		}
+	}
+
 	// Save the fossil collection if it is not empty.
-	if !collection.IsEmpty() && !dryRun {
+	if !collection.IsEmpty() && !dryRun && !exclusive {
 		collection.EndTime = time.Now().Unix()
 
 		collectionNumber := maxCollectionNumber + 1
@@ -2028,7 +2096,7 @@ func (manager *SnapshotManager) PruneSnapshots(selfID string, snapshotID string,
 
 // pruneSnapshots in non-exhaustive mode, only chunks that exist in the
 // snapshots to be deleted but not other are identified as unreferenced chunks.
-func (manager *SnapshotManager) pruneSnapshots(allSnapshots map[string][]*Snapshot, collection *FossilCollection, logFile io.Writer, dryRun, exclusive bool) bool {
+func (manager *SnapshotManager) pruneSnapshotsNonExhaustive(allSnapshots map[string][]*Snapshot, collection *FossilCollection, logFile io.Writer, dryRun, exclusive bool) bool {
 	targetChunks := make(map[string]bool)
 
 	// Now build all chunks referened by snapshot not deleted
@@ -2085,18 +2153,7 @@ func (manager *SnapshotManager) pruneSnapshots(allSnapshots map[string][]*Snapsh
 			continue
 		}
 
-		chunkPath, exist, _, err := manager.storage.FindChunk(0, chunk, false)
-		if err != nil {
-			LOG_ERROR("CHUNK_FIND", "Failed to locate the path for the chunk %s: %v", chunk, err)
-			return false
-		}
-
-		if !exist {
-			LOG_WARN("CHUNK_MISSING", "The chunk %s does not exist", chunk)
-			continue
-		}
-
-		manager.fossilizeChunk(chunk, chunkPath, exclusive, collection)
+		manager.fossilizeChunk(chunk, "", exclusive)
 		if exclusive {
 			fmt.Fprintf(logFile, "Deleted chunk %s (exclusive mode)\n", chunk)
 		} else {
@@ -2158,12 +2215,7 @@ func (manager *SnapshotManager) pruneSnapshotsExhaustive(referencedFossils map[s
 
 			if exclusive {
 				// In exclusive mode, we assume no other restore operation is running concurrently.
-				err := manager.storage.DeleteFile(0, chunkDir+file)
-				if err != nil {
-					LOG_ERROR("CHUNK_TEMPORARY", "Failed to remove the temporary file %s: %v", file, err)
-					return false
-				}
-				LOG_DEBUG("CHUNK_TEMPORARY", "Deleted temporary file %s", file)
+				manager.chunkOperator.Delete("", chunkDir+file)
 				fmt.Fprintf(logFile, "Deleted temporary %s\n", file)
 			} else {
 				collection.AddTemporary(file)
@@ -2173,19 +2225,33 @@ func (manager *SnapshotManager) pruneSnapshotsExhaustive(referencedFossils map[s
 			// This is a fossil.  If it is unreferenced, it can be a result of failing to save the fossil
 			// collection file after making it a fossil.
 			if _, found := referencedFossils[file]; !found {
-				if dryRun {
-					LOG_INFO("FOSSIL_UNREFERENCED", "Found unreferenced fossil %s", file)
-					continue
-				}
 
 				chunk := strings.Replace(file, "/", "", -1)
 				chunk = strings.Replace(chunk, ".fsl", "", -1)
 
 				if _, found := referencedChunks[chunk]; found {
-					manager.resurrectChunk(chunkDir+file, chunk)
+
+					if dryRun {
+						LOG_INFO("FOSSIL_REFERENCED", "Found referenced fossil %s", file)
+						continue
+					}
+
+					manager.chunkOperator.Resurrect(chunk, chunkDir+file)
+					fmt.Fprintf(logFile, "Found referenced fossil %s\n", file)
+
 				} else {
-					collection.AddFossil(chunkDir + file)
-					LOG_DEBUG("FOSSIL_FIND", "Found unreferenced fossil %s", file)
+
+					if dryRun {
+						LOG_INFO("FOSSIL_UNREFERENCED", "Found unreferenced fossil %s", file)
+						continue
+					}
+
+					if exclusive {
+						manager.chunkOperator.Delete(chunk, chunkDir+file)
+					} else {
+						collection.AddFossil(chunkDir + file)
+						LOG_DEBUG("FOSSIL_FIND", "Found unreferenced fossil %s", file)
+					}
 					fmt.Fprintf(logFile, "Found unreferenced fossil %s\n", file)
 				}
 			}
@@ -2196,7 +2262,7 @@ func (manager *SnapshotManager) pruneSnapshotsExhaustive(referencedFossils map[s
 		chunk := strings.Replace(file, "/", "", -1)
 
 		if !chunkRegex.MatchString(chunk) {
-			LOG_WARN("CHUNK_UNKONWN_FILE", "File %s is not a chunk", file)
+			LOG_WARN("CHUNK_UNKNOWN_FILE", "File %s is not a chunk", file)
 			continue
 		}
 
@@ -2206,7 +2272,7 @@ func (manager *SnapshotManager) pruneSnapshotsExhaustive(referencedFossils map[s
 				continue
 			}
 
-			manager.fossilizeChunk(chunk, chunkDir+file, exclusive, collection)
+			manager.fossilizeChunk(chunk, chunkDir+file, exclusive)
 			if exclusive {
 				fmt.Fprintf(logFile, "Deleted chunk %s (exclusive mode)\n", chunk)
 			} else {
@@ -2222,14 +2288,8 @@ func (manager *SnapshotManager) pruneSnapshotsExhaustive(referencedFossils map[s
 			}
 
 			// This is a redundant chunk file (for instance D3/495A8D and D3/49/5A8D )
-			err := manager.storage.DeleteFile(0, chunkDir+file)
-			if err != nil {
-				LOG_WARN("CHUNK_DELETE", "Failed to remove the redundant chunk file %s: %v", file, err)
-			} else {
-				LOG_TRACE("CHUNK_DELETE", "Removed the redundant chunk file %s", file)
-				fmt.Fprintf(logFile, "Deleted redundant chunk %s\n", file)
-			}
-
+			manager.chunkOperator.Delete(chunk, chunkDir+file)
+			fmt.Fprintf(logFile, "Deleted redundant chunk %s\n", file)
 		} else {
 			referencedChunks[chunk] = true
 			LOG_DEBUG("CHUNK_KEEP", "Chunk %s is referenced", chunk)
@@ -2364,7 +2424,7 @@ func (manager *SnapshotManager) DownloadFile(path string, derivationKey string)
 	}
 
 	if len(derivationKey) > 64 {
-		derivationKey = derivationKey[len(derivationKey) - 64:]
+		derivationKey = derivationKey[len(derivationKey)-64:]
 	}
 
 	err = manager.fileChunk.Decrypt(manager.config.FileKey, derivationKey)
@@ -2398,7 +2458,7 @@ func (manager *SnapshotManager) UploadFile(path string, derivationKey string, co
 	}
 
 	if len(derivationKey) > 64 {
-		derivationKey = derivationKey[len(derivationKey) - 64:]
+		derivationKey = derivationKey[len(derivationKey)-64:]
 	}
 
 	err := manager.fileChunk.Encrypt(manager.config.FileKey, derivationKey)

src/duplicacy_snapshotmanager_test.go

@@ -9,6 +9,7 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"path"
 	"strings"
@@ -143,6 +144,15 @@ func uploadRandomChunk(manager *SnapshotManager, chunkSize int) string {
 	return uploadTestChunk(manager, content)
 }
 
+func uploadRandomChunks(manager *SnapshotManager, chunkSize int, numberOfChunks int) []string {
+	chunkList := make([]string, 0)
+	for i := 0; i < numberOfChunks; i++ {
+		chunkHash := uploadRandomChunk(manager, chunkSize)
+		chunkList = append(chunkList, chunkHash)
+	}
+	return chunkList
+}
+
 func createTestSnapshot(manager *SnapshotManager, snapshotID string, revision int, startTime int64, endTime int64, chunkHashes []string, tag string) {
 
 	snapshot := &Snapshot{
@@ -253,11 +263,11 @@ func TestPruneSingleRepository(t *testing.T) {
 	checkTestSnapshots(snapshotManager, 4, 0)
 
 	t.Logf("Removing snapshot repository1 revisions 1 and 2 with --exclusive")
-	snapshotManager.PruneSnapshots("repository1", "repository1", []int{1, 2}, []string{}, []string{}, false, true, []string{}, false, false, false)
+	snapshotManager.PruneSnapshots("repository1", "repository1", []int{1, 2}, []string{}, []string{}, false, true, []string{}, false, false, false, 1)
 	checkTestSnapshots(snapshotManager, 2, 0)
 
 	t.Logf("Removing snapshot repository1 revision 3 without --exclusive")
-	snapshotManager.PruneSnapshots("repository1", "repository1", []int{3}, []string{}, []string{}, false, false, []string{}, false, false, false)
+	snapshotManager.PruneSnapshots("repository1", "repository1", []int{3}, []string{}, []string{}, false, false, []string{}, false, false, false, 1)
 	checkTestSnapshots(snapshotManager, 1, 2)
 
 	t.Logf("Creating 1 snapshot")
@@ -266,7 +276,7 @@ func TestPruneSingleRepository(t *testing.T) {
 	checkTestSnapshots(snapshotManager, 2, 2)
 
 	t.Logf("Prune without removing any snapshots -- fossils will be deleted")
-	snapshotManager.PruneSnapshots("repository1", "repository1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false)
+	snapshotManager.PruneSnapshots("repository1", "repository1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false, 1)
 	checkTestSnapshots(snapshotManager, 2, 0)
 }
 
@@ -293,11 +303,11 @@ func TestPruneSingleHost(t *testing.T) {
 	checkTestSnapshots(snapshotManager, 3, 0)
 
 	t.Logf("Removing snapshot vm1@host1 revision 1 without --exclusive")
-	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{1}, []string{}, []string{}, false, false, []string{}, false, false, false)
+	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{1}, []string{}, []string{}, false, false, []string{}, false, false, false, 1)
 	checkTestSnapshots(snapshotManager, 2, 2)
 
 	t.Logf("Prune without removing any snapshots -- no fossils will be deleted")
-	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false)
+	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false, 1)
 	checkTestSnapshots(snapshotManager, 2, 2)
 
 	t.Logf("Creating 1 snapshot")
@@ -306,7 +316,7 @@ func TestPruneSingleHost(t *testing.T) {
 	checkTestSnapshots(snapshotManager, 3, 2)
 
 	t.Logf("Prune without removing any snapshots -- fossils will be deleted")
-	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false)
+	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false, 1)
 	checkTestSnapshots(snapshotManager, 3, 0)
 
 }
@@ -334,11 +344,11 @@ func TestPruneMultipleHost(t *testing.T) {
 	checkTestSnapshots(snapshotManager, 3, 0)
 
 	t.Logf("Removing snapshot vm1@host1 revision 1 without --exclusive")
-	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{1}, []string{}, []string{}, false, false, []string{}, false, false, false)
+	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{1}, []string{}, []string{}, false, false, []string{}, false, false, false, 1)
 	checkTestSnapshots(snapshotManager, 2, 2)
 
 	t.Logf("Prune without removing any snapshots -- no fossils will be deleted")
-	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false)
+	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false, 1)
 	checkTestSnapshots(snapshotManager, 2, 2)
 
 	t.Logf("Creating 1 snapshot")
@@ -347,7 +357,7 @@ func TestPruneMultipleHost(t *testing.T) {
 	checkTestSnapshots(snapshotManager, 3, 2)
 
 	t.Logf("Prune without removing any snapshots -- no fossils will be deleted")
-	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false)
+	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false, 1)
 	checkTestSnapshots(snapshotManager, 3, 2)
 
 	t.Logf("Creating 1 snapshot")
@@ -356,7 +366,7 @@ func TestPruneMultipleHost(t *testing.T) {
 	checkTestSnapshots(snapshotManager, 4, 2)
 
 	t.Logf("Prune without removing any snapshots -- fossils will be deleted")
-	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false)
+	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false, 1)
 	checkTestSnapshots(snapshotManager, 4, 0)
 }
 
@@ -381,7 +391,7 @@ func TestPruneAndResurrect(t *testing.T) {
 	checkTestSnapshots(snapshotManager, 2, 0)
 
 	t.Logf("Removing snapshot vm1@host1 revision 1 without --exclusive")
-	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{1}, []string{}, []string{}, false, false, []string{}, false, false, false)
+	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{1}, []string{}, []string{}, false, false, []string{}, false, false, false, 1)
 	checkTestSnapshots(snapshotManager, 1, 2)
 
 	t.Logf("Creating 1 snapshot")
@@ -390,7 +400,7 @@ func TestPruneAndResurrect(t *testing.T) {
 	checkTestSnapshots(snapshotManager, 2, 2)
 
 	t.Logf("Prune without removing any snapshots -- one fossil will be resurrected")
-	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false)
+	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false, 1)
 	checkTestSnapshots(snapshotManager, 2, 0)
 }
 
@@ -418,11 +428,11 @@ func TestPruneWithInactiveHost(t *testing.T) {
 	checkTestSnapshots(snapshotManager, 3, 0)
 
 	t.Logf("Removing snapshot vm1@host1 revision 1")
-	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{1}, []string{}, []string{}, false, false, []string{}, false, false, false)
+	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{1}, []string{}, []string{}, false, false, []string{}, false, false, false, 1)
 	checkTestSnapshots(snapshotManager, 2, 2)
 
 	t.Logf("Prune without removing any snapshots -- no fossils will be deleted")
-	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false)
+	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false, 1)
 	checkTestSnapshots(snapshotManager, 2, 2)
 
 	t.Logf("Creating 1 snapshot")
@@ -431,7 +441,7 @@ func TestPruneWithInactiveHost(t *testing.T) {
 	checkTestSnapshots(snapshotManager, 3, 2)
 
 	t.Logf("Prune without removing any snapshots -- fossils will be deleted")
-	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false)
+	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false, 1)
 	checkTestSnapshots(snapshotManager, 3, 0)
 }
 
@@ -459,15 +469,15 @@ func TestPruneWithRetentionPolicy(t *testing.T) {
 	checkTestSnapshots(snapshotManager, 30, 0)
 
 	t.Logf("Removing snapshot vm1@host1 0:20 with --exclusive")
-	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{"0:20"}, false, true, []string{}, false, false, false)
+	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{"0:20"}, false, true, []string{}, false, false, false, 1)
 	checkTestSnapshots(snapshotManager, 19, 0)
 
 	t.Logf("Removing snapshot vm1@host1 -k 0:20 with --exclusive")
-	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{"0:20"}, false, true, []string{}, false, false, false)
+	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{"0:20"}, false, true, []string{}, false, false, false, 1)
 	checkTestSnapshots(snapshotManager, 19, 0)
 
 	t.Logf("Removing snapshot vm1@host1 -k 3:14 -k 2:7 with --exclusive")
-	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{"3:14", "2:7"}, false, true, []string{}, false, false, false)
+	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{"3:14", "2:7"}, false, true, []string{}, false, false, false, 1)
 	checkTestSnapshots(snapshotManager, 12, 0)
 }
 
@@ -490,7 +500,7 @@ func TestPruneWithRetentionPolicyAndTag(t *testing.T) {
 	t.Logf("Creating 30 snapshots")
 	for i := 0; i < 30; i++ {
 		tag := "auto"
-		if i % 3 == 0 {
+		if i%3 == 0 {
 			tag = "manual"
 		}
 		createTestSnapshot(snapshotManager, "vm1@host1", i+1, now-int64(30-i)*day-3600, now-int64(30-i)*day-60, []string{chunkHashes[i]}, tag)
@@ -499,7 +509,7 @@ func TestPruneWithRetentionPolicyAndTag(t *testing.T) {
 	checkTestSnapshots(snapshotManager, 30, 0)
 
 	t.Logf("Removing snapshot vm1@host1 0:20 with --exclusive and --tag manual")
-	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{"manual"}, []string{"0:7"}, false, true, []string{}, false, false, false)
+	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{"manual"}, []string{"0:7"}, false, true, []string{}, false, false, false, 1)
 	checkTestSnapshots(snapshotManager, 22, 0)
 }
 
@@ -527,11 +537,151 @@ func TestPruneWithFossils(t *testing.T) {
 
 	t.Logf("Prune without removing any snapshots but with --exhaustive")
 	// The unreferenced fossil shouldn't be removed
-	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, true, false, []string{}, false, false, false)
+	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, true, false, []string{}, false, false, false, 1)
 	checkTestSnapshots(snapshotManager, 2, 1)
 
 	t.Logf("Prune without removing any snapshots but with --exclusive")
 	// Now the unreferenced fossil should be removed
-	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, true, []string{}, false, false, false)
+	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, true, []string{}, false, false, false, 1)
 	checkTestSnapshots(snapshotManager, 2, 0)
 }
+
+func TestPruneMultipleThread(t *testing.T) {
+
+	setTestingT(t)
+
+	testDir := path.Join(os.TempDir(), "duplicacy_test", "snapshot_test")
+
+	snapshotManager := createTestSnapshotManager(testDir)
+
+	chunkSize := 1024
+	numberOfChunks := 256
+	numberOfThreads := 4
+
+	chunkList1 := uploadRandomChunks(snapshotManager, chunkSize, numberOfChunks)
+	chunkList2 := uploadRandomChunks(snapshotManager, chunkSize, numberOfChunks)
+
+	now := time.Now().Unix()
+	day := int64(24 * 3600)
+	t.Logf("Creating 2 snapshots")
+	createTestSnapshot(snapshotManager, "repository1", 1, now-4*day-3600, now-3*day-60, chunkList1, "tag")
+	createTestSnapshot(snapshotManager, "repository1", 2, now-3*day-3600, now-2*day-60, chunkList2, "tag")
+	checkTestSnapshots(snapshotManager, 2, 0)
+
+	t.Logf("Removing snapshot revisions 1 with --exclusive")
+	snapshotManager.PruneSnapshots("repository1", "repository1", []int{1}, []string{}, []string{}, false, true, []string{}, false, false, false, numberOfThreads)
+	checkTestSnapshots(snapshotManager, 1, 0)
+
+	t.Logf("Creating 1 more snapshot")
+	chunkList3 := uploadRandomChunks(snapshotManager, chunkSize, numberOfChunks)
+	createTestSnapshot(snapshotManager, "repository1", 3, now-2*day-3600, now-1*day-60, chunkList3, "tag")
+
+	t.Logf("Removing snapshot repository1 revision 2 without --exclusive")
+	snapshotManager.PruneSnapshots("repository1", "repository1", []int{2}, []string{}, []string{}, false, false, []string{}, false, false, false, numberOfThreads)
+
+	t.Logf("Prune without removing any snapshots but with --exclusive")
+	snapshotManager.PruneSnapshots("repository1", "repository1", []int{}, []string{}, []string{}, false, true, []string{}, false, false, false, numberOfThreads)
+	checkTestSnapshots(snapshotManager, 1, 0)
+}
+
+// A snapshot not seen by a fossil collection should always be consider a new snapshot in the fossil deletion step
+func TestPruneNewSnapshots(t *testing.T) {
+	setTestingT(t)
+
+	testDir := path.Join(os.TempDir(), "duplicacy_test", "snapshot_test")
+
+	snapshotManager := createTestSnapshotManager(testDir)
+
+	chunkSize := 1024
+	chunkHash1 := uploadRandomChunk(snapshotManager, chunkSize)
+	chunkHash2 := uploadRandomChunk(snapshotManager, chunkSize)
+	chunkHash3 := uploadRandomChunk(snapshotManager, chunkSize)
+	chunkHash4 := uploadRandomChunk(snapshotManager, chunkSize)
+
+	now := time.Now().Unix()
+	day := int64(24 * 3600)
+	t.Logf("Creating 3 snapshots")
+	createTestSnapshot(snapshotManager, "vm1@host1", 1, now-3*day-3600, now-3*day-60, []string{chunkHash1, chunkHash2}, "tag")
+	createTestSnapshot(snapshotManager, "vm1@host1", 2, now-2*day-3600, now-2*day-60, []string{chunkHash2, chunkHash3}, "tag")
+	createTestSnapshot(snapshotManager, "vm2@host1", 1, now-2*day-3600, now-2*day-60, []string{chunkHash3, chunkHash4}, "tag")
+	checkTestSnapshots(snapshotManager, 3, 0)
+
+	t.Logf("Prune snapshot 1")
+	// chunkHash1 should be marked as fossil
+	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{1}, []string{}, []string{}, false, false, []string{}, false, false, false, 1)
+	checkTestSnapshots(snapshotManager, 2, 2)
+
+	chunkHash5 := uploadRandomChunk(snapshotManager, chunkSize)
+	// Create another snapshot of vm1 that brings back chunkHash1
+	createTestSnapshot(snapshotManager, "vm1@host1", 3, now-0*day-3600, now-0*day-60, []string{chunkHash1, chunkHash3}, "tag")
+	// Create another snapshot of vm2 so the fossil collection will be processed by next prune
+	createTestSnapshot(snapshotManager, "vm2@host1", 2, now+3600, now+3600*2, []string{chunkHash4, chunkHash5}, "tag")
+
+	// Now chunkHash1 wil be resurrected
+	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false, 1)
+	checkTestSnapshots(snapshotManager, 4, 0)
+	snapshotManager.CheckSnapshots("vm1@host1", []int{2, 3}, "", false, false, false, false, false)
+}
+
+// A fossil collection left by an aborted prune should be ignored if any supposedly deleted snapshot exists
+func TestPruneGhostSnapshots(t *testing.T) {
+	setTestingT(t)
+
+	EnableStackTrace()
+
+	testDir := path.Join(os.TempDir(), "duplicacy_test", "snapshot_test")
+
+	snapshotManager := createTestSnapshotManager(testDir)
+
+	chunkSize := 1024
+	chunkHash1 := uploadRandomChunk(snapshotManager, chunkSize)
+	chunkHash2 := uploadRandomChunk(snapshotManager, chunkSize)
+	chunkHash3 := uploadRandomChunk(snapshotManager, chunkSize)
+
+	now := time.Now().Unix()
+	day := int64(24 * 3600)
+	t.Logf("Creating 2 snapshots")
+	createTestSnapshot(snapshotManager, "vm1@host1", 1, now-3*day-3600, now-3*day-60, []string{chunkHash1, chunkHash2}, "tag")
+	createTestSnapshot(snapshotManager, "vm1@host1", 2, now-2*day-3600, now-2*day-60, []string{chunkHash2, chunkHash3}, "tag")
+	checkTestSnapshots(snapshotManager, 2, 0)
+
+	snapshot1, err := ioutil.ReadFile(path.Join(testDir, "snapshots", "vm1@host1", "1"))
+	if err != nil {
+		t.Errorf("Failed to read snapshot file: %v", err)
+	}
+
+	t.Logf("Prune snapshot 1")
+	// chunkHash1 should be marked as fossil
+	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{1}, []string{}, []string{}, false, false, []string{}, false, false, false, 1)
+	checkTestSnapshots(snapshotManager, 1, 2)
+
+	// Recover the snapshot file for revision 1; this is to simulate a scenario where prune may encounter a network error after
+	// leaving the fossil collection but before deleting any snapshots.
+	err = ioutil.WriteFile(path.Join(testDir, "snapshots", "vm1@host1", "1"), snapshot1, 0644)
+	if err != nil {
+		t.Errorf("Failed to write snapshot file: %v", err)
+	}
+
+	// Create another snapshot of vm1 so the fossil collection becomes eligible for processing.
+	chunkHash4 := uploadRandomChunk(snapshotManager, chunkSize)
+	createTestSnapshot(snapshotManager, "vm1@host1", 3, now-day-3600, now-day-60, []string{chunkHash3, chunkHash4}, "tag")
+
+	// Run the prune again but the fossil collection should be igored, since revision 1 still exists
+	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false, 1)
+	checkTestSnapshots(snapshotManager, 3, 2)
+	snapshotManager.CheckSnapshots("vm1@host1", []int{1, 2, 3}, "", false, false, false, true /*searchFossils*/, false)
+
+	// Prune snapshot 1 again
+	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{1}, []string{}, []string{}, false, false, []string{}, false, false, false, 1)
+	checkTestSnapshots(snapshotManager, 2, 2)
+
+	// Create another snapshot
+	chunkHash5 := uploadRandomChunk(snapshotManager, chunkSize)
+	createTestSnapshot(snapshotManager, "vm1@host1", 4, now+3600, now+3600*2, []string{chunkHash5, chunkHash5}, "tag")
+	checkTestSnapshots(snapshotManager, 3, 2)
+
+	// Run the prune again and this time the fossil collection will be processed and the fossils removed
+	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false, 1)
+	checkTestSnapshots(snapshotManager, 3, 0)
+	snapshotManager.CheckSnapshots("vm1@host1", []int{2, 3, 4}, "", false, false, false, false, false)
+}

src/duplicacy_storage.go

@@ -89,7 +89,7 @@ func (storage *StorageBase) SetRateLimits(downloadRateLimit int, uploadRateLimit
 }
 
 // SetDefaultNestingLevels sets the default read and write levels.  This is usually called by
-// derived storages to set the levels with old values so that storages initialied by ealier versions
+// derived storages to set the levels with old values so that storages initialized by earlier versions
 // will continue to work.
 func (storage *StorageBase) SetDefaultNestingLevels(readLevels []int, writeLevel int) {
 	storage.readLevels = readLevels
@@ -291,6 +291,7 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
 
 		// If ssh_key_file is set, skip password-based login
 		keyFile := GetPasswordFromPreference(preference, "ssh_key_file")
+		passphrase := ""
 
 		password := ""
 		passwordCallback := func() (string, error) {
@@ -348,7 +349,20 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
 				} else {
 					key, err = ssh.ParsePrivateKey(content)
 					if err != nil {
-						LOG_INFO("SSH_PUBLICKEY", "Failed to parse the private key file %s: %v", keyFile, err)
+						if strings.Contains(err.Error(), "cannot decode encrypted private keys") {
+							LOG_TRACE("SSH_PUBLICKEY", "The private key file is encrypted")
+							passphrase = GetPassword(preference, "ssh_passphrase", "Enter the passphrase to decrypt the private key file:", false, resetPassword)
+							if len(passphrase) == 0 {
+								LOG_INFO("SSH_PUBLICKEY", "No passphrase to descrypt the private key file %s", keyFile)
+							} else {
+								key, err = ssh.ParsePrivateKeyWithPassphrase(content, []byte(passphrase))
+								if err != nil {
+									LOG_INFO("SSH_PUBLICKEY", "Failed to parse the encrypted private key file %s: %v", keyFile, err)
+								}
+							}
+						} else {
+							LOG_INFO("SSH_PUBLICKEY", "Failed to parse the private key file %s: %v", keyFile, err)
+						}
 					}
 				}
 			}
@@ -410,6 +424,9 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
 
 		if keyFile != "" {
 			SavePassword(preference, "ssh_key_file", keyFile)
+			if passphrase != "" {
+				SavePassword(preference, "ssh_passphrase", passphrase)
+			}
 		} else if password != "" {
 			SavePassword(preference, "ssh_password", password)
 		}
@@ -509,11 +526,12 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
 		return dropboxStorage
 	} else if matched[1] == "b2" {
 		bucket := matched[3]
+		storageDir := matched[5]
 
-		accountID := GetPassword(preference, "b2_id", "Enter Backblaze Account ID:", true, resetPassword)
-		applicationKey := GetPassword(preference, "b2_key", "Enter Backblaze Application Key:", true, resetPassword)
+		accountID := GetPassword(preference, "b2_id", "Enter Backblaze account or application id:", true, resetPassword)
+		applicationKey := GetPassword(preference, "b2_key", "Enter corresponding Backblaze application key:", true, resetPassword)
 
-		b2Storage, err := CreateB2Storage(accountID, applicationKey, bucket, threads)
+		b2Storage, err := CreateB2Storage(accountID, applicationKey, bucket, storageDir, threads)
 		if err != nil {
 			LOG_ERROR("STORAGE_CREATE", "Failed to load the Backblaze B2 storage at %s: %v", storageURL, err)
 			return nil
@@ -609,7 +627,7 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
 	} else if matched[1] == "webdav" || matched[1] == "webdav-http" {
 		server := matched[3]
 		username := matched[2]
-		username = username[:len(username) - 1]
+		username = username[:len(username)-1]
 		storageDir := matched[5]
 		port := 0
 		useHTTP := matched[1] == "webdav-http"

src/duplicacy_storage_test.go

@@ -80,12 +80,12 @@ func loadStorage(localStoragePath string, threads int) (Storage, error) {
 		return storage, err
 	} else if testStorageName == "s3" {
 		storage, err := CreateS3Storage(config["region"], config["endpoint"], config["bucket"], config["directory"], config["access_key"], config["secret_key"], threads, true, false)
-		return storage, err
 		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
+		return storage, err
 	} else if testStorageName == "wasabi" {
 		storage, err := CreateWasabiStorage(config["region"], config["endpoint"], config["bucket"], config["directory"], config["access_key"], config["secret_key"], threads)
-		return storage, err
 		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
+		return storage, err
 	} else if testStorageName == "s3c" {
 		storage, err := CreateS3CStorage(config["region"], config["endpoint"], config["bucket"], config["directory"], config["access_key"], config["secret_key"], threads)
 		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
@@ -107,7 +107,7 @@ func loadStorage(localStoragePath string, threads int) (Storage, error) {
 		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
 		return storage, err
 	} else if testStorageName == "b2" {
-		storage, err := CreateB2Storage(config["account"], config["key"], config["bucket"], threads)
+		storage, err := CreateB2Storage(config["account"], config["key"], config["bucket"], config["directory"], threads)
 		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
 		return storage, err
 	} else if testStorageName == "gcs-s3" {
@@ -153,10 +153,7 @@ func loadStorage(localStoragePath string, threads int) (Storage, error) {
 		}
 		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
 		return storage, err
-	} else {
-		return nil, fmt.Errorf("Invalid storage named: %s", testStorageName)
 	}
-
 	return nil, fmt.Errorf("Invalid storage named: %s", testStorageName)
 }
 
@@ -299,7 +296,8 @@ func TestStorage(t *testing.T) {
 
 	LOG_INFO("STORAGE_TEST", "storage: %s", testStorageName)
 
-	storage, err := loadStorage(testDir, 1)
+	threads := 8
+	storage, err := loadStorage(testDir, threads)
 	if err != nil {
 		t.Errorf("Failed to create storage: %v", err)
 		return
@@ -329,16 +327,16 @@ func TestStorage(t *testing.T) {
 	storage.CreateDirectory(0, "shared")
 
 	// Upload to the same directory by multiple goroutines
-	count := 8
+	count := threads
 	finished := make(chan int, count)
 	for i := 0; i < count; i++ {
-		go func(name string) {
-			err := storage.UploadFile(0, name, []byte("this is a test file"))
+		go func(threadIndex int, name string) {
+			err := storage.UploadFile(threadIndex, name, []byte("this is a test file"))
 			if err != nil {
 				t.Errorf("Error to upload '%s': %v", name, err)
 			}
 			finished <- 0
-		}(fmt.Sprintf("shared/a/b/c/%d", i))
+		}(i, fmt.Sprintf("shared/a/b/c/%d", i))
 	}
 
 	for i := 0; i < count; i++ {
@@ -387,7 +385,6 @@ func TestStorage(t *testing.T) {
 
 	snapshotIDs := []string{}
 	for _, snapshotDir := range snapshotDirs {
-		LOG_INFO("debug", "snapshot dir: %s", snapshotDir)
 		if len(snapshotDir) > 0 && snapshotDir[len(snapshotDir)-1] == '/' {
 			snapshotIDs = append(snapshotIDs, snapshotDir[:len(snapshotDir)-1])
 		}

src/duplicacy_swiftstorage.go

@@ -21,7 +21,7 @@ type SwiftStorage struct {
 	threads    int
 }
 
-// CreateSwiftStorage creates an OpenStack Swift storage object.  storageURL is in the form of 
+// CreateSwiftStorage creates an OpenStack Swift storage object.  storageURL is in the form of
 // `user@authURL/container/path?arg1=value1&arg2=value2``
 func CreateSwiftStorage(storageURL string, key string, threads int) (storage *SwiftStorage, err error) {
 

src/duplicacy_utils.go

@@ -10,10 +10,7 @@ import (
 	"fmt"
 	"io"
 	"os"
-	"path"
-	"path/filepath"
 	"regexp"
-	"runtime"
 	"strconv"
 	"strings"
 	"time"
@@ -58,7 +55,7 @@ func IsEmptyFilter(pattern string) bool {
 }
 
 func IsUnspecifiedFilter(pattern string) bool {
-	if pattern[0] != '+' && pattern[0] != '-' && pattern[0] != 'i' && pattern[0] != 'e' {
+	if pattern[0] != '+' && pattern[0] != '-' && !strings.HasPrefix(pattern, "i:") && !strings.HasPrefix(pattern, "e:")  {
 		return true
 	} else {
 		return false
@@ -176,6 +173,15 @@ func GetPasswordFromPreference(preference Preference, passwordType string) strin
 		if password, found := os.LookupEnv(name); found && password != "" {
 			return password
 		}
+
+		re := regexp.MustCompile(`[^a-zA-Z0-9_]`)
+		namePlain := re.ReplaceAllString(name, "_")
+		if namePlain != name {
+			LOG_DEBUG("PASSWORD_ENV_VAR", "Reading the environment variable %s", namePlain)
+			if password, found := os.LookupEnv(namePlain); found && password != "" {
+				return password
+			}
+		}
 	}
 
 	// If the password is stored in the preference, there is no need to include the storage name
@@ -390,19 +396,6 @@ func MatchPath(filePath string, patterns []string) (included bool) {
 	}
 }
 
-func joinPath(components ...string) string {
-
-	combinedPath := path.Join(components...)
-	if len(combinedPath) > 257 && runtime.GOOS == "windows" {
-		combinedPath = `\\?\` + filepath.Join(components...)
-		// If the path is on a samba drive we must use the UNC format
-		if strings.HasPrefix(combinedPath, `\\?\\\`) {
-			combinedPath = `\\?\UNC\` + combinedPath[6:]
-		}
-	}
-	return combinedPath
-}
-
 func PrettyNumber(number int64) string {
 
 	G := int64(1024 * 1024 * 1024)
@@ -467,10 +460,3 @@ func AtoSize(sizeString string) int {
 
 	return size
 }
-
-func MinInt(x, y int) int {
-	if x < y {
-		return x
-	}
-	return y
-}

src/duplicacy_utils_others.go

@@ -9,6 +9,7 @@ package duplicacy
 import (
 	"bytes"
 	"os"
+	"path"
 	"path/filepath"
 	"syscall"
 
@@ -35,7 +36,7 @@ func SetOwner(fullPath string, entry *Entry, fileInfo *os.FileInfo) bool {
 	stat, ok := (*fileInfo).Sys().(*syscall.Stat_t)
 	if ok && stat != nil && (int(stat.Uid) != entry.UID || int(stat.Gid) != entry.GID) {
 		if entry.UID != -1 && entry.GID != -1 {
-			err := os.Chown(fullPath, entry.UID, entry.GID)
+			err := os.Lchown(fullPath, entry.UID, entry.GID)
 			if err != nil {
 				LOG_ERROR("RESTORE_CHOWN", "Failed to change uid or gid: %v", err)
 				return false
@@ -83,3 +84,11 @@ func (entry *Entry) SetAttributesToFile(fullPath string) {
 	}
 
 }
+
+func joinPath(components ...string) string {
+	return path.Join(components...)
+}
+
+func SplitDir(fullPath string) (dir string, file string) {
+	return path.Split(fullPath)
+}

src/duplicacy_utils_test.go

@@ -92,6 +92,17 @@ func TestMatchPattern(t *testing.T) {
 		}
 	}
 
+	for _, pattern := range []string{ "+", "-", "i:", "e:", "+a", "-a", "i:a", "e:a"} {
+		if IsUnspecifiedFilter(pattern) {
+			t.Errorf("pattern %s has a specified filter", pattern)
+		}
+	}
+
+	for _, pattern := range []string{ "i", "e", "ia", "ib", "a", "b"} {
+		if !IsUnspecifiedFilter(pattern) {
+			t.Errorf("pattern %s does not have a specified filter", pattern)
+		}
+	}
 }
 
 func TestRateLimit(t *testing.T) {

src/duplicacy_utils_windows.go

@@ -7,6 +7,8 @@ package duplicacy
 import (
 	"fmt"
 	"os"
+	"path/filepath"
+	"strings"
 	"syscall"
 	"unsafe"
 )
@@ -114,3 +116,18 @@ func (entry *Entry) ReadAttributes(top string) {
 func (entry *Entry) SetAttributesToFile(fullPath string) {
 
 }
+
+func joinPath(components ...string) string {
+
+	combinedPath := `\\?\` + filepath.Join(components...)
+	// If the path is on a samba drive we must use the UNC format
+	if strings.HasPrefix(combinedPath, `\\?\\\`) {
+		combinedPath = `\\?\UNC\` + combinedPath[6:]
+	}
+	return combinedPath
+}
+
+func SplitDir(fullPath string) (dir string, file string) {
+	i := strings.LastIndex(fullPath, "\\")
+	return fullPath[:i+1], fullPath[i+1:]
+}

src/duplicacy_wasabistorage.go

@@ -93,43 +93,49 @@ func (storage *WasabiStorage) DeleteFile(
 // rename.  It's designed to get the job done with as few dependencies
 // on other packages as possible rather than being somethng
 // general-purpose and reusable.
-func (storage *WasabiStorage) MoveFile(
-	threadIndex int, from string, to string,
-) (err error) {
+func (storage *WasabiStorage) MoveFile(threadIndex int, from string, to string) (err error) {
 
-	// The from path includes the bucket
-	from_path := fmt.Sprintf("/%s/%s/%s", storage.bucket, storage.storageDir, from)
+	var fromPath string
+	// The from path includes the bucket.  Take care not to include an empty storageDir
+	// string as Wasabi's backend will return 404 on URLs with double slashes.
+	if storage.storageDir == "" {
+		fromPath = fmt.Sprintf("/%s/%s", storage.bucket, from)
+	} else {
+		fromPath = fmt.Sprintf("/%s/%s/%s", storage.bucket, storage.storageDir, from)
+	}
 
-	object := fmt.Sprintf("https://%s@%s%s",
-		storage.region, storage.endpoint, from_path)
+	object := fmt.Sprintf("https://%s@%s%s", storage.region, storage.endpoint, fromPath)
 
+	toPath := to
 	// The object's new name is relative to the top of the bucket.
-	new_name := fmt.Sprintf("%s/%s", storage.storageDir, to)
+	if storage.storageDir != "" {
+		toPath = fmt.Sprintf("%s/%s", storage.storageDir, to)
+	}
 
 	timestamp := time.Now().Format(time.RFC1123Z)
 
-	signing_string := fmt.Sprintf("MOVE\n\n\n%s\n%s", timestamp, from_path)
+	signingString := fmt.Sprintf("MOVE\n\n\n%s\n%s", timestamp, fromPath)
 
 	signer := hmac.New(sha1.New, []byte(storage.secret))
-	signer.Write([]byte(signing_string))
+	signer.Write([]byte(signingString))
 
 	signature := base64.StdEncoding.EncodeToString(signer.Sum(nil))
 
 	authorization := fmt.Sprintf("AWS %s:%s", storage.key, signature)
 
-	request, error := http.NewRequest("MOVE", object, nil)
-	if error != nil {
-		return error
+	request, err := http.NewRequest("MOVE", object, nil)
+	if err != nil {
+		return err
 	}
 	request.Header.Add("Authorization", authorization)
 	request.Header.Add("Date", timestamp)
-	request.Header.Add("Destination", new_name)
+	request.Header.Add("Destination", toPath)
 	request.Header.Add("Host", storage.endpoint)
 	request.Header.Add("Overwrite", "true")
 
-	response, error := storage.client.Do(request)
-	if error != nil {
-		return error
+	response, err := storage.client.Do(request)
+	if err != nil {
+		return err
 	}
 	defer response.Body.Close()
 

src/duplicacy_webdavstorage.go

@@ -19,9 +19,9 @@ import (
 	"net/http"
 	//"net/http/httputil"
 	"strconv"
+	"strings"
 	"sync"
 	"time"
-	"strings"
 )
 
 type WebDAVStorage struct {
@@ -42,14 +42,14 @@ type WebDAVStorage struct {
 
 var (
 	errWebDAVAuthorizationFailure = errors.New("Authentication failed")
-	errWebDAVMovedPermanently = errors.New("Moved permanently")
-	errWebDAVNotExist = errors.New("Path does not exist")
-	errWebDAVMaximumBackoff = errors.New("Maximum backoff reached")
-	errWebDAVMethodNotAllowed = errors.New("Method not allowed")
+	errWebDAVMovedPermanently     = errors.New("Moved permanently")
+	errWebDAVNotExist             = errors.New("Path does not exist")
+	errWebDAVMaximumBackoff       = errors.New("Maximum backoff reached")
+	errWebDAVMethodNotAllowed     = errors.New("Method not allowed")
 )
 
 func CreateWebDAVStorage(host string, port int, username string, password string, storageDir string, useHTTP bool, threads int) (storage *WebDAVStorage, err error) {
-	if storageDir[len(storageDir)-1] != '/' {
+	if len(storageDir) > 0 && storageDir[len(storageDir)-1] != '/' {
 		storageDir += "/"
 	}
 
@@ -59,7 +59,7 @@ func CreateWebDAVStorage(host string, port int, username string, password string
 		username:   username,
 		password:   password,
 		storageDir: "",
-		useHTTP:    false,
+		useHTTP:    useHTTP,
 
 		client:         http.DefaultClient,
 		threads:        threads,
@@ -68,7 +68,7 @@ func CreateWebDAVStorage(host string, port int, username string, password string
 
 	// Make sure it doesn't follow redirect
 	storage.client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
-        return http.ErrUseLastResponse
+		return http.ErrUseLastResponse
 	}
 
 	exist, isDir, _, err := storage.GetFileInfo(0, storageDir)
@@ -151,6 +151,10 @@ func (storage *WebDAVStorage) sendRequest(method string, uri string, depth int,
 			request.Header.Set(key, value)
 		}
 
+		if method == "PUT" {
+			request.ContentLength = int64(len(data))
+		}
+
 		//requestDump, err := httputil.DumpRequest(request, true)
 		//LOG_INFO("debug", "Request: %s", requestDump)
 
@@ -313,6 +317,7 @@ func (storage *WebDAVStorage) ListFiles(threadIndex int, dir string) (files []st
 
 // GetFileInfo returns the information about the file or directory at 'filePath'.
 func (storage *WebDAVStorage) GetFileInfo(threadIndex int, filePath string) (exist bool, isDir bool, size int64, err error) {
+
 	properties, err := storage.getProperties(filePath, 0, "getcontentlength", "resourcetype")
 	if err != nil {
 		if err == errWebDAVNotExist {
@@ -325,11 +330,18 @@ func (storage *WebDAVStorage) GetFileInfo(threadIndex int, filePath string) (exi
 		return false, false, 0, err
 	}
 
-	if m, exist := properties["/" + storage.storageDir + filePath]; !exist {
+	m, exist := properties["/"+storage.storageDir+filePath]
+
+	// If no properties exist for the given filePath, remove the trailing / from filePath and search again
+	if !exist && filePath != "" && filePath[len(filePath) - 1] == '/' {
+		m, exist = properties["/"+storage.storageDir+filePath[:len(filePath) - 1]]
+	}
+
+	if !exist {
 		return false, false, 0, nil
 	} else if resourceType, exist := m["resourcetype"]; exist && strings.Contains(resourceType, "collection") {
 		return true, true, 0, nil
-	} else if length, exist := m["getcontentlength"]; exist && length != ""{
+	} else if length, exist := m["getcontentlength"]; exist && length != "" {
 		value, _ := strconv.Atoi(length)
 		return true, false, int64(value), nil
 	} else {