Add src/duplicacy_utils_freebsd.go

Add exclude_by_attribute preference to exclude files based on xattr

.github/ISSUE_TEMPLATE.md

@@ -1,5 +1,17 @@
-Please submit an issue for bug reports or feature requests.  If you have any questions please post them on https://forum.duplicacy.com.
+---
+name: Please use the official forum
+about: Please use the official forum instead of Github
+title: 'Please use the official forum'
+labels: ''
+assignees: ''
 
-When you're reporting a bug, please specify the OS, version, command line arguments, or any info that you think is helpful for the diagnosis.  If Duplicacy reports an error, please post the program output here.
+---
 
-Note that this repository hosts the CLI version of Duplicacy only.  If you're reporting anything related to the GUI version, please visit https://forum.duplicacy.com.
+
+Please **use the [Duplicacy Forum](https://forum.duplicacy.com/)** when reporting bugs, making feature requests, asking for help or simply praising Duplicacy for its ease of use.
+
+We strongly encourage you to create an account on the forum and use that platform for discussion as there is a higher chance that someone there will talk to you. 
+
+There is a handful of people watching the Github Issues and we are in the process of moving **all** of them to the forum as well. Most likely you will not receive an answer here or it will be very slow and you will be pointed to the forum.
+
+We have already created a comprehensive [Guide](https://forum.duplicacy.com/t/duplicacy-user-guide/1197), and a [How-To](https://forum.duplicacy.com/c/how-to) category which stores more wisdom than these issues on Github.

ACKNOWLEDGEMENTS.md

@@ -14,3 +14,4 @@ Duplicacy is based on the following open source projects:
 |https://github.com/pcwizz/xattr            |  BSD-2-Clause |
 |https://github.com/minio/blake2b-simd      |  Apache-2.0 |
 |https://github.com/go-ole/go-ole           |  MIT |
+https://github.com/ncw/swift                |  MIT |

Gopkg.lock

@@ -7,17 +7,11 @@
   revision = "2d3a6656c17a60b0815b7e06ab0be04eacb6e613"
   version = "v0.16.0"
 
-[[projects]]
-  name = "github.com/Azure/azure-sdk-for-go"
-  packages = ["version"]
-  revision = "b7fadebe0e7f5c5720986080a01495bd8d27be37"
-  version = "v14.2.0"
-
 [[projects]]
   name = "github.com/Azure/go-autorest"
-  packages = ["autorest","autorest/adal","autorest/azure","autorest/date"]
-  revision = "0ae36a9e544696de46fdadb7b0d5fb38af48c063"
-  version = "v10.2.0"
+  packages = ["autorest","autorest/adal","autorest/azure","autorest/date","logger","version"]
+  revision = "9bc4033dd347c7f416fca46b2f42a043dc1fbdf6"
+  version = "v10.15.5"
 
 [[projects]]
   branch = "master"
@@ -27,9 +21,9 @@
 
 [[projects]]
   name = "github.com/aws/aws-sdk-go"
-  packages = ["aws","aws/awserr","aws/awsutil","aws/client","aws/client/metadata","aws/corehandlers","aws/credentials","aws/credentials/ec2rolecreds","aws/credentials/endpointcreds","aws/credentials/stscreds","aws/defaults","aws/ec2metadata","aws/endpoints","aws/request","aws/session","aws/signer/v4","internal/shareddefaults","private/protocol","private/protocol/query","private/protocol/query/queryutil","private/protocol/rest","private/protocol/restxml","private/protocol/xml/xmlutil","service/s3","service/sts"]
-  revision = "a32b1dcd091264b5dee7b386149b6cc3823395c9"
-  version = "v1.12.31"
+  packages = ["aws","aws/arn","aws/awserr","aws/awsutil","aws/client","aws/client/metadata","aws/corehandlers","aws/credentials","aws/credentials/ec2rolecreds","aws/credentials/endpointcreds","aws/credentials/processcreds","aws/credentials/stscreds","aws/csm","aws/defaults","aws/ec2metadata","aws/endpoints","aws/request","aws/session","aws/signer/v4","internal/context","internal/ini","internal/s3err","internal/sdkio","internal/sdkmath","internal/sdkrand","internal/sdkuri","internal/shareddefaults","internal/strings","internal/sync/singleflight","private/protocol","private/protocol/eventstream","private/protocol/eventstream/eventstreamapi","private/protocol/json/jsonutil","private/protocol/query","private/protocol/query/queryutil","private/protocol/rest","private/protocol/restxml","private/protocol/xml/xmlutil","service/s3","service/s3/internal/arn","service/sts","service/sts/stsiface"]
+  revision = "851d5ffb66720c2540cc68020d4d8708950686c8"
+  version = "v1.30.7"
 
 [[projects]]
   name = "github.com/bkaradzic/go-lz4"
@@ -40,14 +34,14 @@
 [[projects]]
   name = "github.com/dgrijalva/jwt-go"
   packages = ["."]
-  revision = "dbeaa9332f19a944acb5736b4456cfcc02140e29"
-  version = "v3.1.0"
+  revision = "06ea1031745cb8b3dab3f6a236daf2b0aa468b7e"
+  version = "v3.2.0"
 
 [[projects]]
   branch = "master"
   name = "github.com/gilbertchen/azure-sdk-for-go"
-  packages = ["storage"]
-  revision = "bbf89bd4d716c184f158d1e1428c2dbef4a18307"
+  packages = ["storage","version"]
+  revision = "8fd4663cab7c7c1c46d00449291c92ad23b0d0d9"
 
 [[projects]]
   branch = "master"
@@ -56,10 +50,9 @@
   revision = "1de0a1836ce9c3ae1bf737a0869c4f04f28a7f98"
 
 [[projects]]
-  branch = "master"
   name = "github.com/gilbertchen/go-dropbox"
   packages = ["."]
-  revision = "90711b603312b1f973f3a5da3793ac4f1e5c2f2a"
+  revision = "0baa9015ac2547d8b69b2e88c709aa90cfb8fbc1"
 
 [[projects]]
   name = "github.com/gilbertchen/go-ole"
@@ -71,7 +64,7 @@
   branch = "master"
   name = "github.com/gilbertchen/go.dbus"
   packages = ["."]
-  revision = "9e442e6378618c083fd3b85b703ffd202721fb17"
+  revision = "8591994fa32f1dbe3fa9486bc6f4d4361ac16649"
 
 [[projects]]
   branch = "master"
@@ -98,33 +91,45 @@
   revision = "68e7a6806b0137a396d7d05601d7403ae1abac58"
 
 [[projects]]
-  name = "github.com/go-ini/ini"
-  packages = ["."]
-  revision = "32e4c1e6bc4e7d0d8451aa6b75200d19e37a536a"
-  version = "v1.32.0"
+  branch = "master"
+  name = "github.com/golang/groupcache"
+  packages = ["lru"]
+  revision = "8c9f03a8e57eb486e42badaed3fb287da51807ba"
 
 [[projects]]
-  branch = "master"
   name = "github.com/golang/protobuf"
   packages = ["proto","protoc-gen-go/descriptor","ptypes","ptypes/any","ptypes/duration","ptypes/timestamp"]
-  revision = "1e59b77b52bf8e4b449a57e6f79f21226d571845"
+  revision = "84668698ea25b64748563aa20726db66a6b8d299"
+  version = "v1.3.5"
 
 [[projects]]
   name = "github.com/googleapis/gax-go"
-  packages = ["."]
-  revision = "317e0006254c44a0ac427cc52a0e083ff0b9622f"
-  version = "v2.0.0"
+  packages = [".","v2"]
+  revision = "c8a15bac9b9fe955bd9f900272f9a306465d28cf"
+  version = "v2.0.3"
 
 [[projects]]
   name = "github.com/jmespath/go-jmespath"
   packages = ["."]
-  revision = "0b12d6b5"
+  revision = "c2b33e84"
+
+[[projects]]
+  name = "github.com/klauspost/cpuid"
+  packages = ["."]
+  revision = "750c0591dbbd50ef88371c665ad49e426a4b830b"
+  version = "v1.3.1"
+
+[[projects]]
+  name = "github.com/klauspost/reedsolomon"
+  packages = ["."]
+  revision = "7daa20bf74337a939c54f892a2eca9d9b578eb7f"
+  version = "v1.9.9"
 
 [[projects]]
-  branch = "master"
   name = "github.com/kr/fs"
   packages = ["."]
-  revision = "2788f0dbd16903de03cb8186e5c7d97b69ad387b"
+  revision = "1455def202f6e05b95cc7bfc7e8ae67ae5141eba"
+  version = "v0.1.0"
 
 [[projects]]
   name = "github.com/marstr/guid"
@@ -138,23 +143,35 @@
   packages = ["."]
   revision = "3f5f724cb5b182a5c278d6d3d55b40e7f8c2efb4"
 
+[[projects]]
+  name = "github.com/minio/highwayhash"
+  packages = ["."]
+  revision = "86a2a969d04373bf05ca722517d30fb1c9a3e4f9"
+  version = "v1.0.1"
+
 [[projects]]
   branch = "master"
+  name = "github.com/mmcloughlin/avo"
+  packages = ["attr","build","buildtags","gotypes","internal/prnt","internal/stack","ir","operand","pass","printer","reg","src","x86"]
+  revision = "443f81d771042b019379ae4bfcd0a591cb47c88a"
+
+[[projects]]
   name = "github.com/ncw/swift"
   packages = ["."]
-  revision = "ae9f0ea1605b9aa6434ed5c731ca35d83ba67c55"
+  revision = "3e1a09f21340e4828e7265aa89f4dc1495fa7ccc"
+  version = "v1.0.50"
 
 [[projects]]
   name = "github.com/pkg/errors"
   packages = ["."]
-  revision = "645ef00459ed84a119197bfb8d8205042c6df63d"
-  version = "v0.8.0"
+  revision = "614d223910a179a466c1767a985424175c39b465"
+  version = "v0.9.1"
 
 [[projects]]
   name = "github.com/pkg/sftp"
   packages = ["."]
-  revision = "98203f5a8333288eb3163b7c667d4260fe1333e9"
-  version = "1.0.0"
+  revision = "5616182052227b951e76d9c9b79a616c608bd91b"
+  version = "v1.11.0"
 
 [[projects]]
   name = "github.com/satori/go.uuid"
@@ -168,63 +185,86 @@
   packages = ["."]
   revision = "a98ad7ee00ec53921f08832bc06ecf7fd600e6a1"
 
+[[projects]]
+  name = "go.opencensus.io"
+  packages = [".","internal","internal/tagencoding","metric/metricdata","metric/metricproducer","plugin/ochttp","plugin/ochttp/propagation/b3","resource","stats","stats/internal","stats/view","tag","trace","trace/internal","trace/propagation","trace/tracestate"]
+  revision = "d835ff86be02193d324330acdb7d65546b05f814"
+  version = "v0.22.3"
+
 [[projects]]
   branch = "master"
   name = "golang.org/x/crypto"
-  packages = ["curve25519","ed25519","ed25519/internal/edwards25519","pbkdf2","ssh","ssh/agent","ssh/terminal"]
-  revision = "9f005a07e0d31d45e6656d241bb5c0f2efd4bc94"
+  packages = ["blowfish","chacha20","curve25519","ed25519","ed25519/internal/edwards25519","internal/subtle","pbkdf2","poly1305","ssh","ssh/agent","ssh/internal/bcrypt_pbkdf","ssh/terminal"]
+  revision = "056763e48d71961566155f089ac0f02f1dda9b5a"
+
+[[projects]]
+  name = "golang.org/x/mod"
+  packages = ["semver"]
+  revision = "859b3ef565e237f9f1a0fb6b55385c497545680d"
+  version = "v0.3.0"
 
 [[projects]]
   branch = "master"
   name = "golang.org/x/net"
-  packages = ["context","context/ctxhttp","http2","http2/hpack","idna","internal/timeseries","lex/httplex","trace"]
-  revision = "9dfe39835686865bff950a07b394c12a98ddc811"
+  packages = ["context","context/ctxhttp","http/httpguts","http2","http2/hpack","idna","internal/timeseries","trace"]
+  revision = "d3edc9973b7eb1fb302b0ff2c62357091cea9a30"
 
 [[projects]]
-  branch = "master"
   name = "golang.org/x/oauth2"
   packages = [".","google","internal","jws","jwt"]
-  revision = "f95fa95eaa936d9d87489b15d1d18b97c1ba9c28"
+  revision = "bf48bf16ab8d622ce64ec6ce98d2c98f916b6303"
 
 [[projects]]
   branch = "master"
   name = "golang.org/x/sys"
-  packages = ["unix","windows"]
-  revision = "82aafbf43bf885069dc71b7e7c2f9d7a614d47da"
+  packages = ["cpu","unix","windows"]
+  revision = "59c9f1ba88faf592b225274f69c5ef1e4ebacf82"
 
 [[projects]]
-  branch = "master"
   name = "golang.org/x/text"
-  packages = ["collate","collate/build","internal/colltab","internal/gen","internal/tag","internal/triegen","internal/ucd","language","secure/bidirule","transform","unicode/bidi","unicode/cldr","unicode/norm","unicode/rangetable"]
-  revision = "88f656faf3f37f690df1a32515b479415e1a6769"
+  packages = ["collate","collate/build","internal/colltab","internal/gen","internal/language","internal/language/compact","internal/tag","internal/triegen","internal/ucd","language","secure/bidirule","transform","unicode/bidi","unicode/cldr","unicode/norm","unicode/rangetable"]
+  revision = "342b2e1fbaa52c93f31447ad2c6abc048c63e475"
+  version = "v0.3.2"
 
 [[projects]]
   branch = "master"
+  name = "golang.org/x/tools"
+  packages = ["go/ast/astutil","go/gcexportdata","go/internal/gcimporter","go/internal/packagesdriver","go/packages","go/types/typeutil","internal/event","internal/event/core","internal/event/keys","internal/event/label","internal/gocommand","internal/packagesinternal","internal/typesinternal"]
+  revision = "5d1fdd8fa3469142b9369713b23d8413d6d83189"
+
+[[projects]]
+  branch = "master"
+  name = "golang.org/x/xerrors"
+  packages = [".","internal"]
+  revision = "5ec99f83aff198f5fbd629d6c8d8eb38a04218ca"
+
+[[projects]]
   name = "google.golang.org/api"
-  packages = ["drive/v3","gensupport","googleapi","googleapi/internal/uritemplates","googleapi/transport","internal","iterator","option","storage/v1","transport/http"]
-  revision = "17b5f22a248d6d3913171c1a557552ace0d9c806"
+  packages = ["drive/v3","googleapi","googleapi/transport","internal","internal/gensupport","internal/third_party/uritemplates","iterator","option","option/internaloption","storage/v1","transport/cert","transport/http","transport/http/internal/propagation"]
+  revision = "52f0532eadbcc6f6b82d6f5edf66e610d10bfde6"
+  version = "v0.21.0"
 
 [[projects]]
   name = "google.golang.org/appengine"
   packages = [".","internal","internal/app_identity","internal/base","internal/datastore","internal/log","internal/modules","internal/remote_api","internal/urlfetch","urlfetch"]
-  revision = "150dc57a1b433e64154302bdc40b6bb8aefa313a"
-  version = "v1.0.0"
+  revision = "971852bfffca25b069c31162ae8f247a3dba083b"
+  version = "v1.6.5"
 
 [[projects]]
   branch = "master"
   name = "google.golang.org/genproto"
-  packages = ["googleapis/api/annotations","googleapis/iam/v1","googleapis/rpc/status"]
-  revision = "891aceb7c239e72692819142dfca057bdcbfcb96"
+  packages = ["googleapis/api/annotations","googleapis/iam/v1","googleapis/rpc/status","googleapis/type/expr"]
+  revision = "baae70f3302d3efdff74db41e48a5d476d036906"
 
 [[projects]]
   name = "google.golang.org/grpc"
-  packages = [".","balancer","balancer/roundrobin","codes","connectivity","credentials","encoding","grpclb/grpc_lb_v1/messages","grpclog","internal","keepalive","metadata","naming","peer","resolver","resolver/dns","resolver/passthrough","stats","status","tap","transport"]
-  revision = "5a9f7b402fe85096d2e1d0383435ee1876e863d0"
-  version = "v1.8.0"
+  packages = [".","attributes","backoff","balancer","balancer/base","balancer/roundrobin","binarylog/grpc_binarylog_v1","codes","connectivity","credentials","credentials/internal","encoding","encoding/proto","grpclog","internal","internal/backoff","internal/balancerload","internal/binarylog","internal/buffer","internal/channelz","internal/envconfig","internal/grpclog","internal/grpcrand","internal/grpcsync","internal/grpcutil","internal/resolver/dns","internal/resolver/passthrough","internal/syscall","internal/transport","keepalive","metadata","naming","peer","resolver","serviceconfig","stats","status","tap"]
+  revision = "ac54eec90516cee50fc6b9b113b34628a85f976f"
+  version = "v1.28.1"
 
 [solve-meta]
   analyzer-name = "dep"
   analyzer-version = 1
-  inputs-digest = "eff5ae2d9507f0d62cd2e5bdedebb5c59d64f70f476b087c01c35d4a5e1be72d"
+  inputs-digest = "0e6ea2be64dedc36cb9192f1d410917ea72896302011e55b6df5e4c00c1c2f1c"
   solver-name = "gps-cdcl"
   solver-version = 1

Gopkg.toml

@@ -31,7 +31,7 @@
 
 [[constraint]]
   name = "github.com/aws/aws-sdk-go"
-  version = "1.12.31"
+  version = "1.30.7"
 
 [[constraint]]
   name = "github.com/bkaradzic/go-lz4"
@@ -46,8 +46,8 @@
   name = "github.com/gilbertchen/cli"
 
 [[constraint]]
-  branch = "master"
   name = "github.com/gilbertchen/go-dropbox"
+  revision = "0baa9015ac2547d8b69b2e88c709aa90cfb8fbc1"
 
 [[constraint]]
   name = "github.com/gilbertchen/go-ole"
@@ -75,7 +75,7 @@
 
 [[constraint]]
   name = "github.com/pkg/sftp"
-  version = "1.0.0"
+  version = "1.10.1"
 
 [[constraint]]
   branch = "master"
@@ -86,9 +86,13 @@
   name = "golang.org/x/net"
 
 [[constraint]]
-  branch = "master"
   name = "golang.org/x/oauth2"
+  revision = "bf48bf16ab8d622ce64ec6ce98d2c98f916b6303"
 
 [[constraint]]
-  branch = "master"
   name = "google.golang.org/api"
+  version = "0.21.0"
+
+[[constraint]]
+  name = "google.golang.org/grpc"
+  version = "1.28.0"

LICENSE.md

@@ -1,8 +1,7 @@
 Copyright © 2017 Acrosync LLC
 
 * Free for personal use or commercial trial
-* Non-trial commercial use requires per-user CLI licenses available from [duplicacy.com](https://duplicacy.com/buy) at a cost of $20 per year
-* A user is defined as the computer account that creates or edits the files to be backed up; if a backup contains files created or edited by multiple users for commercial purposes, one CLI license is required for each user
+* Non-trial commercial use requires per-computer CLI licenses available from [duplicacy.com](https://duplicacy.com/buy.html) at a cost of $50 per year
 * The computer with a valid commercial license for the GUI version may run the CLI version without a CLI license
 * CLI licenses are not required to restore or manage backups; only the backup command requires valid CLI licenses
 * Modification and redistribution are permitted, but commercial use of derivative works is subject to the same requirements of this license

README.md

@@ -90,8 +90,7 @@ The following table compares the feature lists of all these backup tools:
 ## License
 
 * Free for personal use or commercial trial
-* Non-trial commercial use requires per-user CLI licenses available from [duplicacy.com](https://duplicacy.com/buy) at a cost of $20 per year
-* A user is defined as the computer account that creates or edits the files to be backed up; if a backup contains files created or edited by multiple users for commercial purposes, one CLI license is required for each user
+* Non-trial commercial use requires per-computer CLI licenses available from [duplicacy.com](https://duplicacy.com/buy.html) at a cost of $50 per year
 * The computer with a valid commercial license for the GUI version may run the CLI version without a CLI license
 * CLI licenses are not required to restore or manage backups; only the backup command requires valid CLI licenses
 * Modification and redistribution are permitted, but commercial use of derivative works is subject to the same requirements of this license

duplicacy/duplicacy_main.go

@@ -159,6 +159,10 @@ func setGlobalOptions(context *cli.Context) {
 		}()
 	}
 
+	for _, logID := range context.GlobalStringSlice("suppress") {
+		duplicacy.SuppressLog(logID)
+	}
+
 	duplicacy.RunInBackground = context.GlobalBool("background")
 }
 
@@ -201,13 +205,29 @@ func runScript(context *cli.Context, storageName string, phase string) bool {
 	}
 
 	if err != nil {
-		duplicacy.LOG_WARN("SCRIPT_ERROR", "Failed to run script: %v", err)
+		duplicacy.LOG_ERROR("SCRIPT_ERROR", "Failed to run %s script: %v", script, err)
 		return false
 	}
 
 	return true
 }
 
+func loadRSAPrivateKey(keyFile string, passphrase string, preference *duplicacy.Preference, backupManager *duplicacy.BackupManager, resetPasswords bool) {
+	if keyFile == "" {
+		return
+	}
+
+	prompt := fmt.Sprintf("Enter the passphrase for %s:", keyFile)
+	if passphrase == "" {
+		passphrase = duplicacy.GetPassword(*preference, "rsa_passphrase", prompt, false, resetPasswords)
+		backupManager.LoadRSAPrivateKey(keyFile, passphrase)
+		duplicacy.SavePassword(*preference, "rsa_passphrase", passphrase)
+	} else {
+		backupManager.LoadRSAPrivateKey(keyFile, passphrase)
+	}
+
+}
+
 func initRepository(context *cli.Context) {
 	configRepository(context, true)
 }
@@ -319,6 +339,11 @@ func configRepository(context *cli.Context, init bool) {
 	if preference.Encrypted {
 		prompt := fmt.Sprintf("Enter storage password for %s:", preference.StorageURL)
 		storagePassword = duplicacy.GetPassword(preference, "password", prompt, false, true)
+	} else {
+		if context.String("key") != "" {
+			duplicacy.LOG_ERROR("STORAGE_CONFIG", "RSA encryption can't be enabled with an unencrypted storage")
+			return
+		}
 	}
 
 	existingConfig, _, err := duplicacy.DownloadConfig(storage, storagePassword)
@@ -433,8 +458,26 @@ func configRepository(context *cli.Context, init bool) {
 		if iterations == 0 {
 			iterations = duplicacy.CONFIG_DEFAULT_ITERATIONS
 		}
+
+		dataShards := 0
+		parityShards := 0
+		shards := context.String("erasure-coding")
+		if shards != "" {
+			shardsRegex := regexp.MustCompile(`^([0-9]+):([0-9]+)$`)
+			matched := shardsRegex.FindStringSubmatch(shards)
+			if matched == nil {
+				duplicacy.LOG_ERROR("STORAGE_ERASURECODE", "Invalid erasure coding parameters: %s", shards)
+			} else {
+				dataShards, _ = strconv.Atoi(matched[1])
+				parityShards, _ = strconv.Atoi(matched[2])
+				if dataShards == 0 || dataShards > 256 || parityShards == 0 || parityShards > dataShards {
+					duplicacy.LOG_ERROR("STORAGE_ERASURECODE", "Invalid erasure coding parameters: %s", shards)
+				}
+			}
+		}
+
 		duplicacy.ConfigStorage(storage, iterations, compressionLevel, averageChunkSize, maximumChunkSize,
-			minimumChunkSize, storagePassword, otherConfig, bitCopy)
+			minimumChunkSize, storagePassword, otherConfig, bitCopy, context.String("key"), dataShards, parityShards)
 	}
 
 	duplicacy.Preferences = append(duplicacy.Preferences, preference)
@@ -532,7 +575,18 @@ func setPreference(context *cli.Context) {
 		newPreference.DoNotSavePassword = triBool.IsTrue()
 	}
 
-	newPreference.NobackupFile = context.String("nobackup-file")
+	if context.String("nobackup-file") != "" {
+		newPreference.NobackupFile = context.String("nobackup-file")
+	}
+
+	if context.String("filters") != "" {
+		newPreference.FiltersFile = context.String("filters")
+	}
+
+	triBool = context.Generic("exclude-by-attribute").(*TriBool)
+	if triBool.IsSet() {
+		newPreference.ExcludeByAttribute = triBool.IsTrue()
+	}
 
 	key := context.String("key")
 	value := context.String("value")
@@ -715,7 +769,7 @@ func backupRepository(context *cli.Context) {
 	uploadRateLimit := context.Int("limit-rate")
 	enumOnly := context.Bool("enum-only")
 	storage.SetRateLimits(0, uploadRateLimit)
-	backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, preference.NobackupFile)
+	backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, preference.NobackupFile, preference.FiltersFile, preference.ExcludeByAttribute)
 	duplicacy.SavePassword(*preference, "password", password)
 
 	backupManager.SetupSnapshotCache(preference.Name)
@@ -768,6 +822,7 @@ func restoreRepository(context *cli.Context) {
 	setOwner := !context.Bool("ignore-owner")
 
 	showStatistics := context.Bool("stats")
+	persist := context.Bool("persist")
 
 	var patterns []string
 	for _, pattern := range context.Args() {
@@ -782,33 +837,27 @@ func restoreRepository(context *cli.Context) {
 			pattern = pattern[1:]
 		}
 
-		if duplicacy.IsUnspecifiedFilter(pattern) {
-			pattern = "+" + pattern
-		}
-
-		if duplicacy.IsEmptyFilter(pattern) {
-			continue
-		}
-
-		if strings.HasPrefix(pattern, "i:") || strings.HasPrefix(pattern, "e:") {
-			valid, err := duplicacy.IsValidRegex(pattern[2:])
-			if !valid || err != nil {
-				duplicacy.LOG_ERROR("SNAPSHOT_FILTER", "Invalid regular expression encountered for filter: \"%s\", error: %v", pattern, err)
-			}
-		}
-
 		patterns = append(patterns, pattern)
-
 	}
 
+	patterns = duplicacy.ProcessFilterLines(patterns, make([]string, 0))
+
 	duplicacy.LOG_DEBUG("REGEX_DEBUG", "There are %d compiled regular expressions stored", len(duplicacy.RegexMap))
 
+	duplicacy.LOG_INFO("SNAPSHOT_FILTER", "Loaded %d include/exclude pattern(s)", len(patterns))
+
 	storage.SetRateLimits(context.Int("limit-rate"), 0)
-	backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, preference.NobackupFile)
+	backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, preference.NobackupFile, preference.FiltersFile, preference.ExcludeByAttribute)
 	duplicacy.SavePassword(*preference, "password", password)
 
+	loadRSAPrivateKey(context.String("key"), context.String("key-passphrase"), preference, backupManager, false)
+
 	backupManager.SetupSnapshotCache(preference.Name)
-	backupManager.Restore(repository, revision, true, quickMode, threads, overwrite, deleteMode, setOwner, showStatistics, patterns)
+	failed := backupManager.Restore(repository, revision, true, quickMode, threads, overwrite, deleteMode, setOwner, showStatistics, patterns, persist)
+	if failed > 0 {
+		duplicacy.LOG_ERROR("RESTORE_FAIL", "%d file(s) were not restored correctly", failed)
+		return
+	}
 
 	runScript(context, preference.Name, "post")
 }
@@ -844,7 +893,7 @@ func listSnapshots(context *cli.Context) {
 	tag := context.String("t")
 	revisions := getRevisions(context)
 
-	backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, preference.NobackupFile)
+	backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, "", "", preference.ExcludeByAttribute)
 	duplicacy.SavePassword(*preference, "password", password)
 
 	id := preference.SnapshotID
@@ -857,6 +906,9 @@ func listSnapshots(context *cli.Context) {
 	showFiles := context.Bool("files")
 	showChunks := context.Bool("chunks")
 
+	// list doesn't need to decrypt file chunks; but we need -key here so we can reset the passphrase for the private key
+	loadRSAPrivateKey(context.String("key"), "", preference, backupManager, resetPassword)
+
 	backupManager.SetupSnapshotCache(preference.Name)
 	backupManager.SnapshotManager.ListSnapshots(id, revisions, tag, showFiles, showChunks)
 
@@ -879,7 +931,12 @@ func checkSnapshots(context *cli.Context) {
 
 	runScript(context, preference.Name, "pre")
 
-	storage := duplicacy.CreateStorage(*preference, false, 1)
+	threads := context.Int("threads")
+	if threads < 1 {
+		threads = 1
+	}
+
+	storage := duplicacy.CreateStorage(*preference, false, threads)
 	if storage == nil {
 		return
 	}
@@ -892,9 +949,11 @@ func checkSnapshots(context *cli.Context) {
 	tag := context.String("t")
 	revisions := getRevisions(context)
 
-	backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, preference.NobackupFile)
+	backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, "", "", false)
 	duplicacy.SavePassword(*preference, "password", password)
 
+	loadRSAPrivateKey(context.String("key"), context.String("key-passphrase"), preference, backupManager, false)
+
 	id := preference.SnapshotID
 	if context.Bool("all") {
 		id = ""
@@ -905,11 +964,13 @@ func checkSnapshots(context *cli.Context) {
 	showStatistics := context.Bool("stats")
 	showTabular := context.Bool("tabular")
 	checkFiles := context.Bool("files")
+	checkChunks := context.Bool("chunks")
 	searchFossils := context.Bool("fossils")
 	resurrect := context.Bool("resurrect")
+	persist := context.Bool("persist")
 
 	backupManager.SetupSnapshotCache(preference.Name)
-	backupManager.SnapshotManager.CheckSnapshots(id, revisions, tag, showStatistics, showTabular, checkFiles, searchFossils, resurrect)
+	backupManager.SnapshotManager.CheckSnapshots(id, revisions, tag, showStatistics, showTabular, checkFiles, checkChunks, searchFossils, resurrect, threads, persist)
 
 	runScript(context, preference.Name, "post")
 }
@@ -947,9 +1008,12 @@ func printFile(context *cli.Context) {
 		snapshotID = context.String("id")
 	}
 
-	backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, preference.NobackupFile)
+
+	backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, "", "", false)
 	duplicacy.SavePassword(*preference, "password", password)
 
+	loadRSAPrivateKey(context.String("key"), context.String("key-passphrase"), preference, backupManager, false)
+
 	backupManager.SetupSnapshotCache(preference.Name)
 
 	file := ""
@@ -1003,11 +1067,13 @@ func diff(context *cli.Context) {
 	}
 
 	compareByHash := context.Bool("hash")
-	backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, preference.NobackupFile)
+	backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, "", "", false)
 	duplicacy.SavePassword(*preference, "password", password)
 
+	loadRSAPrivateKey(context.String("key"), context.String("key-passphrase"), preference, backupManager, false)
+
 	backupManager.SetupSnapshotCache(preference.Name)
-	backupManager.SnapshotManager.Diff(repository, snapshotID, revisions, path, compareByHash, preference.NobackupFile)
+	backupManager.SnapshotManager.Diff(repository, snapshotID, revisions, path, compareByHash, preference.NobackupFile, preference.FiltersFile, preference.ExcludeByAttribute)
 
 	runScript(context, preference.Name, "post")
 }
@@ -1046,7 +1112,7 @@ func showHistory(context *cli.Context) {
 
 	revisions := getRevisions(context)
 	showLocalHash := context.Bool("hash")
-	backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, preference.NobackupFile)
+	backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, "", "", false)
 	duplicacy.SavePassword(*preference, "password", password)
 
 	backupManager.SetupSnapshotCache(preference.Name)
@@ -1109,7 +1175,7 @@ func pruneSnapshots(context *cli.Context) {
 		os.Exit(ArgumentExitCode)
 	}
 
-	backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, preference.NobackupFile)
+	backupManager := duplicacy.CreateBackupManager(preference.SnapshotID, storage, repository, password, "", "", false)
 	duplicacy.SavePassword(*preference, "password", password)
 
 	backupManager.SetupSnapshotCache(preference.Name)
@@ -1129,9 +1195,14 @@ func copySnapshots(context *cli.Context) {
 		os.Exit(ArgumentExitCode)
 	}
 
-	threads := context.Int("threads")
-	if threads < 1 {
-		threads = 1
+	uploadingThreads := context.Int("threads")
+	if uploadingThreads < 1 {
+		uploadingThreads = 1
+	}
+
+	downloadingThreads := context.Int("download-threads")
+	if downloadingThreads < 1 {
+		downloadingThreads = 1
 	}
 
 	repository, source := getRepositoryPreference(context, context.String("from"))
@@ -1139,7 +1210,7 @@ func copySnapshots(context *cli.Context) {
 	runScript(context, source.Name, "pre")
 
 	duplicacy.LOG_INFO("STORAGE_SET", "Source storage set to %s", source.StorageURL)
-	sourceStorage := duplicacy.CreateStorage(*source, false, threads)
+	sourceStorage := duplicacy.CreateStorage(*source, false, downloadingThreads)
 	if sourceStorage == nil {
 		return
 	}
@@ -1149,10 +1220,12 @@ func copySnapshots(context *cli.Context) {
 		sourcePassword = duplicacy.GetPassword(*source, "password", "Enter source storage password:", false, false)
 	}
 
-	sourceManager := duplicacy.CreateBackupManager(source.SnapshotID, sourceStorage, repository, sourcePassword, source.NobackupFile)
+	sourceManager := duplicacy.CreateBackupManager(source.SnapshotID, sourceStorage, repository, sourcePassword, "", "", false)
 	sourceManager.SetupSnapshotCache(source.Name)
 	duplicacy.SavePassword(*source, "password", sourcePassword)
 
+	loadRSAPrivateKey(context.String("key"), context.String("key-passphrase"), source, sourceManager, false)
+
 	_, destination := getRepositoryPreference(context, context.String("to"))
 
 	if destination.Name == source.Name {
@@ -1167,7 +1240,7 @@ func copySnapshots(context *cli.Context) {
 	}
 
 	duplicacy.LOG_INFO("STORAGE_SET", "Destination storage set to %s", destination.StorageURL)
-	destinationStorage := duplicacy.CreateStorage(*destination, false, threads)
+	destinationStorage := duplicacy.CreateStorage(*destination, false, uploadingThreads)
 	if destinationStorage == nil {
 		return
 	}
@@ -1182,7 +1255,7 @@ func copySnapshots(context *cli.Context) {
 	destinationStorage.SetRateLimits(0, context.Int("upload-limit-rate"))
 
 	destinationManager := duplicacy.CreateBackupManager(destination.SnapshotID, destinationStorage, repository,
-		destinationPassword, destination.NobackupFile)
+		                                                  destinationPassword, "", "", false)
 	duplicacy.SavePassword(*destination, "password", destinationPassword)
 	destinationManager.SetupSnapshotCache(destination.Name)
 
@@ -1192,7 +1265,7 @@ func copySnapshots(context *cli.Context) {
 		snapshotID = context.String("id")
 	}
 
-	sourceManager.CopySnapshots(destinationManager, snapshotID, revisions, threads)
+	sourceManager.CopySnapshots(destinationManager, snapshotID, revisions, uploadingThreads, downloadingThreads)
 	runScript(context, source.Name, "post")
 }
 
@@ -1360,6 +1433,16 @@ func main() {
 					Usage:    "initialize a new repository at the specified path rather than the current working directory",
 					Argument: "<path>",
 				},
+				cli.StringFlag{
+					Name:     "key",
+					Usage:    "the RSA public key to encrypt file chunks",
+					Argument: "<public key>",
+				},
+				cli.StringFlag{
+					Name:     "erasure-coding",
+					Usage:    "enable erasure coding to protect against storage corruption",
+					Argument: "<data shards>:<parity shards>",
+				},
 			},
 			Usage:     "Initialize the storage if necessary and the current directory as the repository",
 			ArgsUsage: "<snapshot id> <storage url>",
@@ -1467,6 +1550,20 @@ func main() {
 					Usage:    "restore from the specified storage instead of the default one",
 					Argument: "<storage name>",
 				},
+				cli.StringFlag{
+					Name:     "key",
+					Usage:    "the RSA private key to decrypt file chunks",
+					Argument: "<private key>",
+				},
+				cli.BoolFlag{
+					Name:  "persist",
+					Usage: "continue processing despite chunk errors or existing files (without -overwrite), reporting any affected files",
+        },
+				cli.StringFlag{
+					Name:     "key-passphrase",
+					Usage:    "the passphrase to decrypt the RSA private key",
+					Argument: "<private key passphrase>",
+				},
 			},
 			Usage:     "Restore the repository to a previously saved snapshot",
 			ArgsUsage: "[--] [pattern] ...",
@@ -1512,6 +1609,11 @@ func main() {
 					Usage:    "retrieve snapshots from the specified storage",
 					Argument: "<storage name>",
 				},
+				cli.StringFlag{
+					Name:     "key",
+					Usage:    "the RSA private key to decrypt file chunks",
+					Argument: "<private key>",
+				},
 			},
 			Usage:     "List snapshots",
 			ArgsUsage: " ",
@@ -1551,6 +1653,10 @@ func main() {
 					Name:  "files",
 					Usage: "verify the integrity of every file",
 				},
+				cli.BoolFlag{
+					Name:  "chunks",
+					Usage: "verify the integrity of every chunk",
+				},
 				cli.BoolFlag{
 					Name:  "stats",
 					Usage: "show deduplication statistics (imply -all and all revisions)",
@@ -1564,6 +1670,26 @@ func main() {
 					Usage:    "retrieve snapshots from the specified storage",
 					Argument: "<storage name>",
 				},
+				cli.StringFlag{
+					Name:     "key",
+					Usage:    "the RSA private key to decrypt file chunks",
+					Argument: "<private key>",
+				},
+				cli.StringFlag{
+					Name:     "key-passphrase",
+					Usage:    "the passphrase to decrypt the RSA private key",
+					Argument: "<private key passphrase>",
+				},
+				cli.IntFlag{
+					Name:     "threads",
+					Value:    1,
+					Usage:    "number of threads used to verify chunks",
+					Argument: "<n>",
+				},
+				cli.BoolFlag{
+					Name:  "persist",
+					Usage: "continue processing despite chunk errors, reporting any affected (corrupted) files",
+				},
 			},
 			Usage:     "Check the integrity of snapshots",
 			ArgsUsage: " ",
@@ -1587,6 +1713,16 @@ func main() {
 					Usage:    "retrieve the file from the specified storage",
 					Argument: "<storage name>",
 				},
+				cli.StringFlag{
+					Name:     "key",
+					Usage:    "the RSA private key to decrypt file chunks",
+					Argument: "<private key>",
+				},
+				cli.StringFlag{
+					Name:     "key-passphrase",
+					Usage:    "the passphrase to decrypt the RSA private key",
+					Argument: "<private key passphrase>",
+				},
 			},
 			Usage:     "Print to stdout the specified file, or the snapshot content if no file is specified",
 			ArgsUsage: "[<file>]",
@@ -1615,6 +1751,16 @@ func main() {
 					Usage:    "retrieve files from the specified storage",
 					Argument: "<storage name>",
 				},
+				cli.StringFlag{
+					Name:     "key",
+					Usage:    "the RSA private key to decrypt file chunks",
+					Argument: "<private key>",
+				},
+				cli.StringFlag{
+					Name:     "key-passphrase",
+					Usage:    "the passphrase to decrypt the RSA private key",
+					Argument: "<private key passphrase>",
+				},
 			},
 			Usage:     "Compare two snapshots or two revisions of a file",
 			ArgsUsage: "[<file>]",
@@ -1779,6 +1925,16 @@ func main() {
 					Usage:    "specify the path of the repository (instead of the current working directory)",
 					Argument: "<path>",
 				},
+				cli.StringFlag{
+					Name:     "key",
+					Usage:    "the RSA public key to encrypt file chunks",
+					Argument: "<public key>",
+				},
+				cli.StringFlag{
+					Name:     "erasure-coding",
+					Usage:    "enable erasure coding to protect against storage corruption",
+					Argument: "<data shards>:<parity shards>",
+				},
 			},
 			Usage:     "Add an additional storage to be used for the existing repository",
 			ArgsUsage: "<storage name> <snapshot id> <storage url>",
@@ -1818,6 +1974,12 @@ func main() {
 					Argument: "<file name>",
 					Value:    "",
 				},
+				cli.GenericFlag{
+					Name:  "exclude-by-attribute",
+					Usage: "Exclude files based on file attributes. (macOS only, com_apple_backup_excludeItem)",
+					Value: &TriBool{},
+					Arg:   "true",
+				},
 				cli.StringFlag{
 					Name:  "key",
 					Usage: "add a key/password whose value is supplied by the -value option",
@@ -1831,6 +1993,11 @@ func main() {
 					Usage:    "use the specified storage instead of the default one",
 					Argument: "<storage name>",
 				},
+				cli.StringFlag{
+					Name:     "filters",
+					Usage:    "specify the path of the filters file containing include/exclude patterns",
+					Argument: "<file path>",
+				},
 			},
 			Usage:     "Change the options for the default or specified storage",
 			ArgsUsage: " ",
@@ -1877,6 +2044,22 @@ func main() {
 					Usage:    "number of uploading threads",
 					Argument: "<n>",
 				},
+				cli.IntFlag{
+					Name:     "download-threads",
+					Value:    1,
+					Usage:    "number of downloading threads",
+					Argument: "<n>",
+				},
+				cli.StringFlag{
+					Name:     "key",
+					Usage:    "the RSA private key to decrypt file chunks from the source storage",
+					Argument: "<private key>",
+				},
+				cli.StringFlag{
+					Name:     "key-passphrase",
+					Usage:    "the passphrase to decrypt the RSA private key",
+					Argument: "<private key passphrase>",
+				},
 			},
 			Usage:     "Copy snapshots between compatible storages",
 			ArgsUsage: " ",
@@ -1985,13 +2168,18 @@ func main() {
 			Name:  "comment",
 			Usage: "add a comment to identify the process",
 		},
+		cli.StringSliceFlag{
+			Name:  "suppress, s",
+			Usage: "suppress logs with the specified id",
+			Argument: "<id>",
+		},
 	}
 
 	app.HideVersion = true
 	app.Name = "duplicacy"
 	app.HelpName = "duplicacy"
 	app.Usage = "A new generation cloud backup tool based on lock-free deduplication"
-	app.Version = "2.1.2" + " (" + GitCommit + ")"
+	app.Version = "2.7.0" + " (" + GitCommit + ")"
 
 	// If the program is interrupted, call the RunAtError function.
 	c := make(chan os.Signal, 1)

src/duplicacy_azurestorage.go

@@ -166,9 +166,21 @@ func (storage *AzureStorage) DownloadFile(threadIndex int, filePath string, chun
 
 // UploadFile writes 'content' to the file at 'filePath'.
 func (storage *AzureStorage) UploadFile(threadIndex int, filePath string, content []byte) (err error) {
-	reader := CreateRateLimitedReader(content, storage.UploadRateLimit/len(storage.containers))
-	blob := storage.containers[threadIndex].GetBlobReference(filePath)
-	return blob.CreateBlockBlobFromReader(reader, nil)
+
+	tries := 0
+
+	for {
+		reader := CreateRateLimitedReader(content, storage.UploadRateLimit/len(storage.containers))
+		blob := storage.containers[threadIndex].GetBlobReference(filePath)
+		err = blob.CreateBlockBlobFromReader(reader, nil)
+
+		if err == nil || !strings.Contains(err.Error(), "write: broken pipe") || tries >= 3 {
+			return err
+		}
+
+		LOG_INFO("AZURE_RETRY", "Connection unexpectedly terminated: %v; retrying", err)
+		tries++
+	}
 
 }
 

src/duplicacy_b2client.go

@@ -5,19 +5,22 @@
 package duplicacy
 
 import (
-	"bytes"
-	"crypto/sha1"
-	"encoding/base64"
-	"encoding/hex"
-	"encoding/json"
-	"fmt"
 	"io"
-	"io/ioutil"
-	"math/rand"
-	"net/http"
+	"os"
+	"fmt"
+	"bytes"
+	"time"
+	"sync"
 	"strconv"
 	"strings"
-	"time"
+	"net/url"
+	"net/http"
+	"math/rand"
+	"io/ioutil"
+	"crypto/sha1"
+	"encoding/hex"
+	"encoding/json"
+	"encoding/base64"
 )
 
 type B2Error struct {
@@ -39,67 +42,115 @@ var B2AuthorizationURL = "https://api.backblazeb2.com/b2api/v1/b2_authorize_acco
 
 type B2Client struct {
 	HTTPClient         *http.Client
+
 	AccountID          string
 	ApplicationKeyID   string
 	ApplicationKey     string
+	BucketName         string
+	BucketID           string
+	StorageDir         string
+
+	Lock               sync.Mutex
 	AuthorizationToken string
 	APIURL             string
 	DownloadURL        string
-	BucketName         string
-	BucketID           string
+	IsAuthorized       bool
 
-	UploadURL   string
-	UploadToken string
+	UploadURLs         []string
+	UploadTokens       []string
 
-	TestMode bool
+	Threads            int
+	MaximumRetries     int
+	TestMode           bool
+
+	LastAuthorizationTime int64
 }
 
-func NewB2Client(applicationKeyID string, applicationKey string) *B2Client {
+// URL encode the given path but keep the slashes intact
+func B2Escape(path string) string {
+	var components []string
+	for _, c := range strings.Split(path, "/") {
+		components = append(components, url.QueryEscape(c))
+	}
+	return strings.Join(components, "/")
+}
+
+func NewB2Client(applicationKeyID string, applicationKey string, downloadURL string, storageDir string, threads int) *B2Client {
+
+	for storageDir != "" && storageDir[0] == '/' {
+		storageDir = storageDir[1:]
+	}
+
+	if storageDir != "" && storageDir[len(storageDir) - 1] != '/' {
+		storageDir += "/"
+	}
+
+	maximumRetries := 15
+	if value, found := os.LookupEnv("DUPLICACY_B2_RETRIES"); found && value != "" {
+		maximumRetries, _ = strconv.Atoi(value)
+		LOG_INFO("B2_RETRIES", "Setting maximum retries for B2 to %d", maximumRetries)
+	}
+
 	client := &B2Client{
 		HTTPClient:       http.DefaultClient,
 		ApplicationKeyID: applicationKeyID,
 		ApplicationKey:   applicationKey,
+		DownloadURL:      downloadURL,
+		StorageDir:       storageDir,
+		UploadURLs:       make([]string, threads),
+		UploadTokens:     make([]string, threads),
+		Threads:          threads,
+		MaximumRetries:   maximumRetries,
 	}
 	return client
 }
 
-func (client *B2Client) retry(backoff int, response *http.Response) int {
+func (client *B2Client) getAPIURL() string {
+	client.Lock.Lock()
+	defer client.Lock.Unlock()
+	return client.APIURL
+}
+
+func (client *B2Client) getDownloadURL() string {
+	client.Lock.Lock()
+	defer client.Lock.Unlock()
+	return client.DownloadURL
+}
+
+func (client *B2Client) retry(retries int, response *http.Response) int {
 	if response != nil {
 		if backoffList, found := response.Header["Retry-After"]; found && len(backoffList) > 0 {
 			retryAfter, _ := strconv.Atoi(backoffList[0])
 			if retryAfter >= 1 {
 				time.Sleep(time.Duration(retryAfter) * time.Second)
-				return 0
+				return 1
 			}
 		}
 	}
-	if backoff == 0 {
-		backoff = 1
-	} else {
-		backoff *= 2
+
+	if retries >= client.MaximumRetries + 1 {
+		return 0
 	}
-	time.Sleep(time.Duration(backoff) * time.Second)
-	return backoff
+	retries++
+	delay := 1 << uint(retries)
+	if delay > 64 {
+		delay = 64
+	}
+	delayInSeconds := (rand.Float32() + 1.0) * float32(delay) / 2.0
+
+	time.Sleep(time.Duration(delayInSeconds) * time.Second)
+	return retries
 }
 
-func (client *B2Client) call(url string, method string, requestHeaders map[string]string, input interface{}) (io.ReadCloser, http.Header, int64, error) {
-
-	switch method {
-	case http.MethodGet:
-		break
-	case http.MethodHead:
-		break
-	case http.MethodPost:
-		break
-	default:
-		return nil, nil, 0, fmt.Errorf("unhandled http request method: " + method)
-	}
+func (client *B2Client) call(threadIndex int, requestURL string, method string, requestHeaders map[string]string, input interface{}) (
+	                         io.ReadCloser, http.Header, int64, error) {
 
 	var response *http.Response
 
-	backoff := 0
-	for i := 0; i < 8; i++ {
-		var inputReader *bytes.Reader
+	retries := 0
+	for {
+		var inputReader io.Reader
+		isUpload := false
 
 		switch input.(type) {
 		default:
@@ -108,21 +159,43 @@ func (client *B2Client) call(url string, method string, requestHeaders map[strin
 				return nil, nil, 0, err
 			}
 			inputReader = bytes.NewReader(jsonInput)
-		case []byte:
-			inputReader = bytes.NewReader(input.([]byte))
 		case int:
 			inputReader = bytes.NewReader([]byte(""))
+		case []byte:
+			isUpload = true
+			inputReader = bytes.NewReader(input.([]byte))
+		case *RateLimitedReader:
+			isUpload = true
+			rateLimitedReader := input.(*RateLimitedReader)
+			rateLimitedReader.Reset()
+			inputReader = rateLimitedReader
 		}
 
-		request, err := http.NewRequest(method, url, inputReader)
+
+		if isUpload {
+			if client.UploadURLs[threadIndex] == "" || client.UploadTokens[threadIndex] == "" {
+				err := client.getUploadURL(threadIndex)
+				if err != nil {
+					return nil, nil, 0, err
+				}
+			}
+			requestURL = client.UploadURLs[threadIndex]
+		}
+
+		request, err := http.NewRequest(method, requestURL, inputReader)
 		if err != nil {
 			return nil, nil, 0, err
 		}
 
-		if url == B2AuthorizationURL {
+		if requestURL == B2AuthorizationURL {
 			request.Header.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(client.ApplicationKeyID+":"+client.ApplicationKey)))
+		} else if isUpload {
+			request.ContentLength, _ = strconv.ParseInt(requestHeaders["Content-Length"], 10, 64)
+			request.Header.Set("Authorization", client.UploadTokens[threadIndex])
 		} else {
+			client.Lock.Lock()
 			request.Header.Set("Authorization", client.AuthorizationToken)
+			client.Lock.Unlock()
 		}
 
 		if requestHeaders != nil {
@@ -133,7 +206,9 @@ func (client *B2Client) call(url string, method string, requestHeaders map[strin
 
 		if client.TestMode {
 			r := rand.Float32()
-			if r < 0.5 {
+			if r < 0.5 && isUpload {
+				request.Header.Set("X-Bz-Test-Mode", "fail_some_uploads")
+			} else if r < 0.75 {
 				request.Header.Set("X-Bz-Test-Mode", "expire_some_account_authorization_tokens")
 			} else {
 				request.Header.Set("X-Bz-Test-Mode", "force_cap_exceeded")
@@ -142,28 +217,51 @@ func (client *B2Client) call(url string, method string, requestHeaders map[strin
 
 		response, err = client.HTTPClient.Do(request)
 		if err != nil {
-			if url != B2AuthorizationURL {
-				LOG_DEBUG("BACKBLAZE_CALL", "URL request '%s' returned an error: %v", url, err)
-				backoff = client.retry(backoff, response)
-				continue
+
+			// Don't retry when the first authorization request fails
+			if requestURL == B2AuthorizationURL && !client.IsAuthorized {
+				return nil, nil, 0, err
 			}
-			return nil, nil, 0, err
+
+			LOG_TRACE("BACKBLAZE_CALL", "[%d] URL request '%s' returned an error: %v", threadIndex, requestURL, err)
+
+			retries = client.retry(retries, response)
+			if retries <= 0 {
+				return nil, nil, 0, err
+			}
+
+			// Clear the upload url to requrest a new one on retry
+			if isUpload {
+				client.UploadURLs[threadIndex] = ""
+				client.UploadTokens[threadIndex] = ""
+			}
+			continue
+
 		}
 
 		if response.StatusCode < 300 {
 			return response.Body, response.Header, response.ContentLength, nil
 		}
 
-		LOG_DEBUG("BACKBLAZE_CALL", "URL request '%s %s' returned status code %d", method, url, response.StatusCode)
+		e := &B2Error{}
+		if err := json.NewDecoder(response.Body).Decode(e); err != nil {
+			LOG_TRACE("BACKBLAZE_CALL", "[%d] URL request '%s %s' returned status code %d", threadIndex, method, requestURL, response.StatusCode)
+		} else {
+			LOG_TRACE("BACKBLAZE_CALL", "[%d] URL request '%s %s' returned %d %s", threadIndex, method, requestURL, response.StatusCode, e.Message)
+		}
 
-		io.Copy(ioutil.Discard, response.Body)
 		response.Body.Close()
+
 		if response.StatusCode == 401 {
-			if url == B2AuthorizationURL {
+			if requestURL == B2AuthorizationURL {
 				return nil, nil, 0, fmt.Errorf("Authorization failure")
 			}
-			client.AuthorizeAccount()
-			continue
+
+			// Attempt authorization again.  If authorization is actually not done, run the random backoff
+			_, allowed := client.AuthorizeAccount(threadIndex)
+			if allowed {
+				continue
+			}
 		} else if response.StatusCode == 403 {
 			if !client.TestMode {
 				return nil, nil, 0, fmt.Errorf("B2 cap exceeded")
@@ -176,32 +274,21 @@ func (client *B2Client) call(url string, method string, requestHeaders map[strin
 		} else if response.StatusCode == 416 {
 			if http.MethodHead == method {
 				// 416 Requested Range Not Satisfiable
-				return nil, nil, 0, fmt.Errorf("URL request '%s' returned status code %d", url, response.StatusCode)
+				return nil, nil, 0, fmt.Errorf("URL request '%s' returned %d %s", requestURL, response.StatusCode, e.Message)
 			}
-		} else if response.StatusCode == 429 || response.StatusCode == 408 {
-			backoff = client.retry(backoff, response)
-			continue
-		} else if response.StatusCode >= 500 && response.StatusCode <= 599 {
-			backoff = client.retry(backoff, response)
-			continue
-		} else {
-			LOG_INFO("BACKBLAZE_CALL", "URL request '%s' returned status code %d", url, response.StatusCode)
-			backoff = client.retry(backoff, response)
-			continue
 		}
 
-		defer response.Body.Close()
-
-		e := &B2Error{}
-
-		if err := json.NewDecoder(response.Body).Decode(e); err != nil {
-			return nil, nil, 0, err
+		retries = client.retry(retries, response)
+		if retries <= 0 {
+			return nil, nil, 0, fmt.Errorf("URL request '%s' returned %d %s", requestURL, response.StatusCode, e.Message)
 		}
 
-		return nil, nil, 0, e
+		if isUpload {
+			client.UploadURLs[threadIndex] = ""
+			client.UploadTokens[threadIndex] = ""
+		}
 	}
 
-	return nil, nil, 0, fmt.Errorf("Maximum backoff reached")
 }
 
 type B2AuthorizeAccountOutput struct {
@@ -211,11 +298,18 @@ type B2AuthorizeAccountOutput struct {
 	DownloadURL        string
 }
 
-func (client *B2Client) AuthorizeAccount() (err error) {
+func (client *B2Client) AuthorizeAccount(threadIndex int) (err error, allowed bool) {
+	client.Lock.Lock()
+	defer client.Lock.Unlock()
 
-	readCloser, _, _, err := client.call(B2AuthorizationURL, http.MethodPost, nil, make(map[string]string))
+	// Don't authorize if the previous one was done less than 30 seconds ago
+	if client.LastAuthorizationTime != 0 && client.LastAuthorizationTime > time.Now().Unix() - 30 {
+		return nil, false
+	}
+
+	readCloser, _, _, err := client.call(threadIndex, B2AuthorizationURL, http.MethodPost, nil, make(map[string]string))
 	if err != nil {
-		return err
+		return err, true
 	}
 
 	defer readCloser.Close()
@@ -223,7 +317,7 @@ func (client *B2Client) AuthorizeAccount() (err error) {
 	output := &B2AuthorizeAccountOutput{}
 
 	if err = json.NewDecoder(readCloser).Decode(&output); err != nil {
-		return err
+		return err, true
 	}
 
 	// The account id may be different from the application key id so we're getting the account id from the returned
@@ -232,9 +326,15 @@ func (client *B2Client) AuthorizeAccount() (err error) {
 
 	client.AuthorizationToken = output.AuthorizationToken
 	client.APIURL = output.APIURL
-	client.DownloadURL = output.DownloadURL
+	if client.DownloadURL == "" {
+		client.DownloadURL = output.DownloadURL
+	}
+	LOG_INFO("BACKBLAZE_URL", "download URL is: %s", client.DownloadURL)
+	client.IsAuthorized = true
 
-	return nil
+	client.LastAuthorizationTime = time.Now().Unix()
+
+	return nil, true
 }
 
 type ListBucketOutput struct {
@@ -248,10 +348,11 @@ func (client *B2Client) FindBucket(bucketName string) (err error) {
 
 	input := make(map[string]string)
 	input["accountId"] = client.AccountID
+	input["bucketName"] = bucketName
 
-	url := client.APIURL + "/b2api/v1/b2_list_buckets"
+	url := client.getAPIURL() + "/b2api/v1/b2_list_buckets"
 
-	readCloser, _, _, err := client.call(url, http.MethodPost, nil, input)
+	readCloser, _, _, err := client.call(0, url, http.MethodPost, nil, input)
 	if err != nil {
 		return err
 	}
@@ -293,7 +394,7 @@ type B2ListFileNamesOutput struct {
 	NextFileId   string
 }
 
-func (client *B2Client) ListFileNames(startFileName string, singleFile bool, includeVersions bool) (files []*B2Entry, err error) {
+func (client *B2Client) ListFileNames(threadIndex int, startFileName string, singleFile bool, includeVersions bool) (files []*B2Entry, err error) {
 
 	maxFileCount := 1000
 	if singleFile {
@@ -311,20 +412,21 @@ func (client *B2Client) ListFileNames(startFileName string, singleFile bool, inc
 
 	input := make(map[string]interface{})
 	input["bucketId"] = client.BucketID
-	input["startFileName"] = startFileName
+	input["startFileName"] = client.StorageDir + startFileName
 	input["maxFileCount"] = maxFileCount
+	input["prefix"] = client.StorageDir
 
 	for {
-		url := client.APIURL + "/b2api/v1/b2_list_file_names"
+		apiURL := client.getAPIURL() + "/b2api/v1/b2_list_file_names"
 		requestHeaders := map[string]string{}
 		requestMethod := http.MethodPost
 		var requestInput interface{}
 		requestInput = input
 		if includeVersions {
-			url = client.APIURL + "/b2api/v1/b2_list_file_versions"
+			apiURL = client.getAPIURL() + "/b2api/v1/b2_list_file_versions"
 		} else if singleFile {
 			// handle a single file with no versions as a special case to download the last byte of the file
-			url = client.DownloadURL + "/file/" + client.BucketName + "/" + startFileName
+			apiURL = client.getDownloadURL() + "/file/" + client.BucketName + "/" + B2Escape(client.StorageDir + startFileName)
 			// requesting byte -1 works for empty files where 0-0 fails with a 416 error
 			requestHeaders["Range"] = "bytes=-1"
 			// HEAD request
@@ -334,7 +436,7 @@ func (client *B2Client) ListFileNames(startFileName string, singleFile bool, inc
 		var readCloser io.ReadCloser
 		var responseHeader http.Header
 		var err error
-		readCloser, responseHeader, _, err = client.call(url, requestMethod, requestHeaders, requestInput)
+		readCloser, responseHeader, _, err = client.call(threadIndex, apiURL, requestMethod, requestHeaders, requestInput)
 		if err != nil {
 			return nil, err
 		}
@@ -347,7 +449,7 @@ func (client *B2Client) ListFileNames(startFileName string, singleFile bool, inc
 
 		if singleFile && !includeVersions {
 			if responseHeader == nil {
-				LOG_DEBUG("BACKBLAZE_LIST", "b2_download_file_by_name did not return headers")
+				LOG_DEBUG("BACKBLAZE_LIST", "%s did not return headers", apiURL)
 				return []*B2Entry{}, nil
 			}
 			requiredHeaders := []string{
@@ -361,11 +463,17 @@ func (client *B2Client) ListFileNames(startFileName string, singleFile bool, inc
 				}
 			}
 			if len(missingKeys) > 0 {
-				return nil, fmt.Errorf("b2_download_file_by_name missing headers: %s", missingKeys)
+				return nil, fmt.Errorf("%s missing headers: %s", apiURL, missingKeys)
 			}
 			// construct the B2Entry from the response headers of the download request
 			fileID := responseHeader.Get("x-bz-file-id")
 			fileName := responseHeader.Get("x-bz-file-name")
+			unescapedFileName, err := url.QueryUnescape(fileName)
+			if err == nil {
+				fileName = unescapedFileName
+			} else {
+				LOG_WARN("BACKBLAZE_UNESCAPE", "Failed to unescape the file name %s", fileName)
+			}
 			fileAction := "upload"
 			// byte range that is returned: "bytes #-#/#
 			rangeString := responseHeader.Get("Content-Range")
@@ -378,14 +486,14 @@ func (client *B2Client) ListFileNames(startFileName string, singleFile bool, inc
 				// this should only execute if the requested file is empty and the range request didn't result in a Content-Range header
 				fileSize, _ = strconv.ParseInt(lengthString, 0, 64)
 				if fileSize != 0 {
-					return nil, fmt.Errorf("b2_download_file_by_name returned non-zero file length")
+					return nil, fmt.Errorf("%s returned non-zero file length", apiURL)
 				}
 			} else {
-				return nil, fmt.Errorf("could not parse b2_download_file_by_name headers")
+				return nil, fmt.Errorf("could not parse headers returned by %s", apiURL)
 			}
 			fileUploadTimestamp, _ := strconv.ParseInt(responseHeader.Get("X-Bz-Upload-Timestamp"), 0, 64)
 
-			return []*B2Entry{{fileID, fileName, fileAction, fileSize, fileUploadTimestamp}}, nil
+			return []*B2Entry{{fileID, fileName[len(client.StorageDir):], fileAction, fileSize, fileUploadTimestamp}}, nil
 		}
 
 		if err = json.NewDecoder(readCloser).Decode(&output); err != nil {
@@ -394,31 +502,27 @@ func (client *B2Client) ListFileNames(startFileName string, singleFile bool, inc
 
 		ioutil.ReadAll(readCloser)
 
-		if startFileName == "" {
-			files = append(files, output.Files...)
-		} else {
-			for _, file := range output.Files {
-				if singleFile {
-					if file.FileName == startFileName {
-						files = append(files, file)
-						if !includeVersions {
-							output.NextFileName = ""
-							break
-						}
-					} else {
+		for _, file := range output.Files {
+			file.FileName = file.FileName[len(client.StorageDir):]
+			if singleFile {
+				if file.FileName == startFileName {
+					files = append(files, file)
+					if !includeVersions {
 						output.NextFileName = ""
 						break
 					}
 				} else {
-					if strings.HasPrefix(file.FileName, startFileName) {
-						files = append(files, file)
-					} else {
-						output.NextFileName = ""
-						break
-					}
+					output.NextFileName = ""
+					break
+				}
+			} else {
+				if strings.HasPrefix(file.FileName, startFileName) {
+					files = append(files, file)
+				} else {
+					output.NextFileName = ""
+					break
 				}
 			}
-
 		}
 
 		if len(output.NextFileName) == 0 {
@@ -434,14 +538,14 @@ func (client *B2Client) ListFileNames(startFileName string, singleFile bool, inc
 	return files, nil
 }
 
-func (client *B2Client) DeleteFile(fileName string, fileID string) (err error) {
+func (client *B2Client) DeleteFile(threadIndex int, fileName string, fileID string) (err error) {
 
 	input := make(map[string]string)
-	input["fileName"] = fileName
+	input["fileName"] = client.StorageDir + fileName
 	input["fileId"] = fileID
 
-	url := client.APIURL + "/b2api/v1/b2_delete_file_version"
-	readCloser, _, _, err := client.call(url, http.MethodPost, make(map[string]string), input)
+	url := client.getAPIURL() + "/b2api/v1/b2_delete_file_version"
+	readCloser, _, _, err := client.call(threadIndex, url, http.MethodPost, make(map[string]string), input)
 	if err != nil {
 		return err
 	}
@@ -454,14 +558,14 @@ type B2HideFileOutput struct {
 	FileID string
 }
 
-func (client *B2Client) HideFile(fileName string) (fileID string, err error) {
+func (client *B2Client) HideFile(threadIndex int, fileName string) (fileID string, err error) {
 
 	input := make(map[string]string)
 	input["bucketId"] = client.BucketID
-	input["fileName"] = fileName
+	input["fileName"] = client.StorageDir + fileName
 
-	url := client.APIURL + "/b2api/v1/b2_hide_file"
-	readCloser, _, _, err := client.call(url, http.MethodPost, make(map[string]string), input)
+	url := client.getAPIURL() + "/b2api/v1/b2_hide_file"
+	readCloser, _, _, err := client.call(threadIndex, url, http.MethodPost, make(map[string]string), input)
 	if err != nil {
 		return "", err
 	}
@@ -478,11 +582,11 @@ func (client *B2Client) HideFile(fileName string) (fileID string, err error) {
 	return output.FileID, nil
 }
 
-func (client *B2Client) DownloadFile(filePath string) (io.ReadCloser, int64, error) {
+func (client *B2Client) DownloadFile(threadIndex int, filePath string) (io.ReadCloser, int64, error) {
 
-	url := client.DownloadURL + "/file/" + client.BucketName + "/" + filePath
+	url := client.getDownloadURL() + "/file/" + client.BucketName + "/" + B2Escape(client.StorageDir + filePath)
 
-	readCloser, _, len, err := client.call(url, http.MethodGet, make(map[string]string), 0)
+	readCloser, _, len, err := client.call(threadIndex, url, http.MethodGet, make(map[string]string), 0)
 	return readCloser, len, err
 }
 
@@ -492,12 +596,12 @@ type B2GetUploadArgumentOutput struct {
 	AuthorizationToken string
 }
 
-func (client *B2Client) getUploadURL() error {
+func (client *B2Client) getUploadURL(threadIndex int) error {
 	input := make(map[string]string)
 	input["bucketId"] = client.BucketID
 
-	url := client.APIURL + "/b2api/v1/b2_get_upload_url"
-	readCloser, _, _, err := client.call(url, http.MethodPost, make(map[string]string), input)
+	url := client.getAPIURL() + "/b2api/v1/b2_get_upload_url"
+	readCloser, _, _, err := client.call(threadIndex, url, http.MethodPost, make(map[string]string), input)
 	if err != nil {
 		return err
 	}
@@ -510,96 +614,29 @@ func (client *B2Client) getUploadURL() error {
 		return err
 	}
 
-	client.UploadURL = output.UploadURL
-	client.UploadToken = output.AuthorizationToken
+	client.UploadURLs[threadIndex] = output.UploadURL
+	client.UploadTokens[threadIndex] = output.AuthorizationToken
 
 	return nil
 }
 
-func (client *B2Client) UploadFile(filePath string, content []byte, rateLimit int) (err error) {
+func (client *B2Client) UploadFile(threadIndex int, filePath string, content []byte, rateLimit int) (err error) {
 
 	hasher := sha1.New()
 	hasher.Write(content)
 	hash := hex.EncodeToString(hasher.Sum(nil))
 
 	headers := make(map[string]string)
-	headers["X-Bz-File-Name"] = filePath
+	headers["X-Bz-File-Name"] = B2Escape(client.StorageDir + filePath)
+	headers["Content-Length"] = fmt.Sprintf("%d", len(content))
 	headers["Content-Type"] = "application/octet-stream"
 	headers["X-Bz-Content-Sha1"] = hash
 
-	var response *http.Response
-
-	backoff := 0
-	for i := 0; i < 8; i++ {
-
-		if client.UploadURL == "" || client.UploadToken == "" {
-			err = client.getUploadURL()
-			if err != nil {
-				return err
-			}
-		}
-
-		request, err := http.NewRequest("POST", client.UploadURL, CreateRateLimitedReader(content, rateLimit))
-		if err != nil {
-			return err
-		}
-		request.ContentLength = int64(len(content))
-
-		request.Header.Set("Authorization", client.UploadToken)
-		request.Header.Set("X-Bz-File-Name", filePath)
-		request.Header.Set("Content-Type", "application/octet-stream")
-		request.Header.Set("X-Bz-Content-Sha1", hash)
-
-		for key, value := range headers {
-			request.Header.Set(key, value)
-		}
-
-		if client.TestMode {
-			r := rand.Float32()
-			if r < 0.8 {
-				request.Header.Set("X-Bz-Test-Mode", "fail_some_uploads")
-			} else if r < 0.9 {
-				request.Header.Set("X-Bz-Test-Mode", "expire_some_account_authorization_tokens")
-			} else {
-				request.Header.Set("X-Bz-Test-Mode", "force_cap_exceeded")
-			}
-		}
-
-		response, err = client.HTTPClient.Do(request)
-		if err != nil {
-			LOG_DEBUG("BACKBLAZE_UPLOAD", "URL request '%s' returned an error: %v", client.UploadURL, err)
-			backoff = client.retry(backoff, response)
-			client.UploadURL = ""
-			client.UploadToken = ""
-			continue
-		}
-
-		io.Copy(ioutil.Discard, response.Body)
-		response.Body.Close()
-
-		if response.StatusCode < 300 {
-			return nil
-		}
-
-		LOG_DEBUG("BACKBLAZE_UPLOAD", "URL request '%s' returned status code %d", client.UploadURL, response.StatusCode)
-
-		if response.StatusCode == 401 {
-			LOG_INFO("BACKBLAZE_UPLOAD", "Re-authorization required")
-			client.UploadURL = ""
-			client.UploadToken = ""
-			continue
-		} else if response.StatusCode == 403 {
-			if !client.TestMode {
-				return fmt.Errorf("B2 cap exceeded")
-			}
-			continue
-		} else {
-			LOG_INFO("BACKBLAZE_UPLOAD", "URL request '%s' returned status code %d", client.UploadURL, response.StatusCode)
-			backoff = client.retry(backoff, response)
-			client.UploadURL = ""
-			client.UploadToken = ""
-		}
+	readCloser, _, _, err := client.call(threadIndex, "", http.MethodPost, headers, CreateRateLimitedReader(content, rateLimit))
+	if err != nil {
+		return err
 	}
 
-	return fmt.Errorf("Maximum backoff reached")
+	readCloser.Close()
+	return nil
 }

src/duplicacy_b2client_test.go

@@ -37,7 +37,7 @@ func createB2ClientForTest(t *testing.T) (*B2Client, string) {
 		return nil, ""
 	}
 
-	return NewB2Client(b2["account"], b2["key"]), b2["bucket"]
+	return NewB2Client(b2["account"], b2["key"], "", b2["directory"], 1), b2["bucket"]
 
 }
 
@@ -50,7 +50,7 @@ func TestB2Client(t *testing.T) {
 
 	b2Client.TestMode = true
 
-	err := b2Client.AuthorizeAccount()
+	err, _ := b2Client.AuthorizeAccount(0)
 	if err != nil {
 		t.Errorf("Failed to authorize the b2 account: %v", err)
 		return
@@ -64,14 +64,14 @@ func TestB2Client(t *testing.T) {
 
 	testDirectory := "b2client_test/"
 
-	files, err := b2Client.ListFileNames(testDirectory, false, false)
+	files, err := b2Client.ListFileNames(0, testDirectory, false, false)
 	if err != nil {
 		t.Errorf("Failed to list files: %v", err)
 		return
 	}
 
 	for _, file := range files {
-		err = b2Client.DeleteFile(file.FileName, file.FileID)
+		err = b2Client.DeleteFile(0, file.FileName, file.FileID)
 		if err != nil {
 			t.Errorf("Failed to delete file '%s': %v", file.FileName, err)
 		}
@@ -90,14 +90,14 @@ func TestB2Client(t *testing.T) {
 		hash := sha256.Sum256(content)
 		name := hex.EncodeToString(hash[:])
 
-		err = b2Client.UploadFile(testDirectory+name, content, 100)
+		err = b2Client.UploadFile(0, testDirectory+name, content, 100)
 		if err != nil {
 			t.Errorf("Error uploading file '%s': %v", name, err)
 			return
 		}
 	}
 
-	files, err = b2Client.ListFileNames(testDirectory, false, false)
+	files, err = b2Client.ListFileNames(0, testDirectory, false, false)
 	if err != nil {
 		t.Errorf("Failed to list files: %v", err)
 		return
@@ -105,7 +105,7 @@ func TestB2Client(t *testing.T) {
 
 	for _, file := range files {
 
-		readCloser, _, err := b2Client.DownloadFile(file.FileName)
+		readCloser, _, err := b2Client.DownloadFile(0, file.FileName)
 		if err != nil {
 			t.Errorf("Error downloading file '%s': %v", file.FileName, err)
 			return
@@ -125,7 +125,7 @@ func TestB2Client(t *testing.T) {
 	}
 
 	for _, file := range files {
-		err = b2Client.DeleteFile(file.FileName, file.FileID)
+		err = b2Client.DeleteFile(0, file.FileName, file.FileID)
 		if err != nil {
 			t.Errorf("Failed to delete file '%s': %v", file.FileName, err)
 		}

src/duplicacy_b2storage.go

@@ -11,32 +11,26 @@ import (
 type B2Storage struct {
 	StorageBase
 
-	clients []*B2Client
+	client *B2Client
 }
 
 // CreateB2Storage creates a B2 storage object.
-func CreateB2Storage(accountID string, applicationKey string, bucket string, threads int) (storage *B2Storage, err error) {
+func CreateB2Storage(accountID string, applicationKey string, downloadURL string, bucket string, storageDir string, threads int) (storage *B2Storage, err error) {
 
-	var clients []*B2Client
+	client := NewB2Client(accountID, applicationKey, downloadURL, storageDir, threads)
 
-	for i := 0; i < threads; i++ {
-		client := NewB2Client(accountID, applicationKey)
+	err, _ = client.AuthorizeAccount(0)
+	if err != nil {
+		return nil, err
+	}
 
-		err = client.AuthorizeAccount()
-		if err != nil {
-			return nil, err
-		}
-
-		err = client.FindBucket(bucket)
-		if err != nil {
-			return nil, err
-		}
-
-		clients = append(clients, client)
+	err = client.FindBucket(bucket)
+	if err != nil {
+		return nil, err
 	}
 
 	storage = &B2Storage{
-		clients: clients,
+		client: client,
 	}
 
 	storage.DerivedStorage = storage
@@ -56,7 +50,7 @@ func (storage *B2Storage) ListFiles(threadIndex int, dir string) (files []string
 		includeVersions = true
 	}
 
-	entries, err := storage.clients[threadIndex].ListFileNames(dir, false, includeVersions)
+	entries, err := storage.client.ListFileNames(threadIndex, dir, false, includeVersions)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -102,7 +96,7 @@ func (storage *B2Storage) DeleteFile(threadIndex int, filePath string) (err erro
 
 	if strings.HasSuffix(filePath, ".fsl") {
 		filePath = filePath[:len(filePath)-len(".fsl")]
-		entries, err := storage.clients[threadIndex].ListFileNames(filePath, true, true)
+		entries, err := storage.client.ListFileNames(threadIndex, filePath, true, true)
 		if err != nil {
 			return err
 		}
@@ -116,7 +110,7 @@ func (storage *B2Storage) DeleteFile(threadIndex int, filePath string) (err erro
 
 			toBeDeleted = true
 
-			err = storage.clients[threadIndex].DeleteFile(filePath, entry.FileID)
+			err = storage.client.DeleteFile(threadIndex, filePath, entry.FileID)
 			if err != nil {
 				return err
 			}
@@ -125,7 +119,7 @@ func (storage *B2Storage) DeleteFile(threadIndex int, filePath string) (err erro
 		return nil
 
 	} else {
-		entries, err := storage.clients[threadIndex].ListFileNames(filePath, true, false)
+		entries, err := storage.client.ListFileNames(threadIndex, filePath, true, false)
 		if err != nil {
 			return err
 		}
@@ -133,7 +127,7 @@ func (storage *B2Storage) DeleteFile(threadIndex int, filePath string) (err erro
 		if len(entries) == 0 {
 			return nil
 		}
-		return storage.clients[threadIndex].DeleteFile(filePath, entries[0].FileID)
+		return storage.client.DeleteFile(threadIndex, filePath, entries[0].FileID)
 	}
 }
 
@@ -160,10 +154,10 @@ func (storage *B2Storage) MoveFile(threadIndex int, from string, to string) (err
 	}
 
 	if filePath == from {
-		_, err = storage.clients[threadIndex].HideFile(from)
+		_, err = storage.client.HideFile(threadIndex, from)
 		return err
 	} else {
-		entries, err := storage.clients[threadIndex].ListFileNames(filePath, true, true)
+		entries, err := storage.client.ListFileNames(threadIndex, filePath, true, true)
 		if err != nil {
 			return err
 		}
@@ -171,7 +165,7 @@ func (storage *B2Storage) MoveFile(threadIndex int, from string, to string) (err
 			return nil
 		}
 
-		return storage.clients[threadIndex].DeleteFile(filePath, entries[0].FileID)
+		return storage.client.DeleteFile(threadIndex, filePath, entries[0].FileID)
 	}
 }
 
@@ -188,7 +182,7 @@ func (storage *B2Storage) GetFileInfo(threadIndex int, filePath string) (exist b
 		filePath = filePath[:len(filePath)-len(".fsl")]
 	}
 
-	entries, err := storage.clients[threadIndex].ListFileNames(filePath, true, isFossil)
+	entries, err := storage.client.ListFileNames(threadIndex, filePath, true, isFossil)
 	if err != nil {
 		return false, false, 0, err
 	}
@@ -210,22 +204,20 @@ func (storage *B2Storage) GetFileInfo(threadIndex int, filePath string) (exist b
 // DownloadFile reads the file at 'filePath' into the chunk.
 func (storage *B2Storage) DownloadFile(threadIndex int, filePath string, chunk *Chunk) (err error) {
 
-	filePath = strings.Replace(filePath, " ", "%20", -1)
-	readCloser, _, err := storage.clients[threadIndex].DownloadFile(filePath)
+	readCloser, _, err := storage.client.DownloadFile(threadIndex, filePath)
 	if err != nil {
 		return err
 	}
 
 	defer readCloser.Close()
 
-	_, err = RateLimitedCopy(chunk, readCloser, storage.DownloadRateLimit/len(storage.clients))
+	_, err = RateLimitedCopy(chunk, readCloser, storage.DownloadRateLimit/storage.client.Threads)
 	return err
 }
 
 // UploadFile writes 'content' to the file at 'filePath'.
 func (storage *B2Storage) UploadFile(threadIndex int, filePath string, content []byte) (err error) {
-	filePath = strings.Replace(filePath, " ", "%20", -1)
-	return storage.clients[threadIndex].UploadFile(filePath, content, storage.UploadRateLimit/len(storage.clients))
+	return storage.client.UploadFile(threadIndex, filePath, content, storage.UploadRateLimit/storage.client.Threads)
 }
 
 // If a local snapshot cache is needed for the storage to avoid downloading/uploading chunks too often when
@@ -243,7 +235,5 @@ func (storage *B2Storage) IsFastListing() bool { return true }
 
 // Enable the test mode.
 func (storage *B2Storage) EnableTestMode() {
-	for _, client := range storage.clients {
-		client.TestMode = true
-	}
+	storage.client.TestMode = true
 }

src/duplicacy_backupmanager.go

@@ -35,6 +35,11 @@ type BackupManager struct {
 	config *Config // contains a number of options
 
 	nobackupFile string // don't backup directory when this file name is found
+
+  filtersFile string  // the path to the filters file
+
+  excludeByAttribute bool // don't backup file based on file attribute
+
 }
 
 func (manager *BackupManager) SetDryRun(dryRun bool) {
@@ -44,7 +49,7 @@ func (manager *BackupManager) SetDryRun(dryRun bool) {
 // CreateBackupManager creates a backup manager using the specified 'storage'.  'snapshotID' is a unique id to
 // identify snapshots created for this repository.  'top' is the top directory of the repository.  'password' is the
 // master key which can be nil if encryption is not enabled.
-func CreateBackupManager(snapshotID string, storage Storage, top string, password string, nobackupFile string) *BackupManager {
+func CreateBackupManager(snapshotID string, storage Storage, top string, password string, nobackupFile string, filtersFile string, excludeByAttribute bool) *BackupManager {
 
 	config, _, err := DownloadConfig(storage, password)
 	if err != nil {
@@ -67,6 +72,10 @@ func CreateBackupManager(snapshotID string, storage Storage, top string, passwor
 		config: config,
 
 		nobackupFile: nobackupFile,
+
+		filtersFile: filtersFile,
+
+		excludeByAttribute: excludeByAttribute,
 	}
 
 	if IsDebugging() {
@@ -76,6 +85,11 @@ func CreateBackupManager(snapshotID string, storage Storage, top string, passwor
 	return backupManager
 }
 
+// loadRSAPrivateKey loads the specifed private key file for decrypting file chunks
+func (manager *BackupManager) LoadRSAPrivateKey(keyFile string, passphrase string) {
+	manager.config.loadRSAPrivateKey(keyFile, passphrase)
+}
+
 // SetupSnapshotCache creates the snapshot cache, which is merely a local storage under the default .duplicacy
 // directory
 func (manager *BackupManager) SetupSnapshotCache(storageName string) bool {
@@ -103,6 +117,7 @@ func (manager *BackupManager) SetupSnapshotCache(storageName string) bool {
 	return true
 }
 
+
 // setEntryContent sets the 4 content pointers for each entry in 'entries'.  'offset' indicates the value
 // to be added to the StartChunk and EndChunk points, used when intending to append 'entries' to the
 // original unchanged entry list.
@@ -176,6 +191,19 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
 
 	LOG_DEBUG("BACKUP_PARAMETERS", "top: %s, quick: %t, tag: %s", top, quickMode, tag)
 
+	if manager.config.DataShards != 0 && manager.config.ParityShards != 0 {
+		LOG_INFO("BACKUP_ERASURECODING", "Erasure coding is enabled with %d data shards and %d parity shards",
+		         manager.config.DataShards, manager.config.ParityShards)
+	}
+
+	if manager.config.rsaPublicKey != nil && len(manager.config.FileKey) > 0 {
+		LOG_INFO("BACKUP_KEY", "RSA encryption is enabled")
+	}
+
+	if manager.excludeByAttribute {
+		LOG_INFO("BACKUP_EXCLUDE", "Exclude files with no-backup attributes")
+	}
+
 	remoteSnapshot := manager.SnapshotManager.downloadLatestSnapshot(manager.snapshotID)
 	if remoteSnapshot == nil {
 		remoteSnapshot = CreateEmptySnapshot(manager.snapshotID)
@@ -188,7 +216,8 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
 	defer DeleteShadowCopy()
 
 	LOG_INFO("BACKUP_INDEXING", "Indexing %s", top)
-	localSnapshot, skippedDirectories, skippedFiles, err := CreateSnapshotFromDirectory(manager.snapshotID, shadowTop, manager.nobackupFile)
+	localSnapshot, skippedDirectories, skippedFiles, err := CreateSnapshotFromDirectory(manager.snapshotID, shadowTop,
+		                                                                                manager.nobackupFile, manager.filtersFile, manager.excludeByAttribute)
 	if err != nil {
 		LOG_ERROR("SNAPSHOT_LIST", "Failed to list the directory %s: %v", top, err)
 		return false
@@ -198,6 +227,11 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
 		return true
 	}
 
+	if len(localSnapshot.Files) == 0 {
+		LOG_ERROR("SNAPSHOT_EMPTY", "No files under the repository to be backed up")
+		return false
+	}
+
 	// This cache contains all chunks referenced by last snasphot. Any other chunks will lead to a call to
 	// UploadChunk.
 	chunkCache := make(map[string]bool)
@@ -502,6 +536,11 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
 				chunkID := chunk.GetID()
 				chunkSize := chunk.GetLength()
 
+				if chunkSize == 0 {
+					LOG_DEBUG("CHUNK_EMPTY", "Ignored chunk %s of size 0", chunkID)
+					return
+				}
+
 				chunkIndex++
 
 				_, found := chunkCache[chunkID]
@@ -723,7 +762,7 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
 // the same as 'top'.  'quickMode' will bypass files with unchanged sizes and timestamps.  'deleteMode' will
 // remove local files that don't exist in the snapshot. 'patterns' is used to include/exclude certain files.
 func (manager *BackupManager) Restore(top string, revision int, inPlace bool, quickMode bool, threads int, overwrite bool,
-	deleteMode bool, setOwner bool, showStatistics bool, patterns []string) bool {
+	deleteMode bool, setOwner bool, showStatistics bool, patterns []string, allowFailures bool) int {
 
 	startTime := time.Now().Unix()
 
@@ -746,7 +785,7 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
 		err = os.Mkdir(top, 0744)
 		if err != nil {
 			LOG_ERROR("RESTORE_MKDIR", "Can't create the directory to be restored: %v", err)
-			return false
+			return 0
 		}
 	}
 
@@ -754,16 +793,17 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
 	err = os.Mkdir(path.Join(top, DUPLICACY_DIRECTORY), 0744)
 	if err != nil && !os.IsExist(err) {
 		LOG_ERROR("RESTORE_MKDIR", "Failed to create the preference directory: %v", err)
-		return false
+		return 0
 	}
 
 	remoteSnapshot := manager.SnapshotManager.DownloadSnapshot(manager.snapshotID, revision)
 	manager.SnapshotManager.DownloadSnapshotContents(remoteSnapshot, patterns, true)
 
-	localSnapshot, _, _, err := CreateSnapshotFromDirectory(manager.snapshotID, top, manager.nobackupFile)
+	localSnapshot, _, _, err := CreateSnapshotFromDirectory(manager.snapshotID, top, manager.nobackupFile,
+		                                                    manager.filtersFile, manager.excludeByAttribute)
 	if err != nil {
 		LOG_ERROR("SNAPSHOT_LIST", "Failed to list the repository: %v", err)
-		return false
+		return 0
 	}
 
 	LOG_INFO("RESTORE_START", "Restoring %s to revision %d", top, revision)
@@ -790,6 +830,11 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
 
 	var totalFileSize int64
 	var downloadedFileSize int64
+	var failedFiles int
+	var skippedFileSize int64
+	var skippedFiles int64
+	
+	var downloadedFiles []*Entry
 
 	i := 0
 	for _, entry := range remoteSnapshot.Files {
@@ -807,6 +852,9 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
 				if compare == 0 {
 					i++
 					if quickMode && local.IsSameAs(entry) {
+						LOG_TRACE("RESTORE_SKIP", "File %s unchanged (by size and timestamp)", local.Path)
+						skippedFileSize += entry.Size
+						skippedFiles++
 						skipped = true
 					}
 				}
@@ -836,7 +884,7 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
 			err = os.Symlink(entry.Link, fullPath)
 			if err != nil {
 				LOG_ERROR("RESTORE_SYMLINK", "Can't create symlink %s: %v", entry.Path, err)
-				return false
+				return 0
 			}
 			entry.RestoreMetadata(fullPath, nil, setOwner)
 			LOG_TRACE("DOWNLOAD_DONE", "Symlink %s updated", entry.Path)
@@ -845,7 +893,7 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
 
 			if err == nil && !stat.IsDir() {
 				LOG_ERROR("RESTORE_NOTDIR", "The path %s is not a directory", fullPath)
-				return false
+				return 0
 			}
 
 			if os.IsNotExist(err) {
@@ -854,7 +902,7 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
 				err = os.MkdirAll(fullPath, 0700)
 				if err != nil && !os.IsExist(err) {
 					LOG_ERROR("RESTORE_MKDIR", "%v", err)
-					return false
+					return 0
 				}
 			}
 		} else {
@@ -872,14 +920,13 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
 	// Sort entries by their starting chunks in order to linearize the access to the chunk chain.
 	sort.Sort(ByChunk(fileEntries))
 
-	chunkDownloader := CreateChunkDownloader(manager.config, manager.storage, nil, showStatistics, threads)
+	chunkDownloader := CreateChunkDownloader(manager.config, manager.storage, nil, showStatistics, threads, allowFailures)
 	chunkDownloader.AddFiles(remoteSnapshot, fileEntries)
 
 	chunkMaker := CreateChunkMaker(manager.config, true)
 
 	startDownloadingTime := time.Now().Unix()
 
-	var downloadedFiles []*Entry
 	// Now download files one by one
 	for _, file := range fileEntries {
 
@@ -889,16 +936,21 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
 			if quickMode {
 				if file.IsSameAsFileInfo(stat) {
 					LOG_TRACE("RESTORE_SKIP", "File %s unchanged (by size and timestamp)", file.Path)
+					skippedFileSize += file.Size
+					skippedFiles++
 					continue
 				}
 			}
 
 			if file.Size == 0 && file.IsSameAsFileInfo(stat) {
 				LOG_TRACE("RESTORE_SKIP", "File %s unchanged (size 0)", file.Path)
+				skippedFileSize += file.Size
+				skippedFiles++
 				continue
 			}
 		} else {
-			err = os.MkdirAll(path.Dir(fullPath), 0744)
+			parent, _ := SplitDir(fullPath)
+			err = os.MkdirAll(parent, 0744)
 			if err != nil {
 				LOG_ERROR("DOWNLOAD_MKDIR", "Failed to create directory: %v", err)
 			}
@@ -909,22 +961,39 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
 			newFile, err := os.OpenFile(fullPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, file.GetPermissions())
 			if err != nil {
 				LOG_ERROR("DOWNLOAD_OPEN", "Failed to create empty file: %v", err)
-				return false
+				return 0
 			}
 			newFile.Close()
 
 			file.RestoreMetadata(fullPath, nil, setOwner)
 			if !showStatistics {
 				LOG_INFO("DOWNLOAD_DONE", "Downloaded %s (0)", file.Path)
+				downloadedFileSize += file.Size
+				downloadedFiles = append(downloadedFiles, file)
 			}
 
 			continue
 		}
 
-		if manager.RestoreFile(chunkDownloader, chunkMaker, file, top, inPlace, overwrite, showStatistics,
-			totalFileSize, downloadedFileSize, startDownloadingTime) {
+		downloaded, err := manager.RestoreFile(chunkDownloader, chunkMaker, file, top, inPlace, overwrite, showStatistics,
+			totalFileSize, downloadedFileSize, startDownloadingTime, allowFailures)
+		if err != nil {
+			// RestoreFile returned an error; if allowFailures is false RestoerFile would error out and not return so here
+			// we just need to show a warning
+			failedFiles++
+			LOG_WARN("DOWNLOAD_FAIL", "Failed to restore %s: %v", file.Path, err)
+			continue
+		}
+
+		// No error
+		if downloaded {
+			// No error, file was restored
 			downloadedFileSize += file.Size
 			downloadedFiles = append(downloadedFiles, file)
+		} else {
+			// No error, file was skipped
+			skippedFileSize += file.Size
+			skippedFiles++
 		}
 		file.RestoreMetadata(fullPath, nil, setOwner)
 	}
@@ -952,11 +1021,16 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
 		}
 	}
 
+	if failedFiles > 0 {
+		return failedFiles
+	}
+
 	LOG_INFO("RESTORE_END", "Restored %s to revision %d", top, revision)
 	if showStatistics {
 		LOG_INFO("RESTORE_STATS", "Files: %d total, %s bytes", len(fileEntries), PrettySize(totalFileSize))
 		LOG_INFO("RESTORE_STATS", "Downloaded %d file, %s bytes, %d chunks",
 			len(downloadedFiles), PrettySize(downloadedFileSize), chunkDownloader.numberOfDownloadedChunks)
+		LOG_INFO("RESTORE_STATS", "Skipped %d file, %s bytes", skippedFiles, PrettySize(skippedFileSize))
 	}
 
 	runningTime := time.Now().Unix() - startTime
@@ -968,7 +1042,7 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
 
 	chunkDownloader.Stop()
 
-	return true
+	return 0
 }
 
 // fileEncoder encodes one file at a time to avoid loading the full json description of the entire file tree
@@ -1128,8 +1202,11 @@ func (manager *BackupManager) UploadSnapshot(chunkMaker *ChunkMaker, uploader *C
 // Restore downloads a file from the storage.  If 'inPlace' is false, the download file is saved first to a temporary
 // file under the .duplicacy directory and then replaces the existing one.  Otherwise, the existing file will be
 // overwritten directly.
+// Return: true, nil:    Restored file; 
+//         false, nil:   Skipped file; 
+//         false, error: Failure to restore file (only if allowFailures == true)
 func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chunkMaker *ChunkMaker, entry *Entry, top string, inPlace bool, overwrite bool,
-	showStatistics bool, totalFileSize int64, downloadedFileSize int64, startTime int64) bool {
+	showStatistics bool, totalFileSize int64, downloadedFileSize int64, startTime int64, allowFailures bool) (bool, error) {
 
 	LOG_TRACE("DOWNLOAD_START", "Downloading %s", entry.Path)
 
@@ -1174,7 +1251,7 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
 				existingFile, err = os.OpenFile(fullPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
 				if err != nil {
 					LOG_ERROR("DOWNLOAD_CREATE", "Failed to create the file %s for in-place writing: %v", fullPath, err)
-					return false
+					return false, nil
 				}
 
 				n := int64(1)
@@ -1186,30 +1263,24 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
 				_, err = existingFile.Seek(entry.Size-n, 0)
 				if err != nil {
 					LOG_ERROR("DOWNLOAD_CREATE", "Failed to resize the initial file %s for in-place writing: %v", fullPath, err)
-					return false
+					return false, nil
 				}
 				_, err = existingFile.Write([]byte("\x00\x00")[:n])
 				if err != nil {
 					LOG_ERROR("DOWNLOAD_CREATE", "Failed to initialize the sparse file %s for in-place writing: %v", fullPath, err)
-					return false
+					return false, nil
 				}
 				existingFile.Close()
 				existingFile, err = os.Open(fullPath)
 				if err != nil {
 					LOG_ERROR("DOWNLOAD_OPEN", "Can't reopen the initial file just created: %v", err)
-					return false
+					return false, nil
 				}
 				isNewFile = true
 			}
 		} else {
 			LOG_TRACE("DOWNLOAD_OPEN", "Can't open the existing file: %v", err)
 		}
-	} else {
-		if !overwrite {
-			LOG_ERROR("DOWNLOAD_OVERWRITE",
-				"File %s already exists.  Please specify the -overwrite option to continue", entry.Path)
-			return false
-		}
 	}
 
 	// The key in this map is the number of zeroes.  The value is the corresponding hash.
@@ -1278,7 +1349,7 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
 					}
 					if err != nil {
 						LOG_ERROR("DOWNLOAD_SPLIT", "Failed to read existing file: %v", err)
-						return false
+						return false, nil
 					}
 				}
 				if count > 0 {
@@ -1299,6 +1370,19 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
 			}
 
 			fileHash = hex.EncodeToString(fileHasher.Sum(nil))
+
+			if fileHash == entry.Hash && fileHash != "" {
+				LOG_TRACE("DOWNLOAD_SKIP", "File %s unchanged (by hash)", entry.Path)
+				return false, nil
+			}
+
+			// fileHash != entry.Hash, warn/error depending on -overwrite option
+			if !overwrite {
+				LOG_WERROR(allowFailures, "DOWNLOAD_OVERWRITE",
+							"File %s already exists.  Please specify the -overwrite option to overwrite", entry.Path)
+				return false, fmt.Errorf("file exists")
+			}
+
 		} else {
 			// If it is not inplace, we want to reuse any chunks in the existing file regardless their offets, so
 			// we run the chunk maker to split the original file.
@@ -1318,9 +1402,11 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
 					return nil, false
 				})
 		}
+
+		// This is an additional check comparing fileHash to entry.Hash above, so this should no longer occur
 		if fileHash == entry.Hash && fileHash != "" {
 			LOG_TRACE("DOWNLOAD_SKIP", "File %s unchanged (by hash)", entry.Path)
-			return false
+			return false, nil
 		}
 	}
 
@@ -1348,7 +1434,7 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
 			existingFile, err = os.OpenFile(fullPath, os.O_RDWR, 0)
 			if err != nil {
 				LOG_ERROR("DOWNLOAD_OPEN", "Failed to open the file %s for in-place writing", fullPath)
-				return false
+				return false, nil
 			}
 		}
 
@@ -1380,7 +1466,7 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
 			_, err = existingFile.Seek(offset, 0)
 			if err != nil {
 				LOG_ERROR("DOWNLOAD_SEEK", "Failed to set the offset to %d for file %s: %v", offset, fullPath, err)
-				return false
+				return false, nil
 			}
 
 			// Check if the chunk is available in the existing file
@@ -1390,17 +1476,20 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
 				_, err := io.CopyN(hasher, existingFile, int64(existingLengths[j]))
 				if err != nil {
 					LOG_ERROR("DOWNLOAD_READ", "Failed to read the existing chunk %s: %v", hash, err)
-					return false
+					return false, nil
 				}
 				if IsDebugging() {
 					LOG_DEBUG("DOWNLOAD_UNCHANGED", "Chunk %s is unchanged", manager.config.GetChunkIDFromHash(hash))
 				}
 			} else {
 				chunk := chunkDownloader.WaitForChunk(i)
+				if chunk.isBroken {
+					return false, fmt.Errorf("chunk %s is corrupted", manager.config.GetChunkIDFromHash(hash))
+				}
 				_, err = existingFile.Write(chunk.GetBytes()[start:end])
 				if err != nil {
 					LOG_ERROR("DOWNLOAD_WRITE", "Failed to write to the file: %v", err)
-					return false
+					return false, nil
 				}
 				hasher.Write(chunk.GetBytes()[start:end])
 			}
@@ -1411,15 +1500,15 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
 		// Must truncate the file if the new size is smaller
 		if err = existingFile.Truncate(offset); err != nil {
 			LOG_ERROR("DOWNLOAD_TRUNCATE", "Failed to truncate the file at %d: %v", offset, err)
-			return false
+			return false, nil
 		}
 
 		// Verify the download by hash
 		hash := hex.EncodeToString(hasher.Sum(nil))
 		if hash != entry.Hash && hash != "" && entry.Hash != "" && !strings.HasPrefix(entry.Hash, "#") {
-			LOG_ERROR("DOWNLOAD_HASH", "File %s has a mismatched hash: %s instead of %s (in-place)",
+			LOG_WERROR(allowFailures, "DOWNLOAD_HASH", "File %s has a mismatched hash: %s instead of %s (in-place)",
 				fullPath, "", entry.Hash)
-			return false
+			return false, fmt.Errorf("file corrupt (hash mismatch)")
 		}
 
 	} else {
@@ -1428,7 +1517,7 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
 		newFile, err = os.OpenFile(temporaryPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
 		if err != nil {
 			LOG_ERROR("DOWNLOAD_OPEN", "Failed to open file for writing: %v", err)
-			return false
+			return false, nil
 		}
 
 		hasher := manager.config.NewFileHasher()
@@ -1466,6 +1555,9 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
 
 			if !hasLocalCopy {
 				chunk := chunkDownloader.WaitForChunk(i)
+				if chunk.isBroken {
+					return false, fmt.Errorf("chunk %s is corrupted", manager.config.GetChunkIDFromHash(hash))
+				}
 				// If the chunk was downloaded from the storage, we may still need a portion of it.
 				start := 0
 				if i == entry.StartChunk {
@@ -1481,7 +1573,7 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
 			_, err = newFile.Write(data)
 			if err != nil {
 				LOG_ERROR("DOWNLOAD_WRITE", "Failed to write file: %v", err)
-				return false
+				return false, nil
 			}
 
 			hasher.Write(data)
@@ -1490,9 +1582,9 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
 
 		hash := hex.EncodeToString(hasher.Sum(nil))
 		if hash != entry.Hash && hash != "" && entry.Hash != "" && !strings.HasPrefix(entry.Hash, "#") {
-			LOG_ERROR("DOWNLOAD_HASH", "File %s has a mismatched hash: %s instead of %s",
+			LOG_WERROR(allowFailures, "DOWNLOAD_HASH", "File %s has a mismatched hash: %s instead of %s",
 				entry.Path, hash, entry.Hash)
-			return false
+			return false, fmt.Errorf("file corrupt (hash mismatch)")
 		}
 
 		if existingFile != nil {
@@ -1506,31 +1598,40 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
 		err = os.Remove(fullPath)
 		if err != nil && !os.IsNotExist(err) {
 			LOG_ERROR("DOWNLOAD_REMOVE", "Failed to remove the old file: %v", err)
-			return false
+			return false, nil
 		}
 
 		err = os.Rename(temporaryPath, fullPath)
 		if err != nil {
 			LOG_ERROR("DOWNLOAD_RENAME", "Failed to rename the file %s to %s: %v", temporaryPath, fullPath, err)
-			return false
+			return false, nil
 		}
 	}
 
 	if !showStatistics {
 		LOG_INFO("DOWNLOAD_DONE", "Downloaded %s (%d)", entry.Path, entry.Size)
 	}
-	return true
+	return true, nil
 }
 
 // CopySnapshots copies the specified snapshots from one storage to the other.
 func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapshotID string,
-	revisionsToBeCopied []int, threads int) bool {
+	revisionsToBeCopied []int, uploadingThreads int, downloadingThreads int) bool {
 
 	if !manager.config.IsCompatiableWith(otherManager.config) {
 		LOG_ERROR("CONFIG_INCOMPATIBLE", "Two storages are not compatible for the copy operation")
 		return false
 	}
 
+	if otherManager.config.DataShards != 0 && otherManager.config.ParityShards != 0 {
+		LOG_INFO("BACKUP_ERASURECODING", "Erasure coding is enabled for the destination storage with %d data shards and %d parity shards",
+		         otherManager.config.DataShards, otherManager.config.ParityShards)
+	}
+
+	if otherManager.config.rsaPublicKey != nil && len(otherManager.config.FileKey) > 0 {
+		LOG_INFO("BACKUP_KEY", "RSA encryption is enabled for the destination")
+	}
+
 	if snapshotID == "" && len(revisionsToBeCopied) > 0 {
 		LOG_ERROR("SNAPSHOT_ERROR", "You must specify the snapshot id when one or more revisions are specified.")
 		return false
@@ -1610,6 +1711,9 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho
 		return true
 	}
 
+	// These two maps store hashes of chunks in the source and destination storages, respectively.  Note that
+	// the value of 'chunks' is used to indicated if the chunk is a snapshot chunk, while the value of 'otherChunks' 
+	// is not used.
 	chunks := make(map[string]bool)
 	otherChunks := make(map[string]bool)
 
@@ -1622,21 +1726,15 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho
 		LOG_TRACE("SNAPSHOT_COPY", "Copying snapshot %s at revision %d", snapshot.ID, snapshot.Revision)
 
 		for _, chunkHash := range snapshot.FileSequence {
-			if _, found := chunks[chunkHash]; !found {
-				chunks[chunkHash] = true
-			}
+			chunks[chunkHash] = true  // The chunk is a snapshot chunk
 		}
 
 		for _, chunkHash := range snapshot.ChunkSequence {
-			if _, found := chunks[chunkHash]; !found {
-				chunks[chunkHash] = true
-			}
+			chunks[chunkHash] = true  // The chunk is a snapshot chunk
 		}
 
 		for _, chunkHash := range snapshot.LengthSequence {
-			if _, found := chunks[chunkHash]; !found {
-				chunks[chunkHash] = true
-			}
+			chunks[chunkHash] = true  // The chunk is a snapshot chunk
 		}
 
 		description := manager.SnapshotManager.DownloadSequence(snapshot.ChunkSequence)
@@ -1649,9 +1747,11 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho
 
 		for _, chunkHash := range snapshot.ChunkHashes {
 			if _, found := chunks[chunkHash]; !found {
-				chunks[chunkHash] = true
+				chunks[chunkHash] = false  // The chunk is a file chunk
 			}
 		}
+
+		snapshot.ChunkHashes = nil
 	}
 
 	otherChunkFiles, otherChunkSizes := otherManager.SnapshotManager.ListAllFiles(otherManager.storage, "chunks/")
@@ -1670,62 +1770,64 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho
 
 	LOG_DEBUG("SNAPSHOT_COPY", "Found %d chunks on destination storage", len(otherChunks))
 
-	chunksToCopy := 0
-	chunksToSkip := 0
+	var chunksToCopy []string
 
 	for chunkHash := range chunks {
 		otherChunkID := otherManager.config.GetChunkIDFromHash(chunkHash)
-		if _, found := otherChunks[otherChunkID]; found {
-			chunksToSkip++
-		} else {
-			chunksToCopy++
+		if _, found := otherChunks[otherChunkID]; !found {
+			chunksToCopy = append(chunksToCopy, chunkHash)
 		}
 	}
 
-	LOG_DEBUG("SNAPSHOT_COPY", "Chunks to copy = %d, to skip = %d, total = %d", chunksToCopy, chunksToSkip, chunksToCopy+chunksToSkip)
-	LOG_DEBUG("SNAPSHOT_COPY", "Total chunks in source snapshot revisions = %d\n", len(chunks))
+	LOG_INFO("SNAPSHOT_COPY", "Chunks to copy: %d, to skip: %d, total: %d", len(chunksToCopy), len(chunks) - len(chunksToCopy), len(chunks))
 
-	chunkDownloader := CreateChunkDownloader(manager.config, manager.storage, nil, false, threads)
+	chunkDownloader := CreateChunkDownloader(manager.config, manager.storage, nil, false, downloadingThreads, false)
 
-	chunkUploader := CreateChunkUploader(otherManager.config, otherManager.storage, nil, threads,
+	var uploadedBytes int64
+	startTime := time.Now()
+
+	copiedChunks := 0
+	chunkUploader := CreateChunkUploader(otherManager.config, otherManager.storage, nil, uploadingThreads,
 		func(chunk *Chunk, chunkIndex int, skipped bool, chunkSize int, uploadSize int) {
-			if skipped {
-				LOG_INFO("SNAPSHOT_COPY", "Chunk %s (%d/%d) exists at the destination", chunk.GetID(), chunkIndex, len(chunks))
-			} else {
-				LOG_INFO("SNAPSHOT_COPY", "Chunk %s (%d/%d) copied to the destination", chunk.GetID(), chunkIndex, len(chunks))
+			action := "Skipped"
+			if !skipped {
+				copiedChunks++
+				action = "Copied"
 			}
+
+			atomic.AddInt64(&uploadedBytes, int64(chunkSize))
+
+			elapsedTime := time.Now().Sub(startTime).Seconds()
+			speed := int64(float64(atomic.LoadInt64(&uploadedBytes)) / elapsedTime)
+			remainingTime := int64(float64(len(chunksToCopy) - chunkIndex - 1) / float64(chunkIndex + 1) * elapsedTime)
+			percentage := float64(chunkIndex + 1) / float64(len(chunksToCopy)) * 100.0
+			LOG_INFO("COPY_PROGRESS", "%s chunk %s (%d/%d) %sB/s %s %.1f%%",
+					action, chunk.GetID(), chunkIndex + 1, len(chunksToCopy),
+					PrettySize(speed), PrettyTime(remainingTime), percentage)
 			otherManager.config.PutChunk(chunk)
 		})
 
 	chunkUploader.Start()
 
-	totalCopied := 0
-	totalSkipped := 0
-	chunkIndex := 0
-
-	for chunkHash := range chunks {
-		chunkIndex++
+	for _, chunkHash := range chunksToCopy {
+		chunkDownloader.AddChunk(chunkHash)
+	}
+	for i, chunkHash := range chunksToCopy {
 		chunkID := manager.config.GetChunkIDFromHash(chunkHash)
 		newChunkID := otherManager.config.GetChunkIDFromHash(chunkHash)
-		if _, found := otherChunks[newChunkID]; !found {
-			LOG_DEBUG("SNAPSHOT_COPY", "Copying chunk %s to %s", chunkID, newChunkID)
-			i := chunkDownloader.AddChunk(chunkHash)
-			chunk := chunkDownloader.WaitForChunk(i)
-			newChunk := otherManager.config.GetChunk()
-			newChunk.Reset(true)
-			newChunk.Write(chunk.GetBytes())
-			chunkUploader.StartChunk(newChunk, chunkIndex)
-			totalCopied++
-		} else {
-			LOG_INFO("SNAPSHOT_COPY", "Chunk %s (%d/%d) skipped at the destination", chunkID, chunkIndex, len(chunks))
-			totalSkipped++
-		}
+		LOG_DEBUG("SNAPSHOT_COPY", "Copying chunk %s to %s", chunkID, newChunkID)
+		chunk := chunkDownloader.WaitForChunk(i)
+		newChunk := otherManager.config.GetChunk()
+		newChunk.Reset(true)
+		newChunk.Write(chunk.GetBytes())
+		newChunk.isSnapshot = chunks[chunkHash]
+		chunkUploader.StartChunk(newChunk, i)
 	}
 
 	chunkDownloader.Stop()
 	chunkUploader.Stop()
 
-	LOG_INFO("SNAPSHOT_COPY", "Copy complete, %d total chunks, %d chunks copied, %d skipped", totalCopied+totalSkipped, totalCopied, totalSkipped)
+	LOG_INFO("SNAPSHOT_COPY", "Copied %d new chunks and skipped %d existing chunks", copiedChunks, len(chunks) - copiedChunks)
 
 	for _, snapshot := range snapshots {
 		if revisionMap[snapshot.ID][snapshot.Revision] == false {

src/duplicacy_backupmanager_test.go

@@ -169,6 +169,12 @@ func getFileHash(path string) (hash string) {
 	return hex.EncodeToString(hasher.Sum(nil))
 }
 
+func assertRestoreFailures(t *testing.T, failedFiles int, expectedFailedFiles int) {
+	if failedFiles != expectedFailedFiles {
+		t.Errorf("Failed to restore %d instead of %d file(s)", failedFiles, expectedFailedFiles)
+	}
+}
+
 func TestBackupManager(t *testing.T) {
 
 	rand.Seed(time.Now().UnixNano())
@@ -226,12 +232,20 @@ func TestBackupManager(t *testing.T) {
 	cleanStorage(storage)
 
 	time.Sleep(time.Duration(delay) * time.Second)
+
+	dataShards := 0
+	parityShards := 0
+	if testErasureCoding {
+		dataShards = 5
+		parityShards = 2
+	}
+
 	if testFixedChunkSize {
-		if !ConfigStorage(storage, 16384, 100, 64*1024, 64*1024, 64*1024, password, nil, false) {
+		if !ConfigStorage(storage, 16384, 100, 64*1024, 64*1024, 64*1024, password, nil, false, "", dataShards, parityShards) {
 			t.Errorf("Failed to initialize the storage")
 		}
 	} else {
-		if !ConfigStorage(storage, 16384, 100, 64*1024, 256*1024, 16*1024, password, nil, false) {
+		if !ConfigStorage(storage, 16384, 100, 64*1024, 256*1024, 16*1024, password, nil, false, "", dataShards, parityShards) {
 			t.Errorf("Failed to initialize the storage")
 		}
 	}
@@ -239,15 +253,16 @@ func TestBackupManager(t *testing.T) {
 	time.Sleep(time.Duration(delay) * time.Second)
 
 	SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy")
-	backupManager := CreateBackupManager("host1", storage, testDir, password, "")
+	backupManager := CreateBackupManager("host1", storage, testDir, password, "", "", false)
 	backupManager.SetupSnapshotCache("default")
 
 	SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy")
 	backupManager.Backup(testDir+"/repository1" /*quickMode=*/, true, threads, "first", false, false, 0, false)
 	time.Sleep(time.Duration(delay) * time.Second)
 	SetDuplicacyPreferencePath(testDir + "/repository2/.duplicacy")
-	backupManager.Restore(testDir+"/repository2", threads /*inPlace=*/, false /*quickMode=*/, false, threads /*overwrite=*/, true,
-		/*deleteMode=*/ false /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, nil)
+	failedFiles := backupManager.Restore(testDir+"/repository2", threads /*inPlace=*/, false /*quickMode=*/, false, threads /*overwrite=*/, true,
+		/*deleteMode=*/ false /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, nil /*allowFailures=*/, false)
+	assertRestoreFailures(t, failedFiles, 0)
 
 	for _, f := range []string{"file1", "file2", "dir1/file3"} {
 		if _, err := os.Stat(testDir + "/repository2/" + f); os.IsNotExist(err) {
@@ -270,8 +285,9 @@ func TestBackupManager(t *testing.T) {
 	backupManager.Backup(testDir+"/repository1" /*quickMode=*/, true, threads, "second", false, false, 0, false)
 	time.Sleep(time.Duration(delay) * time.Second)
 	SetDuplicacyPreferencePath(testDir + "/repository2/.duplicacy")
-	backupManager.Restore(testDir+"/repository2", 2 /*inPlace=*/, true /*quickMode=*/, true, threads /*overwrite=*/, true,
-		/*deleteMode=*/ false /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, nil)
+	failedFiles = backupManager.Restore(testDir+"/repository2", 2 /*inPlace=*/, true /*quickMode=*/, true, threads /*overwrite=*/, true,
+		/*deleteMode=*/ false /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, nil /*allowFailures=*/, false)
+	assertRestoreFailures(t, failedFiles, 0)
 
 	for _, f := range []string{"file1", "file2", "dir1/file3"} {
 		hash1 := getFileHash(testDir + "/repository1/" + f)
@@ -298,8 +314,9 @@ func TestBackupManager(t *testing.T) {
 	createRandomFile(testDir+"/repository2/dir5/file5", 100)
 
 	SetDuplicacyPreferencePath(testDir + "/repository2/.duplicacy")
-	backupManager.Restore(testDir+"/repository2", 3 /*inPlace=*/, true /*quickMode=*/, false, threads /*overwrite=*/, true,
-		/*deleteMode=*/ true /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, nil)
+	failedFiles = backupManager.Restore(testDir+"/repository2", 3 /*inPlace=*/, true /*quickMode=*/, false, threads /*overwrite=*/, true,
+		/*deleteMode=*/ true /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, nil /*allowFailures=*/, false)
+	assertRestoreFailures(t, failedFiles, 0)
 
 	for _, f := range []string{"file1", "file2", "dir1/file3"} {
 		hash1 := getFileHash(testDir + "/repository1/" + f)
@@ -325,8 +342,9 @@ func TestBackupManager(t *testing.T) {
 	os.Remove(testDir + "/repository1/file2")
 	os.Remove(testDir + "/repository1/dir1/file3")
 	SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy")
-	backupManager.Restore(testDir+"/repository1", 3 /*inPlace=*/, true /*quickMode=*/, false, threads /*overwrite=*/, true,
-		/*deleteMode=*/ false /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, []string{"+file2", "+dir1/file3", "-*"})
+	failedFiles = backupManager.Restore(testDir+"/repository1", 3 /*inPlace=*/, true /*quickMode=*/, false, threads /*overwrite=*/, true,
+		/*deleteMode=*/ false /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, []string{"+file2", "+dir1/file3", "-*"} /*allowFailures=*/, false)
+	assertRestoreFailures(t, failedFiles, 0)
 
 	for _, f := range []string{"file1", "file2", "dir1/file3"} {
 		hash1 := getFileHash(testDir + "/repository1/" + f)
@@ -341,7 +359,7 @@ func TestBackupManager(t *testing.T) {
 		t.Errorf("Expected 3 snapshots but got %d", numberOfSnapshots)
 	}
 	backupManager.SnapshotManager.CheckSnapshots( /*snapshotID*/ "host1" /*revisions*/, []int{1, 2, 3} /*tag*/, "",
-		/*showStatistics*/ false /*showTabular*/, false /*checkFiles*/, false /*searchFossils*/, false /*resurrect*/, false)
+		/*showStatistics*/ false /*showTabular*/, false /*checkFiles*/, false /*checkChunks*/, false /*searchFossils*/, false /*resurrect*/, false, 1 /*allowFailures*/, false)
 	backupManager.SnapshotManager.PruneSnapshots("host1", "host1" /*revisions*/, []int{1} /*tags*/, nil /*retentions*/, nil,
 		/*exhaustive*/ false /*exclusive=*/, false /*ignoredIDs*/, nil /*dryRun*/, false /*deleteOnly*/, false /*collectOnly*/, false, 1)
 	numberOfSnapshots = backupManager.SnapshotManager.ListSnapshots( /*snapshotID*/ "host1" /*revisionsToList*/, nil /*tag*/, "" /*showFiles*/, false /*showChunks*/, false)
@@ -349,7 +367,7 @@ func TestBackupManager(t *testing.T) {
 		t.Errorf("Expected 2 snapshots but got %d", numberOfSnapshots)
 	}
 	backupManager.SnapshotManager.CheckSnapshots( /*snapshotID*/ "host1" /*revisions*/, []int{2, 3} /*tag*/, "",
-		/*showStatistics*/ false /*showTabular*/, false /*checkFiles*/, false /*searchFossils*/, false /*resurrect*/, false)
+		/*showStatistics*/ false /*showTabular*/, false /*checkFiles*/, false /*checkChunks*/, false /*searchFossils*/, false /*resurrect*/, false, 1 /*allowFailures*/, false)
 	backupManager.Backup(testDir+"/repository1" /*quickMode=*/, false, threads, "fourth", false, false, 0, false)
 	backupManager.SnapshotManager.PruneSnapshots("host1", "host1" /*revisions*/, nil /*tags*/, nil /*retentions*/, nil,
 		/*exhaustive*/ false /*exclusive=*/, true /*ignoredIDs*/, nil /*dryRun*/, false /*deleteOnly*/, false /*collectOnly*/, false, 1)
@@ -358,9 +376,348 @@ func TestBackupManager(t *testing.T) {
 		t.Errorf("Expected 3 snapshots but got %d", numberOfSnapshots)
 	}
 	backupManager.SnapshotManager.CheckSnapshots( /*snapshotID*/ "host1" /*revisions*/, []int{2, 3, 4} /*tag*/, "",
-		/*showStatistics*/ false /*showTabular*/, false /*checkFiles*/, false /*searchFossils*/, false /*resurrect*/, false)
+		/*showStatistics*/ false /*showTabular*/, false /*checkFiles*/, false /*checkChunks*/, false /*searchFossils*/, false /*resurrect*/, false, 1 /*allowFailures*/, false)
 
 	/*buf := make([]byte, 1<<16)
 	  runtime.Stack(buf, true)
 	  fmt.Printf("%s", buf)*/
 }
+
+// Create file with random file with certain seed
+func createRandomFileSeeded(path string, maxSize int, seed int64) {
+	rand.Seed(seed)
+	file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
+	if err != nil {
+		LOG_ERROR("RANDOM_FILE", "Can't open %s for writing: %v", path, err)
+		return
+	}
+
+	defer file.Close()
+
+	size := maxSize/2 + rand.Int()%(maxSize/2)
+
+	buffer := make([]byte, 32*1024)
+	for size > 0 {
+		bytes := size
+		if bytes > cap(buffer) {
+			bytes = cap(buffer)
+		}
+		rand.Read(buffer[:bytes])
+		bytes, err = file.Write(buffer[:bytes])
+		if err != nil {
+			LOG_ERROR("RANDOM_FILE", "Failed to write to %s: %v", path, err)
+			return
+		}
+		size -= bytes
+	}
+}
+
+func corruptFile(path string, start int, length int, seed int64) {
+	rand.Seed(seed)
+
+	file, err := os.OpenFile(path, os.O_WRONLY, 0644)
+	if err != nil {
+		LOG_ERROR("CORRUPT_FILE", "Can't open %s for writing: %v", path, err)
+		return
+	}
+
+	defer func() {
+		if file != nil {
+			file.Close()
+		}
+	}()
+
+	_, err = file.Seek(int64(start), 0)
+	if err != nil {
+		LOG_ERROR("CORRUPT_FILE", "Can't seek to the offset %d: %v", start, err)
+		return
+	}
+
+	buffer := make([]byte, length)
+	rand.Read(buffer)
+
+	_, err = file.Write(buffer)
+	if err != nil {
+		LOG_ERROR("CORRUPT_FILE", "Failed to write to %s: %v", path, err)
+		return
+	}
+}
+
+func TestPersistRestore(t *testing.T) {
+	// We want deterministic output here so we can test the expected files are corrupted by missing or corrupt chunks
+	// There use rand functions with fixed seed, and known keys
+
+	setTestingT(t)
+	SetLoggingLevel(INFO)
+
+	defer func() {
+		if r := recover(); r != nil {
+			switch e := r.(type) {
+			case Exception:
+				t.Errorf("%s %s", e.LogID, e.Message)
+				debug.PrintStack()
+			default:
+				t.Errorf("%v", e)
+				debug.PrintStack()
+			}
+		}
+	}()
+
+	testDir := path.Join(os.TempDir(), "duplicacy_test")
+	os.RemoveAll(testDir)
+	os.MkdirAll(testDir, 0700)
+	os.Mkdir(testDir+"/repository1", 0700)
+	os.Mkdir(testDir+"/repository1/dir1", 0700)
+	os.Mkdir(testDir+"/repository1/.duplicacy", 0700)
+	os.Mkdir(testDir+"/repository2", 0700)
+	os.Mkdir(testDir+"/repository2/.duplicacy", 0700)
+	os.Mkdir(testDir+"/repository3", 0700)
+	os.Mkdir(testDir+"/repository3/.duplicacy", 0700)
+
+	maxFileSize := 1000000
+	//maxFileSize := 200000
+
+	createRandomFileSeeded(testDir+"/repository1/file1", maxFileSize,1)
+	createRandomFileSeeded(testDir+"/repository1/file2", maxFileSize,2)
+	createRandomFileSeeded(testDir+"/repository1/dir1/file3", maxFileSize,3)
+
+	threads := 1
+
+	password := "duplicacy"
+
+	// We want deterministic output, plus ability to test encrypted storage
+	// So make unencrypted storage with default keys, and encrypted as bit-identical copy of this but with password
+	unencStorage, err := loadStorage(testDir+"/unenc_storage", threads)
+	if err != nil {
+		t.Errorf("Failed to create storage: %v", err)
+		return
+	}
+	delay := 0
+	if _, ok := unencStorage.(*ACDStorage); ok {
+		delay = 1
+	}
+	if _, ok := unencStorage.(*OneDriveStorage); ok {
+		delay = 5
+	}
+
+	time.Sleep(time.Duration(delay) * time.Second)
+	cleanStorage(unencStorage)
+
+	if !ConfigStorage(unencStorage, 16384, 100, 64*1024, 256*1024, 16*1024, "", nil, false, "", 0, 0) {
+		t.Errorf("Failed to initialize the unencrypted storage")
+	}
+	time.Sleep(time.Duration(delay) * time.Second)
+	unencConfig, _, err := DownloadConfig(unencStorage, "")
+	if err != nil {
+		t.Errorf("Failed to download storage config: %v", err)
+		return
+	}
+
+	// Make encrypted storage
+	storage, err := loadStorage(testDir+"/enc_storage", threads)
+	if err != nil {
+		t.Errorf("Failed to create encrypted storage: %v", err)
+		return
+	}
+	time.Sleep(time.Duration(delay) * time.Second)
+	cleanStorage(storage)
+
+	if !ConfigStorage(storage, 16384, 100, 64*1024, 256*1024, 16*1024, password, unencConfig, true, "", 0, 0) {
+		t.Errorf("Failed to initialize the encrypted storage")
+	}
+	time.Sleep(time.Duration(delay) * time.Second)
+
+	// do unencrypted backup
+	SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy")
+	unencBackupManager := CreateBackupManager("host1", unencStorage, testDir, "", "", "", false)
+	unencBackupManager.SetupSnapshotCache("default")
+
+	SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy")
+	unencBackupManager.Backup(testDir+"/repository1" /*quickMode=*/, true, threads, "first", false, false, 0, false)
+	time.Sleep(time.Duration(delay) * time.Second)
+
+
+	// do encrypted backup
+	SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy")
+	encBackupManager := CreateBackupManager("host1", storage, testDir, password, "", "", false)
+	encBackupManager.SetupSnapshotCache("default")
+
+	SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy")
+	encBackupManager.Backup(testDir+"/repository1" /*quickMode=*/, true, threads, "first", false, false, 0, false)
+	time.Sleep(time.Duration(delay) * time.Second)
+
+
+	// check snapshots
+	unencBackupManager.SnapshotManager.CheckSnapshots( /*snapshotID*/ "host1" /*revisions*/, []int{1} /*tag*/, "",
+		/*showStatistics*/ true /*showTabular*/, false /*checkFiles*/, true /*checkChunks*/, false,
+		/*searchFossils*/ false /*resurrect*/, false, 1 /*allowFailures*/, false)
+
+	encBackupManager.SnapshotManager.CheckSnapshots( /*snapshotID*/ "host1" /*revisions*/, []int{1} /*tag*/, "",
+		/*showStatistics*/ true /*showTabular*/, false /*checkFiles*/, true /*checkChunks*/, false,
+		 /*searchFossils*/ false /*resurrect*/, false, 1 /*allowFailures*/, false)
+		
+	// check functions
+	checkAllUncorrupted := func(cmpRepository string) {
+			for _, f := range []string{"file1", "file2", "dir1/file3"} {
+				if _, err := os.Stat(testDir + cmpRepository + "/" + f); os.IsNotExist(err) {
+					t.Errorf("File %s does not exist", f)
+					continue
+				}
+
+				hash1 := getFileHash(testDir + "/repository1/" + f)
+				hash2 := getFileHash(testDir + cmpRepository + "/" + f)
+				if hash1 != hash2 {
+					t.Errorf("File %s has different hashes: %s vs %s", f, hash1, hash2)
+				}
+		}
+	}
+	checkMissingFile := func(cmpRepository string, expectMissing string) {
+			for _, f := range []string{"file1", "file2", "dir1/file3"} {
+				_, err := os.Stat(testDir + cmpRepository + "/" + f)
+				if err==nil {
+					if f==expectMissing {
+						t.Errorf("File %s exists, expected to be missing", f)
+					}
+					continue
+				}
+				if os.IsNotExist(err) {
+					if f!=expectMissing {
+						t.Errorf("File %s does not exist", f)
+					}
+					continue
+				}
+
+				hash1 := getFileHash(testDir + "/repository1/" + f)
+				hash2 := getFileHash(testDir + cmpRepository + "/" + f)
+				if hash1 != hash2 {
+					t.Errorf("File %s has different hashes: %s vs %s", f, hash1, hash2)
+				}
+		}
+	}
+	checkCorruptedFile  := func(cmpRepository string, expectCorrupted string) {
+		for _, f := range []string{"file1", "file2", "dir1/file3"} {
+			if _, err := os.Stat(testDir + cmpRepository + "/" + f); os.IsNotExist(err) {
+				t.Errorf("File %s does not exist", f)
+				continue
+			}
+	
+			hash1 := getFileHash(testDir + "/repository1/" + f)
+			hash2 := getFileHash(testDir + cmpRepository + "/" + f)
+			if (f==expectCorrupted) {
+				if hash1 == hash2 {
+					t.Errorf("File %s has same hashes, expected to be corrupted: %s vs %s", f, hash1, hash2)
+				}
+	
+			} else {
+				if hash1 != hash2 {
+					t.Errorf("File %s has different hashes: %s vs %s", f, hash1, hash2)
+				}
+			}
+		}
+	}
+
+	// test restore all uncorrupted to repository3
+	SetDuplicacyPreferencePath(testDir + "/repository3/.duplicacy")
+	failedFiles := unencBackupManager.Restore(testDir+"/repository3", threads /*inPlace=*/, true /*quickMode=*/, false, threads /*overwrite=*/, false,
+		 /*deleteMode=*/ false /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, nil /*allowFailures=*/, false)
+	assertRestoreFailures(t, failedFiles, 0)
+	checkAllUncorrupted("/repository3")
+
+	// test for corrupt files and -persist
+	// corrupt a chunk 
+	chunkToCorrupt1 := "/4d/538e5dfd2b08e782bfeb56d1360fb5d7eb9d8c4b2531cc2fca79efbaec910c"
+		// this should affect file1
+	chunkToCorrupt2 := "/2b/f953a766d0196ce026ae259e76e3c186a0e4bcd3ce10f1571d17f86f0a5497"
+		// this should affect dir1/file3
+	
+	for i := 0; i < 2; i++ {
+		if i==0 {
+			// test corrupt chunks
+			corruptFile(testDir+"/unenc_storage"+"/chunks"+chunkToCorrupt1, 128, 128, 4)
+			corruptFile(testDir+"/enc_storage"+"/chunks"+chunkToCorrupt2, 128, 128, 4)
+		} else {
+			// test missing chunks
+			os.Remove(testDir+"/unenc_storage"+"/chunks"+chunkToCorrupt1)
+			os.Remove(testDir+"/enc_storage"+"/chunks"+chunkToCorrupt2)
+		}
+		
+		// check snapshots with --persist (allowFailures == true)
+		// this would cause a panic and os.Exit from duplicacy_log if allowFailures == false
+		unencBackupManager.SnapshotManager.CheckSnapshots( /*snapshotID*/ "host1" /*revisions*/, []int{1} /*tag*/, "",
+			/*showStatistics*/ true /*showTabular*/, false /*checkFiles*/, true /*checkChunks*/, false,
+			/*searchFossils*/ false /*resurrect*/, false, 1 /*allowFailures*/, true)
+
+		encBackupManager.SnapshotManager.CheckSnapshots( /*snapshotID*/ "host1" /*revisions*/, []int{1} /*tag*/, "",
+			/*showStatistics*/ true /*showTabular*/, false /*checkFiles*/, true /*checkChunks*/, false,
+			/*searchFossils*/ false /*resurrect*/, false, 1 /*allowFailures*/, true)
+
+		
+		// test restore corrupted, inPlace = true, corrupted files will have hash failures
+		os.RemoveAll(testDir+"/repository2")
+		SetDuplicacyPreferencePath(testDir + "/repository2/.duplicacy")
+		failedFiles = unencBackupManager.Restore(testDir+"/repository2", threads /*inPlace=*/, true /*quickMode=*/, false, threads /*overwrite=*/, false,
+			/*deleteMode=*/ false /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, nil /*allowFailures=*/, true)
+		assertRestoreFailures(t, failedFiles, 1)
+
+		// check restore, expect file1 to be corrupted
+		checkCorruptedFile("/repository2", "file1")
+
+		
+		os.RemoveAll(testDir+"/repository2")
+		SetDuplicacyPreferencePath(testDir + "/repository2/.duplicacy")
+		failedFiles = encBackupManager.Restore(testDir+"/repository2", threads /*inPlace=*/, true /*quickMode=*/, false, threads /*overwrite=*/, false,
+			/*deleteMode=*/ false /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, nil /*allowFailures=*/, true)
+		assertRestoreFailures(t, failedFiles, 1)
+
+		// check restore, expect file3 to be corrupted
+		checkCorruptedFile("/repository2", "dir1/file3")
+
+		//SetLoggingLevel(DEBUG)
+		// test restore corrupted, inPlace = false, corrupted files will be missing
+		os.RemoveAll(testDir+"/repository2")
+		SetDuplicacyPreferencePath(testDir + "/repository2/.duplicacy")
+		failedFiles = unencBackupManager.Restore(testDir+"/repository2", threads /*inPlace=*/, false /*quickMode=*/, false, threads /*overwrite=*/, false,
+			/*deleteMode=*/ false /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, nil /*allowFailures=*/, true)
+		assertRestoreFailures(t, failedFiles, 1)
+
+		// check restore, expect file1 to be corrupted
+		checkMissingFile("/repository2", "file1")
+
+		
+		os.RemoveAll(testDir+"/repository2")
+		SetDuplicacyPreferencePath(testDir + "/repository2/.duplicacy")
+		failedFiles = encBackupManager.Restore(testDir+"/repository2", threads /*inPlace=*/, false /*quickMode=*/, false, threads /*overwrite=*/, false,
+			/*deleteMode=*/ false /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, nil /*allowFailures=*/, true)
+		assertRestoreFailures(t, failedFiles, 1)
+
+		// check restore, expect file3 to be corrupted
+		checkMissingFile("/repository2", "dir1/file3")
+
+		// test restore corrupted files from different backups, inPlace = true
+		// with overwrite=true, corrupted file1 from unenc will be restored correctly from enc
+		// the latter will not touch the existing file3 with correct hash
+		os.RemoveAll(testDir+"/repository2")
+		failedFiles = unencBackupManager.Restore(testDir+"/repository2", threads /*inPlace=*/, true /*quickMode=*/, false, threads /*overwrite=*/, false,
+			/*deleteMode=*/ false /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, nil /*allowFailures=*/, true)
+		assertRestoreFailures(t, failedFiles, 1)
+
+		failedFiles = encBackupManager.Restore(testDir+"/repository2", threads /*inPlace=*/, true /*quickMode=*/, false, threads /*overwrite=*/, true,
+			/*deleteMode=*/ false /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, nil /*allowFailures=*/, true)
+		assertRestoreFailures(t, failedFiles, 0)
+		checkAllUncorrupted("/repository2")
+
+		// restore to repository3, with overwrite and allowFailures (true/false), quickMode = false (use hashes)
+		// should always succeed as uncorrupted files already exist with correct hash, so these will be ignored
+		SetDuplicacyPreferencePath(testDir + "/repository3/.duplicacy")
+		failedFiles = unencBackupManager.Restore(testDir+"/repository3", threads /*inPlace=*/, true /*quickMode=*/, false, threads /*overwrite=*/, true,
+			/*deleteMode=*/ false /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, nil /*allowFailures=*/, false)
+		assertRestoreFailures(t, failedFiles, 0)
+		checkAllUncorrupted("/repository3")
+
+		failedFiles = unencBackupManager.Restore(testDir+"/repository3", threads /*inPlace=*/, true /*quickMode=*/, false, threads /*overwrite=*/, true,
+			/*deleteMode=*/ false /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, nil /*allowFailures=*/, true)
+		assertRestoreFailures(t, failedFiles, 0)
+		checkAllUncorrupted("/repository3")
+	}
+
+}

src/duplicacy_benchmark.go

@@ -41,7 +41,7 @@ func benchmarkSplit(reader *bytes.Reader, fileSize int64, chunkSize int, compres
 				if encryption {
 					key = "0123456789abcdef0123456789abcdef"
 				}
-				err := chunk.Encrypt([]byte(key), "")
+				err := chunk.Encrypt([]byte(key), "", false)
 				if err != nil {
 					LOG_ERROR("BENCHMARK_ENCRYPT", "Failed to encrypt the chunk: %v", err)
 				}

src/duplicacy_chunk.go

@@ -8,11 +8,13 @@ import (
 	"bytes"
 	"compress/zlib"
 	"crypto/aes"
+	"crypto/rsa"
 	"crypto/cipher"
 	"crypto/hmac"
 	"crypto/rand"
 	"crypto/sha256"
 	"encoding/hex"
+	"encoding/binary"
 	"fmt"
 	"hash"
 	"io"
@@ -20,6 +22,8 @@ import (
 	"runtime"
 
 	"github.com/bkaradzic/go-lz4"
+	"github.com/minio/highwayhash"
+	"github.com/klauspost/reedsolomon"
 )
 
 // A chunk needs to acquire a new buffer and return the old one for every encrypt/decrypt operation, therefore
@@ -60,10 +64,20 @@ type Chunk struct {
 
 	config *Config // Every chunk is associated with a Config object.  Which hashing algorithm to use is determined
 	// by the config
+
+	isSnapshot bool // Indicates if the chunk is a snapshot chunk (instead of a file chunk).  This is only used by RSA
+	                // encryption, where a snapshot chunk is not encrypted by RSA
+	
+	isBroken bool // Indicates the chunk did not download correctly. This is only used for -persist (allowFailures) mode
 }
 
 // Magic word to identify a duplicacy format encrypted file, plus a version number.
-var ENCRYPTION_HEADER = "duplicacy\000"
+var ENCRYPTION_BANNER = "duplicacy\000"
+
+// RSA encrypted chunks start with "duplicacy\002"
+var ENCRYPTION_VERSION_RSA byte = 2
+
+var ERASURE_CODING_BANNER = "duplicacy\003"
 
 // CreateChunk creates a new chunk.
 func CreateChunk(config *Config, bufferNeeded bool) *Chunk {
@@ -113,6 +127,8 @@ func (chunk *Chunk) Reset(hashNeeded bool) {
 	chunk.hash = nil
 	chunk.id = ""
 	chunk.size = 0
+	chunk.isSnapshot = false
+	chunk.isBroken = false
 }
 
 // Write implements the Writer interface.
@@ -170,7 +186,7 @@ func (chunk *Chunk) VerifyID() {
 
 // Encrypt encrypts the plain data stored in the chunk buffer.  If derivationKey is not nil, the actual
 // encryption key will be HMAC-SHA256(encryptionKey, derivationKey).
-func (chunk *Chunk) Encrypt(encryptionKey []byte, derivationKey string) (err error) {
+func (chunk *Chunk) Encrypt(encryptionKey []byte, derivationKey string, isSnapshot bool) (err error) {
 
 	var aesBlock cipher.Block
 	var gcm cipher.AEAD
@@ -186,8 +202,17 @@ func (chunk *Chunk) Encrypt(encryptionKey []byte, derivationKey string) (err err
 	if len(encryptionKey) > 0 {
 
 		key := encryptionKey
-
-		if len(derivationKey) > 0 {
+		usingRSA := false
+		// Enable RSA encryption only when the chunk is not a snapshot chunk
+		if chunk.config.rsaPublicKey != nil && !isSnapshot && !chunk.isSnapshot {
+			randomKey := make([]byte, 32)
+			_, err := rand.Read(randomKey)
+			if err != nil {
+				return err
+			}
+			key = randomKey
+			usingRSA = true
+		} else if len(derivationKey) > 0 {
 			hasher := chunk.config.NewKeyedHasher([]byte(derivationKey))
 			hasher.Write(encryptionKey)
 			key = hasher.Sum(nil)
@@ -204,7 +229,21 @@ func (chunk *Chunk) Encrypt(encryptionKey []byte, derivationKey string) (err err
 		}
 
 		// Start with the magic number and the version number.
-		encryptedBuffer.Write([]byte(ENCRYPTION_HEADER))
+		if usingRSA {
+			// RSA encryption starts "duplicacy\002"
+			encryptedBuffer.Write([]byte(ENCRYPTION_BANNER)[:len(ENCRYPTION_BANNER) - 1])
+			encryptedBuffer.Write([]byte{ENCRYPTION_VERSION_RSA})
+
+			// Then the encrypted key
+			encryptedKey, err := rsa.EncryptOAEP(sha256.New(), rand.Reader, chunk.config.rsaPublicKey, key, nil)
+			if err != nil {
+				return err
+			}
+			binary.Write(encryptedBuffer, binary.LittleEndian, uint16(len(encryptedKey)))
+			encryptedBuffer.Write(encryptedKey)
+		} else {
+			encryptedBuffer.Write([]byte(ENCRYPTION_BANNER))
+		}
 
 		// Followed by the nonce
 		nonce = make([]byte, gcm.NonceSize())
@@ -214,10 +253,9 @@ func (chunk *Chunk) Encrypt(encryptionKey []byte, derivationKey string) (err err
 		}
 		encryptedBuffer.Write(nonce)
 		offset = encryptedBuffer.Len()
-
 	}
 
-	// offset is either 0 or the length of header + nonce
+	// offset is either 0 or the length of banner + nonce
 
 	if chunk.config.CompressionLevel >= -1 && chunk.config.CompressionLevel <= 9 {
 		deflater, _ := zlib.NewWriterLevel(encryptedBuffer, chunk.config.CompressionLevel)
@@ -242,26 +280,79 @@ func (chunk *Chunk) Encrypt(encryptionKey []byte, derivationKey string) (err err
 		return fmt.Errorf("Invalid compression level: %d", chunk.config.CompressionLevel)
 	}
 
-	if len(encryptionKey) == 0 {
-		chunk.buffer, encryptedBuffer = encryptedBuffer, chunk.buffer
-		return nil
+	if len(encryptionKey) > 0 {
+
+		// PKCS7 is used.  The sizes of compressed chunks leak information about the original chunks so we want the padding sizes
+		// to be the maximum allowed by PKCS7
+		dataLength := encryptedBuffer.Len() - offset
+		paddingLength := 256 - dataLength%256
+
+		encryptedBuffer.Write(bytes.Repeat([]byte{byte(paddingLength)}, paddingLength))
+		encryptedBuffer.Write(bytes.Repeat([]byte{0}, gcm.Overhead()))
+
+		// The encrypted data will be appended to the duplicacy banner and the once.
+		encryptedBytes := gcm.Seal(encryptedBuffer.Bytes()[:offset], nonce,
+			encryptedBuffer.Bytes()[offset:offset+dataLength+paddingLength], nil)
+
+		encryptedBuffer.Truncate(len(encryptedBytes))
 	}
 
-	// PKCS7 is used.  Compressed chunk sizes leaks information about the original chunks so we want the padding sizes
-	// to be the maximum allowed by PKCS7
-	dataLength := encryptedBuffer.Len() - offset
-	paddingLength := 256 - dataLength%256
+	if chunk.config.DataShards == 0 || chunk.config.ParityShards == 0 {
+		chunk.buffer, encryptedBuffer = encryptedBuffer, chunk.buffer
+		return
+	}
 
-	encryptedBuffer.Write(bytes.Repeat([]byte{byte(paddingLength)}, paddingLength))
-	encryptedBuffer.Write(bytes.Repeat([]byte{0}, gcm.Overhead()))
+	// Start erasure coding
+	encoder, err := reedsolomon.New(chunk.config.DataShards, chunk.config.ParityShards)
+	if err != nil {
+		return err
+	}
+	chunkSize := len(encryptedBuffer.Bytes())
+	shardSize := (chunkSize + chunk.config.DataShards - 1) / chunk.config.DataShards
+	// Append zeros to make the last shard to have the same size as other
+	encryptedBuffer.Write(make([]byte, shardSize * chunk.config.DataShards - chunkSize))
+	// Grow the buffer for parity shards
+	encryptedBuffer.Grow(shardSize * chunk.config.ParityShards)
+	// Now create one slice for each shard, reusing the data in the buffer
+	data := make([][]byte, chunk.config.DataShards + chunk.config.ParityShards)
+	for i := 0; i < chunk.config.DataShards + chunk.config.ParityShards; i++ {
+		data[i] = encryptedBuffer.Bytes()[i * shardSize: (i + 1) * shardSize]
+	}
+	// This populates the parity shard
+	encoder.Encode(data)
 
-	// The encrypted data will be appended to the duplicacy header and the once.
-	encryptedBytes := gcm.Seal(encryptedBuffer.Bytes()[:offset], nonce,
-		encryptedBuffer.Bytes()[offset:offset+dataLength+paddingLength], nil)
+	// Prepare the chunk to be uploaded
+	chunk.buffer.Reset()
+	// First the banner
+	chunk.buffer.Write([]byte(ERASURE_CODING_BANNER))
+	// Then the header which includes the chunk size, data/parity and a 2-byte checksum
+	header := make([]byte, 14)
+	binary.LittleEndian.PutUint64(header[0:], uint64(chunkSize))
+	binary.LittleEndian.PutUint16(header[8:], uint16(chunk.config.DataShards))
+	binary.LittleEndian.PutUint16(header[10:], uint16(chunk.config.ParityShards))
+	header[12] = header[0] ^ header[2] ^ header[4] ^ header[6] ^ header[8] ^ header[10]
+	header[13] = header[1] ^ header[3] ^ header[5] ^ header[7] ^ header[9] ^ header[11]
+	chunk.buffer.Write(header)
+	// Calculate the highway hash for each shard
+	hashKey := make([]byte, 32)
+	for _, part := range data {
+		hasher, err := highwayhash.New(hashKey)
+		if err != nil {
+			return err
+		}
+		_, err = hasher.Write(part)
+		if err != nil {
+			return err
+		}
+		chunk.buffer.Write(hasher.Sum(nil))
+	}
 
-	encryptedBuffer.Truncate(len(encryptedBytes))
-
-	chunk.buffer, encryptedBuffer = encryptedBuffer, chunk.buffer
+	// Copy the data
+	for _, part := range data {
+		chunk.buffer.Write(part)
+	}
+	// Append the header again for redundancy
+	chunk.buffer.Write(header)
 
 	return nil
 
@@ -291,6 +382,122 @@ func (chunk *Chunk) Decrypt(encryptionKey []byte, derivationKey string) (err err
 	}()
 
 	chunk.buffer, encryptedBuffer = encryptedBuffer, chunk.buffer
+	bannerLength := len(ENCRYPTION_BANNER)
+
+	if len(encryptedBuffer.Bytes()) > bannerLength && string(encryptedBuffer.Bytes()[:bannerLength]) == ERASURE_CODING_BANNER {
+
+		// The chunk was encoded with erasure coding
+		if len(encryptedBuffer.Bytes()) < bannerLength + 14 {
+			return fmt.Errorf("Erasure coding header truncated (%d bytes)", len(encryptedBuffer.Bytes()))
+		}
+		// Check the header checksum
+		header := encryptedBuffer.Bytes()[bannerLength: bannerLength + 14]
+		if header[12] != header[0] ^ header[2] ^ header[4] ^ header[6] ^ header[8] ^ header[10] ||
+		   header[13] != header[1] ^ header[3] ^ header[5] ^ header[7] ^ header[9] ^ header[11] {
+			return fmt.Errorf("Erasure coding header corrupted (%x)", header)
+		}
+
+		// Read the parameters
+		chunkSize := int(binary.LittleEndian.Uint64(header[0:8]))
+		dataShards := int(binary.LittleEndian.Uint16(header[8:10]))
+		parityShards := int(binary.LittleEndian.Uint16(header[10:12]))
+		shardSize := (chunkSize + chunk.config.DataShards - 1) / chunk.config.DataShards
+		// This is the length the chunk file should have
+		expectedLength := bannerLength + 2 * len(header) + (dataShards + parityShards) * (shardSize + 32)
+		// The minimum length that can be recovered from
+		minimumLength := bannerLength + len(header) + (dataShards + parityShards) * 32 + dataShards * shardSize
+		LOG_DEBUG("CHUNK_ERASURECODE", "Chunk size: %d bytes, data size: %d, parity: %d/%d", chunkSize, len(encryptedBuffer.Bytes()), dataShards, parityShards)
+		if len(encryptedBuffer.Bytes()) > expectedLength {
+			LOG_WARN("CHUNK_ERASURECODE", "Chunk has %d bytes (instead of %d)", len(encryptedBuffer.Bytes()), expectedLength)
+		} else if len(encryptedBuffer.Bytes()) == expectedLength {
+			// Correct size; fall through
+		} else if len(encryptedBuffer.Bytes()) > minimumLength {
+			LOG_WARN("CHUNK_ERASURECODE", "Chunk is truncated (%d out of %d bytes)", len(encryptedBuffer.Bytes()), expectedLength)
+		} else {
+			return fmt.Errorf("Not enough chunk data for recovery; chunk size: %d bytes, data size: %d, parity: %d/%d", chunkSize, len(encryptedBuffer.Bytes()), dataShards, parityShards)
+		}
+
+		// Where the hashes start
+		hashOffset := bannerLength + len(header)
+		// Where the data start
+		dataOffset := hashOffset + (dataShards + parityShards) * 32
+
+		data := make([][]byte, dataShards + parityShards)
+		recoveryNeeded := false
+		hashKey := make([]byte, 32)
+		availableShards := 0
+		for i := 0; i < dataShards + parityShards; i++ {
+			start := dataOffset + i * shardSize
+			if start + shardSize > len(encryptedBuffer.Bytes()) {
+				// the current shard is incomplete
+				break
+			}
+			// Now verify the hash
+			hasher, err := highwayhash.New(hashKey)
+			if err != nil {
+				return err
+			}
+			_, err = hasher.Write(encryptedBuffer.Bytes()[start: start + shardSize])
+			if err != nil {
+				return err
+			}
+			if bytes.Compare(hasher.Sum(nil), encryptedBuffer.Bytes()[hashOffset + i * 32: hashOffset + (i + 1) * 32]) != 0 {
+				if i < dataShards {
+					recoveryNeeded = true
+				}
+			} else {
+				// The shard is good
+				data[i] = encryptedBuffer.Bytes()[start: start + shardSize]
+				availableShards++
+				if availableShards >= dataShards {
+					// We have enough shards to recover; skip the remaining shards
+					break
+				}
+			}
+		}
+
+		if !recoveryNeeded {
+			// Remove the padding zeros from the last shard
+			encryptedBuffer.Truncate(dataOffset + chunkSize)
+			// Skip the header and hashes
+			encryptedBuffer.Read(encryptedBuffer.Bytes()[:dataOffset])
+		} else {
+			if availableShards < dataShards {
+				return fmt.Errorf("Not enough chunk data for recover; only %d out of %d shards are complete", availableShards, dataShards + parityShards)
+			}
+
+			// Show the validity of shards using a string of * and -
+			slots := ""
+			for _, part := range data {
+				if len(part) != 0 {
+					slots += "*"
+				} else {
+					slots += "-"
+				}
+			}
+
+			LOG_WARN("CHUNK_ERASURECODE", "Recovering a %d byte chunk from %d byte shards: %s", chunkSize, shardSize, slots)
+			encoder, err := reedsolomon.New(dataShards, parityShards)
+			if err != nil {
+				return err
+			}
+			err = encoder.Reconstruct(data)
+			if err != nil {
+				return err
+			}
+			LOG_DEBUG("CHUNK_ERASURECODE", "Chunk data successfully recovered")
+			buffer := AllocateChunkBuffer()
+			buffer.Reset()
+			for i := 0; i < dataShards; i++ {
+				buffer.Write(data[i])
+			}
+			buffer.Truncate(chunkSize)
+
+			ReleaseChunkBuffer(encryptedBuffer)
+			encryptedBuffer = buffer
+		}
+
+	}
 
 	if len(encryptionKey) > 0 {
 
@@ -308,6 +515,41 @@ func (chunk *Chunk) Decrypt(encryptionKey []byte, derivationKey string) (err err
 			key = hasher.Sum(nil)
 		}
 
+		if len(encryptedBuffer.Bytes()) < bannerLength + 12 {
+			return fmt.Errorf("No enough encrypted data (%d bytes) provided", len(encryptedBuffer.Bytes()))
+		}
+
+		if string(encryptedBuffer.Bytes()[:bannerLength-1]) != ENCRYPTION_BANNER[:bannerLength-1] {
+			return fmt.Errorf("The storage doesn't seem to be encrypted")
+		}
+
+		encryptionVersion := encryptedBuffer.Bytes()[bannerLength-1]
+		if encryptionVersion != 0 && encryptionVersion != ENCRYPTION_VERSION_RSA {
+			return fmt.Errorf("Unsupported encryption version %d", encryptionVersion)
+		}
+
+		if encryptionVersion == ENCRYPTION_VERSION_RSA {
+			if chunk.config.rsaPrivateKey == nil {
+				LOG_ERROR("CHUNK_DECRYPT", "An RSA private key is required to decrypt the chunk")
+				return fmt.Errorf("An RSA private key is required to decrypt the chunk")
+			}
+
+			encryptedKeyLength := binary.LittleEndian.Uint16(encryptedBuffer.Bytes()[bannerLength:bannerLength+2])
+
+			if len(encryptedBuffer.Bytes()) < bannerLength + 14 + int(encryptedKeyLength) {
+				return fmt.Errorf("No enough encrypted data (%d bytes) provided", len(encryptedBuffer.Bytes()))
+			}
+
+			encryptedKey := encryptedBuffer.Bytes()[bannerLength + 2:bannerLength + 2 + int(encryptedKeyLength)]
+			bannerLength += 2 + int(encryptedKeyLength)
+
+			decryptedKey, err := rsa.DecryptOAEP(sha256.New(), rand.Reader, chunk.config.rsaPrivateKey, encryptedKey, nil)
+			if err != nil {
+				return err
+			}
+			key = decryptedKey
+		}
+
 		aesBlock, err := aes.NewCipher(key)
 		if err != nil {
 			return err
@@ -318,22 +560,8 @@ func (chunk *Chunk) Decrypt(encryptionKey []byte, derivationKey string) (err err
 			return err
 		}
 
-		headerLength := len(ENCRYPTION_HEADER)
-		offset = headerLength + gcm.NonceSize()
-
-		if len(encryptedBuffer.Bytes()) < offset {
-			return fmt.Errorf("No enough encrypted data (%d bytes) provided", len(encryptedBuffer.Bytes()))
-		}
-
-		if string(encryptedBuffer.Bytes()[:headerLength-1]) != ENCRYPTION_HEADER[:headerLength-1] {
-			return fmt.Errorf("The storage doesn't seem to be encrypted")
-		}
-
-		if encryptedBuffer.Bytes()[headerLength-1] != 0 {
-			return fmt.Errorf("Unsupported encryption version %d", encryptedBuffer.Bytes()[headerLength-1])
-		}
-
-		nonce := encryptedBuffer.Bytes()[headerLength:offset]
+		offset = bannerLength + gcm.NonceSize()
+		nonce := encryptedBuffer.Bytes()[bannerLength:offset]
 
 		decryptedBytes, err := gcm.Open(encryptedBuffer.Bytes()[:offset], nonce,
 			encryptedBuffer.Bytes()[offset:], nil)

src/duplicacy_chunk_test.go

@@ -7,11 +7,51 @@ package duplicacy
 import (
 	"bytes"
 	crypto_rand "crypto/rand"
+	"crypto/rsa"
 	"math/rand"
 	"testing"
 )
 
-func TestChunk(t *testing.T) {
+func TestErasureCoding(t *testing.T) {
+	key := []byte("duplicacydefault")
+
+	config := CreateConfig()
+	config.HashKey = key
+	config.IDKey = key
+	config.MinimumChunkSize = 100
+	config.CompressionLevel = DEFAULT_COMPRESSION_LEVEL
+	config.DataShards = 5
+    config.ParityShards = 2
+
+	chunk := CreateChunk(config, true)
+	chunk.Reset(true)
+	data := make([]byte, 100)
+	for i := 0; i < len(data); i++ {
+		data[i] = byte(i)
+	}
+	chunk.Write(data)
+	err := chunk.Encrypt([]byte(""), "", false)
+	if err != nil {
+		t.Errorf("Failed to encrypt the test data: %v", err)
+		return
+	}
+
+	encryptedData := make([]byte, chunk.GetLength())
+	copy(encryptedData, chunk.GetBytes())
+
+	crypto_rand.Read(encryptedData[280:300])
+
+	chunk.Reset(false)
+	chunk.Write(encryptedData)
+	err = chunk.Decrypt([]byte(""), "")
+	if err != nil {
+		t.Errorf("Failed to decrypt the data: %v", err)
+		return
+	}
+	return
+}
+
+func TestChunkBasic(t *testing.T) {
 
 	key := []byte("duplicacydefault")
 
@@ -22,7 +62,19 @@ func TestChunk(t *testing.T) {
 	config.CompressionLevel = DEFAULT_COMPRESSION_LEVEL
 	maxSize := 1000000
 
-	remainderLength := -1
+	if testRSAEncryption {
+		privateKey, err := rsa.GenerateKey(crypto_rand.Reader, 2048)
+		if err != nil {
+			t.Errorf("Failed to generate a random private key: %v", err)
+		}
+		config.rsaPrivateKey = privateKey
+		config.rsaPublicKey = privateKey.Public().(*rsa.PublicKey)
+	}
+
+	if testErasureCoding {
+		config.DataShards = 5
+		config.ParityShards = 2
+	}
 
 	for i := 0; i < 500; i++ {
 
@@ -37,7 +89,7 @@ func TestChunk(t *testing.T) {
 		hash := chunk.GetHash()
 		id := chunk.GetID()
 
-		err := chunk.Encrypt(key, "")
+		err := chunk.Encrypt(key, "", false)
 		if err != nil {
 			t.Errorf("Failed to encrypt the data: %v", err)
 			continue
@@ -46,10 +98,14 @@ func TestChunk(t *testing.T) {
 		encryptedData := make([]byte, chunk.GetLength())
 		copy(encryptedData, chunk.GetBytes())
 
-		if remainderLength == -1 {
-			remainderLength = len(encryptedData) % 256
-		} else if len(encryptedData)%256 != remainderLength {
-			t.Errorf("Incorrect padding size")
+		if testErasureCoding {
+			offset := 24 + 32 * 7
+			start := rand.Int() % (len(encryptedData) - offset) + offset
+			length := (len(encryptedData) - offset) / 7
+			if start + length > len(encryptedData) {
+				length = len(encryptedData) - start
+			}
+			crypto_rand.Read(encryptedData[start: start+length])
 		}
 
 		chunk.Reset(false)

src/duplicacy_chunkdownloader.go

@@ -36,6 +36,7 @@ type ChunkDownloader struct {
 	snapshotCache  *FileStorage // Used as cache if not nil; usually for downloading snapshot chunks
 	showStatistics bool         // Show a stats log for each chunk if true
 	threads        int          // Number of threads
+	allowFailures  bool         // Whether to failfast on download error, or continue
 
 	taskList       []ChunkDownloadTask // The list of chunks to be downloaded
 	completedTasks map[int]bool        // Store downloaded chunks
@@ -51,15 +52,18 @@ type ChunkDownloader struct {
 	numberOfDownloadedChunks  int   // The number of chunks that have been downloaded
 	numberOfDownloadingChunks int   // The number of chunks still being downloaded
 	numberOfActiveChunks      int   // The number of chunks that is being downloaded or has been downloaded but not reclaimed
+
+	NumberOfFailedChunks      int   // The number of chunks that can't be downloaded
 }
 
-func CreateChunkDownloader(config *Config, storage Storage, snapshotCache *FileStorage, showStatistics bool, threads int) *ChunkDownloader {
+func CreateChunkDownloader(config *Config, storage Storage, snapshotCache *FileStorage, showStatistics bool, threads int, allowFailures bool) *ChunkDownloader {
 	downloader := &ChunkDownloader{
 		config:         config,
 		storage:        storage,
 		snapshotCache:  snapshotCache,
 		showStatistics: showStatistics,
 		threads:        threads,
+		allowFailures:  allowFailures,
 
 		taskList:       nil,
 		completedTasks: make(map[int]bool),
@@ -126,6 +130,7 @@ func (downloader *ChunkDownloader) AddFiles(snapshot *Snapshot, files []*Entry)
 
 // AddChunk adds a single chunk the download list.
 func (downloader *ChunkDownloader) AddChunk(chunkHash string) int {
+
 	task := ChunkDownloadTask{
 		chunkIndex:    len(downloader.taskList),
 		chunkHash:     chunkHash,
@@ -197,6 +202,16 @@ func (downloader *ChunkDownloader) Reclaim(chunkIndex int) {
 	downloader.lastChunkIndex = chunkIndex
 }
 
+// Return the chunk last downloaded and its hash
+func (downloader *ChunkDownloader) GetLastDownloadedChunk() (chunk *Chunk, chunkHash string) {
+	if downloader.lastChunkIndex >= len(downloader.taskList) {
+		return nil, ""
+	}
+
+	task := downloader.taskList[downloader.lastChunkIndex]
+	return task.chunk, task.chunkHash
+}
+
 // WaitForChunk waits until the specified chunk is ready
 func (downloader *ChunkDownloader) WaitForChunk(chunkIndex int) (chunk *Chunk) {
 
@@ -220,7 +235,7 @@ func (downloader *ChunkDownloader) WaitForChunk(chunkIndex int) (chunk *Chunk) {
 		}
 		task := &downloader.taskList[i]
 		if !task.needed {
-			continue
+			break
 		}
 
 		if !task.isDownloading {
@@ -239,10 +254,57 @@ func (downloader *ChunkDownloader) WaitForChunk(chunkIndex int) (chunk *Chunk) {
 		downloader.taskList[completion.chunkIndex].chunk = completion.chunk
 		downloader.numberOfDownloadedChunks++
 		downloader.numberOfDownloadingChunks--
+		if completion.chunk.isBroken {
+			downloader.NumberOfFailedChunks++
+		}
 	}
 	return downloader.taskList[chunkIndex].chunk
 }
 
+// WaitForCompletion waits until all chunks have been downloaded
+func (downloader *ChunkDownloader) WaitForCompletion() {
+
+	// Tasks in completedTasks have not been counted by numberOfActiveChunks
+	downloader.numberOfActiveChunks -= len(downloader.completedTasks)
+
+	// find the completed task with the largest index; we'll start from the next index
+	for index := range downloader.completedTasks {
+		if downloader.lastChunkIndex < index {
+			downloader.lastChunkIndex = index
+		}
+	}
+
+	// Looping until there isn't a download task in progress
+	for downloader.numberOfActiveChunks > 0 || downloader.lastChunkIndex + 1 < len(downloader.taskList) {
+
+		// Wait for a completion event first
+		if downloader.numberOfActiveChunks > 0 {
+			completion := <-downloader.completionChannel
+			downloader.config.PutChunk(completion.chunk)
+			downloader.numberOfActiveChunks--
+			downloader.numberOfDownloadedChunks++
+			downloader.numberOfDownloadingChunks--
+			if completion.chunk.isBroken {
+				downloader.NumberOfFailedChunks++
+			}
+		}
+
+		// Pass the tasks one by one to the download queue
+		if downloader.lastChunkIndex + 1 < len(downloader.taskList) {
+			task := &downloader.taskList[downloader.lastChunkIndex + 1]
+			if task.isDownloading {
+				downloader.lastChunkIndex++
+				continue
+			}
+			downloader.taskQueue <- *task
+			task.isDownloading = true
+			downloader.numberOfDownloadingChunks++
+			downloader.numberOfActiveChunks++
+			downloader.lastChunkIndex++
+		}
+	}
+}
+
 // Stop terminates all downloading goroutines
 func (downloader *ChunkDownloader) Stop() {
 	for downloader.numberOfDownloadingChunks > 0 {
@@ -251,7 +313,10 @@ func (downloader *ChunkDownloader) Stop() {
 		downloader.taskList[completion.chunkIndex].chunk = completion.chunk
 		downloader.numberOfDownloadedChunks++
 		downloader.numberOfDownloadingChunks--
-	}
+		if completion.chunk.isBroken {
+			downloader.NumberOfFailedChunks++
+		}
+}
 
 	for i := range downloader.completedTasks {
 		downloader.config.PutChunk(downloader.taskList[i].chunk)
@@ -305,13 +370,22 @@ func (downloader *ChunkDownloader) Download(threadIndex int, task ChunkDownloadT
 	// will be set up before the encryption
 	chunk.Reset(false)
 
+	// If failures are allowed, complete the task properly
+	completeFailedChunk := func(chunk *Chunk) {
+		if downloader.allowFailures {
+			chunk.isBroken = true
+			downloader.completionChannel <- ChunkDownloadCompletion{chunk: chunk, chunkIndex: task.chunkIndex}
+		}
+	}
+
 	const MaxDownloadAttempts = 3
 	for downloadAttempt := 0; ; downloadAttempt++ {
 
 		// Find the chunk by ID first.
 		chunkPath, exist, _, err := downloader.storage.FindChunk(threadIndex, chunkID, false)
 		if err != nil {
-			LOG_ERROR("DOWNLOAD_CHUNK", "Failed to find the chunk %s: %v", chunkID, err)
+			completeFailedChunk(chunk)
+			LOG_WERROR(downloader.allowFailures, "DOWNLOAD_CHUNK", "Failed to find the chunk %s: %v", chunkID, err)
 			return false
 		}
 
@@ -319,7 +393,8 @@ func (downloader *ChunkDownloader) Download(threadIndex int, task ChunkDownloadT
 			// No chunk is found.  Have to find it in the fossil pool again.
 			fossilPath, exist, _, err := downloader.storage.FindChunk(threadIndex, chunkID, true)
 			if err != nil {
-				LOG_ERROR("DOWNLOAD_CHUNK", "Failed to find the chunk %s: %v", chunkID, err)
+				completeFailedChunk(chunk)
+				LOG_WERROR(downloader.allowFailures, "DOWNLOAD_CHUNK", "Failed to find the chunk %s: %v", chunkID, err)
 				return false
 			}
 
@@ -341,11 +416,12 @@ func (downloader *ChunkDownloader) Download(threadIndex int, task ChunkDownloadT
 					continue
 				}
 
+				completeFailedChunk(chunk)
 				// A chunk is not found.  This is a serious error and hopefully it will never happen.
 				if err != nil {
-					LOG_FATAL("DOWNLOAD_CHUNK", "Chunk %s can't be found: %v", chunkID, err)
+					LOG_WERROR(downloader.allowFailures,  "DOWNLOAD_CHUNK", "Chunk %s can't be found: %v", chunkID, err)
 				} else {
-					LOG_FATAL("DOWNLOAD_CHUNK", "Chunk %s can't be found", chunkID)
+					LOG_WERROR(downloader.allowFailures, "DOWNLOAD_CHUNK", "Chunk %s can't be found", chunkID)
 				}
 				return false
 			}
@@ -354,7 +430,8 @@ func (downloader *ChunkDownloader) Download(threadIndex int, task ChunkDownloadT
 			// downloading again.
 			err = downloader.storage.MoveFile(threadIndex, fossilPath, chunkPath)
 			if err != nil {
-				LOG_FATAL("DOWNLOAD_CHUNK", "Failed to resurrect chunk %s: %v", chunkID, err)
+				completeFailedChunk(chunk)
+				LOG_WERROR(downloader.allowFailures, "DOWNLOAD_CHUNK", "Failed to resurrect chunk %s: %v", chunkID, err)
 				return false
 			}
 
@@ -371,7 +448,8 @@ func (downloader *ChunkDownloader) Download(threadIndex int, task ChunkDownloadT
 				chunk.Reset(false)
 				continue
 			} else {
-				LOG_ERROR("DOWNLOAD_CHUNK", "Failed to download the chunk %s: %v", chunkID, err)
+				completeFailedChunk(chunk)
+				LOG_WERROR(downloader.allowFailures, "DOWNLOAD_CHUNK", "Failed to download the chunk %s: %v", chunkID, err)
 				return false
 			}
 		}
@@ -383,7 +461,8 @@ func (downloader *ChunkDownloader) Download(threadIndex int, task ChunkDownloadT
 				chunk.Reset(false)
 				continue
 			} else {
-				LOG_ERROR("DOWNLOAD_DECRYPT", "Failed to decrypt the chunk %s: %v", chunkID, err)
+				completeFailedChunk(chunk)
+				LOG_WERROR(downloader.allowFailures, "DOWNLOAD_DECRYPT", "Failed to decrypt the chunk %s: %v", chunkID, err)
 				return false
 			}
 		}
@@ -395,7 +474,8 @@ func (downloader *ChunkDownloader) Download(threadIndex int, task ChunkDownloadT
 				chunk.Reset(false)
 				continue
 			} else {
-				LOG_FATAL("DOWNLOAD_CORRUPTED", "The chunk %s has a hash id of %s", chunkID, actualChunkID)
+				completeFailedChunk(chunk)
+				LOG_WERROR(downloader.allowFailures, "DOWNLOAD_CORRUPTED", "The chunk %s has a hash id of %s", chunkID, actualChunkID)
 				return false
 			}
 		}

src/duplicacy_chunkuploader.go

@@ -128,7 +128,7 @@ func (uploader *ChunkUploader) Upload(threadIndex int, task ChunkUploadTask) boo
 	}
 
 	// Encrypt the chunk only after we know that it must be uploaded.
-	err = chunk.Encrypt(uploader.config.ChunkKey, chunk.GetHash())
+	err = chunk.Encrypt(uploader.config.ChunkKey, chunk.GetHash(), uploader.snapshotCache != nil)
 	if err != nil {
 		LOG_ERROR("UPLOAD_CHUNK", "Failed to encrypt the chunk %s: %v", chunkID, err)
 		return false

src/duplicacy_chunkuploader_test.go

@@ -101,7 +101,7 @@ func TestUploaderAndDownloader(t *testing.T) {
 
 	chunkUploader.Stop()
 
-	chunkDownloader := CreateChunkDownloader(config, storage, nil, true, testThreads)
+	chunkDownloader := CreateChunkDownloader(config, storage, nil, true, testThreads, false)
 	chunkDownloader.totalChunkSize = int64(totalFileSize)
 
 	for _, chunk := range chunks {

src/duplicacy_config.go

@@ -9,15 +9,21 @@ import (
 	"crypto/hmac"
 	"crypto/rand"
 	"crypto/sha256"
+	"crypto/rsa"
+	"crypto/x509"
 	"encoding/binary"
 	"encoding/hex"
 	"encoding/json"
+	"encoding/pem"
 	"fmt"
 	"hash"
 	"os"
+	"strings"
 	"runtime"
 	"runtime/debug"
 	"sync/atomic"
+	"io/ioutil"
+	"reflect"
 
 	blake2 "github.com/minio/blake2b-simd"
 )
@@ -29,8 +35,8 @@ var DEFAULT_KEY = []byte("duplicacy")
 // standard zlib levels of -1 to 9.
 var DEFAULT_COMPRESSION_LEVEL = 100
 
-// The new header of the config file (to differentiate from the old format where the salt and iterations are fixed)
-var CONFIG_HEADER = "duplicacy\001"
+// The new banner of the config file (to differentiate from the old format where the salt and iterations are fixed)
+var CONFIG_BANNER = "duplicacy\001"
 
 // The length of the salt used in the new format
 var CONFIG_SALT_LENGTH = 32
@@ -65,6 +71,14 @@ type Config struct {
 	// for encrypting a non-chunk file
 	FileKey []byte `json:"-"`
 
+	// for erasure coding
+	DataShards int `json:'data-shards'`
+	ParityShards int `json:'parity-shards'`
+
+	// for RSA encryption
+	rsaPrivateKey *rsa.PrivateKey
+	rsaPublicKey *rsa.PublicKey
+
 	chunkPool      chan *Chunk
 	numberOfChunks int32
 	dryRun         bool
@@ -80,10 +94,15 @@ type jsonableConfig struct {
 	IDKey     string `json:"id-key"`
 	ChunkKey  string `json:"chunk-key"`
 	FileKey   string `json:"file-key"`
+	RSAPublicKey string `json:"rsa-public-key"`
 }
 
 func (config *Config) MarshalJSON() ([]byte, error) {
 
+	publicKey := []byte {}
+	if config.rsaPublicKey != nil {
+		publicKey, _ = x509.MarshalPKIXPublicKey(config.rsaPublicKey)
+	}
 	return json.Marshal(&jsonableConfig{
 		aliasedConfig: (*aliasedConfig)(config),
 		ChunkSeed:     hex.EncodeToString(config.ChunkSeed),
@@ -91,6 +110,7 @@ func (config *Config) MarshalJSON() ([]byte, error) {
 		IDKey:         hex.EncodeToString(config.IDKey),
 		ChunkKey:      hex.EncodeToString(config.ChunkKey),
 		FileKey:       hex.EncodeToString(config.FileKey),
+		RSAPublicKey:  hex.EncodeToString(publicKey),
 	})
 }
 
@@ -120,6 +140,19 @@ func (config *Config) UnmarshalJSON(description []byte) (err error) {
 		return fmt.Errorf("Invalid representation of the file key in the config")
 	}
 
+	if publicKey, err := hex.DecodeString(aliased.RSAPublicKey); err != nil {
+		return fmt.Errorf("Invalid hex encoding of the RSA public key in the config")
+	} else if len(publicKey) > 0 {
+		parsedKey, err := x509.ParsePKIXPublicKey(publicKey)
+		if err != nil {
+			return fmt.Errorf("Invalid RSA public key in the config: %v", err)
+		}
+		config.rsaPublicKey = parsedKey.(*rsa.PublicKey)
+		if config.rsaPublicKey == nil {
+			return fmt.Errorf("Unsupported public key type %s in the config", reflect.TypeOf(parsedKey))
+		}
+	}
+
 	return nil
 }
 
@@ -140,6 +173,33 @@ func (config *Config) Print() {
 	LOG_INFO("CONFIG_INFO", "Maximum chunk size: %d", config.MaximumChunkSize)
 	LOG_INFO("CONFIG_INFO", "Minimum chunk size: %d", config.MinimumChunkSize)
 	LOG_INFO("CONFIG_INFO", "Chunk seed: %x", config.ChunkSeed)
+
+	LOG_TRACE("CONFIG_INFO", "Hash key: %x", config.HashKey)
+	LOG_TRACE("CONFIG_INFO", "ID key: %x", config.IDKey)
+
+	if len(config.ChunkKey) > 0 {
+		LOG_TRACE("CONFIG_INFO", "File chunks are encrypted")
+	}
+
+	if len(config.FileKey) > 0 {
+		LOG_TRACE("CONFIG_INFO", "Metadata chunks are encrypted")
+	}
+
+	if config.DataShards != 0 && config.ParityShards != 0 {
+		LOG_TRACE("CONFIG_INFO", "Data shards: %d, parity shards: %d", config.DataShards, config.ParityShards)
+	}
+
+	if config.rsaPublicKey != nil {
+		pkisPublicKey, _ := x509.MarshalPKIXPublicKey(config.rsaPublicKey)
+
+		publicKey := pem.EncodeToMemory(&pem.Block{
+			Type:  "PUBLIC KEY",
+			Bytes: pkisPublicKey,
+		})
+
+		LOG_TRACE("CONFIG_INFO", "RSA public key: %s", publicKey)
+	}
+
 }
 
 func CreateConfigFromParameters(compressionLevel int, averageChunkSize int, maximumChunkSize int, mininumChunkSize int,
@@ -335,11 +395,11 @@ func DownloadConfig(storage Storage, password string) (config *Config, isEncrypt
 		return nil, false, err
 	}
 
-	if len(configFile.GetBytes()) < len(ENCRYPTION_HEADER) {
+	if len(configFile.GetBytes()) < len(ENCRYPTION_BANNER) {
 		return nil, false, fmt.Errorf("The storage has an invalid config file")
 	}
 
-	if string(configFile.GetBytes()[:len(ENCRYPTION_HEADER)-1]) == ENCRYPTION_HEADER[:len(ENCRYPTION_HEADER)-1] && len(password) == 0 {
+	if string(configFile.GetBytes()[:len(ENCRYPTION_BANNER)-1]) == ENCRYPTION_BANNER[:len(ENCRYPTION_BANNER)-1] && len(password) == 0 {
 		return nil, true, fmt.Errorf("The storage is likely to have been initialized with a password before")
 	}
 
@@ -347,23 +407,23 @@ func DownloadConfig(storage Storage, password string) (config *Config, isEncrypt
 
 	if len(password) > 0 {
 
-		if string(configFile.GetBytes()[:len(ENCRYPTION_HEADER)]) == ENCRYPTION_HEADER {
+		if string(configFile.GetBytes()[:len(ENCRYPTION_BANNER)]) == ENCRYPTION_BANNER {
 			// This is the old config format with a static salt and a fixed number of iterations
 			masterKey = GenerateKeyFromPassword(password, DEFAULT_KEY, CONFIG_DEFAULT_ITERATIONS)
 			LOG_TRACE("CONFIG_FORMAT", "Using a static salt and %d iterations for key derivation", CONFIG_DEFAULT_ITERATIONS)
-		} else if string(configFile.GetBytes()[:len(CONFIG_HEADER)]) == CONFIG_HEADER {
+		} else if string(configFile.GetBytes()[:len(CONFIG_BANNER)]) == CONFIG_BANNER {
 			// This is the new config format with a random salt and a configurable number of iterations
 			encryptedLength := len(configFile.GetBytes()) - CONFIG_SALT_LENGTH - 4
 
 			// Extract the salt and the number of iterations
-			saltStart := configFile.GetBytes()[len(CONFIG_HEADER):]
+			saltStart := configFile.GetBytes()[len(CONFIG_BANNER):]
 			iterations := binary.LittleEndian.Uint32(saltStart[CONFIG_SALT_LENGTH : CONFIG_SALT_LENGTH+4])
 			LOG_TRACE("CONFIG_ITERATIONS", "Using %d iterations for key derivation", iterations)
 			masterKey = GenerateKeyFromPassword(password, saltStart[:CONFIG_SALT_LENGTH], int(iterations))
 
-			// Copy to a temporary buffer to replace the header and remove the salt and the number of riterations
+			// Copy to a temporary buffer to replace the banner and remove the salt and the number of riterations
 			var encrypted bytes.Buffer
-			encrypted.Write([]byte(ENCRYPTION_HEADER))
+			encrypted.Write([]byte(ENCRYPTION_BANNER))
 			encrypted.Write(saltStart[CONFIG_SALT_LENGTH+4:])
 
 			configFile.Reset(false)
@@ -372,7 +432,7 @@ func DownloadConfig(storage Storage, password string) (config *Config, isEncrypt
 				LOG_ERROR("CONFIG_DOWNLOAD", "Encrypted config has %d bytes instead of expected %d bytes", len(configFile.GetBytes()), encryptedLength)
 			}
 		} else {
-			return nil, true, fmt.Errorf("The config file has an invalid header")
+			return nil, true, fmt.Errorf("The config file has an invalid banner")
 		}
 
 		// Decrypt the config file.  masterKey == nil means no encryption.
@@ -430,21 +490,21 @@ func UploadConfig(storage Storage, config *Config, password string, iterations i
 
 	if len(password) > 0 {
 		// Encrypt the config file with masterKey.  If masterKey is nil then no encryption is performed.
-		err = chunk.Encrypt(masterKey, "")
+		err = chunk.Encrypt(masterKey, "", true)
 		if err != nil {
 			LOG_ERROR("CONFIG_CREATE", "Failed to create the config file: %v", err)
 			return false
 		}
 
-		// The new encrypted format for config is CONFIG_HEADER + salt + #iterations + encrypted content
+		// The new encrypted format for config is CONFIG_BANNER + salt + #iterations + encrypted content
 		encryptedLength := len(chunk.GetBytes()) + CONFIG_SALT_LENGTH + 4
 
-		// Copy to a temporary buffer to replace the header and add the salt and the number of iterations
+		// Copy to a temporary buffer to replace the banner and add the salt and the number of iterations
 		var encrypted bytes.Buffer
-		encrypted.Write([]byte(CONFIG_HEADER))
+		encrypted.Write([]byte(CONFIG_BANNER))
 		encrypted.Write(salt)
 		binary.Write(&encrypted, binary.LittleEndian, uint32(iterations))
-		encrypted.Write(chunk.GetBytes()[len(ENCRYPTION_HEADER):])
+		encrypted.Write(chunk.GetBytes()[len(ENCRYPTION_BANNER):])
 
 		chunk.Reset(false)
 		chunk.Write(encrypted.Bytes())
@@ -477,7 +537,7 @@ func UploadConfig(storage Storage, config *Config, password string, iterations i
 // it simply creates a file named 'config' that stores various parameters as well as a set of keys if encryption
 // is enabled.
 func ConfigStorage(storage Storage, iterations int, compressionLevel int, averageChunkSize int, maximumChunkSize int,
-	minimumChunkSize int, password string, copyFrom *Config, bitCopy bool) bool {
+	minimumChunkSize int, password string, copyFrom *Config, bitCopy bool, keyFile string, dataShards int, parityShards int) bool {
 
 	exist, _, _, err := storage.GetFileInfo(0, "config")
 	if err != nil {
@@ -496,5 +556,129 @@ func ConfigStorage(storage Storage, iterations int, compressionLevel int, averag
 		return false
 	}
 
+	if keyFile != "" {
+		config.loadRSAPublicKey(keyFile)
+	}
+
+	config.DataShards = dataShards
+	config.ParityShards = parityShards
+
 	return UploadConfig(storage, config, password, iterations)
 }
+
+func (config *Config) loadRSAPublicKey(keyFile string) {
+	encodedKey := []byte(keyFile)
+	var err error
+
+	// keyFile may be the actually key, in which case we don't need to read from a file
+	if !strings.Contains(keyFile, "-----BEGIN") {
+		encodedKey, err = ioutil.ReadFile(keyFile)
+		if err != nil {
+			LOG_ERROR("BACKUP_KEY", "Failed to read the public key file: %v", err)
+			return
+		}
+	}
+
+	decodedKey, _ := pem.Decode(encodedKey)
+	if decodedKey == nil {
+		LOG_ERROR("RSA_PUBLIC", "unrecognized public key in %s", keyFile)
+		return
+	}
+	if decodedKey.Type != "PUBLIC KEY" {
+		LOG_ERROR("RSA_PUBLIC", "Unsupported public key type %s in %s", decodedKey.Type, keyFile)
+		return
+	}
+
+	parsedKey, err := x509.ParsePKIXPublicKey(decodedKey.Bytes)
+	if err != nil {
+		LOG_ERROR("RSA_PUBLIC", "Failed to parse the public key in %s: %v", keyFile, err)
+		return
+	}
+
+	key, ok := parsedKey.(*rsa.PublicKey)
+	if !ok {
+		LOG_ERROR("RSA_PUBLIC", "Unsupported public key type %s in %s", reflect.TypeOf(parsedKey), keyFile)
+		return
+	}
+
+	config.rsaPublicKey = key
+}
+
+// loadRSAPrivateKey loads the specifed private key file for decrypting file chunks
+func (config *Config) loadRSAPrivateKey(keyFile string, passphrase string) {
+
+	if config.rsaPublicKey == nil {
+		LOG_ERROR("RSA_PUBLIC", "The storage was not encrypted by an RSA key")
+		return
+	}
+
+	encodedKey := []byte(keyFile)
+	var err error
+
+	// keyFile may be the actually key, in which case we don't need to read from a file
+	if !strings.Contains(keyFile, "-----BEGIN") {
+		encodedKey, err = ioutil.ReadFile(keyFile)
+		if err != nil {
+			LOG_ERROR("RSA_PRIVATE", "Failed to read the private key file: %v", err)
+			return
+		}
+	}
+
+	decodedKey, _ := pem.Decode(encodedKey)
+	if decodedKey == nil {
+		LOG_ERROR("RSA_PRIVATE", "unrecognized private key in %s", keyFile)
+		return
+	}
+	if decodedKey.Type != "RSA PRIVATE KEY" {
+		LOG_ERROR("RSA_PRIVATE", "Unsupported private key type %s in %s", decodedKey.Type, keyFile)
+		return
+	}
+
+	var decodedKeyBytes []byte
+	if passphrase != "" {
+		decodedKeyBytes, err = x509.DecryptPEMBlock(decodedKey, []byte(passphrase))
+	} else {
+		decodedKeyBytes = decodedKey.Bytes
+	}
+
+	var parsedKey interface{}
+	if parsedKey, err = x509.ParsePKCS1PrivateKey(decodedKeyBytes); err != nil {
+		if parsedKey, err = x509.ParsePKCS8PrivateKey(decodedKeyBytes); err != nil {
+			LOG_ERROR("RSA_PRIVATE", "Failed to parse the private key in %s: %v", keyFile, err)
+			return
+		}
+	}
+
+	key, ok := parsedKey.(*rsa.PrivateKey)
+	if !ok {
+		LOG_ERROR("RSA_PRIVATE", "Unsupported private key type %s in %s", reflect.TypeOf(parsedKey), keyFile)
+		return
+	}
+
+	data := make([]byte, 32)
+	_, err = rand.Read(data)
+	if err != nil {
+		LOG_ERROR("RSA_PRIVATE", "Failed to generate random data for testing the private key: %v", err)
+		return
+	}
+
+	// Now test if the private key matches the public key
+	encryptedData, err := rsa.EncryptOAEP(sha256.New(), rand.Reader, config.rsaPublicKey, data, nil)
+	if err != nil {
+		LOG_ERROR("RSA_PRIVATE", "Failed to encrypt random data with the public key: %v", err)
+		return
+	}
+
+	decryptedData, err := rsa.DecryptOAEP(sha256.New(), rand.Reader, key, encryptedData, nil)
+	if err != nil {
+		LOG_ERROR("RSA_PRIVATE", "Incorrect private key: %v", err)
+		return
+	}
+
+	if !bytes.Equal(data, decryptedData) {
+		LOG_ERROR("RSA_PRIVATE", "Decrypted data do not match the original data")
+		return
+	}
+
+	config.rsaPrivateKey = key
+}

src/duplicacy_dropboxstorage.go

@@ -6,6 +6,7 @@ package duplicacy
 
 import (
 	"fmt"
+	"io/ioutil"
 	"strings"
 
 	"github.com/gilbertchen/go-dropbox"
@@ -199,6 +200,7 @@ func (storage *DropboxStorage) DownloadFile(threadIndex int, filePath string, ch
 	}
 
 	defer output.Body.Close()
+	defer ioutil.ReadAll(output.Body)
 
 	_, err = RateLimitedCopy(chunk, output.Body, storage.DownloadRateLimit/len(storage.clients))
 	return err

src/duplicacy_entry.go

@@ -443,7 +443,7 @@ func (files FileInfoCompare) Less(i, j int) bool {
 
 // ListEntries returns a list of entries representing file and subdirectories under the directory 'path'.  Entry paths
 // are normalized as relative to 'top'.  'patterns' are used to exclude or include certain files.
-func ListEntries(top string, path string, fileList *[]*Entry, patterns []string, nobackupFile string, discardAttributes bool) (directoryList []*Entry,
+func ListEntries(top string, path string, fileList *[]*Entry, patterns []string, nobackupFile string, discardAttributes bool, excludeByAttribute bool) (directoryList []*Entry,
 	skippedFiles []string, err error) {
 
 	LOG_DEBUG("LIST_ENTRIES", "Listing %s", path)
@@ -490,7 +490,7 @@ func ListEntries(top string, path string, fileList *[]*Entry, patterns []string,
 		}
 		if entry.IsLink() {
 			isRegular := false
-			isRegular, entry.Link, err = Readlink(filepath.Join(top, entry.Path))
+			isRegular, entry.Link, err = Readlink(joinPath(top, entry.Path))
 			if err != nil {
 				LOG_WARN("LIST_LINK", "Failed to read the symlink %s: %v", entry.Path, err)
 				skippedFiles = append(skippedFiles, entry.Path)
@@ -500,7 +500,7 @@ func ListEntries(top string, path string, fileList *[]*Entry, patterns []string,
 			if isRegular {
 				entry.Mode ^= uint32(os.ModeSymlink)
 			} else if path == "" && (filepath.IsAbs(entry.Link) || filepath.HasPrefix(entry.Link, `\\`)) && !strings.HasPrefix(entry.Link, normalizedTop) {
-				stat, err := os.Stat(filepath.Join(top, entry.Path))
+				stat, err := os.Stat(joinPath(top, entry.Path))
 				if err != nil {
 					LOG_WARN("LIST_LINK", "Failed to read the symlink: %v", err)
 					skippedFiles = append(skippedFiles, entry.Path)
@@ -513,6 +513,9 @@ func ListEntries(top string, path string, fileList *[]*Entry, patterns []string,
 					// path from f.Name(); note that a "/" is append assuming a symbolic link is always a directory
 					newEntry.Path = filepath.Join(normalizedPath, f.Name()) + "/"
 				}
+				if len(patterns) > 0 && !MatchPath(newEntry.Path, patterns) {
+					continue
+				}
 				entry = newEntry
 			}
 		}
@@ -521,6 +524,11 @@ func ListEntries(top string, path string, fileList *[]*Entry, patterns []string,
 			entry.ReadAttributes(top)
 		}
 
+		if excludeByAttribute && excludedByAttribute(entry.Attributes) {
+			LOG_DEBUG("LIST_EXCLUDE", "%s is excluded by attribute", entry.Path)
+			continue
+		}
+
 		if f.Mode()&(os.ModeNamedPipe|os.ModeSocket|os.ModeDevice) != 0 {
 			LOG_WARN("LIST_SKIP", "Skipped non-regular file %s", entry.Path)
 			skippedFiles = append(skippedFiles, entry.Path)

src/duplicacy_entry_test.go

@@ -9,8 +9,12 @@ import (
 	"math/rand"
 	"os"
 	"path/filepath"
+	"runtime"
 	"sort"
+	"strings"
 	"testing"
+
+	"github.com/gilbertchen/xattr"
 )
 
 func TestEntrySort(t *testing.T) {
@@ -173,7 +177,7 @@ func TestEntryList(t *testing.T) {
 		directory := directories[len(directories)-1]
 		directories = directories[:len(directories)-1]
 		entries = append(entries, directory)
-		subdirectories, _, err := ListEntries(testDir, directory.Path, &entries, nil, "", false)
+		subdirectories, _, err := ListEntries(testDir, directory.Path, &entries, nil, "", false, false)
 		if err != nil {
 			t.Errorf("ListEntries(%s, %s) returned an error: %s", testDir, directory.Path, err)
 		}
@@ -216,3 +220,110 @@ func TestEntryList(t *testing.T) {
 	}
 
 }
+
+// TestEntryExcludeByAttribute tests the excludeByAttribute parameter to the ListEntries function
+func TestEntryExcludeByAttribute(t *testing.T) {
+
+	if !(runtime.GOOS == "darwin" || runtime.GOOS == "linux") {
+		t.Skip("skipping test not darwin or linux")
+	}
+
+	testDir := filepath.Join(os.TempDir(), "duplicacy_test")
+
+	os.RemoveAll(testDir)
+	os.MkdirAll(testDir, 0700)
+
+	// Files or folders named with "exclude" below will have the exclusion attribute set on them
+	// When ListEntries is called with excludeByAttribute true, they should be excluded.
+	DATA := [...]string{
+		"excludefile",
+		"includefile",
+		"excludedir/",
+		"excludedir/file",
+		"includedir/",
+		"includedir/includefile",
+		"includedir/excludefile",
+	}
+
+	for _, file := range DATA {
+		fullPath := filepath.Join(testDir, file)
+		if file[len(file)-1] == '/' {
+			err := os.Mkdir(fullPath, 0700)
+			if err != nil {
+				t.Errorf("Mkdir(%s) returned an error: %s", fullPath, err)
+			}
+			continue
+		}
+
+		err := ioutil.WriteFile(fullPath, []byte(file), 0700)
+		if err != nil {
+			t.Errorf("WriteFile(%s) returned an error: %s", fullPath, err)
+		}
+	}
+
+	for _, file := range DATA {
+		fullPath := filepath.Join(testDir, file)
+		if strings.Contains(file, "exclude") {
+			xattr.Setxattr(fullPath, "com.apple.metadata:com_apple_backup_excludeItem", []byte("com.apple.backupd"))
+		}
+	}
+
+	for _, excludeByAttribute := range [2]bool{true, false} {
+		t.Logf("testing excludeByAttribute: %t", excludeByAttribute)
+		directories := make([]*Entry, 0, 4)
+		directories = append(directories, CreateEntry("", 0, 0, 0))
+
+		entries := make([]*Entry, 0, 4)
+
+		for len(directories) > 0 {
+			directory := directories[len(directories)-1]
+			directories = directories[:len(directories)-1]
+			entries = append(entries, directory)
+			subdirectories, _, err := ListEntries(testDir, directory.Path, &entries, nil, "", false, excludeByAttribute)
+			if err != nil {
+				t.Errorf("ListEntries(%s, %s) returned an error: %s", testDir, directory.Path, err)
+			}
+			directories = append(directories, subdirectories...)
+		}
+
+		entries = entries[1:]
+
+		for _, entry := range entries {
+			t.Logf("entry: %s", entry.Path)
+		}
+
+		i := 0
+		for _, file := range DATA {
+			entryFound := false
+			var entry *Entry
+			for _, entry = range entries {
+				if entry.Path == file {
+					entryFound = true
+					break
+				}
+			}
+
+			if excludeByAttribute && strings.Contains(file, "exclude") {
+				if entryFound {
+					t.Errorf("file: %s, expected to be excluded but wasn't. attributes: %v", file, entry.Attributes)
+					i++
+				} else {
+					t.Logf("file: %s, excluded", file)
+				}
+			} else {
+				if entryFound {
+					t.Logf("file: %s, included. attributes: %v", file, entry.Attributes)
+					i++
+				} else {
+					t.Errorf("file: %s, expected to be included but wasn't", file)
+				}
+			}
+		}
+
+	}
+
+	if !t.Failed() {
+		os.RemoveAll(testDir)
+	}
+
+}

src/duplicacy_filefabricstorage.go

@@ -0,0 +1,618 @@
+// Copyright (c) Storage Made Easy. All rights reserved.
+// 
+// This storage backend is contributed by Storage Made Easy (https://storagemadeeasy.com/) to be used in
+// Duplicacy and its derivative works.
+//
+
+package duplicacy
+
+import (
+	"io"
+	"fmt"
+	"time"
+	"sync"
+	"bytes"
+	"errors"
+	"strings"
+	"net/url"
+	"net/http"
+	"math/rand"
+	"io/ioutil"
+	"encoding/xml"
+	"path/filepath"
+	"mime/multipart"
+)
+
+// The XML element representing a file returned by the File Fabric server
+type FileFabricFile struct {
+	XMLName     xml.Name
+	ID          string     `xml:"fi_id"`
+	Path        string     `xml:"path"`
+	Size        int64      `xml:"fi_size"`
+	Type        int        `xml:"fi_type"`
+}
+
+// The XML element representing a file list returned by the server
+type FileFabricFileList struct {
+	XMLName xml.Name `xml:"files"`
+	Files []FileFabricFile `xml:",any"`
+}
+
+type FileFabricStorage struct {
+	StorageBase
+
+	endpoint           string            // the server
+	authToken          string            // the authentication token
+	accessToken        string            // the access token (as returned by getTokenByAuthToken)
+	storageDir         string            // the path of the storage directory
+	storageDirID       string            // the id of 'storageDir'
+
+	client             *http.Client      // the default http client
+	threads            int               // number of threads
+	maxRetries         int               // maximum number of tries
+	directoryCache     map[string]string // stores ids for directories known to this backend
+	directoryCacheLock sync.Mutex        // lock for accessing directoryCache
+
+	isAuthorized       bool
+	testMode           bool
+}
+
+var (
+	errFileFabricAuthorizationFailure = errors.New("Authentication failure")
+	errFileFabricDirectoryExists = errors.New("Directory exists")
+)
+
+// The general server response
+type FileFabricResponse struct {
+	Status string `xml:"status"`
+	Message string `xml:"statusmessage"`
+}
+
+// Check the server response and return an error representing the error message it contains
+func checkFileFabricResponse(response FileFabricResponse, actionFormat string, actionArguments ...interface{}) error {
+
+	action := fmt.Sprintf(actionFormat, actionArguments...)
+	if response.Status == "ok" && response.Message == "Success" {
+		return nil
+	} else if response.Status == "error_data" {
+		if response.Message == "Folder with same name already exists." {
+			return errFileFabricDirectoryExists
+		}
+	}
+
+	return fmt.Errorf("Failed to %s (status: %s, message: %s)", action, response.Status, response.Message)
+}
+
+// Create a File Fabric storage backend
+func CreateFileFabricStorage(endpoint string, token string, storageDir string, threads int) (storage *FileFabricStorage, err error) {
+
+	if len(storageDir) > 0 && storageDir[len(storageDir)-1] != '/' {
+		storageDir += "/"
+	}
+
+	storage = &FileFabricStorage{
+
+		endpoint:       endpoint,
+		authToken:      token,
+		client:         http.DefaultClient,
+		threads:        threads,
+		directoryCache: make(map[string]string),
+		maxRetries:     12,
+	}
+
+	err = storage.getAccessToken()
+	if err != nil {
+		return nil, err
+	}
+
+	storageDirID, isDir, _, err := storage.getFileInfo(0, storageDir)
+	if err != nil {
+		return nil, err
+	}
+	if storageDirID == "" {
+		return nil, fmt.Errorf("Storage path %s does not exist", storageDir)
+	}
+	if !isDir {
+		return nil, fmt.Errorf("Storage path %s is not a directory", storageDir)
+	}
+	storage.storageDir = storageDir
+	storage.storageDirID = storageDirID
+
+	for _, dir := range []string{"snapshots", "chunks"} {
+		storage.CreateDirectory(0, dir)
+	}
+
+	storage.DerivedStorage = storage
+	storage.SetDefaultNestingLevels([]int{0}, 0)
+	return storage, nil
+}
+
+// Retrieve the access token using an auth token
+func (storage *FileFabricStorage) getAccessToken() (error) {
+
+    formData := url.Values { "authtoken": {storage.authToken},}
+	readCloser, _, _, err := storage.sendRequest(0, http.MethodPost, storage.getAPIURL("getTokenByAuthToken"), nil, formData)
+	if err != nil {
+		return err
+	}
+
+	defer readCloser.Close()
+	defer io.Copy(ioutil.Discard, readCloser)	
+
+    var output struct {
+		FileFabricResponse
+        Token string `xml:"token"`
+    }
+
+    err = xml.NewDecoder(readCloser).Decode(&output)
+    if err != nil {
+        return err
+    }
+
+	err = checkFileFabricResponse(output.FileFabricResponse, "request the access token")
+	if err != nil {
+		return err
+	}
+
+	storage.accessToken = output.Token
+    return nil
+}
+
+// Determine if we should retry based on the number of retries given by 'retry' and if so calculate the delay with exponential backoff
+func (storage *FileFabricStorage) shouldRetry(retry int, messageFormat string, messageArguments ...interface{}) bool {
+	message := fmt.Sprintf(messageFormat, messageArguments...)
+
+	if retry >= storage.maxRetries {
+		LOG_WARN("FILEFABRIC_REQUEST", "%s", message)
+		return false
+	}
+	backoff := 1 << uint(retry)
+	if backoff > 60 {
+		backoff = 60
+	}
+	delay := rand.Intn(backoff*500) + backoff*500
+	LOG_INFO("FILEFABRIC_RETRY", "%s; retrying after %.1f seconds", message, float32(delay) / 1000.0)
+	time.Sleep(time.Duration(delay) * time.Millisecond)
+	return true
+}
+
+// Send a request to the server
+func (storage *FileFabricStorage) sendRequest(threadIndex int, method string, requestURL string, requestHeaders map[string]string, input interface{}) ( io.ReadCloser, http.Header, int64, error) {
+
+	var response *http.Response
+
+	for retries := 0; ; retries++ {
+		var inputReader io.Reader
+
+		switch input.(type) {
+		case url.Values:
+			values := input.(url.Values)
+			inputReader = strings.NewReader(values.Encode())
+			if requestHeaders == nil {
+				requestHeaders = make(map[string]string)
+			}
+			requestHeaders["Content-Type"] = "application/x-www-form-urlencoded"
+		case *RateLimitedReader:
+			rateLimitedReader := input.(*RateLimitedReader)
+			rateLimitedReader.Reset()
+			inputReader = rateLimitedReader
+		default:
+			LOG_FATAL("FILEFABRIC_REQUEST", "Input type is not supported")
+			return nil, nil, 0, fmt.Errorf("Input type is not supported")
+		}
+
+		request, err := http.NewRequest(method, requestURL, inputReader)
+		if err != nil {
+			return nil, nil, 0, err
+		}
+
+		if requestHeaders != nil {
+			for key, value := range requestHeaders {
+				request.Header.Set(key, value)
+			}
+		}
+
+		if _, ok := input.(*RateLimitedReader); ok {
+			request.ContentLength = input.(*RateLimitedReader).Length()
+		}
+
+		response, err = storage.client.Do(request)
+		if err != nil {
+			if !storage.shouldRetry(retries, "[%d] %s %s returned an error: %v", threadIndex, method, requestURL, err) {
+				return nil, nil, 0, err
+			}
+			continue
+		}
+
+		if response.StatusCode < 300 {
+			return response.Body, response.Header, response.ContentLength, nil
+		}
+
+		defer response.Body.Close()
+		defer io.Copy(ioutil.Discard, response.Body)	
+
+		var output struct {
+			Status string `xml:"status"`
+			Message string `xml:"statusmessage"`
+		}
+	
+		err = xml.NewDecoder(response.Body).Decode(&output)
+		if err != nil {
+			if !storage.shouldRetry(retries, "[%d] %s %s returned an invalid response: %v", threadIndex, method, requestURL, err) {
+				return nil, nil, 0, err
+			}
+			continue
+		}
+
+		if !storage.shouldRetry(retries, "[%d] %s %s returned status: %s, message: %s", threadIndex, method, requestURL, output.Status, output.Message) {
+			return nil, nil, 0, err
+		}
+	}
+
+}
+
+func (storage *FileFabricStorage) getAPIURL(function string) string {
+	if storage.accessToken == "" {
+		return "https://" + storage.endpoint + "/api/*/" + function + "/"
+	} else {
+		return "https://" + storage.endpoint + "/api/" + storage.accessToken + "/" + function + "/"
+	}
+}
+
+// ListFiles return the list of files and subdirectories under 'dir'.  A subdirectories returned must have a trailing '/', with
+// a size of 0.  If 'dir' is 'snapshots', only subdirectories will be returned.  If 'dir' is 'snapshots/repository_id', then only
+// files will be returned.  If 'dir' is 'chunks', the implementation can return the list either recusively or non-recusively.
+func (storage *FileFabricStorage) ListFiles(threadIndex int, dir string) (files []string, sizes []int64, err error) {
+	if dir != "" && dir[len(dir)-1] != '/' {
+		dir += "/"
+	}
+
+	dirID, _, _, err := storage.getFileInfo(threadIndex, dir)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	if dirID == "" {
+		return nil, nil, nil
+	}
+
+	lastID := ""
+
+	for {
+		formData := url.Values { "marker": {lastID}, "limit": {"1000"}, "includefolders": {"n"}, "fi_pid" : {dirID}}
+		if dir == "snapshots/" {
+			formData["includefolders"] = []string{"y"}
+		}
+		if storage.testMode {
+			formData["limit"] = []string{"5"}
+		}
+
+		readCloser, _, _, err := storage.sendRequest(threadIndex, http.MethodPost, storage.getAPIURL("getListOfFiles"), nil, formData)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		defer readCloser.Close()
+		defer io.Copy(ioutil.Discard, readCloser)	
+
+		var output struct {
+			FileFabricResponse
+			FileList FileFabricFileList `xml:"files"`
+			Truncated int `xml:"truncated"`
+		}
+
+		err = xml.NewDecoder(readCloser).Decode(&output)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		err = checkFileFabricResponse(output.FileFabricResponse, "list the storage directory '%s'", dir)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		if dir == "snapshots/" {
+			for _, file := range output.FileList.Files {
+				if file.Type == 1 {
+					files = append(files, file.Path + "/")
+				}
+				lastID = file.ID
+			}
+		} else {
+			for _, file := range output.FileList.Files {
+				if file.Type == 0 {
+					files = append(files, file.Path)
+					sizes = append(sizes, file.Size)
+				}
+				lastID = file.ID
+			}
+		}
+
+		if output.Truncated != 1 {
+			break
+		}
+	}
+	return files, sizes, nil
+}
+
+// getFileInfo returns the information about the file or directory at 'filePath'.
+func (storage *FileFabricStorage) getFileInfo(threadIndex int, filePath string) (fileID string, isDir bool, size int64, err error) {
+
+	formData := url.Values { "path" : {storage.storageDir + filePath}}
+
+	readCloser, _, _, err := storage.sendRequest(threadIndex, http.MethodPost, storage.getAPIURL("checkPathExists"), nil, formData)
+	if err != nil {
+		return "", false, 0, err
+	}
+
+	defer readCloser.Close()
+	defer io.Copy(ioutil.Discard, readCloser)	
+
+	var output struct {
+		FileFabricResponse
+		File FileFabricFile `xml:"file"`
+		Exists string `xml:"exists"`
+	}
+
+	err = xml.NewDecoder(readCloser).Decode(&output)
+	if err != nil {
+		return "", false, 0, err
+	}
+
+	err = checkFileFabricResponse(output.FileFabricResponse, "get the info on '%s'", filePath)
+	if err != nil {
+		return "", false, 0, err
+	}
+
+	if output.Exists != "y" {
+		return "", false, 0, nil
+	} else {
+		if output.File.Type == 1 {
+			for filePath != "" && filePath[len(filePath)-1] == '/' {
+				filePath = filePath[:len(filePath)-1]
+			}
+			
+			storage.directoryCacheLock.Lock()
+			storage.directoryCache[filePath] = output.File.ID
+			storage.directoryCacheLock.Unlock()
+		}
+		return output.File.ID, output.File.Type == 1, output.File.Size, nil
+	}
+}
+
+// GetFileInfo returns the information about the file or directory at 'filePath'.  This is a function required by the Storage interface.
+func (storage *FileFabricStorage) GetFileInfo(threadIndex int, filePath string) (exist bool, isDir bool, size int64, err error) {
+
+	fileID := ""
+	fileID, isDir, size, err = storage.getFileInfo(threadIndex, filePath)
+	return fileID != "", isDir, size, err
+}
+
+// DeleteFile deletes the file or directory at 'filePath'.
+func (storage *FileFabricStorage) DeleteFile(threadIndex int, filePath string) (err error) {
+
+	fileID, _, _, _ := storage.getFileInfo(threadIndex, filePath)
+	if fileID == "" {
+		return nil
+	}
+
+	formData := url.Values { "fi_id" : {fileID}}
+
+	readCloser, _, _, err := storage.sendRequest(threadIndex, http.MethodPost, storage.getAPIURL("doDeleteFile"), nil, formData)
+	if err != nil {
+		return err
+	}
+
+	defer readCloser.Close()
+	defer io.Copy(ioutil.Discard, readCloser)	
+
+	var output FileFabricResponse
+
+	err = xml.NewDecoder(readCloser).Decode(&output)
+	if err != nil {
+		return err
+	}
+
+	err = checkFileFabricResponse(output, "delete file '%s'", filePath)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// MoveFile renames the file.
+func (storage *FileFabricStorage) MoveFile(threadIndex int, from string, to string) (err error) {
+	fileID, _, _, _ := storage.getFileInfo(threadIndex, from)
+	if fileID == "" {
+		return nil
+	}
+
+	formData := url.Values { "fi_id" : {fileID}, "fi_name": {filepath.Base(to)},}
+
+	readCloser, _, _, err := storage.sendRequest(threadIndex, http.MethodPost, storage.getAPIURL("doRenameFile"), nil, formData)
+	if err != nil {
+		return err
+	}
+
+	defer readCloser.Close()
+	defer io.Copy(ioutil.Discard, readCloser)	
+
+	var output FileFabricResponse
+
+	err = xml.NewDecoder(readCloser).Decode(&output)
+	if err != nil {
+		return err
+	}
+
+	err = checkFileFabricResponse(output, "rename file '%s' to '%s'", from, to)
+	if err != nil {
+		return err
+	}
+	
+	return nil
+}
+
+// createParentDirectory creates the parent directory if it doesn't exist in the cache.
+func (storage *FileFabricStorage) createParentDirectory(threadIndex int, dir string) (parentID string, err error) {
+
+	found := strings.LastIndex(dir, "/")
+	if found == -1 {
+		return storage.storageDirID, nil
+	}
+	parent := dir[:found]
+
+	storage.directoryCacheLock.Lock()
+	parentID = storage.directoryCache[parent]
+	storage.directoryCacheLock.Unlock()
+
+	if parentID != "" {
+		return parentID, nil
+	}
+
+	parentID, err = storage.createDirectory(threadIndex, parent)
+	if err != nil {
+		if err == errFileFabricDirectoryExists {
+		    var isDir bool
+			parentID, isDir, _, err = storage.getFileInfo(threadIndex, parent)
+			if err != nil {
+				return "", err
+			}
+			if isDir == false {
+				return "", fmt.Errorf("'%s' in the storage is a file", parent)
+			}
+			storage.directoryCacheLock.Lock()
+			storage.directoryCache[parent] = parentID
+			storage.directoryCacheLock.Unlock()
+			return parentID, nil
+		} else {
+			return "", err
+		}
+	}
+	return parentID, nil
+}
+
+// createDirectory creates a new directory.
+func (storage *FileFabricStorage) createDirectory(threadIndex int, dir string) (dirID string, err error) {
+	for dir != "" && dir[len(dir)-1] == '/' {
+		dir = dir[:len(dir)-1]
+	}
+
+	parentID, err := storage.createParentDirectory(threadIndex, dir)
+	if err != nil {
+		return "", err
+	}
+
+	formData := url.Values { "fi_name": {filepath.Base(dir)}, "fi_pid" : {parentID}}
+
+	readCloser, _, _, err := storage.sendRequest(threadIndex, http.MethodPost, storage.getAPIURL("doCreateNewFolder"), nil, formData)
+	if err != nil {
+		return "", err
+	}
+
+	defer readCloser.Close()
+	defer io.Copy(ioutil.Discard, readCloser)	
+
+	var output struct {
+		FileFabricResponse
+		File FileFabricFile `xml:"file"`
+	}
+
+	err = xml.NewDecoder(readCloser).Decode(&output)
+	if err != nil {
+		return "", err
+	}
+
+	err = checkFileFabricResponse(output.FileFabricResponse, "create directory '%s'", dir)
+	if err != nil {
+		return "", err
+	}
+
+	storage.directoryCacheLock.Lock()
+	storage.directoryCache[dir] = output.File.ID
+	storage.directoryCacheLock.Unlock()
+
+	return output.File.ID, nil
+}
+
+func (storage *FileFabricStorage) CreateDirectory(threadIndex int, dir string) (err error) {
+	_, err = storage.createDirectory(threadIndex, dir)
+	if err == errFileFabricDirectoryExists {
+		return nil
+	}
+	return err
+}
+
+// DownloadFile reads the file at 'filePath' into the chunk.
+func (storage *FileFabricStorage) DownloadFile(threadIndex int, filePath string, chunk *Chunk) (err error) {
+	formData := url.Values { "fi_id" : {storage.storageDir + filePath}}
+
+	readCloser, _, _, err := storage.sendRequest(threadIndex, http.MethodPost, storage.getAPIURL("getFile"), nil, formData)
+	if err != nil {
+		return err
+	}
+
+	defer readCloser.Close()
+	defer io.Copy(ioutil.Discard, readCloser)	
+	_, err = RateLimitedCopy(chunk, readCloser, storage.DownloadRateLimit/storage.threads)
+	return err
+}
+
+// UploadFile writes 'content' to the file at 'filePath'.
+func (storage *FileFabricStorage) UploadFile(threadIndex int, filePath string, content []byte) (err error) {
+
+	parentID, err := storage.createParentDirectory(threadIndex, filePath)
+	if err != nil {
+		return err
+	}
+
+	fileName := filepath.Base(filePath)
+    requestBody := &bytes.Buffer{}
+    writer := multipart.NewWriter(requestBody)
+    part, _ := writer.CreateFormFile("file_1", fileName)
+    part.Write(content)
+
+    writer.WriteField("file_name1", fileName)
+	writer.WriteField("fi_pid", parentID)
+	writer.WriteField("fi_structtype", "g")
+    writer.Close()
+
+	headers := make(map[string]string)
+	headers["Content-Type"] = writer.FormDataContentType()
+
+	rateLimitedReader := CreateRateLimitedReader(requestBody.Bytes(), storage.UploadRateLimit/storage.threads)
+	readCloser, _, _, err := storage.sendRequest(threadIndex, http.MethodPost, storage.getAPIURL("doUploadFiles"), headers, rateLimitedReader)
+
+	defer readCloser.Close()
+	defer io.Copy(ioutil.Discard, readCloser)	
+
+	var output FileFabricResponse
+
+	err = xml.NewDecoder(readCloser).Decode(&output)
+	if err != nil {
+		return err
+	}
+
+	err = checkFileFabricResponse(output, "upload file '%s'", filePath)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// If a local snapshot cache is needed for the storage to avoid downloading/uploading chunks too often when
+// managing snapshots.
+func (storage *FileFabricStorage) IsCacheNeeded() bool { return true }
+
+// If the 'MoveFile' method is implemented.
+func (storage *FileFabricStorage) IsMoveFileImplemented() bool { return true }
+
+// If the storage can guarantee strong consistency.
+func (storage *FileFabricStorage) IsStrongConsistent() bool { return false }
+
+// If the storage supports fast listing of files names.
+func (storage *FileFabricStorage) IsFastListing() bool { return false }
+
+// Enable the test mode.
+func (storage *FileFabricStorage) EnableTestMode() { storage.testMode = true }

src/duplicacy_filestorage.go

@@ -12,6 +12,7 @@ import (
 	"os"
 	"path"
 	"strings"
+	"syscall"
 	"time"
 )
 
@@ -190,10 +191,13 @@ func (storage *FileStorage) UploadFile(threadIndex int, filePath string, content
 		return err
 	}
 
-	err = file.Sync()
-	if err != nil {
-		file.Close()
-		return err
+	if err = file.Sync(); err != nil {
+		pathErr, ok := err.(*os.PathError)
+		isNotSupported := ok && pathErr.Op == "sync" && pathErr.Err == syscall.ENOTSUP
+		if !isNotSupported {
+			_ = file.Close()
+			return err
+		}
 	}
 
 	err = file.Close()

src/duplicacy_gcdstorage.go

@@ -20,13 +20,16 @@ import (
 
 	"golang.org/x/net/context"
 	"golang.org/x/oauth2"
+	"golang.org/x/oauth2/google"
 	"google.golang.org/api/drive/v3"
 	"google.golang.org/api/googleapi"
+	"google.golang.org/api/option"
 )
 
 var (
 	GCDFileMimeType      = "application/octet-stream"
 	GCDDirectoryMimeType = "application/vnd.google-apps.folder"
+	GCDUserDrive         = "root"
 )
 
 type GCDStorage struct {
@@ -37,6 +40,7 @@ type GCDStorage struct {
 	idCacheLock sync.Mutex
 	backoffs    []int // desired backoff time in seconds for each thread
 	attempts    []int // number of failed attempts since last success for each thread
+	driveID     string // the ID of the shared drive or 'root' (GCDUserDrive) if the user's drive
 
 	createDirectoryLock sync.Mutex
 	isConnected         bool
@@ -78,6 +82,14 @@ func (storage *GCDStorage) shouldRetry(threadIndex int, err error) (bool, error)
 			// User Rate Limit Exceeded
 			message = e.Message
 			retry = true
+		} else if e.Code == 408 {
+			// Request timeout
+			message = e.Message
+			retry = true
+		} else if e.Code == 400 && strings.Contains(e.Message, "failedPrecondition") {
+			// Daily quota exceeded
+			message = e.Message
+			retry = true
 		} else if e.Code == 401 {
 			// Only retry on authorization error when storage has been connected before
 			if storage.isConnected {
@@ -187,7 +199,11 @@ func (storage *GCDStorage) listFiles(threadIndex int, parentID string, listFiles
 		var err error
 
 		for {
-			fileList, err = storage.service.Files.List().Q(query).Fields("nextPageToken", "files(name, mimeType, id, size)").PageToken(startToken).PageSize(maxCount).Do()
+			q := storage.service.Files.List().Q(query).Fields("nextPageToken", "files(name, mimeType, id, size)").PageToken(startToken).PageSize(maxCount)
+			if storage.driveID != GCDUserDrive {
+				q = q.DriveId(storage.driveID).IncludeItemsFromAllDrives(true).Corpora("drive").SupportsAllDrives(true)
+			}
+			fileList, err = q.Do()
 			if retry, e := storage.shouldRetry(threadIndex, err); e == nil && !retry {
 				break
 			} else if retry {
@@ -215,7 +231,11 @@ func (storage *GCDStorage) listByName(threadIndex int, parentID string, name str
 
 	for {
 		query := "name = '" + name + "' and '" + parentID + "' in parents and trashed = false "
-		fileList, err = storage.service.Files.List().Q(query).Fields("files(name, mimeType, id, size)").Do()
+		q := storage.service.Files.List().Q(query).Fields("files(name, mimeType, id, size)")
+		if storage.driveID != GCDUserDrive {
+			q = q.DriveId(storage.driveID).IncludeItemsFromAllDrives(true).Corpora("drive").SupportsAllDrives(true)
+		}
+		fileList, err = q.Do()
 
 		if retry, e := storage.shouldRetry(threadIndex, err); e == nil && !retry {
 			break
@@ -244,7 +264,7 @@ func (storage *GCDStorage) getIDFromPath(threadIndex int, filePath string, creat
 		return fileID, nil
 	}
 
-	fileID := "root"
+	fileID := storage.driveID
 
 	if rootID, ok := storage.findPathID(""); ok {
 		fileID = rootID
@@ -299,37 +319,85 @@ func (storage *GCDStorage) getIDFromPath(threadIndex int, filePath string, creat
 }
 
 // CreateGCDStorage creates a GCD storage object.
-func CreateGCDStorage(tokenFile string, storagePath string, threads int) (storage *GCDStorage, err error) {
+func CreateGCDStorage(tokenFile string, driveID string, storagePath string, threads int) (storage *GCDStorage, err error) {
+
+	ctx := context.Background()
 
 	description, err := ioutil.ReadFile(tokenFile)
 	if err != nil {
 		return nil, err
 	}
 
-	gcdConfig := &GCDConfig{}
-	if err := json.Unmarshal(description, gcdConfig); err != nil {
-		return nil, err
-	}
+	var object map[string]interface{}
 
-	oauth2Config := oauth2.Config{
-		ClientID:     gcdConfig.ClientID,
-		ClientSecret: gcdConfig.ClientSecret,
-		Endpoint:     gcdConfig.Endpoint,
-	}
-
-	authClient := oauth2Config.Client(context.Background(), &gcdConfig.Token)
-
-	service, err := drive.New(authClient)
+	err = json.Unmarshal(description, &object)
 	if err != nil {
 		return nil, err
 	}
 
+	isServiceAccount := false
+	if value, ok := object["type"]; ok {
+		if authType, ok := value.(string); ok && authType == "service_account" {
+			isServiceAccount = true
+		}
+	}
+
+	var tokenSource oauth2.TokenSource
+
+	if isServiceAccount {
+		config, err := google.JWTConfigFromJSON(description, drive.DriveScope)
+		if err != nil {
+			return nil, err
+		}
+		tokenSource = config.TokenSource(ctx)
+	} else {
+		gcdConfig := &GCDConfig{}
+		if err := json.Unmarshal(description, gcdConfig); err != nil {
+			return nil, err
+		}
+
+		config := oauth2.Config{
+			ClientID:     gcdConfig.ClientID,
+			ClientSecret: gcdConfig.ClientSecret,
+			Endpoint:     gcdConfig.Endpoint,
+		}
+		tokenSource = config.TokenSource(ctx, &gcdConfig.Token)
+	}
+
+	service, err := drive.NewService(ctx, option.WithTokenSource(tokenSource))
+	if err != nil {
+		return nil, err
+	}
+
+	if len(driveID) == 0 {
+		driveID = GCDUserDrive
+	} else {
+		driveList, err := drive.NewTeamdrivesService(service).List().Do()
+		if err != nil {
+			return nil, fmt.Errorf("Failed to look up the drive id: %v", err)
+		}
+
+		found := false
+		for _, teamDrive := range driveList.TeamDrives {
+			if teamDrive.Id == driveID || teamDrive.Name == driveID {
+				driveID = teamDrive.Id
+				found = true
+				break
+			}
+		}
+
+		if !found {
+			return nil, fmt.Errorf("%s is not the id or name of a shared drive", driveID)
+		}
+	}
+
 	storage = &GCDStorage{
 		service:         service,
 		numberOfThreads: threads,
 		idCache:         make(map[string]string),
 		backoffs:        make([]int, threads),
 		attempts:        make([]int, threads),
+		driveID:         driveID,
 	}
 
 	for i := range storage.backoffs {
@@ -337,6 +405,7 @@ func CreateGCDStorage(tokenFile string, storagePath string, threads int) (storag
 		storage.attempts[i] = 0
 	}
 
+	storage.savePathID("", driveID)
 	storagePathID, err := storage.getIDFromPath(0, storagePath, true)
 	if err != nil {
 		return nil, err
@@ -411,39 +480,76 @@ func (storage *GCDStorage) ListFiles(threadIndex int, dir string) ([]string, []i
 		}
 		return files, nil, nil
 	} else {
-		files := []string{}
-		sizes := []int64{}
+		lock := sync.Mutex {}
+		allFiles := []string{}
+		allSizes := []int64{}
+
+		errorChannel := make(chan error)
+		directoryChannel := make(chan string)
+		activeWorkers := 0
 
 		parents := []string{"chunks", "fossils"}
-		for i := 0; i < len(parents); i++ {
-			parent := parents[i]
-			pathID, ok := storage.findPathID(parent)
-			if !ok {
-				continue
-			}
-			entries, err := storage.listFiles(threadIndex, pathID, true, true)
-			if err != nil {
-				return nil, nil, err
-			}
-			for _, entry := range entries {
-				if entry.MimeType != GCDDirectoryMimeType {
-					name := entry.Name
-					if strings.HasPrefix(parent, "fossils") {
-						name = parent + "/" + name + ".fsl"
-						name = name[len("fossils/"):]
-					} else {
-						name = parent + "/" + name
-						name = name[len("chunks/"):]
+		for len(parents) > 0 || activeWorkers > 0 {
+
+			if len(parents) > 0 && activeWorkers < storage.numberOfThreads {
+				parent := parents[0]
+				parents = parents[1:]
+				activeWorkers++
+				go func(parent string) {
+					pathID, ok := storage.findPathID(parent)
+					if !ok {
+						return
+					}
+					entries, err := storage.listFiles(threadIndex, pathID, true, true)
+					if err != nil {
+						errorChannel <- err
+						return
+					}
+
+					LOG_DEBUG("GCD_STORAGE", "Listing %s; %d items returned", parent, len(entries))
+
+					files := []string {}
+					sizes := []int64 {}
+					for _, entry := range entries {
+						if entry.MimeType != GCDDirectoryMimeType {
+							name := entry.Name
+							if strings.HasPrefix(parent, "fossils") {
+								name = parent + "/" + name + ".fsl"
+								name = name[len("fossils/"):]
+							} else {
+								name = parent + "/" + name
+								name = name[len("chunks/"):]
+							}
+							files = append(files, name)
+							sizes = append(sizes, entry.Size)
+						} else {
+							directoryChannel <- parent+"/"+entry.Name
+							storage.savePathID(parent+"/"+entry.Name, entry.Id)
+						}
+					}
+					lock.Lock()
+					allFiles = append(allFiles, files...)
+					allSizes = append(allSizes, sizes...)
+					lock.Unlock()
+					directoryChannel <- ""
+				} (parent)
+			}
+
+			if activeWorkers > 0 {
+				select {
+				case err := <- errorChannel:
+					return nil, nil, err
+				case directory := <- directoryChannel:
+					if directory == "" {
+						activeWorkers--
+					} else {
+						parents = append(parents, directory)
 					}
-					files = append(files, name)
-					sizes = append(sizes, entry.Size)
-				} else {
-					parents = append(parents, parent+"/"+entry.Name)
-					storage.savePathID(parent+"/"+entry.Name, entry.Id)
 				}
 			}
 		}
-		return files, sizes, nil
+
+		return allFiles, allSizes, nil
 	}
 
 }
@@ -458,7 +564,7 @@ func (storage *GCDStorage) DeleteFile(threadIndex int, filePath string) (err err
 	}
 
 	for {
-		err = storage.service.Files.Delete(fileID).Fields("id").Do()
+		err = storage.service.Files.Delete(fileID).SupportsAllDrives(true).Fields("id").Do()
 		if retry, err := storage.shouldRetry(threadIndex, err); err == nil && !retry {
 			storage.deletePathID(filePath)
 			return nil
@@ -504,7 +610,7 @@ func (storage *GCDStorage) MoveFile(threadIndex int, from string, to string) (er
 	}
 
 	for {
-		_, err = storage.service.Files.Update(fileID, nil).AddParents(toParentID).RemoveParents(fromParentID).Do()
+		_, err = storage.service.Files.Update(fileID, nil).SupportsAllDrives(true).AddParents(toParentID).RemoveParents(fromParentID).Do()
 		if retry, err := storage.shouldRetry(threadIndex, err); err == nil && !retry {
 			break
 		} else if retry {
@@ -555,7 +661,7 @@ func (storage *GCDStorage) CreateDirectory(threadIndex int, dir string) (err err
 			Parents:  []string{parentID},
 		}
 
-		file, err = storage.service.Files.Create(file).Fields("id").Do()
+		file, err = storage.service.Files.Create(file).SupportsAllDrives(true).Fields("id").Do()
 		if retry, err := storage.shouldRetry(threadIndex, err); err == nil && !retry {
 			break
 		} else {
@@ -626,7 +732,7 @@ func (storage *GCDStorage) DownloadFile(threadIndex int, filePath string, chunk
 	for {
 		// AcknowledgeAbuse(true) lets the download proceed even if GCD thinks that it contains malware.
 		// TODO: Should this prompt the user or log a warning?
-		req := storage.service.Files.Get(fileID)
+		req := storage.service.Files.Get(fileID).SupportsAllDrives(true)
 		if e, ok := err.(*googleapi.Error); ok {
 			if strings.Contains(err.Error(), "cannotDownloadAbusiveFile") || len(e.Errors) > 0 && e.Errors[0].Reason == "cannotDownloadAbusiveFile" {
 				LOG_WARN("GCD_STORAGE", "%s is marked as abusive, will download anyway.", filePath)
@@ -672,7 +778,7 @@ func (storage *GCDStorage) UploadFile(threadIndex int, filePath string, content
 
 	for {
 		reader := CreateRateLimitedReader(content, storage.UploadRateLimit/storage.numberOfThreads)
-		_, err = storage.service.Files.Create(file).Media(reader).Fields("id").Do()
+		_, err = storage.service.Files.Create(file).SupportsAllDrives(true).Media(reader).Fields("id").Do()
 		if retry, err := storage.shouldRetry(threadIndex, err); err == nil && !retry {
 			break
 		} else if retry {

src/duplicacy_log.go

@@ -7,10 +7,12 @@ package duplicacy
 import (
 	"fmt"
 	"os"
+	"log"
 	"runtime/debug"
 	"sync"
 	"testing"
 	"time"
+	"regexp"
 )
 
 const (
@@ -43,6 +45,13 @@ func setTestingT(t *testing.T) {
 	testingT = t
 }
 
+// Contains the ids of logs that won't be displayed
+var suppressedLogs map[string]bool = map[string]bool{}
+
+func SuppressLog(id string) {
+	suppressedLogs[id] = true
+}
+
 func getLevelName(level int) string {
 	switch level {
 	case DEBUG:
@@ -98,6 +107,15 @@ func LOG_ERROR(logID string, format string, v ...interface{}) {
 	logf(ERROR, logID, format, v...)
 }
 
+func LOG_WERROR(isWarning bool, logID string, format string, v ...interface{}) {
+	if isWarning {
+		logf(WARN, logID, format, v...)
+	} else {
+		logf(ERROR, logID, format, v...)
+	}
+}
+
+
 func LOG_FATAL(logID string, format string, v ...interface{}) {
 	logf(FATAL, logID, format, v...)
 }
@@ -143,6 +161,12 @@ func logf(level int, logID string, format string, v ...interface{}) {
 		defer logMutex.Unlock()
 
 		if level >= loggingLevel {
+			if level <= ERROR && len(suppressedLogs) > 0 {
+				if _, found := suppressedLogs[logID]; found {
+					return
+				}
+			}
+
 			if printLogHeader {
 				fmt.Printf("%s %s %s %s\n",
 					now.Format("2006-01-02 15:04:05.000"), getLevelName(level), logID, message)
@@ -161,6 +185,32 @@ func logf(level int, logID string, format string, v ...interface{}) {
 	}
 }
 
+// Set up logging for libraries that Duplicacy depends on.  They can call 'log.Printf("[ID] message")'
+// to produce logs in Duplicacy's format
+type Logger struct {
+	formatRegex *regexp.Regexp
+}
+
+func (logger *Logger) Write(line []byte) (n int, err error) {
+	n = len(line)
+	for len(line) > 0 && line[len(line) - 1] == '\n' {
+		line = line[:len(line) - 1]
+	}
+	matched := logger.formatRegex.FindStringSubmatch(string(line))
+	if matched != nil {
+		LOG_INFO(matched[1], "%s", matched[2])
+	} else {
+		LOG_INFO("LOG_DEFAULT", "%s", line)
+	}
+
+    return
+}
+
+func init() {
+	log.SetFlags(0)
+	log.SetOutput(&Logger{ formatRegex: regexp.MustCompile(`^\[(.+)\]\s*(.+)`) })
+}
+
 const (
 	duplicacyExitCode = 100
 	otherExitCode     = 101

src/duplicacy_oneclient.go

@@ -13,8 +13,10 @@ import (
 	"math/rand"
 	"net/http"
 	"strings"
+	"strconv"
 	"sync"
 	"time"
+	"path/filepath"
 
 	"golang.org/x/oauth2"
 )
@@ -32,9 +34,6 @@ type OneDriveErrorResponse struct {
 	Error OneDriveError `json:"error"`
 }
 
-var OneDriveRefreshTokenURL = "https://duplicacy.com/one_refresh"
-var OneDriveAPIURL = "https://api.onedrive.com/v1.0"
-
 type OneDriveClient struct {
 	HTTPClient *http.Client
 
@@ -44,9 +43,13 @@ type OneDriveClient struct {
 
 	IsConnected bool
 	TestMode    bool
+
+	IsBusiness bool
+	RefreshTokenURL string
+	APIURL string
 }
 
-func NewOneDriveClient(tokenFile string) (*OneDriveClient, error) {
+func NewOneDriveClient(tokenFile string, isBusiness bool) (*OneDriveClient, error) {
 
 	description, err := ioutil.ReadFile(tokenFile)
 	if err != nil {
@@ -63,6 +66,15 @@ func NewOneDriveClient(tokenFile string) (*OneDriveClient, error) {
 		TokenFile:  tokenFile,
 		Token:      token,
 		TokenLock:  &sync.Mutex{},
+		IsBusiness: isBusiness,
+	}
+
+	if isBusiness {
+		client.RefreshTokenURL = "https://duplicacy.com/odb_refresh"
+		client.APIURL = "https://graph.microsoft.com/v1.0/me"
+	} else {
+		client.RefreshTokenURL = "https://duplicacy.com/one_refresh"
+		client.APIURL = "https://api.onedrive.com/v1.0"
 	}
 
 	client.RefreshToken(false)
@@ -75,7 +87,7 @@ func (client *OneDriveClient) call(url string, method string, input interface{},
 	var response *http.Response
 
 	backoff := 1
-	for i := 0; i < 8; i++ {
+	for i := 0; i < 12; i++ {
 
 		LOG_DEBUG("ONEDRIVE_CALL", "%s %s", method, url)
 
@@ -106,9 +118,10 @@ func (client *OneDriveClient) call(url string, method string, input interface{},
 
 		if reader, ok := inputReader.(*RateLimitedReader); ok {
 			request.ContentLength = reader.Length()
+			request.Header.Set("Content-Range", fmt.Sprintf("bytes 0-%d/%d", reader.Length() - 1, reader.Length()))
 		}
 
-		if url != OneDriveRefreshTokenURL {
+		if url != client.RefreshTokenURL {
 			client.TokenLock.Lock()
 			request.Header.Set("Authorization", "Bearer "+client.Token.AccessToken)
 			client.TokenLock.Unlock()
@@ -117,6 +130,8 @@ func (client *OneDriveClient) call(url string, method string, input interface{},
 			request.Header.Set("Content-Type", contentType)
 		}
 
+		request.Header.Set("User-Agent", "ISV|Acrosync|Duplicacy/2.0")
+
 		response, err = client.HTTPClient.Do(request)
 		if err != nil {
 			if client.IsConnected {
@@ -133,6 +148,9 @@ func (client *OneDriveClient) call(url string, method string, input interface{},
 					time.Sleep(retryAfter * time.Millisecond)
 				}
 				backoff *= 2
+				if backoff > 256 {
+					backoff = 256
+				}
 				continue
 			}
 			return nil, 0, err
@@ -152,7 +170,7 @@ func (client *OneDriveClient) call(url string, method string, input interface{},
 
 		if response.StatusCode == 401 {
 
-			if url == OneDriveRefreshTokenURL {
+			if url == client.RefreshTokenURL {
 				return nil, 0, OneDriveError{Status: response.StatusCode, Message: "Authorization error when refreshing token"}
 			}
 
@@ -161,11 +179,23 @@ func (client *OneDriveClient) call(url string, method string, input interface{},
 				return nil, 0, err
 			}
 			continue
+		} else if response.StatusCode == 409 {
+			return nil, 0, OneDriveError{Status: response.StatusCode, Message: "Conflict"}
 		} else if response.StatusCode > 401 && response.StatusCode != 404 {
-			retryAfter := time.Duration(rand.Float32() * 1000.0 * float32(backoff))
-			LOG_INFO("ONEDRIVE_RETRY", "Response code: %d; retry after %d milliseconds", response.StatusCode, retryAfter)
-			time.Sleep(retryAfter * time.Millisecond)
+			delay := int((rand.Float32() * 0.5 + 0.5) * 1000.0 * float32(backoff))
+			if backoffList, found := response.Header["Retry-After"]; found && len(backoffList) > 0 {
+				retryAfter, _ := strconv.Atoi(backoffList[0])
+				if retryAfter * 1000 > delay {
+					delay = retryAfter * 1000
+				}
+			}
+
+			LOG_INFO("ONEDRIVE_RETRY", "Response code: %d; retry after %d milliseconds", response.StatusCode, delay)
+			time.Sleep(time.Duration(delay) * time.Millisecond)
 			backoff *= 2
+			if backoff > 256 {
+				backoff = 256
+			}
 			continue
 		} else {
 			if err := json.NewDecoder(response.Body).Decode(errorResponse); err != nil {
@@ -188,7 +218,7 @@ func (client *OneDriveClient) RefreshToken(force bool) (err error) {
 		return nil
 	}
 
-	readCloser, _, err := client.call(OneDriveRefreshTokenURL, "POST", client.Token, "")
+	readCloser, _, err := client.call(client.RefreshTokenURL, "POST", client.Token, "")
 	if err != nil {
 		return fmt.Errorf("failed to refresh the access token: %v", err)
 	}
@@ -228,9 +258,9 @@ func (client *OneDriveClient) ListEntries(path string) ([]OneDriveEntry, error)
 
 	entries := []OneDriveEntry{}
 
-	url := OneDriveAPIURL + "/drive/root:/" + path + ":/children"
+	url := client.APIURL + "/drive/root:/" + path + ":/children"
 	if path == "" {
-		url = OneDriveAPIURL + "/drive/root/children"
+		url = client.APIURL + "/drive/root/children"
 	}
 	if client.TestMode {
 		url += "?top=8"
@@ -266,7 +296,7 @@ func (client *OneDriveClient) ListEntries(path string) ([]OneDriveEntry, error)
 
 func (client *OneDriveClient) GetFileInfo(path string) (string, bool, int64, error) {
 
-	url := OneDriveAPIURL + "/drive/root:/" + path
+	url := client.APIURL + "/drive/root:/" + path
 	url += "?select=id,name,size,folder"
 
 	readCloser, _, err := client.call(url, "GET", 0, "")
@@ -291,28 +321,95 @@ func (client *OneDriveClient) GetFileInfo(path string) (string, bool, int64, err
 
 func (client *OneDriveClient) DownloadFile(path string) (io.ReadCloser, int64, error) {
 
-	url := OneDriveAPIURL + "/drive/items/root:/" + path + ":/content"
+	url := client.APIURL + "/drive/items/root:/" + path + ":/content"
 
 	return client.call(url, "GET", 0, "")
 }
 
 func (client *OneDriveClient) UploadFile(path string, content []byte, rateLimit int) (err error) {
 
-	url := OneDriveAPIURL + "/drive/root:/" + path + ":/content"
+	// Upload file using the simple method; this is only possible for OneDrive Personal or if the file
+	// is smaller than 4MB for OneDrive Business
+	if !client.IsBusiness || (client.TestMode && rand.Int() % 2 == 0) {
+		url := client.APIURL + "/drive/root:/" + path + ":/content"
 
-	readCloser, _, err := client.call(url, "PUT", CreateRateLimitedReader(content, rateLimit), "application/octet-stream")
+		readCloser, _, err := client.call(url, "PUT", CreateRateLimitedReader(content, rateLimit), "application/octet-stream")
+		if err != nil {
+			return err
+		}
 
+		readCloser.Close()
+		return nil
+	}
+
+	// For large files, create an upload session first
+	uploadURL, err := client.CreateUploadSession(path)
 	if err != nil {
 		return err
 	}
 
+	return client.UploadFileSession(uploadURL, content, rateLimit)
+}
+
+func (client *OneDriveClient) CreateUploadSession(path string) (uploadURL string, err error) {
+
+	type CreateUploadSessionItem struct {
+		ConflictBehavior string `json:"@microsoft.graph.conflictBehavior"`
+		Name string `json:"name"`
+	}
+
+	input := map[string]interface{} {
+		"item": CreateUploadSessionItem {
+			ConflictBehavior: "replace",
+			Name: filepath.Base(path),
+		},
+	}
+
+	readCloser, _, err := client.call(client.APIURL + "/drive/root:/" + path + ":/createUploadSession", "POST", input, "application/json")
+	if err != nil {
+		return "", err
+	}
+
+	type CreateUploadSessionOutput struct {
+		UploadURL string `json:"uploadUrl"`
+	}
+
+	output := &CreateUploadSessionOutput{}
+
+	if err = json.NewDecoder(readCloser).Decode(&output); err != nil {
+		return "", err
+	}
+
+	readCloser.Close()
+	return output.UploadURL, nil
+}
+
+func (client *OneDriveClient) UploadFileSession(uploadURL string, content []byte, rateLimit int) (err error) {
+
+	readCloser, _, err := client.call(uploadURL, "PUT", CreateRateLimitedReader(content, rateLimit), "")
+	if err != nil {
+		return err
+	}
+	type UploadFileSessionOutput struct {
+		Size int `json:"size"`
+	}
+	output := &UploadFileSessionOutput{}
+
+	if err = json.NewDecoder(readCloser).Decode(&output); err != nil {
+		return fmt.Errorf("Failed to complete the file upload session: %v", err)
+	}
+
+	if output.Size != len(content) {
+		return fmt.Errorf("Uploaded %d bytes out of %d bytes", output.Size, len(content))
+	}
+
 	readCloser.Close()
 	return nil
 }
 
 func (client *OneDriveClient) DeleteFile(path string) error {
 
-	url := OneDriveAPIURL + "/drive/root:/" + path
+	url := client.APIURL + "/drive/root:/" + path
 
 	readCloser, _, err := client.call(url, "DELETE", 0, "")
 	if err != nil {
@@ -325,7 +422,7 @@ func (client *OneDriveClient) DeleteFile(path string) error {
 
 func (client *OneDriveClient) MoveFile(path string, parent string) error {
 
-	url := OneDriveAPIURL + "/drive/root:/" + path
+	url := client.APIURL + "/drive/root:/" + path
 
 	parentReference := make(map[string]string)
 	parentReference["path"] = "/drive/root:/" + parent
@@ -335,6 +432,20 @@ func (client *OneDriveClient) MoveFile(path string, parent string) error {
 
 	readCloser, _, err := client.call(url, "PATCH", parameters, "application/json")
 	if err != nil {
+		if e, ok := err.(OneDriveError); ok && e.Status == 400 {
+			// The destination directory doesn't exist; trying to create it...
+			dir := filepath.Dir(parent)
+			if dir == "." {
+				dir = ""
+			}
+			client.CreateDirectory(dir, filepath.Base(parent))
+			readCloser, _, err = client.call(url, "PATCH", parameters, "application/json")
+			if err != nil {
+				return nil
+			}
+
+		}
+
 		return err
 	}
 
@@ -344,24 +455,29 @@ func (client *OneDriveClient) MoveFile(path string, parent string) error {
 
 func (client *OneDriveClient) CreateDirectory(path string, name string) error {
 
-	url := OneDriveAPIURL + "/root/children"
+	url := client.APIURL + "/root/children"
 
 	if path != "" {
 
-		parentID, isDir, _, err := client.GetFileInfo(path)
+		pathID, isDir, _, err := client.GetFileInfo(path)
 		if err != nil {
 			return err
 		}
 
-		if parentID == "" {
-			return fmt.Errorf("The path '%s' does not exist", path)
+		if pathID == "" {
+			dir := filepath.Dir(path)
+			if dir != "." {
+				// The parent directory doesn't exist; trying to create it...
+				client.CreateDirectory(dir, filepath.Base(path))
+				isDir = true
+			}
 		}
 
 		if !isDir {
 			return fmt.Errorf("The path '%s' is not a directory", path)
 		}
 
-		url = OneDriveAPIURL + "/drive/items/" + parentID + "/children"
+		url = client.APIURL + "/drive/root:/" + path + ":/children"
 	}
 
 	parameters := make(map[string]interface{})
@@ -370,6 +486,11 @@ func (client *OneDriveClient) CreateDirectory(path string, name string) error {
 
 	readCloser, _, err := client.call(url, "POST", parameters, "application/json")
 	if err != nil {
+		if e, ok := err.(OneDriveError); ok && e.Status == 409 {
+			// This error usually means the directory already exists
+			LOG_TRACE("ONEDRIVE_MKDIR", "The directory '%s/%s' already exists", path, name)
+			return nil
+		}
 		return err
 	}
 

src/duplicacy_oneclient_test.go

@@ -17,7 +17,7 @@ import (
 
 func TestOneDriveClient(t *testing.T) {
 
-	oneDriveClient, err := NewOneDriveClient("one-token.json")
+	oneDriveClient, err := NewOneDriveClient("one-token.json", false)
 	if err != nil {
 		t.Errorf("Failed to create the OneDrive client: %v", err)
 		return

src/duplicacy_onestorage.go

@@ -19,13 +19,13 @@ type OneDriveStorage struct {
 }
 
 // CreateOneDriveStorage creates an OneDrive storage object.
-func CreateOneDriveStorage(tokenFile string, storagePath string, threads int) (storage *OneDriveStorage, err error) {
+func CreateOneDriveStorage(tokenFile string, isBusiness bool, storagePath string, threads int) (storage *OneDriveStorage, err error) {
 
 	for len(storagePath) > 0 && storagePath[len(storagePath)-1] == '/' {
 		storagePath = storagePath[:len(storagePath)-1]
 	}
 
-	client, err := NewOneDriveClient(tokenFile)
+	client, err := NewOneDriveClient(tokenFile, isBusiness)
 	if err != nil {
 		return nil, err
 	}
@@ -80,6 +80,7 @@ func (storage *OneDriveStorage) convertFilePath(filePath string) string {
 
 // ListFiles return the list of files and subdirectories under 'dir' (non-recursively)
 func (storage *OneDriveStorage) ListFiles(threadIndex int, dir string) ([]string, []int64, error) {
+
 	for len(dir) > 0 && dir[len(dir)-1] == '/' {
 		dir = dir[:len(dir)-1]
 	}

src/duplicacy_preference.go

@@ -25,6 +25,8 @@ type Preference struct {
 	DoNotSavePassword bool              `json:"no_save_password"`
 	NobackupFile      string            `json:"nobackup_file"`
 	Keys              map[string]string `json:"keys"`
+	FiltersFile       string            `json:"filters"`
+	ExcludeByAttribute bool             `json:"exclude_by_attribute"`
 }
 
 var preferencePath string

src/duplicacy_s3storage.go

@@ -210,7 +210,7 @@ func (storage *S3Storage) DownloadFile(threadIndex int, filePath string, chunk *
 
 	defer output.Body.Close()
 
-	_, err = RateLimitedCopy(chunk, output.Body, storage.DownloadRateLimit/len(storage.bucket))
+	_, err = RateLimitedCopy(chunk, output.Body, storage.DownloadRateLimit/storage.numberOfThreads)
 	return err
 
 }
@@ -225,7 +225,7 @@ func (storage *S3Storage) UploadFile(threadIndex int, filePath string, content [
 			Bucket:      aws.String(storage.bucket),
 			Key:         aws.String(storage.storageDir + filePath),
 			ACL:         aws.String(s3.ObjectCannedACLPrivate),
-			Body:        CreateRateLimitedReader(content, storage.UploadRateLimit/len(storage.bucket)),
+			Body:        CreateRateLimitedReader(content, storage.UploadRateLimit/storage.numberOfThreads),
 			ContentType: aws.String("application/duplicacy"),
 		}
 

src/duplicacy_sftpstorage.go

@@ -43,10 +43,10 @@ func CreateSFTPStorageWithPassword(server string, port int, username string, sto
 		return nil
 	}
 
-	return CreateSFTPStorage(server, port, username, storageDir, minimumNesting, authMethods, hostKeyCallback, threads)
+	return CreateSFTPStorage(false, server, port, username, storageDir, minimumNesting, authMethods, hostKeyCallback, threads)
 }
 
-func CreateSFTPStorage(server string, port int, username string, storageDir string, minimumNesting int,
+func CreateSFTPStorage(compatibilityMode bool, server string, port int, username string, storageDir string, minimumNesting int,
 	authMethods []ssh.AuthMethod,
 	hostKeyCallback func(hostname string, remote net.Addr,
 		key ssh.PublicKey) error, threads int) (storage *SFTPStorage, err error) {
@@ -57,8 +57,21 @@ func CreateSFTPStorage(server string, port int, username string, storageDir stri
 		HostKeyCallback: hostKeyCallback,
 	}
 
-	if server == "sftp.hidrive.strato.com" {
-		sftpConfig.Ciphers = []string{"aes128-ctr", "aes256-ctr"}
+	if compatibilityMode {
+		sftpConfig.Ciphers = []string{
+			"aes128-ctr", "aes192-ctr", "aes256-ctr",
+			"aes128-gcm@openssh.com",
+			"chacha20-poly1305@openssh.com",
+			"arcfour256", "arcfour128", "arcfour",
+			"aes128-cbc",
+			"3des-cbc",
+		}
+		sftpConfig.KeyExchanges = [] string {
+			"curve25519-sha256@libssh.org",
+			"ecdh-sha2-nistp256", "ecdh-sha2-nistp384", "ecdh-sha2-nistp521",
+			"diffie-hellman-group1-sha1", "diffie-hellman-group14-sha1",
+			"diffie-hellman-group-exchange-sha1", "diffie-hellman-group-exchange-sha256",
+		}
 	}
 
 	serverAddress := fmt.Sprintf("%s:%d", server, port)
@@ -91,7 +104,7 @@ func CreateSFTPStorage(server string, port int, username string, storageDir stri
 		storageDir:      storageDir,
 		minimumNesting:  minimumNesting,
 		numberOfThreads: threads,
-		numberOfTries:   6,
+		numberOfTries:   8,
 		serverAddress:   serverAddress,
 		sftpConfig:      sftpConfig,
 	}
@@ -129,22 +142,19 @@ func (storage *SFTPStorage) retry(f func () error) error {
 			delay *= 2
 
 			storage.clientLock.Lock()
-			if storage.client != nil {
-				storage.client.Close()
-				storage.client = nil
-			}
-
 			connection, err := ssh.Dial("tcp", storage.serverAddress, storage.sftpConfig)
 			if err != nil {
+				LOG_WARN("SFT_RECONNECT", "Failed to connect to %s: %v; retrying", storage.serverAddress, err)
 				storage.clientLock.Unlock()
-				return err
+				continue
 			}
 
 			client, err := sftp.NewClient(connection)
 			if err != nil {
+				LOG_WARN("SFT_RECONNECT", "Failed to create a new SFTP client to %s: %v; retrying", storage.serverAddress, err)
 				connection.Close()
 				storage.clientLock.Unlock()
-				return err
+				continue
 			}
 			storage.client = client
 			storage.clientLock.Unlock()
@@ -275,36 +285,19 @@ func (storage *SFTPStorage) UploadFile(threadIndex int, filePath string, content
 	fullPath := path.Join(storage.storageDir, filePath)
 
 	dirs := strings.Split(filePath, "/")
-	if len(dirs) > 1 {
-		fullDir := path.Dir(fullPath)
-		err = storage.retry(func() error {
-			_, err := storage.getSFTPClient().Stat(fullDir)
-			return err
-		})
-		if err != nil {
-			// The error may be caused by a non-existent fullDir, or a broken connection.  In either case,
-			// we just assume it is the former because there isn't a way to tell which is the case.
-			for i := range dirs[1 : len(dirs)-1] {
-				subDir := path.Join(storage.storageDir, path.Join(dirs[0:i+2]...))
-				// We don't check the error; just keep going blindly but always store the last err
-				err = storage.getSFTPClient().Mkdir(subDir)
-			}
+	fullDir := path.Dir(fullPath)
+	return storage.retry(func() error {
 
-			// If there is an error creating the dirs, we check fullDir one more time, because another thread
-			// may happen to create the same fullDir ahead of this thread
-			if err != nil {
-				err = storage.retry(func() error {
-					_, err := storage.getSFTPClient().Stat(fullDir)
-					return err
-				})
-				if err != nil {
-					return err
+		if len(dirs) > 1 {
+			_, err := storage.getSFTPClient().Stat(fullDir)
+			if os.IsNotExist(err) {
+				for i := range dirs[1 : len(dirs)-1] {
+					subDir := path.Join(storage.storageDir, path.Join(dirs[0:i+2]...))
+					// We don't check the error; just keep going blindly
+					storage.getSFTPClient().Mkdir(subDir)
 				}
 			}
 		}
-	}
-
-	return storage.retry(func() error {
 
 		letters := "abcdefghijklmnopqrstuvwxyz"
 		suffix := make([]byte, 8)
@@ -325,7 +318,11 @@ func (storage *SFTPStorage) UploadFile(threadIndex int, filePath string, content
 			file.Close()
 			return err
 		}
-		file.Close()
+
+		err = file.Close()
+		if err != nil {
+			return err
+		}
 
 		err = storage.getSFTPClient().Rename(temporaryFile, fullPath)
 		if err != nil {

src/duplicacy_shadowcopy_darwin.go

@@ -13,6 +13,7 @@ import (
 	"io/ioutil"
 	"os"
 	"os/exec"
+	"regexp"
 	"strings"
 	"syscall"
 	"time"
@@ -123,11 +124,11 @@ func CreateShadowCopy(top string, shadowCopy bool, timeoutInSeconds int) (shadow
 	}
 	deviceIdRepository, err := GetPathDeviceId(top)
 	if err != nil {
-		LOG_ERROR("VSS_INIT", "Unable to get device ID of path: ", top)
+		LOG_ERROR("VSS_INIT", "Unable to get device ID of path: %s", top)
 		return top
 	}
 	if deviceIdLocal != deviceIdRepository {
-		LOG_WARN("VSS_PATH", "VSS not supported for non-local repository path: ", top)
+		LOG_WARN("VSS_PATH", "VSS not supported for non-local repository path: %s", top)
 		return top
 	}
 
@@ -145,22 +146,37 @@ func CreateShadowCopy(top string, shadowCopy bool, timeoutInSeconds int) (shadow
 	// Use tmutil to create snapshot
 	tmutilOutput, err := CommandWithTimeout(timeoutInSeconds, "tmutil", "snapshot")
 	if err != nil {
-		LOG_ERROR("VSS_CREATE", "Error while calling tmutil: ", err)
+		LOG_ERROR("VSS_CREATE", "Error while calling tmutil: %v", err)
 		return top
 	}
 
 	colonPos := strings.IndexByte(tmutilOutput, ':')
 	if colonPos < 0 {
-		LOG_ERROR("VSS_CREATE", "Snapshot creation failed: ", tmutilOutput)
+		LOG_ERROR("VSS_CREATE", "Snapshot creation failed: %s", tmutilOutput)
 		return top
 	}
 	snapshotDate = strings.TrimSpace(tmutilOutput[colonPos+1:])
 
+	tmutilOutput, err = CommandWithTimeout(timeoutInSeconds, "tmutil", "listlocalsnapshots", ".")
+	if err != nil {
+		LOG_ERROR("VSS_CREATE", "Error while calling 'tmutil listlocalsnapshots': %v", err)
+		return top
+	}
+	snapshotName := "com.apple.TimeMachine." + snapshotDate
+ 
+	r := regexp.MustCompile(`(?m)^(.+` + snapshotDate + `.*)$`)
+	snapshotNames := r.FindStringSubmatch(tmutilOutput)
+	if len(snapshotNames) > 0 {
+		snapshotName = snapshotNames[0]
+	} else {
+		LOG_WARN("VSS_CREATE", "Error while using 'tmutil listlocalsnapshots' to find snapshot name. Will fallback to 'com.apple.TimeMachine.SNAPSHOT_DATE'")
+	}
+ 
 	// Mount snapshot as readonly and hide from GUI i.e. Finder
 	_, err = CommandWithTimeout(timeoutInSeconds,
-		"/sbin/mount", "-t", "apfs", "-o", "nobrowse,-r,-s=com.apple.TimeMachine."+snapshotDate, "/", snapshotPath)
+		"/sbin/mount", "-t", "apfs", "-o", "nobrowse,-r,-s="+snapshotName, "/", snapshotPath)
 	if err != nil {
-		LOG_ERROR("VSS_CREATE", "Error while mounting snapshot: ", err)
+		LOG_ERROR("VSS_CREATE", "Error while mounting snapshot: %v", err)
 		return top
 	}
 

src/duplicacy_snapshot.go

@@ -11,6 +11,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"time"
@@ -57,7 +58,7 @@ func CreateEmptySnapshot(id string) (snapshto *Snapshot) {
 
 // CreateSnapshotFromDirectory creates a snapshot from the local directory 'top'.  Only 'Files'
 // will be constructed, while 'ChunkHashes' and 'ChunkLengths' can only be populated after uploading.
-func CreateSnapshotFromDirectory(id string, top string, nobackupFile string) (snapshot *Snapshot, skippedDirectories []string,
+func CreateSnapshotFromDirectory(id string, top string, nobackupFile string, filtersFile string, excludeByAttribute bool) (snapshot *Snapshot, skippedDirectories []string,
 	skippedFiles []string, err error) {
 
 	snapshot = &Snapshot{
@@ -68,47 +69,10 @@ func CreateSnapshotFromDirectory(id string, top string, nobackupFile string) (sn
 
 	var patterns []string
 
-	patternFile, err := ioutil.ReadFile(path.Join(GetDuplicacyPreferencePath(), "filters"))
-	if err == nil {
-		for _, pattern := range strings.Split(string(patternFile), "\n") {
-			pattern = strings.TrimSpace(pattern)
-			if len(pattern) == 0 {
-				continue
-			}
-
-			if pattern[0] == '#' {
-				continue
-			}
-
-			if IsUnspecifiedFilter(pattern) {
-				pattern = "+" + pattern
-			}
-
-			if IsEmptyFilter(pattern) {
-				continue
-			}
-
-			if strings.HasPrefix(pattern, "i:") || strings.HasPrefix(pattern, "e:") {
-				valid, err := IsValidRegex(pattern[2:])
-				if !valid || err != nil {
-					LOG_ERROR("SNAPSHOT_FILTER", "Invalid regular expression encountered for filter: \"%s\", error: %v", pattern, err)
-				}
-			}
-
-			patterns = append(patterns, pattern)
-		}
-
-		LOG_DEBUG("REGEX_DEBUG", "There are %d compiled regular expressions stored", len(RegexMap))
-
-		LOG_INFO("SNAPSHOT_FILTER", "Loaded %d include/exclude pattern(s)", len(patterns))
-
-		if IsTracing() {
-			for _, pattern := range patterns {
-				LOG_TRACE("SNAPSHOT_PATTERN", "Pattern: %s", pattern)
-			}
-		}
-
+	if filtersFile == "" {
+		filtersFile = joinPath(GetDuplicacyPreferencePath(), "filters")
 	}
+	patterns = ProcessFilters(filtersFile)
 
 	directories := make([]*Entry, 0, 256)
 	directories = append(directories, CreateEntry("", 0, 0, 0))
@@ -125,8 +89,12 @@ func CreateSnapshotFromDirectory(id string, top string, nobackupFile string) (sn
 		directory := directories[len(directories)-1]
 		directories = directories[:len(directories)-1]
 		snapshot.Files = append(snapshot.Files, directory)
-		subdirectories, skipped, err := ListEntries(top, directory.Path, &snapshot.Files, patterns, nobackupFile, snapshot.discardAttributes)
+		subdirectories, skipped, err := ListEntries(top, directory.Path, &snapshot.Files, patterns, nobackupFile, snapshot.discardAttributes, excludeByAttribute)
 		if err != nil {
+			if directory.Path == "" {
+				LOG_ERROR("LIST_FAILURE", "Failed to list the repository root: %v", err)
+				return nil, nil, nil, err
+			}
 			LOG_WARN("LIST_FAILURE", "Failed to list subdirectory: %v", err)
 			skippedDirectories = append(skippedDirectories, directory.Path)
 			continue
@@ -150,6 +118,103 @@ func CreateSnapshotFromDirectory(id string, top string, nobackupFile string) (sn
 	return snapshot, skippedDirectories, skippedFiles, nil
 }
 
+func AppendPattern(patterns []string, new_pattern string) (new_patterns []string) {
+	for _, pattern := range patterns {
+		if pattern == new_pattern {
+			LOG_INFO("SNAPSHOT_FILTER", "Ignoring duplicate pattern: %s ...", new_pattern)
+			return patterns
+		}
+	}
+	new_patterns = append(patterns, new_pattern)
+	return new_patterns
+}
+func ProcessFilters(filtersFile string) (patterns []string) {
+	patterns = ProcessFilterFile(filtersFile, make([]string, 0))
+
+	LOG_DEBUG("REGEX_DEBUG", "There are %d compiled regular expressions stored", len(RegexMap))
+
+	LOG_INFO("SNAPSHOT_FILTER", "Loaded %d include/exclude pattern(s)", len(patterns))
+
+	if IsTracing() {
+		for _, pattern := range patterns {
+			LOG_TRACE("SNAPSHOT_PATTERN", "Pattern: %s", pattern)
+		}
+
+	}
+
+	return patterns
+}
+
+func ProcessFilterFile(patternFile string, includedFiles []string) (patterns []string) {
+	for _, file := range includedFiles {
+		if file == patternFile {
+			// cycle in include mechanism discovered.
+			LOG_ERROR("SNAPSHOT_FILTER", "The filter file %s has already been included", patternFile)
+			return patterns
+		}
+	}
+	includedFiles = append(includedFiles, patternFile)
+	LOG_INFO("SNAPSHOT_FILTER", "Parsing filter file %s", patternFile)
+	patternFileContent, err := ioutil.ReadFile(patternFile)
+	if err == nil {
+		patternFileLines := strings.Split(string(patternFileContent), "\n")
+		patterns = ProcessFilterLines(patternFileLines, includedFiles)
+	}
+	return patterns
+}
+
+func ProcessFilterLines(patternFileLines []string, includedFiles []string) (patterns []string) {
+	for _, pattern := range patternFileLines {
+		pattern = strings.TrimSpace(pattern)
+		if len(pattern) == 0 {
+			continue
+		}
+
+		if strings.HasPrefix(pattern, "@") {
+			patternIncludeFile := strings.TrimSpace(pattern[1:])
+			if patternIncludeFile == "" {
+				continue
+			}
+			if ! filepath.IsAbs(patternIncludeFile) {
+				basePath := ""
+				if len(includedFiles) == 0 {
+					basePath, _ = os.Getwd()
+				} else {
+					basePath = filepath.Dir(includedFiles[len(includedFiles)-1])
+				}
+				patternIncludeFile = joinPath(basePath, patternIncludeFile)
+			}
+			for _, pattern := range ProcessFilterFile(patternIncludeFile, includedFiles) {
+				patterns = AppendPattern(patterns, pattern)
+			}
+			continue
+		}
+
+		if pattern[0] == '#' {
+			continue
+		}
+
+		if IsUnspecifiedFilter(pattern) {
+			pattern = "+" + pattern
+		}
+
+		if IsEmptyFilter(pattern) {
+			continue
+		}
+
+		if strings.HasPrefix(pattern, "i:") || strings.HasPrefix(pattern, "e:") {
+			valid, err := IsValidRegex(pattern[2:])
+			if !valid || err != nil {
+				LOG_ERROR("SNAPSHOT_FILTER", "Invalid regular expression encountered for filter: \"%s\", error: %v", pattern, err)
+			}
+		}
+
+		patterns = AppendPattern(patterns, pattern)
+	}
+
+	return patterns
+}
+
 // This is the struct used to save/load incomplete snapshots
 type IncompleteSnapshot struct {
 	Files        []*Entry

src/duplicacy_snapshotmanager.go

@@ -270,7 +270,7 @@ func (reader *sequenceReader) Read(data []byte) (n int, err error) {
 
 func (manager *SnapshotManager) CreateChunkDownloader() {
 	if manager.chunkDownloader == nil {
-		manager.chunkDownloader = CreateChunkDownloader(manager.config, manager.storage, manager.snapshotCache, false, 1)
+		manager.chunkDownloader = CreateChunkDownloader(manager.config, manager.storage, manager.snapshotCache, false, 1, false)
 	}
 }
 
@@ -381,6 +381,13 @@ func (manager *SnapshotManager) DownloadSnapshotContents(snapshot *Snapshot, pat
 	return true
 }
 
+// ClearSnapshotContents removes contents loaded by DownloadSnapshotContents
+func (manager *SnapshotManager) ClearSnapshotContents(snapshot *Snapshot) {
+	snapshot.ChunkHashes = nil
+	snapshot.ChunkLengths = nil
+	snapshot.Files = nil
+}
+
 // CleanSnapshotCache removes all files not referenced by the specified 'snapshot' in the snapshot cache.
 func (manager *SnapshotManager) CleanSnapshotCache(latestSnapshot *Snapshot, allSnapshots map[string][]*Snapshot) bool {
 
@@ -653,6 +660,51 @@ func (manager *SnapshotManager) GetSnapshotChunks(snapshot *Snapshot, keepChunkH
 	return chunks
 }
 
+// GetSnapshotChunkHashes has an option to retrieve chunk hashes in addition to chunk ids.
+func (manager *SnapshotManager) GetSnapshotChunkHashes(snapshot *Snapshot, chunkHashes *map[string]bool, chunkIDs map[string]bool) {
+
+	for _, chunkHash := range snapshot.FileSequence {
+		if chunkHashes != nil {
+			(*chunkHashes)[chunkHash] = true
+		}
+		chunkIDs[manager.config.GetChunkIDFromHash(chunkHash)] = true
+	}
+
+	for _, chunkHash := range snapshot.ChunkSequence {
+		if chunkHashes != nil {
+			(*chunkHashes)[chunkHash] = true
+		}
+		chunkIDs[manager.config.GetChunkIDFromHash(chunkHash)] = true
+	}
+
+	for _, chunkHash := range snapshot.LengthSequence {
+		if chunkHashes != nil {
+			(*chunkHashes)[chunkHash] = true
+		}
+		chunkIDs[manager.config.GetChunkIDFromHash(chunkHash)] = true
+	}
+
+	if len(snapshot.ChunkHashes) == 0 {
+
+		description := manager.DownloadSequence(snapshot.ChunkSequence)
+		err := snapshot.LoadChunks(description)
+		if err != nil {
+			LOG_ERROR("SNAPSHOT_CHUNK", "Failed to load chunks for snapshot %s at revision %d: %v",
+				snapshot.ID, snapshot.Revision, err)
+			return
+		}
+	}
+
+	for _, chunkHash := range snapshot.ChunkHashes {
+		if chunkHashes != nil {
+			(*chunkHashes)[chunkHash] = true
+		}
+		chunkIDs[manager.config.GetChunkIDFromHash(chunkHash)] = true
+	}
+
+	snapshot.ClearChunks()
+}
+
 // ListSnapshots shows the information about a snapshot.
 func (manager *SnapshotManager) ListSnapshots(snapshotID string, revisionsToList []int, tag string,
 	showFiles bool, showChunks bool) int {
@@ -757,10 +809,12 @@ func (manager *SnapshotManager) ListSnapshots(snapshotID string, revisionsToList
 
 // ListSnapshots shows the information about a snapshot.
 func (manager *SnapshotManager) CheckSnapshots(snapshotID string, revisionsToCheck []int, tag string, showStatistics bool, showTabular bool,
-	checkFiles bool, searchFossils bool, resurrect bool) bool {
+	checkFiles bool, checkChunks, searchFossils bool, resurrect bool, threads int, allowFailures bool) bool {
 
-	LOG_DEBUG("LIST_PARAMETERS", "id: %s, revisions: %v, tag: %s, showStatistics: %t, checkFiles: %t, searchFossils: %t, resurrect: %t",
-		snapshotID, revisionsToCheck, tag, showStatistics, checkFiles, searchFossils, resurrect)
+	manager.chunkDownloader = CreateChunkDownloader(manager.config, manager.storage, manager.snapshotCache, false, threads, allowFailures)
+
+	LOG_DEBUG("LIST_PARAMETERS", "id: %s, revisions: %v, tag: %s, showStatistics: %t, showTabular: %t, checkFiles: %t, searchFossils: %t, resurrect: %t",
+		snapshotID, revisionsToCheck, tag, showStatistics, showTabular, checkFiles, searchFossils, resurrect)
 
 	snapshotMap := make(map[string][]*Snapshot)
 	var err error
@@ -774,6 +828,8 @@ func (manager *SnapshotManager) CheckSnapshots(snapshotID string, revisionsToChe
 	// Store the index of the snapshot that references each chunk; if the chunk is shared by multiple chunks, the index is -1
 	chunkSnapshotMap := make(map[string]int)
 
+	emptyChunks := 0
+
 	LOG_INFO("SNAPSHOT_CHECK", "Listing all chunks")
 	allChunks, allSizes := manager.ListAllFiles(manager.storage, chunkDir)
 
@@ -788,9 +844,14 @@ func (manager *SnapshotManager) CheckSnapshots(snapshotID string, revisionsToChe
 
 		chunk = strings.Replace(chunk, "/", "", -1)
 		chunkSizeMap[chunk] = allSizes[i]
+
+		if allSizes[i] == 0 && !strings.HasSuffix(chunk, ".tmp") {
+			LOG_WARN("SNAPSHOT_CHECK", "Chunk %s has a size of 0", chunk)
+			emptyChunks++
+		}
 	}
 
-	if snapshotID == "" || showStatistics {
+	if snapshotID == "" || showStatistics || showTabular {
 		snapshotIDs, err := manager.ListSnapshotIDs()
 		if err != nil {
 			LOG_ERROR("SNAPSHOT_LIST", "Failed to list all snapshots: %v", err)
@@ -810,7 +871,7 @@ func (manager *SnapshotManager) CheckSnapshots(snapshotID string, revisionsToChe
 	for snapshotID = range snapshotMap {
 
 		revisions := revisionsToCheck
-		if len(revisions) == 0 || showStatistics {
+		if len(revisions) == 0 || showStatistics || showTabular {
 			revisions, err = manager.ListSnapshotRevisions(snapshotID)
 			if err != nil {
 				LOG_ERROR("SNAPSHOT_LIST", "Failed to list all revisions for snapshot %s: %v", snapshotID, err)
@@ -839,6 +900,12 @@ func (manager *SnapshotManager) CheckSnapshots(snapshotID string, revisionsToChe
 	}
 	LOG_INFO("SNAPSHOT_CHECK", "Total chunk size is %s in %d chunks", PrettyNumber(totalChunkSize), len(chunkSizeMap))
 
+	var allChunkHashes *map[string]bool
+	if checkChunks && !checkFiles {
+		m := make(map[string]bool)
+		allChunkHashes = &m
+	}
+
 	for snapshotID = range snapshotMap {
 
 		for _, snapshot := range snapshotMap[snapshotID] {
@@ -846,13 +913,12 @@ func (manager *SnapshotManager) CheckSnapshots(snapshotID string, revisionsToChe
 			if checkFiles {
 				manager.DownloadSnapshotContents(snapshot, nil, false)
 				manager.VerifySnapshot(snapshot)
+				manager.ClearSnapshotContents(snapshot)
 				continue
 			}
 
 			chunks := make(map[string]bool)
-			for _, chunkID := range manager.GetSnapshotChunks(snapshot, false) {
-				chunks[chunkID] = true
-			}
+			manager.GetSnapshotChunkHashes(snapshot, allChunkHashes, chunks)
 
 			missingChunks := 0
 			for chunkID := range chunks {
@@ -860,6 +926,20 @@ func (manager *SnapshotManager) CheckSnapshots(snapshotID string, revisionsToChe
 				_, found := chunkSizeMap[chunkID]
 
 				if !found {
+
+					// Look up the chunk again in case it actually exists, but only if there aren't
+					// too many missing chunks.
+					if missingChunks < 100 {
+						_, exist, _, err := manager.storage.FindChunk(0, chunkID, false)
+						if err != nil {
+							LOG_WARN("SNAPSHOT_VALIDATE", "Failed to check the existence of chunk %s: %v",
+							         chunkID, err)
+						} else if exist {
+							LOG_INFO("SNAPSHOT_VALIDATE", "Chunk %s is confirmed to exist", chunkID)
+							continue
+						}
+					}
+
 					if !searchFossils {
 						missingChunks += 1
 						LOG_WARN("SNAPSHOT_VALIDATE",
@@ -870,7 +950,7 @@ func (manager *SnapshotManager) CheckSnapshots(snapshotID string, revisionsToChe
 
 					chunkPath, exist, size, err := manager.storage.FindChunk(0, chunkID, true)
 					if err != nil {
-						LOG_ERROR("SNAPSHOT_VALIDATE", "Failed to check the existence of chunk %s: %v",
+						LOG_ERROR("SNAPSHOT_VALIDATE", "Failed to check the existence of fossil %s: %v",
 							chunkID, err)
 						return false
 					}
@@ -926,12 +1006,61 @@ func (manager *SnapshotManager) CheckSnapshots(snapshotID string, revisionsToChe
 		return false
 	}
 
+	if emptyChunks > 0 {
+		LOG_ERROR("SNAPSHOT_CHECK", "%d chunks have a size of 0", emptyChunks)
+		return false
+	}
+
 	if showTabular {
 		manager.ShowStatisticsTabular(snapshotMap, chunkSizeMap, chunkUniqueMap, chunkSnapshotMap)
 	} else if showStatistics {
 		manager.ShowStatistics(snapshotMap, chunkSizeMap, chunkUniqueMap, chunkSnapshotMap)
 	}
 
+	if checkChunks && !checkFiles {
+		manager.chunkDownloader.snapshotCache = nil
+		LOG_INFO("SNAPSHOT_VERIFY", "Verifying %d chunks", len(*allChunkHashes))
+
+		startTime := time.Now()
+		var chunkHashes []string
+
+		// The index of the first chunk to add to the downloader, which may have already downloaded
+		// some metadata chunks so the index doesn't start with 0.
+		chunkIndex := -1
+
+		for chunkHash := range *allChunkHashes {
+			chunkHashes = append(chunkHashes, chunkHash)
+			if chunkIndex == -1 {
+				chunkIndex = manager.chunkDownloader.AddChunk(chunkHash)
+			} else {
+				manager.chunkDownloader.AddChunk(chunkHash)
+			}
+		}
+
+		var downloadedChunkSize int64
+		totalChunks := len(*allChunkHashes)
+		for i := 0; i < totalChunks; i++ {
+			chunk := manager.chunkDownloader.WaitForChunk(i + chunkIndex)
+			if chunk.isBroken {
+				continue
+			}
+			downloadedChunkSize += int64(chunk.GetLength())
+
+			elapsedTime := time.Now().Sub(startTime).Seconds()
+			speed := int64(float64(downloadedChunkSize) / elapsedTime)
+			remainingTime := int64(float64(totalChunks - i - 1) / float64(i + 1) * elapsedTime)
+			percentage := float64(i + 1) / float64(totalChunks) * 100.0
+			LOG_INFO("VERIFY_PROGRESS", "Verified chunk %s (%d/%d), %sB/s %s %.1f%%",
+					 manager.config.GetChunkIDFromHash(chunkHashes[i]), i + 1, totalChunks,
+					 PrettySize(speed), PrettyTime(remainingTime), percentage)
+		}
+
+		if manager.chunkDownloader.NumberOfFailedChunks > 0 {
+			LOG_ERROR("SNAPSHOT_VERIFY", "%d out of %d chunks are corrupted", manager.chunkDownloader.NumberOfFailedChunks, totalChunks)
+		} else {
+			LOG_INFO("SNAPSHOT_VERIFY", "All %d chunks have been successfully verified", totalChunks)
+		}
+	}
 	return true
 }
 
@@ -998,18 +1127,20 @@ func (manager *SnapshotManager) ShowStatisticsTabular(snapshotMap map[string][]*
 		earliestSeenChunks := make(map[string]int)
 
 		for _, snapshot := range snapshotList {
-			for _, chunkID := range manager.GetSnapshotChunks(snapshot, true) {
+			for _, chunkID := range manager.GetSnapshotChunks(snapshot, false) {
 				if earliestSeenChunks[chunkID] == 0 {
 					earliestSeenChunks[chunkID] = math.MaxInt32
 				}
-				earliestSeenChunks[chunkID] = MinInt(earliestSeenChunks[chunkID], snapshot.Revision)
+				if earliestSeenChunks[chunkID] > snapshot.Revision {
+					earliestSeenChunks[chunkID] = snapshot.Revision
+				}
 			}
 		}
 
 		for _, snapshot := range snapshotList {
 
 			chunks := make(map[string]bool)
-			for _, chunkID := range manager.GetSnapshotChunks(snapshot, true) {
+			for _, chunkID := range manager.GetSnapshotChunks(snapshot, false) {
 				chunks[chunkID] = true
 				snapshotChunks[chunkID] = true
 			}
@@ -1178,7 +1309,6 @@ func (manager *SnapshotManager) RetrieveFile(snapshot *Snapshot, file *Entry, ou
 	}
 
 	var chunk *Chunk
-	currentHash := ""
 
 	for i := file.StartChunk; i <= file.EndChunk; i++ {
 		start := 0
@@ -1191,10 +1321,12 @@ func (manager *SnapshotManager) RetrieveFile(snapshot *Snapshot, file *Entry, ou
 		}
 
 		hash := snapshot.ChunkHashes[i]
-		if currentHash != hash {
+		lastChunk, lastChunkHash := manager.chunkDownloader.GetLastDownloadedChunk()
+		if lastChunkHash != hash {
 			i := manager.chunkDownloader.AddChunk(hash)
 			chunk = manager.chunkDownloader.WaitForChunk(i)
-			currentHash = hash
+		} else {
+			chunk = lastChunk
 		}
 
 		output(chunk.GetBytes()[start:end])
@@ -1269,21 +1401,20 @@ func (manager *SnapshotManager) PrintFile(snapshotID string, revision int, path
 	}
 
 	file := manager.FindFile(snapshot, path, false)
-	var content []byte
-	if !manager.RetrieveFile(snapshot, file, func(chunk []byte) { content = append(content, chunk...) }) {
+	if !manager.RetrieveFile(snapshot, file, func(chunk []byte) {
+			fmt.Printf("%s", chunk)
+		}) {
 		LOG_ERROR("SNAPSHOT_RETRIEVE", "File %s is corrupted in snapshot %s at revision %d",
 			path, snapshot.ID, snapshot.Revision)
 		return false
 	}
 
-	fmt.Printf("%s", string(content))
-
 	return true
 }
 
 // Diff compares two snapshots, or two revision of a file if the file argument is given.
 func (manager *SnapshotManager) Diff(top string, snapshotID string, revisions []int,
-	filePath string, compareByHash bool, nobackupFile string) bool {
+	filePath string, compareByHash bool, nobackupFile string, filtersFile string, excludeByAttribute bool) bool {
 
 	LOG_DEBUG("DIFF_PARAMETERS", "top: %s, id: %s, revision: %v, path: %s, compareByHash: %t",
 		top, snapshotID, revisions, filePath, compareByHash)
@@ -1296,7 +1427,7 @@ func (manager *SnapshotManager) Diff(top string, snapshotID string, revisions []
 	if len(revisions) <= 1 {
 		// Only scan the repository if filePath is not provided
 		if len(filePath) == 0 {
-			rightSnapshot, _, _, err = CreateSnapshotFromDirectory(snapshotID, top, nobackupFile)
+			rightSnapshot, _, _, err = CreateSnapshotFromDirectory(snapshotID, top, nobackupFile, filtersFile, excludeByAttribute)
 			if err != nil {
 				LOG_ERROR("SNAPSHOT_LIST", "Failed to list the directory %s: %v", top, err)
 				return false
@@ -1467,7 +1598,11 @@ func (manager *SnapshotManager) Diff(top string, snapshotID string, revisions []
 						same = right.IsSameAs(left)
 					}
 				} else {
-					same = left.Hash == right.Hash
+					if left.Size == 0 && right.Size == 0 {
+						same = true
+					} else {
+						same = left.Hash == right.Hash
+					}
 				}
 
 				if !same {
@@ -1838,7 +1973,7 @@ func (manager *SnapshotManager) PruneSnapshots(selfID string, snapshotID string,
 				if _, found := newChunks[chunk]; found {
 					// The fossil is referenced so it can't be deleted.
 					if dryRun {
-						LOG_INFO("FOSSIL_RESURRECT", "Fossil %s would be resurrected: %v", chunk)
+						LOG_INFO("FOSSIL_RESURRECT", "Fossil %s would be resurrected", chunk)
 						continue
 					}
 
@@ -2446,7 +2581,7 @@ func (manager *SnapshotManager) UploadFile(path string, derivationKey string, co
 		derivationKey = derivationKey[len(derivationKey)-64:]
 	}
 
-	err := manager.fileChunk.Encrypt(manager.config.FileKey, derivationKey)
+	err := manager.fileChunk.Encrypt(manager.config.FileKey, derivationKey, true)
 	if err != nil {
 		LOG_ERROR("UPLOAD_File", "Failed to encrypt the file %s: %v", path, err)
 		return false

src/duplicacy_snapshotmanager_test.go

@@ -620,7 +620,7 @@ func TestPruneNewSnapshots(t *testing.T) {
 	// Now chunkHash1 wil be resurrected
 	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false, 1)
 	checkTestSnapshots(snapshotManager, 4, 0)
-	snapshotManager.CheckSnapshots("vm1@host1", []int{2, 3}, "", false, false, false, false, false)
+	snapshotManager.CheckSnapshots("vm1@host1", []int{2, 3}, "", false, false, false, false, false, false, 1, false)
 }
 
 // A fossil collection left by an aborted prune should be ignored if any supposedly deleted snapshot exists
@@ -669,7 +669,7 @@ func TestPruneGhostSnapshots(t *testing.T) {
 	// Run the prune again but the fossil collection should be igored, since revision 1 still exists
 	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false, 1)
 	checkTestSnapshots(snapshotManager, 3, 2)
-	snapshotManager.CheckSnapshots("vm1@host1", []int{1, 2, 3}, "", false, false, false, true /*searchFossils*/, false)
+	snapshotManager.CheckSnapshots("vm1@host1", []int{1, 2, 3}, "", false, false, false, false, true /*searchFossils*/, false, 1, false)
 
 	// Prune snapshot 1 again
 	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{1}, []string{}, []string{}, false, false, []string{}, false, false, false, 1)
@@ -683,5 +683,5 @@ func TestPruneGhostSnapshots(t *testing.T) {
 	// Run the prune again and this time the fossil collection will be processed and the fossils removed
 	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false, 1)
 	checkTestSnapshots(snapshotManager, 3, 0)
-	snapshotManager.CheckSnapshots("vm1@host1", []int{2, 3, 4}, "", false, false, false, false, false)
+	snapshotManager.CheckSnapshots("vm1@host1", []int{2, 3, 4}, "", false, false, false, false, false, false, 1, false)
 }

src/duplicacy_storage.go

@@ -268,7 +268,7 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
 	if matched == nil {
 		LOG_ERROR("STORAGE_CREATE", "Unrecognizable storage URL: %s", storageURL)
 		return nil
-	} else if matched[1] == "sftp" {
+	} else if matched[1] == "sftp" || matched[1] == "sftpc" {
 		server := matched[3]
 		username := matched[2]
 		storageDir := matched[5]
@@ -336,7 +336,7 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
 			keyFile = GetPassword(preference, "ssh_key_file", "Enter the path of the private key file:",
 				true, resetPassword)
 
-			var key ssh.Signer
+			var keySigner ssh.Signer
 			var err error
 
 			if keyFile == "" {
@@ -347,15 +347,15 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
 				if err != nil {
 					LOG_INFO("SSH_PUBLICKEY", "Failed to read the private key file: %v", err)
 				} else {
-					key, err = ssh.ParsePrivateKey(content)
+					keySigner, err = ssh.ParsePrivateKey(content)
 					if err != nil {
-						if strings.Contains(err.Error(), "cannot decode encrypted private keys") {
+						if _, ok := err.(*ssh.PassphraseMissingError); ok {
 							LOG_TRACE("SSH_PUBLICKEY", "The private key file is encrypted")
 							passphrase = GetPassword(preference, "ssh_passphrase", "Enter the passphrase to decrypt the private key file:", false, resetPassword)
 							if len(passphrase) == 0 {
 								LOG_INFO("SSH_PUBLICKEY", "No passphrase to descrypt the private key file %s", keyFile)
 							} else {
-								key, err = ssh.ParsePrivateKeyWithPassphrase(content, []byte(passphrase))
+								keySigner, err = ssh.ParsePrivateKeyWithPassphrase(content, []byte(passphrase))
 								if err != nil {
 									LOG_INFO("SSH_PUBLICKEY", "Failed to parse the encrypted private key file %s: %v", keyFile, err)
 								}
@@ -364,11 +364,35 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
 							LOG_INFO("SSH_PUBLICKEY", "Failed to parse the private key file %s: %v", keyFile, err)
 						}
 					}
+
+					if keySigner != nil {
+						certFile := keyFile + "-cert.pub"
+						if stat, err := os.Stat(certFile); err == nil && !stat.IsDir() {
+							LOG_DEBUG("SSH_CERTIFICATE", "Attempting to use ssh certificate from file %s", certFile)
+							var content []byte
+							content, err = ioutil.ReadFile(certFile)
+							if err != nil {
+								LOG_INFO("SSH_CERTIFICATE", "Failed to read ssh certificate file %s: %v", certFile, err)
+							} else {
+								pubKey, _, _, _, err := ssh.ParseAuthorizedKey(content)
+								if err != nil {
+									LOG_INFO("SSH_CERTIFICATE", "Failed parse ssh certificate file %s: %v", certFile, err)
+								} else {
+									certSigner, err := ssh.NewCertSigner(pubKey.(*ssh.Certificate), keySigner)
+									if err != nil {
+										LOG_INFO("SSH_CERTIFICATE", "Failed to create certificate signer: %v", err)
+									} else {
+										keySigner = certSigner
+									}
+								}
+							}
+						}
+					}
 				}
 			}
 
-			if key != nil {
-				signers = append(signers, key)
+			if keySigner != nil {
+				signers = append(signers, keySigner)
 			}
 
 			if len(signers) > 0 {
@@ -416,7 +440,7 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
 			return checkHostKey(hostname, remote, key)
 		}
 
-		sftpStorage, err := CreateSFTPStorage(server, port, username, storageDir, 2, authMethods, hostKeyChecker, threads)
+		sftpStorage, err := CreateSFTPStorage(matched[1] == "sftpc", server, port, username, storageDir, 2, authMethods, hostKeyChecker, threads)
 		if err != nil {
 			LOG_ERROR("STORAGE_CREATE", "Failed to load the SFTP storage at %s: %v", storageURL, err)
 			return nil
@@ -526,11 +550,30 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
 		return dropboxStorage
 	} else if matched[1] == "b2" {
 		bucket := matched[3]
+		storageDir := matched[5]
 
-		accountID := GetPassword(preference, "b2_id", "Enter Backblaze Account ID:", true, resetPassword)
-		applicationKey := GetPassword(preference, "b2_key", "Enter Backblaze Application Key:", true, resetPassword)
+		accountID := GetPassword(preference, "b2_id", "Enter Backblaze account or application id:", true, resetPassword)
+		applicationKey := GetPassword(preference, "b2_key", "Enter corresponding Backblaze application key:", true, resetPassword)
 
-		b2Storage, err := CreateB2Storage(accountID, applicationKey, bucket, threads)
+		b2Storage, err := CreateB2Storage(accountID, applicationKey, "", bucket, storageDir, threads)
+		if err != nil {
+			LOG_ERROR("STORAGE_CREATE", "Failed to load the Backblaze B2 storage at %s: %v", storageURL, err)
+			return nil
+		}
+		SavePassword(preference, "b2_id", accountID)
+		SavePassword(preference, "b2_key", applicationKey)
+		return b2Storage
+	} else if matched[1] == "b2-custom" {
+		b2customUrlRegex := regexp.MustCompile(`^b2-custom://([^/]+)/([^/]+)(/(.+))?`)
+		matched := b2customUrlRegex.FindStringSubmatch(storageURL)
+		downloadURL := "https://" + matched[1]
+		bucket := matched[2]
+		storageDir := matched[4]
+
+		accountID := GetPassword(preference, "b2_id", "Enter Backblaze account or application id:", true, resetPassword)
+		applicationKey := GetPassword(preference, "b2_key", "Enter corresponding Backblaze application key:", true, resetPassword)
+
+		b2Storage, err := CreateB2Storage(accountID, applicationKey, downloadURL, bucket, storageDir, threads)
 		if err != nil {
 			LOG_ERROR("STORAGE_CREATE", "Failed to load the Backblaze B2 storage at %s: %v", storageURL, err)
 			return nil
@@ -581,26 +624,35 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
 		SavePassword(preference, "gcs_token", tokenFile)
 		return gcsStorage
 	} else if matched[1] == "gcd" {
+		// Handle writing directly to the root of the drive
+		// For gcd://driveid@/, driveid@ is match[3] not match[2]
+		if matched[2] == "" && strings.HasSuffix(matched[3], "@") {
+			matched[2], matched[3]  = matched[3], matched[2]
+		}
+		driveID := matched[2]
+		if driveID != "" {
+			driveID = driveID[:len(driveID)-1]
+		}
 		storagePath := matched[3] + matched[4]
 		prompt := fmt.Sprintf("Enter the path of the Google Drive token file (downloadable from https://duplicacy.com/gcd_start):")
 		tokenFile := GetPassword(preference, "gcd_token", prompt, true, resetPassword)
-		gcdStorage, err := CreateGCDStorage(tokenFile, storagePath, threads)
+		gcdStorage, err := CreateGCDStorage(tokenFile, driveID, storagePath, threads)
 		if err != nil {
 			LOG_ERROR("STORAGE_CREATE", "Failed to load the Google Drive storage at %s: %v", storageURL, err)
 			return nil
 		}
 		SavePassword(preference, "gcd_token", tokenFile)
 		return gcdStorage
-	} else if matched[1] == "one" {
+	} else if matched[1] == "one" || matched[1] == "odb" {
 		storagePath := matched[3] + matched[4]
 		prompt := fmt.Sprintf("Enter the path of the OneDrive token file (downloadable from https://duplicacy.com/one_start):")
-		tokenFile := GetPassword(preference, "one_token", prompt, true, resetPassword)
-		oneDriveStorage, err := CreateOneDriveStorage(tokenFile, storagePath, threads)
+		tokenFile := GetPassword(preference, matched[1] + "_token", prompt, true, resetPassword)
+		oneDriveStorage, err := CreateOneDriveStorage(tokenFile, matched[1] == "odb", storagePath, threads)
 		if err != nil {
 			LOG_ERROR("STORAGE_CREATE", "Failed to load the OneDrive storage at %s: %v", storageURL, err)
 			return nil
 		}
-		SavePassword(preference, "one_token", tokenFile)
+		SavePassword(preference, matched[1] + "_token", tokenFile)
 		return oneDriveStorage
 	} else if matched[1] == "hubic" {
 		storagePath := matched[3] + matched[4]
@@ -626,6 +678,10 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
 	} else if matched[1] == "webdav" || matched[1] == "webdav-http" {
 		server := matched[3]
 		username := matched[2]
+		if username == "" {
+			LOG_ERROR("STORAGE_CREATE", "No username is provided to access the WebDAV storage")
+			return nil
+		}
 		username = username[:len(username)-1]
 		storageDir := matched[5]
 		port := 0
@@ -646,6 +702,18 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
 		}
 		SavePassword(preference, "webdav_password", password)
 		return webDAVStorage
+	} else if matched[1] == "fabric" {
+		endpoint := matched[3]
+		storageDir := matched[5]
+		prompt := fmt.Sprintf("Enter the token for accessing the Storage Made Easy File Fabric storage:")
+		token := GetPassword(preference, "fabric_token", prompt, true, resetPassword)
+		smeStorage, err := CreateFileFabricStorage(endpoint, token, storageDir, threads)
+		if err != nil {
+			LOG_ERROR("STORAGE_CREATE", "Failed to load the File Fabric storage at %s: %v", storageURL, err)
+			return nil
+		}
+		SavePassword(preference, "fabric_token", token)
+		return smeStorage
 	} else {
 		LOG_ERROR("STORAGE_CREATE", "The storage type '%s' is not supported", matched[1])
 		return nil

src/duplicacy_storage_test.go

@@ -27,6 +27,8 @@ var testRateLimit int
 var testQuickMode bool
 var testThreads int
 var testFixedChunkSize bool
+var testRSAEncryption bool
+var testErasureCoding bool
 
 func init() {
 	flag.StringVar(&testStorageName, "storage", "", "the test storage to use")
@@ -34,6 +36,8 @@ func init() {
 	flag.BoolVar(&testQuickMode, "quick", false, "quick test")
 	flag.IntVar(&testThreads, "threads", 1, "number of downloading/uploading threads")
 	flag.BoolVar(&testFixedChunkSize, "fixed-chunk-size", false, "fixed chunk size")
+	flag.BoolVar(&testRSAEncryption, "rsa", false, "enable RSA encryption")
+	flag.BoolVar(&testErasureCoding, "erasure-coding", false, "enable Erasure Coding")
 	flag.Parse()
 }
 
@@ -107,7 +111,7 @@ func loadStorage(localStoragePath string, threads int) (Storage, error) {
 		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
 		return storage, err
 	} else if testStorageName == "b2" {
-		storage, err := CreateB2Storage(config["account"], config["key"], config["bucket"], threads)
+		storage, err := CreateB2Storage(config["account"], config["key"], "", config["bucket"], config["directory"], threads)
 		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
 		return storage, err
 	} else if testStorageName == "gcs-s3" {
@@ -131,11 +135,23 @@ func loadStorage(localStoragePath string, threads int) (Storage, error) {
 		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
 		return storage, err
 	} else if testStorageName == "gcd" {
-		storage, err := CreateGCDStorage(config["token_file"], config["storage_path"], threads)
+		storage, err := CreateGCDStorage(config["token_file"], "", config["storage_path"], threads)
+		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
+		return storage, err
+	} else if testStorageName == "gcd-shared" {
+		storage, err := CreateGCDStorage(config["token_file"], config["drive"], config["storage_path"], threads)
 		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
 		return storage, err
 	} else if testStorageName == "one" {
-		storage, err := CreateOneDriveStorage(config["token_file"], config["storage_path"], threads)
+		storage, err := CreateOneDriveStorage(config["token_file"], false, config["storage_path"], threads)
+		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
+		return storage, err
+	} else if testStorageName == "odb" {
+		storage, err := CreateOneDriveStorage(config["token_file"], true, config["storage_path"], threads)
+		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
+		return storage, err
+	} else if testStorageName == "one" {
+		storage, err := CreateOneDriveStorage(config["token_file"], false, config["storage_path"], threads)
 		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
 		return storage, err
 	} else if testStorageName == "hubic" {
@@ -296,7 +312,8 @@ func TestStorage(t *testing.T) {
 
 	LOG_INFO("STORAGE_TEST", "storage: %s", testStorageName)
 
-	storage, err := loadStorage(testDir, 1)
+	threads := 8
+	storage, err := loadStorage(testDir, threads)
 	if err != nil {
 		t.Errorf("Failed to create storage: %v", err)
 		return
@@ -326,16 +343,16 @@ func TestStorage(t *testing.T) {
 	storage.CreateDirectory(0, "shared")
 
 	// Upload to the same directory by multiple goroutines
-	count := 8
+	count := threads
 	finished := make(chan int, count)
 	for i := 0; i < count; i++ {
-		go func(name string) {
-			err := storage.UploadFile(0, name, []byte("this is a test file"))
+		go func(threadIndex int, name string) {
+			err := storage.UploadFile(threadIndex, name, []byte("this is a test file"))
 			if err != nil {
 				t.Errorf("Error to upload '%s': %v", name, err)
 			}
 			finished <- 0
-		}(fmt.Sprintf("shared/a/b/c/%d", i))
+		}(i, fmt.Sprintf("shared/a/b/c/%d", i))
 	}
 
 	for i := 0; i < count; i++ {
@@ -384,7 +401,6 @@ func TestStorage(t *testing.T) {
 
 	snapshotIDs := []string{}
 	for _, snapshotDir := range snapshotDirs {
-		LOG_INFO("debug", "snapshot dir: %s", snapshotDir)
 		if len(snapshotDir) > 0 && snapshotDir[len(snapshotDir)-1] == '/' {
 			snapshotIDs = append(snapshotIDs, snapshotDir[:len(snapshotDir)-1])
 		}

src/duplicacy_swiftstorage.go

@@ -129,6 +129,11 @@ func CreateSwiftStorage(storageURL string, key string, threads int) (storage *Sw
 		TrustId:        arguments["trust_id"],
 	}
 
+	err = connection.Authenticate()
+	if err != nil {
+		return nil, err
+	}
+
 	_, _, err = connection.Container(container)
 	if err != nil {
 		return nil, err

src/duplicacy_utils.go

@@ -55,7 +55,7 @@ func IsEmptyFilter(pattern string) bool {
 }
 
 func IsUnspecifiedFilter(pattern string) bool {
-	if pattern[0] != '+' && pattern[0] != '-' && pattern[0] != 'i' && pattern[0] != 'e' {
+	if pattern[0] != '+' && pattern[0] != '-' && !strings.HasPrefix(pattern, "i:") && !strings.HasPrefix(pattern, "e:")  {
 		return true
 	} else {
 		return false
@@ -434,7 +434,7 @@ func PrettyTime(seconds int64) string {
 			seconds/day, (seconds%day)/3600, (seconds%3600)/60, seconds%60)
 	} else if seconds > day {
 		return fmt.Sprintf("1 day %02d:%02d:%02d", (seconds%day)/3600, (seconds%3600)/60, seconds%60)
-	} else if seconds > 0 {
+	} else if seconds >= 0 {
 		return fmt.Sprintf("%02d:%02d:%02d", seconds/3600, (seconds%3600)/60, seconds%60)
 	} else {
 		return "n/a"
@@ -460,10 +460,3 @@ func AtoSize(sizeString string) int {
 
 	return size
 }
-
-func MinInt(x, y int) int {
-	if x < y {
-		return x
-	}
-	return y
-}

src/duplicacy_utils_darwin.go

@@ -0,0 +1,14 @@
+// Copyright (c) Acrosync LLC. All rights reserved.
+// Free for personal use and commercial trial
+// Commercial use requires per-user licenses available from https://duplicacy.com
+
+package duplicacy
+
+import (
+	"strings"
+)
+
+func excludedByAttribute(attirbutes map[string][]byte) bool {
+	value, ok := attirbutes["com.apple.metadata:com_apple_backup_excludeItem"]
+	return ok && strings.Contains(string(value), "com.apple.backupd")
+}

src/duplicacy_utils_freebsd.go

@@ -0,0 +1,13 @@
+// Copyright (c) Acrosync LLC. All rights reserved.
+// Free for personal use and commercial trial
+// Commercial use requires per-user licenses available from https://duplicacy.com
+
+package duplicacy
+
+import (
+)
+
+func excludedByAttribute(attirbutes map[string][]byte) bool {
+	_, ok := attirbutes["duplicacy_exclude"]
+	return ok
+}

src/duplicacy_utils_linux.go

@@ -0,0 +1,13 @@
+// Copyright (c) Acrosync LLC. All rights reserved.
+// Free for personal use and commercial trial
+// Commercial use requires per-user licenses available from https://duplicacy.com
+
+package duplicacy
+
+import (
+)
+
+func excludedByAttribute(attirbutes map[string][]byte) bool {
+	_, ok := attirbutes["duplicacy_exclude"]
+	return ok
+}

src/duplicacy_utils_others.go

@@ -88,3 +88,7 @@ func (entry *Entry) SetAttributesToFile(fullPath string) {
 func joinPath(components ...string) string {
 	return path.Join(components...)
 }
+
+func SplitDir(fullPath string) (dir string, file string) {
+	return path.Split(fullPath)
+}

src/duplicacy_utils_test.go

@@ -92,6 +92,17 @@ func TestMatchPattern(t *testing.T) {
 		}
 	}
 
+	for _, pattern := range []string{ "+", "-", "i:", "e:", "+a", "-a", "i:a", "e:a"} {
+		if IsUnspecifiedFilter(pattern) {
+			t.Errorf("pattern %s has a specified filter", pattern)
+		}
+	}
+
+	for _, pattern := range []string{ "i", "e", "ia", "ib", "a", "b"} {
+		if !IsUnspecifiedFilter(pattern) {
+			t.Errorf("pattern %s does not have a specified filter", pattern)
+		}
+	}
 }
 
 func TestRateLimit(t *testing.T) {

src/duplicacy_utils_windows.go

@@ -126,3 +126,12 @@ func joinPath(components ...string) string {
 	}
 	return combinedPath
 }
+
+func SplitDir(fullPath string) (dir string, file string) {
+	i := strings.LastIndex(fullPath, "\\")
+	return fullPath[:i+1], fullPath[i+1:]
+}
+
+func excludedByAttribute(attirbutes map[string][]byte) bool {
+	return false
+}

src/duplicacy_wasabistorage.go

@@ -93,49 +93,49 @@ func (storage *WasabiStorage) DeleteFile(
 // rename.  It's designed to get the job done with as few dependencies
 // on other packages as possible rather than being somethng
 // general-purpose and reusable.
-func (storage *WasabiStorage) MoveFile(
-	threadIndex int, from string, to string,
-) (err error) {
+func (storage *WasabiStorage) MoveFile(threadIndex int, from string, to string) (err error) {
 
-	var from_path string
+	var fromPath string
 	// The from path includes the bucket.  Take care not to include an empty storageDir
 	// string as Wasabi's backend will return 404 on URLs with double slashes.
 	if storage.storageDir == "" {
-		from_path = fmt.Sprintf("/%s/%s", storage.bucket, from)
+		fromPath = fmt.Sprintf("/%s/%s", storage.bucket, from)
 	} else {
-		from_path = fmt.Sprintf("/%s/%s/%s", storage.bucket, storage.storageDir, from)
+		fromPath = fmt.Sprintf("/%s/%s/%s", storage.bucket, storage.storageDir, from)
 	}
 
-	object := fmt.Sprintf("https://%s@%s%s",
-		storage.region, storage.endpoint, from_path)
+	object := fmt.Sprintf("https://%s@%s%s", storage.region, storage.endpoint, fromPath)
 
+	toPath := to
 	// The object's new name is relative to the top of the bucket.
-	new_name := fmt.Sprintf("%s/%s", storage.storageDir, to)
+	if storage.storageDir != "" {
+		toPath = fmt.Sprintf("%s/%s", storage.storageDir, to)
+	}
 
 	timestamp := time.Now().Format(time.RFC1123Z)
 
-	signing_string := fmt.Sprintf("MOVE\n\n\n%s\n%s", timestamp, from_path)
+	signingString := fmt.Sprintf("MOVE\n\n\n%s\n%s", timestamp, fromPath)
 
 	signer := hmac.New(sha1.New, []byte(storage.secret))
-	signer.Write([]byte(signing_string))
+	signer.Write([]byte(signingString))
 
 	signature := base64.StdEncoding.EncodeToString(signer.Sum(nil))
 
 	authorization := fmt.Sprintf("AWS %s:%s", storage.key, signature)
 
-	request, error := http.NewRequest("MOVE", object, nil)
-	if error != nil {
-		return error
+	request, err := http.NewRequest("MOVE", object, nil)
+	if err != nil {
+		return err
 	}
 	request.Header.Add("Authorization", authorization)
 	request.Header.Add("Date", timestamp)
-	request.Header.Add("Destination", new_name)
+	request.Header.Add("Destination", toPath)
 	request.Header.Add("Host", storage.endpoint)
 	request.Header.Add("Overwrite", "true")
 
-	response, error := storage.client.Do(request)
-	if error != nil {
-		return error
+	response, err := storage.client.Do(request)
+	if err != nil {
+		return err
 	}
 	defer response.Body.Close()
 

src/duplicacy_webdavstorage.go

@@ -14,7 +14,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"math/rand"
 	"net/http"
 	//"net/http/httputil"
@@ -22,6 +21,7 @@ import (
 	"strings"
 	"sync"
 	"time"
+	"io/ioutil"
 )
 
 type WebDAVStorage struct {
@@ -49,7 +49,7 @@ var (
 )
 
 func CreateWebDAVStorage(host string, port int, username string, password string, storageDir string, useHTTP bool, threads int) (storage *WebDAVStorage, err error) {
-	if storageDir[len(storageDir)-1] != '/' {
+	if len(storageDir) > 0 && storageDir[len(storageDir)-1] != '/' {
 		storageDir += "/"
 	}
 
@@ -59,7 +59,7 @@ func CreateWebDAVStorage(host string, port int, username string, password string
 		username:   username,
 		password:   password,
 		storageDir: "",
-		useHTTP:    false,
+		useHTTP:    useHTTP,
 
 		client:         http.DefaultClient,
 		threads:        threads,
@@ -128,7 +128,12 @@ func (storage *WebDAVStorage) sendRequest(method string, uri string, depth int,
 			dataReader = bytes.NewReader(data)
 		} else if method == "PUT" {
 			headers["Content-Type"] = "application/octet-stream"
-			dataReader = CreateRateLimitedReader(data, storage.UploadRateLimit/storage.threads)
+			headers["Content-Length"] = fmt.Sprintf("%d", len(data))
+			if storage.UploadRateLimit <= 0 {
+				dataReader = bytes.NewReader(data)
+			} else {
+				dataReader = CreateRateLimitedReader(data, storage.UploadRateLimit/storage.threads)
+			}
 		} else if method == "MOVE" {
 			headers["Destination"] = storage.createConnectionString(string(data))
 			headers["Content-Type"] = "application/octet-stream"
@@ -151,12 +156,16 @@ func (storage *WebDAVStorage) sendRequest(method string, uri string, depth int,
 			request.Header.Set(key, value)
 		}
 
+		if method == "PUT" {
+			request.ContentLength = int64(len(data))
+		}
+
 		//requestDump, err := httputil.DumpRequest(request, true)
 		//LOG_INFO("debug", "Request: %s", requestDump)
 
 		response, err := storage.client.Do(request)
 		if err != nil {
-			LOG_TRACE("WEBDAV_RETRY", "URL request '%s %s' returned an error (%v)", method, uri, err)
+			LOG_TRACE("WEBDAV_ERROR", "URL request '%s %s' returned an error (%v)", method, uri, err)
 			backoff = storage.retry(backoff)
 			continue
 		}
@@ -165,11 +174,13 @@ func (storage *WebDAVStorage) sendRequest(method string, uri string, depth int,
 			return response.Body, response.Header, nil
 		}
 
+		io.Copy(ioutil.Discard, response.Body)
+		response.Body.Close()
+
 		if response.StatusCode == 301 {
 			return nil, nil, errWebDAVMovedPermanently
 		}
 
-		response.Body.Close()
 		if response.StatusCode == 404 {
 			// Retry if it is UPLOAD, otherwise return immediately
 			if method != "PUT" {
@@ -210,53 +221,57 @@ type WebDAVMultiStatus struct {
 
 func (storage *WebDAVStorage) getProperties(uri string, depth int, properties ...string) (map[string]WebDAVProperties, error) {
 
-	propfind := "<prop>"
-	for _, p := range properties {
-		propfind += fmt.Sprintf("<%s/>", p)
-	}
-	propfind += "</prop>"
+	maxTries := 3
+	for tries := 0; ; tries++ {
+		propfind := "<prop>"
+		for _, p := range properties {
+			propfind += fmt.Sprintf("<%s/>", p)
+		}
+		propfind += "</prop>"
 
-	body := fmt.Sprintf(`<?xml version="1.0" encoding="utf-8" ?><propfind xmlns="DAV:">%s</propfind>`, propfind)
+		body := fmt.Sprintf(`<?xml version="1.0" encoding="utf-8" ?><propfind xmlns="DAV:">%s</propfind>`, propfind)
 
-	readCloser, _, err := storage.sendRequest("PROPFIND", uri, depth, []byte(body))
-	if err != nil {
-		return nil, err
-	}
-	defer readCloser.Close()
-	content, err := ioutil.ReadAll(readCloser)
-	if err != nil {
-		return nil, err
-	}
+		readCloser, _, err := storage.sendRequest("PROPFIND", uri, depth, []byte(body))
+		if err != nil {
+			return nil, err
+		}
+		defer readCloser.Close()
+		defer io.Copy(ioutil.Discard, readCloser)
 
-	object := WebDAVMultiStatus{}
-	err = xml.Unmarshal(content, &object)
-	if err != nil {
-		return nil, err
-	}
-
-	if object.Responses == nil || len(object.Responses) == 0 {
-		return nil, errors.New("no WebDAV responses")
-	}
-
-	responses := make(map[string]WebDAVProperties)
-
-	for _, responseTag := range object.Responses {
-
-		if responseTag.PropStat == nil || responseTag.PropStat.Prop == nil || responseTag.PropStat.Prop.PropList == nil {
-			return nil, errors.New("no WebDAV properties")
+		object := WebDAVMultiStatus{}
+		err = xml.NewDecoder(readCloser).Decode(&object)
+		if err != nil {
+			if strings.Contains(err.Error(), "unexpected EOF") && tries < maxTries {
+				LOG_WARN("WEBDAV_RETRY", "Retrying on %v", err)
+				continue
+			}
+			return nil, err
 		}
 
-		properties := make(WebDAVProperties)
-		for _, prop := range responseTag.PropStat.Prop.PropList {
-			properties[prop.XMLName.Local] = prop.Value
+		if object.Responses == nil || len(object.Responses) == 0 {
+			return nil, errors.New("no WebDAV responses")
 		}
 
-		responseKey := responseTag.Href
-		responses[responseKey] = properties
+		responses := make(map[string]WebDAVProperties)
 
+		for _, responseTag := range object.Responses {
+
+			if responseTag.PropStat == nil || responseTag.PropStat.Prop == nil || responseTag.PropStat.Prop.PropList == nil {
+				return nil, errors.New("no WebDAV properties")
+			}
+
+			properties := make(WebDAVProperties)
+			for _, prop := range responseTag.PropStat.Prop.PropList {
+				properties[prop.XMLName.Local] = prop.Value
+			}
+
+			responseKey := responseTag.Href
+			responses[responseKey] = properties
+
+		}
+
+		return responses, nil
 	}
-
-	return responses, nil
 }
 
 // ListFiles return the list of files and subdirectories under 'dir'.  A subdirectories returned must have a trailing '/', with
@@ -305,6 +320,12 @@ func (storage *WebDAVStorage) ListFiles(threadIndex int, dir string) (files []st
 			}
 			files = append(files, file)
 			sizes = append(sizes, int64(0))
+
+			// Add the directory to the directory cache
+			storage.directoryCacheLock.Lock()
+			storage.directoryCache[dir + file] = 1
+			storage.directoryCacheLock.Unlock()
+
 		}
 	}
 
@@ -313,6 +334,7 @@ func (storage *WebDAVStorage) ListFiles(threadIndex int, dir string) (files []st
 
 // GetFileInfo returns the information about the file or directory at 'filePath'.
 func (storage *WebDAVStorage) GetFileInfo(threadIndex int, filePath string) (exist bool, isDir bool, size int64, err error) {
+
 	properties, err := storage.getProperties(filePath, 0, "getcontentlength", "resourcetype")
 	if err != nil {
 		if err == errWebDAVNotExist {
@@ -325,7 +347,14 @@ func (storage *WebDAVStorage) GetFileInfo(threadIndex int, filePath string) (exi
 		return false, false, 0, err
 	}
 
-	if m, exist := properties["/"+storage.storageDir+filePath]; !exist {
+	m, exist := properties["/"+storage.storageDir+filePath]
+
+	// If no properties exist for the given filePath, remove the trailing / from filePath and search again
+	if !exist && filePath != "" && filePath[len(filePath) - 1] == '/' {
+		m, exist = properties["/"+storage.storageDir+filePath[:len(filePath) - 1]]
+	}
+
+	if !exist {
 		return false, false, 0, nil
 	} else if resourceType, exist := m["resourcetype"]; exist && strings.Contains(resourceType, "collection") {
 		return true, true, 0, nil
@@ -343,6 +372,7 @@ func (storage *WebDAVStorage) DeleteFile(threadIndex int, filePath string) (err
 	if err != nil {
 		return err
 	}
+	io.Copy(ioutil.Discard, readCloser)
 	readCloser.Close()
 	return nil
 }
@@ -353,6 +383,7 @@ func (storage *WebDAVStorage) MoveFile(threadIndex int, from string, to string)
 	if err != nil {
 		return err
 	}
+	io.Copy(ioutil.Discard, readCloser)
 	readCloser.Close()
 	return nil
 }
@@ -366,21 +397,7 @@ func (storage *WebDAVStorage) createParentDirectory(threadIndex int, dir string)
 	}
 	parent := dir[:found]
 
-	storage.directoryCacheLock.Lock()
-	_, exist := storage.directoryCache[parent]
-	storage.directoryCacheLock.Unlock()
-
-	if exist {
-		return nil
-	}
-
-	err = storage.CreateDirectory(threadIndex, parent)
-	if err == nil {
-		storage.directoryCacheLock.Lock()
-		storage.directoryCache[parent] = 1
-		storage.directoryCacheLock.Unlock()
-	}
-	return err
+	return storage.CreateDirectory(threadIndex, parent)
 }
 
 // CreateDirectory creates a new directory.
@@ -393,18 +410,35 @@ func (storage *WebDAVStorage) CreateDirectory(threadIndex int, dir string) (err
 		return nil
 	}
 
+	storage.directoryCacheLock.Lock()
+	_, exist := storage.directoryCache[dir]
+	storage.directoryCacheLock.Unlock()
+
+	if exist {
+		return nil
+	}
+
 	// If there is an error in creating the parent directory, proceed anyway
 	storage.createParentDirectory(threadIndex, dir)
 
 	readCloser, _, err := storage.sendRequest("MKCOL", dir, 0, []byte(""))
 	if err != nil {
-		if err == errWebDAVMethodNotAllowed || err == errWebDAVMovedPermanently {
+		if err == errWebDAVMethodNotAllowed || err == errWebDAVMovedPermanently || err == io.EOF {
 			// We simply ignore these errors and assume that the directory already exists
+			LOG_TRACE("WEBDAV_MKDIR", "Can't create directory %s: %v; error ignored", dir, err)
+			storage.directoryCacheLock.Lock()
+			storage.directoryCache[dir] = 1
+			storage.directoryCacheLock.Unlock()
 			return nil
 		}
 		return err
 	}
+	io.Copy(ioutil.Discard, readCloser)
 	readCloser.Close()
+
+	storage.directoryCacheLock.Lock()
+	storage.directoryCache[dir] = 1
+	storage.directoryCacheLock.Unlock()
 	return nil
 }
 
@@ -429,6 +463,7 @@ func (storage *WebDAVStorage) UploadFile(threadIndex int, filePath string, conte
 	if err != nil {
 		return err
 	}
+	io.Copy(ioutil.Discard, readCloser)
 	readCloser.Close()
 	return nil
 }