Merge 6df5e682f1 into 065ae50868

Parse the storage URL first, then use a regex to ensure the server name is
correctly identified starting after the final '@'.

.github/ISSUE_TEMPLATE.md

@@ -0,0 +1,17 @@
+---
+name: Please use the official forum
+about: Please use the official forum instead of Github
+title: 'Please use the official forum'
+labels: ''
+assignees: ''
+
+---
+
+
+Please **use the [Duplicacy Forum](https://forum.duplicacy.com/)** when reporting bugs, making feature requests, asking for help or simply praising Duplicacy for its ease of use.
+
+We strongly encourage you to create an account on the forum and use that platform for discussion as there is a higher chance that someone there will talk to you. 
+
+There is a handful of people watching the Github Issues and we are in the process of moving **all** of them to the forum as well. Most likely you will not receive an answer here or it will be very slow and you will be pointed to the forum.
+
+We have already created a comprehensive [Guide](https://forum.duplicacy.com/t/duplicacy-user-guide/1197), and a [How-To](https://forum.duplicacy.com/c/how-to) category which stores more wisdom than these issues on Github.

ACKNOWLEDGEMENTS.md

@@ -14,3 +14,4 @@ Duplicacy is based on the following open source projects:
 |https://github.com/pcwizz/xattr            |  BSD-2-Clause |
 |https://github.com/minio/blake2b-simd      |  Apache-2.0 |
 |https://github.com/go-ole/go-ole           |  MIT |
+https://github.com/ncw/swift                |  MIT |

Gopkg.lock

@@ -1,212 +0,0 @@
-# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
-
-
-[[projects]]
-  name = "cloud.google.com/go"
-  packages = ["compute/metadata","iam","internal","internal/optional","internal/version","storage"]
-  revision = "2d3a6656c17a60b0815b7e06ab0be04eacb6e613"
-  version = "v0.16.0"
-
-[[projects]]
-  name = "github.com/Azure/go-autorest"
-  packages = ["autorest","autorest/adal","autorest/azure","autorest/date"]
-  revision = "c67b24a8e30d876542a85022ebbdecf0e5a935e8"
-  version = "v9.4.1"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/aryann/difflib"
-  packages = ["."]
-  revision = "e206f873d14a916d3d26c40ab667bca123f365a3"
-
-[[projects]]
-  name = "github.com/aws/aws-sdk-go"
-  packages = ["aws","aws/awserr","aws/awsutil","aws/client","aws/client/metadata","aws/corehandlers","aws/credentials","aws/credentials/ec2rolecreds","aws/credentials/endpointcreds","aws/credentials/stscreds","aws/defaults","aws/ec2metadata","aws/endpoints","aws/request","aws/session","aws/signer/v4","internal/shareddefaults","private/protocol","private/protocol/query","private/protocol/query/queryutil","private/protocol/rest","private/protocol/restxml","private/protocol/xml/xmlutil","service/s3","service/sts"]
-  revision = "a32b1dcd091264b5dee7b386149b6cc3823395c9"
-  version = "v1.12.31"
-
-[[projects]]
-  name = "github.com/bkaradzic/go-lz4"
-  packages = ["."]
-  revision = "74ddf82598bc4745b965729e9c6a463bedd33049"
-  version = "v1.0.0"
-
-[[projects]]
-  name = "github.com/dgrijalva/jwt-go"
-  packages = ["."]
-  revision = "dbeaa9332f19a944acb5736b4456cfcc02140e29"
-  version = "v3.1.0"
-
-[[projects]]
-  name = "github.com/gilbertchen/azure-sdk-for-go"
-  packages = ["storage"]
-  revision = "2d49bb8f2cee530cc16f1f1a9f0aae763dee257d"
-  version = "v10.2.1-beta"
-
-[[projects]]
-  name = "github.com/gilbertchen/cli"
-  packages = ["."]
-  revision = "565493f259bf868adb54d45d5f4c68d405117adf"
-  version = "v1.2.0"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/gilbertchen/go-dropbox"
-  packages = ["."]
-  revision = "90711b603312b1f973f3a5da3793ac4f1e5c2f2a"
-
-[[projects]]
-  name = "github.com/gilbertchen/go-ole"
-  packages = ["."]
-  revision = "0e87ea779d9deb219633b828a023b32e1244dd57"
-  version = "v1.2.0"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/gilbertchen/go.dbus"
-  packages = ["."]
-  revision = "9e442e6378618c083fd3b85b703ffd202721fb17"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/gilbertchen/goamz"
-  packages = ["aws","s3"]
-  revision = "eada9f4e8cc2a45db775dee08a2c37597ce4760a"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/gilbertchen/gopass"
-  packages = ["."]
-  revision = "bf9dde6d0d2c004a008c27aaee91170c786f6db8"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/gilbertchen/keyring"
-  packages = ["."]
-  revision = "8855f5632086e51468cd7ce91056f8da69687ef6"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/gilbertchen/xattr"
-  packages = ["."]
-  revision = "68e7a6806b0137a396d7d05601d7403ae1abac58"
-
-[[projects]]
-  name = "github.com/go-ini/ini"
-  packages = ["."]
-  revision = "32e4c1e6bc4e7d0d8451aa6b75200d19e37a536a"
-  version = "v1.32.0"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/golang/protobuf"
-  packages = ["proto","protoc-gen-go/descriptor","ptypes","ptypes/any","ptypes/duration","ptypes/timestamp"]
-  revision = "1e59b77b52bf8e4b449a57e6f79f21226d571845"
-
-[[projects]]
-  name = "github.com/googleapis/gax-go"
-  packages = ["."]
-  revision = "317e0006254c44a0ac427cc52a0e083ff0b9622f"
-  version = "v2.0.0"
-
-[[projects]]
-  name = "github.com/jmespath/go-jmespath"
-  packages = ["."]
-  revision = "0b12d6b5"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/kr/fs"
-  packages = ["."]
-  revision = "2788f0dbd16903de03cb8186e5c7d97b69ad387b"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/minio/blake2b-simd"
-  packages = ["."]
-  revision = "3f5f724cb5b182a5c278d6d3d55b40e7f8c2efb4"
-
-[[projects]]
-  name = "github.com/pkg/errors"
-  packages = ["."]
-  revision = "645ef00459ed84a119197bfb8d8205042c6df63d"
-  version = "v0.8.0"
-
-[[projects]]
-  name = "github.com/pkg/sftp"
-  packages = ["."]
-  revision = "98203f5a8333288eb3163b7c667d4260fe1333e9"
-  version = "1.0.0"
-
-[[projects]]
-  name = "github.com/satori/uuid"
-  packages = ["."]
-  revision = "879c5887cd475cd7864858769793b2ceb0d44feb"
-  version = "v1.1.0"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/vaughan0/go-ini"
-  packages = ["."]
-  revision = "a98ad7ee00ec53921f08832bc06ecf7fd600e6a1"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/crypto"
-  packages = ["curve25519","ed25519","ed25519/internal/edwards25519","pbkdf2","ssh","ssh/agent","ssh/terminal"]
-  revision = "9f005a07e0d31d45e6656d241bb5c0f2efd4bc94"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/net"
-  packages = ["context","context/ctxhttp","http2","http2/hpack","idna","internal/timeseries","lex/httplex","trace"]
-  revision = "9dfe39835686865bff950a07b394c12a98ddc811"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/oauth2"
-  packages = [".","google","internal","jws","jwt"]
-  revision = "f95fa95eaa936d9d87489b15d1d18b97c1ba9c28"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/sys"
-  packages = ["unix","windows"]
-  revision = "82aafbf43bf885069dc71b7e7c2f9d7a614d47da"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/text"
-  packages = ["collate","collate/build","internal/colltab","internal/gen","internal/tag","internal/triegen","internal/ucd","language","secure/bidirule","transform","unicode/bidi","unicode/cldr","unicode/norm","unicode/rangetable"]
-  revision = "88f656faf3f37f690df1a32515b479415e1a6769"
-
-[[projects]]
-  branch = "master"
-  name = "google.golang.org/api"
-  packages = ["drive/v3","gensupport","googleapi","googleapi/internal/uritemplates","googleapi/transport","internal","iterator","option","storage/v1","transport/http"]
-  revision = "17b5f22a248d6d3913171c1a557552ace0d9c806"
-
-[[projects]]
-  name = "google.golang.org/appengine"
-  packages = [".","internal","internal/app_identity","internal/base","internal/datastore","internal/log","internal/modules","internal/remote_api","internal/urlfetch","urlfetch"]
-  revision = "150dc57a1b433e64154302bdc40b6bb8aefa313a"
-  version = "v1.0.0"
-
-[[projects]]
-  branch = "master"
-  name = "google.golang.org/genproto"
-  packages = ["googleapis/api/annotations","googleapis/iam/v1","googleapis/rpc/status"]
-  revision = "891aceb7c239e72692819142dfca057bdcbfcb96"
-
-[[projects]]
-  name = "google.golang.org/grpc"
-  packages = [".","balancer","balancer/roundrobin","codes","connectivity","credentials","encoding","grpclb/grpc_lb_v1/messages","grpclog","internal","keepalive","metadata","naming","peer","resolver","resolver/dns","resolver/passthrough","stats","status","tap","transport"]
-  revision = "5a9f7b402fe85096d2e1d0383435ee1876e863d0"
-  version = "v1.8.0"
-
-[solve-meta]
-  analyzer-name = "dep"
-  analyzer-version = 1
-  inputs-digest = "95a162eedee5e915fbd1917c3ba5021e646aa2f13a542c7cbeb02bcf30a3acb9"
-  solver-name = "gps-cdcl"
-  solver-version = 1

Gopkg.toml

@@ -1,94 +0,0 @@
-
-# Gopkg.toml example
-#
-# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md
-# for detailed Gopkg.toml documentation.
-#
-# required = ["github.com/user/thing/cmd/thing"]
-# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"]
-#
-# [[constraint]]
-#   name = "github.com/user/project"
-#   version = "1.0.0"
-#
-# [[constraint]]
-#   name = "github.com/user/project2"
-#   branch = "dev"
-#   source = "github.com/myfork/project2"
-#
-# [[override]]
-#  name = "github.com/x/y"
-#  version = "2.4.0"
-
-
-[[constraint]]
-  name = "cloud.google.com/go"
-  version = "0.16.0"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/aryann/difflib"
-
-[[constraint]]
-  name = "github.com/aws/aws-sdk-go"
-  version = "1.12.31"
-
-[[constraint]]
-  name = "github.com/bkaradzic/go-lz4"
-  version = "1.0.0"
-
-[[constraint]]
-  name = "github.com/gilbertchen/azure-sdk-for-go"
-  version = "10.2.1-beta"
-
-[[constraint]]
-  name = "github.com/gilbertchen/cli"
-  version = "1.2.0"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/gilbertchen/go-dropbox"
-
-[[constraint]]
-  name = "github.com/gilbertchen/go-ole"
-  version = "1.2.0"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/gilbertchen/goamz"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/gilbertchen/gopass"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/gilbertchen/keyring"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/gilbertchen/xattr"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/minio/blake2b-simd"
-
-[[constraint]]
-  name = "github.com/pkg/sftp"
-  version = "1.0.0"
-
-[[constraint]]
-  branch = "master"
-  name = "golang.org/x/crypto"
-
-[[constraint]]
-  branch = "master"
-  name = "golang.org/x/net"
-
-[[constraint]]
-  branch = "master"
-  name = "golang.org/x/oauth2"
-
-[[constraint]]
-  branch = "master"
-  name = "google.golang.org/api"

LICENSE.md

@@ -1,8 +1,7 @@
 Copyright © 2017 Acrosync LLC
 
 * Free for personal use or commercial trial
-* Non-trial commercial use requires per-user CLI licenses available from [duplicacy.com](https://duplicacy.com/buy) at a cost of $20 per year
-* A user is defined as the computer account that creates or edits the files to be backed up; if a backup contains files created or edited by multiple users for commercial purposes, one CLI license is required for each user
+* Non-trial commercial use requires per-computer CLI licenses available from [duplicacy.com](https://duplicacy.com/buy.html) at a cost of $50 per year
 * The computer with a valid commercial license for the GUI version may run the CLI version without a CLI license
 * CLI licenses are not required to restore or manage backups; only the backup command requires valid CLI licenses
 * Modification and redistribution are permitted, but commercial use of derivative works is subject to the same requirements of this license

README.md

@@ -1,134 +1,100 @@
 # Duplicacy: A lock-free deduplication cloud backup tool
 
-Duplicacy is a new generation cross-platform cloud backup tool based on the idea of [Lock-Free Deduplication](https://github.com/gilbertchen/duplicacy/wiki/Lock-Free-Deduplication).  It is the only cloud backup tool that allows multiple computers to back up to the same storage simultaneously without using any locks (thus readily amenable to various cloud storage services).
+Duplicacy is a new generation cross-platform cloud backup tool based on the idea of [Lock-Free Deduplication](https://github.com/gilbertchen/duplicacy/wiki/Lock-Free-Deduplication).
 
-The repository hosts source code, design documents, and binary releases of the command line version.  There is also a Duplicacy GUI frontend built for Windows and Mac OS X available from https://duplicacy.com.
+Our paper explaining the inner workings of Duplicacy has been accepted by [IEEE Transactions on Cloud Computing](https://ieeexplore.ieee.org/document/9310668) and will appear in a future issue this year.  The final draft version is available [here](https://github.com/gilbertchen/duplicacy/blob/master/duplicacy_paper.pdf) for those who don't have IEEE subscriptions. 
+
+This repository hosts source code, design documents, and binary releases of the command line version of Duplicacy.  There is also a Web GUI frontend built for Windows, macOS, and Linux, available from https://duplicacy.com.
 
 There is a special edition of Duplicacy developed for VMware vSphere (ESXi) named [Vertical Backup](https://www.verticalbackup.com) that can back up virtual machine files on ESXi to local drives, network or cloud storages.
 
 ## Features
 
-Duplicacy currently supports major cloud storage providers (Amazon S3, Google Cloud Storage, Microsoft Azure, Dropbox, Backblaze B2, Google Drive, Microsoft OneDrive, and Hubic) and offers all essential features of a modern backup tool:
+There are 3 core advantages of Duplicacy over any other open-source or commercial backup tools:
 
-* Incremental backup: only back up what has been changed
-* Full snapshot: although each backup is incremental, it must behave like a full snapshot for easy restore and deletion
-* Deduplication: identical files must be stored as one copy (file-level deduplication), and identical parts from different files must be stored as one copy (block-level deduplication)
-* Encryption: encrypt not only file contents but also file paths, sizes, times, etc.
-* Deletion: every backup can be deleted independently without affecting others
-* Concurrent access: multiple clients can back up to the same storage at the same time
-* Snapshot migration: all or selected snapshots can be migrated from one storage to another
+* Duplicacy is the *only* cloud backup tool that allows multiple computers to back up to the same cloud storage, taking advantage of cross-computer deduplication whenever possible, without direct communication among them.  This feature turns any cloud storage server supporting only a basic set of file operations into a sophisticated deduplication-aware server.  
 
-The key idea of **[Lock-Free Deduplication](https://github.com/gilbertchen/duplicacy/wiki/Lock-Free-Deduplication)** can be summarized as follows:
+* Unlike other chunk-based backup tools where chunks are grouped into pack files and a chunk database is used to track which chunks are stored inside each pack file, Duplicacy takes a database-less approach where every chunk is saved independently using its hash as the file name to facilitate quick lookups.  The avoidance of a centralized chunk database not only produces a simpler and less error-prone implementation, but also makes it easier to develop advanced features, such as [Asymmetric Encryption](https://github.com/gilbertchen/duplicacy/wiki/RSA-encryption) for stronger encryption and [Erasure Coding](https://github.com/gilbertchen/duplicacy/wiki/Erasure-coding) for resilient data protection.
+
+* Duplicacy is fast.  While the performance wasn't the top-priority design goal, Duplicacy has been shown to outperform other backup tools by a considerable margin, as indicated by the following results obtained from a [benchmarking experiment](https://github.com/gilbertchen/benchmarking) backing up the [Linux code base](https://github.com/torvalds/linux) using Duplicacy and 3 other open-source backup tools.
+
+[![Comparison of Duplicacy, restic, Attic, duplicity](https://github.com/gilbertchen/duplicacy/blob/master/images/duplicacy_benchmark_speed.png "Comparison of Duplicacy, restic, Attic, duplicity")](https://github.com/gilbertchen/benchmarking)
 
-* Use variable-size chunking algorithm to split files into chunks
-* Store each chunk in the storage using a file name derived from its hash, and rely on the file system API to manage chunks without using a centralized indexing database
-* Apply a *two-step fossil collection* algorithm to remove chunks that become unreferenced after a backup is deleted
 
 ## Getting Started
 
 * [A brief introduction](https://github.com/gilbertchen/duplicacy/wiki/Quick-Start)
 * [Command references](https://github.com/gilbertchen/duplicacy/wiki)
+* [Building from source](https://github.com/gilbertchen/duplicacy/wiki/Installation)
 
 ## Storages
 
-With Duplicacy, you can back up files to local or networked drives, SFTP servers, or many cloud storage providers.  The following table compares the costs of all cloud storages supported by Duplicacy.
+Duplicacy currently provides the following storage backends:
 
-| Type         |   Storage (monthly)    |   Upload           |    Download    |    API Charge   |
-|:------------:|:-------------:|:------------------:|:--------------:|:-----------:|
-| Amazon S3    | $0.023/GB | free | $0.09/GB | [yes](https://aws.amazon.com/s3/pricing/) |
-| Wasabi       | $3.99 first 1TB <br> $0.0039/GB additional | free | $.04/GB | no |
-| DigitalOcean Spaces| $5 first 250GB <br> $0.02/GB additional | free | first 1TB free <br> $0.01/GB additional| no |
-| Backblaze B2 | $0.005/GB | free | $0.02/GB | [yes](https://www.backblaze.com/b2/b2-transactions-price.html) |
-| Google Cloud Storage| $0.026/GB | free |$ 0.12/GB | [yes](https://cloud.google.com/storage/pricing) |
-| Google Drive | 15GB free <br> $1.99/100GB <br> $9.99/TB | free | free | no |
-| Microsoft Azure | $0.0184/GB | free | free | [yes](https://azure.microsoft.com/en-us/pricing/details/storage/blobs/) |
-| Microsoft OneDrive | 5GB free <br> $1.99/50GB <br> $5.83/TB | free | free | no |
-| Dropbox | 2GB free <br> $8.25/TB | free | free | no |
-| Hubic | 25GB free <br> €1/100GB <br> €5/10TB | free | free | no |
+* Local disk
+* SFTP
+* Dropbox
+* Amazon S3
+* Wasabi
+* DigitalOcean Spaces
+* Google Cloud Storage
+* Microsoft Azure
+* Backblaze B2
+* Google Drive
+* Microsoft OneDrive
+* Hubic
+* OpenStack Swift
+* WebDAV (under beta testing)
+* pcloud (via WebDAV)
+* Box.com (via WebDAV)
+* File Fabric by [Storage Made Easy](https://storagemadeeasy.com/)
 
 Please consult the [wiki page](https://github.com/gilbertchen/duplicacy/wiki/Storage-Backends) on how to set up Duplicacy to work with each cloud storage.
 
-It should be noted that their performances vary a lot.  A [performance comparison](https://github.com/gilbertchen/cloud-storage-comparison) of these storages measured the running times (in seconds) of backing up and restoring the [Linux code base](https://github.com/torvalds/linux) as follows:
+For reference, the following chart shows the running times (in seconds) of backing up the [Linux code base](https://github.com/torvalds/linux) to each of those supported storages:
 
-| Storage              | initial backup | 2nd | 3rd | initial restore | 2nd | 3rd |
-|:--------------------:|:------:|:----:|:-----:|:----:|:-----:|:----:|
-| SFTP                 |  31.5  | 6.6  | 20.6  | 22.5  | 7.8  | 18.4 |
-| Amazon S3            |  41.1  | 5.9  | 21.9  | 27.7  | 7.6  | 23.5 |
-| Wasabi               |  38.7  | 5.7  | 31.7  | 25.7  | 6.5  | 23.2 | 
-| DigitalOcean Spaces  |  51.6  | 7.1  | 31.7  | 29.3  | 6.4  | 27.6 | 
-| Backblaze B2         |  106.7 | 24.0 | 88.2  | 67.9  | 14.4 | 39.1 | 
-| Google Cloud Storage |  76.9  | 11.9 | 33.1  | 39.5  | 9.9  | 26.2 | 
-| Google Drive         |  139.3 | 14.7 | 45.2  | 129.4 | 17.8 | 54.4 |
-| Microsoft Azure      |  35.0  | 5.4  | 20.4  | 30.7  | 7.1  | 21.5 | 
-| Microsoft OneDrive   |  250.0 | 31.6 | 80.2  | 333.4 | 26.2 | 82.0 | 
-| Dropbox              |  267.2 | 35.8 | 113.7 | 164.0 | 31.6 | 80.3 |
 
-For more details please visit https://github.com/gilbertchen/cloud-storage-comparison.
+[![Comparison of Cloud Storages](https://github.com/gilbertchen/duplicacy/blob/master/images/duplicacy_benchmark_cloud.png "Comparison of Cloud Storages")](https://github.com/gilbertchen/cloud-storage-comparison)
 
-## Feature Comparison with Other Backup Tools
+
+For complete benchmark results please visit https://github.com/gilbertchen/cloud-storage-comparison.
+
+## Comparison with Other Backup Tools
 
 [duplicity](http://duplicity.nongnu.org) works by applying the rsync algorithm (or more specific, the [librsync](https://github.com/librsync/librsync) library)
 to find the differences from previous backups and only then uploading the differences.  It is the only existing backup tool with extensive cloud support -- the [long list](http://duplicity.nongnu.org/duplicity.1.html#sect7) of storage backends covers almost every cloud provider one can think of.  However, duplicity's biggest flaw lies in its incremental model -- a chain of dependent backups starts with a full backup followed by a number of incremental ones, and ends when another full backup is uploaded.  Deleting one backup will render useless all the subsequent backups on the same chain.  Periodic full backups are required, in order to make previous backups disposable.
 
 [bup](https://github.com/bup/bup) also uses librsync to split files into chunks but save chunks in the git packfile format.  It doesn't support any cloud storage, or deletion of old backups.
 
-[Obnam](http://obnam.org) got the incremental backup model right in the sense that every incremental backup is actually a full snapshot.  Although Obnam also splits files into chunks, it does not adopt either the rsync algorithm or the variable-size chunking algorithm.  As a result, deletions or insertions of a few bytes will foil the
-[deduplication](http://obnam.org/faq/dedup).
-Deletion of old backups is possible, but no cloud storages are supported.
-Multiple clients can back up to the same storage, but only sequential access is granted by the [locking on-disk data structures](http://obnam.org/locking/).
-It is unclear if the lack of cloud backends is due to difficulties in porting the locking data structures to cloud storage APIs.
+[Duplicati](https://duplicati.com) is one of the first backup tools that adopt the chunk-based approach to split files into chunks which are then uploaded to the storage.  The chunk-based approach got the incremental backup model right in the sense that every incremental backup is actually a full snapshot.  As Duplicati splits files into fixed-size chunks,  deletions or insertions of a few bytes will foil the deduplication.  Cloud support is extensive, but multiple clients can't back up to the same storage location.
 
-[Attic](https://attic-backup.org) has been acclaimed by some as the [Holy Grail of backups](https://www.stavros.io/posts/holy-grail-backups).  It follows the same incremental backup model as Obnam, but embraces the variable-size chunk algorithm for better performance and better deduplication.  Deletions of old backup is also supported.  However, no cloud backends are implemented, as in Obnam.  Although concurrent backups from multiple clients to the same storage is in theory possible by the use of locking, it is
+[Attic](https://attic-backup.org) has been acclaimed by some as the [Holy Grail of backups](https://www.stavros.io/posts/holy-grail-backups).  It follows the same incremental backup model like Duplicati but embraces the variable-size chunk algorithm for better performance and higher deduplication efficiency (not susceptible to byte insertion and deletion any more).  Deletions of old backup are also supported.  However, no cloud backends are implemented.  Although concurrent backups from multiple clients to the same storage is in theory possible by the use of locking, it is
 [not recommended](http://librelist.com/browser//attic/2014/11/11/backing-up-multiple-servers-into-a-single-repository/#e96345aa5a3469a87786675d65da492b) by the developer due to chunk indices being kept in a local cache.
-Concurrent access is not only a convenience; it is a necessity for better deduplication.  For instance, if multiple machines with the same OS installed can back up their entire drives to the same storage, only one copy of the system files needs to be stored, greatly reducing the storage space regardless of the number of machines.  Attic still adopts the traditional approach of using a centralized indexing database to manage chunks, and relies heavily on caching to improve performance.  The presence of exclusive locking makes it hard to be adapted for cloud storage APIs and reduces the level of deduplication.
+Concurrent access is not only a convenience; it is a necessity for better deduplication.  For instance, if multiple machines with the same OS installed can back up their entire drives to the same storage, only one copy of the system files needs to be stored, greatly reducing the storage space regardless of the number of machines.  Attic still adopts the traditional approach of using a centralized indexing database to manage chunks and relies heavily on caching to improve performance.  The presence of exclusive locking makes it hard to be extended to cloud storages.
 
-[restic](https://restic.github.io) is a more recent addition.  It is worth mentioning here because, like Duplicacy, it is written in Go.  It uses a format similar to the git packfile format.  Multiple clients backing up to the same storage are still guarded by
-[locks](https://github.com/restic/restic/blob/master/doc/Design.md#locks).  A prune operation will therefore completely block all other clients connected to the storage from doing their regular backups.  Moreover, since most cloud storage services do not provide a locking service, the best effort is to use some basic file operations to simulate a lock, but distributed locking is known to be a hard problem and it is unclear how reliable restic's lock implementation is.  A faulty implementation may cause a prune operation to accidentally delete data still in use, resulting in unrecoverable data loss.  This is the exact problem that we avoided by taking the lock-free approach.
+[restic](https://restic.github.io) is a more recent addition. It uses a format similar to the git packfile format.  Multiple clients backing up to the same storage are still guarded by
+[locks](https://github.com/restic/restic/blob/master/doc/Design.md#locks), and because a chunk database is used, deduplication isn't real-time (different clients sharing the same files will upload different copies of the same chunks).  A prune operation will completely block all other clients connected to the storage from doing their regular backups.  Moreover, since most cloud storage services do not provide a locking service, the best effort is to use some basic file operations to simulate a lock, but distributed locking is known to be a hard problem and it is unclear how reliable restic's lock implementation is.  A faulty implementation may cause a prune operation to accidentally delete data still in use, resulting in unrecoverable data loss.  This is the exact problem that we avoided by taking the lock-free approach.
 
 
 The following table compares the feature lists of all these backup tools:
 
 
-| Feature/Tool       | duplicity | bup | Obnam             | Attic           | restic            | **Duplicacy** |
+| Feature/Tool       | duplicity | bup | Duplicati         | Attic           | restic            | **Duplicacy** |
 |:------------------:|:---------:|:---:|:-----------------:|:---------------:|:-----------------:|:-------------:|
 | Incremental Backup | Yes       | Yes | Yes               | Yes             | Yes               | **Yes**       |
 | Full Snapshot      | No        | Yes | Yes               | Yes             | Yes               | **Yes**       |
+| Compression        | Yes       | Yes | Yes               | Yes             | No                | **Yes**       |
 | Deduplication      | Weak      | Yes | Weak              | Yes             | Yes               | **Yes**       |
 | Encryption         | Yes       | Yes | Yes               | Yes             | Yes               | **Yes**       |
 | Deletion           | No        | No  | Yes               | Yes             | No                | **Yes**       |
-| Concurrent Access  | No        | No  | Exclusive locking | Not recommended | Exclusive locking | **Lock-free** |
-| Cloud Support      | Extensive | No  | No                | No              | S3, B2, OpenStack | **S3, GCS, Azure, Dropbox, Backblaze B2, Google Drive, OneDrive, and Hubic**|
+| Concurrent Access  | No        | No  | No                | Not recommended | Exclusive locking | **Lock-free** |
+| Cloud Support      | Extensive | No  | Extensive         | No              | Limited           | **Extensive** |
 | Snapshot Migration | No        | No  | No                | No              | No                | **Yes**       |
 
-
-## Performance Comparison with Other Backup Tools
-
-Duplicacy is not only more feature-rich but also faster than other backup tools.  The following table lists the running times in seconds of backing up the [Linux code base](https://github.com/torvalds/linux) using Duplicacy and 3 other tools.  Clearly Duplicacy is the fastest by a significant margin.
-
-
-|                    |   Duplicacy  |   restic   |   Attic    |  duplicity  |
-|:------------------:|:----------------:|:----------:|:----------:|:-----------:|
-| Initial backup | 13.7 | 20.7 | 26.9 | 44.2 |
-| 2nd backup | 4.8  |  8.0 | 15.4 | 19.5 |
-| 3rd backup | 6.9  | 11.9 | 19.6 | 29.8 |
-| 4th backup | 3.3  | 7.0  | 13.7 | 18.6 |
-| 5th backup | 9.9  | 11.4 | 19.9 | 28.0 |
-| 6th backup | 3.8  | 8.0  | 16.8 | 22.0 |
-| 7th backup | 5.1  | 7.8  | 14.3 | 21.6 |
-| 8th backup | 9.5  | 13.5 | 18.3 | 35.0 |
-| 9th backup | 4.3  | 9.0  | 15.7 | 24.9 |
-| 10th backup | 7.9 | 20.2 | 32.2 | 35.0 |
-| 11th backup | 4.6 | 9.1  | 16.8 | 28.1 |
-| 12th backup | 7.4 | 12.0 | 21.7 | 37.4 |
-
-
-For more details and other speed comparison results, please visit https://github.com/gilbertchen/benchmarking.  There you can also find test scripts that you can use to run your own experiments.
-
 ## License
 
 * Free for personal use or commercial trial
-* Non-trial commercial use requires per-user CLI licenses available from [duplicacy.com](https://duplicacy.com/buy) at a cost of $20 per year
-* A user is defined as the computer account that creates or edits the files to be backed up; if a backup contains files created or edited by multiple users for commercial purposes, one CLI license is required for each user
+* Non-trial commercial use requires per-computer CLI licenses available from [duplicacy.com](https://duplicacy.com/buy.html) at a cost of $50 per year
 * The computer with a valid commercial license for the GUI version may run the CLI version without a CLI license
 * CLI licenses are not required to restore or manage backups; only the backup command requires valid CLI licenses
 * Modification and redistribution are permitted, but commercial use of derivative works is subject to the same requirements of this license

go.mod

@@ -0,0 +1,78 @@
+module github.com/gilbertchen/duplicacy
+
+go 1.19
+
+require (
+	cloud.google.com/go v0.38.0
+	github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a
+	github.com/aws/aws-sdk-go v1.30.7
+	github.com/bkaradzic/go-lz4 v1.0.0
+	github.com/gilbertchen/azure-sdk-for-go v14.1.2-0.20180323033227-8fd4663cab7c+incompatible
+	github.com/gilbertchen/cli v1.2.1-0.20160223210219-1de0a1836ce9
+	github.com/gilbertchen/go-dropbox v0.0.0-20230321030224-087ef8db1916
+	github.com/gilbertchen/go-ole v1.2.0
+	github.com/gilbertchen/goamz v0.0.0-20170712012135-eada9f4e8cc2
+	github.com/gilbertchen/gopass v0.0.0-20170109162249-bf9dde6d0d2c
+	github.com/gilbertchen/highwayhash v0.0.0-20221109044721-eeab1f4799d8
+	github.com/gilbertchen/keyring v0.0.0-20221004152639-1661cbebc508
+	github.com/gilbertchen/xattr v0.0.0-20160926155429-68e7a6806b01
+	github.com/hirochachacha/go-smb2 v1.1.0
+	github.com/klauspost/compress v1.16.3
+	github.com/klauspost/reedsolomon v1.9.9
+	github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1
+	github.com/minio/highwayhash v1.0.2
+	github.com/ncw/swift/v2 v2.0.1
+	github.com/pkg/sftp v1.11.0
+	github.com/pkg/xattr v0.4.1
+	github.com/vmihailenco/msgpack v4.0.4+incompatible
+	golang.org/x/crypto v0.12.0
+	golang.org/x/net v0.10.0
+	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
+	google.golang.org/api v0.21.0
+	storj.io/uplink v1.12.1
+)
+
+require (
+	github.com/Azure/go-autorest v10.15.5+incompatible // indirect
+	github.com/calebcase/tmpfile v1.0.3 // indirect
+	github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect
+	github.com/dnaeon/go-vcr v1.2.0 // indirect
+	github.com/flynn/noise v1.0.0 // indirect
+	github.com/geoffgarside/ber v1.1.0 // indirect
+	github.com/goamz/goamz v0.0.0-20180131231218-8b901b531db8 // indirect
+	github.com/godbus/dbus v4.1.0+incompatible // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
+	github.com/golang/protobuf v1.5.0 // indirect
+	github.com/googleapis/gax-go/v2 v2.0.5 // indirect
+	github.com/jmespath/go-jmespath v0.3.0 // indirect
+	github.com/jtolio/eventkit v0.0.0-20221004135224-074cf276595b // indirect
+	github.com/jtolio/noiseconn v0.0.0-20230111204749-d7ec1a08b0b8 // indirect
+	github.com/klauspost/cpuid v1.3.1 // indirect
+	github.com/klauspost/cpuid/v2 v2.0.12 // indirect
+	github.com/kr/fs v0.1.0 // indirect
+	github.com/marstr/guid v1.1.0 // indirect
+	github.com/mmcloughlin/avo v0.0.0-20200803215136-443f81d77104 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/satori/go.uuid v1.2.0 // indirect
+	github.com/segmentio/go-env v1.1.0 // indirect
+	github.com/spacemonkeygo/monkit/v3 v3.0.22 // indirect
+	github.com/vaughan0/go-ini v0.0.0-20130923145212-a98ad7ee00ec // indirect
+	github.com/vivint/infectious v0.0.0-20200605153912-25a574ae18a3 // indirect
+	github.com/zeebo/blake3 v0.2.3 // indirect
+	github.com/zeebo/errs v1.3.0 // indirect
+	go.opencensus.io v0.22.3 // indirect
+	golang.org/x/mod v0.10.0 // indirect
+	golang.org/x/sync v0.3.0 // indirect
+	golang.org/x/sys v0.11.0 // indirect
+	golang.org/x/term v0.11.0 // indirect
+	golang.org/x/text v0.12.0 // indirect
+	golang.org/x/tools v0.9.1 // indirect
+	google.golang.org/appengine v1.6.5 // indirect
+	google.golang.org/genproto v0.0.0-20200409111301-baae70f3302d // indirect
+	google.golang.org/grpc v1.28.1 // indirect
+	google.golang.org/protobuf v1.28.1 // indirect
+	storj.io/common v0.0.0-20230920095429-0ce0a575e6f8 // indirect
+	storj.io/drpc v0.0.33 // indirect
+	storj.io/picobuf v0.0.2-0.20230906122608-c4ba17033c6c // indirect
+)

go.sum

@@ -0,0 +1,302 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.38.0 h1:ROfEUZz+Gh5pa62DJWXSaonyu3StP6EA6lPEXPI6mCo=
+cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
+github.com/Azure/go-autorest v10.15.5+incompatible h1:vdxx6wM1rVkKt/3niByPVjguoLWkWImOcJNvEykgBzY=
+github.com/Azure/go-autorest v10.15.5+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a h1:pv34s756C4pEXnjgPfGYgdhg/ZdajGhyOvzx8k+23nw=
+github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A=
+github.com/aws/aws-sdk-go v1.30.7 h1:IaXfqtioP6p9SFAnNfsqdNczbR5UNbYqvcZUSsCAdTY=
+github.com/aws/aws-sdk-go v1.30.7/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
+github.com/bkaradzic/go-lz4 v1.0.0 h1:RXc4wYsyz985CkXXeX04y4VnZFGG8Rd43pRaHsOXAKk=
+github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4=
+github.com/calebcase/tmpfile v1.0.3 h1:BZrOWZ79gJqQ3XbAQlihYZf/YCV0H4KPIdM5K5oMpJo=
+github.com/calebcase/tmpfile v1.0.3/go.mod h1:UAUc01aHeC+pudPagY/lWvt2qS9ZO5Zzof6/tIUzqeI=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI=
+github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
+github.com/dsnet/try v0.0.3 h1:ptR59SsrcFUYbT/FhAbKTV6iLkeD6O18qfIWRml2fqI=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/flynn/noise v1.0.0 h1:DlTHqmzmvcEiKj+4RYo/imoswx/4r6iBlCMfVtrMXpQ=
+github.com/flynn/noise v1.0.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag=
+github.com/geoffgarside/ber v1.1.0 h1:qTmFG4jJbwiSzSXoNJeHcOprVzZ8Ulde2Rrrifu5U9w=
+github.com/geoffgarside/ber v1.1.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc=
+github.com/gilbertchen/azure-sdk-for-go v14.1.2-0.20180323033227-8fd4663cab7c+incompatible h1:2fZxTUw5D9uGWnYTsU/obVavn+1qTF+TsVok3U8uN2Q=
+github.com/gilbertchen/azure-sdk-for-go v14.1.2-0.20180323033227-8fd4663cab7c+incompatible/go.mod h1:qsVRCpBUm2l0eMUeI9wZ47yzra2+lv2YkGhMZpzBVUc=
+github.com/gilbertchen/cli v1.2.1-0.20160223210219-1de0a1836ce9 h1:uMgtTp4sRJ7kMQMF3xEKeFntf3XatwkLNL/byj8v97g=
+github.com/gilbertchen/cli v1.2.1-0.20160223210219-1de0a1836ce9/go.mod h1:WOnN3JdZiZwUaYtLH2DRxe5PpD43wuOIvc/Wem/39M0=
+github.com/gilbertchen/go-dropbox v0.0.0-20230321030224-087ef8db1916 h1:7VpJiGwW51MB7yJ5e27Ar/ej8Yu7WuU2SEo409qPoNs=
+github.com/gilbertchen/go-dropbox v0.0.0-20230321030224-087ef8db1916/go.mod h1:85+2CRHC/klHy4vEM+TYtbhDo2wMjPa4JNdVzUHsDIk=
+github.com/gilbertchen/go-ole v1.2.0 h1:ay65uwxo6w8UVOxN0+fuCqUXGaXxbmkGs5m4uY6e1Zw=
+github.com/gilbertchen/go-ole v1.2.0/go.mod h1:NNiozp7QxhyGmHxxNdFKIcVaINvJFTAjBJ2gYzh8fsg=
+github.com/gilbertchen/goamz v0.0.0-20170712012135-eada9f4e8cc2 h1:VDPwi3huqeJBtymgLOvPAP4S2gbSSK/UrWVwRbRAmnw=
+github.com/gilbertchen/goamz v0.0.0-20170712012135-eada9f4e8cc2/go.mod h1:AoxJeh8meXUrSWBLiq9BJvYMd9RAAGgEUU0gSkNedRY=
+github.com/gilbertchen/gopass v0.0.0-20170109162249-bf9dde6d0d2c h1:0SR0aXvil/eQReU0olxp/j04B+Y/47fjDMotIxaAgKo=
+github.com/gilbertchen/gopass v0.0.0-20170109162249-bf9dde6d0d2c/go.mod h1:HDsXH7AAfDsfYYX0te4zsNbnwVvZ2RtLEOCjN4y84jw=
+github.com/gilbertchen/highwayhash v0.0.0-20221109044721-eeab1f4799d8 h1:ijgl4Y+OKCIFiCPk/Rf9tb6PrarVqitu5TynpyCmRK0=
+github.com/gilbertchen/highwayhash v0.0.0-20221109044721-eeab1f4799d8/go.mod h1:0lQcVva56+L1PuUFXLOsJ6arJQaU0baIH8q+IegeBhg=
+github.com/gilbertchen/keyring v0.0.0-20221004152639-1661cbebc508 h1:SqTyk5KkNXp7zTdTttIZSDcTrL5uau4K/2OpKvgBZVI=
+github.com/gilbertchen/keyring v0.0.0-20221004152639-1661cbebc508/go.mod h1:w/pisxUZezf2XzU9Ewjphcf6q1mZtOzKPHhJiuc8cag=
+github.com/gilbertchen/xattr v0.0.0-20160926155429-68e7a6806b01 h1:LqwS9qL6SrDkp0g0iwUkETrDdtB9gTKaIbSn9imUq5o=
+github.com/gilbertchen/xattr v0.0.0-20160926155429-68e7a6806b01/go.mod h1:TMlibuxKfkdtHyltooAw7+DHqRpaXs9nxaffk00Sh1Q=
+github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
+github.com/goamz/goamz v0.0.0-20180131231218-8b901b531db8 h1:G1U0vew/vA/1/hBmf1XNeyIzJJbPFVv+kb+HPl6rj6c=
+github.com/goamz/goamz v0.0.0-20180131231218-8b901b531db8/go.mod h1:/Ya1YZsqLQp17bDgHdyE9/XBR1uIH1HKasTvLxcoM/A=
+github.com/godbus/dbus v4.1.0+incompatible h1:WqqLRTsQic3apZUK9qC5sGNfXthmPXzUZ7nQPrNITa4=
+github.com/godbus/dbus v4.1.0+incompatible/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw=
+github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
+github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.5.0 h1:LUVKkCeviFUMKqHa4tXIIij/lbhnMbP7Fn5wKdKkRh4=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20211108044417-e9b028704de0 h1:rsq1yB2xiFLDYYaYdlGBsSkwVzsCo500wMhxvW5A/bk=
+github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
+github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM=
+github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hirochachacha/go-smb2 v1.1.0 h1:b6hs9qKIql9eVXAiN0M2wSFY5xnhbHAQoCwRKbaRTZI=
+github.com/hirochachacha/go-smb2 v1.1.0/go.mod h1:8F1A4d5EZzrGu5R7PU163UcMRDJQl4FtcxjBfsY8TZE=
+github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc=
+github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/jtolio/eventkit v0.0.0-20221004135224-074cf276595b h1:tO4MX3k5bvV0Sjv5jYrxStMTJxf1m/TW24XRyHji4aU=
+github.com/jtolio/eventkit v0.0.0-20221004135224-074cf276595b/go.mod h1:q7yMR8BavTz/gBNtIT/uF487LMgcuEpNGKISLAjNQes=
+github.com/jtolio/noiseconn v0.0.0-20230111204749-d7ec1a08b0b8 h1:+A1uT26XjTsxiUUZjAAuveILWWy+Sy2TPX8OIgGvPQE=
+github.com/jtolio/noiseconn v0.0.0-20230111204749-d7ec1a08b0b8/go.mod h1:f0ijQHcvHYAuxX6JA/JUr/Z0FVn12D9REaT/HAWVgP4=
+github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.16.3 h1:XuJt9zzcnaz6a16/OU53ZjWp/v7/42WcR5t2a0PcNQY=
+github.com/klauspost/compress v1.16.3/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/klauspost/cpuid v1.2.4/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
+github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s=
+github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4=
+github.com/klauspost/cpuid/v2 v2.0.12 h1:p9dKCg8i4gmOxtv35DvrYoWqYzQrvEVdjQ762Y0OqZE=
+github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
+github.com/klauspost/reedsolomon v1.9.9 h1:qCL7LZlv17xMixl55nq2/Oa1Y86nfO8EqDfv2GHND54=
+github.com/klauspost/reedsolomon v1.9.9/go.mod h1:O7yFFHiQwDR6b2t63KPUpccPtNdp5ADgh1gg4fd12wo=
+github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
+github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/marstr/guid v1.1.0 h1:/M4H/1G4avsieL6BbUwCOBzulmoeKVP5ux/3mQNnbyI=
+github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho=
+github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g=
+github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ=
+github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g=
+github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY=
+github.com/mmcloughlin/avo v0.0.0-20200803215136-443f81d77104 h1:ULR/QWMgcgRiZLUjSSJMU+fW+RDMstRdmnDWj9Q+AsA=
+github.com/mmcloughlin/avo v0.0.0-20200803215136-443f81d77104/go.mod h1:wqKykBG2QzQDJEzvRkcS8x6MiSJkF52hXZsXcjaB3ls=
+github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8=
+github.com/ncw/swift/v2 v2.0.1 h1:q1IN8hNViXEv8Zvg3Xdis4a3c4IlIGezkYz09zQL5J0=
+github.com/ncw/swift/v2 v2.0.1/go.mod h1:z0A9RVdYPjNjXVo2pDOPxZ4eu3oarO1P91fTItcb+Kg=
+github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/sftp v1.11.0 h1:4Zv0OGbpkg4yNuUtH0s8rvoYxRCNyT29NVUo6pgPmxI=
+github.com/pkg/sftp v1.11.0/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=
+github.com/pkg/xattr v0.4.1 h1:dhclzL6EqOXNaPDWqoeb9tIxATfBSmjqL0b4DpSjwRw=
+github.com/pkg/xattr v0.4.1/go.mod h1:W2cGD0TBEus7MkUgv0tNZ9JutLtVO3cXu+IBRuHqnFs=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/quic-go/qtls-go1-20 v0.3.2 h1:rRgN3WfnKbyik4dBV8A6girlJVxGand/d+jVKbQq5GI=
+github.com/quic-go/quic-go v0.38.0 h1:T45lASr5q/TrVwt+jrVccmqHhPL2XuSyoCLVCpfOSLc=
+github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
+github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
+github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
+github.com/segmentio/go-env v1.1.0 h1:AGJ7OnCx9M5NWpkYPGYELS6III/pFSnAs1GvKWStiEo=
+github.com/segmentio/go-env v1.1.0/go.mod h1:pEKO2ieHe8zF098OMaAHw21SajMuONlnI/vJNB3pB7I=
+github.com/spacemonkeygo/monkit/v3 v3.0.22 h1:4/g8IVItBDKLdVnqrdHZrCVPpIrwDBzl1jrV0IHQHDU=
+github.com/spacemonkeygo/monkit/v3 v3.0.22/go.mod h1:XkZYGzknZwkD0AKUnZaSXhRiVTLCkq7CWVa3IsE72gA=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
+github.com/vaughan0/go-ini v0.0.0-20130923145212-a98ad7ee00ec h1:DGmKwyZwEB8dI7tbLt/I/gQuP559o/0FrAkHKlQM/Ks=
+github.com/vaughan0/go-ini v0.0.0-20130923145212-a98ad7ee00ec/go.mod h1:owBmyHYMLkxyrugmfwE/DLJyW8Ro9mkphwuVErQ0iUw=
+github.com/vivint/infectious v0.0.0-20200605153912-25a574ae18a3 h1:zMsHhfK9+Wdl1F7sIKLyx3wrOFofpb3rWFbA4HgcK5k=
+github.com/vivint/infectious v0.0.0-20200605153912-25a574ae18a3/go.mod h1:R0Gbuw7ElaGSLOZUSwBm/GgVwMd30jWxBDdAyMOeTuc=
+github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI=
+github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
+github.com/zeebo/assert v1.3.1 h1:vukIABvugfNMZMQO1ABsyQDJDTVQbn+LWSMy1ol1h6A=
+github.com/zeebo/assert v1.3.1/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
+github.com/zeebo/blake3 v0.2.3 h1:TFoLXsjeXqRNFxSbk35Dk4YtszE/MQQGK10BH4ptoTg=
+github.com/zeebo/blake3 v0.2.3/go.mod h1:mjJjZpnsyIVtVgTOSpJ9vmRE4wgDeyt2HU3qXvvKCaQ=
+github.com/zeebo/errs v1.3.0 h1:hmiaKqgYZzcVgRL1Vkc1Mn2914BbzB0IBxs+ebeutGs=
+github.com/zeebo/errs v1.3.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4=
+github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo=
+github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4=
+go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
+go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8=
+go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+golang.org/x/arch v0.0.0-20190909030613-46d78d1859ac/go.mod h1:flIaEI6LNU6xOCD5PaJvn9wGP0agmIOqjrtsKGRguv4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
+golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk=
+golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20221205204356-47842c84f3db h1:D/cFflL63o2KSLJIwjlcIt8PR064j/xsmdEJL/YvY/o=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk=
+golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=
+golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
+golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181021155630-eda9bb28ed51/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM=
+golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0=
+golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc=
+golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20200425043458-8463f397d07c/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.9.1 h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo=
+golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
+google.golang.org/api v0.21.0 h1:zS+Q/CJJnVlXpXQVIz+lH0ZT2lBuT2ac7XD8Y/3w6hY=
+google.golang.org/api v0.21.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM=
+google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200409111301-baae70f3302d h1:I7Vuu5Ejagca+VcgfBINHke3xwjCTYnIG4Q57fv0wYY=
+google.golang.org/genproto v0.0.0-20200409111301-baae70f3302d/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.28.1 h1:C1QC6KzgSiLyBabDi87BbjaGreoRgGUF5nOyvfrAZ1k=
+google.golang.org/grpc v1.28.1/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=
+google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
+storj.io/common v0.0.0-20230920095429-0ce0a575e6f8 h1:i+bWPhVnNL6z/TLW3vDZytB6/0bsvJM0a1GhLCxrlxQ=
+storj.io/common v0.0.0-20230920095429-0ce0a575e6f8/go.mod h1:ZmeGPzRb2sm705Nwt/WwuH3e6mliShfvvoUNy1bb9v4=
+storj.io/drpc v0.0.33 h1:yCGZ26r66ZdMP0IcTYsj7WDAUIIjzXk6DJhbhvt9FHI=
+storj.io/drpc v0.0.33/go.mod h1:vR804UNzhBa49NOJ6HeLjd2H3MakC1j5Gv8bsOQT6N4=
+storj.io/picobuf v0.0.2-0.20230906122608-c4ba17033c6c h1:or/DtG5uaZpzimL61ahlgAA+MTYn/U3txz4fe+XBFUg=
+storj.io/picobuf v0.0.2-0.20230906122608-c4ba17033c6c/go.mod h1:JCuc3C0gzCJHQ4J6SOx/Yjg+QTpX0D+Fvs5H46FETCk=
+storj.io/uplink v1.12.1 h1:bDc2dI6Q7EXcvPJLZuH9jIOTIf2oKxvW3xKEA+Y5EI0=
+storj.io/uplink v1.12.1/go.mod h1:1+czctHG25pMzcUp4Mds6QnoJ7LvbgYA5d1qlpFFexg=

src/duplicacy_acdclient_test.go

@@ -1,153 +0,0 @@
-// Copyright (c) Acrosync LLC. All rights reserved.
-// Free for personal use and commercial trial
-// Commercial use requires per-user licenses available from https://duplicacy.com
-
-package duplicacy
-
-import (
-	"crypto/sha256"
-	"encoding/hex"
-	"fmt"
-	"io"
-	"testing"
-
-	crypto_rand "crypto/rand"
-	"math/rand"
-)
-
-func TestACDClient(t *testing.T) {
-
-	acdClient, err := NewACDClient("acd-token.json")
-	if err != nil {
-		t.Errorf("Failed to create the ACD client: %v", err)
-		return
-	}
-
-	acdClient.TestMode = true
-
-	rootID, _, _, err := acdClient.ListByName("", "")
-	if err != nil {
-		t.Errorf("Failed to get the root node: %v", err)
-		return
-	}
-
-	if rootID == "" {
-		t.Errorf("No root node")
-		return
-	}
-
-	testID, _, _, err := acdClient.ListByName(rootID, "test")
-	if err != nil {
-		t.Errorf("Failed to list the test directory: %v", err)
-		return
-	}
-	if testID == "" {
-		testID, err = acdClient.CreateDirectory(rootID, "test")
-		if err != nil {
-			t.Errorf("Failed to create the test directory: %v", err)
-			return
-		}
-	}
-
-	test1ID, _, _, err := acdClient.ListByName(testID, "test1")
-	if err != nil {
-		t.Errorf("Failed to list the test1 directory: %v", err)
-		return
-	}
-	if test1ID == "" {
-		test1ID, err = acdClient.CreateDirectory(testID, "test1")
-		if err != nil {
-			t.Errorf("Failed to create the test1 directory: %v", err)
-			return
-		}
-	}
-
-	test2ID, _, _, err := acdClient.ListByName(testID, "test2")
-	if err != nil {
-		t.Errorf("Failed to list the test2 directory: %v", err)
-		return
-	}
-	if test2ID == "" {
-		test2ID, err = acdClient.CreateDirectory(testID, "test2")
-		if err != nil {
-			t.Errorf("Failed to create the test2 directory: %v", err)
-			return
-		}
-	}
-
-	fmt.Printf("test1: %s, test2: %s\n", test1ID, test2ID)
-
-	numberOfFiles := 20
-	maxFileSize := 64 * 1024
-
-	for i := 0; i < numberOfFiles; i++ {
-		content := make([]byte, rand.Int()%maxFileSize+1)
-		_, err = crypto_rand.Read(content)
-		if err != nil {
-			t.Errorf("Error generating random content: %v", err)
-			return
-		}
-
-		hasher := sha256.New()
-		hasher.Write(content)
-		filename := hex.EncodeToString(hasher.Sum(nil))
-
-		fmt.Printf("file: %s\n", filename)
-
-		_, err = acdClient.UploadFile(test1ID, filename, content, 100)
-		if err != nil {
-			/*if e, ok := err.(ACDError); !ok || e.Status != 409 */ {
-				t.Errorf("Failed to upload the file %s: %v", filename, err)
-				return
-			}
-		}
-	}
-
-	entries, err := acdClient.ListEntries(test1ID, true, false)
-	if err != nil {
-		t.Errorf("Error list randomly generated files: %v", err)
-		return
-	}
-
-	for _, entry := range entries {
-		err = acdClient.MoveFile(entry.ID, test1ID, test2ID)
-		if err != nil {
-			t.Errorf("Failed to move %s: %v", entry.Name, err)
-			return
-		}
-	}
-
-	entries, err = acdClient.ListEntries(test2ID, true, false)
-	if err != nil {
-		t.Errorf("Error list randomly generated files: %v", err)
-		return
-	}
-
-	for _, entry := range entries {
-		readCloser, _, err := acdClient.DownloadFile(entry.ID)
-		if err != nil {
-			t.Errorf("Error downloading file %s: %v", entry.Name, err)
-			return
-		}
-
-		hasher := sha256.New()
-		io.Copy(hasher, readCloser)
-		hash := hex.EncodeToString(hasher.Sum(nil))
-
-		if hash != entry.Name {
-			t.Errorf("File %s, hash %s", entry.Name, hash)
-		}
-
-		readCloser.Close()
-	}
-
-	for _, entry := range entries {
-
-		err = acdClient.DeleteFile(entry.ID)
-		if err != nil {
-			t.Errorf("Failed to delete the file %s: %v", entry.Name, err)
-			return
-		}
-	}
-
-}

src/duplicacy_acdstorage.go

@@ -55,7 +55,7 @@ func CreateACDStorage(tokenFile string, storagePath string, threads int) (storag
 				return nil, err
 			}
 		} else if !isDir {
-			return nil, fmt.Errorf("%s/%s is not a directory", storagePath+"/"+dir)
+			return nil, fmt.Errorf("%s is not a directory", storagePath+"/"+dir)
 		}
 		storage.idCache[dir] = dirID
 	}

src/duplicacy_azurestorage.go

@@ -104,7 +104,7 @@ func (azureStorage *AzureStorage) ListFiles(threadIndex int, dir string) (files
 
 	if dir == "snapshots/" {
 
-		for subDir, _ := range subDirs {
+		for subDir := range subDirs {
 			files = append(files, subDir)
 		}
 
@@ -166,9 +166,21 @@ func (storage *AzureStorage) DownloadFile(threadIndex int, filePath string, chun
 
 // UploadFile writes 'content' to the file at 'filePath'.
 func (storage *AzureStorage) UploadFile(threadIndex int, filePath string, content []byte) (err error) {
-	reader := CreateRateLimitedReader(content, storage.UploadRateLimit/len(storage.containers))
-	blob := storage.containers[threadIndex].GetBlobReference(filePath)
-	return blob.CreateBlockBlobFromReader(reader, nil)
+
+	tries := 0
+
+	for {
+		reader := CreateRateLimitedReader(content, storage.UploadRateLimit/len(storage.containers))
+		blob := storage.containers[threadIndex].GetBlobReference(filePath)
+		err = blob.CreateBlockBlobFromReader(reader, nil)
+
+		if err == nil || !strings.Contains(err.Error(), "write: broken pipe") || tries >= 3 {
+			return err
+		}
+
+		LOG_INFO("AZURE_RETRY", "Connection unexpectedly terminated: %v; retrying", err)
+		tries++
+	}
 
 }
 

src/duplicacy_b2client.go

@@ -5,19 +5,22 @@
 package duplicacy
 
 import (
-	"bytes"
-	"crypto/sha1"
-	"encoding/base64"
-	"encoding/hex"
-	"encoding/json"
-	"fmt"
 	"io"
-	"io/ioutil"
-	"math/rand"
-	"net/http"
+	"os"
+	"fmt"
+	"bytes"
+	"time"
+	"sync"
 	"strconv"
 	"strings"
-	"time"
+	"net/url"
+	"net/http"
+	"math/rand"
+	"io/ioutil"
+	"crypto/sha1"
+	"encoding/hex"
+	"encoding/json"
+	"encoding/base64"
 )
 
 type B2Error struct {
@@ -39,66 +42,115 @@ var B2AuthorizationURL = "https://api.backblazeb2.com/b2api/v1/b2_authorize_acco
 
 type B2Client struct {
 	HTTPClient         *http.Client
+
 	AccountID          string
+	ApplicationKeyID   string
 	ApplicationKey     string
+	BucketName         string
+	BucketID           string
+	StorageDir         string
+
+	Lock               sync.Mutex
 	AuthorizationToken string
 	APIURL             string
 	DownloadURL        string
-	BucketName         string
-	BucketID           string
+	IsAuthorized       bool
 
-	UploadURL   string
-	UploadToken string
+	UploadURLs         []string
+	UploadTokens       []string
 
-	TestMode bool
+	Threads            int
+	MaximumRetries     int
+	TestMode           bool
+
+	LastAuthorizationTime int64
 }
 
-func NewB2Client(accountID string, applicationKey string) *B2Client {
+// URL encode the given path but keep the slashes intact
+func B2Escape(path string) string {
+	var components []string
+	for _, c := range strings.Split(path, "/") {
+		components = append(components, url.QueryEscape(c))
+	}
+	return strings.Join(components, "/")
+}
+
+func NewB2Client(applicationKeyID string, applicationKey string, downloadURL string, storageDir string, threads int) *B2Client {
+
+	for storageDir != "" && storageDir[0] == '/' {
+		storageDir = storageDir[1:]
+	}
+
+	if storageDir != "" && storageDir[len(storageDir) - 1] != '/' {
+		storageDir += "/"
+	}
+
+	maximumRetries := 15
+	if value, found := os.LookupEnv("DUPLICACY_B2_RETRIES"); found && value != "" {
+		maximumRetries, _ = strconv.Atoi(value)
+		LOG_INFO("B2_RETRIES", "Setting maximum retries for B2 to %d", maximumRetries)
+	}
+
 	client := &B2Client{
-		HTTPClient:     http.DefaultClient,
-		AccountID:      accountID,
-		ApplicationKey: applicationKey,
+		HTTPClient:       http.DefaultClient,
+		ApplicationKeyID: applicationKeyID,
+		ApplicationKey:   applicationKey,
+		DownloadURL:      downloadURL,
+		StorageDir:       storageDir,
+		UploadURLs:       make([]string, threads),
+		UploadTokens:     make([]string, threads),
+		Threads:          threads,
+		MaximumRetries:   maximumRetries,
 	}
 	return client
 }
 
-func (client *B2Client) retry(backoff int, response *http.Response) int {
+func (client *B2Client) getAPIURL() string {
+	client.Lock.Lock()
+	defer client.Lock.Unlock()
+	return client.APIURL
+}
+
+func (client *B2Client) getDownloadURL() string {
+	client.Lock.Lock()
+	defer client.Lock.Unlock()
+	return client.DownloadURL
+}
+
+func (client *B2Client) retry(retries int, response *http.Response) int {
 	if response != nil {
 		if backoffList, found := response.Header["Retry-After"]; found && len(backoffList) > 0 {
 			retryAfter, _ := strconv.Atoi(backoffList[0])
 			if retryAfter >= 1 {
 				time.Sleep(time.Duration(retryAfter) * time.Second)
-				return 0
+				return 1
 			}
 		}
 	}
-	if backoff == 0 {
-		backoff = 1
-	} else {
-		backoff *= 2
+
+	if retries >= client.MaximumRetries + 1 {
+		return 0
 	}
-	time.Sleep(time.Duration(backoff) * time.Second)
-	return backoff
+	retries++
+	delay := 1 << uint(retries)
+	if delay > 64 {
+		delay = 64
+	}
+	delayInSeconds := (rand.Float32() + 1.0) * float32(delay) / 2.0
+
+	time.Sleep(time.Duration(delayInSeconds) * time.Second)
+	return retries
 }
 
-func (client *B2Client) call(url string, method string, requestHeaders map[string]string, input interface{}) (io.ReadCloser, http.Header, int64, error) {
-
-	switch method {
-	case http.MethodGet:
-		break
-	case http.MethodHead:
-		break
-	case http.MethodPost:
-		break
-	default:
-		return nil, nil, 0, fmt.Errorf("unhandled http request method: " + method)
-	}
+func (client *B2Client) call(threadIndex int, requestURL string, method string, requestHeaders map[string]string, input interface{}) (
+	                         io.ReadCloser, http.Header, int64, error) {
 
 	var response *http.Response
 
-	backoff := 0
-	for i := 0; i < 8; i++ {
-		var inputReader *bytes.Reader
+	retries := 0
+	for {
+		var inputReader io.Reader
+		isUpload := false
 
 		switch input.(type) {
 		default:
@@ -107,21 +159,43 @@ func (client *B2Client) call(url string, method string, requestHeaders map[strin
 				return nil, nil, 0, err
 			}
 			inputReader = bytes.NewReader(jsonInput)
-		case []byte:
-			inputReader = bytes.NewReader(input.([]byte))
 		case int:
 			inputReader = bytes.NewReader([]byte(""))
+		case []byte:
+			isUpload = true
+			inputReader = bytes.NewReader(input.([]byte))
+		case *RateLimitedReader:
+			isUpload = true
+			rateLimitedReader := input.(*RateLimitedReader)
+			rateLimitedReader.Reset()
+			inputReader = rateLimitedReader
 		}
 
-		request, err := http.NewRequest(method, url, inputReader)
+
+		if isUpload {
+			if client.UploadURLs[threadIndex] == "" || client.UploadTokens[threadIndex] == "" {
+				err := client.getUploadURL(threadIndex)
+				if err != nil {
+					return nil, nil, 0, err
+				}
+			}
+			requestURL = client.UploadURLs[threadIndex]
+		}
+
+		request, err := http.NewRequest(method, requestURL, inputReader)
 		if err != nil {
 			return nil, nil, 0, err
 		}
 
-		if url == B2AuthorizationURL {
-			request.Header.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(client.AccountID+":"+client.ApplicationKey)))
+		if requestURL == B2AuthorizationURL {
+			request.Header.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(client.ApplicationKeyID+":"+client.ApplicationKey)))
+		} else if isUpload {
+			request.ContentLength, _ = strconv.ParseInt(requestHeaders["Content-Length"], 10, 64)
+			request.Header.Set("Authorization", client.UploadTokens[threadIndex])
 		} else {
+			client.Lock.Lock()
 			request.Header.Set("Authorization", client.AuthorizationToken)
+			client.Lock.Unlock()
 		}
 
 		if requestHeaders != nil {
@@ -132,7 +206,9 @@ func (client *B2Client) call(url string, method string, requestHeaders map[strin
 
 		if client.TestMode {
 			r := rand.Float32()
-			if r < 0.5 {
+			if r < 0.5 && isUpload {
+				request.Header.Set("X-Bz-Test-Mode", "fail_some_uploads")
+			} else if r < 0.75 {
 				request.Header.Set("X-Bz-Test-Mode", "expire_some_account_authorization_tokens")
 			} else {
 				request.Header.Set("X-Bz-Test-Mode", "force_cap_exceeded")
@@ -141,28 +217,51 @@ func (client *B2Client) call(url string, method string, requestHeaders map[strin
 
 		response, err = client.HTTPClient.Do(request)
 		if err != nil {
-			if url != B2AuthorizationURL {
-				LOG_DEBUG("BACKBLAZE_CALL", "URL request '%s' returned an error: %v", url, err)
-				backoff = client.retry(backoff, response)
-				continue
+
+			// Don't retry when the first authorization request fails
+			if requestURL == B2AuthorizationURL && !client.IsAuthorized {
+				return nil, nil, 0, err
 			}
-			return nil, nil, 0, err
+
+			LOG_TRACE("BACKBLAZE_CALL", "[%d] URL request '%s' returned an error: %v", threadIndex, requestURL, err)
+
+			retries = client.retry(retries, response)
+			if retries <= 0 {
+				return nil, nil, 0, err
+			}
+
+			// Clear the upload url to requrest a new one on retry
+			if isUpload {
+				client.UploadURLs[threadIndex] = ""
+				client.UploadTokens[threadIndex] = ""
+			}
+			continue
+
 		}
 
 		if response.StatusCode < 300 {
 			return response.Body, response.Header, response.ContentLength, nil
 		}
 
-		LOG_DEBUG("BACKBLAZE_CALL", "URL request '%s' returned status code %d", url, response.StatusCode)
+		e := &B2Error{}
+		if err := json.NewDecoder(response.Body).Decode(e); err != nil {
+			LOG_TRACE("BACKBLAZE_CALL", "[%d] URL request '%s %s' returned status code %d", threadIndex, method, requestURL, response.StatusCode)
+		} else {
+			LOG_TRACE("BACKBLAZE_CALL", "[%d] URL request '%s %s' returned %d %s", threadIndex, method, requestURL, response.StatusCode, e.Message)
+		}
 
-		io.Copy(ioutil.Discard, response.Body)
 		response.Body.Close()
+
 		if response.StatusCode == 401 {
-			if url == B2AuthorizationURL {
+			if requestURL == B2AuthorizationURL {
 				return nil, nil, 0, fmt.Errorf("Authorization failure")
 			}
-			client.AuthorizeAccount()
-			continue
+
+			// Attempt authorization again.  If authorization is actually not done, run the random backoff
+			_, allowed := client.AuthorizeAccount(threadIndex)
+			if allowed {
+				continue
+			}
 		} else if response.StatusCode == 403 {
 			if !client.TestMode {
 				return nil, nil, 0, fmt.Errorf("B2 cap exceeded")
@@ -170,38 +269,26 @@ func (client *B2Client) call(url string, method string, requestHeaders map[strin
 			continue
 		} else if response.StatusCode == 404 {
 			if http.MethodHead == method {
-				LOG_DEBUG("BACKBLAZE_CALL", "URL request '%s' returned status code %d", url, response.StatusCode)
 				return nil, nil, 0, nil
 			}
 		} else if response.StatusCode == 416 {
 			if http.MethodHead == method {
 				// 416 Requested Range Not Satisfiable
-				return nil, nil, 0, fmt.Errorf("URL request '%s' returned status code %d", url, response.StatusCode)
+				return nil, nil, 0, fmt.Errorf("URL request '%s' returned %d %s", requestURL, response.StatusCode, e.Message)
 			}
-		} else if response.StatusCode == 429 || response.StatusCode == 408 {
-			backoff = client.retry(backoff, response)
-			continue
-		} else if response.StatusCode >= 500 && response.StatusCode <= 599 {
-			backoff = client.retry(backoff, response)
-			continue
-		} else {
-			LOG_INFO("BACKBLAZE_CALL", "URL request '%s' returned status code %d", url, response.StatusCode)
-			backoff = client.retry(backoff, response)
-			continue
 		}
 
-		defer response.Body.Close()
-
-		e := &B2Error{}
-
-		if err := json.NewDecoder(response.Body).Decode(e); err != nil {
-			return nil, nil, 0, err
+		retries = client.retry(retries, response)
+		if retries <= 0 {
+			return nil, nil, 0, fmt.Errorf("URL request '%s' returned %d %s", requestURL, response.StatusCode, e.Message)
 		}
 
-		return nil, nil, 0, e
+		if isUpload {
+			client.UploadURLs[threadIndex] = ""
+			client.UploadTokens[threadIndex] = ""
+		}
 	}
 
-	return nil, nil, 0, fmt.Errorf("Maximum backoff reached")
 }
 
 type B2AuthorizeAccountOutput struct {
@@ -211,11 +298,18 @@ type B2AuthorizeAccountOutput struct {
 	DownloadURL        string
 }
 
-func (client *B2Client) AuthorizeAccount() (err error) {
+func (client *B2Client) AuthorizeAccount(threadIndex int) (err error, allowed bool) {
+	client.Lock.Lock()
+	defer client.Lock.Unlock()
 
-	readCloser, _, _, err := client.call(B2AuthorizationURL, http.MethodPost, nil, make(map[string]string))
+	// Don't authorize if the previous one was done less than 30 seconds ago
+	if client.LastAuthorizationTime != 0 && client.LastAuthorizationTime > time.Now().Unix() - 30 {
+		return nil, false
+	}
+
+	readCloser, _, _, err := client.call(threadIndex, B2AuthorizationURL, http.MethodPost, nil, make(map[string]string))
 	if err != nil {
-		return err
+		return err, true
 	}
 
 	defer readCloser.Close()
@@ -223,18 +317,28 @@ func (client *B2Client) AuthorizeAccount() (err error) {
 	output := &B2AuthorizeAccountOutput{}
 
 	if err = json.NewDecoder(readCloser).Decode(&output); err != nil {
-		return err
+		return err, true
 	}
 
+	// The account id may be different from the application key id so we're getting the account id from the returned
+	// json object here, which is needed by the b2_list_buckets call.
+	client.AccountID = output.AccountID
+
 	client.AuthorizationToken = output.AuthorizationToken
 	client.APIURL = output.APIURL
-	client.DownloadURL = output.DownloadURL
+	if client.DownloadURL == "" {
+		client.DownloadURL = output.DownloadURL
+	}
+	LOG_INFO("BACKBLAZE_URL", "Download URL is: %s", client.DownloadURL)
+	client.IsAuthorized = true
 
-	return nil
+	client.LastAuthorizationTime = time.Now().Unix()
+
+	return nil, true
 }
 
 type ListBucketOutput struct {
-	AccoundID  string
+	AccountID  string
 	BucketID   string
 	BucketName string
 	BucketType string
@@ -244,10 +348,11 @@ func (client *B2Client) FindBucket(bucketName string) (err error) {
 
 	input := make(map[string]string)
 	input["accountId"] = client.AccountID
+	input["bucketName"] = bucketName
 
-	url := client.APIURL + "/b2api/v1/b2_list_buckets"
+	url := client.getAPIURL() + "/b2api/v1/b2_list_buckets"
 
-	readCloser, _, _, err := client.call(url, http.MethodPost, nil, input)
+	readCloser, _, _, err := client.call(0, url, http.MethodPost, nil, input)
 	if err != nil {
 		return err
 	}
@@ -289,7 +394,7 @@ type B2ListFileNamesOutput struct {
 	NextFileId   string
 }
 
-func (client *B2Client) ListFileNames(startFileName string, singleFile bool, includeVersions bool) (files []*B2Entry, err error) {
+func (client *B2Client) ListFileNames(threadIndex int, startFileName string, singleFile bool, includeVersions bool) (files []*B2Entry, err error) {
 
 	maxFileCount := 1000
 	if singleFile {
@@ -307,20 +412,21 @@ func (client *B2Client) ListFileNames(startFileName string, singleFile bool, inc
 
 	input := make(map[string]interface{})
 	input["bucketId"] = client.BucketID
-	input["startFileName"] = startFileName
+	input["startFileName"] = client.StorageDir + startFileName
 	input["maxFileCount"] = maxFileCount
+	input["prefix"] = client.StorageDir
 
 	for {
-		url := client.APIURL + "/b2api/v1/b2_list_file_names"
+		apiURL := client.getAPIURL() + "/b2api/v1/b2_list_file_names"
 		requestHeaders := map[string]string{}
 		requestMethod := http.MethodPost
 		var requestInput interface{}
 		requestInput = input
 		if includeVersions {
-			url = client.APIURL + "/b2api/v1/b2_list_file_versions"
+			apiURL = client.getAPIURL() + "/b2api/v1/b2_list_file_versions"
 		} else if singleFile {
 			// handle a single file with no versions as a special case to download the last byte of the file
-			url = client.DownloadURL + "/file/" + client.BucketName + "/" + startFileName
+			apiURL = client.getDownloadURL() + "/file/" + client.BucketName + "/" + B2Escape(client.StorageDir + startFileName)
 			// requesting byte -1 works for empty files where 0-0 fails with a 416 error
 			requestHeaders["Range"] = "bytes=-1"
 			// HEAD request
@@ -330,7 +436,7 @@ func (client *B2Client) ListFileNames(startFileName string, singleFile bool, inc
 		var readCloser io.ReadCloser
 		var responseHeader http.Header
 		var err error
-		readCloser, responseHeader, _, err = client.call(url, requestMethod, requestHeaders, requestInput)
+		readCloser, responseHeader, _, err = client.call(threadIndex, apiURL, requestMethod, requestHeaders, requestInput)
 		if err != nil {
 			return nil, err
 		}
@@ -343,7 +449,7 @@ func (client *B2Client) ListFileNames(startFileName string, singleFile bool, inc
 
 		if singleFile && !includeVersions {
 			if responseHeader == nil {
-				LOG_DEBUG("BACKBLAZE_LIST", "b2_download_file_by_name did not return headers")
+				LOG_DEBUG("BACKBLAZE_LIST", "%s did not return headers", apiURL)
 				return []*B2Entry{}, nil
 			}
 			requiredHeaders := []string{
@@ -357,11 +463,17 @@ func (client *B2Client) ListFileNames(startFileName string, singleFile bool, inc
 				}
 			}
 			if len(missingKeys) > 0 {
-				return nil, fmt.Errorf("b2_download_file_by_name missing headers: %s", missingKeys)
+				return nil, fmt.Errorf("%s missing headers: %s", apiURL, missingKeys)
 			}
 			// construct the B2Entry from the response headers of the download request
 			fileID := responseHeader.Get("x-bz-file-id")
 			fileName := responseHeader.Get("x-bz-file-name")
+			unescapedFileName, err := url.QueryUnescape(fileName)
+			if err == nil {
+				fileName = unescapedFileName
+			} else {
+				LOG_WARN("BACKBLAZE_UNESCAPE", "Failed to unescape the file name %s", fileName)
+			}
 			fileAction := "upload"
 			// byte range that is returned: "bytes #-#/#
 			rangeString := responseHeader.Get("Content-Range")
@@ -374,14 +486,14 @@ func (client *B2Client) ListFileNames(startFileName string, singleFile bool, inc
 				// this should only execute if the requested file is empty and the range request didn't result in a Content-Range header
 				fileSize, _ = strconv.ParseInt(lengthString, 0, 64)
 				if fileSize != 0 {
-					return nil, fmt.Errorf("b2_download_file_by_name returned non-zero file length")
+					return nil, fmt.Errorf("%s returned non-zero file length", apiURL)
 				}
 			} else {
-				return nil, fmt.Errorf("could not parse b2_download_file_by_name headers")
+				return nil, fmt.Errorf("could not parse headers returned by %s", apiURL)
 			}
 			fileUploadTimestamp, _ := strconv.ParseInt(responseHeader.Get("X-Bz-Upload-Timestamp"), 0, 64)
 
-			return []*B2Entry{&B2Entry{fileID, fileName, fileAction, fileSize, fileUploadTimestamp}}, nil
+			return []*B2Entry{{fileID, fileName[len(client.StorageDir):], fileAction, fileSize, fileUploadTimestamp}}, nil
 		}
 
 		if err = json.NewDecoder(readCloser).Decode(&output); err != nil {
@@ -390,31 +502,27 @@ func (client *B2Client) ListFileNames(startFileName string, singleFile bool, inc
 
 		ioutil.ReadAll(readCloser)
 
-		if startFileName == "" {
-			files = append(files, output.Files...)
-		} else {
-			for _, file := range output.Files {
-				if singleFile {
-					if file.FileName == startFileName {
-						files = append(files, file)
-						if !includeVersions {
-							output.NextFileName = ""
-							break
-						}
-					} else {
+		for _, file := range output.Files {
+			file.FileName = file.FileName[len(client.StorageDir):]
+			if singleFile {
+				if file.FileName == startFileName {
+					files = append(files, file)
+					if !includeVersions {
 						output.NextFileName = ""
 						break
 					}
 				} else {
-					if strings.HasPrefix(file.FileName, startFileName) {
-						files = append(files, file)
-					} else {
-						output.NextFileName = ""
-						break
-					}
+					output.NextFileName = ""
+					break
+				}
+			} else {
+				if strings.HasPrefix(file.FileName, startFileName) {
+					files = append(files, file)
+				} else {
+					output.NextFileName = ""
+					break
 				}
 			}
-
 		}
 
 		if len(output.NextFileName) == 0 {
@@ -430,14 +538,14 @@ func (client *B2Client) ListFileNames(startFileName string, singleFile bool, inc
 	return files, nil
 }
 
-func (client *B2Client) DeleteFile(fileName string, fileID string) (err error) {
+func (client *B2Client) DeleteFile(threadIndex int, fileName string, fileID string) (err error) {
 
 	input := make(map[string]string)
-	input["fileName"] = fileName
+	input["fileName"] = client.StorageDir + fileName
 	input["fileId"] = fileID
 
-	url := client.APIURL + "/b2api/v1/b2_delete_file_version"
-	readCloser, _, _, err := client.call(url, http.MethodPost, make(map[string]string), input)
+	url := client.getAPIURL() + "/b2api/v1/b2_delete_file_version"
+	readCloser, _, _, err := client.call(threadIndex, url, http.MethodPost, make(map[string]string), input)
 	if err != nil {
 		return err
 	}
@@ -450,14 +558,14 @@ type B2HideFileOutput struct {
 	FileID string
 }
 
-func (client *B2Client) HideFile(fileName string) (fileID string, err error) {
+func (client *B2Client) HideFile(threadIndex int, fileName string) (fileID string, err error) {
 
 	input := make(map[string]string)
 	input["bucketId"] = client.BucketID
-	input["fileName"] = fileName
+	input["fileName"] = client.StorageDir + fileName
 
-	url := client.APIURL + "/b2api/v1/b2_hide_file"
-	readCloser, _, _, err := client.call(url, http.MethodPost, make(map[string]string), input)
+	url := client.getAPIURL() + "/b2api/v1/b2_hide_file"
+	readCloser, _, _, err := client.call(threadIndex, url, http.MethodPost, make(map[string]string), input)
 	if err != nil {
 		return "", err
 	}
@@ -474,11 +582,29 @@ func (client *B2Client) HideFile(fileName string) (fileID string, err error) {
 	return output.FileID, nil
 }
 
-func (client *B2Client) DownloadFile(filePath string) (io.ReadCloser, int64, error) {
+func (client *B2Client) DownloadFile(threadIndex int, filePath string) (io.ReadCloser, int64, error) {
 
-	url := client.DownloadURL + "/file/" + client.BucketName + "/" + filePath
+	if !strings.HasSuffix(filePath, ".fsl") {
+		url := client.getDownloadURL() + "/file/" + client.BucketName + "/" + B2Escape(client.StorageDir + filePath)
 
-	readCloser, _, len, err := client.call(url, http.MethodGet, make(map[string]string), 0)
+		readCloser, _, len, err := client.call(threadIndex, url, http.MethodGet, make(map[string]string), 0)
+		return readCloser, len, err
+	}
+
+	// We're trying to download a fossil file.  We need to find the file ID of the last 'upload' of the file.
+	filePath = strings.TrimSuffix(filePath, ".fsl")
+	entries, err := client.ListFileNames(threadIndex, filePath, true, true)
+	fileId := ""
+	for _, entry := range entries {
+		if entry.FileName == filePath && entry.Action == "upload" && entry.Size > 0 {
+			fileId = entry.FileID
+			break
+		}
+	}
+
+	// Proceed with the b2_download_file_by_id call
+	url := client.getAPIURL() + "/b2api/v1/b2_download_file_by_id?fileId=" + fileId
+	readCloser, _, len, err := client.call(threadIndex, url, http.MethodGet, make(map[string]string), 0)
 	return readCloser, len, err
 }
 
@@ -488,12 +614,12 @@ type B2GetUploadArgumentOutput struct {
 	AuthorizationToken string
 }
 
-func (client *B2Client) getUploadURL() error {
+func (client *B2Client) getUploadURL(threadIndex int) error {
 	input := make(map[string]string)
 	input["bucketId"] = client.BucketID
 
-	url := client.APIURL + "/b2api/v1/b2_get_upload_url"
-	readCloser, _, _, err := client.call(url, http.MethodPost, make(map[string]string), input)
+	url := client.getAPIURL() + "/b2api/v1/b2_get_upload_url"
+	readCloser, _, _, err := client.call(threadIndex, url, http.MethodPost, make(map[string]string), input)
 	if err != nil {
 		return err
 	}
@@ -506,96 +632,29 @@ func (client *B2Client) getUploadURL() error {
 		return err
 	}
 
-	client.UploadURL = output.UploadURL
-	client.UploadToken = output.AuthorizationToken
+	client.UploadURLs[threadIndex] = output.UploadURL
+	client.UploadTokens[threadIndex] = output.AuthorizationToken
 
 	return nil
 }
 
-func (client *B2Client) UploadFile(filePath string, content []byte, rateLimit int) (err error) {
+func (client *B2Client) UploadFile(threadIndex int, filePath string, content []byte, rateLimit int) (err error) {
 
 	hasher := sha1.New()
 	hasher.Write(content)
 	hash := hex.EncodeToString(hasher.Sum(nil))
 
 	headers := make(map[string]string)
-	headers["X-Bz-File-Name"] = filePath
+	headers["X-Bz-File-Name"] = B2Escape(client.StorageDir + filePath)
+	headers["Content-Length"] = fmt.Sprintf("%d", len(content))
 	headers["Content-Type"] = "application/octet-stream"
 	headers["X-Bz-Content-Sha1"] = hash
 
-	var response *http.Response
-
-	backoff := 0
-	for i := 0; i < 8; i++ {
-
-		if client.UploadURL == "" || client.UploadToken == "" {
-			err = client.getUploadURL()
-			if err != nil {
-				return err
-			}
-		}
-
-		request, err := http.NewRequest("POST", client.UploadURL, CreateRateLimitedReader(content, rateLimit))
-		if err != nil {
-			return err
-		}
-		request.ContentLength = int64(len(content))
-
-		request.Header.Set("Authorization", client.UploadToken)
-		request.Header.Set("X-Bz-File-Name", filePath)
-		request.Header.Set("Content-Type", "application/octet-stream")
-		request.Header.Set("X-Bz-Content-Sha1", hash)
-
-		for key, value := range headers {
-			request.Header.Set(key, value)
-		}
-
-		if client.TestMode {
-			r := rand.Float32()
-			if r < 0.8 {
-				request.Header.Set("X-Bz-Test-Mode", "fail_some_uploads")
-			} else if r < 0.9 {
-				request.Header.Set("X-Bz-Test-Mode", "expire_some_account_authorization_tokens")
-			} else {
-				request.Header.Set("X-Bz-Test-Mode", "force_cap_exceeded")
-			}
-		}
-
-		response, err = client.HTTPClient.Do(request)
-		if err != nil {
-			LOG_DEBUG("BACKBLAZE_UPLOAD", "URL request '%s' returned an error: %v", client.UploadURL, err)
-			backoff = client.retry(backoff, response)
-			client.UploadURL = ""
-			client.UploadToken = ""
-			continue
-		}
-
-		io.Copy(ioutil.Discard, response.Body)
-		response.Body.Close()
-
-		if response.StatusCode < 300 {
-			return nil
-		}
-
-		LOG_DEBUG("BACKBLAZE_UPLOAD", "URL request '%s' returned status code %d", client.UploadURL, response.StatusCode)
-
-		if response.StatusCode == 401 {
-			LOG_INFO("BACKBLAZE_UPLOAD", "Re-authorizatoin required")
-			client.UploadURL = ""
-			client.UploadToken = ""
-			continue
-		} else if response.StatusCode == 403 {
-			if !client.TestMode {
-				return fmt.Errorf("B2 cap exceeded")
-			}
-			continue
-		} else {
-			LOG_INFO("BACKBLAZE_UPLOAD", "URL request '%s' returned status code %d", client.UploadURL, response.StatusCode)
-			backoff = client.retry(backoff, response)
-			client.UploadURL = ""
-			client.UploadToken = ""
-		}
+	readCloser, _, _, err := client.call(threadIndex, "", http.MethodPost, headers, CreateRateLimitedReader(content, rateLimit))
+	if err != nil {
+		return err
 	}
 
-	return fmt.Errorf("Maximum backoff reached")
+	readCloser.Close()
+	return nil
 }

src/duplicacy_b2client_test.go

@@ -1,133 +0,0 @@
-// Copyright (c) Acrosync LLC. All rights reserved.
-// Free for personal use and commercial trial
-// Commercial use requires per-user licenses available from https://duplicacy.com
-
-package duplicacy
-
-import (
-	"crypto/sha256"
-	"encoding/hex"
-	"encoding/json"
-	"testing"
-
-	crypto_rand "crypto/rand"
-	"io"
-	"io/ioutil"
-	"math/rand"
-)
-
-func createB2ClientForTest(t *testing.T) (*B2Client, string) {
-	config, err := ioutil.ReadFile("test_storage.conf")
-	if err != nil {
-		t.Errorf("Failed to read config file: %v", err)
-		return nil, ""
-	}
-
-	storages := make(map[string]map[string]string)
-
-	err = json.Unmarshal(config, &storages)
-	if err != nil {
-		t.Errorf("Failed to parse config file: %v", err)
-		return nil, ""
-	}
-
-	b2, found := storages["b2"]
-	if !found {
-		t.Errorf("Failed to find b2 config")
-		return nil, ""
-	}
-
-	return NewB2Client(b2["account"], b2["key"]), b2["bucket"]
-
-}
-
-func TestB2Client(t *testing.T) {
-
-	b2Client, bucket := createB2ClientForTest(t)
-	if b2Client == nil {
-		return
-	}
-
-	b2Client.TestMode = true
-
-	err := b2Client.AuthorizeAccount()
-	if err != nil {
-		t.Errorf("Failed to authorize the b2 account: %v", err)
-		return
-	}
-
-	err = b2Client.FindBucket(bucket)
-	if err != nil {
-		t.Errorf("Failed to find bucket '%s': %v", bucket, err)
-		return
-	}
-
-	testDirectory := "b2client_test/"
-
-	files, err := b2Client.ListFileNames(testDirectory, false, false)
-	if err != nil {
-		t.Errorf("Failed to list files: %v", err)
-		return
-	}
-
-	for _, file := range files {
-		err = b2Client.DeleteFile(file.FileName, file.FileID)
-		if err != nil {
-			t.Errorf("Failed to delete file '%s': %v", file.FileName, err)
-		}
-	}
-
-	maxSize := 10000
-	for i := 0; i < 20; i++ {
-		size := rand.Int()%maxSize + 1
-		content := make([]byte, size)
-		_, err := crypto_rand.Read(content)
-		if err != nil {
-			t.Errorf("Error generating random content: %v", err)
-			return
-		}
-
-		hash := sha256.Sum256(content)
-		name := hex.EncodeToString(hash[:])
-
-		err = b2Client.UploadFile(testDirectory+name, content, 100)
-		if err != nil {
-			t.Errorf("Error uploading file '%s': %v", name, err)
-			return
-		}
-	}
-
-	files, err = b2Client.ListFileNames(testDirectory, false, false)
-	if err != nil {
-		t.Errorf("Failed to list files: %v", err)
-		return
-	}
-
-	for _, file := range files {
-
-		readCloser, _, err := b2Client.DownloadFile(file.FileName)
-		if err != nil {
-			t.Errorf("Error downloading file '%s': %v", file.FileName, err)
-			return
-		}
-
-		defer readCloser.Close()
-
-		hasher := sha256.New()
-		_, err = io.Copy(hasher, readCloser)
-
-		hash := hex.EncodeToString(hasher.Sum(nil))
-
-		if testDirectory+hash != file.FileName {
-			t.Errorf("File %s has hash %s", file.FileName, hash)
-		}
-
-	}
-
-	for _, file := range files {
-		err = b2Client.DeleteFile(file.FileName, file.FileID)
-		if err != nil {
-			t.Errorf("Failed to delete file '%s': %v", file.FileName, err)
-		}
-	}
-}

src/duplicacy_b2storage.go

@@ -11,32 +11,26 @@ import (
 type B2Storage struct {
 	StorageBase
 
-	clients []*B2Client
+	client *B2Client
 }
 
 // CreateB2Storage creates a B2 storage object.
-func CreateB2Storage(accountID string, applicationKey string, bucket string, threads int) (storage *B2Storage, err error) {
+func CreateB2Storage(accountID string, applicationKey string, downloadURL string, bucket string, storageDir string, threads int) (storage *B2Storage, err error) {
 
-	var clients []*B2Client
+	client := NewB2Client(accountID, applicationKey, downloadURL, storageDir, threads)
 
-	for i := 0; i < threads; i++ {
-		client := NewB2Client(accountID, applicationKey)
+	err, _ = client.AuthorizeAccount(0)
+	if err != nil {
+		return nil, err
+	}
 
-		err = client.AuthorizeAccount()
-		if err != nil {
-			return nil, err
-		}
-
-		err = client.FindBucket(bucket)
-		if err != nil {
-			return nil, err
-		}
-
-		clients = append(clients, client)
+	err = client.FindBucket(bucket)
+	if err != nil {
+		return nil, err
 	}
 
 	storage = &B2Storage{
-		clients: clients,
+		client: client,
 	}
 
 	storage.DerivedStorage = storage
@@ -56,7 +50,7 @@ func (storage *B2Storage) ListFiles(threadIndex int, dir string) (files []string
 		includeVersions = true
 	}
 
-	entries, err := storage.clients[threadIndex].ListFileNames(dir, false, includeVersions)
+	entries, err := storage.client.ListFileNames(threadIndex, dir, false, includeVersions)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -71,7 +65,7 @@ func (storage *B2Storage) ListFiles(threadIndex int, dir string) (files []string
 			subDirs[subDir+"/"] = true
 		}
 
-		for subDir, _ := range subDirs {
+		for subDir := range subDirs {
 			files = append(files, subDir)
 		}
 	} else if dir == "chunks" {
@@ -102,7 +96,7 @@ func (storage *B2Storage) DeleteFile(threadIndex int, filePath string) (err erro
 
 	if strings.HasSuffix(filePath, ".fsl") {
 		filePath = filePath[:len(filePath)-len(".fsl")]
-		entries, err := storage.clients[threadIndex].ListFileNames(filePath, true, true)
+		entries, err := storage.client.ListFileNames(threadIndex, filePath, true, true)
 		if err != nil {
 			return err
 		}
@@ -116,7 +110,7 @@ func (storage *B2Storage) DeleteFile(threadIndex int, filePath string) (err erro
 
 			toBeDeleted = true
 
-			err = storage.clients[threadIndex].DeleteFile(filePath, entry.FileID)
+			err = storage.client.DeleteFile(threadIndex, filePath, entry.FileID)
 			if err != nil {
 				return err
 			}
@@ -125,7 +119,7 @@ func (storage *B2Storage) DeleteFile(threadIndex int, filePath string) (err erro
 		return nil
 
 	} else {
-		entries, err := storage.clients[threadIndex].ListFileNames(filePath, true, false)
+		entries, err := storage.client.ListFileNames(threadIndex, filePath, true, false)
 		if err != nil {
 			return err
 		}
@@ -133,7 +127,7 @@ func (storage *B2Storage) DeleteFile(threadIndex int, filePath string) (err erro
 		if len(entries) == 0 {
 			return nil
 		}
-		return storage.clients[threadIndex].DeleteFile(filePath, entries[0].FileID)
+		return storage.client.DeleteFile(threadIndex, filePath, entries[0].FileID)
 	}
 }
 
@@ -160,10 +154,10 @@ func (storage *B2Storage) MoveFile(threadIndex int, from string, to string) (err
 	}
 
 	if filePath == from {
-		_, err = storage.clients[threadIndex].HideFile(from)
+		_, err = storage.client.HideFile(threadIndex, from)
 		return err
 	} else {
-		entries, err := storage.clients[threadIndex].ListFileNames(filePath, true, true)
+		entries, err := storage.client.ListFileNames(threadIndex, filePath, true, true)
 		if err != nil {
 			return err
 		}
@@ -171,7 +165,7 @@ func (storage *B2Storage) MoveFile(threadIndex int, from string, to string) (err
 			return nil
 		}
 
-		return storage.clients[threadIndex].DeleteFile(filePath, entries[0].FileID)
+		return storage.client.DeleteFile(threadIndex, filePath, entries[0].FileID)
 	}
 }
 
@@ -188,7 +182,7 @@ func (storage *B2Storage) GetFileInfo(threadIndex int, filePath string) (exist b
 		filePath = filePath[:len(filePath)-len(".fsl")]
 	}
 
-	entries, err := storage.clients[threadIndex].ListFileNames(filePath, true, isFossil)
+	entries, err := storage.client.ListFileNames(threadIndex, filePath, true, isFossil)
 	if err != nil {
 		return false, false, 0, err
 	}
@@ -210,20 +204,20 @@ func (storage *B2Storage) GetFileInfo(threadIndex int, filePath string) (exist b
 // DownloadFile reads the file at 'filePath' into the chunk.
 func (storage *B2Storage) DownloadFile(threadIndex int, filePath string, chunk *Chunk) (err error) {
 
-	readCloser, _, err := storage.clients[threadIndex].DownloadFile(filePath)
+	readCloser, _, err := storage.client.DownloadFile(threadIndex, filePath)
 	if err != nil {
 		return err
 	}
 
 	defer readCloser.Close()
 
-	_, err = RateLimitedCopy(chunk, readCloser, storage.DownloadRateLimit/len(storage.clients))
+	_, err = RateLimitedCopy(chunk, readCloser, storage.DownloadRateLimit/storage.client.Threads)
 	return err
 }
 
 // UploadFile writes 'content' to the file at 'filePath'.
 func (storage *B2Storage) UploadFile(threadIndex int, filePath string, content []byte) (err error) {
-	return storage.clients[threadIndex].UploadFile(filePath, content, storage.UploadRateLimit/len(storage.clients))
+	return storage.client.UploadFile(threadIndex, filePath, content, storage.UploadRateLimit/storage.client.Threads)
 }
 
 // If a local snapshot cache is needed for the storage to avoid downloading/uploading chunks too often when
@@ -241,7 +235,5 @@ func (storage *B2Storage) IsFastListing() bool { return true }
 
 // Enable the test mode.
 func (storage *B2Storage) EnableTestMode() {
-	for _, client := range storage.clients {
-		client.TestMode = true
-	}
+	storage.client.TestMode = true
 }

src/duplicacy_backupmanager_test.go

@@ -169,6 +169,12 @@ func getFileHash(path string) (hash string) {
 	return hex.EncodeToString(hasher.Sum(nil))
 }
 
+func assertRestoreFailures(t *testing.T, failedFiles int, expectedFailedFiles int) {
+	if failedFiles != expectedFailedFiles {
+		t.Errorf("Failed to restore %d instead of %d file(s)", failedFiles, expectedFailedFiles)
+	}
+}
+
 func TestBackupManager(t *testing.T) {
 
 	rand.Seed(time.Now().UnixNano())
@@ -226,12 +232,20 @@ func TestBackupManager(t *testing.T) {
 	cleanStorage(storage)
 
 	time.Sleep(time.Duration(delay) * time.Second)
-	if testFixedChunkSize {
-		if !ConfigStorage(storage, 16384, 100, 64*1024, 64*1024, 64*1024, password, nil) {
+
+	dataShards := 0
+	parityShards := 0
+	if *testErasureCoding {
+		dataShards = 5
+		parityShards = 2
+	}
+
+	if *testFixedChunkSize {
+		if !ConfigStorage(storage, 16384, 100, 64*1024, 64*1024, 64*1024, password, nil, false, "", dataShards, parityShards) {
 			t.Errorf("Failed to initialize the storage")
 		}
 	} else {
-		if !ConfigStorage(storage, 16384, 100, 64*1024, 256*1024, 16*1024, password, nil) {
+		if !ConfigStorage(storage, 16384, 100, 64*1024, 256*1024, 16*1024, password, nil, false, "", dataShards, parityShards) {
 			t.Errorf("Failed to initialize the storage")
 		}
 	}
@@ -239,15 +253,16 @@ func TestBackupManager(t *testing.T) {
 	time.Sleep(time.Duration(delay) * time.Second)
 
 	SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy")
-	backupManager := CreateBackupManager("host1", storage, testDir, password)
+	backupManager := CreateBackupManager("host1", storage, testDir, password, "", "", false)
 	backupManager.SetupSnapshotCache("default")
 
 	SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy")
-	backupManager.Backup(testDir+"/repository1" /*quickMode=*/, true, threads, "first", false, false)
+	backupManager.Backup(testDir+"/repository1" /*quickMode=*/, true, threads, "first", false, false, 0, false, 1024, 1024)
 	time.Sleep(time.Duration(delay) * time.Second)
 	SetDuplicacyPreferencePath(testDir + "/repository2/.duplicacy")
-	backupManager.Restore(testDir+"/repository2", threads, /*inPlace=*/false, /*quickMode=*/false, threads, /*overwrite=*/true,
-		/*deleteMode=*/false, /*setowner=*/false, /*showStatistics=*/false, /*patterns=*/ nil)
+	failedFiles := backupManager.Restore(testDir+"/repository2", threads /*inPlace=*/, false /*quickMode=*/, false, threads /*overwrite=*/, true,
+		/*deleteMode=*/ false /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, nil /*allowFailures=*/, false)
+	assertRestoreFailures(t, failedFiles, 0)
 
 	for _, f := range []string{"file1", "file2", "dir1/file3"} {
 		if _, err := os.Stat(testDir + "/repository2/" + f); os.IsNotExist(err) {
@@ -267,11 +282,12 @@ func TestBackupManager(t *testing.T) {
 	modifyFile(testDir+"/repository1/dir1/file3", 0.3)
 
 	SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy")
-	backupManager.Backup(testDir+"/repository1" /*quickMode=*/, true, threads, "second", false, false)
+	backupManager.Backup(testDir+"/repository1" /*quickMode=*/, true, threads, "second", false, false, 0, false, 1024, 1024)
 	time.Sleep(time.Duration(delay) * time.Second)
 	SetDuplicacyPreferencePath(testDir + "/repository2/.duplicacy")
-	backupManager.Restore(testDir+"/repository2", 2, /*inPlace=*/true, /*quickMode=*/true, threads, /*overwrite=*/true,
-		/*deleteMode=*/false, /*setowner=*/false, /*showStatistics=*/false, /*patterns=*/nil)
+	failedFiles = backupManager.Restore(testDir+"/repository2", 2 /*inPlace=*/, true /*quickMode=*/, true, threads /*overwrite=*/, true,
+		/*deleteMode=*/ false /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, nil /*allowFailures=*/, false)
+	assertRestoreFailures(t, failedFiles, 0)
 
 	for _, f := range []string{"file1", "file2", "dir1/file3"} {
 		hash1 := getFileHash(testDir + "/repository1/" + f)
@@ -287,7 +303,7 @@ func TestBackupManager(t *testing.T) {
 	os.Mkdir(testDir+"/repository1/dir2/dir3", 0700)
 	os.Mkdir(testDir+"/repository1/dir4", 0700)
 	SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy")
-	backupManager.Backup(testDir+"/repository1" /*quickMode=*/, false, threads, "third", false, false)
+	backupManager.Backup(testDir+"/repository1" /*quickMode=*/, false, threads, "third", false, false, 0, false, 1024, 1024)
 	time.Sleep(time.Duration(delay) * time.Second)
 
 	// Create some directories and files under repository2 that will be deleted during restore
@@ -298,8 +314,9 @@ func TestBackupManager(t *testing.T) {
 	createRandomFile(testDir+"/repository2/dir5/file5", 100)
 
 	SetDuplicacyPreferencePath(testDir + "/repository2/.duplicacy")
-	backupManager.Restore(testDir+"/repository2", 3, /*inPlace=*/true, /*quickMode=*/false, threads, /*overwrite=*/true,
-		/*deleteMode=*/true, /*setowner=*/false, /*showStatistics=*/false, /*patterns=*/nil)
+	failedFiles = backupManager.Restore(testDir+"/repository2", 3 /*inPlace=*/, true /*quickMode=*/, false, threads /*overwrite=*/, true,
+		/*deleteMode=*/ true /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, nil /*allowFailures=*/, false)
+	assertRestoreFailures(t, failedFiles, 0)
 
 	for _, f := range []string{"file1", "file2", "dir1/file3"} {
 		hash1 := getFileHash(testDir + "/repository1/" + f)
@@ -325,8 +342,9 @@ func TestBackupManager(t *testing.T) {
 	os.Remove(testDir + "/repository1/file2")
 	os.Remove(testDir + "/repository1/dir1/file3")
 	SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy")
-	backupManager.Restore(testDir+"/repository1", 3, /*inPlace=*/true, /*quickMode=*/false, threads, /*overwrite=*/true,
-		/*deleteMode=*/false,  /*setowner=*/false, /*showStatistics=*/false, /*patterns=*/[]string{"+file2", "+dir1/file3", "-*"})
+	failedFiles = backupManager.Restore(testDir+"/repository1", 3 /*inPlace=*/, true /*quickMode=*/, false, threads /*overwrite=*/, true,
+		/*deleteMode=*/ false /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, []string{"+file2", "+dir1/file3", "-*"} /*allowFailures=*/, false)
+	assertRestoreFailures(t, failedFiles, 0)
 
 	for _, f := range []string{"file1", "file2", "dir1/file3"} {
 		hash1 := getFileHash(testDir + "/repository1/" + f)
@@ -340,27 +358,377 @@ func TestBackupManager(t *testing.T) {
 	if numberOfSnapshots != 3 {
 		t.Errorf("Expected 3 snapshots but got %d", numberOfSnapshots)
 	}
-	backupManager.SnapshotManager.CheckSnapshots( /*snapshotID*/ "host1" /*revisions*/, []int{1, 2, 3} /*tag*/, "",
-		/*showStatistics*/ false /*showTabular*/, false /*checkFiles*/, false /*searchFossils*/, false /*resurrect*/, false)
+			
+	backupManager.SnapshotManager.CheckSnapshots( /*snapshotID*/ "host1", /*revisions*/ []int{1, 2, 3}, /*tag*/ "", /*showStatistics*/ false,
+	    /*showTabular*/ false, /*checkFiles*/ false, /*checkChunks*/ false, /*searchFossils*/ false, /*resurrect*/ false, /*rewiret*/ false, 1,  /*allowFailures*/false)
 	backupManager.SnapshotManager.PruneSnapshots("host1", "host1" /*revisions*/, []int{1} /*tags*/, nil /*retentions*/, nil,
-		/*exhaustive*/ false /*exclusive=*/, false /*ignoredIDs*/, nil /*dryRun*/, false /*deleteOnly*/, false /*collectOnly*/, false)
+		/*exhaustive*/ false /*exclusive=*/, false /*ignoredIDs*/, nil /*dryRun*/, false /*deleteOnly*/, false /*collectOnly*/, false, 1)
 	numberOfSnapshots = backupManager.SnapshotManager.ListSnapshots( /*snapshotID*/ "host1" /*revisionsToList*/, nil /*tag*/, "" /*showFiles*/, false /*showChunks*/, false)
 	if numberOfSnapshots != 2 {
 		t.Errorf("Expected 2 snapshots but got %d", numberOfSnapshots)
 	}
-	backupManager.SnapshotManager.CheckSnapshots( /*snapshotID*/ "host1" /*revisions*/, []int{2, 3} /*tag*/, "",
-		/*showStatistics*/ false /*showTabular*/, false /*checkFiles*/, false /*searchFossils*/, false /*resurrect*/, false)
-	backupManager.Backup(testDir+"/repository1" /*quickMode=*/, false, threads, "fourth", false, false)
+	backupManager.SnapshotManager.CheckSnapshots( /*snapshotID*/ "host1", /*revisions*/ []int{2, 3}, /*tag*/ "", /*showStatistics*/ false,
+	     /*showTabular*/ false, /*checkFiles*/ false, /*checkChunks*/ false, /*searchFossils*/ false, /*resurrect*/ false, /*rewiret*/ false, 1, /*allowFailures*/ false)
+	backupManager.Backup(testDir+"/repository1" /*quickMode=*/, false, threads, "fourth", false, false, 0, false, 1024, 1024)
 	backupManager.SnapshotManager.PruneSnapshots("host1", "host1" /*revisions*/, nil /*tags*/, nil /*retentions*/, nil,
-		/*exhaustive*/ false /*exclusive=*/, true /*ignoredIDs*/, nil /*dryRun*/, false /*deleteOnly*/, false /*collectOnly*/, false)
+		/*exhaustive*/ false /*exclusive=*/, true /*ignoredIDs*/, nil /*dryRun*/, false /*deleteOnly*/, false /*collectOnly*/, false, 1)
 	numberOfSnapshots = backupManager.SnapshotManager.ListSnapshots( /*snapshotID*/ "host1" /*revisionsToList*/, nil /*tag*/, "" /*showFiles*/, false /*showChunks*/, false)
 	if numberOfSnapshots != 3 {
 		t.Errorf("Expected 3 snapshots but got %d", numberOfSnapshots)
 	}
-	backupManager.SnapshotManager.CheckSnapshots( /*snapshotID*/ "host1" /*revisions*/, []int{2, 3, 4} /*tag*/, "",
-		/*showStatistics*/ false /*showTabular*/, false /*checkFiles*/, false /*searchFossils*/, false /*resurrect*/, false)
+	backupManager.SnapshotManager.CheckSnapshots( /*snapshotID*/ "host1", /*revisions*/ []int{2, 3, 4}, /*tag*/ "", /*showStatistics*/ false,
+	    /*showTabular*/ false, /*checkFiles*/ false, /*checkChunks*/ false, /*searchFossils*/ false, /*resurrect*/ false, /*rewiret*/ false, 1, /*allowFailures*/ false)
 
 	/*buf := make([]byte, 1<<16)
 	  runtime.Stack(buf, true)
 	  fmt.Printf("%s", buf)*/
 }
+
+// Create file with random file with certain seed
+func createRandomFileSeeded(path string, maxSize int, seed int64) {
+	rand.Seed(seed)
+	file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
+	if err != nil {
+		LOG_ERROR("RANDOM_FILE", "Can't open %s for writing: %v", path, err)
+		return
+	}
+
+	defer file.Close()
+
+	size := maxSize/2 + rand.Int()%(maxSize/2)
+
+	buffer := make([]byte, 32*1024)
+	for size > 0 {
+		bytes := size
+		if bytes > cap(buffer) {
+			bytes = cap(buffer)
+		}
+		rand.Read(buffer[:bytes])
+		bytes, err = file.Write(buffer[:bytes])
+		if err != nil {
+			LOG_ERROR("RANDOM_FILE", "Failed to write to %s: %v", path, err)
+			return
+		}
+		size -= bytes
+	}
+}
+
+func corruptFile(path string, start int, length int, seed int64) {
+	rand.Seed(seed)
+
+	file, err := os.OpenFile(path, os.O_WRONLY, 0644)
+	if err != nil {
+		LOG_ERROR("CORRUPT_FILE", "Can't open %s for writing: %v", path, err)
+		return
+	}
+
+	defer func() {
+		if file != nil {
+			file.Close()
+		}
+	}()
+
+	_, err = file.Seek(int64(start), 0)
+	if err != nil {
+		LOG_ERROR("CORRUPT_FILE", "Can't seek to the offset %d: %v", start, err)
+		return
+	}
+
+	buffer := make([]byte, length)
+	rand.Read(buffer)
+
+	_, err = file.Write(buffer)
+	if err != nil {
+		LOG_ERROR("CORRUPT_FILE", "Failed to write to %s: %v", path, err)
+		return
+	}
+}
+
+func TestPersistRestore(t *testing.T) {
+	// We want deterministic output here so we can test the expected files are corrupted by missing or corrupt chunks
+	// There use rand functions with fixed seed, and known keys
+
+	setTestingT(t)
+	SetLoggingLevel(INFO)
+
+	defer func() {
+		if r := recover(); r != nil {
+			switch e := r.(type) {
+			case Exception:
+				t.Errorf("%s %s", e.LogID, e.Message)
+				debug.PrintStack()
+			default:
+				t.Errorf("%v", e)
+				debug.PrintStack()
+			}
+		}
+	}()
+
+	testDir := path.Join(os.TempDir(), "duplicacy_test")
+	os.RemoveAll(testDir)
+	os.MkdirAll(testDir, 0700)
+	os.Mkdir(testDir+"/repository1", 0700)
+	os.Mkdir(testDir+"/repository1/dir1", 0700)
+	os.Mkdir(testDir+"/repository1/.duplicacy", 0700)
+	os.Mkdir(testDir+"/repository2", 0700)
+	os.Mkdir(testDir+"/repository2/.duplicacy", 0700)
+	os.Mkdir(testDir+"/repository3", 0700)
+	os.Mkdir(testDir+"/repository3/.duplicacy", 0700)
+
+	maxFileSize := 1000000
+	//maxFileSize := 200000
+
+	createRandomFileSeeded(testDir+"/repository1/file1", maxFileSize,1)
+	createRandomFileSeeded(testDir+"/repository1/file2", maxFileSize,2)
+	createRandomFileSeeded(testDir+"/repository1/dir1/file3", maxFileSize,3)
+
+	threads := 1
+
+	password := "duplicacy"
+
+	// We want deterministic output, plus ability to test encrypted storage
+	// So make unencrypted storage with default keys, and encrypted as bit-identical copy of this but with password
+	unencStorage, err := loadStorage(testDir+"/unenc_storage", threads)
+	if err != nil {
+		t.Errorf("Failed to create storage: %v", err)
+		return
+	}
+	delay := 0
+	if _, ok := unencStorage.(*ACDStorage); ok {
+		delay = 1
+	}
+	if _, ok := unencStorage.(*OneDriveStorage); ok {
+		delay = 5
+	}
+
+	time.Sleep(time.Duration(delay) * time.Second)
+	cleanStorage(unencStorage)
+
+	if !ConfigStorage(unencStorage, 16384, 100, 64*1024, 256*1024, 16*1024, "", nil, false, "", 0, 0) {
+		t.Errorf("Failed to initialize the unencrypted storage")
+	}
+	time.Sleep(time.Duration(delay) * time.Second)
+	unencConfig, _, err := DownloadConfig(unencStorage, "")
+	if err != nil {
+		t.Errorf("Failed to download storage config: %v", err)
+		return
+	}
+
+	// Make encrypted storage
+	storage, err := loadStorage(testDir+"/enc_storage", threads)
+	if err != nil {
+		t.Errorf("Failed to create encrypted storage: %v", err)
+		return
+	}
+	time.Sleep(time.Duration(delay) * time.Second)
+	cleanStorage(storage)
+
+	if !ConfigStorage(storage, 16384, 100, 64*1024, 256*1024, 16*1024, password, unencConfig, true, "", 0, 0) {
+		t.Errorf("Failed to initialize the encrypted storage")
+	}
+	time.Sleep(time.Duration(delay) * time.Second)
+
+	// do unencrypted backup
+	SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy")
+	unencBackupManager := CreateBackupManager("host1", unencStorage, testDir, "", "", "", false)
+	unencBackupManager.SetupSnapshotCache("default")
+
+	SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy")
+	unencBackupManager.Backup(testDir+"/repository1" /*quickMode=*/, true, threads, "first", false, false, 0, false, 1024, 1024)
+	time.Sleep(time.Duration(delay) * time.Second)
+
+
+	// do encrypted backup
+	SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy")
+	encBackupManager := CreateBackupManager("host1", storage, testDir, password, "", "", false)
+	encBackupManager.SetupSnapshotCache("default")
+
+	SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy")
+	encBackupManager.Backup(testDir+"/repository1" /*quickMode=*/, true, threads, "first", false, false, 0, false, 1024, 1024)
+	time.Sleep(time.Duration(delay) * time.Second)
+
+
+	// check snapshots
+	unencBackupManager.SnapshotManager.CheckSnapshots( /*snapshotID*/ "host1", /*revisions*/ []int{1}, /*tag*/ "",
+		/*showStatistics*/ true, /*showTabular*/ false, /*checkFiles*/ true, /*checkChunks*/ false,
+		/*searchFossils*/ false, /*resurrect*/ false, /*rewiret*/ false, 1, /*allowFailures*/ false)
+
+	encBackupManager.SnapshotManager.CheckSnapshots( /*snapshotID*/ "host1", /*revisions*/ []int{1}, /*tag*/ "",
+		/*showStatistics*/ true, /*showTabular*/ false, /*checkFiles*/ true, /*checkChunks*/ false,
+		 /*searchFossils*/ false, /*resurrect*/ false, /*rewiret*/ false, 1, /*allowFailures*/ false)
+		
+	// check functions
+	checkAllUncorrupted := func(cmpRepository string) {
+			for _, f := range []string{"file1", "file2", "dir1/file3"} {
+				if _, err := os.Stat(testDir + cmpRepository + "/" + f); os.IsNotExist(err) {
+					t.Errorf("File %s does not exist", f)
+					continue
+				}
+
+				hash1 := getFileHash(testDir + "/repository1/" + f)
+				hash2 := getFileHash(testDir + cmpRepository + "/" + f)
+				if hash1 != hash2 {
+					t.Errorf("File %s has different hashes: %s vs %s", f, hash1, hash2)
+				}
+		}
+	}
+	checkMissingFile := func(cmpRepository string, expectMissing string) {
+			for _, f := range []string{"file1", "file2", "dir1/file3"} {
+				_, err := os.Stat(testDir + cmpRepository + "/" + f)
+				if err==nil {
+					if f==expectMissing {
+						t.Errorf("File %s exists, expected to be missing", f)
+					}
+					continue
+				}
+				if os.IsNotExist(err) {
+					if f!=expectMissing {
+						t.Errorf("File %s does not exist", f)
+					}
+					continue
+				}
+
+				hash1 := getFileHash(testDir + "/repository1/" + f)
+				hash2 := getFileHash(testDir + cmpRepository + "/" + f)
+				if hash1 != hash2 {
+					t.Errorf("File %s has different hashes: %s vs %s", f, hash1, hash2)
+				}
+		}
+	}
+	checkCorruptedFile  := func(cmpRepository string, expectCorrupted string) {
+		for _, f := range []string{"file1", "file2", "dir1/file3"} {
+			if _, err := os.Stat(testDir + cmpRepository + "/" + f); os.IsNotExist(err) {
+				t.Errorf("File %s does not exist", f)
+				continue
+			}
+	
+			hash1 := getFileHash(testDir + "/repository1/" + f)
+			hash2 := getFileHash(testDir + cmpRepository + "/" + f)
+			if (f==expectCorrupted) {
+				if hash1 == hash2 {
+					t.Errorf("File %s has same hashes, expected to be corrupted: %s vs %s", f, hash1, hash2)
+				}
+	
+			} else {
+				if hash1 != hash2 {
+					t.Errorf("File %s has different hashes: %s vs %s", f, hash1, hash2)
+				}
+			}
+		}
+	}
+
+	// test restore all uncorrupted to repository3
+	SetDuplicacyPreferencePath(testDir + "/repository3/.duplicacy")
+	failedFiles := unencBackupManager.Restore(testDir+"/repository3", threads /*inPlace=*/, true /*quickMode=*/, false, threads /*overwrite=*/, false,
+		 /*deleteMode=*/ false /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, nil /*allowFailures=*/, false)
+	assertRestoreFailures(t, failedFiles, 0)
+	checkAllUncorrupted("/repository3")
+
+	// test for corrupt files and -persist
+	// corrupt a chunk 
+	chunkToCorrupt1 := "/4d/538e5dfd2b08e782bfeb56d1360fb5d7eb9d8c4b2531cc2fca79efbaec910c"
+		// this should affect file1
+	chunkToCorrupt2 := "/2b/f953a766d0196ce026ae259e76e3c186a0e4bcd3ce10f1571d17f86f0a5497"
+		// this should affect dir1/file3
+	
+	for i := 0; i < 2; i++ {
+		if i==0 {
+			// test corrupt chunks
+			corruptFile(testDir+"/unenc_storage"+"/chunks"+chunkToCorrupt1, 128, 128, 4)
+			corruptFile(testDir+"/enc_storage"+"/chunks"+chunkToCorrupt2, 128, 128, 4)
+		} else {
+			// test missing chunks
+			os.Remove(testDir+"/unenc_storage"+"/chunks"+chunkToCorrupt1)
+			os.Remove(testDir+"/enc_storage"+"/chunks"+chunkToCorrupt2)
+		}
+
+		// This is to make sure that allowFailures is set to true.  Note that this is not needed
+		// in the production code because chunkOperator can be only recreated multiple time in tests.
+		if unencBackupManager.SnapshotManager.chunkOperator != nil {
+			unencBackupManager.SnapshotManager.chunkOperator.allowFailures = true
+		}
+
+		if encBackupManager.SnapshotManager.chunkOperator != nil {
+			encBackupManager.SnapshotManager.chunkOperator.allowFailures = true
+		}
+
+		// check snapshots with --persist (allowFailures == true)
+		// this would cause a panic and os.Exit from duplicacy_log if allowFailures == false
+		unencBackupManager.SnapshotManager.CheckSnapshots( /*snapshotID*/ "host1", /*revisions*/ []int{1}, /*tag*/ "",
+			/*showStatistics*/ true, /*showTabular*/ false, /*checkFiles*/ true, /*checkChunks*/ false,
+			/*searchFossils*/ false, /*resurrect*/ false, /*rewrite*/ false, 1, /*allowFailures*/ true)
+
+		encBackupManager.SnapshotManager.CheckSnapshots( /*snapshotID*/ "host1", /*revisions*/ []int{1}, /*tag*/ "",
+			/*showStatistics*/ true, /*showTabular*/ false, /*checkFiles*/ true, /*checkChunks*/ false,
+			/*searchFossils*/ false, /*resurrect*/ false, /*rewrite*/ false, 1, /*allowFailures*/ true)
+
+	
+		// test restore corrupted, inPlace = true, corrupted files will have hash failures
+		os.RemoveAll(testDir+"/repository2")
+		SetDuplicacyPreferencePath(testDir + "/repository2/.duplicacy")
+		failedFiles = unencBackupManager.Restore(testDir+"/repository2", threads /*inPlace=*/, true /*quickMode=*/, false, threads /*overwrite=*/, false,
+			/*deleteMode=*/ false /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, nil /*allowFailures=*/, true)
+		assertRestoreFailures(t, failedFiles, 1)
+
+		// check restore, expect file1 to be corrupted
+		checkCorruptedFile("/repository2", "file1")
+
+		
+		os.RemoveAll(testDir+"/repository2")
+		SetDuplicacyPreferencePath(testDir + "/repository2/.duplicacy")
+		failedFiles = encBackupManager.Restore(testDir+"/repository2", threads /*inPlace=*/, true /*quickMode=*/, false, threads /*overwrite=*/, false,
+			/*deleteMode=*/ false /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, nil /*allowFailures=*/, true)
+		assertRestoreFailures(t, failedFiles, 1)
+
+		// check restore, expect file3 to be corrupted
+		checkCorruptedFile("/repository2", "dir1/file3")
+
+		//SetLoggingLevel(DEBUG)
+		// test restore corrupted, inPlace = false, corrupted files will be missing
+		os.RemoveAll(testDir+"/repository2")
+		SetDuplicacyPreferencePath(testDir + "/repository2/.duplicacy")
+		failedFiles = unencBackupManager.Restore(testDir+"/repository2", threads /*inPlace=*/, false /*quickMode=*/, false, threads /*overwrite=*/, false,
+			/*deleteMode=*/ false /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, nil /*allowFailures=*/, true)
+		assertRestoreFailures(t, failedFiles, 1)
+
+		// check restore, expect file1 to be corrupted
+		checkMissingFile("/repository2", "file1")
+
+		
+		os.RemoveAll(testDir+"/repository2")
+		SetDuplicacyPreferencePath(testDir + "/repository2/.duplicacy")
+		failedFiles = encBackupManager.Restore(testDir+"/repository2", threads /*inPlace=*/, false /*quickMode=*/, false, threads /*overwrite=*/, false,
+			/*deleteMode=*/ false /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, nil /*allowFailures=*/, true)
+		assertRestoreFailures(t, failedFiles, 1)
+
+		// check restore, expect file3 to be corrupted
+		checkMissingFile("/repository2", "dir1/file3")
+
+		// test restore corrupted files from different backups, inPlace = true
+		// with overwrite=true, corrupted file1 from unenc will be restored correctly from enc
+		// the latter will not touch the existing file3 with correct hash
+		os.RemoveAll(testDir+"/repository2")
+		failedFiles = unencBackupManager.Restore(testDir+"/repository2", threads /*inPlace=*/, true /*quickMode=*/, false, threads /*overwrite=*/, false,
+			/*deleteMode=*/ false /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, nil /*allowFailures=*/, true)
+		assertRestoreFailures(t, failedFiles, 1)
+
+		failedFiles = encBackupManager.Restore(testDir+"/repository2", threads /*inPlace=*/, true /*quickMode=*/, false, threads /*overwrite=*/, true,
+			/*deleteMode=*/ false /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, nil /*allowFailures=*/, true)
+		assertRestoreFailures(t, failedFiles, 0)
+		checkAllUncorrupted("/repository2")
+
+		// restore to repository3, with overwrite and allowFailures (true/false), quickMode = false (use hashes)
+		// should always succeed as uncorrupted files already exist with correct hash, so these will be ignored
+		SetDuplicacyPreferencePath(testDir + "/repository3/.duplicacy")
+		failedFiles = unencBackupManager.Restore(testDir+"/repository3", threads /*inPlace=*/, true /*quickMode=*/, false, threads /*overwrite=*/, true,
+			/*deleteMode=*/ false /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, nil /*allowFailures=*/, false)
+		assertRestoreFailures(t, failedFiles, 0)
+		checkAllUncorrupted("/repository3")
+
+		failedFiles = unencBackupManager.Restore(testDir+"/repository3", threads /*inPlace=*/, true /*quickMode=*/, false, threads /*overwrite=*/, true,
+			/*deleteMode=*/ false /*setowner=*/, false /*showStatistics=*/, false /*patterns=*/, nil /*allowFailures=*/, true)
+		assertRestoreFailures(t, failedFiles, 0)
+		checkAllUncorrupted("/repository3")
+	}
+
+}

src/duplicacy_benchmark.go

@@ -0,0 +1,235 @@
+// Copyright (c) Acrosync LLC. All rights reserved.
+// Free for personal use and commercial trial
+// Commercial use requires per-user licenses available from https://duplicacy.com
+
+package duplicacy
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"math/rand"
+	"os"
+	"path/filepath"
+	"time"
+)
+
+func benchmarkSplit(reader *bytes.Reader, fileSize int64, chunkSize int, compression bool, encryption bool, annotation string) {
+
+	config := CreateConfig()
+	config.CompressionLevel = DEFAULT_COMPRESSION_LEVEL
+	config.AverageChunkSize = chunkSize
+	config.MaximumChunkSize = chunkSize * 4
+	config.MinimumChunkSize = chunkSize / 4
+	config.ChunkSeed = []byte("duplicacy")
+
+	config.HashKey = DEFAULT_KEY
+	config.IDKey = DEFAULT_KEY
+
+	maker := CreateFileChunkMaker(config, false)
+
+	startTime := float64(time.Now().UnixNano()) / 1e9
+	numberOfChunks := 0
+	reader.Seek(0, os.SEEK_SET)
+
+	chunkFunc := func(chunk *Chunk) {
+		if compression {
+			key := ""
+			if encryption {
+				key = "0123456789abcdef0123456789abcdef"
+			}
+			err := chunk.Encrypt([]byte(key), "", false)
+			if err != nil {
+				LOG_ERROR("BENCHMARK_ENCRYPT", "Failed to encrypt the chunk: %v", err)
+			}
+		}
+		config.PutChunk(chunk)
+		numberOfChunks++
+	}
+
+	maker.AddData(reader, chunkFunc)
+	maker.AddData(nil, chunkFunc)
+
+	runningTime := float64(time.Now().UnixNano())/1e9 - startTime
+	speed := int64(float64(fileSize) / runningTime)
+	LOG_INFO("BENCHMARK_SPLIT", "Split %s bytes into %d chunks %s in %.2fs: %s/s", PrettySize(fileSize), numberOfChunks, annotation,
+		runningTime, PrettySize(speed))
+
+	return
+}
+
+func benchmarkRun(threads int, chunkCount int, job func(threadIndex int, chunkIndex int)) {
+	indexChannel := make(chan int, chunkCount)
+	stopChannel := make(chan int, threads)
+	finishChannel := make(chan int, threads)
+
+	// Start the uploading goroutines
+	for i := 0; i < threads; i++ {
+		go func(threadIndex int) {
+			defer CatchLogException()
+			for {
+				select {
+				case chunkIndex := <-indexChannel:
+					job(threadIndex, chunkIndex)
+					finishChannel <- 0
+				case <-stopChannel:
+					return
+				}
+			}
+		}(i)
+	}
+
+	for i := 0; i < chunkCount; i++ {
+		indexChannel <- i
+	}
+
+	for i := 0; i < chunkCount; i++ {
+		<-finishChannel
+	}
+
+	for i := 0; i < threads; i++ {
+		stopChannel <- 0
+	}
+}
+
+func Benchmark(localDirectory string, storage Storage, fileSize int64, chunkSize int, chunkCount int, uploadThreads int, downloadThreads int) bool {
+
+	filename := filepath.Join(localDirectory, "benchmark.dat")
+
+	defer func() {
+		os.Remove(filename)
+	}()
+
+	LOG_INFO("BENCHMARK_GENERATE", "Generating %s byte random data in memory", PrettySize(fileSize))
+	data := make([]byte, fileSize)
+	_, err := rand.Read(data)
+	if err != nil {
+		LOG_ERROR("BENCHMARK_RAND", "Failed to generate random data: %v", err)
+		return false
+	}
+
+	startTime := float64(time.Now().UnixNano()) / 1e9
+	LOG_INFO("BENCHMARK_WRITE", "Writing random data to local disk")
+	err = ioutil.WriteFile(filename, data, 0600)
+	if err != nil {
+		LOG_ERROR("BENCHMARK_WRITE", "Failed to write the random data: %v", err)
+		return false
+	}
+
+	runningTime := float64(time.Now().UnixNano())/1e9 - startTime
+	speed := int64(float64(fileSize) / runningTime)
+	LOG_INFO("BENCHMARK_WRITE", "Wrote %s bytes in %.2fs: %s/s", PrettySize(fileSize), runningTime, PrettySize(speed))
+
+	startTime = float64(time.Now().UnixNano()) / 1e9
+	LOG_INFO("BENCHMARK_READ", "Reading the random data from local disk")
+	file, err := os.Open(filename)
+	if err != nil {
+		LOG_ERROR("BENCHMARK_OPEN", "Failed to open the random data file: %v", err)
+		return false
+	}
+	segment := make([]byte, 1024*1024)
+	for err == nil {
+		_, err = file.Read(segment)
+	}
+	if err != io.EOF {
+		LOG_ERROR("BENCHMARK_OPEN", "Failed to read the random data file: %v", err)
+		return false
+	}
+	file.Close()
+	runningTime = float64(time.Now().UnixNano())/1e9 - startTime
+	speed = int64(float64(fileSize) / runningTime)
+	LOG_INFO("BENCHMARK_READ", "Read %s bytes in %.2fs: %s/s", PrettySize(fileSize), runningTime, PrettySize(speed))
+
+	buffer := bytes.NewReader(data)
+	benchmarkSplit(buffer, fileSize, chunkSize, false, false, "without compression/encryption")
+	benchmarkSplit(buffer, fileSize, chunkSize, true, false, "with compression but without encryption")
+	benchmarkSplit(buffer, fileSize, chunkSize, true, true, "with compression and encryption")
+
+	storage.CreateDirectory(0, "benchmark")
+	existingFiles, _, err := storage.ListFiles(0, "benchmark/")
+	if err != nil {
+		LOG_ERROR("BENCHMARK_LIST", "Failed to list the benchmark directory: %v", err)
+		return false
+	}
+
+	var existingChunks []string
+	for _, f := range existingFiles {
+		if len(f) > 0 && f[len(f)-1] != '/' {
+			existingChunks = append(existingChunks, "benchmark/"+f)
+		}
+	}
+
+	if len(existingChunks) > 0 {
+		LOG_INFO("BENCHMARK_DELETE", "Deleting %d temporary files from previous benchmark runs", len(existingChunks))
+		benchmarkRun(uploadThreads, len(existingChunks), func(threadIndex int, chunkIndex int) {
+			storage.DeleteFile(threadIndex, existingChunks[chunkIndex])
+		})
+	}
+
+	chunks := make([][]byte, chunkCount)
+	chunkHashes := make([]string, chunkCount)
+	LOG_INFO("BENCHMARK_GENERATE", "Generating %d chunks", chunkCount)
+	for i := 0; i < chunkCount; i++ {
+		chunks[i] = make([]byte, chunkSize)
+		_, err = rand.Read(chunks[i])
+		if err != nil {
+			LOG_ERROR("BENCHMARK_RAND", "Failed to generate random data: %v", err)
+			return false
+		}
+		hashInBytes := sha256.Sum256(chunks[i])
+		chunkHashes[i] = hex.EncodeToString(hashInBytes[:])
+
+	}
+
+	startTime = float64(time.Now().UnixNano()) / 1e9
+	benchmarkRun(uploadThreads, chunkCount, func(threadIndex int, chunkIndex int) {
+		err := storage.UploadFile(threadIndex, fmt.Sprintf("benchmark/chunk%d", chunkIndex), chunks[chunkIndex])
+		if err != nil {
+			LOG_ERROR("BENCHMARK_UPLOAD", "Failed to upload the chunk: %v", err)
+			return
+		}
+	})
+
+	runningTime = float64(time.Now().UnixNano())/1e9 - startTime
+	speed = int64(float64(chunkSize*chunkCount) / runningTime)
+	LOG_INFO("BENCHMARK_UPLOAD", "Uploaded %s bytes in %.2fs: %s/s", PrettySize(int64(chunkSize*chunkCount)), runningTime, PrettySize(speed))
+
+	config := CreateConfig()
+
+	startTime = float64(time.Now().UnixNano()) / 1e9
+	hashError := false
+	benchmarkRun(downloadThreads, chunkCount, func(threadIndex int, chunkIndex int) {
+		chunk := config.GetChunk()
+		chunk.Reset(false)
+		err := storage.DownloadFile(threadIndex, fmt.Sprintf("benchmark/chunk%d", chunkIndex), chunk)
+		if err != nil {
+			LOG_ERROR("BENCHMARK_DOWNLOAD", "Failed to download the chunk: %v", err)
+			return
+		}
+
+		hashInBytes := sha256.Sum256(chunk.GetBytes())
+		hash := hex.EncodeToString(hashInBytes[:])
+		if hash != chunkHashes[chunkIndex] {
+			LOG_WARN("BENCHMARK_HASH", "Chunk %d has mismatched hashes: %s != %s", chunkIndex, chunkHashes[chunkIndex], hash)
+			hashError = true
+		}
+
+		config.PutChunk(chunk)
+	})
+
+	runningTime = float64(time.Now().UnixNano())/1e9 - startTime
+	speed = int64(float64(chunkSize*chunkCount) / runningTime)
+	LOG_INFO("BENCHMARK_DOWNLOAD", "Downloaded %s bytes in %.2fs: %s/s", PrettySize(int64(chunkSize*chunkCount)), runningTime, PrettySize(speed))
+
+	if !hashError {
+		benchmarkRun(uploadThreads, chunkCount, func(threadIndex int, chunkIndex int) {
+			storage.DeleteFile(threadIndex, fmt.Sprintf("benchmark/chunk%d", chunkIndex))
+		})
+		LOG_INFO("BENCHMARK_DELETE", "Deleted %d temporary files from the storage", chunkCount)
+	}
+
+	return true
+}

src/duplicacy_chunk.go

@@ -8,15 +8,29 @@ import (
 	"bytes"
 	"compress/zlib"
 	"crypto/aes"
+	"crypto/rsa"
 	"crypto/cipher"
+	"crypto/hmac"
 	"crypto/rand"
+	"crypto/sha256"
 	"encoding/hex"
+	"encoding/binary"
 	"fmt"
 	"hash"
 	"io"
+	"os"
 	"runtime"
 
 	"github.com/bkaradzic/go-lz4"
+	"github.com/minio/highwayhash"
+	"github.com/klauspost/reedsolomon"
+	"github.com/klauspost/compress/zstd"
+
+	// This is a fork of github.com/minio/highwayhash at 1.0.1 that computes incorrect hash on
+	// arm64 machines.  We need this fork to be able to read the chunks created by Duplicacy
+	// CLI 3.0.1 which unfortunately relies on incorrect hashes to determine if each shard is valid.
+	wronghighwayhash "github.com/gilbertchen/highwayhash"
+
 )
 
 // A chunk needs to acquire a new buffer and return the old one for every encrypt/decrypt operation, therefore
@@ -57,10 +71,20 @@ type Chunk struct {
 
 	config *Config // Every chunk is associated with a Config object.  Which hashing algorithm to use is determined
 	// by the config
+
+	isMetadata bool // Indicates if the chunk is a metadata chunk (instead of a file chunk).  This is primarily used by RSA
+	                // encryption, where a metadata chunk is not encrypted by RSA
+	
+	isBroken bool // Indicates the chunk did not download correctly. This is only used for -persist (allowFailures) mode
 }
 
 // Magic word to identify a duplicacy format encrypted file, plus a version number.
-var ENCRYPTION_HEADER = "duplicacy\000"
+var ENCRYPTION_BANNER = "duplicacy\000"
+
+// RSA encrypted chunks start with "duplicacy\002"
+var ENCRYPTION_VERSION_RSA byte = 2
+
+var ERASURE_CODING_BANNER = "duplicacy\003"
 
 // CreateChunk creates a new chunk.
 func CreateChunk(config *Config, bufferNeeded bool) *Chunk {
@@ -110,6 +134,8 @@ func (chunk *Chunk) Reset(hashNeeded bool) {
 	chunk.hash = nil
 	chunk.id = ""
 	chunk.size = 0
+	chunk.isMetadata = false
+	chunk.isBroken = false
 }
 
 // Write implements the Writer interface.
@@ -167,7 +193,7 @@ func (chunk *Chunk) VerifyID() {
 
 // Encrypt encrypts the plain data stored in the chunk buffer.  If derivationKey is not nil, the actual
 // encryption key will be HMAC-SHA256(encryptionKey, derivationKey).
-func (chunk *Chunk) Encrypt(encryptionKey []byte, derivationKey string) (err error) {
+func (chunk *Chunk) Encrypt(encryptionKey []byte, derivationKey string, isMetadata bool) (err error) {
 
 	var aesBlock cipher.Block
 	var gcm cipher.AEAD
@@ -183,8 +209,17 @@ func (chunk *Chunk) Encrypt(encryptionKey []byte, derivationKey string) (err err
 	if len(encryptionKey) > 0 {
 
 		key := encryptionKey
-
-		if len(derivationKey) > 0 {
+		usingRSA := false
+		// Enable RSA encryption only when the chunk is not a metadata chunk
+		if chunk.config.rsaPublicKey != nil && !isMetadata && !chunk.isMetadata {
+			randomKey := make([]byte, 32)
+			_, err := rand.Read(randomKey)
+			if err != nil {
+				return err
+			}
+			key = randomKey
+			usingRSA = true
+		} else if len(derivationKey) > 0 {
 			hasher := chunk.config.NewKeyedHasher([]byte(derivationKey))
 			hasher.Write(encryptionKey)
 			key = hasher.Sum(nil)
@@ -201,7 +236,21 @@ func (chunk *Chunk) Encrypt(encryptionKey []byte, derivationKey string) (err err
 		}
 
 		// Start with the magic number and the version number.
-		encryptedBuffer.Write([]byte(ENCRYPTION_HEADER))
+		if usingRSA {
+			// RSA encryption starts "duplicacy\002"
+			encryptedBuffer.Write([]byte(ENCRYPTION_BANNER)[:len(ENCRYPTION_BANNER) - 1])
+			encryptedBuffer.Write([]byte{ENCRYPTION_VERSION_RSA})
+
+			// Then the encrypted key
+			encryptedKey, err := rsa.EncryptOAEP(sha256.New(), rand.Reader, chunk.config.rsaPublicKey, key, nil)
+			if err != nil {
+				return err
+			}
+			binary.Write(encryptedBuffer, binary.LittleEndian, uint16(len(encryptedKey)))
+			encryptedBuffer.Write(encryptedKey)
+		} else {
+			encryptedBuffer.Write([]byte(ENCRYPTION_BANNER))
+		}
 
 		// Followed by the nonce
 		nonce = make([]byte, gcm.NonceSize())
@@ -211,15 +260,46 @@ func (chunk *Chunk) Encrypt(encryptionKey []byte, derivationKey string) (err err
 		}
 		encryptedBuffer.Write(nonce)
 		offset = encryptedBuffer.Len()
-
 	}
 
-	// offset is either 0 or the length of header + nonce
+	// offset is either 0 or the length of banner + nonce
 
 	if chunk.config.CompressionLevel >= -1 && chunk.config.CompressionLevel <= 9 {
 		deflater, _ := zlib.NewWriterLevel(encryptedBuffer, chunk.config.CompressionLevel)
 		deflater.Write(chunk.buffer.Bytes())
 		deflater.Close()
+	} else if chunk.config.CompressionLevel >= ZSTD_COMPRESSION_LEVEL_FASTEST && chunk.config.CompressionLevel <= ZSTD_COMPRESSION_LEVEL_BEST  {
+		encryptedBuffer.Write([]byte("ZSTD"))
+
+		compressionLevel := zstd.SpeedDefault
+		if chunk.config.CompressionLevel == ZSTD_COMPRESSION_LEVEL_FASTEST {
+			compressionLevel = zstd.SpeedFastest
+		} else if chunk.config.CompressionLevel == ZSTD_COMPRESSION_LEVEL_BETTER {
+			compressionLevel = zstd.SpeedBetterCompression
+		} else if chunk.config.CompressionLevel == ZSTD_COMPRESSION_LEVEL_BEST {
+			compressionLevel = zstd.SpeedBestCompression
+		}
+
+		deflater, err := zstd.NewWriter(encryptedBuffer, zstd.WithEncoderLevel(compressionLevel))
+		if err != nil {
+			return err
+		}
+
+		// Make sure we have enough space in encryptedBuffer
+		availableLength := encryptedBuffer.Cap() - len(encryptedBuffer.Bytes())
+		maximumLength := deflater.MaxEncodedSize(chunk.buffer.Len())
+		if availableLength < maximumLength {
+			encryptedBuffer.Grow(maximumLength - availableLength)
+		}
+		_, err = deflater.Write(chunk.buffer.Bytes())
+		if err != nil {
+			return fmt.Errorf("ZSTD compression error: %v", err)
+		}
+
+		err = deflater.Close()
+		if err != nil {
+			return fmt.Errorf("ZSTD compression error: %v", err)
+		}
 	} else if chunk.config.CompressionLevel == DEFAULT_COMPRESSION_LEVEL {
 		encryptedBuffer.Write([]byte("LZ4 "))
 		// Make sure we have enough space in encryptedBuffer
@@ -239,38 +319,99 @@ func (chunk *Chunk) Encrypt(encryptionKey []byte, derivationKey string) (err err
 		return fmt.Errorf("Invalid compression level: %d", chunk.config.CompressionLevel)
 	}
 
-	if len(encryptionKey) == 0 {
+	if len(encryptionKey) > 0 {
+
+		// PKCS7 is used.  The sizes of compressed chunks leak information about the original chunks so we want the padding sizes
+		// to be the maximum allowed by PKCS7
+		dataLength := encryptedBuffer.Len() - offset
+		paddingLength := 256 - dataLength%256
+
+		encryptedBuffer.Write(bytes.Repeat([]byte{byte(paddingLength)}, paddingLength))
+		encryptedBuffer.Write(bytes.Repeat([]byte{0}, gcm.Overhead()))
+
+		// The encrypted data will be appended to the duplicacy banner and the once.
+		encryptedBytes := gcm.Seal(encryptedBuffer.Bytes()[:offset], nonce,
+			encryptedBuffer.Bytes()[offset:offset+dataLength+paddingLength], nil)
+
+		encryptedBuffer.Truncate(len(encryptedBytes))
+	}
+
+	if chunk.config.DataShards == 0 || chunk.config.ParityShards == 0 {
 		chunk.buffer, encryptedBuffer = encryptedBuffer, chunk.buffer
-		return nil
+		return
 	}
 
-	// PKCS7 is used.  Compressed chunk sizes leaks information about the original chunks so we want the padding sizes
-	// to be the maximum allowed by PKCS7
-	dataLength := encryptedBuffer.Len() - offset
-	paddingLength := dataLength % 256
-	if paddingLength == 0 {
-		paddingLength = 256
+	// Start erasure coding
+	encoder, err := reedsolomon.New(chunk.config.DataShards, chunk.config.ParityShards)
+	if err != nil {
+		return err
+	}
+	chunkSize := len(encryptedBuffer.Bytes())
+	shardSize := (chunkSize + chunk.config.DataShards - 1) / chunk.config.DataShards
+	// Append zeros to make the last shard to have the same size as other
+	encryptedBuffer.Write(make([]byte, shardSize * chunk.config.DataShards - chunkSize))
+	// Grow the buffer for parity shards
+	encryptedBuffer.Grow(shardSize * chunk.config.ParityShards)
+	// Now create one slice for each shard, reusing the data in the buffer
+	data := make([][]byte, chunk.config.DataShards + chunk.config.ParityShards)
+	for i := 0; i < chunk.config.DataShards + chunk.config.ParityShards; i++ {
+		data[i] = encryptedBuffer.Bytes()[i * shardSize: (i + 1) * shardSize]
+	}
+	// This populates the parity shard
+	encoder.Encode(data)
+
+	// Prepare the chunk to be uploaded
+	chunk.buffer.Reset()
+	// First the banner
+	chunk.buffer.Write([]byte(ERASURE_CODING_BANNER))
+	// Then the header which includes the chunk size, data/parity and a 2-byte checksum
+	header := make([]byte, 14)
+	binary.LittleEndian.PutUint64(header[0:], uint64(chunkSize))
+	binary.LittleEndian.PutUint16(header[8:], uint16(chunk.config.DataShards))
+	binary.LittleEndian.PutUint16(header[10:], uint16(chunk.config.ParityShards))
+	header[12] = header[0] ^ header[2] ^ header[4] ^ header[6] ^ header[8] ^ header[10]
+	header[13] = header[1] ^ header[3] ^ header[5] ^ header[7] ^ header[9] ^ header[11]
+	chunk.buffer.Write(header)
+	// Calculate the highway hash for each shard
+	hashKey := make([]byte, 32)
+	for _, part := range data {
+		hasher, err := highwayhash.New(hashKey)
+		if err != nil {
+			return err
+		}
+		_, err = hasher.Write(part)
+		if err != nil {
+			return err
+		}
+		chunk.buffer.Write(hasher.Sum(nil))
 	}
 
-	encryptedBuffer.Write(bytes.Repeat([]byte{byte(paddingLength)}, paddingLength))
-	encryptedBuffer.Write(bytes.Repeat([]byte{0}, gcm.Overhead()))
-
-	// The encrypted data will be appended to the duplicacy header and the once.
-	encryptedBytes := gcm.Seal(encryptedBuffer.Bytes()[:offset], nonce,
-		encryptedBuffer.Bytes()[offset:offset+dataLength+paddingLength], nil)
-
-	encryptedBuffer.Truncate(len(encryptedBytes))
-
-	chunk.buffer, encryptedBuffer = encryptedBuffer, chunk.buffer
+	// Copy the data
+	for _, part := range data {
+		chunk.buffer.Write(part)
+	}
+	// Append the header again for redundancy
+	chunk.buffer.Write(header)
 
 	return nil
+}
 
+// This is to ensure compatibility with Vertical Backup, which still uses HMAC-SHA256 (instead of HMAC-BLAKE2) to
+// derive the key used to encrypt/decrypt files and chunks.
+
+var DecryptWithHMACSHA256 = false
+
+func init() {
+	if value, found := os.LookupEnv("DUPLICACY_DECRYPT_WITH_HMACSHA256"); found && value != "0" {
+		DecryptWithHMACSHA256 = true
+	}
 }
 
 // Decrypt decrypts the encrypted data stored in the chunk buffer.  If derivationKey is not nil, the actual
 // encryption key will be HMAC-SHA256(encryptionKey, derivationKey).
-func (chunk *Chunk) Decrypt(encryptionKey []byte, derivationKey string) (err error) {
+func (chunk *Chunk) Decrypt(encryptionKey []byte, derivationKey string) (err error, rewriteNeeded bool) {
 
+	rewriteNeeded = false
 	var offset int
 
 	encryptedBuffer := AllocateChunkBuffer()
@@ -280,49 +421,213 @@ func (chunk *Chunk) Decrypt(encryptionKey []byte, derivationKey string) (err err
 	}()
 
 	chunk.buffer, encryptedBuffer = encryptedBuffer, chunk.buffer
+	bannerLength := len(ENCRYPTION_BANNER)
+
+	if len(encryptedBuffer.Bytes()) > bannerLength && string(encryptedBuffer.Bytes()[:bannerLength]) == ERASURE_CODING_BANNER {
+
+		// The chunk was encoded with erasure coding
+		if len(encryptedBuffer.Bytes()) < bannerLength + 14 {
+			return fmt.Errorf("Erasure coding header truncated (%d bytes)", len(encryptedBuffer.Bytes())), false
+		}
+		// Check the header checksum
+		header := encryptedBuffer.Bytes()[bannerLength: bannerLength + 14]
+		if header[12] != header[0] ^ header[2] ^ header[4] ^ header[6] ^ header[8] ^ header[10] ||
+		   header[13] != header[1] ^ header[3] ^ header[5] ^ header[7] ^ header[9] ^ header[11] {
+			return fmt.Errorf("Erasure coding header corrupted (%x)", header), false
+		}
+
+		// Read the parameters
+		chunkSize := int(binary.LittleEndian.Uint64(header[0:8]))
+		dataShards := int(binary.LittleEndian.Uint16(header[8:10]))
+		parityShards := int(binary.LittleEndian.Uint16(header[10:12]))
+		shardSize := (chunkSize + chunk.config.DataShards - 1) / chunk.config.DataShards
+		// This is the length the chunk file should have
+		expectedLength := bannerLength + 2 * len(header) + (dataShards + parityShards) * (shardSize + 32)
+		// The minimum length that can be recovered from
+		minimumLength := bannerLength + len(header) + (dataShards + parityShards) * 32 + dataShards * shardSize
+		LOG_DEBUG("CHUNK_ERASURECODE", "Chunk size: %d bytes, data size: %d, parity: %d/%d", chunkSize, len(encryptedBuffer.Bytes()), dataShards, parityShards)
+		if len(encryptedBuffer.Bytes()) > expectedLength {
+			LOG_WARN("CHUNK_ERASURECODE", "Chunk has %d bytes (instead of %d)", len(encryptedBuffer.Bytes()), expectedLength)
+		} else if len(encryptedBuffer.Bytes()) == expectedLength {
+			// Correct size; fall through
+		} else if len(encryptedBuffer.Bytes()) > minimumLength {
+			LOG_WARN("CHUNK_ERASURECODE", "Chunk is truncated (%d out of %d bytes)", len(encryptedBuffer.Bytes()), expectedLength)
+		} else {
+			return fmt.Errorf("Not enough chunk data for recovery; chunk size: %d bytes, data size: %d, parity: %d/%d", chunkSize, len(encryptedBuffer.Bytes()), dataShards, parityShards), false
+		}
+
+		// Where the hashes start
+		hashOffset := bannerLength + len(header)
+		// Where the data start
+		dataOffset := hashOffset + (dataShards + parityShards) * 32
+
+		data := make([][]byte, dataShards + parityShards)
+		recoveryNeeded := false
+		hashKey := make([]byte, 32)
+		availableShards := 0
+		wrongHashDetected := false
+
+		for i := 0; i < dataShards + parityShards; i++ {
+			start := dataOffset + i * shardSize
+			if start + shardSize > len(encryptedBuffer.Bytes()) {
+				// the current shard is incomplete
+				break
+			}
+			// Now verify the hash
+			hasher, err := highwayhash.New(hashKey)
+			if err != nil {
+				return err, false
+			}
+			_, err = hasher.Write(encryptedBuffer.Bytes()[start: start + shardSize])
+			if err != nil {
+				return err, false
+			}
+
+			matched := bytes.Compare(hasher.Sum(nil), encryptedBuffer.Bytes()[hashOffset + i * 32: hashOffset + (i + 1) * 32]) == 0
+
+			if !matched && runtime.GOARCH == "arm64" {
+				hasher, err := wronghighwayhash.New(hashKey)
+				if err == nil {
+					_, err = hasher.Write(encryptedBuffer.Bytes()[start: start + shardSize])
+					if err == nil {
+						matched = bytes.Compare(hasher.Sum(nil), encryptedBuffer.Bytes()[hashOffset + i * 32: hashOffset + (i + 1) * 32]) == 0
+						if matched && !wrongHashDetected {
+							LOG_WARN("CHUNK_ERASURECODE", "Hash for shard %d was calculated with a wrong version of highwayhash", i)
+							wrongHashDetected = true
+							rewriteNeeded = true
+						}
+					}
+				}
+			}
+
+			if !matched {
+				if i < dataShards {
+					recoveryNeeded = true
+					rewriteNeeded = true
+				}
+			} else {
+				// The shard is good
+				data[i] = encryptedBuffer.Bytes()[start: start + shardSize]
+				availableShards++
+				if availableShards >= dataShards {
+					// We have enough shards to recover; skip the remaining shards
+					break
+				}
+			}
+		}
+
+		if !recoveryNeeded {
+			// Remove the padding zeros from the last shard
+			encryptedBuffer.Truncate(dataOffset + chunkSize)
+			// Skip the header and hashes
+			encryptedBuffer.Read(encryptedBuffer.Bytes()[:dataOffset])
+		} else {
+			if availableShards < dataShards {
+				return fmt.Errorf("Not enough chunk data for recover; only %d out of %d shards are complete", availableShards, dataShards + parityShards), false
+			}
+
+			// Show the validity of shards using a string of * and -
+			slots := ""
+			for _, part := range data {
+				if len(part) != 0 {
+					slots += "*"
+				} else {
+					slots += "-"
+				}
+			}
+
+			LOG_WARN("CHUNK_ERASURECODE", "Recovering a %d byte chunk from %d byte shards: %s", chunkSize, shardSize, slots)
+			encoder, err := reedsolomon.New(dataShards, parityShards)
+			if err != nil {
+				return err, false
+			}
+			err = encoder.Reconstruct(data)
+			if err != nil {
+				return err, false
+			}
+			LOG_DEBUG("CHUNK_ERASURECODE", "Chunk data successfully recovered")
+			buffer := AllocateChunkBuffer()
+			buffer.Reset()
+			for i := 0; i < dataShards; i++ {
+				buffer.Write(data[i])
+			}
+			buffer.Truncate(chunkSize)
+
+			ReleaseChunkBuffer(encryptedBuffer)
+			encryptedBuffer = buffer
+		}
+
+	}
 
 	if len(encryptionKey) > 0 {
 
 		key := encryptionKey
 
 		if len(derivationKey) > 0 {
-			hasher := chunk.config.NewKeyedHasher([]byte(derivationKey))
+			var hasher hash.Hash
+			if DecryptWithHMACSHA256 {
+				hasher = hmac.New(sha256.New, []byte(derivationKey))
+			} else {
+				hasher = chunk.config.NewKeyedHasher([]byte(derivationKey))
+			}
+
 			hasher.Write(encryptionKey)
 			key = hasher.Sum(nil)
 		}
 
+		if len(encryptedBuffer.Bytes()) < bannerLength + 12 {
+			return fmt.Errorf("No enough encrypted data (%d bytes) provided", len(encryptedBuffer.Bytes())), false
+		}
+
+		if string(encryptedBuffer.Bytes()[:bannerLength-1]) != ENCRYPTION_BANNER[:bannerLength-1] {
+			return fmt.Errorf("The storage doesn't seem to be encrypted"), false
+		}
+
+		encryptionVersion := encryptedBuffer.Bytes()[bannerLength-1]
+		if encryptionVersion != 0 && encryptionVersion != ENCRYPTION_VERSION_RSA {
+			return fmt.Errorf("Unsupported encryption version %d", encryptionVersion), false
+		}
+
+		if encryptionVersion == ENCRYPTION_VERSION_RSA {
+			if chunk.config.rsaPrivateKey == nil {
+				LOG_ERROR("CHUNK_DECRYPT", "An RSA private key is required to decrypt the chunk")
+				return fmt.Errorf("An RSA private key is required to decrypt the chunk"), false
+			}
+
+			encryptedKeyLength := binary.LittleEndian.Uint16(encryptedBuffer.Bytes()[bannerLength:bannerLength+2])
+
+			if len(encryptedBuffer.Bytes()) < bannerLength + 14 + int(encryptedKeyLength) {
+				return fmt.Errorf("No enough encrypted data (%d bytes) provided", len(encryptedBuffer.Bytes())), false
+			}
+
+			encryptedKey := encryptedBuffer.Bytes()[bannerLength + 2:bannerLength + 2 + int(encryptedKeyLength)]
+			bannerLength += 2 + int(encryptedKeyLength)
+
+			decryptedKey, err := rsa.DecryptOAEP(sha256.New(), rand.Reader, chunk.config.rsaPrivateKey, encryptedKey, nil)
+			if err != nil {
+				return err, false
+			}
+			key = decryptedKey
+		}
+
 		aesBlock, err := aes.NewCipher(key)
 		if err != nil {
-			return err
+			return err, false
 		}
 
 		gcm, err := cipher.NewGCM(aesBlock)
 		if err != nil {
-			return err
+			return err, false
 		}
 
-		headerLength := len(ENCRYPTION_HEADER)
-		offset = headerLength + gcm.NonceSize()
-
-		if len(encryptedBuffer.Bytes()) < offset {
-			return fmt.Errorf("No enough encrypted data (%d bytes) provided", len(encryptedBuffer.Bytes()))
-		}
-
-		if string(encryptedBuffer.Bytes()[:headerLength-1]) != ENCRYPTION_HEADER[:headerLength-1] {
-			return fmt.Errorf("The storage doesn't seem to be encrypted")
-		}
-
-		if encryptedBuffer.Bytes()[headerLength-1] != 0 {
-			return fmt.Errorf("Unsupported encryption version %d", encryptedBuffer.Bytes()[headerLength-1])
-		}
-
-		nonce := encryptedBuffer.Bytes()[headerLength:offset]
+		offset = bannerLength + gcm.NonceSize()
+		nonce := encryptedBuffer.Bytes()[bannerLength:offset]
 
 		decryptedBytes, err := gcm.Open(encryptedBuffer.Bytes()[:offset], nonce,
 			encryptedBuffer.Bytes()[offset:], nil)
 
 		if err != nil {
-			return err
+			return err, false
 		}
 
 		paddingLength := int(decryptedBytes[len(decryptedBytes)-1])
@@ -330,14 +635,14 @@ func (chunk *Chunk) Decrypt(encryptionKey []byte, derivationKey string) (err err
 			paddingLength = 256
 		}
 		if len(decryptedBytes) <= paddingLength {
-			return fmt.Errorf("Incorrect padding length %d out of %d bytes", paddingLength, len(decryptedBytes))
+			return fmt.Errorf("Incorrect padding length %d out of %d bytes", paddingLength, len(decryptedBytes)), false
 		}
 
 		for i := 0; i < paddingLength; i++ {
 			padding := decryptedBytes[len(decryptedBytes)-1-i]
 			if padding != byte(paddingLength) {
 				return fmt.Errorf("Incorrect padding of length %d: %x", paddingLength,
-					decryptedBytes[len(decryptedBytes)-paddingLength:])
+					decryptedBytes[len(decryptedBytes)-paddingLength:]), false
 			}
 		}
 
@@ -351,18 +656,36 @@ func (chunk *Chunk) Decrypt(encryptionKey []byte, derivationKey string) (err err
 		chunk.buffer.Reset()
 		decompressed, err := lz4.Decode(chunk.buffer.Bytes(), encryptedBuffer.Bytes()[4:])
 		if err != nil {
-			return err
+			return err, false
 		}
 
 		chunk.buffer.Write(decompressed)
 		chunk.hasher = chunk.config.NewKeyedHasher(chunk.config.HashKey)
 		chunk.hasher.Write(decompressed)
 		chunk.hash = nil
-		return nil
+		return nil, rewriteNeeded
 	}
+
+	if len(compressed) > 4 && string(compressed[:4]) == "ZSTD" {
+		chunk.buffer.Reset()
+		chunk.hasher = chunk.config.NewKeyedHasher(chunk.config.HashKey)
+		chunk.hash = nil
+
+		encryptedBuffer.Read(encryptedBuffer.Bytes()[:4])
+		inflater, err := zstd.NewReader(encryptedBuffer)
+		if err != nil {
+			return err, false
+		}
+		defer inflater.Close()
+		if _, err = io.Copy(chunk, inflater); err != nil {
+			return err, false
+		}
+		return nil, rewriteNeeded
+	}
+
 	inflater, err := zlib.NewReader(encryptedBuffer)
 	if err != nil {
-		return err
+		return err, false
 	}
 
 	defer inflater.Close()
@@ -372,9 +695,9 @@ func (chunk *Chunk) Decrypt(encryptionKey []byte, derivationKey string) (err err
 	chunk.hash = nil
 
 	if _, err = io.Copy(chunk, inflater); err != nil {
-		return err
+		return err, false
 	}
 
-	return nil
+	return nil, rewriteNeeded
 
 }

src/duplicacy_chunk_test.go

@@ -7,11 +7,51 @@ package duplicacy
 import (
 	"bytes"
 	crypto_rand "crypto/rand"
+	"crypto/rsa"
 	"math/rand"
 	"testing"
 )
 
-func TestChunk(t *testing.T) {
+func TestErasureCoding(t *testing.T) {
+	key := []byte("duplicacydefault")
+
+	config := CreateConfig()
+	config.HashKey = key
+	config.IDKey = key
+	config.MinimumChunkSize = 100
+	config.CompressionLevel = DEFAULT_COMPRESSION_LEVEL
+	config.DataShards = 5
+    config.ParityShards = 2
+
+	chunk := CreateChunk(config, true)
+	chunk.Reset(true)
+	data := make([]byte, 100)
+	for i := 0; i < len(data); i++ {
+		data[i] = byte(i)
+	}
+	chunk.Write(data)
+	err := chunk.Encrypt([]byte(""), "", false)
+	if err != nil {
+		t.Errorf("Failed to encrypt the test data: %v", err)
+		return
+	}
+
+	encryptedData := make([]byte, chunk.GetLength())
+	copy(encryptedData, chunk.GetBytes())
+
+	crypto_rand.Read(encryptedData[280:300])
+
+	chunk.Reset(false)
+	chunk.Write(encryptedData)
+	err, _ = chunk.Decrypt([]byte(""), "")
+	if err != nil {
+		t.Errorf("Failed to decrypt the data: %v", err)
+		return
+	}
+	return
+}
+
+func TestChunkBasic(t *testing.T) {
 
 	key := []byte("duplicacydefault")
 
@@ -22,6 +62,20 @@ func TestChunk(t *testing.T) {
 	config.CompressionLevel = DEFAULT_COMPRESSION_LEVEL
 	maxSize := 1000000
 
+	if *testRSAEncryption {
+		privateKey, err := rsa.GenerateKey(crypto_rand.Reader, 2048)
+		if err != nil {
+			t.Errorf("Failed to generate a random private key: %v", err)
+		}
+		config.rsaPrivateKey = privateKey
+		config.rsaPublicKey = privateKey.Public().(*rsa.PublicKey)
+	}
+
+	if *testErasureCoding {
+		config.DataShards = 5
+		config.ParityShards = 2
+	}
+
 	for i := 0; i < 500; i++ {
 
 		size := rand.Int() % maxSize
@@ -35,7 +89,7 @@ func TestChunk(t *testing.T) {
 		hash := chunk.GetHash()
 		id := chunk.GetID()
 
-		err := chunk.Encrypt(key, "")
+		err := chunk.Encrypt(key, "", false)
 		if err != nil {
 			t.Errorf("Failed to encrypt the data: %v", err)
 			continue
@@ -44,9 +98,19 @@ func TestChunk(t *testing.T) {
 		encryptedData := make([]byte, chunk.GetLength())
 		copy(encryptedData, chunk.GetBytes())
 
+		if *testErasureCoding {
+			offset := 24 + 32 * 7
+			start := rand.Int() % (len(encryptedData) - offset) + offset
+			length := (len(encryptedData) - offset) / 7
+			if start + length > len(encryptedData) {
+				length = len(encryptedData) - start
+			}
+			crypto_rand.Read(encryptedData[start: start+length])
+		}
+
 		chunk.Reset(false)
 		chunk.Write(encryptedData)
-		err = chunk.Decrypt(key, "")
+		err, _ = chunk.Decrypt(key, "")
 		if err != nil {
 			t.Errorf("Failed to decrypt the data: %v", err)
 			continue
@@ -63,7 +127,7 @@ func TestChunk(t *testing.T) {
 		}
 
 		if bytes.Compare(plainData, decryptedData) != 0 {
-			t.Logf("orginal length: %d, decrypted length: %d", len(plainData), len(decryptedData))
+			t.Logf("Original length: %d, decrypted length: %d", len(plainData), len(decryptedData))
 			t.Errorf("Original data:\n%x\nDecrypted data:\n%x\n", plainData, decryptedData)
 		}
 

src/duplicacy_chunkdownloader.go

@@ -5,7 +5,6 @@
 package duplicacy
 
 import (
-	"io"
 	"sync/atomic"
 	"time"
 )
@@ -20,73 +19,47 @@ type ChunkDownloadTask struct {
 	isDownloading bool   // 'true' means the chunk has been downloaded or is being downloaded
 }
 
-// ChunkDownloadCompletion represents the nofication when a chunk has been downloaded.
 type ChunkDownloadCompletion struct {
-	chunkIndex int    // The index of this chunk in the chunk list
-	chunk      *Chunk // The chunk that has been downloaded
+	chunk      *Chunk
+	chunkIndex int
 }
 
-// ChunkDownloader is capable of performing multi-threaded downloading.  Chunks to be downloaded are first organized
+// ChunkDownloader is a wrapper of ChunkOperator and is only used by the restore procedure.capable of performing multi-threaded downloading.  Chunks to be downloaded are first organized
 // as a list of ChunkDownloadTasks, with only the chunkHash field initialized.  When a chunk is needed, the
 // corresponding ChunkDownloadTask is sent to the dowloading goroutine.  Once a chunk is downloaded, it will be
 // inserted in the completed task list.
 type ChunkDownloader struct {
-	config         *Config      // Associated config
-	storage        Storage      // Download from this storage
-	snapshotCache  *FileStorage // Used as cache if not nil; usually for downloading snapshot chunks
-	showStatistics bool         // Show a stats log for each chunk if true
-	threads        int          // Number of threads
+
+	operator                  *ChunkOperator
+
+	totalChunkSize            int64 // Total chunk size
+	downloadedChunkSize       int64 // Downloaded chunk size
 
 	taskList       []ChunkDownloadTask // The list of chunks to be downloaded
 	completedTasks map[int]bool        // Store downloaded chunks
 	lastChunkIndex int                 // a monotonically increasing number indicating the last chunk to be downloaded
 
-	taskQueue         chan ChunkDownloadTask       // Downloading goroutines are waiting on this channel for input
-	stopChannel       chan bool                    // Used to stop the dowloading goroutines
 	completionChannel chan ChunkDownloadCompletion // A downloading goroutine sends back the chunk via this channel after downloading
 
 	startTime                 int64 // The time it starts downloading
-	totalChunkSize            int64 // Total chunk size
-	downloadedChunkSize       int64 // Downloaded chunk size
 	numberOfDownloadedChunks  int   // The number of chunks that have been downloaded
 	numberOfDownloadingChunks int   // The number of chunks still being downloaded
 	numberOfActiveChunks      int   // The number of chunks that is being downloaded or has been downloaded but not reclaimed
 }
 
-func CreateChunkDownloader(config *Config, storage Storage, snapshotCache *FileStorage, showStatistics bool, threads int) *ChunkDownloader {
+func CreateChunkDownloader(operator *ChunkOperator) *ChunkDownloader {
 	downloader := &ChunkDownloader{
-		config:         config,
-		storage:        storage,
-		snapshotCache:  snapshotCache,
-		showStatistics: showStatistics,
-		threads:        threads,
+		operator:       operator,
 
 		taskList:       nil,
 		completedTasks: make(map[int]bool),
 		lastChunkIndex: 0,
 
-		taskQueue:         make(chan ChunkDownloadTask, threads),
-		stopChannel:       make(chan bool),
 		completionChannel: make(chan ChunkDownloadCompletion),
 
 		startTime: time.Now().Unix(),
 	}
 
-	// Start the downloading goroutines
-	for i := 0; i < downloader.threads; i++ {
-		go func(threadIndex int) {
-			defer CatchLogException()
-			for {
-				select {
-				case task := <-downloader.taskQueue:
-					downloader.Download(threadIndex, task)
-				case <-downloader.stopChannel:
-					return
-				}
-			}
-		}(i)
-	}
-
 	return downloader
 }
 
@@ -122,25 +95,7 @@ func (downloader *ChunkDownloader) AddFiles(snapshot *Snapshot, files []*Entry)
 			maximumChunks = file.EndChunk - file.StartChunk
 		}
 	}
-}
-
-// AddChunk adds a single chunk the download list.
-func (downloader *ChunkDownloader) AddChunk(chunkHash string) int {
-	task := ChunkDownloadTask{
-		chunkIndex:    len(downloader.taskList),
-		chunkHash:     chunkHash,
-		chunkLength:   0,
-		needed:        true,
-		isDownloading: false,
-	}
-	downloader.taskList = append(downloader.taskList, task)
-	if downloader.numberOfActiveChunks < downloader.threads {
-		downloader.taskQueue <- task
-		downloader.numberOfDownloadingChunks++
-		downloader.numberOfActiveChunks++
-		downloader.taskList[len(downloader.taskList)-1].isDownloading = true
-	}
-	return len(downloader.taskList) - 1
+	downloader.operator.totalChunkSize = downloader.totalChunkSize
 }
 
 // Prefetch adds up to 'threads' chunks needed by a file to the download list
@@ -153,20 +108,22 @@ func (downloader *ChunkDownloader) Prefetch(file *Entry) {
 		task := &downloader.taskList[i]
 		if task.needed {
 			if !task.isDownloading {
-				if downloader.numberOfActiveChunks >= downloader.threads {
+				if downloader.numberOfActiveChunks >= downloader.operator.threads {
 					return
 				}
 
 				LOG_DEBUG("DOWNLOAD_PREFETCH", "Prefetching %s chunk %s", file.Path,
-					downloader.config.GetChunkIDFromHash(task.chunkHash))
-				downloader.taskQueue <- *task
+					downloader.operator.config.GetChunkIDFromHash(task.chunkHash))
+				downloader.operator.DownloadAsync(task.chunkHash, i, false, func (chunk *Chunk, chunkIndex int) {
+					downloader.completionChannel <- ChunkDownloadCompletion { chunk: chunk, chunkIndex: chunkIndex }
+				})
 				task.isDownloading = true
 				downloader.numberOfDownloadingChunks++
 				downloader.numberOfActiveChunks++
 			}
 		} else {
 			LOG_DEBUG("DOWNLOAD_PREFETCH", "%s chunk %s is not needed", file.Path,
-				downloader.config.GetChunkIDFromHash(task.chunkHash))
+				downloader.operator.config.GetChunkIDFromHash(task.chunkHash))
 		}
 	}
 }
@@ -174,33 +131,52 @@ func (downloader *ChunkDownloader) Prefetch(file *Entry) {
 // Reclaim releases the downloaded chunk to the chunk pool
 func (downloader *ChunkDownloader) Reclaim(chunkIndex int) {
 
-	if downloader.lastChunkIndex == chunkIndex {
+	if downloader.lastChunkIndex >= chunkIndex {
 		return
 	}
 
-	for i, _ := range downloader.completedTasks {
+	for i := range downloader.completedTasks {
 		if i < chunkIndex && downloader.taskList[i].chunk != nil {
-			downloader.config.PutChunk(downloader.taskList[i].chunk)
+			downloader.operator.config.PutChunk(downloader.taskList[i].chunk)
 			downloader.taskList[i].chunk = nil
 			delete(downloader.completedTasks, i)
 			downloader.numberOfActiveChunks--
 		}
 	}
 
+	for i := downloader.lastChunkIndex; i < chunkIndex; i++ {
+		// These chunks are never downloaded if 'isDownloading' is false; note that 'isDownloading' isn't reset to
+		// false after a chunk has been downloaded
+		if !downloader.taskList[i].isDownloading {
+			atomic.AddInt64(&downloader.totalChunkSize, -int64(downloader.taskList[i].chunkLength))
+		}
+	}
 	downloader.lastChunkIndex = chunkIndex
 }
 
+// Return the chunk last downloaded and its hash
+func (downloader *ChunkDownloader) GetLastDownloadedChunk() (chunk *Chunk, chunkHash string) {
+	if downloader.lastChunkIndex >= len(downloader.taskList) {
+		return nil, ""
+	}
+
+	task := downloader.taskList[downloader.lastChunkIndex]
+	return task.chunk, task.chunkHash
+}
+
 // WaitForChunk waits until the specified chunk is ready
 func (downloader *ChunkDownloader) WaitForChunk(chunkIndex int) (chunk *Chunk) {
 
-	// Reclain any chunk not needed
+	// Reclaim any chunk not needed
 	downloader.Reclaim(chunkIndex)
 
 	// If we haven't started download the specified chunk, download it now
 	if !downloader.taskList[chunkIndex].isDownloading {
 		LOG_DEBUG("DOWNLOAD_FETCH", "Fetching chunk %s",
-			downloader.config.GetChunkIDFromHash(downloader.taskList[chunkIndex].chunkHash))
-		downloader.taskQueue <- downloader.taskList[chunkIndex]
+			downloader.operator.config.GetChunkIDFromHash(downloader.taskList[chunkIndex].chunkHash))
+		downloader.operator.DownloadAsync(downloader.taskList[chunkIndex].chunkHash, chunkIndex, false, func (chunk *Chunk, chunkIndex int) {
+			downloader.completionChannel <- ChunkDownloadCompletion { chunk: chunk, chunkIndex: chunkIndex }
+		})
 		downloader.taskList[chunkIndex].isDownloading = true
 		downloader.numberOfDownloadingChunks++
 		downloader.numberOfActiveChunks++
@@ -208,7 +184,7 @@ func (downloader *ChunkDownloader) WaitForChunk(chunkIndex int) (chunk *Chunk) {
 
 	// We also need to look ahead and prefetch other chunks as many as permitted by the number of threads
 	for i := chunkIndex + 1; i < len(downloader.taskList); i++ {
-		if downloader.numberOfActiveChunks >= downloader.threads {
+		if downloader.numberOfActiveChunks >= downloader.operator.threads {
 			break
 		}
 		task := &downloader.taskList[i]
@@ -217,8 +193,10 @@ func (downloader *ChunkDownloader) WaitForChunk(chunkIndex int) (chunk *Chunk) {
 		}
 
 		if !task.isDownloading {
-			LOG_DEBUG("DOWNLOAD_PREFETCH", "Prefetching chunk %s", downloader.config.GetChunkIDFromHash(task.chunkHash))
-			downloader.taskQueue <- *task
+			LOG_DEBUG("DOWNLOAD_PREFETCH", "Prefetching chunk %s", downloader.operator.config.GetChunkIDFromHash(task.chunkHash))
+			downloader.operator.DownloadAsync(task.chunkHash, task.chunkIndex, false, func (chunk *Chunk, chunkIndex int) {
+				downloader.completionChannel <- ChunkDownloadCompletion { chunk: chunk, chunkIndex: chunkIndex }
+			})
 			task.isDownloading = true
 			downloader.numberOfDownloadingChunks++
 			downloader.numberOfActiveChunks++
@@ -236,165 +214,45 @@ func (downloader *ChunkDownloader) WaitForChunk(chunkIndex int) (chunk *Chunk) {
 	return downloader.taskList[chunkIndex].chunk
 }
 
-// Stop terminates all downloading goroutines
-func (downloader *ChunkDownloader) Stop() {
-	for downloader.numberOfDownloadingChunks > 0 {
-		completion := <-downloader.completionChannel
-		downloader.completedTasks[completion.chunkIndex] = true
-		downloader.taskList[completion.chunkIndex].chunk = completion.chunk
-		downloader.numberOfDownloadedChunks++
-		downloader.numberOfDownloadingChunks--
-	}
+// WaitForCompletion waits until all chunks have been downloaded
+func (downloader *ChunkDownloader) WaitForCompletion() {
 
-	for i, _ := range downloader.completedTasks {
-		downloader.config.PutChunk(downloader.taskList[i].chunk)
-		downloader.taskList[i].chunk = nil
-		downloader.numberOfActiveChunks--
-	}
+	// Tasks in completedTasks have not been counted by numberOfActiveChunks
+	downloader.numberOfActiveChunks -= len(downloader.completedTasks)
 
-	for i := 0; i < downloader.threads; i++ {
-		downloader.stopChannel <- true
-	}
-}
-
-// Download downloads a chunk from the storage.
-func (downloader *ChunkDownloader) Download(threadIndex int, task ChunkDownloadTask) bool {
-
-	cachedPath := ""
-	chunk := downloader.config.GetChunk()
-	chunkID := downloader.config.GetChunkIDFromHash(task.chunkHash)
-
-	if downloader.snapshotCache != nil && downloader.storage.IsCacheNeeded() {
-
-		var exist bool
-		var err error
-
-		// Reset the chunk with a hasher -- we're reading from the cache where chunk are not encrypted or compressed
-		chunk.Reset(true)
-
-		cachedPath, exist, _, err = downloader.snapshotCache.FindChunk(threadIndex, chunkID, false)
-		if err != nil {
-			LOG_WARN("DOWNLOAD_CACHE", "Failed to find the cache path for the chunk %s: %v", chunkID, err)
-		} else if exist {
-			err = downloader.snapshotCache.DownloadFile(0, cachedPath, chunk)
-			if err != nil {
-				LOG_WARN("DOWNLOAD_CACHE", "Failed to load the chunk %s from the snapshot cache: %v", chunkID, err)
-			} else {
-				actualChunkID := chunk.GetID()
-				if actualChunkID != chunkID {
-					LOG_WARN("DOWNLOAD_CACHE_CORRUPTED",
-						"The chunk %s load from the snapshot cache has a hash id of %s", chunkID, actualChunkID)
-				} else {
-					LOG_DEBUG("CHUNK_CACHE", "Chunk %s has been loaded from the snapshot cache", chunkID)
-
-					downloader.completionChannel <- ChunkDownloadCompletion{chunk: chunk, chunkIndex: task.chunkIndex}
-					return false
-				}
-			}
+	// find the completed task with the largest index; we'll start from the next index
+	for index := range downloader.completedTasks {
+		if downloader.lastChunkIndex < index {
+			downloader.lastChunkIndex = index
 		}
 	}
 
-	// Reset the chunk without a hasher -- the downloaded content will be encrypted and/or compressed and the hasher
-	// will be set up before the encryption
-	chunk.Reset(false)
+	// Looping until there isn't a download task in progress
+	for downloader.numberOfActiveChunks > 0 || downloader.lastChunkIndex + 1 < len(downloader.taskList) {
 
-	// Find the chunk by ID first.
-	chunkPath, exist, _, err := downloader.storage.FindChunk(threadIndex, chunkID, false)
-	if err != nil {
-		LOG_ERROR("DOWNLOAD_CHUNK", "Failed to find the chunk %s: %v", chunkID, err)
-		return false
-	}
-
-	if !exist {
-		// No chunk is found.  Have to find it in the fossil pool again.
-		chunkPath, exist, _, err = downloader.storage.FindChunk(threadIndex, chunkID, true)
-		if err != nil {
-			LOG_ERROR("DOWNLOAD_CHUNK", "Failed to find the chunk %s: %v", chunkID, err)
-			return false
+		// Wait for a completion event first
+		if downloader.numberOfActiveChunks > 0 {
+			completion := <-downloader.completionChannel
+			downloader.operator.config.PutChunk(completion.chunk)
+			downloader.numberOfActiveChunks--
+			downloader.numberOfDownloadedChunks++
+			downloader.numberOfDownloadingChunks--
 		}
 
-		if !exist {
-			// A chunk is not found.  This is a serious error and hopefully it will never happen.
-			if err != nil {
-				LOG_FATAL("DOWNLOAD_CHUNK", "Chunk %s can't be found: %v", chunkID, err)
-			} else {
-				LOG_FATAL("DOWNLOAD_CHUNK", "Chunk %s can't be found", chunkID)
-			}
-			return false
-		}
-		LOG_DEBUG("CHUNK_FOSSIL", "Chunk %s has been marked as a fossil", chunkID)
-	}
-
-	const MaxDownloadAttempts = 3
-	for downloadAttempt := 0; ; downloadAttempt++ {
-		err = downloader.storage.DownloadFile(threadIndex, chunkPath, chunk)
-		if err != nil {
-			if err == io.ErrUnexpectedEOF && downloadAttempt < MaxDownloadAttempts {
-				LOG_WARN("DOWNLOAD_RETRY", "Failed to download the chunk %s: %v; retrying", chunkID, err)
-				chunk.Reset(false)
+		// Pass the tasks one by one to the download queue
+		if downloader.lastChunkIndex + 1 < len(downloader.taskList) {
+			task := &downloader.taskList[downloader.lastChunkIndex + 1]
+			if task.isDownloading {
+				downloader.lastChunkIndex++
 				continue
-			} else {
-				LOG_ERROR("DOWNLOAD_CHUNK", "Failed to download the chunk %s: %v", chunkID, err)
-				return false
 			}
-		}
-
-		err = chunk.Decrypt(downloader.config.ChunkKey, task.chunkHash)
-		if err != nil {
-			if downloadAttempt < MaxDownloadAttempts {
-				LOG_WARN("DOWNLOAD_RETRY", "Failed to decrypt the chunk %s: %v; retrying", chunkID, err)
-				chunk.Reset(false)
-				continue
-			} else {
-				LOG_ERROR("DOWNLOAD_DECRYPT", "Failed to decrypt the chunk %s: %v", chunkID, err)
-				return false
-			}
-		}
-
-		actualChunkID := chunk.GetID()
-		if actualChunkID != chunkID {
-			if downloadAttempt < MaxDownloadAttempts {
-				LOG_WARN("DOWNLOAD_RETRY", "The chunk %s has a hash id of %s; retrying", chunkID, actualChunkID)
-				chunk.Reset(false)
-				continue
-			} else {
-				LOG_FATAL("DOWNLOAD_CORRUPTED", "The chunk %s has a hash id of %s", chunkID, actualChunkID)
-				return false
-			}
-		}
-
-		break
-	}
-
-	if len(cachedPath) > 0 {
-		// Save a copy to the local snapshot cache
-		err = downloader.snapshotCache.UploadFile(threadIndex, cachedPath, chunk.GetBytes())
-		if err != nil {
-			LOG_WARN("DOWNLOAD_CACHE", "Failed to add the chunk %s to the snapshot cache: %v", chunkID, err)
+			downloader.operator.DownloadAsync(task.chunkHash, task.chunkIndex, false, func (chunk *Chunk, chunkIndex int) {
+				downloader.completionChannel <- ChunkDownloadCompletion { chunk: chunk, chunkIndex: chunkIndex }
+			})
+			task.isDownloading = true
+			downloader.numberOfDownloadingChunks++
+			downloader.numberOfActiveChunks++
+			downloader.lastChunkIndex++
 		}
 	}
-
-	downloadedChunkSize := atomic.AddInt64(&downloader.downloadedChunkSize, int64(chunk.GetLength()))
-
-	if (downloader.showStatistics || IsTracing()) && downloader.totalChunkSize > 0 {
-
-		now := time.Now().Unix()
-		if now <= downloader.startTime {
-			now = downloader.startTime + 1
-		}
-		speed := downloadedChunkSize / (now - downloader.startTime)
-		remainingTime := int64(0)
-		if speed > 0 {
-			remainingTime = (downloader.totalChunkSize-downloadedChunkSize)/speed + 1
-		}
-		percentage := float32(downloadedChunkSize * 1000 / downloader.totalChunkSize)
-		LOG_INFO("DOWNLOAD_PROGRESS", "Downloaded chunk %d size %d, %sB/s %s %.1f%%",
-			task.chunkIndex+1, chunk.GetLength(),
-			PrettySize(speed), PrettyTime(remainingTime), percentage/10)
-	} else {
-		LOG_DEBUG("CHUNK_DOWNLOAD", "Chunk %s has been downloaded", chunkID)
-	}
-
-	downloader.completionChannel <- ChunkDownloadCompletion{chunk: chunk, chunkIndex: task.chunkIndex}
-	return true
-}
+}

src/duplicacy_chunkmaker.go

@@ -25,15 +25,20 @@ type ChunkMaker struct {
 	bufferSize  int
 	bufferStart int
 
+	minimumReached bool
+	hashSum uint64
+	chunk *Chunk
+
 	config *Config
 
 	hashOnly      bool
 	hashOnlyChunk *Chunk
+
 }
 
 // CreateChunkMaker creates a chunk maker.  'randomSeed' is used to generate the character-to-integer table needed by
 // buzhash.
-func CreateChunkMaker(config *Config, hashOnly bool) *ChunkMaker {
+func CreateFileChunkMaker(config *Config, hashOnly bool) *ChunkMaker {
 	size := 1
 	for size*2 <= config.AverageChunkSize {
 		size *= 2
@@ -67,6 +72,33 @@ func CreateChunkMaker(config *Config, hashOnly bool) *ChunkMaker {
 	}
 
 	maker.buffer = make([]byte, 2*config.MinimumChunkSize)
+	maker.bufferStart = 0
+	maker.bufferSize = 0
+
+	maker.startNewChunk()
+
+	return maker
+}
+
+// CreateMetaDataChunkMaker creates a chunk maker that always uses the variable-sized chunking algorithm
+func CreateMetaDataChunkMaker(config *Config, chunkSize int) *ChunkMaker {
+
+	size := 1
+	for size*2 <= chunkSize {
+		size *= 2
+	}
+
+	if size != chunkSize {
+		LOG_FATAL("CHUNK_SIZE", "Invalid metadata chunk size: %d is not a power of 2", chunkSize)
+		return nil
+	}
+
+	maker := CreateFileChunkMaker(config, false)
+	maker.hashMask = uint64(chunkSize - 1)
+	maker.maximumChunkSize = chunkSize * 4
+	maker.minimumChunkSize = chunkSize / 4
+	maker.bufferCapacity = 2 * maker.minimumChunkSize
+	maker.buffer = make([]byte, maker.bufferCapacity)
 
 	return maker
 }
@@ -90,62 +122,50 @@ func (maker *ChunkMaker) buzhashUpdate(sum uint64, out byte, in byte, length int
 	return rotateLeftByOne(sum) ^ rotateLeft(maker.randomTable[out], uint(length)) ^ maker.randomTable[in]
 }
 
-// ForEachChunk reads data from 'reader'.  If EOF is encountered, it will call 'nextReader' to ask for next file.  If
-// 'nextReader' returns false, it will process remaining data in the buffer and then quit.  When a chunk is identified,
-// it will call 'endOfChunk' to return the chunk size and a boolean flag indicating if it is the last chunk.
-func (maker *ChunkMaker) ForEachChunk(reader io.Reader, endOfChunk func(chunk *Chunk, final bool),
-	nextReader func(size int64, hash string) (io.Reader, bool)) {
+func (maker *ChunkMaker) startNewChunk() (chunk *Chunk) {
+	maker.hashSum = 0
+	maker.minimumReached = false
+	if maker.hashOnly {
+		maker.chunk = maker.hashOnlyChunk
+		maker.chunk.Reset(true)
+	} else {
+		maker.chunk = maker.config.GetChunk()
+		maker.chunk.Reset(true)
+	}
+	return
+}
 
-	maker.bufferStart = 0
-	maker.bufferSize = 0
-
-	var minimumReached bool
-	var hashSum uint64
-	var chunk *Chunk
+func (maker *ChunkMaker) AddData(reader io.Reader, sendChunk func(*Chunk)) (int64, string) {
 
+	isEOF := false
 	fileSize := int64(0)
 	fileHasher := maker.config.NewFileHasher()
 
-	// Start a new chunk.
-	startNewChunk := func() {
-		hashSum = 0
-		minimumReached = false
-		if maker.hashOnly {
-			chunk = maker.hashOnlyChunk
-			chunk.Reset(true)
-		} else {
-			chunk = maker.config.GetChunk()
-			chunk.Reset(true)
-		}
-	}
-
 	// Move data from the buffer to the chunk.
 	fill := func(count int) {
+
 		if maker.bufferStart+count < maker.bufferCapacity {
-			chunk.Write(maker.buffer[maker.bufferStart : maker.bufferStart+count])
+			maker.chunk.Write(maker.buffer[maker.bufferStart : maker.bufferStart+count])
 			maker.bufferStart += count
 			maker.bufferSize -= count
 		} else {
-			chunk.Write(maker.buffer[maker.bufferStart:])
-			chunk.Write(maker.buffer[:count-(maker.bufferCapacity-maker.bufferStart)])
+			maker.chunk.Write(maker.buffer[maker.bufferStart:])
+			maker.chunk.Write(maker.buffer[:count-(maker.bufferCapacity-maker.bufferStart)])
 			maker.bufferStart = count - (maker.bufferCapacity - maker.bufferStart)
 			maker.bufferSize -= count
 		}
 	}
 
-	startNewChunk()
-
 	var err error
 
-	isEOF := false
-
 	if maker.minimumChunkSize == maker.maximumChunkSize {
 
-		if maker.bufferCapacity < maker.minimumChunkSize {
-			maker.buffer = make([]byte, maker.minimumChunkSize)
+		if reader == nil {
+			return 0, ""
 		}
 
 		for {
+			maker.startNewChunk()
 			maker.bufferStart = 0
 			for maker.bufferStart < maker.minimumChunkSize && !isEOF {
 				count, err := reader.Read(maker.buffer[maker.bufferStart:maker.minimumChunkSize])
@@ -153,7 +173,7 @@ func (maker *ChunkMaker) ForEachChunk(reader io.Reader, endOfChunk func(chunk *C
 				if err != nil {
 					if err != io.EOF {
 						LOG_ERROR("CHUNK_MAKER", "Failed to read %d bytes: %s", count, err.Error())
-						return
+						return 0, ""
 					} else {
 						isEOF = true
 					}
@@ -161,26 +181,15 @@ func (maker *ChunkMaker) ForEachChunk(reader io.Reader, endOfChunk func(chunk *C
 				maker.bufferStart += count
 			}
 
-			fileHasher.Write(maker.buffer[:maker.bufferStart])
-			fileSize += int64(maker.bufferStart)
-			chunk.Write(maker.buffer[:maker.bufferStart])
+			if maker.bufferStart > 0 {
+				fileHasher.Write(maker.buffer[:maker.bufferStart])
+				fileSize += int64(maker.bufferStart)
+				maker.chunk.Write(maker.buffer[:maker.bufferStart])
+				sendChunk(maker.chunk)
+			}
 
 			if isEOF {
-				var ok bool
-				reader, ok = nextReader(fileSize, hex.EncodeToString(fileHasher.Sum(nil)))
-				if !ok {
-					endOfChunk(chunk, true)
-					return
-				} else {
-					endOfChunk(chunk, false)
-					startNewChunk()
-					fileSize = 0
-					fileHasher = maker.config.NewFileHasher()
-					isEOF = false
-				}
-			} else {
-				endOfChunk(chunk, false)
-				startNewChunk()
+				return fileSize, hex.EncodeToString(fileHasher.Sum(nil))
 			}
 		}
 
@@ -189,7 +198,7 @@ func (maker *ChunkMaker) ForEachChunk(reader io.Reader, endOfChunk func(chunk *C
 	for {
 
 		// If the buffer still has some space left and EOF is not seen, read more data.
-		for maker.bufferSize < maker.bufferCapacity && !isEOF {
+		for maker.bufferSize < maker.bufferCapacity && !isEOF && reader != nil {
 			start := maker.bufferStart + maker.bufferSize
 			count := maker.bufferCapacity - start
 			if start >= maker.bufferCapacity {
@@ -201,7 +210,7 @@ func (maker *ChunkMaker) ForEachChunk(reader io.Reader, endOfChunk func(chunk *C
 
 			if err != nil && err != io.EOF {
 				LOG_ERROR("CHUNK_MAKER", "Failed to read %d bytes: %s", count, err.Error())
-				return
+				return 0, ""
 			}
 
 			maker.bufferSize += count
@@ -210,54 +219,55 @@ func (maker *ChunkMaker) ForEachChunk(reader io.Reader, endOfChunk func(chunk *C
 
 			// if EOF is seen, try to switch to next file and continue
 			if err == io.EOF {
-				var ok bool
-				reader, ok = nextReader(fileSize, hex.EncodeToString(fileHasher.Sum(nil)))
-				if !ok {
-					isEOF = true
-				} else {
-					fileSize = 0
-					fileHasher = maker.config.NewFileHasher()
-					isEOF = false
-				}
+				isEOF = true
+				break
 			}
 		}
 
 		// No eough data to meet the minimum chunk size requirement, so just return as a chunk.
 		if maker.bufferSize < maker.minimumChunkSize {
-			fill(maker.bufferSize)
-			endOfChunk(chunk, true)
-			return
+			if reader == nil {
+				fill(maker.bufferSize)
+				if maker.chunk.GetLength() > 0 {
+					sendChunk(maker.chunk)
+				}
+				return 0, ""
+			} else if isEOF {
+				return fileSize, hex.EncodeToString(fileHasher.Sum(nil))
+			} else {
+				continue
+			}
 		}
 
 		// Minimum chunk size has been reached.  Calculate the buzhash for the minimum size chunk.
-		if !minimumReached {
+		if !maker.minimumReached {
 
 			bytes := maker.minimumChunkSize
 
 			if maker.bufferStart+bytes < maker.bufferCapacity {
-				hashSum = maker.buzhashSum(0, maker.buffer[maker.bufferStart:maker.bufferStart+bytes])
+				maker.hashSum = maker.buzhashSum(0, maker.buffer[maker.bufferStart:maker.bufferStart+bytes])
 			} else {
-				hashSum = maker.buzhashSum(0, maker.buffer[maker.bufferStart:])
-				hashSum = maker.buzhashSum(hashSum,
+				maker.hashSum = maker.buzhashSum(0, maker.buffer[maker.bufferStart:])
+				maker.hashSum = maker.buzhashSum(maker.hashSum,
 					maker.buffer[:bytes-(maker.bufferCapacity-maker.bufferStart)])
 			}
 
-			if (hashSum & maker.hashMask) == 0 {
+			if (maker.hashSum & maker.hashMask) == 0 {
 				// This is a minimum size chunk
 				fill(bytes)
-				endOfChunk(chunk, false)
-				startNewChunk()
+				sendChunk(maker.chunk)
+				maker.startNewChunk()
 				continue
 			}
 
-			minimumReached = true
+			maker.minimumReached = true
 		}
 
 		// Now check the buzhash of the data in the buffer, shifting one byte at a time.
 		bytes := maker.bufferSize - maker.minimumChunkSize
-		isEOC := false
-		maxSize := maker.maximumChunkSize - chunk.GetLength()
-		for i := 0; i < maker.bufferSize-maker.minimumChunkSize; i++ {
+		isEOC := false // chunk boundary found
+		maxSize := maker.maximumChunkSize - maker.chunk.GetLength()
+		for i := 0; i < bytes; i++ {
 			out := maker.bufferStart + i
 			if out >= maker.bufferCapacity {
 				out -= maker.bufferCapacity
@@ -267,8 +277,8 @@ func (maker *ChunkMaker) ForEachChunk(reader io.Reader, endOfChunk func(chunk *C
 				in -= maker.bufferCapacity
 			}
 
-			hashSum = maker.buzhashUpdate(hashSum, maker.buffer[out], maker.buffer[in], maker.minimumChunkSize)
-			if (hashSum&maker.hashMask) == 0 || i == maxSize-maker.minimumChunkSize-1 {
+			maker.hashSum = maker.buzhashUpdate(maker.hashSum, maker.buffer[out], maker.buffer[in], maker.minimumChunkSize)
+			if (maker.hashSum&maker.hashMask) == 0 || i == maxSize-maker.minimumChunkSize-1 {
 				// A chunk is completed.
 				bytes = i + 1 + maker.minimumChunkSize
 				isEOC = true
@@ -277,21 +287,20 @@ func (maker *ChunkMaker) ForEachChunk(reader io.Reader, endOfChunk func(chunk *C
 		}
 
 		fill(bytes)
-
 		if isEOC {
-			if isEOF && maker.bufferSize == 0 {
-				endOfChunk(chunk, true)
-				return
+			sendChunk(maker.chunk)
+			maker.startNewChunk()
+		} else {
+			if reader == nil {
+				fill(maker.minimumChunkSize)
+				sendChunk(maker.chunk)
+				maker.startNewChunk()
+				return 0, ""
 			}
-			endOfChunk(chunk, false)
-			startNewChunk()
-			continue
 		}
 
 		if isEOF {
-			fill(maker.bufferSize)
-			endOfChunk(chunk, true)
-			return
+			return fileSize, hex.EncodeToString(fileHasher.Sum(nil))
 		}
 	}
 }

src/duplicacy_chunkmaker_test.go

@@ -7,14 +7,12 @@ package duplicacy
 import (
 	"bytes"
 	crypto_rand "crypto/rand"
-	"io"
 	"math/rand"
 	"sort"
 	"testing"
 )
 
-func splitIntoChunks(content []byte, n, averageChunkSize, maxChunkSize, minChunkSize,
-	bufferCapacity int) ([]string, int) {
+func splitIntoChunks(content []byte, n, averageChunkSize, maxChunkSize, minChunkSize int) ([]string, int) {
 
 	config := CreateConfig()
 
@@ -27,14 +25,12 @@ func splitIntoChunks(content []byte, n, averageChunkSize, maxChunkSize, minChunk
 	config.HashKey = DEFAULT_KEY
 	config.IDKey = DEFAULT_KEY
 
-	maker := CreateChunkMaker(config, false)
+	maker := CreateFileChunkMaker(config, false)
 
 	var chunks []string
 	totalChunkSize := 0
 	totalFileSize := int64(0)
 
-	//LOG_INFO("CHUNK_SPLIT", "bufferCapacity: %d", bufferCapacity)
-
 	buffers := make([]*bytes.Buffer, n)
 	sizes := make([]int, n)
 	sizes[0] = 0
@@ -42,7 +38,7 @@ func splitIntoChunks(content []byte, n, averageChunkSize, maxChunkSize, minChunk
 		same := true
 		for same {
 			same = false
-			sizes[i] = rand.Int() % n
+			sizes[i] = rand.Int() % len(content)
 			for j := 0; j < i; j++ {
 				if sizes[i] == sizes[j] {
 					same = true
@@ -59,22 +55,17 @@ func splitIntoChunks(content []byte, n, averageChunkSize, maxChunkSize, minChunk
 	}
 	buffers[n-1] = bytes.NewBuffer(content[sizes[n-1]:])
 
-	i := 0
+	chunkFunc := func(chunk *Chunk) {
+		chunks = append(chunks, chunk.GetHash())
+		totalChunkSize += chunk.GetLength()
+		config.PutChunk(chunk)
+	}
 
-	maker.ForEachChunk(buffers[0],
-		func(chunk *Chunk, final bool) {
-			//LOG_INFO("CHUNK_SPLIT", "i: %d, chunk: %s, size: %d", i, chunk.GetHash(), size)
-			chunks = append(chunks, chunk.GetHash())
-			totalChunkSize += chunk.GetLength()
-		},
-		func(size int64, hash string) (io.Reader, bool) {
-			totalFileSize += size
-			i++
-			if i >= len(buffers) {
-				return nil, false
-			}
-			return buffers[i], true
-		})
+	for _, buffer := range buffers {
+		fileSize, _ := maker.AddData(buffer, chunkFunc)
+		totalFileSize += fileSize
+	}
+	maker.AddData(nil, chunkFunc)
 
 	if totalFileSize != int64(totalChunkSize) {
 		LOG_ERROR("CHUNK_SPLIT", "total chunk size: %d, total file size: %d", totalChunkSize, totalFileSize)
@@ -96,35 +87,28 @@ func TestChunkMaker(t *testing.T) {
 			continue
 		}
 
-		chunkArray1, totalSize1 := splitIntoChunks(content, 10, 32, 64, 16, 32)
+		chunkArray1, totalSize1 := splitIntoChunks(content, 10, 32, 64, 16)
 
-		capacities := [...]int{32, 33, 34, 61, 62, 63, 64, 65, 66, 126, 127, 128, 129, 130,
-			255, 256, 257, 511, 512, 513, 1023, 1024, 1025,
-			32, 48, 64, 128, 256, 512, 1024, 2048}
 
-		//capacities := [...]int { 32 }
+		for _, n := range [...]int{6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16} {
+			chunkArray2, totalSize2 := splitIntoChunks(content, n, 32, 64, 16)
 
-		for _, capacity := range capacities {
+			if totalSize1 != totalSize2 {
+				t.Errorf("[size %d] total size is %d instead of %d",
+					size, totalSize2, totalSize1)
+			}
 
-			for _, n := range [...]int{6, 7, 8, 9, 10} {
-				chunkArray2, totalSize2 := splitIntoChunks(content, n, 32, 64, 16, capacity)
-
-				if totalSize1 != totalSize2 {
-					t.Errorf("[size %d, capacity %d] total size is %d instead of %d",
-						size, capacity, totalSize2, totalSize1)
-				}
-
-				if len(chunkArray1) != len(chunkArray2) {
-					t.Errorf("[size %d, capacity %d] number of chunks is %d instead of %d",
-						size, capacity, len(chunkArray2), len(chunkArray1))
-				} else {
-					for i := 0; i < len(chunkArray1); i++ {
-						if chunkArray1[i] != chunkArray2[i] {
-							t.Errorf("[size %d, capacity %d, chunk %d] chunk is different", size, capacity, i)
-						}
+			if len(chunkArray1) != len(chunkArray2) {
+				t.Errorf("[size %d] number of chunks is %d instead of %d",
+					size, len(chunkArray2), len(chunkArray1))
+			} else {
+				for i := 0; i < len(chunkArray1); i++ {
+					if chunkArray1[i] != chunkArray2[i] {
+						t.Errorf("[size %d, chunk %d] chunk is different", size, i)
 					}
 				}
 			}
+
 		}
 	}
 

src/duplicacy_chunkoperator.go

@@ -0,0 +1,579 @@
+// Copyright (c) Acrosync LLC. All rights reserved.
+// Free for personal use and commercial trial
+// Commercial use requires per-user licenses available from https://duplicacy.com
+
+package duplicacy
+
+import (
+	"io"
+	"sync"
+	"sync/atomic"
+	"time"
+)
+
+// These are operations that ChunkOperator will perform.
+const (
+	ChunkOperationDownload  = 0
+	ChunkOperationUpload    = 1
+	ChunkOperationDelete    = 2
+	ChunkOperationFossilize = 3
+	ChunkOperationResurrect = 4
+	ChunkOperationFind = 5
+)
+
+// ChunkTask is used to pass parameters for different kinds of chunk operations.
+type ChunkTask struct {
+	operation  int    // The type of operation
+	chunkID    string // The chunk id
+	chunkHash  string // The chunk hash
+	chunkIndex int    // The chunk index
+	filePath   string // The path of the chunk file; it may be empty
+
+	isMetadata bool
+	chunk      *Chunk
+
+	completionFunc func(chunk *Chunk, chunkIndex int)
+}
+
+// ChunkOperator is capable of performing multi-threaded operations on chunks.
+type ChunkOperator struct {
+	config *Config               // Associated config
+	storage Storage              // This storage
+	snapshotCache  *FileStorage
+	showStatistics bool
+	threads  int                 // Number of threads
+	taskQueue chan ChunkTask     // Operating goroutines are waiting on this channel for input
+	stopChannel chan bool        // Used to stop all the goroutines
+
+	numberOfActiveTasks int64    // The number of chunks that are being operated on
+
+	fossils []string             // For fossilize operation, the paths of the fossils are stored in this slice
+	collectionLock *sync.Mutex   // The lock for accessing 'fossils'
+
+	startTime int64              // The time it starts downloading
+	totalChunkSize int64         // Total chunk size
+	downloadedChunkSize int64    // Downloaded chunk size
+
+	allowFailures bool           // Whether to fail on download error, or continue
+	NumberOfFailedChunks int64   // The number of chunks that can't be downloaded
+
+	rewriteChunks bool           // Whether to rewrite corrupted chunks when erasure coding is enabled
+
+	UploadCompletionFunc func(chunk *Chunk, chunkIndex int, inCache bool, chunkSize int, uploadSize int)
+}
+
+// CreateChunkOperator creates a new ChunkOperator.
+func CreateChunkOperator(config *Config, storage Storage, snapshotCache *FileStorage, showStatistics bool, rewriteChunks bool, threads int,
+						 allowFailures bool) *ChunkOperator {
+
+	operator := &ChunkOperator{
+		config: config,
+		storage: storage,
+		snapshotCache: snapshotCache,
+		showStatistics: showStatistics,
+		threads: threads,
+
+		taskQueue:   make(chan ChunkTask, threads),
+		stopChannel: make(chan bool),
+
+		collectionLock: &sync.Mutex{},
+		startTime: time.Now().Unix(),
+		allowFailures: allowFailures,
+		rewriteChunks: rewriteChunks,
+	}
+
+	// Start the operator goroutines
+	for i := 0; i < operator.threads; i++ {
+		go func(threadIndex int) {
+			defer CatchLogException()
+			for {
+				select {
+				case task := <-operator.taskQueue:
+					operator.Run(threadIndex, task)
+				case <-operator.stopChannel:
+					return
+				}
+			}
+		}(i)
+	}
+
+	return operator
+}
+
+func (operator *ChunkOperator) Stop() {
+	if atomic.LoadInt64(&operator.numberOfActiveTasks) < 0 {
+		return
+	}
+
+	for atomic.LoadInt64(&operator.numberOfActiveTasks) > 0 {
+		time.Sleep(100 * time.Millisecond)
+	}
+	for i := 0; i < operator.threads; i++ {
+		operator.stopChannel <- false
+	}
+
+	// Assign -1 to numberOfActiveTasks so Stop() can be called multiple times
+	atomic.AddInt64(&operator.numberOfActiveTasks, int64(-1))
+}
+
+func (operator *ChunkOperator) WaitForCompletion() {
+
+	for atomic.LoadInt64(&operator.numberOfActiveTasks) > 0 {
+		time.Sleep(100 * time.Millisecond)
+	}
+}
+
+func (operator *ChunkOperator) AddTask(operation int, chunkID string, chunkHash string, filePath string, chunkIndex int, chunk *Chunk, isMetadata bool, completionFunc func(*Chunk, int))  {
+
+	task := ChunkTask {
+		operation:  operation,
+		chunkID:    chunkID,
+		chunkHash:  chunkHash,
+		chunkIndex: chunkIndex,
+		filePath:   filePath,
+		chunk:      chunk,
+		isMetadata: isMetadata,
+		completionFunc: completionFunc,
+	}
+
+	operator.taskQueue <- task
+	atomic.AddInt64(&operator.numberOfActiveTasks, int64(1))
+
+	return
+}
+
+func (operator *ChunkOperator) Download(chunkHash string, chunkIndex int, isMetadata bool) *Chunk {
+	chunkID := operator.config.GetChunkIDFromHash(chunkHash)
+	completionChannel := make(chan *Chunk)
+	completionFunc := func(chunk *Chunk, chunkIndex int) {
+		completionChannel <- chunk
+	}
+	operator.AddTask(ChunkOperationDownload, chunkID, chunkHash, "", chunkIndex, nil, isMetadata, completionFunc)
+	return <- completionChannel
+}
+
+func (operator *ChunkOperator) DownloadAsync(chunkHash string, chunkIndex int, isMetadata bool, completionFunc func(*Chunk, int)) {
+	chunkID := operator.config.GetChunkIDFromHash(chunkHash)
+	operator.AddTask(ChunkOperationDownload, chunkID, chunkHash, "", chunkIndex, nil, isMetadata, completionFunc)
+}
+
+func (operator *ChunkOperator) Upload(chunk *Chunk, chunkIndex int, isMetadata bool) {
+	chunkHash := chunk.GetHash()
+	chunkID := operator.config.GetChunkIDFromHash(chunkHash)
+	operator.AddTask(ChunkOperationUpload, chunkID, chunkHash, "", chunkIndex, chunk, isMetadata, nil)
+}
+
+func (operator *ChunkOperator) Delete(chunkID string, filePath string) {
+	operator.AddTask(ChunkOperationDelete, chunkID, "", filePath, 0, nil, false, nil)
+}
+
+func (operator *ChunkOperator) Fossilize(chunkID string, filePath string) {
+	operator.AddTask(ChunkOperationFossilize, chunkID, "", filePath, 0, nil, false, nil)
+}
+
+func (operator *ChunkOperator) Resurrect(chunkID string, filePath string) {
+	operator.AddTask(ChunkOperationResurrect, chunkID, "", filePath, 0, nil, false, nil)
+}
+
+func (operator *ChunkOperator) Run(threadIndex int, task ChunkTask) {
+	defer func() {
+		atomic.AddInt64(&operator.numberOfActiveTasks, int64(-1))
+	}()
+
+	if task.operation == ChunkOperationDownload {
+		operator.DownloadChunk(threadIndex, task)
+		return
+	} else if task.operation == ChunkOperationUpload {
+		operator.UploadChunk(threadIndex, task)
+		return
+	}
+
+	// task.filePath may be empty.  If so, find the chunk first.
+	if task.operation == ChunkOperationDelete || task.operation == ChunkOperationFossilize {
+		if task.filePath == "" {
+			filePath, exist, _, err := operator.storage.FindChunk(threadIndex, task.chunkID, false)
+			if err != nil {
+				LOG_ERROR("CHUNK_FIND", "Failed to locate the path for the chunk %s: %v", task.chunkID, err)
+				return
+			} else if !exist {
+				if task.operation == ChunkOperationDelete {
+					LOG_WARN("CHUNK_FIND", "Chunk %s does not exist in the storage", task.chunkID)
+					return
+				}
+
+				fossilPath, exist, _, _ := operator.storage.FindChunk(threadIndex, task.chunkID, true)
+				if exist {
+					LOG_WARN("CHUNK_FOSSILIZE", "Chunk %s is already a fossil", task.chunkID)
+					operator.collectionLock.Lock()
+					operator.fossils = append(operator.fossils, fossilPath)
+					operator.collectionLock.Unlock()
+				} else {
+					LOG_ERROR("CHUNK_FIND", "Chunk %s does not exist in the storage", task.chunkID)
+				}
+				return
+			}
+			task.filePath = filePath
+		}
+	}
+
+	if task.operation == ChunkOperationFind {
+		_, exist, _, err := operator.storage.FindChunk(threadIndex, task.chunkID, false)
+		if err != nil {
+			LOG_ERROR("CHUNK_FIND", "Failed to locate the path for the chunk %s: %v", task.chunkID, err)
+		} else if !exist {
+			LOG_ERROR("CHUNK_FIND", "Chunk %s does not exist in the storage", task.chunkID)
+		} else {
+			LOG_DEBUG("CHUNK_FIND", "Chunk %s exists in the storage", task.chunkID)
+		}
+	} else if task.operation == ChunkOperationDelete {
+		err := operator.storage.DeleteFile(threadIndex, task.filePath)
+		if err != nil {
+			LOG_WARN("CHUNK_DELETE", "Failed to remove the file %s: %v", task.filePath, err)
+		} else {
+			if task.chunkID != "" {
+				LOG_INFO("CHUNK_DELETE", "The chunk %s has been permanently removed", task.chunkID)
+			} else {
+				LOG_INFO("CHUNK_DELETE", "Deleted file %s from the storage", task.filePath)
+			}
+		}
+	} else if task.operation == ChunkOperationFossilize {
+
+		fossilPath := task.filePath + ".fsl"
+
+		err := operator.storage.MoveFile(threadIndex, task.filePath, fossilPath)
+		if err != nil {
+			if _, exist, _, _ := operator.storage.FindChunk(threadIndex, task.chunkID, true); exist {
+				err := operator.storage.DeleteFile(threadIndex, task.filePath)
+				if err == nil {
+					LOG_TRACE("CHUNK_DELETE", "Deleted chunk file %s as the fossil already exists", task.chunkID)
+				}
+				operator.collectionLock.Lock()
+				operator.fossils = append(operator.fossils, fossilPath)
+				operator.collectionLock.Unlock()
+			} else {
+				LOG_ERROR("CHUNK_DELETE", "Failed to fossilize the chunk %s: %v", task.chunkID, err)
+			}
+		} else {
+			LOG_TRACE("CHUNK_FOSSILIZE", "The chunk %s has been marked as a fossil", task.chunkID)
+			operator.collectionLock.Lock()
+			operator.fossils = append(operator.fossils, fossilPath)
+			operator.collectionLock.Unlock()
+		}
+	} else if task.operation == ChunkOperationResurrect {
+		chunkPath, exist, _, err := operator.storage.FindChunk(threadIndex, task.chunkID, false)
+		if err != nil {
+			LOG_ERROR("CHUNK_FIND", "Failed to locate the path for the chunk %s: %v", task.chunkID, err)
+		}
+
+		if exist {
+			operator.storage.DeleteFile(threadIndex, task.filePath)
+			LOG_INFO("FOSSIL_RESURRECT", "The chunk %s already exists", task.chunkID)
+		} else {
+			err := operator.storage.MoveFile(threadIndex, task.filePath, chunkPath)
+			if err != nil {
+				LOG_ERROR("FOSSIL_RESURRECT", "Failed to resurrect the chunk %s from the fossil %s: %v",
+					task.chunkID, task.filePath, err)
+			} else {
+				LOG_INFO("FOSSIL_RESURRECT", "The chunk %s has been resurrected", task.filePath)
+			}
+		}
+	}
+}
+
+// Download downloads a chunk from the storage.
+func (operator *ChunkOperator) DownloadChunk(threadIndex int, task ChunkTask) {
+
+	cachedPath := ""
+	chunk := operator.config.GetChunk()
+	chunk.isMetadata = task.isMetadata
+	chunkID := task.chunkID
+
+	defer func() {
+		if chunk != nil {
+			operator.config.PutChunk(chunk)
+		}
+	} ()
+
+	if task.isMetadata && operator.snapshotCache != nil {
+
+		var exist bool
+		var err error
+
+		// Reset the chunk with a hasher -- we're reading from the cache where chunk are not encrypted or compressed
+		chunk.Reset(true)
+
+		cachedPath, exist, _, err = operator.snapshotCache.FindChunk(threadIndex, chunkID, false)
+		if err != nil {
+			LOG_WARN("DOWNLOAD_CACHE", "Failed to find the cache path for the chunk %s: %v", chunkID, err)
+		} else if exist {
+			err = operator.snapshotCache.DownloadFile(0, cachedPath, chunk)
+			if err != nil {
+				LOG_WARN("DOWNLOAD_CACHE", "Failed to load the chunk %s from the snapshot cache: %v", chunkID, err)
+			} else {
+				actualChunkID := chunk.GetID()
+				if actualChunkID != chunkID {
+					LOG_WARN("DOWNLOAD_CACHE_CORRUPTED",
+						"The chunk %s load from the snapshot cache has a hash id of %s", chunkID, actualChunkID)
+				} else {
+					LOG_DEBUG("CHUNK_CACHE", "Chunk %s has been loaded from the snapshot cache", chunkID)
+
+					task.completionFunc(chunk, task.chunkIndex)
+					chunk = nil
+					return
+				}
+			}
+		}
+	}
+
+	// Reset the chunk without a hasher -- the downloaded content will be encrypted and/or compressed and the hasher
+	// will be set up before the encryption
+	chunk.Reset(false)
+	chunk.isMetadata = task.isMetadata
+
+	// If failures are allowed, complete the task properly
+	completeFailedChunk := func() {
+
+		atomic.AddInt64(&operator.NumberOfFailedChunks, 1)
+		if operator.allowFailures {
+			chunk.isBroken = true
+			task.completionFunc(chunk, task.chunkIndex)
+		}
+	}
+
+	chunkPath := ""
+	fossilPath := ""
+	filePath := ""
+
+	const MaxDownloadAttempts = 3
+	for downloadAttempt := 0; ; downloadAttempt++ {
+
+		exist := false
+		var err error
+
+		// Find the chunk by ID first.
+		chunkPath, exist, _, err = operator.storage.FindChunk(threadIndex, chunkID, false)
+		if err != nil {
+			completeFailedChunk()
+			LOG_WERROR(operator.allowFailures, "DOWNLOAD_CHUNK", "Failed to find the chunk %s: %v", chunkID, err)
+			return
+		}
+
+		if exist {
+			filePath = chunkPath
+		} else {
+			// No chunk is found.  Have to find it in the fossil pool again.
+			fossilPath, exist, _, err = operator.storage.FindChunk(threadIndex, chunkID, true)
+			if err != nil {
+				completeFailedChunk()
+				LOG_WERROR(operator.allowFailures, "DOWNLOAD_CHUNK", "Failed to find the chunk %s: %v", chunkID, err)
+				return
+			}
+
+			if !exist {
+
+				retry := false
+
+				// Retry for Hubic or WebDAV as it may return 404 even when the chunk exists
+				if _, ok := operator.storage.(*HubicStorage); ok {
+					retry = true
+				}
+
+				if _, ok := operator.storage.(*WebDAVStorage); ok {
+					retry = true
+				}
+
+				if retry && downloadAttempt < MaxDownloadAttempts {
+					LOG_WARN("DOWNLOAD_RETRY", "Failed to find the chunk %s; retrying", chunkID)
+					continue
+				}
+
+				// A chunk is not found.  This is a serious error and hopefully it will never happen.
+				completeFailedChunk()
+				if err != nil {
+					LOG_WERROR(operator.allowFailures, "DOWNLOAD_CHUNK", "Chunk %s can't be found: %v", chunkID, err)
+				} else {
+					LOG_WERROR(operator.allowFailures, "DOWNLOAD_CHUNK", "Chunk %s can't be found", chunkID)
+				}
+				return
+			}
+
+			filePath = fossilPath
+			LOG_WARN("DOWNLOAD_FOSSIL", "Chunk %s is a fossil", chunkID)
+		}
+
+		err = operator.storage.DownloadFile(threadIndex, filePath, chunk)
+		if err != nil {
+			_, isHubic := operator.storage.(*HubicStorage)
+			// Retry on EOF or if it is a Hubic backend as it may return 404 even when the chunk exists
+			if (err == io.ErrUnexpectedEOF || isHubic) && downloadAttempt < MaxDownloadAttempts {
+				LOG_WARN("DOWNLOAD_RETRY", "Failed to download the chunk %s: %v; retrying", chunkID, err)
+				chunk.Reset(false)
+				chunk.isMetadata = task.isMetadata
+				continue
+			} else {
+				completeFailedChunk()
+				LOG_WERROR(operator.allowFailures, "DOWNLOAD_CHUNK", "Failed to download the chunk %s: %v", chunkID, err)
+				return
+			}
+		}
+
+		rewriteNeeded := false
+		err, rewriteNeeded = chunk.Decrypt(operator.config.ChunkKey, task.chunkHash)
+		if err != nil {
+			if downloadAttempt < MaxDownloadAttempts {
+				LOG_WARN("DOWNLOAD_RETRY", "Failed to decrypt the chunk %s: %v; retrying", chunkID, err)
+				chunk.Reset(false)
+				chunk.isMetadata = task.isMetadata
+				continue
+			} else {
+				completeFailedChunk()
+				LOG_WERROR(operator.allowFailures, "DOWNLOAD_DECRYPT", "Failed to decrypt the chunk %s: %v", chunkID, err)
+				return
+			}
+		}
+
+		actualChunkID := chunk.GetID()
+		if actualChunkID != chunkID {
+			if downloadAttempt < MaxDownloadAttempts {
+				LOG_WARN("DOWNLOAD_RETRY", "The chunk %s has a hash id of %s; retrying", chunkID, actualChunkID)
+				chunk.Reset(false)
+				chunk.isMetadata = task.isMetadata
+				continue
+			} else {
+				completeFailedChunk()
+				LOG_WERROR(operator.allowFailures, "DOWNLOAD_CORRUPTED", "The chunk %s has a hash id of %s", chunkID, actualChunkID)
+				return
+			}
+		}
+
+		if rewriteNeeded && operator.rewriteChunks {
+
+			if filePath != fossilPath {
+				fossilPath = filePath + ".fsl"
+				err := operator.storage.MoveFile(threadIndex, chunkPath, fossilPath)
+				if err != nil {
+					LOG_WARN("CHUNK_REWRITE", "Failed to fossilize the chunk %s: %v", task.chunkID, err)
+				} else {
+					LOG_TRACE("CHUNK_REWRITE", "The existing chunk %s has been marked as a fossil for rewrite", task.chunkID)
+					operator.collectionLock.Lock()
+					operator.fossils = append(operator.fossils, fossilPath)
+					operator.collectionLock.Unlock()
+				}
+			}
+
+			newChunk := operator.config.GetChunk()
+			newChunk.Reset(true)
+			newChunk.Write(chunk.GetBytes())
+			// Encrypt the chunk only after we know that it must be uploaded.
+			err = newChunk.Encrypt(operator.config.ChunkKey, chunk.GetHash(), task.isMetadata)
+			if err == nil {
+				// Re-upload the chunk
+				err = operator.storage.UploadFile(threadIndex, chunkPath, newChunk.GetBytes())
+				if err != nil {
+					LOG_WARN("CHUNK_REWRITE", "Failed to re-upload the chunk %s: %v", chunkID, err)
+				} else {
+					LOG_INFO("CHUNK_REWRITE", "The chunk %s has been re-uploaded", chunkID)
+				}
+			}
+			operator.config.PutChunk(newChunk)
+		}
+
+		break
+	}
+
+	if chunk.isMetadata && len(cachedPath) > 0 {
+		// Save a copy to the local snapshot cache
+		err := operator.snapshotCache.UploadFile(threadIndex, cachedPath, chunk.GetBytes())
+		if err != nil {
+			LOG_WARN("DOWNLOAD_CACHE", "Failed to add the chunk %s to the snapshot cache: %v", chunkID, err)
+		}
+	}
+
+	downloadedChunkSize := atomic.AddInt64(&operator.downloadedChunkSize, int64(chunk.GetLength()))
+
+	if (operator.showStatistics || IsTracing()) && operator.totalChunkSize > 0 {
+
+		now := time.Now().Unix()
+		if now <= operator.startTime {
+			now = operator.startTime + 1
+		}
+		speed := downloadedChunkSize / (now - operator.startTime)
+		remainingTime := int64(0)
+		if speed > 0 {
+			remainingTime = (operator.totalChunkSize-downloadedChunkSize)/speed + 1
+		}
+		percentage := float32(downloadedChunkSize * 1000 / operator.totalChunkSize)
+		LOG_INFO("DOWNLOAD_PROGRESS", "Downloaded chunk %d size %d, %sB/s %s %.1f%%",
+			task.chunkIndex+1, chunk.GetLength(),
+			PrettySize(speed), PrettyTime(remainingTime), percentage/10)
+	} else {
+		LOG_DEBUG("CHUNK_DOWNLOAD", "Chunk %s has been downloaded", chunkID)
+	}
+
+	task.completionFunc(chunk, task.chunkIndex)
+	chunk = nil
+	return
+}
+
+// UploadChunk is called by the task goroutines to perform the actual uploading
+func (operator *ChunkOperator) UploadChunk(threadIndex int, task ChunkTask) bool {
+
+	chunk := task.chunk
+	chunkID := task.chunkID
+	chunkSize := chunk.GetLength()
+
+	// For a snapshot chunk, verify that its chunk id is correct
+	if task.isMetadata {
+		chunk.VerifyID()
+	}
+
+	if task.isMetadata && operator.snapshotCache != nil && operator.storage.IsCacheNeeded() {
+		// Save a copy to the local snapshot.
+		chunkPath, exist, _, err := operator.snapshotCache.FindChunk(threadIndex, chunkID, false)
+		if err != nil {
+			LOG_WARN("UPLOAD_CACHE", "Failed to find the cache path for the chunk %s: %v", chunkID, err)
+		} else if exist {
+			LOG_DEBUG("CHUNK_CACHE", "Chunk %s already exists in the snapshot cache", chunkID)
+		} else if err = operator.snapshotCache.UploadFile(threadIndex, chunkPath, chunk.GetBytes()); err != nil {
+			LOG_WARN("UPLOAD_CACHE", "Failed to save the chunk %s to the snapshot cache: %v", chunkID, err)
+		} else {
+			LOG_DEBUG("CHUNK_CACHE", "Chunk %s has been saved to the snapshot cache", chunkID)
+		}
+	}
+
+	// This returns the path the chunk file should be at.
+	chunkPath, exist, _, err := operator.storage.FindChunk(threadIndex, chunkID, false)
+	if err != nil {
+		LOG_ERROR("UPLOAD_CHUNK", "Failed to find the path for the chunk %s: %v", chunkID, err)
+		return false
+	}
+
+	if exist {
+		// Chunk deduplication by name in effect here.
+		LOG_DEBUG("CHUNK_DUPLICATE", "Chunk %s already exists", chunkID)
+
+		operator.UploadCompletionFunc(chunk, task.chunkIndex, false, chunkSize, 0)
+		return false
+	}
+
+	// Encrypt the chunk only after we know that it must be uploaded.
+	err = chunk.Encrypt(operator.config.ChunkKey, chunk.GetHash(), task.isMetadata)
+	if err != nil {
+		LOG_ERROR("UPLOAD_CHUNK", "Failed to encrypt the chunk %s: %v", chunkID, err)
+		return false
+	}
+
+	if !operator.config.dryRun {
+		err = operator.storage.UploadFile(threadIndex, chunkPath, chunk.GetBytes())
+		if err != nil {
+			LOG_ERROR("UPLOAD_CHUNK", "Failed to upload the chunk %s: %v", chunkID, err)
+			return false
+		}
+		LOG_DEBUG("CHUNK_UPLOAD", "Chunk %s has been uploaded", chunkID)
+	} else {
+		LOG_DEBUG("CHUNK_UPLOAD", "Uploading was skipped for chunk %s", chunkID)
+	}
+
+	operator.UploadCompletionFunc(chunk, task.chunkIndex, false, chunkSize, chunk.GetLength())
+	return true
+}

src/duplicacy_chunkoperator_test.go

@@ -15,11 +15,11 @@ import (
 	"math/rand"
 )
 
-func TestUploaderAndDownloader(t *testing.T) {
+func TestChunkOperator(t *testing.T) {
 
 	rand.Seed(time.Now().UnixNano())
 	setTestingT(t)
-	SetLoggingLevel(INFO)
+	SetLoggingLevel(DEBUG)
 
 	defer func() {
 		if r := recover(); r != nil {
@@ -38,7 +38,7 @@ func TestUploaderAndDownloader(t *testing.T) {
 	os.RemoveAll(testDir)
 	os.MkdirAll(testDir, 0700)
 
-	t.Logf("storage: %s", testStorageName)
+	t.Logf("storage: %s", *testStorageName)
 
 	storage, err := loadStorage(testDir, 1)
 	if err != nil {
@@ -46,7 +46,7 @@ func TestUploaderAndDownloader(t *testing.T) {
 		return
 	}
 	storage.EnableTestMode()
-	storage.SetRateLimits(testRateLimit, testRateLimit)
+	storage.SetRateLimits(*testRateLimit, *testRateLimit)
 
 	for _, dir := range []string{"chunks", "snapshots"} {
 		err = storage.CreateDirectory(0, dir)
@@ -59,7 +59,7 @@ func TestUploaderAndDownloader(t *testing.T) {
 	numberOfChunks := 100
 	maxChunkSize := 64 * 1024
 
-	if testQuickMode {
+	if *testQuickMode {
 		numberOfChunks = 10
 	}
 
@@ -87,35 +87,25 @@ func TestUploaderAndDownloader(t *testing.T) {
 		totalFileSize += chunk.GetLength()
 	}
 
-	completionFunc := func(chunk *Chunk, chunkIndex int, skipped bool, chunkSize int, uploadSize int) {
+	chunkOperator := CreateChunkOperator(config, storage, nil, false, false, *testThreads, false)
+	chunkOperator.UploadCompletionFunc = func(chunk *Chunk, chunkIndex int, skipped bool, chunkSize int, uploadSize int) {
 		t.Logf("Chunk %s size %d (%d/%d) uploaded", chunk.GetID(), chunkSize, chunkIndex, len(chunks))
 	}
 
-	chunkUploader := CreateChunkUploader(config, storage, nil, testThreads, nil)
-	chunkUploader.completionFunc = completionFunc
-	chunkUploader.Start()
-
 	for i, chunk := range chunks {
-		chunkUploader.StartChunk(chunk, i)
+		chunkOperator.Upload(chunk, i, false)
 	}
 
-	chunkUploader.Stop()
-
-	chunkDownloader := CreateChunkDownloader(config, storage, nil, true, testThreads)
-	chunkDownloader.totalChunkSize = int64(totalFileSize)
-
-	for _, chunk := range chunks {
-		chunkDownloader.AddChunk(chunk.GetHash())
-	}
+	chunkOperator.WaitForCompletion()
 
 	for i, chunk := range chunks {
-		downloaded := chunkDownloader.WaitForChunk(i)
+		downloaded := chunkOperator.Download(chunk.GetHash(), i, false)
 		if downloaded.GetID() != chunk.GetID() {
-			t.Error("Uploaded: %s, downloaded: %s", chunk.GetID(), downloaded.GetID())
+			t.Errorf("Uploaded: %s, downloaded: %s", chunk.GetID(), downloaded.GetID())
 		}
 	}
 
-	chunkDownloader.Stop()
+	chunkOperator.Stop()
 
 	for _, file := range listChunks(storage) {
 		err = storage.DeleteFile(0, "chunks/"+file)

src/duplicacy_chunkuploader.go

@@ -1,151 +0,0 @@
-// Copyright (c) Acrosync LLC. All rights reserved.
-// Free for personal use and commercial trial
-// Commercial use requires per-user licenses available from https://duplicacy.com
-
-package duplicacy
-
-import (
-	"sync/atomic"
-	"time"
-)
-
-// ChunkUploadTask represents a chunk to be uploaded.
-type ChunkUploadTask struct {
-	chunk      *Chunk
-	chunkIndex int
-}
-
-// ChunkUploader uploads chunks to the storage using one or more uploading goroutines.  Chunks are added
-// by the call to StartChunk(), and then passed to the uploading goroutines.  The completion function is
-// called when the downloading is completed.  Note that ChunkUploader does not release chunks to the
-// chunk pool; instead
-type ChunkUploader struct {
-	config        *Config              // Associated config
-	storage       Storage              // Download from this storage
-	snapshotCache *FileStorage         // Used as cache if not nil; usually for uploading snapshot chunks
-	threads       int                  // Number of uploading goroutines
-	taskQueue     chan ChunkUploadTask // Uploading goroutines are listening on this channel for upload jobs
-	stopChannel   chan bool            // Used to terminate uploading goroutines
-
-	numberOfUploadingTasks int32 // The number of uploading tasks
-
-	// Uploading goroutines call this function after having downloaded chunks
-	completionFunc func(chunk *Chunk, chunkIndex int, skipped bool, chunkSize int, uploadSize int)
-}
-
-// CreateChunkUploader creates a chunk uploader.
-func CreateChunkUploader(config *Config, storage Storage, snapshotCache *FileStorage, threads int,
-	completionFunc func(chunk *Chunk, chunkIndex int, skipped bool, chunkSize int, uploadSize int)) *ChunkUploader {
-	uploader := &ChunkUploader{
-		config:         config,
-		storage:        storage,
-		snapshotCache:  snapshotCache,
-		threads:        threads,
-		taskQueue:      make(chan ChunkUploadTask, 1),
-		stopChannel:    make(chan bool),
-		completionFunc: completionFunc,
-	}
-
-	return uploader
-}
-
-// Starts starts uploading goroutines.
-func (uploader *ChunkUploader) Start() {
-	for i := 0; i < uploader.threads; i++ {
-		go func(threadIndex int) {
-			defer CatchLogException()
-			for {
-				select {
-				case task := <-uploader.taskQueue:
-					uploader.Upload(threadIndex, task)
-				case <-uploader.stopChannel:
-					return
-				}
-			}
-		}(i)
-	}
-}
-
-// StartChunk sends a chunk to be uploaded to  a waiting uploading goroutine.  It may block if all uploading goroutines are busy.
-func (uploader *ChunkUploader) StartChunk(chunk *Chunk, chunkIndex int) {
-	atomic.AddInt32(&uploader.numberOfUploadingTasks, 1)
-	uploader.taskQueue <- ChunkUploadTask{
-		chunk:      chunk,
-		chunkIndex: chunkIndex,
-	}
-}
-
-// Stop stops all uploading goroutines.
-func (uploader *ChunkUploader) Stop() {
-	for atomic.LoadInt32(&uploader.numberOfUploadingTasks) > 0 {
-		time.Sleep(100 * time.Millisecond)
-	}
-	for i := 0; i < uploader.threads; i++ {
-		uploader.stopChannel <- false
-	}
-}
-
-// Upload is called by the uploading goroutines to perform the actual uploading
-func (uploader *ChunkUploader) Upload(threadIndex int, task ChunkUploadTask) bool {
-
-	chunk := task.chunk
-	chunkSize := chunk.GetLength()
-	chunkID := chunk.GetID()
-
-	// For a snapshot chunk, verify that its chunk id is correct
-	if uploader.snapshotCache != nil {
-		chunk.VerifyID()
-	}
-
-	if uploader.snapshotCache != nil && uploader.storage.IsCacheNeeded() {
-		// Save a copy to the local snapshot.
-		chunkPath, exist, _, err := uploader.snapshotCache.FindChunk(threadIndex, chunkID, false)
-		if err != nil {
-			LOG_WARN("UPLOAD_CACHE", "Failed to find the cache path for the chunk %s: %v", chunkID, err)
-		} else if exist {
-			LOG_DEBUG("CHUNK_CACHE", "Chunk %s already exists in the snapshot cache", chunkID)
-		} else if err = uploader.snapshotCache.UploadFile(threadIndex, chunkPath, chunk.GetBytes()); err != nil {
-			LOG_WARN("UPLOAD_CACHE", "Failed to save the chunk %s to the snapshot cache: %v", chunkID, err)
-		} else {
-			LOG_DEBUG("CHUNK_CACHE", "Chunk %s has been saved to the snapshot cache", chunkID)
-		}
-	}
-
-	// This returns the path the chunk file should be at.
-	chunkPath, exist, _, err := uploader.storage.FindChunk(threadIndex, chunkID, false)
-	if err != nil {
-		LOG_ERROR("UPLOAD_CHUNK", "Failed to find the path for the chunk %s: %v", chunkID, err)
-		return false
-	}
-
-	if exist {
-		// Chunk deduplication by name in effect here.
-		LOG_DEBUG("CHUNK_DUPLICATE", "Chunk %s already exists", chunkID)
-
-		uploader.completionFunc(chunk, task.chunkIndex, true, chunkSize, 0)
-		atomic.AddInt32(&uploader.numberOfUploadingTasks, -1)
-		return false
-	}
-
-	// Encrypt the chunk only after we know that it must be uploaded.
-	err = chunk.Encrypt(uploader.config.ChunkKey, chunk.GetHash())
-	if err != nil {
-		LOG_ERROR("UPLOAD_CHUNK", "Failed to encrypt the chunk %s: %v", chunkID, err)
-		return false
-	}
-
-	if !uploader.config.dryRun {
-		err = uploader.storage.UploadFile(threadIndex, chunkPath, chunk.GetBytes())
-		if err != nil {
-			LOG_ERROR("UPLOAD_CHUNK", "Failed to upload the chunk %s: %v", chunkID, err)
-			return false
-		}
-		LOG_DEBUG("CHUNK_UPLOAD", "Chunk %s has been uploaded", chunkID)
-	} else {
-		LOG_DEBUG("CHUNK_UPLOAD", "Uploading was skipped for chunk %s", chunkID)
-	}
-
-	uploader.completionFunc(chunk, task.chunkIndex, false, chunkSize, chunk.GetLength())
-	atomic.AddInt32(&uploader.numberOfUploadingTasks, -1)
-	return true
-}

src/duplicacy_config.go

@@ -9,15 +9,21 @@ import (
 	"crypto/hmac"
 	"crypto/rand"
 	"crypto/sha256"
+	"crypto/rsa"
+	"crypto/x509"
 	"encoding/binary"
 	"encoding/hex"
 	"encoding/json"
+	"encoding/pem"
 	"fmt"
 	"hash"
 	"os"
+	"strings"
 	"runtime"
 	"runtime/debug"
 	"sync/atomic"
+	"io/ioutil"
+	"reflect"
 
 	blake2 "github.com/minio/blake2b-simd"
 )
@@ -29,8 +35,21 @@ var DEFAULT_KEY = []byte("duplicacy")
 // standard zlib levels of -1 to 9.
 var DEFAULT_COMPRESSION_LEVEL = 100
 
-// The new header of the config file (to differentiate from the old format where the salt and iterations are fixed)
-var CONFIG_HEADER = "duplicacy\001"
+// zstd compression levels starting from 200
+var ZSTD_COMPRESSION_LEVEL_FASTEST = 200
+var ZSTD_COMPRESSION_LEVEL_DEFAULT = 201
+var ZSTD_COMPRESSION_LEVEL_BETTER = 202
+var ZSTD_COMPRESSION_LEVEL_BEST = 203
+
+var ZSTD_COMPRESSION_LEVELS = map[string]int {
+    "fastest": ZSTD_COMPRESSION_LEVEL_FASTEST,
+	"default": ZSTD_COMPRESSION_LEVEL_DEFAULT,
+	"better":  ZSTD_COMPRESSION_LEVEL_BETTER,
+	"best":    ZSTD_COMPRESSION_LEVEL_BEST,
+}
+
+// The new banner of the config file (to differentiate from the old format where the salt and iterations are fixed)
+var CONFIG_BANNER = "duplicacy\001"
 
 // The length of the salt used in the new format
 var CONFIG_SALT_LENGTH = 32
@@ -65,6 +84,14 @@ type Config struct {
 	// for encrypting a non-chunk file
 	FileKey []byte `json:"-"`
 
+	// for erasure coding
+	DataShards int `json:'data-shards'`
+	ParityShards int `json:'parity-shards'`
+
+	// for RSA encryption
+	rsaPrivateKey *rsa.PrivateKey
+	rsaPublicKey *rsa.PublicKey
+
 	chunkPool      chan *Chunk
 	numberOfChunks int32
 	dryRun         bool
@@ -80,10 +107,15 @@ type jsonableConfig struct {
 	IDKey     string `json:"id-key"`
 	ChunkKey  string `json:"chunk-key"`
 	FileKey   string `json:"file-key"`
+	RSAPublicKey string `json:"rsa-public-key"`
 }
 
 func (config *Config) MarshalJSON() ([]byte, error) {
 
+	publicKey := []byte {}
+	if config.rsaPublicKey != nil {
+		publicKey, _ = x509.MarshalPKIXPublicKey(config.rsaPublicKey)
+	}
 	return json.Marshal(&jsonableConfig{
 		aliasedConfig: (*aliasedConfig)(config),
 		ChunkSeed:     hex.EncodeToString(config.ChunkSeed),
@@ -91,6 +123,7 @@ func (config *Config) MarshalJSON() ([]byte, error) {
 		IDKey:         hex.EncodeToString(config.IDKey),
 		ChunkKey:      hex.EncodeToString(config.ChunkKey),
 		FileKey:       hex.EncodeToString(config.FileKey),
+		RSAPublicKey:  hex.EncodeToString(publicKey),
 	})
 }
 
@@ -120,13 +153,25 @@ func (config *Config) UnmarshalJSON(description []byte) (err error) {
 		return fmt.Errorf("Invalid representation of the file key in the config")
 	}
 
+	if publicKey, err := hex.DecodeString(aliased.RSAPublicKey); err != nil {
+		return fmt.Errorf("Invalid hex encoding of the RSA public key in the config")
+	} else if len(publicKey) > 0 {
+		parsedKey, err := x509.ParsePKIXPublicKey(publicKey)
+		if err != nil {
+			return fmt.Errorf("Invalid RSA public key in the config: %v", err)
+		}
+		config.rsaPublicKey = parsedKey.(*rsa.PublicKey)
+		if config.rsaPublicKey == nil {
+			return fmt.Errorf("Unsupported public key type %s in the config", reflect.TypeOf(parsedKey))
+		}
+	}
+
 	return nil
 }
 
-func (config *Config) IsCompatiableWith(otherConfig *Config) bool {
+func (config *Config) IsCompatibleWith(otherConfig *Config) bool {
 
-	return config.CompressionLevel == otherConfig.CompressionLevel &&
-		config.AverageChunkSize == otherConfig.AverageChunkSize &&
+	return config.AverageChunkSize == otherConfig.AverageChunkSize &&
 		config.MaximumChunkSize == otherConfig.MaximumChunkSize &&
 		config.MinimumChunkSize == otherConfig.MinimumChunkSize &&
 		bytes.Equal(config.ChunkSeed, otherConfig.ChunkSeed) &&
@@ -140,6 +185,41 @@ func (config *Config) Print() {
 	LOG_INFO("CONFIG_INFO", "Maximum chunk size: %d", config.MaximumChunkSize)
 	LOG_INFO("CONFIG_INFO", "Minimum chunk size: %d", config.MinimumChunkSize)
 	LOG_INFO("CONFIG_INFO", "Chunk seed: %x", config.ChunkSeed)
+
+	LOG_TRACE("CONFIG_INFO", "Hash key: %x", config.HashKey)
+	LOG_TRACE("CONFIG_INFO", "ID key: %x", config.IDKey)
+
+	if len(config.ChunkKey) > 0 {
+		LOG_TRACE("CONFIG_INFO", "File chunks are encrypted")
+	}
+
+	if len(config.FileKey) > 0 {
+		LOG_TRACE("CONFIG_INFO", "Metadata chunks are encrypted")
+	}
+
+	if config.DataShards != 0 && config.ParityShards != 0 {
+		LOG_TRACE("CONFIG_INFO", "Data shards: %d, parity shards: %d", config.DataShards, config.ParityShards)
+	}
+
+	if config.rsaPublicKey != nil {
+		pkisPublicKey, _ := x509.MarshalPKIXPublicKey(config.rsaPublicKey)
+
+		publicKey := pem.EncodeToMemory(&pem.Block{
+			Type:  "PUBLIC KEY",
+			Bytes: pkisPublicKey,
+		})
+
+		LOG_TRACE("CONFIG_INFO", "RSA public key: %s", publicKey)
+	}
+
+}
+
+func (config *Config) PrintCompressionLevel() {
+	for name, level := range ZSTD_COMPRESSION_LEVELS {
+		if level == config.CompressionLevel {
+			LOG_INFO("COMPRESSION_LEVEL", "Zstd compression is enabled (level: %s)", name)
+		}
+	}
 }
 
 func CreateConfigFromParameters(compressionLevel int, averageChunkSize int, maximumChunkSize int, mininumChunkSize int,
@@ -174,7 +254,6 @@ func CreateConfigFromParameters(compressionLevel int, averageChunkSize int, maxi
 	}
 
 	if copyFrom != nil {
-		config.CompressionLevel = copyFrom.CompressionLevel
 
 		config.AverageChunkSize = copyFrom.AverageChunkSize
 		config.MaximumChunkSize = copyFrom.MaximumChunkSize
@@ -184,6 +263,8 @@ func CreateConfigFromParameters(compressionLevel int, averageChunkSize int, maxi
 		config.HashKey = copyFrom.HashKey
 
 		if bitCopy {
+			config.CompressionLevel = copyFrom.CompressionLevel
+
 			config.IDKey = copyFrom.IDKey
 			config.ChunkKey = copyFrom.ChunkKey
 			config.FileKey = copyFrom.FileKey
@@ -234,7 +315,10 @@ func (config *Config) PutChunk(chunk *Chunk) {
 }
 
 func (config *Config) NewKeyedHasher(key []byte) hash.Hash {
-	if config.CompressionLevel == DEFAULT_COMPRESSION_LEVEL {
+	// Early versions of Duplicacy used SHA256 as the hash function for chunk IDs at the time when
+	// only zlib compression was supported.  Later SHA256 was replaced by Blake2b and LZ4 was used
+	// for compression (with compression level set to 100).
+	if config.CompressionLevel >= DEFAULT_COMPRESSION_LEVEL {
 		hasher, err := blake2.New(&blake2.Config{Size: 32, Key: key})
 		if err != nil {
 			LOG_ERROR("HASH_KEY", "Invalid hash key: %x", key)
@@ -279,7 +363,7 @@ func (hasher *DummyHasher) BlockSize() int {
 func (config *Config) NewFileHasher() hash.Hash {
 	if SkipFileHash {
 		return &DummyHasher{}
-	} else if config.CompressionLevel == DEFAULT_COMPRESSION_LEVEL {
+	} else if config.CompressionLevel >= DEFAULT_COMPRESSION_LEVEL {
 		hasher, _ := blake2.New(&blake2.Config{Size: 32})
 		return hasher
 	} else {
@@ -335,11 +419,11 @@ func DownloadConfig(storage Storage, password string) (config *Config, isEncrypt
 		return nil, false, err
 	}
 
-	if len(configFile.GetBytes()) < len(ENCRYPTION_HEADER) {
+	if len(configFile.GetBytes()) < len(ENCRYPTION_BANNER) {
 		return nil, false, fmt.Errorf("The storage has an invalid config file")
 	}
 
-	if string(configFile.GetBytes()[:len(ENCRYPTION_HEADER)-1]) == ENCRYPTION_HEADER[:len(ENCRYPTION_HEADER)-1] && len(password) == 0 {
+	if string(configFile.GetBytes()[:len(ENCRYPTION_BANNER)-1]) == ENCRYPTION_BANNER[:len(ENCRYPTION_BANNER)-1] && len(password) == 0 {
 		return nil, true, fmt.Errorf("The storage is likely to have been initialized with a password before")
 	}
 
@@ -347,23 +431,23 @@ func DownloadConfig(storage Storage, password string) (config *Config, isEncrypt
 
 	if len(password) > 0 {
 
-		if string(configFile.GetBytes()[:len(ENCRYPTION_HEADER)]) == ENCRYPTION_HEADER {
+		if string(configFile.GetBytes()[:len(ENCRYPTION_BANNER)]) == ENCRYPTION_BANNER {
 			// This is the old config format with a static salt and a fixed number of iterations
 			masterKey = GenerateKeyFromPassword(password, DEFAULT_KEY, CONFIG_DEFAULT_ITERATIONS)
 			LOG_TRACE("CONFIG_FORMAT", "Using a static salt and %d iterations for key derivation", CONFIG_DEFAULT_ITERATIONS)
-		} else if string(configFile.GetBytes()[:len(CONFIG_HEADER)]) == CONFIG_HEADER {
+		} else if string(configFile.GetBytes()[:len(CONFIG_BANNER)]) == CONFIG_BANNER {
 			// This is the new config format with a random salt and a configurable number of iterations
 			encryptedLength := len(configFile.GetBytes()) - CONFIG_SALT_LENGTH - 4
 
 			// Extract the salt and the number of iterations
-			saltStart := configFile.GetBytes()[len(CONFIG_HEADER):]
+			saltStart := configFile.GetBytes()[len(CONFIG_BANNER):]
 			iterations := binary.LittleEndian.Uint32(saltStart[CONFIG_SALT_LENGTH : CONFIG_SALT_LENGTH+4])
 			LOG_TRACE("CONFIG_ITERATIONS", "Using %d iterations for key derivation", iterations)
 			masterKey = GenerateKeyFromPassword(password, saltStart[:CONFIG_SALT_LENGTH], int(iterations))
 
-			// Copy to a temporary buffer to replace the header and remove the salt and the number of riterations
+			// Copy to a temporary buffer to replace the banner and remove the salt and the number of riterations
 			var encrypted bytes.Buffer
-			encrypted.Write([]byte(ENCRYPTION_HEADER))
+			encrypted.Write([]byte(ENCRYPTION_BANNER))
 			encrypted.Write(saltStart[CONFIG_SALT_LENGTH+4:])
 
 			configFile.Reset(false)
@@ -372,11 +456,11 @@ func DownloadConfig(storage Storage, password string) (config *Config, isEncrypt
 				LOG_ERROR("CONFIG_DOWNLOAD", "Encrypted config has %d bytes instead of expected %d bytes", len(configFile.GetBytes()), encryptedLength)
 			}
 		} else {
-			return nil, true, fmt.Errorf("The config file has an invalid header")
+			return nil, true, fmt.Errorf("The config file has an invalid banner")
 		}
 
 		// Decrypt the config file.  masterKey == nil means no encryption.
-		err = configFile.Decrypt(masterKey, "")
+		err, _ = configFile.Decrypt(masterKey, "")
 		if err != nil {
 			return nil, false, fmt.Errorf("Failed to retrieve the config file: %v", err)
 		}
@@ -430,21 +514,21 @@ func UploadConfig(storage Storage, config *Config, password string, iterations i
 
 	if len(password) > 0 {
 		// Encrypt the config file with masterKey.  If masterKey is nil then no encryption is performed.
-		err = chunk.Encrypt(masterKey, "")
+		err = chunk.Encrypt(masterKey, "", true)
 		if err != nil {
 			LOG_ERROR("CONFIG_CREATE", "Failed to create the config file: %v", err)
 			return false
 		}
 
-		// The new encrypted format for config is CONFIG_HEADER + salt + #iterations + encrypted content
+		// The new encrypted format for config is CONFIG_BANNER + salt + #iterations + encrypted content
 		encryptedLength := len(chunk.GetBytes()) + CONFIG_SALT_LENGTH + 4
 
-		// Copy to a temporary buffer to replace the header and add the salt and the number of iterations
+		// Copy to a temporary buffer to replace the banner and add the salt and the number of iterations
 		var encrypted bytes.Buffer
-		encrypted.Write([]byte(CONFIG_HEADER))
+		encrypted.Write([]byte(CONFIG_BANNER))
 		encrypted.Write(salt)
 		binary.Write(&encrypted, binary.LittleEndian, uint32(iterations))
-		encrypted.Write(chunk.GetBytes()[len(ENCRYPTION_HEADER):])
+		encrypted.Write(chunk.GetBytes()[len(ENCRYPTION_BANNER):])
 
 		chunk.Reset(false)
 		chunk.Write(encrypted.Bytes())
@@ -477,7 +561,7 @@ func UploadConfig(storage Storage, config *Config, password string, iterations i
 // it simply creates a file named 'config' that stores various parameters as well as a set of keys if encryption
 // is enabled.
 func ConfigStorage(storage Storage, iterations int, compressionLevel int, averageChunkSize int, maximumChunkSize int,
-	minimumChunkSize int, password string, copyFrom *Config, bitCopy bool) bool {
+	minimumChunkSize int, password string, copyFrom *Config, bitCopy bool, keyFile string, dataShards int, parityShards int) bool {
 
 	exist, _, _, err := storage.GetFileInfo(0, "config")
 	if err != nil {
@@ -496,5 +580,129 @@ func ConfigStorage(storage Storage, iterations int, compressionLevel int, averag
 		return false
 	}
 
+	if keyFile != "" {
+		config.loadRSAPublicKey(keyFile)
+	}
+
+	config.DataShards = dataShards
+	config.ParityShards = parityShards
+
 	return UploadConfig(storage, config, password, iterations)
 }
+
+func (config *Config) loadRSAPublicKey(keyFile string) {
+	encodedKey := []byte(keyFile)
+	var err error
+
+	// keyFile may be the actually key, in which case we don't need to read from a file
+	if !strings.Contains(keyFile, "-----BEGIN") {
+		encodedKey, err = ioutil.ReadFile(keyFile)
+		if err != nil {
+			LOG_ERROR("BACKUP_KEY", "Failed to read the public key file: %v", err)
+			return
+		}
+	}
+
+	decodedKey, _ := pem.Decode(encodedKey)
+	if decodedKey == nil {
+		LOG_ERROR("RSA_PUBLIC", "unrecognized public key in %s", keyFile)
+		return
+	}
+	if decodedKey.Type != "PUBLIC KEY" {
+		LOG_ERROR("RSA_PUBLIC", "Unsupported public key type %s in %s", decodedKey.Type, keyFile)
+		return
+	}
+
+	parsedKey, err := x509.ParsePKIXPublicKey(decodedKey.Bytes)
+	if err != nil {
+		LOG_ERROR("RSA_PUBLIC", "Failed to parse the public key in %s: %v", keyFile, err)
+		return
+	}
+
+	key, ok := parsedKey.(*rsa.PublicKey)
+	if !ok {
+		LOG_ERROR("RSA_PUBLIC", "Unsupported public key type %s in %s", reflect.TypeOf(parsedKey), keyFile)
+		return
+	}
+
+	config.rsaPublicKey = key
+}
+
+// loadRSAPrivateKey loads the specifed private key file for decrypting file chunks
+func (config *Config) loadRSAPrivateKey(keyFile string, passphrase string) {
+
+	if config.rsaPublicKey == nil {
+		LOG_ERROR("RSA_PUBLIC", "The storage was not encrypted by an RSA key")
+		return
+	}
+
+	encodedKey := []byte(keyFile)
+	var err error
+
+	// keyFile may be the actually key, in which case we don't need to read from a file
+	if !strings.Contains(keyFile, "-----BEGIN") {
+		encodedKey, err = ioutil.ReadFile(keyFile)
+		if err != nil {
+			LOG_ERROR("RSA_PRIVATE", "Failed to read the private key file: %v", err)
+			return
+		}
+	}
+
+	decodedKey, _ := pem.Decode(encodedKey)
+	if decodedKey == nil {
+		LOG_ERROR("RSA_PRIVATE", "unrecognized private key in %s", keyFile)
+		return
+	}
+	if decodedKey.Type != "RSA PRIVATE KEY" {
+		LOG_ERROR("RSA_PRIVATE", "Unsupported private key type %s in %s", decodedKey.Type, keyFile)
+		return
+	}
+
+	var decodedKeyBytes []byte
+	if passphrase != "" {
+		decodedKeyBytes, err = x509.DecryptPEMBlock(decodedKey, []byte(passphrase))
+	} else {
+		decodedKeyBytes = decodedKey.Bytes
+	}
+
+	var parsedKey interface{}
+	if parsedKey, err = x509.ParsePKCS1PrivateKey(decodedKeyBytes); err != nil {
+		if parsedKey, err = x509.ParsePKCS8PrivateKey(decodedKeyBytes); err != nil {
+			LOG_ERROR("RSA_PRIVATE", "Failed to parse the private key in %s: %v", keyFile, err)
+			return
+		}
+	}
+
+	key, ok := parsedKey.(*rsa.PrivateKey)
+	if !ok {
+		LOG_ERROR("RSA_PRIVATE", "Unsupported private key type %s in %s", reflect.TypeOf(parsedKey), keyFile)
+		return
+	}
+
+	data := make([]byte, 32)
+	_, err = rand.Read(data)
+	if err != nil {
+		LOG_ERROR("RSA_PRIVATE", "Failed to generate random data for testing the private key: %v", err)
+		return
+	}
+
+	// Now test if the private key matches the public key
+	encryptedData, err := rsa.EncryptOAEP(sha256.New(), rand.Reader, config.rsaPublicKey, data, nil)
+	if err != nil {
+		LOG_ERROR("RSA_PRIVATE", "Failed to encrypt random data with the public key: %v", err)
+		return
+	}
+
+	decryptedData, err := rsa.DecryptOAEP(sha256.New(), rand.Reader, key, encryptedData, nil)
+	if err != nil {
+		LOG_ERROR("RSA_PRIVATE", "Incorrect private key: %v", err)
+		return
+	}
+
+	if !bytes.Equal(data, decryptedData) {
+		LOG_ERROR("RSA_PRIVATE", "Decrypted data do not match the original data")
+		return
+	}
+
+	config.rsaPrivateKey = key
+}

src/duplicacy_dropboxstorage.go

@@ -6,6 +6,7 @@ package duplicacy
 
 import (
 	"fmt"
+	"io/ioutil"
 	"strings"
 
 	"github.com/gilbertchen/go-dropbox"
@@ -20,11 +21,11 @@ type DropboxStorage struct {
 }
 
 // CreateDropboxStorage creates a dropbox storage object.
-func CreateDropboxStorage(accessToken string, storageDir string, minimumNesting int, threads int) (storage *DropboxStorage, err error) {
+func CreateDropboxStorage(refreshToken string, storageDir string, minimumNesting int, threads int) (storage *DropboxStorage, err error) {
 
 	var clients []*dropbox.Files
 	for i := 0; i < threads; i++ {
-		client := dropbox.NewFiles(dropbox.NewConfig(accessToken))
+		client := dropbox.NewFiles(dropbox.NewConfig("", refreshToken, "https://duplicacy.com/dropbox_refresh"))
 		clients = append(clients, client)
 	}
 
@@ -199,6 +200,7 @@ func (storage *DropboxStorage) DownloadFile(threadIndex int, filePath string, ch
 	}
 
 	defer output.Body.Close()
+	defer ioutil.ReadAll(output.Body)
 
 	_, err = RateLimitedCopy(chunk, output.Body, storage.DownloadRateLimit/len(storage.clients))
 	return err

src/duplicacy_entry.go

@@ -16,12 +16,20 @@ import (
 	"strconv"
 	"strings"
 	"time"
+	"bytes"
+	"crypto/sha256"
+
+    "github.com/vmihailenco/msgpack"
+
 )
 
 // This is the hidden directory in the repository for storing various files.
 var DUPLICACY_DIRECTORY = ".duplicacy"
 var DUPLICACY_FILE = ".duplicacy"
 
+// Mask for file permission bits
+var fileModeMask = os.ModePerm | os.ModeSetuid | os.ModeSetgid | os.ModeSticky
+
 // Regex for matching 'StartChunk:StartOffset:EndChunk:EndOffset'
 var contentRegex = regexp.MustCompile(`^([0-9]+):([0-9]+):([0-9]+):([0-9]+)`)
 
@@ -42,7 +50,7 @@ type Entry struct {
 	EndChunk    int
 	EndOffset   int
 
-	Attributes map[string][]byte
+	Attributes *map[string][]byte
 }
 
 // CreateEntry creates an entry from file properties.
@@ -90,6 +98,27 @@ func CreateEntryFromFileInfo(fileInfo os.FileInfo, directory string) *Entry {
 	return entry
 }
 
+func (entry *Entry) Copy() *Entry {
+	return &Entry{
+		Path: entry.Path,
+		Size: entry.Size,
+		Time: entry.Time,
+		Mode: entry.Mode,
+		Link: entry.Link,
+		Hash: entry.Hash,
+
+		UID: entry.UID,
+		GID: entry.GID,
+
+		StartChunk: entry.StartChunk,
+		StartOffset: entry.StartOffset,
+		EndChunk: entry.EndChunk,
+		EndOffset: entry.EndOffset,
+
+		Attributes: entry.Attributes,
+	}
+}
+
 // CreateEntryFromJSON creates an entry from a json description.
 func (entry *Entry) UnmarshalJSON(description []byte) (err error) {
 
@@ -172,17 +201,17 @@ func (entry *Entry) UnmarshalJSON(description []byte) (err error) {
 		if attributes, ok := value.(map[string]interface{}); !ok {
 			return fmt.Errorf("Attributes are invalid for file '%s' in the snapshot", entry.Path)
 		} else {
-			entry.Attributes = make(map[string][]byte)
+			entry.Attributes = &map[string][]byte{}
 			for name, object := range attributes {
 				if object == nil {
-					entry.Attributes[name] = []byte("")
+					(*entry.Attributes)[name] = []byte("")
 				} else if attributeInBase64, ok := object.(string); !ok {
 					return fmt.Errorf("Attribute '%s' is invalid for file '%s' in the snapshot", name, entry.Path)
 				} else if attribute, err := base64.StdEncoding.DecodeString(attributeInBase64); err != nil {
 					return fmt.Errorf("Failed to decode attribute '%s' for file '%s' in the snapshot: %v",
 						name, entry.Path, err)
 				} else {
-					entry.Attributes[name] = attribute
+					(*entry.Attributes)[name] = attribute
 				}
 			}
 		}
@@ -241,7 +270,7 @@ func (entry *Entry) convertToObject(encodeName bool) map[string]interface{} {
 		object["gid"] = entry.GID
 	}
 
-	if len(entry.Attributes) > 0 {
+	if entry.Attributes != nil && len(*entry.Attributes) > 0 {
 		object["attributes"] = entry.Attributes
 	}
 
@@ -256,6 +285,197 @@ func (entry *Entry) MarshalJSON() ([]byte, error) {
 	return description, err
 }
 
+var _ msgpack.CustomEncoder = (*Entry)(nil)
+var _ msgpack.CustomDecoder = (*Entry)(nil)
+
+func (entry *Entry) EncodeMsgpack(encoder *msgpack.Encoder) error {
+
+	err := encoder.EncodeString(entry.Path)
+	if err != nil {
+		return err
+	}
+
+	err = encoder.EncodeInt(entry.Size)
+	if err != nil {
+		return err
+	}
+
+	err = encoder.EncodeInt(entry.Time)
+	if err != nil {
+		return err
+	}
+
+	err = encoder.EncodeInt(int64(entry.Mode))
+	if err != nil {
+		return err
+	}
+
+	err = encoder.EncodeString(entry.Link)
+	if err != nil {
+		return err
+	}
+
+	err = encoder.EncodeString(entry.Hash)
+	if err != nil {
+		return err
+	}
+
+	err = encoder.EncodeInt(int64(entry.StartChunk))
+	if err != nil {
+		return err
+	}
+
+	err = encoder.EncodeInt(int64(entry.StartOffset))
+	if err != nil {
+		return err
+	}
+
+	err = encoder.EncodeInt(int64(entry.EndChunk))
+	if err != nil {
+		return err
+	}
+
+	err = encoder.EncodeInt(int64(entry.EndOffset))
+	if err != nil {
+		return err
+	}
+
+	err = encoder.EncodeInt(int64(entry.UID))
+	if err != nil {
+		return err
+	}
+
+	err = encoder.EncodeInt(int64(entry.GID))
+	if err != nil {
+		return err
+	}
+
+	var numberOfAttributes int64
+	if entry.Attributes != nil {
+		numberOfAttributes = int64(len(*entry.Attributes))
+	}
+
+	err = encoder.EncodeInt(numberOfAttributes)
+	if err != nil {
+		return err
+	}
+
+	if entry.Attributes != nil {
+		attributes := make([]string, numberOfAttributes)
+        i := 0
+        for attribute := range *entry.Attributes {
+            attributes[i] = attribute
+            i++
+        }
+        sort.Strings(attributes)
+		for _, attribute := range attributes {
+			err = encoder.EncodeString(attribute)
+			if err != nil {
+				return err
+			}
+			err = encoder.EncodeString(string((*entry.Attributes)[attribute]))
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+    return nil
+}
+
+func (entry *Entry) DecodeMsgpack(decoder *msgpack.Decoder) error {
+
+	var err error
+
+	entry.Path, err = decoder.DecodeString()
+	if err != nil {
+		return err
+	}
+
+	entry.Size, err = decoder.DecodeInt64()
+	if err != nil {
+		return err
+	}
+
+	entry.Time, err = decoder.DecodeInt64()
+	if err != nil {
+		return err
+	}
+
+	mode, err := decoder.DecodeInt64()
+	if err != nil {
+		return err
+	}
+	entry.Mode = uint32(mode)
+
+	entry.Link, err = decoder.DecodeString()
+	if err != nil {
+		return err
+	}
+
+	entry.Hash, err = decoder.DecodeString()
+	if err != nil {
+		return err
+	}
+
+	startChunk, err := decoder.DecodeInt()
+	if err != nil {
+		return err
+	}
+	entry.StartChunk = int(startChunk)
+
+	startOffset, err := decoder.DecodeInt()
+	if err != nil {
+		return err
+	}
+	entry.StartOffset = int(startOffset)
+
+	endChunk, err := decoder.DecodeInt()
+	if err != nil {
+		return err
+	}
+	entry.EndChunk = int(endChunk)
+
+	endOffset, err := decoder.DecodeInt()
+	if err != nil {
+		return err
+	}
+	entry.EndOffset = int(endOffset)
+
+	uid, err := decoder.DecodeInt()
+	if err != nil {
+		return err
+	}
+	entry.UID = int(uid)
+
+	gid, err := decoder.DecodeInt()
+	if err != nil {
+		return err
+	}
+	entry.GID = int(gid)
+
+	numberOfAttributes, err := decoder.DecodeInt()
+	if err != nil {
+		return err
+	}
+
+	if numberOfAttributes > 0 {
+		entry.Attributes = &map[string][]byte{}
+		for i := 0; i < numberOfAttributes; i++ {
+			attribute, err := decoder.DecodeString()
+			if err != nil {
+				return err
+			}
+			value, err := decoder.DecodeString()
+			if err != nil {
+				return err
+			}
+			(*entry.Attributes)[attribute] = []byte(value)
+		}
+	}
+	return nil
+}
+
 func (entry *Entry) IsFile() bool {
 	return entry.Mode&uint32(os.ModeType) == 0
 }
@@ -268,8 +488,25 @@ func (entry *Entry) IsLink() bool {
 	return entry.Mode&uint32(os.ModeSymlink) != 0
 }
 
+func (entry *Entry) IsComplete() bool {
+	return entry.Size >= 0
+}
+
 func (entry *Entry) GetPermissions() os.FileMode {
-	return os.FileMode(entry.Mode) & os.ModePerm
+	return os.FileMode(entry.Mode) & fileModeMask
+}
+
+func (entry *Entry) GetParent() string {
+	path := entry.Path
+	if path != "" && path[len(path) - 1] == '/' {
+		path = path[:len(path) - 1]
+	}
+	i := strings.LastIndex(path, "/")
+	if i == -1 {
+		return ""
+	} else {
+		return path[:i]
+	}
 }
 
 func (entry *Entry) IsSameAs(other *Entry) bool {
@@ -289,7 +526,7 @@ func (entry *Entry) String(maxSizeDigits int) string {
 func (entry *Entry) RestoreMetadata(fullPath string, fileInfo *os.FileInfo, setOwner bool) bool {
 
 	if fileInfo == nil {
-		stat, err := os.Stat(fullPath)
+		stat, err := os.Lstat(fullPath)
 		fileInfo = &stat
 		if err != nil {
 			LOG_ERROR("RESTORE_STAT", "Failed to retrieve the file info: %v", err)
@@ -297,7 +534,15 @@ func (entry *Entry) RestoreMetadata(fullPath string, fileInfo *os.FileInfo, setO
 		}
 	}
 
-	if (*fileInfo).Mode()&os.ModePerm != entry.GetPermissions() {
+	// Note that chown can remove setuid/setgid bits so should be called before chmod
+	if setOwner {
+		if !SetOwner(fullPath, entry, fileInfo) {
+			return false
+		}
+	}
+
+	// Only set the permission if the file is not a symlink
+	if !entry.IsLink() && (*fileInfo).Mode()&fileModeMask != entry.GetPermissions() {
 		err := os.Chmod(fullPath, entry.GetPermissions())
 		if err != nil {
 			LOG_ERROR("RESTORE_CHMOD", "Failed to set the file permissions: %v", err)
@@ -305,7 +550,8 @@ func (entry *Entry) RestoreMetadata(fullPath string, fileInfo *os.FileInfo, setO
 		}
 	}
 
-	if (*fileInfo).ModTime().Unix() != entry.Time {
+	// Only set the time if the file is not a symlink
+	if !entry.IsLink() && (*fileInfo).ModTime().Unix() != entry.Time {
 		modifiedTime := time.Unix(entry.Time, 0)
 		err := os.Chtimes(fullPath, modifiedTime, modifiedTime)
 		if err != nil {
@@ -314,60 +560,71 @@ func (entry *Entry) RestoreMetadata(fullPath string, fileInfo *os.FileInfo, setO
 		}
 	}
 
-	if len(entry.Attributes) > 0 {
+	if entry.Attributes != nil && len(*entry.Attributes) > 0 {
 		entry.SetAttributesToFile(fullPath)
 	}
 
-	if setOwner {
-		return SetOwner(fullPath, entry, fileInfo)
-	} else {
-		return true
-	}
+	return true
 }
 
 // Return -1 if 'left' should appear before 'right', 1 if opposite, and 0 if they are the same.
 // Files are always arranged before subdirectories under the same parent directory.
-func (left *Entry) Compare(right *Entry) int {
-
-	path1 := left.Path
-	path2 := right.Path
 
+func ComparePaths(left string, right string) int {
 	p := 0
-	for ; p < len(path1) && p < len(path2); p++ {
-		if path1[p] != path2[p] {
+	for ; p < len(left) && p < len(right); p++ {
+		if left[p] != right[p] {
 			break
 		}
 	}
 
-	// c1, c2 is the first byte that differs
+	// c1, c2 are the first bytes that differ
 	var c1, c2 byte
-	if p < len(path1) {
-		c1 = path1[p]
+	if p < len(left) {
+		c1 = left[p]
 	}
-	if p < len(path2) {
-		c2 = path2[p]
+	if p < len(right) {
+		c2 = right[p]
 	}
 
-	// c3, c4 indicates how the current component ends
-	// c3 == '/':  the current component is a directory
-	// c3 != '/':  the current component is the last one
+	// c3, c4 indicate how the current component ends
+	// c3 == '/':  the current component is a directory; c3 != '/':  the current component is the last one
 	c3 := c1
-	for i := p; c3 != '/' && i < len(path1); i++ {
-		c3 = path1[i]
+
+	// last1, last2 means if the current compoent is the last component
+	last1 := true
+	for i := p; i < len(left); i++ {
+		c3 = left[i]
+		if c3 == '/' {
+			last1 = i == len(left) - 1
+			break
+		}
 	}
 
 	c4 := c2
-	for i := p; c4 != '/' && i < len(path2); i++ {
-		c4 = path2[i]
+	last2 := true
+	for i := p; i < len(right); i++ {
+		c4 = right[i]
+		if c4 == '/' {
+			last2 = i == len(right) - 1
+			break
+		}
+	}
+
+	if last1 != last2 {
+		if last1 {
+			return -1
+		} else {
+			return 1
+		}
 	}
 
 	if c3 == '/' {
 		if c4 == '/' {
 			// We are comparing two directory components
 			if c1 == '/' {
-				// left is shorter
-				// Note that c2 maybe smaller than c1 but c1 is '/' which is counted
-				// as 0
+				// left is shorter; note that c2 maybe smaller than c1 but c1 should be treated as 0 therefore
+				// this is a special case that must be handled separately
 				return -1
 			} else if c2 == '/' {
 				// right is shorter
@@ -389,6 +646,10 @@ func (left *Entry) Compare(right *Entry) int {
 	}
 }
 
+func (left *Entry) Compare(right *Entry) int {
+	return ComparePaths(left.Path, right.Path)
+}
+
 // This is used to sort entries by their names.
 type ByName []*Entry
 
@@ -435,7 +696,7 @@ func (files FileInfoCompare) Less(i, j int) bool {
 
 // ListEntries returns a list of entries representing file and subdirectories under the directory 'path'.  Entry paths
 // are normalized as relative to 'top'.  'patterns' are used to exclude or include certain files.
-func ListEntries(top string, path string, fileList *[]*Entry, patterns []string, discardAttributes bool) (directoryList []*Entry,
+func ListEntries(top string, path string, patterns []string, nobackupFile string, excludeByAttribute bool, listingChannel chan *Entry) (directoryList []*Entry,
 	skippedFiles []string, err error) {
 
 	LOG_DEBUG("LIST_ENTRIES", "Listing %s", path)
@@ -449,6 +710,15 @@ func ListEntries(top string, path string, fileList *[]*Entry, patterns []string,
 		return directoryList, nil, err
 	}
 
+	// This binary search works because ioutil.ReadDir returns files sorted by Name() by default
+	if nobackupFile != "" {
+		ii := sort.Search(len(files), func(ii int) bool { return strings.Compare(files[ii].Name(), nobackupFile) >= 0 })
+		if ii < len(files) && files[ii].Name() == nobackupFile {
+			LOG_DEBUG("LIST_NOBACKUP", "%s is excluded due to nobackup file", path)
+			return directoryList, skippedFiles, nil
+		}
+	}
+
 	normalizedPath := path
 	if len(normalizedPath) > 0 && normalizedPath[len(normalizedPath)-1] != '/' {
 		normalizedPath += "/"
@@ -461,20 +731,17 @@ func ListEntries(top string, path string, fileList *[]*Entry, patterns []string,
 
 	sort.Sort(FileInfoCompare(files))
 
-	entries := make([]*Entry, 0, 4)
-
 	for _, f := range files {
 		if f.Name() == DUPLICACY_DIRECTORY {
 			continue
 		}
 		entry := CreateEntryFromFileInfo(f, normalizedPath)
 		if len(patterns) > 0 && !MatchPath(entry.Path, patterns) {
-			LOG_DEBUG("LIST_EXCLUDE", "%s is excluded", entry.Path)
 			continue
 		}
 		if entry.IsLink() {
 			isRegular := false
-			isRegular, entry.Link, err = Readlink(filepath.Join(top, entry.Path))
+			isRegular, entry.Link, err = Readlink(joinPath(top, entry.Path))
 			if err != nil {
 				LOG_WARN("LIST_LINK", "Failed to read the symlink %s: %v", entry.Path, err)
 				skippedFiles = append(skippedFiles, entry.Path)
@@ -483,8 +750,8 @@ func ListEntries(top string, path string, fileList *[]*Entry, patterns []string,
 
 			if isRegular {
 				entry.Mode ^= uint32(os.ModeSymlink)
-			} else if path == "" && filepath.IsAbs(entry.Link) && !strings.HasPrefix(entry.Link, normalizedTop) {
-				stat, err := os.Stat(filepath.Join(top, entry.Path))
+			} else if path == "" && (filepath.IsAbs(entry.Link) || filepath.HasPrefix(entry.Link, `\\`)) && !strings.HasPrefix(entry.Link, normalizedTop) {
+				stat, err := os.Stat(joinPath(top, entry.Path))
 				if err != nil {
 					LOG_WARN("LIST_LINK", "Failed to read the symlink: %v", err)
 					skippedFiles = append(skippedFiles, entry.Path)
@@ -497,12 +764,18 @@ func ListEntries(top string, path string, fileList *[]*Entry, patterns []string,
 					// path from f.Name(); note that a "/" is append assuming a symbolic link is always a directory
 					newEntry.Path = filepath.Join(normalizedPath, f.Name()) + "/"
 				}
+				if len(patterns) > 0 && !MatchPath(newEntry.Path, patterns) {
+					continue
+				}
 				entry = newEntry
 			}
 		}
 
-		if !discardAttributes {
-			entry.ReadAttributes(top)
+		entry.ReadAttributes(top)
+
+		if excludeByAttribute && entry.Attributes != nil && excludedByAttribute(*entry.Attributes) {
+			LOG_DEBUG("LIST_EXCLUDE", "%s is excluded by attribute", entry.Path)
+			continue
 		}
 
 		if f.Mode()&(os.ModeNamedPipe|os.ModeSocket|os.ModeDevice) != 0 {
@@ -511,20 +784,20 @@ func ListEntries(top string, path string, fileList *[]*Entry, patterns []string,
 			continue
 		}
 
-		entries = append(entries, entry)
+		if entry.IsDir() {
+			directoryList = append(directoryList, entry)
+		} else {
+			listingChannel <- entry
+		}
 	}
 
 	// For top level directory we need to sort again because symlinks may have been changed
 	if path == "" {
-		sort.Sort(ByName(entries))
+		sort.Sort(ByName(directoryList))
 	}
 
-	for _, entry := range entries {
-		if entry.IsDir() {
-			directoryList = append(directoryList, entry)
-		} else {
-			*fileList = append(*fileList, entry)
-		}
+	for _, entry := range directoryList {
+		listingChannel <- entry
 	}
 
 	for i, j := 0, len(directoryList)-1; i < j; i, j = i+1, j-1 {
@@ -573,3 +846,100 @@ func (entry *Entry) Diff(chunkHashes []string, chunkLengths []int,
 
 	return modifiedLength
 }
+
+func (entry *Entry) EncodeWithHash(encoder *msgpack.Encoder) error {
+	entryBytes, err := msgpack.Marshal(entry)
+	if err != nil {
+		return err
+	}
+	hash := sha256.Sum256(entryBytes)
+	err = encoder.EncodeBytes(entryBytes)
+	if err != nil {
+		return err
+	}
+	err = encoder.EncodeBytes(hash[:])
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func DecodeEntryWithHash(decoder *msgpack.Decoder) (*Entry, error) {
+	entryBytes, err := decoder.DecodeBytes()
+	if err != nil {
+		return nil, err
+	}
+	hashBytes, err := decoder.DecodeBytes()
+	if err != nil {
+		return nil, err
+	}
+	expectedHash := sha256.Sum256(entryBytes)
+	if bytes.Compare(expectedHash[:], hashBytes) != 0 {
+		return nil, fmt.Errorf("corrupted file metadata")
+	}
+
+	var entry Entry
+	err = msgpack.Unmarshal(entryBytes, &entry)
+	if err != nil {
+		return nil, err
+	}
+	return &entry, nil
+}
+
+func (entry *Entry) check(chunkLengths []int) error {
+
+	if entry.Size < 0 {
+		return fmt.Errorf("The file %s hash an invalid size (%d)", entry.Path, entry.Size)
+	}
+
+	if !entry.IsFile() || entry.Size == 0 {
+		return nil
+	}
+
+	if entry.StartChunk < 0 {
+		return fmt.Errorf("The file %s starts at chunk %d", entry.Path, entry.StartChunk)
+	}
+
+	if entry.EndChunk >= len(chunkLengths) {
+		return fmt.Errorf("The file %s ends at chunk %d while the number of chunks is %d",
+			entry.Path, entry.EndChunk, len(chunkLengths))
+	}
+
+	if entry.EndChunk < entry.StartChunk {
+		return fmt.Errorf("The file %s starts at chunk %d and ends at chunk %d",
+			entry.Path, entry.StartChunk, entry.EndChunk)
+	}
+
+	if entry.StartOffset >= chunkLengths[entry.StartChunk] {
+		return fmt.Errorf("The file %s starts at offset %d of chunk %d of length %d",
+			entry.Path, entry.StartOffset, entry.StartChunk, chunkLengths[entry.StartChunk])
+	}
+
+	if entry.EndOffset > chunkLengths[entry.EndChunk] {
+		return fmt.Errorf("The file %s ends at offset %d of chunk %d of length %d",
+			entry.Path, entry.EndOffset, entry.EndChunk, chunkLengths[entry.EndChunk])
+	}
+
+	fileSize := int64(0)
+
+	for i := entry.StartChunk; i <= entry.EndChunk; i++ {
+
+		start := 0
+		if i == entry.StartChunk {
+			start = entry.StartOffset
+		}
+		end := chunkLengths[i]
+		if i == entry.EndChunk {
+			end = entry.EndOffset
+		}
+
+		fileSize += int64(end - start)
+	}
+
+	if entry.Size != fileSize {
+		return fmt.Errorf("The file %s has a size of %d but the total size of chunks is %d",
+			entry.Path, entry.Size, fileSize)
+	}
+
+	return nil
+}

src/duplicacy_entry_test.go

@@ -9,8 +9,15 @@ import (
 	"math/rand"
 	"os"
 	"path/filepath"
+	"runtime"
 	"sort"
+	"strings"
 	"testing"
+	"bytes"
+	"encoding/json"
+
+	"github.com/gilbertchen/xattr"
+    "github.com/vmihailenco/msgpack"
 )
 
 func TestEntrySort(t *testing.T) {
@@ -23,19 +30,19 @@ func TestEntrySort(t *testing.T) {
 		"\xBB\xDDfile",
 		"\xFF\xDDfile",
 		"ab/",
+		"ab-/",
+		"ab0/",
+		"ab1/",
 		"ab/c",
 		"ab+/c-",
 		"ab+/c0",
 		"ab+/c/",
-		"ab+/c/d",
 		"ab+/c+/",
-		"ab+/c+/d",
 		"ab+/c0/",
+		"ab+/c/d",
+		"ab+/c+/d",
 		"ab+/c0/d",
-		"ab-/",
 		"ab-/c",
-		"ab0/",
-		"ab1/",
 		"ab1/c",
 		"ab1/\xBB\xDDfile",
 		"ab1/\xFF\xDDfile",
@@ -82,7 +89,7 @@ func TestEntrySort(t *testing.T) {
 	}
 }
 
-func TestEntryList(t *testing.T) {
+func TestEntryOrder(t *testing.T) {
 
 	testDir := filepath.Join(os.TempDir(), "duplicacy_test")
 	os.RemoveAll(testDir)
@@ -94,16 +101,16 @@ func TestEntryList(t *testing.T) {
 		"ab0",
 		"ab1",
 		"ab+/",
+		"ab2/",
+		"ab3/",
 		"ab+/c",
 		"ab+/c+",
 		"ab+/c1",
 		"ab+/c-/",
-		"ab+/c-/d",
 		"ab+/c0/",
+		"ab+/c-/d",
 		"ab+/c0/d",
-		"ab2/",
 		"ab2/c",
-		"ab3/",
 		"ab3/c",
 	}
 
@@ -168,18 +175,24 @@ func TestEntryList(t *testing.T) {
 	directories = append(directories, CreateEntry("", 0, 0, 0))
 
 	entries := make([]*Entry, 0, 4)
+    entryChannel := make(chan *Entry, 1024)
+	entries = append(entries, CreateEntry("", 0, 0, 0))
 
 	for len(directories) > 0 {
 		directory := directories[len(directories)-1]
 		directories = directories[:len(directories)-1]
-		entries = append(entries, directory)
-		subdirectories, _, err := ListEntries(testDir, directory.Path, &entries, nil, false)
+		subdirectories, _, err := ListEntries(testDir, directory.Path, nil, "", false, entryChannel)
 		if err != nil {
 			t.Errorf("ListEntries(%s, %s) returned an error: %s", testDir, directory.Path, err)
 		}
 		directories = append(directories, subdirectories...)
 	}
 
+	close(entryChannel)
+	for entry := range entryChannel {
+		entries = append(entries, entry)
+	}
+
 	entries = entries[1:]
 
 	for _, entry := range entries {
@@ -216,3 +229,147 @@ func TestEntryList(t *testing.T) {
 	}
 
 }
+
+// TestEntryExcludeByAttribute tests the excludeByAttribute parameter to the ListEntries function
+func TestEntryExcludeByAttribute(t *testing.T) {
+
+	if !(runtime.GOOS == "darwin" || runtime.GOOS == "linux") {
+		t.Skip("skipping test not darwin or linux")
+	}
+
+	testDir := filepath.Join(os.TempDir(), "duplicacy_test")
+
+	os.RemoveAll(testDir)
+	os.MkdirAll(testDir, 0700)
+
+	// Files or folders named with "exclude" below will have the exclusion attribute set on them
+	// When ListEntries is called with excludeByAttribute true, they should be excluded.
+	DATA := [...]string{
+		"excludefile",
+		"includefile",
+		"excludedir/",
+		"excludedir/file",
+		"includedir/",
+		"includedir/includefile",
+		"includedir/excludefile",
+	}
+
+	for _, file := range DATA {
+		fullPath := filepath.Join(testDir, file)
+		if file[len(file)-1] == '/' {
+			err := os.Mkdir(fullPath, 0700)
+			if err != nil {
+				t.Errorf("Mkdir(%s) returned an error: %s", fullPath, err)
+			}
+			continue
+		}
+
+		err := ioutil.WriteFile(fullPath, []byte(file), 0700)
+		if err != nil {
+			t.Errorf("WriteFile(%s) returned an error: %s", fullPath, err)
+		}
+	}
+
+	for _, file := range DATA {
+		fullPath := filepath.Join(testDir, file)
+		if strings.Contains(file, "exclude") {
+			xattr.Setxattr(fullPath, "com.apple.metadata:com_apple_backup_excludeItem", []byte("com.apple.backupd"))
+		}
+	}
+
+	for _, excludeByAttribute := range [2]bool{true, false} {
+		t.Logf("testing excludeByAttribute: %t", excludeByAttribute)
+		directories := make([]*Entry, 0, 4)
+		directories = append(directories, CreateEntry("", 0, 0, 0))
+
+		entries := make([]*Entry, 0, 4)
+		entryChannel := make(chan *Entry, 1024)
+		entries = append(entries, CreateEntry("", 0, 0, 0))
+
+		for len(directories) > 0 {
+			directory := directories[len(directories)-1]
+			directories = directories[:len(directories)-1]
+			subdirectories, _, err := ListEntries(testDir, directory.Path, nil, "", excludeByAttribute, entryChannel)
+			if err != nil {
+				t.Errorf("ListEntries(%s, %s) returned an error: %s", testDir, directory.Path, err)
+			}
+			directories = append(directories, subdirectories...)
+		}
+
+		close(entryChannel)
+
+		for entry := range entryChannel {
+			entries = append(entries, entry)
+		}
+
+		entries = entries[1:]
+
+		for _, entry := range entries {
+			t.Logf("entry: %s", entry.Path)
+		}
+
+		i := 0
+		for _, file := range DATA {
+			entryFound := false
+			var entry *Entry
+			for _, entry = range entries {
+				if entry.Path == file {
+					entryFound = true
+					break
+				}
+			}
+
+			if excludeByAttribute && strings.Contains(file, "exclude") {
+				if entryFound {
+					t.Errorf("file: %s, expected to be excluded but wasn't. attributes: %v", file, entry.Attributes)
+					i++
+				} else {
+					t.Logf("file: %s, excluded", file)
+				}
+			} else {
+				if entryFound {
+					t.Logf("file: %s, included. attributes: %v", file, entry.Attributes)
+					i++
+				} else {
+					t.Errorf("file: %s, expected to be included but wasn't", file)
+				}
+			}
+		}
+
+	}
+
+	if !t.Failed() {
+		os.RemoveAll(testDir)
+	}
+
+}
+
+func TestEntryEncoding(t *testing.T) {
+	buffer := new(bytes.Buffer)
+	encoder := msgpack.NewEncoder(buffer)
+
+	entry1 := CreateEntry("abcd", 1, 2, 0700)
+	err := encoder.Encode(entry1)
+	if err != nil {
+		t.Errorf("Failed to encode the entry: %v", err)
+		return
+	}
+
+	t.Logf("msgpack size: %d\n", len(buffer.Bytes()))
+	decoder := msgpack.NewDecoder(buffer)
+
+	description, _ := json.Marshal(entry1)
+	t.Logf("json size: %d\n", len(description))
+
+	var entry2 Entry
+	err = decoder.Decode(&entry2)
+	if err != nil {
+		t.Errorf("Failed to decode the entry: %v", err)
+		return
+	}
+
+	if entry1.Path != entry2.Path || entry1.Size != entry2.Size || entry1.Time != entry2.Time {
+		t.Error("Decoded entry is different than the original one")
+	}
+
+}

src/duplicacy_entrylist.go

@@ -0,0 +1,574 @@
+// Copyright (c) Acrosync LLC. All rights reserved.
+// Free for personal use and commercial trial
+// Commercial use requires per-user licenses available from https://duplicacy.com
+
+package duplicacy
+
+import (
+	"encoding/hex"
+	"encoding/binary"
+	"fmt"
+	"os"
+	"io"
+	"path"
+	"crypto/sha256"
+	"crypto/rand"	
+	"sync"
+
+    "github.com/vmihailenco/msgpack"
+)
+
+// This struct stores information about a file entry that has been modified
+type ModifiedEntry struct {
+	Path string
+	Size int64
+	Hash string
+}
+
+// EntryList is basically a list of entries, which can be kept in the memory, or serialized to a disk file,
+// depending on if maximumInMemoryEntries is reached.
+//
+// The idea behind the on-disk entry list is that entries are written to a disk file as they are coming in.
+// Entries that have been modified and thus need to be uploaded will have their Incomplete bit set (i.e., 
+// with a size of -1).  When the limit is reached, entries are moved to a disk file but ModifiedEntries and
+// UploadedChunks are still kept in memory.  When later entries are read from the entry list, incomplete 
+// entries are back-annotated with info from ModifiedEntries and UploadedChunk* before sending them out.
+
+type EntryList struct {
+	onDiskFile *os.File                     // the file to store entries
+	encoder *msgpack.Encoder                // msgpack encoder for entry serialization
+	entries []*Entry                        // in-memory entry list
+	
+	SnapshotID string                       // the snapshot id
+	Token string                            // this unique random token makes sure we read/write 
+	                                        // the same entry list
+	ModifiedEntries []ModifiedEntry         // entries that will be uploaded
+
+	UploadedChunkHashes []string            // chunks from entries that have been uploaded
+	UploadedChunkLengths []int              // chunk lengths from entries that have been uploaded
+	uploadedChunkLock sync.Mutex            // lock for UploadedChunkHashes and UploadedChunkLengths
+
+	PreservedChunkHashes []string           // chunks from entries not changed
+	PreservedChunkLengths []int             // chunk lengths from entries not changed
+
+	Checksum string                         // checksum of all entries to detect disk corruption
+
+	maximumInMemoryEntries int              // max in-memory entries
+	NumberOfEntries int64                   // number of entries (not including directories and links)
+	cachePath string                        // the directory for the on-disk file
+
+	// These 3 variables are used in entry infomation back-annotation
+	modifiedEntryIndex int                  // points to the current modified entry
+	uploadedChunkIndex int                  // counter for upload chunks
+	uploadedChunkOffset int                 // the start offset for the current modified entry
+
+}
+
+// Create a new entry list
+func CreateEntryList(snapshotID string, cachePath string, maximumInMemoryEntries int) (*EntryList, error) {
+
+	token := make([]byte, 16)
+	_, err := rand.Read(token)
+	if err != nil {
+		return nil, fmt.Errorf("Failed to create a random token: %v", err)
+	}
+
+	entryList := &EntryList {
+		SnapshotID: snapshotID,
+		maximumInMemoryEntries: maximumInMemoryEntries,
+		cachePath: cachePath,
+		Token: string(token),
+	}
+
+	return entryList, nil
+
+}
+
+// Create the on-disk entry list file
+func (entryList *EntryList)createOnDiskFile() error {
+	file, err := os.OpenFile(path.Join(entryList.cachePath, "incomplete_files"), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
+	if err != nil {
+		return fmt.Errorf("Failed to create on disk entry list: %v", err)
+	}
+
+	entryList.onDiskFile = file
+	entryList.encoder = msgpack.NewEncoder(file)
+
+	err = entryList.encoder.EncodeString(entryList.Token)
+	if err != nil {
+		return fmt.Errorf("Failed to create on disk entry list: %v", err)
+	}
+
+	for _, entry := range entryList.entries {
+		err = entry.EncodeWithHash(entryList.encoder)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Add an entry to the entry list
+func (entryList *EntryList)AddEntry(entry *Entry) error {
+
+	if !entry.IsDir() && !entry.IsLink() {
+		entryList.NumberOfEntries++
+	}
+
+	if !entry.IsComplete() {
+		if entry.IsDir() || entry.IsLink() {
+			entry.Size = 0
+		} else {
+			modifiedEntry := ModifiedEntry {
+				Path: entry.Path,
+				Size: -1,
+			}
+		
+			entryList.ModifiedEntries = append(entryList.ModifiedEntries, modifiedEntry)
+		}
+	}
+
+	if entryList.onDiskFile != nil {
+		return entry.EncodeWithHash(entryList.encoder)
+	} else {
+		entryList.entries = append(entryList.entries, entry)
+		if entryList.maximumInMemoryEntries >= 0 && len(entryList.entries) > entryList.maximumInMemoryEntries {
+			err := entryList.createOnDiskFile()
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+// Add a preserved chunk that belongs to files that have not been modified
+func (entryList *EntryList)AddPreservedChunk(chunkHash string, chunkSize int) {
+	entryList.PreservedChunkHashes = append(entryList.PreservedChunkHashes, chunkHash)
+	entryList.PreservedChunkLengths = append(entryList.PreservedChunkLengths, chunkSize)
+}
+
+// Add a chunk just uploaded (that belongs to files that have been modified)
+func (entryList *EntryList)AddUploadedChunk(chunkIndex int, chunkHash string, chunkSize int) {
+	entryList.uploadedChunkLock.Lock()
+
+	for len(entryList.UploadedChunkHashes) <= chunkIndex {
+		entryList.UploadedChunkHashes = append(entryList.UploadedChunkHashes, "")
+	}
+
+	for len(entryList.UploadedChunkLengths) <= chunkIndex {
+		entryList.UploadedChunkLengths = append(entryList.UploadedChunkLengths, 0)
+	}
+
+	entryList.UploadedChunkHashes[chunkIndex] = chunkHash
+	entryList.UploadedChunkLengths[chunkIndex] = chunkSize
+	entryList.uploadedChunkLock.Unlock()
+}
+
+// Close the on-disk file
+func (entryList *EntryList) CloseOnDiskFile() error {
+
+	if entryList.onDiskFile == nil {
+		return nil
+	}
+
+	err := entryList.onDiskFile.Sync()
+	if err != nil {
+		return err
+	}
+
+	err = entryList.onDiskFile.Close()
+	if err != nil {
+		return err
+	}
+
+	entryList.onDiskFile = nil
+	return nil
+}
+
+// Return the length of the `index`th chunk
+func (entryList *EntryList) getChunkLength(index int) int {
+	if index < len(entryList.PreservedChunkLengths) {
+		return entryList.PreservedChunkLengths[index]
+	} else {
+		return entryList.UploadedChunkLengths[index - len(entryList.PreservedChunkLengths)]
+	}
+}
+
+// Sanity check for each entry
+func (entryList *EntryList) checkEntry(entry *Entry) error {
+
+	if entry.Size < 0 {
+		return fmt.Errorf("the file %s hash an invalid size (%d)", entry.Path, entry.Size)
+	}
+
+	if !entry.IsFile() || entry.Size == 0 {
+		return nil
+	}
+
+	numberOfChunks := len(entryList.PreservedChunkLengths) + len(entryList.UploadedChunkLengths) 
+
+	if entry.StartChunk < 0 {
+		return fmt.Errorf("the file %s starts at chunk %d", entry.Path, entry.StartChunk)
+	}
+
+	if entry.EndChunk >= numberOfChunks {
+		return fmt.Errorf("the file %s ends at chunk %d while the number of chunks is %d",
+			entry.Path, entry.EndChunk, numberOfChunks)
+	}
+
+	if entry.EndChunk < entry.StartChunk {
+		return fmt.Errorf("the file %s starts at chunk %d and ends at chunk %d",
+			entry.Path, entry.StartChunk, entry.EndChunk)
+	}
+
+	if entry.StartOffset >= entryList.getChunkLength(entry.StartChunk) {
+		return fmt.Errorf("the file %s starts at offset %d of chunk %d with a length of %d",
+			entry.Path, entry.StartOffset, entry.StartChunk, entryList.getChunkLength(entry.StartChunk))
+	}
+
+	if entry.EndOffset > entryList.getChunkLength(entry.EndChunk) {
+		return fmt.Errorf("the file %s ends at offset %d of chunk %d with a length of %d",
+			entry.Path, entry.EndOffset, entry.EndChunk, entryList.getChunkLength(entry.EndChunk))
+	}
+
+	fileSize := int64(0)
+
+	for i := entry.StartChunk; i <= entry.EndChunk; i++ {
+
+		start := 0
+		if i == entry.StartChunk {
+			start = entry.StartOffset
+		}
+		end := entryList.getChunkLength(i)
+		if i == entry.EndChunk {
+			end = entry.EndOffset
+		}
+
+		fileSize += int64(end - start)
+	}
+
+	if entry.Size != fileSize {
+		return fmt.Errorf("the file %s has a size of %d but the total size of chunks is %d",
+			entry.Path, entry.Size, fileSize)
+	}
+
+	return nil
+}
+
+// An incomplete entry (with a size of -1) does not have 'startChunk', 'startOffset', 'endChunk', and 'endOffset'.  This function
+// is to fill in these information before sending the entry out.
+func (entryList *EntryList) fillAndSendEntry(entry *Entry, entryOut func(*Entry)error) (skipped bool, err error) {
+
+	if entry.IsComplete() {
+		err := entryList.checkEntry(entry)
+		if err != nil {
+			return false, err
+		}
+		return false, entryOut(entry)
+	}
+
+	if entryList.modifiedEntryIndex >= len(entryList.ModifiedEntries) {
+		return false, fmt.Errorf("Unexpected file index %d (%d modified files)", entryList.modifiedEntryIndex, len(entryList.ModifiedEntries))
+	}
+
+	modifiedEntry := &entryList.ModifiedEntries[entryList.modifiedEntryIndex]
+	entryList.modifiedEntryIndex++
+
+	if modifiedEntry.Path != entry.Path {
+		return false, fmt.Errorf("Unexpected file path %s when expecting %s", modifiedEntry.Path, entry.Path)
+	}
+
+	if modifiedEntry.Size <= 0 {
+		return true, nil
+	}
+
+	entry.Size = modifiedEntry.Size
+	entry.Hash = modifiedEntry.Hash
+
+	entry.StartChunk = entryList.uploadedChunkIndex + len(entryList.PreservedChunkHashes)
+	entry.StartOffset = entryList.uploadedChunkOffset
+	entry.EndChunk = entry.StartChunk
+	endOffset := int64(entry.StartOffset) + entry.Size
+
+	for entryList.uploadedChunkIndex < len(entryList.UploadedChunkLengths) && endOffset > int64(entryList.UploadedChunkLengths[entryList.uploadedChunkIndex]) {
+		endOffset -= int64(entryList.UploadedChunkLengths[entryList.uploadedChunkIndex])
+		entry.EndChunk++
+		entryList.uploadedChunkIndex++
+	}
+
+	if entryList.uploadedChunkIndex >= len(entryList.UploadedChunkLengths) {
+		return false, fmt.Errorf("File %s has not been completely uploaded", entry.Path)
+	}
+
+	entry.EndOffset = int(endOffset)
+	entryList.uploadedChunkOffset = entry.EndOffset 
+	if entry.EndOffset == entryList.UploadedChunkLengths[entryList.uploadedChunkIndex] {
+		entryList.uploadedChunkIndex++
+		entryList.uploadedChunkOffset = 0
+	}
+
+	err = entryList.checkEntry(entry)
+	if err != nil {
+		return false, err
+	}
+
+	return false, entryOut(entry)
+}
+
+// Iterate through the entries in this entry list
+func (entryList *EntryList) ReadEntries(entryOut func(*Entry)error) (error) {
+
+	entryList.modifiedEntryIndex = 0
+	entryList.uploadedChunkIndex = 0
+	entryList.uploadedChunkOffset = 0
+
+	if entryList.onDiskFile == nil {
+		for _, entry := range entryList.entries {
+			skipped, err := entryList.fillAndSendEntry(entry.Copy(), entryOut)
+			if err != nil {
+				return err
+			}
+			if skipped {
+				continue
+			}
+		}
+	} else {
+		_, err := entryList.onDiskFile.Seek(0, os.SEEK_SET)
+		if err != nil {
+			return err
+		}
+		decoder := msgpack.NewDecoder(entryList.onDiskFile)
+
+		_, err = decoder.DecodeString()
+		if err != nil {
+			return err
+		}
+	
+    	for _, err = decoder.PeekCode(); err == nil; _, err = decoder.PeekCode() {	
+			entry, err := DecodeEntryWithHash(decoder)
+			if err != nil {
+				return err
+			}
+			skipped, err := entryList.fillAndSendEntry(entry, entryOut)
+			if err != nil {
+				return err
+			}
+			if skipped {
+				continue
+			}
+		}
+
+		if err != io.EOF {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// When saving an incomplete snapshot, the on-disk entry list ('incomplete_files') is renamed to
+// 'incomplete_snapshot', and this EntryList struct is saved as 'incomplete_chunks'.
+func (entryList *EntryList) SaveIncompleteSnapshot() {
+	entryList.uploadedChunkLock.Lock()
+	defer entryList.uploadedChunkLock.Unlock()
+
+	if entryList.onDiskFile == nil {
+		err := entryList.createOnDiskFile()
+		if err != nil {
+			LOG_WARN("INCOMPLETE_SAVE", "Failed to create the incomplete snapshot file: %v", err)
+			return
+		}
+
+		for _, entry := range entryList.entries {
+
+			err = entry.EncodeWithHash(entryList.encoder)
+			if err != nil {
+				LOG_WARN("INCOMPLETE_SAVE", "Failed to save the entry %s: %v", entry.Path, err)
+				return
+			}
+		}
+	}
+
+	err := entryList.onDiskFile.Close()
+	if err != nil {
+		LOG_WARN("INCOMPLETE_SAVE", "Failed to close the on-disk file: %v", err)
+		return
+	}
+
+	filePath := path.Join(entryList.cachePath, "incomplete_snapshot")
+	if _, err := os.Stat(filePath); err == nil {
+		err = os.Remove(filePath)
+		if err != nil {
+			LOG_WARN("INCOMPLETE_REMOVE", "Failed to remove previous incomplete snapshot: %v", err)
+		}
+	}
+
+	err = os.Rename(path.Join(entryList.cachePath, "incomplete_files"), filePath)
+	if err != nil {
+		LOG_WARN("INCOMPLETE_SAVE", "Failed to rename the incomplete snapshot file: %v", err)
+		return
+	}
+
+	chunkFile := path.Join(entryList.cachePath, "incomplete_chunks")
+	file, err := os.OpenFile(chunkFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
+	if err != nil {
+		LOG_WARN("INCOMPLETE_SAVE", "Failed to create the incomplete chunk file: %v", err)
+		return
+	}
+
+	defer file.Close()
+	encoder := msgpack.NewEncoder(file)
+
+	entryList.Checksum = entryList.CalculateChecksum()
+
+	err = encoder.Encode(entryList)
+	if err != nil {
+		LOG_WARN("INCOMPLETE_SAVE", "Failed to save the incomplete snapshot: %v", err)
+		return
+	}
+
+	LOG_INFO("INCOMPLETE_SAVE", "Incomplete snapshot saved to %s", filePath)	
+}
+
+// Calculate a checksum for this entry list
+func (entryList *EntryList) CalculateChecksum() string{
+
+	hasher := sha256.New()
+	for _, s := range entryList.UploadedChunkHashes {
+		hasher.Write([]byte(s))
+	}
+
+	buffer := make([]byte, 8)
+	for _, i := range entryList.UploadedChunkLengths {
+		binary.LittleEndian.PutUint64(buffer, uint64(i))
+		hasher.Write(buffer)
+	}
+
+	for _, s := range entryList.PreservedChunkHashes {
+		hasher.Write([]byte(s))
+	}
+
+	for _, i := range entryList.PreservedChunkLengths {
+		binary.LittleEndian.PutUint64(buffer, uint64(i))
+		hasher.Write(buffer)
+	}
+
+	for _, entry := range entryList.ModifiedEntries {
+		binary.LittleEndian.PutUint64(buffer, uint64(entry.Size))
+		hasher.Write(buffer)
+		hasher.Write([]byte(entry.Hash))
+	}
+
+	return hex.EncodeToString(hasher.Sum(nil))
+}
+
+// Check if all chunks exist in 'chunkCache'
+func (entryList *EntryList) CheckChunks(config *Config, chunkCache map[string]bool) bool {
+	for _, chunkHash := range entryList.UploadedChunkHashes {
+		chunkID := config.GetChunkIDFromHash(chunkHash)
+		if _, ok := chunkCache[chunkID]; !ok {
+			return false
+		}
+	}
+	for _, chunkHash := range entryList.PreservedChunkHashes {
+		chunkID := config.GetChunkIDFromHash(chunkHash)
+		if _, ok := chunkCache[chunkID]; !ok {
+			return false
+		}
+	}
+
+	return true
+
+}
+
+// Recover the on disk file from 'incomplete_snapshot', and restore the EntryList struct
+// from 'incomplete_chunks'
+func loadIncompleteSnapshot(snapshotID string, cachePath string) *EntryList {
+
+	onDiskFilePath := path.Join(cachePath, "incomplete_snapshot")
+	entryListFilePath := path.Join(cachePath, "incomplete_chunks")
+
+	if _, err := os.Stat(onDiskFilePath); os.IsNotExist(err) {
+		return nil
+	}
+
+	if _, err := os.Stat(entryListFilePath); os.IsNotExist(err) {
+		return nil
+	}
+
+	entryList := &EntryList {}
+	entryListFile, err := os.OpenFile(entryListFilePath, os.O_RDONLY, 0600)
+	if err != nil {
+		LOG_WARN("INCOMPLETE_LOAD", "Failed to open the incomplete snapshot: %v", err)
+		return nil
+	}
+
+	defer entryListFile.Close()
+	decoder := msgpack.NewDecoder(entryListFile)
+	err = decoder.Decode(&entryList)
+	if err != nil {
+		LOG_WARN("INCOMPLETE_LOAD", "Failed to load the incomplete snapshot: %v", err)
+		return nil
+	}
+
+	checksum := entryList.CalculateChecksum()
+	if checksum != entryList.Checksum {
+		LOG_WARN("INCOMPLETE_LOAD", "Failed to load the incomplete snapshot: checksum mismatched")
+		return nil
+	}
+
+	onDiskFile, err := os.OpenFile(onDiskFilePath, os.O_RDONLY, 0600)
+	if err != nil {
+		LOG_WARN("INCOMPLETE_LOAD", "Failed to open the on disk file for the incomplete snapshot: %v", err)
+		return nil
+	}
+
+	decoder = msgpack.NewDecoder(onDiskFile)
+	token, err := decoder.DecodeString()
+	if err != nil {
+		LOG_WARN("INCOMPLETE_LOAD", "Failed to read the token for the incomplete snapshot: %v", err)
+		onDiskFile.Close()
+		return nil
+	}
+
+	if token != entryList.Token {
+		LOG_WARN("INCOMPLETE_LOAD", "Mismatched tokens in the incomplete snapshot")
+		onDiskFile.Close()
+		return nil
+	}
+
+	entryList.onDiskFile = onDiskFile
+
+	for i, hash := range entryList.UploadedChunkHashes {
+		if len(hash) == 0 {
+			// An empty hash means the chunk has not been uploaded in previous run
+			entryList.UploadedChunkHashes = entryList.UploadedChunkHashes[0:i]
+			entryList.UploadedChunkLengths = entryList.UploadedChunkLengths[0:i]
+			break
+		}
+	}
+
+	LOG_INFO("INCOMPLETE_LOAD", "Previous incomplete backup contains %d files and %d chunks",
+	         entryList.NumberOfEntries, len(entryList.PreservedChunkLengths) + len(entryList.UploadedChunkHashes))
+
+	return entryList
+}
+
+// Delete the two incomplete files.
+func deleteIncompleteSnapshot(cachePath string) {
+
+	for _, file := range []string{"incomplete_snapshot", "incomplete_chunks", "incomplete_files"} {
+		filePath := path.Join(cachePath, file)
+		if _, err := os.Stat(filePath); err == nil {
+			err = os.Remove(filePath)
+			if err != nil {
+				LOG_WARN("INCOMPLETE_REMOVE", "Failed to remove the incomplete snapshot: %v", err)
+				return
+			}
+		}
+	}
+
+
+}

src/duplicacy_entrylist_test.go

@@ -0,0 +1,179 @@
+// Copyright (c) Acrosync LLC. All rights reserved.
+// Free for personal use and commercial trial
+// Commercial use requires per-user licenses available from https://duplicacy.com
+
+package duplicacy
+
+import (
+	"os"
+	"path"
+	"time"
+	"testing"
+	"math/rand"
+)
+
+
+func generateRandomString(length int) string {
+	var letters = []rune("abcdefghijklmnopqrstuvwxyz")
+    b := make([]rune, length)
+    for i := range b {
+        b[i] = letters[rand.Intn(len(letters))]
+    }
+    return string(b)
+}
+
+var fileSizeGenerator = rand.NewZipf(rand.New(rand.NewSource(time.Now().UnixNano())), 1.2, 1.0, 1024)
+
+func generateRandomFileSize() int64 {
+	return int64(fileSizeGenerator.Uint64() + 1)
+}
+
+func generateRandomChunks(totalFileSize int64) (chunks []string, lengths []int) {
+
+	totalChunkSize := int64(0)
+
+	for totalChunkSize < totalFileSize {
+		chunks = append(chunks, generateRandomString(64))
+		chunkSize := int64(1 + (rand.Int() % 64))
+		if chunkSize + totalChunkSize > totalFileSize {
+			chunkSize = totalFileSize - totalChunkSize
+		}
+		lengths = append(lengths, int(chunkSize))
+		totalChunkSize += chunkSize
+	}
+	return chunks, lengths
+
+}
+
+func getPreservedChunks(entries []*Entry, chunks []string, lengths []int) (preservedChunks []string, preservedChunkLengths []int) {
+	lastPreservedChunk := -1
+	for i := range entries {
+		if entries[i].Size < 0 {
+			continue
+		}
+		delta := entries[i].StartChunk - len(chunks)
+		if lastPreservedChunk != entries[i].StartChunk {
+			lastPreservedChunk = entries[i].StartChunk
+			preservedChunks = append(preservedChunks, chunks[entries[i].StartChunk])
+			preservedChunkLengths = append(preservedChunkLengths, lengths[entries[i].StartChunk])
+			delta++
+		}
+		for j := entries[i].StartChunk + 1; i <= entries[i].EndChunk; i++ {
+			preservedChunks = append(preservedChunks, chunks[j])
+			preservedChunkLengths = append(preservedChunkLengths, lengths[j])
+			lastPreservedChunk = j
+		}
+	}
+
+	return
+}
+
+func testEntryList(t *testing.T, numberOfEntries int, maximumInMemoryEntries int) {
+
+	entries := make([]*Entry, 0, numberOfEntries)
+	entrySizes := make([]int64, 0)
+
+	for i := 0; i < numberOfEntries; i++ {
+		entry:= CreateEntry(generateRandomString(16), -1, 0, 0700)
+		entries = append(entries, entry)
+		entrySizes = append(entrySizes, generateRandomFileSize())
+	}
+
+	totalFileSize := int64(0)
+	for _, size := range entrySizes {
+		totalFileSize += size
+	}
+
+	testDir := path.Join(os.TempDir(), "duplicacy_test")
+	os.RemoveAll(testDir)
+	os.MkdirAll(testDir, 0700)
+
+	os.MkdirAll(testDir + "/list1", 0700)
+	os.MkdirAll(testDir + "/list2", 0700)
+	os.MkdirAll(testDir + "/list3", 0700)
+	os.MkdirAll(testDir + "/list1", 0700)
+
+	// For the first entry list, all entries are new
+	entryList, _ := CreateEntryList("test", testDir + "/list1", maximumInMemoryEntries)
+	for _, entry := range entries {
+		entryList.AddEntry(entry)
+	}
+	uploadedChunks, uploadedChunksLengths := generateRandomChunks(totalFileSize)
+	for i, chunk := range uploadedChunks {
+		entryList.AddUploadedChunk(i, chunk, uploadedChunksLengths[i])
+	}
+
+	for i := range entryList.ModifiedEntries {
+		entryList.ModifiedEntries[i].Size = entrySizes[i]
+	}
+
+	totalEntries := 0
+	err := entryList.ReadEntries(func(entry *Entry) error {
+		totalEntries++
+		return nil
+	})
+
+	if err != nil {
+		t.Errorf("ReadEntries returned an error: %s", err)
+		return
+	}
+
+	if totalEntries != numberOfEntries {
+		t.Errorf("EntryList contains %d entries instead of %d", totalEntries, numberOfEntries)
+		return
+	}
+
+	// For the second entry list, half of the entries are new
+	for i := range entries {
+		if rand.Int() % 1 == 0 {
+			entries[i].Size = -1
+		} else {
+			entries[i].Size = entrySizes[i]
+		}
+	}
+
+	preservedChunks, preservedChunkLengths := getPreservedChunks(entries, uploadedChunks, uploadedChunksLengths)
+	entryList, _ = CreateEntryList("test", testDir + "/list2", maximumInMemoryEntries)
+	for _, entry := range entries {
+		entryList.AddEntry(entry)
+	}
+	for i, chunk := range preservedChunks {
+		entryList.AddPreservedChunk(chunk, preservedChunkLengths[i])
+	}
+
+	totalFileSize = 0
+	for i := range entryList.ModifiedEntries {
+		fileSize := generateRandomFileSize()		
+		entryList.ModifiedEntries[i].Size = fileSize
+		totalFileSize += fileSize
+	}
+
+	uploadedChunks, uploadedChunksLengths = generateRandomChunks(totalFileSize)
+	for i, chunk := range uploadedChunks {
+		entryList.AddUploadedChunk(i, chunk, uploadedChunksLengths[i])
+	}
+
+	totalEntries = 0
+	err = entryList.ReadEntries(func(entry *Entry) error {
+		totalEntries++
+		return nil
+	})
+
+	if err != nil {
+		t.Errorf("ReadEntries returned an error: %s", err)
+		return
+	}
+
+	if totalEntries != numberOfEntries {
+		t.Errorf("EntryList contains %d entries instead of %d", totalEntries, numberOfEntries)
+		return
+	}
+
+}
+
+
+func TestEntryList(t *testing.T) {
+	testEntryList(t, 1024, 1024)
+	testEntryList(t, 1024, 512)
+	testEntryList(t, 1024, 0)
+}

src/duplicacy_filefabricstorage.go

@@ -0,0 +1,618 @@
+// Copyright (c) Storage Made Easy. All rights reserved.
+// 
+// This storage backend is contributed by Storage Made Easy (https://storagemadeeasy.com/) to be used in
+// Duplicacy and its derivative works.
+//
+
+package duplicacy
+
+import (
+	"io"
+	"fmt"
+	"time"
+	"sync"
+	"bytes"
+	"errors"
+	"strings"
+	"net/url"
+	"net/http"
+	"math/rand"
+	"io/ioutil"
+	"encoding/xml"
+	"path/filepath"
+	"mime/multipart"
+)
+
+// The XML element representing a file returned by the File Fabric server
+type FileFabricFile struct {
+	XMLName     xml.Name
+	ID          string     `xml:"fi_id"`
+	Path        string     `xml:"path"`
+	Size        int64      `xml:"fi_size"`
+	Type        int        `xml:"fi_type"`
+}
+
+// The XML element representing a file list returned by the server
+type FileFabricFileList struct {
+	XMLName xml.Name `xml:"files"`
+	Files []FileFabricFile `xml:",any"`
+}
+
+type FileFabricStorage struct {
+	StorageBase
+
+	endpoint           string            // the server
+	authToken          string            // the authentication token
+	accessToken        string            // the access token (as returned by getTokenByAuthToken)
+	storageDir         string            // the path of the storage directory
+	storageDirID       string            // the id of 'storageDir'
+
+	client             *http.Client      // the default http client
+	threads            int               // number of threads
+	maxRetries         int               // maximum number of tries
+	directoryCache     map[string]string // stores ids for directories known to this backend
+	directoryCacheLock sync.Mutex        // lock for accessing directoryCache
+
+	isAuthorized       bool
+	testMode           bool
+}
+
+var (
+	errFileFabricAuthorizationFailure = errors.New("Authentication failure")
+	errFileFabricDirectoryExists = errors.New("Directory exists")
+)
+
+// The general server response
+type FileFabricResponse struct {
+	Status string `xml:"status"`
+	Message string `xml:"statusmessage"`
+}
+
+// Check the server response and return an error representing the error message it contains
+func checkFileFabricResponse(response FileFabricResponse, actionFormat string, actionArguments ...interface{}) error {
+
+	action := fmt.Sprintf(actionFormat, actionArguments...)
+	if response.Status == "ok" && response.Message == "Success" {
+		return nil
+	} else if response.Status == "error_data" {
+		if response.Message == "Folder with same name already exists." {
+			return errFileFabricDirectoryExists
+		}
+	}
+
+	return fmt.Errorf("Failed to %s (status: %s, message: %s)", action, response.Status, response.Message)
+}
+
+// Create a File Fabric storage backend
+func CreateFileFabricStorage(endpoint string, token string, storageDir string, threads int) (storage *FileFabricStorage, err error) {
+
+	if len(storageDir) > 0 && storageDir[len(storageDir)-1] != '/' {
+		storageDir += "/"
+	}
+
+	storage = &FileFabricStorage{
+
+		endpoint:       endpoint,
+		authToken:      token,
+		client:         http.DefaultClient,
+		threads:        threads,
+		directoryCache: make(map[string]string),
+		maxRetries:     12,
+	}
+
+	err = storage.getAccessToken()
+	if err != nil {
+		return nil, err
+	}
+
+	storageDirID, isDir, _, err := storage.getFileInfo(0, storageDir)
+	if err != nil {
+		return nil, err
+	}
+	if storageDirID == "" {
+		return nil, fmt.Errorf("Storage path %s does not exist", storageDir)
+	}
+	if !isDir {
+		return nil, fmt.Errorf("Storage path %s is not a directory", storageDir)
+	}
+	storage.storageDir = storageDir
+	storage.storageDirID = storageDirID
+
+	for _, dir := range []string{"snapshots", "chunks"} {
+		storage.CreateDirectory(0, dir)
+	}
+
+	storage.DerivedStorage = storage
+	storage.SetDefaultNestingLevels([]int{0}, 0)
+	return storage, nil
+}
+
+// Retrieve the access token using an auth token
+func (storage *FileFabricStorage) getAccessToken() (error) {
+
+    formData := url.Values { "authtoken": {storage.authToken},}
+	readCloser, _, _, err := storage.sendRequest(0, http.MethodPost, storage.getAPIURL("getTokenByAuthToken"), nil, formData)
+	if err != nil {
+		return err
+	}
+
+	defer readCloser.Close()
+	defer io.Copy(ioutil.Discard, readCloser)	
+
+    var output struct {
+		FileFabricResponse
+        Token string `xml:"token"`
+    }
+
+    err = xml.NewDecoder(readCloser).Decode(&output)
+    if err != nil {
+        return err
+    }
+
+	err = checkFileFabricResponse(output.FileFabricResponse, "request the access token")
+	if err != nil {
+		return err
+	}
+
+	storage.accessToken = output.Token
+    return nil
+}
+
+// Determine if we should retry based on the number of retries given by 'retry' and if so calculate the delay with exponential backoff
+func (storage *FileFabricStorage) shouldRetry(retry int, messageFormat string, messageArguments ...interface{}) bool {
+	message := fmt.Sprintf(messageFormat, messageArguments...)
+
+	if retry >= storage.maxRetries {
+		LOG_WARN("FILEFABRIC_REQUEST", "%s", message)
+		return false
+	}
+	backoff := 1 << uint(retry)
+	if backoff > 60 {
+		backoff = 60
+	}
+	delay := rand.Intn(backoff*500) + backoff*500
+	LOG_INFO("FILEFABRIC_RETRY", "%s; retrying after %.1f seconds", message, float32(delay) / 1000.0)
+	time.Sleep(time.Duration(delay) * time.Millisecond)
+	return true
+}
+
+// Send a request to the server
+func (storage *FileFabricStorage) sendRequest(threadIndex int, method string, requestURL string, requestHeaders map[string]string, input interface{}) ( io.ReadCloser, http.Header, int64, error) {
+
+	var response *http.Response
+
+	for retries := 0; ; retries++ {
+		var inputReader io.Reader
+
+		switch input.(type) {
+		case url.Values:
+			values := input.(url.Values)
+			inputReader = strings.NewReader(values.Encode())
+			if requestHeaders == nil {
+				requestHeaders = make(map[string]string)
+			}
+			requestHeaders["Content-Type"] = "application/x-www-form-urlencoded"
+		case *RateLimitedReader:
+			rateLimitedReader := input.(*RateLimitedReader)
+			rateLimitedReader.Reset()
+			inputReader = rateLimitedReader
+		default:
+			LOG_FATAL("FILEFABRIC_REQUEST", "Input type is not supported")
+			return nil, nil, 0, fmt.Errorf("Input type is not supported")
+		}
+
+		request, err := http.NewRequest(method, requestURL, inputReader)
+		if err != nil {
+			return nil, nil, 0, err
+		}
+
+		if requestHeaders != nil {
+			for key, value := range requestHeaders {
+				request.Header.Set(key, value)
+			}
+		}
+
+		if _, ok := input.(*RateLimitedReader); ok {
+			request.ContentLength = input.(*RateLimitedReader).Length()
+		}
+
+		response, err = storage.client.Do(request)
+		if err != nil {
+			if !storage.shouldRetry(retries, "[%d] %s %s returned an error: %v", threadIndex, method, requestURL, err) {
+				return nil, nil, 0, err
+			}
+			continue
+		}
+
+		if response.StatusCode < 300 {
+			return response.Body, response.Header, response.ContentLength, nil
+		}
+
+		defer response.Body.Close()
+		defer io.Copy(ioutil.Discard, response.Body)	
+
+		var output struct {
+			Status string `xml:"status"`
+			Message string `xml:"statusmessage"`
+		}
+	
+		err = xml.NewDecoder(response.Body).Decode(&output)
+		if err != nil {
+			if !storage.shouldRetry(retries, "[%d] %s %s returned an invalid response: %v", threadIndex, method, requestURL, err) {
+				return nil, nil, 0, err
+			}
+			continue
+		}
+
+		if !storage.shouldRetry(retries, "[%d] %s %s returned status: %s, message: %s", threadIndex, method, requestURL, output.Status, output.Message) {
+			return nil, nil, 0, err
+		}
+	}
+
+}
+
+func (storage *FileFabricStorage) getAPIURL(function string) string {
+	if storage.accessToken == "" {
+		return "https://" + storage.endpoint + "/api/*/" + function + "/"
+	} else {
+		return "https://" + storage.endpoint + "/api/" + storage.accessToken + "/" + function + "/"
+	}
+}
+
+// ListFiles return the list of files and subdirectories under 'dir'.  A subdirectories returned must have a trailing '/', with
+// a size of 0.  If 'dir' is 'snapshots', only subdirectories will be returned.  If 'dir' is 'snapshots/repository_id', then only
+// files will be returned.  If 'dir' is 'chunks', the implementation can return the list either recusively or non-recusively.
+func (storage *FileFabricStorage) ListFiles(threadIndex int, dir string) (files []string, sizes []int64, err error) {
+	if dir != "" && dir[len(dir)-1] != '/' {
+		dir += "/"
+	}
+
+	dirID, _, _, err := storage.getFileInfo(threadIndex, dir)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	if dirID == "" {
+		return nil, nil, nil
+	}
+
+	lastID := ""
+
+	for {
+		formData := url.Values { "marker": {lastID}, "limit": {"1000"}, "includefolders": {"n"}, "fi_pid" : {dirID}}
+		if dir == "snapshots/" {
+			formData["includefolders"] = []string{"y"}
+		}
+		if storage.testMode {
+			formData["limit"] = []string{"5"}
+		}
+
+		readCloser, _, _, err := storage.sendRequest(threadIndex, http.MethodPost, storage.getAPIURL("getListOfFiles"), nil, formData)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		defer readCloser.Close()
+		defer io.Copy(ioutil.Discard, readCloser)	
+
+		var output struct {
+			FileFabricResponse
+			FileList FileFabricFileList `xml:"files"`
+			Truncated int `xml:"truncated"`
+		}
+
+		err = xml.NewDecoder(readCloser).Decode(&output)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		err = checkFileFabricResponse(output.FileFabricResponse, "list the storage directory '%s'", dir)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		if dir == "snapshots/" {
+			for _, file := range output.FileList.Files {
+				if file.Type == 1 {
+					files = append(files, file.Path + "/")
+				}
+				lastID = file.ID
+			}
+		} else {
+			for _, file := range output.FileList.Files {
+				if file.Type == 0 {
+					files = append(files, file.Path)
+					sizes = append(sizes, file.Size)
+				}
+				lastID = file.ID
+			}
+		}
+
+		if output.Truncated != 1 {
+			break
+		}
+	}
+	return files, sizes, nil
+}
+
+// getFileInfo returns the information about the file or directory at 'filePath'.
+func (storage *FileFabricStorage) getFileInfo(threadIndex int, filePath string) (fileID string, isDir bool, size int64, err error) {
+
+	formData := url.Values { "path" : {storage.storageDir + filePath}}
+
+	readCloser, _, _, err := storage.sendRequest(threadIndex, http.MethodPost, storage.getAPIURL("checkPathExists"), nil, formData)
+	if err != nil {
+		return "", false, 0, err
+	}
+
+	defer readCloser.Close()
+	defer io.Copy(ioutil.Discard, readCloser)	
+
+	var output struct {
+		FileFabricResponse
+		File FileFabricFile `xml:"file"`
+		Exists string `xml:"exists"`
+	}
+
+	err = xml.NewDecoder(readCloser).Decode(&output)
+	if err != nil {
+		return "", false, 0, err
+	}
+
+	err = checkFileFabricResponse(output.FileFabricResponse, "get the info on '%s'", filePath)
+	if err != nil {
+		return "", false, 0, err
+	}
+
+	if output.Exists != "y" {
+		return "", false, 0, nil
+	} else {
+		if output.File.Type == 1 {
+			for filePath != "" && filePath[len(filePath)-1] == '/' {
+				filePath = filePath[:len(filePath)-1]
+			}
+			
+			storage.directoryCacheLock.Lock()
+			storage.directoryCache[filePath] = output.File.ID
+			storage.directoryCacheLock.Unlock()
+		}
+		return output.File.ID, output.File.Type == 1, output.File.Size, nil
+	}
+}
+
+// GetFileInfo returns the information about the file or directory at 'filePath'.  This is a function required by the Storage interface.
+func (storage *FileFabricStorage) GetFileInfo(threadIndex int, filePath string) (exist bool, isDir bool, size int64, err error) {
+
+	fileID := ""
+	fileID, isDir, size, err = storage.getFileInfo(threadIndex, filePath)
+	return fileID != "", isDir, size, err
+}
+
+// DeleteFile deletes the file or directory at 'filePath'.
+func (storage *FileFabricStorage) DeleteFile(threadIndex int, filePath string) (err error) {
+
+	fileID, _, _, _ := storage.getFileInfo(threadIndex, filePath)
+	if fileID == "" {
+		return nil
+	}
+
+	formData := url.Values { "fi_id" : {fileID}}
+
+	readCloser, _, _, err := storage.sendRequest(threadIndex, http.MethodPost, storage.getAPIURL("doDeleteFile"), nil, formData)
+	if err != nil {
+		return err
+	}
+
+	defer readCloser.Close()
+	defer io.Copy(ioutil.Discard, readCloser)	
+
+	var output FileFabricResponse
+
+	err = xml.NewDecoder(readCloser).Decode(&output)
+	if err != nil {
+		return err
+	}
+
+	err = checkFileFabricResponse(output, "delete file '%s'", filePath)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// MoveFile renames the file.
+func (storage *FileFabricStorage) MoveFile(threadIndex int, from string, to string) (err error) {
+	fileID, _, _, _ := storage.getFileInfo(threadIndex, from)
+	if fileID == "" {
+		return nil
+	}
+
+	formData := url.Values { "fi_id" : {fileID}, "fi_name": {filepath.Base(to)},}
+
+	readCloser, _, _, err := storage.sendRequest(threadIndex, http.MethodPost, storage.getAPIURL("doRenameFile"), nil, formData)
+	if err != nil {
+		return err
+	}
+
+	defer readCloser.Close()
+	defer io.Copy(ioutil.Discard, readCloser)	
+
+	var output FileFabricResponse
+
+	err = xml.NewDecoder(readCloser).Decode(&output)
+	if err != nil {
+		return err
+	}
+
+	err = checkFileFabricResponse(output, "rename file '%s' to '%s'", from, to)
+	if err != nil {
+		return err
+	}
+	
+	return nil
+}
+
+// createParentDirectory creates the parent directory if it doesn't exist in the cache.
+func (storage *FileFabricStorage) createParentDirectory(threadIndex int, dir string) (parentID string, err error) {
+
+	found := strings.LastIndex(dir, "/")
+	if found == -1 {
+		return storage.storageDirID, nil
+	}
+	parent := dir[:found]
+
+	storage.directoryCacheLock.Lock()
+	parentID = storage.directoryCache[parent]
+	storage.directoryCacheLock.Unlock()
+
+	if parentID != "" {
+		return parentID, nil
+	}
+
+	parentID, err = storage.createDirectory(threadIndex, parent)
+	if err != nil {
+		if err == errFileFabricDirectoryExists {
+		    var isDir bool
+			parentID, isDir, _, err = storage.getFileInfo(threadIndex, parent)
+			if err != nil {
+				return "", err
+			}
+			if isDir == false {
+				return "", fmt.Errorf("'%s' in the storage is a file", parent)
+			}
+			storage.directoryCacheLock.Lock()
+			storage.directoryCache[parent] = parentID
+			storage.directoryCacheLock.Unlock()
+			return parentID, nil
+		} else {
+			return "", err
+		}
+	}
+	return parentID, nil
+}
+
+// createDirectory creates a new directory.
+func (storage *FileFabricStorage) createDirectory(threadIndex int, dir string) (dirID string, err error) {
+	for dir != "" && dir[len(dir)-1] == '/' {
+		dir = dir[:len(dir)-1]
+	}
+
+	parentID, err := storage.createParentDirectory(threadIndex, dir)
+	if err != nil {
+		return "", err
+	}
+
+	formData := url.Values { "fi_name": {filepath.Base(dir)}, "fi_pid" : {parentID}}
+
+	readCloser, _, _, err := storage.sendRequest(threadIndex, http.MethodPost, storage.getAPIURL("doCreateNewFolder"), nil, formData)
+	if err != nil {
+		return "", err
+	}
+
+	defer readCloser.Close()
+	defer io.Copy(ioutil.Discard, readCloser)	
+
+	var output struct {
+		FileFabricResponse
+		File FileFabricFile `xml:"file"`
+	}
+
+	err = xml.NewDecoder(readCloser).Decode(&output)
+	if err != nil {
+		return "", err
+	}
+
+	err = checkFileFabricResponse(output.FileFabricResponse, "create directory '%s'", dir)
+	if err != nil {
+		return "", err
+	}
+
+	storage.directoryCacheLock.Lock()
+	storage.directoryCache[dir] = output.File.ID
+	storage.directoryCacheLock.Unlock()
+
+	return output.File.ID, nil
+}
+
+func (storage *FileFabricStorage) CreateDirectory(threadIndex int, dir string) (err error) {
+	_, err = storage.createDirectory(threadIndex, dir)
+	if err == errFileFabricDirectoryExists {
+		return nil
+	}
+	return err
+}
+
+// DownloadFile reads the file at 'filePath' into the chunk.
+func (storage *FileFabricStorage) DownloadFile(threadIndex int, filePath string, chunk *Chunk) (err error) {
+	formData := url.Values { "fi_id" : {storage.storageDir + filePath}}
+
+	readCloser, _, _, err := storage.sendRequest(threadIndex, http.MethodPost, storage.getAPIURL("getFile"), nil, formData)
+	if err != nil {
+		return err
+	}
+
+	defer readCloser.Close()
+	defer io.Copy(ioutil.Discard, readCloser)	
+	_, err = RateLimitedCopy(chunk, readCloser, storage.DownloadRateLimit/storage.threads)
+	return err
+}
+
+// UploadFile writes 'content' to the file at 'filePath'.
+func (storage *FileFabricStorage) UploadFile(threadIndex int, filePath string, content []byte) (err error) {
+
+	parentID, err := storage.createParentDirectory(threadIndex, filePath)
+	if err != nil {
+		return err
+	}
+
+	fileName := filepath.Base(filePath)
+    requestBody := &bytes.Buffer{}
+    writer := multipart.NewWriter(requestBody)
+    part, _ := writer.CreateFormFile("file_1", fileName)
+    part.Write(content)
+
+    writer.WriteField("file_name1", fileName)
+	writer.WriteField("fi_pid", parentID)
+	writer.WriteField("fi_structtype", "g")
+    writer.Close()
+
+	headers := make(map[string]string)
+	headers["Content-Type"] = writer.FormDataContentType()
+
+	rateLimitedReader := CreateRateLimitedReader(requestBody.Bytes(), storage.UploadRateLimit/storage.threads)
+	readCloser, _, _, err := storage.sendRequest(threadIndex, http.MethodPost, storage.getAPIURL("doUploadFiles"), headers, rateLimitedReader)
+
+	defer readCloser.Close()
+	defer io.Copy(ioutil.Discard, readCloser)	
+
+	var output FileFabricResponse
+
+	err = xml.NewDecoder(readCloser).Decode(&output)
+	if err != nil {
+		return err
+	}
+
+	err = checkFileFabricResponse(output, "upload file '%s'", filePath)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// If a local snapshot cache is needed for the storage to avoid downloading/uploading chunks too often when
+// managing snapshots.
+func (storage *FileFabricStorage) IsCacheNeeded() bool { return true }
+
+// If the 'MoveFile' method is implemented.
+func (storage *FileFabricStorage) IsMoveFileImplemented() bool { return true }
+
+// If the storage can guarantee strong consistency.
+func (storage *FileFabricStorage) IsStrongConsistent() bool { return false }
+
+// If the storage supports fast listing of files names.
+func (storage *FileFabricStorage) IsFastListing() bool { return false }
+
+// Enable the test mode.
+func (storage *FileFabricStorage) EnableTestMode() { storage.testMode = true }

src/duplicacy_filereader.go

@@ -34,7 +34,7 @@ func CreateFileReader(top string, files []*Entry) *FileReader {
 	return reader
 }
 
-// NextFile switchs to the next file in the file reader.
+// NextFile switches to the next file in the file reader.
 func (reader *FileReader) NextFile() bool {
 
 	if reader.CurrentFile != nil {

src/duplicacy_filestorage.go

@@ -12,6 +12,7 @@ import (
 	"os"
 	"path"
 	"strings"
+	"syscall"
 	"time"
 )
 
@@ -78,7 +79,7 @@ func (storage *FileStorage) ListFiles(threadIndex int, dir string) (files []stri
 
 	for _, f := range list {
 		name := f.Name()
-		if f.IsDir() && name[len(name)-1] != '/' {
+		if (f.IsDir() || f.Mode() & os.ModeSymlink != 0) && name[len(name)-1] != '/' {
 			name += "/"
 		}
 		files = append(files, name)
@@ -164,8 +165,8 @@ func (storage *FileStorage) UploadFile(threadIndex int, filePath string, content
 				return err
 			}
 		} else {
-			if !stat.IsDir() {
-				fmt.Errorf("The path %s is not a directory", dir)
+			if !stat.IsDir() && stat.Mode() & os.ModeSymlink == 0 {
+				return fmt.Errorf("The path %s is not a directory or symlink", dir)
 			}
 		}
 	}
@@ -190,7 +191,19 @@ func (storage *FileStorage) UploadFile(threadIndex int, filePath string, content
 		return err
 	}
 
-	file.Close()
+	if err = file.Sync(); err != nil {
+		pathErr, ok := err.(*os.PathError)
+		isNotSupported := ok && pathErr.Op == "sync" && pathErr.Err == syscall.ENOTSUP
+		if !isNotSupported {
+			_ = file.Close()
+			return err
+		}
+	}
+
+	err = file.Close()
+	if err != nil {
+		return err
+	}
 
 	err = os.Rename(temporaryFile, fullPath)
 	if err != nil {

src/duplicacy_gcdstorage.go

@@ -20,18 +20,28 @@ import (
 
 	"golang.org/x/net/context"
 	"golang.org/x/oauth2"
+	"golang.org/x/oauth2/google"
 	"google.golang.org/api/drive/v3"
 	"google.golang.org/api/googleapi"
+	"google.golang.org/api/option"
+)
+
+var (
+	GCDFileMimeType      = "application/octet-stream"
+	GCDDirectoryMimeType = "application/vnd.google-apps.folder"
+	GCDUserDrive         = "root"
 )
 
 type GCDStorage struct {
 	StorageBase
 
 	service     *drive.Service
-	idCache     map[string]string
+	idCache     map[string]string // only directories are saved in this cache
 	idCacheLock sync.Mutex
 	backoffs    []int // desired backoff time in seconds for each thread
 	attempts    []int // number of failed attempts since last success for each thread
+	driveID     string // the ID of the shared drive or 'root' (GCDUserDrive) if the user's drive
+	spaces      string // 'appDataFolder' if scope is drive.appdata; 'drive' otherwise
 
 	createDirectoryLock sync.Mutex
 	isConnected         bool
@@ -73,6 +83,14 @@ func (storage *GCDStorage) shouldRetry(threadIndex int, err error) (bool, error)
 			// User Rate Limit Exceeded
 			message = e.Message
 			retry = true
+		} else if e.Code == 408 {
+			// Request timeout
+			message = e.Message
+			retry = true
+		} else if e.Code == 400 && strings.Contains(e.Message, "failedPrecondition") {
+			// Daily quota exceeded
+			message = e.Message
+			retry = true
 		} else if e.Code == 401 {
 			// Only retry on authorization error when storage has been connected before
 			if storage.isConnected {
@@ -165,7 +183,7 @@ func (storage *GCDStorage) listFiles(threadIndex int, parentID string, listFiles
 
 	startToken := ""
 
-	query := "'" + parentID + "' in parents "
+	query := "'" + parentID + "' in parents and trashed = false "
 	if listFiles && !listDirectories {
 		query += "and mimeType != 'application/vnd.google-apps.folder'"
 	} else if !listFiles && !listDirectories {
@@ -182,7 +200,11 @@ func (storage *GCDStorage) listFiles(threadIndex int, parentID string, listFiles
 		var err error
 
 		for {
-			fileList, err = storage.service.Files.List().Q(query).Fields("nextPageToken", "files(name, mimeType, id, size)").PageToken(startToken).PageSize(maxCount).Do()
+			q := storage.service.Files.List().Q(query).Fields("nextPageToken", "files(name, mimeType, id, size)").PageToken(startToken).PageSize(maxCount).Spaces(storage.spaces)
+			if storage.driveID != GCDUserDrive {
+				q = q.DriveId(storage.driveID).IncludeItemsFromAllDrives(true).Corpora("drive").SupportsAllDrives(true)
+			}
+			fileList, err = q.Do()
 			if retry, e := storage.shouldRetry(threadIndex, err); e == nil && !retry {
 				break
 			} else if retry {
@@ -209,8 +231,12 @@ func (storage *GCDStorage) listByName(threadIndex int, parentID string, name str
 	var err error
 
 	for {
-		query := "name = '" + name + "' and '" + parentID + "' in parents"
-		fileList, err = storage.service.Files.List().Q(query).Fields("files(name, mimeType, id, size)").Do()
+		query := "name = '" + name + "' and '" + parentID + "' in parents and trashed = false "
+		q := storage.service.Files.List().Q(query).Fields("files(name, mimeType, id, size)").Spaces(storage.spaces)
+		if storage.driveID != GCDUserDrive {
+			q = q.DriveId(storage.driveID).IncludeItemsFromAllDrives(true).Corpora("drive").SupportsAllDrives(true)
+		}
+		fileList, err = q.Do()
 
 		if retry, e := storage.shouldRetry(threadIndex, err); e == nil && !retry {
 			break
@@ -227,7 +253,30 @@ func (storage *GCDStorage) listByName(threadIndex int, parentID string, name str
 
 	file := fileList.Files[0]
 
-	return file.Id, file.MimeType == "application/vnd.google-apps.folder", file.Size, nil
+	return file.Id, file.MimeType == GCDDirectoryMimeType, file.Size, nil
+}
+
+// Returns the id of the shared folder with the given name if it exists
+func (storage *GCDStorage) findSharedFolder(threadIndex int, name string) (string, error) {
+
+	query := "name = '" + name + "' and sharedWithMe and trashed = false and mimeType = 'application/vnd.google-apps.folder'"
+	q := storage.service.Files.List().Q(query).Fields("files(name, mimeType, id, size)").Spaces(storage.spaces)
+	if storage.driveID != GCDUserDrive {
+		q = q.DriveId(storage.driveID).IncludeItemsFromAllDrives(true).Corpora("drive").SupportsAllDrives(true)
+	}
+
+	fileList, err := q.Do()
+	if err != nil {
+		return "", err
+	}
+
+	if len(fileList.Files) == 0 {
+		return "", nil
+	}
+
+	file := fileList.Files[0]
+
+	return file.Id, nil
 }
 
 // getIDFromPath returns the id of the given path.  If 'createDirectories' is true, create the given path and all its
@@ -239,7 +288,7 @@ func (storage *GCDStorage) getIDFromPath(threadIndex int, filePath string, creat
 		return fileID, nil
 	}
 
-	fileID := "root"
+	fileID := storage.driveID
 
 	if rootID, ok := storage.findPathID(""); ok {
 		fileID = rootID
@@ -283,7 +332,7 @@ func (storage *GCDStorage) getIDFromPath(threadIndex int, filePath string, creat
 			}
 			fileID = currentID
 			continue
-		} else {
+		} else if isDir {
 			storage.savePathID(current, fileID)
 		}
 		if i != len(names)-1 && !isDir {
@@ -294,37 +343,97 @@ func (storage *GCDStorage) getIDFromPath(threadIndex int, filePath string, creat
 }
 
 // CreateGCDStorage creates a GCD storage object.
-func CreateGCDStorage(tokenFile string, storagePath string, threads int) (storage *GCDStorage, err error) {
+func CreateGCDStorage(tokenFile string, driveID string, storagePath string, threads int) (storage *GCDStorage, err error) {
+
+	ctx := context.Background()
 
 	description, err := ioutil.ReadFile(tokenFile)
 	if err != nil {
 		return nil, err
 	}
 
-	gcdConfig := &GCDConfig{}
-	if err := json.Unmarshal(description, gcdConfig); err != nil {
-		return nil, err
-	}
+	var object map[string]interface{}
 
-	oauth2Config := oauth2.Config{
-		ClientID:     gcdConfig.ClientID,
-		ClientSecret: gcdConfig.ClientSecret,
-		Endpoint:     gcdConfig.Endpoint,
-	}
-
-	authClient := oauth2Config.Client(context.Background(), &gcdConfig.Token)
-
-	service, err := drive.New(authClient)
+	err = json.Unmarshal(description, &object)
 	if err != nil {
 		return nil, err
 	}
 
+	isServiceAccount := false
+	if value, ok := object["type"]; ok {
+		if authType, ok := value.(string); ok && authType == "service_account" {
+			isServiceAccount = true
+		}
+	}
+
+	var tokenSource oauth2.TokenSource
+
+	scope := drive.DriveScope
+
+	if isServiceAccount {
+
+		if newScope, ok := object["scope"]; ok {
+			scope = newScope.(string)
+		}
+
+		config, err := google.JWTConfigFromJSON(description, scope)
+		if err != nil {
+			return nil, err
+		}
+
+		if subject, ok := object["subject"]; ok {
+		    config.Subject = subject.(string)
+		}
+
+		tokenSource = config.TokenSource(ctx)
+	} else {
+		gcdConfig := &GCDConfig{}
+		if err := json.Unmarshal(description, gcdConfig); err != nil {
+			return nil, err
+		}
+
+		config := oauth2.Config{
+			ClientID:     gcdConfig.ClientID,
+			ClientSecret: gcdConfig.ClientSecret,
+			Endpoint:     gcdConfig.Endpoint,
+		}
+		tokenSource = config.TokenSource(ctx, &gcdConfig.Token)
+	}
+
+	service, err := drive.NewService(ctx, option.WithTokenSource(tokenSource))
+	if err != nil {
+		return nil, err
+	}
+
+	if len(driveID) == 0 {
+		driveID = GCDUserDrive
+	} else {
+		// In case the driveID is a name, convert it to an id
+		query := fmt.Sprintf("name='%s'", driveID)
+		driveList, err := drive.NewDrivesService(service).List().Q(query).Do()
+		if err == nil {
+			found := false
+			for _, drive := range driveList.Drives {
+				if drive.Name == driveID {
+					found = true
+					driveID = drive.Id
+					break
+				}
+			}
+			if !found {
+				return nil, fmt.Errorf("%s is not the id or name of a shared drive", driveID)
+			}
+		}
+	}
+
 	storage = &GCDStorage{
 		service:         service,
 		numberOfThreads: threads,
 		idCache:         make(map[string]string),
 		backoffs:        make([]int, threads),
 		attempts:        make([]int, threads),
+		driveID:         driveID,
+		spaces:          "drive",
 	}
 
 	for i := range storage.backoffs {
@@ -332,11 +441,33 @@ func CreateGCDStorage(tokenFile string, storagePath string, threads int) (storag
 		storage.attempts[i] = 0
 	}
 
-	storagePathID, err := storage.getIDFromPath(0, storagePath, false)
-	if err != nil {
-		return nil, err
+
+	if scope == drive.DriveAppdataScope {
+		storage.spaces = "appDataFolder"
+		storage.savePathID("", "appDataFolder")
+	} else {
+		storage.savePathID("", driveID)
 	}
 
+	storagePathID := ""
+
+	// When using service acount, check if storagePath is a shared folder which takes priority over regular folders.
+	if isServiceAccount && !strings.Contains(storagePath, "/") {
+		storagePathID, err = storage.findSharedFolder(0, storagePath)
+		if err != nil {
+			LOG_WARN("GCD_STORAGE", "Failed to check if %s is a shared folder: %v", storagePath, err)
+		}
+	}
+
+	if storagePathID == "" {
+		storagePathID, err = storage.getIDFromPath(0, storagePath, true)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// Reset the id cache and start with 'storagePathID' as the root
+	storage.idCache = make(map[string]string)
 	storage.idCache[""] = storagePathID
 
 	for _, dir := range []string{"chunks", "snapshots", "fossils"} {
@@ -383,7 +514,7 @@ func (storage *GCDStorage) ListFiles(threadIndex int, dir string) ([]string, []i
 			subDirs = append(subDirs, file.Name+"/")
 		}
 		return subDirs, nil, nil
-	} else if strings.HasPrefix(dir, "snapshots/") {
+	} else if strings.HasPrefix(dir, "snapshots/") || strings.HasPrefix(dir, "benchmark") {
 		pathID, err := storage.getIDFromPath(threadIndex, dir, false)
 		if err != nil {
 			return nil, nil, err
@@ -400,44 +531,80 @@ func (storage *GCDStorage) ListFiles(threadIndex int, dir string) ([]string, []i
 		files := []string{}
 
 		for _, entry := range entries {
-			storage.savePathID(dir+"/"+entry.Name, entry.Id)
 			files = append(files, entry.Name)
 		}
 		return files, nil, nil
 	} else {
-		files := []string{}
-		sizes := []int64{}
+		lock := sync.Mutex {}
+		allFiles := []string{}
+		allSizes := []int64{}
+
+		errorChannel := make(chan error)
+		directoryChannel := make(chan string)
+		activeWorkers := 0
 
 		parents := []string{"chunks", "fossils"}
-		for i := 0; i < len(parents); i++ {
-			parent := parents[i]
-			pathID, ok := storage.findPathID(parent)
-			if !ok {
-				continue
-			}
-			entries, err := storage.listFiles(threadIndex, pathID, true, true)
-			if err != nil {
-				return nil, nil, err
-			}
-			for _, entry := range entries {
-				if entry.MimeType != "application/vnd.google-apps.folder" {
-					name := entry.Name
-					if strings.HasPrefix(parent, "fossils") {
-						name = parent + "/" + name + ".fsl"
-						name = name[len("fossils/"):]
-					} else {
-						name = parent + "/" + name
-						name = name[len("chunks/"):]
+		for len(parents) > 0 || activeWorkers > 0 {
+
+			if len(parents) > 0 && activeWorkers < storage.numberOfThreads {
+				parent := parents[0]
+				parents = parents[1:]
+				activeWorkers++
+				go func(parent string) {
+					pathID, ok := storage.findPathID(parent)
+					if !ok {
+						return
+					}
+					entries, err := storage.listFiles(threadIndex, pathID, true, true)
+					if err != nil {
+						errorChannel <- err
+						return
+					}
+
+					LOG_DEBUG("GCD_STORAGE", "Listing %s; %d items returned", parent, len(entries))
+
+					files := []string {}
+					sizes := []int64 {}
+					for _, entry := range entries {
+						if entry.MimeType != GCDDirectoryMimeType {
+							name := entry.Name
+							if strings.HasPrefix(parent, "fossils") {
+								name = parent + "/" + name + ".fsl"
+								name = name[len("fossils/"):]
+							} else {
+								name = parent + "/" + name
+								name = name[len("chunks/"):]
+							}
+							files = append(files, name)
+							sizes = append(sizes, entry.Size)
+						} else {
+							directoryChannel <- parent+"/"+entry.Name
+							storage.savePathID(parent+"/"+entry.Name, entry.Id)
+						}
+					}
+					lock.Lock()
+					allFiles = append(allFiles, files...)
+					allSizes = append(allSizes, sizes...)
+					lock.Unlock()
+					directoryChannel <- ""
+				} (parent)
+			}
+
+			if activeWorkers > 0 {
+				select {
+				case err := <- errorChannel:
+					return nil, nil, err
+				case directory := <- directoryChannel:
+					if directory == "" {
+						activeWorkers--
+					} else {
+						parents = append(parents, directory)
 					}
-					files = append(files, name)
-					sizes = append(sizes, entry.Size)
-				} else {
-					parents = append(parents, parent+"/"+entry.Name)
 				}
-				storage.savePathID(parent+"/"+entry.Name, entry.Id)
 			}
 		}
-		return files, sizes, nil
+
+		return allFiles, allSizes, nil
 	}
 
 }
@@ -452,7 +619,7 @@ func (storage *GCDStorage) DeleteFile(threadIndex int, filePath string) (err err
 	}
 
 	for {
-		err = storage.service.Files.Delete(fileID).Fields("id").Do()
+		err = storage.service.Files.Delete(fileID).SupportsAllDrives(true).Fields("id").Do()
 		if retry, err := storage.shouldRetry(threadIndex, err); err == nil && !retry {
 			storage.deletePathID(filePath)
 			return nil
@@ -474,9 +641,12 @@ func (storage *GCDStorage) MoveFile(threadIndex int, from string, to string) (er
 	from = storage.convertFilePath(from)
 	to = storage.convertFilePath(to)
 
-	fileID, ok := storage.findPathID(from)
-	if !ok {
-		return fmt.Errorf("Attempting to rename file %s with unknown id", from)
+	fileID, err := storage.getIDFromPath(threadIndex, from, false)
+	if err != nil {
+		return fmt.Errorf("Failed to retrieve the id of '%s': %v", from, err)
+	}
+	if fileID == "" {
+		return fmt.Errorf("The file '%s' to be moved does not exist", from)
 	}
 
 	fromParent := path.Dir(from)
@@ -495,7 +665,7 @@ func (storage *GCDStorage) MoveFile(threadIndex int, from string, to string) (er
 	}
 
 	for {
-		_, err = storage.service.Files.Update(fileID, nil).AddParents(toParentID).RemoveParents(fromParentID).Do()
+		_, err = storage.service.Files.Update(fileID, nil).SupportsAllDrives(true).AddParents(toParentID).RemoveParents(fromParentID).Do()
 		if retry, err := storage.shouldRetry(threadIndex, err); err == nil && !retry {
 			break
 		} else if retry {
@@ -505,8 +675,6 @@ func (storage *GCDStorage) MoveFile(threadIndex int, from string, to string) (er
 		}
 	}
 
-	storage.savePathID(to, storage.getPathID(from))
-	storage.deletePathID(from)
 	return nil
 }
 
@@ -539,21 +707,22 @@ func (storage *GCDStorage) CreateDirectory(threadIndex int, dir string) (err err
 	}
 	name := path.Base(dir)
 
-	file := &drive.File{
-		Name:     name,
-		MimeType: "application/vnd.google-apps.folder",
-		Parents:  []string{parentID},
-	}
+	var file *drive.File
 
 	for {
-		file, err = storage.service.Files.Create(file).Fields("id").Do()
+		file = &drive.File{
+			Name:     name,
+			MimeType: GCDDirectoryMimeType,
+			Parents:  []string{parentID},
+		}
+
+		file, err = storage.service.Files.Create(file).SupportsAllDrives(true).Fields("id").Do()
 		if retry, err := storage.shouldRetry(threadIndex, err); err == nil && !retry {
 			break
 		} else {
 
 			// Check if the directory has already been created by other thread
-			exist, _, _, newErr := storage.GetFileInfo(threadIndex, dir)
-			if newErr == nil && exist {
+			if _, ok := storage.findPathID(dir); ok {
 				return nil
 			}
 
@@ -577,42 +746,35 @@ func (storage *GCDStorage) GetFileInfo(threadIndex int, filePath string) (exist
 	filePath = storage.convertFilePath(filePath)
 
 	fileID, ok := storage.findPathID(filePath)
-	if !ok {
-		dir := path.Dir(filePath)
-		if dir == "." {
-			dir = ""
-		}
-		dirID, err := storage.getIDFromPath(threadIndex, dir, false)
-		if err != nil {
-			return false, false, 0, err
-		}
-		if dirID == "" {
-			return false, false, 0, nil
-		}
-
-		fileID, isDir, size, err = storage.listByName(threadIndex, dirID, path.Base(filePath))
-		if fileID != "" {
-			storage.savePathID(filePath, fileID)
-		}
-		return fileID != "", isDir, size, err
+	if ok {
+		// Only directories are saved in the case so this must be a directory
+		return true, true, 0, nil
 	}
 
-	for {
-		file, err := storage.service.Files.Get(fileID).Fields("id, mimeType").Do()
-		if retry, err := storage.shouldRetry(threadIndex, err); err == nil && !retry {
-			return true, file.MimeType == "application/vnd.google-apps.folder", file.Size, nil
-		} else if retry {
-			continue
-		} else {
-			return false, false, 0, err
-		}
+	dir := path.Dir(filePath)
+	if dir == "." {
+		dir = ""
 	}
+	dirID, err := storage.getIDFromPath(threadIndex, dir, false)
+	if err != nil {
+		return false, false, 0, err
+	}
+	if dirID == "" {
+		return false, false, 0, nil
+	}
+
+	fileID, isDir, size, err = storage.listByName(threadIndex, dirID, path.Base(filePath))
+	if fileID != "" && isDir {
+		storage.savePathID(filePath, fileID)
+	}
+	return fileID != "", isDir, size, err
+
 }
 
 // DownloadFile reads the file at 'filePath' into the chunk.
 func (storage *GCDStorage) DownloadFile(threadIndex int, filePath string, chunk *Chunk) (err error) {
 	// We never download the fossil so there is no need to convert the path
-	fileID, err := storage.getIDFromPath(threadIndex, filePath, false)
+	fileID, err := storage.getIDFromPath(threadIndex, storage.convertFilePath(filePath), false)
 	if err != nil {
 		return err
 	}
@@ -623,13 +785,22 @@ func (storage *GCDStorage) DownloadFile(threadIndex int, filePath string, chunk
 	var response *http.Response
 
 	for {
-		response, err = storage.service.Files.Get(fileID).Download()
-		if retry, err := storage.shouldRetry(threadIndex, err); err == nil && !retry {
+		// AcknowledgeAbuse(true) lets the download proceed even if GCD thinks that it contains malware.
+		// TODO: Should this prompt the user or log a warning?
+		req := storage.service.Files.Get(fileID).SupportsAllDrives(true)
+		if e, ok := err.(*googleapi.Error); ok {
+			if strings.Contains(err.Error(), "cannotDownloadAbusiveFile") || len(e.Errors) > 0 && e.Errors[0].Reason == "cannotDownloadAbusiveFile" {
+				LOG_WARN("GCD_STORAGE", "%s is marked as abusive, will download anyway.", filePath)
+				req = req.AcknowledgeAbuse(true)
+			}
+		}
+		response, err = req.Download()
+		if retry, retry_err := storage.shouldRetry(threadIndex, err); retry_err == nil && !retry {
 			break
 		} else if retry {
 			continue
 		} else {
-			return err
+			return retry_err
 		}
 	}
 
@@ -656,13 +827,13 @@ func (storage *GCDStorage) UploadFile(threadIndex int, filePath string, content
 
 	file := &drive.File{
 		Name:     path.Base(filePath),
-		MimeType: "application/octet-stream",
+		MimeType: GCDFileMimeType,
 		Parents:  []string{parentID},
 	}
 
 	for {
 		reader := CreateRateLimitedReader(content, storage.UploadRateLimit/storage.numberOfThreads)
-		_, err = storage.service.Files.Create(file).Media(reader).Fields("id").Do()
+		_, err = storage.service.Files.Create(file).SupportsAllDrives(true).Media(reader).Fields("id").Do()
 		if retry, err := storage.shouldRetry(threadIndex, err); err == nil && !retry {
 			break
 		} else if retry {

src/duplicacy_hubicclient.go

@@ -72,7 +72,7 @@ func NewHubicClient(tokenFile string) (*HubicClient, error) {
 					KeepAlive: 30 * time.Second,
 				}).Dial,
 				TLSHandshakeTimeout:   60 * time.Second,
-				ResponseHeaderTimeout: 30 * time.Second,
+				ResponseHeaderTimeout: 300 * time.Second,
 				ExpectContinueTimeout: 10 * time.Second,
 			},
 		},
@@ -82,7 +82,7 @@ func NewHubicClient(tokenFile string) (*HubicClient, error) {
 		CredentialLock: &sync.Mutex{},
 	}
 
-	err = client.RefreshToken()
+	err = client.RefreshToken(false)
 	if err != nil {
 		return nil, err
 	}
@@ -100,7 +100,7 @@ func (client *HubicClient) call(url string, method string, input interface{}, ex
 	var response *http.Response
 
 	backoff := 1
-	for i := 0; i < 8; i++ {
+	for i := 0; i < 11; i++ {
 
 		LOG_DEBUG("HUBIC_CALL", "%s %s", method, url)
 
@@ -151,6 +151,13 @@ func (client *HubicClient) call(url string, method string, input interface{}, ex
 
 		response, err = client.HTTPClient.Do(request)
 		if err != nil {
+			if url != HubicCredentialURL {
+				retryAfter := time.Duration((0.5 + rand.Float32()) * 1000.0 * float32(backoff))
+				LOG_INFO("HUBIC_CALL", "%s %s returned an error: %v; retry after %d milliseconds", method, url, err, retryAfter)
+				time.Sleep(retryAfter * time.Millisecond)
+				backoff *= 2
+				continue
+			}
 			return nil, 0, "", err
 		}
 
@@ -179,7 +186,7 @@ func (client *HubicClient) call(url string, method string, input interface{}, ex
 				return nil, 0, "", HubicError{Status: response.StatusCode, Message: "Authorization error when retrieving credentials"}
 			}
 
-			err = client.RefreshToken()
+			err = client.RefreshToken(true)
 			if err != nil {
 				return nil, 0, "", err
 			}
@@ -190,7 +197,13 @@ func (client *HubicClient) call(url string, method string, input interface{}, ex
 			}
 			continue
 		} else if response.StatusCode >= 500 && response.StatusCode < 600 {
-			retryAfter := time.Duration(rand.Float32() * 1000.0 * float32(backoff))
+			retryAfter := time.Duration((0.5 + rand.Float32()) * 1000.0 * float32(backoff))
+			LOG_INFO("HUBIC_RETRY", "Response status: %d; retry after %d milliseconds", response.StatusCode, retryAfter)
+			time.Sleep(retryAfter * time.Millisecond)
+			backoff *= 2
+			continue
+		} else if response.StatusCode == 408 {
+			retryAfter := time.Duration((0.5 + rand.Float32()) * 1000.0 * float32(backoff))
 			LOG_INFO("HUBIC_RETRY", "Response status: %d; retry after %d milliseconds", response.StatusCode, retryAfter)
 			time.Sleep(retryAfter * time.Millisecond)
 			backoff *= 2
@@ -203,11 +216,11 @@ func (client *HubicClient) call(url string, method string, input interface{}, ex
 	return nil, 0, "", fmt.Errorf("Maximum number of retries reached")
 }
 
-func (client *HubicClient) RefreshToken() (err error) {
+func (client *HubicClient) RefreshToken(force bool) (err error) {
 	client.TokenLock.Lock()
 	defer client.TokenLock.Unlock()
 
-	if client.Token.Valid() {
+	if !force && client.Token.Valid() {
 		return nil
 	}
 

src/duplicacy_hubicclient_test.go

@@ -1,149 +0,0 @@
-// Copyright (c) Acrosync LLC. All rights reserved.
-// Free for personal use and commercial trial
-// Commercial use requires per-user licenses available from https://duplicacy.com
-
-package duplicacy
-
-import (
-	"crypto/sha256"
-	"encoding/hex"
-	"fmt"
-	"io"
-	"testing"
-
-	crypto_rand "crypto/rand"
-	"math/rand"
-)
-
-func TestHubicClient(t *testing.T) {
-
-	hubicClient, err := NewHubicClient("hubic-token.json")
-	if err != nil {
-		t.Errorf("Failed to create the Hubic client: %v", err)
-		return
-	}
-
-	hubicClient.TestMode = true
-
-	existingFiles, err := hubicClient.ListEntries("")
-	for _, file := range existingFiles {
-		fmt.Printf("name: %s, isDir: %t\n", file.Name, file.Type == "application/directory")
-	}
-
-	testExists, _, _, err := hubicClient.GetFileInfo("test")
-	if err != nil {
-		t.Errorf("Failed to list the test directory: %v", err)
-		return
-	}
-	if !testExists {
-		err = hubicClient.CreateDirectory("test")
-		if err != nil {
-			t.Errorf("Failed to create the test directory: %v", err)
-			return
-		}
-	}
-
-	test1Exists, _, _, err := hubicClient.GetFileInfo("test/test1")
-	if err != nil {
-		t.Errorf("Failed to list the test1 directory: %v", err)
-		return
-	}
-	if !test1Exists {
-		err = hubicClient.CreateDirectory("test/test1")
-		if err != nil {
-			t.Errorf("Failed to create the test1 directory: %v", err)
-			return
-		}
-	}
-
-	test2Exists, _, _, err := hubicClient.GetFileInfo("test/test2")
-	if err != nil {
-		t.Errorf("Failed to list the test2 directory: %v", err)
-		return
-	}
-	if !test2Exists {
-		err = hubicClient.CreateDirectory("test/test2")
-		if err != nil {
-			t.Errorf("Failed to create the test2 directory: %v", err)
-			return
-		}
-	}
-
-	numberOfFiles := 20
-	maxFileSize := 64 * 1024
-
-	for i := 0; i < numberOfFiles; i++ {
-		content := make([]byte, rand.Int()%maxFileSize+1)
-		_, err = crypto_rand.Read(content)
-		if err != nil {
-			t.Errorf("Error generating random content: %v", err)
-			return
-		}
-
-		hasher := sha256.New()
-		hasher.Write(content)
-		filename := hex.EncodeToString(hasher.Sum(nil))
-
-		fmt.Printf("file: %s\n", filename)
-
-		err = hubicClient.UploadFile("test/test1/"+filename, content, 100)
-		if err != nil {
-			/*if e, ok := err.(ACDError); !ok || e.Status != 409 */ {
-				t.Errorf("Failed to upload the file %s: %v", filename, err)
-				return
-			}
-		}
-	}
-
-	entries, err := hubicClient.ListEntries("test/test1")
-	if err != nil {
-		t.Errorf("Error list randomly generated files: %v", err)
-		return
-	}
-
-	for _, entry := range entries {
-
-		exists, isDir, size, err := hubicClient.GetFileInfo("test/test1/" + entry.Name)
-		fmt.Printf("%s exists: %t, isDir: %t, size: %d, err: %v\n", "test/test1/"+entry.Name, exists, isDir, size, err)
-
-		err = hubicClient.MoveFile("test/test1/"+entry.Name, "test/test2/"+entry.Name)
-		if err != nil {
-			t.Errorf("Failed to move %s: %v", entry.Name, err)
-			return
-		}
-	}
-
-	entries, err = hubicClient.ListEntries("test/test2")
-	if err != nil {
-		t.Errorf("Error list randomly generated files: %v", err)
-		return
-	}
-
-	for _, entry := range entries {
-		readCloser, _, err := hubicClient.DownloadFile("test/test2/" + entry.Name)
-		if err != nil {
-			t.Errorf("Error downloading file %s: %v", entry.Name, err)
-			return
-		}
-
-		hasher := sha256.New()
-		io.Copy(hasher, readCloser)
-		hash := hex.EncodeToString(hasher.Sum(nil))
-
-		if hash != entry.Name {
-			t.Errorf("File %s, hash %s", entry.Name, hash)
-		}
-
-		readCloser.Close()
-	}
-
-	for _, entry := range entries {
-
-		err = hubicClient.DeleteFile("test/test2/" + entry.Name)
-		if err != nil {
-			t.Errorf("Failed to delete the file %s: %v", entry.Name, err)
-			return
-		}
-	}
-
-}

src/duplicacy_hubicstorage.go

@@ -106,17 +106,19 @@ func (storage *HubicStorage) ListFiles(threadIndex int, dir string) ([]string, [
 	} else {
 		files := []string{}
 		sizes := []int64{}
-		entries, err := storage.client.ListEntries(storage.storageDir + "/chunks")
+		entries, err := storage.client.ListEntries(storage.storageDir + "/" + dir)
 		if err != nil {
 			return nil, nil, err
 		}
 
 		for _, entry := range entries {
 			if entry.Type == "application/directory" {
-				continue
+				files = append(files, entry.Name+"/")
+				sizes = append(sizes, 0)
+			} else {
+				files = append(files, entry.Name)
+				sizes = append(sizes, entry.Size)
 			}
-			files = append(files, entry.Name)
-			sizes = append(sizes, entry.Size)
 		}
 		return files, sizes, nil
 	}

src/duplicacy_keyring_windows.go

@@ -136,6 +136,16 @@ func keyringSet(key string, value string) bool {
 	if value == "" {
 		keyring[key] = nil
 	} else {
+
+		// Check if the value to be set is the same as the existing one
+		existingEncryptedValue := keyring[key]
+		if len(existingEncryptedValue) > 0 {
+			existingValue, err := keyringDecrypt(existingEncryptedValue)
+			if err == nil && string(existingValue) == value {
+				return true
+			}
+		}
+
 		encryptedValue, err := keyringEncrypt([]byte(value))
 		if err != nil {
 			LOG_DEBUG("KEYRING_ENCRYPT", "Failed to encrypt the value: %v", err)

src/duplicacy_log.go

@@ -7,10 +7,12 @@ package duplicacy
 import (
 	"fmt"
 	"os"
+	"log"
 	"runtime/debug"
 	"sync"
 	"testing"
 	"time"
+	"regexp"
 )
 
 const (
@@ -43,6 +45,13 @@ func setTestingT(t *testing.T) {
 	testingT = t
 }
 
+// Contains the ids of logs that won't be displayed
+var suppressedLogs map[string]bool = map[string]bool{}
+
+func SuppressLog(id string) {
+	suppressedLogs[id] = true
+}
+
 func getLevelName(level int) string {
 	switch level {
 	case DEBUG:
@@ -98,6 +107,15 @@ func LOG_ERROR(logID string, format string, v ...interface{}) {
 	logf(ERROR, logID, format, v...)
 }
 
+func LOG_WERROR(isWarning bool, logID string, format string, v ...interface{}) {
+	if isWarning {
+		logf(WARN, logID, format, v...)
+	} else {
+		logf(ERROR, logID, format, v...)
+	}
+}
+
+
 func LOG_FATAL(logID string, format string, v ...interface{}) {
 	logf(FATAL, logID, format, v...)
 }
@@ -129,7 +147,7 @@ func logf(level int, logID string, format string, v ...interface{}) {
 	// fmt.Printf("%s %s %s %s\n", now.Format("2006-01-02 15:04:05.000"), getLevelName(level), logID, message)
 
 	if testingT != nil {
-		if level < WARN {
+		if level <= WARN {
 			if level >= loggingLevel {
 				testingT.Logf("%s %s %s %s\n",
 					now.Format("2006-01-02 15:04:05.000"), getLevelName(level), logID, message)
@@ -143,6 +161,12 @@ func logf(level int, logID string, format string, v ...interface{}) {
 		defer logMutex.Unlock()
 
 		if level >= loggingLevel {
+			if level <= ERROR && len(suppressedLogs) > 0 {
+				if _, found := suppressedLogs[logID]; found {
+					return
+				}
+			}
+
 			if printLogHeader {
 				fmt.Printf("%s %s %s %s\n",
 					now.Format("2006-01-02 15:04:05.000"), getLevelName(level), logID, message)
@@ -161,6 +185,32 @@ func logf(level int, logID string, format string, v ...interface{}) {
 	}
 }
 
+// Set up logging for libraries that Duplicacy depends on.  They can call 'log.Printf("[ID] message")'
+// to produce logs in Duplicacy's format
+type Logger struct {
+	formatRegex *regexp.Regexp
+}
+
+func (logger *Logger) Write(line []byte) (n int, err error) {
+	n = len(line)
+	for len(line) > 0 && line[len(line) - 1] == '\n' {
+		line = line[:len(line) - 1]
+	}
+	matched := logger.formatRegex.FindStringSubmatch(string(line))
+	if matched != nil {
+		LOG_INFO(matched[1], "%s", matched[2])
+	} else {
+		LOG_INFO("LOG_DEFAULT", "%s", line)
+	}
+
+    return
+}
+
+func init() {
+	log.SetFlags(0)
+	log.SetOutput(&Logger{ formatRegex: regexp.MustCompile(`^\[(.+)\]\s*(.+)`) })
+}
+
 const (
 	duplicacyExitCode = 100
 	otherExitCode     = 101

src/duplicacy_oneclient.go

@@ -5,6 +5,7 @@
 package duplicacy
 
 import (
+	"context"
 	"bytes"
 	"encoding/json"
 	"fmt"
@@ -13,8 +14,10 @@ import (
 	"math/rand"
 	"net/http"
 	"strings"
+	"strconv"
 	"sync"
 	"time"
+	"path/filepath"
 
 	"golang.org/x/oauth2"
 )
@@ -32,21 +35,23 @@ type OneDriveErrorResponse struct {
 	Error OneDriveError `json:"error"`
 }
 
-var OneDriveRefreshTokenURL = "https://duplicacy.com/one_refresh"
-var OneDriveAPIURL = "https://api.onedrive.com/v1.0"
-
 type OneDriveClient struct {
 	HTTPClient *http.Client
 
 	TokenFile string
 	Token     *oauth2.Token
+	OAConfig  *oauth2.Config
 	TokenLock *sync.Mutex
 
 	IsConnected bool
 	TestMode    bool
+
+	IsBusiness bool
+	RefreshTokenURL string
+	APIURL string
 }
 
-func NewOneDriveClient(tokenFile string) (*OneDriveClient, error) {
+func NewOneDriveClient(tokenFile string, isBusiness bool, client_id string, client_secret string, drive_id string) (*OneDriveClient, error) {
 
 	description, err := ioutil.ReadFile(tokenFile)
 	if err != nil {
@@ -62,9 +67,38 @@ func NewOneDriveClient(tokenFile string) (*OneDriveClient, error) {
 		HTTPClient: http.DefaultClient,
 		TokenFile:  tokenFile,
 		Token:      token,
+		OAConfig:	nil,
 		TokenLock:  &sync.Mutex{},
+		IsBusiness: isBusiness,
 	}
 
+	if (client_id != "") {
+	    oneOauthConfig := oauth2.Config{
+	        ClientID: 		client_id,
+	        ClientSecret: 	client_secret,
+	        Scopes:       	[]string{"Files.ReadWrite", "offline_access"},
+	        Endpoint: oauth2.Endpoint{
+	            AuthURL:  "https://login.microsoftonline.com/common/oauth2/v2.0/authorize",
+	            TokenURL: "https://login.microsoftonline.com/common/oauth2/v2.0/token",
+	        },
+	    }
+
+		client.OAConfig = &oneOauthConfig
+	}
+
+	if isBusiness {
+		client.RefreshTokenURL = "https://duplicacy.com/odb_refresh"
+		client.APIURL = "https://graph.microsoft.com/v1.0/me/drive"
+		if drive_id != "" {
+			client.APIURL = "https://graph.microsoft.com/v1.0/drives/"+drive_id
+		}
+	} else {
+		client.RefreshTokenURL = "https://duplicacy.com/one_refresh"
+		client.APIURL = "https://api.onedrive.com/v1.0/drive"
+	}
+
+	client.RefreshToken(false)
+
 	return client, nil
 }
 
@@ -73,7 +107,7 @@ func (client *OneDriveClient) call(url string, method string, input interface{},
 	var response *http.Response
 
 	backoff := 1
-	for i := 0; i < 8; i++ {
+	for i := 0; i < 12; i++ {
 
 		LOG_DEBUG("ONEDRIVE_CALL", "%s %s", method, url)
 
@@ -104,9 +138,10 @@ func (client *OneDriveClient) call(url string, method string, input interface{},
 
 		if reader, ok := inputReader.(*RateLimitedReader); ok {
 			request.ContentLength = reader.Length()
+			request.Header.Set("Content-Range", fmt.Sprintf("bytes 0-%d/%d", reader.Length() - 1, reader.Length()))
 		}
 
-		if url != OneDriveRefreshTokenURL {
+		if url != client.RefreshTokenURL {
 			client.TokenLock.Lock()
 			request.Header.Set("Authorization", "Bearer "+client.Token.AccessToken)
 			client.TokenLock.Unlock()
@@ -115,6 +150,8 @@ func (client *OneDriveClient) call(url string, method string, input interface{},
 			request.Header.Set("Content-Type", contentType)
 		}
 
+		request.Header.Set("User-Agent", "ISV|Acrosync|Duplicacy/2.0")
+
 		response, err = client.HTTPClient.Do(request)
 		if err != nil {
 			if client.IsConnected {
@@ -131,6 +168,9 @@ func (client *OneDriveClient) call(url string, method string, input interface{},
 					time.Sleep(retryAfter * time.Millisecond)
 				}
 				backoff *= 2
+				if backoff > 256 {
+					backoff = 256
+				}
 				continue
 			}
 			return nil, 0, err
@@ -150,20 +190,32 @@ func (client *OneDriveClient) call(url string, method string, input interface{},
 
 		if response.StatusCode == 401 {
 
-			if url == OneDriveRefreshTokenURL {
+			if url == client.RefreshTokenURL {
 				return nil, 0, OneDriveError{Status: response.StatusCode, Message: "Authorization error when refreshing token"}
 			}
 
-			err = client.RefreshToken()
+			err = client.RefreshToken(true)
 			if err != nil {
 				return nil, 0, err
 			}
 			continue
+		} else if response.StatusCode == 409 {
+			return nil, 0, OneDriveError{Status: response.StatusCode, Message: "Conflict"}
 		} else if response.StatusCode > 401 && response.StatusCode != 404 {
-			retryAfter := time.Duration(rand.Float32() * 1000.0 * float32(backoff))
-			LOG_INFO("ONEDRIVE_RETRY", "Response code: %d; retry after %d milliseconds", response.StatusCode, retryAfter)
-			time.Sleep(retryAfter * time.Millisecond)
+			delay := int((rand.Float32() * 0.5 + 0.5) * 1000.0 * float32(backoff))
+			if backoffList, found := response.Header["Retry-After"]; found && len(backoffList) > 0 {
+				retryAfter, _ := strconv.Atoi(backoffList[0])
+				if retryAfter * 1000 > delay {
+					delay = retryAfter * 1000
+				}
+			}
+
+			LOG_INFO("ONEDRIVE_RETRY", "Response code: %d; retry after %d milliseconds", response.StatusCode, delay)
+			time.Sleep(time.Duration(delay) * time.Millisecond)
 			backoff *= 2
+			if backoff > 256 {
+				backoff = 256
+			}
 			continue
 		} else {
 			if err := json.NewDecoder(response.Body).Decode(errorResponse); err != nil {
@@ -178,23 +230,33 @@ func (client *OneDriveClient) call(url string, method string, input interface{},
 	return nil, 0, fmt.Errorf("Maximum number of retries reached")
 }
 
-func (client *OneDriveClient) RefreshToken() (err error) {
+func (client *OneDriveClient) RefreshToken(force bool) (err error) {
 	client.TokenLock.Lock()
 	defer client.TokenLock.Unlock()
 
-	if client.Token.Valid() {
+	if !force && client.Token.Valid() {
 		return nil
 	}
 
-	readCloser, _, err := client.call(OneDriveRefreshTokenURL, "POST", client.Token, "")
-	if err != nil {
-		return fmt.Errorf("failed to refresh the access token: %v", err)
-	}
+	if (client.OAConfig == nil) {
+		readCloser, _, err := client.call(client.RefreshTokenURL, "POST", client.Token, "")
+		if err != nil {
+			return fmt.Errorf("failed to refresh the access token: %v", err)
+		}
 
-	defer readCloser.Close()
+		defer readCloser.Close()
 
-	if err = json.NewDecoder(readCloser).Decode(client.Token); err != nil {
-		return err
+		if err = json.NewDecoder(readCloser).Decode(client.Token); err != nil {
+			return err
+		}
+	} else {
+		ctx := context.Background()
+		tokenSource := client.OAConfig.TokenSource(ctx, client.Token)
+	    token, err := tokenSource.Token()
+	    if err != nil {
+	        return fmt.Errorf("failed to refresh the access token: %v", err)
+	    }
+	    client.Token = token
 	}
 
 	description, err := json.Marshal(client.Token)
@@ -226,9 +288,9 @@ func (client *OneDriveClient) ListEntries(path string) ([]OneDriveEntry, error)
 
 	entries := []OneDriveEntry{}
 
-	url := OneDriveAPIURL + "/drive/root:/" + path + ":/children"
+	url := client.APIURL + "/root:/" + path + ":/children"
 	if path == "" {
-		url = OneDriveAPIURL + "/drive/root/children"
+		url = client.APIURL + "/root/children"
 	}
 	if client.TestMode {
 		url += "?top=8"
@@ -264,7 +326,8 @@ func (client *OneDriveClient) ListEntries(path string) ([]OneDriveEntry, error)
 
 func (client *OneDriveClient) GetFileInfo(path string) (string, bool, int64, error) {
 
-	url := OneDriveAPIURL + "/drive/root:/" + path
+	url := client.APIURL + "/root:/" + path
+	if path == "" { url = client.APIURL + "/root" }
 	url += "?select=id,name,size,folder"
 
 	readCloser, _, err := client.call(url, "GET", 0, "")
@@ -289,28 +352,95 @@ func (client *OneDriveClient) GetFileInfo(path string) (string, bool, int64, err
 
 func (client *OneDriveClient) DownloadFile(path string) (io.ReadCloser, int64, error) {
 
-	url := OneDriveAPIURL + "/drive/items/root:/" + path + ":/content"
+	url := client.APIURL + "/items/root:/" + path + ":/content"
 
 	return client.call(url, "GET", 0, "")
 }
 
 func (client *OneDriveClient) UploadFile(path string, content []byte, rateLimit int) (err error) {
 
-	url := OneDriveAPIURL + "/drive/root:/" + path + ":/content"
+	// Upload file using the simple method; this is only possible for OneDrive Personal or if the file
+	// is smaller than 4MB for OneDrive Business
+	if !client.IsBusiness || (client.TestMode && rand.Int() % 2 == 0) {
+		url := client.APIURL + "/root:/" + path + ":/content"
 
-	readCloser, _, err := client.call(url, "PUT", CreateRateLimitedReader(content, rateLimit), "application/octet-stream")
+		readCloser, _, err := client.call(url, "PUT", CreateRateLimitedReader(content, rateLimit), "application/octet-stream")
+		if err != nil {
+			return err
+		}
 
+		readCloser.Close()
+		return nil
+	}
+
+	// For large files, create an upload session first
+	uploadURL, err := client.CreateUploadSession(path)
 	if err != nil {
 		return err
 	}
 
+	return client.UploadFileSession(uploadURL, content, rateLimit)
+}
+
+func (client *OneDriveClient) CreateUploadSession(path string) (uploadURL string, err error) {
+
+	type CreateUploadSessionItem struct {
+		ConflictBehavior string `json:"@microsoft.graph.conflictBehavior"`
+		Name string `json:"name"`
+	}
+
+	input := map[string]interface{} {
+		"item": CreateUploadSessionItem {
+			ConflictBehavior: "replace",
+			Name: filepath.Base(path),
+		},
+	}
+
+	readCloser, _, err := client.call(client.APIURL + "/root:/" + path + ":/createUploadSession", "POST", input, "application/json")
+	if err != nil {
+		return "", err
+	}
+
+	type CreateUploadSessionOutput struct {
+		UploadURL string `json:"uploadUrl"`
+	}
+
+	output := &CreateUploadSessionOutput{}
+
+	if err = json.NewDecoder(readCloser).Decode(&output); err != nil {
+		return "", err
+	}
+
+	readCloser.Close()
+	return output.UploadURL, nil
+}
+
+func (client *OneDriveClient) UploadFileSession(uploadURL string, content []byte, rateLimit int) (err error) {
+
+	readCloser, _, err := client.call(uploadURL, "PUT", CreateRateLimitedReader(content, rateLimit), "")
+	if err != nil {
+		return err
+	}
+	type UploadFileSessionOutput struct {
+		Size int `json:"size"`
+	}
+	output := &UploadFileSessionOutput{}
+
+	if err = json.NewDecoder(readCloser).Decode(&output); err != nil {
+		return fmt.Errorf("Failed to complete the file upload session: %v", err)
+	}
+
+	if output.Size != len(content) {
+		return fmt.Errorf("Uploaded %d bytes out of %d bytes", output.Size, len(content))
+	}
+
 	readCloser.Close()
 	return nil
 }
 
 func (client *OneDriveClient) DeleteFile(path string) error {
 
-	url := OneDriveAPIURL + "/drive/root:/" + path
+	url := client.APIURL + "/root:/" + path
 
 	readCloser, _, err := client.call(url, "DELETE", 0, "")
 	if err != nil {
@@ -323,16 +453,30 @@ func (client *OneDriveClient) DeleteFile(path string) error {
 
 func (client *OneDriveClient) MoveFile(path string, parent string) error {
 
-	url := OneDriveAPIURL + "/drive/root:/" + path
+	url := client.APIURL + "/root:/" + path
 
 	parentReference := make(map[string]string)
-	parentReference["path"] = "/drive/root:/" + parent
+	parentReference["path"] = "/root:/" + parent
 
 	parameters := make(map[string]interface{})
 	parameters["parentReference"] = parentReference
 
 	readCloser, _, err := client.call(url, "PATCH", parameters, "application/json")
 	if err != nil {
+		if e, ok := err.(OneDriveError); ok && e.Status == 400 {
+			// The destination directory doesn't exist; trying to create it...
+			dir := filepath.Dir(parent)
+			if dir == "." {
+				dir = ""
+			}
+			client.CreateDirectory(dir, filepath.Base(parent))
+			readCloser, _, err = client.call(url, "PATCH", parameters, "application/json")
+			if err != nil {
+				return nil
+			}
+
+		}
+
 		return err
 	}
 
@@ -342,24 +486,29 @@ func (client *OneDriveClient) MoveFile(path string, parent string) error {
 
 func (client *OneDriveClient) CreateDirectory(path string, name string) error {
 
-	url := OneDriveAPIURL + "/root/children"
+	url := client.APIURL + "/root/children"
 
 	if path != "" {
 
-		parentID, isDir, _, err := client.GetFileInfo(path)
+		pathID, isDir, _, err := client.GetFileInfo(path)
 		if err != nil {
 			return err
 		}
 
-		if parentID == "" {
-			return fmt.Errorf("The path '%s' does not exist", path)
+		if pathID == "" {
+			dir := filepath.Dir(path)
+			if dir != "." {
+				// The parent directory doesn't exist; trying to create it...
+				client.CreateDirectory(dir, filepath.Base(path))
+				isDir = true
+			}
 		}
 
 		if !isDir {
 			return fmt.Errorf("The path '%s' is not a directory", path)
 		}
 
-		url = OneDriveAPIURL + "/drive/items/" + parentID + "/children"
+		url = client.APIURL + "/root:/" + path + ":/children"
 	}
 
 	parameters := make(map[string]interface{})
@@ -368,6 +517,11 @@ func (client *OneDriveClient) CreateDirectory(path string, name string) error {
 
 	readCloser, _, err := client.call(url, "POST", parameters, "application/json")
 	if err != nil {
+		if e, ok := err.(OneDriveError); ok && e.Status == 409 {
+			// This error usually means the directory already exists
+			LOG_TRACE("ONEDRIVE_MKDIR", "The directory '%s/%s' already exists", path, name)
+			return nil
+		}
 		return err
 	}
 

src/duplicacy_oneclient_test.go

@@ -1,145 +0,0 @@
-// Copyright (c) Acrosync LLC. All rights reserved.
-// Free for personal use and commercial trial
-// Commercial use requires per-user licenses available from https://duplicacy.com
-
-package duplicacy
-
-import (
-	"crypto/sha256"
-	"encoding/hex"
-	"fmt"
-	"io"
-	"testing"
-
-	crypto_rand "crypto/rand"
-	"math/rand"
-)
-
-func TestOneDriveClient(t *testing.T) {
-
-	oneDriveClient, err := NewOneDriveClient("one-token.json")
-	if err != nil {
-		t.Errorf("Failed to create the OneDrive client: %v", err)
-		return
-	}
-
-	oneDriveClient.TestMode = true
-
-	existingFiles, err := oneDriveClient.ListEntries("")
-	for _, file := range existingFiles {
-		fmt.Printf("name: %s, isDir: %t\n", file.Name, len(file.Folder) != 0)
-	}
-
-	testID, _, _, err := oneDriveClient.GetFileInfo("test")
-	if err != nil {
-		t.Errorf("Failed to list the test directory: %v", err)
-		return
-	}
-	if testID == "" {
-		err = oneDriveClient.CreateDirectory("", "test")
-		if err != nil {
-			t.Errorf("Failed to create the test directory: %v", err)
-			return
-		}
-	}
-
-	test1ID, _, _, err := oneDriveClient.GetFileInfo("test/test1")
-	if err != nil {
-		t.Errorf("Failed to list the test1 directory: %v", err)
-		return
-	}
-	if test1ID == "" {
-		err = oneDriveClient.CreateDirectory("test", "test1")
-		if err != nil {
-			t.Errorf("Failed to create the test1 directory: %v", err)
-			return
-		}
-	}
-
-	test2ID, _, _, err := oneDriveClient.GetFileInfo("test/test2")
-	if err != nil {
-		t.Errorf("Failed to list the test2 directory: %v", err)
-		return
-	}
-	if test2ID == "" {
-		err = oneDriveClient.CreateDirectory("test", "test2")
-		if err != nil {
-			t.Errorf("Failed to create the test2 directory: %v", err)
-			return
-		}
-	}
-
-	numberOfFiles := 20
-	maxFileSize := 64 * 1024
-
-	for i := 0; i < numberOfFiles; i++ {
-		content := make([]byte, rand.Int()%maxFileSize+1)
-		_, err = crypto_rand.Read(content)
-		if err != nil {
-			t.Errorf("Error generating random content: %v", err)
-			return
-		}
-
-		hasher := sha256.New()
-		hasher.Write(content)
-		filename := hex.EncodeToString(hasher.Sum(nil))
-
-		fmt.Printf("file: %s\n", filename)
-
-		err = oneDriveClient.UploadFile("test/test1/"+filename, content, 100)
-		if err != nil {
-			/*if e, ok := err.(ACDError); !ok || e.Status != 409 */ {
-				t.Errorf("Failed to upload the file %s: %v", filename, err)
-				return
-			}
-		}
-	}
-
-	entries, err := oneDriveClient.ListEntries("test/test1")
-	if err != nil {
-		t.Errorf("Error list randomly generated files: %v", err)
-		return
-	}
-
-	for _, entry := range entries {
-		err = oneDriveClient.MoveFile("test/test1/"+entry.Name, "test/test2")
-		if err != nil {
-			t.Errorf("Failed to move %s: %v", entry.Name, err)
-			return
-		}
-	}
-
-	entries, err = oneDriveClient.ListEntries("test/test2")
-	if err != nil {
-		t.Errorf("Error list randomly generated files: %v", err)
-		return
-	}
-
-	for _, entry := range entries {
-		readCloser, _, err := oneDriveClient.DownloadFile("test/test2/" + entry.Name)
-		if err != nil {
-			t.Errorf("Error downloading file %s: %v", entry.Name, err)
-			return
-		}
-
-		hasher := sha256.New()
-		io.Copy(hasher, readCloser)
-		hash := hex.EncodeToString(hasher.Sum(nil))
-
-		if hash != entry.Name {
-			t.Errorf("File %s, hash %s", entry.Name, hash)
-		}
-
-		readCloser.Close()
-	}
-
-	for _, entry := range entries {
-
-		err = oneDriveClient.DeleteFile("test/test2/" + entry.Name)
-		if err != nil {
-			t.Errorf("Failed to delete the file %s: %v", entry.Name, err)
-			return
-		}
-	}
-
-}

src/duplicacy_onestorage.go

@@ -19,13 +19,13 @@ type OneDriveStorage struct {
 }
 
 // CreateOneDriveStorage creates an OneDrive storage object.
-func CreateOneDriveStorage(tokenFile string, storagePath string, threads int) (storage *OneDriveStorage, err error) {
+func CreateOneDriveStorage(tokenFile string, isBusiness bool, storagePath string, threads int, client_id string, client_secret string, drive_id string) (storage *OneDriveStorage, err error) {
 
 	for len(storagePath) > 0 && storagePath[len(storagePath)-1] == '/' {
 		storagePath = storagePath[:len(storagePath)-1]
 	}
 
-	client, err := NewOneDriveClient(tokenFile)
+	client, err := NewOneDriveClient(tokenFile, isBusiness, client_id, client_secret, drive_id)
 	if err != nil {
 		return nil, err
 	}
@@ -80,6 +80,7 @@ func (storage *OneDriveStorage) convertFilePath(filePath string) string {
 
 // ListFiles return the list of files and subdirectories under 'dir' (non-recursively)
 func (storage *OneDriveStorage) ListFiles(threadIndex int, dir string) ([]string, []int64, error) {
+
 	for len(dir) > 0 && dir[len(dir)-1] == '/' {
 		dir = dir[:len(dir)-1]
 	}
@@ -97,7 +98,7 @@ func (storage *OneDriveStorage) ListFiles(threadIndex int, dir string) ([]string
 			}
 		}
 		return subDirs, nil, nil
-	} else if strings.HasPrefix(dir, "snapshots/") {
+	} else if strings.HasPrefix(dir, "snapshots/") || strings.HasPrefix(dir, "benchmark") {
 		entries, err := storage.client.ListEntries(storage.storageDir + "/" + dir)
 		if err != nil {
 			return nil, nil, err

src/duplicacy_preference.go

@@ -17,12 +17,16 @@ import (
 type Preference struct {
 	Name              string            `json:"name"`
 	SnapshotID        string            `json:"id"`
+	RepositoryPath    string            `json:"repository"`
 	StorageURL        string            `json:"storage"`
 	Encrypted         bool              `json:"encrypted"`
 	BackupProhibited  bool              `json:"no_backup"`
 	RestoreProhibited bool              `json:"no_restore"`
 	DoNotSavePassword bool              `json:"no_save_password"`
+	NobackupFile      string            `json:"nobackup_file"`
 	Keys              map[string]string `json:"keys"`
+	FiltersFile       string            `json:"filters"`
+	ExcludeByAttribute bool             `json:"exclude_by_attribute"`
 }
 
 var preferencePath string

src/duplicacy_s3storage.go

@@ -2,6 +2,11 @@
 // Free for personal use and commercial trial
 // Commercial use requires per-user licenses available from https://duplicacy.com
 
+// NOTE: The code in the Wasabi storage module relies on all functions
+// in this one except MoveFile(), IsMoveFileImplemented() and
+// IsStrongConsistent().  Changes to the API here will need to be
+// reflected there.
+
 package duplicacy
 
 import (
@@ -205,7 +210,7 @@ func (storage *S3Storage) DownloadFile(threadIndex int, filePath string, chunk *
 
 	defer output.Body.Close()
 
-	_, err = RateLimitedCopy(chunk, output.Body, storage.DownloadRateLimit/len(storage.bucket))
+	_, err = RateLimitedCopy(chunk, output.Body, storage.DownloadRateLimit/storage.numberOfThreads)
 	return err
 
 }
@@ -220,7 +225,7 @@ func (storage *S3Storage) UploadFile(threadIndex int, filePath string, content [
 			Bucket:      aws.String(storage.bucket),
 			Key:         aws.String(storage.storageDir + filePath),
 			ACL:         aws.String(s3.ObjectCannedACLPrivate),
-			Body:        CreateRateLimitedReader(content, storage.UploadRateLimit/len(storage.bucket)),
+			Body:        CreateRateLimitedReader(content, storage.UploadRateLimit/storage.numberOfThreads),
 			ContentType: aws.String("application/duplicacy"),
 		}
 
@@ -232,8 +237,6 @@ func (storage *S3Storage) UploadFile(threadIndex int, filePath string, content [
 		LOG_INFO("S3_RETRY", "Retrying on %s: %v", reflect.TypeOf(err), err)
 		attempts += 1
 	}
-
-	return err
 }
 
 // If a local snapshot cache is needed for the storage to avoid downloading/uploading chunks too often when

src/duplicacy_sambastorage.go

@@ -0,0 +1,250 @@
+// Copyright (c) Acrosync LLC. All rights reserved.
+// Free for personal use and commercial trial
+// Commercial use requires per-user licenses available from https://duplicacy.com
+
+package duplicacy
+
+import (
+	"io"
+	"os"
+	"fmt"
+	"net"
+	"path"
+	"time"
+	"strings"
+	"syscall"
+	"math/rand"
+
+	"github.com/hirochachacha/go-smb2"
+)
+
+// SambaStorage is a local on-disk file storage implementing the Storage interface.
+type SambaStorage struct {
+	StorageBase
+
+	share           *smb2.Share
+	storageDir      string
+	numberOfThreads int
+}
+
+// CreateSambaStorage creates a file storage.
+func CreateSambaStorage(server string, port int, username string, password string, shareName string, storageDir string, threads int) (storage *SambaStorage, err error) {
+
+	connection, err := net.Dial("tcp", fmt.Sprintf("%s:%d", server, port))
+	if err != nil {
+		return nil, err
+	}
+
+	dialer := &smb2.Dialer{
+		Initiator: &smb2.NTLMInitiator{
+			User:     username,
+			Password: password,
+		},
+	}
+
+	client, err := dialer.Dial(connection)
+	if err != nil {
+		return nil, err
+	}
+
+	share, err := client.Mount(shareName)
+	if err != nil {
+		return nil, err
+	}
+
+	// Random number fo generating the temporary chunk file suffix.
+	rand.Seed(time.Now().UnixNano())
+
+	storage = &SambaStorage{
+		share:           share,
+		numberOfThreads: threads,
+	}
+
+	exist, isDir, _, err := storage.GetFileInfo(0, storageDir)
+	if err != nil {
+		return nil, fmt.Errorf("Failed to check the storage path %s: %v", storageDir, err)
+	}
+
+	if !exist {
+		return nil, fmt.Errorf("The storage path %s does not exist", storageDir)
+	}
+
+	if !isDir {
+		return nil, fmt.Errorf("The storage path %s is not a directory", storageDir)
+	}
+
+	storage.storageDir = storageDir
+	storage.DerivedStorage = storage
+	storage.SetDefaultNestingLevels([]int{2, 3}, 2)
+	return storage, nil
+}
+
+// ListFiles return the list of files and subdirectories under 'dir' (non-recursively).
+func (storage *SambaStorage) ListFiles(threadIndex int, dir string) (files []string, sizes []int64, err error) {
+
+	fullPath := path.Join(storage.storageDir, dir)
+
+	list, err := storage.share.ReadDir(fullPath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, nil, nil
+		}
+		return nil, nil, err
+	}
+
+	for _, f := range list {
+		name := f.Name()
+		if (f.IsDir() || f.Mode() & os.ModeSymlink != 0) && name[len(name)-1] != '/' {
+			name += "/"
+		}
+		files = append(files, name)
+		sizes = append(sizes, f.Size())
+	}
+
+	return files, sizes, nil
+}
+
+// DeleteFile deletes the file or directory at 'filePath'.
+func (storage *SambaStorage) DeleteFile(threadIndex int, filePath string) (err error) {
+	err = storage.share.Remove(path.Join(storage.storageDir, filePath))
+	if err == nil || os.IsNotExist(err) {
+		return nil
+	} else {
+		return err
+	}
+}
+
+// MoveFile renames the file.
+func (storage *SambaStorage) MoveFile(threadIndex int, from string, to string) (err error) {
+	return storage.share.Rename(path.Join(storage.storageDir, from), path.Join(storage.storageDir, to))
+}
+
+// CreateDirectory creates a new directory.
+func (storage *SambaStorage) CreateDirectory(threadIndex int, dir string) (err error) {
+	fmt.Printf("Creating directory %s\n", dir)
+	err = storage.share.Mkdir(path.Join(storage.storageDir, dir), 0744)
+	if err != nil && os.IsExist(err) {
+		return nil
+	} else {
+		return err
+	}
+}
+
+// GetFileInfo returns the information about the file or directory at 'filePath'.
+func (storage *SambaStorage) GetFileInfo(threadIndex int, filePath string) (exist bool, isDir bool, size int64, err error) {
+	stat, err := storage.share.Stat(path.Join(storage.storageDir, filePath))
+	if err != nil {
+		if os.IsNotExist(err) {
+			return false, false, 0, nil
+		} else {
+			return false, false, 0, err
+		}
+	}
+
+	return true, stat.IsDir(), stat.Size(), nil
+}
+
+// DownloadFile reads the file at 'filePath' into the chunk.
+func (storage *SambaStorage) DownloadFile(threadIndex int, filePath string, chunk *Chunk) (err error) {
+
+	file, err := storage.share.Open(path.Join(storage.storageDir, filePath))
+
+	if err != nil {
+		return err
+	}
+
+	defer file.Close()
+	if _, err = RateLimitedCopy(chunk, file, storage.DownloadRateLimit/storage.numberOfThreads); err != nil {
+		return err
+	}
+
+	return nil
+
+}
+
+// UploadFile writes 'content' to the file at 'filePath'
+func (storage *SambaStorage) UploadFile(threadIndex int, filePath string, content []byte) (err error) {
+
+	fullPath := path.Join(storage.storageDir, filePath)
+
+	if len(strings.Split(filePath, "/")) > 2 {
+		dir := path.Dir(fullPath)
+		stat, err := storage.share.Stat(dir)
+		if err != nil {
+			if !os.IsNotExist(err) {
+				return err
+			}
+			err = storage.share.MkdirAll(dir, 0744)
+			if err != nil {
+				return err
+			}
+		} else {
+			if !stat.IsDir() && stat.Mode() & os.ModeSymlink == 0 {
+				return fmt.Errorf("The path %s is not a directory or symlink", dir)
+			}
+		}
+	}
+
+	letters := "abcdefghijklmnopqrstuvwxyz"
+	suffix := make([]byte, 8)
+	for i := range suffix {
+		suffix[i] = letters[rand.Intn(len(letters))]
+	}
+
+	temporaryFile := fullPath + "." + string(suffix) + ".tmp"
+
+	file, err := storage.share.Create(temporaryFile)
+	if err != nil {
+		return err
+	}
+
+	reader := CreateRateLimitedReader(content, storage.UploadRateLimit/storage.numberOfThreads)
+	_, err = io.Copy(file, reader)
+	if err != nil {
+		file.Close()
+		return err
+	}
+
+	if err = file.Sync(); err != nil {
+		pathErr, ok := err.(*os.PathError)
+		isNotSupported := ok && pathErr.Op == "sync" && pathErr.Err == syscall.ENOTSUP
+		if !isNotSupported {
+			_ = file.Close()
+			return err
+		}
+	}
+
+	err = file.Close()
+	if err != nil {
+		return err
+	}
+
+	err = storage.share.Rename(temporaryFile, fullPath)
+	if err != nil {
+
+		if _, e := storage.share.Stat(fullPath); e == nil {
+			storage.share.Remove(temporaryFile)
+			return nil
+		} else {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// If a local snapshot cache is needed for the storage to avoid downloading/uploading chunks too often when
+// managing snapshots.
+func (storage *SambaStorage) IsCacheNeeded() bool { return true }
+
+// If the 'MoveFile' method is implemented.
+func (storage *SambaStorage) IsMoveFileImplemented() bool { return true }
+
+// If the storage can guarantee strong consistency.
+func (storage *SambaStorage) IsStrongConsistent() bool { return true }
+
+// If the storage supports fast listing of files names.
+func (storage *SambaStorage) IsFastListing() bool { return false }
+
+// Enable the test mode.
+func (storage *SambaStorage) EnableTestMode() {}

src/duplicacy_sftpstorage.go

@@ -14,6 +14,7 @@ import (
 	"runtime"
 	"strings"
 	"time"
+	"sync"
 
 	"github.com/pkg/sftp"
 	"golang.org/x/crypto/ssh"
@@ -23,9 +24,13 @@ type SFTPStorage struct {
 	StorageBase
 
 	client          *sftp.Client
+	clientLock      sync.Mutex
 	minimumNesting  int // The minimum level of directories to dive into before searching for the chunk file.
 	storageDir      string
 	numberOfThreads int
+	numberOfTries   int
+	serverAddress   string
+	sftpConfig      *ssh.ClientConfig
 }
 
 func CreateSFTPStorageWithPassword(server string, port int, username string, storageDir string,
@@ -38,10 +43,10 @@ func CreateSFTPStorageWithPassword(server string, port int, username string, sto
 		return nil
 	}
 
-	return CreateSFTPStorage(server, port, username, storageDir, minimumNesting, authMethods, hostKeyCallback, threads)
+	return CreateSFTPStorage(false, server, port, username, storageDir, minimumNesting, authMethods, hostKeyCallback, threads)
 }
 
-func CreateSFTPStorage(server string, port int, username string, storageDir string, minimumNesting int,
+func CreateSFTPStorage(compatibilityMode bool, server string, port int, username string, storageDir string, minimumNesting int,
 	authMethods []ssh.AuthMethod,
 	hostKeyCallback func(hostname string, remote net.Addr,
 		key ssh.PublicKey) error, threads int) (storage *SFTPStorage, err error) {
@@ -52,8 +57,21 @@ func CreateSFTPStorage(server string, port int, username string, storageDir stri
 		HostKeyCallback: hostKeyCallback,
 	}
 
-	if server == "sftp.hidrive.strato.com" {
-		sftpConfig.Ciphers = []string{"aes128-cbc", "aes128-ctr", "aes256-ctr"}
+	if compatibilityMode {
+		sftpConfig.Ciphers = []string{
+			"aes128-ctr", "aes192-ctr", "aes256-ctr",
+			"aes128-gcm@openssh.com",
+			"chacha20-poly1305@openssh.com",
+			"arcfour256", "arcfour128", "arcfour",
+			"aes128-cbc",
+			"3des-cbc",
+		}
+		sftpConfig.KeyExchanges = [] string {
+			"curve25519-sha256@libssh.org",
+			"ecdh-sha2-nistp256", "ecdh-sha2-nistp384", "ecdh-sha2-nistp521",
+			"diffie-hellman-group1-sha1", "diffie-hellman-group14-sha1",
+			"diffie-hellman-group-exchange-sha1", "diffie-hellman-group-exchange-sha256",
+		}
 	}
 
 	serverAddress := fmt.Sprintf("%s:%d", server, port)
@@ -86,6 +104,9 @@ func CreateSFTPStorage(server string, port int, username string, storageDir stri
 		storageDir:      storageDir,
 		minimumNesting:  minimumNesting,
 		numberOfThreads: threads,
+		numberOfTries:   8,
+		serverAddress:   serverAddress,
+		sftpConfig:      sftpConfig,
 	}
 
 	// Random number fo generating the temporary chunk file suffix.
@@ -99,13 +120,57 @@ func CreateSFTPStorage(server string, port int, username string, storageDir stri
 }
 
 func CloseSFTPStorage(storage *SFTPStorage) {
-	storage.client.Close()
+	if storage.client != nil {
+		storage.client.Close()
+		storage.client = nil
+	}
 }
 
+func (storage *SFTPStorage) getSFTPClient() *sftp.Client {
+	storage.clientLock.Lock()
+	defer storage.clientLock.Unlock()
+	return storage.client
+}
+
+func (storage *SFTPStorage) retry(f func () error) error {
+	delay := time.Second
+	for i := 0;; i++ {
+		err := f()
+		if err != nil && strings.Contains(err.Error(), "EOF") && i < storage.numberOfTries {
+			LOG_WARN("SFTP_RETRY", "Encountered an error (%v); retry after %d second(s)", err, delay/time.Second)
+			time.Sleep(delay)
+			delay *= 2
+
+			storage.clientLock.Lock()
+			connection, err := ssh.Dial("tcp", storage.serverAddress, storage.sftpConfig)
+			if err != nil {
+				LOG_WARN("SFTP_RECONNECT", "Failed to connect to %s: %v; retrying", storage.serverAddress, err)
+				storage.clientLock.Unlock()
+				continue
+			}
+
+			client, err := sftp.NewClient(connection)
+			if err != nil {
+				LOG_WARN("SFTP_RECONNECT", "Failed to create a new SFTP client to %s: %v; retrying", storage.serverAddress, err)
+				connection.Close()
+				storage.clientLock.Unlock()
+				continue
+			}
+			storage.client = client
+			storage.clientLock.Unlock()
+			continue
+		}
+		return err
+	}
+}
 // ListFiles return the list of files and subdirectories under 'file' (non-recursively)
 func (storage *SFTPStorage) ListFiles(threadIndex int, dirPath string) (files []string, sizes []int64, err error) {
 
-	entries, err := storage.client.ReadDir(path.Join(storage.storageDir, dirPath))
+	var entries []os.FileInfo
+	err = storage.retry(func() error {
+		entries, err = storage.getSFTPClient().ReadDir(path.Join(storage.storageDir, dirPath))
+		return err
+	})
 	if err != nil {
 		return nil, nil, err
 	}
@@ -126,7 +191,11 @@ func (storage *SFTPStorage) ListFiles(threadIndex int, dirPath string) (files []
 // DeleteFile deletes the file or directory at 'filePath'.
 func (storage *SFTPStorage) DeleteFile(threadIndex int, filePath string) (err error) {
 	fullPath := path.Join(storage.storageDir, filePath)
-	fileInfo, err := storage.client.Stat(fullPath)
+	var fileInfo os.FileInfo
+	err = storage.retry(func() error {
+		fileInfo, err = storage.getSFTPClient().Stat(fullPath)
+		return err
+	})
 	if err != nil {
 		if os.IsNotExist(err) {
 			LOG_TRACE("SFTP_STORAGE", "File %s has disappeared before deletion", filePath)
@@ -137,33 +206,47 @@ func (storage *SFTPStorage) DeleteFile(threadIndex int, filePath string) (err er
 	if fileInfo == nil {
 		return nil
 	}
-	return storage.client.Remove(path.Join(storage.storageDir, filePath))
+	return storage.retry(func() error { return storage.getSFTPClient().Remove(path.Join(storage.storageDir, filePath)) })
 }
 
 // MoveFile renames the file.
 func (storage *SFTPStorage) MoveFile(threadIndex int, from string, to string) (err error) {
 	toPath := path.Join(storage.storageDir, to)
-	fileInfo, err := storage.client.Stat(toPath)
+	var fileInfo os.FileInfo
+	err = storage.retry(func() error {
+		fileInfo, err = storage.getSFTPClient().Stat(toPath)
+		return err
+	})
 	if fileInfo != nil {
 		return fmt.Errorf("The destination file %s already exists", toPath)
 	}
-	return storage.client.Rename(path.Join(storage.storageDir, from),
-		path.Join(storage.storageDir, to))
+	err = storage.retry(func() error { return storage.getSFTPClient().Rename(path.Join(storage.storageDir, from),
+		path.Join(storage.storageDir, to)) })
+	return err
 }
 
 // CreateDirectory creates a new directory.
 func (storage *SFTPStorage) CreateDirectory(threadIndex int, dirPath string) (err error) {
 	fullPath := path.Join(storage.storageDir, dirPath)
-	fileInfo, err := storage.client.Stat(fullPath)
+	var fileInfo os.FileInfo
+	err = storage.retry(func() error {
+		fileInfo, err = storage.getSFTPClient().Stat(fullPath)
+		return err
+	})
 	if fileInfo != nil && fileInfo.IsDir() {
 		return nil
 	}
-	return storage.client.Mkdir(path.Join(storage.storageDir, dirPath))
+	return storage.retry(func() error { return storage.getSFTPClient().Mkdir(path.Join(storage.storageDir, dirPath)) })
 }
 
 // GetFileInfo returns the information about the file or directory at 'filePath'.
 func (storage *SFTPStorage) GetFileInfo(threadIndex int, filePath string) (exist bool, isDir bool, size int64, err error) {
-	fileInfo, err := storage.client.Stat(path.Join(storage.storageDir, filePath))
+	var fileInfo os.FileInfo
+	err = storage.retry(func() error {
+		fileInfo, err = storage.getSFTPClient().Stat(path.Join(storage.storageDir, filePath))
+		return err
+	})
+
 	if err != nil {
 		if os.IsNotExist(err) {
 			return false, false, 0, nil
@@ -181,18 +264,19 @@ func (storage *SFTPStorage) GetFileInfo(threadIndex int, filePath string) (exist
 
 // DownloadFile reads the file at 'filePath' into the chunk.
 func (storage *SFTPStorage) DownloadFile(threadIndex int, filePath string, chunk *Chunk) (err error) {
-	file, err := storage.client.Open(path.Join(storage.storageDir, filePath))
+	return storage.retry(func() error {
+		file, err := storage.getSFTPClient().Open(path.Join(storage.storageDir, filePath))
 
-	if err != nil {
-		return err
-	}
+		if err != nil {
+			return err
+		}
 
-	defer file.Close()
-	if _, err = RateLimitedCopy(chunk, file, storage.DownloadRateLimit/storage.numberOfThreads); err != nil {
-		return err
-	}
-
-	return nil
+		defer file.Close()
+		if _, err = RateLimitedCopy(chunk, file, storage.DownloadRateLimit/storage.numberOfThreads); err != nil {
+			return err
+		}
+		return nil
+	})
 }
 
 // UploadFile writes 'content' to the file at 'filePath'.
@@ -201,62 +285,57 @@ func (storage *SFTPStorage) UploadFile(threadIndex int, filePath string, content
 	fullPath := path.Join(storage.storageDir, filePath)
 
 	dirs := strings.Split(filePath, "/")
-	if len(dirs) > 1 {
-		fullDir := path.Dir(fullPath)
-		_, err := storage.client.Stat(fullDir)
-		if err != nil {
-			// The error may be caused by a non-existent fullDir, or a broken connection.  In either case,
-			// we just assume it is the former because there isn't a way to tell which is the case.
-			for i, _ := range dirs[1 : len(dirs)-1] {
-				subDir := path.Join(storage.storageDir, path.Join(dirs[0:i+2]...))
-				// We don't check the error; just keep going blindly but always store the last err
-				err = storage.client.Mkdir(subDir)
-			}
+	fullDir := path.Dir(fullPath)
+	return storage.retry(func() error {
 
-			// If there is an error creating the dirs, we check fullDir one more time, because another thread
-			// may happen to create the same fullDir ahead of this thread
-			if err != nil {
-				_, err := storage.client.Stat(fullDir)
-				if err != nil {
-					return err
+		if len(dirs) > 1 {
+			_, err := storage.getSFTPClient().Stat(fullDir)
+			if os.IsNotExist(err) {
+				for i := range dirs[1 : len(dirs)-1] {
+					subDir := path.Join(storage.storageDir, path.Join(dirs[0:i+2]...))
+					// We don't check the error; just keep going blindly
+					storage.getSFTPClient().Mkdir(subDir)
 				}
 			}
 		}
-	}
 
-	letters := "abcdefghijklmnopqrstuvwxyz"
-	suffix := make([]byte, 8)
-	for i := range suffix {
-		suffix[i] = letters[rand.Intn(len(letters))]
-	}
-
-	temporaryFile := fullPath + "." + string(suffix) + ".tmp"
-
-	file, err := storage.client.OpenFile(temporaryFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC)
-	if err != nil {
-		return err
-	}
-
-	reader := CreateRateLimitedReader(content, storage.UploadRateLimit/storage.numberOfThreads)
-	_, err = io.Copy(file, reader)
-	if err != nil {
-		file.Close()
-		return err
-	}
-	file.Close()
-
-	err = storage.client.Rename(temporaryFile, fullPath)
-	if err != nil {
-
-		if _, err = storage.client.Stat(fullPath); err == nil {
-			storage.client.Remove(temporaryFile)
-			return nil
-		} else {
-			return fmt.Errorf("Uploaded file but failed to store it at %s: %v", fullPath, err)
+		letters := "abcdefghijklmnopqrstuvwxyz"
+		suffix := make([]byte, 8)
+		for i := range suffix {
+			suffix[i] = letters[rand.Intn(len(letters))]
 		}
-	}
 
-	return nil
+		temporaryFile := fullPath + "." + string(suffix) + ".tmp"
+
+		file, err := storage.getSFTPClient().OpenFile(temporaryFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC)
+		if err != nil {
+			return err
+		}
+
+		reader := CreateRateLimitedReader(content, storage.UploadRateLimit/storage.numberOfThreads)
+		_, err = io.Copy(file, reader)
+		if err != nil {
+			file.Close()
+			return err
+		}
+
+		err = file.Close()
+		if err != nil {
+			return err
+		}
+
+		err = storage.getSFTPClient().Rename(temporaryFile, fullPath)
+		if err != nil {
+			if _, err = storage.getSFTPClient().Stat(fullPath); err == nil {
+				storage.getSFTPClient().Remove(temporaryFile)
+				return nil
+			} else {
+				return fmt.Errorf("Uploaded file but failed to store it at %s: %v", fullPath, err)
+			}
+		}
+
+		return nil
+	})
 }
 
 // If a local snapshot cache is needed for the storage to avoid downloading/uploading chunks too often when

src/duplicacy_shadowcopy.go

@@ -3,10 +3,11 @@
 // Commercial use requires per-user licenses available from https://duplicacy.com
 
 // +build !windows
+// +build !darwin
 
 package duplicacy
 
-func CreateShadowCopy(top string, shadowCopy bool) (shadowTop string) {
+func CreateShadowCopy(top string, shadowCopy bool, timeoutInSeconds int) (shadowTop string) {
 	return top
 }
 

src/duplicacy_shadowcopy_darwin.go

@@ -0,0 +1,186 @@
+//
+// Shadow copy module for Mac OSX using APFS snapshot
+//
+//
+// This module copyright 2018 Adam Marcus (https://github.com/amarcu5)
+// and may be distributed under the same terms as Duplicacy.
+
+package duplicacy
+
+import (
+	"context"
+	"errors"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"regexp"
+	"syscall"
+	"time"
+)
+
+var snapshotPath string
+var snapshotDate string
+
+// Converts char array to string
+func CharsToString(ca []int8) string {
+
+	len := len(ca)
+	ba := make([]byte, len)
+
+	for i, v := range ca {
+		ba[i] = byte(v)
+		if ba[i] == 0 {
+			len = i
+			break
+		}
+	}
+
+	return string(ba[:len])
+}
+
+// Get ID of device containing path
+func GetPathDeviceId(path string) (deviceId int32, err error) {
+
+	stat := syscall.Stat_t{}
+
+	err = syscall.Stat(path, &stat)
+	if err != nil {
+		return 0, err
+	}
+
+	return stat.Dev, nil
+}
+
+// Executes shell command with timeout and returns stdout
+func CommandWithTimeout(timeoutInSeconds int, name string, arg ...string) (output string, err error) {
+
+	ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeoutInSeconds)*time.Second)
+	defer cancel()
+
+	cmd := exec.CommandContext(ctx, name, arg...)
+	out, err := cmd.Output()
+
+	if ctx.Err() == context.DeadlineExceeded {
+		err = errors.New("Command '" + name + "' timed out")
+	}
+
+	output = string(out)
+	return output, err
+}
+
+func DeleteShadowCopy() {
+
+	if snapshotPath == "" {
+		return
+	}
+
+	err := exec.Command("/sbin/umount", "-f", snapshotPath).Run()
+	if err != nil {
+		LOG_WARN("VSS_DELETE", "Error while unmounting snapshot: %v", err)
+		return
+	}
+
+	err = exec.Command("tmutil", "deletelocalsnapshots", snapshotDate).Run()
+	if err != nil {
+		LOG_WARN("VSS_DELETE", "Error while deleting local snapshot: %v", err)
+		return
+	}
+
+	err = os.RemoveAll(snapshotPath)
+	if err != nil {
+		LOG_WARN("VSS_DELETE", "Error while deleting temporary mount directory: %v", err)
+		return
+	}
+
+	LOG_INFO("VSS_DELETE", "Shadow copy unmounted and deleted at %s", snapshotPath)
+
+	snapshotPath = ""
+}
+
+func CreateShadowCopy(top string, shadowCopy bool, timeoutInSeconds int) (shadowTop string) {
+
+	if !shadowCopy {
+		return top
+	}
+
+	// Check repository filesystem is APFS
+	stat := syscall.Statfs_t{}
+	err := syscall.Statfs(top, &stat)
+	if err != nil {
+		LOG_ERROR("VSS_INIT", "Unable to determine filesystem of repository path")
+		return top
+	}
+	if CharsToString(stat.Fstypename[:]) != "apfs" {
+		LOG_WARN("VSS_INIT", "VSS requires APFS filesystem")
+		return top
+	}
+
+	// Check path is local as tmutil snapshots will not support APFS formatted external drives
+	deviceIdLocal, err := GetPathDeviceId("/")
+	if err != nil {
+		LOG_ERROR("VSS_INIT", "Unable to get device ID of path: /")
+		return top
+	}
+	deviceIdRepository, err := GetPathDeviceId(top)
+	if err != nil {
+		LOG_ERROR("VSS_INIT", "Unable to get device ID of path: %s", top)
+		return top
+	}
+	if deviceIdLocal != deviceIdRepository {
+		LOG_WARN("VSS_PATH", "VSS not supported for non-local repository path: %s", top)
+		return top
+	}
+
+	if timeoutInSeconds <= 60 {
+		timeoutInSeconds = 60
+	}
+
+	// Create mount point
+	snapshotPath, err = ioutil.TempDir("/tmp/", "snp_")
+	if err != nil {
+		LOG_ERROR("VSS_CREATE", "Failed to create temporary mount directory")
+		return top
+	}
+
+	// Use tmutil to create snapshot
+	tmutilOutput, err := CommandWithTimeout(timeoutInSeconds, "tmutil", "snapshot")
+	if err != nil {
+		LOG_ERROR("VSS_CREATE", "Error while calling tmutil: %v", err)
+		return top
+	}
+
+	snapshotDateRegex := regexp.MustCompile(`:\s+([0-9\-]+)`)
+	matched := snapshotDateRegex.FindStringSubmatch(tmutilOutput)
+	if matched == nil {
+		LOG_ERROR("VSS_CREATE", "Snapshot creation failed: %s", tmutilOutput)
+		return top
+	}
+	snapshotDate = matched[1]
+
+	tmutilOutput, err = CommandWithTimeout(timeoutInSeconds, "tmutil", "listlocalsnapshots", ".")
+	if err != nil {
+		LOG_ERROR("VSS_CREATE", "Error while calling 'tmutil listlocalsnapshots': %v", err)
+		return top
+	}
+	snapshotName := "com.apple.TimeMachine." + snapshotDate
+ 
+	snapshotNameRegex := regexp.MustCompile(`(?m)^(.+` + snapshotDate + `.*)$`)
+	matched = snapshotNameRegex.FindStringSubmatch(tmutilOutput)
+	if len(matched) > 0 {
+		snapshotName = matched[0]
+	} else {
+		LOG_INFO("VSS_CREATE", "Can't find the snapshot name with 'tmutil listlocalsnapshots'; fallback to %s", snapshotName)
+	}
+ 
+	// Mount snapshot as readonly and hide from GUI i.e. Finder
+	_, err = CommandWithTimeout(timeoutInSeconds,
+		"/sbin/mount", "-t", "apfs", "-o", "nobrowse,-r,-s="+snapshotName, "/System/Volumes/Data", snapshotPath)
+	if err != nil {
+		LOG_ERROR("VSS_CREATE", "Error while mounting snapshot: %v", err)
+		return top
+	}
+
+	LOG_INFO("VSS_DONE", "Shadow copy created and mounted at %s", snapshotPath)
+
+	return snapshotPath + top
+}

src/duplicacy_shadowcopy_windows.go

@@ -353,12 +353,15 @@ func DeleteShadowCopy() {
 	ole.CoUninitialize()
 }
 
-func CreateShadowCopy(top string, shadowCopy bool) (shadowTop string) {
+func CreateShadowCopy(top string, shadowCopy bool, timeoutInSeconds int) (shadowTop string) {
 
 	if !shadowCopy {
 		return top
 	}
 
+	if timeoutInSeconds <= 60 {
+		timeoutInSeconds = 60
+	}
 	ole.CoInitialize(0)
 	defer ole.CoUninitialize()
 
@@ -416,7 +419,7 @@ func CreateShadowCopy(top string, shadowCopy bool) (shadowTop string) {
 		return top
 	}
 
-	if !async.Wait(60) {
+	if !async.Wait(timeoutInSeconds) {
 		LOG_ERROR("VSS_GATHER", "Shadow copy creation failed: GatherWriterMetadata didn't finish properly")
 		return top
 	}
@@ -456,7 +459,7 @@ func CreateShadowCopy(top string, shadowCopy bool) (shadowTop string) {
 		return top
 	}
 
-	if !async.Wait(60) {
+	if !async.Wait(timeoutInSeconds) {
 		LOG_ERROR("VSS_PREPARE", "Shadow copy creation failed: PrepareForBackup didn't finish properly")
 		return top
 	}
@@ -473,7 +476,7 @@ func CreateShadowCopy(top string, shadowCopy bool) (shadowTop string) {
 		return top
 	}
 
-	if !async.Wait(180) {
+	if !async.Wait(timeoutInSeconds) {
 		LOG_ERROR("VSS_SNAPSHOT", "Shadow copy creation failed: DoSnapshotSet didn't finish properly")
 		return top
 	}

src/duplicacy_snapshot.go

@@ -8,16 +8,21 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"os"
-	"path"
-	"strconv"
+	"path/filepath"
 	"strings"
 	"time"
+	"sort"
+
+    "github.com/vmihailenco/msgpack"
+
 )
 
 // Snapshot represents a backup of the repository.
 type Snapshot struct {
+	Version       int
 	ID            string // the snapshot id; must be different for different repositories
 	Revision      int    // the revision number
 	Options       string // options used to create this snapshot (some not included)
@@ -36,212 +41,266 @@ type Snapshot struct {
 	// A sequence of chunks whose aggregated content is the json representation of 'ChunkLengths'.
 	LengthSequence []string
 
-	Files []*Entry // list of files and subdirectories
-
 	ChunkHashes  []string // a sequence of chunks representing the file content
 	ChunkLengths []int    // the length of each chunk
 
 	Flag bool // used to mark certain snapshots for deletion or copy
 
-	discardAttributes bool
 }
 
 // CreateEmptySnapshot creates an empty snapshot.
 func CreateEmptySnapshot(id string) (snapshto *Snapshot) {
 	return &Snapshot{
+		Version:   1,
 		ID:        id,
 		Revision:  0,
 		StartTime: time.Now().Unix(),
 	}
 }
 
-// CreateSnapshotFromDirectory creates a snapshot from the local directory 'top'.  Only 'Files'
-// will be constructed, while 'ChunkHashes' and 'ChunkLengths' can only be populated after uploading.
-func CreateSnapshotFromDirectory(id string, top string) (snapshot *Snapshot, skippedDirectories []string,
-	skippedFiles []string, err error) {
+type DirectoryListing struct {
+	directory string
+	files *[]Entry
+}
 
-	snapshot = &Snapshot{
-		ID:        id,
-		Revision:  0,
-		StartTime: time.Now().Unix(),
-	}
+func (snapshot *Snapshot) ListLocalFiles(top string, nobackupFile string,
+								         filtersFile string, excludeByAttribute bool, listingChannel chan *Entry,
+								         skippedDirectories *[]string, skippedFiles *[]string) {
 
 	var patterns []string
 
-	patternFile, err := ioutil.ReadFile(path.Join(GetDuplicacyPreferencePath(), "filters"))
-	if err == nil {
-		for _, pattern := range strings.Split(string(patternFile), "\n") {
-			pattern = strings.TrimSpace(pattern)
-			if len(pattern) == 0 {
-				continue
-			}
-
-			if pattern[0] == '#' {
-				continue
-			}
-
-			if IsUnspecifiedFilter(pattern) {
-				pattern = "+" + pattern
-			}
-
-			if IsEmptyFilter(pattern) {
-				continue
-			}
-
-			if strings.HasPrefix(pattern, "i:") || strings.HasPrefix(pattern, "e:") {
-				valid, err := IsValidRegex(pattern[2:])
-				if !valid || err != nil {
-					LOG_ERROR("SNAPSHOT_FILTER", "Invalid regular expression encountered for filter: \"%s\", error: %v", pattern, err)
-				}
-			}
-
-			patterns = append(patterns, pattern)
-		}
-
-		LOG_DEBUG("REGEX_DEBUG", "There are %d compiled regular expressions stored", len(RegexMap))
-
-		LOG_INFO("SNAPSHOT_FILTER", "Loaded %d include/exclude pattern(s)", len(patterns))
-
-		if IsTracing() {
-			for _, pattern := range patterns {
-				LOG_TRACE("SNAPSHOT_PATTERN", "Pattern: %s", pattern)
-			}
-		}
-
+	if filtersFile == "" {
+		filtersFile = joinPath(GetDuplicacyPreferencePath(), "filters")
 	}
+	patterns = ProcessFilters(filtersFile)
 
 	directories := make([]*Entry, 0, 256)
 	directories = append(directories, CreateEntry("", 0, 0, 0))
 
-	snapshot.Files = make([]*Entry, 0, 256)
-
-	attributeThreshold := 1024 * 1024
-	if attributeThresholdValue, found := os.LookupEnv("DUPLICACY_ATTRIBUTE_THRESHOLD"); found && attributeThresholdValue != "" {
-		attributeThreshold, _ = strconv.Atoi(attributeThresholdValue)
-	}
-
 	for len(directories) > 0 {
 
 		directory := directories[len(directories)-1]
 		directories = directories[:len(directories)-1]
-		snapshot.Files = append(snapshot.Files, directory)
-		subdirectories, skipped, err := ListEntries(top, directory.Path, &snapshot.Files, patterns, snapshot.discardAttributes)
+		subdirectories, skipped, err := ListEntries(top, directory.Path, patterns, nobackupFile, excludeByAttribute, listingChannel)
 		if err != nil {
-			LOG_WARN("LIST_FAILURE", "Failed to list subdirectory: %v", err)
-			skippedDirectories = append(skippedDirectories, directory.Path)
+			if directory.Path == "" {
+				LOG_ERROR("LIST_FAILURE", "Failed to list the repository root: %v", err)
+				return
+			}
+			LOG_WARN("LIST_FAILURE", "Failed to list subdirectory %s: %v", directory.Path, err)
+			if skippedDirectories != nil {
+				*skippedDirectories = append(*skippedDirectories, directory.Path)
+			}
 			continue
 		}
 
 		directories = append(directories, subdirectories...)
-		skippedFiles = append(skippedFiles, skipped...)
 
-		if !snapshot.discardAttributes && len(snapshot.Files) > attributeThreshold {
-			LOG_INFO("LIST_ATTRIBUTES", "Discarding file attributes")
-			snapshot.discardAttributes = true
-			for _, file := range snapshot.Files {
-				file.Attributes = nil
+		if skippedFiles != nil {
+			*skippedFiles = append(*skippedFiles, skipped...)
+		}
+
+	}
+	close(listingChannel)
+}
+
+func (snapshot *Snapshot)ListRemoteFiles(config *Config, chunkOperator *ChunkOperator, entryOut func(*Entry) bool) {
+
+	var chunks []string
+	for _, chunkHash := range snapshot.FileSequence {
+		chunks = append(chunks, chunkOperator.config.GetChunkIDFromHash(chunkHash))
+	}
+
+	var chunk *Chunk
+	reader := NewSequenceReader(snapshot.FileSequence, func(chunkHash string) []byte {
+		if chunk != nil {
+			config.PutChunk(chunk)
+		}
+		chunk = chunkOperator.Download(chunkHash, 0, true)
+		return chunk.GetBytes()
+	})
+
+	defer func() {
+		if chunk != nil {
+			config.PutChunk(chunk)
+		}
+	} ()
+
+	// Normally if Version is 0 then the snapshot is created by CLI v2 but unfortunately CLI 3.0.1 does not set the
+	// version bit correctly when copying old backups.  So we need to check the first byte -- if it is '[' then it is
+	// the old format.  The new format starts with a string encoded in msgpack and the first byte can't be '['.
+	if snapshot.Version == 0 || reader.GetFirstByte() == '['{
+		LOG_INFO("SNAPSHOT_VERSION", "snapshot %s at revision %d is encoded in an old version format", snapshot.ID, snapshot.Revision)
+		files := make([]*Entry, 0)
+		decoder := json.NewDecoder(reader)
+
+		// read open bracket
+		_, err := decoder.Token()
+		if err != nil {
+			LOG_ERROR("SNAPSHOT_PARSE", "Failed to open the snapshot %s at revision %d: not a list of entries",
+				snapshot.ID, snapshot.Revision)
+			return
+		}
+
+		for decoder.More() {
+			var entry Entry
+			err = decoder.Decode(&entry)
+			if err != nil {
+				LOG_ERROR("SNAPSHOT_PARSE", "Failed to load files specified in the snapshot %s at revision %d: %v",
+					snapshot.ID, snapshot.Revision, err)
+				return
+			}
+			files = append(files, &entry)
+		}
+
+		sort.Sort(ByName(files))
+
+		for _, file := range files {
+			if !entryOut(file) {
+				return
 			}
 		}
-	}
+	} else if snapshot.Version == 1 {
+		decoder := msgpack.NewDecoder(reader)
 
-	// Remove the root entry
-	snapshot.Files = snapshot.Files[1:]
+		lastEndChunk := 0
 
-	return snapshot, skippedDirectories, skippedFiles, nil
-}
+		// while the array contains values
+		for _, err := decoder.PeekCode(); err != io.EOF; _, err = decoder.PeekCode() {
+			if err != nil {
+				LOG_ERROR("SNAPSHOT_PARSE", "Failed to parse the snapshot %s at revision %d: %v",
+					snapshot.ID, snapshot.Revision, err)
+				return
+			}
+			var entry Entry
+			err = decoder.Decode(&entry)
+			if err != nil {
+				LOG_ERROR("SNAPSHOT_PARSE", "Failed to load the snapshot %s at revision %d: %v",
+					snapshot.ID, snapshot.Revision, err)
+				return
+			}
 
-// This is the struct used to save/load incomplete snapshots
-type IncompleteSnapshot struct {
-	Files        []*Entry
-	ChunkHashes  []string
-	ChunkLengths []int
-}
+			if entry.IsFile() {
+				entry.StartChunk += lastEndChunk
+				entry.EndChunk += entry.StartChunk
+				lastEndChunk = entry.EndChunk
+			}
 
-// LoadIncompleteSnapshot loads the incomplete snapshot if it exists
-func LoadIncompleteSnapshot() (snapshot *Snapshot) {
-	snapshotFile := path.Join(GetDuplicacyPreferencePath(), "incomplete")
-	description, err := ioutil.ReadFile(snapshotFile)
-	if err != nil {
-		LOG_DEBUG("INCOMPLETE_LOCATE", "Failed to locate incomplete snapshot: %v", err)
-		return nil
-	}
+			err = entry.check(snapshot.ChunkLengths)
+			if err != nil {
+				LOG_ERROR("SNAPSHOT_ENTRY", "Failed to load the snapshot %s at revision %d: %v",
+					snapshot.ID, snapshot.Revision, err)
+				return
+			}
 
-	var incompleteSnapshot IncompleteSnapshot
-
-	err = json.Unmarshal(description, &incompleteSnapshot)
-	if err != nil {
-		LOG_DEBUG("INCOMPLETE_PARSE", "Failed to parse incomplete snapshot: %v", err)
-		return nil
-	}
-
-	var chunkHashes []string
-	for _, chunkHash := range incompleteSnapshot.ChunkHashes {
-		hash, err := hex.DecodeString(chunkHash)
-		if err != nil {
-			LOG_DEBUG("INCOMPLETE_DECODE", "Failed to decode incomplete snapshot: %v", err)
-			return nil
+			if !entryOut(&entry) {
+				return
+			}
 		}
-		chunkHashes = append(chunkHashes, string(hash))
-	}
 
-	snapshot = &Snapshot{
-		Files:        incompleteSnapshot.Files,
-		ChunkHashes:  chunkHashes,
-		ChunkLengths: incompleteSnapshot.ChunkLengths,
-	}
-	LOG_INFO("INCOMPLETE_LOAD", "Incomplete snapshot loaded from %s", snapshotFile)
-	return snapshot
-}
-
-// SaveIncompleteSnapshot saves the incomplete snapshot under the preference directory
-func SaveIncompleteSnapshot(snapshot *Snapshot) {
-	var files []*Entry
-	for _, file := range snapshot.Files {
-		// All unprocessed files will have a size of -1
-		if file.Size >= 0 {
-			file.Attributes = nil
-			files = append(files, file)
-		} else {
-			break
-		}
-	}
-	var chunkHashes []string
-	for _, chunkHash := range snapshot.ChunkHashes {
-		chunkHashes = append(chunkHashes, hex.EncodeToString([]byte(chunkHash)))
-	}
-
-	incompleteSnapshot := IncompleteSnapshot{
-		Files:        files,
-		ChunkHashes:  chunkHashes,
-		ChunkLengths: snapshot.ChunkLengths,
-	}
-
-	description, err := json.MarshalIndent(incompleteSnapshot, "", "  ")
-	if err != nil {
-		LOG_WARN("INCOMPLETE_ENCODE", "Failed to encode the incomplete snapshot: %v", err)
+	} else {
+		LOG_ERROR("SNAPSHOT_VERSION", "snapshot %s at revision %d is encoded in unsupported version %d format",
+				  snapshot.ID, snapshot.Revision, snapshot.Version)
 		return
 	}
 
-	snapshotFile := path.Join(GetDuplicacyPreferencePath(), "incomplete")
-	err = ioutil.WriteFile(snapshotFile, description, 0644)
-	if err != nil {
-		LOG_WARN("INCOMPLETE_WRITE", "Failed to save the incomplete snapshot: %v", err)
-		return
-	}
-
-	LOG_INFO("INCOMPLETE_SAVE", "Incomplete snapshot saved to %s", snapshotFile)
 }
 
-func RemoveIncompleteSnapshot() {
-	snapshotFile := path.Join(GetDuplicacyPreferencePath(), "incomplete")
-	if stat, err := os.Stat(snapshotFile); err == nil && !stat.IsDir() {
-		err = os.Remove(snapshotFile)
-		if err != nil {
-			LOG_INFO("INCOMPLETE_SAVE", "Failed to remove ncomplete snapshot: %v", err)
-		} else {
-			LOG_INFO("INCOMPLETE_SAVE", "Removed incomplete snapshot %s", snapshotFile)
+func AppendPattern(patterns []string, new_pattern string) (new_patterns []string) {
+	for _, pattern := range patterns {
+		if pattern == new_pattern {
+			LOG_INFO("SNAPSHOT_FILTER", "Ignoring duplicate pattern: %s ...", new_pattern)
+			return patterns
 		}
 	}
+	new_patterns = append(patterns, new_pattern)
+	return new_patterns
+}
+func ProcessFilters(filtersFile string) (patterns []string) {
+	patterns = ProcessFilterFile(filtersFile, make([]string, 0))
+
+	LOG_DEBUG("REGEX_DEBUG", "There are %d compiled regular expressions stored", len(RegexMap))
+
+	LOG_INFO("SNAPSHOT_FILTER", "Loaded %d include/exclude pattern(s)", len(patterns))
+
+	if IsTracing() {
+		for _, pattern := range patterns {
+			LOG_TRACE("SNAPSHOT_PATTERN", "Pattern: %s", pattern)
+		}
+
+	}
+
+	return patterns
+}
+
+func ProcessFilterFile(patternFile string, includedFiles []string) (patterns []string) {
+	for _, file := range includedFiles {
+		if file == patternFile {
+			// cycle in include mechanism discovered.
+			LOG_ERROR("SNAPSHOT_FILTER", "The filter file %s has already been included", patternFile)
+			return patterns
+		}
+	}
+	includedFiles = append(includedFiles, patternFile)
+	LOG_INFO("SNAPSHOT_FILTER", "Parsing filter file %s", patternFile)
+	patternFileContent, err := ioutil.ReadFile(patternFile)
+	if err == nil {
+		patternFileLines := strings.Split(string(patternFileContent), "\n")
+		patterns = ProcessFilterLines(patternFileLines, includedFiles)
+	}
+	return patterns
+}
+
+func ProcessFilterLines(patternFileLines []string, includedFiles []string) (patterns []string) {
+	for _, pattern := range patternFileLines {
+		pattern = strings.TrimSpace(pattern)
+		if len(pattern) == 0 {
+			continue
+		}
+
+		if strings.HasPrefix(pattern, "@") {
+			patternIncludeFile := strings.TrimSpace(pattern[1:])
+			if patternIncludeFile == "" {
+				continue
+			}
+			if ! filepath.IsAbs(patternIncludeFile) {
+				basePath := ""
+				if len(includedFiles) == 0 {
+					basePath, _ = os.Getwd()
+				} else {
+					basePath = filepath.Dir(includedFiles[len(includedFiles)-1])
+				}
+				patternIncludeFile = joinPath(basePath, patternIncludeFile)
+			}
+			for _, pattern := range ProcessFilterFile(patternIncludeFile, includedFiles) {
+				patterns = AppendPattern(patterns, pattern)
+			}
+			continue
+		}
+
+		if pattern[0] == '#' {
+			continue
+		}
+
+		if IsUnspecifiedFilter(pattern) {
+			pattern = "+" + pattern
+		}
+
+		if IsEmptyFilter(pattern) {
+			continue
+		}
+
+		if strings.HasPrefix(pattern, "i:") || strings.HasPrefix(pattern, "e:") {
+			valid, err := IsValidRegex(pattern[2:])
+			if !valid || err != nil {
+				LOG_ERROR("SNAPSHOT_FILTER", "Invalid regular expression encountered for filter: \"%s\", error: %v", pattern, err)
+			}
+		}
+
+		patterns = AppendPattern(patterns, pattern)
+	}
+
+	return patterns
 }
 
 // CreateSnapshotFromDescription creates a snapshot from json decription.
@@ -256,6 +315,14 @@ func CreateSnapshotFromDescription(description []byte) (snapshot *Snapshot, err
 
 	snapshot = &Snapshot{}
 
+	if value, ok := root["version"]; !ok {
+		snapshot.Version = 0
+	} else if version, ok := value.(float64); !ok {
+		return nil, fmt.Errorf("Invalid version is specified in the snapshot")
+	} else {
+		snapshot.Version = int(version)
+	}
+
 	if value, ok := root["id"]; !ok {
 		return nil, fmt.Errorf("No id is specified in the snapshot")
 	} else if snapshot.ID, ok = value.(string); !ok {
@@ -357,6 +424,11 @@ func (snapshot *Snapshot) LoadChunks(description []byte) (err error) {
 	return err
 }
 
+// ClearChunks removes loaded chunks from memory
+func (snapshot *Snapshot) ClearChunks() {
+	snapshot.ChunkHashes = nil
+}
+
 // LoadLengths construct 'ChunkLengths' from the json description.
 func (snapshot *Snapshot) LoadLengths(description []byte) (err error) {
 	return json.Unmarshal(description, &snapshot.ChunkLengths)
@@ -367,6 +439,7 @@ func (snapshot *Snapshot) MarshalJSON() ([]byte, error) {
 
 	object := make(map[string]interface{})
 
+	object["version"] = snapshot.Version
 	object["id"] = snapshot.ID
 	object["revision"] = snapshot.Revision
 	object["options"] = snapshot.Options
@@ -388,9 +461,7 @@ func (snapshot *Snapshot) MarshalJSON() ([]byte, error) {
 // MarshalSequence creates a json represetion for the specified chunk sequence.
 func (snapshot *Snapshot) MarshalSequence(sequenceType string) ([]byte, error) {
 
-	if sequenceType == "files" {
-		return json.Marshal(snapshot.Files)
-	} else if sequenceType == "chunks" {
+	if sequenceType == "chunks" {
 		return json.Marshal(encodeSequence(snapshot.ChunkHashes))
 	} else {
 		return json.Marshal(snapshot.ChunkLengths)
@@ -419,3 +490,4 @@ func encodeSequence(sequence []string) []string {
 
 	return sequenceInHex
 }
+

src/duplicacy_snapshotmanager_test.go

@@ -9,6 +9,7 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"path"
 	"strings"
@@ -107,24 +108,26 @@ func createTestSnapshotManager(testDir string) *SnapshotManager {
 	snapshotCache.CreateDirectory(0, "snapshots")
 
 	snapshotManager.snapshotCache = snapshotCache
+
+	SetDuplicacyPreferencePath(testDir + "/.duplicacy")
+
 	return snapshotManager
 }
 
 func uploadTestChunk(manager *SnapshotManager, content []byte) string {
 
-	completionFunc := func(chunk *Chunk, chunkIndex int, skipped bool, chunkSize int, uploadSize int) {
+	chunkOperator := CreateChunkOperator(manager.config, manager.storage, nil, false, false, *testThreads, false)
+	chunkOperator.UploadCompletionFunc = func(chunk *Chunk, chunkIndex int, skipped bool, chunkSize int, uploadSize int) {
 		LOG_INFO("UPLOAD_CHUNK", "Chunk %s size %d uploaded", chunk.GetID(), chunkSize)
 	}
 
-	chunkUploader := CreateChunkUploader(manager.config, manager.storage, nil, testThreads, nil)
-	chunkUploader.completionFunc = completionFunc
-	chunkUploader.Start()
-
 	chunk := CreateChunk(manager.config, true)
 	chunk.Reset(true)
 	chunk.Write(content)
-	chunkUploader.StartChunk(chunk, 0)
-	chunkUploader.Stop()
+
+	chunkOperator.Upload(chunk, 0, false)
+	chunkOperator.WaitForCompletion()
+	chunkOperator.Stop()
 
 	return chunk.GetHash()
 }
@@ -140,7 +143,16 @@ func uploadRandomChunk(manager *SnapshotManager, chunkSize int) string {
 	return uploadTestChunk(manager, content)
 }
 
-func createTestSnapshot(manager *SnapshotManager, snapshotID string, revision int, startTime int64, endTime int64, chunkHashes []string) {
+func uploadRandomChunks(manager *SnapshotManager, chunkSize int, numberOfChunks int) []string {
+	chunkList := make([]string, 0)
+	for i := 0; i < numberOfChunks; i++ {
+		chunkHash := uploadRandomChunk(manager, chunkSize)
+		chunkList = append(chunkList, chunkHash)
+	}
+	return chunkList
+}
+
+func createTestSnapshot(manager *SnapshotManager, snapshotID string, revision int, startTime int64, endTime int64, chunkHashes []string, tag string) {
 
 	snapshot := &Snapshot{
 		ID:          snapshotID,
@@ -148,6 +160,7 @@ func createTestSnapshot(manager *SnapshotManager, snapshotID string, revision in
 		StartTime:   startTime,
 		EndTime:     endTime,
 		ChunkHashes: chunkHashes,
+		Tag:         tag,
 	}
 
 	var chunkHashesInHex []string
@@ -166,6 +179,12 @@ func createTestSnapshot(manager *SnapshotManager, snapshotID string, revision in
 
 func checkTestSnapshots(manager *SnapshotManager, expectedSnapshots int, expectedFossils int) {
 
+	manager.CreateChunkOperator(false, false, 1, false)
+	defer func() {
+		manager.chunkOperator.Stop()
+		manager.chunkOperator = nil
+	}()
+
 	var snapshotIDs []string
 	var err error
 
@@ -199,7 +218,7 @@ func checkTestSnapshots(manager *SnapshotManager, expectedSnapshots int, expecte
 			snapshot := manager.DownloadSnapshot(snapshotID, revision)
 			numberOfSnapshots++
 
-			for _, chunk := range manager.GetSnapshotChunks(snapshot) {
+			for _, chunk := range manager.GetSnapshotChunks(snapshot, false) {
 				chunks[chunk] = true
 			}
 		}
@@ -222,7 +241,7 @@ func checkTestSnapshots(manager *SnapshotManager, expectedSnapshots int, expecte
 	}
 }
 
-func TestSingleRepositoryPrune(t *testing.T) {
+func TestPruneSingleRepository(t *testing.T) {
 
 	setTestingT(t)
 
@@ -238,34 +257,35 @@ func TestSingleRepositoryPrune(t *testing.T) {
 
 	now := time.Now().Unix()
 	day := int64(24 * 3600)
-	t.Logf("Creating 1 snapshot")
-	createTestSnapshot(snapshotManager, "repository1", 1, now-3*day-3600, now-3*day-60, []string{chunkHash1, chunkHash2})
-	checkTestSnapshots(snapshotManager, 1, 2)
+	t.Logf("Creating 2 snapshots")
+	createTestSnapshot(snapshotManager, "repository1", 1, now-4*day-3600, now-3*day-60, []string{chunkHash1, chunkHash2}, "tag")
+	createTestSnapshot(snapshotManager, "repository1", 2, now-4*day-3600, now-3*day-60, []string{chunkHash1, chunkHash2}, "tag")
+	checkTestSnapshots(snapshotManager, 2, 2)
 
 	t.Logf("Creating 2 snapshots")
-	createTestSnapshot(snapshotManager, "repository1", 2, now-2*day-3600, now-2*day-60, []string{chunkHash2, chunkHash3})
-	createTestSnapshot(snapshotManager, "repository1", 3, now-1*day-3600, now-1*day-60, []string{chunkHash3, chunkHash4})
-	checkTestSnapshots(snapshotManager, 3, 0)
+	createTestSnapshot(snapshotManager, "repository1", 3, now-2*day-3600, now-2*day-60, []string{chunkHash2, chunkHash3}, "tag")
+	createTestSnapshot(snapshotManager, "repository1", 4, now-1*day-3600, now-1*day-60, []string{chunkHash3, chunkHash4}, "tag")
+	checkTestSnapshots(snapshotManager, 4, 0)
 
-	t.Logf("Removing snapshot repository1 revision 1 with --exclusive")
-	snapshotManager.PruneSnapshots("repository1", "repository1", []int{1}, []string{}, []string{}, false, true, []string{}, false, false, false)
+	t.Logf("Removing snapshot repository1 revisions 1 and 2 with --exclusive")
+	snapshotManager.PruneSnapshots("repository1", "repository1", []int{1, 2}, []string{}, []string{}, false, true, []string{}, false, false, false, 1)
 	checkTestSnapshots(snapshotManager, 2, 0)
 
-	t.Logf("Removing snapshot repository1 revision 2 without --exclusive")
-	snapshotManager.PruneSnapshots("repository1", "repository1", []int{2}, []string{}, []string{}, false, false, []string{}, false, false, false)
+	t.Logf("Removing snapshot repository1 revision 3 without --exclusive")
+	snapshotManager.PruneSnapshots("repository1", "repository1", []int{3}, []string{}, []string{}, false, false, []string{}, false, false, false, 1)
 	checkTestSnapshots(snapshotManager, 1, 2)
 
 	t.Logf("Creating 1 snapshot")
 	chunkHash5 := uploadRandomChunk(snapshotManager, chunkSize)
-	createTestSnapshot(snapshotManager, "repository1", 4, now+1*day-3600, now+1*day, []string{chunkHash4, chunkHash5})
+	createTestSnapshot(snapshotManager, "repository1", 5, now+1*day-3600, now+1*day, []string{chunkHash4, chunkHash5}, "tag")
 	checkTestSnapshots(snapshotManager, 2, 2)
 
 	t.Logf("Prune without removing any snapshots -- fossils will be deleted")
-	snapshotManager.PruneSnapshots("repository1", "repository1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false)
+	snapshotManager.PruneSnapshots("repository1", "repository1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false, 1)
 	checkTestSnapshots(snapshotManager, 2, 0)
 }
 
-func TestSingleHostPrune(t *testing.T) {
+func TestPruneSingleHost(t *testing.T) {
 
 	setTestingT(t)
 
@@ -282,31 +302,31 @@ func TestSingleHostPrune(t *testing.T) {
 	now := time.Now().Unix()
 	day := int64(24 * 3600)
 	t.Logf("Creating 3 snapshots")
-	createTestSnapshot(snapshotManager, "vm1@host1", 1, now-3*day-3600, now-3*day-60, []string{chunkHash1, chunkHash2})
-	createTestSnapshot(snapshotManager, "vm1@host1", 2, now-2*day-3600, now-2*day-60, []string{chunkHash2, chunkHash3})
-	createTestSnapshot(snapshotManager, "vm2@host1", 1, now-3*day-3600, now-3*day-60, []string{chunkHash3, chunkHash4})
+	createTestSnapshot(snapshotManager, "vm1@host1", 1, now-3*day-3600, now-3*day-60, []string{chunkHash1, chunkHash2}, "tag")
+	createTestSnapshot(snapshotManager, "vm1@host1", 2, now-2*day-3600, now-2*day-60, []string{chunkHash2, chunkHash3}, "tag")
+	createTestSnapshot(snapshotManager, "vm2@host1", 1, now-3*day-3600, now-3*day-60, []string{chunkHash3, chunkHash4}, "tag")
 	checkTestSnapshots(snapshotManager, 3, 0)
 
 	t.Logf("Removing snapshot vm1@host1 revision 1 without --exclusive")
-	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{1}, []string{}, []string{}, false, false, []string{}, false, false, false)
+	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{1}, []string{}, []string{}, false, false, []string{}, false, false, false, 1)
 	checkTestSnapshots(snapshotManager, 2, 2)
 
 	t.Logf("Prune without removing any snapshots -- no fossils will be deleted")
-	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false)
+	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false, 1)
 	checkTestSnapshots(snapshotManager, 2, 2)
 
 	t.Logf("Creating 1 snapshot")
 	chunkHash5 := uploadRandomChunk(snapshotManager, chunkSize)
-	createTestSnapshot(snapshotManager, "vm2@host1", 2, now+1*day-3600, now+1*day, []string{chunkHash4, chunkHash5})
+	createTestSnapshot(snapshotManager, "vm2@host1", 2, now+1*day-3600, now+1*day, []string{chunkHash4, chunkHash5}, "tag")
 	checkTestSnapshots(snapshotManager, 3, 2)
 
 	t.Logf("Prune without removing any snapshots -- fossils will be deleted")
-	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false)
+	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false, 1)
 	checkTestSnapshots(snapshotManager, 3, 0)
 
 }
 
-func TestMultipleHostPrune(t *testing.T) {
+func TestPruneMultipleHost(t *testing.T) {
 
 	setTestingT(t)
 
@@ -323,35 +343,35 @@ func TestMultipleHostPrune(t *testing.T) {
 	now := time.Now().Unix()
 	day := int64(24 * 3600)
 	t.Logf("Creating 3 snapshot")
-	createTestSnapshot(snapshotManager, "vm1@host1", 1, now-3*day-3600, now-3*day-60, []string{chunkHash1, chunkHash2})
-	createTestSnapshot(snapshotManager, "vm1@host1", 2, now-2*day-3600, now-2*day-60, []string{chunkHash2, chunkHash3})
-	createTestSnapshot(snapshotManager, "vm2@host2", 1, now-3*day-3600, now-3*day-60, []string{chunkHash3, chunkHash4})
+	createTestSnapshot(snapshotManager, "vm1@host1", 1, now-3*day-3600, now-3*day-60, []string{chunkHash1, chunkHash2}, "tag")
+	createTestSnapshot(snapshotManager, "vm1@host1", 2, now-2*day-3600, now-2*day-60, []string{chunkHash2, chunkHash3}, "tag")
+	createTestSnapshot(snapshotManager, "vm2@host2", 1, now-3*day-3600, now-3*day-60, []string{chunkHash3, chunkHash4}, "tag")
 	checkTestSnapshots(snapshotManager, 3, 0)
 
 	t.Logf("Removing snapshot vm1@host1 revision 1 without --exclusive")
-	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{1}, []string{}, []string{}, false, false, []string{}, false, false, false)
+	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{1}, []string{}, []string{}, false, false, []string{}, false, false, false, 1)
 	checkTestSnapshots(snapshotManager, 2, 2)
 
 	t.Logf("Prune without removing any snapshots -- no fossils will be deleted")
-	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false)
+	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false, 1)
 	checkTestSnapshots(snapshotManager, 2, 2)
 
 	t.Logf("Creating 1 snapshot")
 	chunkHash5 := uploadRandomChunk(snapshotManager, chunkSize)
-	createTestSnapshot(snapshotManager, "vm2@host2", 2, now+1*day-3600, now+1*day, []string{chunkHash4, chunkHash5})
+	createTestSnapshot(snapshotManager, "vm2@host2", 2, now+1*day-3600, now+1*day, []string{chunkHash4, chunkHash5}, "tag")
 	checkTestSnapshots(snapshotManager, 3, 2)
 
 	t.Logf("Prune without removing any snapshots -- no fossils will be deleted")
-	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false)
+	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false, 1)
 	checkTestSnapshots(snapshotManager, 3, 2)
 
 	t.Logf("Creating 1 snapshot")
 	chunkHash6 := uploadRandomChunk(snapshotManager, chunkSize)
-	createTestSnapshot(snapshotManager, "vm1@host1", 3, now+1*day-3600, now+1*day, []string{chunkHash5, chunkHash6})
+	createTestSnapshot(snapshotManager, "vm1@host1", 3, now+1*day-3600, now+1*day, []string{chunkHash5, chunkHash6}, "tag")
 	checkTestSnapshots(snapshotManager, 4, 2)
 
 	t.Logf("Prune without removing any snapshots -- fossils will be deleted")
-	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false)
+	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false, 1)
 	checkTestSnapshots(snapshotManager, 4, 0)
 }
 
@@ -371,25 +391,25 @@ func TestPruneAndResurrect(t *testing.T) {
 	now := time.Now().Unix()
 	day := int64(24 * 3600)
 	t.Logf("Creating 2 snapshots")
-	createTestSnapshot(snapshotManager, "vm1@host1", 1, now-3*day-3600, now-3*day-60, []string{chunkHash1, chunkHash2})
-	createTestSnapshot(snapshotManager, "vm1@host1", 2, now-2*day-3600, now-2*day-60, []string{chunkHash2, chunkHash3})
+	createTestSnapshot(snapshotManager, "vm1@host1", 1, now-3*day-3600, now-3*day-60, []string{chunkHash1, chunkHash2}, "tag")
+	createTestSnapshot(snapshotManager, "vm1@host1", 2, now-2*day-3600, now-2*day-60, []string{chunkHash2, chunkHash3}, "tag")
 	checkTestSnapshots(snapshotManager, 2, 0)
 
 	t.Logf("Removing snapshot vm1@host1 revision 1 without --exclusive")
-	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{1}, []string{}, []string{}, false, false, []string{}, false, false, false)
+	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{1}, []string{}, []string{}, false, false, []string{}, false, false, false, 1)
 	checkTestSnapshots(snapshotManager, 1, 2)
 
 	t.Logf("Creating 1 snapshot")
 	chunkHash4 := uploadRandomChunk(snapshotManager, chunkSize)
-	createTestSnapshot(snapshotManager, "vm1@host1", 4, now+1*day-3600, now+1*day, []string{chunkHash4, chunkHash1})
+	createTestSnapshot(snapshotManager, "vm1@host1", 4, now+1*day-3600, now+1*day, []string{chunkHash4, chunkHash1}, "tag")
 	checkTestSnapshots(snapshotManager, 2, 2)
 
 	t.Logf("Prune without removing any snapshots -- one fossil will be resurrected")
-	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false)
+	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false, 1)
 	checkTestSnapshots(snapshotManager, 2, 0)
 }
 
-func TestInactiveHostPrune(t *testing.T) {
+func TestPruneWithInactiveHost(t *testing.T) {
 
 	setTestingT(t)
 
@@ -406,31 +426,31 @@ func TestInactiveHostPrune(t *testing.T) {
 	now := time.Now().Unix()
 	day := int64(24 * 3600)
 	t.Logf("Creating 3 snapshot")
-	createTestSnapshot(snapshotManager, "vm1@host1", 1, now-3*day-3600, now-3*day-60, []string{chunkHash1, chunkHash2})
-	createTestSnapshot(snapshotManager, "vm1@host1", 2, now-2*day-3600, now-2*day-60, []string{chunkHash2, chunkHash3})
+	createTestSnapshot(snapshotManager, "vm1@host1", 1, now-3*day-3600, now-3*day-60, []string{chunkHash1, chunkHash2}, "tag")
+	createTestSnapshot(snapshotManager, "vm1@host1", 2, now-2*day-3600, now-2*day-60, []string{chunkHash2, chunkHash3}, "tag")
 	// Host2 is inactive
-	createTestSnapshot(snapshotManager, "vm2@host2", 1, now-7*day-3600, now-7*day-60, []string{chunkHash3, chunkHash4})
+	createTestSnapshot(snapshotManager, "vm2@host2", 1, now-7*day-3600, now-7*day-60, []string{chunkHash3, chunkHash4}, "tag")
 	checkTestSnapshots(snapshotManager, 3, 0)
 
 	t.Logf("Removing snapshot vm1@host1 revision 1")
-	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{1}, []string{}, []string{}, false, false, []string{}, false, false, false)
+	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{1}, []string{}, []string{}, false, false, []string{}, false, false, false, 1)
 	checkTestSnapshots(snapshotManager, 2, 2)
 
 	t.Logf("Prune without removing any snapshots -- no fossils will be deleted")
-	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false)
+	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false, 1)
 	checkTestSnapshots(snapshotManager, 2, 2)
 
 	t.Logf("Creating 1 snapshot")
 	chunkHash5 := uploadRandomChunk(snapshotManager, chunkSize)
-	createTestSnapshot(snapshotManager, "vm1@host1", 3, now+1*day-3600, now+1*day, []string{chunkHash4, chunkHash5})
+	createTestSnapshot(snapshotManager, "vm1@host1", 3, now+1*day-3600, now+1*day, []string{chunkHash4, chunkHash5}, "tag")
 	checkTestSnapshots(snapshotManager, 3, 2)
 
 	t.Logf("Prune without removing any snapshots -- fossils will be deleted")
-	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false)
+	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false, 1)
 	checkTestSnapshots(snapshotManager, 3, 0)
 }
 
-func TestRetentionPolicy(t *testing.T) {
+func TestPruneWithRetentionPolicy(t *testing.T) {
 
 	setTestingT(t)
 
@@ -448,20 +468,225 @@ func TestRetentionPolicy(t *testing.T) {
 	day := int64(24 * 3600)
 	t.Logf("Creating 30 snapshots")
 	for i := 0; i < 30; i++ {
-		createTestSnapshot(snapshotManager, "vm1@host1", i+1, now-int64(30-i)*day-3600, now-int64(30-i)*day-60, []string{chunkHashes[i]})
+		createTestSnapshot(snapshotManager, "vm1@host1", i+1, now-int64(30-i)*day-3600, now-int64(30-i)*day-60, []string{chunkHashes[i]}, "tag")
 	}
 
 	checkTestSnapshots(snapshotManager, 30, 0)
 
 	t.Logf("Removing snapshot vm1@host1 0:20 with --exclusive")
-	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{"0:20"}, false, true, []string{}, false, false, false)
+	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{"0:20"}, false, true, []string{}, false, false, false, 1)
 	checkTestSnapshots(snapshotManager, 19, 0)
 
 	t.Logf("Removing snapshot vm1@host1 -k 0:20 with --exclusive")
-	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{"0:20"}, false, true, []string{}, false, false, false)
+	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{"0:20"}, false, true, []string{}, false, false, false, 1)
 	checkTestSnapshots(snapshotManager, 19, 0)
 
 	t.Logf("Removing snapshot vm1@host1 -k 3:14 -k 2:7 with --exclusive")
-	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{"3:14", "2:7"}, false, true, []string{}, false, false, false)
+	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{"3:14", "2:7"}, false, true, []string{}, false, false, false, 1)
 	checkTestSnapshots(snapshotManager, 12, 0)
 }
+
+func TestPruneWithRetentionPolicyAndTag(t *testing.T) {
+
+	setTestingT(t)
+
+	testDir := path.Join(os.TempDir(), "duplicacy_test", "snapshot_test")
+
+	snapshotManager := createTestSnapshotManager(testDir)
+
+	chunkSize := 1024
+	var chunkHashes []string
+	for i := 0; i < 30; i++ {
+		chunkHashes = append(chunkHashes, uploadRandomChunk(snapshotManager, chunkSize))
+	}
+
+	now := time.Now().Unix()
+	day := int64(24 * 3600)
+	t.Logf("Creating 30 snapshots")
+	for i := 0; i < 30; i++ {
+		tag := "auto"
+		if i%3 == 0 {
+			tag = "manual"
+		}
+		createTestSnapshot(snapshotManager, "vm1@host1", i+1, now-int64(30-i)*day-3600, now-int64(30-i)*day-60, []string{chunkHashes[i]}, tag)
+	}
+
+	checkTestSnapshots(snapshotManager, 30, 0)
+
+	t.Logf("Removing snapshot vm1@host1 0:20 with --exclusive and --tag manual")
+	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{"manual"}, []string{"0:7"}, false, true, []string{}, false, false, false, 1)
+	checkTestSnapshots(snapshotManager, 22, 0)
+}
+
+// Test that an unreferenced fossil shouldn't be removed as it may be the result of another prune job in-progress.
+func TestPruneWithFossils(t *testing.T) {
+	setTestingT(t)
+
+	testDir := path.Join(os.TempDir(), "duplicacy_test", "snapshot_test")
+
+	snapshotManager := createTestSnapshotManager(testDir)
+
+	chunkSize := 1024
+	chunkHash1 := uploadRandomChunk(snapshotManager, chunkSize)
+	chunkHash2 := uploadRandomChunk(snapshotManager, chunkSize)
+	chunkHash3 := uploadRandomChunk(snapshotManager, chunkSize)
+	// Create an unreferenced fossil
+	snapshotManager.storage.UploadFile(0, "chunks/113b6a2350dcfd836829c47304dd330fa6b58b93dd7ac696c6b7b913e6868662.fsl", []byte("this is a test fossil"))
+
+	now := time.Now().Unix()
+	day := int64(24 * 3600)
+	t.Logf("Creating 2 snapshots")
+	createTestSnapshot(snapshotManager, "vm1@host1", 1, now-3*day-3600, now-3*day-60, []string{chunkHash1, chunkHash2}, "tag")
+	createTestSnapshot(snapshotManager, "vm1@host1", 2, now-2*day-3600, now-2*day-60, []string{chunkHash2, chunkHash3}, "tag")
+	checkTestSnapshots(snapshotManager, 2, 1)
+
+	t.Logf("Prune without removing any snapshots but with --exhaustive")
+	// The unreferenced fossil shouldn't be removed
+	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, true, false, []string{}, false, false, false, 1)
+	checkTestSnapshots(snapshotManager, 2, 1)
+
+	t.Logf("Prune without removing any snapshots but with --exclusive")
+	// Now the unreferenced fossil should be removed
+	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, true, []string{}, false, false, false, 1)
+	checkTestSnapshots(snapshotManager, 2, 0)
+}
+
+func TestPruneMultipleThread(t *testing.T) {
+
+	setTestingT(t)
+
+	testDir := path.Join(os.TempDir(), "duplicacy_test", "snapshot_test")
+
+	snapshotManager := createTestSnapshotManager(testDir)
+
+	chunkSize := 1024
+	numberOfChunks := 256
+	numberOfThreads := 4
+
+	chunkList1 := uploadRandomChunks(snapshotManager, chunkSize, numberOfChunks)
+	chunkList2 := uploadRandomChunks(snapshotManager, chunkSize, numberOfChunks)
+
+	now := time.Now().Unix()
+	day := int64(24 * 3600)
+	t.Logf("Creating 2 snapshots")
+	createTestSnapshot(snapshotManager, "repository1", 1, now-4*day-3600, now-3*day-60, chunkList1, "tag")
+	createTestSnapshot(snapshotManager, "repository1", 2, now-3*day-3600, now-2*day-60, chunkList2, "tag")
+	checkTestSnapshots(snapshotManager, 2, 0)
+
+	t.Logf("Removing snapshot revisions 1 with --exclusive")
+	snapshotManager.PruneSnapshots("repository1", "repository1", []int{1}, []string{}, []string{}, false, true, []string{}, false, false, false, numberOfThreads)
+	checkTestSnapshots(snapshotManager, 1, 0)
+
+	t.Logf("Creating 1 more snapshot")
+	chunkList3 := uploadRandomChunks(snapshotManager, chunkSize, numberOfChunks)
+	createTestSnapshot(snapshotManager, "repository1", 3, now-2*day-3600, now-1*day-60, chunkList3, "tag")
+
+	t.Logf("Removing snapshot repository1 revision 2 without --exclusive")
+	snapshotManager.PruneSnapshots("repository1", "repository1", []int{2}, []string{}, []string{}, false, false, []string{}, false, false, false, numberOfThreads)
+
+	t.Logf("Prune without removing any snapshots but with --exclusive")
+	snapshotManager.PruneSnapshots("repository1", "repository1", []int{}, []string{}, []string{}, false, true, []string{}, false, false, false, numberOfThreads)
+	checkTestSnapshots(snapshotManager, 1, 0)
+}
+
+// A snapshot not seen by a fossil collection should always be consider a new snapshot in the fossil deletion step
+func TestPruneNewSnapshots(t *testing.T) {
+	setTestingT(t)
+
+	testDir := path.Join(os.TempDir(), "duplicacy_test", "snapshot_test")
+
+	snapshotManager := createTestSnapshotManager(testDir)
+
+	chunkSize := 1024
+	chunkHash1 := uploadRandomChunk(snapshotManager, chunkSize)
+	chunkHash2 := uploadRandomChunk(snapshotManager, chunkSize)
+	chunkHash3 := uploadRandomChunk(snapshotManager, chunkSize)
+	chunkHash4 := uploadRandomChunk(snapshotManager, chunkSize)
+
+	now := time.Now().Unix()
+	day := int64(24 * 3600)
+	t.Logf("Creating 3 snapshots")
+	createTestSnapshot(snapshotManager, "vm1@host1", 1, now-3*day-3600, now-3*day-60, []string{chunkHash1, chunkHash2}, "tag")
+	createTestSnapshot(snapshotManager, "vm1@host1", 2, now-2*day-3600, now-2*day-60, []string{chunkHash2, chunkHash3}, "tag")
+	createTestSnapshot(snapshotManager, "vm2@host1", 1, now-2*day-3600, now-2*day-60, []string{chunkHash3, chunkHash4}, "tag")
+	checkTestSnapshots(snapshotManager, 3, 0)
+
+	t.Logf("Prune snapshot 1")
+	// chunkHash1 should be marked as fossil
+	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{1}, []string{}, []string{}, false, false, []string{}, false, false, false, 1)
+	checkTestSnapshots(snapshotManager, 2, 2)
+
+	chunkHash5 := uploadRandomChunk(snapshotManager, chunkSize)
+	// Create another snapshot of vm1 that brings back chunkHash1
+	createTestSnapshot(snapshotManager, "vm1@host1", 3, now-0*day-3600, now-0*day-60, []string{chunkHash1, chunkHash3}, "tag")
+	// Create another snapshot of vm2 so the fossil collection will be processed by next prune
+	createTestSnapshot(snapshotManager, "vm2@host1", 2, now+3600, now+3600*2, []string{chunkHash4, chunkHash5}, "tag")
+
+	// Now chunkHash1 wil be resurrected
+	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false, 1)
+	checkTestSnapshots(snapshotManager, 4, 0)
+	snapshotManager.CheckSnapshots("vm1@host1", []int{2, 3}, "", false, false, false, false, false, false, false, 1, false)
+}
+
+// A fossil collection left by an aborted prune should be ignored if any supposedly deleted snapshot exists
+func TestPruneGhostSnapshots(t *testing.T) {
+	setTestingT(t)
+
+	EnableStackTrace()
+
+	testDir := path.Join(os.TempDir(), "duplicacy_test", "snapshot_test")
+
+	snapshotManager := createTestSnapshotManager(testDir)
+
+	chunkSize := 1024
+	chunkHash1 := uploadRandomChunk(snapshotManager, chunkSize)
+	chunkHash2 := uploadRandomChunk(snapshotManager, chunkSize)
+	chunkHash3 := uploadRandomChunk(snapshotManager, chunkSize)
+
+	now := time.Now().Unix()
+	day := int64(24 * 3600)
+	t.Logf("Creating 2 snapshots")
+	createTestSnapshot(snapshotManager, "vm1@host1", 1, now-3*day-3600, now-3*day-60, []string{chunkHash1, chunkHash2}, "tag")
+	createTestSnapshot(snapshotManager, "vm1@host1", 2, now-2*day-3600, now-2*day-60, []string{chunkHash2, chunkHash3}, "tag")
+	checkTestSnapshots(snapshotManager, 2, 0)
+
+	snapshot1, err := ioutil.ReadFile(path.Join(testDir, "snapshots", "vm1@host1", "1"))
+	if err != nil {
+		t.Errorf("Failed to read snapshot file: %v", err)
+	}
+
+	t.Logf("Prune snapshot 1")
+	// chunkHash1 should be marked as fossil
+	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{1}, []string{}, []string{}, false, false, []string{}, false, false, false, 1)
+	checkTestSnapshots(snapshotManager, 1, 2)
+
+	// Recover the snapshot file for revision 1; this is to simulate a scenario where prune may encounter a network error after
+	// leaving the fossil collection but before deleting any snapshots.
+	err = ioutil.WriteFile(path.Join(testDir, "snapshots", "vm1@host1", "1"), snapshot1, 0644)
+	if err != nil {
+		t.Errorf("Failed to write snapshot file: %v", err)
+	}
+
+	// Create another snapshot of vm1 so the fossil collection becomes eligible for processing.
+	chunkHash4 := uploadRandomChunk(snapshotManager, chunkSize)
+	createTestSnapshot(snapshotManager, "vm1@host1", 3, now-day-3600, now-day-60, []string{chunkHash3, chunkHash4}, "tag")
+
+	// Run the prune again but the fossil collection should be igored, since revision 1 still exists
+	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false, 1)
+	checkTestSnapshots(snapshotManager, 3, 2)
+	snapshotManager.CheckSnapshots("vm1@host1", []int{1, 2, 3}, "", false, false, false, false, true /*searchFossils*/, false, false, 1, false)
+
+	// Prune snapshot 1 again
+	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{1}, []string{}, []string{}, false, false, []string{}, false, false, false, 1)
+	checkTestSnapshots(snapshotManager, 2, 2)
+
+	// Create another snapshot
+	chunkHash5 := uploadRandomChunk(snapshotManager, chunkSize)
+	createTestSnapshot(snapshotManager, "vm1@host1", 4, now+3600, now+3600*2, []string{chunkHash5, chunkHash5}, "tag")
+	checkTestSnapshots(snapshotManager, 3, 2)
+
+	// Run the prune again and this time the fossil collection will be processed and the fossils removed
+	snapshotManager.PruneSnapshots("vm1@host1", "vm1@host1", []int{}, []string{}, []string{}, false, false, []string{}, false, false, false, 1)
+	checkTestSnapshots(snapshotManager, 3, 0)
+	snapshotManager.CheckSnapshots("vm1@host1", []int{2, 3, 4}, "", false, false, false, false, false, false, false, 1, false)
+}

src/duplicacy_storage.go

@@ -89,7 +89,7 @@ func (storage *StorageBase) SetRateLimits(downloadRateLimit int, uploadRateLimit
 }
 
 // SetDefaultNestingLevels sets the default read and write levels.  This is usually called by
-// derived storages to set the levels with old values so that storages initialied by ealier versions
+// derived storages to set the levels with old values so that storages initialized by earlier versions
 // will continue to work.
 func (storage *StorageBase) SetDefaultNestingLevels(readLevels []int, writeLevel int) {
 	storage.readLevels = readLevels
@@ -109,7 +109,7 @@ func (storage *StorageBase) SetNestingLevels(config *Config) {
 		exist, _, _, err := storage.DerivedStorage.GetFileInfo(0, "nesting")
 		if err == nil && exist {
 			nestingFile := CreateChunk(CreateConfig(), true)
-			if storage.DerivedStorage.DownloadFile(0, "config", nestingFile) == nil {
+			if storage.DerivedStorage.DownloadFile(0, "nesting", nestingFile) == nil {
 				var nesting struct {
 					ReadLevels []int `json:"read-levels"`
 					WriteLevel int   `json:"write-level"`
@@ -261,14 +261,15 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
 		return fileStorage
 	}
 
-	urlRegex := regexp.MustCompile(`^(\w+)://([\w\-]+@)?([^/]+)(/(.+))?`)
+	// Added \! to matched[2] because OneDrive drive ids contain ! (e.g. "b!xxx")
+	urlRegex := regexp.MustCompile(`^([\w-]+)://([\w\-@\.\!]+@)?([^/]+)(/(.+))?`)
 
 	matched := urlRegex.FindStringSubmatch(storageURL)
 
 	if matched == nil {
 		LOG_ERROR("STORAGE_CREATE", "Unrecognizable storage URL: %s", storageURL)
 		return nil
-	} else if matched[1] == "sftp" {
+	} else if matched[1] == "sftp" || matched[1] == "sftpc" {
 		server := matched[3]
 		username := matched[2]
 		storageDir := matched[5]
@@ -291,6 +292,7 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
 
 		// If ssh_key_file is set, skip password-based login
 		keyFile := GetPasswordFromPreference(preference, "ssh_key_file")
+		passphrase := ""
 
 		password := ""
 		passwordCallback := func() (string, error) {
@@ -335,7 +337,7 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
 			keyFile = GetPassword(preference, "ssh_key_file", "Enter the path of the private key file:",
 				true, resetPassword)
 
-			var key ssh.Signer
+			var keySigner ssh.Signer
 			var err error
 
 			if keyFile == "" {
@@ -346,15 +348,52 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
 				if err != nil {
 					LOG_INFO("SSH_PUBLICKEY", "Failed to read the private key file: %v", err)
 				} else {
-					key, err = ssh.ParsePrivateKey(content)
+					keySigner, err = ssh.ParsePrivateKey(content)
 					if err != nil {
-						LOG_INFO("SSH_PUBLICKEY", "Failed to parse the private key file %s: %v", keyFile, err)
+						if _, ok := err.(*ssh.PassphraseMissingError); ok {
+							LOG_TRACE("SSH_PUBLICKEY", "The private key file is encrypted")
+							passphrase = GetPassword(preference, "ssh_passphrase", "Enter the passphrase to decrypt the private key file:", false, resetPassword)
+							if len(passphrase) == 0 {
+								LOG_INFO("SSH_PUBLICKEY", "No passphrase to descrypt the private key file %s", keyFile)
+							} else {
+								keySigner, err = ssh.ParsePrivateKeyWithPassphrase(content, []byte(passphrase))
+								if err != nil {
+									LOG_INFO("SSH_PUBLICKEY", "Failed to parse the encrypted private key file %s: %v", keyFile, err)
+								}
+							}
+						} else {
+							LOG_INFO("SSH_PUBLICKEY", "Failed to parse the private key file %s: %v", keyFile, err)
+						}
+					}
+
+					if keySigner != nil {
+						certFile := keyFile + "-cert.pub"
+						if stat, err := os.Stat(certFile); err == nil && !stat.IsDir() {
+							LOG_DEBUG("SSH_CERTIFICATE", "Attempting to use ssh certificate from file %s", certFile)
+							var content []byte
+							content, err = ioutil.ReadFile(certFile)
+							if err != nil {
+								LOG_INFO("SSH_CERTIFICATE", "Failed to read ssh certificate file %s: %v", certFile, err)
+							} else {
+								pubKey, _, _, _, err := ssh.ParseAuthorizedKey(content)
+								if err != nil {
+									LOG_INFO("SSH_CERTIFICATE", "Failed parse ssh certificate file %s: %v", certFile, err)
+								} else {
+									certSigner, err := ssh.NewCertSigner(pubKey.(*ssh.Certificate), keySigner)
+									if err != nil {
+										LOG_INFO("SSH_CERTIFICATE", "Failed to create certificate signer: %v", err)
+									} else {
+										keySigner = certSigner
+									}
+								}
+							}
+						}
 					}
 				}
 			}
 
-			if key != nil {
-				signers = append(signers, key)
+			if keySigner != nil {
+				signers = append(signers, keySigner)
 			}
 
 			if len(signers) > 0 {
@@ -402,7 +441,7 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
 			return checkHostKey(hostname, remote, key)
 		}
 
-		sftpStorage, err := CreateSFTPStorage(server, port, username, storageDir, 2, authMethods, hostKeyChecker, threads)
+		sftpStorage, err := CreateSFTPStorage(matched[1] == "sftpc", server, port, username, storageDir, 2, authMethods, hostKeyChecker, threads)
 		if err != nil {
 			LOG_ERROR("STORAGE_CREATE", "Failed to load the SFTP storage at %s: %v", storageURL, err)
 			return nil
@@ -410,6 +449,9 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
 
 		if keyFile != "" {
 			SavePassword(preference, "ssh_key_file", keyFile)
+			if passphrase != "" {
+				SavePassword(preference, "ssh_passphrase", passphrase)
+			}
 		} else if password != "" {
 			SavePassword(preference, "ssh_password", password)
 		}
@@ -461,9 +503,45 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
 		SavePassword(preference, "s3_secret", secretKey)
 
 		return storage
+
+	} else if matched[1] == "wasabi" {
+
+		region := matched[2]
+		endpoint := matched[3]
+		bucket := matched[5]
+
+		if region != "" {
+			region = region[:len(region)-1]
+		}
+
+		key := GetPassword(preference, "wasabi_key",
+			"Enter Wasabi key:", true, resetPassword)
+		secret := GetPassword(preference, "wasabi_secret",
+			"Enter Wasabi secret:", true, resetPassword)
+
+		storageDir := ""
+		if strings.Contains(bucket, "/") {
+			firstSlash := strings.Index(bucket, "/")
+			storageDir = bucket[firstSlash+1:]
+			bucket = bucket[:firstSlash]
+		}
+
+		storage, err := CreateWasabiStorage(region, endpoint,
+			bucket, storageDir, key, secret, threads)
+
+		if err != nil {
+			LOG_ERROR("STORAGE_CREATE", "Failed to load the Wasabi storage at %s: %v", storageURL, err)
+			return nil
+		}
+
+		SavePassword(preference, "wasabi_key", key)
+		SavePassword(preference, "wasabi_secret", secret)
+
+		return storage
+
 	} else if matched[1] == "dropbox" {
 		storageDir := matched[3] + matched[5]
-		token := GetPassword(preference, "dropbox_token", "Enter Dropbox access token:", true, resetPassword)
+		token := GetPassword(preference, "dropbox_token", "Enter Dropbox refresh token:", true, resetPassword)
 		dropboxStorage, err := CreateDropboxStorage(token, storageDir, 1, threads)
 		if err != nil {
 			LOG_ERROR("STORAGE_CREATE", "Failed to load the dropbox storage: %v", err)
@@ -473,11 +551,30 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
 		return dropboxStorage
 	} else if matched[1] == "b2" {
 		bucket := matched[3]
+		storageDir := matched[5]
 
-		accountID := GetPassword(preference, "b2_id", "Enter Backblaze Account ID:", true, resetPassword)
-		applicationKey := GetPassword(preference, "b2_key", "Enter Backblaze Application Key:", true, resetPassword)
+		accountID := GetPassword(preference, "b2_id", "Enter Backblaze account or application id:", true, resetPassword)
+		applicationKey := GetPassword(preference, "b2_key", "Enter corresponding Backblaze application key:", true, resetPassword)
 
-		b2Storage, err := CreateB2Storage(accountID, applicationKey, bucket, threads)
+		b2Storage, err := CreateB2Storage(accountID, applicationKey, "", bucket, storageDir, threads)
+		if err != nil {
+			LOG_ERROR("STORAGE_CREATE", "Failed to load the Backblaze B2 storage at %s: %v", storageURL, err)
+			return nil
+		}
+		SavePassword(preference, "b2_id", accountID)
+		SavePassword(preference, "b2_key", applicationKey)
+		return b2Storage
+	} else if matched[1] == "b2-custom" {
+		b2customUrlRegex := regexp.MustCompile(`^b2-custom://([^/]+)/([^/]+)(/(.+))?`)
+		matched := b2customUrlRegex.FindStringSubmatch(storageURL)
+		downloadURL := "https://" + matched[1]
+		bucket := matched[2]
+		storageDir := matched[4]
+
+		accountID := GetPassword(preference, "b2_id", "Enter Backblaze account or application id:", true, resetPassword)
+		applicationKey := GetPassword(preference, "b2_key", "Enter corresponding Backblaze application key:", true, resetPassword)
+
+		b2Storage, err := CreateB2Storage(accountID, applicationKey, downloadURL, bucket, storageDir, threads)
 		if err != nil {
 			LOG_ERROR("STORAGE_CREATE", "Failed to load the Backblaze B2 storage at %s: %v", storageURL, err)
 			return nil
@@ -528,26 +625,60 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
 		SavePassword(preference, "gcs_token", tokenFile)
 		return gcsStorage
 	} else if matched[1] == "gcd" {
+		// Handle writing directly to the root of the drive
+		// For gcd://driveid@/, driveid@ is match[3] not match[2]
+		if matched[2] == "" && strings.HasSuffix(matched[3], "@") {
+			matched[2], matched[3]  = matched[3], matched[2]
+		}
+		driveID := matched[2]
+		if driveID != "" {
+			driveID = driveID[:len(driveID)-1]
+		}
 		storagePath := matched[3] + matched[4]
 		prompt := fmt.Sprintf("Enter the path of the Google Drive token file (downloadable from https://duplicacy.com/gcd_start):")
 		tokenFile := GetPassword(preference, "gcd_token", prompt, true, resetPassword)
-		gcdStorage, err := CreateGCDStorage(tokenFile, storagePath, threads)
+		gcdStorage, err := CreateGCDStorage(tokenFile, driveID, storagePath, threads)
 		if err != nil {
 			LOG_ERROR("STORAGE_CREATE", "Failed to load the Google Drive storage at %s: %v", storageURL, err)
 			return nil
 		}
 		SavePassword(preference, "gcd_token", tokenFile)
 		return gcdStorage
-	} else if matched[1] == "one" {
+	} else if matched[1] == "one" || matched[1] == "odb" {
+		// Handle writing directly to the root of the drive
+		// For odb://drive_id@/, drive_id@ is match[3] not match[2]
+		if matched[2] == "" && strings.HasSuffix(matched[3], "@") {
+			matched[2], matched[3]  = matched[3], matched[2]
+		}
+		drive_id := matched[2]
+		if len(drive_id) > 0 {
+			drive_id = drive_id[:len(drive_id)-1]
+		}
 		storagePath := matched[3] + matched[4]
 		prompt := fmt.Sprintf("Enter the path of the OneDrive token file (downloadable from https://duplicacy.com/one_start):")
-		tokenFile := GetPassword(preference, "one_token", prompt, true, resetPassword)
-		oneDriveStorage, err := CreateOneDriveStorage(tokenFile, storagePath, threads)
+		tokenFile := GetPassword(preference, matched[1] + "_token", prompt, true, resetPassword)
+
+		// client_id, just like tokenFile, can be stored in preferences
+		//prompt = fmt.Sprintf("Enter client_id for custom Azure app (if empty will use duplicacy.com one):")
+		client_id := GetPasswordFromPreference(preference, matched[1] + "_client_id")
+		client_secret := ""
+
+		if client_id != "" {
+			// client_secret should go into keyring
+			prompt = fmt.Sprintf("Enter client_secret for custom Azure app (if empty will use duplicacy.com one):")
+			client_secret = GetPassword(preference, matched[1] + "_client_secret", prompt, true, resetPassword)
+		}
+
+		oneDriveStorage, err := CreateOneDriveStorage(tokenFile, matched[1] == "odb", storagePath, threads, client_id, client_secret, drive_id)
 		if err != nil {
 			LOG_ERROR("STORAGE_CREATE", "Failed to load the OneDrive storage at %s: %v", storageURL, err)
 			return nil
 		}
-		SavePassword(preference, "one_token", tokenFile)
+
+		SavePassword(preference, matched[1] + "_token", tokenFile)
+		if client_id != "" {
+			SavePassword(preference, matched[1] + "_client_secret", client_secret)
+		}
 		return oneDriveStorage
 	} else if matched[1] == "hubic" {
 		storagePath := matched[3] + matched[4]
@@ -560,6 +691,111 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
 		}
 		SavePassword(preference, "hubic_token", tokenFile)
 		return hubicStorage
+	} else if matched[1] == "swift" {
+		prompt := fmt.Sprintf("Enter the OpenStack Swift key:")
+		key := GetPassword(preference, "swift_key", prompt, true, resetPassword)
+		swiftStorage, err := CreateSwiftStorage(storageURL[8:], key, threads)
+		if err != nil {
+			LOG_ERROR("STORAGE_CREATE", "Failed to load the OpenStack Swift storage at %s: %v", storageURL, err)
+			return nil
+		}
+		SavePassword(preference, "swift_key", key)
+		return swiftStorage
+	} else if matched[1] == "webdav" || matched[1] == "webdav-http" {
+		server := matched[3]
+		username := matched[2]
+		if username == "" {
+			LOG_ERROR("STORAGE_CREATE", "No username is provided to access the WebDAV storage")
+			return nil
+		}
+		username = username[:len(username)-1]
+		storageDir := matched[5]
+		port := 0
+		useHTTP := matched[1] == "webdav-http"
+
+		if strings.Contains(server, ":") {
+			index := strings.Index(server, ":")
+			port, _ = strconv.Atoi(server[index+1:])
+			server = server[:index]
+		}
+
+		prompt := fmt.Sprintf("Enter the WebDAV password:")
+		password := GetPassword(preference, "webdav_password", prompt, true, resetPassword)
+		webDAVStorage, err := CreateWebDAVStorage(server, port, username, password, storageDir, useHTTP, threads)
+		if err != nil {
+			LOG_ERROR("STORAGE_CREATE", "Failed to load the WebDAV storage at %s: %v", storageURL, err)
+			return nil
+		}
+		SavePassword(preference, "webdav_password", password)
+		return webDAVStorage
+	} else if matched[1] == "fabric" {
+		endpoint := matched[3]
+		storageDir := matched[5]
+		prompt := fmt.Sprintf("Enter the token for accessing the Storage Made Easy File Fabric storage:")
+		token := GetPassword(preference, "fabric_token", prompt, true, resetPassword)
+		smeStorage, err := CreateFileFabricStorage(endpoint, token, storageDir, threads)
+		if err != nil {
+			LOG_ERROR("STORAGE_CREATE", "Failed to load the File Fabric storage at %s: %v", storageURL, err)
+			return nil
+		}
+		SavePassword(preference, "fabric_token", token)
+		return smeStorage
+	} else if matched[1] == "storj" {
+		satellite := matched[2] + matched[3]
+		bucket := matched[5]
+		storageDir := ""
+		index := strings.Index(bucket, "/")
+		if index >= 0 {
+			storageDir = bucket[index + 1:]
+			bucket = bucket[:index]
+		}
+		apiKey := GetPassword(preference, "storj_key", "Enter the API access key:", true, resetPassword)
+		passphrase := GetPassword(preference, "storj_passphrase", "Enter the passphrase:", true, resetPassword)
+		storjStorage, err := CreateStorjStorage(satellite, apiKey, passphrase, bucket, storageDir, threads)
+		if err != nil {
+			LOG_ERROR("STORAGE_CREATE", "Failed to load the Storj storage at %s: %v", storageURL, err)
+			return nil
+		}
+		SavePassword(preference, "storj_key", apiKey)
+		SavePassword(preference, "storj_passphrase", passphrase)
+		return storjStorage
+	} else if matched[1] == "smb" {
+		server := matched[3]
+		username := matched[2]
+		if username == "" {
+			LOG_ERROR("STORAGE_CREATE", "No username is provided to access the SAMBA storage")
+			return nil
+		}
+		username = username[:len(username)-1]
+		storageDir := matched[5]
+		port := 445
+
+		if strings.Contains(server, ":") {
+			index := strings.Index(server, ":")
+			port, _ = strconv.Atoi(server[index+1:])
+			server = server[:index]
+		}
+
+		if !strings.Contains(storageDir, "/") {
+			LOG_ERROR("STORAGE_CREATE", "No share name specified for the SAMBA storage")
+			return nil
+		}
+
+		index := strings.Index(storageDir, "/")
+		shareName := storageDir[:index]
+		storageDir = storageDir[index+1:]
+
+		prompt := fmt.Sprintf("Enter the SAMBA password:")
+		password := GetPassword(preference, "smb_password", prompt, true, resetPassword)
+		sambaStorage, err := CreateSambaStorage(server, port, username, password, shareName, storageDir, threads)
+		if err != nil {
+			LOG_ERROR("STORAGE_CREATE", "Failed to load the SAMBA storage at %s: %v", storageURL, err)
+			return nil
+		}
+		SavePassword(preference, "smb_password", password)
+		return sambaStorage
+
+
 	} else {
 		LOG_ERROR("STORAGE_CREATE", "The storage type '%s' is not supported", matched[1])
 		return nil

src/duplicacy_storage_test.go

@@ -22,24 +22,17 @@ import (
 	"math/rand"
 )
 
-var testStorageName string
-var testRateLimit int
-var testQuickMode bool
-var testThreads int
-var testFixedChunkSize bool
-
-func init() {
-	flag.StringVar(&testStorageName, "storage", "", "the test storage to use")
-	flag.IntVar(&testRateLimit, "limit-rate", 0, "maximum transfer speed in kbytes/sec")
-	flag.BoolVar(&testQuickMode, "quick", false, "quick test")
-	flag.IntVar(&testThreads, "threads", 1, "number of downloading/uploading threads")
-	flag.BoolVar(&testFixedChunkSize, "fixed-chunk-size", false, "fixed chunk size")
-	flag.Parse()
-}
+var testStorageName = flag.String("storage", "", "the test storage to use")
+var testRateLimit = flag.Int("limit-rate", 0, "maximum transfer speed in kbytes/sec")
+var testQuickMode = flag.Bool("quick", false, "quick test")
+var testThreads = flag.Int("threads", 1, "number of downloading/uploading threads")
+var testFixedChunkSize = flag.Bool("fixed-chunk-size", false, "fixed chunk size")
+var testRSAEncryption = flag.Bool("rsa", false, "enable RSA encryption")
+var testErasureCoding = flag.Bool("erasure-coding", false, "enable Erasure Coding")
 
 func loadStorage(localStoragePath string, threads int) (Storage, error) {
 
-	if testStorageName == "" || testStorageName == "file" {
+	if *testStorageName == "" || *testStorageName == "file" {
 		storage, err := CreateFileStorage(localStoragePath, false, threads)
 		if storage != nil {
 			// Use a read level of at least 2 because this will catch more errors than a read level of 1.
@@ -60,87 +53,147 @@ func loadStorage(localStoragePath string, threads int) (Storage, error) {
 		return nil, err
 	}
 
-	config, found := configs[testStorageName]
+	config, found := configs[*testStorageName]
 	if !found {
-		return nil, fmt.Errorf("No storage named '%s' found", testStorageName)
+		return nil, fmt.Errorf("No storage named '%s' found", *testStorageName)
 	}
 
-	if testStorageName == "flat" {
+	if *testStorageName == "flat" {
 		storage, err := CreateFileStorage(localStoragePath, false, threads)
 		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
 		return storage, err
-	} else if testStorageName == "samba" {
+	} else if *testStorageName == "samba" {
 		storage, err := CreateFileStorage(localStoragePath, true, threads)
 		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
 		return storage, err
-	} else if testStorageName == "sftp" {
+	} else if *testStorageName == "sftp" {
 		port, _ := strconv.Atoi(config["port"])
 		storage, err := CreateSFTPStorageWithPassword(config["server"], port, config["username"], config["directory"], 2, config["password"], threads)
 		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
 		return storage, err
-	} else if testStorageName == "s3" || testStorageName == "wasabi" {
+	} else if *testStorageName == "s3" {
 		storage, err := CreateS3Storage(config["region"], config["endpoint"], config["bucket"], config["directory"], config["access_key"], config["secret_key"], threads, true, false)
 		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
 		return storage, err
-	} else if testStorageName == "s3c" {
+	} else if *testStorageName == "wasabi" {
+		storage, err := CreateWasabiStorage(config["region"], config["endpoint"], config["bucket"], config["directory"], config["access_key"], config["secret_key"], threads)
+		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
+		return storage, err
+	} else if *testStorageName == "s3c" {
 		storage, err := CreateS3CStorage(config["region"], config["endpoint"], config["bucket"], config["directory"], config["access_key"], config["secret_key"], threads)
 		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
 		return storage, err
-	} else if testStorageName == "digitalocean" {
+	} else if *testStorageName == "digitalocean" {
 		storage, err := CreateS3CStorage(config["region"], config["endpoint"], config["bucket"], config["directory"], config["access_key"], config["secret_key"], threads)
 		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
 		return storage, err
-	} else if testStorageName == "minio" {
+	} else if *testStorageName == "minio" {
 		storage, err := CreateS3Storage(config["region"], config["endpoint"], config["bucket"], config["directory"], config["access_key"], config["secret_key"], threads, false, true)
 		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
 		return storage, err
-	} else if testStorageName == "minios" {
+	} else if *testStorageName == "minios" {
 		storage, err := CreateS3Storage(config["region"], config["endpoint"], config["bucket"], config["directory"], config["access_key"], config["secret_key"], threads, true, true)
 		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
 		return storage, err
-	} else if testStorageName == "dropbox" {
+	} else if *testStorageName == "dropbox" {
 		storage, err := CreateDropboxStorage(config["token"], config["directory"], 1, threads)
 		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
 		return storage, err
-	} else if testStorageName == "b2" {
-		storage, err := CreateB2Storage(config["account"], config["key"], config["bucket"], threads)
+	} else if *testStorageName == "b2" {
+		storage, err := CreateB2Storage(config["account"], config["key"], "", config["bucket"], config["directory"], threads)
 		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
 		return storage, err
-	} else if testStorageName == "gcs-s3" {
+	} else if *testStorageName == "gcs-s3" {
 		storage, err := CreateS3Storage(config["region"], config["endpoint"], config["bucket"], config["directory"], config["access_key"], config["secret_key"], threads, true, false)
 		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
 		return storage, err
-	} else if testStorageName == "gcs" {
+	} else if *testStorageName == "gcs" {
 		storage, err := CreateGCSStorage(config["token_file"], config["bucket"], config["directory"], threads)
 		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
 		return storage, err
-	} else if testStorageName == "gcs-sa" {
+	} else if *testStorageName == "gcs-sa" {
 		storage, err := CreateGCSStorage(config["token_file"], config["bucket"], config["directory"], threads)
 		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
 		return storage, err
-	} else if testStorageName == "azure" {
+	} else if *testStorageName == "azure" {
 		storage, err := CreateAzureStorage(config["account"], config["key"], config["container"], threads)
 		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
 		return storage, err
-	} else if testStorageName == "acd" {
+	} else if *testStorageName == "acd" {
 		storage, err := CreateACDStorage(config["token_file"], config["storage_path"], threads)
 		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
 		return storage, err
-	} else if testStorageName == "gcd" {
-		storage, err := CreateGCDStorage(config["token_file"], config["storage_path"], threads)
+	} else if *testStorageName == "gcd" {
+		storage, err := CreateGCDStorage(config["token_file"], "", config["storage_path"], threads)
 		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
 		return storage, err
-	} else if testStorageName == "one" {
-		storage, err := CreateOneDriveStorage(config["token_file"], config["storage_path"], threads)
+	} else if *testStorageName == "gcd-shared" {
+		storage, err := CreateGCDStorage(config["token_file"], config["drive"], config["storage_path"], threads)
 		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
 		return storage, err
-	} else if testStorageName == "hubic" {
+	} else if *testStorageName == "gcd-impersonate" {
+		storage, err := CreateGCDStorage(config["token_file"], config["drive"], config["storage_path"], threads)
+		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
+		return storage, err
+	} else if *testStorageName == "one" {
+		storage, err := CreateOneDriveStorage(config["token_file"], false, config["storage_path"], threads, "", "", "")
+		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
+		return storage, err
+	} else if *testStorageName == "odb" {
+		storage, err := CreateOneDriveStorage(config["token_file"], true, config["storage_path"], threads, "", "", "")
+		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
+		return storage, err
+	} else if *testStorageName == "one" {
+		storage, err := CreateOneDriveStorage(config["token_file"], false, config["storage_path"], threads, "", "", "")
+		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
+		return storage, err
+	} else if *testStorageName == "hubic" {
 		storage, err := CreateHubicStorage(config["token_file"], config["storage_path"], threads)
 		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
 		return storage, err
-	} else {
-		return nil, fmt.Errorf("Invalid storage named: %s", testStorageName)
+	} else if *testStorageName == "memset" {
+		storage, err := CreateSwiftStorage(config["storage_url"], config["key"], threads)
+		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
+		return storage, err
+	} else if *testStorageName == "pcloud" || *testStorageName == "box" {
+		storage, err := CreateWebDAVStorage(config["host"], 0, config["username"], config["password"], config["storage_path"], false, threads)
+		if err != nil {
+			return nil, err
+		}
+		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
+		return storage, err
+	} else if *testStorageName == "fabric" {
+		storage, err := CreateFileFabricStorage(config["endpoint"], config["token"], config["storage_path"], threads)
+		if err != nil {
+			return nil, err
+		}
+		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
+		return storage, err
+	} else if *testStorageName == "storj" {
+		storage, err := CreateStorjStorage(config["satellite"], config["key"], config["passphrase"], config["bucket"], config["storage_path"], threads)
+		if err != nil {
+			return nil, err
+		}
+		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
+		return storage, err
+	} else if *testStorageName == "storj" {
+		storage, err := CreateStorjStorage(config["satellite"], config["key"], config["passphrase"], config["bucket"], config["storage_path"], threads)
+		if err != nil {
+			return nil, err
+		}
+		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
+		return storage, err
+	} else if *testStorageName == "smb" {
+		port, _ := strconv.Atoi(config["port"])
+		storage, err := CreateSambaStorage(config["server"], port, config["username"], config["password"], config["share"], config["storage_path"], threads)
+		if err != nil {
+			return nil, err
+		}
+		storage.SetDefaultNestingLevels([]int{2, 3}, 2)
+		return storage, err
 	}
+
+	return nil, fmt.Errorf("Invalid storage named: %s", *testStorageName)
 }
 
 func cleanStorage(storage Storage) {
@@ -280,15 +333,16 @@ func TestStorage(t *testing.T) {
 	os.RemoveAll(testDir)
 	os.MkdirAll(testDir, 0700)
 
-	LOG_INFO("STORAGE_TEST", "storage: %s", testStorageName)
+	LOG_INFO("STORAGE_TEST", "storage: %s", *testStorageName)
 
-	storage, err := loadStorage(testDir, 1)
+	threads := 8
+	storage, err := loadStorage(testDir, threads)
 	if err != nil {
 		t.Errorf("Failed to create storage: %v", err)
 		return
 	}
 	storage.EnableTestMode()
-	storage.SetRateLimits(testRateLimit, testRateLimit)
+	storage.SetRateLimits(*testRateLimit, *testRateLimit)
 
 	delay := 0
 	if _, ok := storage.(*ACDStorage); ok {
@@ -312,16 +366,16 @@ func TestStorage(t *testing.T) {
 	storage.CreateDirectory(0, "shared")
 
 	// Upload to the same directory by multiple goroutines
-	count := 8
+	count := threads
 	finished := make(chan int, count)
 	for i := 0; i < count; i++ {
-		go func(name string) {
-			err := storage.UploadFile(0, name, []byte("this is a test file"))
+		go func(threadIndex int, name string) {
+			err := storage.UploadFile(threadIndex, name, []byte("this is a test file"))
 			if err != nil {
 				t.Errorf("Error to upload '%s': %v", name, err)
 			}
 			finished <- 0
-		}(fmt.Sprintf("shared/a/b/c/%d", i))
+		}(i, fmt.Sprintf("shared/a/b/c/%d", i))
 	}
 
 	for i := 0; i < count; i++ {
@@ -410,7 +464,7 @@ func TestStorage(t *testing.T) {
 	numberOfFiles := 10
 	maxFileSize := 64 * 1024
 
-	if testQuickMode {
+	if *testQuickMode {
 		numberOfFiles = 2
 	}
 
@@ -496,7 +550,7 @@ func TestStorage(t *testing.T) {
 	} else {
 		err = storage.DeleteFile(0, filePath)
 		if err != nil {
-			t.Errorf("Failed to delete file %s: %v", filePath)
+			t.Errorf("Failed to delete file %s: %v", filePath, err)
 		} else {
 			time.Sleep(time.Duration(delay) * time.Second)
 			filePath, exist, _, err = storage.FindChunk(0, chunks[1], true)
@@ -545,7 +599,7 @@ func TestCleanStorage(t *testing.T) {
 	os.RemoveAll(testDir)
 	os.MkdirAll(testDir, 0700)
 
-	LOG_INFO("STORAGE_TEST", "storage: %s", testStorageName)
+	LOG_INFO("STORAGE_TEST", "storage: %s", *testStorageName)
 
 	storage, err := loadStorage(testDir, 1)
 	if err != nil {
@@ -583,12 +637,11 @@ func TestCleanStorage(t *testing.T) {
 	storage.DeleteFile(0, "config")
 	LOG_INFO("DELETE_FILE", "Deleted config")
 
-
 	files, _, err := storage.ListFiles(0, "chunks/")
 	for _, file := range files {
-			if len(file) > 0 && file[len(file)-1] != '/' {
-				LOG_DEBUG("FILE_EXIST", "File %s exists after deletion", file)
-			}
+		if len(file) > 0 && file[len(file)-1] != '/' {
+			LOG_DEBUG("FILE_EXIST", "File %s exists after deletion", file)
+		}
 	}
 
 }

src/duplicacy_storjstorage.go

@@ -0,0 +1,184 @@
+// Copyright (c) Acrosync LLC. All rights reserved.
+// Free for personal use and commercial trial
+// Commercial use requires per-user licenses available from https://duplicacy.com
+
+package duplicacy
+
+import (
+    "fmt"
+    "io"
+    "context"
+
+    "storj.io/uplink"
+)
+
+// StorjStorage is a storage backend for Storj.
+type StorjStorage struct {
+    StorageBase
+
+    project         *uplink.Project
+    bucket          string
+    storageDir      string
+    numberOfThreads int
+}
+
+// CreateStorjStorage creates a Storj storage.
+func CreateStorjStorage(satellite string, apiKey string, passphrase string,
+                        bucket string, storageDir string, threads int) (storage *StorjStorage, err error) {
+
+    ctx := context.Background()
+    access, err := uplink.RequestAccessWithPassphrase(ctx, satellite, apiKey, passphrase)
+    if err != nil {
+        return nil, fmt.Errorf("cannot request the access grant: %v", err)
+    }
+
+    project, err := uplink.OpenProject(ctx, access)
+    if err != nil {
+        return nil, fmt.Errorf("cannot open the project: %v", err)
+    }
+
+    _, err = project.StatBucket(ctx, bucket)
+    if err != nil {
+        return nil, fmt.Errorf("cannot found the bucket: %v", err)
+    }
+
+    if storageDir != "" && storageDir[len(storageDir) - 1] != '/' {
+        storageDir += "/"
+    }
+
+    storage = &StorjStorage {
+        project: project,
+        bucket: bucket,
+        storageDir: storageDir,
+        numberOfThreads: threads,
+    }
+
+    storage.DerivedStorage = storage
+    storage.SetDefaultNestingLevels([]int{2, 3}, 2)
+    return storage, nil
+}
+
+// ListFiles return the list of files and subdirectories under 'dir' (non-recursively).
+func (storage *StorjStorage) ListFiles(threadIndex int, dir string) (
+                                         files []string, sizes []int64, err error) {
+
+    fullPath := storage.storageDir + dir
+    if fullPath != "" && fullPath[len(fullPath) - 1] != '/' {
+        fullPath += "/"
+    }
+
+    options := uplink.ListObjectsOptions {
+        Prefix: fullPath,
+        System: true, // request SystemMetadata which includes ContentLength 
+    }
+    objects := storage.project.ListObjects(context.Background(), storage.bucket, &options)
+    for objects.Next() {
+        if objects.Err() != nil {
+            return nil, nil, objects.Err()
+        }
+        item := objects.Item()
+        name := item.Key[len(fullPath):]
+        size := item.System.ContentLength
+        if item.IsPrefix {
+            if name != "" && name[len(name) - 1] != '/' {
+                name += "/"
+                size = 0
+            }
+        }
+        files = append(files, name)
+        sizes = append(sizes, size)
+    }
+
+    return files, sizes, nil
+}
+
+// DeleteFile deletes the file or directory at 'filePath'.
+func (storage *StorjStorage) DeleteFile(threadIndex int, filePath string) (err error) {
+
+    _, err = storage.project.DeleteObject(context.Background(), storage.bucket,
+                                          storage.storageDir + filePath)
+    return err
+}
+
+// MoveFile renames the file.
+func (storage *StorjStorage) MoveFile(threadIndex int, from string, to string) (err error) {
+    err = storage.project.MoveObject(context.Background(), storage.bucket,
+                                     storage.storageDir + from, storage.bucket, storage.storageDir + to, nil)
+
+    return err
+}
+
+// CreateDirectory creates a new directory.
+func (storage *StorjStorage) CreateDirectory(threadIndex int, dir string) (err error) {
+    return nil
+}
+
+// GetFileInfo returns the information about the file or directory at 'filePath'.
+func (storage *StorjStorage) GetFileInfo(threadIndex int, filePath string) (
+                                         exist bool, isDir bool, size int64, err error) {
+    info, err := storage.project.StatObject(context.Background(), storage.bucket,
+                                            storage.storageDir + filePath)
+
+    if info == nil {
+        return false, false, 0, nil
+    } else if err != nil {
+        return false, false, 0, err
+    } else {
+        return true, info.IsPrefix, info.System.ContentLength, nil
+    }
+}
+
+// DownloadFile reads the file at 'filePath' into the chunk.
+func (storage *StorjStorage) DownloadFile(threadIndex int, filePath string, chunk *Chunk) (err error) {
+    file, err := storage.project.DownloadObject(context.Background(), storage.bucket,
+                                                storage.storageDir + filePath, nil)
+    if err != nil {
+        return err
+    }
+    defer file.Close()
+    
+    if _, err = RateLimitedCopy(chunk, file, storage.DownloadRateLimit/storage.numberOfThreads); err != nil {
+        return err
+    }
+
+    return nil    
+}
+
+// UploadFile writes 'content' to the file at 'filePath'
+func (storage *StorjStorage) UploadFile(threadIndex int, filePath string, content []byte) (err error) {
+
+    file, err := storage.project.UploadObject(context.Background(), storage.bucket,
+                                              storage.storageDir + filePath, nil)
+    if err != nil {
+        return err
+    }
+                                
+    reader := CreateRateLimitedReader(content, storage.UploadRateLimit/storage.numberOfThreads)
+    _, err = io.Copy(file, reader)
+    if err != nil {
+        return err
+    }
+
+    err = file.Commit()
+    if err != nil {
+        return err
+    }
+
+    return nil
+}
+
+// If a local snapshot cache is needed for the storage to avoid downloading/uploading chunks too often when
+// managing snapshots.
+func (storage *StorjStorage) IsCacheNeeded() bool { return true }
+
+// If the 'MoveFile' method is implemented.
+func (storage *StorjStorage) IsMoveFileImplemented() bool { return true }
+
+// If the storage can guarantee strong consistency.
+func (storage *StorjStorage) IsStrongConsistent() bool { return false }
+
+// If the storage supports fast listing of files names.
+func (storage *StorjStorage) IsFastListing() bool { return true }
+
+// Enable the test mode.
+func (storage *StorjStorage) EnableTestMode() {}

src/duplicacy_swiftstorage.go

@@ -0,0 +1,266 @@
+// Copyright (c) Acrosync LLC. All rights reserved.
+// Free for personal use and commercial trial
+// Commercial use requires per-user licenses available from https://duplicacy.com
+
+package duplicacy
+
+import (
+	"context"
+	"strconv"
+	"strings"
+	"time"
+	"regexp"
+
+	"github.com/ncw/swift/v2"
+)
+
+type SwiftStorage struct {
+	StorageBase
+
+	ctx        context.Context
+	connection *swift.Connection
+	container  string
+	storageDir string
+	threads    int
+}
+
+// CreateSwiftStorage creates an OpenStack Swift storage object.  storageURL is in the form of
+// `user@authURL/container/path?arg1=value1&arg2=value2``
+func CreateSwiftStorage(storageURL string, key string, threads int) (storage *SwiftStorage, err error) {
+
+	// This is the map to store all arguments
+	arguments := make(map[string]string)
+
+	// Check if there are arguments provided as a query string
+	if strings.Contains(storageURL, "?") {
+		urlAndArguments := strings.SplitN(storageURL, "?", 2)
+		storageURL = urlAndArguments[0]
+		for _, pair := range strings.Split(urlAndArguments[1], "&") {
+			if strings.Contains(pair, "=") {
+				keyAndValue := strings.Split(pair, "=")
+				arguments[keyAndValue[0]] = keyAndValue[1]
+			}
+		}
+	}
+
+	// The version is used to split authURL and container/path
+	versions := []string{"/v1/", "/v1.0/", "/v2/", "/v2.0/", "/v3/", "/v3.0/", "/v4/", "/v4.0/"}
+	storageDir := ""
+	for _, version := range versions {
+		if strings.Contains(storageURL, version) {
+			urlAndStorageDir := strings.SplitN(storageURL, version, 2)
+			storageURL = urlAndStorageDir[0] + version[0:len(version)-1]
+			storageDir = urlAndStorageDir[1]
+		}
+	}
+
+	// Take out the user name if there is one
+	if strings.Contains(storageURL, "@") {
+		// Use regex to split the username and the rest of the URL
+		lineRegex := regexp.MustCompile(`^(.+)@([^@]+)$`)
+		match := lineRegex.FindStringSubmatch(storageURL)
+		if match != nil {
+			arguments["user"] = match[1]
+			storageURL = match[2]
+		}
+	}
+
+	// If no container/path is specified, find them from the arguments
+	if storageDir == "" {
+		storageDir = arguments["storage_dir"]
+	}
+
+	// Now separate the container name from the storage path
+	container := ""
+	if strings.Contains(storageDir, "/") {
+		containerAndStorageDir := strings.SplitN(storageDir, "/", 2)
+		container = containerAndStorageDir[0]
+		storageDir = containerAndStorageDir[1]
+		if len(storageDir) > 0 && storageDir[len(storageDir)-1] != '/' {
+			storageDir += "/"
+		}
+	} else {
+		container = storageDir
+		storageDir = ""
+	}
+
+	// Number of retries on err
+	retries := 4
+	if value, ok := arguments["retries"]; ok {
+		retries, _ = strconv.Atoi(value)
+	}
+
+	// Connect channel timeout
+	connectionTimeout := 10
+	if value, ok := arguments["connection_timeout"]; ok {
+		connectionTimeout, _ = strconv.Atoi(value)
+	}
+
+	// Data channel timeout
+	timeout := 60
+	if value, ok := arguments["timeout"]; ok {
+		timeout, _ = strconv.Atoi(value)
+	}
+
+	// Auth version; default to auto-detect
+	authVersion := 0
+	if value, ok := arguments["auth_version"]; ok {
+		authVersion, _ = strconv.Atoi(value)
+	}
+
+	// Allow http to be used by setting "protocol=http" in arguments
+	if _, ok := arguments["protocol"]; !ok {
+		arguments["protocol"] = "https"
+	}
+
+	ctx, _ := context.WithTimeout(context.Background(), time.Duration(timeout)*time.Second)
+
+	// Please refer to https://godoc.org/github.com/ncw/swift#Connection
+	connection := swift.Connection{
+		Domain:         arguments["domain"],
+		DomainId:       arguments["domain_id"],
+		UserName:       arguments["user"],
+		UserId:         arguments["user_id"],
+		ApiKey:         key,
+		AuthUrl:        arguments["protocol"] + "://" + storageURL,
+		Retries:        retries,
+		UserAgent:      arguments["user_agent"],
+		ConnectTimeout: time.Duration(connectionTimeout) * time.Second,
+		Timeout:        time.Duration(timeout) * time.Second,
+		Region:         arguments["region"],
+		AuthVersion:    authVersion,
+		Internal:       false,
+		Tenant:         arguments["tenant"],
+		TenantId:       arguments["tenant_id"],
+		EndpointType:   swift.EndpointType(arguments["endpiont_type"]),
+		TenantDomain:   arguments["tenant_domain"],
+		TenantDomainId: arguments["tenant_domain_id"],
+		TrustId:        arguments["trust_id"],
+	}
+
+	err = connection.Authenticate(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	_, _, err = connection.Container(ctx, container)
+	if err != nil {
+		return nil, err
+	}
+
+	storage = &SwiftStorage{
+		ctx:        ctx,
+		connection: &connection,
+		container:  container,
+		storageDir: storageDir,
+		threads:    threads,
+	}
+
+	storage.DerivedStorage = storage
+	storage.SetDefaultNestingLevels([]int{1}, 1)
+	return storage, nil
+}
+
+// ListFiles return the list of files and subdirectories under 'dir' (non-recursively)
+func (storage *SwiftStorage) ListFiles(threadIndex int, dir string) (files []string, sizes []int64, err error) {
+	if len(dir) > 0 && dir[len(dir)-1] != '/' {
+		dir += "/"
+	}
+	isSnapshotDir := dir == "snapshots/"
+	dir = storage.storageDir + dir
+
+	options := swift.ObjectsOpts{
+		Prefix: dir,
+		Limit:  1000,
+	}
+
+	if isSnapshotDir {
+		options.Delimiter = '/'
+	}
+
+	objects, err := storage.connection.ObjectsAll(storage.ctx, storage.container, &options)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	for _, obj := range objects {
+		if isSnapshotDir {
+			if obj.SubDir != "" {
+				files = append(files, obj.SubDir[len(dir):])
+				sizes = append(sizes, 0)
+			}
+		} else {
+			files = append(files, obj.Name[len(dir):])
+			sizes = append(sizes, obj.Bytes)
+		}
+	}
+
+	return files, sizes, nil
+}
+
+// DeleteFile deletes the file or directory at 'filePath'.
+func (storage *SwiftStorage) DeleteFile(threadIndex int, filePath string) (err error) {
+	return storage.connection.ObjectDelete(storage.ctx, storage.container, storage.storageDir+filePath)
+}
+
+// MoveFile renames the file.
+func (storage *SwiftStorage) MoveFile(threadIndex int, from string, to string) (err error) {
+	return storage.connection.ObjectMove(storage.ctx, storage.container, storage.storageDir+from,
+		storage.container, storage.storageDir+to)
+}
+
+// CreateDirectory creates a new directory.
+func (storage *SwiftStorage) CreateDirectory(threadIndex int, dir string) (err error) {
+	// Does nothing as directories do not exist in OpenStack Swift
+	return nil
+}
+
+// GetFileInfo returns the information about the file or directory at 'filePath'.
+func (storage *SwiftStorage) GetFileInfo(threadIndex int, filePath string) (exist bool, isDir bool, size int64, err error) {
+	object, _, err := storage.connection.Object(storage.ctx, storage.container, storage.storageDir+filePath)
+
+	if err != nil {
+		if err == swift.ObjectNotFound {
+			return false, false, 0, nil
+		} else {
+			return false, false, 0, err
+		}
+	}
+
+	return true, false, object.Bytes, nil
+}
+
+// DownloadFile reads the file at 'filePath' into the chunk.
+func (storage *SwiftStorage) DownloadFile(threadIndex int, filePath string, chunk *Chunk) (err error) {
+
+	file, _, err := storage.connection.ObjectOpen(storage.ctx, storage.container, storage.storageDir+filePath, false, nil)
+	if err != nil {
+		return err
+	}
+	_, err = RateLimitedCopy(chunk, file, storage.DownloadRateLimit/storage.threads)
+	return err
+}
+
+// UploadFile writes 'content' to the file at 'filePath'.
+func (storage *SwiftStorage) UploadFile(threadIndex int, filePath string, content []byte) (err error) {
+	reader := CreateRateLimitedReader(content, storage.UploadRateLimit/storage.threads)
+	_, err = storage.connection.ObjectPut(storage.ctx, storage.container, storage.storageDir+filePath, reader, true, "", "application/duplicacy", nil)
+	return err
+}
+
+// If a local snapshot cache is needed for the storage to avoid downloading/uploading chunks too often when
+// managing snapshots.
+func (storage *SwiftStorage) IsCacheNeeded() bool { return true }
+
+// If the 'MoveFile' method is implemented.
+func (storage *SwiftStorage) IsMoveFileImplemented() bool { return true }
+
+// If the storage can guarantee strong consistency.
+func (storage *SwiftStorage) IsStrongConsistent() bool { return false }
+
+// If the storage supports fast listing of files names.
+func (storage *SwiftStorage) IsFastListing() bool { return true }
+
+// Enable the test mode.
+func (storage *SwiftStorage) EnableTestMode() {
+}

src/duplicacy_utils.go

@@ -10,13 +10,11 @@ import (
 	"fmt"
 	"io"
 	"os"
-	"path"
-	"path/filepath"
 	"regexp"
-	"runtime"
 	"strconv"
 	"strings"
 	"time"
+	"runtime"
 
 	"github.com/gilbertchen/gopass"
 	"golang.org/x/crypto/pbkdf2"
@@ -58,7 +56,7 @@ func IsEmptyFilter(pattern string) bool {
 }
 
 func IsUnspecifiedFilter(pattern string) bool {
-	if pattern[0] != '+' && pattern[0] != '-' && pattern[0] != 'i' && pattern[0] != 'e' {
+	if pattern[0] != '+' && pattern[0] != '-' && !strings.HasPrefix(pattern, "i:") && !strings.HasPrefix(pattern, "e:")  {
 		return true
 	} else {
 		return false
@@ -176,6 +174,15 @@ func GetPasswordFromPreference(preference Preference, passwordType string) strin
 		if password, found := os.LookupEnv(name); found && password != "" {
 			return password
 		}
+
+		re := regexp.MustCompile(`[^a-zA-Z0-9_]`)
+		namePlain := re.ReplaceAllString(name, "_")
+		if namePlain != name {
+			LOG_DEBUG("PASSWORD_ENV_VAR", "Reading the environment variable %s", namePlain)
+			if password, found := os.LookupEnv(namePlain); found && password != "" {
+				return password
+			}
+		}
 	}
 
 	// If the password is stored in the preference, there is no need to include the storage name
@@ -344,11 +351,13 @@ func MatchPath(filePath string, patterns []string) (included bool) {
 	for _, pattern := range patterns {
 		if pattern[0] == '+' {
 			if matchPattern(filePath, pattern[1:]) {
+				LOG_DEBUG("PATTERN_INCLUDE", "%s is included by pattern %s", filePath, pattern)
 				return true
 			}
 		} else if pattern[0] == '-' {
 			allIncludes = false
 			if matchPattern(filePath, pattern[1:]) {
+				LOG_DEBUG("PATTERN_EXCLUDE", "%s is excluded by pattern %s", filePath, pattern)
 				return false
 			}
 		} else if strings.HasPrefix(pattern, "i:") || strings.HasPrefix(pattern, "e:") {
@@ -363,7 +372,14 @@ func MatchPath(filePath string, patterns []string) (included bool) {
 				matched = re.MatchString(filePath)
 			}
 			if matched {
-				return strings.HasPrefix(pattern, "i:")
+				if strings.HasPrefix(pattern, "i:") {
+					LOG_DEBUG("PATTERN_INCLUDE", "%s is included by pattern %s", filePath, pattern)
+					return true
+				} else {
+					LOG_DEBUG("PATTERN_EXCLUDE", "%s is excluded by pattern %s", filePath, pattern)
+					return false
+
+				}
 			} else {
 				if strings.HasPrefix(pattern, "e:") {
 					allIncludes = false
@@ -372,20 +388,13 @@ func MatchPath(filePath string, patterns []string) (included bool) {
 		}
 	}
 
-	return !allIncludes
-}
-
-func joinPath(components ...string) string {
-
-	combinedPath := path.Join(components...)
-	if len(combinedPath) > 257 && runtime.GOOS == "windows" {
-		combinedPath = `\\?\` + filepath.Join(components...)
-		// If the path is on a samba drive we must use the UNC format
-		if strings.HasPrefix(combinedPath, `\\?\\\`) {
-			combinedPath = `\\?\UNC\` + combinedPath[6:]
-		}
+	if allIncludes {
+		LOG_DEBUG("PATTERN_EXCLUDE", "%s is excluded", filePath)
+		return false
+	} else {
+		LOG_DEBUG("PATTERN_INCLUDE", "%s is included", filePath)
+		return true
 	}
-	return combinedPath
 }
 
 func PrettyNumber(number int64) string {
@@ -426,7 +435,7 @@ func PrettyTime(seconds int64) string {
 			seconds/day, (seconds%day)/3600, (seconds%3600)/60, seconds%60)
 	} else if seconds > day {
 		return fmt.Sprintf("1 day %02d:%02d:%02d", (seconds%day)/3600, (seconds%3600)/60, seconds%60)
-	} else if seconds > 0 {
+	} else if seconds >= 0 {
 		return fmt.Sprintf("%02d:%02d:%02d", seconds/3600, (seconds%3600)/60, seconds%60)
 	} else {
 		return "n/a"
@@ -453,9 +462,15 @@ func AtoSize(sizeString string) int {
 	return size
 }
 
-func MinInt(x, y int) int {
-	if x < y {
-		return x
+func PrintMemoryUsage() {
+
+	for {
+		var m runtime.MemStats
+		runtime.ReadMemStats(&m)
+
+		LOG_INFO("MEMORY_STATS", "Currently allocated: %s, total allocated: %s, system memory: %s, number of GCs: %d",
+		         PrettySize(int64(m.Alloc)), PrettySize(int64(m.TotalAlloc)), PrettySize(int64(m.Sys)), m.NumGC)
+
+		time.Sleep(time.Second)
 	}
-	return y
-}
+}

src/duplicacy_utils_darwin.go

@@ -0,0 +1,14 @@
+// Copyright (c) Acrosync LLC. All rights reserved.
+// Free for personal use and commercial trial
+// Commercial use requires per-user licenses available from https://duplicacy.com
+
+package duplicacy
+
+import (
+	"strings"
+)
+
+func excludedByAttribute(attirbutes map[string][]byte) bool {
+	value, ok := attirbutes["com.apple.metadata:com_apple_backup_excludeItem"]
+	return ok && strings.Contains(string(value), "com.apple.backupd")
+}

src/duplicacy_utils_freebsd.go

@@ -0,0 +1,13 @@
+// Copyright (c) Acrosync LLC. All rights reserved.
+// Free for personal use and commercial trial
+// Commercial use requires per-user licenses available from https://duplicacy.com
+
+package duplicacy
+
+import (
+)
+
+func excludedByAttribute(attirbutes map[string][]byte) bool {
+	_, ok := attirbutes["duplicacy_exclude"]
+	return ok
+}

src/duplicacy_utils_linux.go

@@ -0,0 +1,13 @@
+// Copyright (c) Acrosync LLC. All rights reserved.
+// Free for personal use and commercial trial
+// Commercial use requires per-user licenses available from https://duplicacy.com
+
+package duplicacy
+
+import (
+)
+
+func excludedByAttribute(attirbutes map[string][]byte) bool {
+	_, ok := attirbutes["user.duplicacy_exclude"]
+	return ok
+}

src/duplicacy_utils_others.go

@@ -9,10 +9,11 @@ package duplicacy
 import (
 	"bytes"
 	"os"
+	"path"
 	"path/filepath"
 	"syscall"
 
-	"github.com/gilbertchen/xattr"
+	"github.com/pkg/xattr"
 )
 
 func Readlink(path string) (isRegular bool, s string, err error) {
@@ -35,7 +36,7 @@ func SetOwner(fullPath string, entry *Entry, fileInfo *os.FileInfo) bool {
 	stat, ok := (*fileInfo).Sys().(*syscall.Stat_t)
 	if ok && stat != nil && (int(stat.Uid) != entry.UID || int(stat.Gid) != entry.GID) {
 		if entry.UID != -1 && entry.GID != -1 {
-			err := os.Chown(fullPath, entry.UID, entry.GID)
+			err := os.Lchown(fullPath, entry.UID, entry.GID)
 			if err != nil {
 				LOG_ERROR("RESTORE_CHOWN", "Failed to change uid or gid: %v", err)
 				return false
@@ -49,37 +50,46 @@ func SetOwner(fullPath string, entry *Entry, fileInfo *os.FileInfo) bool {
 func (entry *Entry) ReadAttributes(top string) {
 
 	fullPath := filepath.Join(top, entry.Path)
-	attributes, _ := xattr.Listxattr(fullPath)
+	attributes, _ := xattr.List(fullPath)
 	if len(attributes) > 0 {
-		entry.Attributes = make(map[string][]byte)
+		entry.Attributes = &map[string][]byte{}
 		for _, name := range attributes {
-			attribute, err := xattr.Getxattr(fullPath, name)
+			attribute, err := xattr.Get(fullPath, name)
 			if err == nil {
-				entry.Attributes[name] = attribute
+				(*entry.Attributes)[name] = attribute
 			}
 		}
 	}
 }
 
 func (entry *Entry) SetAttributesToFile(fullPath string) {
-	names, _ := xattr.Listxattr(fullPath)
+	names, _ := xattr.List(fullPath)
 
 	for _, name := range names {
 
-		newAttribute, found := entry.Attributes[name]
+
+		newAttribute, found := (*entry.Attributes)[name]
 		if found {
-			oldAttribute, _ := xattr.Getxattr(fullPath, name)
-			if bytes.Equal(oldAttribute, newAttribute) {
-				xattr.Setxattr(fullPath, name, newAttribute)
+			oldAttribute, _ := xattr.Get(fullPath, name)
+			if !bytes.Equal(oldAttribute, newAttribute) {
+				xattr.Set(fullPath, name, newAttribute)
 			}
-			delete(entry.Attributes, name)
+			delete(*entry.Attributes, name)
 		} else {
-			xattr.Removexattr(fullPath, name)
+			xattr.Remove(fullPath, name)
 		}
 	}
 
-	for name, attribute := range entry.Attributes {
-		xattr.Setxattr(fullPath, name, attribute)
+	for name, attribute := range *entry.Attributes {
+		xattr.Set(fullPath, name, attribute)
 	}
 
 }
+
+func joinPath(components ...string) string {
+	return path.Join(components...)
+}
+
+func SplitDir(fullPath string) (dir string, file string) {
+	return path.Split(fullPath)
+}

src/duplicacy_utils_test.go

@@ -92,6 +92,17 @@ func TestMatchPattern(t *testing.T) {
 		}
 	}
 
+	for _, pattern := range []string{ "+", "-", "i:", "e:", "+a", "-a", "i:a", "e:a"} {
+		if IsUnspecifiedFilter(pattern) {
+			t.Errorf("pattern %s has a specified filter", pattern)
+		}
+	}
+
+	for _, pattern := range []string{ "i", "e", "ia", "ib", "a", "b"} {
+		if !IsUnspecifiedFilter(pattern) {
+			t.Errorf("pattern %s does not have a specified filter", pattern)
+		}
+	}
 }
 
 func TestRateLimit(t *testing.T) {
@@ -112,7 +123,7 @@ func TestRateLimit(t *testing.T) {
 		return
 	}
 	if int(n) != len(content) {
-		t.Errorf("Wrote %s bytes instead of %s", n, len(content))
+		t.Errorf("Wrote %d bytes instead of %d", n, len(content))
 		return
 	}
 
@@ -127,7 +138,7 @@ func TestRateLimit(t *testing.T) {
 		return
 	}
 	if int(n) != len(content) {
-		t.Errorf("Copied %s bytes instead of %s", n, len(content))
+		t.Errorf("Copied %d bytes instead of %d", n, len(content))
 		return
 	}
 

src/duplicacy_utils_windows.go

@@ -7,6 +7,8 @@ package duplicacy
 import (
 	"fmt"
 	"os"
+	"path/filepath"
+	"strings"
 	"syscall"
 	"unsafe"
 )
@@ -114,3 +116,22 @@ func (entry *Entry) ReadAttributes(top string) {
 func (entry *Entry) SetAttributesToFile(fullPath string) {
 
 }
+
+func joinPath(components ...string) string {
+
+	combinedPath := `\\?\` + filepath.Join(components...)
+	// If the path is on a samba drive we must use the UNC format
+	if strings.HasPrefix(combinedPath, `\\?\\\`) {
+		combinedPath = `\\?\UNC\` + combinedPath[6:]
+	}
+	return combinedPath
+}
+
+func SplitDir(fullPath string) (dir string, file string) {
+	i := strings.LastIndex(fullPath, "\\")
+	return fullPath[:i+1], fullPath[i+1:]
+}
+
+func excludedByAttribute(attirbutes map[string][]byte) bool {
+	return false
+}

src/duplicacy_wasabistorage.go

@@ -0,0 +1,192 @@
+//
+// Storage module for Wasabi (https://www.wasabi.com)
+//
+
+// Wasabi is nominally compatible with AWS S3, but the copy-and-delete
+// method used for renaming objects creates additional expense under
+// Wasabi's billing system.  This module is a pass-through to the
+// existing S3 module for everything other than that one operation.
+//
+// This module copyright 2017 Mark Feit (https://github.com/markfeit)
+// and may be distributed under the same terms as Duplicacy.
+
+package duplicacy
+
+import (
+	"crypto/hmac"
+	"crypto/sha1"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"net/http"
+	"time"
+)
+
+type WasabiStorage struct {
+	StorageBase
+
+	s3         *S3Storage
+	region     string
+	endpoint   string
+	bucket     string
+	storageDir string
+	key        string
+	secret     string
+	client     *http.Client
+}
+
+// See the Storage interface in duplicacy_storage.go for function
+// descriptions.
+
+func CreateWasabiStorage(
+	regionName string, endpoint string,
+	bucketName string, storageDir string,
+	accessKey string, secretKey string,
+	threads int,
+) (storage *WasabiStorage, err error) {
+
+	s3storage, error := CreateS3Storage(regionName, endpoint, bucketName,
+		storageDir, accessKey, secretKey, threads,
+		true,  // isSSLSupported
+		false, // isMinioCompatible
+	)
+
+	if err != nil {
+		return nil, error
+	}
+
+	wasabi := &WasabiStorage{
+
+		// Pass-through to existing S3 module
+		s3: s3storage,
+
+		// Local copies required for renaming
+		region:     regionName,
+		endpoint:   endpoint,
+		bucket:     bucketName,
+		storageDir: storageDir,
+		key:        accessKey,
+		secret:     secretKey,
+		client:     &http.Client{},
+	}
+
+	wasabi.DerivedStorage = wasabi
+	wasabi.SetDefaultNestingLevels([]int{0}, 0)
+
+	return wasabi, nil
+}
+
+func (storage *WasabiStorage) ListFiles(
+	threadIndex int, dir string,
+) (files []string, sizes []int64, err error) {
+	return storage.s3.ListFiles(threadIndex, dir)
+}
+
+func (storage *WasabiStorage) DeleteFile(
+	threadIndex int, filePath string,
+) (err error) {
+	return storage.s3.DeleteFile(threadIndex, filePath)
+
+}
+
+// This is a lightweight implementation of a call to Wasabi for a
+// rename.  It's designed to get the job done with as few dependencies
+// on other packages as possible rather than being somethng
+// general-purpose and reusable.
+func (storage *WasabiStorage) MoveFile(threadIndex int, from string, to string) (err error) {
+
+	var fromPath string
+	// The from path includes the bucket.  Take care not to include an empty storageDir
+	// string as Wasabi's backend will return 404 on URLs with double slashes.
+	if storage.storageDir == "" {
+		fromPath = fmt.Sprintf("/%s/%s", storage.bucket, from)
+	} else {
+		fromPath = fmt.Sprintf("/%s/%s/%s", storage.bucket, storage.storageDir, from)
+	}
+
+	object := fmt.Sprintf("https://%s@%s%s", storage.region, storage.endpoint, fromPath)
+
+	toPath := to
+	// The object's new name is relative to the top of the bucket.
+	if storage.storageDir != "" {
+		toPath = fmt.Sprintf("%s/%s", storage.storageDir, to)
+	}
+
+	timestamp := time.Now().Format(time.RFC1123Z)
+
+	signingString := fmt.Sprintf("MOVE\n\n\n%s\n%s", timestamp, fromPath)
+
+	signer := hmac.New(sha1.New, []byte(storage.secret))
+	signer.Write([]byte(signingString))
+
+	signature := base64.StdEncoding.EncodeToString(signer.Sum(nil))
+
+	authorization := fmt.Sprintf("AWS %s:%s", storage.key, signature)
+
+	request, err := http.NewRequest("MOVE", object, nil)
+	if err != nil {
+		return err
+	}
+	request.Header.Add("Authorization", authorization)
+	request.Header.Add("Date", timestamp)
+	request.Header.Add("Destination", toPath)
+	request.Header.Add("Host", storage.endpoint)
+	request.Header.Add("Overwrite", "true")
+
+	response, err := storage.client.Do(request)
+	if err != nil {
+		return err
+	}
+	defer response.Body.Close()
+
+	if response.StatusCode != 200 {
+		return errors.New(response.Status)
+	}
+
+	return nil
+}
+
+func (storage *WasabiStorage) CreateDirectory(
+	threadIndex int, dir string,
+) (err error) {
+	return storage.s3.CreateDirectory(threadIndex, dir)
+}
+
+func (storage *WasabiStorage) GetFileInfo(
+	threadIndex int, filePath string,
+) (exist bool, isDir bool, size int64, err error) {
+	return storage.s3.GetFileInfo(threadIndex, filePath)
+}
+
+func (storage *WasabiStorage) DownloadFile(
+	threadIndex int, filePath string, chunk *Chunk,
+) (err error) {
+	return storage.s3.DownloadFile(threadIndex, filePath, chunk)
+}
+
+func (storage *WasabiStorage) UploadFile(
+	threadIndex int, filePath string, content []byte,
+) (err error) {
+	return storage.s3.UploadFile(threadIndex, filePath, content)
+}
+
+func (storage *WasabiStorage) IsCacheNeeded() bool {
+	return storage.s3.IsCacheNeeded()
+}
+
+func (storage *WasabiStorage) IsMoveFileImplemented() bool {
+	// This is implemented locally since S3 does a copy and delete
+	return true
+}
+
+func (storage *WasabiStorage) IsStrongConsistent() bool {
+	// Wasabi has it, S3 doesn't.
+	return true
+}
+
+func (storage *WasabiStorage) IsFastListing() bool {
+	return storage.s3.IsFastListing()
+}
+
+func (storage *WasabiStorage) EnableTestMode() {
+}

src/duplicacy_webdavstorage.go

@@ -0,0 +1,485 @@
+// Copyright (c) Acrosync LLC. All rights reserved.
+// Free for personal use and commercial trial
+// Commercial use requires per-user licenses available from https://duplicacy.com
+//
+//
+// This storage backend is based on the work by Yuri Karamani from https://github.com/karamani/webdavclnt,
+// released under the MIT license.
+//
+package duplicacy
+
+import (
+	"bytes"
+	"encoding/xml"
+	"errors"
+	"fmt"
+	"io"
+	"math/rand"
+	"net/http"
+	//"net/http/httputil"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+	"io/ioutil"
+)
+
+type WebDAVStorage struct {
+	StorageBase
+
+	host       string
+	port       int
+	username   string
+	password   string
+	storageDir string
+	useHTTP    bool
+
+	client             *http.Client
+	threads            int
+	directoryCache     map[string]int // stores directories known to exist by this backend
+	directoryCacheLock sync.Mutex     // lock for accessing directoryCache
+}
+
+var (
+	errWebDAVAuthorizationFailure = errors.New("Authentication failed")
+	errWebDAVMovedPermanently     = errors.New("Moved permanently")
+	errWebDAVNotExist             = errors.New("Path does not exist")
+	errWebDAVMaximumBackoff       = errors.New("Maximum backoff reached")
+	errWebDAVMethodNotAllowed     = errors.New("Method not allowed")
+)
+
+func CreateWebDAVStorage(host string, port int, username string, password string, storageDir string, useHTTP bool, threads int) (storage *WebDAVStorage, err error) {
+	if len(storageDir) > 0 && storageDir[len(storageDir)-1] != '/' {
+		storageDir += "/"
+	}
+
+	storage = &WebDAVStorage{
+		host:       host,
+		port:       port,
+		username:   username,
+		password:   password,
+		storageDir: "",
+		useHTTP:    useHTTP,
+
+		client:         http.DefaultClient,
+		threads:        threads,
+		directoryCache: make(map[string]int),
+	}
+
+	// Make sure it doesn't follow redirect
+	storage.client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
+		return http.ErrUseLastResponse
+	}
+
+	exist, isDir, _, err := storage.GetFileInfo(0, storageDir)
+	if err != nil {
+		return nil, err
+	}
+	if !exist {
+		return nil, fmt.Errorf("Storage path %s does not exist", storageDir)
+	}
+	if !isDir {
+		return nil, fmt.Errorf("Storage path %s is not a directory", storageDir)
+	}
+	storage.storageDir = storageDir
+
+	for _, dir := range []string{"snapshots", "chunks"} {
+		storage.CreateDirectory(0, dir)
+	}
+
+	storage.DerivedStorage = storage
+	storage.SetDefaultNestingLevels([]int{0}, 0)
+	return storage, nil
+}
+
+func (storage *WebDAVStorage) createConnectionString(uri string) string {
+
+	url := storage.host
+
+	if storage.useHTTP {
+		url = "http://" + url
+	} else {
+		url = "https://" + url
+	}
+
+	if storage.port > 0 {
+		url += fmt.Sprintf(":%d", storage.port)
+	}
+	return url + "/" + storage.storageDir + uri
+}
+
+func (storage *WebDAVStorage) retry(backoff int) int {
+	delay := rand.Intn(backoff*500) + backoff*500
+	time.Sleep(time.Duration(delay) * time.Millisecond)
+	backoff *= 2
+	return backoff
+}
+
+func (storage *WebDAVStorage) sendRequest(method string, uri string, depth int, data []byte) (io.ReadCloser, http.Header, error) {
+
+	backoff := 1
+	for i := 0; i < 8; i++ {
+
+		var dataReader io.Reader
+		headers := make(map[string]string)
+		if method == "PROPFIND" {
+			headers["Content-Type"] = "application/xml"
+			headers["Depth"] = fmt.Sprintf("%d", depth)
+			dataReader = bytes.NewReader(data)
+		} else if method == "PUT" {
+			headers["Content-Type"] = "application/octet-stream"
+			headers["Content-Length"] = fmt.Sprintf("%d", len(data))
+			if storage.UploadRateLimit <= 0 {
+				dataReader = bytes.NewReader(data)
+			} else {
+				dataReader = CreateRateLimitedReader(data, storage.UploadRateLimit/storage.threads)
+			}
+		} else if method == "MOVE" {
+			headers["Destination"] = storage.createConnectionString(string(data))
+			headers["Content-Type"] = "application/octet-stream"
+			dataReader = bytes.NewReader([]byte(""))
+		} else {
+			headers["Content-Type"] = "application/octet-stream"
+			dataReader = bytes.NewReader(data)
+		}
+
+		request, err := http.NewRequest(method, storage.createConnectionString(uri), dataReader)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		if len(storage.username) > 0 {
+			request.SetBasicAuth(storage.username, storage.password)
+		}
+
+		for key, value := range headers {
+			request.Header.Set(key, value)
+		}
+
+		if method == "PUT" {
+			request.ContentLength = int64(len(data))
+		}
+
+		//requestDump, err := httputil.DumpRequest(request, true)
+		//LOG_INFO("debug", "Request: %s", requestDump)
+
+		response, err := storage.client.Do(request)
+		if err != nil {
+			LOG_TRACE("WEBDAV_ERROR", "URL request '%s %s' returned an error (%v)", method, uri, err)
+			backoff = storage.retry(backoff)
+			continue
+		}
+
+		if response.StatusCode < 300 {
+			return response.Body, response.Header, nil
+		}
+
+		io.Copy(ioutil.Discard, response.Body)
+		response.Body.Close()
+
+		if response.StatusCode == 301 {
+			return nil, nil, errWebDAVMovedPermanently
+		}
+
+		if response.StatusCode == 404 {
+			// Retry if it is UPLOAD, otherwise return immediately
+			if method != "PUT" {
+				return nil, nil, errWebDAVNotExist
+			}
+		} else if response.StatusCode == 405 {
+			return nil, nil, errWebDAVMethodNotAllowed
+		}
+		LOG_INFO("WEBDAV_RETRY", "URL request '%s %s' returned status code %d", method, uri, response.StatusCode)
+		backoff = storage.retry(backoff)
+	}
+	return nil, nil, errWebDAVMaximumBackoff
+}
+
+type WebDAVProperties map[string]string
+
+type WebDAVPropValue struct {
+	XMLName xml.Name `xml:""`
+	Value   string   `xml:",innerxml"`
+}
+
+type WebDAVProp struct {
+	PropList []WebDAVPropValue `xml:",any"`
+}
+
+type WebDAVPropStat struct {
+	Prop *WebDAVProp `xml:"prop"`
+}
+
+type WebDAVResponse struct {
+	Href     string          `xml:"href"`
+	PropStat *WebDAVPropStat `xml:"propstat"`
+}
+
+type WebDAVMultiStatus struct {
+	Responses []WebDAVResponse `xml:"response"`
+}
+
+func (storage *WebDAVStorage) getProperties(uri string, depth int, properties ...string) (map[string]WebDAVProperties, error) {
+
+	maxTries := 3
+	for tries := 0; ; tries++ {
+		propfind := "<prop>"
+		for _, p := range properties {
+			propfind += fmt.Sprintf("<%s/>", p)
+		}
+		propfind += "</prop>"
+
+		body := fmt.Sprintf(`<?xml version="1.0" encoding="utf-8" ?><propfind xmlns="DAV:">%s</propfind>`, propfind)
+
+		readCloser, _, err := storage.sendRequest("PROPFIND", uri, depth, []byte(body))
+		if err != nil {
+			return nil, err
+		}
+		defer readCloser.Close()
+		defer io.Copy(ioutil.Discard, readCloser)
+
+		object := WebDAVMultiStatus{}
+		err = xml.NewDecoder(readCloser).Decode(&object)
+		if err != nil {
+			if strings.Contains(err.Error(), "unexpected EOF") && tries < maxTries {
+				LOG_WARN("WEBDAV_RETRY", "Retrying on %v", err)
+				continue
+			}
+			return nil, err
+		}
+
+		if object.Responses == nil || len(object.Responses) == 0 {
+			return nil, errors.New("no WebDAV responses")
+		}
+
+		responses := make(map[string]WebDAVProperties)
+
+		for _, responseTag := range object.Responses {
+
+			if responseTag.PropStat == nil || responseTag.PropStat.Prop == nil || responseTag.PropStat.Prop.PropList == nil {
+				return nil, errors.New("no WebDAV properties")
+			}
+
+			properties := make(WebDAVProperties)
+			for _, prop := range responseTag.PropStat.Prop.PropList {
+				properties[prop.XMLName.Local] = prop.Value
+			}
+
+			responseKey := responseTag.Href
+			responses[responseKey] = properties
+
+		}
+
+		return responses, nil
+	}
+}
+
+// ListFiles return the list of files and subdirectories under 'dir'.  A subdirectories returned must have a trailing '/', with
+// a size of 0.  If 'dir' is 'snapshots', only subdirectories will be returned.  If 'dir' is 'snapshots/repository_id', then only
+// files will be returned.  If 'dir' is 'chunks', the implementation can return the list either recusively or non-recusively.
+func (storage *WebDAVStorage) ListFiles(threadIndex int, dir string) (files []string, sizes []int64, err error) {
+	if dir[len(dir)-1] != '/' {
+		dir += "/"
+	}
+	properties, err := storage.getProperties(dir, 1, "getcontentlength", "resourcetype")
+	if err != nil {
+		return nil, nil, err
+	}
+
+	prefixLength := len(storage.storageDir) + len(dir) + 1
+
+	for file, m := range properties {
+		if len(file) <= prefixLength {
+			continue
+		}
+
+		isDir := false
+		size := 0
+		if resourceType, exist := m["resourcetype"]; exist && strings.Contains(resourceType, "collection") {
+			isDir = true
+		} else if length, exist := m["getcontentlength"]; exist {
+			if length == "" {
+				isDir = true
+			} else {
+				size, _ = strconv.Atoi(length)
+			}
+		} else {
+			continue
+		}
+
+		if !isDir {
+			if dir != "snapshots/" {
+				files = append(files, file[prefixLength:])
+				sizes = append(sizes, int64(size))
+			}
+		} else {
+			// This is a dir
+			file := file[prefixLength:]
+			if file[len(file)-1] != '/' {
+				file += "/"
+			}
+			files = append(files, file)
+			sizes = append(sizes, int64(0))
+
+			// Add the directory to the directory cache
+			storage.directoryCacheLock.Lock()
+			storage.directoryCache[dir + file] = 1
+			storage.directoryCacheLock.Unlock()
+
+		}
+	}
+
+	return files, sizes, nil
+}
+
+// GetFileInfo returns the information about the file or directory at 'filePath'.
+func (storage *WebDAVStorage) GetFileInfo(threadIndex int, filePath string) (exist bool, isDir bool, size int64, err error) {
+
+	properties, err := storage.getProperties(filePath, 0, "getcontentlength", "resourcetype")
+	if err != nil {
+		if err == errWebDAVNotExist {
+			return false, false, 0, nil
+		}
+		if err == errWebDAVMovedPermanently {
+			// This must be a directory
+			return true, true, 0, nil
+		}
+		return false, false, 0, err
+	}
+
+	m, exist := properties["/"+storage.storageDir+filePath]
+
+	// If no properties exist for the given filePath, remove the trailing / from filePath and search again
+	if !exist && filePath != "" && filePath[len(filePath) - 1] == '/' {
+		m, exist = properties["/"+storage.storageDir+filePath[:len(filePath) - 1]]
+	}
+
+	if !exist {
+		return false, false, 0, nil
+	} else if resourceType, exist := m["resourcetype"]; exist && strings.Contains(resourceType, "collection") {
+		return true, true, 0, nil
+	} else if length, exist := m["getcontentlength"]; exist && length != "" {
+		value, _ := strconv.Atoi(length)
+		return true, false, int64(value), nil
+	} else {
+		return true, true, 0, nil
+	}
+}
+
+// DeleteFile deletes the file or directory at 'filePath'.
+func (storage *WebDAVStorage) DeleteFile(threadIndex int, filePath string) (err error) {
+	readCloser, _, err := storage.sendRequest("DELETE", filePath, 0, []byte(""))
+	if err != nil {
+		return err
+	}
+	io.Copy(ioutil.Discard, readCloser)
+	readCloser.Close()
+	return nil
+}
+
+// MoveFile renames the file.
+func (storage *WebDAVStorage) MoveFile(threadIndex int, from string, to string) (err error) {
+	readCloser, _, err := storage.sendRequest("MOVE", from, 0, []byte(to))
+	if err != nil {
+		return err
+	}
+	io.Copy(ioutil.Discard, readCloser)
+	readCloser.Close()
+	return nil
+}
+
+// createParentDirectory creates the parent directory if it doesn't exist in the cache
+func (storage *WebDAVStorage) createParentDirectory(threadIndex int, dir string) (err error) {
+
+	found := strings.LastIndex(dir, "/")
+	if found == -1 {
+		return nil
+	}
+	parent := dir[:found]
+
+	return storage.CreateDirectory(threadIndex, parent)
+}
+
+// CreateDirectory creates a new directory.
+func (storage *WebDAVStorage) CreateDirectory(threadIndex int, dir string) (err error) {
+	for dir != "" && dir[len(dir)-1] == '/' {
+		dir = dir[:len(dir)-1]
+	}
+
+	if dir == "" {
+		return nil
+	}
+
+	storage.directoryCacheLock.Lock()
+	_, exist := storage.directoryCache[dir]
+	storage.directoryCacheLock.Unlock()
+
+	if exist {
+		return nil
+	}
+
+	// If there is an error in creating the parent directory, proceed anyway
+	storage.createParentDirectory(threadIndex, dir)
+
+	readCloser, _, err := storage.sendRequest("MKCOL", dir, 0, []byte(""))
+	if err != nil {
+		if err == errWebDAVMethodNotAllowed || err == errWebDAVMovedPermanently || err == io.EOF {
+			// We simply ignore these errors and assume that the directory already exists
+			LOG_TRACE("WEBDAV_MKDIR", "Can't create directory %s: %v; error ignored", dir, err)
+			storage.directoryCacheLock.Lock()
+			storage.directoryCache[dir] = 1
+			storage.directoryCacheLock.Unlock()
+			return nil
+		}
+		return err
+	}
+	io.Copy(ioutil.Discard, readCloser)
+	readCloser.Close()
+
+	storage.directoryCacheLock.Lock()
+	storage.directoryCache[dir] = 1
+	storage.directoryCacheLock.Unlock()
+	return nil
+}
+
+// DownloadFile reads the file at 'filePath' into the chunk.
+func (storage *WebDAVStorage) DownloadFile(threadIndex int, filePath string, chunk *Chunk) (err error) {
+	readCloser, _, err := storage.sendRequest("GET", filePath, 0, nil)
+	if err != nil {
+		return err
+	}
+
+	_, err = RateLimitedCopy(chunk, readCloser, storage.DownloadRateLimit/storage.threads)
+	return err
+}
+
+// UploadFile writes 'content' to the file at 'filePath'.
+func (storage *WebDAVStorage) UploadFile(threadIndex int, filePath string, content []byte) (err error) {
+
+	// If there is an error in creating the parent directory, proceed anyway
+	storage.createParentDirectory(threadIndex, filePath)
+
+	readCloser, _, err := storage.sendRequest("PUT", filePath, 0, content)
+	if err != nil {
+		return err
+	}
+	io.Copy(ioutil.Discard, readCloser)
+	readCloser.Close()
+	return nil
+}
+
+// If a local snapshot cache is needed for the storage to avoid downloading/uploading chunks too often when
+// managing snapshots.
+func (storage *WebDAVStorage) IsCacheNeeded() bool { return true }
+
+// If the 'MoveFile' method is implemented.
+func (storage *WebDAVStorage) IsMoveFileImplemented() bool { return true }
+
+// If the storage can guarantee strong consistency.
+func (storage *WebDAVStorage) IsStrongConsistent() bool { return false }
+
+// If the storage supports fast listing of files names.
+func (storage *WebDAVStorage) IsFastListing() bool { return false }
+
+// Enable the test mode.
+func (storage *WebDAVStorage) EnableTestMode() {}