jgaunt/duplicacy
1
0
Fork 0
mirror of https://github.com/gilbertchen/duplicacy synced 2025-12-10 21:33:19 +00:00
Code Issues Releases Wiki Activity

Run goimports on all source files

Browse Source
This commit is contained in:
Gilbert Chen
2017-09-20 23:07:43 -04:00
parent 978212fd75
commit 923cd0aa63
49 changed files with 13570 additions and 13631 deletions
Download Patch File Download Diff File Expand all files Collapse all files

253
duplicacy/duplicacy_main.go
View File

@@ -5,22 +5,23 @@
package main
import (
"os"
"encoding/json"
"fmt"
"os"
"os/exec"
"os/signal"
"path"
"path/filepath"
"regexp"
"strings"
"strconv"
"runtime"
"os/exec"
"os/signal"
"encoding/json"
"strconv"
"strings"
"github.com/gilbertchen/cli"
"github.com/gilbertchen/duplicacy/src"
"io/ioutil"
"github.com/gilbertchen/duplicacy/src"
)
const (
@@ -79,7 +80,7 @@ func getRepositoryPreference(context *cli.Context, storageName string) (reposito
return repository, preference
}
func getRevisions(context *cli.Context) (revisions[] int) {
func getRevisions(context *cli.Context) (revisions []int) {
flags := context.StringSlice("r")
@@ -148,8 +149,8 @@ func runScript(context *cli.Context, storageName string, phase string) bool {
preferencePath := duplicacy.GetDuplicacyPreferencePath()
scriptDir, _ := filepath.Abs(path.Join(preferencePath, "scripts"))
scriptNames := []string { phase + "-" + context.Command.Name,
storageName + "-" + phase + "-" + context.Command.Name }
scriptNames := []string{phase + "-" + context.Command.Name,
storageName + "-" + phase + "-" + context.Command.Name}
script := ""
for _, scriptName := range scriptNames {
@@ -239,7 +240,6 @@ func configRepository(context *cli.Context, init bool) {
preferencePath = path.Join(repository, duplicacy.DUPLICACY_DIRECTORY) // TOKEEP
}
if stat, _ := os.Stat(path.Join(preferencePath, "preferences")); stat != nil {
duplicacy.LOG_ERROR("REPOSITORY_INIT", "The repository %s has already been initialized", repository)
return
@@ -273,10 +273,10 @@ func configRepository(context *cli.Context, init bool) {
}
}
preference := duplicacy.Preference {
preference := duplicacy.Preference{
Name: storageName,
SnapshotID : snapshotID,
StorageURL : storageURL,
SnapshotID: snapshotID,
StorageURL: storageURL,
Encrypted: context.Bool("encrypt"),
}
@@ -318,7 +318,7 @@ func configRepository(context *cli.Context, init bool) {
}
size := 1
for size * 2 <= averageChunkSize {
for size*2 <= averageChunkSize {
size *= 2
}
@@ -678,7 +678,7 @@ func restoreRepository(context *cli.Context) {
deleteMode := context.Bool("delete")
showStatistics := context.Bool("stats")
var patterns [] string
var patterns []string
for _, pattern := range context.Args() {
pattern = strings.TrimSpace(pattern)
@@ -890,7 +890,6 @@ func diff(context *cli.Context) {
return
}
password := ""
if preference.Encrypted {
password = duplicacy.GetPassword(*preference, "password", "Enter storage password:", false, false)
@@ -1058,7 +1057,6 @@ func copySnapshots(context *cli.Context) {
sourceManager.SetupSnapshotCache(source.Name)
duplicacy.SavePassword(*source, "password", sourcePassword)
_, destination := getRepositoryPreference(context, context.String("to"))
if destination.Name == source.Name {
@@ -1072,7 +1070,6 @@ func copySnapshots(context *cli.Context) {
return
}
duplicacy.LOG_INFO("STORAGE_SET", "Destination storage set to %s", destination.StorageURL)
destinationStorage := duplicacy.CreateStorage(*destination, false, threads)
if destinationStorage == nil {
@@ -1082,7 +1079,7 @@ func copySnapshots(context *cli.Context) {
destinationPassword := ""
if destination.Encrypted {
destinationPassword = duplicacy.GetPassword(*destination, "password",
"Enter destination storage password:",false, false)
"Enter destination storage password:", false, false)
}
sourceStorage.SetRateLimits(context.Int("download-limit-rate"), 0)
@@ -1121,7 +1118,7 @@ func infoStorage(context *cli.Context) {
}
isEncrypted := context.Bool("e")
preference := duplicacy.Preference {
preference := duplicacy.Preference{
Name: "default",
SnapshotID: "default",
StorageURL: context.Args()[0],
@@ -1154,26 +1151,26 @@ func main() {
app := cli.NewApp()
app.Commands = []cli.Command {
app.Commands = []cli.Command{
{
Name: "init",
Flags: []cli.Flag {
cli.BoolFlag {
Flags: []cli.Flag{
cli.BoolFlag{
Name: "encrypt, e",
Usage: "encrypt the storage with a password",
},
cli.StringFlag {
cli.StringFlag{
Name: "chunk-size, c",
Value: "4M",
Usage: "the average size of chunks",
Argument: "4M",
},
cli.StringFlag {
cli.StringFlag{
Name: "max-chunk-size, max",
Usage: "the maximum size of chunks (defaults to chunk-size * 4)",
Argument: "16M",
},
cli.StringFlag {
cli.StringFlag{
Name: "min-chunk-size, min",
Usage: "the minimum size of chunks (defaults to chunk-size / 4)",
Argument: "1M",
@@ -1190,41 +1187,41 @@ func main() {
},
{
Name: "backup",
Flags: []cli.Flag {
cli.BoolFlag {
Flags: []cli.Flag{
cli.BoolFlag{
Name: "hash",
Usage: "detect file differences by hash (rather than size and timestamp)",
},
cli.StringFlag {
cli.StringFlag{
Name: "t",
Usage: "assign a tag to the backup",
Argument: "<tag>",
},
cli.BoolFlag {
cli.BoolFlag{
Name: "stats",
Usage: "show statistics during and after backup",
},
cli.IntFlag {
cli.IntFlag{
Name: "threads",
Value: 1,
Usage: "number of uploading threads",
Argument: "<n>",
},
cli.IntFlag {
cli.IntFlag{
Name: "limit-rate",
Value: 0,
Usage: "the maximum upload rate (in kilobytes/sec)",
Argument: "<kB/s>",
},
cli.BoolFlag {
cli.BoolFlag{
Name: "dry-run",
Usage: "Dry run for testing, don't backup anything. Use with -stats and -d",
},
cli.BoolFlag {
cli.BoolFlag{
Name: "vss",
Usage: "enable the Volume Shadow Copy service (Windows only)",
},
cli.StringFlag {
cli.StringFlag{
Name: "storage",
Usage: "backup to the specified storage instead of the default one",
Argument: "<storage name>",
@@ -1237,41 +1234,41 @@ func main() {
{
Name: "restore",
Flags: []cli.Flag {
cli.IntFlag {
Flags: []cli.Flag{
cli.IntFlag{
Name: "r",
Usage: "the revision number of the snapshot (required)",
Argument: "<revision>",
},
cli.BoolFlag {
cli.BoolFlag{
Name: "hash",
Usage: "detect file differences by hash (rather than size and timestamp)",
},
cli.BoolFlag {
cli.BoolFlag{
Name: "overwrite",
Usage: "overwrite existing files in the repository",
},
cli.BoolFlag {
cli.BoolFlag{
Name: "delete",
Usage: "delete files not in the snapshot",
},
cli.BoolFlag {
cli.BoolFlag{
Name: "stats",
Usage: "show statistics during and after restore",
},
cli.IntFlag {
cli.IntFlag{
Name: "threads",
Value: 1,
Usage: "number of downloading threads",
Argument: "<n>",
},
cli.IntFlag {
cli.IntFlag{
Name: "limit-rate",
Value: 0,
Usage: "the maximum download rate (in kilobytes/sec)",
Argument: "<kB/s>",
},
cli.StringFlag {
cli.StringFlag{
Name: "storage",
Usage: "restore from the specified storage instead of the default one",
Argument: "<storage name>",
@@ -1284,39 +1281,39 @@ func main() {
{
Name: "list",
Flags: []cli.Flag {
cli.BoolFlag {
Flags: []cli.Flag{
cli.BoolFlag{
Name: "all, a",
Usage: "list snapshots with any id",
},
cli.StringFlag {
cli.StringFlag{
Name: "id",
Usage: "list snapshots with the specified id rather than the default one",
Argument: "<snapshot id>",
},
cli.StringSliceFlag {
cli.StringSliceFlag{
Name: "r",
Usage: "the revision number of the snapshot",
Argument: "<revision>",
},
cli.StringFlag {
cli.StringFlag{
Name: "t",
Usage: "list snapshots with the specified tag",
Argument: "<tag>",
},
cli.BoolFlag {
cli.BoolFlag{
Name: "files",
Usage: "print the file list in each snapshot",
},
cli.BoolFlag {
cli.BoolFlag{
Name: "chunks",
Usage: "print chunks in each snapshot or all chunks if no snapshot specified",
},
cli.BoolFlag {
cli.BoolFlag{
Name: "reset-passwords",
Usage: "take passwords from input rather than keychain/keyring",
},
cli.StringFlag {
cli.StringFlag{
Name: "storage",
Usage: "retrieve snapshots from the specified storage",
Argument: "<storage name>",
@@ -1328,47 +1325,47 @@ func main() {
},
{
Name: "check",
Flags: []cli.Flag {
cli.BoolFlag {
Flags: []cli.Flag{
cli.BoolFlag{
Name: "all, a",
Usage: "check snapshots with any id",
},
cli.StringFlag {
cli.StringFlag{
Name: "id",
Usage: "check snapshots with the specified id rather than the default one",
Argument: "<snapshot id>",
},
cli.StringSliceFlag {
cli.StringSliceFlag{
Name: "r",
Usage: "the revision number of the snapshot",
Argument: "<revision>",
},
cli.StringFlag {
cli.StringFlag{
Name: "t",
Usage: "check snapshots with the specified tag",
Argument: "<tag>",
},
cli.BoolFlag {
cli.BoolFlag{
Name: "fossils",
Usage: "search fossils if a chunk can't be found",
},
cli.BoolFlag {
cli.BoolFlag{
Name: "resurrect",
Usage: "turn referenced fossils back into chunks",
},
cli.BoolFlag {
cli.BoolFlag{
Name: "files",
Usage: "verify the integrity of every file",
},
cli.BoolFlag {
cli.BoolFlag{
Name: "stats",
Usage: "show deduplication statistics (imply -all and all revisions)",
},
cli.BoolFlag {
cli.BoolFlag{
Name: "tabular",
Usage: "show tabular usage and deduplication statistics (imply -stats, -all, and all revisions)",
},
cli.StringFlag {
cli.StringFlag{
Name: "storage",
Usage: "retrieve snapshots from the specified storage",
Argument: "<storage name>",
@@ -1380,18 +1377,18 @@ func main() {
},
{
Name: "cat",
Flags: []cli.Flag {
cli.StringFlag {
Flags: []cli.Flag{
cli.StringFlag{
Name: "id",
Usage: "retrieve from the snapshot with the specified id",
Argument: "<snapshot id>",
},
cli.IntFlag {
cli.IntFlag{
Name: "r",
Usage: "the revision number of the snapshot",
Argument: "<revision>",
},
cli.StringFlag {
cli.StringFlag{
Name: "storage",
Usage: "retrieve the file from the specified storage",
Argument: "<storage name>",
@@ -1404,22 +1401,22 @@ func main() {
{
Name: "diff",
Flags: []cli.Flag {
cli.StringFlag {
Flags: []cli.Flag{
cli.StringFlag{
Name: "id",
Usage: "diff snapshots with the specified id",
Argument: "<snapshot id>",
},
cli.IntSliceFlag {
cli.IntSliceFlag{
Name: "r",
Usage: "the revision number of the snapshot",
Argument: "<revision>",
},
cli.BoolFlag {
cli.BoolFlag{
Name: "hash",
Usage: "compute the hashes of on-disk files",
},
cli.StringFlag {
cli.StringFlag{
Name: "storage",
Usage: "retrieve files from the specified storage",
Argument: "<storage name>",
@@ -1432,22 +1429,22 @@ func main() {
{
Name: "history",
Flags: []cli.Flag {
cli.StringFlag {
Flags: []cli.Flag{
cli.StringFlag{
Name: "id",
Usage: "find the file in the snapshot with the specified id",
Argument: "<snapshot id>",
},
cli.StringSliceFlag {
cli.StringSliceFlag{
Name: "r",
Usage: "show history of the specified revisions",
Argument: "<revision>",
},
cli.BoolFlag {
cli.BoolFlag{
Name: "hash",
Usage: "show the hash of the on-disk file",
},
cli.StringFlag {
cli.StringFlag{
Name: "storage",
Usage: "retrieve files from the specified storage",
Argument: "<storage name>",
@@ -1460,57 +1457,57 @@ func main() {
{
Name: "prune",
Flags: []cli.Flag {
cli.StringFlag {
Flags: []cli.Flag{
cli.StringFlag{
Name: "id",
Usage: "delete snapshots with the specified id instead of the default one",
Argument: "<snapshot id>",
},
cli.BoolFlag {
cli.BoolFlag{
Name: "all, a",
Usage: "match against all snapshot IDs",
},
cli.StringSliceFlag {
cli.StringSliceFlag{
Name: "r",
Usage: "delete snapshots with the specified revisions",
Argument: "<revision>",
},
cli.StringSliceFlag {
cli.StringSliceFlag{
Name: "t",
Usage: "delete snapshots with the specifed tags",
Argument: "<tag>",
},
cli.StringSliceFlag {
cli.StringSliceFlag{
Name: "keep",
Usage: "keep 1 snapshot every n days for snapshots older than m days",
Argument: "<n:m>",
},
cli.BoolFlag {
cli.BoolFlag{
Name: "exhaustive",
Usage: "remove all unreferenced chunks (not just those referenced by deleted snapshots)",
},
cli.BoolFlag {
cli.BoolFlag{
Name: "exclusive",
Usage: "assume exclusive acess to the storage (disable two-step fossil collection)",
},
cli.BoolFlag {
cli.BoolFlag{
Name: "dry-run, d",
Usage: "show what would have been deleted",
},
cli.BoolFlag {
cli.BoolFlag{
Name: "delete-only",
Usage: "delete fossils previously collected (if deletable) and don't collect fossils",
},
cli.BoolFlag {
cli.BoolFlag{
Name: "collect-only",
Usage: "identify and collect fossils, but don't delete fossils previously collected",
},
cli.StringSliceFlag {
cli.StringSliceFlag{
Name: "ignore",
Usage: "ignore snapshots with the specified id when deciding if fossils can be deleted",
Argument: "<id>",
},
cli.StringFlag {
cli.StringFlag{
Name: "storage",
Usage: "prune snapshots from the specified storage",
Argument: "<storage name>",
@@ -1521,11 +1518,10 @@ func main() {
Action: pruneSnapshots,
},
{
Name: "password",
Flags: []cli.Flag {
cli.StringFlag {
Flags: []cli.Flag{
cli.StringFlag{
Name: "storage",
Usage: "change the password used to access the specified storage",
Argument: "<storage name>",
@@ -1538,28 +1534,28 @@ func main() {
{
Name: "add",
Flags: []cli.Flag {
cli.BoolFlag {
Flags: []cli.Flag{
cli.BoolFlag{
Name: "encrypt, e",
Usage: "Encrypt the storage with a password",
},
cli.StringFlag {
cli.StringFlag{
Name: "chunk-size, c",
Value: "4M",
Usage: "the average size of chunks",
Argument: "4M",
},
cli.StringFlag {
cli.StringFlag{
Name: "max-chunk-size, max",
Usage: "the maximum size of chunks (defaults to chunk-size * 4)",
Argument: "16M",
},
cli.StringFlag {
cli.StringFlag{
Name: "min-chunk-size, min",
Usage: "the minimum size of chunks (defaults to chunk-size / 4)",
Argument: "1M",
},
cli.StringFlag {
cli.StringFlag{
Name: "copy",
Usage: "make the new storage compatible with an existing one to allow for copy operations",
Argument: "<storage name>",
@@ -1572,40 +1568,40 @@ func main() {
{
Name: "set",
Flags: []cli.Flag {
cli.GenericFlag {
Flags: []cli.Flag{
cli.GenericFlag{
Name: "encrypt, e",
Usage: "encrypt the storage with a password",
Value: &TriBool{},
Arg: "true",
},
cli.GenericFlag {
cli.GenericFlag{
Name: "no-backup",
Usage: "backup to this storage is prohibited",
Value: &TriBool{},
Arg: "true",
},
cli.GenericFlag {
cli.GenericFlag{
Name: "no-restore",
Usage: "restore from this storage is prohibited",
Value: &TriBool{},
Arg: "true",
},
cli.GenericFlag {
cli.GenericFlag{
Name: "no-save-password",
Usage: "don't save password or access keys to keychain/keyring",
Value: &TriBool{},
Arg: "true",
},
cli.StringFlag {
cli.StringFlag{
Name: "key",
Usage: "add a key/password whose value is supplied by the -value option",
},
cli.StringFlag {
cli.StringFlag{
Name: "value",
Usage: "the value of the key/password",
},
cli.StringFlag {
cli.StringFlag{
Name: "storage",
Usage: "use the specified storage instead of the default one",
Argument: "<storage name>",
@@ -1617,40 +1613,40 @@ func main() {
},
{
Name: "copy",
Flags: []cli.Flag {
cli.StringFlag {
Flags: []cli.Flag{
cli.StringFlag{
Name: "id",
Usage: "copy snapshots with the specified id instead of all snapshot ids",
Argument: "<snapshot id>",
},
cli.StringSliceFlag {
cli.StringSliceFlag{
Name: "r",
Usage: "copy snapshots with the specified revisions",
Argument: "<revision>",
},
cli.StringFlag {
cli.StringFlag{
Name: "from",
Usage: "copy snapshots from the specified storage",
Argument: "<storage name>",
},
cli.StringFlag {
cli.StringFlag{
Name: "to",
Usage: "copy snapshots to the specified storage",
Argument: "<storage name>",
},
cli.IntFlag {
cli.IntFlag{
Name: "download-limit-rate",
Value: 0,
Usage: "the maximum download rate (in kilobytes/sec)",
Argument: "<kB/s>",
},
cli.IntFlag {
cli.IntFlag{
Name: "upload-limit-rate",
Value: 0,
Usage: "the maximum upload rate (in kilobytes/sec)",
Argument: "<kB/s>",
},
cli.IntFlag {
cli.IntFlag{
Name: "threads",
Value: 1,
Usage: "number of downloading threads",
@@ -1664,17 +1660,17 @@ func main() {
{
Name: "info",
Flags: []cli.Flag {
cli.BoolFlag {
Flags: []cli.Flag{
cli.BoolFlag{
Name: "encrypt, e",
Usage: "The storage is encrypted with a password",
},
cli.StringFlag {
cli.StringFlag{
Name: "repository",
Usage: "retrieve saved passwords from the specified repository",
Argument: "<repository directory>",
},
cli.BoolFlag {
cli.BoolFlag{
Name: "reset-passwords",
Usage: "take passwords from input rather than keychain/keyring",
},
@@ -1683,31 +1679,30 @@ func main() {
ArgsUsage: "<storage url>",
Action: infoStorage,
},
}
app.Flags = []cli.Flag {
cli.BoolFlag {
app.Flags = []cli.Flag{
cli.BoolFlag{
Name: "verbose, v",
Usage: "show more detailed information",
},
cli.BoolFlag {
cli.BoolFlag{
Name: "debug, d",
Usage: "show even more detailed information, useful for debugging",
},
cli.BoolFlag {
cli.BoolFlag{
Name: "log",
Usage: "enable log-style output",
},
cli.BoolFlag {
cli.BoolFlag{
Name: "stack",
Usage: "print the stack trace when an error occurs",
},
cli.BoolFlag {
cli.BoolFlag{
Name: "no-script",
Usage: "do not run script before or after command execution",
},
cli.BoolFlag {
cli.BoolFlag{
Name: "background",
Usage: "read passwords, tokens, or keys only from keychain/keyring or env",
},

52
src/duplicacy_acdclient.go
View File

@@ -5,16 +5,16 @@
package duplicacy
import (
"fmt"
"time"
"bytes"
"sync"
"io/ioutil"
"encoding/json"
"fmt"
"io"
"net/http"
"mime/multipart"
"io/ioutil"
"math/rand"
"mime/multipart"
"net/http"
"sync"
"time"
"golang.org/x/oauth2"
)
@@ -29,6 +29,7 @@ func (err ACDError) Error() string {
}
var ACDRefreshTokenURL = "https://duplicacy.com/acd_refresh"
type ACDClient struct {
HTTPClient *http.Client
@@ -42,7 +43,6 @@ type ACDClient struct {
TestMode bool
}
func NewACDClient(tokenFile string) (*ACDClient, error) {
description, err := ioutil.ReadFile(tokenFile)
@@ -106,7 +106,7 @@ func (client *ACDClient) call(url string, method string, input interface{}, cont
if url != ACDRefreshTokenURL {
client.TokenLock.Lock()
request.Header.Set("Authorization", "Bearer " + client.Token.AccessToken)
request.Header.Set("Authorization", "Bearer "+client.Token.AccessToken)
client.TokenLock.Unlock()
}
if contentType != "" {
@@ -126,20 +126,20 @@ func (client *ACDClient) call(url string, method string, input interface{}, cont
buffer := new(bytes.Buffer)
buffer.ReadFrom(response.Body)
response.Body.Close()
return nil, 0, ACDError { Status: response.StatusCode, Message: buffer.String()}
return nil, 0, ACDError{Status: response.StatusCode, Message: buffer.String()}
}
if response.StatusCode == 400 {
defer response.Body.Close()
e := &ACDError {
e := &ACDError{
Status: response.StatusCode,
}
if err := json.NewDecoder(response.Body).Decode(e); err == nil {
return nil, 0, e
} else {
return nil, 0, ACDError { Status: response.StatusCode, Message: "Bad input parameter"}
return nil, 0, ACDError{Status: response.StatusCode, Message: "Bad input parameter"}
}
}
@@ -148,7 +148,7 @@ func (client *ACDClient) call(url string, method string, input interface{}, cont
if response.StatusCode == 401 {
if url == ACDRefreshTokenURL {
return nil, 0, ACDError { Status: response.StatusCode, Message: "Unauthorized"}
return nil, 0, ACDError{Status: response.StatusCode, Message: "Unauthorized"}
}
err = client.RefreshToken()
@@ -158,15 +158,15 @@ func (client *ACDClient) call(url string, method string, input interface{}, cont
continue
} else if response.StatusCode == 403 {
return nil, 0, ACDError { Status: response.StatusCode, Message: "Forbidden"}
return nil, 0, ACDError{Status: response.StatusCode, Message: "Forbidden"}
} else if response.StatusCode == 404 {
return nil, 0, ACDError { Status: response.StatusCode, Message: "Resource not found"}
return nil, 0, ACDError{Status: response.StatusCode, Message: "Resource not found"}
} else if response.StatusCode == 409 {
return nil, 0, ACDError { Status: response.StatusCode, Message: "Conflict"}
return nil, 0, ACDError{Status: response.StatusCode, Message: "Conflict"}
} else if response.StatusCode == 411 {
return nil, 0, ACDError { Status: response.StatusCode, Message: "Length required"}
return nil, 0, ACDError{Status: response.StatusCode, Message: "Length required"}
} else if response.StatusCode == 412 {
return nil, 0, ACDError { Status: response.StatusCode, Message: "Precondition failed"}
return nil, 0, ACDError{Status: response.StatusCode, Message: "Precondition failed"}
} else if response.StatusCode == 429 || response.StatusCode == 500 {
reason := "Too many requests"
if response.StatusCode == 500 {
@@ -178,9 +178,9 @@ func (client *ACDClient) call(url string, method string, input interface{}, cont
backoff *= 2
continue
} else if response.StatusCode == 503 {
return nil, 0, ACDError { Status: response.StatusCode, Message: "Service unavailable"}
return nil, 0, ACDError{Status: response.StatusCode, Message: "Service unavailable"}
} else {
return nil, 0, ACDError { Status: response.StatusCode, Message: "Unknown error"}
return nil, 0, ACDError{Status: response.StatusCode, Message: "Unknown error"}
}
}
@@ -231,7 +231,7 @@ func (client *ACDClient) GetEndpoint() (err error) {
defer readCloser.Close()
output := &ACDGetEndpointOutput {}
output := &ACDGetEndpointOutput{}
if err = json.NewDecoder(readCloser).Decode(&output); err != nil {
return err
@@ -287,7 +287,7 @@ func (client *ACDClient) ListEntries(parentID string, listFiles bool) ([]ACDEntr
defer readCloser.Close()
output := &ACDListEntriesOutput {}
output := &ACDListEntriesOutput{}
if err = json.NewDecoder(readCloser).Decode(&output); err != nil {
return nil, err
@@ -321,7 +321,7 @@ func (client *ACDClient) ListByName(parentID string, name string) (string, bool,
defer readCloser.Close()
output := &ACDListEntriesOutput {}
output := &ACDListEntriesOutput{}
if err = json.NewDecoder(readCloser).Decode(&output); err != nil {
return "", false, 0, err
@@ -351,7 +351,7 @@ func (client *ACDClient) UploadFile(parentID string, name string, content []byte
metadata := make(map[string]interface{})
metadata["name"] = name
metadata["kind"] = "FILE"
metadata["parents"] = []string{ parentID }
metadata["parents"] = []string{parentID}
metadataJSON, err := json.Marshal(metadata)
if err != nil {
@@ -389,7 +389,7 @@ func (client *ACDClient) UploadFile(parentID string, name string, content []byte
defer readCloser.Close()
entry := ACDEntry {}
entry := ACDEntry{}
if err = json.NewDecoder(readCloser).Decode(&entry); err != nil {
return "", err
}
@@ -434,7 +434,7 @@ func (client *ACDClient) CreateDirectory(parentID string, name string) (string,
parameters := make(map[string]interface{})
parameters["name"] = name
parameters["kind"] = "FOLDER"
parameters["parents"] = []string {parentID}
parameters["parents"] = []string{parentID}
readCloser, _, err := client.call(url, "POST", parameters, "")
if err != nil {
@@ -443,7 +443,7 @@ func (client *ACDClient) CreateDirectory(parentID string, name string) (string,
defer readCloser.Close()
entry := ACDEntry {}
entry := ACDEntry{}
if err = json.NewDecoder(readCloser).Decode(&entry); err != nil {
return "", err
}

8
src/duplicacy_acdclient_test.go
View File

@@ -5,11 +5,11 @@
package duplicacy
import (
"io"
"fmt"
"testing"
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
"testing"
crypto_rand "crypto/rand"
"math/rand"
@@ -81,7 +81,7 @@ func TestACDClient(t *testing.T) {
maxFileSize := 64 * 1024
for i := 0; i < numberOfFiles; i++ {
content := make([]byte, rand.Int() % maxFileSize + 1)
content := make([]byte, rand.Int()%maxFileSize+1)
_, err = crypto_rand.Read(content)
if err != nil {
t.Errorf("Error generating random content: %v", err)

48
src/duplicacy_acdstorage.go
View File

@@ -28,7 +28,7 @@ func CreateACDStorage(tokenFile string, storagePath string, threads int) (storag
return nil, err
}
storage = &ACDStorage {
storage = &ACDStorage{
client: client,
idCache: make(map[string]string),
idCacheLock: &sync.Mutex{},
@@ -42,7 +42,7 @@ func CreateACDStorage(tokenFile string, storagePath string, threads int) (storag
storage.idCache[""] = storagePathID
for _, dir := range []string { "chunks", "fossils", "snapshots" } {
for _, dir := range []string{"chunks", "fossils", "snapshots"} {
dirID, isDir, _, err := client.ListByName(storagePathID, dir)
if err != nil {
return nil, err
@@ -53,7 +53,7 @@ func CreateACDStorage(tokenFile string, storagePath string, threads int) (storag
return nil, err
}
} else if !isDir {
return nil, fmt.Errorf("%s/%s is not a directory", storagePath + "/" + dir)
return nil, fmt.Errorf("%s/%s is not a directory", storagePath+"/"+dir)
}
storage.idCache[dir] = dirID
}
@@ -88,10 +88,9 @@ func (storage *ACDStorage) deletePathID(path string) {
storage.idCacheLock.Unlock()
}
func (storage *ACDStorage) convertFilePath(filePath string) (string) {
func (storage *ACDStorage) convertFilePath(filePath string) string {
if strings.HasPrefix(filePath, "chunks/") && strings.HasSuffix(filePath, ".fsl") {
return "fossils/" + filePath[len("chunks/"):len(filePath) - len(".fsl")]
return "fossils/" + filePath[len("chunks/"):len(filePath)-len(".fsl")]
}
return filePath
}
@@ -113,13 +112,13 @@ func (storage *ACDStorage) getIDFromPath(threadIndex int, path string) (fileID s
return "", false, 0, err
}
if parentID == "" {
if i == len(names) - 1 {
if i == len(names)-1 {
return "", false, 0, nil
} else {
return "", false, 0, fmt.Errorf("File path '%s' does not exist", path)
}
}
if i != len(names) - 1 && !isDir {
if i != len(names)-1 && !isDir {
return "", false, 0, fmt.Errorf("Invalid path %s", path)
}
}
@@ -131,8 +130,8 @@ func (storage *ACDStorage) getIDFromPath(threadIndex int, path string) (fileID s
func (storage *ACDStorage) ListFiles(threadIndex int, dir string) ([]string, []int64, error) {
var err error
for len(dir) > 0 && dir[len(dir) - 1] == '/' {
dir = dir[:len(dir) - 1]
for len(dir) > 0 && dir[len(dir)-1] == '/' {
dir = dir[:len(dir)-1]
}
if dir == "snapshots" {
@@ -146,7 +145,7 @@ func (storage *ACDStorage) ListFiles(threadIndex int, dir string) ([]string, []i
for _, entry := range entries {
storage.savePathID(entry.Name, entry.ID)
subDirs = append(subDirs, entry.Name + "/")
subDirs = append(subDirs, entry.Name+"/")
}
return subDirs, nil, nil
} else if strings.HasPrefix(dir, "snapshots/") {
@@ -170,27 +169,26 @@ func (storage *ACDStorage) ListFiles(threadIndex int, dir string) ([]string, []i
files := []string{}
for _, entry := range entries {
storage.savePathID(dir + "/" + entry.Name, entry.ID)
storage.savePathID(dir+"/"+entry.Name, entry.ID)
files = append(files, entry.Name)
}
return files, nil, nil
} else {
files := []string{}
sizes := []int64{}
for _, parent := range []string {"chunks", "fossils" } {
for _, parent := range []string{"chunks", "fossils"} {
entries, err := storage.client.ListEntries(storage.getPathID(parent), true)
if err != nil {
return nil, nil, err
}
for _, entry := range entries {
name := entry.Name
if parent == "fossils" {
name += ".fsl"
}
storage.savePathID(parent + "/" + entry.Name, entry.ID)
storage.savePathID(parent+"/"+entry.Name, entry.ID)
files = append(files, name)
sizes = append(sizes, entry.Size)
}
@@ -259,8 +257,8 @@ func (storage *ACDStorage) MoveFile(threadIndex int, from string, to string) (er
// CreateDirectory creates a new directory.
func (storage *ACDStorage) CreateDirectory(threadIndex int, dir string) (err error) {
for len(dir) > 0 && dir[len(dir) - 1] == '/' {
dir = dir[:len(dir) - 1]
for len(dir) > 0 && dir[len(dir)-1] == '/' {
dir = dir[:len(dir)-1]
}
if dir == "chunks" || dir == "snapshots" {
@@ -288,8 +286,8 @@ func (storage *ACDStorage) CreateDirectory(threadIndex int, dir string) (err err
// GetFileInfo returns the information about the file or directory at 'filePath'.
func (storage *ACDStorage) GetFileInfo(threadIndex int, filePath string) (exist bool, isDir bool, size int64, err error) {
for len(filePath) > 0 && filePath[len(filePath) - 1] == '/' {
filePath = filePath[:len(filePath) - 1]
for len(filePath) > 0 && filePath[len(filePath)-1] == '/' {
filePath = filePath[:len(filePath)-1]
}
filePath = storage.convertFilePath(filePath)
@@ -348,7 +346,7 @@ func (storage *ACDStorage) DownloadFile(threadIndex int, filePath string, chunk
defer readCloser.Close()
_, err = RateLimitedCopy(chunk, readCloser, storage.DownloadRateLimit / storage.numberOfThreads)
_, err = RateLimitedCopy(chunk, readCloser, storage.DownloadRateLimit/storage.numberOfThreads)
return err
}
@@ -373,7 +371,7 @@ func (storage *ACDStorage) UploadFile(threadIndex int, filePath string, content
storage.savePathID(parent, parentID)
}
fileID, err := storage.client.UploadFile(parentID, path.Base(filePath), content, storage.UploadRateLimit / storage.numberOfThreads)
fileID, err := storage.client.UploadFile(parentID, path.Base(filePath), content, storage.UploadRateLimit/storage.numberOfThreads)
if err == nil {
storage.savePathID(filePath, fileID)
return nil
@@ -389,16 +387,16 @@ func (storage *ACDStorage) UploadFile(threadIndex int, filePath string, content
// If a local snapshot cache is needed for the storage to avoid downloading/uploading chunks too often when
// managing snapshots.
func (storage *ACDStorage) IsCacheNeeded() (bool) { return true }
func (storage *ACDStorage) IsCacheNeeded() bool { return true }
// If the 'MoveFile' method is implemented.
func (storage *ACDStorage) IsMoveFileImplemented() (bool) { return true }
func (storage *ACDStorage) IsMoveFileImplemented() bool { return true }
// If the storage can guarantee strong consistency.
func (storage *ACDStorage) IsStrongConsistent() (bool) { return true }
func (storage *ACDStorage) IsStrongConsistent() bool { return true }
// If the storage supports fast listing of files names.
func (storage *ACDStorage) IsFastListing() (bool) { return true }
func (storage *ACDStorage) IsFastListing() bool { return true }
// Enable the test mode.
func (storage *ACDStorage) EnableTestMode() {}

20
src/duplicacy_azurestorage.go
View File

@@ -43,7 +43,7 @@ func CreateAzureStorage(accountName string, accountKey string,
return nil, fmt.Errorf("container %s does not exist", containerName)
}
azureStorage = &AzureStorage {
azureStorage = &AzureStorage{
containers: containers,
}
@@ -62,12 +62,12 @@ func (azureStorage *AzureStorage) ListFiles(threadIndex int, dir string) (files
Timeout uint
}
if len(dir) > 0 && dir[len(dir) - 1] != '/' {
if len(dir) > 0 && dir[len(dir)-1] != '/' {
dir += "/"
}
dirLength := len(dir)
parameters := storage.ListBlobsParameters {
parameters := storage.ListBlobsParameters{
Prefix: dir,
Delimiter: "",
}
@@ -84,7 +84,7 @@ func (azureStorage *AzureStorage) ListFiles(threadIndex int, dir string) (files
if dir == "snapshots/" {
for _, blob := range results.Blobs {
name := strings.Split(blob.Name[dirLength:], "/")[0]
subDirs[name + "/"] = true
subDirs[name+"/"] = true
}
} else {
for _, blob := range results.Blobs {
@@ -175,13 +175,13 @@ func (storage *AzureStorage) DownloadFile(threadIndex int, filePath string, chun
defer readCloser.Close()
_, err = RateLimitedCopy(chunk, readCloser, storage.DownloadRateLimit / len(storage.containers))
_, err = RateLimitedCopy(chunk, readCloser, storage.DownloadRateLimit/len(storage.containers))
return err
}
// UploadFile writes 'content' to the file at 'filePath'.
func (storage *AzureStorage) UploadFile(threadIndex int, filePath string, content []byte) (err error) {
reader := CreateRateLimitedReader(content, storage.UploadRateLimit / len(storage.containers))
reader := CreateRateLimitedReader(content, storage.UploadRateLimit/len(storage.containers))
blob := storage.containers[threadIndex].GetBlobReference(filePath)
return blob.CreateBlockBlobFromReader(reader, nil)
@@ -189,16 +189,16 @@ func (storage *AzureStorage) UploadFile(threadIndex int, filePath string, conten
// If a local snapshot cache is needed for the storage to avoid downloading/uploading chunks too often when
// managing snapshots.
func (storage *AzureStorage) IsCacheNeeded() (bool) { return true }
func (storage *AzureStorage) IsCacheNeeded() bool { return true }
// If the 'MoveFile' method is implemented.
func (storage *AzureStorage) IsMoveFileImplemented() (bool) { return true }
func (storage *AzureStorage) IsMoveFileImplemented() bool { return true }
// If the storage can guarantee strong consistency.
func (storage *AzureStorage) IsStrongConsistent() (bool) { return true }
func (storage *AzureStorage) IsStrongConsistent() bool { return true }
// If the storage supports fast listing of files names.
func (storage *AzureStorage) IsFastListing() (bool) { return true }
func (storage *AzureStorage) IsFastListing() bool { return true }
// Enable the test mode.
func (storage *AzureStorage) EnableTestMode() {}

35
src/duplicacy_b2client.go
View File

@@ -5,19 +5,19 @@
package duplicacy
import (
"fmt"
"time"
"bytes"
"strconv"
"io/ioutil"
"encoding/json"
"crypto/sha1"
"encoding/base64"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"net/http"
"strings"
"crypto/sha1"
"io/ioutil"
"math/rand"
"net/http"
"strconv"
"strings"
"time"
)
type B2Error struct {
@@ -51,7 +51,6 @@ type B2Client struct {
UploadToken string
TestMode bool
}
func NewB2Client(accountID string, applicationKey string) *B2Client {
@@ -111,7 +110,7 @@ func (client *B2Client) call(url string, input interface{}) (io.ReadCloser, int6
}
if url == B2AuthorizationURL {
request.Header.Set("Authorization", "Basic " + base64.StdEncoding.EncodeToString([]byte(client.AccountID + ":" + client.ApplicationKey)))
request.Header.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(client.AccountID+":"+client.ApplicationKey)))
} else {
request.Header.Set("Authorization", client.AuthorizationToken)
}
@@ -168,8 +167,7 @@ func (client *B2Client) call(url string, input interface{}) (io.ReadCloser, int6
defer response.Body.Close()
e := &B2Error {
}
e := &B2Error{}
if err := json.NewDecoder(response.Body).Decode(e); err != nil {
return nil, 0, err
@@ -197,7 +195,7 @@ func (client *B2Client) AuthorizeAccount() (err error) {
defer readCloser.Close()
output := &B2AuthorizeAccountOutput {}
output := &B2AuthorizeAccountOutput{}
if err = json.NewDecoder(readCloser).Decode(&output); err != nil {
return err
@@ -299,8 +297,7 @@ func (client *B2Client) ListFileNames(startFileName string, singleFile bool, inc
defer readCloser.Close()
output := B2ListFileNamesOutput {
}
output := B2ListFileNamesOutput{}
if err = json.NewDecoder(readCloser).Decode(&output); err != nil {
return nil, err
@@ -382,7 +379,7 @@ func (client *B2Client) HideFile(fileName string) (fileID string, err error) {
defer readCloser.Close()
output := & B2HideFileOutput {}
output := &B2HideFileOutput{}
if err = json.NewDecoder(readCloser).Decode(&output); err != nil {
return "", err
@@ -405,7 +402,7 @@ type B2GetUploadArgumentOutput struct {
AuthorizationToken string
}
func (client *B2Client) getUploadURL() (error) {
func (client *B2Client) getUploadURL() error {
input := make(map[string]string)
input["bucketId"] = client.BucketID
@@ -417,7 +414,7 @@ func (client *B2Client) getUploadURL() (error) {
defer readCloser.Close()
output := & B2GetUploadArgumentOutput {}
output := &B2GetUploadArgumentOutput{}
if err = json.NewDecoder(readCloser).Decode(&output); err != nil {
return err
@@ -431,7 +428,6 @@ func (client *B2Client) getUploadURL() (error) {
func (client *B2Client) UploadFile(filePath string, content []byte, rateLimit int) (err error) {
hasher := sha1.New()
hasher.Write(content)
hash := hex.EncodeToString(hasher.Sum(nil))
@@ -517,4 +513,3 @@ func (client *B2Client) UploadFile(filePath string, content []byte, rateLimit in
return fmt.Errorf("Maximum backoff reached")
}

10
src/duplicacy_b2client_test.go
View File

@@ -5,15 +5,15 @@
package duplicacy
import (
"testing"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"testing"
crypto_rand "crypto/rand"
"math/rand"
"io"
"io/ioutil"
"math/rand"
)
func createB2ClientForTest(t *testing.T) (*B2Client, string) {
@@ -79,7 +79,7 @@ func TestB2Client(t *testing.T) {
maxSize := 10000
for i := 0; i < 20; i++ {
size := rand.Int() % maxSize + 1
size := rand.Int()%maxSize + 1
content := make([]byte, size)
_, err := crypto_rand.Read(content)
if err != nil {
@@ -90,7 +90,7 @@ func TestB2Client(t *testing.T) {
hash := sha256.Sum256(content)
name := hex.EncodeToString(hash[:])
err = b2Client.UploadFile(testDirectory + name, content, 100)
err = b2Client.UploadFile(testDirectory+name, content, 100)
if err != nil {
t.Errorf("Error uploading file '%s': %v", name, err)
return
@@ -118,7 +118,7 @@ func TestB2Client(t *testing.T) {
hash := hex.EncodeToString(hasher.Sum(nil))
if testDirectory + hash != file.FileName {
if testDirectory+hash != file.FileName {
t.Errorf("File %s has hash %s", file.FileName, hash)
}

32
src/duplicacy_b2storage.go
View File

@@ -35,7 +35,7 @@ func CreateB2Storage(accountID string, applicationKey string, bucket string, thr
clients = append(clients, client)
}
storage = &B2Storage {
storage = &B2Storage{
clients: clients,
}
return storage, nil
@@ -43,8 +43,8 @@ func CreateB2Storage(accountID string, applicationKey string, bucket string, thr
// ListFiles return the list of files and subdirectories under 'dir' (non-recursively)
func (storage *B2Storage) ListFiles(threadIndex int, dir string) (files []string, sizes []int64, err error) {
for len(dir) > 0 && dir[len(dir) - 1] == '/' {
dir = dir[:len(dir) - 1]
for len(dir) > 0 && dir[len(dir)-1] == '/' {
dir = dir[:len(dir)-1]
}
length := len(dir) + 1
@@ -65,7 +65,7 @@ func (storage *B2Storage) ListFiles(threadIndex int, dir string) (files []string
for _, entry := range entries {
name := entry.FileName[length:]
subDir := strings.Split(name, "/")[0]
subDirs[subDir + "/"] = true
subDirs[subDir+"/"] = true
}
for subDir, _ := range subDirs {
@@ -79,7 +79,7 @@ func (storage *B2Storage) ListFiles(threadIndex int, dir string) (files []string
}
lastFile = entry.FileName
if entry.Action == "hide" {
files = append(files, entry.FileName[length:] + ".fsl")
files = append(files, entry.FileName[length:]+".fsl")
} else {
files = append(files, entry.FileName[length:])
}
@@ -98,7 +98,7 @@ func (storage *B2Storage) ListFiles(threadIndex int, dir string) (files []string
func (storage *B2Storage) DeleteFile(threadIndex int, filePath string) (err error) {
if strings.HasSuffix(filePath, ".fsl") {
filePath = filePath[:len(filePath) - len(".fsl")]
filePath = filePath[:len(filePath)-len(".fsl")]
entries, err := storage.clients[threadIndex].ListFileNames(filePath, true, true)
if err != nil {
return err
@@ -107,7 +107,7 @@ func (storage *B2Storage) DeleteFile(threadIndex int, filePath string) (err erro
toBeDeleted := false
for _, entry := range entries {
if entry.FileName != filePath || (!toBeDeleted && entry.Action != "hide" ) {
if entry.FileName != filePath || (!toBeDeleted && entry.Action != "hide") {
continue
}
@@ -141,12 +141,12 @@ func (storage *B2Storage) MoveFile(threadIndex int, from string, to string) (err
if strings.HasSuffix(from, ".fsl") {
filePath = to
if from != to + ".fsl" {
if from != to+".fsl" {
filePath = ""
}
} else if strings.HasSuffix(to, ".fsl") {
filePath = from
if to != from + ".fsl" {
if to != from+".fsl" {
filePath = ""
}
}
@@ -182,7 +182,7 @@ func (storage *B2Storage) GetFileInfo(threadIndex int, filePath string) (exist b
isFossil := false
if strings.HasSuffix(filePath, ".fsl") {
isFossil = true
filePath = filePath[:len(filePath) - len(".fsl")]
filePath = filePath[:len(filePath)-len(".fsl")]
}
entries, err := storage.clients[threadIndex].ListFileNames(filePath, true, isFossil)
@@ -225,27 +225,27 @@ func (storage *B2Storage) DownloadFile(threadIndex int, filePath string, chunk *
defer readCloser.Close()
_, err = RateLimitedCopy(chunk, readCloser, storage.DownloadRateLimit / len(storage.clients))
_, err = RateLimitedCopy(chunk, readCloser, storage.DownloadRateLimit/len(storage.clients))
return err
}
// UploadFile writes 'content' to the file at 'filePath'.
func (storage *B2Storage) UploadFile(threadIndex int, filePath string, content []byte) (err error) {
return storage.clients[threadIndex].UploadFile(filePath, content, storage.UploadRateLimit / len(storage.clients))
return storage.clients[threadIndex].UploadFile(filePath, content, storage.UploadRateLimit/len(storage.clients))
}
// If a local snapshot cache is needed for the storage to avoid downloading/uploading chunks too often when
// managing snapshots.
func (storage *B2Storage) IsCacheNeeded() (bool) { return true }
func (storage *B2Storage) IsCacheNeeded() bool { return true }
// If the 'MoveFile' method is implemented.
func (storage *B2Storage) IsMoveFileImplemented() (bool) { return true }
func (storage *B2Storage) IsMoveFileImplemented() bool { return true }
// If the storage can guarantee strong consistency.
func (storage *B2Storage) IsStrongConsistent() (bool) { return true }
func (storage *B2Storage) IsStrongConsistent() bool { return true }
// If the storage supports fast listing of files names.
func (storage *B2Storage) IsFastListing() (bool) { return true }
func (storage *B2Storage) IsFastListing() bool { return true }
// Enable the test mode.
func (storage *B2Storage) EnableTestMode() {

150
src/duplicacy_backupmanager.go
View File

@@ -5,21 +5,21 @@
package duplicacy
import (
"encoding/json"
"bytes"
"os"
"io"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"os"
"path"
"time"
"path/filepath"
"runtime"
"sort"
"strconv"
"strings"
"sync"
"sync/atomic"
"strings"
"strconv"
"runtime"
"encoding/hex"
"path/filepath"
"time"
)
// BackupManager performs the two major operations, backup and restore, and passes other operations, mostly related to
@@ -35,12 +35,10 @@ type BackupManager struct {
config *Config // contains a number of options
}
func (manager *BackupManager) SetDryRun(dryRun bool) {
manager.config.dryRun = dryRun
}
// CreateBackupManager creates a backup manager using the specified 'storage'. 'snapshotID' is a unique id to
// identify snapshots created for this repository. 'top' is the top directory of the repository. 'password' is the
// master key which can be nil if encryption is not enabled.
@@ -58,7 +56,7 @@ func CreateBackupManager(snapshotID string, storage Storage, top string, passwor
snapshotManager := CreateSnapshotManager(config, storage)
backupManager := &BackupManager {
backupManager := &BackupManager{
snapshotID: snapshotID,
storage: storage,
@@ -87,7 +85,7 @@ func (manager *BackupManager) SetupSnapshotCache(storageName string) bool {
return false
}
for _, subdir := range [] string { "chunks", "snapshots" } {
for _, subdir := range []string{"chunks", "snapshots"} {
err := os.Mkdir(path.Join(cacheDir, subdir), 0744)
if err != nil && !os.IsExist(err) {
LOG_ERROR("BACKUP_CACHE", "Failed to create the snapshot cache subdir: %v", err)
@@ -106,7 +104,7 @@ func (manager *BackupManager) SetupSnapshotCache(storageName string) bool {
//
// This function assumes the Size field of each entry is equal to the length of the chunk content that belong
// to the file.
func setEntryContent(entries[] *Entry, chunkLengths[]int, offset int) {
func setEntryContent(entries []*Entry, chunkLengths []int, offset int) {
if len(entries) == 0 {
return
}
@@ -120,7 +118,7 @@ func setEntryContent(entries[] *Entry, chunkLengths[]int, offset int) {
entries[i].StartOffset = 0
for j, length := range chunkLengths {
for totalChunkSize + int64(length) >= totalFileSize {
for totalChunkSize+int64(length) >= totalFileSize {
entries[i].EndChunk = j + offset
entries[i].EndOffset = int(totalFileSize - totalChunkSize)
@@ -131,7 +129,7 @@ func setEntryContent(entries[] *Entry, chunkLengths[]int, offset int) {
// If the current file ends at the end of the current chunk, the next file will
// start at the next chunk
if totalChunkSize + int64(length) == totalFileSize {
if totalChunkSize+int64(length) == totalFileSize {
entries[i].StartChunk = j + 1 + offset
entries[i].StartOffset = 0
} else {
@@ -217,7 +215,7 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
allChunks, _ := manager.SnapshotManager.ListAllFiles(manager.storage, "chunks/")
for _, chunk := range allChunks {
if len(chunk) == 0 || chunk[len(chunk) - 1] == '/' {
if len(chunk) == 0 || chunk[len(chunk)-1] == '/' {
continue
}
@@ -230,7 +228,6 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
}
}
if incompleteSnapshot != nil {
// This is the last chunk from the incomplete snapshot that can be found in the cache
@@ -256,8 +253,8 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
incompleteSnapshot.Files = files
// Remove incomplete chunks (they may not have been uploaded)
incompleteSnapshot.ChunkHashes = incompleteSnapshot.ChunkHashes[:lastCompleteChunk + 1]
incompleteSnapshot.ChunkLengths = incompleteSnapshot.ChunkLengths[:lastCompleteChunk + 1]
incompleteSnapshot.ChunkHashes = incompleteSnapshot.ChunkHashes[:lastCompleteChunk+1]
incompleteSnapshot.ChunkLengths = incompleteSnapshot.ChunkLengths[:lastCompleteChunk+1]
remoteSnapshot = incompleteSnapshot
LOG_INFO("FILE_SKIP", "Skipped %d files from previous incomplete backup", len(files))
}
@@ -276,8 +273,8 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
var totalModifiedFileSize int64 // total size of modified files
var uploadedModifiedFileSize int64 // portions that have been uploaded (including cache hits)
var modifiedEntries [] *Entry // Files that has been modified or newly created
var preservedEntries [] *Entry // Files unchanges
var modifiedEntries []*Entry // Files that has been modified or newly created
var preservedEntries []*Entry // Files unchanges
// If the quick mode is disable and there isn't an incomplete snapshot from last (failed) backup,
// we simply treat all files as if they were new, and break them into chunks.
@@ -361,7 +358,7 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
entry.EndChunk -= deletedChunks
}
var uploadedEntries [] *Entry
var uploadedEntries []*Entry
var uploadedChunkHashes []string
var uploadedChunkLengths []int
var uploadedChunkLock = &sync.Mutex{}
@@ -398,7 +395,6 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
LOG_INFO("SNAPSHOT_FAIL", "Will abort the backup on chunk %d", chunkToFail)
}
chunkMaker := CreateChunkMaker(manager.config, false)
chunkUploader := CreateChunkUploader(manager.config, manager.storage, nil, threads, nil)
@@ -472,11 +468,11 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
speed := uploadedModifiedFileSize / (now - startUploadingTime)
remainingTime := int64(0)
if speed > 0 {
remainingTime = (totalModifiedFileSize - uploadedModifiedFileSize) / speed + 1
remainingTime = (totalModifiedFileSize-uploadedModifiedFileSize)/speed + 1
}
percentage := float32(uploadedModifiedFileSize * 1000 / totalModifiedFileSize)
LOG_INFO("UPLOAD_PROGRESS", "%s chunk %d size %d, %sB/s %s %.1f%%", action, chunkIndex,
chunkSize, PrettySize(speed), PrettyTime(remainingTime), percentage / 10)
chunkSize, PrettySize(speed), PrettyTime(remainingTime), percentage/10)
}
atomic.AddInt64(&numberOfCollectedChunks, 1)
@@ -488,7 +484,7 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
// Break files into chunks
chunkMaker.ForEachChunk(
fileReader.CurrentFile,
func (chunk *Chunk, final bool) {
func(chunk *Chunk, final bool) {
hash := chunk.GetHash()
chunkID := chunk.GetID()
@@ -498,7 +494,7 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
_, found := chunkCache[chunkID]
if found {
if time.Now().Unix() - lastUploadingTime > keepUploadAlive {
if time.Now().Unix()-lastUploadingTime > keepUploadAlive {
LOG_INFO("UPLOAD_KEEPALIVE", "Skip chunk cache to keep connection alive")
found = false
}
@@ -524,7 +520,7 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
}
},
func (fileSize int64, hash string) (io.Reader, bool) {
func(fileSize int64, hash string) (io.Reader, bool) {
// Must lock here because the RunAtError function called by other threads may access uploadedEntries
uploadedChunkLock.Lock()
@@ -647,8 +643,8 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
if showStatistics {
LOG_INFO("BACKUP_STATS", "Files: %d total, %s bytes; %d new, %s bytes",
len(preservedEntries) + len(uploadedEntries),
PrettyNumber(preservedFileSize + uploadedFileSize),
len(preservedEntries)+len(uploadedEntries),
PrettyNumber(preservedFileSize+uploadedFileSize),
len(uploadedEntries), PrettyNumber(uploadedFileSize))
LOG_INFO("BACKUP_STATS", "File chunks: %d total, %s bytes; %d new, %s bytes, %s bytes uploaded",
@@ -662,17 +658,17 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
PrettyNumber(totalUploadedSnapshotChunkBytes))
LOG_INFO("BACKUP_STATS", "All chunks: %d total, %s bytes; %d new, %s bytes, %s bytes uploaded",
len(localSnapshot.ChunkHashes) + totalSnapshotChunks,
PrettyNumber(totalFileChunkLength + totalSnapshotChunkLength),
int(numberOfNewFileChunks) + numberOfNewSnapshotChunks,
PrettyNumber(totalUploadedFileChunkLength + totalUploadedSnapshotChunkLength),
PrettyNumber(totalUploadedFileChunkBytes + totalUploadedSnapshotChunkBytes))
len(localSnapshot.ChunkHashes)+totalSnapshotChunks,
PrettyNumber(totalFileChunkLength+totalSnapshotChunkLength),
int(numberOfNewFileChunks)+numberOfNewSnapshotChunks,
PrettyNumber(totalUploadedFileChunkLength+totalUploadedSnapshotChunkLength),
PrettyNumber(totalUploadedFileChunkBytes+totalUploadedSnapshotChunkBytes))
now := time.Now().Unix()
if now == startTime {
now = startTime + 1
}
LOG_INFO("BACKUP_STATS", "Total running time: %s", PrettyTime(now - startTime))
LOG_INFO("BACKUP_STATS", "Total running time: %s", PrettyTime(now-startTime))
}
skipped := ""
@@ -696,7 +692,7 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
}
if len(skipped) > 0 {
if len(skippedDirectories) + len(skippedFiles) == 1 {
if len(skippedDirectories)+len(skippedFiles) == 1 {
skipped += " was"
} else {
skipped += " were"
@@ -715,7 +711,7 @@ func (manager *BackupManager) Backup(top string, quickMode bool, threads int, ta
// the same as 'top'. 'quickMode' will bypass files with unchanged sizes and timestamps. 'deleteMode' will
// remove local files that don't exist in the snapshot. 'patterns' is used to include/exclude certain files.
func (manager *BackupManager) Restore(top string, revision int, inPlace bool, quickMode bool, threads int, overwrite bool,
deleteMode bool, showStatistics bool, patterns [] string) bool {
deleteMode bool, showStatistics bool, patterns []string) bool {
startTime := time.Now().Unix()
@@ -736,7 +732,7 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
_, err := os.Stat(top)
if os.IsNotExist(err) {
err = os.Mkdir(top, 0744)
if err != nil{
if err != nil {
LOG_ERROR("RESTORE_MKDIR", "Can't create the directory to be restored: %v", err)
return false
}
@@ -760,7 +756,7 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
LOG_INFO("RESTORE_START", "Restoring %s to revision %d", top, revision)
var includedFiles [] *Entry
var includedFiles []*Entry
// Include/exclude some files if needed
if len(patterns) > 0 {
@@ -781,7 +777,7 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
var extraFiles []string
// These will store files to be downloaded.
fileEntries := make([]*Entry, 0, len(remoteSnapshot.Files) / 2)
fileEntries := make([]*Entry, 0, len(remoteSnapshot.Files)/2)
var totalFileSize int64
var downloadedFileSize int64
@@ -817,7 +813,7 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
if entry.IsLink() {
stat, err := os.Lstat(fullPath)
if stat != nil {
if stat.Mode() & os.ModeSymlink != 0 {
if stat.Mode()&os.ModeSymlink != 0 {
isRegular, link, err := Readlink(fullPath)
if err == nil && link == entry.Link && !isRegular {
continue
@@ -900,7 +896,7 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
// Handle zero size files.
if file.Size == 0 {
newFile, err := os.OpenFile(fullPath, os.O_WRONLY | os.O_CREATE | os.O_TRUNC, file.GetPermissions())
newFile, err := os.OpenFile(fullPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, file.GetPermissions())
if err != nil {
LOG_ERROR("DOWNLOAD_OPEN", "Failed to create empty file: %v", err)
return false
@@ -924,11 +920,10 @@ func (manager *BackupManager) Restore(top string, revision int, inPlace bool, qu
file.RestoreMetadata(fullPath, nil)
}
if deleteMode && len(patterns) == 0 {
// Reverse the order to make sure directories are empty before being deleted
for i := range extraFiles {
file := extraFiles[len(extraFiles) - 1 - i]
file := extraFiles[len(extraFiles)-1-i]
fullPath := joinPath(top, file)
os.Remove(fullPath)
LOG_INFO("RESTORE_DELETE", "Deleted %s", file)
@@ -988,7 +983,7 @@ func (encoder *fileEncoder) NextFile() (io.Reader, bool) {
if encoder.currentIndex == len(encoder.files) {
return nil, false
}
if encoder.currentIndex == len(encoder.files) - 1 {
if encoder.currentIndex == len(encoder.files)-1 {
encoder.buffer.Write([]byte("]"))
encoder.currentIndex++
return encoder, true
@@ -1019,7 +1014,7 @@ func (encoder *fileEncoder) NextFile() (io.Reader, bool) {
// UploadSnapshot uploads the specified snapshot to the storage. It turns Files, ChunkHashes, and ChunkLengths into
// sequences of chunks, and uploads these chunks, and finally the snapshot file.
func (manager *BackupManager) UploadSnapshot(chunkMaker *ChunkMaker, uploader *ChunkUploader, top string, snapshot *Snapshot,
chunkCache map[string]bool, ) ( totalSnapshotChunkSize int64,
chunkCache map[string]bool) (totalSnapshotChunkSize int64,
numberOfNewSnapshotChunks int, totalUploadedSnapshotChunkSize int64,
totalUploadedSnapshotChunkBytes int64) {
@@ -1046,10 +1041,10 @@ func (manager *BackupManager) UploadSnapshot(chunkMaker *ChunkMaker, uploader *C
// uploadSequenceFunc uploads chunks read from 'reader'.
uploadSequenceFunc := func(reader io.Reader,
nextReader func(size int64, hash string)(io.Reader, bool)) (sequence[]string) {
nextReader func(size int64, hash string) (io.Reader, bool)) (sequence []string) {
chunkMaker.ForEachChunk(reader,
func (chunk *Chunk, final bool) {
func(chunk *Chunk, final bool) {
totalSnapshotChunkSize += int64(chunk.GetLength())
chunkID := chunk.GetID()
if _, found := chunkCache[chunkID]; found {
@@ -1064,7 +1059,7 @@ func (manager *BackupManager) UploadSnapshot(chunkMaker *ChunkMaker, uploader *C
return sequence
}
sequences := []string { "chunks", "lengths" }
sequences := []string{"chunks", "lengths"}
// The file list is assumed not to be too large when fixed-size chunking is used
if chunkMaker.minimumChunkSize == chunkMaker.maximumChunkSize {
sequences = append(sequences, "files")
@@ -1081,7 +1076,7 @@ func (manager *BackupManager) UploadSnapshot(chunkMaker *ChunkMaker, uploader *C
}
sequence := uploadSequenceFunc(bytes.NewReader(contents),
func (fileSize int64, hash string) (io.Reader, bool) {
func(fileSize int64, hash string) (io.Reader, bool) {
return nil, false
})
snapshot.SetSequence(sequenceType, sequence)
@@ -1090,7 +1085,7 @@ func (manager *BackupManager) UploadSnapshot(chunkMaker *ChunkMaker, uploader *C
// File sequence may be too big to fit into the memory. So we encode files one by one and take advantages of
// the multi-reader capability of the chunk maker.
if chunkMaker.minimumChunkSize != chunkMaker.maximumChunkSize {
encoder := fileEncoder {
encoder := fileEncoder{
top: top,
readAttributes: snapshot.discardAttributes,
files: snapshot.Files,
@@ -1100,7 +1095,7 @@ func (manager *BackupManager) UploadSnapshot(chunkMaker *ChunkMaker, uploader *C
encoder.buffer.Write([]byte("["))
sequence := uploadSequenceFunc(encoder,
func (fileSize int64, hash string) (io.Reader, bool) {
func(fileSize int64, hash string) (io.Reader, bool) {
return encoder.NextFile()
})
snapshot.SetSequence("files", sequence)
@@ -1147,24 +1142,24 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
if temporaryPath != fullPath {
os.Remove(temporaryPath)
}
} ()
}()
// These are used to break the existing file into chunks.
var existingChunks [] string
var existingLengths [] int
var existingChunks []string
var existingLengths []int
// These are to enable fast lookup of what chunks are available in the existing file.
offsetMap := make(map[string] int64)
lengthMap := make(map[string] int)
offsetMap := make(map[string]int64)
lengthMap := make(map[string]int)
var offset int64
existingFile, err = os.Open(fullPath)
if err != nil {
if os.IsNotExist(err) {
// macOS has no sparse file support
if inPlace && entry.Size > 100 * 1024 * 1024 && runtime.GOOS != "darwin" {
if inPlace && entry.Size > 100*1024*1024 && runtime.GOOS != "darwin" {
// Create an empty sparse file
existingFile, err = os.OpenFile(fullPath, os.O_WRONLY | os.O_CREATE | os.O_TRUNC, 0600)
existingFile, err = os.OpenFile(fullPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
if err != nil {
LOG_ERROR("DOWNLOAD_CREATE", "Failed to create the file %s for in-place writing: %v", fullPath, err)
return false
@@ -1173,10 +1168,10 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
n := int64(1)
// There is a go bug on Windows (https://github.com/golang/go/issues/21681) that causes Seek to fail
// if the lower 32 bit of the offset argument is 0xffffffff. Therefore we need to avoid that value by increasing n.
if uint32(entry.Size) == 0 && (entry.Size >> 32) > 0 {
if uint32(entry.Size) == 0 && (entry.Size>>32) > 0 {
n = int64(2)
}
_, err = existingFile.Seek(entry.Size - n, 0)
_, err = existingFile.Seek(entry.Size-n, 0)
if err != nil {
LOG_ERROR("DOWNLOAD_CREATE", "Failed to resize the initial file %s for in-place writing: %v", fullPath, err)
return false
@@ -1211,11 +1206,11 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
// In inplace mode, we only consider chunks in the existing file with the same offsets, so we
// break the original file at offsets retrieved from the backup
fileHasher := manager.config.NewFileHasher()
buffer := make([]byte, 64 * 1024)
buffer := make([]byte, 64*1024)
err = nil
// We set to read one more byte so the file hash will be different if the file to be restored is a
// truncated portion of the existing file
for i := entry.StartChunk; i <= entry.EndChunk + 1; i++ {
for i := entry.StartChunk; i <= entry.EndChunk+1; i++ {
hasher := manager.config.NewKeyedHasher(manager.config.HashKey)
chunkSize := 1 // the size of extra chunk beyond EndChunk
if i == entry.StartChunk {
@@ -1264,7 +1259,7 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
// we run the chunk maker to split the original file.
chunkMaker.ForEachChunk(
existingFile,
func (chunk *Chunk, final bool) {
func(chunk *Chunk, final bool) {
hash := chunk.GetHash()
chunkSize := chunk.GetLength()
existingChunks = append(existingChunks, hash)
@@ -1273,7 +1268,7 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
lengthMap[hash] = chunkSize
offset += int64(chunkSize)
},
func (fileSize int64, hash string) (io.Reader, bool) {
func(fileSize int64, hash string) (io.Reader, bool) {
fileHash = hash
return nil, false
})
@@ -1298,7 +1293,7 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
if existingFile == nil {
// Create an empty file
existingFile, err = os.OpenFile(fullPath, os.O_WRONLY | os.O_CREATE | os.O_TRUNC, 0600)
existingFile, err = os.OpenFile(fullPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
if err != nil {
LOG_ERROR("DOWNLOAD_CREATE", "Failed to create the file %s for in-place writing", fullPath)
}
@@ -1357,12 +1352,12 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
}
} else {
chunk := chunkDownloader.WaitForChunk(i)
_, err = existingFile.Write(chunk.GetBytes()[start: end])
_, err = existingFile.Write(chunk.GetBytes()[start:end])
if err != nil {
LOG_ERROR("DOWNLOAD_WRITE", "Failed to write to the file: %v", err)
return false
}
hasher.Write(chunk.GetBytes()[start: end])
hasher.Write(chunk.GetBytes()[start:end])
}
offset += int64(end - start)
@@ -1385,7 +1380,7 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
} else {
// Create the temporary file.
newFile, err = os.OpenFile(temporaryPath, os.O_WRONLY | os.O_CREATE | os.O_TRUNC, 0600)
newFile, err = os.OpenFile(temporaryPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
if err != nil {
LOG_ERROR("DOWNLOAD_OPEN", "Failed to open file for writing: %v", err)
return false
@@ -1435,7 +1430,7 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
if i == entry.EndChunk {
end = entry.EndOffset
}
data = chunk.GetBytes()[start: end]
data = chunk.GetBytes()[start:end]
}
_, err = newFile.Write(data)
@@ -1455,7 +1450,6 @@ func (manager *BackupManager) RestoreFile(chunkDownloader *ChunkDownloader, chun
return false
}
if existingFile != nil {
existingFile.Close()
existingFile = nil
@@ -1508,8 +1502,8 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho
revisionMap[snapshotID][revision] = true
}
var snapshots [] *Snapshot
var snapshotIDs [] string
var snapshots []*Snapshot
var snapshotIDs []string
var err error
if snapshotID == "" {
@@ -1519,7 +1513,7 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho
return false
}
} else {
snapshotIDs = [] string { snapshotID }
snapshotIDs = []string{snapshotID}
}
for _, id := range snapshotIDs {
@@ -1643,7 +1637,7 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho
}
}
LOG_DEBUG("SNAPSHOT_COPY", "Chunks to copy = %d, to skip = %d, total = %d", chunksToCopy, chunksToSkip, chunksToCopy + chunksToSkip)
LOG_DEBUG("SNAPSHOT_COPY", "Chunks to copy = %d, to skip = %d, total = %d", chunksToCopy, chunksToSkip, chunksToCopy+chunksToSkip)
LOG_DEBUG("SNAPSHOT_COPY", "Total chunks in source snapshot revisions = %d\n", len(chunks))
chunkDownloader := CreateChunkDownloader(manager.config, manager.storage, nil, false, threads)
@@ -1686,7 +1680,7 @@ func (manager *BackupManager) CopySnapshots(otherManager *BackupManager, snapsho
chunkDownloader.Stop()
chunkUploader.Stop()
LOG_INFO("SNAPSHOT_COPY", "Copy complete, %d total chunks, %d chunks copied, %d skipped", totalCopied + totalSkipped, totalCopied, totalSkipped)
LOG_INFO("SNAPSHOT_COPY", "Copy complete, %d total chunks, %d chunks copied, %d skipped", totalCopied+totalSkipped, totalCopied, totalSkipped)
for _, snapshot := range snapshots {
if revisionMap[snapshot.ID][snapshot.Revision] == false {

113
src/duplicacy_backupmanager_test.go
View File

@@ -5,21 +5,21 @@
package duplicacy
import (
"os"
crypto_rand "crypto/rand"
"crypto/sha256"
"encoding/hex"
"io"
"math/rand"
"os"
"path"
"testing"
"math/rand"
"encoding/hex"
"time"
"crypto/sha256"
crypto_rand "crypto/rand"
"runtime/debug"
)
func createRandomFile(path string, maxSize int) {
file, err := os.OpenFile(path, os.O_WRONLY | os.O_CREATE | os.O_TRUNC, 0644)
file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
if err != nil {
LOG_ERROR("RANDOM_FILE", "Can't open %s for writing: %v", path, err)
return
@@ -27,9 +27,9 @@ func createRandomFile(path string, maxSize int) {
defer file.Close()
size := maxSize / 2 + rand.Int() % (maxSize / 2)
size := maxSize/2 + rand.Int()%(maxSize/2)
buffer := make([]byte, 32 * 1024)
buffer := make([]byte, 32*1024)
for size > 0 {
bytes := size
if bytes > cap(buffer) {
@@ -65,7 +65,7 @@ func modifyFile(path string, portion float32) {
if file != nil {
file.Close()
}
} ()
}()
size, err := file.Seek(0, 2)
if err != nil {
@@ -73,7 +73,7 @@ func modifyFile(path string, portion float32) {
return
}
length := int (float32(size) * portion)
length := int(float32(size) * portion)
start := rand.Int() % (int(size) - length)
_, err = file.Seek(int64(start), 0)
@@ -186,28 +186,28 @@ func TestBackupManager(t *testing.T) {
debug.PrintStack()
}
}
} ()
}()
testDir := path.Join(os.TempDir(), "duplicacy_test")
os.RemoveAll(testDir)
os.MkdirAll(testDir, 0700)
os.Mkdir(testDir + "/repository1", 0700)
os.Mkdir(testDir + "/repository1/dir1", 0700)
os.Mkdir(testDir + "/repository1/.duplicacy", 0700)
os.Mkdir(testDir + "/repository2", 0700)
os.Mkdir(testDir + "/repository2/.duplicacy", 0700)
os.Mkdir(testDir+"/repository1", 0700)
os.Mkdir(testDir+"/repository1/dir1", 0700)
os.Mkdir(testDir+"/repository1/.duplicacy", 0700)
os.Mkdir(testDir+"/repository2", 0700)
os.Mkdir(testDir+"/repository2/.duplicacy", 0700)
maxFileSize := 1000000
//maxFileSize := 200000
createRandomFile(testDir + "/repository1/file1", maxFileSize)
createRandomFile(testDir + "/repository1/file2", maxFileSize)
createRandomFile(testDir + "/repository1/dir1/file3", maxFileSize)
createRandomFile(testDir+"/repository1/file1", maxFileSize)
createRandomFile(testDir+"/repository1/file2", maxFileSize)
createRandomFile(testDir+"/repository1/dir1/file3", maxFileSize)
threads := 1
storage, err := loadStorage(testDir + "/storage", threads)
storage, err := loadStorage(testDir+"/storage", threads)
if err != nil {
t.Errorf("Failed to create storage: %v", err)
return
@@ -227,16 +227,15 @@ func TestBackupManager(t *testing.T) {
time.Sleep(time.Duration(delay) * time.Second)
if testFixedChunkSize {
if !ConfigStorage(storage, 100, 64 * 1024, 64 * 1024, 64 * 1024, password, nil) {
if !ConfigStorage(storage, 100, 64*1024, 64*1024, 64*1024, password, nil) {
t.Errorf("Failed to initialize the storage")
}
} else {
if !ConfigStorage(storage, 100, 64 * 1024, 256 * 1024, 16 * 1024, password, nil) {
if !ConfigStorage(storage, 100, 64*1024, 256*1024, 16*1024, password, nil) {
t.Errorf("Failed to initialize the storage")
}
}
time.Sleep(time.Duration(delay) * time.Second)
SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy")
@@ -244,13 +243,13 @@ func TestBackupManager(t *testing.T) {
backupManager.SetupSnapshotCache("default")
SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy")
backupManager.Backup(testDir + "/repository1", /*quickMode=*/true, threads, "first", false, false)
backupManager.Backup(testDir+"/repository1" /*quickMode=*/, true, threads, "first", false, false)
time.Sleep(time.Duration(delay) * time.Second)
SetDuplicacyPreferencePath(testDir + "/repository2/.duplicacy")
backupManager.Restore(testDir + "/repository2", threads, /*inPlace=*/false, /*quickMode=*/false, threads, /*overwrite=*/true,
/*deleteMode=*/false, /*showStatistics=*/false, /*patterns=*/nil)
backupManager.Restore(testDir+"/repository2", threads /*inPlace=*/, false /*quickMode=*/, false, threads /*overwrite=*/, true,
/*deleteMode=*/ false /*showStatistics=*/, false /*patterns=*/, nil)
for _, f := range []string{ "file1", "file2", "dir1/file3" } {
for _, f := range []string{"file1", "file2", "dir1/file3"} {
if _, err := os.Stat(testDir + "/repository2/" + f); os.IsNotExist(err) {
t.Errorf("File %s does not exist", f)
continue
@@ -263,18 +262,18 @@ func TestBackupManager(t *testing.T) {
}
}
modifyFile(testDir + "/repository1/file1", 0.1)
modifyFile(testDir + "/repository1/file2", 0.2)
modifyFile(testDir + "/repository1/dir1/file3", 0.3)
modifyFile(testDir+"/repository1/file1", 0.1)
modifyFile(testDir+"/repository1/file2", 0.2)
modifyFile(testDir+"/repository1/dir1/file3", 0.3)
SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy")
backupManager.Backup(testDir + "/repository1", /*quickMode=*/true, threads, "second", false, false)
backupManager.Backup(testDir+"/repository1" /*quickMode=*/, true, threads, "second", false, false)
time.Sleep(time.Duration(delay) * time.Second)
SetDuplicacyPreferencePath(testDir + "/repository2/.duplicacy")
backupManager.Restore(testDir + "/repository2", 2, /*inPlace=*/true, /*quickMode=*/true, threads, /*overwrite=*/true,
/*deleteMode=*/false, /*showStatistics=*/false, /*patterns=*/nil)
backupManager.Restore(testDir+"/repository2", 2 /*inPlace=*/, true /*quickMode=*/, true, threads /*overwrite=*/, true,
/*deleteMode=*/ false /*showStatistics=*/, false /*patterns=*/, nil)
for _, f := range []string{ "file1", "file2", "dir1/file3" } {
for _, f := range []string{"file1", "file2", "dir1/file3"} {
hash1 := getFileHash(testDir + "/repository1/" + f)
hash2 := getFileHash(testDir + "/repository2/" + f)
if hash1 != hash2 {
@@ -284,25 +283,25 @@ func TestBackupManager(t *testing.T) {
// Truncate file2 and add a few empty directories
truncateFile(testDir + "/repository1/file2")
os.Mkdir(testDir + "/repository1/dir2", 0700)
os.Mkdir(testDir + "/repository1/dir2/dir3", 0700)
os.Mkdir(testDir + "/repository1/dir4", 0700)
os.Mkdir(testDir+"/repository1/dir2", 0700)
os.Mkdir(testDir+"/repository1/dir2/dir3", 0700)
os.Mkdir(testDir+"/repository1/dir4", 0700)
SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy")
backupManager.Backup(testDir + "/repository1", /*quickMode=*/false, threads, "third", false, false)
backupManager.Backup(testDir+"/repository1" /*quickMode=*/, false, threads, "third", false, false)
time.Sleep(time.Duration(delay) * time.Second)
// Create some directories and files under repository2 that will be deleted during restore
os.Mkdir(testDir + "/repository2/dir5", 0700)
os.Mkdir(testDir + "/repository2/dir5/dir6", 0700)
os.Mkdir(testDir + "/repository2/dir7", 0700)
createRandomFile(testDir + "/repository2/file4", 100)
createRandomFile(testDir + "/repository2/dir5/file5", 100)
os.Mkdir(testDir+"/repository2/dir5", 0700)
os.Mkdir(testDir+"/repository2/dir5/dir6", 0700)
os.Mkdir(testDir+"/repository2/dir7", 0700)
createRandomFile(testDir+"/repository2/file4", 100)
createRandomFile(testDir+"/repository2/dir5/file5", 100)
SetDuplicacyPreferencePath(testDir + "/repository2/.duplicacy")
backupManager.Restore(testDir + "/repository2", 3, /*inPlace=*/true, /*quickMode=*/false, threads, /*overwrite=*/true,
/*deleteMode=*/true, /*showStatistics=*/false, /*patterns=*/nil)
backupManager.Restore(testDir+"/repository2", 3 /*inPlace=*/, true /*quickMode=*/, false, threads /*overwrite=*/, true,
/*deleteMode=*/ true /*showStatistics=*/, false /*patterns=*/, nil)
for _, f := range []string{ "file1", "file2", "dir1/file3" } {
for _, f := range []string{"file1", "file2", "dir1/file3"} {
hash1 := getFileHash(testDir + "/repository1/" + f)
hash2 := getFileHash(testDir + "/repository2/" + f)
if hash1 != hash2 {
@@ -311,25 +310,25 @@ func TestBackupManager(t *testing.T) {
}
// These files/dirs should not exist because deleteMode == true
checkExistence(t, testDir + "/repository2/dir5", false, false);
checkExistence(t, testDir + "/repository2/dir5/dir6", false, false);
checkExistence(t, testDir + "/repository2/dir7", false, false);
checkExistence(t, testDir + "/repository2/file4", false, false);
checkExistence(t, testDir + "/repository2/dir5/file5", false, false);
checkExistence(t, testDir+"/repository2/dir5", false, false)
checkExistence(t, testDir+"/repository2/dir5/dir6", false, false)
checkExistence(t, testDir+"/repository2/dir7", false, false)
checkExistence(t, testDir+"/repository2/file4", false, false)
checkExistence(t, testDir+"/repository2/dir5/file5", false, false)
// These empty dirs should exist
checkExistence(t, testDir + "/repository2/dir2", true, true);
checkExistence(t, testDir + "/repository2/dir2/dir3", true, true);
checkExistence(t, testDir + "/repository2/dir4", true, true);
checkExistence(t, testDir+"/repository2/dir2", true, true)
checkExistence(t, testDir+"/repository2/dir2/dir3", true, true)
checkExistence(t, testDir+"/repository2/dir4", true, true)
// Remove file2 and dir1/file3 and restore them from revision 3
os.Remove(testDir + "/repository1/file2")
os.Remove(testDir + "/repository1/dir1/file3")
SetDuplicacyPreferencePath(testDir + "/repository1/.duplicacy")
backupManager.Restore(testDir + "/repository1", 3, /*inPlace=*/true, /*quickMode=*/false, threads, /*overwrite=*/true,
/*deleteMode=*/false, /*showStatistics=*/false, /*patterns=*/[]string{"+file2", "+dir1/file3", "-*"})
backupManager.Restore(testDir+"/repository1", 3 /*inPlace=*/, true /*quickMode=*/, false, threads /*overwrite=*/, true,
/*deleteMode=*/ false /*showStatistics=*/, false /*patterns=*/, []string{"+file2", "+dir1/file3", "-*"})
for _, f := range []string{ "file1", "file2", "dir1/file3" } {
for _, f := range []string{"file1", "file2", "dir1/file3"} {
hash1 := getFileHash(testDir + "/repository1/" + f)
hash2 := getFileHash(testDir + "/repository2/" + f)
if hash1 != hash2 {

52
src/duplicacy_chunk.go
View File

@@ -5,28 +5,27 @@
package duplicacy
import (
"io"
"fmt"
"hash"
"bytes"
"runtime"
"crypto/cipher"
"compress/zlib"
"crypto/aes"
"crypto/cipher"
"crypto/rand"
"encoding/hex"
"compress/zlib"
"fmt"
"hash"
"io"
"runtime"
"github.com/bkaradzic/go-lz4"
)
// A chunk needs to acquire a new buffer and return the old one for every encrypt/decrypt operation, therefore
// we maintain a pool of previously used buffers.
var chunkBufferPool chan *bytes.Buffer = make(chan *bytes.Buffer, runtime.NumCPU() * 16)
var chunkBufferPool chan *bytes.Buffer = make(chan *bytes.Buffer, runtime.NumCPU()*16)
func AllocateChunkBuffer() (buffer *bytes.Buffer) {
select {
case buffer = <- chunkBufferPool:
case buffer = <-chunkBufferPool:
default:
buffer = new(bytes.Buffer)
}
@@ -76,9 +75,9 @@ func CreateChunk(config *Config, bufferNeeded bool) *Chunk {
}
}
return &Chunk {
buffer : buffer,
config : config,
return &Chunk{
buffer: buffer,
config: config,
}
}
@@ -92,7 +91,7 @@ func (chunk *Chunk) GetLength() int {
}
// GetBytes returns data available in this chunk
func (chunk *Chunk) GetBytes() [] byte {
func (chunk *Chunk) GetBytes() []byte {
return chunk.buffer.Bytes()
}
@@ -114,7 +113,7 @@ func (chunk *Chunk) Reset(hashNeeded bool) {
}
// Write implements the Writer interface.
func (chunk *Chunk) Write(p []byte) (int, error){
func (chunk *Chunk) Write(p []byte) (int, error) {
// buffer may be nil, when the chunk is used for computing the hash only.
if chunk.buffer == nil {
@@ -132,7 +131,7 @@ func (chunk *Chunk) Write(p []byte) (int, error){
// GetHash returns the chunk hash.
func (chunk *Chunk) GetHash() string {
if (len(chunk.hash) == 0) {
if len(chunk.hash) == 0 {
chunk.hash = chunk.hasher.Sum(nil)
}
@@ -179,7 +178,7 @@ func (chunk *Chunk) Encrypt(encryptionKey []byte, derivationKey string) (err err
encryptedBuffer.Reset()
defer func() {
ReleaseChunkBuffer(encryptedBuffer)
} ()
}()
if len(encryptionKey) > 0 {
@@ -229,7 +228,7 @@ func (chunk *Chunk) Encrypt(encryptionKey []byte, derivationKey string) (err err
if availableLength < maximumLength {
encryptedBuffer.Grow(maximumLength - availableLength)
}
written, err := lz4.Encode(encryptedBuffer.Bytes()[offset + 4:], chunk.buffer.Bytes())
written, err := lz4.Encode(encryptedBuffer.Bytes()[offset+4:], chunk.buffer.Bytes())
if err != nil {
return fmt.Errorf("LZ4 compression error: %v", err)
}
@@ -258,7 +257,7 @@ func (chunk *Chunk) Encrypt(encryptionKey []byte, derivationKey string) (err err
// The encrypted data will be appended to the duplicacy header and the once.
encryptedBytes := gcm.Seal(encryptedBuffer.Bytes()[:offset], nonce,
encryptedBuffer.Bytes()[offset: offset + dataLength + paddingLength], nil)
encryptedBuffer.Bytes()[offset:offset+dataLength+paddingLength], nil)
encryptedBuffer.Truncate(len(encryptedBytes))
@@ -278,7 +277,7 @@ func (chunk *Chunk) Decrypt(encryptionKey []byte, derivationKey string) (err err
encryptedBuffer.Reset()
defer func() {
ReleaseChunkBuffer(encryptedBuffer)
} ()
}()
chunk.buffer, encryptedBuffer = encryptedBuffer, chunk.buffer
@@ -309,15 +308,15 @@ func (chunk *Chunk) Decrypt(encryptionKey []byte, derivationKey string) (err err
return fmt.Errorf("No enough encrypted data (%d bytes) provided", len(encryptedBuffer.Bytes()))
}
if string(encryptedBuffer.Bytes()[:headerLength - 1]) != ENCRYPTION_HEADER[:headerLength - 1] {
if string(encryptedBuffer.Bytes()[:headerLength-1]) != ENCRYPTION_HEADER[:headerLength-1] {
return fmt.Errorf("The storage doesn't seem to be encrypted")
}
if encryptedBuffer.Bytes()[headerLength - 1] != 0 {
return fmt.Errorf("Unsupported encryption version %d", encryptedBuffer.Bytes()[headerLength - 1])
if encryptedBuffer.Bytes()[headerLength-1] != 0 {
return fmt.Errorf("Unsupported encryption version %d", encryptedBuffer.Bytes()[headerLength-1])
}
nonce := encryptedBuffer.Bytes()[headerLength: offset]
nonce := encryptedBuffer.Bytes()[headerLength:offset]
decryptedBytes, err := gcm.Open(encryptedBuffer.Bytes()[:offset], nonce,
encryptedBuffer.Bytes()[offset:], nil)
@@ -326,7 +325,7 @@ func (chunk *Chunk) Decrypt(encryptionKey []byte, derivationKey string) (err err
return err
}
paddingLength := int(decryptedBytes[len(decryptedBytes) - 1])
paddingLength := int(decryptedBytes[len(decryptedBytes)-1])
if paddingLength == 0 {
paddingLength = 256
}
@@ -335,10 +334,10 @@ func (chunk *Chunk) Decrypt(encryptionKey []byte, derivationKey string) (err err
}
for i := 0; i < paddingLength; i++ {
padding := decryptedBytes[len(decryptedBytes) - 1 - i]
padding := decryptedBytes[len(decryptedBytes)-1-i]
if padding != byte(paddingLength) {
return fmt.Errorf("Incorrect padding of length %d: %x", paddingLength,
decryptedBytes[len(decryptedBytes) - paddingLength:])
decryptedBytes[len(decryptedBytes)-paddingLength:])
}
}
@@ -379,4 +378,3 @@ func (chunk *Chunk) Decrypt(encryptionKey []byte, derivationKey string) (err err
return nil
}

3
src/duplicacy_chunk_test.go
View File

@@ -5,10 +5,10 @@
package duplicacy
import (
"testing"
"bytes"
crypto_rand "crypto/rand"
"math/rand"
"testing"
)
func TestChunk(t *testing.T) {
@@ -67,7 +67,6 @@ func TestChunk(t *testing.T) {
t.Errorf("Original data:\n%x\nDecrypted data:\n%x\n", plainData, decryptedData)
}
}
}

39
src/duplicacy_chunkdownloader.go
View File

@@ -36,7 +36,7 @@ type ChunkDownloader struct {
showStatistics bool // Show a stats log for each chunk if true
threads int // Number of threads
taskList [] ChunkDownloadTask // The list of chunks to be downloaded
taskList []ChunkDownloadTask // The list of chunks to be downloaded
completedTasks map[int]bool // Store downloaded chunks
lastChunkIndex int // a monotonically increasing number indicating the last chunk to be downloaded
@@ -53,7 +53,7 @@ type ChunkDownloader struct {
}
func CreateChunkDownloader(config *Config, storage Storage, snapshotCache *FileStorage, showStatistics bool, threads int) *ChunkDownloader {
downloader := &ChunkDownloader {
downloader := &ChunkDownloader{
config: config,
storage: storage,
snapshotCache: snapshotCache,
@@ -77,20 +77,20 @@ func CreateChunkDownloader(config *Config, storage Storage, snapshotCache *FileS
defer CatchLogException()
for {
select {
case task := <- downloader.taskQueue:
case task := <-downloader.taskQueue:
downloader.Download(threadIndex, task)
case <- downloader.stopChannel:
case <-downloader.stopChannel:
return
}
}
} (i)
}(i)
}
return downloader
}
// AddFiles adds chunks needed by the specified files to the download list.
func (downloader *ChunkDownloader) AddFiles(snapshot *Snapshot, files [] *Entry) {
func (downloader *ChunkDownloader) AddFiles(snapshot *Snapshot, files []*Entry) {
downloader.taskList = nil
lastChunkIndex := -1
@@ -102,7 +102,7 @@ func (downloader *ChunkDownloader) AddFiles(snapshot *Snapshot, files [] *Entry)
}
for i := file.StartChunk; i <= file.EndChunk; i++ {
if lastChunkIndex != i {
task := ChunkDownloadTask {
task := ChunkDownloadTask{
chunkIndex: len(downloader.taskList),
chunkHash: snapshot.ChunkHashes[i],
chunkLength: snapshot.ChunkLengths[i],
@@ -111,13 +111,13 @@ func (downloader *ChunkDownloader) AddFiles(snapshot *Snapshot, files [] *Entry)
downloader.taskList = append(downloader.taskList, task)
downloader.totalChunkSize += int64(snapshot.ChunkLengths[i])
} else {
downloader.taskList[len(downloader.taskList) - 1].needed = true
downloader.taskList[len(downloader.taskList)-1].needed = true
}
lastChunkIndex = i
}
file.StartChunk = len(downloader.taskList) - (file.EndChunk - file.StartChunk) - 1
file.EndChunk = len(downloader.taskList) - 1
if file.EndChunk - file.StartChunk > maximumChunks {
if file.EndChunk-file.StartChunk > maximumChunks {
maximumChunks = file.EndChunk - file.StartChunk
}
}
@@ -125,7 +125,7 @@ func (downloader *ChunkDownloader) AddFiles(snapshot *Snapshot, files [] *Entry)
// AddChunk adds a single chunk the download list.
func (downloader *ChunkDownloader) AddChunk(chunkHash string) int {
task := ChunkDownloadTask {
task := ChunkDownloadTask{
chunkIndex: len(downloader.taskList),
chunkHash: chunkHash,
chunkLength: 0,
@@ -137,7 +137,7 @@ func (downloader *ChunkDownloader) AddChunk(chunkHash string) int {
downloader.taskQueue <- task
downloader.numberOfDownloadingChunks++
downloader.numberOfActiveChunks++
downloader.taskList[len(downloader.taskList) - 1].isDownloading = true
downloader.taskList[len(downloader.taskList)-1].isDownloading = true
}
return len(downloader.taskList) - 1
}
@@ -163,7 +163,7 @@ func (downloader *ChunkDownloader) Prefetch(file *Entry) {
downloader.numberOfDownloadingChunks++
downloader.numberOfActiveChunks++
}
} else{
} else {
LOG_DEBUG("DOWNLOAD_PREFETCH", "%s chunk %s is not needed", file.Path,
downloader.config.GetChunkIDFromHash(task.chunkHash))
}
@@ -226,7 +226,7 @@ func (downloader *ChunkDownloader) WaitForChunk(chunkIndex int) (chunk *Chunk) {
// Now wait until the chunk to be downloaded appears in the completed tasks
for _, found := downloader.completedTasks[chunkIndex]; !found; _, found = downloader.completedTasks[chunkIndex] {
completion := <- downloader.completionChannel
completion := <-downloader.completionChannel
downloader.completedTasks[completion.chunkIndex] = true
downloader.taskList[completion.chunkIndex].chunk = completion.chunk
downloader.numberOfDownloadedChunks++
@@ -238,7 +238,7 @@ func (downloader *ChunkDownloader) WaitForChunk(chunkIndex int) (chunk *Chunk) {
// Stop terminates all downloading goroutines
func (downloader *ChunkDownloader) Stop() {
for downloader.numberOfDownloadingChunks > 0 {
completion := <- downloader.completionChannel
completion := <-downloader.completionChannel
downloader.completedTasks[completion.chunkIndex] = true
downloader.taskList[completion.chunkIndex].chunk = completion.chunk
downloader.numberOfDownloadedChunks++
@@ -286,7 +286,7 @@ func (downloader *ChunkDownloader) Download(threadIndex int, task ChunkDownloadT
} else {
LOG_DEBUG("CHUNK_CACHE", "Chunk %s has been loaded from the snapshot cache", chunkID)
downloader.completionChannel <- ChunkDownloadCompletion{ chunk: chunk, chunkIndex:task.chunkIndex }
downloader.completionChannel <- ChunkDownloadCompletion{chunk: chunk, chunkIndex: task.chunkIndex}
return false
}
}
@@ -330,7 +330,6 @@ func (downloader *ChunkDownloader) Download(threadIndex int, task ChunkDownloadT
return false
}
err = chunk.Decrypt(downloader.config.ChunkKey, task.chunkHash)
if err != nil {
LOG_ERROR("UPLOAD_CHUNK", "Failed to decrypt the chunk %s: %v", chunkID, err)
@@ -362,16 +361,16 @@ func (downloader *ChunkDownloader) Download(threadIndex int, task ChunkDownloadT
speed := downloadedChunkSize / (now - downloader.startTime)
remainingTime := int64(0)
if speed > 0 {
remainingTime = (downloader.totalChunkSize - downloadedChunkSize) / speed + 1
remainingTime = (downloader.totalChunkSize-downloadedChunkSize)/speed + 1
}
percentage := float32(downloadedChunkSize * 1000 / downloader.totalChunkSize)
LOG_INFO("DOWNLOAD_PROGRESS", "Downloaded chunk %d size %d, %sB/s %s %.1f%%",
task.chunkIndex + 1, chunk.GetLength(),
PrettySize(speed), PrettyTime(remainingTime), percentage / 10)
task.chunkIndex+1, chunk.GetLength(),
PrettySize(speed), PrettyTime(remainingTime), percentage/10)
} else {
LOG_DEBUG("CHUNK_DOWNLOAD", "Chunk %s has been downloaded", chunkID)
}
downloader.completionChannel <- ChunkDownloadCompletion{ chunk: chunk, chunkIndex:task.chunkIndex }
downloader.completionChannel <- ChunkDownloadCompletion{chunk: chunk, chunkIndex: task.chunkIndex}
return true
}

45
src/duplicacy_chunkmaker.go
View File

@@ -5,10 +5,10 @@
package duplicacy
import (
"io"
"crypto/sha256"
"encoding/hex"
"encoding/binary"
"encoding/hex"
"io"
)
// ChunkMaker breaks data into chunks using buzhash. To save memory, the chunk maker only use a circular buffer
@@ -35,7 +35,7 @@ type ChunkMaker struct {
// buzhash.
func CreateChunkMaker(config *Config, hashOnly bool) *ChunkMaker {
size := 1
for size * 2 <= config.AverageChunkSize {
for size*2 <= config.AverageChunkSize {
size *= 2
}
@@ -44,7 +44,7 @@ func CreateChunkMaker(config *Config, hashOnly bool) *ChunkMaker {
return nil
}
maker := &ChunkMaker {
maker := &ChunkMaker{
hashMask: uint64(config.AverageChunkSize - 1),
maximumChunkSize: config.MaximumChunkSize,
minimumChunkSize: config.MinimumChunkSize,
@@ -61,12 +61,12 @@ func CreateChunkMaker(config *Config, hashOnly bool) *ChunkMaker {
for i := 0; i < 64; i++ {
for j := 0; j < 4; j++ {
maker.randomTable[4 * i + j] = binary.LittleEndian.Uint64(randomData[8 * j : 8 * j + 8])
maker.randomTable[4*i+j] = binary.LittleEndian.Uint64(randomData[8*j : 8*j+8])
}
randomData = sha256.Sum256(randomData[:])
}
maker.buffer = make([]byte, 2 * config.MinimumChunkSize)
maker.buffer = make([]byte, 2*config.MinimumChunkSize)
return maker
}
@@ -79,7 +79,7 @@ func rotateLeftByOne(value uint64) uint64 {
return (value << 1) | (value >> 63)
}
func (maker *ChunkMaker) buzhashSum(sum uint64, data [] byte) uint64 {
func (maker *ChunkMaker) buzhashSum(sum uint64, data []byte) uint64 {
for i := 0; i < len(data); i++ {
sum = rotateLeftByOne(sum) ^ maker.randomTable[data[i]]
}
@@ -94,7 +94,7 @@ func (maker *ChunkMaker) buzhashUpdate(sum uint64, out byte, in byte, length int
// 'nextReader' returns false, it will process remaining data in the buffer and then quit. When a chunk is identified,
// it will call 'endOfChunk' to return the chunk size and a boolean flag indicating if it is the last chunk.
func (maker *ChunkMaker) ForEachChunk(reader io.Reader, endOfChunk func(chunk *Chunk, final bool),
nextReader func(size int64, hash string)(io.Reader, bool)) {
nextReader func(size int64, hash string) (io.Reader, bool)) {
maker.bufferStart = 0
maker.bufferSize = 0
@@ -121,13 +121,13 @@ func (maker *ChunkMaker) ForEachChunk(reader io.Reader, endOfChunk func(chunk *C
// Move data from the buffer to the chunk.
fill := func(count int) {
if maker.bufferStart + count < maker.bufferCapacity {
chunk.Write(maker.buffer[maker.bufferStart : maker.bufferStart + count])
if maker.bufferStart+count < maker.bufferCapacity {
chunk.Write(maker.buffer[maker.bufferStart : maker.bufferStart+count])
maker.bufferStart += count
maker.bufferSize -= count
} else {
chunk.Write(maker.buffer[maker.bufferStart :])
chunk.Write(maker.buffer[: count - (maker.bufferCapacity - maker.bufferStart)])
chunk.Write(maker.buffer[maker.bufferStart:])
chunk.Write(maker.buffer[:count-(maker.bufferCapacity-maker.bufferStart)])
maker.bufferStart = count - (maker.bufferCapacity - maker.bufferStart)
maker.bufferSize -= count
}
@@ -148,7 +148,7 @@ func (maker *ChunkMaker) ForEachChunk(reader io.Reader, endOfChunk func(chunk *C
for {
maker.bufferStart = 0
for maker.bufferStart < maker.minimumChunkSize && !isEOF {
count, err := reader.Read(maker.buffer[maker.bufferStart : maker.minimumChunkSize])
count, err := reader.Read(maker.buffer[maker.bufferStart:maker.minimumChunkSize])
if err != nil {
if err != io.EOF {
@@ -197,7 +197,7 @@ func (maker *ChunkMaker) ForEachChunk(reader io.Reader, endOfChunk func(chunk *C
count = maker.bufferStart - start
}
count, err = reader.Read(maker.buffer[start : start + count])
count, err = reader.Read(maker.buffer[start : start+count])
if err != nil && err != io.EOF {
LOG_ERROR("CHUNK_MAKER", "Failed to read %d bytes: %s", count, err.Error())
@@ -205,7 +205,7 @@ func (maker *ChunkMaker) ForEachChunk(reader io.Reader, endOfChunk func(chunk *C
}
maker.bufferSize += count
fileHasher.Write(maker.buffer[start : start + count])
fileHasher.Write(maker.buffer[start : start+count])
fileSize += int64(count)
// if EOF is seen, try to switch to next file and continue
@@ -229,18 +229,17 @@ func (maker *ChunkMaker) ForEachChunk(reader io.Reader, endOfChunk func(chunk *C
return
}
// Minimum chunk size has been reached. Calculate the buzhash for the minimum size chunk.
if (!minimumReached) {
if !minimumReached {
bytes := maker.minimumChunkSize
if maker.bufferStart + bytes < maker.bufferCapacity {
hashSum = maker.buzhashSum(0, maker.buffer[maker.bufferStart : maker.bufferStart + bytes])
if maker.bufferStart+bytes < maker.bufferCapacity {
hashSum = maker.buzhashSum(0, maker.buffer[maker.bufferStart:maker.bufferStart+bytes])
} else {
hashSum = maker.buzhashSum(0, maker.buffer[maker.bufferStart :])
hashSum = maker.buzhashSum(0, maker.buffer[maker.bufferStart:])
hashSum = maker.buzhashSum(hashSum,
maker.buffer[: bytes - (maker.bufferCapacity - maker.bufferStart)])
maker.buffer[:bytes-(maker.bufferCapacity-maker.bufferStart)])
}
if (hashSum & maker.hashMask) == 0 {
@@ -258,7 +257,7 @@ func (maker *ChunkMaker) ForEachChunk(reader io.Reader, endOfChunk func(chunk *C
bytes := maker.bufferSize - maker.minimumChunkSize
isEOC := false
maxSize := maker.maximumChunkSize - chunk.GetLength()
for i := 0; i < maker.bufferSize - maker.minimumChunkSize; i++ {
for i := 0; i < maker.bufferSize-maker.minimumChunkSize; i++ {
out := maker.bufferStart + i
if out >= maker.bufferCapacity {
out -= maker.bufferCapacity
@@ -269,7 +268,7 @@ func (maker *ChunkMaker) ForEachChunk(reader io.Reader, endOfChunk func(chunk *C
}
hashSum = maker.buzhashUpdate(hashSum, maker.buffer[out], maker.buffer[in], maker.minimumChunkSize)
if (hashSum & maker.hashMask) == 0 || i == maxSize - maker.minimumChunkSize - 1 {
if (hashSum&maker.hashMask) == 0 || i == maxSize-maker.minimumChunkSize-1 {
// A chunk is completed.
bytes = i + 1 + maker.minimumChunkSize
isEOC = true

31
src/duplicacy_chunkmaker_test.go
View File

@@ -5,12 +5,12 @@
package duplicacy
import (
"testing"
"bytes"
crypto_rand "crypto/rand"
"math/rand"
"io"
"math/rand"
"sort"
"testing"
)
func splitIntoChunks(content []byte, n, averageChunkSize, maxChunkSize, minChunkSize,
@@ -29,14 +29,14 @@ func splitIntoChunks(content []byte, n, averageChunkSize, maxChunkSize, minChunk
maker := CreateChunkMaker(config, false)
var chunks [] string
var chunks []string
totalChunkSize := 0
totalFileSize := int64(0)
//LOG_INFO("CHUNK_SPLIT", "bufferCapacity: %d", bufferCapacity)
buffers := make([] *bytes.Buffer, n)
sizes := make([] int, n)
buffers := make([]*bytes.Buffer, n)
sizes := make([]int, n)
sizes[0] = 0
for i := 1; i < n; i++ {
same := true
@@ -54,20 +54,20 @@ func splitIntoChunks(content []byte, n, averageChunkSize, maxChunkSize, minChunk
sort.Sort(sort.IntSlice(sizes))
for i := 0; i < n - 1; i++ {
buffers[i] = bytes.NewBuffer(content[sizes[i] : sizes[i + 1]])
for i := 0; i < n-1; i++ {
buffers[i] = bytes.NewBuffer(content[sizes[i]:sizes[i+1]])
}
buffers[n - 1] = bytes.NewBuffer(content[sizes[n - 1]:])
buffers[n-1] = bytes.NewBuffer(content[sizes[n-1]:])
i := 0
maker.ForEachChunk(buffers[0],
func (chunk *Chunk, final bool) {
func(chunk *Chunk, final bool) {
//LOG_INFO("CHUNK_SPLIT", "i: %d, chunk: %s, size: %d", i, chunk.GetHash(), size)
chunks = append(chunks, chunk.GetHash())
totalChunkSize += chunk.GetLength()
},
func (size int64, hash string) (io.Reader, bool) {
func(size int64, hash string) (io.Reader, bool) {
totalFileSize += size
i++
if i >= len(buffers) {
@@ -76,7 +76,7 @@ func splitIntoChunks(content []byte, n, averageChunkSize, maxChunkSize, minChunk
return buffers[i], true
})
if (totalFileSize != int64(totalChunkSize)) {
if totalFileSize != int64(totalChunkSize) {
LOG_ERROR("CHUNK_SPLIT", "total chunk size: %d, total file size: %d", totalChunkSize, totalFileSize)
}
return chunks, totalChunkSize
@@ -84,9 +84,8 @@ func splitIntoChunks(content []byte, n, averageChunkSize, maxChunkSize, minChunk
func TestChunkMaker(t *testing.T) {
//sizes := [...] int { 64 }
sizes := [...] int { 64, 256, 1024, 1024 * 10 }
sizes := [...]int{64, 256, 1024, 1024 * 10}
for _, size := range sizes {
@@ -99,15 +98,15 @@ func TestChunkMaker(t *testing.T) {
chunkArray1, totalSize1 := splitIntoChunks(content, 10, 32, 64, 16, 32)
capacities := [...]int { 32, 33, 34, 61, 62, 63, 64, 65, 66, 126, 127, 128, 129, 130,
capacities := [...]int{32, 33, 34, 61, 62, 63, 64, 65, 66, 126, 127, 128, 129, 130,
255, 256, 257, 511, 512, 513, 1023, 1024, 1025,
32, 48, 64, 128, 256, 512, 1024, 2048, }
32, 48, 64, 128, 256, 512, 1024, 2048}
//capacities := [...]int { 32 }
for _, capacity := range capacities {
for _, n := range [...]int { 6, 7, 8, 9, 10 } {
for _, n := range [...]int{6, 7, 8, 9, 10} {
chunkArray2, totalSize2 := splitIntoChunks(content, n, 32, 64, 16, capacity)
if totalSize1 != totalSize2 {

12
src/duplicacy_chunkuploader.go
View File

@@ -11,7 +11,7 @@ import (
// ChunkUploadTask represents a chunk to be uploaded.
type ChunkUploadTask struct {
chunk * Chunk
chunk *Chunk
chunkIndex int
}
@@ -36,7 +36,7 @@ type ChunkUploader struct {
// CreateChunkUploader creates a chunk uploader.
func CreateChunkUploader(config *Config, storage Storage, snapshotCache *FileStorage, threads int,
completionFunc func(chunk *Chunk, chunkIndex int, skipped bool, chunkSize int, uploadSize int)) *ChunkUploader {
uploader := &ChunkUploader {
uploader := &ChunkUploader{
config: config,
storage: storage,
snapshotCache: snapshotCache,
@@ -56,20 +56,20 @@ func (uploader *ChunkUploader) Start() {
defer CatchLogException()
for {
select {
case task := <- uploader.taskQueue:
case task := <-uploader.taskQueue:
uploader.Upload(threadIndex, task)
case <- uploader.stopChannel:
case <-uploader.stopChannel:
return
}
}
} (i)
}(i)
}
}
// StartChunk sends a chunk to be uploaded to a waiting uploading goroutine. It may block if all uploading goroutines are busy.
func (uploader *ChunkUploader) StartChunk(chunk *Chunk, chunkIndex int) {
atomic.AddInt32(&uploader.numberOfUploadingTasks, 1)
uploader.taskQueue <- ChunkUploadTask {
uploader.taskQueue <- ChunkUploadTask{
chunk: chunk,
chunkIndex: chunkIndex,
}

16
src/duplicacy_chunkuploader_test.go
View File

@@ -6,10 +6,10 @@ package duplicacy
import (
"os"
"time"
"path"
"testing"
"runtime/debug"
"testing"
"time"
crypto_rand "crypto/rand"
"math/rand"
@@ -32,7 +32,7 @@ func TestUploaderAndDownloader(t *testing.T) {
debug.PrintStack()
}
}
} ()
}()
testDir := path.Join(os.TempDir(), "duplicacy_test", "storage_test")
os.RemoveAll(testDir)
@@ -48,7 +48,7 @@ func TestUploaderAndDownloader(t *testing.T) {
storage.EnableTestMode()
storage.SetRateLimits(testRateLimit, testRateLimit)
for _, dir := range []string { "chunks", "snapshots" } {
for _, dir := range []string{"chunks", "snapshots"} {
err = storage.CreateDirectory(0, dir)
if err != nil {
t.Errorf("Failed to create directory %s: %v", dir, err)
@@ -56,7 +56,6 @@ func TestUploaderAndDownloader(t *testing.T) {
}
}
numberOfChunks := 100
maxChunkSize := 64 * 1024
@@ -68,11 +67,11 @@ func TestUploaderAndDownloader(t *testing.T) {
config := CreateConfig()
config.MinimumChunkSize = 100
config.chunkPool = make(chan *Chunk, numberOfChunks * 2)
config.chunkPool = make(chan *Chunk, numberOfChunks*2)
totalFileSize := 0
for i := 0; i < numberOfChunks; i++ {
content := make([]byte, rand.Int() % maxChunkSize + 1)
content := make([]byte, rand.Int()%maxChunkSize+1)
_, err = crypto_rand.Read(content)
if err != nil {
t.Errorf("Error generating random content: %v", err)
@@ -102,7 +101,6 @@ func TestUploaderAndDownloader(t *testing.T) {
chunkUploader.Stop()
chunkDownloader := CreateChunkDownloader(config, storage, nil, true, testThreads)
chunkDownloader.totalChunkSize = int64(totalFileSize)
@@ -120,7 +118,7 @@ func TestUploaderAndDownloader(t *testing.T) {
chunkDownloader.Stop()
for _, file := range listChunks(storage) {
err = storage.DeleteFile(0, "chunks/" + file)
err = storage.DeleteFile(0, "chunks/"+file)
if err != nil {
t.Errorf("Failed to delete the file %s: %v", file, err)
return

47
src/duplicacy_config.go
View File

@@ -5,18 +5,18 @@
package duplicacy
import (
"encoding/json"
"bytes"
"os"
"crypto/hmac"
"crypto/rand"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"hash"
"os"
"runtime"
"runtime/debug"
"sync/atomic"
"crypto/rand"
"crypto/hmac"
"crypto/sha256"
"encoding/hex"
blake2 "github.com/minio/blake2b-simd"
)
@@ -58,7 +58,7 @@ type Config struct {
dryRun bool
}
// Create an alias to avoid recursive calls on Config.MarshalJSON
// Create an alias to avoid recursive calls on Config.MarshalJSON
type aliasedConfig Config
type jsonableConfig struct {
@@ -70,9 +70,9 @@ type jsonableConfig struct {
FileKey string `json:"file-key"`
}
func (config *Config) MarshalJSON() ([] byte, error) {
func (config *Config) MarshalJSON() ([]byte, error) {
return json.Marshal(&jsonableConfig {
return json.Marshal(&jsonableConfig{
aliasedConfig: (*aliasedConfig)(config),
ChunkSeed: hex.EncodeToString(config.ChunkSeed),
HashKey: hex.EncodeToString(config.HashKey),
@@ -84,7 +84,7 @@ func (config *Config) MarshalJSON() ([] byte, error) {
func (config *Config) UnmarshalJSON(description []byte) (err error) {
aliased := &jsonableConfig {
aliased := &jsonableConfig{
aliasedConfig: (*aliasedConfig)(config),
}
@@ -133,7 +133,7 @@ func (config *Config) Print() {
func CreateConfigFromParameters(compressionLevel int, averageChunkSize int, maximumChunkSize int, mininumChunkSize int,
isEncrypted bool, copyFrom *Config) (config *Config) {
config = &Config {
config = &Config{
CompressionLevel: compressionLevel,
AverageChunkSize: averageChunkSize,
MaximumChunkSize: maximumChunkSize,
@@ -142,7 +142,7 @@ func CreateConfigFromParameters(compressionLevel int, averageChunkSize int, maxi
if isEncrypted {
// Randomly generate keys
keys := make([]byte, 32 * 5)
keys := make([]byte, 32*5)
_, err := rand.Read(keys)
if err != nil {
LOG_ERROR("CONFIG_KEY", "Failed to generate random keys: %v", err)
@@ -171,26 +171,26 @@ func CreateConfigFromParameters(compressionLevel int, averageChunkSize int, maxi
config.HashKey = copyFrom.HashKey
}
config.chunkPool = make(chan *Chunk, runtime.NumCPU() * 16)
config.chunkPool = make(chan *Chunk, runtime.NumCPU()*16)
return config
}
func CreateConfig() (config *Config) {
return &Config {
return &Config{
HashKey: DEFAULT_KEY,
IDKey: DEFAULT_KEY,
CompressionLevel: DEFAULT_COMPRESSION_LEVEL,
chunkPool: make(chan *Chunk, runtime.NumCPU() * 16),
chunkPool: make(chan *Chunk, runtime.NumCPU()*16),
}
}
func (config *Config) GetChunk() (chunk *Chunk) {
select {
case chunk = <- config.chunkPool :
case chunk = <-config.chunkPool:
default:
numberOfChunks := atomic.AddInt32(&config.numberOfChunks, 1)
if numberOfChunks >= int32(runtime.NumCPU() * 16) {
if numberOfChunks >= int32(runtime.NumCPU()*16) {
LOG_WARN("CONFIG_CHUNK", "%d chunks have been allocated", numberOfChunks)
if _, found := os.LookupEnv("DUPLICACY_CHUNK_DEBUG"); found {
debug.PrintStack()
@@ -201,7 +201,7 @@ func (config *Config) GetChunk() (chunk *Chunk) {
return chunk
}
func (config *Config) PutChunk(chunk *Chunk){
func (config *Config) PutChunk(chunk *Chunk) {
if chunk == nil {
return
@@ -216,7 +216,7 @@ func (config *Config) PutChunk(chunk *Chunk){
func (config *Config) NewKeyedHasher(key []byte) hash.Hash {
if config.CompressionLevel == DEFAULT_COMPRESSION_LEVEL {
hasher, err := blake2.New(&blake2.Config{ Size: 32, Key:key })
hasher, err := blake2.New(&blake2.Config{Size: 32, Key: key})
if err != nil {
LOG_ERROR("HASH_KEY", "Invalid hash key: %x", key)
}
@@ -259,9 +259,9 @@ func (hasher *DummyHasher) BlockSize() int {
func (config *Config) NewFileHasher() hash.Hash {
if SkipFileHash {
return &DummyHasher {}
return &DummyHasher{}
} else if config.CompressionLevel == DEFAULT_COMPRESSION_LEVEL {
hasher, _ := blake2.New(&blake2.Config{ Size: 32 })
hasher, _ := blake2.New(&blake2.Config{Size: 32})
return hasher
} else {
return sha256.New()
@@ -344,7 +344,7 @@ func DownloadConfig(storage Storage, password string) (config *Config, isEncrypt
}
func UploadConfig(storage Storage, config *Config, password string) (bool) {
func UploadConfig(storage Storage, config *Config, password string) bool {
// This is the key to encrypt the config file.
var masterKey []byte
@@ -390,7 +390,7 @@ func UploadConfig(storage Storage, config *Config, password string) (bool) {
config.Print()
}
for _, subDir := range []string {"chunks", "snapshots"} {
for _, subDir := range []string{"chunks", "snapshots"} {
err = storage.CreateDirectory(0, subDir)
if err != nil {
LOG_ERROR("CONFIG_MKDIR", "Failed to create storage subdirectory: %v", err)
@@ -417,7 +417,6 @@ func ConfigStorage(storage Storage, compressionLevel int, averageChunkSize int,
return false
}
config := CreateConfigFromParameters(compressionLevel, averageChunkSize, maximumChunkSize, minimumChunkSize, len(password) > 0,
copyFrom)
if config == nil {

57
src/duplicacy_dropboxstorage.go
View File

@@ -8,6 +8,7 @@ import (
"fmt"
"path"
"strings"
"github.com/gilbertchen/go-dropbox"
)
@@ -31,11 +32,11 @@ func CreateDropboxStorage(accessToken string, storageDir string, threads int) (s
storageDir = "/" + storageDir
}
if len(storageDir) > 1 && storageDir[len(storageDir) - 1] == '/' {
storageDir = storageDir[:len(storageDir) - 1]
if len(storageDir) > 1 && storageDir[len(storageDir)-1] == '/' {
storageDir = storageDir[:len(storageDir)-1]
}
storage = &DropboxStorage {
storage = &DropboxStorage{
clients: clients,
storageDir: storageDir,
}
@@ -55,13 +56,13 @@ func (storage *DropboxStorage) ListFiles(threadIndex int, dir string) (files []s
dir = "/" + dir
}
if len(dir) > 1 && dir[len(dir) - 1] == '/' {
dir = dir[:len(dir) - 1]
if len(dir) > 1 && dir[len(dir)-1] == '/' {
dir = dir[:len(dir)-1]
}
input := &dropbox.ListFolderInput {
Path : storage.storageDir + dir,
Recursive : false,
input := &dropbox.ListFolderInput{
Path: storage.storageDir + dir,
Recursive: false,
IncludeMediaInfo: false,
IncludeDeleted: false,
}
@@ -85,7 +86,7 @@ func (storage *DropboxStorage) ListFiles(threadIndex int, dir string) (files []s
if output.HasMore {
output, err = storage.clients[threadIndex].ListFolderContinue(
&dropbox.ListFolderContinueInput { Cursor: output.Cursor, })
&dropbox.ListFolderContinueInput{Cursor: output.Cursor})
} else {
break
@@ -102,7 +103,7 @@ func (storage *DropboxStorage) DeleteFile(threadIndex int, filePath string) (err
filePath = "/" + filePath
}
input := &dropbox.DeleteInput {
input := &dropbox.DeleteInput{
Path: storage.storageDir + filePath,
}
_, err = storage.clients[threadIndex].Delete(input)
@@ -123,7 +124,7 @@ func (storage *DropboxStorage) MoveFile(threadIndex int, from string, to string)
if to != "" && to[0] != '/' {
to = "/" + to
}
input := &dropbox.MoveInput {
input := &dropbox.MoveInput{
FromPath: storage.storageDir + from,
ToPath: storage.storageDir + to,
}
@@ -137,12 +138,12 @@ func (storage *DropboxStorage) CreateDirectory(threadIndex int, dir string) (err
dir = "/" + dir
}
if len(dir) > 1 && dir[len(dir) - 1] == '/' {
dir = dir[:len(dir) - 1]
if len(dir) > 1 && dir[len(dir)-1] == '/' {
dir = dir[:len(dir)-1]
}
input := &dropbox.CreateFolderInput {
Path : storage.storageDir + dir,
input := &dropbox.CreateFolderInput{
Path: storage.storageDir + dir,
}
_, err = storage.clients[threadIndex].CreateFolder(input)
@@ -161,7 +162,7 @@ func (storage *DropboxStorage) GetFileInfo(threadIndex int, filePath string) (ex
filePath = "/" + filePath
}
input := &dropbox.GetMetadataInput {
input := &dropbox.GetMetadataInput{
Path: storage.storageDir + filePath,
IncludeMediaInfo: false,
}
@@ -191,9 +192,9 @@ func (storage *DropboxStorage) FindChunk(threadIndex int, chunkID string, isFoss
// The minimum level of directories to dive into before searching for the chunk file.
minimumLevel := 1
for level := 0; level * 2 < len(chunkID); level ++ {
for level := 0; level*2 < len(chunkID); level++ {
if level >= minimumLevel {
filePath = path.Join(dir, chunkID[2 * level:]) + suffix
filePath = path.Join(dir, chunkID[2*level:]) + suffix
var size int64
exist, _, size, err = storage.GetFileInfo(threadIndex, filePath)
if err != nil {
@@ -205,7 +206,7 @@ func (storage *DropboxStorage) FindChunk(threadIndex int, chunkID string, isFoss
}
// Find the subdirectory the chunk file may reside.
subDir := path.Join(dir, chunkID[2 * level: 2 * level + 2])
subDir := path.Join(dir, chunkID[2*level:2*level+2])
exist, _, _, err = storage.GetFileInfo(threadIndex, subDir)
if err != nil {
return "", false, 0, err
@@ -228,7 +229,7 @@ func (storage *DropboxStorage) FindChunk(threadIndex int, chunkID string, isFoss
}
// Teh chunk must be under this subdirectory but it doesn't exist.
return path.Join(dir, chunkID[2 * level:])[1:] + suffix, false, 0, nil
return path.Join(dir, chunkID[2*level:])[1:] + suffix, false, 0, nil
}
@@ -245,7 +246,7 @@ func (storage *DropboxStorage) DownloadFile(threadIndex int, filePath string, ch
filePath = "/" + filePath
}
input := &dropbox.DownloadInput {
input := &dropbox.DownloadInput{
Path: storage.storageDir + filePath,
}
@@ -256,7 +257,7 @@ func (storage *DropboxStorage) DownloadFile(threadIndex int, filePath string, ch
defer output.Body.Close()
_, err = RateLimitedCopy(chunk, output.Body, storage.DownloadRateLimit / len(storage.clients))
_, err = RateLimitedCopy(chunk, output.Body, storage.DownloadRateLimit/len(storage.clients))
return err
}
@@ -267,12 +268,12 @@ func (storage *DropboxStorage) UploadFile(threadIndex int, filePath string, cont
filePath = "/" + filePath
}
input := &dropbox.UploadInput {
input := &dropbox.UploadInput{
Path: storage.storageDir + filePath,
Mode: dropbox.WriteModeOverwrite,
AutoRename: false,
Mute: true,
Reader: CreateRateLimitedReader(content, storage.UploadRateLimit / len(storage.clients)),
Reader: CreateRateLimitedReader(content, storage.UploadRateLimit/len(storage.clients)),
}
_, err = storage.clients[threadIndex].Upload(input)
@@ -281,16 +282,16 @@ func (storage *DropboxStorage) UploadFile(threadIndex int, filePath string, cont
// If a local snapshot cache is needed for the storage to avoid downloading/uploading chunks too often when
// managing snapshots.
func (storage *DropboxStorage) IsCacheNeeded() (bool) { return true }
func (storage *DropboxStorage) IsCacheNeeded() bool { return true }
// If the 'MoveFile' method is implemented.
func (storage *DropboxStorage) IsMoveFileImplemented() (bool) { return true }
func (storage *DropboxStorage) IsMoveFileImplemented() bool { return true }
// If the storage can guarantee strong consistency.
func (storage *DropboxStorage) IsStrongConsistent() (bool) { return false }
func (storage *DropboxStorage) IsStrongConsistent() bool { return false }
// If the storage supports fast listing of files names.
func (storage *DropboxStorage) IsFastListing() (bool) { return false }
func (storage *DropboxStorage) IsFastListing() bool { return false }
// Enable the test mode.
func (storage *DropboxStorage) EnableTestMode() {}

93
src/duplicacy_entry.go
View File

@@ -4,22 +4,20 @@
package duplicacy
import (
"os"
"fmt"
"path/filepath"
"io/ioutil"
"sort"
"regexp"
"strconv"
"time"
"encoding/json"
"encoding/base64"
"strings"
"encoding/json"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"regexp"
"runtime"
"sort"
"strconv"
"strings"
"time"
)
// This is the hidden directory in the repository for storing various files.
var DUPLICACY_DIRECTORY = ".duplicacy"
var DUPLICACY_FILE = ".duplicacy"
@@ -50,18 +48,18 @@ type Entry struct {
// CreateEntry creates an entry from file properties.
func CreateEntry(path string, size int64, time int64, mode uint32) *Entry {
if len(path) > 0 && path[len(path) - 1] != '/' && (mode & uint32(os.ModeDir)) != 0 {
if len(path) > 0 && path[len(path)-1] != '/' && (mode&uint32(os.ModeDir)) != 0 {
path += "/"
}
return &Entry {
Path : path,
Size : size,
Time : time,
Mode : mode,
return &Entry{
Path: path,
Size: size,
Time: time,
Mode: mode,
UID : -1,
GID : -1,
UID: -1,
GID: -1,
}
}
@@ -72,15 +70,15 @@ func CreateEntryFromFileInfo(fileInfo os.FileInfo, directory string) *Entry {
mode := fileInfo.Mode()
if mode & os.ModeDir != 0 && mode & os.ModeSymlink != 0 {
if mode&os.ModeDir != 0 && mode&os.ModeSymlink != 0 {
mode ^= os.ModeDir
}
if path[len(path) - 1] != '/' && mode & os.ModeDir != 0 {
if path[len(path)-1] != '/' && mode&os.ModeDir != 0 {
path += "/"
}
entry := &Entry {
entry := &Entry{
Path: path,
Size: fileInfo.Size(),
Time: fileInfo.ModTime().Unix(),
@@ -95,14 +93,14 @@ func CreateEntryFromFileInfo(fileInfo os.FileInfo, directory string) *Entry {
// CreateEntryFromJSON creates an entry from a json description.
func (entry *Entry) UnmarshalJSON(description []byte) (err error) {
var object map[string]interface {}
var object map[string]interface{}
err = json.Unmarshal(description, &object)
if err != nil {
return err
}
var value interface {}
var value interface{}
var ok bool
if value, ok = object["name"]; ok {
@@ -171,7 +169,7 @@ func (entry *Entry) UnmarshalJSON(description []byte) (err error) {
}
if value, ok = object["attributes"]; ok {
if attributes, ok := value.(map[string]interface {}); !ok {
if attributes, ok := value.(map[string]interface{}); !ok {
return fmt.Errorf("Attributes are invalid for file '%s' in the snapshot", entry.Path)
} else {
entry.Attributes = make(map[string][]byte)
@@ -251,7 +249,7 @@ func (entry *Entry) convertToObject(encodeName bool) map[string]interface{} {
}
// MarshalJSON returns the json description of an entry.
func (entry *Entry) MarshalJSON() ([] byte, error) {
func (entry *Entry) MarshalJSON() ([]byte, error) {
object := entry.convertToObject(true)
description, err := json.Marshal(object)
@@ -259,15 +257,15 @@ func (entry *Entry) MarshalJSON() ([] byte, error) {
}
func (entry *Entry) IsFile() bool {
return entry.Mode & uint32(os.ModeType) == 0
return entry.Mode&uint32(os.ModeType) == 0
}
func (entry *Entry) IsDir() bool {
return entry.Mode & uint32(os.ModeDir) != 0
return entry.Mode&uint32(os.ModeDir) != 0
}
func (entry *Entry) IsLink() bool {
return entry.Mode & uint32(os.ModeSymlink) != 0
return entry.Mode&uint32(os.ModeSymlink) != 0
}
func (entry *Entry) GetPermissions() os.FileMode {
@@ -275,12 +273,12 @@ func (entry *Entry) GetPermissions() os.FileMode {
}
func (entry *Entry) IsSameAs(other *Entry) bool {
return entry.Size == other.Size && entry.Time <= other.Time + 1 && entry.Time >= other.Time - 1
return entry.Size == other.Size && entry.Time <= other.Time+1 && entry.Time >= other.Time-1
}
func (entry *Entry) IsSameAsFileInfo(other os.FileInfo) bool {
time := other.ModTime().Unix()
return entry.Size == other.Size() && entry.Time <= time + 1 && entry.Time >= time - 1
return entry.Size == other.Size() && entry.Time <= time+1 && entry.Time >= time-1
}
func (entry *Entry) String(maxSizeDigits int) string {
@@ -299,7 +297,7 @@ func (entry *Entry) RestoreMetadata(fullPath string, fileInfo *os.FileInfo) bool
}
}
if (*fileInfo).Mode() & os.ModePerm != entry.GetPermissions() {
if (*fileInfo).Mode()&os.ModePerm != entry.GetPermissions() {
err := os.Chmod(fullPath, entry.GetPermissions())
if err != nil {
LOG_ERROR("RESTORE_CHMOD", "Failed to set the file permissions: %v", err)
@@ -323,7 +321,6 @@ func (entry *Entry) RestoreMetadata(fullPath string, fileInfo *os.FileInfo) bool
return SetOwner(fullPath, entry, fileInfo)
}
// Return -1 if 'left' should appear before 'right', 1 if opposite, and 0 if they are the same.
// Files are always arranged before subdirectories under the same parent directory.
func (left *Entry) Compare(right *Entry) int {
@@ -417,14 +414,14 @@ func (files FileInfoCompare) Less(i, j int) bool {
left := files[i]
right := files[j]
if left.IsDir() && left.Mode() & os.ModeSymlink == 0 {
if right.IsDir() && right.Mode() & os.ModeSymlink == 0 {
if left.IsDir() && left.Mode()&os.ModeSymlink == 0 {
if right.IsDir() && right.Mode()&os.ModeSymlink == 0 {
return left.Name() < right.Name()
} else {
return false
}
} else {
if right.IsDir() && right.Mode() & os.ModeSymlink == 0 {
if right.IsDir() && right.Mode()&os.ModeSymlink == 0 {
return true
} else {
return left.Name() < right.Name()
@@ -434,8 +431,8 @@ func (files FileInfoCompare) Less(i, j int) bool {
// ListEntries returns a list of entries representing file and subdirectories under the directory 'path'. Entry paths
// are normalized as relative to 'top'. 'patterns' are used to exclude or include certain files.
func ListEntries(top string, path string, fileList *[]*Entry, patterns [] string, discardAttributes bool) (directoryList []*Entry,
skippedFiles [] string, err error) {
func ListEntries(top string, path string, fileList *[]*Entry, patterns []string, discardAttributes bool) (directoryList []*Entry,
skippedFiles []string, err error) {
LOG_DEBUG("LIST_ENTRIES", "Listing %s", path)
@@ -449,12 +446,12 @@ func ListEntries(top string, path string, fileList *[]*Entry, patterns [] string
}
normalizedPath := path
if len(normalizedPath) > 0 && normalizedPath[len(normalizedPath) - 1] != '/' {
if len(normalizedPath) > 0 && normalizedPath[len(normalizedPath)-1] != '/' {
normalizedPath += "/"
}
normalizedTop := top
if normalizedTop != "" && normalizedTop[len(normalizedTop) - 1] != '/' {
if normalizedTop != "" && normalizedTop[len(normalizedTop)-1] != '/' {
normalizedTop += "/"
}
@@ -475,7 +472,7 @@ func ListEntries(top string, path string, fileList *[]*Entry, patterns [] string
isRegular := false
isRegular, entry.Link, err = Readlink(filepath.Join(top, entry.Path))
if err != nil {
LOG_WARN("LIST_LINK", "Failed to read the symlink %s: %v", entry.Path, err )
LOG_WARN("LIST_LINK", "Failed to read the symlink %s: %v", entry.Path, err)
skippedFiles = append(skippedFiles, entry.Path)
continue
}
@@ -485,7 +482,7 @@ func ListEntries(top string, path string, fileList *[]*Entry, patterns [] string
} else if path == "" && filepath.IsAbs(entry.Link) && !strings.HasPrefix(entry.Link, normalizedTop) {
stat, err := os.Stat(filepath.Join(top, entry.Path))
if err != nil {
LOG_WARN("LIST_LINK", "Failed to read the symlink: %v", err )
LOG_WARN("LIST_LINK", "Failed to read the symlink: %v", err)
skippedFiles = append(skippedFiles, entry.Path)
continue
}
@@ -504,7 +501,7 @@ func ListEntries(top string, path string, fileList *[]*Entry, patterns [] string
entry.ReadAttributes(top)
}
if f.Mode() & (os.ModeNamedPipe | os.ModeSocket | os.ModeDevice) != 0 {
if f.Mode()&(os.ModeNamedPipe|os.ModeSocket|os.ModeDevice) != 0 {
LOG_WARN("LIST_SKIP", "Skipped non-regular file %s", entry.Path)
skippedFiles = append(skippedFiles, entry.Path)
continue
@@ -526,7 +523,7 @@ func ListEntries(top string, path string, fileList *[]*Entry, patterns [] string
}
}
for i, j := 0, len(directoryList) - 1; i < j; i, j = i + 1, j - 1 {
for i, j := 0, len(directoryList)-1; i < j; i, j = i+1, j-1 {
directoryList[i], directoryList[j] = directoryList[j], directoryList[i]
}
@@ -534,8 +531,8 @@ func ListEntries(top string, path string, fileList *[]*Entry, patterns [] string
}
// Diff returns how many bytes remain unmodifiled between two files.
func (entry *Entry) Diff(chunkHashes[]string, chunkLengths[]int,
otherHashes[]string, otherLengths [] int) (modifiedLength int64) {
func (entry *Entry) Diff(chunkHashes []string, chunkLengths []int,
otherHashes []string, otherLengths []int) (modifiedLength int64) {
var offset1, offset2 int64
i1 := entry.StartChunk
@@ -559,7 +556,7 @@ func (entry *Entry) Diff(chunkHashes[]string, chunkLengths[]int,
offset2 += int64(otherLengths[i2])
i2++
} else {
if chunkHashes[i1] == otherHashes[i2] && end - start == otherLengths[i2] {
if chunkHashes[i1] == otherHashes[i2] && end-start == otherLengths[i2] {
} else {
modifiedLength += int64(chunkLengths[i1])
}

32
src/duplicacy_entry_test.go
View File

@@ -5,17 +5,17 @@
package duplicacy
import (
"testing"
"io/ioutil"
"math/rand"
"os"
"path/filepath"
"math/rand"
"sort"
"testing"
)
func TestEntrySort(t *testing.T) {
DATA := [...]string {
DATA := [...]string{
"ab",
"ab-",
"ab0",
@@ -44,15 +44,15 @@ func TestEntrySort(t *testing.T) {
var entry1, entry2 *Entry
for i, p1 := range DATA {
if p1[len(p1) - 1] == '/' {
entry1 = CreateEntry(p1, 0, 0, 0700 | uint32(os.ModeDir))
if p1[len(p1)-1] == '/' {
entry1 = CreateEntry(p1, 0, 0, 0700|uint32(os.ModeDir))
} else {
entry1 = CreateEntry(p1, 0, 0, 0700)
}
for j, p2 := range DATA {
if p2[len(p2) - 1] == '/' {
entry2 = CreateEntry(p2, 0, 0, 0700 | uint32(os.ModeDir))
if p2[len(p2)-1] == '/' {
entry2 = CreateEntry(p2, 0, 0, 0700|uint32(os.ModeDir))
} else {
entry2 = CreateEntry(p2, 0, 0, 0700)
}
@@ -88,7 +88,7 @@ func TestEntryList(t *testing.T) {
os.RemoveAll(testDir)
os.MkdirAll(testDir, 0700)
DATA := [...]string {
DATA := [...]string{
"ab",
"ab-",
"ab0",
@@ -107,19 +107,18 @@ func TestEntryList(t *testing.T) {
"ab3/c",
}
var entry1, entry2 *Entry
for i, p1 := range DATA {
if p1[len(p1) - 1] == '/' {
entry1 = CreateEntry(p1, 0, 0, 0700 | uint32(os.ModeDir))
if p1[len(p1)-1] == '/' {
entry1 = CreateEntry(p1, 0, 0, 0700|uint32(os.ModeDir))
} else {
entry1 = CreateEntry(p1, 0, 0, 0700)
}
for j, p2 := range DATA {
if p2[len(p2) - 1] == '/' {
entry2 = CreateEntry(p2, 0, 0, 0700 | uint32(os.ModeDir))
if p2[len(p2)-1] == '/' {
entry2 = CreateEntry(p2, 0, 0, 0700|uint32(os.ModeDir))
} else {
entry2 = CreateEntry(p2, 0, 0, 0700)
}
@@ -151,7 +150,7 @@ func TestEntryList(t *testing.T) {
for _, file := range DATA {
fullPath := filepath.Join(testDir, file)
if file[len(file) - 1] == '/' {
if file[len(file)-1] == '/' {
err := os.Mkdir(fullPath, 0700)
if err != nil {
t.Errorf("Mkdir(%s) returned an error: %s", fullPath, err)
@@ -171,8 +170,8 @@ func TestEntryList(t *testing.T) {
entries := make([]*Entry, 0, 4)
for len(directories) > 0 {
directory := directories[len(directories) - 1]
directories = directories[:len(directories) - 1]
directory := directories[len(directories)-1]
directories = directories[:len(directories)-1]
entries = append(entries, directory)
subdirectories, _, err := ListEntries(testDir, directory.Path, &entries, nil, false)
if err != nil {
@@ -217,4 +216,3 @@ func TestEntryList(t *testing.T) {
}
}

14
src/duplicacy_filereader.go
View File

@@ -11,19 +11,19 @@ import (
// FileReader wraps a number of files and turns them into a series of readers.
type FileReader struct {
top string
files [] *Entry
files []*Entry
CurrentFile *os.File
CurrentIndex int
CurrentEntry *Entry
SkippedFiles [] string
SkippedFiles []string
}
// CreateFileReader creates a file reader.
func CreateFileReader(top string, files[] *Entry) (*FileReader) {
func CreateFileReader(top string, files []*Entry) *FileReader {
reader := &FileReader {
reader := &FileReader{
top: top,
files: files,
CurrentIndex: -1,
@@ -35,7 +35,7 @@ func CreateFileReader(top string, files[] *Entry) (*FileReader) {
}
// NextFile switchs to the next file in the file reader.
func (reader *FileReader) NextFile() bool{
func (reader *FileReader) NextFile() bool {
if reader.CurrentFile != nil {
reader.CurrentFile.Close()
@@ -68,7 +68,3 @@ func (reader *FileReader) NextFile() bool{
reader.CurrentFile = nil
return false
}

42
src/duplicacy_filestorage.go
View File

@@ -5,13 +5,13 @@
package duplicacy
import (
"os"
"fmt"
"path"
"io"
"io/ioutil"
"time"
"math/rand"
"os"
"path"
"time"
)
// FileStorage is a local on-disk file storage implementing the Storage interface.
@@ -45,12 +45,12 @@ func CreateFileStorage(storageDir string, minimumLevel int, isCacheNeeded bool,
}
}
for storageDir[len(storageDir) - 1] == '/' {
storageDir = storageDir[:len(storageDir) - 1]
for storageDir[len(storageDir)-1] == '/' {
storageDir = storageDir[:len(storageDir)-1]
}
storage = &FileStorage {
storageDir : storageDir,
storage = &FileStorage{
storageDir: storageDir,
minimumLevel: minimumLevel,
isCacheNeeded: isCacheNeeded,
numberOfThreads: threads,
@@ -77,7 +77,7 @@ func (storage *FileStorage) ListFiles(threadIndex int, dir string) (files []stri
for _, f := range list {
name := f.Name()
if f.IsDir() && name[len(name) - 1] != '/' {
if f.IsDir() && name[len(name)-1] != '/' {
name += "/"
}
files = append(files, name)
@@ -136,23 +136,23 @@ func (storage *FileStorage) FindChunk(threadIndex int, chunkID string, isFossil
suffix = ".fsl"
}
for level := 0; level * 2 < len(chunkID); level ++ {
for level := 0; level*2 < len(chunkID); level++ {
if level >= storage.minimumLevel {
filePath = path.Join(dir, chunkID[2 * level:]) + suffix
filePath = path.Join(dir, chunkID[2*level:]) + suffix
// Use Lstat() instead of Stat() since 1) Stat() doesn't work for deduplicated disks on Windows and 2) there isn't
// really a need to follow the link if filePath is a link.
stat, err := os.Lstat(filePath)
if err != nil {
LOG_DEBUG("FS_FIND", "File %s can't be found: %v", filePath, err)
} else if stat.IsDir() {
return filePath[len(storage.storageDir) + 1:], false, 0, fmt.Errorf("The path %s is a directory", filePath)
return filePath[len(storage.storageDir)+1:], false, 0, fmt.Errorf("The path %s is a directory", filePath)
} else {
return filePath[len(storage.storageDir) + 1:], true, stat.Size(), nil
return filePath[len(storage.storageDir)+1:], true, stat.Size(), nil
}
}
// Find the subdirectory the chunk file may reside.
subDir := path.Join(dir, chunkID[2 * level: 2 * level + 2])
subDir := path.Join(dir, chunkID[2*level:2*level+2])
stat, err := os.Stat(subDir)
if err == nil && stat.IsDir() {
dir = subDir
@@ -179,7 +179,7 @@ func (storage *FileStorage) FindChunk(threadIndex int, chunkID string, isFossil
}
// The chunk must be under this subdirectory but it doesn't exist.
return path.Join(dir, chunkID[2 * level:])[len(storage.storageDir) + 1:] + suffix, false, 0, nil
return path.Join(dir, chunkID[2*level:])[len(storage.storageDir)+1:] + suffix, false, 0, nil
}
@@ -197,7 +197,7 @@ func (storage *FileStorage) DownloadFile(threadIndex int, filePath string, chunk
}
defer file.Close()
if _, err = RateLimitedCopy(chunk, file, storage.DownloadRateLimit / storage.numberOfThreads); err != nil {
if _, err = RateLimitedCopy(chunk, file, storage.DownloadRateLimit/storage.numberOfThreads); err != nil {
return err
}
@@ -218,12 +218,12 @@ func (storage *FileStorage) UploadFile(threadIndex int, filePath string, content
temporaryFile := fullPath + "." + string(suffix) + ".tmp"
file, err := os.OpenFile(temporaryFile, os.O_WRONLY | os.O_CREATE | os.O_TRUNC, 0644)
file, err := os.OpenFile(temporaryFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
if err != nil {
return err
}
reader := CreateRateLimitedReader(content, storage.UploadRateLimit / storage.numberOfThreads)
reader := CreateRateLimitedReader(content, storage.UploadRateLimit/storage.numberOfThreads)
_, err = io.Copy(file, reader)
if err != nil {
file.Close()
@@ -248,16 +248,16 @@ func (storage *FileStorage) UploadFile(threadIndex int, filePath string, content
// If a local snapshot cache is needed for the storage to avoid downloading/uploading chunks too often when
// managing snapshots.
func (storage *FileStorage) IsCacheNeeded () (bool) { return storage.isCacheNeeded }
func (storage *FileStorage) IsCacheNeeded() bool { return storage.isCacheNeeded }
// If the 'MoveFile' method is implemented.
func (storage *FileStorage) IsMoveFileImplemented() (bool) { return true }
func (storage *FileStorage) IsMoveFileImplemented() bool { return true }
// If the storage can guarantee strong consistency.
func (storage *FileStorage) IsStrongConsistent() (bool) { return true }
func (storage *FileStorage) IsStrongConsistent() bool { return true }
// If the storage supports fast listing of files names.
func (storage *FileStorage) IsFastListing() (bool) { return false }
func (storage *FileStorage) IsFastListing() bool { return false }
// Enable the test mode.
func (storage *FileStorage) EnableTestMode() {}

78
src/duplicacy_gcdstorage.go
View File

@@ -5,18 +5,18 @@
package duplicacy
import (
"io"
"encoding/json"
"fmt"
"net"
"path"
"time"
"sync"
"strings"
"net/http"
"net/url"
"io"
"io/ioutil"
"math/rand"
"encoding/json"
"net"
"net/http"
"net/url"
"path"
"strings"
"sync"
"time"
"golang.org/x/net/context"
"golang.org/x/oauth2"
@@ -35,7 +35,6 @@ type GCDStorage struct {
isConnected bool
numberOfThreads int
TestMode bool
}
type GCDConfig struct {
@@ -96,9 +95,9 @@ func (storage *GCDStorage) shouldRetry(threadIndex int, err error) (bool, error)
return true, nil
}
func (storage *GCDStorage) convertFilePath(filePath string) (string) {
func (storage *GCDStorage) convertFilePath(filePath string) string {
if strings.HasPrefix(filePath, "chunks/") && strings.HasSuffix(filePath, ".fsl") {
return "fossils/" + filePath[len("chunks/"):len(filePath) - len(".fsl")]
return "fossils/" + filePath[len("chunks/"):len(filePath)-len(".fsl")]
}
return filePath
}
@@ -135,7 +134,7 @@ func (storage *GCDStorage) listFiles(threadIndex int, parentID string, listFiles
return nil, fmt.Errorf("No parent ID provided")
}
files := []*drive.File {}
files := []*drive.File{}
startToken := ""
@@ -174,7 +173,6 @@ func (storage *GCDStorage) listFiles(threadIndex int, parentID string, listFiles
}
}
return files, nil
}
@@ -238,7 +236,7 @@ func (storage *GCDStorage) getIDFromPath(threadIndex int, path string) (string,
if fileID == "" {
return "", fmt.Errorf("Path %s doesn't exist", path)
}
if i != len(names) - 1 && !isDir {
if i != len(names)-1 && !isDir {
return "", fmt.Errorf("Invalid path %s", path)
}
}
@@ -253,7 +251,7 @@ func CreateGCDStorage(tokenFile string, storagePath string, threads int) (storag
return nil, err
}
gcdConfig := &GCDConfig {}
gcdConfig := &GCDConfig{}
if err := json.Unmarshal(description, gcdConfig); err != nil {
return nil, err
}
@@ -271,7 +269,7 @@ func CreateGCDStorage(tokenFile string, storagePath string, threads int) (storag
return nil, err
}
storage = &GCDStorage {
storage = &GCDStorage{
service: service,
numberOfThreads: threads,
idCache: make(map[string]string),
@@ -286,7 +284,7 @@ func CreateGCDStorage(tokenFile string, storagePath string, threads int) (storag
storage.idCache[""] = storagePathID
for _, dir := range []string { "chunks", "snapshots", "fossils" } {
for _, dir := range []string{"chunks", "snapshots", "fossils"} {
dirID, isDir, _, err := storage.listByName(0, storagePathID, dir)
if err != nil {
return nil, err
@@ -297,7 +295,7 @@ func CreateGCDStorage(tokenFile string, storagePath string, threads int) (storag
return nil, err
}
} else if !isDir {
return nil, fmt.Errorf("%s/%s is not a directory", storagePath + "/" + dir)
return nil, fmt.Errorf("%s/%s is not a directory", storagePath+"/"+dir)
} else {
storage.idCache[dir] = dirID
}
@@ -311,8 +309,8 @@ func CreateGCDStorage(tokenFile string, storagePath string, threads int) (storag
// ListFiles return the list of files and subdirectories under 'dir' (non-recursively)
func (storage *GCDStorage) ListFiles(threadIndex int, dir string) ([]string, []int64, error) {
for len(dir) > 0 && dir[len(dir) - 1] == '/' {
dir = dir[:len(dir) - 1]
for len(dir) > 0 && dir[len(dir)-1] == '/' {
dir = dir[:len(dir)-1]
}
if dir == "snapshots" {
@@ -325,8 +323,8 @@ func (storage *GCDStorage) ListFiles(threadIndex int, dir string) ([]string, []i
subDirs := []string{}
for _, file := range files {
storage.savePathID("snapshots/" + file.Name, file.Id)
subDirs = append(subDirs, file.Name + "/")
storage.savePathID("snapshots/"+file.Name, file.Id)
subDirs = append(subDirs, file.Name+"/")
}
return subDirs, nil, nil
} else if strings.HasPrefix(dir, "snapshots/") {
@@ -343,7 +341,7 @@ func (storage *GCDStorage) ListFiles(threadIndex int, dir string) ([]string, []i
files := []string{}
for _, entry := range entries {
storage.savePathID(dir + "/" + entry.Name, entry.Id)
storage.savePathID(dir+"/"+entry.Name, entry.Id)
files = append(files, entry.Name)
}
return files, nil, nil
@@ -351,7 +349,7 @@ func (storage *GCDStorage) ListFiles(threadIndex int, dir string) ([]string, []i
files := []string{}
sizes := []int64{}
for _, parent := range []string { "chunks", "fossils" } {
for _, parent := range []string{"chunks", "fossils"} {
entries, err := storage.listFiles(threadIndex, storage.getPathID(parent), true)
if err != nil {
return nil, nil, err
@@ -362,7 +360,7 @@ func (storage *GCDStorage) ListFiles(threadIndex int, dir string) ([]string, []i
if parent == "fossils" {
name += ".fsl"
}
storage.savePathID(parent + "/" + entry.Name, entry.Id)
storage.savePathID(parent+"/"+entry.Name, entry.Id)
files = append(files, name)
sizes = append(sizes, entry.Size)
}
@@ -438,8 +436,8 @@ func (storage *GCDStorage) MoveFile(threadIndex int, from string, to string) (er
// CreateDirectory creates a new directory.
func (storage *GCDStorage) CreateDirectory(threadIndex int, dir string) (err error) {
for len(dir) > 0 && dir[len(dir) - 1] == '/' {
dir = dir[:len(dir) - 1]
for len(dir) > 0 && dir[len(dir)-1] == '/' {
dir = dir[:len(dir)-1]
}
exist, isDir, _, err := storage.GetFileInfo(threadIndex, dir)
@@ -462,10 +460,10 @@ func (storage *GCDStorage) CreateDirectory(threadIndex int, dir string) (err err
name = dir[len("snapshots/"):]
}
file := &drive.File {
file := &drive.File{
Name: name,
MimeType: "application/vnd.google-apps.folder",
Parents: []string { parentID },
Parents: []string{parentID},
}
for {
@@ -485,8 +483,8 @@ func (storage *GCDStorage) CreateDirectory(threadIndex int, dir string) (err err
// GetFileInfo returns the information about the file or directory at 'filePath'.
func (storage *GCDStorage) GetFileInfo(threadIndex int, filePath string) (exist bool, isDir bool, size int64, err error) {
for len(filePath) > 0 && filePath[len(filePath) - 1] == '/' {
filePath = filePath[:len(filePath) - 1]
for len(filePath) > 0 && filePath[len(filePath)-1] == '/' {
filePath = filePath[:len(filePath)-1]
}
// GetFileInfo is never called on a fossil
@@ -568,7 +566,7 @@ func (storage *GCDStorage) DownloadFile(threadIndex int, filePath string, chunk
defer response.Body.Close()
_, err = RateLimitedCopy(chunk, response.Body, storage.DownloadRateLimit / storage.numberOfThreads)
_, err = RateLimitedCopy(chunk, response.Body, storage.DownloadRateLimit/storage.numberOfThreads)
return err
}
@@ -591,14 +589,14 @@ func (storage *GCDStorage) UploadFile(threadIndex int, filePath string, content
storage.savePathID(parent, parentID)
}
file := &drive.File {
file := &drive.File{
Name: path.Base(filePath),
MimeType: "application/octet-stream",
Parents: []string { parentID },
Parents: []string{parentID},
}
for {
reader := CreateRateLimitedReader(content, storage.UploadRateLimit / storage.numberOfThreads)
reader := CreateRateLimitedReader(content, storage.UploadRateLimit/storage.numberOfThreads)
_, err = storage.service.Files.Create(file).Media(reader).Fields("id").Do()
if retry, err := storage.shouldRetry(threadIndex, err); err == nil && !retry {
break
@@ -614,16 +612,16 @@ func (storage *GCDStorage) UploadFile(threadIndex int, filePath string, content
// If a local snapshot cache is needed for the storage to avoid downloading/uploading chunks too often when
// managing snapshots.
func (storage *GCDStorage) IsCacheNeeded() (bool) { return true }
func (storage *GCDStorage) IsCacheNeeded() bool { return true }
// If the 'MoveFile' method is implemented.
func (storage *GCDStorage) IsMoveFileImplemented() (bool) { return true }
func (storage *GCDStorage) IsMoveFileImplemented() bool { return true }
// If the storage can guarantee strong consistency.
func (storage *GCDStorage) IsStrongConsistent() (bool) { return false }
func (storage *GCDStorage) IsStrongConsistent() bool { return false }
// If the storage supports fast listing of files names.
func (storage *GCDStorage) IsFastListing() (bool) { return true }
func (storage *GCDStorage) IsFastListing() bool { return true }
// Enable the test mode.
func (storage *GCDStorage) EnableTestMode() { storage.TestMode = true }

46
src/duplicacy_gcsstorage.go
View File

@@ -5,22 +5,22 @@
package duplicacy
import (
"io"
"fmt"
"net"
"time"
"net/url"
"math/rand"
"io/ioutil"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"math/rand"
"net"
"net/url"
"time"
gcs "cloud.google.com/go/storage"
"golang.org/x/net/context"
"golang.org/x/oauth2"
"golang.org/x/oauth2/google"
gcs "cloud.google.com/go/storage"
"google.golang.org/api/googleapi"
"google.golang.org/api/iterator"
"google.golang.org/api/option"
"google.golang.org/api/googleapi"
)
type GCSStorage struct {
@@ -31,7 +31,6 @@ type GCSStorage struct {
numberOfThreads int
TestMode bool
}
type GCSConfig struct {
@@ -51,7 +50,7 @@ func CreateGCSStorage(tokenFile string, bucketName string, storageDir string, th
return nil, err
}
var object map[string]interface {}
var object map[string]interface{}
err = json.Unmarshal(description, &object)
if err != nil {
@@ -74,7 +73,7 @@ func CreateGCSStorage(tokenFile string, bucketName string, storageDir string, th
}
tokenSource = config.TokenSource(ctx)
} else {
gcsConfig := &GCSConfig {}
gcsConfig := &GCSConfig{}
if err := json.Unmarshal(description, gcsConfig); err != nil {
return nil, err
}
@@ -92,11 +91,11 @@ func CreateGCSStorage(tokenFile string, bucketName string, storageDir string, th
bucket := client.Bucket(bucketName)
if len(storageDir) > 0 && storageDir[len(storageDir) - 1] != '/' {
if len(storageDir) > 0 && storageDir[len(storageDir)-1] != '/' {
storageDir += "/"
}
storage = &GCSStorage {
storage = &GCSStorage{
bucket: bucket,
storageDir: storageDir,
numberOfThreads: threads,
@@ -149,14 +148,13 @@ func (storage *GCSStorage) shouldRetry(backoff *int, err error) (bool, error) {
return true, nil
}
// ListFiles return the list of files and subdirectories under 'dir' (non-recursively)
func (storage *GCSStorage) ListFiles(threadIndex int, dir string) ([]string, []int64, error) {
for len(dir) > 0 && dir[len(dir) - 1] == '/' {
dir = dir[:len(dir) - 1]
for len(dir) > 0 && dir[len(dir)-1] == '/' {
dir = dir[:len(dir)-1]
}
query := gcs.Query {
query := gcs.Query{
Prefix: storage.storageDir + dir + "/",
}
dirOnly := false
@@ -260,7 +258,7 @@ func (storage *GCSStorage) DownloadFile(threadIndex int, filePath string, chunk
return err
}
defer readCloser.Close()
_, err = RateLimitedCopy(chunk, readCloser, storage.DownloadRateLimit / storage.numberOfThreads)
_, err = RateLimitedCopy(chunk, readCloser, storage.DownloadRateLimit/storage.numberOfThreads)
return err
}
@@ -271,7 +269,7 @@ func (storage *GCSStorage) UploadFile(threadIndex int, filePath string, content
for {
writeCloser := storage.bucket.Object(storage.storageDir + filePath).NewWriter(context.Background())
defer writeCloser.Close()
reader := CreateRateLimitedReader(content, storage.UploadRateLimit / storage.numberOfThreads)
reader := CreateRateLimitedReader(content, storage.UploadRateLimit/storage.numberOfThreads)
_, err = io.Copy(writeCloser, reader)
if retry, e := storage.shouldRetry(&backoff, err); e == nil && !retry {
@@ -288,16 +286,16 @@ func (storage *GCSStorage) UploadFile(threadIndex int, filePath string, content
// If a local snapshot cache is needed for the storage to avoid downloading/uploading chunks too often when
// managing snapshots.
func (storage *GCSStorage) IsCacheNeeded() (bool) { return true }
func (storage *GCSStorage) IsCacheNeeded() bool { return true }
// If the 'MoveFile' method is implemented.
func (storage *GCSStorage) IsMoveFileImplemented() (bool) { return true }
func (storage *GCSStorage) IsMoveFileImplemented() bool { return true }
// If the storage can guarantee strong consistency.
func (storage *GCSStorage) IsStrongConsistent() (bool) { return true }
func (storage *GCSStorage) IsStrongConsistent() bool { return true }
// If the storage supports fast listing of files names.
func (storage *GCSStorage) IsFastListing() (bool) { return true }
func (storage *GCSStorage) IsFastListing() bool { return true }
// Enable the test mode.
func (storage *GCSStorage) EnableTestMode() { storage.TestMode = true }

62
src/duplicacy_hubicclient.go
View File

@@ -5,18 +5,18 @@
package duplicacy
import (
"fmt"
"net"
"time"
"sync"
"bytes"
"strings"
"io/ioutil"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"math/rand"
"net"
"net/http"
net_url "net/url"
"math/rand"
"strings"
"sync"
"time"
"golang.org/x/oauth2"
)
@@ -65,8 +65,8 @@ func NewHubicClient(tokenFile string) (*HubicClient, error) {
}
client := &HubicClient{
HTTPClient: &http.Client {
Transport: &http.Transport {
HTTPClient: &http.Client{
Transport: &http.Transport{
Dial: (&net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
@@ -137,7 +137,7 @@ func (client *HubicClient) call(url string, method string, input interface{}, ex
if url == HubicCredentialURL {
client.TokenLock.Lock()
request.Header.Set("Authorization", "Bearer " + client.Token.AccessToken)
request.Header.Set("Authorization", "Bearer "+client.Token.AccessToken)
client.TokenLock.Unlock()
} else if url != HubicRefreshTokenURL {
client.CredentialLock.Lock()
@@ -172,11 +172,11 @@ func (client *HubicClient) call(url string, method string, input interface{}, ex
if response.StatusCode == 401 {
if url == HubicRefreshTokenURL {
return nil, 0, "", HubicError { Status: response.StatusCode, Message: "Authorization error when refreshing token"}
return nil, 0, "", HubicError{Status: response.StatusCode, Message: "Authorization error when refreshing token"}
}
if url == HubicCredentialURL {
return nil, 0, "", HubicError { Status: response.StatusCode, Message: "Authorization error when retrieving credentials"}
return nil, 0, "", HubicError{Status: response.StatusCode, Message: "Authorization error when retrieving credentials"}
}
err = client.RefreshToken()
@@ -196,7 +196,7 @@ func (client *HubicClient) call(url string, method string, input interface{}, ex
backoff *= 2
continue
} else {
return nil, 0, "", HubicError { Status: response.StatusCode, Message: "Hubic API error"}
return nil, 0, "", HubicError{Status: response.StatusCode, Message: "Hubic API error"}
}
}
@@ -264,7 +264,7 @@ type HubicEntry struct {
func (client *HubicClient) ListEntries(path string) ([]HubicEntry, error) {
if len(path) > 0 && path[len(path) - 1] != '/' {
if len(path) > 0 && path[len(path)-1] != '/' {
path += "/"
}
@@ -308,8 +308,8 @@ func (client *HubicClient) ListEntries(path string) ([]HubicEntry, error) {
marker = entry.Name
} else {
marker = entry.Subdir
for len(entry.Subdir) > 0 && entry.Subdir[len(entry.Subdir) - 1] == '/' {
entry.Subdir = entry.Subdir[:len(entry.Subdir) - 1]
for len(entry.Subdir) > 0 && entry.Subdir[len(entry.Subdir)-1] == '/' {
entry.Subdir = entry.Subdir[:len(entry.Subdir)-1]
}
entry.Name = entry.Subdir
entry.Type = "application/directory"
@@ -329,8 +329,8 @@ func (client *HubicClient) ListEntries(path string) ([]HubicEntry, error) {
func (client *HubicClient) GetFileInfo(path string) (bool, bool, int64, error) {
for len(path) > 0 && path[len(path) - 1] == '/' {
path = path[:len(path) - 1]
for len(path) > 0 && path[len(path)-1] == '/' {
path = path[:len(path)-1]
}
client.CredentialLock.Lock()
@@ -353,8 +353,8 @@ func (client *HubicClient) GetFileInfo(path string) (bool, bool, int64, error) {
func (client *HubicClient) DownloadFile(path string) (io.ReadCloser, int64, error) {
for len(path) > 0 && path[len(path) - 1] == '/' {
path = path[:len(path) - 1]
for len(path) > 0 && path[len(path)-1] == '/' {
path = path[:len(path)-1]
}
client.CredentialLock.Lock()
@@ -367,8 +367,8 @@ func (client *HubicClient) DownloadFile(path string) (io.ReadCloser, int64, erro
func (client *HubicClient) UploadFile(path string, content []byte, rateLimit int) (err error) {
for len(path) > 0 && path[len(path) - 1] == '/' {
path = path[:len(path) - 1]
for len(path) > 0 && path[len(path)-1] == '/' {
path = path[:len(path)-1]
}
client.CredentialLock.Lock()
@@ -390,8 +390,8 @@ func (client *HubicClient) UploadFile(path string, content []byte, rateLimit int
func (client *HubicClient) DeleteFile(path string) error {
for len(path) > 0 && path[len(path) - 1] == '/' {
path = path[:len(path) - 1]
for len(path) > 0 && path[len(path)-1] == '/' {
path = path[:len(path)-1]
}
client.CredentialLock.Lock()
@@ -410,12 +410,12 @@ func (client *HubicClient) DeleteFile(path string) error {
func (client *HubicClient) MoveFile(from string, to string) error {
for len(from) > 0 && from[len(from) - 1] == '/' {
from = from[:len(from) - 1]
for len(from) > 0 && from[len(from)-1] == '/' {
from = from[:len(from)-1]
}
for len(to) > 0 && to[len(to) - 1] == '/' {
to = to[:len(to) - 1]
for len(to) > 0 && to[len(to)-1] == '/' {
to = to[:len(to)-1]
}
client.CredentialLock.Lock()
@@ -436,10 +436,10 @@ func (client *HubicClient) MoveFile(from string, to string) error {
return client.DeleteFile(from)
}
func (client *HubicClient) CreateDirectory(path string) (error) {
func (client *HubicClient) CreateDirectory(path string) error {
for len(path) > 0 && path[len(path) - 1] == '/' {
path = path[:len(path) - 1]
for len(path) > 0 && path[len(path)-1] == '/' {
path = path[:len(path)-1]
}
client.CredentialLock.Lock()

14
src/duplicacy_hubicclient_test.go
View File

@@ -5,11 +5,11 @@
package duplicacy
import (
"io"
"fmt"
"testing"
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
"testing"
crypto_rand "crypto/rand"
"math/rand"
@@ -73,7 +73,7 @@ func TestHubicClient(t *testing.T) {
maxFileSize := 64 * 1024
for i := 0; i < numberOfFiles; i++ {
content := make([]byte, rand.Int() % maxFileSize + 1)
content := make([]byte, rand.Int()%maxFileSize+1)
_, err = crypto_rand.Read(content)
if err != nil {
t.Errorf("Error generating random content: %v", err)
@@ -86,7 +86,7 @@ func TestHubicClient(t *testing.T) {
fmt.Printf("file: %s\n", filename)
err = hubicClient.UploadFile("test/test1/" + filename, content, 100)
err = hubicClient.UploadFile("test/test1/"+filename, content, 100)
if err != nil {
/*if e, ok := err.(ACDError); !ok || e.Status != 409 */ {
t.Errorf("Failed to upload the file %s: %v", filename, err)
@@ -104,9 +104,9 @@ func TestHubicClient(t *testing.T) {
for _, entry := range entries {
exists, isDir, size, err := hubicClient.GetFileInfo("test/test1/" + entry.Name)
fmt.Printf("%s exists: %t, isDir: %t, size: %d, err: %v\n", "test/test1/" + entry.Name, exists, isDir, size, err)
fmt.Printf("%s exists: %t, isDir: %t, size: %d, err: %v\n", "test/test1/"+entry.Name, exists, isDir, size, err)
err = hubicClient.MoveFile("test/test1/" + entry.Name, "test/test2/" + entry.Name)
err = hubicClient.MoveFile("test/test1/"+entry.Name, "test/test2/"+entry.Name)
if err != nil {
t.Errorf("Failed to move %s: %v", entry.Name, err)
return

34
src/duplicacy_hubicstorage.go
View File

@@ -20,8 +20,8 @@ type HubicStorage struct {
// CreateHubicStorage creates an Hubic storage object.
func CreateHubicStorage(tokenFile string, storagePath string, threads int) (storage *HubicStorage, err error) {
for len(storagePath) > 0 && storagePath[len(storagePath) - 1] == '/' {
storagePath = storagePath[:len(storagePath) - 1]
for len(storagePath) > 0 && storagePath[len(storagePath)-1] == '/' {
storagePath = storagePath[:len(storagePath)-1]
}
client, err := NewHubicClient(tokenFile)
@@ -42,13 +42,13 @@ func CreateHubicStorage(tokenFile string, storagePath string, threads int) (stor
return nil, fmt.Errorf("Path '%s' is not a directory", storagePath)
}
storage = &HubicStorage {
storage = &HubicStorage{
client: client,
storageDir: storagePath,
numberOfThreads: threads,
}
for _, path := range []string { "chunks", "snapshots" } {
for _, path := range []string{"chunks", "snapshots"} {
dir := storagePath + "/" + path
exists, isDir, _, err := client.GetFileInfo(dir)
if err != nil {
@@ -70,8 +70,8 @@ func CreateHubicStorage(tokenFile string, storagePath string, threads int) (stor
// ListFiles return the list of files and subdirectories under 'dir' (non-recursively)
func (storage *HubicStorage) ListFiles(threadIndex int, dir string) ([]string, []int64, error) {
for len(dir) > 0 && dir[len(dir) - 1] == '/' {
dir = dir[:len(dir) - 1]
for len(dir) > 0 && dir[len(dir)-1] == '/' {
dir = dir[:len(dir)-1]
}
if dir == "snapshots" {
@@ -83,7 +83,7 @@ func (storage *HubicStorage) ListFiles(threadIndex int, dir string) ([]string, [
subDirs := []string{}
for _, entry := range entries {
if entry.Type == "application/directory" {
subDirs = append(subDirs, entry.Name + "/")
subDirs = append(subDirs, entry.Name+"/")
}
}
return subDirs, nil, nil
@@ -142,8 +142,8 @@ func (storage *HubicStorage) MoveFile(threadIndex int, from string, to string) (
// CreateDirectory creates a new directory.
func (storage *HubicStorage) CreateDirectory(threadIndex int, dir string) (err error) {
for len(dir) > 0 && dir[len(dir) - 1] == '/' {
dir = dir[:len(dir) - 1]
for len(dir) > 0 && dir[len(dir)-1] == '/' {
dir = dir[:len(dir)-1]
}
return storage.client.CreateDirectory(storage.storageDir + "/" + dir)
@@ -152,8 +152,8 @@ func (storage *HubicStorage) CreateDirectory(threadIndex int, dir string) (err e
// GetFileInfo returns the information about the file or directory at 'filePath'.
func (storage *HubicStorage) GetFileInfo(threadIndex int, filePath string) (exist bool, isDir bool, size int64, err error) {
for len(filePath) > 0 && filePath[len(filePath) - 1] == '/' {
filePath = filePath[:len(filePath) - 1]
for len(filePath) > 0 && filePath[len(filePath)-1] == '/' {
filePath = filePath[:len(filePath)-1]
}
return storage.client.GetFileInfo(storage.storageDir + "/" + filePath)
}
@@ -179,27 +179,27 @@ func (storage *HubicStorage) DownloadFile(threadIndex int, filePath string, chun
defer readCloser.Close()
_, err = RateLimitedCopy(chunk, readCloser, storage.DownloadRateLimit / storage.numberOfThreads)
_, err = RateLimitedCopy(chunk, readCloser, storage.DownloadRateLimit/storage.numberOfThreads)
return err
}
// UploadFile writes 'content' to the file at 'filePath'.
func (storage *HubicStorage) UploadFile(threadIndex int, filePath string, content []byte) (err error) {
return storage.client.UploadFile(storage.storageDir + "/" + filePath, content, storage.UploadRateLimit / storage.numberOfThreads)
return storage.client.UploadFile(storage.storageDir+"/"+filePath, content, storage.UploadRateLimit/storage.numberOfThreads)
}
// If a local snapshot cache is needed for the storage to avoid downloading/uploading chunks too often when
// managing snapshots.
func (storage *HubicStorage) IsCacheNeeded() (bool) { return true }
func (storage *HubicStorage) IsCacheNeeded() bool { return true }
// If the 'MoveFile' method is implemented.
func (storage *HubicStorage) IsMoveFileImplemented() (bool) { return true }
func (storage *HubicStorage) IsMoveFileImplemented() bool { return true }
// If the storage can guarantee strong consistency.
func (storage *HubicStorage) IsStrongConsistent() (bool) { return false }
func (storage *HubicStorage) IsStrongConsistent() bool { return false }
// If the storage supports fast listing of files names.
func (storage *HubicStorage) IsFastListing() (bool) { return true }
func (storage *HubicStorage) IsFastListing() bool { return true }
// Enable the test mode.
func (storage *HubicStorage) EnableTestMode() {

12
src/duplicacy_keyring_windows.go
View File

@@ -5,10 +5,10 @@
package duplicacy
import (
"encoding/json"
"io/ioutil"
"syscall"
"unsafe"
"io/ioutil"
"encoding/json"
)
var keyringFile string
@@ -33,11 +33,11 @@ func SetKeyringFile(path string) {
func keyringEncrypt(value []byte) ([]byte, error) {
dataIn := DATA_BLOB {
dataIn := DATA_BLOB{
pbData: &value[0],
cbData: uint32(len(value)),
}
dataOut := DATA_BLOB {}
dataOut := DATA_BLOB{}
r, _, err := procEncryptData.Call(uintptr(unsafe.Pointer(&dataIn)),
0, 0, 0, 0, 0, uintptr(unsafe.Pointer(&dataOut)))
@@ -57,11 +57,11 @@ func keyringEncrypt(value []byte) ([]byte, error) {
func keyringDecrypt(value []byte) ([]byte, error) {
dataIn := DATA_BLOB {
dataIn := DATA_BLOB{
pbData: &value[0],
cbData: uint32(len(value)),
}
dataOut := DATA_BLOB {}
dataOut := DATA_BLOB{}
r, _, err := procDecryptData.Call(uintptr(unsafe.Pointer(&dataIn)),
0, 0, 0, 0, 0, uintptr(unsafe.Pointer(&dataOut)))

6
src/duplicacy_log.go
View File

@@ -5,12 +5,12 @@
package duplicacy
import (
"os"
"fmt"
"time"
"os"
"runtime/debug"
"sync"
"testing"
"runtime/debug"
"time"
)
const (

30
src/duplicacy_oneclient.go
View File

@@ -5,16 +5,16 @@
package duplicacy
import (
"fmt"
"time"
"sync"
"bytes"
"strings"
"io/ioutil"
"encoding/json"
"fmt"
"io"
"net/http"
"io/ioutil"
"math/rand"
"net/http"
"strings"
"sync"
"time"
"golang.org/x/oauth2"
)
@@ -108,7 +108,7 @@ func (client *OneDriveClient) call(url string, method string, input interface{},
if url != OneDriveRefreshTokenURL {
client.TokenLock.Lock()
request.Header.Set("Authorization", "Bearer " + client.Token.AccessToken)
request.Header.Set("Authorization", "Bearer "+client.Token.AccessToken)
client.TokenLock.Unlock()
}
if contentType != "" {
@@ -121,7 +121,7 @@ func (client *OneDriveClient) call(url string, method string, input interface{},
if strings.Contains(err.Error(), "TLS handshake timeout") {
// Give a long timeout regardless of backoff when a TLS timeout happens, hoping that
// idle connections are not to be reused on reconnect.
retryAfter := time.Duration(rand.Float32() * 60000 + 180000)
retryAfter := time.Duration(rand.Float32()*60000 + 180000)
LOG_INFO("ONEDRIVE_RETRY", "TLS handshake timeout; retry after %d milliseconds", retryAfter)
time.Sleep(retryAfter * time.Millisecond)
} else {
@@ -144,14 +144,14 @@ func (client *OneDriveClient) call(url string, method string, input interface{},
defer response.Body.Close()
errorResponse := &OneDriveErrorResponse {
Error: OneDriveError { Status: response.StatusCode },
errorResponse := &OneDriveErrorResponse{
Error: OneDriveError{Status: response.StatusCode},
}
if response.StatusCode == 401 {
if url == OneDriveRefreshTokenURL {
return nil, 0, OneDriveError { Status: response.StatusCode, Message: "Authorization error when refreshing token"}
return nil, 0, OneDriveError{Status: response.StatusCode, Message: "Authorization error when refreshing token"}
}
err = client.RefreshToken()
@@ -167,7 +167,7 @@ func (client *OneDriveClient) call(url string, method string, input interface{},
continue
} else {
if err := json.NewDecoder(response.Body).Decode(errorResponse); err != nil {
return nil, 0, OneDriveError { Status: response.StatusCode, Message: fmt.Sprintf("Unexpected response"), }
return nil, 0, OneDriveError{Status: response.StatusCode, Message: fmt.Sprintf("Unexpected response")}
}
errorResponse.Error.Status = response.StatusCode
@@ -213,7 +213,7 @@ func (client *OneDriveClient) RefreshToken() (err error) {
type OneDriveEntry struct {
ID string
Name string
Folder map[string] interface {}
Folder map[string]interface{}
Size int64
}
@@ -245,7 +245,7 @@ func (client *OneDriveClient) ListEntries(path string) ([]OneDriveEntry, error)
defer readCloser.Close()
output := &OneDriveListEntriesOutput {}
output := &OneDriveListEntriesOutput{}
if err = json.NewDecoder(readCloser).Decode(&output); err != nil {
return nil, err
@@ -340,7 +340,7 @@ func (client *OneDriveClient) MoveFile(path string, parent string) error {
return nil
}
func (client *OneDriveClient) CreateDirectory(path string, name string) (error) {
func (client *OneDriveClient) CreateDirectory(path string, name string) error {
url := OneDriveAPIURL + "/root/children"

13
src/duplicacy_oneclient_test.go
View File

@@ -5,11 +5,11 @@
package duplicacy
import (
"io"
"fmt"
"testing"
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
"testing"
crypto_rand "crypto/rand"
"math/rand"
@@ -30,7 +30,6 @@ func TestOneDriveClient(t *testing.T) {
fmt.Printf("name: %s, isDir: %t\n", file.Name, len(file.Folder) != 0)
}
testID, _, _, err := oneDriveClient.GetFileInfo("test")
if err != nil {
t.Errorf("Failed to list the test directory: %v", err)
@@ -74,7 +73,7 @@ func TestOneDriveClient(t *testing.T) {
maxFileSize := 64 * 1024
for i := 0; i < numberOfFiles; i++ {
content := make([]byte, rand.Int() % maxFileSize + 1)
content := make([]byte, rand.Int()%maxFileSize+1)
_, err = crypto_rand.Read(content)
if err != nil {
t.Errorf("Error generating random content: %v", err)
@@ -87,7 +86,7 @@ func TestOneDriveClient(t *testing.T) {
fmt.Printf("file: %s\n", filename)
err = oneDriveClient.UploadFile("test/test1/" + filename, content, 100)
err = oneDriveClient.UploadFile("test/test1/"+filename, content, 100)
if err != nil {
/*if e, ok := err.(ACDError); !ok || e.Status != 409 */ {
t.Errorf("Failed to upload the file %s: %v", filename, err)
@@ -103,7 +102,7 @@ func TestOneDriveClient(t *testing.T) {
}
for _, entry := range entries {
err = oneDriveClient.MoveFile("test/test1/" + entry.Name, "test/test2")
err = oneDriveClient.MoveFile("test/test1/"+entry.Name, "test/test2")
if err != nil {
t.Errorf("Failed to move %s: %v", entry.Name, err)
return

42
src/duplicacy_onestorage.go
View File

@@ -21,8 +21,8 @@ type OneDriveStorage struct {
// CreateOneDriveStorage creates an OneDrive storage object.
func CreateOneDriveStorage(tokenFile string, storagePath string, threads int) (storage *OneDriveStorage, err error) {
for len(storagePath) > 0 && storagePath[len(storagePath) - 1] == '/' {
storagePath = storagePath[:len(storagePath) - 1]
for len(storagePath) > 0 && storagePath[len(storagePath)-1] == '/' {
storagePath = storagePath[:len(storagePath)-1]
}
client, err := NewOneDriveClient(tokenFile)
@@ -43,13 +43,13 @@ func CreateOneDriveStorage(tokenFile string, storagePath string, threads int) (s
return nil, fmt.Errorf("Path '%s' is not a directory", storagePath)
}
storage = &OneDriveStorage {
storage = &OneDriveStorage{
client: client,
storageDir: storagePath,
numberOfThread: threads,
}
for _, path := range []string { "chunks", "fossils", "snapshots" } {
for _, path := range []string{"chunks", "fossils", "snapshots"} {
dir := storagePath + "/" + path
dirID, isDir, _, err := client.GetFileInfo(dir)
if err != nil {
@@ -71,8 +71,8 @@ func CreateOneDriveStorage(tokenFile string, storagePath string, threads int) (s
// ListFiles return the list of files and subdirectories under 'dir' (non-recursively)
func (storage *OneDriveStorage) ListFiles(threadIndex int, dir string) ([]string, []int64, error) {
for len(dir) > 0 && dir[len(dir) - 1] == '/' {
dir = dir[:len(dir) - 1]
for len(dir) > 0 && dir[len(dir)-1] == '/' {
dir = dir[:len(dir)-1]
}
if dir == "snapshots" {
@@ -84,7 +84,7 @@ func (storage *OneDriveStorage) ListFiles(threadIndex int, dir string) ([]string
subDirs := []string{}
for _, entry := range entries {
if len(entry.Folder) > 0 {
subDirs = append(subDirs, entry.Name + "/")
subDirs = append(subDirs, entry.Name+"/")
}
}
return subDirs, nil, nil
@@ -105,7 +105,7 @@ func (storage *OneDriveStorage) ListFiles(threadIndex int, dir string) ([]string
} else {
files := []string{}
sizes := []int64{}
for _, parent := range []string {"chunks", "fossils" } {
for _, parent := range []string{"chunks", "fossils"} {
entries, err := storage.client.ListEntries(storage.storageDir + "/" + parent)
if err != nil {
return nil, nil, err
@@ -128,7 +128,7 @@ func (storage *OneDriveStorage) ListFiles(threadIndex int, dir string) ([]string
// DeleteFile deletes the file or directory at 'filePath'.
func (storage *OneDriveStorage) DeleteFile(threadIndex int, filePath string) (err error) {
if strings.HasSuffix(filePath, ".fsl") && strings.HasPrefix(filePath, "chunks/") {
filePath = "fossils/" + filePath[len("chunks/"):len(filePath) - len(".fsl")]
filePath = "fossils/" + filePath[len("chunks/"):len(filePath)-len(".fsl")]
}
err = storage.client.DeleteFile(storage.storageDir + "/" + filePath)
@@ -144,7 +144,7 @@ func (storage *OneDriveStorage) MoveFile(threadIndex int, from string, to string
fromPath := storage.storageDir + "/" + from
toParent := storage.storageDir + "/fossils"
if strings.HasSuffix(from, ".fsl") {
fromPath = storage.storageDir + "/fossils/" + from[len("chunks/"):len(from) - len(".fsl")]
fromPath = storage.storageDir + "/fossils/" + from[len("chunks/"):len(from)-len(".fsl")]
toParent = storage.storageDir + "/chunks"
}
@@ -161,8 +161,8 @@ func (storage *OneDriveStorage) MoveFile(threadIndex int, from string, to string
// CreateDirectory creates a new directory.
func (storage *OneDriveStorage) CreateDirectory(threadIndex int, dir string) (err error) {
for len(dir) > 0 && dir[len(dir) - 1] == '/' {
dir = dir[:len(dir) - 1]
for len(dir) > 0 && dir[len(dir)-1] == '/' {
dir = dir[:len(dir)-1]
}
parent := path.Dir(dir)
@@ -170,15 +170,15 @@ func (storage *OneDriveStorage) CreateDirectory(threadIndex int, dir string) (er
if parent == "." {
return storage.client.CreateDirectory(storage.storageDir, dir)
} else {
return storage.client.CreateDirectory(storage.storageDir + "/" + parent, path.Base(dir))
return storage.client.CreateDirectory(storage.storageDir+"/"+parent, path.Base(dir))
}
}
// GetFileInfo returns the information about the file or directory at 'filePath'.
func (storage *OneDriveStorage) GetFileInfo(threadIndex int, filePath string) (exist bool, isDir bool, size int64, err error) {
for len(filePath) > 0 && filePath[len(filePath) - 1] == '/' {
filePath = filePath[:len(filePath) - 1]
for len(filePath) > 0 && filePath[len(filePath)-1] == '/' {
filePath = filePath[:len(filePath)-1]
}
fileID, isDir, size, err := storage.client.GetFileInfo(storage.storageDir + "/" + filePath)
return fileID != "", isDir, size, err
@@ -207,13 +207,13 @@ func (storage *OneDriveStorage) DownloadFile(threadIndex int, filePath string, c
defer readCloser.Close()
_, err = RateLimitedCopy(chunk, readCloser, storage.DownloadRateLimit / storage.numberOfThread)
_, err = RateLimitedCopy(chunk, readCloser, storage.DownloadRateLimit/storage.numberOfThread)
return err
}
// UploadFile writes 'content' to the file at 'filePath'.
func (storage *OneDriveStorage) UploadFile(threadIndex int, filePath string, content []byte) (err error) {
err = storage.client.UploadFile(storage.storageDir + "/" + filePath, content, storage.UploadRateLimit / storage.numberOfThread)
err = storage.client.UploadFile(storage.storageDir+"/"+filePath, content, storage.UploadRateLimit/storage.numberOfThread)
if e, ok := err.(OneDriveError); ok && e.Status == 409 {
LOG_TRACE("ONEDRIVE_UPLOAD", "File %s already exists", filePath)
@@ -225,16 +225,16 @@ func (storage *OneDriveStorage) UploadFile(threadIndex int, filePath string, con
// If a local snapshot cache is needed for the storage to avoid downloading/uploading chunks too often when
// managing snapshots.
func (storage *OneDriveStorage) IsCacheNeeded() (bool) { return true }
func (storage *OneDriveStorage) IsCacheNeeded() bool { return true }
// If the 'MoveFile' method is implemented.
func (storage *OneDriveStorage) IsMoveFileImplemented() (bool) { return true }
func (storage *OneDriveStorage) IsMoveFileImplemented() bool { return true }
// If the storage can guarantee strong consistency.
func (storage *OneDriveStorage) IsStrongConsistent() (bool) { return false }
func (storage *OneDriveStorage) IsStrongConsistent() bool { return false }
// If the storage supports fast listing of files names.
func (storage *OneDriveStorage) IsFastListing() (bool) { return true }
func (storage *OneDriveStorage) IsFastListing() bool { return true }
// Enable the test mode.
func (storage *OneDriveStorage) EnableTestMode() {

12
src/duplicacy_preference.go
View File

@@ -5,12 +5,12 @@
package duplicacy
import (
"strings"
"encoding/json"
"path"
"io/ioutil"
"reflect"
"os"
"path"
"reflect"
"strings"
)
// Preference stores options for each storage.
@@ -26,7 +26,7 @@ type Preference struct {
}
var preferencePath string
var Preferences [] Preference
var Preferences []Preference
func LoadPreferences(repository string) bool {
@@ -91,7 +91,7 @@ func SetDuplicacyPreferencePath(p string) {
preferencePath = p
}
func SavePreferences() (bool) {
func SavePreferences() bool {
description, err := json.MarshalIndent(Preferences, "", " ")
if err != nil {
LOG_ERROR("PREFERENCE_MARSHAL", "Failed to marshal the repository preferences: %v", err)
@@ -108,7 +108,7 @@ func SavePreferences() (bool) {
return true
}
func FindPreference(name string) (*Preference) {
func FindPreference(name string) *Preference {
for i, preference := range Preferences {
if preference.Name == name || preference.StorageURL == name {
return &Preferences[i]

41
src/duplicacy_s3cstorage.go
View File

@@ -6,6 +6,7 @@ package duplicacy
import (
"time"
"github.com/gilbertchen/goamz/aws"
"github.com/gilbertchen/goamz/s3"
)
@@ -30,10 +31,10 @@ func CreateS3CStorage(regionName string, endpoint string, bucketName string, sto
}
region = aws.Regions[regionName]
} else {
region = aws.Region { Name: regionName, S3Endpoint:"https://" + endpoint }
region = aws.Region{Name: regionName, S3Endpoint: "https://" + endpoint}
}
auth := aws.Auth{ AccessKey: accessKey, SecretKey: secretKey }
auth := aws.Auth{AccessKey: accessKey, SecretKey: secretKey}
var buckets []*s3.Bucket
for i := 0; i < threads; i++ {
@@ -48,11 +49,11 @@ func CreateS3CStorage(regionName string, endpoint string, bucketName string, sto
buckets = append(buckets, bucket)
}
if len(storageDir) > 0 && storageDir[len(storageDir) - 1] != '/' {
if len(storageDir) > 0 && storageDir[len(storageDir)-1] != '/' {
storageDir += "/"
}
storage = &S3CStorage {
storage = &S3CStorage{
buckets: buckets,
storageDir: storageDir,
}
@@ -62,13 +63,13 @@ func CreateS3CStorage(regionName string, endpoint string, bucketName string, sto
// ListFiles return the list of files and subdirectories under 'dir' (non-recursively)
func (storage *S3CStorage) ListFiles(threadIndex int, dir string) (files []string, sizes []int64, err error) {
if len(dir) > 0 && dir[len(dir) - 1] != '/' {
if len(dir) > 0 && dir[len(dir)-1] != '/' {
dir += "/"
}
dirLength := len(storage.storageDir + dir)
if dir == "snapshots/" {
results, err := storage.buckets[threadIndex].List(storage.storageDir + dir, "/", "", 100)
results, err := storage.buckets[threadIndex].List(storage.storageDir+dir, "/", "", 100)
if err != nil {
return nil, nil, err
}
@@ -80,7 +81,7 @@ func (storage *S3CStorage) ListFiles(threadIndex int, dir string) (files []strin
} else if dir == "chunks/" {
marker := ""
for {
results, err := storage.buckets[threadIndex].List(storage.storageDir + dir, "", marker, 1000)
results, err := storage.buckets[threadIndex].List(storage.storageDir+dir, "", marker, 1000)
if err != nil {
return nil, nil, err
}
@@ -94,13 +95,13 @@ func (storage *S3CStorage) ListFiles(threadIndex int, dir string) (files []strin
break
}
marker = results.Contents[len(results.Contents) - 1].Key
marker = results.Contents[len(results.Contents)-1].Key
}
return files, sizes, nil
} else {
results, err := storage.buckets[threadIndex].List(storage.storageDir + dir, "", "", 1000)
results, err := storage.buckets[threadIndex].List(storage.storageDir+dir, "", "", 1000)
if err != nil {
return nil, nil, err
}
@@ -120,8 +121,8 @@ func (storage *S3CStorage) DeleteFile(threadIndex int, filePath string) (err err
// MoveFile renames the file.
func (storage *S3CStorage) MoveFile(threadIndex int, from string, to string) (err error) {
options := s3.CopyOptions { ContentType: "application/duplicacy" }
_, err = storage.buckets[threadIndex].PutCopy(storage.storageDir + to, s3.Private, options, storage.buckets[threadIndex].Name + "/" + storage.storageDir + from)
options := s3.CopyOptions{ContentType: "application/duplicacy"}
_, err = storage.buckets[threadIndex].PutCopy(storage.storageDir+to, s3.Private, options, storage.buckets[threadIndex].Name+"/"+storage.storageDir+from)
if err != nil {
return nil
}
@@ -137,7 +138,7 @@ func (storage *S3CStorage) CreateDirectory(threadIndex int, dir string) (err err
// GetFileInfo returns the information about the file or directory at 'filePath'.
func (storage *S3CStorage) GetFileInfo(threadIndex int, filePath string) (exist bool, isDir bool, size int64, err error) {
response, err := storage.buckets[threadIndex].Head(storage.storageDir + filePath, nil)
response, err := storage.buckets[threadIndex].Head(storage.storageDir+filePath, nil)
if err != nil {
if e, ok := err.(*s3.Error); ok && (e.StatusCode == 403 || e.StatusCode == 404) {
return false, false, 0, nil
@@ -182,7 +183,7 @@ func (storage *S3CStorage) DownloadFile(threadIndex int, filePath string, chunk
defer readCloser.Close()
_, err = RateLimitedCopy(chunk, readCloser, storage.DownloadRateLimit / len(storage.buckets))
_, err = RateLimitedCopy(chunk, readCloser, storage.DownloadRateLimit/len(storage.buckets))
return err
}
@@ -190,23 +191,23 @@ func (storage *S3CStorage) DownloadFile(threadIndex int, filePath string, chunk
// UploadFile writes 'content' to the file at 'filePath'.
func (storage *S3CStorage) UploadFile(threadIndex int, filePath string, content []byte) (err error) {
options := s3.Options { }
reader := CreateRateLimitedReader(content, storage.UploadRateLimit / len(storage.buckets))
return storage.buckets[threadIndex].PutReader(storage.storageDir + filePath, reader, int64(len(content)), "application/duplicacy", s3.Private, options)
options := s3.Options{}
reader := CreateRateLimitedReader(content, storage.UploadRateLimit/len(storage.buckets))
return storage.buckets[threadIndex].PutReader(storage.storageDir+filePath, reader, int64(len(content)), "application/duplicacy", s3.Private, options)
}
// If a local snapshot cache is needed for the storage to avoid downloading/uploading chunks too often when
// managing snapshots.
func (storage *S3CStorage) IsCacheNeeded () (bool) { return true }
func (storage *S3CStorage) IsCacheNeeded() bool { return true }
// If the 'MoveFile' method is implemented.
func (storage *S3CStorage) IsMoveFileImplemented() (bool) { return true }
func (storage *S3CStorage) IsMoveFileImplemented() bool { return true }
// If the storage can guarantee strong consistency.
func (storage *S3CStorage) IsStrongConsistent() (bool) { return false }
func (storage *S3CStorage) IsStrongConsistent() bool { return false }
// If the storage supports fast listing of files names.
func (storage *S3CStorage) IsFastListing() (bool) { return true }
func (storage *S3CStorage) IsFastListing() bool { return true }
// Enable the test mode.
func (storage *S3CStorage) EnableTestMode() {}

40
src/duplicacy_s3storage.go
View File

@@ -5,8 +5,8 @@
package duplicacy
import (
"strings"
"reflect"
"strings"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
@@ -34,7 +34,7 @@ func CreateS3Storage(regionName string, endpoint string, bucketName string, stor
auth := credentials.NewStaticCredentials(accessKey, secretKey, token)
if regionName == "" && endpoint == "" {
defaultRegionConfig := &aws.Config {
defaultRegionConfig := &aws.Config{
Region: aws.String("us-east-1"),
Credentials: auth,
}
@@ -53,7 +53,7 @@ func CreateS3Storage(regionName string, endpoint string, bucketName string, stor
}
}
config := &aws.Config {
config := &aws.Config{
Region: aws.String(regionName),
Credentials: auth,
Endpoint: aws.String(endpoint),
@@ -61,11 +61,11 @@ func CreateS3Storage(regionName string, endpoint string, bucketName string, stor
DisableSSL: aws.Bool(!isSSLSupported),
}
if len(storageDir) > 0 && storageDir[len(storageDir) - 1] != '/' {
if len(storageDir) > 0 && storageDir[len(storageDir)-1] != '/' {
storageDir += "/"
}
storage = &S3Storage {
storage = &S3Storage{
client: s3.New(session.New(config)),
bucket: bucketName,
storageDir: storageDir,
@@ -77,13 +77,13 @@ func CreateS3Storage(regionName string, endpoint string, bucketName string, stor
// ListFiles return the list of files and subdirectories under 'dir' (non-recursively)
func (storage *S3Storage) ListFiles(threadIndex int, dir string) (files []string, sizes []int64, err error) {
if len(dir) > 0 && dir[len(dir) - 1] != '/' {
if len(dir) > 0 && dir[len(dir)-1] != '/' {
dir += "/"
}
if dir == "snapshots/" {
dir = storage.storageDir + dir
input := s3.ListObjectsInput {
input := s3.ListObjectsInput{
Bucket: aws.String(storage.bucket),
Prefix: aws.String(dir),
Delimiter: aws.String("/"),
@@ -103,7 +103,7 @@ func (storage *S3Storage) ListFiles(threadIndex int, dir string) (files []string
dir = storage.storageDir + dir
marker := ""
for {
input := s3.ListObjectsInput {
input := s3.ListObjectsInput{
Bucket: aws.String(storage.bucket),
Prefix: aws.String(dir),
MaxKeys: aws.Int64(1000),
@@ -124,7 +124,7 @@ func (storage *S3Storage) ListFiles(threadIndex int, dir string) (files []string
break
}
marker = *output.Contents[len(output.Contents) - 1].Key
marker = *output.Contents[len(output.Contents)-1].Key
}
return files, sizes, nil
}
@@ -133,7 +133,7 @@ func (storage *S3Storage) ListFiles(threadIndex int, dir string) (files []string
// DeleteFile deletes the file or directory at 'filePath'.
func (storage *S3Storage) DeleteFile(threadIndex int, filePath string) (err error) {
input := &s3.DeleteObjectInput {
input := &s3.DeleteObjectInput{
Bucket: aws.String(storage.bucket),
Key: aws.String(storage.storageDir + filePath),
}
@@ -144,7 +144,7 @@ func (storage *S3Storage) DeleteFile(threadIndex int, filePath string) (err erro
// MoveFile renames the file.
func (storage *S3Storage) MoveFile(threadIndex int, from string, to string) (err error) {
input := &s3.CopyObjectInput {
input := &s3.CopyObjectInput{
Bucket: aws.String(storage.bucket),
CopySource: aws.String(storage.bucket + "/" + storage.storageDir + from),
Key: aws.String(storage.storageDir + to),
@@ -167,7 +167,7 @@ func (storage *S3Storage) CreateDirectory(threadIndex int, dir string) (err erro
// GetFileInfo returns the information about the file or directory at 'filePath'.
func (storage *S3Storage) GetFileInfo(threadIndex int, filePath string) (exist bool, isDir bool, size int64, err error) {
input := &s3.HeadObjectInput {
input := &s3.HeadObjectInput{
Bucket: aws.String(storage.bucket),
Key: aws.String(storage.storageDir + filePath),
}
@@ -210,7 +210,7 @@ func (storage *S3Storage) FindChunk(threadIndex int, chunkID string, isFossil bo
// DownloadFile reads the file at 'filePath' into the chunk.
func (storage *S3Storage) DownloadFile(threadIndex int, filePath string, chunk *Chunk) (err error) {
input := &s3.GetObjectInput {
input := &s3.GetObjectInput{
Bucket: aws.String(storage.bucket),
Key: aws.String(storage.storageDir + filePath),
}
@@ -222,7 +222,7 @@ func (storage *S3Storage) DownloadFile(threadIndex int, filePath string, chunk *
defer output.Body.Close()
_, err = RateLimitedCopy(chunk, output.Body, storage.DownloadRateLimit / len(storage.bucket))
_, err = RateLimitedCopy(chunk, output.Body, storage.DownloadRateLimit/len(storage.bucket))
return err
}
@@ -233,11 +233,11 @@ func (storage *S3Storage) UploadFile(threadIndex int, filePath string, content [
attempts := 0
for {
input := &s3.PutObjectInput {
input := &s3.PutObjectInput{
Bucket: aws.String(storage.bucket),
Key: aws.String(storage.storageDir + filePath),
ACL: aws.String(s3.ObjectCannedACLPrivate),
Body: CreateRateLimitedReader(content, storage.UploadRateLimit / len(storage.bucket)),
Body: CreateRateLimitedReader(content, storage.UploadRateLimit/len(storage.bucket)),
ContentType: aws.String("application/duplicacy"),
}
@@ -255,16 +255,16 @@ func (storage *S3Storage) UploadFile(threadIndex int, filePath string, content [
// If a local snapshot cache is needed for the storage to avoid downloading/uploading chunks too often when
// managing snapshots.
func (storage *S3Storage) IsCacheNeeded () (bool) { return true }
func (storage *S3Storage) IsCacheNeeded() bool { return true }
// If the 'MoveFile' method is implemented.
func (storage *S3Storage) IsMoveFileImplemented() (bool) { return true }
func (storage *S3Storage) IsMoveFileImplemented() bool { return true }
// If the storage can guarantee strong consistency.
func (storage *S3Storage) IsStrongConsistent() (bool) { return false }
func (storage *S3Storage) IsStrongConsistent() bool { return false }
// If the storage supports fast listing of files names.
func (storage *S3Storage) IsFastListing() (bool) { return true }
func (storage *S3Storage) IsFastListing() bool { return true }
// Enable the test mode.
func (storage *S3Storage) EnableTestMode() {}

53
src/duplicacy_sftpstorage.go
View File

@@ -7,15 +7,15 @@ package duplicacy
import (
"fmt"
"io"
"os"
"net"
"path"
"time"
"runtime"
"math/rand"
"net"
"os"
"path"
"runtime"
"time"
"golang.org/x/crypto/ssh"
"github.com/pkg/sftp"
"golang.org/x/crypto/ssh"
)
type SFTPStorage struct {
@@ -29,8 +29,7 @@ type SFTPStorage struct {
func CreateSFTPStorageWithPassword(server string, port int, username string, storageDir string,
password string, threads int) (storage *SFTPStorage, err error) {
authMethods := [] ssh.AuthMethod { ssh.Password(password) }
authMethods := []ssh.AuthMethod{ssh.Password(password)}
hostKeyCallback := func(hostname string, remote net.Addr,
key ssh.PublicKey) error {
@@ -41,7 +40,7 @@ func CreateSFTPStorageWithPassword(server string, port int, username string, sto
}
func CreateSFTPStorage(server string, port int, username string, storageDir string,
authMethods [] ssh.AuthMethod,
authMethods []ssh.AuthMethod,
hostKeyCallback func(hostname string, remote net.Addr,
key ssh.PublicKey) error, threads int) (storage *SFTPStorage, err error) {
@@ -52,7 +51,7 @@ func CreateSFTPStorage(server string, port int, username string, storageDir stri
}
if server == "sftp.hidrive.strato.com" {
config.Ciphers = []string {"aes128-cbc", "aes128-ctr", "aes256-ctr"}
config.Ciphers = []string{"aes128-cbc", "aes128-ctr", "aes256-ctr"}
}
serverAddress := fmt.Sprintf("%s:%d", server, port)
@@ -67,8 +66,8 @@ func CreateSFTPStorage(server string, port int, username string, storageDir stri
return nil, err
}
for storageDir[len(storageDir) - 1] == '/' {
storageDir = storageDir[:len(storageDir) - 1]
for storageDir[len(storageDir)-1] == '/' {
storageDir = storageDir[:len(storageDir)-1]
}
fileInfo, err := client.Stat(storageDir)
@@ -80,7 +79,7 @@ func CreateSFTPStorage(server string, port int, username string, storageDir stri
return nil, fmt.Errorf("The storage path %s is not a directory", storageDir)
}
storage = &SFTPStorage {
storage = &SFTPStorage{
client: client,
storageDir: storageDir,
numberOfThreads: threads,
@@ -108,7 +107,7 @@ func (storage *SFTPStorage) ListFiles(threadIndex int, dirPath string) (files []
for _, entry := range entries {
name := entry.Name()
if entry.IsDir() && name[len(name) - 1] != '/' {
if entry.IsDir() && name[len(name)-1] != '/' {
name += "/"
}
@@ -188,18 +187,18 @@ func (storage *SFTPStorage) FindChunk(threadIndex int, chunkID string, isFossil
// The minimum level of directories to dive into before searching for the chunk file.
minimumLevel := 2
for level := 0; level * 2 < len(chunkID); level ++ {
for level := 0; level*2 < len(chunkID); level++ {
if level >= minimumLevel {
filePath = path.Join(dir, chunkID[2 * level:]) + suffix
filePath = path.Join(dir, chunkID[2*level:]) + suffix
if stat, err := storage.client.Stat(filePath); err == nil && !stat.IsDir() {
return filePath[len(storage.storageDir) + 1:], true, stat.Size(), nil
return filePath[len(storage.storageDir)+1:], true, stat.Size(), nil
} else if err == nil && stat.IsDir() {
return filePath[len(storage.storageDir) + 1:], true, 0, fmt.Errorf("The path %s is a directory", filePath)
return filePath[len(storage.storageDir)+1:], true, 0, fmt.Errorf("The path %s is a directory", filePath)
}
}
// Find the subdirectory the chunk file may reside.
subDir := path.Join(dir, chunkID[2 * level: 2 * level + 2])
subDir := path.Join(dir, chunkID[2*level:2*level+2])
stat, err := storage.client.Stat(subDir)
if err == nil && stat.IsDir() {
dir = subDir
@@ -227,7 +226,7 @@ func (storage *SFTPStorage) FindChunk(threadIndex int, chunkID string, isFossil
}
// Teh chunk must be under this subdirectory but it doesn't exist.
return path.Join(dir, chunkID[2 * level:])[len(storage.storageDir) + 1:] + suffix, false, 0, nil
return path.Join(dir, chunkID[2*level:])[len(storage.storageDir)+1:] + suffix, false, 0, nil
}
@@ -245,7 +244,7 @@ func (storage *SFTPStorage) DownloadFile(threadIndex int, filePath string, chunk
}
defer file.Close()
if _, err = RateLimitedCopy(chunk, file, storage.DownloadRateLimit / storage.numberOfThreads); err != nil {
if _, err = RateLimitedCopy(chunk, file, storage.DownloadRateLimit/storage.numberOfThreads); err != nil {
return err
}
@@ -265,12 +264,12 @@ func (storage *SFTPStorage) UploadFile(threadIndex int, filePath string, content
temporaryFile := fullPath + "." + string(suffix) + ".tmp"
file, err := storage.client.OpenFile(temporaryFile, os.O_WRONLY | os.O_CREATE | os.O_TRUNC)
file, err := storage.client.OpenFile(temporaryFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC)
if err != nil {
return err
}
reader := CreateRateLimitedReader(content, storage.UploadRateLimit / storage.numberOfThreads)
reader := CreateRateLimitedReader(content, storage.UploadRateLimit/storage.numberOfThreads)
_, err = io.Copy(file, reader)
if err != nil {
file.Close()
@@ -294,16 +293,16 @@ func (storage *SFTPStorage) UploadFile(threadIndex int, filePath string, content
// If a local snapshot cache is needed for the storage to avoid downloading/uploading chunks too often when
// managing snapshots.
func (storage *SFTPStorage) IsCacheNeeded () (bool) { return true }
func (storage *SFTPStorage) IsCacheNeeded() bool { return true }
// If the 'MoveFile' method is implemented.
func (storage *SFTPStorage) IsMoveFileImplemented() (bool) { return true }
func (storage *SFTPStorage) IsMoveFileImplemented() bool { return true }
// If the storage can guarantee strong consistency.
func (storage *SFTPStorage) IsStrongConsistent() (bool) { return true }
func (storage *SFTPStorage) IsStrongConsistent() bool { return true }
// If the storage supports fast listing of files names.
func (storage *SFTPStorage) IsFastListing() (bool) { return false }
func (storage *SFTPStorage) IsFastListing() bool { return false }
// Enable the test mode.
func (storage *SFTPStorage) EnableTestMode() {}

26
src/duplicacy_shadowcopy_windows.go
View File

@@ -5,11 +5,11 @@
package duplicacy
import (
"syscall"
"unsafe"
"time"
"os"
"runtime"
"syscall"
"time"
"unsafe"
ole "github.com/gilbertchen/go-ole"
)
@@ -28,7 +28,7 @@ type IVSSAsyncVtbl struct {
queryStatus uintptr
}
func (async *IVSSAsync) VTable() * IVSSAsyncVtbl {
func (async *IVSSAsync) VTable() *IVSSAsyncVtbl {
return (*IVSSAsyncVtbl)(unsafe.Pointer(async.RawVTable))
}
@@ -55,7 +55,7 @@ func (async *IVSSAsync) Wait(seconds int) bool {
if status == VSS_S_ASYNC_FINISHED {
return true
}
if time.Now().Unix() - startTime > int64(seconds) {
if time.Now().Unix()-startTime > int64(seconds) {
LOG_WARN("IVSSASYNC_TIMEOUT", "IVssAsync is pending for more than %d seconds\n", seconds)
return false
}
@@ -77,7 +77,6 @@ func getIVSSAsync(unknown *ole.IUnknown, iid *ole.GUID) (async *IVSSAsync) {
return
}
//665c1d5f-c218-414d-a05d-7fef5f9d5c86
var IID_IVSS = &ole.GUID{0x665c1d5f, 0xc218, 0x414d, [8]byte{0xa0, 0x5d, 0x7f, 0xef, 0x5f, 0x9d, 0x5c, 0x86}}
@@ -137,7 +136,7 @@ type IVSSVtbl struct {
queryRevertStatus uintptr
}
func (vss *IVSS) VTable() * IVSSVtbl {
func (vss *IVSS) VTable() *IVSSVtbl {
return (*IVSSVtbl)(unsafe.Pointer(vss.RawVTable))
}
@@ -238,7 +237,7 @@ type SnapshotProperties struct {
Status int
}
func (vss *IVSS) GetSnapshotProperties(snapshotSetID ole.GUID, properties *SnapshotProperties) (int) {
func (vss *IVSS) GetSnapshotProperties(snapshotSetID ole.GUID, properties *SnapshotProperties) int {
var ret uintptr
if runtime.GOARCH == "386" {
address := uint(uintptr(unsafe.Pointer(&snapshotSetID)))
@@ -292,8 +291,7 @@ func (vss *IVSS) DeleteSnapshots(snapshotID ole.GUID) (int, int, ole.GUID) {
return int(ret), int(deleted), deletedGUID
}
func uint16ArrayToString(p *uint16) (string) {
func uint16ArrayToString(p *uint16) string {
if p == nil {
return ""
}
@@ -481,9 +479,7 @@ func CreateShadowCopy(top string, shadowCopy bool) (shadowTop string) {
}
async.Release()
properties := SnapshotProperties {
}
properties := SnapshotProperties{}
ret = vssBackupComponent.GetSnapshotProperties(snapshotID, &properties)
if ret != 0 {
@@ -512,7 +508,7 @@ func CreateShadowCopy(top string, shadowCopy bool) (shadowTop string) {
preferencePath := GetDuplicacyPreferencePath()
shadowLink = preferencePath + "\\shadow"
os.Remove(shadowLink)
err = os.Symlink(snapshotPath + "\\", shadowLink)
err = os.Symlink(snapshotPath+"\\", shadowLink)
if err != nil {
LOG_ERROR("VSS_SYMLINK", "Failed to create a symbolic link to the shadow copy just created: %v", err)
return top
@@ -521,5 +517,3 @@ func CreateShadowCopy(top string, shadowCopy bool) (shadowTop string) {
return shadowLink + "\\" + top[2:]
}

56
src/duplicacy_snapshot.go
View File

@@ -5,15 +5,15 @@
package duplicacy
import (
"os"
"fmt"
"time"
"path"
"strings"
"strconv"
"io/ioutil"
"encoding/json"
"encoding/hex"
"encoding/json"
"fmt"
"io/ioutil"
"os"
"path"
"strconv"
"strings"
"time"
)
// Snapshot represents a backup of the repository.
@@ -47,10 +47,10 @@ type Snapshot struct {
}
// CreateEmptySnapshot creates an empty snapshot.
func CreateEmptySnapshot (id string) (snapshto *Snapshot) {
func CreateEmptySnapshot(id string) (snapshto *Snapshot) {
return &Snapshot{
ID : id,
Revision : 0,
ID: id,
Revision: 0,
StartTime: time.Now().Unix(),
}
}
@@ -60,8 +60,8 @@ func CreateEmptySnapshot (id string) (snapshto *Snapshot) {
func CreateSnapshotFromDirectory(id string, top string) (snapshot *Snapshot, skippedDirectories []string,
skippedFiles []string, err error) {
snapshot = &Snapshot {
ID : id,
snapshot = &Snapshot{
ID: id,
Revision: 0,
StartTime: time.Now().Unix(),
}
@@ -122,8 +122,8 @@ func CreateSnapshotFromDirectory(id string, top string) (snapshot *Snapshot, ski
for len(directories) > 0 {
directory := directories[len(directories) - 1]
directories = directories[:len(directories) - 1]
directory := directories[len(directories)-1]
directories = directories[:len(directories)-1]
snapshot.Files = append(snapshot.Files, directory)
subdirectories, skipped, err := ListEntries(top, directory.Path, &snapshot.Files, patterns, snapshot.discardAttributes)
if err != nil {
@@ -152,9 +152,9 @@ func CreateSnapshotFromDirectory(id string, top string) (snapshot *Snapshot, ski
// This is the struct used to save/load incomplete snapshots
type IncompleteSnapshot struct {
Files [] *Entry
Files []*Entry
ChunkHashes []string
ChunkLengths [] int
ChunkLengths []int
}
// LoadIncompleteSnapshot loads the incomplete snapshot if it exists
@@ -184,7 +184,7 @@ func LoadIncompleteSnapshot() (snapshot *Snapshot) {
chunkHashes = append(chunkHashes, string(hash))
}
snapshot = &Snapshot {
snapshot = &Snapshot{
Files: incompleteSnapshot.Files,
ChunkHashes: chunkHashes,
ChunkLengths: incompleteSnapshot.ChunkLengths,
@@ -210,7 +210,7 @@ func SaveIncompleteSnapshot(snapshot *Snapshot) {
chunkHashes = append(chunkHashes, hex.EncodeToString([]byte(chunkHash)))
}
incompleteSnapshot := IncompleteSnapshot {
incompleteSnapshot := IncompleteSnapshot{
Files: files,
ChunkHashes: chunkHashes,
ChunkLengths: snapshot.ChunkLengths,
@@ -247,14 +247,14 @@ func RemoveIncompleteSnapshot() {
// CreateSnapshotFromDescription creates a snapshot from json decription.
func CreateSnapshotFromDescription(description []byte) (snapshot *Snapshot, err error) {
var root map[string] interface{}
var root map[string]interface{}
err = json.Unmarshal(description, &root)
if err != nil {
return nil, err
}
snapshot = &Snapshot {}
snapshot = &Snapshot{}
if value, ok := root["id"]; !ok {
return nil, fmt.Errorf("No id is specified in the snapshot")
@@ -308,7 +308,7 @@ func CreateSnapshotFromDescription(description []byte) (snapshot *Snapshot, err
}
}
for _, sequenceType := range []string { "files", "chunks", "lengths" } {
for _, sequenceType := range []string{"files", "chunks", "lengths"} {
if value, ok := root[sequenceType]; !ok {
return nil, fmt.Errorf("No %s are specified in the snapshot", sequenceType)
} else if _, ok = value.([]interface{}); !ok {
@@ -336,7 +336,7 @@ func CreateSnapshotFromDescription(description []byte) (snapshot *Snapshot, err
// LoadChunks construct 'ChunkHashes' from the json description.
func (snapshot *Snapshot) LoadChunks(description []byte) (err error) {
var root [] interface {}
var root []interface{}
err = json.Unmarshal(description, &root)
if err != nil {
return err
@@ -363,7 +363,7 @@ func (snapshot *Snapshot) LoadLengths(description []byte) (err error) {
}
// MarshalJSON creates a json representation of the snapshot.
func (snapshot *Snapshot) MarshalJSON() ([] byte, error) {
func (snapshot *Snapshot) MarshalJSON() ([]byte, error) {
object := make(map[string]interface{})
@@ -386,7 +386,7 @@ func (snapshot *Snapshot) MarshalJSON() ([] byte, error) {
}
// MarshalSequence creates a json represetion for the specified chunk sequence.
func (snapshot *Snapshot) MarshalSequence(sequenceType string) ([] byte, error) {
func (snapshot *Snapshot) MarshalSequence(sequenceType string) ([]byte, error) {
if sequenceType == "files" {
return json.Marshal(snapshot.Files)
@@ -398,7 +398,7 @@ func (snapshot *Snapshot) MarshalSequence(sequenceType string) ([] byte, error)
}
// SetSequence assign a chunk sequence to the specified field.
func (snapshot *Snapshot) SetSequence(sequenceType string, sequence [] string) {
func (snapshot *Snapshot) SetSequence(sequenceType string, sequence []string) {
if sequenceType == "files" {
snapshot.FileSequence = sequence
} else if sequenceType == "chunks" {
@@ -409,7 +409,7 @@ func (snapshot *Snapshot) SetSequence(sequenceType string, sequence [] string) {
}
// encodeSequence turns a sequence of binary hashes into a sequence of hex hashes.
func encodeSequence(sequence[] string) ([] string) {
func encodeSequence(sequence []string) []string {
sequenceInHex := make([]string, len(sequence))
@@ -419,5 +419,3 @@ func encodeSequence(sequence[] string) ([] string) {
return sequenceInHex
}

184
src/duplicacy_snapshotmanager.go
View File

@@ -5,21 +5,21 @@
package duplicacy
import (
"io"
"os"
"fmt"
"text/tabwriter"
"sort"
"bytes"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"math"
"os"
"path"
"regexp"
"sort"
"strconv"
"strings"
"text/tabwriter"
"time"
"math"
"path"
"io/ioutil"
"encoding/json"
"encoding/hex"
"github.com/aryann/difflib"
)
@@ -35,7 +35,7 @@ type FossilCollection struct {
EndTime int64 `json:"end_time"`
// The lastest revision for each snapshot id when the fossil collection was created.
LastRevisions map[string] int `json:"last_revisions"`
LastRevisions map[string]int `json:"last_revisions"`
// Fossils (i.e., chunks not referenced by any snapshots)
Fossils []string `json:"fossils"`
@@ -45,23 +45,23 @@ type FossilCollection struct {
}
// CreateFossilCollection creates an empty fossil collection
func CreateFossilCollection(allSnapshots map[string][] *Snapshot) *FossilCollection{
func CreateFossilCollection(allSnapshots map[string][]*Snapshot) *FossilCollection {
lastRevisions := make(map[string] int)
lastRevisions := make(map[string]int)
for id, snapshots := range allSnapshots {
lastRevisions[id] = snapshots[len(snapshots) - 1].Revision
lastRevisions[id] = snapshots[len(snapshots)-1].Revision
}
return &FossilCollection {
LastRevisions : lastRevisions,
return &FossilCollection{
LastRevisions: lastRevisions,
}
}
// IsDeletable determines if the previously collected fossils are safe to be permanently removed. If so, it will
// also returns a number of snapshots that were created during or after these fossils were being collected.
// Therefore, some fossils may be referenced by these new snapshots and they must be resurrected.
func (collection *FossilCollection) IsDeletable(isStrongConsistent bool, ignoredIDs [] string,
allSnapshots map[string][] *Snapshot) (isDeletable bool, newSnapshots []*Snapshot) {
func (collection *FossilCollection) IsDeletable(isStrongConsistent bool, ignoredIDs []string,
allSnapshots map[string][]*Snapshot) (isDeletable bool, newSnapshots []*Snapshot) {
hasNewSnapshot := make(map[string]bool)
lastSnapshotTime := make(map[string]int64)
@@ -111,7 +111,7 @@ func (collection *FossilCollection) IsDeletable(isStrongConsistent bool, ignored
// If this snapshot ends before this fossil collection, then it is still possible that another snapshot
// might be in progress (although very unlikely). So we only deem it deletable if that is not the case.
if snapshot.EndTime > collection.EndTime + int64(extraTime){
if snapshot.EndTime > collection.EndTime+int64(extraTime) {
hasNewSnapshot[hostID] = true
newSnapshots = append(newSnapshots, snapshot)
break
@@ -126,7 +126,7 @@ func (collection *FossilCollection) IsDeletable(isStrongConsistent bool, ignored
LOG_TRACE("SNAPSHOT_NO_NEW", "No new snapshot from %s since the fossil collection step", snapshotID)
}
lastSnapshot := allSnapshots[snapshotID][len(allSnapshots[snapshotID]) - 1]
lastSnapshot := allSnapshots[snapshotID][len(allSnapshots[snapshotID])-1]
if lastSnapshot.EndTime > lastSnapshotTime[hostID] {
lastSnapshotTime[hostID] = lastSnapshot.EndTime
}
@@ -140,7 +140,7 @@ func (collection *FossilCollection) IsDeletable(isStrongConsistent bool, ignored
// snapshot id during the last 7 days. A snapshot created at the roughly same time as this fossil
// collection would have finsihed already, while a snapshot currently being created does not affect
// this fossil collection.
if lastSnapshotTime[hostID] > 0 && lastSnapshotTime[hostID] < time.Now().Unix() - maxSnapshotRunningTime * secondsInDay {
if lastSnapshotTime[hostID] > 0 && lastSnapshotTime[hostID] < time.Now().Unix()-maxSnapshotRunningTime*secondsInDay {
LOG_INFO("SNAPSHOT_INACTIVE", "Ignore snapshot %s whose last revision was created %d days ago",
hostID, maxSnapshotRunningTime)
continue
@@ -174,13 +174,12 @@ type SnapshotManager struct {
snapshotCache *FileStorage
chunkDownloader *ChunkDownloader
}
// CreateSnapshotManager creates a snapshot manager
func CreateSnapshotManager(config *Config, storage Storage) *SnapshotManager {
manager := &SnapshotManager {
manager := &SnapshotManager{
config: config,
storage: storage,
fileChunk: CreateChunk(config, true),
@@ -231,14 +230,14 @@ func (manager *SnapshotManager) DownloadSnapshot(snapshotID string, revision int
// sequenceReader loads the chunks pointed to by 'sequence' one by one as needed. This avoid loading all chunks into
// the memory before passing them to the json unmarshaller.
type sequenceReader struct {
sequence [] string
sequence []string
buffer *bytes.Buffer
index int
refillFunc func(hash string) ([]byte)
refillFunc func(hash string) []byte
}
// Read reads a new chunk using the refill function when there is no more data in the buffer
func (reader *sequenceReader)Read(data []byte) (n int, err error) {
func (reader *sequenceReader) Read(data []byte) (n int, err error) {
if len(reader.buffer.Bytes()) == 0 {
if reader.index < len(reader.sequence) {
newData := reader.refillFunc(reader.sequence[reader.index])
@@ -274,17 +273,17 @@ func (manager *SnapshotManager) DownloadSnapshotFileSequence(snapshot *Snapshot,
manager.CreateChunkDownloader()
reader := sequenceReader {
reader := sequenceReader{
sequence: snapshot.FileSequence,
buffer: new(bytes.Buffer),
refillFunc: func (chunkHash string) ([]byte) {
refillFunc: func(chunkHash string) []byte {
i := manager.chunkDownloader.AddChunk(chunkHash)
chunk := manager.chunkDownloader.WaitForChunk(i)
return chunk.GetBytes()
},
}
files := make([] *Entry, 0)
files := make([]*Entry, 0)
decoder := json.NewDecoder(&reader)
// read open bracket
@@ -315,7 +314,6 @@ func (manager *SnapshotManager) DownloadSnapshotFileSequence(snapshot *Snapshot,
return true
}
// DownloadSnapshotSequence downloads the content represented by a sequence of chunks, and then unmarshal the content
// using the specified 'loadFunction'. It purpose is to decode the chunk sequences representing chunk hashes or chunk lengths
// in a snapshot.
@@ -331,7 +329,6 @@ func (manager *SnapshotManager) DownloadSnapshotSequence(snapshot *Snapshot, seq
content := manager.DownloadSequence(sequence)
if len(content) == 0 {
LOG_ERROR("SNAPSHOT_PARSE", "Failed to load %s specified in the snapshot %s at revision %d",
sequenceType, snapshot.ID, snapshot.Revision)
@@ -367,7 +364,7 @@ func (manager *SnapshotManager) DownloadSnapshotContents(snapshot *Snapshot, pat
}
// CleanSnapshotCache removes all files not referenced by the specified 'snapshot' in the snapshot cache.
func (manager *SnapshotManager) CleanSnapshotCache(latestSnapshot *Snapshot, allSnapshots map[string] []*Snapshot) bool {
func (manager *SnapshotManager) CleanSnapshotCache(latestSnapshot *Snapshot, allSnapshots map[string][]*Snapshot) bool {
if allSnapshots == nil {
// If the 'fossils' directory exists then don't clean the cache as all snapshots will be needed later
@@ -409,7 +406,7 @@ func (manager *SnapshotManager) CleanSnapshotCache(latestSnapshot *Snapshot, all
allCachedSnapshots, _ := manager.ListAllFiles(manager.snapshotCache, "snapshots/")
for _, snapshotFile := range allCachedSnapshots {
if snapshotFile[len(snapshotFile) - 1] == '/' {
if snapshotFile[len(snapshotFile)-1] == '/' {
continue
}
@@ -460,7 +457,7 @@ func (manager *SnapshotManager) CleanSnapshotCache(latestSnapshot *Snapshot, all
allFiles, _ := manager.ListAllFiles(manager.snapshotCache, "chunks/")
for _, file := range allFiles {
if file[len(file) - 1] != '/' {
if file[len(file)-1] != '/' {
chunkID := strings.Replace(file, "/", "", -1)
if _, found := chunks[chunkID]; !found {
LOG_DEBUG("SNAPSHOT_CLEAN", "Delete chunk %s from the snapshot cache", chunkID)
@@ -478,7 +475,7 @@ func (manager *SnapshotManager) CleanSnapshotCache(latestSnapshot *Snapshot, all
}
// ListSnapshotIDs returns all snapshot ids.
func (manager *SnapshotManager) ListSnapshotIDs() (snapshotIDs [] string, err error) {
func (manager *SnapshotManager) ListSnapshotIDs() (snapshotIDs []string, err error) {
LOG_TRACE("SNAPSHOT_LIST_IDS", "Listing all snapshot ids")
@@ -488,8 +485,8 @@ func (manager *SnapshotManager) ListSnapshotIDs() (snapshotIDs [] string, err er
}
for _, dir := range dirs {
if len(dir) > 0 && dir[len(dir) - 1] == '/' {
snapshotIDs = append(snapshotIDs, dir[:len(dir) - 1])
if len(dir) > 0 && dir[len(dir)-1] == '/' {
snapshotIDs = append(snapshotIDs, dir[:len(dir)-1])
}
}
@@ -497,7 +494,7 @@ func (manager *SnapshotManager) ListSnapshotIDs() (snapshotIDs [] string, err er
}
// ListSnapshotRevisions returns the list of all revisions given a snapshot id.
func (manager *SnapshotManager) ListSnapshotRevisions(snapshotID string) (revisions [] int, err error) {
func (manager *SnapshotManager) ListSnapshotRevisions(snapshotID string) (revisions []int, err error) {
LOG_TRACE("SNAPSHOT_LIST_REVISIONS", "Listing revisions for snapshot %s", snapshotID)
@@ -519,7 +516,7 @@ func (manager *SnapshotManager) ListSnapshotRevisions(snapshotID string) (revisi
}
for _, file := range files {
if len(file) > 0 && file[len(file) - 1] != '/' {
if len(file) > 0 && file[len(file)-1] != '/' {
revision, err := strconv.Atoi(file)
if err == nil {
revisions = append(revisions, revision)
@@ -571,8 +568,8 @@ func (manager *SnapshotManager) ListAllFiles(storage Storage, top string) (allFi
for len(directories) > 0 {
dir := directories[len(directories) - 1]
directories = directories[:len(directories) - 1]
dir := directories[len(directories)-1]
directories = directories[:len(directories)-1]
LOG_TRACE("LIST_FILES", "Listing %s", dir)
@@ -588,8 +585,8 @@ func (manager *SnapshotManager) ListAllFiles(storage Storage, top string) (allFi
}
for i, file := range files {
if len(file) > 0 && file[len(file) - 1] == '/' {
directories = append(directories, dir + file)
if len(file) > 0 && file[len(file)-1] == '/' {
directories = append(directories, dir+file)
} else {
allFiles = append(allFiles, (dir + file)[len(top):])
allSizes = append(allSizes, sizes[i])
@@ -619,7 +616,7 @@ func (manager *SnapshotManager) ListAllFiles(storage Storage, top string) (allFi
}
// GetSnapshotChunks returns all chunks referenced by a given snapshot.
func (manager *SnapshotManager) GetSnapshotChunks(snapshot *Snapshot) (chunks [] string) {
func (manager *SnapshotManager) GetSnapshotChunks(snapshot *Snapshot) (chunks []string) {
for _, chunkHash := range snapshot.FileSequence {
chunks = append(chunks, manager.config.GetChunkIDFromHash(chunkHash))
@@ -658,7 +655,7 @@ func (manager *SnapshotManager) ListSnapshots(snapshotID string, revisionsToList
LOG_DEBUG("LIST_PARAMETERS", "id: %s, revisions: %v, tag: %s, showFiles: %t, showChunks: %t",
snapshotID, revisionsToList, tag, showFiles, showChunks)
var snapshotIDs [] string
var snapshotIDs []string
var err error
if snapshotID == "" {
@@ -668,7 +665,7 @@ func (manager *SnapshotManager) ListSnapshots(snapshotID string, revisionsToList
return 0
}
} else {
snapshotIDs = []string { snapshotID }
snapshotIDs = []string{snapshotID}
}
numberOfSnapshots := 0
@@ -715,7 +712,7 @@ func (manager *SnapshotManager) ListSnapshots(snapshotID string, revisionsToList
totalFiles++
totalFileSize += file.Size
if file.Size > maxSize {
maxSize = maxSize * 10 + 9
maxSize = maxSize*10 + 9
maxSizeDigits += 1
}
if file.EndChunk > lastChunk {
@@ -732,7 +729,7 @@ func (manager *SnapshotManager) ListSnapshots(snapshotID string, revisionsToList
metaChunks := len(snapshot.FileSequence) + len(snapshot.ChunkSequence) + len(snapshot.LengthSequence)
LOG_INFO("SNAPSHOT_STATS", "Files: %d, total size: %d, file chunks: %d, metadata chunks: %d",
totalFiles, totalFileSize, lastChunk + 1, metaChunks)
totalFiles, totalFileSize, lastChunk+1, metaChunks)
}
if showChunks {
@@ -756,7 +753,7 @@ func (manager *SnapshotManager) CheckSnapshots(snapshotID string, revisionsToChe
LOG_DEBUG("LIST_PARAMETERS", "id: %s, revisions: %v, tag: %s, showStatistics: %t, checkFiles: %t, searchFossils: %t, resurrect: %t",
snapshotID, revisionsToCheck, tag, showStatistics, checkFiles, searchFossils, resurrect)
snapshotMap := make(map[string] [] *Snapshot)
snapshotMap := make(map[string][]*Snapshot)
var err error
// Stores the chunk file size for each chunk
@@ -772,7 +769,7 @@ func (manager *SnapshotManager) CheckSnapshots(snapshotID string, revisionsToChe
allChunks, allSizes := manager.ListAllFiles(manager.storage, "chunks/")
for i, chunk := range allChunks {
if len(chunk) == 0 || chunk[len(chunk) - 1] == '/'{
if len(chunk) == 0 || chunk[len(chunk)-1] == '/' {
continue
}
@@ -799,7 +796,6 @@ func (manager *SnapshotManager) CheckSnapshots(snapshotID string, revisionsToChe
snapshotMap[snapshotID] = nil
}
snapshotIDIndex := 0
for snapshotID, _ = range snapshotMap {
@@ -863,7 +859,7 @@ func (manager *SnapshotManager) CheckSnapshots(snapshotID string, revisionsToChe
if resurrect {
manager.resurrectChunk(chunkPath, chunkID)
} else {
LOG_WARN("SNAPHOST_FOSSIL", "Chunk %s referenced by snapshot %s at revision %d " +
LOG_WARN("SNAPHOST_FOSSIL", "Chunk %s referenced by snapshot %s at revision %d "+
"has been marked as a fossil", chunkID, snapshotID, revision)
}
@@ -908,7 +904,7 @@ func (manager *SnapshotManager) CheckSnapshots(snapshotID string, revisionsToChe
}
// Print snapshot and revision statistics
func (manager *SnapshotManager) ShowStatistics(snapshotMap map[string] [] *Snapshot, chunkSizeMap map[string]int64, chunkUniqueMap map[string]bool,
func (manager *SnapshotManager) ShowStatistics(snapshotMap map[string][]*Snapshot, chunkSizeMap map[string]int64, chunkUniqueMap map[string]bool,
chunkSnapshotMap map[string]int) {
for snapshotID, snapshotList := range snapshotMap {
@@ -957,7 +953,7 @@ func (manager *SnapshotManager) ShowStatistics(snapshotMap map[string] [] *Snaps
}
// Print snapshot and revision statistics in tabular format
func (manager *SnapshotManager) ShowStatisticsTabular(snapshotMap map[string] [] *Snapshot, chunkSizeMap map[string]int64, chunkUniqueMap map[string]bool,
func (manager *SnapshotManager) ShowStatisticsTabular(snapshotMap map[string][]*Snapshot, chunkSizeMap map[string]int64, chunkUniqueMap map[string]bool,
chunkSnapshotMap map[string]int) {
tableBuffer := new(bytes.Buffer)
tableWriter := tabwriter.NewWriter(tableBuffer, 0, 0, 1, ' ', tabwriter.AlignRight|tabwriter.Debug)
@@ -1099,7 +1095,7 @@ func (manager *SnapshotManager) VerifySnapshot(snapshot *Snapshot) bool {
return false
}
files := make([]*Entry, 0, len(snapshot.Files) / 2)
files := make([]*Entry, 0, len(snapshot.Files)/2)
for _, file := range snapshot.Files {
if file.IsFile() && file.Size != 0 {
files = append(files, file)
@@ -1109,7 +1105,7 @@ func (manager *SnapshotManager) VerifySnapshot(snapshot *Snapshot) bool {
sort.Sort(ByChunk(files))
corruptedFiles := 0
for _, file := range files {
if !manager.RetrieveFile(snapshot, file, func([]byte) {} ) {
if !manager.RetrieveFile(snapshot, file, func([]byte) {}) {
corruptedFiles++
}
LOG_TRACE("SNAPSHOT_VERIFY", "%s", file.Path)
@@ -1127,7 +1123,7 @@ func (manager *SnapshotManager) VerifySnapshot(snapshot *Snapshot) bool {
}
// RetrieveFile retrieve the file in the specifed snapshot.
func (manager *SnapshotManager) RetrieveFile(snapshot *Snapshot, file *Entry, output func([]byte)()) bool {
func (manager *SnapshotManager) RetrieveFile(snapshot *Snapshot, file *Entry, output func([]byte)) bool {
if file.Size == 0 {
return true
@@ -1181,7 +1177,7 @@ func (manager *SnapshotManager) RetrieveFile(snapshot *Snapshot, file *Entry, ou
}
// FindFile returns the file entry that has the given file name.
func (manager *SnapshotManager) FindFile(snapshot *Snapshot, filePath string, suppressError bool) (*Entry) {
func (manager *SnapshotManager) FindFile(snapshot *Snapshot, filePath string, suppressError bool) *Entry {
for _, entry := range snapshot.Files {
if entry.Path == filePath {
return entry
@@ -1232,7 +1228,7 @@ func (manager *SnapshotManager) PrintFile(snapshotID string, revision int, path
}
file := manager.FindFile(snapshot, path, false)
var content [] byte
var content []byte
if !manager.RetrieveFile(snapshot, file, func(chunk []byte) { content = append(content, chunk...) }) {
LOG_ERROR("SNAPSHOT_RETRIEVE", "File %s is corrupted in snapshot %s at revision %d",
path, snapshot.ID, snapshot.Revision)
@@ -1280,7 +1276,6 @@ func (manager *SnapshotManager) Diff(top string, snapshotID string, revisions []
leftSnapshot = manager.DownloadSnapshot(snapshotID, revisions[0])
}
if len(filePath) > 0 {
manager.DownloadSnapshotContents(leftSnapshot, nil)
@@ -1320,7 +1315,7 @@ func (manager *SnapshotManager) Diff(top string, snapshotID string, revisions []
after := 10
before := 10
var buffer [] string
var buffer []string
on := false
distance := 0
@@ -1372,19 +1367,19 @@ func (manager *SnapshotManager) Diff(top string, snapshotID string, revisions []
// Find the max Size value in order for pretty alignment.
for _, file := range leftSnapshot.Files {
for !file.IsDir() && file.Size > maxSize {
maxSize = maxSize * 10 + 9
maxSize = maxSize*10 + 9
maxSizeDigits += 1
}
}
for _, file := range rightSnapshot.Files {
for !file.IsDir() && file.Size > maxSize {
maxSize = maxSize * 10 + 9
maxSize = maxSize*10 + 9
maxSizeDigits += 1
}
}
buffer := make([]byte, 32 * 1024)
buffer := make([]byte, 32*1024)
var i, j int
for i < len(leftSnapshot.Files) || j < len(rightSnapshot.Files) {
@@ -1486,7 +1481,6 @@ func (manager *SnapshotManager) ShowHistory(top string, snapshotID string, revis
LOG_INFO("SNAPSHOT_HISTORY", "%7d:", revision)
}
}
stat, err := os.Stat(joinPath(top, filePath))
@@ -1497,7 +1491,7 @@ func (manager *SnapshotManager) ShowHistory(top string, snapshotID string, revis
modifiedFlag = "*"
}
if showLocalHash {
localFile.Hash = manager.config.ComputeFileHash(joinPath(top, filePath), make([]byte, 32 * 1024))
localFile.Hash = manager.config.ComputeFileHash(joinPath(top, filePath), make([]byte, 32*1024))
if lastVersion.Hash != localFile.Hash {
modifiedFlag = "*"
}
@@ -1512,7 +1506,7 @@ func (manager *SnapshotManager) ShowHistory(top string, snapshotID string, revis
// fossilizeChunk turns the chunk into a fossil.
func (manager *SnapshotManager) fossilizeChunk(chunkID string, filePath string,
exclusive bool, collection *FossilCollection) (bool) {
exclusive bool, collection *FossilCollection) bool {
if exclusive {
err := manager.storage.DeleteFile(0, filePath)
if err != nil {
@@ -1548,7 +1542,7 @@ func (manager *SnapshotManager) fossilizeChunk(chunkID string, filePath string,
}
// resurrectChunk turns the fossil back into a chunk
func (manager *SnapshotManager) resurrectChunk(fossilPath string, chunkID string) (bool) {
func (manager *SnapshotManager) resurrectChunk(fossilPath string, chunkID string) bool {
chunkPath, exist, _, err := manager.storage.FindChunk(0, chunkID, false)
if err != nil {
LOG_ERROR("CHUNK_FIND", "Failed to locate the path for the chunk %s: %v", chunkID, err)
@@ -1571,8 +1565,6 @@ func (manager *SnapshotManager) resurrectChunk(fossilPath string, chunkID string
return true
}
// PruneSnapshots deletes snapshots by revisions, tags, or a retention policy. The main idea is two-step
// fossil collection.
// 1. Delete snapshots specified by revision, retention policy, with a tag. Find any resulting unreferenced
@@ -1594,7 +1586,7 @@ func (manager *SnapshotManager) PruneSnapshots(selfID string, snapshotID string,
dryRun bool, deleteOnly bool, collectOnly bool) bool {
LOG_DEBUG("DELETE_PARAMETERS",
"id: %s, revisions: %v, tags: %v, retentions: %v, exhaustive: %t, exclusive: %t, " +
"id: %s, revisions: %v, tags: %v, retentions: %v, exhaustive: %t, exclusive: %t, "+
"dryrun: %t, deleteOnly: %t, collectOnly: %t",
snapshotID, revisionsToBeDeleted, tags, retentions,
exhaustive, exclusive, dryRun, deleteOnly, collectOnly)
@@ -1607,13 +1599,13 @@ func (manager *SnapshotManager) PruneSnapshots(selfID string, snapshotID string,
logDir := path.Join(preferencePath, "logs")
os.Mkdir(logDir, 0700)
logFileName := path.Join(logDir, time.Now().Format("prune-log-20060102-150405"))
logFile, err := os.OpenFile(logFileName, os.O_WRONLY | os.O_CREATE | os.O_TRUNC, 0600)
logFile, err := os.OpenFile(logFileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
defer func() {
if logFile != nil {
logFile.Close()
}
} ()
}()
// A retention policy is specified in the form 'interval:age', where both 'interval' and 'age' are numbers of
// days. A retention policy applies to a snapshot if the snapshot is older than the age. For snapshots older
@@ -1629,7 +1621,7 @@ func (manager *SnapshotManager) PruneSnapshots(selfID string, snapshotID string,
Age int
Interval int
}
var retentionPolicies [] RetentionPolicy
var retentionPolicies []RetentionPolicy
// Parse the retention policy if needed.
if len(revisionsToBeDeleted) == 0 && len(retentions) > 0 {
@@ -1654,9 +1646,9 @@ func (manager *SnapshotManager) PruneSnapshots(selfID string, snapshotID string,
return false
}
policy := RetentionPolicy {
Age : age,
Interval : interval,
policy := RetentionPolicy{
Age: age,
Interval: interval,
}
retentionPolicies = append(retentionPolicies, policy)
@@ -1668,7 +1660,7 @@ func (manager *SnapshotManager) PruneSnapshots(selfID string, snapshotID string,
}
for i, policy := range retentionPolicies {
if i == 0 || policy.Age < retentionPolicies[i - 1].Age {
if i == 0 || policy.Age < retentionPolicies[i-1].Age {
if policy.Interval == 0 {
LOG_INFO("RETENTION_POLICY", "Keep no snapshots older than %d days", policy.Age)
} else {
@@ -1679,7 +1671,7 @@ func (manager *SnapshotManager) PruneSnapshots(selfID string, snapshotID string,
}
}
allSnapshots := make(map[string] [] *Snapshot)
allSnapshots := make(map[string][]*Snapshot)
// We must find all snapshots for all ids even if only one snapshot is specified to be deleted,
// because we need to find out which chunks are not referenced.
@@ -1697,7 +1689,7 @@ func (manager *SnapshotManager) PruneSnapshots(selfID string, snapshotID string,
}
sort.Ints(revisions)
var snapshots [] *Snapshot
var snapshots []*Snapshot
for _, revision := range revisions {
snapshot := manager.DownloadSnapshot(id, revision)
if snapshot != nil {
@@ -1731,7 +1723,7 @@ func (manager *SnapshotManager) PruneSnapshots(selfID string, snapshotID string,
}
matched := collectionRegex.FindStringSubmatch(collectionName)
if matched == nil{
if matched == nil {
continue
}
@@ -1874,7 +1866,7 @@ func (manager *SnapshotManager) PruneSnapshots(selfID string, snapshotID string,
i := 0
for j, snapshot := range snapshots {
if !exclusive && j == len(snapshots) - 1 {
if !exclusive && j == len(snapshots)-1 {
continue
}
@@ -1886,7 +1878,7 @@ func (manager *SnapshotManager) PruneSnapshots(selfID string, snapshotID string,
// Find out which retent policy applies based on the age.
for i < len(retentionPolicies) &&
int(now - snapshot.StartTime) <retentionPolicies[i].Age * secondsInDay {
int(now-snapshot.StartTime) < retentionPolicies[i].Age*secondsInDay {
i++
lastSnapshotTime = 0
}
@@ -1897,7 +1889,7 @@ func (manager *SnapshotManager) PruneSnapshots(selfID string, snapshotID string,
snapshot.Flag = true
toBeDeleted++
} else if lastSnapshotTime != 0 &&
int(snapshot.StartTime - lastSnapshotTime) < retentionPolicies[i].Interval * secondsInDay - 600 {
int(snapshot.StartTime-lastSnapshotTime) < retentionPolicies[i].Interval*secondsInDay-600 {
// Delete the snapshot if it is too close to the last kept one. Note that a tolerance of 10
// minutes was subtracted from the interval.
snapshot.Flag = true
@@ -1935,7 +1927,7 @@ func (manager *SnapshotManager) PruneSnapshots(selfID string, snapshotID string,
for _, snapshots := range allSnapshots {
if len(snapshots) > 0 {
latest := snapshots[len(snapshots) - 1]
latest := snapshots[len(snapshots)-1]
if latest.Flag && !exclusive {
LOG_ERROR("SNAPSHOT_DELETE",
"The latest snapshot %s at revision %d can't be deleted in non-exclusive mode",
@@ -1966,7 +1958,7 @@ func (manager *SnapshotManager) PruneSnapshots(selfID string, snapshotID string,
// In exhaustive, we scan the entire chunk tree to find dangling chunks and temporaries.
allFiles, _ := manager.ListAllFiles(manager.storage, chunkDir)
for _, file := range allFiles {
if file[len(file) - 1] == '/' {
if file[len(file)-1] == '/' {
continue
}
@@ -1981,7 +1973,7 @@ func (manager *SnapshotManager) PruneSnapshots(selfID string, snapshotID string,
if exclusive {
// In exclusive mode, we assume no other restore operation is running concurrently.
err := manager.storage.DeleteFile(0, chunkDir + file)
err := manager.storage.DeleteFile(0, chunkDir+file)
if err != nil {
LOG_ERROR("CHUNK_TEMPORARY", "Failed to remove the temporary file %s: %v", file, err)
return false
@@ -2006,9 +1998,9 @@ func (manager *SnapshotManager) PruneSnapshots(selfID string, snapshotID string,
chunk = strings.Replace(chunk, ".fsl", "", -1)
if _, found := referencedChunks[chunk]; found {
manager.resurrectChunk(chunkDir + file, chunk)
manager.resurrectChunk(chunkDir+file, chunk)
} else {
err := manager.storage.DeleteFile(0, chunkDir + file)
err := manager.storage.DeleteFile(0, chunkDir+file)
if err != nil {
LOG_WARN("FOSSIL_DELETE", "Failed to remove the unreferenced fossil %s: %v", file, err)
} else {
@@ -2035,7 +2027,7 @@ func (manager *SnapshotManager) PruneSnapshots(selfID string, snapshotID string,
continue
}
manager.fossilizeChunk(chunk, chunkDir + file, exclusive, collection)
manager.fossilizeChunk(chunk, chunkDir+file, exclusive, collection)
if exclusive {
fmt.Fprintf(logFile, "Deleted chunk %s (exclusive mode)\n", chunk)
} else {
@@ -2053,7 +2045,7 @@ func (manager *SnapshotManager) PruneSnapshots(selfID string, snapshotID string,
}
// This is a redundant chunk file (for instance D3/495A8D and D3/49/5A8D )
err := manager.storage.DeleteFile(0, chunkDir + file)
err := manager.storage.DeleteFile(0, chunkDir+file)
if err != nil {
LOG_WARN("CHUNK_DELETE", "Failed to remove the redundant chunk file %s: %v", file, err)
} else {
@@ -2113,7 +2105,6 @@ func (manager *SnapshotManager) PruneSnapshots(selfID string, snapshotID string,
}
}
}
// Save the fossil collection if it is not empty.
@@ -2169,7 +2160,7 @@ func (manager *SnapshotManager) PruneSnapshots(selfID string, snapshotID string,
var latestSnapshot *Snapshot = nil
if len(allSnapshots[selfID]) > 0 {
latestSnapshot = allSnapshots[selfID][len(allSnapshots[selfID]) - 1]
latestSnapshot = allSnapshots[selfID][len(allSnapshots[selfID])-1]
}
if latestSnapshot != nil && !latestSnapshot.Flag {
@@ -2181,7 +2172,6 @@ func (manager *SnapshotManager) PruneSnapshots(selfID string, snapshotID string,
return true
}
// CheckSnapshot performs sanity checks on the given snapshot.
func (manager *SnapshotManager) CheckSnapshot(snapshot *Snapshot) (err error) {
@@ -2233,7 +2223,7 @@ func (manager *SnapshotManager) CheckSnapshot(snapshot *Snapshot) (err error) {
entry.Path, entry.StartChunk, lastChunk)
}
if entry.StartChunk > lastChunk + 1 {
if entry.StartChunk > lastChunk+1 {
return fmt.Errorf("The file %s starts at chunk %d while the last chunk is %d",
entry.Path, entry.StartChunk, lastChunk)
}
@@ -2275,11 +2265,11 @@ func (manager *SnapshotManager) CheckSnapshot(snapshot *Snapshot) (err error) {
}
if len(entries) > 0 && entries[0].StartChunk != 0 {
return fmt.Errorf("The first file starts at chunk %d", entries[0].StartChunk )
return fmt.Errorf("The first file starts at chunk %d", entries[0].StartChunk)
}
// There may be a last chunk whose size is 0 so we allow this to happen
if lastChunk < numberOfChunks - 2 {
if lastChunk < numberOfChunks-2 {
return fmt.Errorf("The last file ends at chunk %d but the number of chunks is %d", lastChunk, numberOfChunks)
}

98
src/duplicacy_snapshotmanager_test.go
View File

@@ -5,19 +5,19 @@
package duplicacy
import (
"testing"
"os"
"crypto/rand"
"encoding/hex"
"encoding/json"
"fmt"
"time"
"os"
"path"
"strings"
"crypto/rand"
"encoding/json"
"encoding/hex"
"testing"
"time"
)
func createDummySnapshot(snapshotID string, revision int, endTime int64) * Snapshot {
return &Snapshot {
func createDummySnapshot(snapshotID string, revision int, endTime int64) *Snapshot {
return &Snapshot{
ID: snapshotID,
Revision: revision,
EndTime: endTime,
@@ -31,15 +31,15 @@ func TestIsDeletable(t *testing.T) {
now := time.Now().Unix()
day := int64(3600 * 24)
allSnapshots := make(map[string][] *Snapshot)
allSnapshots["host1"] = append([]*Snapshot{}, createDummySnapshot("host1", 1, now - 2 * day))
allSnapshots["host2"] = append([]*Snapshot{}, createDummySnapshot("host2", 1, now - 2 * day))
allSnapshots["host1"] = append(allSnapshots["host1"], createDummySnapshot("host1", 2, now - 1 * day))
allSnapshots["host2"] = append(allSnapshots["host2"], createDummySnapshot("host2", 2, now - 1 * day))
allSnapshots := make(map[string][]*Snapshot)
allSnapshots["host1"] = append([]*Snapshot{}, createDummySnapshot("host1", 1, now-2*day))
allSnapshots["host2"] = append([]*Snapshot{}, createDummySnapshot("host2", 1, now-2*day))
allSnapshots["host1"] = append(allSnapshots["host1"], createDummySnapshot("host1", 2, now-1*day))
allSnapshots["host2"] = append(allSnapshots["host2"], createDummySnapshot("host2", 2, now-1*day))
collection := & FossilCollection {
collection := &FossilCollection{
EndTime: now - day - 3600,
LastRevisions: make(map[string] int),
LastRevisions: make(map[string]int),
}
collection.LastRevisions["host1"] = 1
@@ -51,21 +51,21 @@ func TestIsDeletable(t *testing.T) {
}
collection.LastRevisions["host3"] = 1
allSnapshots["host3"] = append([]*Snapshot{}, createDummySnapshot("host3", 1, now - 2 * day))
allSnapshots["host3"] = append([]*Snapshot{}, createDummySnapshot("host3", 1, now-2*day))
isDeletable, newSnapshots = collection.IsDeletable(true, nil, allSnapshots)
if isDeletable {
t.Errorf("Scenario 2: should not be deletable")
}
allSnapshots["host3"] = append(allSnapshots["host3"], createDummySnapshot("host3", 2, now - day))
allSnapshots["host3"] = append(allSnapshots["host3"], createDummySnapshot("host3", 2, now-day))
isDeletable, newSnapshots = collection.IsDeletable(true, nil, allSnapshots)
if !isDeletable || len(newSnapshots) != 3 {
t.Errorf("Scenario 3: should be deletable, 3 new snapshots")
}
collection.LastRevisions["host4"] = 1
allSnapshots["host4"] = append([]*Snapshot{}, createDummySnapshot("host4", 1, now - 8 * day))
allSnapshots["host4"] = append([]*Snapshot{}, createDummySnapshot("host4", 1, now-8*day))
isDeletable, newSnapshots = collection.IsDeletable(true, nil, allSnapshots)
if !isDeletable || len(newSnapshots) != 3 {
@@ -73,17 +73,17 @@ func TestIsDeletable(t *testing.T) {
}
collection.LastRevisions["repository1@host5"] = 1
allSnapshots["repository1@host5"] = append([]*Snapshot{}, createDummySnapshot("repository1@host5", 1, now - 3 * day))
allSnapshots["repository1@host5"] = append([]*Snapshot{}, createDummySnapshot("repository1@host5", 1, now-3*day))
collection.LastRevisions["repository2@host5"] = 1
allSnapshots["repository2@host5"] = append([]*Snapshot{}, createDummySnapshot("repository2@host5", 1, now - 2 * day))
allSnapshots["repository2@host5"] = append([]*Snapshot{}, createDummySnapshot("repository2@host5", 1, now-2*day))
isDeletable, newSnapshots = collection.IsDeletable(true, nil, allSnapshots)
if isDeletable {
t.Errorf("Scenario 5: should not be deletable")
}
allSnapshots["repository1@host5"] = append(allSnapshots["repository1@host5"], createDummySnapshot("repository1@host5", 2, now - day))
allSnapshots["repository1@host5"] = append(allSnapshots["repository1@host5"], createDummySnapshot("repository1@host5", 2, now-day))
isDeletable, newSnapshots = collection.IsDeletable(true, nil, allSnapshots)
if !isDeletable || len(newSnapshots) != 4 {
t.Errorf("Scenario 6: should be deletable, 4 new snapshots")
@@ -142,7 +142,7 @@ func uploadRandomChunk(manager *SnapshotManager, chunkSize int) string {
func createTestSnapshot(manager *SnapshotManager, snapshotID string, revision int, startTime int64, endTime int64, chunkHashes []string) {
snapshot := &Snapshot {
snapshot := &Snapshot{
ID: snapshotID,
Revision: revision,
StartTime: startTime,
@@ -150,29 +150,29 @@ func createTestSnapshot(manager *SnapshotManager, snapshotID string, revision in
ChunkHashes: chunkHashes,
}
var chunkHashesInHex [] string
var chunkHashesInHex []string
for _, chunkHash := range chunkHashes {
chunkHashesInHex = append(chunkHashesInHex, hex.EncodeToString([]byte(chunkHash)))
}
sequence, _ := json.Marshal(chunkHashesInHex)
snapshot.ChunkSequence = []string { uploadTestChunk(manager, sequence) }
snapshot.ChunkSequence = []string{uploadTestChunk(manager, sequence)}
description, _ := snapshot.MarshalJSON()
path := fmt.Sprintf("snapshots/%s/%d", snapshotID, snapshot.Revision)
manager.storage.CreateDirectory(0, "snapshots/" + snapshotID)
manager.storage.CreateDirectory(0, "snapshots/"+snapshotID)
manager.UploadFile(path, path, description)
}
func checkTestSnapshots(manager *SnapshotManager, expectedSnapshots int, expectedFossils int) {
var snapshotIDs [] string
var snapshotIDs []string
var err error
chunks := make(map[string]bool)
files, _ := manager.ListAllFiles(manager.storage, "chunks/")
for _, file := range files {
if file[len(file) - 1] == '/' {
if file[len(file)-1] == '/' {
continue
}
chunk := strings.Replace(file, "/", "", -1)
@@ -239,12 +239,12 @@ func TestSingleRepositoryPrune(t *testing.T) {
now := time.Now().Unix()
day := int64(24 * 3600)
t.Logf("Creating 1 snapshot")
createTestSnapshot(snapshotManager, "repository1", 1, now - 3 * day - 3600, now - 3 * day - 60, []string { chunkHash1, chunkHash2 })
createTestSnapshot(snapshotManager, "repository1", 1, now-3*day-3600, now-3*day-60, []string{chunkHash1, chunkHash2})
checkTestSnapshots(snapshotManager, 1, 2)
t.Logf("Creating 2 snapshots")
createTestSnapshot(snapshotManager, "repository1", 2, now - 2 * day - 3600, now - 2 * day - 60, []string { chunkHash2, chunkHash3 })
createTestSnapshot(snapshotManager, "repository1", 3, now - 1 * day - 3600, now - 1 * day - 60, []string { chunkHash3, chunkHash4 })
createTestSnapshot(snapshotManager, "repository1", 2, now-2*day-3600, now-2*day-60, []string{chunkHash2, chunkHash3})
createTestSnapshot(snapshotManager, "repository1", 3, now-1*day-3600, now-1*day-60, []string{chunkHash3, chunkHash4})
checkTestSnapshots(snapshotManager, 3, 0)
t.Logf("Removing snapshot repository1 revision 1 with --exclusive")
@@ -257,7 +257,7 @@ func TestSingleRepositoryPrune(t *testing.T) {
t.Logf("Creating 1 snapshot")
chunkHash5 := uploadRandomChunk(snapshotManager, chunkSize)
createTestSnapshot(snapshotManager, "repository1", 4, now + 1 * day - 3600 , now + 1 * day, []string { chunkHash4, chunkHash5 })
createTestSnapshot(snapshotManager, "repository1", 4, now+1*day-3600, now+1*day, []string{chunkHash4, chunkHash5})
checkTestSnapshots(snapshotManager, 2, 2)
t.Logf("Prune without removing any snapshots -- fossils will be deleted")
@@ -282,9 +282,9 @@ func TestSingleHostPrune(t *testing.T) {
now := time.Now().Unix()
day := int64(24 * 3600)
t.Logf("Creating 3 snapshots")
createTestSnapshot(snapshotManager, "vm1@host1", 1, now - 3 * day - 3600, now - 3 * day - 60, []string { chunkHash1, chunkHash2 })
createTestSnapshot(snapshotManager, "vm1@host1", 2, now - 2 * day - 3600, now - 2 * day - 60, []string { chunkHash2, chunkHash3 })
createTestSnapshot(snapshotManager, "vm2@host1", 1, now - 3 * day - 3600, now - 3 * day - 60, []string { chunkHash3, chunkHash4 })
createTestSnapshot(snapshotManager, "vm1@host1", 1, now-3*day-3600, now-3*day-60, []string{chunkHash1, chunkHash2})
createTestSnapshot(snapshotManager, "vm1@host1", 2, now-2*day-3600, now-2*day-60, []string{chunkHash2, chunkHash3})
createTestSnapshot(snapshotManager, "vm2@host1", 1, now-3*day-3600, now-3*day-60, []string{chunkHash3, chunkHash4})
checkTestSnapshots(snapshotManager, 3, 0)
t.Logf("Removing snapshot vm1@host1 revision 1 without --exclusive")
@@ -297,7 +297,7 @@ func TestSingleHostPrune(t *testing.T) {
t.Logf("Creating 1 snapshot")
chunkHash5 := uploadRandomChunk(snapshotManager, chunkSize)
createTestSnapshot(snapshotManager, "vm2@host1", 2, now + 1 * day - 3600 , now + 1 * day, []string { chunkHash4, chunkHash5 })
createTestSnapshot(snapshotManager, "vm2@host1", 2, now+1*day-3600, now+1*day, []string{chunkHash4, chunkHash5})
checkTestSnapshots(snapshotManager, 3, 2)
t.Logf("Prune without removing any snapshots -- fossils will be deleted")
@@ -323,9 +323,9 @@ func TestMultipleHostPrune(t *testing.T) {
now := time.Now().Unix()
day := int64(24 * 3600)
t.Logf("Creating 3 snapshot")
createTestSnapshot(snapshotManager, "vm1@host1", 1, now - 3 * day - 3600, now - 3 * day - 60, []string { chunkHash1, chunkHash2 })
createTestSnapshot(snapshotManager, "vm1@host1", 2, now - 2 * day - 3600, now - 2 * day - 60, []string { chunkHash2, chunkHash3 })
createTestSnapshot(snapshotManager, "vm2@host2", 1, now - 3 * day - 3600, now - 3 * day - 60, []string { chunkHash3, chunkHash4 })
createTestSnapshot(snapshotManager, "vm1@host1", 1, now-3*day-3600, now-3*day-60, []string{chunkHash1, chunkHash2})
createTestSnapshot(snapshotManager, "vm1@host1", 2, now-2*day-3600, now-2*day-60, []string{chunkHash2, chunkHash3})
createTestSnapshot(snapshotManager, "vm2@host2", 1, now-3*day-3600, now-3*day-60, []string{chunkHash3, chunkHash4})
checkTestSnapshots(snapshotManager, 3, 0)
t.Logf("Removing snapshot vm1@host1 revision 1 without --exclusive")
@@ -338,7 +338,7 @@ func TestMultipleHostPrune(t *testing.T) {
t.Logf("Creating 1 snapshot")
chunkHash5 := uploadRandomChunk(snapshotManager, chunkSize)
createTestSnapshot(snapshotManager, "vm2@host2", 2, now + 1 * day - 3600 , now + 1 * day, []string {chunkHash4, chunkHash5})
createTestSnapshot(snapshotManager, "vm2@host2", 2, now+1*day-3600, now+1*day, []string{chunkHash4, chunkHash5})
checkTestSnapshots(snapshotManager, 3, 2)
t.Logf("Prune without removing any snapshots -- no fossils will be deleted")
@@ -347,7 +347,7 @@ func TestMultipleHostPrune(t *testing.T) {
t.Logf("Creating 1 snapshot")
chunkHash6 := uploadRandomChunk(snapshotManager, chunkSize)
createTestSnapshot(snapshotManager, "vm1@host1", 3, now + 1 * day - 3600 , now + 1 * day, []string {chunkHash5, chunkHash6})
createTestSnapshot(snapshotManager, "vm1@host1", 3, now+1*day-3600, now+1*day, []string{chunkHash5, chunkHash6})
checkTestSnapshots(snapshotManager, 4, 2)
t.Logf("Prune without removing any snapshots -- fossils will be deleted")
@@ -371,8 +371,8 @@ func TestPruneAndResurrect(t *testing.T) {
now := time.Now().Unix()
day := int64(24 * 3600)
t.Logf("Creating 2 snapshots")
createTestSnapshot(snapshotManager, "vm1@host1", 1, now - 3 * day - 3600, now - 3 * day - 60, []string { chunkHash1, chunkHash2})
createTestSnapshot(snapshotManager, "vm1@host1", 2, now - 2 * day - 3600, now - 2 * day - 60, []string { chunkHash2, chunkHash3})
createTestSnapshot(snapshotManager, "vm1@host1", 1, now-3*day-3600, now-3*day-60, []string{chunkHash1, chunkHash2})
createTestSnapshot(snapshotManager, "vm1@host1", 2, now-2*day-3600, now-2*day-60, []string{chunkHash2, chunkHash3})
checkTestSnapshots(snapshotManager, 2, 0)
t.Logf("Removing snapshot vm1@host1 revision 1 without --exclusive")
@@ -381,7 +381,7 @@ func TestPruneAndResurrect(t *testing.T) {
t.Logf("Creating 1 snapshot")
chunkHash4 := uploadRandomChunk(snapshotManager, chunkSize)
createTestSnapshot(snapshotManager, "vm1@host1", 4, now + 1 * day - 3600 , now + 1 * day, []string { chunkHash4, chunkHash1})
createTestSnapshot(snapshotManager, "vm1@host1", 4, now+1*day-3600, now+1*day, []string{chunkHash4, chunkHash1})
checkTestSnapshots(snapshotManager, 2, 2)
t.Logf("Prune without removing any snapshots -- one fossil will be resurrected")
@@ -406,10 +406,10 @@ func TestInactiveHostPrune(t *testing.T) {
now := time.Now().Unix()
day := int64(24 * 3600)
t.Logf("Creating 3 snapshot")
createTestSnapshot(snapshotManager, "vm1@host1", 1, now - 3 * day - 3600, now - 3 * day - 60, []string { chunkHash1, chunkHash2} )
createTestSnapshot(snapshotManager, "vm1@host1", 2, now - 2 * day - 3600, now - 2 * day - 60, []string { chunkHash2, chunkHash3} )
createTestSnapshot(snapshotManager, "vm1@host1", 1, now-3*day-3600, now-3*day-60, []string{chunkHash1, chunkHash2})
createTestSnapshot(snapshotManager, "vm1@host1", 2, now-2*day-3600, now-2*day-60, []string{chunkHash2, chunkHash3})
// Host2 is inactive
createTestSnapshot(snapshotManager, "vm2@host2", 1, now - 7 * day - 3600, now - 7 * day - 60, []string { chunkHash3, chunkHash4} )
createTestSnapshot(snapshotManager, "vm2@host2", 1, now-7*day-3600, now-7*day-60, []string{chunkHash3, chunkHash4})
checkTestSnapshots(snapshotManager, 3, 0)
t.Logf("Removing snapshot vm1@host1 revision 1")
@@ -422,7 +422,7 @@ func TestInactiveHostPrune(t *testing.T) {
t.Logf("Creating 1 snapshot")
chunkHash5 := uploadRandomChunk(snapshotManager, chunkSize)
createTestSnapshot(snapshotManager, "vm1@host1", 3, now + 1 * day - 3600 , now + 1 * day, []string { chunkHash4, chunkHash5} )
createTestSnapshot(snapshotManager, "vm1@host1", 3, now+1*day-3600, now+1*day, []string{chunkHash4, chunkHash5})
checkTestSnapshots(snapshotManager, 3, 2)
t.Logf("Prune without removing any snapshots -- fossils will be deleted")
@@ -439,7 +439,7 @@ func TestRetentionPolicy(t *testing.T) {
snapshotManager := createTestSnapshotManager(testDir)
chunkSize := 1024
var chunkHashes [] string
var chunkHashes []string
for i := 0; i < 30; i++ {
chunkHashes = append(chunkHashes, uploadRandomChunk(snapshotManager, chunkSize))
}
@@ -448,7 +448,7 @@ func TestRetentionPolicy(t *testing.T) {
day := int64(24 * 3600)
t.Logf("Creating 30 snapshots")
for i := 0; i < 30; i++ {
createTestSnapshot(snapshotManager, "vm1@host1", i + 1, now - int64(30 - i) * day - 3600, now - int64(30 - i) * day - 60, []string { chunkHashes[i] })
createTestSnapshot(snapshotManager, "vm1@host1", i+1, now-int64(30-i)*day-3600, now-int64(30-i)*day-60, []string{chunkHashes[i]})
}
checkTestSnapshots(snapshotManager, 30, 0)

47
src/duplicacy_storage.go
View File

@@ -6,14 +6,14 @@ package duplicacy
import (
"fmt"
"regexp"
"strings"
"strconv"
"os"
"net"
"path"
"io/ioutil"
"net"
"os"
"path"
"regexp"
"runtime"
"strconv"
"strings"
"golang.org/x/crypto/ssh"
"golang.org/x/crypto/ssh/agent"
@@ -47,16 +47,16 @@ type Storage interface {
// If a local snapshot cache is needed for the storage to avoid downloading/uploading chunks too often when
// managing snapshots.
IsCacheNeeded() (bool)
IsCacheNeeded() bool
// If the 'MoveFile' method is implemented.
IsMoveFileImplemented() (bool)
IsMoveFileImplemented() bool
// If the storage can guarantee strong consistency.
IsStrongConsistent() (bool)
IsStrongConsistent() bool
// If the storage supports fast listing of files names.
IsFastListing() (bool)
IsFastListing() bool
// Enable the test mode.
EnableTestMode()
@@ -79,7 +79,7 @@ func checkHostKey(hostname string, remote net.Addr, key ssh.PublicKey) error {
preferencePath := GetDuplicacyPreferencePath()
hostFile := path.Join(preferencePath, "known_hosts")
file, err := os.OpenFile(hostFile, os.O_RDWR | os.O_CREATE, 0600)
file, err := os.OpenFile(hostFile, os.O_RDWR|os.O_CREATE, 0600)
if err != nil {
return err
}
@@ -96,7 +96,7 @@ func checkHostKey(hostname string, remote net.Addr, key ssh.PublicKey) error {
keyString = strings.Replace(keyString, "\n", "", -1)
remoteAddress := remote.String()
if strings.HasSuffix(remoteAddress, ":22") {
remoteAddress = remoteAddress[:len(remoteAddress) - len(":22")]
remoteAddress = remoteAddress[:len(remoteAddress)-len(":22")]
}
for i, line := range strings.Split(string(content), "\n") {
@@ -187,7 +187,7 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
if strings.Contains(server, ":") {
index := strings.Index(server, ":")
port, _ = strconv.Atoi(server[index + 1:])
port, _ = strconv.Atoi(server[index+1:])
server = server[:index]
}
@@ -197,7 +197,7 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
}
if username != "" {
username = username[:len(username) - 1]
username = username[:len(username)-1]
}
// If ssh_key_file is set, skip password-based login
@@ -210,12 +210,12 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
return password, nil
}
keyboardInteractive := func (user, instruction string, questions []string, echos []bool) (answers []string,
keyboardInteractive := func(user, instruction string, questions []string, echos []bool) (answers []string,
err error) {
if len(questions) == 1 {
LOG_DEBUG("SSH_INTERACTIVE", "Attempting keyboard interactive login")
password = GetPassword(preference, "ssh_password", "Enter SSH password:", false, resetPassword)
answers = []string { password }
answers = []string{password}
return answers, nil
} else {
return nil, nil
@@ -225,7 +225,7 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
publicKeysCallback := func() ([]ssh.Signer, error) {
LOG_DEBUG("SSH_PUBLICKEY", "Attempting public key authentication")
signers := []ssh.Signer {}
signers := []ssh.Signer{}
agentSock := os.Getenv("SSH_AUTH_SOCK")
if agentSock != "" {
@@ -274,13 +274,12 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
}
authMethods := [] ssh.AuthMethod {
}
passwordAuthMethods := [] ssh.AuthMethod {
authMethods := []ssh.AuthMethod{}
passwordAuthMethods := []ssh.AuthMethod{
ssh.PasswordCallback(passwordCallback),
ssh.KeyboardInteractive(keyboardInteractive),
}
keyFileAuthMethods := [] ssh.AuthMethod {
keyFileAuthMethods := []ssh.AuthMethod{
ssh.PublicKeysCallback(publicKeysCallback),
}
if keyFile != "" {
@@ -298,7 +297,7 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
keyFileKey = preference.Name + "_" + keyFileKey
}
authMethods = [] ssh.AuthMethod {}
authMethods = []ssh.AuthMethod{}
if keyringGet(passwordKey) != "" {
authMethods = append(authMethods, ssh.PasswordCallback(passwordCallback))
authMethods = append(authMethods, ssh.KeyboardInteractive(keyboardInteractive))
@@ -333,7 +332,7 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
bucket := matched[5]
if region != "" {
region = region[:len(region) - 1]
region = region[:len(region)-1]
}
if strings.EqualFold(endpoint, "amazon") || strings.EqualFold(endpoint, "amazon.com") {
@@ -343,7 +342,7 @@ func CreateStorage(preference Preference, resetPassword bool, threads int) (stor
storageDir := ""
if strings.Contains(bucket, "/") {
firstSlash := strings.Index(bucket, "/")
storageDir = bucket[firstSlash + 1:]
storageDir = bucket[firstSlash+1:]
bucket = bucket[:firstSlash]
}

77
src/duplicacy_storage_test.go
View File

@@ -5,19 +5,19 @@
package duplicacy
import (
"os"
"fmt"
"time"
"flag"
"path"
"testing"
"strings"
"strconv"
"io/ioutil"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"flag"
"fmt"
"io/ioutil"
"os"
"path"
"runtime/debug"
"strconv"
"strings"
"testing"
"time"
crypto_rand "crypto/rand"
"math/rand"
@@ -111,8 +111,8 @@ func cleanStorage(storage Storage) {
LOG_INFO("STORAGE_LIST", "Listing snapshots in the storage")
for len(directories) > 0 {
dir := directories[len(directories) - 1]
directories = directories[:len(directories) - 1]
dir := directories[len(directories)-1]
directories = directories[:len(directories)-1]
files, _, err := storage.ListFiles(0, dir)
if err != nil {
@@ -121,10 +121,10 @@ func cleanStorage(storage Storage) {
}
for _, file := range files {
if len(file) > 0 && file[len(file) - 1] == '/' {
directories = append(directories, dir + file)
if len(file) > 0 && file[len(file)-1] == '/' {
directories = append(directories, dir+file)
} else {
snapshots = append(snapshots, dir + file)
snapshots = append(snapshots, dir+file)
}
}
}
@@ -135,7 +135,7 @@ func cleanStorage(storage Storage) {
}
for _, chunk := range listChunks(storage) {
storage.DeleteFile(0, "chunks/" + chunk)
storage.DeleteFile(0, "chunks/"+chunk)
}
storage.DeleteFile(0, "config")
@@ -151,8 +151,8 @@ func listChunks(storage Storage) (chunks []string) {
for len(directories) > 0 {
dir := directories[len(directories) - 1]
directories = directories[:len(directories) - 1]
dir := directories[len(directories)-1]
directories = directories[:len(directories)-1]
files, _, err := storage.ListFiles(0, dir)
if err != nil {
@@ -161,8 +161,8 @@ func listChunks(storage Storage) (chunks []string) {
}
for _, file := range files {
if len(file) > 0 && file[len(file) - 1] == '/' {
directories = append(directories, dir + file)
if len(file) > 0 && file[len(file)-1] == '/' {
directories = append(directories, dir+file)
} else {
chunk := dir + file
chunk = chunk[len("chunks/"):]
@@ -185,7 +185,7 @@ func moveChunk(t *testing.T, storage Storage, chunkID string, isFossil bool, del
to := filePath + ".fsl"
if isFossil {
to = filePath[:len(filePath) - len(".fsl")]
to = filePath[:len(filePath)-len(".fsl")]
}
err = storage.MoveFile(0, filePath, to)
@@ -232,7 +232,7 @@ func TestStorage(t *testing.T) {
debug.PrintStack()
}
}
} ()
}()
testDir := path.Join(os.TempDir(), "duplicacy_test", "storage_test")
os.RemoveAll(testDir)
@@ -256,7 +256,7 @@ func TestStorage(t *testing.T) {
delay = 2
}
for _, dir := range []string { "chunks", "snapshots" } {
for _, dir := range []string{"chunks", "snapshots"} {
err = storage.CreateDirectory(0, dir)
if err != nil {
t.Errorf("Failed to create directory %s: %v", dir, err)
@@ -299,10 +299,10 @@ func TestStorage(t *testing.T) {
return
}
snapshotIDs := []string {}
snapshotIDs := []string{}
for _, snapshotDir := range snapshotDirs {
if len(snapshotDir) > 0 && snapshotDir[len(snapshotDir) - 1] == '/' {
snapshotIDs = append(snapshotIDs, snapshotDir[:len(snapshotDir) - 1])
if len(snapshotDir) > 0 && snapshotDir[len(snapshotDir)-1] == '/' {
snapshotIDs = append(snapshotIDs, snapshotDir[:len(snapshotDir)-1])
}
}
@@ -312,13 +312,13 @@ func TestStorage(t *testing.T) {
}
for _, snapshotID := range snapshotIDs {
snapshots, _, err := storage.ListFiles(0, "snapshots/" + snapshotID)
snapshots, _, err := storage.ListFiles(0, "snapshots/"+snapshotID)
if err != nil {
t.Errorf("Failed to list snapshots for %s: %v", snapshotID, err)
return
}
for _, snapshot := range snapshots {
storage.DeleteFile(0, "snapshots/" + snapshotID + "/" + snapshot)
storage.DeleteFile(0, "snapshots/"+snapshotID+"/"+snapshot)
}
}
@@ -326,7 +326,7 @@ func TestStorage(t *testing.T) {
storage.DeleteFile(0, "config")
for _, file := range []string { "snapshots/repository1/1", "snapshots/repository2/1"} {
for _, file := range []string{"snapshots/repository1/1", "snapshots/repository2/1"} {
exist, _, _, err := storage.GetFileInfo(0, file)
if err != nil {
t.Errorf("Failed to get file info for %s: %v", file, err)
@@ -348,7 +348,7 @@ func TestStorage(t *testing.T) {
chunks := []string{}
for i := 0; i < numberOfFiles; i++ {
content := make([]byte, rand.Int() % maxFileSize + 1)
content := make([]byte, rand.Int()%maxFileSize+1)
_, err = crypto_rand.Read(content)
if err != nil {
t.Errorf("Error generating random content: %v", err)
@@ -377,7 +377,7 @@ func TestStorage(t *testing.T) {
LOG_INFO("STORAGE_CHUNK", "Uploaded chunk: %s, size: %d", chunkID, len(content))
}
allChunks := [] string {}
allChunks := []string{}
for _, file := range listChunks(storage) {
file = strings.Replace(file, "/", "", -1)
if len(file) == 64 {
@@ -392,11 +392,10 @@ func TestStorage(t *testing.T) {
config := CreateConfig()
config.MinimumChunkSize = 100
config.chunkPool = make(chan *Chunk, numberOfFiles * 2)
config.chunkPool = make(chan *Chunk, numberOfFiles*2)
chunk := CreateChunk(config, true)
for _, chunkID := range chunks {
chunk.Reset(false)
@@ -450,7 +449,7 @@ func TestStorage(t *testing.T) {
for _, file := range allChunks {
err = storage.DeleteFile(0, "chunks/" + file)
err = storage.DeleteFile(0, "chunks/"+file)
if err != nil {
t.Errorf("Failed to delete the file %s: %v", file, err)
return
@@ -474,7 +473,7 @@ func TestCleanStorage(t *testing.T) {
debug.PrintStack()
}
}
} ()
}()
testDir := path.Join(os.TempDir(), "duplicacy_test", "storage_test")
os.RemoveAll(testDir)
@@ -494,8 +493,8 @@ func TestCleanStorage(t *testing.T) {
for len(directories) > 0 {
dir := directories[len(directories) - 1]
directories = directories[:len(directories) - 1]
dir := directories[len(directories)-1]
directories = directories[:len(directories)-1]
LOG_INFO("LIST_FILES", "Listing %s", dir)
@@ -506,10 +505,10 @@ func TestCleanStorage(t *testing.T) {
}
for _, file := range files {
if len(file) > 0 && file[len(file) - 1] == '/' {
directories = append(directories, dir + file)
if len(file) > 0 && file[len(file)-1] == '/' {
directories = append(directories, dir+file)
} else {
storage.DeleteFile(0, dir + file)
storage.DeleteFile(0, dir+file)
LOG_INFO("DELETE_FILE", "Deleted file %s", file)
}
}

72
src/duplicacy_utils.go
View File

@@ -5,21 +5,21 @@
package duplicacy
import (
"fmt"
"os"
"bufio"
"crypto/sha256"
"fmt"
"io"
"time"
"os"
"path"
"path/filepath"
"regexp"
"strings"
"strconv"
"runtime"
"crypto/sha256"
"strconv"
"strings"
"time"
"golang.org/x/crypto/pbkdf2"
"github.com/gilbertchen/gopass"
"golang.org/x/crypto/pbkdf2"
)
var RunInBackground bool = false
@@ -41,8 +41,8 @@ func init() {
}
func CreateRateLimitedReader(content []byte, rate int) (*RateLimitedReader) {
return &RateLimitedReader {
func CreateRateLimitedReader(content []byte, rate int) *RateLimitedReader {
return &RateLimitedReader{
Content: content,
Rate: float64(rate * 1024),
Next: 0,
@@ -84,7 +84,7 @@ func IsValidRegex(pattern string) (valid bool, err error) {
}
}
func (reader *RateLimitedReader) Length() (int64) {
func (reader *RateLimitedReader) Length() int64 {
return int64(len(reader.Content))
}
@@ -123,19 +123,19 @@ func (reader *RateLimitedReader) Read(p []byte) (n int, err error) {
}
elapsed := time.Since(reader.StartTime).Seconds()
delay := float64(reader.Next) / reader.Rate - elapsed
end := reader.Next + int(reader.Rate / 5)
delay := float64(reader.Next)/reader.Rate - elapsed
end := reader.Next + int(reader.Rate/5)
if delay > 0 {
time.Sleep(time.Duration(delay * float64(time.Second)))
} else {
end += - int(delay * reader.Rate)
end += -int(delay * reader.Rate)
}
if end > len(reader.Content) {
end = len(reader.Content)
}
n = copy(p, reader.Content[reader.Next : end])
n = copy(p, reader.Content[reader.Next:end])
reader.Next += n
return n, nil
}
@@ -145,7 +145,7 @@ func RateLimitedCopy(writer io.Writer, reader io.Reader, rate int) (written int6
return io.Copy(writer, reader)
}
for range time.Tick(time.Second / 5) {
n, err := io.CopyN(writer, reader, int64(rate * 1024 / 5))
n, err := io.CopyN(writer, reader, int64(rate*1024/5))
written += n
if err != nil {
if err == io.EOF {
@@ -164,7 +164,7 @@ func GenerateKeyFromPassword(password string) []byte {
}
// Get password from preference, env, but don't start any keyring request
func GetPasswordFromPreference(preference Preference, passwordType string) (string) {
func GetPasswordFromPreference(preference Preference, passwordType string) string {
passwordID := passwordType
if preference.Name != "default" {
passwordID = preference.Name + "_" + passwordID
@@ -196,9 +196,9 @@ func GetPasswordFromPreference(preference Preference, passwordType string) (stri
// GetPassword attempts to get the password from KeyChain/KeyRing, environment variables, or keyboard input.
func GetPassword(preference Preference, passwordType string, prompt string,
showPassword bool, resetPassword bool) (string) {
showPassword bool, resetPassword bool) string {
passwordID := passwordType
password := GetPasswordFromPreference(preference,passwordType)
password := GetPasswordFromPreference(preference, passwordType)
if password != "" {
return password
}
@@ -331,7 +331,7 @@ func matchPattern(text string, pattern string) bool {
// either '+' or '-', whereas '-' indicates exclusion and '+' indicates inclusion. Wildcards like '*' and '?' may
// appear in the patterns. In case no matching pattern is found, the file will be excluded if all patterns are
// include patterns, and included otherwise.
func MatchPath(filePath string, patterns [] string) (included bool) {
func MatchPath(filePath string, patterns []string) (included bool) {
var re *regexp.Regexp = nil
var found bool
@@ -386,52 +386,52 @@ func joinPath(components ...string) string {
return combinedPath
}
func PrettyNumber(number int64) (string) {
func PrettyNumber(number int64) string {
G := int64(1024 * 1024 * 1024)
M := int64(1024 * 1024)
K := int64(1024)
if number > 1000 * G {
return fmt.Sprintf("%dG", number / G)
if number > 1000*G {
return fmt.Sprintf("%dG", number/G)
} else if number > G {
return fmt.Sprintf("%d,%03dM", number / (1000 * M), (number / M) % 1000)
return fmt.Sprintf("%d,%03dM", number/(1000*M), (number/M)%1000)
} else if number > M {
return fmt.Sprintf("%d,%03dK", number / (1000 * K), (number / K) % 1000)
return fmt.Sprintf("%d,%03dK", number/(1000*K), (number/K)%1000)
} else if number > K {
return fmt.Sprintf("%dK", number / K)
return fmt.Sprintf("%dK", number/K)
} else {
return fmt.Sprintf("%d", number)
}
}
func PrettySize(size int64) (string) {
if size > 1024 * 1024 {
return fmt.Sprintf("%.2fM", float64(size) / (1024.0 * 1024.0))
func PrettySize(size int64) string {
if size > 1024*1024 {
return fmt.Sprintf("%.2fM", float64(size)/(1024.0*1024.0))
} else if size > 1024 {
return fmt.Sprintf("%.0fK", float64(size) / 1024.0)
return fmt.Sprintf("%.0fK", float64(size)/1024.0)
} else {
return fmt.Sprintf("%d", size)
}
}
func PrettyTime(seconds int64) (string) {
func PrettyTime(seconds int64) string {
day := int64(3600 * 24)
if seconds > day * 2 {
if seconds > day*2 {
return fmt.Sprintf("%d days %02d:%02d:%02d",
seconds / day, (seconds % day) / 3600, (seconds % 3600) / 60, seconds % 60)
seconds/day, (seconds%day)/3600, (seconds%3600)/60, seconds%60)
} else if seconds > day {
return fmt.Sprintf("1 day %02d:%02d:%02d", (seconds % day) / 3600, (seconds % 3600) / 60, seconds % 60)
return fmt.Sprintf("1 day %02d:%02d:%02d", (seconds%day)/3600, (seconds%3600)/60, seconds%60)
} else if seconds > 0 {
return fmt.Sprintf("%02d:%02d:%02d", seconds / 3600, (seconds % 3600) / 60, seconds % 60)
return fmt.Sprintf("%02d:%02d:%02d", seconds/3600, (seconds%3600)/60, seconds%60)
} else {
return "n/a"
}
}
func AtoSize(sizeString string) (int) {
func AtoSize(sizeString string) int {
sizeString = strings.ToLower(sizeString)
sizeRegex := regexp.MustCompile(`^([0-9]+)([mk])?$`)
@@ -451,7 +451,7 @@ func AtoSize(sizeString string) (int) {
return size
}
func MinInt(x, y int) (int) {
func MinInt(x, y int) int {
if x < y {
return x
}

6
src/duplicacy_utils_others.go
View File

@@ -7,10 +7,10 @@
package duplicacy
import (
"os"
"bytes"
"syscall"
"os"
"path/filepath"
"syscall"
"github.com/gilbertchen/xattr"
)
@@ -31,7 +31,7 @@ func GetOwner(entry *Entry, fileInfo *os.FileInfo) {
}
}
func SetOwner(fullPath string, entry *Entry, fileInfo *os.FileInfo) (bool) {
func SetOwner(fullPath string, entry *Entry, fileInfo *os.FileInfo) bool {
stat, ok := (*fileInfo).Sys().(*syscall.Stat_t)
if ok && stat != nil && (int(stat.Uid) != entry.UID || int(stat.Gid) != entry.GID) {
if entry.UID != -1 && entry.GID != -1 {

105
src/duplicacy_utils_test.go
View File

@@ -5,15 +5,14 @@
package duplicacy
import (
"bytes"
"io"
"io/ioutil"
"time"
"bytes"
crypto_rand "crypto/rand"
"testing"
)
func TestMatchPattern(t *testing.T) {
@@ -21,70 +20,70 @@ func TestMatchPattern(t *testing.T) {
// Test cases were copied from Matching Wildcards: An Empirical Way to Tame an Algorithm
// By Kirk J. Krauss, October 07, 2014
DATA := [] struct {
DATA := []struct {
text string
pattern string
matched bool
} {
}{
// Cases with repeating character sequences.
{ "abcccd", "*ccd", true },
{ "mississipissippi", "*issip*ss*", true },
{ "xxxx*zzzzzzzzy*f", "xxxx*zzy*fffff", false },
{ "xxxx*zzzzzzzzy*f", "xxx*zzy*f", true },
{ "xxxxzzzzzzzzyf", "xxxx*zzy*fffff", false },
{ "xxxxzzzzzzzzyf", "xxxx*zzy*f", true },
{ "xyxyxyzyxyz", "xy*z*xyz", true },
{ "mississippi", "*sip*", true },
{ "xyxyxyxyz", "xy*xyz", true },
{ "mississippi", "mi*sip*", true },
{ "ababac", "*abac*", true },
{ "ababac", "*abac*", true },
{ "aaazz", "a*zz*", true },
{ "a12b12", "*12*23", false },
{ "a12b12", "a12b", false },
{ "a12b12", "*12*12*", true },
{"abcccd", "*ccd", true},
{"mississipissippi", "*issip*ss*", true},
{"xxxx*zzzzzzzzy*f", "xxxx*zzy*fffff", false},
{"xxxx*zzzzzzzzy*f", "xxx*zzy*f", true},
{"xxxxzzzzzzzzyf", "xxxx*zzy*fffff", false},
{"xxxxzzzzzzzzyf", "xxxx*zzy*f", true},
{"xyxyxyzyxyz", "xy*z*xyz", true},
{"mississippi", "*sip*", true},
{"xyxyxyxyz", "xy*xyz", true},
{"mississippi", "mi*sip*", true},
{"ababac", "*abac*", true},
{"ababac", "*abac*", true},
{"aaazz", "a*zz*", true},
{"a12b12", "*12*23", false},
{"a12b12", "a12b", false},
{"a12b12", "*12*12*", true},
// More double wildcard scenarios.
{ "XYXYXYZYXYz", "XY*Z*XYz", true },
{ "missisSIPpi", "*SIP*", true },
{ "mississipPI", "*issip*PI", true },
{ "xyxyxyxyz", "xy*xyz", true },
{ "miSsissippi", "mi*sip*", true },
{ "miSsissippi", "mi*Sip*", false },
{ "abAbac", "*Abac*", true },
{ "abAbac", "*Abac*", true },
{ "aAazz", "a*zz*", true },
{ "A12b12", "*12*23", false },
{ "a12B12", "*12*12*", true },
{ "oWn", "*oWn*", true },
{"XYXYXYZYXYz", "XY*Z*XYz", true},
{"missisSIPpi", "*SIP*", true},
{"mississipPI", "*issip*PI", true},
{"xyxyxyxyz", "xy*xyz", true},
{"miSsissippi", "mi*sip*", true},
{"miSsissippi", "mi*Sip*", false},
{"abAbac", "*Abac*", true},
{"abAbac", "*Abac*", true},
{"aAazz", "a*zz*", true},
{"A12b12", "*12*23", false},
{"a12B12", "*12*12*", true},
{"oWn", "*oWn*", true},
// Completely tame (no wildcards) cases.
{ "bLah", "bLah", true },
{ "bLah", "bLaH", false },
{"bLah", "bLah", true},
{"bLah", "bLaH", false},
// Simple mixed wildcard tests suggested by IBMer Marlin Deckert.
{ "a", "*?", true },
{ "ab", "*?", true },
{ "abc", "*?", true },
{"a", "*?", true},
{"ab", "*?", true},
{"abc", "*?", true},
// More mixed wildcard tests including coverage for false positives.
{ "a", "??", false },
{ "ab", "?*?", true },
{ "ab", "*?*?*", true },
{ "abc", "?*?*?", true },
{ "abc", "?*?*&?", false },
{ "abcd", "?b*??", true },
{ "abcd", "?a*??", false },
{ "abcd", "?*?c?", true },
{ "abcd", "?*?d?", false },
{ "abcde", "?*b*?*d*?", true },
{"a", "??", false},
{"ab", "?*?", true},
{"ab", "*?*?*", true},
{"abc", "?*?*?", true},
{"abc", "?*?*&?", false},
{"abcd", "?b*??", true},
{"abcd", "?a*??", false},
{"abcd", "?*?c?", true},
{"abcd", "?*?d?", false},
{"abcde", "?*b*?*d*?", true},
// Single-character-match cases.
{ "bLah", "bL?h", true },
{ "bLaaa", "bLa?", false },
{ "bLah", "bLa?", true },
{ "bLaH", "?Lah", false },
{ "bLaH", "?LaH", true },
{"bLah", "bL?h", true},
{"bLaaa", "bLa?", false},
{"bLah", "bLa?", true},
{"bLaH", "?Lah", false},
{"bLaH", "?LaH", true},
}
for _, data := range DATA {
@@ -96,7 +95,7 @@ func TestMatchPattern(t *testing.T) {
}
func TestRateLimit(t *testing.T) {
content := make([]byte, 100 * 1024)
content := make([]byte, 100*1024)
_, err := crypto_rand.Read(content)
if err != nil {
t.Errorf("Error generating random content: %v", err)

13
src/duplicacy_utils_windows.go
View File

@@ -5,8 +5,8 @@
package duplicacy
import (
"os"
"fmt"
"os"
"syscall"
"unsafe"
)
@@ -36,6 +36,7 @@ type reparseDataBuffer struct {
// GenericReparseBuffer
reparseBuffer byte
}
const (
FSCTL_GET_REPARSE_POINT = 0x900A8
MAXIMUM_REPARSE_DATA_BUFFER_SIZE = 16 * 1024
@@ -75,17 +76,17 @@ func Readlink(path string) (isRegular bool, s string, err error) {
data := (*symbolicLinkReparseBuffer)(unsafe.Pointer(&rdb.reparseBuffer))
p := (*[0xffff]uint16)(unsafe.Pointer(&data.PathBuffer[0]))
if data.PrintNameLength > 0 {
s = syscall.UTF16ToString(p[data.PrintNameOffset/2 : (data.PrintNameLength + data.PrintNameOffset)/2])
s = syscall.UTF16ToString(p[data.PrintNameOffset/2 : (data.PrintNameLength+data.PrintNameOffset)/2])
} else {
s = syscall.UTF16ToString(p[data.SubstituteNameOffset/2 : (data.SubstituteNameLength + data.SubstituteNameOffset)/2])
s = syscall.UTF16ToString(p[data.SubstituteNameOffset/2 : (data.SubstituteNameLength+data.SubstituteNameOffset)/2])
}
case IO_REPARSE_TAG_MOUNT_POINT:
data := (*mountPointReparseBuffer)(unsafe.Pointer(&rdb.reparseBuffer))
p := (*[0xffff]uint16)(unsafe.Pointer(&data.PathBuffer[0]))
if data.PrintNameLength > 0 {
s = syscall.UTF16ToString(p[data.PrintNameOffset/2 : (data.PrintNameLength + data.PrintNameOffset)/2])
s = syscall.UTF16ToString(p[data.PrintNameOffset/2 : (data.PrintNameLength+data.PrintNameOffset)/2])
} else {
s = syscall.UTF16ToString(p[data.SubstituteNameOffset/2 : (data.SubstituteNameLength + data.SubstituteNameOffset)/2])
s = syscall.UTF16ToString(p[data.SubstituteNameOffset/2 : (data.SubstituteNameLength+data.SubstituteNameOffset)/2])
}
case IO_REPARSE_TAG_DEDUP:
return true, "", nil
@@ -103,7 +104,7 @@ func GetOwner(entry *Entry, fileInfo *os.FileInfo) {
entry.GID = -1
}
func SetOwner(fullPath string, entry *Entry, fileInfo *os.FileInfo) (bool) {
func SetOwner(fullPath string, entry *Entry, fileInfo *os.FileInfo) bool {
return true
}