duplicacy/duplicacy_config.go

// Copyright (c) Acrosync LLC. All rights reserved.
// Licensed under the Fair Source License 0.9 (https://fair.io/)
// User Limitation: 5 users

package duplicacy

import (
    "encoding/json"
    "bytes"
    "os"
    "fmt"
    "hash"
    "runtime"
    "runtime/debug"
    "sync/atomic"
    "crypto/rand"
    "crypto/hmac"
    "crypto/sha256"
    "encoding/hex"

    blake2 "github.com/minio/blake2b-simd"
)

// If encryption is turned off, use this key for HMAC-SHA256 or chunk ID generation etc.
var DEFAULT_KEY = []byte("duplicacy")

// The new default compression level is 100.  However, in the early versions we use the
// standard zlib levels of -1 to 9.
var DEFAULT_COMPRESSION_LEVEL = 100

type Config struct {
    CompressionLevel int                   `json:"compression-level"`
    AverageChunkSize int                   `json:"average-chunk-size"`
    MaximumChunkSize int                   `json:"max-chunk-size"`
    MinimumChunkSize int                   `json:"min-chunk-size"`

    ChunkSeed []byte                       `json:"chunk-seed"`

    // Use HMAC-SHA256(hashKey, plaintext) as the chunk hash.
    // Use HMAC-SHA256(idKey, chunk hash) as the file name of the chunk
    // For chunks, use HMAC-SHA256(chunkKey, chunk hash) as the encryption key
    // For files, use HMAC-SHA256(fileKey, file path) as the encryption key

    // the HMAC-SHA256 key of the chunk data
    HashKey  []byte                        `json:"-"`

    // used to generate an id from the chunk hash
    IDKey    []byte                        `json:"-"`

    // for encrypting a chunk
    ChunkKey []byte                        `json:"-"`

    // for encrypting a non-chunk file
    FileKey  []byte                        `json:"-"`

    chunkPool chan *Chunk                  `json:"-"`
    numberOfChunks int32
}

    // Create an alias to avoid recursive calls on Config.MarshalJSON
type aliasedConfig Config

type jsonableConfig struct {
    *aliasedConfig
    ChunkSeed string `json:"chunk-seed"`
    HashKey   string `json:"hash-key"`
    IDKey     string `json:"id-key"`
    ChunkKey  string `json:"chunk-key"`
    FileKey   string `json:"file-key"`
}

func (config *Config) MarshalJSON() ([] byte, error) {

    return json.Marshal(&jsonableConfig {
        aliasedConfig: (*aliasedConfig)(config),
        ChunkSeed:     hex.EncodeToString(config.ChunkSeed),
        HashKey:       hex.EncodeToString(config.HashKey),
        IDKey:         hex.EncodeToString(config.IDKey),
        ChunkKey:      hex.EncodeToString(config.ChunkKey),
        FileKey:       hex.EncodeToString(config.FileKey),
    })
}

func (config *Config) UnmarshalJSON(description []byte) (err error) {

    aliased := &jsonableConfig {
        aliasedConfig: (*aliasedConfig)(config),
    }

    if err = json.Unmarshal(description, &aliased); err != nil {
        return err
    }

    if config.ChunkSeed, err = hex.DecodeString(aliased.ChunkSeed); err != nil {
        return fmt.Errorf("Invalid representation of the chunk seed in the config")
    }
    if config.HashKey, err = hex.DecodeString(aliased.HashKey); err != nil {
        return fmt.Errorf("Invalid representation of the hash key in the config")
    }
    if config.IDKey, err = hex.DecodeString(aliased.IDKey); err != nil {
        return fmt.Errorf("Invalid representation of the id key in the config")
    }
    if config.ChunkKey, err = hex.DecodeString(aliased.ChunkKey); err != nil {
        return fmt.Errorf("Invalid representation of the chunk key in the config")
    }
    if config.FileKey, err = hex.DecodeString(aliased.FileKey); err != nil {
        return fmt.Errorf("Invalid representation of the file key in the config")
    }

    return nil
}

func (config *Config) IsCompatiableWith(otherConfig *Config) bool {

    return config.CompressionLevel == otherConfig.CompressionLevel &&
           config.AverageChunkSize == otherConfig.AverageChunkSize &&
           config.MaximumChunkSize == otherConfig.MaximumChunkSize &&
           config.MinimumChunkSize == otherConfig.MinimumChunkSize &&
           bytes.Equal(config.ChunkSeed, otherConfig.ChunkSeed)    &&
           bytes.Equal(config.HashKey, otherConfig.HashKey)
}

func (config *Config) Print() {

    LOG_INFO("CONFIG_INFO", "Compression level: %d", config.CompressionLevel)
    LOG_INFO("CONFIG_INFO", "Average chunk size: %d", config.AverageChunkSize)
    LOG_INFO("CONFIG_INFO", "Maximum chunk size: %d", config.MaximumChunkSize)
    LOG_INFO("CONFIG_INFO", "Minimum chunk size: %d", config.MinimumChunkSize)
    LOG_INFO("CONFIG_INFO", "Chunk seed: %x", config.ChunkSeed)
}

func CreateConfigFromParameters(compressionLevel int, averageChunkSize int, maximumChunkSize int, mininumChunkSize int,
                  isEncrypted bool, copyFrom *Config) (config *Config) {

    config = &Config {
        CompressionLevel: compressionLevel,
        AverageChunkSize: averageChunkSize,
        MaximumChunkSize: maximumChunkSize,
        MinimumChunkSize: mininumChunkSize,
    }

    if isEncrypted {
        // Randomly generate keys
        keys := make([]byte, 32 * 5)
        _, err := rand.Read(keys)
        if err != nil {
            LOG_ERROR("CONFIG_KEY", "Failed to generate random keys: %v", err)
            return nil
        }

        config.ChunkSeed = keys[:32]
        config.HashKey   = keys[32:64]
        config.IDKey     = keys[64:96]
        config.ChunkKey  = keys[96:128]
        config.FileKey   = keys[128:]
    } else {
        config.ChunkSeed = DEFAULT_KEY
        config.HashKey   = DEFAULT_KEY
        config.IDKey     = DEFAULT_KEY
    }

    if copyFrom != nil {
        config.CompressionLevel = copyFrom.CompressionLevel

        config.AverageChunkSize = copyFrom.AverageChunkSize
        config.MaximumChunkSize = copyFrom.MaximumChunkSize
        config.MinimumChunkSize = copyFrom.MinimumChunkSize

        config.ChunkSeed        = copyFrom.ChunkSeed
        config.HashKey          = copyFrom.HashKey
    }

    config.chunkPool = make(chan *Chunk, runtime.NumCPU() * 16)

    return config
}

func CreateConfig() (config *Config) {
    return &Config {
        HashKey: DEFAULT_KEY,
        IDKey: DEFAULT_KEY,
        CompressionLevel: DEFAULT_COMPRESSION_LEVEL,
        chunkPool: make(chan *Chunk, runtime.NumCPU() * 16),
    }
}

func (config *Config) GetChunk() (chunk *Chunk) {
    select {
    case chunk = <- config.chunkPool :
    default:
        numberOfChunks := atomic.AddInt32(&config.numberOfChunks, 1)
        if numberOfChunks >= int32(runtime.NumCPU() * 16) {
            LOG_WARN("CONFIG_CHUNK", "%d chunks have been allocated", numberOfChunks)
            if _, found := os.LookupEnv("DUPLICACY_CHUNK_DEBUG"); found {
                debug.PrintStack()
            }
        }
        chunk = CreateChunk(config, true)
    }
    return chunk
}

func (config *Config) PutChunk(chunk *Chunk){

    if chunk == nil {
        return
    }

    select {
    case config.chunkPool <- chunk:
    default:
        LOG_INFO("CHUNK_BUFFER", "Discarding a free chunk due to a full pool")
    }
}

func (config *Config) NewKeyedHasher(key []byte) hash.Hash {
    if config.CompressionLevel == DEFAULT_COMPRESSION_LEVEL {
        hasher, err := blake2.New(&blake2.Config{ Size: 32, Key:key })
        if err != nil {
            LOG_ERROR("HASH_KEY", "Invalid hash key: %x", key)
        }
        return hasher
    } else {
        return hmac.New(sha256.New, key)
    }
}

func (config *Config) NewFileHasher() hash.Hash {
    if config.CompressionLevel == DEFAULT_COMPRESSION_LEVEL {
        hasher, _ := blake2.New(&blake2.Config{ Size: 32 })
        return hasher
    } else {
        return sha256.New()
    }
}

// Calculate the file hash using the corresponding hasher
func (config *Config) ComputeFileHash(path string, buffer []byte) string {

    file, err := os.Open(path)
      if err != nil {
        return ""
      }

    hasher := config.NewFileHasher()
    defer file.Close()

    count := 1
    for count > 0 {
        count, err = file.Read(buffer)
        hasher.Write(buffer[:count])
    }

      return hex.EncodeToString(hasher.Sum(nil))
}

// GetChunkIDFromHash creates a chunk id from the chunk hash.  The chunk id will be used as the name of the chunk
// file, so it is publicly exposed.  The chunk hash is the HMAC-SHA256 of what is contained in the chunk and should
// never be exposed.
func (config *Config) GetChunkIDFromHash(hash string) string {
    hasher := config.NewKeyedHasher(config.IDKey)
    hasher.Write([]byte(hash))
    return hex.EncodeToString(hasher.Sum(nil))
}

func DownloadConfig(storage Storage, password string) (config *Config, isEncrypted bool, err error) {
    // Although the default key is passed to the function call the key is not actually used since there is no need to
    // calculate the hash or id of the config file.
    configFile := CreateChunk(CreateConfig(), true)

    exist, _, _, err := storage.GetFileInfo(0, "config")
    if err != nil {
        return nil, false, err
    }

    if !exist {
        return nil, false, nil
    }

    err = storage.DownloadFile(0, "config", configFile)
    if err != nil {
        return nil, false, err
    }

    var masterKey []byte

    if len(password) > 0 {
        masterKey = GenerateKeyFromPassword(password)

        // Decrypt the config file.  masterKey == nil means no encryption.
        err = configFile.Decrypt(masterKey, "")
        if err != nil {
            return nil, false, fmt.Errorf("Failed to retrieve the config file: %v", err)
        }
    }

    config = CreateConfig()

    err = json.Unmarshal(configFile.GetBytes(), config)

    if err != nil {
        if bytes.Equal(configFile.GetBytes()[:9], []byte("duplicacy")) {
            return nil, true, fmt.Errorf("The storage is likely to have been initialized with a password before")
        } else {
            return nil, false, fmt.Errorf("Failed to parse the config file: %v", err)
        }
    }

    return config, false, nil

}

func UploadConfig(storage Storage, config *Config, password string) (bool) {

    // This is the key to encrypt the config file.
    var masterKey []byte

    if len(password) > 0 {

        if len(password) < 8 {
            LOG_ERROR("CONFIG_PASSWORD", "The password must be at least 8 characters")
            return false
        }

        masterKey = GenerateKeyFromPassword(password)
    }

    description, err := json.MarshalIndent(config, "", "    ")
    if err != nil {
        LOG_ERROR("CONFIG_MARSHAL", "Failed to marshal the config: %v", err)
        return false
    }

    // Although the default key is passed to the function call the key is not actually used since there is no need to
    // calculate the hash or id of the config file.
    chunk := CreateChunk(CreateConfig(), true)
    chunk.Write(description)

    if len(password) > 0 {
        // Encrypt the config file with masterKey.  If masterKey is nil then no encryption is performed.
        err = chunk.Encrypt(masterKey, "")

        if err != nil {
            LOG_ERROR("CONFIG_CREATE", "Failed to create the config file: %v", err)
            return false
        }
    }

    err = storage.UploadFile(0, "config", chunk.GetBytes())
    if err != nil {
        LOG_ERROR("CONFIG_INIT", "Failed to configure the storage: %v", err)
        return false
    }

    if IsTracing() {
        config.Print()
    }

    for _, subDir := range []string {"chunks", "snapshots"} {
        err = storage.CreateDirectory(0, subDir)
        if err != nil {
            LOG_ERROR("CONFIG_MKDIR", "Failed to create storage subdirectory: %v", err)
        }
    }

    return true
}

// ConfigStorage makes the general storage space available for storing duplicacy format snapshots.  In essence,
// it simply creates a file named 'config' that stores various parameters as well as a set of keys if encryption
// is enabled.
func ConfigStorage(storage Storage, compressionLevel int, averageChunkSize int, maximumChunkSize int,
                   minimumChunkSize int, password string, copyFrom *Config) bool {

    exist, _, _, err := storage.GetFileInfo(0, "config")
    if err != nil {
        LOG_ERROR("CONFIG_INIT", "Failed to check if there is an existing config file: %v", err)
        return false
    }

    if exist {
        LOG_INFO("CONFIG_EXIST", "The storage has already been configured")
        return false
    }


    config := CreateConfigFromParameters(compressionLevel, averageChunkSize, maximumChunkSize, minimumChunkSize, len(password) > 0,
                           copyFrom)
    if config == nil {
        return false
    }

    return UploadConfig(storage, config, password)
}