jgaunt/duplicacy
1
0
Fork 0
mirror of https://github.com/gilbertchen/duplicacy synced 2025-12-14 23:33:18 +00:00
Code Issues Releases Wiki Activity

Merge branch 'master' into tbp/google_rate_limit_exceeded

Browse Source 
* master:
  Run goimports on all source files
  restored original stats output, enabled option to switch to tabular
  added -tabular to check options
  moving "func min" to MinInt in utils
  reorder check -stats columns
  print additional, table-formatted stats for CHECK

# Conflicts:
#	src/duplicacy_gcdstorage.go
This commit is contained in:
TheBestPessimist
2017-09-21 08:15:02 +03:00
parent d20ea41cd0 923cd0aa63
commit 46917ddf6b
49 changed files with 13156 additions and 13114 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
src/duplicacy_gcdstorage.go
View File

@@ -5,18 +5,18 @@
package duplicacy
import (
"io"
"fmt"
"net"
"path"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"math/rand"
"net"
"net/http"
"net/url"
"path"
"strings"
"sync"
"time"
"sync"
"strings"
"net/http"
"net/url"
"io/ioutil"
"math/rand"
"encoding/json"
"golang.org/x/net/context"
"golang.org/x/oauth2"
@@ -28,10 +28,10 @@ import (
type GCDStorage struct {
RateLimitedStorage
service *drive.Service
idCache map[string]string
idCacheLock *sync.Mutex
backoffs []float64
service *drive.Service
idCache map[string]string
idCacheLock *sync.Mutex
backoffs []float64
backoffsRetries []int
isConnected bool
@@ -101,22 +101,20 @@ func (storage *GCDStorage) shouldRetry(threadIndex int, err error) (bool, error)
return false, err
}
if storage.backoffs[threadIndex] < LIMIT_BACKOFF_TIME {
if storage.backoffs[threadIndex] < LIMIT_BACKOFF_TIME {
storage.backoffs[threadIndex] *= 2.0
} else {
storage.backoffs[threadIndex] = LIMIT_BACKOFF_TIME
storage.backoffsRetries[threadIndex] += 1
}
delay := storage.backoffs[threadIndex]*rand.Float64() + storage.backoffs[threadIndex]*rand.Float64()
if storage.backoffs[threadIndex] >= LIMIT_BACKOFF_TIME {
delay := storage.backoffs[threadIndex] * rand.Float64() + storage.backoffs[threadIndex]*rand.Float64()
if storage.backoffs[threadIndex] >= LIMIT_BACKOFF_TIME {
callerChain:=findCallerChain()
LOG_INFO("GCD_RETRY", "Thread: %3d. Message: %s. Retrying after %6.2f seconds. Current backoff: %6.2f. Number of retries: %2d. caller chain: %s", threadIndex, message, delay, storage.backoffs[threadIndex], storage.backoffsRetries[threadIndex], callerChain)
LOG_INFO("GCD_RETRY", "Thread: %3d. Message:%s. Retrying after %6.2f seconds. Current backoff: %6.2f. Number of retries: %2d. caller chain: %s", threadIndex, message, delay, storage.backoffs[threadIndex], storage.backoffsRetries[threadIndex], callerChain)
}
time.Sleep(time.Duration(delay * float64(time.Second)))
time.Sleep(time.Duration(delay * float64(time.Second)))
return true, nil
return true, nil
}
func findCallerChain() string {
@@ -336,10 +334,10 @@ func CreateGCDStorage(tokenFile string, storagePath string, threads int) (storag
storage.backoffs[b] = 0.1 * float64(storage.numberOfThreads) // at the first error, we should still sleep some amount
}
storagePathID, err := storage.getIDFromPath(0, storagePath)
if err != nil {
return nil, err
}
storagePathID, err := storage.getIDFromPath(0, storagePath)
if err != nil {
return nil, err
}
storage.idCache[""] = storagePathID
@@ -671,16 +669,16 @@ func (storage *GCDStorage) UploadFile(threadIndex int, filePath string, content
// If a local snapshot cache is needed for the storage to avoid downloading/uploading chunks too often when
// managing snapshots.
func (storage *GCDStorage) IsCacheNeeded() (bool) { return true }
func (storage *GCDStorage) IsCacheNeeded() bool { return true }
// If the 'MoveFile' method is implemented.
func (storage *GCDStorage) IsMoveFileImplemented() (bool) { return true }
func (storage *GCDStorage) IsMoveFileImplemented() bool { return true }
// If the storage can guarantee strong consistency.
func (storage *GCDStorage) IsStrongConsistent() (bool) { return false }
func (storage *GCDStorage) IsStrongConsistent() bool { return false }
// If the storage supports fast listing of files names.
func (storage *GCDStorage) IsFastListing() (bool) { return true }
func (storage *GCDStorage) IsFastListing() bool { return true }
// Enable the test mode.
func (storage *GCDStorage) EnableTestMode() { storage.TestMode = true }