jgaunt/duplicacy
1
0
Fork 0
mirror of https://github.com/gilbertchen/duplicacy synced 2025-12-15 15:53:26 +00:00
Code Issues Releases Wiki Activity

Add a method for debugging which shows the method call chains, to find out where most of the retries come from

Browse Source
This commit is contained in:
TheBestPessimist
2017-09-20 18:52:46 +03:00
parent ef19a3705f
commit d20ea41cd0
1 changed files with 63 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
src/duplicacy_gcdstorage.go
View File

@@ -22,6 +22,7 @@ import (
"golang.org/x/oauth2"
"google.golang.org/api/drive/v3"
"google.golang.org/api/googleapi"
"runtime"
)
type GCDStorage struct {
@@ -46,7 +47,7 @@ type GCDConfig struct {
}
func (storage *GCDStorage) shouldRetry(threadIndex int, err error) (bool, error) {
const LIMIT_BACKOFF_TIME = 32
const LIMIT_BACKOFF_TIME = 64
const MAX_NUMBER_OF_RETRIES = 15
minimumSleepRatio := 0.1
maximumSleepRatio := 0.2
@@ -57,15 +58,7 @@ func (storage *GCDStorage) shouldRetry(threadIndex int, err error) (bool, error)
retry := false
message := ""
if err == nil {
/**
logic for said calculus is here: https://stackoverflow.com/questions/1527803/generating-random-whole-numbers-in-javascript-in-a-specific-range
i chose 0.1*thread number as a minimum sleep time
and 0.2*thread number as a maximum sleep time
for the first sleep of the first backoff of the threads.
This would mean that both when the program is started, and when multiple threads retry, google won't be ddosed :^)
*/
storage.backoffs[threadIndex] = rand.Float64()*(maximumSleep-minimumSleep+1) + minimumSleep
storage.backoffs[threadIndex] = computeInitialBackoff(minimumSleep, maximumSleep)
storage.backoffsRetries[threadIndex] = 0
return false, nil
} else if e, ok := err.(*googleapi.Error); ok {
@@ -103,7 +96,7 @@ func (storage *GCDStorage) shouldRetry(threadIndex int, err error) (bool, error)
if !retry || storage.backoffsRetries[threadIndex] >= MAX_NUMBER_OF_RETRIES {
LOG_INFO("GCD_RETRY", "Thread: %03d. Maximum number of retries reached. Backoff time: %.2f. Number of retries: %d", threadIndex, storage.backoffs[threadIndex], storage.backoffsRetries[threadIndex])
storage.backoffs[threadIndex] = rand.Float64()*(maximumSleep-minimumSleep+1) + minimumSleep
storage.backoffs[threadIndex] = computeInitialBackoff(minimumSleep, maximumSleep)
storage.backoffsRetries[threadIndex] = 0
return false, err
}
@@ -116,12 +109,46 @@ func (storage *GCDStorage) shouldRetry(threadIndex int, err error) (bool, error)
}
delay := storage.backoffs[threadIndex]*rand.Float64() + storage.backoffs[threadIndex]*rand.Float64()
LOG_INFO("GCD_RETRY", "Thread: %03d. Message: %s. Retrying after %.2f seconds. Current backoff: %.2f. Number of retries: %d", threadIndex, message, delay, storage.backoffs[threadIndex], storage.backoffsRetries[threadIndex])
if storage.backoffs[threadIndex] >= LIMIT_BACKOFF_TIME {
callerChain:=findCallerChain()
LOG_INFO("GCD_RETRY", "Thread: %3d. Message: %s. Retrying after %6.2f seconds. Current backoff: %6.2f. Number of retries: %2d. caller chain: %s", threadIndex, message, delay, storage.backoffs[threadIndex], storage.backoffsRetries[threadIndex], callerChain)
}
time.Sleep(time.Duration(delay * float64(time.Second)))
return true, nil
}
func findCallerChain() string {
callerStack := ""
pc := make([]uintptr, 15)
n := runtime.Callers(1, pc)
frames := runtime.CallersFrames(pc[:n])
for {
frame, more := frames.Next()
if strings.Contains(frame.File, "runtime/") {
break
}
callerStack += "->" + frame.Function
if !more {
break
}
}
return callerStack
}
/*
logic for said calculus is here: https://stackoverflow.com/questions/1527803/generating-random-whole-numbers-in-javascript-in-a-specific-range
chose 0.1*thread number as a minimum sleep time
and 0.2*thread number as a maximum sleep time
for the first sleep of the first backoff of the threads.
This would mean that both when the program is started, and when multiple threads retry, google won't be ddosed :^)
*/
func computeInitialBackoff(minimumSleep float64, maximumSleep float64) float64 {
return rand.Float64()*(maximumSleep-minimumSleep+1) + minimumSleep
}
func (storage *GCDStorage) convertFilePath(filePath string) string {
if strings.HasPrefix(filePath, "chunks/") && strings.HasSuffix(filePath, ".fsl") {
return "fossils/" + filePath[len("chunks/"):len(filePath)-len(".fsl")]
@@ -296,13 +323,13 @@ func CreateGCDStorage(tokenFile string, storagePath string, threads int) (storag
return nil, err
}
storage = &GCDStorage{
storage = &GCDStorage {
service: service,
numberOfThreads: threads,
idCache: make(map[string]string),
idCacheLock: &sync.Mutex{},
backoffs: make([]float64, threads),
backoffsRetries: make([]int, threads),
backoffsRetries:make([]int, threads),
}
for b := range storage.backoffs {