Exponential Backoff should work now.

Maximum sleep is 32*2.

src/duplicacy_gcdstorage.go

@@ -30,12 +30,12 @@ type GCDStorage struct {
 	service         *drive.Service
 	idCache         map[string]string
 	idCacheLock     *sync.Mutex
-    backoffs []int
+	backoffs        []float64
+	backoffsRetries []int
 
 	isConnected     bool
 	numberOfThreads int
 	TestMode        bool
-
 }
 
 type GCDConfig struct {
@@ -46,11 +46,27 @@ type GCDConfig struct {
 }
 
 func (storage *GCDStorage) shouldRetry(threadIndex int, err error) (bool, error) {
+	const LIMIT_BACKOFF_TIME = 32
+	const MAX_NUMBER_OF_RETRIES = 15
+	minimumSleepRatio := 0.1
+	maximumSleepRatio := 0.2
+	minimumSleep := float64(storage.numberOfThreads) * minimumSleepRatio
+	maximumSleep := float64(storage.numberOfThreads) * maximumSleepRatio
+	rand.Seed(time.Now().UnixNano()) 			// unsure if this is needed
 
 	retry := false
 	message := ""
 	if err == nil {
-        storage.backoffs[threadIndex] = 1
+		/**
+		  logic for said calculus is here: https://stackoverflow.com/questions/1527803/generating-random-whole-numbers-in-javascript-in-a-specific-range
+		  i chose 0.1*thread number as a minimum sleep time
+		      and 0.2*thread number as a maximum sleep time
+		  for the first sleep of the first backoff of the threads.
+		  This would mean that both when the program is started, and when multiple threads retry, google won't be ddosed :^)
+		*/
+
+		storage.backoffs[threadIndex] = rand.Float64()*(maximumSleep-minimumSleep+1) + minimumSleep
+		storage.backoffsRetries[threadIndex] = 0
 		return false, nil
 	} else if e, ok := err.(*googleapi.Error); ok {
 		if 500 <= e.Code && e.Code < 600 {
@@ -63,8 +79,9 @@ func (storage *GCDStorage) shouldRetry(threadIndex int, err error) (bool, error)
 			retry = true
 		} else if e.Code == 403 {
 			// User Rate Limit Exceeded
-            message = "User Rate Limit Exceeded"
+			message = e.Message 	// "User Rate Limit Exceeded"
 			retry = true
+
 		} else if e.Code == 401 {
 			// Only retry on authorization error when storage has been connected before
 			if storage.isConnected {
@@ -84,19 +101,27 @@ func (storage *GCDStorage) shouldRetry(threadIndex int, err error) (bool, error)
 		retry = err.Temporary()
 	}
 
-    if !retry || storage.backoffs[threadIndex] >= 256 {
-        storage.backoffs[threadIndex] = 1
+	if !retry || storage.backoffsRetries[threadIndex] >= MAX_NUMBER_OF_RETRIES {
+		LOG_INFO("GCD_RETRY", "Thread: %03d. Maximum number of retries reached. Backoff time: %.2f. Number of retries: %d", threadIndex, storage.backoffs[threadIndex], storage.backoffsRetries[threadIndex])
+		storage.backoffs[threadIndex] = rand.Float64()*(maximumSleep-minimumSleep+1) + minimumSleep
+		storage.backoffsRetries[threadIndex] = 0
 		return false, err
 	}
 
-    delay := float32(storage.backoffs[threadIndex]) * rand.Float32()
-    LOG_DEBUG("GCD_RETRY", "%s; retrying after %.2f seconds", message, delay)
-    time.Sleep(time.Duration(float32(storage.backoffs[threadIndex]) * float32(time.Second)))
-    storage.backoffs[threadIndex] *= 2
+	delay := storage.backoffs[threadIndex]*rand.Float64() + storage.backoffs[threadIndex]*rand.Float64()
+	LOG_INFO("GCD_RETRY", "Thread: %03d. Message: %s. Retrying after %.2f seconds. current backoff: %.2f.  Number of retries: %d", threadIndex, message, delay, storage.backoffs[threadIndex], storage.backoffsRetries[threadIndex])
+	time.Sleep(time.Duration(delay * float64(time.Second)))
+
+	if storage.backoffs[threadIndex] < LIMIT_BACKOFF_TIME {
+			storage.backoffs[threadIndex] *= 2.0
+	} else {
+		storage.backoffs[threadIndex] = LIMIT_BACKOFF_TIME
+		storage.backoffsRetries[threadIndex] += 1
+	}
 	return true, nil
 }
 
-func (storage *GCDStorage) convertFilePath(filePath string) (string) {
+func (storage *GCDStorage) convertFilePath(filePath string) string {
 	if strings.HasPrefix(filePath, "chunks/") && strings.HasSuffix(filePath, ".fsl") {
 		return "fossils/" + filePath[len("chunks/"):len(filePath)-len(".fsl")]
 	}
@@ -174,7 +199,6 @@ func (storage *GCDStorage) listFiles(threadIndex int, parentID string, listFiles
 		}
 	}
 
-
 	return files, nil
 }
 
@@ -276,7 +300,8 @@ func CreateGCDStorage(tokenFile string, storagePath string, threads int) (storag
 		numberOfThreads: threads,
 		idCache:         make(map[string]string),
 		idCacheLock:     &sync.Mutex{},
-        backoffs: make([]int, threads),
+		backoffs:        make([]float64, threads),
+		backoffsRetries: make([]int, threads),
 	}
 
 	storagePathID, err := storage.getIDFromPath(0, storagePath)