S3: Use (custom) pacer, to retry operations when reasonable - fixes #2503

lib/pacer/pacer.go

@@ -59,6 +59,12 @@ const (
 	//
 	// See https://developers.google.com/drive/v2/web/handle-errors#exponential-backoff
 	GoogleDrivePacer
+
+	// S3Pacer is a specialised pacer for S3
+	//
+	// It is basically the defaultPacer, but allows the sleep time to go to 0
+	// when things are going well.
+	S3Pacer
 )
 
 // Paced is a function which is called by the Call and CallNoRetry
@@ -185,6 +191,8 @@ func (p *Pacer) SetPacer(t Type) *Pacer {
 		p.calculatePace = p.acdPacer
 	case GoogleDrivePacer:
 		p.calculatePace = p.drivePacer
+	case S3Pacer:
+		p.calculatePace = p.s3Pacer
 	default:
 		p.calculatePace = p.defaultPacer
 	}
@@ -309,6 +317,46 @@ func (p *Pacer) drivePacer(retry bool) {
 	}
 }
 
+// s3Pacer implements a pacer compatible with our expectations of S3, where it tries to not
+// delay at all between successful calls, but backs off in the default fashion in response
+// to any errors.
+// The assumption is that errors should be exceedingly rare (S3 seems to have largely solved
+//  the sort of scability questions rclone is likely to run into), and in the happy case
+//  it can handle calls with no delays between them.
+//
+// Basically defaultPacer, but with some handling of sleepTime going to/from 0ms
+// Ignores minSleep entirely
+//
+// Call with p.mu held
+func (p *Pacer) s3Pacer(retry bool) {
+	oldSleepTime := p.sleepTime
+	if retry {
+		if p.attackConstant == 0 {
+			p.sleepTime = p.maxSleep
+		} else {
+			if p.sleepTime == 0 {
+				p.sleepTime = p.minSleep
+			} else {
+				p.sleepTime = (p.sleepTime << p.attackConstant) / ((1 << p.attackConstant) - 1)
+			}
+		}
+		if p.sleepTime > p.maxSleep {
+			p.sleepTime = p.maxSleep
+		}
+		if p.sleepTime != oldSleepTime {
+			fs.Debugf("pacer", "Rate limited, increasing sleep to %v", p.sleepTime)
+		}
+	} else {
+		p.sleepTime = (p.sleepTime<<p.decayConstant - p.sleepTime) >> p.decayConstant
+		if p.sleepTime < p.minSleep {
+			p.sleepTime = 0
+		}
+		if p.sleepTime != oldSleepTime {
+			fs.Debugf("pacer", "Reducing sleep to %v", p.sleepTime)
+		}
+	}
+}
+
 // endCall implements the pacing algorithm
 //
 // This should calculate a new sleepTime.  It takes a boolean as to

lib/pacer/pacer_test.go

@@ -340,6 +340,32 @@ func TestGoogleDrivePacer(t *testing.T) {
 	}
 }
 
+func TestS3Pacer(t *testing.T) {
+	p := New().SetMinSleep(10 * time.Millisecond).SetPacer(S3Pacer).SetMaxSleep(time.Second).SetDecayConstant(2)
+	for _, test := range []struct {
+		in    time.Duration
+		retry bool
+		want  time.Duration
+	}{
+		{0, true, 10 * time.Millisecond},                      //Things were going ok, we failed once, back off to minSleep
+		{10 * time.Millisecond, true, 20 * time.Millisecond},  //Another fail, double the backoff
+		{10 * time.Millisecond, false, 0},                     //Things start going ok when we're at minSleep; should result in no sleep
+		{12 * time.Millisecond, false, 0},                     //*near* minsleep and going ok, decay would take below minSleep, should go to 0
+		{0, false, 0},                                         //Things have been going ok; not retrying should keep sleep at 0
+		{time.Second, true, time.Second},                      //Check maxSleep is enforced
+		{(3 * time.Second) / 4, true, time.Second},            //Check attack heading to maxSleep doesn't exceed maxSleep
+		{time.Second, false, 750 * time.Millisecond},          //Check decay from maxSleep
+		{48 * time.Millisecond, false, 36 * time.Millisecond}, //Check simple decay above minSleep
+	} {
+		p.sleepTime = test.in
+		p.s3Pacer(test.retry)
+		got := p.sleepTime
+		if got != test.want {
+			t.Errorf("bad sleep for %v with retry %v:  want %v got %v", test.in, test.retry, test.want, got)
+		}
+	}
+}
+
 func TestEndCall(t *testing.T) {
 	p := New().SetMaxConnections(5)
 	emptyTokens(p)