Add small backoff jitter to retrriable requests to avoid stampeding herd problem.

Karl Rieb · Karl Rieb · commit 85ffb864c8b0 · 2016-07-01T13:20:36.000-07:00
diff --git a/src/main/java/com/dropbox/core/DbxRequestUtil.java b/src/main/java/com/dropbox/core/DbxRequestUtil.java
@@ -13,6 +13,7 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
+import java.util.Random;
 
 import com.dropbox.core.stone.StoneSerializer;
 import com.dropbox.core.http.HttpRequestor;
@@ -27,6 +28,7 @@
 /*>>> import checkers.nullness.quals.Nullable; */
 
 public final class DbxRequestUtil {
+    private static final Random RAND = new Random();
 
     public static String encodeUrlParam(String s) {
         try {
@@ -442,6 +444,10 @@ public static <T, E extends Throwable> T runAndRetry(int maxRetries, RequestMake
                 throw thrown;
             }
 
+            // add a random jitter to the backoff to avoid stampeding herd. This is especially
+            // useful for ServerExceptions, where backoff is 0.
+            backoff += RAND.nextInt(1000);
+
             if (backoff > 0L) {
                 try {
                     Thread.sleep(backoff);
diff --git a/src/main/java/com/dropbox/core/v2/DbxRawClientV2.java b/src/main/java/com/dropbox/core/v2/DbxRawClientV2.java
@@ -28,6 +28,7 @@
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
+import java.util.Random;
 
 /*>>> import checkers.nullness.quals.NonNull; */
 /*>>> import checkers.nullness.quals.Nullable; */
@@ -49,6 +50,7 @@ public abstract class DbxRawClientV2 {
     // The HTTP status codes returned for errors specific to particular API calls.
     private static final List<Integer> FUNCTION_SPECIFIC_ERROR_CODES = Arrays.asList(403, 404, 409);
     private static final JsonFactory JSON = new JsonFactory();
+    private static final Random RAND = new Random();
 
     private final DbxRequestConfig requestConfig;
     private final DbxHost host;
@@ -259,21 +261,26 @@ private static <T> T executeRetriable(int maxRetries, RetriableExecution<T> exec
             } catch (RetryException ex) {
                 if (retries < maxRetries) {
                     ++retries;
-                    sleepQuietly(ex.getBackoffMillis());
+                    sleepQuietlyWithJitter(ex.getBackoffMillis());
                 } else {
                     throw ex;
                 }
             }
         }
     }
 
-    private static void sleepQuietly(long millis) {
-        if (millis <= 0) {
+    private static void sleepQuietlyWithJitter(long millis) {
+        // add a small jitter to the sleep to avoid stampeding herd problem, especially when millis
+        // is 0.
+        long jitter = RAND.nextInt(1000);
+        long sleepMillis = millis + jitter;
+
+        if (sleepMillis <= 0) {
             return;
         }
 
         try {
-            Thread.sleep(millis);
+            Thread.sleep(sleepMillis);
         } catch (InterruptedException ex) {
             // preserve interrupt
             Thread.currentThread().interrupt();
