Thomas Vachuska
Committed by Gerrit Code Review

Increased number of retries and added variable delay to better withstand StorageException timeout.

Change-Id: Ic33d4e66e2787c717f7e3c90879d725a6e3e74d7
...@@ -62,8 +62,8 @@ import java.util.function.Function; ...@@ -62,8 +62,8 @@ import java.util.function.Function;
62 62
63 import static com.google.common.io.ByteStreams.toByteArray; 63 import static com.google.common.io.ByteStreams.toByteArray;
64 import static java.util.concurrent.TimeUnit.MILLISECONDS; 64 import static java.util.concurrent.TimeUnit.MILLISECONDS;
65 -import static org.onlab.util.Tools.delay;
66 import static org.onlab.util.Tools.groupedThreads; 65 import static org.onlab.util.Tools.groupedThreads;
66 +import static org.onlab.util.Tools.randomDelay;
67 import static org.onosproject.app.ApplicationEvent.Type.*; 67 import static org.onosproject.app.ApplicationEvent.Type.*;
68 import static org.onosproject.store.app.GossipApplicationStore.InternalState.*; 68 import static org.onosproject.store.app.GossipApplicationStore.InternalState.*;
69 import static org.onosproject.store.service.EventuallyConsistentMapEvent.Type.PUT; 69 import static org.onosproject.store.service.EventuallyConsistentMapEvent.Type.PUT;
...@@ -83,11 +83,10 @@ public class GossipApplicationStore extends ApplicationArchive ...@@ -83,11 +83,10 @@ public class GossipApplicationStore extends ApplicationArchive
83 83
84 private static final MessageSubject APP_BITS_REQUEST = new MessageSubject("app-bits-request"); 84 private static final MessageSubject APP_BITS_REQUEST = new MessageSubject("app-bits-request");
85 85
86 - private static final int MAX_LOAD_RETRIES = 3; 86 + private static final int MAX_LOAD_RETRIES = 5;
87 private static final int RETRY_DELAY_MS = 2_000; 87 private static final int RETRY_DELAY_MS = 2_000;
88 88
89 private static final int FETCH_TIMEOUT_MS = 10_000; 89 private static final int FETCH_TIMEOUT_MS = 10_000;
90 - private static final int LOAD_TIMEOUT_MS = 5_000;
91 90
92 public enum InternalState { 91 public enum InternalState {
93 INSTALLED, ACTIVATED, DEACTIVATED 92 INSTALLED, ACTIVATED, DEACTIVATED
...@@ -169,7 +168,7 @@ public class GossipApplicationStore extends ApplicationArchive ...@@ -169,7 +168,7 @@ public class GossipApplicationStore extends ApplicationArchive
169 } 168 }
170 } catch (Exception e) { 169 } catch (Exception e) {
171 log.warn("Unable to load application {} from disk; retrying", name); 170 log.warn("Unable to load application {} from disk; retrying", name);
172 - delay(RETRY_DELAY_MS); // FIXME: This is a deliberate hack; fix in Drake 171 + randomDelay(RETRY_DELAY_MS); // FIXME: This is a deliberate hack; fix in Drake
173 } 172 }
174 } 173 }
175 } 174 }
......
...@@ -16,7 +16,7 @@ import org.slf4j.Logger; ...@@ -16,7 +16,7 @@ import org.slf4j.Logger;
16 16
17 import java.util.Map; 17 import java.util.Map;
18 18
19 -import static org.onlab.util.Tools.delay; 19 +import static org.onlab.util.Tools.randomDelay;
20 import static org.slf4j.LoggerFactory.getLogger; 20 import static org.slf4j.LoggerFactory.getLogger;
21 21
22 /** 22 /**
...@@ -26,7 +26,7 @@ import static org.slf4j.LoggerFactory.getLogger; ...@@ -26,7 +26,7 @@ import static org.slf4j.LoggerFactory.getLogger;
26 @Service 26 @Service
27 public class ConsistentIdBlockStore implements IdBlockStore { 27 public class ConsistentIdBlockStore implements IdBlockStore {
28 28
29 - private static final int MAX_TRIES = 3; 29 + private static final int MAX_TRIES = 5;
30 private static final int RETRY_DELAY_MS = 2_000; 30 private static final int RETRY_DELAY_MS = 2_000;
31 31
32 private final Logger log = getLogger(getClass()); 32 private final Logger log = getLogger(getClass());
...@@ -63,7 +63,7 @@ public class ConsistentIdBlockStore implements IdBlockStore { ...@@ -63,7 +63,7 @@ public class ConsistentIdBlockStore implements IdBlockStore {
63 log.warn("Unable to allocate ID block due to {}; retrying...", 63 log.warn("Unable to allocate ID block due to {}; retrying...",
64 e.getMessage()); 64 e.getMessage());
65 exc = e; 65 exc = e;
66 - delay(RETRY_DELAY_MS); // FIXME: This is a deliberate hack; fix in Drake 66 + randomDelay(RETRY_DELAY_MS); // FIXME: This is a deliberate hack; fix in Drake
67 } 67 }
68 } 68 }
69 throw new IllegalStateException("Unable to allocate ID block", exc); 69 throw new IllegalStateException("Unable to allocate ID block", exc);
......
...@@ -39,6 +39,7 @@ import java.util.Arrays; ...@@ -39,6 +39,7 @@ import java.util.Arrays;
39 import java.util.Collection; 39 import java.util.Collection;
40 import java.util.Dictionary; 40 import java.util.Dictionary;
41 import java.util.List; 41 import java.util.List;
42 +import java.util.Random;
42 import java.util.concurrent.CompletableFuture; 43 import java.util.concurrent.CompletableFuture;
43 import java.util.concurrent.ExecutionException; 44 import java.util.concurrent.ExecutionException;
44 import java.util.concurrent.Future; 45 import java.util.concurrent.Future;
...@@ -62,6 +63,8 @@ public abstract class Tools { ...@@ -62,6 +63,8 @@ public abstract class Tools {
62 63
63 private static final Logger log = getLogger(Tools.class); 64 private static final Logger log = getLogger(Tools.class);
64 65
66 + private static Random random = new Random();
67 +
65 /** 68 /**
66 * Returns a thread factory that produces threads named according to the 69 * Returns a thread factory that produces threads named according to the
67 * supplied name pattern. 70 * supplied name pattern.
...@@ -195,6 +198,20 @@ public abstract class Tools { ...@@ -195,6 +198,20 @@ public abstract class Tools {
195 } 198 }
196 199
197 /** 200 /**
201 + * Suspends the current thread for a random number of millis between 0 and
202 + * the indicated limit.
203 + *
204 + * @param ms max number of millis
205 + */
206 + public static void randomDelay(int ms) {
207 + try {
208 + Thread.sleep(random.nextInt(ms));
209 + } catch (InterruptedException e) {
210 + throw new RuntimeException("Interrupted", e);
211 + }
212 + }
213 +
214 + /**
198 * Suspends the current thread for a specified number of millis and nanos. 215 * Suspends the current thread for a specified number of millis and nanos.
199 * 216 *
200 * @param ms number of millis 217 * @param ms number of millis
......