Fix SpringOpenTTP thread leak

- It was creating 2 theads everytime a Device connected.
  Now uses thread pool shared across Devices, where threads will die out on idle.
  Should resolve ONOS-3579

Change-Id: I490b2ef677853677fbd151af27f6ac2be563774c

@@ -15,6 +15,7 @@
  */
  */
 package org.onosproject.driver.pipeline;
 package org.onosproject.driver.pipeline;
 
 
+import static java.util.concurrent.Executors.newScheduledThreadPool;
 import static org.onlab.util.Tools.groupedThreads;
 import static org.onlab.util.Tools.groupedThreads;
 import static org.slf4j.LoggerFactory.getLogger;
 import static org.slf4j.LoggerFactory.getLogger;
 
 
@@ -86,8 +87,8 @@ import java.util.Collections;
 import java.util.List;
 import java.util.List;
 import java.util.Objects;
 import java.util.Objects;
 import java.util.Set;
 import java.util.Set;
-import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 import java.util.stream.Collectors;
 
 
@@ -97,6 +98,11 @@ import java.util.stream.Collectors;
 public class SpringOpenTTP extends AbstractHandlerBehaviour
 public class SpringOpenTTP extends AbstractHandlerBehaviour
         implements Pipeliner {
         implements Pipeliner {
 
 
+    /**
+     * GroupCheck delay.
+     */
+    private static final int CHECK_DELAY = 500;
+
     // Default table ID - compatible with CpqD switch
     // Default table ID - compatible with CpqD switch
     private static final int TABLE_VLAN = 0;
     private static final int TABLE_VLAN = 0;
     private static final int TABLE_TMAC = 1;
     private static final int TABLE_TMAC = 1;
@@ -118,7 +124,7 @@ public class SpringOpenTTP extends AbstractHandlerBehaviour
     protected int aclTableId = TABLE_ACL;
     protected int aclTableId = TABLE_ACL;
     protected int srcMacTableId = TABLE_SMAC;
     protected int srcMacTableId = TABLE_SMAC;
 
 
-    protected final Logger log = getLogger(getClass());
+    private static final Logger log = getLogger(SpringOpenTTP.class);
 
 
     private ServiceDirectory serviceDirectory;
     private ServiceDirectory serviceDirectory;
     private FlowRuleService flowRuleService;
     private FlowRuleService flowRuleService;
@@ -130,11 +136,19 @@ public class SpringOpenTTP extends AbstractHandlerBehaviour
 
 
     private Cache<GroupKey, NextObjective> pendingGroups;
     private Cache<GroupKey, NextObjective> pendingGroups;
 
 
-    private ScheduledExecutorService groupChecker = Executors
-            .newScheduledThreadPool(2,
+    private static final ScheduledExecutorService GROUP_CHECKER
+        = newScheduledThreadPool(2,
                                  groupedThreads("onos/pipeliner",
                                  groupedThreads("onos/pipeliner",
-                                                   "spring-open-%d",
-                                                   log));
+                                                "spring-open-%d", log));
+    static {
+        // ONOS-3579 workaround, let core threads die out on idle
+        if (GROUP_CHECKER instanceof ScheduledThreadPoolExecutor) {
+            ScheduledThreadPoolExecutor executor = (ScheduledThreadPoolExecutor) GROUP_CHECKER;
+            executor.setKeepAliveTime(CHECK_DELAY * 2, TimeUnit.MILLISECONDS);
+            executor.allowCoreThreadTimeOut(true);
+        }
+    }
+
     protected KryoNamespace appKryo = new KryoNamespace.Builder()
     protected KryoNamespace appKryo = new KryoNamespace.Builder()
             .register(KryoNamespaces.API)
             .register(KryoNamespaces.API)
             .register(GroupKey.class)
             .register(GroupKey.class)
@@ -158,9 +172,6 @@ public class SpringOpenTTP extends AbstractHandlerBehaviour
                                      }
                                      }
                                  }).build();
                                  }).build();
 
 
-        groupChecker.scheduleAtFixedRate(new GroupChecker(), 0, 500,
-                                         TimeUnit.MILLISECONDS);
-
         coreService = serviceDirectory.get(CoreService.class);
         coreService = serviceDirectory.get(CoreService.class);
         flowRuleService = serviceDirectory.get(FlowRuleService.class);
         flowRuleService = serviceDirectory.get(FlowRuleService.class);
         groupService = serviceDirectory.get(GroupService.class);
         groupService = serviceDirectory.get(GroupService.class);
@@ -342,6 +353,7 @@ public class SpringOpenTTP extends AbstractHandlerBehaviour
                                 + " in dev:{}", nextObjective.id(), deviceId);
                                 + " in dev:{}", nextObjective.id(), deviceId);
                         pendingGroups.put(key, nextObjective);
                         pendingGroups.put(key, nextObjective);
                         groupService.addGroup(groupDescription);
                         groupService.addGroup(groupDescription);
+                        verifyPendingGroupLater();
                     }
                     }
                 }
                 }
                 break;
                 break;
@@ -367,6 +379,7 @@ public class SpringOpenTTP extends AbstractHandlerBehaviour
                             + "in device {}", nextObjective.id(), deviceId);
                             + "in device {}", nextObjective.id(), deviceId);
                     pendingGroups.put(key, nextObjective);
                     pendingGroups.put(key, nextObjective);
                     groupService.addGroup(groupDescription);
                     groupService.addGroup(groupDescription);
+                    verifyPendingGroupLater();
                 }
                 }
                 break;
                 break;
             case FAILOVER:
             case FAILOVER:
@@ -1130,6 +1143,10 @@ public class SpringOpenTTP extends AbstractHandlerBehaviour
         }
         }
     }
     }
 
 
+    private void verifyPendingGroupLater() {
+        GROUP_CHECKER.schedule(new GroupChecker(), CHECK_DELAY, TimeUnit.MILLISECONDS);
+    }
+
     private class GroupChecker implements Runnable {
     private class GroupChecker implements Runnable {
 
 
         @Override
         @Override
@@ -1158,6 +1175,13 @@ public class SpringOpenTTP extends AbstractHandlerBehaviour
                                                         obj.id(),
                                                         obj.id(),
                                                         new SpringOpenGroup(key, null));
                                                         new SpringOpenGroup(key, null));
                     });
                     });
+
+            if (!pendingGroups.asMap().isEmpty()) {
+                // Periodically execute only if entry remains in pendingGroups.
+                // Iterating pendingGroups trigger cleanUp and expiration,
+                // which will eventually empty the pendingGroups.
+                verifyPendingGroupLater();
+            }
         }
         }
     }
     }
 
 

@@ -15,6 +15,8 @@
  */
  */
 package org.onosproject.driver.pipeline;
 package org.onosproject.driver.pipeline;
 
 
+import static org.slf4j.LoggerFactory.getLogger;
+
 import java.util.Collection;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Collections;
 import java.util.List;
 import java.util.List;
@@ -42,6 +44,7 @@ import org.onosproject.net.flowobjective.FilteringObjective;
 import org.onosproject.net.flowobjective.ForwardingObjective;
 import org.onosproject.net.flowobjective.ForwardingObjective;
 import org.onosproject.net.flowobjective.ObjectiveError;
 import org.onosproject.net.flowobjective.ObjectiveError;
 import org.onosproject.net.group.Group;
 import org.onosproject.net.group.Group;
+import org.slf4j.Logger;
 
 
 /**
 /**
  * Spring-open driver implementation for Dell hardware switches.
  * Spring-open driver implementation for Dell hardware switches.
@@ -55,6 +58,8 @@ public class SpringOpenTTPDell extends SpringOpenTTP {
     private static final int DELL_TABLE_MPLS = 25;
     private static final int DELL_TABLE_MPLS = 25;
     private static final int DELL_TABLE_ACL = 40;
     private static final int DELL_TABLE_ACL = 40;
 
 
+    private final Logger log = getLogger(getClass());
+
     //TODO: Store this info in the distributed store.
     //TODO: Store this info in the distributed store.
     private MacAddress deviceTMac = null;
     private MacAddress deviceTMac = null;