Merge branch 'master' of ssh://gerrit.onlab.us:29418/onos-next

+package org.onlab.onos.cluster;
+
+import org.onlab.onos.net.DeviceId;
+import org.onlab.onos.net.MastershipRole;
+
+/**
+ * Service for administering the inventory of device masterships.
+ */
+public interface MastershipAdminService {
+
+    /**
+     * Applies the current mastership role for the specified device.
+     *
+     * @param instance controller instance identifier
+     * @param deviceId device identifier
+     * @param role     requested role
+     */
+    void setRole(InstanceId instance, DeviceId deviceId, MastershipRole role);
+
+}

@@ -4,7 +4,7 @@ import org.onlab.onos.event.AbstractEvent;
 import org.onlab.onos.net.DeviceId;
 
 /**
- * Describes infrastructure device event.
+ * Describes a device mastership event.
  */
 public class MastershipEvent extends AbstractEvent<MastershipEvent.Type, DeviceId> {
 

@@ -49,6 +49,6 @@ public interface MastershipService {
      *
      * @param listener the mastership listener
      */
-    void removeListemer(MastershipListener listener);
+    void removeListener(MastershipListener listener);
 
 }

+package org.onlab.onos.cluster;
+
+import java.util.Set;
+
+import org.onlab.onos.net.DeviceId;
+import org.onlab.onos.net.MastershipRole;
+
+/**
+ * Manages inventory of mastership roles for devices, across controller instances.
+ */
+public interface MastershipStore {
+
+    // three things to map: InstanceId, DeviceId, MastershipRole
+
+    /**
+     * Sets a device's role for a specified controller instance.
+     *
+     * @param instance controller instance identifier
+     * @param deviceId device identifier
+     * @param role     new role
+     * @return a mastership event
+     */
+    MastershipEvent setRole(
+            InstanceId instance, DeviceId deviceId, MastershipRole role);
+
+    /**
+     * Adds or updates the mastership information for a device.
+     *
+     * @param instance controller instance identifier
+     * @param deviceId device identifier
+     * @param role     new role
+     * @return a mastership event
+     */
+    MastershipEvent addOrUpdateDevice(
+            InstanceId instance, DeviceId deviceId, MastershipRole role);
+
+    /**
+     * Returns the master for a device.
+     *
+     * @param deviceId the device identifier
+     * @return the instance identifier of the master
+     */
+    InstanceId getMaster(DeviceId deviceId);
+
+    /**
+     * Returns the devices that a controller instance is master of.
+     *
+     * @param  instanceId the instance identifier
+     * @return a set of device identifiers
+     */
+    Set<DeviceId> getDevices(InstanceId instanceId);
+
+    /**
+     * Returns the role of a device for a specific controller instance.
+     *
+     * @param instanceId the instance identifier
+     * @param deviceId the device identifiers
+     * @return the role
+     */
+    MastershipRole getRole(InstanceId instanceId, DeviceId deviceId);
+}

@@ -13,7 +13,9 @@ public interface DeviceAdminService {
      *
      * @param deviceId device identifier
      * @param role     requested role
+     * @deprecated Will be removed in favor of MastershipAdminService.setRole()
      */
+    @Deprecated
     void setRole(DeviceId deviceId, MastershipRole role);
 
     /**

 package org.onlab.onos.net.device.impl;
 
+import com.hazelcast.core.HazelcastInstance;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -23,6 +24,7 @@ import org.onlab.onos.net.device.PortDescription;
 import org.onlab.onos.net.provider.AbstractProvider;
 import org.onlab.onos.net.provider.ProviderId;
 import org.onlab.onos.event.impl.TestEventDispatcher;
+import org.onlab.onos.store.StoreService;
 import org.onlab.onos.store.device.impl.DistributedDeviceStore;
 
 import com.google.common.collect.Iterables;
@@ -76,7 +78,6 @@ public class DistributedDeviceManagerTest {
         service = mgr;
         admin = mgr;
         registry = mgr;
-        dstore = new DistributedDeviceStore();
         // FIXME should be reading the hazelcast.xml
         Config config = new Config();
         // avoid accidentally joining other cluster
@@ -88,7 +89,7 @@ public class DistributedDeviceManagerTest {
         config.getNetworkConfig().getJoin()
             .getMulticastConfig()
             .setEnabled(false);
-        dstore.theInstance = Hazelcast.newHazelcastInstance(config);
+        dstore = new TestDistributedDeviceStore(Hazelcast.newHazelcastInstance(config));
         dstore.activate();
         mgr.store = dstore;
         mgr.eventDispatcher = new TestEventDispatcher();
@@ -280,4 +281,14 @@ public class DistributedDeviceManagerTest {
         }
     }
 
+    private class TestDistributedDeviceStore extends DistributedDeviceStore {
+        public TestDistributedDeviceStore(final HazelcastInstance hazelcastInstance) {
+            storeService = new StoreService() {
+                @Override
+                public HazelcastInstance getHazelcastInstance() {
+                    return hazelcastInstance;
+                }
+            };
+        }
+    }
 }

+package org.onlab.onos.store;
+
+import com.hazelcast.core.HazelcastInstance;
+
+/**
+ * Bootstrap service to get a handle on a share Hazelcast instance.
+ */
+public interface StoreService {
+
+    /**
+     * Returns the shared Hazelcast instance for use as a distributed store
+     * backing.
+     *
+     * @return shared Hazelcast instance
+     */
+    HazelcastInstance getHazelcastInstance();
+
+}

@@ -40,6 +40,7 @@ import org.onlab.onos.net.device.DeviceEvent;
 import org.onlab.onos.net.device.DeviceStore;
 import org.onlab.onos.net.device.PortDescription;
 import org.onlab.onos.net.provider.ProviderId;
+import org.onlab.onos.store.StoreService;
 import org.onlab.util.KryoPool;
 import org.slf4j.Logger;
 
@@ -187,12 +188,15 @@ public class DistributedDeviceStore implements DeviceStore {
 
     // FIXME change to protected once we remove DistributedDeviceManagerTest.
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected StoreService storeService;
+
     /*protected*/public HazelcastInstance theInstance;
 
 
     @Activate
     public void activate() {
         log.info("Started");
+        theInstance = storeService.getHazelcastInstance();
 
         // IMap event handler needs value
         final boolean includeValue = true;

+package org.onlab.onos.store.impl;
+
+import com.hazelcast.core.Hazelcast;
+import com.hazelcast.core.HazelcastInstance;
+import org.apache.felix.scr.annotations.Activate;
+import org.apache.felix.scr.annotations.Component;
+import org.apache.felix.scr.annotations.Deactivate;
+import org.apache.felix.scr.annotations.Service;
+import org.onlab.onos.store.StoreService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Auxiliary bootstrap of distributed store.
+ */
+@Component(immediate = true)
+@Service
+public class StoreManager implements StoreService {
+
+    private final Logger log = LoggerFactory.getLogger(getClass());
+
+    protected HazelcastInstance instance;
+
+    @Activate
+    public void activate() {
+        instance = Hazelcast.newHazelcastInstance();
+        log.info("Started");
+    }
+
+    @Deactivate
+    public void deactivate() {
+        log.info("Stopped");
+    }
+
+    @Override
+    public HazelcastInstance getHazelcastInstance() {
+        return instance;
+    }
+}

@@ -10,9 +10,13 @@
         <bundle>mvn:com.google.guava/guava/18.0</bundle>
         <bundle>mvn:io.netty/netty/3.9.2.Final</bundle>
 
+        <bundle>mvn:com.hazelcast/hazelcast/3.3</bundle>
+        <bundle>mvn:com.eclipsesource.minimal-json/minimal-json/0.9.1</bundle>
+
         <bundle>mvn:com.esotericsoftware.kryo/kryo/2.24.0</bundle>
         <bundle>mvn:com.esotericsoftware/minlog/1.3.0</bundle>
         <bundle>mvn:org.objenesis/objenesis/2.1</bundle>
+        <bundle>mvn:de.javakaffee/kryo-serializers/0.27</bundle>
     </feature>
 
     <feature name="onos-thirdparty-web" version="1.0.0"
@@ -42,6 +46,13 @@
              description="ONOS core components">
         <feature>onos-api</feature>
         <bundle>mvn:org.onlab.onos/onos-core-net/1.0.0-SNAPSHOT</bundle>
+        <bundle>mvn:org.onlab.onos/onos-core-store/1.0.0-SNAPSHOT</bundle>
+    </feature>
+
+    <feature name="onos-core-trivial" version="1.0.0"
+             description="ONOS core components">
+        <feature>onos-api</feature>
+        <bundle>mvn:org.onlab.onos/onos-core-net/1.0.0-SNAPSHOT</bundle>
         <bundle>mvn:org.onlab.onos/onos-core-trivial/1.0.0-SNAPSHOT</bundle>
     </feature>
 

@@ -139,6 +139,11 @@
               <version>3.3</version>
             </dependency>
             <dependency>
+                <groupId>com.eclipsesource.minimal-json</groupId>
+                <artifactId>minimal-json</artifactId>
+                <version>0.9.1</version>
+            </dependency>
+            <dependency>
               <groupId>com.esotericsoftware.kryo</groupId>
               <artifactId>kryo</artifactId>
               <version>2.24.0</version>

@@ -26,7 +26,7 @@ mkdir bin
 
 # Stage the ONOS admin scripts and patch in Karaf service wrapper extras
 cp -r $ONOS_ROOT/tools/package/bin .
-cp -r $ONOS_ROOT/tools/package/wrapper/* $KARAF_DIST
+cp -r $ONOS_ROOT/tools/package/debian $ONOS_STAGE/debian
 cp -r $ONOS_ROOT/tools/package/etc/* $KARAF_DIST/etc
 
 # Stage the ONOS bundles
@@ -40,7 +40,7 @@ perl -pi.old -e "s|^(featuresRepositories=.*)|\1,mvn:org.apache.karaf.cellar/apa
     $ONOS_STAGE/$KARAF_DIST/etc/org.apache.karaf.features.cfg 
 
 # Patch the Apache Karaf distribution file to load ONOS features
-perl -pi.old -e 's|^(featuresBoot=.*)|\1,wrapper,cellar|' \
+perl -pi.old -e 's|^(featuresBoot=.*)|\1,cellar|' \
     $ONOS_STAGE/$KARAF_DIST/etc/org.apache.karaf.features.cfg 
 
 # ONOS Patching ----------------------------------------------------------------
@@ -49,15 +49,10 @@ perl -pi.old -e 's|^(featuresBoot=.*)|\1,wrapper,cellar|' \
 perl -pi.old -e "s|^(featuresRepositories=.*)|\1,mvn:org.onlab.onos/onos-features/$ONOS_VERSION/xml/features|" \
     $ONOS_STAGE/$KARAF_DIST/etc/org.apache.karaf.features.cfg 
 
-# Patch the Apache Karaf distribution file to load ONOS features
-#perl -pi.old -e 's|^(featuresBoot=.*)|\1,onos-api,onos-core,onos-cli,onos-rest,onos-gui,onos-openflow,onos-app-tvue,onos-app-fwd|' \
-#    $ONOS_STAGE/$KARAF_DIST/etc/org.apache.karaf.features.cfg 
-
 # Patch the Apache Karaf distribution with ONOS branding bundle
 cp $M2_REPO/org/onlab/onos/onos-branding/$ONOS_VERSION/onos-branding-*.jar \
     $ONOS_STAGE/$KARAF_DIST/lib
 
-
 # Now package up the ONOS tar file
 cd $ONOS_STAGE_ROOT
 COPYFILE_DISABLE=1 tar zcf $ONOS_TAR $ONOS_BITS

@@ -3,7 +3,7 @@
 # ONOS command-line client
 #-------------------------------------------------------------------------------
 
-export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64/
+export JAVA_HOME=${JAVA_HOME:-/usr/lib/jvm/java-7-openjdk-amd64/}
 
 cd $(dirname $0)/../apache-karaf-*/bin
 ./client -h localhost "$@"

@@ -3,8 +3,8 @@
 # Starts ONOS Apache Karaf container
 #-------------------------------------------------------------------------------
 
-export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64/
+export JAVA_HOME=${JAVA_HOME:-/usr/lib/jvm/java-7-openjdk-amd64/}
 
-cd $(dirname $0)/../apache-karaf-*/bin
-./karaf "$@"
+cd /opt/onos
+/opt/onos/apache-karaf-3.0.1/bin/karaf "$@"
 

+description  "Open Networking Operating System"
+author       "ON.Lab"
+
+start on (net-device-up
+          and local-filesystems
+          and runlevel [2345])
+stop on runlevel [016]
+
+console output
+kill timeout 60
+respawn
+
+env LANG=en_US.UTF-8
+env JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64
+
+script
+  [ -f /opt/onos/options ] && . /opt/onos/options
+  start-stop-daemon --signal INT --start --chuid sdn \
+    --exec /opt/onos/bin/onos-ctl -- $ONOS_OPTS \
+        >/opt/onos/var/stdout.log 2>/opt/onos/var/stderr.log
+end script

-# ------------------------------------------------------------------------
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ------------------------------------------------------------------------
-
-#********************************************************************
-# Wrapper Properties
-#********************************************************************
-set.default.JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64/
-set.default.KARAF_HOME=/opt/onos/apache-karaf-3.0.1
-set.default.KARAF_BASE=/opt/onos/apache-karaf-3.0.1
-set.default.KARAF_DATA=/opt/onos/apache-karaf-3.0.1/data
-set.default.KARAF_ETC=/opt/onos/apache-karaf-3.0.1/etc
-
-# Java Application
-wrapper.working.dir=%KARAF_BASE%
-wrapper.java.command=%JAVA_HOME%/bin/java
-wrapper.java.mainclass=org.apache.karaf.wrapper.internal.Main
-wrapper.java.classpath.1=%KARAF_HOME%/lib/karaf-wrapper.jar
-wrapper.java.classpath.2=%KARAF_HOME%/lib/karaf.jar
-wrapper.java.classpath.3=%KARAF_HOME%/lib/karaf-jmx-boot.jar
-wrapper.java.classpath.4=%KARAF_HOME%/lib/karaf-jaas-boot.jar
-wrapper.java.classpath.5=%KARAF_HOME%/lib/karaf-wrapper-main.jar
-wrapper.java.classpath.6=%KARAF_HOME%/lib/karaf-org.osgi.core.jar
-wrapper.java.library.path.1=%KARAF_HOME%/lib/
-
-# Application Parameters.  Add parameters as needed starting from 1
-#wrapper.app.parameter.1=
-
-# JVM Parameters            
-# note that n is the parameter number starting from 1.
-wrapper.java.additional.1=-Dkaraf.home=%KARAF_HOME%
-wrapper.java.additional.2=-Dkaraf.base=%KARAF_BASE%
-wrapper.java.additional.3=-Dkaraf.data=%KARAF_DATA%
-wrapper.java.additional.4=-Dkaraf.etc=%KARAF_ETC%
-wrapper.java.additional.5=-Dcom.sun.management.jmxremote
-wrapper.java.additional.6=-Djavax.management.builder.initial=org.apache.karaf.management.boot.KarafMBeanServerBuilder
-wrapper.java.additional.7=-Dkaraf.startLocalConsole=false
-wrapper.java.additional.8=-Dkaraf.startRemoteShell=true
-wrapper.java.additional.9=-Djava.endorsed.dirs=%JAVA_HOME%/jre/lib/endorsed:%JAVA_HOME%/lib/endorsed:%KARAF_HOME%/lib/endorsed
-wrapper.java.additional.10=-Djava.ext.dirs=%JAVA_HOME%/jre/lib/ext:%JAVA_HOME%/lib/ext:%KARAF_HOME%/lib/ext
-
-# Uncomment to enable jmx
-#wrapper.java.additional.n=-Dcom.sun.management.jmxremote.port=1616
-#wrapper.java.additional.n=-Dcom.sun.management.jmxremote.authenticate=false
-#wrapper.java.additional.n=-Dcom.sun.management.jmxremote.ssl=false
-
-# Uncomment to enable YourKit profiling
-#wrapper.java.additional.n=-Xrunyjpagent
-
-# Uncomment to enable remote debugging
-#wrapper.java.additional.n=-Xdebug -Xnoagent -Djava.compiler=NONE
-#wrapper.java.additional.n=-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=5005
-
-# Initial Java Heap Size (in MB)
-#wrapper.java.initmemory=3
-
-# Maximum Java Heap Size (in MB)
-wrapper.java.maxmemory=512
-
-
-#********************************************************************
-# Wrapper Logging Properties
-#********************************************************************
-# Format of output for the console.  (See docs for formats)
-wrapper.console.format=PM
-
-# Log Level for console output.  (See docs for log levels)
-wrapper.console.loglevel=INFO
-
-# Log file to use for wrapper output logging.
-wrapper.logfile=%KARAF_DATA%/log/wrapper.log
-
-# Format of output for the log file.  (See docs for formats)
-wrapper.logfile.format=LPTM
-
-# Log Level for log file output.  (See docs for log levels)
-wrapper.logfile.loglevel=INFO
-
-# Maximum size that the log file will be allowed to grow to before
-#  the log is rolled. Size is specified in bytes.  The default value
-#  of 0, disables log rolling.  May abbreviate with the 'k' (kb) or
-#  'm' (mb) suffix.  For example: 10m = 10 megabytes.
-wrapper.logfile.maxsize=10m
-
-# Maximum number of rolled log files which will be allowed before old
-#  files are deleted.  The default value of 0 implies no limit.
-wrapper.logfile.maxfiles=5
-
-# Log Level for sys/event log output.  (See docs for log levels)
-wrapper.syslog.loglevel=NONE
-
-#********************************************************************
-# Wrapper Windows Properties
-#********************************************************************
-# Title to use when running as a console
-wrapper.console.title=onos
-
-#********************************************************************
-# Wrapper Windows NT/2000/XP Service Properties
-#********************************************************************
-# WARNING - Do not modify any of these properties when an application
-#  using this configuration file has been installed as a service.
-#  Please uninstall the service before modifying this section.  The
-#  service can then be reinstalled.
-
-# Name of the service
-wrapper.ntservice.name=onos
-
-# Display name of the service
-wrapper.ntservice.displayname=onos
-
-# Description of the service
-wrapper.ntservice.description=ONOS
-
-# Service dependencies.  Add dependencies as needed starting from 1
-wrapper.ntservice.dependency.1=
-
-# Mode in which the service is installed.  AUTO_START or DEMAND_START
-wrapper.ntservice.starttype=AUTO_START
-
-# Allow the service to interact with the desktop.
-wrapper.ntservice.interactive=false

@@ -12,9 +12,6 @@ LOG=$ONOS_INSTALL_DIR/config.log
 onos=$ONOS_INSTALL_DIR/bin/onos
 
 ssh $remote "
-    echo 'Starting...'
-    nohup $ONOS_INSTALL_DIR/bin/onos-ctl server </dev/null | 1>/opt/onos/svc.log 2>&1 &
-
     # Wait until we reach the run-level 100
     echo 'Waiting for cluster bootstrap...'
     running=""
@@ -22,29 +19,13 @@ ssh $remote "
         $onos bundle:list 2>>$LOG | grep -q 'START LEVEL 100' && running=1 || sleep 2
     done
 
-    # Now create group onos and join it, while quitting the default one
-    if ! $onos cluster:group-list 2>>$LOG | cut -d \\  -f3 | grep -q onos; then
-        echo 'Creating ONOS group...'
-        installRole=primary
-        $onos cluster:group-create onos 1>>$LOG 2>&1
-    fi
-
-    echo 'Configuring group membership...'
-    node=\$($onos cluster:node-list 2>>$LOG | grep '^x' | cut -d \\  -f3)
-    $onos cluster:group-join onos \$node 1>>$LOG 2>&1
-    $onos cluster:group-quit default \$node 1>>$LOG 2>&1
-
-    if [ X\$installRole = Xprimary ]; then
     echo 'Installing ONOS bundles...'
-        $onos cluster:feature-install onos onos-api 1>>$LOG 2>&1
-        $onos cluster:feature-install onos onos-core 1>>$LOG 2>&1
-        $onos cluster:feature-install onos onos-openflow 1>>$LOG 2>&1
-        $onos cluster:feature-install onos onos-cli 1>>$LOG 2>&1
-      # $onos cluster:feature-install onos onos-gui 1>>$LOG 2>&1
-      # $onos cluster:feature-install onos onos-rest 1>>$LOG 2>&1
-        $onos cluster:feature-install onos onos-app-tvue 1>>$LOG 2>&1
-        $onos cluster:feature-install onos onos-app-fwd 1>>$LOG 2>&1
-    fi
- 
-    echo 'Started...'
+    $onos cluster:feature-install default onos-api 1>>$LOG 2>&1
+    $onos cluster:feature-install default onos-core 1>>$LOG 2>&1
+    $onos cluster:feature-install default onos-openflow 1>>$LOG 2>&1
+    $onos cluster:feature-install default onos-cli 1>>$LOG 2>&1
+  # $onos cluster:feature-install default onos-gui 1>>$LOG 2>&1
+  # $onos cluster:feature-install default onos-rest 1>>$LOG 2>&1
+    $onos cluster:feature-install default onos-app-tvue 1>>$LOG 2>&1
+    $onos cluster:feature-install default onos-app-fwd 1>>$LOG 2>&1
 "

@@ -20,6 +20,13 @@ ssh $remote "
     sudo mkdir -p $ONOS_INSTALL_DIR && sudo chown sdn:sdn $ONOS_INSTALL_DIR
     tar zxmf /tmp/$ONOS_BITS.tar.gz -C $ONOS_INSTALL_DIR --strip-components=1
 
-    # Make a link to the log file directory.
+    # Make a link to the log file directory and make a home for auxiliaries
     ln -s $ONOS_INSTALL_DIR/$KARAF_DIST/data/log /opt/onos/log
+    mkdir $ONOS_INSTALL_DIR/var
+
+    # Install the upstart configuration file.
+    sudo cp $ONOS_INSTALL_DIR/debian/onos.conf /etc/init/onos.conf
+
+    # Ignite the ONOS service.
+    sudo service onos start
 "

@@ -9,6 +9,7 @@
 remote=$ONOS_USER@${1:-$OCI}
 
 ssh $remote "
+    sudo service onos stop 1>/dev/null 2>/dev/null
     [ -f $ONOS_INSTALL_DIR/bin/onos ] && \
         $ONOS_INSTALL_DIR/bin/onos halt 2>/dev/null
     sudo rm -fr $ONOS_INSTALL_DIR

@@ -35,6 +35,11 @@
             <artifactId>commons-lang3</artifactId>
         </dependency>
 
+        <!-- TODO: do we still need this here? -->
+        <dependency>
+            <groupId>com.eclipsesource.minimal-json</groupId>
+            <artifactId>minimal-json</artifactId>
+        </dependency>
         <dependency>
           <groupId>com.esotericsoftware.kryo</groupId>
           <artifactId>kryo</artifactId>