Merge branch 'master' of ssh://gerrit.onlab.us:29418/onos-next

@@ -31,6 +31,11 @@
       <version>2.4.2</version>
       <scope>provided</scope>
     </dependency>
+
+    <dependency>
+      <groupId>com.google.guava</groupId>
+      <artifactId>guava</artifactId>
+    </dependency>
   </dependencies>
 
 </project>

+package org.onlab.onos.sdnip;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import java.util.Set;
+
+import org.apache.commons.lang.NotImplementedException;
+import org.onlab.onos.net.ConnectPoint;
+import org.onlab.onos.net.host.HostService;
+import org.onlab.onos.net.host.PortAddresses;
+import org.onlab.onos.sdnip.config.Interface;
+import org.onlab.packet.IpAddress;
+
+import com.google.common.collect.Sets;
+
+
+
+/**
+ * Provides IntefaceService using PortAddresses data from the HostService.
+ */
+public class HostServiceBasedInterfaceService implements InterfaceService {
+
+    private final HostService hostService;
+
+    public HostServiceBasedInterfaceService(HostService hostService) {
+        this.hostService = checkNotNull(hostService);
+    }
+
+    @Override
+    public Set<Interface> getInterfaces() {
+        Set<PortAddresses> addresses = hostService.getAddressBindings();
+        Set<Interface> interfaces = Sets.newHashSetWithExpectedSize(addresses.size());
+        for (PortAddresses a : addresses) {
+            interfaces.add(new Interface(a));
+        }
+        return interfaces;
+    }
+
+    @Override
+    public Interface getInterface(ConnectPoint connectPoint) {
+        checkNotNull(connectPoint);
+
+        PortAddresses portAddresses =
+                hostService.getAddressBindingsForPort(connectPoint);
+
+        if (!portAddresses.ips().isEmpty()) {
+            return new Interface(portAddresses);
+        }
+
+        return null;
+    }
+
+    @Override
+    public Interface getMatchingInterface(IpAddress ipAddress) {
+        // TODO implement
+        throw new NotImplementedException("getMatchingInteface is not yet implemented");
+    }
+
+}

+package org.onlab.onos.sdnip;
+
+import java.util.Set;
+
+import org.onlab.onos.net.ConnectPoint;
+import org.onlab.onos.sdnip.config.Interface;
+import org.onlab.packet.IpAddress;
+
+/**
+ * Provides information about the interfaces in the network.
+ */
+public interface InterfaceService {
+    /**
+     * Retrieves the entire set of interfaces in the network.
+     *
+     * @return the set of interfaces
+     */
+    Set<Interface> getInterfaces();
+
+    /**
+     * Retrieves the interface associated with the given connect point.
+     *
+     * @param connectPoint the connect point to retrieve interface information
+     * for
+     * @return the interface
+     */
+    Interface getInterface(ConnectPoint connectPoint);
+
+    /**
+     * Retrieves the interface that matches the given IP address. Matching
+     * means that the IP address is in one of the interface's assigned subnets.
+     *
+     * @param ipAddress IP address to match
+     * @return the matching interface
+     */
+    Interface getMatchingInterface(IpAddress ipAddress);
+}

+package org.onlab.onos.sdnip;
+
+import java.util.List;
+
+import org.onlab.onos.net.ConnectPoint;
+import org.onlab.onos.net.flow.DefaultTrafficSelector;
+import org.onlab.onos.net.flow.DefaultTrafficTreatment;
+import org.onlab.onos.net.flow.TrafficSelector;
+import org.onlab.onos.net.flow.TrafficTreatment;
+import org.onlab.onos.net.intent.IntentId;
+import org.onlab.onos.net.intent.IntentService;
+import org.onlab.onos.net.intent.PointToPointIntent;
+import org.onlab.onos.sdnip.config.BgpPeer;
+import org.onlab.onos.sdnip.config.BgpSpeaker;
+import org.onlab.onos.sdnip.config.Interface;
+import org.onlab.onos.sdnip.config.InterfaceAddress;
+import org.onlab.onos.sdnip.config.SdnIpConfigService;
+import org.onlab.packet.Ethernet;
+import org.onlab.packet.IPv4;
+import org.onlab.packet.IpAddress;
+import org.onlab.packet.IpPrefix;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Manages the connectivity requirements between peers.
+ */
+public class PeerConnectivity {
+
+    private static final Logger log = LoggerFactory.getLogger(
+            PeerConnectivity.class);
+
+    // TODO these shouldn't be defined here
+    private static final short BGP_PORT = 179;
+    private static final int IPV4_BIT_LENGTH = 32;
+
+    private final SdnIpConfigService configInfoService;
+    private final InterfaceService interfaceService;
+    private final IntentService intentService;
+
+    // TODO this sucks.
+    private int intentId = 0;
+
+    public PeerConnectivity(SdnIpConfigService configInfoService,
+            InterfaceService interfaceService, IntentService intentService) {
+        this.configInfoService = configInfoService;
+        this.interfaceService = interfaceService;
+        this.intentService = intentService;
+    }
+
+    public void start() {
+        // TODO are any of these errors?
+        if (interfaceService.getInterfaces().isEmpty()) {
+
+            log.warn("The interface in configuration file is empty. "
+                    + "Thus, the SDN-IP application can not be started.");
+        } else if (configInfoService.getBgpPeers().isEmpty()) {
+
+            log.warn("The BGP peer in configuration file is empty."
+                    + "Thus, the SDN-IP application can not be started.");
+        } else if (configInfoService.getBgpSpeakers() == null) {
+
+            log.error("The BGP speaker in configuration file is empty. "
+                    + "Thus, the SDN-IP application can not be started.");
+            return;
+        }
+
+        setupBgpPaths();
+        setupIcmpPaths();
+    }
+
+    /**
+     * Sets up paths for all {@link BgpSpeaker}s and all external peers.
+     * <p/>
+     * Run a loop for all BGP speakers and a loop for all BGP peers outside.
+     * Push intents for paths from each BGP speaker to all peers. Push intents
+     * for paths from all peers to each BGP speaker.
+     */
+    private void setupBgpPaths() {
+        for (BgpSpeaker bgpSpeaker : configInfoService.getBgpSpeakers()
+                .values()) {
+            log.debug("Start to set up BGP paths for BGP speaker: {}",
+                    bgpSpeaker);
+            ConnectPoint bgpdConnectPoint = bgpSpeaker.connectPoint();
+
+            List<InterfaceAddress> interfaceAddresses =
+                    bgpSpeaker.interfaceAddresses();
+
+            for (BgpPeer bgpPeer : configInfoService.getBgpPeers().values()) {
+
+                log.debug("Start to set up BGP paths between BGP speaker: {} "
+                        + "to BGP peer: {}", bgpSpeaker, bgpPeer);
+
+                Interface peerInterface = interfaceService.getInterface(
+                        bgpPeer.connectPoint());
+                if (peerInterface == null) {
+                    log.error("Can not find the corresponding Interface from "
+                            + "configuration for BGP peer {}",
+                            bgpPeer.ipAddress());
+                    continue;
+                }
+
+                IpAddress bgpdAddress = null;
+                for (InterfaceAddress interfaceAddress : interfaceAddresses) {
+                    if (interfaceAddress.connectPoint().equals(
+                            peerInterface.connectPoint())) {
+                        bgpdAddress = interfaceAddress.ipAddress();
+                        break;
+                    }
+                }
+                if (bgpdAddress == null) {
+                    log.debug("There is no interface IP address for bgpPeer: {}"
+                            + " on interface {}", bgpPeer, bgpPeer.connectPoint());
+                    return;
+                }
+
+                IpAddress bgpdPeerAddress = bgpPeer.ipAddress();
+                ConnectPoint bgpdPeerConnectPoint = peerInterface.connectPoint();
+
+                // install intent for BGP path from BGPd to BGP peer matching
+                // destination TCP port 179
+
+                // TODO: The usage of PacketMatchBuilder will be improved, then we
+                // only need to new the PacketMatchBuilder once.
+                // By then, the code here will be improved accordingly.
+                 TrafficSelector selector = DefaultTrafficSelector.builder()
+                        .matchEthType(Ethernet.TYPE_IPV4)
+                        .matchIPProtocol(IPv4.PROTOCOL_TCP)
+                        .matchIPSrc(IpPrefix.valueOf(bgpdAddress.toRealInt(), IPV4_BIT_LENGTH))
+                        .matchIPDst(IpPrefix.valueOf(bgpdPeerAddress.toRealInt(), IPV4_BIT_LENGTH))
+                        .matchTcpDst(BGP_PORT)
+                        .build();
+
+                TrafficTreatment treatment = DefaultTrafficTreatment.builder()
+                        .build();
+
+                PointToPointIntent intentMatchDstTcpPort = new PointToPointIntent(
+                        nextIntentId(), selector, treatment,
+                        bgpdConnectPoint, bgpdPeerConnectPoint);
+                intentService.submit(intentMatchDstTcpPort);
+                log.debug("Submitted BGP path intent matching dst TCP port 179 "
+                        + "from BGPd {} to peer {}: {}",
+                        bgpdAddress, bgpdPeerAddress, intentMatchDstTcpPort);
+
+                // install intent for BGP path from BGPd to BGP peer matching
+                // source TCP port 179
+                selector = DefaultTrafficSelector.builder()
+                        .matchEthType(Ethernet.TYPE_IPV4)
+                        .matchIPProtocol(IPv4.PROTOCOL_TCP)
+                        .matchIPSrc(IpPrefix.valueOf(bgpdAddress.toRealInt(), IPV4_BIT_LENGTH))
+                        .matchIPDst(IpPrefix.valueOf(bgpdPeerAddress.toRealInt(), IPV4_BIT_LENGTH))
+                        .matchTcpSrc(BGP_PORT)
+                        .build();
+
+                PointToPointIntent intentMatchSrcTcpPort = new PointToPointIntent(
+                        nextIntentId(), selector, treatment,
+                        bgpdConnectPoint, bgpdPeerConnectPoint);
+                intentService.submit(intentMatchSrcTcpPort);
+                log.debug("Submitted BGP path intent matching src TCP port 179"
+                        + "from BGPd {} to peer {}: {}",
+                        bgpdAddress, bgpdPeerAddress, intentMatchSrcTcpPort);
+
+                // install intent for reversed BGP path from BGP peer to BGPd
+                // matching destination TCP port 179
+                selector = DefaultTrafficSelector.builder()
+                        .matchEthType(Ethernet.TYPE_IPV4)
+                        .matchIPProtocol(IPv4.PROTOCOL_TCP)
+                        .matchIPSrc(IpPrefix.valueOf(bgpdPeerAddress.toRealInt(), IPV4_BIT_LENGTH))
+                        .matchIPDst(IpPrefix.valueOf(bgpdAddress.toRealInt(), IPV4_BIT_LENGTH))
+                        .matchTcpDst(BGP_PORT)
+                        .build();
+
+                PointToPointIntent reversedIntentMatchDstTcpPort = new PointToPointIntent(
+                        nextIntentId(), selector, treatment,
+                        bgpdPeerConnectPoint, bgpdConnectPoint);
+                intentService.submit(reversedIntentMatchDstTcpPort);
+                log.debug("Submitted BGP path intent matching dst TCP port 179"
+                        + "from BGP peer {} to BGPd {} : {}",
+                        bgpdPeerAddress, bgpdAddress, reversedIntentMatchDstTcpPort);
+
+                // install intent for reversed BGP path from BGP peer to BGPd
+                // matching source TCP port 179
+                selector = DefaultTrafficSelector.builder()
+                        .matchEthType(Ethernet.TYPE_IPV4)
+                        .matchIPProtocol(IPv4.PROTOCOL_TCP)
+                        .matchIPSrc(IpPrefix.valueOf(bgpdPeerAddress.toRealInt(), IPV4_BIT_LENGTH))
+                        .matchIPDst(IpPrefix.valueOf(bgpdAddress.toRealInt(), IPV4_BIT_LENGTH))
+                        .matchTcpSrc(BGP_PORT)
+                        .build();
+
+                PointToPointIntent reversedIntentMatchSrcTcpPort = new PointToPointIntent(
+                        nextIntentId(), selector, treatment,
+                        bgpdPeerConnectPoint, bgpdConnectPoint);
+                intentService.submit(reversedIntentMatchSrcTcpPort);
+                log.debug("Submitted BGP path intent matching src TCP port 179"
+                        + "from BGP peer {} to BGPd {} : {}",
+                        bgpdPeerAddress, bgpdAddress, reversedIntentMatchSrcTcpPort);
+
+            }
+        }
+    }
+
+    /**
+     * Sets up ICMP paths between each {@link BgpSpeaker} and all BGP peers
+     * located in other external networks.
+     * <p/>
+     * Run a loop for all BGP speakers and a loop for all BGP Peers. Push
+     * intents for paths from each BGP speaker to all peers. Push intents
+     * for paths from all peers to each BGP speaker.
+     */
+    private void setupIcmpPaths() {
+        for (BgpSpeaker bgpSpeaker : configInfoService.getBgpSpeakers()
+                .values()) {
+            log.debug("Start to set up ICMP paths for BGP speaker: {}",
+                    bgpSpeaker);
+            ConnectPoint bgpdConnectPoint = bgpSpeaker.connectPoint();
+            List<InterfaceAddress> interfaceAddresses = bgpSpeaker
+                    .interfaceAddresses();
+
+            for (BgpPeer bgpPeer : configInfoService.getBgpPeers().values()) {
+
+                Interface peerInterface = interfaceService.getInterface(
+                        bgpPeer.connectPoint());
+
+                if (peerInterface == null) {
+                    log.error("Can not find the corresponding Interface from "
+                            + "configuration for BGP peer {}",
+                            bgpPeer.ipAddress());
+                    continue;
+                }
+                IpAddress bgpdAddress = null;
+                for (InterfaceAddress interfaceAddress : interfaceAddresses) {
+                    if (interfaceAddress.connectPoint().equals(
+                            peerInterface.connectPoint())) {
+                        bgpdAddress = interfaceAddress.ipAddress();
+                        break;
+                    }
+
+                }
+                if (bgpdAddress == null) {
+                    log.debug("There is no IP address for bgpPeer: {} on "
+                            + "interface port: {}", bgpPeer,
+                            bgpPeer.connectPoint());
+                    return;
+                }
+
+                IpAddress bgpdPeerAddress = bgpPeer.ipAddress();
+                ConnectPoint bgpdPeerConnectPoint = peerInterface.connectPoint();
+
+                // install intent for ICMP path from BGPd to BGP peer
+                TrafficSelector selector = DefaultTrafficSelector.builder()
+                        .matchEthType(Ethernet.TYPE_IPV4)
+                        .matchIPProtocol(IPv4.PROTOCOL_ICMP)
+                        .matchIPSrc(IpPrefix.valueOf(bgpdAddress.toRealInt(), IPV4_BIT_LENGTH))
+                        .matchIPDst(IpPrefix.valueOf(bgpdPeerAddress.toRealInt(), IPV4_BIT_LENGTH))
+                        .build();
+
+                TrafficTreatment treatment = DefaultTrafficTreatment.builder()
+                        .build();
+
+                PointToPointIntent intent = new PointToPointIntent(
+                        nextIntentId(), selector, treatment,
+                        bgpdConnectPoint, bgpdPeerConnectPoint);
+                intentService.submit(intent);
+                log.debug("Submitted ICMP path intent from BGPd {} to peer {} :"
+                        + " {}", bgpdAddress, bgpdPeerAddress, intent);
+
+                // install intent for reversed ICMP path from BGP peer to BGPd
+                selector = DefaultTrafficSelector.builder()
+                        .matchEthType(Ethernet.TYPE_IPV4)
+                        .matchIPProtocol(IPv4.PROTOCOL_ICMP)
+                        .matchIPSrc(IpPrefix.valueOf(bgpdPeerAddress.toRealInt(), IPV4_BIT_LENGTH))
+                        .matchIPDst(IpPrefix.valueOf(bgpdAddress.toRealInt(), IPV4_BIT_LENGTH))
+                        .build();
+
+                PointToPointIntent reversedIntent = new PointToPointIntent(
+                        nextIntentId(), selector, treatment,
+                        bgpdPeerConnectPoint, bgpdConnectPoint);
+                intentService.submit(reversedIntent);
+                log.debug("Submitted ICMP path intent from BGP peer {} to BGPd"
+                        + " {} : {}",
+                        bgpdPeerAddress, bgpdAddress, reversedIntent);
+            }
+        }
+    }
+
+    private IntentId nextIntentId() {
+        return new IntentId(intentId++);
+    }
+}

@@ -5,6 +5,11 @@ import static org.slf4j.LoggerFactory.getLogger;
 import org.apache.felix.scr.annotations.Activate;
 import org.apache.felix.scr.annotations.Component;
 import org.apache.felix.scr.annotations.Deactivate;
+import org.apache.felix.scr.annotations.Reference;
+import org.apache.felix.scr.annotations.ReferenceCardinality;
+import org.onlab.onos.net.host.HostService;
+import org.onlab.onos.net.intent.IntentService;
+import org.onlab.onos.sdnip.config.SdnIpConfigReader;
 import org.slf4j.Logger;
 
 /**
@@ -15,9 +20,27 @@ public class SdnIp {
 
     private final Logger log = getLogger(getClass());
 
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected IntentService intentService;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected HostService hostService;
+
+    private SdnIpConfigReader config;
+    private PeerConnectivity peerConnectivity;
+
     @Activate
     protected void activate() {
         log.debug("SDN-IP started");
+
+        config = new SdnIpConfigReader();
+        config.init();
+
+        InterfaceService interfaceService = new HostServiceBasedInterfaceService(hostService);
+
+        peerConnectivity = new PeerConnectivity(config, interfaceService, intentService);
+        peerConnectivity.start();
+
     }
 
     @Deactivate

+package org.onlab.onos.sdnip.config;
+
+import java.util.Objects;
+
+import org.codehaus.jackson.annotate.JsonProperty;
+import org.onlab.onos.net.ConnectPoint;
+import org.onlab.onos.net.DeviceId;
+import org.onlab.onos.net.PortNumber;
+import org.onlab.packet.IpAddress;
+
+import com.google.common.base.MoreObjects;
+
+/**
+ * Configuration details for a BGP peer.
+ */
+public class BgpPeer {
+    private final ConnectPoint connectPoint;
+    private final IpAddress ipAddress;
+
+    /**
+     * Creates a new BgpPeer.
+     *
+     * @param dpid the DPID of the switch the peer is attached at, as a String
+     * @param port the port the peer is attached at
+     * @param ipAddress the IP address of the peer as a String
+     */
+    public BgpPeer(@JsonProperty("attachmentDpid") String dpid,
+                   @JsonProperty("attachmentPort") int port,
+                   @JsonProperty("ipAddress") String ipAddress) {
+        this.connectPoint = new ConnectPoint(
+                DeviceId.deviceId(SdnIpConfigReader.dpidToUri(dpid)),
+                PortNumber.portNumber(port));
+        this.ipAddress = IpAddress.valueOf(ipAddress);
+    }
+
+    /**
+     * Gets the connection point of the peer.
+     *
+     * @return the connection point
+     */
+    public ConnectPoint connectPoint() {
+        return connectPoint;
+    }
+
+    /**
+     * Gets the IP address of the peer.
+     *
+     * @return the IP address
+     */
+    public IpAddress ipAddress() {
+        return ipAddress;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(connectPoint, ipAddress);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == this) {
+            return true;
+        }
+
+        if (!(obj instanceof BgpPeer)) {
+            return false;
+        }
+
+        BgpPeer that = (BgpPeer) obj;
+        return Objects.equals(this.connectPoint, that.connectPoint)
+                && Objects.equals(this.ipAddress, that.ipAddress);
+    }
+
+    @Override
+    public String toString() {
+        return MoreObjects.toStringHelper(getClass())
+                .add("connectPoint", connectPoint)
+                .add("ipAddress", ipAddress)
+                .toString();
+    }
+}

+package org.onlab.onos.sdnip.config;
+
+import java.util.List;
+import java.util.Objects;
+
+import org.codehaus.jackson.annotate.JsonCreator;
+import org.codehaus.jackson.annotate.JsonProperty;
+import org.onlab.onos.net.ConnectPoint;
+import org.onlab.onos.net.DeviceId;
+import org.onlab.onos.net.PortNumber;
+import org.onlab.packet.MacAddress;
+
+import com.google.common.base.MoreObjects;
+
+/**
+ * Represents a BGP daemon in SDN network.
+ * <p/>
+ * Each BGP speaker has a attachment point, which includes a switch DPID and a
+ * switch port. Each BGP speaker has one MAC address and several IP addresses,
+ * which are used to peer with BGP peers outside the SDN network. For each
+ * peer outside the SDN network, we configure a different IP address to BGP
+ * speaker inside the SDN network.
+ * <p/>
+ * Each BGP speaker has a name, which is a unique identifying String that is
+ * used to reference this speaker in the configuration.
+ */
+public class BgpSpeaker {
+    private final String name;
+    private final ConnectPoint connectPoint;
+    private final MacAddress macAddress;
+    private List<InterfaceAddress> interfaceAddresses;
+
+    /**
+     * Class constructor used by the JSON library to create an object.
+     *
+     * @param name the name of the BGP speaker inside SDN network
+     * @param attachmentDpid the DPID where the BGP speaker is attached to
+     * @param attachmentPort the port where the BGP speaker is attached to
+     * @param macAddress the MAC address of the BGP speaker
+     */
+    @JsonCreator
+    public BgpSpeaker(@JsonProperty("name") String name,
+            @JsonProperty("attachmentDpid") String attachmentDpid,
+            @JsonProperty("attachmentPort") int attachmentPort,
+            @JsonProperty("macAddress") String macAddress) {
+
+        this.name = name;
+        this.macAddress = MacAddress.valueOf(macAddress);
+        this.connectPoint = new ConnectPoint(
+                DeviceId.deviceId(SdnIpConfigReader.dpidToUri(attachmentDpid)),
+                PortNumber.portNumber(attachmentPort));
+    }
+
+    /**
+     * Sets the addresses we configured for the BGP speaker on all virtual
+     * {@link Interface}s.
+     *
+     * @param interfaceAddresses a list of IP addresses of the BGP speaker
+     * configured on all virtual interfaces
+     */
+    @JsonProperty("interfaceAddresses")
+    public void setInterfaceAddresses(
+            List<InterfaceAddress> interfaceAddresses) {
+        this.interfaceAddresses = interfaceAddresses;
+    }
+
+    /**
+     * Gets the BGP speaker name.
+     *
+     * @return the BGP speaker name
+     */
+    public String name() {
+        return name;
+    }
+
+    /**
+     * Gets the connect point where the BGP speaker is attached.
+     *
+     * @return the connect point
+     */
+    public ConnectPoint connectPoint() {
+        return connectPoint;
+    }
+
+    /**
+     * Gets the MAC address of the BGP speaker.
+     *
+     * @return the MAC address
+     */
+    public MacAddress macAddress() {
+        return macAddress;
+    }
+
+    /**
+     * Gets all IP addresses configured on all {@link Interface}s of the
+     * BGP speaker.
+     *
+     * @return a list of IP addresses of the BGP speaker configured on all
+     * virtual interfaces
+     */
+    public List<InterfaceAddress> interfaceAddresses() {
+        return interfaceAddresses;
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        if (!(other instanceof BgpSpeaker)) {
+            return false;
+        }
+
+        BgpSpeaker otherBgpSpeaker = (BgpSpeaker) other;
+
+        return  name.equals(otherBgpSpeaker.name) &&
+                connectPoint.equals(
+                        otherBgpSpeaker.connectPoint) &&
+                macAddress.equals(otherBgpSpeaker.macAddress) &&
+                interfaceAddresses.equals(otherBgpSpeaker.interfaceAddresses);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(name, connectPoint, macAddress,
+                interfaceAddresses);
+
+    }
+
+    @Override
+    public String toString() {
+        return MoreObjects.toStringHelper(getClass())
+                .add("speakerName", name)
+                .add("connectPoint", connectPoint)
+                .add("macAddress", macAddress)
+                .add("interfaceAddresses", interfaceAddresses)
+                .toString();
+    }
+}

+package org.onlab.onos.sdnip.config;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.codehaus.jackson.annotate.JsonProperty;
+
+/**
+ * Contains the configuration data for SDN-IP that has been read from a
+ * JSON-formatted configuration file.
+ */
+public class Configuration {
+    // We call the BGP routers in our SDN network the BGP speakers, and call
+    // the BGP routers outside our SDN network the BGP peers.
+    private List<BgpSpeaker> bgpSpeakers;
+    private List<BgpPeer> peers;
+
+    /**
+     * Default constructor.
+     */
+    public Configuration() {
+    }
+
+    /**
+     * Gets a list of bgpSpeakers in the system, represented by
+     * {@link BgpSpeaker} objects.
+     *
+     * @return the list of BGP speakers
+     */
+    public List<BgpSpeaker> getBgpSpeakers() {
+        return Collections.unmodifiableList(bgpSpeakers);
+    }
+
+    /**
+     * Sets a list of bgpSpeakers in the system.
+     *
+     * @param bgpSpeakers the list of BGP speakers
+     */
+    @JsonProperty("bgpSpeakers")
+    public void setBgpSpeakers(List<BgpSpeaker> bgpSpeakers) {
+        this.bgpSpeakers = bgpSpeakers;
+    }
+
+    /**
+     * Gets a list of BGP peers we are configured to peer with. Peers are
+     * represented by {@link BgpPeer} objects.
+     *
+     * @return the list of BGP peers
+     */
+    public List<BgpPeer> getPeers() {
+        return Collections.unmodifiableList(peers);
+    }
+
+    /**
+     * Sets a list of BGP peers we are configured to peer with.
+     *
+     * @param peers the list of BGP peers
+     */
+    @JsonProperty("bgpPeers")
+    public void setPeers(List<BgpPeer> peers) {
+        this.peers = peers;
+    }
+
+}

+package org.onlab.onos.sdnip.config;
+
+import java.util.Objects;
+import java.util.Set;
+
+import org.onlab.onos.net.ConnectPoint;
+import org.onlab.onos.net.host.PortAddresses;
+import org.onlab.packet.IpPrefix;
+import org.onlab.packet.MacAddress;
+
+import com.google.common.base.MoreObjects;
+import com.google.common.collect.Sets;
+
+/**
+ * An Interface is a set of addresses that are logically mapped to a switch
+ * port in the network.
+ */
+public class Interface {
+    private final ConnectPoint connectPoint;
+    private final Set<IpPrefix> ipAddresses;
+    private final MacAddress macAddress;
+
+    /**
+     * Creates an Interface based on a connection point, a set of IP addresses
+     * and a MAC address.
+     *
+     * @param connectPoint the connect point this interface is mapped to
+     * @param prefixAddress the IP addresses for the interface
+     * @param macAddress the MAC address of the interface
+     */
+    public Interface(ConnectPoint connectPoint, Set<IpPrefix> prefixAddress,
+                     MacAddress macAddress) {
+        this.connectPoint = connectPoint;
+        this.ipAddresses = Sets.newHashSet(prefixAddress);
+        this.macAddress = macAddress;
+    }
+
+    /**
+     * Creates an Interface based on a PortAddresses object.
+     *
+     * @param portAddresses the PortAddresses object to turn into an Interface
+     */
+    public Interface(PortAddresses portAddresses) {
+        connectPoint = portAddresses.connectPoint();
+        ipAddresses = Sets.newHashSet(portAddresses.ips());
+        macAddress = portAddresses.mac();
+    }
+
+    /**
+     * Retrieves the connection point that this interface maps to.
+     *
+     * @return the connection point
+     */
+    public ConnectPoint connectPoint() {
+        return connectPoint;
+    }
+
+    /**
+     * Retrieves the set of IP addresses that are assigned to the interface.
+     *
+     * @return the set of IP addresses
+     */
+   public Set<IpPrefix> ips() {
+        return ipAddresses;
+    }
+
+   /**
+    * Retrieves the MAC address that is assigned to the interface.
+    *
+    * @return the MAC address
+    */
+   public MacAddress mac() {
+       return macAddress;
+   }
+
+    @Override
+    public boolean equals(Object other) {
+        if (!(other instanceof Interface)) {
+            return false;
+        }
+
+        Interface otherInterface = (Interface) other;
+
+        return  connectPoint.equals(otherInterface.connectPoint) &&
+                ipAddresses.equals(otherInterface.ipAddresses) &&
+                macAddress.equals(otherInterface.macAddress);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(connectPoint, ipAddresses, macAddress);
+    }
+
+    @Override
+    public String toString() {
+        return MoreObjects.toStringHelper(getClass())
+                .add("connectPoint", connectPoint)
+                .add("ipAddresses", ipAddresses)
+                .add("macAddress", macAddress)
+                .toString();
+    }
+}

+package org.onlab.onos.sdnip.config;
+
+import java.util.Objects;
+
+import org.codehaus.jackson.annotate.JsonProperty;
+import org.onlab.onos.net.ConnectPoint;
+import org.onlab.onos.net.DeviceId;
+import org.onlab.onos.net.PortNumber;
+import org.onlab.packet.IpAddress;
+
+import com.google.common.base.MoreObjects;
+
+/**
+ * Represents an address of a {@link BgpSpeaker} configured on an
+ * {@link Interface}.
+ * <p/>
+ * Each InterfaceAddress includes the interface name and an IP address.
+ */
+public class InterfaceAddress {
+    private final ConnectPoint connectPoint;
+    private final IpAddress ipAddress;
+
+    /**
+     * Creates an InterfaceAddress object.
+     *
+     * @param dpid the DPID of the interface as a String
+     * @param port the port of the interface
+     * @param ipAddress the IP address of a {@link BgpSpeaker} configured on
+     * the interface
+     */
+    public InterfaceAddress(@JsonProperty("interfaceDpid") String dpid,
+                            @JsonProperty("interfacePort") int port,
+                            @JsonProperty("ipAddress") String ipAddress) {
+        this.connectPoint = new ConnectPoint(
+                DeviceId.deviceId(SdnIpConfigReader.dpidToUri(dpid)),
+                PortNumber.portNumber(port));
+        this.ipAddress = IpAddress.valueOf(ipAddress);
+    }
+
+    /**
+     * Gets the connection point of the peer.
+     *
+     * @return the connection point
+     */
+    public ConnectPoint connectPoint() {
+        return connectPoint;
+    }
+
+    /**
+     * Gets the IP address of a BGP speaker configured on an {@link Interface}.
+     *
+     * @return the IP address
+     */
+    public IpAddress ipAddress() {
+        return ipAddress;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(connectPoint, ipAddress);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == this) {
+            return true;
+        }
+
+        if (!(obj instanceof InterfaceAddress)) {
+            return false;
+        }
+
+        InterfaceAddress that = (InterfaceAddress) obj;
+        return Objects.equals(this.connectPoint, that.connectPoint)
+                && Objects.equals(this.ipAddress, that.ipAddress);
+    }
+
+    @Override
+    public String toString() {
+        return MoreObjects.toStringHelper(getClass())
+                .add("connectPoint", connectPoint)
+                .add("ipAddress", ipAddress)
+                .toString();
+    }
+}

+package org.onlab.onos.sdnip.config;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.codehaus.jackson.map.ObjectMapper;
+import org.onlab.packet.IpAddress;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * SDN-IP Config Reader provides IConfigInfoService
+ * by reading from an SDN-IP configuration file.
+ * It must be enabled on the nodes within the cluster
+ * not running SDN-IP.
+ * <p/>
+ * TODO: As a long term solution, a module providing
+ * general network configuration to ONOS nodes should be used.
+ */
+public class SdnIpConfigReader implements SdnIpConfigService {
+
+    private static final Logger log = LoggerFactory.getLogger(SdnIpConfigReader.class);
+
+    private static final String DEFAULT_CONFIG_FILE = "config/sdnip.json";
+    private String configFileName = DEFAULT_CONFIG_FILE;
+    //private Map<String, Interface> interfaces;
+    // We call the BGP routers in our SDN network the BGP speakers, and call
+    // the BGP routers outside our SDN network the BGP peers.
+    private Map<String, BgpSpeaker> bgpSpeakers;
+    private Map<IpAddress, BgpPeer> bgpPeers;
+    //private InvertedRadixTree<Interface> interfaceRoutes;
+
+    /**
+     * Reads the info contained in the configuration file.
+     *
+     * @param configFilename The name of configuration file for SDN-IP application.
+     */
+    private void readConfiguration(String configFilename) {
+        File gatewaysFile = new File(configFilename);
+        ObjectMapper mapper = new ObjectMapper();
+
+        try {
+            Configuration config = mapper.readValue(gatewaysFile, Configuration.class);
+            /*interfaces = new ConcurrentHashMap<>();
+            for (Interface intf : config.getInterfaces()) {
+                interfaces.put(intf.getName(), intf);
+            }*/
+            bgpSpeakers = new ConcurrentHashMap<>();
+            for (BgpSpeaker speaker : config.getBgpSpeakers()) {
+                bgpSpeakers.put(speaker.name(), speaker);
+            }
+            bgpPeers = new ConcurrentHashMap<>();
+            for (BgpPeer peer : config.getPeers()) {
+                bgpPeers.put(peer.ipAddress(), peer);
+            }
+        } catch (IOException e) {
+            log.error("Error reading JSON file", e);
+            //throw new ConfigurationRuntimeException("Error in JSON file", e);
+        }
+
+        // Populate the interface InvertedRadixTree
+        /*for (Interface intf : interfaces.values()) {
+            Ip4Prefix prefix = intf.getIp4Prefix();
+            String binaryString = RouteEntry.createBinaryString(prefix);
+            interfaceRoutes.put(binaryString, intf);
+        }*/
+    }
+
+    /**
+     * To find the Interface which has longest matchable IP prefix (sub-network
+     *  prefix) to next hop IP address.
+     *
+     * @param address the IP address of next hop router
+     * @return the Interface which has longest matchable IP prefix
+     */
+    /*private Interface longestInterfacePrefixMatch(IpAddress address) {
+        Ip4Prefix prefixToSearchFor =
+            new Ip4Prefix(address, (short) Ip4Address.BIT_LENGTH);
+        String binaryString = RouteEntry.createBinaryString(prefixToSearchFor);
+
+        Iterator<Interface> it =
+            interfaceRoutes.getValuesForKeysPrefixing(binaryString).iterator();
+        Interface intf = null;
+        // Find the last prefix, which will be the longest prefix
+        while (it.hasNext()) {
+            intf = it.next();
+        }
+
+        return intf;
+    }*/
+
+    /*@Override
+    public Interface getOutgoingInterface(IpAddress dstIpAddress) {
+        return longestInterfacePrefixMatch(dstIpAddress);
+    }*/
+
+    public void init() {
+        //interfaceRoutes = new ConcurrentInvertedRadixTree<>(
+                //new DefaultByteArrayNodeFactory());
+
+        // Reading config values
+        /*String configFilenameParameter = context.getConfigParams(this).get("configfile");
+        if (configFilenameParameter != null) {
+            currentConfigFilename = configFilenameParameter;
+        }*/
+        log.debug("Config file set to {}", configFileName);
+
+        readConfiguration(configFileName);
+    }
+
+    /*@Override
+    public Map<String, Interface> getInterfaces() {
+        return Collections.unmodifiableMap(interfaces);
+    }*/
+
+    @Override
+    public Map<String, BgpSpeaker> getBgpSpeakers() {
+        return Collections.unmodifiableMap(bgpSpeakers);
+    }
+
+    @Override
+    public Map<IpAddress, BgpPeer> getBgpPeers() {
+        return Collections.unmodifiableMap(bgpPeers);
+    }
+
+    static String dpidToUri(String dpid) {
+        return "of:" + dpid.replace(":", "");
+    }
+}

+package org.onlab.onos.sdnip.config;
+
+import java.util.Map;
+
+import org.onlab.packet.IpAddress;
+
+/**
+ * Provides information about the layer 3 properties of the network.
+ * This is based on IP addresses configured on ports in the network.
+ */
+public interface SdnIpConfigService {
+
+    /**
+     * Gets the list of virtual external-facing interfaces.
+     *
+     * @return the map of interface names to interface objects
+     */
+    //public Map<String, Interface> getInterfaces();
+
+    /**
+     * Gets the list of BGP speakers inside the SDN network.
+     *
+     * @return the map of BGP speaker names to BGP speaker objects
+     */
+    public Map<String, BgpSpeaker> getBgpSpeakers();
+
+    /**
+     * Gets the list of configured BGP peers.
+     *
+     * @return the map from peer IP address to BgpPeer object
+     */
+    public Map<IpAddress, BgpPeer> getBgpPeers();
+
+    /**
+     * Gets the Interface object for the interface that packets
+     * to dstIpAddress will be sent out of. Returns null if dstIpAddress is not
+     * in a directly connected network, or if no interfaces are configured.
+     *
+     * @param dstIpAddress destination IP address that we want to match to
+     *                     an outgoing interface
+     * @return the Interface object if one is found, otherwise null
+     */
+    //public Interface getOutgoingInterface(IpAddress dstIpAddress);
+
+}

+/**
+ * SDN-IP configuration.
+ */
+package org.onlab.onos.sdnip.config;
\ No newline at end of file

@@ -45,7 +45,7 @@
             <action class="org.onlab.onos.cli.net.DeviceRoleCommand"/>
             <completers>
                 <ref component-id="deviceIdCompleter"/>
-                <ref component-id="roleCompleter"/>
+                <ref component-id="nodeIdCompleter"/>
                 <ref component-id="roleCompleter"/>
                 <null/>
             </completers>

@@ -20,6 +20,7 @@ public class DefaultHost extends AbstractElement implements Host {
     private final MacAddress mac;
     private final VlanId vlan;
     private final HostLocation location;
+    // FIXME: should be IpAddress
     private final Set<IpPrefix> ips;
 
     /**

@@ -38,6 +38,7 @@ public interface Host extends Element {
      *
      * @return set of IP addresses; empty if no IP address is bound
      */
+    // FIXME: Switch to IpAddress
     Set<IpPrefix> ipAddresses();
 
     /**

@@ -4,8 +4,6 @@ import static com.google.common.base.Preconditions.checkNotNull;
 
 import java.util.Objects;
 
-import org.onlab.onos.net.link.LinkDescription;
-
 import com.google.common.base.MoreObjects;
 
 // TODO Consider renaming.
@@ -69,16 +67,6 @@ public final class LinkKey {
         return new LinkKey(link.src(), link.dst());
     }
 
-    /**
-     * Creates a link identifier for the specified link.
-     *
-     * @param desc link description
-     * @return a link identifier
-     */
-    public static LinkKey linkKey(LinkDescription desc) {
-        return new LinkKey(desc.src(), desc.dst());
-    }
-
     @Override
     public int hashCode() {
         return Objects.hash(src(), dst);

 package org.onlab.onos.net.host;
 
-import java.util.Set;
-
 import org.onlab.onos.net.ConnectPoint;
 import org.onlab.onos.net.HostId;
 
@@ -47,20 +45,4 @@ public interface HostAdminService {
      */
     void clearAddresses(ConnectPoint connectPoint);
 
-    /**
-     * Returns the addresses information for all connection points.
-     *
-     * @return the set of address bindings for all connection points
-     */
-    Set<PortAddresses> getAddressBindings();
-
-    /**
-     * Retrieves the addresses that have been bound to the given connection
-     * point.
-     *
-     * @param connectPoint the connection point to retrieve address bindings
-     * for
-     * @return addresses bound to the port
-     */
-    PortAddresses getAddressBindingsForPort(ConnectPoint connectPoint);
 }

@@ -37,6 +37,7 @@ public interface HostDescription extends Description {
      *
      * @return host IP address
      */
+    // FIXME: Switch to IpAddress
     IpPrefix ipAddress();
 
 }

@@ -110,6 +110,23 @@ public interface HostService {
     void requestMac(IpAddress ip);
 
     /**
+     * Returns the addresses information for all connection points.
+     *
+     * @return the set of address bindings for all connection points
+     */
+    Set<PortAddresses> getAddressBindings();
+
+    /**
+     * Retrieves the addresses that have been bound to the given connection
+     * point.
+     *
+     * @param connectPoint the connection point to retrieve address bindings
+     * for
+     * @return addresses bound to the port
+     */
+    PortAddresses getAddressBindingsForPort(ConnectPoint connectPoint);
+
+    /**
      * Adds the specified host listener.
      *
      * @param listener host listener

@@ -29,6 +29,7 @@ public interface HostStore extends Store<HostEvent, HostStoreDelegate> {
     HostEvent createOrUpdateHost(ProviderId providerId, HostId hostId,
                                  HostDescription hostDescription);
 
+    // FIXME: API to remove only IpAddress is missing
     /**
      * Removes the specified host from the inventory.
      *
@@ -81,6 +82,7 @@ public interface HostStore extends Store<HostEvent, HostStoreDelegate> {
      * @param ip ip address
      * @return set of hosts with the given IP
      */
+    // FIXME: Switch to IpAddress
     Set<Host> getHosts(IpPrefix ip);
 
     /**

@@ -17,6 +17,7 @@ import com.google.common.base.MoreObjects;
 public class PortAddresses {
 
     private final ConnectPoint connectPoint;
+    // TODO: Should this be IpAddress or IpPrefix?
     private final Set<IpPrefix> ipAddresses;
     private final MacAddress macAddress;
 

 package org.onlab.onos.net.proxyarp;
 
+import org.onlab.onos.net.ConnectPoint;
 import org.onlab.onos.net.packet.PacketContext;
 import org.onlab.packet.Ethernet;
 import org.onlab.packet.IpPrefix;
@@ -23,8 +24,9 @@ public interface ProxyArpService {
      * will be flooded at all edge ports.
      *
      * @param eth an arp request
+     * @param inPort the port the request was received on
      */
-    void reply(Ethernet eth);
+    void reply(Ethernet eth, ConnectPoint inPort);
 
     /**
      * Forwards an ARP request to its destination. Floods at the edge the ARP request if the

@@ -75,4 +75,14 @@ public class HostServiceAdapter implements HostService {
     public void removeListener(HostListener listener) {
     }
 
+    @Override
+    public Set<PortAddresses> getAddressBindings() {
+        return null;
+    }
+
+    @Override
+    public PortAddresses getAddressBindingsForPort(ConnectPoint connectPoint) {
+        return null;
+    }
+
 }

@@ -5,6 +5,7 @@ import static com.google.common.base.Preconditions.checkNotNull;
 import static org.slf4j.LoggerFactory.getLogger;
 
 import java.nio.ByteBuffer;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map.Entry;
 import java.util.Set;
@@ -15,6 +16,7 @@ import org.apache.felix.scr.annotations.Deactivate;
 import org.apache.felix.scr.annotations.Reference;
 import org.apache.felix.scr.annotations.ReferenceCardinality;
 import org.apache.felix.scr.annotations.Service;
+import org.onlab.onos.net.ConnectPoint;
 import org.onlab.onos.net.Device;
 import org.onlab.onos.net.Host;
 import org.onlab.onos.net.HostId;
@@ -27,6 +29,7 @@ import org.onlab.onos.net.device.DeviceService;
 import org.onlab.onos.net.flow.DefaultTrafficTreatment;
 import org.onlab.onos.net.flow.TrafficTreatment;
 import org.onlab.onos.net.host.HostService;
+import org.onlab.onos.net.host.PortAddresses;
 import org.onlab.onos.net.link.LinkEvent;
 import org.onlab.onos.net.link.LinkListener;
 import org.onlab.onos.net.link.LinkService;
@@ -37,7 +40,9 @@ import org.onlab.onos.net.packet.PacketService;
 import org.onlab.onos.net.proxyarp.ProxyArpService;
 import org.onlab.packet.ARP;
 import org.onlab.packet.Ethernet;
+import org.onlab.packet.IpAddress;
 import org.onlab.packet.IpPrefix;
+import org.onlab.packet.MacAddress;
 import org.onlab.packet.VlanId;
 import org.slf4j.Logger;
 
@@ -101,12 +106,46 @@ public class ProxyArpManager implements ProxyArpService {
     }
 
     @Override
-    public void reply(Ethernet eth) {
+    public void reply(Ethernet eth, ConnectPoint inPort) {
         checkNotNull(eth, REQUEST_NULL);
         checkArgument(eth.getEtherType() == Ethernet.TYPE_ARP,
                 REQUEST_NOT_ARP);
         ARP arp = (ARP) eth.getPayload();
         checkArgument(arp.getOpCode() == ARP.OP_REQUEST, NOT_ARP_REQUEST);
+        checkNotNull(inPort);
+
+        // If the source address matches one of our external addresses
+        // it could be a request from an internal host to an external
+        // address. Forward it over to the correct port.
+        IpAddress source = IpAddress.valueOf(arp.getSenderProtocolAddress());
+        PortAddresses sourceAddresses = findOutsidePortInSubnet(source);
+        if (sourceAddresses != null && !isOutsidePort(inPort)) {
+            for (IpPrefix subnet : sourceAddresses.ips()) {
+                if (subnet.toIpAddress().equals(source)) {
+                    sendTo(eth, sourceAddresses.connectPoint());
+                    return;
+                }
+            }
+        }
+
+        // If the request came from outside the network, only reply if it was
+        // for one of our external addresses.
+        if (isOutsidePort(inPort)) {
+            IpAddress target = IpAddress.valueOf(arp.getTargetProtocolAddress());
+            PortAddresses addresses = hostService.getAddressBindingsForPort(inPort);
+
+            for (IpPrefix interfaceAddress : addresses.ips()) {
+                if (interfaceAddress.toIpAddress().equals(target)) {
+                    Ethernet arpReply = buildArpReply(interfaceAddress,
+                            addresses.mac(), eth);
+                    sendTo(arpReply, inPort);
+                }
+            }
+
+            return;
+        }
+
+        // Continue with normal proxy ARP case
 
         VlanId vlan = VlanId.vlanId(eth.getVlanID());
         Set<Host> hosts = hostService.getHostsByIp(IpPrefix.valueOf(arp
@@ -128,12 +167,62 @@ public class ProxyArpManager implements ProxyArpService {
             return;
         }
 
-        Ethernet arpReply = buildArpReply(dst, eth);
+        Ethernet arpReply = buildArpReply(dst.ipAddresses().iterator().next(),
+                dst.mac(), eth);
         // TODO: check send status with host service.
+        sendTo(arpReply, src.location());
+    }
+
+    /**
+     * Outputs the given packet out the given port.
+     *
+     * @param packet the packet to send
+     * @param outPort the port to send it out
+     */
+    private void sendTo(Ethernet packet, ConnectPoint outPort) {
+        if (internalPorts.containsEntry(
+                deviceService.getDevice(outPort.deviceId()), outPort.port())) {
+            // Sanity check to make sure we don't send the packet out an
+            // internal port and create a loop (could happen due to
+            // misconfiguration).
+            return;
+        }
+
         TrafficTreatment.Builder builder = DefaultTrafficTreatment.builder();
-        builder.setOutput(src.location().port());
-        packetService.emit(new DefaultOutboundPacket(src.location().deviceId(),
-                builder.build(), ByteBuffer.wrap(arpReply.serialize())));
+        builder.setOutput(outPort.port());
+        packetService.emit(new DefaultOutboundPacket(outPort.deviceId(),
+                builder.build(), ByteBuffer.wrap(packet.serialize())));
+    }
+
+    /**
+     * Finds the port with an address in the subnet of the target address, if
+     * one exists.
+     *
+     * @param target the target address to find a matching external port for
+     * @return a PortAddresses object containing the external addresses if one
+     * was found, otherwise null.
+     */
+    private PortAddresses findOutsidePortInSubnet(IpAddress target) {
+        for (PortAddresses addresses : hostService.getAddressBindings()) {
+            for (IpPrefix prefix : addresses.ips()) {
+                if (prefix.contains(target)) {
+                    return new PortAddresses(addresses.connectPoint(),
+                            Collections.singleton(prefix), addresses.mac());
+                }
+            }
+        }
+        return null;
+    }
+
+    /**
+     * Returns whether the given port is an outside-facing port with an IP
+     * address configured.
+     *
+     * @param port the port to check
+     * @return true if the port is an outside-facing port, otherwise false
+     */
+    private boolean isOutsidePort(ConnectPoint port) {
+        return !hostService.getAddressBindingsForPort(port).ips().isEmpty();
     }
 
     @Override
@@ -167,7 +256,7 @@ public class ProxyArpManager implements ProxyArpService {
             if (arp.getOpCode() == ARP.OP_REPLY) {
                 forward(ethPkt);
             } else if (arp.getOpCode() == ARP.OP_REQUEST) {
-                reply(ethPkt);
+                reply(ethPkt, context.inPacket().receivedFrom());
             }
             context.block();
             return true;
@@ -185,12 +274,16 @@ public class ProxyArpManager implements ProxyArpService {
 
         synchronized (externalPorts) {
             for (Entry<Device, PortNumber> entry : externalPorts.entries()) {
+                ConnectPoint cp = new ConnectPoint(entry.getKey().id(), entry.getValue());
+                if (isOutsidePort(cp)) {
+                    continue;
+                }
+
                 builder = DefaultTrafficTreatment.builder();
                 builder.setOutput(entry.getValue());
                 packetService.emit(new DefaultOutboundPacket(entry.getKey().id(),
                         builder.build(), buf));
             }
-
         }
     }
 
@@ -234,15 +327,19 @@ public class ProxyArpManager implements ProxyArpService {
     }
 
     /**
-     * Builds an arp reply based on a request.
-     * @param h the host we want to send to
-     * @param request the arp request we got
-     * @return an ethernet frame containing the arp reply
+     * Builds an ARP reply based on a request.
+     *
+     * @param srcIp the IP address to use as the reply source
+     * @param srcMac the MAC address to use as the reply source
+     * @param request the ARP request we got
+     * @return an Ethernet frame containing the ARP reply
      */
-    private Ethernet buildArpReply(Host h, Ethernet request) {
+    private Ethernet buildArpReply(IpPrefix srcIp, MacAddress srcMac,
+            Ethernet request) {
+
         Ethernet eth = new Ethernet();
         eth.setDestinationMACAddress(request.getSourceMACAddress());
-        eth.setSourceMACAddress(h.mac().getAddress());
+        eth.setSourceMACAddress(srcMac.getAddress());
         eth.setEtherType(Ethernet.TYPE_ARP);
         eth.setVlanID(request.getVlanID());
 
@@ -253,12 +350,12 @@ public class ProxyArpManager implements ProxyArpService {
 
         arp.setProtocolAddressLength((byte) IpPrefix.INET_LEN);
         arp.setHardwareAddressLength((byte) Ethernet.DATALAYER_ADDRESS_LENGTH);
-        arp.setSenderHardwareAddress(h.mac().getAddress());
+        arp.setSenderHardwareAddress(srcMac.getAddress());
         arp.setTargetHardwareAddress(request.getSourceMACAddress());
 
         arp.setTargetProtocolAddress(((ARP) request.getPayload())
                 .getSenderProtocolAddress());
-        arp.setSenderProtocolAddress(h.ipAddresses().iterator().next().toRealInt());
+        arp.setSenderProtocolAddress(srcIp.toRealInt());
         eth.setPayload(arp);
         return eth;
     }

@@ -13,6 +13,7 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;
+import java.util.Set;
 
 import org.junit.Before;
 import org.junit.Test;
@@ -31,6 +32,7 @@ import org.onlab.onos.net.device.DeviceService;
 import org.onlab.onos.net.flow.instructions.Instruction;
 import org.onlab.onos.net.flow.instructions.Instructions.OutputInstruction;
 import org.onlab.onos.net.host.HostService;
+import org.onlab.onos.net.host.PortAddresses;
 import org.onlab.onos.net.link.LinkListener;
 import org.onlab.onos.net.link.LinkService;
 import org.onlab.onos.net.packet.OutboundPacket;
@@ -50,12 +52,13 @@ import com.google.common.collect.Sets;
  */
 public class ProxyArpManagerTest {
 
-    private static final int NUM_DEVICES = 4;
+    private static final int NUM_DEVICES = 6;
     private static final int NUM_PORTS_PER_DEVICE = 3;
-    private static final int NUM_FLOOD_PORTS = 4;
+    private static final int NUM_ADDRESS_PORTS = NUM_DEVICES / 2;
+    private static final int NUM_FLOOD_PORTS = 3;
 
-    private static final IpPrefix IP1 = IpPrefix.valueOf("10.0.0.1/24");
-    private static final IpPrefix IP2 = IpPrefix.valueOf("10.0.0.2/24");
+    private static final IpPrefix IP1 = IpPrefix.valueOf("192.168.1.1/24");
+    private static final IpPrefix IP2 = IpPrefix.valueOf("192.168.1.2/24");
 
     private static final ProviderId PID = new ProviderId("of", "foo");
 
@@ -104,6 +107,9 @@ public class ProxyArpManagerTest {
      * The default topology is a unidirectional ring topology. Each switch has
      * 3 ports. Ports 2 and 3 have the links to neighbor switches, and port 1
      * is free (edge port).
+     * The first half of the switches have IP addresses configured on their
+     * free ports (port 1). The second half of the switches have no IP
+     * addresses configured.
      */
     private void createTopology() {
         deviceService = createMock(DeviceService.class);
@@ -114,6 +120,7 @@ public class ProxyArpManagerTest {
 
         createDevices(NUM_DEVICES, NUM_PORTS_PER_DEVICE);
         createLinks(NUM_DEVICES);
+        addAddressBindings();
     }
 
     /**
@@ -138,10 +145,11 @@ public class ProxyArpManagerTest {
                 ports.add(port);
             }
 
-            expect(deviceService.getPorts(devId)).andReturn(ports);
+            expect(deviceService.getPorts(devId)).andReturn(ports).anyTimes();
+            expect(deviceService.getDevice(devId)).andReturn(device).anyTimes();
         }
 
-        expect(deviceService.getDevices()).andReturn(devices);
+        expect(deviceService.getDevices()).andReturn(devices).anyTimes();
         replay(deviceService);
     }
 
@@ -173,6 +181,31 @@ public class ProxyArpManagerTest {
         replay(linkService);
     }
 
+    private void addAddressBindings() {
+        Set<PortAddresses> addresses = Sets.newHashSet();
+
+        for (int i = 1; i <= NUM_ADDRESS_PORTS; i++) {
+            ConnectPoint cp = new ConnectPoint(getDeviceId(i), P1);
+            IpPrefix prefix1 = IpPrefix.valueOf("10.0." + (2 * i - 1) + ".1/24");
+            IpPrefix prefix2 = IpPrefix.valueOf("10.0." + (2 * i) + ".1/24");
+            PortAddresses pa = new PortAddresses(cp,
+                    Sets.newHashSet(prefix1, prefix2), MacAddress.valueOf(i));
+            addresses.add(pa);
+
+            expect(hostService.getAddressBindingsForPort(cp))
+                    .andReturn(pa).anyTimes();
+        }
+
+        expect(hostService.getAddressBindings()).andReturn(addresses).anyTimes();
+
+        for (int i = 1; i <= NUM_FLOOD_PORTS; i++) {
+            ConnectPoint cp = new ConnectPoint(getDeviceId(i + NUM_ADDRESS_PORTS),
+                    P1);
+            expect(hostService.getAddressBindingsForPort(cp))
+                    .andReturn(new PortAddresses(cp, null, null)).anyTimes();
+        }
+    }
+
     /**
      * Tests {@link ProxyArpManager#known(IpPrefix)} in the case where the
      * IP address is not known.
@@ -210,10 +243,10 @@ public class ProxyArpManagerTest {
      */
     @Test
     public void testReplyKnown() {
-        Host replyer = new DefaultHost(PID, HID1, MAC1, VLAN1, LOC2,
+        Host replyer = new DefaultHost(PID, HID1, MAC1, VLAN1, getLocation(4),
                 Collections.singleton(IP1));
 
-        Host requestor = new DefaultHost(PID, HID2, MAC2, VLAN1, LOC1,
+        Host requestor = new DefaultHost(PID, HID2, MAC2, VLAN1, getLocation(5),
                 Collections.singleton(IP2));
 
         expect(hostService.getHostsByIp(IpPrefix.valueOf(IP1.toOctets())))
@@ -224,11 +257,11 @@ public class ProxyArpManagerTest {
 
         Ethernet arpRequest = buildArp(ARP.OP_REQUEST, MAC2, null, IP2, IP1);
 
-        proxyArp.reply(arpRequest);
+        proxyArp.reply(arpRequest, getLocation(5));
 
         assertEquals(1, packetService.packets.size());
         Ethernet arpReply = buildArp(ARP.OP_REPLY, MAC1, MAC2, IP1, IP2);
-        verifyPacketOut(arpReply, LOC1, packetService.packets.get(0));
+        verifyPacketOut(arpReply, getLocation(5), packetService.packets.get(0));
     }
 
     /**
@@ -238,7 +271,7 @@ public class ProxyArpManagerTest {
      */
     @Test
     public void testReplyUnknown() {
-        Host requestor = new DefaultHost(PID, HID2, MAC2, VLAN1, LOC1,
+        Host requestor = new DefaultHost(PID, HID2, MAC2, VLAN1, getLocation(5),
                 Collections.singleton(IP2));
 
         expect(hostService.getHostsByIp(IpPrefix.valueOf(IP1.toOctets())))
@@ -249,7 +282,7 @@ public class ProxyArpManagerTest {
 
         Ethernet arpRequest = buildArp(ARP.OP_REQUEST, MAC2, null, IP2, IP1);
 
-        proxyArp.reply(arpRequest);
+        proxyArp.reply(arpRequest, getLocation(5));
 
         verifyFlood(arpRequest);
     }
@@ -262,10 +295,10 @@ public class ProxyArpManagerTest {
      */
     @Test
     public void testReplyDifferentVlan() {
-        Host replyer = new DefaultHost(PID, HID1, MAC1, VLAN2, LOC2,
+        Host replyer = new DefaultHost(PID, HID1, MAC1, VLAN2, getLocation(4),
                 Collections.singleton(IP1));
 
-        Host requestor = new DefaultHost(PID, HID2, MAC2, VLAN1, LOC1,
+        Host requestor = new DefaultHost(PID, HID2, MAC2, VLAN1, getLocation(5),
                 Collections.singleton(IP2));
 
         expect(hostService.getHostsByIp(IpPrefix.valueOf(IP1.toOctets())))
@@ -276,11 +309,84 @@ public class ProxyArpManagerTest {
 
         Ethernet arpRequest = buildArp(ARP.OP_REQUEST, MAC2, null, IP2, IP1);
 
-        proxyArp.reply(arpRequest);
+        proxyArp.reply(arpRequest, getLocation(5));
 
         verifyFlood(arpRequest);
     }
 
+    @Test
+    public void testReplyToRequestForUs() {
+        IpPrefix theirIp = IpPrefix.valueOf("10.0.1.254/24");
+        IpPrefix ourFirstIp = IpPrefix.valueOf("10.0.1.1/24");
+        IpPrefix ourSecondIp = IpPrefix.valueOf("10.0.2.1/24");
+        MacAddress ourMac = MacAddress.valueOf(1L);
+
+        Host requestor = new DefaultHost(PID, HID2, MAC2, VLAN1, LOC1,
+                Collections.singleton(theirIp));
+
+        expect(hostService.getHost(HID2)).andReturn(requestor);
+        replay(hostService);
+
+        Ethernet arpRequest = buildArp(ARP.OP_REQUEST, MAC2, null, theirIp, ourFirstIp);
+
+        proxyArp.reply(arpRequest, LOC1);
+
+        assertEquals(1, packetService.packets.size());
+        Ethernet arpReply = buildArp(ARP.OP_REPLY, ourMac, MAC2, ourFirstIp, theirIp);
+        verifyPacketOut(arpReply, LOC1, packetService.packets.get(0));
+
+        // Test a request for the second address on that port
+        packetService.packets.clear();
+        arpRequest = buildArp(ARP.OP_REQUEST, MAC2, null, theirIp, ourSecondIp);
+
+        proxyArp.reply(arpRequest, LOC1);
+
+        assertEquals(1, packetService.packets.size());
+        arpReply = buildArp(ARP.OP_REPLY, ourMac, MAC2, ourSecondIp, theirIp);
+        verifyPacketOut(arpReply, LOC1, packetService.packets.get(0));
+    }
+
+    @Test
+    public void testReplyExternalPortBadRequest() {
+        replay(hostService); // no further host service expectations
+
+        IpPrefix theirIp = IpPrefix.valueOf("10.0.1.254/24");
+
+        // Request for a valid external IP address but coming in the wrong port
+        Ethernet arpRequest = buildArp(ARP.OP_REQUEST, MAC1, null, theirIp,
+                IpPrefix.valueOf("10.0.3.1"));
+        proxyArp.reply(arpRequest, LOC1);
+        assertEquals(0, packetService.packets.size());
+
+        // Request for a valid internal IP address but coming in an external port
+        packetService.packets.clear();
+        arpRequest = buildArp(ARP.OP_REQUEST, MAC1, null, theirIp, IP1);
+        proxyArp.reply(arpRequest, LOC1);
+        assertEquals(0, packetService.packets.size());
+    }
+
+    @Test
+    public void testReplyToRequestFromUs() {
+        replay(hostService); // no further host service expectations
+
+        IpPrefix ourIp = IpPrefix.valueOf("10.0.1.1/24");
+        MacAddress ourMac = MacAddress.valueOf(1L);
+        IpPrefix theirIp = IpPrefix.valueOf("10.0.1.100/24");
+
+        // This is a request from something inside our network (like a BGP
+        // daemon) to an external host.
+        Ethernet arpRequest = buildArp(ARP.OP_REQUEST, ourMac, null, ourIp, theirIp);
+        proxyArp.reply(arpRequest, getLocation(5));
+
+        assertEquals(1, packetService.packets.size());
+        verifyPacketOut(arpRequest, getLocation(1), packetService.packets.get(0));
+
+        // The same request from a random external port should fail
+        packetService.packets.clear();
+        proxyArp.reply(arpRequest, getLocation(2));
+        assertEquals(0, packetService.packets.size());
+    }
+
     /**
      * Tests {@link ProxyArpManager#forward(Ethernet)} in the case where the
      * destination host is known.
@@ -338,7 +444,8 @@ public class ProxyArpManagerTest {
             });
 
         for (int i = 0; i < NUM_FLOOD_PORTS; i++) {
-            ConnectPoint cp = new ConnectPoint(getDeviceId(i + 1), PortNumber.portNumber(1));
+            ConnectPoint cp = new ConnectPoint(getDeviceId(NUM_ADDRESS_PORTS + i + 1),
+                    PortNumber.portNumber(1));
 
             OutboundPacket outboundPacket = packetService.packets.get(i);
             verifyPacketOut(packet, cp, outboundPacket);
@@ -372,6 +479,10 @@ public class ProxyArpManagerTest {
         return DeviceId.deviceId("" + i);
     }
 
+    private static HostLocation getLocation(int i) {
+        return new HostLocation(new ConnectPoint(getDeviceId(i), P1), 123L);
+    }
+
     /**
      * Builds an ARP packet with the given parameters.
      *

 package org.onlab.onos.net.topology.impl;
 
 import com.google.common.collect.ImmutableSet;
+
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -21,10 +22,12 @@ import org.onlab.onos.net.topology.TopologyProviderService;
 
 import java.util.List;
 import java.util.Set;
+import java.util.concurrent.Phaser;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.onlab.junit.TestTools.assertAfter;
+import static org.junit.Assert.*;
+import static org.hamcrest.Matchers.*;
 import static org.onlab.onos.net.NetTestTools.device;
 import static org.onlab.onos.net.NetTestTools.link;
 import static org.onlab.onos.net.device.DeviceEvent.Type.DEVICE_ADDED;
@@ -41,6 +44,9 @@ public class DefaultTopologyProviderTest {
     private TestLinkService linkService = new TestLinkService();
     private TestTopoProviderService providerService;
 
+    // phase corresponds to number of topologyChanged called
+    private Phaser topologyChangedCounts = new Phaser(1);
+
     @Before
     public void setUp() {
         provider.deviceService = deviceService;
@@ -66,27 +72,24 @@ public class DefaultTopologyProviderTest {
     }
 
     @Test
-    public void basics() {
-        assertAfter(100, new Runnable() {
-            @Override
-            public void run() {
+    public void basics() throws InterruptedException, TimeoutException {
+        assertEquals(1, topologyChangedCounts.awaitAdvanceInterruptibly(0, 1, TimeUnit.SECONDS));
         validateSubmission();
     }
-        });
-    }
 
     @Test
-    public void eventDriven() {
-        assertAfter(100, new Runnable() {
-            @Override
-            public void run() {
+    public void eventDriven() throws InterruptedException, TimeoutException {
+        assertEquals(1, topologyChangedCounts.awaitAdvanceInterruptibly(0, 1, TimeUnit.SECONDS));
         validateSubmission();
+
         deviceService.post(new DeviceEvent(DEVICE_ADDED, device("z"), null));
         linkService.post(new LinkEvent(LINK_ADDED, link("z", 1, "a", 4)));
+        assertThat(topologyChangedCounts.awaitAdvanceInterruptibly(1, 1, TimeUnit.SECONDS),
+                is(greaterThanOrEqualTo(2)));
+        // Note: posting event, to trigger topologyChanged call,
+        // but dummy topology will not change.
         validateSubmission();
     }
-        });
-    }
 
 
     private class TestTopoRegistry implements TopologyProviderRegistry {
@@ -119,6 +122,7 @@ public class DefaultTopologyProviderTest {
         @Override
         public void topologyChanged(GraphDescription graphDescription, List<Event> reasons) {
             graphDesc = graphDescription;
+            topologyChangedCounts.arrive();
         }
     }
 

@@ -18,7 +18,6 @@ import org.onlab.onos.cluster.DefaultControllerNode;
 import org.onlab.onos.cluster.NodeId;
 import org.onlab.onos.store.AbstractStore;
 import org.onlab.onos.store.cluster.messaging.ClusterCommunicationAdminService;
-import org.onlab.onos.store.cluster.messaging.impl.ClusterCommunicationManager;
 import org.onlab.packet.IpPrefix;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -48,7 +47,7 @@ public class DistributedClusterStore
     private final Map<NodeId, State> states = new ConcurrentHashMap<>();
     private final Cache<NodeId, ControllerNode> livenessCache = CacheBuilder.newBuilder()
             .maximumSize(1000)
-            .expireAfterWrite(ClusterCommunicationManager.HEART_BEAT_INTERVAL_MILLIS * 3, TimeUnit.MILLISECONDS)
+            .expireAfterWrite(/*ClusterCommunicationManager.HEART_BEAT_INTERVAL_MILLIS * */3, TimeUnit.MILLISECONDS)
             .removalListener(new LivenessCacheRemovalListener()).build();
 
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)

-package org.onlab.onos.store.cluster.messaging;
-
-import org.onlab.onos.cluster.ControllerNode;
-import org.onlab.onos.store.cluster.impl.ClusterNodesDelegate;
-
-// TODO: This service interface can be removed, once we properly start
-// using ClusterService
-/**
- * Service for administering communications manager.
- */
-public interface ClusterCommunicationAdminService {
-
-    /**
-     * Initialize.
-     */
-    void initialize(ControllerNode localNode, ClusterNodesDelegate nodesDelegate);
-
-    /**
-     * Adds the node to the list of monitored nodes.
-     *
-     * @param node node to be added
-     */
-    void addNode(ControllerNode node);
-
-    /**
-     * Removes the node from the list of monitored nodes.
-     *
-     * @param node node to be removed
-     */
-    void removeNode(ControllerNode node);
-}
\ No newline at end of file

-package org.onlab.onos.store.cluster.messaging;
-
-// FIXME: not used any more? remove
-/**
- * Service for encoding & decoding intra-cluster message payload.
- */
-public interface SerializationService {
-
-    /**
-     * Decodes the specified byte buffer to obtain the message within.
-     *
-     * @param buffer byte buffer with message(s)
-     * @return parsed message
-     */
-    <T> T decode(byte[] data);
-
-    /**
-     * Encodes the specified message into the given byte buffer.
-     *
-     * @param message message to be encoded
-     * @param buffer byte buffer to receive the message data
-     */
-    byte[] encode(Object message);
-
-}

@@ -4,8 +4,6 @@ import static com.google.common.base.Preconditions.checkArgument;
 
 import java.io.IOException;
 import java.util.Set;
-import java.util.Timer;
-import java.util.TimerTask;
 import org.apache.felix.scr.annotations.Activate;
 import org.apache.felix.scr.annotations.Component;
 import org.apache.felix.scr.annotations.Deactivate;
@@ -16,9 +14,6 @@ import org.onlab.onos.cluster.ClusterService;
 import org.onlab.onos.cluster.ControllerNode;
 import org.onlab.onos.cluster.NodeId;
 import org.onlab.onos.store.cluster.impl.ClusterMembershipEvent;
-import org.onlab.onos.store.cluster.impl.ClusterMembershipEventType;
-import org.onlab.onos.store.cluster.impl.ClusterNodesDelegate;
-import org.onlab.onos.store.cluster.messaging.ClusterCommunicationAdminService;
 import org.onlab.onos.store.cluster.messaging.ClusterCommunicationService;
 import org.onlab.onos.store.cluster.messaging.ClusterMessage;
 import org.onlab.onos.store.cluster.messaging.ClusterMessageHandler;
@@ -39,19 +34,13 @@ import org.slf4j.LoggerFactory;
 @Component(immediate = true)
 @Service
 public class ClusterCommunicationManager
-        implements ClusterCommunicationService, ClusterCommunicationAdminService {
+        implements ClusterCommunicationService {
 
     private final Logger log = LoggerFactory.getLogger(getClass());
 
-    private ControllerNode localNode;
-
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
     private ClusterService clusterService;
 
-    private ClusterNodesDelegate nodesDelegate;
-    private final Timer timer = new Timer("onos-controller-heatbeats");
-    public static final long HEART_BEAT_INTERVAL_MILLIS = 1000L;
-
     // TODO: This probably should not be a OSGi service.
     private MessagingService messagingService;
 
@@ -72,7 +61,7 @@ public class ClusterCommunicationManager
 
     @Activate
     public void activate() {
-        localNode = clusterService.getLocalNode();
+        ControllerNode localNode = clusterService.getLocalNode();
         NettyMessagingService netty = new NettyMessagingService(localNode.tcpPort());
         // FIXME: workaround until it becomes a service.
         try {
@@ -92,8 +81,9 @@ public class ClusterCommunicationManager
     }
 
     @Override
-    public boolean broadcast(ClusterMessage message) {
+    public boolean broadcast(ClusterMessage message) throws IOException {
         boolean ok = true;
+        final ControllerNode localNode = clusterService.getLocalNode();
         for (ControllerNode node : clusterService.getNodes()) {
             if (!node.equals(localNode)) {
                 ok = unicast(message, node.id()) && ok;
@@ -103,8 +93,9 @@ public class ClusterCommunicationManager
     }
 
     @Override
-    public boolean multicast(ClusterMessage message, Set<NodeId> nodes) {
+    public boolean multicast(ClusterMessage message, Set<NodeId> nodes) throws IOException {
         boolean ok = true;
+        final ControllerNode localNode = clusterService.getLocalNode();
         for (NodeId nodeId : nodes) {
             if (!nodeId.equals(localNode.id())) {
                 ok = unicast(message, nodeId) && ok;
@@ -114,7 +105,7 @@ public class ClusterCommunicationManager
     }
 
     @Override
-    public boolean unicast(ClusterMessage message, NodeId toNodeId) {
+    public boolean unicast(ClusterMessage message, NodeId toNodeId) throws IOException {
         ControllerNode node = clusterService.getNode(toNodeId);
         checkArgument(node != null, "Unknown nodeId: %s", toNodeId);
         Endpoint nodeEp = new Endpoint(node.ip().toString(), node.tcpPort());
@@ -124,9 +115,8 @@ public class ClusterCommunicationManager
             return true;
         } catch (IOException e) {
             log.error("Failed to send cluster message to nodeId: " + toNodeId, e);
+            throw e;
         }
-
-        return false;
     }
 
     @Override
@@ -135,61 +125,6 @@ public class ClusterCommunicationManager
         messagingService.registerHandler(subject.value(), new InternalClusterMessageHandler(subscriber));
     }
 
-    @Override
-    public void initialize(ControllerNode localNode,
-            ClusterNodesDelegate delegate) {
-        this.localNode = localNode;
-        this.nodesDelegate = delegate;
-        this.addSubscriber(new MessageSubject("CLUSTER_MEMBERSHIP_EVENT"), new ClusterMemebershipEventHandler());
-        timer.schedule(new KeepAlive(), 0, HEART_BEAT_INTERVAL_MILLIS);
-    }
-
-    @Override
-    public void addNode(ControllerNode node) {
-        //members.put(node.id(), node);
-    }
-
-    @Override
-    public void removeNode(ControllerNode node) {
-        broadcast(new ClusterMessage(
-                localNode.id(),
-                new MessageSubject("CLUSTER_MEMBERSHIP_EVENT"),
-                SERIALIZER.encode(new ClusterMembershipEvent(ClusterMembershipEventType.LEAVING_MEMBER, node))));
-        //members.remove(node.id());
-    }
-
-    // Sends a heart beat to all peers.
-    private class KeepAlive extends TimerTask {
-
-        @Override
-        public void run() {
-            broadcast(new ClusterMessage(
-                localNode.id(),
-                new MessageSubject("CLUSTER_MEMBERSHIP_EVENT"),
-                SERIALIZER.encode(new ClusterMembershipEvent(ClusterMembershipEventType.HEART_BEAT, localNode))));
-        }
-    }
-
-    private class ClusterMemebershipEventHandler implements ClusterMessageHandler {
-
-        @Override
-        public void handle(ClusterMessage message) {
-
-            ClusterMembershipEvent event = SERIALIZER.decode(message.payload());
-            ControllerNode node = event.node();
-            if (event.type() == ClusterMembershipEventType.HEART_BEAT) {
-                log.info("Node {} sent a hearbeat", node.id());
-                nodesDelegate.nodeDetected(node.id(), node.ip(), node.tcpPort());
-            } else if (event.type() == ClusterMembershipEventType.LEAVING_MEMBER) {
-                log.info("Node {} is leaving", node.id());
-                nodesDelegate.nodeRemoved(node.id());
-            } else if (event.type() == ClusterMembershipEventType.UNREACHABLE_MEMBER) {
-                log.info("Node {} is unreachable", node.id());
-                nodesDelegate.nodeVanished(node.id());
-            }
-        }
-    }
-
     private final class InternalClusterMessageHandler implements MessageHandler {
 
         private final ClusterMessageHandler handler;

-package org.onlab.onos.store.cluster.messaging.impl;
-
-import org.apache.felix.scr.annotations.Activate;
-import org.apache.felix.scr.annotations.Component;
-import org.apache.felix.scr.annotations.Deactivate;
-import org.apache.felix.scr.annotations.Service;
-import org.onlab.onos.store.cluster.messaging.MessageSubject;
-import org.onlab.onos.store.cluster.messaging.SerializationService;
-import org.onlab.onos.store.serializers.KryoPoolUtil;
-import org.onlab.util.KryoPool;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-//FIXME: not used any more? remove
-/**
- * Factory for parsing messages sent between cluster members.
- */
-@Component(immediate = true)
-@Service
-public class MessageSerializer implements SerializationService {
-
-    private final Logger log = LoggerFactory.getLogger(getClass());
-
-    private static final int METADATA_LENGTH = 12; // 8 + 4
-    private static final int LENGTH_OFFSET = 8;
-
-    private static final long MARKER = 0xfeedcafebeaddeadL;
-
-    private KryoPool serializerPool;
-
-    @Activate
-    public void activate() {
-        setupKryoPool();
-        log.info("Started");
-    }
-
-    @Deactivate
-    public void deactivate() {
-        log.info("Stopped");
-    }
-
-    /**
-     * Sets up the common serialzers pool.
-     */
-    protected void setupKryoPool() {
-        serializerPool = KryoPool.newBuilder()
-                .register(KryoPoolUtil.API)
-                // TODO: Should MessageSubject be in API bundle?
-                .register(MessageSubject.class)
-                .build()
-                .populate(1);
-    }
-
-
-    @Override
-    public <T> T decode(byte[] data) {
-        return serializerPool.deserialize(data);
-    }
-
-    @Override
-    public byte[] encode(Object payload) {
-        return serializerPool.serialize(payload);
-    }
-}

@@ -86,14 +86,11 @@ public class GossipDeviceStore
 
     private final Logger log = getLogger(getClass());
 
-    public static final String DEVICE_NOT_FOUND = "Device with ID %s not found";
+    private static final String DEVICE_NOT_FOUND = "Device with ID %s not found";
 
-    // TODO: Check if inner Map can be replaced with plain Map.
     // innerMap is used to lock a Device, thus instance should never be replaced.
-
     // collection of Description given from various providers
-    private final ConcurrentMap<DeviceId,
-                            ConcurrentMap<ProviderId, DeviceDescriptions>>
+    private final ConcurrentMap<DeviceId, Map<ProviderId, DeviceDescriptions>>
                                 deviceDescs = Maps.newConcurrentMap();
 
     // cache of Device and Ports generated by compositing descriptions from providers
@@ -208,9 +205,9 @@ public class GossipDeviceStore
         final Timestamped<DeviceDescription> deltaDesc = new Timestamped<>(deviceDescription, newTimestamp);
         final DeviceEvent event;
         final Timestamped<DeviceDescription> mergedDesc;
-        synchronized (getDeviceDescriptions(deviceId)) {
+        synchronized (getOrCreateDeviceDescriptionsMap(deviceId)) {
             event = createOrUpdateDeviceInternal(providerId, deviceId, deltaDesc);
-            mergedDesc = getDeviceDescriptions(deviceId).get(providerId).getDeviceDesc();
+            mergedDesc = getOrCreateDeviceDescriptionsMap(deviceId).get(providerId).getDeviceDesc();
         }
         if (event != null) {
             log.info("Notifying peers of a device update topology event for providerId: {} and deviceId: {}",
@@ -230,8 +227,8 @@ public class GossipDeviceStore
                                     Timestamped<DeviceDescription> deltaDesc) {
 
         // Collection of DeviceDescriptions for a Device
-        ConcurrentMap<ProviderId, DeviceDescriptions> providerDescs
-            = getDeviceDescriptions(deviceId);
+        Map<ProviderId, DeviceDescriptions> providerDescs
+            = getOrCreateDeviceDescriptionsMap(deviceId);
 
         synchronized (providerDescs) {
             // locking per device
@@ -241,9 +238,7 @@ public class GossipDeviceStore
                 return null;
             }
 
-            DeviceDescriptions descs
-                = createIfAbsentUnchecked(providerDescs, providerId,
-                    new InitDeviceDescs(deltaDesc));
+            DeviceDescriptions descs = getOrCreateProviderDeviceDescriptions(providerDescs, providerId, deltaDesc);
 
             final Device oldDevice = devices.get(deviceId);
             final Device newDevice;
@@ -338,7 +333,7 @@ public class GossipDeviceStore
     private DeviceEvent markOfflineInternal(DeviceId deviceId, Timestamp timestamp) {
 
         Map<ProviderId, DeviceDescriptions> providerDescs
-            = getDeviceDescriptions(deviceId);
+            = getOrCreateDeviceDescriptionsMap(deviceId);
 
         // locking device
         synchronized (providerDescs) {
@@ -401,9 +396,9 @@ public class GossipDeviceStore
         final List<DeviceEvent> events;
         final Timestamped<List<PortDescription>> merged;
 
-        synchronized (getDeviceDescriptions(deviceId)) {
+        synchronized (getOrCreateDeviceDescriptionsMap(deviceId)) {
             events = updatePortsInternal(providerId, deviceId, timestampedInput);
-            final DeviceDescriptions descs = getDeviceDescriptions(deviceId).get(providerId);
+            final DeviceDescriptions descs = getOrCreateDeviceDescriptionsMap(deviceId).get(providerId);
             List<PortDescription> mergedList =
                     FluentIterable.from(portDescriptions)
                 .transform(new Function<PortDescription, PortDescription>() {
@@ -435,7 +430,7 @@ public class GossipDeviceStore
         Device device = devices.get(deviceId);
         checkArgument(device != null, DEVICE_NOT_FOUND, deviceId);
 
-        ConcurrentMap<ProviderId, DeviceDescriptions> descsMap = deviceDescs.get(deviceId);
+        Map<ProviderId, DeviceDescriptions> descsMap = deviceDescs.get(deviceId);
         checkArgument(descsMap != null, DEVICE_NOT_FOUND, deviceId);
 
         List<DeviceEvent> events = new ArrayList<>();
@@ -539,10 +534,34 @@ public class GossipDeviceStore
                 NewConcurrentHashMap.<PortNumber, Port>ifNeeded());
     }
 
-    private ConcurrentMap<ProviderId, DeviceDescriptions> getDeviceDescriptions(
+    private Map<ProviderId, DeviceDescriptions> getOrCreateDeviceDescriptionsMap(
             DeviceId deviceId) {
-        return createIfAbsentUnchecked(deviceDescs, deviceId,
-                NewConcurrentHashMap.<ProviderId, DeviceDescriptions>ifNeeded());
+        Map<ProviderId, DeviceDescriptions> r;
+        r = deviceDescs.get(deviceId);
+        if (r == null) {
+            r = new HashMap<ProviderId, DeviceDescriptions>();
+            final Map<ProviderId, DeviceDescriptions> concurrentlyAdded;
+            concurrentlyAdded = deviceDescs.putIfAbsent(deviceId, r);
+            if (concurrentlyAdded != null) {
+                r = concurrentlyAdded;
+            }
+        }
+        return r;
+    }
+
+    // Guarded by deviceDescs value (=Device lock)
+    private DeviceDescriptions getOrCreateProviderDeviceDescriptions(
+            Map<ProviderId, DeviceDescriptions> device,
+            ProviderId providerId, Timestamped<DeviceDescription> deltaDesc) {
+
+        synchronized (device) {
+            DeviceDescriptions r = device.get(providerId);
+            if (r == null) {
+                r = new DeviceDescriptions(deltaDesc);
+                device.put(providerId, r);
+            }
+            return r;
+        }
     }
 
     @Override
@@ -555,9 +574,9 @@ public class GossipDeviceStore
             = new Timestamped<>(portDescription, newTimestamp);
         final DeviceEvent event;
         final Timestamped<PortDescription> mergedDesc;
-        synchronized (getDeviceDescriptions(deviceId)) {
+        synchronized (getOrCreateDeviceDescriptionsMap(deviceId)) {
             event = updatePortStatusInternal(providerId, deviceId, deltaDesc);
-            mergedDesc = getDeviceDescriptions(deviceId).get(providerId)
+            mergedDesc = getOrCreateDeviceDescriptionsMap(deviceId).get(providerId)
                             .getPortDesc(portDescription.portNumber());
         }
         if (event != null) {
@@ -579,7 +598,7 @@ public class GossipDeviceStore
         Device device = devices.get(deviceId);
         checkArgument(device != null, DEVICE_NOT_FOUND, deviceId);
 
-        ConcurrentMap<ProviderId, DeviceDescriptions> descsMap = deviceDescs.get(deviceId);
+        Map<ProviderId, DeviceDescriptions> descsMap = deviceDescs.get(deviceId);
         checkArgument(descsMap != null, DEVICE_NOT_FOUND, deviceId);
 
         synchronized (descsMap) {
@@ -591,7 +610,7 @@ public class GossipDeviceStore
 
             DeviceDescriptions descs = descsMap.get(providerId);
             // assuming all providers must to give DeviceDescription
-            checkArgument(descs != null,
+            verify(descs != null,
                     "Device description for Device ID %s from Provider %s was not found",
                     deviceId, providerId);
 
@@ -661,7 +680,7 @@ public class GossipDeviceStore
     private DeviceEvent removeDeviceInternal(DeviceId deviceId,
                                              Timestamp timestamp) {
 
-        Map<ProviderId, DeviceDescriptions> descs = getDeviceDescriptions(deviceId);
+        Map<ProviderId, DeviceDescriptions> descs = getOrCreateDeviceDescriptionsMap(deviceId);
         synchronized (descs) {
             // accept removal request if given timestamp is newer than
             // the latest Timestamp from Primary provider
@@ -751,14 +770,14 @@ public class GossipDeviceStore
      *
      * @param device device the port is on
      * @param number port number
-     * @param providerDescs Collection of Descriptions from multiple providers
+     * @param descsMap Collection of Descriptions from multiple providers
      * @return Port instance
      */
     private Port composePort(Device device, PortNumber number,
-                ConcurrentMap<ProviderId, DeviceDescriptions> providerDescs) {
+                Map<ProviderId, DeviceDescriptions> descsMap) {
 
-        ProviderId primary = pickPrimaryPID(providerDescs);
-        DeviceDescriptions primDescs = providerDescs.get(primary);
+        ProviderId primary = pickPrimaryPID(descsMap);
+        DeviceDescriptions primDescs = descsMap.get(primary);
         // if no primary, assume not enabled
         // TODO: revisit this default port enabled/disabled behavior
         boolean isEnabled = false;
@@ -770,7 +789,7 @@ public class GossipDeviceStore
             annotations = merge(annotations, portDesc.value().annotations());
         }
 
-        for (Entry<ProviderId, DeviceDescriptions> e : providerDescs.entrySet()) {
+        for (Entry<ProviderId, DeviceDescriptions> e : descsMap.entrySet()) {
             if (e.getKey().equals(primary)) {
                 continue;
             }
@@ -893,41 +912,48 @@ public class GossipDeviceStore
     private DeviceAntiEntropyAdvertisement createAdvertisement() {
         final NodeId self = clusterService.getLocalNode().id();
 
-        Map<DeviceFragmentId, Timestamp> devices = new HashMap<>(deviceDescs.size());
-        final int portsPerDevice = 8; // random guess to minimize reallocation
-        Map<PortFragmentId, Timestamp> ports = new HashMap<>(devices.size() * portsPerDevice);
-        Map<DeviceId, Timestamp> offline = new HashMap<>(devices.size());
+        final int numDevices = deviceDescs.size();
+        Map<DeviceFragmentId, Timestamp> adDevices = new HashMap<>(numDevices);
+        final int portsPerDevice = 8; // random factor to minimize reallocation
+        Map<PortFragmentId, Timestamp> adPorts = new HashMap<>(numDevices * portsPerDevice);
+        Map<DeviceId, Timestamp> adOffline = new HashMap<>(numDevices);
 
-        for (Entry<DeviceId, ConcurrentMap<ProviderId, DeviceDescriptions>>
+        for (Entry<DeviceId, Map<ProviderId, DeviceDescriptions>>
             provs : deviceDescs.entrySet()) {
 
+            // for each Device...
             final DeviceId deviceId = provs.getKey();
-            final ConcurrentMap<ProviderId, DeviceDescriptions> devDescs = provs.getValue();
+            final Map<ProviderId, DeviceDescriptions> devDescs = provs.getValue();
             synchronized (devDescs) {
 
-                offline.put(deviceId, this.offline.get(deviceId));
+                // send device offline timestamp
+                Timestamp lOffline = this.offline.get(deviceId);
+                if (lOffline != null) {
+                    adOffline.put(deviceId, lOffline);
+                }
 
                 for (Entry<ProviderId, DeviceDescriptions>
                         prov : devDescs.entrySet()) {
 
+                    // for each Provider Descriptions...
                     final ProviderId provId = prov.getKey();
                     final DeviceDescriptions descs = prov.getValue();
 
-                    devices.put(new DeviceFragmentId(deviceId, provId),
+                    adDevices.put(new DeviceFragmentId(deviceId, provId),
                             descs.getDeviceDesc().timestamp());
 
                     for (Entry<PortNumber, Timestamped<PortDescription>>
                             portDesc : descs.getPortDescs().entrySet()) {
 
                         final PortNumber number = portDesc.getKey();
-                        ports.put(new PortFragmentId(deviceId, provId, number),
+                        adPorts.put(new PortFragmentId(deviceId, provId, number),
                                 portDesc.getValue().timestamp());
                     }
                 }
             }
         }
 
-        return new DeviceAntiEntropyAdvertisement(self, devices, ports, offline);
+        return new DeviceAntiEntropyAdvertisement(self, adDevices, adPorts, adOffline);
     }
 
     /**
@@ -950,7 +976,7 @@ public class GossipDeviceStore
         Collection<DeviceFragmentId> reqDevices = new ArrayList<>();
         Collection<PortFragmentId> reqPorts = new ArrayList<>();
 
-        for (Entry<DeviceId, ConcurrentMap<ProviderId, DeviceDescriptions>> de : deviceDescs.entrySet()) {
+        for (Entry<DeviceId, Map<ProviderId, DeviceDescriptions>> de : deviceDescs.entrySet()) {
             final DeviceId deviceId = de.getKey();
             final Map<ProviderId, DeviceDescriptions> lDevice = de.getValue();
 
@@ -1199,7 +1225,7 @@ public class GossipDeviceStore
 
         @Override
         public void handle(ClusterMessage message) {
-            log.info("Received Device advertisement from peer: {}", message.sender());
+            log.debug("Received Device Anti-Entropy advertisement from peer: {}", message.sender());
             DeviceAntiEntropyAdvertisement advertisement = SERIALIZER.decode(message.payload());
             handleAdvertisement(advertisement);
         }

@@ -4,10 +4,14 @@ import com.google.common.collect.HashMultimap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Multimap;
 import com.google.common.collect.Sets;
+
 import org.apache.felix.scr.annotations.Activate;
 import org.apache.felix.scr.annotations.Component;
 import org.apache.felix.scr.annotations.Deactivate;
+import org.apache.felix.scr.annotations.Reference;
+import org.apache.felix.scr.annotations.ReferenceCardinality;
 import org.apache.felix.scr.annotations.Service;
+import org.onlab.onos.cluster.ClusterService;
 import org.onlab.onos.net.Annotations;
 import org.onlab.onos.net.ConnectPoint;
 import org.onlab.onos.net.DefaultHost;
@@ -15,6 +19,7 @@ import org.onlab.onos.net.DeviceId;
 import org.onlab.onos.net.Host;
 import org.onlab.onos.net.HostId;
 import org.onlab.onos.net.HostLocation;
+import org.onlab.onos.net.host.HostClockService;
 import org.onlab.onos.net.host.HostDescription;
 import org.onlab.onos.net.host.HostEvent;
 import org.onlab.onos.net.host.HostStore;
@@ -22,11 +27,21 @@ import org.onlab.onos.net.host.HostStoreDelegate;
 import org.onlab.onos.net.host.PortAddresses;
 import org.onlab.onos.net.provider.ProviderId;
 import org.onlab.onos.store.AbstractStore;
+import org.onlab.onos.store.Timestamp;
+import org.onlab.onos.store.cluster.messaging.ClusterCommunicationService;
+import org.onlab.onos.store.cluster.messaging.ClusterMessage;
+import org.onlab.onos.store.cluster.messaging.ClusterMessageHandler;
+import org.onlab.onos.store.cluster.messaging.MessageSubject;
+import org.onlab.onos.store.common.impl.Timestamped;
+import org.onlab.onos.store.serializers.DistributedStoreSerializers;
+import org.onlab.onos.store.serializers.KryoSerializer;
 import org.onlab.packet.IpPrefix;
 import org.onlab.packet.MacAddress;
 import org.onlab.packet.VlanId;
+import org.onlab.util.KryoPool;
 import org.slf4j.Logger;
 
+import java.io.IOException;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
@@ -35,14 +50,14 @@ import java.util.concurrent.ConcurrentHashMap;
 import static org.onlab.onos.net.host.HostEvent.Type.*;
 import static org.slf4j.LoggerFactory.getLogger;
 
+//TODO: multi-provider, annotation not supported.
 /**
- * Manages inventory of end-station hosts using trivial in-memory
- * implementation.
+ * Manages inventory of end-station hosts in distributed data store
+ * that uses optimistic replication and gossip based techniques.
  */
-//FIXME: I LIE I AM NOT DISTRIBUTED
 @Component(immediate = true)
 @Service
-public class DistributedHostStore
+public class GossipHostStore
         extends AbstractStore<HostEvent, HostStoreDelegate>
         implements HostStore {
 
@@ -51,14 +66,41 @@ public class DistributedHostStore
     // Host inventory
     private final Map<HostId, StoredHost> hosts = new ConcurrentHashMap<>(2000000, 0.75f, 16);
 
+    private final Map<HostId, Timestamped<Host>> removedHosts = new ConcurrentHashMap<>(2000000, 0.75f, 16);
+
     // Hosts tracked by their location
     private final Multimap<ConnectPoint, Host> locations = HashMultimap.create();
 
     private final Map<ConnectPoint, PortAddresses> portAddresses =
             new ConcurrentHashMap<>();
 
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected HostClockService hostClockService;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected ClusterCommunicationService clusterCommunicator;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected ClusterService clusterService;
+
+    private static final KryoSerializer SERIALIZER = new KryoSerializer() {
+        @Override
+        protected void setupKryoPool() {
+            serializerPool = KryoPool.newBuilder()
+                    .register(DistributedStoreSerializers.COMMON)
+                    .register(InternalHostRemovedEvent.class)
+                    .build()
+                    .populate(1);
+        }
+    };
+
     @Activate
     public void activate() {
+        clusterCommunicator.addSubscriber(
+                GossipHostStoreMessageSubjects.HOST_UPDATED, new InternalHostEventListener());
+        clusterCommunicator.addSubscriber(
+                GossipHostStoreMessageSubjects.HOST_REMOVED, new InternalHostRemovedEventListener());
+
         log.info("Started");
     }
 
@@ -70,34 +112,60 @@ public class DistributedHostStore
     @Override
     public HostEvent createOrUpdateHost(ProviderId providerId, HostId hostId,
                                         HostDescription hostDescription) {
+        Timestamp timestamp = hostClockService.getTimestamp(hostId);
+        HostEvent event = createOrUpdateHostInternal(providerId, hostId, hostDescription, timestamp);
+        if (event != null) {
+            log.info("Notifying peers of a host topology event for providerId: "
+                    + "{}; hostId: {}; hostDescription: {}", providerId, hostId, hostDescription);
+            try {
+                notifyPeers(new InternalHostEvent(providerId, hostId, hostDescription, timestamp));
+            } catch (IOException e) {
+                log.error("Failed to notify peers of a host topology event for providerId: "
+                        + "{}; hostId: {}; hostDescription: {}", providerId, hostId, hostDescription);
+            }
+        }
+        return event;
+    }
+
+    private HostEvent createOrUpdateHostInternal(ProviderId providerId, HostId hostId,
+                                        HostDescription hostDescription, Timestamp timestamp) {
         StoredHost host = hosts.get(hostId);
         if (host == null) {
-            return createHost(providerId, hostId, hostDescription);
+            return createHost(providerId, hostId, hostDescription, timestamp);
         }
-        return updateHost(providerId, host, hostDescription);
+        return updateHost(providerId, host, hostDescription, timestamp);
     }
 
     // creates a new host and sends HOST_ADDED
     private HostEvent createHost(ProviderId providerId, HostId hostId,
-                                 HostDescription descr) {
+                                 HostDescription descr, Timestamp timestamp) {
+        synchronized (this) {
+            // If this host was previously removed, first ensure
+            // this new request is "newer"
+            if (removedHosts.containsKey(hostId)) {
+                if (removedHosts.get(hostId).isNewer(timestamp)) {
+                    return null;
+                } else {
+                    removedHosts.remove(hostId);
+                }
+            }
             StoredHost newhost = new StoredHost(providerId, hostId,
                     descr.hwAddress(),
                     descr.vlan(),
-                                            descr.location(),
+                    new Timestamped<>(descr.location(), timestamp),
                     ImmutableSet.of(descr.ipAddress()));
-        synchronized (this) {
             hosts.put(hostId, newhost);
             locations.put(descr.location(), newhost);
-        }
             return new HostEvent(HOST_ADDED, newhost);
         }
+    }
 
     // checks for type of update to host, sends appropriate event
     private HostEvent updateHost(ProviderId providerId, StoredHost host,
-                                 HostDescription descr) {
+                                 HostDescription descr, Timestamp timestamp) {
         HostEvent event;
-        if (!host.location().equals(descr.location())) {
-            host.setLocation(descr.location());
+        if (!host.location.isNewer(timestamp) && !host.location().equals(descr.location())) {
+            host.setLocation(new Timestamped<>(descr.location(), timestamp));
             return new HostEvent(HOST_MOVED, host);
         }
 
@@ -109,7 +177,7 @@ public class DistributedHostStore
         addresses.add(descr.ipAddress());
         StoredHost updated = new StoredHost(providerId, host.id(),
                                             host.mac(), host.vlan(),
-                                            descr.location(), addresses);
+                                            host.location, addresses);
         event = new HostEvent(HOST_UPDATED, updated);
         synchronized (this) {
             hosts.put(host.id(), updated);
@@ -121,10 +189,25 @@ public class DistributedHostStore
 
     @Override
     public HostEvent removeHost(HostId hostId) {
+        Timestamp timestamp = hostClockService.getTimestamp(hostId);
+        HostEvent event = removeHostInternal(hostId, timestamp);
+        if (event != null) {
+            log.info("Notifying peers of a host removed topology event for hostId: {}", hostId);
+            try {
+                notifyPeers(new InternalHostRemovedEvent(hostId, timestamp));
+            } catch (IOException e) {
+                log.info("Failed to notify peers of a host removed topology event for hostId: {}", hostId);
+            }
+        }
+        return event;
+    }
+
+    private HostEvent removeHostInternal(HostId hostId, Timestamp timestamp) {
         synchronized (this) {
             Host host = hosts.remove(hostId);
             if (host != null) {
                 locations.remove((host.location()), host);
+                removedHosts.put(hostId, new Timestamped<>(host, timestamp));
                 return new HostEvent(HOST_REMOVED, host);
             }
             return null;
@@ -270,7 +353,7 @@ public class DistributedHostStore
 
     // Auxiliary extension to allow location to mutate.
     private class StoredHost extends DefaultHost {
-        private HostLocation location;
+        private Timestamped<HostLocation> location;
 
         /**
          * Creates an end-station host using the supplied information.
@@ -284,19 +367,71 @@ public class DistributedHostStore
          * @param annotations optional key/value annotations
          */
         public StoredHost(ProviderId providerId, HostId id,
-                          MacAddress mac, VlanId vlan, HostLocation location,
+                          MacAddress mac, VlanId vlan, Timestamped<HostLocation> location,
                           Set<IpPrefix> ips, Annotations... annotations) {
-            super(providerId, id, mac, vlan, location, ips, annotations);
+            super(providerId, id, mac, vlan, location.value(), ips, annotations);
             this.location = location;
         }
 
-        void setLocation(HostLocation location) {
+        void setLocation(Timestamped<HostLocation> location) {
             this.location = location;
         }
 
         @Override
         public HostLocation location() {
-            return location;
+            return location.value();
+        }
+    }
+
+    private void notifyPeers(InternalHostRemovedEvent event) throws IOException {
+        broadcastMessage(GossipHostStoreMessageSubjects.HOST_REMOVED, event);
+    }
+
+    private void notifyPeers(InternalHostEvent event) throws IOException {
+        broadcastMessage(GossipHostStoreMessageSubjects.HOST_UPDATED, event);
+    }
+
+    private void broadcastMessage(MessageSubject subject, Object event) throws IOException {
+        ClusterMessage message = new ClusterMessage(
+                clusterService.getLocalNode().id(),
+                subject,
+                SERIALIZER.encode(event));
+        clusterCommunicator.broadcast(message);
+    }
+
+    private void notifyDelegateIfNotNull(HostEvent event) {
+        if (event != null) {
+            notifyDelegate(event);
+        }
+    }
+
+    private class InternalHostEventListener implements ClusterMessageHandler {
+        @Override
+        public void handle(ClusterMessage message) {
+
+            log.info("Received host update event from peer: {}", message.sender());
+            InternalHostEvent event = (InternalHostEvent) SERIALIZER.decode(message.payload());
+
+            ProviderId providerId = event.providerId();
+            HostId hostId = event.hostId();
+            HostDescription hostDescription = event.hostDescription();
+            Timestamp timestamp = event.timestamp();
+
+            notifyDelegateIfNotNull(createOrUpdateHostInternal(providerId, hostId, hostDescription, timestamp));
+        }
+    }
+
+    private class InternalHostRemovedEventListener implements ClusterMessageHandler {
+        @Override
+        public void handle(ClusterMessage message) {
+
+            log.info("Received host removed event from peer: {}", message.sender());
+            InternalHostRemovedEvent event = (InternalHostRemovedEvent) SERIALIZER.decode(message.payload());
+
+            HostId hostId = event.hostId();
+            Timestamp timestamp = event.timestamp();
+
+            notifyDelegateIfNotNull(removeHostInternal(hostId, timestamp));
         }
     }
 }

+package org.onlab.onos.store.host.impl;
+
+import org.onlab.onos.store.cluster.messaging.MessageSubject;
+
+public final class GossipHostStoreMessageSubjects {
+    private GossipHostStoreMessageSubjects() {}
+    public static final MessageSubject HOST_UPDATED = new MessageSubject("peer-host-updated");
+    public static final MessageSubject HOST_REMOVED = new MessageSubject("peer-host-removed");
+}

+package org.onlab.onos.store.host.impl;
+
+import org.onlab.onos.net.HostId;
+import org.onlab.onos.net.host.HostDescription;
+import org.onlab.onos.net.provider.ProviderId;
+import org.onlab.onos.store.Timestamp;
+
+/**
+ * Information published by GossipHostStore to notify peers of a host
+ * change (create/update) event.
+ */
+public class InternalHostEvent {
+
+    private final ProviderId providerId;
+    private final HostId hostId;
+    private final HostDescription hostDescription;
+    private final Timestamp timestamp;
+
+    public InternalHostEvent(ProviderId providerId, HostId hostId,
+                             HostDescription hostDescription, Timestamp timestamp) {
+        this.providerId = providerId;
+        this.hostId = hostId;
+        this.hostDescription = hostDescription;
+        this.timestamp = timestamp;
+    }
+
+    public ProviderId providerId() {
+        return providerId;
+    }
+
+    public HostId hostId() {
+        return hostId;
+    }
+
+    public HostDescription hostDescription() {
+        return hostDescription;
+    }
+
+    public Timestamp timestamp() {
+        return timestamp;
+    }
+
+    // Needed for serialization.
+    @SuppressWarnings("unused")
+    private InternalHostEvent() {
+        providerId = null;
+        hostId = null;
+        hostDescription = null;
+        timestamp = null;
+    }
+}

+package org.onlab.onos.store.host.impl;
+
+import org.onlab.onos.net.HostId;
+import org.onlab.onos.store.Timestamp;
+
+/**
+ * Information published by GossipHostStore to notify peers of a host
+ * removed event.
+ */
+public class InternalHostRemovedEvent {
+
+    private final HostId hostId;
+    private final Timestamp timestamp;
+
+    public InternalHostRemovedEvent(HostId hostId, Timestamp timestamp) {
+        this.hostId = hostId;
+        this.timestamp = timestamp;
+    }
+
+    public HostId hostId() {
+        return hostId;
+    }
+
+    public Timestamp timestamp() {
+        return timestamp;
+    }
+
+    // for serialization.
+    @SuppressWarnings("unused")
+    private InternalHostRemovedEvent() {
+        hostId = null;
+        timestamp = null;
+    }
+}

@@ -9,7 +9,6 @@ import com.google.common.collect.Maps;
 import com.google.common.collect.SetMultimap;
 
 import org.apache.commons.lang3.RandomUtils;
-import org.apache.commons.lang3.concurrent.ConcurrentUtils;
 import org.apache.felix.scr.annotations.Activate;
 import org.apache.felix.scr.annotations.Component;
 import org.apache.felix.scr.annotations.Deactivate;
@@ -46,7 +45,6 @@ import org.onlab.onos.store.common.impl.Timestamped;
 import org.onlab.onos.store.serializers.DistributedStoreSerializers;
 import org.onlab.onos.store.serializers.KryoSerializer;
 import org.onlab.util.KryoPool;
-import org.onlab.util.NewConcurrentHashMap;
 import org.slf4j.Logger;
 
 import java.io.IOException;
@@ -87,7 +85,7 @@ public class GossipLinkStore
     private final Logger log = getLogger(getClass());
 
     // Link inventory
-    private final ConcurrentMap<LinkKey, ConcurrentMap<ProviderId, Timestamped<LinkDescription>>> linkDescs =
+    private final ConcurrentMap<LinkKey, Map<ProviderId, Timestamped<LinkDescription>>> linkDescs =
         new ConcurrentHashMap<>();
 
     // Link instance cache
@@ -238,7 +236,7 @@ public class GossipLinkStore
 
         final Timestamped<LinkDescription> deltaDesc = new Timestamped<>(linkDescription, newTimestamp);
 
-        LinkKey key = linkKey(linkDescription);
+        LinkKey key = linkKey(linkDescription.src(), linkDescription.dst());
         final LinkEvent event;
         final Timestamped<LinkDescription> mergedDesc;
         synchronized (getLinkDescriptions(key)) {
@@ -265,8 +263,9 @@ public class GossipLinkStore
             ProviderId providerId,
             Timestamped<LinkDescription> linkDescription) {
 
-        LinkKey key = linkKey(linkDescription.value());
-        ConcurrentMap<ProviderId, Timestamped<LinkDescription>> descs = getLinkDescriptions(key);
+        LinkKey key = linkKey(linkDescription.value().src(),
+                              linkDescription.value().dst());
+        Map<ProviderId, Timestamped<LinkDescription>> descs = getLinkDescriptions(key);
 
         synchronized (descs) {
             // if the link was previously removed, we should proceed if and
@@ -293,12 +292,12 @@ public class GossipLinkStore
 
     // Guarded by linkDescs value (=locking each Link)
     private Timestamped<LinkDescription> createOrUpdateLinkDescription(
-            ConcurrentMap<ProviderId, Timestamped<LinkDescription>> existingLinkDescriptions,
+            Map<ProviderId, Timestamped<LinkDescription>> descs,
             ProviderId providerId,
             Timestamped<LinkDescription> linkDescription) {
 
         // merge existing attributes and merge
-        Timestamped<LinkDescription> existingLinkDescription = existingLinkDescriptions.get(providerId);
+        Timestamped<LinkDescription> existingLinkDescription = descs.get(providerId);
         if (existingLinkDescription != null && existingLinkDescription.isNewer(linkDescription)) {
             return null;
         }
@@ -313,7 +312,7 @@ public class GossipLinkStore
                         linkDescription.value().type(), merged),
                     linkDescription.timestamp());
         }
-        return existingLinkDescriptions.put(providerId, newLinkDescription);
+        return descs.put(providerId, newLinkDescription);
     }
 
     // Creates and stores the link and returns the appropriate event.
@@ -379,7 +378,7 @@ public class GossipLinkStore
     }
 
     private LinkEvent removeLinkInternal(LinkKey key, Timestamp timestamp) {
-        ConcurrentMap<ProviderId, Timestamped<LinkDescription>> linkDescriptions =
+        Map<ProviderId, Timestamped<LinkDescription>> linkDescriptions =
                 getLinkDescriptions(key);
         synchronized (linkDescriptions) {
             // accept removal request if given timestamp is newer than
@@ -408,10 +407,10 @@ public class GossipLinkStore
      * @return primary ProviderID, or randomly chosen one if none exists
      */
     private ProviderId pickPrimaryProviderId(
-            ConcurrentMap<ProviderId, Timestamped<LinkDescription>> providerDescs) {
+            Map<ProviderId, Timestamped<LinkDescription>> linkDescriptions) {
 
         ProviderId fallBackPrimary = null;
-        for (Entry<ProviderId, Timestamped<LinkDescription>> e : providerDescs.entrySet()) {
+        for (Entry<ProviderId, Timestamped<LinkDescription>> e : linkDescriptions.entrySet()) {
             if (!e.getKey().isAncillary()) {
                 return e.getKey();
             } else if (fallBackPrimary == null) {
@@ -422,9 +421,9 @@ public class GossipLinkStore
         return fallBackPrimary;
     }
 
-    private Link composeLink(ConcurrentMap<ProviderId, Timestamped<LinkDescription>> linkDescriptions) {
-        ProviderId primaryProviderId = pickPrimaryProviderId(linkDescriptions);
-        Timestamped<LinkDescription> base = linkDescriptions.get(primaryProviderId);
+    private Link composeLink(Map<ProviderId, Timestamped<LinkDescription>> descs) {
+        ProviderId primaryProviderId = pickPrimaryProviderId(descs);
+        Timestamped<LinkDescription> base = descs.get(primaryProviderId);
 
         ConnectPoint src = base.value().src();
         ConnectPoint dst = base.value().dst();
@@ -432,7 +431,7 @@ public class GossipLinkStore
         DefaultAnnotations annotations = DefaultAnnotations.builder().build();
         annotations = merge(annotations, base.value().annotations());
 
-        for (Entry<ProviderId, Timestamped<LinkDescription>> e : linkDescriptions.entrySet()) {
+        for (Entry<ProviderId, Timestamped<LinkDescription>> e : descs.entrySet()) {
             if (primaryProviderId.equals(e.getKey())) {
                 continue;
             }
@@ -449,9 +448,20 @@ public class GossipLinkStore
         return new DefaultLink(primaryProviderId , src, dst, type, annotations);
     }
 
-    private ConcurrentMap<ProviderId, Timestamped<LinkDescription>> getLinkDescriptions(LinkKey key) {
-        return ConcurrentUtils.createIfAbsentUnchecked(linkDescs, key,
-                NewConcurrentHashMap.<ProviderId, Timestamped<LinkDescription>>ifNeeded());
+    private Map<ProviderId, Timestamped<LinkDescription>> getLinkDescriptions(LinkKey key) {
+        Map<ProviderId, Timestamped<LinkDescription>> r;
+        r = linkDescs.get(key);
+        if (r != null) {
+            return r;
+        }
+        r = new HashMap<>();
+        final Map<ProviderId, Timestamped<LinkDescription>> concurrentlyAdded;
+        concurrentlyAdded = linkDescs.putIfAbsent(key, r);
+        if (concurrentlyAdded != null) {
+            return concurrentlyAdded;
+        } else {
+            return r;
+        }
     }
 
     private Timestamped<LinkDescription> getLinkDescription(LinkKey key, ProviderId providerId) {
@@ -470,13 +480,13 @@ public class GossipLinkStore
         }
     }
 
+    private static final class IsPrimary implements Predicate<Provided> {
+
         private static final Predicate<Provided> IS_PRIMARY = new IsPrimary();
-    private static final Predicate<Provided> isPrimary() {
+        public static final Predicate<Provided> isPrimary() {
             return IS_PRIMARY;
         }
 
-    private static final class IsPrimary implements Predicate<Provided> {
-
         @Override
         public boolean apply(Provided input) {
             return !input.providerId().isAncillary();
@@ -581,11 +591,11 @@ public class GossipLinkStore
         Map<LinkFragmentId, Timestamp> linkTimestamps = new HashMap<>(linkDescs.size());
         Map<LinkKey, Timestamp> linkTombstones = new HashMap<>(removedLinks.size());
 
-        for (Entry<LinkKey, ConcurrentMap<ProviderId, Timestamped<LinkDescription>>>
+        for (Entry<LinkKey, Map<ProviderId, Timestamped<LinkDescription>>>
             provs : linkDescs.entrySet()) {
 
             final LinkKey linkKey = provs.getKey();
-            final ConcurrentMap<ProviderId, Timestamped<LinkDescription>> linkDesc = provs.getValue();
+            final Map<ProviderId, Timestamped<LinkDescription>> linkDesc = provs.getValue();
             synchronized (linkDesc) {
                 for (Map.Entry<ProviderId, Timestamped<LinkDescription>> e : linkDesc.entrySet()) {
                     linkTimestamps.put(new LinkFragmentId(linkKey, e.getKey()), e.getValue().timestamp());
@@ -670,7 +680,7 @@ public class GossipLinkStore
 
         @Override
         public void handle(ClusterMessage message) {
-            log.info("Received Link Anti-Entropy advertisement from peer: {}", message.sender());
+            log.debug("Received Link Anti-Entropy advertisement from peer: {}", message.sender());
             LinkAntiEntropyAdvertisement advertisement = SERIALIZER.decode(message.payload());
             handleAntiEntropyAdvertisement(advertisement);
         }

@@ -40,22 +40,18 @@ public class ClusterCommunicationManagerTest {
 
     @Before
     public void setUp() throws Exception {
-        MessageSerializer messageSerializer = new MessageSerializer();
-        messageSerializer.activate();
 
         NettyMessagingService messagingService = new NettyMessagingService();
         messagingService.activate();
 
         ccm1 = new ClusterCommunicationManager();
-//        ccm1.serializationService = messageSerializer;
         ccm1.activate();
 
         ccm2 = new ClusterCommunicationManager();
-//        ccm2.serializationService = messageSerializer;
         ccm2.activate();
 
-        ccm1.initialize(node1, cnd1);
-        ccm2.initialize(node2, cnd2);
+//        ccm1.initialize(node1, cnd1);
+//        ccm2.initialize(node2, cnd2);
     }
 
     @After
@@ -70,7 +66,7 @@ public class ClusterCommunicationManagerTest {
         cnd1.latch = new CountDownLatch(1);
         cnd2.latch = new CountDownLatch(1);
 
-        ccm1.addNode(node2);
+//        ccm1.addNode(node2);
         validateDelegateEvent(cnd1, Op.DETECTED, node2.id());
         validateDelegateEvent(cnd2, Op.DETECTED, node1.id());
     }
@@ -81,7 +77,7 @@ public class ClusterCommunicationManagerTest {
         cnd1.latch = new CountDownLatch(1);
         cnd2.latch = new CountDownLatch(1);
 
-        ccm1.addNode(node2);
+//        ccm1.addNode(node2);
         validateDelegateEvent(cnd1, Op.DETECTED, node2.id());
         validateDelegateEvent(cnd2, Op.DETECTED, node1.id());
 

@@ -151,7 +151,7 @@ public class DistributedLinkStore
     @Override
     public LinkEvent createOrUpdateLink(ProviderId providerId,
                                         LinkDescription linkDescription) {
-        LinkKey key = linkKey(linkDescription);
+        LinkKey key = linkKey(linkDescription.src(), linkDescription.dst());
         Optional<DefaultLink> link = links.getUnchecked(key);
         if (!link.isPresent()) {
             return createLink(providerId, key, linkDescription);

@@ -35,6 +35,7 @@ import java.util.concurrent.ConcurrentHashMap;
 import static org.onlab.onos.net.host.HostEvent.Type.*;
 import static org.slf4j.LoggerFactory.getLogger;
 
+// TODO: multi-provider, annotation not supported.
 /**
  * Manages inventory of end-station hosts using trivial in-memory
  * implementation.

@@ -149,7 +149,7 @@ public class SimpleLinkStore
     @Override
     public LinkEvent createOrUpdateLink(ProviderId providerId,
                                         LinkDescription linkDescription) {
-        LinkKey key = linkKey(linkDescription);
+        LinkKey key = linkKey(linkDescription.src(), linkDescription.dst());
 
         ConcurrentMap<ProviderId, LinkDescription> descs = getLinkDescriptions(key);
         synchronized (descs) {

@@ -67,16 +67,6 @@
         <bundle>mvn:org.onlab.onos/onos-core-hz-cluster/1.0.0-SNAPSHOT</bundle>
     </feature>
 
-    <feature name="onos-core-hazelcast" version="1.0.0"
-             description="ONOS core components built on hazelcast">
-        <feature>onos-api</feature>
-        <bundle>mvn:org.onlab.onos/onos-core-net/1.0.0-SNAPSHOT</bundle>
-        <bundle>mvn:org.onlab.onos/onos-core-hz-common/1.0.0-SNAPSHOT</bundle>
-        <bundle>mvn:org.onlab.onos/onos-core-serializers/1.0.0-SNAPSHOT</bundle>
-        <bundle>mvn:org.onlab.onos/onos-core-hz-cluster/1.0.0-SNAPSHOT</bundle>
-        <bundle>mvn:org.onlab.onos/onos-core-hz-net/1.0.0-SNAPSHOT</bundle>
-    </feature>
-
     <feature name="onos-core-trivial" version="1.0.0"
              description="ONOS core components">
         <feature>onos-api</feature>
@@ -163,4 +153,10 @@
         <bundle>mvn:org.onlab.onos/onos-app-config/1.0.0-SNAPSHOT</bundle>
     </feature>
 
+    <feature name="onos-app-sdnip" version="1.0.0"
+             description="SDN-IP peering application">
+        <feature>onos-api</feature>
+        <bundle>mvn:org.onlab.onos/onos-app-sdnip/1.0.0-SNAPSHOT</bundle>
+    </feature>
+
 </features>

@@ -5,6 +5,7 @@ import org.onlab.onos.openflow.controller.driver.SwitchDriverSubHandshakeComplet
 import org.onlab.onos.openflow.controller.driver.SwitchDriverSubHandshakeNotStarted;
 import org.onlab.onos.openflow.controller.Dpid;
 import org.onlab.onos.openflow.controller.driver.AbstractOpenFlowSwitch;
+import org.projectfloodlight.openflow.protocol.OFBarrierRequest;
 import org.projectfloodlight.openflow.protocol.OFCircuitPortsReply;
 import org.projectfloodlight.openflow.protocol.OFCircuitPortsRequest;
 import org.projectfloodlight.openflow.protocol.OFDescStatsReply;
@@ -21,7 +22,6 @@ import org.projectfloodlight.openflow.protocol.oxm.OFOxmInPort;
 import org.projectfloodlight.openflow.protocol.oxm.OFOxmOchSigid;
 import org.projectfloodlight.openflow.protocol.oxm.OFOxmOchSigidBasic;
 import org.projectfloodlight.openflow.protocol.oxm.OFOxmOchSigtype;
-import org.projectfloodlight.openflow.protocol.oxm.OFOxmOchSigtypeBasic;
 import org.projectfloodlight.openflow.types.CircuitSignalID;
 import org.projectfloodlight.openflow.types.OFPort;
 import org.projectfloodlight.openflow.types.U8;
@@ -119,11 +119,12 @@ public class OFOpticalSwitchImplLINC13 extends AbstractOpenFlowSwitch {
                 processHandshakeOFExperimenterPortDescRequest(
                         (OFCircuitPortsReply) m);
                 driverHandshakeComplete.set(true);
-               /* try {
+                try {
                     testMA();
+                    testReverseMA();
                 } catch (IOException e) {
                     e.printStackTrace();
-                }*/
+                }
                 break;
             default:
                 log.debug("Received message {} during switch-driver " +
@@ -163,22 +164,71 @@ public class OFOpticalSwitchImplLINC13 extends AbstractOpenFlowSwitch {
                           "message " +
                           "{}",
                   circuitPortsRequest.toString());
-        channel.write(Collections.singletonList(circuitPortsRequest));
+        sendMsg(Collections.<OFMessage>singletonList(circuitPortsRequest));
     }
 
 
-
-    //todo for testing
-    public static final U8 SIGNAL_TYPE = U8.of((short) 1);
+    public static final U8 SIGNAL_TYPE = U8.of((short) 10);
     private void testMA() throws IOException {
         log.debug("LINC OE *** Testing MA ");
-        short lambda = 100;
-        if (getId() == 0x0000ffffffffff02L) {
+        short lambda = 1;
+        if (getId() == 0x0000ffffffffff01L) {
             final int inport = 10;
             final int outport = 20;
             //Circuit signal id
             CircuitSignalID sigID = getSignalID(lambda);
 
+            OFOxmOchSigidBasic ofOxmOchSigidBasic =
+                    factory().oxms().ochSigidBasic(sigID);
+
+
+            //Match Port
+            OFOxmInPort fieldPort = factory().oxms()
+                                                .inPort(OFPort.of(inport));
+            OFMatchV3 matchPort =
+                    factory()
+                            .buildMatchV3().
+                            setOxmList(OFOxmList.of(fieldPort)).build();
+
+
+            // Set Action outport ,sigType and sigID
+            List<OFAction> actionList = new ArrayList<>();
+            OFAction actionOutPort =
+                    factory().actions().output(OFPort.of(outport),
+                                                  0xffff);
+
+            OFActionCircuit actionCircuit = factory()
+                    .actions()
+                    .circuit(ofOxmOchSigidBasic);
+
+            actionList.add(actionCircuit);
+            actionList.add(actionOutPort);
+
+            OFInstruction instructionAction =
+                    factory().instructions().buildApplyActions()
+                                .setActions(actionList)
+                                .build();
+            List<OFInstruction> instructions =
+                    Collections.singletonList(instructionAction);
+
+            OFMessage opticalFlowEntry =
+                    factory().buildFlowAdd()
+                                .setMatch(matchPort)
+                                .setPriority(100)
+                                .setInstructions(instructions)
+                                .setXid(getNextTransactionId())
+                                .build();
+            log.debug("Adding optical flow in sw {}", getStringId());
+            List<OFMessage> msglist = new ArrayList<>(1);
+            msglist.add(opticalFlowEntry);
+            write(msglist);
+            sendBarrier(true);
+        } else if (getId() == 0x0000ffffffffff03L) {
+            final int inport = 30;
+            final int outport = 31;
+            //Circuit signal id
+            CircuitSignalID sigID = getSignalID(lambda);
+
             OFOxmOchSigid fieldSigIDMatch = factory().oxms().ochSigid(sigID);
             OFOxmOchSigtype fieldSigType = factory()
                     .oxms()
@@ -187,8 +237,119 @@ public class OFOpticalSwitchImplLINC13 extends AbstractOpenFlowSwitch {
             OFOxmOchSigidBasic ofOxmOchSigidBasic =
                     factory().oxms().ochSigidBasic(sigID);
 
-            OFOxmOchSigtypeBasic ofOxmOchSigtypeBasic =
-                    factory().oxms().ochSigtypeBasic(SIGNAL_TYPE);
+            //Match Port,SigType,SigID
+            OFOxmInPort fieldPort = factory()
+                    .oxms()
+                    .inPort(OFPort.of(inport));
+            OFMatchV3 matchPort = factory()
+                    .buildMatchV3()
+                    .setOxmList(OFOxmList.of(fieldPort,
+                                             fieldSigIDMatch,
+                                             fieldSigType
+                    ))
+                    .build();
+
+            // Set Action outport ,SigType, sigID
+            List<OFAction> actionList = new ArrayList<>();
+            OFAction actionOutPort =
+                    factory().actions().output(OFPort.of(outport),
+                                                  0xffff);
+
+            OFActionCircuit actionCircuit = factory()
+                    .actions()
+                    .circuit(ofOxmOchSigidBasic);
+
+
+
+            //actionList.add(setActionSigType);
+            actionList.add(actionCircuit);
+            actionList.add(actionOutPort);
+
+            OFInstruction instructionAction =
+                    factory().instructions().buildApplyActions()
+                                .setActions(actionList)
+                                .build();
+            List<OFInstruction> instructions =
+                    Collections.singletonList(instructionAction);
+
+            OFMessage opticalFlowEntry =
+                    factory().buildFlowAdd()
+                                .setMatch(matchPort)
+                                .setPriority(100)
+                                .setInstructions(instructions)
+                                .setXid(getNextTransactionId())
+                                .build();
+            log.debug("Adding optical flow in sw {}", getStringId());
+            List<OFMessage> msglist = new ArrayList<>(1);
+            msglist.add(opticalFlowEntry);
+            write(msglist);
+            sendBarrier(true);
+
+        } else if (getId() == 0x0000ffffffffff02L) {
+            final int inport = 21;
+            final int outport = 11;
+            //Circuit signal id
+            CircuitSignalID sigID = getSignalID(lambda);
+
+            OFOxmOchSigid fieldSigIDMatch = factory().oxms().ochSigid(sigID);
+            OFOxmOchSigtype fieldSigType = factory()
+                    .oxms()
+                    .ochSigtype(SIGNAL_TYPE);
+
+
+            //Match Port, sig type and sig id
+            OFOxmInPort fieldPort = factory()
+                    .oxms()
+                    .inPort(OFPort.of(inport));
+            OFMatchV3 matchPort =
+                    factory().buildMatchV3()
+                                .setOxmList(OFOxmList.of(fieldPort,
+                                                         fieldSigIDMatch,
+                                                         fieldSigType
+                                ))
+                                .build();
+
+            // Set Action outport
+            List<OFAction> actionList = new ArrayList<>();
+            OFAction actionOutPort =
+                    factory().actions().output(OFPort.of(outport),
+                                                  0xffff);
+
+            actionList.add(actionOutPort);
+
+            OFInstruction instructionAction =
+                    factory().instructions().buildApplyActions()
+                                .setActions(actionList)
+                                .build();
+            List<OFInstruction> instructions =
+                    Collections.singletonList(instructionAction);
+
+            OFMessage opticalFlowEntry =
+                    factory().buildFlowAdd()
+                                .setMatch(matchPort)
+                                .setPriority(100)
+                                .setInstructions(instructions)
+                                .setXid(getNextTransactionId())
+                                .build();
+            log.debug("Adding optical flow in sw {}", getStringId());
+            List<OFMessage> msglist = new ArrayList<>(1);
+            msglist.add(opticalFlowEntry);
+            write(msglist);
+            sendBarrier(true);
+        }
+
+    }
+    private void testReverseMA() throws IOException {
+        log.debug("LINC OE *** Testing MA ");
+        short lambda = 1;
+        if (getId() == 0x0000ffffffffff02L) {
+            final int inport = 11;
+            final int outport = 21;
+            //Circuit signal id
+            CircuitSignalID sigID = getSignalID(lambda);
+
+            OFOxmOchSigidBasic ofOxmOchSigidBasic =
+                    factory().oxms().ochSigidBasic(sigID);
 
             //Match Port
             OFOxmInPort fieldPort = factory().oxms()
@@ -196,27 +357,20 @@ public class OFOpticalSwitchImplLINC13 extends AbstractOpenFlowSwitch {
             OFMatchV3 matchPort =
                     factory()
                             .buildMatchV3().
-                            setOxmList(OFOxmList.of(fieldPort,
-                                                    fieldSigType,
-                                                    fieldSigIDMatch)).build();
+                            setOxmList(OFOxmList.of(fieldPort)).build();
 
 
             // Set Action outport ,sigType and sigID
             List<OFAction> actionList = new ArrayList<>();
             OFAction actionOutPort =
                     factory().actions().output(OFPort.of(outport),
-                                                  Short.MAX_VALUE);
+                                                  0xffff);
 
             OFActionCircuit actionCircuit = factory()
                     .actions()
                     .circuit(ofOxmOchSigidBasic);
-            OFActionCircuit setActionSigType = factory()
-                    .actions()
-                    .circuit(ofOxmOchSigtypeBasic);
-
-            actionList.add(actionOutPort);
-            actionList.add(setActionSigType);
             actionList.add(actionCircuit);
+            actionList.add(actionOutPort);
 
             OFInstruction instructionAction =
                     factory().instructions().buildApplyActions()
@@ -228,6 +382,7 @@ public class OFOpticalSwitchImplLINC13 extends AbstractOpenFlowSwitch {
             OFMessage opticalFlowEntry =
                     factory().buildFlowAdd()
                                 .setMatch(matchPort)
+                                .setPriority(100)
                                 .setInstructions(instructions)
                                 .setXid(getNextTransactionId())
                                 .build();
@@ -235,9 +390,10 @@ public class OFOpticalSwitchImplLINC13 extends AbstractOpenFlowSwitch {
             List<OFMessage> msglist = new ArrayList<>(1);
             msglist.add(opticalFlowEntry);
             write(msglist);
+            sendBarrier(true);
         } else if (getId() == 0x0000ffffffffff03L) {
-            final int inport = 21;
-            final int outport = 22;
+            final int inport = 31;
+            final int outport = 30;
             //Circuit signal id
             CircuitSignalID sigID = getSignalID(lambda);
 
@@ -249,9 +405,6 @@ public class OFOpticalSwitchImplLINC13 extends AbstractOpenFlowSwitch {
             OFOxmOchSigidBasic ofOxmOchSigidBasic =
                     factory().oxms().ochSigidBasic(sigID);
 
-            OFOxmOchSigtypeBasic ofOxmOchSigtypeBasic =
-                    factory().oxms().ochSigtypeBasic(SIGNAL_TYPE);
-
             //Match Port,SigType,SigID
             OFOxmInPort fieldPort = factory()
                     .oxms()
@@ -259,27 +412,22 @@ public class OFOpticalSwitchImplLINC13 extends AbstractOpenFlowSwitch {
             OFMatchV3 matchPort = factory()
                     .buildMatchV3()
                     .setOxmList(OFOxmList.of(fieldPort,
-                                             fieldSigType,
-                                             fieldSigIDMatch))
+                                             fieldSigIDMatch,
+                                             fieldSigType
+                    ))
                     .build();
 
             // Set Action outport ,SigType, sigID
             List<OFAction> actionList = new ArrayList<>();
             OFAction actionOutPort =
                     factory().actions().output(OFPort.of(outport),
-                                                  Short.MAX_VALUE);
-
-            OFActionCircuit setActionSigType = factory()
-                    .actions()
-                    .circuit(ofOxmOchSigtypeBasic);
+                                                  0xffff);
             OFActionCircuit actionCircuit = factory()
                     .actions()
                     .circuit(ofOxmOchSigidBasic);
 
-
-            actionList.add(actionOutPort);
-            actionList.add(setActionSigType);
             actionList.add(actionCircuit);
+            actionList.add(actionOutPort);
 
             OFInstruction instructionAction =
                     factory().instructions().buildApplyActions()
@@ -291,6 +439,7 @@ public class OFOpticalSwitchImplLINC13 extends AbstractOpenFlowSwitch {
             OFMessage opticalFlowEntry =
                     factory().buildFlowAdd()
                                 .setMatch(matchPort)
+                                .setPriority(100)
                                 .setInstructions(instructions)
                                 .setXid(getNextTransactionId())
                                 .build();
@@ -298,10 +447,11 @@ public class OFOpticalSwitchImplLINC13 extends AbstractOpenFlowSwitch {
             List<OFMessage> msglist = new ArrayList<>(1);
             msglist.add(opticalFlowEntry);
             write(msglist);
+            sendBarrier(true);
 
-        } else if (getId() == 0x0000ffffffffff04L) {
-            final int inport = 23;
-            final int outport = 11;
+        } else if (getId() == 0x0000ffffffffff01L) {
+            final int inport = 20;
+            final int outport = 10;
             //Circuit signal id
             CircuitSignalID sigID = getSignalID(lambda);
 
@@ -318,15 +468,16 @@ public class OFOpticalSwitchImplLINC13 extends AbstractOpenFlowSwitch {
             OFMatchV3 matchPort =
                     factory().buildMatchV3()
                                 .setOxmList(OFOxmList.of(fieldPort,
-                                                         fieldSigType,
-                                                         fieldSigIDMatch))
+                                                         fieldSigIDMatch,
+                                                         fieldSigType
+                                ))
                                 .build();
 
             // Set Action outport
             List<OFAction> actionList = new ArrayList<>();
             OFAction actionOutPort =
                     factory().actions().output(OFPort.of(outport),
-                                                  Short.MAX_VALUE);
+                                                  0xffff);
 
             actionList.add(actionOutPort);
 
@@ -340,6 +491,7 @@ public class OFOpticalSwitchImplLINC13 extends AbstractOpenFlowSwitch {
             OFMessage opticalFlowEntry =
                     factory().buildFlowAdd()
                                 .setMatch(matchPort)
+                                .setPriority(100)
                                 .setInstructions(instructions)
                                 .setXid(getNextTransactionId())
                                 .build();
@@ -347,10 +499,12 @@ public class OFOpticalSwitchImplLINC13 extends AbstractOpenFlowSwitch {
             List<OFMessage> msglist = new ArrayList<>(1);
             msglist.add(opticalFlowEntry);
             write(msglist);
+            sendBarrier(true);
         }
 
     }
 
+
     // Todo remove - for testing purpose only
     private static CircuitSignalID getSignalID(short lambda) {
         byte myGrid = 1;
@@ -365,9 +519,21 @@ public class OFOpticalSwitchImplLINC13 extends AbstractOpenFlowSwitch {
         return signalID;
     }
 
+    private void sendBarrier(boolean finalBarrier) throws IOException {
+        int xid = getNextTransactionId();
+        if (finalBarrier) {
+            barrierXidToWaitFor = xid;
+        }
+        OFBarrierRequest br = factory()
+                .buildBarrierRequest()
+                .setXid(xid)
+                .build();
+        sendMsg(br);
+    }
+
     @Override
     public void write(OFMessage msg) {
-        this.channel.write(msg);
+        this.sendMsg(msg);
     }
 
     @Override

@@ -304,7 +304,9 @@
                 <plugin>
                     <groupId>org.apache.maven.plugins</groupId>
                     <artifactId>maven-compiler-plugin</artifactId>
-                    <version>3.1</version>
+                    <!-- TODO: update once following issue is fixed. -->
+                    <!-- https://jira.codehaus.org/browse/MCOMPILER-205 -->
+                    <version>2.5.1</version>
                     <configuration>
                         <source>1.7</source>
                         <target>1.7</target>

@@ -161,10 +161,10 @@ public class FlowModBuilder {
         switch (l3m.subtype()) {
         case IP_DST:
             ip = (ModIPInstruction) i;
-            return factory.actions().setNwDst(IPv4Address.of(ip.ip().toInt()));
+            return factory.actions().setNwDst(IPv4Address.of(ip.ip().toRealInt()));
         case IP_SRC:
             ip = (ModIPInstruction) i;
-            return factory.actions().setNwSrc(IPv4Address.of(ip.ip().toInt()));
+            return factory.actions().setNwSrc(IPv4Address.of(ip.ip().toRealInt()));
         default:
             log.warn("Unimplemented action type {}.", l3m.subtype());
             break;
@@ -220,21 +220,21 @@ public class FlowModBuilder {
             case IPV4_DST:
                 ip = (IPCriterion) c;
                 if (ip.ip().isMasked()) {
-                    Masked<IPv4Address> maskedIp = Masked.of(IPv4Address.of(ip.ip().toInt()),
-                            IPv4Address.of(ip.ip().netmask().toInt()));
+                    Masked<IPv4Address> maskedIp = Masked.of(IPv4Address.of(ip.ip().toRealInt()),
+                            IPv4Address.of(ip.ip().netmask().toRealInt()));
                     mBuilder.setMasked(MatchField.IPV4_DST, maskedIp);
                 } else {
-                    mBuilder.setExact(MatchField.IPV4_DST, IPv4Address.of(ip.ip().toInt()));
+                    mBuilder.setExact(MatchField.IPV4_DST, IPv4Address.of(ip.ip().toRealInt()));
                 }
                 break;
             case IPV4_SRC:
                 ip = (IPCriterion) c;
                 if (ip.ip().isMasked()) {
-                    Masked<IPv4Address> maskedIp = Masked.of(IPv4Address.of(ip.ip().toInt()),
-                            IPv4Address.of(ip.ip().netmask().toInt()));
+                    Masked<IPv4Address> maskedIp = Masked.of(IPv4Address.of(ip.ip().toRealInt()),
+                            IPv4Address.of(ip.ip().netmask().toRealInt()));
                     mBuilder.setMasked(MatchField.IPV4_SRC, maskedIp);
                 } else {
-                    mBuilder.setExact(MatchField.IPV4_SRC, IPv4Address.of(ip.ip().toInt()));
+                    mBuilder.setExact(MatchField.IPV4_SRC, IPv4Address.of(ip.ip().toRealInt()));
                 }
                 break;
             case IP_PROTO:

@@ -9,6 +9,10 @@ export KARAF_ZIP=${KARAF_ZIP:-~/Downloads/apache-karaf-3.0.1.zip}
 export KARAF_TAR=${KARAF_TAR:-~/Downloads/apache-karaf-3.0.1.tar.gz}
 export KARAF_DIST=$(basename $KARAF_ZIP .zip)
 
+# Add ONOS-specific directories to the exectable PATH
+export PATH="$PATH:$ONOS_ROOT/tools/dev/bin:$ONOS_ROOT/tools/test/bin"
+export PATH="$PATH:$ONOS_ROOT/tools/build"
+
 # Fallback build number us derived from from the user name & time
 export BUILD_NUMBER=${BUILD_NUMBER:-$(id -un)~$(date +'%Y/%m/%d@%H:%M')}
 

 #!/bin/bash
-#-------------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Builds the ONOS from source.
-#-------------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 [ ! -d "$ONOS_ROOT" ] && echo "ONOS_ROOT is not defined" >&2 && exit 1
 . $ONOS_ROOT/tools/build/envDefaults

 #!/bin/bash
-#-------------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Packages ONOS distributable into onos.tar.gz
-#-------------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 [ ! -d "$ONOS_ROOT" ] && echo "ONOS_ROOT is not defined" >&2 && exit 1
 . $ONOS_ROOT/tools/build/envDefaults

 #!/bin/bash
-#-------------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Launches the ONOS tests on the current cell environment.
-#-------------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 [ ! -d "$ONOS_ROOT" ] && echo "ONOS_ROOT is not defined" >&2 && exit 1
 . $ONOS_ROOT/tools/build/envDefaults

@@ -61,15 +61,14 @@ function cell {
     if [ -n "$1" ]; then
         [ ! -f $ONOS_ROOT/tools/test/cells/$1 ] && \
             echo "No such cell: $1" >&2 && return 1
+        unset ONOS_CELL ONOS_NIC ONOS_FEATURES
         unset OC1 OC2 OC3 OC4 OC5 OC6 OC7 OC8 OC9 OCN OCI
         . $ONOS_ROOT/tools/test/cells/$1
-        export OCI=$OC1
-        export ONOS_CELL=$1
         cell
     else
         env | egrep "ONOS_CELL"
         env | egrep "OCI"
-        env | egrep "OC[0-9]+" | sort
+        env | egrep "OC[1-9]+" | sort
         env | egrep "OCN"
         env | egrep "ONOS_" | egrep -v 'ONOS_ROOT|ONOS_CELL'
     fi

 #!/bin/bash
-#------------------------------------------------------------------------------
+# ----------------------------------------------------------------------------
 # Selectively builds only those projects that contained modified Java files.
-#------------------------------------------------------------------------------
+# ----------------------------------------------------------------------------
 
 projects=$(find $ONOS_ROOT -name '*.java' \
     -not -path '*/openflowj/*' -and -not -path '.git/*' \

 #!/bin/bash
-#------------------------------------------------------------------------------
+# ----------------------------------------------------------------------------
 # Echoes project-level directory if a Java file within is newer than the
 # target directory.
-#------------------------------------------------------------------------------
+# ----------------------------------------------------------------------------
 
 javaFile=${1#*\/src\/*\/java/}
 basename=${1/*\//}
@@ -14,4 +14,3 @@ project=${src/src*/}
 target=$project/target
 
 [ $target -nt ${src}$javaFile ] || echo ${src/src*/}
-

 #!/bin/bash
-#------------------------------------------------------------------------------
+# ----------------------------------------------------------------------------
 # Continuously watches the Apache Karaf log; survives 'karaf clean'
-#------------------------------------------------------------------------------
+# ----------------------------------------------------------------------------
 KARAF_LOG=${KARAF_LOG:-~/apache-karaf-3.0.1/data/log/karaf.log}
 
 while true; do
     [ ! -f $KARAF_LOG ] && sleep 2 && continue
     tail -n 512 -f -F $KARAF_LOG
 done
-

 #!/bin/bash
-#-------------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # ONOS command-line client
-#-------------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 export JAVA_HOME=${JAVA_HOME:-/usr/lib/jvm/java-7-openjdk-amd64/}
 
 cd $(dirname $0)/../apache-karaf-*/bin
 ./client -h localhost "$@"
-

 #!/bin/bash
-#-------------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Starts ONOS Apache Karaf container
-#-------------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 export JAVA_HOME=${JAVA_HOME:-/usr/lib/jvm/java-7-openjdk-amd64/}
 export JAVA_OPTS="-Xms256M -Xmx2048M"
 
 cd /opt/onos
 /opt/onos/apache-karaf-3.0.1/bin/karaf "$@"
-

 #!/bin/bash
-#-------------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # ONOS remote command-line client.
-#-------------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
+
+[ ! -d "$ONOS_ROOT" ] && echo "ONOS_ROOT is not defined" >&2 && exit 1
+. $ONOS_ROOT/tools/build/envDefaults
 
 [ "$1" = "-w" ] && shift && onos-wait-for-start $1
 

 #!/bin/bash
-#-------------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Checks the logs of the remote ONOS instance and makes sure they are clean.
-#-------------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 [ ! -d "$ONOS_ROOT" ] && echo "ONOS_ROOT is not defined" >&2 && exit 1
 . $ONOS_ROOT/tools/build/envDefaults

 #!/bin/bash
-#-------------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Remotely configures & starts ONOS for the first time.
-#-------------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 [ ! -d "$ONOS_ROOT" ] && echo "ONOS_ROOT is not defined" >&2 && exit 1
 . $ONOS_ROOT/tools/build/envDefaults
@@ -24,5 +24,4 @@ ssh $remote "
     echo \"onos.ip = \$(ifconfig | grep $ONOS_NIC | cut -d: -f2 | cut -d\\  -f1)\" \
         >> $ONOS_INSTALL_DIR/$KARAF_DIST/etc/system.properties
 "
-
 scp -q $CDEF_FILE $remote:$ONOS_INSTALL_DIR/config/

 #!/bin/bash
-#-------------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Remotely fetches the ONOS test VMs from a local share into ~/Downloads.
-#-------------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 [ ! -d "$ONOS_ROOT" ] && echo "ONOS_ROOT is not defined" >&2 && exit 1
 . $ONOS_ROOT/tools/build/envDefaults

 #!/bin/bash
-#-------------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Launches ONOS GUI on the specified node.
-#-------------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
+
+[ ! -d "$ONOS_ROOT" ] && echo "ONOS_ROOT is not defined" >&2 && exit 1
+. $ONOS_ROOT/tools/build/envDefaults
 
 host=${1:-$OCI}
 host=${host:-localhost}

 #!/bin/bash
-#-------------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Remotely pushes bits to a remote node and installs ONOS on it.
-#-------------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 [ ! -d "$ONOS_ROOT" ] && echo "ONOS_ROOT is not defined" >&2 && exit 1
 . $ONOS_ROOT/tools/build/envDefaults
@@ -18,7 +18,7 @@ ssh $remote "
     [ -d $ONOS_INSTALL_DIR/bin ] && echo \"ONOS is already installed\" && exit 1
 
     # Prepare a landing zone and unroll the bits
-    sudo mkdir -p $ONOS_INSTALL_DIR && sudo chown sdn:sdn $ONOS_INSTALL_DIR
+    sudo mkdir -p $ONOS_INSTALL_DIR && sudo chown ${ONOS_USER}:${ONOS_USER} $ONOS_INSTALL_DIR
     tar zxmf /tmp/$ONOS_BITS.tar.gz -C $ONOS_INSTALL_DIR --strip-components=1
 
     # Make a link to the log file directory and make a home for auxiliaries

 #!/bin/bash
-#-------------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Remotely kills the ONOS service on the specified node.
-#-------------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 [ ! -d "$ONOS_ROOT" ] && echo "ONOS_ROOT is not defined" >&2 && exit 1
 . $ONOS_ROOT/tools/build/envDefaults

+#!/bin/bash
+# -----------------------------------------------------------------------------
+# List available ONOS cells configuration.
+# -----------------------------------------------------------------------------
+
+[ ! -d "$ONOS_ROOT" ] && echo "ONOS_ROOT is not defined" >&2 && exit 1
+. $ONOS_ROOT/tools/build/envDefaults
+
+# Lists available cells
+for cell in $(ls -1 $ONOS_ROOT/tools/test/cells); do
+    if [ ${cell} = "${ONOS_CELL}" ]; then
+        cell_id="${cell} *"
+    else
+        cell_id="${cell}"
+    fi
+    cell_descr="$(grep '^#' $ONOS_ROOT/tools/test/cells/$cell | head -n 1)"
+    printf "%-12s  %s\n" "${cell_id}" "${cell_descr}"
+done

 #!/bin/bash
-#-------------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Monitors remote ONOS log file on the specified node.
-#-------------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 [ ! -d "$ONOS_ROOT" ] && echo "ONOS_ROOT is not defined" >&2 && exit 1
 . $ONOS_ROOT/tools/build/envDefaults

 #!/bin/bash
-#-------------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Remotely patches the ONOS VM to tailor its hostname.
-#-------------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 [ ! -d "$ONOS_ROOT" ] && echo "ONOS_ROOT is not defined" >&2 && exit 1
 . $ONOS_ROOT/tools/build/envDefaults

 #!/bin/bash
-#-------------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Pushes the local id_rsa.pub to the authorized_keys on a remote ONOS node.
-#-------------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 [ ! -d "$ONOS_ROOT" ] && echo "ONOS_ROOT is not defined" >&2 && exit 1
 . $ONOS_ROOT/tools/build/envDefaults

 #!/bin/bash
-#-------------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Pushes the specified bundle to the remote ONOS cell machines and updates it.
-#-------------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 [ ! -d "$ONOS_ROOT" ] && echo "ONOS_ROOT is not defined" >&2 && exit 1
 . $ONOS_ROOT/tools/build/envDefaults

 #!/bin/bash
-#-------------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Remotely administers the ONOS service on the specified node.
-#-------------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 [ ! -d "$ONOS_ROOT" ] && echo "ONOS_ROOT is not defined" >&2 && exit 1
 . $ONOS_ROOT/tools/build/envDefaults

+#!/bin/bash
+# -----------------------------------------------------------------------------
+# Print the configuration of an ONOS cell.
+# -----------------------------------------------------------------------------
+
+[ ! -d "$ONOS_ROOT" ] && echo "ONOS_ROOT is not defined" >&2 && exit 1
+. $ONOS_ROOT/tools/build/envDefaults
+
+function print_usage {
+    echo "Print the configuration of an ONOS cell."
+    echo "If no arguments are specified, it will print the configuration for the default"
+    echo "ONOS cell as specified in the 'ONOS_CELL' environmental variable."
+    echo
+    echo "Optional arguments:"
+    echo "  [cell-name]       Print the configuration of 'cell-name'"
+    echo "  [-h | --help]     Print this help"
+}
+
+if [ "${1}" = "-h" -o "${1}" = "--help" ]; then
+    print_usage
+    exit 0
+fi
+
+if [ -n "${1}" ]; then
+    cell="${1}"
+else
+    if [ -z "${ONOS_CELL}" ]; then
+        echo "Environmental variable 'ONOS_CELL' is not defiled"
+        exit 1
+    else
+        cell="${ONOS_CELL}"
+    fi
+fi
+
+if [ ! -f $ONOS_ROOT/tools/test/cells/${cell} ]; then
+    echo "No such cell: ${cell}"
+    exit 1
+fi
+
+# Load the cell setup
+. $ONOS_ROOT/tools/test/cells/${cell}
+
+echo "ONOS_CELL=${ONOS_CELL}"
+echo "ONOS_NIC=${ONOS_NIC}"
+for n in {1..9}; do
+    ocn="OC${n}"
+    if [ -n "${!ocn}" ]; then
+        echo "$ocn=${!ocn}"
+    fi
+done
+echo "OCN=${OCN}"
+echo "OCI=${OCI}"
+echo "ONOS_FEATURES=${ONOS_FEATURES}"

 #!/bin/bash
-#-------------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Logs in to the remote ONOS node.
-#-------------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 [ ! -d "$ONOS_ROOT" ] && echo "ONOS_ROOT is not defined" >&2 && exit 1
 . $ONOS_ROOT/tools/build/envDefaults

 #!/bin/bash
-#-------------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Verifies connectivity to each node in ONOS cell.
-#-------------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 [ ! -d "$ONOS_ROOT" ] && echo "ONOS_ROOT is not defined" >&2 && exit 1
 . $ONOS_ROOT/tools/build/envDefaults

 #!/bin/bash
-#-------------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Remotely stops & uninstalls ONOS on the specified node.
-#-------------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 [ ! -d "$ONOS_ROOT" ] && echo "ONOS_ROOT is not defined" >&2 && exit 1
 . $ONOS_ROOT/tools/build/envDefaults

 #!/bin/bash
-#-------------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Update bundle on locally running karaf.
-#-------------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 [ ! -d "$ONOS_ROOT" ] && echo "ONOS_ROOT is not defined" >&2 && exit 1
 . $ONOS_ROOT/tools/build/envDefaults

 #!/bin/bash
-#-------------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Verifies connectivity to each node in ONOS cell.
-#-------------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 [ ! -d "$ONOS_ROOT" ] && echo "ONOS_ROOT is not defined" >&2 && exit 1
 . $ONOS_ROOT/tools/build/envDefaults

 #!/bin/bash
-#-------------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Waits for ONOS to reach run-level 100 on the specified remote node.
-#-------------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 [ ! -d "$ONOS_ROOT" ] && echo "ONOS_ROOT is not defined" >&2 && exit 1
 . $ONOS_ROOT/tools/build/envDefaults

 #!/bin/bash
-#-------------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 # Monitors selected set of ONOS commands using the system watch command.
-#-------------------------------------------------------------------------------
+# -----------------------------------------------------------------------------
 
 [ ! -d "$ONOS_ROOT" ] && echo "ONOS_ROOT is not defined" >&2 && exit 1
 . $ONOS_ROOT/tools/build/envDefaults

 # Local VirtualBox-based single ONOS instance & ONOS mininet box
 
+export ONOS_CELL="cbench"
+
 export ONOS_NIC=192.168.56.*
 export OC1="192.168.56.103"
 export OCN="192.168.56.103"
+export OCI="${OC1}"
 
 export ONOS_FEATURES="webconsole,onos-api,onos-core-trivial,onos-cli,onos-openflow,onos-app-fwd"

 # Local VirtualBox-based ONOS instances 1,2 & ONOS mininet box
 
+export ONOS_CELL="local"
+
 export ONOS_NIC=192.168.56.*
 export OC1="192.168.56.101"
 export OC2="192.168.56.102"
-
 export OCN="192.168.56.103"
+export OCI="${OC1}"
 
-
+export ONOS_FEATURES=""

 # ProxMox-based cell of ONOS instance; no mininet-box
 
-export ONOS_FEATURES="webconsole,onos-api,onos-core-trivial,onos-cli,onos-openflow,onos-app-fwd,onos-app-mobility,onos-app-tvue,onos-app-proxyarp"
+export ONOS_CELL="office"
 
 export ONOS_NIC="10.128.4.*"
 export OC1="10.128.4.60"
+export OCI="${OC1}"
 
+export ONOS_FEATURES="webconsole,onos-api,onos-core-trivial,onos-cli,onos-openflow,onos-app-fwd,onos-app-mobility,onos-app-tvue,onos-app-proxyarp"

 # ProxMox-based cell of ONOS instances 1,2 & ONOS mininet box
 
+export ONOS_CELL="prox"
+
 export ONOS_NIC="10.1.9.*"
 export OC1="10.1.9.94"
 export OC2="10.1.9.82"
-
 export OCN="10.1.9.93"
+export OCI="${OC1}"
+
+export ONOS_FEATURES=""

 # Local VirtualBox-based single ONOS instance & ONOS mininet box
 
+export ONOS_CELL="single"
+
 export ONOS_NIC=192.168.56.*
 export OC1="192.168.56.101"
 export OCN="192.168.56.103"
+export OCI="${OC1}"
 
+export ONOS_FEATURES=""

 # Local VirtualBox-based ONOS instances 1,2,3 & ONOS mininet box
 
+export ONOS_CELL="triple"
+
 export ONOS_NIC=192.168.56.*
 export OC1="192.168.56.101"
 export OC2="192.168.56.102"
 export OC3="192.168.56.104"
-
 export OCN="192.168.56.103"
+export OCI="${OC1}"
 
+export ONOS_FEATURES=""

@@ -181,6 +181,15 @@ public final class IpAddress {
         return address;
     }
 
+    public int toRealInt() {
+        int val = 0;
+        for (int i = 0; i < octets.length; i++) {
+          val <<= 8;
+          val |= octets[i] & 0xff;
+        }
+        return val;
+    }
+
     /**
      * Helper for computing the mask value from CIDR.
      *

@@ -3,7 +3,6 @@ package org.onlab.util;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 
-import org.apache.commons.lang3.concurrent.ConcurrentException;
 import org.apache.commons.lang3.concurrent.ConcurrentInitializer;
 
 /**
@@ -27,7 +26,7 @@ public final class NewConcurrentHashMap<K, V>
     }
 
     @Override
-    public ConcurrentMap<K, V> get() throws ConcurrentException {
+    public ConcurrentMap<K, V> get() {
         return new ConcurrentHashMap<>();
     }
 }

@@ -108,5 +108,10 @@ public class IpPrefixTest {
         IpAddress addr = IpAddress.valueOf("192.168.10.1");
 
         assertTrue(intf.contains(addr));
+
+        IpPrefix intf1 = IpPrefix.valueOf("10.0.0.101/24");
+        IpAddress addr1 = IpAddress.valueOf("10.0.0.4");
+
+        assertTrue(intf1.contains(addr1));
     }
 }

@@ -15,7 +15,7 @@ import static org.onlab.util.Tools.namedThreads;
  */
 public abstract class AbstractLoopTest {
 
-    protected static final long MAX_MS_WAIT = 500;
+    protected static final long MAX_MS_WAIT = 1500;
 
     /** Block on specified countdown latch. Return when countdown reaches
      * zero, or fail the test if the {@value #MAX_MS_WAIT} ms timeout expires.

@@ -2,6 +2,9 @@
 <html>
 <head>
     <title>ONOS GUI</title>
+
+    <script src="libs/d3.min.js"></script>
+    <script src="libs/jquery-2.1.1.min.js"></script>
 </head>
 <body>
     <h1>ONOS GUI</h1>