Injecting topology through JSON ConfigProvider works for multi-instance (ONOS-490).

Change-Id: Ib977f4cf9a59ddec360072891fd803c6f9ee84f1

Injecting optical device annotations and ports works for multi-instance (ONOS-870).

Change-Id: Icdde16ef72fc4e47eec7213250b04902083f0537

@@ -179,7 +179,7 @@ public class OpticalPathProvisioner {
 
             if (!IntentState.FAILED.equals(intentService.getIntentState(intent.key()))) {
                    return;
-             }
+            }
 
             List<Intent> intents = Lists.newArrayList();
             if (intent instanceof HostToHostIntent) {

@@ -15,10 +15,10 @@
  */
 package org.onosproject.event;
 
-import static com.google.common.base.MoreObjects.toStringHelper;
-
 import org.joda.time.LocalDateTime;
 
+import static com.google.common.base.MoreObjects.toStringHelper;
+
 /**
  * Base event implementation.
  */
@@ -75,5 +75,4 @@ public class AbstractEvent<T extends Enum, S> implements Event<T, S> {
                 .add("subject", subject())
                 .toString();
     }
-
 }

@@ -47,7 +47,7 @@ public class DefaultDeviceDescription extends AbstractDescription
      * @param hwVersion    device HW version
      * @param swVersion    device SW version
      * @param serialNumber device serial number
-     * @param chassis      chasis id
+     * @param chassis      chassis id
      * @param annotations  optional key/value annotations map
      */
     public DefaultDeviceDescription(URI uri, Type type, String manufacturer,

@@ -306,18 +306,16 @@ public class DeviceManager
                 // TODO: Do we need to explicitly tell the Provider that
                 // this instance is not the MASTER
                 applyRole(deviceId, MastershipRole.STANDBY);
-                return;
+            } else {
+                log.info("Role of this node is MASTER for {}", deviceId);
+                // tell clock provider if this instance is the master
+                deviceClockProviderService.setMastershipTerm(deviceId, term);
+                applyRole(deviceId, MastershipRole.MASTER);
             }
-            log.info("Role of this node is MASTER for {}", deviceId);
-
-            // tell clock provider if this instance is the master
-            deviceClockProviderService.setMastershipTerm(deviceId, term);
 
             DeviceEvent event = store.createOrUpdateDevice(provider().id(),
                                                            deviceId, deviceDescription);
 
-            applyRole(deviceId, MastershipRole.MASTER);
-
             // If there was a change of any kind, tell the provider
             // that this instance is the master.
             if (event != null) {

+package org.onosproject.store.device.impl;
+
+import com.google.common.base.MoreObjects;
+import org.onosproject.net.DeviceId;
+import org.onosproject.net.device.DeviceDescription;
+import org.onosproject.net.provider.ProviderId;
+
+public class DeviceInjectedEvent {
+    private final ProviderId providerId;
+    private final DeviceId deviceId;
+    private final DeviceDescription deviceDescription;
+
+    protected DeviceInjectedEvent(
+            ProviderId providerId,
+            DeviceId deviceId,
+            DeviceDescription deviceDescription) {
+        this.providerId = providerId;
+        this.deviceId = deviceId;
+        this.deviceDescription = deviceDescription;
+    }
+
+    public DeviceId deviceId() {
+        return deviceId;
+    }
+
+    public ProviderId providerId() {
+        return providerId;
+    }
+
+    public DeviceDescription deviceDescription() {
+        return deviceDescription;
+    }
+
+    @Override
+    public String toString() {
+        return MoreObjects.toStringHelper(getClass())
+                .add("providerId", providerId)
+                .add("deviceId", deviceId)
+                .add("deviceDescription", deviceDescription)
+                .toString();
+    }
+
+    // for serializer
+    protected DeviceInjectedEvent() {
+        this.providerId = null;
+        this.deviceId = null;
+        this.deviceDescription = null;
+    }
+}

@@ -34,4 +34,8 @@ public final class GossipDeviceStoreMessageSubjects {
     public static final MessageSubject DEVICE_ADVERTISE = new MessageSubject("peer-device-advertisements");
     // to be used with 3-way anti-entropy process
     public static final MessageSubject DEVICE_REQUEST = new MessageSubject("peer-device-request");
+
+    // Network elements injected (not discovered) by ConfigProvider
+    public static final MessageSubject DEVICE_INJECTED = new MessageSubject("peer-device-injected");
+    public static final MessageSubject PORT_INJECTED = new MessageSubject("peer-port-injected");
 }

+package org.onosproject.store.device.impl;
+
+import com.google.common.base.MoreObjects;
+import org.onosproject.net.DeviceId;
+import org.onosproject.net.device.PortDescription;
+import org.onosproject.net.provider.ProviderId;
+
+import java.util.List;
+
+public class PortInjectedEvent {
+
+    private ProviderId providerId;
+    private DeviceId deviceId;
+    private List<PortDescription> portDescriptions;
+
+    protected PortInjectedEvent(ProviderId providerId, DeviceId deviceId, List<PortDescription> portDescriptions) {
+        this.providerId = providerId;
+        this.deviceId = deviceId;
+        this.portDescriptions = portDescriptions;
+    }
+
+    public DeviceId deviceId() {
+        return deviceId;
+    }
+
+    public ProviderId providerId() {
+        return providerId;
+    }
+
+    public List<PortDescription> portDescriptions() {
+        return portDescriptions;
+    }
+
+    @Override
+    public String toString() {
+        return MoreObjects.toStringHelper(getClass())
+                .add("providerId", providerId)
+                .add("deviceId", deviceId)
+                .add("portDescriptions", portDescriptions)
+                .toString();
+    }
+
+    // for serializer
+    protected PortInjectedEvent() {
+        this.providerId = null;
+        this.deviceId = null;
+        this.portDescriptions = null;
+    }
+
+}

@@ -32,6 +32,7 @@ import org.onlab.util.KryoNamespace;
 import org.onosproject.cluster.ClusterService;
 import org.onosproject.cluster.ControllerNode;
 import org.onosproject.cluster.NodeId;
+import org.onosproject.mastership.MastershipService;
 import org.onosproject.net.AnnotationKeys;
 import org.onosproject.net.AnnotationsUtil;
 import org.onosproject.net.ConnectPoint;
@@ -90,9 +91,7 @@ import static org.onosproject.net.Link.State.INACTIVE;
 import static org.onosproject.net.Link.Type.DIRECT;
 import static org.onosproject.net.Link.Type.INDIRECT;
 import static org.onosproject.net.LinkKey.linkKey;
-import static org.onosproject.net.link.LinkEvent.Type.LINK_ADDED;
-import static org.onosproject.net.link.LinkEvent.Type.LINK_REMOVED;
-import static org.onosproject.net.link.LinkEvent.Type.LINK_UPDATED;
+import static org.onosproject.net.link.LinkEvent.Type.*;
 import static org.onosproject.store.link.impl.GossipLinkStoreMessageSubjects.LINK_ANTI_ENTROPY_ADVERTISEMENT;
 import static org.slf4j.LoggerFactory.getLogger;
 
@@ -106,6 +105,9 @@ public class GossipLinkStore
         extends AbstractStore<LinkEvent, LinkStoreDelegate>
         implements LinkStore {
 
+    // Timeout in milliseconds to process links on remote master node
+    private static final int REMOTE_MASTER_TIMEOUT = 1000;
+
     private final Logger log = getLogger(getClass());
 
     // Link inventory
@@ -131,6 +133,9 @@ public class GossipLinkStore
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
     protected ClusterService clusterService;
 
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected MastershipService mastershipService;
+
     protected static final KryoSerializer SERIALIZER = new KryoSerializer() {
         @Override
         protected void setupKryoPool() {
@@ -141,6 +146,7 @@ public class GossipLinkStore
                     .register(InternalLinkRemovedEvent.class)
                     .register(LinkAntiEntropyAdvertisement.class)
                     .register(LinkFragmentId.class)
+                    .register(LinkInjectedEvent.class)
                     .build();
         }
     };
@@ -161,6 +167,9 @@ public class GossipLinkStore
         clusterCommunicator.addSubscriber(
                 GossipLinkStoreMessageSubjects.LINK_ANTI_ENTROPY_ADVERTISEMENT,
                 new InternalLinkAntiEntropyAdvertisementListener());
+        clusterCommunicator.addSubscriber(
+                GossipLinkStoreMessageSubjects.LINK_INJECTED,
+                new LinkInjectedEventListener());
 
         executor = Executors.newCachedThreadPool(namedThreads("onos-link-fg-%d"));
 
@@ -270,27 +279,52 @@ public class GossipLinkStore
     public LinkEvent createOrUpdateLink(ProviderId providerId,
                                         LinkDescription linkDescription) {
 
-        DeviceId dstDeviceId = linkDescription.dst().deviceId();
-        Timestamp newTimestamp = deviceClockService.getTimestamp(dstDeviceId);
+        final DeviceId dstDeviceId = linkDescription.dst().deviceId();
+        final NodeId localNode = clusterService.getLocalNode().id();
+        final NodeId dstNode = mastershipService.getMasterFor(dstDeviceId);
 
-        final Timestamped<LinkDescription> deltaDesc = new Timestamped<>(linkDescription, newTimestamp);
+        // Process link update only if we're the master of the destination node,
+        // otherwise signal the actual master.
+        LinkEvent linkEvent = null;
+        if (localNode.equals(dstNode)) {
 
-        LinkKey key = linkKey(linkDescription.src(), linkDescription.dst());
-        final LinkEvent event;
-        final Timestamped<LinkDescription> mergedDesc;
-        Map<ProviderId, Timestamped<LinkDescription>> map = getOrCreateLinkDescriptions(key);
-        synchronized (map) {
-            event = createOrUpdateLinkInternal(providerId, deltaDesc);
-            mergedDesc = map.get(providerId);
-        }
+            Timestamp newTimestamp = deviceClockService.getTimestamp(dstDeviceId);
+
+            final Timestamped<LinkDescription> deltaDesc = new Timestamped<>(linkDescription, newTimestamp);
+
+            LinkKey key = linkKey(linkDescription.src(), linkDescription.dst());
+            final Timestamped<LinkDescription> mergedDesc;
+            Map<ProviderId, Timestamped<LinkDescription>> map = getOrCreateLinkDescriptions(key);
+
+            synchronized (map) {
+                linkEvent = createOrUpdateLinkInternal(providerId, deltaDesc);
+                mergedDesc = map.get(providerId);
+            }
+
+            if (linkEvent != null) {
+                log.info("Notifying peers of a link update topology event from providerId: "
+                                + "{}  between src: {} and dst: {}",
+                        providerId, linkDescription.src(), linkDescription.dst());
+                notifyPeers(new InternalLinkEvent(providerId, mergedDesc));
+            }
+
+        } else {
+
+            LinkInjectedEvent linkInjectedEvent = new LinkInjectedEvent(providerId, linkDescription);
+            ClusterMessage linkInjectedMessage = new ClusterMessage(localNode,
+                    GossipLinkStoreMessageSubjects.LINK_INJECTED, SERIALIZER.encode(linkInjectedEvent));
+
+            try {
+                clusterCommunicator.unicast(linkInjectedMessage, dstNode);
+            } catch (IOException e) {
+                log.warn("Failed to process link update between src: {} and dst: {} " +
+                                "(cluster messaging failed: {})",
+                        linkDescription.src(), linkDescription.dst(), e);
+            }
 
-        if (event != null) {
-            log.info("Notifying peers of a link update topology event from providerId: "
-                    + "{}  between src: {} and dst: {}",
-                    providerId, linkDescription.src(), linkDescription.dst());
-            notifyPeers(new InternalLinkEvent(providerId, mergedDesc));
         }
-        return event;
+
+        return linkEvent;
     }
 
     @Override
@@ -318,7 +352,7 @@ public class GossipLinkStore
             Timestamped<LinkDescription> linkDescription) {
 
         final LinkKey key = linkKey(linkDescription.value().src(),
-                                    linkDescription.value().dst());
+                linkDescription.value().dst());
         Map<ProviderId, Timestamped<LinkDescription>> descs = getOrCreateLinkDescriptions(key);
 
         synchronized (descs) {
@@ -397,7 +431,7 @@ public class GossipLinkStore
             !AnnotationsUtil.isEqual(oldLink.annotations(), newLink.annotations())) {
 
             links.put(key, newLink);
-            // strictly speaking following can be ommitted
+            // strictly speaking following can be omitted
             srcLinks.put(oldLink.src().deviceId(), key);
             dstLinks.put(oldLink.dst().deviceId(), key);
             return new LinkEvent(LINK_UPDATED, newLink);
@@ -848,4 +882,25 @@ public class GossipLinkStore
             });
         }
     }
+
+    private final class LinkInjectedEventListener
+            implements ClusterMessageHandler {
+        @Override
+        public void handle(ClusterMessage message) {
+
+            log.trace("Received injected link event from peer: {}", message.sender());
+            LinkInjectedEvent linkInjectedEvent = SERIALIZER.decode(message.payload());
+
+            ProviderId providerId = linkInjectedEvent.providerId();
+            LinkDescription linkDescription = linkInjectedEvent.linkDescription();
+
+            executor.submit(new Runnable() {
+
+                @Override
+                public void run() {
+                    createOrUpdateLink(providerId, linkDescription);
+                }
+            });
+        }
+    }
 }

@@ -15,7 +15,7 @@
  */
 package org.onosproject.store.link.impl;
 
-import org.onosproject.store.cluster.messaging.MessageSubject;
+        import org.onosproject.store.cluster.messaging.MessageSubject;
 
 /**
  * MessageSubjects used by GossipLinkStore peer-peer communication.
@@ -30,4 +30,6 @@ public final class GossipLinkStoreMessageSubjects {
             new MessageSubject("peer-link-removed");
     public static final MessageSubject LINK_ANTI_ENTROPY_ADVERTISEMENT =
             new MessageSubject("link-enti-entropy-advertisement");
+    public static final MessageSubject LINK_INJECTED =
+            new MessageSubject("peer-link-injected");
 }

+package org.onosproject.store.link.impl;
+
+import com.google.common.base.MoreObjects;
+import org.onosproject.net.link.LinkDescription;
+import org.onosproject.net.provider.ProviderId;
+
+public class LinkInjectedEvent {
+
+    ProviderId providerId;
+    LinkDescription linkDescription;
+
+    public LinkInjectedEvent(ProviderId providerId, LinkDescription linkDescription) {
+        this.providerId = providerId;
+        this.linkDescription = linkDescription;
+    }
+
+    public ProviderId providerId() {
+        return providerId;
+    }
+
+    public LinkDescription linkDescription() {
+        return linkDescription;
+    }
+
+    @Override
+    public String toString() {
+        return MoreObjects.toStringHelper(getClass())
+                .add("providerId", providerId)
+                .add("linkDescription", linkDescription)
+                .toString();
+    }
+
+    // for serializer
+    protected LinkInjectedEvent() {
+        this.providerId = null;
+        this.linkDescription = null;
+    }
+}

@@ -27,6 +27,8 @@ import org.onlab.packet.IpAddress;
 import org.onosproject.cluster.ControllerNode;
 import org.onosproject.cluster.DefaultControllerNode;
 import org.onosproject.cluster.NodeId;
+import org.onosproject.mastership.MastershipService;
+import org.onosproject.mastership.MastershipServiceAdapter;
 import org.onosproject.mastership.MastershipTerm;
 import org.onosproject.net.ConnectPoint;
 import org.onosproject.net.DefaultAnnotations;
@@ -115,7 +117,7 @@ public class GossipLinkStoreTest {
     private DeviceClockManager deviceClockManager;
     private DeviceClockService deviceClockService;
     private ClusterCommunicationService clusterCommunicator;
-
+    private MastershipService mastershipService;
 
     @BeforeClass
     public static void setUpBeforeClass() throws Exception {
@@ -146,11 +148,13 @@ public class GossipLinkStoreTest {
         linkStoreImpl.deviceClockService = deviceClockService;
         linkStoreImpl.clusterCommunicator = clusterCommunicator;
         linkStoreImpl.clusterService = new TestClusterService();
+        linkStoreImpl.mastershipService = new TestMastershipService();
         linkStoreImpl.activate();
         linkStore = linkStoreImpl;
 
         verify(clusterCommunicator);
         reset(clusterCommunicator);
+
     }
 
     @After
@@ -602,4 +606,11 @@ public class GossipLinkStoreTest {
             nodeStates.put(NID2, ACTIVE);
         }
     }
+
+    private final class TestMastershipService extends MastershipServiceAdapter {
+        @Override
+        public NodeId getMasterFor(DeviceId deviceId) {
+            return NID1;
+        }
+    }
 }