Add GossipLinkStoreTest

Change-Id: If3d9777583a38d911b19bb1bc50212ccb621918a

@@ -23,6 +23,7 @@ import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
 
+import static org.junit.Assert.assertEquals;
 import static org.onlab.onos.net.DeviceId.deviceId;
 import static org.onlab.onos.net.HostId.hostId;
 import static org.onlab.onos.net.PortNumber.portNumber;
@@ -85,4 +86,23 @@ public final class NetTestTools {
         return new DefaultPath(PID, links, ids.length);
     }
 
+
+    /**
+     * Verifies that Annotations created by merging {@code annotations} is
+     * equal to actual Annotations.
+     *
+     * @param actual Annotations to check
+     * @param annotations
+     */
+    public static void assertAnnotationsEquals(Annotations actual, SparseAnnotations... annotations) {
+        DefaultAnnotations expected = DefaultAnnotations.builder().build();
+        for (SparseAnnotations a : annotations) {
+            expected = DefaultAnnotations.merge(expected, a);
+        }
+        assertEquals(expected.keys(), actual.keys());
+        for (String key : expected.keys()) {
+            assertEquals(expected.value(key), actual.value(key));
+        }
+    }
+
 }

@@ -122,7 +122,7 @@ public class GossipLinkStore
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
     protected ClusterService clusterService;
 
-    private static final KryoSerializer SERIALIZER = new KryoSerializer() {
+    protected static final KryoSerializer SERIALIZER = new KryoSerializer() {
         @Override
         protected void setupKryoPool() {
             serializerPool = KryoNamespace.newBuilder()

+package org.onlab.onos.store.cluster;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+import org.onlab.onos.cluster.ClusterEventListener;
+import org.onlab.onos.cluster.ClusterService;
+import org.onlab.onos.cluster.ControllerNode;
+import org.onlab.onos.cluster.NodeId;
+import org.onlab.onos.cluster.ControllerNode.State;
+
+import com.google.common.collect.Sets;
+
+public abstract class StaticClusterService implements ClusterService {
+
+    protected final Map<NodeId, ControllerNode> nodes = new HashMap<>();
+    protected final Map<NodeId, ControllerNode.State> nodeStates = new HashMap<>();
+    protected ControllerNode localNode;
+
+    @Override
+    public ControllerNode getLocalNode() {
+        return localNode;
+    }
+
+    @Override
+    public Set<ControllerNode> getNodes() {
+        return Sets.newHashSet(nodes.values());
+    }
+
+    @Override
+    public ControllerNode getNode(NodeId nodeId) {
+        return nodes.get(nodeId);
+    }
+
+    @Override
+    public State getState(NodeId nodeId) {
+        return nodeStates.get(nodeId);
+    }
+
+    @Override
+    public void addListener(ClusterEventListener listener) {
+    }
+
+    @Override
+    public void removeListener(ClusterEventListener listener) {
+    }
+}
\ No newline at end of file

@@ -41,10 +41,8 @@ import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Ignore;
 import org.junit.Test;
-import org.onlab.onos.cluster.ClusterEventListener;
 import org.onlab.onos.cluster.ClusterService;
 import org.onlab.onos.cluster.ControllerNode;
-import org.onlab.onos.cluster.ControllerNode.State;
 import org.onlab.onos.cluster.DefaultControllerNode;
 import org.onlab.onos.cluster.NodeId;
 import org.onlab.onos.mastership.MastershipServiceAdapter;
@@ -65,6 +63,7 @@ import org.onlab.onos.net.device.DeviceStore;
 import org.onlab.onos.net.device.DeviceStoreDelegate;
 import org.onlab.onos.net.device.PortDescription;
 import org.onlab.onos.net.provider.ProviderId;
+import org.onlab.onos.store.cluster.StaticClusterService;
 import org.onlab.onos.store.cluster.messaging.ClusterCommunicationService;
 import org.onlab.onos.store.cluster.messaging.ClusterMessage;
 import org.onlab.onos.store.cluster.messaging.ClusterMessageHandler;
@@ -133,6 +132,7 @@ public class GossipDeviceStoreTest {
     private DeviceClockManager deviceClockManager;
     private DeviceClockService deviceClockService;
     private ClusterCommunicationService clusterCommunicator;
+
     @BeforeClass
     public static void setUpBeforeClass() throws Exception {
     }
@@ -838,45 +838,15 @@ public class GossipDeviceStoreTest {
         }
     }
 
-    private static final class TestClusterService implements ClusterService {
-
-        private final Map<NodeId, ControllerNode> nodes = new HashMap<>();
-        private final Map<NodeId, ControllerNode.State> nodeStates = new HashMap<>();
+    private static final class TestClusterService extends StaticClusterService {
 
         public TestClusterService() {
+            localNode = ONOS1;
             nodes.put(NID1, ONOS1);
             nodeStates.put(NID1, ACTIVE);
 
             nodes.put(NID2, ONOS2);
             nodeStates.put(NID2, ACTIVE);
         }
-
-        @Override
-        public ControllerNode getLocalNode() {
-            return GossipDeviceStoreTest.ONOS1;
-        }
-
-        @Override
-        public Set<ControllerNode> getNodes() {
-            return Sets.newHashSet(nodes.values());
-        }
-
-        @Override
-        public ControllerNode getNode(NodeId nodeId) {
-            return nodes.get(nodeId);
-        }
-
-        @Override
-        public State getState(NodeId nodeId) {
-            return nodeStates.get(nodeId);
-        }
-
-        @Override
-        public void addListener(ClusterEventListener listener) {
-        }
-
-        @Override
-        public void removeListener(ClusterEventListener listener) {
-        }
     }
 }

+/*
+ * Copyright 2014 Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onlab.onos.store.link.impl;
+
+import com.google.common.collect.Iterables;
+
+import org.easymock.Capture;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.onlab.onos.cluster.ControllerNode;
+import org.onlab.onos.cluster.DefaultControllerNode;
+import org.onlab.onos.cluster.NodeId;
+import org.onlab.onos.mastership.MastershipTerm;
+import org.onlab.onos.net.ConnectPoint;
+import org.onlab.onos.net.DefaultAnnotations;
+import org.onlab.onos.net.DeviceId;
+import org.onlab.onos.net.Link;
+import org.onlab.onos.net.Link.Type;
+import org.onlab.onos.net.LinkKey;
+import org.onlab.onos.net.PortNumber;
+import org.onlab.onos.net.SparseAnnotations;
+import org.onlab.onos.net.device.DeviceClockService;
+import org.onlab.onos.net.link.DefaultLinkDescription;
+import org.onlab.onos.net.link.LinkDescription;
+import org.onlab.onos.net.link.LinkEvent;
+import org.onlab.onos.net.link.LinkStore;
+import org.onlab.onos.net.link.LinkStoreDelegate;
+import org.onlab.onos.net.provider.ProviderId;
+import org.onlab.onos.store.cluster.StaticClusterService;
+import org.onlab.onos.store.cluster.messaging.ClusterCommunicationService;
+import org.onlab.onos.store.cluster.messaging.ClusterMessage;
+import org.onlab.onos.store.cluster.messaging.ClusterMessageHandler;
+import org.onlab.onos.store.cluster.messaging.MessageSubject;
+import org.onlab.onos.store.device.impl.DeviceClockManager;
+import org.onlab.packet.IpAddress;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.capture;
+import static org.easymock.EasyMock.createNiceMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.expectLastCall;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.reset;
+import static org.easymock.EasyMock.verify;
+import static org.junit.Assert.*;
+import static org.onlab.onos.cluster.ControllerNode.State.ACTIVE;
+import static org.onlab.onos.net.DeviceId.deviceId;
+import static org.onlab.onos.net.Link.Type.*;
+import static org.onlab.onos.net.link.LinkEvent.Type.*;
+import static org.onlab.onos.net.NetTestTools.assertAnnotationsEquals;
+
+/**
+ * Test of the GossipLinkStoreTest implementation.
+ */
+public class GossipLinkStoreTest {
+
+    private static final ProviderId PID = new ProviderId("of", "foo");
+    private static final ProviderId PIDA = new ProviderId("of", "bar", true);
+    private static final DeviceId DID1 = deviceId("of:foo");
+    private static final DeviceId DID2 = deviceId("of:bar");
+
+    private static final PortNumber P1 = PortNumber.portNumber(1);
+    private static final PortNumber P2 = PortNumber.portNumber(2);
+    private static final PortNumber P3 = PortNumber.portNumber(3);
+
+    private static final SparseAnnotations A1 = DefaultAnnotations.builder()
+            .set("A1", "a1")
+            .set("B1", "b1")
+            .build();
+    private static final SparseAnnotations A1_2 = DefaultAnnotations.builder()
+            .remove("A1")
+            .set("B3", "b3")
+            .build();
+    private static final SparseAnnotations A2 = DefaultAnnotations.builder()
+            .set("A2", "a2")
+            .set("B2", "b2")
+            .build();
+    private static final SparseAnnotations A2_2 = DefaultAnnotations.builder()
+            .remove("A2")
+            .set("B4", "b4")
+            .build();
+
+    // local node
+    private static final NodeId NID1 = new NodeId("local");
+    private static final ControllerNode ONOS1 =
+            new DefaultControllerNode(NID1, IpAddress.valueOf("127.0.0.1"));
+
+    // remote node
+    private static final NodeId NID2 = new NodeId("remote");
+    private static final ControllerNode ONOS2 =
+            new DefaultControllerNode(NID2, IpAddress.valueOf("127.0.0.2"));
+
+    private GossipLinkStore linkStoreImpl;
+    private LinkStore linkStore;
+
+    private DeviceClockManager deviceClockManager;
+    private DeviceClockService deviceClockService;
+    private ClusterCommunicationService clusterCommunicator;
+
+
+    @BeforeClass
+    public static void setUpBeforeClass() throws Exception {
+    }
+
+    @AfterClass
+    public static void tearDownAfterClass() throws Exception {
+    }
+
+    @Before
+    public void setUp() throws Exception {
+        deviceClockManager = new DeviceClockManager();
+        deviceClockManager.activate();
+        deviceClockService = deviceClockManager;
+
+        // set initial terms
+        deviceClockManager.setMastershipTerm(DID1, MastershipTerm.of(NID1, 1));
+        deviceClockManager.setMastershipTerm(DID2, MastershipTerm.of(NID1, 2));
+
+        // TODO mock clusterCommunicator
+        clusterCommunicator = createNiceMock(ClusterCommunicationService.class);
+        clusterCommunicator.addSubscriber(anyObject(MessageSubject.class),
+                                    anyObject(ClusterMessageHandler.class));
+        expectLastCall().anyTimes();
+        replay(clusterCommunicator);
+
+        linkStoreImpl = new GossipLinkStore();
+        linkStoreImpl.deviceClockService = deviceClockService;
+        linkStoreImpl.clusterCommunicator = clusterCommunicator;
+        linkStoreImpl.clusterService = new TestClusterService();
+        linkStoreImpl.activate();
+        linkStore = linkStoreImpl;
+
+        verify(clusterCommunicator);
+        reset(clusterCommunicator);
+    }
+
+    @After
+    public void tearDown() throws Exception {
+        linkStoreImpl.deactivate();
+    }
+
+    private void putLink(DeviceId srcId, PortNumber srcNum,
+                         DeviceId dstId, PortNumber dstNum, Type type,
+                         SparseAnnotations... annotations) {
+        ConnectPoint src = new ConnectPoint(srcId, srcNum);
+        ConnectPoint dst = new ConnectPoint(dstId, dstNum);
+        reset(clusterCommunicator);
+        try {
+            expect(clusterCommunicator.broadcast(anyObject(ClusterMessage.class)))
+                .andReturn(true).anyTimes();
+        } catch (IOException e) {
+            fail("Should never reach here");
+        }
+        replay(clusterCommunicator);
+        linkStore.createOrUpdateLink(PID, new DefaultLinkDescription(src, dst, type, annotations));
+        verify(clusterCommunicator);
+    }
+
+    private void resetCommunicatorExpectingNoBroadcast(
+                                                       Capture<ClusterMessage> bcast) {
+        bcast.reset();
+        reset(clusterCommunicator);
+        replay(clusterCommunicator);
+    }
+
+    private void resetCommunicatorExpectingSingleBroadcast(
+                                                           Capture<ClusterMessage> bcast) {
+
+        bcast.reset();
+        reset(clusterCommunicator);
+        try {
+            expect(clusterCommunicator.broadcast(capture(bcast))).andReturn(true).once();
+        } catch (IOException e) {
+            fail("Should never reach here");
+        }
+        replay(clusterCommunicator);
+    }
+
+    private void putLink(LinkKey key, Type type, SparseAnnotations... annotations) {
+        putLink(key.src().deviceId(), key.src().port(),
+                key.dst().deviceId(), key.dst().port(),
+                type, annotations);
+    }
+
+    private static void assertLink(DeviceId srcId, PortNumber srcNum,
+                            DeviceId dstId, PortNumber dstNum, Type type,
+                            Link link) {
+        assertEquals(srcId, link.src().deviceId());
+        assertEquals(srcNum, link.src().port());
+        assertEquals(dstId, link.dst().deviceId());
+        assertEquals(dstNum, link.dst().port());
+        assertEquals(type, link.type());
+    }
+
+    private static void assertLink(LinkKey key, Type type, Link link) {
+        assertLink(key.src().deviceId(), key.src().port(),
+                   key.dst().deviceId(), key.dst().port(),
+                   type, link);
+    }
+
+    @Test
+    public final void testGetLinkCount() {
+        assertEquals("initialy empty", 0, linkStore.getLinkCount());
+
+        putLink(DID1, P1, DID2, P2, DIRECT);
+        putLink(DID2, P2, DID1, P1, DIRECT);
+        putLink(DID1, P1, DID2, P2, DIRECT);
+
+        assertEquals("expecting 2 unique link", 2, linkStore.getLinkCount());
+    }
+
+    @Test
+    public final void testGetLinks() {
+        assertEquals("initialy empty", 0,
+                Iterables.size(linkStore.getLinks()));
+
+        LinkKey linkId1 = LinkKey.linkKey(new ConnectPoint(DID1, P1), new ConnectPoint(DID2, P2));
+        LinkKey linkId2 = LinkKey.linkKey(new ConnectPoint(DID2, P2), new ConnectPoint(DID1, P1));
+
+        putLink(linkId1, DIRECT);
+        putLink(linkId2, DIRECT);
+        putLink(linkId1, DIRECT);
+
+        assertEquals("expecting 2 unique link", 2,
+                Iterables.size(linkStore.getLinks()));
+
+        Map<LinkKey, Link> links = new HashMap<>();
+        for (Link link : linkStore.getLinks()) {
+            links.put(LinkKey.linkKey(link), link);
+        }
+
+        assertLink(linkId1, DIRECT, links.get(linkId1));
+        assertLink(linkId2, DIRECT, links.get(linkId2));
+    }
+
+    @Test
+    public final void testGetDeviceEgressLinks() {
+        LinkKey linkId1 = LinkKey.linkKey(new ConnectPoint(DID1, P1), new ConnectPoint(DID2, P2));
+        LinkKey linkId2 = LinkKey.linkKey(new ConnectPoint(DID2, P2), new ConnectPoint(DID1, P1));
+        LinkKey linkId3 = LinkKey.linkKey(new ConnectPoint(DID1, P2), new ConnectPoint(DID2, P3));
+
+        putLink(linkId1, DIRECT);
+        putLink(linkId2, DIRECT);
+        putLink(linkId3, DIRECT);
+
+        // DID1,P1 => DID2,P2
+        // DID2,P2 => DID1,P1
+        // DID1,P2 => DID2,P3
+
+        Set<Link> links1 = linkStore.getDeviceEgressLinks(DID1);
+        assertEquals(2, links1.size());
+        // check
+
+        Set<Link> links2 = linkStore.getDeviceEgressLinks(DID2);
+        assertEquals(1, links2.size());
+        assertLink(linkId2, DIRECT, links2.iterator().next());
+    }
+
+    @Test
+    public final void testGetDeviceIngressLinks() {
+        LinkKey linkId1 = LinkKey.linkKey(new ConnectPoint(DID1, P1), new ConnectPoint(DID2, P2));
+        LinkKey linkId2 = LinkKey.linkKey(new ConnectPoint(DID2, P2), new ConnectPoint(DID1, P1));
+        LinkKey linkId3 = LinkKey.linkKey(new ConnectPoint(DID1, P2), new ConnectPoint(DID2, P3));
+
+        putLink(linkId1, DIRECT);
+        putLink(linkId2, DIRECT);
+        putLink(linkId3, DIRECT);
+
+        // DID1,P1 => DID2,P2
+        // DID2,P2 => DID1,P1
+        // DID1,P2 => DID2,P3
+
+        Set<Link> links1 = linkStore.getDeviceIngressLinks(DID2);
+        assertEquals(2, links1.size());
+        // check
+
+        Set<Link> links2 = linkStore.getDeviceIngressLinks(DID1);
+        assertEquals(1, links2.size());
+        assertLink(linkId2, DIRECT, links2.iterator().next());
+    }
+
+    @Test
+    public final void testGetLink() {
+        ConnectPoint src = new ConnectPoint(DID1, P1);
+        ConnectPoint dst = new ConnectPoint(DID2, P2);
+        LinkKey linkId1 = LinkKey.linkKey(src, dst);
+
+        putLink(linkId1, DIRECT);
+
+        Link link = linkStore.getLink(src, dst);
+        assertLink(linkId1, DIRECT, link);
+
+        assertNull("There shouldn't be reverese link",
+                linkStore.getLink(dst, src));
+    }
+
+    @Test
+    public final void testGetEgressLinks() {
+        final ConnectPoint d1P1 = new ConnectPoint(DID1, P1);
+        final ConnectPoint d2P2 = new ConnectPoint(DID2, P2);
+        LinkKey linkId1 = LinkKey.linkKey(d1P1, d2P2);
+        LinkKey linkId2 = LinkKey.linkKey(d2P2, d1P1);
+        LinkKey linkId3 = LinkKey.linkKey(new ConnectPoint(DID1, P2), new ConnectPoint(DID2, P3));
+
+        putLink(linkId1, DIRECT);
+        putLink(linkId2, DIRECT);
+        putLink(linkId3, DIRECT);
+
+        // DID1,P1 => DID2,P2
+        // DID2,P2 => DID1,P1
+        // DID1,P2 => DID2,P3
+
+        Set<Link> links1 = linkStore.getEgressLinks(d1P1);
+        assertEquals(1, links1.size());
+        assertLink(linkId1, DIRECT, links1.iterator().next());
+
+        Set<Link> links2 = linkStore.getEgressLinks(d2P2);
+        assertEquals(1, links2.size());
+        assertLink(linkId2, DIRECT, links2.iterator().next());
+    }
+
+    @Test
+    public final void testGetIngressLinks() {
+        final ConnectPoint d1P1 = new ConnectPoint(DID1, P1);
+        final ConnectPoint d2P2 = new ConnectPoint(DID2, P2);
+        LinkKey linkId1 = LinkKey.linkKey(d1P1, d2P2);
+        LinkKey linkId2 = LinkKey.linkKey(d2P2, d1P1);
+        LinkKey linkId3 = LinkKey.linkKey(new ConnectPoint(DID1, P2), new ConnectPoint(DID2, P3));
+
+        putLink(linkId1, DIRECT);
+        putLink(linkId2, DIRECT);
+        putLink(linkId3, DIRECT);
+
+        // DID1,P1 => DID2,P2
+        // DID2,P2 => DID1,P1
+        // DID1,P2 => DID2,P3
+
+        Set<Link> links1 = linkStore.getIngressLinks(d2P2);
+        assertEquals(1, links1.size());
+        assertLink(linkId1, DIRECT, links1.iterator().next());
+
+        Set<Link> links2 = linkStore.getIngressLinks(d1P1);
+        assertEquals(1, links2.size());
+        assertLink(linkId2, DIRECT, links2.iterator().next());
+    }
+
+    @Test
+    public final void testCreateOrUpdateLink() {
+        ConnectPoint src = new ConnectPoint(DID1, P1);
+        ConnectPoint dst = new ConnectPoint(DID2, P2);
+
+        Capture<ClusterMessage> bcast = new Capture<>();
+
+        // add link
+        resetCommunicatorExpectingSingleBroadcast(bcast);
+        final DefaultLinkDescription linkDescription = new DefaultLinkDescription(src, dst, INDIRECT);
+        LinkEvent event = linkStore.createOrUpdateLink(PID,
+                    linkDescription);
+        verifyLinkBroadcastMessage(PID, NID1, src, dst, INDIRECT, bcast);
+
+        assertLink(DID1, P1, DID2, P2, INDIRECT, event.subject());
+        assertEquals(LINK_ADDED, event.type());
+
+        // update link type
+        resetCommunicatorExpectingSingleBroadcast(bcast);
+        LinkEvent event2 = linkStore.createOrUpdateLink(PID,
+                new DefaultLinkDescription(src, dst, DIRECT));
+        verifyLinkBroadcastMessage(PID, NID1, src, dst, DIRECT, bcast);
+
+        assertLink(DID1, P1, DID2, P2, DIRECT, event2.subject());
+        assertEquals(LINK_UPDATED, event2.type());
+
+        // no change
+        resetCommunicatorExpectingSingleBroadcast(bcast);
+        LinkEvent event3 = linkStore.createOrUpdateLink(PID,
+                new DefaultLinkDescription(src, dst, DIRECT));
+        verifyNoBroadcastMessage(bcast);
+
+        assertNull("No change event expected", event3);
+    }
+
+    private void verifyNoBroadcastMessage(Capture<ClusterMessage> bcast) {
+        assertFalse("No broadcast expected", bcast.hasCaptured());
+    }
+
+    private void verifyLinkBroadcastMessage(ProviderId providerId,
+                                            NodeId sender,
+                                            ConnectPoint src,
+                                            ConnectPoint dst,
+                                            Type type,
+                                            Capture<ClusterMessage> actualMsg) {
+        verify(clusterCommunicator);
+        assertTrue(actualMsg.hasCaptured());
+        assertEquals(sender, actualMsg.getValue().sender());
+        assertEquals(GossipLinkStoreMessageSubjects.LINK_UPDATE,
+                     actualMsg.getValue().subject());
+        InternalLinkEvent linkEvent
+            = GossipLinkStore.SERIALIZER.decode(actualMsg.getValue().payload());
+        assertEquals(providerId, linkEvent.providerId());
+        assertLinkDescriptionEquals(src, dst, type, linkEvent.linkDescription().value());
+
+    }
+
+    private static void assertLinkDescriptionEquals(ConnectPoint src,
+                                             ConnectPoint dst,
+                                             Type type,
+                                             LinkDescription actual) {
+        assertEquals(src, actual.src());
+        assertEquals(dst, actual.dst());
+        assertEquals(type, actual.type());
+        // TODO check annotations
+    }
+
+    @Test
+    public final void testCreateOrUpdateLinkAncillary() {
+        ConnectPoint src = new ConnectPoint(DID1, P1);
+        ConnectPoint dst = new ConnectPoint(DID2, P2);
+
+        Capture<ClusterMessage> bcast = new Capture<>();
+
+        // add Ancillary link
+        resetCommunicatorExpectingSingleBroadcast(bcast);
+        LinkEvent event = linkStore.createOrUpdateLink(PIDA,
+                    new DefaultLinkDescription(src, dst, INDIRECT, A1));
+        verifyLinkBroadcastMessage(PIDA, NID1, src, dst, INDIRECT, bcast);
+
+        assertNotNull("Ancillary only link is ignored", event);
+
+        // add Primary link
+        resetCommunicatorExpectingSingleBroadcast(bcast);
+        LinkEvent event2 = linkStore.createOrUpdateLink(PID,
+                new DefaultLinkDescription(src, dst, INDIRECT, A2));
+        verifyLinkBroadcastMessage(PID, NID1, src, dst, INDIRECT, bcast);
+
+        assertLink(DID1, P1, DID2, P2, INDIRECT, event2.subject());
+        assertAnnotationsEquals(event2.subject().annotations(), A2, A1);
+        assertEquals(LINK_UPDATED, event2.type());
+
+        // update link type
+        resetCommunicatorExpectingSingleBroadcast(bcast);
+        LinkEvent event3 = linkStore.createOrUpdateLink(PID,
+                new DefaultLinkDescription(src, dst, DIRECT, A2));
+        verifyLinkBroadcastMessage(PID, NID1, src, dst, DIRECT, bcast);
+
+        assertLink(DID1, P1, DID2, P2, DIRECT, event3.subject());
+        assertAnnotationsEquals(event3.subject().annotations(), A2, A1);
+        assertEquals(LINK_UPDATED, event3.type());
+
+
+        // no change
+        resetCommunicatorExpectingNoBroadcast(bcast);
+        LinkEvent event4 = linkStore.createOrUpdateLink(PID,
+                new DefaultLinkDescription(src, dst, DIRECT));
+        verifyNoBroadcastMessage(bcast);
+
+        assertNull("No change event expected", event4);
+
+        // update link annotation (Primary)
+        resetCommunicatorExpectingSingleBroadcast(bcast);
+        LinkEvent event5 = linkStore.createOrUpdateLink(PID,
+                new DefaultLinkDescription(src, dst, DIRECT, A2_2));
+        verifyLinkBroadcastMessage(PID, NID1, src, dst, DIRECT, bcast);
+
+        assertLink(DID1, P1, DID2, P2, DIRECT, event5.subject());
+        assertAnnotationsEquals(event5.subject().annotations(), A2, A2_2, A1);
+        assertEquals(LINK_UPDATED, event5.type());
+
+        // update link annotation (Ancillary)
+        resetCommunicatorExpectingSingleBroadcast(bcast);
+        LinkEvent event6 = linkStore.createOrUpdateLink(PIDA,
+                new DefaultLinkDescription(src, dst, DIRECT, A1_2));
+        verifyLinkBroadcastMessage(PIDA, NID1, src, dst, DIRECT, bcast);
+
+        assertLink(DID1, P1, DID2, P2, DIRECT, event6.subject());
+        assertAnnotationsEquals(event6.subject().annotations(), A2, A2_2, A1, A1_2);
+        assertEquals(LINK_UPDATED, event6.type());
+
+        // update link type (Ancillary) : ignored
+        resetCommunicatorExpectingNoBroadcast(bcast);
+        LinkEvent event7 = linkStore.createOrUpdateLink(PIDA,
+                new DefaultLinkDescription(src, dst, EDGE));
+        verifyNoBroadcastMessage(bcast);
+        assertNull("Ancillary change other than annotation is ignored", event7);
+    }
+
+
+    @Test
+    public final void testRemoveLink() {
+        final ConnectPoint d1P1 = new ConnectPoint(DID1, P1);
+        final ConnectPoint d2P2 = new ConnectPoint(DID2, P2);
+        LinkKey linkId1 = LinkKey.linkKey(d1P1, d2P2);
+        LinkKey linkId2 = LinkKey.linkKey(d2P2, d1P1);
+
+        putLink(linkId1, DIRECT, A1);
+        putLink(linkId2, DIRECT, A2);
+
+        // DID1,P1 => DID2,P2
+        // DID2,P2 => DID1,P1
+        // DID1,P2 => DID2,P3
+
+        LinkEvent event = linkStore.removeLink(d1P1, d2P2);
+        assertEquals(LINK_REMOVED, event.type());
+        assertAnnotationsEquals(event.subject().annotations(), A1);
+        LinkEvent event2 = linkStore.removeLink(d1P1, d2P2);
+        assertNull(event2);
+
+        assertLink(linkId2, DIRECT, linkStore.getLink(d2P2, d1P1));
+        assertAnnotationsEquals(linkStore.getLink(d2P2, d1P1).annotations(), A2);
+
+        // annotations, etc. should not survive remove
+        putLink(linkId1, DIRECT);
+        assertLink(linkId1, DIRECT, linkStore.getLink(d1P1, d2P2));
+        assertAnnotationsEquals(linkStore.getLink(d1P1, d2P2).annotations());
+    }
+
+    @Test
+    public final void testAncillaryVisible() {
+        ConnectPoint src = new ConnectPoint(DID1, P1);
+        ConnectPoint dst = new ConnectPoint(DID2, P2);
+
+        // add Ancillary link
+        linkStore.createOrUpdateLink(PIDA,
+                    new DefaultLinkDescription(src, dst, INDIRECT, A1));
+
+        // Ancillary only link should not be visible
+        assertEquals(1, linkStore.getLinkCount());
+        assertNotNull(linkStore.getLink(src, dst));
+    }
+
+    // If Delegates should be called only on remote events,
+    // then Simple* should never call them, thus not test required.
+    @Ignore("Ignore until Delegate spec. is clear.")
+    @Test
+    public final void testEvents() throws InterruptedException {
+
+        final ConnectPoint d1P1 = new ConnectPoint(DID1, P1);
+        final ConnectPoint d2P2 = new ConnectPoint(DID2, P2);
+        final LinkKey linkId1 = LinkKey.linkKey(d1P1, d2P2);
+
+        final CountDownLatch addLatch = new CountDownLatch(1);
+        LinkStoreDelegate checkAdd = new LinkStoreDelegate() {
+            @Override
+            public void notify(LinkEvent event) {
+                assertEquals(LINK_ADDED, event.type());
+                assertLink(linkId1, INDIRECT, event.subject());
+                addLatch.countDown();
+            }
+        };
+        final CountDownLatch updateLatch = new CountDownLatch(1);
+        LinkStoreDelegate checkUpdate = new LinkStoreDelegate() {
+            @Override
+            public void notify(LinkEvent event) {
+                assertEquals(LINK_UPDATED, event.type());
+                assertLink(linkId1, DIRECT, event.subject());
+                updateLatch.countDown();
+            }
+        };
+        final CountDownLatch removeLatch = new CountDownLatch(1);
+        LinkStoreDelegate checkRemove = new LinkStoreDelegate() {
+            @Override
+            public void notify(LinkEvent event) {
+                assertEquals(LINK_REMOVED, event.type());
+                assertLink(linkId1, DIRECT, event.subject());
+                removeLatch.countDown();
+            }
+        };
+
+        linkStore.setDelegate(checkAdd);
+        putLink(linkId1, INDIRECT);
+        assertTrue("Add event fired", addLatch.await(1, TimeUnit.SECONDS));
+
+        linkStore.unsetDelegate(checkAdd);
+        linkStore.setDelegate(checkUpdate);
+        putLink(linkId1, DIRECT);
+        assertTrue("Update event fired", updateLatch.await(1, TimeUnit.SECONDS));
+
+        linkStore.unsetDelegate(checkUpdate);
+        linkStore.setDelegate(checkRemove);
+        linkStore.removeLink(d1P1, d2P2);
+        assertTrue("Remove event fired", removeLatch.await(1, TimeUnit.SECONDS));
+    }
+
+    private static final class TestClusterService extends StaticClusterService {
+
+        public TestClusterService() {
+            localNode = ONOS1;
+            nodes.put(NID1, ONOS1);
+            nodeStates.put(NID1, ACTIVE);
+
+            nodes.put(NID2, ONOS2);
+            nodeStates.put(NID2, ACTIVE);
+        }
+    }
+}

@@ -23,6 +23,7 @@ import static org.junit.Assert.*;
 import static org.onlab.onos.net.Device.Type.SWITCH;
 import static org.onlab.onos.net.DeviceId.deviceId;
 import static org.onlab.onos.net.device.DeviceEvent.Type.*;
+import static org.onlab.onos.net.NetTestTools.assertAnnotationsEquals;
 
 import java.util.Arrays;
 import java.util.HashMap;
@@ -38,7 +39,6 @@ import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Ignore;
 import org.junit.Test;
-import org.onlab.onos.net.Annotations;
 import org.onlab.onos.net.DefaultAnnotations;
 import org.onlab.onos.net.Device;
 import org.onlab.onos.net.DeviceId;
@@ -56,6 +56,7 @@ import org.onlab.onos.net.provider.ProviderId;
 
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Sets;
+
 import org.onlab.packet.ChassisId;
 
 /**
@@ -146,25 +147,6 @@ public class SimpleDeviceStoreTest {
         assertEquals(SN, device.serialNumber());
     }
 
-    // TODO slice this out somewhere
-    /**
-     * Verifies that Annotations created by merging {@code annotations} is
-     * equal to actual Annotations.
-     *
-     * @param actual Annotations to check
-     * @param annotations
-     */
-    public static void assertAnnotationsEquals(Annotations actual, SparseAnnotations... annotations) {
-        DefaultAnnotations expected = DefaultAnnotations.builder().build();
-        for (SparseAnnotations a : annotations) {
-            expected = DefaultAnnotations.merge(expected, a);
-        }
-        assertEquals(expected.keys(), actual.keys());
-        for (String key : expected.keys()) {
-            assertEquals(expected.value(key), actual.value(key));
-        }
-    }
-
     @Test
     public final void testGetDeviceCount() {
         assertEquals("initialy empty", 0, deviceStore.getDeviceCount());

@@ -16,6 +16,7 @@
 package org.onlab.onos.store.trivial.impl;
 
 import com.google.common.collect.Iterables;
+
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
@@ -46,7 +47,7 @@ import static org.junit.Assert.*;
 import static org.onlab.onos.net.DeviceId.deviceId;
 import static org.onlab.onos.net.Link.Type.*;
 import static org.onlab.onos.net.link.LinkEvent.Type.*;
-import static org.onlab.onos.store.trivial.impl.SimpleDeviceStoreTest.assertAnnotationsEquals;
+import static org.onlab.onos.net.NetTestTools.assertAnnotationsEquals;
 
 /**
  * Test of the simple LinkStore implementation.