Device Anti-Entropy

- create Advertisement
- handler for Advertisement
- register handler, background thread to send advertisement

Change-Id: I99e8a7d68747970c34b3c25c6d0489769d251446

@@ -4,6 +4,8 @@ import org.onlab.onos.net.AbstractDescription;
 import org.onlab.onos.net.PortNumber;
 import org.onlab.onos.net.PortNumber;
 import org.onlab.onos.net.SparseAnnotations;
 import org.onlab.onos.net.SparseAnnotations;
 
 
+import com.google.common.base.MoreObjects;
+
 /**
 /**
  * Default implementation of immutable port description.
  * Default implementation of immutable port description.
  */
  */
@@ -48,6 +50,15 @@ public class DefaultPortDescription extends AbstractDescription
         return isEnabled;
         return isEnabled;
     }
     }
 
 
+    @Override
+    public String toString() {
+        return MoreObjects.toStringHelper(getClass())
+                .add("number", number)
+                .add("isEnabled", isEnabled)
+                .add("annotations", annotations())
+                .toString();
+    }
+
     // default constructor for serialization
     // default constructor for serialization
     private DefaultPortDescription() {
     private DefaultPortDescription() {
         this.number = null;
         this.number = null;

-package org.onlab.onos.store.common.impl;
-
-import java.util.Map;
-
-import org.onlab.onos.cluster.NodeId;
-import org.onlab.onos.store.Timestamp;
-
-import com.google.common.collect.ImmutableMap;
-
-/**
- * Anti-Entropy advertisement message.
- * <p>
- * Message to advertise the information this node holds.
- *
- * @param <ID> ID type
- */
-public class AntiEntropyAdvertisement<ID> {
-
-    private final NodeId sender;
-    private final ImmutableMap<ID, Timestamp> advertisement;
-
-    /**
-     * Creates anti-entropy advertisement message.
-     *
-     * @param sender sender of this message
-     * @param advertisement timestamp information of the data sender holds
-     */
-    public AntiEntropyAdvertisement(NodeId sender, Map<ID, Timestamp> advertisement) {
-        this.sender = sender;
-        this.advertisement = ImmutableMap.copyOf(advertisement);
-    }
-
-    public NodeId sender() {
-        return sender;
-    }
-
-    public ImmutableMap<ID, Timestamp> advertisement() {
-        return advertisement;
-    }
-
-    // Default constructor for serializer
-    protected AntiEntropyAdvertisement() {
-        this.sender = null;
-        this.advertisement = null;
-    }
-}

-package org.onlab.onos.store.common.impl;
-
-import java.util.Map;
-import java.util.Set;
-
-import org.onlab.onos.cluster.NodeId;
-import org.onlab.onos.store.device.impl.VersionedValue;
-
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSet;
-
-/**
- * Anti-Entropy reply message.
- * <p>
- * Message to send in reply to advertisement or another reply.
- * Suggest to the sender about the more up-to-date data this node has,
- * and request for more recent data that the receiver has.
- */
-public class AntiEntropyReply<ID, V extends VersionedValue<?>> {
-
-    private final NodeId sender;
-    private final ImmutableMap<ID, V> suggestion;
-    private final ImmutableSet<ID> request;
-
-    /**
-     * Creates a reply to anti-entropy message.
-     *
-     * @param sender sender of this message
-     * @param suggestion collection of more recent values, sender had
-     * @param request Collection of identifiers
-     */
-    public AntiEntropyReply(NodeId sender,
-                            Map<ID, V> suggestion,
-                            Set<ID> request) {
-        this.sender = sender;
-        this.suggestion = ImmutableMap.copyOf(suggestion);
-        this.request = ImmutableSet.copyOf(request);
-    }
-
-    public NodeId sender() {
-        return sender;
-    }
-
-    /**
-     * Returns collection of values, which the recipient of this reply is likely
-     * to be missing or has outdated version.
-     *
-     * @return
-     */
-    public ImmutableMap<ID, V> suggestion() {
-        return suggestion;
-    }
-
-    /**
-     * Returns collection of identifier to request.
-     *
-     * @return collection of identifier to request
-     */
-    public ImmutableSet<ID> request() {
-        return request;
-    }
-
-    /**
-     * Checks if reply contains any suggestion or request.
-     *
-     * @return true if nothing is suggested and requested
-     */
-    public boolean isEmpty() {
-        return suggestion.isEmpty() && request.isEmpty();
-    }
-
-    // Default constructor for serializer
-    protected AntiEntropyReply() {
-        this.sender = null;
-        this.suggestion = null;
-        this.request = null;
-    }
-}

+package org.onlab.onos.store.common.impl;
+
+import org.onlab.onos.cluster.ControllerNode;
+import org.onlab.onos.cluster.NodeId;
+
+import com.google.common.base.Function;
+
+/**
+ * Function to convert ControllerNode to NodeId.
+ */
+public final class ControllerNodeToNodeId
+    implements Function<ControllerNode, NodeId> {
+
+    private static final ControllerNodeToNodeId INSTANCE = new ControllerNodeToNodeId();
+
+    @Override
+    public NodeId apply(ControllerNode input) {
+        return input.id();
+    }
+
+    public static ControllerNodeToNodeId toNodeId() {
+        return INSTANCE;
+    }
+}

@@ -30,6 +30,7 @@ public final class Timestamped<T> {
 
 
     /**
     /**
      * Returns the value.
      * Returns the value.
+     *
      * @return value
      * @return value
      */
      */
     public T value() {
     public T value() {
@@ -38,6 +39,7 @@ public final class Timestamped<T> {
 
 
     /**
     /**
      * Returns the time stamp.
      * Returns the time stamp.
+     *
      * @return time stamp
      * @return time stamp
      */
      */
     public Timestamp timestamp() {
     public Timestamp timestamp() {
@@ -51,7 +53,16 @@ public final class Timestamped<T> {
      * @return true if this instance is newer.
      * @return true if this instance is newer.
      */
      */
     public boolean isNewer(Timestamped<T> other) {
     public boolean isNewer(Timestamped<T> other) {
-        return this.timestamp.compareTo(checkNotNull(other).timestamp()) > 0;
+        return isNewer(checkNotNull(other).timestamp());
+    }
+
+    /**
+     * Tests if this timestamp is newer thatn the specified timestamp.
+     * @param timestamp to compare agains
+     * @return true if this instance is newer
+     */
+    public boolean isNewer(Timestamp timestamp) {
+        return this.timestamp.compareTo(checkNotNull(timestamp)) > 0;
     }
     }
 
 
     @Override
     @Override

-package org.onlab.onos.store.device.impl;
-
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.onlab.onos.cluster.NodeId;
-import org.onlab.onos.net.Device;
-import org.onlab.onos.net.DeviceId;
-import org.onlab.onos.store.Timestamp;
-import org.onlab.onos.store.common.impl.AntiEntropyAdvertisement;
-
-// TODO DeviceID needs to be changed to something like (ProviderID, DeviceID)
-// TODO: Handle Port as part of these messages, or separate messages for Ports?
-
-public class DeviceAntiEntropyAdvertisement
-    extends AntiEntropyAdvertisement<DeviceId> {
-
-
-    public DeviceAntiEntropyAdvertisement(NodeId sender,
-            Map<DeviceId, Timestamp> advertisement) {
-        super(sender, advertisement);
-    }
-
-    // May need to add ProviderID, etc.
-    public static DeviceAntiEntropyAdvertisement create(
-            NodeId self,
-            Collection<VersionedValue<Device>> localValues) {
-
-        Map<DeviceId, Timestamp> ads = new HashMap<>(localValues.size());
-        for (VersionedValue<Device> e : localValues) {
-            ads.put(e.entity().id(), e.timestamp());
-        }
-        return new DeviceAntiEntropyAdvertisement(self, ads);
-    }
-
-    // For serializer
-    protected DeviceAntiEntropyAdvertisement() {}
-}

-package org.onlab.onos.store.device.impl;
-
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-import org.onlab.onos.cluster.NodeId;
-import org.onlab.onos.net.Device;
-import org.onlab.onos.net.DeviceId;
-import org.onlab.onos.store.Timestamp;
-import org.onlab.onos.store.common.impl.AntiEntropyReply;
-
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSet;
-
-public class DeviceAntiEntropyReply
-    extends AntiEntropyReply<DeviceId, VersionedValue<Device>> {
-
-
-    public DeviceAntiEntropyReply(NodeId sender,
-            Map<DeviceId, VersionedValue<Device>> suggestion,
-            Set<DeviceId> request) {
-        super(sender, suggestion, request);
-    }
-
-    /**
-     * Creates a reply to Anti-Entropy advertisement.
-     *
-     * @param advertisement to respond to
-     * @param self node identifier representing local node
-     * @param localValues local values held on this node
-     * @return reply message
-     */
-    public static DeviceAntiEntropyReply reply(
-            DeviceAntiEntropyAdvertisement advertisement,
-            NodeId self,
-            Collection<VersionedValue<Device>> localValues
-            ) {
-
-        ImmutableMap<DeviceId, Timestamp> ads = advertisement.advertisement();
-
-        ImmutableMap.Builder<DeviceId, VersionedValue<Device>>
-            sug = ImmutableMap.builder();
-
-        Set<DeviceId> req = new HashSet<>(ads.keySet());
-
-        for (VersionedValue<Device> e : localValues) {
-            final DeviceId id = e.entity().id();
-            final Timestamp local = e.timestamp();
-            final Timestamp theirs = ads.get(id);
-            if (theirs == null) {
-                // they don't have it, suggest
-                sug.put(id, e);
-                // don't need theirs
-                req.remove(id);
-            } else if (local.compareTo(theirs) < 0) {
-                // they got older one, suggest
-                sug.put(id, e);
-                // don't need theirs
-                req.remove(id);
-            } else if (local.equals(theirs)) {
-                // same, don't need theirs
-                req.remove(id);
-            }
-        }
-
-        return new DeviceAntiEntropyReply(self, sug.build(), req);
-    }
-
-    /**
-     * Creates a reply to request for values held locally.
-     *
-     * @param requests message containing the request
-     * @param self node identifier representing local node
-     * @param localValues local valeds held on this node
-     * @return reply message
-     */
-    public static DeviceAntiEntropyReply reply(
-            DeviceAntiEntropyReply requests,
-            NodeId self,
-            Map<DeviceId, VersionedValue<Device>> localValues
-            ) {
-
-        Set<DeviceId> reqs = requests.request();
-
-        Map<DeviceId, VersionedValue<Device>> requested = new HashMap<>(reqs.size());
-        for (DeviceId id : reqs) {
-            final VersionedValue<Device> value = localValues.get(id);
-            if (value != null) {
-                requested.put(id, value);
-            }
-        }
-
-        Set<DeviceId> empty = ImmutableSet.of();
-        return new DeviceAntiEntropyReply(self, requested, empty);
-    }
-
-    // For serializer
-    protected DeviceAntiEntropyReply() {}
-}

+package org.onlab.onos.store.device.impl;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static org.onlab.onos.net.DefaultAnnotations.union;
+
+import java.util.Collections;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import org.onlab.onos.net.PortNumber;
+import org.onlab.onos.net.SparseAnnotations;
+import org.onlab.onos.net.device.DefaultDeviceDescription;
+import org.onlab.onos.net.device.DefaultPortDescription;
+import org.onlab.onos.net.device.DeviceDescription;
+import org.onlab.onos.net.device.PortDescription;
+import org.onlab.onos.store.Timestamp;
+import org.onlab.onos.store.common.impl.Timestamped;
+
+/*
+ * Collection of Description of a Device and Ports, given from a Provider.
+ */
+class DeviceDescriptions {
+
+    private volatile Timestamped<DeviceDescription> deviceDesc;
+
+    private final ConcurrentMap<PortNumber, Timestamped<PortDescription>> portDescs;
+
+    public DeviceDescriptions(Timestamped<DeviceDescription> desc) {
+        this.deviceDesc = checkNotNull(desc);
+        this.portDescs = new ConcurrentHashMap<>();
+    }
+
+    public Timestamp getLatestTimestamp() {
+        Timestamp latest = deviceDesc.timestamp();
+        for (Timestamped<PortDescription> desc : portDescs.values()) {
+            if (desc.timestamp().compareTo(latest) > 0) {
+                latest = desc.timestamp();
+            }
+        }
+        return latest;
+    }
+
+    public Timestamped<DeviceDescription> getDeviceDesc() {
+        return deviceDesc;
+    }
+
+    public Timestamped<PortDescription> getPortDesc(PortNumber number) {
+        return portDescs.get(number);
+    }
+
+    public Map<PortNumber, Timestamped<PortDescription>> getPortDescs() {
+        return Collections.unmodifiableMap(portDescs);
+    }
+
+    /**
+     * Puts DeviceDescription, merging annotations as necessary.
+     *
+     * @param newDesc new DeviceDescription
+     */
+    public synchronized void putDeviceDesc(Timestamped<DeviceDescription> newDesc) {
+        Timestamped<DeviceDescription> oldOne = deviceDesc;
+        Timestamped<DeviceDescription> newOne = newDesc;
+        if (oldOne != null) {
+            SparseAnnotations merged = union(oldOne.value().annotations(),
+                                             newDesc.value().annotations());
+            newOne = new Timestamped<DeviceDescription>(
+                    new DefaultDeviceDescription(newDesc.value(), merged),
+                    newDesc.timestamp());
+        }
+        deviceDesc = newOne;
+    }
+
+    /**
+     * Puts PortDescription, merging annotations as necessary.
+     *
+     * @param newDesc new PortDescription
+     */
+    public synchronized void putPortDesc(Timestamped<PortDescription> newDesc) {
+        Timestamped<PortDescription> oldOne = portDescs.get(newDesc.value().portNumber());
+        Timestamped<PortDescription> newOne = newDesc;
+        if (oldOne != null) {
+            SparseAnnotations merged = union(oldOne.value().annotations(),
+                                             newDesc.value().annotations());
+            newOne = new Timestamped<PortDescription>(
+                    new DefaultPortDescription(newDesc.value(), merged),
+                    newDesc.timestamp());
+        }
+        portDescs.put(newOne.value().portNumber(), newOne);
+    }
+}

@@ -2,6 +2,7 @@ package org.onlab.onos.store.device.impl;
 
 
 import org.onlab.onos.store.cluster.messaging.MessageSubject;
 import org.onlab.onos.store.cluster.messaging.MessageSubject;
 
 
+// TODO: add prefix to assure uniqueness.
 /**
 /**
  * MessageSubjects used by GossipDeviceStore peer-peer communication.
  * MessageSubjects used by GossipDeviceStore peer-peer communication.
  */
  */
@@ -14,4 +15,8 @@ public final class GossipDeviceStoreMessageSubjects {
     public static final MessageSubject DEVICE_REMOVED = new MessageSubject("peer-device-removed");
     public static final MessageSubject DEVICE_REMOVED = new MessageSubject("peer-device-removed");
     public static final MessageSubject PORT_UPDATE = new MessageSubject("peer-port-update");
     public static final MessageSubject PORT_UPDATE = new MessageSubject("peer-port-update");
     public static final MessageSubject PORT_STATUS_UPDATE = new MessageSubject("peer-port-status-update");
     public static final MessageSubject PORT_STATUS_UPDATE = new MessageSubject("peer-port-status-update");
+
+    public static final MessageSubject DEVICE_ADVERTISE = new MessageSubject("peer-device-advertisements");
+    // to be used with 3-way anti-entropy process
+    public static final MessageSubject DEVICE_REQUEST = new MessageSubject("peer-device-request");
 }
 }

+package org.onlab.onos.store.device.impl;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import org.apache.commons.lang3.concurrent.ConcurrentException;
+import org.apache.commons.lang3.concurrent.ConcurrentInitializer;
+import org.onlab.onos.net.device.DeviceDescription;
+import org.onlab.onos.store.common.impl.Timestamped;
+
+// FIXME: consider removing this class
+public final class InitDeviceDescs
+    implements ConcurrentInitializer<DeviceDescriptions> {
+
+    private final Timestamped<DeviceDescription> deviceDesc;
+
+    public InitDeviceDescs(Timestamped<DeviceDescription> deviceDesc) {
+        this.deviceDesc = checkNotNull(deviceDesc);
+    }
+    @Override
+    public DeviceDescriptions get() throws ConcurrentException {
+        return new DeviceDescriptions(deviceDesc);
+    }
+}
\ No newline at end of file

@@ -5,6 +5,8 @@ import org.onlab.onos.net.device.DeviceDescription;
 import org.onlab.onos.net.provider.ProviderId;
 import org.onlab.onos.net.provider.ProviderId;
 import org.onlab.onos.store.common.impl.Timestamped;
 import org.onlab.onos.store.common.impl.Timestamped;
 
 
+import com.google.common.base.MoreObjects;
+
 /**
 /**
  * Information published by GossipDeviceStore to notify peers of a device
  * Information published by GossipDeviceStore to notify peers of a device
  * change event.
  * change event.
@@ -36,6 +38,15 @@ public class InternalDeviceEvent {
         return deviceDescription;
         return deviceDescription;
     }
     }
 
 
+    @Override
+    public String toString() {
+        return MoreObjects.toStringHelper(getClass())
+                .add("providerId", providerId)
+                .add("deviceId", deviceId)
+                .add("deviceDescription", deviceDescription)
+                .toString();
+    }
+
     // for serializer
     // for serializer
     protected InternalDeviceEvent() {
     protected InternalDeviceEvent() {
         this.providerId = null;
         this.providerId = null;

@@ -3,6 +3,8 @@ package org.onlab.onos.store.device.impl;
 import org.onlab.onos.net.DeviceId;
 import org.onlab.onos.net.DeviceId;
 import org.onlab.onos.store.Timestamp;
 import org.onlab.onos.store.Timestamp;
 
 
+import com.google.common.base.MoreObjects;
+
 /**
 /**
  * Information published by GossipDeviceStore to notify peers of a device
  * Information published by GossipDeviceStore to notify peers of a device
  * going offline.
  * going offline.
@@ -30,6 +32,14 @@ public class InternalDeviceOfflineEvent {
         return timestamp;
         return timestamp;
     }
     }
 
 
+    @Override
+    public String toString() {
+        return MoreObjects.toStringHelper(getClass())
+                .add("deviceId", deviceId)
+                .add("timestamp", timestamp)
+                .toString();
+    }
+
     // for serializer
     // for serializer
     @SuppressWarnings("unused")
     @SuppressWarnings("unused")
     private InternalDeviceOfflineEvent() {
     private InternalDeviceOfflineEvent() {

@@ -3,6 +3,8 @@ package org.onlab.onos.store.device.impl;
 import org.onlab.onos.net.DeviceId;
 import org.onlab.onos.net.DeviceId;
 import org.onlab.onos.store.Timestamp;
 import org.onlab.onos.store.Timestamp;
 
 
+import com.google.common.base.MoreObjects;
+
 /**
 /**
  * Information published by GossipDeviceStore to notify peers of a device
  * Information published by GossipDeviceStore to notify peers of a device
  * being administratively removed.
  * being administratively removed.
@@ -30,6 +32,14 @@ public class InternalDeviceRemovedEvent {
         return timestamp;
         return timestamp;
     }
     }
 
 
+    @Override
+    public String toString() {
+        return MoreObjects.toStringHelper(getClass())
+                .add("deviceId", deviceId)
+                .add("timestamp", timestamp)
+                .toString();
+    }
+
     // for serializer
     // for serializer
     @SuppressWarnings("unused")
     @SuppressWarnings("unused")
     private InternalDeviceRemovedEvent() {
     private InternalDeviceRemovedEvent() {

@@ -7,6 +7,8 @@ import org.onlab.onos.net.device.PortDescription;
 import org.onlab.onos.net.provider.ProviderId;
 import org.onlab.onos.net.provider.ProviderId;
 import org.onlab.onos.store.common.impl.Timestamped;
 import org.onlab.onos.store.common.impl.Timestamped;
 
 
+import com.google.common.base.MoreObjects;
+
 /**
 /**
  * Information published by GossipDeviceStore to notify peers of a port
  * Information published by GossipDeviceStore to notify peers of a port
  * change event.
  * change event.
@@ -38,6 +40,15 @@ public class InternalPortEvent {
         return portDescriptions;
         return portDescriptions;
     }
     }
 
 
+    @Override
+    public String toString() {
+        return MoreObjects.toStringHelper(getClass())
+                .add("providerId", providerId)
+                .add("deviceId", deviceId)
+                .add("portDescriptions", portDescriptions)
+                .toString();
+    }
+
     // for serializer
     // for serializer
     protected InternalPortEvent() {
     protected InternalPortEvent() {
         this.providerId = null;
         this.providerId = null;

@@ -5,6 +5,8 @@ import org.onlab.onos.net.device.PortDescription;
 import org.onlab.onos.net.provider.ProviderId;
 import org.onlab.onos.net.provider.ProviderId;
 import org.onlab.onos.store.common.impl.Timestamped;
 import org.onlab.onos.store.common.impl.Timestamped;
 
 
+import com.google.common.base.MoreObjects;
+
 /**
 /**
  * Information published by GossipDeviceStore to notify peers of a port
  * Information published by GossipDeviceStore to notify peers of a port
  * status change event.
  * status change event.
@@ -36,6 +38,15 @@ public class InternalPortStatusEvent {
         return portDescription;
         return portDescription;
     }
     }
 
 
+    @Override
+    public String toString() {
+        return MoreObjects.toStringHelper(getClass())
+                .add("providerId", providerId)
+                .add("deviceId", deviceId)
+                .add("portDescription", portDescription)
+                .toString();
+    }
+
     // for serializer
     // for serializer
     protected InternalPortStatusEvent() {
     protected InternalPortStatusEvent() {
         this.providerId = null;
         this.providerId = null;

@@ -35,6 +35,7 @@ public class InternalPortStatusEventSerializer extends Serializer<InternalPortSt
                                Class<InternalPortStatusEvent> type) {
                                Class<InternalPortStatusEvent> type) {
         ProviderId providerId = (ProviderId) kryo.readClassAndObject(input);
         ProviderId providerId = (ProviderId) kryo.readClassAndObject(input);
         DeviceId deviceId = (DeviceId) kryo.readClassAndObject(input);
         DeviceId deviceId = (DeviceId) kryo.readClassAndObject(input);
+        @SuppressWarnings("unchecked")
         Timestamped<PortDescription> portDescription = (Timestamped<PortDescription>) kryo.readClassAndObject(input);
         Timestamped<PortDescription> portDescription = (Timestamped<PortDescription>) kryo.readClassAndObject(input);
 
 
         return new InternalPortStatusEvent(providerId, deviceId, portDescription);
         return new InternalPortStatusEvent(providerId, deviceId, portDescription);

+package org.onlab.onos.store.device.impl.peermsg;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import java.util.Map;
+
+import org.onlab.onos.cluster.NodeId;
+import org.onlab.onos.net.DeviceId;
+import org.onlab.onos.store.Timestamp;
+
+
+/**
+ * Device Advertisement message.
+ */
+public class DeviceAntiEntropyAdvertisement {
+
+    private final NodeId sender;
+    private final Map<DeviceFragmentId, Timestamp> deviceFingerPrints;
+    private final Map<PortFragmentId, Timestamp> portFingerPrints;
+    private final Map<DeviceId, Timestamp> offline;
+
+
+    public DeviceAntiEntropyAdvertisement(NodeId sender,
+                Map<DeviceFragmentId, Timestamp> devices,
+                Map<PortFragmentId, Timestamp> ports,
+                Map<DeviceId, Timestamp> offline) {
+        this.sender = checkNotNull(sender);
+        this.deviceFingerPrints = checkNotNull(devices);
+        this.portFingerPrints = checkNotNull(ports);
+        this.offline = checkNotNull(offline);
+    }
+
+    public NodeId sender() {
+        return sender;
+    }
+
+    public Map<DeviceFragmentId, Timestamp> deviceFingerPrints() {
+        return deviceFingerPrints;
+    }
+
+    public Map<PortFragmentId, Timestamp> ports() {
+        return portFingerPrints;
+    }
+
+    public Map<DeviceId, Timestamp> offline() {
+        return offline;
+    }
+
+    // For serializer
+    @SuppressWarnings("unused")
+    private DeviceAntiEntropyAdvertisement() {
+        this.sender = null;
+        this.deviceFingerPrints = null;
+        this.portFingerPrints = null;
+        this.offline = null;
+    }
+}

+package org.onlab.onos.store.device.impl.peermsg;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import java.util.Collection;
+
+import org.onlab.onos.cluster.NodeId;
+
+/**
+ * Message to request for other peers information.
+ */
+public class DeviceAntiEntropyRequest {
+
+    private final NodeId sender;
+    private final Collection<DeviceFragmentId> devices;
+    private final Collection<PortFragmentId> ports;
+
+    public DeviceAntiEntropyRequest(NodeId sender,
+                                   Collection<DeviceFragmentId> devices,
+                                   Collection<PortFragmentId> ports) {
+
+        this.sender = checkNotNull(sender);
+        this.devices = checkNotNull(devices);
+        this.ports = checkNotNull(ports);
+    }
+
+    public NodeId sender() {
+        return sender;
+    }
+
+    public Collection<DeviceFragmentId> devices() {
+        return devices;
+    }
+
+    public Collection<PortFragmentId> ports() {
+        return ports;
+    }
+
+    // For serializer
+    @SuppressWarnings("unused")
+    private DeviceAntiEntropyRequest() {
+        this.sender = null;
+        this.devices = null;
+        this.ports = null;
+    }
+}

+package org.onlab.onos.store.device.impl.peermsg;
+
+import java.util.Objects;
+
+import org.onlab.onos.net.DeviceId;
+import org.onlab.onos.net.provider.ProviderId;
+
+import com.google.common.base.MoreObjects;
+
+/**
+ * Identifier for DeviceDesctiption from a Provider.
+ */
+public final class DeviceFragmentId {
+    public final ProviderId providerId;
+    public final DeviceId deviceId;
+
+    public DeviceFragmentId(DeviceId deviceId, ProviderId providerId) {
+        this.providerId = providerId;
+        this.deviceId = deviceId;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(providerId, deviceId);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (!(obj instanceof DeviceFragmentId)) {
+            return false;
+        }
+        DeviceFragmentId that = (DeviceFragmentId) obj;
+        return Objects.equals(this.deviceId, that.deviceId) &&
+               Objects.equals(this.providerId, that.providerId);
+    }
+
+    @Override
+    public String toString() {
+        return MoreObjects.toStringHelper(getClass())
+                .add("providerId", providerId)
+                .add("deviceId", deviceId)
+                .toString();
+    }
+
+    // for serializer
+    @SuppressWarnings("unused")
+    private DeviceFragmentId() {
+        this.providerId = null;
+        this.deviceId = null;
+    }
+}
\ No newline at end of file

+package org.onlab.onos.store.device.impl.peermsg;
+
+import java.util.Objects;
+
+import org.onlab.onos.net.DeviceId;
+import org.onlab.onos.net.PortNumber;
+import org.onlab.onos.net.provider.ProviderId;
+
+import com.google.common.base.MoreObjects;
+
+/**
+ * Identifier for PortDescription from a Provider.
+ */
+public final class PortFragmentId {
+    public final ProviderId providerId;
+    public final DeviceId deviceId;
+    public final PortNumber portNumber;
+
+    public PortFragmentId(DeviceId deviceId, ProviderId providerId,
+                          PortNumber portNumber) {
+        this.providerId = providerId;
+        this.deviceId = deviceId;
+        this.portNumber = portNumber;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(providerId, deviceId, portNumber);
+    };
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (!(obj instanceof PortFragmentId)) {
+            return false;
+        }
+        PortFragmentId that = (PortFragmentId) obj;
+        return Objects.equals(this.deviceId, that.deviceId) &&
+               Objects.equals(this.portNumber, that.portNumber) &&
+               Objects.equals(this.providerId, that.providerId);
+    }
+
+    @Override
+    public String toString() {
+        return MoreObjects.toStringHelper(getClass())
+                .add("providerId", providerId)
+                .add("deviceId", deviceId)
+                .add("portNumber", portNumber)
+                .toString();
+    }
+
+    // for serializer
+    @SuppressWarnings("unused")
+    private PortFragmentId() {
+        this.providerId = null;
+        this.deviceId = null;
+        this.portNumber = null;
+    }
+}
\ No newline at end of file

+/**
+ * Structure and utilities used for inter-Node messaging.
+ */
+package org.onlab.onos.store.device.impl.peermsg;

@@ -31,7 +31,6 @@ import org.onlab.onos.net.provider.ProviderId;
 import org.onlab.onos.store.AbstractStore;
 import org.onlab.onos.store.AbstractStore;
 import org.onlab.onos.store.ClockService;
 import org.onlab.onos.store.ClockService;
 import org.onlab.onos.store.Timestamp;
 import org.onlab.onos.store.Timestamp;
-import org.onlab.onos.store.device.impl.VersionedValue;
 import org.slf4j.Logger;
 import org.slf4j.Logger;
 
 
 import com.google.common.collect.HashMultimap;
 import com.google.common.collect.HashMultimap;

-package org.onlab.onos.store.device.impl;
+package org.onlab.onos.store.link.impl;
 
 
 import java.util.Objects;
 import java.util.Objects;
 
 
 import org.onlab.onos.store.Timestamp;
 import org.onlab.onos.store.Timestamp;
 
 
+// TODO: remove once we stop using this
 /**
 /**
  * Wrapper class for a entity that is versioned
  * Wrapper class for a entity that is versioned
  * and can either be up or down.
  * and can either be up or down.

+package org.onlab.onos.store.serializers;
+
+import org.onlab.onos.store.common.impl.MastershipBasedTimestamp;
+import org.onlab.onos.store.common.impl.Timestamped;
+import org.onlab.util.KryoPool;
+
+public final class DistributedStoreSerializers {
+
+    /**
+     * KryoPool which can serialize ON.lab misc classes.
+     */
+    public static final KryoPool COMMON = KryoPool.newBuilder()
+            .register(KryoPoolUtil.API)
+            .register(Timestamped.class)
+            .register(MastershipBasedTimestamp.class, new MastershipBasedTimestampSerializer())
+            .build();
+
+    // avoid instantiation
+    private DistributedStoreSerializers() {}
+}

@@ -25,6 +25,7 @@ import org.onlab.onos.net.PortNumber;
 import org.onlab.onos.net.device.DefaultDeviceDescription;
 import org.onlab.onos.net.device.DefaultDeviceDescription;
 import org.onlab.onos.net.device.DefaultPortDescription;
 import org.onlab.onos.net.device.DefaultPortDescription;
 import org.onlab.onos.net.provider.ProviderId;
 import org.onlab.onos.net.provider.ProviderId;
+import org.onlab.onos.store.Timestamp;
 import org.onlab.packet.IpAddress;
 import org.onlab.packet.IpAddress;
 import org.onlab.packet.IpPrefix;
 import org.onlab.packet.IpPrefix;
 import org.onlab.util.KryoPool;
 import org.onlab.util.KryoPool;
@@ -63,7 +64,9 @@ public final class KryoPoolUtil {
                     Port.class,
                     Port.class,
                     DefaultPortDescription.class,
                     DefaultPortDescription.class,
                     Element.class,
                     Element.class,
-                    Link.Type.class
+                    Link.Type.class,
+                    Timestamp.class
+
                     )
                     )
             .register(URI.class, new URISerializer())
             .register(URI.class, new URISerializer())
             .register(NodeId.class, new NodeIdSerializer())
             .register(NodeId.class, new NodeIdSerializer())

 package org.onlab.onos.store.serializers;
 package org.onlab.onos.store.serializers;
 
 
 import org.onlab.util.KryoPool;
 import org.onlab.util.KryoPool;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 import java.nio.ByteBuffer;
 import java.nio.ByteBuffer;
 
 
 /**
 /**
@@ -11,10 +8,8 @@ import java.nio.ByteBuffer;
  */
  */
 public class KryoSerializer implements StoreSerializer {
 public class KryoSerializer implements StoreSerializer {
 
 
-    private final Logger log = LoggerFactory.getLogger(getClass());
     protected KryoPool serializerPool;
     protected KryoPool serializerPool;
 
 
-
     public KryoSerializer() {
     public KryoSerializer() {
         setupKryoPool();
         setupKryoPool();
     }
     }