Initial implementation: The init extended flow rule and store interface APIs

The APIs are for supporting service data to install on network devices.
This is related to JIRA ticket ID ONOS-869.

Updated API code and added implementation code files.

Modified API for supporting payload abstruction, and added routing mechanism for pushing flow rules to devices.
Added more javadoc, and fixed some minor issues.

Updated javadoc, removed unnecessary method, and test code.

Change-Id: I105defc92a9e01b30601fcb56a9dafa086d4adc0

@@ -172,7 +172,6 @@ public class DefaultFlowRule implements FlowRule {
         return treatment;
     }
 
-
     @Override
     /*
      * The priority and statistics can change on a given treatment and selector

@@ -25,7 +25,7 @@ import java.util.Set;
 public class FlowRuleBatchRequest {
 
     /**
-     * This id is used to cary to id of the original
+     * This id is used to carry to id of the original
      * FlowOperations and track where this batch operation
      * came from. The id is unique cluster wide.
      */
@@ -37,8 +37,6 @@ public class FlowRuleBatchRequest {
     public FlowRuleBatchRequest(long batchId, Set<FlowRuleBatchEntry> ops) {
         this.batchId = batchId;
         this.ops = Collections.unmodifiableSet(ops);
-
-
     }
 
     public Set<FlowRuleBatchEntry> ops() {

+/*
+ * Copyright 2015 Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.net.flowext;
+
+import org.onosproject.core.ApplicationId;
+import org.onosproject.core.DefaultGroupId;
+import org.onosproject.core.GroupId;
+import org.onosproject.net.DeviceId;
+import org.onosproject.net.flow.DefaultFlowRule;
+import org.onosproject.net.flow.FlowRule;
+import org.onosproject.net.flow.TrafficSelector;
+import org.onosproject.net.flow.TrafficTreatment;
+
+import java.util.Objects;
+
+import static com.google.common.base.MoreObjects.toStringHelper;
+
+/**
+ * Experimental extension to the flow rule subsystem; still under development.
+ * A temporary flow rule extend implementation, It will cover current onos flow rule and other flow extension.
+ */
+public class DefaultFlowRuleExt
+        extends DefaultFlowRule implements FlowRuleExt {
+
+    private FlowEntryExtension flowEntryExtension;
+
+    public DefaultFlowRuleExt(DeviceId deviceId, TrafficSelector selector,
+                              TrafficTreatment treatment, int priority, long flowId,
+                              int timeout, boolean permanent) {
+        super(deviceId, selector, treatment, priority, flowId, timeout, permanent);
+    }
+
+    public DefaultFlowRuleExt(DeviceId deviceId, TrafficSelector selector,
+                              TrafficTreatment treatment, int priority, ApplicationId appId,
+                              int timeout, boolean permanent) {
+        this(deviceId, selector, treatment, priority, appId, new DefaultGroupId(0),
+             timeout, permanent);
+    }
+
+    public DefaultFlowRuleExt(DeviceId deviceId, TrafficSelector selector,
+                              TrafficTreatment treatment, int priority, ApplicationId appId,
+                              GroupId groupId, int timeout, boolean permanent) {
+        super(deviceId, selector, treatment, priority, appId, groupId, timeout, permanent);
+    }
+
+    public DefaultFlowRuleExt(FlowRule rule) {
+        super(rule);
+    }
+
+    public DefaultFlowRuleExt(ApplicationId appId, DeviceId deviceId, FlowEntryExtension data) {
+        this(deviceId, null, null, FlowRule.MIN_PRIORITY, appId, 0, false);
+        this.flowEntryExtension = data;
+    }
+
+    @Override
+    public FlowEntryExtension getFlowEntryExt() {
+        return this.flowEntryExtension;
+    }
+
+    @Override
+    public int hashCode() {
+        return 31 * super.hashCode() + Objects.hash(flowEntryExtension);
+    }
+
+    public int hash() {
+        return 31 * super.hashCode() + Objects.hash(flowEntryExtension);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+        if (!super.equals(obj)) {
+            return false;
+        }
+        final DefaultFlowRuleExt other = (DefaultFlowRuleExt) obj;
+        return Objects.equals(this.flowEntryExtension, other.flowEntryExtension);
+    }
+
+    @Override
+    public String toString() {
+        return toStringHelper(this)
+                // TODO there might be a better way to grab super's string
+                .add("id", Long.toHexString(id().value()))
+                .add("deviceId", deviceId())
+                .add("priority", priority())
+                .add("selector", selector().criteria())
+                .add("treatment", treatment() == null ? "N/A" : treatment().instructions())
+                        //.add("created", created)
+                .add("flowEntryExtension", flowEntryExtension)
+                .toString();
+    }
+}

+/*
+ * Copyright 2015 Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.net.flowext;
+
+import java.nio.ByteBuffer;
+import java.util.Objects;
+
+/**
+ * Experimental extension to the flow rule subsystem; still under development.
+ * Represents a generic abstraction of the service data. User app can customize whatever it needs to install on devices.
+ */
+public class DownStreamFlowEntry implements FlowEntryExtension {
+
+    /**
+     * temporarily only have byte stream, but it will be extract more abstract information from it later.
+     */
+    private final ByteBuffer payload;
+
+    public DownStreamFlowEntry(ByteBuffer data) {
+        this.payload = data;
+    }
+
+    /**
+     * Get the payload of flowExtension.
+     *
+     * @return the byte steam value of payload.
+     */
+//   @Override
+//   public ByteBuffer getPayload() {
+    // TODO Auto-generated method stub
+//       return payload;
+//   }
+
+    /**
+     * Returns a hash code value for the object.
+     * It use payload as parameter to hash.
+     *
+     * @return a hash code value for this object.
+     */
+    @Override
+    public int hashCode() {
+        return Objects.hash(payload);
+    }
+
+    /**
+     * Indicates whether some other object is "equal to" this one.
+     *
+     * @param obj the reference object with which to compare.
+     * @return {@code true} if this object is the same as the obj
+     * argument; {@code false} otherwise.
+     */
+    @Override
+    public boolean equals(Object obj) {
+        if (obj instanceof DownStreamFlowEntry) {
+            DownStreamFlowEntry packet = (DownStreamFlowEntry) obj;
+            return Objects.equals(this.payload, packet.payload);
+        } else {
+            return false;
+        }
+    }
+
+    /**
+     * Returns a string representation of the object.
+     *
+     * @return a string representation of the object.
+     */
+    @Override
+    public String toString() {
+        String obj = new String(payload.array());
+        return obj;
+    }
+}

+/*
+ * Copyright 2015 Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.net.flowext;
+
+
+/**
+ * Experimental extension to the flow rule subsystem; still under development.
+ * Represents a generic abstraction of the service data. User app can customize whatever it needs to install on devices.
+ */
+public interface FlowEntryExtension {
+    // some abstraction of the service data, like length, type, etc, will be added here later
+
+}

+/*
+ * Copyright 2015 Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.net.flowext;
+
+import com.google.common.base.MoreObjects;
+import org.onosproject.net.flow.CompletedBatchOperation;
+import org.onosproject.net.flow.FlowRule;
+
+import java.util.Set;
+
+/**
+ * Experimental extension to the flow rule subsystem; still under development.
+ * <p>
+ * Representation of a completed flow rule batch operation.
+ * </p>
+ */
+//TODO explain the purpose of this class beyond FlowRuleProvider
+public class FlowExtCompletedOperation extends CompletedBatchOperation {
+    // the batchId is provided by application, once one flow rule of this batch failed
+    // all the batch should withdraw
+    private final long batchId;
+
+    public FlowExtCompletedOperation(long batchId, boolean success, Set<FlowRule> failures) {
+        super(success, failures, null);
+        this.batchId = batchId;
+    }
+
+    /**
+     * Returns the BatchId of this BatchOperation.
+     *
+     * @return the number of Batch
+     */
+    public long getBatchId() {
+        return batchId;
+    }
+
+    /**
+     * Returns a string representation of the object.
+     *
+     * @return a string representation of the object.
+     */
+    @Override
+    public String toString() {
+        return MoreObjects.toStringHelper(getClass())
+                .add("success?", isSuccess())
+                .add("failedItems", failedIds())
+                .toString();
+    }
+}
\ No newline at end of file

+/*
+ * Copyright 2015 Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.net.flowext;
+
+import org.onosproject.net.flow.FlowRule;
+
+/**
+ * Experimental extension to the flow rule subsystem; still under development.
+ * <p>
+ * FlowRule extended for current FlowRule API.
+ * </p>
+ */
+public interface FlowRuleExt extends FlowRule {
+    /**
+     * Get the flow entry extension.
+     *
+     * @return FlowEntryExtension value.
+     */
+    FlowEntryExtension getFlowEntryExt();
+}

+/*
+ * Copyright 2015 Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.net.flowext;
+
+import org.onosproject.net.flow.FlowRuleBatchEvent;
+import org.onosproject.net.flow.FlowRuleBatchRequest;
+
+import java.util.concurrent.Future;
+
+/**
+ * Experimental extension to the flow rule subsystem; still under development.
+ * Represents a router-like mechanism which is in charge of sending flow rule to master;
+ * <p>
+ * The Router is in charge of sending flow rule to master;
+ * the core component of routing-like mechanism.
+ * </p>
+ */
+public interface FlowRuleExtRouter {
+
+    /**
+     * apply the sub batch of flow extension rules.
+     *
+     * @param batchOperation batch of flow rules.
+     *                       A batch can contain flow rules for a single device only.
+     * @return Future response indicating success/failure of the batch operation
+     * all the way down to the device.
+     */
+    Future<FlowExtCompletedOperation> applySubBatch(FlowRuleBatchRequest batchOperation);
+
+    /**
+     * Invoked on the completion of a storeBatch operation.
+     *
+     * @param event flow rule batch event
+     */
+    void batchOperationComplete(FlowRuleBatchEvent event);
+
+    /**
+     * Register the listener to monitor Router,
+     * The Router find master to send downStream.
+     *
+     * @param listener the listener to register
+     */
+    public void addListener(FlowRuleExtRouterListener listener);
+
+    /**
+     * Remove the listener of Router.
+     *
+     * @param listener the listener to remove
+     */
+    public void removeListener(FlowRuleExtRouterListener listener);
+}

+/*
+ * Copyright 2015 Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.net.flowext;
+
+import org.onosproject.net.flow.FlowRuleBatchEvent;
+
+/**
+ * Experimental extension to the flow rule subsystem; still under development.
+ * The monitor module of the router.
+ * <p>
+ * The monitor module of router.
+ * </p>
+ */
+public interface FlowRuleExtRouterListener {
+
+    /**
+     * Notify monitor the router has down its work.
+     *
+     * @param event the event to notify
+     */
+    void notify(FlowRuleBatchEvent event);
+}

+/*
+ * Copyright 2015 Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.net.flowext;
+
+import org.onosproject.net.flow.FlowRuleBatchRequest;
+import org.onosproject.net.flow.FlowRuleService;
+
+import java.util.concurrent.Future;
+
+/**
+ * Experimental extension to the flow rule subsystem; still under development.
+ * Service for injecting extended flow rules into the environment.
+ * This service just send the packet downstream. It won't store the
+ * flowRuleExtension in cache.
+ */
+public interface FlowRuleExtService extends FlowRuleService {
+    /**
+     * Applies a batch operation of FlowRules.
+     * this batch can be divided into many sub-batch by deviceId, and application
+     * gives a batchId, it means once one flowRule apply failed, all flow rules should
+     * withdraw.
+     *
+     * @param batch batch operation to apply
+     * @return future indicating the state of the batch operation
+     */
+    Future<FlowExtCompletedOperation> applyBatch(FlowRuleBatchRequest batch);
+}

+/*
+ * Copyright 2015 Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Experimental extension to the flow rule subsystem; still under development.
+ * <p>
+ * This package is an extension for the current ONOS flow rule API.
+ * Its main purpose is to support external applications to push service data to network elements.
+ * The service data could be any kind of service related data or commands required for corresponding service
+ * setup and other operations as defined by application and its communicating device.
+ * </p>
+ */
+package org.onosproject.net.flowext;

+/*
+ * Copyright 2015 Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.net.flowext.impl;
+
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.Sets;
+import org.apache.felix.scr.annotations.Activate;
+import org.apache.felix.scr.annotations.Component;
+import org.apache.felix.scr.annotations.Deactivate;
+import org.apache.felix.scr.annotations.Reference;
+import org.apache.felix.scr.annotations.ReferenceCardinality;
+import org.apache.felix.scr.annotations.Service;
+import org.onosproject.event.AbstractListenerRegistry;
+import org.onosproject.event.EventDeliveryService;
+import org.onosproject.net.DeviceId;
+import org.onosproject.net.device.DeviceService;
+import org.onosproject.net.flow.FlowRule;
+import org.onosproject.net.flow.FlowRuleBatchEntry;
+import org.onosproject.net.flow.FlowRuleBatchEvent;
+import org.onosproject.net.flow.FlowRuleBatchRequest;
+import org.onosproject.net.flow.FlowRuleEvent;
+import org.onosproject.net.flow.FlowRuleListener;
+import org.onosproject.net.flow.FlowRuleProvider;
+import org.onosproject.net.flow.impl.FlowRuleManager;
+import org.onosproject.net.flowext.FlowExtCompletedOperation;
+import org.onosproject.net.flowext.FlowRuleExtRouter;
+import org.onosproject.net.flowext.FlowRuleExtRouterListener;
+import org.onosproject.net.flowext.FlowRuleExtService;
+import org.slf4j.Logger;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.CancellationException;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.onlab.util.Tools.namedThreads;
+import static org.slf4j.LoggerFactory.getLogger;
+
+/**
+ * Experimental extension to the flow rule subsystem; still under development.
+ */
+@Component(immediate = true)
+@Service
+public class FlowRuleExtManager extends FlowRuleManager
+        implements FlowRuleExtService {
+
+    enum BatchState {
+        STARTED, FINISHED, CANCELLED
+    }
+
+    public static final String FLOW_RULE_NULL = "FlowRule cannot be null";
+    private final Logger log = getLogger(getClass());
+
+    private final AbstractListenerRegistry<FlowRuleEvent, FlowRuleListener>
+            listenerRegistry = new AbstractListenerRegistry<>();
+
+    private ExecutorService futureService;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected FlowRuleExtRouter router;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected EventDeliveryService eventDispatcher;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected DeviceService deviceService;
+
+    InternalFlowRuleExtRouterListener routerListener = new InternalFlowRuleExtRouterListener();
+
+    @Activate
+    public void activate() {
+        futureService = Executors.newFixedThreadPool(
+                32, namedThreads("provider-future-listeners-%d"));
+        eventDispatcher.addSink(FlowRuleEvent.class, listenerRegistry);
+        router.addListener(routerListener);
+        log.info("Started");
+    }
+
+    @Deactivate
+    public void deactivate() {
+        futureService.shutdownNow();
+        eventDispatcher.removeSink(FlowRuleEvent.class);
+        router.removeListener(routerListener);
+        log.info("Stopped");
+    }
+
+    /**
+     * Applies a batch operation of FlowRules.
+     * this batch can be divided into many sub-batch by deviceId
+     *
+     * @param batch batch operation to apply
+     * @return future indicating the state of the batch operation
+     */
+    @Override
+    public Future<FlowExtCompletedOperation> applyBatch(FlowRuleBatchRequest batch) {
+        // TODO group the Collection into sub-Collection by deviceId
+        Multimap<DeviceId, FlowRuleBatchEntry> perDeviceBatches = ArrayListMultimap
+                .create();
+        for (FlowRuleBatchEntry fbe : batch.ops()) {
+            FlowRule flowRule = fbe.target();
+            perDeviceBatches.put(flowRule.deviceId(), fbe);
+        }
+
+        List<Future<FlowExtCompletedOperation>> futures = Lists.newArrayList();
+        for (DeviceId deviceId : perDeviceBatches.keySet()) {
+            Collection<FlowRuleBatchEntry> flows = perDeviceBatches.get(deviceId);
+            //FIXME if there can be collisions, than converting the collection to a set will drop flow rules
+            FlowRuleBatchRequest subBatch = new FlowRuleBatchRequest(batch.batchId(), Sets.newHashSet(flows));
+            Future<FlowExtCompletedOperation> future = router.applySubBatch(subBatch);
+            futures.add(future);
+        }
+        return new FlowRuleBatchFuture(batch.batchId(), futures);
+    }
+
+    /**
+     * Batch futures include all flow extension entries in one batch.
+     * Using for transaction and will use in next-step.
+     */
+    private class FlowRuleBatchFuture
+            implements Future<FlowExtCompletedOperation> {
+
+        private final List<Future<FlowExtCompletedOperation>> futures;
+        private final long batchId;
+        private final AtomicReference<BatchState> state;
+        private FlowExtCompletedOperation overall;
+
+        public FlowRuleBatchFuture(long batchId, List<Future<FlowExtCompletedOperation>> futures) {
+            this.futures = futures;
+            this.batchId = batchId;
+            state = new AtomicReference<FlowRuleExtManager.BatchState>();
+            state.set(BatchState.STARTED);
+        }
+
+        /**
+         * Attempts to cancel execution of this task.
+         *
+         * @param mayInterruptIfRunning {@code true} if the thread executing this
+         *                              task should be interrupted; otherwise, in-progress tasks are allowed
+         *                              to complete
+         * @return {@code false} if the task could not be cancelled,
+         * typically because it has already completed normally;
+         * {@code true} otherwise
+         */
+        @Override
+        public boolean cancel(boolean mayInterruptIfRunning) {
+            if (state.get() == BatchState.FINISHED) {
+                return false;
+            }
+            if (log.isDebugEnabled()) {
+                log.debug("Cancelling FlowRuleBatchFuture",
+                          new RuntimeException("Just printing backtrace"));
+            }
+            if (!state.compareAndSet(BatchState.STARTED, BatchState.CANCELLED)) {
+                return false;
+            }
+            cleanUpBatch();
+            for (Future<FlowExtCompletedOperation> f : futures) {
+                f.cancel(mayInterruptIfRunning);
+            }
+            return true;
+        }
+
+        /**
+         * Judge whether the task cancelled completely.
+         *
+         * @return {@code true} if this task was cancelled before it completed
+         */
+        @Override
+        public boolean isCancelled() {
+            return state.get() == BatchState.CANCELLED;
+        }
+
+        /**
+         * Judge whether the task finished completely.
+         *
+         * @return {@code true} if this task completed
+         */
+        @Override
+        public boolean isDone() {
+            return state.get() == BatchState.FINISHED;
+        }
+
+        /**
+         * Get the result of apply flow extension rules.
+         * If the task isn't finished, the thread block here.
+         */
+        @Override
+        public FlowExtCompletedOperation get()
+                throws InterruptedException, ExecutionException {
+
+            if (isDone()) {
+                return overall;
+            }
+            boolean success = true;
+            Set<FlowRule> failed = Sets.newHashSet();
+            FlowExtCompletedOperation completed;
+            for (Future<FlowExtCompletedOperation> future : futures) {
+                completed = future.get();
+                success = validateBatchOperation(failed, completed);
+            }
+            return finalizeBatchOperation(success, failed);
+        }
+
+        /**
+         * Waits if necessary for at most the given time for the computation
+         * to complete, and then retrieves its result, if available. In here,
+         * the maximum of time out is sum of given time for every computation.
+         *
+         * @param timeout the maximum time to wait
+         * @param unit    the time unit of the timeout argument
+         * @return the computed result
+         * @throws CancellationException if the computation was cancelled
+         * @throws ExecutionException    if the computation threw an
+         *                               exception
+         * @throws InterruptedException  if the current thread was interrupted
+         *                               while waiting
+         * @throws TimeoutException      if the wait timed out
+         */
+        @Override
+        public FlowExtCompletedOperation get(long timeout, TimeUnit unit)
+                throws InterruptedException, ExecutionException,
+                TimeoutException {
+
+            if (isDone()) {
+                return overall;
+            }
+            boolean success = true;
+            Set<FlowRule> failed = Sets.newHashSet();
+            FlowExtCompletedOperation completed;
+            for (Future<FlowExtCompletedOperation> future : futures) {
+                completed = future.get(timeout, unit);
+                success = validateBatchOperation(failed, completed);
+            }
+            return finalizeBatchOperation(success, failed);
+        }
+
+        /**
+         * Confirm whether the batch operation success.
+         *
+         * @param failed    using to populate failed entries
+         * @param completed the result of apply flow extension entries
+         * @return {@code true} if all entries applies successful
+         */
+        private boolean validateBatchOperation(Set<FlowRule> failed,
+                                               FlowExtCompletedOperation completed) {
+
+            if (isCancelled()) {
+                throw new CancellationException();
+            }
+            if (!completed.isSuccess()) {
+                log.warn("FlowRuleBatch failed: {}", completed);
+                failed.addAll(completed.failedItems());
+                cleanUpBatch();
+                cancelAllSubBatches();
+                return false;
+            }
+            return true;
+        }
+
+        /**
+         * Once one subBatch failed, cancel the rest of them.
+         */
+        private void cancelAllSubBatches() {
+            for (Future<FlowExtCompletedOperation> f : futures) {
+                f.cancel(true);
+            }
+        }
+
+        /**
+         * Construct the result of batch operation.
+         *
+         * @param success the result of batch operation
+         * @param failed  the failed entries of batch operation
+         * @return FlowExtCompletedOperation of batch operation
+         */
+        private FlowExtCompletedOperation finalizeBatchOperation(boolean success,
+                                                                 Set<FlowRule> failed) {
+            synchronized (this) {
+                if (!state.compareAndSet(BatchState.STARTED,
+                                         BatchState.FINISHED)) {
+                    if (state.get() == BatchState.FINISHED) {
+                        return overall;
+                    }
+                    throw new CancellationException();
+                }
+                overall = new FlowExtCompletedOperation(batchId, success, failed);
+                return overall;
+            }
+        }
+
+        private void cleanUpBatch() {
+        }
+    }
+
+    /**
+     * South Bound API to south plug-in.
+     */
+    private class InternalFlowRuleExtRouterListener
+            implements FlowRuleExtRouterListener {
+        @Override
+        public void notify(FlowRuleBatchEvent event) {
+            // Request has been forwarded to MASTER Node
+            for (FlowRuleBatchEntry entry : event.subject().ops()) {
+                switch (entry.operator()) {
+                    case ADD:
+                        eventDispatcher
+                                .post(new FlowRuleEvent(FlowRuleEvent.Type.RULE_ADD_REQUESTED,
+                                                        entry.target()));
+                        break;
+                    // FALLTHROUGH
+                    case REMOVE:
+                    case MODIFY:
+                    default:
+                        // TODO not implemented
+                        break;
+                }
+            }
+            // send it
+            FlowRuleProvider flowRuleProvider = getProvider(event.subject().ops()
+                                                                    .iterator().next().target().deviceId());
+            // TODO we may want to specify a deviceId
+            flowRuleProvider.executeBatch(event.subject().asBatchOperation(null));
+            // do not have transaction, assume it install success
+            // temporarily
+            FlowExtCompletedOperation result = new FlowExtCompletedOperation(
+                    event.subject().batchId(), true, Collections.emptySet());
+            futureService.submit(() -> {
+                router.batchOperationComplete(FlowRuleBatchEvent
+                                                      .completed(event.subject(), result));
+            });
+        }
+    }
+}

+/*
+ * Copyright 2015 Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Experimental extension to the flow rule subsystem; still under development.
+ * <p>
+ * This package is an extension for the current ONOS flow rule subsystem.
+ * Its main purpose is to support external applications to push service data to network elements.
+ * The service data could be any kind of service related data or commands required for corresponding service
+ * setup and other operations as defined by application and its communicating device.
+ * </p>
+ */
+package org.onosproject.net.flowext.impl;

+/*
+ * Copyright 2015 Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.store.flowext.impl;
+
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.SettableFuture;
+import org.apache.felix.scr.annotations.Activate;
+import org.apache.felix.scr.annotations.Component;
+import org.apache.felix.scr.annotations.Deactivate;
+import org.apache.felix.scr.annotations.Reference;
+import org.apache.felix.scr.annotations.ReferenceCardinality;
+import org.apache.felix.scr.annotations.Service;
+import org.onlab.util.KryoNamespace;
+import org.onosproject.cluster.ClusterService;
+import org.onosproject.net.DeviceId;
+import org.onosproject.net.device.DeviceService;
+import org.onosproject.net.flow.CompletedBatchOperation;
+import org.onosproject.net.flow.FlowRuleBatchEntry;
+import org.onosproject.net.flow.FlowRuleBatchEvent;
+import org.onosproject.net.flow.FlowRuleBatchRequest;
+import org.onosproject.net.flowext.DefaultFlowRuleExt;
+import org.onosproject.net.flowext.DownStreamFlowEntry;
+import org.onosproject.net.flowext.FlowExtCompletedOperation;
+import org.onosproject.net.flowext.FlowRuleExtRouter;
+import org.onosproject.net.flowext.FlowRuleExtRouterListener;
+import org.onosproject.store.cluster.messaging.ClusterCommunicationService;
+import org.onosproject.store.cluster.messaging.ClusterMessage;
+import org.onosproject.store.cluster.messaging.ClusterMessageHandler;
+import org.onosproject.store.flow.ReplicaInfo;
+import org.onosproject.store.flow.ReplicaInfoEventListener;
+import org.onosproject.store.flow.ReplicaInfoService;
+import org.onosproject.store.serializers.DecodeTo;
+import org.onosproject.store.serializers.KryoSerializer;
+import org.onosproject.store.serializers.StoreSerializer;
+import org.onosproject.store.serializers.impl.DistributedStoreSerializers;
+import org.slf4j.Logger;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+import static org.onlab.util.Tools.namedThreads;
+import static org.onosproject.store.flowext.impl.FlowExtRouterMessageSubjects.APPLY_EXTEND_FLOWS;
+import static org.slf4j.LoggerFactory.getLogger;
+
+/**
+ * Experimental extension to the flow rule subsystem; still under development.
+ * Implement a simple routing-like mechanism to directly send service data to its master and push to device.
+ * This Router does not save any flow rule extension data in cache, it focus on routing mechanism.
+ */
+@Component(immediate = true)
+@Service
+public class DefaultFlowRuleExtRouter
+        implements FlowRuleExtRouter {
+
+    private final Logger log = getLogger(getClass());
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected ReplicaInfoService replicaInfoManager;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected ClusterCommunicationService clusterCommunicator;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected ClusterService clusterService;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected DeviceService deviceService;
+
+    private int pendingFutureTimeoutMinutes = 5;
+
+    protected Set<FlowRuleExtRouterListener> routerListener = new HashSet<>();
+    private Cache<Long, SettableFuture<FlowExtCompletedOperation>> pendingExtendFutures = CacheBuilder
+            .newBuilder()
+            .expireAfterWrite(pendingFutureTimeoutMinutes, TimeUnit.MINUTES)
+                    // .removalListener(new TimeoutFuture())
+            .build();
+
+    private final ExecutorService futureListeners = Executors
+            .newCachedThreadPool(namedThreads("flowstore-peer-responders"));
+
+    protected static final StoreSerializer SERIALIZER = new KryoSerializer() {
+        @Override
+        protected void setupKryoPool() {
+            serializerPool = KryoNamespace.newBuilder()
+                    .register(DistributedStoreSerializers.STORE_COMMON)
+                    .nextId(DistributedStoreSerializers.STORE_CUSTOM_BEGIN)
+                    .register(FlowExtCompletedOperation.class)
+                    .register(FlowRuleBatchRequest.class)
+                    .register(DownStreamFlowEntry.class)
+                    .register(DefaultFlowRuleExt.class)
+                    .build();
+        }
+    };
+
+    private ReplicaInfoEventListener replicaInfoEventListener;
+
+    @Activate
+    public void activate() {
+        clusterCommunicator.addSubscriber(APPLY_EXTEND_FLOWS,
+            new ClusterMessageHandler() {
+
+              @Override
+              public void handle(ClusterMessage message) {
+                  // decode the extended flow entry and store them in memory.
+                  FlowRuleBatchRequest operation = SERIALIZER.decode(message.payload());
+                  log.info("received batch request {}", operation);
+                  final ListenableFuture<FlowExtCompletedOperation> f = applyBatchInternal(operation);
+                  f.addListener(new Runnable() {
+                      @Override
+                      public void run() {
+                          FlowExtCompletedOperation result = Futures.getUnchecked(f);
+                          try {
+                              message.respond(SERIALIZER.encode(result));
+                          } catch (IOException e) {
+                              log.error("Failed to respond back", e);
+                          }
+                      }
+                  }, futureListeners);
+              }
+            });
+
+        replicaInfoManager.addListener(replicaInfoEventListener);
+
+        log.info("Started");
+    }
+
+    @Deactivate
+    public void deactivate() {
+        clusterCommunicator.removeSubscriber(APPLY_EXTEND_FLOWS);
+        replicaInfoManager.removeListener(replicaInfoEventListener);
+        log.info("Stopped");
+    }
+
+    /**
+     * apply the sub batch of flow extension rules.
+     *
+     * @param batchOperation batch of flow rules.
+     *                       A batch can contain flow rules for a single device only.
+     * @return Future response indicating success/failure of the batch operation
+     * all the way down to the device.
+     */
+    @Override
+    public Future<FlowExtCompletedOperation> applySubBatch(FlowRuleBatchRequest batchOperation) {
+        // TODO Auto-generated method stub
+        if (batchOperation.ops().isEmpty()) {
+            return Futures.immediateFuture(new FlowExtCompletedOperation(
+                    batchOperation.batchId(), true, Collections.emptySet()));
+        }
+        // get the deviceId all the collection belongs to
+        DeviceId deviceId = getBatchDeviceId(batchOperation.ops());
+
+        if (deviceId == null) {
+            log.error("This Batch exists more than two deviceId");
+            return null;
+        }
+        ReplicaInfo replicaInfo = replicaInfoManager
+                .getReplicaInfoFor(deviceId);
+
+        if (replicaInfo.master().get()
+                .equals(clusterService.getLocalNode().id())) {
+            return applyBatchInternal(batchOperation);
+        }
+
+        log.trace("Forwarding storeBatch to {}, which is the primary (master) for device {}",
+                  replicaInfo.master().orNull(), deviceId);
+
+        ClusterMessage message = new ClusterMessage(clusterService
+                    .getLocalNode().id(), APPLY_EXTEND_FLOWS, SERIALIZER.encode(batchOperation));
+
+        try {
+            ListenableFuture<byte[]> responseFuture = clusterCommunicator
+                    .sendAndReceive(message, replicaInfo.master().get());
+            // here should add another decode process
+            return Futures.transform(responseFuture,
+                                     new DecodeTo<FlowExtCompletedOperation>(SERIALIZER));
+        } catch (IOException e) {
+            return Futures.immediateFailedFuture(e);
+        }
+    }
+
+    /**
+     * apply the batch in local node.
+     * It means this instance is master of the device the flow entry belongs to.
+     *
+     * @param batchOperation a collection of flow entry, all they should send down to one device
+     * @return Future response indicating success/failure of the batch operation
+     * all the way down to the device.
+     */
+    private ListenableFuture<FlowExtCompletedOperation> applyBatchInternal(FlowRuleBatchRequest batchOperation) {
+        SettableFuture<FlowExtCompletedOperation> r = SettableFuture.create();
+        pendingExtendFutures.put(batchOperation.batchId(), r);
+        // here should notify manager to complete
+        notify(batchOperation);
+        return r;
+    }
+
+    /**
+     * Get the deviceId of this batch.
+     * The whole Batch should belong to one deviceId.
+     *
+     * @param batchOperation a collection of flow entry, all they should send down to one device
+     * @return the deviceId the whole batch belongs to
+     */
+    private DeviceId getBatchDeviceId(Collection<FlowRuleBatchEntry> batchOperation) {
+        Iterator<FlowRuleBatchEntry> head = batchOperation.iterator();
+        FlowRuleBatchEntry headOp = head.next();
+        boolean sameId = true;
+        for (FlowRuleBatchEntry operation : batchOperation) {
+            if (operation.target().deviceId() != headOp.target().deviceId()) {
+                log.warn("this batch does not apply on one device Id ");
+                sameId = false;
+                break;
+            }
+        }
+        return sameId ? headOp.target().deviceId() : null;
+    }
+
+    /**
+     * Notify the listener of Router to do some reaction.
+     *
+     * @param request the requested operation to do
+     */
+    public void notify(FlowRuleBatchRequest request) {
+        for (FlowRuleExtRouterListener listener : routerListener) {
+            listener.notify(FlowRuleBatchEvent
+                                    // TODO fill in the deviceId
+                                    .requested(request, null));
+        }
+    }
+
+    /**
+     * Invoked on the completion of a storeBatch operation.
+     *
+     * @param event flow rule batch event
+     */
+    @Override
+    public void batchOperationComplete(FlowRuleBatchEvent event) {
+        // TODO Auto-generated method stub
+        final Long batchId = event.subject().batchId();
+        SettableFuture<FlowExtCompletedOperation> future = pendingExtendFutures
+                .getIfPresent(batchId);
+        if (future != null) {
+            FlowRuleBatchRequest request = event.subject();
+            CompletedBatchOperation result = event.result();
+            FlowExtCompletedOperation completed =
+                    new FlowExtCompletedOperation(request.batchId(), result.isSuccess(), result.failedItems());
+            future.set(completed);
+            pendingExtendFutures.invalidate(batchId);
+        }
+    }
+
+    /**
+     * Register the listener to monitor Router,
+     * The Router find master to send downStream.
+     *
+     * @param listener the listener to register
+     */
+    @Override
+    public void addListener(FlowRuleExtRouterListener listener) {
+        routerListener.add(listener);
+    }
+
+    /**
+     * Remove the listener of Router.
+     *
+     * @param listener the listener to remove
+     */
+    @Override
+    public void removeListener(FlowRuleExtRouterListener listener) {
+        routerListener.remove(listener);
+    }
+}
\ No newline at end of file

+/*
+ * Copyright 2015 Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.store.flowext.impl;
+
+import org.onosproject.store.cluster.messaging.MessageSubject;
+
+/**
+ * Experimental extension to the flow rule subsystem; still under development.
+ * MessageSubjects used by DefaultFlowRuleExtRouter peer-peer communication.
+ */
+public final class FlowExtRouterMessageSubjects {
+    private FlowExtRouterMessageSubjects() {
+    }
+
+    /**
+     * The subject of routing extended flow to specified device.
+     */
+    public static final MessageSubject APPLY_EXTEND_FLOWS
+            = new MessageSubject("peer-forward-apply-batch-extension");
+}

+/*
+ * Copyright 2015 Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Experimental extension to the flow rule subsystem; still under development.
+ * <p>
+ * Implementation of the distributed flow extension rule router using p2p synchronization
+ * protocol. The Router is the core component of routing flow rules to specified device.
+ * This package is still experimental at this point in time.
+ * </p>
+ */
+package org.onosproject.store.flowext.impl;