[ONOS-4481] Integrate scalable gateway to openstackRouting

 - Modified openstack_routing to use scalable gateway application
 - Add GatewayNode Builder method

Change-Id: Ib12d52009eeb89ab59f01cafe794894aac80985e

@@ -3,6 +3,7 @@ COMPILE_DEPS = [
     '//core/store/serializers:onos-core-serializers',
     '//apps/openstackinterface/api:onos-apps-openstackinterface-api',
     '//apps/openstacknetworking/api:onos-apps-openstacknetworking-api',
+    '//apps/scalablegateway:onos-apps-scalablegateway',
 ]
 
 BUNDLES = [

@@ -52,6 +52,11 @@
             <artifactId>onos-app-openstackinterface-api</artifactId>
             <version>${project.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.onosproject</groupId>
+            <artifactId>onos-scalablegateway</artifactId>
+            <version>${project.version}</version>
+        </dependency>
     </dependencies>
 </project>
 

@@ -40,6 +40,7 @@ import org.onosproject.openstackinterface.OpenstackPort;
 import org.onosproject.openstacknetworking.OpenstackNetworkingConfig;
 import org.onosproject.openstacknetworking.OpenstackPortInfo;
 import org.onosproject.openstacknetworking.OpenstackSwitchingService;
+import org.onosproject.scalablegateway.api.ScalableGatewayService;
 import org.slf4j.Logger;
 
 import java.nio.ByteBuffer;
@@ -62,13 +63,14 @@ public class OpenstackIcmpHandler {
     private final Map<String, OpenstackPortInfo> icmpInfoMap = Maps.newHashMap();
     private final OpenstackSwitchingService openstackSwitchingService;
     private final OpenstackInterfaceService openstackService;
+    private final ScalableGatewayService gatewayService;
     private final OpenstackNetworkingConfig config;
     private static final MacAddress GATEWAY_MAC = MacAddress.valueOf("1f:1f:1f:1f:1f:1f");
     private static final String NETWORK_ROUTER_INTERFACE = "network:router_interface";
     private static final String PORTNAME = "portName";
     private static final String NETWORK_ROUTER_GATEWAY = "network:router_gateway";
     private static final String NETWORK_FLOATING_IP = "network:floatingip";
-
+    private static final String EXTERNAL_NODE_NULL = "There is no external node about this deviceId []";
     /**
      * Default constructor.
      *
@@ -77,15 +79,17 @@ public class OpenstackIcmpHandler {
      * @param openstackService          openstackInterface service
      * @param config                    openstackRoutingConfig
      * @param openstackSwitchingService openstackSwitching service
+     * @param gatewayService scalable gateway service
      */
     OpenstackIcmpHandler(PacketService packetService, DeviceService deviceService,
                          OpenstackInterfaceService openstackService, OpenstackNetworkingConfig config,
-                         OpenstackSwitchingService openstackSwitchingService) {
+                         OpenstackSwitchingService openstackSwitchingService, ScalableGatewayService gatewayService) {
         this.packetService = packetService;
         this.deviceService = deviceService;
         this.openstackService = checkNotNull(openstackService);
         this.config = checkNotNull(config);
         this.openstackSwitchingService = checkNotNull(openstackSwitchingService);
+        this.gatewayService = gatewayService;
     }
 
     /**
@@ -99,10 +103,13 @@ public class OpenstackIcmpHandler {
                 .matchIPProtocol(IPv4.PROTOCOL_ICMP)
                 .build();
 
+        Map<DeviceId, PortNumber> externalInfoMap = getExternalInfo();
+
+        externalInfoMap.keySet().forEach(deviceId ->
                 packetService.requestPackets(icmpSelector,
                         PacketPriority.CONTROL,
                         appId,
-                Optional.of(DeviceId.deviceId(config.gatewayBridgeId())));
+                        Optional.of(deviceId)));
     }
 
     /**
@@ -223,14 +230,19 @@ public class OpenstackIcmpHandler {
         Ethernet icmpRequestEth = new Ethernet();
 
         icmpRequestEth.setEtherType(Ethernet.TYPE_IPV4)
-                .setSourceMACAddress(MacAddress.valueOf(config.gatewayExternalInterfaceMac()))
-                .setDestinationMACAddress(MacAddress.valueOf(config.physicalRouterMac()))
                 .setPayload(icmpRequestIpv4);
 
-        TrafficTreatment treatment = DefaultTrafficTreatment.builder()
-                .setOutput(getPortForAnnotationPortName(DeviceId.deviceId(config.gatewayBridgeId()),
-                        config.gatewayExternalInterfaceName()))
-                .build();
+        TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
+
+        Map<DeviceId, PortNumber> externalInforMap = getExternalInfo();
+
+        if (externalInforMap.size() == 0 || !externalInforMap.containsKey(deviceId)) {
+            log.error(EXTERNAL_NODE_NULL, deviceId.toString());
+            return;
+        }
+        tBuilder.setOutput(externalInforMap.get(deviceId));
+
+        TrafficTreatment treatment = tBuilder.build();
 
         OutboundPacket packet = new DefaultOutboundPacket(deviceId,
                 treatment, ByteBuffer.wrap(icmpRequestEth.serialize()));
@@ -323,4 +335,10 @@ public class OpenstackIcmpHandler {
         }
         return true;
     }
+    private Map<DeviceId, PortNumber> getExternalInfo() {
+        Map<DeviceId, PortNumber> externalInfoMap = Maps.newHashMap();
+        gatewayService.getGatewayDeviceIds().forEach(deviceId ->
+                externalInfoMap.putIfAbsent(deviceId, gatewayService.getGatewayExternalPorts(deviceId).get(0)));
+        return externalInfoMap;
+    }
 }

@@ -30,10 +30,11 @@ import org.onosproject.net.packet.InboundPacket;
 import org.onosproject.net.packet.PacketContext;
 import org.onosproject.net.packet.PacketService;
 import org.onosproject.openstackinterface.OpenstackInterfaceService;
-import org.onosproject.openstackinterface.OpenstackNetwork;
 import org.onosproject.openstackinterface.OpenstackPort;
 import org.onosproject.openstackinterface.OpenstackRouter;
 import org.onosproject.openstacknetworking.OpenstackNetworkingConfig;
+import org.onosproject.scalablegateway.api.GatewayNode;
+import org.onosproject.scalablegateway.api.ScalableGatewayService;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -60,6 +61,7 @@ public class OpenstackPnatHandler implements Runnable {
     private OpenstackNetworkingConfig config;
 
     private static final String DEVICE_OWNER_ROUTER_INTERFACE = "network:router_interface";
+    private static final String EXTERNAL_PORT_NULL = "There is no external port in this deviceId []";
 
     OpenstackPnatHandler(OpenstackRoutingRulePopulator rulePopulator, PacketContext context,
                          int portNum, OpenstackPort openstackPort, Port port, OpenstackNetworkingConfig config) {
@@ -84,16 +86,17 @@ public class OpenstackPnatHandler implements Runnable {
 
         OpenstackRouter router = getOpenstackRouter(openstackPort);
 
+        MacAddress externalMac = MacAddress.NONE;
+        MacAddress routerMac = MacAddress.NONE;
+
         rulePopulator.populatePnatFlowRules(inboundPacket, openstackPort, portNum,
-                getExternalIp(router), MacAddress.valueOf(config.gatewayExternalInterfaceMac()),
-                MacAddress.valueOf(config.physicalRouterMac()));
+                getExternalIp(router), externalMac, routerMac);
 
         packetOut((Ethernet) ethernet.clone(), inboundPacket.receivedFrom().deviceId(), portNum, router);
     }
 
     private OpenstackRouter getOpenstackRouter(OpenstackPort openstackPort) {
         OpenstackInterfaceService networkingService = getService(OpenstackInterfaceService.class);
-        OpenstackNetwork network = networkingService.network(openstackPort.networkId());
 
         OpenstackPort port = networkingService.ports()
                 .stream()
@@ -146,11 +149,16 @@ public class OpenstackPnatHandler implements Runnable {
         iPacket.resetChecksum();
         iPacket.setParent(ethernet);
         ethernet.setPayload(iPacket);
-        ethernet.setSourceMACAddress(config.gatewayExternalInterfaceMac())
-                .setDestinationMACAddress(config.physicalRouterMac());
+        ScalableGatewayService gatewayService = getService(ScalableGatewayService.class);
+        GatewayNode gatewayNode = gatewayService.getGatewayNode(deviceId);
+        if (gatewayNode.getGatewayExternalInterfaceNames().size() == 0) {
+            log.error(EXTERNAL_PORT_NULL, deviceId.toString());
+            return;
+        }
+        treatment.setOutput(gatewayService.getGatewayExternalPorts(deviceId).get(0));
+
         ethernet.resetChecksum();
 
-        treatment.setOutput(port.number());
 
         packetService.emit(new DefaultOutboundPacket(deviceId, treatment.build(),
                 ByteBuffer.wrap(ethernet.serialize())));

@@ -15,6 +15,7 @@
  */
 package org.onosproject.openstacknetworking.routing;
 
+import com.google.common.collect.Lists;
 import org.onlab.packet.ARP;
 import org.onlab.packet.EthType;
 import org.onlab.packet.Ethernet;
@@ -34,9 +35,11 @@ import org.onosproject.net.packet.PacketService;
 import org.onosproject.openstackinterface.OpenstackInterfaceService;
 import org.onosproject.openstackinterface.OpenstackPort;
 import org.onosproject.openstacknetworking.OpenstackNetworkingConfig;
+import org.onosproject.scalablegateway.api.ScalableGatewayService;
 import org.slf4j.Logger;
 
 import java.nio.ByteBuffer;
+import java.util.List;
 import java.util.Optional;
 
 import static com.google.common.base.Preconditions.checkNotNull;
@@ -51,21 +54,23 @@ public class OpenstackRoutingArpHandler {
     private final PacketService packetService;
     private final OpenstackInterfaceService openstackService;
     private final OpenstackNetworkingConfig config;
+    private final ScalableGatewayService gatewayService;
     private static final String NETWORK_ROUTER_GATEWAY = "network:router_gateway";
     private static final String NETWORK_FLOATING_IP = "network:floatingip";
 
     /**
      * Default constructor.
-     *
      *  @param packetService packet service
      * @param openstackService openstackInterface service
      * @param config openstackRoutingConfig
+     * @param gatewayService
      */
     OpenstackRoutingArpHandler(PacketService packetService, OpenstackInterfaceService openstackService,
-                               OpenstackNetworkingConfig config) {
+                               OpenstackNetworkingConfig config, ScalableGatewayService gatewayService) {
         this.packetService = packetService;
         this.openstackService = checkNotNull(openstackService);
         this.config = checkNotNull(config);
+        this.gatewayService = gatewayService;
     }
 
     /**
@@ -79,10 +84,12 @@ public class OpenstackRoutingArpHandler {
                 .matchEthType(EthType.EtherType.ARP.ethType().toShort())
                 .build();
 
+        getExternalInfo().forEach(deviceId ->
                 packetService.requestPackets(arpSelector,
                         PacketPriority.CONTROL,
                         appId,
-                Optional.of(DeviceId.deviceId(config.gatewayBridgeId())));
+                        Optional.of(deviceId))
+        );
     }
 
     /**
@@ -138,4 +145,10 @@ public class OpenstackRoutingArpHandler {
         }
         return port.macAddress();
     }
+
+    private List<DeviceId> getExternalInfo() {
+        List<DeviceId> externalInfoList = Lists.newArrayList();
+        gatewayService.getGatewayDeviceIds().forEach(externalInfoList::add);
+        return externalInfoList;
+    }
 }

@@ -56,6 +56,7 @@ import org.onosproject.openstacknetworking.OpenstackPortInfo;
 import org.onosproject.openstacknetworking.OpenstackRoutingService;
 import org.onosproject.openstacknetworking.OpenstackSubjectFactories;
 import org.onosproject.openstacknetworking.OpenstackSwitchingService;
+import org.onosproject.scalablegateway.api.ScalableGatewayService;
 import org.onosproject.store.serializers.KryoNamespaces;
 import org.onosproject.store.service.ConsistentMap;
 import org.onosproject.store.service.Serializer;
@@ -113,6 +114,8 @@ public class OpenstackRoutingManager implements OpenstackRoutingService {
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
     protected StorageService storageService;
 
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected ScalableGatewayService gatewayService;
 
     private ApplicationId appId;
     private ConsistentMap<Integer, String> tpPortNumMap; // Map<PortNum, allocated VM`s Mac & destionation Ip address>
@@ -444,8 +447,7 @@ public class OpenstackRoutingManager implements OpenstackRoutingService {
 
             if (context.isHandled()) {
                 return;
-            } else if (!context.inPacket().receivedFrom().deviceId().toString()
-                    .equals(config.gatewayBridgeId())) {
+            } else if (!checkGatewayNode(context.inPacket().receivedFrom().deviceId())) {
                 return;
             }
 
@@ -472,16 +474,19 @@ public class OpenstackRoutingManager implements OpenstackRoutingService {
                         }
                     default:
                         int portNum = getPortNum(ethernet.getSourceMAC(), iPacket.getDestinationAddress());
-                        Optional<Port> port =
-                                getExternalPort(pkt.receivedFrom().deviceId(), config.gatewayExternalInterfaceName());
-                        if (port.isPresent()) {
+                        DeviceId deviceId = pkt.receivedFrom().deviceId();
+                        Port port = null;
+                        port = deviceService.getPort(deviceId,
+                                gatewayService.getGatewayExternalPorts(deviceId).get(0));
+                        if (port != null) {
                             OpenstackPort openstackPort = getOpenstackPort(ethernet.getSourceMAC(),
                                     Ip4Address.valueOf(iPacket.getSourceAddress()));
                             l3EventExecutorService.execute(new OpenstackPnatHandler(rulePopulator, context,
-                                    portNum, openstackPort, port.get(), config));
+                                    portNum, openstackPort, port, config));
                         } else {
                             log.warn("There`s no external interface");
                         }
+
                         break;
                 }
             } else if (ethernet.getEtherType() == Ethernet.TYPE_ARP) {
@@ -511,6 +516,10 @@ public class OpenstackRoutingManager implements OpenstackRoutingService {
 
     }
 
+    private boolean checkGatewayNode(DeviceId deviceId) {
+        return gatewayService.getGatewayDeviceIds().contains(deviceId);
+    }
+
     private void clearPortNumMap() {
         tpPortNumMap.entrySet().forEach(e -> {
             if (System.currentTimeMillis() - e.getValue().creationTime() > PNAT_PORT_EXPIRE_TIME) {
@@ -600,21 +609,12 @@ public class OpenstackRoutingManager implements OpenstackRoutingService {
             return;
         }
 
-        checkNotNull(config.physicalRouterMac());
-        checkNotNull(config.gatewayBridgeId());
-        checkNotNull(config.gatewayExternalInterfaceMac());
-        checkNotNull(config.gatewayExternalInterfaceName());
-
-        log.warn("Configured info: {}, {}, {}, {}", config.physicalRouterMac(), config.gatewayBridgeId(),
-                config.gatewayExternalInterfaceMac(), config.gatewayExternalInterfaceName());
-
-        rulePopulator = new OpenstackRoutingRulePopulator(appId,
-                openstackService, flowObjectiveService, deviceService, driverService, config);
-
+        rulePopulator = new OpenstackRoutingRulePopulator(appId, openstackService, flowObjectiveService,
+                deviceService, driverService, config, gatewayService);
         openstackIcmpHandler = new OpenstackIcmpHandler(packetService, deviceService,
-                openstackService, config, openstackSwitchingService);
-        openstackArpHandler = new OpenstackRoutingArpHandler(packetService, openstackService, config);
-
+                openstackService, config, openstackSwitchingService, gatewayService);
+        openstackArpHandler = new OpenstackRoutingArpHandler(packetService, openstackService,
+                config, gatewayService);
         openstackIcmpHandler.requestPacket(appId);
         openstackArpHandler.requestPacket(appId);
 

+{
+    "apps" : {
+        "org.onosproject.scalablegateway" : {
+            "scalablegateway" : {
+              "nodes" : [
+              {
+                "bridgeId" : "of:0000000000000003",
+                "dataPlaneIp" : "10.62.148.17",
+                "gatewayExternalInterfaceName" : "veth0"
+                }]
+            }
+        }
+    }
+}

@@ -66,6 +66,15 @@ public final class GatewayNode {
     }
 
     /**
+     * Returns GatewayNode builder object.
+     *
+     * @return GatewayNode builder
+     */
+    public static GatewayNode.Builder builder() {
+        return new Builder();
+    }
+
+    /**
      * GatewayNode Builder class.
      */
     public static final class Builder {

@@ -17,6 +17,7 @@ package org.onosproject.scalablegateway.api;
 
 
 import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
 import org.onlab.packet.Ip4Address;
@@ -28,9 +29,10 @@ import org.slf4j.Logger;
 import java.util.Collections;
 import java.util.List;
 import java.util.Set;
+import java.util.stream.StreamSupport;
 
-import static org.slf4j.LoggerFactory.getLogger;
 import static org.onosproject.net.config.Config.FieldPresence.MANDATORY;
+import static org.slf4j.LoggerFactory.getLogger;
 
 /**
  * Configuration object for OpensatckNode service.
@@ -79,9 +81,21 @@ public class GatewayNodeConfig extends Config<ApplicationId> {
 
     @Override
     public boolean isValid() {
-        return hasOnlyFields(NODES, BRIDGE_ID, DATAPLANE_IP, EXTERNAL_INTERFACE_NAME) &&
-                isIpAddress(DATAPLANE_IP, MANDATORY) && isString(BRIDGE_ID, MANDATORY) &&
-                isString(EXTERNAL_INTERFACE_NAME, MANDATORY);
+        JsonNode jsonNodes = object.get(NODES);
+
+        if (jsonNodes == null) {
+            return false;
+        }
+
+        return hasOnlyFields(NODES)
+                && StreamSupport.stream(jsonNodes.spliterator(), false).allMatch(this::checkValid);
+    }
+
+    private boolean checkValid(JsonNode jsonNode) {
+        ObjectNode objectNode = (ObjectNode) jsonNode;
+        return isString(objectNode, BRIDGE_ID, MANDATORY)
+                && isIpAddress(objectNode, DATAPLANE_IP, MANDATORY)
+                && isString(objectNode, EXTERNAL_INTERFACE_NAME, MANDATORY);
     }
 
 }