홍길동 / onos

Go to a project
Toggle navigation Toggle navigation pinning
Kyuhwi Choi
Committed by Gerrit Code Review
92d9ea44 1 parent b4bf7c89

[ONOS-4481] Integrate scalable gateway to openstackRouting 

 - Modified openstack_routing to use scalable gateway application
 - Add GatewayNode Builder method

Change-Id: Ib12d52009eeb89ab59f01cafe794894aac80985e
Inline Side-by-side
Showing 10 changed files with 265 additions and 147 deletions
...@@ -3,6 +3,7 @@ COMPILE_DEPS = [ ...@@ -3,6 +3,7 @@ COMPILE_DEPS = [
3 '//core/store/serializers:onos-core-serializers', 3 '//core/store/serializers:onos-core-serializers',
4 '//apps/openstackinterface/api:onos-apps-openstackinterface-api', 4 '//apps/openstackinterface/api:onos-apps-openstackinterface-api',
5 '//apps/openstacknetworking/api:onos-apps-openstacknetworking-api', 5 '//apps/openstacknetworking/api:onos-apps-openstacknetworking-api',
6 + '//apps/scalablegateway:onos-apps-scalablegateway',
6 ] 7 ]
7 8
8 BUNDLES = [ 9 BUNDLES = [
......
...@@ -52,6 +52,11 @@ ...@@ -52,6 +52,11 @@
52 <artifactId>onos-app-openstackinterface-api</artifactId> 52 <artifactId>onos-app-openstackinterface-api</artifactId>
53 <version>${project.version}</version> 53 <version>${project.version}</version>
54 </dependency> 54 </dependency>
55 + <dependency>
56 + <groupId>org.onosproject</groupId>
57 + <artifactId>onos-scalablegateway</artifactId>
58 + <version>${project.version}</version>
59 + </dependency>
55 </dependencies> 60 </dependencies>
56 </project> 61 </project>
57 62
......
...@@ -40,6 +40,7 @@ import org.onosproject.openstackinterface.OpenstackPort; ...@@ -40,6 +40,7 @@ import org.onosproject.openstackinterface.OpenstackPort;
40 import org.onosproject.openstacknetworking.OpenstackNetworkingConfig; 40 import org.onosproject.openstacknetworking.OpenstackNetworkingConfig;
41 import org.onosproject.openstacknetworking.OpenstackPortInfo; 41 import org.onosproject.openstacknetworking.OpenstackPortInfo;
42 import org.onosproject.openstacknetworking.OpenstackSwitchingService; 42 import org.onosproject.openstacknetworking.OpenstackSwitchingService;
43 +import org.onosproject.scalablegateway.api.ScalableGatewayService;
43 import org.slf4j.Logger; 44 import org.slf4j.Logger;
44 45
45 import java.nio.ByteBuffer; 46 import java.nio.ByteBuffer;
...@@ -62,13 +63,14 @@ public class OpenstackIcmpHandler { ...@@ -62,13 +63,14 @@ public class OpenstackIcmpHandler {
62 private final Map<String, OpenstackPortInfo> icmpInfoMap = Maps.newHashMap(); 63 private final Map<String, OpenstackPortInfo> icmpInfoMap = Maps.newHashMap();
63 private final OpenstackSwitchingService openstackSwitchingService; 64 private final OpenstackSwitchingService openstackSwitchingService;
64 private final OpenstackInterfaceService openstackService; 65 private final OpenstackInterfaceService openstackService;
66 + private final ScalableGatewayService gatewayService;
65 private final OpenstackNetworkingConfig config; 67 private final OpenstackNetworkingConfig config;
66 private static final MacAddress GATEWAY_MAC = MacAddress.valueOf("1f:1f:1f:1f:1f:1f"); 68 private static final MacAddress GATEWAY_MAC = MacAddress.valueOf("1f:1f:1f:1f:1f:1f");
67 private static final String NETWORK_ROUTER_INTERFACE = "network:router_interface"; 69 private static final String NETWORK_ROUTER_INTERFACE = "network:router_interface";
68 private static final String PORTNAME = "portName"; 70 private static final String PORTNAME = "portName";
69 private static final String NETWORK_ROUTER_GATEWAY = "network:router_gateway"; 71 private static final String NETWORK_ROUTER_GATEWAY = "network:router_gateway";
70 private static final String NETWORK_FLOATING_IP = "network:floatingip"; 72 private static final String NETWORK_FLOATING_IP = "network:floatingip";
71 - 73 + private static final String EXTERNAL_NODE_NULL = "There is no external node about this deviceId []";
72 /** 74 /**
73 * Default constructor. 75 * Default constructor.
74 * 76 *
...@@ -77,15 +79,17 @@ public class OpenstackIcmpHandler { ...@@ -77,15 +79,17 @@ public class OpenstackIcmpHandler {
77 * @param openstackService openstackInterface service 79 * @param openstackService openstackInterface service
78 * @param config openstackRoutingConfig 80 * @param config openstackRoutingConfig
79 * @param openstackSwitchingService openstackSwitching service 81 * @param openstackSwitchingService openstackSwitching service
82 + * @param gatewayService scalable gateway service
80 */ 83 */
81 OpenstackIcmpHandler(PacketService packetService, DeviceService deviceService, 84 OpenstackIcmpHandler(PacketService packetService, DeviceService deviceService,
82 OpenstackInterfaceService openstackService, OpenstackNetworkingConfig config, 85 OpenstackInterfaceService openstackService, OpenstackNetworkingConfig config,
83 - OpenstackSwitchingService openstackSwitchingService) { 86 + OpenstackSwitchingService openstackSwitchingService, ScalableGatewayService gatewayService) {
84 this.packetService = packetService; 87 this.packetService = packetService;
85 this.deviceService = deviceService; 88 this.deviceService = deviceService;
86 this.openstackService = checkNotNull(openstackService); 89 this.openstackService = checkNotNull(openstackService);
87 this.config = checkNotNull(config); 90 this.config = checkNotNull(config);
88 this.openstackSwitchingService = checkNotNull(openstackSwitchingService); 91 this.openstackSwitchingService = checkNotNull(openstackSwitchingService);
92 + this.gatewayService = gatewayService;
89 } 93 }
90 94
91 /** 95 /**
...@@ -99,10 +103,13 @@ public class OpenstackIcmpHandler { ...@@ -99,10 +103,13 @@ public class OpenstackIcmpHandler {
99 .matchIPProtocol(IPv4.PROTOCOL_ICMP) 103 .matchIPProtocol(IPv4.PROTOCOL_ICMP)
100 .build(); 104 .build();
101 105
102 - packetService.requestPackets(icmpSelector, 106 + Map<DeviceId, PortNumber> externalInfoMap = getExternalInfo();
103 - PacketPriority.CONTROL, 107 +
104 - appId, 108 + externalInfoMap.keySet().forEach(deviceId ->
105 - Optional.of(DeviceId.deviceId(config.gatewayBridgeId()))); 109 + packetService.requestPackets(icmpSelector,
110 + PacketPriority.CONTROL,
111 + appId,
112 + Optional.of(deviceId)));
106 } 113 }
107 114
108 /** 115 /**
...@@ -223,14 +230,19 @@ public class OpenstackIcmpHandler { ...@@ -223,14 +230,19 @@ public class OpenstackIcmpHandler {
223 Ethernet icmpRequestEth = new Ethernet(); 230 Ethernet icmpRequestEth = new Ethernet();
224 231
225 icmpRequestEth.setEtherType(Ethernet.TYPE_IPV4) 232 icmpRequestEth.setEtherType(Ethernet.TYPE_IPV4)
226 - .setSourceMACAddress(MacAddress.valueOf(config.gatewayExternalInterfaceMac()))
227 - .setDestinationMACAddress(MacAddress.valueOf(config.physicalRouterMac()))
228 .setPayload(icmpRequestIpv4); 233 .setPayload(icmpRequestIpv4);
229 234
230 - TrafficTreatment treatment = DefaultTrafficTreatment.builder() 235 + TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
231 - .setOutput(getPortForAnnotationPortName(DeviceId.deviceId(config.gatewayBridgeId()), 236 +
232 - config.gatewayExternalInterfaceName())) 237 + Map<DeviceId, PortNumber> externalInforMap = getExternalInfo();
233 - .build(); 238 +
239 + if (externalInforMap.size() == 0 || !externalInforMap.containsKey(deviceId)) {
240 + log.error(EXTERNAL_NODE_NULL, deviceId.toString());
241 + return;
242 + }
243 + tBuilder.setOutput(externalInforMap.get(deviceId));
244 +
245 + TrafficTreatment treatment = tBuilder.build();
234 246
235 OutboundPacket packet = new DefaultOutboundPacket(deviceId, 247 OutboundPacket packet = new DefaultOutboundPacket(deviceId,
236 treatment, ByteBuffer.wrap(icmpRequestEth.serialize())); 248 treatment, ByteBuffer.wrap(icmpRequestEth.serialize()));
...@@ -323,4 +335,10 @@ public class OpenstackIcmpHandler { ...@@ -323,4 +335,10 @@ public class OpenstackIcmpHandler {
323 } 335 }
324 return true; 336 return true;
325 } 337 }
326 -}
...\ No newline at end of file ...\ No newline at end of file
338 + private Map<DeviceId, PortNumber> getExternalInfo() {
339 + Map<DeviceId, PortNumber> externalInfoMap = Maps.newHashMap();
340 + gatewayService.getGatewayDeviceIds().forEach(deviceId ->
341 + externalInfoMap.putIfAbsent(deviceId, gatewayService.getGatewayExternalPorts(deviceId).get(0)));
342 + return externalInfoMap;
343 + }
344 +}
......
...@@ -30,10 +30,11 @@ import org.onosproject.net.packet.InboundPacket; ...@@ -30,10 +30,11 @@ import org.onosproject.net.packet.InboundPacket;
30 import org.onosproject.net.packet.PacketContext; 30 import org.onosproject.net.packet.PacketContext;
31 import org.onosproject.net.packet.PacketService; 31 import org.onosproject.net.packet.PacketService;
32 import org.onosproject.openstackinterface.OpenstackInterfaceService; 32 import org.onosproject.openstackinterface.OpenstackInterfaceService;
33 -import org.onosproject.openstackinterface.OpenstackNetwork;
34 import org.onosproject.openstackinterface.OpenstackPort; 33 import org.onosproject.openstackinterface.OpenstackPort;
35 import org.onosproject.openstackinterface.OpenstackRouter; 34 import org.onosproject.openstackinterface.OpenstackRouter;
36 import org.onosproject.openstacknetworking.OpenstackNetworkingConfig; 35 import org.onosproject.openstacknetworking.OpenstackNetworkingConfig;
36 +import org.onosproject.scalablegateway.api.GatewayNode;
37 +import org.onosproject.scalablegateway.api.ScalableGatewayService;
37 import org.slf4j.Logger; 38 import org.slf4j.Logger;
38 import org.slf4j.LoggerFactory; 39 import org.slf4j.LoggerFactory;
39 40
...@@ -60,6 +61,7 @@ public class OpenstackPnatHandler implements Runnable { ...@@ -60,6 +61,7 @@ public class OpenstackPnatHandler implements Runnable {
60 private OpenstackNetworkingConfig config; 61 private OpenstackNetworkingConfig config;
61 62
62 private static final String DEVICE_OWNER_ROUTER_INTERFACE = "network:router_interface"; 63 private static final String DEVICE_OWNER_ROUTER_INTERFACE = "network:router_interface";
64 + private static final String EXTERNAL_PORT_NULL = "There is no external port in this deviceId []";
63 65
64 OpenstackPnatHandler(OpenstackRoutingRulePopulator rulePopulator, PacketContext context, 66 OpenstackPnatHandler(OpenstackRoutingRulePopulator rulePopulator, PacketContext context,
65 int portNum, OpenstackPort openstackPort, Port port, OpenstackNetworkingConfig config) { 67 int portNum, OpenstackPort openstackPort, Port port, OpenstackNetworkingConfig config) {
...@@ -84,16 +86,17 @@ public class OpenstackPnatHandler implements Runnable { ...@@ -84,16 +86,17 @@ public class OpenstackPnatHandler implements Runnable {
84 86
85 OpenstackRouter router = getOpenstackRouter(openstackPort); 87 OpenstackRouter router = getOpenstackRouter(openstackPort);
86 88
89 + MacAddress externalMac = MacAddress.NONE;
90 + MacAddress routerMac = MacAddress.NONE;
91 +
87 rulePopulator.populatePnatFlowRules(inboundPacket, openstackPort, portNum, 92 rulePopulator.populatePnatFlowRules(inboundPacket, openstackPort, portNum,
88 - getExternalIp(router), MacAddress.valueOf(config.gatewayExternalInterfaceMac()), 93 + getExternalIp(router), externalMac, routerMac);
89 - MacAddress.valueOf(config.physicalRouterMac()));
90 94
91 packetOut((Ethernet) ethernet.clone(), inboundPacket.receivedFrom().deviceId(), portNum, router); 95 packetOut((Ethernet) ethernet.clone(), inboundPacket.receivedFrom().deviceId(), portNum, router);
92 } 96 }
93 97
94 private OpenstackRouter getOpenstackRouter(OpenstackPort openstackPort) { 98 private OpenstackRouter getOpenstackRouter(OpenstackPort openstackPort) {
95 OpenstackInterfaceService networkingService = getService(OpenstackInterfaceService.class); 99 OpenstackInterfaceService networkingService = getService(OpenstackInterfaceService.class);
96 - OpenstackNetwork network = networkingService.network(openstackPort.networkId());
97 100
98 OpenstackPort port = networkingService.ports() 101 OpenstackPort port = networkingService.ports()
99 .stream() 102 .stream()
...@@ -146,11 +149,16 @@ public class OpenstackPnatHandler implements Runnable { ...@@ -146,11 +149,16 @@ public class OpenstackPnatHandler implements Runnable {
146 iPacket.resetChecksum(); 149 iPacket.resetChecksum();
147 iPacket.setParent(ethernet); 150 iPacket.setParent(ethernet);
148 ethernet.setPayload(iPacket); 151 ethernet.setPayload(iPacket);
149 - ethernet.setSourceMACAddress(config.gatewayExternalInterfaceMac()) 152 + ScalableGatewayService gatewayService = getService(ScalableGatewayService.class);
150 - .setDestinationMACAddress(config.physicalRouterMac()); 153 + GatewayNode gatewayNode = gatewayService.getGatewayNode(deviceId);
154 + if (gatewayNode.getGatewayExternalInterfaceNames().size() == 0) {
155 + log.error(EXTERNAL_PORT_NULL, deviceId.toString());
156 + return;
157 + }
158 + treatment.setOutput(gatewayService.getGatewayExternalPorts(deviceId).get(0));
159 +
151 ethernet.resetChecksum(); 160 ethernet.resetChecksum();
152 161
153 - treatment.setOutput(port.number());
154 162
155 packetService.emit(new DefaultOutboundPacket(deviceId, treatment.build(), 163 packetService.emit(new DefaultOutboundPacket(deviceId, treatment.build(),
156 ByteBuffer.wrap(ethernet.serialize()))); 164 ByteBuffer.wrap(ethernet.serialize())));
......
...@@ -15,6 +15,7 @@ ...@@ -15,6 +15,7 @@
15 */ 15 */
16 package org.onosproject.openstacknetworking.routing; 16 package org.onosproject.openstacknetworking.routing;
17 17
18 +import com.google.common.collect.Lists;
18 import org.onlab.packet.ARP; 19 import org.onlab.packet.ARP;
19 import org.onlab.packet.EthType; 20 import org.onlab.packet.EthType;
20 import org.onlab.packet.Ethernet; 21 import org.onlab.packet.Ethernet;
...@@ -34,9 +35,11 @@ import org.onosproject.net.packet.PacketService; ...@@ -34,9 +35,11 @@ import org.onosproject.net.packet.PacketService;
34 import org.onosproject.openstackinterface.OpenstackInterfaceService; 35 import org.onosproject.openstackinterface.OpenstackInterfaceService;
35 import org.onosproject.openstackinterface.OpenstackPort; 36 import org.onosproject.openstackinterface.OpenstackPort;
36 import org.onosproject.openstacknetworking.OpenstackNetworkingConfig; 37 import org.onosproject.openstacknetworking.OpenstackNetworkingConfig;
38 +import org.onosproject.scalablegateway.api.ScalableGatewayService;
37 import org.slf4j.Logger; 39 import org.slf4j.Logger;
38 40
39 import java.nio.ByteBuffer; 41 import java.nio.ByteBuffer;
42 +import java.util.List;
40 import java.util.Optional; 43 import java.util.Optional;
41 44
42 import static com.google.common.base.Preconditions.checkNotNull; 45 import static com.google.common.base.Preconditions.checkNotNull;
...@@ -51,21 +54,23 @@ public class OpenstackRoutingArpHandler { ...@@ -51,21 +54,23 @@ public class OpenstackRoutingArpHandler {
51 private final PacketService packetService; 54 private final PacketService packetService;
52 private final OpenstackInterfaceService openstackService; 55 private final OpenstackInterfaceService openstackService;
53 private final OpenstackNetworkingConfig config; 56 private final OpenstackNetworkingConfig config;
57 + private final ScalableGatewayService gatewayService;
54 private static final String NETWORK_ROUTER_GATEWAY = "network:router_gateway"; 58 private static final String NETWORK_ROUTER_GATEWAY = "network:router_gateway";
55 private static final String NETWORK_FLOATING_IP = "network:floatingip"; 59 private static final String NETWORK_FLOATING_IP = "network:floatingip";
56 60
57 /** 61 /**
58 * Default constructor. 62 * Default constructor.
59 - * 63 + * @param packetService packet service
60 - * @param packetService packet service
61 * @param openstackService openstackInterface service 64 * @param openstackService openstackInterface service
62 * @param config openstackRoutingConfig 65 * @param config openstackRoutingConfig
66 + * @param gatewayService
63 */ 67 */
64 OpenstackRoutingArpHandler(PacketService packetService, OpenstackInterfaceService openstackService, 68 OpenstackRoutingArpHandler(PacketService packetService, OpenstackInterfaceService openstackService,
65 - OpenstackNetworkingConfig config) { 69 + OpenstackNetworkingConfig config, ScalableGatewayService gatewayService) {
66 this.packetService = packetService; 70 this.packetService = packetService;
67 this.openstackService = checkNotNull(openstackService); 71 this.openstackService = checkNotNull(openstackService);
68 this.config = checkNotNull(config); 72 this.config = checkNotNull(config);
73 + this.gatewayService = gatewayService;
69 } 74 }
70 75
71 /** 76 /**
...@@ -79,10 +84,12 @@ public class OpenstackRoutingArpHandler { ...@@ -79,10 +84,12 @@ public class OpenstackRoutingArpHandler {
79 .matchEthType(EthType.EtherType.ARP.ethType().toShort()) 84 .matchEthType(EthType.EtherType.ARP.ethType().toShort())
80 .build(); 85 .build();
81 86
82 - packetService.requestPackets(arpSelector, 87 + getExternalInfo().forEach(deviceId ->
83 - PacketPriority.CONTROL, 88 + packetService.requestPackets(arpSelector,
84 - appId, 89 + PacketPriority.CONTROL,
85 - Optional.of(DeviceId.deviceId(config.gatewayBridgeId()))); 90 + appId,
91 + Optional.of(deviceId))
92 + );
86 } 93 }
87 94
88 /** 95 /**
...@@ -138,4 +145,10 @@ public class OpenstackRoutingArpHandler { ...@@ -138,4 +145,10 @@ public class OpenstackRoutingArpHandler {
138 } 145 }
139 return port.macAddress(); 146 return port.macAddress();
140 } 147 }
148 +
149 + private List<DeviceId> getExternalInfo() {
150 + List<DeviceId> externalInfoList = Lists.newArrayList();
151 + gatewayService.getGatewayDeviceIds().forEach(externalInfoList::add);
152 + return externalInfoList;
153 + }
141 } 154 }
......
...@@ -56,6 +56,7 @@ import org.onosproject.openstacknetworking.OpenstackPortInfo; ...@@ -56,6 +56,7 @@ import org.onosproject.openstacknetworking.OpenstackPortInfo;
56 import org.onosproject.openstacknetworking.OpenstackRoutingService; 56 import org.onosproject.openstacknetworking.OpenstackRoutingService;
57 import org.onosproject.openstacknetworking.OpenstackSubjectFactories; 57 import org.onosproject.openstacknetworking.OpenstackSubjectFactories;
58 import org.onosproject.openstacknetworking.OpenstackSwitchingService; 58 import org.onosproject.openstacknetworking.OpenstackSwitchingService;
59 +import org.onosproject.scalablegateway.api.ScalableGatewayService;
59 import org.onosproject.store.serializers.KryoNamespaces; 60 import org.onosproject.store.serializers.KryoNamespaces;
60 import org.onosproject.store.service.ConsistentMap; 61 import org.onosproject.store.service.ConsistentMap;
61 import org.onosproject.store.service.Serializer; 62 import org.onosproject.store.service.Serializer;
...@@ -113,6 +114,8 @@ public class OpenstackRoutingManager implements OpenstackRoutingService { ...@@ -113,6 +114,8 @@ public class OpenstackRoutingManager implements OpenstackRoutingService {
113 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY) 114 @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
114 protected StorageService storageService; 115 protected StorageService storageService;
115 116
117 + @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
118 + protected ScalableGatewayService gatewayService;
116 119
117 private ApplicationId appId; 120 private ApplicationId appId;
118 private ConsistentMap<Integer, String> tpPortNumMap; // Map<PortNum, allocated VM`s Mac & destionation Ip address> 121 private ConsistentMap<Integer, String> tpPortNumMap; // Map<PortNum, allocated VM`s Mac & destionation Ip address>
...@@ -444,8 +447,7 @@ public class OpenstackRoutingManager implements OpenstackRoutingService { ...@@ -444,8 +447,7 @@ public class OpenstackRoutingManager implements OpenstackRoutingService {
444 447
445 if (context.isHandled()) { 448 if (context.isHandled()) {
446 return; 449 return;
447 - } else if (!context.inPacket().receivedFrom().deviceId().toString() 450 + } else if (!checkGatewayNode(context.inPacket().receivedFrom().deviceId())) {
448 - .equals(config.gatewayBridgeId())) {
449 return; 451 return;
450 } 452 }
451 453
...@@ -472,16 +474,19 @@ public class OpenstackRoutingManager implements OpenstackRoutingService { ...@@ -472,16 +474,19 @@ public class OpenstackRoutingManager implements OpenstackRoutingService {
472 } 474 }
473 default: 475 default:
474 int portNum = getPortNum(ethernet.getSourceMAC(), iPacket.getDestinationAddress()); 476 int portNum = getPortNum(ethernet.getSourceMAC(), iPacket.getDestinationAddress());
475 - Optional<Port> port = 477 + DeviceId deviceId = pkt.receivedFrom().deviceId();
476 - getExternalPort(pkt.receivedFrom().deviceId(), config.gatewayExternalInterfaceName()); 478 + Port port = null;
477 - if (port.isPresent()) { 479 + port = deviceService.getPort(deviceId,
480 + gatewayService.getGatewayExternalPorts(deviceId).get(0));
481 + if (port != null) {
478 OpenstackPort openstackPort = getOpenstackPort(ethernet.getSourceMAC(), 482 OpenstackPort openstackPort = getOpenstackPort(ethernet.getSourceMAC(),
479 Ip4Address.valueOf(iPacket.getSourceAddress())); 483 Ip4Address.valueOf(iPacket.getSourceAddress()));
480 l3EventExecutorService.execute(new OpenstackPnatHandler(rulePopulator, context, 484 l3EventExecutorService.execute(new OpenstackPnatHandler(rulePopulator, context,
481 - portNum, openstackPort, port.get(), config)); 485 + portNum, openstackPort, port, config));
482 } else { 486 } else {
483 log.warn("There`s no external interface"); 487 log.warn("There`s no external interface");
484 } 488 }
489 +
485 break; 490 break;
486 } 491 }
487 } else if (ethernet.getEtherType() == Ethernet.TYPE_ARP) { 492 } else if (ethernet.getEtherType() == Ethernet.TYPE_ARP) {
...@@ -511,6 +516,10 @@ public class OpenstackRoutingManager implements OpenstackRoutingService { ...@@ -511,6 +516,10 @@ public class OpenstackRoutingManager implements OpenstackRoutingService {
511 516
512 } 517 }
513 518
519 + private boolean checkGatewayNode(DeviceId deviceId) {
520 + return gatewayService.getGatewayDeviceIds().contains(deviceId);
521 + }
522 +
514 private void clearPortNumMap() { 523 private void clearPortNumMap() {
515 tpPortNumMap.entrySet().forEach(e -> { 524 tpPortNumMap.entrySet().forEach(e -> {
516 if (System.currentTimeMillis() - e.getValue().creationTime() > PNAT_PORT_EXPIRE_TIME) { 525 if (System.currentTimeMillis() - e.getValue().creationTime() > PNAT_PORT_EXPIRE_TIME) {
...@@ -600,21 +609,12 @@ public class OpenstackRoutingManager implements OpenstackRoutingService { ...@@ -600,21 +609,12 @@ public class OpenstackRoutingManager implements OpenstackRoutingService {
600 return; 609 return;
601 } 610 }
602 611
603 - checkNotNull(config.physicalRouterMac()); 612 + rulePopulator = new OpenstackRoutingRulePopulator(appId, openstackService, flowObjectiveService,
604 - checkNotNull(config.gatewayBridgeId()); 613 + deviceService, driverService, config, gatewayService);
605 - checkNotNull(config.gatewayExternalInterfaceMac());
606 - checkNotNull(config.gatewayExternalInterfaceName());
607 -
608 - log.warn("Configured info: {}, {}, {}, {}", config.physicalRouterMac(), config.gatewayBridgeId(),
609 - config.gatewayExternalInterfaceMac(), config.gatewayExternalInterfaceName());
610 -
611 - rulePopulator = new OpenstackRoutingRulePopulator(appId,
612 - openstackService, flowObjectiveService, deviceService, driverService, config);
613 -
614 openstackIcmpHandler = new OpenstackIcmpHandler(packetService, deviceService, 614 openstackIcmpHandler = new OpenstackIcmpHandler(packetService, deviceService,
615 - openstackService, config, openstackSwitchingService); 615 + openstackService, config, openstackSwitchingService, gatewayService);
616 - openstackArpHandler = new OpenstackRoutingArpHandler(packetService, openstackService, config); 616 + openstackArpHandler = new OpenstackRoutingArpHandler(packetService, openstackService,
617 - 617 + config, gatewayService);
618 openstackIcmpHandler.requestPacket(appId); 618 openstackIcmpHandler.requestPacket(appId);
619 openstackArpHandler.requestPacket(appId); 619 openstackArpHandler.requestPacket(appId);
620 620
......
...@@ -15,6 +15,7 @@ ...@@ -15,6 +15,7 @@
15 */ 15 */
16 package org.onosproject.openstacknetworking.routing; 16 package org.onosproject.openstacknetworking.routing;
17 17
18 +import com.google.common.collect.Lists;
18 import org.onlab.packet.Ethernet; 19 import org.onlab.packet.Ethernet;
19 import org.onlab.packet.IPv4; 20 import org.onlab.packet.IPv4;
20 import org.onlab.packet.Ip4Address; 21 import org.onlab.packet.Ip4Address;
...@@ -25,6 +26,7 @@ import org.onlab.packet.TCP; ...@@ -25,6 +26,7 @@ import org.onlab.packet.TCP;
25 import org.onlab.packet.TpPort; 26 import org.onlab.packet.TpPort;
26 import org.onlab.packet.UDP; 27 import org.onlab.packet.UDP;
27 import org.onosproject.core.ApplicationId; 28 import org.onosproject.core.ApplicationId;
29 +import org.onosproject.core.GroupId;
28 import org.onosproject.net.Device; 30 import org.onosproject.net.Device;
29 import org.onosproject.net.DeviceId; 31 import org.onosproject.net.DeviceId;
30 import org.onosproject.net.Port; 32 import org.onosproject.net.Port;
...@@ -55,6 +57,7 @@ import org.onosproject.openstackinterface.OpenstackFloatingIP; ...@@ -55,6 +57,7 @@ import org.onosproject.openstackinterface.OpenstackFloatingIP;
55 import org.onosproject.openstacknetworking.OpenstackNetworkingConfig; 57 import org.onosproject.openstacknetworking.OpenstackNetworkingConfig;
56 import org.onosproject.openstacknetworking.OpenstackPortInfo; 58 import org.onosproject.openstacknetworking.OpenstackPortInfo;
57 import org.onosproject.openstacknetworking.OpenstackRoutingService; 59 import org.onosproject.openstacknetworking.OpenstackRoutingService;
60 +import org.onosproject.scalablegateway.api.ScalableGatewayService;
58 import org.slf4j.Logger; 61 import org.slf4j.Logger;
59 import org.slf4j.LoggerFactory; 62 import org.slf4j.LoggerFactory;
60 63
...@@ -77,6 +80,7 @@ public class OpenstackRoutingRulePopulator { ...@@ -77,6 +80,7 @@ public class OpenstackRoutingRulePopulator {
77 private final DeviceService deviceService; 80 private final DeviceService deviceService;
78 private final DriverService driverService; 81 private final DriverService driverService;
79 private final OpenstackNetworkingConfig config; 82 private final OpenstackNetworkingConfig config;
83 + private final ScalableGatewayService gatewayService;
80 84
81 private static final String PORTNAME_PREFIX_TUNNEL = "vxlan"; 85 private static final String PORTNAME_PREFIX_TUNNEL = "vxlan";
82 private static final String PORTNAME = "portName"; 86 private static final String PORTNAME = "portName";
...@@ -84,6 +88,7 @@ public class OpenstackRoutingRulePopulator { ...@@ -84,6 +88,7 @@ public class OpenstackRoutingRulePopulator {
84 88
85 private static final String PORTNOTNULL = "Port can not be null"; 89 private static final String PORTNOTNULL = "Port can not be null";
86 private static final String DEVICENOTNULL = "Device can not be null"; 90 private static final String DEVICENOTNULL = "Device can not be null";
91 + private static final String EXTPORTNOTNULL = "External port can not be null";
87 private static final String TUNNEL_DESTINATION = "tunnelDst"; 92 private static final String TUNNEL_DESTINATION = "tunnelDst";
88 private static final int ROUTING_RULE_PRIORITY = 25000; 93 private static final int ROUTING_RULE_PRIORITY = 25000;
89 private static final int FLOATING_RULE_PRIORITY = 42000; 94 private static final int FLOATING_RULE_PRIORITY = 42000;
...@@ -107,16 +112,19 @@ public class OpenstackRoutingRulePopulator { ...@@ -107,16 +112,19 @@ public class OpenstackRoutingRulePopulator {
107 * @param deviceService DeviceService 112 * @param deviceService DeviceService
108 * @param driverService DriverService 113 * @param driverService DriverService
109 * @param config Configuration for openstack environment 114 * @param config Configuration for openstack environment
115 + * @param gatewayService scalable gateway service
110 */ 116 */
111 public OpenstackRoutingRulePopulator(ApplicationId appId, OpenstackInterfaceService openstackService, 117 public OpenstackRoutingRulePopulator(ApplicationId appId, OpenstackInterfaceService openstackService,
112 FlowObjectiveService flowObjectiveService, DeviceService deviceService, 118 FlowObjectiveService flowObjectiveService, DeviceService deviceService,
113 - DriverService driverService, OpenstackNetworkingConfig config) { 119 + DriverService driverService, OpenstackNetworkingConfig config,
120 + ScalableGatewayService gatewayService) {
114 this.appId = appId; 121 this.appId = appId;
115 this.flowObjectiveService = flowObjectiveService; 122 this.flowObjectiveService = flowObjectiveService;
116 this.openstackService = checkNotNull(openstackService); 123 this.openstackService = checkNotNull(openstackService);
117 this.deviceService = deviceService; 124 this.deviceService = deviceService;
118 this.driverService = driverService; 125 this.driverService = driverService;
119 this.config = config; 126 this.config = config;
127 + this.gatewayService = gatewayService;
120 } 128 }
121 129
122 /** 130 /**
...@@ -155,9 +163,6 @@ public class OpenstackRoutingRulePopulator { ...@@ -155,9 +163,6 @@ public class OpenstackRoutingRulePopulator {
155 .matchIPDst(IpPrefix.valueOf(iPacket.getDestinationAddress(), 32)); 163 .matchIPDst(IpPrefix.valueOf(iPacket.getDestinationAddress(), 32));
156 164
157 TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder(); 165 TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
158 - tBuilder.setEthSrc(externalInterface)
159 - .setEthDst(externalRouter)
160 - .setIpSrc(externalIp);
161 166
162 switch (iPacket.getProtocol()) { 167 switch (iPacket.getProtocol()) {
163 case IPv4.PROTOCOL_TCP: 168 case IPv4.PROTOCOL_TCP:
...@@ -177,19 +182,20 @@ public class OpenstackRoutingRulePopulator { ...@@ -177,19 +182,20 @@ public class OpenstackRoutingRulePopulator {
177 break; 182 break;
178 } 183 }
179 184
180 - Port port = checkNotNull(getPortOfExternalInterface(), PORTNOTNULL); 185 + tBuilder.setIpSrc(externalIp);
181 - tBuilder.setOutput(port.number()); 186 + gatewayService.getGatewayNodes().forEach(node -> {
182 - 187 + tBuilder.setOutput(gatewayService.getGatewayExternalPorts(node.getGatewayDeviceId()).get(0));
183 - ForwardingObjective fo = DefaultForwardingObjective.builder() 188 + ForwardingObjective fo = DefaultForwardingObjective.builder()
184 - .withSelector(sBuilder.build()) 189 + .withSelector(sBuilder.build())
185 - .withTreatment(tBuilder.build()) 190 + .withTreatment(tBuilder.build())
186 - .withFlag(ForwardingObjective.Flag.VERSATILE) 191 + .withFlag(ForwardingObjective.Flag.VERSATILE)
187 - .withPriority(PNAT_RULE_PRIORITY) 192 + .withPriority(PNAT_RULE_PRIORITY)
188 - .makeTemporary(PNAT_TIMEOUT) 193 + .makeTemporary(PNAT_TIMEOUT)
189 - .fromApp(appId) 194 + .fromApp(appId)
190 - .add(); 195 + .add();
191 - 196 +
192 - flowObjectiveService.forward(inboundPacket.receivedFrom().deviceId(), fo); 197 + flowObjectiveService.forward(node.getGatewayDeviceId(), fo);
198 + });
193 199
194 } 200 }
195 201
...@@ -202,7 +208,6 @@ public class OpenstackRoutingRulePopulator { ...@@ -202,7 +208,6 @@ public class OpenstackRoutingRulePopulator {
202 208
203 private void populatePnatIncomingFlowRules(long vni, Ip4Address externalIp) { 209 private void populatePnatIncomingFlowRules(long vni, Ip4Address externalIp) {
204 IPv4 iPacket = (IPv4) inboundPacket.parsed().getPayload(); 210 IPv4 iPacket = (IPv4) inboundPacket.parsed().getPayload();
205 - DeviceId deviceId = inboundPacket.receivedFrom().deviceId();
206 211
207 TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder(); 212 TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
208 sBuilder.matchEthType(Ethernet.TYPE_IPV4) 213 sBuilder.matchEthType(Ethernet.TYPE_IPV4)
...@@ -232,19 +237,29 @@ public class OpenstackRoutingRulePopulator { ...@@ -232,19 +237,29 @@ public class OpenstackRoutingRulePopulator {
232 break; 237 break;
233 } 238 }
234 239
235 - tBuilder.extension(buildNiciraExtenstion(deviceId, getHostIpfromOpenstackPort(openstackPort)), deviceId) 240 + getGatewayNodeList().forEach(node -> {
236 - .setOutput(getTunnelPort(deviceId)); 241 + DeviceId deviceId = node.id();
237 - 242 + tBuilder.extension(buildNiciraExtenstion(deviceId, getHostIpfromOpenstackPort(openstackPort)), deviceId)
238 - ForwardingObjective fo = DefaultForwardingObjective.builder() 243 + .setOutput(getTunnelPort(deviceId));
239 - .withSelector(sBuilder.build()) 244 +
240 - .withTreatment(tBuilder.build()) 245 + ForwardingObjective fo = DefaultForwardingObjective.builder()
241 - .withFlag(ForwardingObjective.Flag.VERSATILE) 246 + .withSelector(sBuilder.build())
242 - .withPriority(PNAT_RULE_PRIORITY) 247 + .withTreatment(tBuilder.build())
243 - .makeTemporary(PNAT_TIMEOUT) 248 + .withFlag(ForwardingObjective.Flag.VERSATILE)
244 - .fromApp(appId) 249 + .withPriority(PNAT_RULE_PRIORITY)
245 - .add(); 250 + .makeTemporary(PNAT_TIMEOUT)
251 + .fromApp(appId)
252 + .add();
253 +
254 + flowObjectiveService.forward(deviceId, fo);
255 + });
256 + }
246 257
247 - flowObjectiveService.forward(inboundPacket.receivedFrom().deviceId(), fo); 258 + private List<Device> getGatewayNodeList() {
259 + List<Device> devices = Lists.newArrayList();
260 + gatewayService.getGatewayDeviceIds().forEach(deviceId ->
261 + devices.add(checkNotNull(deviceService.getDevice(deviceId))));
262 + return devices;
248 } 263 }
249 264
250 private Ip4Address getHostIpfromOpenstackPort(OpenstackPort openstackPort) { 265 private Ip4Address getHostIpfromOpenstackPort(OpenstackPort openstackPort) {
...@@ -255,15 +270,15 @@ public class OpenstackRoutingRulePopulator { ...@@ -255,15 +270,15 @@ public class OpenstackRoutingRulePopulator {
255 private Device getDevicefromOpenstackPort(OpenstackPort openstackPort) { 270 private Device getDevicefromOpenstackPort(OpenstackPort openstackPort) {
256 String openstackPortName = PORTNAME_PREFIX_VM + openstackPort.id().substring(0, 11); 271 String openstackPortName = PORTNAME_PREFIX_VM + openstackPort.id().substring(0, 11);
257 Device device = StreamSupport.stream(deviceService.getDevices().spliterator(), false) 272 Device device = StreamSupport.stream(deviceService.getDevices().spliterator(), false)
258 - .filter(d -> findPortinDevice(d, openstackPortName)) 273 + .filter(d -> findPortinDevice(d.id(), openstackPortName))
259 .iterator() 274 .iterator()
260 .next(); 275 .next();
261 checkNotNull(device, DEVICENOTNULL); 276 checkNotNull(device, DEVICENOTNULL);
262 return device; 277 return device;
263 } 278 }
264 279
265 - private boolean findPortinDevice(Device d, String openstackPortName) { 280 + private boolean findPortinDevice(DeviceId deviceId, String openstackPortName) {
266 - Port port = deviceService.getPorts(d.id()) 281 + Port port = deviceService.getPorts(deviceId)
267 .stream() 282 .stream()
268 .filter(p -> p.isEnabled() && p.annotations().value(PORTNAME).equals(openstackPortName)) 283 .filter(p -> p.isEnabled() && p.annotations().value(PORTNAME).equals(openstackPortName))
269 .findAny() 284 .findAny()
...@@ -274,13 +289,13 @@ public class OpenstackRoutingRulePopulator { ...@@ -274,13 +289,13 @@ public class OpenstackRoutingRulePopulator {
274 /** 289 /**
275 * Builds Nicira extension for tagging remoteIp of vxlan. 290 * Builds Nicira extension for tagging remoteIp of vxlan.
276 * 291 *
277 - * @param id Device Id of vxlan source device 292 + * @param deviceId Device Id of vxlan source device
278 * @param hostIp Remote Ip of vxlan destination device 293 * @param hostIp Remote Ip of vxlan destination device
279 * @return NiciraExtension Treatment 294 * @return NiciraExtension Treatment
280 */ 295 */
281 - public ExtensionTreatment buildNiciraExtenstion(DeviceId id, Ip4Address hostIp) { 296 + public ExtensionTreatment buildNiciraExtenstion(DeviceId deviceId, Ip4Address hostIp) {
282 - Driver driver = driverService.getDriver(id); 297 + Driver driver = driverService.getDriver(deviceId);
283 - DriverHandler driverHandler = new DefaultDriverHandler(new DefaultDriverData(driver, id)); 298 + DriverHandler driverHandler = new DefaultDriverHandler(new DefaultDriverData(driver, deviceId));
284 ExtensionTreatmentResolver resolver = driverHandler.behaviour(ExtensionTreatmentResolver.class); 299 ExtensionTreatmentResolver resolver = driverHandler.behaviour(ExtensionTreatmentResolver.class);
285 300
286 ExtensionTreatment extensionInstruction = 301 ExtensionTreatment extensionInstruction =
...@@ -329,7 +344,6 @@ public class OpenstackRoutingRulePopulator { ...@@ -329,7 +344,6 @@ public class OpenstackRoutingRulePopulator {
329 } 344 }
330 345
331 private void populateRuleGatewaytoController(long vni) { 346 private void populateRuleGatewaytoController(long vni) {
332 - Device gatewayDevice = getGatewayNode();
333 TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder(); 347 TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
334 TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder(); 348 TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
335 349
...@@ -346,26 +360,24 @@ public class OpenstackRoutingRulePopulator { ...@@ -346,26 +360,24 @@ public class OpenstackRoutingRulePopulator {
346 .fromApp(appId) 360 .fromApp(appId)
347 .add(); 361 .add();
348 362
349 - flowObjectiveService.forward(gatewayDevice.id(), fo); 363 + getGatewayNodeList().forEach(device -> flowObjectiveService.forward(device.id(), fo));
350 } 364 }
351 365
352 private void populateComputeNodeRules(long vni) { 366 private void populateComputeNodeRules(long vni) {
353 - Device gatewayDevice = getGatewayNode();
354 -
355 StreamSupport.stream(deviceService.getDevices().spliterator(), false) 367 StreamSupport.stream(deviceService.getDevices().spliterator(), false)
356 .filter(d -> !checkGatewayNode(d.id())) 368 .filter(d -> !checkGatewayNode(d.id()))
357 - .forEach(d -> populateRuleToGateway(d, gatewayDevice, vni)); 369 + .forEach(d -> populateRuleToGatewayBySgw(d.id(),
370 + gatewayService.getGroupIdForGatewayLoadBalance(d.id()), vni));
358 } 371 }
359 372
360 - private void populateRuleToGateway(Device d, Device gatewayDevice, long vni) { 373 + private void populateRuleToGatewayBySgw(DeviceId deviceId, GroupId groupId, long vni) {
361 TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder(); 374 TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
362 TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder(); 375 TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
363 376
364 sBuilder.matchEthType(Ethernet.TYPE_IPV4) 377 sBuilder.matchEthType(Ethernet.TYPE_IPV4)
365 .matchTunnelId(vni) 378 .matchTunnelId(vni)
366 .matchEthDst(GATEWAYMAC); 379 .matchEthDst(GATEWAYMAC);
367 - tBuilder.extension(buildNiciraExtenstion(d.id(), config.nodes().get(gatewayDevice.id())), d.id()) 380 + tBuilder.group(groupId);
368 - .setOutput(getTunnelPort(d.id()));
369 381
370 ForwardingObjective fo = DefaultForwardingObjective.builder() 382 ForwardingObjective fo = DefaultForwardingObjective.builder()
371 .withSelector(sBuilder.build()) 383 .withSelector(sBuilder.build())
...@@ -375,7 +387,28 @@ public class OpenstackRoutingRulePopulator { ...@@ -375,7 +387,28 @@ public class OpenstackRoutingRulePopulator {
375 .fromApp(appId) 387 .fromApp(appId)
376 .add(); 388 .add();
377 389
378 - flowObjectiveService.forward(d.id(), fo); 390 + flowObjectiveService.forward(deviceId, fo);
391 + }
392 +
393 + private void populateRuleToGateway(DeviceId deviceId, Device gatewayDevice, long vni) {
394 + TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
395 + TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
396 +
397 + sBuilder.matchEthType(Ethernet.TYPE_IPV4)
398 + .matchTunnelId(vni)
399 + .matchEthDst(GATEWAYMAC);
400 + tBuilder.extension(buildNiciraExtenstion(deviceId, config.nodes().get(gatewayDevice.id())), deviceId)
401 + .setOutput(getTunnelPort(deviceId));
402 +
403 + ForwardingObjective fo = DefaultForwardingObjective.builder()
404 + .withSelector(sBuilder.build())
405 + .withTreatment(tBuilder.build())
406 + .withFlag(ForwardingObjective.Flag.SPECIFIC)
407 + .withPriority(ROUTING_RULE_PRIORITY)
408 + .fromApp(appId)
409 + .add();
410 +
411 + flowObjectiveService.forward(deviceId, fo);
379 } 412 }
380 413
381 private Device getGatewayNode() { 414 private Device getGatewayNode() {
...@@ -383,7 +416,7 @@ public class OpenstackRoutingRulePopulator { ...@@ -383,7 +416,7 @@ public class OpenstackRoutingRulePopulator {
383 } 416 }
384 417
385 private boolean checkGatewayNode(DeviceId deviceId) { 418 private boolean checkGatewayNode(DeviceId deviceId) {
386 - return deviceId.toString().equals(config.gatewayBridgeId()); 419 + return gatewayService.getGatewayDeviceIds().stream().anyMatch(dId -> dId.equals(deviceId));
387 } 420 }
388 421
389 private long getVni(String netId) { 422 private long getVni(String netId) {
...@@ -412,7 +445,7 @@ public class OpenstackRoutingRulePopulator { ...@@ -412,7 +445,7 @@ public class OpenstackRoutingRulePopulator {
412 445
413 } 446 }
414 447
415 - private void removeRule(DeviceId id, TrafficSelector.Builder sBuilder, 448 + private void removeRule(DeviceId deviceId, TrafficSelector.Builder sBuilder,
416 ForwardingObjective.Flag flag, int priority) { 449 ForwardingObjective.Flag flag, int priority) {
417 TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder(); 450 TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
418 451
...@@ -424,7 +457,7 @@ public class OpenstackRoutingRulePopulator { ...@@ -424,7 +457,7 @@ public class OpenstackRoutingRulePopulator {
424 .fromApp(appId) 457 .fromApp(appId)
425 .remove(); 458 .remove();
426 459
427 - flowObjectiveService.forward(id, fo); 460 + flowObjectiveService.forward(deviceId, fo);
428 } 461 }
429 462
430 /** 463 /**
...@@ -442,8 +475,6 @@ public class OpenstackRoutingRulePopulator { ...@@ -442,8 +475,6 @@ public class OpenstackRoutingRulePopulator {
442 475
443 private void populateFloatingIpIncomingRules(OpenstackFloatingIP floatingIP, OpenstackPort port) { 476 private void populateFloatingIpIncomingRules(OpenstackFloatingIP floatingIP, OpenstackPort port) {
444 DeviceId portDeviceId = getDevicefromOpenstackPort(port).id(); 477 DeviceId portDeviceId = getDevicefromOpenstackPort(port).id();
445 - Device gatewayNode = getGatewayNode();
446 - Device portNode = deviceService.getDevice(portDeviceId);
447 478
448 TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder(); 479 TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
449 TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder(); 480 TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
...@@ -451,29 +482,28 @@ public class OpenstackRoutingRulePopulator { ...@@ -451,29 +482,28 @@ public class OpenstackRoutingRulePopulator {
451 sBuilder.matchEthType(Ethernet.TYPE_IPV4) 482 sBuilder.matchEthType(Ethernet.TYPE_IPV4)
452 .matchIPDst(IpPrefix.valueOf(floatingIP.floatingIpAddress(), PREFIX_LENGTH)); 483 .matchIPDst(IpPrefix.valueOf(floatingIP.floatingIpAddress(), PREFIX_LENGTH));
453 484
454 - tBuilder.setEthSrc(GATEWAYMAC) 485 + getGatewayNodeList().forEach(device -> {
455 - .setEthDst(port.macAddress()) 486 + DeviceId deviceId = device.id();
456 - .setIpDst(floatingIP.fixedIpAddress()) 487 + tBuilder.setEthSrc(GATEWAYMAC)
457 - .setTunnelId(getVni(port.networkId())) 488 + .setEthDst(port.macAddress())
458 - .extension(buildNiciraExtenstion(gatewayNode.id(), 489 + .setIpDst(floatingIP.fixedIpAddress())
459 - config.nodes().get(portNode.id())), gatewayNode.id()) 490 + .setTunnelId(getVni(port.networkId()))
460 - .setOutput(getTunnelPort(gatewayNode.id())); 491 + .extension(buildNiciraExtenstion(deviceId, config.nodes().get(portDeviceId)), deviceId)
461 - 492 + .setOutput(getTunnelPort(deviceId));
462 - ForwardingObjective fo = DefaultForwardingObjective.builder() 493 +
463 - .withSelector(sBuilder.build()) 494 + ForwardingObjective fo = DefaultForwardingObjective.builder()
464 - .withTreatment(tBuilder.build()) 495 + .withSelector(sBuilder.build())
465 - .withFlag(ForwardingObjective.Flag.VERSATILE) 496 + .withTreatment(tBuilder.build())
466 - .withPriority(FLOATING_RULE_PRIORITY) 497 + .withFlag(ForwardingObjective.Flag.VERSATILE)
467 - .fromApp(appId) 498 + .withPriority(FLOATING_RULE_PRIORITY)
468 - .add(); 499 + .fromApp(appId)
469 - 500 + .add();
470 - flowObjectiveService.forward(getGatewayNode().id(), fo); 501 +
471 - 502 + flowObjectiveService.forward(deviceId, fo);
503 + });
472 } 504 }
473 505
474 private void populateFloatingIpOutgoingRules(OpenstackFloatingIP floatingIP, OpenstackPort port) { 506 private void populateFloatingIpOutgoingRules(OpenstackFloatingIP floatingIP, OpenstackPort port) {
475 - Port outputPort = checkNotNull(getPortOfExternalInterface(), PORTNOTNULL);
476 -
477 TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder(); 507 TrafficSelector.Builder sBuilder = DefaultTrafficSelector.builder();
478 TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder(); 508 TrafficTreatment.Builder tBuilder = DefaultTrafficTreatment.builder();
479 509
...@@ -481,20 +511,25 @@ public class OpenstackRoutingRulePopulator { ...@@ -481,20 +511,25 @@ public class OpenstackRoutingRulePopulator {
481 .matchTunnelId(getVni(port.networkId())) 511 .matchTunnelId(getVni(port.networkId()))
482 .matchIPSrc(IpPrefix.valueOf(floatingIP.fixedIpAddress(), 32)); 512 .matchIPSrc(IpPrefix.valueOf(floatingIP.fixedIpAddress(), 32));
483 513
484 - tBuilder.setIpSrc(floatingIP.floatingIpAddress()) 514 + getGatewayNodeList().forEach(device -> {
485 - .setEthSrc(MacAddress.valueOf(config.gatewayExternalInterfaceMac())) 515 + DeviceId deviceId = device.id();
486 - .setEthDst(MacAddress.valueOf(config.physicalRouterMac())) 516 + tBuilder.setIpSrc(floatingIP.floatingIpAddress())
487 - .setOutput(outputPort.number()); 517 + .setOutput(getExternalPortNum(deviceId));
488 518
489 - ForwardingObjective fo = DefaultForwardingObjective.builder() 519 + ForwardingObjective fo = DefaultForwardingObjective.builder()
490 - .withSelector(sBuilder.build()) 520 + .withSelector(sBuilder.build())
491 - .withTreatment(tBuilder.build()) 521 + .withTreatment(tBuilder.build())
492 - .withFlag(ForwardingObjective.Flag.VERSATILE) 522 + .withFlag(ForwardingObjective.Flag.VERSATILE)
493 - .withPriority(FLOATING_RULE_PRIORITY) 523 + .withPriority(FLOATING_RULE_PRIORITY)
494 - .fromApp(appId) 524 + .fromApp(appId)
495 - .add(); 525 + .add();
496 526
497 - flowObjectiveService.forward(getGatewayNode().id(), fo); 527 + flowObjectiveService.forward(deviceId, fo);
528 + });
529 + }
530 +
531 + private PortNumber getExternalPortNum(DeviceId deviceId) {
532 + return checkNotNull(gatewayService.getGatewayExternalPorts(deviceId).get(0), PORTNOTNULL);
498 } 533 }
499 534
500 /** 535 /**
...@@ -514,38 +549,39 @@ public class OpenstackRoutingRulePopulator { ...@@ -514,38 +549,39 @@ public class OpenstackRoutingRulePopulator {
514 sIncomingBuilder.matchEthType(Ethernet.TYPE_IPV4) 549 sIncomingBuilder.matchEthType(Ethernet.TYPE_IPV4)
515 .matchIPDst(IpPrefix.valueOf(floatingIP.floatingIpAddress(), PREFIX_LENGTH)); 550 .matchIPDst(IpPrefix.valueOf(floatingIP.floatingIpAddress(), PREFIX_LENGTH));
516 551
517 - removeRule(getGatewayNode().id(), sOutgoingBuilder, ForwardingObjective.Flag.VERSATILE, FLOATING_RULE_PRIORITY); 552 + getGatewayNodeList().forEach(device -> {
518 - removeRule(getGatewayNode().id(), sIncomingBuilder, ForwardingObjective.Flag.VERSATILE, FLOATING_RULE_PRIORITY); 553 + removeRule(device.id(), sOutgoingBuilder, ForwardingObjective.Flag.VERSATILE, FLOATING_RULE_PRIORITY);
554 + removeRule(device.id(), sIncomingBuilder, ForwardingObjective.Flag.VERSATILE, FLOATING_RULE_PRIORITY);
555 + });
519 } 556 }
520 557
521 /** 558 /**
522 * Populates L3 rules for east to west traffic. 559 * Populates L3 rules for east to west traffic.
523 * 560 *
524 - * @param p target VM 561 + * @param openstackPort target VM
525 * @param targetList target openstackRouterInterfaces 562 * @param targetList target openstackRouterInterfaces
526 */ 563 */
527 - public void populateL3Rules(OpenstackPort p, List<OpenstackRouterInterface> targetList) { 564 + public void populateL3Rules(OpenstackPort openstackPort, List<OpenstackRouterInterface> targetList) {
528 - Device device = getDevicefromOpenstackPort(p); 565 + Device device = getDevicefromOpenstackPort(openstackPort);
529 - Port port = getPortFromOpenstackPort(device, p); 566 + Port port = getPortFromOpenstackPort(device, openstackPort);
530 - Ip4Address vmIp = (Ip4Address) p.fixedIps().values().iterator().next(); 567 + Ip4Address vmIp = openstackPort.fixedIps().values().iterator().next();
531 568
532 if (port == null) { 569 if (port == null) {
533 return; 570 return;
534 } 571 }
535 572
536 targetList.forEach(routerInterface -> { 573 targetList.forEach(routerInterface -> {
537 - OpenstackPort openstackPort = openstackService.port(routerInterface.portId()); 574 + long vni = getVni(openstackService.port(routerInterface.portId()).networkId());
538 - long vni = getVni(openstackPort.networkId());
539 575
540 if (vmIp == null) { 576 if (vmIp == null) {
541 return; 577 return;
542 } 578 }
543 579
544 - populateL3RulestoSameNode(vmIp, p, port, device, vni); 580 + populateL3RulestoSameNode(vmIp, openstackPort, port, device, vni);
545 581
546 deviceService.getAvailableDevices().forEach(d -> { 582 deviceService.getAvailableDevices().forEach(d -> {
547 - if (!d.equals(device) && !d.equals(getGatewayNode())) { 583 + if (!d.equals(device) && !checkGatewayNode(d.id())) {
548 - populateL3RulestoDifferentNode(vmIp, vni, d.id(), getHostIpfromOpenstackPort(p)); 584 + populateL3RulestoDifferentNode(vmIp, vni, d.id(), getHostIpfromOpenstackPort(openstackPort));
549 } 585 }
550 }); 586 });
551 587
...@@ -618,7 +654,7 @@ public class OpenstackRoutingRulePopulator { ...@@ -618,7 +654,7 @@ public class OpenstackRoutingRulePopulator {
618 OpenstackRoutingService routingService = getService(OpenstackRoutingService.class); 654 OpenstackRoutingService routingService = getService(OpenstackRoutingService.class);
619 655
620 deviceService.getAvailableDevices().forEach(d -> { 656 deviceService.getAvailableDevices().forEach(d -> {
621 - if (!d.equals(getGatewayNode())) { 657 + if (!checkGatewayNode(d.id())) {
622 routerInterfaces.forEach(routerInterface -> { 658 routerInterfaces.forEach(routerInterface -> {
623 String networkId = routingService.networkIdForRouterInterface(routerInterface.portId()); 659 String networkId = routingService.networkIdForRouterInterface(routerInterface.portId());
624 long vni = getVni(networkId); 660 long vni = getVni(networkId);
......
1 +{
2 + "apps" : {
3 + "org.onosproject.scalablegateway" : {
4 + "scalablegateway" : {
5 + "nodes" : [
6 + {
7 + "bridgeId" : "of:0000000000000003",
8 + "dataPlaneIp" : "10.62.148.17",
9 + "gatewayExternalInterfaceName" : "veth0"
10 + }]
11 + }
12 + }
13 + }
14 +}
...@@ -66,6 +66,15 @@ public final class GatewayNode { ...@@ -66,6 +66,15 @@ public final class GatewayNode {
66 } 66 }
67 67
68 /** 68 /**
69 + * Returns GatewayNode builder object.
70 + *
71 + * @return GatewayNode builder
72 + */
73 + public static GatewayNode.Builder builder() {
74 + return new Builder();
75 + }
76 +
77 + /**
69 * GatewayNode Builder class. 78 * GatewayNode Builder class.
70 */ 79 */
71 public static final class Builder { 80 public static final class Builder {
......
...@@ -17,6 +17,7 @@ package org.onosproject.scalablegateway.api; ...@@ -17,6 +17,7 @@ package org.onosproject.scalablegateway.api;
17 17
18 18
19 import com.fasterxml.jackson.databind.JsonNode; 19 import com.fasterxml.jackson.databind.JsonNode;
20 +import com.fasterxml.jackson.databind.node.ObjectNode;
20 import com.google.common.collect.Lists; 21 import com.google.common.collect.Lists;
21 import com.google.common.collect.Sets; 22 import com.google.common.collect.Sets;
22 import org.onlab.packet.Ip4Address; 23 import org.onlab.packet.Ip4Address;
...@@ -28,9 +29,10 @@ import org.slf4j.Logger; ...@@ -28,9 +29,10 @@ import org.slf4j.Logger;
28 import java.util.Collections; 29 import java.util.Collections;
29 import java.util.List; 30 import java.util.List;
30 import java.util.Set; 31 import java.util.Set;
32 +import java.util.stream.StreamSupport;
31 33
32 -import static org.slf4j.LoggerFactory.getLogger;
33 import static org.onosproject.net.config.Config.FieldPresence.MANDATORY; 34 import static org.onosproject.net.config.Config.FieldPresence.MANDATORY;
35 +import static org.slf4j.LoggerFactory.getLogger;
34 36
35 /** 37 /**
36 * Configuration object for OpensatckNode service. 38 * Configuration object for OpensatckNode service.
...@@ -79,9 +81,21 @@ public class GatewayNodeConfig extends Config<ApplicationId> { ...@@ -79,9 +81,21 @@ public class GatewayNodeConfig extends Config<ApplicationId> {
79 81
80 @Override 82 @Override
81 public boolean isValid() { 83 public boolean isValid() {
82 - return hasOnlyFields(NODES, BRIDGE_ID, DATAPLANE_IP, EXTERNAL_INTERFACE_NAME) && 84 + JsonNode jsonNodes = object.get(NODES);
83 - isIpAddress(DATAPLANE_IP, MANDATORY) && isString(BRIDGE_ID, MANDATORY) && 85 +
84 - isString(EXTERNAL_INTERFACE_NAME, MANDATORY); 86 + if (jsonNodes == null) {
87 + return false;
88 + }
89 +
90 + return hasOnlyFields(NODES)
91 + && StreamSupport.stream(jsonNodes.spliterator(), false).allMatch(this::checkValid);
92 + }
93 +
94 + private boolean checkValid(JsonNode jsonNode) {
95 + ObjectNode objectNode = (ObjectNode) jsonNode;
96 + return isString(objectNode, BRIDGE_ID, MANDATORY)
97 + && isIpAddress(objectNode, DATAPLANE_IP, MANDATORY)
98 + && isString(objectNode, EXTERNAL_INTERFACE_NAME, MANDATORY);
85 } 99 }
86 100
87 } 101 }
......