Carmelo Cascone
Committed by Thomas Vachuska

Various changes in BMv2 driver and provider modules (onos1.6 cherry-pick)

Driver notable changes:
- Implemented new behaviors, removed deprecated ones
- Removed flow rule translator classes (now under protocol module)
- Improved FlowRuleProgrammable: now it uses BMv2TableEntryService
	to lookup/bind flow rules with BMv2 table entries, retrieves flow
	statistics, better exception handling when adding/replacing/removing
	table entries.
- Improved PacketProgrammable: better exception handling and logging

Provider notable changes:
- Bmv2DeviceProvider: detects and notifies device configuration
	changes and reboots to Bmv2DeviceContextService, added support for
	periodic polling of port statistics
- Bmv2PacketProvider: implemented workaround for OutboundPackets with
	flood treatment

Change-Id: I79b756b533d4afb6b70025a137b2e811fd42a4e8
Showing 31 changed files with 744 additions and 1529 deletions
......@@ -20,6 +20,6 @@
<feature name="${project.artifactId}" version="${project.version}"
description="${project.description}">
<bundle>mvn:${project.groupId}/${project.artifactId}/${project.version}</bundle>
<bundle>mvn:${project.groupId}/onos-bmv2-protocol/${project.version}</bundle>
<bundle>mvn:${project.groupId}/onos-bmv2-protocol-api/${project.version}</bundle>
</feature>
</features>
......
......@@ -22,11 +22,11 @@
<artifactId>onos-drivers-general</artifactId>
<groupId>org.onosproject</groupId>
<version>1.7.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>onos-drivers-bmv2</artifactId>
<version>1.7.0-SNAPSHOT</version>
<packaging>bundle</packaging>
......@@ -36,7 +36,7 @@
<onos.app.name>org.onosproject.drivers.bmv2</onos.app.name>
<onos.app.origin>ON.Lab</onos.app.origin>
<onos.app.category>Drivers</onos.app.category>
<onos.app.title>BMv2 Device Drivers</onos.app.title>
<onos.app.title>BMv2 Drivers</onos.app.title>
<onos.app.url>http://onosproject.org</onos.app.url>
<onos.app.requires>
org.onosproject.bmv2
......@@ -46,7 +46,7 @@
<dependencies>
<dependency>
<groupId>org.onosproject</groupId>
<artifactId>onos-bmv2-protocol</artifactId>
<artifactId>onos-bmv2-protocol-api</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
......
/*
* Copyright 2016-present Open Networking Laboratory
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.onosproject.drivers.bmv2;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import org.onlab.packet.ChassisId;
import org.onosproject.bmv2.api.runtime.Bmv2DeviceAgent;
import org.onosproject.bmv2.api.runtime.Bmv2RuntimeException;
import org.onosproject.bmv2.api.service.Bmv2Controller;
import org.onosproject.net.AnnotationKeys;
import org.onosproject.net.DefaultAnnotations;
import org.onosproject.net.DeviceId;
import org.onosproject.net.PortNumber;
import org.onosproject.net.device.DefaultDeviceDescription;
import org.onosproject.net.device.DefaultPortDescription;
import org.onosproject.net.device.DeviceDescription;
import org.onosproject.net.device.DeviceDescriptionDiscovery;
import org.onosproject.net.device.PortDescription;
import org.onosproject.net.driver.AbstractHandlerBehaviour;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.math.BigInteger;
import java.util.List;
import static org.onosproject.bmv2.api.runtime.Bmv2Device.*;
import static org.onosproject.net.Device.Type.SWITCH;
/**
* Implementation of the device description discovery behaviour for BMv2.
*/
public class Bmv2DeviceDescriptionDiscovery extends AbstractHandlerBehaviour implements DeviceDescriptionDiscovery {
private static final String JSON_CONFIG_MD5 = "bmv2JsonConfigMd5";
private static final String PROCESS_INSTANCE_ID = "bmv2ProcessInstanceId";
private final Logger log = LoggerFactory.getLogger(this.getClass());
private Bmv2Controller controller;
private boolean init() {
controller = handler().get(Bmv2Controller.class);
if (controller == null) {
log.warn("Failed to get a BMv2 controller");
return false;
}
return true;
}
@Override
public DeviceDescription discoverDeviceDetails() {
if (!init()) {
return null;
}
DeviceId deviceId = handler().data().deviceId();
Bmv2DeviceAgent deviceAgent;
try {
deviceAgent = controller.getAgent(deviceId);
} catch (Bmv2RuntimeException e) {
log.error("Failed to connect to Bmv2 device", e);
return null;
}
DefaultAnnotations.Builder annotationsBuilder = DefaultAnnotations.builder();
try {
String md5 = deviceAgent.getJsonConfigMd5();
BigInteger i = new BigInteger(1, md5.getBytes());
annotationsBuilder.set(JSON_CONFIG_MD5, String.format("%1$032X", i).toLowerCase());
} catch (Bmv2RuntimeException e) {
log.warn("Unable to dump JSON configuration from {}: {}", deviceId, e.explain());
}
try {
int instanceId = deviceAgent.getProcessInstanceId();
annotationsBuilder.set(PROCESS_INSTANCE_ID, String.valueOf(instanceId));
} catch (Bmv2RuntimeException e) {
log.warn("Unable to get process instance ID from {}: {}", deviceId, e.explain());
}
annotationsBuilder.set(AnnotationKeys.PROTOCOL, PROTOCOL);
return new DefaultDeviceDescription(deviceId.uri(),
SWITCH,
MANUFACTURER,
HW_VERSION,
SW_VERSION,
SERIAL_NUMBER,
new ChassisId(),
annotationsBuilder.build());
}
@Override
public List<PortDescription> discoverPortDetails() {
if (!init()) {
return null;
}
DeviceId deviceId = handler().data().deviceId();
Bmv2DeviceAgent deviceAgent;
try {
deviceAgent = controller.getAgent(deviceId);
} catch (Bmv2RuntimeException e) {
log.error("Failed to connect to Bmv2 device", e);
return null;
}
List<PortDescription> portDescriptions = Lists.newArrayList();
try {
deviceAgent.getPortsInfo().forEach(p -> {
PortNumber portNumber = PortNumber.portNumber((long) p.number(), p.ifaceName());
portDescriptions.add(new DefaultPortDescription(portNumber, p.isUp(), DefaultAnnotations.EMPTY));
});
} catch (Bmv2RuntimeException e) {
log.error("Unable to get port descriptions of {}: {}", deviceId, e);
}
return ImmutableList.copyOf(portDescriptions);
}
}
......@@ -20,7 +20,7 @@ import org.apache.felix.scr.annotations.Component;
import org.onosproject.net.driver.AbstractDriverLoader;
/**
* Loader for barefoot drivers from specific xml.
* Loader for BMv2 drivers from xml file.
*/
@Component(immediate = true)
public class Bmv2DriversLoader extends AbstractDriverLoader {
......
......@@ -14,7 +14,29 @@
* limitations under the License.
*/
package org.onosproject.drivers.bmv2;
import org.onosproject.bmv2.api.runtime.Bmv2ExtensionSelector;
import org.onosproject.net.behaviour.ExtensionSelectorResolver;
import org.onosproject.net.driver.AbstractHandlerBehaviour;
import org.onosproject.net.flow.criteria.ExtensionSelector;
import org.onosproject.net.flow.criteria.ExtensionSelectorType;
import java.util.Collections;
import static org.onosproject.net.flow.criteria.ExtensionSelectorType.ExtensionSelectorTypes.BMV2_MATCH_PARAMS;
/**
* Translators of ONOS abstractions to BMv2 model-dependent abstractions.
* Implementation of the extension selector resolver behaviour for BMv2.
*/
package org.onosproject.drivers.bmv2.translators;
\ No newline at end of file
public class Bmv2ExtensionSelectorResolver extends AbstractHandlerBehaviour implements ExtensionSelectorResolver {
@Override
public ExtensionSelector getExtensionSelector(ExtensionSelectorType type) {
if (type.equals(BMV2_MATCH_PARAMS.type())) {
return new Bmv2ExtensionSelector(Collections.emptyMap());
}
return null;
}
}
......
......@@ -14,14 +14,26 @@
* limitations under the License.
*/
package org.onosproject.drivers.bmv2.translators;
package org.onosproject.drivers.bmv2;
import org.onosproject.bmv2.api.runtime.Bmv2ExtensionTreatment;
import org.onosproject.net.behaviour.ExtensionTreatmentResolver;
import org.onosproject.net.driver.AbstractHandlerBehaviour;
import org.onosproject.net.flow.instructions.ExtensionTreatment;
import org.onosproject.net.flow.instructions.ExtensionTreatmentType;
import static org.onosproject.net.flow.instructions.ExtensionTreatmentType.ExtensionTreatmentTypes.BMV2_ACTION;
/**
* BMv2 flow rule translator exception.
* Implementation of the extension treatment resolver behavior for BMv2.
*/
public class Bmv2FlowRuleTranslatorException extends Exception {
public class Bmv2ExtensionTreatmentResolver extends AbstractHandlerBehaviour implements ExtensionTreatmentResolver {
Bmv2FlowRuleTranslatorException(String msg) {
super(msg);
@Override
public ExtensionTreatment getExtensionInstruction(ExtensionTreatmentType type) {
if (type.equals(BMV2_ACTION.type())) {
return new Bmv2ExtensionTreatment(null);
}
return null;
}
}
......
......@@ -16,28 +16,25 @@
package org.onosproject.drivers.bmv2;
import com.eclipsesource.json.Json;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.commons.lang3.tuple.Triple;
import org.onosproject.bmv2.api.model.Bmv2Model;
import org.onosproject.bmv2.api.runtime.Bmv2Client;
import org.onosproject.bmv2.api.context.Bmv2Configuration;
import org.onosproject.bmv2.api.context.Bmv2DeviceContext;
import org.onosproject.bmv2.api.context.Bmv2FlowRuleTranslator;
import org.onosproject.bmv2.api.context.Bmv2FlowRuleTranslatorException;
import org.onosproject.bmv2.api.context.Bmv2Interpreter;
import org.onosproject.bmv2.api.runtime.Bmv2DeviceAgent;
import org.onosproject.bmv2.api.runtime.Bmv2FlowRuleWrapper;
import org.onosproject.bmv2.api.runtime.Bmv2MatchKey;
import org.onosproject.bmv2.api.runtime.Bmv2ParsedTableEntry;
import org.onosproject.bmv2.api.runtime.Bmv2RuntimeException;
import org.onosproject.bmv2.api.runtime.Bmv2TableEntry;
import org.onosproject.bmv2.ctl.Bmv2ThriftClient;
import org.onosproject.drivers.bmv2.translators.Bmv2DefaultFlowRuleTranslator;
import org.onosproject.drivers.bmv2.translators.Bmv2FlowRuleTranslator;
import org.onosproject.drivers.bmv2.translators.Bmv2FlowRuleTranslatorException;
import org.onosproject.drivers.bmv2.translators.Bmv2SimpleTranslatorConfig;
import org.onosproject.net.Device;
import org.onosproject.bmv2.api.runtime.Bmv2TableEntryReference;
import org.onosproject.bmv2.api.service.Bmv2Controller;
import org.onosproject.bmv2.api.service.Bmv2DeviceContextService;
import org.onosproject.bmv2.api.service.Bmv2TableEntryService;
import org.onosproject.net.DeviceId;
import org.onosproject.net.device.DeviceService;
import org.onosproject.net.driver.AbstractHandlerBehaviour;
import org.onosproject.net.flow.DefaultFlowEntry;
import org.onosproject.net.flow.FlowEntry;
......@@ -50,97 +47,130 @@ import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Set;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import static org.onosproject.bmv2.api.runtime.Bmv2RuntimeException.Code.*;
import static org.onosproject.net.flow.FlowEntry.FlowEntryState.ADDED;
/**
* Flow rule programmable device behaviour implementation for BMv2.
* Implementation of the flow rule programmable behaviour for BMv2.
*/
public class Bmv2FlowRuleProgrammable extends AbstractHandlerBehaviour
implements FlowRuleProgrammable {
private static final Logger LOG =
LoggerFactory.getLogger(Bmv2FlowRuleProgrammable.class);
// There's no Bmv2 client method to poll flow entries from the device. Use a local store.
// FIXME: this information should be distributed across instances of the cluster.
private static final ConcurrentMap<Triple<DeviceId, String, Bmv2MatchKey>, Pair<Long, TimestampedFlowRule>>
ENTRIES_MAP = Maps.newConcurrentMap();
// Cache model objects instead of parsing the JSON each time.
private static final LoadingCache<String, Bmv2Model> MODEL_CACHE = CacheBuilder.newBuilder()
.expireAfterAccess(60, TimeUnit.SECONDS)
.build(new CacheLoader<String, Bmv2Model>() {
@Override
public Bmv2Model load(String jsonString) throws Exception {
// Expensive call.
return Bmv2Model.parse(Json.parse(jsonString).asObject());
}
});
public class Bmv2FlowRuleProgrammable extends AbstractHandlerBehaviour implements FlowRuleProgrammable {
private final Logger log = LoggerFactory.getLogger(this.getClass());
// Needed to synchronize operations over the same table entry.
private static final ConcurrentMap<Bmv2TableEntryReference, Boolean> ENTRY_LOCKS = Maps.newConcurrentMap();
private Bmv2Controller controller;
private Bmv2TableEntryService tableEntryService;
private Bmv2DeviceContextService contextService;
private boolean init() {
controller = handler().get(Bmv2Controller.class);
tableEntryService = handler().get(Bmv2TableEntryService.class);
contextService = handler().get(Bmv2DeviceContextService.class);
if (controller == null) {
log.warn("Failed to get a BMv2 controller");
return false;
}
if (tableEntryService == null) {
log.warn("Failed to get a BMv2 table entry service");
return false;
}
if (contextService == null) {
log.warn("Failed to get a BMv2 device context service");
return false;
}
return true;
}
@Override
public Collection<FlowEntry> getFlowEntries() {
if (!init()) {
return Collections.emptyList();
}
DeviceId deviceId = handler().data().deviceId();
Bmv2Client deviceClient;
Bmv2DeviceAgent deviceAgent;
try {
deviceClient = Bmv2ThriftClient.of(deviceId);
deviceAgent = controller.getAgent(deviceId);
} catch (Bmv2RuntimeException e) {
LOG.error("Failed to connect to Bmv2 device", e);
log.error("Failed to get BMv2 device agent: {}", e.explain());
return Collections.emptyList();
}
Bmv2Model model = getTranslator(deviceId).config().model();
Bmv2DeviceContext context = contextService.getContext(deviceId);
if (context == null) {
log.warn("Unable to get device context for {}", deviceId);
}
Bmv2Interpreter interpreter = context.interpreter();
Bmv2Configuration configuration = context.configuration();
List<FlowEntry> entryList = Lists.newArrayList();
model.tables().forEach(table -> {
// For each table declared in the model for this device, do:
try {
// Bmv2 doesn't support proper polling for table entries, but only a string based table dump.
// The trick here is to first dump the entry ids currently installed in the device for a given table,
// and then filter ENTRIES_MAP based on the retrieved values.
Set<Long> installedEntryIds = Sets.newHashSet(deviceClient.getInstalledEntryIds(table.name()));
ENTRIES_MAP.forEach((key, value) -> {
if (key.getLeft() == deviceId && key.getMiddle() == table.name()
&& value != null) {
long entryId = value.getKey();
// Filter entries_map for this device and table.
if (installedEntryIds.contains(entryId)) {
// Entry is installed.
long bytes = 0L;
long packets = 0L;
if (table.hasCounters()) {
// Read counter values from device.
try {
Pair<Long, Long> counterValue = deviceClient.readTableEntryCounter(table.name(),
entryId);
bytes = counterValue.getLeft();
packets = counterValue.getRight();
} catch (Bmv2RuntimeException e) {
LOG.warn("Unable to get counter values for entry {} of table {} of device {}: {}",
entryId, table.name(), deviceId, e.toString());
}
}
TimestampedFlowRule tsRule = value.getRight();
FlowEntry entry = new DefaultFlowEntry(tsRule.rule(), ADDED,
tsRule.lifeInSeconds(), packets, bytes);
entryList.add(entry);
} else {
// No such entry on device, can remove from local store.
ENTRIES_MAP.remove(key);
configuration.tables().forEach(table -> {
// For each table in the configuration AND exposed by the interpreter.
if (!interpreter.tableIdMap().inverse().containsKey(table.name())) {
return;
}
// Bmv2 doesn't support proper polling for table entries, but only a string based table dump.
// The trick here is to first dump the entries currently installed in the device for a given table,
// and then query a service for the corresponding, previously applied, flow rule.
List<Bmv2ParsedTableEntry> installedEntries = tableEntryService.getTableEntries(deviceId, table.name());
installedEntries.forEach(parsedEntry -> {
Bmv2TableEntryReference entryRef = new Bmv2TableEntryReference(deviceId,
table.name(),
parsedEntry.matchKey());
ENTRY_LOCKS.compute(entryRef, (key, value) -> {
Bmv2FlowRuleWrapper frWrapper = tableEntryService.lookupEntryReference(entryRef);
if (frWrapper == null) {
log.warn("missing reference from table entry service, BUG? " +
"deviceId={}, tableName={}, matchKey={}",
deviceId, table.name(), entryRef.matchKey());
return null;
}
long remoteEntryId = parsedEntry.entryId();
long localEntryId = frWrapper.entryId();
if (remoteEntryId != localEntryId) {
log.warn("getFlowEntries(): inconsistent entry id! BUG? Updating it... remote={}, local={}",
remoteEntryId, localEntryId);
frWrapper = new Bmv2FlowRuleWrapper(frWrapper.rule(), remoteEntryId,
frWrapper.creationDate());
tableEntryService.bindEntryReference(entryRef, frWrapper);
}
long bytes = 0L;
long packets = 0L;
if (table.hasCounters()) {
// Read counter values from device.
try {
Pair<Long, Long> counterValue = deviceAgent.readTableEntryCounter(table.name(),
remoteEntryId);
bytes = counterValue.getLeft();
packets = counterValue.getRight();
} catch (Bmv2RuntimeException e) {
log.warn("Unable to get counters for entry {}/{} of device {}: {}",
table.name(), remoteEntryId, deviceId, e.explain());
}
}
FlowEntry entry = new DefaultFlowEntry(frWrapper.rule(), ADDED, frWrapper.lifeInSeconds(),
packets, bytes);
entryList.add(entry);
return true;
});
} catch (Bmv2RuntimeException e) {
LOG.error("Unable to get flow entries for table {} of device {}: {}",
table.name(), deviceId, e.toString());
}
});
});
return Collections.unmodifiableCollection(entryList);
......@@ -160,17 +190,27 @@ public class Bmv2FlowRuleProgrammable extends AbstractHandlerBehaviour
private Collection<FlowRule> processFlowRules(Collection<FlowRule> rules, Operation operation) {
if (!init()) {
return Collections.emptyList();
}
DeviceId deviceId = handler().data().deviceId();
Bmv2Client deviceClient;
Bmv2DeviceAgent deviceAgent;
try {
deviceClient = Bmv2ThriftClient.of(deviceId);
deviceAgent = controller.getAgent(deviceId);
} catch (Bmv2RuntimeException e) {
LOG.error("Failed to connect to Bmv2 device", e);
log.error("Failed to get BMv2 device agent: {}", e.explain());
return Collections.emptyList();
}
Bmv2FlowRuleTranslator translator = getTranslator(deviceId);
Bmv2DeviceContext context = contextService.getContext(deviceId);
if (context == null) {
log.error("Unable to get device context for {}", deviceId);
return Collections.emptyList();
}
Bmv2FlowRuleTranslator translator = tableEntryService.getFlowRuleTranslator();
List<FlowRule> processedFlowRules = Lists.newArrayList();
......@@ -179,120 +219,114 @@ public class Bmv2FlowRuleProgrammable extends AbstractHandlerBehaviour
Bmv2TableEntry bmv2Entry;
try {
bmv2Entry = translator.translate(rule);
bmv2Entry = translator.translate(rule, context);
} catch (Bmv2FlowRuleTranslatorException e) {
LOG.error("Unable to translate flow rule: {}", e.getMessage());
log.warn("Unable to translate flow rule: {} - {}", e.getMessage(), rule);
continue;
}
String tableName = bmv2Entry.tableName();
Triple<DeviceId, String, Bmv2MatchKey> entryKey = Triple.of(deviceId, tableName, bmv2Entry.matchKey());
Bmv2TableEntryReference entryRef = new Bmv2TableEntryReference(deviceId, tableName, bmv2Entry.matchKey());
/*
From here on threads are synchronized over entryKey, i.e. serialize operations
over the same matchKey of a specific table and device.
*/
ENTRIES_MAP.compute(entryKey, (key, value) -> {
ENTRY_LOCKS.compute(entryRef, (key, value) -> {
// Get from store
Bmv2FlowRuleWrapper frWrapper = tableEntryService.lookupEntryReference(entryRef);
try {
if (operation == Operation.APPLY) {
// Apply entry
long entryId;
if (value != null) {
if (frWrapper != null) {
// Existing entry.
entryId = value.getKey();
try {
// Tentatively delete entry before re-adding.
// It might not exist on device due to inconsistencies.
deviceClient.deleteTableEntry(bmv2Entry.tableName(), entryId);
value = null;
} catch (Bmv2RuntimeException e) {
// Silently drop exception as we can probably fix this by re-adding the entry.
}
entryId = frWrapper.entryId();
// Tentatively delete entry before re-adding.
// It might not exist on device due to inconsistencies.
silentlyRemove(deviceAgent, entryRef.tableName(), entryId);
}
// Add entry.
entryId = deviceClient.addTableEntry(bmv2Entry);
value = Pair.of(entryId, new TimestampedFlowRule(rule));
entryId = doAddEntry(deviceAgent, bmv2Entry);
frWrapper = new Bmv2FlowRuleWrapper(rule, entryId, new Date());
} else {
// Remove entry
if (value == null) {
if (frWrapper == null) {
// Entry not found in map, how come?
LOG.debug("Trying to remove entry, but entry ID not found: " + entryKey);
forceRemove(deviceAgent, entryRef.tableName(), entryRef.matchKey());
} else {
deviceClient.deleteTableEntry(tableName, value.getKey());
value = null;
long entryId = frWrapper.entryId();
doRemove(deviceAgent, entryRef.tableName(), entryId, entryRef.matchKey());
}
frWrapper = null;
}
// If here, no exceptions... things went well :)
processedFlowRules.add(rule);
} catch (Bmv2RuntimeException e) {
LOG.warn("Unable to {} flow rule: {}", operation.name().toLowerCase(), e.toString());
log.warn("Unable to {} flow rule: {}", operation.name(), e.explain());
}
// Update binding in table entry service.
if (frWrapper != null) {
tableEntryService.bindEntryReference(entryRef, frWrapper);
return true;
} else {
tableEntryService.unbindEntryReference(entryRef);
return null;
}
return value;
});
}
return processedFlowRules;
}
/**
* Gets the appropriate flow rule translator based on the device running configuration.
*
* @param deviceId a device id
* @return a flow rule translator
*/
private Bmv2FlowRuleTranslator getTranslator(DeviceId deviceId) {
DeviceService deviceService = handler().get(DeviceService.class);
if (deviceService == null) {
LOG.error("Unable to get device service");
return null;
private long doAddEntry(Bmv2DeviceAgent agent, Bmv2TableEntry entry) throws Bmv2RuntimeException {
try {
return agent.addTableEntry(entry);
} catch (Bmv2RuntimeException e) {
if (e.getCode() != TABLE_DUPLICATE_ENTRY) {
forceRemove(agent, entry.tableName(), entry.matchKey());
return agent.addTableEntry(entry);
} else {
throw e;
}
}
}
Device device = deviceService.getDevice(deviceId);
if (device == null) {
LOG.error("Unable to get device {}", deviceId);
return null;
private void doRemove(Bmv2DeviceAgent agent, String tableName, long entryId, Bmv2MatchKey matchKey)
throws Bmv2RuntimeException {
try {
agent.deleteTableEntry(tableName, entryId);
} catch (Bmv2RuntimeException e) {
if (e.getCode() == TABLE_INVALID_HANDLE || e.getCode() == TABLE_EXPIRED_HANDLE) {
// entry is not there with the declared ID, try with a forced remove.
forceRemove(agent, tableName, matchKey);
} else {
throw e;
}
}
}
String jsonString = device.annotations().value("bmv2JsonConfigValue");
if (jsonString == null) {
LOG.error("Unable to read bmv2 JSON config from device {}", deviceId);
return null;
private void forceRemove(Bmv2DeviceAgent agent, String tableName, Bmv2MatchKey matchKey)
throws Bmv2RuntimeException {
// Find the entryID (expensive call!)
for (Bmv2ParsedTableEntry pEntry : tableEntryService.getTableEntries(agent.deviceId(), tableName)) {
if (pEntry.matchKey().equals(matchKey)) {
// Remove entry and drop exceptions.
silentlyRemove(agent, tableName, pEntry.entryId());
break;
}
}
}
Bmv2Model model;
private void silentlyRemove(Bmv2DeviceAgent agent, String tableName, long entryId) {
try {
model = MODEL_CACHE.get(jsonString);
} catch (ExecutionException e) {
LOG.error("Unable to parse bmv2 JSON config for device {}:", deviceId, e.getCause());
return null;
agent.deleteTableEntry(tableName, entryId);
} catch (Bmv2RuntimeException e) {
// do nothing
}
// TODO: get translator config dynamically.
// Now it's hardcoded, selection should be based on the device bmv2 model.
Bmv2FlowRuleTranslator.TranslatorConfig translatorConfig = new Bmv2SimpleTranslatorConfig(model);
return new Bmv2DefaultFlowRuleTranslator(translatorConfig);
}
private enum Operation {
APPLY, REMOVE
}
private class TimestampedFlowRule {
private final FlowRule rule;
private final Date addedDate;
public TimestampedFlowRule(FlowRule rule) {
this.rule = rule;
this.addedDate = new Date();
}
public FlowRule rule() {
return rule;
}
public long lifeInSeconds() {
return (new Date().getTime() - addedDate.getTime()) / 1000;
}
}
}
\ No newline at end of file
......
......@@ -17,56 +17,59 @@
package org.onosproject.drivers.bmv2;
import org.onlab.util.ImmutableByteSequence;
import org.onosproject.bmv2.api.runtime.Bmv2Client;
import org.onosproject.bmv2.api.runtime.Bmv2DeviceAgent;
import org.onosproject.bmv2.api.runtime.Bmv2RuntimeException;
import org.onosproject.bmv2.ctl.Bmv2ThriftClient;
import org.onosproject.bmv2.api.service.Bmv2Controller;
import org.onosproject.net.DeviceId;
import org.onosproject.net.driver.AbstractHandlerBehaviour;
import org.onosproject.net.flow.TrafficTreatment;
import org.onosproject.net.flow.instructions.Instructions;
import org.onosproject.net.packet.OutboundPacket;
import org.onosproject.net.packet.PacketProgrammable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.List;
import static java.lang.Math.toIntExact;
import static org.onosproject.net.PortNumber.FLOOD;
import static java.util.stream.Collectors.toList;
import static org.onosproject.net.flow.instructions.Instruction.Type.OUTPUT;
import static org.onosproject.net.flow.instructions.Instructions.OutputInstruction;
/**
* Packet programmable device behaviour implementation for BMv2.
* Implementation of the packet programmable behaviour for BMv2.
*/
public class Bmv2PacketProgrammable extends AbstractHandlerBehaviour implements PacketProgrammable {
private static final Logger LOG =
LoggerFactory.getLogger(Bmv2PacketProgrammable.class);
private final Logger log = LoggerFactory.getLogger(this.getClass());
@Override
public void emit(OutboundPacket packet) {
TrafficTreatment treatment = packet.treatment();
treatment.allInstructions().forEach(inst -> {
if (inst.type().equals(OUTPUT)) {
Instructions.OutputInstruction outInst = (Instructions.OutputInstruction) inst;
if (outInst.port().isLogical()) {
if (outInst.port() == FLOOD) {
// TODO: implement flood
LOG.info("Flood not implemented", outInst);
}
LOG.info("Output on logical port not supported: {}", outInst);
} else {
try {
long longPort = outInst.port().toLong();
int portNumber = toIntExact(longPort);
send(portNumber, packet);
} catch (ArithmeticException e) {
LOG.error("Port number overflow! Cannot send packet on port {} (long), as the bmv2" +
" device only accepts int port values.");
}
}
// BMv2 supports only OUTPUT instructions.
List<OutputInstruction> outInstructions = treatment.allInstructions()
.stream()
.filter(i -> i.type().equals(OUTPUT))
.map(i -> (OutputInstruction) i)
.collect(toList());
if (treatment.allInstructions().size() != outInstructions.size()) {
// There are other instructions that are not of type OUTPUT
log.warn("Dropping emit request, treatment nor supported: {}", treatment);
return;
}
outInstructions.forEach(outInst -> {
if (outInst.port().isLogical()) {
log.warn("Dropping emit request, logical port not supported: {}", outInst.port());
} else {
LOG.info("Instruction type not supported: {}", inst.type().name());
try {
int portNumber = toIntExact(outInst.port().toLong());
send(portNumber, packet);
} catch (ArithmeticException e) {
log.error("Dropping emit request, port number too big: {}", outInst.port().toLong());
}
}
});
}
......@@ -75,19 +78,25 @@ public class Bmv2PacketProgrammable extends AbstractHandlerBehaviour implements
DeviceId deviceId = handler().data().deviceId();
Bmv2Client deviceClient;
Bmv2Controller controller = handler().get(Bmv2Controller.class);
if (controller == null) {
log.error("Failed to get BMv2 controller");
return;
}
Bmv2DeviceAgent deviceAgent;
try {
deviceClient = Bmv2ThriftClient.of(deviceId);
deviceAgent = controller.getAgent(deviceId);
} catch (Bmv2RuntimeException e) {
LOG.error("Failed to connect to Bmv2 device", e);
log.error("Failed to get Bmv2 device agent for {}: {}", deviceId, e.explain());
return;
}
ImmutableByteSequence bs = ImmutableByteSequence.copyFrom(packet.data());
try {
deviceClient.transmitPacket(port, bs);
deviceAgent.transmitPacket(port, bs);
} catch (Bmv2RuntimeException e) {
LOG.info("Unable to push packet to device: deviceId={}, packet={}", deviceId, bs);
log.warn("Unable to emit packet trough {}: {}", deviceId, e.explain());
}
}
}
......
......@@ -37,8 +37,8 @@ public class Bmv2Pipeliner extends AbstractHandlerBehaviour implements Pipeliner
@Override
public void init(DeviceId deviceId, PipelinerContext context) {
// TODO: get multi-table pipeliner dynamically based on BMv2 device running model
// Right now we only support single table pipelines
// TODO: get multi-table pipeliner dynamically based on BMv2 device running model (hard).
// Right now we are able to map flow objectives only in the first table of the pipeline.
pipeliner = new DefaultSingleTablePipeline();
pipeliner.init(deviceId, context);
}
......
/*
* Copyright 2014-2016 Open Networking Laboratory
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.onosproject.drivers.bmv2;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import org.onosproject.bmv2.api.runtime.Bmv2Client;
import org.onosproject.bmv2.api.runtime.Bmv2RuntimeException;
import org.onosproject.bmv2.ctl.Bmv2ThriftClient;
import org.onosproject.net.DefaultAnnotations;
import org.onosproject.net.PortNumber;
import org.onosproject.net.SparseAnnotations;
import org.onosproject.net.behaviour.PortDiscovery;
import org.onosproject.net.device.DefaultPortDescription;
import org.onosproject.net.device.PortDescription;
import org.onosproject.net.driver.AbstractHandlerBehaviour;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Collections;
import java.util.List;
public class Bmv2PortDiscovery extends AbstractHandlerBehaviour
implements PortDiscovery {
private final Logger log =
LoggerFactory.getLogger(this.getClass());
@Override
public List<PortDescription> getPorts() {
Bmv2Client deviceClient;
try {
deviceClient = Bmv2ThriftClient.of(handler().data().deviceId());
} catch (Bmv2RuntimeException e) {
log.error("Failed to connect to Bmv2 device", e);
return Collections.emptyList();
}
List<PortDescription> portDescriptions = Lists.newArrayList();
try {
deviceClient.getPortsInfo().forEach(
p -> {
DefaultAnnotations.Builder builder =
DefaultAnnotations.builder();
p.getExtraProperties().forEach(builder::set);
SparseAnnotations annotations = builder.build();
portDescriptions.add(new DefaultPortDescription(
PortNumber.portNumber(
(long) p.portNumber(),
p.ifaceName()),
p.isUp(),
annotations
));
});
} catch (Bmv2RuntimeException e) {
log.error("Unable to get port description from Bmv2 device", e);
}
return ImmutableList.copyOf(portDescriptions);
}
}
/*
* Copyright 2016-present Open Networking Laboratory
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.onosproject.drivers.bmv2.translators;
import com.google.common.annotations.Beta;
import org.onlab.util.ImmutableByteSequence;
import org.onosproject.bmv2.api.model.Bmv2ModelField;
import org.onosproject.bmv2.api.model.Bmv2ModelTable;
import org.onosproject.bmv2.api.model.Bmv2ModelTableKey;
import org.onosproject.bmv2.api.runtime.Bmv2Action;
import org.onosproject.bmv2.api.runtime.Bmv2ExtensionSelector;
import org.onosproject.bmv2.api.runtime.Bmv2ExtensionTreatment;
import org.onosproject.bmv2.api.runtime.Bmv2LpmMatchParam;
import org.onosproject.bmv2.api.runtime.Bmv2MatchKey;
import org.onosproject.bmv2.api.runtime.Bmv2TableEntry;
import org.onosproject.bmv2.api.runtime.Bmv2TernaryMatchParam;
import org.onosproject.net.flow.FlowRule;
import org.onosproject.net.flow.TrafficSelector;
import org.onosproject.net.flow.TrafficTreatment;
import org.onosproject.net.flow.criteria.Criterion;
import org.onosproject.net.flow.criteria.EthCriterion;
import org.onosproject.net.flow.criteria.EthTypeCriterion;
import org.onosproject.net.flow.criteria.ExtensionCriterion;
import org.onosproject.net.flow.criteria.ExtensionSelector;
import org.onosproject.net.flow.criteria.ExtensionSelectorType.ExtensionSelectorTypes;
import org.onosproject.net.flow.criteria.PortCriterion;
import org.onosproject.net.flow.instructions.ExtensionTreatment;
import org.onosproject.net.flow.instructions.ExtensionTreatmentType.ExtensionTreatmentTypes;
import org.onosproject.net.flow.instructions.Instruction;
import org.onosproject.net.flow.instructions.Instructions;
import org.onosproject.net.flow.instructions.Instructions.ExtensionInstructionWrapper;
/**
* Default Bmv2 flow rule translator implementation.
* <p>
* Flow rules are translated into {@link Bmv2TableEntry BMv2 table entries} according to the following logic:
* <ul>
* <li> table name: obtained from the Bmv2 model using the flow rule table ID;
* <li> match key: if the flow rule selector defines only a criterion of type {@link Criterion.Type#EXTENSION EXTENSION}
* , then the latter is expected to contain a {@link Bmv2ExtensionSelector Bmv2ExtensionSelector}, which should provide
* a match key already formatted for the given table; otherwise a match key is built using the
* {@link TranslatorConfig#fieldToCriterionTypeMap() mapping} defined by this translator configuration.
* <li> action: if the flow rule treatment contains only one instruction of type
* {@link Instruction.Type#EXTENSION EXTENSION}, then the latter is expected to contain a {@link Bmv2ExtensionTreatment}
* , which should provide a {@link Bmv2Action} already formatted for the given table; otherwise, an action is
* {@link TranslatorConfig#buildAction(TrafficTreatment) built} using this translator configuration.
* <li> priority: the same as the flow rule.
* <li> timeout: if the table supports timeout, use the same as the flow rule, otherwise none (i.e. permanent entry).
* </ul>
*/
@Beta
public class Bmv2DefaultFlowRuleTranslator implements Bmv2FlowRuleTranslator {
private final TranslatorConfig config;
public Bmv2DefaultFlowRuleTranslator(TranslatorConfig config) {
this.config = config;
}
private static Bmv2TernaryMatchParam buildTernaryParam(Bmv2ModelField field, Criterion criterion, int byteWidth)
throws Bmv2FlowRuleTranslatorException {
// Value and mask will be filled according to criterion type
ImmutableByteSequence value;
ImmutableByteSequence mask = null;
switch (criterion.type()) {
case IN_PORT:
// FIXME: allow port numbers of variable bit length (based on model), truncating when necessary
short port = (short) ((PortCriterion) criterion).port().toLong();
value = ImmutableByteSequence.copyFrom(port);
break;
case ETH_DST:
EthCriterion c = (EthCriterion) criterion;
value = ImmutableByteSequence.copyFrom(c.mac().toBytes());
if (c.mask() != null) {
mask = ImmutableByteSequence.copyFrom(c.mask().toBytes());
}
break;
case ETH_SRC:
EthCriterion c2 = (EthCriterion) criterion;
value = ImmutableByteSequence.copyFrom(c2.mac().toBytes());
if (c2.mask() != null) {
mask = ImmutableByteSequence.copyFrom(c2.mask().toBytes());
}
break;
case ETH_TYPE:
short ethType = ((EthTypeCriterion) criterion).ethType().toShort();
value = ImmutableByteSequence.copyFrom(ethType);
break;
// TODO: implement building for other criterion types (easy with DefaultCriterion of ONOS-4034)
default:
throw new Bmv2FlowRuleTranslatorException("Feature not implemented, ternary builder for criterion" +
"type: " + criterion.type().name());
}
if (mask == null) {
// no mask, all ones
mask = ImmutableByteSequence.ofOnes(byteWidth);
}
return new Bmv2TernaryMatchParam(value, mask);
}
private static Bmv2MatchKey getMatchKeyFromExtension(ExtensionCriterion criterion)
throws Bmv2FlowRuleTranslatorException {
ExtensionSelector extSelector = criterion.extensionSelector();
if (extSelector.type() == ExtensionSelectorTypes.P4_BMV2_MATCH_KEY.type()) {
if (extSelector instanceof Bmv2ExtensionSelector) {
return ((Bmv2ExtensionSelector) extSelector).matchKey();
} else {
throw new Bmv2FlowRuleTranslatorException("Unable to decode extension selector " + extSelector);
}
} else {
throw new Bmv2FlowRuleTranslatorException("Unsupported extension selector type " + extSelector.type());
}
}
private static Bmv2Action getActionFromExtension(Instructions.ExtensionInstructionWrapper inst)
throws Bmv2FlowRuleTranslatorException {
ExtensionTreatment extTreatment = inst.extensionInstruction();
if (extTreatment.type() == ExtensionTreatmentTypes.P4_BMV2_ACTION.type()) {
if (extTreatment instanceof Bmv2ExtensionTreatment) {
return ((Bmv2ExtensionTreatment) extTreatment).getAction();
} else {
throw new Bmv2FlowRuleTranslatorException("Unable to decode treatment extension: " + extTreatment);
}
} else {
throw new Bmv2FlowRuleTranslatorException("Unsupported treatment extension type: " + extTreatment.type());
}
}
private static Bmv2MatchKey buildMatchKey(TranslatorConfig config, TrafficSelector selector, Bmv2ModelTable table)
throws Bmv2FlowRuleTranslatorException {
Bmv2MatchKey.Builder matchKeyBuilder = Bmv2MatchKey.builder();
for (Bmv2ModelTableKey key : table.keys()) {
String fieldName = key.field().header().name() + "." + key.field().type().name();
int byteWidth = (int) Math.ceil((double) key.field().type().bitWidth() / 8.0);
Criterion.Type criterionType = config.fieldToCriterionTypeMap().get(fieldName);
if (criterionType == null || selector.getCriterion(criterionType) == null) {
// A mapping is not available or the selector doesn't have such a type
switch (key.matchType()) {
case TERNARY:
// Wildcard field
matchKeyBuilder.withWildcard(byteWidth);
break;
case LPM:
// LPM with prefix 0
matchKeyBuilder.add(new Bmv2LpmMatchParam(ImmutableByteSequence.ofZeros(byteWidth), 0));
break;
default:
throw new Bmv2FlowRuleTranslatorException("Match field not supported: " + fieldName);
}
// Next key
continue;
}
Criterion criterion = selector.getCriterion(criterionType);
Bmv2TernaryMatchParam matchParam = null;
switch (key.matchType()) {
case TERNARY:
matchParam = buildTernaryParam(key.field(), criterion, byteWidth);
break;
default:
// TODO: implement other match param builders (exact, LPM, etc.)
throw new Bmv2FlowRuleTranslatorException("Feature not implemented, match param builder: "
+ key.matchType().name());
}
matchKeyBuilder.add(matchParam);
}
return matchKeyBuilder.build();
}
@Override
public Bmv2TableEntry translate(FlowRule rule)
throws Bmv2FlowRuleTranslatorException {
int tableId = rule.tableId();
Bmv2ModelTable table = config.model().table(tableId);
if (table == null) {
throw new Bmv2FlowRuleTranslatorException("Unknown table ID: " + tableId);
}
/* Translate selector */
TrafficSelector selector = rule.selector();
Bmv2MatchKey bmv2MatchKey = null;
// If selector has only 1 criterion of type extension, use that
Criterion criterion = selector.getCriterion(Criterion.Type.EXTENSION);
if (criterion != null) {
if (selector.criteria().size() == 1) {
bmv2MatchKey = getMatchKeyFromExtension((ExtensionCriterion) criterion);
} else {
throw new Bmv2FlowRuleTranslatorException("Unable to translate traffic selector, found multiple " +
"criteria of which one is an extension: " +
selector.toString());
}
}
if (bmv2MatchKey == null) {
// not an extension
bmv2MatchKey = buildMatchKey(config, selector, table);
}
/* Translate treatment */
TrafficTreatment treatment = rule.treatment();
Bmv2Action bmv2Action = null;
// If treatment has only 1 instruction of type extension, use that
for (Instruction inst : treatment.allInstructions()) {
if (inst.type() == Instruction.Type.EXTENSION) {
if (treatment.allInstructions().size() == 1) {
bmv2Action = getActionFromExtension((ExtensionInstructionWrapper) inst);
} else {
throw new Bmv2FlowRuleTranslatorException("Unable to translate traffic treatment, found multiple " +
"instructions of which one is an extension: " +
selector.toString());
}
}
}
if (bmv2Action == null) {
// No extension, use config to build action
bmv2Action = config.buildAction(treatment);
}
if (bmv2Action == null) {
// Config returned null
throw new Bmv2FlowRuleTranslatorException("Unable to translate treatment: " + treatment);
}
Bmv2TableEntry.Builder tableEntryBuilder = Bmv2TableEntry.builder();
// In BMv2 0 is the highest priority, i.e. the opposite than ONOS.
int newPriority = Integer.MAX_VALUE - rule.priority();
tableEntryBuilder
.withTableName(table.name())
.withPriority(newPriority)
.withMatchKey(bmv2MatchKey)
.withAction(bmv2Action);
if (!rule.isPermanent()) {
if (table.hasTimeouts()) {
tableEntryBuilder.withTimeout((double) rule.timeout());
}
//FIXME: add warn log or exception?
}
return tableEntryBuilder.build();
}
@Override
public TranslatorConfig config() {
return this.config;
}
}
/*
* Copyright 2016-present Open Networking Laboratory
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.onosproject.drivers.bmv2.translators;
import com.google.common.annotations.Beta;
import org.onosproject.bmv2.api.model.Bmv2Model;
import org.onosproject.net.flow.criteria.Criterion;
import java.util.Map;
import static org.onosproject.drivers.bmv2.translators.Bmv2FlowRuleTranslator.TranslatorConfig;
/**
* Default implementation of a BMv2 flow rule translator configuration.
*/
@Beta
public abstract class Bmv2DefaultTranslatorConfig implements TranslatorConfig {
private final Bmv2Model model;
private final Map<String, Criterion.Type> fieldMap;
/**
* Creates a new translator configuration.
*
* @param model a BMv2 packet processing model
* @param fieldMap a field-to-criterion type map
*/
protected Bmv2DefaultTranslatorConfig(Bmv2Model model, Map<String, Criterion.Type> fieldMap) {
this.model = model;
this.fieldMap = fieldMap;
}
@Override
public Bmv2Model model() {
return this.model;
}
@Override
public Map<String, Criterion.Type> fieldToCriterionTypeMap() {
return this.fieldMap;
}
}
/*
* Copyright 2016-present Open Networking Laboratory
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.onosproject.drivers.bmv2.translators;
import com.google.common.annotations.Beta;
import org.onosproject.bmv2.api.model.Bmv2Model;
import org.onosproject.bmv2.api.runtime.Bmv2Action;
import org.onosproject.bmv2.api.runtime.Bmv2TableEntry;
import org.onosproject.net.flow.FlowRule;
import org.onosproject.net.flow.TrafficTreatment;
import org.onosproject.net.flow.criteria.Criterion;
import java.util.Map;
/**
* Translator of ONOS flow rules to BMv2 table entries. Translation depends on a
* {@link TranslatorConfig translator configuration}.
*/
@Beta
public interface Bmv2FlowRuleTranslator {
/**
* Returns a new BMv2 table entry equivalent to the given flow rule.
*
* @param rule a flow rule
* @return a BMv2 table entry
* @throws Bmv2FlowRuleTranslatorException if the flow rule cannot be
* translated
*/
Bmv2TableEntry translate(FlowRule rule) throws Bmv2FlowRuleTranslatorException;
/**
* Returns the configuration of this translator.
*
* @return a translator configuration
*/
TranslatorConfig config();
/**
* BMv2 flow rules translator configuration. Such a configuration is used to
* generate table entries that are compatible with a given {@link Bmv2Model}.
*/
@Beta
interface TranslatorConfig {
/**
* Return the {@link Bmv2Model} associated with this configuration.
*
* @return a BMv2 model
*/
Bmv2Model model();
/**
* Returns a map describing a one-to-one relationship between BMv2
* header field names and ONOS criterion types. Header field names are
* formatted using the notation {@code header_name.field_name}
* representing a specific header field instance extracted by the BMv2
* parser (e.g. {@code ethernet.dstAddr}).
*
* @return a map where the keys represent BMv2 header field names and
* values are criterion types
*/
Map<String, Criterion.Type> fieldToCriterionTypeMap();
/**
* Return a BMv2 action that is equivalent to the given ONOS traffic
* treatment.
*
* @param treatment a traffic treatment
* @return a BMv2 action object
* @throws Bmv2FlowRuleTranslatorException if the treatment cannot be
* translated to a BMv2 action
*/
Bmv2Action buildAction(TrafficTreatment treatment)
throws Bmv2FlowRuleTranslatorException;
}
}
/*
* Copyright 2016-present Open Networking Laboratory
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.onosproject.drivers.bmv2.translators;
import com.google.common.annotations.Beta;
import com.google.common.collect.ImmutableMap;
import org.onlab.util.ImmutableByteSequence;
import org.onosproject.bmv2.api.model.Bmv2Model;
import org.onosproject.bmv2.api.runtime.Bmv2Action;
import org.onosproject.net.PortNumber;
import org.onosproject.net.flow.TrafficTreatment;
import org.onosproject.net.flow.criteria.Criterion;
import org.onosproject.net.flow.instructions.Instruction;
import org.onosproject.net.flow.instructions.Instructions;
import java.util.Map;
/**
* Implementation of a Bmv2 flow rule translator configuration for the
* simple.p4 model.
*/
@Beta
public class Bmv2SimpleTranslatorConfig extends Bmv2DefaultTranslatorConfig {
// Lazily populate field map.
private static final Map<String, Criterion.Type> FIELD_MAP = ImmutableMap.of(
"standard_metadata.ingress_port", Criterion.Type.IN_PORT,
"ethernet.dstAddr", Criterion.Type.ETH_DST,
"ethernet.srcAddr", Criterion.Type.ETH_SRC,
"ethernet.etherType", Criterion.Type.ETH_TYPE);
/**
* Creates a new simple pipeline translator configuration.
*/
public Bmv2SimpleTranslatorConfig(Bmv2Model model) {
// Populate fieldMap.
super(model, FIELD_MAP);
}
private static Bmv2Action buildDropAction() {
return Bmv2Action.builder()
.withName("_drop")
.build();
}
private static Bmv2Action buildPushToCpAction() {
return Bmv2Action.builder()
.withName("send_to_cpu")
.build();
}
private static Bmv2Action buildFwdAction(Instructions.OutputInstruction inst)
throws Bmv2FlowRuleTranslatorException {
Bmv2Action.Builder actionBuilder = Bmv2Action.builder();
actionBuilder.withName("fwd");
if (inst.port().isLogical()) {
if (inst.port() == PortNumber.CONTROLLER) {
return buildPushToCpAction();
} else {
throw new Bmv2FlowRuleTranslatorException(
"Output logic port number not supported: " + inst);
}
}
actionBuilder.addParameter(
ImmutableByteSequence.copyFrom((short) inst.port().toLong()));
return actionBuilder.build();
}
@Override
public Bmv2Action buildAction(TrafficTreatment treatment)
throws Bmv2FlowRuleTranslatorException {
if (treatment.allInstructions().size() == 0) {
// No instructions means drop.
return buildDropAction();
} else if (treatment.allInstructions().size() > 1) {
// Otherwise, we understand treatments with only 1 instruction.
throw new Bmv2FlowRuleTranslatorException(
"Treatment not supported, more than 1 instructions found: "
+ treatment.toString());
}
Instruction instruction = treatment.allInstructions().get(0);
switch (instruction.type()) {
case OUTPUT:
return buildFwdAction((Instructions.OutputInstruction) instruction);
case NOACTION:
return buildDropAction();
default:
throw new Bmv2FlowRuleTranslatorException(
"Instruction type not supported: "
+ instruction.type().name());
}
}
}
......@@ -16,14 +16,18 @@
~ limitations under the License.
-->
<drivers>
<driver name="bmv2-thrift" manufacturer="p4.org" hwVersion="bmv2" swVersion="unknown">
<behaviour api="org.onosproject.net.behaviour.PortDiscovery"
impl="org.onosproject.drivers.bmv2.Bmv2PortDiscovery"/>
<driver name="bmv2-thrift" manufacturer="p4.org" hwVersion="bmv2" swVersion="n/a">
<behaviour api="org.onosproject.net.device.DeviceDescriptionDiscovery"
impl="org.onosproject.drivers.bmv2.Bmv2DeviceDescriptionDiscovery"/>
<behaviour api="org.onosproject.net.flow.FlowRuleProgrammable"
impl="org.onosproject.drivers.bmv2.Bmv2FlowRuleProgrammable"/>
<behaviour api="org.onosproject.net.behaviour.Pipeliner"
impl="org.onosproject.drivers.bmv2.Bmv2Pipeliner"/>
<behaviour api="org.onosproject.net.packet.PacketProgrammable"
impl="org.onosproject.drivers.bmv2.Bmv2PacketProgrammable"/>
<behaviour api="org.onosproject.net.behaviour.ExtensionSelectorResolver"
impl="org.onosproject.drivers.bmv2.Bmv2ExtensionSelectorResolver"/>
<behaviour api="org.onosproject.net.behaviour.ExtensionTreatmentResolver"
impl="org.onosproject.drivers.bmv2.Bmv2ExtensionTreatmentResolver"/>
</driver>
</drivers>
......
/*
* Copyright 2014-2016 Open Networking Laboratory
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.onosproject.drivers.bmv2;
import com.eclipsesource.json.Json;
import com.eclipsesource.json.JsonObject;
import com.google.common.testing.EqualsTester;
import org.junit.Before;
import org.junit.Test;
import org.onlab.packet.MacAddress;
import org.onosproject.bmv2.api.model.Bmv2Model;
import org.onosproject.bmv2.api.runtime.Bmv2TableEntry;
import org.onosproject.bmv2.api.runtime.Bmv2TernaryMatchParam;
import org.onosproject.core.ApplicationId;
import org.onosproject.core.DefaultApplicationId;
import org.onosproject.drivers.bmv2.translators.Bmv2DefaultFlowRuleTranslator;
import org.onosproject.drivers.bmv2.translators.Bmv2FlowRuleTranslator;
import org.onosproject.drivers.bmv2.translators.Bmv2SimpleTranslatorConfig;
import org.onosproject.net.DeviceId;
import org.onosproject.net.PortNumber;
import org.onosproject.net.flow.DefaultFlowRule;
import org.onosproject.net.flow.DefaultTrafficSelector;
import org.onosproject.net.flow.DefaultTrafficTreatment;
import org.onosproject.net.flow.FlowRule;
import org.onosproject.net.flow.TrafficSelector;
import org.onosproject.net.flow.TrafficTreatment;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.Random;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.MatcherAssert.assertThat;
/**
* Tests for {@link Bmv2DefaultFlowRuleTranslator}.
*/
public class Bmv2DefaultFlowRuleTranslatorTest {
private static final String JSON_CONFIG_PATH = "/simple.json";
private Random random = new Random();
private Bmv2Model model;
private Bmv2FlowRuleTranslator.TranslatorConfig config;
private Bmv2FlowRuleTranslator translator;
@Before
public void setUp() throws Exception {
InputStream inputStream = Bmv2SimpleTranslatorConfig.class
.getResourceAsStream(JSON_CONFIG_PATH);
InputStreamReader reader = new InputStreamReader(inputStream);
BufferedReader bufReader = new BufferedReader(reader);
JsonObject json = null;
try {
json = Json.parse(bufReader).asObject();
} catch (IOException e) {
throw new RuntimeException("Unable to parse JSON file: " + e.getMessage());
}
this.model = Bmv2Model.parse(json);
this.config = new Bmv2SimpleTranslatorConfig(model);
this.translator = new Bmv2DefaultFlowRuleTranslator(config);
}
@Test
public void testCompiler() throws Exception {
DeviceId deviceId = DeviceId.NONE;
ApplicationId appId = new DefaultApplicationId(1, "test");
int tableId = 0;
MacAddress ethDstMac = MacAddress.valueOf(random.nextLong());
MacAddress ethSrcMac = MacAddress.valueOf(random.nextLong());
short ethType = (short) (0x0000FFFF & random.nextInt());
short outPort = (short) random.nextInt(65);
short inPort = (short) random.nextInt(65);
int timeout = random.nextInt(100);
int priority = random.nextInt(100);
TrafficSelector matchInPort1 = DefaultTrafficSelector
.builder()
.matchInPort(PortNumber.portNumber(inPort))
.matchEthDst(ethDstMac)
.matchEthSrc(ethSrcMac)
.matchEthType(ethType)
.build();
TrafficTreatment outPort2 = DefaultTrafficTreatment
.builder()
.setOutput(PortNumber.portNumber(outPort))
.build();
FlowRule rule1 = DefaultFlowRule.builder()
.forDevice(deviceId)
.forTable(tableId)
.fromApp(appId)
.withSelector(matchInPort1)
.withTreatment(outPort2)
.makeTemporary(timeout)
.withPriority(priority)
.build();
FlowRule rule2 = DefaultFlowRule.builder()
.forDevice(deviceId)
.forTable(tableId)
.fromApp(appId)
.withSelector(matchInPort1)
.withTreatment(outPort2)
.makeTemporary(timeout)
.withPriority(priority)
.build();
Bmv2TableEntry entry1 = translator.translate(rule1);
Bmv2TableEntry entry2 = translator.translate(rule1);
// check equality, i.e. same rules must produce same entries
new EqualsTester()
.addEqualityGroup(rule1, rule2)
.addEqualityGroup(entry1, entry2)
.testEquals();
int numMatchParams = model.table(0).keys().size();
// parse values stored in entry1
Bmv2TernaryMatchParam inPortParam = (Bmv2TernaryMatchParam) entry1.matchKey().matchParams().get(0);
Bmv2TernaryMatchParam ethDstParam = (Bmv2TernaryMatchParam) entry1.matchKey().matchParams().get(1);
Bmv2TernaryMatchParam ethSrcParam = (Bmv2TernaryMatchParam) entry1.matchKey().matchParams().get(2);
Bmv2TernaryMatchParam ethTypeParam = (Bmv2TernaryMatchParam) entry1.matchKey().matchParams().get(3);
double expectedTimeout = (double) (model.table(0).hasTimeouts() ? rule1.timeout() : -1);
// check that the number of parameters in the entry is the same as the number of table keys
assertThat("Incorrect number of match parameters",
entry1.matchKey().matchParams().size(), is(equalTo(numMatchParams)));
// check that values stored in entry are the same used for the flow rule
assertThat("Incorrect inPort match param value",
inPortParam.value().asReadOnlyBuffer().getShort(), is(equalTo(inPort)));
assertThat("Incorrect ethDestMac match param value",
ethDstParam.value().asArray(), is(equalTo(ethDstMac.toBytes())));
assertThat("Incorrect ethSrcMac match param value",
ethSrcParam.value().asArray(), is(equalTo(ethSrcMac.toBytes())));
assertThat("Incorrect ethType match param value",
ethTypeParam.value().asReadOnlyBuffer().getShort(), is(equalTo(ethType)));
assertThat("Incorrect priority value",
entry1.priority(), is(equalTo(Integer.MAX_VALUE - rule1.priority())));
assertThat("Incorrect timeout value",
entry1.timeout(), is(equalTo(expectedTimeout)));
}
}
\ No newline at end of file
{
"header_types": [
{
"name": "standard_metadata_t",
"id": 0,
"fields": [
[
"ingress_port",
9
],
[
"packet_length",
32
],
[
"egress_spec",
9
],
[
"egress_port",
9
],
[
"egress_instance",
32
],
[
"instance_type",
32
],
[
"clone_spec",
32
],
[
"_padding",
5
]
],
"length_exp": null,
"max_length": null
},
{
"name": "ethernet_t",
"id": 1,
"fields": [
[
"dstAddr",
48
],
[
"srcAddr",
48
],
[
"etherType",
16
]
],
"length_exp": null,
"max_length": null
},
{
"name": "intrinsic_metadata_t",
"id": 2,
"fields": [
[
"ingress_global_timestamp",
32
],
[
"lf_field_list",
32
],
[
"mcast_grp",
16
],
[
"egress_rid",
16
]
],
"length_exp": null,
"max_length": null
}
],
"headers": [
{
"name": "standard_metadata",
"id": 0,
"header_type": "standard_metadata_t",
"metadata": true
},
{
"name": "ethernet",
"id": 1,
"header_type": "ethernet_t",
"metadata": false
},
{
"name": "intrinsic_metadata",
"id": 2,
"header_type": "intrinsic_metadata_t",
"metadata": true
}
],
"header_stacks": [],
"parsers": [
{
"name": "parser",
"id": 0,
"init_state": "start",
"parse_states": [
{
"name": "start",
"id": 0,
"parser_ops": [],
"transition_key": [],
"transitions": [
{
"value": "default",
"mask": null,
"next_state": "parse_ethernet"
}
]
},
{
"name": "parse_ethernet",
"id": 1,
"parser_ops": [
{
"op": "extract",
"parameters": [
{
"type": "regular",
"value": "ethernet"
}
]
}
],
"transition_key": [],
"transitions": [
{
"value": "default",
"mask": null,
"next_state": null
}
]
}
]
}
],
"deparsers": [
{
"name": "deparser",
"id": 0,
"order": [
"ethernet"
]
}
],
"meter_arrays": [],
"actions": [
{
"name": "flood",
"id": 0,
"runtime_data": [],
"primitives": [
{
"op": "modify_field",
"parameters": [
{
"type": "field",
"value": [
"intrinsic_metadata",
"mcast_grp"
]
},
{
"type": "field",
"value": [
"standard_metadata",
"ingress_port"
]
}
]
}
]
},
{
"name": "_drop",
"id": 1,
"runtime_data": [],
"primitives": [
{
"op": "modify_field",
"parameters": [
{
"type": "field",
"value": [
"standard_metadata",
"egress_spec"
]
},
{
"type": "hexstr",
"value": "0x1ff"
}
]
}
]
},
{
"name": "fwd",
"id": 2,
"runtime_data": [
{
"name": "port",
"bitwidth": 9
}
],
"primitives": [
{
"op": "modify_field",
"parameters": [
{
"type": "field",
"value": [
"standard_metadata",
"egress_spec"
]
},
{
"type": "runtime_data",
"value": 0
}
]
}
]
},
{
"name": "push_to_cp",
"id": 3,
"runtime_data": [],
"primitives": [
{
"op": "modify_field",
"parameters": [
{
"type": "field",
"value": [
"standard_metadata",
"egress_spec"
]
},
{
"type": "hexstr",
"value": "0x200"
}
]
}
]
}
],
"pipelines": [
{
"name": "ingress",
"id": 0,
"init_table": "table0",
"tables": [
{
"name": "table0",
"id": 0,
"match_type": "ternary",
"type": "simple",
"max_size": 16384,
"with_counters": false,
"direct_meters": null,
"support_timeout": false,
"key": [
{
"match_type": "ternary",
"target": [
"standard_metadata",
"ingress_port"
],
"mask": null
},
{
"match_type": "ternary",
"target": [
"ethernet",
"dstAddr"
],
"mask": null
},
{
"match_type": "ternary",
"target": [
"ethernet",
"srcAddr"
],
"mask": null
},
{
"match_type": "ternary",
"target": [
"ethernet",
"etherType"
],
"mask": null
}
],
"actions": [
"fwd",
"flood",
"push_to_cp",
"_drop"
],
"next_tables": {
"fwd": null,
"flood": null,
"push_to_cp": null,
"_drop": null
},
"default_action": null,
"base_default_next": null
}
],
"conditionals": []
},
{
"name": "egress",
"id": 1,
"init_table": null,
"tables": [],
"conditionals": []
}
],
"calculations": [],
"checksums": [],
"learn_lists": [],
"field_lists": [],
"counter_arrays": [],
"register_arrays": [],
"force_arith": [
[
"standard_metadata",
"ingress_port"
],
[
"standard_metadata",
"packet_length"
],
[
"standard_metadata",
"egress_spec"
],
[
"standard_metadata",
"egress_port"
],
[
"standard_metadata",
"egress_instance"
],
[
"standard_metadata",
"instance_type"
],
[
"standard_metadata",
"clone_spec"
],
[
"standard_metadata",
"_padding"
],
[
"intrinsic_metadata",
"ingress_global_timestamp"
],
[
"intrinsic_metadata",
"lf_field_list"
],
[
"intrinsic_metadata",
"mcast_grp"
],
[
"intrinsic_metadata",
"egress_rid"
]
]
}
\ No newline at end of file
......@@ -41,7 +41,7 @@
<module>utilities</module>
<module>lumentum</module>
<module>bti</module>
<!--<module>bmv2</module>-->
<module>bmv2</module>
<module>corsa</module>
<module>optical</module>
</modules>
......
......@@ -21,7 +21,8 @@
<parent>
<artifactId>onos-bmv2-protocol</artifactId>
<groupId>org.onosproject</groupId>
<version>1.6.0-SNAPSHOT</version>
<version>1.7.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
......
......@@ -21,7 +21,8 @@
<parent>
<artifactId>onos-bmv2-protocol</artifactId>
<groupId>org.onosproject</groupId>
<version>1.6.0-SNAPSHOT</version>
<version>1.7.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
......
......@@ -22,6 +22,7 @@
<artifactId>onos-protocols</artifactId>
<groupId>org.onosproject</groupId>
<version>1.7.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
......
......@@ -21,7 +21,8 @@
<parent>
<artifactId>onos-bmv2-protocol</artifactId>
<groupId>org.onosproject</groupId>
<version>1.6.0-SNAPSHOT</version>
<version>1.7.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
......
......@@ -22,7 +22,9 @@
<bundle>mvn:${project.groupId}/onos-bmv2-provider-device/${project.version}</bundle>
<bundle>mvn:${project.groupId}/onos-bmv2-provider-packet/${project.version}</bundle>
<bundle>mvn:org.apache.thrift/libthrift/0.9.3</bundle>
<bundle>mvn:${project.groupId}/onos-bmv2-protocol/${project.version}</bundle>
<bundle>mvn:${project.groupId}/onos-bmv2-protocol-api/${project.version}</bundle>
<bundle>mvn:${project.groupId}/onos-bmv2-protocol-ctl/${project.version}</bundle>
<bundle>mvn:${project.groupId}/onos-bmv2-protocol-thrift-api/${project.version}</bundle>
</feature>
</features>
......
......@@ -22,6 +22,7 @@
<artifactId>onos-bmv2-providers</artifactId>
<groupId>org.onosproject</groupId>
<version>1.7.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
......@@ -33,7 +34,7 @@
<properties>
<onos.app.name>org.onosproject.bmv2</onos.app.name>
<onos.app.title>BMv2 Provider</onos.app.title>
<onos.app.title>BMv2 Providers</onos.app.title>
<onos.app.category>Provider</onos.app.category>
</properties>
......
......@@ -22,6 +22,7 @@
<artifactId>onos-bmv2-providers</artifactId>
<groupId>org.onosproject</groupId>
<version>1.7.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
......@@ -33,12 +34,12 @@
<dependencies>
<dependency>
<groupId>org.onosproject</groupId>
<artifactId>onos-drivers-bmv2</artifactId>
<artifactId>onos-core-common</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.onosproject</groupId>
<artifactId>onos-core-common</artifactId>
<artifactId>onos-bmv2-protocol-api</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
......
......@@ -23,36 +23,38 @@ import org.apache.felix.scr.annotations.ReferenceCardinality;
import org.jboss.netty.util.HashedWheelTimer;
import org.jboss.netty.util.Timeout;
import org.jboss.netty.util.TimerTask;
import org.onlab.packet.ChassisId;
import org.onlab.util.HexString;
import org.onlab.util.Timer;
import org.onosproject.bmv2.api.runtime.Bmv2ControlPlaneServer;
import org.onosproject.bmv2.api.runtime.Bmv2Device;
import org.onosproject.bmv2.api.runtime.Bmv2RuntimeException;
import org.onosproject.bmv2.ctl.Bmv2ThriftClient;
import org.onosproject.bmv2.api.service.Bmv2Controller;
import org.onosproject.bmv2.api.service.Bmv2DeviceContextService;
import org.onosproject.bmv2.api.service.Bmv2DeviceListener;
import org.onosproject.common.net.AbstractDeviceProvider;
import org.onosproject.core.ApplicationId;
import org.onosproject.core.CoreService;
import org.onosproject.incubator.net.config.basics.ConfigException;
import org.onosproject.net.AnnotationKeys;
import org.onosproject.net.DefaultAnnotations;
import org.onosproject.net.Device;
import org.onosproject.net.DeviceId;
import org.onosproject.net.MastershipRole;
import org.onosproject.net.PortNumber;
import org.onosproject.net.behaviour.PortDiscovery;
import org.onosproject.net.config.ConfigFactory;
import org.onosproject.net.config.NetworkConfigEvent;
import org.onosproject.net.config.NetworkConfigListener;
import org.onosproject.net.config.NetworkConfigRegistry;
import org.onosproject.net.device.DefaultDeviceDescription;
import org.onosproject.net.device.DeviceDescription;
import org.onosproject.net.device.DeviceDescriptionDiscovery;
import org.onosproject.net.device.DeviceService;
import org.onosproject.net.device.PortDescription;
import org.onosproject.net.device.PortStatistics;
import org.onosproject.net.driver.DefaultDriverData;
import org.onosproject.net.driver.DefaultDriverHandler;
import org.onosproject.net.driver.Driver;
import org.onosproject.net.provider.ProviderId;
import org.slf4j.Logger;
import java.util.Collection;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
......@@ -60,9 +62,9 @@ import java.util.concurrent.TimeUnit;
import static org.onlab.util.Tools.groupedThreads;
import static org.onosproject.bmv2.api.runtime.Bmv2Device.*;
import static org.onosproject.bmv2.ctl.Bmv2ThriftClient.forceDisconnectOf;
import static org.onosproject.bmv2.ctl.Bmv2ThriftClient.ping;
import static org.onosproject.net.config.basics.SubjectFactories.APP_SUBJECT_FACTORY;
import static org.onosproject.provider.bmv2.device.impl.Bmv2PortStatisticsGetter.getPortStatistics;
import static org.onosproject.provider.bmv2.device.impl.Bmv2PortStatisticsGetter.initCounters;
import static org.slf4j.LoggerFactory.getLogger;
/**
......@@ -71,11 +73,24 @@ import static org.slf4j.LoggerFactory.getLogger;
@Component(immediate = true)
public class Bmv2DeviceProvider extends AbstractDeviceProvider {
private static final Logger LOG = getLogger(Bmv2DeviceProvider.class);
private static final String APP_NAME = "org.onosproject.bmv2";
private static final String UNKNOWN = "unknown";
private static final int POLL_INTERVAL = 5; // seconds
private static final int POLL_INTERVAL = 5_000; // milliseconds
private final Logger log = getLogger(this.getClass());
private final ExecutorService executorService = Executors
.newFixedThreadPool(16, groupedThreads("onos/bmv2", "device-discovery", log));
private final NetworkConfigListener cfgListener = new InternalNetworkConfigListener();
private final ConfigFactory cfgFactory = new InternalConfigFactory();
private final ConcurrentMap<DeviceId, DeviceDescription> activeDevices = Maps.newConcurrentMap();
private final DevicePoller devicePoller = new DevicePoller();
private final InternalDeviceListener deviceListener = new InternalDeviceListener();
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
protected NetworkConfigRegistry netCfgService;
......@@ -87,16 +102,11 @@ public class Bmv2DeviceProvider extends AbstractDeviceProvider {
protected DeviceService deviceService;
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
protected Bmv2ControlPlaneServer controlPlaneServer;
protected Bmv2Controller controller;
private final ExecutorService deviceDiscoveryExecutor = Executors
.newFixedThreadPool(5, groupedThreads("onos/bmv2", "device-discovery", LOG));
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
protected Bmv2DeviceContextService contextService;
private final NetworkConfigListener cfgListener = new InternalNetworkConfigListener();
private final ConfigFactory cfgFactory = new InternalConfigFactory();
private final ConcurrentMap<DeviceId, Boolean> activeDevices = Maps.newConcurrentMap();
private final DevicePoller devicePoller = new DevicePoller();
private final InternalHelloListener helloListener = new InternalHelloListener();
private ApplicationId appId;
/**
......@@ -111,7 +121,7 @@ public class Bmv2DeviceProvider extends AbstractDeviceProvider {
appId = coreService.registerApplication(APP_NAME);
netCfgService.registerConfigFactory(cfgFactory);
netCfgService.addListener(cfgListener);
controlPlaneServer.addHelloListener(helloListener);
controller.addDeviceListener(deviceListener);
devicePoller.start();
super.activate();
}
......@@ -119,16 +129,16 @@ public class Bmv2DeviceProvider extends AbstractDeviceProvider {
@Override
protected void deactivate() {
devicePoller.stop();
controlPlaneServer.removeHelloListener(helloListener);
controller.removeDeviceListener(deviceListener);
try {
activeDevices.forEach((did, value) -> {
deviceDiscoveryExecutor.execute(() -> disconnectDevice(did));
executorService.execute(() -> disconnectDevice(did));
});
deviceDiscoveryExecutor.awaitTermination(1000, TimeUnit.MILLISECONDS);
executorService.awaitTermination(1000, TimeUnit.MILLISECONDS);
} catch (InterruptedException e) {
LOG.error("Device discovery threads did not terminate");
log.error("Device discovery threads did not terminate");
}
deviceDiscoveryExecutor.shutdownNow();
executorService.shutdownNow();
netCfgService.unregisterConfigFactory(cfgFactory);
netCfgService.removeListener(cfgListener);
super.deactivate();
......@@ -137,14 +147,12 @@ public class Bmv2DeviceProvider extends AbstractDeviceProvider {
@Override
public void triggerProbe(DeviceId deviceId) {
// Asynchronously trigger probe task.
deviceDiscoveryExecutor.execute(() -> executeProbe(deviceId));
executorService.execute(() -> executeProbe(deviceId));
}
private void executeProbe(DeviceId did) {
boolean reachable = isReachable(did);
LOG.debug("Probed device: id={}, reachable={}",
did.toString(),
reachable);
log.debug("Probed device: id={}, reachable={}", did.toString(), reachable);
if (reachable) {
discoverDevice(did);
} else {
......@@ -154,85 +162,108 @@ public class Bmv2DeviceProvider extends AbstractDeviceProvider {
@Override
public void roleChanged(DeviceId deviceId, MastershipRole newRole) {
LOG.debug("roleChanged() is not yet implemented");
log.debug("roleChanged() is not yet implemented");
// TODO: implement mastership handling
}
@Override
public boolean isReachable(DeviceId deviceId) {
return ping(deviceId);
return controller.isReacheable(deviceId);
}
@Override
public void changePortState(DeviceId deviceId, PortNumber portNumber, boolean enable) {
LOG.debug("changePortState() is not yet implemented");
// TODO: implement port handling
log.warn("changePortState() not supported");
}
private void discoverDevice(DeviceId did) {
LOG.debug("Starting device discovery... deviceId={}", did);
// Atomically notify device to core and update port information.
activeDevices.compute(did, (k, v) -> {
if (!deviceService.isAvailable(did)) {
// Device not available in the core, connect it now.
DefaultAnnotations.Builder annotationsBuilder = DefaultAnnotations.builder()
.set(AnnotationKeys.PROTOCOL, SCHEME);
dumpJsonConfigToAnnotations(did, annotationsBuilder);
DeviceDescription descr = new DefaultDeviceDescription(
did.uri(), Device.Type.SWITCH, MANUFACTURER, HW_VERSION,
UNKNOWN, UNKNOWN, new ChassisId(), annotationsBuilder.build());
// Reset device state (cleanup entries, etc.)
resetDeviceState(did);
providerService.deviceConnected(did, descr);
log.debug("Starting device discovery... deviceId={}", did);
activeDevices.compute(did, (k, lastDescription) -> {
DeviceDescription thisDescription = getDeviceDescription(did);
if (thisDescription != null) {
boolean descriptionChanged = lastDescription != null &&
(!Objects.equals(thisDescription, lastDescription) ||
!Objects.equals(thisDescription.annotations(), lastDescription.annotations()));
if (descriptionChanged || !deviceService.isAvailable(did)) {
// Device description changed or device not available in the core.
if (contextService.notifyDeviceChange(did)) {
// Configuration is the expected one, we can proceed notifying the core.
resetDeviceState(did);
initPortCounters(did);
providerService.deviceConnected(did, thisDescription);
updatePortsAndStats(did);
}
}
return thisDescription;
} else {
log.warn("Unable to get device description for {}", did);
return lastDescription;
}
updatePorts(did);
return true;
});
}
private void dumpJsonConfigToAnnotations(DeviceId did, DefaultAnnotations.Builder builder) {
// TODO: store json config string somewhere else, possibly in a Bmv2Controller (see ONOS-4419)
private DeviceDescription getDeviceDescription(DeviceId did) {
Device device = deviceService.getDevice(did);
DeviceDescriptionDiscovery discovery = null;
if (device == null) {
// Device not yet in the core. Manually get a driver.
Driver driver = driverService.getDriver(MANUFACTURER, HW_VERSION, SW_VERSION);
if (driver.hasBehaviour(DeviceDescriptionDiscovery.class)) {
discovery = driver.createBehaviour(new DefaultDriverHandler(new DefaultDriverData(driver, did)),
DeviceDescriptionDiscovery.class);
}
} else if (device.is(DeviceDescriptionDiscovery.class)) {
discovery = device.as(DeviceDescriptionDiscovery.class);
}
if (discovery == null) {
log.warn("No DeviceDescriptionDiscovery behavior for device {}", did);
return null;
} else {
return discovery.discoverDeviceDetails();
}
}
private void resetDeviceState(DeviceId did) {
try {
String md5 = Bmv2ThriftClient.of(did).getJsonConfigMd5();
// Convert to hex string for readability.
md5 = HexString.toHexString(md5.getBytes());
String jsonString = Bmv2ThriftClient.of(did).dumpJsonConfig();
builder.set("bmv2JsonConfigMd5", md5);
builder.set("bmv2JsonConfigValue", jsonString);
controller.getAgent(did).resetState();
} catch (Bmv2RuntimeException e) {
LOG.warn("Unable to dump device JSON config from device {}: {}", did, e.toString());
log.warn("Unable to reset {}: {}", did, e.toString());
}
}
private void resetDeviceState(DeviceId did) {
private void initPortCounters(DeviceId did) {
try {
Bmv2ThriftClient.of(did).resetState();
initCounters(controller.getAgent(did));
} catch (Bmv2RuntimeException e) {
LOG.warn("Unable to reset {}: {}", did, e.toString());
log.warn("Unable to init counter on {}: {}", did, e.explain());
}
}
private void updatePorts(DeviceId did) {
private void updatePortsAndStats(DeviceId did) {
Device device = deviceService.getDevice(did);
if (device.is(PortDiscovery.class)) {
PortDiscovery portConfig = device.as(PortDiscovery.class);
List<PortDescription> portDescriptions = portConfig.getPorts();
providerService.updatePorts(did, portDescriptions);
if (device.is(DeviceDescriptionDiscovery.class)) {
DeviceDescriptionDiscovery discovery = device.as(DeviceDescriptionDiscovery.class);
List<PortDescription> portDescriptions = discovery.discoverPortDetails();
if (portDescriptions != null) {
providerService.updatePorts(did, portDescriptions);
}
} else {
LOG.warn("No PortDiscovery behavior for device {}", did);
log.warn("No DeviceDescriptionDiscovery behavior for device {}", did);
}
try {
Collection<PortStatistics> portStats = getPortStatistics(controller.getAgent(did),
deviceService.getPorts(did));
providerService.updatePortStatistics(did, portStats);
} catch (Bmv2RuntimeException e) {
log.warn("Unable to get port statistics for {}: {}", did, e.explain());
}
}
private void disconnectDevice(DeviceId did) {
LOG.debug("Trying to disconnect device from core... deviceId={}", did);
// Atomically disconnect device.
log.debug("Trying to disconnect device from core... deviceId={}", did);
activeDevices.compute(did, (k, v) -> {
if (deviceService.isAvailable(did)) {
providerService.deviceDisconnected(did);
// Make sure to close the transport session with device. Do we really need this?
forceDisconnectOf(did);
}
return null;
});
......@@ -269,10 +300,10 @@ public class Bmv2DeviceProvider extends AbstractDeviceProvider {
triggerProbe(bmv2Device.asDeviceId());
});
} catch (ConfigException e) {
LOG.error("Unable to read config: " + e);
log.error("Unable to read config: " + e);
}
} else {
LOG.error("Unable to read config (was null)");
log.error("Unable to read config (was null)");
}
}
......@@ -285,18 +316,18 @@ public class Bmv2DeviceProvider extends AbstractDeviceProvider {
}
/**
* Listener triggered by Bmv2ControlPlaneServer each time a hello message is received.
* Listener triggered by the BMv2 controller each time a hello message is received.
*/
private class InternalHelloListener implements Bmv2ControlPlaneServer.HelloListener {
private class InternalDeviceListener implements Bmv2DeviceListener {
@Override
public void handleHello(Bmv2Device device) {
public void handleHello(Bmv2Device device, int instanceId, String jsonConfigMd5) {
log.debug("Received hello from {}", device);
triggerProbe(device.asDeviceId());
}
}
/**
* Task that periodically trigger device probes.
* Task that periodically trigger device probes to check for device status and update port informations.
*/
private class DevicePoller implements TimerTask {
......@@ -304,20 +335,31 @@ public class Bmv2DeviceProvider extends AbstractDeviceProvider {
private Timeout timeout;
@Override
public void run(Timeout timeout) throws Exception {
if (timeout.isCancelled()) {
public void run(Timeout tout) throws Exception {
if (tout.isCancelled()) {
return;
}
log.debug("Executing polling on {} devices...", activeDevices.size());
activeDevices.forEach((did, value) -> triggerProbe(did));
timeout.getTimer().newTimeout(this, POLL_INTERVAL, TimeUnit.SECONDS);
activeDevices.keySet()
.stream()
// Filter out devices not yet created in the core.
.filter(did -> deviceService.getDevice(did) != null)
.forEach(did -> executorService.execute(() -> pollingTask(did)));
tout.getTimer().newTimeout(this, POLL_INTERVAL, TimeUnit.MILLISECONDS);
}
private void pollingTask(DeviceId deviceId) {
if (isReachable(deviceId)) {
updatePortsAndStats(deviceId);
} else {
disconnectDevice(deviceId);
}
}
/**
* Starts the collector.
*/
synchronized void start() {
LOG.info("Starting device poller...");
synchronized void start() {
log.info("Starting device poller...");
timeout = timer.newTimeout(this, 1, TimeUnit.SECONDS);
}
......@@ -325,7 +367,7 @@ public class Bmv2DeviceProvider extends AbstractDeviceProvider {
* Stops the collector.
*/
synchronized void stop() {
LOG.info("Stopping device poller...");
log.info("Stopping device poller...");
timeout.cancel();
}
}
......
/*
* Copyright 2016-present Open Networking Laboratory
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.onosproject.provider.bmv2.device.impl;
import com.google.common.collect.Lists;
import org.apache.commons.lang3.tuple.Pair;
import org.onosproject.bmv2.api.runtime.Bmv2Action;
import org.onosproject.bmv2.api.runtime.Bmv2DeviceAgent;
import org.onosproject.bmv2.api.runtime.Bmv2RuntimeException;
import org.onosproject.net.Port;
import org.onosproject.net.device.DefaultPortStatistics;
import org.onosproject.net.device.PortStatistics;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Collection;
import java.util.List;
/**
* Utility class to read port statistics from a BMv2 device.
*/
final class Bmv2PortStatisticsGetter {
// TODO: make counters configuration dependent
private static final String TABLE_NAME = "port_count_table";
private static final String ACTION_NAME = "count_packet";
private static final String EGRESS_COUNTER = "egress_port_counter";
private static final String INGRESS_COUNTER = "ingress_port_counter";
private static final Logger log = LoggerFactory.getLogger(Bmv2PortStatisticsGetter.class);
private Bmv2PortStatisticsGetter() {
// ban constructor.
}
/**
* Returns a collection of port statistics for given ports using the given BMv2 device agent.
*
* @param deviceAgent a device agent
* @param ports a collection of ports
* @return a collection of port statistics
*/
static Collection<PortStatistics> getPortStatistics(Bmv2DeviceAgent deviceAgent, Collection<Port> ports) {
List<PortStatistics> ps = Lists.newArrayList();
for (Port port : ports) {
int portNumber = (int) port.number().toLong();
try {
Pair<Long, Long> egressCounter = deviceAgent.readCounter(EGRESS_COUNTER, portNumber);
Pair<Long, Long> ingressCounter = deviceAgent.readCounter(INGRESS_COUNTER, portNumber);
ps.add(DefaultPortStatistics.builder()
.setPort(portNumber)
.setBytesSent(egressCounter.getLeft())
.setPacketsSent(egressCounter.getRight())
.setBytesReceived(ingressCounter.getLeft())
.setPacketsReceived(ingressCounter.getRight())
.build());
} catch (Bmv2RuntimeException e) {
log.info("Unable to read port statistics from {}: {}", port, e.explain());
}
}
return ps;
}
/**
* Initialize port counters on the given device agent.
*
* @param deviceAgent a device agent.
*/
static void initCounters(Bmv2DeviceAgent deviceAgent) {
try {
deviceAgent.setTableDefaultAction(TABLE_NAME, Bmv2Action.builder().withName(ACTION_NAME).build());
} catch (Bmv2RuntimeException e) {
log.debug("Failed to provision counters on {}: {}", deviceAgent.deviceId(), e.explain());
}
}
}
......@@ -22,6 +22,7 @@
<artifactId>onos-bmv2-providers</artifactId>
<groupId>org.onosproject</groupId>
<version>1.7.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
......@@ -34,12 +35,12 @@
<dependencies>
<dependency>
<groupId>org.onosproject</groupId>
<artifactId>onos-drivers-bmv2</artifactId>
<artifactId>onos-core-common</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.onosproject</groupId>
<artifactId>onos-core-common</artifactId>
<artifactId>onos-bmv2-protocol-api</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
......
......@@ -23,12 +23,14 @@ import org.apache.felix.scr.annotations.Reference;
import org.apache.felix.scr.annotations.ReferenceCardinality;
import org.onlab.packet.Ethernet;
import org.onlab.util.ImmutableByteSequence;
import org.onosproject.bmv2.api.runtime.Bmv2ControlPlaneServer;
import org.onosproject.bmv2.api.runtime.Bmv2Device;
import org.onosproject.bmv2.api.service.Bmv2Controller;
import org.onosproject.bmv2.api.service.Bmv2PacketListener;
import org.onosproject.core.CoreService;
import org.onosproject.net.ConnectPoint;
import org.onosproject.net.Device;
import org.onosproject.net.DeviceId;
import org.onosproject.net.Port;
import org.onosproject.net.PortNumber;
import org.onosproject.net.device.DeviceService;
import org.onosproject.net.flow.DefaultTrafficTreatment;
......@@ -49,6 +51,12 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.nio.ByteBuffer;
import java.util.Optional;
import static org.onosproject.net.PortNumber.FLOOD;
import static org.onosproject.net.flow.DefaultTrafficTreatment.emptyTreatment;
import static org.onosproject.net.flow.instructions.Instruction.Type.OUTPUT;
import static org.onosproject.net.flow.instructions.Instructions.OutputInstruction;
/**
* Implementation of a packet provider for BMv2.
......@@ -56,11 +64,11 @@ import java.nio.ByteBuffer;
@Component(immediate = true)
public class Bmv2PacketProvider extends AbstractProvider implements PacketProvider {
private static final Logger LOG = LoggerFactory.getLogger(Bmv2PacketProvider.class);
private final Logger log = LoggerFactory.getLogger(Bmv2PacketProvider.class);
private static final String APP_NAME = "org.onosproject.bmv2";
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
protected Bmv2ControlPlaneServer controlPlaneServer;
protected Bmv2Controller controller;
@Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
protected CoreService coreService;
......@@ -86,28 +94,28 @@ public class Bmv2PacketProvider extends AbstractProvider implements PacketProvid
protected void activate() {
providerService = providerRegistry.register(this);
coreService.registerApplication(APP_NAME);
controlPlaneServer.addPacketListener(packetListener);
LOG.info("Started");
controller.addPacketListener(packetListener);
log.info("Started");
}
@Deactivate
public void deactivate() {
controlPlaneServer.removePacketListener(packetListener);
controller.removePacketListener(packetListener);
providerRegistry.unregister(this);
providerService = null;
LOG.info("Stopped");
log.info("Stopped");
}
@Override
public void emit(OutboundPacket packet) {
if (packet != null) {
DeviceId did = packet.sendThrough();
Device device = deviceService.getDevice(did);
DeviceId deviceId = packet.sendThrough();
Device device = deviceService.getDevice(deviceId);
if (device.is(PacketProgrammable.class)) {
PacketProgrammable packetProgrammable = device.as(PacketProgrammable.class);
packetProgrammable.emit(packet);
} else {
LOG.info("Unable to send packet, no PacketProgrammable behavior for device {}", did);
log.info("No PacketProgrammable behavior for device {}", deviceId);
}
}
}
......@@ -117,47 +125,75 @@ public class Bmv2PacketProvider extends AbstractProvider implements PacketProvid
*/
private class Bmv2PacketContext extends DefaultPacketContext {
public Bmv2PacketContext(long time, InboundPacket inPkt, OutboundPacket outPkt, boolean block) {
Bmv2PacketContext(long time, InboundPacket inPkt, OutboundPacket outPkt, boolean block) {
super(time, inPkt, outPkt, block);
}
@Override
public void send() {
if (!this.block()) {
if (this.outPacket().treatment() == null) {
TrafficTreatment treatment = (this.treatmentBuilder() == null)
? DefaultTrafficTreatment.emptyTreatment()
: this.treatmentBuilder().build();
OutboundPacket newPkt = new DefaultOutboundPacket(this.outPacket().sendThrough(),
treatment,
this.outPacket().data());
emit(newPkt);
} else {
emit(outPacket());
}
if (this.block()) {
log.info("Unable to send, packet context not blocked");
return;
}
DeviceId deviceId = outPacket().sendThrough();
ByteBuffer rawData = outPacket().data();
TrafficTreatment treatment;
if (outPacket().treatment() == null) {
treatment = (treatmentBuilder() == null) ? emptyTreatment() : treatmentBuilder().build();
} else {
treatment = outPacket().treatment();
}
// BMv2 doesn't support FLOOD for packet-outs.
// Workaround here is to perform multiple emits, one for each device port != packet inPort.
Optional<OutputInstruction> floodInst = treatment.allInstructions()
.stream()
.filter(i -> i.type().equals(OUTPUT))
.map(i -> (OutputInstruction) i)
.filter(i -> i.port().equals(FLOOD))
.findAny();
if (floodInst.isPresent() && treatment.allInstructions().size() == 1) {
// Only one instruction and is FLOOD. Do the trick.
PortNumber inPort = inPacket().receivedFrom().port();
deviceService.getPorts(outPacket().sendThrough())
.stream()
.map(Port::number)
.filter(port -> !port.equals(inPort))
.map(outPort -> DefaultTrafficTreatment.builder().setOutput(outPort).build())
.map(outTreatment -> new DefaultOutboundPacket(deviceId, outTreatment, rawData))
.forEach(Bmv2PacketProvider.this::emit);
} else {
LOG.info("Unable to send, packet context not blocked");
// Not FLOOD treatment, what to do is up to driver.
emit(new DefaultOutboundPacket(deviceId, treatment, rawData));
}
}
}
/**
* Internal packet listener to get packet events from the Bmv2ControlPlaneServer.
* Internal packet listener to handle packet-in events received from the BMv2 controller.
*/
private class InternalPacketListener implements Bmv2ControlPlaneServer.PacketListener {
private class InternalPacketListener implements Bmv2PacketListener {
@Override
public void handlePacketIn(Bmv2Device device, int inputPort, long reason, int tableId, int contextId,
ImmutableByteSequence packet) {
Ethernet ethPkt = new Ethernet();
ethPkt.deserialize(packet.asArray(), 0, packet.size());
Ethernet eth = new Ethernet();
eth.deserialize(packet.asArray(), 0, packet.size());
DeviceId deviceId = device.asDeviceId();
ConnectPoint receivedFrom = new ConnectPoint(deviceId, PortNumber.portNumber(inputPort));
ByteBuffer rawData = ByteBuffer.wrap(packet.asArray());
InboundPacket inPkt = new DefaultInboundPacket(receivedFrom, ethPkt, rawData);
OutboundPacket outPkt = new DefaultOutboundPacket(deviceId, null, rawData);
InboundPacket inPkt = new DefaultInboundPacket(new ConnectPoint(device.asDeviceId(),
PortNumber.portNumber(inputPort)),
eth, ByteBuffer.wrap(packet.asArray()));
OutboundPacket outPkt = new DefaultOutboundPacket(device.asDeviceId(), null,
ByteBuffer.wrap(packet.asArray()));
PacketContext pktCtx = new Bmv2PacketContext(System.currentTimeMillis(), inPkt, outPkt, false);
providerService.processPacket(pktCtx);
}
}
......
......@@ -22,6 +22,7 @@
<artifactId>onos-providers</artifactId>
<groupId>org.onosproject</groupId>
<version>1.7.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
......
......@@ -46,7 +46,7 @@
<module>lldpcommon</module>
<module>lldp</module>
<module>netcfglinks</module>
<!--<module>bmv2</module>-->
<module>bmv2</module>
<module>isis</module>
</modules>
......