홍길동 / onos

Go to a project
Toggle navigation Toggle navigation pinning
Thomas Vachuska
Committed by Gerrit Code Review
1eff3a60 1 parent 2199c30c

Initial implementation of shared test cell warden. 

Change-Id: Ia973d514fe1dd11ffe4cdb7c902cc43a9c2eb626
Inline Side-by-side
Showing 8 changed files with 732 additions and 10 deletions
......@@ -103,10 +103,41 @@ function setPrimaryInstance {
echo $OCI
}
# ON.Lab shared test cell warden address
export CELL_WARDEN="10.254.1.19"
# Applies the settings in the specified cell file or lists current cell definition
# if no cell file is given.
function cell {
if [ -n "$1" ]; then
cell=$1
case "$cell" in
"borrow")
aux="/tmp/cell-$$"
curl -sS -X POST "http://$CELL_WARDEN:4321/?user=$(id -un)&duration=${2:-60}" \
-d "$(cat ~/.ssh/id_rsa.pub)" > $aux
. $aux
rm -f $aux
export ONOS_INSTANCES=$(env | grep 'OC[0-9]*=' | sort | cut -d= -f2)
setPrimaryInstance 1 >/dev/null
cell
;;
"return")
curl -sS -X DELETE "http://$CELL_WARDEN:4321/?user=$(id -un)"
;;
"status")
curl -sS "http://$CELL_WARDEN:4321/"
;;
"")
env | egrep "ONOS_CELL"
env | egrep "OCI"
env | egrep "OC[0-9]+" | sort
env | egrep "OC[NT]"
env | egrep "ONOS_" | egrep -v 'ONOS_ROOT|ONOS_CELL|ONOS_INSTANCES' | sort
;;
*)
[ ! -f $ONOS_ROOT/tools/test/cells/$1 ] && \
echo "No such cell: $1" >&2 && return 1
unset ONOS_CELL ONOS_NIC ONOS_IP ONOS_APPS ONOS_BOOT_FEATURES
......@@ -121,16 +152,10 @@ function cell {
export ONOS_INSTANCES=$(env | grep 'OC[0-9]*=' | sort | cut -d= -f2)
setPrimaryInstance 1 >/dev/null
cell
else
env | egrep "ONOS_CELL"
env | egrep "OCI"
env | egrep "OC[0-9]+" | sort
env | egrep "OC[NT]"
env | egrep "ONOS_" | egrep -v 'ONOS_ROOT|ONOS_CELL|ONOS_INSTANCES' | sort
fi
esac
}
cell $ONOS_CELL > /dev/null
[ -n "$ONOS_CELL" ] && cell $ONOS_CELL > /dev/null
# Lists available cells
function cells {
......
......@@ -40,7 +40,7 @@ complete -F _stl-opts stl
function _cell-opts () {
local cur=${COMP_WORDS[COMP_CWORD]}
if [ $COMP_CWORD -eq 1 ]; then
COMPREPLY=( $( compgen -W "$(cd $ONOS_ROOT/tools/test/cells && ls -1)" -- $cur ) )
COMPREPLY=( $( compgen -W "$(cd $ONOS_ROOT/tools/test/cells && ls -1) borrow return status" -- $cur ) )
fi
}
......
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Copyright 2015-present Open Networking Laboratory
~
~ Licensed under the Apache License, Version 2.0 (the "License");
~ you may not use this file except in compliance with the License.
~ You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.onosproject</groupId>
<artifactId>onlab-utils</artifactId>
<version>1.6.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<artifactId>onlab-warden</artifactId>
<packaging>jar</packaging>
<description>System Test Cell Warden</description>
<dependencies>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-server</artifactId>
<version>8.1.18.v20150929</version>
</dependency>
<dependency>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-servlet</artifactId>
<version>8.1.18.v20150929</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>2.4.3</version>
<configuration>
<transformers>
<transformer
implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass>org.onlab.warden.Main</mainClass>
</transformer>
</transformers>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
</excludes>
</filter>
</filters>
</configuration>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
/*
* Copyright 2016 Open Networking Laboratory
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.onlab.warden;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.servlet.ServletHandler;
import org.eclipse.jetty.util.log.Logger;
/**
* Main program for executing scenario test warden.
*/
public final class Main {
// Public construction forbidden
private Main(String[] args) {
}
/**
* Main entry point for the cell warden.
*
* @param args command-line arguments
*/
public static void main(String[] args) {
Main main = new Main(args);
main.run();
}
// Runs the warden processing
private void run() {
startWebServer();
}
// Initiates a web-server.
private static void startWebServer() {
WardenServlet.warden = new Warden();
org.eclipse.jetty.util.log.Log.setLog(new NullLogger());
Server server = new Server(4321);
ServletHandler handler = new ServletHandler();
server.setHandler(handler);
handler.addServletWithMapping(WardenServlet.class, "/*");
try {
server.start();
} catch (Exception e) {
print("Warden already active...");
}
}
private static void print(String s) {
System.out.println(s);
}
// Logger to quiet Jetty down
private static class NullLogger implements Logger {
@Override
public String getName() {
return "quiet";
}
@Override
public void warn(String msg, Object... args) {
}
@Override
public void warn(Throwable thrown) {
}
@Override
public void warn(String msg, Throwable thrown) {
}
@Override
public void info(String msg, Object... args) {
}
@Override
public void info(Throwable thrown) {
}
@Override
public void info(String msg, Throwable thrown) {
}
@Override
public boolean isDebugEnabled() {
return false;
}
@Override
public void setDebugEnabled(boolean enabled) {
}
@Override
public void debug(String msg, Object... args) {
}
@Override
public void debug(Throwable thrown) {
}
@Override
public void debug(String msg, Throwable thrown) {
}
@Override
public Logger getLogger(String name) {
return this;
}
@Override
public void ignore(Throwable ignored) {
}
}
}
/*
* Copyright 2016 Open Networking Laboratory
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.onlab.warden;
import static com.google.common.base.Preconditions.checkState;
/**
* Cell reservation record.
*/
final class Reservation {
final String cellName;
final String userName;
final long time;
final int duration;
// Creates a new reservation record
Reservation(String cellName, String userName, long time, int duration) {
this.cellName = cellName;
this.userName = userName;
this.time = time;
this.duration = duration;
}
/**
* Decodes reservation record from the specified line.
*
* @param line string line
*/
Reservation(String line) {
String[] fields = line.trim().split("\t");
checkState(fields.length == 4, "Incorrect reservation encoding");
this.cellName = fields[0];
this.userName = fields[1];
this.time = Long.parseLong(fields[2]);
this.duration = Integer.parseInt(fields[3]);
}
/**
* Encodes reservation record into a string line.
*
* @return encoded string
*/
String encode() {
return String.format("%s\t%s\t%s\t%s\n", cellName, userName, time, duration);
}
}
/*
* Copyright 2016 Open Networking Laboratory
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.onlab.warden;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.common.io.ByteStreams;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintWriter;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Random;
import java.util.Set;
import java.util.Timer;
import java.util.TimerTask;
import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import static com.google.common.base.Preconditions.*;
/**
* Warden for tracking use of shared test cells.
*/
class Warden {
private static final String CELL_NOT_NULL = "Cell name cannot be null";
private static final String USER_NOT_NULL = "User name cannot be null";
private static final String KEY_NOT_NULL = "User key cannot be null";
private static final String UTF_8 = "UTF-8";
private static final long TIMEOUT = 3;
private static final String AUTHORIZED_KEYS = "authorized_keys";
private static final int MAX_MINUTES = 240; // 4 hours max
private static final int MINUTE = 60_000; // 1 minute
private final File log = new File("warden.log");
private final File cells = new File("cells");
private final File supported = new File(cells, "supported");
private final File reserved = new File(cells, "reserved");
private final Random random = new Random();
private final Timer timer = new Timer("cell-pruner", true);
/**
* Creates a new cell warden.
*/
Warden() {
random.setSeed(System.currentTimeMillis());
timer.schedule(new Reposessor(), MINUTE / 4, MINUTE);
}
/**
* Returns list of names of supported cells.
*
* @return list of cell names
*/
Set<String> getCells() {
String[] list = supported.list();
return list != null ? ImmutableSet.copyOf(list) : ImmutableSet.of();
}
/**
* Returns list of names of available cells.
*
* @return list of cell names
*/
Set<String> getAvailableCells() {
Set<String> available = new HashSet<>(getCells());
available.removeAll(getReservedCells());
return ImmutableSet.copyOf(available);
}
/**
* Returns list of names of reserved cells.
*
* @return list of cell names
*/
Set<String> getReservedCells() {
String[] list = reserved.list();
return list != null ? ImmutableSet.copyOf(list) : ImmutableSet.of();
}
/**
* Returns reservation for the specified user.
*
* @param userName user name
* @return cell reservation record or null if user does not have one
*/
Reservation currentUserReservation(String userName) {
checkNotNull(userName, USER_NOT_NULL);
for (String cellName : getReservedCells()) {
Reservation reservation = currentCellReservation(cellName);
if (reservation != null && userName.equals(reservation.userName)) {
return reservation;
}
}
return null;
}
/**
* Returns the name of the user who reserved the given cell.
*
* @param cellName cell name
* @return cell reservation record or null if cell is not reserved
*/
Reservation currentCellReservation(String cellName) {
checkNotNull(cellName, CELL_NOT_NULL);
File cellFile = new File(reserved, cellName);
if (!cellFile.exists()) {
return null;
}
try (InputStream stream = new FileInputStream(cellFile)) {
return new Reservation(new String(ByteStreams.toByteArray(stream), "UTF-8"));
} catch (IOException e) {
throw new IllegalStateException("Unable to get current user for cell " + cellName, e);
}
}
/**
* Reserves a cell for the specified user and their public access key.
*
* @param userName user name
* @param sshKey user ssh public key
* @param minutes number of minutes for reservation
* @return reserved cell definition
*/
synchronized String borrowCell(String userName, String sshKey, int minutes) {
checkNotNull(userName, USER_NOT_NULL);
checkNotNull(sshKey, KEY_NOT_NULL);
checkArgument(minutes > 0, "Number of minutes must be positive");
checkArgument(minutes < MAX_MINUTES, "Number of minutes must be less than %d", MAX_MINUTES);
long now = System.currentTimeMillis();
Reservation reservation = currentUserReservation(userName);
if (reservation == null) {
Set<String> cells = getAvailableCells();
checkState(!cells.isEmpty(), "No cells are presently available");
String cellName = ImmutableList.copyOf(cells).get(random.nextInt(cells.size()));
reservation = new Reservation(cellName, userName, now, minutes);
} else {
reservation = new Reservation(reservation.cellName, userName, now, minutes);
}
reserveCell(reservation.cellName, reservation);
installUserKeys(reservation.cellName, userName, sshKey);
log(userName, reservation.cellName, "borrowed for " + minutes + " minutes");
return getCellDefinition(reservation.cellName);
}
/**
* Reserves the specified cell for the user the source file and writes the
* specified content to the target file.
*
* @param cellName cell name
* @param reservation cell reservation record
*/
private void reserveCell(String cellName, Reservation reservation) {
try (FileOutputStream stream = new FileOutputStream(new File(reserved, cellName))) {
stream.write(reservation.encode().getBytes(UTF_8));
} catch (IOException e) {
throw new IllegalStateException("Unable to reserve cell " + cellName, e);
}
}
/**
* Returns the specified cell for the specified user and their public access key.
*
* @param userName user name
*/
synchronized void returnCell(String userName) {
checkNotNull(userName, USER_NOT_NULL);
Reservation reservation = currentUserReservation(userName);
checkState(reservation != null, "User %s has no cell reservations", userName);
checkState(new File(reserved, reservation.cellName).delete(),
"Unable to return cell %s", reservation.cellName);
uninstallUserKeys(reservation.cellName);
log(userName, reservation.cellName, "returned");
}
/**
* Reads the definition of the specified cell.
*
* @param cellName cell name
* @return cell definition
*/
String getCellDefinition(String cellName) {
File cellFile = new File(supported, cellName);
try (InputStream stream = new FileInputStream(cellFile)) {
return new String(ByteStreams.toByteArray(stream), UTF_8);
} catch (IOException e) {
throw new IllegalStateException("Unable to definition for cell " + cellName, e);
}
}
// Returns list of cell hosts, i.e. OC#, OCN
private List<String> cellHosts(String cellName) {
ImmutableList.Builder<String> builder = ImmutableList.builder();
Pattern pattern = Pattern.compile("export OC[0-9N]=(.*)");
for (String line : getCellDefinition(cellName).split("\n")) {
Matcher matcher = pattern.matcher(line);
if (matcher.matches()) {
builder.add(matcher.group(1).replaceAll("[\"']", ""));
}
}
return builder.build();
}
// Installs the specified user's key on all hosts of the given cell.
private void installUserKeys(String cellName, String userName, String sshKey) {
File authKeysFile = authKeys(sshKey);
for (String host : cellHosts(cellName)) {
installAuthorizedKey(host, authKeysFile.getPath());
}
checkState(authKeysFile.delete(), "Unable to install user keys");
}
// Uninstalls the user keys on the specified cell
private void uninstallUserKeys(String cellName) {
for (String host : cellHosts(cellName)) {
installAuthorizedKey(host, AUTHORIZED_KEYS);
}
}
// Installs the authorized keys on the specified host.
private void installAuthorizedKey(String host, String authorizedKeysFile) {
String cmd = "scp " + authorizedKeysFile + " sdn@" + host + ":.ssh/authorized_keys";
try {
Process process = Runtime.getRuntime().exec(cmd);
process.waitFor(TIMEOUT, TimeUnit.SECONDS);
} catch (Exception e) {
throw new IllegalStateException("Unable to set authorized keys for host " + host);
}
}
// Returns the file containing authorized keys that incudes the specified key.
private File authKeys(String sshKey) {
File keysFile = new File(AUTHORIZED_KEYS);
try {
File tmp = File.createTempFile("warden-", ".auth");
tmp.deleteOnExit();
try (InputStream stream = new FileInputStream(keysFile);
PrintWriter output = new PrintWriter(tmp)) {
String baseKeys = new String(ByteStreams.toByteArray(stream), UTF_8);
output.println(baseKeys);
output.println(sshKey);
return tmp;
} catch (IOException e) {
throw new IllegalStateException("Unable to generate authorized keys", e);
}
} catch (IOException e) {
throw new IllegalStateException("Unable to generate authorized keys", e);
}
}
// Creates an audit log entry.
void log(String userName, String cellName, String action) {
try (FileOutputStream fos = new FileOutputStream(log, true);
PrintWriter pw = new PrintWriter(fos)) {
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
pw.println(String.format("%s\t%s\t%s\t%s", format.format(new Date()),
userName, cellName, action));
pw.flush();
} catch (IOException e) {
throw new IllegalStateException("Unable to log reservation action", e);
}
}
// Task for re-possessing overdue cells
private class Reposessor extends TimerTask {
@Override
public void run() {
long now = System.currentTimeMillis();
for (String cellName : getReservedCells()) {
Reservation reservation = currentCellReservation(cellName);
if (reservation != null &&
(reservation.time + reservation.duration * MINUTE) < now) {
try {
returnCell(reservation.userName);
} catch (Exception e) {
e.printStackTrace();
}
}
}
}
}
}
/*
* Copyright 2016 Open Networking Laboratory
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.onlab.warden;
import com.google.common.io.ByteStreams;
import org.eclipse.jetty.server.Response;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.io.PrintWriter;
import java.text.SimpleDateFormat;
import java.util.Date;
import static com.google.common.base.Strings.isNullOrEmpty;
/**
* Web socket servlet capable of creating web sockets for the STC monitor.
*/
public class WardenServlet extends HttpServlet {
static Warden warden;
@Override
protected void doGet(HttpServletRequest req, HttpServletResponse resp)
throws ServletException, IOException {
resp.setContentType("text/plain; charset=UTF-8");
SimpleDateFormat fmt = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
try (PrintWriter out = resp.getWriter()) {
for (String cellName : warden.getCells()) {
Reservation reservation = warden.currentCellReservation(cellName);
if (reservation != null) {
long expiration = reservation.time + reservation.duration * 60_000;
out.println(String.format("%-10s\t%-10s\t%s\t%s\t%s minutes", cellName,
reservation.userName,
fmt.format(new Date(reservation.time)),
fmt.format(new Date(expiration)),
reservation.duration));
} else {
out.println(String.format("%-10s\t%-10s", cellName, "available"));
}
}
} catch (Exception e) {
resp.setStatus(Response.SC_INTERNAL_SERVER_ERROR);
e.printStackTrace();
}
}
@Override
protected void doPost(HttpServletRequest req, HttpServletResponse resp)
throws ServletException, IOException {
try (PrintWriter out = resp.getWriter()) {
String sshKey = new String(ByteStreams.toByteArray(req.getInputStream()), "UTF-8");
String userName = req.getParameter("user");
String sd = req.getParameter("duration");
int duration = isNullOrEmpty(sd) ? 60 : Integer.parseInt(sd);
String cellDefinition = warden.borrowCell(userName, sshKey, duration);
out.println(cellDefinition);
} catch (Exception e) {
resp.setStatus(Response.SC_INTERNAL_SERVER_ERROR);
e.printStackTrace();
}
}
@Override
protected void doDelete(HttpServletRequest req, HttpServletResponse resp)
throws ServletException, IOException {
try (PrintWriter out = resp.getWriter()) {
String userName = req.getParameter("user");
warden.returnCell(userName);
} catch (Exception e) {
resp.setStatus(Response.SC_INTERNAL_SERVER_ERROR);
e.printStackTrace();
}
}
}
/*
* Copyright 2016 Open Networking Laboratory
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Cell warden to coordinate borrowing and returning test cells.
*/
package org.onlab.warden;
\ No newline at end of file