Updated SDN-IP to use the Intent framework batch-based intents.
Also, created a local cache of IPv4-to-MAC address mapping, to avoid relatively costly hostService.getHostsByIp() calls per added route. Change-Id: I8abed378985708e883fd99e85c54b01f38756515
Showing
10 changed files
with
195 additions
and
139 deletions
... | @@ -26,6 +26,7 @@ import java.util.concurrent.ExecutorService; | ... | @@ -26,6 +26,7 @@ import java.util.concurrent.ExecutorService; |
26 | import java.util.concurrent.Executors; | 26 | import java.util.concurrent.Executors; |
27 | import java.util.concurrent.Semaphore; | 27 | import java.util.concurrent.Semaphore; |
28 | 28 | ||
29 | +import org.apache.commons.lang3.tuple.Pair; | ||
29 | import org.onlab.onos.core.ApplicationId; | 30 | import org.onlab.onos.core.ApplicationId; |
30 | import org.onlab.onos.net.flow.criteria.Criteria.IPCriterion; | 31 | import org.onlab.onos.net.flow.criteria.Criteria.IPCriterion; |
31 | import org.onlab.onos.net.flow.criteria.Criterion; | 32 | import org.onlab.onos.net.flow.criteria.Criterion; |
... | @@ -116,7 +117,7 @@ public class IntentSynchronizer { | ... | @@ -116,7 +117,7 @@ public class IntentSynchronizer { |
116 | // | 117 | // |
117 | log.debug("SDN-IP Intent Synchronizer shutdown: " + | 118 | log.debug("SDN-IP Intent Synchronizer shutdown: " + |
118 | "withdrawing all intents..."); | 119 | "withdrawing all intents..."); |
119 | - IntentOperations.Builder builder = IntentOperations.builder(); | 120 | + IntentOperations.Builder builder = IntentOperations.builder(appId); |
120 | for (Intent intent : intentService.getIntents()) { | 121 | for (Intent intent : intentService.getIntents()) { |
121 | // Skip the intents from other applications | 122 | // Skip the intents from other applications |
122 | if (!intent.appId().equals(appId)) { | 123 | if (!intent.appId().equals(appId)) { |
... | @@ -234,51 +235,84 @@ public class IntentSynchronizer { | ... | @@ -234,51 +235,84 @@ public class IntentSynchronizer { |
234 | } | 235 | } |
235 | 236 | ||
236 | /** | 237 | /** |
237 | - * Submits a multi-point-to-single-point intent. | 238 | + * Updates multi-point-to-single-point route intents. |
238 | * | 239 | * |
239 | - * @param prefix the IPv4 matching prefix for the intent to submit | 240 | + * @param submitIntents the intents to submit |
240 | - * @param intent the intent to submit | 241 | + * @param withdrawPrefixes the IPv4 matching prefixes for the intents |
242 | + * to withdraw | ||
241 | */ | 243 | */ |
242 | - void submitRouteIntent(Ip4Prefix prefix, | 244 | + void updateRouteIntents( |
243 | - MultiPointToSinglePointIntent intent) { | 245 | + Collection<Pair<Ip4Prefix, MultiPointToSinglePointIntent>> submitIntents, |
246 | + Collection<Ip4Prefix> withdrawPrefixes) { | ||
247 | + | ||
248 | + // | ||
249 | + // NOTE: Semantically, we MUST withdraw existing intents before | ||
250 | + // submitting new intents. | ||
251 | + // | ||
244 | synchronized (this) { | 252 | synchronized (this) { |
245 | - MultiPointToSinglePointIntent oldIntent = | 253 | + MultiPointToSinglePointIntent intent; |
246 | - routeIntents.put(prefix, intent); | ||
247 | 254 | ||
248 | - if (isElectedLeader && isActivatedLeader) { | 255 | + log.debug("SDN-IP submitting intents = {} withdrawing = {}", |
249 | - if (oldIntent != null) { | 256 | + submitIntents.size(), withdrawPrefixes.size()); |
250 | - // | 257 | + |
251 | - // TODO: Short-term solution to explicitly withdraw | 258 | + // |
252 | - // instead of using "replace" operation. | 259 | + // Prepare the Intent batch operations for the intents to withdraw |
253 | - // | 260 | + // |
254 | - log.debug("SDN-IP Withdrawing old intent: {}", oldIntent); | 261 | + IntentOperations.Builder withdrawBuilder = |
255 | - intentService.withdraw(oldIntent); | 262 | + IntentOperations.builder(appId); |
263 | + for (Ip4Prefix prefix : withdrawPrefixes) { | ||
264 | + intent = routeIntents.remove(prefix); | ||
265 | + if (intent == null) { | ||
266 | + log.debug("SDN-IP No intent in routeIntents to delete " + | ||
267 | + "for prefix: {}", prefix); | ||
268 | + continue; | ||
269 | + } | ||
270 | + if (isElectedLeader && isActivatedLeader) { | ||
271 | + log.debug("SDN-IP Withdrawing intent: {}", intent); | ||
272 | + withdrawBuilder.addWithdrawOperation(intent.id()); | ||
256 | } | 273 | } |
257 | - log.debug("SDN-IP Submitting intent: {}", intent); | ||
258 | - intentService.submit(intent); | ||
259 | } | 274 | } |
260 | - } | ||
261 | - } | ||
262 | 275 | ||
263 | - /** | 276 | + // |
264 | - * Withdraws a multi-point-to-single-point intent. | 277 | + // Prepare the Intent batch operations for the intents to submit |
265 | - * | 278 | + // |
266 | - * @param prefix the IPv4 matching prefix for the intent to withdraw. | 279 | + IntentOperations.Builder submitBuilder = |
267 | - */ | 280 | + IntentOperations.builder(appId); |
268 | - void withdrawRouteIntent(Ip4Prefix prefix) { | 281 | + for (Pair<Ip4Prefix, MultiPointToSinglePointIntent> pair : |
269 | - synchronized (this) { | 282 | + submitIntents) { |
270 | - MultiPointToSinglePointIntent intent = | 283 | + Ip4Prefix prefix = pair.getLeft(); |
271 | - routeIntents.remove(prefix); | 284 | + intent = pair.getRight(); |
272 | - | 285 | + MultiPointToSinglePointIntent oldIntent = |
273 | - if (intent == null) { | 286 | + routeIntents.put(prefix, intent); |
274 | - log.debug("SDN-IP no intent in routeIntents to delete for " + | 287 | + if (isElectedLeader && isActivatedLeader) { |
275 | - "prefix: {}", prefix); | 288 | + if (oldIntent != null) { |
276 | - return; | 289 | + // |
290 | + // TODO: Short-term solution to explicitly withdraw | ||
291 | + // instead of using "replace" operation. | ||
292 | + // | ||
293 | + log.debug("SDN-IP Withdrawing old intent: {}", | ||
294 | + oldIntent); | ||
295 | + withdrawBuilder.addWithdrawOperation(oldIntent.id()); | ||
296 | + } | ||
297 | + log.debug("SDN-IP Submitting intent: {}", intent); | ||
298 | + submitBuilder.addSubmitOperation(intent); | ||
299 | + } | ||
277 | } | 300 | } |
278 | 301 | ||
302 | + // | ||
303 | + // Submit the Intent operations | ||
304 | + // | ||
279 | if (isElectedLeader && isActivatedLeader) { | 305 | if (isElectedLeader && isActivatedLeader) { |
280 | - log.debug("SDN-IP Withdrawing intent: {}", intent); | 306 | + IntentOperations intentOperations = withdrawBuilder.build(); |
281 | - intentService.withdraw(intent); | 307 | + if (!intentOperations.operations().isEmpty()) { |
308 | + log.debug("SDN-IP Withdrawing intents executed"); | ||
309 | + intentService.execute(intentOperations); | ||
310 | + } | ||
311 | + intentOperations = submitBuilder.build(); | ||
312 | + if (!intentOperations.operations().isEmpty()) { | ||
313 | + log.debug("SDN-IP Submitting intents executed"); | ||
314 | + intentService.execute(intentOperations); | ||
315 | + } | ||
282 | } | 316 | } |
283 | } | 317 | } |
284 | } | 318 | } | ... | ... |
... | @@ -15,6 +15,8 @@ | ... | @@ -15,6 +15,8 @@ |
15 | */ | 15 | */ |
16 | package org.onlab.onos.sdnip; | 16 | package org.onlab.onos.sdnip; |
17 | 17 | ||
18 | +import java.util.Collection; | ||
19 | + | ||
18 | /** | 20 | /** |
19 | * An interface to receive route updates from route providers. | 21 | * An interface to receive route updates from route providers. |
20 | */ | 22 | */ |
... | @@ -22,7 +24,7 @@ public interface RouteListener { | ... | @@ -22,7 +24,7 @@ public interface RouteListener { |
22 | /** | 24 | /** |
23 | * Receives a route update from a route provider. | 25 | * Receives a route update from a route provider. |
24 | * | 26 | * |
25 | - * @param routeUpdate the updated route information | 27 | + * @param routeUpdates the collection with updated route information |
26 | */ | 28 | */ |
27 | - public void update(RouteUpdate routeUpdate); | 29 | + public void update(Collection<RouteUpdate> routeUpdates); |
28 | } | 30 | } | ... | ... |
This diff is collapsed. Click to expand it.
... | @@ -22,6 +22,7 @@ import java.net.InetAddress; | ... | @@ -22,6 +22,7 @@ import java.net.InetAddress; |
22 | import java.net.InetSocketAddress; | 22 | import java.net.InetSocketAddress; |
23 | import java.net.SocketAddress; | 23 | import java.net.SocketAddress; |
24 | import java.util.Collection; | 24 | import java.util.Collection; |
25 | +import java.util.LinkedList; | ||
25 | import java.util.concurrent.ConcurrentHashMap; | 26 | import java.util.concurrent.ConcurrentHashMap; |
26 | import java.util.concurrent.ConcurrentMap; | 27 | import java.util.concurrent.ConcurrentMap; |
27 | import java.util.concurrent.Executors; | 28 | import java.util.concurrent.Executors; |
... | @@ -248,18 +249,28 @@ public class BgpSessionManager { | ... | @@ -248,18 +249,28 @@ public class BgpSessionManager { |
248 | synchronized void routeUpdates(BgpSession bgpSession, | 249 | synchronized void routeUpdates(BgpSession bgpSession, |
249 | Collection<BgpRouteEntry> addedBgpRouteEntries, | 250 | Collection<BgpRouteEntry> addedBgpRouteEntries, |
250 | Collection<BgpRouteEntry> deletedBgpRouteEntries) { | 251 | Collection<BgpRouteEntry> deletedBgpRouteEntries) { |
252 | + Collection<RouteUpdate> routeUpdates = new LinkedList<>(); | ||
253 | + RouteUpdate routeUpdate; | ||
254 | + | ||
251 | if (isShutdown) { | 255 | if (isShutdown) { |
252 | return; // Ignore any leftover updates if shutdown | 256 | return; // Ignore any leftover updates if shutdown |
253 | } | 257 | } |
254 | // Process the deleted route entries | 258 | // Process the deleted route entries |
255 | for (BgpRouteEntry bgpRouteEntry : deletedBgpRouteEntries) { | 259 | for (BgpRouteEntry bgpRouteEntry : deletedBgpRouteEntries) { |
256 | - processDeletedRoute(bgpSession, bgpRouteEntry); | 260 | + routeUpdate = processDeletedRoute(bgpSession, bgpRouteEntry); |
261 | + if (routeUpdate != null) { | ||
262 | + routeUpdates.add(routeUpdate); | ||
263 | + } | ||
257 | } | 264 | } |
258 | 265 | ||
259 | // Process the added/updated route entries | 266 | // Process the added/updated route entries |
260 | for (BgpRouteEntry bgpRouteEntry : addedBgpRouteEntries) { | 267 | for (BgpRouteEntry bgpRouteEntry : addedBgpRouteEntries) { |
261 | - processAddedRoute(bgpSession, bgpRouteEntry); | 268 | + routeUpdate = processAddedRoute(bgpSession, bgpRouteEntry); |
269 | + if (routeUpdate != null) { | ||
270 | + routeUpdates.add(routeUpdate); | ||
271 | + } | ||
262 | } | 272 | } |
273 | + routeListener.update(routeUpdates); | ||
263 | } | 274 | } |
264 | 275 | ||
265 | /** | 276 | /** |
... | @@ -268,9 +279,11 @@ public class BgpSessionManager { | ... | @@ -268,9 +279,11 @@ public class BgpSessionManager { |
268 | * @param bgpSession the BGP session the route entry update was | 279 | * @param bgpSession the BGP session the route entry update was |
269 | * received on | 280 | * received on |
270 | * @param bgpRouteEntry the added/updated route entry | 281 | * @param bgpRouteEntry the added/updated route entry |
282 | + * @return the result route update that should be forwarded to the | ||
283 | + * Route Listener, or null if no route update should be forwarded | ||
271 | */ | 284 | */ |
272 | - private void processAddedRoute(BgpSession bgpSession, | 285 | + private RouteUpdate processAddedRoute(BgpSession bgpSession, |
273 | - BgpRouteEntry bgpRouteEntry) { | 286 | + BgpRouteEntry bgpRouteEntry) { |
274 | RouteUpdate routeUpdate; | 287 | RouteUpdate routeUpdate; |
275 | BgpRouteEntry bestBgpRouteEntry = | 288 | BgpRouteEntry bestBgpRouteEntry = |
276 | bgpRoutes.get(bgpRouteEntry.prefix()); | 289 | bgpRoutes.get(bgpRouteEntry.prefix()); |
... | @@ -284,9 +297,7 @@ public class BgpSessionManager { | ... | @@ -284,9 +297,7 @@ public class BgpSessionManager { |
284 | bgpRoutes.put(bgpRouteEntry.prefix(), bgpRouteEntry); | 297 | bgpRoutes.put(bgpRouteEntry.prefix(), bgpRouteEntry); |
285 | routeUpdate = | 298 | routeUpdate = |
286 | new RouteUpdate(RouteUpdate.Type.UPDATE, bgpRouteEntry); | 299 | new RouteUpdate(RouteUpdate.Type.UPDATE, bgpRouteEntry); |
287 | - // Forward the result route updates to the Route Listener | 300 | + return routeUpdate; |
288 | - routeListener.update(routeUpdate); | ||
289 | - return; | ||
290 | } | 301 | } |
291 | 302 | ||
292 | // | 303 | // |
... | @@ -296,7 +307,7 @@ public class BgpSessionManager { | ... | @@ -296,7 +307,7 @@ public class BgpSessionManager { |
296 | // | 307 | // |
297 | if (bestBgpRouteEntry.getBgpSession() != | 308 | if (bestBgpRouteEntry.getBgpSession() != |
298 | bgpRouteEntry.getBgpSession()) { | 309 | bgpRouteEntry.getBgpSession()) { |
299 | - return; | 310 | + return null; // Nothing to do |
300 | } | 311 | } |
301 | 312 | ||
302 | // Find the next best route | 313 | // Find the next best route |
... | @@ -315,8 +326,7 @@ public class BgpSessionManager { | ... | @@ -315,8 +326,7 @@ public class BgpSessionManager { |
315 | bgpRoutes.put(bestBgpRouteEntry.prefix(), bestBgpRouteEntry); | 326 | bgpRoutes.put(bestBgpRouteEntry.prefix(), bestBgpRouteEntry); |
316 | routeUpdate = new RouteUpdate(RouteUpdate.Type.UPDATE, | 327 | routeUpdate = new RouteUpdate(RouteUpdate.Type.UPDATE, |
317 | bestBgpRouteEntry); | 328 | bestBgpRouteEntry); |
318 | - // Forward the result route updates to the Route Listener | 329 | + return routeUpdate; |
319 | - routeListener.update(routeUpdate); | ||
320 | } | 330 | } |
321 | 331 | ||
322 | /** | 332 | /** |
... | @@ -325,9 +335,11 @@ public class BgpSessionManager { | ... | @@ -325,9 +335,11 @@ public class BgpSessionManager { |
325 | * @param bgpSession the BGP session the route entry update was | 335 | * @param bgpSession the BGP session the route entry update was |
326 | * received on | 336 | * received on |
327 | * @param bgpRouteEntry the deleted route entry | 337 | * @param bgpRouteEntry the deleted route entry |
338 | + * @return the result route update that should be forwarded to the | ||
339 | + * Route Listener, or null if no route update should be forwarded | ||
328 | */ | 340 | */ |
329 | - private void processDeletedRoute(BgpSession bgpSession, | 341 | + private RouteUpdate processDeletedRoute(BgpSession bgpSession, |
330 | - BgpRouteEntry bgpRouteEntry) { | 342 | + BgpRouteEntry bgpRouteEntry) { |
331 | RouteUpdate routeUpdate; | 343 | RouteUpdate routeUpdate; |
332 | BgpRouteEntry bestBgpRouteEntry = | 344 | BgpRouteEntry bestBgpRouteEntry = |
333 | bgpRoutes.get(bgpRouteEntry.prefix()); | 345 | bgpRoutes.get(bgpRouteEntry.prefix()); |
... | @@ -340,7 +352,7 @@ public class BgpSessionManager { | ... | @@ -340,7 +352,7 @@ public class BgpSessionManager { |
340 | // because we need to check whether this is same object. | 352 | // because we need to check whether this is same object. |
341 | // | 353 | // |
342 | if (bgpRouteEntry != bestBgpRouteEntry) { | 354 | if (bgpRouteEntry != bestBgpRouteEntry) { |
343 | - return; // Nothing to do | 355 | + return null; // Nothing to do |
344 | } | 356 | } |
345 | 357 | ||
346 | // | 358 | // |
... | @@ -353,9 +365,7 @@ public class BgpSessionManager { | ... | @@ -353,9 +365,7 @@ public class BgpSessionManager { |
353 | bestBgpRouteEntry); | 365 | bestBgpRouteEntry); |
354 | routeUpdate = new RouteUpdate(RouteUpdate.Type.UPDATE, | 366 | routeUpdate = new RouteUpdate(RouteUpdate.Type.UPDATE, |
355 | bestBgpRouteEntry); | 367 | bestBgpRouteEntry); |
356 | - // Forward the result route updates to the Route Listener | 368 | + return routeUpdate; |
357 | - routeListener.update(routeUpdate); | ||
358 | - return; | ||
359 | } | 369 | } |
360 | 370 | ||
361 | // | 371 | // |
... | @@ -364,8 +374,7 @@ public class BgpSessionManager { | ... | @@ -364,8 +374,7 @@ public class BgpSessionManager { |
364 | bgpRoutes.remove(bgpRouteEntry.prefix()); | 374 | bgpRoutes.remove(bgpRouteEntry.prefix()); |
365 | routeUpdate = new RouteUpdate(RouteUpdate.Type.DELETE, | 375 | routeUpdate = new RouteUpdate(RouteUpdate.Type.DELETE, |
366 | bgpRouteEntry); | 376 | bgpRouteEntry); |
367 | - // Forward the result route updates to the Route Listener | 377 | + return routeUpdate; |
368 | - routeListener.update(routeUpdate); | ||
369 | } | 378 | } |
370 | 379 | ||
371 | /** | 380 | /** | ... | ... |
... | @@ -325,13 +325,13 @@ public class IntentSyncTest extends AbstractIntentTest { | ... | @@ -325,13 +325,13 @@ public class IntentSyncTest extends AbstractIntentTest { |
325 | .andReturn(IntentState.WITHDRAWING).anyTimes(); | 325 | .andReturn(IntentState.WITHDRAWING).anyTimes(); |
326 | expect(intentService.getIntents()).andReturn(intents).anyTimes(); | 326 | expect(intentService.getIntents()).andReturn(intents).anyTimes(); |
327 | 327 | ||
328 | - IntentOperations.Builder builder = IntentOperations.builder(null); //FIXME null | 328 | + IntentOperations.Builder builder = IntentOperations.builder(APPID); |
329 | builder.addWithdrawOperation(intent2.id()); | 329 | builder.addWithdrawOperation(intent2.id()); |
330 | builder.addWithdrawOperation(intent4.id()); | 330 | builder.addWithdrawOperation(intent4.id()); |
331 | intentService.execute(TestIntentServiceHelper.eqExceptId( | 331 | intentService.execute(TestIntentServiceHelper.eqExceptId( |
332 | builder.build())); | 332 | builder.build())); |
333 | 333 | ||
334 | - builder = IntentOperations.builder(null); //FIXME null | 334 | + builder = IntentOperations.builder(APPID); |
335 | builder.addSubmitOperation(intent3); | 335 | builder.addSubmitOperation(intent3); |
336 | builder.addSubmitOperation(intent4Update); | 336 | builder.addSubmitOperation(intent4Update); |
337 | builder.addSubmitOperation(intent6); | 337 | builder.addSubmitOperation(intent6); | ... | ... |
... | @@ -566,7 +566,7 @@ public class PeerConnectivityManagerTest extends AbstractIntentTest { | ... | @@ -566,7 +566,7 @@ public class PeerConnectivityManagerTest extends AbstractIntentTest { |
566 | reset(intentService); | 566 | reset(intentService); |
567 | 567 | ||
568 | // Setup the expected intents | 568 | // Setup the expected intents |
569 | - IntentOperations.Builder builder = IntentOperations.builder(null); //FIXME null | 569 | + IntentOperations.Builder builder = IntentOperations.builder(APPID); |
570 | for (Intent intent : intentList) { | 570 | for (Intent intent : intentList) { |
571 | builder.addSubmitOperation(intent); | 571 | builder.addSubmitOperation(intent); |
572 | } | 572 | } |
... | @@ -601,9 +601,8 @@ public class PeerConnectivityManagerTest extends AbstractIntentTest { | ... | @@ -601,9 +601,8 @@ public class PeerConnectivityManagerTest extends AbstractIntentTest { |
601 | replay(configInfoService); | 601 | replay(configInfoService); |
602 | 602 | ||
603 | reset(intentService); | 603 | reset(intentService); |
604 | - IntentOperations.Builder builder = IntentOperations.builder(null); //FIXME null | 604 | + IntentOperations.Builder builder = IntentOperations.builder(APPID); |
605 | - intentService.execute(TestIntentServiceHelper.eqExceptId( | 605 | + intentService.execute(builder.build()); |
606 | - builder.build())); | ||
607 | replay(intentService); | 606 | replay(intentService); |
608 | peerConnectivityManager.start(); | 607 | peerConnectivityManager.start(); |
609 | verify(intentService); | 608 | verify(intentService); |
... | @@ -627,9 +626,8 @@ public class PeerConnectivityManagerTest extends AbstractIntentTest { | ... | @@ -627,9 +626,8 @@ public class PeerConnectivityManagerTest extends AbstractIntentTest { |
627 | replay(configInfoService); | 626 | replay(configInfoService); |
628 | 627 | ||
629 | reset(intentService); | 628 | reset(intentService); |
630 | - IntentOperations.Builder builder = IntentOperations.builder(null); //FIXME null | 629 | + IntentOperations.Builder builder = IntentOperations.builder(APPID); |
631 | - intentService.execute(TestIntentServiceHelper.eqExceptId( | 630 | + intentService.execute(builder.build()); |
632 | - builder.build())); | ||
633 | replay(intentService); | 631 | replay(intentService); |
634 | peerConnectivityManager.start(); | 632 | peerConnectivityManager.start(); |
635 | verify(intentService); | 633 | verify(intentService); | ... | ... |
... | @@ -25,6 +25,7 @@ import static org.easymock.EasyMock.verify; | ... | @@ -25,6 +25,7 @@ import static org.easymock.EasyMock.verify; |
25 | import static org.junit.Assert.assertEquals; | 25 | import static org.junit.Assert.assertEquals; |
26 | import static org.junit.Assert.assertTrue; | 26 | import static org.junit.Assert.assertTrue; |
27 | 27 | ||
28 | +import java.util.Collections; | ||
28 | import java.util.HashMap; | 29 | import java.util.HashMap; |
29 | import java.util.HashSet; | 30 | import java.util.HashSet; |
30 | import java.util.Map; | 31 | import java.util.Map; |
... | @@ -50,6 +51,7 @@ import org.onlab.onos.net.host.HostListener; | ... | @@ -50,6 +51,7 @@ import org.onlab.onos.net.host.HostListener; |
50 | import org.onlab.onos.net.host.HostService; | 51 | import org.onlab.onos.net.host.HostService; |
51 | import org.onlab.onos.net.host.InterfaceIpAddress; | 52 | import org.onlab.onos.net.host.InterfaceIpAddress; |
52 | import org.onlab.onos.net.intent.Intent; | 53 | import org.onlab.onos.net.intent.Intent; |
54 | +import org.onlab.onos.net.intent.IntentOperations; | ||
53 | import org.onlab.onos.net.intent.IntentService; | 55 | import org.onlab.onos.net.intent.IntentService; |
54 | import org.onlab.onos.net.intent.MultiPointToSinglePointIntent; | 56 | import org.onlab.onos.net.intent.MultiPointToSinglePointIntent; |
55 | import org.onlab.onos.net.intent.AbstractIntentTest; | 57 | import org.onlab.onos.net.intent.AbstractIntentTest; |
... | @@ -234,7 +236,7 @@ public class RouterTest extends AbstractIntentTest { | ... | @@ -234,7 +236,7 @@ public class RouterTest extends AbstractIntentTest { |
234 | * This method tests adding a route entry. | 236 | * This method tests adding a route entry. |
235 | */ | 237 | */ |
236 | @Test | 238 | @Test |
237 | - public void testProcessRouteAdd() throws TestUtilsException { | 239 | + public void testRouteAdd() throws TestUtilsException { |
238 | // Construct a route entry | 240 | // Construct a route entry |
239 | RouteEntry routeEntry = new RouteEntry( | 241 | RouteEntry routeEntry = new RouteEntry( |
240 | Ip4Prefix.valueOf("1.1.1.0/24"), | 242 | Ip4Prefix.valueOf("1.1.1.0/24"), |
... | @@ -261,13 +263,19 @@ public class RouterTest extends AbstractIntentTest { | ... | @@ -261,13 +263,19 @@ public class RouterTest extends AbstractIntentTest { |
261 | 263 | ||
262 | // Set up test expectation | 264 | // Set up test expectation |
263 | reset(intentService); | 265 | reset(intentService); |
264 | - intentService.submit(TestIntentServiceHelper.eqExceptId(intent)); | 266 | + // Setup the expected intents |
267 | + IntentOperations.Builder builder = IntentOperations.builder(APPID); | ||
268 | + builder.addSubmitOperation(intent); | ||
269 | + intentService.execute(TestIntentServiceHelper.eqExceptId( | ||
270 | + builder.build())); | ||
265 | replay(intentService); | 271 | replay(intentService); |
266 | 272 | ||
267 | - // Call the processRouteAdd() method in Router class | 273 | + // Call the processRouteUpdates() method in Router class |
268 | intentSynchronizer.leaderChanged(true); | 274 | intentSynchronizer.leaderChanged(true); |
269 | TestUtils.setField(intentSynchronizer, "isActivatedLeader", true); | 275 | TestUtils.setField(intentSynchronizer, "isActivatedLeader", true); |
270 | - router.processRouteAdd(routeEntry); | 276 | + RouteUpdate routeUpdate = new RouteUpdate(RouteUpdate.Type.UPDATE, |
277 | + routeEntry); | ||
278 | + router.processRouteUpdates(Collections.<RouteUpdate>singletonList(routeUpdate)); | ||
271 | 279 | ||
272 | // Verify | 280 | // Verify |
273 | assertEquals(router.getRoutes().size(), 1); | 281 | assertEquals(router.getRoutes().size(), 1); |
... | @@ -289,32 +297,16 @@ public class RouterTest extends AbstractIntentTest { | ... | @@ -289,32 +297,16 @@ public class RouterTest extends AbstractIntentTest { |
289 | @Test | 297 | @Test |
290 | public void testRouteUpdate() throws TestUtilsException { | 298 | public void testRouteUpdate() throws TestUtilsException { |
291 | // Firstly add a route | 299 | // Firstly add a route |
292 | - testProcessRouteAdd(); | 300 | + testRouteAdd(); |
301 | + | ||
302 | + Intent addedIntent = | ||
303 | + intentSynchronizer.getRouteIntents().iterator().next(); | ||
293 | 304 | ||
294 | // Construct the existing route entry | 305 | // Construct the existing route entry |
295 | RouteEntry routeEntry = new RouteEntry( | 306 | RouteEntry routeEntry = new RouteEntry( |
296 | Ip4Prefix.valueOf("1.1.1.0/24"), | 307 | Ip4Prefix.valueOf("1.1.1.0/24"), |
297 | Ip4Address.valueOf("192.168.10.1")); | 308 | Ip4Address.valueOf("192.168.10.1")); |
298 | 309 | ||
299 | - // Construct the existing MultiPointToSinglePointIntent intent | ||
300 | - TrafficSelector.Builder selectorBuilder = | ||
301 | - DefaultTrafficSelector.builder(); | ||
302 | - selectorBuilder.matchEthType(Ethernet.TYPE_IPV4).matchIPDst( | ||
303 | - routeEntry.prefix()); | ||
304 | - | ||
305 | - TrafficTreatment.Builder treatmentBuilder = | ||
306 | - DefaultTrafficTreatment.builder(); | ||
307 | - treatmentBuilder.setEthDst(MacAddress.valueOf("00:00:00:00:00:01")); | ||
308 | - | ||
309 | - Set<ConnectPoint> ingressPoints = new HashSet<ConnectPoint>(); | ||
310 | - ingressPoints.add(SW2_ETH1); | ||
311 | - ingressPoints.add(SW3_ETH1); | ||
312 | - | ||
313 | - MultiPointToSinglePointIntent intent = | ||
314 | - new MultiPointToSinglePointIntent(APPID, | ||
315 | - selectorBuilder.build(), treatmentBuilder.build(), | ||
316 | - ingressPoints, SW1_ETH1); | ||
317 | - | ||
318 | // Start to construct a new route entry and new intent | 310 | // Start to construct a new route entry and new intent |
319 | RouteEntry routeEntryUpdate = new RouteEntry( | 311 | RouteEntry routeEntryUpdate = new RouteEntry( |
320 | Ip4Prefix.valueOf("1.1.1.0/24"), | 312 | Ip4Prefix.valueOf("1.1.1.0/24"), |
... | @@ -343,14 +335,23 @@ public class RouterTest extends AbstractIntentTest { | ... | @@ -343,14 +335,23 @@ public class RouterTest extends AbstractIntentTest { |
343 | 335 | ||
344 | // Set up test expectation | 336 | // Set up test expectation |
345 | reset(intentService); | 337 | reset(intentService); |
346 | - intentService.withdraw(TestIntentServiceHelper.eqExceptId(intent)); | 338 | + // Setup the expected intents |
347 | - intentService.submit(TestIntentServiceHelper.eqExceptId(intentNew)); | 339 | + IntentOperations.Builder builder = IntentOperations.builder(APPID); |
340 | + builder.addWithdrawOperation(addedIntent.id()); | ||
341 | + intentService.execute(TestIntentServiceHelper.eqExceptId( | ||
342 | + builder.build())); | ||
343 | + builder = IntentOperations.builder(APPID); | ||
344 | + builder.addSubmitOperation(intentNew); | ||
345 | + intentService.execute(TestIntentServiceHelper.eqExceptId( | ||
346 | + builder.build())); | ||
348 | replay(intentService); | 347 | replay(intentService); |
349 | 348 | ||
350 | - // Call the processRouteAdd() method in Router class | 349 | + // Call the processRouteUpdates() method in Router class |
351 | intentSynchronizer.leaderChanged(true); | 350 | intentSynchronizer.leaderChanged(true); |
352 | TestUtils.setField(intentSynchronizer, "isActivatedLeader", true); | 351 | TestUtils.setField(intentSynchronizer, "isActivatedLeader", true); |
353 | - router.processRouteAdd(routeEntryUpdate); | 352 | + RouteUpdate routeUpdate = new RouteUpdate(RouteUpdate.Type.UPDATE, |
353 | + routeEntryUpdate); | ||
354 | + router.processRouteUpdates(Collections.<RouteUpdate>singletonList(routeUpdate)); | ||
354 | 355 | ||
355 | // Verify | 356 | // Verify |
356 | assertEquals(router.getRoutes().size(), 1); | 357 | assertEquals(router.getRoutes().size(), 1); |
... | @@ -368,43 +369,33 @@ public class RouterTest extends AbstractIntentTest { | ... | @@ -368,43 +369,33 @@ public class RouterTest extends AbstractIntentTest { |
368 | * This method tests deleting a route entry. | 369 | * This method tests deleting a route entry. |
369 | */ | 370 | */ |
370 | @Test | 371 | @Test |
371 | - public void testProcessRouteDelete() throws TestUtilsException { | 372 | + public void testRouteDelete() throws TestUtilsException { |
372 | // Firstly add a route | 373 | // Firstly add a route |
373 | - testProcessRouteAdd(); | 374 | + testRouteAdd(); |
375 | + | ||
376 | + Intent addedIntent = | ||
377 | + intentSynchronizer.getRouteIntents().iterator().next(); | ||
374 | 378 | ||
375 | // Construct the existing route entry | 379 | // Construct the existing route entry |
376 | RouteEntry routeEntry = new RouteEntry( | 380 | RouteEntry routeEntry = new RouteEntry( |
377 | Ip4Prefix.valueOf("1.1.1.0/24"), | 381 | Ip4Prefix.valueOf("1.1.1.0/24"), |
378 | Ip4Address.valueOf("192.168.10.1")); | 382 | Ip4Address.valueOf("192.168.10.1")); |
379 | 383 | ||
380 | - // Construct the existing MultiPointToSinglePointIntent intent | ||
381 | - TrafficSelector.Builder selectorBuilder = | ||
382 | - DefaultTrafficSelector.builder(); | ||
383 | - selectorBuilder.matchEthType(Ethernet.TYPE_IPV4).matchIPDst( | ||
384 | - routeEntry.prefix()); | ||
385 | - | ||
386 | - TrafficTreatment.Builder treatmentBuilder = | ||
387 | - DefaultTrafficTreatment.builder(); | ||
388 | - treatmentBuilder.setEthDst(MacAddress.valueOf("00:00:00:00:00:01")); | ||
389 | - | ||
390 | - Set<ConnectPoint> ingressPoints = new HashSet<ConnectPoint>(); | ||
391 | - ingressPoints.add(SW2_ETH1); | ||
392 | - ingressPoints.add(SW3_ETH1); | ||
393 | - | ||
394 | - MultiPointToSinglePointIntent intent = | ||
395 | - new MultiPointToSinglePointIntent(APPID, | ||
396 | - selectorBuilder.build(), treatmentBuilder.build(), | ||
397 | - ingressPoints, SW1_ETH1); | ||
398 | - | ||
399 | // Set up expectation | 384 | // Set up expectation |
400 | reset(intentService); | 385 | reset(intentService); |
401 | - intentService.withdraw(TestIntentServiceHelper.eqExceptId(intent)); | 386 | + // Setup the expected intents |
387 | + IntentOperations.Builder builder = IntentOperations.builder(APPID); | ||
388 | + builder.addWithdrawOperation(addedIntent.id()); | ||
389 | + intentService.execute(TestIntentServiceHelper.eqExceptId( | ||
390 | + builder.build())); | ||
402 | replay(intentService); | 391 | replay(intentService); |
403 | 392 | ||
404 | - // Call route deleting method in Router class | 393 | + // Call the processRouteUpdates() method in Router class |
405 | intentSynchronizer.leaderChanged(true); | 394 | intentSynchronizer.leaderChanged(true); |
406 | TestUtils.setField(intentSynchronizer, "isActivatedLeader", true); | 395 | TestUtils.setField(intentSynchronizer, "isActivatedLeader", true); |
407 | - router.processRouteDelete(routeEntry); | 396 | + RouteUpdate routeUpdate = new RouteUpdate(RouteUpdate.Type.DELETE, |
397 | + routeEntry); | ||
398 | + router.processRouteUpdates(Collections.<RouteUpdate>singletonList(routeUpdate)); | ||
408 | 399 | ||
409 | // Verify | 400 | // Verify |
410 | assertEquals(router.getRoutes().size(), 0); | 401 | assertEquals(router.getRoutes().size(), 0); |
... | @@ -428,10 +419,12 @@ public class RouterTest extends AbstractIntentTest { | ... | @@ -428,10 +419,12 @@ public class RouterTest extends AbstractIntentTest { |
428 | reset(intentService); | 419 | reset(intentService); |
429 | replay(intentService); | 420 | replay(intentService); |
430 | 421 | ||
431 | - // Call the processRouteAdd() method in Router class | 422 | + // Call the processRouteUpdates() method in Router class |
432 | intentSynchronizer.leaderChanged(true); | 423 | intentSynchronizer.leaderChanged(true); |
433 | TestUtils.setField(intentSynchronizer, "isActivatedLeader", true); | 424 | TestUtils.setField(intentSynchronizer, "isActivatedLeader", true); |
434 | - router.processRouteAdd(routeEntry); | 425 | + RouteUpdate routeUpdate = new RouteUpdate(RouteUpdate.Type.UPDATE, |
426 | + routeEntry); | ||
427 | + router.processRouteUpdates(Collections.<RouteUpdate>singletonList(routeUpdate)); | ||
435 | 428 | ||
436 | // Verify | 429 | // Verify |
437 | assertEquals(router.getRoutes().size(), 1); | 430 | assertEquals(router.getRoutes().size(), 1); | ... | ... |
... | @@ -24,6 +24,7 @@ import static org.easymock.EasyMock.verify; | ... | @@ -24,6 +24,7 @@ import static org.easymock.EasyMock.verify; |
24 | import static org.junit.Assert.assertEquals; | 24 | import static org.junit.Assert.assertEquals; |
25 | import static org.junit.Assert.assertTrue; | 25 | import static org.junit.Assert.assertTrue; |
26 | 26 | ||
27 | +import java.util.Collections; | ||
27 | import java.util.HashMap; | 28 | import java.util.HashMap; |
28 | import java.util.HashSet; | 29 | import java.util.HashSet; |
29 | import java.util.Map; | 30 | import java.util.Map; |
... | @@ -50,6 +51,7 @@ import org.onlab.onos.net.host.HostEvent; | ... | @@ -50,6 +51,7 @@ import org.onlab.onos.net.host.HostEvent; |
50 | import org.onlab.onos.net.host.HostService; | 51 | import org.onlab.onos.net.host.HostService; |
51 | import org.onlab.onos.net.host.InterfaceIpAddress; | 52 | import org.onlab.onos.net.host.InterfaceIpAddress; |
52 | import org.onlab.onos.net.intent.Intent; | 53 | import org.onlab.onos.net.intent.Intent; |
54 | +import org.onlab.onos.net.intent.IntentOperations; | ||
53 | import org.onlab.onos.net.intent.IntentService; | 55 | import org.onlab.onos.net.intent.IntentService; |
54 | import org.onlab.onos.net.intent.MultiPointToSinglePointIntent; | 56 | import org.onlab.onos.net.intent.MultiPointToSinglePointIntent; |
55 | import org.onlab.onos.net.intent.AbstractIntentTest; | 57 | import org.onlab.onos.net.intent.AbstractIntentTest; |
... | @@ -196,7 +198,7 @@ public class RouterTestWithAsyncArp extends AbstractIntentTest { | ... | @@ -196,7 +198,7 @@ public class RouterTestWithAsyncArp extends AbstractIntentTest { |
196 | * This method tests adding a route entry. | 198 | * This method tests adding a route entry. |
197 | */ | 199 | */ |
198 | @Test | 200 | @Test |
199 | - public void testProcessRouteAdd() throws TestUtilsException { | 201 | + public void testRouteAdd() throws TestUtilsException { |
200 | 202 | ||
201 | // Construct a route entry | 203 | // Construct a route entry |
202 | RouteEntry routeEntry = new RouteEntry( | 204 | RouteEntry routeEntry = new RouteEntry( |
... | @@ -214,13 +216,18 @@ public class RouterTestWithAsyncArp extends AbstractIntentTest { | ... | @@ -214,13 +216,18 @@ public class RouterTestWithAsyncArp extends AbstractIntentTest { |
214 | replay(hostService); | 216 | replay(hostService); |
215 | 217 | ||
216 | reset(intentService); | 218 | reset(intentService); |
217 | - intentService.submit(TestIntentServiceHelper.eqExceptId(intent)); | 219 | + IntentOperations.Builder builder = IntentOperations.builder(APPID); |
220 | + builder.addSubmitOperation(intent); | ||
221 | + intentService.execute(TestIntentServiceHelper.eqExceptId( | ||
222 | + builder.build())); | ||
218 | replay(intentService); | 223 | replay(intentService); |
219 | 224 | ||
220 | - // Call the processRouteAdd() method in Router class | 225 | + // Call the processRouteUpdates() method in Router class |
221 | intentSynchronizer.leaderChanged(true); | 226 | intentSynchronizer.leaderChanged(true); |
222 | TestUtils.setField(intentSynchronizer, "isActivatedLeader", true); | 227 | TestUtils.setField(intentSynchronizer, "isActivatedLeader", true); |
223 | - router.processRouteAdd(routeEntry); | 228 | + RouteUpdate routeUpdate = new RouteUpdate(RouteUpdate.Type.UPDATE, |
229 | + routeEntry); | ||
230 | + router.processRouteUpdates(Collections.<RouteUpdate>singletonList(routeUpdate)); | ||
224 | 231 | ||
225 | Host host = new DefaultHost(ProviderId.NONE, HostId.NONE, | 232 | Host host = new DefaultHost(ProviderId.NONE, HostId.NONE, |
226 | MacAddress.valueOf("00:00:00:00:00:01"), VlanId.NONE, | 233 | MacAddress.valueOf("00:00:00:00:00:01"), VlanId.NONE, |
... | @@ -299,14 +306,22 @@ public class RouterTestWithAsyncArp extends AbstractIntentTest { | ... | @@ -299,14 +306,22 @@ public class RouterTestWithAsyncArp extends AbstractIntentTest { |
299 | replay(hostService); | 306 | replay(hostService); |
300 | 307 | ||
301 | reset(intentService); | 308 | reset(intentService); |
302 | - intentService.withdraw(TestIntentServiceHelper.eqExceptId(intent)); | 309 | + IntentOperations.Builder builder = IntentOperations.builder(APPID); |
303 | - intentService.submit(TestIntentServiceHelper.eqExceptId(intentNew)); | 310 | + builder.addWithdrawOperation(intent.id()); |
311 | + intentService.execute(TestIntentServiceHelper.eqExceptId( | ||
312 | + builder.build())); | ||
313 | + builder = IntentOperations.builder(APPID); | ||
314 | + builder.addSubmitOperation(intentNew); | ||
315 | + intentService.execute(TestIntentServiceHelper.eqExceptId( | ||
316 | + builder.build())); | ||
304 | replay(intentService); | 317 | replay(intentService); |
305 | 318 | ||
306 | - // Call the processRouteAdd() method in Router class | 319 | + // Call the processRouteUpdates() method in Router class |
307 | intentSynchronizer.leaderChanged(true); | 320 | intentSynchronizer.leaderChanged(true); |
308 | TestUtils.setField(intentSynchronizer, "isActivatedLeader", true); | 321 | TestUtils.setField(intentSynchronizer, "isActivatedLeader", true); |
309 | - router.processRouteAdd(routeEntryUpdate); | 322 | + RouteUpdate routeUpdate = new RouteUpdate(RouteUpdate.Type.UPDATE, |
323 | + routeEntryUpdate); | ||
324 | + router.processRouteUpdates(Collections.<RouteUpdate>singletonList(routeUpdate)); | ||
310 | 325 | ||
311 | Host host = new DefaultHost(ProviderId.NONE, HostId.NONE, | 326 | Host host = new DefaultHost(ProviderId.NONE, HostId.NONE, |
312 | MacAddress.valueOf("00:00:00:00:00:02"), VlanId.NONE, | 327 | MacAddress.valueOf("00:00:00:00:00:02"), VlanId.NONE, |
... | @@ -334,7 +349,7 @@ public class RouterTestWithAsyncArp extends AbstractIntentTest { | ... | @@ -334,7 +349,7 @@ public class RouterTestWithAsyncArp extends AbstractIntentTest { |
334 | * This method tests deleting a route entry. | 349 | * This method tests deleting a route entry. |
335 | */ | 350 | */ |
336 | @Test | 351 | @Test |
337 | - public void testProcessRouteDelete() throws TestUtilsException { | 352 | + public void testRouteDelete() throws TestUtilsException { |
338 | 353 | ||
339 | // Construct the existing route entry | 354 | // Construct the existing route entry |
340 | RouteEntry routeEntry = new RouteEntry( | 355 | RouteEntry routeEntry = new RouteEntry( |
... | @@ -351,13 +366,18 @@ public class RouterTestWithAsyncArp extends AbstractIntentTest { | ... | @@ -351,13 +366,18 @@ public class RouterTestWithAsyncArp extends AbstractIntentTest { |
351 | 366 | ||
352 | // Set up expectation | 367 | // Set up expectation |
353 | reset(intentService); | 368 | reset(intentService); |
354 | - intentService.withdraw(TestIntentServiceHelper.eqExceptId(intent)); | 369 | + IntentOperations.Builder builder = IntentOperations.builder(APPID); |
370 | + builder.addWithdrawOperation(intent.id()); | ||
371 | + intentService.execute(TestIntentServiceHelper.eqExceptId( | ||
372 | + builder.build())); | ||
355 | replay(intentService); | 373 | replay(intentService); |
356 | 374 | ||
357 | - // Call route deleting method in Router class | 375 | + // Call the processRouteUpdates() method in Router class |
358 | intentSynchronizer.leaderChanged(true); | 376 | intentSynchronizer.leaderChanged(true); |
359 | TestUtils.setField(intentSynchronizer, "isActivatedLeader", true); | 377 | TestUtils.setField(intentSynchronizer, "isActivatedLeader", true); |
360 | - router.processRouteDelete(routeEntry); | 378 | + RouteUpdate routeUpdate = new RouteUpdate(RouteUpdate.Type.DELETE, |
379 | + routeEntry); | ||
380 | + router.processRouteUpdates(Collections.<RouteUpdate>singletonList(routeUpdate)); | ||
361 | 381 | ||
362 | // Verify | 382 | // Verify |
363 | assertEquals(router.getRoutes().size(), 0); | 383 | assertEquals(router.getRoutes().size(), 0); | ... | ... |
... | @@ -236,9 +236,7 @@ public class SdnIpTest extends AbstractIntentTest { | ... | @@ -236,9 +236,7 @@ public class SdnIpTest extends AbstractIntentTest { |
236 | TestUtils.setField(intentSynchronizer, "isActivatedLeader", true); | 236 | TestUtils.setField(intentSynchronizer, "isActivatedLeader", true); |
237 | 237 | ||
238 | // Add route updates | 238 | // Add route updates |
239 | - for (RouteUpdate update : routeUpdates) { | 239 | + router.processRouteUpdates(routeUpdates); |
240 | - router.processRouteAdd(update.routeEntry()); | ||
241 | - } | ||
242 | 240 | ||
243 | latch.await(5000, TimeUnit.MILLISECONDS); | 241 | latch.await(5000, TimeUnit.MILLISECONDS); |
244 | 242 | ||
... | @@ -304,17 +302,19 @@ public class SdnIpTest extends AbstractIntentTest { | ... | @@ -304,17 +302,19 @@ public class SdnIpTest extends AbstractIntentTest { |
304 | TestUtils.setField(intentSynchronizer, "isActivatedLeader", true); | 302 | TestUtils.setField(intentSynchronizer, "isActivatedLeader", true); |
305 | 303 | ||
306 | // Send the add updates first | 304 | // Send the add updates first |
307 | - for (RouteUpdate update : routeUpdates) { | 305 | + router.processRouteUpdates(routeUpdates); |
308 | - router.processRouteAdd(update.routeEntry()); | ||
309 | - } | ||
310 | 306 | ||
311 | // Give some time to let the intents be submitted | 307 | // Give some time to let the intents be submitted |
312 | installCount.await(5000, TimeUnit.MILLISECONDS); | 308 | installCount.await(5000, TimeUnit.MILLISECONDS); |
313 | 309 | ||
314 | // Send the DELETE updates | 310 | // Send the DELETE updates |
311 | + List<RouteUpdate> deleteRouteUpdates = new ArrayList<>(); | ||
315 | for (RouteUpdate update : routeUpdates) { | 312 | for (RouteUpdate update : routeUpdates) { |
316 | - router.processRouteDelete(update.routeEntry()); | 313 | + RouteUpdate deleteUpdate = new RouteUpdate(RouteUpdate.Type.DELETE, |
314 | + update.routeEntry()); | ||
315 | + deleteRouteUpdates.add(deleteUpdate); | ||
317 | } | 316 | } |
317 | + router.processRouteUpdates(deleteRouteUpdates); | ||
318 | 318 | ||
319 | deleteCount.await(5000, TimeUnit.MILLISECONDS); | 319 | deleteCount.await(5000, TimeUnit.MILLISECONDS); |
320 | 320 | ... | ... |
... | @@ -82,7 +82,7 @@ public class BgpSessionManagerTest { | ... | @@ -82,7 +82,7 @@ public class BgpSessionManagerTest { |
82 | */ | 82 | */ |
83 | private class DummyRouteListener implements RouteListener { | 83 | private class DummyRouteListener implements RouteListener { |
84 | @Override | 84 | @Override |
85 | - public void update(RouteUpdate routeUpdate) { | 85 | + public void update(Collection<RouteUpdate> routeUpdate) { |
86 | // Nothing to do | 86 | // Nothing to do |
87 | } | 87 | } |
88 | } | 88 | } | ... | ... |
-
Please register or login to post a comment