[LISPMAP-68] Mapping updates over a period of time throw exception and SMRs are not sent

Duplicate — Thu, 19 Oct 2017 21:29:14 +0000

When making use of ODL for an extended period of time, subscribing requesters and updating mappings for those subscribers, eventually a mapping update will throw the exception below and the SMRs will not be sent.

How to reproduce: Set up a scenario with two xTRs and two RTRs in parallel, switch traffic from one RTR to the other, with different time gaps between the mapping updates, until the problem appears.

2015-07-29 23:06:51,163 | ERROR | lt-dispatcher-17 | DataChangeListener | 157 - org.opendaylight.controller.sal-distributed-datastore - 1.2.1.SNAPSHOT | Error notifying listener org.opendaylight.lispflowmapping.implementation.mdsal.MappingDataListener
java.util.ConcurrentModificationException
at java.util.HashMap$HashIterator.nextEntry(HashMap.java:922)[:1.7.0_80]
at java.util.HashMap$KeyIterator.next(HashMap.java:956)[:1.7.0_80]
at org.opendaylight.lispflowmapping.implementation.lisp.MapServer.handleSmr(MapServer.java:324)[226:org.opendaylight.lispflowmapping.mappingservice.implementation:1.2.1.SNAPSHOT]
at org.opendaylight.lispflowmapping.implementation.lisp.MapServer.sendSmrs(MapServer.java:302)[226:org.opendaylight.lispflowmapping.mappingservice.implementation:1.2.1.SNAPSHOT]
at org.opendaylight.lispflowmapping.implementation.lisp.MapServer.handleMapRegister(MapServer.java:143)[226:org.opendaylight.lispflowmapping.mappingservice.implementation:1.2.1.SNAPSHOT]
at org.opendaylight.lispflowmapping.implementation.LispMappingService.handleMapRegister(LispMappingService.java:318)[226:org.opendaylight.lispflowmapping.mappingservice.implementation:1.2.1.SNAPSHOT]
at org.opendaylight.lispflowmapping.implementation.mdsal.MappingDataListener.onDataChanged(MappingDataListener.java:84)[226:org.opendaylight.lispflowmapping.mappingservice.implementation:1.2.1.SNAPSHOT]
at org.opendaylight.controller.md.sal.binding.impl.AbstractForwardedDataBroker$TranslatingDataChangeInvoker.onDataChanged(AbstractForwardedDataBroker.java:134)[135:org.opendaylight.controller.sal-binding-broker-impl:1.2.1.SNAPSHOT]
at org.opendaylight.controller.cluster.datastore.DataChangeListener.dataChanged(DataChangeListener.java:66)[157:org.opendaylight.controller.sal-distributed-datastore:1.2.1.SNAPSHOT]
at org.opendaylight.controller.cluster.datastore.DataChangeListener.handleReceive(DataChangeListener.java:39)[157:org.opendaylight.controller.sal-distributed-datastore:1.2.1.SNAPSHOT]
at org.opendaylight.controller.cluster.common.actor.AbstractUntypedActor.onReceive(AbstractUntypedActor.java:34)[149:org.opendaylight.controller.sal-clustering-commons:1.2.1.SNAPSHOT]
at akka.actor.UntypedActor$$anonfun$receive$1.applyOrElse(UntypedActor.scala:167)[142:com.typesafe.akka.actor:2.3.10]
at akka.actor.Actor$class.aroundReceive(Actor.scala:467)[142:com.typesafe.akka.actor:2.3.10]
at akka.actor.UntypedActor.aroundReceive(UntypedActor.scala:97)[142:com.typesafe.akka.actor:2.3.10]
at akka.actor.ActorCell.receiveMessage(ActorCell.scala:516)[142:com.typesafe.akka.actor:2.3.10]
at akka.actor.ActorCell.invoke(ActorCell.scala:487)[142:com.typesafe.akka.actor:2.3.10]
at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:254)[142:com.typesafe.akka.actor:2.3.10]
at akka.dispatch.Mailbox.run(Mailbox.scala:221)[142:com.typesafe.akka.actor:2.3.10]
at akka.dispatch.Mailbox.exec(Mailbox.scala:231)[142:com.typesafe.akka.actor:2.3.10]
at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)[139:org.scala-lang.scala-library:2.10.4.v20140209-180020-VFINAL-b66a39653b]
at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.pollAndExecAll(ForkJoinPool.java:1253)[139:org.scala-lang.scala-library:2.10.4.v20140209-180020-VFINAL-b66a39653b]
at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1346)[139:org.scala-lang.scala-library:2.10.4.v20140209-180020-VFINAL-b66a39653b]
at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)[139:org.scala-lang.scala-library:2.10.4.v20140209-180020-VFINAL-b66a39653b]
at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)[139:org.scala-lang.scala-library:2.10.4.v20140209-180020-VFINAL-b66a39653b]
2015-07-29 23:07:43,323 | ERROR | ult-dispatcher-2 | DataChangeListener | 157 - org.opendaylight.controller.sal-distributed-datastore - 1.2.1.SNAPSHOT | Error notifying listener org.opendaylight.lispflowmapping.implementation.mdsal.MappingDataListener
java.util.ConcurrentModificationException
at java.util.HashMap$HashIterator.nextEntry(HashMap.java:922)[:1.7.0_80]
at java.util.HashMap$KeyIterator.next(HashMap.java:956)[:1.7.0_80]
at org.opendaylight.lispflowmapping.implementation.lisp.MapServer.handleSmr(MapServer.java:324)[226:org.opendaylight.lispflowmapping.mappingservice.implementation:1.2.1.SNAPSHOT]
at org.opendaylight.lispflowmapping.implementation.lisp.MapServer.sendSmrs(MapServer.java:302)[226:org.opendaylight.lispflowmapping.mappingservice.implementation:1.2.1.SNAPSHOT]
at org.opendaylight.lispflowmapping.implementation.lisp.MapServer.handleMapRegister(MapServer.java:143)[226:org.opendaylight.lispflowmapping.mappingservice.implementation:1.2.1.SNAPSHOT]
at org.opendaylight.lispflowmapping.implementation.LispMappingService.handleMapRegister(LispMappingService.java:318)[226:org.opendaylight.lispflowmapping.mappingservice.implementation:1.2.1.SNAPSHOT]
at org.opendaylight.lispflowmapping.implementation.mdsal.MappingDataListener.onDataChanged(MappingDataListener.java:84)[226:org.opendaylight.lispflowmapping.mappingservice.implementation:1.2.1.SNAPSHOT]
at org.opendaylight.controller.md.sal.binding.impl.AbstractForwardedDataBroker$TranslatingDataChangeInvoker.onDataChanged(AbstractForwardedDataBroker.java:134)[135:org.opendaylight.controller.sal-binding-broker-impl:1.2.1.SNAPSHOT]
at org.opendaylight.controller.cluster.datastore.DataChangeListener.dataChanged(DataChangeListener.java:66)[157:org.opendaylight.controller.sal-distributed-datastore:1.2.1.SNAPSHOT]
at org.opendaylight.controller.cluster.datastore.DataChangeListener.handleReceive(DataChangeListener.java:39)[157:org.opendaylight.controller.sal-distributed-datastore:1.2.1.SNAPSHOT]
at org.opendaylight.controller.cluster.common.actor.AbstractUntypedActor.onReceive(AbstractUntypedActor.java:34)[149:org.opendaylight.controller.sal-clustering-commons:1.2.1.SNAPSHOT]
at akka.actor.UntypedActor$$anonfun$receive$1.applyOrElse(UntypedActor.scala:167)[142:com.typesafe.akka.actor:2.3.10]
at akka.actor.Actor$class.aroundReceive(Actor.scala:467)[142:com.typesafe.akka.actor:2.3.10]
at akka.actor.UntypedActor.aroundReceive(UntypedActor.scala:97)[142:com.typesafe.akka.actor:2.3.10]
at akka.actor.ActorCell.receiveMessage(ActorCell.scala:516)[142:com.typesafe.akka.actor:2.3.10]
at akka.actor.ActorCell.invoke(ActorCell.scala:487)[142:com.typesafe.akka.actor:2.3.10]
at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:254)[142:com.typesafe.akka.actor:2.3.10]
at akka.dispatch.Mailbox.run(Mailbox.scala:221)[142:com.typesafe.akka.actor:2.3.10]
at akka.dispatch.Mailbox.exec(Mailbox.scala:231)[142:com.typesafe.akka.actor:2.3.10]
at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)[139:org.scala-lang.scala-library:2.10.4.v20140209-180020-VFINAL-b66a39653b]
at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)[139:org.scala-lang.scala-library:2.10.4.v20140209-180020-VFINAL-b66a39653b]
at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)[139:org.scala-lang.scala-library:2.10.4.v20140209-180020-VFINAL-b66a39653b]
at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)[139:org.scala-lang.scala-library:2.10.4.v20140209-180020-VFINAL-b66a39653b]
2015-07-29 23:08:37,530 | ERROR | ult-dispatcher-3 | DataChangeListener | 157 - org.opendaylight.controller.sal-distributed-datastore - 1.2.1.SNAPSHOT | Error notifying listener org.opendaylight.lispflowmapping.implementation.mdsal.MappingDataListener
java.util.ConcurrentModificationException
at java.util.HashMap$HashIterator.nextEntry(HashMap.java:922)[:1.7.0_80]
at java.util.HashMap$KeyIterator.next(HashMap.java:956)[:1.7.0_80]
at org.opendaylight.lispflowmapping.implementation.lisp.MapServer.handleSmr(MapServer.java:324)[226:org.opendaylight.lispflowmapping.mappingservice.implementation:1.2.1.SNAPSHOT]
at org.opendaylight.lispflowmapping.implementation.lisp.MapServer.sendSmrs(MapServer.java:302)[226:org.opendaylight.lispflowmapping.mappingservice.implementation:1.2.1.SNAPSHOT]
at org.opendaylight.lispflowmapping.implementation.lisp.MapServer.handleMapRegister(MapServer.java:143)[226:org.opendaylight.lispflowmapping.mappingservice.implementation:1.2.1.SNAPSHOT]
at org.opendaylight.lispflowmapping.implementation.LispMappingService.handleMapRegister(LispMappingService.java:318)[226:org.opendaylight.lispflowmapping.mappingservice.implementation:1.2.1.SNAPSHOT]
at org.opendaylight.lispflowmapping.implementation.mdsal.MappingDataListener.onDataChanged(MappingDataListener.java:84)[226:org.opendaylight.lispflowmapping.mappingservice.implementation:1.2.1.SNAPSHOT]
at org.opendaylight.controller.md.sal.binding.impl.AbstractForwardedDataBroker$TranslatingDataChangeInvoker.onDataChanged(AbstractForwardedDataBroker.java:134)[135:org.opendaylight.controller.sal-binding-broker-impl:1.2.1.SNAPSHOT]
at org.opendaylight.controller.cluster.datastore.DataChangeListener.dataChanged(DataChangeListener.java:66)[157:org.opendaylight.controller.sal-distributed-datastore:1.2.1.SNAPSHOT]
at org.opendaylight.controller.cluster.datastore.DataChangeListener.handleReceive(DataChangeListener.java:39)[157:org.opendaylight.controller.sal-distributed-datastore:1.2.1.SNAPSHOT]
at org.opendaylight.controller.cluster.common.actor.AbstractUntypedActor.onReceive(AbstractUntypedActor.java:34)[149:org.opendaylight.controller.sal-clustering-commons:1.2.1.SNAPSHOT]
at akka.actor.UntypedActor$$anonfun$receive$1.applyOrElse(UntypedActor.scala:167)[142:com.typesafe.akka.actor:2.3.10]
at akka.actor.Actor$class.aroundReceive(Actor.scala:467)[142:com.typesafe.akka.actor:2.3.10]
at akka.actor.UntypedActor.aroundReceive(UntypedActor.scala:97)[142:com.typesafe.akka.actor:2.3.10]
at akka.actor.ActorCell.receiveMessage(ActorCell.scala:516)[142:com.typesafe.akka.actor:2.3.10]
at akka.actor.ActorCell.invoke(ActorCell.scala:487)[142:com.typesafe.akka.actor:2.3.10]
at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:254)[142:com.typesafe.akka.actor:2.3.10]
at akka.dispatch.Mailbox.run(Mailbox.scala:221)[142:com.typesafe.akka.actor:2.3.10]
at akka.dispatch.Mailbox.exec(Mailbox.scala:231)[142:com.typesafe.akka.actor:2.3.10]
at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)[139:org.scala-lang.scala-library:2.10.4.v20140209-180020-VFINAL-b66a39653b]
at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.pollAndExecAll(ForkJoinPool.java:1253)[139:org.scala-lang.scala-library:2.10.4.v20140209-180020-VFINAL-b66a39653b]
at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1346)[139:org.scala-lang.scala-library:2.10.4.v20140209-180020-VFINAL-b66a39653b]
at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)[139:org.scala-lang.scala-library:2.10.4.v20140209-180020-VFINAL-b66a39653b]
at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)[139:org.scala-lang.scala-library:2.10.4.v20140209-180020-VFINAL-b66a39653b]
2015-07-29 23:08:49,471 | ERROR | lt-dispatcher-16 | DataChangeListener | 157 - org.opendaylight.controller.sal-distributed-datastore - 1.2.1.SNAPSHOT | Error notifying listener org.opendaylight.lispflowmapping.implementation.mdsal.MappingDataListener
java.util.ConcurrentModificationException
at java.util.HashMap$HashIterator.nextEntry(HashMap.java:922)[:1.7.0_80]
at java.util.HashMap$KeyIterator.next(HashMap.java:956)[:1.7.0_80]
at org.opendaylight.lispflowmapping.implementation.lisp.MapServer.handleSmr(MapServer.java:324)[226:org.opendaylight.lispflowmapping.mappingservice.implementation:1.2.1.SNAPSHOT]
at org.opendaylight.lispflowmapping.implementation.lisp.MapServer.sendSmrs(MapServer.java:302)[226:org.opendaylight.lispflowmapping.mappingservice.implementation:1.2.1.SNAPSHOT]
at org.opendaylight.lispflowmapping.implementation.lisp.MapServer.handleMapRegister(MapServer.java:143)[226:org.opendaylight.lispflowmapping.mappingservice.implementation:1.2.1.SNAPSHOT]
at org.opendaylight.lispflowmapping.implementation.LispMappingService.handleMapRegister(LispMappingService.java:318)[226:org.opendaylight.lispflowmapping.mappingservice.implementation:1.2.1.SNAPSHOT]
at org.opendaylight.lispflowmapping.implementation.mdsal.MappingDataListener.onDataChanged(MappingDataListener.java:84)[226:org.opendaylight.lispflowmapping.mappingservice.implementation:1.2.1.SNAPSHOT]
at org.opendaylight.controller.md.sal.binding.impl.AbstractForwardedDataBroker$TranslatingDataChangeInvoker.onDataChanged(AbstractForwardedDataBroker.java:134)[135:org.opendaylight.controller.sal-binding-broker-impl:1.2.1.SNAPSHOT]
at org.opendaylight.controller.cluster.datastore.DataChangeListener.dataChanged(DataChangeListener.java:66)[157:org.opendaylight.controller.sal-distributed-datastore:1.2.1.SNAPSHOT]
at org.opendaylight.controller.cluster.datastore.DataChangeListener.handleReceive(DataChangeListener.java:39)[157:org.opendaylight.controller.sal-distributed-datastore:1.2.1.SNAPSHOT]
at org.opendaylight.controller.cluster.common.actor.AbstractUntypedActor.onReceive(AbstractUntypedActor.java:34)[149:org.opendaylight.controller.sal-clustering-commons:1.2.1.SNAPSHOT]
at akka.actor.UntypedActor$$anonfun$receive$1.applyOrElse(UntypedActor.scala:167)[142:com.typesafe.akka.actor:2.3.10]
at akka.actor.Actor$class.aroundReceive(Actor.scala:467)[142:com.typesafe.akka.actor:2.3.10]
at akka.actor.UntypedActor.aroundReceive(UntypedActor.scala:97)[142:com.typesafe.akka.actor:2.3.10]
at akka.actor.ActorCell.receiveMessage(ActorCell.scala:516)[142:com.typesafe.akka.actor:2.3.10]
at akka.actor.ActorCell.invoke(ActorCell.scala:487)[142:com.typesafe.akka.actor:2.3.10]
at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:254)[142:com.typesafe.akka.actor:2.3.10]
at akka.dispatch.Mailbox.run(Mailbox.scala:221)[142:com.typesafe.akka.actor:2.3.10]
at akka.dispatch.Mailbox.exec(Mailbox.scala:231)[142:com.typesafe.akka.actor:2.3.10]
at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)[139:org.scala-lang.scala-library:2.10.4.v20140209-180020-VFINAL-b66a39653b]
at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)[139:org.scala-lang.scala-library:2.10.4.v20140209-180020-VFINAL-b66a39653b]
at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)[139:org.scala-lang.scala-library:2.10.4.v20140209-180020-VFINAL-b66a39653b]
at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)[139:org.scala-lang.scala-library:2.10.4.v20140209-180020-VFINAL-b66a39653b]
2015-07-29 23:08:52,229 | ERROR | lt-dispatcher-17 | DataChangeListener | 157 - org.opendaylight.controller.sal-distributed-datastore - 1.2.1.SNAPSHOT | Error notifying listener org.opendaylight.lispflowmapping.implementation.mdsal.MappingDataListener
java.util.ConcurrentModificationException
at java.util.HashMap$HashIterator.nextEntry(HashMap.java:922)[:1.7.0_80]
at java.util.HashMap$KeyIterator.next(HashMap.java:956)[:1.7.0_80]
at org.opendaylight.lispflowmapping.implementation.lisp.MapServer.handleSmr(MapServer.java:324)[226:org.opendaylight.lispflowmapping.mappingservice.implementation:1.2.1.SNAPSHOT]
at org.opendaylight.lispflowmapping.implementation.lisp.MapServer.sendSmrs(MapServer.java:302)[226:org.opendaylight.lispflowmapping.mappingservice.implementation:1.2.1.SNAPSHOT]
at org.opendaylight.lispflowmapping.implementation.lisp.MapServer.handleMapRegister(MapServer.java:143)[226:org.opendaylight.lispflowmapping.mappingservice.implementation:1.2.1.SNAPSHOT]
at org.opendaylight.lispflowmapping.implementation.LispMappingService.handleMapRegister(LispMappingService.java:318)[226:org.opendaylight.lispflowmapping.mappingservice.implementation:1.2.1.SNAPSHOT]
at org.opendaylight.lispflowmapping.implementation.mdsal.MappingDataListener.onDataChanged(MappingDataListener.java:84)[226:org.opendaylight.lispflowmapping.mappingservice.implementation:1.2.1.SNAPSHOT]
at org.opendaylight.controller.md.sal.binding.impl.AbstractForwardedDataBroker$TranslatingDataChangeInvoker.onDataChanged(AbstractForwardedDataBroker.java:134)[135:org.opendaylight.controller.sal-binding-broker-impl:1.2.1.SNAPSHOT]
at org.opendaylight.controller.cluster.datastore.DataChangeListener.dataChanged(DataChangeListener.java:66)[157:org.opendaylight.controller.sal-distributed-datastore:1.2.1.SNAPSHOT]
at org.opendaylight.controller.cluster.datastore.DataChangeListener.handleReceive(DataChangeListener.java:39)[157:org.opendaylight.controller.sal-distributed-datastore:1.2.1.SNAPSHOT]
at org.opendaylight.controller.cluster.common.actor.AbstractUntypedActor.onReceive(AbstractUntypedActor.java:34)[149:org.opendaylight.controller.sal-clustering-commons:1.2.1.SNAPSHOT]
at akka.actor.UntypedActor$$anonfun$receive$1.applyOrElse(UntypedActor.scala:167)[142:com.typesafe.akka.actor:2.3.10]
at akka.actor.Actor$class.aroundReceive(Actor.scala:467)[142:com.typesafe.akka.actor:2.3.10]
at akka.actor.UntypedActor.aroundReceive(UntypedActor.scala:97)[142:com.typesafe.akka.actor:2.3.10]
at akka.actor.ActorCell.receiveMessage(ActorCell.scala:516)[142:com.typesafe.akka.actor:2.3.10]
at akka.actor.ActorCell.invoke(ActorCell.scala:487)[142:com.typesafe.akka.actor:2.3.10]
at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:254)[142:com.typesafe.akka.actor:2.3.10]
at akka.dispatch.Mailbox.run(Mailbox.scala:221)[142:com.typesafe.akka.actor:2.3.10]
at akka.dispatch.Mailbox.exec(Mailbox.scala:231)[142:com.typesafe.akka.actor:2.3.10]
at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)[139:org.scala-lang.scala-library:2.10.4.v20140209-180020-VFINAL-b66a39653b]
at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)[139:org.scala-lang.scala-library:2.10.4.v20140209-180020-VFINAL-b66a39653b]
at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)[139:org.scala-lang.scala-library:2.10.4.v20140209-180020-VFINAL-b66a39653b]
at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)[139:org.scala-lang.scala-library:2.10.4.v20140209-180020-VFINAL-b66a39653b]

OpenDaylight JIRA

[LISPMAP-68] Mapping updates over a period of time throw exception and SMRs are not sent