[MDSAL-362] Thread deadlock between mdsal and bgpcep when running Controller Clustering CSIT job

Relates — Mon, 20 Aug 2018 18:23:09 +0000
I've seen this issue in 4 job runs so far. Added other job links in subsequent comments.
Logs for the run can be found here:
https://logs.opendaylight.org/releng/vex-yul-odl-jenkins-1/builder-copy-sandbox-logs/178/controller-csit-3node-clustering-vpickard-all-oxygen/17/
For reference, here are the logs from the 2 deadlocked threads:
2018-07-03T18:42:02,391 | ERROR | infrautils.metrics.ThreadsWatcher-0 | ThreadsWatcher                   | 357 - org.opendaylight.infrautils.metrics-impl - 1.3.3.SNAPSHOT | Deadlocked thread stack trace: opendaylight-cluster-data-akka.actor.default-dispatcher-18 locked on org.opendaylight.bgpcep.pcep.topology.provider.config.PCEPTopologyProviderBean$PCEPTopologyProviderBeanCSS@3acd01d1 (owned by opendaylight-cluster-data-notification-dispatcher-61):
	 at org.opendaylight.bgpcep.pcep.topology.provider.config.PCEPTopologyProviderBean$PCEPTopologyProviderBeanCSS.instantiateServiceInstance(PCEPTopologyProviderBean.java:157)
	 at org.opendaylight.mdsal.singleton.dom.impl.ClusterSingletonServiceGroupImpl.lambda$startServices$0(ClusterSingletonServiceGroupImpl.java:648)
	 at org.opendaylight.mdsal.singleton.dom.impl.ClusterSingletonServiceGroupImpl$$Lambda$1173/320400615.accept(Unknown Source)
	 at java.util.ArrayList.forEach(ArrayList.java:1257)
	 at org.opendaylight.mdsal.singleton.dom.impl.ClusterSingletonServiceGroupImpl.startServices(ClusterSingletonServiceGroupImpl.java:645)
	 at org.opendaylight.mdsal.singleton.dom.impl.ClusterSingletonServiceGroupImpl.cleanupCandidateOwnershipChanged(ClusterSingletonServiceGroupImpl.java:506)
	 at org.opendaylight.mdsal.singleton.dom.impl.ClusterSingletonServiceGroupImpl.lockedOwnershipChanged(ClusterSingletonServiceGroupImpl.java:453)
	 at org.opendaylight.mdsal.singleton.dom.impl.ClusterSingletonServiceGroupImpl.ownershipChanged(ClusterSingletonServiceGroupImpl.java:433)
	 at org.opendaylight.mdsal.singleton.dom.impl.AbstractClusterSingletonServiceProviderImpl.ownershipChanged(AbstractClusterSingletonServiceProviderImpl.java:234)
	 at org.opendaylight.mdsal.singleton.dom.impl.DOMClusterSingletonServiceProviderImpl.ownershipChanged(DOMClusterSingletonServiceProviderImpl.java:23)
	 at org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnershipListenerActor.onEntityOwnershipChanged(EntityOwnershipListenerActor.java:44)
	 at org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnershipListenerActor.handleReceive(EntityOwnershipListenerActor.java:33)
	 at org.opendaylight.controller.cluster.common.actor.AbstractUntypedActor.onReceive(AbstractUntypedActor.java:38)
	 at akka.actor.UntypedActor$$anonfun$receive$1.applyOrElse(UntypedActor.scala:167)
	 at akka.actor.Actor.aroundReceive(Actor.scala:517)
	 at akka.actor.Actor.aroundReceive$(Actor.scala:515)
	 at akka.actor.UntypedActor.aroundReceive(UntypedActor.scala:97)
	 at akka.actor.ActorCell.receiveMessage(ActorCell.scala:590)
	 at akka.actor.ActorCell.invoke(ActorCell.scala:559)
	 at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:257)
	 at akka.dispatch.Mailbox.run(Mailbox.scala:224)
	 at akka.dispatch.Mailbox.exec(Mailbox.scala:234)
	 at akka.dispatch.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
	 at akka.dispatch.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
	 at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
	 at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)

2018-07-03T18:42:02,392 | ERROR | infrautils.metrics.ThreadsWatcher-0 | ThreadsWatcher                   | 357 - org.opendaylight.infrautils.metrics-impl - 1.3.3.SNAPSHOT | Deadlocked thread stack trace: opendaylight-cluster-data-notification-dispatcher-61 locked on java.util.concurrent.locks.ReentrantLock$FairSync@340ae920 (owned by opendaylight-cluster-data-akka.actor.default-dispatcher-18):
	 at sun.misc.Unsafe.park(Native Method)
	 at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
	 at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
	 at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:870)
	 at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:1199)
	 at java.util.concurrent.locks.ReentrantLock$FairSync.lock(ReentrantLock.java:224)
	 at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock.java:285)
	 at org.opendaylight.mdsal.singleton.dom.impl.ClusterSingletonServiceGroupImpl.unregisterService(ClusterSingletonServiceGroupImpl.java:393)
	 at org.opendaylight.mdsal.singleton.dom.impl.AbstractClusterSingletonServiceProviderImpl.removeRegistration(AbstractClusterSingletonServiceProviderImpl.java:151)
	 at org.opendaylight.mdsal.singleton.dom.impl.AbstractClusterSingletonServiceProviderImpl$1.removeRegistration(AbstractClusterSingletonServiceProviderImpl.java:123)
	 at org.opendaylight.yangtools.concepts.AbstractRegistration.close(AbstractRegistration.java:41)
	 at org.opendaylight.bgpcep.pcep.topology.provider.config.PCEPTopologyProviderBean$PCEPTopologyProviderBeanCSS.close(PCEPTopologyProviderBean.java:186)
	 at org.opendaylight.bgpcep.pcep.topology.provider.config.PCEPTopologyProviderBean.close(PCEPTopologyProviderBean.java:88)
	 at org.opendaylight.bgpcep.pcep.topology.provider.config.PCEPTopologyDeployerImpl.closeTopology(PCEPTopologyDeployerImpl.java:151)
	 at org.opendaylight.bgpcep.pcep.topology.provider.config.PCEPTopologyDeployerImpl.updateTopologyProvider(PCEPTopologyDeployerImpl.java:92)
	 at org.opendaylight.bgpcep.pcep.topology.provider.config.PCEPTopologyDeployerImpl.lambda$onDataTreeChanged$0(PCEPTopologyDeployerImpl.java:75)
	 at org.opendaylight.bgpcep.pcep.topology.provider.config.PCEPTopologyDeployerImpl$$Lambda$1179/1398407903.accept(Unknown Source)
	 at java.util.Iterator.forEachRemaining(Iterator.java:116)
	 at org.opendaylight.bgpcep.pcep.topology.provider.config.PCEPTopologyDeployerImpl.onDataTreeChanged(PCEPTopologyDeployerImpl.java:71)
	 at org.opendaylight.controller.md.sal.binding.impl.BindingDOMDataTreeChangeListenerAdapter.onDataTreeChanged(BindingDOMDataTreeChangeListenerAdapter.java:41)
	 at org.opendaylight.controller.cluster.datastore.DataTreeChangeListenerActor.dataChanged(DataTreeChangeListenerActor.java:67)
	 at org.opendaylight.controller.cluster.datastore.DataTreeChangeListenerActor.handleReceive(DataTreeChangeListenerActor.java:41)
	 at org.opendaylight.controller.cluster.common.actor.AbstractUntypedActor.onReceive(AbstractUntypedActor.java:38)
	 at akka.actor.UntypedActor$$anonfun$receive$1.applyOrElse(UntypedActor.scala:167)
	 at akka.actor.Actor.aroundReceive(Actor.scala:517)
	 at akka.actor.Actor.aroundReceive$(Actor.scala:515)
	 at akka.actor.UntypedActor.aroundReceive(UntypedActor.scala:97)
	 at akka.actor.ActorCell.receiveMessage(ActorCell.scala:590)
	 at akka.actor.ActorCell.invoke(ActorCell.scala:559)
	 at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:257)
	 at akka.dispatch.Mailbox.run(Mailbox.scala:224)
	 at akka.dispatch.Mailbox.exec(Mailbox.scala:234)
	 at akka.dispatch.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
	 at akka.dispatch.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
	 at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
	 at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
OpenDaylight JIRA

[MDSAL-362] Thread deadlock between mdsal and bgpcep when running Controller Clustering CSIT job
