global_rpc kill - 501 ODL_SYSTEM_IP=10.30.170.22 ODL_SYSTEM_1_IP=10.30.170.22 ODL_SYSTEM_2_IP=10.30.170.37 ODL_SYSTEM_3_IP=10.30.170.53 odl3 ==== 2 different threads, each thread trying to obtain 2 locks. 1st thread, grabs: lock 1 - held lock 2 - trying to obtain 2nd thread lock 2 - held lock 1 trying to obtain 2018-07-03T18:42:02,025 | INFO | opendaylight-cluster-data-notification-dispatcher-61 | PCEPTopologyDeployerImpl | 239 - org.opendaylight.bgpcep.pcep-tunnel-provider - 0.9.3.SNAPSHOT | Updating Topology Uri [_value=pcep-topology] 2018-07-03T18:42:02,025 | INFO | opendaylight-cluster-data-notification-dispatcher-61 | PCEPTopologyDeployerImpl | 239 - org.opendaylight.bgpcep.pcep-tunnel-provider - 0.9.3.SNAPSHOT | Removing Topology Uri [_value=pcep-topology] 2018-07-03T18:42:02,025 | INFO | opendaylight-cluster-data-notification-dispatcher-61 | PCEPTopologyProviderBean | 235 - org.opendaylight.bgpcep.pcep-topology-provider - 0.9.3.SNAPSHOT | Close PCEP Topology Provider Singleton Service pcep-topology-service-group 018-07-03T18:42:27,058 | INFO | pipe-log:log "ROBOT MESSAGE: Starting suite /w/workspace/controller-csit-3node-clustering-vpickard-all-oxygen/test/csit/suites/controller/singleton_service/global_rpc_kill.robot" | core | 131 - org.apache.karaf.log.core - 4.1.5 | ROBOT MESSAGE: Starting suite /w/workspace/controller-csit-3node-clustering-vpickard-all-oxygen/test/csit/suites/controller/singleton_service/global_rpc_kill.robot 2018-07-03T18:42:27,390 | ERROR | infrautils.metrics.ThreadsWatcher-0 | ThreadsWatcher | 357 - org.opendaylight.infrautils.metrics-impl - 1.3.3.SNAPSHOT | Oh nose - there are 2 deadlocked threads!! :-( 2018-07-03T18:42:02,391 | ERROR | infrautils.metrics.ThreadsWatcher-0 | ThreadsWatcher | 357 - org.opendaylight.infrautils.metrics-impl - 1.3.3.SNAPSHOT | Deadlocked thread stack trace: opendaylight-cluster-data-akka.actor.default-dispatcher-18 locked on org.opendaylight.bgpcep.pcep.topology.provider.config.PCEPTopologyProviderBean$PCEPTopologyProviderBeanCSS@3acd01d1 (owned by opendaylight-cluster-data-notification-dispatcher-61): *** synchronized - attempts to grab internal lock on PCEPTopologyProviderBean held by thread below ** at org.opendaylight.bgpcep.pcep.topology.provider.config.PCEPTopologyProviderBean$PCEPTopologyProviderBeanCSS.instantiateServiceInstance(PCEPTopologyProviderBean.java:157) at org.opendaylight.mdsal.singleton.dom.impl.ClusterSingletonServiceGroupImpl.lambda$startServices$0(ClusterSingletonServiceGroupImpl.java:648) at org.opendaylight.mdsal.singleton.dom.impl.ClusterSingletonServiceGroupImpl$$Lambda$1173/320400615.accept(Unknown Source) at java.util.ArrayList.forEach(ArrayList.java:1257) at org.opendaylight.mdsal.singleton.dom.impl.ClusterSingletonServiceGroupImpl.startServices(ClusterSingletonServiceGroupImpl.java:645) at org.opendaylight.mdsal.singleton.dom.impl.ClusterSingletonServiceGroupImpl.cleanupCandidateOwnershipChanged(ClusterSingletonServiceGroupImpl.java:506) at org.opendaylight.mdsal.singleton.dom.impl.ClusterSingletonServiceGroupImpl.lockedOwnershipChanged(ClusterSingletonServiceGroupImpl.java:453) ** has lock held here - Reentrant Lock *** at org.opendaylight.mdsal.singleton.dom.impl.ClusterSingletonServiceGroupImpl.ownershipChanged(ClusterSingletonServiceGroupImpl.java:433) at org.opendaylight.mdsal.singleton.dom.impl.AbstractClusterSingletonServiceProviderImpl.ownershipChanged(AbstractClusterSingletonServiceProviderImpl.java:234) at org.opendaylight.mdsal.singleton.dom.impl.DOMClusterSingletonServiceProviderImpl.ownershipChanged(DOMClusterSingletonServiceProviderImpl.java:23) at org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnershipListenerActor.onEntityOwnershipChanged(EntityOwnershipListenerActor.java:44) at org.opendaylight.controller.cluster.datastore.entityownership.EntityOwnershipListenerActor.handleReceive(EntityOwnershipListenerActor.java:33) at org.opendaylight.controller.cluster.common.actor.AbstractUntypedActor.onReceive(AbstractUntypedActor.java:38) at akka.actor.UntypedActor$$anonfun$receive$1.applyOrElse(UntypedActor.scala:167) at akka.actor.Actor.aroundReceive(Actor.scala:517) at akka.actor.Actor.aroundReceive$(Actor.scala:515) at akka.actor.UntypedActor.aroundReceive(UntypedActor.scala:97) at akka.actor.ActorCell.receiveMessage(ActorCell.scala:590) at akka.actor.ActorCell.invoke(ActorCell.scala:559) at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:257) at akka.dispatch.Mailbox.run(Mailbox.scala:224) at akka.dispatch.Mailbox.exec(Mailbox.scala:234) at akka.dispatch.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260) at akka.dispatch.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339) at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) 2018-07-03T18:42:02,392 | ERROR | infrautils.metrics.ThreadsWatcher-0 | ThreadsWatcher | 357 - org.opendaylight.infrautils.metrics-impl - 1.3.3.SNAPSHOT | Deadlocked thread stack trace: opendaylight-cluster-data-notification-dispatcher-61 locked on java.util.concurrent.locks.ReentrantLock$FairSync@340ae920 (owned by opendaylight-cluster-data-akka.actor.default-dispatcher-18): at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:870) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:1199) at java.util.concurrent.locks.ReentrantLock$FairSync.lock(ReentrantLock.java:224) at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock.java:285) ** attempts to grab same lock already held in thread above ** at org.opendaylight.mdsal.singleton.dom.impl.ClusterSingletonServiceGroupImpl.unregisterService(ClusterSingletonServiceGroupImpl.java:393) at org.opendaylight.mdsal.singleton.dom.impl.AbstractClusterSingletonServiceProviderImpl.removeRegistration(AbstractClusterSingletonServiceProviderImpl.java:151) at org.opendaylight.mdsal.singleton.dom.impl.AbstractClusterSingletonServiceProviderImpl$1.removeRegistration(AbstractClusterSingletonServiceProviderImpl.java:123) at org.opendaylight.yangtools.concepts.AbstractRegistration.close(AbstractRegistration.java:41) ** synchronized - internal lock on PCEPTopologyProviderBean ** at org.opendaylight.bgpcep.pcep.topology.provider.config.PCEPTopologyProviderBean$PCEPTopologyProviderBeanCSS.close(PCEPTopologyProviderBean.java:186) at org.opendaylight.bgpcep.pcep.topology.provider.config.PCEPTopologyProviderBean.close(PCEPTopologyProviderBean.java:88) at org.opendaylight.bgpcep.pcep.topology.provider.config.PCEPTopologyDeployerImpl.closeTopology(PCEPTopologyDeployerImpl.java:151) at org.opendaylight.bgpcep.pcep.topology.provider.config.PCEPTopologyDeployerImpl.updateTopologyProvider(PCEPTopologyDeployerImpl.java:92) at org.opendaylight.bgpcep.pcep.topology.provider.config.PCEPTopologyDeployerImpl.lambda$onDataTreeChanged$0(PCEPTopologyDeployerImpl.java:75) at org.opendaylight.bgpcep.pcep.topology.provider.config.PCEPTopologyDeployerImpl$$Lambda$1179/1398407903.accept(Unknown Source) at java.util.Iterator.forEachRemaining(Iterator.java:116) at org.opendaylight.bgpcep.pcep.topology.provider.config.PCEPTopologyDeployerImpl.onDataTreeChanged(PCEPTopologyDeployerImpl.java:71) at org.opendaylight.controller.md.sal.binding.impl.BindingDOMDataTreeChangeListenerAdapter.onDataTreeChanged(BindingDOMDataTreeChangeListenerAdapter.java:41) at org.opendaylight.controller.cluster.datastore.DataTreeChangeListenerActor.dataChanged(DataTreeChangeListenerActor.java:67) at org.opendaylight.controller.cluster.datastore.DataTreeChangeListenerActor.handleReceive(DataTreeChangeListenerActor.java:41) at org.opendaylight.controller.cluster.common.actor.AbstractUntypedActor.onReceive(AbstractUntypedActor.java:38) at akka.actor.UntypedActor$$anonfun$receive$1.applyOrElse(UntypedActor.scala:167) at akka.actor.Actor.aroundReceive(Actor.scala:517) at akka.actor.Actor.aroundReceive$(Actor.scala:515) at akka.actor.UntypedActor.aroundReceive(UntypedActor.scala:97) at akka.actor.ActorCell.receiveMessage(ActorCell.scala:590) at akka.actor.ActorCell.invoke(ActorCell.scala:559) at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:257) at akka.dispatch.Mailbox.run(Mailbox.scala:224) at akka.dispatch.Mailbox.exec(Mailbox.scala:234) at akka.dispatch.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260) at akka.dispatch.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339) at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)