[BGPCEP-731] Deadlock occurred in BGP

Mon, 3 Aug 2020 10:01:59 +0000

Found one Java-level deadlock:
=============================
"opendaylight-cluster-data-akka.actor.default-dispatcher-3191":
waiting to lock monitor 0x00007f3390f99e78 (object 0x0000000618c00150, a org.opendaylight.protocol.bgp.rib.impl.RIBImpl),
which is held by "CommitFutures-1349"
"CommitFutures-1349":
waiting to lock monitor 0x00007f34ca575b48 (object 0x000000068342c220, a org.opendaylight.controller.md.sal.dom.broker.impl.PingPongTransactionChain),
which is held by "CommitFutures-1326"
"CommitFutures-1326":
waiting to lock monitor 0x00007f3390f99e78 (object 0x0000000618c00150, a org.opendaylight.protocol.bgp.rib.impl.RIBImpl),
which is held by "CommitFutures-1349"

Java stack information for the threads listed above:
===================================================
"opendaylight-cluster-data-akka.actor.default-dispatcher-3191":
at org.opendaylight.protocol.bgp.rib.impl.RIBImpl.onTransactionChainFailed(RIBImpl.java:264)

waiting to lock <0x0000000618c00150> (a org.opendaylight.protocol.bgp.rib.impl.RIBImpl)
at org.opendaylight.controller.md.sal.dom.broker.impl.PingPongTransactionChain$1.onTransactionChainFailed(PingPongTransactionChain.java:104)
at org.opendaylight.controller.cluster.databroker.DOMBrokerTransactionChain.transactionFailed(DOMBrokerTransactionChain.java:137)
at org.opendaylight.controller.cluster.databroker.DOMBrokerTransactionChain.access$100(DOMBrokerTransactionChain.java:30)
at org.opendaylight.controller.cluster.databroker.DOMBrokerTransactionChain$1.onFailure(DOMBrokerTransactionChain.java:98)
at com.google.common.util.concurrent.Futures$6.run(Futures.java:1764)
at java.util.concurrent.ThreadPoolExecutor$CallerRunsPolicy.rejectedExecution(ThreadPoolExecutor.java:2022)
at org.opendaylight.yangtools.util.concurrent.CountingRejectedExecutionHandler.rejectedExecution(CountingRejectedExecutionHandler.java:41)
at org.opendaylight.yangtools.util.concurrent.CachedThreadPoolExecutor$RejectedTaskHandler.rejectedExecution(CachedThreadPoolExecutor.java:218)
at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:823)
at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1369)
at org.opendaylight.controller.cluster.datastore.ConcurrentDOMDataBroker$AsyncNotifyingSettableFuture$DelegatingRunnable.run(ConcurrentDOMDataBroker.java:317)
at com.google.common.util.concurrent.MoreExecutors$DirectExecutor.execute(MoreExecutors.java:456)
at com.google.common.util.concurrent.AbstractFuture.executeListener(AbstractFuture.java:817)
at com.google.common.util.concurrent.AbstractFuture.complete(AbstractFuture.java:753)
at com.google.common.util.concurrent.AbstractFuture.setException(AbstractFuture.java:634)
at org.opendaylight.controller.cluster.datastore.ConcurrentDOMDataBroker$AsyncNotifyingSettableFuture.setException(ConcurrentDOMDataBroker.java:296)
at org.opendaylight.controller.cluster.datastore.ConcurrentDOMDataBroker$4.onSuccess(ConcurrentDOMDataBroker.java:231)
at org.opendaylight.controller.cluster.datastore.ConcurrentDOMDataBroker$4.onSuccess(ConcurrentDOMDataBroker.java:227)
at com.google.common.util.concurrent.Futures$6.run(Futures.java:1773)
at com.google.common.util.concurrent.MoreExecutors$DirectExecutor.execute(MoreExecutors.java:456)
at com.google.common.util.concurrent.AbstractFuture.executeListener(AbstractFuture.java:817)
at com.google.common.util.concurrent.AbstractFuture.addListener(AbstractFuture.java:595)
at com.google.common.util.concurrent.AbstractFuture$TrustedFuture.addListener(AbstractFuture.java:96)
at com.google.common.util.concurrent.Futures.addCallback(Futures.java:1776)
at org.opendaylight.controller.cluster.datastore.ConcurrentDOMDataBroker.handleException(ConcurrentDOMDataBroker.java:227)
at org.opendaylight.controller.cluster.datastore.ConcurrentDOMDataBroker.access$000(ConcurrentDOMDataBroker.java:44)
at org.opendaylight.controller.cluster.datastore.ConcurrentDOMDataBroker$1.onFailure(ConcurrentDOMDataBroker.java:123)
at com.google.common.util.concurrent.Futures$6.run(Futures.java:1764)
at com.google.common.util.concurrent.MoreExecutors$DirectExecutor.execute(MoreExecutors.java:456)
at com.google.common.util.concurrent.AbstractFuture.executeListener(AbstractFuture.java:817)
at com.google.common.util.concurrent.AbstractFuture.complete(AbstractFuture.java:753)
at com.google.common.util.concurrent.AbstractFuture.setException(AbstractFuture.java:634)
at com.google.common.util.concurrent.SettableFuture.setException(SettableFuture.java:53)
at org.opendaylight.controller.cluster.datastore.SingleCommitCohortProxy$1.onComplete(SingleCommitCohortProxy.java:60)
at akka.dispatch.OnComplete.internal(Future.scala:258)
at akka.dispatch.OnComplete.internal(Future.scala:256)
at akka.dispatch.japi$CallbackBridge.apply(Future.scala:186)
at akka.dispatch.japi$CallbackBridge.apply(Future.scala:183)
at scala.concurrent.impl.CallbackRunnable.run(Promise.scala:32)
at akka.dispatch.BatchingExecutor$AbstractBatch.processBatch(BatchingExecutor.scala:55)
at akka.dispatch.BatchingExecutor$BlockableBatch$$anonfun$run$1.apply$mcV$sp(BatchingExecutor.scala:91)
at akka.dispatch.BatchingExecutor$BlockableBatch$$anonfun$run$1.apply(BatchingExecutor.scala:91)
at akka.dispatch.BatchingExecutor$BlockableBatch$$anonfun$run$1.apply(BatchingExecutor.scala:91)
at scala.concurrent.BlockContext$.withBlockContext(BlockContext.scala:72)
at akka.dispatch.BatchingExecutor$BlockableBatch.run(BatchingExecutor.scala:90)
at akka.dispatch.TaskInvocation.run(AbstractDispatcher.scala:39)
at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:409)
at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
"CommitFutures-1349":
at org.opendaylight.controller.md.sal.dom.broker.impl.PingPongTransactionChain.close(PingPongTransactionChain.java:288)
waiting to lock <0x000000068342c220> (a org.opendaylight.controller.md.sal.dom.broker.impl.PingPongTransactionChain)
at org.opendaylight.protocol.bgp.rib.impl.LocRibWriter.close(LocRibWriter.java:138)
at org.opendaylight.protocol.bgp.rib.impl.LocRibWriter.restart(LocRibWriter.java:129)
locked <0x0000000618bfff88> (a org.opendaylight.protocol.bgp.rib.impl.LocRibWriter)
at org.opendaylight.protocol.bgp.rib.impl.RIBImpl.onTransactionChainFailed(RIBImpl.java:268)
locked <0x0000000618c00150> (a org.opendaylight.protocol.bgp.rib.impl.RIBImpl)
at org.opendaylight.controller.md.sal.dom.broker.impl.PingPongTransactionChain$1.onTransactionChainFailed(PingPongTransactionChain.java:104)
at org.opendaylight.controller.cluster.databroker.DOMBrokerTransactionChain.transactionFailed(DOMBrokerTransactionChain.java:137)
at org.opendaylight.controller.cluster.databroker.DOMBrokerTransactionChain.access$100(DOMBrokerTransactionChain.java:30)
at org.opendaylight.controller.cluster.databroker.DOMBrokerTransactionChain$1.onFailure(DOMBrokerTransactionChain.java:98)
at com.google.common.util.concurrent.Futures$6.run(Futures.java:1764)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
"CommitFutures-1326":
at org.opendaylight.protocol.bgp.rib.impl.RIBImpl.onTransactionChainFailed(RIBImpl.java:264)
waiting to lock <0x0000000618c00150> (a org.opendaylight.protocol.bgp.rib.impl.RIBImpl)
at org.opendaylight.controller.md.sal.dom.broker.impl.PingPongTransactionChain$1.onTransactionChainFailed(PingPongTransactionChain.java:104)
at org.opendaylight.controller.cluster.databroker.DOMBrokerTransactionChain.transactionFailed(DOMBrokerTransactionChain.java:137)
at org.opendaylight.controller.cluster.databroker.DOMBrokerTransactionChain.access$100(DOMBrokerTransactionChain.java:30)
at org.opendaylight.controller.cluster.databroker.DOMBrokerTransactionChain$1.onFailure(DOMBrokerTransactionChain.java:98)
at com.google.common.util.concurrent.Futures$6.run(Futures.java:1764)
at org.opendaylight.controller.cluster.datastore.ConcurrentDOMDataBroker$AsyncNotifyingSettableFuture$DelegatingRunnable.run(ConcurrentDOMDataBroker.java:322)
at com.google.common.util.concurrent.MoreExecutors$DirectExecutor.execute(MoreExecutors.java:456)
at com.google.common.util.concurrent.AbstractFuture.executeListener(AbstractFuture.java:817)
at com.google.common.util.concurrent.AbstractFuture.addListener(AbstractFuture.java:595)
at org.opendaylight.controller.cluster.datastore.ConcurrentDOMDataBroker$AsyncNotifyingSettableFuture.addListener(ConcurrentDOMDataBroker.java:280)
at com.google.common.util.concurrent.ForwardingListenableFuture.addListener(ForwardingListenableFuture.java:47)
at com.google.common.util.concurrent.Futures.addCallback(Futures.java:1776)
at com.google.common.util.concurrent.Futures.addCallback(Futures.java:1713)
at org.opendaylight.controller.cluster.databroker.DOMBrokerTransactionChain.submit(DOMBrokerTransactionChain.java:90)
at org.opendaylight.controller.cluster.databroker.AbstractDOMBrokerWriteTransaction.submit(AbstractDOMBrokerWriteTransaction.java:137)
at org.opendaylight.controller.md.sal.dom.broker.impl.PingPongTransactionChain.processTransaction(PingPongTransactionChain.java:197)
at org.opendaylight.controller.md.sal.dom.broker.impl.PingPongTransactionChain.processNextTransaction(PingPongTransactionChain.java:237)
locked <0x000000068342c220> (a org.opendaylight.controller.md.sal.dom.broker.impl.PingPongTransactionChain)
at org.opendaylight.controller.md.sal.dom.broker.impl.PingPongTransactionChain.transactionFailed(PingPongTransactionChain.java:256)
at org.opendaylight.controller.md.sal.dom.broker.impl.PingPongTransactionChain.access$400(PingPongTransactionChain.java:54)
at org.opendaylight.controller.md.sal.dom.broker.impl.PingPongTransactionChain$2.onFailure(PingPongTransactionChain.java:205)
at com.google.common.util.concurrent.Futures$6.run(Futures.java:1764)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)

OpenDaylight JIRA

[BGPCEP-731] Deadlock occurred in BGP