When running perf/scale tests by creating a large number of neutron resources we see OOM on one of the ODLs in a clustered ODL setup. Capturing the stdout of the JVM before it exited due to OOM we see:
[root@overcloud-controller-2 heat-admin]# docker logs opendaylight_api
Exception in thread "CommitFutures-20" java.lang.IllegalStateException: Transaction chain has failed
at com.google.common.base.Preconditions.checkState(Preconditions.java:173)
at org.opendaylight.controller.cluster.databroker.DOMBrokerTransactionChain.checkNotFailed(DOMBrokerTransactionChain.java:73)
at org.opendaylight.controller.cluster.databroker.DOMBrokerTransactionChain.submit(DOMBrokerTransactionChain.java:84)
at org.opendaylight.controller.cluster.databroker.AbstractDOMBrokerWriteTransaction.submit(AbstractDOMBrokerWriteTransaction.java:145)
at org.opendaylight.controller.md.sal.dom.broker.impl.PingPongTransactionChain.processTransaction(PingPongTransactionChain.java:235)
at org.opendaylight.controller.md.sal.dom.broker.impl.PingPongTransactionChain.processNextTransaction(PingPongTransactionChain.java:277)
at org.opendaylight.controller.md.sal.dom.broker.impl.PingPongTransactionChain.transactionFailed(PingPongTransactionChain.java:294)
at org.opendaylight.controller.md.sal.dom.broker.impl.PingPongTransactionChain$2.onFailure(PingPongTransactionChain.java:243)
at com.google.common.util.concurrent.Futures$6.run(Futures.java:1310)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Exception in thread "CommitFutures-24" java.lang.IllegalStateException: Transaction chain has failed
at com.google.common.base.Preconditions.checkState(Preconditions.java:173)
at org.opendaylight.controller.cluster.databroker.DOMBrokerTransactionChain.checkNotFailed(DOMBrokerTransactionChain.java:73)
at org.opendaylight.controller.cluster.databroker.DOMBrokerTransactionChain.submit(DOMBrokerTransactionChain.java:84)
at org.opendaylight.controller.cluster.databroker.AbstractDOMBrokerWriteTransaction.submit(AbstractDOMBrokerWriteTransaction.java:145)
at org.opendaylight.controller.md.sal.dom.broker.impl.PingPongTransactionChain.processTransaction(PingPongTransactionChain.java:235)
at org.opendaylight.controller.md.sal.dom.broker.impl.PingPongTransactionChain.processNextTransaction(PingPongTransactionChain.java:277)
at org.opendaylight.controller.md.sal.dom.broker.impl.PingPongTransactionChain.transactionFailed(PingPongTransactionChain.java:294)
at org.opendaylight.controller.md.sal.dom.broker.impl.PingPongTransactionChain$2.onFailure(PingPongTransactionChain.java:243)
at com.google.common.util.concurrent.Futures$6.run(Futures.java:1310)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Exception in thread "CommitFutures-31" java.lang.IllegalStateException: Transaction chain has failed
at com.google.common.base.Preconditions.checkState(Preconditions.java:173)
at org.opendaylight.controller.cluster.databroker.DOMBrokerTransactionChain.checkNotFailed(DOMBrokerTransactionChain.java:73)
at org.opendaylight.controller.cluster.databroker.DOMBrokerTransactionChain.submit(DOMBrokerTransactionChain.java:84)
at org.opendaylight.controller.cluster.databroker.AbstractDOMBrokerWriteTransaction.submit(AbstractDOMBrokerWriteTransaction.java:145)
at org.opendaylight.controller.md.sal.dom.broker.impl.PingPongTransactionChain.processTransaction(PingPongTransactionChain.java:235)
at org.opendaylight.controller.md.sal.dom.broker.impl.PingPongTransactionChain.processNextTransaction(PingPongTransactionChain.java:277)
at org.opendaylight.controller.md.sal.dom.broker.impl.PingPongTransactionChain.transactionFailed(PingPongTransactionChain.java:294)
at org.opendaylight.controller.md.sal.dom.broker.impl.PingPongTransactionChain$2.onFailure(PingPongTransactionChain.java:243)
at com.google.common.util.concurrent.Futures$6.run(Futures.java:1310)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
opendaylight-user@root>java.lang.OutOfMemoryError: GC overhead limit exceeded
Dumping heap to java_pid1.hprof ...
Heap dump file created [3493848468 bytes in 18.776 secs]
Uncaught error from thread [opendaylight-cluster-data-akka.remote.default-remote-dispatcher-1589] shutting down JVM since 'akka.jvm-exit-on-fatal-error' is enabled for ActorSystem[opendaylig
ht-cluster-data]
java.lang.OutOfMemoryError: GC overhead limit exceeded
at java.util.Arrays.copyOf(Arrays.java:3332)
at java.lang.AbstractStringBuilder.ensureCapacityInternal(AbstractStringBuilder.java:124)
at java.lang.AbstractStringBuilder.append(AbstractStringBuilder.java:448)
at java.lang.StringBuilder.append(StringBuilder.java:136)
at scala.collection.mutable.StringBuilder.append(StringBuilder.scala:200)
at scala.Predef$any2stringadd$.$plus$extension(Predef.scala:281)
at akka.actor.RootActorPath.<init>(ActorPath.scala:274)
at akka.remote.RemoteActorRefProvider.resolveActorRefWithLocalAddress(RemoteActorRefProvider.scala:386)
at akka.remote.transport.AkkaPduProtobufCodec$.decodeMessage(AkkaPduCodec.scala:201)
Uncaught error from thread [opendaylight-cluster-data-shard-dispatcher-84] shutting down JVM since 'akka.jvm-exit-on-fatal-error' is enabled for ActorSystem[opendaylight-cluster-data]
at akka.remote.EndpointReader.akka$remote$EndpointReader$$tryDecodeMessageAndAck(Endpoint.scala:1058)
at akka.remote.EndpointReader$$anonfun$receive$2.applyOrElse(Endpoint.scala:978)
at akka.actor.Actor$class.aroundReceive(Actor.scala:502)
at akka.remote.EndpointActor.aroundReceive(Endpoint.scala:452)
at akka.actor.ActorCell.receiveMessage(ActorCell.scala:526)
at akka.actor.ActorCell.invoke(ActorCell.scala:495)
at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:257)
at akka.dispatch.Mailbox.run(Mailbox.scala:224)
at akka.dispatch.Mailbox.exec(Mailbox.scala:234)
at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
java.lang.OutOfMemoryError: GC overhead limit exceeded
Exception in thread "CommitFutures-9" java.lang.OutOfMemoryError: GC overhead limit exceeded
at java.util.concurrent.SynchronousQueue$TransferStack.snode(SynchronousQueue.java:318)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:361)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at org.opendaylight.yangtools.util.concurrent.CachedThreadPoolExecutor$ExecutorQueue.poll(CachedThreadPoolExecutor.java:164)
at org.opendaylight.yangtools.util.concurrent.CachedThreadPoolExecutor$ExecutorQueue.poll(CachedThreadPoolExecutor.java:125)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)