Akka artery fails with java.lang.OutOfMemoryError: Direct buffer memory

This is tracked here:

https://jenkins.opendaylight.org/releng/view/CSIT-3node/job/controller-csit-3node-clustering-only-carbon/

Looking at the console of a failing test:

https://jenkins.opendaylight.org/releng/view/CSIT-3node/job/controller-csit-3node-clustering-only-carbon/478/console

I can see:

15:30:51 2017-01-10 15:21:02,052 | ERROR | ult-dispatcher-3 | ActorSystemImpl | 203 - com.typesafe.akka.slf4j - 2.4.16 | Uncaught error from thread [opendaylight-cluster-data-akka.remote.default-remote-dispatcher-10] shutting down JVM since 'akka.jvm-exit-on-fatal-error' is enabled
15:30:51 2017-01-10 15:21:02,070 | ERROR | ult-dispatcher-3 | kka://opendaylight-cluster-data) | 203 - com.typesafe.akka.slf4j - 2.4.16 | Aeron error, Direct buffer memory
15:30:51 java.lang.OutOfMemoryError: Direct buffer memory
15:30:51 at java.nio.Bits.reserveMemory(Bits.java:693)[:1.8.0_111]
15:30:51 at java.nio.DirectByteBuffer.<init>(DirectByteBuffer.java:123)[:1.8.0_111]
15:30:51 at java.nio.ByteBuffer.allocateDirect(ByteBuffer.java:311)[:1.8.0_111]
15:30:51 at akka.remote.artery.EnvelopeBufferPool.acquire(BufferPool.scala:40)[215:com.typesafe.akka.remote:2.4.16]
15:30:51 at akka.remote.artery.AeronSource$Fragments$$anon$2.onFragment(AeronSource.scala:59)[215:com.typesafe.akka.remote:2.4.16]
15:30:51 at io.aeron.FragmentAssembler.onFragment(FragmentAssembler.java:82)[218:io.aeron.client:1.0.4]
15:30:51 at io.aeron.logbuffer.TermReader.read(TermReader.java:74)[218:io.aeron.client:1.0.4]
15:30:51 at io.aeron.Image.poll(Image.java:211)[218:io.aeron.client:1.0.4]
15:30:51 at io.aeron.Subscription.poll(Subscription.java:132)[218:io.aeron.client:1.0.4]
15:30:51 at akka.remote.artery.AeronSource$$anonfun$akka$remote$artery$AeronSource$$pollTask$1.apply$mcZ$sp(AeronSource.scala:38)[215:com.typesafe.akka.remote:2.4.16]
15:30:51 at akka.remote.artery.TaskRunner.executeTasks(TaskRunner.scala:171)[215:com.typesafe.akka.remote:2.4.16]
15:30:51 at akka.remote.artery.TaskRunner.run(TaskRunner.scala:150)[215:com.typesafe.akka.remote:2.4.16]
15:30:51 at java.lang.Thread.run(Thread.java:745)[:1.8.0_111]
15:30:51 2017-01-10 15:21:04,161 | ERROR | ult-dispatcher-3 | kka://opendaylight-cluster-data) | 203 - com.typesafe.akka.slf4j - 2.4.16 | Aeron error, Direct buffer memory
15:30:51 java.lang.OutOfMemoryError: Direct buffer memory
15:30:51 at java.nio.Bits.reserveMemory(Bits.java:693)[:1.8.0_111]
15:30:51 at java.nio.DirectByteBuffer.<init>(DirectByteBuffer.java:123)[:1.8.0_111]
15:30:51 at java.nio.ByteBuffer.allocateDirect(ByteBuffer.java:311)[:1.8.0_111]
15:30:51 at akka.remote.artery.EnvelopeBufferPool.acquire(BufferPool.scala:40)[215:com.typesafe.akka.remote:2.4.16]
15:30:51 at akka.remote.artery.AeronSource$Fragments$$anon$2.onFragment(AeronSource.scala:59)[215:com.typesafe.akka.remote:2.4.16]
15:30:51 at io.aeron.FragmentAssembler.onFragment(FragmentAssembler.java:82)[218:io.aeron.client:1.0.4]
15:30:51 at io.aeron.logbuffer.TermReader.read(TermReader.java:74)[218:io.aeron.client:1.0.4]
15:30:51 at io.aeron.Image.poll(Image.java:211)[218:io.aeron.client:1.0.4]
15:30:51 at io.aeron.Subscription.poll(Subscription.java:127)[218:io.aeron.client:1.0.4]
15:30:51 at akka.remote.artery.AeronSource$$anonfun$akka$remote$artery$AeronSource$$pollTask$1.apply$mcZ$sp(AeronSource.scala:38)[215:com.typesafe.akka.remote:2.4.16]
15:30:51 at akka.remote.artery.TaskRunner.executeTasks(TaskRunner.scala:171)[215:com.typesafe.akka.remote:2.4.16]
15:30:51 at akka.remote.artery.TaskRunner.run(TaskRunner.scala:150)[215:com.typesafe.akka.remote:2.4.16]
15:30:51 at java.lang.Thread.run(Thread.java:745)[:1.8.0_111]
15:30:51 2017-01-10 15:21:04,909 | ERROR | ult-dispatcher-3 | kka://opendaylight-cluster-data) | 203 - com.typesafe.akka.slf4j - 2.4.16 | Aeron error, Direct buffer memory
15:30:51 java.lang.OutOfMemoryError: Direct buffer memory
15:30:51 at java.nio.Bits.reserveMemory(Bits.java:693)[:1.8.0_111]
15:30:51 at java.nio.DirectByteBuffer.<init>(DirectByteBuffer.java:123)[:1.8.0_111]
15:30:51 at java.nio.ByteBuffer.allocateDirect(ByteBuffer.java:311)[:1.8.0_111]
15:30:51 at akka.remote.artery.EnvelopeBufferPool.acquire(BufferPool.scala:40)[215:com.typesafe.akka.remote:2.4.16]
15:30:51 at akka.remote.artery.AeronSource$Fragments$$anon$2.onFragment(AeronSource.scala:59)[215:com.typesafe.akka.remote:2.4.16]
15:30:51 at io.aeron.FragmentAssembler.onFragment(FragmentAssembler.java:82)[218:io.aeron.client:1.0.4]
15:30:51 at io.aeron.logbuffer.TermReader.read(TermReader.java:74)[218:io.aeron.client:1.0.4]
15:30:51 at io.aeron.Image.poll(Image.java:211)[218:io.aeron.client:1.0.4]
15:30:51 at io.aeron.Subscription.poll(Subscription.java:127)[218:io.aeron.client:1.0.4]
15:30:51 at akka.remote.artery.AeronSource$$anonfun$akka$remote$artery$AeronSource$$pollTask$1.apply$mcZ$sp(AeronSource.scala:38)[215:com.typesafe.akka.remote:2.4.16]
15:30:51 at akka.remote.artery.TaskRunner.executeTasks(TaskRunner.scala:171)[215:com.typesafe.akka.remote:2.4.16]
15:30:51 at akka.remote.artery.TaskRunner.run(TaskRunner.scala:150)[215:com.typesafe.akka.remote:2.4.16]
15:30:51 at java.lang.Thread.run(Thread.java:745)[:1.8.0_111]
15:30:51 2017-01-10 15:21:05,638 | ERROR | ult-dispatcher-3 | kka://opendaylight-cluster-data) | 203 - com.typesafe.akka.slf4j - 2.4.16 | Aeron error, Direct buffer memory
15:30:51 java.lang.OutOfMemoryError: Direct buffer memory
15:30:51 at java.nio.Bits.reserveMemory(Bits.java:693)[:1.8.0_111]
15:30:51 at java.nio.DirectByteBuffer.<init>(DirectByteBuffer.java:123)[:1.8.0_111]
15:30:51 at java.nio.ByteBuffer.allocateDirect(ByteBuffer.java:311)[:1.8.0_111]
15:30:51 at akka.remote.artery.EnvelopeBufferPool.acquire(BufferPool.scala:40)[215:com.typesafe.akka.remote:2.4.16]
15:30:51 at akka.remote.artery.AeronSource$Fragments$$anon$2.onFragment(AeronSource.scala:59)[215:com.typesafe.akka.remote:2.4.16]
15:30:51 at io.aeron.FragmentAssembler.onFragment(FragmentAssembler.java:82)[218:io.aeron.client:1.0.4]
15:30:51 at io.aeron.logbuffer.TermReader.read(TermReader.java:74)[218:io.aeron.client:1.0.4]
15:30:51 at io.aeron.Image.poll(Image.java:211)[218:io.aeron.client:1.0.4]
15:30:51 at io.aeron.Subscription.poll(Subscription.java:127)[218:io.aeron.client:1.0.4]
15:30:51 at akka.remote.artery.AeronSource$$anonfun$akka$remote$artery$AeronSource$$pollTask$1.apply$mcZ$sp(AeronSource.scala:38)[215:com.typesafe.akka.remote:2.4.16]
15:30:51 at akka.remote.artery.TaskRunner.executeTasks(TaskRunner.scala:171)[215:com.typesafe.akka.remote:2.4.16]
15:30:51 at akka.remote.artery.TaskRunner.run(TaskRunner.scala:150)[215:com.typesafe.akka.remote:2.4.16]
15:30:51 at java.lang.Thread.run(Thread.java:745)[:1.8.0_111]
15:30:51 2017-01-10 15:21:06,401 | ERROR | lt-dispatcher-14 | kka://opendaylight-cluster-data) | 203 - com.typesafe.akka.slf4j - 2.4.16 | Aeron error, Direct buffer memory
15:30:51 java.lang.OutOfMemoryError: Direct buffer memory
15:30:51 at java.nio.Bits.reserveMemory(Bits.java:693)[:1.8.0_111]
15:30:51 at java.nio.DirectByteBuffer.<init>(DirectByteBuffer.java:123)[:1.8.0_111]
15:30:51 at java.nio.ByteBuffer.allocateDirect(ByteBuffer.java:311)[:1.8.0_111]
15:30:51 at akka.remote.artery.EnvelopeBufferPool.acquire(BufferPool.scala:40)[215:com.typesafe.akka.remote:2.4.16]
15:30:51 at akka.remote.artery.AeronSource$Fragments$$anon$2.onFragment(AeronSource.scala:59)[215:com.typesafe.akka.remote:2.4.16]
15:30:51 at io.aeron.FragmentAssembler.onFragment(FragmentAssembler.java:82)[218:io.aeron.client:1.0.4]
15:30:51 at io.aeron.logbuffer.TermReader.read(TermReader.java:74)[218:io.aeron.client:1.0.4]
15:30:51 at io.aeron.Image.poll(Image.java:211)[218:io.aeron.client:1.0.4]
15:30:51 at io.aeron.Subscription.poll(Subscription.java:127)[218:io.aeron.client:1.0.4]
15:30:51 at akka.remote.artery.AeronSource$$anonfun$akka$remote$artery$AeronSource$$pollTask$1.apply$mcZ$sp(AeronSource.scala:38)[215:com.typesafe.akka.remote:2.4.16]
15:30:51 at akka.remote.artery.TaskRunner.executeTasks(TaskRunner.scala:171)[215:com.typesafe.akka.remote:2.4.16]
15:30:51 at akka.remote.artery.TaskRunner.run(TaskRunner.scala:150)[215:com.typesafe.akka.remote:2.4.16]
15:30:51 at java.lang.Thread.run(Thread.java:745)[:1.8.0_111]
15:30:51 2017-01-10 15:21:07,461 | WARN | saction-30-31'}} | DeadlockMonitor | 151 - org.opendaylight.controller.config-manager - 0.6.0.SNAPSHOT | ModuleIdentifier

{factoryName='binding-broker-impl', instanceName='binding-broker-impl'}

did not finish after 9920 ms

We can also see the used config files in the console:

15:20:16 Dump akka.conf
15:20:16
15:20:16 odl-cluster-data {
15:20:16 akka {
15:20:16 remote {
15:20:16 artery

{ 15:20:16 enabled = on 15:20:16 canonical.hostname = "10.29.12.251" 15:20:16 canonical.port = 2550 15:20:16 }

15:20:16 }
15:20:16
15:20:16 cluster

{ 15:20:16 seed-nodes = ["akka://opendaylight-cluster-data@10.29.12.251:2550", 15:20:16 "akka://opendaylight-cluster-data@10.29.12.96:2550", 15:20:16 "akka://opendaylight-cluster-data@10.29.12.146:2550"] 15:20:16 15:20:16 roles = ["member-1"] 15:20:16 15:20:16 }

15:20:16
15:20:16 persistence {
15:20:16 # By default the snapshots/journal directories live in KARAF_HOME. You can choose to put it somewhere else by
15:20:16 # modifying the following two properties. The directory location specified may be a relative or absolute path.
15:20:16 # The relative path is always relative to KARAF_HOME.
15:20:16
15:20:16 # snapshot-store.local.dir = "target/snapshots"
15:20:16 # journal.leveldb.dir = "target/journal"
15:20:16
15:20:16 journal {
15:20:16 leveldb

{ 15:20:16 # Set native = off to use a Java-only implementation of leveldb. 15:20:16 # Note that the Java-only version is not currently considered by Akka to be production quality. 15:20:16 15:20:16 # native = off 15:20:16 }

15:20:16 }
15:20:16 }
15:20:16 }
15:20:16 }
15:20:16 Dump modules.conf
15:20:16 modules = [
15:20:16
15:20:16

{ 15:20:16 name = "inventory" 15:20:16 namespace = "urn:opendaylight:inventory" 15:20:16 shard-strategy = "module" 15:20:16 }

,
15:20:16

{ 15:20:16 name = "topology" 15:20:16 namespace = "urn:TBD:params:xml:ns:yang:network-topology" 15:20:16 shard-strategy = "module" 15:20:16 }

,
15:20:16

{ 15:20:16 name = "toaster" 15:20:16 namespace = "http://netconfcentral.org/ns/toaster" 15:20:16 shard-strategy = "module" 15:20:16 }

,
15:20:16

{ 15:20:16 name = "car" 15:20:16 namespace = "urn:opendaylight:params:xml:ns:yang:controller:config:sal-clustering-it:car" 15:20:16 shard-strategy = "module" 15:20:16 }

,
15:20:16

{ 15:20:16 name = "people" 15:20:16 namespace = "urn:opendaylight:params:xml:ns:yang:controller:config:sal-clustering-it:people" 15:20:16 shard-strategy = "module" 15:20:16 }

,
15:20:16

{ 15:20:16 name = "car-people" 15:20:16 namespace = "urn:opendaylight:params:xml:ns:yang:controller:config:sal-clustering-it:car-people" 15:20:16 shard-strategy = "module" 15:20:16 }

15:20:16 ]
15:20:16 Dump module-shards.conf
15:20:16 module-shards = [
15:20:16 {
15:20:16 name = "default"
15:20:16 shards = [
15:20:16

{ 15:20:16 name = "default" 15:20:16 replicas = ["member-1", 15:20:16 "member-2", 15:20:16 "member-3"] 15:20:16 }

15:20:16 ]
15:20:16 },
15:20:16 {
15:20:16 name = "inventory"
15:20:16 shards = [
15:20:16

{ 15:20:16 name="inventory" 15:20:16 replicas = ["member-1", 15:20:16 "member-2", 15:20:16 "member-3"] 15:20:16 }

15:20:16 ]
15:20:16 },
15:20:16 {
15:20:16 name = "topology"
15:20:16 shards = [
15:20:16

{ 15:20:16 name="topology" 15:20:16 replicas = ["member-1", 15:20:16 "member-2", 15:20:16 "member-3"] 15:20:16 }

15:20:16 ]
15:20:16 },
15:20:16 {
15:20:16 name = "toaster"
15:20:16 shards = [
15:20:16

{ 15:20:16 name="toaster" 15:20:16 replicas = ["member-1", 15:20:16 "member-2", 15:20:16 "member-3"] 15:20:16 }

15:20:16 ]
15:20:16 },
15:20:16 {
15:20:16 name = "car"
15:20:16 shards = [
15:20:16

{ 15:20:16 name="car" 15:20:16 replicas = ["member-1", 15:20:16 "member-2", 15:20:16 "member-3"] 15:20:16 }

15:20:16 ]
15:20:16 },
15:20:16 {
15:20:16 name = "people"
15:20:16 shards = [
15:20:16

{ 15:20:16 name="people" 15:20:16 replicas = ["member-1", 15:20:16 "member-2", 15:20:16 "member-3"] 15:20:16 }

15:20:16 ]
15:20:16 },
15:20:16 {
15:20:16 name = "car-people"
15:20:16 shards = [
15:20:16

{ 15:20:16 name="car-people" 15:20:16 replicas = ["member-1", 15:20:16 "member-2", 15:20:16 "member-3"] 15:20:16 }

15:20:16 ]
15:20:16 }
15:20:16 ]