Testested scenario: leader isolation , tell-based=true

-write-transactions rpc called on each node of the cluster
-leader of the default config shard isolated (iptables)
-remaining nodes elected new leader
-rejoin of isolated node happened approximately 31-33s after isolation
-after another 40s jvm went down

https://jenkins.opendaylight.org/sandbox/job/controller-csit-3node-clustering-only-carbon-4th/8/

https://logs.opendaylight.org/sandbox/jenkins091/controller-csit-3node-clustering-only-carbon-4th/8/archives/odl3_karaf_console.log.gz

Uncaught error from thread [opendaylight-cluster-data-akka.actor.default-dispatcher-5] shutting down JVM since 'akka.jvm-exit-on-fatal-error' is enabled for ActorSystem[opendaylight-cluster-data]
java.lang.StackOverflowError
at com.romix.scala.collection.concurrent.TrieMap$Equiv.equiv(TrieMap.java:994)
at com.romix.scala.collection.concurrent.TrieMap$INode.equal(TrieMap.java:173)
at com.romix.scala.collection.concurrent.TrieMap$INode.rec_remove(TrieMap.java:509)
at com.romix.scala.collection.concurrent.TrieMap$INode.rec_remove(TrieMap.java:499)
at com.romix.scala.collection.concurrent.TrieMap$INode.rec_remove(TrieMap.java:502)
at com.romix.scala.collection.concurrent.TrieMap$INode.rec_remove(TrieMap.java:499)
at com.romix.scala.collection.concurrent.TrieMap$INode.rec_remove(TrieMap.java:502)
at com.romix.scala.collection.concurrent.TrieMap$INode.rec_remove(TrieMap.java:499)
at com.romix.scala.collection.concurrent.TrieMap$INode.rec_remove(TrieMap.java:502)
at com.romix.scala.collection.concurrent.TrieMap.removehc(TrieMap.java:1213)
at com.romix.scala.collection.concurrent.TrieMap.remove(TrieMap.java:1396)
at org.opendaylight.yangtools.util.ReadWriteTrieMap.remove(ReadWriteTrieMap.java:88)
at org.opendaylight.yangtools.yang.data.impl.schema.builder.impl.ImmutableMapNodeBuilder.withoutChild(ImmutableMapNodeBuilder.java:74)
at org.opendaylight.yangtools.yang.data.impl.schema.builder.impl.ImmutableMapNodeBuilder.removeChild(ImmutableMapNodeBuilder.java:109)
at org.opendaylight.yangtools.yang.data.impl.schema.tree.AbstractNodeContainerModificationStrategy.mutateChildren(AbstractNodeContainerModificationStrategy.java:146)
at org.opendaylight.yangtools.yang.data.impl.schema.tree.AbstractNodeContainerModificationStrategy.applyTouch(AbstractNodeContainerModificationStrategy.java:252)
at org.opendaylight.yangtools.yang.data.impl.schema.tree.SchemaAwareApplyOperation.apply(SchemaAwareApplyOperation.java:200)
at org.opendaylight.yangtools.yang.data.impl.schema.tree.AbstractNodeContainerModificationStrategy.mutateChildren(AbstractNodeContainerModificationStrategy.java:139)
at org.opendaylight.yangtools.yang.data.impl.schema.tree.AbstractNodeContainerModificationStrategy.applyTouch(AbstractNodeContainerModificationStrategy.java:252)
at org.opendaylight.yangtools.yang.data.impl.schema.tree.ListEntryModificationStrategy.applyTouch(ListEntryModificationStrategy.java:55)
at org.opendaylight.yangtools.yang.data.impl.schema.tree.SchemaAwareApplyOperation.apply(SchemaAwareApplyOperation.java:200)
at org.opendaylight.yangtools.yang.data.impl.schema.tree.AbstractNodeContainerModificationStrategy.mutateChildren(AbstractNodeContainerModificationStrategy.java:139)
at org.opendaylight.yangtools.yang.data.impl.schema.tree.AbstractNodeContainerModificationStrategy.applyTouch(AbstractNodeContainerModificationStrategy.java:252)
at org.opendaylight.yangtools.yang.data.impl.schema.tree.SchemaAwareApplyOperation.apply(SchemaAwareApplyOperation.java:200)
at org.opendaylight.yangtools.yang.data.impl.schema.tree.AbstractNodeContainerModificationStrategy.mutateChildren(AbstractNodeContainerModificationStrategy.java:139)
at org.opendaylight.yangtools.yang.data.impl.schema.tree.AbstractNodeContainerModificationStrategy.applyTouch(AbstractNodeContainerModificationStrategy.java:252)
at org.opendaylight.yangtools.yang.data.impl.schema.tree.SchemaAwareApplyOperation.apply(SchemaAwareApplyOperation.java:200)
at org.opendaylight.yangtools.yang.data.impl.schema.tree.StructuralContainerModificationStrategy.apply(StructuralContainerModificationStrategy.java:71)
at org.opendaylight.yangtools.yang.data.impl.schema.tree.AbstractNodeContainerModificationStrategy.mutateChildren(AbstractNodeContainerModificationStrategy.java:139)
at org.opendaylight.yangtools.yang.data.impl.schema.tree.AbstractNodeContainerModificationStrategy.applyTouch(AbstractNodeContainerModificationStrategy.java:252)
at org.opendaylight.yangtools.yang.data.impl.schema.tree.SchemaAwareApplyOperation.apply(SchemaAwareApplyOperation.java:200)
at org.opendaylight.yangtools.yang.data.impl.schema.tree.RootModificationApplyOperation.apply(RootModificationApplyOperation.java:78)
at org.opendaylight.yangtools.yang.data.impl.schema.tree.AbstractDataTreeTip.prepare(AbstractDataTreeTip.java:53)
at org.opendaylight.controller.cluster.datastore.ShardDataTree.startPreCommit(ShardDataTree.java:809)
at org.opendaylight.controller.cluster.datastore.SimpleShardDataTreeCohort.preCommit(SimpleShardDataTreeCohort.java:115)
at org.opendaylight.controller.cluster.datastore.ChainedCommitCohort.preCommit(ChainedCommitCohort.java:63)
at org.opendaylight.controller.cluster.datastore.FrontendReadWriteTransaction.successfulDirectCanCommit(FrontendReadWriteTransaction.java:212)
at org.opendaylight.controller.cluster.datastore.FrontendReadWriteTransaction$5.onSuccess(FrontendReadWriteTransaction.java:200)
at org.opendaylight.controller.cluster.datastore.FrontendReadWriteTransaction$5.onSuccess(FrontendReadWriteTransaction.java:197)
at org.opendaylight.controller.cluster.datastore.SimpleShardDataTreeCohort.successfulCanCommit(SimpleShardDataTreeCohort.java:183)
at org.opendaylight.controller.cluster.datastore.ShardDataTree.lambda$processNextPendingTransaction$0(ShardDataTree.java:718)
at org.opendaylight.controller.cluster.datastore.ShardDataTree.processNextPending(ShardDataTree.java:762)
at org.opendaylight.controller.cluster.datastore.ShardDataTree.processNextPendingTransaction(ShardDataTree.java:707)
at org.opendaylight.controller.cluster.datastore.ShardDataTree.startPreCommit(ShardDataTree.java:828)
at org.opendaylight.controller.cluster.datastore.SimpleShardDataTreeCohort.preCommit(SimpleShardDataTreeCohort.java:115)
at org.opendaylight.controller.cluster.datastore.ChainedCommitCohort.preCommit(ChainedCommitCohort.java:63)
at org.opendaylight.controller.cluster.datastore.FrontendReadWriteTransaction.successfulDirectCanCommit(FrontendReadWriteTransaction.java:212)
at org.opendaylight.controller.cluster.datastore.FrontendReadWriteTransaction$5.onSuccess(FrontendReadWriteTransaction.java:200)
at org.opendaylight.controller.cluster.datastore.FrontendReadWriteTransaction$5.onSuccess(FrontendReadWriteTransaction.java:197)
at org.opendaylight.controller.cluster.datastore.SimpleShardDataTreeCohort.successfulCanCommit(SimpleShardDataTreeCohort.java:183)
at org.opendaylight.controller.cluster.datastore.ShardDataTree.lambda$processNextPendingTransaction$0(ShardDataTree.java:718)
at org.opendaylight.controller.cluster.datastore.ShardDataTree.processNextPending(ShardDataTree.java:762)
at org.opendaylight.controller.cluster.datastore.ShardDataTree.processNextPendingTransaction(ShardDataTree.java:707)
at org.opendaylight.controller.cluster.datastore.ShardDataTree.startPreCommit(ShardDataTree.java:828)
at org.opendaylight.controller.cluster.datastore.SimpleShardDataTreeCohort.preCommit(SimpleShardDataTreeCohort.java:115)
at org.opendaylight.controller.cluster.datastore.ChainedCommitCohort.preCommit(ChainedCommitCohort.java:63)
at org.opendaylight.controller.cluster.datastore.FrontendReadWriteTransaction.successfulDirectCanCommit(FrontendReadWriteTransaction.java:212)
at org.opendaylight.controller.cluster.datastore.FrontendReadWriteTransaction$5.onSuccess(FrontendReadWriteTransaction.java:200)
at org.opendaylight.controller.cluster.datastore.FrontendReadWriteTransaction$5.onSuccess(FrontendReadWriteTransaction.java:197)
at org.opendaylight.controller.cluster.datastore.SimpleShardDataTreeCohort.successfulCanCommit(SimpleShardDataTreeCohort.java:183)
at org.opendaylight.controller.cluster.datastore.ShardDataTree.lambda$processNextPendingTransaction$0(ShardDataTree.java:718)
at org.opendaylight.controller.cluster.datastore.ShardDataTree.processNextPending(ShardDataTree.java:762)
at org.opendaylight.controller.cluster.datastore.ShardDataTree.processNextPendingTransaction(ShardDataTree.java:707)
at org.opendaylight.controller.cluster.datastore.ShardDataTree.startPreCommit(ShardDataTree.java:828)
at org.opendaylight.controller.cluster.datastore.SimpleShardDataTreeCohort.preCommit(SimpleShardDataTreeCohort.java:115)
at org.opendaylight.controller.cluster.datastore.ChainedCommitCohort.preCommit(ChainedCommitCohort.java:63)
at org.opendaylight.controller.cluster.datastore.FrontendReadWriteTransaction.successfulDirectCanCommit(FrontendReadWriteTransaction.java:212)
at org.opendaylight.controller.cluster.datastore.FrontendReadWriteTransaction$5.onSuccess(FrontendReadWriteTransaction.java:200)
at org.opendaylight.controller.cluster.datastore.FrontendReadWriteTransaction$5.onSuccess(FrontendReadWriteTransaction.java:197)
at org.opendaylight.controller.cluster.datastore.SimpleShardDataTreeCohort.successfulCanCommit(SimpleShardDataTreeCohort.java:183)
at org.opendaylight.controller.cluster.datastore.ShardDataTree.lambda$processNextPendingTransaction$0(ShardDataTree.java:718)
at org.opendaylight.controller.cluster.datastore.ShardDataTree.processNextPending(ShardDataTree.java:762)
at org.opendaylight.controller.cluster.datastore.ShardDataTree.processNextPendingTransaction(ShardDataTree.java:707)
at org.opendaylight.controller.cluster.datastore.ShardDataTree.startPreCommit(ShardDataTree.java:828)
at org.opendaylight.controller.cluster.datastore.SimpleShardDataTreeCohort.preCommit(SimpleShardDataTreeCohort.java:115)
at org.opendaylight.controller.cluster.datastore.ChainedCommitCohort.preCommit(ChainedCommitCohort.java:63)
at org.opendaylight.controller.cluster.datastore.FrontendReadWriteTransaction.successfulDirectCanCommit(FrontendReadWriteTransaction.java:212)
at org.opendaylight.controller.cluster.datastore.FrontendReadWriteTransaction$5.onSuccess(FrontendReadWriteTransaction.java:200)
at org.opendaylight.controller.cluster.datastore.FrontendReadWriteTransaction$5.onSuccess(FrontendReadWriteTransaction.java:197)
at org.opendaylight.controller.cluster.datastore.SimpleShardDataTreeCohort.successfulCanCommit(SimpleShardDataTreeCohort.java:183)
at org.opendaylight.controller.cluster.datastore.ShardDataTree.lambda$processNextPendingTransaction$0(ShardDataTree.java:718)
at org.opendaylight.controller.cluster.datastore.ShardDataTree.processNextPending(ShardDataTree.java:762)
at org.opendaylight.controller.cluster.datastore.ShardDataTree.processNextPendingTransaction(ShardDataTree.java:707)
at org.opendaylight.controller.cluster.datastore.ShardDataTree.startPreCommit(ShardDataTree.java:828)
at org.opendaylight.controller.cluster.datastore.SimpleShardDataTreeCohort.preCommit(SimpleShardDataTreeCohort.java:115)
at org.opendaylight.controller.cluster.datastore.ChainedCommitCohort.preCommit(ChainedCommitCohort.java:63)
at org.opendaylight.controller.cluster.datastore.FrontendReadWriteTransaction.successfulDirectCanCommit(FrontendReadWriteTransaction.java:212)
at org.opendaylight.controller.cluster.datastore.FrontendReadWriteTransaction$5.onSuccess(FrontendReadWriteTransaction.java:200)
at org.opendaylight.controller.cluster.datastore.FrontendReadWriteTransaction$5.onSuccess(FrontendReadWriteTransaction.java:197)
at org.opendaylight.controller.cluster.datastore.SimpleShardDataTreeCohort.successfulCanCommit(SimpleShardDataTreeCohort.java:183)
at org.opendaylight.controller.cluster.datastore.ShardDataTree.lambda$processNextPendingTransaction$0(ShardDataTree.java:718)
at org.opendaylight.controller.cluster.datastore.ShardDataTree.processNextPending(Shar