Exceptions seen : org.opendaylight.controller.cluster.datastore.exceptions.NoShardLeaderException: Could not process ready local transaction member-1-datastore-operational-fe-4-txn-403-0.

In 3 nodes setup we are seeing exceptions after multiple bringing up and down of ODL.

logs:

https://jenkins.opendaylight.org/releng/view/netvirt-csit/job/netvirt-csit-3node-0cmb-1ctl-2cmp-openstack-queens-upstream-stateful-sodium/393/

Karaf Logs:

https://logs.opendaylight.org/releng/vex-yul-odl-jenkins-1/netvirt-csit-3node-0cmb-1ctl-2cmp-openstack-queens-upstream-stateful-sodium/393/odl_1/odl1_karaf.log.gz

Karaf exceptions:

org.opendaylight.controller.cluster.datastore.exceptions.NoShardLeaderException: Could not process ready local transaction member-1-datastore-operational-fe-4-txn-403-0. Shard member-1-shard-default-operational currently has no leader. Try again later.
at org.opendaylight.controller.cluster.datastore.ShardTransactionMessageRetrySupport$MessageInfo.timedOut(ShardTransactionMessageRetrySupport.java:102) ~[246:org.opendaylight.controller.sal-distributed-datastore:1.10.2]
at org.opendaylight.controller.cluster.datastore.ShardTransactionMessageRetrySupport.onTimerMessage(ShardTransactionMessageRetrySupport.java:72) ~[246:org.opendaylight.controller.sal-distributed-datastore:1.10.2]
at org.opendaylight.controller.cluster.datastore.Shard.handleNonRaftCommand(Shard.java:369) ~[246:org.opendaylight.controller.sal-distributed-datastore:1.10.2]
at org.opendaylight.controller.cluster.raft.RaftActor.handleCommand(RaftActor.java:270) ~[231:org.opendaylight.controller.sal-akka-raft:1.10.2]
at akka.japi.pf.UnitCaseStatement.apply(CaseStatements.scala:24) ~[41:com.typesafe.akka.actor:2.5.25]
at akka.japi.pf.UnitCaseStatement.apply(CaseStatements.scala:20) ~[41:com.typesafe.akka.actor:2.5.25]
at scala.PartialFunction.applyOrElse(PartialFunction.scala:127) ~[481:org.scala-lang.scala-library:2.12.10.v20190904-150159-VFINAL-61701c2]
at scala.PartialFunction.applyOrElse$(PartialFunction.scala:126) ~[481:org.scala-lang.scala-library:2.12.10.v20190904-150159-VFINAL-61701c2]
at akka.japi.pf.UnitCaseStatement.applyOrElse(CaseStatements.scala:20) ~[41:com.typesafe.akka.actor:2.5.25]
at scala.PartialFunction$OrElse.apply(PartialFunction.scala:172) ~[481:org.scala-lang.scala-library:2.12.10.v20190904-150159-VFINAL-61701c2]
at scala.PartialFunction.applyOrElse(PartialFunction.scala:127) ~[481:org.scala-lang.scala-library:2.12.10.v20190904-150159-VFINAL-61701c2]
at scala.PartialFunction.applyOrElse$(PartialFunction.scala:126) ~[481:org.scala-lang.scala-library:2.12.10.v20190904-150159-VFINAL-61701c2]
at akka.japi.pf.UnitCaseStatement.applyOrElse(CaseStatements.scala:20) ~[41:com.typesafe.akka.actor:2.5.25]
at scala.PartialFunction$OrElse.apply(PartialFunction.scala:172) ~[481:org.scala-lang.scala-library:2.12.10.v20190904-150159-VFINAL-61701c2]
at org.opendaylight.controller.cluster.common.actor.MeteringBehavior.apply(MeteringBehavior.java:93) ~[239:org.opendaylight.controller.sal-clustering-commons:1.10.2]
at org.opendaylight.controller.cluster.common.actor.MeteringBehavior.apply(MeteringBehavior.java:29) ~[239:org.opendaylight.controller.sal-clustering-commons:1.10.2]
at scala.PartialFunction.applyOrElse(PartialFunction.scala:127) ~[481:org.scala-lang.scala-library:2.12.10.v20190904-150159-VFINAL-61701c2]
at scala.PartialFunction.applyOrElse$(PartialFunction.scala:126) ~[481:org.scala-lang.scala-library:2.12.10.v20190904-150159-VFINAL-61701c2]
at scala.runtime.AbstractPartialFunction.applyOrElse(AbstractPartialFunction.scala:30) ~[481:org.scala-lang.scala-library:2.12.10.v20190904-150159-VFINAL-61701c2]
at akka.actor.Actor.aroundReceive(Actor.scala:539) ~[41:com.typesafe.akka.actor:2.5.25]
at akka.actor.Actor.aroundReceive$(Actor.scala:537) ~[41:com.typesafe.akka.actor:2.5.25]
at akka.persistence.AbstractPersistentActor.akka$persistence$Eventsourced$$super$aroundReceive(PersistentActor.scala:455) ~[44:com.typesafe.akka.persistence:2.5.25]
at akka.persistence.Eventsourced$$anon$4.stateReceive(Eventsourced.scala:898) ~[44:com.typesafe.akka.persistence:2.5.25]
at akka.persistence.Eventsourced.aroundReceive(Eventsourced.scala:250) ~[44:com.typesafe.akka.persistence:2.5.25]
at akka.persistence.Eventsourced.aroundReceive$(Eventsourced.scala:249) ~[44:com.typesafe.akka.persistence:2.5.25]
at akka.persistence.AbstractPersistentActor.aroundReceive(PersistentActor.scala:455) ~[44:com.typesafe.akka.persistence:2.5.25]
at akka.actor.ActorCell.receiveMessage(ActorCell.scala:612) ~[41:com.typesafe.akka.actor:2.5.25]
at akka.actor.ActorCell.invoke(ActorCell.scala:581) ~[41:com.typesafe.akka.actor:2.5.25]
at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:268) ~[41:com.typesafe.akka.actor:2.5.25]
at akka.dispatch.Mailbox.run(Mailbox.scala:229) ~[41:com.typesafe.akka.actor:2.5.25]
at akka.dispatch.Mailbox.exec(Mailbox.scala:241) ~[41:com.typesafe.akka.actor:2.5.25]
at akka.dispatch.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260) [41:com.typesafe.akka.actor:2.5.25]
at akka.dispatch.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339) [41:com.typesafe.akka.actor:2.5.25]
at akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) [41:com.typesafe.akka.actor:2.5.25]
at akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) [41:com.typesafe.akka.actor:2.5.25]