<!-- 
RSS generated by JIRA (8.20.10#820010-sha1:ace47f9899e9ee25d7157d59aa17ab06aee30d3d) at Wed Feb 07 19:56:07 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>OpenDaylight JIRA</title>
    <link>https://jira.opendaylight.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>8.20.10</version>
        <build-number>820010</build-number>
        <build-date>22-06-2022</build-date>
    </build-info>


<item>
            <title>[CONTROLLER-1664] C: OutOfOrderRequestException: Expecting request 0 (while create-prefix-shard)</title>
                <link>https://jira.opendaylight.org/browse/CONTROLLER-1664</link>
                <project id="10113" key="CONTROLLER">controller</project>
                    <description>&lt;p&gt;&lt;a href=&quot;https://logs.opendaylight.org/sandbox/jenkins091/controller-csit-3node-clustering-only-carbon-4th/5/archives/odl1_karaf.log.gz&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://logs.opendaylight.org/sandbox/jenkins091/controller-csit-3node-clustering-only-carbon-4th/5/archives/odl1_karaf.log.gz&lt;/a&gt;&lt;/p&gt;


&lt;p&gt;At the bottom of the log file:&lt;br/&gt;
2017-05-09 05:02:42,423 | WARN  | monPool-worker-0 | PrefixShardHandler               | 257 - org.opendaylight.controller.samples.clustering-it-provider - 1.5.0.SNAPSHOT | Shard&lt;span class=&quot;error&quot;&gt;&amp;#91;/(tag:opendaylight.org,2017:controller:yang:lowlevel:target?revision=2017-02-15)id-ints&amp;#93;&lt;/span&gt; creation failed:&lt;br/&gt;
org.opendaylight.controller.cluster.sharding.DOMDataTreeShardCreationFailedException: Unable to create a cds shard.&lt;br/&gt;
        at org.opendaylight.controller.cluster.sharding.DistributedShardedDOMDataTree$2.onFailure(DistributedShardedDOMDataTree.java:366)&lt;br/&gt;
        at com.google.common.util.concurrent.Futures$6.run(Futures.java:1310)&lt;span class=&quot;error&quot;&gt;&amp;#91;57:com.google.guava:18.0.0&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at com.google.common.util.concurrent.MoreExecutors$DirectExecutor.execute(MoreExecutors.java:457)&lt;span class=&quot;error&quot;&gt;&amp;#91;57:com.google.guava:18.0.0&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at com.google.common.util.concurrent.ExecutionList.executeListener(ExecutionList.java:156)&lt;span class=&quot;error&quot;&gt;&amp;#91;57:com.google.guava:18.0.0&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at com.google.common.util.concurrent.ExecutionList.execute(ExecutionList.java:145)&lt;span class=&quot;error&quot;&gt;&amp;#91;57:com.google.guava:18.0.0&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at com.google.common.util.concurrent.AbstractFuture.setException(AbstractFuture.java:202)&lt;span class=&quot;error&quot;&gt;&amp;#91;57:com.google.guava:18.0.0&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at com.google.common.util.concurrent.Futures$ChainingListenableFuture.run(Futures.java:902)&lt;span class=&quot;error&quot;&gt;&amp;#91;57:com.google.guava:18.0.0&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at com.google.common.util.concurrent.MoreExecutors$DirectExecutor.execute(MoreExecutors.java:457)&lt;span class=&quot;error&quot;&gt;&amp;#91;57:com.google.guava:18.0.0&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at com.google.common.util.concurrent.ExecutionList.executeListener(ExecutionList.java:156)&lt;span class=&quot;error&quot;&gt;&amp;#91;57:com.google.guava:18.0.0&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at com.google.common.util.concurrent.ExecutionList.execute(ExecutionList.java:145)&lt;span class=&quot;error&quot;&gt;&amp;#91;57:com.google.guava:18.0.0&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at com.google.common.util.concurrent.AbstractFuture.setException(AbstractFuture.java:202)&lt;span class=&quot;error&quot;&gt;&amp;#91;57:com.google.guava:18.0.0&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at com.google.common.util.concurrent.Futures$ChainingListenableFuture.run(Futures.java:902)&lt;span class=&quot;error&quot;&gt;&amp;#91;57:com.google.guava:18.0.0&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at com.google.common.util.concurrent.MoreExecutors$DirectExecutor.execute(MoreExecutors.java:457)&lt;span class=&quot;error&quot;&gt;&amp;#91;57:com.google.guava:18.0.0&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at com.google.common.util.concurrent.ExecutionList.executeListener(ExecutionList.java:156)&lt;span class=&quot;error&quot;&gt;&amp;#91;57:com.google.guava:18.0.0&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at com.google.common.util.concurrent.ExecutionList.execute(ExecutionList.java:145)&lt;span class=&quot;error&quot;&gt;&amp;#91;57:com.google.guava:18.0.0&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at com.google.common.util.concurrent.AbstractFuture.setException(AbstractFuture.java:202)&lt;span class=&quot;error&quot;&gt;&amp;#91;57:com.google.guava:18.0.0&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at com.google.common.util.concurrent.SettableFuture.setException(SettableFuture.java:68)&lt;span class=&quot;error&quot;&gt;&amp;#91;57:com.google.guava:18.0.0&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at org.opendaylight.controller.cluster.databroker.actors.dds.AbstractProxyTransaction.lambda$directCommit$1(AbstractProxyTransaction.java:353)&lt;br/&gt;
        at org.opendaylight.controller.cluster.access.client.ConnectionEntry.complete(ConnectionEntry.java:50)&lt;span class=&quot;error&quot;&gt;&amp;#91;197:org.opendaylight.controller.cds-access-client:1.1.0.SNAPSHOT&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at org.opendaylight.controller.cluster.access.client.AbstractClientConnection.receiveResponse(AbstractClientConnection.java:300)&lt;span class=&quot;error&quot;&gt;&amp;#91;197:org.opendaylight.controller.cds-access-client:1.1.0.SNAPSHOT&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at org.opendaylight.controller.cluster.access.client.ClientActorBehavior.onResponse(ClientActorBehavior.java:143)&lt;span class=&quot;error&quot;&gt;&amp;#91;197:org.opendaylight.controller.cds-access-client:1.1.0.SNAPSHOT&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at org.opendaylight.controller.cluster.access.client.ClientActorBehavior.onRequestFailure(ClientActorBehavior.java:155)&lt;span class=&quot;error&quot;&gt;&amp;#91;197:org.opendaylight.controller.cds-access-client:1.1.0.SNAPSHOT&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at org.opendaylight.controller.cluster.access.client.ClientActorBehavior.internalOnRequestFailure(ClientActorBehavior.java:179)&lt;span class=&quot;error&quot;&gt;&amp;#91;197:org.opendaylight.controller.cds-access-client:1.1.0.SNAPSHOT&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at org.opendaylight.controller.cluster.access.client.ClientActorBehavior.onReceiveCommand(ClientActorBehavior.java:124)&lt;span class=&quot;error&quot;&gt;&amp;#91;197:org.opendaylight.controller.cds-access-client:1.1.0.SNAPSHOT&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at org.opendaylight.controller.cluster.access.client.ClientActorBehavior.onReceiveCommand(ClientActorBehavior.java:42)&lt;span class=&quot;error&quot;&gt;&amp;#91;197:org.opendaylight.controller.cds-access-client:1.1.0.SNAPSHOT&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at org.opendaylight.controller.cluster.access.client.AbstractClientActor.onReceiveCommand(AbstractClientActor.java:59)&lt;span class=&quot;error&quot;&gt;&amp;#91;197:org.opendaylight.controller.cds-access-client:1.1.0.SNAPSHOT&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at akka.persistence.UntypedPersistentActor.onReceive(PersistentActor.scala:170)&lt;span class=&quot;error&quot;&gt;&amp;#91;180:com.typesafe.akka.persistence:2.4.17&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at akka.actor.UntypedActor$$anonfun$receive$1.applyOrElse(UntypedActor.scala:165)&lt;span class=&quot;error&quot;&gt;&amp;#91;173:com.typesafe.akka.actor:2.4.17&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at akka.actor.Actor$class.aroundReceive(Actor.scala:497)&lt;span class=&quot;error&quot;&gt;&amp;#91;173:com.typesafe.akka.actor:2.4.17&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at akka.persistence.UntypedPersistentActor.akka$persistence$Eventsourced$$super$aroundReceive(PersistentActor.scala:168)&lt;span class=&quot;error&quot;&gt;&amp;#91;180:com.typesafe.akka.persistence:2.4.17&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at akka.persistence.Eventsourced$$anon$1.stateReceive(Eventsourced.scala:664)&lt;span class=&quot;error&quot;&gt;&amp;#91;180:com.typesafe.akka.persistence:2.4.17&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at akka.persistence.Eventsourced$class.aroundReceive(Eventsourced.scala:183)&lt;span class=&quot;error&quot;&gt;&amp;#91;180:com.typesafe.akka.persistence:2.4.17&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at akka.persistence.UntypedPersistentActor.aroundReceive(PersistentActor.scala:168)&lt;span class=&quot;error&quot;&gt;&amp;#91;180:com.typesafe.akka.persistence:2.4.17&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at akka.actor.ActorCell.receiveMessage(ActorCell.scala:526)&lt;span class=&quot;error&quot;&gt;&amp;#91;173:com.typesafe.akka.actor:2.4.17&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at akka.actor.ActorCell.invoke(ActorCell.scala:495)&lt;span class=&quot;error&quot;&gt;&amp;#91;173:com.typesafe.akka.actor:2.4.17&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:257)&lt;span class=&quot;error&quot;&gt;&amp;#91;173:com.typesafe.akka.actor:2.4.17&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at akka.dispatch.Mailbox.run(Mailbox.scala:224)&lt;span class=&quot;error&quot;&gt;&amp;#91;173:com.typesafe.akka.actor:2.4.17&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at akka.dispatch.Mailbox.exec(Mailbox.scala:234)&lt;span class=&quot;error&quot;&gt;&amp;#91;173:com.typesafe.akka.actor:2.4.17&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)&lt;span class=&quot;error&quot;&gt;&amp;#91;169:org.scala-lang.scala-library:2.11.8.v20160304-115712-1706a37eb8&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)&lt;span class=&quot;error&quot;&gt;&amp;#91;169:org.scala-lang.scala-library:2.11.8.v20160304-115712-1706a37eb8&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)&lt;span class=&quot;error&quot;&gt;&amp;#91;169:org.scala-lang.scala-library:2.11.8.v20160304-115712-1706a37eb8&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)&lt;span class=&quot;error&quot;&gt;&amp;#91;169:org.scala-lang.scala-library:2.11.8.v20160304-115712-1706a37eb8&amp;#93;&lt;/span&gt;&lt;br/&gt;
Caused by: org.opendaylight.controller.cluster.access.commands.OutOfOrderRequestException: Expecting request 0&lt;br/&gt;
        at org.opendaylight.controller.cluster.datastore.LeaderFrontendState.checkRequestSequence(LeaderFrontendState.java:96)&lt;span class=&quot;error&quot;&gt;&amp;#91;199:org.opendaylight.controller.sal-distributed-datastore:1.5.0.SNAPSHOT&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at org.opendaylight.controller.cluster.datastore.LeaderFrontendState.handleTransactionRequest(LeaderFrontendState.java:182)&lt;span class=&quot;error&quot;&gt;&amp;#91;199:org.opendaylight.controller.sal-distributed-datastore:1.5.0.SNAPSHOT&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at org.opendaylight.controller.cluster.datastore.Shard.handleRequest(Shard.java:450)&lt;span class=&quot;error&quot;&gt;&amp;#91;199:org.opendaylight.controller.sal-distributed-datastore:1.5.0.SNAPSHOT&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at org.opendaylight.controller.cluster.datastore.Shard.handleNonRaftCommand(Shard.java:283)&lt;span class=&quot;error&quot;&gt;&amp;#91;199:org.opendaylight.controller.sal-distributed-datastore:1.5.0.SNAPSHOT&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at org.opendaylight.controller.cluster.raft.RaftActor.handleCommand(RaftActor.java:270)&lt;span class=&quot;error&quot;&gt;&amp;#91;193:org.opendaylight.controller.sal-akka-raft:1.5.0.SNAPSHOT&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at org.opendaylight.controller.cluster.common.actor.AbstractUntypedPersistentActor.onReceiveCommand(AbstractUntypedPersistentActor.java:31)&lt;span class=&quot;error&quot;&gt;&amp;#91;192:org.opendaylight.controller.sal-clustering-commons:1.5.0.SNAPSHOT&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at akka.persistence.UntypedPersistentActor.onReceive(PersistentActor.scala:170)&lt;span class=&quot;error&quot;&gt;&amp;#91;180:com.typesafe.akka.persistence:2.4.17&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at org.opendaylight.controller.cluster.common.actor.MeteringBehavior.apply(MeteringBehavior.java:104)&lt;span class=&quot;error&quot;&gt;&amp;#91;192:org.opendaylight.controller.sal-clustering-commons:1.5.0.SNAPSHOT&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at akka.actor.ActorCell$$anonfun$become$1.applyOrElse(ActorCell.scala:544)&lt;span class=&quot;error&quot;&gt;&amp;#91;173:com.typesafe.akka.actor:2.4.17&amp;#93;&lt;/span&gt;&lt;br/&gt;
        ... 14 more&lt;/p&gt;</description>
                <environment>&lt;p&gt;Operating System: All&lt;br/&gt;
Platform: All&lt;/p&gt;</environment>
        <key id="26218">CONTROLLER-1664</key>
            <summary>C: OutOfOrderRequestException: Expecting request 0 (while create-prefix-shard)</summary>
                <type id="10104" iconUrl="https://jira.opendaylight.org/secure/viewavatar?size=xsmall&amp;avatarId=10303&amp;avatarType=issuetype">Bug</type>
                                                <status id="5" iconUrl="https://jira.opendaylight.org/images/icons/statuses/resolved.png" description="A resolution has been taken, and it is awaiting verification by reporter. From here issues are either reopened, or are closed.">Resolved</status>
                    <statusCategory id="3" key="done" colorName="green"/>
                                    <resolution id="10000">Done</resolution>
                                        <assignee username="rovarga">Robert Varga</assignee>
                                    <reporter username="pgubka@cisco.com">Peter Gubka</reporter>
                        <labels>
                    </labels>
                <created>Tue, 9 May 2017 08:37:43 +0000</created>
                <updated>Tue, 25 Jul 2023 08:24:35 +0000</updated>
                            <resolved>Fri, 26 May 2017 11:44:28 +0000</resolved>
                                                                    <component>clustering</component>
                        <due></due>
                            <votes>0</votes>
                                    <watches>4</watches>
                                                                                                                <comments>
                            <comment id="52121" author="pgubka@cisco.com" created="Tue, 9 May 2017 09:15:28 +0000"  >&lt;p&gt;The same exception happened for write-transactions rpc in remote to remote explicit leader movement scenario&lt;br/&gt;
&lt;a href=&quot;https://logs.opendaylight.org/sandbox/jenkins091/controller-csit-3node-clustering-only-carbon-2nd/12/archives/log.html.gz#s1-s1-t2-k2-k8&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://logs.opendaylight.org/sandbox/jenkins091/controller-csit-3node-clustering-only-carbon-2nd/12/archives/log.html.gz#s1-s1-t2-k2-k8&lt;/a&gt;&lt;/p&gt;

&lt;p&gt;Transaction producer was on node3 and default hsard leader moved form node1 to node2.&lt;/p&gt;

&lt;p&gt;Node1 logs:&lt;br/&gt;
&lt;a href=&quot;https://logs.opendaylight.org/sandbox/jenkins091/controller-csit-3node-clustering-only-carbon-2nd/12/archives/odl1_karaf.log.gz&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://logs.opendaylight.org/sandbox/jenkins091/controller-csit-3node-clustering-only-carbon-2nd/12/archives/odl1_karaf.log.gz&lt;/a&gt;&lt;br/&gt;
2017-05-09 04:57:22,379 | INFO  | h for user karaf | command                          | 266 - org.apache.karaf.log.command - 3.0.8 | ROBOT MESSAGE: Starting test Remote_To_Remote_Movement&lt;/p&gt;</comment>
                            <comment id="52122" author="pgubka@cisco.com" created="Tue, 9 May 2017 11:09:21 +0000"  >&lt;p&gt;Same http status 500 with &quot;OutOfOrderRequestException: Expecting request 0&quot; is returned for clean leader shutdown scenario when remove-shard-replica is used.&lt;br/&gt;
Replica is removed from on leader node and no matter where producer runs (same or different node)&lt;/p&gt;

&lt;p&gt;Robot and logs:&lt;br/&gt;
&lt;a href=&quot;https://logs.opendaylight.org/sandbox/jenkins091/controller-csit-3node-clustering-only-carbon-3rd/8&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://logs.opendaylight.org/sandbox/jenkins091/controller-csit-3node-clustering-only-carbon-3rd/8&lt;/a&gt;&lt;/p&gt;

&lt;p&gt;See logs after&lt;br/&gt;
2017-05-09 04:53:22,244 | INFO  | h for user karaf | command                          | 266 - org.apache.karaf.log.command - 3.0.8 | ROBOT MESSAGE: Starting test Local_Leader_Shutdown&lt;br/&gt;
or&lt;br/&gt;
2017-05-09 04:54:45,023 | INFO  | h for user karaf | command                          | 266 - org.apache.karaf.log.command - 3.0.8 | ROBOT MESSAGE: Starting test Remote_Leader_Shutdown&lt;/p&gt;</comment>
                            <comment id="52123" author="rovarga" created="Tue, 9 May 2017 14:59:09 +0000"  >&lt;p&gt;I&apos;ll need debug logs for org.opendaylight.controller.cluster.databroker.actors a org.opendaylight.controller.cluster.datastore.&lt;/p&gt;</comment>
                            <comment id="52124" author="pgubka@cisco.com" created="Wed, 10 May 2017 12:26:02 +0000"  >&lt;p&gt;(In reply to Robert Varga from comment #3)&lt;br/&gt;
&amp;gt; I&apos;ll need debug logs for&lt;br/&gt;
&amp;gt; org.opendaylight.controller.cluster.databroker.actors a&lt;br/&gt;
&amp;gt; org.opendaylight.controller.cluster.datastore.&lt;/p&gt;

&lt;p&gt;For simplicity i tried to replicate the scenario explicit leader movement with module shards. I reduced the suite to just one test case.&lt;/p&gt;

&lt;p&gt;Without debug logging &lt;br/&gt;
&lt;a href=&quot;https://jenkins.opendaylight.org/sandbox/job/controller-csit-3node-clustering-only-carbon-3rd/10/&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://jenkins.opendaylight.org/sandbox/job/controller-csit-3node-clustering-only-carbon-3rd/10/&lt;/a&gt;&lt;br/&gt;
the exception is received as expected.&lt;/p&gt;


&lt;p&gt;With debug logging i have not succeeded to replicate the problem. &lt;br/&gt;
&lt;a href=&quot;https://jenkins.opendaylight.org/sandbox/job/controller-csit-3node-clustering-only-carbon-3rd/11/&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://jenkins.opendaylight.org/sandbox/job/controller-csit-3node-clustering-only-carbon-3rd/11/&lt;/a&gt;&lt;br/&gt;
write-transactions just timed out from robot side. And it generated huge logs. &lt;/p&gt;

&lt;p&gt;Locally when switch debug logging i am not able to replicate it too.&lt;/p&gt;</comment>
                            <comment id="52125" author="rovarga" created="Fri, 12 May 2017 11:31:10 +0000"  >&lt;p&gt;Aside from the usual replay issues, there seems to be a problem with concurrent access:&lt;/p&gt;

&lt;p&gt;java.util.ConcurrentModificationException&lt;br/&gt;
        at java.util.ArrayDeque$DeqIterator.next(ArrayDeque.java:643)&lt;span class=&quot;error&quot;&gt;&amp;#91;:1.8.0_121&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at com.google.common.collect.Iterators$5.next(Iterators.java:558)&lt;span class=&quot;error&quot;&gt;&amp;#91;57:com.google.guava:18.0.0&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at org.opendaylight.controller.cluster.databroker.actors.dds.AbstractProxyTransaction.replayMessages(AbstractProxyTransaction.java:507)&lt;span class=&quot;error&quot;&gt;&amp;#91;199:org.opendaylight.controller.sal-distributed-datastore:1.5.0.SNAPSHOT&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at org.opendaylight.controller.cluster.databroker.actors.dds.ProxyHistory$ReconnectCohort.replayRequests(ProxyHistory.java:232)&lt;span class=&quot;error&quot;&gt;&amp;#91;199:org.opendaylight.controller.sal-distributed-datastore:1.5.0.SNAPSHOT&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at org.opendaylight.controller.cluster.databroker.actors.dds.AbstractClientHistory$1.replayRequests(AbstractClientHistory.java:310)&lt;span class=&quot;error&quot;&gt;&amp;#91;199:org.opendaylight.controller.sal-distributed-datastore:1.5.0.SNAPSHOT&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at org.opendaylight.controller.cluster.databroker.actors.dds.AbstractDataStoreClientBehavior.lambda$connectionUp$0(AbstractDataStoreClientBehavior.java:147)&lt;span class=&quot;error&quot;&gt;&amp;#91;199:org.opendaylight.controller.sal-distributed-datastore:1.5.0.SNAPSHOT&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at org.opendaylight.controller.cluster.access.client.ClientActorBehavior.backendConnectFinished(ClientActorBehavior.java:280)&lt;span class=&quot;error&quot;&gt;&amp;#91;197:org.opendaylight.controller.cds-access-client:1.1.0.SNAPSHOT&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at org.opendaylight.controller.cluster.access.client.ClientActorBehavior.lambda$null$3(ClientActorBehavior.java:329)&lt;span class=&quot;error&quot;&gt;&amp;#91;197:org.opendaylight.controller.cds-access-client:1.1.0.SNAPSHOT&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at org.opendaylight.controller.cluster.access.client.ClientActorBehavior.onReceiveCommand(ClientActorBehavior.java:118)&lt;span class=&quot;error&quot;&gt;&amp;#91;197:org.opendaylight.controller.cds-access-client:1.1.0.SNAPSHOT&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at org.opendaylight.controller.cluster.access.client.ClientActorBehavior.onReceiveCommand(ClientActorBehavior.java:42)&lt;span class=&quot;error&quot;&gt;&amp;#91;197:org.opendaylight.controller.cds-access-client:1.1.0.SNAPSHOT&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at org.opendaylight.controller.cluster.access.client.AbstractClientActor.onReceiveCommand(AbstractClientActor.java:59)&lt;span class=&quot;error&quot;&gt;&amp;#91;197:org.opendaylight.controller.cds-access-client:1.1.0.SNAPSHOT&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at akka.persistence.UntypedPersistentActor.onReceive(PersistentActor.scala:170)&lt;span class=&quot;error&quot;&gt;&amp;#91;180:com.typesafe.akka.persistence:2.4.17&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at akka.actor.UntypedActor$$anonfun$receive$1.applyOrElse(UntypedActor.scala:165)&lt;span class=&quot;error&quot;&gt;&amp;#91;173:com.typesafe.akka.actor:2.4.17&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at akka.actor.Actor$class.aroundReceive(Actor.scala:497)&lt;span class=&quot;error&quot;&gt;&amp;#91;173:com.typesafe.akka.actor:2.4.17&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at akka.persistence.UntypedPersistentActor.akka$persistence$Eventsourced$$super$aroundReceive(PersistentActor.scala:168)&lt;span class=&quot;error&quot;&gt;&amp;#91;180:com.typesafe.akka.persistence:2.4.17&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at akka.persistence.Eventsourced$$anon$1.stateReceive(Eventsourced.scala:664)&lt;span class=&quot;error&quot;&gt;&amp;#91;180:com.typesafe.akka.persistence:2.4.17&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at akka.persistence.Eventsourced$class.aroundReceive(Eventsourced.scala:183)&lt;span class=&quot;error&quot;&gt;&amp;#91;180:com.typesafe.akka.persistence:2.4.17&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at akka.persistence.UntypedPersistentActor.aroundReceive(PersistentActor.scala:168)&lt;span class=&quot;error&quot;&gt;&amp;#91;180:com.typesafe.akka.persistence:2.4.17&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at akka.actor.ActorCell.receiveMessage(ActorCell.scala:526)&lt;span class=&quot;error&quot;&gt;&amp;#91;173:com.typesafe.akka.actor:2.4.17&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at akka.actor.ActorCell.invoke(ActorCell.scala:495)&lt;span class=&quot;error&quot;&gt;&amp;#91;173:com.typesafe.akka.actor:2.4.17&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:257)&lt;span class=&quot;error&quot;&gt;&amp;#91;173:com.typesafe.akka.actor:2.4.17&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at akka.dispatch.Mailbox.run(Mailbox.scala:224)&lt;span class=&quot;error&quot;&gt;&amp;#91;173:com.typesafe.akka.actor:2.4.17&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at akka.dispatch.Mailbox.exec(Mailbox.scala:234)&lt;span class=&quot;error&quot;&gt;&amp;#91;173:com.typesafe.akka.actor:2.4.17&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)&lt;span class=&quot;error&quot;&gt;&amp;#91;169:org.scala-lang.scala-library:2.11.8.v20160304-115712-1706a37eb8&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)&lt;span class=&quot;error&quot;&gt;&amp;#91;169:org.scala-lang.scala-library:2.11.8.v20160304-115712-1706a37eb8&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)&lt;span class=&quot;error&quot;&gt;&amp;#91;169:org.scala-lang.scala-library:2.11.8.v20160304-115712-1706a37eb8&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)&lt;span class=&quot;error&quot;&gt;&amp;#91;169:org.scala-lang.scala-library:2.11.8.v20160304-115712-1706a37eb8&amp;#93;&lt;/span&gt;&lt;/p&gt;

&lt;p&gt;This wrecks the replay, hence we are probably missing some messages. A concurrent modification should not be occuring, as the queue should be protected by locks in this code path.&lt;/p&gt;</comment>
                            <comment id="52126" author="rovarga" created="Sun, 14 May 2017 18:49:38 +0000"  >&lt;p&gt;This affects prefix-based shards with RESTCONF&apos;s POST operation and requires introduction of a new message to the protocol, hence raising priority.&lt;/p&gt;

&lt;p&gt;Note this affects both creation and normal operations on those shards, as well as the data store with tell-based protocol.&lt;/p&gt;</comment>
                            <comment id="52127" author="rovarga" created="Sun, 14 May 2017 21:42:58 +0000"  >&lt;p&gt;&lt;a href=&quot;https://git.opendaylight.org/gerrit/57028&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://git.opendaylight.org/gerrit/57028&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="52128" author="rovarga" created="Tue, 16 May 2017 16:39:48 +0000"  >&lt;p&gt;Transmit queue fix: &lt;a href=&quot;https://git.opendaylight.org/gerrit/57152&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://git.opendaylight.org/gerrit/57152&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="52129" author="anipbu" created="Wed, 17 May 2017 18:18:03 +0000"  >&lt;p&gt;We are looking to build Carbon RC2 tomorrow 5/18 at 23:59 UTC time assuming there are no blocker bugs.  Is there an ETA for when a fix can be merged and this bug resolved for stable/carbon branch?&lt;/p&gt;</comment>
                            <comment id="52130" author="anipbu" created="Wed, 17 May 2017 23:03:46 +0000"  >&lt;p&gt;Robert, do we need to merge this patch for Carbon Blocker &lt;a href=&quot;https://jira.opendaylight.org/browse/CONTROLLER-1664&quot; title=&quot;C: OutOfOrderRequestException: Expecting request 0 (while create-prefix-shard)&quot; class=&quot;issue-link&quot; data-issue-key=&quot;CONTROLLER-1664&quot;&gt;&lt;del&gt;CONTROLLER-1664&lt;/del&gt;&lt;/a&gt;? &lt;a href=&quot;https://git.opendaylight.org/gerrit/#/c/57289/&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://git.opendaylight.org/gerrit/#/c/57289/&lt;/a&gt;&lt;/p&gt;

&lt;p&gt;Also, it appears multiple patches have been assigned to this Bug ID.  Do you have a list of all relevant patches that must be merged to fix &lt;a href=&quot;https://jira.opendaylight.org/browse/CONTROLLER-1664&quot; title=&quot;C: OutOfOrderRequestException: Expecting request 0 (while create-prefix-shard)&quot; class=&quot;issue-link&quot; data-issue-key=&quot;CONTROLLER-1664&quot;&gt;&lt;del&gt;CONTROLLER-1664&lt;/del&gt;&lt;/a&gt;?&lt;/p&gt;</comment>
                            <comment id="52131" author="rovarga" created="Thu, 18 May 2017 08:25:35 +0000"  >&lt;p&gt;An, I have split the formerly second patch into three for easier review:&lt;/p&gt;

&lt;p&gt;&lt;a href=&quot;https://git.opendaylight.org/gerrit/57289&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://git.opendaylight.org/gerrit/57289&lt;/a&gt;&lt;br/&gt;
&lt;a href=&quot;https://git.opendaylight.org/gerrit/57293&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://git.opendaylight.org/gerrit/57293&lt;/a&gt;&lt;br/&gt;
&lt;a href=&quot;https://git.opendaylight.org/gerrit/57028&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://git.opendaylight.org/gerrit/57028&lt;/a&gt;&lt;/p&gt;

&lt;p&gt;I have updated the spreadsheet with the patch IDs and added their sequences into notes for easy reference.&lt;/p&gt;</comment>
                            <comment id="52132" author="pgubka@cisco.com" created="Fri, 19 May 2017 18:06:29 +0000"  >&lt;p&gt;The exception is present in the logs, e.g. &lt;a href=&quot;https://logs.opendaylight.org/releng/jenkins092/controller-csit-3node-clustering-only-carbon/698/archives/odl2_karaf.log.gz&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://logs.opendaylight.org/releng/jenkins092/controller-csit-3node-clustering-only-carbon/698/archives/odl2_karaf.log.gz&lt;/a&gt;&lt;/p&gt;

&lt;p&gt;2017-05-19 09:41:13,386 | ERROR | lt-dispatcher-25 | DistributedShardedDOMDataTree    | 199 - org.opendaylight.controller.sal-distributed-datastore - 1.5.0.Carbon | Removal of shard DOMDataTreeIdentifier&lt;/p&gt;
{datastore=CONFIGURATION, root=/(tag:opendaylight.org,2017:controller:yang:lowlevel:target?revision=2017-02-15)id-ints}
&lt;p&gt; from configuration failed.&lt;br/&gt;
org.opendaylight.controller.cluster.access.commands.OutOfOrderRequestException: Expecting request 0&lt;/p&gt;

&lt;p&gt;But it was not propagated to robot via restcong responses. I havent found any in the robot log.&lt;br/&gt;
&lt;a href=&quot;https://logs.opendaylight.org/releng/jenkins092/controller-csit-3node-clustering-only-carbon/698/archives/log.html.gz&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://logs.opendaylight.org/releng/jenkins092/controller-csit-3node-clustering-only-carbon/698/archives/log.html.gz&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="52133" author="pgubka@cisco.com" created="Tue, 23 May 2017 11:29:08 +0000"  >&lt;p&gt;The presence of the OutOfOrderRequestException: Expecting request 0 is here:&lt;br/&gt;
&lt;a href=&quot;https://logs.opendaylight.org/sandbox/jenkins091/controller-csit-3node-clustering-only-carbon-2nd/6/archives/odl2_karaf.log.gz&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://logs.opendaylight.org/sandbox/jenkins091/controller-csit-3node-clustering-only-carbon-2nd/6/archives/odl2_karaf.log.gz&lt;/a&gt;&lt;/p&gt;

&lt;p&gt;Now it is not happening during the prefix shard creation, but during the prefix shard removal.&lt;br/&gt;
Test details:&lt;br/&gt;
1) Prefix shard created and produce-transactions started on shard leader node&lt;br/&gt;
2) Shard leader node isolated&lt;br/&gt;
3) produce-transactions rpc should have returned an error, but it did not&lt;br/&gt;
4) Isolated node rejoined&lt;br/&gt;
5) Produce-transactions finished with TimeoutException&lt;br/&gt;
&lt;a href=&quot;https://logs.opendaylight.org/sandbox/jenkins091/controller-csit-3node-clustering-only-carbon-2nd/6/archives/log.html.gz#s1-s2-t1-k2-k21-k4&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://logs.opendaylight.org/sandbox/jenkins091/controller-csit-3node-clustering-only-carbon-2nd/6/archives/log.html.gz#s1-s2-t1-k2-k21-k4&lt;/a&gt;&lt;br/&gt;
6) Prefix shards removal invoked (and on the node2 appeared OutOfOrderRequestException&lt;/p&gt;

&lt;p&gt;Logs have debug logging on for loggers org.opendaylight.controller.cluster.access.client&lt;br/&gt;
org.opendaylight.controller.cluster.databroker.actors.dds&lt;/p&gt;</comment>
                            <comment id="52134" author="rovarga" created="Tue, 23 May 2017 12:26:40 +0000"  >&lt;p&gt;Okay, the logs now shed some more light on this.&lt;/p&gt;

&lt;p&gt;There are two issues here:&lt;/p&gt;

&lt;p&gt;1) PrefixedShardConfigWriter is creating a snapshot, invoking an exist and not closing it. This causes the snapshot to exist for the duration of the transaction chain, across all reconnects, when it is sending only IncrementTransactionSequenceRequest.&lt;/p&gt;

&lt;p&gt;2) IncrementTransactionSequenceRequest does not have a record of who is originating it, i.e. whether it is a read-only snapshot or a transaction.&lt;/p&gt;

&lt;p&gt;Hence when the shard is being deleted, we are opening a read-write transaction and sending a request with correct sequence, but the transaction chain state is incorrect, because it saw only IncrementTransactionSequenceRequest, which allocated a transaction (not a snapshot) and that was never closed, leading to a splat:&lt;/p&gt;

&lt;p&gt;Caused by: java.lang.IllegalStateException: Transaction ReadWriteShardDataTreeTransaction{id=member-2-datastore-Shard-prefix-configuration-shard-fe-0-chn-1-txn-1-0, closed=false, snapshot=MutableDataTree [modification=NodeModification &lt;span class=&quot;error&quot;&gt;&amp;#91;identifier=(urn:ietf:params:xml:ns:netconf:base:1.0)data, modificationType=NONE, childModification={}&amp;#93;&lt;/span&gt;]} is open&lt;/p&gt;

&lt;p&gt;which leaves the backend still expecting request 0 when it receives the second request, which triggers the OutOfOrderRequestException.&lt;/p&gt;

&lt;p&gt;&lt;a href=&quot;https://git.opendaylight.org/gerrit/57681&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://git.opendaylight.org/gerrit/57681&lt;/a&gt; fixes PrefixedShardConfigWriter to clean up the read-only transaction, which will cause the first transaction to be closed/purged and hide the issue.&lt;/p&gt;

&lt;p&gt;The second fix involves fixing up IncrementTransactionSequenceRequest and will be posted soon.&lt;/p&gt;</comment>
                            <comment id="52135" author="rovarga" created="Tue, 23 May 2017 13:15:59 +0000"  >&lt;p&gt;&lt;a href=&quot;https://git.opendaylight.org/gerrit/57692&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://git.opendaylight.org/gerrit/57692&lt;/a&gt;&lt;/p&gt;</comment>
                    </comments>
                <issuelinks>
                            <issuelinktype id="10000">
                    <name>Blocks</name>
                                            <outwardlinks description="blocks">
                                        <issuelink>
            <issuekey id="26207">CONTROLLER-1653</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="26236">CONTROLLER-1682</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="26240">CONTROLLER-1686</issuekey>
        </issuelink>
                            </outwardlinks>
                                                                <inwardlinks description="is blocked by">
                                        <issuelink>
            <issuekey id="26249">CONTROLLER-1695</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                            <issuelinktype id="10002">
                    <name>Duplicate</name>
                                                                <inwardlinks description="is duplicated by">
                                        <issuelink>
            <issuekey id="26236">CONTROLLER-1682</issuekey>
        </issuelink>
            <issuelink>
            <issuekey id="26238">CONTROLLER-1684</issuekey>
        </issuelink>
                            </inwardlinks>
                                    </issuelinktype>
                    </issuelinks>
                <attachments>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                                            <customfield id="customfield_11400" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                        <customfield id="customfield_10208" key="com.atlassian.jira.plugin.system.customfieldtypes:textfield">
                        <customfieldname>External issue ID</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>8402</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10201" key="com.atlassian.jira.plugin.system.customfieldtypes:url">
                        <customfieldname>External issue URL</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[https://bugs.opendaylight.org/show_bug.cgi?id=8402]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10206" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Issue Type</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10300"><![CDATA[Bug]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_10204" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>ODL SR Target Milestone</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10359"><![CDATA[Carbon-RC2]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_10000" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>0|i02ryv:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                </customfields>
    </item>
</channel>
</rss>