<!-- 
RSS generated by JIRA (8.20.10#820010-sha1:ace47f9899e9ee25d7157d59aa17ab06aee30d3d) at Wed Feb 07 19:56:15 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>OpenDaylight JIRA</title>
    <link>https://jira.opendaylight.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>8.20.10</version>
        <build-number>820010</build-number>
        <build-date>22-06-2022</build-date>
    </build-info>


<item>
            <title>[CONTROLLER-1708] write-transactions does not respond (no evident reason)</title>
                <link>https://jira.opendaylight.org/browse/CONTROLLER-1708</link>
                <project id="10113" key="CONTROLLER">controller</project>
                    <description>&lt;p&gt;This is basically &lt;a href=&quot;https://jira.opendaylight.org/browse/CONTROLLER-1672&quot; title=&quot;Restconf slow to respond when the member is under load&quot; class=&quot;issue-link&quot; data-issue-key=&quot;CONTROLLER-1672&quot;&gt;&lt;del&gt;CONTROLLER-1672&lt;/del&gt;&lt;/a&gt; but that evolved into issues with datastore and jolokia reads; this is just a missing RPC response.&lt;br/&gt;
&lt;a href=&quot;https://jira.opendaylight.org/browse/CONTROLLER-1665&quot; title=&quot;C: write-transactions does not return if local leader moved&quot; class=&quot;issue-link&quot; data-issue-key=&quot;CONTROLLER-1665&quot;&gt;&lt;del&gt;CONTROLLER-1665&lt;/del&gt;&lt;/a&gt; is basically the same, it that was seen in different scenarios and is perhaps fixed by now, so not sure if this is a duplicate.&lt;/p&gt;

&lt;p&gt;This happened on Sandbox, when the original module-based leader member-1 was isolated for a brief time. The unresponsive member was member-2, which stayed a Follower, but it is not clear whether it was considering member-3 as a Leader.&lt;br/&gt;
Karaf.log &lt;span class=&quot;error&quot;&gt;&amp;#91;0&amp;#93;&lt;/span&gt; is huge, the test started at 17:52:27,543&lt;/p&gt;

&lt;p&gt;Writers on the other two members have reported &lt;span class=&quot;error&quot;&gt;&amp;#91;1&amp;#93;&lt;/span&gt; success.&lt;/p&gt;

&lt;p&gt;&lt;span class=&quot;error&quot;&gt;&amp;#91;0&amp;#93;&lt;/span&gt; &lt;a href=&quot;https://logs.opendaylight.org/sandbox/jenkins091/controller-csit-3node-clustering-only-carbon/6/odl2_karaf.log.gz&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://logs.opendaylight.org/sandbox/jenkins091/controller-csit-3node-clustering-only-carbon/6/odl2_karaf.log.gz&lt;/a&gt;&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;1&amp;#93;&lt;/span&gt; &lt;a href=&quot;https://logs.opendaylight.org/sandbox/jenkins091/controller-csit-3node-clustering-only-carbon/6/log.html.gz#s1-s10-t1-k2-k25-k1-k1&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://logs.opendaylight.org/sandbox/jenkins091/controller-csit-3node-clustering-only-carbon/6/log.html.gz#s1-s10-t1-k2-k25-k1-k1&lt;/a&gt;&lt;/p&gt;</description>
                <environment>&lt;p&gt;Operating System: All&lt;br/&gt;
Platform: All&lt;/p&gt;</environment>
        <key id="26262">CONTROLLER-1708</key>
            <summary>write-transactions does not respond (no evident reason)</summary>
                <type id="10104" iconUrl="https://jira.opendaylight.org/secure/viewavatar?size=xsmall&amp;avatarId=10303&amp;avatarType=issuetype">Bug</type>
                                                <status id="5" iconUrl="https://jira.opendaylight.org/images/icons/statuses/resolved.png" description="A resolution has been taken, and it is awaiting verification by reporter. From here issues are either reopened, or are closed.">Resolved</status>
                    <statusCategory id="3" key="done" colorName="green"/>
                                    <resolution id="10000">Done</resolution>
                                        <assignee username="rovarga">Robert Varga</assignee>
                                    <reporter username="vrpolak">Vratko Polak</reporter>
                        <labels>
                    </labels>
                <created>Wed, 7 Jun 2017 11:43:57 +0000</created>
                <updated>Tue, 25 Jul 2023 08:24:42 +0000</updated>
                            <resolved>Mon, 24 Jul 2017 15:13:05 +0000</resolved>
                                                                    <component>clustering</component>
                        <due></due>
                            <votes>0</votes>
                                    <watches>2</watches>
                                                                                                                <comments>
                            <comment id="52375" author="vrpolak" created="Fri, 9 Jun 2017 14:37:24 +0000"  >&lt;p&gt;This is now happening &lt;span class=&quot;error&quot;&gt;&amp;#91;2&amp;#93;&lt;/span&gt; on RelEng. The unresponsive member-2 is the one which stayed follower.&lt;br/&gt;
Still nothing obvious in karaf.log &lt;span class=&quot;error&quot;&gt;&amp;#91;3&amp;#93;&lt;/span&gt; between 06:26:02,391 and 06:35:27,863.&lt;/p&gt;

&lt;p&gt;&lt;span class=&quot;error&quot;&gt;&amp;#91;2&amp;#93;&lt;/span&gt; &lt;a href=&quot;https://logs.opendaylight.org/releng/jenkins092/controller-csit-3node-clustering-only-carbon/737/log.html.gz#s1-s28-t1-k2-k25-k1-k1&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://logs.opendaylight.org/releng/jenkins092/controller-csit-3node-clustering-only-carbon/737/log.html.gz#s1-s28-t1-k2-k25-k1-k1&lt;/a&gt;&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;3&amp;#93;&lt;/span&gt; &lt;a href=&quot;https://logs.opendaylight.org/releng/jenkins092/controller-csit-3node-clustering-only-carbon/737/odl2_karaf.log.gz&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://logs.opendaylight.org/releng/jenkins092/controller-csit-3node-clustering-only-carbon/737/odl2_karaf.log.gz&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="52376" author="vrpolak" created="Fri, 9 Jun 2017 14:41:26 +0000"  >&lt;p&gt;Note that this time Robot waits for: duration writer should work, plus three 125s timeouts plus 10 seconds just to be sure something is really wrong.&lt;/p&gt;</comment>
                            <comment id="52377" author="vrpolak" created="Thu, 15 Jun 2017 06:52:06 +0000"  >&lt;p&gt;This is still happening, does not matter if the isolation time is brief &lt;span class=&quot;error&quot;&gt;&amp;#91;4&amp;#93;&lt;/span&gt; or long &lt;span class=&quot;error&quot;&gt;&amp;#91;5&amp;#93;&lt;/span&gt;. Only the member which stays follower acts this way, and only for module-based shards (only tell-based protocol is tested).&lt;/p&gt;

&lt;p&gt;Prefix-based shard tests either pass, or run into &lt;a href=&quot;https://jira.opendaylight.org/browse/CONTROLLER-1687&quot; title=&quot;Transaction writer does not fail expected time after isolation&quot; class=&quot;issue-link&quot; data-issue-key=&quot;CONTROLLER-1687&quot;&gt;&lt;del&gt;CONTROLLER-1687&lt;/del&gt;&lt;/a&gt;.&lt;/p&gt;

&lt;p&gt;&lt;span class=&quot;error&quot;&gt;&amp;#91;4&amp;#93;&lt;/span&gt; &lt;a href=&quot;https://logs.opendaylight.org/releng/jenkins092/controller-csit-3node-clustering-only-carbon/749/log.html.gz#s1-s28-t1-k2-k25-k1-k2-k3&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://logs.opendaylight.org/releng/jenkins092/controller-csit-3node-clustering-only-carbon/749/log.html.gz#s1-s28-t1-k2-k25-k1-k2-k3&lt;/a&gt;&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;5&amp;#93;&lt;/span&gt; &lt;a href=&quot;https://logs.opendaylight.org/releng/jenkins092/controller-csit-3node-clustering-only-carbon/749/log.html.gz#s1-s28-t3-k2-k25-k1-k9-k2&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://logs.opendaylight.org/releng/jenkins092/controller-csit-3node-clustering-only-carbon/749/log.html.gz#s1-s28-t3-k2-k25-k1-k9-k2&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="52378" author="vrpolak" created="Tue, 27 Jun 2017 13:57:23 +0000"  >&lt;p&gt;Sandbox run &lt;span class=&quot;error&quot;&gt;&amp;#91;6&amp;#93;&lt;/span&gt; with verbose debug logs, codebase &lt;span class=&quot;error&quot;&gt;&amp;#91;7&amp;#93;&lt;/span&gt; with first two failures as they happen on releng.&lt;/p&gt;

&lt;p&gt;&lt;span class=&quot;error&quot;&gt;&amp;#91;6&amp;#93;&lt;/span&gt; &lt;a href=&quot;https://logs.opendaylight.org/sandbox/jenkins091/controller-csit-3node-clustering-li-only-carbon/5/&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://logs.opendaylight.org/sandbox/jenkins091/controller-csit-3node-clustering-li-only-carbon/5/&lt;/a&gt;&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;7&amp;#93;&lt;/span&gt; &lt;a href=&quot;https://git.opendaylight.org/gerrit/#/c/59527/1&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://git.opendaylight.org/gerrit/#/c/59527/1&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="52379" author="vrpolak" created="Fri, 30 Jun 2017 12:56:14 +0000"  >&lt;p&gt;This is still happening, but in the test (3 writers, previous Leader isolated longer than 2 minutes) where &lt;a href=&quot;https://jira.opendaylight.org/browse/CONTROLLER-1734&quot; title=&quot;RequestTimeoutException after ~250s after brief isolation&quot; class=&quot;issue-link&quot; data-issue-key=&quot;CONTROLLER-1734&quot;&gt;&lt;del&gt;CONTROLLER-1734&lt;/del&gt;&lt;/a&gt; appears too. Releng &lt;span class=&quot;error&quot;&gt;&amp;#91;8&amp;#93;&lt;/span&gt; and Sandbox &lt;span class=&quot;error&quot;&gt;&amp;#91;9&amp;#93;&lt;/span&gt; had different members (relative to leader placement) show this, so it would be better to fix &lt;a href=&quot;https://jira.opendaylight.org/browse/CONTROLLER-1734&quot; title=&quot;RequestTimeoutException after ~250s after brief isolation&quot; class=&quot;issue-link&quot; data-issue-key=&quot;CONTROLLER-1734&quot;&gt;&lt;del&gt;CONTROLLER-1734&lt;/del&gt;&lt;/a&gt; first. This may even be an extreme case of &lt;a href=&quot;https://jira.opendaylight.org/browse/CONTROLLER-1734&quot; title=&quot;RequestTimeoutException after ~250s after brief isolation&quot; class=&quot;issue-link&quot; data-issue-key=&quot;CONTROLLER-1734&quot;&gt;&lt;del&gt;CONTROLLER-1734&lt;/del&gt;&lt;/a&gt; when the RequestTimeout would came after very long time.&lt;/p&gt;

&lt;p&gt;&lt;span class=&quot;error&quot;&gt;&amp;#91;8&amp;#93;&lt;/span&gt; &lt;a href=&quot;https://logs.opendaylight.org/releng/jenkins092/controller-csit-3node-clustering-only-carbon/767/log.html.gz#s1-s28-t3-k2-k25-k1-k8&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://logs.opendaylight.org/releng/jenkins092/controller-csit-3node-clustering-only-carbon/767/log.html.gz#s1-s28-t3-k2-k25-k1-k8&lt;/a&gt;&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;9&amp;#93;&lt;/span&gt; &lt;a href=&quot;https://logs.opendaylight.org/sandbox/jenkins091/controller-csit-3node-clustering-li-only-carbon/17/log.html.gz#s1-s2-t3-k2-k25-k1-k8&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://logs.opendaylight.org/sandbox/jenkins091/controller-csit-3node-clustering-li-only-carbon/17/log.html.gz#s1-s2-t3-k2-k25-k1-k8&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="52380" author="vrpolak" created="Mon, 3 Jul 2017 13:23:40 +0000"  >&lt;p&gt;Recent Sandbox occurrence: &lt;span class=&quot;error&quot;&gt;&amp;#91;10&amp;#93;&lt;/span&gt;.&lt;br/&gt;
I mean member-1, other members exhibit &lt;a href=&quot;https://jira.opendaylight.org/browse/CONTROLLER-1734&quot; title=&quot;RequestTimeoutException after ~250s after brief isolation&quot; class=&quot;issue-link&quot; data-issue-key=&quot;CONTROLLER-1734&quot;&gt;&lt;del&gt;CONTROLLER-1734&lt;/del&gt;&lt;/a&gt; instead.&lt;br/&gt;
Currently that is the only test case showing this Bug.&lt;/p&gt;

&lt;p&gt;&lt;span class=&quot;error&quot;&gt;&amp;#91;10&amp;#93;&lt;/span&gt; &lt;a href=&quot;https://logs.opendaylight.org/sandbox/jenkins091/controller-csit-3node-clustering-li-only-carbon/1/log.html.gz#s1-s2-t3-k2-k25-k1-k8&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://logs.opendaylight.org/sandbox/jenkins091/controller-csit-3node-clustering-li-only-carbon/1/log.html.gz#s1-s2-t3-k2-k25-k1-k8&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="52381" author="vrpolak" created="Fri, 7 Jul 2017 11:14:44 +0000"  >&lt;p&gt;Based on one Sandbox run, code &lt;span class=&quot;error&quot;&gt;&amp;#91;11&amp;#93;&lt;/span&gt; may be fixing &lt;span class=&quot;error&quot;&gt;&amp;#91;12&amp;#93;&lt;/span&gt; this Bug.&lt;/p&gt;

&lt;p&gt;&lt;span class=&quot;error&quot;&gt;&amp;#91;11&amp;#93;&lt;/span&gt; &lt;a href=&quot;https://git.opendaylight.org/gerrit/#/c/60033/5&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://git.opendaylight.org/gerrit/#/c/60033/5&lt;/a&gt;&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;12&amp;#93;&lt;/span&gt; &lt;a href=&quot;https://logs.opendaylight.org/sandbox/jenkins091/controller-csit-3node-clustering-li-only-carbon/4/log.html.gz#s1-s2-t3-k2-k25-k1-k8&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://logs.opendaylight.org/sandbox/jenkins091/controller-csit-3node-clustering-li-only-carbon/4/log.html.gz#s1-s2-t3-k2-k25-k1-k8&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="52382" author="vrpolak" created="Mon, 10 Jul 2017 11:21:09 +0000"  >&lt;p&gt;&amp;gt; code &lt;span class=&quot;error&quot;&gt;&amp;#91;11&amp;#93;&lt;/span&gt; may be fixing &lt;span class=&quot;error&quot;&gt;&amp;#91;12&amp;#93;&lt;/span&gt; this Bug.&lt;/p&gt;

&lt;p&gt;No. This week Sandbox run on the same .zip shows &lt;span class=&quot;error&quot;&gt;&amp;#91;13&amp;#93;&lt;/span&gt; read timeout again.&lt;/p&gt;

&lt;p&gt;&lt;span class=&quot;error&quot;&gt;&amp;#91;13&amp;#93;&lt;/span&gt; &lt;a href=&quot;https://logs.opendaylight.org/sandbox/jenkins091/controller-csit-3node-clustering-li-only-carbon/1/log.html.gz#s1-s2-t3-k2-k25-k1-k8&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://logs.opendaylight.org/sandbox/jenkins091/controller-csit-3node-clustering-li-only-carbon/1/log.html.gz#s1-s2-t3-k2-k25-k1-k8&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="52383" author="vrpolak" created="Mon, 17 Jul 2017 14:40:28 +0000"  >&lt;p&gt;This week Sandbox (codebase &lt;span class=&quot;error&quot;&gt;&amp;#91;14&amp;#93;&lt;/span&gt;) fails &lt;span class=&quot;error&quot;&gt;&amp;#91;15&amp;#93;&lt;/span&gt;, huge log &lt;span class=&quot;error&quot;&gt;&amp;#91;16&amp;#93;&lt;/span&gt;.&lt;/p&gt;

&lt;p&gt;As far as I can see, every future completed successfully, no idea why writer has not returned success. Will retry with extended logs &lt;span class=&quot;error&quot;&gt;&amp;#91;17&amp;#93;&lt;/span&gt;.&lt;/p&gt;

&lt;p&gt;&lt;span class=&quot;error&quot;&gt;&amp;#91;14&amp;#93;&lt;/span&gt; &lt;a href=&quot;https://git.opendaylight.org/gerrit/#/c/59506/32&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://git.opendaylight.org/gerrit/#/c/59506/32&lt;/a&gt;&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;15&amp;#93;&lt;/span&gt; &lt;a href=&quot;https://logs.opendaylight.org/sandbox/jenkins091/controller-csit-3node-clustering-li-only-carbon/2/log.html.gz#s1-s2-t3-k2-k25-k1-k8&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://logs.opendaylight.org/sandbox/jenkins091/controller-csit-3node-clustering-li-only-carbon/2/log.html.gz#s1-s2-t3-k2-k25-k1-k8&lt;/a&gt;&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;16&amp;#93;&lt;/span&gt; &lt;a href=&quot;https://logs.opendaylight.org/sandbox/jenkins091/controller-csit-3node-clustering-li-only-carbon/2/odl2_karaf.log.gz&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://logs.opendaylight.org/sandbox/jenkins091/controller-csit-3node-clustering-li-only-carbon/2/odl2_karaf.log.gz&lt;/a&gt;&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;17&amp;#93;&lt;/span&gt; &lt;a href=&quot;https://git.opendaylight.org/gerrit/#/c/60187/2&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://git.opendaylight.org/gerrit/#/c/60187/2&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="52384" author="vrpolak" created="Mon, 17 Jul 2017 16:14:59 +0000"  >&lt;p&gt;&amp;gt; every future completed successfully&lt;/p&gt;

&lt;p&gt;That was a case when this Bug hits the member which stayed follower.&lt;/p&gt;

&lt;p&gt;In the next run &lt;span class=&quot;error&quot;&gt;&amp;#91;18&amp;#93;&lt;/span&gt; however, this bug happened on the old leader after rejoin (after long isolation).&lt;br/&gt;
We may have two different bugs with the same Robot symptom here.&lt;/p&gt;

&lt;p&gt;&lt;span class=&quot;error&quot;&gt;&amp;#91;18&amp;#93;&lt;/span&gt; &lt;a href=&quot;https://logs.opendaylight.org/sandbox/jenkins091/controller-csit-3node-clustering-li-only-carbon/4/&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://logs.opendaylight.org/sandbox/jenkins091/controller-csit-3node-clustering-li-only-carbon/4/&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="52385" author="vrpolak" created="Tue, 18 Jul 2017 10:44:20 +0000"  >&lt;p&gt;Sandbox keeps reproducing &lt;span class=&quot;error&quot;&gt;&amp;#91;19&amp;#93;&lt;/span&gt; the variant with old leader failure.&lt;br/&gt;
In the karaf.log &lt;span class=&quot;error&quot;&gt;&amp;#91;20&amp;#93;&lt;/span&gt; I see the second writer got stuck on line 147 &lt;span class=&quot;error&quot;&gt;&amp;#91;21&amp;#93;&lt;/span&gt; and also that ReconnectingClientConnection has never reconnected after rejoin.&lt;/p&gt;

&lt;p&gt;I will keep looking into this some more.&lt;/p&gt;

&lt;p&gt;&lt;span class=&quot;error&quot;&gt;&amp;#91;19&amp;#93;&lt;/span&gt; &lt;a href=&quot;https://logs.opendaylight.org/sandbox/jenkins091/controller-csit-3node-clustering-li-only-carbon/8/log.html.gz#s1-s2-t1-k2-k25-k1-k8&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://logs.opendaylight.org/sandbox/jenkins091/controller-csit-3node-clustering-li-only-carbon/8/log.html.gz#s1-s2-t1-k2-k25-k1-k8&lt;/a&gt;&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;20&amp;#93;&lt;/span&gt; &lt;a href=&quot;https://logs.opendaylight.org/sandbox/jenkins091/controller-csit-3node-clustering-li-only-carbon/8/odl3_karaf.log.gz&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://logs.opendaylight.org/sandbox/jenkins091/controller-csit-3node-clustering-li-only-carbon/8/odl3_karaf.log.gz&lt;/a&gt;&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;21&amp;#93;&lt;/span&gt; &lt;a href=&quot;https://git.opendaylight.org/gerrit/#/c/60187/5/opendaylight/md-sal/samples/clustering-test-app/provider/src/main/java/org/opendaylight/controller/clustering/it/provider/impl/WriteTransactionsHandler.java@146&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://git.opendaylight.org/gerrit/#/c/60187/5/opendaylight/md-sal/samples/clustering-test-app/provider/src/main/java/org/opendaylight/controller/clustering/it/provider/impl/WriteTransactionsHandler.java@146&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="52386" author="vrpolak" created="Thu, 20 Jul 2017 11:58:07 +0000"  >&lt;p&gt;&amp;gt; ReconnectingClientConnection has never reconnected after rejoin.&lt;/p&gt;

&lt;p&gt;Looks like in lockedCheckTimeout we are completing (failing) connection entries, which leads to purge request entering the queue and sleeping the thread.&lt;/p&gt;

&lt;p&gt;Segment of karaf.log &lt;span class=&quot;error&quot;&gt;&amp;#91;22&amp;#93;&lt;/span&gt;:&lt;/p&gt;

&lt;p&gt;2017-07-20 10:38:35,854 | DEBUG | lt-dispatcher-25 | AbstractClientConnection         | 197 - org.opendaylight.control&lt;br/&gt;
ler.cds-access-client - 1.1.2.SNAPSHOT | member-1-frontend-datastore-config: Connection ReconnectingClientConnection{c&lt;br/&gt;
lient=ClientIdentifier&lt;/p&gt;
{frontend=member-1-frontend-datastore-config, generation=0}, cookie=0, backend=ShardBackendInfo{&lt;br/&gt;
actor=Actor&lt;a href=&quot;#879061049&quot; target=&quot;_blank&quot; rel=&quot;noopener&quot;&gt;akka://opendaylight-cluster-data/user/shardmanager-config/member-1-shard-default-config#879061049&lt;/a&gt;, sessio&lt;br/&gt;
nId=0, version=BORON, maxMessages=1000, cookie=0, shard=default, dataTree=present}} timed out entry TransmittedConnect&lt;br/&gt;
ionEntry{request=CommitLocalTransactionRequest{target=member-1-datastore-config-fe-0-chn-2-txn-4402-0, sequence=0, rep
lyTo=Actor[akka://opendaylight-cluster-data/user/$a#-451310230], coordinated=false}, enqueuedTicks=443234413299, sessi&lt;br/&gt;
onId=0, txSequence=7862}&lt;br/&gt;
2017-07-20 10:38:35,854 | WARN  | lt-dispatcher-25 | ConcurrentDOMDataBroker          | 199 - org.opendaylight.controller.sal-distributed-datastore - 1.5.2.SNAPSHOT | Tx: DOM-CHAIN-1-4402 Error during phase CAN_COMMIT, starting Abort&lt;br/&gt;
org.opendaylight.controller.cluster.access.client.RequestTimeoutException: Timed out after 120.022340136seconds&lt;br/&gt;
        at org.opendaylight.controller.cluster.access.client.AbstractClientConnection.lockedCheckTimeout(AbstractClientConnection.java:390)&lt;span class=&quot;error&quot;&gt;&amp;#91;197:org.opendaylight.controller.cds-access-client:1.1.2.SNAPSHOT&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at org.opendaylight.controller.cluster.access.client.AbstractClientConnection.runTimer(AbstractClientConnection.java:321)&lt;span class=&quot;error&quot;&gt;&amp;#91;197:org.opendaylight.controller.cds-access-client:1.1.2.SNAPSHOT&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at org.opendaylight.controller.cluster.access.client.ClientActorBehavior.onReceiveCommand(ClientActorBehavior.java:123)&lt;span class=&quot;error&quot;&gt;&amp;#91;197:org.opendaylight.controller.cds-access-client:1.1.2.SNAPSHOT&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at org.opendaylight.controller.cluster.access.client.ClientActorBehavior.onReceiveCommand(ClientActorBehavior.java:46)&lt;span class=&quot;error&quot;&gt;&amp;#91;197:org.opendaylight.controller.cds-access-client:1.1.2.SNAPSHOT&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at org.opendaylight.controller.cluster.access.client.AbstractClientActor.onReceiveCommand(AbstractClientActor.java:60)&lt;span class=&quot;error&quot;&gt;&amp;#91;197:org.opendaylight.controller.cds-access-client:1.1.2.SNAPSHOT&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at akka.persistence.UntypedPersistentActor.onReceive(PersistentActor.scala:170)&lt;span class=&quot;error&quot;&gt;&amp;#91;180:com.typesafe.akka.persistence:2.4.18&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at akka.actor.UntypedActor$$anonfun$receive$1.applyOrElse(UntypedActor.scala:165)&lt;span class=&quot;error&quot;&gt;&amp;#91;173:com.typesafe.akka.actor:2.4.18&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at akka.actor.Actor$class.aroundReceive(Actor.scala:502)&lt;span class=&quot;error&quot;&gt;&amp;#91;173:com.typesafe.akka.actor:2.4.18&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at akka.persistence.UntypedPersistentActor.akka$persistence$Eventsourced$$super$aroundReceive(PersistentActor.scala:168)&lt;span class=&quot;error&quot;&gt;&amp;#91;180:com.typesafe.akka.persistence:2.4.18&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at akka.persistence.Eventsourced$$anon$1.stateReceive(Eventsourced.scala:727)&lt;span class=&quot;error&quot;&gt;&amp;#91;180:com.typesafe.akka.persistence:2.4.18&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at akka.persistence.Eventsourced$class.aroundReceive(Eventsourced.scala:183)&lt;span class=&quot;error&quot;&gt;&amp;#91;180:com.typesafe.akka.persistence:2.4.18&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at akka.persistence.UntypedPersistentActor.aroundReceive(PersistentActor.scala:168)&lt;span class=&quot;error&quot;&gt;&amp;#91;180:com.typesafe.akka.persistence:2.4.18&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at akka.actor.ActorCell.receiveMessage(ActorCell.scala:526)&lt;span class=&quot;error&quot;&gt;&amp;#91;173:com.typesafe.akka.actor:2.4.18&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at akka.actor.ActorCell.invoke(ActorCell.scala:495)&lt;span class=&quot;error&quot;&gt;&amp;#91;173:com.typesafe.akka.actor:2.4.18&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:257)&lt;span class=&quot;error&quot;&gt;&amp;#91;173:com.typesafe.akka.actor:2.4.18&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at akka.dispatch.Mailbox.run(Mailbox.scala:224)&lt;span class=&quot;error&quot;&gt;&amp;#91;173:com.typesafe.akka.actor:2.4.18&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at akka.dispatch.Mailbox.exec(Mailbox.scala:234)&lt;span class=&quot;error&quot;&gt;&amp;#91;173:com.typesafe.akka.actor:2.4.18&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)&lt;span class=&quot;error&quot;&gt;&amp;#91;169:org.scala-lang.scala-library:2.11.11.v20170413-090219-8a413ba7cc&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)&lt;span class=&quot;error&quot;&gt;&amp;#91;169:org.scala-lang.scala-library:2.11.11.v20170413-090219-8a413ba7cc&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)&lt;span class=&quot;error&quot;&gt;&amp;#91;169:org.scala-lang.scala-library:2.11.11.v20170413-090219-8a413ba7cc&amp;#93;&lt;/span&gt;&lt;br/&gt;
        at scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)&lt;span class=&quot;error&quot;&gt;&amp;#91;169:org.scala-lang.scala-library:2.11.11.v20170413-090219-8a413ba7cc&amp;#93;&lt;/span&gt;&lt;br/&gt;
2017-07-20 10:38:35,855 | DEBUG | lt-dispatcher-25 | AbstractProxyTransaction         | 199 - org.opendaylight.controller.sal-distributed-datastore - 1.5.2.SNAPSHOT | Transaction LocalReadWriteProxyTransaction{identifier=member-1-datastore-config-fe-0-chn-2-txn-4402-0, state=FLUSHED} directCommit completed&lt;br/&gt;
2017-07-20 10:38:35,855 | DEBUG | lt-dispatcher-25 | AbstractProxyTransaction         | 199 - org.opendaylight.controller.sal-distributed-datastore - 1.5.2.SNAPSHOT | LocalReadWriteProxyTransaction{identifier=member-1-datastore-config-fe-0-chn-2-txn-4402-0, state=FLUSHED}: initiating purge&lt;br/&gt;
2017-07-20 10:38:35,855 | DEBUG | lt-dispatcher-25 | AbstractProxyTransaction         | 199 - org.opendaylight.controller.sal-distributed-datastore - 1.5.2.SNAPSHOT | Transaction LocalReadWriteProxyTransaction{identifier=member-1-datastore-config-fe-0-chn-2-txn-4402-0, state=DONE} allocated sequence 1&lt;br/&gt;
2017-07-20 10:38:35,855 | DEBUG | lt-dispatcher-25 | AbstractProxyTransaction         | 199 - org.opendaylight.controller.sal-distributed-datastore - 1.5.2.SNAPSHOT | Transaction proxy LocalReadWriteProxyTransaction{identifier=member-1-datastore-config-fe-0-chn-2-txn-4402-0, state=DONE} enqueing request TransactionPurgeRequest{target=member-1-datastore-config-fe-0-chn-2-txn-4402-0, sequence=1, replyTo=Actor[akka://opendaylight-cluster-data/user/$a#-451310230]} callback org.opendaylight.controller.cluster.databroker.actors.dds.AbstractProxyTransaction$$Lambda$152/951909138@6759f4d3&lt;br/&gt;
2017-07-20 10:38:35,855 | DEBUG | lt-dispatcher-25 | AbstractClientConnection         | 197 - org.opendaylight.controller.cds-access-client - 1.1.2.SNAPSHOT | member-1-frontend-datastore-config: connection ConnectedClientConnection{client=ClientIdentifier{frontend=member-1-frontend-datastore-config, generation=0}
&lt;p&gt;, cookie=0, backend=ShardBackendInfo{actor=Actor&lt;a href=&quot;#879061049&quot; target=&quot;_blank&quot; rel=&quot;noopener&quot;&gt;akka://opendaylight-cluster-data/user/shardmanager-config/member-1-shard-default-config#879061049&lt;/a&gt;, sessionId=0, version=BORON, maxMessages=1000, cookie=0, shard=default, dataTree=present}} has a successor, not scheduling timer&lt;br/&gt;
2017-07-20 10:38:35,855 | TRACE | lt-dispatcher-25 | TransmitQueue                    | 197 - org.opendaylight.controller.cds-access-client - 1.1.2.SNAPSHOT | Queue is at capacity, delayed sending of request TransactionPurgeRequest&lt;/p&gt;
{target=member-1-datastore-config-fe-0-chn-2-txn-4402-0, sequence=1, replyTo=Actor[akka://opendaylight-cluster-data/user/$a#-451310230]}
&lt;p&gt;2017-07-20 10:38:35,855 | DEBUG | lt-dispatcher-25 | AbstractClientConnection         | 197 - org.opendaylight.controller.cds-access-client - 1.1.2.SNAPSHOT | member-1-frontend-datastore-config: Sleeping for 498ms on connection ReconnectingClientConnection{client=ClientIdentifier&lt;/p&gt;
{frontend=member-1-frontend-datastore-config, generation=0}
&lt;p&gt;, cookie=0, backend=ShardBackendInfo{actor=Actor&lt;a href=&quot;#879061049&quot; target=&quot;_blank&quot; rel=&quot;noopener&quot;&gt;akka://opendaylight-cluster-data/user/shardmanager-config/member-1-shard-default-config#879061049&lt;/a&gt;, sessionId=0, version=BORON, maxMessages=1000, cookie=0, shard=default, dataTree=present}}&lt;/p&gt;

&lt;p&gt;&lt;span class=&quot;error&quot;&gt;&amp;#91;22&amp;#93;&lt;/span&gt; &lt;a href=&quot;https://logs.opendaylight.org/sandbox/jenkins091/controller-csit-3node-clustering-li-only-carbon/13/odl1_karaf.log.gz&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://logs.opendaylight.org/sandbox/jenkins091/controller-csit-3node-clustering-li-only-carbon/13/odl1_karaf.log.gz&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="52387" author="rovarga" created="Thu, 20 Jul 2017 12:28:53 +0000"  >&lt;p&gt;The problem now is the TransmitQueue:233, which switches from enqueue to send mechanics. This needs to be investigated more.&lt;/p&gt;</comment>
                            <comment id="52388" author="rovarga" created="Thu, 20 Jul 2017 15:55:37 +0000"  >&lt;p&gt;&lt;a href=&quot;https://git.opendaylight.org/gerrit/60614&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://git.opendaylight.org/gerrit/60614&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="52389" author="vrpolak" created="Fri, 21 Jul 2017 11:22:41 +0000"  >&lt;p&gt;&amp;gt; &lt;a href=&quot;https://git.opendaylight.org/gerrit/60614&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://git.opendaylight.org/gerrit/60614&lt;/a&gt;&lt;/p&gt;

&lt;p&gt;Merged, but the Bug is still present. Big karaf.log &lt;span class=&quot;error&quot;&gt;&amp;#91;24&amp;#93;&lt;/span&gt; with additional &lt;span class=&quot;error&quot;&gt;&amp;#91;25&amp;#93;&lt;/span&gt; logs.&lt;/p&gt;

&lt;p&gt;The scenario starts with transaction-writers on each node. Then Leader is isolated for more than 120 seconds, its writer fails as expected. Then the member is rejoined and a new writer is started on that member. That is the writer which never returns any RPC result.&lt;/p&gt;

&lt;p&gt;Previously that was because it got stuck in &quot;write only the top list&quot; step (possibly because ReconnectingClientConnection never connected after the rejoin). More details after I take a look into the log.&lt;/p&gt;

&lt;p&gt;&lt;span class=&quot;error&quot;&gt;&amp;#91;24&amp;#93;&lt;/span&gt; &lt;a href=&quot;https://logs.opendaylight.org/sandbox/jenkins091/controller-csit-3node-clustering-li-only-carbon/16/odl1_karaf.log.gz&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://logs.opendaylight.org/sandbox/jenkins091/controller-csit-3node-clustering-li-only-carbon/16/odl1_karaf.log.gz&lt;/a&gt;&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;25&amp;#93;&lt;/span&gt; &lt;a href=&quot;https://git.opendaylight.org/gerrit/#/c/60187/9&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://git.opendaylight.org/gerrit/#/c/60187/9&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="52390" author="vrpolak" created="Fri, 21 Jul 2017 13:31:36 +0000"  >&lt;p&gt;&amp;gt; More details after I take a look into the log.&lt;/p&gt;

&lt;p&gt;&amp;gt;&amp;gt; ReconnectingClientConnection never connected after the rejoin&lt;/p&gt;

&lt;p&gt;Still happens.&lt;/p&gt;

&lt;p&gt;Contrary to previous attempt at 11:02:32,894, the attempt at 11:02:53,934 never failed (nor resolved the new backend), so ReconnectingClientConnection has never been replaced by ConnectedClientConnection after rejoin (started at 11:03:16,111).&lt;/p&gt;

&lt;p&gt;2017-07-21 11:02:53,934 | DEBUG | lt-dispatcher-26 | ClientActorBehavior              | 197 - org.opendaylight.control&lt;br/&gt;
ler.cds-access-client - 1.1.2.SNAPSHOT | member-1-frontend-datastore-config: resolving shard 0 connection Reconnecting&lt;br/&gt;
ClientConnection{client=ClientIdentifier&lt;/p&gt;
{frontend=member-1-frontend-datastore-config, generation=0}
&lt;p&gt;, cookie=0, backend&lt;br/&gt;
=ShardBackendInfo{actor=Actor[akka://opendaylight-cluster-data/user/shardmanager-config/member-1-shard-default-config#&lt;br/&gt;
-208338531], sessionId=0, version=BORON, maxMessages=1000, cookie=0, shard=default, dataTree=present}}&lt;/p&gt;

&lt;p&gt;&amp;gt;&amp;gt; it got stuck in &quot;write only the top list&quot; step&lt;/p&gt;

&lt;p&gt;This time it got stuck after &quot;Submitting list merge&quot;.&lt;/p&gt;

&lt;p&gt;2017-07-21 11:03:29,669 | TRACE | qtp608302457-332 | WriteTransactionsHandler         | 257 - org.opendaylight.controller.samples.clustering-it-provider - 1.5.2.SNAPSHOT | Submitting list merge.&lt;/p&gt;

&lt;p&gt;Currently I have no idea why any of this happens, I will experiment some more.&lt;/p&gt;</comment>
                            <comment id="52391" author="vrpolak" created="Fri, 21 Jul 2017 15:36:11 +0000"  >&lt;p&gt;I have partially good news.&lt;/p&gt;

&lt;p&gt;The &lt;a href=&quot;https://jira.opendaylight.org/browse/CONTROLLER-1687&quot; title=&quot;Transaction writer does not fail expected time after isolation&quot; class=&quot;issue-link&quot; data-issue-key=&quot;CONTROLLER-1687&quot;&gt;&lt;del&gt;CONTROLLER-1687&lt;/del&gt;&lt;/a&gt; fix &lt;span class=&quot;error&quot;&gt;&amp;#91;26&amp;#93;&lt;/span&gt; makes this bug disappear as well.&lt;br/&gt;
The bad part of the news is that a client application which handles callbacks from a single thread shared with writes (incurring sleeps if backend is not fast enough) can cause this bug.&lt;/p&gt;

&lt;p&gt;Should a separate bug be open if the aforementioned fix gets merged?&lt;/p&gt;

&lt;p&gt;&lt;span class=&quot;error&quot;&gt;&amp;#91;26&amp;#93;&lt;/span&gt; &lt;a href=&quot;https://git.opendaylight.org/gerrit/#/c/60606/7&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://git.opendaylight.org/gerrit/#/c/60606/7&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="52392" author="vrpolak" created="Mon, 24 Jul 2017 10:02:49 +0000"  >&lt;p&gt;&amp;gt; client application which handles callbacks from a single thread&lt;/p&gt;

&lt;p&gt;That should not make a difference.&lt;/p&gt;

&lt;p&gt;The test passed &lt;span class=&quot;error&quot;&gt;&amp;#91;27&amp;#93;&lt;/span&gt; today on releng, but the test was not always failing before and I believe this Bug is not fixed yet. Will create more runs on Sandbox.&lt;/p&gt;

&lt;p&gt;&lt;span class=&quot;error&quot;&gt;&amp;#91;27&amp;#93;&lt;/span&gt; &lt;a href=&quot;https://logs.opendaylight.org/releng/jenkins092/controller-csit-3node-clustering-only-carbon/791/log.html.gz#s1-s28-t3-k2-k25-k1-k8&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://logs.opendaylight.org/releng/jenkins092/controller-csit-3node-clustering-only-carbon/791/log.html.gz#s1-s28-t3-k2-k25-k1-k8&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="52393" author="vrpolak" created="Mon, 24 Jul 2017 15:13:05 +0000"  >&lt;p&gt;&amp;gt;&amp;gt;&amp;gt; &lt;a href=&quot;https://git.opendaylight.org/gerrit/60614&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://git.opendaylight.org/gerrit/60614&lt;/a&gt;&lt;br/&gt;
&amp;gt;&amp;gt; &lt;br/&gt;
&amp;gt;&amp;gt; Merged, but the Bug is still present.&lt;/p&gt;

&lt;p&gt;&amp;gt; I believe this Bug is not fixed yet.&lt;/p&gt;

&lt;p&gt;Under the weight of Sandbox passes, now I believe that last week I was testing some different build by mistake, and &lt;span class=&quot;error&quot;&gt;&amp;#91;28&amp;#93;&lt;/span&gt; has fixed this Bug indeed.&lt;/p&gt;

&lt;p&gt;&lt;span class=&quot;error&quot;&gt;&amp;#91;28&amp;#93;&lt;/span&gt; &lt;a href=&quot;https://git.opendaylight.org/gerrit/#/c/60614/2&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://git.opendaylight.org/gerrit/#/c/60614/2&lt;/a&gt;&lt;/p&gt;</comment>
                    </comments>
                    <attachments>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                                            <customfield id="customfield_11400" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                        <customfield id="customfield_10208" key="com.atlassian.jira.plugin.system.customfieldtypes:textfield">
                        <customfieldname>External issue ID</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>8619</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10201" key="com.atlassian.jira.plugin.system.customfieldtypes:url">
                        <customfieldname>External issue URL</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[https://bugs.opendaylight.org/show_bug.cgi?id=8619]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10206" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Issue Type</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10300"><![CDATA[Bug]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                        <customfield id="customfield_10204" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>ODL SR Target Milestone</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10337"><![CDATA[Carbon-SR2]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                    <customfield id="customfield_10000" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>0|i02s8n:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                </customfields>
    </item>
</channel>
</rss>