<!-- 
RSS generated by JIRA (8.20.10#820010-sha1:ace47f9899e9ee25d7157d59aa17ab06aee30d3d) at Wed Feb 07 20:22:44 UTC 2024

It is possible to restrict the fields that are returned in this document by specifying the 'field' parameter in your request.
For example, to request only the issue key and summary append 'field=key&field=summary' to the URL of your request.
-->
<rss version="0.92" >
<channel>
    <title>OpenDaylight JIRA</title>
    <link>https://jira.opendaylight.org</link>
    <description>This file is an XML representation of an issue</description>
    <language>en-us</language>    <build-info>
        <version>8.20.10</version>
        <build-number>820010</build-number>
        <build-date>22-06-2022</build-date>
    </build-info>


<item>
            <title>[NETVIRT-896] CSIT Sporadic failures - tempest.api.network precommit_delete failure on security group delete</title>
                <link>https://jira.opendaylight.org/browse/NETVIRT-896</link>
                <project id="10144" key="NETVIRT">netvirt</project>
                    <description>&lt;p&gt;this csit job:&lt;/p&gt;

&lt;p&gt;&lt;a href=&quot;https://logs.opendaylight.org/releng/jenkins092/netvirt-csit-1node-openstack-pike-upstream-stateful-nitrogen/3/log.html.gz&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://logs.opendaylight.org/releng/jenkins092/netvirt-csit-1node-openstack-pike-upstream-stateful-nitrogen/3/log.html.gz&lt;/a&gt;&lt;/p&gt;

&lt;p&gt;tempest output snippet:&lt;/p&gt;

{2}

&lt;p&gt;tearDownClass (tempest.api.network.test_extensions.ExtensionsTestJSON) &lt;span class=&quot;error&quot;&gt;&amp;#91;0.000000s&amp;#93;&lt;/span&gt; ... FAILED&lt;/p&gt;

&lt;p&gt;Captured traceback:&lt;br/&gt;
 ~~~~~~~~~~~~~~~~~~~&lt;br/&gt;
 Traceback (most recent call last):&lt;/p&gt;
&lt;div class=&quot;preformatted panel&quot; style=&quot;border-width: 1px;&quot;&gt;&lt;div class=&quot;preformattedContent panelContent&quot;&gt;
&lt;pre&gt;File &quot;tempest/test.py&quot;, line 190, in tearDownClass
 six.reraise(etype, value, trace)
 File &quot;tempest/test.py&quot;, line 173, in tearDownClass
 teardown()
 File &quot;tempest/test.py&quot;, line 477, in clear_credentials
 cls._creds_provider.clear_creds()
 File &quot;tempest/lib/common/dynamic_creds.py&quot;, line 459, in clear_creds
 self._cleanup_default_secgroup(creds.tenant_id)
 File &quot;tempest/lib/common/dynamic_creds.py&quot;, line 406, in _cleanup_default_secgroup
 nsg_client.delete_security_group(secgroup[&apos;id&apos;])
 File &quot;tempest/lib/services/network/security_groups_client.py&quot;, line 59, in delete_security_group
 return self.delete_resource(uri)
 File &quot;tempest/lib/services/network/base.py&quot;, line 41, in delete_resource
 resp, body = self.delete(req_uri)
 File &quot;tempest/lib/common/rest_client.py&quot;, line 301, in delete
 return self.request(&apos;DELETE&apos;, url, extra_headers, headers, body)
 File &quot;tempest/lib/common/rest_client.py&quot;, line 659, in request
 self._error_checker(resp, resp_body)
 File &quot;tempest/lib/common/rest_client.py&quot;, line 780, in _error_checker
 raise exceptions.Conflict(resp_body, resp=resp)
 tempest.lib.exceptions.Conflict: An object with that identifier already exists
 Details:
{u&apos;message&apos;: u&apos;Security Group 462df320-f024-4810-85a7-5b1195ee3602 cannot perform precommit_delete due to Callback networking_odl.common.callback.OdlSecurityGroupsHandler.sg_callback_precommit--9223372036854618144 failed with &quot;argument of type \&apos;NoneType\&apos; is not iterable&quot;.&apos;, u&apos;type&apos;: u&apos;SecurityGroupInUse&apos;, u&apos;detail&apos;: u&apos;&apos;}
&lt;/pre&gt;
&lt;/div&gt;&lt;/div&gt;
&lt;p&gt;&#160;&lt;/p&gt;</description>
                <environment>&lt;p&gt;Operating System: All&lt;br/&gt;
Platform: All&lt;/p&gt;</environment>
        <key id="20817">NETVIRT-896</key>
            <summary>CSIT Sporadic failures - tempest.api.network precommit_delete failure on security group delete</summary>
                <type id="10104" iconUrl="https://jira.opendaylight.org/secure/viewavatar?size=xsmall&amp;avatarId=10303&amp;avatarType=issuetype">Bug</type>
                                            <priority id="3" iconUrl="https://jira.opendaylight.org/images/icons/priorities/major.svg">Medium</priority>
                        <status id="5" iconUrl="https://jira.opendaylight.org/images/icons/statuses/resolved.png" description="A resolution has been taken, and it is awaiting verification by reporter. From here issues are either reopened, or are closed.">Resolved</status>
                    <statusCategory id="3" key="done" colorName="green"/>
                                    <resolution id="10000">Done</resolution>
                                        <assignee username="yamahata">Isaku Yamahata</assignee>
                                    <reporter username="jluhrsen">Jamo Luhrsen</reporter>
                        <labels>
                            <label>csit:failures</label>
                    </labels>
                <created>Fri, 8 Sep 2017 22:52:07 +0000</created>
                <updated>Thu, 3 May 2018 14:37:15 +0000</updated>
                            <resolved>Thu, 5 Apr 2018 21:20:15 +0000</resolved>
                                    <version>Nitrogen</version>
                    <version>Oxygen</version>
                                    <fixVersion>Carbon-SR3</fixVersion>
                                    <component>General</component>
                        <due></due>
                            <votes>0</votes>
                                    <watches>9</watches>
                                                                                                                <comments>
                            <comment id="38404" author="jluhrsen" created="Mon, 18 Sep 2017 23:44:39 +0000"  >&lt;p&gt;&lt;a href=&quot;https://logs.opendaylight.org/releng/jenkins092/netvirt-csit-1node-openstack-pike-upstream-stateful-carbon/16/log.html.gz#s1-s2-s1&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://logs.opendaylight.org/releng/jenkins092/netvirt-csit-1node-openstack-pike-upstream-stateful-carbon/16/log.html.gz#s1-s2-s1&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="38405" author="jluhrsen" created="Wed, 27 Sep 2017 18:50:55 +0000"  >&lt;p&gt;another example:&lt;/p&gt;

&lt;p&gt;snippet from the neutron log &lt;span class=&quot;error&quot;&gt;&amp;#91;0&amp;#93;&lt;/span&gt;:&lt;/p&gt;

&lt;p&gt;2017-09-27 14:36:11.991 15977 DEBUG networking_odl.journal.journal &lt;span class=&quot;error&quot;&gt;&amp;#91;req-feaf99a4-cde1-4f5e-8f45-907690caa0a2 - -&amp;#93;&lt;/span&gt; Start processing journal entries sync_pending_entries /opt/stack/networking-odl/networking_odl/journal/journal.py:221&lt;br/&gt;
2017-09-27 14:36:12.010 15977 DEBUG neutron.api.rpc.handlers.resources_rpc &lt;span class=&quot;error&quot;&gt;&amp;#91;req-96f23e4d-1b20-47bc-8c27-e8badfa60e43 tempest-RoutersTest-1789903320 tempest-RoutersTest-1789903320&amp;#93;&lt;/span&gt; neutron.api.rpc.handlers.resources_rpc.ResourcesPushRpcApi method push called with arguments (&amp;lt;neutron_lib.context.Context object at 0xa9cacd0&amp;gt;, [Port(admin_state_up=True,allowed_address_pairs=[],binding=PortBinding,binding_levels=[],created_at=2017-09-27T14:36:07Z,data_plane_status=&amp;lt;?&amp;gt;,description=&apos;&apos;,device_id=&apos;4f943f20-2fb8-4a03-b4fb-a67955677cde&apos;,device_owner=&apos;network:router_interface&apos;,dhcp_options=[],distributed_binding=None,dns=None,fixed_ips=&lt;span class=&quot;error&quot;&gt;&amp;#91;IPAllocation&amp;#93;&lt;/span&gt;,id=3d3cacc1-f87b-4bbc-96af-cc612de0f0be,mac_address=fa:16:3e:f3:40:5f,name=&apos;&apos;,network_id=79f4139a-7bbb-4577-8905-c3202156d36d,project_id=&apos;e59173ac2305426f9a849edff8b58b0a&apos;,qos_policy_id=None,revision_number=4,security=PortSecurity(3d3cacc1-f87b-4bbc-96af-cc612de0f0be),security_group_ids=set([]),status=&apos;DOWN&apos;,updated_at=2017-09-27T14:36:10Z)], &apos;updated&apos;) {} wrapper /usr/lib/python2.7/site-packages/oslo_log/helpers.py:66&lt;br/&gt;
2017-09-27 14:36:12.015 15977 DEBUG neutron_lib.callbacks.manager &lt;span class=&quot;error&quot;&gt;&amp;#91;req-112bdc5c-9147-4ee8-9959-63083496c140 admin admin&amp;#93;&lt;/span&gt; Callback networking_odl.common.callback.OdlSecurityGroupsHandler.sg_callback_precommit--9223372036846803568 raised argument of type &apos;NoneType&apos; is not iterable _notify_loop /usr/lib/python2.7/site-packages/neutron_lib/callbacks/manager.py:184&lt;br/&gt;
2017-09-27 14:36:12.020 15976 DEBUG neutron.quota.resource &lt;span class=&quot;error&quot;&gt;&amp;#91;req-db17e372-3bb5-4f33-84f4-f0f9ec2390d2 tempest-SecGroupIPv6Test-1702529477 tempest-SecGroupIPv6Test-1702529477&amp;#93;&lt;/span&gt; Synchronizing usage tracker for tenant:4047e9db93a744f48d093e3c3d9e914b on resource:security_group_rule resync /opt/stack/neutron/neutron/quota/resource.py:245&lt;br/&gt;
2017-09-27 14:36:12.023 15977 DEBUG oslo_concurrency.lockutils &lt;span class=&quot;error&quot;&gt;&amp;#91;req-96f23e4d-1b20-47bc-8c27-e8badfa60e43 tempest-RoutersTest-1789903320 tempest-RoutersTest-1789903320&amp;#93;&lt;/span&gt; Lock &quot;event-dispatch&quot; released by &quot;neutron.plugins.ml2.ovo_rpc.dispatch_events&quot; :: held 0.143s inner /usr/lib/python2.7/site-packages/oslo_concurrency/lockutils.py:282&lt;br/&gt;
2017-09-27 14:36:12.027 15976 DEBUG networking_odl.journal.journal &lt;span class=&quot;error&quot;&gt;&amp;#91;req-996ea9ed-d917-4ec8-98c7-06ca9a443272 - -&amp;#93;&lt;/span&gt; No journal entries to process sync_pending_entries /opt/stack/networking-odl/networking_odl/journal/journal.py:225&lt;br/&gt;
2017-09-27 14:36:12.027 15977 INFO neutron.api.v2.resource &lt;span class=&quot;error&quot;&gt;&amp;#91;req-112bdc5c-9147-4ee8-9959-63083496c140 admin admin&amp;#93;&lt;/span&gt; delete failed (client error): There was a conflict when trying to complete your request.&lt;br/&gt;
2017-09-27 14:36:12.029 15977 INFO neutron.wsgi &lt;span class=&quot;error&quot;&gt;&amp;#91;req-112bdc5c-9147-4ee8-9959-63083496c140 admin admin&amp;#93;&lt;/span&gt; 10.29.13.105 &quot;DELETE /v2.0/security-groups/90cf7332-acac-40ca-87f2-abb2484cc9f1 HTTP/1.1&quot; status: 409  len: 536 time: 1.0451260&lt;/p&gt;

&lt;p&gt;nothing stood out to me in the karaf.log &lt;span class=&quot;error&quot;&gt;&amp;#91;1&amp;#93;&lt;/span&gt;. for the comments here, the tempest log is here &lt;span class=&quot;error&quot;&gt;&amp;#91;2&amp;#93;&lt;/span&gt;&lt;/p&gt;



&lt;p&gt;&lt;span class=&quot;error&quot;&gt;&amp;#91;0&amp;#93;&lt;/span&gt; &lt;a href=&quot;https://logs.opendaylight.org/releng/jenkins092/netvirt-csit-1node-openstack-pike-upstream-stateful-carbon/27/control_1/q-svc.log.2017-09-27-132116.gz&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://logs.opendaylight.org/releng/jenkins092/netvirt-csit-1node-openstack-pike-upstream-stateful-carbon/27/control_1/q-svc.log.2017-09-27-132116.gz&lt;/a&gt;&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;1&amp;#93;&lt;/span&gt; &lt;a href=&quot;https://logs.opendaylight.org/releng/jenkins092/netvirt-csit-1node-openstack-pike-upstream-stateful-carbon/27/odl1_karaf.log.gz&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://logs.opendaylight.org/releng/jenkins092/netvirt-csit-1node-openstack-pike-upstream-stateful-carbon/27/odl1_karaf.log.gz&lt;/a&gt;&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;2&amp;#93;&lt;/span&gt; &lt;a href=&quot;https://logs.opendaylight.org/releng/jenkins092/netvirt-csit-1node-openstack-pike-upstream-stateful-carbon/27/tempest/tempest.log.gz&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://logs.opendaylight.org/releng/jenkins092/netvirt-csit-1node-openstack-pike-upstream-stateful-carbon/27/tempest/tempest.log.gz&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="38406" author="aswins" created="Thu, 28 Sep 2017 07:33:03 +0000"  >&lt;p&gt;From initial look this looks like the cleanup of the resources are failing. It is not the same test case that is failing always. In run &lt;span class=&quot;error&quot;&gt;&amp;#91;1&amp;#93;&lt;/span&gt; it is the tempest.api.network.test_extensions.ExtensionsTestJSON and in the run &lt;span class=&quot;error&quot;&gt;&amp;#91;2&amp;#93;&lt;/span&gt; it is tempest.api.network.test_networks.NetworksIpV6Test. So are we too fast in doing the clean up?&lt;/p&gt;

&lt;p&gt;The error states the SG we try to clean in use , which should mean it is associated with a port.&lt;/p&gt;

&lt;p&gt;Are the tests run in order always or  it purely random?&lt;/p&gt;


&lt;p&gt;&lt;span class=&quot;error&quot;&gt;&amp;#91;1&amp;#93;&lt;/span&gt;&lt;br/&gt;
&lt;a href=&quot;https://logs.opendaylight.org/releng/jenkins092/netvirt-csit-1node-openstack-pike-upstream-stateful-nitrogen/3/log.html.gz&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://logs.opendaylight.org/releng/jenkins092/netvirt-csit-1node-openstack-pike-upstream-stateful-nitrogen/3/log.html.gz&lt;/a&gt;&lt;br/&gt;
&lt;span class=&quot;error&quot;&gt;&amp;#91;2&amp;#93;&lt;/span&gt;&lt;br/&gt;
&lt;a href=&quot;https://logs.opendaylight.org/releng/jenkins092/netvirt-csit-1node-openstack-pike-upstream-stateful-carbon/27/control_1/q-svc.log.2017-09-27-132116.gz&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://logs.opendaylight.org/releng/jenkins092/netvirt-csit-1node-openstack-pike-upstream-stateful-carbon/27/control_1/q-svc.log.2017-09-27-132116.gz&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="38407" author="jluhrsen" created="Thu, 28 Sep 2017 14:29:58 +0000"  >&lt;p&gt;(In reply to Aswin Suryanarayanan from comment #3)&lt;br/&gt;
&amp;gt; From initial look this looks like the cleanup of the resources are failing.&lt;br/&gt;
&amp;gt; It is not the same test case that is failing always. In run &lt;span class=&quot;error&quot;&gt;&amp;#91;1&amp;#93;&lt;/span&gt; it is the&lt;br/&gt;
&amp;gt; tempest.api.network.test_extensions.ExtensionsTestJSON and in the run &lt;span class=&quot;error&quot;&gt;&amp;#91;2&amp;#93;&lt;/span&gt; it&lt;br/&gt;
&amp;gt; is tempest.api.network.test_networks.NetworksIpV6Test. So are we too fast in&lt;br/&gt;
&amp;gt; doing the clean up?&lt;br/&gt;
&amp;gt; &lt;br/&gt;
&amp;gt; The error states the SG we try to clean in use , which should mean it is&lt;br/&gt;
&amp;gt; associated with a port.&lt;br/&gt;
&amp;gt; &lt;br/&gt;
&amp;gt; Are the tests run in order always or  it purely random?&lt;br/&gt;
&amp;gt; &lt;br/&gt;
&amp;gt; &lt;br/&gt;
&amp;gt; &lt;span class=&quot;error&quot;&gt;&amp;#91;1&amp;#93;&lt;/span&gt;&lt;br/&gt;
&amp;gt; &lt;a href=&quot;https://logs.opendaylight.org/releng/jenkins092/netvirt-csit-1node-openstack-&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://logs.opendaylight.org/releng/jenkins092/netvirt-csit-1node-openstack-&lt;/a&gt;&lt;br/&gt;
&amp;gt; pike-upstream-stateful-nitrogen/3/log.html.gz&lt;br/&gt;
&amp;gt; &lt;span class=&quot;error&quot;&gt;&amp;#91;2&amp;#93;&lt;/span&gt;&lt;br/&gt;
&amp;gt; &lt;a href=&quot;https://logs.opendaylight.org/releng/jenkins092/netvirt-csit-1node-openstack-&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://logs.opendaylight.org/releng/jenkins092/netvirt-csit-1node-openstack-&lt;/a&gt;&lt;br/&gt;
&amp;gt; pike-upstream-stateful-carbon/27/control_1/q-svc.log.2017-09-27-132116.gz&lt;/p&gt;

&lt;p&gt;Aswin,&lt;/p&gt;

&lt;p&gt;good catch on realizing the same precommit error on SG was happening in different test&lt;br/&gt;
cases. AFAIK, by default tempest will run it&apos;s sub-tests in parallel so I guess that&lt;br/&gt;
probably means there is no guaranteed order. And I guess it&apos;s not happening every&lt;br/&gt;
time either.&lt;/p&gt;

&lt;p&gt;This may be a tough one to debug. Maybe we need some sort of extra log level turned&lt;br/&gt;
on for a specific module?&lt;/p&gt;

&lt;p&gt;I can look in to running the tests in sequence to see if that guarantees we never see&lt;br/&gt;
the failure, but we will need to fix this anyway, so not sure if it&apos;s worth the&lt;br/&gt;
effort.&lt;/p&gt;

&lt;p&gt;any other ideas for debugging?&lt;/p&gt;

&lt;p&gt;JamO&lt;/p&gt;</comment>
                            <comment id="38408" author="aswins" created="Tue, 3 Oct 2017 10:07:51 +0000"  >&lt;p&gt;This is not observed after run 32&lt;span class=&quot;error&quot;&gt;&amp;#91;1&amp;#93;&lt;/span&gt;. Is there any thing changed here? or it is a part of random behavior?  I am not able to reproduce this locally either.&lt;/p&gt;

&lt;p&gt;&lt;span class=&quot;error&quot;&gt;&amp;#91;1&amp;#93;&lt;/span&gt;https://logs.opendaylight.org/releng/jenkins092/netvirt-csit-1node-openstack-pike-upstream-stateful-nitrogen/36/log.html.gz&lt;/p&gt;</comment>
                            <comment id="38409" author="aswins" created="Tue, 3 Oct 2017 15:59:38 +0000"  >&lt;p&gt;It is back again&lt;br/&gt;
&lt;a href=&quot;https://logs.opendaylight.org/releng/jenkins092/netvirt-csit-1node-openstack-pike-upstream-stateful-nitrogen/38/log.html.gz#s1-s2-s1&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://logs.opendaylight.org/releng/jenkins092/netvirt-csit-1node-openstack-pike-upstream-stateful-nitrogen/38/log.html.gz#s1-s2-s1&lt;/a&gt;&lt;/p&gt;


&lt;p&gt;Jamo,&lt;/p&gt;

&lt;p&gt; Given the sporadic nature of the failure, the parallel execution of tempest tests could be a reason. Like when SG is associated with a port in a test another test tries to delete/clean the SGs? But I don&apos;t have an answer why it is only on pike.&lt;/p&gt;</comment>
                            <comment id="38410" author="jluhrsen" created="Tue, 3 Oct 2017 16:44:11 +0000"  >&lt;p&gt;(In reply to Aswin Suryanarayanan from comment #6)&lt;br/&gt;
&amp;gt; It is back again&lt;br/&gt;
&amp;gt; &lt;a href=&quot;https://logs.opendaylight.org/releng/jenkins092/netvirt-csit-1node-openstack-&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://logs.opendaylight.org/releng/jenkins092/netvirt-csit-1node-openstack-&lt;/a&gt;&lt;br/&gt;
&amp;gt; pike-upstream-stateful-nitrogen/38/log.html.gz#s1-s2-s1&lt;/p&gt;

&lt;p&gt;yeah, here is a good graphic to illustrate it&apos;s sporadic nature:&lt;/p&gt;

&lt;p&gt;&lt;a href=&quot;https://jenkins.opendaylight.org/releng/user/jluhrsen/my-views/view/netvirt%20csit/job/netvirt-csit-1node-openstack-pike-upstream-stateful-nitrogen/38/robot/netvirt-1node-openstack.txt/Tempest/Tempest/tempest.api.network/graph?zoomSignificant=false&amp;amp;failedOnly=false&amp;amp;criticalOnly=false&amp;amp;maxBuildsToShow=0&amp;amp;hd=true&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://jenkins.opendaylight.org/releng/user/jluhrsen/my-views/view/netvirt%20csit/job/netvirt-csit-1node-openstack-pike-upstream-stateful-nitrogen/38/robot/netvirt-1node-openstack.txt/Tempest/Tempest/tempest.api.network/graph?zoomSignificant=false&amp;amp;failedOnly=false&amp;amp;criticalOnly=false&amp;amp;maxBuildsToShow=0&amp;amp;hd=true&lt;/a&gt;&lt;/p&gt;


&lt;p&gt;&amp;gt; Jamo,&lt;br/&gt;
&amp;gt; &lt;br/&gt;
&amp;gt;  Given the sporadic nature of the failure, the parallel execution of tempest&lt;br/&gt;
&amp;gt; tests could be a reason. Like when SG is associated with a port in a test&lt;br/&gt;
&amp;gt; another test tries to delete/clean the SGs? But I don&apos;t have an answer why&lt;br/&gt;
&amp;gt; it is only on pike.&lt;/p&gt;

&lt;p&gt;From what I know about tempest, it&apos;s usually using unique names/ids for&lt;br/&gt;
the things each test is creating/updating/deleting. So, I think we should&lt;br/&gt;
assume that even running in parallel, there would not be one test case&lt;br/&gt;
modifying a SG that is being used by another test case. I could be wrong,&lt;br/&gt;
but that seems a reasonable assumption. Otherwise, I think a lot of the&lt;br/&gt;
community using tempest in parallel would run in to problems all the time.&lt;/p&gt;</comment>
                            <comment id="38411" author="jluhrsen" created="Tue, 3 Oct 2017 16:44:50 +0000"  >&lt;p&gt;(In reply to Jamo Luhrsen from comment #7)&lt;br/&gt;
&amp;gt; (In reply to Aswin Suryanarayanan from comment #6)&lt;br/&gt;
&amp;gt; &amp;gt; It is back again&lt;br/&gt;
&amp;gt; &amp;gt; &lt;a href=&quot;https://logs.opendaylight.org/releng/jenkins092/netvirt-csit-1node-openstack-&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://logs.opendaylight.org/releng/jenkins092/netvirt-csit-1node-openstack-&lt;/a&gt;&lt;br/&gt;
&amp;gt; &amp;gt; pike-upstream-stateful-nitrogen/38/log.html.gz#s1-s2-s1&lt;br/&gt;
&amp;gt; &lt;br/&gt;
&amp;gt; yeah, here is a good graphic to illustrate it&apos;s sporadic nature:&lt;br/&gt;
&amp;gt; &lt;br/&gt;
&amp;gt; &lt;a href=&quot;https://jenkins.opendaylight.org/releng/user/jluhrsen/my-views/view/&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://jenkins.opendaylight.org/releng/user/jluhrsen/my-views/view/&lt;/a&gt;&lt;br/&gt;
&amp;gt; netvirt%20csit/job/netvirt-csit-1node-openstack-pike-upstream-stateful-&lt;br/&gt;
&amp;gt; nitrogen/38/robot/netvirt-1node-openstack.txt/Tempest/Tempest/tempest.api.&lt;br/&gt;
&amp;gt; network/&lt;br/&gt;
&amp;gt; graph?zoomSignificant=false&amp;amp;failedOnly=false&amp;amp;criticalOnly=false&amp;amp;maxBuildsToSh&lt;br/&gt;
&amp;gt; ow=0&amp;amp;hd=true&lt;br/&gt;
&amp;gt; &lt;br/&gt;
&amp;gt; &lt;br/&gt;
&amp;gt; &amp;gt; Jamo,&lt;br/&gt;
&amp;gt; &amp;gt; &lt;br/&gt;
&amp;gt; &amp;gt;  Given the sporadic nature of the failure, the parallel execution of tempest&lt;br/&gt;
&amp;gt; &amp;gt; tests could be a reason. Like when SG is associated with a port in a test&lt;br/&gt;
&amp;gt; &amp;gt; another test tries to delete/clean the SGs? But I don&apos;t have an answer why&lt;br/&gt;
&amp;gt; &amp;gt; it is only on pike.&lt;br/&gt;
&amp;gt; &lt;br/&gt;
&amp;gt; From what I know about tempest, it&apos;s usually using unique names/ids for&lt;br/&gt;
&amp;gt; the things each test is creating/updating/deleting. So, I think we should&lt;br/&gt;
&amp;gt; assume that even running in parallel, there would not be one test case&lt;br/&gt;
&amp;gt; modifying a SG that is being used by another test case. I could be wrong,&lt;br/&gt;
&amp;gt; but that seems a reasonable assumption. Otherwise, I think a lot of the&lt;br/&gt;
&amp;gt; community using tempest in parallel would run in to problems all the time.&lt;/p&gt;

&lt;p&gt;are there any debugs we can enable on the ODL side that might help?&lt;/p&gt;</comment>
                            <comment id="38412" author="aswins" created="Thu, 5 Oct 2017 14:30:24 +0000"  >&lt;p&gt;(In reply to Jamo Luhrsen from comment #8)&lt;br/&gt;
&amp;gt; (In reply to Jamo Luhrsen from comment #7)&lt;br/&gt;
&amp;gt; &amp;gt; (In reply to Aswin Suryanarayanan from comment #6)&lt;br/&gt;
&amp;gt; &amp;gt; &amp;gt; It is back again&lt;br/&gt;
&amp;gt; &amp;gt; &amp;gt; &lt;a href=&quot;https://logs.opendaylight.org/releng/jenkins092/netvirt-csit-1node-openstack-&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://logs.opendaylight.org/releng/jenkins092/netvirt-csit-1node-openstack-&lt;/a&gt;&lt;br/&gt;
&amp;gt; &amp;gt; &amp;gt; pike-upstream-stateful-nitrogen/38/log.html.gz#s1-s2-s1&lt;br/&gt;
&amp;gt; &amp;gt; &lt;br/&gt;
&amp;gt; &amp;gt; yeah, here is a good graphic to illustrate it&apos;s sporadic nature:&lt;br/&gt;
&amp;gt; &amp;gt; &lt;br/&gt;
&amp;gt; &amp;gt; &lt;a href=&quot;https://jenkins.opendaylight.org/releng/user/jluhrsen/my-views/view/&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://jenkins.opendaylight.org/releng/user/jluhrsen/my-views/view/&lt;/a&gt;&lt;br/&gt;
&amp;gt; &amp;gt; netvirt%20csit/job/netvirt-csit-1node-openstack-pike-upstream-stateful-&lt;br/&gt;
&amp;gt; &amp;gt; nitrogen/38/robot/netvirt-1node-openstack.txt/Tempest/Tempest/tempest.api.&lt;br/&gt;
&amp;gt; &amp;gt; network/&lt;br/&gt;
&amp;gt; &amp;gt; graph?zoomSignificant=false&amp;amp;failedOnly=false&amp;amp;criticalOnly=false&amp;amp;maxBuildsToSh&lt;br/&gt;
&amp;gt; &amp;gt; ow=0&amp;amp;hd=true&lt;br/&gt;
&amp;gt; &amp;gt; &lt;br/&gt;
&amp;gt; &amp;gt; &lt;br/&gt;
&amp;gt; &amp;gt; &amp;gt; Jamo,&lt;br/&gt;
&amp;gt; &amp;gt; &amp;gt; &lt;br/&gt;
&amp;gt; &amp;gt; &amp;gt;  Given the sporadic nature of the failure, the parallel execution of tempest&lt;br/&gt;
&amp;gt; &amp;gt; &amp;gt; tests could be a reason. Like when SG is associated with a port in a test&lt;br/&gt;
&amp;gt; &amp;gt; &amp;gt; another test tries to delete/clean the SGs? But I don&apos;t have an answer why&lt;br/&gt;
&amp;gt; &amp;gt; &amp;gt; it is only on pike.&lt;br/&gt;
&amp;gt; &amp;gt; &lt;br/&gt;
&amp;gt; &amp;gt; From what I know about tempest, it&apos;s usually using unique names/ids for&lt;br/&gt;
&amp;gt; &amp;gt; the things each test is creating/updating/deleting. So, I think we should&lt;br/&gt;
&amp;gt; &amp;gt; assume that even running in parallel, there would not be one test case&lt;br/&gt;
&amp;gt; &amp;gt; modifying a SG that is being used by another test case. I could be wrong,&lt;br/&gt;
&amp;gt; &amp;gt; but that seems a reasonable assumption. Otherwise, I think a lot of the&lt;br/&gt;
&amp;gt; &amp;gt; community using tempest in parallel would run in to problems all the time.&lt;br/&gt;
&amp;gt; &lt;br/&gt;
&amp;gt; are there any debugs we can enable on the ODL side that might help?&lt;/p&gt;

&lt;p&gt;My assumption is this related to some neutron/networking odl. Didn&apos;t see anything interesting in karaf log either. So nothing comes in mind that could help from ODL side&lt;/p&gt;</comment>
                            <comment id="38413" author="jluhrsen" created="Thu, 5 Oct 2017 16:59:22 +0000"  >&lt;p&gt;(In reply to Aswin Suryanarayanan from comment #9)&lt;br/&gt;
&amp;gt; (In reply to Jamo Luhrsen from comment #8)&lt;br/&gt;
&amp;gt; &amp;gt; (In reply to Jamo Luhrsen from comment #7)&lt;br/&gt;
&amp;gt; &amp;gt; &amp;gt; (In reply to Aswin Suryanarayanan from comment #6)&lt;br/&gt;
&amp;gt; &amp;gt; &amp;gt; &amp;gt; It is back again&lt;br/&gt;
&amp;gt; &amp;gt; &amp;gt; &amp;gt; &lt;a href=&quot;https://logs.opendaylight.org/releng/jenkins092/netvirt-csit-1node-openstack-&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://logs.opendaylight.org/releng/jenkins092/netvirt-csit-1node-openstack-&lt;/a&gt;&lt;br/&gt;
&amp;gt; &amp;gt; &amp;gt; &amp;gt; pike-upstream-stateful-nitrogen/38/log.html.gz#s1-s2-s1&lt;br/&gt;
&amp;gt; &amp;gt; &amp;gt; &lt;br/&gt;
&amp;gt; &amp;gt; &amp;gt; yeah, here is a good graphic to illustrate it&apos;s sporadic nature:&lt;br/&gt;
&amp;gt; &amp;gt; &amp;gt; &lt;br/&gt;
&amp;gt; &amp;gt; &amp;gt; &lt;a href=&quot;https://jenkins.opendaylight.org/releng/user/jluhrsen/my-views/view/&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://jenkins.opendaylight.org/releng/user/jluhrsen/my-views/view/&lt;/a&gt;&lt;br/&gt;
&amp;gt; &amp;gt; &amp;gt; netvirt%20csit/job/netvirt-csit-1node-openstack-pike-upstream-stateful-&lt;br/&gt;
&amp;gt; &amp;gt; &amp;gt; nitrogen/38/robot/netvirt-1node-openstack.txt/Tempest/Tempest/tempest.api.&lt;br/&gt;
&amp;gt; &amp;gt; &amp;gt; network/&lt;br/&gt;
&amp;gt; &amp;gt; &amp;gt; graph?zoomSignificant=false&amp;amp;failedOnly=false&amp;amp;criticalOnly=false&amp;amp;maxBuildsToSh&lt;br/&gt;
&amp;gt; &amp;gt; &amp;gt; ow=0&amp;amp;hd=true&lt;br/&gt;
&amp;gt; &amp;gt; &amp;gt; &lt;br/&gt;
&amp;gt; &amp;gt; &amp;gt; &lt;br/&gt;
&amp;gt; &amp;gt; &amp;gt; &amp;gt; Jamo,&lt;br/&gt;
&amp;gt; &amp;gt; &amp;gt; &amp;gt; &lt;br/&gt;
&amp;gt; &amp;gt; &amp;gt; &amp;gt;  Given the sporadic nature of the failure, the parallel execution of tempest&lt;br/&gt;
&amp;gt; &amp;gt; &amp;gt; &amp;gt; tests could be a reason. Like when SG is associated with a port in a test&lt;br/&gt;
&amp;gt; &amp;gt; &amp;gt; &amp;gt; another test tries to delete/clean the SGs? But I don&apos;t have an answer why&lt;br/&gt;
&amp;gt; &amp;gt; &amp;gt; &amp;gt; it is only on pike.&lt;br/&gt;
&amp;gt; &amp;gt; &amp;gt; &lt;br/&gt;
&amp;gt; &amp;gt; &amp;gt; From what I know about tempest, it&apos;s usually using unique names/ids for&lt;br/&gt;
&amp;gt; &amp;gt; &amp;gt; the things each test is creating/updating/deleting. So, I think we should&lt;br/&gt;
&amp;gt; &amp;gt; &amp;gt; assume that even running in parallel, there would not be one test case&lt;br/&gt;
&amp;gt; &amp;gt; &amp;gt; modifying a SG that is being used by another test case. I could be wrong,&lt;br/&gt;
&amp;gt; &amp;gt; &amp;gt; but that seems a reasonable assumption. Otherwise, I think a lot of the&lt;br/&gt;
&amp;gt; &amp;gt; &amp;gt; community using tempest in parallel would run in to problems all the time.&lt;br/&gt;
&amp;gt; &amp;gt; &lt;br/&gt;
&amp;gt; &amp;gt; are there any debugs we can enable on the ODL side that might help?&lt;br/&gt;
&amp;gt; &lt;br/&gt;
&amp;gt; My assumption is this related to some neutron/networking odl. Didn&apos;t see&lt;br/&gt;
&amp;gt; anything interesting in karaf log either. So nothing comes in mind that&lt;br/&gt;
&amp;gt; could help from ODL side&lt;/p&gt;

&lt;p&gt;random thought:&lt;br/&gt;
  I wonder if you could artificially load your local environment where you are &lt;br/&gt;
trying unsuccessfully to reproduce this issue. There has always been some&lt;br/&gt;
concern that our shared upstream cloud environment for our CSIT systems can&lt;br/&gt;
be heavily loaded at times. I wonder if that could cause certain timing situations&lt;br/&gt;
that would uncover some races?&lt;/p&gt;</comment>
                            <comment id="59766" author="aswins" created="Tue, 24 Oct 2017 13:28:01 +0000"  >&lt;p&gt;There was a bug related to this fixed in tempest&lt;span class=&quot;error&quot;&gt;&amp;#91;1&amp;#93;&lt;/span&gt;. But  we should be having the latest?&lt;/p&gt;

&lt;p&gt;&lt;span class=&quot;error&quot;&gt;&amp;#91;1&amp;#93;&lt;/span&gt;https://bugs.launchpad.net/tempest/+bug/1623691&lt;/p&gt;</comment>
                            <comment id="59885" author="jluhrsen" created="Thu, 2 Nov 2017 04:52:25 +0000"  >&lt;p&gt;I saw this in the sandbox tonight (11/1/2017). I&apos;ll keep it open for a while and keep checking&lt;br/&gt;
in releng.&lt;/p&gt;</comment>
                            <comment id="60573" author="huzhj" created="Tue, 2 Jan 2018 05:57:23 +0000"  >&lt;p&gt;We also encounter this problem yesterday in OPNFV functest &lt;span class=&quot;error&quot;&gt;&amp;#91;1&amp;#93;&lt;/span&gt;, and the log is &lt;span class=&quot;error&quot;&gt;&amp;#91;2&amp;#93;&lt;/span&gt;, related ODL version is &lt;span class=&quot;error&quot;&gt;&amp;#91;3&amp;#93;&lt;/span&gt;.&lt;/p&gt;

&lt;p&gt;&lt;span class=&quot;error&quot;&gt;&amp;#91;1&amp;#93;&lt;/span&gt;&lt;a href=&quot;https://build.opnfv.org/ci/job/functest-daisy-baremetal-daily-master/240/console&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://build.opnfv.org/ci/job/functest-daisy-baremetal-daily-master/240/console&lt;/a&gt;&lt;/p&gt;

&lt;p&gt;&lt;span class=&quot;error&quot;&gt;&amp;#91;2&amp;#93;&lt;/span&gt;&lt;a href=&quot;http://artifacts.opnfv.org/logs/functest/zte-pod3/2018-01-01_13-53-07/functest.log&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;http://artifacts.opnfv.org/logs/functest/zte-pod3/2018-01-01_13-53-07/functest.log&lt;/a&gt;&lt;/p&gt;

&lt;p&gt;&lt;span class=&quot;error&quot;&gt;&amp;#91;3&amp;#93;&lt;/span&gt;&lt;a href=&quot;http://cbs.centos.org/repos/nfv7-opendaylight-6-candidate/x86_64/os/Packages/opendaylight-6.3.0-0.1.20171211rel2065.el7.noarch.rpm&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;http://cbs.centos.org/repos/nfv7-opendaylight-6-candidate/x86_64/os/Packages/opendaylight-6.3.0-0.1.20171211rel2065.el7.noarch.rpm&lt;/a&gt;&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;</comment>
                            <comment id="60638" author="shague@redhat.com" created="Tue, 9 Jan 2018 11:56:32 +0000"  >&lt;p&gt;latest runs still have failure 1/8/18:&#160;&lt;a href=&quot;https://logs.opendaylight.org/releng/vex-yul-odl-jenkins-1/netvirt-csit-1node-openstack-pike-upstream-stateful-oxygen/114/log_05_tempest.html.gz&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://logs.opendaylight.org/releng/vex-yul-odl-jenkins-1/netvirt-csit-1node-openstack-pike-upstream-stateful-oxygen/114/log_05_tempest.html.gz&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="60650" author="jluhrsen" created="Tue, 9 Jan 2018 19:00:11 +0000"  >&lt;p&gt;Using a local deployment (tripleo based), I can reproduce this with the following tempest run:&lt;/p&gt;

&lt;p&gt;&#160;&#160;&#160; --concurrency 2 --regex &apos;(tempest.api.network.test_security_groups)&apos;&lt;/p&gt;

&lt;p&gt;that runs 43 tests and sometimes one or more will come back failed with this kind of error:&lt;/p&gt;

&lt;p&gt;Details: {u&apos;message&apos;: u&apos;Security Group aa4f7835-44d9-489c-861b-552d25922d81 cannot perform precommit_delete due to Callback networking_odl.common.callback.OdlSecurityGroupsHandler.sg_callback_precommit-765033 failed with &quot;argument of type \&apos;NoneType\&apos; is not iterable&quot;.&apos;, u&apos;type&apos;: u&apos;SecurityGroupInUse&apos;, u&apos;detail&apos;: u&apos;&apos;}&lt;/p&gt;

&lt;p&gt;If concurrency is set to 1, I can&apos;t reproduce the problem.&lt;/p&gt;</comment>
                            <comment id="60659" author="yamahata" created="Wed, 10 Jan 2018 16:50:03 +0000"  >&lt;p&gt;Can you please enable debug option for neutron and get log? Unfortunately the root cause exception is masked.&lt;/p&gt;

&lt;p&gt;Also can you try the following patch? this patch is based on pure guess, though.&lt;/p&gt;</comment>
                            <comment id="60660" author="yamahata" created="Wed, 10 Jan 2018 16:51:30 +0000"  >&lt;p&gt;diff --git a/networking_odl/ml2/mech_driver_v2.py b/networking_odl/ml2/mech_driver_v2.py&lt;br/&gt;
index 44306f8ed..5a773b080 100644&lt;br/&gt;
&amp;#8212; a/networking_odl/ml2/mech_driver_v2.py&lt;br/&gt;
+++ b/networking_odl/ml2/mech_driver_v2.py&lt;br/&gt;
@@ -194,7 +194,8 @@ class OpenDaylightMechanismDriver(api.MechanismDriver):&lt;/p&gt;
&lt;ol&gt;
	&lt;li&gt;be deleted. At NeutronDB layer rules are auto deleted with&lt;/li&gt;
	&lt;li&gt;cascade=&apos;all,delete&apos;.&lt;br/&gt;
         if (object_type == odl_const.ODL_SG and&lt;/li&gt;
&lt;/ol&gt;
&lt;ul class=&quot;alternate&quot; type=&quot;square&quot;&gt;
	&lt;li&gt;operation == odl_const.ODL_DELETE):&lt;br/&gt;
+                operation == odl_const.ODL_DELETE and&lt;br/&gt;
+                kwargs&lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;#39;security_group_rule_ids&amp;#39;&amp;#93;&lt;/span&gt;):&lt;br/&gt;
             for rule_id in kwargs&lt;span class=&quot;error&quot;&gt;&amp;#91;&amp;#39;security_group_rule_ids&amp;#39;&amp;#93;&lt;/span&gt;:&lt;br/&gt;
                 journal.record(context, odl_const.ODL_SG_RULE,&lt;br/&gt;
                                rule_id, odl_const.ODL_DELETE, &lt;span class=&quot;error&quot;&gt;&amp;#91;object_uuid&amp;#93;&lt;/span&gt;)&lt;/li&gt;
&lt;/ul&gt;
</comment>
                            <comment id="60661" author="yamahata" created="Wed, 10 Jan 2018 16:52:25 +0000"  >&lt;p&gt; &lt;span class=&quot;nobr&quot;&gt;&lt;a href=&quot;https://jira.opendaylight.org/secure/attachment/14355/14355_sg-precommit-delete.diff&quot; title=&quot;sg-precommit-delete.diff attached to NETVIRT-896&quot;&gt;sg-precommit-delete.diff&lt;sup&gt;&lt;img class=&quot;rendericon&quot; src=&quot;https://jira.opendaylight.org/images/icons/link_attachment_7.gif&quot; height=&quot;7&quot; width=&quot;7&quot; align=&quot;absmiddle&quot; alt=&quot;&quot; border=&quot;0&quot;/&gt;&lt;/sup&gt;&lt;/a&gt;&lt;/span&gt; &lt;/p&gt;</comment>
                            <comment id="60662" author="yamahata" created="Wed, 10 Jan 2018 17:02:03 +0000"  >&lt;p&gt; &lt;span class=&quot;nobr&quot;&gt;&lt;a href=&quot;https://jira.opendaylight.org/secure/attachment/14355/14355_sg-precommit-delete.diff&quot; title=&quot;sg-precommit-delete.diff attached to NETVIRT-896&quot;&gt;sg-precommit-delete.diff&lt;sup&gt;&lt;img class=&quot;rendericon&quot; src=&quot;https://jira.opendaylight.org/images/icons/link_attachment_7.gif&quot; height=&quot;7&quot; width=&quot;7&quot; align=&quot;absmiddle&quot; alt=&quot;&quot; border=&quot;0&quot;/&gt;&lt;/sup&gt;&lt;/a&gt;&lt;/span&gt; &lt;/p&gt;</comment>
                            <comment id="60676" author="jluhrsen" created="Fri, 12 Jan 2018 21:52:03 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.opendaylight.org/secure/ViewProfile.jspa?name=yamahata&quot; class=&quot;user-hover&quot; rel=&quot;yamahata&quot;&gt;yamahata&lt;/a&gt;, I applied the changes to my setup, but the problem persists.&lt;/p&gt;

&lt;p&gt;Attached is the neutron server log (DEBUG logs) when running with your changes. I ran this tempest command,&lt;br/&gt;
which seems to be using 4 workers (concurrency).&lt;/p&gt;

&lt;p&gt;tempest run --regex &apos;(tempest.api.network.test_security_groups)&apos;&lt;/p&gt;

&lt;p&gt;I can easily try more patches/fixes, if you have things to try.&lt;/p&gt;</comment>
                            <comment id="60679" author="yamahata" created="Sat, 13 Jan 2018 02:37:37 +0000"  >&lt;p&gt; &lt;span class=&quot;nobr&quot;&gt;&lt;a href=&quot;https://jira.opendaylight.org/secure/attachment/14358/14358_sg-precommit-exception.diff&quot; title=&quot;sg-precommit-exception.diff attached to NETVIRT-896&quot;&gt;sg-precommit-exception.diff&lt;sup&gt;&lt;img class=&quot;rendericon&quot; src=&quot;https://jira.opendaylight.org/images/icons/link_attachment_7.gif&quot; height=&quot;7&quot; width=&quot;7&quot; align=&quot;absmiddle&quot; alt=&quot;&quot; border=&quot;0&quot;/&gt;&lt;/sup&gt;&lt;/a&gt;&lt;/span&gt; &lt;br/&gt;
Okay, the log helped much. Can you please the patch?&lt;br/&gt;
If it works, I&apos;m going to bake it.&lt;/p&gt;</comment>
                            <comment id="60686" author="yamahata" created="Mon, 15 Jan 2018 15:29:56 +0000"  >&lt;p&gt;the patch was uploaded to &lt;a href=&quot;https://review.openstack.org/#/c/533706/&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://review.openstack.org/#/c/533706/&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="60773" author="huzhj" created="Mon, 22 Jan 2018 02:24:44 +0000"  >&lt;p&gt;Hello,&lt;/p&gt;

&lt;p&gt;Thanks for catching this bug, want to add that this also affects OpenStack Pike + ODL Carbon&#160;&lt;/p&gt;

&lt;p&gt;&lt;a href=&quot;http://artifacts.opnfv.org/logs/functest/zte-pod3/2018-01-21_19-53-22/functest.log&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;http://artifacts.opnfv.org/logs/functest/zte-pod3/2018-01-21_19-53-22/functest.log&lt;/a&gt;&lt;/p&gt;

&lt;p&gt;So when the patch is ready, could it be cherry-picked to networking-odl&apos;s stable/pike branch?&lt;/p&gt;</comment>
                            <comment id="60806" author="jluhrsen" created="Tue, 23 Jan 2018 20:02:09 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.opendaylight.org/secure/ViewProfile.jspa?name=yamahata&quot; class=&quot;user-hover&quot; rel=&quot;yamahata&quot;&gt;yamahata&lt;/a&gt;, local tests with your &lt;a href=&quot;https://review.openstack.org/#/c/533706/&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;patch&lt;/a&gt; are passing for me. I&apos;ve run the tempest.api.network.test_security_groups bucket&lt;br/&gt;
5 times back to back without failure using concurrency == 4. (NOTE: I hit it every time without the patch). Also, the full bucket&lt;br/&gt;
of tempest.api.network is also passing fine.&lt;/p&gt;

&lt;p&gt;looking forward to getting your patch merged and ODL CSIT not hitting this any more.&lt;/p&gt;

&lt;p&gt;Much appreciated!&lt;/p&gt;</comment>
                            <comment id="60833" author="jluhrsen" created="Thu, 25 Jan 2018 18:58:55 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.opendaylight.org/secure/ViewProfile.jspa?name=CaseyCain&quot; class=&quot;user-hover&quot; rel=&quot;CaseyCain&quot;&gt;CaseyCain&lt;/a&gt;, can we get someone to update this Jira with an explanation of why it is a blocker?&lt;/p&gt;

&lt;p&gt;BTW, in case it&apos;s not clear in the comments above, this bug has it&apos;s fix coming on the openstack&lt;br/&gt;
side, not OpenDaylight side. I don&apos;t know how fast that will happen.&lt;/p&gt;</comment>
                            <comment id="60835" author="huzhj" created="Fri, 26 Jan 2018 01:41:09 +0000"  >&lt;p&gt;&lt;a href=&quot;https://jira.opendaylight.org/secure/ViewProfile.jspa?name=jluhrsen&quot; class=&quot;user-hover&quot; rel=&quot;jluhrsen&quot;&gt;jluhrsen&lt;/a&gt; The reason why I think it was an&#160;blocker for OPNFV MS3.2 becasue I saw it happened many times in&#160;OPNFV functest (with Daisy installer which deploy OpenStack Pike + ODL carbon), for example,&#160;&lt;a href=&quot;http://artifacts.opnfv.org/logs/functest/zte-pod3/2018-01-21_19-53-22/functest.log&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;http://artifacts.opnfv.org/logs/functest/zte-pod3/2018-01-21_19-53-22/functest.log&lt;/a&gt;&lt;/p&gt;

&lt;p&gt;&#160;&lt;/p&gt;

&lt;p&gt;Following the JIRA ticket I got the solution patch (&lt;a href=&quot;https://review.openstack.org/#/c/533706/),&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://review.openstack.org/#/c/533706/)&lt;/a&gt;&#160;and&#160;&#160;I have patched it dowstream into Daisy&apos;s repo first(&lt;a href=&quot;https://gerrit.opnfv.org/gerrit/#/c/51031/).&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://gerrit.opnfv.org/gerrit/#/c/51031/).&lt;/a&gt;&#160;And functest for Daisy&apos;s sdn-nofeature scenario is stable now. So from Daisy Installer side, it is no longer a blocker for&#160;OPNFV MS3.2.&#160;Looking forward to getting&#160;the patch merged upstream. &lt;img class=&quot;emoticon&quot; src=&quot;https://jira.opendaylight.org/images/icons/emoticons/smile.png&quot; height=&quot;16&quot; width=&quot;16&quot; align=&quot;absmiddle&quot; alt=&quot;&quot; border=&quot;0&quot;/&gt;&lt;/p&gt;</comment>
                            <comment id="60840" author="jaicaa" created="Fri, 26 Jan 2018 14:19:07 +0000"  >&lt;p&gt;For sake of completeness, this patch got rid of the problem also:&lt;/p&gt;

&lt;p&gt;&lt;a href=&quot;https://review.openstack.org/#/c/536935/&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://review.openstack.org/#/c/536935/&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="60846" author="jluhrsen" created="Fri, 26 Jan 2018 22:20:21 +0000"  >&lt;p&gt;Thanks &lt;a href=&quot;https://jira.opendaylight.org/secure/ViewProfile.jspa?name=jaicaa&quot; class=&quot;user-hover&quot; rel=&quot;jaicaa&quot;&gt;jaicaa&lt;/a&gt;.&lt;/p&gt;

&lt;p&gt;any chance we can get this ported to pike? he&apos;s my try for that:&lt;br/&gt;
&#160; &lt;a href=&quot;https://review.openstack.org/#/c/538352/&quot; class=&quot;external-link&quot; target=&quot;_blank&quot; rel=&quot;nofollow noopener&quot;&gt;https://review.openstack.org/#/c/538352/&lt;/a&gt;&lt;/p&gt;</comment>
                            <comment id="61016" author="klou" created="Fri, 9 Feb 2018 23:53:48 +0000"  >&lt;p&gt;Changing severity to major as we have&#160;confirmation that this is not a blocker.&lt;/p&gt;</comment>
                    </comments>
                    <attachments>
                            <attachment id="14357" name="server.log.xz" size="78128" author="jluhrsen" created="Fri, 12 Jan 2018 21:53:24 +0000"/>
                            <attachment id="14355" name="sg-precommit-delete.diff" size="2454" author="yamahata" created="Wed, 10 Jan 2018 17:02:02 +0000"/>
                            <attachment id="14354" name="sg-precommit-delete.diff" size="803" author="yamahata" created="Wed, 10 Jan 2018 16:52:23 +0000"/>
                            <attachment id="14358" name="sg-precommit-exception.diff" size="2185" author="yamahata" created="Sat, 13 Jan 2018 02:37:03 +0000"/>
                    </attachments>
                <subtasks>
                    </subtasks>
                <customfields>
                                                                            <customfield id="customfield_11400" key="com.atlassian.jira.plugins.jira-development-integration-plugin:devsummary">
                        <customfieldname>Development</customfieldname>
                        <customfieldvalues>
                            
                        </customfieldvalues>
                    </customfield>
                                                                                                                        <customfield id="customfield_10208" key="com.atlassian.jira.plugin.system.customfieldtypes:textfield">
                        <customfieldname>External issue ID</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>9130</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10201" key="com.atlassian.jira.plugin.system.customfieldtypes:url">
                        <customfieldname>External issue URL</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue><![CDATA[https://bugs.opendaylight.org/show_bug.cgi?id=9130]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                                                                                                                                                                                                                                                                                                                <customfield id="customfield_10204" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>ODL SR Target Milestone</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10337"><![CDATA[Carbon-SR2]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                        <customfield id="customfield_10202" key="com.atlassian.jira.plugin.system.customfieldtypes:select">
                        <customfieldname>Priority</customfieldname>
                        <customfieldvalues>
                                <customfieldvalue key="10312"><![CDATA[High]]></customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                <customfield id="customfield_10000" key="com.pyxis.greenhopper.jira:gh-lexo-rank">
                        <customfieldname>Rank</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>0|i01umn:</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                            <customfield id="customfield_10203" key="com.atlassian.jira.plugin.system.customfieldtypes:textfield">
                        <customfieldname>Status Whiteboard</customfieldname>
                        <customfieldvalues>
                            <customfieldvalue>csit:sporadic_failures</customfieldvalue>

                        </customfieldvalues>
                    </customfield>
                                                                                                                        </customfields>
    </item>
</channel>
</rss>