root@cic-1:~# neutron bgpvpn-router-assoc-list VPN1 +--------------------------------------+--------------------------------------+ | id | router_id | +--------------------------------------+--------------------------------------+ | f7c90e18-c0db-40a2-b54e-923cc87a9ef0 | aaa7f0bb-cf03-4e32-ba90-4e2fa49789bd | +--------------------------------------+--------------------------------------+ root@cic-1:~# date Wed Aug 8 10:55:33 UTC 2018 Initial extra route conifg root@cic-1:~# neutron router-update R1 --routes type=dict list=true destination=20.0.0.0/24,nexthop=11.1.2.3 destination=2111:12f:862:18::/64,nexthop=2001:db8:abcd:11:f816:3eff:fe47:5af1 destination=20.0.0.0/24,nexthop=12.23.34.4 destination=2111:12f:862:18::/64,nexthop=2025:caf:3214:12:f816:3eff:fef1:8ac5 destination=20.0.0.0/24,nexthop=13.5.4.3 destination=2111:12f:862:18::/64,nexthop=2033:456a:f012:13:f816:3eff:feeb:b485 destination=20.0.0.0/24,nexthop=14.42.62.4 destination=2111:12f:862:18::/64,nexthop=2056:212:d562:14:f816:3eff:fee9:c346 destination=20.0.0.0/24,nexthop=15.8.9.3 destination=2111:12f:862:18::/64,nexthop=2062:321:543:15:f816:3eff:feb8:9957 destination=20.0.0.0/24,nexthop=16.6.67.4 destination=2111:12f:862:18::/64,nexthop=2078:fde:2cd:16:f816:3eff:fe30:3d70 destination=20.0.0.0/24,nexthop=17.7.6.3 destination=2111:12f:862:18::/64,nexthop=2123:13f:565:17:f816:3eff:feb8:26fa destination=20.0.0.0/24,nexthop=18.6.6.4 destination=2111:12f:862:18::/64,nexthop=2234:777:222:18:f816:3eff:fe8f:cfa3 Updated router: R1 root@cic-1:~# Step-1:Dissociate router from VPN : Load balance is WORKING root@cic-1:~# neutron bgpvpn-router-assoc-delete f7c90e18-c0db-40a2-b54e-923cc87a9ef0 VPN1 Deleted router_association: f7c90e18-c0db-40a2-b54e-923cc87a9ef0 root@cic-1:~# Step-2: Associate the router back to VPN : Load balancing FAILS. root@cic-1:~# neutron bgpvpn-router-assoc-create VPN1 --router aaa7f0bb-cf03-4e32-ba90-4e2fa49789bd Created a new router_association: +-----------+--------------------------------------+ | Field | Value | +-----------+--------------------------------------+ | id | ee0529a3-4408-44bd-bb56-199ec72e16f4 | | router_id | aaa7f0bb-cf03-4e32-ba90-4e2fa49789bd | | tenant_id | b0bb63eaae6b4a64980490b59a37a3d9 | +-----------+--------------------------------------+ root@cic-1:~# neutron router-show R1 +-----------------------+---------------------------------------------------------------------------------------------+ | Field | Value | +-----------------------+---------------------------------------------------------------------------------------------+ | admin_state_up | True | | description | | | distributed | False | | external_gateway_info | | | id | aaa7f0bb-cf03-4e32-ba90-4e2fa49789bd | | name | R1 | | routes | {"destination": "20.0.0.0/24", "nexthop": "11.1.2.3"} | | | {"destination": "20.0.0.0/24", "nexthop": "12.23.34.4"} | | | {"destination": "20.0.0.0/24", "nexthop": "13.5.4.3"} | | | {"destination": "20.0.0.0/24", "nexthop": "14.42.62.4"} | | | {"destination": "20.0.0.0/24", "nexthop": "15.8.9.3"} | | | {"destination": "20.0.0.0/24", "nexthop": "16.6.67.4"} | | | {"destination": "20.0.0.0/24", "nexthop": "17.7.6.3"} | | | {"destination": "20.0.0.0/24", "nexthop": "18.6.6.4"} | | | {"destination": "2111:12f:862:18::/64", "nexthop": "2001:db8:abcd:11:f816:3eff:fe47:5af1"} | | | {"destination": "2111:12f:862:18::/64", "nexthop": "2025:caf:3214:12:f816:3eff:fef1:8ac5"} | | | {"destination": "2111:12f:862:18::/64", "nexthop": "2033:456a:f012:13:f816:3eff:feeb:b485"} | | | {"destination": "2111:12f:862:18::/64", "nexthop": "2056:212:d562:14:f816:3eff:fee9:c346"} | | | {"destination": "2111:12f:862:18::/64", "nexthop": "2062:321:543:15:f816:3eff:feb8:9957"} | | | {"destination": "2111:12f:862:18::/64", "nexthop": "2078:fde:2cd:16:f816:3eff:fe30:3d70"} | | | {"destination": "2111:12f:862:18::/64", "nexthop": "2123:13f:565:17:f816:3eff:feb8:26fa"} | | | {"destination": "2111:12f:862:18::/64", "nexthop": "2234:777:222:18:f816:3eff:fe8f:cfa3"} | | status | ACTIVE | | tenant_id | b0bb63eaae6b4a64980490b59a37a3d9 | +-----------------------+---------------------------------------------------------------------------------------------+ root@cic-1:~# neutron bgpvpn-router-assoc-list VPN1 +--------------------------------------+--------------------------------------+ | id | router_id | +--------------------------------------+--------------------------------------+ | ee0529a3-4408-44bd-bb56-199ec72e16f4 | aaa7f0bb-cf03-4e32-ba90-4e2fa49789bd | +--------------------------------------+--------------------------------------+ root@cic-1:~# neutron router-list +--------------------------------------+------+-----------------------+-------------+ | id | name | external_gateway_info | distributed | +--------------------------------------+------+-----------------------+-------------+ | aaa7f0bb-cf03-4e32-ba90-4e2fa49789bd | R1 | null | False | +--------------------------------------+------+-----------------------+-------------+ root@cic-1:~# root@cic-1:~# neutron bgpvpn-show VPN1 +----------------------+--------------------------------------+ | Field | Value | +----------------------+--------------------------------------+ | export_targets | 100:1 | | id | a0b7227d-6c66-47a5-bd00-7c4d1b260509 | | import_targets | 100:11 | | name | VPN1 | | networks | | | route_distinguishers | 100:21 | | | 100:22 | | | 100:23 | | | 100:24 | | | 100:25 | | | 100:26 | | | 100:27 | | | 100:28 | | route_targets | | | routers | aaa7f0bb-cf03-4e32-ba90-4e2fa49789bd | | tenant_id | b0bb63eaae6b4a64980490b59a37a3d9 | | type | l3 | +----------------------+--------------------------------------+ root@cic-1:~# On Compute-0-2: ============== After Step-1: Working root@compute-0-2:~# sh table.sh 21 | grep 2111 cookie=0x8000003, duration=69.039s, table=21, n_packets=0, n_bytes=0, priority=74,ipv6,metadata=0x30d40/0xfffffe,ipv6_dst=2111:12f:862:18::/64 actions=group:150078 root@compute-0-2:~# sudo ovs-ofctl dump-groups br-int -O Openflow13 | grep 150078 group_id=150078,type=select,bucket=actions=group:150075,bucket=actions=group:150077,bucket=actions=group:150076,bucket=actions=group:150082,bucket=actions=set_field:0x420->tun_id,set_field:fa:16:3e:47:5a:f1->eth_dst,output:186,bucket=actions=set_field:0x430->tun_id,set_field:fa:16:3e:b8:26:fa->eth_dst,output:186,bucket=actions=set_field:0x427->tun_id,set_field:fa:16:3e:b8:99:57->eth_dst,output:186,bucket=actions=set_field:0x434->tun_id,set_field:fa:16:3e:eb:b4:85->eth_dst,output:186 After Step-2: Not working root@compute-0-2:~# sh table.sh 21 | grep 2111 cookie=0x8000003, duration=172.879s, table=21, n_packets=0, n_bytes=0, priority=74,ipv6,metadata=0x30d9e/0xfffffe,ipv6_dst=2111:12f:862:18::/64 actions=group:150118 root@compute-0-2:~# sudo ovs-ofctl dump-groups br-int -O Openflow13 | grep 150118 group_id=150118,type=select,bucket=actions=group:150123,bucket=actions=set_field:0x18747->tun_id,set_field:fa:16:3e:47:5a:f1->eth_dst,output:186,bucket=actions=set_field:0x18747->tun_id,set_field:fa:16:3e:b8:26:fa->eth_dst,output:186,bucket=actions=set_field:0x18747->tun_id,set_field:fa:16:3e:b8:99:57->eth_dst,output:186,bucket=actions=set_field:0x18747->tun_id,set_field:fa:16:3e:eb:b4:85->eth_dst,output:186 root@compute-0-2:~# sh table.sh 20 | grep 100167 cookie=0x8000002, duration=5894.392s, table=20, n_packets=0, n_bytes=0, priority=10,mpls,mpls_label=100167 actions=pop_mpls:0x86dd,group:150119 root@compute-0-2:~# sudo ovs-ofctl dump-groups br-int -O Openflow13 | grep 150119 group_id=150119,type=select,bucket=actions=group:150123 On compute-0-1: ============== After Step-1: WORKING root@compute-0-1:~# sh table.sh 21 | grep 2111 cookie=0x8000003, duration=27.687s, table=21, n_packets=0, n_bytes=0, priority=74,ipv6,metadata=0x30d40/0xfffffe,ipv6_dst=2111:12f:862:18::/64 actions=group:150078 root@compute-0-1:~# sudo ovs-ofctl dump-groups br-int -O Openflow13 | grep 150078 group_id=150078,type=select,bucket=actions=group:150090,bucket=actions=group:150084,bucket=actions=group:150094,bucket=actions=group:150080,bucket=actions=set_field:0x44d->tun_id,set_field:fa:16:3e:30:3d:70->eth_dst,output:88,bucket=actions=set_field:0x438->tun_id,set_field:fa:16:3e:8f:cf:a3->eth_dst,output:88,bucket=actions=set_field:0x3f0->tun_id,set_field:fa:16:3e:f1:8a:c5->eth_dst,output:88,bucket=actions=set_field:0x408->tun_id,set_field:fa:16:3e:e9:c3:46->eth_dst,output:88 After Step-2: WORKING root@compute-0-1:~# sh table.sh 21 | grep 2111 cookie=0x8000003, duration=20.020s, table=21, n_packets=0, n_bytes=0, priority=74,ipv6,metadata=0x30d9e/0xfffffe,ipv6_dst=2111:12f:862:18::/64 actions=group:150118 root@compute-0-1:~# sudo ovs-ofctl dump-groups br-int -O Openflow13 | grep 150118 group_id=150118,type=select,bucket=actions=group:150121,bucket=actions=group:150122,bucket=actions=group:150126,bucket=actions=group:150120,bucket=actions=set_field:0x18747->tun_id,set_field:fa:16:3e:8f:cf:a3->eth_dst,output:88,bucket=actions=set_field:0x18747->tun_id,set_field:fa:16:3e:30:3d:70->eth_dst,output:88,bucket=actions=set_field:0x18747->tun_id,set_field:fa:16:3e:e9:c3:46->eth_dst,output:88,bucket=actions=set_field:0x18747->tun_id,set_field:fa:16:3e:f1:8a:c5->eth_dst,output:88 root@compute-0-1:~# sh table.sh 20 | grep 100167 cookie=0x8000002, duration=5876.602s, table=20, n_packets=0, n_bytes=0, priority=10,mpls,mpls_label=100167 actions=pop_mpls:0x86dd,group:150119 root@compute-0-1:~# sudo ovs-ofctl dump-groups br-int -O Openflow13 | grep 150119 group_id=150119,type=select,bucket=actions=group:150121,bucket=actions=group:150122,bucket=actions=group:150126,bucket=actions=group:150120 cli>display fib Enter password for user cscadm: RD Prefix NextHop Label Origin ------------------------------------------------------------------- 100:28 2062:321:543:15:f816:3eff:fe55:690c/128 192.168.113.11 100184 l 100:28 13.5.4.4/32 192.168.113.11 100192 l 100:28 2001:db8:abcd:11::/64 192.168.113.11 100154 c 100:28 14.42.62.0/24 192.168.113.11 100161 c 100:28 2062:321:543:15:f816:3eff:feb8:9957/128 192.168.113.10 100179 l 100:28 2078:fde:2cd:16::/64 192.168.113.11 100157 c 100:28 14.42.62.1/32 0.0.0.0 100089 l 100:28 2033:456a:f012:13::/64 192.168.113.11 100160 c 100:28 2001:db8:abcd:11:f816:3eff:fe47:5af1/128 192.168.113.10 100189 l 100:28 16.6.67.3/32 192.168.113.10 100186 l 100:28 2001:8::3/128 192.168.113.94 20001 b 100:28 2078:fde:2cd:16:f816:3eff:fe30:3d70/128 192.168.113.11 100177 l 100:28 2234:777:222:18::/64 192.168.113.11 100151 c 100:28 2078:fde:2cd:16::1/128 0.0.0.0 100087 l 100:28 2050:123:456::/64 192.168.113.94 20001 b 100:28 12.23.34.4/32 192.168.113.11 100175 l 100:28 15.8.9.1/32 0.0.0.0 100070 l 100:28 18.6.6.3/32 192.168.113.10 100164 l 100:28 13.5.4.3/32 192.168.113.10 100190 l 100:28 50.1.1.0/24 192.168.113.94 20000 b 100:28 2056:212:d562:14::/64 192.168.113.11 100149 c 100:28 15.8.9.0/24 192.168.113.11 100150 c 100:28 17.7.6.4/32 192.168.113.11 100163 l 100:28 11.1.2.3/32 192.168.113.10 100188 l 100:28 2062:321:543:15::1/128 0.0.0.0 100090 l 100:28 2123:13f:565:17::/64 192.168.113.11 100146 c 100:28 2234:777:222:18::1/128 0.0.0.0 100068 l 100:28 20.0.0.0/24 192.168.113.11 100168 s 100:28 20.0.0.0/24 192.168.113.10 100168 s 100:28 2056:212:d562:14:f816:3eff:fee9:c346/128 192.168.113.11 100187 l 100:28 2025:caf:3214:12::/64 192.168.113.11 100148 c 100:28 10.1.1.0/24 192.168.113.94 20000 b 100:28 17.7.6.0/24 192.168.113.11 100156 c 100:28 2001:db8:abcd:11::1/128 0.0.0.0 100074 l 100:28 2078:fde:2cd:16:f816:3eff:fec8:3a92/128 192.168.113.10 100181 l 100:28 17.7.6.1/32 0.0.0.0 100086 l 100:28 2111:12f:862:18::/64 192.168.113.11 100167 s 100:28 2111:12f:862:18::/64 192.168.113.10 100167 s 100:28 15.8.9.4/32 192.168.113.11 100182 l 100:28 2033:456a:f012:13::1/128 0.0.0.0 100091 l 100:28 2123:13f:565:17::1/128 0.0.0.0 100064 l 100:28 16.6.67.0/24 192.168.113.11 100153 c 100:28 17.7.6.3/32 192.168.113.10 100169 l 100:28 2025:caf:3214:12::1/128 0.0.0.0 100066 l 100:28 14.42.62.3/32 192.168.113.10 100170 l 100:28 2234:777:222:18:f816:3eff:fe11:c9b6/128 192.168.113.10 100165 l 100:28 16.6.67.1/32 0.0.0.0 100073 l 100:28 2033:456a:f012:13:f816:3eff:fe97:a747/128 192.168.113.11 100191 l 100:28 2062:321:543:15::/64 192.168.113.11 100158 c 100:28 15.8.9.3/32 192.168.113.10 100178 l 100:28 2001:db8:abcd:11:f816:3eff:fed1:c53a/128 192.168.113.11 100180 l 100:28 2234:777:222:18:f816:3eff:fe8f:cfa3/128 192.168.113.11 100173 l 100:28 2056:212:d562:14::1/128 0.0.0.0 100069 l 100:28 18.6.6.1/32 0.0.0.0 100065 l 100:28 18.6.6.0/24 192.168.113.11 100147 c 100:28 2004:ae12::/64 192.168.113.94 20001 b 100:28 2033:456a:f012:13:f816:3eff:feeb:b485/128 192.168.113.10 100193 l 100:28 13.5.4.0/24 192.168.113.11 100159 c 100:28 2123:13f:565:17:f816:3eff:feca:bcf3/128 192.168.113.11 100162 l 100:28 14.42.62.4/32 192.168.113.11 100185 l 100:28 1.1.1.1/32 192.168.113.94 20000 b 100:28 11.1.2.5/32 192.168.113.11 100183 l 100:28 13.5.4.1/32 0.0.0.0 100088 l 100:28 16.6.67.4/32 192.168.113.11 100176 l 100:28 11.1.2.0/24 192.168.113.11 100155 c 100:28 2025:caf:3214:12:f816:3eff:fef1:8ac5/128 192.168.113.11 100172 l 100:28 2123:13f:565:17:f816:3eff:feb8:26fa/128 192.168.113.10 100166 l 100:28 2056:212:d562:14:f816:3eff:feda:3855/128 192.168.113.10 100171 l 100:28 11.1.2.1/32 0.0.0.0 100077 l 100:28 12.23.34.1/32 0.0.0.0 100071 l 100:28 12.23.34.0/24 192.168.113.11 100152 c 100:28 18.6.6.4/32 192.168.113.11 100174 l From datastore: http://localhost:8181/restconf/operational/odl-l3vpn:vpn-to-extraroutes/ { "vpn-to-extraroutes": { "vpn": [ { "extra-routes": [ { "routes": [ { "nexthop-ip-list": [ "2025:caf:3214:12:f816:3eff:fef1:8ac5" ], "prefix": "2111:12f:862:18::/64" }, { "nexthop-ip-list": [ "12.23.34.4", "14.42.62.4" ], "prefix": "20.0.0.0/24" } ], "vrf-id": "100:26" }, { "routes": [ { "nexthop-ip-list": [ "2234:777:222:18:f816:3eff:fe8f:cfa3", "2078:fde:2cd:16:f816:3eff:fe30:3d70", "2056:212:d562:14:f816:3eff:fee9:c346" ], "prefix": "2111:12f:862:18::/64" }, { "nexthop-ip-list": [ "18.6.6.4", "16.6.67.4" ], "prefix": "20.0.0.0/24" } ], "vrf-id": "100:27" }, { "routes": [ { "nexthop-ip-list": [ "2001:db8:abcd:11:f816:3eff:fe47:5af1", "2123:13f:565:17:f816:3eff:feb8:26fa", "2062:321:543:15:f816:3eff:feb8:9957", "2033:456a:f012:13:f816:3eff:feeb:b485" ], "prefix": "2111:12f:862:18::/64" }, { "nexthop-ip-list": [ "17.7.6.3", "15.8.9.3", "13.5.4.3", "11.1.2.3" ], "prefix": "20.0.0.0/24" } ], "vrf-id": "100:28" } ], "vpn-name": "a0b7227d-6c66-47a5-bd00-7c4d1b260509" }, { "extra-routes": [ { "vrf-id": "262452554464335" }, { "vrf-id": "2975427776581" } ], "vpn-name": "aaa7f0bb-cf03-4e32-ba90-4e2fa49789bd" } ] } } cli>display all-dpns Number of nodes: 3 NodeId NodeName -------------------------------------------------------------------------- 2975427776581 compute-0-2 33387521770575 compute-0-3 262452554464335 compute-0-1 http://localhost:8181/restconf/config/odl-fib:extraroute-rds-map/ { "extraroute-rds-map": { "extraroute-rds": [ { "vpnid": 100000 }, { "dest-prefixes": [ { "allocated-rds": [ { "nexthop": "12.23.34.4", "rd": "100:26" }, { "nexthop": "16.6.67.4", "rd": "100:27" }, { "nexthop": "17.7.6.3", "rd": "100:28" }, { "nexthop": "15.8.9.3", "rd": "100:28" }, { "nexthop": "14.42.62.4", "rd": "100:26" }, { "nexthop": "13.5.4.3", "rd": "100:28" }, { "nexthop": "11.1.2.3", "rd": "100:28" }, { "nexthop": "18.6.6.4", "rd": "100:27" } ], "dest-prefix": "20.0.0.0/24" }, { "allocated-rds": [ { "nexthop": "2234:777:222:18:f816:3eff:fe8f:cfa3", "rd": "100:27" }, { "nexthop": "2078:fde:2cd:16:f816:3eff:fe30:3d70", "rd": "100:27" }, { "nexthop": "2001:db8:abcd:11:f816:3eff:fe47:5af1", "rd": "100:28" }, { "nexthop": "2062:321:543:15:f816:3eff:feb8:9957", "rd": "100:28" }, { "nexthop": "2025:caf:3214:12:f816:3eff:fef1:8ac5", "rd": "100:26" }, { "nexthop": "2056:212:d562:14:f816:3eff:fee9:c346", "rd": "100:27" }, { "nexthop": "2123:13f:565:17:f816:3eff:feb8:26fa", "rd": "100:28" }, { "nexthop": "2033:456a:f012:13:f816:3eff:feeb:b485", "rd": "100:28" } ], "dest-prefix": "2111:12f:862:18::/64" } ], "vpnid": 100047 } ] } }