mlnx-sn2100: syncd crash on config reload and orchagent termination

[nikos-github]

This is with the default port_config.ini and the speed of the ports is changed in config_db.json.

Dec 5 05:18:40.925407 spine-1 INFO syncd#supervisord: syncd Dec 05 05:18:40 NOTICE SAI_RIF: mlnx_sai_rif.c[532]- mlnx_create_router_interface: Create rif, #0 VIRTUAL_ROUTER_ID=VIRTUAL_ROUTER,(0:0),0,0000,0 #1 TYPE=LOOPBACK
Dec 5 05:18:40.926324 spine-1 INFO swss.sh[6499]: 2018-12-05 05:18:40,925 INFO success: buffermgrd entered RUNNING state, process has stayed up for > than 1 seconds (startsecs)
Dec 5 05:18:40.932312 spine-1 INFO swss#supervisord: start.sh buffermgrd: started
Dec 5 05:18:40.935389 spine-1 INFO syncd#supervisord: syncd Dec 05 05:18:40 ERROR EMAD_TRANSPORT: EMAD [2900002AE62946] completed with status : [BAD PARAMETER]
Dec 5 05:18:40.935472 spine-1 ERR syncd#syncd: :- processEvent: attr: SAI_ROUTER_INTERFACE_ATTR_VIRTUAL_ROUTER_ID: oid:0x3000000000002
Dec 5 05:18:40.935535 spine-1 INFO syncd#supervisord: syncd Dec 05 05:18:40 ERROR REG_BULK: sx_reg_layer_sxd_reg_call - Error at access reg call reg_id 0x8002, error: FW RETURN VALUE != SUCCESS
Dec 5 05:18:40.935594 spine-1 INFO syncd#supervisord: syncd Dec 05 05:18:40 ERROR REG_BULK: __reg_bulk_post_completion_cb Error got in completion data sxd_status [15]-[FW RETURN VALUE != SUCCESS].
Dec 5 05:18:40.935646 spine-1 INFO syncd#supervisord: syncd Dec 05 05:18:40 ERROR ROUTER: Failed to RITR set: [FW RETURN VALUE != SUCCESS].
Dec 5 05:18:40.935703 spine-1 NOTICE swss#orchagent: :- handle_switch_shutdown_request: switch shutdown request
Dec 5 05:18:40.935703 spine-1 INFO syncd#supervisord: syncd Dec 05 05:18:40 ERROR ROUTER: Failed to create router interface RITR Failed , err = Driver's Return Status is Non-Zero
Dec 5 05:18:40.935703 spine-1 INFO syncd#supervisord: syncd Dec 05 05:18:40 ERROR ROUTER: Failed to create router interface in HWD layer, err= Driver's Return Status is Non-Zero.
Dec 5 05:18:40.935703 spine-1 INFO syncd#supervisord: syncd Dec 05 05:18:40 ERROR ROUTER: Failed to create rif Loopback, err = Driver's Return Status is Non-Zero
Dec 5 05:18:40.935703 spine-1 ERR syncd#syncd: :- processEvent: attr: SAI_ROUTER_INTERFACE_ATTR_TYPE: SAI_ROUTER_INTERFACE_TYPE_LOOPBACK
Dec 5 05:18:40.935703 spine-1 ERR syncd#syncd: :- processEvent: failed to execute api: create, key: SAI_OBJECT_TYPE_ROUTER_INTERFACE:oid:0x60000000002e1, status: SAI_STATUS_FAILURE
Dec 5 05:18:40.935778 spine-1 INFO syncd#supervisord: syncd Dec 05 05:18:40 ERROR SAI_RIF: mlnx_sai_rif.c[422]- mlnx_rif_sx_init: Failed to create router interface - Driver's Return Status is Non-Zero.
Dec 5 05:18:40.935778 spine-1 ERR syncd#syncd: :- syncd_main: Runtime error: :- processEvent: failed to execute api: create, key: SAI_OBJECT_TYPE_ROUTER_INTERFACE:oid:0x60000000002e1, status: SAI_STATUS_FAILURE
Dec 5 05:18:40.935778 spine-1 NOTICE syncd#syncd: :- exit_and_notify: sending switch_shutdown_request notification to OA
Dec 5 05:18:40.936160 spine-1 NOTICE syncd#syncd: :- exit_and_notify: notification send successfull
Dec 5 05:18:40.936344 spine-1 WARNING syncd#syncd: :- exit_and_notify: sleep forever to keep data plane active
Dec 5 05:18:40.937219 spine-1 INFO swss#supervisord: orchagent terminate called after throwing an instance of 'std::invalid_argument'
Dec 5 05:18:40.937397 spine-1 INFO swss#supervisord: orchagent what(): parse error - unexpected end of input
Dec 5 05:18:41.128649 spine-1 INFO swss.sh[6499]: 2018-12-05 05:18:41,127 INFO exited: orchagent (terminated by SIGABRT (core dumped); not expected)

config_db.json:

root@spine-1:/home/admin# cat /etc/sonic/config_db.json

{
"DEVICE_METADATA": {
"localhost": {
"bgp_asn": "",
"docker_routing_config_mode": "split",
"hostname": "spine-1",
"hwsku": "ACS-MSN2100",
"mac": "ec:0d:9a:4e:5a:00",
"platform": "x86_64-mlnx_msn2100-r0",
"type": "LeafRouter"
}
},
"INTERFACE": {
"Ethernet0|172.16.0.2/31": {},
"Ethernet12|172.16.0.8/31": {},
"Ethernet4|172.16.0.4/31": {},
"Ethernet8|172.16.0.6/31": {}
},
"LOOPBACK_INTERFACE": {
"Loopback0|172.16.0.18/32": {}
},
"PORT": {
"Ethernet0": {
"admin_status": "up",
"description": "facing_leaf-1:Ethernet1/1",
"lanes": "0,1,2,3",
"speed": "40000"
},
"Ethernet12": {
"admin_status": "up",
"description": "facing_leaf-2:Ethernet50/1",
"lanes": "12,13,14,15",
"speed": "40000"
},
"Ethernet16": {
"admin_status": "up",
"description": "",
"lanes": "16,17,18,19",
"speed": "100000"
},
"Ethernet20": {
"admin_status": "up",
"description": "",
"lanes": "20,21,22,23",
"speed": "100000"
},
"Ethernet24": {
"admin_status": "up",
"description": "",
"lanes": "24,25,26,27",
"speed": "100000"
},
"Ethernet28": {
"admin_status": "up",
"description": "",
"lanes": "28,29,30,31",
"speed": "100000"
},
"Ethernet32": {
"admin_status": "up",
"description": "",
"lanes": "32,33,34,35",
"speed": "100000"
},
"Ethernet36": {
"admin_status": "up",
"description": "",
"lanes": "36,37,38,39",
"speed": "100000"
},
"Ethernet4": {
"admin_status": "up",
"description": "facing_leaf-1:Ethernet1/2",
"lanes": "4,5,6,7",
"speed": "40000"
},
"Ethernet40": {
"admin_status": "up",
"description": "",
"lanes": "40,41,42,43",
"speed": "100000"
},
"Ethernet44": {
"admin_status": "up",
"description": "",
"lanes": "44,45,46,47",
"speed": "100000"
},
"Ethernet48": {
"admin_status": "up",
"description": "",
"lanes": "48,49,50,51",
"speed": "100000"
},
"Ethernet52": {
"admin_status": "up",
"description": "",
"lanes": "52,53,54,55",
"speed": "100000"
},
"Ethernet56": {
"admin_status": "up",
"description": "",
"lanes": "56,57,58,59",
"speed": "100000"
},
"Ethernet60": {
"admin_status": "up",
"description": "",
"lanes": "60,61,62,63",
"speed": "100000"
},
"Ethernet8": {
"admin_status": "up",
"description": "facing_leaf-2:Ethernet49/1",
"lanes": "8,9,10,11",
"speed": "40000"
}
},
"PORTCHANNEL": {},
"PORTCHANNEL_INTERFACE": {},
"VLAN": {},
"VLAN_INTERFACE": {},
"VLAN_MEMBER": {}
}
root@spine-1:/home/admin#

root@spine-1:/home/admin# show int status
Interface Lanes Speed MTU Alias Oper Admin

---

Ethernet0 0,1,2,3 40G 9100 Ethernet0 up up
Ethernet4 4,5,6,7 40G 9100 Ethernet4 up up
Ethernet8 8,9,10,11 40G 9100 Ethernet8 up up
Ethernet12 12,13,14,15 40G 9100 Ethernet12 up up
Ethernet16 16,17,18,19 100G 9100 Ethernet16 down up
Ethernet20 20,21,22,23 100G 9100 Ethernet20 down up
Ethernet24 24,25,26,27 100G 9100 Ethernet24 down up
Ethernet28 28,29,30,31 100G 9100 Ethernet28 down up
Ethernet32 32,33,34,35 100G 9100 Ethernet32 down up
Ethernet36 36,37,38,39 100G 9100 Ethernet36 down up
Ethernet40 40,41,42,43 100G 9100 Ethernet40 down up
Ethernet44 44,45,46,47 100G 9100 Ethernet44 down up
Ethernet48 48,49,50,51 100G 9100 Ethernet48 down up
Ethernet52 52,53,54,55 100G 9100 Ethernet52 down up
Ethernet56 56,57,58,59 100G 9100 Ethernet56 down up
Ethernet60 60,61,62,63 100G 9100 Ethernet60 down up