diff --git a/translib/api_tests_app.go b/translib/api_tests_app.go index 6b6e84249b2c..8c7525f2a40b 100644 --- a/translib/api_tests_app.go +++ b/translib/api_tests_app.go @@ -137,6 +137,8 @@ func (app *apiTests) processGet(dbs [db.MaxDB]*db.DB, fmtType TranslibFmtType) ( resp["message"] = app.echoMsg resp["path"] = app.path resp["depth"] = app.depth + resp["content"] = app.content + resp["fields"] = app.fields gr.Payload, err = json.Marshal(&resp) return gr, err diff --git a/translib/app_interface.go b/translib/app_interface.go index e15cc3a4e09d..225332bc12e0 100644 --- a/translib/app_interface.go +++ b/translib/app_interface.go @@ -60,10 +60,20 @@ type appData struct { // These include RESTCONF query parameters like - depth, fields etc. type appOptions struct { - // depth limits subtree levels in the response data. - // 0 indicates unlimited depth. - // Valid for GET API only. - depth uint + // depth limits subtree levels in the response data. + // 0 indicates unlimited depth. + // Valid for GET API only. + depth uint + + // content query parameter value receved from the URI + // possible value is one of 'config', 'nonconfig','all','state' or 'operational' + // Valid for GET API only. + content string + + //fields query parameters + // paths of the fields that needs to be filtered in GET payload response + // Valid for GET API only. + fields []string // deleteEmptyEntry indicates if the db entry should be deleted upon // deletion of last field. This is a non standard option. diff --git a/translib/common_app.go b/translib/common_app.go index 9523a32427ad..770c373acef2 100644 --- a/translib/common_app.go +++ b/translib/common_app.go @@ -419,8 +419,24 @@ func (app *CommonApp) processGet(dbs [db.MaxDB]*db.DB, fmtType TranslibFmtType) origXfmrYgotRoot, _ := ygot.DeepCopy((*app.ygotRoot).(ygot.GoStruct)) isEmptyPayload := false appYgotStruct := (*app.ygotRoot).(ygot.GoStruct) - payload, isEmptyPayload, err = transformer.GetAndXlateFromDB(app.pathInfo.Path, &appYgotStruct, dbs, txCache) + var qParams transformer.QueryParams + qParams, err = transformer.NewQueryParams(app.depth, app.content, app.fields) if err != nil { + log.Warning("transformer.NewQueryParams() returned : ", err) + resPayload = []byte("{}") + break + } + payload, isEmptyPayload, err = transformer.GetAndXlateFromDB(app.pathInfo.Path, &appYgotStruct, dbs, txCache, qParams) + if err != nil { + // target URI for list GET request with QP content!=all and node's content-type mismatches the requested content-type, return empty payload + if isEmptyPayload && qParams.IsContentEnabled() && transformer.IsListNode(app.pathInfo.Path) { + if err.Error() == transformer.QUERY_CONTENT_MISMATCH_ERR { + err = nil + } + } + if err != nil { + log.Warning("transformer.GetAndXlateFromDB() returned : ", err) + } resPayload = payload break } @@ -429,6 +445,12 @@ func (app *CommonApp) processGet(dbs [db.MaxDB]*db.DB, fmtType TranslibFmtType) resPayload = payload break } + if isEmptyPayload && (app.depth == 1) && !transformer.IsLeafNode(app.pathInfo.Path) && !transformer.IsLeafListNode(app.pathInfo.Path) { + // target URI for Container or list GET request with depth = 1, returns empty payload + resPayload = payload + break + } + targetObj, tgtObjCastOk := (*app.ygotTarget).(ygot.GoStruct) if !tgtObjCastOk { /*For ygotTarget populated by tranlib, for query on leaf level and list(without instance) level, @@ -476,13 +498,17 @@ func (app *CommonApp) processGet(dbs [db.MaxDB]*db.DB, fmtType TranslibFmtType) err = tlerr.NotFound("Resource not found") break } - resPayload = payload - log.Info("No data available") - //TODO: Return not found error - //err = tlerr.NotFound("Resource not found") - break + if !qParams.IsEnabled() { + resPayload = payload + log.Info("No data available") + //TODO: Return not found error + //err = tlerr.NotFound("Resource not found") + break + } + } + if !qParams.IsEnabled() { + resYgot = appYgotStruct } - resYgot = appYgotStruct } } if resYgot != nil { diff --git a/translib/transformer/sflow_openconfig_test.go b/translib/transformer/sflow_openconfig_test.go index ae000b313ead..d112cfa8360c 100644 --- a/translib/transformer/sflow_openconfig_test.go +++ b/translib/transformer/sflow_openconfig_test.go @@ -68,7 +68,7 @@ func Test_node_on_openconfig_sflow(t *testing.T) { loadDB(db.ConfigDB, pre_req_map) expected_get_json := "{\"openconfig-sampling-sflow:state\":{\"agent\":\"Ethernet8\",\"enabled\":true,\"polling-interval\":300}}" url = "/openconfig-sampling-sflow:sampling/sflow/state" - t.Run("Test get on sflow node", processGetRequest(url, expected_get_json, false)) + t.Run("Test get on sflow node", processGetRequest(url, nil, expected_get_json, false)) time.Sleep(1 * time.Second) unloadDB(db.ConfigDB, cleanuptbl) t.Log("\n\n+++++++++++++ Done Performing Get on Sflow node ++++++++++++") @@ -154,7 +154,7 @@ func Test_node_openconfig_sflow_collector(t *testing.T) { loadDB(db.ConfigDB, pre_req_map) expected_get_json := "{ \"openconfig-sampling-sflow:collectors\":{\"collector\":[{\"address\":\"3.3.3.3\",\"config\":{\"address\":\"3.3.3.3\",\"network-instance\":\"default\",\"port\":6666},\"network-instance\":\"default\",\"port\":6666,\"state\":{\"address\":\"3.3.3.3\",\"network-instance\":\"default\",\"port\":6666}}]}}" url = "/openconfig-sampling-sflow:sampling/sflow/collectors" - t.Run("Test get on collector node for sflow", processGetRequest(url, expected_get_json, false)) + t.Run("Test get on collector node for sflow", processGetRequest(url, nil, expected_get_json, false)) time.Sleep(1 * time.Second) cleanuptbl = map[string]interface{}{"SFLOW_COLLECTOR": map[string]interface{}{"3.3.3.3_6666_default": ""}} unloadDB(db.ConfigDB, cleanuptbl) @@ -215,7 +215,7 @@ func Test_node_openconfig_sflow_interface(t *testing.T) { loadDB(db.ApplDB, non_pre_req_map) expected_get_json := "{\"openconfig-sampling-sflow:state\":{\"enabled\":true,\"name\":\"Ethernet8\",\"sampling-rate\":30000}}" url = "/openconfig-sampling-sflow:sampling/sflow/interfaces/interface[name=Ethernet8]/state" - t.Run("Test get on sflow interface", processGetRequest(url, expected_get_json, false)) + t.Run("Test get on sflow interface", processGetRequest(url, nil, expected_get_json, false)) time.Sleep(2 * time.Second) cleanuptbl = map[string]interface{}{"SFLOW_SESSION": map[string]interface{}{"Ethernet8": ""}} unloadDB(db.ConfigDB, cleanuptbl) diff --git a/translib/transformer/sflow_sonic_test.go b/translib/transformer/sflow_sonic_test.go index 79081daa3623..7c3a7655ed26 100644 --- a/translib/transformer/sflow_sonic_test.go +++ b/translib/transformer/sflow_sonic_test.go @@ -58,7 +58,7 @@ func Test_node_sonic_sflow(t *testing.T) { // Verify global configurations url = "/sonic-sflow:sonic-sflow/SFLOW/global" url_body_json = "{\"sonic-sflow:global\":{\"admin_state\":\"up\",\"agent_id\":\"Ethernet4\",\"polling_interval\":100}}" - t.Run("Verify sFlow global configurations", processGetRequest(url, url_body_json, false)) + t.Run("Verify sFlow global configurations", processGetRequest(url, nil, url_body_json, false)) //Delete sflow global configurations url = "/sonic-sflow:sonic-sflow/SFLOW" @@ -68,7 +68,7 @@ func Test_node_sonic_sflow(t *testing.T) { //Verify deleted sflow global configuration url = "/sonic-sflow:sonic-sflow/SFLOW" url_body_json = "{}" - t.Run("Verify delete on sflow node", processGetRequest(url, url_body_json, false)) + t.Run("Verify delete on sflow node", processGetRequest(url, nil, url_body_json, false)) } func Test_node_sonic_sflow_collector(t *testing.T) { @@ -82,7 +82,7 @@ func Test_node_sonic_sflow_collector(t *testing.T) { // Verify sFlow collector configurations url = "/sonic-sflow:sonic-sflow/SFLOW_COLLECTOR/SFLOW_COLLECTOR_LIST[name=1.1.1.1_6343_default]" - t.Run("Verify sFlow collector", processGetRequest(url, url_body_json, false)) + t.Run("Verify sFlow collector", processGetRequest(url, nil, url_body_json, false)) // Set collector ip url = "/sonic-sflow:sonic-sflow/SFLOW_COLLECTOR/SFLOW_COLLECTOR_LIST[name=1.1.1.1_6343_default]/collector_ip" @@ -104,7 +104,7 @@ func Test_node_sonic_sflow_collector(t *testing.T) { //Verify collector port url = "/sonic-sflow:sonic-sflow/SFLOW_COLLECTOR/SFLOW_COLLECTOR_LIST" url_body_json = "{}" - t.Run("Verify delete on sFlow collector", processGetRequest(url, url_body_json, false)) + t.Run("Verify delete on sFlow collector", processGetRequest(url, nil, url_body_json, false)) } func Test_node_sonic_sflow_interface(t *testing.T) { @@ -137,7 +137,7 @@ func Test_node_sonic_sflow_interface(t *testing.T) { url_body_json = "{}" err_str := "Resource not found" expected_err := tlerr.NotFoundError{Format: err_str} - t.Run("Verify delete on sFlow Interface", processGetRequest(url, url_body_json, true, expected_err)) + t.Run("Verify delete on sFlow Interface", processGetRequest(url, nil, url_body_json, true, expected_err)) //Delete sflow global configurations url = "/sonic-sflow:sonic-sflow/SFLOW" @@ -147,5 +147,5 @@ func Test_node_sonic_sflow_interface(t *testing.T) { //Verify deleted sflow global configuration url = "/sonic-sflow:sonic-sflow/SFLOW" url_body_json = "{}" - t.Run("Verify delete on sFlow collector", processGetRequest(url, url_body_json, false)) + t.Run("Verify delete on sFlow collector", processGetRequest(url, nil, url_body_json, false)) } diff --git a/translib/transformer/test/openconfig-test-xfmr-annot.yang b/translib/transformer/test/openconfig-test-xfmr-annot.yang index 3190ba6db11c..97e96ba08674 100644 --- a/translib/transformer/test/openconfig-test-xfmr-annot.yang +++ b/translib/transformer/test/openconfig-test-xfmr-annot.yang @@ -22,24 +22,12 @@ module openconfig-test-xfmr-annot { } } - deviation /oc-test-xfmr:test-xfmr/oc-test-xfmr:test-sensor-groups/oc-test-xfmr:test-sensor-group/oc-test-xfmr:config/oc-test-xfmr:id { - deviate add { - sonic-ext:field-transformer "test_sensor_group_id_field_xfmr"; - } - } - deviation /oc-test-xfmr:test-xfmr/oc-test-xfmr:test-sensor-groups/oc-test-xfmr:test-sensor-group/oc-test-xfmr:config/oc-test-xfmr:group-colors { deviate add { sonic-ext:field-name "colors"; } } - deviation /oc-test-xfmr:test-xfmr/oc-test-xfmr:test-sensor-groups/oc-test-xfmr:test-sensor-group/oc-test-xfmr:state/oc-test-xfmr:id { - deviate add { - sonic-ext:field-transformer "test_sensor_group_id_field_xfmr"; - } - } - deviation /oc-test-xfmr:test-xfmr/oc-test-xfmr:test-sensor-groups/oc-test-xfmr:test-sensor-group/oc-test-xfmr:state/oc-test-xfmr:group-colors { deviate add { sonic-ext:field-name "colors"; @@ -90,6 +78,12 @@ module openconfig-test-xfmr-annot { } } + deviation /oc-test-xfmr:test-xfmr/oc-test-xfmr:test-sensor-groups/oc-test-xfmr:test-sensor-group/oc-test-xfmr:test-sensor-components/oc-test-xfmr:test-sensor-component { + deviate add { + sonic-ext:table-name "TEST_SENSOR_COMPONENT_TABLE"; + } + } + deviation /oc-test-xfmr:test-xfmr/oc-test-xfmr:test-sets/oc-test-xfmr:test-set { deviate add { sonic-ext:table-name "TEST_SET_TABLE"; diff --git a/translib/transformer/test/openconfig-test-xfmr.yang b/translib/transformer/test/openconfig-test-xfmr.yang index f04fff4e9551..e0de7db7bd26 100644 --- a/translib/transformer/test/openconfig-test-xfmr.yang +++ b/translib/transformer/test/openconfig-test-xfmr.yang @@ -127,6 +127,7 @@ module openconfig-test-xfmr { } } } + uses test-sensor-components-top; } } } @@ -274,7 +275,9 @@ module openconfig-test-xfmr { } } } + //////////////////////////// + grouping test-sensor-group-config { description "Config parameters related to the test sensor groups"; @@ -370,6 +373,126 @@ module openconfig-test-xfmr { } + /////////////////// + grouping test-sensor-component-config { + description + "Configuration data for sensor-components"; + + leaf name { + type string; + description + "Device name for the sensor component. "; + } + leaf type { + type enumeration { + enum TYPE1 { + description + "Component Type 1."; + } + enum TYPE2 { + description + "Component Type 2."; + } + enum TYPE3 { + description + "Component Type 3."; + } + } + } + leaf version { + type string; + description + "Version of the Component. "; + } + leaf description { + type string; + description + "Description, or comment, for the test sensor component"; + } + } + + grouping test-sensor-component-state { + description + "Operational State data for sensor components"; + leaf mfg-name { + type string; + description + "System-supplied identifier for the manufacturer of the component."; + } + + leaf mfg-date { + type string; + description + "System-supplied representation of the component's + manufacturing date."; + } + + leaf hardware-version { + type string; + description + "For hardware components, this is the hardware revision of + the component."; + } + + leaf firmware-version { + type string; + description + "For hardware components, this is the version of associated + firmware that is running on the component, if applicable."; + } + } + + grouping test-sensor-components-top { + container test-sensor-components { + description + "Enclosing container for test sensor component references"; + + list test-sensor-component { + key "name type version"; + description + "List of sensor component references"; + + leaf name { + type leafref { + path "../config/name"; + } + description + "Reference to the name of component"; + } + + leaf type { + type leafref { + path "../config/type"; + } + description + "Reference to the type of component"; + } + + leaf version { + type leafref { + path "../config/version"; + } + description + "Reference to the version of component"; + } + + container config { + description + "Configuration parameters to configure a sensor component"; + uses test-sensor-component-config; + } + + container state { + config false; + description + "Operational state parameters of a sensor component"; + uses test-sensor-component-config; + uses test-sensor-component-state; + } + } + } + } + grouping interfaces-config { description "Configuration data for interface references"; @@ -424,10 +547,8 @@ module openconfig-test-xfmr { } } } - /////////////////// - // data definition statements container test-xfmr { diff --git a/translib/transformer/test/sonic-test-xfmr.yang b/translib/transformer/test/sonic-test-xfmr.yang index afe30ad0e396..6008baade449 100644 --- a/translib/transformer/test/sonic-test-xfmr.yang +++ b/translib/transformer/test/sonic-test-xfmr.yang @@ -117,6 +117,29 @@ module sonic-test-xfmr { } } + container TEST_SENSOR_COMPONENT_TABLE { + + list TEST_SENSOR_COMPONENT_TABLE_LIST { + key "name type version"; + + leaf name { + type string; + } + + leaf type { + type string; + } + + leaf version { + type string; + } + + leaf description { + type string; + } + } + } + container TEST_SET_TABLE { list TEST_SET_TABLE_LIST { diff --git a/translib/transformer/testxfmryang_test.go b/translib/transformer/testxfmryang_test.go index bf8dc4b2d054..b3951ad9fe58 100644 --- a/translib/transformer/testxfmryang_test.go +++ b/translib/transformer/testxfmryang_test.go @@ -73,7 +73,7 @@ func Test_node_exercising_subtree_xfmr_and_virtual_table(t *testing.T) { loadDB(db.ConfigDB, pre_req_map) expected_get_json := "{\"openconfig-test-xfmr:ingress-test-set\":[{\"config\":{\"set-name\":\"TestSet_03\",\"type\":\"openconfig-test-xfmr:TEST_SET_IPV6\"},\"set-name\":\"TestSet_03\",\"state\":{\"set-name\":\"TestSet_03\",\"type\":\"openconfig-test-xfmr:TEST_SET_IPV6\"},\"type\":\"openconfig-test-xfmr:TEST_SET_IPV6\"}]}" url = "/openconfig-test-xfmr:test-xfmr/interfaces/interface[id=Eth_1]/ingress-test-sets/ingress-test-set[set-name=TestSet_03][type=TEST_SET_IPV6]" - t.Run("Test get on node exercising subtree-xfmr and virtual table.", processGetRequest(url, expected_get_json, false)) + t.Run("Test get on node exercising subtree-xfmr and virtual table.", processGetRequest(url, nil, expected_get_json, false)) cleanuptbl = map[string]interface{}{"TEST_SET_TABLE": map[string]interface{}{"TestSet_03_TEST_SET_IPV6": ""}} unloadDB(db.ConfigDB, cleanuptbl) t.Log("\n\n+++++++++++++ Done Performing Get on Yang Node Exercising Subtree-Xfmr and Virtual Table ++++++++++++") @@ -124,7 +124,7 @@ func Test_node_exercising_tableName_key_and_field_xfmr(t *testing.T) { loadDB(db.ConfigDB, pre_req_map) expected_get_json := "{\"openconfig-test-xfmr:test-sets\":{\"test-set\":[{\"config\":{\"description\":\"TestSet_03Description\",\"name\":\"TestSet_03\",\"type\":\"openconfig-test-xfmr:TEST_SET_IPV6\"},\"name\":\"TestSet_03\",\"state\":{\"description\":\"TestSet_03Description\",\"name\":\"TestSet_03\",\"type\":\"openconfig-test-xfmr:TEST_SET_IPV6\"},\"type\":\"openconfig-test-xfmr:TEST_SET_IPV6\"}]}}" url = "/openconfig-test-xfmr:test-xfmr/test-sets" - t.Run("Test get on node exercising Table-Name, Key-Xfmr and Field-Xfmr.", processGetRequest(url, expected_get_json, false)) + t.Run("Test get on node exercising Table-Name, Key-Xfmr and Field-Xfmr.", processGetRequest(url, nil, expected_get_json, false)) time.Sleep(1 * time.Second) cleanuptbl = map[string]interface{}{"TEST_SET_TABLE": map[string]interface{}{"TestSet_03_TEST_SET_IPV6": ""}} unloadDB(db.ConfigDB, cleanuptbl) @@ -179,7 +179,7 @@ func Test_node_with_child_tableXfmr_keyXfmr_fieldNameXfmrs_nonConfigDB_data(t *t get_expected := "{\"openconfig-test-xfmr:test-sensor-group\":[{\"config\":{\"color-hold-time\":10,\"group-colors\":[\"red\",\"blue\",\"green\"],\"id\":\"test_group_1\"},\"id\":\"test_group_1\",\"state\":{\"color-hold-time\":10,\"counters\":{\"frame-in\":12345,\"frame-out\":678910},\"group-colors\":[\"red\",\"blue\",\"green\"],\"id\":\"test_group_1\"},\"test-sensor-types\":{\"test-sensor-type\":[{\"config\":{\"exclude-filter\":\"filterB\",\"type\":\"sensora_testA\"},\"state\":{\"exclude-filter\":\"filterB\",\"type\":\"sensora_testA\"},\"type\":\"sensora_testA\"}]}}]}" - t.Run("Verify_get_on_node_with_child_table_key_field_xfmrs", processGetRequest(url, get_expected, false)) + t.Run("Verify_get_on_node_with_child_table_key_field_xfmrs", processGetRequest(url, nil, get_expected, false)) // Teardown unloadDB(db.ConfigDB, cleanuptbl) @@ -308,7 +308,7 @@ func Test_sonic_yang_node_operations(t *testing.T) { loadDB(db.CountersDB, prereq) get_expected := "{\"sonic-test-xfmr:TEST_SENSOR_MODE_TABLE\":{\"TEST_SENSOR_MODE_TABLE_LIST\":[{\"description\":\"Test sensor mode\",\"id\":3543,\"mode\":\"mode:testsensor123\"}]}}" - t.Run("Get on Sonic table with key xfmr", processGetRequest(url, get_expected, false)) + t.Run("Get on Sonic table with key xfmr", processGetRequest(url, nil, get_expected, false)) // Teardown unloadDB(db.CountersDB, cleanuptbl) @@ -348,147 +348,519 @@ func Test_leaflist_node(t *testing.T) { unloadDB(db.ConfigDB, cleanuptbl) time.Sleep(1 * time.Second) t.Log("\n\n+++++++++++++ Done Performing Put/Replace on Yang leaf-list Node demonstrating leaf-list contents swap ++++++++++++") +} + +func Test_node_exercising_singleton_container_and_keyname_mapping(t *testing.T) { + var pre_req_map, expected_map, cleanuptbl map[string]interface{} + var url, url_body_json string + t.Log("\n\n+++++++++++++ Performing Set on Yang Node Exercising Mapping to Sonic-Yang Singleton Container and Key-name ++++++++++++") + url = "/openconfig-test-xfmr:test-xfmr/global-sensor" + url_body_json = "{ \"openconfig-test-xfmr:mode\": \"testmode\", \"openconfig-test-xfmr:description\": \"testdescription\"}" + expected_map = map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": map[string]interface{}{"mode": "testmode", "description": "testdescription"}}} + cleanuptbl = map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": ""}} loadDB(db.ConfigDB, pre_req_map) time.Sleep(1 * time.Second) + t.Run("Test set on yang node exercising mapping to sonic singleton conatiner and key-name.", processSetRequest(url, url_body_json, "POST", false, nil)) + time.Sleep(1 * time.Second) + t.Run("Verify set on yang node exercising mapping to sonic-yang singleton conatiner and key-name.", verifyDbResult(rclient, "TEST_SENSOR_GLOBAL|global_sensor", expected_map, false)) + unloadDB(db.ConfigDB, cleanuptbl) + time.Sleep(1 * time.Second) + t.Log("\n\n+++++++++++++ Done Performing Set on Yang Node Exercising Mapping to Sonic-Yang Singleton Container and Key-name ++++++++++++") + t.Log("\n\n+++++++++++++ Performing Delete on Yang Node Exercising Mapping to Sonic-Yang Singleton Container and Key-name ++++++++++++") + pre_req_map = map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": map[string]interface{}{ + "mode": "testmode", + "description": "testdescription"}}} loadDB(db.ConfigDB, pre_req_map) time.Sleep(1 * time.Second) -} - -func Test_node_exercising_singleton_container_and_keyname_mapping(t *testing.T) { - var pre_req_map, expected_map, cleanuptbl map[string]interface{} - var url, url_body_json string - - t.Log("\n\n+++++++++++++ Performing Set on Yang Node Exercising Mapping to Sonic-Yang Singleton Container and Key-name ++++++++++++") - url = "/openconfig-test-xfmr:test-xfmr/global-sensor" - url_body_json = "{ \"openconfig-test-xfmr:mode\": \"testmode\", \"openconfig-test-xfmr:description\": \"testdescription\"}" - expected_map = map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": map[string]interface{}{"mode": "testmode", "description":"testdescription"}}} - cleanuptbl = map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": ""}} - loadDB(db.ConfigDB, pre_req_map) - time.Sleep(1 * time.Second) - t.Run("Test set on yang node exercising mapping to sonic singleton conatiner and key-name.", processSetRequest(url, url_body_json, "POST", false, nil)) - time.Sleep(1 * time.Second) - t.Run("Verify set on yang node exercising mapping to sonic-yang singleton conatiner and key-name.", verifyDbResult(rclient, "TEST_SENSOR_GLOBAL|global_sensor", expected_map, false)) - unloadDB(db.ConfigDB, cleanuptbl) - time.Sleep(1 * time.Second) - t.Log("\n\n+++++++++++++ Done Performing Set on Yang Node Exercising Mapping to Sonic-Yang Singleton Container and Key-name ++++++++++++") - - t.Log("\n\n+++++++++++++ Performing Delete on Yang Node Exercising Mapping to Sonic-Yang Singleton Container and Key-name ++++++++++++") - pre_req_map = map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": map[string]interface{}{ - "mode": "testmode", - "description": "testdescription"}}} - loadDB(db.ConfigDB, pre_req_map) - time.Sleep(1 * time.Second) - url = "/openconfig-test-xfmr:test-xfmr/global-sensor/description" - t.Run("Test delete on node exercising mapping to sonic-yang singleton conatiner and key-name.", processDeleteRequest(url, false)) - time.Sleep(1 * time.Second) - expected_map = map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": map[string]interface{}{ - "mode": "testmode"}}} - t.Run("Verify delete on node exercising mapping to sonic-yang singleton conatiner and key-name.", verifyDbResult(rclient, "TEST_SENSOR_GLOBAL|global_sensor", expected_map, false)) - unloadDB(db.ConfigDB, cleanuptbl) - time.Sleep(1 * time.Second) - t.Log("\n\n+++++++++++++ Done Performing Delete on Yang Node Exercising Mapping to Sonic-Yang Singleton Container and Key-name ++++++++++++") + url = "/openconfig-test-xfmr:test-xfmr/global-sensor/description" + t.Run("Test delete on node exercising mapping to sonic-yang singleton conatiner and key-name.", processDeleteRequest(url, false)) + time.Sleep(1 * time.Second) + expected_map = map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": map[string]interface{}{ + "mode": "testmode"}}} + t.Run("Verify delete on node exercising mapping to sonic-yang singleton conatiner and key-name.", verifyDbResult(rclient, "TEST_SENSOR_GLOBAL|global_sensor", expected_map, false)) + unloadDB(db.ConfigDB, cleanuptbl) + time.Sleep(1 * time.Second) + t.Log("\n\n+++++++++++++ Done Performing Delete on Yang Node Exercising Mapping to Sonic-Yang Singleton Container and Key-name ++++++++++++") t.Log("\n\n+++++++++++++ Performing Get on Yang Node Exercising Mapping to Sonic-Yang Singleton Container and Key-name ++++++++++++") - pre_req_map = map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": map[string]interface{}{ - "mode": "testmode", - "description": "testdescription"}}} - loadDB(db.ConfigDB, pre_req_map) - expected_get_json := "{\"openconfig-test-xfmr:global-sensor\": {\"description\": \"testdescription\",\"mode\": \"testmode\"}}" - url = "/openconfig-test-xfmr:test-xfmr/global-sensor" - t.Run("Test get on node exercising mapping to sonic-yang singleton conatiner and key-name.", processGetRequest(url, expected_get_json, false)) - time.Sleep(1 * time.Second) - unloadDB(db.ConfigDB, cleanuptbl) - t.Log("\n\n+++++++++++++ Done Performing Get on Yang Node Exercising mapping to sonic-yang singleton conatiner and key-name ++++++++++++") + pre_req_map = map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": map[string]interface{}{ + "mode": "testmode", + "description": "testdescription"}}} + loadDB(db.ConfigDB, pre_req_map) + expected_get_json := "{\"openconfig-test-xfmr:global-sensor\": {\"description\": \"testdescription\",\"mode\": \"testmode\"}}" + url = "/openconfig-test-xfmr:test-xfmr/global-sensor" + t.Run("Test get on node exercising mapping to sonic-yang singleton conatiner and key-name.", processGetRequest(url, nil, expected_get_json, false)) + time.Sleep(1 * time.Second) + unloadDB(db.ConfigDB, cleanuptbl) + t.Log("\n\n+++++++++++++ Done Performing Get on Yang Node Exercising mapping to sonic-yang singleton conatiner and key-name ++++++++++++") } func Test_singleton_sonic_yang_node_operations(t *testing.T) { - cleanuptbl := map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": ""}} - url := "/sonic-test-xfmr:sonic-test-xfmr/TEST_SENSOR_GLOBAL" + cleanuptbl := map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": ""}} + url := "/sonic-test-xfmr:sonic-test-xfmr/TEST_SENSOR_GLOBAL" + + t.Log("++++++++++++++ Test_create_on_sonic_singleton_container_yang_node +++++++++++++") + + // Setup - Prerequisite + unloadDB(db.ConfigDB, cleanuptbl) + + // Payload + post_payload := "{ \"sonic-test-xfmr:global_sensor\": { \"mode\": \"testmode\", \"description\": \"testdescp\" }}" + post_sensor_global_expected := map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": map[string]interface{}{"mode": "testmode", "description": "testdescp"}}} + + t.Run("Create on singleton sonic table yang node", processSetRequest(url, post_payload, "POST", false)) + time.Sleep(1 * time.Second) + t.Run("Verify Create on singleton sonic table yang node", verifyDbResult(rclient, "TEST_SENSOR_GLOBAL|global_sensor", post_sensor_global_expected, false)) + + // Teardown + unloadDB(db.ConfigDB, cleanuptbl) + + t.Log("++++++++++++++ Test_patch_on_sonic_singleton_container_node +++++++++++++") + + prereq := map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": map[string]interface{}{"mode": "testmode", "description": "testdescp"}}} + url = "/sonic-test-xfmr:sonic-test-xfmr/TEST_SENSOR_GLOBAL/global_sensor" + + // Setup - Prerequisite + loadDB(db.ConfigDB, prereq) + + // Payload + patch_payload := "{ \"sonic-test-xfmr:global_sensor\": { \"mode\": \"testmode\", \"description\": \"test description\" }}" + patch_sensor_global_expected := map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": map[string]interface{}{"mode": "testmode", "description": "test description"}}} + + t.Run("Patch on singleton sonic container yang node", processSetRequest(url, patch_payload, "PATCH", false)) + time.Sleep(1 * time.Second) + t.Run("Verify patch on singleton sonic container yang node", verifyDbResult(rclient, "TEST_SENSOR_GLOBAL|global_sensor", patch_sensor_global_expected, false)) + + // Teardown + unloadDB(db.ConfigDB, cleanuptbl) + + t.Log("++++++++++++++ Test_replace_on_sonic_singleton_container +++++++++++++") + + url = "/sonic-test-xfmr:sonic-test-xfmr/TEST_SENSOR_GLOBAL/global_sensor/mode" + + // Setup - Prerequisite + loadDB(db.ConfigDB, prereq) + + // Payload + put_payload := "{ \"sonic-test-xfmr:mode\": \"test_mode_1\"}" + put_sensor_global_expected := map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": map[string]interface{}{"mode": "test_mode_1", "description": "testdescp"}}} + + t.Run("Put on singleton sonic yang node", processSetRequest(url, put_payload, "PUT", false)) + time.Sleep(1 * time.Second) + t.Run("Verify put on singleton sonic yang node", verifyDbResult(rclient, "TEST_SENSOR_GLOBAL|global_sensor", put_sensor_global_expected, false)) + + // Teardown + unloadDB(db.ConfigDB, cleanuptbl) + + t.Log("++++++++++++++ Test_delete_on_singleton_sonic_container +++++++++++++") + + url = "/sonic-test-xfmr:sonic-test-xfmr/TEST_SENSOR_GLOBAL/global_sensor" + + // Setup - Prerequisite + loadDB(db.ConfigDB, prereq) + + delete_expected := make(map[string]interface{}) + + t.Run("Delete on singleton sonic container", processDeleteRequest(url, false)) + time.Sleep(1 * time.Second) + t.Run("Verify delete on sonic singleton container", verifyDbResult(rclient, "TEST_SENSOR_GLOBAL|global_sensor", delete_expected, false)) + + // Teardown + unloadDB(db.ConfigDB, cleanuptbl) + + t.Log("++++++++++++++ Test_get_on_sonic_singleton_container +++++++++++++") + + prereq = map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": map[string]interface{}{"mode": "mode_test", "description": "test description for single container"}}} + url = "/sonic-test-xfmr:sonic-test-xfmr/TEST_SENSOR_GLOBAL" + + // Setup - Prerequisite + loadDB(db.ConfigDB, prereq) + + get_expected := "{\"sonic-test-xfmr:TEST_SENSOR_GLOBAL\":{ \"global_sensor\": { \"mode\": \"mode_test\", \"description\": \"test description for single container\" }}}" + t.Run("Get on Sonic singleton container", processGetRequest(url, nil, get_expected, false)) + + // Teardown + unloadDB(db.ConfigDB, cleanuptbl) +} + +// Query parameter UT cases + +func Test_Query_Params_OC_Yang_Get(t *testing.T) { + + var qp queryParamsUT + qp.depth = 3 + + cleanuptbl := map[string]interface{}{"TEST_SENSOR_GROUP": map[string]interface{}{"test_group_1": ""}, "TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": ""}, "TEST_SENSOR_A_TABLE": map[string]interface{}{"test_group_1|sensor_type_a_testA": ""}} + prereq := map[string]interface{}{"TEST_SENSOR_GROUP": map[string]interface{}{"test_group_1": map[string]interface{}{"colors@": "red,blue,green", "color-hold-time": "30"}}} + prereq_cntr := map[string]interface{}{"TEST_SENSOR_GROUP_COUNTERS": map[string]interface{}{"test_group_1": map[string]interface{}{"frame-in": "3435", "frame-out": "3452"}}} + + // Setup - Prerequisite - None + unloadDB(db.ConfigDB, cleanuptbl) + unloadDB(db.CountersDB, prereq_cntr) + loadDB(db.ConfigDB, prereq) + loadDB(db.CountersDB, prereq_cntr) + + t.Log("++++++++++++++ Test_Query_Depth3_Container_Get +++++++++++++") + url := "/openconfig-test-xfmr:test-xfmr" + get_expected := "{}" + t.Run("Test_Query_Depth3_Get", processGetRequest(url, &qp, get_expected, false)) + + t.Log("++++++++++++++ Test_Query_Depth4_Container_Get +++++++++++++") + qp.depth = 4 + get_expected = "{\"openconfig-test-xfmr:test-xfmr\":{\"test-sensor-groups\":{\"test-sensor-group\":[{\"id\":\"test_group_1\"}]}}}" + t.Run("Test_Query_Depth4_Get", processGetRequest(url, &qp, get_expected, false)) + + t.Log("++++++++++++++ Test_Query_Depth8_Container_Get +++++++++++++") + qp.depth = 8 + get_expected = "{\"openconfig-test-xfmr:test-xfmr\":{\"test-sensor-groups\":{\"test-sensor-group\":[{\"config\":{\"color-hold-time\":30,\"group-colors\":[\"red\",\"blue\",\"green\"],\"id\":\"test_group_1\"},\"id\":\"test_group_1\",\"state\":{\"color-hold-time\":30,\"counters\":{\"frame-in\":3435,\"frame-out\":3452},\"group-colors\":[\"red\",\"blue\",\"green\"],\"id\":\"test_group_1\"}}]}}}" + t.Run("Test_Query_Depth8_Get", processGetRequest(url, &qp, get_expected, false)) + + t.Log("++++++++++++++ Test_Query_Depth0_Container_Get +++++++++++++") + qp.depth = 0 + t.Run("Test_Query_Depth0_Get", processGetRequest(url, &qp, get_expected, false)) + + t.Log("++++++++++++++ Test_Query_Depth3_List_Get +++++++++++++") + url = "/openconfig-test-xfmr:test-xfmr/test-sensor-groups/test-sensor-group" + qp.depth = 3 + get_expected = "{\"openconfig-test-xfmr:test-sensor-group\":[{\"config\":{\"color-hold-time\":30,\"group-colors\":[\"red\",\"blue\",\"green\"],\"id\":\"test_group_1\"},\"id\":\"test_group_1\",\"state\":{\"color-hold-time\":30,\"group-colors\":[\"red\",\"blue\",\"green\"],\"id\":\"test_group_1\"}}]}" + t.Run("Test_Query_Depth3_List_Get", processGetRequest(url, &qp, get_expected, false)) + + t.Log("++++++++++++++ Test_Query_Depth3_List_Instance_Get +++++++++++++") + url = "/openconfig-test-xfmr:test-xfmr/test-sensor-groups/test-sensor-group[id=test_group_1]" + t.Run("Test_Query_Depth3_List_Instance_Get", processGetRequest(url, &qp, get_expected, false)) + + t.Log("++++++++++++++ Test_Query_Depth2_Leaf_Get +++++++++++++") + url = "/openconfig-test-xfmr:test-xfmr/test-sensor-groups/test-sensor-group[id=test_group_1]/config/color-hold-time" + qp.depth = 2 + get_expected = "{\"openconfig-test-xfmr:color-hold-time\":30}" + t.Run("Test_Query_Depth2_Leaf_Get", processGetRequest(url, &qp, get_expected, false)) + + t.Log("++++++++++++++ Test_Query_Content_All_Get +++++++++++++") + // Reset Depth + qp.depth = 0 + qp.content = "all" + url = "/openconfig-test-xfmr:test-xfmr/test-sensor-groups/test-sensor-group[id=test_group_1]" + get_expected = "{\"openconfig-test-xfmr:test-sensor-group\":[{\"config\":{\"color-hold-time\":30,\"group-colors\":[\"red\",\"blue\",\"green\"],\"id\":\"test_group_1\"},\"id\":\"test_group_1\",\"state\":{\"color-hold-time\":30,\"counters\":{\"frame-in\":3435,\"frame-out\":3452},\"group-colors\":[\"red\",\"blue\",\"green\"],\"id\":\"test_group_1\"}}]}" + t.Run("Test_Query_Content_All_Get", processGetRequest(url, &qp, get_expected, false)) + + t.Log("++++++++++++++ Test_Query_Content_Config_Get +++++++++++++") + qp.content = "config" + url = "/openconfig-test-xfmr:test-xfmr/test-sensor-groups/test-sensor-group" + get_expected = "{\"openconfig-test-xfmr:test-sensor-group\":[{\"config\":{\"color-hold-time\":30,\"group-colors\":[\"red\",\"blue\",\"green\"],\"id\":\"test_group_1\"},\"id\":\"test_group_1\"}]}" + t.Run("Test_Query_Content_Config_Get", processGetRequest(url, &qp, get_expected, false)) + + t.Log("++++++++++++++ Test_Query_Content_NonConfig_Get +++++++++++++") + qp.content = "nonconfig" + url = "/openconfig-test-xfmr:test-xfmr" + get_expected = "{\"openconfig-test-xfmr:test-xfmr\":{\"test-sensor-groups\":{\"test-sensor-group\":[{\"id\":\"test_group_1\",\"state\":{\"color-hold-time\":30,\"counters\":{\"frame-in\":3435,\"frame-out\":3452},\"group-colors\":[\"red\",\"blue\",\"green\"],\"id\":\"test_group_1\"}}]}}}" + t.Run("Test_Query_Content_NonConfig_Get", processGetRequest(url, &qp, get_expected, false)) + + t.Log("++++++++++++++ Test_Query_Content_Operational_Get +++++++++++++") + qp.content = "operational" + url = "/openconfig-test-xfmr:test-xfmr/test-sensor-groups" + get_expected = "{\"openconfig-test-xfmr:test-sensor-groups\":{\"test-sensor-group\":[{\"id\":\"test_group_1\",\"state\":{\"counters\":{\"frame-in\":3435,\"frame-out\":3452}}}]}}" + t.Run("Test_Query_Content_Operational_Get", processGetRequest(url, &qp, get_expected, false)) + t.Log("++++++++++++++ Test_Query_Content_Mismatch_Leaf_Get +++++++++++++") + qp.content = "config" + url = "/openconfig-test-xfmr:test-xfmr/test-sensor-groups/test-sensor-group[id==test_group_1]/state/counters/frame-in" + get_expected = "{}" + expected_err := tlerr.InvalidArgsError{Format: "Bad Request - requested content type doesn't match content type of terminal node uri."} + t.Run("Test_Query_Content_Mismatch_Leaf_Get", processGetRequest(url, &qp, get_expected, true, expected_err)) + + t.Log("++++++++++++++ Test_Query_Content_Mismatch_Container_Get +++++++++++++") + qp.content = "nonconfig" + url = "/openconfig-test-xfmr:test-xfmr/test-sensor-groups/test-sensor-group[id==test_group_1]/config" + get_expected = "{}" + t.Run("Test_Query_Content_Mismatch_Container_Get1", processGetRequest(url, &qp, get_expected, false)) + qp.content = "config" + url = "/openconfig-test-xfmr:test-xfmr/test-sensor-groups/test-sensor-group[id=test_group_1]/state" + t.Run("Test_Query_Content_Mismatch_Container_Get2", processGetRequest(url, &qp, get_expected, false)) + + t.Log("++++++++++++++ Test_Query_Depth3_Content_Config_List_Get +++++++++++++") + qp.depth = 3 + qp.content = "config" + url = "/openconfig-test-xfmr:test-xfmr/test-sensor-groups/test-sensor-group" + get_expected = "{\"openconfig-test-xfmr:test-sensor-group\":[{\"config\":{\"color-hold-time\":30,\"group-colors\":[\"red\",\"blue\",\"green\"],\"id\":\"test_group_1\"},\"id\":\"test_group_1\"}]}" + t.Run("Test_Query_Depth3_Content_Config_List_Get", processGetRequest(url, &qp, get_expected, false)) + + t.Log("++++++++++++++ Test_Query_Depth4_Content_All_Container_Get +++++++++++++") + qp.depth = 4 + qp.content = "all" + url = "/openconfig-test-xfmr:test-xfmr/test-sensor-groups" + get_expected = "{\"openconfig-test-xfmr:test-sensor-groups\":{\"test-sensor-group\":[{\"config\":{\"color-hold-time\":30,\"group-colors\":[\"red\",\"blue\",\"green\"],\"id\":\"test_group_1\"},\"id\":\"test_group_1\",\"state\":{\"color-hold-time\":30,\"group-colors\":[\"red\",\"blue\",\"green\"],\"id\":\"test_group_1\"}}]}}" + t.Run("Test_Query_Depth4_Content_All_Container_Get", processGetRequest(url, &qp, get_expected, false)) + + t.Log("++++++++++++++ Test_Query_Depth3_Content_Nonconfig_ListInstance_Get +++++++++++++") + qp.depth = 3 + qp.content = "nonconfig" + url = "/openconfig-test-xfmr:test-xfmr/test-sensor-groups/test-sensor-group[id=test_group_1]" + get_expected = "{\"openconfig-test-xfmr:test-sensor-group\":[{\"id\":\"test_group_1\",\"state\":{\"color-hold-time\":30,\"group-colors\":[\"red\",\"blue\",\"green\"],\"id\":\"test_group_1\"}}]}" + t.Run("Test_Query_Depth3_Content_NonConfig_ListInstance_Get", processGetRequest(url, &qp, get_expected, false)) + + t.Log("++++++++++++++ Test_Query_Fields_Leaf_Get +++++++++++++") + // Reset Depth and Content + qp.depth = 0 + qp.content = "" + qp.fields = []string{"config/color-hold-time"} + get_expected = "{\"openconfig-test-xfmr:test-sensor-group\":[{\"config\":{\"color-hold-time\":30},\"id\":\"test_group_1\"}]}" + url = "/openconfig-test-xfmr:test-xfmr/test-sensor-groups/test-sensor-group[id=test_group_1]" + t.Run("Test_Query_Fields_Leaf_Get", processGetRequest(url, &qp, get_expected, false)) + + t.Log("++++++++++++++ Test_Query_Fields_MultiLeaf_Get +++++++++++++") + qp.fields = []string{"state/color-hold-time", "state/counters/frame-in"} + url = "/openconfig-test-xfmr:test-xfmr/test-sensor-groups/test-sensor-group" + get_expected = "{\"openconfig-test-xfmr:test-sensor-group\":[{\"id\":\"test_group_1\",\"state\":{\"color-hold-time\":30,\"counters\":{\"frame-in\":3435}}}]}" + t.Run("Test_Query_Fields_MultiLeaf_Get", processGetRequest(url, &qp, get_expected, false)) + + t.Log("++++++++++++++ Test_Query_Fields_Container_Get +++++++++++++") + qp.fields = []string{"counters"} + url = "/openconfig-test-xfmr:test-xfmr/test-sensor-groups/test-sensor-group[id=test_group_1]/state" + get_expected = "{\"openconfig-test-xfmr:state\":{\"counters\":{\"frame-in\":3435,\"frame-out\":3452}}}" + t.Run("Test_Query_Fields_Container_Get", processGetRequest(url, &qp, get_expected, false)) + + t.Log("++++++++++++++ Test_Query_Fields_Error_IncorrectField_Get +++++++++++++") + qp.fields = []string{"state/color-hold-times"} + url = "/openconfig-test-xfmr:test-xfmr/test-sensor-groups/test-sensor-group[id=test_group_1]" + get_expected = "{}" + expected_err = tlerr.InvalidArgsError{Format: "Invalid field name/path: state/color-hold-times"} + t.Run("Test_Query_Fields_Error_IncorrectField_Get", processGetRequest(url, &qp, get_expected, true, expected_err)) + + t.Log("++++++++++++++ Test_Query_Fields_Error_Leaf_Get +++++++++++++") + qp.fields = []string{"color-hold-time"} + url = "/openconfig-test-xfmr:test-xfmr/test-sensor-groups/test-sensor-group[id=test_group_1]/config/color-hold-time" + get_expected = "{}" + expected_err = tlerr.InvalidArgsError{Format: "Bad Request - fields query parameter specified on a terminal node uri."} + t.Run("Test_Query_Fields_Error_Leaf_Get", processGetRequest(url, &qp, get_expected, true, expected_err)) - t.Log("++++++++++++++ Test_create_on_sonic_singleton_container_yang_node +++++++++++++") + // Teardown + unloadDB(db.ConfigDB, prereq) + unloadDB(db.CountersDB, prereq_cntr) +} - // Setup - Prerequisite - unloadDB(db.ConfigDB, cleanuptbl) +/* sonic yang GET operation query-parameter tests */ +func Test_sonic_yang_content_query_parameter_operations(t *testing.T) { + var qp queryParamsUT + + t.Log("++++++++++++++ Test_content_all_query_parameter_on_sonic_yang +++++++++++++") + prereq_sensor_global := map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": map[string]interface{}{"description": "testdescription"}}} + prereq_sensor_mode := map[string]interface{}{"TEST_SENSOR_MODE_TABLE": map[string]interface{}{"mode:testsensor123:3543": map[string]interface{}{"description": "Test sensor mode"}}} + url := "/sonic-test-xfmr:sonic-test-xfmr" + qp.content = "all" + //Setup + loadDB(db.ConfigDB, prereq_sensor_global) + loadDB(db.CountersDB, prereq_sensor_mode) + get_expected := "{\"sonic-test-xfmr:sonic-test-xfmr\":{\"TEST_SENSOR_GLOBAL\":{\"global_sensor\":{\"description\":\"testdescription\"}},\"TEST_SENSOR_MODE_TABLE\":{\"TEST_SENSOR_MODE_TABLE_LIST\":[{\"description\":\"Test sensor mode\",\"id\":3543,\"mode\":\"mode:testsensor123\"}]}}}" + t.Run("Sonic yang query parameter content=all", processGetRequest(url, &qp, get_expected, false)) + + t.Log("++++++++++++++ Test_content_config_query_parameter_on_sonic_yang +++++++++++++") + url = "/sonic-test-xfmr:sonic-test-xfmr" + qp.content = "config" + get_expected = "{\"sonic-test-xfmr:sonic-test-xfmr\":{\"TEST_SENSOR_GLOBAL\":{\"global_sensor\":{\"description\":\"testdescription\"}}}}" + t.Run("Sonic yang query parameter content=config", processGetRequest(url, &qp, get_expected, false)) - // Payload - post_payload := "{ \"sonic-test-xfmr:global_sensor\": { \"mode\": \"testmode\", \"description\": \"testdescp\" }}" - post_sensor_global_expected := map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": map[string]interface{}{"mode": "testmode", "description": "testdescp"}}} + t.Log("++++++++++++++ Test_content_nonconfig_query_parameter_on_sonic_yang +++++++++++++") + url = "/sonic-test-xfmr:sonic-test-xfmr" + qp.content = "nonconfig" + get_expected = "{\"sonic-test-xfmr:sonic-test-xfmr\":{\"TEST_SENSOR_MODE_TABLE\":{\"TEST_SENSOR_MODE_TABLE_LIST\":[{\"description\":\"Test sensor mode\",\"id\":3543,\"mode\":\"mode:testsensor123\"}]}}}" + t.Run("Sonic yang query parameter content=nonconfig", processGetRequest(url, &qp, get_expected, false)) + + t.Log("++++++++++++++ Test_content_mismatch_error_query_parameter_on_sonic_yang +++++++++++++") + url = "/sonic-test-xfmr:sonic-test-xfmr/TEST_SENSOR_MODE_TABLE/TEST_SENSOR_MODE_TABLE_LIST[id=3543][mode=testsensor123]/description" + qp.content = "config" + get_expected = "{\"sonic-test-xfmr:sonic-test-xfmr\":{\"TEST_SENSOR_GLOBAL\":{\"global_sensor\":{\"description\":\"testdescription\"}},\"TEST_SENSOR_MODE_TABLE\":{\"TEST_SENSOR_MODE_TABLE_LIST\":[{\"description\":\"Test sensor mode\",\"id\":3543,\"mode\":\"mode:testsensor123\"}]}}}" + get_expected = "{}" + exp_err := tlerr.InvalidArgsError{Format: "Bad Request - requested content type doesn't match content type of terminal node uri."} + t.Run("Sonic yang query parameter simple terminal node content mismatch error.", processGetRequest(url, &qp, get_expected, true, exp_err)) + // Teardown + unloadDB(db.ConfigDB, prereq_sensor_global) + unloadDB(db.CountersDB, prereq_sensor_mode) - t.Run("Create on singleton sonic table yang node", processSetRequest(url, post_payload, "POST", false)) - time.Sleep(1 * time.Second) - t.Run("Verify Create on singleton sonic table yang node", verifyDbResult(rclient, "TEST_SENSOR_GLOBAL|global_sensor", post_sensor_global_expected, false)) +} - // Teardown - unloadDB(db.ConfigDB, cleanuptbl) +func Test_sonic_yang_depth_query_parameter_operations(t *testing.T) { + var qp queryParamsUT + + t.Log("++++++++++++++ Test_depth_level_1_query_parameter_on_sonic_yang +++++++++++++") + prereq_sensor_global := map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": map[string]interface{}{"description": "testdescription"}}} + url := "/sonic-test-xfmr:sonic-test-xfmr/TEST_SENSOR_GLOBAL/global_sensor/description" + qp.depth = 1 + //Setup + loadDB(db.ConfigDB, prereq_sensor_global) + get_expected := "{\"sonic-test-xfmr:description\":\"testdescription\"}" + t.Run("Sonic yang query parameter depth=1", processGetRequest(url, &qp, get_expected, false)) + + t.Log("++++++++++++++ Test_depth_level_2_query_parameter_on_sonic_yang +++++++++++++") + url = "/sonic-test-xfmr:sonic-test-xfmr/TEST_SENSOR_GLOBAL/global_sensor" + qp.depth = 2 + get_expected = "{\"sonic-test-xfmr:global_sensor\":{\"description\":\"testdescription\"}}" + t.Run("Sonic yang query parameter depth=2", processGetRequest(url, &qp, get_expected, false)) + + t.Log("++++++++++++++ Test_depth_level_4_query_parameter_on_sonic_yang +++++++++++++") + url = "/sonic-test-xfmr:sonic-test-xfmr" + qp.depth = 4 + get_expected = "{\"sonic-test-xfmr:sonic-test-xfmr\":{\"TEST_SENSOR_GLOBAL\":{\"global_sensor\":{\"description\":\"testdescription\"}}}}" + t.Run("Sonic yang query parameter depth=4", processGetRequest(url, &qp, get_expected, false)) + // Teardown + unloadDB(db.ConfigDB, prereq_sensor_global) +} - t.Log("++++++++++++++ Test_patch_on_sonic_singleton_container_node +++++++++++++") +func Test_sonic_yang_content_plus_depth_query_parameter_operations(t *testing.T) { + var qp queryParamsUT + + prereq_sensor_global := map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": map[string]interface{}{"description": "testdescription"}}} + prereq_sensor_mode := map[string]interface{}{"TEST_SENSOR_MODE_TABLE": map[string]interface{}{"mode:testsensor123:3543": map[string]interface{}{"description": "Test sensor mode"}}} + t.Log("++++++++++++++ Test_content_all_depth_level_4_query_parameter_on_sonic_yang +++++++++++++") + url := "/sonic-test-xfmr:sonic-test-xfmr" + qp.depth = 4 + qp.content = "all" + //Setup + loadDB(db.ConfigDB, prereq_sensor_global) + loadDB(db.CountersDB, prereq_sensor_mode) + get_expected := "{\"sonic-test-xfmr:sonic-test-xfmr\":{\"TEST_SENSOR_GLOBAL\":{\"global_sensor\":{\"description\":\"testdescription\"}},\"TEST_SENSOR_MODE_TABLE\":{\"TEST_SENSOR_MODE_TABLE_LIST\":[{\"description\":\"Test sensor mode\",\"id\":3543,\"mode\":\"mode:testsensor123\"}]}}}" + t.Run("Sonic yang query parameter content=all depth=4", processGetRequest(url, &qp, get_expected, false)) + + t.Log("++++++++++++++ Test_content_config_depth_level_4_query_parameter_on_sonic_yang +++++++++++++") + url = "/sonic-test-xfmr:sonic-test-xfmr" + qp.depth = 4 + qp.content = "config" + get_expected = "{\"sonic-test-xfmr:sonic-test-xfmr\":{\"TEST_SENSOR_GLOBAL\":{\"global_sensor\":{\"description\":\"testdescription\"}}}}" + t.Run("Sonic yang query parameter content=config depth=4", processGetRequest(url, &qp, get_expected, false)) - prereq := map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": map[string]interface{}{"mode": "testmode", "description": "testdescp"}}} - url = "/sonic-test-xfmr:sonic-test-xfmr/TEST_SENSOR_GLOBAL/global_sensor" + t.Log("++++++++++++++ Test_content_nonconfig_depth_level_4_query_parameter_on_sonic_yang +++++++++++++") + url = "/sonic-test-xfmr:sonic-test-xfmr" + qp.depth = 4 + qp.content = "nonconfig" + get_expected = "{\"sonic-test-xfmr:sonic-test-xfmr\":{\"TEST_SENSOR_MODE_TABLE\":{\"TEST_SENSOR_MODE_TABLE_LIST\":[{\"description\":\"Test sensor mode\",\"id\":3543,\"mode\":\"mode:testsensor123\"}]}}}" + t.Run("Sonic yang query parameter content=nonconfig depth=4", processGetRequest(url, &qp, get_expected, false)) + // Teardown + unloadDB(db.ConfigDB, prereq_sensor_global) + unloadDB(db.CountersDB, prereq_sensor_mode) +} - // Setup - Prerequisite - loadDB(db.ConfigDB, prereq) +func Test_sonic_yang_fields_query_parameter_operations(t *testing.T) { + var qp queryParamsUT + qp.fields = make([]string, 0) - // Payload - patch_payload := "{ \"sonic-test-xfmr:global_sensor\": { \"mode\": \"testmode\", \"description\": \"test description\" }}" - patch_sensor_global_expected := map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": map[string]interface{}{"mode": "testmode", "description": "test description"}}} + t.Log("++++++++++++++ Test_fields(single_field)_query_parameter_on_sonic_yang +++++++++++++") + prereq := map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": map[string]interface{}{"description": "testdescription"}}, "TEST_SENSOR_A_TABLE": map[string]interface{}{"test_group_1|sensor_type_a_testA": map[string]interface{}{"exclude_filter": "filter_filterB", "description_a": "test group1 sensor type a descriptionXYZ"}, "test_group_2|sensor_type_a_testB": map[string]interface{}{"exclude_filter": "filter_filterA", "description_a": "test group2 sensor type a descriptionB"}}} + url := "/sonic-test-xfmr:sonic-test-xfmr/TEST_SENSOR_A_TABLE/TEST_SENSOR_A_TABLE_LIST" + qp.fields = append(qp.fields, "exclude_filter") + loadDB(db.ConfigDB, prereq) + get_expected := "{\"sonic-test-xfmr:TEST_SENSOR_A_TABLE_LIST\":[{\"exclude_filter\": \"filter_filterB\",\"id\": \"test_group_1\",\"type\": \"sensor_type_a_testA\" },{\"exclude_filter\": \"filter_filterA\",\"id\": \"test_group_2\",\"type\": \"sensor_type_a_testB\"}]}" + t.Run("Sonic yang query parameter fields(single field) ", processGetRequest(url, &qp, get_expected, false)) - t.Run("Patch on singleton sonic container yang node", processSetRequest(url, patch_payload, "PATCH", false)) - time.Sleep(1 * time.Second) - t.Run("Verify patch on singleton sonic container yang node", verifyDbResult(rclient, "TEST_SENSOR_GLOBAL|global_sensor", patch_sensor_global_expected, false)) + t.Log("++++++++++++++ Test_fields(multiple_fields)_query_parameter_on_sonic_yang +++++++++++++") + url = "/sonic-test-xfmr:sonic-test-xfmr" + qp.fields = make([]string, 0) + qp.fields = append(qp.fields, "TEST_SENSOR_GLOBAL/global_sensor/description", "TEST_SENSOR_A_TABLE") + get_expected = "{\"sonic-test-xfmr:sonic-test-xfmr\": {\"TEST_SENSOR_A_TABLE\": {\"TEST_SENSOR_A_TABLE_LIST\": [{\"description_a\": \"test group1 sensor type a descriptionXYZ\",\"exclude_filter\": \"filter_filterB\",\"id\": \"test_group_1\",\"type\": \"sensor_type_a_testA\"},{\"description_a\": \"test group2 sensor type a descriptionB\",\"exclude_filter\": \"filter_filterA\",\"id\": \"test_group_2\",\"type\": \"sensor_type_a_testB\"}]},\"TEST_SENSOR_GLOBAL\": {\"global_sensor\": {\"description\": \"testdescription\"}}}}" + t.Run("Sonic yang query parameter fields(multiple field) ", processGetRequest(url, &qp, get_expected, false)) + + t.Log("++++++++++++++ Test_invalid_fields_query_parameter_on_sonic_yang +++++++++++++") + url = "/sonic-test-xfmr:sonic-test-xfmr/TEST_SENSOR_GLOBAL" + qp.fields = make([]string, 0) + qp.fields = append(qp.fields, "global_sensor/desc") + get_expected = "{}" + exp_err := tlerr.InvalidArgsError{Format: "Invalid field name/path: global_sensor/desc"} + t.Run("Sonic yang query parameter invalid fields", processGetRequest(url, &qp, get_expected, true, exp_err)) + + t.Log("++++++++++++++ Test_invalid_fields_query_parameter_target_on_sonic_yang +++++++++++++") + url = "/sonic-test-xfmr:sonic-test-xfmr/TEST_SENSOR_A_TABLE/TEST_SENSOR_A_TABLE_LIST[id=test_group_1][type=sensor_type_a_testA]/exclude_filter" + qp.fields = make([]string, 0) + qp.fields = append(qp.fields, "exclude_filter") + get_expected = "{}" + exp_err = tlerr.InvalidArgsError{Format: "Bad Request - fields query parameter specified on a terminal node uri."} + t.Run("Sonic yang invalid fields query parameter request target", processGetRequest(url, &qp, get_expected, true, exp_err)) + // Teardown + unloadDB(db.ConfigDB, prereq) +} - // Teardown - unloadDB(db.ConfigDB, cleanuptbl) +func Test_OC_Sonic_OneOnOne_Composite_KeyMapping(t *testing.T) { - t.Log("++++++++++++++ Test_replace_on_sonic_singleton_container +++++++++++++") + parent_prereq := map[string]interface{}{"TEST_SENSOR_GROUP": map[string]interface{}{"test_group_1": map[string]interface{}{"NULL": "NULL"}}} + prereq := map[string]interface{}{"TEST_SENSOR_COMPONENT_TABLE": map[string]interface{}{"FAN|TYPE1|14.31": map[string]interface{}{"description": "Test fan sensor type1 v14.31"}}} - url = "/sonic-test-xfmr:sonic-test-xfmr/TEST_SENSOR_GLOBAL/global_sensor/mode" + // Setup - Prerequisite + unloadDB(db.ConfigDB, prereq) + loadDB(db.ConfigDB, parent_prereq) - // Setup - Prerequisite - loadDB(db.ConfigDB, prereq) + url := "/openconfig-test-xfmr:test-xfmr/test-sensor-groups/test-sensor-group[id=test_group_1]/test-sensor-components" - // Payload - put_payload := "{ \"sonic-test-xfmr:mode\": \"test_mode_1\"}" - put_sensor_global_expected := map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": map[string]interface{}{"mode": "test_mode_1", "description": "testdescp"}}} + t.Log("++++++++++++++ Test_Set_OC_Sonic_OneOnOne_Composite_KeyMapping +++++++++++++") - t.Run("Put on singleton sonic yang node", processSetRequest(url, put_payload, "PUT", false)) - time.Sleep(1 * time.Second) - t.Run("Verify put on singleton sonic yang node", verifyDbResult(rclient, "TEST_SENSOR_GLOBAL|global_sensor", put_sensor_global_expected, false)) + url_body_json := "{\"openconfig-test-xfmr:test-sensor-component\":[{\"config\":{\"name\":\"FAN\",\"type\":\"TYPE1\",\"version\":\"14.31\",\"description\":\"Test fan sensor type1 v14.31\"},\"name\":\"FAN\",\"type\":\"TYPE1\",\"version\":\"14.31\"}]}" - // Teardown - unloadDB(db.ConfigDB, cleanuptbl) + expected_map := map[string]interface{}{"TEST_SENSOR_COMPONENT_TABLE": map[string]interface{}{"FAN|TYPE1|14.31": map[string]interface{}{"description": "Test fan sensor type1 v14.31"}}} + t.Run("SET on OC_Sonic_OneOnOne_Composite_KeyMapping", processSetRequest(url, url_body_json, "POST", false, nil)) + time.Sleep(1 * time.Second) + t.Run("Test OC-Sonic one-one composite key mapping", verifyDbResult(rclient, "TEST_SENSOR_COMPONENT_TABLE|FAN|TYPE1|14.31", expected_map, false)) - t.Log("++++++++++++++ Test_delete_on_singleton_sonic_container +++++++++++++") + // Teardown + unloadDB(db.ConfigDB, prereq) + unloadDB(db.ConfigDB, parent_prereq) - url = "/sonic-test-xfmr:sonic-test-xfmr/TEST_SENSOR_GLOBAL/global_sensor" + t.Log("++++++++++++++ Test_Get_OC_Sonic_OneOnOne_Composite_KeyMapping +++++++++++++") - // Setup - Prerequisite - loadDB(db.ConfigDB, prereq) + loadDB(db.ConfigDB, parent_prereq) + loadDB(db.ConfigDB, prereq) - delete_expected := make(map[string]interface{}) + url = "/openconfig-test-xfmr:test-xfmr/test-sensor-groups/test-sensor-group[id=test_group_1]/test-sensor-components" - t.Run("Delete on singleton sonic container", processDeleteRequest(url, false)) - time.Sleep(1 * time.Second) - t.Run("Verify delete on sonic singleton container", verifyDbResult(rclient, "TEST_SENSOR_GLOBAL|global_sensor", delete_expected, false)) + get_expected := "{\"openconfig-test-xfmr:test-sensor-components\":{\"test-sensor-component\":[{\"config\":{\"description\":\"Test fan sensor type1 v14.31\",\"name\":\"FAN\",\"type\":\"TYPE1\",\"version\":\"14.31\"},\"name\":\"FAN\",\"state\":{\"description\":\"Test fan sensor type1 v14.31\",\"name\":\"FAN\",\"type\":\"TYPE1\",\"version\":\"14.31\"},\"type\":\"TYPE1\",\"version\":\"14.31\"}]}}" + t.Run("GET on List_OC_Sonic_OneOnOne_Composite_KeyMapping", processGetRequest(url, nil, get_expected, false)) - // Teardown - unloadDB(db.ConfigDB, cleanuptbl) + // Teardown + unloadDB(db.ConfigDB, prereq) + unloadDB(db.ConfigDB, parent_prereq) +} - t.Log("++++++++++++++ Test_get_on_sonic_singleton_container +++++++++++++") +/*Test OC List having config container with leaves, that are referenced by list key-leafs and have no annotation. + Also covers the list's state container that have leaves same as list keys */ +func Test_NodeWithListHavingConfigLeafRefByKey_OC_Yang(t *testing.T) { - prereq = map[string]interface{}{"TEST_SENSOR_GLOBAL": map[string]interface{}{"global_sensor": map[string]interface{}{"mode": "mode_test", "description": "test description for single container"}}} - url = "/sonic-test-xfmr:sonic-test-xfmr/TEST_SENSOR_GLOBAL" + t.Log("++++++++++++++ Test_set_on_OC_yang_node_with_list_having_config_leaf_referenced_by_list_key +++++++++++++") + pre_req := map[string]interface{}{"TEST_SENSOR_GROUP": map[string]interface{}{"test_group_1": map[string]interface{}{"color-hold-time": "10"}}} + url := "/openconfig-test-xfmr:test-xfmr/test-sensor-groups" + // Payload + post_payload := "{\"openconfig-test-xfmr:test-sensor-group\":[ { \"id\" : \"test_group_1\", \"config\": { \"id\": \"test_group_1\"} } ]}" + post_sensor_group_expected := map[string]interface{}{"TEST_SENSOR_GROUP": map[string]interface{}{"test_group_1": map[string]interface{}{"NULL": "NULL", "color-hold-time": "10"}}} + t.Run("Set on OC-Yang node with list having config leaf referenced by list key.", processSetRequest(url, post_payload, "POST", false)) + time.Sleep(1 * time.Second) + t.Run("Verify set on OC-Yang node with list having config leaf referenced by list key.", verifyDbResult(rclient, "TEST_SENSOR_GROUP|test_group_1", post_sensor_group_expected, false)) + // Teardown + unloadDB(db.ConfigDB, pre_req) - // Setup - Prerequisite - loadDB(db.ConfigDB, prereq) + t.Log("++++++++++++++ Test get on OC yang node with list having config leaf referenced by list key and state leaf same as list key +++++++++++++") + url = "/openconfig-test-xfmr:test-xfmr/test-sensor-groups/test-sensor-group" + // Setup - Prerequisite + loadDB(db.ConfigDB, pre_req) + // Payload + get_expected := "{\"openconfig-test-xfmr:test-sensor-group\":[{\"config\":{\"color-hold-time\":10,\"id\":\"test_group_1\"},\"id\":\"test_group_1\",\"state\":{\"color-hold-time\":10,\"id\":\"test_group_1\"}}]}" + t.Run("Verify get on OC yang node with list having config leaf referenced by list key and state leaf same list key", processGetRequest(url, nil, get_expected, false)) + // Teardown + unloadDB(db.ConfigDB, pre_req) - get_expected := "{\"sonic-test-xfmr:TEST_SENSOR_GLOBAL\":{ \"global_sensor\": { \"mode\": \"mode_test\", \"description\": \"test description for single container\" }}}" - t.Run("Get on Sonic singleton container", processGetRequest(url, get_expected, false)) + t.Log("++++++++++++++ GET on OC YANG config container leaf that is referenced by immediate parent list's key and has no app annotations +++++++++++++") + url = "/openconfig-test-xfmr:test-xfmr/test-sensor-groups/test-sensor-group[id=test_group_1]/state/id" + // Setup - Prerequisite + loadDB(db.ConfigDB, pre_req) + // Payload + get_expected = "{\"openconfig-test-xfmr:id\":\"test_group_1\"}" + t.Run("Get on leaf in OC config container, with no app annotation, and is referenced by immediate parent list's key leaf", processGetRequest(url, nil, get_expected, false)) + // Teardown + unloadDB(db.ConfigDB, pre_req) - // Teardown - unloadDB(db.ConfigDB, cleanuptbl) + t.Log("++++++++++++++ GET on OC YANG State container leaf that is same as immediate parent list's key and has no app annotations +++++++++++++") + url = "/openconfig-test-xfmr:test-xfmr/test-sensor-groups/test-sensor-group[id=test_group_1]/state/id" + // Setup - Prerequisite + loadDB(db.ConfigDB, pre_req) + // Payload + get_expected = "{\"openconfig-test-xfmr:id\":\"test_group_1\"}" + t.Run("Get on leaf in OC state container, with no app annotation, and is same as immediate parent list's key leaf", processGetRequest(url, nil, get_expected, false)) + // Teardown + unloadDB(db.ConfigDB, pre_req) } diff --git a/translib/transformer/utils_test.go b/translib/transformer/utils_test.go index 4dd4c20fc475..ef3e0db3b965 100644 --- a/translib/transformer/utils_test.go +++ b/translib/transformer/utils_test.go @@ -30,6 +30,12 @@ import ( "github.com/go-redis/redis/v7" ) +type queryParamsUT struct { + depth uint + content string + fields []string +} + func checkErr(t *testing.T, err error, expErr error) { if err.Error() != expErr.Error() { t.Fatalf("Error %v, Expect Err: %v", err, expErr) @@ -38,12 +44,22 @@ func checkErr(t *testing.T, err error, expErr error) { } } -func processGetRequest(url string, expectedRespJson string, errorCase bool, expErr ...error) func(*testing.T) { +func processGetRequest(url string, qparams *queryParamsUT, expectedRespJson string, errorCase bool, expErr ...error) func(*testing.T) { return func(t *testing.T) { var expectedMap map[string]interface{} var receivedMap map[string]interface{} + var qp QueryParameters - response, err := Get(GetRequest{Path: url, User: UserRoles{Name: "admin", Roles: []string{"admin"}} }) + qp.Depth = 0 + qp.Content = "" + qp.Fields = make([]string, 0) + + if qparams != nil { + qp.Depth = qparams.depth + qp.Content = qparams.content + qp.Fields = qparams.fields + } + response, err := Get(GetRequest{Path: url, User: UserRoles{Name: "admin", Roles: []string{"admin"}}, QueryParams: qp}) if err != nil { if !errorCase { t.Fatalf("Error %v received for Url: %s", err, url) diff --git a/translib/transformer/xconst.go b/translib/transformer/xconst.go index 3bbfded343c5..1aae4d9e5c0e 100644 --- a/translib/transformer/xconst.go +++ b/translib/transformer/xconst.go @@ -32,6 +32,12 @@ const ( IANA_MDL_PFX = "iana-" PATH_XFMR_RET_ARGS = 1 PATH_XFMR_RET_ERR_INDX = 0 + + YANG_CONTAINER_NM_CONFIG = "config" + CONFIG_CNT_SUFFIXED_XPATH = "/config" + STATE_CNT_SUFFIXED_XPATH = "/state" + CONFIG_CNT_WITHIN_XPATH = "/config/" + STATE_CNT_WITHIN_XPATH = "/state/" ) const ( @@ -52,6 +58,18 @@ const ( XFMR_DEFAULT_ENABLE ) +const ( + QUERY_CONTENT_ALL ContentType = iota + QUERY_CONTENT_CONFIG + QUERY_CONTENT_NONCONFIG + QUERY_CONTENT_OPERATIONAL +) + +const ( + QUERY_CONTENT_MISMATCH_ERR = "Query Parameter Content mismatch" + QUERY_PARAMETER_SBT_PRUNING_ERR = "Query Parameter processing unsuccessful" +) + const ( GET Operation = iota + 1 CREATE diff --git a/translib/transformer/xfmr_interface.go b/translib/transformer/xfmr_interface.go index f3e4cd2d200b..7c4a63c7f86d 100644 --- a/translib/transformer/xfmr_interface.go +++ b/translib/transformer/xfmr_interface.go @@ -51,6 +51,7 @@ type XfmrParams struct { isVirtualTbl *bool pCascadeDelTbl *[]string //used to populate list of tables needed cascade delete by subtree overloaded methods yangDefValMap map[string]map[string]db.Value + queryParams QueryParams invokeCRUSubtreeOnce *bool } diff --git a/translib/transformer/xfmr_testxfmr_callbacks.go b/translib/transformer/xfmr_testxfmr_callbacks.go index e310c6d2131b..c4acdc5476dd 100644 --- a/translib/transformer/xfmr_testxfmr_callbacks.go +++ b/translib/transformer/xfmr_testxfmr_callbacks.go @@ -19,7 +19,6 @@ //go:build xfmrtest // +build xfmrtest - package transformer import ( @@ -50,7 +49,6 @@ func init() { XlateFuncBind("DbToYang_test_set_key_xfmr", DbToYang_test_set_key_xfmr) // Key leafrefed Field transformer functions - XlateFuncBind("DbToYang_test_sensor_group_id_field_xfmr", DbToYang_test_sensor_group_id_field_xfmr) XlateFuncBind("DbToYang_test_sensor_type_field_xfmr", DbToYang_test_sensor_type_field_xfmr) XlateFuncBind("DbToYang_test_set_name_field_xfmr", DbToYang_test_set_name_field_xfmr) @@ -68,7 +66,7 @@ func init() { XlateFuncBind("Subscribe_test_port_bindings_xfmr", Subscribe_test_port_bindings_xfmr) // Sonic yang Key transformer functions - XlateFuncBind("DbToYang_test_sensor_mode_key_xfmr", DbToYang_test_sensor_mode_key_xfmr) + XlateFuncBind("DbToYang_test_sensor_mode_key_xfmr", DbToYang_test_sensor_mode_key_xfmr) } const ( @@ -109,24 +107,24 @@ var test_pre_xfmr PreXfmrFunc = func(inParams XfmrParams) error { var test_post_xfmr PostXfmrFunc = func(inParams XfmrParams) (map[string]map[string]db.Value, error) { - pathInfo := NewPathInfo(inParams.uri) - groupId := pathInfo.Var("id") + pathInfo := NewPathInfo(inParams.uri) + groupId := pathInfo.Var("id") retDbDataMap := (*inParams.dbDataMap)[inParams.curDb] log.Info("Entering test_post_xfmr Request URI path = ", inParams.requestUri) if inParams.oper == UPDATE { xpath, _, _ := XfmrRemoveXPATHPredicates(inParams.requestUri) if xpath == "/openconfig-test-xfmr:test-xfmr/test-sensor-groups/test-sensor-group/config/color-hold-time" { - holdTime := retDbDataMap["TEST_SENSOR_GROUP"][groupId].Field["color-hold-time"] - key := groupId + "|" + "sensor_type_a_post" + holdTime - subOpCreateMap := make(map[db.DBNum]map[string]map[string]db.Value) - subOpCreateMap[db.ConfigDB] = make(map[string]map[string]db.Value) - subOpCreateMap[db.ConfigDB]["TEST_SENSOR_A_TABLE"] = make(map[string]db.Value) - subOpCreateMap[db.ConfigDB]["TEST_SENSOR_A_TABLE"][key] = db.Value{Field: make(map[string]string)} - subOpCreateMap[db.ConfigDB]["TEST_SENSOR_A_TABLE"][key].Field["description_a"] = "Added instance in post xfmr" - inParams.subOpDataMap[CREATE] = &subOpCreateMap - } - } + holdTime := retDbDataMap["TEST_SENSOR_GROUP"][groupId].Field["color-hold-time"] + key := groupId + "|" + "sensor_type_a_post" + holdTime + subOpCreateMap := make(map[db.DBNum]map[string]map[string]db.Value) + subOpCreateMap[db.ConfigDB] = make(map[string]map[string]db.Value) + subOpCreateMap[db.ConfigDB]["TEST_SENSOR_A_TABLE"] = make(map[string]db.Value) + subOpCreateMap[db.ConfigDB]["TEST_SENSOR_A_TABLE"][key] = db.Value{Field: make(map[string]string)} + subOpCreateMap[db.ConfigDB]["TEST_SENSOR_A_TABLE"][key].Field["description_a"] = "Added instance in post xfmr" + inParams.subOpDataMap[CREATE] = &subOpCreateMap + } + } return retDbDataMap, nil } @@ -262,19 +260,6 @@ var DbToYang_test_set_key_xfmr KeyXfmrDbToYang = func(inParams XfmrParams) (map[ } -var DbToYang_test_sensor_group_id_field_xfmr FieldXfmrDbtoYang = func(inParams XfmrParams) (map[string]interface{}, error) { - var err error - result := make(map[string]interface{}) - log.Info("DbToYang_test_sensor_group_id_field_xfmr - inParams.uri ", inParams.uri) - - if len(inParams.key) > 0 { - result["id"] = inParams.key - } - log.Info("DbToYang_test_sensor_group_id_field_xfmr returns ", result) - - return result, err -} - var DbToYang_test_sensor_type_field_xfmr FieldXfmrDbtoYang = func(inParams XfmrParams) (map[string]interface{}, error) { var err error result := make(map[string]interface{}) @@ -482,7 +467,7 @@ var YangToDb_test_port_bindings_xfmr SubTreeXfmrYangToDb = func(inParams XfmrPar testSetName := getTestSetKeyStrFromOCKey(inTestSetKey.SetName, inTestSetKey.Type) testSetInterfacesMap[testSetName] = append(testSetInterfacesMap[testSetName], *intf.Id) _, ok := testSetTableMap[testSetName] - if !ok && inParams.oper == DELETE { + if !ok && inParams.oper == DELETE { return res_map, tlerr.NotFound("Binding not found for test set %v on %v", inTestSetKey.SetName, *intf.Id) } if inParams.oper == DELETE { @@ -687,20 +672,20 @@ func convertSonicTestSetTypeToOC(testSetType string) ocbinds.E_OpenconfigTestXfm //Sonic yang key transformer functions var DbToYang_test_sensor_mode_key_xfmr SonicKeyXfmrDbToYang = func(inParams SonicXfmrParams) (map[string]interface{}, error) { - res_map := make(map[string]interface{}) - /* from DB-key string(inParams.key) extract mode and id to fill into the res_map + res_map := make(map[string]interface{}) + /* from DB-key string(inParams.key) extract mode and id to fill into the res_map * db key contains the separator as well eg: "mode:test123:3545" - */ - log.Info("DbToYang_test_sensor_mode_key_xfmr: key", inParams.key) - if len(inParams.key) > 0 { - /*split id and mode */ + */ + log.Info("DbToYang_test_sensor_mode_key_xfmr: key", inParams.key) + if len(inParams.key) > 0 { + /*split id and mode */ temp := strings.SplitN(inParams.key, ":", 3) if len(temp) >= 3 { res_map["mode"] = temp[0] + ":" + temp[1] - id := temp[2] + id := temp[2] i64, _ := strconv.ParseUint(id, 10, 32) i32 := uint32(i64) - res_map["id"] = i32 + res_map["id"] = i32 } else if len(temp) == 2 { res_map["mode"] = temp[0] res_map["id"] = temp[1] @@ -708,7 +693,7 @@ var DbToYang_test_sensor_mode_key_xfmr SonicKeyXfmrDbToYang = func(inParams Soni errStr := "Invalid Key in uri." return res_map, tlerr.InvalidArgsError{Format: errStr} } - } - log.Info("DbToYang_test_sensor_mode_key_xfmr: res_map - ", res_map) - return res_map, nil + } + log.Info("DbToYang_test_sensor_mode_key_xfmr: res_map - ", res_map) + return res_map, nil } diff --git a/translib/transformer/xlate.go b/translib/transformer/xlate.go index e6aaf51a9e2e..fdc7c2beb5b5 100644 --- a/translib/transformer/xlate.go +++ b/translib/transformer/xlate.go @@ -196,7 +196,7 @@ func updateDbDataMapAndKeyCache(dbKeyStr string, data *db.Value, spec *KeySpec, dbTblKeyGetCache[spec.DbNum][spec.Ts.Name][dbKeyStr] = readOk } -func XlateUriToKeySpec(uri string, requestUri string, ygRoot *ygot.GoStruct, t *interface{}, txCache interface{}) (*[]KeySpec, error) { +func XlateUriToKeySpec(uri string, requestUri string, ygRoot *ygot.GoStruct, t *interface{}, txCache interface{}, qParams QueryParams) (*[]KeySpec, error) { var err error var retdbFormat = make([]KeySpec, 0) @@ -214,17 +214,23 @@ func XlateUriToKeySpec(uri string, requestUri string, ygRoot *ygot.GoStruct, t * } } - retdbFormat = fillSonicKeySpec(xpath, tableName, keyStr) + retdbFormat = fillSonicKeySpec(xpath, tableName, keyStr, qParams.content) } else { + var reqUriXpath string /* Extract the xpath and key from input xpath */ retData, _ := xpathKeyExtract(nil, ygRoot, GET, uri, requestUri, nil, nil, txCache, nil) - retdbFormat = fillKeySpecs(retData.xpath, retData.dbKey, &retdbFormat) + if requestUri == uri { + reqUriXpath = retData.xpath + } else { + reqUriXpath, _, _ = XfmrRemoveXPATHPredicates(requestUri) + } + retdbFormat = fillKeySpecs(reqUriXpath, &qParams, retData.xpath, retData.dbKey, &retdbFormat) } return &retdbFormat, err } -func fillKeySpecs(yangXpath string, keyStr string, retdbFormat *[]KeySpec) []KeySpec { +func fillKeySpecs(reqUriXpath string, qParams *QueryParams, yangXpath string, keyStr string, retdbFormat *[]KeySpec) []KeySpec { var err error if xYangSpecMap == nil { return *retdbFormat @@ -261,8 +267,10 @@ func fillKeySpecs(yangXpath string, keyStr string, retdbFormat *[]KeySpec) []Key if chlen > 0 { children := make([]KeySpec, 0) for _, childXpath := range xDbSpecMap[child].yangXpath { - children = fillKeySpecs(childXpath, "", &children) - dbFormat.Child = append(dbFormat.Child, children...) + if isChildTraversalRequired(reqUriXpath, qParams, childXpath) { + children = fillKeySpecs(reqUriXpath, qParams, childXpath, "", &children) + dbFormat.Child = append(dbFormat.Child, children...) + } } } } @@ -276,7 +284,9 @@ func fillKeySpecs(yangXpath string, keyStr string, retdbFormat *[]KeySpec) []Key chlen := len(xDbSpecMap[child].yangXpath) if chlen > 0 { for _, childXpath := range xDbSpecMap[child].yangXpath { - *retdbFormat = fillKeySpecs(childXpath, "", retdbFormat) + if isChildTraversalRequired(reqUriXpath, qParams, childXpath) { + *retdbFormat = fillKeySpecs(reqUriXpath, qParams, childXpath, "", retdbFormat) + } } } } @@ -287,7 +297,7 @@ func fillKeySpecs(yangXpath string, keyStr string, retdbFormat *[]KeySpec) []Key return *retdbFormat } -func fillSonicKeySpec(xpath string, tableName string, keyStr string) []KeySpec { +func fillSonicKeySpec(xpath string, tableName string, keyStr string, content ContentType) []KeySpec { var retdbFormat = make([]KeySpec, 0) @@ -296,6 +306,9 @@ func fillSonicKeySpec(xpath string, tableName string, keyStr string) []KeySpec { dbFormat.Ts.Name = tableName cdb := db.ConfigDB if _, ok := xDbSpecMap[tableName]; ok { + if (xDbSpecMap[tableName].dbEntry == nil) || ((content == QUERY_CONTENT_CONFIG) && (xDbSpecMap[tableName].dbEntry.ReadOnly())) || ((content == QUERY_CONTENT_NONCONFIG) && (!xDbSpecMap[tableName].dbEntry.ReadOnly())) { + return retdbFormat + } cdb = xDbSpecMap[tableName].dbIndex } dbFormat.DbNum = cdb @@ -313,6 +326,9 @@ func fillSonicKeySpec(xpath string, tableName string, keyStr string) []KeySpec { for dir := range dbInfo.dbEntry.Dir { _, ok := xDbSpecMap[dir] if ok && xDbSpecMap[dir].yangType == YANG_CONTAINER { + if (xDbSpecMap[dir].dbEntry == nil) || ((content == QUERY_CONTENT_CONFIG) && (xDbSpecMap[dir].dbEntry.ReadOnly())) || ((content == QUERY_CONTENT_NONCONFIG) && (!xDbSpecMap[dir].dbEntry.ReadOnly())) { + continue + } cdb := xDbSpecMap[dir].dbIndex dbFormat := KeySpec{} dbFormat.Ts.Name = dir @@ -387,14 +403,42 @@ func XlateToDb(path string, oper int, d *db.DB, yg *ygot.GoStruct, yt *interface return result, yangDefValMap, yangAuxValMap, err } -func GetAndXlateFromDB(uri string, ygRoot *ygot.GoStruct, dbs [db.MaxDB]*db.DB, txCache interface{}) ([]byte, bool, error) { +func GetAndXlateFromDB(uri string, ygRoot *ygot.GoStruct, dbs [db.MaxDB]*db.DB, txCache interface{}, qParams QueryParams) ([]byte, bool, error) { var err error var payload []byte var inParamsForGet xlateFromDbParams + var processReq bool + inParamsForGet.queryParams = qParams xfmrLogInfo("received xpath = ", uri) requestUri := uri - keySpec, _ := XlateUriToKeySpec(uri, requestUri, ygRoot, nil, txCache) + if len(qParams.fields) > 0 { + xfmrLogDebug("Process fields QP") //todo + yngNdType, nd_err := getYangNodeTypeFromUri(uri) + if nd_err != nil { + return []byte("{}"), false, nd_err + } else if (yngNdType == YANG_LEAF) || (yngNdType == YANG_LEAF_LIST) { + err = tlerr.InvalidArgsError{Format: "Bad Request - fields query parameter specified on a terminal node uri."} + return []byte("{}"), false, err + } + } else { + processReq, err = contentQParamTgtEval(uri, qParams) + if err != nil { + return []byte("{}"), false, err + } + if !processReq { + xfmrLogInfo("further processing of request not needed due to content query param.") + /* translib fills requested list-instance into ygot, but when there is content-mismatch + we have to send empty payload response.So distinguish this case in common_app we send this err + */ + if IsListNode(uri) { + return []byte("{}"), true, tlerr.InternalError{Format: QUERY_CONTENT_MISMATCH_ERR} + } + return []byte("{}"), true, err + } + } + + keySpec, _ := XlateUriToKeySpec(uri, requestUri, ygRoot, nil, txCache, qParams) var dbresult = make(RedisDbMap) for i := db.ApplDB; i < db.MaxDB; i++ { dbresult[i] = make(map[string]map[string]db.Value) @@ -510,7 +554,8 @@ func XlateFromDb(uri string, ygRoot *ygot.GoStruct, dbs [db.MaxDB]*db.DB, data R } } dbTblKeyGetCache := inParamsForGet.dbTblKeyGetCache - inParamsForGet = formXlateFromDbParams(dbs[cdb], dbs, cdb, ygRoot, uri, requestUri, xpath, GET, "", "", &dbData, txCache, nil, false) + qparams := inParamsForGet.queryParams + inParamsForGet = formXlateFromDbParams(dbs[cdb], dbs, cdb, ygRoot, uri, requestUri, xpath, GET, "", "", &dbData, txCache, nil, false, qparams, nil) inParamsForGet.xfmrDbTblKeyCache = make(map[string]tblKeyCache) inParamsForGet.dbTblKeyGetCache = dbTblKeyGetCache payload, isEmptyPayload, err := dbDataToYangJsonCreate(inParamsForGet) @@ -726,6 +771,15 @@ func IsLeafListNode(uri string) bool { return result } +func IsListNode(uri string) bool { + result := false + yngNdType, err := getYangNodeTypeFromUri(uri) + if (err == nil) && (yngNdType == YANG_LIST) { + result = true + } + return result +} + func tableKeysToBeSorted(tblNm string) bool { /* function to decide whether to sort table keys. Required when a sonic table has more than 1 lists diff --git a/translib/transformer/xlate_datastructs.go b/translib/transformer/xlate_datastructs.go index 1f8868a67db8..25a7686f3af8 100644 --- a/translib/transformer/xlate_datastructs.go +++ b/translib/transformer/xlate_datastructs.go @@ -93,7 +93,9 @@ type xlateFromDbParams struct { resultMap map[string]interface{} validate bool xfmrDbTblKeyCache map[string]tblKeyCache + queryParams QueryParams dbTblKeyGetCache map[db.DBNum]map[string]map[string]bool + listKeysMap map[string]interface{} } type xlateToParams struct { @@ -122,8 +124,35 @@ type xlateToParams struct { invokeCRUSubtreeOnceMap map[string]map[string]bool } +type contentQPSpecMapInfo struct { + yangType yangElementType + yangName string + isReadOnly bool + isOperationalNd bool + hasNonTerminalNd bool + hasChildOperationalNd bool + isOcMdl bool +} + +type qpSubtreePruningErr struct { + subtreePath string +} + type Operation int + +type ContentType uint8 + +type QueryParams struct { + depthEnabled bool + curDepth uint + content ContentType + fields []string + fieldsFillAll bool + allowFieldsXpath map[string]bool + tgtFieldsXpathMap map[string][]string +} + type ygotUnMarshalCtx struct { ygParentObj *ygot.GoStruct relUri string @@ -135,4 +164,5 @@ type ygotUnMarshalCtx struct { type ygotXlator struct { ygotCtx *ygotUnMarshalCtx -} \ No newline at end of file +} + diff --git a/translib/transformer/xlate_from_db.go b/translib/transformer/xlate_from_db.go index 18342fc6b6e2..76d204af1c6b 100644 --- a/translib/transformer/xlate_from_db.go +++ b/translib/transformer/xlate_from_db.go @@ -247,6 +247,15 @@ func sonicDbToYangTerminalNodeFill(field string, inParamsForGet xlateFromDbParam resField := field value := "" + if len(inParamsForGet.queryParams.fields) > 0 { + curFldXpath := inParamsForGet.tbl + "/" + field + if _, ok := inParamsForGet.queryParams.tgtFieldsXpathMap[curFldXpath]; !ok { + if !inParamsForGet.queryParams.fieldsFillAll { + return + } + } + } + if inParamsForGet.dbDataMap != nil { tblInstFields, dbDataExists := (*inParamsForGet.dbDataMap)[inParamsForGet.curDb][inParamsForGet.tbl][inParamsForGet.tblKey] if dbDataExists { @@ -311,7 +320,7 @@ func sonicDbToYangListFill(inParamsForGet xlateFromDbParams) []typeMapOfInterfac curMap := make(map[string]interface{}) sonicKeyDataAdd(dbIdx, yangKeys, table, xDbSpecMap[xpath].dbEntry.Name, keyStr, curMap) if len(curMap) > 0 { - linParamsForGet := formXlateFromDbParams(inParamsForGet.dbs[dbIdx], inParamsForGet.dbs, dbIdx, inParamsForGet.ygRoot, inParamsForGet.uri, inParamsForGet.requestUri, xpath, inParamsForGet.oper, table, keyStr, dbDataMap, inParamsForGet.txCache, curMap, inParamsForGet.validate) + linParamsForGet := formXlateFromDbParams(inParamsForGet.dbs[dbIdx], inParamsForGet.dbs, dbIdx, inParamsForGet.ygRoot, inParamsForGet.uri, inParamsForGet.requestUri, xpath, inParamsForGet.oper, table, keyStr, dbDataMap, inParamsForGet.txCache, curMap, inParamsForGet.validate, inParamsForGet.queryParams, nil) sonicDbToYangDataFill(linParamsForGet) curMap = linParamsForGet.resultMap dbDataMap = linParamsForGet.dbDataMap @@ -364,7 +373,7 @@ func sonicDbToYangDataFill(inParamsForGet xlateFromDbParams) { fldName = fldName + "@" } curUri := inParamsForGet.uri + "/" + yangChldName - linParamsForGet := formXlateFromDbParams(nil, inParamsForGet.dbs, dbIdx, inParamsForGet.ygRoot, curUri, inParamsForGet.requestUri, chldXpath, inParamsForGet.oper, table, key, dbDataMap, inParamsForGet.txCache, resultMap, inParamsForGet.validate) + linParamsForGet := formXlateFromDbParams(nil, inParamsForGet.dbs, dbIdx, inParamsForGet.ygRoot, curUri, inParamsForGet.requestUri, chldXpath, inParamsForGet.oper, table, key, dbDataMap, inParamsForGet.txCache, resultMap, inParamsForGet.validate, inParamsForGet.queryParams, nil) dbEntry := yangNode.dbEntry.Dir[yangChldName] sonicDbToYangTerminalNodeFill(fldName, linParamsForGet, dbEntry) resultMap = linParamsForGet.resultMap @@ -372,14 +381,37 @@ func sonicDbToYangDataFill(inParamsForGet xlateFromDbParams) { } else if chldYangType == YANG_CONTAINER { curMap := make(map[string]interface{}) curUri := uri + "/" + yangChldName + if len(inParamsForGet.queryParams.fields) > 0 { + if _, ok := inParamsForGet.queryParams.tgtFieldsXpathMap[chldXpath]; ok { + inParamsForGet.queryParams.fieldsFillAll = true + } else if _, ok := inParamsForGet.queryParams.allowFieldsXpath[chldXpath]; !ok { + if !inParamsForGet.queryParams.fieldsFillAll { + for path := range inParamsForGet.queryParams.tgtFieldsXpathMap { + if strings.HasPrefix(chldXpath, path) { + inParamsForGet.queryParams.fieldsFillAll = true + } + } + if !inParamsForGet.queryParams.fieldsFillAll { + continue + } + } + } + } // container can have a static key, so extract key for current container _, curKey, curTable := sonicXpathKeyExtract(curUri) if _, specmapOk := xDbSpecMap[curTable]; !specmapOk || xDbSpecMap[curTable].dbEntry == nil { xfmrLogDebug("Yang entry not found for %v", curTable) continue } + if inParamsForGet.queryParams.content != QUERY_CONTENT_ALL { + processReq, _ := sonicContentQParamYangNodeProcess(curUri, xDbSpecMap[curTable].yangType, xDbSpecMap[curTable].dbEntry.ReadOnly(), inParamsForGet.queryParams) + if !processReq { + xfmrLogDebug("Further traversal not needed due to content query param, for of URI - %v", curUri) + continue + } + } d := inParamsForGet.dbs[xDbSpecMap[curTable].dbIndex] - linParamsForGet := formXlateFromDbParams(d, inParamsForGet.dbs, xDbSpecMap[curTable].dbIndex, inParamsForGet.ygRoot, curUri, inParamsForGet.requestUri, chldXpath, inParamsForGet.oper, curTable, curKey, dbDataMap, inParamsForGet.txCache, curMap, inParamsForGet.validate) + linParamsForGet := formXlateFromDbParams(d, inParamsForGet.dbs, xDbSpecMap[curTable].dbIndex, inParamsForGet.ygRoot, curUri, inParamsForGet.requestUri, chldXpath, inParamsForGet.oper, curTable, curKey, dbDataMap, inParamsForGet.txCache, curMap, inParamsForGet.validate, inParamsForGet.queryParams, nil) sonicDbToYangDataFill(linParamsForGet) curMap = linParamsForGet.resultMap dbDataMap = linParamsForGet.dbDataMap @@ -391,6 +423,7 @@ func sonicDbToYangDataFill(inParamsForGet xlateFromDbParams) { } else { xfmrLogDebug("Empty container for xpath(%v)", curUri) } + inParamsForGet.queryParams.fieldsFillAll = false inParamsForGet.dbDataMap = linParamsForGet.dbDataMap inParamsForGet.resultMap = resultMap } else if chldYangType == YANG_LIST { @@ -398,6 +431,22 @@ func sonicDbToYangDataFill(inParamsForGet xlateFromDbParams) { curUri := uri + "/" + yangChldName inParamsForGet.uri = curUri inParamsForGet.xpath = chldXpath + if len(inParamsForGet.queryParams.fields) > 0 { + if _, ok := inParamsForGet.queryParams.tgtFieldsXpathMap[chldXpath]; ok { + inParamsForGet.queryParams.fieldsFillAll = true + } else if _, ok := inParamsForGet.queryParams.allowFieldsXpath[chldXpath]; !ok { + if !inParamsForGet.queryParams.fieldsFillAll { + for path := range inParamsForGet.queryParams.tgtFieldsXpathMap { + if strings.HasPrefix(chldXpath, path) { + inParamsForGet.queryParams.fieldsFillAll = true + } + } + if !inParamsForGet.queryParams.fieldsFillAll { + continue + } + } + } + } mapSlice = sonicDbToYangListFill(inParamsForGet) dbDataMap = inParamsForGet.dbDataMap if len(key) > 0 && len(mapSlice) == 1 { // Single instance query. Don't return array of maps @@ -410,6 +459,7 @@ func sonicDbToYangDataFill(inParamsForGet xlateFromDbParams) { } else { xfmrLogDebug("Empty list for xpath(%v)", curUri) } + inParamsForGet.queryParams.fieldsFillAll = false inParamsForGet.resultMap = resultMap } else if chldYangType == YANG_CHOICE || chldYangType == YANG_CASE { inParamsForGet.xpath = chldXpath @@ -430,6 +480,7 @@ func sonicDbToYangDataFill(inParamsForGet xlateFromDbParams) { func directDbToYangJsonCreate(inParamsForGet xlateFromDbParams) (string, bool, error) { var err error uri := inParamsForGet.uri + jsonData := "{}" dbDataMap := inParamsForGet.dbDataMap resultMap := inParamsForGet.resultMap xpath, key, table := sonicXpathKeyExtract(uri) @@ -438,7 +489,25 @@ func directDbToYangJsonCreate(inParamsForGet xlateFromDbParams) (string, bool, e inParamsForGet.tblKey = key fieldName := "" - if len(xpath) > 0 { + traverse := true + if inParamsForGet.queryParams.depthEnabled { + pathList := strings.Split(xpath, "/") + // Depth at requested URI starts at 1. Hence reduce the reqDepth calculated by 1 + reqDepth := (len(pathList) - 1) + int(inParamsForGet.queryParams.curDepth) - 1 + xfmrLogInfo("xpath: %v ,Sonic Yang len(pathlist) %v, reqDepth %v, sonic Field Index %v", xpath, len(pathList), reqDepth, SONIC_FIELD_INDEX) + if reqDepth < SONIC_FIELD_INDEX { + traverse = false + } + } + + if len(inParamsForGet.queryParams.fields) > 0 { + flderr := validateAndFillSonicQpFields(inParamsForGet) + if flderr != nil { + return jsonData, true, flderr + } + } + + if traverse && len(xpath) > 0 { tokens := strings.Split(xpath, "/") if len(tokens) > SONIC_FIELD_INDEX { // Request is at levelf/leaflist level @@ -453,7 +522,7 @@ func directDbToYangJsonCreate(inParamsForGet xlateFromDbParams) (string, bool, e // Request is at table level xpath = table xfmrLogDebug("Request is at table level container - %v.", uri) - } else { + } else { // Request is at top level container prefixed by module name xfmrLogDebug("Request is at top level container - %v.", uri) } @@ -476,7 +545,7 @@ func directDbToYangJsonCreate(inParamsForGet xlateFromDbParams) (string, bool, e if yangType == YANG_LEAF_LIST { fieldName = fieldName + "@" } - linParamsForGet := formXlateFromDbParams(nil, inParamsForGet.dbs, cdb, inParamsForGet.ygRoot, xpath, inParamsForGet.requestUri, uri, inParamsForGet.oper, table, key, dbDataMap, inParamsForGet.txCache, resultMap, inParamsForGet.validate) + linParamsForGet := formXlateFromDbParams(nil, inParamsForGet.dbs, cdb, inParamsForGet.ygRoot, xpath, inParamsForGet.requestUri, uri, inParamsForGet.oper, table, key, dbDataMap, inParamsForGet.txCache, resultMap, inParamsForGet.validate, inParamsForGet.queryParams, nil) sonicDbToYangTerminalNodeFill(fieldName, linParamsForGet, dbEntry) resultMap = linParamsForGet.resultMap } else if yangType == YANG_CONTAINER { @@ -500,7 +569,7 @@ func directDbToYangJsonCreate(inParamsForGet xlateFromDbParams) (string, bool, e jsonMapData, _ := json.Marshal(resultMap) isEmptyPayload := isJsonDataEmpty(string(jsonMapData)) - jsonData := fmt.Sprintf("%v", string(jsonMapData)) + jsonData = fmt.Sprintf("%v", string(jsonMapData)) if isEmptyPayload { log.Warning("No data available") } @@ -589,7 +658,7 @@ func dbDataFromTblXfmrGet(tbl string, inParams XfmrParams, dbDataMap *map[db.DBN return nil } -func yangListDataFill(inParamsForGet xlateFromDbParams, isFirstCall bool) error { +func yangListDataFill(inParamsForGet xlateFromDbParams, isFirstCall bool, isOcMdl bool) error { var tblList []string dbs := inParamsForGet.dbs ygRoot := inParamsForGet.ygRoot @@ -605,6 +674,11 @@ func yangListDataFill(inParamsForGet xlateFromDbParams, isFirstCall bool) error _, ok := xYangSpecMap[xpath] if ok { + // Do not handle list if the curdepth is 1 + if inParamsForGet.queryParams.depthEnabled && (inParamsForGet.queryParams.curDepth <= 1) { + return nil + } + if xYangSpecMap[xpath].xfmrTbl != nil { xfmrTblFunc := *xYangSpecMap[xpath].xfmrTbl if len(xfmrTblFunc) > 0 { @@ -631,10 +705,15 @@ func yangListDataFill(inParamsForGet xlateFromDbParams, isFirstCall bool) error // Handling for case: Parent list is not associated with a tableName but has children containers/lists having tableNames. if tblKey != "" { var mapSlice []typeMapOfInterface - instMap, err := yangListInstanceDataFill(inParamsForGet, isFirstCall) + instMap, err := yangListInstanceDataFill(inParamsForGet, isFirstCall, isOcMdl) dbDataMap = inParamsForGet.dbDataMap if err != nil { xfmrLogDebug("Error(%v) returned for %v", err, uri) + // abort GET request if QP Subtree Pruning API returns error + _, qpSbtPruneErrOk := err.(*qpSubtreePruningErr) + if qpSbtPruneErrOk { + return err + } } else if (instMap != nil) && (len(instMap) > 0) { mapSlice = append(mapSlice, instMap) } @@ -658,7 +737,19 @@ func yangListDataFill(inParamsForGet xlateFromDbParams, isFirstCall bool) error } if len(tblList) == 0 { - xfmrLogInfo("Unable to traverse list as no table information available at URI %v. Please check if table mapping available", uri) + if (strings.HasSuffix(uri, "]") || strings.HasSuffix(uri, "]/")) && (len(xYangSpecMap[xpath].xfmrFunc) > 0 && xYangSpecMap[xpath].hasChildSubTree) { + err := yangDataFill(inParamsForGet, isOcMdl) + if err != nil { + xfmrLogInfo("yangListDataFill: error in its child subtree traversal for the xpath: %v", xpath) + _, qpSbtPruneErrOk := err.(*qpSubtreePruningErr) + if qpSbtPruneErrOk { + return err + } + } + return nil + } else { + xfmrLogInfo("Unable to traverse list as no table information available at URI %v. Please check if table mapping available", uri) + } } for _, tbl = range tblList { @@ -670,10 +761,15 @@ func yangListDataFill(inParamsForGet xlateFromDbParams, isFirstCall bool) error var mapSlice []typeMapOfInterface for dbKey := range tblData { inParamsForGet.tblKey = dbKey - instMap, err := yangListInstanceDataFill(inParamsForGet, isFirstCall) + instMap, err := yangListInstanceDataFill(inParamsForGet, isFirstCall, isOcMdl) dbDataMap = inParamsForGet.dbDataMap if err != nil { xfmrLogDebug("Error(%v) returned for %v", err, uri) + // abort GET request if QP Subtree Pruning API returns error + _, qpSbtPruneErrOk := err.(*qpSubtreePruningErr) + if qpSbtPruneErrOk { + return err + } } else if (instMap != nil) && (len(instMap) > 0) { mapSlice = append(mapSlice, instMap) } @@ -709,7 +805,7 @@ func yangListDataFill(inParamsForGet xlateFromDbParams, isFirstCall bool) error return nil } -func yangListInstanceDataFill(inParamsForGet xlateFromDbParams, isFirstCall bool) (typeMapOfInterface, error) { +func yangListInstanceDataFill(inParamsForGet xlateFromDbParams, isFirstCall bool, isOcMdl bool) (typeMapOfInterface, error) { var err error curMap := make(map[string]interface{}) @@ -737,18 +833,31 @@ func yangListInstanceDataFill(inParamsForGet xlateFromDbParams, isFirstCall bool (len(xYangSpecMap[parentXpath].xfmrFunc) > 0 && (xYangSpecMap[parentXpath].xfmrFunc != xYangSpecMap[xpath].xfmrFunc)))) { xfmrLogDebug("Parent subtree already handled cur uri: %v", xpath) inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, curUri, requestUri, GET, dbKey, dbDataMap, nil, nil, txCache) + inParams.queryParams = inParamsForGet.queryParams err := xfmrHandlerFunc(inParams, xYangSpecMap[xpath].xfmrFunc) inParamsForGet.ygRoot = ygRoot inParamsForGet.dbDataMap = dbDataMap if err != nil { xfmrLogDebug("Error returned by %v: %v", xYangSpecMap[xpath].xfmrFunc, err) + // abort GET request if QP Subtree Pruning API returns error + _, qpSbtPruneErrOk := err.(*qpSubtreePruningErr) + if qpSbtPruneErrOk { + return curMap, err + } } } if xYangSpecMap[xpath].hasChildSubTree { - linParamsForGet := formXlateFromDbParams(dbs[cdb], dbs, cdb, ygRoot, curUri, requestUri, xpath, inParamsForGet.oper, tbl, dbKey, dbDataMap, inParamsForGet.txCache, curMap, inParamsForGet.validate) + linParamsForGet := formXlateFromDbParams(dbs[cdb], dbs, cdb, ygRoot, curUri, requestUri, xpath, inParamsForGet.oper, tbl, dbKey, dbDataMap, inParamsForGet.txCache, curMap, inParamsForGet.validate, inParamsForGet.queryParams, nil) linParamsForGet.xfmrDbTblKeyCache = inParamsForGet.xfmrDbTblKeyCache linParamsForGet.dbTblKeyGetCache = inParamsForGet.dbTblKeyGetCache - yangDataFill(linParamsForGet) + err := yangDataFill(linParamsForGet, isOcMdl) + if err != nil { + // abort GET request if QP Subtree Pruning API returns error + _, qpSbtPruneErrOk := err.(*qpSubtreePruningErr) + if qpSbtPruneErrOk { + return nil, err + } + } curMap = linParamsForGet.resultMap dbDataMap = linParamsForGet.dbDataMap ygRoot = linParamsForGet.ygRoot @@ -761,16 +870,27 @@ func yangListInstanceDataFill(inParamsForGet xlateFromDbParams, isFirstCall bool xpathKeyExtRet, _ := xpathKeyExtract(dbs[cdb], ygRoot, GET, curUri, requestUri, dbDataMap, nil, txCache, inParamsForGet.xfmrDbTblKeyCache) keyFromCurUri := xpathKeyExtRet.dbKey inParamsForGet.ygRoot = ygRoot + var listKeyMap map[string]interface{} if dbKey == keyFromCurUri || keyFromCurUri == "" { if dbKey == keyFromCurUri { + listKeyMap = make(map[string]interface{}) for k, kv := range curKeyMap { curMap[k] = kv + listKeyMap[k] = kv } + } - linParamsForGet := formXlateFromDbParams(dbs[cdb], dbs, cdb, ygRoot, curUri, requestUri, xpathKeyExtRet.xpath, inParamsForGet.oper, tbl, dbKey, dbDataMap, inParamsForGet.txCache, curMap, inParamsForGet.validate) + linParamsForGet := formXlateFromDbParams(dbs[cdb], dbs, cdb, ygRoot, curUri, requestUri, xpathKeyExtRet.xpath, inParamsForGet.oper, tbl, dbKey, dbDataMap, inParamsForGet.txCache, curMap, inParamsForGet.validate, inParamsForGet.queryParams, listKeyMap) linParamsForGet.xfmrDbTblKeyCache = inParamsForGet.xfmrDbTblKeyCache linParamsForGet.dbTblKeyGetCache = inParamsForGet.dbTblKeyGetCache - yangDataFill(linParamsForGet) + err := yangDataFill(linParamsForGet, isOcMdl) + if err != nil { + // abort GET request if QP Subtree Pruning API returns error + _, qpSbtPruneErrOk := err.(*qpSubtreePruningErr) + if qpSbtPruneErrOk { + return curMap, err + } + } curMap = linParamsForGet.resultMap dbDataMap = linParamsForGet.dbDataMap ygRoot = linParamsForGet.ygRoot @@ -797,6 +917,13 @@ func terminalNodeProcess(inParamsForGet xlateFromDbParams, terminalNodeQuery boo dbDataMap := inParamsForGet.dbDataMap txCache := inParamsForGet.txCache + if uri != requestUri { + //Chk should be already done in yangDataFill + if inParamsForGet.queryParams.depthEnabled && inParamsForGet.queryParams.curDepth == 0 { + return resFldValMap, err + } + } + _, ok := xYangSpecMap[xpath] if !ok || yangEntry == nil { logStr := fmt.Sprintf("No YANG entry found for xpath %v.", xpath) @@ -807,6 +934,7 @@ func terminalNodeProcess(inParamsForGet xlateFromDbParams, terminalNodeQuery boo cdb := xYangSpecMap[xpath].dbIndex if len(xYangSpecMap[xpath].xfmrField) > 0 { inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, uri, requestUri, GET, tblKey, dbDataMap, nil, nil, txCache) + inParams.queryParams = inParamsForGet.queryParams fldValMap, err := leafXfmrHandlerFunc(inParams, xYangSpecMap[xpath].xfmrField) inParamsForGet.ygRoot = ygRoot inParamsForGet.dbDataMap = dbDataMap @@ -837,10 +965,12 @@ func terminalNodeProcess(inParamsForGet xlateFromDbParams, terminalNodeQuery boo if dbFldName == XFMR_NONE_STRING { return resFldValMap, err } - /* if there is no transformer extension/annotation then it means leaf-list in YANG is also leaflist in db */ - if len(dbFldName) > 0 && !xYangSpecMap[xpath].isKey { + fillLeafFromUriKey := false + yangDataType := yangEntry.Type.Kind + if terminalNodeQuery && xYangSpecMap[xpath].isKey { //GET request for list key-leaf(direct child of list) + fillLeafFromUriKey = true + } else if len(dbFldName) > 0 && !xYangSpecMap[xpath].isKey { yangType := xYangSpecMap[xpath].yangType - yngTerminalNdDtType := yangEntry.Type.Kind if yangType == YANG_LEAF_LIST { dbFldName += "@" val, ok := (*dbDataMap)[cdb][tbl][tblKey].Field[dbFldName] @@ -880,7 +1010,7 @@ func terminalNodeProcess(inParamsForGet xlateFromDbParams, terminalNodeQuery boo } return resFldValMap, err } else { - resLst := processLfLstDbToYang(xpath, val, yngTerminalNdDtType) + resLst := processLfLstDbToYang(xpath, val, yangDataType) resFldValMap[yangEntry.Name] = resLst } } else { @@ -892,15 +1022,57 @@ func terminalNodeProcess(inParamsForGet xlateFromDbParams, terminalNodeQuery boo } else { val, ok := (*dbDataMap)[cdb][tbl][tblKey].Field[dbFldName] if ok { - resVal, _, err := DbToYangType(yngTerminalNdDtType, xpath, val) + resVal, _, err := DbToYangType(yangDataType, xpath, val) if err != nil { log.Warning("Conversion of DB value type to YANG type for field didn't happen. Field-xfmr recommended if data types differ. Field xpath", xpath) } else { resFldValMap[yangEntry.Name] = resVal } } else { - xfmrLogDebug("Field value does not exist in DB for - %v", uri) - err = tlerr.NotFoundError{Format: "Resource not found"} + resNotFound := true + if xYangSpecMap[xpath].isRefByKey { + /*config container leaf is referenced by list-key that exists already before reaching here + state container leaf is not referenced by list-key, some state containers are mapped to + non-config DB(different from list level DB mapping), so check instance existence before + filling from uri*/ + _, stateContainerInstanceOk := (*dbDataMap)[cdb][tbl][tblKey] + if !yangEntry.ReadOnly() || stateContainerInstanceOk { + fillLeafFromUriKey = true + resNotFound = false + } + } + if resNotFound { + xfmrLogDebug("Field value does not exist in DB for - %v", uri) + err = tlerr.NotFoundError{Format: "Resource not found"} + } + } + } + } else if (len(dbFldName) == 0) && (xYangSpecMap[xpath].isRefByKey) { + _, stateContainerInstanceOk := (*dbDataMap)[cdb][tbl][tblKey] + if !yangEntry.ReadOnly() || stateContainerInstanceOk { + fillLeafFromUriKey = true + } + } + if fillLeafFromUriKey { + extractKeyLeafFromUri := true + xfmrLogDebug("Inferring isRefByKey Leaf %v value from list instance/keys", xpath) + if len(inParamsForGet.listKeysMap) > 0 { + if resVal, keyLeafExists := inParamsForGet.listKeysMap[yangEntry.Name]; keyLeafExists { + resFldValMap = make(map[string]interface{}) + resFldValMap[yangEntry.Name] = resVal + xfmrLogDebug("Filled isRefByKey leaf value from list keys map") + extractKeyLeafFromUri = false + } + } + if extractKeyLeafFromUri { + xfmrLogDebug("Filling isRefByKey leaf valuea from uri string.") + val := extractLeafValFromUriKey(uri, yangEntry.Name) + resVal, _, err := DbToYangType(yangDataType, xpath, val) + if err != nil { + log.Warning("Conversion of DB value type to YANG type for field didn't happen. Field-xfmr recommended if data types differ. Field xpath", xpath) + } else { + resFldValMap = make(map[string]interface{}) + resFldValMap[yangEntry.Name] = resVal } } } @@ -908,7 +1080,7 @@ func terminalNodeProcess(inParamsForGet xlateFromDbParams, terminalNodeQuery boo return resFldValMap, err } -func yangDataFill(inParamsForGet xlateFromDbParams) error { +func yangDataFill(inParamsForGet xlateFromDbParams, isOcMdl bool) error { var err error validate := inParamsForGet.validate isValid := validate @@ -925,8 +1097,19 @@ func yangDataFill(inParamsForGet xlateFromDbParams) error { yangNode, ok := xYangSpecMap[xpath] if ok && yangNode.yangEntry != nil { + if inParamsForGet.queryParams.depthEnabled { + // If fields are available, we need to evaluate the depth after fields level is met + // The depth upto the fields level is considered at level 1 + inParamsForGet.queryParams.curDepth = inParamsForGet.queryParams.curDepth - 1 + log.Infof("yangDataFill curdepth: %v, Path : %v", inParamsForGet.queryParams.curDepth, xpath) + if inParamsForGet.queryParams.curDepth == 0 { + return err + } + } + for yangChldName := range yangNode.yangEntry.Dir { chldXpath := xpath + "/" + yangChldName + chFieldsFillAll := inParamsForGet.queryParams.fieldsFillAll if xYangSpecMap[chldXpath] != nil && xYangSpecMap[chldXpath].nameWithMod != nil { chldUri = uri + "/" + *(xYangSpecMap[chldXpath].nameWithMod) } else { @@ -935,6 +1118,30 @@ func yangDataFill(inParamsForGet xlateFromDbParams) error { inParamsForGet.xpath = chldXpath inParamsForGet.uri = chldUri if xYangSpecMap[chldXpath] != nil && yangNode.yangEntry.Dir[yangChldName] != nil { + chldYangType := xYangSpecMap[chldXpath].yangType + if inParamsForGet.queryParams.content != QUERY_CONTENT_ALL { + yangNdInfo := contentQPSpecMapInfo{ + yangType: chldYangType, + yangName: yangChldName, + isReadOnly: yangNode.yangEntry.Dir[yangChldName].ReadOnly(), + isOperationalNd: xYangSpecMap[chldXpath].operationalQP, + hasNonTerminalNd: xYangSpecMap[chldXpath].hasNonTerminalNode, + hasChildOperationalNd: xYangSpecMap[chldXpath].hasChildOpertnlNd, + isOcMdl: isOcMdl, + } + processReq, _ := contentQParamYangNodeProcess(chldUri, yangNdInfo, inParamsForGet.queryParams) + if !processReq { + xfmrLogDebug("Further traversal not needed due to content query param, for of URI - %v", chldUri) + continue + } + } + + if inParamsForGet.queryParams.depthEnabled && inParamsForGet.queryParams.curDepth == 1 { + if (chldYangType == YANG_CONTAINER) || (chldYangType == YANG_LIST) { + continue + } + } + cdb := xYangSpecMap[chldXpath].dbIndex inParamsForGet.curDb = cdb if len(xYangSpecMap[chldXpath].validateFunc) > 0 && !validate { @@ -942,6 +1149,7 @@ func yangDataFill(inParamsForGet xlateFromDbParams) error { inParamsForGet.ygRoot = ygRoot // TODO - handle non CONFIG-DB inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, chldUri, requestUri, GET, xpathKeyExtRet.dbKey, dbDataMap, nil, nil, txCache) + inParams.queryParams = inParamsForGet.queryParams res := validateHandlerFunc(inParams, xYangSpecMap[chldXpath].validateFunc) if !res { xfmrLogDebug("Further traversal not needed. Validate xfmr returns false for URI %v", chldUri) @@ -953,11 +1161,18 @@ func yangDataFill(inParamsForGet xlateFromDbParams) error { inParamsForGet.dbDataMap = dbDataMap inParamsForGet.ygRoot = ygRoot } - chldYangType := xYangSpecMap[chldXpath].yangType if chldYangType == YANG_LEAF || chldYangType == YANG_LEAF_LIST { if len(xYangSpecMap[xpath].xfmrFunc) > 0 { continue } + if !xYangSpecMap[chldXpath].isKey && len(inParamsForGet.queryParams.fields) > 0 { + if _, ok := inParamsForGet.queryParams.tgtFieldsXpathMap[chldXpath]; !ok { + if !inParamsForGet.queryParams.fieldsFillAll { + xfmrLogDebug("Skip processing URI due to fields QP procesing - %v", chldUri) + continue + } + } + } yangEntry := yangNode.yangEntry.Dir[yangChldName] fldValMap, err := terminalNodeProcess(inParamsForGet, false, yangEntry) dbDataMap = inParamsForGet.dbDataMap @@ -975,6 +1190,23 @@ func yangDataFill(inParamsForGet xlateFromDbParams) error { chtbl := xpathKeyExtRet.tableName inParamsForGet.ygRoot = ygRoot + if len(inParamsForGet.queryParams.fields) > 0 { + if _, ok := inParamsForGet.queryParams.tgtFieldsXpathMap[chldXpath]; ok { + chFieldsFillAll = true + } else if _, ok := inParamsForGet.queryParams.allowFieldsXpath[chldXpath]; !ok { + if !inParamsForGet.queryParams.fieldsFillAll { + for path := range inParamsForGet.queryParams.tgtFieldsXpathMap { + if strings.HasPrefix(chldXpath, path) { + chFieldsFillAll = true + } + } + if !chFieldsFillAll { + continue + } + } + } + } + if _, ok := (*dbDataMap)[cdb][chtbl][tblKey]; !ok && len(chtbl) > 0 { qdbMapHasTblData := false qdbMapHasTblKeyData := false @@ -1019,11 +1251,17 @@ func yangDataFill(inParamsForGet xlateFromDbParams) error { (len(xYangSpecMap[xpath].xfmrFunc) > 0 && (xYangSpecMap[xpath].xfmrFunc != xYangSpecMap[chldXpath].xfmrFunc)) { inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, chldUri, requestUri, GET, "", dbDataMap, nil, nil, txCache) + inParams.queryParams = inParamsForGet.queryParams err := xfmrHandlerFunc(inParams, xYangSpecMap[chldXpath].xfmrFunc) inParamsForGet.dbDataMap = dbDataMap inParamsForGet.ygRoot = ygRoot if err != nil { xfmrLogDebug("Error returned by %v: %v", xYangSpecMap[xpath].xfmrFunc, err) + // abort GET request if QP Subtree Pruning API returns error + _, qpSbtPruneErrOk := err.(*qpSubtreePruningErr) + if qpSbtPruneErrOk { + return err + } } } if !xYangSpecMap[chldXpath].hasChildSubTree { @@ -1032,10 +1270,18 @@ func yangDataFill(inParamsForGet xlateFromDbParams) error { } } cmap2 := make(map[string]interface{}) - linParamsForGet := formXlateFromDbParams(dbs[cdb], dbs, cdb, ygRoot, chldUri, requestUri, chldXpath, inParamsForGet.oper, chtbl, tblKey, dbDataMap, inParamsForGet.txCache, cmap2, inParamsForGet.validate) + linParamsForGet := formXlateFromDbParams(dbs[cdb], dbs, cdb, ygRoot, chldUri, requestUri, chldXpath, inParamsForGet.oper, chtbl, tblKey, dbDataMap, inParamsForGet.txCache, cmap2, inParamsForGet.validate, inParamsForGet.queryParams, inParamsForGet.listKeysMap) linParamsForGet.xfmrDbTblKeyCache = inParamsForGet.xfmrDbTblKeyCache linParamsForGet.dbTblKeyGetCache = inParamsForGet.dbTblKeyGetCache - err = yangDataFill(linParamsForGet) + linParamsForGet.queryParams.fieldsFillAll = chFieldsFillAll + err = yangDataFill(linParamsForGet, isOcMdl) + if err != nil { + // abort GET request if QP Subtree Pruning API returns error + _, qpSbtPruneErrOk := err.(*qpSubtreePruningErr) + if qpSbtPruneErrOk { + return err + } + } cmap2 = linParamsForGet.resultMap dbDataMap = linParamsForGet.dbDataMap ygRoot = linParamsForGet.ygRoot @@ -1059,9 +1305,15 @@ func yangDataFill(inParamsForGet xlateFromDbParams) error { (len(xYangSpecMap[xpath].xfmrFunc) > 0 && (xYangSpecMap[xpath].xfmrFunc != xYangSpecMap[chldXpath].xfmrFunc)) { inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, chldUri, requestUri, GET, "", dbDataMap, nil, nil, txCache) + inParams.queryParams = inParamsForGet.queryParams err := xfmrHandlerFunc(inParams, xYangSpecMap[chldXpath].xfmrFunc) if err != nil { xfmrLogDebug("Error returned by %v: %v", xYangSpecMap[chldXpath].xfmrFunc, err) + // abort GET request if QP Subtree Pruning API returns error + _, qpSbtPruneErrOk := err.(*qpSubtreePruningErr) + if qpSbtPruneErrOk { + return err + } } inParamsForGet.dbDataMap = dbDataMap inParamsForGet.ygRoot = ygRoot @@ -1083,10 +1335,17 @@ func yangDataFill(inParamsForGet xlateFromDbParams) error { inParamsForGet.dbDataMap = dbDataMap } } - linParamsForGet := formXlateFromDbParams(dbs[cdb], dbs, cdb, ygRoot, chldUri, requestUri, chldXpath, inParamsForGet.oper, lTblName, xpathKeyExtRet.dbKey, dbDataMap, inParamsForGet.txCache, resultMap, inParamsForGet.validate) + linParamsForGet := formXlateFromDbParams(dbs[cdb], dbs, cdb, ygRoot, chldUri, requestUri, chldXpath, inParamsForGet.oper, lTblName, xpathKeyExtRet.dbKey, dbDataMap, inParamsForGet.txCache, resultMap, inParamsForGet.validate, inParamsForGet.queryParams, nil) linParamsForGet.xfmrDbTblKeyCache = inParamsForGet.xfmrDbTblKeyCache linParamsForGet.dbTblKeyGetCache = inParamsForGet.dbTblKeyGetCache - yangListDataFill(linParamsForGet, false) + err := yangListDataFill(linParamsForGet, false, isOcMdl) + if err != nil { + // abort GET request if QP Subtree Pruning API returns error + _, qpSbtPruneErrOk := err.(*qpSubtreePruningErr) + if qpSbtPruneErrOk { + return err + } + } resultMap = linParamsForGet.resultMap dbDataMap = linParamsForGet.dbDataMap ygRoot = linParamsForGet.ygRoot @@ -1095,6 +1354,14 @@ func yangDataFill(inParamsForGet xlateFromDbParams) error { inParamsForGet.ygRoot = ygRoot } else if chldYangType == YANG_CHOICE || chldYangType == YANG_CASE { + err := yangDataFill(inParamsForGet, isOcMdl) + if err != nil { + // abort GET request if QP Subtree Pruning API returns error + _, qpSbtPruneErrOk := err.(*qpSubtreePruningErr) + if qpSbtPruneErrOk { + return err + } + } resultMap = inParamsForGet.resultMap dbDataMap = inParamsForGet.dbDataMap } else { @@ -1111,6 +1378,7 @@ func dbDataToYangJsonCreate(inParamsForGet xlateFromDbParams) (string, bool, err var err error var fldSbtErr error // used only when direct query on leaf/leaf-list having subtree var fldErr error //used only when direct query on leaf/leaf-list having field transformer + var isOcMdl bool jsonData := "{}" resultMap := make(map[string]interface{}) d := inParamsForGet.d @@ -1134,12 +1402,27 @@ func dbDataToYangJsonCreate(inParamsForGet xlateFromDbParams) (string, bool, err inParamsForGet.ygRoot = ygRoot yangNode, ok := xYangSpecMap[xpathKeyExtRet.xpath] if ok { + yangType := yangNode.yangType + //Check if fields are valid + if len(inParamsForGet.queryParams.fields) > 0 { + flderr := validateAndFillQpFields(inParamsForGet) + if flderr != nil { + return jsonData, true, flderr + } + } + + //Check if the request depth is 1 + if inParamsForGet.queryParams.depthEnabled && inParamsForGet.queryParams.curDepth == 1 && (yangType == YANG_CONTAINER || yangType == YANG_LIST || yangType == YANG_MODULE) { + return jsonData, true, err + } + /* Invoke pre-xfmr is present for the YANG module */ moduleName := "/" + strings.Split(uri, "/")[1] xfmrLogInfo("Module name for URI %s is %s", uri, moduleName) if xYangModSpecMap != nil { if modSpecInfo, specOk := xYangModSpecMap[moduleName]; specOk && (len(modSpecInfo.xfmrPre) > 0) { inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, uri, requestUri, GET, "", dbDataMap, nil, nil, txCache) + inParams.queryParams = inParamsForGet.queryParams err = preXfmrHandlerFunc(modSpecInfo.xfmrPre, inParams) xfmrLogInfo("Invoked pre transformer: %v, dbDataMap: %v ", modSpecInfo.xfmrPre, dbDataMap) if err != nil { @@ -1151,13 +1434,13 @@ func dbDataToYangJsonCreate(inParamsForGet xlateFromDbParams) (string, bool, err } } - yangType := yangNode.yangType validateHandlerFlag := false tableXfmrFlag := false IsValidate := false if len(xYangSpecMap[xpathKeyExtRet.xpath].validateFunc) > 0 { inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, uri, requestUri, GET, xpathKeyExtRet.dbKey, dbDataMap, nil, nil, txCache) + inParams.queryParams = inParamsForGet.queryParams res := validateHandlerFunc(inParams, xYangSpecMap[xpathKeyExtRet.xpath].validateFunc) inParamsForGet.dbDataMap = dbDataMap inParamsForGet.ygRoot = ygRoot @@ -1170,6 +1453,9 @@ func dbDataToYangJsonCreate(inParamsForGet xlateFromDbParams) (string, bool, err } inParamsForGet.validate = IsValidate isList := false + if strings.HasPrefix(requestUri, "/"+OC_MDL_PFX) { + isOcMdl = true + } switch yangType { case YANG_LIST: isList = true @@ -1226,13 +1512,22 @@ func dbDataToYangJsonCreate(inParamsForGet xlateFromDbParams) (string, bool, err } if len(xYangSpecMap[xpathKeyExtRet.xpath].xfmrFunc) > 0 { inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, uri, requestUri, GET, "", dbDataMap, nil, nil, txCache) - fldSbtErr = xfmrHandlerFunc(inParams, xYangSpecMap[xpathKeyExtRet.xpath].xfmrFunc) - if fldSbtErr != nil { + inParams.queryParams = inParamsForGet.queryParams + xfmrFuncErr := xfmrHandlerFunc(inParams, xYangSpecMap[xpathKeyExtRet.xpath].xfmrFunc) + if xfmrFuncErr != nil { /*For request Uri pointing to leaf/leaf-list having subtree, error will be propagated - to handle check of leaf/leaf-list-instance existence in DB , which will be performed - by subtree + to handle check of leaf/leaf-list-instance existence in DB , which will be performed + by subtree + Error will also be propagated if QP Pruning API for subtree returns error */ - xfmrLogInfo("Error returned by %v: %v", xYangSpecMap[xpathKeyExtRet.xpath].xfmrFunc, err) + _, qpSbtPruneErrOk := xfmrFuncErr.(*qpSubtreePruningErr) + + if qpSbtPruneErrOk { + err = xfmrFuncErr + } else { + // propagate err from subtree callback + fldSbtErr = xfmrFuncErr + } inParamsForGet.ygRoot = ygRoot break } @@ -1268,9 +1563,13 @@ func dbDataToYangJsonCreate(inParamsForGet xlateFromDbParams) (string, bool, err } if len(xYangSpecMap[xpathKeyExtRet.xpath].xfmrFunc) > 0 { inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, uri, requestUri, GET, "", dbDataMap, nil, nil, txCache) + inParams.queryParams = inParamsForGet.queryParams err := xfmrHandlerFunc(inParams, xYangSpecMap[xpathKeyExtRet.xpath].xfmrFunc) if err != nil { - xfmrLogInfo("Error returned by %v: %v", xYangSpecMap[xpathKeyExtRet.xpath].xfmrFunc, err) + qpSbtPruneErr, qpSbtPruneErrOk := err.(*qpSubtreePruningErr) + if qpSbtPruneErrOk { + err = tlerr.InternalError{Format: QUERY_PARAMETER_SBT_PRUNING_ERR, Path: qpSbtPruneErr.subtreePath} + } return jsonData, true, err } inParamsForGet.dbDataMap = dbDataMap @@ -1281,7 +1580,7 @@ func dbDataToYangJsonCreate(inParamsForGet xlateFromDbParams) (string, bool, err } } inParamsForGet.resultMap = make(map[string]interface{}) - err = yangDataFill(inParamsForGet) + err = yangDataFill(inParamsForGet, isOcMdl) if err != nil { xfmrLogInfo("Empty container(\"%v\").\r\n", uri) } @@ -1291,8 +1590,14 @@ func dbDataToYangJsonCreate(inParamsForGet xlateFromDbParams) (string, bool, err isFirstCall := true if len(xYangSpecMap[xpathKeyExtRet.xpath].xfmrFunc) > 0 { inParams := formXfmrInputRequest(dbs[cdb], dbs, cdb, ygRoot, uri, requestUri, GET, "", dbDataMap, nil, nil, txCache) + inParams.queryParams = inParamsForGet.queryParams err := xfmrHandlerFunc(inParams, xYangSpecMap[xpathKeyExtRet.xpath].xfmrFunc) if err != nil { + qpSbtPruneErr, qpSbtPruneErrOk := err.(*qpSubtreePruningErr) + if qpSbtPruneErrOk { + err = tlerr.InternalError{Format: QUERY_PARAMETER_SBT_PRUNING_ERR, Path: qpSbtPruneErr.subtreePath} + return jsonData, true, err + } if ((strings.HasSuffix(uri, "]")) || (strings.HasSuffix(uri, "]/"))) && (uri == requestUri) { // The error handling here is for the deferred resource check error being handled by the subtree for virtual table cases. log.Warningf("Subtree at list instance level returns error %v for URI - %v", err, uri) @@ -1312,7 +1617,7 @@ func dbDataToYangJsonCreate(inParamsForGet xlateFromDbParams) (string, bool, err } } inParamsForGet.resultMap = make(map[string]interface{}) - err = yangListDataFill(inParamsForGet, isFirstCall) + err = yangListDataFill(inParamsForGet, isFirstCall, isOcMdl) if err != nil { xfmrLogInfo("yangListDataFill failed for list case(\"%v\").\r\n", uri) } @@ -1344,6 +1649,12 @@ func dbDataToYangJsonCreate(inParamsForGet xlateFromDbParams) (string, bool, err */ return jsonData, isEmptyPayload, fldErr } + if err != nil { + qpSbtPruneErr, qpSbtPruneErrOk := err.(*qpSubtreePruningErr) + if qpSbtPruneErrOk { + return jsonData, isEmptyPayload, tlerr.InternalError{Format: QUERY_PARAMETER_SBT_PRUNING_ERR, Path: qpSbtPruneErr.subtreePath} + } + } return jsonData, isEmptyPayload, nil } diff --git a/translib/transformer/xlate_to_db.go b/translib/transformer/xlate_to_db.go index 4c11fcb556de..6e04a8ead42b 100644 --- a/translib/transformer/xlate_to_db.go +++ b/translib/transformer/xlate_to_db.go @@ -184,21 +184,37 @@ func mapFillDataUtil(xlateParams xlateToParams) error { } if len(xpathInfo.fieldName) == 0 { - xfmrLogInfo("Field for yang-path(\"%v\") not found in DB.", xlateParams.xpath) + if xpathInfo.isRefByKey || (xpathInfo.isKey && strings.HasPrefix(xlateParams.uri, "/"+IETF_MDL_PFX)) { + dataToDBMapAdd(xlateParams.tableName, xlateParams.keyName, xlateParams.result, "NULL", "NULL") + xfmrLogDebug("%v - Either an OC YANG leaf path referenced by list key or IETF YANG list key, ", xlateParams.xpath, + "maps to no actual field in sonic yang so dummy row added to create instance in DB.") + } else { + xfmrLogInfo("Field for YANG path(\"%v\") not found in DB.", xlateParams.xpath) + } return nil } fieldName := xpathInfo.fieldName valueStr := "" fieldXpath := xlateParams.tableName + "/" + fieldName - _, ok = xDbSpecMap[fieldXpath] + xDbSpecInfo, ok := xDbSpecMap[fieldXpath] dbEntry := getYangEntryForXPath(fieldXpath) - if !ok || (dbEntry == nil) { + if !ok || (xDbSpecInfo == nil) || (dbEntry == nil) { logStr := fmt.Sprintf("Failed to find the xDbSpecMap: xpath(\"%v\").", fieldXpath) log.Warning(logStr) return nil } - + if xDbSpecInfo.isKey { + if xpathInfo.isRefByKey || (xpathInfo.isKey && strings.HasPrefix(xlateParams.uri, "/"+IETF_MDL_PFX)) { + // apps use this leaf in payload to create an instance, redis needs atleast one field:val to create an instance + dataToDBMapAdd(xlateParams.tableName, xlateParams.keyName, xlateParams.result, "NULL", "NULL") // redis needs atleast one field:val to create an instance + xfmrLogDebug("%v - Either an OC YANG leaf path referenced by list key or IETF YANG list key, ", xlateParams.xpath, + "maps to key in sonic YANG - %v so dummy row added to create instance in DB.", fieldXpath) + } else { + xfmrLogInfo("OC YANG leaf path(%v), maps to key in sonic YANG - %v which cannot be filled as a field in DB instance", xlateParams.xpath, fieldXpath) + } + return nil + } if xpathInfo.yangType == YANG_LEAF_LIST { /* Both YANG side and DB side('@' suffix field) the data type is leaf-list */ xfmrLogDebug("Yang type and DB type is Leaflist for field = %v", xlateParams.xpath) @@ -506,7 +522,7 @@ func dbMapDefaultFieldValFill(xlateParams xlateToParams, tblUriList []string) er } } else if len(childNode.fieldName) > 0 { var xfmrErr error - if _, ok := xDbSpecMap[tblName+"/"+childNode.fieldName]; ok { + if xDbSpecInfo, ok := xDbSpecMap[tblName+"/"+childNode.fieldName]; ok && (xDbSpecInfo != nil) && (!xDbSpecInfo.isKey) { if tblXfmrPresent { chldTblNm, ctErr := tblNameFromTblXfmrGet(*childNode.xfmrTbl, inParamsTblXfmr, xlateParams.xfmrDbTblKeyCache) xfmrLogDebug("Table transformer %v for xpath %v returned table %v", *childNode.xfmrTbl, childXpath, chldTblNm) @@ -532,6 +548,18 @@ func dbMapDefaultFieldValFill(xlateParams xlateToParams, tblUriList []string) er } } } + } else if childNode.isRefByKey { + /* this case occurs only for static table-name, table-xfmr always has field-name + assigned by infra when there no explicit annotation from user. + Also key-leaf in config container has no default value, so here we handle only + REPLACE yangAuxValMap filling */ + if xlateParams.oper != REPLACE { + continue + } + _, ok := xlateParams.result[tblName][dbKey].Field["NULL"] + if !ok { + dataToDBMapAdd(tblName, dbKey, xlateParams.yangAuxValMap, "NULL", "") + } } } } @@ -827,15 +855,15 @@ func yangReqToDbMapCreate(xlateParams xlateToParams) error { curUri, _ := uriWithKeyCreate(xlateParams.uri, xlateParams.xpath, data) _, ok := xYangSpecMap[xlateParams.xpath] if ok && len(xYangSpecMap[xlateParams.xpath].xfmrKey) > 0 { - /* key transformer present */ + // key transformer present curYgotNode, nodeErr := yangNodeForUriGet(curUri, xlateParams.ygRoot) if nodeErr != nil { curYgotNode = nil } - inParams := formXfmrInputRequest(xlateParams.d, dbs, db.MaxDB, xlateParams.ygRoot, curUri, xlateParams.requestUri, xlateParams.oper, "", nil, xlateParams.subOpDataMap, curYgotNode, xlateParams.txCache) + inParams := formXfmrInputRequest(xlateParams.d, dbs, db.ConfigDB, xlateParams.ygRoot, curUri, xlateParams.requestUri, xlateParams.oper, "", nil, xlateParams.subOpDataMap, curYgotNode, xlateParams.txCache) ktRetData, err := keyXfmrHandler(inParams, xYangSpecMap[xlateParams.xpath].xfmrKey) - //if key transformer is called without key values in curUri ignore the error + // if key transformer is called without key values in curUri ignore the error if err != nil && strings.HasSuffix(curUri, "]") { if xlateParams.xfmrErr != nil && *xlateParams.xfmrErr == nil { *xlateParams.xfmrErr = err @@ -846,7 +874,7 @@ func yangReqToDbMapCreate(xlateParams xlateToParams) error { } else if ok && xYangSpecMap[xlateParams.xpath].keyName != nil { curKey = *xYangSpecMap[xlateParams.xpath].keyName } else { - curKey = keyCreate(xlateParams.keyName, xlateParams.xpath, data, xlateParams.d.Opts.KeySeparator) + curKey = keyCreate(xlateParams, curUri, data) } curXlateParams := formXlateToDbParam(xlateParams.d, xlateParams.ygRoot, xlateParams.oper, curUri, xlateParams.requestUri, xlateParams.xpath, curKey, data, xlateParams.resultMap, xlateParams.result, xlateParams.txCache, xlateParams.tblXpathMap, xlateParams.subOpDataMap, xlateParams.pCascadeDelTbl, xlateParams.xfmrErr, "", "", "", xlateParams.invokeCRUSubtreeOnceMap) retErr = yangReqToDbMapCreate(curXlateParams) @@ -969,7 +997,7 @@ func yangReqToDbMapCreate(xlateParams xlateToParams) error { _, ok := xYangSpecMap[xpath] // Process the terminal node only if the targetUri is at terminal Node or if the leaf is not at parent level if ok && strings.HasPrefix(curXpath, reqXpath) { - if (!xYangSpecMap[xpath].isKey) || (len(xYangSpecMap[xpath].xfmrField) > 0) { + if (!xYangSpecMap[xpath].isKey) || (len(xYangSpecMap[xpath].xfmrField) > 0) || (xYangSpecMap[xpath].isKey && strings.HasPrefix(xlateParams.uri, "/"+IETF_MDL_PFX)) { if len(xYangSpecMap[xpath].xfmrFunc) == 0 { value := jData.MapIndex(key).Interface() xfmrLogDebug("data field: key(\"%v\"), value(\"%v\").", key, value) @@ -1047,11 +1075,11 @@ func verifyParentTableSonic(d *db.DB, dbs [db.MaxDB]*db.DB, oper Operation, uri tableExists, derr = dbTableExists(d, table, dbKey, oper) if hasSingletonContainer && oper == DELETE { // Special case when we delete at container that does'nt exist. Return true to skip translation. - if !tableExists { - return true, derr - } else { - return true, nil - } + if !tableExists { + return true, derr + } else { + return true, nil + } } if derr != nil { return false, derr diff --git a/translib/transformer/xlate_utils.go b/translib/transformer/xlate_utils.go index 8f490ebd047d..82911a872a36 100644 --- a/translib/transformer/xlate_utils.go +++ b/translib/transformer/xlate_utils.go @@ -46,15 +46,38 @@ func initRegex() { } /* Create db key from data xpath(request) */ -func keyCreate(keyPrefix string, xpath string, data interface{}, dbKeySep string) string { - _, ok := xYangSpecMap[xpath] +func keyCreate(xlateParams xlateToParams, curUri string, data interface{}) string { + keyPrefix := xlateParams.keyName + var dbs [db.MaxDB]*db.DB + dbKeySep := xlateParams.d.Opts.KeySeparator + xpathInfo, ok := xYangSpecMap[xlateParams.xpath] if ok { - if xYangSpecMap[xpath].yangEntry != nil { - yangEntry := xYangSpecMap[xpath].yangEntry + var tableName string + var err error + tblPtr := xpathInfo.tableName + if tblPtr != nil && *tblPtr != XFMR_NONE_STRING { + tableName = *tblPtr + } else if xpathInfo.xfmrTbl != nil { + inParams := formXfmrInputRequest(xlateParams.d, dbs, db.ConfigDB, xlateParams.ygRoot, curUri, xlateParams.requestUri, xlateParams.oper, "", nil, xlateParams.subOpDataMap, nil, xlateParams.txCache) + tableName, err = tblNameFromTblXfmrGet(*xpathInfo.xfmrTbl, inParams, nil) + if err != nil { + if xlateParams.xfmrErr != nil && *xlateParams.xfmrErr == nil { + *xlateParams.xfmrErr = err + } + } + } + if hasSameOcSonicKeys(curUri, xlateParams.xpath, tableName) { + // If the oc and sonic yangs have same keys then discard the parent key accumulated. Else the parent key is concatenated with the key generated by infra + keyPrefix = "" + xfmrLogDebug("1:1 mapping case. Concatenating YANG keys at current list", curUri) + } + + if xpathInfo.yangEntry != nil { + yangEntry := xpathInfo.yangEntry delim := dbKeySep - if len(xYangSpecMap[xpath].delim) > 0 { - delim = xYangSpecMap[xpath].delim - xfmrLogDebug("key concatenater(\"%v\") found for xpath %v ", delim, xpath) + if len(xpathInfo.delim) > 0 { + delim = xpathInfo.delim + xfmrLogDebug("key concatenater(\"%v\") found for xpath %v ", delim, xlateParams.xpath) } if len(keyPrefix) > 0 { @@ -65,7 +88,7 @@ func keyCreate(keyPrefix string, xpath string, data interface{}, dbKeySep string if i > 0 { keyVal = keyVal + delim } - fieldXpath := xpath + "/" + k + fieldXpath := xlateParams.xpath + "/" + k fVal, err := unmarshalJsonToDbData(yangEntry.Dir[k], fieldXpath, k, data.(map[string]interface{})[k]) if err != nil { log.Warningf("Couldn't unmarshal Json to DbData: path(\"%v\") error (\"%v\").", fieldXpath, err) @@ -181,8 +204,7 @@ func dbKeyToYangDataConvert(uri string, requestUri string, xpath string, tableNa } keyNameList := yangKeyFromEntryGet(xYangSpecMap[xpath].yangEntry) - id := xYangSpecMap[xpath].keyLevel - keyDataList := strings.SplitN(dbKey, dbKeySep, int(id)) + keyDataList := strings.SplitN(dbKey, dbKeySep, -1) uriWithKey := fmt.Sprintf("%v", xpath) uriWithKeyCreate := true if len(keyDataList) == 0 { @@ -224,34 +246,38 @@ func dbKeyToYangDataConvert(uri string, requestUri string, xpath string, tableNa } rmap := make(map[string]interface{}) - if len(keyNameList) > 1 { - log.Warningf("No key transformer found for multi element YANG key mapping to a single redis key string, for URI %v", uri) + keyNameValSame := len(keyNameList) == len(keyDataList) + if len(keyNameList) > 1 && !keyNameValSame && !hasSameOcSonicKeys(uri, xpath, tableName) { + log.Warningf("No key transformer found for multi element yang key mapping to a single redis key string, for uri %v", uri) errStr := fmt.Sprintf("Error processing key for list %v", uri) err = fmt.Errorf("%v", errStr) return rmap, uriWithKey, err } - keyXpath := xpath + "/" + keyNameList[0] - yangEntry, ok := xYangSpecMap[xpath].yangEntry.Dir[keyNameList[0]] - if !ok || yangEntry == nil { - errStr := fmt.Sprintf("Failed to find key xpath %v in xYangSpecMap or is nil, needed to fetch the yangEntry data-type", keyXpath) - err = fmt.Errorf("%v", errStr) - return rmap, uriWithKey, err - } - yngTerminalNdDtType := yangEntry.Type.Kind - resVal, _, err := DbToYangType(yngTerminalNdDtType, keyXpath, keyDataList[0]) - if err != nil { - err = fmt.Errorf("Failed in convert DB value type to YANG type for field %v. Key-xfmr recommended if data types differ", keyXpath) - return rmap, uriWithKey, err - } else { - rmap[keyNameList[0]] = resVal - } - if uriWithKeyCreate { - if reflect.TypeOf(resVal).Kind() == reflect.String { - resVal = escapeKeyValForSplitPathAndNewPathInfo(resVal.(string)) + + for i := range keyNameList { + keyXpath := xpath + "/" + keyNameList[i] + yangEntry, ok := xYangSpecMap[xpath].yangEntry.Dir[keyNameList[i]] + if !ok || yangEntry == nil { + errStr := fmt.Sprintf("Failed to find key xpath %v in xYangSpecMap or is nil, needed to fetch the yangEntry data-type", keyXpath) + err = fmt.Errorf("%v", errStr) + return rmap, uriWithKey, err + } + yngTerminalNdDtType := yangEntry.Type.Kind + resVal, _, err := DbToYangType(yngTerminalNdDtType, keyXpath, keyDataList[i]) + if err != nil { + err = fmt.Errorf("Failure in converting Db value type to yang type for field %v", keyXpath) + return rmap, uriWithKey, err + } else { + rmap[keyNameList[i]] = resVal + } + if uriWithKeyCreate { + if reflect.TypeOf(resVal).Kind() == reflect.String { + resVal = escapeKeyValForSplitPathAndNewPathInfo(resVal.(string)) + } + uriWithKey += fmt.Sprintf("[%v=%v]", keyNameList[i], resVal) } - uriWithKey += fmt.Sprintf("[%v=%v]", keyNameList[0], resVal) } - + xfmrLogDebug("dbKeyToYangDataConvert: uri %v dbkey: %v, uriWithkey %v, rmap :%v", uri, dbKey, uriWithKey, rmap) return rmap, uriWithKey, nil } @@ -749,6 +775,24 @@ func xpathKeyExtract(d *db.DB, ygRoot *ygot.GoStruct, oper Operation, path strin will be concatenated with respective default DB type key-delimiter */ if (yangType == YANG_LIST) && (xpathInfo.yangEntry != nil) { + var tableName string + tblPtr := xpathInfo.tableName + if tblPtr != nil && *tblPtr != XFMR_NONE_STRING { + tableName = *tblPtr + } else if xpathInfo.xfmrTbl != nil { + inParams := formXfmrInputRequest(d, dbs, cdb, ygRoot, curPathWithKey, requestUri, oper, "", nil, subOpDataMap, nil, txCache) + if oper == GET { + inParams.dbDataMap = dbDataMap + } + tableName, err = tblNameFromTblXfmrGet(*xpathInfo.xfmrTbl, inParams, xfmrTblKeyCache) + if err != nil && oper != GET { + return retData, err + } + } + if hasSameOcSonicKeys(curPathWithKey, yangXpath, tableName) { + // If the oc and sonic yangs have same keys then discard the parent key accumulated. Else the parent key is concatenated with the key generated by infra + keyStr = "" + } xfmrLogDebug("No key-xfmr at list %v, 1:1 mapping case. Concatenating YANG keys", curPathWithKey) keyNmList := strings.Split(xpathInfo.yangEntry.Key, " ") pathInfo := NewPathInfo("/" + k) @@ -1160,7 +1204,7 @@ func dbDataXfmrHandler(resultMap map[Operation]map[db.DBNum]map[string]map[strin return nil } -func formXlateFromDbParams(d *db.DB, dbs [db.MaxDB]*db.DB, cdb db.DBNum, ygRoot *ygot.GoStruct, uri string, requestUri string, xpath string, oper Operation, tbl string, tblKey string, dbDataMap *RedisDbMap, txCache interface{}, resultMap map[string]interface{}, validate bool) xlateFromDbParams { +func formXlateFromDbParams(d *db.DB, dbs [db.MaxDB]*db.DB, cdb db.DBNum, ygRoot *ygot.GoStruct, uri string, requestUri string, xpath string, oper Operation, tbl string, tblKey string, dbDataMap *RedisDbMap, txCache interface{}, resultMap map[string]interface{}, validate bool, qParams QueryParams, listKeysMap map[string]interface{}) xlateFromDbParams { var inParamsForGet xlateFromDbParams inParamsForGet.d = d inParamsForGet.dbs = dbs @@ -1176,6 +1220,8 @@ func formXlateFromDbParams(d *db.DB, dbs [db.MaxDB]*db.DB, cdb db.DBNum, ygRoot inParamsForGet.txCache = txCache inParamsForGet.resultMap = resultMap inParamsForGet.validate = validate + inParamsForGet.queryParams = qParams + inParamsForGet.listKeysMap = listKeysMap return inParamsForGet } @@ -1559,6 +1605,123 @@ func getYangNodeTypeFromUri(uri string) (yangElementType, error) { return yangNodeType, nil } +func NewQueryParams(depth uint, content string, fields []string) (QueryParams, error) { + var err error + var qparams QueryParams + + xfmrLogInfo("NewQueryParams: depth:%v, content: %v, fields: %v", depth, content, fields) + qpDepthContentPresent := false + if depth > 0 { + qparams.depthEnabled = true + qparams.curDepth = depth + } else { + qparams.curDepth = 0 + } + if len(content) > 0 { + content = strings.ToLower(content) + if content == "all" { + qparams.content = QUERY_CONTENT_ALL + } else if content == "config" { + qparams.content = QUERY_CONTENT_CONFIG + } else if content == "nonconfig" || content == "state" { + qparams.content = QUERY_CONTENT_NONCONFIG + } else if content == "operational" { + qparams.content = QUERY_CONTENT_OPERATIONAL + } + } else { + qparams.content = QUERY_CONTENT_ALL + } + if len(fields) > 0 { + if qpDepthContentPresent { + err = tlerr.NotSupported("Fields query parameter is not supported along with other query parameters.") + return qparams, err + } + + // Need to fill based on how the field values are filled + // the fields can be xpaths for the fields to be filtered out + qparams.fields = fields + qparams.tgtFieldsXpathMap = make(map[string][]string) + qparams.allowFieldsXpath = make(map[string]bool) + } + return qparams, nil +} + +func isChildTraversalRequired(xpath string, qParams *QueryParams, childXpath string) bool { + traverse := false + if len(xpath) == 0 || len(childXpath) == 0 { + return traverse + } + if !strings.HasPrefix(childXpath, xpath) { + return traverse + } + if qParams == nil || (!qParams.depthEnabled && qParams.content == QUERY_CONTENT_ALL) { + // traversal required for all levels + return true + } + if strings.HasPrefix(childXpath, xpath) { + if qParams.content != QUERY_CONTENT_ALL { + if xYangSpecInfo, specOk := xYangSpecMap[childXpath]; specOk { + var yangEntry *yang.Entry + yangType := xYangSpecInfo.yangType + if yangType == YANG_LEAF || yangType == YANG_LEAF_LIST { + yangEntry = getYangEntryForXPath(childXpath) + } else { + yangEntry = xYangSpecInfo.yangEntry + } + if yangEntry != nil { + var isOcMdl bool + if strings.HasPrefix(childXpath, "/"+OC_MDL_PFX) { + isOcMdl = true + } + yangNdInfo := contentQPSpecMapInfo{ + yangType: yangType, + yangName: yangEntry.Name, + isReadOnly: yangEntry.ReadOnly(), + isOperationalNd: xYangSpecInfo.operationalQP, + hasNonTerminalNd: xYangSpecInfo.hasNonTerminalNode, + hasChildOperationalNd: xYangSpecInfo.hasChildOpertnlNd, + isOcMdl: isOcMdl, + } + traverse, _ = contentQParamYangNodeProcess(childXpath, yangNdInfo, *qParams) + } else { + xfmrLogInfo("yang entry is nil for xpath ", childXpath) + } + } + if !traverse { + xfmrLogInfo("Child traversal not needed for xpath %v due to content QP(%v)", childXpath, qParams.content) + } + } + if !qParams.depthEnabled { + return traverse + } + + xpathList := strings.Split(xpath, "/") + xpathList = xpathList[1:] + childXpathList := strings.Split(childXpath, "/") + childXpathList = childXpathList[1:] + + // Evaluate traversal required for childXpath w.r t requested depth + depthDiff := uint(len(childXpathList) - len(xpathList)) + // The depth begins from 1 which is already included in the xpath considered + // Hence the difference in depth to be considered is 1 less than request depth + reqDiff := qParams.curDepth - 1 + if depthDiff < reqDiff { + // If the childXpath is non terminal container/list then table read is required + traverse = true + } else if depthDiff == reqDiff { + // When depth is met and the childXpath is container/list then table read is not required + // only for terminal nodes DB read would be required + if xspecInfo, ok := xYangSpecMap[childXpath]; ok { + yangType := xspecInfo.yangType + if yangType == YANG_LEAF_LIST || yangType == YANG_LEAF { + traverse = true + } + } + } + } // else not a child YANG path + return traverse +} + func getXfmrSpecInfoFromUri(uri string) (interface{}, error) { // function to extract xfmr spec info for a given sonic uri/path var err error @@ -1614,6 +1777,377 @@ func getXfmrSpecInfoFromUri(uri string) (interface{}, error) { return specInfo, err } +func contentQParamTgtEval(uri string, qParams QueryParams) (bool, error) { + /*function to evaluate target URI , based on content query parameter to decide : + 1.)Return bad-request if Target Uri points to leaf/leaf-list and content + query-param doesn't match to the read-only flag of that YANG node or + doesn't match to the operationalQP flag in spec map. + 2.)Don't further process URI if its of OC Yang and points to container named + "config" and content query-param is non-config or operational + 3.)Don't further process URI if its read-only container or list and content + query-param is config + Return values are true/false(whether to process request further) and error. + */ + var err error + var specInfo interface{} + var specInfoOk, isSonicUri bool + var yangEntry *yang.Entry + var yangType yangElementType + var dbSpecInfo *dbInfo + var xYangSpecInfo *yangXpathInfo + var processReq bool = true + + yangXpath, _, _ := XfmrRemoveXPATHPredicates(uri) + if qParams.content == QUERY_CONTENT_ALL { + err = nil + processReq = true + goto exitContentTgtEval + } + if (qParams.content == QUERY_CONTENT_OPERATIONAL) && (isSonicYang(uri) || !strings.HasPrefix(uri, "/"+OC_MDL_PFX)) { + processReq = false + err = tlerr.InvalidArgsError{Format: "Bad Request - operational content type is invalid for sonic and non-openconfig YANG request."} + goto exitContentTgtEval + } + specInfo, err = getXfmrSpecInfoFromUri(uri) + if err != nil { + processReq = false + goto exitContentTgtEval + } + + if isSonicYang(uri) { + isSonicUri = true + dbSpecInfo, specInfoOk = specInfo.(*dbInfo) + if specInfoOk { + yangType = dbSpecInfo.yangType + if yangType == YANG_LEAF || yangType == YANG_LEAF_LIST { + pathList := strings.Split(yangXpath, "/") + if len(pathList) > SONIC_FIELD_INDEX { + dbXpath := pathList[SONIC_TABLE_INDEX] + "/" + pathList[SONIC_FIELD_INDEX] + yangEntry = getYangEntryForXPath(dbXpath) + } + } else { + yangEntry = dbSpecInfo.dbEntry + } + } + } else { + xYangSpecInfo, specInfoOk = specInfo.(*yangXpathInfo) + if specInfoOk { + yangType = xYangSpecInfo.yangType + if yangType == YANG_LEAF || yangType == YANG_LEAF_LIST { + yangEntry = getYangEntryForXPath(yangXpath) + } else { + yangEntry = xYangSpecInfo.yangEntry + } + } + } + + if !specInfoOk || (yangEntry == nil) { + processReq = false + err = fmt.Errorf("no YANG meta-data for translation for URI - %v", uri) + xfmrLogInfo("%v", err) + goto exitContentTgtEval + } + if !isSonicUri { + var isOcMdl bool + if strings.HasPrefix(uri, "/"+OC_MDL_PFX) { + isOcMdl = true + } + yangNdInfo := contentQPSpecMapInfo{ + yangType: yangType, + yangName: yangEntry.Name, + isReadOnly: yangEntry.ReadOnly(), + isOperationalNd: xYangSpecInfo.operationalQP, + hasNonTerminalNd: xYangSpecInfo.hasNonTerminalNode, + hasChildOperationalNd: xYangSpecInfo.hasChildOpertnlNd, + isOcMdl: isOcMdl, + } + processReq, err = contentQParamYangNodeProcess(uri, yangNdInfo, qParams) + } else { + xfmrLogDebug("Sonic YANG content query param target URI eval") + //top container query + if strings.Count(uri, "/") == 1 { + processReq = true + goto exitContentTgtEval + } + processReq, err = sonicContentQParamYangNodeProcess(uri, yangType, yangEntry.ReadOnly(), qParams) + } + +exitContentTgtEval: + xfmrLogDebug("content target eval returning - process request further %v, error - %v", processReq, err) + return processReq, err + +} + +var rejectComplexYangNodeForQParamContent = func(content ContentType, isOcMdl bool, hasChildOpertnlNd bool) bool { + /* IETF model containeri/list having none RO child nodes */ + ietf_pure_rw_complex_node := (content == QUERY_CONTENT_NONCONFIG) && (!isOcMdl) && (!hasChildOpertnlNd) + + /* containeri/list having none opearational child nodes */ + no_operational_children_complex_node := (content == QUERY_CONTENT_OPERATIONAL) && (!hasChildOpertnlNd) + + if ietf_pure_rw_complex_node || no_operational_children_complex_node { + return true + } + return false +} + +func contentQParamYangNodeProcess(uri string, yangNdInfo contentQPSpecMapInfo, qParams QueryParams) (bool, error) { + /*function to decide whether to traverse a Non-Sonic(OC & Ietf) YANG node based + on content query params QUERY_CONTENT_CONFIG/QUERY_CONTENT_NONCONFIG/QUERY_CONTENT_OPERATIONAL. + */ + var processReq bool + var err error + var yangNdType yangElementType + content_mismatch_err := "Bad Request - requested content type doesn't match content type of terminal node uri." + + if uri == "" { + processReq = false + err = fmt.Errorf("Empty URI sent") + goto contentQParamYangNodeProcessExit + } + yangNdType = yangNdInfo.yangType + if (qParams.content == QUERY_CONTENT_CONFIG) && (yangNdInfo.isReadOnly) { //applies to leaf, leaf-list, container and list + processReq = false + if yangNdType == YANG_LEAF || yangNdType == YANG_LEAF_LIST { + err = tlerr.InvalidArgsError{Format: content_mismatch_err} + } + goto contentQParamYangNodeProcessExit + } + + switch yangNdType { + case YANG_LEAF, YANG_LEAF_LIST: + if (yangNdInfo.isReadOnly) && (qParams.content == QUERY_CONTENT_OPERATIONAL) { + if yangNdInfo.isOperationalNd { + processReq = true + } else { + processReq = false + err = tlerr.InvalidArgsError{Format: content_mismatch_err} + } + } else if ((qParams.content == QUERY_CONTENT_NONCONFIG) || (qParams.content == QUERY_CONTENT_OPERATIONAL)) && (!yangNdInfo.isReadOnly) { + processReq = false + err = tlerr.InvalidArgsError{Format: content_mismatch_err} + } else { + processReq = true + } + case YANG_CONTAINER: + containerNm := yangNdInfo.yangName + if strings.Contains(containerNm, ":") { + containerNm = strings.SplitN(containerNm, ":", 2)[1] + } + + var oc_config_or_terminal_rw_container bool + /* OC model container either having name "config" or is a terminal RW container for content=nonconfig|operational */ + if (qParams.content == QUERY_CONTENT_NONCONFIG) || (qParams.content == QUERY_CONTENT_OPERATIONAL) { + if yangNdInfo.isOcMdl { + if (containerNm == YANG_CONTAINER_NM_CONFIG) || ((!yangNdInfo.hasNonTerminalNd) && (!yangNdInfo.isReadOnly)) { + oc_config_or_terminal_rw_container = true + } + } + } + if oc_config_or_terminal_rw_container || rejectComplexYangNodeForQParamContent(qParams.content, yangNdInfo.isOcMdl, yangNdInfo.hasChildOperationalNd) { + processReq = false + } else { + processReq = true + } + case YANG_LIST: + var oc_terminal_rw_list bool + + /* OC Model terminal RW list */ + if (qParams.content == QUERY_CONTENT_NONCONFIG) || (qParams.content == QUERY_CONTENT_OPERATIONAL) { + if yangNdInfo.isOcMdl { + if (!yangNdInfo.hasNonTerminalNd) && (!yangNdInfo.isReadOnly) { + oc_terminal_rw_list = true + } + } + } + if oc_terminal_rw_list || rejectComplexYangNodeForQParamContent(qParams.content, yangNdInfo.isOcMdl, yangNdInfo.hasChildOperationalNd) { + processReq = false + } else { + processReq = true + } + default: + if ((qParams.content == QUERY_CONTENT_NONCONFIG) || (qParams.content == QUERY_CONTENT_OPERATIONAL)) && !yangNdInfo.isReadOnly { + processReq = false + } else { + processReq = true + } + xfmrLogDebug("default content QP processing for node type %v, for URI %v", yangNdType, uri) + } +contentQParamYangNodeProcessExit: + xfmrLogDebug("returning process request - %v, error -%v , for URI - %v", processReq, err, uri) + return processReq, err +} + +func sonicContentQParamYangNodeProcess(uri string, yangNdType yangElementType, ReadOnly bool, qParams QueryParams) (bool, error) { + var processReq bool + var err error + content_mismatch_err := "Bad Request - requested content type doesn't match content type of terminal node uri." + + if uri == "" { + processReq = false + err = fmt.Errorf("Empty URI sent") + goto sonicContentQParamYangNodeProcessExit + } + if (qParams.content == QUERY_CONTENT_CONFIG) && (ReadOnly) { + processReq = false + if yangNdType == YANG_LEAF || yangNdType == YANG_LEAF_LIST { + err = tlerr.InvalidArgsError{Format: content_mismatch_err} + } + goto sonicContentQParamYangNodeProcessExit + } + + switch yangNdType { + case YANG_LEAF, YANG_LEAF_LIST: + if (qParams.content == QUERY_CONTENT_NONCONFIG) && !ReadOnly { + processReq = false + err = tlerr.InvalidArgsError{Format: content_mismatch_err} + } else { + processReq = true + } + case YANG_CONTAINER, YANG_LIST: + /*as per sonic YANG guidelines when inner container(table) and list are ReadWrite + then they DON"T have ReadOnly elements + */ + if (qParams.content == QUERY_CONTENT_NONCONFIG) && !ReadOnly { + processReq = false + } else { + processReq = true + } + default: + if (qParams.content == QUERY_CONTENT_NONCONFIG) && !ReadOnly { + processReq = false + } else { + processReq = true + } + xfmrLogDebug("default content QP processing for node type %v, for URI %v", yangNdType, uri) + } +sonicContentQParamYangNodeProcessExit: + xfmrLogDebug("returning process request - %v, error -%v , for URI - %v", processReq, err, uri) + return processReq, err +} + +/* Validate "fields" query parameter for sonic YANG */ +func validateAndFillSonicQpFields(inParamsForGet xlateFromDbParams) error { + var err error + curAllowedXpath := "" + if len(strings.Split(inParamsForGet.xpath, "/")) > 2 { + curAllowedXpath = strings.Split(inParamsForGet.xpath, "/")[2] + } + if len(curAllowedXpath) > 0 { + inParamsForGet.queryParams.allowFieldsXpath[curAllowedXpath] = true + } + for _, field := range inParamsForGet.queryParams.fields { + curXpath := inParamsForGet.tbl + curAllowedXpath = strings.Join(strings.Split(inParamsForGet.xpath, "/")[2:], "/") + fpath := strings.Split(field, "/") + curTable := inParamsForGet.tbl + for _, p := range fpath { + if strings.Contains(p, "[") { + err = tlerr.NotSupported("Yang node type list not supported in fields query parameter(%v).", field) + return err + } + if len(p) == 0 { + continue + } + if len(curXpath) > 0 { + curXpath += "/" + p + } else { + curXpath = p + curAllowedXpath = p + curTable = p + } + yNode, ok := xDbSpecMap[curXpath] + if !ok || yNode == nil { + err = tlerr.InvalidArgs("Invalid field name/path: %v", field) + return err + } + /* check for list */ + if yNode.yangType == YANG_LIST { + err = tlerr.NotSupported("Yang node type list not supported in fields query parameter(%v).", field) + return err + } else if yNode.yangType == YANG_CONTAINER && (yNode.dbEntry != nil) && (yNode.dbEntry.Parent != nil) && (curTable == yNode.dbEntry.Parent.Name) { + // singleton container case + curAllowedXpath = curXpath + curXpath = curTable + } + inParamsForGet.queryParams.allowFieldsXpath[curAllowedXpath] = true + } + if _, ok := inParamsForGet.queryParams.tgtFieldsXpathMap[curXpath]; !ok { + if xDbSpecMap[curXpath].yangType == YANG_LEAF_LIST { + curXpath = curXpath + string('@') + } + inParamsForGet.queryParams.tgtFieldsXpathMap[curXpath] = []string{} + } + } + return err +} + +/* Validate "fields" query parameter */ +func validateAndFillQpFields(inParamsForGet xlateFromDbParams) error { + var err error + for _, field := range inParamsForGet.queryParams.fields { + curXpath := inParamsForGet.xpath + fpath := strings.Split(field, "/") + curSubtree := xYangSpecMap[curXpath].xfmrFunc + fpathPrefix := "" + subtreeXpath := "" + subTreeFld := "" + if len(curSubtree) > 0 { + subtreeXpath = curXpath + subTreeFld = field + } + for _, p := range fpath { + /* check for list instance */ + if strings.Contains(p, "[") { + err = tlerr.NotSupported("Yang node type list not in fields query parameter(%v).", field) + return err + } + if len(p) == 0 { + continue + } + if strings.Contains(p, ":") { + p = p[strings.LastIndex(p, ":")+1:] + } + curXpath += "/" + p + yNode, ok := xYangSpecMap[curXpath] + if !ok { + err = tlerr.InvalidArgs("Invalid field name/path: %v", field) + return err + } + /* check for list */ + if yNode.yangType == YANG_LIST { + err = tlerr.NotSupported("Yang node type list not in fields query parameter(%v).", field) + return err + } + + fpathPrefix += p + "/" + if len(yNode.xfmrFunc) > 0 && (curSubtree == "" || curSubtree != yNode.xfmrFunc) { + subtreeXpath = curXpath + if field == strings.TrimSuffix(fpathPrefix, "/") { + subTreeFld = "" + } else { + subTreeFld = strings.Replace(field, fpathPrefix, "", 1) + } + curSubtree = yNode.xfmrFunc + } + inParamsForGet.queryParams.allowFieldsXpath[curXpath] = true + } + + if len(subtreeXpath) > 0 { + /* xpaths handled by subtree-xfmr */ + if _, ok := inParamsForGet.queryParams.tgtFieldsXpathMap[subtreeXpath]; !ok { + inParamsForGet.queryParams.tgtFieldsXpathMap[subtreeXpath] = []string{} + } + inParamsForGet.queryParams.tgtFieldsXpathMap[subtreeXpath] = + append(inParamsForGet.queryParams.tgtFieldsXpathMap[subtreeXpath], subTreeFld) + } else { + /* xpaths handled by infra */ + inParamsForGet.queryParams.tgtFieldsXpathMap[curXpath] = []string{} + } + } + return err +} + func escapeKeyVal(val string) string { val = strings.Replace(val, "]", "\\]", -1) val = strings.Replace(val, "/", "\\/", -1) @@ -1770,6 +2304,43 @@ done: return entry } +func hasSameOcSonicKeys(listUri string, xpath string, table string) bool { + sameKeys := false + if len(listUri) == 0 || len(table) == 0 { + xfmrLogDebug("Empty uri / sonic table received") + return sameKeys + } + + xfmrLogDebug("hasSameOcSonicKeys:listUri: %v, xpath %v, table: %v", listUri, xpath, table) + if xspec, ok := xYangSpecMap[xpath]; ok { + if dbSpec, tableok := xDbSpecMap[table]; tableok { + if dbSpec.dbEntry != nil { + /* Traverse the lists to find if we have atleast one list with matching keys */ + /* For lists with not matching keys key-xfmr is always expected */ + ocKeyNameList := xspec.yangEntry.Key + if len(dbSpec.dbEntry.Dir) > 1 { + // Do not handle multi list cases. Key xfmr mandatory here + return false + } + for k := range dbSpec.dbEntry.Dir { + sncKeyNameList := dbSpec.dbEntry.Dir[k].Key + xfmrLogDebug("hasSameOcSonicKeys: ocKeyNameList : %v, sncKeyNameList: %v", ocKeyNameList, sncKeyNameList) + if len(ocKeyNameList) == len(sncKeyNameList) { + ocKey := strings.ToLower(ocKeyNameList) + ocKey = strings.ReplaceAll(ocKey, "-", "_") + sncKey := strings.ToLower(sncKeyNameList) + sncKey = strings.ReplaceAll(sncKey, "-", "_") + if ocKey == sncKey { + return true + } + } + } + } + } + } + return sameKeys +} + func (inPm XfmrParams) String() string { return fmt.Sprintf("{oper: %v, uri: %v, requestUri: %v, "+ "DB Name at current node: %v, table: %v, key: %v, dbDataMap: %v, subOpDataMap: %v, yangDefValMap: %v "+ @@ -1824,11 +2395,11 @@ func SonicUriHasSingletonContainer(uri string) bool { } xpath, _, err := XfmrRemoveXPATHPredicates(uri) - if err != nil || len(xpath) == 0 { - return hasSingletonContainer - } + if err != nil || len(xpath) == 0 { + return hasSingletonContainer + } - pathList := strings.Split(xpath, "/") + pathList := strings.Split(xpath, "/") if len(pathList) > SONIC_TBL_CHILD_INDEX { tblChldXpath := pathList[SONIC_TABLE_INDEX] + "/" + pathList[SONIC_TBL_CHILD_INDEX] @@ -1840,3 +2411,82 @@ func SonicUriHasSingletonContainer(uri string) bool { } return hasSingletonContainer } + +// IsEnabled : Exported version. translib.common_app needs this. +func (qp *QueryParams) IsEnabled() bool { + return qp.isEnabled() +} + +func (qp *QueryParams) isEnabled() bool { + return qp.isDepthEnabled() || + qp.isContentEnabled() || + qp.isFieldsEnabled() +} + +func (qp *QueryParams) isDepthEnabled() bool { + return qp.depthEnabled +} + +func (qp *QueryParams) IsContentEnabled() bool { + // IsEnabled : Exported version. translib.common_app needs this. + return qp.isContentEnabled() +} + +func (qp *QueryParams) isContentEnabled() bool { + return qp.content != QUERY_CONTENT_ALL +} + +func (qp *QueryParams) isFieldsEnabled() bool { + return len(qp.fields) != 0 +} + +func (ct ContentType) String() string { + ret := "Unknown" + switch ct { + case QUERY_CONTENT_ALL: + ret = "ALL" + case QUERY_CONTENT_CONFIG: + ret = "CONFIG" + case QUERY_CONTENT_NONCONFIG: + ret = "NONCONFIG" + case QUERY_CONTENT_OPERATIONAL: + ret = "OPERATIONAL" + } + return ret +} + +func (qp QueryParams) String() string { + qp_ptr := &qp + if qp_ptr.isEnabled() { + return fmt.Sprintf("QueryParams{depthEnabled: %v, curDepth: %v, content: %v, fields: %v, fieldsFillAll: %v, "+ + "allowFieldsXpath: %v, tgtFieldsXpathMap: %v}", qp.depthEnabled, qp.curDepth, qp.content, + qp.fields, qp.fieldsFillAll, qp.allowFieldsXpath, qp.tgtFieldsXpathMap) + } else { + return "" + } +} + +func (e *qpSubtreePruningErr) Error() string { + return fmt.Sprintf("Query Parameter pruning unsuccessful for subtree - %s", e.subtreePath) +} + +func extractLeafValFromUriKey(uri string, keyLeafNm string) string { + /* function to extract string value for key leaf from last list in uri + string value will have unescaped characters if they were escaped in + URI string + */ + var keyLeafVal string + xfmrLogDebug("Extract value for leaf %v from URI %v", keyLeafNm, uri) + uriElementList := splitUri(uri) + for idx := len(uriElementList) - 1; idx >= 0; idx-- { + pathInfo := NewPathInfo("/" + uriElementList[idx]) + if (pathInfo != nil) && (len(pathInfo.Vars) > 0) { + if pathInfo.HasVar(keyLeafNm) { + keyLeafVal = pathInfo.Var(keyLeafNm) + } + break + } + } + xfmrLogDebug("Returning value %v for leaf %v", keyLeafVal, keyLeafNm) + return keyLeafVal +} diff --git a/translib/transformer/xlate_xfmr_handler.go b/translib/transformer/xlate_xfmr_handler.go index 64ae3e9d5e30..0373ef41e71a 100644 --- a/translib/transformer/xlate_xfmr_handler.go +++ b/translib/transformer/xlate_xfmr_handler.go @@ -21,6 +21,7 @@ package transformer import ( "github.com/Azure/sonic-mgmt-common/translib/db" log "github.com/golang/glog" + "github.com/openconfig/ygot/ygot" ) func xfmrHandlerFunc(inParams XfmrParams, xfmrFuncNm string) error { @@ -48,9 +49,27 @@ func xfmrHandlerFunc(inParams XfmrParams, xfmrFuncNm string) error { } } } + if (err == nil) && inParams.queryParams.isEnabled() { + log.Infof("xfmrPruneQP: func %v URI %v, requestUri %v", + xfmrFuncNm, inParams.uri, inParams.requestUri) + err = xfmrPruneQP(inParams.ygRoot, inParams.queryParams, + inParams.uri, inParams.requestUri) + if err != nil { + xfmrLogInfo("xfmrPruneQP: returned error %v", err) + // following will allow xfmr to distinguish subtree vs pruning API err to abort GET request + err = &qpSubtreePruningErr{subtreePath: inParams.uri} + } + } return err } +/*Place holder for xfmrPruneQP API */ +func xfmrPruneQP(ygRoot *ygot.GoStruct, queryParams QueryParams, uri string, + requestUri string) error { + // TODO + return nil +} + func leafXfmrHandlerFunc(inParams XfmrParams, xfmrFieldFuncNm string) (map[string]interface{}, error) { const ( DBTY_FLD_XFMR_RET_ARGS = 2 diff --git a/translib/transformer/xspec.go b/translib/transformer/xspec.go index ceb873a38349..896b67cb0290 100644 --- a/translib/transformer/xspec.go +++ b/translib/transformer/xspec.go @@ -68,12 +68,15 @@ type yangXpathInfo struct { cascadeDel int8 virtualTbl *bool nameWithMod *string + operationalQP bool + hasChildOpertnlNd bool yangType yangElementType xfmrPath string compositeFields []string dbKeyCompCnt int subscriptionFlags utils.Bits isDataSrcDynamic *bool + isRefByKey bool } type dbInfo struct { @@ -91,6 +94,7 @@ type dbInfo struct { hasXfmrFn bool cascadeDel int8 yangType yangElementType + isKey bool } type moduleAnnotInfo struct { @@ -215,6 +219,29 @@ func childSubTreePresenceFlagSet(xpath string) { } } +func childOperationalNodeFlagSet(xpath string) { + if xpathData, ok := xYangSpecMap[xpath]; ok { + yangType := xpathData.yangType + if (yangType != YANG_LEAF) && (yangType != YANG_LEAF_LIST) { + xpathData.hasChildOpertnlNd = true + } + } + + parXpath := parentXpathGet(xpath) + for { + if parXpath == "" { + break + } + if parXpathData, ok := xYangSpecMap[parXpath]; ok { + if parXpathData.hasChildOpertnlNd { + break + } + parXpathData.hasChildOpertnlNd = true + } + parXpath = parentXpathGet(parXpath) + } +} + /* Recursive api to fill the map with yang details */ func yangToDbMapFill(keyLevel uint8, xYangSpecMap map[string]*yangXpathInfo, entry *yang.Entry, xpathPrefix string, xpathFull string) { xpath := "" @@ -389,8 +416,12 @@ func yangToDbMapFill(keyLevel uint8, xYangSpecMap map[string]*yangXpathInfo, ent if ((yangType == YANG_LEAF) || (yangType == YANG_LEAF_LIST)) && (len(xpathData.fieldName) > 0) && (xpathData.tableName != nil) { dbPath := *xpathData.tableName + "/" + xpathData.fieldName - if xDbSpecMap[dbPath] != nil { + _, ok := xDbSpecMap[dbPath] + if ok && xDbSpecMap[dbPath] != nil { xDbSpecMap[dbPath].yangXpath = append(xDbSpecMap[dbPath].yangXpath, xpath) + if xDbSpecMap[dbPath].isKey { + xpathData.fieldName = "" + } } } @@ -401,6 +432,7 @@ func yangToDbMapFill(keyLevel uint8, xYangSpecMap map[string]*yangXpathInfo, ent /* create list with current keys */ keyXpath := make([]string, len(strings.Split(entry.Key, " "))) + isOcMdl := strings.HasPrefix(xpath, "/"+OC_MDL_PFX) for id, keyName := range strings.Split(entry.Key, " ") { keyXpath[id] = xpath + "/" + keyName if _, ok := xYangSpecMap[xpath+"/"+keyName]; !ok { @@ -410,6 +442,34 @@ func yangToDbMapFill(keyLevel uint8, xYangSpecMap map[string]*yangXpathInfo, ent xYangSpecMap[xpath+"/"+keyName] = keyXpathData } xYangSpecMap[xpath+"/"+keyName].isKey = true + if isOcMdl { + var keyLfsInContainerXpaths []string + if configContEntry, ok := entry.Dir["config"]; ok && configContEntry != nil { //OC Mdl list has config container + if _, keyLfOk := configContEntry.Dir[keyName]; keyLfOk { + keyLfsInContainerXpaths = append(keyLfsInContainerXpaths, xpath+CONFIG_CNT_WITHIN_XPATH+keyName) + } + } + + /* Mark OC Model list state-container leaves that are also list key-leaves,as isRefByKey,even though there + is no yang leaf-reference from list-key leaves.This will enable xfmr infra to fill them and elliminate + app annotation + */ + if stateContEntry, ok := entry.Dir["state"]; ok && stateContEntry != nil { //OC Mdl list has state container + if _, keyLfOk := stateContEntry.Dir[keyName]; keyLfOk { + keyLfsInContainerXpaths = append(keyLfsInContainerXpaths, xpath+STATE_CNT_WITHIN_XPATH+keyName) + } + } + + for _, keyLfInContXpath := range keyLfsInContainerXpaths { + if _, ok := xYangSpecMap[keyLfInContXpath]; !ok { + xYangSpecMap[keyLfInContXpath] = new(yangXpathInfo) + xYangSpecMap[keyLfInContXpath].subscribeMinIntvl = XFMR_INVALID + xYangSpecMap[keyLfInContXpath].dbIndex = db.ConfigDB // default value + } + xYangSpecMap[keyLfInContXpath].isRefByKey = true + } + + } } xpathData.keyXpath = make(map[int]*[]string, (parentKeyLen + 1)) @@ -465,6 +525,44 @@ func yangToDbMapFill(keyLevel uint8, xYangSpecMap map[string]*yangXpathInfo, ent } } +/* find and set operation query parameter nodes */ +var xlateUpdateQueryParamInfo = func() { + + for xpath := range xYangSpecMap { + if xYangSpecMap[xpath].yangEntry != nil { + readOnly := xYangSpecMap[xpath].yangEntry.ReadOnly() + if xYangSpecMap[xpath].yangType == YANG_LEAF || xYangSpecMap[xpath].yangType == YANG_LEAF_LIST { + xYangSpecMap[xpath].yangEntry = nil //memory optimization - don't cache for leafy nodes + } + if !readOnly { + continue + } + } + + if strings.Contains(xpath, STATE_CNT_WITHIN_XPATH) { + cfgXpath := strings.Replace(xpath, STATE_CNT_WITHIN_XPATH, CONFIG_CNT_WITHIN_XPATH, -1) + if strings.HasSuffix(cfgXpath, STATE_CNT_SUFFIXED_XPATH) { + suffix_idx := strings.LastIndex(cfgXpath, STATE_CNT_SUFFIXED_XPATH) + cfgXpath = cfgXpath[:suffix_idx] + CONFIG_CNT_SUFFIXED_XPATH + } + if _, ok := xYangSpecMap[cfgXpath]; !ok { + xYangSpecMap[xpath].operationalQP = true + childOperationalNodeFlagSet(xpath) + } + } else if strings.HasSuffix(xpath, STATE_CNT_SUFFIXED_XPATH) { + suffix_idx := strings.LastIndex(xpath, STATE_CNT_SUFFIXED_XPATH) + cfgXpath := xpath[:suffix_idx] + CONFIG_CNT_SUFFIXED_XPATH + if _, ok := xYangSpecMap[cfgXpath]; !ok { + xYangSpecMap[xpath].operationalQP = true + childOperationalNodeFlagSet(xpath) + } + } else { + xYangSpecMap[xpath].operationalQP = true + childOperationalNodeFlagSet(xpath) + } + } +} + /* Build lookup table based of yang xpath */ func yangToDbMapBuild(entries map[string]*yang.Entry) { if entries == nil { @@ -489,6 +587,7 @@ func yangToDbMapBuild(entries map[string]*yang.Entry) { jsonfile := YangPath + TblInfoJsonFile xlateJsonTblInfoLoad(sonicOrdTblListMap, jsonfile) + xlateUpdateQueryParamInfo() sonicLeafRefMap = nil } @@ -559,11 +658,13 @@ func dbMapFill(tableName string, curPath string, moduleNm string, xDbSpecMap map dbXpath = tableName + "/" + entry.Name if tblSpecInfo, tblOk = xDbSpecMap[tableName]; tblOk { tblDbIndex = xDbSpecMap[tableName].dbIndex - } + } } xDbSpecPath = dbXpath - xDbSpecMap[dbXpath] = new(dbInfo) - xDbSpecMap[dbXpath].dbIndex = tblDbIndex + if _, ok := xDbSpecMap[dbXpath]; !ok { + xDbSpecMap[dbXpath] = new(dbInfo) + } + xDbSpecMap[dbXpath].dbIndex = tblDbIndex xDbSpecMap[dbXpath].yangType = entryType xDbSpecMap[dbXpath].dbEntry = entry xDbSpecMap[dbXpath].module = moduleNm @@ -592,6 +693,13 @@ func dbMapFill(tableName string, curPath string, moduleNm string, xDbSpecMap map } else if tblOk && (entryType == YANG_LIST && len(entry.Key) != 0) { tblSpecInfo.listName = append(tblSpecInfo.listName, entry.Name) xDbSpecMap[dbXpath].keyList = append(xDbSpecMap[dbXpath].keyList, strings.Split(entry.Key, " ")...) + for _, keyVal := range xDbSpecMap[dbXpath].keyList { + dbXpathForKeyLeaf := tableName + "/" + keyVal + if _, ok := xDbSpecMap[dbXpathForKeyLeaf]; !ok { + xDbSpecMap[dbXpathForKeyLeaf] = new(dbInfo) + } + xDbSpecMap[dbXpathForKeyLeaf].isKey = true + } } else if entryType == YANG_LEAF || entryType == YANG_LEAF_LIST { /* TODO - Uncomment this line once subscription changes for memory optimization ready xDbSpecMap[dbXpath].dbEntry = nil //memory optimization - don't cache for leafy nodes @@ -1114,6 +1222,9 @@ func mapPrint(fileName string) { fmt.Fprintf(fp, " %d. %#v\r\n", i, kd) } fmt.Fprintf(fp, "\r\n isKey : %v\r\n", d.isKey) + fmt.Fprintf(fp, "\r\n isRefByKey : %v\r\n", d.isRefByKey) + fmt.Fprintf(fp, "\r\n operQP : %v\r\n", d.operationalQP) + fmt.Fprintf(fp, "\r\n hasChildOperQP : %v\r\n", d.hasChildOpertnlNd) fmt.Fprintf(fp, "\r\n isDataSrcDynamic: ") if d.isDataSrcDynamic != nil { fmt.Fprintf(fp, "%v", *d.isDataSrcDynamic) @@ -1140,6 +1251,7 @@ func dbMapPrint(fname string) { for k, v := range xDbSpecMap { fmt.Fprintf(fp, " field:%v: \r\n", k) fmt.Fprintf(fp, " type :%v \r\n", getYangTypeStrId(v.yangType)) + fmt.Fprintf(fp, " isKey :%v \r\n", v.isKey) fmt.Fprintf(fp, " db-type :%v \r\n", v.dbIndex) fmt.Fprintf(fp, " hasXfmrFn:%v \r\n", v.hasXfmrFn) fmt.Fprintf(fp, " module :%v \r\n", v.module) diff --git a/translib/translib.go b/translib/translib.go index 191cf427afee..5105065504cd 100644 --- a/translib/translib.go +++ b/translib/translib.go @@ -79,16 +79,19 @@ type SetResponse struct { Err error } +type QueryParameters struct { + Depth uint // range 1 to 65535, default is 0 i.e. all + Content string // all, config, non-config(REST)/state(GNMI), operational(GNMI only) + Fields []string // list of fields from NBI +} + type GetRequest struct { Path string FmtType TranslibFmtType User UserRoles AuthEnabled bool ClientVersion Version - - // Depth limits the depth of data subtree in the response - // payload. Default value 0 indicates there is no limit. - Depth uint + QueryParams QueryParameters } type GetResponse struct { @@ -452,7 +455,7 @@ func Get(req GetRequest) (GetResponse, error) { return resp, err } - opts := appOptions{depth: req.Depth} + opts := appOptions{depth: req.QueryParams.Depth, content: req.QueryParams.Content, fields: req.QueryParams.Fields} err = appInitialize(app, appInfo, path, nil, &opts, GET) if err != nil {