Skip to content

Commit

Permalink
CVL Changes sonic-net#9: Clean up CVL code and integrate custom xpath…
Browse files Browse the repository at this point in the history 
… engine for leafref, when, must expression evaluation (sonic-net#33)

Clean up CVL code
YIN schema reading using libyang API
Integrate with xpath custom engine for leafref, when, must expression evaluation
Add delete constraint check
Add max-element constraint
Add CVL trace log to /tmp/cvl.log based on config file
  • Loading branch information
@dutta-partha
dutta-partha authored Jan 12, 2021
1 parent 86c1108 commit 4aa0899
Show file tree
Hide file tree
Showing 10 changed files with 1,493 additions and 1,518 deletions.
1,307 changes: 350 additions & 957 deletions cvl/cvl.go
View file

Large diffs are not rendered by default.

376 changes: 209 additions & 167 deletions cvl/cvl_api.go
View file

Large diffs are not rendered by default.

105 changes: 77 additions & 28 deletions cvl/cvl_cache.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,15 +20,30 @@
package cvl
import (
"encoding/json"
"github.com/go-redis/redis"
"github.com/go-redis/redis/v7"
//lint:ignore ST1001 This is safe to dot import for util package
. "github.com/Azure/sonic-mgmt-common/cvl/internal/util"
"github.com/Azure/sonic-mgmt-common/cvl/internal/yparser"
"time"
"runtime"
"github.com/antchfx/jsonquery"
"github.com/antchfx/xmlquery"
)

func (c *CVL) addTableEntryToCache(tableName string, redisKey string) {
if (tableName == "" || redisKey == "") {
return
}

if (c.tmpDbCache[tableName] == nil) {
c.tmpDbCache[tableName] = map[string]interface{}{redisKey: nil}
} else {
tblMap := c.tmpDbCache[tableName]
tblMap.(map[string]interface{})[redisKey] =nil
c.tmpDbCache[tableName] = tblMap
}
}

// Fetch dependent data from validated data cache,
// Returns the data and flag to indicate that if requested data
// is found in update request, the data should be merged with Redis data
Expand All @@ -37,20 +52,18 @@ func (c *CVL) fetchDataFromRequestCache(tableName string, key string) (d map[str
pd := &d
pm := &m

TRACE_LOG(INFO_API, TRACE_CACHE,
TRACE_LOG(TRACE_CACHE,
"Returning data from request cache, data = %v, merge needed = %v",
*pd, *pm)
}()

cfgDataArr := c.requestCache[tableName][key]
if (cfgDataArr != nil) {
for _, cfgReqData := range cfgDataArr {
//Delete request doesn't have depedent data
if (cfgReqData.reqData.VOp == OP_CREATE) {
return cfgReqData.reqData.Data, false
} else if (cfgReqData.reqData.VOp == OP_UPDATE) {
return cfgReqData.reqData.Data, true
}
for _, cfgReqData := range cfgDataArr {
//Delete request doesn't have depedent data
if (cfgReqData.reqData.VOp == OP_CREATE) {
return cfgReqData.reqData.Data, false
} else if (cfgReqData.reqData.VOp == OP_UPDATE) {
return cfgReqData.reqData.Data, true
}
}

Expand All @@ -60,7 +73,7 @@ func (c *CVL) fetchDataFromRequestCache(tableName string, key string) (d map[str
//Fetch given table entries using pipeline
func (c *CVL) fetchTableDataToTmpCache(tableName string, dbKeys map[string]interface{}) int {

TRACE_LOG(INFO_API, TRACE_CACHE, "\n%v, Entered fetchTableDataToTmpCache", time.Now())
TRACE_LOG(TRACE_CACHE, "\n%v, Entered fetchTableDataToTmpCache", time.Now())

totalCount := len(dbKeys)
if (totalCount == 0) {
Expand Down Expand Up @@ -102,25 +115,27 @@ func (c *CVL) fetchTableDataToTmpCache(tableName string, dbKeys map[string]inter
redisKey := tableName + modelInfo.tableInfo[tableName].redisKeyDelim + dbKey
//Check in validated cache first and add as dependent data
if entry, mergeNeeded := c.fetchDataFromRequestCache(tableName, dbKey); (entry != nil) {
c.tmpDbCache[tableName].(map[string]interface{})[dbKey] = entry
entryFetched = entryFetched + 1
//Entry found in validated cache, so skip fetching from Redis
//if merging is not required with Redis DB
if (mergeNeeded == false) {
continue
}
entryFetched = entryFetched + 1
//Entry found in validated cache, so skip fetching from Redis
//if merging is not required with Redis DB
if !mergeNeeded {
fieldMap := c.checkFieldMap(&entry)
c.tmpDbCache[tableName].(map[string]interface{})[dbKey] = fieldMap
continue
}
c.tmpDbCache[tableName].(map[string]interface{})[dbKey] = entry
}

//Otherwise fetch it from Redis
mCmd[dbKey] = pipe.HGetAll(redisKey) //write into pipeline
if mCmd[dbKey] == nil {
CVL_LOG(ERROR, "Failed pipe.HGetAll('%s')", redisKey)
CVL_LOG(WARNING, "Failed pipe.HGetAll('%s')", redisKey)
}
}

_, err := pipe.Exec()
if err != nil {
CVL_LOG(ERROR, "Failed to fetch details for table %s", tableName)
CVL_LOG(WARNING, "Failed to fetch details for table %s", tableName)
return 0
}
pipe.Close()
Expand All @@ -130,6 +145,11 @@ func (c *CVL) fetchTableDataToTmpCache(tableName string, dbKeys map[string]inter

for key, val := range mCmd {
res, err := val.Result()

if (mapTable == nil) {
break
}

if (err != nil || len(res) == 0) {
//no data found, don't keep blank entry
delete(mapTable.(map[string]interface{}), key)
Expand All @@ -155,14 +175,14 @@ func (c *CVL) fetchTableDataToTmpCache(tableName string, dbKeys map[string]inter
runtime.Gosched()
}

TRACE_LOG(INFO_API, TRACE_CACHE,"\n%v, Exiting fetchTableDataToTmpCache", time.Now())
TRACE_LOG(TRACE_CACHE,"\n%v, Exiting fetchTableDataToTmpCache", time.Now())

return entryFetched
}

//populate redis data to cache
func (c *CVL) fetchDataToTmpCache() *yparser.YParserNode {
TRACE_LOG(INFO_API, TRACE_CACHE, "\n%v, Entered fetchToTmpCache", time.Now())
TRACE_LOG(TRACE_CACHE, "\n%v, Entered fetchToTmpCache", time.Now())

entryToFetch := 0
var root *yparser.YParserNode = nil
Expand Down Expand Up @@ -191,7 +211,7 @@ func (c *CVL) fetchDataToTmpCache() *yparser.YParserNode {
if Tracing {
jsonDataBytes, _ := json.Marshal(c.tmpDbCache)
jsonData := string(jsonDataBytes)
TRACE_LOG(INFO_API, TRACE_CACHE, "Top Node=%v\n", jsonData)
TRACE_LOG(TRACE_CACHE, "Top Node=%v\n", jsonData)
}

data, err := jsonquery.ParseJsonMap(&c.tmpDbCache)
Expand All @@ -202,7 +222,7 @@ func (c *CVL) fetchDataToTmpCache() *yparser.YParserNode {

//Build yang tree for each table and cache it
for jsonNode := data.FirstChild; jsonNode != nil; jsonNode=jsonNode.NextSibling {
TRACE_LOG(INFO_API, TRACE_CACHE, "Top Node=%v\n", jsonNode.Data)
TRACE_LOG(TRACE_CACHE, "Top Node=%v\n", jsonNode.Data)
//Visit each top level list in a loop for creating table data
topNode, _ := c.generateTableData(true, jsonNode)
if (root == nil) {
Expand All @@ -212,15 +232,46 @@ func (c *CVL) fetchDataToTmpCache() *yparser.YParserNode {
return nil
}
}

//Generate YANG data for Yang Validator
topYangNode, cvlYErrObj := c.generateYangListData(jsonNode, true)
if topYangNode == nil {
cvlYErrObj.ErrCode = CVL_SYNTAX_ERROR
CVL_LOG(WARNING, "Unable to translate cache data to YANG format")
return nil
}

//Create a full document and merge with main YANG data
doc := &xmlquery.Node{Type: xmlquery.DocumentNode}
doc.FirstChild = topYangNode
doc.LastChild = topYangNode
topYangNode.Parent = doc

if (IsTraceLevelSet(TRACE_CACHE)) {
TRACE_LOG(TRACE_CACHE, "Before cache merge = %s, source = %s",
c.yv.root.OutputXML(false),
doc.OutputXML(false))
}

if c.mergeYangData(c.yv.root, doc) != CVL_SUCCESS {
CVL_LOG(WARNING, "Unable to merge translated YANG data while " +
"translating from cache data to YANG format")
cvlYErrObj.ErrCode = CVL_SYNTAX_ERROR
return nil
}
if (IsTraceLevelSet(TRACE_CACHE)) {
TRACE_LOG(TRACE_CACHE, "After cache merge = %s",
c.yv.root.OutputXML(false))
}
}
} // until all dependent data is fetched

if root != nil && Tracing {
dumpStr := c.yp.NodeDump(root)
TRACE_LOG(INFO_API, TRACE_CACHE, "Dependent Data = %v\n", dumpStr)
TRACE_LOG(TRACE_CACHE, "Dependent Data = %v\n", dumpStr)
}

TRACE_LOG(INFO_API, TRACE_CACHE, "\n%v, Exiting fetchToTmpCache", time.Now())
TRACE_LOG(TRACE_CACHE, "\n%v, Exiting fetchToTmpCache", time.Now())
return root
}

Expand All @@ -230,5 +281,3 @@ func (c *CVL) clearTmpDbCache() {
delete(c.tmpDbCache, key)
}
}


Loading

0 comments on commit 4aa0899

Please sign in to comment.