Skip to content

Commit

Permalink
[Yang][cfggen] update sonic-cfggen to generate config_db from Yang da…
Browse files Browse the repository at this point in the history
…ta (#7712)

Why I did it
This PR adds changes in sonic-config-engine to consume configuration data in SONiC Yang schema and generate config_db entries

How I did it
Add a new file sonic_yang_cfg_generator .
This file has the functions to

parse yang data json and convert them in config_db json format.
Validate the converted config_db entries to make sure all the dependencies and constraints are met.
Add a new option -Y to the sonic-cfggen command for this purpose

Add unit tests

This capability is support only in sonic-config-engine Python3 package only
  • Loading branch information
arlakshm authored Jun 10, 2021
1 parent 714894c commit 4d07bbb
Show file tree
Hide file tree
Showing 11 changed files with 800 additions and 38 deletions.
31 changes: 16 additions & 15 deletions files/build_templates/sonic_debian_extension.j2
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,22 @@ if [[ $CONFIGURED_ARCH == armhf || $CONFIGURED_ARCH == arm64 ]]; then
sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install libxslt-dev libz-dev
fi

# Install sonic-yang-models Python 3 package, install dependencies
sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/libyang_*.deb
sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/libyang-cpp_*.deb
sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/python2-yang_*.deb
sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/python3-yang_*.deb
SONIC_YANG_MODEL_PY3_WHEEL_NAME=$(basename {{sonic_yang_models_py3_wheel_path}})
sudo cp {{sonic_yang_models_py3_wheel_path}} $FILESYSTEM_ROOT/$SONIC_YANG_MODEL_PY3_WHEEL_NAME
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install $SONIC_YANG_MODEL_PY3_WHEEL_NAME
sudo rm -rf $FILESYSTEM_ROOT/$SONIC_YANG_MODEL_PY3_WHEEL_NAME

# Install sonic-yang-mgmt Python3 package
SONIC_YANG_MGMT_PY3_WHEEL_NAME=$(basename {{sonic_yang_mgmt_py3_wheel_path}})
sudo cp {{sonic_yang_mgmt_py3_wheel_path}} $FILESYSTEM_ROOT/$SONIC_YANG_MGMT_PY3_WHEEL_NAME
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install $SONIC_YANG_MGMT_PY3_WHEEL_NAME
sudo rm -rf $FILESYSTEM_ROOT/$SONIC_YANG_MGMT_PY3_WHEEL_NAME

# Install SONiC config engine Python 2 package
CONFIG_ENGINE_PY2_WHEEL_NAME=$(basename {{config_engine_py2_wheel_path}})
sudo cp {{config_engine_py2_wheel_path}} $FILESYSTEM_ROOT/$CONFIG_ENGINE_PY2_WHEEL_NAME
Expand All @@ -184,21 +200,6 @@ sudo cp {{config_engine_py3_wheel_path}} $FILESYSTEM_ROOT/$CONFIG_ENGINE_PY3_WHE
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install $CONFIG_ENGINE_PY3_WHEEL_NAME
sudo rm -rf $FILESYSTEM_ROOT/$CONFIG_ENGINE_PY3_WHEEL_NAME

# Install sonic-yang-models py3 package, install dependencies
sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/libyang_*.deb
sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/libyang-cpp_*.deb
sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/python2-yang_*.deb
sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/python3-yang_*.deb
SONIC_YANG_MODEL_PY3_WHEEL_NAME=$(basename {{sonic_yang_models_py3_wheel_path}})
sudo cp {{sonic_yang_models_py3_wheel_path}} $FILESYSTEM_ROOT/$SONIC_YANG_MODEL_PY3_WHEEL_NAME
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install $SONIC_YANG_MODEL_PY3_WHEEL_NAME
sudo rm -rf $FILESYSTEM_ROOT/$SONIC_YANG_MODEL_PY3_WHEEL_NAME

# Install sonic-yang-mgmt Python3 package
SONIC_YANG_MGMT_PY3_WHEEL_NAME=$(basename {{sonic_yang_mgmt_py3_wheel_path}})
sudo cp {{sonic_yang_mgmt_py3_wheel_path}} $FILESYSTEM_ROOT/$SONIC_YANG_MGMT_PY3_WHEEL_NAME
sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install $SONIC_YANG_MGMT_PY3_WHEEL_NAME
sudo rm -rf $FILESYSTEM_ROOT/$SONIC_YANG_MGMT_PY3_WHEEL_NAME

# Install sonic-platform-common Python 2 package
PLATFORM_COMMON_PY2_WHEEL_NAME=$(basename {{platform_common_py2_wheel_path}})
Expand Down
10 changes: 8 additions & 2 deletions rules/docker-config-engine-buster.mk
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,15 @@
DOCKER_CONFIG_ENGINE_BUSTER = docker-config-engine-buster.gz
$(DOCKER_CONFIG_ENGINE_BUSTER)_PATH = $(DOCKERS_PATH)/docker-config-engine-buster

$(DOCKER_CONFIG_ENGINE_BUSTER)_DEPENDS += $(LIBSWSSCOMMON) $(PYTHON3_SWSSCOMMON)
$(DOCKER_CONFIG_ENGINE_BUSTER)_DEPENDS += $(LIBSWSSCOMMON) \
$(LIBYANG) \
$(LIBYANG_CPP) \
$(LIBYANG_PY3) \
$(PYTHON3_SWSSCOMMON)
$(DOCKER_CONFIG_ENGINE_BUSTER)_PYTHON_WHEELS += $(SWSSSDK_PY3)
$(DOCKER_CONFIG_ENGINE_BUSTER)_PYTHON_WHEELS += $(SONIC_PY_COMMON_PY3)
$(DOCKER_CONFIG_ENGINE_BUSTER)_PYTHON_WHEELS += $(SONIC_PY_COMMON_PY3) \
$(SONIC_YANG_MGMT_PY3) \
$(SONIC_YANG_MODELS_PY3)
$(DOCKER_CONFIG_ENGINE_BUSTER)_PYTHON_WHEELS += $(SONIC_CONFIG_ENGINE_PY3)
$(DOCKER_CONFIG_ENGINE_BUSTER)_LOAD_DOCKERS += $(DOCKER_BASE_BUSTER)
$(DOCKER_CONFIG_ENGINE_BUSTER)_FILES += $(SWSS_VARS_TEMPLATE)
Expand Down
9 changes: 7 additions & 2 deletions rules/sonic-config.mk
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,13 @@ SONIC_PYTHON_WHEELS += $(SONIC_CONFIG_ENGINE_PY2)

SONIC_CONFIG_ENGINE_PY3 = sonic_config_engine-1.0-py3-none-any.whl
$(SONIC_CONFIG_ENGINE_PY3)_SRC_PATH = $(SRC_PATH)/sonic-config-engine
$(SONIC_CONFIG_ENGINE_PY3)_DEPENDS += $(SONIC_PY_COMMON_PY3)
$(SONIC_CONFIG_ENGINE_PY3)_DEBS_DEPENDS += $(PYTHON3_SWSSCOMMON)
$(SONIC_CONFIG_ENGINE_PY3)_DEPENDS += $(SONIC_PY_COMMON_PY3) \
$(SONIC_YANG_MGMT_PY3) \
$(SONIC_YANG_MODELS_PY3)
$(SONIC_CONFIG_ENGINE_PY3)_DEBS_DEPENDS += $(LIBYANG) \
$(LIBYANG_CPP) \
$(LIBYANG_PY3) \
$(PYTHON3_SWSSCOMMON)
# Synthetic dependency to avoid building the Python 2 and 3 packages
# simultaneously and any potential conflicts which may arise
$(SONIC_CONFIG_ENGINE_PY3)_DEPENDS += $(SONIC_CONFIG_ENGINE_PY2)
Expand Down
9 changes: 7 additions & 2 deletions rules/sonic_bgpcfgd.mk
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,12 @@ $(SONIC_BGPCFGD)_SRC_PATH = $(SRC_PATH)/sonic-bgpcfgd
# as part of its unit tests.
# TODO: Refactor unit tests so that these dependencies are not needed

$(SONIC_BGPCFGD)_DEPENDS += $(SONIC_CONFIG_ENGINE_PY3)
$(SONIC_BGPCFGD)_DEBS_DEPENDS += $(PYTHON3_SWSSCOMMON)
$(SONIC_BGPCFGD)_DEPENDS += $(SONIC_CONFIG_ENGINE_PY3) \
$(SONIC_YANG_MGMT_PY3) \
$(SONIC_YANG_MODELS_PY3)
$(SONIC_BGPCFGD)_DEBS_DEPENDS += $(LIBYANG) \
$(LIBYANG_CPP) \
$(LIBYANG_PY3) \
$(PYTHON3_SWSSCOMMON)
$(SONIC_BGPCFGD)_PYTHON_VERSION = 3
SONIC_PYTHON_WHEELS += $(SONIC_BGPCFGD)
27 changes: 18 additions & 9 deletions src/sonic-config-engine/setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,9 @@
# Python3 has enum module and so pyangbind should be installed outside
# dependencies section of setuptools followed by uninstall of enum43
# 'pyangbind==0.8.1',
'Jinja2>=2.10'
'Jinja2>=2.10',
'sonic-yang-mgmt>=1.0',
'sonic-yang-models>=1.0'
]
else:
# Python 2-only dependencies
Expand All @@ -34,6 +36,20 @@
'importlib-resources==3.3.1' # importlib-resources v4.0.0 was released 2020-12-23 and drops support for Python 2
]

# Common modules for python2 and python3
py_modules = [
'config_samples',
'lazy_re',
'minigraph',
'openconfig_acl',
'portconfig',
'redis_bcc',
]
if sys.version_info.major == 3:
# Python 3-only modules
py_modules += [
'sonic_yang_cfg_generator'
]

setup(
name = 'sonic-config-engine',
Expand All @@ -42,14 +58,7 @@
author = 'Taoyu Li',
author_email = '[email protected]',
url = 'https://github.com/Azure/sonic-buildimage',
py_modules = [
'config_samples',
'lazy_re',
'minigraph',
'openconfig_acl',
'portconfig',
'redis_bcc',
],
py_modules = py_modules,
scripts = [
'sonic-cfggen',
],
Expand Down
16 changes: 15 additions & 1 deletion src/sonic-config-engine/sonic-cfggen
Original file line number Diff line number Diff line change
Expand Up @@ -48,9 +48,11 @@ from swsscommon.swsscommon import SonicV2Connector, ConfigDBConnector, SonicDBCo

PY3x = sys.version_info >= (3, 0)

#TODO: Remove STR_TYPE, FILE_TYPE once SONiC moves to Python 3.x
# TODO: Remove STR_TYPE, FILE_TYPE once SONiC moves to Python 3.x
# TODO: Remove the import SonicYangCfgDbGenerator once SONiC moves to python3.x
if PY3x:
from io import IOBase
from sonic_yang_cfg_generator import SonicYangCfgDbGenerator
STR_TYPE = str
FILE_TYPE = IOBase
else:
Expand Down Expand Up @@ -258,6 +260,7 @@ def main():
parser=argparse.ArgumentParser(description="Render configuration file from minigraph data and jinja2 template.")
group = parser.add_mutually_exclusive_group()
group.add_argument("-m", "--minigraph", help="minigraph xml file", nargs='?', const='/etc/sonic/minigraph.xml')
group.add_argument("-Y", "--yang", help="yang data json file", nargs='?', const='/etc/sonic/config_yang.json')
group.add_argument("-M", "--device-description", help="device description xml file")
group.add_argument("-k", "--hwsku", help="HwSKU")
parser.add_argument("-n", "--namespace", help="namespace name", nargs='?', const=None, default=None)
Expand Down Expand Up @@ -325,6 +328,17 @@ def main():

_process_json(args, data)

if args.yang is not None:
#TODO: Remove this check onces SONiC moves to python3.x
if PY3x:
yang_file = args.yang
config_db_json = SonicYangCfgDbGenerator().generate_config(
yang_data_file=yang_file)
deep_update(data, config_db_json)
else:
print('-Y/--yang option is not available in Python2', file=sys.stderr)
sys.exit(1)

if args.minigraph is not None:
minigraph = args.minigraph
if platform:
Expand Down
60 changes: 60 additions & 0 deletions src/sonic-config-engine/sonic_yang_cfg_generator.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
"""sonic_yang_cfg_generator
version_added: "1.0"
author: Arvindsrinivasan Lakshmi naraismhan ([email protected])
short_description: Parse sonic_yang data and generate the config_db entries
"""

from __future__ import print_function

import json
import sys


import sonic_yang
# TODO: Remove this once we no longer support Python 2
if sys.version_info.major == 3:
UNICODE_TYPE = str
else:
UNICODE_TYPE = unicode

YANG_MODELS_DIR = "/usr/local/yang-models"
DEFAULT_YANG_DATA_FILE = "/etc/sonic/config_yang.json"


class SonicYangCfgDbGenerator:

def __init__(self, yang_models_dir=YANG_MODELS_DIR):
self.yang_models_dir = yang_models_dir
self.yang_parser = sonic_yang.SonicYang(self.yang_models_dir)
self.yang_parser.loadYangModel()

def get_config_db_from_yang_data(self,
yang_data_file=DEFAULT_YANG_DATA_FILE):
self.yang_data_file = yang_data_file
config_db_json = dict()
with open(self.yang_data_file, "r") as yang_file:
try:
self.yang_data = json.load(yang_file)
config_db_json = self.yang_parser.XlateYangToConfigDB(
yang_data=self.yang_data)
except json.JSONDecodeError as e:
print("Unable to parse Yang data file {} Error: {}".format(
yang_data_file, e))
return config_db_json

def validate_config_db_json(self, config_db_json):
self.yang_parser.loadData(configdbJson=config_db_json)
try:
self.yang_parser.validate_data_tree()
return True
except sonic_yang.SonicYangException as e:
print("yang data in {} is not valid".format(self.yang_data_file))
return False

def generate_config(self, yang_data_file=DEFAULT_YANG_DATA_FILE):
config_db_json = self.get_config_db_from_yang_data(
yang_data_file=yang_data_file)
if self.validate_config_db_json(config_db_json):
return config_db_json
else:
return {}
Loading

0 comments on commit 4d07bbb

Please sign in to comment.