Skip to content

Commit

Permalink
Support golden config in db migrator (#3076)
Browse files Browse the repository at this point in the history
What I did
Need to support golden config in db migrator.

How I did it
If there's golden config json, read from golden config instead of minigraph.
And db migrator will use golden config data to generate new configuration.

How to verify it
Run unit test.
  • Loading branch information
ganglyu authored Dec 22, 2023
1 parent 20d1495 commit bcb10f1
Show file tree
Hide file tree
Showing 6 changed files with 161 additions and 34 deletions.
11 changes: 2 additions & 9 deletions config/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@
from utilities_common.intf_filter import parse_interface_in_filter
from utilities_common import bgp_util
import utilities_common.cli as clicommon
from utilities_common.helper import get_port_pbh_binding, get_port_acl_binding
from utilities_common.helper import get_port_pbh_binding, get_port_acl_binding, update_config
from utilities_common.general import load_db_config, load_module_from_source
from .validated_config_db_connector import ValidatedConfigDBConnector
import utilities_common.multi_asic as multi_asic_util
Expand Down Expand Up @@ -1950,6 +1950,7 @@ def override_config_table(db, input_config_db, dry_run):
ns_config_input = config_input
# Generate sysinfo if missing in ns_config_input
generate_sysinfo(current_config, ns_config_input, ns)
# Use deepcopy by default to avoid modifying input config
updated_config = update_config(current_config, ns_config_input)

yang_enabled = device_info.is_yang_config_validation_enabled(config_db)
Expand Down Expand Up @@ -1985,14 +1986,6 @@ def validate_config_by_cm(cm, config_json, jname):
sys.exit(1)


def update_config(current_config, config_input):
updated_config = copy.deepcopy(current_config)
# Override current config with golden config
for table in config_input:
updated_config[table] = config_input[table]
return updated_config


def override_config_db(config_db, config_input):
# Deserialized golden config to DB recognized format
sonic_cfggen.FormatConverter.to_deserialized(config_input)
Expand Down
88 changes: 67 additions & 21 deletions scripts/db_migrator.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,11 @@
from sonic_py_common import device_info, logger
from swsscommon.swsscommon import SonicV2Connector, ConfigDBConnector, SonicDBConfig
from minigraph import parse_xml
from utilities_common.helper import update_config

INIT_CFG_FILE = '/etc/sonic/init_cfg.json'
MINIGRAPH_FILE = '/etc/sonic/minigraph.xml'
GOLDEN_CFG_FILE = '/etc/sonic/golden_config_db.json'

# mock the redis for unit test purposes #
try:
Expand All @@ -24,10 +26,12 @@
sys.path.insert(0, tests_path)
INIT_CFG_FILE = os.path.join(mocked_db_path, "init_cfg.json")
MINIGRAPH_FILE = os.path.join(mocked_db_path, "minigraph.xml")
GOLDEN_CFG_FILE = os.path.join(mocked_db_path, "golden_config_db.json")
except KeyError:
pass

SYSLOG_IDENTIFIER = 'db_migrator'
DEFAULT_NAMESPACE = ''


# Global logger instance
Expand Down Expand Up @@ -60,15 +64,8 @@ def __init__(self, namespace, socket=None):
self.TABLE_KEY = 'DATABASE'
self.TABLE_FIELD = 'VERSION'

# load config data from minigraph to get the default/hardcoded values from minigraph.py
# this is to avoid duplicating the hardcoded these values in db_migrator
self.minigraph_data = None
try:
if os.path.isfile(MINIGRAPH_FILE):
self.minigraph_data = parse_xml(MINIGRAPH_FILE)
except Exception as e:
log.log_error('Caught exception while trying to parse minigraph: ' + str(e))
pass
# Generate config_src_data from minigraph and golden config
self.generate_config_src(namespace)

db_kwargs = {}
if socket:
Expand Down Expand Up @@ -107,6 +104,55 @@ def __init__(self, namespace, socket=None):
from mellanox_buffer_migrator import MellanoxBufferMigrator
self.mellanox_buffer_migrator = MellanoxBufferMigrator(self.configDB, self.appDB, self.stateDB)

def generate_config_src(self, ns):
'''
Generate config_src_data from minigraph and golden config
This method uses golden_config_data and minigraph_data as local variables,
which means they are not accessible or modifiable from outside this method.
This way, this method ensures that these variables are not changed unintentionally.
Args:
ns: namespace
Returns:
'''
# load config data from golden_config_db.json
golden_config_data = None
try:
if os.path.isfile(GOLDEN_CFG_FILE):
with open(GOLDEN_CFG_FILE) as f:
golden_data = json.load(f)
if ns is None:
golden_config_data = golden_data
else:
if ns == DEFAULT_NAMESPACE:
config_namespace = "localhost"
else:
config_namespace = ns
golden_config_data = golden_data[config_namespace]
except Exception as e:
log.log_error('Caught exception while trying to load golden config: ' + str(e))
pass
# load config data from minigraph to get the default/hardcoded values from minigraph.py
minigraph_data = None
try:
if os.path.isfile(MINIGRAPH_FILE):
minigraph_data = parse_xml(MINIGRAPH_FILE)
except Exception as e:
log.log_error('Caught exception while trying to parse minigraph: ' + str(e))
pass
# When both golden config and minigraph exists, override minigraph config with golden config
# config_src_data is the source of truth for config data
# this is to avoid duplicating the hardcoded these values in db_migrator
self.config_src_data = None
if minigraph_data:
# Shallow copy for better performance
self.config_src_data = minigraph_data
if golden_config_data:
# Shallow copy for better performance
self.config_src_data = update_config(minigraph_data, golden_config_data, False)
elif golden_config_data:
# Shallow copy for better performance
self.config_src_data = golden_config_data

def migrate_pfc_wd_table(self):
'''
Migrate all data entries from table PFC_WD_TABLE to PFC_WD
Expand Down Expand Up @@ -547,9 +593,9 @@ def migrate_vxlan_config(self):

def migrate_restapi(self):
# RESTAPI - add missing key
if not self.minigraph_data or 'RESTAPI' not in self.minigraph_data:
if not self.config_src_data or 'RESTAPI' not in self.config_src_data:
return
restapi_data = self.minigraph_data['RESTAPI']
restapi_data = self.config_src_data['RESTAPI']
log.log_notice('Migrate RESTAPI configuration')
config = self.configDB.get_entry('RESTAPI', 'config')
if not config:
Expand All @@ -560,9 +606,9 @@ def migrate_restapi(self):

def migrate_telemetry(self):
# TELEMETRY - add missing key
if not self.minigraph_data or 'TELEMETRY' not in self.minigraph_data:
if not self.config_src_data or 'TELEMETRY' not in self.config_src_data:
return
telemetry_data = self.minigraph_data['TELEMETRY']
telemetry_data = self.config_src_data['TELEMETRY']
log.log_notice('Migrate TELEMETRY configuration')
gnmi = self.configDB.get_entry('TELEMETRY', 'gnmi')
if not gnmi:
Expand All @@ -573,9 +619,9 @@ def migrate_telemetry(self):

def migrate_console_switch(self):
# CONSOLE_SWITCH - add missing key
if not self.minigraph_data or 'CONSOLE_SWITCH' not in self.minigraph_data:
if not self.config_src_data or 'CONSOLE_SWITCH' not in self.config_src_data:
return
console_switch_data = self.minigraph_data['CONSOLE_SWITCH']
console_switch_data = self.config_src_data['CONSOLE_SWITCH']
log.log_notice('Migrate CONSOLE_SWITCH configuration')
console_mgmt = self.configDB.get_entry('CONSOLE_SWITCH', 'console_mgmt')
if not console_mgmt:
Expand All @@ -584,11 +630,11 @@ def migrate_console_switch(self):

def migrate_device_metadata(self):
# DEVICE_METADATA - synchronous_mode entry
if not self.minigraph_data or 'DEVICE_METADATA' not in self.minigraph_data:
if not self.config_src_data or 'DEVICE_METADATA' not in self.config_src_data:
return
log.log_notice('Migrate DEVICE_METADATA missing configuration')
metadata = self.configDB.get_entry('DEVICE_METADATA', 'localhost')
device_metadata_data = self.minigraph_data["DEVICE_METADATA"]["localhost"]
device_metadata_data = self.config_src_data["DEVICE_METADATA"]["localhost"]
if 'synchronous_mode' not in metadata:
metadata['synchronous_mode'] = device_metadata_data.get("synchronous_mode")
self.configDB.set_entry('DEVICE_METADATA', 'localhost', metadata)
Expand Down Expand Up @@ -628,19 +674,19 @@ def migrate_dns_nameserver(self):
Handle DNS_NAMESERVER table migration. Migrations handled:
If there's no DNS_NAMESERVER in config_DB, load DNS_NAMESERVER from minigraph
"""
if not self.minigraph_data or 'DNS_NAMESERVER' not in self.minigraph_data:
if not self.config_src_data or 'DNS_NAMESERVER' not in self.config_src_data:
return
dns_table = self.configDB.get_table('DNS_NAMESERVER')
if not dns_table:
for addr, config in self.minigraph_data['DNS_NAMESERVER'].items():
for addr, config in self.config_src_data['DNS_NAMESERVER'].items():
self.configDB.set_entry('DNS_NAMESERVER', addr, config)

def migrate_routing_config_mode(self):
# DEVICE_METADATA - synchronous_mode entry
if not self.minigraph_data or 'DEVICE_METADATA' not in self.minigraph_data:
if not self.config_src_data or 'DEVICE_METADATA' not in self.config_src_data:
return
device_metadata_old = self.configDB.get_entry('DEVICE_METADATA', 'localhost')
device_metadata_new = self.minigraph_data['DEVICE_METADATA']['localhost']
device_metadata_new = self.config_src_data['DEVICE_METADATA']['localhost']
# overwrite the routing-config-mode as per minigraph parser
# Criteria for update:
# if config mode is missing in base OS or if base and target modes are not same
Expand Down
7 changes: 7 additions & 0 deletions tests/db_migrator_input/golden_config_db.json.invalid
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
{
"DEVICE_METADATA": {
"localhost": {
"hostname": "SONiC-Golden-Config"
}}
}
}
7 changes: 7 additions & 0 deletions tests/db_migrator_input/golden_config_db.json.test
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
{
"DEVICE_METADATA": {
"localhost": {
"hostname": "SONiC-Golden-Config"
}
}
}
54 changes: 50 additions & 4 deletions tests/db_migrator_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -368,8 +368,8 @@ def test_dns_nameserver_migrator(self):
dbconnector.dedicated_dbs['CONFIG_DB'] = os.path.join(mock_db_path, 'config_db', 'dns-nameserver-input')
import db_migrator
dbmgtr = db_migrator.DBMigrator(None)
# Set minigraph_data to DNS_NAMESERVERS
dbmgtr.minigraph_data = {
# Set config_src_data to DNS_NAMESERVERS
dbmgtr.config_src_data = {
'DNS_NAMESERVER': {
'1.1.1.1': {},
'2001:1001:110:1001::1': {}
Expand Down Expand Up @@ -635,8 +635,8 @@ def test_warm_upgrade__without_mg_to_2_0_2(self):
dbconnector.dedicated_dbs['CONFIG_DB'] = os.path.join(mock_db_path, 'config_db', 'cross_branch_upgrade_to_version_2_0_2_input')
import db_migrator
dbmgtr = db_migrator.DBMigrator(None)
# set minigraph_data to None to mimic the missing minigraph.xml scenario
dbmgtr.minigraph_data = None
# set config_src_data to None to mimic the missing minigraph.xml scenario
dbmgtr.config_src_data = None
dbmgtr.migrate()
dbconnector.dedicated_dbs['CONFIG_DB'] = os.path.join(mock_db_path, 'config_db', 'cross_branch_upgrade_without_mg_2_0_2_expected.json')
expected_db = Db()
Expand Down Expand Up @@ -823,3 +823,49 @@ def test_sflow_migrator(self):
expected_keys = expected_appl_db.get_all(expected_appl_db.APPL_DB, key)
diff = DeepDiff(resulting_keys, expected_keys, ignore_order=True)
assert not diff

class TestGoldenConfig(object):
@classmethod
def setup_class(cls):
os.system("cp %s %s" % (mock_db_path + '/golden_config_db.json.test', mock_db_path + '/golden_config_db.json'))
os.environ['UTILITIES_UNIT_TESTING'] = "2"

@classmethod
def teardown_class(cls):
os.environ['UTILITIES_UNIT_TESTING'] = "0"
os.system("rm %s" % (mock_db_path + '/golden_config_db.json'))

def test_golden_config_hostname(self):
import db_migrator
dbmgtr = db_migrator.DBMigrator(None)
config = dbmgtr.config_src_data
device_metadata = config.get('DEVICE_METADATA', {})
assert device_metadata != {}
host = device_metadata.get('localhost', {})
assert host != {}
hostname = host.get('hostname', '')
# hostname is from golden_config_db.json
assert hostname == 'SONiC-Golden-Config'

class TestGoldenConfigInvalid(object):
@classmethod
def setup_class(cls):
os.system("cp %s %s" % (mock_db_path + '/golden_config_db.json.invalid', mock_db_path + '/golden_config_db.json'))
os.environ['UTILITIES_UNIT_TESTING'] = "2"

@classmethod
def teardown_class(cls):
os.environ['UTILITIES_UNIT_TESTING'] = "0"
os.system("rm %s" % (mock_db_path + '/golden_config_db.json'))

def test_golden_config_hostname(self):
import db_migrator
dbmgtr = db_migrator.DBMigrator(None)
config = dbmgtr.config_src_data
device_metadata = config.get('DEVICE_METADATA', {})
assert device_metadata != {}
host = device_metadata.get('localhost', {})
assert host != {}
hostname = host.get('hostname', '')
# hostname is from minigraph.xml
assert hostname == 'SONiC-Dummy'
28 changes: 28 additions & 0 deletions utilities_common/helper.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
from dump.match_infra import MatchEngine, MatchRequest, ConnectionPool
from dump.match_helper import get_matched_keys
from .db import Db
import copy

def get_port_acl_binding(db_wrap, port, ns):
"""
Expand Down Expand Up @@ -64,3 +65,30 @@ def get_port_pbh_binding(db_wrap, port, ns):
ret = m_engine.fetch(req)
pbh_tables, _ = get_matched_keys(ret)
return pbh_tables


def update_config(current_config, config_input, deepcopy=True):
"""
Override current config with golden config
Shallow copy only copies the references to the original object,
so any changes to one object will also change the other.
Therefore, we should be careful when using shallow copy to avoid unwanted modifications.
Args:
current_config: current config
config_input: input golden config
deepcopy: True for deep copy, False for shallow copy
Returns:
Final config after overriding
"""
if deepcopy:
# Deep copy for safety
updated_config = copy.deepcopy(current_config)
else:
# Shallow copy for better performance
updated_config = current_config
# Override current config with golden config
for table in config_input:
updated_config[table] = config_input[table]
return updated_config

0 comments on commit bcb10f1

Please sign in to comment.