Skip to content

Commit

Permalink
[debug dump util] Base Skeleton and Click Class added (#1668)
Browse files Browse the repository at this point in the history
What I did
HLD for Dump Utility: HLD.

Added the top level CLI command i.e "dump state"
Added corresponding UT's
Added the bash autocompletion support files
Added the implementation for the customization options provided, such as --db, --key-map, --show, --table & --namespace
How I did it
How to verify it
UT's are implemented
  • Loading branch information
vivekrnv authored Sep 7, 2021
1 parent 171eb4f commit 9f2326e
Show file tree
Hide file tree
Showing 6 changed files with 561 additions and 2 deletions.
55 changes: 55 additions & 0 deletions doc/Command-Reference.md
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,7 @@
* [SONiC Package Manager](#sonic-package-manager)
* [SONiC Installer](#sonic-installer)
* [Troubleshooting Commands](#troubleshooting-commands)
* [Debug Dumps](#debug-dumps)
* [Routing Stack](#routing-stack)
* [Quagga BGP Show Commands](#Quagga-BGP-Show-Commands)
* [ZTP Configuration And Show Commands](#ztp-configuration-and-show-commands)
Expand Down Expand Up @@ -9712,6 +9713,60 @@ If the SONiC system was running for quite some time `show techsupport` will prod
admin@sonic:~$ show techsupport --since='hour ago' # Will collect syslog and core files for the last one hour
```
### Debug Dumps
In SONiC, there usually exists a set of tables related/relevant to a particular module. All of these might have to be looked at to confirm whether any configuration update is properly applied and propagated. This utility comes in handy because it prints a unified view of the redis-state for a given module
- Usage:
```
Usage: dump state [OPTIONS] MODULE IDENTIFIER
Dump the redis-state of the identifier for the module specified
Options:
-s, --show Display Modules Available
-d, --db TEXT Only dump from these Databases
-t, --table Print in tabular format [default: False]
-k, --key-map Only fetch the keys matched, don't extract field-value dumps [default: False]
-v, --verbose Prints any intermediate output to stdout useful for dev & troubleshooting [default: False]
-n, --namespace TEXT Dump the redis-state for this namespace. [default: DEFAULT_NAMESPACE]
--help Show this message and exit.
```
- Examples:
```
root@sonic# dump state --show
Module Identifier
-------- ------------
port port_name
copp trap_id
```
```
admin@sonic:~$ dump state copp arp_req --key-map --db ASIC_DB
{
"arp_req": {
"ASIC_DB": {
"keys": [
"ASIC_STATE:SAI_OBJECT_TYPE_HOSTIF_TRAP:oid:0x22000000000c5b",
"ASIC_STATE:SAI_OBJECT_TYPE_HOSTIF_TRAP_GROUP:oid:0x11000000000c59",
"ASIC_STATE:SAI_OBJECT_TYPE_POLICER:oid:0x12000000000c5a",
"ASIC_STATE:SAI_OBJECT_TYPE_QUEUE:oid:0x15000000000626"
],
"tables_not_found": [],
"vidtorid": {
"oid:0x22000000000c5b": "oid:0x200000000022",
"oid:0x11000000000c59": "oid:0x300000011",
"oid:0x12000000000c5a": "oid:0x200000012",
"oid:0x15000000000626": "oid:0x12e0000040015"
}
}
}
}
```
Go Back To [Beginning of the document](#) or [Beginning of this section](#troubleshooting-commands)
## Routing Stack
Expand Down
229 changes: 229 additions & 0 deletions dump/main.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,229 @@
import os
import sys
import json
import re
import click
from tabulate import tabulate
from sonic_py_common import multi_asic
from utilities_common.constants import DEFAULT_NAMESPACE
from dump.match_infra import RedisSource, JsonSource, ConnectionPool
from dump import plugins


# Autocompletion Helper
def get_available_modules(ctx, args, incomplete):
return [k for k in plugins.dump_modules.keys() if incomplete in k]


# Display Modules Callback
def show_modules(ctx, param, value):
if not value or ctx.resilient_parsing:
return
header = ["Module", "Identifier"]
display = []
for mod in plugins.dump_modules:
display.append((mod, plugins.dump_modules[mod].ARG_NAME))
click.echo(tabulate(display, header))
ctx.exit()


@click.group()
def dump():
pass


@dump.command()
@click.pass_context
@click.argument('module', required=True, type=str, autocompletion=get_available_modules)
@click.argument('identifier', required=True, type=str)
@click.option('--show', '-s', is_flag=True, default=False, expose_value=False,
callback=show_modules, help='Display Modules Available', is_eager=True)
@click.option('--db', '-d', multiple=True,
help='Only dump from these Databases or the CONFIG_FILE')
@click.option('--table', '-t', is_flag=True, default=False,
help='Print in tabular format', show_default=True)
@click.option('--key-map', '-k', is_flag=True, default=False, show_default=True,
help="Only fetch the keys matched, don't extract field-value dumps")
@click.option('--verbose', '-v', is_flag=True, default=False, show_default=True,
help="Prints any intermediate output to stdout useful for dev & troubleshooting")
@click.option('--namespace', '-n', default=DEFAULT_NAMESPACE, type=str,
show_default=True, help='Dump the redis-state for this namespace.')
def state(ctx, module, identifier, db, table, key_map, verbose, namespace):
"""
Dump the current state of the identifier for the specified module from Redis DB or CONFIG_FILE
"""
if not multi_asic.is_multi_asic() and namespace != DEFAULT_NAMESPACE:
click.echo("Namespace option is not valid for a single-ASIC device")
ctx.exit()

if multi_asic.is_multi_asic() and (namespace != DEFAULT_NAMESPACE and namespace not in multi_asic.get_namespace_list()):
click.echo("Namespace option is not valid. Choose one of {}".format(multi_asic.get_namespace_list()))
ctx.exit()

if module not in plugins.dump_modules:
click.echo("No Matching Plugin has been Implemented")
ctx.exit()

if verbose:
os.environ["VERBOSE"] = "1"
else:
os.environ["VERBOSE"] = "0"

ctx.module = module
obj = plugins.dump_modules[module]()

if identifier == "all":
ids = obj.get_all_args(namespace)
else:
ids = identifier.split(",")

params = {}
collected_info = {}
params['namespace'] = namespace
for arg in ids:
params[plugins.dump_modules[module].ARG_NAME] = arg
collected_info[arg] = obj.execute(params)

if len(db) > 0:
collected_info = filter_out_dbs(db, collected_info)

vidtorid = extract_rid(collected_info, namespace)

if not key_map:
collected_info = populate_fv(collected_info, module, namespace)

for id in vidtorid.keys():
collected_info[id]["ASIC_DB"]["vidtorid"] = vidtorid[id]

print_dump(collected_info, table, module, identifier, key_map)

return


def extract_rid(info, ns):
r = RedisSource(ConnectionPool())
r.connect("ASIC_DB", ns)
vidtorid = {}
vid_cache = {} # Cache Entries to reduce number of Redis Calls
for arg in info.keys():
mp = get_v_r_map(r, info[arg], vid_cache)
if mp:
vidtorid[arg] = mp
return vidtorid


def get_v_r_map(r, single_dict, vid_cache):
v_r_map = {}
asic_obj_ptrn = "ASIC_STATE:.*:oid:0x\w{1,14}"

if "ASIC_DB" in single_dict and "keys" in single_dict["ASIC_DB"]:
for redis_key in single_dict["ASIC_DB"]["keys"]:
if re.match(asic_obj_ptrn, redis_key):
matches = re.findall(r"oid:0x\w{1,14}", redis_key)
if matches:
vid = matches[0]
if vid in vid_cache:
rid = vid_cache[vid]
else:
rid = r.hget("ASIC_DB", "VIDTORID", vid)
vid_cache[vid] = rid
v_r_map[vid] = rid if rid else "Real ID Not Found"
return v_r_map


# Filter dbs which are not required
def filter_out_dbs(db_list, collected_info):
args_ = list(collected_info.keys())
for arg in args_:
dbs = list(collected_info[arg].keys())
for db in dbs:
if db not in db_list:
del collected_info[arg][db]
return collected_info


def populate_fv(info, module, namespace):
all_dbs = set()
for id in info.keys():
for db_name in info[id].keys():
all_dbs.add(db_name)

db_cfg_file = JsonSource()
db_conn = ConnectionPool().initialize_connector(namespace)
for db_name in all_dbs:
if db_name is "CONFIG_FILE":
db_cfg_file.connect(plugins.dump_modules[module].CONFIG_FILE, namespace)
else:
db_conn.connect(db_name)

final_info = {}
for id in info.keys():
final_info[id] = {}
for db_name in info[id].keys():
final_info[id][db_name] = {}
final_info[id][db_name]["keys"] = []
final_info[id][db_name]["tables_not_found"] = info[id][db_name]["tables_not_found"]
for key in info[id][db_name]["keys"]:
if db_name is "CONFIG_FILE":
fv = db_dict[db_name].get(db_name, key)
else:
fv = db_conn.get_all(db_name, key)
final_info[id][db_name]["keys"].append({key: fv})

return final_info


def get_dict_str(key_obj):
table = []
for pair in key_obj.items():
table.append(list(pair))
return tabulate(table, headers=["field", "value"], tablefmt="psql")


# print dump
def print_dump(collected_info, table, module, identifier, key_map):
if not table:
click.echo(json.dumps(collected_info, indent=4))
return

top_header = [plugins.dump_modules[module].ARG_NAME, "DB_NAME", "DUMP"]
final_collection = []
for ids in collected_info.keys():
for db in collected_info[ids].keys():
total_info = ""

if collected_info[ids][db]["tables_not_found"]:
tabulate_fmt = []
for tab in collected_info[ids][db]["tables_not_found"]:
tabulate_fmt.append([tab])
total_info += tabulate(tabulate_fmt, ["Tables Not Found"], tablefmt="grid")
total_info += "\n"

if not key_map:
values = []
hdrs = ["Keys", "field-value pairs"]
for key_obj in collected_info[ids][db]["keys"]:
if isinstance(key_obj, dict) and key_obj:
key = list(key_obj.keys())[0]
values.append([key, get_dict_str(key_obj[key])])
total_info += str(tabulate(values, hdrs, tablefmt="grid"))
else:
temp = []
for key_ in collected_info[ids][db]["keys"]:
temp.append([key_])
total_info += str(tabulate(temp, headers=["Keys Collected"], tablefmt="grid"))

total_info += "\n"
if "vidtorid" in collected_info[ids][db]:
temp = []
for pair in collected_info[ids][db]["vidtorid"].items():
temp.append(list(pair))
total_info += str(tabulate(temp, headers=["vid", "rid"], tablefmt="grid"))
final_collection.append([ids, db, total_info])

click.echo(tabulate(final_collection, top_header, tablefmt="grid"))
return


if __name__ == '__main__':
dump()
6 changes: 5 additions & 1 deletion setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@
'crm',
'debug',
'generic_config_updater',
'dump',
'dump.plugins',
'pfcwd',
'sfputil',
'ssdutil',
Expand Down Expand Up @@ -71,7 +73,8 @@
'filter_fdb_input/*',
'pfcwd_input/*',
'wm_input/*',
'ecn_input/*']
'ecn_input/*',
'dump_input/*']
},
scripts=[
'scripts/aclshow',
Expand Down Expand Up @@ -143,6 +146,7 @@
'counterpoll = counterpoll.main:cli',
'crm = crm.main:cli',
'debug = debug.main:cli',
'dump = dump.main:dump',
'filter_fdb_entries = fdbutil.filter_fdb_entries:main',
'pfcwd = pfcwd.main:cli',
'sfputil = sfputil.main:cli',
Expand Down
8 changes: 8 additions & 0 deletions sonic-utilities-data/bash_completion.d/dump
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
_dump_completion() {
COMPREPLY=( $( env COMP_WORDS="${COMP_WORDS[*]}" \
COMP_CWORD=$COMP_CWORD \
_DUMP_COMPLETE=complete $1 ) )
return 0
}

complete -F _dump_completion -o default dump
Loading

0 comments on commit 9f2326e

Please sign in to comment.