Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Update storm cmdargs 'type' value to allow any model type name #1923

Merged
merged 18 commits into from
Oct 23, 2020
Merged

Update storm cmdargs 'type' value to allow any model type name #1923

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
ff5b675
add time argument type to storm argparser
Cisphyx Oct 20, 2020
eef9909
add ival
Cisphyx Oct 21, 2020
50088ea
Merge branch 'master' into storm-argparse-time
Cisphyx Oct 21, 2020
b399b19
adjust tests
Cisphyx Oct 21, 2020
d6706ea
Merge branch 'storm-argparse-time' of https://github.com/vertexprojec…
Cisphyx Oct 21, 2020
0e7ec03
Merge branch 'master' into storm-argparse-time
Cisphyx Oct 21, 2020
f4e8cb4
cmdarg type values must be a modeltype
Cisphyx Oct 22, 2020
ca31e57
Merge branch 'master' into storm-argparse-time
Cisphyx Oct 22, 2020
e9b3ef4
formatting
Cisphyx Oct 22, 2020
0f3a0f5
Merge branch 'storm-argparse-time' of https://github.com/vertexprojec…
Cisphyx Oct 22, 2020
f5b15f4
update tests
Cisphyx Oct 22, 2020
7e98605
Merge branch 'master' into storm-argparse-time
Cisphyx Oct 22, 2020
7db5d47
make parser also validate arg types
Cisphyx Oct 22, 2020
bd07132
Merge branch 'storm-argparse-time' of https://github.com/vertexprojec…
Cisphyx Oct 22, 2020
ac15ffc
adjust wording
Cisphyx Oct 22, 2020
0a7d2d8
Merge branch 'master' into storm-argparse-time
Cisphyx Oct 22, 2020
fd679b1
Merge branch 'master' into storm-argparse-time
invisig0th Oct 22, 2020
1a887c0
Merge branch 'master' into storm-argparse-time
invisig0th Oct 22, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 45 additions & 19 deletions synapse/lib/storm.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import synapse.exc as s_exc
import synapse.common as s_common
import synapse.telepath as s_telepath
import synapse.datamodel as s_datamodel

import synapse.lib.ast as s_ast
import synapse.lib.base as s_base
Expand Down Expand Up @@ -193,10 +194,30 @@
'type': 'string',
'pattern': s_grammar.re_scmd
},
'cmdargs': {
'type': ['array', 'null'],
'items': {'$ref': '#/definitions/cmdarg'},
},
'storm': {'type': 'string'}
},
'additionalProperties': True,
'required': ['name', 'storm']
},
'cmdarg': {
'type': 'array',
'items': [
{'type': 'string'},
{
'type': 'object',
'properties': {
'help': {'type': 'string'},
'type': {
'type': 'string',
'enum': list(s_datamodel.Model().types)
},
},
}
]
}
}
})
Expand Down Expand Up @@ -299,7 +320,7 @@
'action': 'store_true'}),
('--readonly', {'help': 'Should the layer be readonly.',
'action': 'store_true'}),
('--growsize', {'help': 'Amount to grow the map size when necessary.', 'type': int}),
('--growsize', {'help': 'Amount to grow the map size when necessary.', 'type': 'int'}),
('--upstream', {'help': 'One or more telepath urls to receive updates from.'}),
('--name', {'help': 'The name of the layer.'}),
),
Expand Down Expand Up @@ -1313,6 +1334,11 @@ def add_argument(self, *names, **opts):

assert len(names)

argtype = opts.get('type')
if argtype is not None and argtype not in s_datamodel.Model().types:
mesg = f'Argument type "{argtype}" is not a valid model type name'
raise s_exc.BadArg(mesg=mesg, argtype=argtype)

dest = self._get_dest(names)
opts.setdefault('dest', dest)
self.allargs.append((names, opts))
Expand Down Expand Up @@ -1448,9 +1474,9 @@ def _get_store(self, name, argdef, todo, opts):
valu = todo.popleft()
if argtype is not None:
try:
valu = argtype(valu)
valu = s_datamodel.Model().type(argtype).norm(valu)[0]
except Exception:
mesg = f'Invalid value for type ({str(argtype)}): {valu}'
mesg = f'Invalid value for type ({argtype}): {valu}'
return self.help(mesg=mesg)

opts[dest] = valu
Expand All @@ -1466,9 +1492,9 @@ def _get_store(self, name, argdef, todo, opts):
valu = todo.popleft()
if argtype is not None:
try:
valu = argtype(valu)
valu = s_datamodel.Model().type(argtype).norm(valu)[0]
except Exception:
mesg = f'Invalid value for type ({str(argtype)}): {valu}'
mesg = f'Invalid value for type ({argtype}): {valu}'
return self.help(mesg=mesg)

opts[dest] = valu
Expand All @@ -1483,9 +1509,9 @@ def _get_store(self, name, argdef, todo, opts):

if argtype is not None:
try:
valu = argtype(valu)
valu = s_datamodel.Model().type(argtype).norm(valu)[0]
except Exception:
mesg = f'Invalid value for type ({str(argtype)}): {valu}'
mesg = f'Invalid value for type ({argtype}): {valu}'
return self.help(mesg=mesg)

vals.append(valu)
Expand All @@ -1506,9 +1532,9 @@ def _get_store(self, name, argdef, todo, opts):
valu = todo.popleft()
if argtype is not None:
try:
valu = argtype(valu)
valu = s_datamodel.Model().type(argtype).norm(valu)[0]
except Exception:
mesg = f'Invalid value for type ({str(argtype)}): {valu}'
mesg = f'Invalid value for type ({argtype}): {valu}'
return self.help(mesg=mesg)

vals.append(valu)
Expand Down Expand Up @@ -1838,7 +1864,7 @@ class LimitCmd(Cmd):

def getArgParser(self):
pars = Cmd.getArgParser(self)
pars.add_argument('count', type=int, help='The maximum number of nodes to yield.')
pars.add_argument('count', type='int', help='The maximum number of nodes to yield.')
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is a breaking change we need to be aware of

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

should only apply to python storm commands ( since pure storm commands had to be serializeable ) so i'd call it an internal change, but yea. def need to be aware.

return pars

async def execStormCmd(self, runt, genr):
Expand Down Expand Up @@ -2218,7 +2244,7 @@ class IdenCmd(Cmd):

def getArgParser(self):
pars = Cmd.getArgParser(self)
pars.add_argument('iden', nargs='*', type=str, default=[],
pars.add_argument('iden', nargs='*', type='str', default=[],
help='Iden to lift nodes by. May be specified multiple times.')
return pars

Expand Down Expand Up @@ -2266,7 +2292,7 @@ async def execStormCmd(self, runt, genr):

def getArgParser(self):
pars = Cmd.getArgParser(self)
pars.add_argument('delay', type=float, default=1, help='Delay in floating point seconds.')
pars.add_argument('delay', type='float', default=1, help='Delay in floating point seconds.')
return pars

class GraphCmd(Cmd):
Expand All @@ -2293,7 +2319,7 @@ class GraphCmd(Cmd):
def getArgParser(self):

pars = Cmd.getArgParser(self)
pars.add_argument('--degrees', type=int, default=1, help='How many degrees to graph out.')
pars.add_argument('--degrees', type='int', default=1, help='How many degrees to graph out.')

pars.add_argument('--pivot', default=[], action='append',
help='Specify a storm pivot for all nodes. (must quote)')
Expand Down Expand Up @@ -2582,13 +2608,13 @@ class SpliceListCmd(Cmd):
def getArgParser(self):
pars = Cmd.getArgParser(self)

pars.add_argument('--maxtimestamp', type=int, default=None,
pars.add_argument('--maxtimestamp', type='int', default=None,
help='Only yield splices which occurred on or before this timestamp.')
pars.add_argument('--mintimestamp', type=int, default=None,
pars.add_argument('--mintimestamp', type='int', default=None,
help='Only yield splices which occurred on or after this timestamp.')
pars.add_argument('--maxtime', type=str, default=None,
pars.add_argument('--maxtime', type='str', default=None,
help='Only yield splices which occurred on or before this time.')
pars.add_argument('--mintime', type=str, default=None,
pars.add_argument('--mintime', type='str', default=None,
help='Only yield splices which occurred on or after this time.')

return pars
Expand Down Expand Up @@ -2885,7 +2911,7 @@ class LiftByVerb(Cmd):

def getArgParser(self):
pars = Cmd.getArgParser(self)
pars.add_argument('verb', type=str, required=True,
pars.add_argument('verb', type='str', required=True,
help='The edge verb to lift nodes by.')
pars.add_argument('--n2', action='store_true', default=False,
help='Lift by the N2 value instead of N1 value.')
Expand Down Expand Up @@ -2952,7 +2978,7 @@ class EdgesDelCmd(Cmd):

def getArgParser(self):
pars = Cmd.getArgParser(self)
pars.add_argument('verb', type=str, help='The verb of light edges to delete.')
pars.add_argument('verb', type='str', help='The verb of light edges to delete.')

pars.add_argument('--n2', action='store_true', default=False,
help='Delete light edges where input node is N2 instead of N1.')
Expand Down
74 changes: 62 additions & 12 deletions synapse/tests/test_lib_storm.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

import synapse.exc as s_exc
import synapse.common as s_common
import synapse.datamodel as s_datamodel

import synapse.lib.storm as s_storm

Expand Down Expand Up @@ -862,7 +863,7 @@ async def test_storm_argv_parser(self):
self.true(pars.exited)

pars = s_storm.Parser()
pars.add_argument('--yada', type=int)
pars.add_argument('--yada', type='int')
self.none(pars.parse_args(['--yada', 'hehe']))
self.true(pars.exited)

Expand All @@ -887,7 +888,7 @@ async def test_storm_argv_parser(self):

# Check formatting for store_true / store_false optargs
pars = s_storm.Parser()
pars.add_argument('--ques', nargs=2, type=int)
pars.add_argument('--ques', nargs=2, type='int')
pars.add_argument('--beep', action='store_true', help='beep beep')
pars.add_argument('--boop', action='store_false', help='boop boop')
pars.help()
Expand All @@ -898,28 +899,28 @@ async def test_storm_argv_parser(self):

# test some nargs type intersections
pars = s_storm.Parser()
pars.add_argument('--ques', nargs='?', type=int)
pars.add_argument('--ques', nargs='?', type='int')
self.none(pars.parse_args(['--ques', 'asdf']))
helptext = '\n'.join(pars.mesgs)
self.isin("Invalid value for type (<class 'int'>): asdf", helptext)
self.isin("Invalid value for type (int): asdf", helptext)

pars = s_storm.Parser()
pars.add_argument('--ques', nargs='*', type=int)
pars.add_argument('--ques', nargs='*', type='int')
self.none(pars.parse_args(['--ques', 'asdf']))
helptext = '\n'.join(pars.mesgs)
self.isin("Invalid value for type (<class 'int'>): asdf", helptext)
self.isin("Invalid value for type (int): asdf", helptext)

pars = s_storm.Parser()
pars.add_argument('--ques', nargs='+', type=int)
pars.add_argument('--ques', nargs='+', type='int')
self.none(pars.parse_args(['--ques', 'asdf']))
helptext = '\n'.join(pars.mesgs)
self.isin("Invalid value for type (<class 'int'>): asdf", helptext)
self.isin("Invalid value for type (int): asdf", helptext)

pars = s_storm.Parser()
pars.add_argument('foo', type=int)
pars.add_argument('foo', type='int')
self.none(pars.parse_args(['asdf']))
helptext = '\n'.join(pars.mesgs)
self.isin("Invalid value for type (<class 'int'>): asdf", helptext)
self.isin("Invalid value for type (int): asdf", helptext)

# argument count mismatch
pars = s_storm.Parser()
Expand All @@ -935,10 +936,59 @@ async def test_storm_argv_parser(self):
self.isin("2 arguments are required for --ques", helptext)

pars = s_storm.Parser()
pars.add_argument('--ques', nargs=2, type=int)
pars.add_argument('--ques', nargs=2, type='int')
self.none(pars.parse_args(['--ques', 'lolz', 'hehe']))
helptext = '\n'.join(pars.mesgs)
self.isin("Invalid value for type (<class 'int'>): lolz", helptext)
self.isin("Invalid value for type (int): lolz", helptext)

# test time argtype
ttyp = s_datamodel.Model().type('time')

pars = s_storm.Parser()
pars.add_argument('--yada', type='time')
args = pars.parse_args(['--yada', '20201021-1day'])
self.nn(args)
self.eq(ttyp.norm('20201021-1day')[0], args.yada)

args = pars.parse_args(['--yada', 1603229675444])
self.nn(args)
self.eq(ttyp.norm(1603229675444)[0], args.yada)

self.none(pars.parse_args(['--yada', 'hehe']))
self.true(pars.exited)
helptext = '\n'.join(pars.mesgs)
self.isin("Invalid value for type (time): hehe", helptext)

# test ival argtype
ityp = s_datamodel.Model().type('ival')

pars = s_storm.Parser()
pars.add_argument('--yada', type='ival')
args = pars.parse_args(['--yada', '20201021-1day'])
self.nn(args)
self.eq(ityp.norm('20201021-1day')[0], args.yada)

args = pars.parse_args(['--yada', 1603229675444])
self.nn(args)
self.eq(ityp.norm(1603229675444)[0], args.yada)

args = pars.parse_args(['--yada', ('20201021', '20201023')])
self.nn(args)
self.eq(ityp.norm(('20201021', '20201023'))[0], args.yada)

args = pars.parse_args(['--yada', (1603229675444, '20201021')])
self.nn(args)
self.eq(ityp.norm((1603229675444, '20201021'))[0], args.yada)

self.none(pars.parse_args(['--yada', 'hehe']))
self.true(pars.exited)
helptext = '\n'.join(pars.mesgs)
self.isin("Invalid value for type (ival): hehe", helptext)

# check adding argument with invalid type
with self.raises(s_exc.BadArg):
pars = s_storm.Parser()
pars.add_argument('--yada', type=int)

async def test_liftby_edge(self):
async with self.getTestCore() as core:
Expand Down
29 changes: 26 additions & 3 deletions synapse/tests/test_lib_stormtypes.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3231,17 +3231,20 @@ async def test_lib_stormtypes_cmdopts(self):
'cmdargs': (
('foo', {}),
('--bar', {'default': False, 'action': 'store_true'}),
('--footime', {'default': False, 'type': 'time'}),
),
'storm': '''
$lib.print($lib.len($cmdopts))
if ($lib.len($cmdopts) = 3) { $lib.print(foo) }
if ($lib.len($cmdopts) = 4) { $lib.print(foo) }

$set = $lib.set()
for ($name, $valu) in $cmdopts { $set.add($valu) }

if ($lib.len($set) = 3) { $lib.print(bar) }
if ($lib.len($set) = 4) { $lib.print(bar) }

if $cmdopts.bar { $lib.print(baz) }

if $cmdopts.footime { $lib.print($cmdopts.footime) }
'''
},
{
Expand All @@ -3256,12 +3259,32 @@ async def test_lib_stormtypes_cmdopts(self):
},
],
}
sadt = {
'name': 'bar',
'desc': 'test',
'version': (0, 0, 1),
'commands': [
{
'name': 'test.badtype',
'cmdargs': [
('--bar', {'type': 'notatype'}),
],
'storm': '''
$cmdopts.foo = hehe
'''
},
],
}
async with self.getTestCore() as core:
await core.addStormPkg(pdef)
msgs = await core.stormlist('test.cmdopts hehe --bar')
msgs = await core.stormlist('test.cmdopts hehe --bar --footime 20200101')
self.stormIsInPrint('foo', msgs)
self.stormIsInPrint('bar', msgs)
self.stormIsInPrint('baz', msgs)
self.stormIsInPrint('1577836800000', msgs)

with self.raises(s_exc.StormRuntimeError):
await core.nodes('test.setboom hehe --bar')

with self.raises(s_exc.SchemaViolation):
await core.addStormPkg(sadt)