diff --git a/scripts/benchmark_cortex.py b/scripts/benchmark_cortex.py index c59e61ea5f..05f5d5a24f 100644 --- a/scripts/benchmark_cortex.py +++ b/scripts/benchmark_cortex.py @@ -59,7 +59,7 @@ Benchmark cortex operations TODO: separate client process, multiple clients -TODO: tagprops, regex, control flow, node data, multiple layers, spawn option, remote layer +TODO: tagprops, regex, control flow, node data, multiple layers, spawn option ''' logger = logging.getLogger(__name__) @@ -179,6 +179,10 @@ def printreport(self, configname: str): def reportdata(self): retn = [] + if self.num_iters < 3: + print('--niters must be > 2 for effective statistics') + return retn + for name, measurements in self.measurements.items(): # ms = ', '.join(f'{m[0]:0.3}' for m in measurements) tottimes = [m[0] for m in measurements[1:]] @@ -229,6 +233,14 @@ async def do00EmptyQuery(self, core: s_cortex.Cortex, prox: s_telepath.Proxy) -> assert count == 0 return self.workfactor // 10 + @benchmark() + async def do00NewQuery(self, core: s_cortex.Cortex, prox: s_telepath.Proxy) -> int: + for i in range(self.workfactor): + count = await acount(prox.eval(f'$x={i}')) + + assert count == 0 + return self.workfactor + @benchmark({'official'}) async def do01SimpleCount(self, core: s_cortex.Cortex, prox: s_telepath.Proxy) -> int: count = await acount(prox.eval('inet:ipv4 | count | spin')) @@ -295,6 +307,36 @@ async def do07AAddNodes(self, core: s_cortex.Cortex, prox: s_telepath.Proxy) -> assert count == self.workfactor return count + @benchmark({'addnodes', }) + async def do07AAddNodesCallStorm(self, core: s_cortex.Cortex, prox: s_telepath.Proxy) -> int: + tags_to_add = '+#test' + count = 0 + + for node in self.testdata.asns2prop: + props_to_add = f":name = {node[1]['props']['name']}" + form, valu = node[0] + + opts = {'vars': {'valu': valu}} + await prox.callStorm(f'[ {form}=$valu {props_to_add} {tags_to_add}] return($node.pack(dorepr=1))', + opts=opts) + count += 1 + assert count == self.workfactor + return count + + @benchmark({'addnodes', }) + async def do07AAddNodesStorm(self, core: s_cortex.Cortex, prox: s_telepath.Proxy) -> int: + tags_to_add = '+#test' + count = 0 + + for node in self.testdata.asns2prop: + props_to_add = f":name = {node[1]['props']['name']}" + form, valu = node[0] + + opts = {'vars': {'valu': valu}} + count += await acount(prox.storm(f'[ {form}=$valu {props_to_add} {tags_to_add}]', opts=opts)) + assert count == self.workfactor + return count + @benchmark({'addnodes'}) async def do07AAddNodesSync(self, core: s_cortex.Cortex, prox: s_telepath.Proxy) -> int: count = await acount(prox.addNodes(self.testdata.asns2)) @@ -354,6 +396,13 @@ async def do10AutoAdds(self, core: s_cortex.Cortex, prox: s_telepath.Proxy) -> i assert count == self.workfactor return self.workfactor + @benchmark() + async def do10SlashAdds(self, core: s_cortex.Cortex, prox: s_telepath.Proxy) -> int: + q = '[ inet:ipv4=1.2.0.0/16 ] | spin' + count = await acount(prox.eval(q)) + assert count == 0 + return 2 ** 16 + @benchmark() async def do10Formatting(self, core: s_cortex.Cortex, prox: s_telepath.Proxy) -> int: ''' @@ -505,8 +554,8 @@ def getParser(): help='Prefix to append to the autogenerated filename for json output.') parser.add_argument('--bench', '-b', nargs='*', default=None, help='Prefixes of which benchmarks to run (defaults to run all)') - parser.add_argument('--tag', '-t', default='official', - help='Tag of which suite to run (defaults to "official")') + parser.add_argument('--tag', '-t', default=None, + help='Tag of which suite to run (defaults to "official" if bench not set)') parser.add_argument('--do-profiling', action='store_true') return parser @@ -519,6 +568,9 @@ def getParser(): print('Error: module "yappi" must be installed to use --do-profiling') sys.exit(1) + if opts.bench is None and opts.tag is None: + opts.tag = 'official' + asyncio.run(benchmarkAll(opts.config, 1, opts.workfactor, opts.tmpdir, jsondir=opts.jsondir, jsonprefix=opts.jsonprefix, niters=opts.niters, bench=opts.bench, do_profiling=opts.do_profiling, tag=opts.tag))