diff --git a/docs/synapse/userguides/storm_ref_lift.ipynb b/docs/synapse/userguides/storm_ref_lift.ipynb index 98b9825ef8..e2767ead10 100644 --- a/docs/synapse/userguides/storm_ref_lift.ipynb +++ b/docs/synapse/userguides/storm_ref_lift.ipynb @@ -48,6 +48,7 @@ "Lift operations retrieve a set of nodes from a Synapse Cortex based on specified criteria. While all lift operations are retrieval operations, they can be broken down into “types” of lifts based on the criteria, comparison operator, or special handler used:\n", "\n", "- `Simple Lifts`_\n", + "- `Try Lifts`_\n", "- `Lifts Using Standard Comparison Operators`_\n", "- `Lifts Using Extended Comparison Operators`_\n", "\n", @@ -623,6 +624,170 @@ "_ = await core.fini()" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "hideCode": true, + "hideOutput": true + }, + "outputs": [], + "source": [ + "# Create a cortex for the Safe Lifts section\n", + "core = await getTempCoreCmdr()" + ] + }, + { + "cell_type": "raw", + "metadata": { + "hideCode": false + }, + "source": [ + "Try Lifts\n", + "---------\n", + "\n", + "Try lifts refer to lifts that \"try\" to perform a Cortex lift operation, and fail silently if :ref:`data-type` normalization is not successful. Try lifts prevent a Cortex from throwing a runtime execution error, and terminating query execution if an invalid Type is encountered.\n", + "\n", + "When lifting nodes by property value using the equals (``=``) comparator, if Type validation fails for a supplied property value,  the Cortex will throw a ``BadTypeValu`` error, and terminate the query as shown below." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "hideCode": true, + "hideOutput": true, + "hidePrompt": false + }, + "outputs": [], + "source": [ + "# Make a FQDN, MD5, IPv4, and email nodes:\n", + "q = '[ inet:fqdn=evil.com inet:dns:a=(evil.com,192.168.0.100) hash:md5=174cc541c8d9e1accef73025293923a6 inet:ipv4=8.8.8.8 inet:email=jane@goodgirl.com inet:email=jack@soso.net]'\n", + "# Execute query and test\n", + "podes = await core.eval(q, num=6, cmdr=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "hideCode": true, + "hideOutput": false, + "hidePrompt": false + }, + "outputs": [], + "source": [ + "# Define and print test query\n", + "q = 'inet:ipv4 = evil.com inet:ipv4 = 8.8.8.8'\n", + "# Execute the query and test\n", + "podes = await core.storm(q, num=0, cmdr=True, suppress_logging=True)" + ] + }, + { + "cell_type": "raw", + "metadata": { + "hidePrompt": false + }, + "source": [ + "To suppress errors, and prevent premature query termination, Storm supports the use of the try operator (``?=``) when performing property value lifts. This operator is useful when you are performing multiple Cortex operations in succession within a single query, lifting nodes using external data that has not been normalized, or lifting nodes during automation, and do not want a query to terminate if an invalid Type is encountered.\n", + "\n", + "\n", + "**Syntax:**\n", + "\n", + "*
[:]* ?= **\n", + "\n", + "**Examples:**\n", + "\n", + "- Try to lift the MD5 node ``174cc541c8d9e1accef73025293923a6``:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "hideCode": true + }, + "outputs": [], + "source": [ + "# Define and print test query\n", + "q = 'hash:md5 ?= 174cc541c8d9e1accef73025293923a6'\n", + "print(q)\n", + "# Execute the query and test\n", + "podes = await core.eval(q, num=1, cmdr=False)" + ] + }, + { + "cell_type": "raw", + "metadata": { + "hideCode": false, + "hideOutput": false + }, + "source": [ + "- Try to lift the DNS nodes whose ``inet:dns:a:ipv4`` secondary property value equals ``'192.168.0.100'``. Notice that an error message is not displayed, despite an invalid IPv4 address ``'192.168.0.1000'`` being entered:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "hideCode": true, + "hideOutput": false + }, + "outputs": [], + "source": [ + "# Define and print test query\n", + "q = 'inet:dns:a:ipv4 ?= 192.168.0.1000'\n", + "print(q)\n", + "# Execute the query and test\n", + "podes = await core.eval(q, num=0, cmdr=True)" + ] + }, + { + "cell_type": "raw", + "metadata": {}, + "source": [ + "- Try to lift the email address nodes ``'jack@soso.net'`` and ``'jane@goodgirl.com'``. Notice that despite the first email address being entered incorrectly, the error message is suppressed, and the query executes to completion." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "hideCode": true, + "hideOutput": false + }, + "outputs": [], + "source": [ + "# Define and print test query\n", + "q = 'inet:email ?= \"jack[at]soso.net\" inet:email ?= \"jane@goodgirl.com\"'\n", + "print(q)\n", + "# Execute the query and test\n", + "podes = await core.eval(q, num=1, cmdr=True)" + ] + }, + { + "cell_type": "raw", + "metadata": {}, + "source": [ + "**Usage Notes:**\n", + "\n", + "- The try operator should be used when you want Storm query execution to continue even if an invalid Type is encountered. \n", + "- It is not recommended to use the try operator when you want to raise an error, or stop query execution if an invalid Type is encountered." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "hideCode": true, + "hideOutput": true + }, + "outputs": [], + "source": [ + "# Close cortex for Safe Lifts section\n", + "_ = await core.fini()" + ] + }, { "cell_type": "raw", "metadata": {}, @@ -2182,7 +2347,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.1" + "version": "3.7.7" } }, "nbformat": 4, diff --git a/synapse/lib/jupyter.py b/synapse/lib/jupyter.py index 676d43ab06..951f2f4d8c 100644 --- a/synapse/lib/jupyter.py +++ b/synapse/lib/jupyter.py @@ -1,6 +1,7 @@ import os import copy import json +import logging import pathlib import contextlib @@ -11,6 +12,10 @@ import synapse.lib.cmdr as s_cmdr import synapse.lib.msgpack as s_msgpack +loggers_to_supress = ( + 'synapse.lib.view', +) + def getDocPath(fn, root=None): ''' Helper for getting a documentation data file paths. @@ -167,28 +172,47 @@ async def runCmdLine(self, text): ''' await self.cmdr.runCmdLine(text) - async def _runStorm(self, text, opts=None, cmdr=False): + @contextlib.contextmanager + def suppress_logging(self, suppress): + ''' + Context manager to suppress specific loggers. + ''' + logs = {} + if not suppress: + yield None + else: + try: + for logname in loggers_to_supress: + logger = logging.getLogger(logname) + if logger is not None: + logs[logname] = (logger, logger.level) + logger.setLevel(logger.level + 100) + yield None + finally: + for (logger, level) in logs.values(): + logger.setLevel(level) + + async def _runStorm(self, text, opts=None, cmdr=False, suppress_logging=False): mesgs = [] + with self.suppress_logging(suppress_logging): + if cmdr: + if self.prefix: + text = ' '.join((self.prefix, text)) - if cmdr: + def onEvent(event): + mesg = event[1].get('mesg') + mesgs.append(mesg) - if self.prefix: - text = ' '.join((self.prefix, text)) + with self.cmdr.onWith('storm:mesg', onEvent): + await self.runCmdLine(text) - def onEvent(event): - mesg = event[1].get('mesg') - mesgs.append(mesg) - - with self.cmdr.onWith('storm:mesg', onEvent): - await self.runCmdLine(text) - - else: - async for mesg in self.core.storm(text, opts=opts): - mesgs.append(mesg) + else: + async for mesg in self.core.storm(text, opts=opts): + mesgs.append(mesg) return mesgs - async def storm(self, text, opts=None, num=None, cmdr=False): + async def storm(self, text, opts=None, num=None, cmdr=False, suppress_logging=False): ''' A helper for executing a storm command and getting a list of storm messages. @@ -197,6 +221,7 @@ async def storm(self, text, opts=None, num=None, cmdr=False): opts (dict): Opt to pass to the cortex during execution. num (int): Number of nodes to expect in the output query. Checks that with an assert statement. cmdr (bool): If True, executes the line via the Cmdr CLI and will send output to outp. + suppress_logging (bool): If True, suppresses some logging related to Storm runtime exceptions. Notes: The opts dictionary will not be used if cmdr=True. @@ -204,7 +229,7 @@ async def storm(self, text, opts=None, num=None, cmdr=False): Returns: list: A list of storm messages. ''' - mesgs = await self._runStorm(text, opts, cmdr) + mesgs = await self._runStorm(text, opts, cmdr, suppress_logging) if num is not None: nodes = [m for m in mesgs if m[0] == 'node'] if len(nodes) != num: diff --git a/synapse/tests/test_lib_jupyter.py b/synapse/tests/test_lib_jupyter.py index cbb52be4d5..66b552c782 100644 --- a/synapse/tests/test_lib_jupyter.py +++ b/synapse/tests/test_lib_jupyter.py @@ -134,6 +134,28 @@ async def test_cmdrcore(self): self.true(outp.expect('cli> help')) self.true(outp.expect('List commands and display help output.')) + async def test_log_supression(self): + + async with self.getTestCoreAndProxy() as (realcore, core): + + outp = self.getTestOutp() + async with await s_jupyter.CmdrCore.anit(core, outp=outp) as cmdrcore: + with self.getAsyncLoggerStream('synapse.lib.view') as stream: + mesgs = await cmdrcore.storm('[test:int=beep]', + num=0, cmdr=False, + suppress_logging=True) + self.stormIsInErr('invalid literal for int', mesgs) + stream.seek(0) + self.notin('Error during storm execution', stream.read()) + + with self.getAsyncLoggerStream('synapse.lib.view', + 'Error during storm execution') as stream: + mesgs = await cmdrcore.storm('[test:int=beep]', + num=0, cmdr=False, + suppress_logging=False) + self.true(await stream.wait(6)) + self.stormIsInErr('invalid literal for int', mesgs) + def test_doc_data(self): with self.getTestDir() as dirn: s_common.gendir(dirn, 'docdata', 'stuff')