Reigndocs

docs/synapse/userguides/storm_ref_lift.ipynb

@@ -48,6 +48,7 @@
     "Lift operations retrieve a set of nodes from a Synapse Cortex based on specified criteria. While all lift operations are retrieval operations, they can be broken down into “types” of lifts based on the criteria, comparison operator, or special handler used:\n",
     "\n",
     "- `Simple Lifts`_\n",
+    "- `Try Lifts`_\n",
     "- `Lifts Using Standard Comparison Operators`_\n",
     "- `Lifts Using Extended Comparison Operators`_\n",
     "\n",
@@ -623,6 +624,170 @@
     "_ = await core.fini()"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "hideCode": true,
+    "hideOutput": true
+   },
+   "outputs": [],
+   "source": [
+    "# Create a cortex for the Safe Lifts section\n",
+    "core = await getTempCoreCmdr()"
+   ]
+  },
+  {
+   "cell_type": "raw",
+   "metadata": {
+    "hideCode": false
+   },
+   "source": [
+    "Try Lifts\n",
+    "---------\n",
+    "\n",
+    "Try lifts refer to lifts that \"try\" to perform a Cortex lift operation, and fail silently if :ref:`data-type` normalization is not successful. Try lifts prevent a Cortex from throwing a runtime execution error, and terminating query execution if an invalid Type is encountered.\n",
+    "\n",
+    "When lifting nodes by property value using the equals (``=``) comparator, if Type validation fails for a supplied property value,  the Cortex will throw a ``BadTypeValu`` error, and terminate the query as shown below."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "hideCode": true,
+    "hideOutput": true,
+    "hidePrompt": false
+   },
+   "outputs": [],
+   "source": [
+    "# Make a FQDN, MD5, IPv4, and email nodes:\n",
+    "q = '[ inet:fqdn=evil.com inet:dns:a=(evil.com,192.168.0.100) hash:md5=174cc541c8d9e1accef73025293923a6 inet:ipv4=8.8.8.8 inet:[email protected] inet:[email protected]]'\n",
+    "# Execute query and test\n",
+    "podes = await core.eval(q, num=6, cmdr=False)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "hideCode": true,
+    "hideOutput": false,
+    "hidePrompt": false
+   },
+   "outputs": [],
+   "source": [
+    "# Define and print test query\n",
+    "q = 'inet:ipv4 = evil.com inet:ipv4 = 8.8.8.8'\n",
+    "# Execute the query and test\n",
+    "podes = await core.storm(q, num=0, cmdr=True, suppress_logging=True)"
+   ]
+  },
+  {
+   "cell_type": "raw",
+   "metadata": {
+    "hidePrompt": false
+   },
+   "source": [
+    "To suppress errors, and prevent premature query termination, Storm supports the use of the try operator (``?=``) when performing property value lifts. This operator is useful when you are performing multiple Cortex operations in succession within a single query, lifting nodes using external data that has not been normalized, or lifting nodes during automation, and do not want a query to terminate if an invalid Type is encountered.\n",
+    "\n",
+    "\n",
+    "**Syntax:**\n",
+    "\n",
+    "*<form>[:<prop>]* ?= *<pval>*\n",
+    "\n",
+    "**Examples:**\n",
+    "\n",
+    "- Try to lift the MD5 node ``174cc541c8d9e1accef73025293923a6``:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "hideCode": true
+   },
+   "outputs": [],
+   "source": [
+    "# Define and print test query\n",
+    "q = 'hash:md5 ?= 174cc541c8d9e1accef73025293923a6'\n",
+    "print(q)\n",
+    "# Execute the query and test\n",
+    "podes = await core.eval(q, num=1, cmdr=False)"
+   ]
+  },
+  {
+   "cell_type": "raw",
+   "metadata": {
+    "hideCode": false,
+    "hideOutput": false
+   },
+   "source": [
+    "- Try to lift the DNS nodes whose ``inet:dns:a:ipv4`` secondary property value equals ``'192.168.0.100'``. Notice that an error message is displayed.despite an invalid IPv4 address ``'192.168.0.1000'`` being entered:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "hideCode": true,
+    "hideOutput": false
+   },
+   "outputs": [],
+   "source": [
+    "# Define and print test query\n",
+    "q = 'inet:dns:a:ipv4 ?= 192.168.0.1000'\n",
+    "print(q)\n",
+    "# Execute the query and test\n",
+    "podes = await core.eval(q, num=0, cmdr=True)"
+   ]
+  },
+  {
+   "cell_type": "raw",
+   "metadata": {},
+   "source": [
+    "- Try to lift the email address nodes ``'[email protected]'`` and ``'[email protected]'``. Notice that despite the first email address being entered incorrectly, the error message is suppressed, and the query executes to completion."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "hideCode": true,
+    "hideOutput": false
+   },
+   "outputs": [],
+   "source": [
+    "# Define and print test query\n",
+    "q = 'inet:email ?= \"jack[at]soso.net\" inet:email ?= \"[email protected]\"'\n",
+    "print(q)\n",
+    "# Execute the query and test\n",
+    "podes = await core.eval(q, num=1, cmdr=True)"
+   ]
+  },
+  {
+   "cell_type": "raw",
+   "metadata": {},
+   "source": [
+    "**Usage Notes:**\n",
+    "\n",
+    "- The try operator should be used when you want Storm query execution to continue even if an invalid Type is encountered. \n",
+    "- It is not recommended to use the try operator when you want to raise an error, or stop query execution if an invalid Type is encountered."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "hideCode": true,
+    "hideOutput": true
+   },
+   "outputs": [],
+   "source": [
+    "# Close cortex for Safe Lifts section\n",
+    "_ = await core.fini()"
+   ]
+  },
   {
    "cell_type": "raw",
    "metadata": {},
@@ -2182,7 +2347,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.7.1"
+   "version": "3.7.7"
   }
  },
  "nbformat": 4,

synapse/lib/jupyter.py

@@ -1,6 +1,7 @@
 import os
 import copy
 import json
+import logging
 import pathlib
 import contextlib
 
@@ -11,6 +12,10 @@
 import synapse.lib.cmdr as s_cmdr
 import synapse.lib.msgpack as s_msgpack
 
+loggers_to_supress = (
+    'synapse.lib.view',
+)
+
 def getDocPath(fn, root=None):
     '''
     Helper for getting a documentation data file paths.
@@ -167,28 +172,47 @@ async def runCmdLine(self, text):
         '''
         await self.cmdr.runCmdLine(text)
 
-    async def _runStorm(self, text, opts=None, cmdr=False):
+    @contextlib.contextmanager
+    def suppress_logging(self, suppress):
+        '''
+        Context manager to suppress specific loggers.
+        '''
+        logs = {}
+        if not suppress:
+            yield None
+        else:
+            try:
+                for logname in loggers_to_supress:
+                    logger = logging.getLogger(logname)
+                    if logger is not None:
+                        logs[logname] = (logger, logger.level)
+                        logger.setLevel(logger.level + 100)
+                yield None
+            finally:
+                for (logger, level) in logs.values():
+                    logger.setLevel(level)
+
+    async def _runStorm(self, text, opts=None, cmdr=False, suppress_logging=False):
         mesgs = []
+        with self.suppress_logging(suppress_logging):
+            if cmdr:
+                if self.prefix:
+                    text = ' '.join((self.prefix, text))
 
-        if cmdr:
+                def onEvent(event):
+                    mesg = event[1].get('mesg')
+                    mesgs.append(mesg)
 
-            if self.prefix:
-                text = ' '.join((self.prefix, text))
+                with self.cmdr.onWith('storm:mesg', onEvent):
+                    await self.runCmdLine(text)
 
-            def onEvent(event):
-                mesg = event[1].get('mesg')
-                mesgs.append(mesg)
-
-            with self.cmdr.onWith('storm:mesg', onEvent):
-                await self.runCmdLine(text)
-
-        else:
-            async for mesg in self.core.storm(text, opts=opts):
-                mesgs.append(mesg)
+            else:
+                async for mesg in self.core.storm(text, opts=opts):
+                    mesgs.append(mesg)
 
         return mesgs
 
-    async def storm(self, text, opts=None, num=None, cmdr=False):
+    async def storm(self, text, opts=None, num=None, cmdr=False, suppress_logging=False):
         '''
         A helper for executing a storm command and getting a list of storm messages.
 
@@ -197,14 +221,15 @@ async def storm(self, text, opts=None, num=None, cmdr=False):
             opts (dict): Opt to pass to the cortex during execution.
             num (int): Number of nodes to expect in the output query. Checks that with an assert statement.
             cmdr (bool): If True, executes the line via the Cmdr CLI and will send output to outp.
+            suppress_logging (bool): If True, suppresses some logging related to Storm runtime exceptions.
 
         Notes:
             The opts dictionary will not be used if cmdr=True.
 
         Returns:
             list: A list of storm messages.
         '''
-        mesgs = await self._runStorm(text, opts, cmdr)
+        mesgs = await self._runStorm(text, opts, cmdr, suppress_logging)
         if num is not None:
             nodes = [m for m in mesgs if m[0] == 'node']
             if len(nodes) != num:

synapse/tests/test_lib_jupyter.py

@@ -134,6 +134,28 @@ async def test_cmdrcore(self):
                 self.true(outp.expect('cli> help'))
                 self.true(outp.expect('List commands and display help output.'))
 
+    async def test_log_supression(self):
+
+        async with self.getTestCoreAndProxy() as (realcore, core):
+
+            outp = self.getTestOutp()
+            async with await s_jupyter.CmdrCore.anit(core, outp=outp) as cmdrcore:
+                with self.getAsyncLoggerStream('synapse.lib.view') as stream:
+                    mesgs = await cmdrcore.storm('[test:int=beep]',
+                                                 num=0, cmdr=False,
+                                                 suppress_logging=True)
+                    self.stormIsInErr('invalid literal for int', mesgs)
+                stream.seek(0)
+                self.notin('Error during storm execution', stream.read())
+
+                with self.getAsyncLoggerStream('synapse.lib.view',
+                                                     'Error during storm execution') as stream:
+                    mesgs = await cmdrcore.storm('[test:int=beep]',
+                                                 num=0, cmdr=False,
+                                                 suppress_logging=False)
+                    self.true(await stream.wait(6))
+                    self.stormIsInErr('invalid literal for int', mesgs)
+
     def test_doc_data(self):
         with self.getTestDir() as dirn:
             s_common.gendir(dirn, 'docdata', 'stuff')