Merge branch 'visi-layer-setinfo-logedits' of https://github.com/vert…

…exproject/synapse into visi-layer-setinfo-logedits

synapse/axon.py

@@ -406,6 +406,7 @@ async def wget(self, url, params=None, headers=None, json=None, body=None, metho
 
                     info = {
                         'ok': True,
+                        'url': str(resp.url),
                         'code': resp.status,
                         'headers': dict(resp.headers),
                     }

synapse/lib/storm.py

@@ -926,7 +926,44 @@ def reqValidPkgdef(pkgdef):
             $kild = $lib.ps.kill($cmdopts.iden)
             $lib.print("kill status: {kild}", kild=$kild)
         ''',
-    }
+    },
+    {
+        'name': 'wget',
+        'descr': 'Retrieve bytes from a URL and store them in the axon. Yields inet:urlfile nodes.',
+        'cmdargs': (
+            ('urls', {'nargs': '*', 'help': 'URLs to download.'}),
+            ('--no-ssl-verify', {'default': False, 'action': 'store_true', 'help': 'Ignore SSL certificate validation errors.'}),
+        ),
+        'storm': '''
+        init {
+            $count = (0)
+        }
+
+        $ssl = (not $cmdopts.no_ssl_verify)
+
+        if $node {
+            $count = ($count + 1)
+            if $cmdopts.urls {
+                $urls = $cmdopts.urls
+            } else {
+                if ($node.form() != "inet:url") {
+                    $lib.warn("wget can only take inet:url nodes as input without args.")
+                    $lib.exit()
+                }
+                $urls = ($node.value(),)
+            }
+            for $url in $urls {
+                -> { yield $lib.axon.urlfile($url, ssl=$ssl) }
+            }
+        }
+
+        if ($count = 0) {
+            for $url in $cmdopts.urls {
+                yield $lib.axon.urlfile($url, ssl=$ssl)
+            }
+        }
+        ''',
+    },
 )
 
 class DmonManager(s_base.Base):
@@ -1977,6 +2014,7 @@ class MergeCmd(Cmd):
     def getArgParser(self):
         pars = Cmd.getArgParser(self)
         pars.add_argument('--apply', default=False, action='store_true', help='Execute the merge changes.')
+        pars.add_argument('--diff', default=False, action='store_true', help='Enumerate all changes in the current layer.')
         return pars
 
     async def execStormCmd(self, runt, genr):
@@ -1988,6 +2026,18 @@ async def execStormCmd(self, runt, genr):
         layr0 = runt.snap.view.layers[0].iden
         layr1 = runt.snap.view.layers[1].iden
 
+        if self.opts.diff:
+
+            async for node, path in genr:
+                yield node, path
+
+            async def diffgenr():
+                async for buid, sode in runt.snap.view.layers[0].getStorNodes():
+                    node = await runt.snap.getNodeByBuid(buid)
+                    yield node, runt.initPath(node)
+
+            genr = diffgenr()
+
         async for node, path in genr:
 
             # the timestamp for the adds/subs of each node merge will match

synapse/lib/stormtypes.py

@@ -1140,6 +1140,7 @@ class LibAxon(Lib):
     def getObjLocals(self):
         return {
             'wget': self.wget,
+            'urlfile': self.urlfile,
         }
 
     async def wget(self, url, headers=None, params=None, method='GET', json=None, body=None, ssl=True):
@@ -1184,6 +1185,47 @@ async def wget(self, url, headers=None, params=None, method='GET', json=None, bo
         axon = self.runt.snap.core.axon
         return await axon.wget(url, headers=headers, params=params, method=method, ssl=ssl, body=body, json=json)
 
+    async def urlfile(self, *args, **kwargs):
+        '''
+        Retrive the target URL using the wget() function and construct an inet:urlfile node from the response.
+
+        Args: see $lib.axon.wget()
+
+        Returns:
+            inet:urlfile node on success.  $lib.null on error.
+        '''
+        resp = await self.wget(*args, **kwargs)
+        code = resp.get('code')
+
+        if code != 200:
+            mesg = f'$lib.axon.urlfile(): HTTP code {code} != 200'
+            await self.runt.warn(mesg, log=False)
+            return
+
+        url = resp.get('url')
+        hashes = resp.get('hashes')
+        props = {
+            'size': resp.get('size'),
+            'md5': hashes.get('md5'),
+            'sha1': hashes.get('sha1'),
+            'sha256': hashes.get('sha256'),
+            '.seen': 'now',
+        }
+
+        sha256 = hashes.get('sha256')
+        filenode = await self.runt.snap.addNode('file:bytes', sha256, props=props)
+
+        if not filenode.get('name'):
+            info = s_urlhelp.chopurl(url)
+            base = info.get('path').strip('/').split('/')[-1]
+            if base:
+                await filenode.set('name', base)
+
+        props = {'.seen': 'now'}
+        urlfile = await self.runt.snap.addNode('inet:urlfile', (url, sha256), props=props)
+
+        return urlfile
+
 @registry.registerLib
 class LibBytes(Lib):
     '''

synapse/tests/test_lib_storm.py

@@ -190,6 +190,27 @@ async def test_lib_storm_basics(self):
             with self.raises(s_exc.AuthDeny):
                 await core.callStorm(wget, opts=opts)
 
+            # test wget runtsafe / per-node / per-node with cmdopt
+            nodes = await core.nodes(f'wget https://127.0.0.1:{port}/api/v1/active')
+            self.len(1, nodes)
+            self.eq(nodes[0].ndef[0], 'inet:urlfile')
+
+            nodes = await core.nodes(f'inet:url=https://127.0.0.1:{port}/api/v1/active | wget')
+            self.len(1, nodes)
+            self.eq(nodes[0].ndef[0], 'inet:urlfile')
+
+            nodes = await core.nodes(f'inet:urlfile:url=https://127.0.0.1:{port}/api/v1/active | wget :url')
+            self.len(1, nodes)
+            self.eq(nodes[0].ndef[0], 'inet:urlfile')
+
+            # check that the file name got set...
+            nodes = await core.nodes(f'wget https://127.0.0.1:{port}/api/v1/active | -> file:bytes +:name=active')
+            self.len(1, nodes)
+            self.eq(nodes[0].ndef[0], 'file:bytes')
+
+            msgs = await core.stormlist(f'wget https://127.0.0.1:{port}/api/v1/newp')
+            self.stormIsInWarn('HTTP code 404', msgs)
+
             await visi.addRule((True, ('storm', 'lib', 'axon', 'wget')))
             resp = await core.callStorm(wget, opts=opts)
             self.true(resp['ok'])
@@ -318,6 +339,13 @@ async def test_lib_storm_basics(self):
             self.stormIsInPrint("aade791ea3263edd78e27d0351e7eed8372471a0434a6f0ba77101b5acf4f9bc inet:ipv4 DATA foo = 'bar'", msgs)
             self.stormIsInPrint('aade791ea3263edd78e27d0351e7eed8372471a0434a6f0ba77101b5acf4f9bc inet:ipv4 +(blahverb)> a0df14eab785847912993519f5606bbe741ad81afb51b81455ac6982a5686436', msgs)
 
+            msgs = await core.stormlist('ps:person | merge --diff', opts=opts)
+            self.stormIsInPrint('aade791ea3263edd78e27d0351e7eed8372471a0434a6f0ba77101b5acf4f9bc inet:ipv4:asn = 99', msgs)
+            self.stormIsInPrint("aade791ea3263edd78e27d0351e7eed8372471a0434a6f0ba77101b5acf4f9bc inet:ipv4#foo = ('2020/01/01 00:00:00.000', '2020/01/01 00:00:00.001')", msgs)
+            self.stormIsInPrint("aade791ea3263edd78e27d0351e7eed8372471a0434a6f0ba77101b5acf4f9bc inet:ipv4#foo:score = 100", msgs)
+            self.stormIsInPrint("aade791ea3263edd78e27d0351e7eed8372471a0434a6f0ba77101b5acf4f9bc inet:ipv4 DATA foo = 'bar'", msgs)
+            self.stormIsInPrint('aade791ea3263edd78e27d0351e7eed8372471a0434a6f0ba77101b5acf4f9bc inet:ipv4 +(blahverb)> a0df14eab785847912993519f5606bbe741ad81afb51b81455ac6982a5686436', msgs)
+
             await core.callStorm('inet:ipv4=11.22.33.44 | merge --apply', opts=opts)
             nodes = await core.nodes('inet:ipv4=11.22.33.44')
             self.len(1, nodes)