Skip to content

Commit

Permalink
Feature storm httpapi params (#1856)
Browse files Browse the repository at this point in the history 
add params support to storm http library
  • Loading branch information
@vEpiphyte
vEpiphyte authored Aug 21, 2020
1 parent 8a13aba commit 062fbd2
Show file tree
Hide file tree
Showing 3 changed files with 123 additions and 125 deletions.
39 changes: 19 additions & 20 deletions synapse/lib/stormhttp.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ def getObjLocals(self):
'request': self._httpRequest,
}

async def _httpEasyGet(self, url, headers=None, ssl_verify=True):
async def _httpEasyGet(self, url, headers=None, ssl_verify=True, params=None):
'''
Get the contents of a given URL.
Expand All @@ -32,24 +32,14 @@ async def _httpEasyGet(self, url, headers=None, ssl_verify=True):
ssl_verify (bool): Perform SSL/TLS verification. Defaults to true.
params (dict): Optional parameters which may be passed to the request.
Returns:
HttpResp: A Storm HttpResp object.
'''
url = await s_stormtypes.toprim(url)
headers = await s_stormtypes.toprim(headers)
return await self._httpRequest('get', url, headers=headers, ssl_verify=ssl_verify, params=params)

kwargs = {}
if not ssl_verify:
kwargs['ssl'] = False
async with aiohttp.ClientSession() as sess:
async with sess.get(url, headers=headers, **kwargs) as resp:
info = {
'code': resp.status,
'body': await resp.content.read(),
}
return HttpResp(info)

async def _httpPost(self, url, headers=None, json=None, body=None, ssl_verify=True):
async def _httpPost(self, url, headers=None, json=None, body=None, ssl_verify=True, params=None):
'''
Post data to a given URL.
Expand All @@ -64,12 +54,16 @@ async def _httpPost(self, url, headers=None, json=None, body=None, ssl_verify=Tr
ssl_verify (bool): Perform SSL/TLS verification. Defaults to true.
params (dict): Optional parameters which may be passed to the request.
Returns:
HttpResp: A Storm HttpResp object.
'''
return await self._httpRequest('POST', url, headers=headers, json=json, body=body, ssl_verify=ssl_verify)
return await self._httpRequest('POST', url, headers=headers, json=json,
body=body, ssl_verify=ssl_verify, params=params)

async def _httpRequest(self, meth, url, headers=None, json=None, body=None, ssl_verify=True):
async def _httpRequest(self, meth, url, headers=None, json=None, body=None, ssl_verify=True,
params=None):
'''
Make an HTTP request using the given HTTP method to the url.
Expand All @@ -86,6 +80,8 @@ async def _httpRequest(self, meth, url, headers=None, json=None, body=None, ssl_
ssl_verify (bool): Perform SSL/TLS verification. Defaults to true.
params (dict): Optional parameters which may be passed to the request.
Returns:
HttpResp: A Storm HttpResp object.
'''
Expand All @@ -95,10 +91,13 @@ async def _httpRequest(self, meth, url, headers=None, json=None, body=None, ssl_
json = await s_stormtypes.toprim(json)
body = await s_stormtypes.toprim(body)
headers = await s_stormtypes.toprim(headers)
params = await s_stormtypes.toprim(params)

kwargs = {}
if not ssl_verify:
kwargs['ssl'] = False
if params:
kwargs['params'] = params

async with aiohttp.ClientSession() as sess:
try:
Expand All @@ -108,9 +107,9 @@ async def _httpRequest(self, meth, url, headers=None, json=None, body=None, ssl_
'body': await resp.content.read()
}
return HttpResp(info)
except ValueError as e:
mesg = f'Error during http post - {str(e)}'
raise s_exc.StormRuntimeError(mesg=mesg, headers=headers, json=json, body=body) from None
except (TypeError, ValueError) as e:
mesg = f'Error during http {meth} - {str(e)}'
raise s_exc.StormRuntimeError(mesg=mesg, headers=headers, json=json, body=body, params=params) from None

@s_stormtypes.registry.registerType
class HttpResp(s_stormtypes.StormType):
Expand Down
97 changes: 73 additions & 24 deletions synapse/tests/test_lib_stormhttp.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,41 +8,77 @@ async def test_storm_http_get(self):
addr, port = await core.addHttpsPort(0)
root = await core.auth.getUserByName('root')
await root.setPasswd('root')
text = '''
$hdr = (
("User-Agent", "Storm HTTP Stuff"),
)
$url = $lib.str.format("https://root:[email protected]:{port}/api/v1/model", port=$port)

for ($name, $fdef) in $lib.inet.http.get($url, headers=$hdr, ssl_verify=$(0)).json().result.forms {
[ test:str=$name ]
}
core.addHttpApi('/api/v0/test', s_test.HttpReflector, {'cell': core})
url = f'https://root:[email protected]:{port}/api/v0/test'
opts = {'vars': {'url': url}}

# Header and params as dict
q = '''
$params=$lib.dict(key=valu, foo=bar)
$hdr = (
("User-Agent", "Storm HTTP Stuff"),
)
$resp = $lib.inet.http.get($url, headers=$hdr, params=$params, ssl_verify=$lib.false)
return ( $resp.json() )
'''
opts = {'vars': {'port': port}}
nodes = await core.nodes(text, opts=opts)
self.len(1, await core.nodes('test:str=inet:ipv4'))
resp = await core.callStorm(q, opts=opts)
data = resp.get('result')
self.eq(data.get('params'), {'key': ('valu',), 'foo': ('bar',)})
self.eq(data.get('headers').get('User-Agent'), 'Storm HTTP Stuff')

# params as list of key/value pairs
q = '''
$params=((foo, bar), (key, valu))
$resp = $lib.inet.http.get($url, params=$params, ssl_verify=$lib.false)
return ( $resp.json() )
'''
resp = await core.callStorm(q, opts=opts)
data = resp.get('result')
self.eq(data.get('params'), {'key': ('valu',), 'foo': ('bar',)})

# params as a urlencoded string
q = '''
$params="foo=bar&key=valu&foo=baz"
$resp = $lib.inet.http.get($url, params=$params, ssl_verify=$lib.false)
return ( $resp.json() )
'''
resp = await core.callStorm(q, opts=opts)
data = resp.get('result')
self.eq(data.get('params'), {'key': ('valu',), 'foo': ('bar', 'baz')})

# Bad param
q = '''
$params=(1138)
$resp = $lib.inet.http.get($url, params=$params, ssl_verify=$lib.false)
return ( $resp.json() )
'''
msgs = await core.stormlist(q, opts=opts)
self.stormIsInErr('Error during http get - Invalid query type', msgs)

async def test_storm_http_request(self):

async with self.getTestCore() as core:
addr, port = await core.addHttpsPort(0)
root = await core.auth.getUserByName('root')
await root.setPasswd('root')
text = '''
$hdr = (
core.addHttpApi('/api/v0/test', s_test.HttpReflector, {'cell': core})
url = f'https://root:[email protected]:{port}/api/v0/test'
opts = {'vars': {'url': url}}
q = '''
$params=$lib.dict(key=valu, foo=bar)
$hdr = (
("User-Agent", "Storm HTTP Stuff"),
)
$url = $lib.str.format("https://root:[email protected]:{port}/api/v1/model", port=$port)
for ($name, $fdef) in $lib.inet.http.request(GET, $url, headers=$hdr, ssl_verify=$(0)).json().result.forms {
[ test:str=$name ]
}
)
$resp = $lib.inet.http.request(GET, $url, headers=$hdr, params=$params, ssl_verify=$lib.false)
return ( $resp.json() )
'''
opts = {'vars': {'port': port}}
nodes = await core.nodes(text, opts=opts)
self.len(1, await core.nodes('test:str=inet:ipv4'))
resp = await core.callStorm(q, opts=opts)
data = resp.get('result')
self.eq(data.get('params'), {'key': ('valu',), 'foo': ('bar',)})
self.eq(data.get('headers').get('User-Agent'), 'Storm HTTP Stuff')

async def test_storm_http_post_api(self):
async def test_storm_http_post(self):

async with self.getTestCore() as core:
addr, port = await core.addHttpsPort(0)
Expand Down Expand Up @@ -73,6 +109,19 @@ async def test_storm_http_post_api(self):
self.len(1, nodes)
self.assertIn('vertex', [u.name for u in core.auth.users()])

core.addHttpApi('/api/v0/test', s_test.HttpReflector, {'cell': core})
url = f'https://root:[email protected]:{port}/api/v0/test'
opts = {'vars': {'url': url, 'buf': b'1234'}}
q = '''
$params=$lib.dict(key=valu, foo=bar)
$resp = $lib.inet.http.post($url, params=$params, body=$buf, ssl_verify=$lib.false)
return ( $resp.json() )
'''
resp = await core.callStorm(q, opts=opts)
data = resp.get('result')
self.eq(data.get('params'), {'key': ('valu',), 'foo': ('bar',)})
self.eq(data.get('body'), 'MTIzNA==')

async def test_storm_http_post_file(self):

async with self.getTestCore() as core:
Expand Down Expand Up @@ -107,4 +156,4 @@ async def test_storm_http_post_file(self):
self.len(1, errs)
err = errs[0]
self.eq(err[0], 'StormRuntimeError')
self.isin('Error during http post - data and json parameters can not be used at the same time', err[1].get('mesg'))
self.isin('Error during http POST - data and json parameters can not be used at the same time', err[1].get('mesg'))
112 changes: 31 additions & 81 deletions synapse/tests/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,9 @@
import synapse.lib.types as s_types
import synapse.lib.module as s_module
import synapse.lib.output as s_output
import synapse.lib.httpapi as s_httpapi
import synapse.lib.msgpack as s_msgpack
import synapse.lib.lmdbslab as s_slab
import synapse.lib.lmdbslab as s_lmdbslab
import synapse.lib.thishost as s_thishost
import synapse.lib.stormtypes as s_stormtypes

Expand Down Expand Up @@ -502,85 +503,6 @@ def expect(self, substr, throw=True):
def clear(self):
self.mesgs.clear()

class TestSteps:
'''
A class to assist with interlocking for multi-thread tests.
Args:
names (list): A list of names of tests steps as strings.
'''
def __init__(self, names):
self.steps = {}
self.names = names

for name in names:
self.steps[name] = threading.Event()

def done(self, step):
'''
Mark the step name as complete.
Args:
step (str): The step name to mark complete
'''
self.steps[step].set()

def wait(self, step, timeout=None):
'''
Wait (up to timeout seconds) for a step to complete.
Args:
step (str): The step name to wait for.
timeout (int): The timeout in seconds (or None)
Returns:
bool: True if the step is completed within the wait timeout.
Raises:
StepTimeout: on wait timeout
'''
if not self.steps[step].wait(timeout=timeout):
raise s_exc.StepTimeout(mesg='timeout waiting for step', step=step)
return True

def step(self, done, wait, timeout=None):
'''
Complete a step and wait for another.
Args:
done (str): The step name to complete.
wait (str): The step name to wait for.
timeout (int): The wait timeout.
'''
self.done(done)
return self.wait(wait, timeout=timeout)

def waitall(self, timeout=None):
'''
Wait for all the steps to be complete.
Args:
timeout (int): The wait timeout (per step).
Returns:
bool: True when all steps have completed within the alloted time.
Raises:
StepTimeout: When the first step fails to complete in the given time.
'''
for name in self.names:
self.wait(name, timeout=timeout)
return True

def clear(self, step):
'''
Clear the event for a given step.
Args:
step (str): The name of the step.
'''
self.steps[step].clear()

class CmdGenerator:

def __init__(self, cmds):
Expand Down Expand Up @@ -669,6 +591,34 @@ async def wait(self, timeout=None):
return await asyncio.Event.wait(self)
return await s_coro.event_wait(self, timeout=timeout)

class HttpReflector(s_httpapi.Handler):
'''Test handler which reflects get/post data back to the caller'''
async def get(self):
resp = {}
if self.request.arguments:
d = collections.defaultdict(list)
resp['params'] = d
for k, items in self.request.arguments.items():
for v in items:
d[k].append(v.decode())
resp['headers'] = dict(self.request.headers)
resp['path'] = self.request.path
self.sendRestRetn(resp)

async def post(self):
resp = {}
if self.request.arguments:
d = collections.defaultdict(list)
resp['params'] = d
for k, items in self.request.arguments.items():
for v in items:
d[k].append(v.decode())
resp['headers'] = dict(self.request.headers)
resp['path'] = self.request.path
if self.request.body:
resp['body'] = s_common.enbase64(self.request.body)
self.sendRestRetn(resp)

s_task.vardefault('applynest', lambda: None)

async def _doubleapply(self, indx, item):
Expand Down Expand Up @@ -1750,7 +1700,7 @@ async def getTestHiveFromDirn(self, dirn):
import synapse.lib.const as s_const
map_size = s_const.gibibyte

async with await s_slab.Slab.anit(dirn, map_size=map_size) as slab:
async with await s_lmdbslab.Slab.anit(dirn, map_size=map_size) as slab:

nexsroot = await s_nexus.NexsRoot.anit(dirn)
await nexsroot.startup(None)
Expand Down

0 comments on commit 062fbd2

Please sign in to comment.