From 430aa4522eb5eb4b423bbb643641a2963c8df658 Mon Sep 17 00:00:00 2001 From: Cisphyx Date: Wed, 2 Dec 2020 09:22:53 -0500 Subject: [PATCH] Prevent overflow errors in integer lifts (#1980) * Prevent overflow errors in integer lifts --- synapse/lib/layer.py | 38 ++++++++++++--- synapse/tests/test_lib_layer.py | 80 +++++++++++++++++++++++++++++++- synapse/tests/test_model_inet.py | 3 +- 3 files changed, 112 insertions(+), 9 deletions(-) diff --git a/synapse/lib/layer.py b/synapse/lib/layer.py index a234f79d92..f0e0ffff0a 100644 --- a/synapse/lib/layer.py +++ b/synapse/lib/layer.py @@ -606,6 +606,8 @@ def __init__(self, layr, stortype, size, signed): if signed: self.offset = 2 ** ((self.size * 8) - 1) - 1 + self.maxval = 2 ** (self.size * 8) - 1 + self.lifters.update({ '=': self._liftIntEq, '<': self._liftIntLt, @@ -625,8 +627,12 @@ def indx(self, valu): return (self.getIntIndx(valu),) async def _liftIntEq(self, liftby, valu): - indx = (valu + self.offset).to_bytes(self.size, 'big') - for item in liftby.buidsByDups(indx): + indx = valu + self.offset + if indx < 0 or indx > self.maxval: + return + + pkey = indx.to_bytes(self.size, 'big') + for item in liftby.buidsByDups(pkey): yield item async def _liftIntGt(self, liftby, valu): @@ -634,7 +640,13 @@ async def _liftIntGt(self, liftby, valu): yield item async def _liftIntGe(self, liftby, valu): - pkeymin = (valu + self.offset).to_bytes(self.size, 'big') + minv = valu + self.offset + if minv > self.maxval: + return + + minv = max(minv, 0) + + pkeymin = minv.to_bytes(self.size, 'big') pkeymax = self.fullbyts for item in liftby.buidsByRange(pkeymin, pkeymax): yield item @@ -644,14 +656,28 @@ async def _liftIntLt(self, liftby, valu): yield item async def _liftIntLe(self, liftby, valu): + maxv = valu + self.offset + if maxv < 0: + return + + maxv = min(maxv, self.maxval) + pkeymin = self.zerobyts - pkeymax = (valu + self.offset).to_bytes(self.size, 'big') + pkeymax = maxv.to_bytes(self.size, 'big') for item in liftby.buidsByRange(pkeymin, pkeymax): yield item async def _liftIntRange(self, liftby, valu): - pkeymin = (valu[0] + self.offset).to_bytes(self.size, 'big') - pkeymax = (valu[1] + self.offset).to_bytes(self.size, 'big') + minv = valu[0] + self.offset + maxv = valu[1] + self.offset + if minv > self.maxval or maxv < 0: + return + + minv = max(minv, 0) + maxv = min(maxv, self.maxval) + + pkeymin = minv.to_bytes(self.size, 'big') + pkeymax = maxv.to_bytes(self.size, 'big') for item in liftby.buidsByRange(pkeymin, pkeymax): yield item diff --git a/synapse/tests/test_lib_layer.py b/synapse/tests/test_lib_layer.py index d8a5206f74..7ad9a41e9b 100644 --- a/synapse/tests/test_lib_layer.py +++ b/synapse/tests/test_lib_layer.py @@ -527,13 +527,91 @@ async def test_layer_splices(self): async with await s_telepath.openurl(url) as layrprox: await self.agenlen(26, layrprox.splices()) + async def test_layer_stortype_int(self): + async with self.getTestCore() as core: + + layr = core.view.layers[0] + tmpdb = layr.layrslab.initdb('temp', dupsort=True) + + stor = s_layer.StorTypeInt(layr, s_layer.STOR_TYPE_I32, 8, True) + minv = -2 ** 63 + 1 + maxv = 2 ** 63 + vals = [minv, 0, 1, maxv] + + indxby = s_layer.IndxBy(layr, b'', tmpdb) + + for key, val in ((stor.indx(v), s_msgpack.en(v)) for v in vals): + layr.layrslab.put(key[0], val, db=tmpdb) + + retn = [s_msgpack.un(valu) async for valu in stor.indxBy(indxby, '=', minv)] + self.eq(retn, [minv]) + + retn = [s_msgpack.un(valu) async for valu in stor.indxBy(indxby, '=', maxv)] + self.eq(retn, [maxv]) + + retn = [s_msgpack.un(valu) async for valu in stor.indxBy(indxby, '<', minv + 1)] + self.eq(retn, [minv]) + + retn = [s_msgpack.un(valu) async for valu in stor.indxBy(indxby, '>', maxv - 1)] + self.eq(retn, [maxv]) + + retn = [s_msgpack.un(valu) async for valu in stor.indxBy(indxby, '<=', minv)] + self.eq(retn, [minv]) + + retn = [s_msgpack.un(valu) async for valu in stor.indxBy(indxby, '>=', maxv)] + self.eq(retn, [maxv]) + + retn = [s_msgpack.un(valu) async for valu in stor.indxBy(indxby, 'range=', (minv, maxv))] + self.eq(retn, vals) + + # Should get no results instead of overflowing + retn = [s_msgpack.un(valu) async for valu in stor.indxBy(indxby, '=', minv - 1)] + self.eq(retn, []) + + retn = [s_msgpack.un(valu) async for valu in stor.indxBy(indxby, '=', maxv + 1)] + self.eq(retn, []) + + retn = [s_msgpack.un(valu) async for valu in stor.indxBy(indxby, '<', minv)] + self.eq(retn, []) + + retn = [s_msgpack.un(valu) async for valu in stor.indxBy(indxby, '>', maxv)] + self.eq(retn, []) + + retn = [s_msgpack.un(valu) async for valu in stor.indxBy(indxby, '<=', minv - 1)] + self.eq(retn, []) + + retn = [s_msgpack.un(valu) async for valu in stor.indxBy(indxby, '>=', maxv + 1)] + self.eq(retn, []) + + retn = [s_msgpack.un(valu) async for valu in stor.indxBy(indxby, 'range=', (minv - 2, minv - 1))] + self.eq(retn, []) + + retn = [s_msgpack.un(valu) async for valu in stor.indxBy(indxby, 'range=', (maxv + 1, maxv + 2))] + self.eq(retn, []) + + # Value is out of range but there are still valid results + retn = [s_msgpack.un(valu) async for valu in stor.indxBy(indxby, '<', maxv + 2)] + self.eq(retn, vals) + + retn = [s_msgpack.un(valu) async for valu in stor.indxBy(indxby, '>', minv - 2)] + self.eq(retn, vals) + + retn = [s_msgpack.un(valu) async for valu in stor.indxBy(indxby, '<=', maxv + 1)] + self.eq(retn, vals) + + retn = [s_msgpack.un(valu) async for valu in stor.indxBy(indxby, '>=', minv - 1)] + self.eq(retn, vals) + + retn = [s_msgpack.un(valu) async for valu in stor.indxBy(indxby, 'range=', (minv - 1, maxv + 1))] + self.eq(retn, vals) + async def test_layer_stortype_float(self): async with self.getTestCore() as core: layr = core.view.layers[0] tmpdb = layr.layrslab.initdb('temp', dupsort=True) - stor = s_layer.StorTypeFloat(s_layer.STOR_TYPE_FLOAT64, 8) + stor = s_layer.StorTypeFloat(layr, s_layer.STOR_TYPE_FLOAT64, 8) vals = [math.nan, -math.inf, -99999.9, -0.0000000001, -42.1, -0.0, 0.0, 0.000001, 42.1, 99999.9, math.inf] indxby = s_layer.IndxBy(layr, b'', tmpdb) diff --git a/synapse/tests/test_model_inet.py b/synapse/tests/test_model_inet.py index 3f9de16a17..353ab6f0a2 100644 --- a/synapse/tests/test_model_inet.py +++ b/synapse/tests/test_model_inet.py @@ -721,8 +721,7 @@ async def test_ipv4(self): # > / < lifts and filters self.len(4, await core.nodes('[inet:ipv4=0 inet:ipv4=1 inet:ipv4=2 inet:ipv4=3]')) # Lifts - with self.raises(OverflowError): - await core.nodes('inet:ipv4<0') + self.len(0, await core.nodes('inet:ipv4<0')) self.len(1, await core.nodes('inet:ipv4<=0')) self.len(1, await core.nodes('inet:ipv4<1')) self.len(3, await core.nodes('inet:ipv4<=2'))