initial model for prices, currencies, securities and exchanges (#1820)

hugenum base type and inital security and exchange model

synapse/common.py

@@ -8,6 +8,7 @@
 import base64
 import shutil
 import struct
+import decimal
 import fnmatch
 import hashlib
 import logging
@@ -145,6 +146,13 @@ def intify(x):
     except (TypeError, ValueError):
         return None
 
+hugectx = decimal.Context(prec=15)
+def hugenum(valu):
+    '''
+    Return a decimal.Decimal with proper precision for use as a synapse hugenum.
+    '''
+    return decimal.Decimal(valu, context=hugectx)
+
 def vertup(vstr):
     '''
     Convert a version string to a tuple.

synapse/datamodel.py

@@ -495,6 +495,10 @@ def __init__(self):
         item = s_types.NodeProp(self, 'nodeprop', info, {})
         self.addBaseType(item)
 
+        info = {'doc': 'A potentially huge/tiny number. [x] <= 170141183460469231731687 with a fractional precision of 15 decimal digits.'}
+        item = s_types.HugeNum(self, 'hugenum', info, {})
+        self.addBaseType(item)
+
         # add the base universal properties...
         self.addUnivProp('seen', ('ival', {}), {
             'doc': 'The time interval for first/last observation of the node.',

synapse/lib/layer.py

@@ -200,6 +200,7 @@ async def getIden(self):
 STOR_TYPE_MINTIME = 21
 
 STOR_TYPE_FLOAT64 = 22
+STOR_TYPE_HUGENUM = 23
 
 # STOR_TYPE_TOMB      = ??
 # STOR_TYPE_FIXED     = ??
@@ -649,6 +650,66 @@ async def _liftIntRange(self, liftby, valu):
         for item in liftby.buidsByRange(pkeymin, pkeymax):
             yield item
 
+class StorTypeHugeNum(StorType):
+
+    def __init__(self, layr, stortype):
+        StorType.__init__(self, layr, STOR_TYPE_HUGENUM)
+        self.lifters.update({
+            '=': self._liftHugeEq,
+            '<': self._liftHugeLt,
+            '>': self._liftHugeGt,
+            '<=': self._liftHugeLe,
+            '>=': self._liftHugeGe,
+            'range=': self._liftHugeRange,
+        })
+
+        self.one = s_common.hugenum(1)
+        self.offset = s_common.hugenum(0x7fffffffffffffffffffffffffffffff)
+
+        self.zerobyts = b'\x00' * 16
+        self.fullbyts = b'\xff' * 16
+
+    def getHugeIndx(self, norm):
+        scaled = s_common.hugenum(norm).scaleb(15)
+        byts = int(scaled + self.offset).to_bytes(16, byteorder='big')
+        return byts
+
+    def indx(self, norm):
+        return (self.getHugeIndx(norm),)
+
+    async def _liftHugeEq(self, liftby, valu):
+        byts = self.getHugeIndx(valu)
+        for item in liftby.buidsByDups(byts):
+            yield item
+
+    async def _liftHugeGt(self, liftby, valu):
+        valu = s_common.hugenum(valu)
+        async for item in self._liftHugeGe(liftby, valu + self.one):
+            yield item
+
+    async def _liftHugeLt(self, liftby, valu):
+        valu = s_common.hugenum(valu)
+        async for item in self._liftHugeLe(liftby, valu - self.one):
+            yield item
+
+    async def _liftHugeGe(self, liftby, valu):
+        pkeymin = self.getHugeIndx(valu)
+        pkeymax = self.fullbyts
+        for item in liftby.buidsByRange(pkeymin, pkeymax):
+            yield item
+
+    async def _liftHugeLe(self, liftby, valu):
+        pkeymin = self.zerobyts
+        pkeymax = self.getHugeIndx(valu)
+        for item in liftby.buidsByRange(pkeymin, pkeymax):
+            yield item
+
+    async def _liftHugeRange(self, liftby, valu):
+        pkeymin = self.getHugeIndx(valu[0])
+        pkeymax = self.getHugeIndx(valu[1])
+        for item in liftby.buidsByRange(pkeymin, pkeymax):
+            yield item
+
 class StorTypeFloat(StorType):
     FloatPacker = struct.Struct('>d')
     fpack = FloatPacker.pack
@@ -975,6 +1036,7 @@ async def __anit__(self, layrinfo, dirn, nexsroot=None, allow_upstream=True):
             StorTypeTime(self), # STOR_TYPE_MINTIME
 
             StorTypeFloat(self, STOR_TYPE_FLOAT64, 8),
+            StorTypeHugeNum(self, STOR_TYPE_HUGENUM),
         ]
 
         self.editors = [

synapse/lib/types.py

@@ -614,6 +614,87 @@ def _normPyBytes(self, valu):
     (16, False): s_layer.STOR_TYPE_U128,
 }
 
+hugemax = 170141183460469231731687
+class HugeNum(Type):
+
+    _opt_defs = (
+        ('norm', True),
+    )
+
+    stortype = s_layer.STOR_TYPE_HUGENUM
+
+    def __init__(self, modl, name, info, opts):
+
+        Type.__init__(self, modl, name, info, opts)
+
+        self.setCmprCtor('>', self._ctorCmprGt)
+        self.setCmprCtor('<', self._ctorCmprLt)
+        self.setCmprCtor('>=', self._ctorCmprGe)
+        self.setCmprCtor('<=', self._ctorCmprLe)
+
+        self.storlifts.update({
+            '<': self._storLiftNorm,
+            '>': self._storLiftNorm,
+            '<=': self._storLiftNorm,
+            '>=': self._storLiftNorm,
+            'range=': self._storLiftRange,
+        })
+
+    def norm(self, valu):
+
+        huge = s_common.hugenum(valu)
+        if huge > hugemax:
+            mesg = f'Value ({valu}) is too large for hugenum.'
+            raise s_exc.BadTypeValu(mesg)
+
+        if abs(huge) > hugemax:
+            mesg = f'Value ({valu}) is too small for hugenum.'
+            raise s_exc.BadTypeValu(mesg)
+
+        if self.opts.get('norm'):
+            huge.normalize(), {}
+        return huge.to_eng_string(), {}
+
+    def _ctorCmprEq(self, text):
+        base = s_common.hugenum(text)
+        def cmpr(valu):
+            valu = s_common.hugenum(valu)
+            return valu == base
+        return cmpr
+
+    def _ctorCmprGt(self, text):
+        base = s_common.hugenum(text)
+        def cmpr(valu):
+            valu = s_common.hugenum(valu)
+            return valu > base
+        return cmpr
+
+    def _ctorCmprLt(self, text):
+        base = s_common.hugenum(text)
+        def cmpr(valu):
+            valu = s_common.hugenum(valu)
+            return valu < base
+        return cmpr
+
+    def _ctorCmprGe(self, text):
+        base = s_common.hugenum(text)
+        def cmpr(valu):
+            valu = s_common.hugenum(valu)
+            return valu >= base
+        return cmpr
+
+    def _ctorCmprLe(self, text):
+        base = s_common.hugenum(text)
+        def cmpr(valu):
+            valu = s_common.hugenum(valu)
+            return valu <= base
+        return cmpr
+
+    def _storLiftRange(self, cmpr, valu):
+        minv, minfo = self.norm(valu[0])
+        maxv, maxfo = self.norm(valu[1])
+        return ((cmpr, (minv, maxv), self.stortype),)
+
 class IntBase(Type):
 
     def __init__(self, modl, name, info, opts):

synapse/models/economic.py

@@ -1,4 +1,9 @@
+import synapse.common as s_common
+
+import synapse.lib.time as s_time
+import synapse.lib.types as s_types
 import synapse.lib.module as s_module
+import synapse.lib.grammar as s_grammar
 
 class EconModule(s_module.CoreModule):
 
@@ -34,6 +39,26 @@ def getModelDefs(self):
                 ('econ:acct:payment', ('guid', {}), {
                     'doc': 'A payment moving currency from one monetary instrument to another.'}),
 
+                ('econ:price', ('hugenum', {'norm': False}), {
+                    'doc': 'The amount of money expected, required, or given in payment for something',
+                    'ex': '2.20'}),
+
+                ('econ:currency', ('str', {'lower': True, 'strip': False}), {
+                    'doc': 'The name of a system of money in general use',
+                    'ex': '2.20'}),
+
+                ('econ:fin:exchange', ('guid', {}), {
+                    'doc': 'A financial exchange where securities are traded.'}),
+
+                ('econ:fin:security', ('guid', {}), {
+                    'doc': 'A financial security which is typically traded on an exchange.'}),
+
+                ('econ:fin:bar', ('guid', {}), {
+                    'doc': 'A sample of the open, close, high, low prices of a security in a specific time window'}),
+
+                ('econ:fin:tick', ('guid', {}), {
+                    'doc': 'A sample of the price of a security at a single moment in time'}),
+
                 # TODO currency / monetary units / crypto currency
                 # econ:acct:bill
                 # econ:goods econ:services
@@ -100,7 +125,12 @@ def getModelDefs(self):
 
                     ('campaign', ('ou:campaign', {}), {
                         'doc': 'The campaign that the purchase was in support of.'}),
-                    # TODO price
+
+                    ('price', ('econ:price', {}), {
+                        'doc': 'The econ:price of the purchase'}),
+
+                    ('currency', ('econ:currency', {}), {
+                        'doc': 'The econ:price of the purchase'}),
                 )),
 
                 ('econ:acquired', {}, (
@@ -134,6 +164,77 @@ def getModelDefs(self):
 
                     ('purchase', ('econ:purchase', {}), {
                         'doc': 'The purchase which the payment was paying for.'}),
+
+                    ('amount', ('econ:price', {}), {
+                        'doc': 'The amount of money transferred in the payment'}),
+
+                    ('currency', ('econ:currency', {}), {
+                        'doc': 'The currency of the payment'}),
+                )),
+
+                ('econ:fin:exchange', {}, (
+
+                    ('name', ('str', {'lower': True, 'strip': True}), {
+                        'doc': 'A simple name for the exchange',
+                        'ex': 'nasdaq'}),
+
+                    ('org', ('ou:org', {}), {
+                        'doc': 'The organization that operates the exchange'}),
+
+                    ('currency', ('econ:currency', {}), {
+                        'doc': 'The currency used for all transactions in the exchange',
+                        'ex': 'usd'}),
+                )),
+
+                ('econ:fin:security', {}, (
+
+                    ('exchange', ('econ:fin:exchange', {}), {
+                        'doc': 'The exchange on which the security is traded'}),
+
+                    ('ticker', ('str', {'lower': True, 'strip': True}), {
+                        'doc': 'The identifier for this security within the exchange'}),
+
+                    ('type', ('str', {'lower': True, 'strip': True}), {
+                        'doc': 'A user defined type such as stock, bond, option, future, or forex'}),
+
+                    ('price', ('econ:price', {}), {
+                        'doc': 'The last known/available price of the security'}),
+
+                    ('time', ('time', {}), {
+                        'doc': 'The time of the last know price sample'}),
+                )),
+
+                ('econ:fin:tick', {}, (
+
+                    ('security', ('econ:fin:security', {}), {
+                        'doc': 'The security measured by the tick'}),
+
+                    ('time', ('time', {}), {
+                        'doc': 'The time the price was sampled'}),
+
+                    ('price', ('econ:price', {}), {
+                        'doc': 'The price of the security at the time'}),
+                )),
+
+                ('econ:fin:bar', {}, (
+
+                    ('security', ('econ:fin:security', {}), {
+                        'doc': 'The security measured by the bar'}),
+
+                    ('ival', ('ival', {}), {
+                        'doc': 'The the interval of measurement'}),
+
+                    ('price:open', ('econ:price', {}), {
+                        'doc': 'The opening price of the security'}),
+
+                    ('price:close', ('econ:price', {}), {
+                        'doc': 'The closing price of the security'}),
+
+                    ('price:low', ('econ:price', {}), {
+                        'doc': 'The low price of the security'}),
+
+                    ('price:high', ('econ:price', {}), {
+                        'doc': 'The high price of the security'}),
                 )),
             ),
         }),)