Created floating pt and bit accurate Dense ProcModels + unit tests. F…

…ixes issues lava-nc#100 and lava-nc#111. (lava-nc#112)

src/lava/proc/dense/models.py

@@ -16,17 +16,95 @@
 @implements(proc=Dense, protocol=LoihiProtocol)
 @requires(CPU)
 @tag('floating_pt')
-class PyDenseModel(PyLoihiProcessModel):
+class PyDenseModelFloat(PyLoihiProcessModel):
+    """Implementation of Conn Process with Dense synaptic connections in
+    floating point precision. This short and simple ProcessModel can be used
+    for quick algorithmic prototyping, without engaging with the nuances of a
+    fixed point implementation.
+    """
+    s_in: PyInPort = LavaPyType(PyInPort.VEC_DENSE, bool, precision=1)
+    a_out: PyOutPort = LavaPyType(PyOutPort.VEC_DENSE, float)
+    a_buff: np.ndarray = LavaPyType(np.ndarray, float)
+    # weights is a 2D matrix of form (num_flat_output_neurons,
+    # num_flat_input_neurons)in C-order (row major).
+    weights: np.ndarray = LavaPyType(np.ndarray, float)
+    weight_exp: float = LavaPyType(float, float)
+    num_weight_bits: float = LavaPyType(float, float)
+    sign_mode: float = LavaPyType(float, float)
+
+    def run_spk(self):
+        # The a_out sent on a each timestep is a buffered value from dendritic
+        # accumulation at timestep t-1. This prevents deadlocking in
+        # networks with recurrent connectivity structures.
+        self.a_out.send(self.a_buff)
+        s_in = self.s_in.recv()
+        self.a_buff = self.weights[:, s_in].sum(axis=1)
+
+
+@implements(proc=Dense, protocol=LoihiProtocol)
+@requires(CPU)
+@tag('bit_accurate_loihi', 'fixed_pt')
+class PyDenseModelBitAcc(PyLoihiProcessModel):
+    """Implementation of Conn Process with Dense synaptic connections that is
+    bit-accurate with Loihi's hardware implementation of Dense, which means,
+    it mimics Loihi behaviour bit-by-bit.
+    """
+
     s_in: PyInPort = LavaPyType(PyInPort.VEC_DENSE, bool, precision=1)
     a_out: PyOutPort = LavaPyType(PyOutPort.VEC_DENSE, np.int32, precision=16)
-    # previously hidden var
+    a_buff: np.ndarray = LavaPyType(np.ndarray, np.int32, precision=16)
+    # weights is a 2D matrix of form (num_flat_output_neurons,
+    # num_flat_input_neurons) in C-order (row major).
     weights: np.ndarray = LavaPyType(np.ndarray, np.int32, precision=8)
+    weight_exp: np.ndarray = LavaPyType(np.ndarray, np.int32, precision=4)
+    num_weight_bits: np.ndarray = LavaPyType(np.ndarray, np.int32, precision=3)
+    sign_mode: np.ndarray = LavaPyType(np.ndarray, np.int32, precision=2)
+
+    def __init__(self):
+        super(PyDenseModelBitAcc, self).__init__()
+        # Flag to determine whether weights have already been scaled.
+        self.weights_set = False
+
+    def _set_wgts(self):
+        wgt_vals = np.copy(self.weights)
+
+        # Saturate the weights according to the sign_mode:
+        # 0 : null
+        # 1 : mixed
+        # 2 : excitatory
+        # 3 : inhibitory
+        mixed_idx = np.equal(self.sign_mode, 1).astype(np.int32)
+        excitatory_idx = np.equal(self.sign_mode, 2).astype(np.int32)
+        inhibitory_idx = np.equal(self.sign_mode, 3).astype(np.int32)
+
+        min_wgt = -2 ** 8 * (mixed_idx + inhibitory_idx)
+        max_wgt = (2 ** 8 - 1) * (mixed_idx + excitatory_idx)
+
+        saturated_wgts = np.clip(wgt_vals, min_wgt, max_wgt)
+
+        # Truncate least significant bits given sign_mode and num_wgt_bits.
+        num_truncate_bits = 8 - self.num_weight_bits + mixed_idx
+
+        truncated_wgts = np.left_shift(
+            np.right_shift(saturated_wgts, num_truncate_bits),
+            num_truncate_bits)
+
+        wgt_vals = truncated_wgts.astype(np.int32)
+        wgts_scaled = np.copy(wgt_vals)
+        self.weights_set = True
+        return wgts_scaled
 
     def run_spk(self):
+        # Since this Process has no learning, weights are assumed to be static
+        # and only require scaling on the first timestep of run_spk().
+        if not self.weights_set:
+            self.weights = self._set_wgts()
+        # The a_out sent on a each timestep is a buffered value from dendritic
+        # accumulation at timestep t-1. This prevents deadlocking in
+        # networks with recurrent connectivity structures.
+        self.a_out.send(self.a_buff)
         s_in = self.s_in.recv()
-        a_out = self.weights[:, s_in].sum(axis=1)
-        self.a_out.send(a_out)
-        self.a_out.flush()
-
-    def run_lrn(self):
-        pass
+        a_accum = self.weights[:, s_in].sum(axis=1)
+        self.a_buff = np.left_shift(a_accum,
+                                    self.weight_exp) if self.weight_exp > 0 \
+            else np.right_shift(a_accum, -self.weight_exp)

src/lava/proc/dense/process.py

@@ -2,22 +2,76 @@
 # SPDX-License-Identifier: BSD-3-Clause
 # See: https://spdx.org/licenses/
 
+import numpy as np
 from lava.magma.core.process.process import AbstractProcess
 from lava.magma.core.process.variable import Var
 from lava.magma.core.process.ports.ports import InPort, OutPort
 
 
 class Dense(AbstractProcess):
-    """Dense connections between neurons.
-    Realizes the following abstract behavior:
-    a_out = W * s_in
+    """Dense connections between neurons. Realizes the      following abstract
+    behavior: a_out = weights * s_in '
+
+    Parameters
+    ----------
+
+    weights:
+    2D Connection weight matrix of form (num_flat_output_neurons,
+    num_flat_input_neurons) in C-order (row major).
+
+    weight_exp:
+    Shared weight exponent used to
+    scale magnitude of weights, if needed. Mostly for fixed point
+    implementations. Unnecessary for floating point implementations.
+    Default value is 0.
+
+    num_weight_bits:
+    Shared weight width/precision used by weight. Mostly for
+    fixed point  implementations. Unnecessary for floating point
+    implementations.
+    Default is for weights to use full 8 bit precision.
+
+    sign_mode:
+    Shared indicator whether synapse is of 'null' (0),
+    ’mixed’ (1, default), ’excitatory’ (2) or  ’inhibitory’ (3) type. If
+    ’mixed’, the sign of the weight is included in the weight bits and
+    the fixed point weight used for inference is scaled by 2.
+    Unnecessary for floating point implementations.
+
+    In the fixed point implementation, weights are scaled according to the
+    following equations:
+    weights = weights * (2 ** w_scale)
+    w_scale =  8 - num_weight_bits + weight_exp + isMixed()
+
+    a_buff:
+    Circular buffer that stores output activations accumulated in current
+    timestep for future timesteps.
+
     """
 
+    # ToDo: (DR) Implement a ProcModel that supports synaptic delays. a_buff
+    # must then be adjusted to the length of the delay.
+
+    # ToDo: (DR) Revisit the implementation of w_scale so that less of this
+    # computation is exposed at the level of the Process.
+
     def __init__(self, **kwargs):
-        # super(AbstractProcess, self).__init__(kwargs)
-        # shape = kwargs.pop("shape")
         super().__init__(**kwargs)
         shape = kwargs.get("shape", (1, 1))
+        if len(shape) != 2:
+            raise AssertionError("Dense Process 'shape' expected a 2D tensor.")
+        weights = kwargs.pop("weights", np.zeros(shape=shape))
+        if len(np.shape(weights)) != 2:
+            raise AssertionError("Dense Process 'weights' expected a 2D "
+                                 "matrix.")
+        weight_exp = kwargs.pop("weight_exp", 0)
+        num_weight_bits = kwargs.pop("num_weight_bits", 8)
+        sign_mode = kwargs.pop("sign_mode", 1)
+
         self.s_in = InPort(shape=(shape[1],))
         self.a_out = OutPort(shape=(shape[0],))
-        self.weights = Var(shape=shape, init=kwargs.pop("weights", 0))
+        self.weights = Var(shape=shape, init=weights)
+        self.weight_exp = Var(shape=(1,), init=weight_exp)
+        self.num_weight_bits = Var(shape=(1,), init=num_weight_bits)
+        self.sign_mode = Var(shape=(1,), init=sign_mode)
+        self.a_buff = Var(shape=(shape[0],), init=0)