From c2afedaa8ad1636fee90294655a2a5966a3b3cf6 Mon Sep 17 00:00:00 2001 From: Danielle Rager <83376999+drager-intel@users.noreply.github.com> Date: Sun, 28 Nov 2021 15:00:07 -0800 Subject: [PATCH] Created floating pt and bit accurate Dense ProcModels + unit tests. Fixes issues #100 and #111. (#112) --- src/lava/proc/dense/models.py | 94 ++++- src/lava/proc/dense/process.py | 66 ++- tests/lava/proc/dense/__init__.py | 0 tests/lava/proc/dense/test_models.py | 553 ++++++++++++++++++++++++++ tests/lava/proc/dense/test_process.py | 47 +++ 5 files changed, 746 insertions(+), 14 deletions(-) create mode 100644 tests/lava/proc/dense/__init__.py create mode 100644 tests/lava/proc/dense/test_models.py create mode 100644 tests/lava/proc/dense/test_process.py diff --git a/src/lava/proc/dense/models.py b/src/lava/proc/dense/models.py index 6ea842f9a..f54f2c65f 100644 --- a/src/lava/proc/dense/models.py +++ b/src/lava/proc/dense/models.py @@ -16,17 +16,95 @@ @implements(proc=Dense, protocol=LoihiProtocol) @requires(CPU) @tag('floating_pt') -class PyDenseModel(PyLoihiProcessModel): +class PyDenseModelFloat(PyLoihiProcessModel): + """Implementation of Conn Process with Dense synaptic connections in + floating point precision. This short and simple ProcessModel can be used + for quick algorithmic prototyping, without engaging with the nuances of a + fixed point implementation. + """ + s_in: PyInPort = LavaPyType(PyInPort.VEC_DENSE, bool, precision=1) + a_out: PyOutPort = LavaPyType(PyOutPort.VEC_DENSE, float) + a_buff: np.ndarray = LavaPyType(np.ndarray, float) + # weights is a 2D matrix of form (num_flat_output_neurons, + # num_flat_input_neurons)in C-order (row major). + weights: np.ndarray = LavaPyType(np.ndarray, float) + weight_exp: float = LavaPyType(float, float) + num_weight_bits: float = LavaPyType(float, float) + sign_mode: float = LavaPyType(float, float) + + def run_spk(self): + # The a_out sent on a each timestep is a buffered value from dendritic + # accumulation at timestep t-1. This prevents deadlocking in + # networks with recurrent connectivity structures. + self.a_out.send(self.a_buff) + s_in = self.s_in.recv() + self.a_buff = self.weights[:, s_in].sum(axis=1) + + +@implements(proc=Dense, protocol=LoihiProtocol) +@requires(CPU) +@tag('bit_accurate_loihi', 'fixed_pt') +class PyDenseModelBitAcc(PyLoihiProcessModel): + """Implementation of Conn Process with Dense synaptic connections that is + bit-accurate with Loihi's hardware implementation of Dense, which means, + it mimics Loihi behaviour bit-by-bit. + """ + s_in: PyInPort = LavaPyType(PyInPort.VEC_DENSE, bool, precision=1) a_out: PyOutPort = LavaPyType(PyOutPort.VEC_DENSE, np.int32, precision=16) - # previously hidden var + a_buff: np.ndarray = LavaPyType(np.ndarray, np.int32, precision=16) + # weights is a 2D matrix of form (num_flat_output_neurons, + # num_flat_input_neurons) in C-order (row major). weights: np.ndarray = LavaPyType(np.ndarray, np.int32, precision=8) + weight_exp: np.ndarray = LavaPyType(np.ndarray, np.int32, precision=4) + num_weight_bits: np.ndarray = LavaPyType(np.ndarray, np.int32, precision=3) + sign_mode: np.ndarray = LavaPyType(np.ndarray, np.int32, precision=2) + + def __init__(self): + super(PyDenseModelBitAcc, self).__init__() + # Flag to determine whether weights have already been scaled. + self.weights_set = False + + def _set_wgts(self): + wgt_vals = np.copy(self.weights) + + # Saturate the weights according to the sign_mode: + # 0 : null + # 1 : mixed + # 2 : excitatory + # 3 : inhibitory + mixed_idx = np.equal(self.sign_mode, 1).astype(np.int32) + excitatory_idx = np.equal(self.sign_mode, 2).astype(np.int32) + inhibitory_idx = np.equal(self.sign_mode, 3).astype(np.int32) + + min_wgt = -2 ** 8 * (mixed_idx + inhibitory_idx) + max_wgt = (2 ** 8 - 1) * (mixed_idx + excitatory_idx) + + saturated_wgts = np.clip(wgt_vals, min_wgt, max_wgt) + + # Truncate least significant bits given sign_mode and num_wgt_bits. + num_truncate_bits = 8 - self.num_weight_bits + mixed_idx + + truncated_wgts = np.left_shift( + np.right_shift(saturated_wgts, num_truncate_bits), + num_truncate_bits) + + wgt_vals = truncated_wgts.astype(np.int32) + wgts_scaled = np.copy(wgt_vals) + self.weights_set = True + return wgts_scaled def run_spk(self): + # Since this Process has no learning, weights are assumed to be static + # and only require scaling on the first timestep of run_spk(). + if not self.weights_set: + self.weights = self._set_wgts() + # The a_out sent on a each timestep is a buffered value from dendritic + # accumulation at timestep t-1. This prevents deadlocking in + # networks with recurrent connectivity structures. + self.a_out.send(self.a_buff) s_in = self.s_in.recv() - a_out = self.weights[:, s_in].sum(axis=1) - self.a_out.send(a_out) - self.a_out.flush() - - def run_lrn(self): - pass + a_accum = self.weights[:, s_in].sum(axis=1) + self.a_buff = np.left_shift(a_accum, + self.weight_exp) if self.weight_exp > 0 \ + else np.right_shift(a_accum, -self.weight_exp) diff --git a/src/lava/proc/dense/process.py b/src/lava/proc/dense/process.py index 322fb9b5e..28b00e5ef 100644 --- a/src/lava/proc/dense/process.py +++ b/src/lava/proc/dense/process.py @@ -2,22 +2,76 @@ # SPDX-License-Identifier: BSD-3-Clause # See: https://spdx.org/licenses/ +import numpy as np from lava.magma.core.process.process import AbstractProcess from lava.magma.core.process.variable import Var from lava.magma.core.process.ports.ports import InPort, OutPort class Dense(AbstractProcess): - """Dense connections between neurons. - Realizes the following abstract behavior: - a_out = W * s_in + """Dense connections between neurons. Realizes the following abstract + behavior: a_out = weights * s_in ' + + Parameters + ---------- + + weights: + 2D Connection weight matrix of form (num_flat_output_neurons, + num_flat_input_neurons) in C-order (row major). + + weight_exp: + Shared weight exponent used to + scale magnitude of weights, if needed. Mostly for fixed point + implementations. Unnecessary for floating point implementations. + Default value is 0. + + num_weight_bits: + Shared weight width/precision used by weight. Mostly for + fixed point implementations. Unnecessary for floating point + implementations. + Default is for weights to use full 8 bit precision. + + sign_mode: + Shared indicator whether synapse is of 'null' (0), + ’mixed’ (1, default), ’excitatory’ (2) or ’inhibitory’ (3) type. If + ’mixed’, the sign of the weight is included in the weight bits and + the fixed point weight used for inference is scaled by 2. + Unnecessary for floating point implementations. + + In the fixed point implementation, weights are scaled according to the + following equations: + weights = weights * (2 ** w_scale) + w_scale = 8 - num_weight_bits + weight_exp + isMixed() + + a_buff: + Circular buffer that stores output activations accumulated in current + timestep for future timesteps. + """ + # ToDo: (DR) Implement a ProcModel that supports synaptic delays. a_buff + # must then be adjusted to the length of the delay. + + # ToDo: (DR) Revisit the implementation of w_scale so that less of this + # computation is exposed at the level of the Process. + def __init__(self, **kwargs): - # super(AbstractProcess, self).__init__(kwargs) - # shape = kwargs.pop("shape") super().__init__(**kwargs) shape = kwargs.get("shape", (1, 1)) + if len(shape) != 2: + raise AssertionError("Dense Process 'shape' expected a 2D tensor.") + weights = kwargs.pop("weights", np.zeros(shape=shape)) + if len(np.shape(weights)) != 2: + raise AssertionError("Dense Process 'weights' expected a 2D " + "matrix.") + weight_exp = kwargs.pop("weight_exp", 0) + num_weight_bits = kwargs.pop("num_weight_bits", 8) + sign_mode = kwargs.pop("sign_mode", 1) + self.s_in = InPort(shape=(shape[1],)) self.a_out = OutPort(shape=(shape[0],)) - self.weights = Var(shape=shape, init=kwargs.pop("weights", 0)) + self.weights = Var(shape=shape, init=weights) + self.weight_exp = Var(shape=(1,), init=weight_exp) + self.num_weight_bits = Var(shape=(1,), init=num_weight_bits) + self.sign_mode = Var(shape=(1,), init=sign_mode) + self.a_buff = Var(shape=(shape[0],), init=0) diff --git a/tests/lava/proc/dense/__init__.py b/tests/lava/proc/dense/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/lava/proc/dense/test_models.py b/tests/lava/proc/dense/test_models.py new file mode 100644 index 000000000..d828f18b2 --- /dev/null +++ b/tests/lava/proc/dense/test_models.py @@ -0,0 +1,553 @@ +# Copyright (C) 2021 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause +# See: https://spdx.org/licenses/ +import unittest +import numpy as np + +from lava.magma.core.decorator import implements, requires, tag +from lava.magma.core.model.py.model import PyLoihiProcessModel +from lava.magma.core.model.py.ports import PyOutPort, PyInPort +from lava.magma.core.model.py.type import LavaPyType +from lava.magma.core.process.ports.ports import OutPort, InPort +from lava.magma.core.process.process import AbstractProcess +from lava.magma.core.process.variable import Var +from lava.magma.core.resources import CPU +from lava.magma.core.run_configs import RunConfig +from lava.magma.core.run_conditions import RunSteps +from lava.magma.core.sync.protocols.loihi_protocol import LoihiProtocol +from lava.proc.dense.process import Dense + + +class DenseRunConfig(RunConfig): + """Run configuration selects appropriate Dense ProcessModel based on tag: + floating point precision or Loihi bit-accurate fixed-point precision""" + + def __init__(self, custom_sync_domains=None, select_tag='fixed_pt'): + super().__init__(custom_sync_domains=custom_sync_domains) + self.select_tag = select_tag + + def select(self, proc, proc_models): + for pm in proc_models: + if self.select_tag in pm.tags: + return pm + raise AssertionError("No legal ProcessModel found.") + + +class VecSendandRecvProcess(AbstractProcess): + """ + Process of a user-defined shape that sends an arbitrary vector + + Process also listens for incoming connections via InPort a_in. This + allows the test Process to validate that network behavior won't deadlock + in the presence of recurrent connections. + + Parameters + ---------- + shape: tuple, shape of the process + vec_to_send: np.ndarray, vector of spike values to send + send_at_times: np.ndarray, vector bools. Send the `vec_to_send` at times + when there is a True + """ + + def __init__(self, **kwargs): + super().__init__(**kwargs) + shape = kwargs.pop("shape", (1,)) + vec_to_send = kwargs.pop("vec_to_send") + send_at_times = kwargs.pop("send_at_times") + num_steps = kwargs.pop("num_steps", 1) + self.shape = shape + self.num_steps = num_steps + self.vec_to_send = Var(shape=shape, init=vec_to_send) + self.send_at_times = Var(shape=(num_steps,), init=send_at_times) + self.s_out = OutPort(shape=shape) + self.a_in = InPort(shape=shape) # enables recurrence test + + +class VecRecvProcess(AbstractProcess): + """ + Process that receives arbitrary vectors + + Parameters + ---------- + shape: tuple, shape of the process + """ + + def __init__(self, **kwargs): + super().__init__(**kwargs) + shape = kwargs.get("shape", (1,)) + self.shape = shape + self.s_in = InPort(shape=(shape[1],)) + self.spk_data = Var(shape=shape, init=0) # This Var expands with time + + +@implements(proc=VecSendandRecvProcess, protocol=LoihiProtocol) +@requires(CPU) +# need the following tag to discover the ProcessModel using DenseRunConfig +@tag('floating_pt') +class PyVecSendModelFloat(PyLoihiProcessModel): + s_out: PyOutPort = LavaPyType(PyOutPort.VEC_DENSE, bool, precision=1) + a_in: PyInPort = LavaPyType(PyInPort.VEC_DENSE, float) + vec_to_send: np.ndarray = LavaPyType(np.ndarray, bool, precision=1) + send_at_times: np.ndarray = LavaPyType(np.ndarray, bool, precision=1) + + def run_spk(self): + """ + Send `spikes_to_send` if current time-step requires it + """ + self.a_in.recv() + + if self.send_at_times[self.current_ts - 1]: + self.s_out.send(self.vec_to_send) + else: + self.s_out.send(np.zeros_like(self.vec_to_send)) + + +@implements(proc=VecSendandRecvProcess, protocol=LoihiProtocol) +@requires(CPU) +# need the following tag to discover the ProcessModel using DenseRunConfig +@tag('fixed_pt') +class PyVecSendModelFixed(PyLoihiProcessModel): + s_out: PyOutPort = LavaPyType(PyOutPort.VEC_DENSE, bool, precision=1) + a_in: PyInPort = LavaPyType(PyInPort.VEC_DENSE, np.int32, precision=16) + vec_to_send: np.ndarray = LavaPyType(np.ndarray, bool, precision=1) + send_at_times: np.ndarray = LavaPyType(np.ndarray, bool, precision=1) + + def run_spk(self): + """ + Send `spikes_to_send` if current time-step requires it + """ + self.a_in.recv() + + if self.send_at_times[self.current_ts - 1]: + self.s_out.send(self.vec_to_send) + else: + self.s_out.send(np.zeros_like(self.vec_to_send)) + + +@implements(proc=VecRecvProcess, protocol=LoihiProtocol) +@requires(CPU) +# need the following tag to discover the ProcessModel using DenseRunConfig +@tag('floating_pt') +class PySpkRecvModelFloat(PyLoihiProcessModel): + s_in: PyInPort = LavaPyType(PyInPort.VEC_DENSE, bool, precision=1) + spk_data: np.ndarray = LavaPyType(np.ndarray, float) + + def run_spk(self): + """Receive spikes and store in an internal variable""" + spk_in = self.s_in.recv() + self.spk_data[self.current_ts - 1, :] = spk_in + + +@implements(proc=VecRecvProcess, protocol=LoihiProtocol) +@requires(CPU) +# need the following tag to discover the ProcessModel using DenseRunConfig +@tag('fixed_pt') +class PySpkRecvModelFixed(PyLoihiProcessModel): + s_in: PyInPort = LavaPyType(PyInPort.VEC_DENSE, bool, precision=1) + spk_data: np.ndarray = LavaPyType(np.ndarray, int, precision=1) + + def run_spk(self): + """Receive spikes and store in an internal variable""" + spk_in = self.s_in.recv() + self.spk_data[self.current_ts - 1, :] = spk_in + + +class TestDenseProcessModelFloat(unittest.TestCase): + """Tests for floating point ProcessModels of Dense""" + + def test_float_pm_buffer(self): + """ + Tests floating point Dense ProcessModel connectivity and temporal + dynamics. All input 'neurons' from the VecSendandRcv fire + once at time t=4, and only 1 connection weight + in the Dense Process is non-zero. The non-zero + connection should have an activation of 1 at timestep t=5. + """ + shape = (3, 4) + num_steps = 6 + # Set up external input to emulate every neuron spiking once on + # timestep 4 + vec_to_send = np.ones((shape[1],), dtype=float) + send_at_times = np.repeat(False, (num_steps,)) + send_at_times[3] = True + sps = VecSendandRecvProcess(shape=(shape[1],), num_steps=num_steps, + vec_to_send=vec_to_send, + send_at_times=send_at_times) + # Set up Dense Process with a single non-zero connection weight at + # entry [2,2] of the connectivity mat. + weights = np.zeros(shape, dtype=float) + weights[2, 2] = 1 + dense = Dense(shape=shape, weights=weights) + # Receive neuron spikes + spr = VecRecvProcess(shape=(num_steps, shape[0])) + sps.s_out.connect(dense.s_in) + dense.a_out.connect(spr.s_in) + # Configure execution and run + rcnd = RunSteps(num_steps=num_steps) + rcfg = DenseRunConfig(select_tag='floating_pt') + dense.run(condition=rcnd, run_cfg=rcfg) + # Gather spike data and stop + spk_data_through_run = spr.spk_data.get() + dense.stop() + # Gold standard for the test + # a_out will be equal to 1 at timestep 5, because the dendritic + # accumulators work on inputs from the previous timestep. + expected_spk_data = np.zeros((num_steps, shape[0])) + expected_spk_data[4, 2] = 1. + self.assertTrue(np.all(expected_spk_data == spk_data_through_run)) + + def test_float_pm_fan_in(self): + """ + Tests floating point Dense ProcessModel dendritic accumulation + behavior when the fan-in to a receiving neuron is greater than 1. + """ + shape = (3, 4) + num_steps = 6 + # Set up external input to emulate every neuron spiking once on + # timestep 4 + vec_to_send = np.ones((shape[1],), dtype=float) + send_at_times = np.repeat(False, (num_steps,)) + send_at_times[3] = True + sps = VecSendandRecvProcess(shape=(shape[1],), num_steps=num_steps, + vec_to_send=vec_to_send, + send_at_times=send_at_times) + # Set up a Dense Process where all input layer neurons project to a + # single output layer neuron. + weights = np.zeros(shape, dtype=float) + weights[2, :] = [2, -3, 4, -5] + dense = Dense(shape=shape, weights=weights) + # Receive neuron spikes + spr = VecRecvProcess(shape=(num_steps, shape[0])) + sps.s_out.connect(dense.s_in) + dense.a_out.connect(spr.s_in) + # Configure execution and run + rcnd = RunSteps(num_steps=num_steps) + rcfg = DenseRunConfig(select_tag='floating_pt') + dense.run(condition=rcnd, run_cfg=rcfg) + # Gather spike data and stop + spk_data_through_run = spr.spk_data.get() + dense.stop() + # Gold standard for the test + # Expected behavior is that a_out corresponding to output + # neuron 3 will be equal to -2=2-3+4-5 at timestep 5. + expected_spk_data = np.zeros((num_steps, shape[0])) + expected_spk_data[4, 2] = -2 + self.assertTrue(np.all(expected_spk_data == spk_data_through_run)) + + def test_float_pm_fan_out(self): + """ + Tests floating point Dense ProcessModel dendritic accumulation + behavior when the fan-out of a projecting neuron is greater than 1. + """ + shape = (3, 4) + num_steps = 6 + # Set up external input to emulate every neuron spiking once on + # timestep t=4. + vec_to_send = np.ones((shape[1],), dtype=float) + send_at_times = np.repeat(False, (num_steps,)) + send_at_times[3] = True + sps = VecSendandRecvProcess(shape=(shape[1],), num_steps=num_steps, + vec_to_send=vec_to_send, + send_at_times=send_at_times) + # Set up a Dense Process where a single input layer neuron projects to + # all output layer neurons. + weights = np.zeros(shape, dtype=float) + weights[:, 2] = [3, 4, 5] + dense = Dense(shape=shape, weights=weights) + # Receive neuron spikes + spr = VecRecvProcess(shape=(num_steps, shape[0])) + sps.s_out.connect(dense.s_in) + dense.a_out.connect(spr.s_in) + # Configure execution and run + rcnd = RunSteps(num_steps=num_steps) + rcfg = DenseRunConfig(select_tag='floating_pt') + dense.run(condition=rcnd, run_cfg=rcfg) + # Gather spike data and stop + spk_data_through_run = spr.spk_data.get() + dense.stop() + # Gold standard for the test + # Expected behavior is that a_out corresponding to output + # neurons 1-3 will be equal to 3, 4, and 5, respectively, at timestep 5. + expected_spk_data = np.zeros((num_steps, shape[0])) + expected_spk_data[4, :] = [3, 4, 5] + self.assertTrue(np.all(expected_spk_data == spk_data_through_run)) + + def test_float_pm_recurrence(self): + """ + Tests that floating Dense ProcessModel has non-blocking dynamics for + recurrent connectivity architectures. + """ + shape = (3, 3) + num_steps = 6 + # Set up external input to emulate every neuron spiking once on + # timestep 4. + vec_to_send = np.ones((shape[1],), dtype=float) + send_at_times = np.repeat(True, (num_steps,)) + sps = VecSendandRecvProcess(shape=(shape[1],), num_steps=num_steps, + vec_to_send=vec_to_send, + send_at_times=send_at_times) + # Set up Dense Process with fully connected recurrent connectivity + # architecture + weights = np.ones(shape, dtype=float) + dense = Dense(shape=shape, weights=weights) + # Receive neuron spikes + sps.s_out.connect(dense.s_in) + dense.a_out.connect(sps.a_in) + # Configure execution and run + rcnd = RunSteps(num_steps=num_steps) + rcfg = DenseRunConfig(select_tag='floating_pt') + dense.run(condition=rcnd, run_cfg=rcfg) + dense.stop() + + +class TestDenseProcessModelFixed(unittest.TestCase): + """Tests for fixed-point, ProcessModels of Dense, which are bit-accurate + with Loihi hardware""" + + def test_bitacc_pm_fan_out_excitatory(self): + """ + Tests fixed-point Dense ProcessModel dendritic accumulation + behavior when the fan-out of a projecting neuron is greater than 1 + and all connections are excitatory (sign_mode = 2). + """ + shape = (3, 4) + num_steps = 6 + # Set up external input to emulate every neuron spiking once on + # timestep 4. + vec_to_send = np.ones((shape[1],), dtype=float) + send_at_times = np.repeat(False, (num_steps,)) + send_at_times[3] = True + sps = VecSendandRecvProcess(shape=(shape[1],), num_steps=num_steps, + vec_to_send=vec_to_send, + send_at_times=send_at_times) + # Set up Dense Process in which a single input neuron projects to all + # output neurons. + weights = np.zeros(shape, dtype=float) + weights[:, 2] = [0.5, 300, 40] + dense = Dense(shape=shape, weights=weights, sign_mode=2) + # Receive neuron spikes + spr = VecRecvProcess(shape=(num_steps, shape[0])) + sps.s_out.connect(dense.s_in) + dense.a_out.connect(spr.s_in) + # Configure execution and run + rcnd = RunSteps(num_steps=num_steps) + rcfg = DenseRunConfig(select_tag='fixed_pt') + dense.run(condition=rcnd, run_cfg=rcfg) + # Gather spike data and stop + spk_data_through_run = spr.spk_data.get() + dense.stop() + # Gold standard for the test + # Expected behavior is that a_out corresponding to output + # neurons 1-3 will be equal to 0, 255, and 40, respectively, + # at timestep 5, because a_out can only have integer values between 0 + # and 255. + expected_spk_data = np.zeros((num_steps, shape[0])) + expected_spk_data[4, :] = [0, 255, 40] + self.assertTrue(np.all(expected_spk_data == spk_data_through_run)) + + def test_bitacc_pm_fan_out_mixed_sign(self): + """ + Tests fixed-point Dense ProcessModel dendritic accumulation + behavior when the fan-out of a projecting neuron is greater than 1 + and connections are both excitatory and inhibitory (sign_mode = 1). + When using mixed sign weights and full 8 bit weight precision, + a_out can take even values from -256 to 254. + """ + shape = (3, 4) + num_steps = 6 + # Set up external input to emulate every neuron spiking once on + # timestep 4. + vec_to_send = np.ones((shape[1],), dtype=float) + send_at_times = np.repeat(False, (num_steps,)) + send_at_times[3] = True + sps = VecSendandRecvProcess(shape=(shape[1],), num_steps=num_steps, + vec_to_send=vec_to_send, + send_at_times=send_at_times) + # Set up Dense Process in which a single input neuron projects to all + # output neurons with both excitatory and inhibitory weights. + weights = np.zeros(shape, dtype=float) + weights[:, 2] = [300, -300, 39] + dense = Dense(shape=shape, weights=weights, sign_mode=1) + # Receive neuron spikes + spr = VecRecvProcess(shape=(num_steps, shape[0])) + sps.s_out.connect(dense.s_in) + dense.a_out.connect(spr.s_in) + # Configure execution and run + rcnd = RunSteps(num_steps=num_steps) + rcfg = DenseRunConfig(select_tag='fixed_pt') + dense.run(condition=rcnd, run_cfg=rcfg) + # Gather spike data and stop + spk_data_through_run = spr.spk_data.get() + dense.stop() + # Gold standard for the test + # Expected behavior is that a_out corresponding to output + # neurons 1-3 will be equal to 254, -256, and 38, respectively, + # at timestep 5, because a_out can only have even values between -256 + # and 254. + expected_spk_data = np.zeros((num_steps, shape[0])) + expected_spk_data[4, :] = [254, -256, 38] + self.assertTrue(np.all(expected_spk_data == spk_data_through_run)) + + def test_bitacc_pm_fan_out_weight_exp(self): + """ + Tests fixed-point Dense ProcessModel dendritic accumulation + behavior when the fan-out of a projecting neuron is greater than 1 + , connections are both excitatory and inhibitory (sign_mode = 1), + and weight_exp = 1. + When using mixed sign weights, full 8 bit weight precision, + and weight_exp = 1, a_out can take even values from -512 to 508. + As a result of setting weight_exp = 1, the expected a_out result is 2x + that of the previous unit test. + """ + + shape = (3, 4) + num_steps = 6 + # Set up external input to emulate every neuron spiking once on + # timestep 4. + vec_to_send = np.ones((shape[1],), dtype=float) + send_at_times = np.repeat(False, (num_steps,)) + send_at_times[3] = True + sps = VecSendandRecvProcess(shape=(shape[1],), num_steps=num_steps, + vec_to_send=vec_to_send, + send_at_times=send_at_times) + # Set up Dense Process in which all input neurons project to a single + # output neuron with mixed sign connection weights. + weights = np.zeros(shape, dtype=float) + weights[:, 2] = [300, -300, 39] + # Set weight_exp = 1. This affects weight scaling. + dense = Dense(shape=shape, weights=weights, weight_exp=1) + # Receive neuron spikes + spr = VecRecvProcess(shape=(num_steps, shape[0])) + sps.s_out.connect(dense.s_in) + dense.a_out.connect(spr.s_in) + # Configure execution and run + rcnd = RunSteps(num_steps=num_steps) + rcfg = DenseRunConfig(select_tag='fixed_pt') + dense.run(condition=rcnd, run_cfg=rcfg) + # Gather spike data and stop + spk_data_through_run = spr.spk_data.get() + dense.stop() + # Gold standard for the test + # Expected behavior is that a_out corresponding to output + # neurons 1-3 will be equal to 508, -512, and 76, respectively, + # at timestep 5, because a_out can only have values between -512 + # and 508 such that a_out % 4 = 0. + expected_spk_data = np.zeros((num_steps, shape[0])) + expected_spk_data[4, :] = [508, -512, 76] + self.assertTrue(np.all(expected_spk_data == spk_data_through_run)) + + def test_bitacc_pm_fan_out_weight_precision(self): + """ + Tests fixed-point Dense ProcessModel dendritic accumulation + behavior when the fan-out of a projecting neuron is greater than 1 + , connections are both excitatory and inhibitory (sign_mode = 1), + and num_weight_bits = 7. + When using mixed sign weights and 7 bit weight precision, + a_out can take values from -256 to 252 such that a_out % 4 = 0. + """ + + shape = (3, 4) + num_steps = 6 + # Set up external input to emulate every neuron spiking once on + # timestep 4. + vec_to_send = np.ones((shape[1],), dtype=float) + send_at_times = np.repeat(False, (num_steps,)) + send_at_times[3] = True + sps = VecSendandRecvProcess(shape=(shape[1],), num_steps=num_steps, + vec_to_send=vec_to_send, + send_at_times=send_at_times) + # Set up Dense Process in which all input neurons project to a single + # output neuron with mixed sign connection weights. + weights = np.zeros(shape, dtype=float) + weights[:, 2] = [300, -300, 39] + # Set num_weight_bits = 7. This affects weight scaling. + dense = Dense(shape=shape, weights=weights, num_weight_bits=7) + # Receive neuron spikes + spr = VecRecvProcess(shape=(num_steps, shape[0])) + sps.s_out.connect(dense.s_in) + dense.a_out.connect(spr.s_in) + # Configure execution and run + rcnd = RunSteps(num_steps=num_steps) + rcfg = DenseRunConfig(select_tag='fixed_pt') + dense.run(condition=rcnd, run_cfg=rcfg) + # Gather spike data and stop + spk_data_through_run = spr.spk_data.get() + dense.stop() + # Gold standard for the test + # Expected behavior is that a_out corresponding to output + # neurons 1-3 will be equal to 252, -256, and 36, respectively, + # at timestep 5, because a_out can only have values between -256 + # and 252 such that a_out % 4 = 0. + expected_spk_data = np.zeros((num_steps, shape[0])) + expected_spk_data[4, :] = [252, -256, 36] + self.assertTrue(np.all(expected_spk_data == spk_data_through_run)) + + def test_bitacc_pm_fan_in_mixed_sign(self): + """ + Tests fixed-point Dense ProcessModel dendritic accumulation + behavior when the fan-in of a receiving neuron is greater than 1 + and connections are both excitatory and inhibitory (sign_mode = 1). + When using mixed sign weights and full 8 bit weight precision, + a_out can take even values from -256 to 254. + """ + shape = (3, 4) + num_steps = 6 + # Set up external input to emulate every neuron spiking once on + # timestep 4. + vec_to_send = np.ones((shape[1],), dtype=float) + send_at_times = np.repeat(False, (num_steps,)) + send_at_times[3] = True + sps = VecSendandRecvProcess(shape=(shape[1],), num_steps=num_steps, + vec_to_send=vec_to_send, + send_at_times=send_at_times) + # Set up Dense Process in which all input layer neurons project to a + # single output layer neuron with both excitatory and inhibitory + # weights. + weights = np.zeros(shape, dtype=float) + weights[2, :] = [300, -300, 39, -0.4] + dense = Dense(shape=shape, weights=weights, sign_mode=1) + # Receive neuron spikes + spr = VecRecvProcess(shape=(num_steps, shape[0])) + sps.s_out.connect(dense.s_in) + dense.a_out.connect(spr.s_in) + # Configure execution and run + rcnd = RunSteps(num_steps=num_steps) + rcfg = DenseRunConfig(select_tag='fixed_pt') + dense.run(condition=rcnd, run_cfg=rcfg) + # Gather spike data and stop + spk_data_through_run = spr.spk_data.get() + dense.stop() + # Gold standard for the test + # Expected behavior is that a_out corresponding to output + # neuron 3 will be equal to 36=254-256+38-0 at timestep 5, because + # weights can only have even values between -256 and 254. + expected_spk_data = np.zeros((num_steps, shape[0])) + expected_spk_data[4, 2] = 36 + self.assertTrue(np.all(expected_spk_data == spk_data_through_run)) + + def test_bitacc_pm_recurrence(self): + """ + Tests that bit accurate Dense ProcessModel has non-blocking dynamics for + recurrent connectivity architectures. + """ + shape = (3, 3) + num_steps = 6 + # Set up external input to emulate every neuron spiking once on + # timestep 4. + vec_to_send = np.ones((shape[1],), dtype=float) + send_at_times = np.repeat(True, (num_steps,)) + sps = VecSendandRecvProcess(shape=(shape[1],), num_steps=num_steps, + vec_to_send=vec_to_send, + send_at_times=send_at_times) + # Set up Dense Process with fully connected recurrent connectivity + # architecture. + weights = np.ones(shape, dtype=float) + dense = Dense(shape=shape, weights=weights) + # Receive neuron spikes + sps.s_out.connect(dense.s_in) + dense.a_out.connect(sps.a_in) + # Configure execution and run + rcnd = RunSteps(num_steps=num_steps) + rcfg = DenseRunConfig(select_tag='fixed_pt') + dense.run(condition=rcnd, run_cfg=rcfg) + dense.stop() diff --git a/tests/lava/proc/dense/test_process.py b/tests/lava/proc/dense/test_process.py new file mode 100644 index 000000000..74d95515f --- /dev/null +++ b/tests/lava/proc/dense/test_process.py @@ -0,0 +1,47 @@ +# Copyright (C) 2021 Intel Corporation +# SPDX-License-Identifier: BSD-3-Clause +# See: https://spdx.org/licenses/ + +import unittest +import numpy as np +from lava.proc.dense.process import Dense + + +class TestConnProcess(unittest.TestCase): + """Tests for Dense class""" + + def test_init(self): + """Tests instantiation of Dense""" + shape = (100, 200) + weights = np.random.randint(100, size=shape) + weight_exp = 2 + num_weight_bits = 7 + sign_mode = 1 + + conn = Dense(shape=shape, weights=weights, weight_exp=weight_exp, + num_weight_bits=num_weight_bits, sign_mode=sign_mode) + + self.assertEqual(np.shape(conn.weights.init), shape) + self.assertIsNone( + np.testing.assert_array_equal(conn.weights.init, weights)) + self.assertEqual(conn.weight_exp.init, weight_exp) + self.assertEqual(conn.num_weight_bits.init, num_weight_bits) + self.assertEqual(conn.sign_mode.init, sign_mode) + + def test_no_in_args(self): + """Tests instantiation of Dense with no input arguments""" + conn = Dense() + self.assertEqual(np.shape(conn.weights.init), (1, 1)) + + def test_input_validation_shape(self): + """Tests input validation on the dimensions of 'shape'. (Must be 2D.)""" + shape = (100, 200, 300) + with self.assertRaises(AssertionError): + Dense(shape=shape) + + def test_input_validation_weights(self): + """Tests input validation on the dimensions of 'weights'. (Must be + 2D.)""" + weights = np.random.randint(100, size=(2, 3, 4)) + with self.assertRaises(AssertionError): + Dense(weights=weights)