From 650c1fc0503a8cd706567cf11f94ae5fcaece1bc Mon Sep 17 00:00:00 2001 From: "Risbud, Sumedh" Date: Wed, 14 Jun 2023 17:13:50 -0700 Subject: [PATCH 1/5] Explicitly keep track of all Processes in the Executable to make sure all Processes are assigned the Runtime Signed-off-by: Risbud, Sumedh --- src/lava/magma/compiler/compiler.py | 3 +++ src/lava/magma/compiler/executable.py | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/lava/magma/compiler/compiler.py b/src/lava/magma/compiler/compiler.py index 1a3beec26..9937fe801 100644 --- a/src/lava/magma/compiler/compiler.py +++ b/src/lava/magma/compiler/compiler.py @@ -128,6 +128,8 @@ def compile( # ProcGroups. proc_group_digraph = ProcGroupDiGraphs(process, run_cfg) proc_groups: ty.List[ProcGroup] = proc_group_digraph.get_proc_groups() + # Get a flattened list of all AbstractProcesses + process_list = list(itertools.chain.from_iterable(proc_groups)) channel_map = ChannelMap.from_proc_groups(proc_groups) proc_builders, channel_map = self._compile_proc_groups( proc_groups, channel_map @@ -161,6 +163,7 @@ def compile( # Package all Builders and NodeConfigs into an Executable. executable = Executable( + process_list, proc_builders, channel_builders, node_configs, diff --git a/src/lava/magma/compiler/executable.py b/src/lava/magma/compiler/executable.py index e1d325615..42d290357 100644 --- a/src/lava/magma/compiler/executable.py +++ b/src/lava/magma/compiler/executable.py @@ -33,6 +33,7 @@ class Executable: # py_builders: ty.Dict[AbstractProcess, NcProcessBuilder] # c_builders: ty.Dict[AbstractProcess, CProcessBuilder] # nc_builders: ty.Dict[AbstractProcess, PyProcessBuilder] + process_list: ty.List[AbstractProcess] # All leaf processes, flat list. proc_builders: ty.Dict[AbstractProcess, 'AbstractProcessBuilder'] channel_builders: ty.List[ChannelBuilderMp] node_configs: ty.List[NodeConfig] @@ -43,5 +44,5 @@ class Executable: ty.Iterable[AbstractChannelBuilder]] = None def assign_runtime_to_all_processes(self, runtime): - for p in self.proc_builders.keys(): + for p in self.process_list: p.runtime = runtime From 1e1e5a220d8c0883e85f87fc032761c6cadcd851 Mon Sep 17 00:00:00 2001 From: "Risbud, Sumedh" Date: Wed, 14 Jun 2023 17:14:58 -0700 Subject: [PATCH 2/5] A utility to measure the size of a python object recursively. Useful for example, to measure the size of NxBoard object after compilation Signed-off-by: Risbud, Sumedh --- src/lava/utils/profiler.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/lava/utils/profiler.py b/src/lava/utils/profiler.py index f62141424..f29f290bc 100644 --- a/src/lava/utils/profiler.py +++ b/src/lava/utils/profiler.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: BSD-3-Clause # See: https://spdx.org/licenses/ +import sys import warnings import typing as ty from lava.magma.core.run_configs import RunConfig, Loihi2HwCfg @@ -17,6 +18,27 @@ "Currently no profiler is available.") +def get_pyobj_size(obj, seen=None): + """Recursively finds size of objects""" + size = sys.getsizeof(obj) + if seen is None: + seen = set() + obj_id = id(obj) + if obj_id in seen: + return 0 + # Important mark as seen *before* entering recursion to gracefully handle + # self-referential objects + seen.add(obj_id) + if isinstance(obj, dict): + size += sum([get_pyobj_size(v, seen) for v in obj.values()]) + size += sum([get_pyobj_size(k, seen) for k in obj.keys()]) + elif hasattr(obj, '__dict__'): + size += get_pyobj_size(obj.__dict__, seen) + elif hasattr(obj, '__iter__') and not isinstance(obj, (str, bytes, bytearray)): + size += sum([get_pyobj_size(i, seen) for i in obj]) + return size + + class Profiler: """Base class for profiling execution time, energy and other metrics on different resources. Depending on the computing From 66a16dee54b2557d3f65cd480c97ee807dc20478 Mon Sep 17 00:00:00 2001 From: "Risbud, Sumedh" Date: Thu, 15 Jun 2023 16:33:27 -0700 Subject: [PATCH 3/5] Removed the utility to estimate Python object size Signed-off-by: Risbud, Sumedh --- src/lava/utils/profiler.py | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/src/lava/utils/profiler.py b/src/lava/utils/profiler.py index f29f290bc..0a35b2c4b 100644 --- a/src/lava/utils/profiler.py +++ b/src/lava/utils/profiler.py @@ -18,27 +18,6 @@ "Currently no profiler is available.") -def get_pyobj_size(obj, seen=None): - """Recursively finds size of objects""" - size = sys.getsizeof(obj) - if seen is None: - seen = set() - obj_id = id(obj) - if obj_id in seen: - return 0 - # Important mark as seen *before* entering recursion to gracefully handle - # self-referential objects - seen.add(obj_id) - if isinstance(obj, dict): - size += sum([get_pyobj_size(v, seen) for v in obj.values()]) - size += sum([get_pyobj_size(k, seen) for k in obj.keys()]) - elif hasattr(obj, '__dict__'): - size += get_pyobj_size(obj.__dict__, seen) - elif hasattr(obj, '__iter__') and not isinstance(obj, (str, bytes, bytearray)): - size += sum([get_pyobj_size(i, seen) for i in obj]) - return size - - class Profiler: """Base class for profiling execution time, energy and other metrics on different resources. Depending on the computing From c890ca737b4478bc5b33dad4e967072bd4f0f501 Mon Sep 17 00:00:00 2001 From: "Risbud, Sumedh" Date: Thu, 15 Jun 2023 16:54:22 -0700 Subject: [PATCH 4/5] Fixed Runtime unittest to accommodate passing an explicit ProcessList to the Executable Signed-off-by: Risbud, Sumedh --- tests/lava/magma/runtime/test_runtime.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/lava/magma/runtime/test_runtime.py b/tests/lava/magma/runtime/test_runtime.py index a70563638..6be322f45 100644 --- a/tests/lava/magma/runtime/test_runtime.py +++ b/tests/lava/magma/runtime/test_runtime.py @@ -27,7 +27,8 @@ def test_runtime_creation(self): def test_executable_node_config_assertion(self): """Tests runtime constructions with expected constraints""" - exe: Executable = Executable(proc_builders={}, + exe: Executable = Executable(process_list=[], + proc_builders={}, channel_builders=[], node_configs=[], sync_domains=[]) @@ -45,7 +46,8 @@ def test_executable_node_config_assertion(self): f"Expected type {expected_type} doesn't match {(type(runtime2))}") runtime2.stop() - exe1: Executable = Executable(proc_builders={}, + exe1: Executable = Executable(process_list=[], + proc_builders={}, channel_builders=[], node_configs=[], sync_domains=[]) From 93b33b56daff74c76a0b9c82dfd88995c63fc1d8 Mon Sep 17 00:00:00 2001 From: "Risbud, Sumedh" Date: Thu, 15 Jun 2023 16:55:42 -0700 Subject: [PATCH 5/5] Removed unused import Signed-off-by: Risbud, Sumedh --- src/lava/utils/profiler.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/lava/utils/profiler.py b/src/lava/utils/profiler.py index 0a35b2c4b..f62141424 100644 --- a/src/lava/utils/profiler.py +++ b/src/lava/utils/profiler.py @@ -2,7 +2,6 @@ # SPDX-License-Identifier: BSD-3-Clause # See: https://spdx.org/licenses/ -import sys import warnings import typing as ty from lava.magma.core.run_configs import RunConfig, Loihi2HwCfg