From 47ff8879a01cfbd96b87177ab073d601cdae12d6 Mon Sep 17 00:00:00 2001 From: Philipp Plank Date: Wed, 31 Jul 2024 10:36:07 -0700 Subject: [PATCH 1/5] enable manual partitioning --- src/lava/magma/compiler/compiler.py | 13 ++++++++++--- .../compiler/subcompilers/py/pyproc_compiler.py | 3 ++- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/lava/magma/compiler/compiler.py b/src/lava/magma/compiler/compiler.py index 4a45b5fe5..e447b488c 100644 --- a/src/lava/magma/compiler/compiler.py +++ b/src/lava/magma/compiler/compiler.py @@ -247,6 +247,9 @@ def _compile_proc_groups( f"Cache {cache_dir}\n") return proc_builders, channel_map + # Get manual partitioning, if available + partitioning = self._compile_config.get("partitioning", None) + # Create the global ChannelMap that is passed between # SubCompilers to communicate about Channels between Processes. @@ -266,7 +269,8 @@ def _compile_proc_groups( subcompilers.append(pg_subcompilers) # Compile this ProcGroup. - self._compile_proc_group(pg_subcompilers, channel_map) + self._compile_proc_group(pg_subcompilers, channel_map, + partitioning) # Flatten the list of all SubCompilers. subcompilers = list(itertools.chain.from_iterable(subcompilers)) @@ -403,7 +407,8 @@ def _create_subcompilers( @staticmethod def _compile_proc_group( - subcompilers: ty.List[AbstractSubCompiler], channel_map: ChannelMap + subcompilers: ty.List[AbstractSubCompiler], channel_map: ChannelMap, + partitioning: ty.Dict[str, ty.Dict] ) -> None: """For a given list of SubCompilers that have been initialized with the Processes of a single ProcGroup, iterate through the compilation @@ -419,6 +424,8 @@ def _compile_proc_group( channel_map : ChannelMap The global ChannelMap that contains information about Channels between Processes. + partitioning: ty.Dict + Optional manual mapping dictionary used by ncproc compiler. """ channel_map_prev = None @@ -431,7 +438,7 @@ def _compile_proc_group( for subcompiler in subcompilers: # Compile the Processes registered with each SubCompiler and # update the ChannelMap. - channel_map = subcompiler.compile(channel_map) + channel_map = subcompiler.compile(channel_map, partitioning) @staticmethod def _extract_proc_builders( diff --git a/src/lava/magma/compiler/subcompilers/py/pyproc_compiler.py b/src/lava/magma/compiler/subcompilers/py/pyproc_compiler.py index 9c74c92d8..965e23409 100644 --- a/src/lava/magma/compiler/subcompilers/py/pyproc_compiler.py +++ b/src/lava/magma/compiler/subcompilers/py/pyproc_compiler.py @@ -89,7 +89,8 @@ def __init__( super().__init__(proc_group, compile_config) self._spike_io_counter_offset: Offset = Offset() - def compile(self, channel_map: ChannelMap) -> ChannelMap: + def compile(self, channel_map: ChannelMap, + partitioning: ty.Dict = {}) -> ChannelMap: return self._update_channel_map(channel_map) def __del__(self): From 88be52eddcdcaad88e3650df4cc7c81b5052f448 Mon Sep 17 00:00:00 2001 From: Philipp Plank Date: Wed, 31 Jul 2024 10:43:15 -0700 Subject: [PATCH 2/5] fix codacy complaint --- src/lava/magma/compiler/subcompilers/py/pyproc_compiler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lava/magma/compiler/subcompilers/py/pyproc_compiler.py b/src/lava/magma/compiler/subcompilers/py/pyproc_compiler.py index 965e23409..c5948a399 100644 --- a/src/lava/magma/compiler/subcompilers/py/pyproc_compiler.py +++ b/src/lava/magma/compiler/subcompilers/py/pyproc_compiler.py @@ -90,7 +90,7 @@ def __init__( self._spike_io_counter_offset: Offset = Offset() def compile(self, channel_map: ChannelMap, - partitioning: ty.Dict = {}) -> ChannelMap: + partitioning: ty.Dict = None) -> ChannelMap: return self._update_channel_map(channel_map) def __del__(self): From a6a56b8e618be0d30465822ba497f4a89d11605e Mon Sep 17 00:00:00 2001 From: Philipp Plank Date: Wed, 31 Jul 2024 11:07:58 -0700 Subject: [PATCH 3/5] fix unit tests --- tests/lava/magma/compiler/test_compiler.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/lava/magma/compiler/test_compiler.py b/tests/lava/magma/compiler/test_compiler.py index 0d4e6d07f..68c7dafae 100644 --- a/tests/lava/magma/compiler/test_compiler.py +++ b/tests/lava/magma/compiler/test_compiler.py @@ -391,7 +391,7 @@ def test_compile_proc_group_single_loop(self) -> None: subcompilers = [py_proc_compiler] # Call the method to be tested. - self.compiler._compile_proc_group(subcompilers, channel_map) + self.compiler._compile_proc_group(subcompilers, channel_map, None) # Check that it called compile() on every SubCompiler instance # exactly once. After that, the while loop should exit because the @@ -424,7 +424,8 @@ def test_compile_proc_group_multiple_loops(self) -> None: subcompilers = [py_proc_compiler] # Call the method to be tested. - self.compiler._compile_proc_group(subcompilers, channel_map) + self.compiler._compile_proc_group(subcompilers, channel_map, + None) # Check that it called compile() on every SubCompiler instance # exactly once. After that, the while loop should exit because the @@ -511,7 +512,7 @@ def test_compile_proc_groups(self) -> None: with py_patch: # Call the method to be tested. proc_builders, channel_map = self.compiler._compile_proc_groups( - proc_groups, channel_map + proc_groups, channel_map, None ) # There should be six Process Builders... From ee67e90fe5d9fd3449a2a59ede32854c15eb312b Mon Sep 17 00:00:00 2001 From: Philipp Plank Date: Thu, 1 Aug 2024 04:24:26 -0700 Subject: [PATCH 4/5] fixed unit tests --- tests/lava/magma/compiler/test_compiler.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/lava/magma/compiler/test_compiler.py b/tests/lava/magma/compiler/test_compiler.py index 68c7dafae..cda571c76 100644 --- a/tests/lava/magma/compiler/test_compiler.py +++ b/tests/lava/magma/compiler/test_compiler.py @@ -217,7 +217,8 @@ def create_patches( and the compile() method returns the given ChannelMap unchanged. .""" - def compile_return(channel_map: ChannelMap) -> ChannelMap: + def compile_return(channel_map: ChannelMap, + partitioning=None) -> ChannelMap: return channel_map py_patch = patch( @@ -397,7 +398,7 @@ def test_compile_proc_group_single_loop(self) -> None: # exactly once. After that, the while loop should exit because the # ChannelMap instance has not changed. for sc in subcompilers: - sc.compile.assert_called_once_with({}) + sc.compile.assert_called_once_with({}, None) def test_compile_proc_group_multiple_loops(self) -> None: """Test whether the correct methods are called on all objects when @@ -431,7 +432,7 @@ def test_compile_proc_group_multiple_loops(self) -> None: # exactly once. After that, the while loop should exit because the # ChannelMap instance has not changed. for sc in subcompilers: - sc.compile.assert_called_with({**channel_map1, **channel_map2}) + sc.compile.assert_called_with({**channel_map1, **channel_map2}, None) self.assertEqual(sc.compile.call_count, 3) def test_extract_proc_builders(self) -> None: @@ -512,7 +513,7 @@ def test_compile_proc_groups(self) -> None: with py_patch: # Call the method to be tested. proc_builders, channel_map = self.compiler._compile_proc_groups( - proc_groups, channel_map, None + proc_groups, channel_map ) # There should be six Process Builders... From f367b1a59ec289465ef81021127c3ff1741e72aa Mon Sep 17 00:00:00 2001 From: Philipp Plank Date: Thu, 1 Aug 2024 04:26:21 -0700 Subject: [PATCH 5/5] fixed linting --- tests/lava/magma/compiler/test_compiler.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/lava/magma/compiler/test_compiler.py b/tests/lava/magma/compiler/test_compiler.py index cda571c76..876def248 100644 --- a/tests/lava/magma/compiler/test_compiler.py +++ b/tests/lava/magma/compiler/test_compiler.py @@ -432,7 +432,8 @@ def test_compile_proc_group_multiple_loops(self) -> None: # exactly once. After that, the while loop should exit because the # ChannelMap instance has not changed. for sc in subcompilers: - sc.compile.assert_called_with({**channel_map1, **channel_map2}, None) + sc.compile.assert_called_with({**channel_map1, **channel_map2}, + None) self.assertEqual(sc.compile.call_count, 3) def test_extract_proc_builders(self) -> None: