NodeLinkManager support dot separated namespace attributes retrieving (…

…#4625)

The `NodeLinksManager` is used by the ``inputs`` and ``outputs``
attributes of the ``ProcessNode`` class to allow users to quickly access
input and output nodes by their link label. Nested namespaces in link
labels are converted to double underscores before it is stored in the
database because it can only store a flat string. This is an
implementation detail, however, and the user should not have to know
about it and should be able to use the nested namespace. However, up
till now, one had to pass the link label as stored in the database, i.e.

    node.inputs.nested__sub__namespace

After this commit, it is now possible to use the more intuitive

    node.inputs.nested.sub.namespace

For backwards compatibility, the old flat link is still supported but
will emit a deprecation warning.

Co-authored-by: Sebastiaan Huber <[email protected]>

aiida/orm/utils/managers.py

@@ -8,11 +8,11 @@
 # For further information please visit http://www.aiida.net               #
 ###########################################################################
 """
-Contain utility classes for "managers", i.e., classes that act allow
+Contain utility classes for "managers", i.e., classes that allow
 to access members of other classes via TAB-completable attributes
 (e.g. the class underlying `calculation.inputs` to allow to do `calculation.inputs.<label>`).
 """
-
+from aiida.common import AttributeDict
 from aiida.common.links import LinkType
 from aiida.common.exceptions import NotExistent, NotExistentAttributeError, NotExistentKeyError
 
@@ -45,23 +45,63 @@ def __init__(self, node, link_type, incoming):
         self._link_type = link_type
         self._incoming = incoming
 
+    def _construct_attribute_dict(self, incoming):
+        """Construct an attribute dict from all links of the node, recreating nested namespaces from flat link labels.
+
+        :param incoming: if True, inspect incoming links, otherwise inspect outgoing links.
+        """
+        if incoming:
+            links = self._node.get_incoming(link_type=self._link_type)
+        else:
+            links = self._node.get_outgoing(link_type=self._link_type)
+
+        return AttributeDict(links.nested())
+
     def _get_keys(self):
         """Return the valid link labels, used e.g. to make getattr() work"""
-        if self._incoming:
-            node_attributes = self._node.get_incoming(link_type=self._link_type).all_link_labels()
-        else:
-            node_attributes = self._node.get_outgoing(link_type=self._link_type).all_link_labels()
-        return node_attributes
+        attribute_dict = self._construct_attribute_dict(self._incoming)
+
+        return attribute_dict.keys()
 
     def _get_node_by_link_label(self, label):
-        """
-        Return the linked node with a given link label
+        """Return the linked node with a given link label.
+
+        Nested namespaces in link labels get represented by double underscores in the database. Up until now, the link
+        manager didn't automatically unroll these again into nested namespaces and so a user was forced to pass the link
+        with double underscores to dereference the corresponding node. For example, when used with the ``inputs``
+        attribute of a ``ProcessNode`` one had to do:
+
+            node.inputs.nested__sub__namespace
+
+        Now it is possible to do
 
-        :param label: the link label connecting the current node to the node to get
+            node.inputs.nested.sub.namespace
+
+        which is more intuitive since the double underscore replacement is just for the database and the user shouldn't
+        even have to know about it. For compatibility we support the old version a bit longer and it will emit a
+        deprecation warning.
+
+        :param label: the link label connecting the current node to the node to get.
         """
-        if self._incoming:
-            return self._node.get_incoming(link_type=self._link_type).get_node_by_label(label)
-        return self._node.get_outgoing(link_type=self._link_type).get_node_by_label(label)
+        attribute_dict = self._construct_attribute_dict(self._incoming)
+        try:
+            node = attribute_dict[label]
+        except KeyError as exception:
+            if '__' in label:
+                import functools
+                import warnings
+                from aiida.common.warnings import AiidaDeprecationWarning
+                warnings.warn(
+                    'dereferencing nodes with links containing double underscores is deprecated, simply replace '
+                    'the double underscores with a single dot instead. For example: \n'
+                    '`self.inputs.some__label` can be written as `self.inputs.some.label` instead.\n'
+                    'Support for double underscores will be removed in the future.', AiidaDeprecationWarning
+                )  # pylint: disable=no-member
+                namespaces = label.split('__')
+                return functools.reduce(lambda d, namespace: d.get(namespace), namespaces, attribute_dict)
+            raise NotExistent from exception
+
+        return node
 
     def __dir__(self):
         """
@@ -81,12 +121,14 @@ def __getattr__(self, name):
         """
         try:
             return self._get_node_by_link_label(label=name)
-        except NotExistent:
+        except NotExistent as exception:
             # Note: in order for TAB-completion to work, we need to raise an exception that also inherits from
             # `AttributeError`, so that `getattr(node.inputs, 'some_label', some_default)` returns `some_default`.
             # Otherwise, the exception is not caught by `getattr` and is propagated, instead of returning the default.
             prefix = 'input' if self._incoming else 'output'
-            raise NotExistentAttributeError(f"Node<{self._node.pk}> does not have an {prefix} with link label '{name}'")
+            raise NotExistentAttributeError(
+                f"Node<{self._node.pk}> does not have an {prefix} with link label '{name}'"
+            ) from exception
 
     def __getitem__(self, name):
         """
@@ -96,12 +138,14 @@ def __getitem__(self, name):
         """
         try:
             return self._get_node_by_link_label(label=name)
-        except NotExistent:
+        except NotExistent as exception:
             # Note: in order for this class to behave as a dictionary, we raise an exception that also inherits from
             # `KeyError` - in this way, users can use the standard construct `try/except KeyError` and this will behave
             # like a standard dictionary.
             prefix = 'input' if self._incoming else 'output'
-            raise NotExistentKeyError(f"Node<{self._node.pk}> does not have an {prefix} with link label '{name}'")
+            raise NotExistentKeyError(
+                f"Node<{self._node.pk}> does not have an {prefix} with link label '{name}'"
+            ) from exception
 
     def __str__(self):
         """Return a string representation of the manager"""
@@ -173,5 +217,5 @@ def __getitem__(self, name):
         """
         try:
             return self._node.get_attribute(name)
-        except AttributeError as err:
-            raise KeyError(str(err))
+        except AttributeError as exception:
+            raise KeyError(str(exception)) from exception

tests/engine/test_process.py

@@ -68,6 +68,8 @@ def test_namespaced_process(self):
         # Check that the link of the process node has the correct link name
         self.assertTrue('some__name__space__a' in proc.node.get_incoming().all_link_labels())
         self.assertEqual(proc.node.get_incoming().get_node_by_label('some__name__space__a'), 5)
+        self.assertEqual(proc.node.inputs.some.name.space.a, 5)
+        self.assertEqual(proc.node.inputs['some']['name']['space']['a'], 5)
 
 
 class ProcessStackTest(Process):

tests/orm/utils/test_managers.py

@@ -9,12 +9,11 @@
 ###########################################################################
 """Tests for the various node managers (.inputs, .outputs, .dict, ...)."""
 # pylint: disable=unused-argument
-
 import pytest
 
 from aiida import orm
 from aiida.common.exceptions import NotExistent, NotExistentAttributeError, NotExistentKeyError
-from aiida.common import LinkType
+from aiida.common import LinkType, AttributeDict
 
 
 def test_dot_dict_manager(clear_database_before_test):
@@ -57,6 +56,8 @@ def test_link_manager(clear_database_before_test):
     inp1.store()
     inp2 = orm.Data()
     inp2.store()
+    inp3 = orm.Data()
+    inp3.store()
 
     # Create calc with inputs
     calc = orm.CalculationNode()
@@ -135,3 +136,38 @@ def test_link_manager(clear_database_before_test):
     # Must raise a KeyError
     with pytest.raises(KeyError):
         _ = calc.outputs['NotExistentLabel']
+
+
+def test_link_manager_with_nested_namespaces(clear_database_before_test):
+    """Test the ``LinkManager`` works with nested namespaces."""
+    inp1 = orm.Data()
+    inp1.store()
+
+    calc = orm.CalculationNode()
+    calc.add_incoming(inp1, link_type=LinkType.INPUT_CALC, link_label='nested__sub__namespace')
+    calc.store()
+
+    # Attach outputs
+    out1 = orm.Data()
+    out1.add_incoming(calc, link_type=LinkType.CREATE, link_label='nested__sub__namespace')
+    out1.store()
+
+    # Check that the recommended way of dereferencing works
+    assert calc.inputs.nested.sub.namespace.uuid == inp1.uuid
+    assert calc.outputs.nested.sub.namespace.uuid == out1.uuid
+
+    # Leafs will return an ``AttributeDict`` instance
+    assert isinstance(calc.outputs.nested.sub, AttributeDict)
+
+    # Check the legacy way still works
+    with pytest.warns(Warning):
+        assert calc.inputs.nested__sub__namespace.uuid == inp1.uuid
+        assert calc.outputs.nested__sub__namespace.uuid == out1.uuid
+
+    # Must raise a AttributeError, otherwise tab competion will not work
+    with pytest.raises(AttributeError):
+        _ = calc.outputs.nested.not_existent
+
+    # Must raise a KeyError
+    with pytest.raises(KeyError):
+        _ = calc.outputs.nested['not_existent']