Migrations for legacy and now illegal default link label _return

aiida/backends/djsite/db/migrations/0043_default_link_label.py

@@ -0,0 +1,45 @@
+# -*- coding: utf-8 -*-
+###########################################################################
+# Copyright (c), The AiiDA team. All rights reserved.                     #
+# This file is part of the AiiDA code.                                    #
+#                                                                         #
+# The code is hosted on GitHub at https://github.com/aiidateam/aiida-core #
+# For further information on the license, see the LICENSE.txt file        #
+# For further information please visit http://www.aiida.net               #
+###########################################################################
+# pylint: disable=invalid-name,too-few-public-methods
+"""Update all link labels with the value `_return` which is the legacy default single link label.
+
+The old process functions used to use `_return` as the default link label, however, since labels that start or end with
+and underscore are illegal because they are used for namespacing.
+"""
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+from __future__ import absolute_import
+
+# Remove when https://github.com/PyCQA/pylint/issues/1931 is fixed
+# pylint: disable=no-name-in-module,import-error
+from django.db import migrations
+from aiida.backends.djsite.db.migrations import upgrade_schema_version
+
+REVISION = '1.0.43'
+DOWN_REVISION = '1.0.42'
+
+
+class Migration(migrations.Migration):
+    """Migrate."""
+
+    dependencies = [
+        ('db', '0042_prepare_schema_reset'),
+    ]
+
+    operations = [
+        migrations.RunSQL(
+            sql=r"""
+                UPDATE db_dblink SET label='result' WHERE label = '_return';
+                """,
+            reverse_sql=''
+        ),
+        upgrade_schema_version(REVISION, DOWN_REVISION)
+    ]

aiida/backends/djsite/db/migrations/__init__.py

@@ -23,7 +23,7 @@ class DeserializationException(AiidaException):
     pass
 
 
-LATEST_MIGRATION = '0042_prepare_schema_reset'
+LATEST_MIGRATION = '0043_default_link_label'
 
 
 def _update_schema_version(version, apps, schema_editor):

aiida/backends/djsite/db/subtests/migrations/test_migrations_0043_default_link_label.py

@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+###########################################################################
+# Copyright (c), The AiiDA team. All rights reserved.                     #
+# This file is part of the AiiDA code.                                    #
+#                                                                         #
+# The code is hosted on GitHub at https://github.com/aiidateam/aiida-core #
+# For further information on the license, see the LICENSE.txt file        #
+# For further information please visit http://www.aiida.net               #
+###########################################################################
+# pylint: disable=import-error,no-name-in-module,invalid-name
+"""Tests for the migrations of legacy process attributes."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from aiida.backends.djsite.db.subtests.migrations.test_migrations_common import TestMigrations
+
+
+class TestSealUnsealedProcessesMigration(TestMigrations):
+    """Test the migration that performs a data migration of legacy `JobCalcState`."""
+
+    migrate_from = '0042_prepare_schema_reset'
+    migrate_to = '0043_default_link_label'
+
+    def setUpBeforeMigration(self):
+        node_process = self.DbNode(
+            node_type='process.calculation.calcjob.CalcJobNode.',
+            user_id=self.default_user.id,
+        )
+        node_process.save()
+        self.node_process_id = node_process.id
+
+        node_data = self.DbNode(
+            node_type='data.dict.Dict.',
+            user_id=self.default_user.id,
+        )
+        node_data.save()
+        self.node_data_id = node_data.id
+
+        link = self.DbLink(input=node_data, output=node_process, type='input', label='_return')
+        link.save()
+
+    def test_data_migrated(self):
+        """Verify that the link label has been renamed."""
+        node = self.load_node(self.node_data_id)
+        link = self.DbLink.objects.get(input=node)
+        self.assertEqual(link.label, 'result')

aiida/backends/djsite/db/subtests/migrations/test_migrations_common.py

@@ -60,6 +60,7 @@ def setUp(self):
         # Reset session after the migration
         sa.get_scoped_session().close()
 
+        self.DbLink = self.apps.get_model('db', 'DbLink')
         self.DbNode = self.apps.get_model('db', 'DbNode')
         self.DbUser = self.apps.get_model('db', 'DbUser')
         self.DbUser.objects.all().delete()

aiida/backends/sqlalchemy/migrations/versions/118349c10896_default_link_label.py

@@ -0,0 +1,49 @@
+# -*- coding: utf-8 -*-
+###########################################################################
+# Copyright (c), The AiiDA team. All rights reserved.                     #
+# This file is part of the AiiDA code.                                    #
+#                                                                         #
+# The code is hosted on GitHub at https://github.com/aiidateam/aiida-core #
+# For further information on the license, see the LICENSE.txt file        #
+# For further information please visit http://www.aiida.net               #
+###########################################################################
+# pylint: disable=invalid-name
+"""Update all link labels with the value `_return` which is the legacy default single link label.
+
+The old process functions used to use `_return` as the default link label, however, since labels that start or end with
+and underscore are illegal because they are used for namespacing.
+
+Revision ID: 118349c10896
+Revises: 91b573400be5
+Create Date: 2019-11-21 09:43:45.006053
+
+"""
+from __future__ import division
+from __future__ import print_function
+from __future__ import absolute_import
+
+# pylint: disable=no-member,no-name-in-module,import-error
+from alembic import op
+from sqlalchemy.sql import text
+
+# revision identifiers, used by Alembic.
+revision = '118349c10896'
+down_revision = '91b573400be5'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    """Migrations for the upgrade."""
+    conn = op.get_bind()
+
+    # The old process functions used to use `_return` as the default link label, however, since labels that start or end
+    # with and underscore are illegal.
+    statement = text("""
+        UPDATE db_dblink SET label='result' WHERE label = '_return';
+    """)
+    conn.execute(statement)
+
+
+def downgrade():
+    """Migrations for the downgrade."""

aiida/backends/sqlalchemy/tests/test_migrations.py

@@ -1610,3 +1610,58 @@ def test_data_migrated(self):
 
             finally:
                 session.close()
+
+
+class TestDefaultLinkLabelMigration(TestMigrationsSQLA):
+    """Test the migration that performs a data migration of legacy default link labels."""
+
+    migrate_from = '91b573400be5'
+    migrate_to = '118349c10896'
+
+    def setUpBeforeMigration(self):
+        from sqlalchemy.orm import Session  # pylint: disable=import-error,no-name-in-module
+
+        DbLink = self.get_auto_base().classes.db_dblink  # pylint: disable=invalid-name
+        DbNode = self.get_auto_base().classes.db_dbnode  # pylint: disable=invalid-name
+        DbUser = self.get_auto_base().classes.db_dbuser  # pylint: disable=invalid-name
+
+        with sa.ENGINE.begin() as connection:
+            try:
+                session = Session(connection.engine)
+
+                user = DbUser(email='{}@aiida.net'.format(self.id()))
+                session.add(user)
+                session.commit()
+
+                node_process = DbNode(node_type='process.calculation.calcjob.CalcJobNode.', user_id=user.id)
+                node_data = DbNode(node_type='data.dict.Dict.', user_id=user.id)
+                link = DbLink(input_id=node_data.id, output_id=node_process.id, type='input', label='_return')
+
+                session.add(node_process)
+                session.add(node_data)
+                session.add(link)
+                session.commit()
+
+                self.node_process_id = node_process.id
+                self.node_data_id = node_data.id
+                self.link_id = link.id
+            except Exception:
+                session.rollback()
+                raise
+            finally:
+                session.close()
+
+    def test_data_migrated(self):
+        """Verify that the attributes for process node have been deleted and `_sealed` has been changed to `sealed`."""
+        from sqlalchemy.orm import Session  # pylint: disable=import-error,no-name-in-module
+
+        DbLink = self.get_auto_base().classes.db_dblink  # pylint: disable=invalid-name
+
+        with sa.ENGINE.begin() as connection:
+            try:
+                session = Session(connection.engine)
+                link = session.query(DbLink).filter(DbLink.id == self.link_id).one()
+                self.assertEqual(link.label, 'result')
+
+            finally:
+                session.close()

aiida/backends/tests/__init__.py

@@ -27,6 +27,7 @@
             'aiida.backends.djsite.db.subtests.migrations.test_migrations_0038_data_migration_legacy_job_calculations',
             'aiida.backends.djsite.db.subtests.migrations.test_migrations_0040_data_migration_legacy_process_attributes',
             'aiida.backends.djsite.db.subtests.migrations.test_migrations_0041_seal_unsealed_processes',
+            'aiida.backends.djsite.db.subtests.migrations.test_migrations_0043_default_link_label',
         ],
     },
     BACKEND_SQLA: {
@@ -162,6 +163,7 @@
         'tools.importexport.migration.v04_to_v05': ['aiida.backends.tests.tools.importexport.migration.test_v04_to_v05'],
         'tools.importexport.migration.v05_to_v06': ['aiida.backends.tests.tools.importexport.migration.test_v05_to_v06'],
         'tools.importexport.migration.v06_to_v07': ['aiida.backends.tests.tools.importexport.migration.test_v06_to_v07'],
+        'tools.importexport.migration.v07_to_v08': ['aiida.backends.tests.tools.importexport.migration.test_v07_to_v08'],
         'tools.importexport.orm.attributes': ['aiida.backends.tests.tools.importexport.orm.test_attributes'],
         'tools.importexport.orm.calculations': ['aiida.backends.tests.tools.importexport.orm.test_calculations'],
         'tools.importexport.orm.codes': ['aiida.backends.tests.tools.importexport.orm.test_codes'],

aiida/backends/tests/tools/importexport/migration/test_migration.py

@@ -479,3 +479,58 @@ def test_v06_to_newest(self, temp_dir):
         builder.append(orm.CalculationNode, tag='parent')
         builder.append(orm.RemoteData, with_incoming='parent')
         self.assertGreater(len(builder.all()), 0)
+
+    @with_temp_dir
+    def test_v07_to_newest(self, temp_dir):
+        """Test migration of exported files from v0.7 to newest export version"""
+        input_file = get_archive_file('export_v0.7_manual.aiida', **self.external_archive)
+        output_file = os.path.join(temp_dir, 'output_file.aiida')
+
+        # Perform the migration
+        migrate_archive(input_file, output_file)
+        metadata, _ = get_json_files(output_file)
+        verify_metadata_version(metadata, version=newest_version)
+
+        # Load the migrated file
+        import_data(output_file, silent=True)
+
+        # Do the necessary checks
+        self.assertEqual(orm.QueryBuilder().append(orm.Node).count(), self.node_count + 2)
+
+        # Verify that CalculationNodes have non-empty attribute dictionaries
+        builder = orm.QueryBuilder().append(orm.CalculationNode)
+        for [calculation] in builder.iterall():
+            self.assertIsInstance(calculation.attributes, dict)
+            self.assertNotEqual(len(calculation.attributes), 0)
+
+        # Verify that the StructureData nodes maintained their (same) label, cell, and kinds
+        builder = orm.QueryBuilder().append(orm.StructureData)
+        self.assertEqual(
+            builder.count(),
+            self.struct_count,
+            msg='There should be {} StructureData, instead {} were/was found'.format(
+                self.struct_count, builder.count()
+            )
+        )
+        for structures in builder.all():
+            structure = structures[0]
+            self.assertEqual(structure.label, self.known_struct_label)
+            self.assertEqual(structure.cell, self.known_cell)
+
+        builder = orm.QueryBuilder().append(orm.StructureData, project=['attributes.kinds'])
+        for [kinds] in builder.iterall():
+            self.assertEqual(len(kinds), len(self.known_kinds))
+            for kind in kinds:
+                self.assertIn(kind, self.known_kinds, msg="Kind '{}' not found in: {}".format(kind, self.known_kinds))
+
+        # Check that there is a StructureData that is an input of a CalculationNode
+        builder = orm.QueryBuilder()
+        builder.append(orm.StructureData, tag='structure')
+        builder.append(orm.CalculationNode, with_incoming='structure')
+        self.assertGreater(len(builder.all()), 0)
+
+        # Check that there is a RemoteData that is the output of a CalculationNode
+        builder = orm.QueryBuilder()
+        builder.append(orm.CalculationNode, tag='parent')
+        builder.append(orm.RemoteData, with_incoming='parent')
+        self.assertGreater(len(builder.all()), 0)