⬆️ Update to aiida-core v1.4.2 (#48)

This commit updates the aiida-core version to 1.4.2

It also introduces the generation/use of a constraint file,
to ensure that plugin installs do not break the aiida-core dependency requirements,
and also adds more testing of the virtualenv consistency (using pip check).

Lastly, it updates the ansible/molecule version pinnings, which have recently been updated.

.ansible-lint

@@ -1,4 +1,6 @@
-warn_list:  # or 'skip_list' to silence them completely
+# use `# noqa xxx` at the end of a line, to ignore a particular error
+# or add to the warn_list, to ignore for the whole project
+warn_list:
   - '106'  # Role name {} does not match ``^[a-z][a-z0-9_]+$`` pattern
   - '208'  # File permissions unset or incorrect
   - '305'  # Use shell only when shell functionality is required

.gitignore

@@ -1,4 +1,5 @@
 *.swp
+.DS_Store
 .galaxy_install_info
 .vscode/
 .tox/

defaults/main.yml

@@ -1,16 +1,19 @@
-aiida_version: "1.3.0"
+aiida_version: "1.4.2"
 aiida_tag: v{{ aiida_version }}
 aiida_extras:
 - rest
 - docs
 - atomic_tools
-- testing
 - notebook
 aiida_data_folder: "${HOME}/.local/share/aiida"
 aiida_templates_folder: "${HOME}/.local/share/aiida"
 aiida_source_folder: "${HOME}/src"
 aiida_examples_folder: "${HOME}"
 aiida_venv: "${HOME}/.virtualenvs/aiida"
+aiida_constraints_file: "{{ aiida_data_folder }}/constraints.txt"
+# these are additional to those extracted from aiida-core and its extras
+aiida_constraints:
+- "nbconvert<6.0"   # required by notebook 5.7.10
 
 # profile
 aiida_profile_name: generic
@@ -58,15 +61,15 @@ aiida_components:
 # Note: If possible, these versions should coincide with the ones in
 # https://github.com/aiidalab/aiidalab-metapkg/blob/master/requirements.txt
 aiida_plugin_versions:
-  aiida_cp2k: "1.1.0"
+  aiida_cp2k: "1.2.0"
   aiida_fleur: "1.1.0"
   aiida_bigdft: "0.2.1a2"
   # aiida_gudhi: "0.1.0a3"
   # aiida_phtools: "0.1.0a1"
-  aiida_quantumespresso: "3.1.0"
+  aiida_quantumespresso: "3.2.0"
   # aiida_raspa: "1.0.0a2"
   aiida_siesta: "1.1.0"
-  aiida_yambo: "1.1.1"
+  aiida_yambo: "1.1.3"
   aiida_wannier90: "2.0.1"
   aiida_wannier90_workflows: "1.0.1"
 

files/aiida-core-constraints.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python
+"""Script to convert aiida setup.json to pip constraints file.
+
+Usage::
+
+    python aiida-core-constraints.py path/to/setup.json extras1 extras2 ...
+"""
+import json
+from pathlib import Path
+import sys
+
+def remove_extra(req):
+    """Extras are not allowed in constraints files, e.g. dep[extra]>0.1 -> dep>0.1 """
+    if "[" not in req or "]" not in req:
+        return req
+    start, end = req.split("[", 1)
+    return start + end.split("]", 1)[1]
+
+
+def main(path, *extras):
+    path = Path(path)
+    data = json.loads(path.read_text("utf8"))
+
+    output = ["# aiida-core requirements: v" + data["version"]]
+    # we can't add this when using the file as a constraint file for actually installing aiida-core
+    # output.append("aiida-core==" + data["version"])
+    output.extend([remove_extra(req) for req in data["install_requires"]])
+
+    for extra in extras:
+        output.append("# " + extra)
+        output.extend([remove_extra(req) for req in data["extras_require"][extra]])
+
+    print("\n".join(output))
+
+if __name__ == "__main__":
+    assert len(sys.argv) > 1, "JSON path required"
+    main(sys.argv[1], *sys.argv[2:])

molecule/default/converge.yml

@@ -4,6 +4,7 @@
 
   vars:
     ansible_python_interpreter: /usr/bin/python3
+    aiida_venv: "${HOME}/.virtualenvs/aiida"
 
   tasks:
   - name: Create dummy pw executable
@@ -28,3 +29,8 @@
         folder: "/usr/bin"
         executable: "pw.x"
         plugin: quantumespresso.pw
+
+  post_tasks:
+  - name: run pip check
+    command: "{{ aiida_venv }}/bin/pip check --no-color"
+    changed_when: false

molecule/default/verify.yml

@@ -20,6 +20,16 @@
       - item.name in aiida_pps.stdout
     with_items: "{{ aiida_pseudopotentials }}"
 
+  - name: print aiida-core version
+    shell: "{{ aiida_venv }}/bin/pip show aiida-core"
+    changed_when: false
+    register: aiida_pip_show
+
+  - name: check aiida-core version
+    assert:
+      that:
+      - aiida_version in aiida_pip_show.stdout
+
   - name: check plugin versions
     include_tasks: tests/test_plugins.yml
     with_dict: "{{ aiida_plugin_versions }}"

requirements.txt

@@ -1,5 +1,5 @@
 # for running
-ansible~=2.9
+ansible~=2.10.0
 # for testing
-docker
-molecule~=3.0
+molecule[docker]~=3.1.0
+docker~=4.2

tasks/aiida-core.yml

@@ -6,6 +6,24 @@
     version: "{{ aiida_tag }}"
   register: git_checkout
 
+- name: Copy pip constrain generation script to folder
+  copy:
+    src: aiida-core-constraints.py
+    dest: "{{ aiida_source_folder }}/aiida-core"
+
+- name: Create pip constraints file contents
+  command: >
+    /usr/bin/python3 '{{ aiida_source_folder }}/aiida-core/aiida-core-constraints.py'
+    {{ aiida_source_folder }}/aiida-core/setup.json
+    {{ aiida_extras | join(' ') }}
+  changed_when: false
+  register: constraints
+
+- name: Write pip constraints file
+  copy:
+    dest: "{{ aiida_constraints_file }}"
+    content: "{{ constraints.stdout }}\n# additional\n{{ aiida_constraints | join('\n') }}"
+
 - name: Install AiiDA git code
   pip:
     name:
@@ -14,6 +32,7 @@
     # According to https://github.com/ansible/ansible/issues/52275
     virtualenv_command: /usr/bin/python3 -m venv
     editable: true
+    extra_args: "-c {{ aiida_constraints_file }}"
   register: aiida_core_install
   notify: reentry scan
   # this guard is necessary because, for some reason, installs with

tasks/aiida-prepare.yml

@@ -75,9 +75,9 @@
   pip:
     name:
     - pip~=20.0
-    - setuptools~=36.2
+    - setuptools
     - wheel
-    - numpy==1.17.4
+    - numpy  # required by pymatgen
     virtualenv: "{{ aiida_venv }}"
     # According to: https://github.com/ansible/ansible/issues/52275
     virtualenv_command: /usr/bin/python3 -m venv

tasks/main.yml

@@ -13,47 +13,47 @@
 - name: Get info on current user
   include_role:
     name: marvel-nccr.current_user
-  tags: always
+  tags: [always]
   when: current_user is undefined
 
 # apt and python packages
 - include_tasks: aiida-deps-rhel.yml
-  tags: aiida_deps
+  tags: [aiida_deps]
   when: ansible_os_family|lower == 'redhat'
 - include_tasks: aiida-deps-debian.yml
-  tags: aiida_deps
+  tags: [aiida_deps]
   when: ansible_os_family|lower == 'debian'
 
 # Start and setup Postgresql and RabbitMQ, create folders
 - import_tasks: aiida-prepare.yml
-  tags: aiida, aiida_prepare
+  tags: [aiida, aiida_prepare]
 
 # install and setup AiiDA, including profile etc.
 - import_tasks: aiida-core.yml
-  tags: aiida,aiida_core
+  tags: [aiida, aiida_core]
 - import_tasks: aiida-daemon.yml
   tags: aiida_daemon,aiida_core
 - import_tasks: aiida-rest.yml
-  tags: aiida_rest,aiida_core
+  tags: [aiida_rest, aiida_core]
 
 # setup computers
 - include_tasks: computers-setup.yml
-  tags: aiida_computers,aiida_core
+  tags: [aiida_computers, aiida_core]
   when: '"computers" in aiida_components'
 
 # setup plugins and codes
 - include_tasks: plugins/main.yml
-  tags: aiida_plugins
+  tags: [aiida_plugins]
   when: '"plugins" in aiida_components'
 
 # Set up pseudopotentials
 - include_tasks: aiida-pps.yml
   vars:
     pp: "{{ item }}"
   with_items: "{{ aiida_pseudopotentials }}"
-  tags: aiida_pps
+  tags: [aiida_pps]
   when: '"pseudopotentials" in aiida_components'
 
 - include_tasks: aiida-examples.yml
-  tags: aiida_examples
+  tags: [aiida_examples]
   when: '"examples" in aiida_components'

tasks/plugins/aiida-bigdft.yml

@@ -3,6 +3,7 @@
     name: aiida-bigdft
     version: "{{ aiida_bigdft_version }}"
     virtualenv: "{{ aiida_venv }}"
+    extra_args: "-c {{ aiida_constraints_file }}"
   register: pip_install
 
 - name: reentry scan
@@ -27,8 +28,7 @@
 - include_role:
     name: release_notes
   vars:
-    section: AiiDA
+    section: "AiiDA Plugins"
     option: aiida-bigdft
-    value: >-
-      aiida-bigdft {{ aiida_bigdft_version }} is installed.
+    value: "{{ aiida_bigdft_version }}"
   when: release_notes is defined and release_notes

tasks/plugins/aiida-cp2k.yml

@@ -3,6 +3,7 @@
     name: aiida-cp2k
     version: "{{ aiida_cp2k_version }}"
     virtualenv: "{{ aiida_venv }}"
+    extra_args: "-c {{ aiida_constraints_file }}"
   register: pip_install
 
 - name: reentry scan
@@ -27,8 +28,7 @@
 - include_role:
     name: release_notes
   vars:
-    section: AiiDA
+    section: "AiiDA Plugins"
     option: aiida-cp2k
-    value: >-
-      aiida-cp2k {{ aiida_cp2k_version }} is installed.
+    value: "{{ aiida_cp2k_version }}"
   when: release_notes is defined and release_notes

tasks/plugins/aiida-fleur.yml

@@ -3,6 +3,7 @@
     name: aiida-fleur
     version: "{{ aiida_fleur_version }}"
     virtualenv: "{{ aiida_venv }}"
+    extra_args: "-c {{ aiida_constraints_file }}"
   register: pip_install
 
 - name: reentry scan
@@ -27,8 +28,7 @@
 - include_role:
     name: release_notes
   vars:
-    section: AiiDA
+    section: "AiiDA Plugins"
     option: aiida-fleur
-    value: >-
-      aiida-fleur {{ aiida_fleur_version }} is installed.
+    value: "{{ aiida_fleur_version }}"
   when: release_notes is defined and release_notes

tasks/plugins/aiida-gudhi.yml

@@ -2,6 +2,7 @@
   pip:
     name: git+https://github.com/ltalirz/aiida-gudhi@v{{ aiida_gudhi_version }}
     virtualenv: "{{ aiida_venv }}"
+    extra_args: "-c {{ aiida_constraints_file }}"
   register: pip_install
 
 - name: reentry scan
@@ -26,9 +27,7 @@
 - include_role:
     name: release_notes
   vars:
-    section: AiiDA
+    section: "AiiDA Plugins"
     option: aiida-gudhi
-    value: >-
-      aiida-gudhi v{{ aiida_gudhi_version }} is installed.
-      See 'verdi code list' for available codes.
+    value: "{{ aiida_gudhi_version }}"
   when: release_notes is defined and release_notes

tasks/plugins/aiida-phtools.yml

@@ -2,6 +2,7 @@
   pip:
     name: git+https://github.com/ltalirz/aiida-phtools@v{{ aiida_phtools_version }}
     virtualenv: "{{ aiida_venv }}"
+    extra_args: "-c {{ aiida_constraints_file }}"
   register: pip_install
 
 - name: reentry scan
@@ -26,9 +27,7 @@
 - include_role:
     name: release_notes
   vars:
-    section: AiiDA
+    section: "AiiDA Plugins"
     option: aiida-phtools
-    value: >-
-      aiida-phtools v{{ aiida_phtools_version }} is installed.
-      See 'verdi code list' for available codes.
+    value: "{{ aiida_phtools_version }}"
   when: release_notes is defined and release_notes

tasks/plugins/aiida-quantumespresso.yml

@@ -3,6 +3,7 @@
     name: aiida-quantumespresso
     version: "{{ aiida_quantumespresso_version }}"
     virtualenv: "{{ aiida_venv }}"
+    extra_args: "-c {{ aiida_constraints_file }}"
   register: aiida_qe_installed
 
 - name: reentry scan
@@ -29,8 +30,7 @@
 - include_role:
     name: release_notes
   vars:
-    section: "AiiDA"
+    section: "AiiDA Plugins"
     option: "aiida-quantumespresso"
-    value: >-
-      aiida-quantumespresso {{ aiida_quantumespresso_version }} is installed.
+    value: "{{ aiida_quantumespresso_version }}"
   when: release_notes is defined and release_notes

tasks/plugins/aiida-raspa.yml

@@ -2,6 +2,7 @@
   pip:
     name: git+https://github.com/yakutovicha/aiida-raspa@v{{ aiida_raspa_version }}
     virtualenv: "{{ aiida_venv }}"
+    extra_args: "-c {{ aiida_constraints_file }}"
   register: pip_install
 
 - name: reentry scan
@@ -26,9 +27,7 @@
 - include_role:
     name: release_notes
   vars:
-    section: AiiDA
+    section: "AiiDA Plugins"
     option: aiida-raspa
-    value: >-
-      aiida-raspa v{{ aiida_raspa_version }} is installed.
-      See 'verdi code list' for available codes.
+    value: "{{ aiida_raspa_version }}"
   when: release_notes is defined and release_notes

tasks/plugins/aiida-siesta.yml

@@ -3,6 +3,7 @@
     name: aiida-siesta
     version: "{{ aiida_siesta_version }}"
     virtualenv: "{{ aiida_venv }}"
+    extra_args: "-c {{ aiida_constraints_file }}"
   register: pip_install
 
 - name: reentry scan
@@ -27,8 +28,7 @@
 - include_role:
     name: release_notes
   vars:
-    section: AiiDA
+    section: "AiiDA Plugins"
     option: aiida-siesta
-    value: >-
-      aiida-siesta {{ aiida_siesta_version }} is installed.
+    value: "{{ aiida_siesta_version }}"
   when: release_notes is defined and release_notes