diff --git a/.github/config/README.md b/.github/config/README.md new file mode 100644 index 0000000000..6b43da3d5b --- /dev/null +++ b/.github/config/README.md @@ -0,0 +1,5 @@ +# AiiDA configuration files + +This folder contains configuration files for AiiDA computers, codes etc. + + - `slurm_rsa`: private key that provides access to the `slurm-ssh` container diff --git a/.github/config/slurm-ssh-config.yaml b/.github/config/slurm-ssh-config.yaml new file mode 100644 index 0000000000..48332209de --- /dev/null +++ b/.github/config/slurm-ssh-config.yaml @@ -0,0 +1,7 @@ +--- +safe_interval: 0 +username: xenon +look_for_keys: true +key_filename: "PLACEHOLDER_SSH_KEY" +key_policy: AutoAddPolicy +port: 5001 diff --git a/.github/config/slurm-ssh.yaml b/.github/config/slurm-ssh.yaml new file mode 100644 index 0000000000..43e5919e5b --- /dev/null +++ b/.github/config/slurm-ssh.yaml @@ -0,0 +1,12 @@ +--- +label: slurm-ssh +description: slurm container +hostname: localhost +transport: ssh +scheduler: slurm +shebang: "#!/bin/bash" +work_dir: /home/{username}/workdir +mpirun_command: "mpirun -np {tot_num_mpiprocs}" +mpiprocs_per_machine: 1 +prepend_text: "" +append_text: "" diff --git a/.github/config/slurm_rsa b/.github/config/slurm_rsa new file mode 100644 index 0000000000..20123b7d8c --- /dev/null +++ b/.github/config/slurm_rsa @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEAnCqpTQFbmi1WPX4uTUFCHAvf61AhvqXUFoJEHQEvtDYibWJZ +bI7LueA2eEKw68oynIfPeinr4+DOnejMG1+HKCWi03DzWoorBOYc0e9i3nxkU93j +hZZsiQZfBgcCenqh2t1ZLbEFdFnCqLDw6gbDH0F3W3NJW0Q30a8HQ01lqdSKyVdf +UghVLCx1HM53BxXEYGU2m2ii+uyoMIsz9TSCJdKXIAb5N4tZYqKPF8q0vf1eP2BB +SUsn4bAHpPqvx3I0HkyR6qV5UT4K91FteULLTJHjK3Y0bBUMOmNQPh0JTmfj/KNB +EtJdlGYE0Tce1XINvhHItSpdFZs8GTnmOzUaVQIDAQABAoIBAEpWsILcm5tX646Y +KzhRUUQCjxP38ChNzhjs57ma3/d8MYU6ZPEdRHN1/Nfgf1Guzcrfh29S11yBnjlj +IQ4CulbtG4ZlZSJ7VSEe3Sc+OiVIt4WIwY7M3VuY8dDvs0lUaQnDhnkOpFcPh28/ +017D20xcoJGi3o+YeK3TELUD+doOeaot4+5TvR0PiLEmyjlnWB1FRkYpGAVDRKKa +F3dSAGf41ygoDOaGmtNmpH/Fn1k9cSDZsRsMKjZQTjgKfX+y/H6eOpORgHYHVmlu +eFIK8+yVVBy5k+m7nTIAUzXm01yJ5fQuT/75EcILUvjloTwmykaTfO1Ez6rNf+BC +VCdD9H0CgYEAyBjEB9vbZ5gDnnkdG0WCr34xPtBztTuVADWz5HorHYFircBUIaJ0 +XOIUioXMmpgSRTzbryAXVznh+g3LeS8QgiGQJoRhIknN8rrRUWd25tgImCMte0eb +bTieJYpvUk8RPan/Arb6f1MLZjWYfJelSw8qQS6R4ydk1L2M78sri/8CgYEAx8vy +KP1e5gGfA42Q0aHvocH7vqbEAOfDK8J+RpT/EoSJ6kSu2oPvblF1CBqHo/nQMhfK +AGbAtWIfy8rs1Md2k+Y+8PXtY8sJJ/HA8laVnEvTHbPSt4X7TtrLx27a8ZWtTNYu +JH/kK8rFBHEGqLnS6VJmqvHKqglp7FIQmHNNaasCgYEApGSMcXR0zqh6mLEic6xp +EOtZZCT4WzZHVTPJxvWEBKqvOtbfh/6jIUhw3dnNXll/8ThtuHRiGLyqZrj8qWQ8 +aN1QRATQlM4UEM7hd8LMUh28+dk03arYDCTO8ULJ8NKa9JF8vGs+ZGsC24c+72Xb +XE5qRcEQBJLx6UKNztiZv1sCgYACHBEuhZ5e5116eCAzVnZlStsRpEkliUzyRVd3 +/1LCK0wZgSgnfoUksQ9/SmhsPtMH9GBZqLwYLjUPvdDKXmDOJvw7Jx2elCJAnbjf +1jI2OEa+ZYuwDGYe6wiDzpPZQS9XRFuwXvlVzQpPhbIAThYACLK002DEctz/dc5f +DbifiQKBgQCdXgr7tdEAmusvIcTRA1KMIOGE5pMGYfbMnDTTIihUfRMJbCnn9sHe +PrDKVVgD3W4hjOABN24KOlCZPtWZfKUKe893ali7mFAIwKNV/AKhQhDgGzJPidqc +6DIL2GhDwqtPIf3b6sI21ZvyAFDROZMKnoL5Q1xbbp5EADi2wPO55Q== +-----END RSA PRIVATE KEY----- diff --git a/.github/system_tests/pytest/test_memory_leaks.py b/.github/system_tests/pytest/test_memory_leaks.py index 7a4edd1d1d..b9f57a7e6a 100644 --- a/.github/system_tests/pytest/test_memory_leaks.py +++ b/.github/system_tests/pytest/test_memory_leaks.py @@ -10,17 +10,23 @@ """Utilities for testing memory leakage.""" from tests.utils import processes as test_processes # pylint: disable=no-name-in-module,import-error from tests.utils.memory import get_instances # pylint: disable=no-name-in-module,import-error -from aiida.engine import processes, run +from aiida.engine import processes, run_get_node from aiida.plugins import CalculationFactory from aiida import orm ArithmeticAddCalculation = CalculationFactory('arithmetic.add') +def run_finished_ok(*args, **kwargs): + """Convenience function to check that run worked fine.""" + _, node = run_get_node(*args, **kwargs) + assert node.is_finished_ok, (node.exit_status, node.exit_message) + + def test_leak_run_process(): """Test whether running a dummy process leaks memory.""" inputs = {'a': orm.Int(2), 'b': orm.Str('test')} - run(test_processes.DummyProcess, **inputs) + run_finished_ok(test_processes.DummyProcess, **inputs) # check that no reference to the process is left in memory # some delay is necessary in order to allow for all callbacks to finish @@ -30,8 +36,25 @@ def test_leak_run_process(): def test_leak_local_calcjob(aiida_local_code_factory): """Test whether running a local CalcJob leaks memory.""" - inputs = {'x': orm.Int(1), 'y': orm.Int(2), 'code': aiida_local_code_factory('arithmetic.add', '/usr/bin/diff')} - run(ArithmeticAddCalculation, **inputs) + inputs = {'x': orm.Int(1), 'y': orm.Int(2), 'code': aiida_local_code_factory('arithmetic.add', '/bin/bash')} + run_finished_ok(ArithmeticAddCalculation, **inputs) + + # check that no reference to the process is left in memory + # some delay is necessary in order to allow for all callbacks to finish + process_instances = get_instances(processes.Process, delay=0.2) + assert not process_instances, f'Memory leak: process instances remain in memory: {process_instances}' + + +def test_leak_ssh_calcjob(): + """Test whether running a CalcJob over SSH leaks memory. + + Note: This relies on the 'slurm-ssh' computer being set up. + """ + code = orm.Code( + input_plugin_name='arithmetic.add', remote_computer_exec=[orm.load_computer('slurm-ssh'), '/bin/bash'] + ) + inputs = {'x': orm.Int(1), 'y': orm.Int(2), 'code': code} + run_finished_ok(ArithmeticAddCalculation, **inputs) # check that no reference to the process is left in memory # some delay is necessary in order to allow for all callbacks to finish diff --git a/.github/workflows/ci-code.yml b/.github/workflows/ci-code.yml index 6ffa0248b7..8ff88267ec 100644 --- a/.github/workflows/ci-code.yml +++ b/.github/workflows/ci-code.yml @@ -70,6 +70,10 @@ jobs: image: rabbitmq:latest ports: - 5672:5672 + slurm: + image: xenonmiddleware/slurm:17 + ports: + - 5001:22 steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/setup.sh b/.github/workflows/setup.sh index 8e890cc36d..ab508b0897 100755 --- a/.github/workflows/setup.sh +++ b/.github/workflows/setup.sh @@ -10,18 +10,28 @@ chmod 755 "${HOME}" # Replace the placeholders in configuration files with actual values CONFIG="${GITHUB_WORKSPACE}/.github/config" +cp "${CONFIG}/slurm_rsa" "${HOME}/.ssh/slurm_rsa" sed -i "s|PLACEHOLDER_BACKEND|${AIIDA_TEST_BACKEND}|" "${CONFIG}/profile.yaml" sed -i "s|PLACEHOLDER_PROFILE|test_${AIIDA_TEST_BACKEND}|" "${CONFIG}/profile.yaml" sed -i "s|PLACEHOLDER_DATABASE_NAME|test_${AIIDA_TEST_BACKEND}|" "${CONFIG}/profile.yaml" sed -i "s|PLACEHOLDER_REPOSITORY|/tmp/test_repository_test_${AIIDA_TEST_BACKEND}/|" "${CONFIG}/profile.yaml" sed -i "s|PLACEHOLDER_WORK_DIR|${GITHUB_WORKSPACE}|" "${CONFIG}/localhost.yaml" sed -i "s|PLACEHOLDER_REMOTE_ABS_PATH_DOUBLER|${CONFIG}/doubler.sh|" "${CONFIG}/doubler.yaml" +sed -i "s|PLACEHOLDER_SSH_KEY|${HOME}/.ssh/slurm_rsa|" "${CONFIG}/slurm-ssh-config.yaml" verdi setup --config "${CONFIG}/profile.yaml" + +# set up localhost computer verdi computer setup --config "${CONFIG}/localhost.yaml" verdi computer configure local localhost --config "${CONFIG}/localhost-config.yaml" +verdi computer test localhost verdi code setup --config "${CONFIG}/doubler.yaml" verdi code setup --config "${CONFIG}/add.yaml" +# set up slurm-ssh computer +verdi computer setup --config "${CONFIG}/slurm-ssh.yaml" +verdi computer configure ssh slurm-ssh --config "${CONFIG}/slurm-ssh-config.yaml" -n # needs slurm container +verdi computer test slurm-ssh --print-traceback + verdi profile setdefault test_${AIIDA_TEST_BACKEND} verdi config runner.poll.interval 0 diff --git a/.github/workflows/test-install.yml b/.github/workflows/test-install.yml index b9fb994ce9..a51c64015b 100644 --- a/.github/workflows/test-install.yml +++ b/.github/workflows/test-install.yml @@ -147,6 +147,10 @@ jobs: image: rabbitmq:latest ports: - 5672:5672 + slurm: + image: xenonmiddleware/slurm:17 + ports: + - 5001:22 steps: - uses: actions/checkout@v2