Exercise multiple ansible versions in CI (kubernetes-sigs#8172)

* Ansible: separate requirements files for supported ansible versions

* Ansible: allow using ansible 2.11

* CI: Exercise Ansible 2.9 and Ansible 2.11 in a basic AIO CI job

* CI: Allow running a reset test outside of idempotency tests and running it in stage1

* CI: move ubuntu18-calico-aio job to stage2 and relay only on ubuntu20 with the variously supported ansible versions for stage1

* CI: add capability to install collections or roles from ansible-galaxy to mitigate missing behavior in older ansible versions

.gitlab-ci.yml

@@ -33,12 +33,13 @@ variables:
   RECOVER_CONTROL_PLANE_TEST: "false"
   RECOVER_CONTROL_PLANE_TEST_GROUPS: "etcd[2:],kube_control_plane[1:]"
   TERRAFORM_VERSION: 1.0.8
+  ANSIBLE_MAJOR_VERSION: "2.10"
 
 before_script:
   - ./tests/scripts/rebase.sh
   - update-alternatives --install /usr/bin/python python /usr/bin/python3 1
-  - python -m pip uninstall -y ansible
-  - python -m pip install -r tests/requirements.txt
+  - python -m pip uninstall -y ansible ansible-base ansible-core
+  - python -m pip install -r tests/requirements-${ANSIBLE_MAJOR_VERSION}.txt
   - mkdir -p /.ssh
 
 .job: &job

.gitlab-ci/packet.yml

@@ -23,15 +23,34 @@
   extends: .packet
 
 packet_ubuntu18-calico-aio:
-  stage: deploy-part1
+  stage: deploy-part2
   extends: .packet_pr
   when: on_success
 
-# Future AIO job
+# The ubuntu20-calico-aio jobs are meant as early stages to prevent running the full CI if something is horribly broken
 packet_ubuntu20-calico-aio:
   stage: deploy-part1
   extends: .packet_pr
   when: on_success
+  variables:
+    RESET_CHECK: "true"
+
+# Exericse ansible variants
+packet_ubuntu20-calico-aio-ansible-2_9:
+  stage: deploy-part1
+  extends: .packet_pr
+  when: on_success
+  variables:
+    ANSIBLE_MAJOR_VERSION: "2.9"
+    RESET_CHECK: "true"
+
+packet_ubuntu20-calico-aio-ansible-2_11:
+  stage: deploy-part1
+  extends: .packet_pr
+  when: on_success
+  variables:
+    ANSIBLE_MAJOR_VERSION: "2.11"
+    RESET_CHECK: "true"
 
 # ### PR JOBS PART2
 

ansible_version.yml

@@ -5,7 +5,7 @@
   vars:
     minimal_ansible_version: 2.9.0
     minimal_ansible_version_2_10: 2.10.11
-    maximal_ansible_version: 2.11.0
+    maximal_ansible_version: 2.12.0
     ansible_connection: local
   tags: always
   tasks:

requirements-2.10.txt

@@ -0,0 +1,10 @@
+ansible==3.4.0
+ansible-base==2.10.15
+cryptography==2.8
+jinja2==2.11.3
+netaddr==0.7.19
+pbr==5.4.4
+jmespath==0.9.5
+ruamel.yaml==0.16.10
+ruamel.yaml.clib==0.2.4
+MarkupSafe==1.1.1

requirements-2.11.txt

@@ -0,0 +1,10 @@
+ansible==4.8.0
+ansible-core==2.11.6
+cryptography==2.8
+jinja2==2.11.3
+netaddr==0.7.19
+pbr==5.4.4
+jmespath==0.9.5
+ruamel.yaml==0.16.10
+ruamel.yaml.clib==0.2.4
+MarkupSafe==1.1.1

requirements-2.9.txt

@@ -0,0 +1,8 @@
+ansible==2.9.27
+jinja2==2.11.3
+netaddr==0.7.19
+pbr==5.4.4
+jmespath==0.9.5
+ruamel.yaml==0.16.10
+ruamel.yaml.clib==0.2.4
+MarkupSafe==1.1.1

requirements-2.9.yml

@@ -0,0 +1,4 @@
+---
+collections:
+- name: community.general
+  version: '<3.0'

requirements.txt

@@ -0,0 +1 @@
+requirements-2.10.txt

tests/files/packet_ubuntu20-calico-aio-ansible-2_11.yml

@@ -0,0 +1 @@
+packet_ubuntu20-calico-aio.yml

tests/files/packet_ubuntu20-calico-aio-ansible-2_9.yml

@@ -0,0 +1 @@
+packet_ubuntu20-calico-aio.yml

tests/requirements-2.10.txt

@@ -0,0 +1,12 @@
+-r ../requirements-2.10.txt
+yamllint==1.19.0
+apache-libcloud==2.2.1
+tox==3.11.1
+dopy==0.3.7
+cryptography==2.8
+ansible-lint==5.0.11
+openshift==0.8.8
+molecule==3.0.6
+molecule-vagrant==0.3
+testinfra==5.2.2
+python-vagrant==0.5.15

tests/requirements-2.11.txt

@@ -0,0 +1,12 @@
+-r ../requirements-2.11.txt
+yamllint==1.19.0
+apache-libcloud==2.2.1
+tox==3.11.1
+dopy==0.3.7
+cryptography==2.8
+ansible-lint==5.0.11
+openshift==0.8.8
+molecule==3.0.6
+molecule-vagrant==0.3
+testinfra==5.2.2
+python-vagrant==0.5.15

tests/requirements-2.9.txt

@@ -0,0 +1,12 @@
+-r ../requirements-2.9.txt
+yamllint==1.19.0
+apache-libcloud==2.2.1
+tox==3.11.1
+dopy==0.3.7
+cryptography==2.8
+ansible-lint==5.0.11
+openshift==0.8.8
+molecule==3.0.6
+molecule-vagrant==0.3
+testinfra==5.2.2
+python-vagrant==0.5.15

tests/requirements.txt

@@ -0,0 +1 @@
+requirements-2.10.txt

tests/scripts/testcases_prepare.sh

@@ -1,9 +1,18 @@
 #!/bin/bash
 set -euxo pipefail
 
-/usr/bin/python -m pip uninstall -y ansible
-/usr/bin/python -m pip install -r tests/requirements.txt
+: ${ANSIBLE_MAJOR_VERSION:=2.10}
+
+/usr/bin/python -m pip uninstall -y ansible ansible-base ansible-core
+/usr/bin/python -m pip install -r tests/requirements-${ANSIBLE_MAJOR_VERSION}.txt
 mkdir -p /.ssh
 mkdir -p cluster-dump
 mkdir -p $HOME/.ssh
 ansible-playbook --version
+
+# in some cases we may need to bring in collections or roles from ansible-galaxy
+# to compensate for missing functionality in older ansible versions
+if [ -f requirements-${ANSIBLE_MAJOR_VERSION}.yml ] ; then
+  ansible-galaxy role install -r requirements-${ANSIBLE_MAJOR_VERSION}.yml
+  ansible-galaxy collection install -r requirements-${ANSIBLE_MAJOR_VERSION}.yml
+fi

tests/scripts/testcases_run.sh

@@ -86,27 +86,26 @@ ansible-playbook --limit "all:!fake_hosts" -e @${CI_TEST_VARS} ${CI_TEST_ADDITIO
 ## Kubernetes conformance tests
 ansible-playbook -i ${ANSIBLE_INVENTORY} -e @${CI_TEST_VARS} ${CI_TEST_ADDITIONAL_VARS} --limit "all:!fake_hosts" tests/testcases/100_check-k8s-conformance.yml $ANSIBLE_LOG_LEVEL
 
-## Idempotency checks 1/5 (repeat deployment)
 if [ "${IDEMPOT_CHECK}" = "true" ]; then
+  ## Idempotency checks 1/5 (repeat deployment)
   ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR} ${CI_TEST_ADDITIONAL_VARS} -e @${CI_TEST_VARS} -e local_release_dir=${PWD}/downloads --limit "all:!fake_hosts" cluster.yml
-fi
 
-## Idempotency checks 2/5 (Advanced DNS checks)
-if [ "${IDEMPOT_CHECK}" = "true" ]; then
+  ## Idempotency checks 2/5 (Advanced DNS checks)
   ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_VARS} ${CI_TEST_ADDITIONAL_VARS} --limit "all:!fake_hosts" tests/testcases/040_check-network-adv.yml
-fi
 
-## Idempotency checks 3/5 (reset deployment)
-if [ "${IDEMPOT_CHECK}" = "true" -a "${RESET_CHECK}" = "true" ]; then
-  ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR}  -e @${CI_TEST_VARS} ${CI_TEST_ADDITIONAL_VARS} -e reset_confirmation=yes --limit "all:!fake_hosts" reset.yml
-fi
+  if [ "${RESET_CHECK}" = "true" ]; then
+    ## Idempotency checks 3/5 (reset deployment)
+    ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR}  -e @${CI_TEST_VARS} ${CI_TEST_ADDITIONAL_VARS} -e reset_confirmation=yes --limit "all:!fake_hosts" reset.yml
+
+    ## Idempotency checks 4/5 (redeploy after reset)
+    ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} ${CI_TEST_ADDITIONAL_VARS} -e local_release_dir=${PWD}/downloads --limit "all:!fake_hosts" cluster.yml
 
-## Idempotency checks 4/5 (redeploy after reset)
-if [ "${IDEMPOT_CHECK}" = "true" -a "${RESET_CHECK}" = "true" ]; then
-  ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} ${CI_TEST_ADDITIONAL_VARS} -e local_release_dir=${PWD}/downloads --limit "all:!fake_hosts" cluster.yml
+    ## Idempotency checks 5/5 (Advanced DNS checks)
+    ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} ${CI_TEST_ADDITIONAL_VARS} --limit "all:!fake_hosts" tests/testcases/040_check-network-adv.yml
+  fi
 fi
 
-## Idempotency checks 5/5 (Advanced DNS checks)
-if [ "${IDEMPOT_CHECK}" = "true" -a "${RESET_CHECK}" = "true" ]; then
-  ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR} -e @${CI_TEST_VARS} ${CI_TEST_ADDITIONAL_VARS} --limit "all:!fake_hosts" tests/testcases/040_check-network-adv.yml
+# Clean up at the end, this is to allow stage1 tests to include cleanup test
+if [ "${RESET_CHECK}" = "true" ]; then
+  ansible-playbook ${ANSIBLE_LOG_LEVEL} -e @${CI_TEST_REGISTRY_MIRROR}  -e @${CI_TEST_VARS} ${CI_TEST_ADDITIONAL_VARS} -e reset_confirmation=yes --limit "all:!fake_hosts" reset.yml
 fi