WIP: RFC: Rework vmcheck to use STI qcow2 inventory

.papr.yml

@@ -15,86 +15,98 @@ tests:
 
 ---
 
-context: f28-primary
+context: f28-primary-1
 inherit: true
 
-cluster:
-  hosts:
-    - name: vmcheck1
-      distro: fedora/28/atomic
-    - name: vmcheck2
-      distro: fedora/28/atomic
-    - name: vmcheck3
-      distro: fedora/28/atomic
-  container:
-    image: registry.fedoraproject.org/fedora:28
+# https://github.com/ostreedev/ostree/pull/1513#issuecomment-378784162
+host:
+  distro: fedora/28/atomic
+  specs:
+    ram: 8192
+#container:
+#  image: registry.fedoraproject.org/fedora:28
+
+tests:
+  - modprobe -r kvm_intel || true
+  - modprobe kvm_intel nested=1
+  - docker run --device /dev/kvm --rm -v $(pwd):/srv/code:z
+    --env VMCHECK_PARALLEL=${VMCHECK_PARALLEL}
+    --env TESTS_OFFSET=${TESTS_OFFSET}
+    --env TESTS_MODULUS=${TESTS_MODULUS}
+    --env CFLAGS="${CFLAGS}" --env ASAN_OPTIONS=${ASAN_OPTIONS}
+    registry.fedoraproject.org/fedora:28 /bin/sh -c "cd /srv/code && ./ci/papr-vmcheck.sh"
 
 env:
-  HOSTS: vmcheck1 vmcheck2 vmcheck3
+  # each VM is (currently) 1024MB, so we'll leave 1G for the OS
+  VMCHECK_PARALLEL: '5'
   # TODO use -fsanitize=address
   CFLAGS: '-fsanitize=undefined -fsanitize-undefined-trap-on-error -O2 -Wp,-D_FORTIFY_SOURCE=2'
   ASAN_OPTIONS: 'detect_leaks=0'  # Right now we're not fully clean, but this gets us use-after-free etc
-  CI_PKGS: rsync
-
-tests:
-  - ci/build-check.sh
-  - ci/vmcheck-provision.sh
-  - make vmcheck
-  # make sure we're aware of any tests that were skipped
-  - "grep -nr '^SKIP: ' vmcheck/ || :"
+  TESTS_OFFSET: '0'
+  TESTS_MODULUS: '3'
 
 timeout: 60m
 
 artifacts:
+  - artifacts/
   - test-suite.log
   - config.log
   - vmcheck
 
 ---
 
+context: f28-primary-2
+inherit: true
+
+# Copy of above, with modified TESTS_OFFSET
+env:
+  VMCHECK_PARALLEL: '5'
+  CFLAGS: '-fsanitize=undefined -fsanitize-undefined-trap-on-error -O2 -Wp,-D_FORTIFY_SOURCE=2'
+  ASAN_OPTIONS: 'detect_leaks=0'  # Right now we're not fully clean, but this gets us use-after-free etc
+  # This one runs the other half of the tests
+  TESTS_OFFSET: '1'
+  TESTS_MODULUS: '2'
+
+---
+
+context: f28-primary-3
+inherit: true
+
+# Copy of above, with modified TESTS_OFFSET
+env:
+  VMCHECK_PARALLEL: '5'
+  CFLAGS: '-fsanitize=undefined -fsanitize-undefined-trap-on-error -O2 -Wp,-D_FORTIFY_SOURCE=2'
+  ASAN_OPTIONS: 'detect_leaks=0'  # Right now we're not fully clean, but this gets us use-after-free etc
+  # This one runs the other half of the tests
+  TESTS_OFFSET: '2'
+  TESTS_MODULUS: '3'
+
+
+---
+
 inherit: true
 
 context: c7-primary
 
 required: true
 
-cluster:
-  hosts:
-    - name: vmcheck1
-      distro: centos/7/atomic/smoketested
-      # XXX: temp hack until smoketested has newer glib2
-      # XXX: and also causes layering-relabel to fail?
-      # https://github.com/projectatomic/rpm-ostree/pull/1406
-      ostree:
-        branch: centos-atomic-host/7/x86_64/devel/continuous
-    - name: vmcheck2
-      distro: centos/7/atomic/smoketested
-      ostree:
-        branch: centos-atomic-host/7/x86_64/devel/continuous
-    - name: vmcheck3
-      distro: centos/7/atomic/smoketested
-      ostree:
-        branch: centos-atomic-host/7/x86_64/devel/continuous
-  container:
-    image: registry.centos.org/centos/centos:7
+# FIXME; temporary workaround
+# https://github.com/ostreedev/ostree/pull/1513#issuecomment-378784162
+host:
+  distro: fedora/28/atomic
+  specs:
+    ram: 4096
+#container:
+#  image: registry.centos.org/centos/centos:7
 
 # We only want the sanitizers on Fedora
 env:
-    HOSTS: vmcheck1 vmcheck2 vmcheck3
     CFLAGS: ''
 
-extra-repos:
-  - name: atomic-centos-continuous
-    baseurl: https://ci.centos.org/artifacts/sig-atomic/rdgo/centos-continuous/build
-    gpgcheck: 0
-
 tests:
-  # we're still on devmapper here; we need to expand rootfs for tests
-  - for vm in vmcheck{1..3}; do ssh $vm lvresize -r -L +5G atomicos/root; done
-  - ci/vmcheck-provision.sh
-  - yum install -y epel-release
-  - ci/build-check.sh
-  - make vmcheck
+  # If we're still on devmapper, we need to expand rootfs for tests
+  - lvresize -r -l +100%FREE atomicos/root
+  - docker run --device /dev/kvm --rm -v $(pwd):/srv/code:z registry.centos.org/centos/centos:7 /bin/sh -c "cd /srv/code && ./ci/papr-vmcheck.sh"
 
 ---
 

Makefile-tests.am

@@ -78,7 +78,7 @@ check-local:
 	@echo "  *** NOTE ***"
 	@echo "  *** NOTE ***"
 
-.PHONY: vmsync vmoverlay vmcheck testenv
+.PHONY: vmsync vmcheck testenv
 
 HOSTS ?= "vmcheck"
 
@@ -88,20 +88,10 @@ vmsync:
 	fi; \
 	env $(BASE_TESTS_ENVIRONMENT) ./tests/vmcheck/sync.sh
 
-vmoverlay: inject-pkglist
-	@set -e; if [ -z "$(SKIP_VMOVERLAY)" ]; then \
-	  if [ -z "$(SKIP_INSTALL)" ]; then \
-	    env $(BASE_TESTS_ENVIRONMENT) ./tests/vmcheck/install.sh; \
-	  fi; \
-	  echo -n "$(HOSTS)" | xargs -P 0 -n 1 -d ' ' -I {} \
-	    env $(BASE_TESTS_ENVIRONMENT) VM={} \
-	      ./tests/vmcheck/overlay.sh; \
-	fi
 
 # set up test environment to somewhat resemble uninstalled tests
-vmcheck: vmoverlay
-	@env VMTESTS=1 $(BASE_TESTS_ENVIRONMENT) PYTHONUNBUFFERED=1 \
-	  tests/vmcheck/multitest.py $(HOSTS)
+vmcheck: inject-pkglist
+	@env VMTESTS=1 $(BASE_TESTS_ENVIRONMENT) tests/vmcheck/vmcheck-run.sh
 
 testenv:
 	@echo "===== ENTERING TESTENV ====="

ci/libbuild.sh

@@ -2,7 +2,13 @@
 
 pkg_upgrade() {
     echo "Running yum -y distro-sync... $(date)"
-    yum -y distro-sync
+    if ! yum -y distro-sync; then
+        rc=$?
+        if test -f /var/log/dnf.log; then
+            grep librepo.LibrepoException /var/log/dnf.log
+        fi
+        exit ${rc}
+    fi
     echo "Done yum -y distro-sync! $(date)"
 }
 

ci/papr-vmcheck.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/bash
+# Install build dependencies, run unit tests and installed tests.
+
+set -xeuo pipefail
+
+dn=$(dirname $0)
+. ${dn}/libbuild.sh
+OS_ID=$(. /etc/os-release && echo "${ID}")
+
+case ${OS_ID} in
+    centos)
+        cat > /etc/yum.repos.d/cahc.repo <<EOF
+[atomic-centos-continuous]
+baseurl=https://ci.centos.org/artifacts/sig-atomic/rdgo/centos-continuous/build
+gpgcheck=0
+EOF
+        yum -y install epel-release
+        yum -y install python36;;
+esac
+
+pkg_upgrade
+${dn}/build-check.sh
+pkg_install git rsync openssh-clients ansible qemu-kvm standard-test-roles parallel
+
+# "Hot patch" this to pick up https://pagure.io/standard-test-roles/pull-request/223
+(cd /usr/share/ansible/inventory && curl -L -O https://pagure.io/fork/walters/standard-test-roles/raw/cpu-cores/f/inventory/standard-inventory-qcow2)
+
+# Ensure we really have KVM loaded
+if ! ls -al /dev/kvm; then
+    dmesg |grep -i kvm | sed -e 's,^,dmesg: ,'
+    fatal "Failed to find /dev/kvm"
+fi
+
+case ${OS_ID} in
+    fedora)
+        curl -Lo fedora-atomic-host.qcow2 https://getfedora.org/atomic_qcow2_latest
+        export TEST_SUBJECTS="$(pwd)/fedora-atomic-host.qcow2";;
+    centos)
+        curl -L https://ci.centos.org/artifacts/sig-atomic/centos-continuous/images-smoketested/cloud/latest/images/centos-atomic-host-7.qcow2.gz | gunzip > centos-atomic-host-7.qcow2
+        export TEST_SUBJECTS="$(pwd)/centos-atomic-host-7.qcow2";;
+    *) fatal "unknown OS_ID=${OS_ID}";;
+esac
+
+make vmcheck

tests/README.md

@@ -18,8 +18,8 @@ Tests are divided into three groups:
   Note: This is intentionally *not* a `Makefile` target because
   it doesn't require building and doesn't use uninstalled binaries.
 
-- Tests in the `vmcheck` directory are oriented around using
-  Vagrant.  Use `make vmcheck` to run them.
+- Tests in the `vmcheck` directory are oriented around testing
+  in disposable virtual machines.  Use `make vmcheck` to run them.
   See also `HACKING.md` in the top directory.
 
 The `common` directory contains files used by multiple

tests/common/libtest.sh

@@ -52,7 +52,7 @@ _cleanup_tmpdir () {
 # Create a tmpdir if we're running as a local test (i.e. through `make check`)
 # or as a `vmcheck` test, which also needs some scratch space on the host.
 if ( test -n "${UNINSTALLEDTESTS:-}" || test -n "${VMTESTS:-}" ) && ! test -f $PWD/.test; then
-   test_tmpdir=$(mktemp -d test.XXXXXX)
+   test_tmpdir=$(mktemp -d /tmp/test-$(basename ${topsrcdir}).XXXXXX)
    touch ${test_tmpdir}/.test
    trap _cleanup_tmpdir EXIT
    cd ${test_tmpdir}

tests/common/libvm.sh

@@ -17,11 +17,83 @@
 # Free Software Foundation, Inc., 59 Temple Place - Suite 330,
 # Boston, MA 02111-1307, USA.
 
-# prepares the VM and library for action
+vm_run_sti() {
+    local vmid=$1
+    sti_qemu_lockf=$(pwd)/sti-qemu.lock
+    mkdir inventory-${vmid}
+    cd inventory-${vmid}
+    # Marker to denote this is really a tmpdir
+    # https://fedoraproject.org/wiki/CI/Tests
+    if test -z "${TEST_SUBJECTS:-}"; then
+        cat <<EOF
+
+error: TEST_SUBJECTS must be set; e.g.:
+
+  curl -Lo fedora-atomic-host.qcow2 'https://getfedora.org/atomic_qcow2_latest'
+  export TEST_SUBJECTS=\$(pwd)/fedora-atomic-host.qcow2
+
+If you're doing interactive development, we recommend caching the qcow2
+somewhere persistent.
+EOF
+        exit 1
+    fi
+    for subj in ${TEST_SUBJECTS}; do
+        ls -al ${subj} && file ${subj}
+    done
+
+    # Required bits
+    rpm -q qemu-kvm standard-test-roles
+
+    # Unset G_DEBUG=fatal-warnings since qemu triggers a warning in C7 today
+    # (A bit embarassing that we ship it that way...)
+    env -u G_DEBUG /usr/share/ansible/inventory/standard-inventory-qcow2 > inventory.json
+    if ! jq '.' inventory.json; then
+        echo "Failed provisioning; JSON:"
+        sed -e 's,^,# ,' < inventory.json
+        exit 1
+    fi
+    python3 -c "
+import json
+with open('inventory.json') as f:
+  d = json.load(f)
+all_hostvars = d['_meta']['hostvars']
+for k in all_hostvars:
+  host = k
+  break
+hostvars = all_hostvars[host]
+
+# Ansible seems to sometimes use ansible_, sometimes ansible_ssh_
+def getkey(k):
+    return hostvars.get('ansible_'+k) or hostvars.get('ansible_ssh_'+k)
+
+print('''Host vmcheck
+  HostName {host}
+  User root
+  Port {port}
+  UserKnownHostsFile /dev/null
+  StrictHostKeyChecking no
+  PasswordAuthentication no
+  IdentityFile {identity}
+  IdentitiesOnly yes
+  LogLevel FATAL
+'''.format(host=getkey('host'),
+           port=getkey('port'),
+           identity=getkey('private_key_file')))
+" > ssh-config
+  export SSH_CONFIG=$(pwd)/ssh-config
+  cd -
+}
+
+# If SSH_CONFIG is set, use that pre-prepared VM.  Otherwise,
+# use the standard test interface
 vm_setup() {
+  local vmid=${1:-}
+  export SSH_CONFIG=${SSH_CONFIG:-${topsrcdir}/ssh-config}
+  if test '!' -f "${SSH_CONFIG}"; then
+    vm_run_sti "${vmid}"
+  fi
 
   export VM=${VM:-vmcheck}
-  export SSH_CONFIG=${SSH_CONFIG:-${topsrcdir}/ssh-config}
   SSHOPTS="-o User=root -o ControlMaster=auto \
            -o ControlPath=/var/tmp/ssh-$VM-$(date +%s%N).sock \
            -o ControlPersist=yes"
@@ -94,6 +166,11 @@ vm_cmd() {
   $SSH "$@"
 }
 
+vm_console_log() {
+    msg=$1
+    vm_cmd /bin/sh -c 'echo "$(date): '"${msg}"' > /dev/ttyS0"'
+}
+
 # Delete anything which we might change between runs
 vm_clean_caches() {
     vm_cmd rm /ostree/repo/refs/heads/rpmostree/pkg/* -rf