diff --git a/.papr.yml b/.papr.yml index 63c887eea9..491b3608f3 100644 --- a/.papr.yml +++ b/.papr.yml @@ -15,86 +15,98 @@ tests: --- -context: f28-primary +context: f28-primary-1 inherit: true -cluster: - hosts: - - name: vmcheck1 - distro: fedora/28/atomic - - name: vmcheck2 - distro: fedora/28/atomic - - name: vmcheck3 - distro: fedora/28/atomic - container: - image: registry.fedoraproject.org/fedora:28 +# https://github.com/ostreedev/ostree/pull/1513#issuecomment-378784162 +host: + distro: fedora/28/atomic + specs: + ram: 8192 +#container: +# image: registry.fedoraproject.org/fedora:28 + +tests: + - modprobe -r kvm_intel || true + - modprobe kvm_intel nested=1 + - docker run --device /dev/kvm --rm -v $(pwd):/srv/code:z + --env VMCHECK_PARALLEL=${VMCHECK_PARALLEL} + --env TESTS_OFFSET=${TESTS_OFFSET} + --env TESTS_MODULUS=${TESTS_MODULUS} + --env CFLAGS="${CFLAGS}" --env ASAN_OPTIONS=${ASAN_OPTIONS} + registry.fedoraproject.org/fedora:28 /bin/sh -c "cd /srv/code && ./ci/papr-vmcheck.sh" env: - HOSTS: vmcheck1 vmcheck2 vmcheck3 + # each VM is (currently) 1024MB, so we'll leave 1G for the OS + VMCHECK_PARALLEL: '5' # TODO use -fsanitize=address CFLAGS: '-fsanitize=undefined -fsanitize-undefined-trap-on-error -O2 -Wp,-D_FORTIFY_SOURCE=2' ASAN_OPTIONS: 'detect_leaks=0' # Right now we're not fully clean, but this gets us use-after-free etc - CI_PKGS: rsync - -tests: - - ci/build-check.sh - - ci/vmcheck-provision.sh - - make vmcheck - # make sure we're aware of any tests that were skipped - - "grep -nr '^SKIP: ' vmcheck/ || :" + TESTS_OFFSET: '0' + TESTS_MODULUS: '3' timeout: 60m artifacts: + - artifacts/ - test-suite.log - config.log - vmcheck --- +context: f28-primary-2 +inherit: true + +# Copy of above, with modified TESTS_OFFSET +env: + VMCHECK_PARALLEL: '5' + CFLAGS: '-fsanitize=undefined -fsanitize-undefined-trap-on-error -O2 -Wp,-D_FORTIFY_SOURCE=2' + ASAN_OPTIONS: 'detect_leaks=0' # Right now we're not fully clean, but this gets us use-after-free etc + # This one runs the other half of the tests + TESTS_OFFSET: '1' + TESTS_MODULUS: '2' + +--- + +context: f28-primary-3 +inherit: true + +# Copy of above, with modified TESTS_OFFSET +env: + VMCHECK_PARALLEL: '5' + CFLAGS: '-fsanitize=undefined -fsanitize-undefined-trap-on-error -O2 -Wp,-D_FORTIFY_SOURCE=2' + ASAN_OPTIONS: 'detect_leaks=0' # Right now we're not fully clean, but this gets us use-after-free etc + # This one runs the other half of the tests + TESTS_OFFSET: '2' + TESTS_MODULUS: '3' + + +--- + inherit: true context: c7-primary required: true -cluster: - hosts: - - name: vmcheck1 - distro: centos/7/atomic/smoketested - # XXX: temp hack until smoketested has newer glib2 - # XXX: and also causes layering-relabel to fail? - # https://github.com/projectatomic/rpm-ostree/pull/1406 - ostree: - branch: centos-atomic-host/7/x86_64/devel/continuous - - name: vmcheck2 - distro: centos/7/atomic/smoketested - ostree: - branch: centos-atomic-host/7/x86_64/devel/continuous - - name: vmcheck3 - distro: centos/7/atomic/smoketested - ostree: - branch: centos-atomic-host/7/x86_64/devel/continuous - container: - image: registry.centos.org/centos/centos:7 +# FIXME; temporary workaround +# https://github.com/ostreedev/ostree/pull/1513#issuecomment-378784162 +host: + distro: fedora/28/atomic + specs: + ram: 4096 +#container: +# image: registry.centos.org/centos/centos:7 # We only want the sanitizers on Fedora env: - HOSTS: vmcheck1 vmcheck2 vmcheck3 CFLAGS: '' -extra-repos: - - name: atomic-centos-continuous - baseurl: https://ci.centos.org/artifacts/sig-atomic/rdgo/centos-continuous/build - gpgcheck: 0 - tests: - # we're still on devmapper here; we need to expand rootfs for tests - - for vm in vmcheck{1..3}; do ssh $vm lvresize -r -L +5G atomicos/root; done - - ci/vmcheck-provision.sh - - yum install -y epel-release - - ci/build-check.sh - - make vmcheck + # If we're still on devmapper, we need to expand rootfs for tests + - lvresize -r -l +100%FREE atomicos/root + - docker run --device /dev/kvm --rm -v $(pwd):/srv/code:z registry.centos.org/centos/centos:7 /bin/sh -c "cd /srv/code && ./ci/papr-vmcheck.sh" --- diff --git a/Makefile-tests.am b/Makefile-tests.am index b18974cb3f..71168cc4aa 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -78,7 +78,7 @@ check-local: @echo " *** NOTE ***" @echo " *** NOTE ***" -.PHONY: vmsync vmoverlay vmcheck testenv +.PHONY: vmsync vmcheck testenv HOSTS ?= "vmcheck" @@ -88,20 +88,10 @@ vmsync: fi; \ env $(BASE_TESTS_ENVIRONMENT) ./tests/vmcheck/sync.sh -vmoverlay: inject-pkglist - @set -e; if [ -z "$(SKIP_VMOVERLAY)" ]; then \ - if [ -z "$(SKIP_INSTALL)" ]; then \ - env $(BASE_TESTS_ENVIRONMENT) ./tests/vmcheck/install.sh; \ - fi; \ - echo -n "$(HOSTS)" | xargs -P 0 -n 1 -d ' ' -I {} \ - env $(BASE_TESTS_ENVIRONMENT) VM={} \ - ./tests/vmcheck/overlay.sh; \ - fi # set up test environment to somewhat resemble uninstalled tests -vmcheck: vmoverlay - @env VMTESTS=1 $(BASE_TESTS_ENVIRONMENT) PYTHONUNBUFFERED=1 \ - tests/vmcheck/multitest.py $(HOSTS) +vmcheck: inject-pkglist + @env VMTESTS=1 $(BASE_TESTS_ENVIRONMENT) tests/vmcheck/vmcheck-run.sh testenv: @echo "===== ENTERING TESTENV =====" diff --git a/ci/libbuild.sh b/ci/libbuild.sh index c15e7fe503..8ee47b9572 100644 --- a/ci/libbuild.sh +++ b/ci/libbuild.sh @@ -2,7 +2,13 @@ pkg_upgrade() { echo "Running yum -y distro-sync... $(date)" - yum -y distro-sync + if ! yum -y distro-sync; then + rc=$? + if test -f /var/log/dnf.log; then + grep librepo.LibrepoException /var/log/dnf.log + fi + exit ${rc} + fi echo "Done yum -y distro-sync! $(date)" } diff --git a/ci/papr-vmcheck.sh b/ci/papr-vmcheck.sh new file mode 100755 index 0000000000..5aaa7b0f41 --- /dev/null +++ b/ci/papr-vmcheck.sh @@ -0,0 +1,44 @@ +#!/usr/bin/bash +# Install build dependencies, run unit tests and installed tests. + +set -xeuo pipefail + +dn=$(dirname $0) +. ${dn}/libbuild.sh +OS_ID=$(. /etc/os-release && echo "${ID}") + +case ${OS_ID} in + centos) + cat > /etc/yum.repos.d/cahc.repo < centos-atomic-host-7.qcow2 + export TEST_SUBJECTS="$(pwd)/centos-atomic-host-7.qcow2";; + *) fatal "unknown OS_ID=${OS_ID}";; +esac + +make vmcheck diff --git a/ci/vmcheck-provision.sh b/ci/vmcheck-provision.sh deleted file mode 100755 index 66fc35239c..0000000000 --- a/ci/vmcheck-provision.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/bash -# Install build dependencies, run unit tests and installed tests. - -set -xeuo pipefail - -dn=$(dirname $0) -. ${dn}/libbuild.sh -pkg_install openssh-clients ansible diff --git a/tests/README.md b/tests/README.md index 5c894c02c0..73e7979fe3 100644 --- a/tests/README.md +++ b/tests/README.md @@ -18,8 +18,8 @@ Tests are divided into three groups: Note: This is intentionally *not* a `Makefile` target because it doesn't require building and doesn't use uninstalled binaries. -- Tests in the `vmcheck` directory are oriented around using - Vagrant. Use `make vmcheck` to run them. +- Tests in the `vmcheck` directory are oriented around testing + in disposable virtual machines. Use `make vmcheck` to run them. See also `HACKING.md` in the top directory. The `common` directory contains files used by multiple diff --git a/tests/common/libtest.sh b/tests/common/libtest.sh index cf6415a6df..7bc28136a9 100644 --- a/tests/common/libtest.sh +++ b/tests/common/libtest.sh @@ -52,7 +52,7 @@ _cleanup_tmpdir () { # Create a tmpdir if we're running as a local test (i.e. through `make check`) # or as a `vmcheck` test, which also needs some scratch space on the host. if ( test -n "${UNINSTALLEDTESTS:-}" || test -n "${VMTESTS:-}" ) && ! test -f $PWD/.test; then - test_tmpdir=$(mktemp -d test.XXXXXX) + test_tmpdir=$(mktemp -d /tmp/test-$(basename ${topsrcdir}).XXXXXX) touch ${test_tmpdir}/.test trap _cleanup_tmpdir EXIT cd ${test_tmpdir} diff --git a/tests/common/libvm.sh b/tests/common/libvm.sh index 6f36ec3aa6..368ddf649a 100644 --- a/tests/common/libvm.sh +++ b/tests/common/libvm.sh @@ -17,11 +17,83 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -# prepares the VM and library for action +vm_run_sti() { + local vmid=$1 + sti_qemu_lockf=$(pwd)/sti-qemu.lock + mkdir inventory-${vmid} + cd inventory-${vmid} + # Marker to denote this is really a tmpdir + # https://fedoraproject.org/wiki/CI/Tests + if test -z "${TEST_SUBJECTS:-}"; then + cat < inventory.json + if ! jq '.' inventory.json; then + echo "Failed provisioning; JSON:" + sed -e 's,^,# ,' < inventory.json + exit 1 + fi + python3 -c " +import json +with open('inventory.json') as f: + d = json.load(f) +all_hostvars = d['_meta']['hostvars'] +for k in all_hostvars: + host = k + break +hostvars = all_hostvars[host] + +# Ansible seems to sometimes use ansible_, sometimes ansible_ssh_ +def getkey(k): + return hostvars.get('ansible_'+k) or hostvars.get('ansible_ssh_'+k) + +print('''Host vmcheck + HostName {host} + User root + Port {port} + UserKnownHostsFile /dev/null + StrictHostKeyChecking no + PasswordAuthentication no + IdentityFile {identity} + IdentitiesOnly yes + LogLevel FATAL +'''.format(host=getkey('host'), + port=getkey('port'), + identity=getkey('private_key_file'))) +" > ssh-config + export SSH_CONFIG=$(pwd)/ssh-config + cd - +} + +# If SSH_CONFIG is set, use that pre-prepared VM. Otherwise, +# use the standard test interface vm_setup() { + local vmid=${1:-} + export SSH_CONFIG=${SSH_CONFIG:-${topsrcdir}/ssh-config} + if test '!' -f "${SSH_CONFIG}"; then + vm_run_sti "${vmid}" + fi export VM=${VM:-vmcheck} - export SSH_CONFIG=${SSH_CONFIG:-${topsrcdir}/ssh-config} SSHOPTS="-o User=root -o ControlMaster=auto \ -o ControlPath=/var/tmp/ssh-$VM-$(date +%s%N).sock \ -o ControlPersist=yes" @@ -94,6 +166,11 @@ vm_cmd() { $SSH "$@" } +vm_console_log() { + msg=$1 + vm_cmd /bin/sh -c 'echo "$(date): '"${msg}"' > /dev/ttyS0"' +} + # Delete anything which we might change between runs vm_clean_caches() { vm_cmd rm /ostree/repo/refs/heads/rpmostree/pkg/* -rf diff --git a/tests/vmcheck/fetch-journal.sh b/tests/vmcheck/fetch-journal.sh deleted file mode 100755 index 5b0f923e1b..0000000000 --- a/tests/vmcheck/fetch-journal.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -set -euo pipefail - -. ${commondir}/libvm.sh - -vm_setup - -if ! vm_ssh_wait 30; then - echo "WARNING: Failed to wait for VM to fetch journal" > ${JOURNAL_LOG} -else - echo "Saving ${JOURNAL_LOG}" - vm_cmd 'journalctl --no-pager || true' > ${JOURNAL_LOG} -fi diff --git a/tests/vmcheck/multitest.py b/tests/vmcheck/multitest.py deleted file mode 100755 index ed4e6a0bcc..0000000000 --- a/tests/vmcheck/multitest.py +++ /dev/null @@ -1,132 +0,0 @@ -#!/bin/env python - -import os -import sys -import glob -import time -import subprocess - - -def main(): - - failed = False - hosts = [] - for host in sys.argv[1:]: - hosts.append(Host(host)) - - if len(hosts) == 0: - print("error: no hosts provided") - sys.exit(1) - - requested_tests_spec = os.environ.get('TESTS') - if requested_tests_spec is not None: - requested_tests = requested_tests_spec.split() - else: - requested_tests = None - - tests = glob.iglob(os.path.join(sys.path[0], "test-*.sh")) - matched_tests = [] - unmatched_tests = [] - for test in tests: - testname = Host._strip_test(test) - if requested_tests is None or testname in requested_tests: - matched_tests.append(test) - else: - unmatched_tests.append(testname) - if len(matched_tests) == 0: - print("error: no tests match '{}': {}".format(requested_tests_spec, unmatched_tests)) - sys.exit(1) - - for test in matched_tests: - host = wait_for_next_available_host(hosts) - rc = host.flush() - failed = failed or rc != 0 - host.dispatch(test) - if len(unmatched_tests) > 0: - print("NOTE: Skipping tests not matching {}: {}".format(requested_tests_spec, unmatched_tests)) - - for host in hosts: - rc = host.flush() - failed = failed or rc != 0 - - # fetch the journal from all the hosts which had a failure - fetcher = os.path.join(sys.path[0], "fetch-journal.sh") - for host in hosts: - if host.saw_fail: - fetcher_env = dict(os.environ) - fetcher_env.update({'VM': host.hostname, - 'JOURNAL_LOG': - "vmcheck/%s.journal.log" % host.hostname}) - subprocess.check_call([fetcher], env=fetcher_env) - - return 1 if failed else 0 - - -def wait_for_next_available_host(hosts): - while True: - for host in hosts: - if host.is_done(): - return host - time.sleep(1) - - -class Host: - - def __init__(self, hostname): - self.hostname = hostname - self.test = "" - self._p = None - self.saw_fail = False - - def is_done(self): - if not self._p: - return True - return self._p.poll() is not None - - def dispatch(self, test): - assert self.is_done() - test = self._strip_test(test) - env = dict(os.environ) - env.update({'TESTS': test, - 'VM': self.hostname, - 'JOURNAL_LOG': "", # we fetch the journal at the end - 'LOG': "vmcheck/%s.out" % test}) - if not os.path.isdir("vmcheck"): - os.mkdir("vmcheck") - testsh = os.path.join(sys.path[0], "test.sh") - self._p = subprocess.Popen([testsh], env=env, - stdout=open("vmcheck/%s.log" % test, 'wb'), - stderr=subprocess.STDOUT) - self.test = test - print "INFO: scheduled", self.test, "on host", self.hostname - - def flush(self): - if not self._p: - return 0 - rc = self._p.wait() - - # just merge the two files - outfile = "vmcheck/{}.out".format(self.test) - if os.path.isfile(outfile): - with open(outfile) as f: - with open("vmcheck/%s.log" % self.test, 'a') as j: - j.write(f.read()) - os.remove(outfile) - - rcs = "PASS" if rc == 0 else ("FAIL (rc %d)" % rc) - print("%s: %s" % (rcs, self.test)) - - self.test = "" - self._p = None - self.saw_fail = self.saw_fail or rc != 0 - return rc - - @staticmethod - def _strip_test(test): - test = os.path.basename(test) - assert test.startswith('test-') and test.endswith('.sh') - return test[5:-3] - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/tests/vmcheck/overlay.sh b/tests/vmcheck/overlay.sh index da7e193131..11ca1c5607 100755 --- a/tests/vmcheck/overlay.sh +++ b/tests/vmcheck/overlay.sh @@ -4,13 +4,6 @@ set -euo pipefail # Execute this code path on the host if test -z "${INSIDE_VM:-}"; then . ${commondir}/libvm.sh - vm_setup - - if ! vm_ssh_wait 30; then - echo "ERROR: A running VM is required for 'make vmcheck'." - exit 1 - fi - vm_rsync vm_cmd env \ @@ -21,9 +14,14 @@ if test -z "${INSIDE_VM:-}"; then fi set -x - # And then this code path in the VM +console_log() { + echo "$(date) overlay: $@" > /dev/ttyS0 +} + +console_log "Starting" + # get details from the current default deployment rpm-ostree status --json > json.txt json_field() { @@ -52,6 +50,8 @@ rm vmcheck -rf ostree checkout $commit vmcheck --fsync=0 rm vmcheck/etc -rf +console_log "Checkout complete" + # Now, overlay our built binaries & config files, unless # explicitly requested not to (with the goal of testing the # tree shipped as is with our existing tests). @@ -98,5 +98,7 @@ fi ostree commit --parent=$commit -b vmcheck --consume --no-bindings \ --link-checkout-speedup ${commit_opts} "${source_opt}" \ --selinux-policy=vmcheck --tree=dir=vmcheck +console_log "Commit complete" ostree admin deploy vmcheck +console_log "Deploy complete" diff --git a/tests/vmcheck/run-one-test.sh b/tests/vmcheck/run-one-test.sh new file mode 100755 index 0000000000..acd6611ee0 --- /dev/null +++ b/tests/vmcheck/run-one-test.sh @@ -0,0 +1,79 @@ +#!/bin/bash +# Wrap a single test-foo.sh by launching a VM, +# and grabbing the journal etc. if it fails. + +set -euo pipefail + +tf=$1 + +dn=$(cd $(dirname $0) && pwd) +. ${commondir}/libvm.sh + +# Configure a subdirectory for our logs; note this also +# configures the standard-inventory-qcow2 console logs. +export TEST_ARTIFACTS=${LOGDIR}/${tf} +rm -rf ${TEST_ARTIFACTS} +mkdir -p ${TEST_ARTIFACTS} +# Redirect our stdout/stderr, since we don't want what GNU parallel does +exec 1>${TEST_ARTIFACTS}/output.txt +exec 2>&1 + +set -x + +# Ensure we have a connection to VM, and overlay our built +# binaries. +vm_setup ${tf} +${dn}/overlay.sh + +# Sanity check connection +if ! vm_ssh_wait 30; then + echo "ERROR: A running VM is required for 'make vmcheck'." + exit 1 +fi + +vm_console_log "SSH ready, preparing test: $tf" + +# remember the csum we're currently on and tag it so that ostree doesn't wipe it +csum_orig=$(vm_get_booted_csum) +vm_cmd ostree rev-parse $csum_orig &> /dev/null # validate +vm_cmd ostree refs vmcheck_orig --delete +vm_cmd ostree refs $csum_orig --create vmcheck_orig + +# Santiy check we're on vmcheck +origin=$(vm_get_booted_deployment_info origin) +test "$origin" = "vmcheck" + +echo "Neutering /etc/yum.repos.d" +# Move the current one to .bak +vm_cmd mv /etc/yum.repos.d{,.bak} +vm_cmd mkdir /etc/yum.repos.d + +if vm_cmd test -f /usr/etc/rpm-ostreed.conf; then + vm_cmd cp -f /usr/etc/rpm-ostreed.conf /etc + # Unless we're doing overrides + if [[ "${VMCHECK_FLAGS:-}" =~ "stage-deployments" ]]; then + vm_cmd 'echo "[Experimental]" >> /etc/rpm-ostreed.conf' + vm_cmd 'echo StageDeployments=true >> /etc/rpm-ostreed.conf' + fi +fi + +origdir=$(pwd) +testdir="$(dirname $(realpath $0))" +cd $testdir + +bn=$(basename ${tf}) +echo "Running $bn..." +vm_cmd logger "vmcheck: running $bn..." + +if ! ${dn}/${tf}; then + echo "FAILED: ${tf}" + echo "FAILED: ${tf}" >> ${LOGDIR}/vmcheck/failed.txt + vm_console_log "FAILED: ${tf}" || true + echo "Attempting to gather journal..." + vm_cmd 'journalctl --no-pager || true' > ${TEST_ARTIFACTS}/journal.txt || true +fi +vm_console_log "SUCCESS: ${tf}" + +# The standard-test-roles qemu instance will exit asynchronously, but that +# causes RAM usage spikes with parallelization in play. +vm_cmd systemd-run sh -c "'sleep 2 && systemctl halt -ff'" || true diff --git a/tests/vmcheck/test.sh b/tests/vmcheck/test.sh deleted file mode 100755 index 356c0ad6a5..0000000000 --- a/tests/vmcheck/test.sh +++ /dev/null @@ -1,236 +0,0 @@ -#!/bin/bash -set -euo pipefail - -. ${commondir}/libvm.sh - -LOG=${LOG:-vmcheck.log} -LOG=$(realpath $LOG) - -# NB: allow JOURNAL_LOG to be empty, which means we never -# fetch it -JOURNAL_LOG=${JOURNAL_LOG-vmcheck-journal.txt} -if [ -n "$JOURNAL_LOG" ]; then - JOURNAL_LOG=$(realpath $JOURNAL_LOG) -fi - -# create ssh-config if needed and export cmds -vm_setup - -# stand up ssh connection and sanity check that it all works -if ! vm_ssh_wait 30; then - echo "ERROR: A running VM is required for 'make vmcheck'." - exit 1 -fi - -echo "VM is running." - -# just error out if we're unlocked -- we use the current deployment as the -# fallback between each test, so we need to be sure it's in a state that works. -# also, the user might have forgotten that these tests are somewhat destructive -# and thus would wipe out unlocked changes, hotfix or not. -unlocked_cur=$(vm_get_booted_deployment_info unlocked) -if [[ $unlocked_cur != none ]]; then - echo "ERROR: VM is unlocked." - exit 1 -fi - -# remember the csum we're currently on and tag it so that ostree doesn't wipe it -csum_orig=$(vm_get_booted_csum) -vm_cmd ostree rev-parse $csum_orig &> /dev/null # validate -vm_cmd ostree refs vmcheck_orig --delete -vm_cmd ostree refs $csum_orig --create vmcheck_orig - -# reboot onto the vmcheck ref if we're not already on it -origin=$(vm_get_booted_deployment_info origin) -if [[ $origin != vmcheck ]]; then - vm_cmd ostree refs vmcheck --delete - vm_cmd ostree refs vmcheck_orig --create vmcheck - vm_cmd ostree admin deploy vmcheck &>> ${LOG} - vm_reboot &>> ${LOG} -fi - -# delete whatever tmp refs the previous testsuite runs may have created -vm_cmd ostree refs vmcheck_tmp vmcheck_remote --delete - -# we bring our own test repo and test packages, so let's neuter any repo that -# comes with the distro to help speed up rpm-ostree metadata fetching since we -# don't cache it (e.g. on Fedora, it takes *forever* to fetch metadata, which we -# have to do dozens of times throughout the suite) -if ! vm_cmd test -f /etc/yum.repos.d/.vmcheck; then - echo "Neutering /etc/yum.repos.d" - # Move the current one to .bak - vm_cmd mv /etc/yum.repos.d{,.bak} - # And create a new one with a .vmcheck as a stamp file so we recognize it - vm_cmd rm /etc/yum.repos.d.tmp -rf - vm_cmd mkdir /etc/yum.repos.d.tmp - vm_cmd touch /etc/yum.repos.d.tmp/.vmcheck - vm_cmd cp -r /etc/yum.repos.d{.tmp,} -else - echo "Keeping existing vmcheck /etc/yum.repos.d" -fi - -# tests expect to run with the default config -# remember the original config, we restore it after the tests -if vm_cmd test -f /etc/rpm-ostreed.conf; then - vm_cmd mv -f /etc/rpm-ostreed.conf{,.bak} -fi -if vm_cmd test -f /usr/etc/rpm-ostreed.conf; then - vm_cmd cp -f /usr/etc/rpm-ostreed.conf /etc - # Unless we're doing overrides - if [[ "${VMCHECK_FLAGS:-}" =~ "stage-deployments" ]]; then - vm_cmd 'echo "[Experimental]" >> /etc/rpm-ostreed.conf' - vm_cmd 'echo StageDeployments=true >> /etc/rpm-ostreed.conf' - fi -fi - -vm_cmd ostree remote delete --if-exists vmcheckmote - -origdir=$(pwd) -echo -n '' > ${LOG} - -testdir="$(dirname $(realpath $0))" -cd $testdir - -colour_print() { - colour=$1; shift - [ ! -t 1 ] || echo -en "\e[${colour}m" - echo -n "$@" - [ ! -t 1 ] || echo -en "\e[0m" - echo -} - -pass_print() { - colour_print 32 "$@" # green -} - -fail_print() { - colour_print 31 "$@" # red -} - -skip_print() { - colour_print 34 "$@" # blue -} - -total=0 -pass=0 -fail=0 -skip=0 -notrun=0 -for tf in $(find . -name 'test-*.sh' | sort); do - - if [ -n "${TESTS+ }" ]; then - tfbn=$(basename "$tf" .sh) - tfbn=" ${tfbn#test-} " - if [[ " $TESTS " != *$tfbn* ]]; then - let "notrun += 1" - continue - fi - fi - - let "total += 1" - - bn=$(basename ${tf}) - printf "Running $bn...\n" - printf "\n==== ${bn} ====\n" >> ${LOG} - vm_cmd logger "vmcheck: running $bn..." - - # do some dirty piping to get some instant feedback and help debugging - if ${tf} |& tee -a ${LOG} \ - | grep -e '^ok ' --line-buffered \ - | xargs -d '\n' -n 1 echo " "; then - pass_print "PASS: $bn" - echo "PASS" >> ${LOG} - let "pass += 1" - else - if test $? = 77; then - skip_print "SKIP: $bn" - echo "SKIP" >> ${LOG} - let "skip += 1" - else - fail_print "FAIL: $bn" - echo "FAIL" >> ${LOG} - let "fail += 1" - fi - fi - - vm_cmd logger "vmcheck: finished $bn..." - if test -n "${VMCHECK_DEBUG:-}"; then - echo "VMCHECK_DEBUG is set, skipping restoration of original deployment" - break - fi - - # nuke all vmcheck and vmcheck_tmp refs and recreate vmcheck from orig - echo "Restoring original vmcheck commit" >> ${LOG} - vm_cmd ostree refs vmcheck vmcheck_tmp vmcheck_remote --delete - vm_cmd ostree refs vmcheck_orig --create vmcheck &>> ${LOG} - - # restore the default config - vm_cmd cp -f /usr/etc/rpm-ostreed.conf /etc/ - - # go back to the original vmcheck deployment if needed - origin_cur=$(vm_get_booted_deployment_info origin) - csum_cur=$(vm_get_booted_csum) - unlocked_cur=$(vm_get_booted_deployment_info unlocked) - live_csum=$(vm_get_booted_deployment_info live-replaced) - if [[ $origin_cur != vmcheck ]] || \ - [[ $csum_orig != $csum_cur ]] || \ - [[ $unlocked_cur != none ]] || \ - [ -n "${live_csum}" ]; then - # redeploy under the name 'vmcheck' so that tests can - # never modify the vmcheck_orig ref itself - vm_cmd ostree admin deploy vmcheck &>> ${LOG} - vm_reboot &>> ${LOG} - else - # make sure we're using the default config vals again - vm_cmd systemctl restart rpm-ostreed - fi - - # make sure to clean up any pending & rollback deployments - vm_rpmostree cleanup -b -p -r -m || : - - # and put back our tmp repo - vm_cmd rm /etc/yum.repos.d -rf - vm_cmd cp -r /etc/yum.repos.d{.tmp,} - - # and clean up any leftovers from our tmp - osname=$(vm_get_booted_deployment_info osname) - vm_cmd rm -rf /ostree/deploy/$osname/var/tmp/vmcheck - vm_cmd ostree remote delete --if-exists vmcheckmote -done - - -if test -z "${VMCHECK_DEBUG:-}"; then - - # put back the original yum repos - if vm_cmd test -f /etc/yum.repos.d/.vmcheck; then - echo "Restoring original /etc/yum.repos.d" - vm_cmd rm -rf /etc/yum.repos.d - vm_cmd mv /etc/yum.repos.d{.bak,} - fi - - # put back the original config if any - vm_cmd rm -f /etc/rpm-ostreed.conf - if vm_cmd test -f /etc/rpm-ostreed.conf.bak; then - vm_cmd mv /etc/rpm-ostreed.conf{.bak,} - fi -fi - -# Gather post-failure system logs if necessary -ALL_LOGS=$LOG -if [ ${fail} -ne 0 ] && [ -n "$JOURNAL_LOG" ]; then - ./fetch-journal.sh - ALL_LOGS="$ALL_LOGS $JOURNAL_LOG" -fi - -# tear down ssh connection if needed -if $SSH -O check &>/dev/null; then - $SSH -O exit &>/dev/null -fi - -[ ${fail} -eq 0 ] && printer=pass || printer=fail -${printer}_print "TOTAL: $total PASS: $pass SKIP: $skip FAIL: $fail" -if test ${notrun} -gt 0; then - echo "NOTICE: Skipped ${notrun} tests not matching \"${TESTS}\"" -fi -echo "See ${ALL_LOGS} for more information." -[ ${fail} -eq 0 ] diff --git a/tests/vmcheck/vmcheck-run.sh b/tests/vmcheck/vmcheck-run.sh new file mode 100755 index 0000000000..0d07c7366e --- /dev/null +++ b/tests/vmcheck/vmcheck-run.sh @@ -0,0 +1,80 @@ +#!/bin/bash +# Run all vmcheck tests using parallel. If +# VMCHECK_PARALLEL is set, it is a maximum number +# jobs to run in parallel. +# +# FIXME: dedup this logic with tests/compose +set -euo pipefail + +export VMCHECK_TIMEOUT=${VMCHECK_TIMEOUT:-10m} + +dn=$(cd $(dirname $0) && pwd) + +# Preparatory work; we have a helper binary +make inject-pkglist +# And finally install the built binaries, which we'll then +# later rsync into the tree. +tests/vmcheck/install.sh + +LOGDIR=${LOGDIR:-$(pwd)/vmcheck} +mkdir -p ${LOGDIR} +export LOGDIR + +# From here, execute in a temporary directory +. ${dn}/../common/libtest.sh + +all_tests="$(cd ${dn} && ls test-*.sh | sort)" +tests="" +# Support: env TESTS=misc make vmcheck +if [ -n "${TESTS+ }" ]; then + for tf in ${all_tests}; do + tfbn=$(basename "$tf" .sh) + tfbn=" ${tfbn#test-} " + if [[ " $TESTS " != *$tfbn* ]]; then + echo "Skipping: ${tf}" + continue + fi + tests="${tests} ${tf}" + done +else + # Used by papr to parallelize, e.g. run in parallel + # TESTS_OFFSET=0 TESTS_MODULUS=3 + # TESTS_OFFSET=1 TESTS_MODULUS=3 + # TESTS_OFFSET=2 TESTS_MODULUS=3 + if [ -n "${TESTS_MODULUS+ }" ]; then + o=0 + i=0 + for tf in ${all_tests}; do + if [[ $(($o < ${TESTS_OFFSET})) == 1 ]]; then + echo "o=$o Skipping: ${tf}" + o=$((${o}+1)) + continue + fi + if [[ $i == 0 ]]; then + tests="${tests} ${tf}" + else + echo "Skipping: ${tf}" + fi + i=$(((${i}+1) % ${TESTS_MODULUS})) + done + else + tests="${all_tests}" + fi +fi + +if test -z "${tests}"; then + fatal "error: No tests match ${TESTS}" +fi + + +echo "vmcheck tests starting: $(date)" +echo "Executing: ${tests}" +echo "Writing logs to ${LOGDIR}" + +# Note we merge stdout/stderr here since I don't see value +# in separating them. +mkdir -p ${LOGDIR}/parallel +(for tf in ${tests}; do echo $tf; done) | \ + parallel -v -j ${VMCHECK_PARALLEL:-1} --progress --halt soon,fail=1 \ + --results ${LOGDIR}/parallel --quote /bin/sh -c "timeout --signal=KILL ${VMCHECK_TIMEOUT} ${dn}/run-one-test.sh {}" +echo "$(date): All vmcheck tests passed"