From 3cdfc62da394f0826ab8f6c15ac0987f905b33f0 Mon Sep 17 00:00:00 2001
From: Dusty Mabe <dusty@dustymabe.com>
Date: Tue, 24 Mar 2020 18:12:33 -0400
Subject: [PATCH] tests: add a test that can be run manually for net testing

This is how I've been doing network testing for the NM in
the initrd PR (https://github.com/coreos/fedora-coreos-config/pull/310).
One day I'd like to convert this into some more sophisticated
test that can be run in our CI but AFAIU right now that's not
possibly because our CI is limited to qemu unprivileged networking.

For now let's put this $somewhere so that we can pull it out
at least when we suspect problems.
---
 tests/manual/fcos-network-testing.sh | 486 +++++++++++++++++++++++++++
 1 file changed, 486 insertions(+)
 create mode 100755 tests/manual/fcos-network-testing.sh

diff --git a/tests/manual/fcos-network-testing.sh b/tests/manual/fcos-network-testing.sh
new file mode 100755
index 000000000..12a9b0cc5
--- /dev/null
+++ b/tests/manual/fcos-network-testing.sh
@@ -0,0 +1,486 @@
+#!/usr/bin/bash
+set -eu -o pipefail
+#set -x
+
+# This script attempts to test networking configuration in various
+# scenarios/configurations. It tries to test what should happen
+# when initramfs networking is passed and/or networking config is
+# passed via Ignition. It also tries to make sure that initramfs
+# configured network is passed properly to the real root when it
+# should be and not when it shouldn't be. See the following issue
+# for more details: https://github.com/coreos/fedora-coreos-tracker/issues/394
+# - Dusty Mabe - dusty@dustymabe.com
+
+fcct_common=\
+'variant: fcos
+version: 1.0.0
+passwd:
+  users:
+    - name: core
+      ssh_authorized_keys:
+        - $sshpubkey
+systemd:
+  units:
+    - name: serial-getty@ttyS0.service
+      dropins:
+      - name: autologin-core.conf
+        contents: |
+          [Service]
+          # Override Execstart in main unit
+          ExecStart=
+          # Add new Execstart with `-` prefix to ignore failure
+          ExecStart=-/usr/sbin/agetty --autologin core --noclear %I $TERM
+          TTYVTDisallocate=no
+storage:
+  files:
+    # pulling from a remote verifies we have networking in the initramfs
+    - path: /home/core/remotefile
+      mode: 0600
+      user:
+        name: core
+      group:
+        name: core
+      contents:
+        source: https://raw.githubusercontent.com/coreos/fedora-coreos-config/8b08bd030ef3968d00d4fea9a0fa3ca3fbabf852/COPYING
+        verification:
+          hash: sha512-d904690e4fc5defb804c2151e397cbe2aeeea821639995610aa377bb2446214c3433616a8708163776941df585b657648f20955e50d4b011ea2a96e7d8e08c66'
+
+fcct_static_eth0='
+    - path: /etc/NetworkManager/system-connections/eth0.nmconnection
+      mode: 0600
+      contents:
+        inline: |
+          [connection]
+          id=eth0
+          type=ethernet
+          interface-name=eth0
+          [ipv4]
+          address1=$ip/$prefix,$gateway
+          dns=$nameserver;
+          dns-search=
+          may-fail=false
+          method=manual
+          [ipv6]
+          method=disabled
+    - path: /etc/NetworkManager/system-connections/eth1.nmconnection
+      mode: 0600
+      contents:
+        inline: |
+          [connection]
+          id=eth1
+          type=ethernet
+          interface-name=eth1
+          [ipv4]
+          method=disabled
+          [ipv6]
+          method=disabled'
+
+fcct_static_team0='
+    - path: /etc/NetworkManager/system-connections/team0.nmconnection
+      mode: 0600
+      contents:
+        inline: |
+          [connection]
+          id=team0
+          type=team
+          interface-name=team0
+          [team]
+          config={"runner": {"name": "activebackup"}, "link_watch": {"name": "ethtool"}}
+          [ipv4]
+          address1=$ip/$prefix,$gateway
+          dns=$nameserver;
+          dns-search=
+          may-fail=false
+          method=manual
+    - path: /etc/NetworkManager/system-connections/team0-slave-eth0.nmconnection
+      mode: 0600
+      contents:
+        inline: |
+          [connection]
+          id=team0-slave-eth0
+          type=ethernet
+          interface-name=eth0
+          master=team0
+          slave-type=team
+          [team-port]
+          config={"prio": 100}
+    - path: /etc/NetworkManager/system-connections/team0-slave-eth1.nmconnection
+      mode: 0600
+      contents:
+        inline: |
+          [connection]
+          id=team0-slave-eth1
+          type=ethernet
+          interface-name=eth1
+          master=team0
+          slave-type=team
+          [team-port]
+          config={"prio": 100}'
+
+fcct_static_bond0='
+    - path: /etc/NetworkManager/system-connections/bond0.nmconnection
+      mode: 0600
+      contents:
+        inline: |
+          [connection]
+          id=bond0
+          type=bond
+          interface-name=bond0
+          [bond]
+          miimon=100
+          mode=active-backup
+          [ipv4]
+          address1=$ip/$prefix,$gateway
+          dns=$nameserver;
+          dns-search=
+          may-fail=false
+          method=manual
+    - path: /etc/NetworkManager/system-connections/bond0-slave-eth0.nmconnection
+      mode: 0600
+      contents:
+        inline: |
+          [connection]
+          id=bond0-slave-eth0
+          type=ethernet
+          interface-name=eth0
+          master=bond0
+          slave-type=bond
+    - path: /etc/NetworkManager/system-connections/bond0-slave-eth1.nmconnection
+      mode: 0600
+      contents:
+        inline: |
+          [connection]
+          id=bond0-slave-eth1
+          type=ethernet
+          interface-name=eth1
+          master=bond0
+          slave-type=bond'
+
+fcct_static_br0='
+    - path: /etc/NetworkManager/system-connections/br0.nmconnection
+      mode: 0600
+      contents:
+        inline: |
+          [connection]
+          id=br0
+          type=bridge
+          interface-name=br0
+          [bridge]
+          [ipv4]
+          address1=$ip/$prefix,$gateway
+          dns=$nameserver;
+          dns-search=
+          may-fail=false
+          method=manual
+    - path: /etc/NetworkManager/system-connections/br0-slave-eth0.nmconnection
+      mode: 0600
+      contents:
+        inline: |
+          [connection]
+          id=br0-slave-eth0
+          type=ethernet
+          interface-name=eth0
+          master=br0
+          slave-type=bridge
+          [bridge-port]
+    - path: /etc/NetworkManager/system-connections/br0-slave-eth1.nmconnection
+      mode: 0600
+      contents:
+        inline: |
+          [connection]
+          id=br0-slave-eth1
+          type=ethernet
+          interface-name=eth1
+          master=br0
+          slave-type=bridge
+          [bridge-port]'
+
+check_requirements() {
+    reqs=(
+        chcon
+        envsubst
+        fcct
+        jq
+        ssh
+        ssh-keygen
+        virsh
+        virt-cat
+        virt-install
+        virt-ls
+    )
+    for req in ${reqs[@]}; do
+        which $req &>/dev/null
+    done
+}
+
+start_vm() {
+    echo "Starting domain: fcos"
+    local disk=$1; shift
+    local ignitionfile=$1; shift
+    local kernel=$1; shift
+    local initramfs=$1; shift
+    local kernel_args=$@
+    virt-install --name fcos --ram 3096 --vcpus 2 --graphics=none --noautoconsole \
+                   --quiet --network bridge=virbr0 --network bridge=virbr0 \
+                   --disk size=20,backing_store=${disk} \
+                   --install kernel=${kernel},initrd=${initramfs},kernel_args_overwrite=yes,kernel_args="${kernel_args}" \
+                   --qemu-commandline="-fw_cfg name=opt/com.coreos/config,file=$ignitionfile"
+}
+
+check_vm() {
+    local interfaces=$1
+    local ip=$2
+    local dev=$3
+    local sshkeyfile=$4
+    local ssh_config=' -o CheckHostIP=no'
+    ssh_config+=' -o UserKnownHostsFile=/dev/null'
+    ssh_config+=' -o StrictHostKeyChecking=no'
+    ssh_config+=" -i $sshkeyfile"
+    local ssh="ssh -q $ssh_config -l core $ip"
+
+    # Wait for system to come up
+    try=10
+
+    echo 'waiting on ssh connection to come up'
+    while true; do
+        $ssh /usr/bin/true && echo && break
+        echo -n '.'
+        sleep 5
+        ((try--))
+        if [ $try -eq 0 ]; then
+            echo -e "\nTimeout while trying to reach $ip" 2>&1
+            return 1
+        fi
+    done
+    # The output gives us something like:
+    #
+    #   ip -j -4 -o address show up | jq .
+    #   [
+    #     {
+    #       "addr_info": [
+    #         {
+    #           "index": 1,
+    #           "dev": "lo",
+    #           "family": "inet",
+    #           "local": "127.0.0.1",
+    #           "prefixlen": 8,
+    #           "scope": "host",
+    #           "label": "lo",
+    #           "valid_life_time": 4294967295,
+    #           "preferred_life_time": 4294967295
+    #         }
+    #       ]
+    #     },
+    #     {
+    #       "addr_info": [
+    #         {
+    #           "index": 5,
+    #           "dev": "bond0",
+    #           "family": "inet",
+    #           "local": "192.168.122.111",
+    #           "prefixlen": 24,
+    #           "broadcast": "192.168.122.255",
+    #           "scope": "global",
+    #           "noprefixroute": true,
+    #           "label": "bond0",
+    #           "valid_life_time": 4294967295,
+    #           "preferred_life_time": 4294967295
+    #         }
+    #       ]
+    #     }
+    #   ]
+    ipinfo=$($ssh ip -j -4 -o address show up)
+    rc=0
+
+    # verify that there are the right number of ipv4 devices "up"
+    if [ $(jq length <<< $ipinfo) != "$((interfaces+1))" ]; then
+        rc=1
+        echo "ERROR: More interfaces up than expected" 1>&2 
+    fi
+    # verify that the first one in loopback
+    if [ $(jq -r .[0].addr_info[0].dev <<< $ipinfo) != 'lo' ]; then
+        rc=1
+        echo "ERROR: The first active interface is not 'lo'" 1>&2 
+    fi
+    # verify that the second one is the expected device
+    if [ $(jq -r .[1].addr_info[0].dev <<< $ipinfo) != "${dev}" ]; then
+        rc=1
+        echo "ERROR: The second active interface is not ${dev}" 1>&2 
+    fi
+    # verify that the second one has the IP we assigned
+    if [ $(jq -r .[1].addr_info[0].local <<< $ipinfo) != "${ip}" ]; then
+        rc=1
+        echo "ERROR: The second active interface does not have expected ip" 1>&2 
+    fi
+
+    if [ "$rc" != '0' ]; then
+        jq -r .[].addr_info[].dev 1>&2 <<< $ipinfo
+        jq -r .[].addr_info[].local 1>&2 <<< $ipinfo
+        true
+    else
+        echo "Networking check for ${dev}/${ip} passed!"
+    fi
+    return $rc
+    #$ssh sudo nmcli connection show
+}
+
+reboot_vm() {
+    echo "Rebooting domain: fcos"
+    # The reboot after a virt-install --install will not boot the VM
+    # back up so `virsh reboot` && `virsh start`
+    virsh reboot fcos 1>/dev/null
+    sleep 5
+    virsh start fcos 1>/dev/null
+}
+
+destroy_vm() {
+    echo "Destroying domain: fcos"
+    # If the domain doesn't exist then return
+    virsh dominfo fcos &>/dev/null || return 0
+    # Destroy domain and recover storage
+    virsh destroy fcos 1>/dev/null
+    virsh undefine --nvram --remove-all-storage fcos 1>/dev/null
+}
+
+
+
+main() {
+    qcow=$1
+    local ip='192.168.122.111'
+    local netmask='255.255.255.0'
+    local prefix='24'
+    local gateway='192.168.122.1'
+    local nameserver='192.168.122.1'
+    local kernel="${PWD}/fcos-nettest-kernel"
+    local initramfs="${PWD}/fcos-nettest-initramfs"
+    local sshkeyfile="${PWD}/fcos-nettest-sshkey"
+    local sshpubkeyfile="${PWD}/fcos-nettest-sshkey.pub"
+    local ignitionfile="${PWD}/fcos-nettest-config.ign"
+    local sshpubkey
+    local fcct
+     
+    check_requirements
+
+    # generate an ssh key to use:
+    rm -f $sshkeyfile $sshpubkeyfile
+    ssh-keygen -N '' -C '' -f $sshkeyfile &>/dev/null
+    sshpubkey=$(cat $sshpubkeyfile)
+
+    # export these values so we can substitute the values
+    # in using the envsubst command
+    export ip prefix nameserver gateway sshpubkey
+
+    # Grab kernel/initramfs from the disk
+    files=$(virt-ls -a $qcow -m /dev/sda1 -R /ostree/)
+    for f in $files; do
+        if [[ "${f}" =~ hmac$ ]]; then
+            # ignore .vmlinuz-5.5.9-200.fc31.x86_64.hmac
+            true
+        elif [[ "${f}" =~ img$ ]]; then
+            # grab initramfs in the form initramfs-5.5.9-200.fc31.x86_64.img
+            virt-cat -a $qcow -m /dev/sda1 "/ostree/${f}" > $initramfs
+        elif [[ "${f}" =~ '/vmlinuz' ]]; then
+            # grab kernel in the form vmlinuz-5.5.9-200.fc31.x86_64
+            virt-cat -a $qcow -m /dev/sda1 "/ostree/${f}" > $kernel
+        fi
+    done
+
+    # Grab kernel arguments from the disk and use them
+    # - strip `options ` from the front of the line
+    # - strip `$ignition_firstboot`
+    common_args=$(virt-cat -a $qcow -m /dev/sda1 /loader.1/entries/ostree-1-fedora-coreos.conf | \
+                  grep -P '^options' | \
+                  sed -e 's/options //' | \
+                  sed -e 's/$ignition_firstboot//')
+    common_args+=' ignition.firstboot' # manually set ignition.firstboot
+   #common_args+=' rd.break=pre-pivot'
+
+    # nameserver= doesn't work as I would expect
+    # https://gitlab.freedesktop.org/NetworkManager/NetworkManager/issues/391
+    devname=eth0
+    x="${common_args} rd.neednet=1 ip=eth0:dhcp"
+    initramfs_dhcp_eth0=$x
+
+    devname=eth0
+    x="${common_args} rd.neednet=1 ip=eth1:off"
+    x+=" ip=${ip}::${gateway}:${netmask}:fcos:${devname}:none:${nameserver}"
+    initramfs_static_eth0=$x
+
+    devname=bond0
+    x="${common_args} rd.neednet=1"
+    x+=" ip=${ip}::${gateway}:${netmask}:fcos:${devname}:none:${nameserver}"
+    x+=" bond=${devname}:eth0,eth1:mode=active-backup,miimon=100"
+    initramfs_static_bond0=$x
+
+    devname=team0
+    x="${common_args} rd.neednet=1"
+    x+=" ip=${ip}::${gateway}:${netmask}:fcos:${devname}:none:${nameserver}"
+    x+=" team=${devname}:eth0,eth1"
+    initramfs_static_team0=$x
+
+    devname=br0
+    x="${common_args} rd.neednet=1"
+    x+=" ip=${ip}::${gateway}:${netmask}:fcos:${devname}:none:${nameserver}"
+    x+=" bridge=${devname}:eth0,eth1"
+    initramfs_static_br0=$x
+
+    fcct_none=$(echo "${fcct_common}" | envsubst)
+    fcct_static_eth0=$(echo "${fcct_common}${fcct_static_eth0}" | envsubst)
+    fcct_static_bond0=$(echo "${fcct_common}${fcct_static_bond0}" | envsubst)
+    fcct_static_team0=$(echo "${fcct_common}${fcct_static_team0}" | envsubst)
+    fcct_static_br0=$(echo "${fcct_common}${fcct_static_br0}" | envsubst)
+
+    # If the VM is still around for whatever reason, destroy it
+    destroy_vm || true
+
+    # Note 'static_team0' initramfs teaming doesn't work so leave it out for now
+    # https://bugzilla.redhat.com/show_bug.cgi?id=1814038#c1
+    # https://bugzilla.redhat.com/show_bug.cgi?id=1784363
+    initramfsloop=(
+        dhcp_eth0
+        static_eth0
+        static_bond0
+       #static_team0
+        static_br0
+    )
+
+    fcctloop=(
+        none
+        static_eth0
+        static_bond0
+        static_team0
+        static_br0
+    )
+        
+    for fcctnet in ${fcctloop[@]}; do
+    for initramfsnet in ${initramfsloop[@]}; do
+            if [ "${fcctnet}" == 'none' ]; then
+                # because we propagate initramfs networking if no real root networking 
+                devname=${initramfsnet##*_}
+                # If we're using dhcp for initramfs and not providing any real root 
+                # networking config then we can't predict the IP. Skip it
+                [ "${initramfsnet}" == 'dhcp_eth0' ] && continue
+            else
+                devname=${fcctnet##*_}
+            fi
+            fcctvar="fcct_${fcctnet}"
+            fcctconfig=${!fcctvar}
+            initramfsvar="initramfs_${initramfsnet}"
+            kernel_args=${!initramfsvar}
+
+            echo -e "\n###### Testing initramfs: ${initramfsnet} + ignition/fcct: ${fcctnet}\n"
+
+            echo "$fcctconfig" | fcct --strict --output $ignitionfile
+            chcon --verbose unconfined_u:object_r:svirt_home_t:s0 $ignitionfile &>/dev/null
+            start_vm $qcow $ignitionfile $kernel $initramfs "${kernel_args}"
+            check_vm 1 $ip $devname $sshkeyfile
+            reboot_vm
+            check_vm 1 $ip $devname $sshkeyfile
+            destroy_vm
+        done
+    done
+}
+
+
+main $@
+