Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Test selecting disk by uuid+label #2877

Merged
merged 13 commits into from
Sep 25, 2024
9 changes: 9 additions & 0 deletions .github/workflows/image-pr.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,15 @@ jobs:
needs:
- core-ubuntu-24-lts

install-target:
uses: ./.github/workflows/reusable-install-test-target.yaml
with:
flavor: ubuntu
flavor_release: "24.04"
secureboot: false
needs:
- core-ubuntu-24-lts

install-secureboot:
uses: ./.github/workflows/reusable-install-test.yaml
secrets: inherit
Expand Down
15 changes: 15 additions & 0 deletions .github/workflows/image.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,19 @@ jobs:
- flavor: opensuse
flavorRelease: leap-15.6
secureboot: false
install-target:
uses: ./.github/workflows/reusable-install-test-target.yaml
with:
flavor: ${{ matrix.flavor }}
flavor_release: ${{ matrix.flavorRelease }}
needs:
- core
strategy:
fail-fast: true
matrix:
include:
- flavor: "ubuntu"
flavorRelease: "24.04"
install-secureboot:
uses: ./.github/workflows/reusable-install-test.yaml
with:
Expand Down Expand Up @@ -413,6 +426,8 @@ jobs:
- core
- standard
- install
- install-target
- install-secureboot
- zfs
- acceptance
- bundles
Expand Down
75 changes: 75 additions & 0 deletions .github/workflows/reusable-install-test-target.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
name: Reusable workflow - install test target

on:
workflow_call:
inputs:
flavor:
required: true
type: string
flavor_release:
required: true
type: string
secureboot:
required: false
type: boolean

permissions: read-all
jobs:
test:
runs-on: kvm
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
- run: |
git fetch --prune --unshallow
- name: Download ISO
id: iso
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
with:
name: kairos-${{ inputs.flavor }}-${{ inputs.flavor_release}}.iso.zip
- name: Install Go
uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5
with:
go-version-file: tests/go.mod
cache-dependency-path: tests/go.sum
- name: Block all traffic to metadata ip # For cloud runners, the metadata ip can interact with our test machines
run: |
sudo iptables -I INPUT -s 169.254.169.254 -j DROP
sudo iptables -I OUTPUT -d 169.254.169.254 -j DROP
- name: Enable KVM group perms
run: |
sudo apt-get update
sudo apt-get install -y libvirt-clients libvirt-daemon-system libvirt-daemon virtinst bridge-utils qemu qemu-system-x86 qemu-system-x86 qemu-utils qemu-kvm acl udev

# https://github.blog/changelog/2023-02-23-hardware-accelerated-android-virtualization-on-actions-windows-and-linux-larger-hosted-runners/
# echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules
# sudo udevadm control --reload-rules
# sudo udevadm trigger --name-match=kvm
# sudo usermod -a -G kvm,libvirt $USER
#
# TODO: Switch back to the above solution when we switch to the github runners
# https://askubuntu.com/a/1081326
sudo setfacl -m u:runner:rwx /dev/kvm
- name: Tests
env:
USE_QEMU: true
KVM: true
MEMORY: 4000
CPUS: 2
DRIVE_SIZE: 30000
CREATE_VM: true
FLAVOR: ${{ inputs.flavor }}
run: |
if [ "${{ inputs.secureboot }}" = "true" ]; then
export FIRMWARE=/usr/share/OVMF/OVMF_CODE.fd
fi
ls *.iso
export ISO=$PWD/$(ls *.iso)
echo "ISO is: $ISO"
cp tests/go.* .
go run github.com/onsi/ginkgo/v2/ginkgo -v --label-filter "install-test-target" --fail-fast -r ./tests
- uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4
if: failure()
with:
name: ${{ inputs.flavor }}.logs.zip
path: tests/**/logs/*
if-no-files-found: warn
167 changes: 167 additions & 0 deletions tests/install_target_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,167 @@
package mos_test

import (
"context"
"fmt"
"github.com/google/uuid"
. "github.com/spectrocloud/peg/matcher"
"github.com/spectrocloud/peg/pkg/machine"
"github.com/spectrocloud/peg/pkg/machine/types"
"os"
"strings"
"time"

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)

var _ = Describe("kairos install test different targets", Label("install-test-target"), func() {

var vm VM
var label string
var diskUUID uuid.UUID
BeforeEach(func() {
label = "TESTDISK"
diskUUID = uuid.New()
stateDir, err := os.MkdirTemp("", "")
Expect(err).ToNot(HaveOccurred())
fmt.Printf("State dir: %s\n", stateDir)

opts := defaultVMOptsNoDrives(stateDir)
opts = append(opts, types.WithDriveSize("40000"))
opts = append(opts, types.WithDriveSize("30000"))

m, err := machine.New(opts...)
Expect(err).ToNot(HaveOccurred())
vm = NewVM(m, stateDir)
_, err = vm.Start(context.Background())
Expect(err).ToNot(HaveOccurred())

vm.EventuallyConnects(1200)
// Format the first disk so it gets an uuid and label
_, err = vm.Sudo(fmt.Sprintf("mkfs.ext4 -L %s -U %s /dev/vda", label, diskUUID.String()))
Expect(err).NotTo(HaveOccurred())
})

AfterEach(func() {
Expect(vm.Destroy(nil)).ToNot(HaveOccurred())
})

// TODO: Install on second disk instead of first and check that it worked.
// Missing the bootindex check, it will only try to boot from the first disk it seems
Context("Selects the disk by uuid/label", func() {
It("Selects the correct disk if using uuid for target", func() {
expectSecureBootEnabled(vm)

err := vm.Scp("assets/config.yaml", "/tmp/config.yaml", "0770")
Expect(err).ToNot(HaveOccurred())

var out string
// Test that install works
By("installing kairos", func() {
installCmd := fmt.Sprintf("kairos-agent --debug manual-install --device /dev/disk/by-uuid/%s /tmp/config.yaml", diskUUID.String())
By(fmt.Sprintf("Running %s", installCmd))
out, err = vm.Sudo(installCmd)
Expect(err).ToNot(HaveOccurred(), out)
Expect(out).Should(ContainSubstring("Running after-install hook"))
vm.Sudo("sync")
})

By("waiting for VM to reboot", func() {

_, _ = vm.Sudo("reboot")
Expect(vm.DetachCD()).ToNot(HaveOccurred())
vm.EventuallyConnects(1200)
})

By("checking that vm has rebooted to 'active'", func() {
Eventually(func() string {
out, _ := vm.Sudo("kairos-agent state boot")
return out
}, 40*time.Minute, 10*time.Second).Should(
Or(
ContainSubstring("active_boot"),
))
})

By("checking corresponding state", func() {
currentVersion, err := vm.Sudo(getVersionCmd)
Expect(err).ToNot(HaveOccurred(), currentVersion)

stateAssertVM(vm, "boot", "active_boot")
stateAssertVM(vm, "oem.mounted", "true")
stateAssertVM(vm, "oem.found", "true")
stateAssertVM(vm, "persistent.mounted", "true")
stateAssertVM(vm, "state.mounted", "true")
stateAssertVM(vm, "oem.type", "ext4")
stateAssertVM(vm, "persistent.type", "ext4")
stateAssertVM(vm, "state.type", "ext4")
stateAssertVM(vm, "oem.mount_point", "/oem")
stateAssertVM(vm, "persistent.mount_point", "/usr/local")
stateAssertVM(vm, "persistent.name", "/dev/vda5")
stateAssertVM(vm, "state.mount_point", "/run/initramfs/cos-state")
stateAssertVM(vm, "oem.read_only", "false")
stateAssertVM(vm, "persistent.read_only", "false")
stateAssertVM(vm, "state.read_only", "true")
stateAssertVM(vm, "kairos.version", strings.ReplaceAll(strings.ReplaceAll(currentVersion, "\r", ""), "\n", ""))
stateContains(vm, "system.os.name", "alpine", "opensuse", "ubuntu", "debian")
stateContains(vm, "kairos.flavor", "alpine", "opensuse", "ubuntu", "debian")
})
})
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't see any check that the correct disk was actually used. We only check that the command didn't fail and that installation occurred. I guess, since we only have one disk, it was indeed selected but would be better if we had 2 disks and we made sure the correct one was selected, to avoid for example the flag being completely ignored and the default disk (bigger one?) gets selected instead.

But it can be tricky:

It("installs on the pre-configured disks", func() {
Expect(installError).ToNot(HaveOccurred(), installationOutput)
By("Rebooting into live CD again")
// In qemu it's tricky to boot the second disk. In multiple disk scenarios,
// setting "-boot=cd" will make qemu try to boot from the first disk and
// then from the cdrom.
// We want to make sure that kairos-agent selected the second disk so we
// simply let it boot from the cdrom again. Hopefully if the installation
// failed, we would see the error from the manual-install command.

I don't have a strong opinion, I'll let you decide.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes indeed, that's a good point. I am letting it work as is because if you pass a wrong or nonexistent disk, the install fails but I'll have a look at peg to see if we can pass multiple disks to the VM, that would make it more robust

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I introduced it for this exact reason. It's the boot order that is hard to do in qemu I think (see the link above).

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

but boot order should not matter no? It will try first all the disks and then livecd? Or it just tries the first disk only?

Copy link
Contributor

@jimmykarily jimmykarily Sep 25, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't recall to be honest. Maybe back then I was no aware of how to setup boot order using bootindex in qemu. We need to look at it again.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You are rigth, it doesnt try to boot from the second one somehow 😭

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I could not make it work, maybe we should introduce a config in peg to choose the boot device or something, would be difficult to have it working for all, but at least we could make it work for qemu

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe we can add a default bootindex to all disks in qemu in peg, as in: first check disks (/dev/vda, /dev/vdb, etc) then cdrom.
This should allow us to try to target the second disk and see if it worked (since vda will be skipped as not being bootable).

It("Selects the correct disk if using label for target", func() {
expectSecureBootEnabled(vm)

err := vm.Scp("assets/config.yaml", "/tmp/config.yaml", "0770")
Expect(err).ToNot(HaveOccurred())

var out string
By("installing kairos", func() {
installCmd := fmt.Sprintf("kairos-agent --debug manual-install --device /dev/disk/by-label/%s /tmp/config.yaml", label)
By(fmt.Sprintf("Running %s", installCmd))
out, err = vm.Sudo(installCmd)
Expect(err).ToNot(HaveOccurred(), out)
Expect(out).Should(ContainSubstring("Running after-install hook"))
vm.Sudo("sync")
})

By("waiting for VM to reboot", func() {
vm.Reboot()
vm.EventuallyConnects(1200)
})

By("checking that vm has rebooted to 'active'", func() {
Eventually(func() string {
out, _ := vm.Sudo("kairos-agent state boot")
return out
}, 40*time.Minute, 10*time.Second).Should(
Or(
ContainSubstring("active_boot"),
))
})

By("checking corresponding state", func() {
currentVersion, err := vm.Sudo(getVersionCmd)
Expect(err).ToNot(HaveOccurred(), currentVersion)

stateAssertVM(vm, "boot", "active_boot")
stateAssertVM(vm, "oem.mounted", "true")
stateAssertVM(vm, "oem.found", "true")
stateAssertVM(vm, "persistent.mounted", "true")
stateAssertVM(vm, "state.mounted", "true")
stateAssertVM(vm, "oem.type", "ext4")
stateAssertVM(vm, "persistent.type", "ext4")
stateAssertVM(vm, "state.type", "ext4")
stateAssertVM(vm, "oem.mount_point", "/oem")
stateAssertVM(vm, "persistent.mount_point", "/usr/local")
stateAssertVM(vm, "persistent.name", "/dev/vda5")
stateAssertVM(vm, "state.mount_point", "/run/initramfs/cos-state")
stateAssertVM(vm, "oem.read_only", "false")
stateAssertVM(vm, "persistent.read_only", "false")
stateAssertVM(vm, "state.read_only", "true")
stateAssertVM(vm, "kairos.version", strings.ReplaceAll(strings.ReplaceAll(currentVersion, "\r", ""), "\n", ""))
stateContains(vm, "system.os.name", "alpine", "opensuse", "ubuntu", "debian")
stateContains(vm, "kairos.flavor", "alpine", "opensuse", "ubuntu", "debian")
})
})
})
})
Loading