From f98481f1f5ad16f278081fd877d8bdfe85a90619 Mon Sep 17 00:00:00 2001 From: Renata Andrade Matos Ravanelli Date: Wed, 11 Jan 2023 23:57:00 +0000 Subject: [PATCH] kola: Refactor testiso tests - This is an initial step in order to try to merge kola tests and testiso, much more work will be needed to create a final solution. Nonetheless, for now let's start by breaking the tests and adding the same denylist for both. - kola testiso now allows: - multipath, 4k, iso, pxe and firmware explicit tests; - Denylist for the pattern above and by test. Example: Add the following patterns in `kola-denylist.yaml` to deny it: - pattern: iso-as-disk.* - pattern: iso-as-iso-live-login.*.4k - pattern: iso-offline-install.ppcfw.* Signed-off-by: Renata Andrade Matos Ravanelli --- mantle/cmd/kola/testiso.go | 387 ++++++++++++++++++------------------- 1 file changed, 191 insertions(+), 196 deletions(-) diff --git a/mantle/cmd/kola/testiso.go b/mantle/cmd/kola/testiso.go index 6d10b22fdb..93c23d6052 100644 --- a/mantle/cmd/kola/testiso.go +++ b/mantle/cmd/kola/testiso.go @@ -29,6 +29,7 @@ import ( "strings" "time" + "github.com/coreos/coreos-assembler/mantle/harness" "github.com/coreos/coreos-assembler/mantle/platform/conf" "github.com/coreos/coreos-assembler/mantle/util" coreosarch "github.com/coreos/stream-metadata-go/arch" @@ -44,7 +45,7 @@ var ( cmdTestIso = &cobra.Command{ RunE: runTestIso, PreRunE: preRun, - Use: "testiso", + Use: "testiso [glob pattern...]", Short: "Test a CoreOS PXE boot or ISO install path", SilenceUsage: true, @@ -52,46 +53,79 @@ var ( instInsecure bool - nopxe bool - noiso bool - - scenarios []string - pxeAppendRootfs bool pxeKernelArgs []string console bool - addNmKeyfile bool + addNmKeyfile bool + enable4k bool + enableMultipath bool + enableUefi bool + enableUefiSecure bool + isOffline bool + + // The iso-as-disk tests are only supported in x86_64 because other + // architectures don't have the required hybrid partition table. + tests_x86_64 = []string{ + "iso-as-disk.bios", + "iso-as-disk.uefi", + "iso-as-disk.uefi-secure", + "iso-as-disk.4k.uefi", + "iso-install.bios", + "iso-live-login.bios", + "iso-live-login.uefi", + "iso-live-login.uefi-secure", + "iso-live-login.4k.uefi", + "iso-offline-install.bios", + "iso-offline-install.mpath.bios", + "iso-offline-install.4k.uefi", + "miniso-install.bios", + "miniso-install.nm.bios.", + "miniso-install.4k.uefi", + "miniso-install.4k.nm.uefi", + "pxe-offline-install.bios", + "pxe-offline-install.4k.uefi", + "pxe-online-install.bios", + "pxe-online-install.4k.uefi", + } + tests_s390x = []string{ + "pxe-online-install.s390fw", + "pxe-offline-install.s390fw", + } + tests_ppc64le = []string{ + "iso-live-login.ppcfw", + "iso-offline-install.ppcfw", + "iso-offline-install.mpath.ppcfw", + "iso-offline-install.4k.ppcfw", + "miniso-install.ppcfw", + "miniso-install.nm.ppcfw", + "miniso-install.4k.ppcfw", + "miniso-install.4k.nm.ppcfw", + "pxe-online-install.ppcfw", + "pxe-offline-install.4k.ppcfw", + } + tests_aarch64 = []string{ + "iso-live-login.uefi", + "iso-live-login.4k.uefi", + "iso-offline-install.uefi", + "iso-offline-install.mpath.uefi", + "iso-offline-install.4k.uefi", + "miniso-install.uefi", + "miniso-install.nm.uefi", + "miniso-install.4k.uefi", + "miniso-install.4k.nm.uefi", + "pxe-offline-install.uefi", + "pxe-offline-install.4k.uefi", + "pxe-online-install.uefi", + "pxe-online-install.4k.uefi", + } ) const ( installTimeout = 10 * time.Minute - - scenarioPXEInstall = "pxe-install" - scenarioISOInstall = "iso-install" - - scenarioMinISOInstall = "miniso-install" - scenarioMinISOInstallNm = "miniso-install-nm" - - scenarioPXEOfflineInstall = "pxe-offline-install" - scenarioISOOfflineInstall = "iso-offline-install" - - scenarioISOLiveLogin = "iso-live-login" - scenarioISOAsDisk = "iso-as-disk" ) -var allScenarios = map[string]bool{ - scenarioPXEInstall: true, - scenarioPXEOfflineInstall: true, - scenarioISOInstall: true, - scenarioISOOfflineInstall: true, - scenarioMinISOInstall: true, - scenarioMinISOInstallNm: true, - scenarioISOLiveLogin: true, - scenarioISOAsDisk: true, -} - var liveOKSignal = "live-test-OK" var liveSignalOKUnit = fmt.Sprintf(`[Unit] Description=TestISO Signal Live ISO Completion @@ -104,7 +138,7 @@ Type=oneshot RemainAfterExit=yes ExecStart=/bin/sh -c '/usr/bin/echo %s >/dev/virtio-ports/testisocompletion' [Install] -# for install scenarios +# for install tests RequiredBy=coreos-installer.target # for iso-as-disk RequiredBy=multi-user.target @@ -191,7 +225,7 @@ Type=oneshot RemainAfterExit=yes ExecStart=/bin/sh -c '! efibootmgr -v | grep -E "(HD|CDROM)\("' [Install] -# for install scenarios +# for install tests RequiredBy=coreos-installer.target # for iso-as-disk RequiredBy=multi-user.target` @@ -233,21 +267,44 @@ RequiredBy=multi-user.target`, nmConnectionId, nmConnectionFile) func init() { cmdTestIso.Flags().BoolVarP(&instInsecure, "inst-insecure", "S", false, "Do not verify signature on metal image") - cmdTestIso.Flags().BoolVarP(&nopxe, "no-pxe", "P", false, "Skip testing live installer PXE") - cmdTestIso.Flags().BoolVarP(&noiso, "no-iso", "", false, "Skip testing live installer ISO") cmdTestIso.Flags().BoolVar(&console, "console", false, "Connect qemu console to terminal, turn off automatic initramfs failure checking") cmdTestIso.Flags().BoolVar(&pxeAppendRootfs, "pxe-append-rootfs", false, "Append rootfs to PXE initrd instead of fetching at runtime") cmdTestIso.Flags().StringSliceVar(&pxeKernelArgs, "pxe-kargs", nil, "Additional kernel arguments for PXE") - cmdTestIso.Flags().BoolVar(&addNmKeyfile, "add-nm-keyfile", false, "Add NetworkManager connection keyfile") - cmdTestIso.Flags().StringSliceVar(&scenarios, "scenarios", []string{scenarioPXEInstall, scenarioISOOfflineInstall, scenarioPXEOfflineInstall, scenarioISOLiveLogin, scenarioISOAsDisk, scenarioMinISOInstall, scenarioMinISOInstallNm}, fmt.Sprintf("Test scenarios (also available: %v)", []string{scenarioISOInstall})) - cmdTestIso.Args = cobra.ExactArgs(0) root.AddCommand(cmdTestIso) } +func liveArtifactExistsInBuild() error { + + if kola.CosaBuild.Meta.BuildArtifacts.LiveIso == nil || kola.CosaBuild.Meta.BuildArtifacts.LiveKernel == nil { + return fmt.Errorf("build %s is missing live artifacts", kola.CosaBuild.Meta.Name) + } + return nil +} + +func getArchPatternsList() []string { + arch := coreosarch.CurrentRpmArch() + var tests []string + switch arch { + case "x86_64": + tests = tests_x86_64 + case "ppc64le": + tests = tests_ppc64le + case "s390x": + tests = tests_s390x + case "aarch64": + tests = tests_aarch64 + } + return tests +} + func newBaseQemuBuilder(outdir string) (*platform.QemuBuilder, error) { builder := platform.NewMetalQemuBuilderDefault() - builder.Firmware = kola.QEMUOptions.Firmware + if enableUefiSecure { + builder.Firmware = "uefi-secure" + } else if enableUefi { + builder.Firmware = "uefi" + } if err := os.MkdirAll(outdir, 0755); err != nil { return nil, err @@ -292,20 +349,20 @@ func newQemuBuilder(outdir string) (*platform.QemuBuilder, *conf.Conf, error) { func newQemuBuilderWithDisk(outdir string) (*platform.QemuBuilder, *conf.Conf, error) { builder, config, err := newQemuBuilder(outdir) + if err != nil { return nil, nil, err } sectorSize := 0 - if kola.QEMUOptions.Native4k { + if enable4k { sectorSize = 4096 } disk := platform.Disk{ - Size: "12G", // Arbitrary - SectorSize: sectorSize, - - MultiPathDisk: kola.QEMUOptions.MultiPathDisk, + Size: "12G", // Arbitrary + SectorSize: sectorSize, + MultiPathDisk: enableMultipath, } //TBD: see if we can remove this and just use AddDisk and inject bootindex during startup @@ -323,57 +380,56 @@ func newQemuBuilderWithDisk(outdir string) (*platform.QemuBuilder, *conf.Conf, e return builder, config, nil } -func runTestIso(cmd *cobra.Command, args []string) error { - if kola.CosaBuild == nil { - return fmt.Errorf("Must provide --build") +// See similar semantics in the `filterTests` of `kola.go`. +func filterTests(tests []string, patterns []string) ([]string, error) { + r := []string{} + for _, test := range tests { + if matches, err := kola.MatchesPatterns(test, patterns); err != nil { + return nil, err + } else if matches { + r = append(r, test) + } } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + return r, nil +} - targetScenarios := make(map[string]bool) - for _, scenario := range scenarios { - if _, ok := allScenarios[scenario]; !ok { - return fmt.Errorf("Unknown scenario: %s", scenario) +func runTestIso(cmd *cobra.Command, args []string) error { + var err error + tests := getArchPatternsList() + if len(args) != 0 { + if tests, err = filterTests(tests, args); err != nil { + return err + } else if len(tests) == 0 { + return harness.SuiteEmpty } - targetScenarios[scenario] = true } - // Only x86_64 supports hybrid ISOs. - if coreosarch.CurrentRpmArch() != "x86_64" { - fmt.Printf("Skipping iso-as-disk; not supported on %s\n", coreosarch.CurrentRpmArch()) - delete(targetScenarios, scenarioISOAsDisk) + if kola.CosaBuild == nil { + return fmt.Errorf("Must provide --build") } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() - // s390x: iso-install does not work because s390x uses an El Torito image - if coreosarch.CurrentRpmArch() == "s390x" { - fmt.Println("Skipping iso-install on s390x") - noiso = true + // Call `ParseDenyListYaml` to populate the `kola.DenylistedTests` var + err = kola.ParseDenyListYaml("qemu") + if err != nil { + plog.Fatal(err) } - if nopxe { - delete(targetScenarios, scenarioPXEInstall) - delete(targetScenarios, scenarioPXEOfflineInstall) - } - if noiso { - delete(targetScenarios, scenarioISOInstall) - delete(targetScenarios, scenarioISOOfflineInstall) - delete(targetScenarios, scenarioMinISOInstall) - delete(targetScenarios, scenarioMinISOInstallNm) - delete(targetScenarios, scenarioISOLiveLogin) - } + finalTests := []string{} + for _, test := range tests { + if !kola.HasString(test, kola.DenylistedTests) { + matchTest, err := kola.MatchesPatterns(test, kola.DenylistedTests) + if err != nil { + return err - // just make it a normal print message so pipelines don't error out for ppc64le and s390x - if len(targetScenarios) == 0 { - fmt.Println("No valid scenarios specified!") - return nil - } - scenarios = []string{} - for scenario := range targetScenarios { - scenarios = append(scenarios, scenario) + } + if !matchTest { + finalTests = append(finalTests, test) + } + } } - fmt.Printf("Testing scenarios: %s\n", scenarios) - var err error // note this reassigns a *global* outputDir, err = kola.SetupOutputDir(outputDir, "testiso") if err != nil { @@ -382,8 +438,6 @@ func runTestIso(cmd *cobra.Command, args []string) error { baseInst := platform.Install{ CosaBuild: kola.CosaBuild, - Native4k: kola.QEMUOptions.Native4k, - MultiPathDisk: kola.QEMUOptions.MultiPathDisk, PxeAppendRootfs: pxeAppendRootfs, NmKeyfiles: make(map[string]string), } @@ -400,116 +454,69 @@ func runTestIso(cmd *cobra.Command, args []string) error { fmt.Printf("Detected development build; disabling signature verification\n") } - ranTest := false + var duration time.Duration - if _, ok := targetScenarios[scenarioPXEInstall]; ok { - if kola.CosaBuild.Meta.BuildArtifacts.LiveKernel == nil { - return fmt.Errorf("build %s has no live installer kernel", kola.CosaBuild.Meta.Name) - } + atLeastOneFailed := false + for _, test := range finalTests { - ranTest = true - instPxe := baseInst // Pretend this is Rust and I wrote .copy() - - duration, err := testPXE(ctx, instPxe, filepath.Join(outputDir, scenarioPXEInstall), false) - printResult(scenarioPXEInstall, duration, err) + // All of these tests require buildextend-live to have been run + err = liveArtifactExistsInBuild() if err != nil { return err } - } - if _, ok := targetScenarios[scenarioPXEOfflineInstall]; ok { - if kola.CosaBuild.Meta.BuildArtifacts.LiveKernel == nil { - return fmt.Errorf("build %s has no live installer kernel", kola.CosaBuild.Meta.Name) - } - ranTest = true - instPxe := baseInst // Pretend this is Rust and I wrote .copy() + addNmKeyfile = false + enable4k = false + enableMultipath = false + enableUefi = false + enableUefiSecure = false + isOffline = false + inst := baseInst // Pretend this is Rust and I wrote .copy() - duration, err := testPXE(ctx, instPxe, filepath.Join(outputDir, scenarioPXEOfflineInstall), true) - printResult(scenarioPXEOfflineInstall, duration, err) - if err != nil { - return err - } - } - if _, ok := targetScenarios[scenarioISOInstall]; ok { - if kola.CosaBuild.Meta.BuildArtifacts.LiveIso == nil { - return fmt.Errorf("build %s has no live ISO", kola.CosaBuild.Meta.Name) - } - ranTest = true - instIso := baseInst // Pretend this is Rust and I wrote .copy() - duration, err := testLiveIso(ctx, instIso, filepath.Join(outputDir, scenarioISOInstall), false, false) - printResult(scenarioISOInstall, duration, err) - if err != nil { - return err - } - } - if _, ok := targetScenarios[scenarioISOOfflineInstall]; ok { - if kola.CosaBuild.Meta.BuildArtifacts.LiveIso == nil { - return fmt.Errorf("build %s has no live ISO", kola.CosaBuild.Meta.Name) + fmt.Printf("Running test: %s\n", test) + components := strings.Split(test, ".") + + if kola.HasString("4k", components) { + enable4k = true + inst.Native4k = true } - ranTest = true - instIso := baseInst // Pretend this is Rust and I wrote .copy() - duration, err := testLiveIso(ctx, instIso, filepath.Join(outputDir, scenarioISOOfflineInstall), true, false) - printResult(scenarioISOOfflineInstall, duration, err) - if err != nil { - return err + if kola.HasString("nm", components) { + addNmKeyfile = true } - } - if _, ok := targetScenarios[scenarioISOLiveLogin]; ok { - if kola.CosaBuild.Meta.BuildArtifacts.LiveIso == nil { - return fmt.Errorf("build %s has no live ISO", kola.CosaBuild.Meta.Name) + if kola.HasString("mpath", components) { + enableMultipath = true + inst.MultiPathDisk = true } - ranTest = true - duration, err := testLiveLogin(ctx, filepath.Join(outputDir, scenarioISOLiveLogin)) - printResult(scenarioISOLiveLogin, duration, err) - if err != nil { - return err + if kola.HasString("uefi-secure", components) { + enableUefiSecure = true + } else if kola.HasString("uefi", components) { + enableUefi = true } - } - if _, ok := targetScenarios[scenarioISOAsDisk]; ok { - if kola.CosaBuild.Meta.BuildArtifacts.LiveIso == nil { - return fmt.Errorf("build %s has no live ISO", kola.CosaBuild.Meta.Name) + if kola.HasString("offline", components) { + isOffline = true } - switch coreosarch.CurrentRpmArch() { - case "x86_64": - ranTest = true - duration, err := testAsDisk(ctx, filepath.Join(outputDir, scenarioISOAsDisk)) - printResult(scenarioISOAsDisk, duration, err) - if err != nil { - return err - } + + switch components[0] { + case "pxe-offline-install", "pxe-online-install": + duration, err = testPXE(ctx, inst, filepath.Join(outputDir, test)) + case "iso-as-disk": + duration, err = testAsDisk(ctx, filepath.Join(outputDir, test)) + case "iso-live-login": + duration, err = testLiveLogin(ctx, filepath.Join(outputDir, test)) + case "iso-install", "iso-offline-install": + duration, err = testLiveIso(ctx, inst, filepath.Join(outputDir, test), false) + case "miniso-install": + duration, err = testLiveIso(ctx, inst, filepath.Join(outputDir, test), true) default: - // no hybrid partition table to boot from - fmt.Printf("%s unsupported on %s; skipping\n", scenarioISOAsDisk, coreosarch.CurrentRpmArch()) - } - } - if _, ok := targetScenarios[scenarioMinISOInstall]; ok { - if kola.CosaBuild.Meta.BuildArtifacts.LiveIso == nil { - return fmt.Errorf("build %s has no live ISO", kola.CosaBuild.Meta.Name) - } - ranTest = true - instIso := baseInst // Pretend this is Rust and I wrote .copy() - duration, err := testLiveIso(ctx, instIso, filepath.Join(outputDir, scenarioMinISOInstall), false, true) - printResult(scenarioMinISOInstall, duration, err) - if err != nil { - return err - } - } - if _, ok := targetScenarios[scenarioMinISOInstallNm]; ok { - if kola.CosaBuild.Meta.BuildArtifacts.LiveIso == nil { - return fmt.Errorf("build %s has no live ISO", kola.CosaBuild.Meta.Name) + plog.Fatalf("Unknown test name:%s", test) } - ranTest = true - instIso := baseInst // Pretend this is Rust and I wrote .copy() - addNmKeyfile = true - duration, err := testLiveIso(ctx, instIso, filepath.Join(outputDir, scenarioMinISOInstallNm), false, true) - printResult(scenarioMinISOInstallNm, duration, err) - if err != nil { - return err + if printResult(test, duration, err) { + atLeastOneFailed = true } } - if !ranTest { - panic("Nothing was tested!") + if atLeastOneFailed { + return harness.SuiteFailed } return nil @@ -589,24 +596,12 @@ func awaitCompletion(ctx context.Context, inst *platform.QemuInstance, outdir st return time.Since(start), err } -func printResult(mode string, duration time.Duration, err error) bool { +func printResult(test string, duration time.Duration, err error) bool { result := "PASS" if err != nil { result = "FAIL" } - variant := []string{kola.QEMUOptions.Firmware} - if kola.QEMUOptions.Native4k { - variant = append(variant, "metal4k") - } else { - variant = append(variant, "metal") - } - if kola.QEMUOptions.MultiPathDisk { - variant = append(variant, "multipath") - } - if addNmKeyfile { - variant = append(variant, "nm-keyfile") - } - fmt.Printf("%s: %s (%s) (%s)\n", result, mode, strings.Join(variant, " + "), duration.Round(time.Millisecond).String()) + fmt.Printf("%s: %s (%s)\n", result, test, duration.Round(time.Millisecond).String()) if err != nil { fmt.Printf(" %s\n", err) return true @@ -614,7 +609,7 @@ func printResult(mode string, duration time.Duration, err error) bool { return false } -func testPXE(ctx context.Context, inst platform.Install, outdir string, offline bool) (time.Duration, error) { +func testPXE(ctx context.Context, inst platform.Install, outdir string) (time.Duration, error) { if addNmKeyfile { return 0, errors.New("--add-nm-keyfile not yet supported for PXE") } @@ -647,7 +642,7 @@ func testPXE(ctx context.Context, inst platform.Install, outdir string, offline liveConfig.AddSystemdUnit("live-signal-ok.service", liveSignalOKUnit, conf.Enable) liveConfig.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) - if offline { + if isOffline { contents := fmt.Sprintf(downloadCheck, kola.CosaBuild.Meta.BuildID, kola.CosaBuild.Meta.OstreeCommit) liveConfig.AddSystemdUnit("coreos-installer-offline-check.service", contents, conf.Enable) } @@ -657,7 +652,7 @@ func testPXE(ctx context.Context, inst platform.Install, outdir string, offline targetConfig.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable) targetConfig.AddSystemdUnit("coreos-test-installer-no-ignition.service", checkNoIgnition, conf.Enable) - mach, err := inst.PXE(pxeKernelArgs, liveConfig, targetConfig, offline) + mach, err := inst.PXE(pxeKernelArgs, liveConfig, targetConfig, isOffline) if err != nil { return 0, errors.Wrapf(err, "running PXE") } @@ -670,7 +665,7 @@ func testPXE(ctx context.Context, inst platform.Install, outdir string, offline return awaitCompletion(ctx, mach.QemuInst, outdir, completionChannel, mach.BootStartedErrorChannel, []string{liveOKSignal, signalCompleteString}) } -func testLiveIso(ctx context.Context, inst platform.Install, outdir string, offline, minimal bool) (time.Duration, error) { +func testLiveIso(ctx context.Context, inst platform.Install, outdir string, minimal bool) (time.Duration, error) { tmpd, err := os.MkdirTemp("", "kola-testiso") if err != nil { return 0, err @@ -715,7 +710,7 @@ func testLiveIso(ctx context.Context, inst platform.Install, outdir string, offl inst.NmKeyfiles[nmConnectionFile] = nmConnection } - mach, err := inst.InstallViaISOEmbed(nil, liveConfig, targetConfig, outdir, offline, minimal) + mach, err := inst.InstallViaISOEmbed(nil, liveConfig, targetConfig, outdir, isOffline, minimal) if err != nil { return 0, errors.Wrapf(err, "running iso install") }