Create kernel_qemu_aarch64_run rule

bazel/linux/defs.bzl

@@ -1,4 +1,4 @@
-load("//bazel/linux:providers.bzl", "KernelBundleInfo", "KernelImageInfo", "KernelModulesInfo", "KernelTreeInfo", "RootfsImageInfo", "RuntimeBundleInfo")
+load("//bazel/linux:providers.bzl", "KernelBundleInfo", "KernelImageInfo", "KernelModulesInfo", "KernelTreeInfo", "RootfsImageInfo", "RuntimeBundleInfo", "BootRamImageInfo")
 load("//bazel/linux/platforms:defs.bzl", "kernel_aarch64_transition", "kernel_aarch64_constraints")
 load("//bazel/linux:utils.bzl", "expand_deps", "get_compatible", "is_module")
 load("//bazel/linux:test.bzl", "kunit_test")
@@ -609,3 +609,41 @@ def kernel_test(name, kernel_image, module, **kwargs):
 
     # Use the UML runner for the legacy tests
     kunit_test(name, kernel_image, module, runner = kernel_uml_run, **kwargs)
+
+def _bootram_image(ctx):
+    return [DefaultInfo(files = depset([ctx.file.image])), BootRamImageInfo(
+        image = ctx.file.image,
+        arch = ctx.attr.arch,
+    )]
+
+bootram_image = rule(
+    doc = """Defines a new bootram image.
+
+This rule exports a file that represents a bootram image so as to launch
+qemu acf machine.  This images is loaded into bootram while
+rest of the image (kernel, rootfs, devicetree) are loaded into DRAM.  On qemu launch
+it starts from BOOTRAM image which does some minimal stuff and jumps to bootloader
+in DRAM which continues rest of the boot
+
+Example:
+
+    bootram_image(
+        # This image file.
+        image = "bootram-img",
+        # Architecture of this image file.
+        arch = "aarch64",
+    )
+""",
+    implementation = _bootram_image,
+    attrs = {
+        "arch": attr.string(
+            doc = "Architecture this image was built for. Will only accept moudules for this arch.",
+            default = "host",
+        ),
+        "image": attr.label(
+            mandatory = True,
+            allow_single_file = True,
+            doc = "File containing the bootram image.",
+        ),
+    },
+)

bazel/linux/providers.bzl

@@ -78,3 +78,11 @@ RuntimeBundleInfo = provider(
         "check": "list of RuntimeInfo, executable (and its runfiles) to run OUTSIDE the VM AFTER the RUN to check if the run was successful",
     },
 )
+
+BootRamImageInfo = provider(
+    doc = """Maintains the information necessary to represent a BootRam image, needed for aarch64 qemu acf-machine boot.""",
+    fields = {
+        "arch": "Architecture this image was built for.",
+        "image": "Path of the image.",
+    },
+)

bazel/linux/qemu.bzl

@@ -17,6 +17,19 @@ DEFAULT_QEMU_FLAGS = [
     "-no-reboot",  # When kernel crashes / exits, don't restart VM.
 ]
 
+
+DEFAULT_QEMU_AARCH64_FLAGS = [
+    "-M",
+    "acf-ref",
+    "-cpu",
+    "neoverse-n1",
+    "-m",
+    "4.1G",
+    "-smp",
+    "cpus=16,sockets=1,clusters=2,cores=4,threads=2",
+    "-nographic",
+ ]
+
 DEFAULT_KERNEL_FLAGS = [
     "rw",
     "console=ttyS0",  # early boot messages to serial, which qemu sends to stdio.
@@ -28,6 +41,8 @@ def _kernel_qemu_run(ctx):
 QEMU_FLAGS={qemu_flags}
 QEMU_SEARCH={qemu_search}
 KERNEL_FLAGS={kernel_flags}
+BOOTRAM_IMAGE={bootram_image}
+ARCH={arch}
 
 for qemu in "${{QEMU_SEARCH[@]}}"; do
     QEMU_BINARY=$(which "$qemu") && break
@@ -38,29 +53,39 @@ test -n "$QEMU_BINARY" || {{
     exit 10
 }}
 
-if [ -n "$ROOTFS" ]; then
-    QEMU_FLAGS+=("-drive" "file=$ROOTFS,if=virtio,cache=none")
-else
-    QEMU_FLAGS+=("-fsdev" "local,security_model=none,multidevs=remap,id=fsdev-fsRoot,path=/")
-    QEMU_FLAGS+=("-device" "virtio-9p-pci,fsdev=fsdev-fsRoot,mount_tag=/dev/root")
-    KERNEL_FLAGS+=("root=/dev/root" "rootfstype=9p" "init=$INIT")
-    KERNEL_FLAGS+=("rootflags=trans=virtio,version=9p2000.L,msize=5000000,cache=mmap,posixacl")
-fi
+if [[ "$ARCH" == "x86" ]]; then
 
-QEMU_FLAGS+=("-fsdev" "local,security_model=none,multidevs=remap,id=fsdev-fsOutputDir,path=$OUTPUT_DIR")
-QEMU_FLAGS+=("-device" "virtio-9p-pci,fsdev=fsdev-fsOutputDir,mount_tag=/dev/output_dir")
+    if [ -n "$ROOTFS" ]; then
+        QEMU_FLAGS+=("-drive" "file=$ROOTFS,if=virtio,cache=none")
+    else
+        QEMU_FLAGS+=("-fsdev" "local,security_model=none,multidevs=remap,id=fsdev-fsRoot,path=/")
+        QEMU_FLAGS+=("-device" "virtio-9p-pci,fsdev=fsdev-fsRoot,mount_tag=/dev/root")
+        KERNEL_FLAGS+=("root=/dev/root" "rootfstype=9p" "init=$INIT")
+        KERNEL_FLAGS+=("rootflags=trans=virtio,version=9p2000.L,msize=5000000,cache=mmap,posixacl")
+    fi
 
-test -z "$KERNEL" || QEMU_FLAGS+=("-kernel" "$KERNEL")
-test -z "$SINGLE" || KERNEL_FLAGS+=("init=/bin/sh")
+    QEMU_FLAGS+=("-fsdev" "local,security_model=none,multidevs=remap,id=fsdev-fsOutputDir,path=$OUTPUT_DIR")
+    QEMU_FLAGS+=("-device" "virtio-9p-pci,fsdev=fsdev-fsOutputDir,mount_tag=/dev/output_dir")
+
+    test -z "$KERNEL" || QEMU_FLAGS+=("-kernel" "$KERNEL")
+    test -z "$SINGLE" || KERNEL_FLAGS+=("init=/bin/sh")
+
+    QEMU_FLAGS+=("-append" "${{KERNEL_FLAGS[*]}} ${{KERNEL_OPTS[*]}}")
+fi
+
+if [[ "$ARCH" == "aarch64" ]] ; then
+    QEMU_FLAGS+=("-device" "loader,file=$BOOTRAM_IMAGE,addr=0")
+    QEMU_FLAGS+=("-device" "loader,file=$KERNEL,addr=0x10000000000")
+fi
 
-QEMU_FLAGS+=("-append" "${{KERNEL_FLAGS[*]}} ${{KERNEL_OPTS[*]}}")
 QEMU_FLAGS+=("${{EMULATOR_OPTS[@]}}")
 
 if [ ${{#WRAPPER_OPTS[@]}} -ne 0 ]; then
     WRAPPER_OPTS=('--' "${{WRAPPER_OPTS[@]}}")
 fi
 
 echo 1>&2 '$' "$QEMU_BINARY" "${{QEMU_FLAGS[@]}}" "${{WRAPPER_OPTS[@]}}"
+
 if [ -z "$INTERACTIVE" -a -z "$SINGLE" ]; then
     "$QEMU_BINARY" "${{QEMU_FLAGS[@]}}" "${{WRAPPER_OPTS[@]}}" </dev/null | tee "$OUTPUT_FILE"
 else
@@ -75,11 +100,19 @@ fi
         runfiles = di.default_runfiles
     qemu_flags = ctx.attr.qemu_defaults + ctx.attr.qemu_flags
 
-    kernel_flags = ctx.attr.kernel_defaults + ctx.attr.kernel_flags
+    if "aarch64" in ctx.attr.archs:
+       # no cmdline kernel flags for aarch64 qemu.  All embeded in the fit image for now
+       arch = "aarch64"
+       kernel_flags = []
+    else:
+       arch = "x86"
+       kernel_flags = ctx.attr.kernel_defaults + ctx.attr.kernel_flags
+
     return create_runner(ctx, ctx.attr.archs, code, runfiles = runfiles, extra = {
         "qemu_search": shell.array_literal(qemu_search),
         "qemu_flags": shell.array_literal(qemu_flags),
         "kernel_flags": shell.array_literal(kernel_flags),
+        "arch": shell.quote(arch),
     })
 
 kernel_qemu_run = rule(
@@ -127,3 +160,49 @@ See the RuntimeBundleInfo provider for details.
         }
     ),
 )
+
+kernel_qemu_aarch64_run = rule(
+    doc = """Runs code in a aarch64 qemu instance.
+
+The code to run is specified by using the "runner" attribute, which
+pretty much provides a self contained directory with an init script.
+See the RuntimeBundleInfo provider for details.
+
+Note to change the kernel, initramfs, kernel cmdline etc. a new FIT image with
+necessary changes has to be created and its KernelImageInfo passed as kernel_image
+to this rule
+""",
+    implementation = _kernel_qemu_run,
+    executable = True,
+    # Note to change the kernel, initramfs, kernel cmdline etc. a new FIT image with
+    # necessary changes has to be created and its KernelImageInfo passed as kernel_image
+    # to this rule
+    attrs = dict(
+        create_runner_attrs(
+            template_init_default = Label("//bazel/linux:templates/init-qemu.template.sh"),
+        ),
+        **{
+            "archs": attr.string_list(
+                default = ["aarch64"],
+                doc = "Architectures supported by this test",
+            ),
+            "qemu_binary": attr.label(
+                doc = "A target defining the qemu binary to run. If unspecified, it will use a search path",
+                executable = True,
+                cfg = "target",
+            ),
+            "qemu_search": attr.string_list(
+                doc = "Qemu binaries to try to run, in turn, until one is found. Ignored if qemu_binary is specified.",
+                default = ["qemu-system-aarch64"],
+            ),
+            "qemu_defaults": attr.string_list(
+                doc = "Default flags to pass to qemu. Use only if you need to change the defaults",
+                default = DEFAULT_QEMU_AARCH64_FLAGS,
+            ),
+            "qemu_flags": attr.string_list(
+                doc = "Additional flags to pass to qemu. Appended to the default flags",
+                default = [],
+            ),
+        }
+    ),
+)

bazel/linux/runner.bzl

@@ -1,4 +1,4 @@
-load("//bazel/linux:providers.bzl", "KernelImageInfo", "RootfsImageInfo", "RuntimeBundleInfo", "RuntimeInfo")
+load("//bazel/linux:providers.bzl", "KernelImageInfo", "RootfsImageInfo", "RuntimeBundleInfo", "RuntimeInfo", "BootRamImageInfo")
 load("//bazel/utils:messaging.bzl", "location", "package")
 load("//bazel/utils:types.bzl", "escape_and_join")
 load("//bazel/utils:files.bzl", "files_to_dir")
@@ -13,6 +13,12 @@ def create_runner_attrs(template_init_default):
             providers = [DefaultInfo, KernelImageInfo],
             doc = "The kernel image that will be used to execute this test. A string like @stefano-s-favourite-kernel-image, referencing a kernel_image(name = 'stefano-s-favourite-kernel-image', ...",
         ),
+       "bootram_image": attr.label(
+            doc = "A target defining the bootram image (used with aarch64 qemu).",
+            allow_single_file = True,
+            mandatory = False,
+            providers = [DefaultInfo, BootRamImageInfo],
+        ),
         "rootfs_image": attr.label(
             mandatory = False,
             providers = [RootfsImageInfo],
@@ -218,6 +224,13 @@ def create_runner(ctx, archs, code, runfiles = None, extra = {}):
         post = "cd {dest}; cp -L %s ./init.sh" % (shell.quote(init.short_path)),
     )
     outside_runfiles = torun.outside_runfiles.merge(ctx.runfiles([runtime_root, ki.image]))
+    bootram_image = ""
+    if ki.arch == "aarch64":
+       # for aarch64 need a image that should be loaded into bootram which is where execution starts
+       if ctx.attr.bootram_image and ctx.attr.bootram_image[BootRamImageInfo].arch == ki.arch:
+          bootram_info = ctx.attr.bootram_image[BootRamImageInfo]
+          bootram_image = bootram_info.image.short_path
+          outside_runfiles = outside_runfiles.merge(ctx.runfiles([bootram_info.image]))
     if runfiles:
       outside_runfiles = outside_runfiles.merge(runfiles)
 
@@ -232,10 +245,12 @@ def create_runner(ctx, archs, code, runfiles = None, extra = {}):
         "cleanups": "\n".join(torun.commands.cleanup),
         "checks": "\n".join(torun.commands.check),
         "kernel": ki.image.short_path,
+        "bootram_image": bootram_image,
         "rootfs": rootfs,
         "init": init.short_path,
         "runtime": runtime_root.short_path,
 	"wrapper_flags": shell.array_literal(ctx.attr.wrapper_flags),
+	"arch": ki.arch,
     }, **extra)
 
     subs["code"] = code.format(**subs)

bazel/linux/templates/runner.template.sh

@@ -14,6 +14,7 @@ RELRUNTIME="{runtime}"
 ROOTFS="{rootfs}"
 KERNEL="{kernel}"
 TARGET="{target}"
+ARCH="{arch}"
 WRAPPER_OPTS={wrapper_flags} # Array, no quotes!
 
 # A script in charge of verifying the output of the run.
@@ -31,13 +32,18 @@ or:
   bazel build $TARGET
   bazel-bin/.../path/to/file [-k option]... [-e option]... [-s|-x|-h]
 
+
 Accepted options:
 
-  -s           Configures /bin/sh as init, drops you in a shell.
+  For aarch64 changing kernel/initramfs or kernel cmdline is done by building
+  a new FIT image with the new kernel/initramfs and for cmdline changes need to change
+  the dts file. So can't use some of these options
+
+  -s           Configures /bin/sh as init, drops you in a shell (only for x86).
                Hint: you can then use the paths shown with -x to manually
                start the init script used.
 
-  -k [value]   Adds one or more command line options to the kernel.
+  -k [value]   Adds one or more command line options to the kernel (only for x86).
 
      For example: "-k ro -k root=/dev/sda -k console=ttyS0"
      will add "ro root=/dev/sda console=ttyS0" to the kernel command line.
@@ -47,7 +53,7 @@ Accepted options:
      For example: "-e'-f' -e/dev/sda -e'-mem' -e2048"
      will add "-f /dev/sda -mem 2048" to the emulator command line.
 
-  -r [value]   Overrides the path to the rootfs.
+  -r [value]   Overrides the path to the rootfs (only for x86).
 
      For example: "-r /tmp/myown.qcow" will ask the emulator to run
      the specified rootfs.
@@ -78,6 +84,7 @@ function onexit {
 }
 
 function showstate {
+    echo 1>&2 "Arch: $ARCH"
     echo 1>&2 "CWD: $(realpath "$PWD")"
     echo 1>&2 "Script: $(realpath "$0")"
     echo 1>&2 "Kernel: $(realpath "$KERNEL")"
@@ -97,7 +104,7 @@ TMPDIR="${TEST_TMPDIR:-$(mktemp -d)}"
 OUTPUT_DIR=${TEST_UNDECLARED_OUTPUTS_DIR:-$(mktemp -d)}
 
 # If TERM variable is exported, then set the number of columns in the VM.
-if [[ "${TERM@a}" == *x* ]]; then
+if [[ "$ARCH" != "aarch64" && "${TERM@a}" == *x* ]]; then
   KERNEL_OPTS+=("COLUMNS=$(tput cols)")
 fi
 
@@ -121,6 +128,11 @@ while getopts "k:e:r:hsx" opt; do
 done
 shift $((OPTIND - 1))
 
+if [[ "$ARCH" = "aarch64" ]]  && [[ -n "$SINGLE" || -n "$ROOTFS" || -n "$KERNEL_OPTS" ]];  then
+    echo "Invalid option provided for AARCH64"
+    exit 1
+fi
+
 WRAPPER_OPTS+=("$@")
 
 showstate