A few fixes to scripts to simulate a Gimlet on commodity machines (#957)

* A few fixes to scripts to simulate a Gimlet on commodity machines

- Renames VNICs from `vioifN` to `netN` to avoid collision when running
  the script inside a VM
- Remove the loopback address from the virtual hardware scripts
- Better handling of existing names
- Remove manual handling of xde kernel driver, delegating to the xde ONU
  consolidation for that
- Cleans up the `omicron-package uninstall` command to avoid removing
  the files installed by the `helios-netdev` publisher for OPTE. This is
  mostly for the `opteadm` tool.

* Address review comments

package/src/bin/omicron-package.rs

@@ -422,16 +422,18 @@ fn remove_all_unless_already_removed<P: AsRef<Path>>(path: P) -> Result<()> {
     Ok(())
 }
 
-fn remove_all_except_databases<P: AsRef<Path>>(path: P) -> Result<()> {
-    const TO_KEEP: [&str; 2] = ["clickhouse", "cockroachdb"];
+fn remove_all_except<P: AsRef<Path>>(path: P, to_keep: &[&str]) -> Result<()> {
     let dir = match path.as_ref().read_dir() {
         Ok(dir) => dir,
         Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(()),
         Err(e) => bail!(e),
     };
     for entry in dir {
         let entry = entry?;
-        if !TO_KEEP.contains(&&*(entry.file_name().to_string_lossy())) {
+        if to_keep.contains(&&*(entry.file_name().to_string_lossy())) {
+            println!(" Keeping: '{}'", entry.path().to_string_lossy());
+        } else {
+            println!(" Removing: '{}'", entry.path().to_string_lossy());
             if entry.metadata()?.is_dir() {
                 remove_all_unless_already_removed(entry.path())?;
             } else {
@@ -452,9 +454,16 @@ fn do_uninstall(
     println!("Uninstalling all packages");
     uninstall_all_packages(config);
     println!("Removing artifacts in: {}", artifact_dir.to_string_lossy());
-    remove_all_except_databases(artifact_dir)?;
-    println!("Removing: {}", install_dir.to_string_lossy());
-    remove_all_unless_already_removed(install_dir)?;
+
+    const ARTIFACTS_TO_KEEP: &[&str] = &["clickhouse", "cockroachdb", "xde"];
+    remove_all_except(artifact_dir, ARTIFACTS_TO_KEEP)?;
+
+    println!(
+        "Removing installed objects in: {}",
+        install_dir.to_string_lossy()
+    );
+    const INSTALLED_OBJECTS_TO_KEEP: &[&str] = &["opte"];
+    remove_all_except(install_dir, INSTALLED_OBJECTS_TO_KEEP)?;
     Ok(())
 }
 

tools/create_virtual_hardware.sh

@@ -50,41 +50,33 @@ function ensure_zpools {
     done
 }
 
+# Return the name of a VNIC link if it exists, or the empty string if not.
+#
+# Arguments:
+#   $1: The name of the VNIC to look for
+function get_vnic_name_if_exists {
+    NAME="$(dladm show-vnic -p -o LINK "$1")"
+    if [[ "$?" -eq 0 ]]; then
+        echo "$NAME"
+    else
+        echo ""
+    fi
+}
+
 # Create VNICs to represent the Chelsio physical links
 #
 # Arguments:
 #   $1: Optional name of the physical link to use. If not provided, use the
 #   first physical link available on the machine.
 function ensure_simulated_chelsios {
     local PHYSICAL_LINK="$1"
-    VNIC_NAMES=("vioif0" "vioif1")
+    VNIC_NAMES=("net0" "net1")
     for VNIC in "${VNIC_NAMES[@]}"; do
-        if [[ -z "$(dladm show-vnic -p -o LINK "$VNIC")" ]]; then
+        if [[ -z "$(get_vnic_name_if_exists "$VNIC")" ]]; then
             dladm create-vnic -t -l "$PHYSICAL_LINK" "$VNIC"
         fi
         success "VNIC $VNIC exists"
-        if [[ -z "$(ipadm show-addr -p -o ADDR "$VNIC/v6")" ]]; then
-            ipadm create-addr -t -T addrconf "$VNIC/v6"
-        fi
-        success "IP address $VNIC/v6 exists"
     done
-
-    # Create an address on the underlay network
-    UNDERLAY_ADDR="lo0/underlay"
-    if [[ -z "$(ipadm show-addr -p -o ADDR "$UNDERLAY_ADDR")" ]]; then
-        ipadm create-addr -t -T static -a fd00:1::1/64 lo0/underlay
-    fi
-    success "IP address $UNDERLAY_ADDR exists"
-}
-
-function ensure_xde_driver {
-    # Always remove the driver first. There seems to be a bug in the driver,
-    # preventing it from showing up in `modinfo` on boot, even if it's actually
-    # installed.
-    if [[ -z "$(modinfo | grep xde)" ]]; then
-        rem_drv xde
-        add_drv xde
-    fi
 }
 
 function ensure_run_as_root {
@@ -97,4 +89,3 @@ function ensure_run_as_root {
 ensure_run_as_root
 ensure_zpools
 ensure_simulated_chelsios "$PHYSICAL_LINK"
-ensure_xde_driver

tools/destroy_virtual_hardware.sh

@@ -27,20 +27,6 @@ function success {
     echo -e "\e[1;36m$1\e[0m"
 }
 
-function try_uninstall_xde {
-    RC=0
-    if ! [[ -z "$(modinfo | grep xde)" ]]; then
-        rem_drv xde
-        RC=$?
-    fi
-
-    if [[ $RC -eq 0 ]]; then
-        success "XDE kernel module uninstalled"
-    else
-        warn "Failed to uninstall XDE kernel module"
-    fi
-}
-
 function try_remove_address {
     local ADDRESS="$1"
     RC=0
@@ -71,9 +57,8 @@ function try_remove_vnic {
 
 function try_remove_vnics {
     try_remove_address "lo0/underlay"
-    VNIC_LINKS=("vioif0" "vioif1")
+    VNIC_LINKS=("net0" "net1")
     for LINK in "${VNIC_LINKS[@]}"; do
-        try_remove_address "$LINK/v6"
         try_remove_vnic "$LINK"
     done
 }
@@ -94,5 +79,5 @@ function try_destroy_zpools {
     done
 }
 
-try_uninstall_xde && try_remove_vnics
+try_remove_vnics
 try_destroy_zpools

tools/install_opte.sh

@@ -19,7 +19,8 @@ SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
 cd "${SOURCE_DIR}/.."
 OMICRON_TOP="$PWD"
 OUT_DIR="$OMICRON_TOP/out"
-mkdir -p "$OUT_DIR"
+XDE_DIR="$OUT_DIR/xde"
+mkdir -p "$XDE_DIR"
 
 # Compute the SHA256 of the path in $1, returning just the sum
 function file_sha {
@@ -30,7 +31,7 @@ function file_sha {
 function download_and_check_sha {
     local URL="$1"
     local FILENAME="$(basename "$URL")"
-    local OUT_PATH="$OUT_DIR/$FILENAME"
+    local OUT_PATH="$XDE_DIR/$FILENAME"
     local SHA="$2"
 
     # Check if the file already exists, with the expected SHA
@@ -52,36 +53,39 @@ function sha_from_url {
     curl -L "$SHA_URL" 2> /dev/null | cut -d ' ' -f 1
 }
 
-OPTE_P5P_URL="https://buildomat.eng.oxide.computer/wg/0/artefact/01G0MRWX9Y0X46HBEJBW245DJY/PM097Agvf89uKmVRZ890z6saoeLp6RCcVsbYRa5PDv9DnLDT/01G0MRX6GMBV34CNANABXZXX25/01G0MSFZZWPFEQBW7JRS7ST99G/opte-0.1.58.p5p"
-OPTE_P5P_SHA_URL="https://buildomat.eng.oxide.computer/wg/0/artefact/01G0MRWX9Y0X46HBEJBW245DJY/PM097Agvf89uKmVRZ890z6saoeLp6RCcVsbYRa5PDv9DnLDT/01G0MRX6GMBV34CNANABXZXX25/01G0MSG01CGP6TH9THNY39G88Z/opte-0.1.58.p5p.sha256"
-OPTE_P5P_REPO_PATH="$OUT_DIR/$(basename "$OPTE_P5P_URL")"
-XDE_URL="https://buildomat.eng.oxide.computer/wg/0/artefact/01G0DM53XR4E008D6ET5T8DXP6/wBWo0Jsg1AG19toIyAY23xAWhzmuNKmAsF6tL18ypZODNuHK/01G0DM5DMQHF5B89VGHZ05Z4E0/01G0DMHNYQ1NS7DBX8VG3JPAP0/xde"
-XDE_SHA_URL="https://buildomat.eng.oxide.computer/wg/0/artefact/01G0DM53XR4E008D6ET5T8DXP6/wBWo0Jsg1AG19toIyAY23xAWhzmuNKmAsF6tL18ypZODNuHK/01G0DM5DMQHF5B89VGHZ05Z4E0/01G0DMHP47353961S3ETXBSD2T/xde.sha256"
+# The `helios-netdev` provides the XDE kernel driver and the `opteadm` userland
+# tool for interacting with it.
+HELIOS_NETDEV_REPO_URL="https://buildomat.eng.oxide.computer/wg/0/artefact/01G11AT7E4XV9J1J54GE2YDJT6/CB4WF4BVgnbvf5NI573z9osAV2LNIKogPtWJ5sfW2cNxUYQO/01G11ATFVTWAC2HSNV148PQ4ER/01G11B5MPQRBX3Q5EF45YDAW6Q/opte-0.1.60.p5p"
+HELIOS_NETDEV_REPO_SHA_URL="https://buildomat.eng.oxide.computer/wg/0/artefact/01G11AT7E4XV9J1J54GE2YDJT6/CB4WF4BVgnbvf5NI573z9osAV2LNIKogPtWJ5sfW2cNxUYQO/01G11ATFVTWAC2HSNV148PQ4ER/01G11B5MR60H4N13NJKGWEEA69/opte-0.1.60.p5p.sha256"
+HELIOS_NETDEV_REPO_PATH="$XDE_DIR/$(basename "$HELIOS_NETDEV_REPO_URL")"
 
-download_and_check_sha "$OPTE_P5P_URL" "$(sha_from_url "$OPTE_P5P_SHA_URL")"
-XDE_SHA="$(sha_from_url "$XDE_SHA_URL")"
-download_and_check_sha "$XDE_URL" "$XDE_SHA"
+# The XDE repo provides a full OS/Net incorporation, with updated kernel bits
+# that the `xde` kernel module and OPTE rely on.
+XDE_REPO_URL="https://buildomat.eng.oxide.computer/wg/0/artefact/01G0ZKH44GQF88GB0GQBG9TQGW/7eOYj8L8E4MLrtvdTgGMyMu5qjYTRheV250bEvh2OkBrggX4/01G0ZKHBQ33K40S5ABZMRNWS5P/01G0ZYDDRXQ3Y4E5SG9QX8N9FK/repo.p5p"
+XDE_REPO_SHA_URL="https://buildomat.eng.oxide.computer/wg/0/artefact/01G0ZKH44GQF88GB0GQBG9TQGW/7eOYj8L8E4MLrtvdTgGMyMu5qjYTRheV250bEvh2OkBrggX4/01G0ZKHBQ33K40S5ABZMRNWS5P/01G0ZYDJDMJAYHFV9Z6XVE30X5/repo.p5p.sha256"
+XDE_REPO_PATH="$XDE_DIR/$(basename "$XDE_REPO_URL")"
 
-# Move the XDE driver into it the expected location to allow operating on it
-# with `add_drv` and `rem_drv`
-DRIVER_DIR="/kernel/drv/amd64"
-XDE_FILENAME="$(basename "$XDE_URL")"
-XDE_PATH="$DRIVER_DIR/$XDE_FILENAME"
-if ! [[ -f "$XDE_PATH" ]] || [[ "$XDE_SHA" != "$(file_sha "$XDE_PATH")" ]]; then
-    echo "Replacing XDE driver"
-    mv -f "$OUT_DIR/$XDE_FILENAME" "$XDE_PATH"
-else
-    echo "XDE driver already exists with correct SHA"
-fi
+# Download and verify the package repositorieies
+download_and_check_sha "$HELIOS_NETDEV_REPO_URL" "$(sha_from_url "$HELIOS_NETDEV_REPO_SHA_URL")"
+download_and_check_sha "$XDE_REPO_URL" "$(sha_from_url "$XDE_REPO_SHA_URL")"
 
-# Add the OPTE P5P package repository (at the top of the search order) and
-# update the OS packages. This may require a reboot.
-pkg set-publisher -p "$OPTE_P5P_REPO_PATH" --search-first
+# Set the `helios-dev` repo as non-sticky, meaning that packages that were
+# originally provided by it may be updated by another repository, if that repo
+# provides newer versions of the packages.
 pkg set-publisher --non-sticky helios-dev
+
+# Add the OPTE and XDE repositories and update packages.
+pkg set-publisher -p "$HELIOS_NETDEV_REPO_PATH" --search-first
+pkg set-publisher -p "$XDE_REPO_PATH" --search-first
+
+# Actually update packages, handling case where no updates are needed
 RC=0
 pkg update || RC=$?;
 if [[ "$RC" -eq 0 ]] || [[ "$RC" -eq 4 ]]; then
-    exit 0
+    return 0
 else
-    exit "$RC"
+    return "$RC"
 fi
+
+# Actually install the xde kernel module and opteadm tool
+pkg install driver/network/opte