From 6638efb33730e2a00c978ce11a0bf9d840b88d3c Mon Sep 17 00:00:00 2001 From: bnaecker Date: Thu, 21 Apr 2022 11:34:17 -0700 Subject: [PATCH] A few fixes to scripts to simulate a Gimlet on commodity machines (#957) * A few fixes to scripts to simulate a Gimlet on commodity machines - Renames VNICs from `vioifN` to `netN` to avoid collision when running the script inside a VM - Remove the loopback address from the virtual hardware scripts - Better handling of existing names - Remove manual handling of xde kernel driver, delegating to the xde ONU consolidation for that - Cleans up the `omicron-package uninstall` command to avoid removing the files installed by the `helios-netdev` publisher for OPTE. This is mostly for the `opteadm` tool. * Address review comments --- package/src/bin/omicron-package.rs | 21 +++++++---- tools/create_virtual_hardware.sh | 39 ++++++++------------- tools/destroy_virtual_hardware.sh | 19 ++-------- tools/install_opte.sh | 56 ++++++++++++++++-------------- 4 files changed, 62 insertions(+), 73 deletions(-) diff --git a/package/src/bin/omicron-package.rs b/package/src/bin/omicron-package.rs index 560b5a7c7c..6a27a29099 100644 --- a/package/src/bin/omicron-package.rs +++ b/package/src/bin/omicron-package.rs @@ -422,8 +422,7 @@ fn remove_all_unless_already_removed>(path: P) -> Result<()> { Ok(()) } -fn remove_all_except_databases>(path: P) -> Result<()> { - const TO_KEEP: [&str; 2] = ["clickhouse", "cockroachdb"]; +fn remove_all_except>(path: P, to_keep: &[&str]) -> Result<()> { let dir = match path.as_ref().read_dir() { Ok(dir) => dir, Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(()), @@ -431,7 +430,10 @@ fn remove_all_except_databases>(path: P) -> Result<()> { }; for entry in dir { let entry = entry?; - if !TO_KEEP.contains(&&*(entry.file_name().to_string_lossy())) { + if to_keep.contains(&&*(entry.file_name().to_string_lossy())) { + println!(" Keeping: '{}'", entry.path().to_string_lossy()); + } else { + println!(" Removing: '{}'", entry.path().to_string_lossy()); if entry.metadata()?.is_dir() { remove_all_unless_already_removed(entry.path())?; } else { @@ -452,9 +454,16 @@ fn do_uninstall( println!("Uninstalling all packages"); uninstall_all_packages(config); println!("Removing artifacts in: {}", artifact_dir.to_string_lossy()); - remove_all_except_databases(artifact_dir)?; - println!("Removing: {}", install_dir.to_string_lossy()); - remove_all_unless_already_removed(install_dir)?; + + const ARTIFACTS_TO_KEEP: &[&str] = &["clickhouse", "cockroachdb", "xde"]; + remove_all_except(artifact_dir, ARTIFACTS_TO_KEEP)?; + + println!( + "Removing installed objects in: {}", + install_dir.to_string_lossy() + ); + const INSTALLED_OBJECTS_TO_KEEP: &[&str] = &["opte"]; + remove_all_except(install_dir, INSTALLED_OBJECTS_TO_KEEP)?; Ok(()) } diff --git a/tools/create_virtual_hardware.sh b/tools/create_virtual_hardware.sh index eac2be4fd8..b33c5a991b 100755 --- a/tools/create_virtual_hardware.sh +++ b/tools/create_virtual_hardware.sh @@ -50,6 +50,19 @@ function ensure_zpools { done } +# Return the name of a VNIC link if it exists, or the empty string if not. +# +# Arguments: +# $1: The name of the VNIC to look for +function get_vnic_name_if_exists { + NAME="$(dladm show-vnic -p -o LINK "$1")" + if [[ "$?" -eq 0 ]]; then + echo "$NAME" + else + echo "" + fi +} + # Create VNICs to represent the Chelsio physical links # # Arguments: @@ -57,34 +70,13 @@ function ensure_zpools { # first physical link available on the machine. function ensure_simulated_chelsios { local PHYSICAL_LINK="$1" - VNIC_NAMES=("vioif0" "vioif1") + VNIC_NAMES=("net0" "net1") for VNIC in "${VNIC_NAMES[@]}"; do - if [[ -z "$(dladm show-vnic -p -o LINK "$VNIC")" ]]; then + if [[ -z "$(get_vnic_name_if_exists "$VNIC")" ]]; then dladm create-vnic -t -l "$PHYSICAL_LINK" "$VNIC" fi success "VNIC $VNIC exists" - if [[ -z "$(ipadm show-addr -p -o ADDR "$VNIC/v6")" ]]; then - ipadm create-addr -t -T addrconf "$VNIC/v6" - fi - success "IP address $VNIC/v6 exists" done - - # Create an address on the underlay network - UNDERLAY_ADDR="lo0/underlay" - if [[ -z "$(ipadm show-addr -p -o ADDR "$UNDERLAY_ADDR")" ]]; then - ipadm create-addr -t -T static -a fd00:1::1/64 lo0/underlay - fi - success "IP address $UNDERLAY_ADDR exists" -} - -function ensure_xde_driver { - # Always remove the driver first. There seems to be a bug in the driver, - # preventing it from showing up in `modinfo` on boot, even if it's actually - # installed. - if [[ -z "$(modinfo | grep xde)" ]]; then - rem_drv xde - add_drv xde - fi } function ensure_run_as_root { @@ -97,4 +89,3 @@ function ensure_run_as_root { ensure_run_as_root ensure_zpools ensure_simulated_chelsios "$PHYSICAL_LINK" -ensure_xde_driver diff --git a/tools/destroy_virtual_hardware.sh b/tools/destroy_virtual_hardware.sh index 13e903a43c..ab8e475a16 100755 --- a/tools/destroy_virtual_hardware.sh +++ b/tools/destroy_virtual_hardware.sh @@ -27,20 +27,6 @@ function success { echo -e "\e[1;36m$1\e[0m" } -function try_uninstall_xde { - RC=0 - if ! [[ -z "$(modinfo | grep xde)" ]]; then - rem_drv xde - RC=$? - fi - - if [[ $RC -eq 0 ]]; then - success "XDE kernel module uninstalled" - else - warn "Failed to uninstall XDE kernel module" - fi -} - function try_remove_address { local ADDRESS="$1" RC=0 @@ -71,9 +57,8 @@ function try_remove_vnic { function try_remove_vnics { try_remove_address "lo0/underlay" - VNIC_LINKS=("vioif0" "vioif1") + VNIC_LINKS=("net0" "net1") for LINK in "${VNIC_LINKS[@]}"; do - try_remove_address "$LINK/v6" try_remove_vnic "$LINK" done } @@ -94,5 +79,5 @@ function try_destroy_zpools { done } -try_uninstall_xde && try_remove_vnics +try_remove_vnics try_destroy_zpools diff --git a/tools/install_opte.sh b/tools/install_opte.sh index 842eab947a..2098a48a92 100755 --- a/tools/install_opte.sh +++ b/tools/install_opte.sh @@ -19,7 +19,8 @@ SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" cd "${SOURCE_DIR}/.." OMICRON_TOP="$PWD" OUT_DIR="$OMICRON_TOP/out" -mkdir -p "$OUT_DIR" +XDE_DIR="$OUT_DIR/xde" +mkdir -p "$XDE_DIR" # Compute the SHA256 of the path in $1, returning just the sum function file_sha { @@ -30,7 +31,7 @@ function file_sha { function download_and_check_sha { local URL="$1" local FILENAME="$(basename "$URL")" - local OUT_PATH="$OUT_DIR/$FILENAME" + local OUT_PATH="$XDE_DIR/$FILENAME" local SHA="$2" # Check if the file already exists, with the expected SHA @@ -52,36 +53,39 @@ function sha_from_url { curl -L "$SHA_URL" 2> /dev/null | cut -d ' ' -f 1 } -OPTE_P5P_URL="https://buildomat.eng.oxide.computer/wg/0/artefact/01G0MRWX9Y0X46HBEJBW245DJY/PM097Agvf89uKmVRZ890z6saoeLp6RCcVsbYRa5PDv9DnLDT/01G0MRX6GMBV34CNANABXZXX25/01G0MSFZZWPFEQBW7JRS7ST99G/opte-0.1.58.p5p" -OPTE_P5P_SHA_URL="https://buildomat.eng.oxide.computer/wg/0/artefact/01G0MRWX9Y0X46HBEJBW245DJY/PM097Agvf89uKmVRZ890z6saoeLp6RCcVsbYRa5PDv9DnLDT/01G0MRX6GMBV34CNANABXZXX25/01G0MSG01CGP6TH9THNY39G88Z/opte-0.1.58.p5p.sha256" -OPTE_P5P_REPO_PATH="$OUT_DIR/$(basename "$OPTE_P5P_URL")" -XDE_URL="https://buildomat.eng.oxide.computer/wg/0/artefact/01G0DM53XR4E008D6ET5T8DXP6/wBWo0Jsg1AG19toIyAY23xAWhzmuNKmAsF6tL18ypZODNuHK/01G0DM5DMQHF5B89VGHZ05Z4E0/01G0DMHNYQ1NS7DBX8VG3JPAP0/xde" -XDE_SHA_URL="https://buildomat.eng.oxide.computer/wg/0/artefact/01G0DM53XR4E008D6ET5T8DXP6/wBWo0Jsg1AG19toIyAY23xAWhzmuNKmAsF6tL18ypZODNuHK/01G0DM5DMQHF5B89VGHZ05Z4E0/01G0DMHP47353961S3ETXBSD2T/xde.sha256" +# The `helios-netdev` provides the XDE kernel driver and the `opteadm` userland +# tool for interacting with it. +HELIOS_NETDEV_REPO_URL="https://buildomat.eng.oxide.computer/wg/0/artefact/01G11AT7E4XV9J1J54GE2YDJT6/CB4WF4BVgnbvf5NI573z9osAV2LNIKogPtWJ5sfW2cNxUYQO/01G11ATFVTWAC2HSNV148PQ4ER/01G11B5MPQRBX3Q5EF45YDAW6Q/opte-0.1.60.p5p" +HELIOS_NETDEV_REPO_SHA_URL="https://buildomat.eng.oxide.computer/wg/0/artefact/01G11AT7E4XV9J1J54GE2YDJT6/CB4WF4BVgnbvf5NI573z9osAV2LNIKogPtWJ5sfW2cNxUYQO/01G11ATFVTWAC2HSNV148PQ4ER/01G11B5MR60H4N13NJKGWEEA69/opte-0.1.60.p5p.sha256" +HELIOS_NETDEV_REPO_PATH="$XDE_DIR/$(basename "$HELIOS_NETDEV_REPO_URL")" -download_and_check_sha "$OPTE_P5P_URL" "$(sha_from_url "$OPTE_P5P_SHA_URL")" -XDE_SHA="$(sha_from_url "$XDE_SHA_URL")" -download_and_check_sha "$XDE_URL" "$XDE_SHA" +# The XDE repo provides a full OS/Net incorporation, with updated kernel bits +# that the `xde` kernel module and OPTE rely on. +XDE_REPO_URL="https://buildomat.eng.oxide.computer/wg/0/artefact/01G0ZKH44GQF88GB0GQBG9TQGW/7eOYj8L8E4MLrtvdTgGMyMu5qjYTRheV250bEvh2OkBrggX4/01G0ZKHBQ33K40S5ABZMRNWS5P/01G0ZYDDRXQ3Y4E5SG9QX8N9FK/repo.p5p" +XDE_REPO_SHA_URL="https://buildomat.eng.oxide.computer/wg/0/artefact/01G0ZKH44GQF88GB0GQBG9TQGW/7eOYj8L8E4MLrtvdTgGMyMu5qjYTRheV250bEvh2OkBrggX4/01G0ZKHBQ33K40S5ABZMRNWS5P/01G0ZYDJDMJAYHFV9Z6XVE30X5/repo.p5p.sha256" +XDE_REPO_PATH="$XDE_DIR/$(basename "$XDE_REPO_URL")" -# Move the XDE driver into it the expected location to allow operating on it -# with `add_drv` and `rem_drv` -DRIVER_DIR="/kernel/drv/amd64" -XDE_FILENAME="$(basename "$XDE_URL")" -XDE_PATH="$DRIVER_DIR/$XDE_FILENAME" -if ! [[ -f "$XDE_PATH" ]] || [[ "$XDE_SHA" != "$(file_sha "$XDE_PATH")" ]]; then - echo "Replacing XDE driver" - mv -f "$OUT_DIR/$XDE_FILENAME" "$XDE_PATH" -else - echo "XDE driver already exists with correct SHA" -fi +# Download and verify the package repositorieies +download_and_check_sha "$HELIOS_NETDEV_REPO_URL" "$(sha_from_url "$HELIOS_NETDEV_REPO_SHA_URL")" +download_and_check_sha "$XDE_REPO_URL" "$(sha_from_url "$XDE_REPO_SHA_URL")" -# Add the OPTE P5P package repository (at the top of the search order) and -# update the OS packages. This may require a reboot. -pkg set-publisher -p "$OPTE_P5P_REPO_PATH" --search-first +# Set the `helios-dev` repo as non-sticky, meaning that packages that were +# originally provided by it may be updated by another repository, if that repo +# provides newer versions of the packages. pkg set-publisher --non-sticky helios-dev + +# Add the OPTE and XDE repositories and update packages. +pkg set-publisher -p "$HELIOS_NETDEV_REPO_PATH" --search-first +pkg set-publisher -p "$XDE_REPO_PATH" --search-first + +# Actually update packages, handling case where no updates are needed RC=0 pkg update || RC=$?; if [[ "$RC" -eq 0 ]] || [[ "$RC" -eq 4 ]]; then - exit 0 + return 0 else - exit "$RC" + return "$RC" fi + +# Actually install the xde kernel module and opteadm tool +pkg install driver/network/opte