From ac10ac380be6df9eea354999c88c3f13bba5abb2 Mon Sep 17 00:00:00 2001 From: Lars Kellogg-Stedman Date: Thu, 23 Jun 2022 18:28:34 -0400 Subject: [PATCH] Configure bond0 interface We want to use bonded interface pairs on these system. The nodes aren't yet wired for it, but setting this up now will allow us to refer to the `bond0` interface in e.g. VLAN configurations (and means we won't have to re-work those later). Because we're using OVNKubernetes, we can't use nmstate [1] to enact the configuration. The recommendation is to apply the configuration using a MachineConfig [2] resource, but this is complicated by the fact that our nodes don't all have the same interface names, and it's not possible to apply node-specific machineconfigs [3]. We work around this solution by: 1. Copying nmconnection files for *all hosts* to *every host*, but placing them in `/etc/mco` (just because that's a convenient available directory, it seems relatively topical, and it's not possible to create new directories using the `directories` section of an ignition config [4]). 2. Installing a systemd unit that runs a shell script at boot that copies the host-specific configs from `/etc/mco` into `/etc/NetworkManager/system-connections`. [1]: https://docs.openshift.com/container-platform/4.10/networking/k8s_nmstate/k8s-nmstate-about-the-k8s-nmstate-operator.html [2]: https://docs.openshift.com/container-platform/4.10/post_installation_configuration/machine-configuration-tasks.html [3]: https://github.com/openshift/machine-config-operator/issues/1720 [4]: https://github.com/openshift/machine-config-operator/blob/master/docs/MachineConfigDaemon.md#supported-vs-unsupported-ignition-config-changes x-branch: feature/bond0 --- .../nerc-ocp-infra/kustomization.yaml | 1 + .../machineconfigs/configure-bond0/Makefile | 10 ++++ .../machineconfigs/configure-bond0/README.md | 7 +++ .../configure-bond0/configure-bond0.yaml | 57 +++++++++++++++++++ .../configure-bond0/kustomization.yaml | 4 ++ .../configure-bond0/src/bond0.nmconnection | 17 ++++++ .../src/configure-bond0-ports.sh | 13 +++++ .../configure-bond0/src/configure-bond0.bu | 50 ++++++++++++++++ .../src/ctl-0-nic1.nmconnection | 10 ++++ .../src/ctl-0-nic2.nmconnection | 10 ++++ .../src/ctl-1-nic1.nmconnection | 10 ++++ .../src/ctl-1-nic2.nmconnection | 10 ++++ .../src/ctl-2-nic1.nmconnection | 10 ++++ .../src/ctl-2-nic2.nmconnection | 10 ++++ 14 files changed, 219 insertions(+) create mode 100644 cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/Makefile create mode 100644 cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/README.md create mode 100644 cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/configure-bond0.yaml create mode 100644 cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/kustomization.yaml create mode 100644 cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/src/bond0.nmconnection create mode 100644 cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/src/configure-bond0-ports.sh create mode 100644 cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/src/configure-bond0.bu create mode 100644 cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/src/ctl-0-nic1.nmconnection create mode 100644 cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/src/ctl-0-nic2.nmconnection create mode 100644 cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/src/ctl-1-nic1.nmconnection create mode 100644 cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/src/ctl-1-nic2.nmconnection create mode 100644 cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/src/ctl-2-nic1.nmconnection create mode 100644 cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/src/ctl-2-nic2.nmconnection diff --git a/cluster-scope/overlays/nerc-ocp-infra/kustomization.yaml b/cluster-scope/overlays/nerc-ocp-infra/kustomization.yaml index 61e931cd..0ad58cb2 100644 --- a/cluster-scope/overlays/nerc-ocp-infra/kustomization.yaml +++ b/cluster-scope/overlays/nerc-ocp-infra/kustomization.yaml @@ -8,6 +8,7 @@ resources: - ../../bundles/acm - ../../bundles/odf - clusterversion.yaml +- machineconfigs/configure-bond0 patches: - path: oauths/cluster_patch.yaml diff --git a/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/Makefile b/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/Makefile new file mode 100644 index 00000000..c60efebb --- /dev/null +++ b/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/Makefile @@ -0,0 +1,10 @@ +MACHINECONFIGS = \ + configure-bond0.yaml + +%.yaml: src/%.bu + butane -o $@ -d src $< + +all: $(MACHINECONFIGS) + +clean: + rm -f $(MACHINECONFIGS) diff --git a/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/README.md b/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/README.md new file mode 100644 index 00000000..f7eb8a10 --- /dev/null +++ b/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/README.md @@ -0,0 +1,7 @@ +Including file content in ignition configs is a pain, because it has to be base64 encoded. The `Makefile` in this directory uses [Butane][] to transpile `MachineConfig` resources, automatically including and encoding content from files. + +Read "[Creating machine configs with Butane][]" in the [OpenShift documentation][] for more information. + +[butane]: https://coreos.github.io/butane/ +[openshift documentation]: https://docs.openshift.com/container-platform/4.10/installing/install_config/installing-customizing.html +[creating machine configs with butane]: https://docs.openshift.com/container-platform/4.10/installing/install_config/installing-customizing.html#installation-special-config-butane_installing-customizing diff --git a/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/configure-bond0.yaml b/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/configure-bond0.yaml new file mode 100644 index 00000000..3d6344b4 --- /dev/null +++ b/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/configure-bond0.yaml @@ -0,0 +1,57 @@ +# Generated by Butane; do not edit +apiVersion: machineconfiguration.openshift.io/v1 +kind: MachineConfig +metadata: + labels: + machineconfiguration.openshift.io/role: master + name: configure-bond0 +spec: + config: + ignition: + version: 3.2.0 + storage: + files: + - contents: + source: data:,%5Bconnection%5D%0Aid%3Dens6f0%0Atype%3Dethernet%0Ainterface-name%3Dens6f0%0Amaster%3Dbond0%0Aslave-type%3Dbond%0Aautoconnect%3Dtrue%0A%0A%5Bethernet%5D%0Amtu%3D9000%0A + path: /etc/mco/ctl-0-nic1.nmconnection + - contents: + source: data:,%5Bconnection%5D%0Aid%3Dens6f1%0Atype%3Dethernet%0Ainterface-name%3Dens6f1%0Amaster%3Dbond0%0Aslave-type%3Dbond%0Aautoconnect%3Dtrue%0A%0A%5Bethernet%5D%0Amtu%3D9000%0A + path: /etc/mco/ctl-0-nic2.nmconnection + - contents: + source: data:,%5Bconnection%5D%0Aid%3Dens5f0%0Atype%3Dethernet%0Ainterface-name%3Dens5f0%0Amaster%3Dbond0%0Aslave-type%3Dbond%0Aautoconnect%3Dtrue%0A%0A%5Bethernet%5D%0Amtu%3D9000%0A + path: /etc/mco/ctl-1-nic1.nmconnection + - contents: + source: data:,%5Bconnection%5D%0Aid%3Dens5f1%0Atype%3Dethernet%0Ainterface-name%3Dens5f1%0Amaster%3Dbond0%0Aslave-type%3Dbond%0Aautoconnect%3Dtrue%0A%0A%5Bethernet%5D%0Amtu%3D9000%0A + path: /etc/mco/ctl-1-nic2.nmconnection + - contents: + source: data:,%5Bconnection%5D%0Aid%3Dens5f0%0Atype%3Dethernet%0Ainterface-name%3Dens5f0%0Amaster%3Dbond0%0Aslave-type%3Dbond%0Aautoconnect%3Dtrue%0A%0A%5Bethernet%5D%0Amtu%3D9000%0A + path: /etc/mco/ctl-2-nic1.nmconnection + - contents: + source: data:,%5Bconnection%5D%0Aid%3Dens5f1%0Atype%3Dethernet%0Ainterface-name%3Dens5f1%0Amaster%3Dbond0%0Aslave-type%3Dbond%0Aautoconnect%3Dtrue%0A%0A%5Bethernet%5D%0Amtu%3D9000%0A + path: /etc/mco/ctl-2-nic2.nmconnection + - contents: + compression: gzip + source: data:;base64,H4sIAAAAAAAC/0zMwQqDMAzG8XvepaOCjO2QJ5EeavsNCzaRmgp7++EQtlvgn+83JRVBsqISqGSeVbIne2/4nlTE0F4xwUmsuHLspteOrXXQD7n9Nbev8cDOA9EEW9AEFqha56f3nmg6tUBVM3iOa5QE1xrVUqoKD+P5UrZjDFRhi2Y+bcpL2pyVCu3Gj/voPX0CAAD//04u+Q/FAAAA + path: /etc/NetworkConfiguration/system-connections/bond0.nmconnection + - contents: + compression: gzip + source: data:;base64,H4sIAAAAAAAC/5SQMU8DMQyFd/+KR9sJcURl7cRQiaVlgK3qkEt81OrFOcUpFFH+O+KEhNgOj3563yd7fuVaUdd6OxClY5SCZoDjGtyW61sux41X/8LF2btVTk3IqhyqZLUmivm250jpdWLFsdr1rabf1X9djqjLBYOvB8hPO4XsFg+PT8/b+826UQkfy5u7zz+aFWImAJAOux2aDosRsd+vUA+sY/Y9YcBsjGZTT8LlAj5LxXKEdEIxKxPNsVY7FUbIpXCoMO5FT2f0vuXeqLDVXDhM/gF9BQAA///ZzNpjrQEAAA== + mode: 493 + path: /etc/mco/configure-bond0-ports.sh + systemd: + units: + - contents: | + [Unit] + Description = Apply node-specific network configuration for bond0 + RequiredBy = NetworkManager.service + After = network-pre.target + Before = NetworkManager.service + + [Service] + Type = oneshot + ExecStart = /etc/mco/configure-bond0-ports.sh + + [Install] + WantedBy = multi-user.service + enabled: true + name: configure-bond0-ports.service diff --git a/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/kustomization.yaml b/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/kustomization.yaml new file mode 100644 index 00000000..ca3b67e2 --- /dev/null +++ b/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- configure-bond0.yaml diff --git a/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/src/bond0.nmconnection b/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/src/bond0.nmconnection new file mode 100644 index 00000000..d7cef358 --- /dev/null +++ b/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/src/bond0.nmconnection @@ -0,0 +1,17 @@ +[connection] +id=bond0 +type=bond +interface-name=bond0 +autoconnect=true +connection.autoconnect-slaves=1 + +[ethernet] +mtu=9000 + +[bond] +mode=balance-rr +miimon=140 + +[ipv4] +method=auto +dhcp-timeout=86400 diff --git a/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/src/configure-bond0-ports.sh b/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/src/configure-bond0-ports.sh new file mode 100644 index 00000000..b14677dc --- /dev/null +++ b/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/src/configure-bond0-ports.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +mkdir -p /etc/NetworkManager/system-connections-disabled +mv /etc/NetworkManager/system-connections/ens*.nmconnection /etc/NetworkManager/system-connections-disabled/ + +for path in /etc/mco/$HOSTNAME-nic{1,2}.nmconnection; do + if [[ -f $path ]]; then + cp "$path" /etc/NetworkManager/system-connections/ || exit 1 + fi +done + +# Ensure correct selinux labels +restorecon /etc/NetworkManager/system-connections diff --git a/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/src/configure-bond0.bu b/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/src/configure-bond0.bu new file mode 100644 index 00000000..5edb96d1 --- /dev/null +++ b/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/src/configure-bond0.bu @@ -0,0 +1,50 @@ +variant: openshift +version: 4.10.0 +metadata: + name: configure-bond0 + labels: + machineconfiguration.openshift.io/role: master +storage: + files: + - path: /etc/mco/ctl-0-nic1.nmconnection + contents: + local: ctl-0-nic1.nmconnection + - path: /etc/mco/ctl-0-nic2.nmconnection + contents: + local: ctl-0-nic2.nmconnection + - path: /etc/mco/ctl-1-nic1.nmconnection + contents: + local: ctl-1-nic1.nmconnection + - path: /etc/mco/ctl-1-nic2.nmconnection + contents: + local: ctl-1-nic2.nmconnection + - path: /etc/mco/ctl-2-nic1.nmconnection + contents: + local: ctl-2-nic1.nmconnection + - path: /etc/mco/ctl-2-nic2.nmconnection + contents: + local: ctl-2-nic2.nmconnection + - path: /etc/NetworkConfiguration/system-connections/bond0.nmconnection + contents: + local: bond0.nmconnection + - path: /etc/mco/configure-bond0-ports.sh + contents: + local: configure-bond0-ports.sh + mode: 0755 +systemd: + units: + - name: configure-bond0-ports.service + enabled: true + contents: | + [Unit] + Description = Apply node-specific network configuration for bond0 + RequiredBy = NetworkManager.service + After = network-pre.target + Before = NetworkManager.service + + [Service] + Type = oneshot + ExecStart = /etc/mco/configure-bond0-ports.sh + + [Install] + WantedBy = multi-user.service diff --git a/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/src/ctl-0-nic1.nmconnection b/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/src/ctl-0-nic1.nmconnection new file mode 100644 index 00000000..8ef6fb2a --- /dev/null +++ b/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/src/ctl-0-nic1.nmconnection @@ -0,0 +1,10 @@ +[connection] +id=ens6f0 +type=ethernet +interface-name=ens6f0 +master=bond0 +slave-type=bond +autoconnect=true + +[ethernet] +mtu=9000 diff --git a/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/src/ctl-0-nic2.nmconnection b/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/src/ctl-0-nic2.nmconnection new file mode 100644 index 00000000..45bbb655 --- /dev/null +++ b/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/src/ctl-0-nic2.nmconnection @@ -0,0 +1,10 @@ +[connection] +id=ens6f1 +type=ethernet +interface-name=ens6f1 +master=bond0 +slave-type=bond +autoconnect=true + +[ethernet] +mtu=9000 diff --git a/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/src/ctl-1-nic1.nmconnection b/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/src/ctl-1-nic1.nmconnection new file mode 100644 index 00000000..744a7729 --- /dev/null +++ b/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/src/ctl-1-nic1.nmconnection @@ -0,0 +1,10 @@ +[connection] +id=ens5f0 +type=ethernet +interface-name=ens5f0 +master=bond0 +slave-type=bond +autoconnect=true + +[ethernet] +mtu=9000 diff --git a/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/src/ctl-1-nic2.nmconnection b/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/src/ctl-1-nic2.nmconnection new file mode 100644 index 00000000..6c017d79 --- /dev/null +++ b/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/src/ctl-1-nic2.nmconnection @@ -0,0 +1,10 @@ +[connection] +id=ens5f1 +type=ethernet +interface-name=ens5f1 +master=bond0 +slave-type=bond +autoconnect=true + +[ethernet] +mtu=9000 diff --git a/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/src/ctl-2-nic1.nmconnection b/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/src/ctl-2-nic1.nmconnection new file mode 100644 index 00000000..744a7729 --- /dev/null +++ b/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/src/ctl-2-nic1.nmconnection @@ -0,0 +1,10 @@ +[connection] +id=ens5f0 +type=ethernet +interface-name=ens5f0 +master=bond0 +slave-type=bond +autoconnect=true + +[ethernet] +mtu=9000 diff --git a/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/src/ctl-2-nic2.nmconnection b/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/src/ctl-2-nic2.nmconnection new file mode 100644 index 00000000..6c017d79 --- /dev/null +++ b/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/src/ctl-2-nic2.nmconnection @@ -0,0 +1,10 @@ +[connection] +id=ens5f1 +type=ethernet +interface-name=ens5f1 +master=bond0 +slave-type=bond +autoconnect=true + +[ethernet] +mtu=9000