From eef1de9a3a7a3b4b8fdb3387e108adfb57834b83 Mon Sep 17 00:00:00 2001 From: tyzbit <3319104+tyzbit@users.noreply.github.com> Date: Sat, 7 Oct 2023 01:31:23 -0400 Subject: [PATCH] feat: (flux) add flux bundle Closes https://github.com/kairos-io/kairos/issues/638 --- README.md | 118 +++++++++++++++++++++++++++----------------- fluxcd/Dockerfile | 10 ++++ fluxcd/bootstrap.sh | 103 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 186 insertions(+), 45 deletions(-) create mode 100644 fluxcd/Dockerfile create mode 100755 fluxcd/bootstrap.sh diff --git a/README.md b/README.md index a820b49..a59eb70 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,3 @@ -


kairos-white-column 5bc2fe34 @@ -10,7 +9,6 @@
- Welcome to the community-bundles repository! This repository builds and pushes Kairos community bundles that can be consumed by Kairos core or derivative images (such as [provider-kairos](https://github.com/kairos-io/provider-kairos) ) to extend Kairos configurations and settings, and to add cloud-config keywords. Please note that these community bundles are not officially supported and are provided on a best-effort basis by the community. @@ -36,8 +34,8 @@ To use a community bundle, you can load it with the bundles block in the Kairos ```yaml bundles: -- targets: - - run://quay.io/kairos/community-bundles: + - targets: + - run://quay.io/kairos/community-bundles: ``` Here is an example of how you might use a community bundle in a Kairos core image: @@ -45,20 +43,20 @@ Here is an example of how you might use a community bundle in a Kairos core imag ```yaml #cloud-config install: - device: "auto" - auto: true - reboot: true - image: "docker:quay.io/kairos/kairos-opensuse:v1.4.0-k3sv1.26.0-k3s1" + device: "auto" + auto: true + reboot: true + image: "docker:quay.io/kairos/kairos-opensuse:v1.4.0-k3sv1.26.0-k3s1" users: -- name: "kairos" - passwd: "kairos" - ssh_authorized_keys: - - ... + - name: "kairos" + passwd: "kairos" + ssh_authorized_keys: + - ... bundles: -- targets: - - run://quay.io/kairos/community-bundles:kubevirt + - targets: + - run://quay.io/kairos/community-bundles:kubevirt k3s: enabled: true @@ -77,8 +75,8 @@ To configure the bundle, use the `calico` block: # Specify the bundle to use bundles: -- targets: - - run://quay.io/kairos/community-bundles:calico_latest + - targets: + - run://quay.io/kairos/community-bundles:calico_latest # Specify calico settings calico: @@ -94,7 +92,7 @@ calico: version: 3.25.0 ``` -Note that specifying `values` and `version` are optional. Specifying `values` allows you to +Note that specifying `values` and `version` are optional. Specifying `values` allows you to [customize the Helm Chart](https://docs.tigera.io/calico/latest/getting-started/kubernetes/helm#customize-the-helm-chart). ### Cert-manager @@ -108,14 +106,46 @@ The bundle does add a `certManager` block, that allow to change the version (cur # Specify the bundle to use bundles: -- targets: - - run://quay.io/kairos/community-bundles:cert-manager_latest + - targets: + - run://quay.io/kairos/community-bundles:cert-manager_latest # Specify cert-manager settings certManager: version: v1.11.0 ``` +### Flux + +This installs Flux and supports bootstrapping the cluster with [FluxCD](https://fluxcd.io/flux/cmd/flux_bootstrap/). It has logic so only one node will do the bootstrapping, +by creating a ConfigMap in the `default` namespace named `flux-bootstrap`. +By default, it will time out after trying for 30 minutes. + +```yaml +#cloud-config + +bundles: + - targets: + # Installs to /usr/local/lib/extensions/flux/ + - container://quay.io/kairos/community-bundles:flux_latest + +stages: + boot: + - name: "Bootstrap with Flux" + commands: + - bash /usr/local/lib/extensions/flux/bootstrap.sh & + +# Specify command-line arguments as keys under a key of `bitbucket_server`, +# `git`, `github` or `gitlab` for the provider to boostrap from. +flux: + env: + GITHUB_TOKEN: abcde1234 + github: + owner: csagan + repository: fleet-infra + path: clusters/cosmos + components-extra: image-reflector-controller,image-automation-controller +``` + ### Kairos The Kairos bundle deploys the [Kairos helm-charts](https://github.com/kairos-io/helm-charts). It installs the `kairos-crds` chart, and allows to enable [entangle-proxy](https://kairos.io/docs/reference/entangle/), [osbuilder](https://kairos.io/docs/advanced/build/), and [entangle](https://kairos.io/docs/reference/entangle/). @@ -127,8 +157,8 @@ By default the bundle will install only the CRDs, components needs to be explici # Specify the bundle to use bundles: -- targets: - - run://quay.io/kairos/community-bundles:kairos_latest + - targets: + - run://quay.io/kairos/community-bundles:kairos_latest # Specify kairos bundle setting kairos: @@ -154,24 +184,22 @@ To configure the bundle, use the `kyverno` block: # Specify the bundle to use bundles: -- targets: - - run://quay.io/kairos/community-bundles:kyverno_latest + - targets: + - run://quay.io/kairos/community-bundles:kyverno_latest # Specify kyverno settings kyverno: - values: - .... + values: .... version: ... ``` -Note that specifying `values` and `version` are optional. Specifying `values` allows you to +Note that specifying `values` and `version` are optional. Specifying `values` allows you to [customize the Helm Chart](https://github.com/kyverno/kyverno/blob/main/charts/kyverno/values.yaml). ### Kubevirt The Kubevirt bundle deploys [Kubevirt](https://github.com/kubevirt/kubevirt) and optionally [kubevirt-manager](https://kubevirt-manager.io/) - The bundle does add a `kubevirt` block, that allow to enable `kubevirt-manager`: ```yaml @@ -179,8 +207,8 @@ The bundle does add a `kubevirt` block, that allow to enable `kubevirt-manager`: # Specify the bundle to use bundles: -- targets: - - run://quay.io/kairos/community-bundles:kubevirt_latest + - targets: + - run://quay.io/kairos/community-bundles:kubevirt_latest # Specify kubevirt settings kubevirt: @@ -198,8 +226,8 @@ To configure the bundle, use the `longhorn` block: # Specify the bundle to use bundles: -- targets: - - run://quay.io/kairos/community-bundles:longhorn_latest + - targets: + - run://quay.io/kairos/community-bundles:longhorn_latest # Specify longhorn settings longhorn: @@ -209,7 +237,7 @@ longhorn: version: 1.4.0 ``` -Note that specifying `values` and `version` are optional. Specifying `values` allows you to +Note that specifying `values` and `version` are optional. Specifying `values` allows you to [customize the Helm Chart](https://longhorn.io/docs/latest/advanced-resources/deploy/customizing-default-settings/#using-helm). ### MetalLB @@ -223,8 +251,8 @@ The bundle does add a `metallb` block, that allow to set up the MetalLB version # Specify the bundle to use bundles: -- targets: - - run://quay.io/kairos/community-bundles:metallb_latest + - targets: + - run://quay.io/kairos/community-bundles:metallb_latest # Specify metallb settings metallb: @@ -239,20 +267,20 @@ Note, you might want to disable the default LoadBalancer of k3s, a full example hostname: kairoslab-{{ trunc 4 .MachineID }} users: -- name: kairos - ssh_authorized_keys: - # Add your github user here! - - github:mudler + - name: kairos + ssh_authorized_keys: + # Add your github user here! + - github:mudler k3s: enable: true args: - - --disable=servicelb + - --disable=servicelb # Specify the bundle to use bundles: -- targets: - - run://quay.io/kairos/community-bundles:metallb_latest + - targets: + - run://quay.io/kairos/community-bundles:metallb_latest # Specify metallb settings metallb: @@ -271,8 +299,8 @@ To configure the bundle, use the `multus` block: ```yaml # Specify the bundle to use bundles: -- targets: - - run://quay.io/kairos/community-bundles:multus_latest + - targets: + - run://quay.io/kairos/community-bundles:multus_latest # Specify multus settings. Here are the defaults: multus: @@ -326,8 +354,8 @@ The bundle does add a `suc` block, that allow to change the version: # Specify the bundle to use bundles: -- targets: - - run://quay.io/kairos/community-bundles:system-upgrade-controller_latest + - targets: + - run://quay.io/kairos/community-bundles:system-upgrade-controller_latest # Specify system-upgrade-controller settings suc: diff --git a/fluxcd/Dockerfile b/fluxcd/Dockerfile new file mode 100644 index 0000000..976fa70 --- /dev/null +++ b/fluxcd/Dockerfile @@ -0,0 +1,10 @@ +FROM alpine as build +ENV VERSION=2.1.1 + +ADD https://github.com/fluxcd/flux2/releases/download/v${VERSION}/flux_${VERSION}_linux_amd64.tar.gz /tmp +RUN tar xzf /tmp/flux_${VERSION}_linux_amd64.tar.gz -C / && \ + rm /tmp/flux_${VERSION}_linux_amd64.tar.gz + +FROM scratch +COPY --from=build /flux /flux +COPY ./bootstrap.sh /bootstrap.sh \ No newline at end of file diff --git a/fluxcd/bootstrap.sh b/fluxcd/bootstrap.sh new file mode 100755 index 0000000..cfaf4fe --- /dev/null +++ b/fluxcd/bootstrap.sh @@ -0,0 +1,103 @@ +#!/bin/bash + +set -x +# Override with the `env` key if necessary +export KUBECONFIG=/etc/rancher/k3s/k3s.yaml + +scriptdir="$( cd -- "$(dirname "$0")" || return >/dev/null 2>&1 ; pwd -P )" +flux="$scriptdir/flux" + +cleanup() { + kubectl delete configmap -n default flux-boostrap +} + +# Don't bootstrap if the cluster is already bootstrapped +if [[ $($flux version 2>/dev/null; echo $?) -eq 0 ]]; then + echo "Flux is already bootstrapped, exiting..." + exit 0 +fi + +# Determine what VCS we need to bootstrap +for vcs in bitbucket_server git github gitlab; do + if [[ $(kairos-agent config get flux.$vcs) != "null" ]]; then + version_control=$vcs + break + fi +done + +if [[ "${version_control}x" == "x" ]]; then + echo "Unable to determine what version control provider to use, exiting..." + exit 1 +fi + +# Get flux envs and settings for our VCS +mapfile -t envs < <(kairos-agent config get "flux.env") +mapfile -t args < <(kairos-agent config get "flux.$version_control") +declare -a cmdline + +# Set environment variables +# These are likely sensitive +set +x +for setting in "${envs[@]}"; do + env=$(echo "$setting" | cut -d: -f1) + value=$(echo "$setting" | cut -d':' -f2 | sed -E 's/^ //g') + if [[ "${value}x" != "x" ]]; then + export "$env"="$value" + fi +done +set -x + +# Set commandline args +for setting in "${args[@]}"; do + arg=$(echo "$setting" | cut -d: -f1) + value=$(echo "$setting" | cut -d':' -f2 | sed -E 's/^ //g') + if [[ "${value}x" != "x" ]]; then + cmdline+=("--$arg" "$value") + fi +done + +# Try to bootstrap Flux for 30 minutes, sleep 15 seconds between attempts +minutes=30 +sleep=15 +retry_attempt=1 +count=0 +active="false" + +if [[ "${#cmdline[@]}" -eq 0 ]]; then + echo "Flux was not configured, not running any Flux commands" + exit 2 +else + while [[ $count -le "$(( minutes * 60 / sleep ))" ]]; do + if [[ "$active" != "true" ]]; then + # Ensure only one host tries to bootstrap, whichever makes the configmap first + if ! timeout 2 kubectl version &> /dev/null; then + echo "Kubernetes API not ready yet, sleeping" + else + if ! kubectl create configmap flux-bootstrap --from-literal=hostname="$(hostname)"; then + echo "Unable to create configmap, another node may be active" + fi + + # The configmap exists but we must finally check if the hostname matches + if [[ "$(kubectl get configmap -n default flux-bootstrap -o jsonpath='{.data.hostname}')" != "$(hostname)" ]]; then + echo "Flux bootstrap ConfigMap exists but another node is active, exiting..." + exit 3 + fi + + # We must be the active node + active="true" + fi + else + if $flux bootstrap "${version_control/_/-}" "${cmdline[@]}"; then + cleanup + exit 0 + fi + fi + + echo "Install attempt $retry_attempt failed, retrying in $sleep seconds" + (( retry_attempt = retry_attempt + 1 )) + sleep $sleep + done +fi + +echo "Failed to bootstrap with Flux, timed out ($minutes minutes)" +exit 4 \ No newline at end of file