From 1ee42feb78ae64ca694bf832be3ee17bde8cbea7 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Tue, 13 Jun 2023 22:52:57 +1000 Subject: [PATCH] [ARO-3194] Vendor dnsmasq into the operator (#2932) vendor in dnsmasq config into the operator, rather than importing the installer --- go.mod | 4 +- .../controllers/dnsmasq/cluster_controller.go | 3 +- pkg/operator/controllers/dnsmasq/dnsmasq.go | 267 ++++++++++++++++++ 3 files changed, 270 insertions(+), 4 deletions(-) create mode 100644 pkg/operator/controllers/dnsmasq/dnsmasq.go diff --git a/go.mod b/go.mod index 97fd3ff57a3..eef2869162e 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,7 @@ require ( github.com/coreos/go-oidc v2.2.1+incompatible github.com/coreos/go-semver v0.3.0 github.com/coreos/go-systemd/v22 v22.3.2 + github.com/coreos/ignition v0.35.0 github.com/coreos/ignition/v2 v2.14.0 github.com/coreos/stream-metadata-go v0.2.0 github.com/form3tech-oss/jwt-go v3.2.5+incompatible @@ -59,6 +60,7 @@ require ( github.com/stretchr/testify v1.7.1 github.com/tebeka/selenium v0.9.9 github.com/ugorji/go/codec v1.2.7 + github.com/vincent-petithory/dataurl v1.0.0 golang.org/x/crypto v0.9.0 golang.org/x/net v0.10.0 golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 @@ -117,7 +119,6 @@ require ( github.com/containers/ocicrypt v1.1.3 // indirect github.com/containers/storage v1.39.0 // indirect github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect - github.com/coreos/ignition v0.35.0 // indirect github.com/coreos/vcontext v0.0.0-20220326205524-7fcaf69e7050 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect @@ -227,7 +228,6 @@ require ( github.com/ulikunitz/xz v0.5.10 // indirect github.com/vbatts/tar-split v0.11.2 // indirect github.com/vbauerster/mpb/v7 v7.4.1 // indirect - github.com/vincent-petithory/dataurl v1.0.0 // indirect github.com/vmware/govmomi v0.27.4 // indirect github.com/xlab/treeprint v1.1.0 // indirect go.etcd.io/bbolt v1.3.6 // indirect diff --git a/pkg/operator/controllers/dnsmasq/cluster_controller.go b/pkg/operator/controllers/dnsmasq/cluster_controller.go index ab1d342e176..b316b3e5709 100644 --- a/pkg/operator/controllers/dnsmasq/cluster_controller.go +++ b/pkg/operator/controllers/dnsmasq/cluster_controller.go @@ -6,7 +6,6 @@ package dnsmasq import ( "context" - "github.com/openshift/installer/pkg/aro/dnsmasq" mcv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" "github.com/sirupsen/logrus" kruntime "k8s.io/apimachinery/pkg/runtime" @@ -89,7 +88,7 @@ func (r *ClusterReconciler) SetupWithManager(mgr ctrl.Manager) error { func reconcileMachineConfigs(ctx context.Context, instance *arov1alpha1.Cluster, dh dynamichelper.Interface, mcps ...mcv1.MachineConfigPool) error { var resources []kruntime.Object for _, mcp := range mcps { - resource, err := dnsmasq.MachineConfig(instance.Spec.Domain, instance.Spec.APIIntIP, instance.Spec.IngressIP, mcp.Name, instance.Spec.GatewayDomains, instance.Spec.GatewayPrivateEndpointIP) + resource, err := dnsmasqMachineConfig(instance.Spec.Domain, instance.Spec.APIIntIP, instance.Spec.IngressIP, mcp.Name, instance.Spec.GatewayDomains, instance.Spec.GatewayPrivateEndpointIP) if err != nil { return err } diff --git a/pkg/operator/controllers/dnsmasq/dnsmasq.go b/pkg/operator/controllers/dnsmasq/dnsmasq.go new file mode 100644 index 00000000000..9182fb7c040 --- /dev/null +++ b/pkg/operator/controllers/dnsmasq/dnsmasq.go @@ -0,0 +1,267 @@ +package dnsmasq + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "bytes" + "encoding/json" + "fmt" + "text/template" + + ign2types "github.com/coreos/ignition/config/v2_2/types" + ignutil "github.com/coreos/ignition/v2/config/util" + mcv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" + "github.com/vincent-petithory/dataurl" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +var t = template.Must(template.New("").Parse(` + +{{ define "dnsmasq.conf" }} +resolv-file=/etc/resolv.conf.dnsmasq +strict-order +address=/api.{{ .ClusterDomain }}/{{ .APIIntIP }} +address=/api-int.{{ .ClusterDomain }}/{{ .APIIntIP }} +address=/.apps.{{ .ClusterDomain }}/{{ .IngressIP }} +{{- range $GatewayDomain := .GatewayDomains }} +address=/{{ $GatewayDomain }}/{{ $.GatewayPrivateEndpointIP }} +{{- end }} +user=dnsmasq +group=dnsmasq +no-hosts +cache-size=0 +{{ end }} + +{{ define "dnsmasq.service" }} +[Unit] +Description=DNS caching server. +After=network-online.target +Before=bootkube.service + +[Service] +# ExecStartPre will create a copy of the customer current resolv.conf file and make it upstream DNS. +# This file is a product of user DNS settings on the VNET. We will replace this file to point to +# dnsmasq instance on the node. dnsmasq will inject certain dns records we need and forward rest of the queries to +# resolv.conf.dnsmasq upstream customer dns. +ExecStartPre=/bin/bash /usr/local/bin/aro-dnsmasq-pre.sh +ExecStart=/usr/sbin/dnsmasq -k +ExecStopPost=/bin/bash -c '/bin/mv /etc/resolv.conf.dnsmasq /etc/resolv.conf; /usr/sbin/restorecon /etc/resolv.conf' +Restart=always + +[Install] +WantedBy=multi-user.target +{{ end }} + +{{ define "aro-dnsmasq-pre.sh" }} +#!/bin/bash +set -euo pipefail + +# This bash script is a part of the ARO DnsMasq configuration +# It's deployed as part of the 99-aro-dns-* machine config +# See https://github.com/Azure/ARO-RP + +# This file can be rerun and the effect is idempotent, output might change if the DHCP configuration changes + +TMPSELFRESOLV=$(mktemp) +TMPNETRESOLV=$(mktemp) + +echo "# Generated for dnsmasq.service - should point to self" > $TMPSELFRESOLV +echo "# Generated for dnsmasq.service - should contain DHCP configured DNS" > $TMPNETRESOLV + +if nmcli device show br-ex; then + echo "OVN mode - br-ex device exists" + #getting DNS search strings + SEARCH_RAW=$(nmcli --get IP4.DOMAIN device show br-ex) + #getting DNS servers + NAMESERVER_RAW=$(nmcli --get IP4.DNS device show br-ex | tr -s " | " "\n") + LOCAL_IPS_RAW=$(nmcli --get IP4.ADDRESS device show br-ex) +else + NETDEV=$(nmcli --get device connection show --active | head -n 1) #there should be only one active device + echo "OVS SDN mode - br-ex not found, using device $NETDEV" + SEARCH_RAW=$(nmcli --get IP4.DOMAIN device show $NETDEV) + NAMESERVER_RAW=$(nmcli --get IP4.DNS device show $NETDEV | tr -s " | " "\n") + LOCAL_IPS_RAW=$(nmcli --get IP4.ADDRESS device show $NETDEV) +fi + +#search line +echo "search $SEARCH_RAW" | tr '\n' ' ' >> $TMPNETRESOLV +echo "" >> $TMPNETRESOLV +echo "search $SEARCH_RAW" | tr '\n' ' ' >> $TMPSELFRESOLV +echo "" >> $TMPSELFRESOLV + +#nameservers as separate lines +echo "$NAMESERVER_RAW" | while read -r line +do + echo "nameserver $line" >> $TMPNETRESOLV +done +# device IPs are returned in address/mask format +echo "$LOCAL_IPS_RAW" | while read -r line +do + echo "nameserver $line" | cut -d'/' -f 1 >> $TMPSELFRESOLV +done + +# done, copying files to destination locations and cleaning up +/bin/cp $TMPNETRESOLV /etc/resolv.conf.dnsmasq +chmod 0744 /etc/resolv.conf.dnsmasq +/bin/cp $TMPSELFRESOLV /etc/resolv.conf +/usr/sbin/restorecon /etc/resolv.conf +/bin/rm $TMPNETRESOLV +/bin/rm $TMPSELFRESOLV +{{ end }} +`)) + +func config(clusterDomain, apiIntIP, ingressIP string, gatewayDomains []string, gatewayPrivateEndpointIP string) ([]byte, error) { + buf := &bytes.Buffer{} + + err := t.ExecuteTemplate(buf, "dnsmasq.conf", &struct { + ClusterDomain string + APIIntIP string + IngressIP string + GatewayDomains []string + GatewayPrivateEndpointIP string + }{ + ClusterDomain: clusterDomain, + APIIntIP: apiIntIP, + IngressIP: ingressIP, + GatewayDomains: gatewayDomains, + GatewayPrivateEndpointIP: gatewayPrivateEndpointIP, + }) + if err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +func service() (string, error) { + buf := &bytes.Buffer{} + + err := t.ExecuteTemplate(buf, "dnsmasq.service", nil) + if err != nil { + return "", err + } + + return buf.String(), nil +} + +func startpre() ([]byte, error) { + buf := &bytes.Buffer{} + + err := t.ExecuteTemplate(buf, "aro-dnsmasq-pre.sh", nil) + if err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +func ignition2Config(clusterDomain, apiIntIP, ingressIP string, gatewayDomains []string, gatewayPrivateEndpointIP string) (*ign2types.Config, error) { + service, err := service() + if err != nil { + return nil, err + } + + config, err := config(clusterDomain, apiIntIP, ingressIP, gatewayDomains, gatewayPrivateEndpointIP) + if err != nil { + return nil, err + } + + startpre, err := startpre() + if err != nil { + return nil, err + } + + return &ign2types.Config{ + Ignition: ign2types.Ignition{ + Version: ign2types.MaxVersion.String(), + }, + Storage: ign2types.Storage{ + Files: []ign2types.File{ + { + Node: ign2types.Node{ + Filesystem: "root", + Overwrite: ignutil.BoolToPtr(true), + Path: "/etc/dnsmasq.conf", + User: &ign2types.NodeUser{ + Name: "root", + }, + }, + FileEmbedded1: ign2types.FileEmbedded1{ + Contents: ign2types.FileContents{ + Source: dataurl.EncodeBytes(config), + }, + Mode: ignutil.IntToPtr(0644), + }, + }, + { + Node: ign2types.Node{ + Filesystem: "root", + Overwrite: ignutil.BoolToPtr(true), + Path: "/usr/local/bin/aro-dnsmasq-pre.sh", + User: &ign2types.NodeUser{ + Name: "root", + }, + }, + FileEmbedded1: ign2types.FileEmbedded1{ + Contents: ign2types.FileContents{ + Source: dataurl.EncodeBytes(startpre), + }, + Mode: ignutil.IntToPtr(0744), + }, + }, + }, + }, + Systemd: ign2types.Systemd{ + Units: []ign2types.Unit{ + { + Contents: service, + Enabled: ignutil.BoolToPtr(true), + Name: "dnsmasq.service", + }, + }, + }, + }, nil +} + +func dnsmasqMachineConfig(clusterDomain, apiIntIP, ingressIP, role string, gatewayDomains []string, gatewayPrivateEndpointIP string) (*mcv1.MachineConfig, error) { + ignConfig, err := ignition2Config(clusterDomain, apiIntIP, ingressIP, gatewayDomains, gatewayPrivateEndpointIP) + if err != nil { + return nil, err + } + + b, err := json.Marshal(ignConfig) + if err != nil { + return nil, err + } + + // canonicalise the machineconfig payload the same way as MCO + var i interface{} + err = json.Unmarshal(b, &i) + if err != nil { + return nil, err + } + + rawExt := runtime.RawExtension{} + rawExt.Raw, err = json.Marshal(i) + if err != nil { + return nil, err + } + + return &mcv1.MachineConfig{ + TypeMeta: metav1.TypeMeta{ + APIVersion: mcv1.SchemeGroupVersion.String(), + Kind: "MachineConfig", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("99-%s-aro-dns", role), + Labels: map[string]string{ + "machineconfiguration.openshift.io/role": role, + }, + }, + Spec: mcv1.MachineConfigSpec{ + Config: rawExt, + }, + }, nil +}