From 3f62386ca59c107083b122b1cde41777e08c8f0e Mon Sep 17 00:00:00 2001 From: Steven Fairchild Date: Tue, 1 Aug 2023 08:10:12 -0400 Subject: [PATCH] Update dnsmasq operator to use embeded files This cleans up the dnsmasq controller package by moving all go templates into the dnsmasq/scripts directory. --- pkg/api/defaults.go | 2 +- pkg/operator/controllers/dnsmasq/dnsmasq.go | 134 +++--------------- .../controllers/dnsmasq/dnsmasq_restart.go | 77 ++++------ pkg/operator/controllers/dnsmasq/scripts.go | 18 +++ .../dnsmasq/scripts/99-dnsmasq-restart.gotmpl | 45 ++++++ .../dnsmasq/scripts/aro-dnsmasq-pre.sh.gotmpl | 56 ++++++++ .../dnsmasq/scripts/dnsmasq.conf.gotmpl | 14 ++ .../dnsmasq/scripts/dnsmasq.service.gotmpl | 19 +++ 8 files changed, 196 insertions(+), 169 deletions(-) create mode 100644 pkg/operator/controllers/dnsmasq/scripts.go create mode 100644 pkg/operator/controllers/dnsmasq/scripts/99-dnsmasq-restart.gotmpl create mode 100644 pkg/operator/controllers/dnsmasq/scripts/aro-dnsmasq-pre.sh.gotmpl create mode 100644 pkg/operator/controllers/dnsmasq/scripts/dnsmasq.conf.gotmpl create mode 100644 pkg/operator/controllers/dnsmasq/scripts/dnsmasq.service.gotmpl diff --git a/pkg/api/defaults.go b/pkg/api/defaults.go index 1f3d452fb48..6dad80f2963 100644 --- a/pkg/api/defaults.go +++ b/pkg/api/defaults.go @@ -79,7 +79,7 @@ func DefaultOperatorFlags() OperatorFlags { "aro.banner.enabled": flagFalse, "aro.checker.enabled": flagTrue, "aro.dnsmasq.enabled": flagTrue, - "aro.restartdnsmasq.enabled": flagFalse, + "aro.restartdnsmasq.enabled": flagTrue, "aro.genevalogging.enabled": flagTrue, "aro.imageconfig.enabled": flagTrue, "aro.ingress.enabled": flagTrue, diff --git a/pkg/operator/controllers/dnsmasq/dnsmasq.go b/pkg/operator/controllers/dnsmasq/dnsmasq.go index 296fe6bf97e..539993b194e 100644 --- a/pkg/operator/controllers/dnsmasq/dnsmasq.go +++ b/pkg/operator/controllers/dnsmasq/dnsmasq.go @@ -5,6 +5,7 @@ package dnsmasq import ( "bytes" + _ "embed" "encoding/json" "fmt" "text/template" @@ -17,105 +18,17 @@ import ( "k8s.io/apimachinery/pkg/runtime" ) -var t = template.Must(template.New("").Parse(` - -{{ define "dnsmasq.conf" }} -resolv-file=/etc/resolv.conf.dnsmasq -strict-order -address=/api.{{ .ClusterDomain }}/{{ .APIIntIP }} -address=/api-int.{{ .ClusterDomain }}/{{ .APIIntIP }} -address=/.apps.{{ .ClusterDomain }}/{{ .IngressIP }} -{{- range $GatewayDomain := .GatewayDomains }} -address=/{{ $GatewayDomain }}/{{ $.GatewayPrivateEndpointIP }} -{{- end }} -user=dnsmasq -group=dnsmasq -no-hosts -cache-size=0 -{{ end }} - -{{ define "dnsmasq.service" }} -[Unit] -Description=DNS caching server. -After=network-online.target -Before=bootkube.service - -[Service] -# ExecStartPre will create a copy of the customer current resolv.conf file and make it upstream DNS. -# This file is a product of user DNS settings on the VNET. We will replace this file to point to -# dnsmasq instance on the node. dnsmasq will inject certain dns records we need and forward rest of the queries to -# resolv.conf.dnsmasq upstream customer dns. -ExecStartPre=/bin/bash /usr/local/bin/aro-dnsmasq-pre.sh -ExecStart=/usr/sbin/dnsmasq -k -ExecStopPost=/bin/bash -c '/bin/mv /etc/resolv.conf.dnsmasq /etc/resolv.conf; /usr/sbin/restorecon /etc/resolv.conf' -Restart=always - -[Install] -WantedBy=multi-user.target -{{ end }} - -{{ define "aro-dnsmasq-pre.sh" }} -#!/bin/bash -set -euo pipefail - -# This bash script is a part of the ARO DnsMasq configuration -# It's deployed as part of the 99-aro-dns-* machine config -# See https://github.com/Azure/ARO-RP - -# This file can be rerun and the effect is idempotent, output might change if the DHCP configuration changes - -TMPSELFRESOLV=$(mktemp) -TMPNETRESOLV=$(mktemp) - -echo "# Generated for dnsmasq.service - should point to self" > $TMPSELFRESOLV -echo "# Generated for dnsmasq.service - should contain DHCP configured DNS" > $TMPNETRESOLV - -if nmcli device show br-ex; then - echo "OVN mode - br-ex device exists" - #getting DNS search strings - SEARCH_RAW=$(nmcli --get IP4.DOMAIN device show br-ex) - #getting DNS servers - NAMESERVER_RAW=$(nmcli --get IP4.DNS device show br-ex | tr -s " | " "\n") - LOCAL_IPS_RAW=$(nmcli --get IP4.ADDRESS device show br-ex) -else - NETDEV=$(nmcli --get device connection show --active | head -n 1) #there should be only one active device - echo "OVS SDN mode - br-ex not found, using device $NETDEV" - SEARCH_RAW=$(nmcli --get IP4.DOMAIN device show $NETDEV) - NAMESERVER_RAW=$(nmcli --get IP4.DNS device show $NETDEV | tr -s " | " "\n") - LOCAL_IPS_RAW=$(nmcli --get IP4.ADDRESS device show $NETDEV) -fi - -#search line -echo "search $SEARCH_RAW" | tr '\n' ' ' >> $TMPNETRESOLV -echo "" >> $TMPNETRESOLV -echo "search $SEARCH_RAW" | tr '\n' ' ' >> $TMPSELFRESOLV -echo "" >> $TMPSELFRESOLV - -#nameservers as separate lines -echo "$NAMESERVER_RAW" | while read -r line -do - echo "nameserver $line" >> $TMPNETRESOLV -done -# device IPs are returned in address/mask format -echo "$LOCAL_IPS_RAW" | while read -r line -do - echo "nameserver $line" | cut -d'/' -f 1 >> $TMPSELFRESOLV -done - -# done, copying files to destination locations and cleaning up -/bin/cp $TMPNETRESOLV /etc/resolv.conf.dnsmasq -chmod 0744 /etc/resolv.conf.dnsmasq -/bin/cp $TMPSELFRESOLV /etc/resolv.conf -/usr/sbin/restorecon /etc/resolv.conf -/bin/rm $TMPNETRESOLV -/bin/rm $TMPSELFRESOLV -{{ end }} -`)) +const ( + configFileName = "dnsmasq.conf" + unitFileName = "dnsmasq.service" + prescriptFileName = "aro-dnsmasq-pre.sh" +) func config(clusterDomain, apiIntIP, ingressIP string, gatewayDomains []string, gatewayPrivateEndpointIP string) ([]byte, error) { + t := template.Must(template.New(configFileName).Parse(preScriptFile)) buf := &bytes.Buffer{} - err := t.ExecuteTemplate(buf, "dnsmasq.conf", &struct { + err := t.ExecuteTemplate(buf, configFileName, &struct { ClusterDomain string APIIntIP string IngressIP string @@ -136,9 +49,10 @@ func config(clusterDomain, apiIntIP, ingressIP string, gatewayDomains []string, } func service() (string, error) { + t := template.Must(template.New(unitFileName).Parse(unitFile)) buf := &bytes.Buffer{} - err := t.ExecuteTemplate(buf, "dnsmasq.service", nil) + err := t.ExecuteTemplate(buf, unitFileName, nil) if err != nil { return "", err } @@ -147,9 +61,10 @@ func service() (string, error) { } func startpre() ([]byte, error) { + t := template.Must(template.New(prescriptFileName).Parse(configFile)) buf := &bytes.Buffer{} - err := t.ExecuteTemplate(buf, "aro-dnsmasq-pre.sh", nil) + err := t.ExecuteTemplate(buf, prescriptFileName, nil) if err != nil { return nil, err } @@ -183,7 +98,7 @@ func ignition2Config(clusterDomain, apiIntIP, ingressIP string, gatewayDomains [ Node: ign2types.Node{ Filesystem: "root", Overwrite: to.BoolPtr(true), - Path: "/etc/dnsmasq.conf", + Path: "/etc/" + configFileName, User: &ign2types.NodeUser{ Name: "root", }, @@ -199,7 +114,7 @@ func ignition2Config(clusterDomain, apiIntIP, ingressIP string, gatewayDomains [ Node: ign2types.Node{ Filesystem: "root", Overwrite: to.BoolPtr(true), - Path: "/usr/local/bin/aro-dnsmasq-pre.sh", + Path: "/usr/local/bin/" + prescriptFileName, User: &ign2types.NodeUser{ Name: "root", }, @@ -218,34 +133,19 @@ func ignition2Config(clusterDomain, apiIntIP, ingressIP string, gatewayDomains [ { Contents: service, Enabled: to.BoolPtr(true), - Name: "dnsmasq.service", + Name: unitFileName, }, }, }, } if restartDnsmasq { - restartDnsmasqRaw, err := nmDispatcherRestartDnsmasq() + restartDnsmasqScript, err := nmDispatcherRestartDnsmasq() if err != nil { return nil, err } - ign.Storage.Files = append(ign.Storage.Files, ign2types.File{ - Node: ign2types.Node{ - Filesystem: "root", - Overwrite: to.BoolPtr(true), - Path: "/etc/NetworkManager/dispatcher.d/99-dnsmasq-restart", - User: &ign2types.NodeUser{ - Name: "root", - }, - }, - FileEmbedded1: ign2types.FileEmbedded1{ - Contents: ign2types.FileContents{ - Source: dataurl.EncodeBytes(restartDnsmasqRaw), - }, - Mode: to.IntPtr(0744), - }, - }) + ign.Storage.Files = append(ign.Storage.Files, restartScriptIgnFile(restartDnsmasqScript)) } return ign, nil diff --git a/pkg/operator/controllers/dnsmasq/dnsmasq_restart.go b/pkg/operator/controllers/dnsmasq/dnsmasq_restart.go index db89630ca34..0d5774801f7 100644 --- a/pkg/operator/controllers/dnsmasq/dnsmasq_restart.go +++ b/pkg/operator/controllers/dnsmasq/dnsmasq_restart.go @@ -6,66 +6,41 @@ package dnsmasq import ( "bytes" "text/template" -) - -const restartTemplate = `{{ define "99-dnsmasq-restart" }} -#!/bin/sh -# This is a NetworkManager dispatcher script to restart dnsmasq -# in the event of a network interface change (e. g. host servicing event https://learn.microsoft.com/en-us/azure/developer/intro/hosting-apps-on-azure) -# this will restart dnsmasq, reapplying our /etc/resolv.conf file and overwriting any modifications made by NetworkManager - -interface=$1 -action=$2 - -log() { - logger -i "$0" -t '99-DNSMASQ-RESTART SCRIPT' "$@" -} - -# log dns configuration information relevant to SRE while troubleshooting -# The line break used here is important for formatting -check_dns_files() { - log "/etc/resolv.conf contents - - $(cat /etc/resolv.conf)" - - log "$(echo -n \"/etc/resolv.conf file metadata: \") $(ls -lZ /etc/resolv.conf)" - - log "/etc/resolv.conf.dnsmasq contents - $(cat /etc/resolv.conf.dnsmasq)" - - log "$(echo -n "/etc/resolv.conf.dnsmasq file metadata: ") $(ls -lZ /etc/resolv.conf.dnsmasq)" -} - -if [[ $interface == eth* && $action == "up" ]] || [[ $interface == eth* && $action == "down" ]] || [[ $interface == enP* && $action == "up" ]] || [[ $interface == enP* && $action == "down" ]]; then - log "$action happened on $interface, connection state is now $CONNECTIVITY_STATE" - log "Pre dnsmasq restart file information" - check_dns_files - log "restarting dnsmasq now" - if systemctl try-restart dnsmasq --wait; then - log "dnsmasq successfully restarted" - log "Post dnsmasq restart file information" - check_dns_files - else - log "failed to restart dnsmasq" - fi -fi + "github.com/Azure/go-autorest/autorest/to" + ign2types "github.com/coreos/ignition/config/v2_2/types" + "github.com/vincent-petithory/dataurl" +) -exit 0 -{{ end }} -` - -func restartDnsmasqTemplate() *template.Template { - return template.Must(template.New("").Parse(restartTemplate)) -} +const restartScriptFileName = "99-dnsmasq-restart" func nmDispatcherRestartDnsmasq() ([]byte, error) { + t := template.Must(template.New(restartScriptFileName).Parse(restartScript)) buf := &bytes.Buffer{} - err := restartDnsmasqTemplate().ExecuteTemplate(buf, "99-dnsmasq-restart", nil) + err := t.ExecuteTemplate(buf, restartScriptFileName, nil) if err != nil { return nil, err } return buf.Bytes(), nil } + +func restartScriptIgnFile(data []byte) ign2types.File { + return ign2types.File{ + Node: ign2types.Node{ + Filesystem: "root", + Overwrite: to.BoolPtr(true), + Path: "/etc/NetworkManager/dispatcher.d/" + restartScriptFileName, + User: &ign2types.NodeUser{ + Name: "root", + }, + }, + FileEmbedded1: ign2types.FileEmbedded1{ + Contents: ign2types.FileContents{ + Source: dataurl.EncodeBytes(data), + }, + Mode: to.IntPtr(0744), + }, + } +} diff --git a/pkg/operator/controllers/dnsmasq/scripts.go b/pkg/operator/controllers/dnsmasq/scripts.go new file mode 100644 index 00000000000..46b79018019 --- /dev/null +++ b/pkg/operator/controllers/dnsmasq/scripts.go @@ -0,0 +1,18 @@ +package dnsmasq + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import _ "embed" + +//go:embed scripts/dnsmasq.conf.gotmpl +var configFile string + +//go:embed scripts/dnsmasq.service.gotmpl +var unitFile string + +//go:embed scripts/aro-dnsmasq-pre.sh.gotmpl +var preScriptFile string + +//go:embed scripts/99-dnsmasq-restart.gotmpl +var restartScript string diff --git a/pkg/operator/controllers/dnsmasq/scripts/99-dnsmasq-restart.gotmpl b/pkg/operator/controllers/dnsmasq/scripts/99-dnsmasq-restart.gotmpl new file mode 100644 index 00000000000..0d98a139d0d --- /dev/null +++ b/pkg/operator/controllers/dnsmasq/scripts/99-dnsmasq-restart.gotmpl @@ -0,0 +1,45 @@ +{{ define "99-dnsmasq-restart" }} +#!/bin/sh +# This is a NetworkManager dispatcher script to restart dnsmasq +# in the event of a network interface change (e. g. host servicing event https://learn.microsoft.com/en-us/azure/developer/intro/hosting-apps-on-azure) +# this will restart dnsmasq, reapplying our /etc/resolv.conf file and overwriting any modifications made by NetworkManager + +interface=$1 +action=$2 + +log() { + logger -i "$0" -t '99-DNSMASQ-RESTART SCRIPT' "$@" +} + +# log dns configuration information relevant to SRE while troubleshooting +# The line break used here is important for formatting +log_dns_files() { + log "/etc/resolv.conf contents + + $(cat /etc/resolv.conf)" + + log "$(echo -n \"/etc/resolv.conf file metadata: \") $(ls -lZ /etc/resolv.conf)" + + log "/etc/resolv.conf.dnsmasq contents + + $(cat /etc/resolv.conf.dnsmasq)" + + log "$(echo -n "/etc/resolv.conf.dnsmasq file metadata: ") $(ls -lZ /etc/resolv.conf.dnsmasq)" +} + +if [[ $interface == eth* && $action == "up" ]] || [[ $interface == eth* && $action == "down" ]] || [[ $interface == enP* && $action == "up" ]] || [[ $interface == enP* && $action == "down" ]]; then + log "$action happened on $interface, connection state is now $CONNECTIVITY_STATE" + log "Pre dnsmasq restart file information" + log_dns_files + log "restarting dnsmasq now" + if systemctl try-restart dnsmasq --wait; then + log "dnsmasq successfully restarted" + log "Post dnsmasq restart file information" + log_dns_files + else + log "failed to restart dnsmasq" + fi +fi + +exit 0 +{{ end }} \ No newline at end of file diff --git a/pkg/operator/controllers/dnsmasq/scripts/aro-dnsmasq-pre.sh.gotmpl b/pkg/operator/controllers/dnsmasq/scripts/aro-dnsmasq-pre.sh.gotmpl new file mode 100644 index 00000000000..6dc2dd85000 --- /dev/null +++ b/pkg/operator/controllers/dnsmasq/scripts/aro-dnsmasq-pre.sh.gotmpl @@ -0,0 +1,56 @@ +{{ define "aro-dnsmasq-pre.sh" }} +#!/bin/bash +set -euo pipefail + +# This bash script is a part of the ARO DnsMasq configuration +# It's deployed as part of the 99-aro-dns-* machine config +# See https://github.com/Azure/ARO-RP + +# This file can be rerun and the effect is idempotent, output might change if the DHCP configuration changes + +TMPSELFRESOLV=$(mktemp) +TMPNETRESOLV=$(mktemp) + +echo "# Generated for dnsmasq.service - should point to self" > $TMPSELFRESOLV +echo "# Generated for dnsmasq.service - should contain DHCP configured DNS" > $TMPNETRESOLV + +if nmcli device show br-ex; then + echo "OVN mode - br-ex device exists" + #getting DNS search strings + SEARCH_RAW=$(nmcli --get IP4.DOMAIN device show br-ex) + #getting DNS servers + NAMESERVER_RAW=$(nmcli --get IP4.DNS device show br-ex | tr -s " | " "\n") + LOCAL_IPS_RAW=$(nmcli --get IP4.ADDRESS device show br-ex) +else + NETDEV=$(nmcli --get device connection show --active | head -n 1) #there should be only one active device + echo "OVS SDN mode - br-ex not found, using device $NETDEV" + SEARCH_RAW=$(nmcli --get IP4.DOMAIN device show $NETDEV) + NAMESERVER_RAW=$(nmcli --get IP4.DNS device show $NETDEV | tr -s " | " "\n") + LOCAL_IPS_RAW=$(nmcli --get IP4.ADDRESS device show $NETDEV) +fi + +#search line +echo "search $SEARCH_RAW" | tr '\n' ' ' >> $TMPNETRESOLV +echo "" >> $TMPNETRESOLV +echo "search $SEARCH_RAW" | tr '\n' ' ' >> $TMPSELFRESOLV +echo "" >> $TMPSELFRESOLV + +#nameservers as separate lines +echo "$NAMESERVER_RAW" | while read -r line +do + echo "nameserver $line" >> $TMPNETRESOLV +done +# device IPs are returned in address/mask format +echo "$LOCAL_IPS_RAW" | while read -r line +do + echo "nameserver $line" | cut -d'/' -f 1 >> $TMPSELFRESOLV +done + +# done, copying files to destination locations and cleaning up +/bin/cp $TMPNETRESOLV /etc/resolv.conf.dnsmasq +chmod 0744 /etc/resolv.conf.dnsmasq +/bin/cp $TMPSELFRESOLV /etc/resolv.conf +/usr/sbin/restorecon /etc/resolv.conf +/bin/rm $TMPNETRESOLV +/bin/rm $TMPSELFRESOLV +{{ end }} \ No newline at end of file diff --git a/pkg/operator/controllers/dnsmasq/scripts/dnsmasq.conf.gotmpl b/pkg/operator/controllers/dnsmasq/scripts/dnsmasq.conf.gotmpl new file mode 100644 index 00000000000..a18c3e44e9c --- /dev/null +++ b/pkg/operator/controllers/dnsmasq/scripts/dnsmasq.conf.gotmpl @@ -0,0 +1,14 @@ +{{ define "dnsmasq.conf" }} +resolv-file=/etc/resolv.conf.dnsmasq +strict-order +address=/api.{{ .ClusterDomain }}/{{ .APIIntIP }} +address=/api-int.{{ .ClusterDomain }}/{{ .APIIntIP }} +address=/.apps.{{ .ClusterDomain }}/{{ .IngressIP }} +{{- range $GatewayDomain := .GatewayDomains }} +address=/{{ $GatewayDomain }}/{{ $.GatewayPrivateEndpointIP }} +{{- end }} +user=dnsmasq +group=dnsmasq +no-hosts +cache-size=0 +{{ end }} \ No newline at end of file diff --git a/pkg/operator/controllers/dnsmasq/scripts/dnsmasq.service.gotmpl b/pkg/operator/controllers/dnsmasq/scripts/dnsmasq.service.gotmpl new file mode 100644 index 00000000000..fabef057c01 --- /dev/null +++ b/pkg/operator/controllers/dnsmasq/scripts/dnsmasq.service.gotmpl @@ -0,0 +1,19 @@ +{{ define "dnsmasq.service" }} +[Unit] +Description=DNS caching server. +After=network-online.target +Before=bootkube.service + +[Service] +# ExecStartPre will create a copy of the customer current resolv.conf file and make it upstream DNS. +# This file is a product of user DNS settings on the VNET. We will replace this file to point to +# dnsmasq instance on the node. dnsmasq will inject certain dns records we need and forward rest of the queries to +# resolv.conf.dnsmasq upstream customer dns. +ExecStartPre=/bin/bash /usr/local/bin/aro-dnsmasq-pre.sh +ExecStart=/usr/sbin/dnsmasq -k +ExecStopPost=/bin/bash -c '/bin/mv /etc/resolv.conf.dnsmasq /etc/resolv.conf; /usr/sbin/restorecon /etc/resolv.conf' +Restart=always + +[Install] +WantedBy=multi-user.target +{{ end }} \ No newline at end of file