Add e2e tests for FQDN NetworkPolicy rule to support applications with cached DNS resolutions.

test/e2e/antreapolicy_test.go

@@ -15,6 +15,7 @@
 package e2e
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
@@ -24,6 +25,7 @@ import (
 	"strings"
 	"sync"
 	"testing"
+	"text/template"
 	"time"
 
 	log "github.com/sirupsen/logrus"
@@ -35,9 +37,11 @@ import (
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/utils/ptr"
 
 	"antrea.io/antrea/pkg/agent/apis"
 	crdv1beta1 "antrea.io/antrea/pkg/apis/crd/v1beta1"
+	agentconfig "antrea.io/antrea/pkg/config/agent"
 	"antrea.io/antrea/pkg/controller/networkpolicy"
 	"antrea.io/antrea/pkg/features"
 	. "antrea.io/antrea/test/e2e/utils"
@@ -5202,3 +5206,252 @@ func testAntreaClusterNetworkPolicyStats(t *testing.T, data *TestData) {
 	}
 	k8sUtils.Cleanup(namespaces)
 }
+
+// TestFQDNCacheMinTTL tests stable FQDN access for applications with cached DNS resolutions
+// when FQDN NetworkPolicy are in use and the FQDN-to-IP resolution changes frequently.
+func TestFQDNCacheMinTTL(t *testing.T) {
+	const (
+		testFQDN = "fqdn-test-pod.lfx.test"
+		dnsPort  = 53
+		dnsTTL   = 5
+	)
+
+	skipIfAntreaPolicyDisabled(t)
+	skipIfNotIPv4Cluster(t)
+	skipIfIPv6Cluster(t)
+	skipIfNotRequired(t, "mode-irrelevant")
+
+	data, err := setupTest(t)
+	if err != nil {
+		t.Fatalf("Error when setting up test: %v", err)
+	}
+	defer teardownTest(t, data)
+
+	// create two agnhost Pods and get their IPv4 addresses. The IP of these Pods will be mapped against the FQDN.
+	podCount := 2
+	agnhostPodIPs := make([]*PodIPs, podCount)
+	for i := 0; i < podCount; i++ {
+		agnhostPodIPs[i] = createHttpAgnhostPod(t, data)
+	}
+
+	// get IPv4 addresses of the agnhost Pods created.
+	agnhostPodOneIP, _ := agnhostPodIPs[0].AsStrings()
+	agnhostPodTwoIP, _ := agnhostPodIPs[1].AsStrings()
+
+	// create customDNS Service and get its ClusterIP.
+	customDNSService, err := data.CreateServiceWithAnnotations("custom-dns-service", data.testNamespace, dnsPort,
+		dnsPort, v1.ProtocolUDP, map[string]string{"app": "custom-dns"}, false,
+		false, v1.ServiceTypeClusterIP, ptr.To[v1.IPFamily](v1.IPv4Protocol), map[string]string{})
+	require.NoError(t, err, "Error creating custom DNS Service")
+	dnsServiceIP := customDNSService.Spec.ClusterIP
+
+	// create a ConfigMap for the custom DNS server, mapping IP of agnhost Pod 1 to the FQDN.
+	configMap := &v1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "custom-dns-config",
+			Namespace: data.testNamespace,
+		},
+		Data: createDNSConfig(t, map[string]string{agnhostPodOneIP: testFQDN}, dnsTTL),
+	}
+	customDNSConfigMap, err := data.CreateConfigMap(configMap)
+	require.NoError(t, err, "failed to create custom DNS ConfigMap")
+
+	createCustomDNSPod(t, data, configMap.Name)
+
+	// set the custom DNS server IP address in Antrea ConfigMap.
+	setDNSServerAddressInAntrea(t, data, dnsServiceIP)
+	defer setDNSServerAddressInAntrea(t, data, "") //reset after the test.
+
+	createPolicyForFQDNCacheMinTTL(t, data, testFQDN, "test-anp-fqdn", "custom-dns", "fqdn-cache-test")
+	require.NoError(t, NewPodBuilder(toolboxPodName, data.testNamespace, ToolboxImage).
+		WithLabels(map[string]string{"app": "fqdn-cache-test"}).
+		WithContainerName(toolboxContainerName).
+		WithCustomDNSConfig(&v1.PodDNSConfig{Nameservers: []string{dnsServiceIP}}).
+		Create(data))
+	require.NoError(t, data.podWaitForRunning(defaultTimeout, toolboxPodName, data.testNamespace))
+
+	curlFQDN := func(target string) (string, error) {
+		cmd := []string{"curl", target}
+		stdout, stderr, err := data.RunCommandFromPod(data.testNamespace, toolboxPodName, toolboxContainerName, cmd)
+		if err != nil {
+			return "", fmt.Errorf("error when running command '%s' on Pod '%s': %v, stdout: <%v>, stderr: <%v>",
+				strings.Join(cmd, " "), toolboxPodName, err, stdout, stderr)
+		}
+		return stdout, nil
+	}
+
+	assert.EventuallyWithT(t, func(t *assert.CollectT) {
+		_, err := curlFQDN(testFQDN)
+		assert.NoError(t, err)
+	}, 2*time.Second, 1*time.Millisecond, "failed to curl test FQDN: ", testFQDN)
+
+	// confirm that the FQDN resolves to the expected IP address and store it to simulate caching of this IP by the client Pod.
+	t.Logf("Resolving FQDN to simulate caching the current IP inside toolbox Pod")
+	resolvedIP, err := data.runDNSQuery(toolboxPodName, toolboxContainerName, data.testNamespace, testFQDN, false, dnsServiceIP)
+	fqdnIP := resolvedIP.String()
+	require.NoError(t, err, "failed to resolve FQDN to an IP from toolbox Pod")
+	require.Equalf(t, agnhostPodOneIP, fqdnIP, "Resolved IP does not match expected value")
+	t.Logf("Successfully received the expected IP %s against the test FQDN", fqdnIP)
+
+	// update the IP address mapped to the FQDN in the custom DNS ConfigMap.
+	t.Logf("Updating host mapping in DNS server config to use new IP: %s", agnhostPodTwoIP)
+	customDNSConfigMap.Data = createDNSConfig(t, map[string]string{agnhostPodTwoIP: testFQDN}, dnsTTL)
+	require.NoError(t, data.UpdateConfigMap(customDNSConfigMap), "failed to update configmap with new IP")
+	t.Logf("Successfully updated DNS ConfigMap with new IP: %s", agnhostPodTwoIP)
+
+	// try to trigger an immediate refresh of the configmap by setting annotations in custom DNS server Pod, this way
+	// we try to bypass the kubelet sync period which may be as long as (1 minute by default) + TTL of ConfigMaps.
+	// Ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/#mounted-configmaps-are-updated-automatically
+	require.NoError(t, data.setPodAnnotation(data.testNamespace, "custom-dns-server", "test.antrea.io/random-value",
+		randSeq(8)), "failed to update custom DNS Pod annotation.")
+
+	// finally verify that Curling the previously cached IP fails after DNS update.
+	// The wait time here should be slightly longer than the reload value specified in the custom DNS configuration.
+	// TODO: This assertion currently verifies the issue described in https://github.com/antrea-io/antrea/issues/6229.
+	// It will need to be updated once minTTL support is implemented.
+	t.Logf("Trying to curl the existing cached IP of the domain: %s", fqdnIP)
+	assert.EventuallyWithT(t, func(t *assert.CollectT) {
+		_, err := curlFQDN(fqdnIP)
+		assert.Error(t, err)
+	}, 10*time.Second, 1*time.Second)
+}
+
+// setDNSServerAddressInAntrea sets or resets the custom DNS server IP address in Antrea ConfigMap.
+func setDNSServerAddressInAntrea(t *testing.T, data *TestData, dnsServiceIP string) {
+	agentChanges := func(config *agentconfig.AgentConfig) {
+		config.DNSServerOverride = dnsServiceIP
+	}
+	err := data.mutateAntreaConfigMap(nil, agentChanges, false, true)
+	require.NoError(t, err, "Error when setting up custom DNS server IP in Antrea configmap")
+
+	t.Logf("DNSServerOverride set to %q in Antrea Agent config", dnsServiceIP)
+}
+
+// createPolicyForFQDNCacheMinTTL creates a FQDN policy in the specified Namespace.
+func createPolicyForFQDNCacheMinTTL(t *testing.T, data *TestData, testFQDN string, fqdnPolicyName, customDNSLabelValue, fqdnPodSelectorLabelValue string) {
+	podSelectorLabel := map[string]string{
+		"app": fqdnPodSelectorLabelValue,
+	}
+	builder := &AntreaNetworkPolicySpecBuilder{}
+	builder = builder.SetName(data.testNamespace, fqdnPolicyName).
+		SetTier(defaultTierName).
+		SetPriority(1.0).
+		SetAppliedToGroup([]ANNPAppliedToSpec{{PodSelector: podSelectorLabel}})
+	builder.AddFQDNRule(testFQDN, ProtocolTCP, ptr.To[int32](80), nil, nil, "AllowForFQDN", nil,
+		crdv1beta1.RuleActionAllow)
+	builder.AddEgress(ProtocolUDP, ptr.To[int32](53), nil, nil, nil, nil,
+		nil, nil, nil, nil, map[string]string{"app": customDNSLabelValue},
+		nil, nil, nil, nil,
+		nil, nil, crdv1beta1.RuleActionAllow, "", "AllowDnsQueries")
+	builder.AddEgress(ProtocolTCP, nil, nil, nil, nil, nil,
+		nil, nil, nil, nil, nil,
+		nil, nil, nil, nil,
+		nil, nil, crdv1beta1.RuleActionReject, "", "DropAllRemainingTraffic")
+
+	annp, err := data.CreateOrUpdateANNP(builder.Get())
+	require.NoError(t, err, "error while deploying Antrea policy")
+	require.NoError(t, data.waitForANNPRealized(t, annp.Namespace, annp.Name, 10*time.Second))
+}
+
+// createHttpAgnhostPod creates an agnhost Pod that serves HTTP requests and returns the IP of Pod created.
+func createHttpAgnhostPod(t *testing.T, data *TestData) *PodIPs {
+	const (
+		agnhostPort          = 80
+		agnhostPodNamePreFix = "agnhost-"
+	)
+	podName := randName(agnhostPodNamePreFix)
+	args := []string{"netexec", "--http-port=" + strconv.Itoa(agnhostPort)}
+	ports := []v1.ContainerPort{
+		{
+			Name:          "http",
+			ContainerPort: agnhostPort,
+			Protocol:      v1.ProtocolTCP,
+		},
+	}
+
+	require.NoError(t, NewPodBuilder(podName, data.testNamespace, agnhostImage).
+		WithArgs(args).
+		WithPorts(ports).
+		WithLabels(map[string]string{"app": "agnhost"}).
+		Create(data))
+	podIPs, err := data.podWaitForIPs(defaultTimeout, podName, data.testNamespace)
+	require.NoError(t, err)
+	return podIPs
+}
+
+// createDNSPod creates the CoreDNS Pod configured to use the custom DNS ConfigMap.
+func createCustomDNSPod(t *testing.T, data *TestData, configName string) {
+	volume := []v1.Volume{
+		{
+			Name: "config-volume",
+			VolumeSource: v1.VolumeSource{
+				ConfigMap: &v1.ConfigMapVolumeSource{
+					LocalObjectReference: v1.LocalObjectReference{
+						Name: configName,
+					},
+					Items: []v1.KeyToPath{
+						{
+							Key:  "Corefile",
+							Path: "Corefile",
+						},
+					},
+				},
+			},
+		},
+	}
+
+	volumeMount := []v1.VolumeMount{
+		{
+			Name:      "config-volume",
+			MountPath: "/etc/coredns",
+		},
+	}
+
+	require.NoError(t, NewPodBuilder("custom-dns-server", data.testNamespace, "coredns/coredns:1.11.3").
+		WithLabels(map[string]string{"app": "custom-dns"}).
+		WithContainerName("coredns").
+		WithArgs([]string{"-conf", "/etc/coredns/Corefile"}).
+		AddVolume(volume).AddVolumeMount(volumeMount).
+		Create(data))
+	require.NoError(t, data.podWaitForRunning(defaultTimeout, "custom-dns-server", data.testNamespace))
+}
+
+// createDNSConfig generates a DNS configuration for the specified IP address and domain name.
+func createDNSConfig(t *testing.T, hosts map[string]string, ttl int) map[string]string {
+	const coreFileTemplate = `lfx.test:53 {
+        errors
+        log
+        health
+        hosts {
+            {{ range $IP, $FQDN := .Hosts }}{{ $IP }} {{ $FQDN }}{{ end }}
+            no_reverse
+            pods verified
+            ttl {{ .TTL }}
+        }
+        loop
+        reload 2s
+    }`
+
+	data := struct {
+		Hosts map[string]string
+		TTL   int
+	}{
+		Hosts: hosts,
+		TTL:   ttl,
+	}
+
+	// Parse the template and generate the config data
+	tmpl, err := template.New("configMapData").Parse(coreFileTemplate)
+	require.NoError(t, err, "error parsing config template")
+
+	var output bytes.Buffer
+	err = tmpl.Execute(&output, data)
+	require.NoError(t, err, "error executing config template")
+
+	configMapData := strings.TrimSpace(output.String())
+	configData := map[string]string{
+		"Corefile": configMapData,
+	}
+
+	return configData
+}

test/e2e/framework.go

@@ -331,6 +331,16 @@ func (p *PodIPs) AsSlice() []*net.IP {
 	return ips
 }
 
+func (p *PodIPs) AsStrings() (ipv4, ipv6 string) {
+	if p.IPv4 != nil {
+		ipv4 = p.IPv4.String()
+	}
+	if p.IPv6 != nil {
+		ipv6 = p.IPv6.String()
+	}
+	return
+}
+
 // workerNodeName returns an empty string if there is no worker Node with the provided idx
 // (including if idx is 0, which is reserved for the control-plane Node)
 func workerNodeName(idx int) string {
@@ -1357,6 +1367,7 @@ type PodBuilder struct {
 	ResourceRequests   corev1.ResourceList
 	ResourceLimits     corev1.ResourceList
 	ReadinessProbe     *corev1.Probe
+	DnsConfig          *corev1.PodDNSConfig
 }
 
 func NewPodBuilder(name, ns, image string) *PodBuilder {
@@ -1481,6 +1492,13 @@ func (b *PodBuilder) WithReadinessProbe(probe *corev1.Probe) *PodBuilder {
 	return b
 }
 
+// WithCustomDNSConfig adds a custom DNS Configuration to the Pod spec.
+// It ensures that the DNSPolicy is set to 'None' and assigns the provided DNSConfig.
+func (b *PodBuilder) WithCustomDNSConfig(dnsConfig *corev1.PodDNSConfig) *PodBuilder {
+	b.DnsConfig = dnsConfig
+	return b
+}
+
 func (b *PodBuilder) Create(data *TestData) error {
 	containerName := b.ContainerName
 	if containerName == "" {
@@ -1523,6 +1541,13 @@ func (b *PodBuilder) Create(data *TestData) error {
 		// tolerate NoSchedule taint if we want Pod to run on control-plane Node
 		podSpec.Tolerations = controlPlaneNoScheduleTolerations()
 	}
+	if b.DnsConfig != nil {
+		// Set DNSPolicy to None to allow custom DNSConfig
+		podSpec.DNSPolicy = corev1.DNSNone
+
+		// Assign the provided DNSConfig to the Pod's DNSConfig field
+		podSpec.DNSConfig = b.DnsConfig
+	}
 	pod := &corev1.Pod{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:        b.Name,
@@ -3220,3 +3245,54 @@ func (data *TestData) GetPodLogs(ctx context.Context, namespace, name, container
 	}
 	return b.String(), nil
 }
+
+func (data *TestData) runDNSQuery(
+	podName string,
+	containerName string,
+	podNamespace string,
+	dstAddr string,
+	useTCP bool,
+	dnsServiceIP string) (net.IP, error) {
+
+	digCmdStr := fmt.Sprintf("dig "+"@"+dnsServiceIP+" +short %s", dstAddr)
+	if useTCP {
+		digCmdStr += " +tcp"
+	}
+
+	digCmd := strings.Fields(digCmdStr)
+	stdout, stderr, err := data.RunCommandFromPod(podNamespace, podName, containerName, digCmd)
+	if err != nil {
+		return nil, fmt.Errorf("error when running dig command in Pod '%s': %v - stdout: %s - stderr: %s", podName, err, stdout, stderr)
+	}
+
+	ipAddress := net.ParseIP(strings.TrimSpace(stdout))
+	if ipAddress != nil {
+		return ipAddress, nil
+	} else {
+		return nil, fmt.Errorf("invalid IP address found %v", stdout)
+	}
+}
+
+// setPodAnnotation Patches a pod by adding an annotation with a specified key and value.
+func (data *TestData) setPodAnnotation(namespace, podName, annotationKey string, annotationValue string) error {
+	annotations := map[string]string{
+		annotationKey: annotationValue,
+	}
+	annotationPatch := map[string]interface{}{
+		"metadata": map[string]interface{}{
+			"annotations": annotations,
+		},
+	}
+
+	patchData, err := json.Marshal(annotationPatch)
+	if err != nil {
+		return err
+	}
+
+	if _, err := data.clientset.CoreV1().Pods(namespace).Patch(context.TODO(), podName, types.MergePatchType, patchData, metav1.PatchOptions{}); err != nil {
+		return err
+	}
+
+	log.Infof("Successfully patched Pod %s in Namespace %s", podName, namespace)
+	return nil
+}