From 98a72eaead2533810fc3d58f28f7a7fac53dc32e Mon Sep 17 00:00:00 2001 From: Hemant Date: Fri, 27 Sep 2024 03:36:33 +0530 Subject: [PATCH 1/5] Enhanced FQDN e2e Test for DNS Cache Expiry - Implemented custom DNS configuration with tunable TTL and reload values to simulate DNS refresh scenarios. Added annotation patches for DNS pods to prompt early configmap reflection and a new `AsStrings()` method for improved handling of Pod IPs. - Refactored utility functions to simplify pod creation and DNS configuration. Utilised `PodBuilder` pattern from framework.go for toolbox setup. - Reduced unnecessary constants and global variables. - Reduced verbosity in logging and refactored functions to improve readability. Organized import statements and addressed initialism conventions (FQDN, IP) for consistency. - Replaced synchronous wait patterns with eventual assertions to enhance test performance. Signed-off-by: Hemant --- test/e2e/fqdn_dns_cache_test.go | 313 ++++++++++++++++++++++++++++ test/e2e/framework.go | 67 +++++- test/e2e/k8s_util.go | 8 + test/e2e/utils/annp_spec_builder.go | 26 +++ 4 files changed, 411 insertions(+), 3 deletions(-) create mode 100644 test/e2e/fqdn_dns_cache_test.go diff --git a/test/e2e/fqdn_dns_cache_test.go b/test/e2e/fqdn_dns_cache_test.go new file mode 100644 index 00000000000..9d46c695e6b --- /dev/null +++ b/test/e2e/fqdn_dns_cache_test.go @@ -0,0 +1,313 @@ +// Copyright 2024 Antrea Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e + +import ( + "bytes" + "fmt" + "strconv" + "strings" + "testing" + "text/template" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + + crdv1beta1 "antrea.io/antrea/pkg/apis/crd/v1beta1" + agentconfig "antrea.io/antrea/pkg/config/agent" + "antrea.io/antrea/test/e2e/utils" +) + +func TestFQDNPolicyWithCachedDNS(t *testing.T) { + const ( + testFQDN = "fqdn-test-pod.lfx.test" + dnsPort = 53 + dnsTTL = 5 + ) + + skipIfAntreaPolicyDisabled(t) + skipIfNotIPv4Cluster(t) + skipIfIPv6Cluster(t) + skipIfNotRequired(t, "mode-irrelevant") + + data, err := setupTest(t) + if err != nil { + t.Fatalf("Error when setting up test: %v", err) + } + defer teardownTest(t, data) + + // create two agnHost Pods and get their IPv4 addresses. The IP of these Pods will be mapped against the FQDN. + podCount := 2 + agnHostPodIPs := make([]*PodIPs, podCount) + for i := 0; i < podCount; i++ { + agnHostPodIPs[i] = createHttpAgnHostPod(t, data) + } + + // get IPv4 addresses of the agnHost pods created. + agnHostPodOneIP, _ := agnHostPodIPs[0].AsStrings() + agnHostPodTwoIP, _ := agnHostPodIPs[1].AsStrings() + + // create customDNS service and get its ClusterIP. + customDnsService, err := data.CreateServiceWithAnnotations("custom-dns-service", data.testNamespace, dnsPort, + dnsPort, corev1.ProtocolUDP, map[string]string{"app": "custom-dns"}, false, + false, corev1.ServiceTypeClusterIP, ptr.To[corev1.IPFamily](corev1.IPv4Protocol), map[string]string{}) + require.NoError(t, err, "Error creating custom DNS Service") + dnsServiceIP := customDnsService.Spec.ClusterIP + + // create a ConfigMap for the custom DNS server, mapping IP of agnHost Pod 1 to the FQDN. + configMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "custom-dns-config", + Namespace: data.testNamespace, + }, + Data: createDnsConfig(t, map[string]string{agnHostPodOneIP: testFQDN}, dnsTTL), + } + customDnsConfigMap, err := data.CreateConfigMap(configMap) + require.NoError(t, err, "failed to create custom DNS ConfigMap") + + createCustomDnsPod(t, data, configMap.Name) + + // set the custom DNS server IP address in Antrea configMap. + setDnsServerAddressInAntrea(t, data, dnsServiceIP) + defer setDnsServerAddressInAntrea(t, data, "") //reset after the test. + + createFqdnPolicyInNamespace(t, data, testFQDN, "test-anp-fqdn", "custom-dns", "fqdn-cache-test") + createToolboxPod(t, data, dnsServiceIP) + + curlFQDN := func(target string) (string, error) { + cmd := []string{"curl", target} + stdout, stderr, err := data.RunCommandFromPod(data.testNamespace, toolboxPodName, toolboxContainerName, cmd) + if err != nil { + return "", fmt.Errorf("error when running command '%s' on Pod '%s': %v, stdout: <%v>, stderr: <%v>", + strings.Join(cmd, " "), toolboxPodName, err, stdout, stderr) + } + return stdout, nil + } + + assert.EventuallyWithT(t, func(collect *assert.CollectT) { + stdout, err := curlFQDN(testFQDN) + assert.NoError(t, err) + t.Logf("Response of curl to FQDN: %s", stdout) + }, 2*time.Second, 100*time.Millisecond, "trying to curl the FQDN: ", testFQDN) + + // confirm that the FQDN resolves to the expected IP address and store it to simulate caching of this IP associated with the FQDN. + t.Logf("Resolving FQDN to simulate caching the current IP inside Toolbox Pod...") + resolvedIP, err := data.runDNSQuery(toolboxPodName, toolboxContainerName, data.testNamespace, testFQDN, false, dnsServiceIP) + fqdnIP := resolvedIP.String() + require.NoError(t, err, "failed to resolve FQDN to an IP from toolbox Pod") + require.Equalf(t, agnHostPodOneIP, fqdnIP, "The IP set against the FQDN in the DNS server should be the same, but got %s instead of %s", fqdnIP, agnHostPodOneIP) + t.Logf("Successfully received the expected IP %s using the dig command against the FQDN", fqdnIP) + + // update the IP address mapped to the FQDN in the custom DNS ConfigMap. + t.Logf("Updating host mapping in DNS server config to use new IP: %s", agnHostPodTwoIP) + customDnsConfigMap.Data = createDnsConfig(t, map[string]string{agnHostPodTwoIP: testFQDN}, dnsTTL) + require.NoError(t, data.UpdateConfigMap(customDnsConfigMap), "failed to update configmap with new IP") + t.Logf("Successfully updated DNS ConfigMap with new IP: %s", agnHostPodTwoIP) + + // try to trigger an immediate refresh of the configmap by setting annotations in custom DNS server Pod, this way + // we try to bypass the kubelet sync period which may be as long as (1 minute by default) + TTL of ConfigMaps. + // Ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/#mounted-configmaps-are-updated-automatically + require.NoError(t, data.setPodAnnotation(data.testNamespace, "custom-dns-server", "test.antrea.io/random-value", + randSeq(8)), "failed to update custom DNS Pod annotation.") + + // finally verify that Curling the previously cached IP fails after DNS update. + // The wait time here should be slightly longer than the reload value specified in the custom DNS configuration. + assert.EventuallyWithT(t, func(collectT *assert.CollectT) { + t.Logf("Trying to curl the existing cached IP of the domain: %s", fqdnIP) + stdout, err := curlFQDN(fqdnIP) + if err != nil { + t.Logf("Curling the cached IP failed") + } else { + t.Logf("Response of curl to cached IP: %+v", stdout) + } + assert.Error(collectT, err) + }, 10*time.Second, 1*time.Second) +} + +// setDnsServerAddressInAntrea sets or resets the custom DNS server IP address in Antrea configMap. +func setDnsServerAddressInAntrea(t *testing.T, data *TestData, dnsServiceIP string) { + agentChanges := func(config *agentconfig.AgentConfig) { + config.DNSServerOverride = dnsServiceIP + } + err := data.mutateAntreaConfigMap(nil, agentChanges, false, true) + require.NoError(t, err, "Error when setting up custom DNS server IP in Antrea configmap") + + if dnsServiceIP == "" { + t.Logf("Removing DNS server IP from antrea agent as part of teardown") + } else { + t.Logf("DNS server value set to %s in antrea \n", dnsServiceIP) + } + +} + +// createFqdnPolicyInNamespace creates an FQDN policy in the specified namespace. +func createFqdnPolicyInNamespace(t *testing.T, data *TestData, testFQDN string, fqdnPolicyName, customDnsLabelValue, fqdnPodSelectorLabelValue string) { + podSelectorLabel := map[string]string{ + "app": fqdnPodSelectorLabelValue, + } + port := int32(80) + udpPort := int32(53) + builder := &utils.AntreaNetworkPolicySpecBuilder{} + builder = builder.SetName(data.testNamespace, fqdnPolicyName). + SetTier(defaultTierName). + SetPriority(1.0). + SetAppliedToGroup([]utils.ANNPAppliedToSpec{{PodSelector: podSelectorLabel}}) + builder.AddFQDNRule(testFQDN, utils.ProtocolTCP, &port, nil, nil, "AllowForFQDN", nil, + crdv1beta1.RuleActionAllow) + builder.AddEgress(utils.ProtocolUDP, &udpPort, nil, nil, nil, nil, + nil, nil, nil, nil, map[string]string{"app": customDnsLabelValue}, + nil, nil, nil, nil, + nil, nil, crdv1beta1.RuleActionAllow, "", "AllowDnsQueries") + builder.AddEgress(utils.ProtocolTCP, nil, nil, nil, nil, nil, + nil, nil, nil, nil, nil, + nil, nil, nil, nil, + nil, nil, crdv1beta1.RuleActionReject, "", "DropAllRemainingTraffic") + + annp, err := data.CreateOrUpdateANNP(builder.Get()) + require.NoError(t, err, "error while deploying antrea policy") + require.NoError(t, data.waitForANNPRealized(t, annp.Namespace, annp.Name, 10*time.Second)) +} + +// createToolBoxPod creates the toolbox Pod with custom DNS settings for test purpose. +func createToolboxPod(t *testing.T, data *TestData, dnsServiceIP string) { + mutateSpecForAddingCustomDNS := func(pod *corev1.Pod) { + pod.Spec.DNSPolicy = corev1.DNSNone + if pod.Spec.DNSConfig == nil { + pod.Spec.DNSConfig = &corev1.PodDNSConfig{} + } + pod.Spec.DNSConfig.Nameservers = []string{dnsServiceIP} + + } + require.NoError(t, NewPodBuilder(toolboxPodName, data.testNamespace, ToolboxImage). + WithLabels(map[string]string{"app": "fqdn-cache-test"}). + WithContainerName(toolboxContainerName). + WithMutateFunc(mutateSpecForAddingCustomDNS). + Create(data)) + require.NoError(t, data.podWaitForRunning(defaultTimeout, toolboxPodName, data.testNamespace)) +} + +// createHttpAgnHostPod creates an agnHost Pod that serves HTTP requests and returns the IP of Pod created. +func createHttpAgnHostPod(t *testing.T, data *TestData) *PodIPs { + const ( + agnHostPort = 80 + agnHostPodNamePreFix = "agnhost-" + ) + podName := randName(agnHostPodNamePreFix) + args := []string{"netexec", "--http-port=" + strconv.Itoa(agnHostPort)} + ports := []corev1.ContainerPort{ + { + Name: "http", + ContainerPort: agnHostPort, + Protocol: corev1.ProtocolTCP, + }, + } + + require.NoError(t, NewPodBuilder(podName, data.testNamespace, agnhostImage). + WithArgs(args). + WithPorts(ports). + WithLabels(map[string]string{"app": "agnhost"}). + Create(data)) + podIPs, err := data.podWaitForIPs(defaultTimeout, podName, data.testNamespace) + require.NoError(t, err) + return podIPs +} + +// createDnsPod creates the CoreDNS Pod configured to use the custom DNS ConfigMap. +func createCustomDnsPod(t *testing.T, data *TestData, configName string) { + volume := []corev1.Volume{ + { + Name: "config-volume", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: configName, + }, + Items: []corev1.KeyToPath{ + { + Key: "Corefile", + Path: "Corefile", + }, + }, + }, + }, + }, + } + + volumeMount := []corev1.VolumeMount{ + { + Name: "config-volume", + MountPath: "/etc/coredns", + }, + } + + require.NoError(t, NewPodBuilder("custom-dns-server", data.testNamespace, "coredns/coredns:1.11.3"). + WithLabels(map[string]string{"app": "custom-dns"}). + WithContainerName("coredns"). + WithArgs([]string{"-conf", "/etc/coredns/Corefile"}). + AddVolume(volume).AddVolumeMount(volumeMount). + Create(data)) + require.NoError(t, data.podWaitForRunning(defaultTimeout, "custom-dns-server", data.testNamespace)) +} + +// createDnsConfig generates a DNS configuration for the specified IP address and domain name. +func createDnsConfig(t *testing.T, hosts map[string]string, ttl int) map[string]string { + const coreFileTemplate = `lfx.test:53 { + errors + log + health + hosts { + {{ range $IP, $FQDN := .Hosts }}{{ $IP }} {{ $FQDN }}{{ end }} + no_reverse + pods verified + ttl {{ .TTL }} + } + loop + reload 2s + }` + + generateConfigData := func() (string, error) { + + data := struct { + Hosts map[string]string + TTL int + }{ + Hosts: hosts, + TTL: ttl, + } + + tmpl, err := template.New("configMapData").Parse(coreFileTemplate) + if err != nil { + return "", err + } + var output bytes.Buffer + err = tmpl.Execute(&output, data) + if err != nil { + return "", err + } + return strings.TrimSpace(output.String()), nil + } + + configMapData, err := generateConfigData() + require.NoError(t, err, "error processing configData template for DNS") + configData := map[string]string{ + "Corefile": configMapData, + } + + return configData +} diff --git a/test/e2e/framework.go b/test/e2e/framework.go index a497b49b53f..99cec75fb95 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -140,9 +140,8 @@ const ( defaultCHDatabaseURL = "tcp://clickhouse-clickhouse.flow-visibility.svc:9000" statefulSetRestartAnnotationKey = "antrea-e2e/restartedAt" - - iperfPort = 5201 - iperfSvcPort = 9999 + iperfPort = 5201 + iperfSvcPort = 9999 ) type ClusterNode struct { @@ -331,6 +330,16 @@ func (p *PodIPs) AsSlice() []*net.IP { return ips } +func (p *PodIPs) AsStrings() (ipv4, ipv6 string) { + if p.IPv4 != nil { + ipv4 = p.IPv4.String() + } + if p.IPv6 != nil { + ipv6 = p.IPv6.String() + } + return +} + // workerNodeName returns an empty string if there is no worker Node with the provided idx // (including if idx is 0, which is reserved for the control-plane Node) func workerNodeName(idx int) string { @@ -3220,3 +3229,55 @@ func (data *TestData) GetPodLogs(ctx context.Context, namespace, name, container } return b.String(), nil } + +func (data *TestData) runDNSQuery( + podName string, + containerName string, + podNamespace string, + dstAddr string, + useTCP bool, + dnsServiceIP string) (net.IP, error) { + + digCmdStr := fmt.Sprintf("dig "+"@"+dnsServiceIP+" +short %s", dstAddr) + if useTCP { + digCmdStr += " +tcp" + } + + digCmd := strings.Fields(digCmdStr) + fmt.Printf("Running: kubectl exec %s -c %s -n %s -- %s", podName, containerName, podNamespace, strings.Join(digCmd, " ")) + stdout, stderr, err := data.RunCommandFromPod(podNamespace, podName, containerName, digCmd) + if err != nil { + return nil, fmt.Errorf("error when running dig command in Pod '%s': %v - stdout: %s - stderr: %s", podName, err, stdout, stderr) + } + + ipAddress := net.ParseIP(strings.TrimSpace(stdout)) + if ipAddress != nil { + return ipAddress, nil + } else { + return nil, fmt.Errorf("invalid IP address found %v", stdout) + } +} + +// setPodAnnotation Patches a pod by adding an annotation with a specified key and value. +func (data *TestData) setPodAnnotation(namespace, podName, annotationKey string, annotationValue string) error { + annotations := map[string]string{ + annotationKey: annotationValue, + } + annotationPatch := map[string]interface{}{ + "metadata": map[string]interface{}{ + "annotations": annotations, + }, + } + + patchData, err := json.Marshal(annotationPatch) + if err != nil { + return err + } + + if _, err := data.clientset.CoreV1().Pods(namespace).Patch(context.TODO(), podName, types.MergePatchType, patchData, metav1.PatchOptions{}); err != nil { + return err + } + + log.Infof("Successfully patched pod %s in namespace %s", podName, namespace) + return nil +} diff --git a/test/e2e/k8s_util.go b/test/e2e/k8s_util.go index d665cce4c82..6fe96c27801 100644 --- a/test/e2e/k8s_util.go +++ b/test/e2e/k8s_util.go @@ -667,6 +667,14 @@ func (data *TestData) UpdateConfigMap(configMap *v1.ConfigMap) error { return err } +func (data *TestData) CreateConfigMap(configMap *v1.ConfigMap) (*v1.ConfigMap, error) { + configMapObject, err := data.clientset.CoreV1().ConfigMaps(configMap.Namespace).Create(context.TODO(), configMap, metav1.CreateOptions{}) + if err != nil { + return nil, err + } + return configMapObject, nil +} + // DeleteService is a convenience function for deleting a Service by Namespace and name. func (data *TestData) DeleteService(ns, name string) error { log.Infof("Deleting Service %s in ns %s", name, ns) diff --git a/test/e2e/utils/annp_spec_builder.go b/test/e2e/utils/annp_spec_builder.go index 670aa584ca4..f3eb1a1b651 100644 --- a/test/e2e/utils/annp_spec_builder.go +++ b/test/e2e/utils/annp_spec_builder.go @@ -214,3 +214,29 @@ func (b *AntreaNetworkPolicySpecBuilder) AddEgressLogging(logLabel string) *Antr } return b } + +func (b *AntreaNetworkPolicySpecBuilder) AddFQDNRule(fqdn string, + protoc AntreaPolicyProtocol, port *int32, portName *string, endPort *int32, name string, + specs []ANNPAppliedToSpec, action crdv1beta1.RuleAction) *AntreaNetworkPolicySpecBuilder { + var appliedTos []crdv1beta1.AppliedTo + + for _, at := range specs { + appliedTos = append(appliedTos, b.GetAppliedToPeer(at.PodSelector, + at.PodSelectorMatchExp, + at.ExternalEntitySelector, + at.ExternalEntitySelectorMatchExp, + at.Group)) + } + + policyPeer := []crdv1beta1.NetworkPolicyPeer{{FQDN: fqdn}} + ports, _ := GenPortsOrProtocols(protoc, port, portName, endPort, nil, nil, nil, nil, nil, nil) + newRule := crdv1beta1.Rule{ + To: policyPeer, + Ports: ports, + Action: &action, + Name: name, + AppliedTo: appliedTos, + } + b.Spec.Egress = append(b.Spec.Egress, newRule) + return b +} From 829f3187d9598d4e2e1a30a0687e71d648b4d9f5 Mon Sep 17 00:00:00 2001 From: Hemant Date: Fri, 8 Nov 2024 21:16:31 +0530 Subject: [PATCH 2/5] Made changes as per contribution guidelines. Add a method to PodBuilder to set custom DNS IP in Pod's DNS Spec. Refactored the code to call above method within main test function. Signed-off-by: Hemant --- test/e2e/fqdn_dns_cache_test.go | 137 +++++++++++++------------------- test/e2e/framework.go | 15 +++- 2 files changed, 70 insertions(+), 82 deletions(-) diff --git a/test/e2e/fqdn_dns_cache_test.go b/test/e2e/fqdn_dns_cache_test.go index 9d46c695e6b..8eba02e85dc 100644 --- a/test/e2e/fqdn_dns_cache_test.go +++ b/test/e2e/fqdn_dns_cache_test.go @@ -52,43 +52,48 @@ func TestFQDNPolicyWithCachedDNS(t *testing.T) { } defer teardownTest(t, data) - // create two agnHost Pods and get their IPv4 addresses. The IP of these Pods will be mapped against the FQDN. + // create two agnhost Pods and get their IPv4 addresses. The IP of these Pods will be mapped against the FQDN. podCount := 2 - agnHostPodIPs := make([]*PodIPs, podCount) + agnhostPodIPs := make([]*PodIPs, podCount) for i := 0; i < podCount; i++ { - agnHostPodIPs[i] = createHttpAgnHostPod(t, data) + agnhostPodIPs[i] = createHttpAgnhostPod(t, data) } - // get IPv4 addresses of the agnHost pods created. - agnHostPodOneIP, _ := agnHostPodIPs[0].AsStrings() - agnHostPodTwoIP, _ := agnHostPodIPs[1].AsStrings() + // get IPv4 addresses of the agnhost Pods created. + agnhostPodOneIP, _ := agnhostPodIPs[0].AsStrings() + agnhostPodTwoIP, _ := agnhostPodIPs[1].AsStrings() - // create customDNS service and get its ClusterIP. - customDnsService, err := data.CreateServiceWithAnnotations("custom-dns-service", data.testNamespace, dnsPort, + // create customDNS Service and get its ClusterIP. + customDNSService, err := data.CreateServiceWithAnnotations("custom-dns-service", data.testNamespace, dnsPort, dnsPort, corev1.ProtocolUDP, map[string]string{"app": "custom-dns"}, false, false, corev1.ServiceTypeClusterIP, ptr.To[corev1.IPFamily](corev1.IPv4Protocol), map[string]string{}) require.NoError(t, err, "Error creating custom DNS Service") - dnsServiceIP := customDnsService.Spec.ClusterIP + dnsServiceIP := customDNSService.Spec.ClusterIP - // create a ConfigMap for the custom DNS server, mapping IP of agnHost Pod 1 to the FQDN. + // create a ConfigMap for the custom DNS server, mapping IP of agnhost Pod 1 to the FQDN. configMap := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: "custom-dns-config", Namespace: data.testNamespace, }, - Data: createDnsConfig(t, map[string]string{agnHostPodOneIP: testFQDN}, dnsTTL), + Data: createDNSConfig(t, map[string]string{agnhostPodOneIP: testFQDN}, dnsTTL), } - customDnsConfigMap, err := data.CreateConfigMap(configMap) + customDNSConfigMap, err := data.CreateConfigMap(configMap) require.NoError(t, err, "failed to create custom DNS ConfigMap") - createCustomDnsPod(t, data, configMap.Name) + createCustomDNSPod(t, data, configMap.Name) - // set the custom DNS server IP address in Antrea configMap. - setDnsServerAddressInAntrea(t, data, dnsServiceIP) - defer setDnsServerAddressInAntrea(t, data, "") //reset after the test. + // set the custom DNS server IP address in Antrea ConfigMap. + setDNSServerAddressInAntrea(t, data, dnsServiceIP) + defer setDNSServerAddressInAntrea(t, data, "") //reset after the test. - createFqdnPolicyInNamespace(t, data, testFQDN, "test-anp-fqdn", "custom-dns", "fqdn-cache-test") - createToolboxPod(t, data, dnsServiceIP) + createFQDNPolicyInNamespace(t, data, testFQDN, "test-anp-fqdn", "custom-dns", "fqdn-cache-test") + require.NoError(t, NewPodBuilder(toolboxPodName, data.testNamespace, ToolboxImage). + WithLabels(map[string]string{"app": "fqdn-cache-test"}). + WithContainerName(toolboxContainerName). + WithCustomDNSConfig(dnsServiceIP). + Create(data)) + require.NoError(t, data.podWaitForRunning(defaultTimeout, toolboxPodName, data.testNamespace)) curlFQDN := func(target string) (string, error) { cmd := []string{"curl", target} @@ -100,25 +105,24 @@ func TestFQDNPolicyWithCachedDNS(t *testing.T) { return stdout, nil } - assert.EventuallyWithT(t, func(collect *assert.CollectT) { - stdout, err := curlFQDN(testFQDN) + assert.EventuallyWithT(t, func(t *assert.CollectT) { + _, err := curlFQDN(testFQDN) assert.NoError(t, err) - t.Logf("Response of curl to FQDN: %s", stdout) - }, 2*time.Second, 100*time.Millisecond, "trying to curl the FQDN: ", testFQDN) + }, 2*time.Second, 1*time.Millisecond, "failed to curl test FQDN: ", testFQDN) - // confirm that the FQDN resolves to the expected IP address and store it to simulate caching of this IP associated with the FQDN. - t.Logf("Resolving FQDN to simulate caching the current IP inside Toolbox Pod...") + // confirm that the FQDN resolves to the expected IP address and store it to simulate caching of this IP by the client Pod. + t.Logf("Resolving FQDN to simulate caching the current IP inside toolbox Pod") resolvedIP, err := data.runDNSQuery(toolboxPodName, toolboxContainerName, data.testNamespace, testFQDN, false, dnsServiceIP) fqdnIP := resolvedIP.String() require.NoError(t, err, "failed to resolve FQDN to an IP from toolbox Pod") - require.Equalf(t, agnHostPodOneIP, fqdnIP, "The IP set against the FQDN in the DNS server should be the same, but got %s instead of %s", fqdnIP, agnHostPodOneIP) - t.Logf("Successfully received the expected IP %s using the dig command against the FQDN", fqdnIP) + require.Equalf(t, agnhostPodOneIP, fqdnIP, "Resolved IP does not match expected value") + t.Logf("Successfully received the expected IP %s against the test FQDN", fqdnIP) // update the IP address mapped to the FQDN in the custom DNS ConfigMap. - t.Logf("Updating host mapping in DNS server config to use new IP: %s", agnHostPodTwoIP) - customDnsConfigMap.Data = createDnsConfig(t, map[string]string{agnHostPodTwoIP: testFQDN}, dnsTTL) - require.NoError(t, data.UpdateConfigMap(customDnsConfigMap), "failed to update configmap with new IP") - t.Logf("Successfully updated DNS ConfigMap with new IP: %s", agnHostPodTwoIP) + t.Logf("Updating host mapping in DNS server config to use new IP: %s", agnhostPodTwoIP) + customDNSConfigMap.Data = createDNSConfig(t, map[string]string{agnhostPodTwoIP: testFQDN}, dnsTTL) + require.NoError(t, data.UpdateConfigMap(customDNSConfigMap), "failed to update configmap with new IP") + t.Logf("Successfully updated DNS ConfigMap with new IP: %s", agnhostPodTwoIP) // try to trigger an immediate refresh of the configmap by setting annotations in custom DNS server Pod, this way // we try to bypass the kubelet sync period which may be as long as (1 minute by default) + TTL of ConfigMaps. @@ -128,36 +132,27 @@ func TestFQDNPolicyWithCachedDNS(t *testing.T) { // finally verify that Curling the previously cached IP fails after DNS update. // The wait time here should be slightly longer than the reload value specified in the custom DNS configuration. - assert.EventuallyWithT(t, func(collectT *assert.CollectT) { - t.Logf("Trying to curl the existing cached IP of the domain: %s", fqdnIP) - stdout, err := curlFQDN(fqdnIP) - if err != nil { - t.Logf("Curling the cached IP failed") - } else { - t.Logf("Response of curl to cached IP: %+v", stdout) - } - assert.Error(collectT, err) + // TODO: This assertion currently verifies the issue described in https://github.com/antrea-io/antrea/issues/6229. It will need to be updated once minTTL support is implemented. + t.Logf("Trying to curl the existing cached IP of the domain: %s", fqdnIP) + assert.EventuallyWithT(t, func(t *assert.CollectT) { + _, err := curlFQDN(fqdnIP) + assert.Error(t, err) }, 10*time.Second, 1*time.Second) } -// setDnsServerAddressInAntrea sets or resets the custom DNS server IP address in Antrea configMap. -func setDnsServerAddressInAntrea(t *testing.T, data *TestData, dnsServiceIP string) { +// setDNSServerAddressInAntrea sets or resets the custom DNS server IP address in Antrea ConfigMap. +func setDNSServerAddressInAntrea(t *testing.T, data *TestData, dnsServiceIP string) { agentChanges := func(config *agentconfig.AgentConfig) { config.DNSServerOverride = dnsServiceIP } err := data.mutateAntreaConfigMap(nil, agentChanges, false, true) require.NoError(t, err, "Error when setting up custom DNS server IP in Antrea configmap") - if dnsServiceIP == "" { - t.Logf("Removing DNS server IP from antrea agent as part of teardown") - } else { - t.Logf("DNS server value set to %s in antrea \n", dnsServiceIP) - } - + t.Logf("DNSServerOverride set to %q in Antrea Agent config", dnsServiceIP) } -// createFqdnPolicyInNamespace creates an FQDN policy in the specified namespace. -func createFqdnPolicyInNamespace(t *testing.T, data *TestData, testFQDN string, fqdnPolicyName, customDnsLabelValue, fqdnPodSelectorLabelValue string) { +// createFQDNPolicyInNamespace creates a FQDN policy in the specified Namespace. +func createFQDNPolicyInNamespace(t *testing.T, data *TestData, testFQDN string, fqdnPolicyName, customDNSLabelValue, fqdnPodSelectorLabelValue string) { podSelectorLabel := map[string]string{ "app": fqdnPodSelectorLabelValue, } @@ -171,7 +166,7 @@ func createFqdnPolicyInNamespace(t *testing.T, data *TestData, testFQDN string, builder.AddFQDNRule(testFQDN, utils.ProtocolTCP, &port, nil, nil, "AllowForFQDN", nil, crdv1beta1.RuleActionAllow) builder.AddEgress(utils.ProtocolUDP, &udpPort, nil, nil, nil, nil, - nil, nil, nil, nil, map[string]string{"app": customDnsLabelValue}, + nil, nil, nil, nil, map[string]string{"app": customDNSLabelValue}, nil, nil, nil, nil, nil, nil, crdv1beta1.RuleActionAllow, "", "AllowDnsQueries") builder.AddEgress(utils.ProtocolTCP, nil, nil, nil, nil, nil, @@ -180,40 +175,22 @@ func createFqdnPolicyInNamespace(t *testing.T, data *TestData, testFQDN string, nil, nil, crdv1beta1.RuleActionReject, "", "DropAllRemainingTraffic") annp, err := data.CreateOrUpdateANNP(builder.Get()) - require.NoError(t, err, "error while deploying antrea policy") + require.NoError(t, err, "error while deploying Antrea policy") require.NoError(t, data.waitForANNPRealized(t, annp.Namespace, annp.Name, 10*time.Second)) } -// createToolBoxPod creates the toolbox Pod with custom DNS settings for test purpose. -func createToolboxPod(t *testing.T, data *TestData, dnsServiceIP string) { - mutateSpecForAddingCustomDNS := func(pod *corev1.Pod) { - pod.Spec.DNSPolicy = corev1.DNSNone - if pod.Spec.DNSConfig == nil { - pod.Spec.DNSConfig = &corev1.PodDNSConfig{} - } - pod.Spec.DNSConfig.Nameservers = []string{dnsServiceIP} - - } - require.NoError(t, NewPodBuilder(toolboxPodName, data.testNamespace, ToolboxImage). - WithLabels(map[string]string{"app": "fqdn-cache-test"}). - WithContainerName(toolboxContainerName). - WithMutateFunc(mutateSpecForAddingCustomDNS). - Create(data)) - require.NoError(t, data.podWaitForRunning(defaultTimeout, toolboxPodName, data.testNamespace)) -} - -// createHttpAgnHostPod creates an agnHost Pod that serves HTTP requests and returns the IP of Pod created. -func createHttpAgnHostPod(t *testing.T, data *TestData) *PodIPs { +// createHttpAgnhostPod creates an agnhost Pod that serves HTTP requests and returns the IP of Pod created. +func createHttpAgnhostPod(t *testing.T, data *TestData) *PodIPs { const ( - agnHostPort = 80 - agnHostPodNamePreFix = "agnhost-" + agnhostPort = 80 + agnhostPodNamePreFix = "agnhost-" ) - podName := randName(agnHostPodNamePreFix) - args := []string{"netexec", "--http-port=" + strconv.Itoa(agnHostPort)} + podName := randName(agnhostPodNamePreFix) + args := []string{"netexec", "--http-port=" + strconv.Itoa(agnhostPort)} ports := []corev1.ContainerPort{ { Name: "http", - ContainerPort: agnHostPort, + ContainerPort: agnhostPort, Protocol: corev1.ProtocolTCP, }, } @@ -228,8 +205,8 @@ func createHttpAgnHostPod(t *testing.T, data *TestData) *PodIPs { return podIPs } -// createDnsPod creates the CoreDNS Pod configured to use the custom DNS ConfigMap. -func createCustomDnsPod(t *testing.T, data *TestData, configName string) { +// createDNSPod creates the CoreDNS Pod configured to use the custom DNS ConfigMap. +func createCustomDNSPod(t *testing.T, data *TestData, configName string) { volume := []corev1.Volume{ { Name: "config-volume", @@ -265,8 +242,8 @@ func createCustomDnsPod(t *testing.T, data *TestData, configName string) { require.NoError(t, data.podWaitForRunning(defaultTimeout, "custom-dns-server", data.testNamespace)) } -// createDnsConfig generates a DNS configuration for the specified IP address and domain name. -func createDnsConfig(t *testing.T, hosts map[string]string, ttl int) map[string]string { +// createDNSConfig generates a DNS configuration for the specified IP address and domain name. +func createDNSConfig(t *testing.T, hosts map[string]string, ttl int) map[string]string { const coreFileTemplate = `lfx.test:53 { errors log diff --git a/test/e2e/framework.go b/test/e2e/framework.go index 99cec75fb95..889e5fde2a3 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -1490,6 +1490,18 @@ func (b *PodBuilder) WithReadinessProbe(probe *corev1.Probe) *PodBuilder { return b } +// WithCustomDNSConfig adds custom DNS IP to the Pod spec. +func (b *PodBuilder) WithCustomDNSConfig(dnsServiceIP string) *PodBuilder { + b.MutateFunc = func(pod *corev1.Pod) { + pod.Spec.DNSPolicy = corev1.DNSNone + if pod.Spec.DNSConfig == nil { + pod.Spec.DNSConfig = &corev1.PodDNSConfig{} + } + pod.Spec.DNSConfig.Nameservers = []string{dnsServiceIP} + } + return b +} + func (b *PodBuilder) Create(data *TestData) error { containerName := b.ContainerName if containerName == "" { @@ -3244,7 +3256,6 @@ func (data *TestData) runDNSQuery( } digCmd := strings.Fields(digCmdStr) - fmt.Printf("Running: kubectl exec %s -c %s -n %s -- %s", podName, containerName, podNamespace, strings.Join(digCmd, " ")) stdout, stderr, err := data.RunCommandFromPod(podNamespace, podName, containerName, digCmd) if err != nil { return nil, fmt.Errorf("error when running dig command in Pod '%s': %v - stdout: %s - stderr: %s", podName, err, stdout, stderr) @@ -3278,6 +3289,6 @@ func (data *TestData) setPodAnnotation(namespace, podName, annotationKey string, return err } - log.Infof("Successfully patched pod %s in namespace %s", podName, namespace) + log.Infof("Successfully patched Pod %s in Namespace %s", podName, namespace) return nil } From def56c60f04b890ecd300d24d01b43dfe25268e6 Mon Sep 17 00:00:00 2001 From: Hemant Date: Mon, 11 Nov 2024 23:37:47 +0530 Subject: [PATCH 3/5] Moved TestFQDNCacheMinTTL to antreapolicy_test.go and modified WithCustomDNSConfig. - moved the FQDNCacheMinTTL test to antreapolicy_test file where other related tests are present and deleted the standalone file that had earlier existsed. - modified WithCustomDNSConfig function to accept DNS config as a parameter of type *corev1.PodDNSConfig. Signed-off-by: Hemant --- test/e2e/antreapolicy_test.go | 253 ++++++++++++++++++++++++++++ test/e2e/fqdn_dns_cache_test.go | 290 -------------------------------- test/e2e/framework.go | 13 +- 3 files changed, 260 insertions(+), 296 deletions(-) delete mode 100644 test/e2e/fqdn_dns_cache_test.go diff --git a/test/e2e/antreapolicy_test.go b/test/e2e/antreapolicy_test.go index 6d49539a4e1..f9152e572fd 100644 --- a/test/e2e/antreapolicy_test.go +++ b/test/e2e/antreapolicy_test.go @@ -15,6 +15,7 @@ package e2e import ( + "bytes" "context" "encoding/json" "fmt" @@ -24,6 +25,7 @@ import ( "strings" "sync" "testing" + "text/template" "time" log "github.com/sirupsen/logrus" @@ -35,9 +37,11 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/utils/ptr" "antrea.io/antrea/pkg/agent/apis" crdv1beta1 "antrea.io/antrea/pkg/apis/crd/v1beta1" + agentconfig "antrea.io/antrea/pkg/config/agent" "antrea.io/antrea/pkg/controller/networkpolicy" "antrea.io/antrea/pkg/features" . "antrea.io/antrea/test/e2e/utils" @@ -5202,3 +5206,252 @@ func testAntreaClusterNetworkPolicyStats(t *testing.T, data *TestData) { } k8sUtils.Cleanup(namespaces) } + +// TestFQDNCacheMinTTL tests stable FQDN access for applications with cached DNS resolutions +// when FQDN NetworkPolicy are in use and the FQDN-to-IP resolution changes frequently. +func TestFQDNCacheMinTTL(t *testing.T) { + const ( + testFQDN = "fqdn-test-pod.lfx.test" + dnsPort = 53 + dnsTTL = 5 + ) + + skipIfAntreaPolicyDisabled(t) + skipIfNotIPv4Cluster(t) + skipIfIPv6Cluster(t) + skipIfNotRequired(t, "mode-irrelevant") + + data, err := setupTest(t) + if err != nil { + t.Fatalf("Error when setting up test: %v", err) + } + defer teardownTest(t, data) + + // create two agnhost Pods and get their IPv4 addresses. The IP of these Pods will be mapped against the FQDN. + podCount := 2 + agnhostPodIPs := make([]*PodIPs, podCount) + for i := 0; i < podCount; i++ { + agnhostPodIPs[i] = createHttpAgnhostPod(t, data) + } + + // get IPv4 addresses of the agnhost Pods created. + agnhostPodOneIP, _ := agnhostPodIPs[0].AsStrings() + agnhostPodTwoIP, _ := agnhostPodIPs[1].AsStrings() + + // create customDNS Service and get its ClusterIP. + customDNSService, err := data.CreateServiceWithAnnotations("custom-dns-service", data.testNamespace, dnsPort, + dnsPort, v1.ProtocolUDP, map[string]string{"app": "custom-dns"}, false, + false, v1.ServiceTypeClusterIP, ptr.To[v1.IPFamily](v1.IPv4Protocol), map[string]string{}) + require.NoError(t, err, "Error creating custom DNS Service") + dnsServiceIP := customDNSService.Spec.ClusterIP + + // create a ConfigMap for the custom DNS server, mapping IP of agnhost Pod 1 to the FQDN. + configMap := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "custom-dns-config", + Namespace: data.testNamespace, + }, + Data: createDNSConfig(t, map[string]string{agnhostPodOneIP: testFQDN}, dnsTTL), + } + customDNSConfigMap, err := data.CreateConfigMap(configMap) + require.NoError(t, err, "failed to create custom DNS ConfigMap") + + createCustomDNSPod(t, data, configMap.Name) + + // set the custom DNS server IP address in Antrea ConfigMap. + setDNSServerAddressInAntrea(t, data, dnsServiceIP) + defer setDNSServerAddressInAntrea(t, data, "") //reset after the test. + + createPolicyForFQDNCacheMinTTL(t, data, testFQDN, "test-anp-fqdn", "custom-dns", "fqdn-cache-test") + require.NoError(t, NewPodBuilder(toolboxPodName, data.testNamespace, ToolboxImage). + WithLabels(map[string]string{"app": "fqdn-cache-test"}). + WithContainerName(toolboxContainerName). + WithCustomDNSConfig(&v1.PodDNSConfig{Nameservers: []string{dnsServiceIP}}). + Create(data)) + require.NoError(t, data.podWaitForRunning(defaultTimeout, toolboxPodName, data.testNamespace)) + + curlFQDN := func(target string) (string, error) { + cmd := []string{"curl", target} + stdout, stderr, err := data.RunCommandFromPod(data.testNamespace, toolboxPodName, toolboxContainerName, cmd) + if err != nil { + return "", fmt.Errorf("error when running command '%s' on Pod '%s': %v, stdout: <%v>, stderr: <%v>", + strings.Join(cmd, " "), toolboxPodName, err, stdout, stderr) + } + return stdout, nil + } + + assert.EventuallyWithT(t, func(t *assert.CollectT) { + _, err := curlFQDN(testFQDN) + assert.NoError(t, err) + }, 2*time.Second, 1*time.Millisecond, "failed to curl test FQDN: ", testFQDN) + + // confirm that the FQDN resolves to the expected IP address and store it to simulate caching of this IP by the client Pod. + t.Logf("Resolving FQDN to simulate caching the current IP inside toolbox Pod") + resolvedIP, err := data.runDNSQuery(toolboxPodName, toolboxContainerName, data.testNamespace, testFQDN, false, dnsServiceIP) + fqdnIP := resolvedIP.String() + require.NoError(t, err, "failed to resolve FQDN to an IP from toolbox Pod") + require.Equalf(t, agnhostPodOneIP, fqdnIP, "Resolved IP does not match expected value") + t.Logf("Successfully received the expected IP %s against the test FQDN", fqdnIP) + + // update the IP address mapped to the FQDN in the custom DNS ConfigMap. + t.Logf("Updating host mapping in DNS server config to use new IP: %s", agnhostPodTwoIP) + customDNSConfigMap.Data = createDNSConfig(t, map[string]string{agnhostPodTwoIP: testFQDN}, dnsTTL) + require.NoError(t, data.UpdateConfigMap(customDNSConfigMap), "failed to update configmap with new IP") + t.Logf("Successfully updated DNS ConfigMap with new IP: %s", agnhostPodTwoIP) + + // try to trigger an immediate refresh of the configmap by setting annotations in custom DNS server Pod, this way + // we try to bypass the kubelet sync period which may be as long as (1 minute by default) + TTL of ConfigMaps. + // Ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/#mounted-configmaps-are-updated-automatically + require.NoError(t, data.setPodAnnotation(data.testNamespace, "custom-dns-server", "test.antrea.io/random-value", + randSeq(8)), "failed to update custom DNS Pod annotation.") + + // finally verify that Curling the previously cached IP fails after DNS update. + // The wait time here should be slightly longer than the reload value specified in the custom DNS configuration. + // TODO: This assertion currently verifies the issue described in https://github.com/antrea-io/antrea/issues/6229. + // It will need to be updated once minTTL support is implemented. + t.Logf("Trying to curl the existing cached IP of the domain: %s", fqdnIP) + assert.EventuallyWithT(t, func(t *assert.CollectT) { + _, err := curlFQDN(fqdnIP) + assert.Error(t, err) + }, 10*time.Second, 1*time.Second) +} + +// setDNSServerAddressInAntrea sets or resets the custom DNS server IP address in Antrea ConfigMap. +func setDNSServerAddressInAntrea(t *testing.T, data *TestData, dnsServiceIP string) { + agentChanges := func(config *agentconfig.AgentConfig) { + config.DNSServerOverride = dnsServiceIP + } + err := data.mutateAntreaConfigMap(nil, agentChanges, false, true) + require.NoError(t, err, "Error when setting up custom DNS server IP in Antrea configmap") + + t.Logf("DNSServerOverride set to %q in Antrea Agent config", dnsServiceIP) +} + +// createPolicyForFQDNCacheMinTTL creates a FQDN policy in the specified Namespace. +func createPolicyForFQDNCacheMinTTL(t *testing.T, data *TestData, testFQDN string, fqdnPolicyName, customDNSLabelValue, fqdnPodSelectorLabelValue string) { + podSelectorLabel := map[string]string{ + "app": fqdnPodSelectorLabelValue, + } + builder := &AntreaNetworkPolicySpecBuilder{} + builder = builder.SetName(data.testNamespace, fqdnPolicyName). + SetTier(defaultTierName). + SetPriority(1.0). + SetAppliedToGroup([]ANNPAppliedToSpec{{PodSelector: podSelectorLabel}}) + builder.AddFQDNRule(testFQDN, ProtocolTCP, ptr.To[int32](80), nil, nil, "AllowForFQDN", nil, + crdv1beta1.RuleActionAllow) + builder.AddEgress(ProtocolUDP, ptr.To[int32](53), nil, nil, nil, nil, + nil, nil, nil, nil, map[string]string{"app": customDNSLabelValue}, + nil, nil, nil, nil, + nil, nil, crdv1beta1.RuleActionAllow, "", "AllowDnsQueries") + builder.AddEgress(ProtocolTCP, nil, nil, nil, nil, nil, + nil, nil, nil, nil, nil, + nil, nil, nil, nil, + nil, nil, crdv1beta1.RuleActionReject, "", "DropAllRemainingTraffic") + + annp, err := data.CreateOrUpdateANNP(builder.Get()) + require.NoError(t, err, "error while deploying Antrea policy") + require.NoError(t, data.waitForANNPRealized(t, annp.Namespace, annp.Name, 10*time.Second)) +} + +// createHttpAgnhostPod creates an agnhost Pod that serves HTTP requests and returns the IP of Pod created. +func createHttpAgnhostPod(t *testing.T, data *TestData) *PodIPs { + const ( + agnhostPort = 80 + agnhostPodNamePreFix = "agnhost-" + ) + podName := randName(agnhostPodNamePreFix) + args := []string{"netexec", "--http-port=" + strconv.Itoa(agnhostPort)} + ports := []v1.ContainerPort{ + { + Name: "http", + ContainerPort: agnhostPort, + Protocol: v1.ProtocolTCP, + }, + } + + require.NoError(t, NewPodBuilder(podName, data.testNamespace, agnhostImage). + WithArgs(args). + WithPorts(ports). + WithLabels(map[string]string{"app": "agnhost"}). + Create(data)) + podIPs, err := data.podWaitForIPs(defaultTimeout, podName, data.testNamespace) + require.NoError(t, err) + return podIPs +} + +// createDNSPod creates the CoreDNS Pod configured to use the custom DNS ConfigMap. +func createCustomDNSPod(t *testing.T, data *TestData, configName string) { + volume := []v1.Volume{ + { + Name: "config-volume", + VolumeSource: v1.VolumeSource{ + ConfigMap: &v1.ConfigMapVolumeSource{ + LocalObjectReference: v1.LocalObjectReference{ + Name: configName, + }, + Items: []v1.KeyToPath{ + { + Key: "Corefile", + Path: "Corefile", + }, + }, + }, + }, + }, + } + + volumeMount := []v1.VolumeMount{ + { + Name: "config-volume", + MountPath: "/etc/coredns", + }, + } + + require.NoError(t, NewPodBuilder("custom-dns-server", data.testNamespace, "coredns/coredns:1.11.3"). + WithLabels(map[string]string{"app": "custom-dns"}). + WithContainerName("coredns"). + WithArgs([]string{"-conf", "/etc/coredns/Corefile"}). + AddVolume(volume).AddVolumeMount(volumeMount). + Create(data)) + require.NoError(t, data.podWaitForRunning(defaultTimeout, "custom-dns-server", data.testNamespace)) +} + +// createDNSConfig generates a DNS configuration for the specified IP address and domain name. +func createDNSConfig(t *testing.T, hosts map[string]string, ttl int) map[string]string { + const coreFileTemplate = `lfx.test:53 { + errors + log + health + hosts { + {{ range $IP, $FQDN := .Hosts }}{{ $IP }} {{ $FQDN }}{{ end }} + no_reverse + pods verified + ttl {{ .TTL }} + } + loop + reload 2s + }` + + data := struct { + Hosts map[string]string + TTL int + }{ + Hosts: hosts, + TTL: ttl, + } + + // Parse the template and generate the config data + tmpl, err := template.New("configMapData").Parse(coreFileTemplate) + require.NoError(t, err, "error parsing config template") + + var output bytes.Buffer + err = tmpl.Execute(&output, data) + require.NoError(t, err, "error executing config template") + + configMapData := strings.TrimSpace(output.String()) + configData := map[string]string{ + "Corefile": configMapData, + } + + return configData +} diff --git a/test/e2e/fqdn_dns_cache_test.go b/test/e2e/fqdn_dns_cache_test.go deleted file mode 100644 index 8eba02e85dc..00000000000 --- a/test/e2e/fqdn_dns_cache_test.go +++ /dev/null @@ -1,290 +0,0 @@ -// Copyright 2024 Antrea Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package e2e - -import ( - "bytes" - "fmt" - "strconv" - "strings" - "testing" - "text/template" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/utils/ptr" - - crdv1beta1 "antrea.io/antrea/pkg/apis/crd/v1beta1" - agentconfig "antrea.io/antrea/pkg/config/agent" - "antrea.io/antrea/test/e2e/utils" -) - -func TestFQDNPolicyWithCachedDNS(t *testing.T) { - const ( - testFQDN = "fqdn-test-pod.lfx.test" - dnsPort = 53 - dnsTTL = 5 - ) - - skipIfAntreaPolicyDisabled(t) - skipIfNotIPv4Cluster(t) - skipIfIPv6Cluster(t) - skipIfNotRequired(t, "mode-irrelevant") - - data, err := setupTest(t) - if err != nil { - t.Fatalf("Error when setting up test: %v", err) - } - defer teardownTest(t, data) - - // create two agnhost Pods and get their IPv4 addresses. The IP of these Pods will be mapped against the FQDN. - podCount := 2 - agnhostPodIPs := make([]*PodIPs, podCount) - for i := 0; i < podCount; i++ { - agnhostPodIPs[i] = createHttpAgnhostPod(t, data) - } - - // get IPv4 addresses of the agnhost Pods created. - agnhostPodOneIP, _ := agnhostPodIPs[0].AsStrings() - agnhostPodTwoIP, _ := agnhostPodIPs[1].AsStrings() - - // create customDNS Service and get its ClusterIP. - customDNSService, err := data.CreateServiceWithAnnotations("custom-dns-service", data.testNamespace, dnsPort, - dnsPort, corev1.ProtocolUDP, map[string]string{"app": "custom-dns"}, false, - false, corev1.ServiceTypeClusterIP, ptr.To[corev1.IPFamily](corev1.IPv4Protocol), map[string]string{}) - require.NoError(t, err, "Error creating custom DNS Service") - dnsServiceIP := customDNSService.Spec.ClusterIP - - // create a ConfigMap for the custom DNS server, mapping IP of agnhost Pod 1 to the FQDN. - configMap := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: "custom-dns-config", - Namespace: data.testNamespace, - }, - Data: createDNSConfig(t, map[string]string{agnhostPodOneIP: testFQDN}, dnsTTL), - } - customDNSConfigMap, err := data.CreateConfigMap(configMap) - require.NoError(t, err, "failed to create custom DNS ConfigMap") - - createCustomDNSPod(t, data, configMap.Name) - - // set the custom DNS server IP address in Antrea ConfigMap. - setDNSServerAddressInAntrea(t, data, dnsServiceIP) - defer setDNSServerAddressInAntrea(t, data, "") //reset after the test. - - createFQDNPolicyInNamespace(t, data, testFQDN, "test-anp-fqdn", "custom-dns", "fqdn-cache-test") - require.NoError(t, NewPodBuilder(toolboxPodName, data.testNamespace, ToolboxImage). - WithLabels(map[string]string{"app": "fqdn-cache-test"}). - WithContainerName(toolboxContainerName). - WithCustomDNSConfig(dnsServiceIP). - Create(data)) - require.NoError(t, data.podWaitForRunning(defaultTimeout, toolboxPodName, data.testNamespace)) - - curlFQDN := func(target string) (string, error) { - cmd := []string{"curl", target} - stdout, stderr, err := data.RunCommandFromPod(data.testNamespace, toolboxPodName, toolboxContainerName, cmd) - if err != nil { - return "", fmt.Errorf("error when running command '%s' on Pod '%s': %v, stdout: <%v>, stderr: <%v>", - strings.Join(cmd, " "), toolboxPodName, err, stdout, stderr) - } - return stdout, nil - } - - assert.EventuallyWithT(t, func(t *assert.CollectT) { - _, err := curlFQDN(testFQDN) - assert.NoError(t, err) - }, 2*time.Second, 1*time.Millisecond, "failed to curl test FQDN: ", testFQDN) - - // confirm that the FQDN resolves to the expected IP address and store it to simulate caching of this IP by the client Pod. - t.Logf("Resolving FQDN to simulate caching the current IP inside toolbox Pod") - resolvedIP, err := data.runDNSQuery(toolboxPodName, toolboxContainerName, data.testNamespace, testFQDN, false, dnsServiceIP) - fqdnIP := resolvedIP.String() - require.NoError(t, err, "failed to resolve FQDN to an IP from toolbox Pod") - require.Equalf(t, agnhostPodOneIP, fqdnIP, "Resolved IP does not match expected value") - t.Logf("Successfully received the expected IP %s against the test FQDN", fqdnIP) - - // update the IP address mapped to the FQDN in the custom DNS ConfigMap. - t.Logf("Updating host mapping in DNS server config to use new IP: %s", agnhostPodTwoIP) - customDNSConfigMap.Data = createDNSConfig(t, map[string]string{agnhostPodTwoIP: testFQDN}, dnsTTL) - require.NoError(t, data.UpdateConfigMap(customDNSConfigMap), "failed to update configmap with new IP") - t.Logf("Successfully updated DNS ConfigMap with new IP: %s", agnhostPodTwoIP) - - // try to trigger an immediate refresh of the configmap by setting annotations in custom DNS server Pod, this way - // we try to bypass the kubelet sync period which may be as long as (1 minute by default) + TTL of ConfigMaps. - // Ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/#mounted-configmaps-are-updated-automatically - require.NoError(t, data.setPodAnnotation(data.testNamespace, "custom-dns-server", "test.antrea.io/random-value", - randSeq(8)), "failed to update custom DNS Pod annotation.") - - // finally verify that Curling the previously cached IP fails after DNS update. - // The wait time here should be slightly longer than the reload value specified in the custom DNS configuration. - // TODO: This assertion currently verifies the issue described in https://github.com/antrea-io/antrea/issues/6229. It will need to be updated once minTTL support is implemented. - t.Logf("Trying to curl the existing cached IP of the domain: %s", fqdnIP) - assert.EventuallyWithT(t, func(t *assert.CollectT) { - _, err := curlFQDN(fqdnIP) - assert.Error(t, err) - }, 10*time.Second, 1*time.Second) -} - -// setDNSServerAddressInAntrea sets or resets the custom DNS server IP address in Antrea ConfigMap. -func setDNSServerAddressInAntrea(t *testing.T, data *TestData, dnsServiceIP string) { - agentChanges := func(config *agentconfig.AgentConfig) { - config.DNSServerOverride = dnsServiceIP - } - err := data.mutateAntreaConfigMap(nil, agentChanges, false, true) - require.NoError(t, err, "Error when setting up custom DNS server IP in Antrea configmap") - - t.Logf("DNSServerOverride set to %q in Antrea Agent config", dnsServiceIP) -} - -// createFQDNPolicyInNamespace creates a FQDN policy in the specified Namespace. -func createFQDNPolicyInNamespace(t *testing.T, data *TestData, testFQDN string, fqdnPolicyName, customDNSLabelValue, fqdnPodSelectorLabelValue string) { - podSelectorLabel := map[string]string{ - "app": fqdnPodSelectorLabelValue, - } - port := int32(80) - udpPort := int32(53) - builder := &utils.AntreaNetworkPolicySpecBuilder{} - builder = builder.SetName(data.testNamespace, fqdnPolicyName). - SetTier(defaultTierName). - SetPriority(1.0). - SetAppliedToGroup([]utils.ANNPAppliedToSpec{{PodSelector: podSelectorLabel}}) - builder.AddFQDNRule(testFQDN, utils.ProtocolTCP, &port, nil, nil, "AllowForFQDN", nil, - crdv1beta1.RuleActionAllow) - builder.AddEgress(utils.ProtocolUDP, &udpPort, nil, nil, nil, nil, - nil, nil, nil, nil, map[string]string{"app": customDNSLabelValue}, - nil, nil, nil, nil, - nil, nil, crdv1beta1.RuleActionAllow, "", "AllowDnsQueries") - builder.AddEgress(utils.ProtocolTCP, nil, nil, nil, nil, nil, - nil, nil, nil, nil, nil, - nil, nil, nil, nil, - nil, nil, crdv1beta1.RuleActionReject, "", "DropAllRemainingTraffic") - - annp, err := data.CreateOrUpdateANNP(builder.Get()) - require.NoError(t, err, "error while deploying Antrea policy") - require.NoError(t, data.waitForANNPRealized(t, annp.Namespace, annp.Name, 10*time.Second)) -} - -// createHttpAgnhostPod creates an agnhost Pod that serves HTTP requests and returns the IP of Pod created. -func createHttpAgnhostPod(t *testing.T, data *TestData) *PodIPs { - const ( - agnhostPort = 80 - agnhostPodNamePreFix = "agnhost-" - ) - podName := randName(agnhostPodNamePreFix) - args := []string{"netexec", "--http-port=" + strconv.Itoa(agnhostPort)} - ports := []corev1.ContainerPort{ - { - Name: "http", - ContainerPort: agnhostPort, - Protocol: corev1.ProtocolTCP, - }, - } - - require.NoError(t, NewPodBuilder(podName, data.testNamespace, agnhostImage). - WithArgs(args). - WithPorts(ports). - WithLabels(map[string]string{"app": "agnhost"}). - Create(data)) - podIPs, err := data.podWaitForIPs(defaultTimeout, podName, data.testNamespace) - require.NoError(t, err) - return podIPs -} - -// createDNSPod creates the CoreDNS Pod configured to use the custom DNS ConfigMap. -func createCustomDNSPod(t *testing.T, data *TestData, configName string) { - volume := []corev1.Volume{ - { - Name: "config-volume", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: configName, - }, - Items: []corev1.KeyToPath{ - { - Key: "Corefile", - Path: "Corefile", - }, - }, - }, - }, - }, - } - - volumeMount := []corev1.VolumeMount{ - { - Name: "config-volume", - MountPath: "/etc/coredns", - }, - } - - require.NoError(t, NewPodBuilder("custom-dns-server", data.testNamespace, "coredns/coredns:1.11.3"). - WithLabels(map[string]string{"app": "custom-dns"}). - WithContainerName("coredns"). - WithArgs([]string{"-conf", "/etc/coredns/Corefile"}). - AddVolume(volume).AddVolumeMount(volumeMount). - Create(data)) - require.NoError(t, data.podWaitForRunning(defaultTimeout, "custom-dns-server", data.testNamespace)) -} - -// createDNSConfig generates a DNS configuration for the specified IP address and domain name. -func createDNSConfig(t *testing.T, hosts map[string]string, ttl int) map[string]string { - const coreFileTemplate = `lfx.test:53 { - errors - log - health - hosts { - {{ range $IP, $FQDN := .Hosts }}{{ $IP }} {{ $FQDN }}{{ end }} - no_reverse - pods verified - ttl {{ .TTL }} - } - loop - reload 2s - }` - - generateConfigData := func() (string, error) { - - data := struct { - Hosts map[string]string - TTL int - }{ - Hosts: hosts, - TTL: ttl, - } - - tmpl, err := template.New("configMapData").Parse(coreFileTemplate) - if err != nil { - return "", err - } - var output bytes.Buffer - err = tmpl.Execute(&output, data) - if err != nil { - return "", err - } - return strings.TrimSpace(output.String()), nil - } - - configMapData, err := generateConfigData() - require.NoError(t, err, "error processing configData template for DNS") - configData := map[string]string{ - "Corefile": configMapData, - } - - return configData -} diff --git a/test/e2e/framework.go b/test/e2e/framework.go index 889e5fde2a3..3b236179395 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -1490,14 +1490,15 @@ func (b *PodBuilder) WithReadinessProbe(probe *corev1.Probe) *PodBuilder { return b } -// WithCustomDNSConfig adds custom DNS IP to the Pod spec. -func (b *PodBuilder) WithCustomDNSConfig(dnsServiceIP string) *PodBuilder { +// WithCustomDNSConfig adds a custom DNS Configuration to the Pod spec. +// It ensures that the DNSPolicy is set to 'None' and assigns the provided DNSConfig. +func (b *PodBuilder) WithCustomDNSConfig(dnsServerConfig *corev1.PodDNSConfig) *PodBuilder { b.MutateFunc = func(pod *corev1.Pod) { + // Set DNSPolicy to None to allow custom DNSConfig pod.Spec.DNSPolicy = corev1.DNSNone - if pod.Spec.DNSConfig == nil { - pod.Spec.DNSConfig = &corev1.PodDNSConfig{} - } - pod.Spec.DNSConfig.Nameservers = []string{dnsServiceIP} + + // Assign the provided DNSConfig to the Pod's DNSConfig field + pod.Spec.DNSConfig = dnsServerConfig } return b } From ab67af111a1f72f2e0ac995e4de57f426cc58f4f Mon Sep 17 00:00:00 2001 From: Hemant Date: Wed, 13 Nov 2024 23:28:35 +0530 Subject: [PATCH 4/5] refactor the approach to set dnsConfig in podBuilder Signed-off-by: Hemant --- test/e2e/framework.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/test/e2e/framework.go b/test/e2e/framework.go index 3b236179395..33643c5d335 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -1366,6 +1366,7 @@ type PodBuilder struct { ResourceRequests corev1.ResourceList ResourceLimits corev1.ResourceList ReadinessProbe *corev1.Probe + DnsConfig *corev1.PodDNSConfig } func NewPodBuilder(name, ns, image string) *PodBuilder { @@ -1492,14 +1493,8 @@ func (b *PodBuilder) WithReadinessProbe(probe *corev1.Probe) *PodBuilder { // WithCustomDNSConfig adds a custom DNS Configuration to the Pod spec. // It ensures that the DNSPolicy is set to 'None' and assigns the provided DNSConfig. -func (b *PodBuilder) WithCustomDNSConfig(dnsServerConfig *corev1.PodDNSConfig) *PodBuilder { - b.MutateFunc = func(pod *corev1.Pod) { - // Set DNSPolicy to None to allow custom DNSConfig - pod.Spec.DNSPolicy = corev1.DNSNone - - // Assign the provided DNSConfig to the Pod's DNSConfig field - pod.Spec.DNSConfig = dnsServerConfig - } +func (b *PodBuilder) WithCustomDNSConfig(dnsConfig *corev1.PodDNSConfig) *PodBuilder { + b.DnsConfig = dnsConfig return b } @@ -1545,6 +1540,13 @@ func (b *PodBuilder) Create(data *TestData) error { // tolerate NoSchedule taint if we want Pod to run on control-plane Node podSpec.Tolerations = controlPlaneNoScheduleTolerations() } + if b.DnsConfig != nil { + // Set DNSPolicy to None to allow custom DNSConfig + podSpec.DNSPolicy = corev1.DNSNone + + // Assign the provided DNSConfig to the Pod's DNSConfig field + podSpec.DNSConfig = b.DnsConfig + } pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: b.Name, From 21b473f81ea70464a39623d8fea599cece6cc670 Mon Sep 17 00:00:00 2001 From: Hemant Date: Thu, 14 Nov 2024 22:36:54 +0530 Subject: [PATCH 5/5] added line for seperation of variables Signed-off-by: Hemant --- test/e2e/framework.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/e2e/framework.go b/test/e2e/framework.go index 33643c5d335..f8467680e03 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -140,8 +140,9 @@ const ( defaultCHDatabaseURL = "tcp://clickhouse-clickhouse.flow-visibility.svc:9000" statefulSetRestartAnnotationKey = "antrea-e2e/restartedAt" - iperfPort = 5201 - iperfSvcPort = 9999 + + iperfPort = 5201 + iperfSvcPort = 9999 ) type ClusterNode struct {