Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

system-test: added IPv6 support to EgressService tests #243

Merged
merged 1 commit into from
Oct 8, 2024
Merged

system-test: added IPv6 support to EgressService tests #243

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
218 changes: 174 additions & 44 deletions tests/system-tests/rdscore/internal/rdscorecommon/egress-service.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package rdscorecommon
import (
"fmt"
"net"
"net/netip"
"os/exec"
"strings"
"time"
Expand Down Expand Up @@ -218,10 +219,6 @@ func verifyPodSourceAddress(clientPods []*pod.Builder, cmdToRun []string, expect
}
}

// func setupWorkloadWithClusterETP() {
//
// }

// VerifyEgressServiceWithClusterETP verifies EgressService with externalTrafficPolicy set to Cluster.
//
//nolint:funlen
Expand Down Expand Up @@ -293,7 +290,15 @@ func VerifyEgressServiceWithClusterETP(ctx SpecContext) {
svcBuilder = svcBuilder.WithAnnotation(map[string]string{
"metallb.universe.tf/address-pool": RDSCoreConfig.EgressServiceDeploy1IPAddrPool})

By("Setting ipFamilyPolicy to 'RequireDualStack'")

glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Setting ipFamilyPolicy to 'RequireDualStack'")

svcBuilder = svcBuilder.WithIPFamily([]corev1.IPFamily{"IPv4", "IPv6"},
corev1.IPFamilyPolicyRequireDualStack)

By("Creating a service")

glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Creating Service object")

svcBuilder, err = svcBuilder.Create()
Expand Down Expand Up @@ -324,7 +329,7 @@ func VerifyEgressServiceWithClusterETP(ctx SpecContext) {
glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Configuring VRF network on EgressService %q in %q namespace",
egrSVCBuilder.Definition.Name, egrSVCBuilder.Definition.Namespace)

egrSVCBuilder = egrSVCBuilder.WithVRFNetwork("1001")
egrSVCBuilder = egrSVCBuilder.WithVRFNetwork(RDSCoreConfig.EgressServiceVRF1Network)
}

By("Creating EgressService")
Expand Down Expand Up @@ -365,15 +370,35 @@ func VerifyEgressServiceWithClusterETP(ctx SpecContext) {
}).WithContext(ctx).WithPolling(15*time.Second).WithTimeout(3*time.Minute).Should(BeTrue(),
"Service does not have LoadBalancer IP address")

loadBalancerIP := svcBuilder.Object.Status.LoadBalancer.Ingress[0].IP
for _, vip := range svcBuilder.Object.Status.LoadBalancer.Ingress {
loadBalancerIP := vip.IP

glog.V(rdscoreparams.RDSCoreLogLevel).Infof("LoadBalancer IP address: %q", loadBalancerIP)
glog.V(rdscoreparams.RDSCoreLogLevel).Infof("LoadBalancer IP address: %q", loadBalancerIP)

cmdToRun := []string{"/bin/bash", "-c",
fmt.Sprintf("curl --connect-timeout 3 -Ls http://%s:%s/clientip",
RDSCoreConfig.EgressServiceRemoteIP, RDSCoreConfig.EgressServiceRemotePort)}
var (
cmdToRun []string
)

myIP, err := netip.ParseAddr(loadBalancerIP)

Expect(err).ToNot(HaveOccurred(), "Failed to parse IP address")

if myIP.Is4() {
glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Processing IPv4 address")

cmdToRun = []string{"/bin/bash", "-c",
fmt.Sprintf("curl --connect-timeout 3 -Ls http://%s:%s/clientip",
RDSCoreConfig.EgressServiceRemoteIP, RDSCoreConfig.EgressServiceRemotePort)}
} else {
glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Processing IPv6 address")

verifyPodSourceAddress(clientPods, cmdToRun, loadBalancerIP)
cmdToRun = []string{"/bin/bash", "-c",
fmt.Sprintf("curl --connect-timeout 3 -Ls http://[%s]:%s/clientip",
RDSCoreConfig.EgressServiceRemoteIPv6, RDSCoreConfig.EgressServiceRemotePort)}
}

verifyPodSourceAddress(clientPods, cmdToRun, loadBalancerIP)
}
}

// VerifyEgressServiceWithLocalETP verifies EgressService with externalTrafficPolicy set to Local.
Expand Down Expand Up @@ -452,6 +477,11 @@ func VerifyEgressServiceWithLocalETP(ctx SpecContext) {
svcBuilder = svcBuilder.WithAnnotation(map[string]string{
"metallb.universe.tf/address-pool": RDSCoreConfig.EgressServiceDeploy2IPAddrPool})

By("Setting ipFamilyPolicy to 'RequireDualStack'")

svcBuilder = svcBuilder.WithIPFamily([]corev1.IPFamily{"IPv4", "IPv6"},
corev1.IPFamilyPolicyRequireDualStack)

By("Creating a service")
glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Creating Service object")

Expand All @@ -468,22 +498,22 @@ func VerifyEgressServiceWithLocalETP(ctx SpecContext) {
egrSVCBuilder := egressservice.NewEgressServiceBuilder(APIClient, egressSVC2Name,
RDSCoreConfig.EgressServiceNS, "LoadBalancerIP")

if len(RDSCoreConfig.EgressServiceDeploy1NodeSelector) != 0 {
if len(RDSCoreConfig.EgressServiceDeploy2NodeSelector) != 0 {
By("Setting nodeSelector for EgressService")

glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Configuring nodeSelector on EgressService %q in %q namespace",
egrSVCBuilder.Definition.Name, egrSVCBuilder.Definition.Namespace)

egrSVCBuilder = egrSVCBuilder.WithNodeLabelSelector(RDSCoreConfig.EgressServiceDeploy1NodeSelector)
egrSVCBuilder = egrSVCBuilder.WithNodeLabelSelector(RDSCoreConfig.EgressServiceDeploy2NodeSelector)
}

if RDSCoreConfig.EgressServiceVRF1Network != "" {
if RDSCoreConfig.EgressServiceVRF2Network != "" {
By("Setting VRF network for EgressService")

glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Configuring VRF network on EgressService %q in %q namespace",
egrSVCBuilder.Definition.Name, egrSVCBuilder.Definition.Namespace)

egrSVCBuilder = egrSVCBuilder.WithVRFNetwork("1001")
egrSVCBuilder = egrSVCBuilder.WithVRFNetwork(RDSCoreConfig.EgressServiceVRF2Network)
}

By("Creating EgressService")
Expand Down Expand Up @@ -524,24 +554,50 @@ func VerifyEgressServiceWithLocalETP(ctx SpecContext) {
fmt.Sprintf("Application pods matching %q label not found in %q namespace",
RDSCoreConfig.EgressServiceNS, egressSVC2Labels))

cmdToRun := []string{"/bin/bash", "-c",
fmt.Sprintf("curl --connect-timeout 3 -Ls http://%s:%s/clientip",
RDSCoreConfig.EgressServiceRemoteIP, RDSCoreConfig.EgressServiceRemotePort)}
for _, vip := range svcBuilder.Object.Status.LoadBalancer.Ingress {
loadBalancerIP := vip.IP

glog.V(rdscoreparams.RDSCoreLogLevel).Infof("LoadBalancer IP address: %q", loadBalancerIP)

var (
cmdToRun []string
expectedIP string
)

loadBalancerIP := svcBuilder.Object.Status.LoadBalancer.Ingress[0].IP
myIP, err := netip.ParseAddr(loadBalancerIP)

glog.V(rdscoreparams.RDSCoreLogLevel).Infof("LoadBalancer IP address: %q", loadBalancerIP)
Expect(err).ToNot(HaveOccurred(), "Failed to parse IP address")

verifyPodSourceAddress(clientPods, cmdToRun, loadBalancerIP)
if myIP.Is4() {
glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Processing IPv4 address")

By(fmt.Sprintf("Accessing workload via LoadBalancer's IP %s", loadBalancerIP))
cmdToRun = []string{"/bin/bash", "-c",
fmt.Sprintf("curl --connect-timeout 3 -Ls http://%s:%s/clientip",
RDSCoreConfig.EgressServiceRemoteIP, RDSCoreConfig.EgressServiceRemotePort)}

glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Accessing workload via LoadBalancer's IP %s", loadBalancerIP)
expectedIP = RDSCoreConfig.EgressServiceRemoteIP
} else {
glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Processing IPv6 address")

verifyIngressIP(loadBalancerIP, RDSCoreConfig.EgressServiceRemoteIP, servicePort)
cmdToRun = []string{"/bin/bash", "-c",
fmt.Sprintf("curl --connect-timeout 3 -Ls http://[%s]:%s/clientip",
RDSCoreConfig.EgressServiceRemoteIPv6, RDSCoreConfig.EgressServiceRemotePort)}

expectedIP = RDSCoreConfig.EgressServiceRemoteIPv6
}

verifyPodSourceAddress(clientPods, cmdToRun, loadBalancerIP)

By(fmt.Sprintf("Accessing workload via LoadBalancer's IP %s", loadBalancerIP))

glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Accessing workload via LoadBalancer's IP %s", loadBalancerIP)

verifyIngressIP(loadBalancerIP, expectedIP, servicePort)
}
}

func verifySourceIP(svcName, svcNS, podLabels string, cmdToRun []string) {
//nolint:funlen
func verifySourceIP(svcName, svcNS, podLabels string, cmdToRun []string, useIPv6 bool) {
By(fmt.Sprintf("Pulling %q service configuration", svcName))

var (
Expand Down Expand Up @@ -591,10 +647,6 @@ func verifySourceIP(svcName, svcNS, podLabels string, cmdToRun []string) {
}).WithContext(ctx).WithPolling(15*time.Second).WithTimeout(3*time.Minute).Should(BeTrue(),
"Service does not have LoadBalancer IP address")

loadBalancerIP := svcBuilder.Object.Status.LoadBalancer.Ingress[0].IP

glog.V(rdscoreparams.RDSCoreLogLevel).Infof("LoadBalancer IP address: %q", loadBalancerIP)

By("Finding pod from app deployment")

clientPods := findPodWithSelector(svcNS, podLabels)
Expand All @@ -603,7 +655,41 @@ func verifySourceIP(svcName, svcNS, podLabels string, cmdToRun []string) {
fmt.Sprintf("Application pods matching %q label not found in %q namespace",
svcName, svcNS))

verifyPodSourceAddress(clientPods, cmdToRun, loadBalancerIP)
By("Processing all LoadBalancer IP addresses")

var trafficValidated bool

for _, vip := range svcBuilder.Object.Status.LoadBalancer.Ingress {
loadBalancerIP := vip.IP

glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Processing %q address", loadBalancerIP)

myIP, err := netip.ParseAddr(loadBalancerIP)

Expect(err).ToNot(HaveOccurred(), "Failed to parse IP address")

if myIP.Is4() && useIPv6 {
glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Skipping %q address as IPv6 is required",
loadBalancerIP)

continue
}

if myIP.Is6() && !useIPv6 {
glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Skipping %q address as IPv4 is required",
loadBalancerIP)

continue
}

trafficValidated = true

glog.V(rdscoreparams.RDSCoreLogLevel).Infof("LoadBalancer IP address: %q", loadBalancerIP)

verifyPodSourceAddress(clientPods, cmdToRun, loadBalancerIP)
}

Expect(trafficValidated).To(BeTrue(), "Traffic wasn't validated")
}

// VerifyEgressServiceConnectivityETPCluster verifies source IP address when external traffic policy
Expand All @@ -613,7 +699,13 @@ func VerifyEgressServiceConnectivityETPCluster() {
fmt.Sprintf("curl --connect-timeout 3 -Ls http://%s:%s/clientip",
RDSCoreConfig.EgressServiceRemoteIP, RDSCoreConfig.EgressServiceRemotePort)}

verifySourceIP(egressSVC1Name, RDSCoreConfig.EgressServiceNS, egressSVC1Labels, cmdToRun)
verifySourceIP(egressSVC1Name, RDSCoreConfig.EgressServiceNS, egressSVC1Labels, cmdToRun, false)

cmdToRun = []string{"/bin/bash", "-c",
fmt.Sprintf("curl --connect-timeout 3 -Ls http://[%s]:%s/clientip",
RDSCoreConfig.EgressServiceRemoteIPv6, RDSCoreConfig.EgressServiceRemotePort)}

verifySourceIP(egressSVC1Name, RDSCoreConfig.EgressServiceNS, egressSVC1Labels, cmdToRun, true)
}

// VerifyEgressServiceConnectivityETPLocal verifies source IP address when external traffic policy
Expand All @@ -623,13 +715,25 @@ func VerifyEgressServiceConnectivityETPLocal() {
fmt.Sprintf("curl --connect-timeout 3 -Ls http://%s:%s/clientip",
RDSCoreConfig.EgressServiceRemoteIP, RDSCoreConfig.EgressServiceRemotePort)}

verifySourceIP(egressSVC2Name, RDSCoreConfig.EgressServiceNS, egressSVC2Labels, cmdToRun)
verifySourceIP(egressSVC2Name, RDSCoreConfig.EgressServiceNS, egressSVC2Labels, cmdToRun, false)

cmdToRun = []string{"/bin/bash", "-c",
fmt.Sprintf("curl --connect-timeout 3 -Ls http://[%s]:%s/clientip",
RDSCoreConfig.EgressServiceRemoteIPv6, RDSCoreConfig.EgressServiceRemotePort)}

verifySourceIP(egressSVC2Name, RDSCoreConfig.EgressServiceNS, egressSVC2Labels, cmdToRun, true)
}

// VerifyEgressServiceIngressConnectivity verifies ingress IP address while accessing backend pods
// via loadbalancer.
func VerifyEgressServiceIngressConnectivity() {
By(fmt.Sprintf("Pulling %q service configuration", egressSVC2Name))
// VerifyEgressServiceETPLocalIngressConnectivity verifies ingress IP address while accessing backend pods
// via loadbalancer with ETP=Local.
func VerifyEgressServiceETPLocalIngressConnectivity() {
verifyEgressServiceIngressConnectivit(egressSVC2Name)
}

// verifyEgressServiceIngressConnectivit shared function to verify backend pods' availability via
// loadbalancer's IP address(es).
func verifyEgressServiceIngressConnectivit(svcName string) {
By(fmt.Sprintf("Pulling %q service configuration", svcName))

var (
svcBuilder *service.Builder
Expand All @@ -639,13 +743,13 @@ func VerifyEgressServiceIngressConnectivity() {

Eventually(func() bool {
glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Pulling %q service from %q namespace",
egressSVC2Name, RDSCoreConfig.EgressServiceNS)
svcName, RDSCoreConfig.EgressServiceNS)

svcBuilder, err = service.Pull(APIClient, egressSVC2Name, RDSCoreConfig.EgressServiceNS)
svcBuilder, err = service.Pull(APIClient, svcName, RDSCoreConfig.EgressServiceNS)

if err != nil {
glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Error pulling %q service from %q namespace: %v",
egressSVC2Name, RDSCoreConfig.EgressServiceNS, err)
svcName, RDSCoreConfig.EgressServiceNS, err)

return false
}
Expand All @@ -655,7 +759,7 @@ func VerifyEgressServiceIngressConnectivity() {

return true
}).WithContext(ctx).WithPolling(5*time.Second).WithTimeout(1*time.Minute).Should(BeTrue(),
fmt.Sprintf("Error obtaining service %q configuration", egressSVC2Name))
fmt.Sprintf("Error obtaining service %q configuration", svcName))

By(fmt.Sprintf("Asserting service %q has LoadBalancer IP address", svcBuilder.Definition.Name))

Expand All @@ -678,11 +782,23 @@ func VerifyEgressServiceIngressConnectivity() {
}).WithContext(ctx).WithPolling(15*time.Second).WithTimeout(3*time.Minute).Should(BeTrue(),
"Service does not have LoadBalancer IP address")

loadBalancerIP := svcBuilder.Object.Status.LoadBalancer.Ingress[0].IP
for _, vip := range svcBuilder.Object.Status.LoadBalancer.Ingress {
loadBalancerIP := vip.IP

glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Accessing workload via LoadBalancer's IP %s", loadBalancerIP)

myIP, err := netip.ParseAddr(loadBalancerIP)

glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Accessing workload via LoadBalancer's IP %s", loadBalancerIP)
Expect(err).ToNot(HaveOccurred(), "Failed to parse IP address")

if myIP.Is4() {
verifyIngressIP(loadBalancerIP, RDSCoreConfig.EgressServiceRemoteIP, servicePort)
}

verifyIngressIP(loadBalancerIP, RDSCoreConfig.EgressServiceRemoteIP, servicePort)
if myIP.Is6() {
verifyIngressIP(loadBalancerIP, RDSCoreConfig.EgressServiceRemoteIPv6, servicePort)
}
}
}

func verifyIngressIP(loadBalancerIP, expectedIP string, servicePort int32) {
Expand All @@ -695,8 +811,22 @@ func verifyIngressIP(loadBalancerIP, expectedIP string, servicePort int32) {
By(fmt.Sprintf("Accessing backend pods via %s IP", loadBalancerIP))

Eventually(func() bool {
myIP, err := netip.ParseAddr(loadBalancerIP)

Expect(err).ToNot(HaveOccurred(), "Failed to parse IP address")

var noramlizedIP string

if myIP.Is4() {
noramlizedIP = loadBalancerIP
}

if myIP.Is6() {
noramlizedIP = fmt.Sprintf("[%s]", loadBalancerIP)
}

cmdExternal := exec.Command("curl", "--connect-timeout", "3", "-s",
fmt.Sprintf("http://%s:%d/clientip", loadBalancerIP, servicePort))
fmt.Sprintf("http://%s:%d/clientip", noramlizedIP, servicePort))

glog.V(rdscoreparams.RDSCoreLogLevel).Infof("Running command: %q", cmdExternal.String())

Expand Down
2 changes: 2 additions & 0 deletions tests/system-tests/rdscore/internal/rdscoreconfig/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -355,6 +355,8 @@ type CoreConfig struct {
//nolint:lll,nolintlint
EgressServiceRemoteIP string `yaml:"rdscore_egress_service_remote_ip" envconfig:"ECO_RDSCORE_EGRESS_SERVICE_REMOTE_IP"`
//nolint:lll,nolintlint
EgressServiceRemoteIPv6 string `yaml:"rdscore_egress_service_remote_ipv6" envconfig:"ECO_RDSCORE_EGRESS_SERVICE_REMOTE_IPV6"`
//nolint:lll,nolintlint
EgressServiceRemotePort string `yaml:"rdscore_egress_service_remote_port" envconfig:"ECO_RDSCORE_EGRESS_SERVICE_REMOTE_PORT"`
//nolint:lll,nolintlint
EgressServiceDeploy1CMD EnvSliceString `yaml:"rdscore_egress_service_deploy_1_cmd" envconfig:"ECO_RDSCORE_EGRESS_SERVICE_DEPLOY_1_CMD"`
Expand Down
1 change: 1 addition & 0 deletions tests/system-tests/rdscore/internal/rdscoreconfig/default.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,7 @@ rdscore_metallb_frr_test_url_ipv6: ''
# Egress Service
rdscore_egress_service_ns: rds-egress-ns
rdscore_egress_service_remote_ip: ''
rdscore_egress_service_remote_ipv6: ''
rdscore_egress_service_remote_port: 9090
#
rdscore_egress_service_deploy_1_cmd:
Expand Down
Loading
Loading