Skip to content

Commit

Permalink
e2e: use external containers for EF connectivity checks to make it
Browse files Browse the repository at this point in the history 
work with ipv6, since github runners don't have any routes for IPv6.
Split current test that checks allow IP and allow CIDR+port into 2
tests to limit the amount of required external containers.
Bonus: the only test that used external containers doesn't need to
create them anymore, as they are created in beforeEach

Signed-off-by: Nadia Pinaeva <[email protected]>
  • Loading branch information
@npinaeva
npinaeva committed May 31, 2024
1 parent 09b08ec commit 2372a08
Showing 1 changed file with 94 additions and 51 deletions.
145 changes: 94 additions & 51 deletions test/e2e/egress_firewall.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,10 @@ var _ = ginkgo.Describe("e2e egress firewall policy validation", func() {
retryInterval = 1 * time.Second
retryTimeout = 30 * time.Second
ciNetworkName = "kind"
externalContainerName1 = "e2e-egress-fw-external-container1"
externalContainerName2 = "e2e-egress-fw-external-container2"
externalContainerPort1 = 1111
externalContainerPort2 = 2222
)

type nodeInfo struct {
Expand All @@ -44,13 +48,15 @@ var _ = ginkgo.Describe("e2e egress firewall policy validation", func() {
}

var (
serverNodeInfo nodeInfo
exFWPermitTcpDnsDest string
singleIPMask string
exFWDenyTcpDnsDest string
exFWPermitTcpWwwDest string
exFWPermitCIDR string
exFWDenyCIDR string
serverNodeInfo nodeInfo
exFWPermitTcpDnsDest string
exFWDenyTcpDnsDest string
exFWPermitTcpWwwDest string
exFWPermitCIDR string
exFWDenyCIDR string
denyAllCIDR string
singleIPMask, subnetMask string
externalContainer1IP, externalContainer2IP string
)

waitForEFApplied := func(namespace string) {
Expand Down Expand Up @@ -154,6 +160,34 @@ var _ = ginkgo.Describe("e2e egress firewall policy validation", func() {
exFWDenyCIDR = "::/0"
singleIPMask = "128"
}

externalContainer1IPV4, externalContainer1IPV6 := createClusterExternalContainer(externalContainerName1, agnhostImage,
[]string{"--network", ciNetworkName, "-p", fmt.Sprintf("%d:%d", externalContainerPort1, externalContainerPort1)},
[]string{"netexec", fmt.Sprintf("--http-port=%d", externalContainerPort1)})

externalContainer2IPV4, externalContainer2IPV6 := createClusterExternalContainer(externalContainerName2, agnhostImage,
[]string{"--network", ciNetworkName, "-p", fmt.Sprintf("%d:%d", externalContainerPort2, externalContainerPort2)},
[]string{"netexec", fmt.Sprintf("--http-port=%d", externalContainerPort2)})

if IsIPv6Cluster(f.ClientSet) {
externalContainer1IP = externalContainer1IPV6
externalContainer2IP = externalContainer2IPV6
} else {
externalContainer1IP = externalContainer1IPV4
externalContainer2IP = externalContainer2IPV4
}

singleIPMask = "32"
subnetMask = "24"
if IsIPv6Cluster(f.ClientSet) {
singleIPMask = "128"
subnetMask = "64"
}

denyAllCIDR = "0.0.0.0/0"
if IsIPv6Cluster(f.ClientSet) {
denyAllCIDR = "::/0"
}
})

ginkgo.AfterEach(func() {
Expand All @@ -168,9 +202,11 @@ var _ = ginkgo.Describe("e2e egress firewall policy validation", func() {
}
}
}
deleteClusterExternalContainer(externalContainerName1)
deleteClusterExternalContainer(externalContainerName2)
})

ginkgo.It("Should validate the egress firewall policy functionality against remote hosts", func() {
ginkgo.It("Should validate the egress firewall policy functionality for allowed IP", func() {
srcPodName := "e2e-egress-fw-src-pod"
// egress firewall crd yaml configuration
var egressFirewallConfig = fmt.Sprintf(`kind: EgressFirewall
Expand All @@ -183,36 +219,60 @@ spec:
- type: Allow
to:
cidrSelector: %s/%s
- type: Allow
- type: Deny
to:
cidrSelector: %s
`, f.Namespace.Name, externalContainer1IP, singleIPMask, denyAllCIDR)
applyEF(egressFirewallConfig, f.Namespace.Name)

// create the pod that will be used as the source for the connectivity test
createSrcPod(srcPodName, serverNodeInfo.name, retryInterval, retryTimeout, f)

// Verify the remote host/port as explicitly allowed by the firewall policy is reachable
ginkgo.By(fmt.Sprintf("Verifying connectivity to an explicitly allowed host %s is permitted as defined "+
"by the external firewall policy", externalContainer1IP))
checkConnectivity(srcPodName, externalContainer1IP, externalContainerPort1, true)

// Verify the remote host/port as implicitly denied by the firewall policy is not reachable
ginkgo.By(fmt.Sprintf("Verifying connectivity to an implicitly denied host %s is not permitted as defined "+
"by the external firewall policy", externalContainer2IP))
checkConnectivity(srcPodName, externalContainer2IP, externalContainerPort2, false)
})

ginkgo.It("Should validate the egress firewall policy functionality for allowed CIDR and port", func() {
srcPodName := "e2e-egress-fw-src-pod"
// egress firewall crd yaml configuration
var egressFirewallConfig = fmt.Sprintf(`kind: EgressFirewall
apiVersion: k8s.ovn.org/v1
metadata:
name: default
namespace: %s
spec:
egress:
- type: Allow
to:
cidrSelector: %s/%s
ports:
- protocol: TCP
port: 80
port: %d
- type: Deny
to:
cidrSelector: %s
`, f.Namespace.Name, exFWPermitTcpDnsDest, singleIPMask, exFWPermitCIDR, exFWDenyCIDR)
`, f.Namespace.Name, externalContainer1IP, subnetMask, externalContainerPort1, denyAllCIDR)
applyEF(egressFirewallConfig, f.Namespace.Name)

// create the pod that will be used as the source for the connectivity test
createSrcPod(srcPodName, serverNodeInfo.name, retryInterval, retryTimeout, f)

// Verify the remote host/port as explicitly allowed by the firewall policy is reachable
ginkgo.By(fmt.Sprintf("Verifying connectivity to an explicitly allowed host %s is permitted as defined by the external firewall policy", exFWPermitTcpDnsDest))
checkConnectivity(srcPodName, exFWPermitTcpDnsDest, 53, true)
ginkgo.By(fmt.Sprintf("Verifying connectivity to an explicitly allowed port on host %s is permitted as "+
"defined by the external firewall policy", externalContainer1IP))
checkConnectivity(srcPodName, externalContainer1IP, externalContainerPort1, true)

// Verify the remote host/port as implicitly denied by the firewall policy is not reachable
ginkgo.By(fmt.Sprintf("Verifying connectivity to an implicitly denied host %s is not permitted as defined by the external firewall policy", exFWDenyTcpDnsDest))
checkConnectivity(srcPodName, exFWDenyTcpDnsDest, 53, false)

// Verify the explicitly allowed host/port tcp port 80 rule is functional
ginkgo.By(fmt.Sprintf("Verifying connectivity to an explicitly allowed host %s is permitted as defined by the external firewall policy", exFWPermitTcpWwwDest))
checkConnectivity(srcPodName, exFWPermitTcpWwwDest, 80, true)

// Verify the remote host/port 443 as implicitly denied by the firewall policy is not reachable
ginkgo.By(fmt.Sprintf("Verifying connectivity to an implicitly denied port on host %s is not permitted as defined by the external firewall policy", exFWPermitTcpWwwDest))
checkConnectivity(srcPodName, exFWPermitTcpWwwDest, 443, false)
ginkgo.By(fmt.Sprintf("Verifying connectivity to an implicitly denied port on host %s is not permitted as "+
"defined by the external firewall policy", externalContainer2IP))
checkConnectivity(srcPodName, externalContainer2IP, externalContainerPort2, false)
})

ginkgo.It("Should validate the egress firewall policy functionality against cluster nodes by using node selector", func() {
Expand Down Expand Up @@ -470,13 +530,6 @@ spec:
efPodPort := 1234
serviceName := "service-for-pods"
servicePort := 31234
externalContainerName := "e2e-egress-fw-external-container"
externalContainerPort := 1234

denyCIDR := "0.0.0.0/0"
if IsIPv6Cluster(f.ClientSet) {
denyCIDR = "::/0"
}

ginkgo.By("Creating the egress firewall pod")
// 1. create nodePort service and external container
Expand All @@ -495,32 +548,22 @@ spec:
err = framework.WaitForServiceEndpointsNum(context.TODO(), f.ClientSet, f.Namespace.Name, serviceName, 1, time.Second, wait.ForeverTestTimeout)
framework.ExpectNoError(err, "failed to validate endpoints for service %s in namespace: %s", serviceName, f.Namespace.Name)

nodeIP := serverNodeInfo.nodeIP
externalContainerIPV4, externalContainerIPV6 := createClusterExternalContainer(externalContainerName, agnhostImage,
[]string{"--network", ciNetworkName, "-p", fmt.Sprintf("%d:%d", externalContainerPort, externalContainerPort)},
[]string{"netexec", fmt.Sprintf("--http-port=%d", externalContainerPort)})
defer deleteClusterExternalContainer(externalContainerName)

// 2. Check connectivity works both ways
// pod -> external container should work
externalContainerIP := externalContainerIPV4
if IsIPv6Cluster(f.ClientSet) {
externalContainerIP = externalContainerIPV6
}
ginkgo.By(fmt.Sprintf("Verifying connectivity from pod %s to external container [%s]:%d",
efPodName, externalContainerIP, externalContainerPort))
checkConnectivity(efPodName, externalContainerIP, externalContainerPort, true)
efPodName, externalContainer1IP, externalContainerPort1))
checkConnectivity(efPodName, externalContainer1IP, externalContainerPort1, true)

// external container -> nodePort svc should work
svc, err := f.ClientSet.CoreV1().Services(f.Namespace.Name).Get(context.TODO(), serviceName, metav1.GetOptions{})
framework.ExpectNoError(err, "failed to fetch service: %s in namespace %s", serviceName, f.Namespace.Name)

nodeIP := serverNodeInfo.nodeIP
ginkgo.By(fmt.Sprintf("Verifying connectivity from external container %s to nodePort svc [%s]:%d",
externalContainerIP, nodeIP, svc.Spec.Ports[0].NodePort))
checkExternalContainerConnectivity(externalContainerName, nodeIP, int(svc.Spec.Ports[0].NodePort))
externalContainer1IP, nodeIP, svc.Spec.Ports[0].NodePort))
checkExternalContainerConnectivity(externalContainerName1, nodeIP, int(svc.Spec.Ports[0].NodePort))

// 3. Apply deny-all egress firewall and wait for it to be applied
// egress firewall crd yaml configuration
var egressFirewallConfig = fmt.Sprintf(`kind: EgressFirewall
apiVersion: k8s.ovn.org/v1
metadata:
Expand All @@ -531,18 +574,18 @@ spec:
- type: Deny
to:
cidrSelector: %s
`, f.Namespace.Name, denyCIDR)
`, f.Namespace.Name, denyAllCIDR)
applyEF(egressFirewallConfig, f.Namespace.Name)

// 4. Check that only inbound traffic is allowed
// pod -> external container should be blocked
ginkgo.By(fmt.Sprintf("Verifying connection from pod %s to external container %s is blocked:%d",
efPodName, externalContainerIP, externalContainerPort))
checkConnectivity(efPodName, externalContainerIP, externalContainerPort, false)
ginkgo.By(fmt.Sprintf("Verifying connectivity from pod %s to external container [%s]:%d is blocked",
efPodName, externalContainer1IP, externalContainerPort1))
checkConnectivity(efPodName, externalContainer1IP, externalContainerPort1, false)

// external container -> nodePort svc should work
ginkgo.By(fmt.Sprintf("Verifying connectivity from external container %s to nodePort svc [%s]:%d",
externalContainerIP, nodeIP, svc.Spec.Ports[0].NodePort))
checkExternalContainerConnectivity(externalContainerName, nodeIP, int(svc.Spec.Ports[0].NodePort))
externalContainer1IP, nodeIP, svc.Spec.Ports[0].NodePort))
checkExternalContainerConnectivity(externalContainerName1, nodeIP, int(svc.Spec.Ports[0].NodePort))
})
})

0 comments on commit 2372a08

Please sign in to comment.