Allow additional CIDRs to be excluded from SNAT

Introduces a mechanism to provide an exclusion list of CIDRs that should
not be SNATed.

Adds an environment variable `AWS_VPC_K8S_CNI_EXCLUDE_SNAT_CIDRS` to
provide a comma separated list of ipv4 CIDRs to exclude from SNAT.
The value of this environment variable will only be use when
`AWS_VPC_K8S_CNI_EXTERNALSNAT` is false.

The SNAT exclusion CIDRs will be used to:
- Generate `iptable` rules to prevent SNATing for packets directed to
the excluded CIDRs
- Generate `ip rule` entries to route packets directed to the excluded
CIDRs through the pod's `eni`. These configuration is done both at the
`cni` plugin level via the `rpc_handler`, and the runtime `network` api.

Given that the list of excluded CIDRs may vary by configuration it was
necessary to include a mechanism to clean up stale SNAT rules. If a
CIDR is removed from the exclusion list, the corresponding `iptable`
rule will be removed as well, and the chain will be adjusted.

Connects #469

Co-authored-by: @rewiko
Co-authored-by: @yorg1st

Add missing README entry

README.md

@@ -151,6 +151,12 @@ Disable (`none`) this functionality if you rely on sequential port allocation fo
 
 *Note*: Any options other than `none` will cause outbound connections to be assigned a source port that's not necessarily part of the ephemeral port range set at the OS level (/proc/sys/net/ipv4/ip_local_port_range). This is relevant for any customers that might have NACLs restricting traffic based on the port range found in ip_local_port_range
 
+`AWS_VPC_K8S_CNI_EXCLUDE_SNAT_CIDRS`
+Type: String
+Default: empty
+Specify a comma separated list of ipv4 CIDRs to exclude from SNAT. For every item in the list an `iptables` rule and off\-VPC
+IP rule will be applied. If an item is not a valid ipv4 range it will be skipped. This should be used when `AWS_VPC_K8S_CNI_EXTERNALSNAT=false`.
+
 `WARM_ENI_TARGET`
 Type: Integer
 Default: `1`

ipamd/rpc_handler.go

@@ -54,12 +54,20 @@ func (s *server) AddNetwork(ctx context.Context, in *pb.AddNetworkRequest) (*pb.
 		pbVPCcidrs = append(pbVPCcidrs, *cidr)
 	}
 
+	useExternalSNAT := s.ipamContext.networkClient.UseExternalSNAT()
+	if !useExternalSNAT {
+		for _, cidr := range s.ipamContext.networkClient.GetExcludeSNATCIDRs() {
+			log.Debugf("CIDR SNAT Exclusion %s", cidr)
+			pbVPCcidrs = append(pbVPCcidrs, cidr)
+		}
+	}
+
 	resp := pb.AddNetworkReply{
 		Success:         err == nil,
 		IPv4Addr:        addr,
 		IPv4Subnet:      "",
 		DeviceNumber:    int32(deviceNumber),
-		UseExternalSNAT: s.ipamContext.networkClient.UseExternalSNAT(),
+		UseExternalSNAT: useExternalSNAT,
 		VPCcidrs:        pbVPCcidrs,
 	}
 

ipamd/rpc_handler_test.go

@@ -0,0 +1,92 @@
+// Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"). You may
+// not use this file except in compliance with the License. A copy of the
+// License is located at
+//
+//     http://aws.amazon.com/apache2.0/
+//
+// or in the "license" file accompanying this file. This file is distributed
+// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+// express or implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package ipamd
+
+import (
+	"context"
+	"github.com/aws/amazon-vpc-cni-k8s/ipamd/datastore"
+	"github.com/aws/aws-sdk-go/aws"
+	"testing"
+
+	pb "github.com/aws/amazon-vpc-cni-k8s/rpc"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestServer_AddNetwork(t *testing.T) {
+	ctrl, mockAWS, mockK8S, mockDocker, mockNetwork, _ := setup(t)
+	defer ctrl.Finish()
+
+	mockContext := &IPAMContext{
+		awsClient:     mockAWS,
+		k8sClient:     mockK8S,
+		maxIPsPerENI:  14,
+		maxENI:        4,
+		warmENITarget: 1,
+		warmIPTarget:  3,
+		dockerClient:  mockDocker,
+		networkClient: mockNetwork,
+		dataStore:     datastore.NewDataStore(),
+	}
+
+	rpcServer := server{ipamContext: mockContext}
+
+	addNetworkRequest := &pb.AddNetworkRequest{
+		Netns:                      "netns",
+		K8S_POD_NAME:               "pod",
+		K8S_POD_NAMESPACE:          "ns",
+		K8S_POD_INFRA_CONTAINER_ID: "cid",
+		IfName:                     "eni",
+	}
+
+	vpcCIDRs := []*string{aws.String(vpcCIDR)}
+	testCases := []struct {
+		name               string
+		useExternalSNAT    bool
+		vpcCIDRs           []*string
+		snatExclusionCIDRs []string
+	}{
+		{
+			"VPC CIDRs",
+			true,
+			vpcCIDRs,
+			nil,
+		},
+		{
+			"SNAT Exclusion CIDRs",
+			false,
+			vpcCIDRs,
+			[]string{"10.12.0.0/16", "10.13.0.0/16"},
+		},
+	}
+	for _, tc := range testCases {
+		mockAWS.EXPECT().GetVPCIPv4CIDRs().Return(tc.vpcCIDRs)
+		mockNetwork.EXPECT().UseExternalSNAT().Return(tc.useExternalSNAT)
+		if !tc.useExternalSNAT {
+			mockNetwork.EXPECT().GetExcludeSNATCIDRs().Return(tc.snatExclusionCIDRs)
+		}
+
+		addNetworkReply, err := rpcServer.AddNetwork(context.TODO(), addNetworkRequest)
+		assert.NoError(t, err, tc.name)
+
+		assert.Equal(t, tc.useExternalSNAT, addNetworkReply.UseExternalSNAT, tc.name)
+
+		var expectedCIDRs []string
+		for _, cidr := range tc.vpcCIDRs {
+			expectedCIDRs = append(expectedCIDRs, *cidr)
+		}
+		expectedCIDRs = append([]string{vpcCIDR}, tc.snatExclusionCIDRs...)
+		assert.Equal(t, expectedCIDRs, addNetworkReply.VPCcidrs, tc.name)
+	}
+}