Skip to content

Commit

Permalink
Implement multiple public IP resolvers
Browse files Browse the repository at this point in the history
The gateway nodes now can be labeled as:

gateway.submariner.io/public-ip=<resolver>[,resolver..]
where resolvers take the form of <method>:<parameter>

the implemented methods are:

- api
- lb
- ipv4
- dns

Signed-off-by: Miguel Angel Ajo <[email protected]>
  • Loading branch information
mangelajo committed May 20, 2021
1 parent 23b6bef commit 0364f74
Show file tree
Hide file tree
Showing 8 changed files with 345 additions and 21 deletions.
8 changes: 3 additions & 5 deletions main.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,9 +52,7 @@ import (
"github.com/submariner-io/submariner/pkg/controllers/tunnel"
"github.com/submariner-io/submariner/pkg/endpoint"
"github.com/submariner-io/submariner/pkg/natdiscovery"
"github.com/submariner-io/submariner/pkg/node"
"github.com/submariner-io/submariner/pkg/types"
"github.com/submariner-io/submariner/pkg/util"
)

var (
Expand Down Expand Up @@ -110,9 +108,9 @@ func main() {
klog.Fatalf("Error creating submariner clientset: %s", err.Error())
}

localNode, err := node.GetLocalNode(cfg)
k8sClient, err := kubernetes.NewForConfig(cfg)
if err != nil {
klog.Fatalf("Error getting information on the local node: %s", err.Error())
klog.Fatalf("Error creating Kubernetes clientset: %s", err.Error())
}

klog.Info("Creating the cable engine")
Expand All @@ -125,7 +123,7 @@ func main() {

submSpec.CableDriver = strings.ToLower(submSpec.CableDriver)

localEndpoint, err := endpoint.GetLocal(submSpec, util.GetLocalIP(), localNode)
localEndpoint, err := endpoint.GetLocal(submSpec, k8sClient)

if err != nil {
klog.Fatalf("Error creating local endpoint object from %#v: %v", submSpec, err)
Expand Down
10 changes: 10 additions & 0 deletions pkg/apis/submariner.io/v1/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -84,12 +84,21 @@ const (
GatewayConfigLabelPrefix = "gateway.submariner.io/"
UDPPortConfig = "udp-port"
NATTDiscoveryPortConfig = "natt-discovery-port"
PublicIP = "public-ip"
)

const (
DefaultNATTDiscoveryPort = "4490"
)

// Valid PublicIP resolvers.
const (
IPv4 = "ipv4" // ipv4:1.2.3.4
LoadBalancer = "lb" // lb:external-gw-lb
API = "api" // api:api.ipify.org
DNS = "dns" // dns:mygateway.dns.name.com
)

// BackendConfig entries which aren't configured via labels, but exposed from the endpoints
const (
PreferredServerConfig = "preferred-server"
Expand All @@ -99,6 +108,7 @@ const (
var ValidGatewayNodeConfig = []string{
UDPPortConfig,
NATTDiscoveryPortConfig,
PublicIP,
}

// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
Expand Down
15 changes: 12 additions & 3 deletions pkg/endpoint/local_endpoint.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,17 +24,26 @@ import (
"time"

"github.com/pkg/errors"
"github.com/rdegges/go-ipify"
"github.com/submariner-io/admiral/pkg/log"
"github.com/submariner-io/admiral/pkg/stringset"
"github.com/submariner-io/submariner/pkg/node"
"github.com/submariner-io/submariner/pkg/util"
v1 "k8s.io/api/core/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/klog"

submv1 "github.com/submariner-io/submariner/pkg/apis/submariner.io/v1"
"github.com/submariner-io/submariner/pkg/types"
)

func GetLocal(submSpec types.SubmarinerSpecification, privateIP string, gwNode *v1.Node) (types.SubmarinerEndpoint, error) {
func GetLocal(submSpec types.SubmarinerSpecification, k8sClient kubernetes.Interface) (types.SubmarinerEndpoint, error) {
privateIP := util.GetLocalIP()

gwNode, err := node.GetLocalNode(k8sClient)
if err != nil {
klog.Fatalf("Error getting information on the local node: %s", err.Error())
}

hostname, err := os.Hostname()
if err != nil {
return types.SubmarinerEndpoint{}, fmt.Errorf("error getting hostname: %v", err)
Expand Down Expand Up @@ -70,7 +79,7 @@ func GetLocal(submSpec types.SubmarinerSpecification, privateIP string, gwNode *
},
}

publicIP, err := ipify.GetIp()
publicIP, err := getPublicIP(submSpec, k8sClient, backendConfig)
if err != nil {
return types.SubmarinerEndpoint{}, fmt.Errorf("could not determine public IP: %v", err)
}
Expand Down
2 changes: 1 addition & 1 deletion pkg/endpoint/local_endpoint_suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package endpoint_test
package endpoint

import (
"testing"
Expand Down
24 changes: 19 additions & 5 deletions pkg/endpoint/local_endpoint_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,23 +13,32 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package endpoint_test
package endpoint

import (
"os"

. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"

"github.com/submariner-io/submariner/pkg/util"
v1 "k8s.io/api/core/v1"
v1meta "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/kubernetes/fake"

"github.com/submariner-io/submariner/pkg/endpoint"
"github.com/submariner-io/submariner/pkg/types"
)

const testNodeName = "this-node"

var _ = Describe("GetLocal", func() {
var submSpec types.SubmarinerSpecification
var client kubernetes.Interface
var testPrivateIP = util.GetLocalIP()
var node *v1.Node

const (
testPrivateIP = "127.0.0.1"
testUDPPort = "1111"
testUDPPortLabel = "udp-port"
testNATTPortLabel = "natt-discovery-port"
Expand All @@ -47,14 +56,19 @@ var _ = Describe("GetLocal", func() {

node = &v1.Node{
ObjectMeta: v1meta.ObjectMeta{
Name: testNodeName,
Labels: map[string]string{
backendConfigPrefix + testNATTPortLabel: "1234",
backendConfigPrefix + testUDPPortLabel: testUDPPort,
}}}

client = fake.NewSimpleClientset(node)

os.Setenv("NODE_NAME", testNodeName)
})

It("should return a valid SubmarinerEndpoint object", func() {
endpoint, err := endpoint.GetLocal(submSpec, testPrivateIP, node)
endpoint, err := GetLocal(submSpec, client)

Expect(err).ToNot(HaveOccurred())
Expect(endpoint.Spec.ClusterID).To(Equal("east"))
Expand All @@ -70,7 +84,7 @@ var _ = Describe("GetLocal", func() {
When("no NAT discovery port label is set on the node", func() {
It("should return a valid SubmarinerEndpoint object", func() {
delete(node.Labels, testNATTPortLabel)
_, err := endpoint.GetLocal(submSpec, testPrivateIP, node)
_, err := GetLocal(submSpec, client)
Expect(err).ToNot(HaveOccurred())
})
})
Expand Down
131 changes: 131 additions & 0 deletions pkg/endpoint/public_ip.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,131 @@
package endpoint

import (
"context"
"io/ioutil"
"net"
"net/http"
"regexp"
"strings"

"github.com/pkg/errors"
v1 "github.com/submariner-io/submariner/pkg/apis/submariner.io/v1"
"github.com/submariner-io/submariner/pkg/types"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/klog"
)

type publicIPResolverFunction func(clientset kubernetes.Interface, namespace, value string) (string, error)

var publicIPMethods = map[string]publicIPResolverFunction{
v1.API: publicAPI,
v1.IPv4: publicIP,
v1.LoadBalancer: publicLoadBalancerIP,
v1.DNS: publicDNSIP,
}

var IPv4RE = regexp.MustCompile(`(?:\d{1,3}\.){3}\d{1,3}`)

func getPublicIP(submSpec types.SubmarinerSpecification, k8sClient kubernetes.Interface, backendConfig map[string]string) (string, error) {
config, ok := backendConfig[v1.PublicIP]
if !ok {
config = "api:api.ipify.org"
}

resolvers := strings.Split(config, ",")
for _, resolver := range resolvers {
resolver = strings.Trim(resolver, " ")
parts := strings.Split(resolver, ":")
if len(parts) != 2 {
return "", errors.Errorf("invalid format for %q label: %q", v1.GatewayConfigLabelPrefix+v1.PublicIP, config)
}

method, ok := publicIPMethods[parts[0]]
if !ok {
return "", errors.Errorf("unknown resolver %q in %q label: %q", parts[0], v1.GatewayConfigLabelPrefix+v1.PublicIP, config)
}

ip, err := method(k8sClient, submSpec.Namespace, parts[1])
if err == nil {
return ip, nil
}

// If this resolved failed we log it, but we fall back to the next one
klog.Errorf("Error resolving public IP with resolver %s : %s", resolver, err.Error())
}

if len(resolvers) > 0 {
return "", errors.Errorf("Unable to resolve public IP by any of the resolver methods: %s", resolvers)
}

return "", nil
}

func publicAPI(clientset kubernetes.Interface, namespace, value string) (string, error) {
url := "https://" + value

//nolint:gosec // we really need to get from a non-predefined const URL
response, err := http.Get(url)
if err != nil {
return "", errors.Wrapf(err, "retrieving public IP from %s", url)
}

defer response.Body.Close()

body, err := ioutil.ReadAll(response.Body)
if err != nil {
return "", errors.Wrapf(err, "reading API response from %s", url)
}

return firstIPv4InString(string(body))
}

func publicIP(clientset kubernetes.Interface, namespace, value string) (string, error) {
return firstIPv4InString(value)
}

func publicLoadBalancerIP(clientset kubernetes.Interface, namespace, loadBalancerName string) (string, error) {
service, err := clientset.CoreV1().Services(namespace).Get(context.TODO(), loadBalancerName, metav1.GetOptions{})
if err != nil {
return "", errors.Wrapf(err, "error getting service %q for the public IP address", loadBalancerName)
}

if len(service.Status.LoadBalancer.Ingress) < 1 {
return "", errors.Errorf("service %q doesn't contain any LoadBalancer ingress", loadBalancerName)
}

ingress := service.Status.LoadBalancer.Ingress[0]
if ingress.IP != "" {
return ingress.IP, nil
}

if ingress.Hostname != "" {
ip, err := publicDNSIP(clientset, namespace, ingress.Hostname)
if err != nil {
return "", errors.Wrapf(err, "error resolving LoadBalancer ingress HostName %q", ingress.Hostname)
}

return ip, nil
}

return "", errors.Errorf("no IP or Hostname for service LoadBalancer %q Ingress", loadBalancerName)
}

func publicDNSIP(clientset kubernetes.Interface, namespace, fqdn string) (string, error) {
ips, err := net.LookupIP(fqdn)
if err != nil {
return "", errors.Wrapf(err, "error resolving DNS hostname %q for public IP", fqdn)
}

return ips[0].String(), nil
}

func firstIPv4InString(body string) (string, error) {
matches := IPv4RE.FindAllString(body, -1)
if len(matches) == 0 {
return "", errors.Errorf("No IPv4 found in: %q", body)
}

return matches[0], nil
}
Loading

0 comments on commit 0364f74

Please sign in to comment.