diff --git a/go.mod b/go.mod index 128348a57..b0231b7e1 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,6 @@ require ( github.com/onsi/gomega v1.12.0 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.10.0 - github.com/rdegges/go-ipify v0.0.0-20150526035502-2d94a6a86c40 github.com/submariner-io/admiral v0.9.0-rc0.0.20210506031438-f6fdcbce358a github.com/submariner-io/shipyard v0.9.1-0.20210506024409-3beff067454a github.com/vishvananda/netlink v1.1.0 diff --git a/go.sum b/go.sum index b102adc8f..4e4cfe955 100644 --- a/go.sum +++ b/go.sum @@ -746,8 +746,6 @@ github.com/prometheus/tsdb v0.7.1 h1:YZcsG11NqnK4czYLrWd9mpEuAJIHVQLwdrleYfszMAA github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a h1:9ZKAASQSHhDYGoxY8uLVpewe1GDZ2vu2Tr/vTdVAkFQ= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= -github.com/rdegges/go-ipify v0.0.0-20150526035502-2d94a6a86c40 h1:31Y7UZ1yTYBU4E79CE52I/1IRi3TqiuwquXGNtZDXWs= -github.com/rdegges/go-ipify v0.0.0-20150526035502-2d94a6a86c40/go.mod h1:j4c6zEU0eMG1oiZPUy+zD4ykX0NIpjZAEOEAviTWC18= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af h1:gu+uRPtBe88sKxUCEXRoeCvVG90TJmwhiqRpvdhQFng= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4 h1:BN/Nyn2nWMoqGRA7G7paDNDqTXE30mXGqzzybrfo05w= diff --git a/main.go b/main.go index 46a00ce2c..ab74b73ae 100644 --- a/main.go +++ b/main.go @@ -54,9 +54,7 @@ import ( "github.com/submariner-io/submariner/pkg/controllers/tunnel" "github.com/submariner-io/submariner/pkg/endpoint" "github.com/submariner-io/submariner/pkg/natdiscovery" - "github.com/submariner-io/submariner/pkg/node" "github.com/submariner-io/submariner/pkg/types" - "github.com/submariner-io/submariner/pkg/util" ) var ( @@ -112,9 +110,9 @@ func main() { klog.Fatalf("Error creating submariner clientset: %s", err.Error()) } - localNode, err := node.GetLocalNode(cfg) + k8sClient, err := kubernetes.NewForConfig(cfg) if err != nil { - klog.Fatalf("Error getting information on the local node: %s", err.Error()) + klog.Fatalf("Error creating Kubernetes clientset: %s", err.Error()) } klog.Info("Creating the cable engine") @@ -127,7 +125,7 @@ func main() { submSpec.CableDriver = strings.ToLower(submSpec.CableDriver) - localEndpoint, err := endpoint.GetLocal(submSpec, util.GetLocalIP(), localNode) + localEndpoint, err := endpoint.GetLocal(submSpec, k8sClient) if err != nil { klog.Fatalf("Error creating local endpoint object from %#v: %v", submSpec, err) diff --git a/pkg/apis/submariner.io/v1/types.go b/pkg/apis/submariner.io/v1/types.go index 833b912eb..4c402273d 100644 --- a/pkg/apis/submariner.io/v1/types.go +++ b/pkg/apis/submariner.io/v1/types.go @@ -87,12 +87,21 @@ const ( GatewayConfigLabelPrefix = "gateway.submariner.io/" UDPPortConfig = "udp-port" NATTDiscoveryPortConfig = "natt-discovery-port" + PublicIP = "public-ip" ) const ( DefaultNATTDiscoveryPort = "4490" ) +// Valid PublicIP resolvers. +const ( + IPv4 = "ipv4" // ipv4:1.2.3.4 + LoadBalancer = "lb" // lb:external-gw-lb + API = "api" // api:api.ipify.org + DNS = "dns" // dns:mygateway.dns.name.com +) + // BackendConfig entries which aren't configured via labels, but exposed from the endpoints const ( PreferredServerConfig = "preferred-server" @@ -102,6 +111,7 @@ const ( var ValidGatewayNodeConfig = []string{ UDPPortConfig, NATTDiscoveryPortConfig, + PublicIP, } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/pkg/endpoint/local_endpoint.go b/pkg/endpoint/local_endpoint.go index b6987b1f1..452c45365 100644 --- a/pkg/endpoint/local_endpoint.go +++ b/pkg/endpoint/local_endpoint.go @@ -26,17 +26,26 @@ import ( "time" "github.com/pkg/errors" - "github.com/rdegges/go-ipify" "github.com/submariner-io/admiral/pkg/log" "github.com/submariner-io/admiral/pkg/stringset" + "github.com/submariner-io/submariner/pkg/node" + "github.com/submariner-io/submariner/pkg/util" v1 "k8s.io/api/core/v1" + "k8s.io/client-go/kubernetes" "k8s.io/klog" submv1 "github.com/submariner-io/submariner/pkg/apis/submariner.io/v1" "github.com/submariner-io/submariner/pkg/types" ) -func GetLocal(submSpec types.SubmarinerSpecification, privateIP string, gwNode *v1.Node) (types.SubmarinerEndpoint, error) { +func GetLocal(submSpec types.SubmarinerSpecification, k8sClient kubernetes.Interface) (types.SubmarinerEndpoint, error) { + privateIP := util.GetLocalIP() + + gwNode, err := node.GetLocalNode(k8sClient) + if err != nil { + klog.Fatalf("Error getting information on the local node: %s", err.Error()) + } + hostname, err := os.Hostname() if err != nil { return types.SubmarinerEndpoint{}, fmt.Errorf("error getting hostname: %v", err) @@ -72,7 +81,7 @@ func GetLocal(submSpec types.SubmarinerSpecification, privateIP string, gwNode * }, } - publicIP, err := ipify.GetIp() + publicIP, err := getPublicIP(submSpec, k8sClient, backendConfig) if err != nil { return types.SubmarinerEndpoint{}, fmt.Errorf("could not determine public IP: %v", err) } diff --git a/pkg/endpoint/local_endpoint_test.go b/pkg/endpoint/local_endpoint_test.go index f560ab167..c913dfe71 100644 --- a/pkg/endpoint/local_endpoint_test.go +++ b/pkg/endpoint/local_endpoint_test.go @@ -18,20 +18,30 @@ limitations under the License. package endpoint_test import ( + "os" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + endpoint "github.com/submariner-io/submariner/pkg/endpoint" + + "github.com/submariner-io/submariner/pkg/util" v1 "k8s.io/api/core/v1" v1meta "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/fake" - "github.com/submariner-io/submariner/pkg/endpoint" "github.com/submariner-io/submariner/pkg/types" ) +const testNodeName = "this-node" + var _ = Describe("GetLocal", func() { var submSpec types.SubmarinerSpecification + var client kubernetes.Interface + var testPrivateIP = util.GetLocalIP() var node *v1.Node + const ( - testPrivateIP = "127.0.0.1" testUDPPort = "1111" testUDPPortLabel = "udp-port" testNATTPortLabel = "natt-discovery-port" @@ -49,14 +59,19 @@ var _ = Describe("GetLocal", func() { node = &v1.Node{ ObjectMeta: v1meta.ObjectMeta{ + Name: testNodeName, Labels: map[string]string{ backendConfigPrefix + testNATTPortLabel: "1234", backendConfigPrefix + testUDPPortLabel: testUDPPort, }}} + + client = fake.NewSimpleClientset(node) + + os.Setenv("NODE_NAME", testNodeName) }) It("should return a valid SubmarinerEndpoint object", func() { - endpoint, err := endpoint.GetLocal(submSpec, testPrivateIP, node) + endpoint, err := endpoint.GetLocal(submSpec, client) Expect(err).ToNot(HaveOccurred()) Expect(endpoint.Spec.ClusterID).To(Equal("east")) @@ -72,7 +87,7 @@ var _ = Describe("GetLocal", func() { When("no NAT discovery port label is set on the node", func() { It("should return a valid SubmarinerEndpoint object", func() { delete(node.Labels, testNATTPortLabel) - _, err := endpoint.GetLocal(submSpec, testPrivateIP, node) + _, err := endpoint.GetLocal(submSpec, client) Expect(err).ToNot(HaveOccurred()) }) }) diff --git a/pkg/endpoint/public_ip.go b/pkg/endpoint/public_ip.go new file mode 100644 index 000000000..e37aa9d79 --- /dev/null +++ b/pkg/endpoint/public_ip.go @@ -0,0 +1,149 @@ +/* +SPDX-License-Identifier: Apache-2.0 + +Copyright Contributors to the Submariner project. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package endpoint + +import ( + "context" + "io/ioutil" + "net" + "net/http" + "regexp" + "strings" + + "github.com/pkg/errors" + v1 "github.com/submariner-io/submariner/pkg/apis/submariner.io/v1" + "github.com/submariner-io/submariner/pkg/types" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/klog" +) + +type publicIPResolverFunction func(clientset kubernetes.Interface, namespace, value string) (string, error) + +var publicIPMethods = map[string]publicIPResolverFunction{ + v1.API: publicAPI, + v1.IPv4: publicIP, + v1.LoadBalancer: publicLoadBalancerIP, + v1.DNS: publicDNSIP, +} + +var IPv4RE = regexp.MustCompile(`(?:\d{1,3}\.){3}\d{1,3}`) + +func getPublicIP(submSpec types.SubmarinerSpecification, k8sClient kubernetes.Interface, backendConfig map[string]string) (string, error) { + config, ok := backendConfig[v1.PublicIP] + if !ok { + config = "api:api.ipify.org,api:api.my-ip.io/ip,api:ip4.seeip.org" + } + + resolvers := strings.Split(config, ",") + for _, resolver := range resolvers { + resolver = strings.Trim(resolver, " ") + parts := strings.Split(resolver, ":") + if len(parts) != 2 { + return "", errors.Errorf("invalid format for %q label: %q", v1.GatewayConfigLabelPrefix+v1.PublicIP, config) + } + + method, ok := publicIPMethods[parts[0]] + if !ok { + return "", errors.Errorf("unknown resolver %q in %q label: %q", parts[0], v1.GatewayConfigLabelPrefix+v1.PublicIP, config) + } + + ip, err := method(k8sClient, submSpec.Namespace, parts[1]) + if err == nil { + return ip, nil + } + + // If this resolved failed we log it, but we fall back to the next one + klog.Errorf("Error resolving public IP with resolver %s : %s", resolver, err.Error()) + } + + if len(resolvers) > 0 { + return "", errors.Errorf("Unable to resolve public IP by any of the resolver methods: %s", resolvers) + } + + return "", nil +} + +func publicAPI(clientset kubernetes.Interface, namespace, value string) (string, error) { + url := "https://" + value + + //nolint:gosec // we really need to get from a non-predefined const URL + response, err := http.Get(url) + if err != nil { + return "", errors.Wrapf(err, "retrieving public IP from %s", url) + } + + defer response.Body.Close() + + body, err := ioutil.ReadAll(response.Body) + if err != nil { + return "", errors.Wrapf(err, "reading API response from %s", url) + } + + return firstIPv4InString(string(body)) +} + +func publicIP(clientset kubernetes.Interface, namespace, value string) (string, error) { + return firstIPv4InString(value) +} + +func publicLoadBalancerIP(clientset kubernetes.Interface, namespace, loadBalancerName string) (string, error) { + service, err := clientset.CoreV1().Services(namespace).Get(context.TODO(), loadBalancerName, metav1.GetOptions{}) + if err != nil { + return "", errors.Wrapf(err, "error getting service %q for the public IP address", loadBalancerName) + } + + if len(service.Status.LoadBalancer.Ingress) < 1 { + return "", errors.Errorf("service %q doesn't contain any LoadBalancer ingress", loadBalancerName) + } + + ingress := service.Status.LoadBalancer.Ingress[0] + if ingress.IP != "" { + return ingress.IP, nil + } + + if ingress.Hostname != "" { + ip, err := publicDNSIP(clientset, namespace, ingress.Hostname) + if err != nil { + return "", errors.Wrapf(err, "error resolving LoadBalancer ingress HostName %q", ingress.Hostname) + } + + return ip, nil + } + + return "", errors.Errorf("no IP or Hostname for service LoadBalancer %q Ingress", loadBalancerName) +} + +func publicDNSIP(clientset kubernetes.Interface, namespace, fqdn string) (string, error) { + ips, err := net.LookupIP(fqdn) + if err != nil { + return "", errors.Wrapf(err, "error resolving DNS hostname %q for public IP", fqdn) + } + + return ips[0].String(), nil +} + +func firstIPv4InString(body string) (string, error) { + matches := IPv4RE.FindAllString(body, -1) + if len(matches) == 0 { + return "", errors.Errorf("No IPv4 found in: %q", body) + } + + return matches[0], nil +} diff --git a/pkg/endpoint/public_ip_test.go b/pkg/endpoint/public_ip_test.go new file mode 100644 index 000000000..1af4e204a --- /dev/null +++ b/pkg/endpoint/public_ip_test.go @@ -0,0 +1,169 @@ +/* +SPDX-License-Identifier: Apache-2.0 + +Copyright Contributors to the Submariner project. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package endpoint + +import ( + "net" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" + v1meta "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" + + "github.com/submariner-io/submariner/pkg/types" +) + +var _ = Describe("firstIPv4InString", func() { + When("the content has an IPv4", func() { + const testIP = "1.2.3.4" + const jsonIP = "{\"ip\": \"" + testIP + "\"}" + + It("should return the IP", func() { + ip, err := firstIPv4InString(jsonIP) + Expect(err).ToNot(HaveOccurred()) + Expect(ip).To(Equal(testIP)) + }) + }) + + When("the content doesn't have an IPv4", func() { + It("should result in error", func() { + ip, err := firstIPv4InString("no IPs here") + Expect(err).To(HaveOccurred()) + Expect(ip).To(Equal("")) + }) + }) +}) + +const ( + testServiceName = "my-loadbalancer" + testNamespace = "namespace" +) + +var _ = Describe("public ip resolvers", func() { + var submSpec types.SubmarinerSpecification + var backendConfig map[string]string + + const ( + publicIPConfig = "public-ip" + testIPDNS = "4.3.2.1" + testIP = "1.2.3.4" + ) + + BeforeEach(func() { + submSpec = types.SubmarinerSpecification{ + Namespace: testNamespace, + } + + backendConfig = map[string]string{} + }) + + When("a LoadBalancer with Ingress IP is specified", func() { + It("should return the IP", func() { + backendConfig[publicIPConfig] = "lb:" + testServiceName + client := fake.NewSimpleClientset(serviceWithIngress(v1.LoadBalancerIngress{Hostname: "", IP: testIP})) + ip, err := getPublicIP(submSpec, client, backendConfig) + Expect(err).ToNot(HaveOccurred()) + Expect(ip).To(Equal(testIP)) + }) + }) + + When("a LoadBalancer with Ingress hostname is specified", func() { + It("should return the IP", func() { + backendConfig[publicIPConfig] = "lb:" + testServiceName + client := fake.NewSimpleClientset(serviceWithIngress(v1.LoadBalancerIngress{Hostname: testIPDNS + ".nip.io", + IP: ""})) + ip, err := getPublicIP(submSpec, client, backendConfig) + Expect(err).ToNot(HaveOccurred()) + Expect(ip).To(Equal(testIPDNS)) + }) + }) + + When("a LoadBalancer with no ingress is specified", func() { + It("should return error", func() { + backendConfig[publicIPConfig] = "lb:" + testServiceName + client := fake.NewSimpleClientset(serviceWithIngress()) + _, err := getPublicIP(submSpec, client, backendConfig) + Expect(err).To(HaveOccurred()) + }) + }) + + When("an IPv4 entry specified", func() { + It("should return the IP", func() { + backendConfig[publicIPConfig] = "ipv4:" + testIP + client := fake.NewSimpleClientset() + ip, err := getPublicIP(submSpec, client, backendConfig) + Expect(err).ToNot(HaveOccurred()) + Expect(ip).To(Equal(testIP)) + }) + }) + + When("a DNS entry specified", func() { + It("should return the IP", func() { + backendConfig[publicIPConfig] = "dns:" + testIPDNS + ".nip.io" + client := fake.NewSimpleClientset() + ip, err := getPublicIP(submSpec, client, backendConfig) + Expect(err).ToNot(HaveOccurred()) + Expect(ip).To(Equal(testIPDNS)) + }) + }) + + When("an API entry specified", func() { + It("should return some IP", func() { + backendConfig[publicIPConfig] = "api:api.ipify.org" + client := fake.NewSimpleClientset() + ip, err := getPublicIP(submSpec, client, backendConfig) + Expect(err).ToNot(HaveOccurred()) + Expect(net.ParseIP(ip)).NotTo(BeNil()) + }) + }) + + When("multiple entries are specified", func() { + It("should return the first working one", func() { + backendConfig[publicIPConfig] = "ipv4:" + testIP + ",dns:" + testIPDNS + ".nip.io" + client := fake.NewSimpleClientset() + ip, err := getPublicIP(submSpec, client, backendConfig) + Expect(err).ToNot(HaveOccurred()) + Expect(ip).To(Equal(testIP)) + }) + }) + + When("multiple entries are specified and the first one doesn't succeed", func() { + It("should return the first working one", func() { + backendConfig[publicIPConfig] = "dns:thisdomaindoesntexistforsure.badbadbad,ipv4:" + testIP + client := fake.NewSimpleClientset() + ip, err := getPublicIP(submSpec, client, backendConfig) + Expect(err).ToNot(HaveOccurred()) + Expect(ip).To(Equal(testIP)) + }) + }) +}) + +func serviceWithIngress(ingress ...v1.LoadBalancerIngress) *v1.Service { + return &v1.Service{ + ObjectMeta: v1meta.ObjectMeta{ + Name: testServiceName, + Namespace: testNamespace, + }, + Status: v1.ServiceStatus{ + LoadBalancer: v1.LoadBalancerStatus{ + Ingress: ingress, + }, + }, + } +} diff --git a/pkg/node/node.go b/pkg/node/node.go index 565a3fd6f..0397432a1 100644 --- a/pkg/node/node.go +++ b/pkg/node/node.go @@ -26,20 +26,14 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" ) -func GetLocalNode(cfg *rest.Config) (*v1.Node, error) { +func GetLocalNode(clientset kubernetes.Interface) (*v1.Node, error) { nodeName, ok := os.LookupEnv("NODE_NAME") if !ok { return nil, errors.New("error reading the NODE_NAME from the environment") } - clientset, err := kubernetes.NewForConfig(cfg) - if err != nil { - return nil, errors.Wrapf(err, "creating Kubernetes clientset") - } - node, err := clientset.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{}) if err != nil { return nil, errors.Wrapf(err, "unable to find local node %q", nodeName)