Skip to content

Commit

Permalink
fix tackle GCE ingress incompatibility, kubernetes#13492
Browse files Browse the repository at this point in the history
  • Loading branch information
Travis Clarke committed Jul 25, 2019
1 parent ab96149 commit 85338af
Show file tree
Hide file tree
Showing 4 changed files with 248 additions and 9 deletions.
25 changes: 23 additions & 2 deletions prow/cmd/tackle/BUILD.bazel
Original file line number Diff line number Diff line change
@@ -1,8 +1,11 @@
load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test")

go_library(
name = "go_default_library",
srcs = ["main.go"],
srcs = [
"ingress.go",
"main.go",
],
importpath = "k8s.io/test-infra/prow/cmd/tackle",
visibility = ["//visibility:public"],
deps = [
Expand All @@ -11,8 +14,12 @@ go_library(
"//prow/github:go_default_library",
"//vendor/github.com/sirupsen/logrus:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/api/extensions/v1beta1:go_default_library",
"//vendor/k8s.io/api/networking/v1beta1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/version:go_default_library",
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
"//vendor/k8s.io/client-go/plugin/pkg/client/auth/gcp:go_default_library",
"//vendor/k8s.io/client-go/rest:go_default_library",
Expand All @@ -21,6 +28,20 @@ go_library(
],
)

go_test(
name = "go_default_test",
srcs = ["ingress_test.go"],
embed = [":go_default_library"],
deps = [
"//vendor/k8s.io/api/extensions/v1beta1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/version:go_default_library",
"//vendor/k8s.io/client-go/discovery/fake:go_default_library",
"//vendor/k8s.io/client-go/kubernetes/fake:go_default_library",
],
)

filegroup(
name = "package-srcs",
srcs = glob(["**"]),
Expand Down
81 changes: 81 additions & 0 deletions prow/cmd/tackle/ingress.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
"github.com/sirupsen/logrus"
extensions "k8s.io/api/extensions/v1beta1"
networking "k8s.io/api/networking/v1beta1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/version"
"k8s.io/client-go/kubernetes"
)

// https://github.com/kubernetes/enhancements/blob/master/keps/sig-network/20190125-ingress-api-group.md#116
// NOTE: GCE ingress controller does not target "networking.k8s.io/v1beta1": https://github.com/kubernetes/ingress-gce/issues/770
const networkingIngressKubeVersion = "v1.16.0"

var runtimeScheme = runtime.NewScheme()

func init() {
extensions.AddToScheme(runtimeScheme)
networking.AddToScheme(runtimeScheme)
}

// networkingIngressAvailable checks if the "networking.k8s.io/v1beta1" resource is available based on Kubernetes version.
func networkingIngressAvailable(kc kubernetes.Interface) bool {
serverVersion, err := kc.Discovery().ServerVersion()
if err != nil {
logrus.WithError(err).Fatal("unexpected error parsing Kubernetes version")
return false
}

return version.CompareKubeAwareVersionStrings(serverVersion.String(), networkingIngressKubeVersion) <= 0
}

// fromExtensions converts a legacy "extensions/v1beta1" IngressList to the newer "networking.k8s.io/v1beta1" IngressList.
func fromExtensions(old *extensions.IngressList) (*networking.IngressList, error) {
networkingIngress := &networking.IngressList{}

err := runtimeScheme.Convert(old, networkingIngress, nil)
if err != nil {
return nil, err
}

return networkingIngress, nil
}

// toIngress normalizes a given IngressList to the newer "networking.k8s.io/v1beta1" type.
func toIngress(obj interface{}) (*networking.IngressList, bool) {
oldVersion, inExtension := obj.(*extensions.IngressList)

if inExtension {
ing, err := fromExtensions(oldVersion)
if err != nil {
logrus.WithError(err).Fatal("unexpected error converting Ingress from extensions package")
return nil, false
}

return ing, true
}

if ing, ok := obj.(*networking.IngressList); ok {
return ing, true
}

return nil, false
}
123 changes: 123 additions & 0 deletions prow/cmd/tackle/ingress_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
"bytes"
extensions "k8s.io/api/extensions/v1beta1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/apimachinery/pkg/version"
discoveryFake "k8s.io/client-go/discovery/fake"
k8sFake "k8s.io/client-go/kubernetes/fake"
"testing"
"time"
)

func TestNetworkingIngressDetectLess(t *testing.T) {
fakeClient := k8sFake.NewSimpleClientset()

fakeClient.Discovery().(*discoveryFake.FakeDiscovery).FakedServerVersion = &version.Info{
GitVersion: "v1.10",
}

actual := networkingIngressAvailable(fakeClient)

if actual != false {
t.Fatalf("networkingIngressAvailable should return `false` for versions < %s", networkingIngressKubeVersion)
}
}

func TestNetworkingIngressDetectEqual(t *testing.T) {
fakeClient := k8sFake.NewSimpleClientset()

fakeClient.Discovery().(*discoveryFake.FakeDiscovery).FakedServerVersion = &version.Info{
GitVersion: networkingIngressKubeVersion,
}

actual := networkingIngressAvailable(fakeClient)

if actual != true {
t.Fatalf("networkingIngressAvailable should return `true` for versions >= %s", networkingIngressKubeVersion)
}
}

func TestNetworkingIngressDetectGreater(t *testing.T) {
fakeClient := k8sFake.NewSimpleClientset()

fakeClient.Discovery().(*discoveryFake.FakeDiscovery).FakedServerVersion = &version.Info{
GitVersion: "v1.20",
}

actual := networkingIngressAvailable(fakeClient)

if actual != true {
t.Fatalf("networkingIngressAvailable should return `true` for versions >= %s", networkingIngressKubeVersion)
}
}

func TestIngressConversion(t *testing.T) {
ings := &extensions.IngressList{
Items: []extensions.Ingress{
{
ObjectMeta: metav1.ObjectMeta{
Name: "old-ingress",
Namespace: "demo",
CreationTimestamp: metav1.NewTime(time.Now()),
},
Spec: extensions.IngressSpec{
Rules: []extensions.IngressRule{
{
Host: "foo.bar",
IngressRuleValue: extensions.IngressRuleValue{
HTTP: &extensions.HTTPIngressRuleValue{
Paths: []extensions.HTTPIngressPath{
{
Backend: extensions.IngressBackend{
ServiceName: "demo",
ServicePort: intstr.FromInt(80),
},
},
},
},
},
},
},
},
},
},
}

new, err := fromExtensions(ings)
if err != nil {
t.Fatalf("unexpected error converting ingress: %v", err)
}

m1, err := new.Marshal()
if err != nil {
t.Fatalf("unexpected error marshalling Ingress: %v", err)
}

m2, err := ings.Marshal()
if err != nil {
t.Fatalf("unexpected error marshalling Ingress: %v", err)
}

if bytes.Compare(m1, m2) != 0 {
t.Fatalf("Expected marshalling of types should be equal")
}
}
28 changes: 21 additions & 7 deletions prow/cmd/tackle/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -574,6 +574,8 @@ func applySecret(ctx, ns, name, key, path string) error {
}

func applyStarter(kc *kubernetes.Clientset, ns, choice, ctx string, overwrite bool) error {
const defaultStarter = "https://raw.githubusercontent.com/kubernetes/test-infra/master/prow/cluster/starter.yaml"

if !strings.HasPrefix(ctx, "gke_") {
// TODO(fejta): maybe do this for us
fmt.Printf("Warning: if %s is not on GKE, you may need to change\n", ctx)
Expand All @@ -583,7 +585,7 @@ func applyStarter(kc *kubernetes.Clientset, ns, choice, ctx string, overwrite bo
choice = prompt("Apply starter.yaml from", "github upstream")
}
if choice == "" || choice == "github" || choice == "upstream" || choice == "github upstream" {
choice = "https://raw.githubusercontent.com/kubernetes/test-infra/master/prow/cluster/starter.yaml"
choice = defaultStarter
fmt.Println("Loading from", choice)
}
_, err := kc.AppsV1().Deployments(ns).Get("plank", metav1.GetOptions{})
Expand All @@ -593,8 +595,8 @@ func applyStarter(kc *kubernetes.Clientset, ns, choice, ctx string, overwrite bo
case err != nil: // unexpected error
return fmt.Errorf("get plank: %v", err)
case !overwrite: // already a plank, confirm overwrite
choice = prompt(fmt.Sprintf("Prow is already deployed to %s in %s, overwrite?", ns, ctx), "no")
switch choice {
overwriteChoice := prompt(fmt.Sprintf("Prow is already deployed to %s in %s, overwrite?", ns, ctx), "no")
switch overwriteChoice {
case "y", "Y", "yes":
// carry on, then
default:
Expand Down Expand Up @@ -626,15 +628,27 @@ func clientConfig(context string) (*rest.Config, error) {
}

func ingress(kc *kubernetes.Clientset, ns, service string) (url.URL, error) {
var obj interface{}
var err error

for {
ings, err := kc.NetworkingV1beta1().Ingresses(ns).List(metav1.ListOptions{})

// Detect ingress API to use based on Kubernetes version
if networkingIngressAvailable(kc) {
obj, err = kc.NetworkingV1beta1().Ingresses(ns).List(metav1.ListOptions{})
} else {
obj, err = kc.ExtensionsV1beta1().Ingresses(ns).List(metav1.ListOptions{})
}

if err != nil {
logrus.WithError(err).Fatal("Could not get ingresses")
time.Sleep(5 * time.Second)
logrus.WithError(err).Fatalf("Could not get ingresses for service: %s", service)
}

ingressList, _ := toIngress(obj)

var best url.URL
points := 0
for _, ing := range ings.Items {
for _, ing := range ingressList.Items {
// does this ingress route to the hook service?
cur := -1
var maybe url.URL
Expand Down

0 comments on commit 85338af

Please sign in to comment.