run presubmits with custom image

add a presubmit job that builds the glbc image from current PR and use it for test in a kubernetes cluster Co-authored-by: Gaurav Ghildiyal <[email protected]>
kubernetes · Mar 23, 2023 · 8529f04 · 8529f04
1 parent 9eaef22
commit 8529f04
Show file tree

Hide file tree

Showing 2 changed files with 105 additions and 23 deletions.
diff --git a/hack/boskos.sh b/hack/boskos.sh
@@ -22,7 +22,7 @@ set -o xtrace
 # acquires a project from boskos
 acquire_project() {
     local project=""
-    local project_type="gce-project"
+    local project_type="ingress-project"
 
     boskos_response=$(curl -X POST "http://boskos.test-pods.svc.cluster.local/acquire?type=${project_type}&state=free&dest=busy&owner=${JOB_NAME}")
 

diff --git a/hack/run-e2e-gce.sh b/hack/run-e2e-gce.sh
@@ -21,7 +21,6 @@ set -o pipefail
 set -o xtrace
 
 export GOPATH="$(go env GOPATH)"
-KUBE_REPO_ROOT=$GOPATH/src/k8s.io/kubernetes
 REPO_ROOT=$(git rev-parse --show-toplevel)
 cd ${REPO_ROOT}
 
@@ -30,10 +29,6 @@ function cleanup {
   if [[ "${CLEANUP_BOSKOS:-}" == "true" ]]; then
     cleanup_boskos
   fi
-  # shellcheck disable=SC2153
-  if [[ "${DELETE_CLUSTER:-}" == "true" ]]; then
-      kubetest2 ${KUBETEST2_ARGS} --down || echo "kubetest2 down failed"
-  fi
 }
 trap cleanup EXIT
 
@@ -58,22 +53,109 @@ if [[ -z "${IMAGE_TAG:-}" ]]; then
   IMAGE_TAG=$(git rev-parse --short HEAD)-$(date +%Y%m%dT%H%M%S)
 fi
 
-export GCE_GLBC_IMAGE=$(go run github.com/google/[email protected] build --tags ${IMAGE_TAG} --base-import-paths --push=true ./cmd/glbc/)
+KO_GLBC_IMAGE=$(go run github.com/google/[email protected] build --tags ${IMAGE_TAG} --base-import-paths --push=true ./cmd/glbc/)
+# remove the sha from the image name, it does not work with gce registry per example
+export GCE_GLBC_IMAGE=$(echo $KO_GLBC_IMAGE | cut -d\@ -f1)
 echo "GCE_GLBC_IMAGE=${GCE_GLBC_IMAGE}"
 
-go install sigs.k8s.io/kubetest2@latest
-go install sigs.k8s.io/kubetest2/kubetest2-gce@latest
-go install sigs.k8s.io/kubetest2/kubetest2-tester-ginkgo@latest
-kubetest2 gce -v 2 \
---repo-root=${KUBE_REPO_ROOT} \
---gcp-project=${GCP_PROJECT} \
---ingress-gce-image=${GCE_GLBC_IMAGE} \
---legacy-mode \
---build \
---up \
---down \
---test=ginkgo \
--- \
---focus-regex='\[Feature:NEG\]|Loadbalancing|LoadBalancers|Ingress' \
---skip-regex='\[Feature:kubemci\]|\[Disruptive\]|\[Feature:IngressScale\]|\[Feature:NetworkPolicy\]' \
---use-built-binaries
+
+export CUSTOM_INGRESS_YAML=$(cat <<EOF
+    apiVersion: v1
+    kind: Pod
+    metadata:
+      name: l7-lb-controller
+      namespace: kube-system
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ""
+        seccomp.security.alpha.kubernetes.io/pod: "docker/default"
+      labels:
+        k8s-app: gcp-lb-controller
+        kubernetes.io/name: "GLBC"
+    spec:
+      priorityClassName: system-cluster-critical
+      securityContext:
+        runAsUser: 0
+      terminationGracePeriodSeconds: 300
+      hostNetwork: true
+      containers:
+      - image: $GCE_GLBC_IMAGE
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 8086
+            scheme: HTTP
+          initialDelaySeconds: 30
+          # healthz reaches out to GCE
+          periodSeconds: 30
+          timeoutSeconds: 15
+          successThreshold: 1
+          failureThreshold: 5
+        name: l7-lb-controller
+        volumeMounts:
+        - mountPath: /etc/gce.conf
+          name: cloudconfig
+          readOnly: true
+        - mountPath: /var/log/glbc.log
+          name: logfile
+          readOnly: false
+        - mountPath: /etc/srv/kubernetes/l7-lb-controller
+          name: srvkube
+          readOnly: true
+        resources:
+          requests:
+            cpu: 10m
+            memory: 50Mi
+        args:
+        - --v=3
+        - --logtostderr=false
+        - --log_file=/var/log/glbc.log
+        - --enable-finalizer-remove
+        - --enable-finalizer-add
+        - --default-backend-service=kube-system/default-http-backend
+        - --kubeconfig=/etc/srv/kubernetes/l7-lb-controller/kubeconfig
+        - --sync-period=600s
+        - --running-in-cluster=false
+        - --config-file-path=/etc/gce.conf
+        - --healthz-port=8086
+        - --gce-ratelimit=ga.Operations.Get,qps,10,100
+        - --gce-ratelimit=alpha.Operations.Get,qps,10,100
+        - --gce-ratelimit=beta.Operations.Get,qps,10,100
+        - --gce-ratelimit=ga.BackendServices.Get,qps,1.8,1
+        - --gce-ratelimit=beta.BackendServices.Get,qps,1.8,1
+        - --gce-ratelimit=ga.HealthChecks.Get,qps,1.8,1
+        - --gce-ratelimit=alpha.HealthChecks.Get,qps,1.8,1
+        - --gce-ratelimit=beta.NetworkEndpointGroups.Get,qps,1.8,1
+        - --gce-ratelimit=beta.NetworkEndpointGroups.AttachNetworkEndpoints,qps,1.8,1
+        - --gce-ratelimit=beta.NetworkEndpointGroups.DetachNetworkEndpoints,qps,1.8,1
+        - --gce-ratelimit=beta.NetworkEndpointGroups.ListNetworkEndpoints,qps,1.8,1
+        - --gce-ratelimit=ga.NetworkEndpointGroups.Get,qps,1.8,1
+        - --gce-ratelimit=ga.NetworkEndpointGroups.AttachNetworkEndpoints,qps,1.8,1
+        - --gce-ratelimit=ga.NetworkEndpointGroups.DetachNetworkEndpoints,qps,1.8,1
+        - --gce-ratelimit=ga.NetworkEndpointGroups.ListNetworkEndpoints,qps,1.8,1
+      volumes:
+      - hostPath:
+          path: /etc/gce.conf
+          type: FileOrCreate
+        name: cloudconfig
+      - hostPath:
+          path: /var/log/glbc.log
+          type: FileOrCreate
+        name: logfile
+      - hostPath:
+          path: /etc/srv/kubernetes/l7-lb-controller
+        name: srvkube
+EOF
+)
+
+
+/workspace/test-infra/scenarios/kubernetes_e2e.py --check-leaked-resources \
+  --cluster= \
+  --env=GCE_ALPHA_FEATURES=NetworkEndpointGroup \
+  --env=KUBE_GCE_ENABLE_IP_ALIASES=true \
+  --extract=ci/latest \
+  --gcp-project="${GCP_PROJECT}" \
+  --gcp-zone=us-west1-b \
+  --ginkgo-parallel=1 \
+  --provider=gce \
+  '--test_args=--ginkgo.focus=\[Feature:Ingress\]|\[Feature:NEG\]' \
+  --timeout=320m