Improve wait routine for m2m oidc configurator (#2)

It was tested with self-hosted runner using custom dockerconfig credentials for debugging.

Signed-off-by: Krzysztof Romanowski <[email protected]>

.github/workflows/kserve_m2m_test.yaml

@@ -39,7 +39,7 @@ jobs:
         run: kustomize build common/kubeflow-namespace/base | kubectl apply -f -
 
       - name: Install Istio with ext auth
-        run: ./tests/gh-actions/install_istio_with_ext_auth.sh*
+        run: ./tests/gh-actions/install_istio_with_ext_auth.sh
 
       - name: Install cert-manager
         run: ./tests/gh-actions/install_cert_manager.sh

.github/workflows/notebook_controller_m2m_test.yaml

@@ -47,7 +47,8 @@ jobs:
       run: |
         kustomize build apps/jupyter/jupyter-web-app/upstream/overlays/istio/ | kubectl apply -f -
         kustomize build apps/jupyter/notebook-controller/upstream/overlays/kubeflow/ | kubectl apply -f -
-        kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout 300s
+        kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout=300s \
+          --field-selector=status.phase!=Succeeded
 
     - name: Create KF Profile
       run: kustomize build common/user-namespace/base | kubectl apply -f -

.github/workflows/pipeline_test.yaml

@@ -60,6 +60,10 @@ jobs:
           nohup kubectl port-forward --namespace istio-system svc/${ingress_gateway_service} 8080:80 &
           while ! curl localhost:8080; do echo waiting for port-forwarding; sleep 1; done; echo port-forwarding ready
 
+      - name: Wait for the kubeflow-m2m-oidc-configurator Job
+        run: |
+          ./tests/gh-actions/wait_for_kubeflow_m2m_oidc_configurator.sh
+
       - name: List and deploy test pipeline with authorized ServiceAccount Token
         run: |
           pip3 install kfp==2.4.0
@@ -116,4 +120,4 @@ jobs:
           ' "${TOKEN}" "${KF_PROFILE}"
 
           echo "Test succeeded. Token from unauthorized ServiceAccount cannot list \
-          piplines in $KF_PROFILE namespace."
+          piplines in $KF_PROFILE namespace."

common/oidc-client/oauth2-proxy/components/configure-self-signed-kubernetes-oidc-issuer/cronjob.kubeflow-m2m-oidc-configurator.yaml

@@ -4,11 +4,11 @@ metadata:
   name: kubeflow-m2m-oidc-configurator
   namespace: istio-system
 spec:
-  schedule: '* * * * *'
+  schedule: '*/5 * * * *'
   concurrencyPolicy: Forbid
   jobTemplate:
     spec:
-      ttlSecondsAfterFinished: 60
+      ttlSecondsAfterFinished: 600
       template:
         metadata:
           labels: {}

tests/gh-actions/install_istio_with_ext_auth.sh

@@ -8,7 +8,8 @@ kustomize build istio-install/overlays/oauth2-proxy | kubectl apply -f -
 cd -
 
 echo "Waiting for all Istio Pods to become ready..."
-kubectl wait --for=condition=Ready pods --all -n istio-system --timeout 300s
+kubectl wait --for=condition=Ready pods --all -n istio-system --timeout=300s \
+  --field-selector=status.phase!=Succeeded
 
 echo "Installing oauth2-proxy..."
 cd common/oidc-client

tests/gh-actions/install_knative-cni.sh

@@ -9,5 +9,6 @@ kustomize build common/knative/knative-serving/base | kubectl apply -f -
 kustomize build common/istio-cni-1-22/cluster-local-gateway/base | kubectl apply -f -
 kustomize build common/istio-cni-1-22/kubeflow-istio-resources/base | kubectl apply -f -
 
-kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout 600s
+kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout=600s \
+  --field-selector=status.phase!=Succeeded
 kubectl patch cm config-domain --patch '{"data":{"example.com":""}}' -n knative-serving

tests/gh-actions/install_knative.sh

@@ -9,5 +9,6 @@ kustomize build common/knative/knative-serving/base | kubectl apply -f -
 kustomize build common/istio-1-22/cluster-local-gateway/base | kubectl apply -f -
 kustomize build common/istio-1-22/kubeflow-istio-resources/base | kubectl apply -f -
 
-kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout 600s
+kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout=600s \
+  --field-selector=status.phase!=Succeeded
 kubectl patch cm config-domain --patch '{"data":{"example.com":""}}' -n knative-serving

tests/gh-actions/install_kserve.sh

@@ -11,4 +11,5 @@ echo "Waiting for crd/clusterservingruntimes.serving.kserve.io to be available .
 kubectl wait --for condition=established --timeout=30s crd/clusterservingruntimes.serving.kserve.io
 kustomize build kserve | kubectl apply -f -
 kustomize build models-web-app/overlays/kubeflow | kubectl apply -f -
-kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout 600s
+kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout=600s \
+  --field-selector=status.phase!=Succeeded

tests/gh-actions/install_pipelines.sh

@@ -7,5 +7,6 @@ echo "Waiting for crd/compositecontrollers.metacontroller.k8s.io to be available
 kubectl wait --for condition=established --timeout=30s crd/compositecontrollers.metacontroller.k8s.io
 kustomize build env/cert-manager/platform-agnostic-multi-user | kubectl apply -f -
 sleep 60
-kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout 600s
+kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout=600s \
+  --field-selector=status.phase!=Succeeded
 cd -