kops grid: create serial test for flaky scenario

Hoping to diagnose out why tests are timing out. Issue kubernetes#20738
justinsb · Feb 4, 2021 · 3047f85 · 3047f85
1 parent de2fa22
commit 3047f85
Show file tree

Hide file tree

Showing 2 changed files with 584 additions and 517 deletions.
diff --git a/config/jobs/kubernetes/kops/build_grid.py b/config/jobs/kubernetes/kops/build_grid.py
@@ -27,7 +27,7 @@
     preset-aws-credential: "true"
   decorate: true
   decoration_config:
-    timeout: 90m
+    timeout: {{job_timeout}}
   spec:
     containers:
     - command:
@@ -42,7 +42,7 @@
       - --env=KOPS_KUBE_RELEASE_URL=https://storage.googleapis.com/kubernetes-release/release
       - --env=KOPS_RUN_TOO_NEW_VERSION=1
       - --extract={{extract}}
-      - --ginkgo-parallel
+      - --ginkgo-parallel={{test_parallelism}}
       - --kops-args={{kops_args}}
       - --kops-feature-flags={{kops_feature_flags}}
       - --kops-image={{kops_image}}
@@ -51,7 +51,7 @@
       - --kops-zones={{kops_zones}}
       - --provider=aws
       - --test_args={{test_args}}
-      - --timeout=60m
+      - --timeout={{test_timeout}}
       image: {{e2e_image}}
       imagePullPolicy: Always
       resources:
@@ -72,7 +72,7 @@
     preset-aws-credential: "true"
   decorate: true
   decoration_config:
-    timeout: 90m
+    timeout: {{job_timeout}}
   extra_refs:
   - org: kubernetes
     repo: kops
@@ -99,7 +99,7 @@
           --test=kops \\
           -- \\
           --test-package-marker={{marker}} \\
-          --parallel 25 \\
+          --parallel {{test_parallelism}} \\
           --skip-regex="{{skip_regex}}"
       env:
       - name: KUBE_SSH_KEY_PATH
@@ -126,6 +126,7 @@
     'kops-grid-scenario-public-jwks',
     'kops-grid-scenario-arm64',
     'kops-grid-scenario-aws-cloud-controller-manager',
+    'kops-grid-scenario-serial-test-for-timeout',
 ]
 
 # These are job tab names of unsupported grid combinations
@@ -214,7 +215,9 @@ def build_test(cloud='aws',
                feature_flags=(),
                extra_flags=None,
                extra_dashboards=None,
-               interval=None):
+               interval=None,
+               test_parallelism=25,
+               test_timeout_minutes=60):
     # pylint: disable=too-many-statements,too-many-branches
 
     if container_runtime == "containerd" and (kops_version == "1.18" or networking in (None, "kopeio")): # pylint: disable=line-too-long
@@ -373,6 +376,10 @@ def expand(s):
     y = y.replace('{{e2e_image}}', e2e_image)
     y = y.replace('{{kops_image}}', kops_image)
 
+    y = y.replace('{{test_parallelism}}', str(test_parallelism))
+    y = y.replace('{{job_timeout}}', str(test_timeout_minutes + 30) + 'm')
+    y = y.replace('{{test_timeout}}', str(test_timeout_minutes) + 'm')
+
     # specific to kubetest2
     if use_kubetest2:
         if networking:
@@ -542,6 +549,15 @@ def generate():
                             '--override=cluster.spec.cloudConfig.awsEBSCSIDriver.enabled=true'],
                extra_dashboards=['provider-aws-cloud-provider-aws', 'kops-misc'])
 
+    # A special test to diagnose test timeouts
+    # cf https://github.com/kubernetes/test-infra/issues/20738
+    build_test(force_name="scenario-serial-test-for-timeout",
+               cloud="aws",
+               networking="calico",
+               distro="amzn2",
+               k8s_version="1.20",
+               test_parallelism=1)
+
     print("")
     print("# %d jobs, total of %d runs per week" % (job_count, runs_per_week))