diff --git a/internal/test/utils/utils.go b/internal/test/utils/utils.go index ff3c4cba..b0f80bf7 100644 --- a/internal/test/utils/utils.go +++ b/internal/test/utils/utils.go @@ -6,18 +6,21 @@ import ( "fmt" "log" "testing" + "time" "github.com/go-logr/logr" "github.com/go-logr/stdr" "github.com/kong/kubernetes-testing-framework/pkg/clusters" "github.com/kong/kubernetes-testing-framework/pkg/environments" "github.com/stretchr/testify/assert" + corev1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" gatewayv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" + "sigs.k8s.io/kustomize/kyaml/yaml" "github.com/kong/blixt/pkg/vars" ) @@ -73,9 +76,13 @@ func NewFakeClientWithGatewayClasses(initObjects ...client.Object) (gatewayv1bet // environment (but deploying Blixt is expected to have already been handled // elsewhere). func WaitForBlixtReadiness(ctx context.Context, env environments.Environment) error { + ticker := time.NewTicker(time.Minute * 3) // TODO: glob for { select { + case <-ticker.C: + simpleDiagDump(ctx, env) case <-ctx.Done(): + simpleDiagDump(ctx, env) if err := ctx.Err(); err != nil { return fmt.Errorf("context completed while waiting for components: %w", err) } @@ -85,6 +92,7 @@ func WaitForBlixtReadiness(ctx context.Context, env environments.Environment) er controlplane, err := env.Cluster().Client().AppsV1().Deployments(vars.DefaultNamespace).Get(ctx, vars.DefaultControlPlaneDeploymentName, metav1.GetOptions{}) if err != nil { + fmt.Printf("Error while checking controlplane components: %s\n", err) return err } if controlplane.Status.AvailableReplicas > 0 { @@ -93,6 +101,7 @@ func WaitForBlixtReadiness(ctx context.Context, env environments.Environment) er dataplane, err := env.Cluster().Client().AppsV1().DaemonSets(vars.DefaultNamespace).Get(ctx, vars.DefaultDataPlaneDaemonSetName, metav1.GetOptions{}) if err != nil { + fmt.Printf("Error while checking dataplane components: %s\n", err) return err } if dataplane.Status.NumberAvailable > 0 { @@ -116,3 +125,42 @@ func DumpDiagnosticsIfFailed(ctx context.Context, t *testing.T, clusters cluster assert.NoError(t, err) } } + +// simpleDiagDump is something more brief than a full cluster diagnostic, +// mainly intended to help quickly diagnose Pod issues. +func simpleDiagDump(ctx context.Context, env environments.Environment) { + fmt.Println("INFO: dumping cluster details") + + deployments, err := env.Cluster().Client().AppsV1().Deployments(corev1.NamespaceAll).List(ctx, corev1.ListOptions{}) + if err == nil { + b, err := yaml.Marshal(deployments) + if err != nil { + fmt.Printf("Warning: failed to marshal YAML report of current deployment status on shutdown: %s\n", err) + } + fmt.Printf("Deployments:\n---\n%s\n", string(b)) + } else { + fmt.Printf("Warning: failed to report current deployment status on shutdown: %s\n", err) + } + + daemonsets, err := env.Cluster().Client().AppsV1().DaemonSets(corev1.NamespaceAll).List(ctx, corev1.ListOptions{}) + if err == nil { + b, err := yaml.Marshal(daemonsets) + if err != nil { + fmt.Printf("Warning: failed to marshal YAML report of current daemonset status on shutdown: %s\n", err) + } + fmt.Printf("Daemonsets:\n---\n%s\n", string(b)) + } else { + fmt.Printf("Warning: failed to report current daemonset status on shutdown: %s\n", err) + } + + pods, err := env.Cluster().Client().CoreV1().Pods(corev1.NamespaceAll).List(ctx, corev1.ListOptions{}) + if err == nil { + b, err := yaml.Marshal(pods) + if err != nil { + fmt.Printf("Warning: failed to marshal YAML report of current pod status on shutdown: %s\n", err) + } + fmt.Printf("Pods:\n---\n%s\n", string(b)) + } else { + fmt.Printf("Warning: failed to report current pod status on shutdown: %s\n", err) + } +}