From b41eb65c3179dd88b3093417dc17e3d132900b76 Mon Sep 17 00:00:00 2001 From: Dale Hamel Date: Thu, 15 Jul 2021 14:45:38 -0400 Subject: [PATCH] feat: Integration test enhancements This enhances the integration test suite in a variety of ways: - Both minikube and KiND are now supported, and run in github actions - Tests run in their own namespace, which is intelligently cleaned up - Images are uploaded to a test backend via docker registry API - Buildkit is used to significantly speed up incremental image builds This commit also fixes a bug where failed jobs would not report their failure. Co-authored-by: Aaron Olson <934893+honkfestival@users.noreply.github.com> Co-authored-by: Zeeshan Qureshi --- .github/workflows/build-and-test.yml | 23 +- Makefile | 74 ++++--- build/Dockerfile.tracerunner | 28 ++- build/hooks/prestop | 29 +++ go.mod | 3 +- go.sum | 1 + integration/cmd_run_test.go | 44 +++- integration/kind_backend.go | 152 +++++++++++++ integration/minikube_backend.go | 118 ++++++++++ integration/suite_test.go | 320 ++++++++++++++++++++++----- pkg/cmd/tracerunner.go | 4 +- pkg/docker/docker.go | 52 +++++ pkg/docker/docker_test.go | 71 ++++++ 13 files changed, 810 insertions(+), 109 deletions(-) create mode 100755 build/hooks/prestop create mode 100644 integration/kind_backend.go create mode 100644 integration/minikube_backend.go create mode 100644 pkg/docker/docker.go create mode 100644 pkg/docker/docker_test.go diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 86a0e5e0..151f3e16 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -1,6 +1,6 @@ name: Kubectl trace build and tests -on: [push, pull_request] +on: [pull_request] jobs: build_and_test: @@ -9,6 +9,9 @@ jobs: matrix: os: [ubuntu-16.04, ubuntu-18.04] # 16.04.4 release has 4.15 kernel # 18.04.3 release has 5.0.0 kernel + env: + - TEST_KUBERNETES_BACKEND: minikube + - TEST_KUBERNETES_BACKEND: kind steps: - uses: actions/checkout@v2 - run: git fetch --prune --unshallow # We want tags @@ -39,13 +42,26 @@ jobs: - name: Build CI image run: | - ./build/scripts/ci-build-image.sh ${{ github.ref }} + ./build/scripts/ci-build-image.sh ${{ github.head_ref }} + + - name: Install minikube + env: ${{matrix.env}} + if: ${{ env.TEST_KUBERNETES_BACKEND == 'minikube' }} + run: | + curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64 + sudo install minikube-linux-amd64 /usr/local/bin/minikube - name: Run integration tests + env: ${{matrix.env}} run: | make integration +# - name: Debug failure over SSH +# if: ${{ failure() }} +# uses: mxschmitt/action-tmate@v3 + - name: Build cross binaries + if: github.ref == 'refs/heads/master' run: | curl -LO https://github.com/goreleaser/goreleaser/releases/latest/download/goreleaser_amd64.deb && sudo dpkg -i goreleaser_amd64.deb make cross @@ -56,6 +72,7 @@ jobs: path: _output/bin/kubectl-trace - uses: actions/upload-artifact@v1 + if: github.ref == 'refs/heads/master' with: name: ${{ matrix.os }}-kubectl-trace-cross-dist path: dist @@ -66,4 +83,4 @@ jobs: env: QUAY_TOKEN: ${{ secrets.QUAY_TOKEN }} run: | - ./build/scripts/ci-release-image.sh ${{ github.ref }} + ./build/scripts/ci-release-image.sh ${{ github.head_ref }} diff --git a/Makefile b/Makefile index b36c8e1c..42fe5ade 100644 --- a/Makefile +++ b/Makefile @@ -4,35 +4,45 @@ GO ?= go DOCKER ?= docker COMMIT_NO := $(shell git rev-parse HEAD 2> /dev/null || true) -GIT_COMMIT := $(if $(shell git status --porcelain --untracked-files=no),${COMMIT_NO}-dirty,${COMMIT_NO}) +GIT_COMMIT := $(if $(shell git status --porcelain --untracked-files=no 2> /dev/null),${COMMIT_NO}-dirty,${COMMIT_NO}) GIT_TAG ?= $(shell git describe 2> /dev/null) GIT_BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD 2>/dev/null) GIT_BRANCH_CLEAN := $(shell echo $(GIT_BRANCH) | sed -e "s/[^[:alnum:]]/-/g") GIT_ORG ?= iovisor -IMAGE_NAME_INIT ?= quay.io/$(GIT_ORG)/kubectl-trace-init -IMAGE_NAME ?= quay.io/$(GIT_ORG)/kubectl-trace-bpftrace +DOCKER_BUILD_PROGRESS ?= auto -IMAGE_TRACERUNNER_BRANCH := $(IMAGE_NAME):$(GIT_BRANCH_CLEAN) -IMAGE_TRACERUNNER_COMMIT := $(IMAGE_NAME):$(GIT_COMMIT) -IMAGE_TRACERUNNER_TAG := $(IMAGE_NAME):$(GIT_TAG) -IMAGE_TRACERUNNER_LATEST := $(IMAGE_NAME):latest +IMAGE_NAME_INITCONTAINER ?= quay.io/$(GIT_ORG)/kubectl-trace-init +IMAGE_NAME_TRACERUNNER ?= quay.io/$(GIT_ORG)/kubectl-trace-runner -IMAGE_INITCONTAINER_BRANCH := $(IMAGE_NAME_INIT):$(GIT_BRANCH_CLEAN) -IMAGE_INITCONTAINER_COMMIT := $(IMAGE_NAME_INIT):$(GIT_COMMIT) -IMAGE_INITCONTAINER_TAG := $(IMAGE_NAME_INIT):$(GIT_TAG) -IMAGE_INITCONTAINER_LATEST := $(IMAGE_NAME_INIT):latest +IMAGE_TRACERUNNER_BRANCH := $(IMAGE_NAME_TRACERUNNER):$(GIT_BRANCH_CLEAN) +IMAGE_TRACERUNNER_COMMIT := $(IMAGE_NAME_TRACERUNNER):$(GIT_COMMIT) +IMAGE_TRACERUNNER_TAG := $(IMAGE_NAME_TRACERUNNER):$(GIT_TAG) +IMAGE_TRACERUNNER_LATEST := $(IMAGE_NAME_TRACERUNNER):latest -IMAGE_BUILD_FLAGS ?= "--no-cache" +IMAGE_INITCONTAINER_BRANCH := $(IMAGE_NAME_INITCONTAINER):$(GIT_BRANCH_CLEAN) +IMAGE_INITCONTAINER_COMMIT := $(IMAGE_NAME_INITCONTAINER):$(GIT_COMMIT) +IMAGE_INITCONTAINER_TAG := $(IMAGE_NAME_INITCONTAINER):$(GIT_TAG) +IMAGE_INITCONTAINER_LATEST := $(IMAGE_NAME_INITCONTAINER):latest -BPFTRACEVERSION ?= "v0.11.1" +IMAGE_BUILD_FLAGS_EXTRA ?= # convenience to allow to specify extra build flags with env var, defaults to nil -LDFLAGS := -ldflags '-X github.com/iovisor/kubectl-trace/pkg/version.buildTime=$(shell date +%s) -X github.com/iovisor/kubectl-trace/pkg/version.gitCommit=${GIT_COMMIT} -X github.com/iovisor/kubectl-trace/pkg/cmd.ImageName=${IMAGE_NAME} -X github.com/iovisor/kubectl-trace/pkg/cmd.ImageTag=${GIT_COMMIT} -X github.com/iovisor/kubectl-trace/pkg/cmd.InitImageName=${IMAGE_NAME_INIT} -X github.com/iovisor/kubectl-trace/pkg/cmd.InitImageTag=${GIT_COMMIT}' +IMG_REPO ?= quay.io/iovisor/ +IMG_SHA ?= latest + +BPFTRACEVERSION ?= "v0.13.0" + +LDFLAGS := -ldflags '-X github.com/iovisor/kubectl-trace/pkg/version.buildTime=$(shell date +%s) -X github.com/iovisor/kubectl-trace/pkg/version.gitCommit=${GIT_COMMIT} -X github.com/iovisor/kubectl-trace/pkg/cmd.ImageName=${IMAGE_NAME_TRACERUNNER} -X github.com/iovisor/kubectl-trace/pkg/cmd.ImageTag=${GIT_COMMIT} -X github.com/iovisor/kubectl-trace/pkg/cmd.InitImageName=${IMAGE_NAME_INITCONTAINER} -X github.com/iovisor/kubectl-trace/pkg/cmd.InitImageTag=${GIT_COMMIT}' TESTPACKAGES := $(shell go list ./... | grep -v github.com/iovisor/kubectl-trace/integration) +TEST_ONLY ?= kubectl_trace ?= _output/bin/kubectl-trace trace_runner ?= _output/bin/trace-runner +trace_uploader ?= _output/bin/trace-uploader + +# ensure variables are available to child invocations of make +export .PHONY: build build: clean ${kubectl_trace} @@ -45,11 +55,11 @@ ${trace_runner}: .PHONY: cross cross: - IMAGE_NAME=$(IMAGE_NAME) GO111MODULE=on goreleaser --snapshot --rm-dist + IMAGE_NAME_TRACERUNNER=$(IMAGE_NAME_TRACERUNNER) GO111MODULE=on goreleaser --snapshot --rm-dist .PHONY: release release: - IMAGE_NAME=$(IMAGE_NAME) GO111MODULE=on goreleaser --rm-dist + IMAGE_NAME_TRACERUNNER=$(IMAGE_NAME_TRACERUNNER) GO111MODULE=on goreleaser --rm-dist .PHONY: clean clean: @@ -59,24 +69,30 @@ clean: .PHONY: image/build-init image/build-init: $(DOCKER) build \ - $(IMAGE_BUILD_FLAGS) \ + --progress=$(DOCKER_BUILD_PROGRESS) \ -t $(IMAGE_INITCONTAINER_BRANCH) \ - -f ./build/Dockerfile.initcontainer ./build - $(DOCKER) tag $(IMAGE_INITCONTAINER_BRANCH) $(IMAGE_INITCONTAINER_COMMIT) - $(DOCKER) tag $(IMAGE_INITCONTAINER_BRANCH) $(IMAGE_INITCONTAINER_TAG) + -f ./build/Dockerfile.initcontainer \ + ${IMAGE_BUILD_FLAGS_EXTRA} \ + ./build + $(DOCKER) tag "$(IMAGE_INITCONTAINER_BRANCH)" "$(IMAGE_INITCONTAINER_COMMIT)" + $(DOCKER) tag "$(IMAGE_INITCONTAINER_BRANCH)" "$(IMAGE_INITCONTAINER_TAG)" + $(DOCKER) tag "$(IMAGE_INITCONTAINER_BRANCH)" "$(IMAGE_INITCONTAINER_LATEST)" .PHONY: image/build image/build: - $(DOCKER) build \ + DOCKER_BUILDKIT=1 $(DOCKER) build \ --build-arg bpftraceversion=$(BPFTRACEVERSION) \ --build-arg GIT_ORG=$(GIT_ORG) \ - $(IMAGE_BUILD_FLAGS) \ + --progress=$(DOCKER_BUILD_PROGRESS) \ + --build-arg BUILDKIT_INLINE_CACHE=1 \ + --cache-from "$(IMAGE_TRACERUNNER_BRANCH)" \ -t "$(IMAGE_TRACERUNNER_BRANCH)" \ - -f build/Dockerfile.tracerunner . - $(DOCKER) tag $(IMAGE_TRACERUNNER_BRANCH) $(IMAGE_TRACERUNNER_COMMIT) - $(DOCKER) tag "$(IMAGE_TRACERUNNER_BRANCH)" $(IMAGE_TRACERUNNER_BRANCH) - $(DOCKER) tag "$(IMAGE_TRACERUNNER_BRANCH)" $(IMAGE_TRACERUNNER_TAG) - + -f build/Dockerfile.tracerunner \ + ${IMAGE_BUILD_FLAGS_EXTRA} \ + . + $(DOCKER) tag "$(IMAGE_TRACERUNNER_BRANCH)" "$(IMAGE_TRACERUNNER_COMMIT)" + $(DOCKER) tag "$(IMAGE_TRACERUNNER_BRANCH)" "$(IMAGE_TRACERUNNER_TAG)" + $(DOCKER) tag "$(IMAGE_TRACERUNNER_BRANCH)" "$(IMAGE_TRACERUNNER_LATEST)" .PHONY: image/push image/push: @@ -99,5 +115,5 @@ test: $(GO) test -v -race $(TESTPACKAGES) .PHONY: integration -integration: - TEST_KUBECTLTRACE_BINARY=$(shell pwd)/$(kubectl_trace) $(GO) test ${LDFLAGS} -v ./integration/... +integration: build image/build image/build-init + TEST_KUBECTLTRACE_BINARY=$(shell pwd)/$(kubectl_trace) $(GO) test ${LDFLAGS} -failfast -count=1 -v ./integration/... -run TestKubectlTraceSuite $(if $(TEST_ONLY),-testify.m $(TEST_ONLY),) diff --git a/build/Dockerfile.tracerunner b/build/Dockerfile.tracerunner index 89083b9e..69b53227 100644 --- a/build/Dockerfile.tracerunner +++ b/build/Dockerfile.tracerunner @@ -1,23 +1,37 @@ +# syntax = docker/dockerfile:1.2 ARG bpftraceversion=v0.13.0 FROM quay.io/iovisor/bpftrace:$bpftraceversion as bpftrace FROM golang:1.15-buster as gobuilder ARG GIT_ORG=iovisor ENV GIT_ORG=$GIT_ORG -RUN apt-get update -RUN apt-get install -y make bash git +RUN apt-get update && apt-get install -y make bash git && apt-get clean -ADD . /go/src/github.com/iovisor/kubectl-trace WORKDIR /go/src/github.com/iovisor/kubectl-trace -RUN make _output/bin/trace-runner +# first copy the go mod files and sync the module cache as this step is expensive +COPY go.* . +RUN go mod download + +# Now copy the rest of the source code one by one +# note any changes in any of these files or subdirectories is expected to bust the cache +# We copy only the code directories, makefile, and git directory in order to prevent +# busting the cache. Due to limitations in docker syntax, this must be done one-per-line +COPY Makefile . +COPY cmd cmd +COPY pkg pkg + +# This buildkit feature reduces the build time from ~50s → 5s by preserving the compiler cache +RUN --mount=type=cache,target=/root/.cache/go-build make _output/bin/trace-runner FROM ubuntu:20.04 -RUN apt-get update -RUN apt-get install -y xz-utils +# Install CA certificates +RUN apt-get update && apt-get install -y ca-certificates && update-ca-certificates && apt-get clean -COPY --from=gobuilder /go/src/github.com/iovisor/kubectl-trace/_output/bin/trace-runner /bin/trace-runner COPY --from=bpftrace /usr/bin/bpftrace /usr/bin/bpftrace +COPY --from=gobuilder /go/src/github.com/iovisor/kubectl-trace/_output/bin/trace-runner /bin/trace-runner + +COPY /build/hooks/prestop /bin/hooks/prestop ENTRYPOINT ["/bin/trace-runner"] diff --git a/build/hooks/prestop b/build/hooks/prestop new file mode 100755 index 00000000..29b9f88b --- /dev/null +++ b/build/hooks/prestop @@ -0,0 +1,29 @@ +#!/bin/bash +set -uo pipefail + +if [[ "$#" -gt "1" ]]; then + echo "usage: prestop [SLEEP_SECONDS]" + exit 41 +fi + +sleep_seconds=0 +if [[ "$#" -eq "1" ]]; then + sleep_seconds="$1" +fi + +tpid=`pgrep --oldest trace-runner` +if [[ -z "$tpid" ]]; then + echo "could not find trace-runner" + exit 21 +fi + +cpid=`pgrep --oldest -P $tpid` +if [[ -z "$cpid" ]]; then + echo "could not find first child of trace-runner" + exit 22 +fi + +kill -SIGINT $cpid + +## Give some time to trace-runner to cleanup before pod kill timeout starts. +sleep $sleep_seconds diff --git a/go.mod b/go.mod index 9a1bbcdf..15bc900c 100644 --- a/go.mod +++ b/go.mod @@ -6,11 +6,10 @@ require ( github.com/evanphx/json-patch v4.9.0+incompatible github.com/fntlnz/mountinfo v0.0.0-20171106231217-40cb42681fad github.com/kr/pretty v0.2.1 // indirect - github.com/pkg/errors v0.9.1 github.com/spf13/cobra v1.1.1 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.6.1 - gotest.tools v2.2.0+incompatible + golang.org/x/mod v0.3.0 k8s.io/api v0.19.3 k8s.io/apimachinery v0.19.3 k8s.io/cli-runtime v0.19.3 diff --git a/go.sum b/go.sum index 5f560f25..4813ab3d 100644 --- a/go.sum +++ b/go.sum @@ -410,6 +410,7 @@ golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKG golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/integration/cmd_run_test.go b/integration/cmd_run_test.go index 4bfae3f9..ee4bf70d 100644 --- a/integration/cmd_run_test.go +++ b/integration/cmd_run_test.go @@ -2,19 +2,45 @@ package integration import ( "regexp" + "time" "github.com/stretchr/testify/assert" + + batchv1 "k8s.io/api/batch/v1" ) -func (k *KubectlTraceSuite) TestRunNode() { - nodes, err := k.provider.ListNodes(k.name) - assert.Nil(k.T(), err) - assert.Equal(k.T(), 1, len(nodes)) +type outputAsserter func(string, []byte) - nodeName := nodes[0].String() +func (k *KubectlTraceSuite) TestRunNode() { + nodeName := k.GetTestNode() bpftraceProgram := `kprobe:do_sys_open { printf("%s: %s\n", comm, str(arg1)) }'` - out := k.KubectlTraceCmd("run", "-e", bpftraceProgram, nodeName) - match, err := regexp.MatchString("trace (\\w+-){4}\\w+ created", out) - assert.Nil(k.T(), err) - assert.True(k.T(), match) + out := k.KubectlTraceCmd("run", "--namespace="+k.namespace(), "--imagename="+k.RunnerImage(), "-e", bpftraceProgram, nodeName) + assert.Regexp(k.T(), "trace (\\w+-){4}\\w+ created", out) +} + +func (k *KubectlTraceSuite) TestReturnErrOnErr() { + nodeName := k.GetTestNode() + + bpftraceProgram := `kprobe:not_a_real_kprobe { printf("%s: %s\n", comm, str(arg1)) }'` + out := k.KubectlTraceCmd("run", "--namespace="+k.namespace(), "--imagename="+k.RunnerImage(), "-e", bpftraceProgram, nodeName) + assert.Regexp(k.T(), regexp.MustCompile("trace [a-f0-9-]{36} created"), out) + + var job batchv1.Job + + for { + jobs := k.GetJobs().Items + assert.Equal(k.T(), 1, len(jobs)) + + job = jobs[0] + if len(job.Status.Conditions) > 0 { + break // on the first condition + } + + time.Sleep(1 * time.Second) + } + + assert.Equal(k.T(), 1, len(job.Status.Conditions)) + assert.Equal(k.T(), "Failed", string(job.Status.Conditions[0].Type)) + assert.Equal(k.T(), int32(0), job.Status.Succeeded, "No jobs in the batch should have succeeded") + assert.Greater(k.T(), job.Status.Failed, int32(1), "There should be at least one failed job") } diff --git a/integration/kind_backend.go b/integration/kind_backend.go new file mode 100644 index 00000000..e319ad51 --- /dev/null +++ b/integration/kind_backend.go @@ -0,0 +1,152 @@ +package integration + +import ( + "fmt" + "github.com/iovisor/kubectl-trace/pkg/docker" + "os/exec" + "strconv" + "strings" + "time" + + "github.com/iovisor/kubectl-trace/pkg/cmd" + "github.com/stretchr/testify/assert" + + "sigs.k8s.io/kind/pkg/cluster" +) + +const ( + KubernetesKindBackend = "kind" + KindClusterName = "kubectl-trace-kind" + KindDefaultRegistryPort = 5000 +) + +const setMaxPodWaitTimeout = 30 + +type kindBackend struct { + suite *KubectlTraceSuite + provider *cluster.Provider + runnerImage string + name string + registryPort int +} + +func (b *kindBackend) SetupBackend() { + var err error + b.name = KindClusterName + + fmt.Println("Setting up KiND backend...") + if RegistryLocalPort == "" { + b.registryPort = KindDefaultRegistryPort + } else { + b.registryPort, err = strconv.Atoi(RegistryLocalPort) + assert.Nil(b.suite.T(), err) + } + + registryConfig := ` +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +containerdConfigPatches: +- |- + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:` + strconv.Itoa(RegistryRemotePort) + `"] + endpoint = ["http://kind-registry:` + strconv.Itoa(RegistryRemotePort) + `"] +` + + b.provider = cluster.NewProvider() + + if ForceFreshBackend != "" { + b.TeardownBackend() + } + + clusters, _ := b.provider.List() + + if !kindClusterExists(clusters, b.name) { + fmt.Printf("Creating new KiND cluster\n") + // Create the cluster + err = b.provider.Create( + b.name, + cluster.CreateWithRetain(false), + cluster.CreateWithWaitForReady(time.Duration(0)), + cluster.CreateWithKubeconfigPath(b.suite.kubeConfigPath), + cluster.CreateWithRawConfig([]byte(registryConfig)), + + // todo > we need a logger + // cluster.ProviderWithLogger(logger), + // runtime.GetDefault(logger), + ) + assert.Nil(b.suite.T(), err) + } + + b.provider.ExportKubeConfig(b.name, b.suite.kubeConfigPath) + + // Start the registry container if not already started + comm := exec.Command("docker", "inspect", "-f", "{{.State.Running}}", "kind-registry") + o, err := comm.CombinedOutput() + if err != nil || strings.TrimSuffix(string(o), "\n") != "true" { + output := b.suite.runWithoutError("docker", "run", "-d", "--restart=always", "-p", fmt.Sprintf("%d:%d", b.registryPort, RegistryRemotePort), "--name", "kind-registry", "registry:2") + fmt.Printf("Started registry: %s\n", output) + } + + // This template is to avoid having to unmarshal the JSON, and will print "true" if there is container called "kind-registry" on the network + networkTemplate := `{{range $_, $value := .Containers }}{{if index $value "Name" | eq "kind-registry"}}true{{end}}{{end}}` + output := b.suite.runWithoutError("docker", "network", "inspect", "kind", "--format", networkTemplate) + if strings.TrimSuffix(output, "\n") != "true" { + // Connect to docker network + output = b.suite.runWithoutError("docker", "network", "connect", "kind", "kind-registry") + fmt.Printf("Connected network: %s\n", output) + } + + // make sure the repository is available + assert.Nil(b.suite.T(), checkRegistryAvailable(b.registryPort)) + + // parse the image of the desired image runner name + parsedImage, err := docker.ParseImageName(cmd.ImageName) + assert.Nil(b.suite.T(), err) + + // set the runner image name with the repository port + b.runnerImage = fmt.Sprintf("localhost:%d/%s/%s:latest", RegistryRemotePort, parsedImage.Repository, parsedImage.Name) + + b.suite.tagAndPushIntegrationImage(cmd.ImageName, cmd.ImageTag) +} + +func (b *kindBackend) TeardownBackend() { + fmt.Printf("Deleting KiND cluster\n") + kubeConfig, err := b.provider.KubeConfig(b.name, false) + assert.Nil(b.suite.T(), err) + err = b.provider.Delete(b.name, kubeConfig) + assert.Nil(b.suite.T(), err) + b.suite.runWithoutError("docker", "rm", "-f", "kind-registry") +} + +func (b *kindBackend) GetBackendNode() string { + nodes, err := b.provider.ListNodes(b.name) + assert.Nil(b.suite.T(), err) + assert.Equal(b.suite.T(), 1, len(nodes)) + nodeName := nodes[0].String() + return nodeName +} + +func (b *kindBackend) RegistryPort() int { + return b.registryPort +} + +func (b *kindBackend) RunnerImage() string { + return b.runnerImage +} + +func (b *kindBackend) RunNodeCommand(command string) error { + comm := exec.Command("docker", "exec", "-i", KindClusterName+"-control-plane", "bash", "-c", command) + o, err := comm.CombinedOutput() + if err != nil { + return fmt.Errorf("Failed to run command '%s', output %s. %v", command, o, err) + } + return nil +} + +func kindClusterExists(clusters []string, cluster string) bool { + for _, n := range clusters { + if cluster == n { + return true + } + } + return false +} diff --git a/integration/minikube_backend.go b/integration/minikube_backend.go new file mode 100644 index 00000000..417bcf57 --- /dev/null +++ b/integration/minikube_backend.go @@ -0,0 +1,118 @@ +package integration + +import ( + "fmt" + "github.com/iovisor/kubectl-trace/pkg/cmd" + "github.com/iovisor/kubectl-trace/pkg/docker" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/mod/semver" + "net" + "os/exec" + "strconv" + "strings" +) + +var ( + MinikubeStartOutput = `Done! kubectl is now configured to use "` + MinikubeProfileName + `"` + MinikubeDeleteOutput = `Removed all traces of the "` + MinikubeProfileName + `" cluster` + MinikubeRegistryOutput = "The 'registry' addon is enabled" +) + +const ( + MinikubeMinimumVersion = "v1.20.0" + KubernetesMinikubeBackend = "minikube" + MinikubeProfileName = "minikube-kubectl-trace" + MinikubeDefaultRegistryPort = 6000 +) + +type minikubeBackend struct { + suite *KubectlTraceSuite + runnerImage string + registryPort int +} + +func (b *minikubeBackend) SetupBackend() { + _, err := exec.LookPath("minikube") + assert.Nil(b.suite.T(), err) + + fmt.Println("Setting up Minikube backend...") + + minikubeVersion := strings.TrimSpace(b.suite.runWithoutError("minikube", "version", "--short")) + fmt.Printf("minikube is version %s, minimum is %s\n", minikubeVersion, MinikubeMinimumVersion) + require.GreaterOrEqual(b.suite.T(), semver.Compare(minikubeVersion, MinikubeMinimumVersion), 0, fmt.Sprintf("minikube %s is too old, upgrade to %s", minikubeVersion, MinikubeMinimumVersion)) + + if RegistryLocalPort == "" { + b.registryPort = MinikubeDefaultRegistryPort + } else { + b.registryPort, err = strconv.Atoi(RegistryLocalPort) + assert.Nil(b.suite.T(), err) + } + + if ForceFreshBackend != "" { + b.TeardownBackend() + } + + output := b.suite.runWithoutError("minikube", "start", "--insecure-registry=localhost:"+strconv.Itoa(RegistryRemotePort), "--profile", MinikubeProfileName) + require.Contains(b.suite.T(), output, MinikubeStartOutput) + + output = b.suite.runWithoutError("minikube", "addons", "enable", "registry", "--profile", MinikubeProfileName) + require.Contains(b.suite.T(), output, MinikubeRegistryOutput) + + // The minikube registry uses a replica controller, so we match on pod label when waiting for it + fmt.Printf("Waiting for minikube registry to be ready...\n") + _ = b.suite.runWithoutError("kubectl", "wait", "pod", "-l", "actual-registry=true", "-n", "kube-system", "--for=condition=Ready", "--timeout=120s") + + minikube_ip := strings.TrimSuffix(b.suite.runWithoutError("minikube", "ip", "--profile", MinikubeProfileName), "\n") + require.NotNil(b.suite.T(), net.ParseIP(minikube_ip)) + + // Start the registry proxy container if not already started + comm := exec.Command("docker", "inspect", "-f", "{{.State.Running}}", "minikube-kubectl-trace-registry-proxy") + o, err := comm.CombinedOutput() + if err != nil || strings.TrimSuffix(string(o), "\n") != "true" { + output := b.suite.runWithoutError("docker", "run", "-d", "--restart=always", "--network=host", "--name", "minikube-kubectl-trace-registry-proxy", "alpine/socat", "TCP-LISTEN:6000,reuseaddr,fork", "TCP:"+minikube_ip+":"+strconv.Itoa(RegistryRemotePort)) + fmt.Printf("Started registry proxy: %s\n", output) + } + + // make sure the repository is available + require.Nil(b.suite.T(), checkRegistryAvailable(b.registryPort)) + + // parse the image of the desired image runner name + parsedImage, err := docker.ParseImageName(cmd.ImageName) + require.Nil(b.suite.T(), err) + + // set the runner image name with the repository port + // TODO: extract the repository logic somewhere? + b.runnerImage = fmt.Sprintf("localhost:%d/%s/%s:latest", RegistryRemotePort, parsedImage.Repository, parsedImage.Name) + + // tag and push the integration image + b.suite.tagAndPushIntegrationImage(cmd.ImageName, cmd.ImageTag) +} + +func (b *minikubeBackend) TeardownBackend() { + fmt.Printf("Deleting minikube cluster...\n") + output := b.suite.runWithoutError("minikube", "delete", "--profile", MinikubeProfileName) + assert.Contains(b.suite.T(), output, MinikubeDeleteOutput) + + b.suite.runWithoutError("docker", "rm", "-f", "minikube-kubectl-trace-registry-proxy") // FIXME allow this to fail if container is missing, or verify it is running first +} +func (b *minikubeBackend) GetBackendNode() string { + return MinikubeProfileName +} + +func (b *minikubeBackend) RegistryPort() int { + return b.registryPort +} + +func (b *minikubeBackend) RunnerImage() string { + return b.runnerImage +} + +func (b *minikubeBackend) RunNodeCommand(command string) error { + comm := exec.Command("docker", "exec", "-i", MinikubeProfileName, "bash", "-c", command) + o, err := comm.CombinedOutput() + if err != nil { + return fmt.Errorf("Failed to run command '%s', output %s. %v", command, o, err) + } + return nil +} diff --git a/integration/suite_test.go b/integration/suite_test.go index f1437de3..42319826 100644 --- a/integration/suite_test.go +++ b/integration/suite_test.go @@ -1,121 +1,327 @@ package integration import ( + "context" "crypto/rand" "fmt" + "io" + "net/http" "os" "os/exec" "path/filepath" + "regexp" "strings" "testing" "time" - "github.com/iovisor/kubectl-trace/pkg/cmd" - "github.com/pkg/errors" + "github.com/iovisor/kubectl-trace/pkg/docker" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - "gotest.tools/icmd" - "sigs.k8s.io/kind/pkg/cluster" - "sigs.k8s.io/kind/pkg/cluster/nodes" - "sigs.k8s.io/kind/pkg/cluster/nodeutils" - "sigs.k8s.io/kind/pkg/fs" + batchv1 "k8s.io/api/batch/v1" + apiv1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" ) var ( - KubectlTraceBinary = os.Getenv("TEST_KUBECTLTRACE_BINARY") + DockerPushOutput = regexp.MustCompile("latest: digest: sha256:[0-9a-f]{64} size: [0-9]+") ) +var ( + KubectlTraceBinary = os.Getenv("TEST_KUBECTLTRACE_BINARY") // allow overriding the kubectl-trace binary used + KubernetesBackend = os.Getenv("TEST_KUBERNETES_BACKEND") // allow specifying which kubernetes backend to use for tests + ForceFreshBackend = os.Getenv("TEST_FORCE_FRESH_BACKEND") // force a fresh kubernetes backend for tests + TeardownBackend = os.Getenv("TEST_TEARDOWN_BACKEND") // force backend to be torn down after test run + RegistryLocalPort = os.Getenv("TEST_REGISTRY_PORT") // override default port for backend's docker registry +) + +const RegistryRemotePort = 5000 +const RegistryWaitTimeout = 60 + +const ( + waitForDeleteTargetSeconds = 60 + waitForTargetPodSeconds = 30 + defaultMaxPods = 110 +) + +const ( + TraceJobsSystemNamespace = "kubectl-trace-system" + IntegrationNamespaceLabel = "kubectl-trace-integration-ns" + IntegrationTargetNamespaceLabel = "kubectl-trace-integration-target" +) + +var ( + ContainerDependencies = []string{ + "quay.io/iovisor/kubectl-trace-init", + } +) + +type TestBackend interface { + SetupBackend() + TeardownBackend() + RunNodeCommand(string) error + GetBackendNode() string + RunnerImage() string + RegistryPort() int +} + +type TestNameSpaceInfo struct { + Namespace string + Passed bool +} + type KubectlTraceSuite struct { suite.Suite - kubeConfigPath string - // kindContext *cluster.Context - - provider *cluster.Provider - name string + testBackend TestBackend + kubeConfigPath string + lastTest string + namespaces map[string]*TestNameSpaceInfo + targetNamespace string } func init() { if KubectlTraceBinary == "" { KubectlTraceBinary = "kubectl-trace" } + + if KubernetesBackend == "" { + KubernetesBackend = KubernetesKindBackend + } +} + +func (k *KubectlTraceSuite) RunnerImage() string { + return k.testBackend.RunnerImage() +} + +func (k *KubectlTraceSuite) GetTestNode() string { + return k.testBackend.GetBackendNode() } func (k *KubectlTraceSuite) SetupSuite() { - var err error - k.name, err = generateClusterName() + path, err := os.Getwd() assert.Nil(k.T(), err) - k.provider = cluster.NewProvider() - // Create the cluster - err = k.provider.Create( - k.name, - cluster.CreateWithRetain(false), - cluster.CreateWithWaitForReady(time.Duration(0)), - cluster.CreateWithKubeconfigPath(k.kubeConfigPath), - - // todo > we need a logger - // cluster.ProviderWithLogger(logger), - // runtime.GetDefault(logger), - ) + // tests are run from /path/to/kubectl-trace-shopify/integration + k.kubeConfigPath = filepath.Join(path, "..", "_output", "kubeconfig") + + switch KubernetesBackend { + case KubernetesKindBackend: + k.testBackend = &kindBackend{ + suite: k, + } + case KubernetesMinikubeBackend: + k.testBackend = &minikubeBackend{ + suite: k, + } + } + + k.testBackend.SetupBackend() + + k.cleanupPreviousRunNamespaces(IntegrationNamespaceLabel) + k.namespaces = make(map[string]*TestNameSpaceInfo) + + fmt.Println("Pushing dependencies...") + for _, image := range ContainerDependencies { + k.tagAndPushIntegrationImage(image, "latest") + } + + fmt.Printf("\x1b[1mKUBECONFIG=%s\x1b[0m\n", k.kubeConfigPath) +} + +func (k *KubectlTraceSuite) teardownTargets() { + k.deleteNamespace(k.targetNamespace) +} + +func checkRegistryAvailable(registryPort int) error { + registry := fmt.Sprintf("http://localhost:%d/v2/", registryPort) + err := fmt.Errorf("registry %s is unavailable", registry) + + attempts := 0 + for err != nil && attempts < RegistryWaitTimeout { + _, err = http.Get(registry) + time.Sleep(1 * time.Second) + attempts++ + } + + if err != nil { + fmt.Printf("Failed waiting for registry to become available after %d seconds\n", attempts) + } + + return err +} + +func (k *KubectlTraceSuite) tagAndPushIntegrationImage(sourceName string, sourceTag string) { + parsedImage, err := docker.ParseImageName(sourceName) + assert.Nil(k.T(), err) + + pushTag := fmt.Sprintf("localhost:%d/%s/%s:latest", k.testBackend.RegistryPort(), parsedImage.Repository, parsedImage.Name) + sourceImage := sourceName + ":" + sourceTag + + output := k.runWithoutError("docker", "tag", sourceImage, pushTag) + assert.Empty(k.T(), output) + + output = k.runWithoutError("docker", "push", pushTag) + assert.Regexp(k.T(), DockerPushOutput, output) +} + +func (k *KubectlTraceSuite) BeforeTest(suiteName, testName string) { + k.lastTest = testName + clientConfig, err := clientcmd.BuildConfigFromFlags("", k.kubeConfigPath) assert.Nil(k.T(), err) - nodes, err := k.provider.ListNodes(k.name) + clientset, err := kubernetes.NewForConfig(clientConfig) assert.Nil(k.T(), err) - // Copy the bpftrace into a tar - dir, err := fs.TempDir("", "image-tar") + namespace, err := generateNamespaceName("kubectl-trace-test") + + k.namespaces[testName] = &TestNameSpaceInfo{Namespace: namespace} assert.Nil(k.T(), err) - defer os.RemoveAll(dir) - imageTarPath := filepath.Join(dir, "image.tar") - err = save(cmd.ImageName+":"+cmd.ImageTag, imageTarPath) + namespaceLabels := map[string]string{ + IntegrationNamespaceLabel: testName, + } + + nsSpec := &apiv1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: k.namespace(), Labels: namespaceLabels}} + _, err = clientset.CoreV1().Namespaces().Create(context.TODO(), nsSpec, metav1.CreateOptions{}) assert.Nil(k.T(), err) +} + +func (k *KubectlTraceSuite) AfterTest(suiteName, testName string) { + k.namespaces[testName].Passed = !k.T().Failed() + + if k.namespaces[testName].Passed { + // delete the namespace if the test passed + k.deleteNamespace(k.namespace()) + } + k.lastTest = "" +} + +func (k *KubectlTraceSuite) cleanupPreviousRunNamespaces(namespaceLabel string) { + clientConfig, err := clientcmd.BuildConfigFromFlags("", k.kubeConfigPath) + clientset, err := kubernetes.NewForConfig(clientConfig) + namespaces, err := clientset.CoreV1().Namespaces().List(context.TODO(), metav1.ListOptions{LabelSelector: namespaceLabel}) + + if err != nil { + fmt.Printf("Error listing previous namespaces %v", err) + } - // Copy the bpftrace image to the nodes - for _, n := range nodes { - err = loadImage(imageTarPath, n) - assert.Nil(k.T(), err) + for _, ns := range namespaces.Items { + fmt.Printf("Cleaning up namespace from previous run %s\n", ns.Name) + k.deleteNamespace(ns.Name) } } -func (k *KubectlTraceSuite) TeardownSuite() { - kubeConfig, err := k.provider.KubeConfig(k.name, false) +func (k *KubectlTraceSuite) deleteNamespace(namespace string) { + clientConfig, err := clientcmd.BuildConfigFromFlags("", k.kubeConfigPath) + assert.Nil(k.T(), err) + + clientset, err := kubernetes.NewForConfig(clientConfig) assert.Nil(k.T(), err) - err = k.provider.Delete(k.name, kubeConfig) + + fg := metav1.DeletePropagationForeground + deleteOptions := metav1.DeleteOptions{PropagationPolicy: &fg} + err = clientset.CoreV1().Namespaces().Delete(context.TODO(), namespace, deleteOptions) assert.Nil(k.T(), err) } +// Reports namespaces of any failed tests for debugging purposes +func (k *KubectlTraceSuite) HandleStats(suiteName string, stats *suite.SuiteInformation) { + if TeardownBackend != "" { + return + } + + for _, v := range stats.TestStats { + if !v.Passed { + namespace := k.namespaces[v.TestName].Namespace + fmt.Printf("\x1b[1m%s failed, namespace %s has been preserved for debugging\x1b[0m\n", v.TestName, namespace) + } + } +} + +func (k *KubectlTraceSuite) TearDownSuite() { + if TeardownBackend != "" { + k.testBackend.TeardownBackend() + } +} + func TestKubectlTraceSuite(t *testing.T) { suite.Run(t, &KubectlTraceSuite{}) } +func (k *KubectlTraceSuite) GetJobs() *batchv1.JobList { + return k.GetJobsInNamespace(k.namespace()) +} + +func (k *KubectlTraceSuite) GetJobsInNamespace(namespace string) *batchv1.JobList { + clientConfig, err := clientcmd.BuildConfigFromFlags("", k.kubeConfigPath) + assert.Nil(k.T(), err) + + clientset, err := kubernetes.NewForConfig(clientConfig) + assert.Nil(k.T(), err) + + jobs, err := clientset.BatchV1().Jobs(namespace).List(context.TODO(), metav1.ListOptions{}) + assert.Nil(k.T(), err) + + return jobs +} + +func (k *KubectlTraceSuite) namespace() string { + if k.lastTest == "" { + require.NotEmpty(k.T(), k.lastTest, "Programming error in test suite: lastTest not set on suite. This condition should be impossible to hit and is a bug if you see this.") + } + + namespaceInfo := k.namespaces[k.lastTest] + return namespaceInfo.Namespace +} + func (k *KubectlTraceSuite) KubectlTraceCmd(args ...string) string { - args = append([]string{fmt.Sprintf("--kubeconfig=%s", k.kubeConfigPath)}, args...) - res := icmd.RunCommand(KubectlTraceBinary, args...) - assert.Equal(k.T(), icmd.Success.ExitCode, res.ExitCode) - return res.Combined() + args = append([]string{fmt.Sprintf("--namespace=%s", k.namespace())}, args...) + return k.runWithoutError(KubectlTraceBinary, args...) } -func generateClusterName() (string, error) { +func generateNamespaceName(baseName string) (string, error) { buf := make([]byte, 10) if _, err := rand.Read(buf); err != nil { return "", err } - return strings.ToLower(fmt.Sprintf("%X", buf)), nil + return strings.ToLower(fmt.Sprintf("%s-%X", baseName, buf)), nil } -// loads an image tarball onto a node -func loadImage(imageTarName string, node nodes.Node) error { - f, err := os.Open(imageTarName) - if err != nil { - return errors.Wrap(err, "failed to open image") - } - defer f.Close() - return nodeutils.LoadImageArchive(node, f) +func (k *KubectlTraceSuite) runWithoutError(command string, args ...string) string { + return k.runWithoutErrorWithStdin("", command, args...) } -// save saves image to dest, as in `docker save` -func save(image, dest string) error { - return exec.Command("docker", "save", "-o", dest, image).Run() +func (k *KubectlTraceSuite) runWithoutErrorWithStdin(input string, command string, args ...string) string { + // prepare the command + comm := exec.Command(command, args...) + + // prepare stdin unless it's empty + if input != "" { + stdin, err := comm.StdinPipe() + if err != nil { + assert.Nilf(k.T(), err, "Could not create the commmand: %s", err.Error()) + } + go func() { + defer stdin.Close() + io.WriteString(stdin, input) + }() + } + + // prepare the environment + comm.Env = os.Environ() + comm.Env = append(comm.Env, fmt.Sprintf("KUBECONFIG=%s", k.kubeConfigPath)) // required to write a unique kubeconfig for the test run) + + // run it + o, err := comm.CombinedOutput() + combined := string(o) + + assert.Nilf(k.T(), err, "Command failed with output %s", combined) + + return combined } + +func int32Ptr(i int32) *int32 { return &i } +func int64Ptr(i int32) *int64 { cast := int64(i); return &cast } diff --git a/pkg/cmd/tracerunner.go b/pkg/cmd/tracerunner.go index c4dffdca..d25f4f07 100644 --- a/pkg/cmd/tracerunner.go +++ b/pkg/cmd/tracerunner.go @@ -40,7 +40,7 @@ func NewTraceRunnerCommand() *cobra.Command { } if err := o.Run(); err != nil { fmt.Fprintln(os.Stdout, err.Error()) - return nil + return err } return nil }, @@ -49,7 +49,7 @@ func NewTraceRunnerCommand() *cobra.Command { cmd.Flags().StringVarP(&o.containerName, "container", "c", o.containerName, "Specify the container") cmd.Flags().StringVarP(&o.podUID, "poduid", "p", o.podUID, "Specify the pod UID") cmd.Flags().StringVarP(&o.programPath, "program", "f", "program.bt", "Specify the bpftrace program path") - cmd.Flags().StringVarP(&o.bpftraceBinaryPath, "bpftracebinary", "b", "/bin/bpftrace", "Specify the bpftrace binary path") + cmd.Flags().StringVarP(&o.bpftraceBinaryPath, "bpftracebinary", "b", "/usr/bin/bpftrace", "Specify the bpftrace binary path") cmd.Flags().BoolVar(&o.inPod, "inpod", false, "Whether or not run this bpftrace in a pod's container process namespace") return cmd } diff --git a/pkg/docker/docker.go b/pkg/docker/docker.go new file mode 100644 index 00000000..920cc3f3 --- /dev/null +++ b/pkg/docker/docker.go @@ -0,0 +1,52 @@ +package docker + +import ( + "fmt" + "strings" +) + +type Image struct { + Hostname string + Repository string + Name string + Tag string +} + +func ParseImageName(imageName string) (*Image, error) { + parts := strings.Split(imageName, "/") + tag := "" + + nameTag := strings.Split(parts[len(parts)-1], ":") + parts[len(parts)-1] = nameTag[0] + + switch len(nameTag) { + case 1: + case 2: + tag = nameTag[1] + default: + return nil, fmt.Errorf("Invalid docker image name '%s'; expected hostname/repository/name[:tag]", imageName) + } + + switch len(parts) { + case 3: + return &Image{ + Hostname: parts[0], + Repository: parts[1], + Name: parts[2], + Tag: tag, + }, nil + case 2: + return &Image{ + Repository: parts[0], + Name: parts[1], + Tag: tag, + }, nil + case 1: + return &Image{ + Name: parts[0], + Tag: tag, + }, nil + default: + return nil, fmt.Errorf("Invalid docker image name '%s'; expected hostname/repository/name[:tag]", imageName) + } +} diff --git a/pkg/docker/docker_test.go b/pkg/docker/docker_test.go new file mode 100644 index 00000000..204e8725 --- /dev/null +++ b/pkg/docker/docker_test.go @@ -0,0 +1,71 @@ +package docker + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestParseImageNameOnlyName(t *testing.T) { + imageName := "testing" + expected := Image{ + Name: "testing", + } + + actual, err := ParseImageName(imageName) + assert.Nil(t, err) + assert.Equal(t, &expected, actual) +} + +func TestParseImageNameWithNameTag(t *testing.T) { + imageName := "testing:latest" + expected := Image{ + Name: "testing", + Tag: "latest", + } + + actual, err := ParseImageName(imageName) + assert.Nil(t, err) + assert.Equal(t, &expected, actual) +} + +func TestParseImageNameWithRepositoryNameTag(t *testing.T) { + imageName := "weird/testing:latest" + expected := Image{ + Repository: "weird", + Name: "testing", + Tag: "latest", + } + + actual, err := ParseImageName(imageName) + assert.Nil(t, err) + assert.Equal(t, &expected, actual) +} + +func TestParseImageNameHostnameRepositoryNameTag(t *testing.T) { + imageName := "quay.io/weird/testing:latest" + expected := Image{ + Hostname: "quay.io", + Repository: "weird", + Name: "testing", + Tag: "latest", + } + + actual, err := ParseImageName(imageName) + assert.Nil(t, err) + assert.Equal(t, &expected, actual) +} + +func TestInvalidImageNames(t *testing.T) { + invalid := []string{ + "https://example.com/nope/sorry:whatever", + "http://example.com/nope/sorry:whatever", + "some/big/long/name", + "another:with:invalid:tags", + } + for _, name := range invalid { + parsed, err := ParseImageName(name) + assert.Nil(t, parsed) + assert.NotNil(t, err) + } +}