Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Make sharded e2e green #2

Open
wants to merge 16 commits into
base: skuznets/use-logical-clusters-1.24
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .ci-operator.yaml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
build_root_image:
namespace: ci
name: kcp-dev-build-root
tag: "1.17"
tag: "1.18"
34 changes: 21 additions & 13 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,10 @@ LDFLAGS := \
all: build
.PHONY: all

.PHONY: require-%
require-%:
@if ! command -v $* 1> /dev/null 2>&1; then echo "$* not found in \$$PATH"; exit 1; fi

pre-build-checks:
ifeq ($(and $(KUBE_MAJOR_VERSION),$(KUBE_MINOR_VERSION)),)
$(info Kubernetes version not set. Ensure jq is installed.)
Expand All @@ -87,6 +91,14 @@ build: pre-build-checks ## Build the project
build-all:
@$(MAKE) build WHAT=./cmd/...

.PHONY: build-kind-images
build-kind-images-ko: require-ko
$(eval SYNCER_IMAGE=$(shell KO_DOCKER_REPO=kind.local ko build --platform=linux/$(ARCH) ./cmd/syncer))
$(eval TEST_IMAGE=$(shell KO_DOCKER_REPO=kind.local ko build --platform=linux/$(ARCH) ./test/e2e/fixtures/kcp-test-image))
build-kind-images: build-kind-images-ko
test -n "$(SYNCER_IMAGE)" || (echo Failed to create syncer image; exit 1)
test -n "$(TEST_IMAGE)" || (echo Failed to create test image; exit 1)

install: WHAT ?= ./cmd/...
install:
go install -ldflags="$(LDFLAGS)" $(WHAT)
Expand Down Expand Up @@ -190,13 +202,11 @@ endif
test-e2e-shared: build-all
kind get kubeconfig > "$(PWD)/kind.kubeconfig"
mkdir -p $(LOG_DIR)
SYNCER_IMAGE=$$(KO_DOCKER_REPO=kind.local ko build --platform=linux/$(ARCH) ./cmd/syncer) && test -n "$${SYNCER_IMAGE}"; \
TEST_IMAGE=$$(KO_DOCKER_REPO=kind.local ko build --platform=linux/$(ARCH) ./test/e2e/fixtures/kcp-test-image) && test -n "$${TEST_IMAGE}"; \
NO_GORUN=1 ./bin/test-server --log-file-path="$(LOG_DIR)/kcp.log" $(TEST_SERVER_ARGS) 2>&1 & PID=$$!; echo "PID $$PID"; \
trap 'kill -TERM $$PID' TERM INT EXIT; \
while [ ! -f .kcp/admin.kubeconfig ]; do sleep 1; done; \
NO_GORUN=1 ./bin/test-server --log-file-path="$(LOG_DIR)/kcp.log" $(TEST_SERVER_ARGS) 2>&1 & PID=$$! && echo "PID $$PID" && \
trap 'kill -TERM $$PID' TERM INT EXIT && \
while [ ! -f .kcp/admin.kubeconfig ]; do sleep 1; done && \
NO_GORUN=1 $(GO_TEST) -race -count $(COUNT) -p $(E2E_PARALLELISM) -parallel $(E2E_PARALLELISM) $(WHAT) $(TEST_ARGS) \
-args --use-default-kcp-server --syncer-image="$${SYNCER_IMAGE}" --kcp-test-image="$${TEST_IMAGE}" --pcluster-kubeconfig="$(PWD)/kind.kubeconfig"
-args --use-default-kcp-server --syncer-image="$(SYNCER_IMAGE)" --kcp-test-image="$(TEST_IMAGE)" --pcluster-kubeconfig="$(PWD)/kind.kubeconfig"

.PHONY: test-e2e-sharded
ifdef USE_GOTESTSUM
Expand All @@ -209,16 +219,14 @@ test-e2e-sharded: LOG_DIR ?= $(ARTIFACT_DIR)/kcp
else
test-e2e-sharded: LOG_DIR ?= .kcp
endif
test-e2e-sharded: build-all
test-e2e-sharded: require-kind require-ko build-all
kind get kubeconfig > "$(PWD)/kind.kubeconfig"
mkdir -p $(LOG_DIR)
SYNCER_IMAGE=$$(KO_DOCKER_REPO=kind.local ko build --platform=linux/$(ARCH) ./cmd/syncer) && test -n "$${SYNCER_IMAGE}"; \
TEST_IMAGE=$$(KO_DOCKER_REPO=kind.local ko build --platform=linux/$(ARCH) ./test/e2e/fixtures/kcp-test-image) && test -n "$${TEST_IMAGE}"; \
NO_GORUN=1 ./bin/sharded-test-server --v=2 --log-dir-path="$(LOG_DIR)" $(TEST_SERVER_ARGS) 2>&1 & PID=$$!; echo "PID $$PID"; \
trap 'kill -TERM $$PID' TERM INT EXIT; \
while [ ! -f .kcp/admin.kubeconfig ]; do sleep 1; done; \
NO_GORUN=1 ./bin/sharded-test-server --v=2 --log-dir-path="$(LOG_DIR)" $(TEST_SERVER_ARGS) 2>&1 & PID=$$!; echo "PID $$PID" && \
trap 'kill -TERM $$PID' TERM INT EXIT && \
while [ ! -f .kcp/admin.kubeconfig ]; do sleep 1; done && \
NO_GORUN=1 $(GO_TEST) -race -count $(COUNT) -p $(E2E_PARALLELISM) -parallel $(E2E_PARALLELISM) $(WHAT) $(TEST_ARGS) \
-args --use-default-kcp-server --root-shard-kubeconfig=$(PWD)/.kcp-0/admin.kubeconfig --syncer-image="$${SYNCER_IMAGE}" --kcp-test-image="$${TEST_IMAGE}" --pcluster-kubeconfig="$(PWD)/kind.kubeconfig"
-args --use-default-kcp-server --root-shard-kubeconfig=$(PWD)/.kcp-0/admin.kubeconfig --syncer-image="$(SYNCER_IMAGE)" --kcp-test-image="$(TEST_IMAGE)" --pcluster-kubeconfig="$(PWD)/kind.kubeconfig"

.PHONY: test
ifdef USE_GOTESTSUM
Expand Down
5 changes: 3 additions & 2 deletions cmd/ingress-controller/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ import (
"os"
"time"

"github.com/kcp-dev/logicalcluster"
"github.com/kcp-dev/logicalcluster/v2"
"github.com/spf13/cobra"
"github.com/spf13/pflag"

Expand All @@ -33,6 +33,7 @@ import (
"k8s.io/component-base/logs"

"github.com/kcp-dev/kcp/pkg/cmd/help"
kcpfeatures "github.com/kcp-dev/kcp/pkg/features"
"github.com/kcp-dev/kcp/pkg/localenvoy/controllers/ingress"
envoycontrolplane "github.com/kcp-dev/kcp/pkg/localenvoy/controlplane"
"github.com/kcp-dev/kcp/pkg/reconciler/workload/ingresssplitter"
Expand All @@ -58,7 +59,7 @@ func main() {
RunE: func(cmd *cobra.Command, args []string) error {
ctx := genericapiserver.SetupSignalContext()

if err := options.Logs.ValidateAndApply(); err != nil {
if err := options.Logs.ValidateAndApply(kcpfeatures.DefaultFeatureGate); err != nil {
return err
}

Expand Down
5 changes: 3 additions & 2 deletions cmd/kcp-front-proxy/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ import (
tenancyv1alpha1 "github.com/kcp-dev/kcp/pkg/apis/tenancy/v1alpha1"
kcpclient "github.com/kcp-dev/kcp/pkg/client/clientset/versioned"
kcpinformers "github.com/kcp-dev/kcp/pkg/client/informers/externalversions"
kcpfeatures "github.com/kcp-dev/kcp/pkg/features"
"github.com/kcp-dev/kcp/pkg/proxy"
"github.com/kcp-dev/kcp/pkg/proxy/index"
"github.com/kcp-dev/kcp/pkg/server"
Expand Down Expand Up @@ -78,7 +79,7 @@ forwards Common Name and Organizations to backend API servers in HTTP headers.
The proxy terminates TLS and communicates with API servers via mTLS. Traffic is
routed based on paths.`,
RunE: func(cmd *cobra.Command, args []string) error {
if err := options.Logs.ValidateAndApply(); err != nil {
if err := options.Logs.ValidateAndApply(kcpfeatures.DefaultFeatureGate); err != nil {
return err
}
if err := options.Complete(); err != nil {
Expand Down Expand Up @@ -156,7 +157,7 @@ routed based on paths.`,
handler = genericapifilters.WithRequestInfo(handler, requestInfoFactory)
handler = genericfilters.WithHTTPLogging(handler)
handler = genericfilters.WithPanicRecovery(handler, requestInfoFactory)
doneCh, err := servingInfo.Serve(handler, time.Second*60, ctx.Done())
doneCh, _, err := servingInfo.Serve(handler, time.Second*60, ctx.Done())
if err != nil {
return err
}
Expand Down
3 changes: 2 additions & 1 deletion cmd/kcp/kcp.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ import (
"k8s.io/component-base/version"

"github.com/kcp-dev/kcp/pkg/cmd/help"
kcpfeatures "github.com/kcp-dev/kcp/pkg/features"
"github.com/kcp-dev/kcp/pkg/server"
"github.com/kcp-dev/kcp/pkg/server/options"
)
Expand Down Expand Up @@ -97,7 +98,7 @@ func main() {
},
RunE: func(cmd *cobra.Command, args []string) error {
// run as early as possible to avoid races later when some components (e.g. grpc) start early using klog
if err := serverOptions.GenericControlPlane.Logs.ValidateAndApply(); err != nil {
if err := serverOptions.GenericControlPlane.Logs.ValidateAndApply(kcpfeatures.DefaultFeatureGate); err != nil {
return err
}

Expand Down
5 changes: 3 additions & 2 deletions cmd/syncer/cmd/syncer.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ package cmd
import (
"context"

"github.com/kcp-dev/logicalcluster"
"github.com/kcp-dev/logicalcluster/v2"
"github.com/spf13/cobra"

"k8s.io/apimachinery/pkg/util/sets"
Expand All @@ -29,6 +29,7 @@ import (
"k8s.io/klog/v2"

synceroptions "github.com/kcp-dev/kcp/cmd/syncer/options"
kcpfeatures "github.com/kcp-dev/kcp/pkg/features"
"github.com/kcp-dev/kcp/pkg/syncer"
)

Expand All @@ -40,7 +41,7 @@ func NewSyncerCommand() *cobra.Command {
Use: "syncer",
Short: "Synchronizes resources in `kcp` assigned to the clusters",
RunE: func(cmd *cobra.Command, args []string) error {
if err := options.Logs.ValidateAndApply(); err != nil {
if err := options.Logs.ValidateAndApply(kcpfeatures.DefaultFeatureGate); err != nil {
return err
}
if err := options.Complete(); err != nil {
Expand Down
5 changes: 3 additions & 2 deletions cmd/virtual-workspaces/command/cmd.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ import (
"net/url"
"time"

"github.com/kcp-dev/logicalcluster"
"github.com/kcp-dev/logicalcluster/v2"
"github.com/spf13/cobra"

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
Expand All @@ -45,6 +45,7 @@ import (
tenancyv1alpha1 "github.com/kcp-dev/kcp/pkg/apis/tenancy/v1alpha1"
kcpclient "github.com/kcp-dev/kcp/pkg/client/clientset/versioned"
kcpinformer "github.com/kcp-dev/kcp/pkg/client/informers/externalversions"
kcpfeatures "github.com/kcp-dev/kcp/pkg/features"
boostrap "github.com/kcp-dev/kcp/pkg/server/bootstrap"
virtualrootapiserver "github.com/kcp-dev/kcp/pkg/virtual/framework/rootapiserver"
)
Expand All @@ -61,7 +62,7 @@ func NewCommand(ctx context.Context, errout io.Writer) *cobra.Command {
Long: "Start a virtual workspace apiserver to managing personal, shared or organization workspaces",

RunE: func(c *cobra.Command, args []string) error {
if err := opts.Logs.ValidateAndApply(); err != nil {
if err := opts.Logs.ValidateAndApply(kcpfeatures.DefaultFeatureGate); err != nil {
return err
}
if err := opts.Validate(); err != nil {
Expand Down
4 changes: 2 additions & 2 deletions config/crds/apis.kcp.dev_apiexports.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -57,11 +57,11 @@ spec:
the API identity in the 'key' file.
properties:
name:
description: Name is unique within a namespace to reference
description: name is unique within a namespace to reference
a secret resource.
type: string
namespace:
description: Namespace defines the space within which the
description: namespace defines the space within which the
secret name must be unique.
type: string
type: object
Expand Down
6 changes: 4 additions & 2 deletions config/crds/bootstrap.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ import (
"sync"
"time"

"github.com/kcp-dev/logicalcluster/v2"

crdhelpers "k8s.io/apiextensions-apiserver/pkg/apihelpers"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
extensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver"
Expand Down Expand Up @@ -132,7 +134,7 @@ func CreateSingle(ctx context.Context, client apiextensionsv1client.CustomResour
return fmt.Errorf("error creating CRD %s: %w", rawCRD.Name, err)
}
} else {
klog.Infof("Bootstrapped CRD %s|%v after %s", crd.GetClusterName(), crd.Name, time.Since(start).String())
klog.Infof("Bootstrapped CRD %s|%v after %s", logicalcluster.From(crd), crd.Name, time.Since(start).String())
}
} else {
return fmt.Errorf("error fetching CRD %s: %w", rawCRD.Name, err)
Expand All @@ -147,7 +149,7 @@ func CreateSingle(ctx context.Context, client apiextensionsv1client.CustomResour
if err != nil {
return err
}
klog.Infof("Updated CRD %s|%v after %s", crd.GetClusterName(), rawCRD.Name, time.Since(start).String())
klog.Infof("Updated CRD %s|%v after %s", logicalcluster.From(crd), rawCRD.Name, time.Since(start).String())
}

return wait.PollImmediateInfiniteWithContext(ctx, 100*time.Millisecond, func(ctx context.Context) (bool, error) {
Expand Down
20 changes: 6 additions & 14 deletions config/helpers/bootstrap.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ import (
"io"
"time"

"github.com/kcp-dev/logicalcluster"
"github.com/kcp-dev/logicalcluster/v2"

extensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver"
apierrors "k8s.io/apimachinery/pkg/api/errors"
Expand All @@ -44,6 +44,7 @@ import (

apisv1alpha1 "github.com/kcp-dev/kcp/pkg/apis/apis/v1alpha1"
tenancyv1alpha1 "github.com/kcp-dev/kcp/pkg/apis/tenancy/v1alpha1"
"github.com/kcp-dev/kcp/pkg/apis/tenancy/v1alpha1/helper"
kcpclient "github.com/kcp-dev/kcp/pkg/client/clientset/versioned"
)

Expand Down Expand Up @@ -169,15 +170,6 @@ func createResourceFromFS(ctx context.Context, client dynamic.Interface, mapper
return fmt.Errorf("could not get REST mapping for %s: %w", gvk, err)
}

// TODO(ncdc): replace with Maru's upcoming helper
logName := func(o metav1.Object) string {
if o.GetNamespace() == "" {
return fmt.Sprintf("%s|%s", o.GetClusterName(), o.GetName())
} else {
return fmt.Sprintf("%s|%s/%s", o.GetClusterName(), o.GetNamespace(), o.GetName())
}
}

upserted, err := client.Resource(m.Resource).Namespace(u.GetNamespace()).Create(ctx, u, metav1.CreateOptions{})
if err != nil {
if apierrors.IsAlreadyExists(err) {
Expand All @@ -190,24 +182,24 @@ func createResourceFromFS(ctx context.Context, client dynamic.Interface, mapper
klog.Infof(
"Skipping update of %s %s because it has the create-only annotation",
gvk,
logName(existing),
helper.QualifiedObjectName(existing),
)

return nil
}

u.SetResourceVersion(existing.GetResourceVersion())
if _, err = client.Resource(m.Resource).Namespace(u.GetNamespace()).Update(ctx, u, metav1.UpdateOptions{}); err != nil {
return fmt.Errorf("could not update %s %s: %w", gvk.Kind, logName(existing), err)
return fmt.Errorf("could not update %s %s: %w", gvk.Kind, helper.QualifiedObjectName(existing), err)
} else {
klog.Infof("Updated %s %s", gvk, logName(existing))
klog.Infof("Updated %s %s", gvk, helper.QualifiedObjectName(existing))
return nil
}
}
return err
}

klog.Infof("Bootstrapped %s %s", gvk.Kind, logName(upserted))
klog.Infof("Bootstrapped %s %s", gvk.Kind, helper.QualifiedObjectName(upserted))

return nil
}
Expand Down
12 changes: 6 additions & 6 deletions docs/developers/controllers.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,12 @@ Here are what keys look like for an object `foo` for both cluster-scoped and nam

|Organization|Workspace|Logical Cluster|Namespace|Key|
|-|-|-|-|-|
|-|-|root|-|root#$#foo|
|-|-|root|default|default/root#$#foo|
|root|my-org|root:my-org|-|root:my-org#$#foo|
|root|my-org|root:my-org|default|default/root:my-org#$#foo|
|my-org|my-workspace|my-org:my-workspace|-|my-org:my-workspace#$#foo|
|my-org|my-workspace|my-org:my-workspace|default|default/my-org:my-workspace#$#foo|
|-|-|root|-|root|foo|
|-|-|root|default|default/root|foo|
|root|my-org|root:my-org|-|root:my-org|foo|
|root|my-org|root:my-org|default|default/root:my-org|foo|
|my-org|my-workspace|my-org:my-workspace|-|my-org:my-workspace|foo|
|my-org|my-workspace|my-org:my-workspace|default|default/my-org:my-workspace|foo|

## Encoding/decoding keys

Expand Down
Loading