From 2de976fa412d37c974a7f0fba654013fc910c300 Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Sat, 23 Jul 2022 10:52:02 +0200 Subject: [PATCH] Makefile/test-e2e-{shared,sharded}: add WORK_DIR --- Makefile | 31 ++++++++++-------- cmd/sharded-test-server/frontproxy.go | 45 ++++++++++++--------------- cmd/sharded-test-server/main.go | 16 +++++----- cmd/sharded-test-server/shard.go | 36 ++++++++++----------- 4 files changed, 63 insertions(+), 65 deletions(-) diff --git a/Makefile b/Makefile index 9015b3017205..063c88e4f3ac 100644 --- a/Makefile +++ b/Makefile @@ -194,19 +194,21 @@ test-e2e-shared: $(GOTESTSUM) endif test-e2e-shared: TEST_ARGS ?= test-e2e-shared: WHAT ?= ./test/e2e... +test-e2e-shared: WORK_DIR ?= . ifdef ARTIFACT_DIR test-e2e-shared: LOG_DIR ?= $(ARTIFACT_DIR)/kcp else -test-e2e-shared: LOG_DIR ?= .kcp +test-e2e-shared: LOG_DIR ?= $(WORK_DIR)/.kcp endif -test-e2e-shared: build-all - kind get kubeconfig > "$(PWD)/kind.kubeconfig" - mkdir -p $(LOG_DIR) +test-e2e-shared: require-kind build-all build-kind-images + mkdir -p "$(LOG_DIR)" "$(WORK_DIR)/.kcp" + kind get kubeconfig > "$(WORK_DIR)/.kcp/kind.kubeconfig" + rm -f "$(WORK_DIR)/.kcp/admin.kubeconfig" NO_GORUN=1 ./bin/test-server --log-file-path="$(LOG_DIR)/kcp.log" $(TEST_SERVER_ARGS) 2>&1 & PID=$$! && echo "PID $$PID" && \ trap 'kill -TERM $$PID' TERM INT EXIT && \ - while [ ! -f .kcp/admin.kubeconfig ]; do sleep 1; done && \ + while [ ! -f "$(WORK_DIR)/.kcp/admin.kubeconfig" ]; do sleep 1; done && \ NO_GORUN=1 $(GO_TEST) -race -count $(COUNT) -p $(E2E_PARALLELISM) -parallel $(E2E_PARALLELISM) $(WHAT) $(TEST_ARGS) \ - -args --use-default-kcp-server --syncer-image="$(SYNCER_IMAGE)" --kcp-test-image="$(TEST_IMAGE)" --pcluster-kubeconfig="$(WORK_DIR)/kind.kubeconfig" + -args --use-default-kcp-server --syncer-image="$(SYNCER_IMAGE)" --kcp-test-image="$(TEST_IMAGE)" --pcluster-kubeconfig="$(abspath $(WORK_DIR)/.kcp/kind.kubeconfig)" .PHONY: test-e2e-sharded ifdef USE_GOTESTSUM @@ -214,19 +216,22 @@ test-e2e-sharded: $(GOTESTSUM) endif test-e2e-sharded: TEST_ARGS ?= test-e2e-sharded: WHAT ?= ./test/e2e... +test-e2e-sharded: WORK_DIR ?= . ifdef ARTIFACT_DIR test-e2e-sharded: LOG_DIR ?= $(ARTIFACT_DIR)/kcp else -test-e2e-sharded: LOG_DIR ?= .kcp +test-e2e-sharded: LOG_DIR ?= $(WORK_DIR)/.kcp endif -test-e2e-sharded: require-kind require-ko build-all - kind get kubeconfig > "$(PWD)/kind.kubeconfig" - mkdir -p $(LOG_DIR) - NO_GORUN=1 ./bin/sharded-test-server --v=2 --log-dir-path="$(LOG_DIR)" $(TEST_SERVER_ARGS) 2>&1 & PID=$$!; echo "PID $$PID" && \ +test-e2e-sharded: require-kind build-all build-kind-images + mkdir -p "$(LOG_DIR)" "$(WORK_DIR)/.kcp" + kind get kubeconfig > "$(WORK_DIR)/.kcp/kind.kubeconfig" + rm -f "$(WORK_DIR)/.kcp/admin.kubeconfig" + NO_GORUN=1 ./bin/sharded-test-server --v=2 --log-dir-path="$(LOG_DIR)" --work-dir-path="$(WORK_DIR)" $(TEST_SERVER_ARGS) 2>&1 & PID=$$!; echo "PID $$PID" && \ trap 'kill -TERM $$PID' TERM INT EXIT && \ - while [ ! -f .kcp/admin.kubeconfig ]; do sleep 1; done && \ + while [ ! -f "$(WORK_DIR)/.kcp/admin.kubeconfig" ]; do sleep 1; done && \ NO_GORUN=1 $(GO_TEST) -race -count $(COUNT) -p $(E2E_PARALLELISM) -parallel $(E2E_PARALLELISM) $(WHAT) $(TEST_ARGS) \ - -args --use-default-kcp-server --root-shard-kubeconfig=$(PWD)/.kcp-0/admin.kubeconfig --syncer-image="$(SYNCER_IMAGE)" --kcp-test-image="$(TEST_IMAGE)" --pcluster-kubeconfig="$(WORK_DIR)/kind.kubeconfig" + -args --use-default-kcp-server --root-shard-kubeconfig=$(PWD)/.kcp-0/admin.kubeconfig \ + --syncer-image="$(SYNCER_IMAGE)" --kcp-test-image="$(TEST_IMAGE)" --pcluster-kubeconfig="$(abspath $(WORK_DIR)/.kcp/kind.kubeconfig)" .PHONY: test ifdef USE_GOTESTSUM diff --git a/cmd/sharded-test-server/frontproxy.go b/cmd/sharded-test-server/frontproxy.go index c3076da2c776..41877a32184c 100644 --- a/cmd/sharded-test-server/frontproxy.go +++ b/cmd/sharded-test-server/frontproxy.go @@ -18,7 +18,6 @@ package main import ( "context" - "flag" "fmt" "io/ioutil" "net/http" @@ -42,7 +41,7 @@ import ( "github.com/kcp-dev/kcp/test/e2e/framework" ) -func startFrontProxy(ctx context.Context, args []string, servingCA *crypto.CA, hostIP string) error { +func startFrontProxy(ctx context.Context, args []string, servingCA *crypto.CA, hostIP string, logDirPath, workDirPath string) error { blue := color.New(color.BgGreen, color.FgBlack).SprintFunc() inverse := color.New(color.BgHiWhite, color.FgGreen).SprintFunc() out := lineprefix.New( @@ -54,7 +53,7 @@ func startFrontProxy(ctx context.Context, args []string, servingCA *crypto.CA, h lineprefix.Color(color.New(color.FgHiWhite)), ) - if err := ioutil.WriteFile(".kcp-front-proxy/mapping.yaml", []byte(` + if err := ioutil.WriteFile(filepath.Join(workDirPath, ".kcp-front-proxy/mapping.yaml"), []byte(` - path: /services/ backend: https://localhost:6444 backend_server_ca: .kcp/serving-ca.crt @@ -70,7 +69,7 @@ func startFrontProxy(ctx context.Context, args []string, servingCA *crypto.CA, h } // write root shard kubeconfig - configLoader := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(&clientcmd.ClientConfigLoadingRules{ExplicitPath: ".kcp-0/admin.kubeconfig"}, nil) + configLoader := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(&clientcmd.ClientConfigLoadingRules{ExplicitPath: filepath.Join(workDirPath, ".kcp-0/admin.kubeconfig")}, nil) raw, err := configLoader.RawConfig() if err != nil { return err @@ -90,18 +89,18 @@ func startFrontProxy(ctx context.Context, args []string, servingCA *crypto.CA, h if err != nil { return fmt.Errorf("failed to create server cert: %w\n", err) } - if err := cert.WriteCertConfigFile(".kcp-front-proxy/apiserver.crt", ".kcp-front-proxy/apiserver.key"); err != nil { + if err := cert.WriteCertConfigFile(filepath.Join(workDirPath, ".kcp-front-proxy/apiserver.crt"), filepath.Join(workDirPath, ".kcp-front-proxy/apiserver.key")); err != nil { return fmt.Errorf("failed to write server cert: %w\n", err) } // run front-proxy command commandLine := append(framework.DirectOrGoRunCommand("kcp-front-proxy"), - "--mapping-file=.kcp-front-proxy/mapping.yaml", - "--root-directory=.kcp-front-proxy", - "--root-kubeconfig=.kcp/root.kubeconfig", - "--client-ca-file=.kcp/client-ca.crt", - "--tls-cert-file=.kcp-front-proxy/apiserver.crt", - "--tls-private-key-file=.kcp-front-proxy/apiserver.key", + fmt.Sprintf("--mapping-file=%s", filepath.Join(workDirPath, ".kcp-front-proxy/mapping.yaml")), + fmt.Sprintf("--root-directory=%s", filepath.Join(workDirPath, ".kcp-front-proxy")), + fmt.Sprintf("--root-kubeconfig=%s", filepath.Join(workDirPath, ".kcp/root.kubeconfig")), + fmt.Sprintf("--client-ca-file=%s", filepath.Join(workDirPath, ".kcp/client-ca.crt")), + fmt.Sprintf("--tls-cert-file=%s", filepath.Join(workDirPath, ".kcp-front-proxy/apiserver.crt")), + fmt.Sprintf("--tls-private-key-file=%s", filepath.Join(workDirPath, ".kcp-front-proxy/apiserver.key")), "--secure-port=6443", ) commandLine = append(commandLine, args...) @@ -109,13 +108,9 @@ func startFrontProxy(ctx context.Context, args []string, servingCA *crypto.CA, h cmd := exec.CommandContext(ctx, commandLine[0], commandLine[1:]...) - logDir := flag.Lookup("log-dir-path").Value.String() - if err != nil { - return err - } - logFilePath := ".kcp-front-proxy/proxy.log" - if logDir != "" { - logFilePath = filepath.Join(logDir, "kcp-front-proxy.log") + logFilePath := filepath.Join(workDirPath, ".kcp-front-proxy/proxy.log") + if logDirPath != "" { + logFilePath = filepath.Join(logDirPath, "kcp-front-proxy.log") } logFile, err := os.OpenFile(logFilePath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0644) @@ -162,7 +157,7 @@ func startFrontProxy(ctx context.Context, args []string, servingCA *crypto.CA, h } // intentionally load again every iteration because it can change - configLoader := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(&clientcmd.ClientConfigLoadingRules{ExplicitPath: ".kcp/admin.kubeconfig"}, + configLoader := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(&clientcmd.ClientConfigLoadingRules{ExplicitPath: filepath.Join(workDirPath, ".kcp/admin.kubeconfig")}, &clientcmd.ConfigOverrides{CurrentContext: "system:admin"}, ) config, err := configLoader.ClientConfig() @@ -199,24 +194,24 @@ func startFrontProxy(ctx context.Context, args []string, servingCA *crypto.CA, h return nil } -func kcpAdminKubeConfig(ctx context.Context, hostIP string) error { +func writeAdminKubeConfig(hostIP string, workDirPath string) error { baseHost := fmt.Sprintf("https://%s:6443", hostIP) var kubeConfig clientcmdapi.Config kubeConfig.AuthInfos = map[string]*clientcmdapi.AuthInfo{ "kcp-admin": { - ClientKey: ".kcp/kcp-admin.key", - ClientCertificate: ".kcp/kcp-admin.crt", + ClientKey: filepath.Join(workDirPath, ".kcp/kcp-admin.key"), + ClientCertificate: filepath.Join(workDirPath, ".kcp/kcp-admin.crt"), }, } kubeConfig.Clusters = map[string]*clientcmdapi.Cluster{ "root": { Server: baseHost + "/clusters/root", - CertificateAuthority: ".kcp/serving-ca.crt", + CertificateAuthority: filepath.Join(workDirPath, ".kcp/serving-ca.crt"), }, "base": { Server: baseHost, - CertificateAuthority: ".kcp/serving-ca.crt", + CertificateAuthority: filepath.Join(workDirPath, ".kcp/serving-ca.crt"), }, } kubeConfig.Contexts = map[string]*clientcmdapi.Context{ @@ -229,5 +224,5 @@ func kcpAdminKubeConfig(ctx context.Context, hostIP string) error { return err } - return clientcmd.WriteToFile(kubeConfig, ".kcp/admin.kubeconfig") + return clientcmd.WriteToFile(kubeConfig, filepath.Join(workDirPath, ".kcp/admin.kubeconfig")) } diff --git a/cmd/sharded-test-server/main.go b/cmd/sharded-test-server/main.go index 045d56a78760..3b76f6540aa5 100644 --- a/cmd/sharded-test-server/main.go +++ b/cmd/sharded-test-server/main.go @@ -33,9 +33,9 @@ import ( ) func main() { - var numberOfShards int - flag.String("log-dir-path", "", "Path to the log files. If empty, log files are stored in the dot directories.") - flag.IntVar(&numberOfShards, "number-of-shards", 1, "The number of shards to create. The first created is assumed root.") + logDirPath := flag.String("log-dir-path", "", "Path to the log files. If empty, log files are stored in the dot directories.") + workDirPath := flag.String("work-dir-path", "", "Path to the working directory where the .kcp* dot directories are created. If empty, the working directory is the current directory.") + numberOfShards := flag.Int("number-of-shards", 1, "The number of shards to create. The first created is assumed root.") // split flags into --proxy-*, --shard-* and everything elese (generic). The former are // passed to the respective components. @@ -51,13 +51,13 @@ func main() { } flag.CommandLine.Parse(genericFlags) // nolint: errcheck - if err := start(proxyFlags, shardFlags, numberOfShards); err != nil { + if err := start(proxyFlags, shardFlags, *logDirPath, *workDirPath, *numberOfShards); err != nil { fmt.Println(err.Error()) os.Exit(1) } } -func start(proxyFlags, shardFlags []string, numberOfShards int) error { +func start(proxyFlags, shardFlags []string, logDirPath, workDirPath string, numberOfShards int) error { ctx, cancelFn := context.WithCancel(genericapiserver.SetupSignalContext()) defer cancelFn() @@ -110,7 +110,7 @@ func start(proxyFlags, shardFlags []string, numberOfShards int) error { // start shards shardsErrCh := make(chan shardErrTuple) for i := 0; i < numberOfShards; i++ { - shardErrCh, err := startShard(ctx, i, shardFlags, servingCA, hostIP.String()) + shardErrCh, err := startShard(ctx, i, shardFlags, servingCA, hostIP.String(), logDirPath, workDirPath) if err != nil { return err } @@ -122,12 +122,12 @@ func start(proxyFlags, shardFlags []string, numberOfShards int) error { } // write kcp-admin kubeconfig talking to the front-proxy with a client-cert - if err := kcpAdminKubeConfig(ctx, hostIP.String()); err != nil { + if err := writeAdminKubeConfig(hostIP.String(), workDirPath); err != nil { return err } // start front-proxy - if err := startFrontProxy(ctx, proxyFlags, servingCA, hostIP.String()); err != nil { + if err := startFrontProxy(ctx, proxyFlags, servingCA, hostIP.String(), logDirPath, workDirPath); err != nil { return err } diff --git a/cmd/sharded-test-server/shard.go b/cmd/sharded-test-server/shard.go index 6cfb5bff981b..4fa73562101e 100644 --- a/cmd/sharded-test-server/shard.go +++ b/cmd/sharded-test-server/shard.go @@ -18,7 +18,6 @@ package main import ( "context" - "flag" "fmt" "path/filepath" @@ -29,7 +28,7 @@ import ( shard "github.com/kcp-dev/kcp/cmd/test-server/kcp" ) -func startShard(ctx context.Context, n int, args []string, servingCA *crypto.CA, hostIP string) (<-chan error, error) { +func startShard(ctx context.Context, n int, args []string, servingCA *crypto.CA, hostIP string, logDirPath, workDirPath string) (<-chan error, error) { // create serving cert hostnames := sets.NewString("localhost", hostIP) klog.Infof("Creating shard server %d serving cert with hostnames %v", n, hostnames) @@ -37,7 +36,7 @@ func startShard(ctx context.Context, n int, args []string, servingCA *crypto.CA, if err != nil { return nil, fmt.Errorf("failed to create server cert: %w", err) } - if err := cert.WriteCertConfigFile(fmt.Sprintf(".kcp-%d/apiserver.crt", n), fmt.Sprintf(".kcp-%d/apiserver.key", n)); err != nil { + if err := cert.WriteCertConfigFile(fmt.Sprintf(".kcp-%d/apiserver.crt", n), filepath.Join(workDirPath, fmt.Sprintf(".kcp-%d/apiserver.key", n))); err != nil { return nil, fmt.Errorf("failed to write server cert: %w", err) } @@ -46,19 +45,18 @@ func startShard(ctx context.Context, n int, args []string, servingCA *crypto.CA, if err != nil { return nil, fmt.Errorf("failed to create server cert: %w", err) } - if err := cert.WriteCertConfigFile(".kcp/apiserver.crt", ".kcp/apiserver.key"); err != nil { + if err := cert.WriteCertConfigFile(filepath.Join(workDirPath, ".kcp/apiserver.crt"), filepath.Join(workDirPath, ".kcp/apiserver.key")); err != nil { return nil, fmt.Errorf("failed to write server cert: %w", err) } - logDir := flag.Lookup("log-dir-path").Value.String() if err != nil { return nil, err } - logFilePath := filepath.Join(fmt.Sprintf(".kcp-%d", n), "kcp.log") - auditFilePath := filepath.Join(fmt.Sprintf(".kcp-%d", n), "audit.log") - if logDir != "" { - logFilePath = filepath.Join(logDir, fmt.Sprintf("kcp-%d.log", n)) - auditFilePath = filepath.Join(logDir, fmt.Sprintf("audit-%d.log", n)) + logFilePath := filepath.Join(workDirPath, fmt.Sprintf(".kcp-%d", n), "kcp.log") + auditFilePath := filepath.Join(workDirPath, fmt.Sprintf(".kcp-%d", n), "audit.log") + if logDirPath != "" { + logFilePath = filepath.Join(logDirPath, fmt.Sprintf("kcp-%d.log", n)) + auditFilePath = filepath.Join(logDirPath, fmt.Sprintf("audit-%d.log", n)) } if n > 0 { @@ -68,23 +66,23 @@ func startShard(ctx context.Context, n int, args []string, servingCA *crypto.CA, } args = append(args, /*fmt.Sprintf("--cluster-workspace-shard-name=kcp-%d", n),*/ - fmt.Sprintf("--root-directory=.kcp-%d", n), - "--client-ca-file=.kcp/client-ca.crt", - "--requestheader-client-ca-file=.kcp/requestheader-ca.crt", + fmt.Sprintf("--root-directory=%s", filepath.Join(workDirPath, fmt.Sprintf(".kcp-%d", n))), + fmt.Sprintf("--client-ca-file=%s", filepath.Join(workDirPath, ".kcp/client-ca.crt")), + fmt.Sprintf("--requestheader-client-ca-file=%s", filepath.Join(workDirPath, ".kcp/requestheader-ca.crt")), "--requestheader-username-headers=X-Remote-User", "--requestheader-group-headers=X-Remote-Group", - "--service-account-key-file=.kcp/service-account.crt", - "--service-account-private-key-file=.kcp/service-account.key", + fmt.Sprintf("--service-account-key-file=%s", filepath.Join(workDirPath, ".kcp/service-account.crt")), + fmt.Sprintf("--service-account-private-key-file=%s", filepath.Join(workDirPath, ".kcp/service-account.key")), "--audit-log-path", auditFilePath, fmt.Sprintf("--shard-external-url=https://%s:%d", hostIP, 6443), - fmt.Sprintf("--tls-cert-file=.kcp-%d/apiserver.crt", n), - fmt.Sprintf("--tls-private-key-file=.kcp-%d/apiserver.key", n), + fmt.Sprintf("--tls-cert-file=%s", filepath.Join(workDirPath, fmt.Sprintf(".kcp-%d/apiserver.crt", n))), + fmt.Sprintf("--tls-private-key-file=%s", filepath.Join(workDirPath, fmt.Sprintf(".kcp-%d/apiserver.key", n))), fmt.Sprintf("--secure-port=%d", 6444+n), ) return shard.Start(ctx, - fmt.Sprintf("kcp-%d", n), // name - fmt.Sprintf(".kcp-%d", n), // runtime directory, etcd data etc. + fmt.Sprintf("kcp-%d", n), // name + filepath.Join(workDirPath, fmt.Sprintf(".kcp-%d", n)), // runtime directory, etcd data etc. logFilePath, args) }