Skip to content

Commit

Permalink
Makefile/test-e2e-{shared,sharded}: add WORK_DIR
Browse files Browse the repository at this point in the history
  • Loading branch information
@sttts
sttts committed Jul 23, 2022
1 parent 3dca191 commit 2de976f
Show file tree
Hide file tree
Showing 4 changed files with 63 additions and 65 deletions.
31 changes: 18 additions & 13 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -194,39 +194,44 @@ test-e2e-shared: $(GOTESTSUM)
endif
test-e2e-shared: TEST_ARGS ?=
test-e2e-shared: WHAT ?= ./test/e2e...
test-e2e-shared: WORK_DIR ?= .
ifdef ARTIFACT_DIR
test-e2e-shared: LOG_DIR ?= $(ARTIFACT_DIR)/kcp
else
test-e2e-shared: LOG_DIR ?= .kcp
test-e2e-shared: LOG_DIR ?= $(WORK_DIR)/.kcp
endif
test-e2e-shared: build-all
kind get kubeconfig > "$(PWD)/kind.kubeconfig"
mkdir -p $(LOG_DIR)
test-e2e-shared: require-kind build-all build-kind-images
mkdir -p "$(LOG_DIR)" "$(WORK_DIR)/.kcp"
kind get kubeconfig > "$(WORK_DIR)/.kcp/kind.kubeconfig"
rm -f "$(WORK_DIR)/.kcp/admin.kubeconfig"
NO_GORUN=1 ./bin/test-server --log-file-path="$(LOG_DIR)/kcp.log" $(TEST_SERVER_ARGS) 2>&1 & PID=$$! && echo "PID $$PID" && \
trap 'kill -TERM $$PID' TERM INT EXIT && \
while [ ! -f .kcp/admin.kubeconfig ]; do sleep 1; done && \
while [ ! -f "$(WORK_DIR)/.kcp/admin.kubeconfig" ]; do sleep 1; done && \
NO_GORUN=1 $(GO_TEST) -race -count $(COUNT) -p $(E2E_PARALLELISM) -parallel $(E2E_PARALLELISM) $(WHAT) $(TEST_ARGS) \
-args --use-default-kcp-server --syncer-image="$(SYNCER_IMAGE)" --kcp-test-image="$(TEST_IMAGE)" --pcluster-kubeconfig="$(WORK_DIR)/kind.kubeconfig"
-args --use-default-kcp-server --syncer-image="$(SYNCER_IMAGE)" --kcp-test-image="$(TEST_IMAGE)" --pcluster-kubeconfig="$(abspath $(WORK_DIR)/.kcp/kind.kubeconfig)"

.PHONY: test-e2e-sharded
ifdef USE_GOTESTSUM
test-e2e-sharded: $(GOTESTSUM)
endif
test-e2e-sharded: TEST_ARGS ?=
test-e2e-sharded: WHAT ?= ./test/e2e...
test-e2e-sharded: WORK_DIR ?= .
ifdef ARTIFACT_DIR
test-e2e-sharded: LOG_DIR ?= $(ARTIFACT_DIR)/kcp
else
test-e2e-sharded: LOG_DIR ?= .kcp
test-e2e-sharded: LOG_DIR ?= $(WORK_DIR)/.kcp
endif
test-e2e-sharded: require-kind require-ko build-all
kind get kubeconfig > "$(PWD)/kind.kubeconfig"
mkdir -p $(LOG_DIR)
NO_GORUN=1 ./bin/sharded-test-server --v=2 --log-dir-path="$(LOG_DIR)" $(TEST_SERVER_ARGS) 2>&1 & PID=$$!; echo "PID $$PID" && \
test-e2e-sharded: require-kind build-all build-kind-images
mkdir -p "$(LOG_DIR)" "$(WORK_DIR)/.kcp"
kind get kubeconfig > "$(WORK_DIR)/.kcp/kind.kubeconfig"
rm -f "$(WORK_DIR)/.kcp/admin.kubeconfig"
NO_GORUN=1 ./bin/sharded-test-server --v=2 --log-dir-path="$(LOG_DIR)" --work-dir-path="$(WORK_DIR)" $(TEST_SERVER_ARGS) 2>&1 & PID=$$!; echo "PID $$PID" && \
trap 'kill -TERM $$PID' TERM INT EXIT && \
while [ ! -f .kcp/admin.kubeconfig ]; do sleep 1; done && \
while [ ! -f "$(WORK_DIR)/.kcp/admin.kubeconfig" ]; do sleep 1; done && \
NO_GORUN=1 $(GO_TEST) -race -count $(COUNT) -p $(E2E_PARALLELISM) -parallel $(E2E_PARALLELISM) $(WHAT) $(TEST_ARGS) \
-args --use-default-kcp-server --root-shard-kubeconfig=$(PWD)/.kcp-0/admin.kubeconfig --syncer-image="$(SYNCER_IMAGE)" --kcp-test-image="$(TEST_IMAGE)" --pcluster-kubeconfig="$(WORK_DIR)/kind.kubeconfig"
-args --use-default-kcp-server --root-shard-kubeconfig=$(PWD)/.kcp-0/admin.kubeconfig \
--syncer-image="$(SYNCER_IMAGE)" --kcp-test-image="$(TEST_IMAGE)" --pcluster-kubeconfig="$(abspath $(WORK_DIR)/.kcp/kind.kubeconfig)"

.PHONY: test
ifdef USE_GOTESTSUM
Expand Down
45 changes: 20 additions & 25 deletions cmd/sharded-test-server/frontproxy.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ package main

import (
"context"
"flag"
"fmt"
"io/ioutil"
"net/http"
Expand All @@ -42,7 +41,7 @@ import (
"github.com/kcp-dev/kcp/test/e2e/framework"
)

func startFrontProxy(ctx context.Context, args []string, servingCA *crypto.CA, hostIP string) error {
func startFrontProxy(ctx context.Context, args []string, servingCA *crypto.CA, hostIP string, logDirPath, workDirPath string) error {
blue := color.New(color.BgGreen, color.FgBlack).SprintFunc()
inverse := color.New(color.BgHiWhite, color.FgGreen).SprintFunc()
out := lineprefix.New(
Expand All @@ -54,7 +53,7 @@ func startFrontProxy(ctx context.Context, args []string, servingCA *crypto.CA, h
lineprefix.Color(color.New(color.FgHiWhite)),
)

if err := ioutil.WriteFile(".kcp-front-proxy/mapping.yaml", []byte(`
if err := ioutil.WriteFile(filepath.Join(workDirPath, ".kcp-front-proxy/mapping.yaml"), []byte(`
- path: /services/
backend: https://localhost:6444
backend_server_ca: .kcp/serving-ca.crt
Expand All @@ -70,7 +69,7 @@ func startFrontProxy(ctx context.Context, args []string, servingCA *crypto.CA, h
}

// write root shard kubeconfig
configLoader := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(&clientcmd.ClientConfigLoadingRules{ExplicitPath: ".kcp-0/admin.kubeconfig"}, nil)
configLoader := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(&clientcmd.ClientConfigLoadingRules{ExplicitPath: filepath.Join(workDirPath, ".kcp-0/admin.kubeconfig")}, nil)
raw, err := configLoader.RawConfig()
if err != nil {
return err
Expand All @@ -90,32 +89,28 @@ func startFrontProxy(ctx context.Context, args []string, servingCA *crypto.CA, h
if err != nil {
return fmt.Errorf("failed to create server cert: %w\n", err)
}
if err := cert.WriteCertConfigFile(".kcp-front-proxy/apiserver.crt", ".kcp-front-proxy/apiserver.key"); err != nil {
if err := cert.WriteCertConfigFile(filepath.Join(workDirPath, ".kcp-front-proxy/apiserver.crt"), filepath.Join(workDirPath, ".kcp-front-proxy/apiserver.key")); err != nil {
return fmt.Errorf("failed to write server cert: %w\n", err)
}

// run front-proxy command
commandLine := append(framework.DirectOrGoRunCommand("kcp-front-proxy"),
"--mapping-file=.kcp-front-proxy/mapping.yaml",
"--root-directory=.kcp-front-proxy",
"--root-kubeconfig=.kcp/root.kubeconfig",
"--client-ca-file=.kcp/client-ca.crt",
"--tls-cert-file=.kcp-front-proxy/apiserver.crt",
"--tls-private-key-file=.kcp-front-proxy/apiserver.key",
fmt.Sprintf("--mapping-file=%s", filepath.Join(workDirPath, ".kcp-front-proxy/mapping.yaml")),
fmt.Sprintf("--root-directory=%s", filepath.Join(workDirPath, ".kcp-front-proxy")),
fmt.Sprintf("--root-kubeconfig=%s", filepath.Join(workDirPath, ".kcp/root.kubeconfig")),
fmt.Sprintf("--client-ca-file=%s", filepath.Join(workDirPath, ".kcp/client-ca.crt")),
fmt.Sprintf("--tls-cert-file=%s", filepath.Join(workDirPath, ".kcp-front-proxy/apiserver.crt")),
fmt.Sprintf("--tls-private-key-file=%s", filepath.Join(workDirPath, ".kcp-front-proxy/apiserver.key")),
"--secure-port=6443",
)
commandLine = append(commandLine, args...)
fmt.Fprintf(out, "running: %v\n", strings.Join(commandLine, " ")) // nolint: errcheck

cmd := exec.CommandContext(ctx, commandLine[0], commandLine[1:]...)

logDir := flag.Lookup("log-dir-path").Value.String()
if err != nil {
return err
}
logFilePath := ".kcp-front-proxy/proxy.log"
if logDir != "" {
logFilePath = filepath.Join(logDir, "kcp-front-proxy.log")
logFilePath := filepath.Join(workDirPath, ".kcp-front-proxy/proxy.log")
if logDirPath != "" {
logFilePath = filepath.Join(logDirPath, "kcp-front-proxy.log")
}

logFile, err := os.OpenFile(logFilePath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0644)
Expand Down Expand Up @@ -162,7 +157,7 @@ func startFrontProxy(ctx context.Context, args []string, servingCA *crypto.CA, h
}

// intentionally load again every iteration because it can change
configLoader := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(&clientcmd.ClientConfigLoadingRules{ExplicitPath: ".kcp/admin.kubeconfig"},
configLoader := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(&clientcmd.ClientConfigLoadingRules{ExplicitPath: filepath.Join(workDirPath, ".kcp/admin.kubeconfig")},
&clientcmd.ConfigOverrides{CurrentContext: "system:admin"},
)
config, err := configLoader.ClientConfig()
Expand Down Expand Up @@ -199,24 +194,24 @@ func startFrontProxy(ctx context.Context, args []string, servingCA *crypto.CA, h
return nil
}

func kcpAdminKubeConfig(ctx context.Context, hostIP string) error {
func writeAdminKubeConfig(hostIP string, workDirPath string) error {
baseHost := fmt.Sprintf("https://%s:6443", hostIP)

var kubeConfig clientcmdapi.Config
kubeConfig.AuthInfos = map[string]*clientcmdapi.AuthInfo{
"kcp-admin": {
ClientKey: ".kcp/kcp-admin.key",
ClientCertificate: ".kcp/kcp-admin.crt",
ClientKey: filepath.Join(workDirPath, ".kcp/kcp-admin.key"),
ClientCertificate: filepath.Join(workDirPath, ".kcp/kcp-admin.crt"),
},
}
kubeConfig.Clusters = map[string]*clientcmdapi.Cluster{
"root": {
Server: baseHost + "/clusters/root",
CertificateAuthority: ".kcp/serving-ca.crt",
CertificateAuthority: filepath.Join(workDirPath, ".kcp/serving-ca.crt"),
},
"base": {
Server: baseHost,
CertificateAuthority: ".kcp/serving-ca.crt",
CertificateAuthority: filepath.Join(workDirPath, ".kcp/serving-ca.crt"),
},
}
kubeConfig.Contexts = map[string]*clientcmdapi.Context{
Expand All @@ -229,5 +224,5 @@ func kcpAdminKubeConfig(ctx context.Context, hostIP string) error {
return err
}

return clientcmd.WriteToFile(kubeConfig, ".kcp/admin.kubeconfig")
return clientcmd.WriteToFile(kubeConfig, filepath.Join(workDirPath, ".kcp/admin.kubeconfig"))
}
16 changes: 8 additions & 8 deletions cmd/sharded-test-server/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,9 @@ import (
)

func main() {
var numberOfShards int
flag.String("log-dir-path", "", "Path to the log files. If empty, log files are stored in the dot directories.")
flag.IntVar(&numberOfShards, "number-of-shards", 1, "The number of shards to create. The first created is assumed root.")
logDirPath := flag.String("log-dir-path", "", "Path to the log files. If empty, log files are stored in the dot directories.")
workDirPath := flag.String("work-dir-path", "", "Path to the working directory where the .kcp* dot directories are created. If empty, the working directory is the current directory.")
numberOfShards := flag.Int("number-of-shards", 1, "The number of shards to create. The first created is assumed root.")

// split flags into --proxy-*, --shard-* and everything elese (generic). The former are
// passed to the respective components.
Expand All @@ -51,13 +51,13 @@ func main() {
}
flag.CommandLine.Parse(genericFlags) // nolint: errcheck

if err := start(proxyFlags, shardFlags, numberOfShards); err != nil {
if err := start(proxyFlags, shardFlags, *logDirPath, *workDirPath, *numberOfShards); err != nil {
fmt.Println(err.Error())
os.Exit(1)
}
}

func start(proxyFlags, shardFlags []string, numberOfShards int) error {
func start(proxyFlags, shardFlags []string, logDirPath, workDirPath string, numberOfShards int) error {
ctx, cancelFn := context.WithCancel(genericapiserver.SetupSignalContext())
defer cancelFn()

Expand Down Expand Up @@ -110,7 +110,7 @@ func start(proxyFlags, shardFlags []string, numberOfShards int) error {
// start shards
shardsErrCh := make(chan shardErrTuple)
for i := 0; i < numberOfShards; i++ {
shardErrCh, err := startShard(ctx, i, shardFlags, servingCA, hostIP.String())
shardErrCh, err := startShard(ctx, i, shardFlags, servingCA, hostIP.String(), logDirPath, workDirPath)
if err != nil {
return err
}
Expand All @@ -122,12 +122,12 @@ func start(proxyFlags, shardFlags []string, numberOfShards int) error {
}

// write kcp-admin kubeconfig talking to the front-proxy with a client-cert
if err := kcpAdminKubeConfig(ctx, hostIP.String()); err != nil {
if err := writeAdminKubeConfig(hostIP.String(), workDirPath); err != nil {
return err
}

// start front-proxy
if err := startFrontProxy(ctx, proxyFlags, servingCA, hostIP.String()); err != nil {
if err := startFrontProxy(ctx, proxyFlags, servingCA, hostIP.String(), logDirPath, workDirPath); err != nil {
return err
}

Expand Down
36 changes: 17 additions & 19 deletions cmd/sharded-test-server/shard.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ package main

import (
"context"
"flag"
"fmt"
"path/filepath"

Expand All @@ -29,15 +28,15 @@ import (
shard "github.com/kcp-dev/kcp/cmd/test-server/kcp"
)

func startShard(ctx context.Context, n int, args []string, servingCA *crypto.CA, hostIP string) (<-chan error, error) {
func startShard(ctx context.Context, n int, args []string, servingCA *crypto.CA, hostIP string, logDirPath, workDirPath string) (<-chan error, error) {
// create serving cert
hostnames := sets.NewString("localhost", hostIP)
klog.Infof("Creating shard server %d serving cert with hostnames %v", n, hostnames)
cert, err := servingCA.MakeServerCert(hostnames, 365)
if err != nil {
return nil, fmt.Errorf("failed to create server cert: %w", err)
}
if err := cert.WriteCertConfigFile(fmt.Sprintf(".kcp-%d/apiserver.crt", n), fmt.Sprintf(".kcp-%d/apiserver.key", n)); err != nil {
if err := cert.WriteCertConfigFile(fmt.Sprintf(".kcp-%d/apiserver.crt", n), filepath.Join(workDirPath, fmt.Sprintf(".kcp-%d/apiserver.key", n))); err != nil {
return nil, fmt.Errorf("failed to write server cert: %w", err)
}

Expand All @@ -46,19 +45,18 @@ func startShard(ctx context.Context, n int, args []string, servingCA *crypto.CA,
if err != nil {
return nil, fmt.Errorf("failed to create server cert: %w", err)
}
if err := cert.WriteCertConfigFile(".kcp/apiserver.crt", ".kcp/apiserver.key"); err != nil {
if err := cert.WriteCertConfigFile(filepath.Join(workDirPath, ".kcp/apiserver.crt"), filepath.Join(workDirPath, ".kcp/apiserver.key")); err != nil {
return nil, fmt.Errorf("failed to write server cert: %w", err)
}

logDir := flag.Lookup("log-dir-path").Value.String()
if err != nil {
return nil, err
}
logFilePath := filepath.Join(fmt.Sprintf(".kcp-%d", n), "kcp.log")
auditFilePath := filepath.Join(fmt.Sprintf(".kcp-%d", n), "audit.log")
if logDir != "" {
logFilePath = filepath.Join(logDir, fmt.Sprintf("kcp-%d.log", n))
auditFilePath = filepath.Join(logDir, fmt.Sprintf("audit-%d.log", n))
logFilePath := filepath.Join(workDirPath, fmt.Sprintf(".kcp-%d", n), "kcp.log")
auditFilePath := filepath.Join(workDirPath, fmt.Sprintf(".kcp-%d", n), "audit.log")
if logDirPath != "" {
logFilePath = filepath.Join(logDirPath, fmt.Sprintf("kcp-%d.log", n))
auditFilePath = filepath.Join(logDirPath, fmt.Sprintf("audit-%d.log", n))
}

if n > 0 {
Expand All @@ -68,23 +66,23 @@ func startShard(ctx context.Context, n int, args []string, servingCA *crypto.CA,
}
args = append(args,
/*fmt.Sprintf("--cluster-workspace-shard-name=kcp-%d", n),*/
fmt.Sprintf("--root-directory=.kcp-%d", n),
"--client-ca-file=.kcp/client-ca.crt",
"--requestheader-client-ca-file=.kcp/requestheader-ca.crt",
fmt.Sprintf("--root-directory=%s", filepath.Join(workDirPath, fmt.Sprintf(".kcp-%d", n))),
fmt.Sprintf("--client-ca-file=%s", filepath.Join(workDirPath, ".kcp/client-ca.crt")),
fmt.Sprintf("--requestheader-client-ca-file=%s", filepath.Join(workDirPath, ".kcp/requestheader-ca.crt")),
"--requestheader-username-headers=X-Remote-User",
"--requestheader-group-headers=X-Remote-Group",
"--service-account-key-file=.kcp/service-account.crt",
"--service-account-private-key-file=.kcp/service-account.key",
fmt.Sprintf("--service-account-key-file=%s", filepath.Join(workDirPath, ".kcp/service-account.crt")),
fmt.Sprintf("--service-account-private-key-file=%s", filepath.Join(workDirPath, ".kcp/service-account.key")),
"--audit-log-path", auditFilePath,
fmt.Sprintf("--shard-external-url=https://%s:%d", hostIP, 6443),
fmt.Sprintf("--tls-cert-file=.kcp-%d/apiserver.crt", n),
fmt.Sprintf("--tls-private-key-file=.kcp-%d/apiserver.key", n),
fmt.Sprintf("--tls-cert-file=%s", filepath.Join(workDirPath, fmt.Sprintf(".kcp-%d/apiserver.crt", n))),
fmt.Sprintf("--tls-private-key-file=%s", filepath.Join(workDirPath, fmt.Sprintf(".kcp-%d/apiserver.key", n))),
fmt.Sprintf("--secure-port=%d", 6444+n),
)

return shard.Start(ctx,
fmt.Sprintf("kcp-%d", n), // name
fmt.Sprintf(".kcp-%d", n), // runtime directory, etcd data etc.
fmt.Sprintf("kcp-%d", n), // name
filepath.Join(workDirPath, fmt.Sprintf(".kcp-%d", n)), // runtime directory, etcd data etc.
logFilePath,
args)
}

0 comments on commit 2de976f

Please sign in to comment.