From 2f80f1e1db1dec17ead3d7b0cbee30568da32b0e Mon Sep 17 00:00:00 2001 From: Maggie Lou Date: Wed, 13 Nov 2024 10:01:37 -0600 Subject: [PATCH] [ci_runner] Clean up deprecated BazelCommands syntax (#7747) `BazelCommands` was replaced with `Steps`, so that we can support arbitrary bash code in our remote runners. --- docs/workflows-config.md | 129 +++------- enterprise/server/cmd/ci_runner/BUILD | 1 + enterprise/server/cmd/ci_runner/main.go | 230 ++---------------- .../test/integration/remote_bazel/BUILD | 5 + .../remote_bazel/remote_bazel_test.go | 194 +++++++++++---- 5 files changed, 220 insertions(+), 339 deletions(-) diff --git a/docs/workflows-config.md b/docs/workflows-config.md index 97fa1b3781f..2a57f6ecf44 100644 --- a/docs/workflows-config.md +++ b/docs/workflows-config.md @@ -23,7 +23,7 @@ BuildBuddy workflows can be configured using a file called `buildbuddy.yaml`, which can be placed at the root of your git repo. `buildbuddy.yaml` consists of multiple **actions**. Each action describes -a list of bazel commands to be run in order, as well as the set of git +a list of commands to be run in order, as well as the set of git events that should trigger these commands. :::note @@ -47,44 +47,43 @@ actions: pull_request: branches: - "*" - bazel_commands: - - "test //..." + steps: + - run: "bazel test //..." ``` This config is equivalent to the default config that we use if you do not have a `buildbuddy.yaml` file at the root of your repo. -### Running shell scripts +### Running bash commands -It is possible to run shell scripts in BuildBuddy Workflows by declaring -an `sh_binary` target in a `BUILD` file, then running that target as a -step in the `bazel_commands` list: +Each step can run arbitrary bash code, which may be useful for running Bazel commands +conditionally, or for installing system dependencies +that aren't available in BuildBuddy's available workflow images. -```bash title="workflow_setup.sh" -#!/usr/bin/env bash -set -eo pipefail -sudo apt-get update && sudo apt-get install -y my-lib -``` +Because workflows are run in [snapshotted microVMs](./rbe-microvms), system +dependencies will be persisted across workflow runs. However, we recommend +fetching dependencies with Bazel whenever possible, rather than relying +on system dependencies. -```python title="BUILD" -sh_binary(name = "workflow_setup", srcs = ["workflow_setup.sh"]) -``` +To specify multiple bash commands, you can either specify a block of bash code within a single step: ```yaml title="buildbuddy.yaml" -actions: - - name: "Test all targets" - # ... - bazel_commands: - - "run :workflow_setup" # runs workflow_setup.sh with Bazel - - "test //..." +# ... +steps: + - run: | + sudo apt-get update && sudo apt-get install -y my-lib + bazel test //... ``` -Setup scripts are occasionally useful for installing system dependencies -that aren't available in BuildBuddy's available workflow images. Because -workflows are run in [snapshotted microVMs](./rbe-microvms), system -dependencies will be persisted across workflow runs. However, we recommend -fetching dependencies with Bazel whenever possible, rather than relying -on system dependencies. +Or you can specify one command per step. Note that each step is run in a separate +bash process, so locally initialized variables will not persist across steps: + +```yaml title="buildbuddy.yaml" +# ... +steps: + - run: sudo apt-get update && sudo apt-get install -y my-lib + - run: bazel test //... +``` ## Bazel configuration @@ -107,7 +106,7 @@ adding them to your `.bazelrc` instead of adding them to your `buildbuddy.yaml`. BuildBuddy also provides a [`bazelrc`](https://bazel.build/docs/bazelrc) file which passes these default options to each bazel invocation listed in -`bazel_commands`: +`steps`: - `--bes_backend` and `--bes_results_url`, so that the results from each Bazel command are viewable with BuildBuddy @@ -135,7 +134,6 @@ configuration steps are the same as when running Bazel locally. See the Trusted workflow executions can access [secrets](secrets) using environment variables. -Environment variables are expanded inline in the `bazel_commands` list. For example, if we have a secret named `REGISTRY_TOKEN` and we want to set the remote header `x-buildbuddy-platform.container-registry-password` to the value of that secret, we can get the secret value using @@ -143,8 +141,8 @@ the value of that secret, we can get the secret value using ```yaml title="buildbuddy.yaml" # ... -bazel_commands: - - "test ... --remote_exec_header=x-buildbuddy-platform.container-registry-password=$REGISTRY_TOKEN" +steps: + - run: "bazel test ... --remote_exec_header=x-buildbuddy-platform.container-registry-password=$REGISTRY_TOKEN" ``` To access the environment variables within `build` or `test` actions, you @@ -156,57 +154,10 @@ or ```yaml title="buildbuddy.yaml" # ... -bazel_commands: - - "test ... --test_env=REGISTRY_TOKEN" -``` - -### Dynamic bazel flags - -Sometimes, you may wish to set a bazel flag using a shell command. For -example, you might want to set image pull credentials using a command like -`aws` that requests an image pull token on the fly. - -To do this, we recommend using a setup script that generates a `bazelrc` -file. - -For example, in `/buildbuddy.yaml`, you would write: - -```yaml title="buildbuddy.yaml" -# ... -bazel_commands: - - bazel run :generate_ci_bazelrc - - bazel --bazelrc=ci.bazelrc test //... +steps: + - run: "bazel test ... --test_env=REGISTRY_TOKEN" ``` -In `/BUILD`, you'd declare an `sh_binary` target for your setup script: - -```python title="/BUILD" -sh_binary(name = "generate_ci_bazelrc", srcs = ["generate_ci_bazelrc.sh"]) -``` - -Then in `/generate_ci_bazelrc.sh`, you'd generate the `ci.bazelrc` file in -the workspace root (make sure to make this file executable with `chmod +x`): - -```shell title="/generate_ci_bazelrc.sh" -#!/usr/bin/env bash -set -e -# Change to the WORKSPACE directory -cd "$BUILD_WORKSPACE_DIRECTORY" -# Run a command to request image pull credentials: -REGISTRY_PASSWORD=$(some-command) -# Write the credentials to ci.bazelrc in the workspace root directory: -echo >ci.bazelrc " -build --remote_exec_header=x-buildbuddy-platform.container-registry-password=${REGISTRY_PASSWORD} -" -``` - -:::tip - -This `generate_ci_bazelrc.sh` script can access workflow secrets using -environment variables. - -::: - ## Merge queue support BuildBuddy workflows are compatible with GitHub's [merge queues](https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/configuring-pull-request-merges/managing-a-merge-queue). @@ -237,8 +188,8 @@ setting: actions: - name: "Test all targets" container_image: "ubuntu-20.04" # <-- add this line - bazel_commands: - - "bazel test //..." + steps: + - run: "bazel test //..." ``` The supported values for `container_image` are `"ubuntu-18.04"` (default) @@ -304,8 +255,8 @@ actions: pull_request: branches: - "*" - bazel_commands: - - "test //... --bes_backend=remote.buildbuddy.io --bes_results_url=https://app.buildbuddy.io/invocation/" + steps: + - run: "bazel test //..." ``` That's it! Whenever any of the configured triggers are matched, one of @@ -331,8 +282,8 @@ Example `buildbuddy.yaml` configuration: actions: - name: "Test" # ... - bazel_commands: - - "test //... --remote_grpc_log=$BUILDBUDDY_ARTIFACTS_DIRECTORY/grpc.log" + steps: + - run: "bazel test //... --remote_grpc_log=$BUILDBUDDY_ARTIFACTS_DIRECTORY/grpc.log" ``` BuildBuddy creates a new artifacts directory for each Bazel command, and @@ -405,11 +356,11 @@ A named group of Bazel commands that run when triggered. - **`bazel_workspace_dir`** (`string`): A subdirectory within the repo containing the bazel workspace for this action. By default, this is assumed to be the repo root directory. -- **`bazel_commands`** (`string` list): Bazel commands to be run in order. +- **`steps`** (list): Bash commands to be run in order. If a command fails, subsequent ones are not run, and the action is reported as failed. Otherwise, the action is reported as succeeded. - Environment variables are expanded, which means that the bazel command - line can reference [secrets](secrets.md) if the workflow execution + Environment variables are expanded, which means that the commands + can reference [secrets](secrets.md) if the workflow execution is trusted. - **`timeout`** (`duration` string, e.g. '30m', '1h'): If set, workflow actions that have been running for longer than this duration will be canceled automatically. This diff --git a/enterprise/server/cmd/ci_runner/BUILD b/enterprise/server/cmd/ci_runner/BUILD index 13f682edd62..69bdb570d2c 100644 --- a/enterprise/server/cmd/ci_runner/BUILD +++ b/enterprise/server/cmd/ci_runner/BUILD @@ -22,6 +22,7 @@ go_library( "//proto:command_line_go_proto", "//proto:remote_execution_go_proto", "//proto:resource_go_proto", + "//proto:runner_go_proto", "//server/real_environment", "//server/remote_cache/cachetools", "//server/remote_cache/digest", diff --git a/enterprise/server/cmd/ci_runner/main.go b/enterprise/server/cmd/ci_runner/main.go index e615ad1a672..52d4aef171f 100644 --- a/enterprise/server/cmd/ci_runner/main.go +++ b/enterprise/server/cmd/ci_runner/main.go @@ -54,6 +54,7 @@ import ( clpb "github.com/buildbuddy-io/buildbuddy/proto/command_line" repb "github.com/buildbuddy-io/buildbuddy/proto/remote_execution" rspb "github.com/buildbuddy-io/buildbuddy/proto/resource" + rnpb "github.com/buildbuddy-io/buildbuddy/proto/runner" gitutil "github.com/buildbuddy-io/buildbuddy/server/util/git" backendLog "github.com/buildbuddy-io/buildbuddy/server/util/log" bspb "google.golang.org/genproto/googleapis/bytestream" @@ -165,11 +166,8 @@ var ( serializedAction = flag.String("serialized_action", "", "If set, run this b64+yaml encoded action, ignoring trigger conditions.") invocationID = flag.String("invocation_id", "", "If set, use the specified invocation ID for the workflow action. Ignored if action_name is not set.") visibility = flag.String("visibility", "", "If set, use the specified value for VISIBILITY build metadata for the workflow invocation.") - // TODO(Maggie): Deprecate this flag when we clean up `BazelCommands` - bazelSubCommand = flag.String("bazel_sub_command", "", "If set, run the bazel command specified by these args and ignore all triggering and configured actions.") - // TODO(Maggie): Deprecate this flag when we clean up `BazelCommands` - recordRunMetadata = flag.Bool("record_run_metadata", false, "Instead of running a target, extract metadata about it and report it in the build event stream.") - timeout = flag.Duration("timeout", 0, "Timeout before all commands will be canceled automatically.") + bazelSubCommand = flag.String("bazel_sub_command", "", "If set, run the bazel command specified by these args and ignore all triggering and configured actions.") + timeout = flag.Duration("timeout", 0, "Timeout before all commands will be canceled automatically.") // Flags to configure setting up git repo triggerEvent = flag.String("trigger_event", "", "Event type that triggered the action runner.") @@ -765,7 +763,6 @@ func run() error { return status.WrapError(err, "failed to extract bazelisk") } *bazelCommand = bazeliskPath - } // Use the bazel wrapper script, which adds some common flags to all @@ -1048,23 +1045,6 @@ func (ar *actionRunner) Run(ctx context.Context, ws *workspace) error { return status.WrapError(err, "failed to get action to run") } - // If the triggering commit merges cleanly with the target branch, the runner - // will execute the configured bazel commands. Otherwise, the runner will - // exit early without running those commands and does not need to create - // invocation streams for them. - if ws.setupError == nil { - for _, bazelCmd := range action.DeprecatedBazelCommands { - iid, err := newUUID() - if err != nil { - return err - } - wfc.Invocation = append(wfc.Invocation, &bespb.WorkflowConfigured_InvocationMetadata{ - InvocationId: iid, - BazelCommand: bazelCmd, - }) - } - } - if !publishedWorkspaceStatus { if err := ar.reporter.Publish(ar.workspaceStatusEvent()); err != nil { return nil @@ -1098,6 +1078,22 @@ func (ar *actionRunner) Run(ctx context.Context, ws *workspace) error { } }() + // Migrate deprecated `action.BazelCommands` to `action.Steps` + if len(action.Steps) > 0 && len(action.DeprecatedBazelCommands) > 0 { + return status.InternalError("only one of `Steps` or `BazelCommands` should be set") + } + if len(action.Steps) == 0 { + action.Steps = make([]*rnpb.Step, 0) + } + for _, cmd := range action.DeprecatedBazelCommands { + if !(strings.HasPrefix(cmd, bazeliskBinaryName) || strings.HasPrefix(cmd, bazelBinaryName)) { + cmd = "bazel " + cmd + } + action.Steps = append(action.Steps, &rnpb.Step{ + Run: cmd, + }) + } + for i, step := range action.Steps { cmdStartTime := time.Now() @@ -1118,12 +1114,10 @@ func (ar *actionRunner) Run(ctx context.Context, ws *workspace) error { return err } - if *recordRunMetadata { - // Provision the directory where we write bazel run scripts - runScriptDir := filepath.Join(ws.rootDir, runScriptDirName) - if err := os.MkdirAll(runScriptDir, 0755); err != nil { - return err - } + // Provision the directory where we write bazel run scripts + runScriptDir := filepath.Join(ws.rootDir, runScriptDirName) + if err := os.MkdirAll(runScriptDir, 0755); err != nil { + return err } runErr := runBashCommand(ctx, step.Run, nil, action.BazelWorkspaceDir, ar.reporter) @@ -1194,7 +1188,6 @@ func (ar *actionRunner) Run(ctx context.Context, ws *workspace) error { // If extracting run information from builds was requested, // extract it and send it via the event stream. - runScriptDir := filepath.Join(ws.rootDir, runScriptDirName) if _, err = os.Stat(runScriptDir); err == nil { err = filepath.Walk(runScriptDir, func(path string, info os.FileInfo, err error) error { if err != nil { @@ -1254,164 +1247,6 @@ func (ar *actionRunner) Run(ctx context.Context, ws *workspace) error { } } - // TODO(Maggie): Consolidate action.BazelCommands with action.Steps - for i, bazelCmd := range action.DeprecatedBazelCommands { - cmdStartTime := time.Now() - if i >= len(wfc.GetInvocation()) { - return status.InternalErrorf("No invocation metadata generated for bazel_commands[%d]; this should never happen", i) - } - iid := wfc.GetInvocation()[i].GetInvocationId() - - // Publish a TargetConfigured event associated with the bazel command so - // that we can render artifacts associated with the "target". - targetLabel := fmt.Sprintf("bazel_commands[%d]", i) - ar.reporter.Publish(&bespb.BuildEvent{ - Id: &bespb.BuildEventId{Id: &bespb.BuildEventId_TargetConfigured{ - TargetConfigured: &bespb.BuildEventId_TargetConfiguredId{ - Label: targetLabel, - }, - }}, - Payload: &bespb.BuildEvent_Configured{Configured: &bespb.TargetConfigured{}}, - }) - - if err := provisionArtifactsDir(ws, i); err != nil { - return err - } - - args, err := ws.bazelArgsWithCustomBazelrc(bazelCmd) - if err != nil { - return status.InvalidArgumentErrorf("failed to parse bazel command: %s", err) - } - if err := printCommandLine(ar.reporter, *bazelCommand, args...); err != nil { - return err - } - // Transparently set the invocation ID from the one we computed ahead of - // time. The UI is expecting this invocation ID so that it can render a - // BuildBuddy invocation URL for each bazel_command that is executed. - args = appendBazelSubcommandArgs(args, fmt.Sprintf("--invocation_id=%s", iid)) - - // Instead of actually running the target, have Bazel write out a run script using the --script_path flag and - // extract run options (i.e. args, runfile information) from the generated run script. - runScript := "" - isRunCmd := getBazelCommand(args) == "run" - if isRunCmd && *recordRunMetadata { - tmpDir, err := os.MkdirTemp("", "bazel-run-script-*") - if err != nil { - return err - } - defer os.RemoveAll(tmpDir) - runScript = filepath.Join(tmpDir, "run.sh") - args = appendBazelSubcommandArgs(args, "--script_path="+runScript) - } - - artifactsDir := artifactsPathForCommand(ws, i) - namedSetID := filepath.Base(artifactsDir) - - runErr := runCommand(ctx, *bazelCommand, expandEnv(args), nil, action.BazelWorkspaceDir, ar.reporter) - exitCode := getExitCode(runErr) - ar.reporter.Publish(&bespb.BuildEvent{ - Id: &bespb.BuildEventId{Id: &bespb.BuildEventId_TargetCompleted{ - TargetCompleted: &bespb.BuildEventId_TargetCompletedId{ - Label: targetLabel, - }, - }}, - Payload: &bespb.BuildEvent_Completed{Completed: &bespb.TargetComplete{ - Success: runErr == nil, - OutputGroup: []*bespb.OutputGroup{ - { - FileSets: []*bespb.BuildEventId_NamedSetOfFilesId{ - {Id: namedSetID}, - }, - }, - }, - }}, - }) - if exitCode != noExitCode { - ar.reporter.Printf("%s(command exited with code %d)%s\n", ansiGray, exitCode, ansiReset) - } - - // If this is a workflow, kill-signal the current process on certain - // exit codes (rather than exiting) so that the workflow action is - // retried. Note that we do this immediately after the Bazel command is - // completed so that the outer workflow invocation gets disconnected - // rather than finishing with an error. - if *workflowID != "" && exitCode == bazelLocalEnvironmentalErrorExitCode { - p, err := os.FindProcess(os.Getpid()) - if err != nil { - return err - } - if err := p.Kill(); err != nil { - return err - } - } - - // If we get an OOM or a Bazel internal error, copy debug outputs to the - // artifacts directory so they get uploaded as workflow artifacts. - if *workflowID != "" && (exitCode == bazelOOMErrorExitCode || exitCode == bazelInternalErrorExitCode) { - jvmOutPath := filepath.Join(ar.rootDir, outputBaseDirName, "server/jvm.out") - if err := os.Link(jvmOutPath, filepath.Join(artifactsDir, "jvm.out")); err != nil { - ar.reporter.Printf("%sfailed to preserve jvm.out: %s%s\n", ansiGray, err, ansiReset) - } - } - if *workflowID != "" && exitCode == bazelOOMErrorExitCode { - heapDumpPath := filepath.Join(ar.rootDir, outputBaseDirName, iid+".heapdump.hprof") - if err := os.Link(heapDumpPath, filepath.Join(artifactsDir, "heapdump.hprof")); err != nil { - ar.reporter.Printf("%sfailed to preserve heapdump.hprof: %s%s\n", ansiGray, err, ansiReset) - } - } - - // Kick off background uploads for the action that just completed - if uploader != nil { - uploader.UploadDirectory(namedSetID, artifactsDir) // does not return an error - } - - // If this is a successfully "bazel run" invocation from which we are extracting run information via - // --script_path, go ahead and extract run information from the script and send it via the event stream. - if exitCode == 0 && runScript != "" { - runInfo, err := processRunScript(ctx, runScript) - if err != nil { - return err - } - e := &bespb.BuildEvent{ - Id: &bespb.BuildEventId{Id: &bespb.BuildEventId_RunTargetAnalyzed{}}, - Payload: &bespb.BuildEvent_RunTargetAnalyzed{RunTargetAnalyzed: &bespb.RunTargetAnalyzed{ - Arguments: runInfo.args, - RunfilesRoot: runInfo.runfilesRoot, - Runfiles: runInfo.runfiles, - RunfileDirectories: runInfo.runfileDirs, - }}, - } - if err := ar.reporter.Publish(e); err != nil { - break - } - } - - duration := time.Since(cmdStartTime) - completedEvent := &bespb.BuildEvent{ - Id: &bespb.BuildEventId{Id: &bespb.BuildEventId_RemoteRunnerStepCompleted{RemoteRunnerStepCompleted: &bespb.BuildEventId_RemoteRunnerStepCompletedId{}}}, - Payload: &bespb.BuildEvent_RemoteRunnerStepCompleted{RemoteRunnerStepCompleted: &bespb.RemoteRunnerStepCompleted{ - ExitCode: int32(exitCode), - StartTime: timestamppb.New(cmdStartTime), - Duration: durationpb.New(duration), - }}, - } - if err := ar.reporter.Publish(completedEvent); err != nil { - break - } - - if runErr != nil { - // Return early if the command failed. - // Note, even though we don't hit the `FlushProgress` call below in this case, - // we'll still flush progress before closing the BEP stream. - return runErr - } - - // Flush progress after every command. - // Stop execution early on BEP failure, but ignore error -- it will surface in `bep.Finish()`. - if err := ar.reporter.FlushProgress(); err != nil { - break - } - } return nil } @@ -1439,17 +1274,6 @@ func (ar *actionRunner) workspaceStatusEvent() *bespb.BuildEvent { } } -// TODO: Clean up when we clean up `BazelCommands` -// Returns the bazel command - i.e. `build` or `run` -func getBazelCommand(bazelArgs []string) string { - for _, arg := range bazelArgs { - if !strings.HasPrefix(arg, "--") { - return arg - } - } - return "" -} - // This should only be used for WorkflowConfiguredEvents--it explicitly labels // actions with no name so that they can be identified later on. func getActionNameForWorkflowConfiguredEvent() (string, error) { @@ -2538,14 +2362,6 @@ func runCommand(ctx context.Context, executable string, args []string, env map[s return err } -func expandEnv(args []string) []string { - out := make([]string, 0, len(args)) - for _, arg := range args { - out = append(out, os.ExpandEnv(arg)) - } - return out -} - func getExitCode(err error) int { if err == nil { return 0 diff --git a/enterprise/server/test/integration/remote_bazel/BUILD b/enterprise/server/test/integration/remote_bazel/BUILD index 785818fa07c..fbb03460dd1 100644 --- a/enterprise/server/test/integration/remote_bazel/BUILD +++ b/enterprise/server/test/integration/remote_bazel/BUILD @@ -27,10 +27,13 @@ go_test( ], deps = [ "//cli/remotebazel", + "//enterprise/server/backends/kms", "//enterprise/server/execution_service", "//enterprise/server/hostedrunner", "//enterprise/server/invocation_search_service", + "//enterprise/server/secrets", "//enterprise/server/test/integration/remote_execution/rbetest", + "//enterprise/server/util/keystore", "//enterprise/server/workflow/service", "//proto:api_key_go_proto", "//proto:buildbuddy_service_go_proto", @@ -38,12 +41,14 @@ go_test( "//proto:eventlog_go_proto", "//proto:invocation_go_proto", "//proto:invocation_status_go_proto", + "//proto:secrets_go_proto", "//proto:user_id_go_proto", "//server/backends/memory_kvstore", "//server/backends/repo_downloader", "//server/interfaces", "//server/tables", "//server/testutil/testenv", + "//server/testutil/testfs", "//server/testutil/testgit", "//server/testutil/testshell", "//server/util/bazel", diff --git a/enterprise/server/test/integration/remote_bazel/remote_bazel_test.go b/enterprise/server/test/integration/remote_bazel/remote_bazel_test.go index d095cfe366c..c52b719799b 100644 --- a/enterprise/server/test/integration/remote_bazel/remote_bazel_test.go +++ b/enterprise/server/test/integration/remote_bazel/remote_bazel_test.go @@ -3,6 +3,7 @@ package remote_bazel_test import ( "bytes" "context" + "crypto/rand" "fmt" "math" "os" @@ -13,16 +14,20 @@ import ( "time" "github.com/buildbuddy-io/buildbuddy/cli/remotebazel" + "github.com/buildbuddy-io/buildbuddy/enterprise/server/backends/kms" "github.com/buildbuddy-io/buildbuddy/enterprise/server/execution_service" "github.com/buildbuddy-io/buildbuddy/enterprise/server/hostedrunner" "github.com/buildbuddy-io/buildbuddy/enterprise/server/invocation_search_service" + "github.com/buildbuddy-io/buildbuddy/enterprise/server/secrets" "github.com/buildbuddy-io/buildbuddy/enterprise/server/test/integration/remote_execution/rbetest" + "github.com/buildbuddy-io/buildbuddy/enterprise/server/util/keystore" "github.com/buildbuddy-io/buildbuddy/enterprise/server/workflow/service" "github.com/buildbuddy-io/buildbuddy/server/backends/memory_kvstore" "github.com/buildbuddy-io/buildbuddy/server/backends/repo_downloader" "github.com/buildbuddy-io/buildbuddy/server/interfaces" "github.com/buildbuddy-io/buildbuddy/server/tables" "github.com/buildbuddy-io/buildbuddy/server/testutil/testenv" + "github.com/buildbuddy-io/buildbuddy/server/testutil/testfs" "github.com/buildbuddy-io/buildbuddy/server/testutil/testgit" "github.com/buildbuddy-io/buildbuddy/server/testutil/testshell" "github.com/buildbuddy-io/buildbuddy/server/util/bazel" @@ -36,6 +41,7 @@ import ( elpb "github.com/buildbuddy-io/buildbuddy/proto/eventlog" inpb "github.com/buildbuddy-io/buildbuddy/proto/invocation" inspb "github.com/buildbuddy-io/buildbuddy/proto/invocation_status" + spb "github.com/buildbuddy-io/buildbuddy/proto/secrets" uidpb "github.com/buildbuddy-io/buildbuddy/proto/user_id" ) @@ -45,7 +51,6 @@ func init() { // startup. Silence the logs to remove the race. *log.LogLevel = "warn" log.Configure() - } func waitForInvocationCreated(t *testing.T, ctx context.Context, bb bbspb.BuildBuddyServiceClient, reqCtx *ctxpb.RequestContext) { @@ -135,7 +140,7 @@ func TestWithPublicRepo(t *testing.T) { require.NoError(t, err) // Run a server and executor locally to run remote bazel against - env, bbServer, _ := runLocalServerAndExecutor(t, "") + env, bbServer, _ := runLocalServerAndExecutor(t, "", "https://github.com/bazelbuild/bazel-gazelle", nil) ctx := env.WithUserID(context.Background(), env.UserID1) reqCtx := &ctxpb.RequestContext{ UserId: &uidpb.UserId{Id: env.UserID1}, @@ -189,16 +194,7 @@ func TestWithPrivateRepo(t *testing.T) { personalAccessToken := os.Getenv("PRIVATE_TEST_REPO_GIT_ACCESS_TOKEN") // Run a server and executor locally to run remote bazel against - env, bbServer, _ := runLocalServerAndExecutor(t, personalAccessToken) - - // Create a workflow for the same repo - will be used to fetch the git token - dbh := env.GetDBHandle() - require.NotNil(t, dbh) - err := dbh.NewQuery(context.Background(), "create_git_repo_for_test").Create(&tables.GitRepository{ - RepoURL: "https://github.com/buildbuddy-io/private-test-repo", - GroupID: env.GroupID1, - }) - require.NoError(t, err) + env, bbServer, _ := runLocalServerAndExecutor(t, personalAccessToken, "https://github.com/buildbuddy-io/private-test-repo", nil) // Run remote bazel exitCode, err := remotebazel.HandleRemoteBazel([]string{ @@ -246,7 +242,7 @@ func TestWithPrivateRepo(t *testing.T) { require.Contains(t, string(logResp.GetBuffer()), "FUTURE OF BUILDS!") } -func runLocalServerAndExecutor(t *testing.T, githubToken string) (*rbetest.Env, *rbetest.BuildBuddyServer, *rbetest.Executor) { +func runLocalServerAndExecutor(t *testing.T, githubToken string, repoURL string, envModifier func(rbeEnv *rbetest.Env, e *testenv.TestEnv)) (*rbetest.Env, *rbetest.BuildBuddyServer, *rbetest.Executor) { env := rbetest.NewRBETestEnv(t) bbServer := env.AddBuildBuddyServerWithOptions(&rbetest.BuildBuddyServerOptions{ EnvModifier: func(e *testenv.TestEnv) { @@ -265,6 +261,10 @@ func runLocalServerAndExecutor(t *testing.T, githubToken string) (*rbetest.Env, require.NoError(t, err) e.SetKeyValStore(keyValStore) e.SetExecutionService(execution_service.NewExecutionService(e)) + + if envModifier != nil { + envModifier(env, e) + } }, }) @@ -272,6 +272,15 @@ func runLocalServerAndExecutor(t *testing.T, githubToken string) (*rbetest.Env, require.Equal(t, 1, len(executors)) flags.Set(t, "executor.enable_bare_runner", true) + // Create a workflow for the repo - will be used to fetch the git token + dbh := env.GetDBHandle() + require.NotNil(t, dbh) + err := dbh.NewQuery(context.Background(), "create_git_repo_for_test").Create(&tables.GitRepository{ + RepoURL: repoURL, + GroupID: env.GroupID1, + }) + require.NoError(t, err) + return env, bbServer, executors[0] } @@ -281,22 +290,13 @@ func TestCancel(t *testing.T) { personalAccessToken := os.Getenv("PRIVATE_TEST_REPO_GIT_ACCESS_TOKEN") // Run a server and executor locally to run remote bazel against - env, bbServer, _ := runLocalServerAndExecutor(t, personalAccessToken) + env, bbServer, _ := runLocalServerAndExecutor(t, personalAccessToken, "https://github.com/buildbuddy-io/private-test-repo", nil) ctx := env.WithUserID(context.Background(), env.UserID1) reqCtx := &ctxpb.RequestContext{ UserId: &uidpb.UserId{Id: env.UserID1}, GroupId: env.GroupID1, } - // Create a workflow for the same repo - will be used to fetch the git token - dbh := env.GetDBHandle() - require.NotNil(t, dbh) - err := dbh.NewQuery(context.Background(), "create_git_repo_for_test").Create(&tables.GitRepository{ - RepoURL: "https://github.com/buildbuddy-io/private-test-repo", - GroupID: env.GroupID1, - }) - require.NoError(t, err) - // Get an API key to authenticate the remote bazel request bbClient := env.GetBuildBuddyServiceClient() apiRsp, err := bbClient.CreateApiKey(ctx, &akpb.CreateApiKeyRequest{ @@ -364,16 +364,7 @@ func TestFetchRemoteBuildOutputs(t *testing.T) { // Run a server and executor locally to run remote bazel against personalAccessToken := os.Getenv("PRIVATE_TEST_REPO_GIT_ACCESS_TOKEN") - env, bbServer, _ := runLocalServerAndExecutor(t, personalAccessToken) - - // Create a workflow for the same repo - will be used to fetch the git token - dbh := env.GetDBHandle() - require.NotNil(t, dbh) - err := dbh.NewQuery(context.Background(), "create_git_repo_for_test").Create(&tables.GitRepository{ - RepoURL: "https://github.com/buildbuddy-io/private-test-repo", - GroupID: env.GroupID1, - }) - require.NoError(t, err) + env, bbServer, _ := runLocalServerAndExecutor(t, personalAccessToken, "https://github.com/buildbuddy-io/private-test-repo", nil) // Run remote bazel randomStr := fmt.Sprintf("%d", time.Now().UnixMilli()) @@ -437,16 +428,7 @@ func TestBuildRemotelyRunLocally(t *testing.T) { // Run a server and executor locally to run remote bazel against personalAccessToken := os.Getenv("PRIVATE_TEST_REPO_GIT_ACCESS_TOKEN") - env, bbServer, _ := runLocalServerAndExecutor(t, personalAccessToken) - - // Create a workflow for the same repo - will be used to fetch the git token - dbh := env.GetDBHandle() - require.NotNil(t, dbh) - err := dbh.NewQuery(context.Background(), "create_git_repo_for_test").Create(&tables.GitRepository{ - RepoURL: "https://github.com/buildbuddy-io/private-test-repo", - GroupID: env.GroupID1, - }) - require.NoError(t, err) + env, bbServer, _ := runLocalServerAndExecutor(t, personalAccessToken, "https://github.com/buildbuddy-io/private-test-repo", nil) // Run remote bazel randomStr := fmt.Sprintf("%d", time.Now().UnixMilli()) @@ -498,3 +480,129 @@ func TestBuildRemotelyRunLocally(t *testing.T) { require.NoError(t, err) require.NotContains(t, string(logResp.GetBuffer()), "Hello! I'm a go program.") } + +func TestAccessingSecrets(t *testing.T) { + clonePrivateTestRepo(t) + + initSecretService, pubKey := setupSecrets(t) + + // Run a server and executor locally to run remote bazel against + personalAccessToken := os.Getenv("PRIVATE_TEST_REPO_GIT_ACCESS_TOKEN") + env, bbServer, _ := runLocalServerAndExecutor(t, personalAccessToken, "https://github.com/buildbuddy-io/private-test-repo", initSecretService) + + bbClient := env.GetBuildBuddyServiceClient() + ctx := env.WithUserID(context.Background(), env.UserID1) + reqCtx := &ctxpb.RequestContext{ + UserId: &uidpb.UserId{Id: env.UserID1}, + GroupId: env.GroupID1, + } + + // Save a secret + saveSecret(t, bbClient, ctx, reqCtx, *pubKey, "SECRET_TARGET", ":hello_world") + + // Run remote bazel + exitCode, err := remotebazel.HandleRemoteBazel([]string{ + fmt.Sprintf("--remote_runner=%s", bbServer.GRPCAddress()), + // Have the ci runner use the "none" isolation type because it's simpler + // to setup than a firecracker runner + "--runner_exec_properties=workload-isolation-type=none", + "--runner_exec_properties=container-image=", + // Initialize secrets as env vars on the runner + "--runner_exec_properties=include-secrets=true", + "--run_remotely=1", + "run", + "$SECRET_TARGET", + "--noenable_bzlmod", + fmt.Sprintf("--remote_header=x-buildbuddy-api-key=%s", env.APIKey1)}) + require.NoError(t, err) + require.Equal(t, 0, exitCode) + + // Check the invocation logs to ensure the bazel command successfully ran + searchRsp, err := bbClient.SearchInvocation(ctx, &inpb.SearchInvocationRequest{ + RequestContext: reqCtx, + Query: &inpb.InvocationQuery{GroupId: env.GroupID1}, + }) + require.NoError(t, err) + + require.Equal(t, 2, len(searchRsp.GetInvocation())) + // Find outer invocation because it will contain run output + var inv *inpb.Invocation + for _, i := range searchRsp.GetInvocation() { + if i.GetRole() == "HOSTED_BAZEL" { + inv = i + } + } + invocationID := inv.InvocationId + + logResp, err := bbClient.GetEventLogChunk(ctx, &elpb.GetEventLogChunkRequest{ + InvocationId: invocationID, + MinLines: math.MaxInt32, + }) + require.NoError(t, err) + require.Contains(t, string(logResp.GetBuffer()), "Build completed successfully") + require.Contains(t, string(logResp.GetBuffer()), "FUTURE OF BUILDS!") +} + +func setupSecrets(t *testing.T) (func(*rbetest.Env, *testenv.TestEnv), *string) { + // Generate the master key + masterKey := make([]byte, 32) + _, err := rand.Read(masterKey) + require.NoError(t, err) + + // Write the master key + masterKeyFile, err := os.OpenFile( + testfs.MakeTempFile(t, testfs.MakeTempDir(t), "master-key-*"), + os.O_WRONLY, + 0, + ) + require.NoError(t, err) + _, err = masterKeyFile.Write(masterKey) + require.NoError(t, err) + err = masterKeyFile.Close() + require.NoError(t, err) + + flags.Set(t, "keystore.master_key_uri", "local-insecure-kms://"+filepath.Base(masterKeyFile.Name())) + flags.Set(t, "keystore.local_insecure_kms_directory", filepath.Dir(masterKeyFile.Name())) + flags.Set(t, "app.enable_secret_service", true) + + pubKeyPtr := new(string) + // Generate a function to initialize the secret service within the server + initSecretService := func(publicEnv *rbetest.Env, e *testenv.TestEnv) { + err = kms.Register(e) + require.NoError(t, err) + err = secrets.Register(e) + require.NoError(t, err) + + pubKey, encPrivKey, err := keystore.GenerateSealedBoxKeys(e) + require.NoError(t, err) + *pubKeyPtr = pubKey + + res := e.GetDBHandle().NewQuery(context.Background(), "update_group_keys_for_test").Raw(` + UPDATE "Groups" SET + public_key = ?, + encrypted_private_key = ? + WHERE group_id = ?`, + pubKey, + encPrivKey, + publicEnv.GroupID1, + ).Exec() + require.NoError(t, res.Error) + } + + return initSecretService, pubKeyPtr +} + +func saveSecret(t *testing.T, bbClient bbspb.BuildBuddyServiceClient, ctx context.Context, reqCtx *ctxpb.RequestContext, publicKey, key, val string) { + encValue, err := keystore.NewAnonymousSealedBox(publicKey, val) + require.NoError(t, err) + + require.NoError(t, err) + _, err = bbClient.UpdateSecret(ctx, &spb.UpdateSecretRequest{ + RequestContext: reqCtx, + Secret: &spb.Secret{ + Name: key, + Value: encValue, + }, + }) + require.NoError(t, err) +}