Skip to content

Commit

Permalink
chore: align ci patterns
Browse files Browse the repository at this point in the history
  • Loading branch information
gregmagolan committed May 14, 2024
1 parent c6c9426 commit 37d7495
Show file tree
Hide file tree
Showing 16 changed files with 223 additions and 153 deletions.
1 change: 1 addition & 0 deletions .aspect/bazelrc/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
user.bazelrc
1 change: 1 addition & 0 deletions .bazelrc
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import %workspace%/.aspect/bazelrc/performance.bazelrc
# For testing our --stamp behavior.
# Normally users would use a --workspace_status_command with a script that calls `git describe`.
build --embed_label=v1.2.3

# Mock versioning command to test the --stamp behavior
build --workspace_status_command="echo BUILD_SCM_VERSION 1.2.3"

Expand Down
17 changes: 8 additions & 9 deletions .github/workflows/ci.bazelrc
Original file line number Diff line number Diff line change
@@ -1,21 +1,20 @@
# Bazel options included on CI using the --bazelrc Bazel flag
# Directories caches by GitHub actions
common:local --disk_cache=~/.cache/bazel-disk-cache
common --repository_cache=~/.cache/bazel-repository-cache

# CI specific caching options
build --repository_cache=~/.cache/bazel-repo
test --test_env=XDG_CACHE_HOME
# Debug where options came from
common --announce_rc

# When no remote cache, use a local one
build:local --disk_cache=~/.cache/bazel
# Allows tests to run bazelisk-in-bazel, since this is the cache folder used
common --test_env=XDG_CACHE_HOME

# Remote build execution
build:rbe --extra_execution_platforms=@aspect_bazel_lib//platforms:x86_64_linux_remote
build:rbe --genrule_strategy=remote
build:rbe --host_platform=@aspect_bazel_lib//platforms:x86_64_linux_remote
build:rbe --jobs=32

# BuildBuddy remote exec
build:rbe --bes_results_url=https://app.buildbuddy.io/invocation/
build:rbe --bes_backend=grpcs://remote.buildbuddy.io
build:rbe --remote_timeout=3600
build:rbe --remote_executor=grpcs://remote.buildbuddy.io

test --test_output=errors
176 changes: 87 additions & 89 deletions .github/workflows/ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -18,78 +18,66 @@ concurrency:
cancel-in-progress: ${{ github.ref_name != 'main' }}

jobs:
# matrix-prep-* steps dynamically generate a bit of JSON depending on whether our action has
# access to repository secrets. When running on a pull_request from a fork, the author is
# untrusted so the secret will be absent. Insanely complex for how simple this requirement is...
# inspired from
# https://stackoverflow.com/questions/65384420/how-to-make-a-github-action-matrix-element-conditional

matrix-prep-config:
# Prepares the 'config' axis of the test matrix
runs-on: ubuntu-latest
env:
BUILDBUDDY_API_KEY: ${{ secrets.BUILDBUDDY_API_KEY }}
steps:
- id: local
run: echo "config=local" >> $GITHUB_OUTPUT
- id: rbe
run: echo "config=rbe" >> $GITHUB_OUTPUT
# Don't run RBE if there is no API key which is the case on forks
if: ${{ env.BUILDBUDDY_API_KEY != '' }}
outputs:
# Will look like '["local", "rbe"]'
configs: ${{ toJSON(steps.*.outputs.config) }}

matrix-prep-bazelversion:
# Prepares the 'bazelversion' axis of the test matrix
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- id: bazel_7
run: echo "bazelversion=$(head -n 1 .bazelversion)" >> $GITHUB_OUTPUT
- id: bazel_6
run: echo "bazelversion=6.5.0" >> $GITHUB_OUTPUT
outputs:
# Will look like '["<version from .bazelversion>", ...]'
bazelversions: ${{ toJSON(steps.*.outputs.bazelversion) }}

matrix-prep-os:
# Prepares the 'os' axis of the test matrix
# Prepares dynamic test matrix values
matrix-prep:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- id: linux
run: echo "os=ubuntu-latest" >> $GITHUB_OUTPUT
- id: macos
run: echo "os=macos-latest" >> $GITHUB_OUTPUT
# Only run on main branch (or PR branches that contain 'macos') to minimize macOS minutes (billed at 10X)
# https://docs.github.com/en/billing/managing-billing-for-github-actions/about-billing-for-github-actions#included-storage-and-minutes
if: ${{ github.ref == 'refs/heads/main' || contains(github.head_ref, 'macos') }}
- id: windows
run: echo "os=windows-latest" >> $GITHUB_OUTPUT
# Only run on main branch (or PR branches that contain 'windows') to minimize Windows minutes (billed at 2X)
- id: bazel-version
name: Prepare 'bazel-version' matrix axis
run: |
v=$(head -n 1 .bazelversion)
m=${v::1}
a=(
"major:$m, version:\"$v\""
"major:6, version:\"6.5.0\""
)
printf -v j '{%s},' "${a[@]}"
echo "res=[${j%,}]" | tee -a $GITHUB_OUTPUT
- id: config
name: Prepare 'config' matrix axis
# Don't run RBE if there is no API key which is the case on forks.
run: |
a=( local )
if [[ "${{ github.ref_name }}" == "main" ]] || [[ "${{ github.head_ref }}" == *"rbe"* ]]; then
if [[ "${{ env.BUILDBUDDY_API_KEY }}" ]]; then
a+=( rbe )
fi
fi
printf -v j '"%s",' "${a[@]}"
echo "res=[${j%,}]" | tee -a $GITHUB_OUTPUT
env:
BUILDBUDDY_API_KEY: ${{ secrets.BUILDBUDDY_API_KEY }}
- id: os
name: Prepare 'os' matrix axis
# Only run MacOS and Windows on main branch (not PRs) to minimize minutes (billed at 10X and 2X respectively)
# https://docs.github.com/en/billing/managing-billing-for-github-actions/about-billing-for-github-actions#included-storage-and-minutes
if: ${{ github.ref == 'refs/heads/main' || contains(github.head_ref, 'windows') }}
run: |
a=( ubuntu )
if [[ "${{ github.ref_name }}" == "main" ]] || [[ "${{ github.head_ref }}" == *"macos"* ]]; then
a+=( macos )
fi
if [[ "${{ github.ref_name }}" == "main" ]] || [[ "${{ github.head_ref }}" == *"windows"* ]]; then
a+=( windows )
fi
printf -v j '"%s",' "${a[@]}"
echo "res=[${j%,}]" | tee -a $GITHUB_OUTPUT
outputs:
# Will look like ["ubuntu-latest", "macos-latest", "windows-latest"]
os: ${{ toJSON(steps.*.outputs.os) }}
bazel-version: ${{ steps.bazel-version.outputs.res }}
config: ${{ steps.config.outputs.res }}
os: ${{ steps.os.outputs.res }}

test:
# The type of runner that the job will run on
runs-on: ${{ matrix.os }}

runs-on: ${{ matrix.os }}-latest
needs:
- matrix-prep-config
- matrix-prep-bazelversion
- matrix-prep-os

- matrix-prep
strategy:
fail-fast: false
matrix:
config: ${{ fromJSON(needs.matrix-prep-config.outputs.configs) }}
bazelversion: ${{ fromJSON(needs.matrix-prep-bazelversion.outputs.bazelversions) }}
os: ${{ fromJSON(needs.matrix-prep-os.outputs.os) }}
bazel-version: ${{ fromJSON(needs.matrix-prep.outputs.bazel-version) }}
bzlmod: [1, 0]
os: ${{ fromJSON(needs.matrix-prep.outputs.os) }}
config: ${{ fromJSON(needs.matrix-prep.outputs.config) }}
folder:
- "."
- "e2e/copy_action"
Expand All @@ -99,35 +87,46 @@ jobs:
- "e2e/smoke"
- "e2e/write_source_files"
exclude:
# Don't test MacOS with RBE to minimize MacOS minutes (billed at 10X)
- config: rbe
os: macos-latest
# Don't test MacOS with Bazel 6 to minimize MacOS minutes (billed at 10X)
- bazelversion: 6.5.0
os: macos-latest
# Don't test Windows with RBE to minimize Windows minutes (billed at 2X)
- config: rbe
os: windows-latest
# Don't test Windows with Bazel 6 to minimize Windows minutes (billed at 2X)
- bazelversion: 6.5.0
os: windows-latest
# Don't test MacOS and Windows against secondary bazel version to minimize minutes (billed at 10X and 2X respectively)
# https://docs.github.com/en/billing/managing-billing-for-github-actions/about-billing-for-github-actions#included-storage-and-minutes
- os: macos
bazel-version:
major: 6
- os: windows
bazel-version:
major: 6
# Don't run RBE tests with Bazel 6 to reduce the size of the test matrix
- bazel-version:
major: 6
config: rbe
# Don't run bzlmod tests with Bazel 6 to reduce the size of the test matrix
- bazel-version:
major: 6
bzlmod: 1
# Don't test RBE with on MacOS (not configured)
- os: macos
config: rbe
# Don't test RBE with on Windows (not configured)
- os: windows
config: rbe
# TODO: green up root Workspace on MacOS & Windows
- folder: .
os: macos-latest
os: macos
- folder: .
os: windows-latest
os: windows

steps:
- uses: actions/checkout@v4

- name: Mount bazel caches
uses: actions/cache@v4
uses: actions/cache@v3
with:
path: |
~/.cache/bazel
~/.cache/bazel-repo
key: bazel-cache-${{ matrix.os }}-${{ matrix.bazelversion }}-${{ matrix.folder }}-${{ matrix.bzlmodEnabled }}-${{ matrix.config }}-${{ hashFiles('.bazelrc', '.bazelversion', '.bazeliskrc', '**/BUILD', '**/BUILD.bazel', '**/*.bzl', 'WORKSPACE', 'WORKSPACE.bazel', 'WORKSPACE.bzlmod', 'MODULE.bazel', 'MODULE.bazel.lock') }}
restore-keys: bazel-cache-${{ matrix.os }}-${{ matrix.bazelversion }}-${{ matrix.folder }}-${{ matrix.bzlmodEnabled }}-${{ matrix.config }}-
~/.cache/bazel-disk-cache
~/.cache/bazel-repository-cache
~/.cache/xdg-cache
key: bazel-cache-${{ matrix.bazel-version.version }}-${{ matrix.bzlmod }}-${{ matrix.os }}-${{ matrix.folder }}-${{ hashFiles('.bazelrc', '.bazelversion', '.bazeliskrc', '**/BUILD', '**/BUILD.bazel', '**/*.bzl', 'WORKSPACE', 'WORKSPACE.bazel', 'WORKSPACE.bzlmod', 'MODULE.bazel') }}
restore-keys: bazel-cache-${{ matrix.bazel-version.version }}-${{ matrix.bzlmod }}-${{ matrix.os }}-${{ matrix.folder }}-

- name: Configure Bazel version
working-directory: ${{ matrix.folder }}
Expand All @@ -136,7 +135,7 @@ jobs:
# Overwrite the .bazelversion instead of using USE_BAZEL_VERSION so that Bazelisk
# still bootstraps Aspect CLI from configuration in .bazeliskrc. Aspect CLI will
# then use .bazelversion to determine which Bazel version to use.
echo "${{ matrix.bazelversion }}" > .bazelversion
echo "${{ matrix.bazel-version.version }}" > .bazelversion
# Delete all the version specific bazelrc files that are used for local development
# since the version we're testing against is dynamic. These are just symlinks and the
# root .bazelrc brings these in with try-imports. In this CI workflows, we explicitly
Expand All @@ -155,7 +154,7 @@ jobs:

# TODO: remove this block once we have Aspect CLI Windows releases
- name: Don't use Aspect CLI on Windows
if: matrix.os == 'windows-latest'
if: matrix.os == 'windows'
working-directory: ${{ matrix.folder }}
shell: bash
run: rm -f .bazeliskrc
Expand All @@ -164,26 +163,25 @@ jobs:
working-directory: ${{ matrix.folder }}
shell: bash
run: |
BAZEL_VERSION=${{ matrix.bazelversion }}
bazel \
--bazelrc=${GITHUB_WORKSPACE//\\/\/}/.aspect/bazelrc/bazel${BAZEL_VERSION::1}.bazelrc \
--bazelrc=${GITHUB_WORKSPACE//\\/\/}/.aspect/bazelrc/bazel${{ matrix.bazel-version.major }}.bazelrc \
--bazelrc=${GITHUB_WORKSPACE//\\/\/}/.aspect/bazelrc/ci.bazelrc \
--bazelrc=${GITHUB_WORKSPACE//\\/\/}/.github/workflows/ci.bazelrc \
test \
--config=${{ matrix.config }} \
--test_tag_filters=-skip-on-bazel${BAZEL_VERSION::1} \
--build_tag_filters=-skip-on-bazel${BAZEL_VERSION::1} \
--test_tag_filters=-skip-on-bazel${{ matrix.bazel-version.major }} \
--build_tag_filters=-skip-on-bazel${{ matrix.bazel-version.major }} \
--enable_bzlmod=${{ matrix.bzlmod }} \
//...
env:
XDG_CACHE_HOME: ~/.cache/xdg-cache # bazelisk will download bazel to here

- name: integration tests
- name: Integration tests
# Don't run integration tests on Windows since they are bash scripts and Windows runs Powershell
if: matrix.folder == '.' && matrix.os != 'windows-latest' && matrix.bazelversion != '6.5.0'
if: matrix.folder == '.' && matrix.os != 'windows' && matrix.bazel-version.major != '6'
# Find all shell scripts within e2e, echo the filename, execute, fail on error
run: find e2e/*.sh -maxdepth 1 -type f -exec sh -c 'echo "\n\n------------------------------- $0 -------------------------------" && BZLMOD_FLAG=${{ steps.set_bzlmod_flag.outputs.bzlmod_flag }} "$0" || kill $PPID' \{\} \;

- name: verify bcr patches
if: matrix.bzlmod == '1' && matrix.os == 'ubuntu-latest'
- name: Verify bcr patches
if: matrix.bzlmod == '1' && matrix.os == 'ubuntu' && matrix.bazel-version.major != '6'
run: patch --dry-run -p1 < .bcr/patches/*.patch
2 changes: 1 addition & 1 deletion .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ jobs:
path: |
~/.cache/bazel
~/.cache/bazel-repo
key: bazel-cache-release-${{ hashFiles('.bazelrc', '.bazelversion', '.bazeliskrc', '**/BUILD', '**/BUILD.bazel', '**/*.bzl', 'WORKSPACE', 'WORKSPACE.bazel', 'WORKSPACE.bzlmod', 'MODULE.bazel', 'MODULE.bazel.lock') }}
key: bazel-cache-release-${{ hashFiles('.bazelrc', '.bazelversion', '.bazeliskrc', '**/BUILD', '**/BUILD.bazel', '**/*.bzl', 'WORKSPACE', 'WORKSPACE.bazel', 'WORKSPACE.bzlmod', 'MODULE.bazel') }}
restore-keys: bazel-cache-release-
- name: bazel test //... (release)
env:
Expand Down
21 changes: 13 additions & 8 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,13 +1,18 @@
bazel-*
**/.terraform/*
.bazelrc.user

test-out/

.idea/
.ijwb/
.vscode
.DS_Store

# Don't commit lockfile for now as it is unstable. Do allow for it to be
# created, however, since it gives a performance boost for local development.
# https://github.com/bazelbuild/bazel/issues/19026
# https://github.com/bazelbuild/bazel/issues/19621
# https://github.com/bazelbuild/bazel/issues/19971
# https://github.com/bazelbuild/bazel/issues/20272
# https://github.com/bazelbuild/bazel/issues/20369
# Bazel's MODULE lockfile isn't ready to check in yet as of Bazel 7.1.
# Do allow for it to be created, however, since it gives a performance boost for local development.
# [Store resolved repository attributes in the Bzlmod lockfile](https://github.com/bazelbuild/bazel/issues/19026)
# [MODULE.bazel.lock file contains user specific paths](https://github.com/bazelbuild/bazel/issues/19621)
# [Consider skipping bazel_tools@_ from lockfile](https://github.com/bazelbuild/bazel/issues/19971)
# [MODULE.bazel.lock file "reads through" already-locked package manager](https://github.com/bazelbuild/bazel/issues/20272)
# [moduleFileHash in MODULE.bazel.lock causes frequent Git merge conflicts](https://github.com/bazelbuild/bazel/issues/20369)
MODULE.bazel.lock
6 changes: 3 additions & 3 deletions MODULE.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -48,12 +48,12 @@ use_repo(host, "aspect_bazel_lib_host")

bazel_dep(
name = "gazelle",
version = "0.33.0",
version = "0.35.0",
# In released versions: dev_dependency = True
)
bazel_dep(
name = "rules_go",
version = "0.41.0",
version = "0.46.0",
repo_name = "io_bazel_rules_go",
# In released versions: dev_dependency = True
)
Expand All @@ -73,5 +73,5 @@ use_repo(
# Development-only dependencies

bazel_dep(name = "bazel_skylib_gazelle_plugin", version = "1.5.0", dev_dependency = True)
bazel_dep(name = "buildifier_prebuilt", version = "6.3.3", dev_dependency = True)
bazel_dep(name = "buildifier_prebuilt", version = "6.4.0", dev_dependency = True)
bazel_dep(name = "bazel_features", version = "0.2.0", dev_dependency = True)
3 changes: 0 additions & 3 deletions WORKSPACE
Original file line number Diff line number Diff line change
Expand Up @@ -75,13 +75,10 @@ buildifier_prebuilt_register_toolchains()
load(
"@aspect_rules_lint//format:repositories.bzl",
"fetch_shfmt",
"fetch_terraform",
)

fetch_shfmt()

fetch_terraform()

load("//.aspect/workflows:deps.bzl", "fetch_workflows_deps")

fetch_workflows_deps()
15 changes: 15 additions & 0 deletions e2e/copy_action/.bazelrc
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
# Import Aspect bazelrc presets
try-import %workspace%/../../.aspect/bazelrc/local/bazel7.bazelrc
import %workspace%/../../.aspect/bazelrc/convenience.bazelrc
import %workspace%/../../.aspect/bazelrc/correctness.bazelrc
import %workspace%/../../.aspect/bazelrc/debug.bazelrc
import %workspace%/../../.aspect/bazelrc/javascript.bazelrc
import %workspace%/../../.aspect/bazelrc/performance.bazelrc

### YOUR PROJECT SPECIFIC OPTIONS GO HERE ###

# Load any settings & overrides specific to the current user from `.aspect/bazelrc/user.bazelrc`.
# This file should appear in `.gitignore` so that settings are not shared with team members. This
# should be last statement in this config so the user configuration is able to overwrite flags from
# this file. See https://bazel.build/configure/best-practices#bazelrc-file.
try-import %workspace%/../../.aspect/bazelrc/user.bazelrc
15 changes: 15 additions & 0 deletions e2e/copy_to_directory/.bazelrc
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
# Import Aspect bazelrc presets
try-import %workspace%/../../.aspect/bazelrc/local/bazel7.bazelrc
import %workspace%/../../.aspect/bazelrc/convenience.bazelrc
import %workspace%/../../.aspect/bazelrc/correctness.bazelrc
import %workspace%/../../.aspect/bazelrc/debug.bazelrc
import %workspace%/../../.aspect/bazelrc/javascript.bazelrc
import %workspace%/../../.aspect/bazelrc/performance.bazelrc

### YOUR PROJECT SPECIFIC OPTIONS GO HERE ###

# Load any settings & overrides specific to the current user from `.aspect/bazelrc/user.bazelrc`.
# This file should appear in `.gitignore` so that settings are not shared with team members. This
# should be last statement in this config so the user configuration is able to overwrite flags from
# this file. See https://bazel.build/configure/best-practices#bazelrc-file.
try-import %workspace%/../../.aspect/bazelrc/user.bazelrc
Loading

0 comments on commit 37d7495

Please sign in to comment.