Skip to content

Add constraint-solver-iterations CI check #4900

Add constraint-solver-iterations CI check

Add constraint-solver-iterations CI check #4900

Workflow file for this run

# SPDX-FileCopyrightText: 2022-2024 Google LLC
#
# SPDX-License-Identifier: Apache-2.0
name: CI
# XXX: What we'd really like is run on every pull request / pull request change,
# which allows GitHub to cancel runs after (force) pushes. Triggering on
# these seems to be broken since January 19th 2023 however. Note that not
# everyone likes auto-cancellation so we should probably recognize [no-skip]
# commits.
on:
schedule:
# Works out to be approx midnight (CET/CEST) accounting for UTC offset and
# GitHub scheduling. See the comment at 'branches-ignore' for more
# information on our development flow. This job will run in context of the
# branch 'staging'.
- cron: '0 19 * * *'
push:
branches-ignore:
# Developers request to merge their changes into 'staging'. In order to
# do so, relatively cheap tests should pass CI. Every night, CI runs our
# full test suite on 'staging'. If these pass, a pull request may be
# created to get 'main' up to date with 'staging'. Note that this should
# be automated in the future, but this will require some engineering.
- 'staging'
- 'main'
env:
HW_SERVER_URL: hoeve.local:3121
S3_PASSWORD: ${{ secrets.S3_PASSWORD }}
SYNTHESIS_BOARD: xilinx.com:kcu105:part0:1.7
concurrency:
group: ${{ github.head_ref || github.run_id }}
cancel-in-progress: true
jobs:
license-year:
name: License year check
runs-on: [self-hosted, compute]
if: ${{ github.event_name != 'schedule' }}
defaults:
run:
shell: git-nix-shell {0} --option connect-timeout 360 --pure --keep "GITHUB_SHA" --keep "S3_PASSWORD"
container:
image: ghcr.io/clash-lang/nixos-bittide-hardware:2024-08-06
options: --memory=11g
steps:
- name: Checkout
uses: actions/checkout@v4
with:
# Check needs access to all branches
fetch-depth: 0
- name: Check license year of all files per commit
run: |
# XXX: We want to pass in the target branch of the PR, but we only run
# on 'push' (see 'on' section in this YAML). Because of this, GH
# sets the target branch to an empty string.
.github/scripts/check_copyright_years.py origin/staging
license-check:
runs-on: [self-hosted, compute]
container:
image: ubuntu:22.04
options: --memory=11g
steps:
- uses: actions/checkout@v4
- name: REUSE Compliance Check
uses: fsfe/reuse-action@v1
lint:
name: Basic linting
runs-on: [self-hosted, compute]
defaults:
run:
shell: git-nix-shell {0} --option connect-timeout 360
container:
image: ghcr.io/clash-lang/nixos-bittide-hardware:2024-08-06
options: --memory=11g
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Install dependencies
run: |
.github/scripts/set_de_mirror.sh
apt-get update
apt-get install -y pcregrep python3-yaml
- name: EOL whitespace
run: |
.github/scripts/check_eol_whitespace.sh
- name: Enforce EOF newline
run: |
.github/scripts/check_missing_eof_newline.sh
- name: Check that all self-hosted jobs run in a docker container
run: |
.github/scripts/self_hosted_docker_check.py
- name: Check that the 'all' job depends on all other jobs
run: |
.github/scripts/all_check.py
- name: Check that all cabal files are formatted correctly
run: |
.github/scripts/cabal-gild.sh check
git diff --exit-code
- name: Check that we don't introduce accidental infinite loops in type checkers
run: |
! grep --include=*.hs -E -r '\-fconstraint-solver-iterations *= *0'
build:
name: Build dependencies
runs-on: [self-hosted, compute]
defaults:
run:
shell: git-nix-shell {0} --option connect-timeout 360 --pure --keep "GITHUB_SHA" --keep "S3_PASSWORD"
container:
image: ghcr.io/clash-lang/nixos-bittide-hardware:2024-08-06
options: --memory=11g
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set CI flags
run: |
cp .github/cabal.project cabal.project.local
- run: cache clean
- run: cache pull cabal --missing-ok --write-cache-found
- run: cache pull cargo --missing-ok
- name: Update Cabal index info
run: |
if [[ $(cat cache_found) == 0 ]]; then
cabal update
else
echo "Restored cache, no need to update cabal package list"
fi
- run: ./cargo.sh fetch
- run: ./cargo.sh build --frozen --release
- run: ./cargo.sh build --frozen
- run: cabal build all
- run: cache push cabal
- run: cache push cargo
- run: cache push build
cc-sim-topologies-matrix:
name: cc-sim-topologies simulation matrix generation
runs-on: [self-hosted, compute]
defaults:
run:
shell: git-nix-shell {0} --option connect-timeout 360 --pure --keep "GITHUB_OUTPUT"
container:
image: ghcr.io/clash-lang/nixos-bittide-hardware:2024-08-06
options: --memory=11g
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Generate matrix
run: |
if [[ "${{ github.event_name }}" == "schedule" || $(.github/scripts/force_expensive_checks.sh) == "true" ]]; then
cp .github/simulation/safe.json sim.json
else
cp .github/simulation/staging.json sim.json
fi
- name: Set simulation matrix
id: set-sim-matrix
run: |
echo "sim_matrix=$(cat sim.json | jq -c)" | tee -a "$GITHUB_OUTPUT"
echo "artifact_names=$(cat sim.json | jq -c '["gen-plots-hs-" + .[].topology]')" | tee -a "$GITHUB_OUTPUT"
outputs:
sim_matrix: ${{ steps.set-sim-matrix.outputs.sim_matrix }}
artifact_names: ${{ steps.set-sim-matrix.outputs.artifact_names }}
cc-sim-topologies:
name: Simulate network
runs-on: [self-hosted, compute]
timeout-minutes: 1200
defaults:
run:
shell: git-nix-shell {0} --option connect-timeout 360 --pure --keep "GITHUB_SHA" --keep "S3_PASSWORD"
needs: [build, lint, cc-sim-topologies-matrix]
strategy:
matrix:
target: ${{ fromJson(needs.cc-sim-topologies-matrix.outputs.sim_matrix) }}
fail-fast: false
container:
image: ghcr.io/clash-lang/nixos-bittide-hardware:2024-08-06
options: --memory=11g
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set CI flags
run: |
cp .github/cabal.project cabal.project.local
- run: cache pull cabal
- run: cache pull build
- name: Plot mesh grid
run: |
export RUNREF="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
mkdir -p _build/reports/simulation
cabal run -- bittide-tools:cc-sim \
+RTS -M${{ matrix.target.hmem }}M -RTS \
--output-directory _build/reports/simulation \
--output-mode pdf \
--get-rusty \
--steps ${{ matrix.target.steps }} \
--samples 1000 \
--max-startup-delay 2000 \
--stop-after-stable ${{ matrix.target.steps }} \
--create-report \
${{ matrix.target.topology }} ${{ matrix.target.args }}
- name: Upload plots
uses: actions/upload-artifact@v4
if: ${{ !cancelled() }}
with:
name: gen-plots-hs-${{ matrix.target.topology }}
path: |
_build
!_build/cargo
retention-days: 14
generate-full-clock-control-simulation-report:
name: Generate clock control simulation report
runs-on: [self-hosted, compute]
if: ${{ !cancelled() }}
defaults:
run:
shell: git-nix-shell {0} --option connect-timeout 360 --pure --keep "GITHUB_SHA" --keep "S3_PASSWORD"
needs: [build, lint, cc-sim-topologies-matrix, cc-sim-topologies, bittide-instances-hardware-in-the-loop]
container:
image: ghcr.io/clash-lang/nixos-bittide-hardware:2024-08-06
options: --memory=11g
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set CI flags
run: |
cp .github/cabal.project cabal.project.local
- run: cache pull cabal
- run: cache pull cargo
- run: cache pull build
- name: Download simulation artifacts
uses: actions/download-artifact@v4
with:
# Note that all artifacts are downloaded, if no 'name' is specified.
# This is still required for the generation of the simulation plots.
# Unfortunately, this is the only working solution that github offers
# right at the moment. Alternative options exist, like
# https://github.com/marketplace/actions/download-multiple-workflow-artifacts,
# but this action is currently broken. Once a working solution
# is available again, 'artifact_names' from
# 'cc-sim-topologies-matrix' can be used to only download
# the required artifacts instead.
path: artifacts
- name: Generate HITL based plots
run: |
./cargo.sh build --frozen --release
export BITTIDE_ARTIFACT_ACCESS_TOKEN="${{ secrets.GITHUB_TOKEN }}"
export RUNREF="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
cabal run -- bittide-tools:cc-plot ${{ github.run_id }}:hwCcTopologyTest _build/plot hitl-topology-plots
- name: Generate clock control reports
run: |
set +f # Enable globbing
mkdir -p reports
pdfunite \
artifacts/*/reports/simulation/report.pdf \
reports/Simulation-Report.pdf
pdfunite \
hitl-topology-plots/*/report.pdf \
reports/HITLT-Report.pdf
rm hitl-topology-plots/*/report.pdf
mkdir -p plot-sources
mv hitl-topology-plots plot-sources/hwCcTopologyTest
- name: Upload Reports
uses: actions/upload-artifact@v4
with:
name: Clock Control Reports
path: reports
retention-days: 14
- name: Upload HITLT Plot Sources
uses: actions/upload-artifact@v4
with:
name: CC-HITLT Plot Sources
path: plot-sources
retention-days: 14
bittide-experiments-tests:
name: bittide-experiments unittests
runs-on: [self-hosted, compute]
defaults:
run:
shell: git-nix-shell {0} --option connect-timeout 360 --pure --keep "GITHUB_SHA" --keep "S3_PASSWORD"
needs: [build, lint]
container:
image: ghcr.io/clash-lang/nixos-bittide-hardware:2024-08-06
options: --memory=11g
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set CI flags
run: |
cp .github/cabal.project cabal.project.local
- run: cache pull cabal
- run: cache pull build
- name: Run unittests
run: |
cabal run bittide-experiments:unittests
bittide-tests:
name: Bittide tests
runs-on: [self-hosted, compute]
defaults:
run:
shell: git-nix-shell {0} --option connect-timeout 360 --pure --keep "GITHUB_SHA" --keep "S3_PASSWORD"
needs: [build, lint]
container:
image: ghcr.io/clash-lang/nixos-bittide-hardware:2024-08-06
options: --memory=11g
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set CI flags
run: |
cp .github/cabal.project cabal.project.local
- run: cache pull cabal
- run: cache pull build
- name: Unittests
run: |
# While Hedgehog can fill many cores with "work", it doesn't actually
# speed up the tests that much (<5% runtime benefits).
cabal run -- bittide:unittests -j 2 +RTS -N4
- name: Bittide Doctests
run: |
cabal run -- bittide:doctests
- name: Bittide.Extra Doctests
run: |
cabal run -- bittide-extra:doctests
- name: Bittide.Extra Unittests
run: |
# While Hedgehog can fill many cores with "work", it doesn't actually
# speed up the tests that much (<5% runtime benefits).
cabal run -- bittide-extra:unittests -j 2 +RTS -N4
rust-lints:
name: Rust Lints
runs-on: [self-hosted, compute]
defaults:
run:
shell: git-nix-shell {0} --option connect-timeout 360 --pure --keep "GITHUB_SHA" --keep "S3_PASSWORD"
container:
image: ghcr.io/clash-lang/nixos-bittide-hardware:2024-08-06
options: --memory=11g
needs: [build]
steps:
- name: Checkout
uses: actions/checkout@v4
- run: cache pull cargo
- name: Rust formatting
run: |
./cargo.sh fmt --all -- --check
- name: Clippy check
run: |
./cargo.sh clippy --all-features -- -Dwarnings
firmware-support-tests:
name: Firmware Support Unit Tests
runs-on: [self-hosted, compute]
defaults:
run:
shell: git-nix-shell {0} --option connect-timeout 360 --pure --keep "GITHUB_SHA" --keep "S3_PASSWORD"
container:
image: ghcr.io/clash-lang/nixos-bittide-hardware:2024-08-06
options: --memory=11g
needs: [build]
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set CI flags
run: |
cp .github/cabal.project cabal.project.local
- run: cache pull cabal
- run: cache pull cargo
- run: cache pull build
- name: Running Tests
run: |
cargo test --all
working-directory: firmware-support/
host-tools-tests:
name: Host Tools Unit Tests
runs-on: [self-hosted, compute]
defaults:
run:
shell: git-nix-shell {0} --option connect-timeout 360 --pure --keep "GITHUB_SHA" --keep "S3_PASSWORD"
container:
image: ghcr.io/clash-lang/nixos-bittide-hardware:2024-08-06
options: --memory=11g
needs: [build]
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set CI flags
run: |
cp .github/cabal.project cabal.project.local
- run: cache pull cargo
- name: Running Tests
run: |
cargo test --all
working-directory: host-tools/
firmware-limit-checks:
name: Firmware Limit Checks
runs-on: [self-hosted, compute]
defaults:
run:
shell: git-nix-shell {0} --option connect-timeout 360 --pure --keep "GITHUB_SHA" --keep "S3_PASSWORD"
container:
image: ghcr.io/clash-lang/nixos-bittide-hardware:2024-08-06
options: --memory=11g
needs: [build]
steps:
- name: Checkout
uses: actions/checkout@v4
- run: cache pull cargo
- run: cache pull build
- name: Install `elf-limits`
run: cargo install --locked --git https://github.com/cuddlefishie/elf-limits
- name: Checking firmware binary limits
run: |
~/.cargo/bin/elf-limits \
--instruction-mem-limit 64K \
--data-mem-limit 64K \
clock-control \
hello \
processing-element-test
working-directory: _build/cargo/firmware-binaries/riscv32imc-unknown-none-elf/release/
bittide-instances-doctests:
name: bittide-instances doctests
runs-on: [self-hosted, compute]
defaults:
run:
shell: git-nix-shell {0} --option connect-timeout 360 --pure --keep "GITHUB_SHA" --keep "S3_PASSWORD"
needs: [build, lint]
container:
image: ghcr.io/clash-lang/nixos-bittide-hardware:2024-08-06
options: --memory=11g
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set CI flags
run: |
cp .github/cabal.project cabal.project.local
- run: cache pull cabal
- run: cache pull build
- name: Doctests
run : |
cabal run -- bittide-instances:doctests
bittide-instances-unittests:
name: bittide-instances unittests
runs-on: [self-hosted, compute]
defaults:
run:
shell: git-nix-shell {0} --option connect-timeout 360 --pure --keep "GITHUB_SHA" --keep "S3_PASSWORD"
needs: [build, lint]
container:
image: ghcr.io/clash-lang/nixos-bittide-hardware:2024-08-06
options: --memory=11g
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set CI flags
run: |
cp .github/cabal.project cabal.project.local
- run: cache pull cabal
- run: cache pull build
- name: Unit tests
run : |
cabal run -- bittide-instances:unittests
bittide-instances-hdl-matrix:
name: bittide-instances synthesis matrix generation
runs-on: [self-hosted, compute]
defaults:
run:
shell: git-nix-shell {0} --option connect-timeout 360 --pure --keep "GITHUB_OUTPUT"
container:
image: ghcr.io/clash-lang/nixos-bittide-hardware:2024-08-06
options: --memory=11g
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Generate matrix
run: |
if [[ "${{ github.event_name }}" == "schedule" || $(.github/scripts/force_expensive_checks.sh) == "true" ]]; then
cp .github/synthesis/all.json checks.json
elif [[ -f .github/synthesis/debug.json ]]; then
cp .github/synthesis/debug.json checks.json
else
cp .github/synthesis/staging.json checks.json
fi
- name: Set test matrix
id: set-matrix
run: |
echo "check_matrix=$(cat checks.json | tr '\n' ' ')" | tee -a "$GITHUB_OUTPUT"
outputs:
check_matrix: ${{ steps.set-matrix.outputs.check_matrix }}
bittide-instances-hardware-in-the-loop-test-matrix:
name: bittide-instances hardware-in-the-loop test matrix generation
runs-on: [self-hosted, compute]
defaults:
run:
shell: git-nix-shell {0} --option connect-timeout 360 --pure --keep "GITHUB_OUTPUT"
container:
image: ghcr.io/clash-lang/nixos-bittide-hardware:2024-08-06
options: --memory=11g
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Generate matrix
run: |
if [[ "${{ github.event_name }}" == "schedule" || $(.github/scripts/force_expensive_checks.sh) == "true" ]]; then
cp .github/synthesis/all.json checks.json
elif [[ -f .github/synthesis/debug.json ]]; then
cp .github/synthesis/debug.json checks.json
else
cp .github/synthesis/staging.json checks.json
fi
jq '[.[] | select(.stage=="test")]' checks.json > checks.json.filtered
cp checks.json.filtered checks.json
- name: Set test matrix
id: set-matrix
run: |
echo "check_matrix=$(cat checks.json | tr '\n' ' ')" | tee -a "$GITHUB_OUTPUT"
outputs:
check_matrix: ${{ steps.set-matrix.outputs.check_matrix }}
bittide-instances-hdl:
name: synth
runs-on: [self-hosted, compute]
defaults:
run:
# We leave out '--pure', as 'with_vivado.sh' relies on basic Ubuntu
shell: git-nix-shell {0} --option connect-timeout 360
needs: [build, lint, bittide-instances-hdl-matrix]
strategy:
matrix:
target: ${{ fromJson(needs.bittide-instances-hdl-matrix.outputs.check_matrix) }}
fail-fast: false
container:
image: ghcr.io/clash-lang/nixos-bittide-hardware:2024-08-06
volumes:
- /opt/tools:/opt/tools
options: --init --mac-address="6c:5a:b0:6c:13:0b" --memory=11g
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set CI flags
run: |
# We use the same flags as the 'build' step to reuse caches. This
# "simulation" config only sets altopts on the Risc core, which we
# don't synthesize yet.
#
# TODO: Either switch to VexRisc or update cache strategy.
#
cp .github/cabal.project cabal.project.local
- run: cache pull cabal
- run: cache pull cargo
- run: cache pull build
- name: HDL generation and synthesis
run : |
target="${{ matrix.target.stage }}"
if [[ "${target}" == "program" ]]; then
echo "Illegal Shake target on CI: program"
exit 1
fi
# Only the local runner connected to the bittide demonstrator (hoeve)
# should perform the jobs which need hardware access. And that runner
# should do only that, so we let other 'compute' runners do the
# bitstream generation.
if [[ "${target}" == "test" ]]; then
target="bitstream"
fi
.github/scripts/with_vivado.sh \
shake ${{ matrix.target.top }}:"${target}"
# Workaround for https://github.com/bittide/bittide-hardware/issues/523
mkdir -p _build/vivado
touch _build/vivado/workaround-issue-523
- name: cache push build-post-synth
run: cache push build-post-synth --prefix="${{ matrix.target.top }}"
- name: Archive build artifacts
uses: actions/upload-artifact@v4
if: ${{ !cancelled() }}
with:
name: _build-${{ matrix.target.top }}
# We don't pack `ip`, as it is generally useless for debugging while
# also taking up a lot of space.
path: |
_build/clash
_build/vivado
_build/.shake.database
_build/.shake.lock
!_build/vivado/*/ip
retention-days: 14
bittide-instances-hardware-in-the-loop:
name: HITL
runs-on: [self-hosted, hardware-access]
defaults:
run:
# We leave out '--pure', as 'with_vivado.sh' relies on basic Ubuntu
shell: git-nix-shell {0} --option connect-timeout 360
needs: [bittide-instances-hdl, bittide-instances-hardware-in-the-loop-test-matrix]
strategy:
matrix:
target: ${{ fromJson(needs.bittide-instances-hardware-in-the-loop-test-matrix.outputs.check_matrix) }}
fail-fast: false
container:
image: ghcr.io/clash-lang/nixos-bittide-hardware:2024-08-06
volumes:
- /opt/tools:/opt/tools
- /dev:/dev
# XXX: User '1001' corresponds to 'bittide' on Linux installation that has
# access to the hardware. We need to set this, lest Docker will write
# files as 'root' and the runner software will be unable to run
# cleaning jobs.
options: >-
--memory=11g
--userns host
--privileged
--user 1001:1001
--group-add plugdev
--group-add dialout
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set CI flags
run: |
cp .github/cabal.project cabal.project.local
- run: cache pull cabal
- run: cache pull cargo
- run: cache pull build
- name: cache pull build-post-synth
run: cache pull build-post-synth --prefix="${{ matrix.target.top }}"
- name: Run tests on hardware
run: |
.github/scripts/with_vivado.sh \
shake ${{ matrix.target.top }}:test --hardware-targets="${{ matrix.target.targets}}"
- name: Archive ILA data
if: ${{ !cancelled() }}
uses: actions/upload-artifact@v4
with:
name: _build-${{ matrix.target.top }}-debug
path: |
_build/vivado/*/ila-data
_build/hitl/*
retention-days: 14
all:
name: All jobs finished
if: ${{ !cancelled() }}
needs: [
bittide-instances-doctests,
bittide-instances-hardware-in-the-loop-test-matrix,
bittide-instances-hardware-in-the-loop,
bittide-instances-hdl-matrix,
bittide-instances-hdl,
bittide-instances-unittests,
bittide-tests,
build,
bittide-experiments-tests,
cc-sim-topologies-matrix,
cc-sim-topologies,
firmware-limit-checks,
firmware-support-tests,
generate-full-clock-control-simulation-report,
host-tools-tests,
license-check,
license-year,
lint,
rust-lints,
]
runs-on: ubuntu-22.04
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Check dependencies for failures
run: |
# Test all dependencies for success/failure
set -x
success="${{ contains(needs.*.result, 'success') }}"
fail="${{ contains(needs.*.result, 'failure') }}"
set +x
# Test whether success/fail variables contain sane values
if [[ "${success}" != "true" && "${success}" != "false" ]]; then exit 1; fi
if [[ "${fail}" != "true" && "${fail}" != "false" ]]; then exit 1; fi
# If this is a debug run, fail even when all dependencies succeeded.
if [[ -f .github/synthesis/debug.json ]]; then
echo "This is a debug run, which may never succeed. Remove '.github/synthesis/debug.json' before trying to merge."
exit 1
fi
# We want to fail if one or more dependencies fail. For safety, we introduce
# a second check: if no dependencies succeeded something weird is going on.
if [[ "${fail}" == "true" || "${success}" == "false" ]]; then
echo "One or more dependency failed, or no dependency succeeded."
exit 1
fi