Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Inspect PR changes to determine if subproject builds are needed. #1572

Merged
merged 3 commits into from
Apr 9, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 25 additions & 1 deletion .github/actions/compute-matrix/compute-matrix.sh
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,30 @@ explode_libs() {
jq -cr 'map(. as $o | {lib: $o.lib[]} + del($o.lib))'
}

# Filter out the libraries that are dirty
filter_libs() {
all_libs=("libcudacxx" "thrust" "cub")
dirty_libs=()
for lib in "${all_libs[@]}"; do
dirty_var_name="${lib^^}_DIRTY"
# If the variable named in dirty_var_name is not set, set it to false:
: "${!dirty_var_name:=false}"
# Declare a nameref to the variable named in dirty_var_name
declare -n lib_dirty="$dirty_var_name"
# echo "${lib^^}_DIRTY: ${lib_dirty}" >> /dev/stderr
if [ "${lib_dirty}" = "true" ]; then
dirty_libs+=("$lib")
fi
done
# echo "Dirty libraries: ${dirty_libs[@]}" >> /dev/stderr

# Construct a regex to filter out the dirty libraries
dirty_lib_regex=$(IFS="|"; echo "${dirty_libs[*]}")
dirty_lib_regex="^(${dirty_lib_regex})\$"
jq_filter="map(select(.lib | test(\"$dirty_lib_regex\")))"
jq -cr "$jq_filter"
}

extract_matrix() {
local file="$1"
local type="$2"
Expand All @@ -29,7 +53,7 @@ extract_matrix() {

write_output "NVRTC_MATRIX" "$(echo "$matrix" | jq '.nvrtc' | explode_std_versions)"

local clang_cuda_matrix="$(echo "$matrix" | jq -cr '.["clang-cuda"]' | explode_std_versions | explode_libs)"
local clang_cuda_matrix="$(echo "$matrix" | jq -cr '.["clang-cuda"]' | explode_std_versions | explode_libs | filter_libs)"
write_output "CLANG_CUDA_MATRIX" "$clang_cuda_matrix"
write_output "CCCL_INFRA_MATRIX" "$(echo "$matrix" | jq -cr '.["cccl-infra"]' )"
}
Expand Down
49 changes: 43 additions & 6 deletions .github/workflows/pr.yml
Original file line number Diff line number Diff line change
Expand Up @@ -35,9 +35,31 @@ permissions:
pull-requests: read

jobs:
inspect-changes:
name: "Inspect Changes"
runs-on: ubuntu-latest
outputs:
LIBCUDACXX_DIRTY: ${{ steps.set-outputs.outputs.LIBCUDACXX_DIRTY }}
CUB_DIRTY: ${{ steps.set-outputs.outputs.CUB_DIRTY }}
THRUST_DIRTY: ${{ steps.set-outputs.outputs.THRUST_DIRTY }}
steps:
- name: Get Base Branch from PR
id: get-pr-info
uses: nv-gha-runners/get-pr-info@main
- name: Checkout repo
uses: actions/checkout@v3
- name: Identify dirty subprojects
id: set-outputs
run: |
./ci/inspect_changes.sh ${BASE_SHA} ${GITHUB_SHA}
env:
BASE_SHA: ${{ fromJSON(steps.get-pr-info.outputs.pr-info).base.sha }}

compute-matrix:
name: Compute matrix
runs-on: ubuntu-latest
needs:
- inspect-changes
outputs:
DEVCONTAINER_VERSION: ${{steps.set-outputs.outputs.DEVCONTAINER_VERSION}}
PER_CUDA_COMPILER_MATRIX: ${{steps.set-outputs.outputs.PER_CUDA_COMPILER_MATRIX}}
Expand All @@ -52,14 +74,20 @@ jobs:
id: set-outputs
run: |
.github/actions/compute-matrix/compute-matrix.sh ci/matrix.yaml pull_request
env:
THRUST_DIRTY: ${{ needs.inspect-changes.outputs.THRUST_DIRTY }}
CUB_DIRTY: ${{ needs.inspect-changes.outputs.CUB_DIRTY }}
LIBCUDACXX_DIRTY: ${{ needs.inspect-changes.outputs.LIBCUDACXX_DIRTY }}

nvrtc:
name: NVRTC CUDA${{matrix.cuda}}
name: libcudacxx NVRTC CUDA${{matrix.cuda}}
permissions:
id-token: write
contents: read
needs: compute-matrix
if: ${{ !contains(github.event.head_commit.message, 'skip-tests') }}
needs:
- compute-matrix
- inspect-changes
if: ${{ !contains(github.event.head_commit.message, 'skip-tests') && needs.inspect-changes.outputs.LIBCUDACXX_DIRTY == 'true' }}
uses: ./.github/workflows/run-as-coder.yml
strategy:
fail-fast: false
Expand All @@ -77,7 +105,10 @@ jobs:
permissions:
id-token: write
contents: read
needs: compute-matrix
needs:
- compute-matrix
- inspect-changes
if: ${{ needs.inspect-changes.outputs.THRUST_DIRTY == 'true' }}
uses: ./.github/workflows/dispatch-build-and-test.yml
strategy:
fail-fast: false
Expand All @@ -94,7 +125,10 @@ jobs:
permissions:
id-token: write
contents: read
needs: compute-matrix
needs:
- compute-matrix
- inspect-changes
if: ${{ needs.inspect-changes.outputs.CUB_DIRTY == 'true' }}
uses: ./.github/workflows/dispatch-build-and-test.yml
strategy:
fail-fast: false
Expand All @@ -111,7 +145,10 @@ jobs:
permissions:
id-token: write
contents: read
needs: compute-matrix
needs:
- compute-matrix
- inspect-changes
if: ${{ needs.inspect-changes.outputs.LIBCUDACXX_DIRTY == 'true' }}
uses: ./.github/workflows/dispatch-build-and-test.yml
strategy:
fail-fast: false
Expand Down
152 changes: 152 additions & 0 deletions ci/inspect_changes.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,152 @@
#!/bin/bash

# Github action script to identify which subprojects are dirty in a PR

set -u

# Usage: inspect_changes.sh <base_sha> <head_sha>
if [ "$#" -ne 2 ]; then
echo "Usage: $0 <base_sha> <head_sha>"
exit 1
fi

base_sha=$1
head_sha=$2

# Github gives the SHA as the current HEAD of the target ref, not the common ancestor.
# Find the common ancestor and use that for the base.
git fetch origin --unshallow -q
git fetch origin $base_sha -q
base_sha=$(git merge-base $head_sha $base_sha)

# Define a list of subproject directories:
subprojects=(
libcudacxx
cub
thrust
)

# ...and their dependencies:
declare -A dependencies=(
[libcudacxx]="cccl"
[cub]="cccl libcudacxx thrust"
[thrust]="cccl libcudacxx cub"
)

write_output() {
local key="$1"
local value="$2"
echo "$key=$value" | tee --append "${GITHUB_OUTPUT:-/dev/null}"
}

dirty_files() {
git diff --name-only "${base_sha}" "${head_sha}"
}

# Return 1 if any files outside of the subproject directories have changed
inspect_cccl() {
subprojs_grep_expr=$(
IFS="|"
echo "(${subprojects[*]})/"
)

if dirty_files | grep -v -E "${subprojs_grep_expr}" | grep -q "."; then
return 1
else
return 0
fi
Comment on lines +53 to +57
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I dont know who, but I guess there is a way to transform this into a

Suggested change
if dirty_files | grep -v -E "${subprojs_grep_expr}" | grep -q "."; then
return 1
else
return 0
fi
return dirty_files | grep -v -E "${subprojs_grep_expr}" | grep -q "."; | SOMETHING

}

# inspect_subdir <subdir>
# Returns 1 if any files in the subdirectory have changed
inspect_subdir() {
local subdir="$1"

if dirty_files | grep -E "^${subdir}/" | grep -q '.'; then
return 1
else
return 0
fi
}

# add_dependencies <subproject>
# if the subproject or any of its dependencies are dirty, return 1
add_dependencies() {
local subproject="$1"

# Check if ${subproject^^}_DIRTY is set to 1, return 1 if it is.
local dirty_flag=${subproject^^}_DIRTY
if [[ ${!dirty_flag} -ne 0 ]]; then
return 1
fi

for dependency in ${dependencies[$subproject]}; do
dirty_flag="${dependency^^}_DIRTY"
if [[ ${!dirty_flag} -ne 0 ]]; then
return 1
fi
done

return 0
}

# write_subproject_status <subproject>
# Write the output <subproject_uppercase>_DIRTY={true|false}
write_subproject_status() {
local subproject="$1"
local dirty_flag=${subproject^^}_DIRTY

if [[ ${!dirty_flag} -ne 0 ]]; then
write_output "${dirty_flag}" "true"
else
write_output "${dirty_flag}" "false"
fi
}

main() {
# Print the list of subprojects and all of their dependencies:
echo "Subprojects: ${subprojects[*]}"
echo
echo "Dependencies:"
for subproject in "${subprojects[@]}"; do
echo " - ${subproject} -> ${dependencies[$subproject]}"
done
echo

echo "Base SHA: ${base_sha}"
echo "HEAD SHA: ${head_sha}"
echo

# Print the list of files that have changed:
echo "Dirty files:"
dirty_files | sed 's/^/ - /'
echo ""

echo "Modifications in project?"
# Assign the return value of `inspect_cccl` to the variable `CCCL_DIRTY`:
inspect_cccl
CCCL_DIRTY=$?
echo "$(if [[ ${CCCL_DIRTY} -eq 0 ]]; then echo " "; else echo "X"; fi) - CCCL Infrastructure"

# Check for changes in each subprojects directory:
for subproject in "${subprojects[@]}"; do
inspect_subdir $subproject
declare ${subproject^^}_DIRTY=$?
echo "$(if [[ ${subproject^^}_DIRTY -eq 0 ]]; then echo " "; else echo "X"; fi) - ${subproject}"
done
echo

echo "Modifications in project or dependencies?"
for subproject in "${subprojects[@]}"; do
add_dependencies ${subproject}
declare ${subproject^^}_DIRTY=$?
echo "$(if [[ ${subproject^^}_DIRTY -eq 0 ]]; then echo " "; else echo "X"; fi) - ${subproject}"
done
echo

for subproject in "${subprojects[@]}"; do
write_subproject_status ${subproject}
done
}

main "$@"
Loading