Skip to content

Commit

Permalink
ci: unify llvm parameters
Browse files Browse the repository at this point in the history
  • Loading branch information
@alandefreitas
alandefreitas committed Aug 30, 2024
1 parent 3500877 commit c08c6cd
Show file tree
Hide file tree
Showing 2 changed files with 107 additions and 113 deletions.
57 changes: 57 additions & 0 deletions .github/llvm-matrix.js
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
const fs = require('fs');
const core = require('@actions/core');
const { exec } = require('child_process');

// Function to check if a file exists on the server
const fileExists = (url) => {
return new Promise((resolve) => {
exec(`curl -s -o /dev/null -w "%{http_code}" -I "${url}"`, (error, stdout) => {
if (error) {
resolve(false);
} else {
resolve(stdout.trim() === '200');
}
});
});
};

// Read the JSON string from the file
const matrixJson = fs.readFileSync('matrix.json', 'utf8');

// Parse the JSON string into an array of objects
const matrixEntries = JSON.parse(matrixJson);

// Create a new array to store unique entries based on llvm-archive-filename
const seenFilenames = new Set();
const uniqueEntries = [];

(async () => {
for (const entry of matrixEntries) {
const filename = entry['llvm-archive-filename'];
if (!seenFilenames.has(filename)) {
seenFilenames.add(filename);
const url = `https://www.mrdocs.com/llvm+clang/${filename}`;
const exists = await fileExists(url);
if (!exists) {
uniqueEntries.push(entry);
}
}
}

// Convert the new array back to a JSON string
const uniqueMatrixJson = JSON.stringify(uniqueEntries);

// Output the filtered JSON string using core.setOutput
core.setOutput('llvm-matrix', uniqueMatrixJson);

// Print matrix to console
console.log(`LLVM Matrix (${uniqueEntries.length} entries):`)
uniqueEntries.forEach(obj => {
console.log(`- ${obj.name}`)
for (const key in obj) {
if (key !== 'name') {
console.log(` ${key}: ${JSON.stringify(obj[key])}`)
}
}
})
})();
163 changes: 50 additions & 113 deletions .github/workflows/ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,13 @@ jobs:
name: Generate Test Matrix
outputs:
matrix: ${{ steps.cpp-matrix.outputs.matrix }}
llvm-matrix: ${{ steps.llvm-matrix.outputs.llvm-matrix }}
steps:
- name: Checkout
uses: actions/checkout@v4

- name: Generate Test Matrix
uses: alandefreitas/cpp-actions/[email protected].4
uses: alandefreitas/cpp-actions/[email protected].5
id: cpp-matrix
with:
compilers: |
Expand All @@ -45,14 +49,34 @@ jobs:
msvc Optimized-Debug: /Ob1 /O2 /Zi
cxxflags: |
msvc Optimized-Debug: /Ob1 /O2 /Zi
extra-values: |
llvm-hash: e1065370aaacb1b1cb48e77d37d376bf024f4a39
llvm-id: {{ substr llvm-hash 0 7 }}
llvm-build-preset-prefix: {{#if optimized-debug}}debwithopt{{else}}{{lowercase build-type}}{{/if}}
llvm-build-preset-suffix: {{#if (ieq os 'windows') }}win{{else}}unix{{/if}}
llvm-build-preset: {{ llvm-build-preset-prefix }}-{{ llvm-build-preset-suffix }}
llvm-compiler-key: {{ compiler }}-{{ version }}
llvm-cache-key: llvm-libcxx-{{ lowercase os }}-{{ llvm-compiler-key }}-{{ llvm-build-preset-prefix }}-{{ llvm-hash }}
llvm-root: ../third-party/llvm-project/install
llvm-archive-basename: llvm-{{ os }}-{{ llvm-build-preset-prefix }}-{{ llvm-id }}
llvm-archive-extension: {{#if (ieq os 'windows') }}7z{{else}}tar.bz2{{/if}}
llvm-archive-filename: {{ llvm-archive-basename }}.{{ llvm-archive-extension }}
output-file: matrix.json

- name: Generate LLVM Test Matrix
id: llvm-matrix
run: |
set -x
npm install @actions/core
node .github/llvm-matrix.js
build:
needs: cpp-matrix

strategy:
fail-fast: false
matrix:
include: ${{ fromJSON(needs.cpp-matrix.outputs.matrix) }}
include: ${{ fromJSON(needs.cpp-matrix.outputs.llvm-matrix) }}

defaults:
run:
Expand All @@ -70,7 +94,7 @@ jobs:
uses: actions/checkout@v4

- name: Setup CMake
uses: alandefreitas/cpp-actions/[email protected].4
uses: alandefreitas/cpp-actions/[email protected].5
id: setup-cmake
with:
version: ${{ matrix.compiler == 'clang' && '3.26' || '>=3.26' }}
Expand All @@ -82,15 +106,15 @@ jobs:
if: ${{ runner.os == 'Windows' }}

- name: Setup C++
uses: alandefreitas/cpp-actions/[email protected].4
uses: alandefreitas/cpp-actions/[email protected].5
id: setup-cpp
with:
compiler: ${{ matrix.compiler }}
version: ${{ matrix.version }}
check-latest: ${{ matrix.compiler == 'clang' }}

- name: Install System Packages
uses: alandefreitas/cpp-actions/[email protected].4
uses: alandefreitas/cpp-actions/[email protected].5
id: package-install
env:
DEBIAN_FRONTEND: 'noninteractive'
Expand Down Expand Up @@ -225,50 +249,12 @@ jobs:
fi
echo -E "libxml2-root=$libxml2_root" >> $GITHUB_OUTPUT
- name: LLVM Parameters
id: llvm-parameters
run: |
llvm_hash=e1065370aaacb1b1cb48e77d37d376bf024f4a39
echo -E "llvm-hash=$llvm_hash" >> $GITHUB_OUTPUT
llvm_id=$(echo $llvm_hash | cut -c1-7)
echo -E "llvm-id=$llvm_id" >> $GITHUB_OUTPUT
llvm_build_preset=${{ runner.os == 'Windows' && 'relwithdebinfo-win' || 'release-unix' }}
if [[ ${{ matrix.optimized-debug }} == 'true' ]]; then
llvm_build_preset=debwithopt-
else
if [[ ${{ runner.os }} == 'Windows' ]]; then
llvm_build_preset=relwithdebinfo-
else
llvm_build_preset=release-
fi
fi
if [[ ${{ runner.os }} == 'Windows' ]]; then
llvm_build_preset+=win
else
llvm_build_preset+=unix
fi
echo -E "llvm-build-preset=$llvm_build_preset" >> $GITHUB_OUTPUT
compiler_and_version=${{ matrix.compiler }}-${{ matrix.version }}
workflow_factor=
if [[ ${{ matrix.build-type }} == 'Debug' ]]; then
workflow_factor=-debug
fi
echo -E "llvm-cache-key=llvm-libcxx-${{ runner.os }}-$compiler_and_version$workflow_factor-$llvm_build_preset-$llvm_hash" >> $GITHUB_OUTPUT
cd ..
llvm_root=$(pwd)/third-party/llvm-project/install
if [[ ${{ runner.os }} == 'Windows' ]]; then
llvm_root=$(echo "$llvm_root" | sed 's/\\/\//g')
llvm_root=$(echo $llvm_root | sed 's|^/d/|D:/|')
echo "$llvm_root"
fi
echo -E "llvm-root=$llvm_root" >> $GITHUB_OUTPUT
- name: LLVM Binaries
id: llvm-cache
uses: actions/cache@v4
with:
path: ${{ steps.llvm-parameters.outputs.llvm-root }}
key: ${{ steps.llvm-parameters.outputs.llvm-cache-key }}
path: ${{ matrix.llvm-root }}
key: ${{ matrix.llvm-cache-key }}

- name: Install LLVM
id: llvm-install
Expand All @@ -287,7 +273,7 @@ jobs:
git config --global advice.detachedHead false
git init
git remote add origin https://github.com/llvm/llvm-project.git
git fetch --depth 1 origin ${{ steps.llvm-parameters.outputs.llvm-hash }}
git fetch --depth 1 origin ${{ matrix.llvm-hash }}
git checkout FETCH_HEAD
# Copy presets
Expand All @@ -298,7 +284,7 @@ jobs:
cd llvm
llvm_root=$(pwd)
cmake --version
cmake -S . -B ./build --preset=${{ steps.llvm-parameters.outputs.llvm-build-preset }} -DCMAKE_CXX_COMPILER=${{ steps.setup-cpp.outputs.cxx }} -DCMAKE_C_COMPILER=${{ steps.setup-cpp.outputs.cc }}
cmake -S . -B ./build --preset=${{ matrix.llvm-build-preset }} -DCMAKE_CXX_COMPILER=${{ steps.setup-cpp.outputs.cxx }} -DCMAKE_C_COMPILER=${{ steps.setup-cpp.outputs.cc }}
if [[ ${{ runner.os }} == 'Linux' ]]; then
cmake --build ./build --target help
fi
Expand Down Expand Up @@ -349,7 +335,7 @@ jobs:
node-version: '18'

- name: CMake Workflow
uses: alandefreitas/cpp-actions/[email protected].4
uses: alandefreitas/cpp-actions/[email protected].5
with:
cmake-version: ${{ matrix.compiler == 'clang' && '3.26' || '>=3.26' }}
cxxstd: ${{ matrix.cxxstd }}
Expand All @@ -363,8 +349,8 @@ jobs:
install-prefix: .local
extra-args: |
-D MRDOCS_BUILD_DOCS=OFF
-D LLVM_ROOT=${{ steps.llvm-parameters.outputs.llvm-root }}
-D Clang_ROOT=${{ steps.llvm-parameters.outputs.llvm-root }}
-D LLVM_ROOT=${{ matrix.llvm-root }}
-D Clang_ROOT=${{ matrix.llvm-root }}
-D duktape_ROOT=${{ steps.duktape-install.outputs.duktape-root }}
-D Duktape_ROOT=${{ steps.duktape-install.outputs.duktape-root }}
-D fmt_ROOT=${{ steps.fmt-install.outputs.fmt-root }}
Expand All @@ -387,7 +373,7 @@ jobs:
retention-days: 1

- name: FlameGraph
uses: alandefreitas/cpp-actions/[email protected].4
uses: alandefreitas/cpp-actions/[email protected].5
if: matrix.time-trace
with:
build-dir: build
Expand Down Expand Up @@ -415,7 +401,7 @@ jobs:
contents: write
steps:
- name: Install packages
uses: alandefreitas/cpp-actions/[email protected].4
uses: alandefreitas/cpp-actions/[email protected].5
id: package-install
with:
apt-get: build-essential asciidoctor cmake bzip2 git
Expand Down Expand Up @@ -448,7 +434,7 @@ jobs:
echo -e "MRDOCS_ROOT=$MRDOCS_ROOT" >> $GITHUB_ENV
- name: Clone Boost.URL
uses: alandefreitas/cpp-actions/[email protected].4
uses: alandefreitas/cpp-actions/[email protected].5
id: boost-url-clone
with:
branch: develop
Expand Down Expand Up @@ -638,7 +624,7 @@ jobs:
git config --global --add safe.directory "$(pwd)"
- name: Create changelog
uses: alandefreitas/cpp-actions/[email protected].4
uses: alandefreitas/cpp-actions/[email protected].5
with:
output-path: CHANGELOG.md
thank-non-regular: ${{ startsWith(github.ref, 'refs/tags/') }}
Expand Down Expand Up @@ -678,7 +664,7 @@ jobs:
strategy:
fail-fast: false
matrix:
include: ${{ fromJSON(needs.cpp-matrix.outputs.matrix) }}
include: ${{ fromJSON(needs.cpp-matrix.outputs.llvm-matrix) }}

defaults:
run:
Expand All @@ -693,66 +679,17 @@ jobs:

steps:
- name: Install packages
uses: alandefreitas/cpp-actions/[email protected].4
uses: alandefreitas/cpp-actions/[email protected].5
id: package-install
with:
apt-get: build-essential asciidoctor cmake bzip2 git curl

- name: LLVM Parameters
id: llvm-parameters
run: |
llvm_hash=e1065370aaacb1b1cb48e77d37d376bf024f4a39
echo -E "llvm-hash=$llvm_hash" >> $GITHUB_OUTPUT
llvm_id=$(echo $llvm_hash | cut -c1-7)
echo -E "llvm-id=$llvm_id" >> $GITHUB_OUTPUT
llvm_build_preset=${{ runner.os == 'Windows' && 'relwithdebinfo-win' || 'release-unix' }}
if [[ ${{ matrix.optimized-debug }} == 'true' ]]; then
llvm_build_preset=debwithopt-
else
if [[ ${{ runner.os }} == 'Windows' ]]; then
llvm_build_preset=relwithdebinfo-
else
llvm_build_preset=release-
fi
fi
if [[ ${{ runner.os }} == 'Windows' ]]; then
llvm_build_preset+=win
else
llvm_build_preset+=unix
fi
echo -E "llvm-build-preset=$llvm_build_preset" >> $GITHUB_OUTPUT
compiler_and_version=${{ matrix.compiler }}-${{ matrix.version }}
workflow_factor=
if [[ ${{ matrix.build-type }} == 'Debug' ]]; then
workflow_factor=-debug
fi
echo -E "llvm-cache-key=llvm-libcxx-${{ runner.os }}-$compiler_and_version$workflow_factor-$llvm_build_preset-$llvm_hash" >> $GITHUB_OUTPUT
llvm_archive_basename=llvm-${{ runner.os }}-$llvm_id
echo -E "llvm-archive-basename=$llvm_archive_basename" >> $GITHUB_OUTPUT
if [[ ${{ runner.os }} == 'Windows' ]]; then
llvm_archive_extension=7z
else
llvm_archive_extension=tar.bz2
fi
echo -E "llvm-archive-extension=$llvm_archive_extension" >> $GITHUB_OUTPUT
llvm_archive_filename=$llvm_archive_basename.$llvm_archive_extension
echo -E "llvm-archive-filename=$llvm_archive_filename" >> $GITHUB_OUTPUT
cd ..
llvm_root=$(pwd)/third-party/llvm-project/install
if [[ ${{ runner.os }} == 'Windows' ]]; then
llvm_root=$(echo "$llvm_root" | sed 's/\\/\//g')
llvm_root=$(echo $llvm_root | sed 's|^/d/|D:/|')
echo "$llvm_root"
fi
echo -E "llvm-root=$llvm_root" >> $GITHUB_OUTPUT
- name: Check website releases
id: website-releases
run: |
set -x
archive_url="https://mrdocs.com/llvm+clang/${{ steps.llvm-parameters.outputs.llvm-archive-filename }}"
http_status=$(curl -s -o /dev/null -w "%{http_code}" "$archive_url")
archive_url="https://mrdocs.com/llvm+clang/${{ matrix.llvm-archive-filename }}"
http_status=$(curl -s -o /dev/null -w "%{http_code}" -I "$archive_url")
if [ "$http_status" -eq 200 ]; then
exists="true"
else
Expand All @@ -765,8 +702,8 @@ jobs:
if: steps.website-releases.outputs.exists != 'true'
uses: actions/cache@v4
with:
path: ${{ steps.llvm-parameters.outputs.llvm-root }}
key: ${{ steps.llvm-parameters.outputs.llvm-cache-key }}
path: ${{ matrix.llvm-root }}
key: ${{ matrix.llvm-cache-key }}

- name: Compress LLVM
id: llvm-upload
Expand All @@ -781,9 +718,9 @@ jobs:
# Use 7z on windows
if [[ ${{ runner.os }} == 'Windows' ]]; then
7z a -t7z -m0=lzma2 -mx=9 -mfb=64 -md=32m -ms=on ${{ steps.llvm-parameters.outputs.llvm-archive-filename }} install
7z a -t7z -m0=lzma2 -mx=9 -mfb=64 -md=32m -ms=on ${{ matrix.llvm-archive-filename }} install
else
tar -cjf ${{ steps.llvm-parameters.outputs.llvm-archive-filename }} -C install .
tar -cjf ${{ matrix.llvm-archive-filename }} -C install .
fi
- name: Publish LLVM on website
Expand All @@ -803,6 +740,6 @@ jobs:
# Remove previous demos associated with this tag
llvm_dir="/var/www/mrdox.com/llvm+clang"
chmod 755 -R ${{ steps.llvm-parameters.outputs.llvm-archive-filename }}
scp -o StrictHostKeyChecking=no $(pwd)/${{ steps.llvm-parameters.outputs.llvm-archive-filename }} [email protected]:$llvm_dir/
chmod 755 -R ${{ matrix.llvm-archive-filename }}
scp -o StrictHostKeyChecking=no $(pwd)/${{ matrix.llvm-archive-filename }} [email protected]:$llvm_dir/

0 comments on commit c08c6cd

Please sign in to comment.