Update CodeQL CLI dependency to 2.12.7. #49
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CodeQL Standard Library Upgrade tests | |
| # Run this workflow every time the "supported_codeql_configs.json" file is changed | |
| on: | |
| pull_request: | |
| branches: | |
| - main | |
| - "rc/**" | |
| - next | |
| paths: | |
| - "supported_codeql_configs.json" | |
| workflow_dispatch: | |
| jobs: | |
| prepare-unit-test-matrix: | |
| name: Prepare CodeQL unit test matrix | |
| runs-on: ubuntu-22.04 | |
| outputs: | |
| matrix: ${{ steps.export-unit-test-matrix.outputs.matrix }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v2 | |
| - name: Export unit test matrix | |
| id: export-unit-test-matrix | |
| run: | | |
| echo "::set-output name=matrix::$( | |
| jq --compact-output \ | |
| '.supported_environment | map([.+{os: "ubuntu-20.04-xl", codeql_standard_library_ident : .codeql_standard_library | sub("\/"; "_")}]) | flatten | {include: .}' \ | |
| supported_codeql_configs.json | |
| )" | |
| run-test-suites: | |
| name: Run unit tests | |
| needs: prepare-unit-test-matrix | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{fromJSON(needs.prepare-unit-test-matrix.outputs.matrix)}} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v2 | |
| - name: Setup Python 3 | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: "3.x" | |
| - name: Cache CodeQL | |
| id: cache-codeql | |
| uses: actions/[email protected] | |
| with: | |
| # A list of files, directories, and wildcard patterns to cache and restore | |
| path: ${{github.workspace}}/codeql_home | |
| # An explicit key for restoring and saving the cache | |
| key: codeql-home-${{matrix.os}}-${{matrix.codeql_cli}}-${{matrix.codeql_standard_library}} | |
| - name: Install CodeQL | |
| if: steps.cache-codeql.outputs.cache-hit != 'true' | |
| uses: ./.github/actions/install-codeql | |
| with: | |
| codeql-cli-version: ${{matrix.codeql_cli}} | |
| codeql-stdlib-version: ${{matrix.codeql_standard_library}} | |
| codeql-home: ${{ github.workspace }}/codeql_home | |
| - name: Run test suites | |
| id: run-test-suites | |
| env: | |
| RUNNER_OS: ${{runner.os}} | |
| RUNNER_TEMP: ${{runner.temp}} | |
| CODEQL_CLI: ${{matrix.codeql_cli}} | |
| CODEQL_STDLIB: ${{matrix.codeql_standard_library}} | |
| CODEQL_STDLIB_IDENT: ${{matrix.codeql_standard_library_ident}} | |
| shell: python | |
| run: | | |
| import os | |
| import sys | |
| import subprocess | |
| import re | |
| import json | |
| from pathlib import Path | |
| def print_error(fmt, *args): | |
| print(f"::error::{fmt}", *args) | |
| def print_error_and_fail(fmt, *args): | |
| print_error(fmt, args) | |
| sys.exit(1) | |
| def print_debug(fmt, *args): | |
| print(f"::debug::{fmt}", *args) | |
| def print_warning(fmt, *args): | |
| print(f"::warning::{fmt}", *args) | |
| def set_output(key, value): | |
| print(f"::set-output name={key}::{value}") | |
| cli_version = os.environ['CODEQL_CLI'] | |
| stdlib_ref = os.environ['CODEQL_STDLIB'] | |
| stdlib_ref_ident = os.environ['CODEQL_STDLIB_IDENT'] | |
| workspace = os.environ['GITHUB_WORKSPACE'] | |
| runner_os = os.environ['RUNNER_OS'] | |
| runner_temp = os.environ['RUNNER_TEMP'] | |
| codeql_home = os.path.join(workspace, 'codeql_home') | |
| codeql_bin = os.path.join(codeql_home, 'codeql', 'codeql') | |
| if runner_os == "Windows": | |
| codeql_bin += ".exe" | |
| test_report_path = os.path.join(runner_temp, f"test_report_{runner_os}_{cli_version}_{stdlib_ref_ident}.json") | |
| os.makedirs(os.path.dirname(test_report_path), exist_ok=True) | |
| with open(test_report_path, 'w') as test_report_file: | |
| codeql_home = os.path.join(workspace, 'codeql_home') | |
| stdlib_path = os.path.join(codeql_home, 'codeql-stdlib') | |
| cpp_test_root = Path(stdlib_path, 'cpp/ql/test') | |
| print(f"Executing tests found (recursively) in the directory '{cpp_test_root}'") | |
| cp = subprocess.run([codeql_bin, "test", "run", "--format=json", cpp_test_root], stdout=test_report_file, stderr=subprocess.PIPE) | |
| if cp.returncode != 0: | |
| print_error_and_fail(f"Failed to run tests with return code {cp.returncode} and error {cp.stderr}") | |
| test_summary = cp.stderr.decode('utf-8') | |
| m = re.match(r"Executing (?P<tests>\d+) tests in (?P<dirs>\d+) directories\.(?P<progress>.*?)\r?\nAll (?P<passed>\d+) tests passed\.", test_summary, re.DOTALL) | |
| if m == None: | |
| print_error_and_fail(f"Unable to parse test summary: {test_summary}") | |
| tests = int(m.group('tests')) | |
| dirs = int(m.group('dirs')) | |
| passed = int(m.group('passed')) | |
| print(f"Executed {tests} tests in {dirs} dirs and {passed} passed.") | |
| test_summary_path = os.path.join(runner_temp, f"test_summary_{runner_os}_{cli_version}_{stdlib_ref_ident}.json") | |
| with open(test_summary_path, 'w') as test_summary_file: | |
| json.dump({ | |
| "os": runner_os, | |
| "cli_version": cli_version, | |
| "stdlib_ref": stdlib_ref, | |
| "tests": tests, | |
| "dirs": dirs, | |
| "passed": passed | |
| }, test_summary_file) | |
| - name: Upload test results | |
| uses: actions/upload-artifact@v2 | |
| with: | |
| name: test-results-${{runner.os}}-${{matrix.codeql_cli}}-${{matrix.codeql_standard_library_ident}} | |
| path: | | |
| ${{runner.temp}}/test_report_${{runner.os}}_${{matrix.codeql_cli}}_${{matrix.codeql_standard_library_ident}}.json | |
| ${{runner.temp}}/test_summary_${{runner.os}}_${{matrix.codeql_cli}}_${{matrix.codeql_standard_library_ident}}.json | |
| if-no-files-found: error | |
| validate-test-results: | |
| name: Validate test results | |
| needs: [run-test-suites] | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: Install Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.9" | |
| - name: Collect test results | |
| uses: actions/download-artifact@v2 | |
| - name: Validate test results | |
| shell: python | |
| run: | | |
| import os | |
| import sys | |
| import json | |
| from pathlib import Path | |
| def print_error(fmt, *args): | |
| print(f"::error::{fmt}", *args) | |
| def print_error_and_fail(fmt, *args): | |
| print_error(fmt, args) | |
| sys.exit(1) | |
| workspace = Path(os.environ['GITHUB_WORKSPACE']) | |
| test_summary_files = workspace.glob("*/test_summary_*.json") | |
| total_tests = 0 | |
| total_dirs = 0 | |
| total_passed = 0 | |
| for test_summary_file_path in test_summary_files: | |
| with open(test_summary_file_path, 'r') as test_summary_file: | |
| test_summary = json.load(test_summary_file) | |
| print(f"os: {test_summary['os']} cli: {test_summary['cli_version']} stdlib: {test_summary['stdlib_ref']} tests: {test_summary['tests']} dirs: {test_summary['dirs']} passed: {test_summary['passed']}") | |
| total_tests += test_summary['tests'] | |
| total_dirs += test_summary['dirs'] | |
| total_passed += test_summary['passed'] | |
| print(f"Total tests: {total_tests} total dirs: {total_dirs} total passed: {total_passed}") | |
| if total_passed != total_tests: | |
| print_error_and_fail(f"{total_tests - total_passed} out of ${total_tests} failed!") | |
| print("All tests passed!") |