ci: add data validation workflow

.github/workflows/tests.yml

@@ -225,3 +225,56 @@ jobs:
           python -m pip install -r test/requirements.txt
           python -m pytest --ignore=test\cunit --pastebin=failed --no-flaky-report -sr a
           deactivate
+
+  validation:
+    runs-on: ubuntu-20.04
+
+    needs: build-linux
+
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.5", "3.6", "3.7", "3.8", "3.9", "3.10", "3.11"]
+
+    env:
+      AUSTIN_TESTS_PYTHON_VERSIONS: ${{ matrix.python-version }}
+
+    name: Data validation with Python ${{ matrix.python-version }}
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Install build dependencies
+        run: |
+          sudo apt-get -y install libunwind-dev binutils-dev libiberty-dev
+
+      - name: Install Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}-dev
+
+      - name: Install Python 3.10
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+
+      - name: Compile Austin
+        run: |
+          autoreconf --install
+          ./configure --enable-debug-symbols true
+          make
+
+      - name: Install runtime dependencies
+        run: |
+          python3.10 -m venv .venv
+          source .venv/bin/activate
+          pip install --upgrade pip
+          pip install -r scripts/requirements-val.txt
+          deactivate
+
+      - name: Run data validation
+        run: |
+          ulimit -c unlimited
+
+          source .venv/bin/activate
+          python scripts/validation.py --ignore-errors
+          deactivate

scripts/benchmark.py

@@ -7,19 +7,15 @@
 from textwrap import wrap
 import typing as t
 from argparse import ArgumentParser
-from itertools import product
 from math import floor, log
 from pathlib import Path
 
 from scipy.stats import ttest_ind
 
-sys.path.insert(0, str(Path(__file__).parent.parent))
+from common import download_release
+
+from test.utils import metadata, target
 
-import tarfile
-from io import BytesIO
-from test.utils import Variant, metadata, target
-from urllib.error import HTTPError
-from urllib.request import urlopen
 
 VERSIONS = ("3.4.1", "3.5.0", "dev")
 SCENARIOS = [
@@ -99,38 +95,6 @@ def get_stats(output: str) -> t.Optional[dict]:
         return None
 
 
-def download_release(version: str, dest: Path, variant_name: str = "austin") -> Variant:
-    if version == "dev":
-        return Variant(f"src/{variant_name}")
-
-    binary_dest = dest / version
-    binary = binary_dest / variant_name
-
-    if not binary.exists():
-        prefix = "https://github.com/p403n1x87/austin/releases/download/"
-        for flavour, v in product({"-gnu", ""}, {"", "v"}):
-            try:
-                with urlopen(
-                    f"{prefix}v{version}/{variant_name}-{v}{version}{flavour}-linux-amd64.tar.xz"
-                ) as stream:
-                    buffer = BytesIO(stream.read())
-                    binary_dest.mkdir(parents=True, exist_ok=True)
-                    tar = tarfile.open(fileobj=buffer, mode="r:xz")
-                    tar.extract(variant_name, str(binary_dest))
-            except HTTPError:
-                continue
-            break
-        else:
-            raise RuntimeError(f"Could not download Austin version {version}")
-
-    variant = Variant(str(binary))
-
-    out = variant("-V").stdout
-    assert f"{variant_name} {version}" in out, (f"{variant_name} {version}", out)
-
-    return variant
-
-
 class Outcome:
     def __init__(self, data: list[float]) -> None:
         self.data = data

scripts/common.py

@@ -0,0 +1,59 @@
+import sys
+from itertools import product
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+import tarfile
+from io import BytesIO
+from test.utils import Variant
+from urllib.error import HTTPError
+from urllib.request import urlopen
+import json
+
+
+def get_latest_release() -> str:
+    with urlopen(
+        "https://api.github.com/repos/p403n1x87/austin/releases/latest"
+    ) as stream:
+        return json.loads(stream.read().decode("utf-8"))["tag_name"].strip("v")
+
+
+def download_release(version: str, dest: Path, variant_name: str = "austin") -> Variant:
+    if version == "dev":
+        return Variant(f"src/{variant_name}")
+
+    binary_dest = dest / version
+    binary = binary_dest / variant_name
+
+    if not binary.exists():
+        prefix = "https://github.com/p403n1x87/austin/releases/download/"
+        for flavour, v in product({"-gnu", ""}, {"", "v"}):
+            try:
+                with urlopen(
+                    f"{prefix}v{version}/{variant_name}-{v}{version}{flavour}-linux-amd64.tar.xz"
+                ) as stream:
+                    buffer = BytesIO(stream.read())
+                    binary_dest.mkdir(parents=True, exist_ok=True)
+                    tar = tarfile.open(fileobj=buffer, mode="r:xz")
+                    tar.extract(variant_name, str(binary_dest))
+            except HTTPError:
+                continue
+            break
+        else:
+            raise RuntimeError(f"Could not download Austin version {version}")
+
+    variant = Variant(str(binary))
+
+    out = variant("-V").stdout
+    assert f"{variant_name} {version}" in out, (f"{variant_name} {version}", out)
+
+    return variant
+
+
+def download_latest(dest: Path, variant_name: str = "austin") -> Variant:
+    return download_release(get_latest_release(), dest, variant_name)
+
+
+def get_dev(variant_name: str = "austin") -> Variant:
+    return download_release("dev", None, variant_name)

scripts/requirements-val.txt

@@ -0,0 +1,3 @@
+austin-python~=1.5
+numpy
+scipy