docs & script for running cargo mutants locally on CI limitation

contrib/tools/local-mutation-testing.sh

@@ -0,0 +1,84 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# Install cargo-mutants
+cargo install --version 24.7.1 cargo-mutants --locked
+
+# Create diff file between current branch and develop branch
+git diff origin/develop...HEAD > git.diff
+
+# Remove git diff files about removed/renamed files
+awk '
+    /^diff --git/ {
+        diff_line = $0
+        getline
+        if ($0 !~ /^(deleted file mode|similarity index)/) {
+            print diff_line
+            print
+        }
+    }
+    !/^(diff --git|deleted file mode|similarity index|rename from|rename to)/ {print}
+' git.diff > processed.diff
+
+# Extract mutants based on the processed diff
+cargo mutants --in-diff processed.diff --list > all_mutants.txt
+
+# Create a directory for organizing mutants
+mkdir -p mutants_by_package
+
+# Organize mutants into files based on their main folder
+while IFS= read -r line; do
+    package=$(echo "$line" | cut -d'/' -f1)
+
+    case $package in
+        "stackslib")
+            echo "$line" >> "mutants_by_package/stackslib.txt"
+            ;;
+        "testnet")
+            echo "$line" >> "mutants_by_package/stacks-node.txt"
+            ;;
+        "stacks-signer")
+            echo "$line" >> "mutants_by_package/stacks-signer.txt"
+            ;;
+        *)
+            echo "$line" >> "mutants_by_package/small-packages.txt"
+            ;;
+    esac
+done < all_mutants.txt
+
+# Function to run mutants for a package
+run_mutants() {
+    local package=$1
+    local threshold=$2
+    local output_dir=$3
+    local mutant_file="mutants_by_package/${package}.txt"
+
+    if [ ! -f "$mutant_file" ]; then
+        echo "No mutants found for $package"
+        return
+    fi
+
+    local regex_pattern=$(sed 's/[][()\.^$*+?{}|]/\\&/g' "$mutant_file" | paste -sd'|' -)
+    local mutant_count=$(cargo mutants -F "$regex_pattern" -E ": replace .{1,2} with .{1,2} in " --list | wc -l)
+
+    if [ "$mutant_count" -gt "$threshold" ]; then
+        echo "Running mutants for $package ($mutant_count mutants)"
+        cargo mutants --timeout-multiplier 1.5 --no-shuffle -vV \
+            -F "$regex_pattern" \
+            -E ": replace .{1,2} with .{1,2} in " \
+            --output "$output_dir" \
+            --test-tool=nextest \
+            --package "$package" \
+            -- --all-targets --test-threads 1
+
+        echo $? > "${output_dir}/exit_code.txt"
+    else
+        echo "Skipping $package, only $mutant_count mutants (threshold: $threshold)"
+    fi
+}
+
+# Run mutants for each wanted package
+run_mutants "stacks-signer" 500 "./stacks-signer_mutants"
+run_mutants "stacks-node" 540 "./stacks-node_mutants"
+run_mutants "stackslib" 72 "./stackslib_mutants"

docs/mutation-testing.md

@@ -0,0 +1,109 @@
+# Mutation Testing
+
+This document describes how to run mutation testing locally to mimic the outcome of a PR, without the CI limitation it provides by timing out after 6 hours.
+[Here is the script](../contrib/tools/local-mutation-testing.sh) to run mutation locally running the mutants created by the changes between the current branch and develop. It does automatically all the steps explained below.
+
+From the root level of the stacks-core repository run
+```sh
+./contrib/tools/local-mutation-testing.sh
+```
+
+## Prerequirements
+
+Install the cargo mutants library
+```sh
+cargo install --version 24.7.1 cargo-mutants --locked
+```
+
+
+## Steps 
+1. be on source branch you would use for the PR.
+2. create diff file comparing this branch with the `develop` branch
+    ```sh
+    git diff origin/develop..HEAD > git.diff
+    ```
+3. clean up the diff file and create auxiliary files
+   ```sh
+    awk '
+        /^diff --git/ {
+            diff_line = $0
+            getline
+            if ($0 !~ /^(deleted file mode|similarity index)/) {
+                print diff_line
+                print
+            }
+        }
+        !/^(diff --git|deleted file mode|similarity index|rename from|rename to)/ {print}
+    ' git.diff > processed.diff
+
+    # Extract mutants based on the processed diff
+    cargo mutants --in-diff processed.diff --list > all_mutants.txt
+
+    # Create a directory for organizing mutants
+    mkdir -p mutants_by_package
+
+    # Organize mutants into files based on their main folder
+    while IFS= read -r line; do
+          package=$(echo "$line" | cut -d'/' -f1)
+
+          case $package in
+            "stackslib")
+              echo "$line" >> "mutants_by_package/stackslib.txt"
+              ;;
+            "testnet")
+              echo "$line" >> "mutants_by_package/stacks-node.txt"
+              ;;
+            "stacks-signer")
+              echo "$line" >> "mutants_by_package/stacks-signer.txt"
+              ;;
+            *)
+              echo "$line" >> "mutants_by_package/small-packages.txt"
+              ;;
+          esac
+        done < all_mutants.txt
+   ```
+4. based on the package required to run the mutants for  
+   a. stackslib package 
+   ```sh
+    regex_pattern=$(sed 's/[][()\.^$*+?{}|]/\\&/g' "mutants_by_package/stackslib.txt" | paste -sd'|' -)
+
+    cargo mutants --timeout-multiplier 1.5 --no-shuffle -vV \
+        -F "$regex_pattern" \
+        -E ": replace .{1,2} with .{1,2} in " \
+        --output "./stackslib_mutants" \
+        --test-tool=nextest \
+        -- --all-targets --test-threads 1
+   ```
+   b. stacks-node (testnet) package
+   ```sh
+    regex_pattern=$(sed 's/[][()\.^$*+?{}|]/\\&/g' "mutants_by_package/testnet.txt" | paste -sd'|' -)
+
+    cargo mutants --timeout-multiplier 1.5 --no-shuffle -vV \
+        -F "$regex_pattern" \
+        -E ": replace .{1,2} with .{1,2} in " \
+        --output "./testnet_mutants" \
+        --test-tool=nextest \
+        -- --all-targets --test-threads 1
+   ```
+   c. stacks-signer
+   ```sh
+    regex_pattern=$(sed 's/[][()\.^$*+?{}|]/\\&/g' "mutants_by_package/stacks-signer.txt" | paste -sd'|' -)
+
+    cargo mutants --timeout-multiplier 1.5 --no-shuffle -vV \
+        -F "$regex_pattern" \
+        -E ": replace .{1,2} with .{1,2} in " \
+        --output "./stacks-signer_mutants" \
+        --test-tool=nextest \
+        -- --all-targets --test-threads 1
+   ```
+   d. all other packages combined  
+    ```sh
+    regex_pattern=$(sed 's/[][()\.^$*+?{}|]/\\&/g' "mutants_by_package/small-packages.txt" | paste -sd'|' -)
+
+    cargo mutants --timeout-multiplier 1.5 --no-shuffle -vV \
+        -F "$regex_pattern" \
+        -E ": replace .{1,2} with .{1,2} in " \
+        --output "./small-packages_mutants" \
+        --test-tool=nextest \
+        -- --all-targets --test-threads 1
+   ```