From 530b6894d1c85a267e61e9e8008971cbca285247 Mon Sep 17 00:00:00 2001 From: ASuciuX Date: Wed, 7 Aug 2024 21:16:15 +0300 Subject: [PATCH 1/7] docs & script for running cargo mutants locally on CI limitation --- contrib/tools/local-mutation-testing.sh | 84 ++++++++++++++++++ docs/mutation-testing.md | 109 ++++++++++++++++++++++++ 2 files changed, 193 insertions(+) create mode 100644 contrib/tools/local-mutation-testing.sh create mode 100644 docs/mutation-testing.md diff --git a/contrib/tools/local-mutation-testing.sh b/contrib/tools/local-mutation-testing.sh new file mode 100644 index 0000000000..a3d563682c --- /dev/null +++ b/contrib/tools/local-mutation-testing.sh @@ -0,0 +1,84 @@ +#!/bin/bash + +set -euo pipefail + +# Install cargo-mutants +cargo install --version 24.7.1 cargo-mutants --locked + +# Create diff file between current branch and develop branch +git diff origin/develop...HEAD > git.diff + +# Remove git diff files about removed/renamed files +awk ' + /^diff --git/ { + diff_line = $0 + getline + if ($0 !~ /^(deleted file mode|similarity index)/) { + print diff_line + print + } + } + !/^(diff --git|deleted file mode|similarity index|rename from|rename to)/ {print} +' git.diff > processed.diff + +# Extract mutants based on the processed diff +cargo mutants --in-diff processed.diff --list > all_mutants.txt + +# Create a directory for organizing mutants +mkdir -p mutants_by_package + +# Organize mutants into files based on their main folder +while IFS= read -r line; do + package=$(echo "$line" | cut -d'/' -f1) + + case $package in + "stackslib") + echo "$line" >> "mutants_by_package/stackslib.txt" + ;; + "testnet") + echo "$line" >> "mutants_by_package/stacks-node.txt" + ;; + "stacks-signer") + echo "$line" >> "mutants_by_package/stacks-signer.txt" + ;; + *) + echo "$line" >> "mutants_by_package/small-packages.txt" + ;; + esac +done < all_mutants.txt + +# Function to run mutants for a package +run_mutants() { + local package=$1 + local threshold=$2 + local output_dir=$3 + local mutant_file="mutants_by_package/${package}.txt" + + if [ ! -f "$mutant_file" ]; then + echo "No mutants found for $package" + return + fi + + local regex_pattern=$(sed 's/[][()\.^$*+?{}|]/\\&/g' "$mutant_file" | paste -sd'|' -) + local mutant_count=$(cargo mutants -F "$regex_pattern" -E ": replace .{1,2} with .{1,2} in " --list | wc -l) + + if [ "$mutant_count" -gt "$threshold" ]; then + echo "Running mutants for $package ($mutant_count mutants)" + cargo mutants --timeout-multiplier 1.5 --no-shuffle -vV \ + -F "$regex_pattern" \ + -E ": replace .{1,2} with .{1,2} in " \ + --output "$output_dir" \ + --test-tool=nextest \ + --package "$package" \ + -- --all-targets --test-threads 1 + + echo $? > "${output_dir}/exit_code.txt" + else + echo "Skipping $package, only $mutant_count mutants (threshold: $threshold)" + fi +} + +# Run mutants for each wanted package +run_mutants "stacks-signer" 500 "./stacks-signer_mutants" +run_mutants "stacks-node" 540 "./stacks-node_mutants" +run_mutants "stackslib" 72 "./stackslib_mutants" diff --git a/docs/mutation-testing.md b/docs/mutation-testing.md new file mode 100644 index 0000000000..7c635f3915 --- /dev/null +++ b/docs/mutation-testing.md @@ -0,0 +1,109 @@ +# Mutation Testing + +This document describes how to run mutation testing locally to mimic the outcome of a PR, without the CI limitation it provides by timing out after 6 hours. +[Here is the script](../contrib/tools/local-mutation-testing.sh) to run mutation locally running the mutants created by the changes between the current branch and develop. It does automatically all the steps explained below. + +From the root level of the stacks-core repository run +```sh +./contrib/tools/local-mutation-testing.sh +``` + +## Prerequirements + +Install the cargo mutants library +```sh +cargo install --version 24.7.1 cargo-mutants --locked +``` + + +## Steps +1. be on source branch you would use for the PR. +2. create diff file comparing this branch with the `develop` branch + ```sh + git diff origin/develop..HEAD > git.diff + ``` +3. clean up the diff file and create auxiliary files + ```sh + awk ' + /^diff --git/ { + diff_line = $0 + getline + if ($0 !~ /^(deleted file mode|similarity index)/) { + print diff_line + print + } + } + !/^(diff --git|deleted file mode|similarity index|rename from|rename to)/ {print} + ' git.diff > processed.diff + + # Extract mutants based on the processed diff + cargo mutants --in-diff processed.diff --list > all_mutants.txt + + # Create a directory for organizing mutants + mkdir -p mutants_by_package + + # Organize mutants into files based on their main folder + while IFS= read -r line; do + package=$(echo "$line" | cut -d'/' -f1) + + case $package in + "stackslib") + echo "$line" >> "mutants_by_package/stackslib.txt" + ;; + "testnet") + echo "$line" >> "mutants_by_package/stacks-node.txt" + ;; + "stacks-signer") + echo "$line" >> "mutants_by_package/stacks-signer.txt" + ;; + *) + echo "$line" >> "mutants_by_package/small-packages.txt" + ;; + esac + done < all_mutants.txt + ``` +4. based on the package required to run the mutants for + a. stackslib package + ```sh + regex_pattern=$(sed 's/[][()\.^$*+?{}|]/\\&/g' "mutants_by_package/stackslib.txt" | paste -sd'|' -) + + cargo mutants --timeout-multiplier 1.5 --no-shuffle -vV \ + -F "$regex_pattern" \ + -E ": replace .{1,2} with .{1,2} in " \ + --output "./stackslib_mutants" \ + --test-tool=nextest \ + -- --all-targets --test-threads 1 + ``` + b. stacks-node (testnet) package + ```sh + regex_pattern=$(sed 's/[][()\.^$*+?{}|]/\\&/g' "mutants_by_package/testnet.txt" | paste -sd'|' -) + + cargo mutants --timeout-multiplier 1.5 --no-shuffle -vV \ + -F "$regex_pattern" \ + -E ": replace .{1,2} with .{1,2} in " \ + --output "./testnet_mutants" \ + --test-tool=nextest \ + -- --all-targets --test-threads 1 + ``` + c. stacks-signer + ```sh + regex_pattern=$(sed 's/[][()\.^$*+?{}|]/\\&/g' "mutants_by_package/stacks-signer.txt" | paste -sd'|' -) + + cargo mutants --timeout-multiplier 1.5 --no-shuffle -vV \ + -F "$regex_pattern" \ + -E ": replace .{1,2} with .{1,2} in " \ + --output "./stacks-signer_mutants" \ + --test-tool=nextest \ + -- --all-targets --test-threads 1 + ``` + d. all other packages combined + ```sh + regex_pattern=$(sed 's/[][()\.^$*+?{}|]/\\&/g' "mutants_by_package/small-packages.txt" | paste -sd'|' -) + + cargo mutants --timeout-multiplier 1.5 --no-shuffle -vV \ + -F "$regex_pattern" \ + -E ": replace .{1,2} with .{1,2} in " \ + --output "./small-packages_mutants" \ + --test-tool=nextest \ + -- --all-targets --test-threads 1 + ``` From 14594ba59e9c95964704b37c15bc3a3189fe1a29 Mon Sep 17 00:00:00 2001 From: ASuciuX <151519329+ASuciuX@users.noreply.github.com> Date: Wed, 7 Aug 2024 21:31:13 +0300 Subject: [PATCH 2/7] Update docs/mutation-testing.md Co-authored-by: wileyj <2847772+wileyj@users.noreply.github.com> --- docs/mutation-testing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/mutation-testing.md b/docs/mutation-testing.md index 7c635f3915..c94b5ca36c 100644 --- a/docs/mutation-testing.md +++ b/docs/mutation-testing.md @@ -1,7 +1,7 @@ # Mutation Testing This document describes how to run mutation testing locally to mimic the outcome of a PR, without the CI limitation it provides by timing out after 6 hours. -[Here is the script](../contrib/tools/local-mutation-testing.sh) to run mutation locally running the mutants created by the changes between the current branch and develop. It does automatically all the steps explained below. +[Here is the script](../contrib/tools/local-mutation-testing.sh) to run the tests locally by running the mutants created by the changes between `HEAD` and develop. It does automatically all the steps explained below. From the root level of the stacks-core repository run ```sh From e21a0f1940f9aef73783e387448237e9e847d097 Mon Sep 17 00:00:00 2001 From: ASuciuX Date: Wed, 7 Aug 2024 22:08:34 +0300 Subject: [PATCH 3/7] changed script to be executable and add capitalization to the numbered lists --- contrib/tools/local-mutation-testing.sh | 0 docs/mutation-testing.md | 19 ++++++++++--------- 2 files changed, 10 insertions(+), 9 deletions(-) mode change 100644 => 100755 contrib/tools/local-mutation-testing.sh diff --git a/contrib/tools/local-mutation-testing.sh b/contrib/tools/local-mutation-testing.sh old mode 100644 new mode 100755 diff --git a/docs/mutation-testing.md b/docs/mutation-testing.md index 7c635f3915..383b3ff8eb 100644 --- a/docs/mutation-testing.md +++ b/docs/mutation-testing.md @@ -1,7 +1,8 @@ # Mutation Testing This document describes how to run mutation testing locally to mimic the outcome of a PR, without the CI limitation it provides by timing out after 6 hours. -[Here is the script](../contrib/tools/local-mutation-testing.sh) to run mutation locally running the mutants created by the changes between the current branch and develop. It does automatically all the steps explained below. +[Here is the script](../contrib/tools/local-mutation-testing.sh) to run mutation locally running the mutants created by the changes between the current branch and develop. +It does automatically all the steps explained below. From the root level of the stacks-core repository run ```sh @@ -17,12 +18,12 @@ cargo install --version 24.7.1 cargo-mutants --locked ## Steps -1. be on source branch you would use for the PR. -2. create diff file comparing this branch with the `develop` branch +1. Be on source branch you would use for the PR. +2. Create diff file comparing this branch with the `develop` branch ```sh git diff origin/develop..HEAD > git.diff ``` -3. clean up the diff file and create auxiliary files +3. Clean up the diff file and create auxiliary files ```sh awk ' /^diff --git/ { @@ -62,8 +63,8 @@ cargo install --version 24.7.1 cargo-mutants --locked esac done < all_mutants.txt ``` -4. based on the package required to run the mutants for - a. stackslib package +4. Based on the package required to run the mutants for + a. Stackslib package ```sh regex_pattern=$(sed 's/[][()\.^$*+?{}|]/\\&/g' "mutants_by_package/stackslib.txt" | paste -sd'|' -) @@ -74,7 +75,7 @@ cargo install --version 24.7.1 cargo-mutants --locked --test-tool=nextest \ -- --all-targets --test-threads 1 ``` - b. stacks-node (testnet) package + b. Stacks-node (testnet) package ```sh regex_pattern=$(sed 's/[][()\.^$*+?{}|]/\\&/g' "mutants_by_package/testnet.txt" | paste -sd'|' -) @@ -85,7 +86,7 @@ cargo install --version 24.7.1 cargo-mutants --locked --test-tool=nextest \ -- --all-targets --test-threads 1 ``` - c. stacks-signer + c. Stacks-signer ```sh regex_pattern=$(sed 's/[][()\.^$*+?{}|]/\\&/g' "mutants_by_package/stacks-signer.txt" | paste -sd'|' -) @@ -96,7 +97,7 @@ cargo install --version 24.7.1 cargo-mutants --locked --test-tool=nextest \ -- --all-targets --test-threads 1 ``` - d. all other packages combined + d. All other packages combined ```sh regex_pattern=$(sed 's/[][()\.^$*+?{}|]/\\&/g' "mutants_by_package/small-packages.txt" | paste -sd'|' -) From 1ad9e5e383d40f8a8461d7ecca6f8de24ccb9bf8 Mon Sep 17 00:00:00 2001 From: ASuciuX Date: Thu, 8 Aug 2024 18:53:53 +0300 Subject: [PATCH 4/7] add `||true` so it runs all packages, not just the first one found --- contrib/tools/local-mutation-testing.sh | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/contrib/tools/local-mutation-testing.sh b/contrib/tools/local-mutation-testing.sh index a3d563682c..80f3e16e3f 100755 --- a/contrib/tools/local-mutation-testing.sh +++ b/contrib/tools/local-mutation-testing.sh @@ -56,7 +56,7 @@ run_mutants() { if [ ! -f "$mutant_file" ]; then echo "No mutants found for $package" - return + return 0 fi local regex_pattern=$(sed 's/[][()\.^$*+?{}|]/\\&/g' "$mutant_file" | paste -sd'|' -) @@ -70,15 +70,17 @@ run_mutants() { --output "$output_dir" \ --test-tool=nextest \ --package "$package" \ - -- --all-targets --test-threads 1 + -- --all-targets --test-threads 1 || true echo $? > "${output_dir}/exit_code.txt" else echo "Skipping $package, only $mutant_count mutants (threshold: $threshold)" fi + + return 0 } # Run mutants for each wanted package -run_mutants "stacks-signer" 500 "./stacks-signer_mutants" -run_mutants "stacks-node" 540 "./stacks-node_mutants" -run_mutants "stackslib" 72 "./stackslib_mutants" +run_mutants "stacks-signer" 500 "./stacks-signer_mutants" || true +run_mutants "stacks-node" 540 "./stacks-node_mutants" || true +run_mutants "stackslib" 72 "./stackslib_mutants" || true From 9d137db19ebb5ffa8107fbace17c0253d2655067 Mon Sep 17 00:00:00 2001 From: ASuciuX Date: Thu, 8 Aug 2024 18:57:02 +0300 Subject: [PATCH 5/7] rust backtrace and bitcoind_test added --- contrib/tools/local-mutation-testing.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/tools/local-mutation-testing.sh b/contrib/tools/local-mutation-testing.sh index 80f3e16e3f..41592b5030 100755 --- a/contrib/tools/local-mutation-testing.sh +++ b/contrib/tools/local-mutation-testing.sh @@ -64,7 +64,7 @@ run_mutants() { if [ "$mutant_count" -gt "$threshold" ]; then echo "Running mutants for $package ($mutant_count mutants)" - cargo mutants --timeout-multiplier 1.5 --no-shuffle -vV \ + RUST_BACKTRACE=1 BITCOIND_TEST=1 cargo mutants --timeout-multiplier 1.5 --no-shuffle -vV \ -F "$regex_pattern" \ -E ": replace .{1,2} with .{1,2} in " \ --output "$output_dir" \ From f129669465da76626c57f4045b5d39fa09610b58 Mon Sep 17 00:00:00 2001 From: ASuciuX Date: Thu, 8 Aug 2024 20:31:53 +0300 Subject: [PATCH 6/7] add example to run one mutant --- docs/mutation-testing.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/mutation-testing.md b/docs/mutation-testing.md index 45110ce7aa..e75ba75bdf 100644 --- a/docs/mutation-testing.md +++ b/docs/mutation-testing.md @@ -108,3 +108,20 @@ cargo install --version 24.7.1 cargo-mutants --locked --test-tool=nextest \ -- --all-targets --test-threads 1 ``` + +## How to run one specific mutant to test it + +Example of output which had a missing mutant +```sh +MISSED stacks-signer/src/runloop.rs:424:9: replace >::run_one_pass -> Option> with None in 3.0s build + 9.3s test +``` + +Example of fix for it +```sh +RUST_BACKTRACE=1 BITCOIND_TEST=1 cargo mutants -vV -F "replace process_stackerdb_event" -E ": replace >::run_one_pass -> Option> with None in " --test-tool=nextest -- --run-ignored all --fail-fast --test-threads 1 +``` + +General command to run +```sh +RUST_BACKTRACE=1 BITCOIND_TEST=1 cargo mutants -vV -F "replace process_stackerdb_event" -E ": replace [modify this] with [modify this] in " --test-tool=nextest -- --run-ignored all --fail-fast --test-threads 1 +``` From 5cc974e2ddce40d4fa25a8a30ac71572b71fc2d6 Mon Sep 17 00:00:00 2001 From: ASuciuX Date: Thu, 8 Aug 2024 22:12:34 +0300 Subject: [PATCH 7/7] adjust the code blocks with line breaks for readability --- contrib/tools/local-mutation-testing.sh | 3 ++- docs/mutation-testing.md | 23 +++++++++++++++++++++-- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/contrib/tools/local-mutation-testing.sh b/contrib/tools/local-mutation-testing.sh index 41592b5030..11da6810e5 100755 --- a/contrib/tools/local-mutation-testing.sh +++ b/contrib/tools/local-mutation-testing.sh @@ -64,7 +64,8 @@ run_mutants() { if [ "$mutant_count" -gt "$threshold" ]; then echo "Running mutants for $package ($mutant_count mutants)" - RUST_BACKTRACE=1 BITCOIND_TEST=1 cargo mutants --timeout-multiplier 1.5 --no-shuffle -vV \ + RUST_BACKTRACE=1 BITCOIND_TEST=1 \ + cargo mutants --timeout-multiplier 1.5 --no-shuffle -vV \ -F "$regex_pattern" \ -E ": replace .{1,2} with .{1,2} in " \ --output "$output_dir" \ diff --git a/docs/mutation-testing.md b/docs/mutation-testing.md index e75ba75bdf..85fcd89a7f 100644 --- a/docs/mutation-testing.md +++ b/docs/mutation-testing.md @@ -68,6 +68,7 @@ cargo install --version 24.7.1 cargo-mutants --locked ```sh regex_pattern=$(sed 's/[][()\.^$*+?{}|]/\\&/g' "mutants_by_package/stackslib.txt" | paste -sd'|' -) + RUST_BACKTRACE=1 BITCOIND_TEST=1 \ cargo mutants --timeout-multiplier 1.5 --no-shuffle -vV \ -F "$regex_pattern" \ -E ": replace .{1,2} with .{1,2} in " \ @@ -79,6 +80,7 @@ cargo install --version 24.7.1 cargo-mutants --locked ```sh regex_pattern=$(sed 's/[][()\.^$*+?{}|]/\\&/g' "mutants_by_package/testnet.txt" | paste -sd'|' -) + RUST_BACKTRACE=1 BITCOIND_TEST=1 \ cargo mutants --timeout-multiplier 1.5 --no-shuffle -vV \ -F "$regex_pattern" \ -E ": replace .{1,2} with .{1,2} in " \ @@ -90,6 +92,7 @@ cargo install --version 24.7.1 cargo-mutants --locked ```sh regex_pattern=$(sed 's/[][()\.^$*+?{}|]/\\&/g' "mutants_by_package/stacks-signer.txt" | paste -sd'|' -) + RUST_BACKTRACE=1 BITCOIND_TEST=1 \ cargo mutants --timeout-multiplier 1.5 --no-shuffle -vV \ -F "$regex_pattern" \ -E ": replace .{1,2} with .{1,2} in " \ @@ -118,10 +121,26 @@ MISSED stacks-signer/src/runloop.rs:424:9: replace >::run_one_pass -> Option> with None in " --test-tool=nextest -- --run-ignored all --fail-fast --test-threads 1 +RUST_BACKTRACE=1 BITCOIND_TEST=1 \ +cargo mutants -vV \ + -F "replace process_stackerdb_event" \ + -E ": replace >::run_one_pass -> Option> with None in " \ + --test-tool=nextest \ + -- \ + --run-ignored all \ + --fail-fast \ + --test-threads 1 ``` General command to run ```sh -RUST_BACKTRACE=1 BITCOIND_TEST=1 cargo mutants -vV -F "replace process_stackerdb_event" -E ": replace [modify this] with [modify this] in " --test-tool=nextest -- --run-ignored all --fail-fast --test-threads 1 +RUST_BACKTRACE=1 BITCOIND_TEST=1 \ +cargo mutants -vV \ + -F "replace process_stackerdb_event" \ + -E ": replace [modify this] with [modify this] in " \ + --test-tool=nextest \ + -- \ + --run-ignored all \ + --fail-fast \ + --test-threads 1 ```