diff --git a/Makefile b/Makefile index d5e06e05b5ab..164adc0ad4d4 100644 --- a/Makefile +++ b/Makefile @@ -60,24 +60,20 @@ tests-code-single-ci: # TODO: How can we limit tests with ONLY_TEST with platformio? tests-code-single-local: @if ! test -n "$(TEST_TARGET)" ; then echo "***ERROR*** Set TEST_TARGET= or use make tests-code-all-local" ; return 1; fi - export PATH=./buildroot/bin/:./buildroot/tests/:${PATH} \ - && export VERBOSE_PLATFORMIO=$(VERBOSE_PLATFORMIO) \ - && exec_pio_test $(TEST_TARGET) + platformio run -t marlin_$(TEST_TARGET) .PHONY: tests-code-single-local tests-code-single-local-docker: @if ! test -n "$(TEST_TARGET)" ; then echo "***ERROR*** Set TEST_TARGET= or use make tests-code-all-local-docker" ; return 1; fi - docker-compose run --rm marlin $(MAKE) tests-code-single-local TEST_TARGET=$(TEST_TARGET) VERBOSE_PLATFORMIO=$(VERBOSE_PLATFORMIO) GIT_RESET_HARD=$(GIT_RESET_HARD) ONLY_TEST="$(ONLY_TEST)" + docker-compose run --rm marlin $(MAKE) tests-code-single-local TEST_TARGET=$(TEST_TARGET) ONLY_TEST="$(ONLY_TEST)" .PHONY: tests-code-single-local-docker tests-code-all-local: - export PATH=./buildroot/bin/:./buildroot/tests/:${PATH} \ - && export VERBOSE_PLATFORMIO=$(VERBOSE_PLATFORMIO) \ - && exec_all_pio_tests + platformio run -t test-marlin .PHONY: tests-code-all-local tests-code-all-local-docker: - docker-compose run --rm marlin $(MAKE) tests-code-all-local VERBOSE_PLATFORMIO=$(VERBOSE_PLATFORMIO) GIT_RESET_HARD=$(GIT_RESET_HARD) + docker-compose run --rm marlin $(MAKE) tests-code-all-local .PHONY: tests-code-all-local-dockerS setup-local-docker: diff --git a/README.md b/README.md index bf8028f0f9d6..0dab55336f79 100644 --- a/README.md +++ b/README.md @@ -113,8 +113,20 @@ Proposed patches should be submitted as a Pull Request against the ([bugfix-2.0. - If you make significant changes, try to run tests locally if you can - It's optional: running all the tests on Windows might take a long time, and they will run anyway on GitHub - If you're running the tests on Linux, or on WSL with the code on a Linux volume, the speed is much faster - - You can use `make tests-all-local`/`make tests-single-local TEST_TARGET=...` - - If you prefer Docker you can use `make tests-all-local-docker`/`make tests-all-local-docker TEST_TARGET=...` + - You can use: + - All: `make tests-config-all-local` + - Only one: `make tests-config-single-local TEST_TARGET=...` + - If you prefer Docker you can use: + - All: `make tests-config-all-local-docker` + - Single: `make tests-config-all-local-docker TEST_TARGET=...` + - To run all unit test suites use: + - Directly: `platformio run -t test-marlin` + - With make: `make tests-code-all-local` + - With Docker & make: `maker tests-code-all-local-docker` + - To run a single unit test suite use: + - Directly: `platformio run -t marlin_` + - With make: `make tests-code-single-local TEST_TARGET=` + - With Docker & make: `maker tests-code-single-local-docker TEST_TARGET=` - If possible, write some new tests. See [the test documentation](test) ### [RepRap.org Wiki Page](https://reprap.org/wiki/Marlin) diff --git a/buildroot/bin/exec_all_pio_tests b/buildroot/bin/exec_all_pio_tests deleted file mode 100755 index b99979d95394..000000000000 --- a/buildroot/bin/exec_all_pio_tests +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash - -shopt nullglob - -function collect() { - tests=(./test/*/) - if [[ "${#tests[@]}" -eq 0 ]]; then - printf "\033[0;31mNo tests found\033[0m\n" - exit 1 - fi - - echo "Collected ${#tests[@]} test suites..." -} - -function run() { - # We need to compile and run each test suite separately, since they will have - # different configuration - total=() - failed=() - for test in ./test/*/; do - total+=("$test") - if ! exec_pio_test "$(basename "$test")"; then - failed+=("$test") - fi - done -} - -function results() { - if [[ ${#failed[@]} -eq 0 ]]; then - printf "\033[0;32mPassed: ${#total[@]}\033[0m\n" - exit 0 - else - printf "\033[0;31mFailed: ${#failed[@]} out of ${#total[@]}\033[0m\n" - for test in "${failed[@]}"; do - printf " * $test\n" - done - exit 1 - fi -} - -collect; -run; -results; diff --git a/buildroot/bin/exec_pio_test b/buildroot/bin/exec_pio_test deleted file mode 100755 index 0ea2c88751f6..000000000000 --- a/buildroot/bin/exec_pio_test +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env bash - -set -e - -function get_configuration() { - if [[ ! -f "$1" ]] ; then - printf "\033[0;31mCouldn't find test \033[0m$1\n" >&2 - return 1 - fi - SC=START_CONFIGURATION - EC=END_CONFIGURATION - - configuration_with_markers=$(sed -n '/'$SC'/,/'$EC'/p;/^'$EC'/q' "$1") - - if ! echo "$configuration_with_markers" | head -n 1 | grep "$SC" > /dev/null ; then - printf "\033[0;31mCouldn't find \033[1;33m$SC\033[0;31m string in test \033[0m$1\n" >&2 - return 1 - fi - - if ! echo "$configuration_with_markers" | tail -n 1 | grep "$EC" > /dev/null ; then - printf "\033[0;31mCouldn't find \033[1;33m$EC\033[0;31m string in test \033[0m$1\n" >&2 - return 1 - fi - - echo "$configuration_with_markers" | tail -n +2 | head -n -1 -} - -function prepare() { - # Restore configuration - restore_configs - - # Set configuration from test - opt_set MOTHERBOARD BOARD_LINUX_RAMPS - if [[ -n "$1" ]] ; then - eval "$1" - fi -} - -function run() { - if [[ -n "$VERBOSE_PLATFORMIO" ]] ; then - verbose='-v' - else - verbose='' - fi - platformio test $verbose -e linux_native -f "$1" -} - -function cleanup() { - if [[ -n "$GIT_RESET_HARD" ]]; then - echo git reset --hard HEAD - else - restore_configs - fi -} - -configuration=$(get_configuration "test/$1/test_all.cpp") -trap cleanup EXIT -prepare "$configuration" -run "$1" diff --git a/buildroot/share/PlatformIO/scripts/collect-code-tests.py b/buildroot/share/PlatformIO/scripts/collect-code-tests.py new file mode 100644 index 000000000000..74309c2ace44 --- /dev/null +++ b/buildroot/share/PlatformIO/scripts/collect-code-tests.py @@ -0,0 +1,107 @@ +# +# collect-code-tests.py +# Convenience script to collect all code tests +# + +import os + +Import("env") +Import("projenv") + + +os.environ['PATH'] = f"./buildroot/bin/:./buildroot/tests/:{os.environ['PATH']}" + + +def collect_test_suites(): + """Get all the test suites""" + from pathlib import Path + return list(Path("./test").glob("test_*/test_all.cpp")) + + +def register_test_suites(): + """Register all the test suites""" + test_suites = collect_test_suites() + for path in test_suites: + name = path.parent.name + configuration = get_configuration(path) + env.AddCustomTarget( + name=f"marlin_{name}", + dependencies=None, + actions=[ + f"echo ====== Configuring for marlin_{name} ======", + "restore_configs", + "opt_set MOTHERBOARD BOARD_LINUX_RAMPS", + ] + configuration + [ + f"echo ====== Finished configuring for marlin_{name} ======", + f"platformio test -e linux_native -f {name}", + f"restore_configs", + ], + title="Marlin: {}".format(name.lower().title().replace("_", " ")), + description=( + f"Rin a Marlin test suite, with the appropriate configuration, " + f"that sits in {path}" + ), + ) + env.AddCustomTarget( + name="test-marlin", + dependencies=None, + actions=[ + f"platformio run -t marlin_{path.parent.name}" + for path in test_suites + ], + title="Marlin: Test all code test suites", + description=( + f"Run all Marlin code test suites ({len(test_suites)} found), each " + f"with the appropriate configuration" + ), + ) + + +def get_configuration(path): + """Get the configuration from a test suite""" + code_lines = path.read_text().splitlines() + + # Find start index + start_indexes = [ + index + for index, line in enumerate(code_lines) + if 'START_CONFIGURATION' in line + ] + if not start_indexes: + raise Exception( + f"Test suite {path.name} did not contain 'START_CONFIGURATION'") + elif len(start_indexes) > 1: + raise Exception( + f"Test suite {path.name} contained too many instances of " + f"'START_CONFIGURATION'") + start_index, = start_indexes + + # Find end index + end_indexes = [ + index + for index, line in enumerate(code_lines) + if 'END_CONFIGURATION' in line + ] + if not end_indexes: + raise Exception( + f"Test suite {path.name} did not contain 'END_CONFIGURATION'") + elif len(end_indexes) > 1: + raise Exception( + f"Test suite {path.name} contained too many instances of " + f"'END_CONFIGURATION'") + end_index, = end_indexes + + # Remove whitespace, empty lines, and commented out lines + configuration_lines = code_lines[start_index + 1:end_index] + configuration_lines = [line.strip() for line in configuration_lines] + configuration_lines = list(filter(None, configuration_lines)) + configuration_lines = [ + line + for line in configuration_lines + if not line.startswith("//") + ] + + return configuration_lines + + +register_test_suites() diff --git a/platformio.ini b/platformio.ini index 22828eb39b37..d442cb5d0de5 100644 --- a/platformio.ini +++ b/platformio.ini @@ -200,6 +200,7 @@ extra_scripts = pre:buildroot/share/PlatformIO/scripts/common-dependencies.py pre:buildroot/share/PlatformIO/scripts/common-cxxflags.py post:buildroot/share/PlatformIO/scripts/common-dependencies-post.py + post:buildroot/share/PlatformIO/scripts/collect-code-tests.py build_flags = -fmax-errors=5 -g -D__MARLIN_FIRMWARE__ -fmerge-all-constants lib_deps =