Skip to content

Commit

Permalink
Switch pytest-xdist algorithm to worksteal (#1355)
Browse files Browse the repository at this point in the history
This PR uses the `worksteal` algorithm to accelerate parallel pytests. See rapidsai/cudf#15207.

Authors:
  - Bradley Dice (https://github.com/bdice)

Approvers:
  - GALI PREM SAGAR (https://github.com/galipremsagar)
  - Jake Awe (https://github.com/AyodeAwe)

URL: #1355
  • Loading branch information
bdice authored Mar 6, 2024
1 parent baa1bbb commit b08f4f7
Show file tree
Hide file tree
Showing 3 changed files with 23 additions and 7 deletions.
6 changes: 3 additions & 3 deletions ci/test_python.sh
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#!/bin/bash
# Copyright (c) 2022-2023, NVIDIA CORPORATION.
# Copyright (c) 2022-2024, NVIDIA CORPORATION.

set -euo pipefail

Expand Down Expand Up @@ -50,7 +50,7 @@ pytest \
--cache-clear \
--junitxml="${RAPIDS_TESTS_DIR}/junit-cuspatial.xml" \
--numprocesses=8 \
--dist=loadscope \
--dist=worksteal \
--cov-config=../.coveragerc \
--cov=cuspatial \
--cov-report=xml:"${RAPIDS_COVERAGE_DIR}/cuspatial-coverage.xml" \
Expand All @@ -65,7 +65,7 @@ pytest \
--cache-clear \
--junitxml="${RAPIDS_TESTS_DIR}/junit-cuproj.xml" \
--numprocesses=8 \
--dist=loadscope \
--dist=worksteal \
--cov-config=../.coveragerc \
--cov=cuproj \
--cov-report=xml:"${RAPIDS_COVERAGE_DIR}/cuproj-coverage.xml" \
Expand Down
12 changes: 10 additions & 2 deletions ci/test_wheel_cuproj.sh
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#!/bin/bash
# Copyright (c) 2023, NVIDIA CORPORATION.
# Copyright (c) 2023-2024, NVIDIA CORPORATION.

set -eou pipefail

Expand All @@ -22,5 +22,13 @@ python -m pip install $(echo ./dist/cuproj*.whl)[test]
if [[ "$(arch)" == "aarch64" && ${RAPIDS_BUILD_TYPE} == "pull-request" ]]; then
python ./ci/wheel_smoke_test_cuproj.py
else
python -m pytest -n 8 ./python/cuproj/cuproj/tests
rapids-logger "pytest cuproj"
pushd python/cuproj/cuproj
python -m pytest \
--cache-clear \
--junitxml="${RAPIDS_TESTS_DIR}/junit-cuproj.xml" \
--numprocesses=8 \
--dist=worksteal \
tests
popd
fi
12 changes: 10 additions & 2 deletions ci/test_wheel_cuspatial.sh
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#!/bin/bash
# Copyright (c) 2023, NVIDIA CORPORATION.
# Copyright (c) 2023-2024, NVIDIA CORPORATION.

set -eou pipefail

Expand All @@ -18,5 +18,13 @@ python -m pip install $(echo ./dist/cuspatial*.whl)[test]
if [[ "$(arch)" == "aarch64" && ${RAPIDS_BUILD_TYPE} == "pull-request" ]]; then
python ./ci/wheel_smoke_test_cuspatial.py
else
python -m pytest -n 8 ./python/cuspatial/cuspatial/tests
rapids-logger "pytest cuspatial"
pushd python/cuspatial/cuspatial
python -m pytest \
--cache-clear \
--junitxml="${RAPIDS_TESTS_DIR}/junit-cuspatial.xml" \
--numprocesses=8 \
--dist=worksteal \
tests
popd
fi

0 comments on commit b08f4f7

Please sign in to comment.