diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 7107af66129e8..02b3814a018b6 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -205,6 +205,7 @@ jobs: HIVE_PROFILE: ${{ matrix.hive }} GITHUB_PREV_SHA: ${{ github.event.before }} SPARK_LOCAL_IP: localhost + SKIP_PACKAGING: true steps: - name: Checkout Spark repository uses: actions/checkout@v3 @@ -344,6 +345,8 @@ jobs: java: - ${{ inputs.java }} modules: + - >- + pyspark-errors - >- pyspark-sql, pyspark-mllib, pyspark-resource, pyspark-testing - >- @@ -353,7 +356,7 @@ jobs: - >- pyspark-pandas-slow - >- - pyspark-connect, pyspark-errors + pyspark-connect - >- pyspark-pandas-connect - >- @@ -366,6 +369,7 @@ jobs: SPARK_LOCAL_IP: localhost SKIP_UNIDOC: true SKIP_MIMA: true + SKIP_PACKAGING: true METASPACE_SIZE: 1g steps: - name: Checkout Spark repository @@ -414,14 +418,20 @@ jobs: python3.9 -m pip list pypy3 -m pip list - name: Install Conda for pip packaging test + if: ${{ matrix.modules == 'pyspark-errors' }} run: | curl -s https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh > miniconda.sh bash miniconda.sh -b -p $HOME/miniconda # Run the tests. - name: Run tests env: ${{ fromJSON(inputs.envs) }} + shell: 'script -q -e -c "bash {0}"' run: | - export PATH=$PATH:$HOME/miniconda/bin + if [[ "$MODULES_TO_TEST" == "pyspark-errors" ]]; then + export PATH=$PATH:$HOME/miniconda/bin + export SKIP_PACKAGING=false + echo "Python Packaging Tests Enabled!" + fi ./dev/run-tests --parallelism 1 --modules "$MODULES_TO_TEST" - name: Upload coverage to Codecov if: fromJSON(inputs.envs).PYSPARK_CODECOV == 'true' @@ -457,6 +467,7 @@ jobs: GITHUB_PREV_SHA: ${{ github.event.before }} SPARK_LOCAL_IP: localhost SKIP_MIMA: true + SKIP_PACKAGING: true steps: - name: Checkout Spark repository uses: actions/checkout@v3 @@ -911,6 +922,7 @@ jobs: SPARK_LOCAL_IP: localhost ORACLE_DOCKER_IMAGE_NAME: gvenzl/oracle-xe:21.3.0 SKIP_MIMA: true + SKIP_PACKAGING: true steps: - name: Checkout Spark repository uses: actions/checkout@v3 diff --git a/dev/run-tests.py b/dev/run-tests.py index c0c281b549e36..9bf3095edb71f 100755 --- a/dev/run-tests.py +++ b/dev/run-tests.py @@ -395,7 +395,7 @@ def run_python_tests(test_modules, parallelism, with_coverage=False): def run_python_packaging_tests(): - if not os.environ.get("SPARK_JENKINS"): + if not os.environ.get("SPARK_JENKINS") and os.environ.get("SKIP_PACKAGING", "false") != "true": set_title_and_block("Running PySpark packaging tests", "BLOCK_PYSPARK_PIP_TESTS") command = [os.path.join(SPARK_HOME, "dev", "run-pip-tests")] run_cmd(command)