[SPARK-48137][INFRA] Run yarn test only in PR builders and Daily CIs

apache · May 6, 2024 · 92fd00c · 92fd00c
1 parent 8094535
commit 92fd00c
Show file tree

Hide file tree

Showing 4 changed files with 15 additions and 3 deletions.
diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
@@ -80,12 +80,14 @@ jobs:
           pyspark=`./dev/is-changed.py -m $pyspark_modules`
           if [[ "${{ github.repository }}" != 'apache/spark' ]]; then
             pandas=$pyspark
+            yarn=`./dev/is-changed.py -m yarn`
             kubernetes=`./dev/is-changed.py -m kubernetes`
             sparkr=`./dev/is-changed.py -m sparkr`
             buf=true
             ui=true
           else
             pandas=false
+            yarn=false
             kubernetes=false
             sparkr=false
             buf=false
@@ -156,7 +158,9 @@ jobs:
             streaming, sql-kafka-0-10, streaming-kafka-0-10, streaming-kinesis-asl,
             kubernetes, hadoop-cloud, spark-ganglia-lgpl, protobuf
           - >-
-            yarn, connect
+            yarn
+          - >-
+            connect
         # Here, we split Hive and SQL tests into some of slow ones and the rest of them.
         included-tags: [""]
         excluded-tags: [""]
@@ -194,6 +198,11 @@ jobs:
             hive: hive2.3
             excluded-tags: org.apache.spark.tags.ExtendedSQLTest,org.apache.spark.tags.SlowSQLTest
             comment: "- other tests"
+        exclude:
+          # Always run if yarn == 'true', even infra-image is skip (such as non-master job)
+          # In practice, the build will run in individual PR, but not against the individual commit
+          # in Apache Spark repository.
+          - modules: ${{ fromJson(needs.precondition.outputs.required).yarn != 'true' && 'yarn' }}
     env:
       MODULES_TO_TEST: ${{ matrix.modules }}
       EXCLUDED_TAGS: ${{ matrix.excluded-tags }}

diff --git a/.github/workflows/build_java21.yml b/.github/workflows/build_java21.yml
@@ -47,6 +47,7 @@ jobs:
           "sparkr": "true",
           "tpcds-1g": "true",
           "docker-integration-tests": "true",
+          "yarn": "true",
           "k8s-integration-tests": "true",
           "buf": "true",
           "ui": "true"

diff --git a/.github/workflows/build_non_ansi.yml b/.github/workflows/build_non_ansi.yml
@@ -44,5 +44,6 @@ jobs:
           "pyspark": "true",
           "sparkr": "true",
           "tpcds-1g": "true",
-          "docker-integration-tests": "true"
+          "docker-integration-tests": "true",
+          "yarn": "true"
         }
diff --git a/.github/workflows/build_rockdb_as_ui_backend.yml b/.github/workflows/build_rockdb_as_ui_backend.yml
@@ -42,5 +42,6 @@ jobs:
         {
           "build": "true",
           "pyspark": "true",
-          "sparkr": "true"
+          "sparkr": "true",
+          "yarn": "true"
         }