Fix tests after GHA runner Java11 migration (#32190)

* Fix tests after GHA runner Java11 migration * Still run HCatalogIO tests on Java8 * Fix Python Flink validates runner test java container * Fix PostCommit Java Examples Dataflow Java on Java8 * Fix HCatalog command
apache · Aug 15, 2024 · d7d9f51 · d7d9f51
1 parent 028e0ee
commit d7d9f51
Show file tree

Hide file tree

Showing 7 changed files with 61 additions and 8 deletions.
diff --git a/.github/trigger_files/beam_PostCommit_Java_Examples_Dataflow_Java.json b/.github/trigger_files/beam_PostCommit_Java_Examples_Dataflow_Java.json
diff --git a/.github/workflows/beam_PostCommit_Java_Hadoop_Versions.yml b/.github/workflows/beam_PostCommit_Java_Hadoop_Versions.yml
@@ -73,11 +73,21 @@ jobs:
       - name: Setup environment
         uses: ./.github/actions/setup-environment-action
         with:
-          java-version: default
-      - name: run validatesRunner script
+          java-version: |
+            8
+            11
+      - name: run javaHadoopVersionsTest script
         uses: ./.github/actions/gradle-command-self-hosted-action
         with:
           gradle-command: :javaHadoopVersionsTest
+      # TODO(https://github.com/apache/beam/issues/32189) remove when embedded hive supports Java11
+      - name: run java8HadoopVersionsTest script
+        uses: ./.github/actions/gradle-command-self-hosted-action
+        with:
+          gradle-command: :sdks:java:io:hcatalog:hadoopVersionsTest
+          arguments: |
+            -PtestJavaVersion=8 \
+            -Pjava8Home=$JAVA_HOME_8_X64 \
       - name: Archive JUnit Test Results
         uses: actions/upload-artifact@v4
         if: ${{ !success() }}

diff --git a/.github/workflows/beam_PreCommit_Java_HCatalog_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_HCatalog_IO_Direct.yml
@@ -87,13 +87,28 @@ jobs:
           github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
       - name: Setup environment
         uses: ./.github/actions/setup-environment-action
+        with:
+          java-version: |
+            8
+            11
       - name: run HCatalog IO build script
         uses: ./.github/actions/gradle-command-self-hosted-action
         with:
           gradle-command: :sdks:java:io:hcatalog:build
           arguments: |
             -PdisableSpotlessCheck=true \
             -PdisableCheckStyle=true \
+      # TODO(https://github.com/apache/beam/issues/32189) remove when embedded hive supports Java11
+      - name: Test HCatalog IO on Java8
+        uses: ./.github/actions/gradle-command-self-hosted-action
+        with:
+          gradle-command: :sdks:java:io:hcatalog:test
+          arguments: |
+            -PdisableSpotlessCheck=true \
+            -PdisableCheckStyle=true \
+            -Dfile.encoding=UTF-8 \
+            -PtestJavaVersion=8 \
+            -Pjava17Home=$JAVA_HOME_8_X64 \
       - name: Archive JUnit Test Results
         uses: actions/upload-artifact@v4
         if: ${{ !success() }}

diff --git a/.github/workflows/beam_PreCommit_Java_IOs_Direct.yml b/.github/workflows/beam_PreCommit_Java_IOs_Direct.yml
@@ -86,6 +86,9 @@ jobs:
           github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
       - name: Setup environment
         uses: ./.github/actions/setup-environment-action
+        java-version: |
+          8
+          11
       - name: run Java IOs PreCommit script
         uses: ./.github/actions/gradle-command-self-hosted-action
         with:
@@ -94,6 +97,17 @@ jobs:
             -PdisableSpotlessCheck=true \
             -PdisableCheckStyle=true \
             -Dfile.encoding=UTF-8 \
+      # TODO(https://github.com/apache/beam/issues/32189) remove when embedded hive supports Java11
+      - name: run Java8 IOs PreCommit script
+        uses: ./.github/actions/gradle-command-self-hosted-action
+        with:
+          gradle-command: :sdks:java:io:hcatalog:build
+          arguments: |
+            -PdisableSpotlessCheck=true \
+            -PdisableCheckStyle=true \
+            -Dfile.encoding=UTF-8 \
+            -PtestJavaVersion=8 \
+            -Pjava17Home=$JAVA_HOME_8_X64 \
       - name: Archive JUnit Test Results
         uses: actions/upload-artifact@v4
         if: ${{ !success() }}

diff --git a/runners/google-cloud-dataflow-java/examples/build.gradle b/runners/google-cloud-dataflow-java/examples/build.gradle
@@ -147,15 +147,15 @@ task javaPostCommit() {
     dependsOn postCommitLegacyWorker
 }
 
-task postCommitLegacyWorkerJava11(type: Test) {
+task postCommitLegacyWorkerJava8(type: Test) {
   dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar"
   def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath
-  systemProperty "java.specification.version", "11"
+  systemProperty "java.specification.version", "8"
   with commonConfig(dataflowWorkerJar: dataflowWorkerJar, runWordCount: 'only')
 }
 
-task java11PostCommit() {
-  dependsOn postCommitLegacyWorkerJava11
+task java8PostCommit() {
+  dependsOn postCommitLegacyWorkerJava8
 }
 
 task postCommitLegacyWorkerJava17(type: Test) {

diff --git a/sdks/java/io/hcatalog/build.gradle b/sdks/java/io/hcatalog/build.gradle
@@ -90,9 +90,20 @@ task hadoopVersionsTest(group: "Verification") {
 }
 
 hadoopVersions.each { kv ->
-  task "hadoopVersion${kv.key}Test"(type: Test, group: "Verification") {
+  tasks.create(name: "hadoopVersion${kv.key}Test", type: Test, group: "Verification") {
     description = "Runs HCatalog tests with Hadoop version $kv.value"
     classpath = configurations."hadoopVersion$kv.key" + sourceSets.test.runtimeClasspath
     include '**/*Test.class'
   }
 }
+
+project.tasks.withType(Test).configureEach {
+  if (JavaVersion.VERSION_1_8.compareTo(JavaVersion.current()) < 0 && project.findProperty('testJavaVersion') != '8') {
+    useJUnit {
+      filter {
+        excludeTestsMatching "org.apache.beam.sdk.io.hcatalog.HCatalogIOTest"
+        excludeTestsMatching "org.apache.beam.sdk.io.hcatalog.HCatalogBeamSchemaTest"
+      }
+    }
+  }
+}
diff --git a/sdks/python/test-suites/portable/common.gradle b/sdks/python/test-suites/portable/common.gradle
@@ -1,3 +1,5 @@
+import static org.apache.beam.gradle.BeamModulePlugin.getSupportedJavaVersion
+
 import org.apache.tools.ant.taskdefs.condition.Os
 
 /*
@@ -39,7 +41,8 @@ def createFlinkRunnerTestTask(String workerType) {
   task.configure {
     dependsOn ":runners:flink:${latestFlinkVersion}:job-server:shadowJar"
     // The Java SDK worker is required to execute external transforms.
-    dependsOn ':sdks:java:container:java8:docker'
+    def suffix = getSupportedJavaVersion()
+    dependsOn ":sdks:java:container:${suffix}:docker"
     if (workerType == 'DOCKER') {
       dependsOn pythonContainerTask
     } else if (workerType == 'PROCESS') {