From bbe211149cf8f683798872e0e4d5843d829db16c Mon Sep 17 00:00:00 2001 From: "Boris G. Tsirkin" Date: Mon, 8 May 2023 13:59:38 +0200 Subject: [PATCH] Updated the xgboost4j-tester. --- jvm-packages/.gitignore | 1 + jvm-packages/xgboost4j-tester/generate_pom.py | 80 ++----------------- .../ml/dmlc/xgboost4j/tester/AppTest.java | 20 ----- tests/ci_build/test_jvm_cross.sh | 57 +++++++------ 4 files changed, 41 insertions(+), 117 deletions(-) delete mode 100644 jvm-packages/xgboost4j-tester/src/test/java/ml/dmlc/xgboost4j/tester/AppTest.java diff --git a/jvm-packages/.gitignore b/jvm-packages/.gitignore index 6d3f7b7cbf4b..856df78fe6af 100644 --- a/jvm-packages/.gitignore +++ b/jvm-packages/.gitignore @@ -1,2 +1,3 @@ tracker.py build.sh +xgboost4j-tester/pom.xml diff --git a/jvm-packages/xgboost4j-tester/generate_pom.py b/jvm-packages/xgboost4j-tester/generate_pom.py index 2cf4103b1b70..9126bb27220e 100644 --- a/jvm-packages/xgboost4j-tester/generate_pom.py +++ b/jvm-packages/xgboost4j-tester/generate_pom.py @@ -8,7 +8,7 @@ 4.0.0 ml.dmlc - xgboost4j-tester_${scala.binary.version} + xgboost4j-tester_{scala_binary_version} 1.0-SNAPSHOT xgboost4j-tester @@ -17,17 +17,14 @@ UTF-8 {maven_compiler_source} {maven_compiler_target} + 4.13.2 {spark_version} {scala_version} + 3.2.15 {scala_binary_version} - - com.esotericsoftware - kryo - 4.0.2 - org.scala-lang scala-compiler @@ -43,40 +40,12 @@ scala-library ${{scala.version}} - - commons-logging - commons-logging - 1.2 - - - com.typesafe.akka - akka-actor_${{scala.binary.version}} - 2.6.20 - compile - - - com.typesafe.akka - akka-testkit_${{scala.binary.version}} - 2.6.20 - test - org.scalatest scalatest_${{scala.binary.version}} - 3.0.8 - test - - - org.scalactic - scalactic_${{scala.binary.version}} - 3.2.15 + ${{scalatest.version}} test - - org.apache.commons - commons-lang3 - 3.9 - org.apache.spark spark-core_${{scala.binary.version}} @@ -98,7 +67,7 @@ junit junit - 4.13.2 + ${{junit.version}} test @@ -128,36 +97,9 @@ - - - maven-clean-plugin - 3.1.0 - - - - maven-resources-plugin - 3.0.2 - - - maven-compiler-plugin - 3.8.0 - - - maven-jar-plugin - 3.0.2 - - - maven-install-plugin - 2.5.2 - - - maven-deploy-plugin - 2.8.2 - org.apache.maven.plugins maven-assembly-plugin - 2.4 jar-with-dependencies @@ -177,22 +119,12 @@ - - - maven-site-plugin - 3.7.1 - - - maven-project-info-reports-plugin - 3.0.0 - org.apache.maven.plugins maven-surefire-plugin - 2.22.1 - ml.dmlc:xgboost4j_2.12 + ml.dmlc:xgboost4j_${{scala.binary.version}} diff --git a/jvm-packages/xgboost4j-tester/src/test/java/ml/dmlc/xgboost4j/tester/AppTest.java b/jvm-packages/xgboost4j-tester/src/test/java/ml/dmlc/xgboost4j/tester/AppTest.java deleted file mode 100644 index 2df69374806a..000000000000 --- a/jvm-packages/xgboost4j-tester/src/test/java/ml/dmlc/xgboost4j/tester/AppTest.java +++ /dev/null @@ -1,20 +0,0 @@ -package ml.dmlc.xgboost4j.tester; - -import static org.junit.Assert.assertTrue; - -import org.junit.Test; - -/** - * Unit test for simple App. - */ -public class AppTest -{ - /** - * Rigorous Test :-) - */ - @Test - public void shouldAnswerWithTrue() - { - assertTrue( true ); - } -} diff --git a/tests/ci_build/test_jvm_cross.sh b/tests/ci_build/test_jvm_cross.sh index 378846d651db..bde573135eb1 100755 --- a/tests/ci_build/test_jvm_cross.sh +++ b/tests/ci_build/test_jvm_cross.sh @@ -12,31 +12,42 @@ xgboost4j_version=$(mvn help:evaluate -Dexpression=project.version -q -DforceStd maven_compiler_source=$(mvn help:evaluate -Dexpression=maven.compiler.source -q -DforceStdout) maven_compiler_target=$(mvn help:evaluate -Dexpression=maven.compiler.target -q -DforceStdout) spark_version=$(mvn help:evaluate -Dexpression=spark.version -q -DforceStdout) -scala_version=$(mvn help:evaluate -Dexpression=scala.version -q -DforceStdout) -scala_binary_version=$(mvn help:evaluate -Dexpression=scala.binary.version -q -DforceStdout) - -# Install XGBoost4J JAR into local Maven repository -mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j/target/xgboost4j_${scala_binary_version}-${xgboost4j_version}.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j_${scala_binary_version} -Dversion=${xgboost4j_version} -Dpackaging=jar -mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j/target/xgboost4j_${scala_binary_version}-${xgboost4j_version}-tests.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j_${scala_binary_version} -Dversion=${xgboost4j_version} -Dpackaging=test-jar -Dclassifier=tests -mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j-spark/target/xgboost4j-spark_${scala_binary_version}-${xgboost4j_version}.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j-spark_${scala_binary_version} -Dversion=${xgboost4j_version} -Dpackaging=jar -mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j-example/target/xgboost4j-example_${scala_binary_version}-${xgboost4j_version}.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j-example_${scala_binary_version} -Dversion=${xgboost4j_version} -Dpackaging=jar - -cd xgboost4j-tester -# Generate pom.xml for XGBoost4J-tester, a dummy project to run XGBoost4J tests -python3 ./generate_pom.py ${xgboost4j_version} ${maven_compiler_source} ${maven_compiler_target} ${spark_version} ${scala_version} ${scala_binary_version} -# Run unit tests with XGBoost4J -mvn --no-transfer-progress package - -# Run integration tests with XGBoost4J -java -jar ./target/xgboost4j-tester_${scala_binary_version}-1.0-SNAPSHOT-jar-with-dependencies.jar - -# Run integration tests with XGBoost4J-Spark -if [ ! -z "$RUN_INTEGRATION_TEST" ] -then + +if [ ! -z "$RUN_INTEGRATION_TEST" ]; then python3 get_iris.py - spark-submit --class ml.dmlc.xgboost4j.scala.example.spark.SparkTraining --master 'local[8]' ./target/xgboost4j-tester_${scala_binary_version}-1.0-SNAPSHOT-jar-with-dependencies.jar ${PWD}/iris.csv - spark-submit --class ml.dmlc.xgboost4j.scala.example.spark.SparkMLlibPipeline --master 'local[8]' ./target/xgboost4j-tester_${scala_binary_version}-1.0-SNAPSHOT-jar-with-dependencies.jar ${PWD}/iris.csv ${PWD}/native_model ${PWD}/pipeline_model fi +# including maven profiles for different scala versions: 2.12 is the default at the moment. +for _maven_profile_string in "" "-Pdefault,scala-2.13"; do + _pwd=`pwd` + scala_version=$(mvn help:evaluate $_maven_profile_string -Dexpression=scala.version -q -DforceStdout) + scala_binary_version=$(mvn help:evaluate $_maven_profile_string -Dexpression=scala.binary.version -q -DforceStdout) + echo $scala_binary_version + echo $scala_version + + # Install XGBoost4J JAR into local Maven repository + mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j/target/xgboost4j_${scala_binary_version}-${xgboost4j_version}.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j_${scala_binary_version} -Dversion=${xgboost4j_version} -Dpackaging=jar + mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j/target/xgboost4j_${scala_binary_version}-${xgboost4j_version}-tests.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j_${scala_binary_version} -Dversion=${xgboost4j_version} -Dpackaging=test-jar -Dclassifier=tests + mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j-spark/target/xgboost4j-spark_${scala_binary_version}-${xgboost4j_version}.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j-spark_${scala_binary_version} -Dversion=${xgboost4j_version} -Dpackaging=jar + mvn --no-transfer-progress install:install-file -Dfile=./xgboost4j-example/target/xgboost4j-example_${scala_binary_version}-${xgboost4j_version}.jar -DgroupId=ml.dmlc -DartifactId=xgboost4j-example_${scala_binary_version} -Dversion=${xgboost4j_version} -Dpackaging=jar + + cd xgboost4j-tester + # Generate pom.xml for XGBoost4J-tester, a dummy project to run XGBoost4J tests + python3 ./generate_pom.py ${xgboost4j_version} ${maven_compiler_source} ${maven_compiler_target} ${spark_version} ${scala_version} ${scala_binary_version} + # Run unit tests with XGBoost4J + mvn --no-transfer-progress clean package + + # Run integration tests with XGBoost4J + java -jar ./target/xgboost4j-tester_${scala_binary_version}-1.0-SNAPSHOT-jar-with-dependencies.jar + + # Run integration tests with XGBoost4J-Spark + if [ ! -z "$RUN_INTEGRATION_TEST" ] + then + spark-submit --class ml.dmlc.xgboost4j.scala.example.spark.SparkTraining --master 'local[8]' ./target/xgboost4j-tester_${scala_binary_version}-1.0-SNAPSHOT-jar-with-dependencies.jar ${PWD}/iris.csv + spark-submit --class ml.dmlc.xgboost4j.scala.example.spark.SparkMLlibPipeline --master 'local[8]' ./target/xgboost4j-tester_${scala_binary_version}-1.0-SNAPSHOT-jar-with-dependencies.jar ${PWD}/iris.csv ${PWD}/native_model ${PWD}/pipeline_model + fi + cd $_pwd +done + set +x set +e