Scala 2.13 support.

dmlc · May 2, 2023 · 3cbd533 · 3cbd533
1 parent 08ce495
commit 3cbd533
Show file tree

Hide file tree

Showing 15 changed files with 113 additions and 42 deletions.
diff --git a/.github/workflows/jvm_tests.yml b/.github/workflows/jvm_tests.yml
@@ -75,3 +75,13 @@ jobs:
       if: matrix.os == 'ubuntu-latest'  # Distributed training doesn't work on Windows
       env:
         RABIT_MOCK: ON
+
+
+    - name: Build and Test XGBoost4J with scala 2.13
+      run: |
+        rm -rfv build/
+        cd jvm-packages
+        mvn -B clean install test -Pdefault,scala-2.13
+      if: matrix.os == 'ubuntu-latest'  # Distributed training doesn't work on Windows
+      env:
+        RABIT_MOCK: ON
diff --git a/jvm-packages/README.md b/jvm-packages/README.md
@@ -36,6 +36,19 @@ XGBoost4J, XGBoost4J-Spark, etc. in maven repository is compiled with g++-4.8.5.
     <version>latest_version_num</version>
 </dependency>
 ```
+or 
+```
+<dependency>
+    <groupId>ml.dmlc</groupId>
+    <artifactId>xgboost4j_2.13</artifactId>
+    <version>latest_version_num</version>
+</dependency>
+<dependency>
+    <groupId>ml.dmlc</groupId>
+    <artifactId>xgboost4j-spark_2.13</artifactId>
+    <version>latest_version_num</version>
+</dependency>
+```
 
 <b>sbt</b>
 ```sbt
@@ -47,7 +60,6 @@ libraryDependencies ++= Seq(
 
 For the latest release version number, please check [here](https://github.com/dmlc/xgboost/releases).
 
-To enable the GPU algorithm (`tree_method='gpu_hist'`), use artifacts `xgboost4j-gpu_2.12` and `xgboost4j-spark-gpu_2.12` instead.
 
 ### Access SNAPSHOT version
 
@@ -85,6 +97,19 @@ Then add XGBoost4J as a dependency:
     <version>latest_version_num-SNAPSHOT</version>
 </dependency>
 ```
+or with scala 2.13 
+```
+<dependency>
+    <groupId>ml.dmlc</groupId>
+    <artifactId>xgboost4j_2.13</artifactId>
+    <version>latest_version_num-SNAPSHOT</version>
+</dependency>
+<dependency>
+    <groupId>ml.dmlc</groupId>
+    <artifactId>xgboost4j-spark_2.13</artifactId>
+    <version>latest_version_num-SNAPSHOT</version>
+</dependency>
+```
 
 <b>sbt</b>
 ```sbt
@@ -96,7 +121,9 @@ libraryDependencies ++= Seq(
 
 For the latest release version number, please check [the repository listing](https://s3-us-west-2.amazonaws.com/xgboost-maven-repo/list.html).
 
+### GPU algorithm
 To enable the GPU algorithm (`tree_method='gpu_hist'`), use artifacts `xgboost4j-gpu_2.12` and `xgboost4j-spark-gpu_2.12` instead.
+Note that scala 2.13 is not supported by the [NVIDIA/spark-rapids#1525](https://github.com/NVIDIA/spark-rapids/issues/1525) yet, so the GPU algorithm can only be used with scala 2.12.
 
 ## Examples
 

diff --git a/jvm-packages/pom.xml b/jvm-packages/pom.xml
@@ -5,7 +5,7 @@
     <modelVersion>4.0.0</modelVersion>
 
     <groupId>ml.dmlc</groupId>
-    <artifactId>xgboost-jvm_2.12</artifactId>
+    <artifactId>xgboost-jvm</artifactId>
     <version>2.0.0-SNAPSHOT</version>
     <packaging>pom</packaging>
     <name>XGBoost JVM Package</name>
@@ -34,6 +34,7 @@
         <maven.compiler.source>1.8</maven.compiler.source>
         <maven.compiler.target>1.8</maven.compiler.target>
         <flink.version>1.17.0</flink.version>
+        <junit.version>4.13.2</junit.version>
         <spark.version>3.4.0</spark.version>
         <scala.version>2.12.17</scala.version>
         <scala.binary.version>2.12</scala.binary.version>
@@ -44,7 +45,9 @@
         <cudf.version>23.04.0</cudf.version>
         <spark.rapids.version>23.04.0</spark.rapids.version>
         <cudf.classifier>cuda11</cudf.classifier>
-    </properties>
+        <scalatest.version>3.2.15</scalatest.version>
+        <scala-collection-compat.version>2.9.0</scala-collection-compat.version>
+      </properties>
     <repositories>
         <repository>
             <id>central_maven</id>
@@ -70,6 +73,14 @@
             </modules>
         </profile>
 
+        <profile>
+            <id>scala-2.13</id>
+            <properties>
+                <scala.binary.version>2.13</scala.binary.version>
+                <scala.version>2.13.10</scala.version>
+            </properties>
+        </profile>
+
         <!-- gpu profile with both cpu and gpu test suites -->
         <profile>
             <id>gpu</id>
@@ -466,6 +477,7 @@
         </plugins>
     </reporting>
     <dependencies>
+
         <dependency>
             <groupId>com.esotericsoftware</groupId>
             <artifactId>kryo</artifactId>
@@ -482,6 +494,11 @@
             <artifactId>scala-library</artifactId>
             <version>${scala.version}</version>
         </dependency>
+        <dependency>
+          <groupId>org.scala-lang.modules</groupId>
+          <artifactId>scala-collection-compat_${scala.binary.version}</artifactId>
+          <version>${scala-collection-compat.version}</version>
+        </dependency>
         <dependency>
             <groupId>commons-logging</groupId>
             <artifactId>commons-logging</artifactId>
@@ -490,13 +507,13 @@
         <dependency>
             <groupId>org.scalatest</groupId>
             <artifactId>scalatest_${scala.binary.version}</artifactId>
-            <version>3.2.15</version>
+            <version>${scalatest.version}</version>
             <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>org.scalactic</groupId>
             <artifactId>scalactic_${scala.binary.version}</artifactId>
-            <version>3.2.15</version>
+            <version>${scalatest.version}</version>
             <scope>test</scope>
         </dependency>
     </dependencies>

diff --git a/jvm-packages/xgboost4j-example/pom.xml b/jvm-packages/xgboost4j-example/pom.xml
@@ -5,10 +5,11 @@
     <modelVersion>4.0.0</modelVersion>
     <parent>
         <groupId>ml.dmlc</groupId>
-        <artifactId>xgboost-jvm_2.12</artifactId>
+        <artifactId>xgboost-jvm</artifactId>
         <version>2.0.0-SNAPSHOT</version>
     </parent>
-    <artifactId>xgboost4j-example_2.12</artifactId>
+    <name>xgboost4j-example</name>
+    <artifactId>xgboost4j-example_${scala.binary.version}</artifactId>
     <version>2.0.0-SNAPSHOT</version>
     <packaging>jar</packaging>
     <build>

diff --git a/...t4j-example/src/main/scala/ml/dmlc/xgboost4j/scala/example/flink/DistTrainWithFlink.scala b/...t4j-example/src/main/scala/ml/dmlc/xgboost4j/scala/example/flink/DistTrainWithFlink.scala
@@ -73,12 +73,13 @@ object DistTrainWithFlink {
           .map(_.f1.f0)
           .returns(testDataTypeHint)
 
-    val paramMap = mapAsJavaMap(Map(
-      ("eta", "0.1".asInstanceOf[AnyRef]),
-      ("max_depth", "2"),
-      ("objective", "binary:logistic"),
-      ("verbosity", "1")
-    ))
+    val paramMap = Map(
+        ("eta", "0.1".asInstanceOf[AnyRef]),
+        ("max_depth", "2"),
+        ("objective", "binary:logistic"),
+        ("verbosity", "1")
+      )
+      .asJava
 
     // number of iterations
     val round = 2

diff --git a/jvm-packages/xgboost4j-flink/pom.xml b/jvm-packages/xgboost4j-flink/pom.xml
@@ -5,9 +5,11 @@
     <modelVersion>4.0.0</modelVersion>
     <parent>
         <groupId>ml.dmlc</groupId>
-        <artifactId>xgboost-jvm_2.12</artifactId>
+        <artifactId>xgboost-jvm</artifactId>
         <version>2.0.0-SNAPSHOT</version>
     </parent>
+
+    <name>xgboost4j-flink</name>
     <artifactId>xgboost4j-flink_${scala.binary.version}</artifactId>
     <version>2.0.0-SNAPSHOT</version>
     <properties>

diff --git a/jvm-packages/xgboost4j-gpu/pom.xml b/jvm-packages/xgboost4j-gpu/pom.xml
@@ -5,10 +5,11 @@
     <modelVersion>4.0.0</modelVersion>
     <parent>
         <groupId>ml.dmlc</groupId>
-        <artifactId>xgboost-jvm_2.12</artifactId>
+        <artifactId>xgboost-jvm</artifactId>
         <version>2.0.0-SNAPSHOT</version>
     </parent>
-    <artifactId>xgboost4j-gpu_2.12</artifactId>
+    <artifactId>xgboost4j-gpu_${scala.binary.version}</artifactId>
+    <name>xgboost4j-gpu</name>
     <version>2.0.0-SNAPSHOT</version>
     <packaging>jar</packaging>
 
@@ -35,13 +36,13 @@
         <dependency>
             <groupId>junit</groupId>
             <artifactId>junit</artifactId>
-            <version>4.13.2</version>
+            <version>${junit.version}</version>
             <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>org.scalatest</groupId>
             <artifactId>scalatest_${scala.binary.version}</artifactId>
-            <version>3.2.15</version>
+            <version>${scalatest.version}</version>
             <scope>provided</scope>
         </dependency>
         <dependency>

diff --git a/jvm-packages/xgboost4j-spark-gpu/pom.xml b/jvm-packages/xgboost4j-spark-gpu/pom.xml
@@ -5,10 +5,11 @@
     <modelVersion>4.0.0</modelVersion>
     <parent>
         <groupId>ml.dmlc</groupId>
-        <artifactId>xgboost-jvm_2.12</artifactId>
+        <artifactId>xgboost-jvm</artifactId>
         <version>2.0.0-SNAPSHOT</version>
     </parent>
-    <artifactId>xgboost4j-spark-gpu_2.12</artifactId>
+    <name>xgboost4j-spark-gpu</name>
+    <artifactId>xgboost4j-spark-gpu_${scala.binary.version}</artifactId>
     <build>
         <plugins>
             <plugin>
@@ -24,7 +25,7 @@
         <dependency>
             <groupId>ml.dmlc</groupId>
             <artifactId>xgboost4j-gpu_${scala.binary.version}</artifactId>
-            <version>2.0.0-SNAPSHOT</version>
+            <version>${project.version}</version>
         </dependency>
         <dependency>
             <groupId>org.apache.spark</groupId>

diff --git a/jvm-packages/xgboost4j-spark/pom.xml b/jvm-packages/xgboost4j-spark/pom.xml
@@ -5,10 +5,11 @@
     <modelVersion>4.0.0</modelVersion>
     <parent>
         <groupId>ml.dmlc</groupId>
-        <artifactId>xgboost-jvm_2.12</artifactId>
+        <artifactId>xgboost-jvm</artifactId>
         <version>2.0.0-SNAPSHOT</version>
     </parent>
-    <artifactId>xgboost4j-spark_2.12</artifactId>
+    <name>xgboost4j-spark</name>
+    <artifactId>xgboost4j-spark_${scala.binary.version}</artifactId>
     <build>
         <plugins>
             <plugin>
@@ -24,7 +25,7 @@
         <dependency>
             <groupId>ml.dmlc</groupId>
             <artifactId>xgboost4j_${scala.binary.version}</artifactId>
-            <version>2.0.0-SNAPSHOT</version>
+            <version>${project.version}</version>
         </dependency>
         <dependency>
             <groupId>org.apache.spark</groupId>

diff --git a/jvm-packages/xgboost4j-tester/generate_pom.py b/jvm-packages/xgboost4j-tester/generate_pom.py
@@ -8,10 +8,10 @@
   <modelVersion>4.0.0</modelVersion>
 
   <groupId>ml.dmlc</groupId>
-  <artifactId>xgboost4j-tester_2.12</artifactId>
+  <artifactId>xgboost4j-tester_${scala.binary.version}</artifactId>
   <version>1.0-SNAPSHOT</version>
 
-  <name>xgboost4j-tester_2.12</name>
+  <name>xgboost4j-tester</name>
 
   <properties>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>

diff --git a/jvm-packages/xgboost4j/pom.xml b/jvm-packages/xgboost4j/pom.xml
@@ -5,10 +5,11 @@
     <modelVersion>4.0.0</modelVersion>
     <parent>
         <groupId>ml.dmlc</groupId>
-        <artifactId>xgboost-jvm_2.12</artifactId>
+        <artifactId>xgboost-jvm</artifactId>
         <version>2.0.0-SNAPSHOT</version>
     </parent>
-    <artifactId>xgboost4j_2.12</artifactId>
+    <name>xgboost4j</name>
+    <artifactId>xgboost4j_${scala.binary.version}</artifactId>
     <version>2.0.0-SNAPSHOT</version>
     <packaging>jar</packaging>
 
@@ -28,13 +29,13 @@
         <dependency>
             <groupId>junit</groupId>
             <artifactId>junit</artifactId>
-            <version>4.13.2</version>
+            <version>${junit.version}</version>
             <scope>test</scope>
         </dependency>
         <dependency>
           <groupId>org.scalatest</groupId>
           <artifactId>scalatest_${scala.binary.version}</artifactId>
-          <version>3.2.15</version>
+          <version>${scalatest.version}</version>
           <scope>provided</scope>
         </dependency>
     </dependencies>

diff --git a/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/EvalTrait.scala b/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/EvalTrait.scala
@@ -37,7 +37,7 @@ trait EvalTrait extends IEvaluation {
    */
   def eval(predicts: Array[Array[Float]], dmat: DMatrix): Float
 
-  private[scala] def eval(predicts: Array[Array[Float]], jdmat: java.DMatrix): Float = {
+  def eval(predicts: Array[Array[Float]], jdmat: java.DMatrix): Float = {
     require(predicts.length == jdmat.getLabel.length, "predicts size and label size must match " +
       s" predicts size: ${predicts.length}, label size: ${jdmat.getLabel.length}")
     eval(predicts, new DMatrix(jdmat))

diff --git a/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/ObjectiveTrait.scala b/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/ObjectiveTrait.scala
@@ -31,7 +31,7 @@ trait ObjectiveTrait extends IObjective {
    */
   def getGradient(predicts: Array[Array[Float]], dtrain: DMatrix): List[Array[Float]]
 
-  private[scala] def getGradient(predicts: Array[Array[Float]], dtrain: JDMatrix):
+  def getGradient(predicts: Array[Array[Float]], dtrain: JDMatrix):
     java.util.List[Array[Float]] = {
     getGradient(predicts, new DMatrix(dtrain)).asJava
   }

diff --git a/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/XGBoost.scala b/jvm-packages/xgboost4j/src/main/scala/ml/dmlc/xgboost4j/scala/XGBoost.scala
@@ -17,12 +17,11 @@
 package ml.dmlc.xgboost4j.scala
 
 import java.io.InputStream
+import ml.dmlc.xgboost4j.java.{XGBoostError, XGBoost => JXGBoost}
 
-import ml.dmlc.xgboost4j.java.{XGBoostError, Booster => JBooster, XGBoost => JXGBoost}
-import scala.collection.JavaConverters._
-
+import scala.jdk.CollectionConverters._
 import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.fs.{FileSystem, Path}
+import org.apache.hadoop.fs.Path
 
 /**
   * XGBoost Scala Training function.
@@ -40,7 +39,12 @@ object XGBoost {
       earlyStoppingRound: Int = 0,
       prevBooster: Booster,
       checkpointParams: Option[ExternalCheckpointParams]): Booster = {
-    val jWatches = watches.mapValues(_.jDMatrix).asJava
+
+    // we have to filter null value for customized obj and eval
+    val jParams: java.util.Map[String, AnyRef] =
+      params.filter(_._2 != null).mapValues(_.toString.asInstanceOf[AnyRef]).toMap.asJava
+
+    val jWatches = watches.mapValues(_.jDMatrix).toMap.asJava
     val jBooster = if (prevBooster == null) {
       null
     } else {
@@ -51,8 +55,7 @@ object XGBoost {
       map(cp => {
           JXGBoost.trainAndSaveCheckpoint(
             dtrain.jDMatrix,
-            // we have to filter null value for customized obj and eval
-            params.filter(_._2 != null).mapValues(_.toString.asInstanceOf[AnyRef]).asJava,
+            jParams,
             numRounds, jWatches, metrics, obj, eval, earlyStoppingRound, jBooster,
             cp.checkpointInterval,
             cp.checkpointPath,
@@ -61,8 +64,7 @@ object XGBoost {
       getOrElse(
         JXGBoost.train(
           dtrain.jDMatrix,
-          // we have to filter null value for customized obj and eval
-          params.filter(_._2 != null).mapValues(_.toString.asInstanceOf[AnyRef]).asJava,
+          jParams,
           numRounds, jWatches, metrics, obj, eval, earlyStoppingRound, jBooster)
       )
     if (prevBooster == null) {