RDB Shredder: bump Spark to 3.0.1 (closes #101)

snowplow · Oct 30, 2020 · dcc0ed0 · dcc0ed0
1 parent 1c977c4
commit dcc0ed0
Show file tree

Hide file tree

Showing 4 changed files with 7 additions and 8 deletions.
diff --git a/build.sbt b/build.sbt
@@ -19,6 +19,7 @@ lazy val common = project.in(file("common"))
   .settings(resolvers ++= Dependencies.resolutionRepos)
   .settings(
     libraryDependencies ++= Seq(
+      Dependencies.slf4j,
       Dependencies.badrows,
       Dependencies.igluClient,
       Dependencies.scalaTracker,
@@ -102,12 +103,6 @@ lazy val shredder = project.in(file("shredder"))
       Dependencies.specs2,
       Dependencies.specs2ScalaCheck,
       Dependencies.scalaCheck
-    ),
-
-    dependencyOverrides ++= Seq(
-      Dependencies.dynamodb,
-      "com.fasterxml.jackson.core" % "jackson-core" % "2.6.7",
-      "com.fasterxml.jackson.core" % "jackson-databind" % "2.6.7.2"
     )
   )
   .dependsOn(common)
diff --git a/project/BuildSettings.scala b/project/BuildSettings.scala
@@ -106,6 +106,7 @@ object BuildSettings {
       case x if x.startsWith("META-INF") => MergeStrategy.discard
       case x if x.endsWith(".html") => MergeStrategy.discard
       case x if x.endsWith("package-info.class") => MergeStrategy.first
+      case x if x.endsWith("module-info.class") => MergeStrategy.discard
       case PathList("com", "google", "common", _) => MergeStrategy.first
       case PathList("org", "apache", "spark", "unused", _) => MergeStrategy.first
       case x =>

diff --git a/project/Dependencies.scala b/project/Dependencies.scala
@@ -29,12 +29,13 @@ object Dependencies {
     val fs2              = "1.0.4"
 
     // Scala (Shredder)
-    val spark            = "2.3.2"
+    val spark            = "3.0.1"
     val eventsManifest   = "0.2.0"
     val schemaDdl        = "0.10.0"
 
     // Java (Loader)
     val postgres         = "42.0.0"
+    val slf4j            = "1.7.30"
     val redshift         = "1.2.36.1060"
     val aws              = "1.11.319"
     val jSch             = "0.1.54"
@@ -78,6 +79,7 @@ object Dependencies {
 
   // Java (Loader)
   val postgres          = "org.postgresql"        % "postgresql"                % V.postgres
+  val slf4j             = "org.slf4j"             % "slf4j-simple"              % V.slf4j
   val redshift          = "com.amazon.redshift"   % "redshift-jdbc42-no-awssdk" % V.redshift
   val redshiftSdk       = "com.amazonaws"         % "aws-java-sdk-redshift"     % V.aws
   val s3                = "com.amazonaws"         % "aws-java-sdk-s3"           % V.aws

diff --git a/shredder/src/main/scala/com.snowplowanalytics.snowplow.storage/spark/ShredJob.scala b/shredder/src/main/scala/com.snowplowanalytics.snowplow.storage/spark/ShredJob.scala
@@ -142,7 +142,8 @@ object ShredJob extends SparkJob {
     classOf[scala.collection.immutable.Map$EmptyMap$],
     classOf[scala.collection.immutable.Set$EmptySet$],
     classOf[org.apache.spark.internal.io.FileCommitProtocol$TaskCommitMessage],
-    classOf[org.apache.spark.sql.execution.datasources.FileFormatWriter$WriteTaskResult],
+    classOf[org.apache.spark.sql.catalyst.InternalRow],
+    classOf[org.apache.spark.sql.execution.datasources.WriteTaskResult],
     classOf[org.apache.spark.sql.execution.datasources.ExecutedWriteSummary],
     classOf[org.apache.spark.sql.execution.datasources.BasicWriteTaskStats]
   )