init commit

.gitignore

@@ -0,0 +1,3 @@
+.idea
+target
+

README.md

@@ -0,0 +1,13 @@
+## RocksDB state storage
+
+This is a repackage of implementation https://issues.apache.org/jira/browse/SPARK-28120, 
+RocksDB state storage, in order to use it with current spark 2.4.x release. 
+
+### How to use
+1. Use "sbt package" to build jar, copy the jar file and rocksdbjni.jar to spark library
+2. Let spark to use the it, by set "spark.sql.streaming.stateStore.providerClass" to "org.apache.spark.sql.execution.streaming.state.rocksdb.RocksDbStateStoreProvider"
+
+### License Apache License Version 2.0
+
+### Reference
+[Original Pull request] (https://github.com/apache/spark/pull/24922)

build.sbt

@@ -0,0 +1,23 @@
+name := "spark-statestore-rocksdb"
+
+
+val sparkVersion = "2.4.3"
+val scalaVersionBase = "2.12"
+val rocksDBVersion = "6.2.2"
+
+version := "2.4.3"
+
+val scalaVersion = s"$scalaVersionBase.8"
+
+javacOptions ++= Seq("-source", "1.8")
+
+scalacOptions += "-target:jvm-1.8"
+
+libraryDependencies ++= Seq(
+  "org.apache.spark" %% "spark-core" % sparkVersion,
+  "org.apache.spark" %% "spark-sql" % sparkVersion,
+  "org.rocksdb" % "rocksdbjni" % rocksDBVersion,
+  "junit" % "junit" % "4.12" % Test,
+  "org.scalatest" %% "scalatest" % "3.0.6" % Test
+)
+

project/build.properties

@@ -0,0 +1 @@
+sbt.version = 1.3.2

src/main/java/org/apache/spark/io/FileUtility.java

@@ -0,0 +1,118 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.io;
+
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
+import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
+import org.apache.commons.compress.utils.IOUtils;
+
+import java.io.BufferedInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+public class FileUtility {
+
+  /**
+   * Extract an input tar file into an output files and directories.
+   * inputTarFileLoc:  the input file location for the tar file
+   * destDirLoc: destination for the extracted files
+   *
+   * throws IllegalStateException
+   */
+  public static final String ENCODING = "utf-8";
+
+  public static void extractTarFile(String inputTarFileLoc, String destDirLoc)
+    throws IllegalStateException {
+    File inputFile = new File(inputTarFileLoc);
+    if (!inputTarFileLoc.endsWith(".tar")) {
+      throw new IllegalStateException(String.format(
+        "Input File[%s] should end with tar extension.", inputTarFileLoc));
+    }
+    File destDir = new File(destDirLoc);
+    if (destDir.exists() && !destDir.delete()) {
+      throw new IllegalStateException(String.format(
+        "Couldn't delete the existing destination directory[%s] ", destDirLoc));
+    } else if (!destDir.mkdir()) {
+      throw new IllegalStateException(String.format(
+        "Couldn't create directory  %s ", destDirLoc));
+    }
+
+    try (InputStream is = new FileInputStream(inputFile);
+         TarArchiveInputStream debInputStream = new TarArchiveInputStream(is, ENCODING)) {
+      TarArchiveEntry entry;
+      while ((entry = (TarArchiveEntry) debInputStream.getNextEntry()) != null) {
+        final File outputFile = new File(destDirLoc, entry.getName());
+        if (entry.isDirectory()) {
+          if (!outputFile.exists() && !outputFile.mkdirs()) {
+            throw new IllegalStateException(String.format(
+              "Couldn't create directory %s.", outputFile.getAbsolutePath()));
+          }
+        } else {
+          try (OutputStream outputFileStream = new FileOutputStream(outputFile)) {
+            IOUtils.copy(debInputStream, outputFileStream);
+          }
+        }
+      }
+    } catch (IOException e){
+      throw new IllegalStateException(String.format(
+        "extractTarFile failed with exception %s.", e.getMessage()));
+    }
+  }
+
+  /**
+   * create a tar file for input source directory location .
+   * source: the source directory location
+   * destFileLoc: destination of the created tarball
+   *
+   * throws IllegalStateException
+   */
+
+  public static void createTarFile(String source, String destFileLoc)
+    throws IllegalStateException {
+    File f = new File(destFileLoc);
+    if (f.exists() && !f.delete()) {
+      throw new IllegalStateException(String.format(
+        "Couldn't delete the destination file location[%s]", destFileLoc));
+    }
+    File folder = new File(source);
+    if (!folder.exists()) {
+      throw new IllegalStateException(String.format(
+        "Source folder[%s] does not exist", source));
+    }
+
+    try (FileOutputStream fos = new FileOutputStream(destFileLoc);
+         TarArchiveOutputStream tarOs = new TarArchiveOutputStream(fos, ENCODING)) {
+      File[] fileNames = folder.listFiles();
+      for (File file : fileNames) {
+        TarArchiveEntry tar_file = new TarArchiveEntry(file.getName());
+        tar_file.setSize(file.length());
+        tarOs.putArchiveEntry(tar_file);
+        try (BufferedInputStream bis = new BufferedInputStream(new FileInputStream(file))) {
+          IOUtils.copy(bis, tarOs);
+          tarOs.closeArchiveEntry();
+        }
+      }
+      tarOs.finish();
+    } catch (IOException e) {
+      throw new IllegalStateException(String.format(
+        "createTarFile failed with exception %s.", e.getMessage()));
+    }
+  }
+
+}