[Java] Package native dependencies into jar (#4367)

.gitignore

@@ -148,6 +148,7 @@ java/**/lib
 java/**/.settings
 java/**/.classpath
 java/**/.project
+java/runtime/native_dependencies/
 
 # python virtual env
 venv

CMakeLists.txt

@@ -152,8 +152,23 @@ if ("${CMAKE_RAY_LANG_JAVA}" STREQUAL "YES")
   get_raylet_library("java" RAYLET_LIBRARY_JAVA)
   add_dependencies(copy_ray ${RAYLET_LIBRARY_JAVA})
 
-  # copy libplasma_java files
+  # Copy java native dependencies.
   add_custom_command(TARGET copy_ray POST_BUILD
-    COMMAND bash -c "mkdir -p ${CMAKE_CURRENT_BINARY_DIR}/src/plasma"
-    COMMAND bash -c "cp ${ARROW_LIBRARY_DIR}/libplasma_java.* ${CMAKE_CURRENT_BINARY_DIR}/src/plasma/")
+    COMMAND mkdir -p ${CMAKE_SOURCE_DIR}/java/runtime/native_dependencies)
+  set(java_native_dependencies
+    "src/ray/thirdparty/redis/src/redis-server"
+    "src/ray/gcs/redis_module/libray_redis_module.so"
+    "src/ray/raylet/raylet"
+    "src/ray/raylet/libraylet_library_java.*")
+  foreach(file ${java_native_dependencies})
+  add_custom_command(TARGET copy_ray POST_BUILD
+    COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/${file}
+      ${CMAKE_SOURCE_DIR}/java/runtime/native_dependencies)
+  endforeach()
+add_custom_command(TARGET copy_ray POST_BUILD
+  COMMAND ${CMAKE_COMMAND} -E copy ${ARROW_HOME}/bin/plasma_store_server
+    ${CMAKE_SOURCE_DIR}/java/runtime/native_dependencies)
+  add_custom_command(TARGET copy_ray POST_BUILD
+    COMMAND $(CMAKE_COMMAND) -E copy ${ARROW_LIBRARY_DIR}/libplasma_java.*
+      ${CMAKE_SOURCE_DIR}/java/runtime/native_dependencies)
 endif()

java/README.rst

@@ -5,7 +5,7 @@ Configuration
 -------------
 Ray will read your configurations in the following order:
 
-* Java system properties: e.g., ``-Dray.home=/path/to/ray``.
+* Java system properties: e.g., ``-Dray.run-mode=SINGLE_PROCESS``.
 * A ``ray.conf`` file in the classpath: `example <https://github.com/ray-project/ray/blob/master/java/example.conf>`_.
 * Customise your own ``ray.conf`` path using system property ``-Dray.config=/path/to/ray.conf``
 

java/example.conf

@@ -6,11 +6,6 @@
 # For config file format, see 'https://github.com/lightbend/config/blob/master/HOCON.md'.
 
 ray {
-  // This is the path to the directory where Ray is installed, e.g.,
-  // something like /home/ubmutu/ray. This can be an absolute path or
-  // a relative path from the current working directory.
-  home = "/path/to/your/ray/home"
-
   // Run mode, available options are:
   //
   // `SINGLE_PROCESS`: Ray is running in one single Java process, without Raylet backend,

java/runtime/pom.xml

@@ -79,6 +79,14 @@
   </dependencies>
 
   <build>
+    <resources>
+      <resource>
+        <directory>src/main/resources</directory>
+      </resource>
+      <resource>
+        <directory>native_dependencies</directory>
+      </resource>
+    </resources>
     <plugins>
       <plugin>
         <groupId>org.apache.maven.plugins</groupId>

java/runtime/src/main/java/org/ray/runtime/RayNativeRuntime.java

@@ -2,8 +2,13 @@
 
 import com.google.common.base.Preconditions;
 import com.google.common.base.Strings;
+import java.io.File;
+import java.io.InputStream;
 import java.lang.reflect.Field;
 import java.nio.ByteBuffer;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -72,11 +77,22 @@ private void resetLibraryPath() {
 
   @Override
   public void start() throws Exception {
-    // Load native libraries.
     try {
+      // Reset library path at runtime.
       resetLibraryPath();
-      System.loadLibrary("raylet_library_java");
-      System.loadLibrary("plasma_java");
+
+      // Load native libraries.
+      String[] libraries = new String[]{"raylet_library_java", "plasma_java"};
+      for (String library : libraries) {
+        String fileName = System.mapLibraryName(library);
+        // Copy the file from resources to a temp dir, and load the native library.
+        File file = File.createTempFile(fileName, "");
+        file.deleteOnExit();
+        InputStream in = RayNativeRuntime.class.getResourceAsStream("/" + fileName);
+        Preconditions.checkNotNull(in, "{} doesn't exist.", fileName);
+        Files.copy(in, Paths.get(file.getAbsolutePath()), StandardCopyOption.REPLACE_EXISTING);
+        System.load(file.getAbsolutePath());
+      }
     } catch (Exception e) {
       LOGGER.error("Failed to load native libraries.", e);
       throw e;

java/runtime/src/main/java/org/ray/runtime/config/RayConfig.java

@@ -30,7 +30,6 @@ public class RayConfig {
   public static final String DEFAULT_CONFIG_FILE = "ray.default.conf";
   public static final String CUSTOM_CONFIG_FILE = "ray.conf";
 
-  public final String rayHome;
   public final String nodeIp;
   public final WorkerMode workerMode;
   public final RunMode runMode;
@@ -56,10 +55,6 @@ public class RayConfig {
   public final String rayletSocketName;
   public final List<String> rayletConfigParameters;
 
-  public final String redisServerExecutablePath;
-  public final String redisModulePath;
-  public final String plasmaStoreExecutablePath;
-  public final String rayletExecutablePath;
   public final String driverResourcePath;
   public final String pythonWorkerCommand;
 
@@ -72,9 +67,6 @@ private void validate() {
     if (workerMode == WorkerMode.WORKER) {
       Preconditions.checkArgument(redisAddress != null,
           "Redis address must be set in worker mode.");
-    } else {
-      Preconditions.checkArgument(!rayHome.isEmpty(),
-          "'ray.home' must be set in driver mode");
     }
   }
 
@@ -87,32 +79,24 @@ private String removeTrailingSlash(String path) {
   }
 
   public RayConfig(Config config) {
-    // worker mode
+    // Worker mode.
     WorkerMode localWorkerMode;
     try {
       localWorkerMode = config.getEnum(WorkerMode.class, "ray.worker.mode");
     } catch (ConfigException.Missing e) {
       localWorkerMode = WorkerMode.DRIVER;
     }
-
     workerMode = localWorkerMode;
     boolean isDriver = workerMode == WorkerMode.DRIVER;
-    // run mode
+    // Run mode.
     runMode = config.getEnum(RunMode.class, "ray.run-mode");
-    // ray home
-    String localRayHome = config.getString("ray.home");
-    if (!localRayHome.startsWith("/")) {
-      // If ray.home isn't an absolute path, prepend it with current work dir.
-      localRayHome = System.getProperty("user.dir") + "/" + localRayHome;
-    }
-    rayHome = removeTrailingSlash(localRayHome);
-    // node ip
+    // Node ip.
     String nodeIp = config.getString("ray.node-ip");
     if (nodeIp.isEmpty()) {
       nodeIp = NetworkUtil.getIpAddress(null);
     }
     this.nodeIp = nodeIp;
-    // resources
+    // Resources.
     resources = ResourceUtil.getResourcesMapFromString(
         config.getString("ray.resources"));
     if (isDriver) {
@@ -127,22 +111,22 @@ public RayConfig(Config config) {
         resources.put("GPU", 0.0);
       }
     }
-    // driver id
+    // Driver id.
     String driverId = config.getString("ray.driver.id");
     if (!driverId.isEmpty()) {
       this.driverId = UniqueId.fromHexString(driverId);
     } else {
       this.driverId = UniqueId.randomId();
     }
-    // log dir
+    // Log dir.
     logDir = removeTrailingSlash(config.getString("ray.log-dir"));
-    // redirect output
+    // Redirect output.
     redirectOutput = config.getBoolean("ray.redirect-output");
-    // custom library path
-    List<String> customLibraryPath = config.getStringList("ray.library.path");
-    // custom classpath
+    // Library path.
+    libraryPath = config.getStringList("ray.library.path");
+    // Custom classpath.
     classpath = config.getStringList("ray.classpath");
-    // custom worker jvm parameters
+    // Custom worker jvm parameters.
     if (config.hasPath("ray.worker.jvm-parameters")) {
       jvmParameters = config.getStringList("ray.worker.jvm-parameters");
     } else {
@@ -155,7 +139,7 @@ public RayConfig(Config config) {
       pythonWorkerCommand = null;
     }
 
-    // redis configurations
+    // Redis configurations.
     String redisAddress = config.getString("ray.redis.address");
     if (!redisAddress.isEmpty()) {
       setRedisAddress(redisAddress);
@@ -167,34 +151,22 @@ public RayConfig(Config config) {
     headRedisPassword = config.getString("ray.redis.head-password");
     redisPassword = config.getString("ray.redis.password");
 
-    // object store configurations
+    // Object store configurations.
     objectStoreSocketName = config.getString("ray.object-store.socket-name");
     objectStoreSize = config.getBytes("ray.object-store.size");
 
-    // raylet socket name
+    // Raylet socket name.
     rayletSocketName = config.getString("ray.raylet.socket-name");
 
-    // raylet parameters
-    rayletConfigParameters = new ArrayList<String>();
+    // Raylet parameters.
+    rayletConfigParameters = new ArrayList<>();
     Config rayletConfig = config.getConfig("ray.raylet.config");
     for (Map.Entry<String,ConfigValue> entry : rayletConfig.entrySet()) {
-      String parameter = entry.getKey() + "," + String.valueOf(entry.getValue().unwrapped());
+      String parameter = entry.getKey() + "," + entry.getValue().unwrapped();
       rayletConfigParameters.add(parameter);
     }
 
-    // library path
-    this.libraryPath = new ImmutableList.Builder<String>().add(
-        rayHome + "/build/src/plasma",
-        rayHome + "/build/src/ray/raylet"
-    ).addAll(customLibraryPath).build();
-
-    redisServerExecutablePath = rayHome +
-        "/build/src/ray/thirdparty/redis/src/redis-server";
-    redisModulePath = rayHome + "/build/src/ray/gcs/redis_module/libray_redis_module.so";
-    plasmaStoreExecutablePath = rayHome + "/build/src/plasma/plasma_store_server";
-    rayletExecutablePath = rayHome + "/build/src/ray/raylet/raylet";
-
-    // driver resource path
+    // Driver resource path.
     if (config.hasPath("ray.driver.resource-path")) {
       driverResourcePath = config.getString("ray.driver.resource-path");
     } else {
@@ -204,7 +176,7 @@ public RayConfig(Config config) {
     // Number of threads that execute tasks.
     numberExecThreadsForDevRuntime = config.getInt("ray.dev-runtime.execution-parallelism");
 
-    // validate config
+    // Validate config.
     validate();
     LOGGER.debug("Created config: {}", this);
   }
@@ -235,7 +207,6 @@ public Integer getRedisPort() {
   @Override
   public String toString() {
     return "RayConfig{"
-        + "rayHome='" + rayHome + '\''
         + ", nodeIp='" + nodeIp + '\''
         + ", workerMode=" + workerMode
         + ", runMode=" + runMode
@@ -255,10 +226,6 @@ public String toString() {
         + ", objectStoreSize=" + objectStoreSize
         + ", rayletSocketName='" + rayletSocketName + '\''
         + ", rayletConfigParameters=" + rayletConfigParameters
-        + ", redisServerExecutablePath='" + redisServerExecutablePath + '\''
-        + ", redisModulePath='" + redisModulePath + '\''
-        + ", plasmaStoreExecutablePath='" + plasmaStoreExecutablePath + '\''
-        + ", rayletExecutablePath='" + rayletExecutablePath + '\''
         + ", driverResourcePath='" + driverResourcePath + '\''
         + ", pythonWorkerCommand='" + pythonWorkerCommand + '\''
         + '}';