From f8d12b04183b01291fe359a62fa6542d349967f2 Mon Sep 17 00:00:00 2001 From: Hao Chen Date: Fri, 15 Mar 2019 12:38:40 +0800 Subject: [PATCH] [Java] Package native dependencies into jar (#4367) --- .gitignore | 1 + CMakeLists.txt | 21 +++++- java/README.rst | 2 +- java/example.conf | 5 -- java/runtime/pom.xml | 8 +++ .../org/ray/runtime/RayNativeRuntime.java | 22 +++++- .../org/ray/runtime/config/RayConfig.java | 71 +++++-------------- .../org/ray/runtime/runner/RunManager.java | 45 ++++++++++-- .../src/main/resources/ray.default.conf | 5 -- .../main/java/org/ray/api/test/BaseTest.java | 24 ++++--- .../api/test/MultiLanguageClusterTest.java | 5 +- .../java/org/ray/api/test/RayConfigTest.java | 14 +--- java/tutorial/README.rst | 2 +- java/tutorial/src/main/resources/ray.conf | 3 +- 14 files changed, 126 insertions(+), 102 deletions(-) diff --git a/.gitignore b/.gitignore index da464a65266a..21d8edfc4f96 100644 --- a/.gitignore +++ b/.gitignore @@ -148,6 +148,7 @@ java/**/lib java/**/.settings java/**/.classpath java/**/.project +java/runtime/native_dependencies/ # python virtual env venv diff --git a/CMakeLists.txt b/CMakeLists.txt index 47f7b2ebe677..84236d4e9ce5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -152,8 +152,23 @@ if ("${CMAKE_RAY_LANG_JAVA}" STREQUAL "YES") get_raylet_library("java" RAYLET_LIBRARY_JAVA) add_dependencies(copy_ray ${RAYLET_LIBRARY_JAVA}) - # copy libplasma_java files + # Copy java native dependencies. add_custom_command(TARGET copy_ray POST_BUILD - COMMAND bash -c "mkdir -p ${CMAKE_CURRENT_BINARY_DIR}/src/plasma" - COMMAND bash -c "cp ${ARROW_LIBRARY_DIR}/libplasma_java.* ${CMAKE_CURRENT_BINARY_DIR}/src/plasma/") + COMMAND mkdir -p ${CMAKE_SOURCE_DIR}/java/runtime/native_dependencies) + set(java_native_dependencies + "src/ray/thirdparty/redis/src/redis-server" + "src/ray/gcs/redis_module/libray_redis_module.so" + "src/ray/raylet/raylet" + "src/ray/raylet/libraylet_library_java.*") + foreach(file ${java_native_dependencies}) + add_custom_command(TARGET copy_ray POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/${file} + ${CMAKE_SOURCE_DIR}/java/runtime/native_dependencies) + endforeach() +add_custom_command(TARGET copy_ray POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy ${ARROW_HOME}/bin/plasma_store_server + ${CMAKE_SOURCE_DIR}/java/runtime/native_dependencies) + add_custom_command(TARGET copy_ray POST_BUILD + COMMAND $(CMAKE_COMMAND) -E copy ${ARROW_LIBRARY_DIR}/libplasma_java.* + ${CMAKE_SOURCE_DIR}/java/runtime/native_dependencies) endif() diff --git a/java/README.rst b/java/README.rst index e01616935787..f774ce57efaa 100644 --- a/java/README.rst +++ b/java/README.rst @@ -5,7 +5,7 @@ Configuration ------------- Ray will read your configurations in the following order: -* Java system properties: e.g., ``-Dray.home=/path/to/ray``. +* Java system properties: e.g., ``-Dray.run-mode=SINGLE_PROCESS``. * A ``ray.conf`` file in the classpath: `example `_. * Customise your own ``ray.conf`` path using system property ``-Dray.config=/path/to/ray.conf`` diff --git a/java/example.conf b/java/example.conf index ca42b336167f..e6aaa37ea8b7 100644 --- a/java/example.conf +++ b/java/example.conf @@ -6,11 +6,6 @@ # For config file format, see 'https://github.com/lightbend/config/blob/master/HOCON.md'. ray { - // This is the path to the directory where Ray is installed, e.g., - // something like /home/ubmutu/ray. This can be an absolute path or - // a relative path from the current working directory. - home = "/path/to/your/ray/home" - // Run mode, available options are: // // `SINGLE_PROCESS`: Ray is running in one single Java process, without Raylet backend, diff --git a/java/runtime/pom.xml b/java/runtime/pom.xml index 8e4fd8059594..4305b074313f 100644 --- a/java/runtime/pom.xml +++ b/java/runtime/pom.xml @@ -79,6 +79,14 @@ + + + src/main/resources + + + native_dependencies + + org.apache.maven.plugins diff --git a/java/runtime/src/main/java/org/ray/runtime/RayNativeRuntime.java b/java/runtime/src/main/java/org/ray/runtime/RayNativeRuntime.java index 4c070ed88d7b..a0fbdf01f28d 100644 --- a/java/runtime/src/main/java/org/ray/runtime/RayNativeRuntime.java +++ b/java/runtime/src/main/java/org/ray/runtime/RayNativeRuntime.java @@ -2,8 +2,13 @@ import com.google.common.base.Preconditions; import com.google.common.base.Strings; +import java.io.File; +import java.io.InputStream; import java.lang.reflect.Field; import java.nio.ByteBuffer; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -72,11 +77,22 @@ private void resetLibraryPath() { @Override public void start() throws Exception { - // Load native libraries. try { + // Reset library path at runtime. resetLibraryPath(); - System.loadLibrary("raylet_library_java"); - System.loadLibrary("plasma_java"); + + // Load native libraries. + String[] libraries = new String[]{"raylet_library_java", "plasma_java"}; + for (String library : libraries) { + String fileName = System.mapLibraryName(library); + // Copy the file from resources to a temp dir, and load the native library. + File file = File.createTempFile(fileName, ""); + file.deleteOnExit(); + InputStream in = RayNativeRuntime.class.getResourceAsStream("/" + fileName); + Preconditions.checkNotNull(in, "{} doesn't exist.", fileName); + Files.copy(in, Paths.get(file.getAbsolutePath()), StandardCopyOption.REPLACE_EXISTING); + System.load(file.getAbsolutePath()); + } } catch (Exception e) { LOGGER.error("Failed to load native libraries.", e); throw e; diff --git a/java/runtime/src/main/java/org/ray/runtime/config/RayConfig.java b/java/runtime/src/main/java/org/ray/runtime/config/RayConfig.java index 6aab89764930..079b9538f7cd 100644 --- a/java/runtime/src/main/java/org/ray/runtime/config/RayConfig.java +++ b/java/runtime/src/main/java/org/ray/runtime/config/RayConfig.java @@ -30,7 +30,6 @@ public class RayConfig { public static final String DEFAULT_CONFIG_FILE = "ray.default.conf"; public static final String CUSTOM_CONFIG_FILE = "ray.conf"; - public final String rayHome; public final String nodeIp; public final WorkerMode workerMode; public final RunMode runMode; @@ -56,10 +55,6 @@ public class RayConfig { public final String rayletSocketName; public final List rayletConfigParameters; - public final String redisServerExecutablePath; - public final String redisModulePath; - public final String plasmaStoreExecutablePath; - public final String rayletExecutablePath; public final String driverResourcePath; public final String pythonWorkerCommand; @@ -72,9 +67,6 @@ private void validate() { if (workerMode == WorkerMode.WORKER) { Preconditions.checkArgument(redisAddress != null, "Redis address must be set in worker mode."); - } else { - Preconditions.checkArgument(!rayHome.isEmpty(), - "'ray.home' must be set in driver mode"); } } @@ -87,32 +79,24 @@ private String removeTrailingSlash(String path) { } public RayConfig(Config config) { - // worker mode + // Worker mode. WorkerMode localWorkerMode; try { localWorkerMode = config.getEnum(WorkerMode.class, "ray.worker.mode"); } catch (ConfigException.Missing e) { localWorkerMode = WorkerMode.DRIVER; } - workerMode = localWorkerMode; boolean isDriver = workerMode == WorkerMode.DRIVER; - // run mode + // Run mode. runMode = config.getEnum(RunMode.class, "ray.run-mode"); - // ray home - String localRayHome = config.getString("ray.home"); - if (!localRayHome.startsWith("/")) { - // If ray.home isn't an absolute path, prepend it with current work dir. - localRayHome = System.getProperty("user.dir") + "/" + localRayHome; - } - rayHome = removeTrailingSlash(localRayHome); - // node ip + // Node ip. String nodeIp = config.getString("ray.node-ip"); if (nodeIp.isEmpty()) { nodeIp = NetworkUtil.getIpAddress(null); } this.nodeIp = nodeIp; - // resources + // Resources. resources = ResourceUtil.getResourcesMapFromString( config.getString("ray.resources")); if (isDriver) { @@ -127,22 +111,22 @@ public RayConfig(Config config) { resources.put("GPU", 0.0); } } - // driver id + // Driver id. String driverId = config.getString("ray.driver.id"); if (!driverId.isEmpty()) { this.driverId = UniqueId.fromHexString(driverId); } else { this.driverId = UniqueId.randomId(); } - // log dir + // Log dir. logDir = removeTrailingSlash(config.getString("ray.log-dir")); - // redirect output + // Redirect output. redirectOutput = config.getBoolean("ray.redirect-output"); - // custom library path - List customLibraryPath = config.getStringList("ray.library.path"); - // custom classpath + // Library path. + libraryPath = config.getStringList("ray.library.path"); + // Custom classpath. classpath = config.getStringList("ray.classpath"); - // custom worker jvm parameters + // Custom worker jvm parameters. if (config.hasPath("ray.worker.jvm-parameters")) { jvmParameters = config.getStringList("ray.worker.jvm-parameters"); } else { @@ -155,7 +139,7 @@ public RayConfig(Config config) { pythonWorkerCommand = null; } - // redis configurations + // Redis configurations. String redisAddress = config.getString("ray.redis.address"); if (!redisAddress.isEmpty()) { setRedisAddress(redisAddress); @@ -167,34 +151,22 @@ public RayConfig(Config config) { headRedisPassword = config.getString("ray.redis.head-password"); redisPassword = config.getString("ray.redis.password"); - // object store configurations + // Object store configurations. objectStoreSocketName = config.getString("ray.object-store.socket-name"); objectStoreSize = config.getBytes("ray.object-store.size"); - // raylet socket name + // Raylet socket name. rayletSocketName = config.getString("ray.raylet.socket-name"); - // raylet parameters - rayletConfigParameters = new ArrayList(); + // Raylet parameters. + rayletConfigParameters = new ArrayList<>(); Config rayletConfig = config.getConfig("ray.raylet.config"); for (Map.Entry entry : rayletConfig.entrySet()) { - String parameter = entry.getKey() + "," + String.valueOf(entry.getValue().unwrapped()); + String parameter = entry.getKey() + "," + entry.getValue().unwrapped(); rayletConfigParameters.add(parameter); } - // library path - this.libraryPath = new ImmutableList.Builder().add( - rayHome + "/build/src/plasma", - rayHome + "/build/src/ray/raylet" - ).addAll(customLibraryPath).build(); - - redisServerExecutablePath = rayHome + - "/build/src/ray/thirdparty/redis/src/redis-server"; - redisModulePath = rayHome + "/build/src/ray/gcs/redis_module/libray_redis_module.so"; - plasmaStoreExecutablePath = rayHome + "/build/src/plasma/plasma_store_server"; - rayletExecutablePath = rayHome + "/build/src/ray/raylet/raylet"; - - // driver resource path + // Driver resource path. if (config.hasPath("ray.driver.resource-path")) { driverResourcePath = config.getString("ray.driver.resource-path"); } else { @@ -204,7 +176,7 @@ public RayConfig(Config config) { // Number of threads that execute tasks. numberExecThreadsForDevRuntime = config.getInt("ray.dev-runtime.execution-parallelism"); - // validate config + // Validate config. validate(); LOGGER.debug("Created config: {}", this); } @@ -235,7 +207,6 @@ public Integer getRedisPort() { @Override public String toString() { return "RayConfig{" - + "rayHome='" + rayHome + '\'' + ", nodeIp='" + nodeIp + '\'' + ", workerMode=" + workerMode + ", runMode=" + runMode @@ -255,10 +226,6 @@ public String toString() { + ", objectStoreSize=" + objectStoreSize + ", rayletSocketName='" + rayletSocketName + '\'' + ", rayletConfigParameters=" + rayletConfigParameters - + ", redisServerExecutablePath='" + redisServerExecutablePath + '\'' - + ", redisModulePath='" + redisModulePath + '\'' - + ", plasmaStoreExecutablePath='" + plasmaStoreExecutablePath + '\'' - + ", rayletExecutablePath='" + rayletExecutablePath + '\'' + ", driverResourcePath='" + driverResourcePath + '\'' + ", pythonWorkerCommand='" + pythonWorkerCommand + '\'' + '}'; diff --git a/java/runtime/src/main/java/org/ray/runtime/runner/RunManager.java b/java/runtime/src/main/java/org/ray/runtime/runner/RunManager.java index f0f1df8befa9..bec4eea591e8 100644 --- a/java/runtime/src/main/java/org/ray/runtime/runner/RunManager.java +++ b/java/runtime/src/main/java/org/ray/runtime/runner/RunManager.java @@ -5,9 +5,14 @@ import com.google.common.collect.Lists; import java.io.File; import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Random; @@ -42,10 +47,13 @@ public class RunManager { private static final int KILL_PROCESS_WAIT_TIMEOUT_SECONDS = 1; + private final Map tempFiles; + public RunManager(RayConfig rayConfig) { this.rayConfig = rayConfig; processes = new ArrayList<>(); random = new Random(); + tempFiles = new HashMap<>(); } public void cleanup() { @@ -61,7 +69,7 @@ public void cleanup() { p.waitFor(KILL_PROCESS_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); } catch (InterruptedException e) { LOGGER.warn("Got InterruptedException while waiting for process {}" + - " to be terminated.", processes.get(i)); + " to be terminated.", processes.get(i)); } if (p.isAlive()) { @@ -76,8 +84,30 @@ private void createTempDirs() { FileUtil.mkDir(new File(rayConfig.objectStoreSocketName).getParentFile()); } + /** + * Copy a file from resources to a temp dir, and return the file object. + */ + private File getTempFile(String fileName) { + File file = tempFiles.get(fileName); + if (file == null) { + try { + file = File.createTempFile(fileName, ""); + file.deleteOnExit(); + try (InputStream in = RunManager.class.getResourceAsStream(fileName)) { + Files.copy(in, Paths.get(file.getCanonicalPath()), StandardCopyOption.REPLACE_EXISTING); + } + file.setExecutable(true); + } catch (IOException e) { + throw new RuntimeException("Couldn't get temp file " + fileName, e); + } + tempFiles.put(fileName, file); + } + return file; + } + /** * Start a process. + * * @param command The command to start the process with. * @param env Environment variables. * @param name Process name. @@ -126,6 +156,7 @@ private void startProcess(List command, Map env, String /** * Start all Ray processes on this node. + * * @param isHead Whether this node is the head node. If true, redis server will be started. */ public void startRayProcesses(boolean isHead) { @@ -171,7 +202,8 @@ private void startRedisServer() { private String startRedisInstance(String ip, int port, String password, Integer shard) { List command = Lists.newArrayList( - rayConfig.redisServerExecutablePath, + // The redis-server executable file. + getTempFile("/redis-server").getAbsolutePath(), "--protected-mode", "no", "--port", @@ -179,7 +211,8 @@ private String startRedisInstance(String ip, int port, String password, Integer "--loglevel", "warning", "--loadmodule", - rayConfig.redisModulePath + // The redis module file. + getTempFile("/libray_redis_module.so").getAbsolutePath() ); if (!StringUtil.isNullOrEmpty(password)) { @@ -216,7 +249,8 @@ private void startRaylet() { // See `src/ray/raylet/main.cc` for the meaning of each parameter. List command = ImmutableList.of( - rayConfig.rayletExecutablePath, + // The raylet executable file. + getTempFile("/raylet").getAbsolutePath(), rayConfig.rayletSocketName, rayConfig.objectStoreSocketName, "0", // The object manager port. @@ -291,7 +325,8 @@ private String buildWorkerCommandRaylet() { private void startObjectStore() { List command = ImmutableList.of( - rayConfig.plasmaStoreExecutablePath, + // The plasma store executable file. + getTempFile("/plasma_store_server").getAbsolutePath(), "-s", rayConfig.objectStoreSocketName, "-m", diff --git a/java/runtime/src/main/resources/ray.default.conf b/java/runtime/src/main/resources/ray.default.conf index 5faeda7cfedf..a2762e76dd60 100644 --- a/java/runtime/src/main/resources/ray.default.conf +++ b/java/runtime/src/main/resources/ray.default.conf @@ -7,11 +7,6 @@ ray { // Basic configurations // ---------------------- - // This is the path to the directory where Ray is installed, e.g., - // something like /home/ubmutu/ray. This can be an absolute path or - // a relative path from the current working directory. - home: "" - // IP of this node. if not provided, IP will be automatically detected. node-ip: "" diff --git a/java/test/src/main/java/org/ray/api/test/BaseTest.java b/java/test/src/main/java/org/ray/api/test/BaseTest.java index b67a8f64c7ce..366ee9a6737f 100644 --- a/java/test/src/main/java/org/ray/api/test/BaseTest.java +++ b/java/test/src/main/java/org/ray/api/test/BaseTest.java @@ -1,6 +1,9 @@ package org.ray.api.test; +import com.google.common.collect.ImmutableList; +import java.io.File; import java.lang.reflect.Method; +import java.util.List; import org.ray.api.Ray; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -11,27 +14,32 @@ public class BaseTest { private static final Logger LOGGER = LoggerFactory.getLogger(BaseTest.class); + private List filesToDelete; + @BeforeMethod public void setUpBase(Method method) { LOGGER.info("===== Running test: " + method.getDeclaringClass().getName() + "." + method.getName()); - System.setProperty("ray.home", "../.."); System.setProperty("ray.resources", "CPU:4,RES-A:4"); Ray.init(); + // These files need to be deleted after each test case. + filesToDelete = ImmutableList.of( + new File(Ray.getRuntimeContext().getRayletSocketName()), + new File(Ray.getRuntimeContext().getObjectStoreSocketName()) + ); + // Make sure the files will be deleted even if the test doesn't exit gracefully. + filesToDelete.forEach(File::deleteOnExit); } @AfterMethod public void tearDownBase() { - // TODO(qwang): This is double check to check that the socket file is removed actually. - // We could not enable this until `systemInfo` enabled. - //File rayletSocketFIle = new File(Ray.systemInfo().rayletSocketName()); Ray.shutdown(); - //remove raylet socket file - //rayletSocketFIle.delete(); + for (File file : filesToDelete) { + file.delete(); + } - // unset system properties - System.clearProperty("ray.home"); + // Unset system properties. System.clearProperty("ray.resources"); } diff --git a/java/test/src/main/java/org/ray/api/test/MultiLanguageClusterTest.java b/java/test/src/main/java/org/ray/api/test/MultiLanguageClusterTest.java index c81a148980f9..ad3033681a68 100644 --- a/java/test/src/main/java/org/ray/api/test/MultiLanguageClusterTest.java +++ b/java/test/src/main/java/org/ray/api/test/MultiLanguageClusterTest.java @@ -66,8 +66,7 @@ public void setUp(Method method) { // Start ray cluster. String testDir = System.getProperty("user.dir"); - String workerOptions = String.format("-Dray.home=%s/../../", testDir); - workerOptions += + String workerOptions = " -classpath " + String.format("%s/../../build/java/*:%s/target/*", testDir, testDir); final List startCommand = ImmutableList.of( "ray", @@ -85,7 +84,6 @@ public void setUp(Method method) { } // Connect to the cluster. - System.setProperty("ray.home", "../.."); System.setProperty("ray.redis.address", "127.0.0.1:6379"); System.setProperty("ray.object-store.socket-name", PLASMA_STORE_SOCKET_NAME); System.setProperty("ray.raylet.socket-name", RAYLET_SOCKET_NAME); @@ -96,7 +94,6 @@ public void setUp(Method method) { public void tearDown() { // Disconnect to the cluster. Ray.shutdown(); - System.clearProperty("ray.home"); System.clearProperty("ray.redis.address"); System.clearProperty("ray.object-store.socket-name"); System.clearProperty("ray.raylet.socket-name"); diff --git a/java/test/src/main/java/org/ray/api/test/RayConfigTest.java b/java/test/src/main/java/org/ray/api/test/RayConfigTest.java index c728eda49eee..26702ebe4c28 100644 --- a/java/test/src/main/java/org/ray/api/test/RayConfigTest.java +++ b/java/test/src/main/java/org/ray/api/test/RayConfigTest.java @@ -10,24 +10,12 @@ public class RayConfigTest { @Test public void testCreateRayConfig() { try { - System.setProperty("ray.home", "/path/to/ray"); System.setProperty("ray.driver.resource-path", "path/to/ray/driver/resource/path"); RayConfig rayConfig = RayConfig.create(); - - Assert.assertEquals("/path/to/ray", rayConfig.rayHome); Assert.assertEquals(WorkerMode.DRIVER, rayConfig.workerMode); - - System.setProperty("ray.home", ""); - rayConfig = RayConfig.create(); - - Assert.assertEquals(System.getProperty("user.dir"), rayConfig.rayHome); - Assert.assertEquals(System.getProperty("user.dir") + - "/build/src/ray/thirdparty/redis/src/redis-server", rayConfig.redisServerExecutablePath); - Assert.assertEquals("path/to/ray/driver/resource/path", rayConfig.driverResourcePath); } finally { - //unset the system property - System.clearProperty("ray.home"); + // Unset system properties. System.clearProperty("ray.driver.resource-path"); } diff --git a/java/tutorial/README.rst b/java/tutorial/README.rst index e099ad25badc..01430605834e 100644 --- a/java/tutorial/README.rst +++ b/java/tutorial/README.rst @@ -12,7 +12,7 @@ To run them, execute the following command under ``ray/java`` folder. .. code-block:: shell - java -Dray.home=.. -classpath "tutorial/target/ray-tutorial-1.0.jar:tutorial/lib/*" org.ray.exercise.Exercise01 + java -classpath "tutorial/target/ray-tutorial-1.0.jar:tutorial/lib/*" org.ray.exercise.Exercise01 `Exercise 1 `_: Define a remote function, and execute multiple remote functions in parallel. diff --git a/java/tutorial/src/main/resources/ray.conf b/java/tutorial/src/main/resources/ray.conf index a6cdf42f8f15..e79b1d232982 100644 --- a/java/tutorial/src/main/resources/ray.conf +++ b/java/tutorial/src/main/resources/ray.conf @@ -1,5 +1,4 @@ -ray{ - home: ".." +ray { run-mode: CLUSTER redirect-output: false }