Merge pull request #413 from surferlocal/dynamic-load_data

Add dynamic load data

.github/workflows/main.yml

@@ -112,4 +112,4 @@ jobs:
       - name: Test mix project with example created from template
         working-directory: rustler_mix
         run: ./test.sh
-        if: "startsWith(matrix.os, 'ubuntu') && matrix.pair.latest"
+        if: "startsWith(matrix.os, 'ubuntu') && matrix.pair.latest"

Cargo.toml

@@ -8,6 +8,7 @@ members = [
   "rustler_tests/native/rustler_test",
   "rustler_tests/native/rustler_bigint_test",
   "rustler_tests/native/deprecated_macros",
+  "rustler_tests/native/dynamic_load",
   "rustler_tests/native/rustler_compile_tests",
   "rustler_benchmarks/native/benchmark",
 ]

rustler_mix/lib/rustler.ex

@@ -43,6 +43,24 @@ defmodule Rustler do
     * `:load_data` - Any valid term. This value is passed into the NIF when it is
       loaded (default: `0`)
 
+    * `:load_data_fun` - `{Module, :function}` to dynamically generate `load_data`.
+      Default value: `nil`.
+
+      This parameter is mutually exclusive with `load_data`
+      which means that `load_data` has to be set to it's default value.
+
+      Example
+
+        defmodule NIF do
+          use Rustler, load_data_fun: {Deployment, :nif_data}
+        end
+
+        defmodule Deployment do
+          def nif_data do
+            :code.priv_dir(:otp_app) |> IO.iodata_to_binary()
+          end
+        end
+
     * `:load_from` - This option allows control over where the final artifact should be
       loaded from at runtime. By default the compiled artifact is loaded from the
       owning `:otp_app`'s `priv/native` directory. This option comes in handy in
@@ -86,6 +104,7 @@ defmodule Rustler do
       if config.lib do
         @load_from config.load_from
         @load_data config.load_data
+        @load_data_fun config.load_data_fun
 
         @before_compile Rustler
       end
@@ -109,11 +128,40 @@ defmodule Rustler do
           |> Application.app_dir(path)
           |> to_charlist()
 
-        :erlang.load_nif(load_path, @load_data)
+        load_data = Rustler.construct_load_data(@load_data, @load_data_fun)
+
+        :erlang.load_nif(load_path, load_data)
       end
     end
   end
 
+  def construct_load_data(load_data, load_data_fun) do
+    default_load_data_value = %Rustler.Compiler.Config{}.load_data
+    default_fun_value = %Rustler.Compiler.Config{}.load_data_fun
+
+    case {load_data, load_data_fun} do
+      {load_data, ^default_fun_value} ->
+        load_data
+
+      {^default_load_data_value, {module, function}}
+      when is_atom(module) and is_atom(function) ->
+        apply(module, function, [])
+
+      {^default_load_data_value, provided_value} ->
+        raise """
+        `load_data` has to be `{Module, :function}`.
+        Instead received: #{inspect(provided_value)}
+        """
+
+      {load_data, load_data_fun} ->
+        raise """
+        Only `load_data` or `load_data_fun` can be provided. Instead received:
+        >>> load_data: #{inspect(load_data)}
+        >>> load_data_fun: #{inspect(load_data_fun)}
+        """
+    end
+  end
+
   @doc false
   def rustler_version, do: "0.27.0"
 

rustler_mix/lib/rustler/compiler/config.ex

@@ -19,6 +19,7 @@ defmodule Rustler.Compiler.Config do
             features: [],
             lib: true,
             load_data: 0,
+            load_data_fun: nil,
             load_from: nil,
             mode: :release,
             otp_app: nil,

rustler_tests/lib/dynamic_data.ex

@@ -0,0 +1,14 @@
+defmodule DynamicData.Config do
+  def nif_data do
+    %{priv_path: :code.priv_dir(:rustler_test) |> IO.iodata_to_binary()}
+  end
+end
+
+defmodule DynamicData do
+  use Rustler,
+    otp_app: :rustler_test,
+    crate: :dynamic_load,
+    load_data_fun: {DynamicData.Config, :nif_data}
+
+  def get_dataset, do: :erlang.nif_error(:nif_not_loaded)
+end

rustler_tests/native/dynamic_load/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "dynamic_load"
+version = "0.1.0"
+edition = "2021"
+
+[lib]
+name = "dynamic_load"
+path = "src/lib.rs"
+crate-type = ["cdylib"]
+
+[dependencies]
+rustler_bigint = { path = "../../../rustler_bigint" }
+rustler = { path = "../../../rustler" }

rustler_tests/native/dynamic_load/src/lib.rs

@@ -0,0 +1,55 @@
+use rustler::Atom;
+use std::{ffi::OsStr, fs::read_to_string, path::PathBuf};
+
+static mut DATASET: Option<Box<str>> = None;
+
+fn initialize_dataset(mut asset_path: PathBuf) {
+    asset_path.push("demo_dataset.txt");
+
+    // https://github.com/elixir-lsp/elixir-ls/issues/604
+    eprintln!("Loading dataset from {:?}.", &asset_path);
+
+    let data = read_to_string(asset_path).unwrap().into_boxed_str();
+    let data = Some(data);
+
+    // Safety: assumes that this function is being called once when
+    // dynamically loading this library.
+    // `load()` is being called exactly once and OTP will not allow any other function call
+    // before this function returns
+    unsafe { DATASET = data };
+}
+
+#[rustler::nif]
+fn get_dataset() -> &'static str {
+    // Safety: see `initialize_dataset()`
+    unsafe { DATASET.as_ref() }.expect("Dataset is not initialized")
+}
+
+fn load<'a>(env: rustler::Env<'a>, args: rustler::Term<'a>) -> bool {
+    let key = Atom::from_str(env, "priv_path").unwrap().to_term(env);
+    let priv_path = args.map_get(key).unwrap();
+    let priv_path = priv_path.into_binary().unwrap().as_slice();
+
+    let asset_path = build_path_buf(priv_path);
+
+    initialize_dataset(asset_path);
+
+    true
+}
+
+#[cfg(unix)]
+fn build_path_buf(priv_path: &[u8]) -> PathBuf {
+    use std::os::unix::prelude::OsStrExt;
+
+    let priv_path = OsStr::from_bytes(priv_path);
+    PathBuf::from(priv_path)
+}
+
+#[cfg(windows)]
+fn build_path_buf(priv_path: &[u8]) -> PathBuf {
+    let string_slice = std::str::from_utf8(priv_path).expect("Data is not valid UTF-8, we could decode it without valid UTF-8 requirements but lets not do that for now because its easier this way");
+    let priv_path = OsStr::new(string_slice);
+    PathBuf::from(priv_path)
+}
+
+rustler::init!("Elixir.DynamicData", [get_dataset], load = load);

rustler_tests/priv/demo_dataset.txt

@@ -0,0 +1 @@
+some random dataset

rustler_tests/test/dynamic_data_test.exs

@@ -0,0 +1,10 @@
+defmodule RustlerTest.DynamicDataTest do
+  use ExUnit.Case, async: true
+
+  test "rust has access to demo_dataset.txt via dynamic priv path" do
+    %{priv_path: path} = DynamicData.Config.nif_data()
+    path = Path.join(path, "demo_dataset.txt")
+
+    assert File.read!(path) == DynamicData.get_dataset()
+  end
+end