From a8edbcdad42b908ed9351390af4fe4ae70430333 Mon Sep 17 00:00:00 2001 From: b5 Date: Wed, 12 Apr 2023 14:59:55 -0400 Subject: [PATCH 01/32] feat: initial support for c bindings --- irohc/.gitignore | 2 + irohc/Cargo.toml | 22 +++++++ irohc/irohc.h | 27 +++++++++ irohc/readme.md | 8 +++ irohc/src/lib.rs | 154 +++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 213 insertions(+) create mode 100644 irohc/.gitignore create mode 100644 irohc/Cargo.toml create mode 100644 irohc/irohc.h create mode 100644 irohc/readme.md create mode 100644 irohc/src/lib.rs diff --git a/irohc/.gitignore b/irohc/.gitignore new file mode 100644 index 0000000000..4fffb2f89c --- /dev/null +++ b/irohc/.gitignore @@ -0,0 +1,2 @@ +/target +/Cargo.lock diff --git a/irohc/Cargo.toml b/irohc/Cargo.toml new file mode 100644 index 0000000000..d77de18a8d --- /dev/null +++ b/irohc/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "irohc" +version = "0.1.0" +edition = "2021" + +[profile.release] +strip = "symbols" + +[lib] +name="irohc" +crate-type = ["cdylib"] + +[target.x86_64-apple-darwin] +rustflags=["-C", "link-arg=-mmacosx-version-min=10.7"] +strip = "symbols" + +[dependencies] +anyhow = "1.0.69" +iroh = { path = "../" } +libc = "0.2.141" +tempfile = "3.4" +tokio = "1.25.0" diff --git a/irohc/irohc.h b/irohc/irohc.h new file mode 100644 index 0000000000..5649a3f98f --- /dev/null +++ b/irohc/irohc.h @@ -0,0 +1,27 @@ +#include +#include +#include +#include +#include + +#if defined _WIN32 || defined _WIN64 +#define IROHC_IMPORT __declspec(dllimport) +#elif defined __linux__ +#define IROHC_IMPORT __attribute__((visibility("default"))) +#else +#define IROHC_IMPORT +#endif + + +extern "C" { + +// exists just to prove _anything_ can be called in rust land +int32_t add_numbers(int32_t number1, int32_t number2); + +// implementation of the "iroh get-ticket" CLI subcommand +// @param ticket is the all-in-one ticket string output, eg: IEkcjuPpomMYB0viuenxrRBlQgIRhWWGnXFLjHN7HzaMIMtCQe_7BBGUta9UV3mAzQcmZeCBJS3GnT-dqxpgREb9AQB_AAAB0SIcO19VsSqBsALDFAhAPW1tkqKg6elXf0b2dfxWxpgkwg +// @param out_path is the path to write the output files to +// @return is 0 for success, 1 for any kind of error +int32_t get_ticket(const char* ticket, const char* out_path); + +} // extern "C" diff --git a/irohc/readme.md b/irohc/readme.md new file mode 100644 index 0000000000..a72bc9c2f7 --- /dev/null +++ b/irohc/readme.md @@ -0,0 +1,8 @@ +# C bindings for Iroh + +Running `cargo build --release` will produce a dynamic libaray. `irohc.h` is a handcrafted header file for that library. Should work on all platforms, I've only tested it on MacOS so far. + +For builds targeting older versions of MacOS, build with with: `MACOSX_DEPLOYMENT_TARGET=10.7 && cargo build --target x86_64-apple-darwin --release`. + +## Why not use [cbindgen](https://github.com/eqrion/cbindgen)? +I (b5) have tried, not much success so far. If someone else wants to give it a go, be my guest. \ No newline at end of file diff --git a/irohc/src/lib.rs b/irohc/src/lib.rs new file mode 100644 index 0000000000..5c91bb7391 --- /dev/null +++ b/irohc/src/lib.rs @@ -0,0 +1,154 @@ +use std::{path::PathBuf, str::FromStr}; + +use anyhow::{Context, Result}; +use iroh::{get, provider::Ticket, Hash}; +use std::ffi::CStr; +use tokio::runtime::Runtime; + +const MAX_CONCURRENT_DIALS: u8 = 16; + +#[no_mangle] +pub extern "C" fn add_numbers(number1: i32, number2: i32) -> i32 { + let result = std::panic::catch_unwind(|| { + println!("Hello from rust!"); + number1 + number2 + }); + if result.is_err() { + eprintln!("error: rust panicked"); + return -1; + } + result.unwrap() +} + +#[no_mangle] +pub extern "C" fn get_ticket( + ticket: *const std::os::raw::c_char, + out_path: *const std::os::raw::c_char, +) -> u32 { + let result = std::panic::catch_unwind(|| { + let tkt = unsafe { + assert!(!ticket.is_null()); + CStr::from_ptr(ticket) + }; + let tkt = tkt.to_str().unwrap(); + let tkt = tkt.parse::().unwrap(); + + let out_path = unsafe { + assert!(!ticket.is_null()); + CStr::from_ptr(out_path) + }; + let out_path = out_path.to_str().unwrap(); + let out_path = PathBuf::from_str(out_path).unwrap(); + println!("temp dir: {:?}", out_path); + + let rt = Runtime::new().unwrap(); + rt.block_on(get_ticket_internal(tkt, Some(out_path))) + .unwrap(); + 0 + }); + if result.is_err() { + eprintln!("error: rust panicked"); + return 1; + } + result.unwrap() +} + +async fn get_ticket_internal(ticket: Ticket, out: Option) -> Result<()> { + let on_connected = || async move { + println!("connected!"); + Ok(()) + }; + let on_collection = |collection: &iroh::blobs::Collection| { + // let name = collection.name().to_string(); + let total_entries = collection.total_entries(); + let size = collection.total_blobs_size(); + async move { + println!( + "downloading collection containing {total_entries} entries totalling {size} bytes" + ); + Ok(()) + } + }; + + let on_blob = |hash: Hash, mut reader, name: String| { + let out = &out; + async move { + let name = if name.is_empty() { + hash.to_string() + } else { + name + }; + + if let Some(ref outpath) = out { + tokio::fs::create_dir_all(outpath) + .await + .context("Unable to create directory {outpath}")?; + let dirpath = std::path::PathBuf::from(outpath); + let filepath = dirpath.join(name); + + // Create temp file + let (temp_file, dup) = tokio::task::spawn_blocking(|| { + let temp_file = tempfile::Builder::new() + .prefix("iroh-tmp-") + .tempfile_in(dirpath) + .context("Failed to create temporary output file")?; + let dup = temp_file.as_file().try_clone()?; + Ok::<_, anyhow::Error>((temp_file, dup)) + }) + .await??; + + let file = tokio::fs::File::from_std(dup); + let mut file_buf = tokio::io::BufWriter::new(file); + tokio::io::copy(&mut reader, &mut file_buf).await?; + + // Rename temp file, to target name + let filepath2 = filepath.clone(); + if let Some(parent) = filepath2.parent() { + tokio::fs::create_dir_all(parent) + .await + .context("Unable to create directory {parent}")?; + } + println!("writing {:?}", &filepath2); + tokio::task::spawn_blocking(|| temp_file.persist(filepath2)) + .await? + .context("Failed to write output file")?; + } else { + // Write to OUT_WRITER + let mut stdout = tokio::io::stdout(); + tokio::io::copy(&mut reader, &mut stdout).await?; + } + + Ok(reader) + } + }; + + let stats = get::run_ticket( + &ticket, + false, + MAX_CONCURRENT_DIALS, + on_connected, + on_collection, + on_blob, + ) + .await?; + // let stats = get::run(hash, token, opts, on_connected, on_collection, on_blob).await?; + println!("Done in {:?}", stats.elapsed); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn add_numbers_test() { + let result = add_numbers(2, 2); + assert_eq!(result, 4); + } + + #[test] + fn get_ticket_test() { + let result = get_ticket(); + assert_eq!(result, 0); + } +} From 47efc1287b01d915d5de2ddde73f9a63aedcb2b9 Mon Sep 17 00:00:00 2001 From: b5 Date: Mon, 24 Apr 2023 16:54:53 -0400 Subject: [PATCH 02/32] refactor: use safer_ffi --- irohc/irohc.h | 27 -------------------- {irohc => libiroh}/.gitignore | 1 + {irohc => libiroh}/Cargo.toml | 8 ++++-- libiroh/c/Makefile | 4 +++ libiroh/c/libiroh.h | 34 ++++++++++++++++++++++++++ libiroh/c/main.c | 16 ++++++++++++ {irohc => libiroh}/readme.md | 2 +- {irohc => libiroh}/src/lib.rs | 46 +++++++++++++++-------------------- 8 files changed, 82 insertions(+), 56 deletions(-) delete mode 100644 irohc/irohc.h rename {irohc => libiroh}/.gitignore (74%) rename {irohc => libiroh}/Cargo.toml (69%) create mode 100644 libiroh/c/Makefile create mode 100644 libiroh/c/libiroh.h create mode 100644 libiroh/c/main.c rename {irohc => libiroh}/readme.md (75%) rename {irohc => libiroh}/src/lib.rs (82%) diff --git a/irohc/irohc.h b/irohc/irohc.h deleted file mode 100644 index 5649a3f98f..0000000000 --- a/irohc/irohc.h +++ /dev/null @@ -1,27 +0,0 @@ -#include -#include -#include -#include -#include - -#if defined _WIN32 || defined _WIN64 -#define IROHC_IMPORT __declspec(dllimport) -#elif defined __linux__ -#define IROHC_IMPORT __attribute__((visibility("default"))) -#else -#define IROHC_IMPORT -#endif - - -extern "C" { - -// exists just to prove _anything_ can be called in rust land -int32_t add_numbers(int32_t number1, int32_t number2); - -// implementation of the "iroh get-ticket" CLI subcommand -// @param ticket is the all-in-one ticket string output, eg: IEkcjuPpomMYB0viuenxrRBlQgIRhWWGnXFLjHN7HzaMIMtCQe_7BBGUta9UV3mAzQcmZeCBJS3GnT-dqxpgREb9AQB_AAAB0SIcO19VsSqBsALDFAhAPW1tkqKg6elXf0b2dfxWxpgkwg -// @param out_path is the path to write the output files to -// @return is 0 for success, 1 for any kind of error -int32_t get_ticket(const char* ticket, const char* out_path); - -} // extern "C" diff --git a/irohc/.gitignore b/libiroh/.gitignore similarity index 74% rename from irohc/.gitignore rename to libiroh/.gitignore index 4fffb2f89c..7fb54d769e 100644 --- a/irohc/.gitignore +++ b/libiroh/.gitignore @@ -1,2 +1,3 @@ /target /Cargo.lock +/c/main \ No newline at end of file diff --git a/irohc/Cargo.toml b/libiroh/Cargo.toml similarity index 69% rename from irohc/Cargo.toml rename to libiroh/Cargo.toml index d77de18a8d..eb16ec1958 100644 --- a/irohc/Cargo.toml +++ b/libiroh/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "irohc" +name = "libiroh" version = "0.1.0" edition = "2021" @@ -7,14 +7,18 @@ edition = "2021" strip = "symbols" [lib] -name="irohc" +name="libiroh" crate-type = ["cdylib"] +[features] +c-headers = ["safer-ffi/headers"] + [target.x86_64-apple-darwin] rustflags=["-C", "link-arg=-mmacosx-version-min=10.7"] strip = "symbols" [dependencies] +safer-ffi = { version = "0.0.10", features = ["proc_macros"] } anyhow = "1.0.69" iroh = { path = "../" } libc = "0.2.141" diff --git a/libiroh/c/Makefile b/libiroh/c/Makefile new file mode 100644 index 0000000000..905677f62a --- /dev/null +++ b/libiroh/c/Makefile @@ -0,0 +1,4 @@ + +build: + cargo build --release + cc main.c -o main -L ../target/release -l libiroh -l pthread -l dl \ No newline at end of file diff --git a/libiroh/c/libiroh.h b/libiroh/c/libiroh.h new file mode 100644 index 0000000000..95c94bcacd --- /dev/null +++ b/libiroh/c/libiroh.h @@ -0,0 +1,34 @@ +/*! \file */ +/******************************************* + * * + * File auto-generated by `::safer_ffi`. * + * * + * Do not manually edit this file. * + * * + *******************************************/ + +#ifndef __RUST_LIBIROH__ +#define __RUST_LIBIROH__ + +#ifdef __cplusplus +extern "C" { +#endif + + +#include +#include + +int32_t add_numbers ( + int32_t number1, + int32_t number2); + +uint32_t get_ticket ( + char const * ticket, + char const * out_path); + + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* __RUST_LIBIROH__ */ diff --git a/libiroh/c/main.c b/libiroh/c/main.c new file mode 100644 index 0000000000..55d8d0190e --- /dev/null +++ b/libiroh/c/main.c @@ -0,0 +1,16 @@ +#include +#include + +#include "libiroh.h" + +int main (int argc, char const * const argv[]) { + if (argc < 3) { + printf("Usage: %s \n", argv[0]); + return 1; + } + + const char *out_path = argv[1]; + const char *ticket = argv[2]; + get_ticket(ticket, out_path); + return EXIT_SUCCESS; +} \ No newline at end of file diff --git a/irohc/readme.md b/libiroh/readme.md similarity index 75% rename from irohc/readme.md rename to libiroh/readme.md index a72bc9c2f7..11b188e3bf 100644 --- a/irohc/readme.md +++ b/libiroh/readme.md @@ -1,6 +1,6 @@ # C bindings for Iroh -Running `cargo build --release` will produce a dynamic libaray. `irohc.h` is a handcrafted header file for that library. Should work on all platforms, I've only tested it on MacOS so far. +Running `cargo build --release` will produce a dynamic libaray. `libiroh.h` is a handcrafted header file for that library. Should work on all platforms, I've only tested it on MacOS so far. For builds targeting older versions of MacOS, build with with: `MACOSX_DEPLOYMENT_TARGET=10.7 && cargo build --target x86_64-apple-darwin --release`. diff --git a/irohc/src/lib.rs b/libiroh/src/lib.rs similarity index 82% rename from irohc/src/lib.rs rename to libiroh/src/lib.rs index 5c91bb7391..758d9c026e 100644 --- a/irohc/src/lib.rs +++ b/libiroh/src/lib.rs @@ -1,14 +1,14 @@ use std::{path::PathBuf, str::FromStr}; +use ::safer_ffi::prelude::*; use anyhow::{Context, Result}; use iroh::{get, provider::Ticket, Hash}; -use std::ffi::CStr; use tokio::runtime::Runtime; const MAX_CONCURRENT_DIALS: u8 = 16; -#[no_mangle] -pub extern "C" fn add_numbers(number1: i32, number2: i32) -> i32 { +#[ffi_export] +fn add_numbers(number1: i32, number2: i32) -> i32 { let result = std::panic::catch_unwind(|| { println!("Hello from rust!"); number1 + number2 @@ -20,29 +20,13 @@ pub extern "C" fn add_numbers(number1: i32, number2: i32) -> i32 { result.unwrap() } -#[no_mangle] -pub extern "C" fn get_ticket( - ticket: *const std::os::raw::c_char, - out_path: *const std::os::raw::c_char, -) -> u32 { +#[ffi_export] +fn get_ticket(ticket: char_p::Ref<'_>, out_path: char_p::Ref<'_>) -> u32 { let result = std::panic::catch_unwind(|| { - let tkt = unsafe { - assert!(!ticket.is_null()); - CStr::from_ptr(ticket) - }; - let tkt = tkt.to_str().unwrap(); - let tkt = tkt.parse::().unwrap(); - - let out_path = unsafe { - assert!(!ticket.is_null()); - CStr::from_ptr(out_path) - }; - let out_path = out_path.to_str().unwrap(); - let out_path = PathBuf::from_str(out_path).unwrap(); - println!("temp dir: {:?}", out_path); - + let ticket = ticket.to_str().parse::().unwrap(); + let out_path = PathBuf::from_str(out_path.to_str()).unwrap(); let rt = Runtime::new().unwrap(); - rt.block_on(get_ticket_internal(tkt, Some(out_path))) + rt.block_on(get_ticket_internal(ticket, Some(out_path))) .unwrap(); 0 }); @@ -140,6 +124,15 @@ async fn get_ticket_internal(ticket: Ticket, out: Option) -> Result<()> mod tests { use super::*; + /// The following test function is necessary for the header generation. + #[::safer_ffi::cfg_headers] + #[test] + fn generate_headers() -> ::std::io::Result<()> { + ::safer_ffi::headers::builder() + .to_file("c/libiroh.h")? + .generate() + } + #[test] fn add_numbers_test() { let result = add_numbers(2, 2); @@ -148,7 +141,8 @@ mod tests { #[test] fn get_ticket_test() { - let result = get_ticket(); - assert_eq!(result, 0); + // TODO + // let result = get_ticket(); + // assert_eq!(result, 0); } } From 39e02a96fa81c59d29cc4a9e1118c4a9ee121897 Mon Sep 17 00:00:00 2001 From: b5 Date: Thu, 4 May 2023 07:21:57 -0400 Subject: [PATCH 03/32] working on iOS support --- libiroh/Cargo.toml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/libiroh/Cargo.toml b/libiroh/Cargo.toml index eb16ec1958..9e893a12e9 100644 --- a/libiroh/Cargo.toml +++ b/libiroh/Cargo.toml @@ -2,20 +2,22 @@ name = "libiroh" version = "0.1.0" edition = "2021" +publish = false [profile.release] strip = "symbols" [lib] -name="libiroh" -crate-type = ["cdylib"] +name="iroh" +crate-type = ["staticlib", "cdylib"] [features] c-headers = ["safer-ffi/headers"] -[target.x86_64-apple-darwin] -rustflags=["-C", "link-arg=-mmacosx-version-min=10.7"] -strip = "symbols" +# needed when building for unreal engine... +# [target.x86_64-apple-darwin] +# rustflags=["-C", "link-arg=-mmacosx-version-min=10.7"] +# strip = "symbols" [dependencies] safer-ffi = { version = "0.0.10", features = ["proc_macros"] } From 151750f2ace51be4b7a79f482e34467772103516 Mon Sep 17 00:00:00 2001 From: b5 Date: Fri, 5 May 2023 14:22:29 -0400 Subject: [PATCH 04/32] WIP: iOS compilation --- libiroh/Cargo.toml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/libiroh/Cargo.toml b/libiroh/Cargo.toml index 9e893a12e9..5d67f12d45 100644 --- a/libiroh/Cargo.toml +++ b/libiroh/Cargo.toml @@ -3,6 +3,7 @@ name = "libiroh" version = "0.1.0" edition = "2021" publish = false +rust-version = "1.64" [profile.release] strip = "symbols" @@ -22,7 +23,7 @@ c-headers = ["safer-ffi/headers"] [dependencies] safer-ffi = { version = "0.0.10", features = ["proc_macros"] } anyhow = "1.0.69" -iroh = { path = "../" } +iroh = { path = "../", default-features = false } libc = "0.2.141" tempfile = "3.4" -tokio = "1.25.0" +tokio = "1.25.0" \ No newline at end of file From 6b72f06ad0e06092cdced0d5aa94cd416ab71a28 Mon Sep 17 00:00:00 2001 From: b5 Date: Fri, 5 May 2023 14:36:06 -0400 Subject: [PATCH 05/32] switch -> iroh-ffi --- {libiroh => iroh-ffi}/.gitignore | 0 {libiroh => iroh-ffi}/Cargo.toml | 7 ++++++- {libiroh => iroh-ffi}/c/Makefile | 0 {libiroh => iroh-ffi}/c/libiroh.h | 0 {libiroh => iroh-ffi}/c/main.c | 0 iroh-ffi/readme.md | 5 +++++ {libiroh => iroh-ffi}/src/lib.rs | 0 libiroh/readme.md | 8 -------- 8 files changed, 11 insertions(+), 9 deletions(-) rename {libiroh => iroh-ffi}/.gitignore (100%) rename {libiroh => iroh-ffi}/Cargo.toml (74%) rename {libiroh => iroh-ffi}/c/Makefile (100%) rename {libiroh => iroh-ffi}/c/libiroh.h (100%) rename {libiroh => iroh-ffi}/c/main.c (100%) create mode 100644 iroh-ffi/readme.md rename {libiroh => iroh-ffi}/src/lib.rs (100%) delete mode 100644 libiroh/readme.md diff --git a/libiroh/.gitignore b/iroh-ffi/.gitignore similarity index 100% rename from libiroh/.gitignore rename to iroh-ffi/.gitignore diff --git a/libiroh/Cargo.toml b/iroh-ffi/Cargo.toml similarity index 74% rename from libiroh/Cargo.toml rename to iroh-ffi/Cargo.toml index 5d67f12d45..fbd34ebb80 100644 --- a/libiroh/Cargo.toml +++ b/iroh-ffi/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "libiroh" +name = "iroh" version = "0.1.0" edition = "2021" publish = false @@ -15,6 +15,11 @@ crate-type = ["staticlib", "cdylib"] [features] c-headers = ["safer-ffi/headers"] +# ios build flags: +# export RUSTFLAGS="-C lto=on -C embed-bitcode=yes" +# export LIBRARY_PATH="/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/lib" +# cargo lipo --release + # needed when building for unreal engine... # [target.x86_64-apple-darwin] # rustflags=["-C", "link-arg=-mmacosx-version-min=10.7"] diff --git a/libiroh/c/Makefile b/iroh-ffi/c/Makefile similarity index 100% rename from libiroh/c/Makefile rename to iroh-ffi/c/Makefile diff --git a/libiroh/c/libiroh.h b/iroh-ffi/c/libiroh.h similarity index 100% rename from libiroh/c/libiroh.h rename to iroh-ffi/c/libiroh.h diff --git a/libiroh/c/main.c b/iroh-ffi/c/main.c similarity index 100% rename from libiroh/c/main.c rename to iroh-ffi/c/main.c diff --git a/iroh-ffi/readme.md b/iroh-ffi/readme.md new file mode 100644 index 0000000000..ed1e92c818 --- /dev/null +++ b/iroh-ffi/readme.md @@ -0,0 +1,5 @@ +# C bindings for Iroh + +Running `cargo build --release` will produce a dynamic library. + +For builds targeting older versions of MacOS, build with with: `MACOSX_DEPLOYMENT_TARGET=10.7 && cargo build --target x86_64-apple-darwin --release`. \ No newline at end of file diff --git a/libiroh/src/lib.rs b/iroh-ffi/src/lib.rs similarity index 100% rename from libiroh/src/lib.rs rename to iroh-ffi/src/lib.rs diff --git a/libiroh/readme.md b/libiroh/readme.md deleted file mode 100644 index 11b188e3bf..0000000000 --- a/libiroh/readme.md +++ /dev/null @@ -1,8 +0,0 @@ -# C bindings for Iroh - -Running `cargo build --release` will produce a dynamic libaray. `libiroh.h` is a handcrafted header file for that library. Should work on all platforms, I've only tested it on MacOS so far. - -For builds targeting older versions of MacOS, build with with: `MACOSX_DEPLOYMENT_TARGET=10.7 && cargo build --target x86_64-apple-darwin --release`. - -## Why not use [cbindgen](https://github.com/eqrion/cbindgen)? -I (b5) have tried, not much success so far. If someone else wants to give it a go, be my guest. \ No newline at end of file From dec7a5f5407a6ff9c4f6aecead4dee7697b14dd8 Mon Sep 17 00:00:00 2001 From: b5 Date: Fri, 5 May 2023 15:47:56 -0400 Subject: [PATCH 06/32] update for get FSM API --- iroh-ffi/Cargo.toml | 6 +- iroh-ffi/src/lib.rs | 309 ++++++++++++++++++++++++++++++++----------- iroh-ffi/src/util.rs | 90 +++++++++++++ 3 files changed, 330 insertions(+), 75 deletions(-) create mode 100644 iroh-ffi/src/util.rs diff --git a/iroh-ffi/Cargo.toml b/iroh-ffi/Cargo.toml index fbd34ebb80..6b2266471f 100644 --- a/iroh-ffi/Cargo.toml +++ b/iroh-ffi/Cargo.toml @@ -28,7 +28,11 @@ c-headers = ["safer-ffi/headers"] [dependencies] safer-ffi = { version = "0.0.10", features = ["proc_macros"] } anyhow = "1.0.69" +blake3 = "1.3.3" iroh = { path = "../", default-features = false } +indicatif = { version = "0.17", features = ["tokio"] } +data-encoding = { version = "2.3.3" } +multibase = { version = "0.9.1"} libc = "0.2.141" tempfile = "3.4" -tokio = "1.25.0" \ No newline at end of file +tokio = "1.25.0" diff --git a/iroh-ffi/src/lib.rs b/iroh-ffi/src/lib.rs index 758d9c026e..814485f289 100644 --- a/iroh-ffi/src/lib.rs +++ b/iroh-ffi/src/lib.rs @@ -2,9 +2,21 @@ use std::{path::PathBuf, str::FromStr}; use ::safer_ffi::prelude::*; use anyhow::{Context, Result}; -use iroh::{get, provider::Ticket, Hash}; +use indicatif::{HumanBytes, HumanDuration}; +use tokio::io::AsyncWriteExt; use tokio::runtime::Runtime; +use crate::util::Blake3Cid; +use iroh::blobs::{Blob, Collection}; +use iroh::get::get_response_machine::{ConnectedNext, EndBlobNext}; +use iroh::get::{self, get_data_path, get_missing_range, get_missing_ranges, pathbuf_from_name}; +use iroh::protocol::{AuthToken, GetRequest}; +use iroh::provider::Ticket; +use iroh::tokio_util::SeekOptimized; +use iroh::{Hash, PeerId}; + +mod util; + const MAX_CONCURRENT_DIALS: u8 = 16; #[ffi_export] @@ -20,14 +32,59 @@ fn add_numbers(number1: i32, number2: i32) -> i32 { result.unwrap() } +#[ffi_export] +fn get( + hash: char_p::Ref<'_>, + auth_token: char_p::Ref<'_>, + peer: char_p::Ref<'_>, + peer_addr: char_p::Ref<'_>, + out_path: char_p::Ref<'_>, +) -> u32 { + let result = std::panic::catch_unwind(|| { + let hash = hash.to_str().parse::().unwrap(); + let token = auth_token.to_str().parse::().unwrap(); + let peer = peer.to_str().parse::().unwrap(); + let peer_addr = peer_addr.to_str().parse().unwrap(); + let out_path = PathBuf::from_str(out_path.to_str()).unwrap(); + let rt = Runtime::new().unwrap(); + + rt.block_on(get_to_dir( + GetInteractive::Hash { + hash, + opts: get::Options { + peer_id: Some(peer), + addr: peer_addr, + keylog: false, + }, + token, + single: false, + }, + out_path, + )) + .unwrap(); + 0 + }); + if result.is_err() { + eprintln!("error: rust panicked"); + return 1; + } + result.unwrap() +} + #[ffi_export] fn get_ticket(ticket: char_p::Ref<'_>, out_path: char_p::Ref<'_>) -> u32 { let result = std::panic::catch_unwind(|| { let ticket = ticket.to_str().parse::().unwrap(); let out_path = PathBuf::from_str(out_path.to_str()).unwrap(); let rt = Runtime::new().unwrap(); - rt.block_on(get_ticket_internal(ticket, Some(out_path))) - .unwrap(); + rt.block_on(get_to_dir( + GetInteractive::Ticket { + ticket, + keylog: false, + }, + out_path, + )) + .unwrap(); 0 }); if result.is_err() { @@ -37,86 +94,190 @@ fn get_ticket(ticket: char_p::Ref<'_>, out_path: char_p::Ref<'_>) -> u32 { result.unwrap() } -async fn get_ticket_internal(ticket: Ticket, out: Option) -> Result<()> { - let on_connected = || async move { - println!("connected!"); - Ok(()) +#[derive(Debug)] +enum GetInteractive { + Ticket { + ticket: Ticket, + keylog: bool, + }, + Hash { + hash: Hash, + opts: get::Options, + token: AuthToken, + single: bool, + }, +} + +impl GetInteractive { + fn hash(&self) -> Hash { + match self { + GetInteractive::Ticket { ticket, .. } => ticket.hash(), + GetInteractive::Hash { hash, .. } => *hash, + } + } + + fn single(&self) -> bool { + match self { + GetInteractive::Ticket { .. } => false, + GetInteractive::Hash { single, .. } => *single, + } + } +} + +/// Get into a file or directory +async fn get_to_dir(get: GetInteractive, out_dir: PathBuf) -> Result<()> { + let hash = get.hash(); + let single = get.single(); + println!("Fetching: {}", Blake3Cid::new(hash)); + println!("[1/3] Connecting ..."); + + let temp_dir = out_dir.join(".iroh-tmp"); + let (query, collection) = if single { + let name = Blake3Cid::new(hash).to_string(); + let query = get_missing_range(&get.hash(), name.as_str(), &temp_dir, &out_dir)?; + (query, vec![Blob { hash, name }]) + } else { + let (query, collection) = get_missing_ranges(get.hash(), &out_dir, &temp_dir)?; + ( + query, + collection.map(|x| x.into_inner()).unwrap_or_default(), + ) + }; + + let init_download_progress = |count: u64, missing_bytes: u64| { + println!("[3/3] Downloading ..."); + println!( + " {} file(s) with total transfer size {}", + count, + HumanBytes(missing_bytes) + ); + }; + + // collection info, in case we won't get a callback with is_root + let collection_info = if collection.is_empty() { + None + } else { + Some((collection.len() as u64, 0)) + }; + + let request = GetRequest::new(get.hash(), query).into(); + let response = match get { + GetInteractive::Ticket { ticket, keylog } => { + get::run_ticket(&ticket, request, keylog, MAX_CONCURRENT_DIALS).await? + } + GetInteractive::Hash { opts, token, .. } => get::run(request, token, opts).await?, }; - let on_collection = |collection: &iroh::blobs::Collection| { - // let name = collection.name().to_string(); - let total_entries = collection.total_entries(); - let size = collection.total_blobs_size(); - async move { - println!( - "downloading collection containing {total_entries} entries totalling {size} bytes" - ); - Ok(()) + let connected = response.next().await?; + println!("[2/3] Requesting ..."); + if let Some((count, missing_bytes)) = collection_info { + init_download_progress(count, missing_bytes); + } + let (mut next, collection) = match connected.next().await? { + ConnectedNext::StartCollection(curr) => { + tokio::fs::create_dir_all(&temp_dir) + .await + .context("unable to create directory {temp_dir}")?; + tokio::fs::create_dir_all(&out_dir) + .await + .context("Unable to create directory {out_dir}")?; + let curr = curr.next(); + let (curr, size) = curr.next().await?; + let mut collection_data = Vec::with_capacity(size as usize); + let curr = curr.concatenate(&mut collection_data, |_, _| {}).await?; + let collection = Collection::from_bytes(&collection_data)?; + init_download_progress(collection.total_entries(), collection.total_blobs_size()); + tokio::fs::write(get_data_path(&temp_dir, hash), collection_data).await?; + (curr.next(), collection.into_inner()) } + ConnectedNext::StartChild(start_child) => { + (EndBlobNext::MoreChildren(start_child), collection) + } + ConnectedNext::Closing(finish) => (EndBlobNext::Closing(finish), collection), }; + // read all the children + let finishing = loop { + let start = match next { + EndBlobNext::MoreChildren(sc) => sc, + EndBlobNext::Closing(finish) => break finish, + }; + let child_offset = start.child_offset() as usize; + let blob = match collection.get(child_offset) { + Some(blob) => blob, + None => break start.finish(), + }; - let on_blob = |hash: Hash, mut reader, name: String| { - let out = &out; - async move { - let name = if name.is_empty() { - hash.to_string() - } else { - name - }; + let hash = blob.hash; + let name = &blob.name; + let name = if name.is_empty() { + PathBuf::from(hash.to_string()) + } else { + pathbuf_from_name(name) + }; + // pb.set_message(format!("Receiving '{}'...", name.display())); + // pb.reset(); + let header = start.next(blob.hash); - if let Some(ref outpath) = out { - tokio::fs::create_dir_all(outpath) - .await - .context("Unable to create directory {outpath}")?; - let dirpath = std::path::PathBuf::from(outpath); - let filepath = dirpath.join(name); - - // Create temp file - let (temp_file, dup) = tokio::task::spawn_blocking(|| { - let temp_file = tempfile::Builder::new() - .prefix("iroh-tmp-") - .tempfile_in(dirpath) - .context("Failed to create temporary output file")?; - let dup = temp_file.as_file().try_clone()?; - Ok::<_, anyhow::Error>((temp_file, dup)) - }) - .await??; - - let file = tokio::fs::File::from_std(dup); - let mut file_buf = tokio::io::BufWriter::new(file); - tokio::io::copy(&mut reader, &mut file_buf).await?; - - // Rename temp file, to target name - let filepath2 = filepath.clone(); - if let Some(parent) = filepath2.parent() { - tokio::fs::create_dir_all(parent) - .await - .context("Unable to create directory {parent}")?; - } - println!("writing {:?}", &filepath2); - tokio::task::spawn_blocking(|| temp_file.persist(filepath2)) - .await? - .context("Failed to write output file")?; + let curr = { + let final_path = out_dir.join(&name); + let tempname = blake3::Hash::from(hash).to_hex(); + let data_path = temp_dir.join(format!("{}.data.part", tempname)); + let outboard_path = temp_dir.join(format!("{}.outboard.part", tempname)); + let data_file = tokio::fs::OpenOptions::new() + .write(true) + .create(true) + .open(&data_path) + .await?; + let mut data_file = SeekOptimized::new(data_file).into(); + let (curr, size) = header.next().await?; + // pb.set_length(size); + let mut outboard_file = if size > 0 { + let outboard_file = tokio::fs::OpenOptions::new() + .write(true) + .create(true) + .open(&outboard_path) + .await?; + let outboard_file = SeekOptimized::new(outboard_file).into(); + Some(outboard_file) } else { - // Write to OUT_WRITER - let mut stdout = tokio::io::stdout(); - tokio::io::copy(&mut reader, &mut stdout).await?; + None + }; + let on_write = |_offset, _size| { + // println!("offset: {}/{}", offset, pb.length().unwrap()); + // pb.set_position(offset); + }; + let curr = curr + .write_all_with_outboard(&mut outboard_file, &mut data_file, on_write) + .await?; + tokio::fs::create_dir_all( + final_path + .parent() + .context("final path should have parent")?, + ) + .await?; + // Flush the data file first, it is the only thing that matters at this point + data_file.into_inner().shutdown().await?; + // Rename temp file, to target name + // once this is done, the file is considered complete + tokio::fs::rename(data_path, final_path).await?; + if let Some(outboard_file) = outboard_file.take() { + // not sure if we have to do this + outboard_file.into_inner().shutdown().await?; + // delete the outboard file + tokio::fs::remove_file(outboard_path).await?; } - - Ok(reader) - } + curr + }; + next = curr.next(); }; + let stats = finishing.next().await?; + tokio::fs::remove_dir_all(temp_dir).await?; + println!( + "Transferred {} in {}, {}/s", + HumanBytes(stats.bytes_read), + HumanDuration(stats.elapsed), + HumanBytes((stats.bytes_read as f64 / stats.elapsed.as_secs_f64()) as u64) + ); - let stats = get::run_ticket( - &ticket, - false, - MAX_CONCURRENT_DIALS, - on_connected, - on_collection, - on_blob, - ) - .await?; - // let stats = get::run(hash, token, opts, on_connected, on_collection, on_blob).await?; - println!("Done in {:?}", stats.elapsed); Ok(()) } diff --git a/iroh-ffi/src/util.rs b/iroh-ffi/src/util.rs new file mode 100644 index 0000000000..a046254902 --- /dev/null +++ b/iroh-ffi/src/util.rs @@ -0,0 +1,90 @@ +//! Utility functions and types. +use std::{fmt, str::FromStr}; + +use anyhow::Result; +use iroh::Hash; + +#[repr(transparent)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Blake3Cid(pub Hash); + +const CID_PREFIX: [u8; 4] = [ + 0x01, // version + 0x55, // raw codec + 0x1e, // hash function, blake3 + 0x20, // hash size, 32 bytes +]; + +impl Blake3Cid { + pub fn new(hash: Hash) -> Self { + Blake3Cid(hash) + } + + // pub fn as_hash(&self) -> &Hash { + // &self.0 + // } + + pub fn as_bytes(&self) -> [u8; 36] { + let hash: [u8; 32] = self.0.as_ref().try_into().unwrap(); + let mut res = [0u8; 36]; + res[0..4].copy_from_slice(&CID_PREFIX); + res[4..36].copy_from_slice(&hash); + res + } + + pub fn from_bytes(bytes: &[u8]) -> anyhow::Result { + anyhow::ensure!( + bytes.len() == 36, + "invalid cid length, expected 36, got {}", + bytes.len() + ); + anyhow::ensure!(bytes[0..4] == CID_PREFIX, "invalid cid prefix"); + let mut hash = [0u8; 32]; + hash.copy_from_slice(&bytes[4..36]); + Ok(Blake3Cid(Hash::from(hash))) + } +} + +impl fmt::Display for Blake3Cid { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // result will be 58 bytes plus prefix + let mut res = [b'b'; 59]; + // write the encoded bytes + data_encoding::BASE32_NOPAD.encode_mut(&self.as_bytes(), &mut res[1..]); + // convert to string, this is guaranteed to succeed + let t = std::str::from_utf8_mut(res.as_mut()).unwrap(); + // hack since data_encoding doesn't have BASE32LOWER_NOPAD as a const + t.make_ascii_lowercase(); + // write the str, no allocations + f.write_str(t) + } +} + +impl FromStr for Blake3Cid { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + let sb = s.as_bytes(); + if sb.len() == 59 && sb[0] == b'b' { + // this is a base32 encoded cid, we can decode it directly + let mut t = [0u8; 58]; + t.copy_from_slice(&sb[1..]); + // hack since data_encoding doesn't have BASE32LOWER_NOPAD as a const + std::str::from_utf8_mut(t.as_mut()) + .unwrap() + .make_ascii_uppercase(); + // decode the bytes + let mut res = [0u8; 36]; + data_encoding::BASE32_NOPAD + .decode_mut(&t, &mut res) + .map_err(|_e| anyhow::anyhow!("invalid base32"))?; + // convert to cid, this will check the prefix + Self::from_bytes(&res) + } else { + // if we want to support all the weird multibase prefixes, we have no choice + // but to use the multibase crate + let (_base, bytes) = multibase::decode(s)?; + Self::from_bytes(bytes.as_ref()) + } + } +} From 34f4310d5094029b798830092e139a26f943257f Mon Sep 17 00:00:00 2001 From: b5 Date: Sat, 6 May 2023 10:22:08 -0400 Subject: [PATCH 07/32] rename iroh lib to iroh_ffi --- iroh-ffi/Cargo.toml | 2 +- iroh-ffi/{c/libiroh.h => iroh.h} | 13 ++++++++++--- iroh-ffi/src/lib.rs | 2 +- 3 files changed, 12 insertions(+), 5 deletions(-) rename iroh-ffi/{c/libiroh.h => iroh.h} (73%) diff --git a/iroh-ffi/Cargo.toml b/iroh-ffi/Cargo.toml index 6b2266471f..c581f99a08 100644 --- a/iroh-ffi/Cargo.toml +++ b/iroh-ffi/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "iroh" +name = "iroh_ffi" version = "0.1.0" edition = "2021" publish = false diff --git a/iroh-ffi/c/libiroh.h b/iroh-ffi/iroh.h similarity index 73% rename from iroh-ffi/c/libiroh.h rename to iroh-ffi/iroh.h index 95c94bcacd..7ac4713393 100644 --- a/iroh-ffi/c/libiroh.h +++ b/iroh-ffi/iroh.h @@ -7,8 +7,8 @@ * * *******************************************/ -#ifndef __RUST_LIBIROH__ -#define __RUST_LIBIROH__ +#ifndef __RUST_IROH__ +#define __RUST_IROH__ #ifdef __cplusplus extern "C" { @@ -22,6 +22,13 @@ int32_t add_numbers ( int32_t number1, int32_t number2); +uint32_t get ( + char const * hash, + char const * auth_token, + char const * peer, + char const * peer_addr, + char const * out_path); + uint32_t get_ticket ( char const * ticket, char const * out_path); @@ -31,4 +38,4 @@ uint32_t get_ticket ( } /* extern "C" */ #endif -#endif /* __RUST_LIBIROH__ */ +#endif /* __RUST_IROH__ */ diff --git a/iroh-ffi/src/lib.rs b/iroh-ffi/src/lib.rs index 814485f289..4680a957c2 100644 --- a/iroh-ffi/src/lib.rs +++ b/iroh-ffi/src/lib.rs @@ -290,7 +290,7 @@ mod tests { #[test] fn generate_headers() -> ::std::io::Result<()> { ::safer_ffi::headers::builder() - .to_file("c/libiroh.h")? + .to_file("iroh.h")? .generate() } From d76f48e192d57071f5c398a142565ef365082663 Mon Sep 17 00:00:00 2001 From: b5 Date: Mon, 15 May 2023 22:25:50 -0400 Subject: [PATCH 08/32] working on swift compilation --- iroh-ffi/Cargo.toml | 2 +- iroh-ffi/{iroh.h => libiroh.h} | 7 +++--- iroh-ffi/src/lib.rs | 39 +++++++--------------------------- 3 files changed, 12 insertions(+), 36 deletions(-) rename iroh-ffi/{iroh.h => libiroh.h} (87%) diff --git a/iroh-ffi/Cargo.toml b/iroh-ffi/Cargo.toml index c581f99a08..fa275295ab 100644 --- a/iroh-ffi/Cargo.toml +++ b/iroh-ffi/Cargo.toml @@ -10,7 +10,7 @@ strip = "symbols" [lib] name="iroh" -crate-type = ["staticlib", "cdylib"] +crate-type = ["staticlib"] [features] c-headers = ["safer-ffi/headers"] diff --git a/iroh-ffi/iroh.h b/iroh-ffi/libiroh.h similarity index 87% rename from iroh-ffi/iroh.h rename to iroh-ffi/libiroh.h index 7ac4713393..23bc301a2f 100644 --- a/iroh-ffi/iroh.h +++ b/iroh-ffi/libiroh.h @@ -7,8 +7,8 @@ * * *******************************************/ -#ifndef __RUST_IROH__ -#define __RUST_IROH__ +#ifndef __RUST_IROH_FFI__ +#define __RUST_IROH_FFI__ #ifdef __cplusplus extern "C" { @@ -24,7 +24,6 @@ int32_t add_numbers ( uint32_t get ( char const * hash, - char const * auth_token, char const * peer, char const * peer_addr, char const * out_path); @@ -38,4 +37,4 @@ uint32_t get_ticket ( } /* extern "C" */ #endif -#endif /* __RUST_IROH__ */ +#endif /* __RUST_IROH_FFI__ */ diff --git a/iroh-ffi/src/lib.rs b/iroh-ffi/src/lib.rs index 4680a957c2..c3a42301b0 100644 --- a/iroh-ffi/src/lib.rs +++ b/iroh-ffi/src/lib.rs @@ -10,7 +10,7 @@ use crate::util::Blake3Cid; use iroh::blobs::{Blob, Collection}; use iroh::get::get_response_machine::{ConnectedNext, EndBlobNext}; use iroh::get::{self, get_data_path, get_missing_range, get_missing_ranges, pathbuf_from_name}; -use iroh::protocol::{AuthToken, GetRequest}; +use iroh::protocol::{GetRequest}; use iroh::provider::Ticket; use iroh::tokio_util::SeekOptimized; use iroh::{Hash, PeerId}; @@ -20,29 +20,14 @@ mod util; const MAX_CONCURRENT_DIALS: u8 = 16; #[ffi_export] -fn add_numbers(number1: i32, number2: i32) -> i32 { - let result = std::panic::catch_unwind(|| { - println!("Hello from rust!"); - number1 + number2 - }); - if result.is_err() { - eprintln!("error: rust panicked"); - return -1; - } - result.unwrap() -} - -#[ffi_export] -fn get( +fn iroh_get( hash: char_p::Ref<'_>, - auth_token: char_p::Ref<'_>, peer: char_p::Ref<'_>, peer_addr: char_p::Ref<'_>, out_path: char_p::Ref<'_>, ) -> u32 { let result = std::panic::catch_unwind(|| { let hash = hash.to_str().parse::().unwrap(); - let token = auth_token.to_str().parse::().unwrap(); let peer = peer.to_str().parse::().unwrap(); let peer_addr = peer_addr.to_str().parse().unwrap(); let out_path = PathBuf::from_str(out_path.to_str()).unwrap(); @@ -56,7 +41,6 @@ fn get( addr: peer_addr, keylog: false, }, - token, single: false, }, out_path, @@ -72,7 +56,7 @@ fn get( } #[ffi_export] -fn get_ticket(ticket: char_p::Ref<'_>, out_path: char_p::Ref<'_>) -> u32 { +fn iroh_get_ticket(ticket: char_p::Ref<'_>, out_path: char_p::Ref<'_>) -> u32 { let result = std::panic::catch_unwind(|| { let ticket = ticket.to_str().parse::().unwrap(); let out_path = PathBuf::from_str(out_path.to_str()).unwrap(); @@ -103,7 +87,6 @@ enum GetInteractive { Hash { hash: Hash, opts: get::Options, - token: AuthToken, single: bool, }, } @@ -165,7 +148,7 @@ async fn get_to_dir(get: GetInteractive, out_dir: PathBuf) -> Result<()> { GetInteractive::Ticket { ticket, keylog } => { get::run_ticket(&ticket, request, keylog, MAX_CONCURRENT_DIALS).await? } - GetInteractive::Hash { opts, token, .. } => get::run(request, token, opts).await?, + GetInteractive::Hash { opts, .. } => get::run(request, opts).await?, }; let connected = response.next().await?; println!("[2/3] Requesting ..."); @@ -173,7 +156,7 @@ async fn get_to_dir(get: GetInteractive, out_dir: PathBuf) -> Result<()> { init_download_progress(count, missing_bytes); } let (mut next, collection) = match connected.next().await? { - ConnectedNext::StartCollection(curr) => { + ConnectedNext::StartRoot(curr) => { tokio::fs::create_dir_all(&temp_dir) .await .context("unable to create directory {temp_dir}")?; @@ -181,9 +164,7 @@ async fn get_to_dir(get: GetInteractive, out_dir: PathBuf) -> Result<()> { .await .context("Unable to create directory {out_dir}")?; let curr = curr.next(); - let (curr, size) = curr.next().await?; - let mut collection_data = Vec::with_capacity(size as usize); - let curr = curr.concatenate(&mut collection_data, |_, _| {}).await?; + let (curr, collection_data) = curr.concatenate_into_vec().await?; let collection = Collection::from_bytes(&collection_data)?; init_download_progress(collection.total_entries(), collection.total_blobs_size()); tokio::fs::write(get_data_path(&temp_dir, hash), collection_data).await?; @@ -241,12 +222,8 @@ async fn get_to_dir(get: GetInteractive, out_dir: PathBuf) -> Result<()> { } else { None }; - let on_write = |_offset, _size| { - // println!("offset: {}/{}", offset, pb.length().unwrap()); - // pb.set_position(offset); - }; let curr = curr - .write_all_with_outboard(&mut outboard_file, &mut data_file, on_write) + .write_all_with_outboard(&mut outboard_file, &mut data_file) .await?; tokio::fs::create_dir_all( final_path @@ -290,7 +267,7 @@ mod tests { #[test] fn generate_headers() -> ::std::io::Result<()> { ::safer_ffi::headers::builder() - .to_file("iroh.h")? + .to_file("libiroh.h")? .generate() } From c998696135ebe73c854376401090a09b57c497d2 Mon Sep 17 00:00:00 2001 From: b5 Date: Mon, 15 May 2023 23:32:09 -0400 Subject: [PATCH 09/32] prefix ffi bindings with 'iroh_' --- iroh-ffi/libiroh.h | 8 ++------ iroh-ffi/src/lib.rs | 7 ------- 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/iroh-ffi/libiroh.h b/iroh-ffi/libiroh.h index 23bc301a2f..d39c30ee04 100644 --- a/iroh-ffi/libiroh.h +++ b/iroh-ffi/libiroh.h @@ -18,17 +18,13 @@ extern "C" { #include #include -int32_t add_numbers ( - int32_t number1, - int32_t number2); - -uint32_t get ( +uint32_t iroh_get ( char const * hash, char const * peer, char const * peer_addr, char const * out_path); -uint32_t get_ticket ( +uint32_t iroh_get_ticket ( char const * ticket, char const * out_path); diff --git a/iroh-ffi/src/lib.rs b/iroh-ffi/src/lib.rs index c3a42301b0..898ef63e6f 100644 --- a/iroh-ffi/src/lib.rs +++ b/iroh-ffi/src/lib.rs @@ -260,7 +260,6 @@ async fn get_to_dir(get: GetInteractive, out_dir: PathBuf) -> Result<()> { #[cfg(test)] mod tests { - use super::*; /// The following test function is necessary for the header generation. #[::safer_ffi::cfg_headers] @@ -271,12 +270,6 @@ mod tests { .generate() } - #[test] - fn add_numbers_test() { - let result = add_numbers(2, 2); - assert_eq!(result, 4); - } - #[test] fn get_ticket_test() { // TODO From f3e9ec03f31e05bf91b930ad04043f0153d23495 Mon Sep 17 00:00:00 2001 From: b5 Date: Fri, 19 May 2023 16:37:34 -0400 Subject: [PATCH 10/32] update c test --- iroh-ffi/Cargo.toml | 2 +- iroh-ffi/c/Makefile | 2 +- iroh-ffi/c/libiroh.h | 36 ++++++++++++++++++++++++++++++++++++ iroh-ffi/c/main.c | 2 +- 4 files changed, 39 insertions(+), 3 deletions(-) create mode 100644 iroh-ffi/c/libiroh.h diff --git a/iroh-ffi/Cargo.toml b/iroh-ffi/Cargo.toml index fa275295ab..c581f99a08 100644 --- a/iroh-ffi/Cargo.toml +++ b/iroh-ffi/Cargo.toml @@ -10,7 +10,7 @@ strip = "symbols" [lib] name="iroh" -crate-type = ["staticlib"] +crate-type = ["staticlib", "cdylib"] [features] c-headers = ["safer-ffi/headers"] diff --git a/iroh-ffi/c/Makefile b/iroh-ffi/c/Makefile index 905677f62a..ed0f2bd125 100644 --- a/iroh-ffi/c/Makefile +++ b/iroh-ffi/c/Makefile @@ -1,4 +1,4 @@ build: cargo build --release - cc main.c -o main -L ../target/release -l libiroh -l pthread -l dl \ No newline at end of file + cc main.c -o main -L ../target/release -l iroh -l pthread -l dl \ No newline at end of file diff --git a/iroh-ffi/c/libiroh.h b/iroh-ffi/c/libiroh.h new file mode 100644 index 0000000000..d39c30ee04 --- /dev/null +++ b/iroh-ffi/c/libiroh.h @@ -0,0 +1,36 @@ +/*! \file */ +/******************************************* + * * + * File auto-generated by `::safer_ffi`. * + * * + * Do not manually edit this file. * + * * + *******************************************/ + +#ifndef __RUST_IROH_FFI__ +#define __RUST_IROH_FFI__ + +#ifdef __cplusplus +extern "C" { +#endif + + +#include +#include + +uint32_t iroh_get ( + char const * hash, + char const * peer, + char const * peer_addr, + char const * out_path); + +uint32_t iroh_get_ticket ( + char const * ticket, + char const * out_path); + + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* __RUST_IROH_FFI__ */ diff --git a/iroh-ffi/c/main.c b/iroh-ffi/c/main.c index 55d8d0190e..50e5290241 100644 --- a/iroh-ffi/c/main.c +++ b/iroh-ffi/c/main.c @@ -11,6 +11,6 @@ int main (int argc, char const * const argv[]) { const char *out_path = argv[1]; const char *ticket = argv[2]; - get_ticket(ticket, out_path); + iroh_get_ticket(ticket, out_path); return EXIT_SUCCESS; } \ No newline at end of file From 450e9694c3fc62884a66f3bf1b214d89c943b50a Mon Sep 17 00:00:00 2001 From: b5 Date: Mon, 22 May 2023 15:10:04 -0400 Subject: [PATCH 11/32] refactor: cleanup makefile, Cargo.toml --- iroh-ffi/.gitignore | 3 ++- iroh-ffi/Cargo.toml | 10 ---------- iroh-ffi/c/Makefile | 8 ++++++-- iroh-ffi/c/libiroh.h | 36 ------------------------------------ iroh-ffi/c/main.c | 2 +- iroh-ffi/libiroh.h | 36 ------------------------------------ iroh-ffi/src/lib.rs | 4 ++-- 7 files changed, 11 insertions(+), 88 deletions(-) delete mode 100644 iroh-ffi/c/libiroh.h delete mode 100644 iroh-ffi/libiroh.h diff --git a/iroh-ffi/.gitignore b/iroh-ffi/.gitignore index 7fb54d769e..d3988d0919 100644 --- a/iroh-ffi/.gitignore +++ b/iroh-ffi/.gitignore @@ -1,3 +1,4 @@ /target /Cargo.lock -/c/main \ No newline at end of file +main +iroh.h \ No newline at end of file diff --git a/iroh-ffi/Cargo.toml b/iroh-ffi/Cargo.toml index c581f99a08..042db5b9b9 100644 --- a/iroh-ffi/Cargo.toml +++ b/iroh-ffi/Cargo.toml @@ -15,16 +15,6 @@ crate-type = ["staticlib", "cdylib"] [features] c-headers = ["safer-ffi/headers"] -# ios build flags: -# export RUSTFLAGS="-C lto=on -C embed-bitcode=yes" -# export LIBRARY_PATH="/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/lib" -# cargo lipo --release - -# needed when building for unreal engine... -# [target.x86_64-apple-darwin] -# rustflags=["-C", "link-arg=-mmacosx-version-min=10.7"] -# strip = "symbols" - [dependencies] safer-ffi = { version = "0.0.10", features = ["proc_macros"] } anyhow = "1.0.69" diff --git a/iroh-ffi/c/Makefile b/iroh-ffi/c/Makefile index ed0f2bd125..cf618124c8 100644 --- a/iroh-ffi/c/Makefile +++ b/iroh-ffi/c/Makefile @@ -1,4 +1,8 @@ -build: +build: iroh.h cargo build --release - cc main.c -o main -L ../target/release -l iroh -l pthread -l dl \ No newline at end of file + $(CC) main.c -o main -L ../target/release -l iroh -l pthread -l dl + +iroh.h: + cargo test --features c-headers -- generate_headers + cp ../iroh.h iroh.h diff --git a/iroh-ffi/c/libiroh.h b/iroh-ffi/c/libiroh.h deleted file mode 100644 index d39c30ee04..0000000000 --- a/iroh-ffi/c/libiroh.h +++ /dev/null @@ -1,36 +0,0 @@ -/*! \file */ -/******************************************* - * * - * File auto-generated by `::safer_ffi`. * - * * - * Do not manually edit this file. * - * * - *******************************************/ - -#ifndef __RUST_IROH_FFI__ -#define __RUST_IROH_FFI__ - -#ifdef __cplusplus -extern "C" { -#endif - - -#include -#include - -uint32_t iroh_get ( - char const * hash, - char const * peer, - char const * peer_addr, - char const * out_path); - -uint32_t iroh_get_ticket ( - char const * ticket, - char const * out_path); - - -#ifdef __cplusplus -} /* extern "C" */ -#endif - -#endif /* __RUST_IROH_FFI__ */ diff --git a/iroh-ffi/c/main.c b/iroh-ffi/c/main.c index 50e5290241..4348e909f4 100644 --- a/iroh-ffi/c/main.c +++ b/iroh-ffi/c/main.c @@ -1,7 +1,7 @@ #include #include -#include "libiroh.h" +#include "iroh.h" int main (int argc, char const * const argv[]) { if (argc < 3) { diff --git a/iroh-ffi/libiroh.h b/iroh-ffi/libiroh.h deleted file mode 100644 index d39c30ee04..0000000000 --- a/iroh-ffi/libiroh.h +++ /dev/null @@ -1,36 +0,0 @@ -/*! \file */ -/******************************************* - * * - * File auto-generated by `::safer_ffi`. * - * * - * Do not manually edit this file. * - * * - *******************************************/ - -#ifndef __RUST_IROH_FFI__ -#define __RUST_IROH_FFI__ - -#ifdef __cplusplus -extern "C" { -#endif - - -#include -#include - -uint32_t iroh_get ( - char const * hash, - char const * peer, - char const * peer_addr, - char const * out_path); - -uint32_t iroh_get_ticket ( - char const * ticket, - char const * out_path); - - -#ifdef __cplusplus -} /* extern "C" */ -#endif - -#endif /* __RUST_IROH_FFI__ */ diff --git a/iroh-ffi/src/lib.rs b/iroh-ffi/src/lib.rs index 898ef63e6f..04e78d5193 100644 --- a/iroh-ffi/src/lib.rs +++ b/iroh-ffi/src/lib.rs @@ -10,7 +10,7 @@ use crate::util::Blake3Cid; use iroh::blobs::{Blob, Collection}; use iroh::get::get_response_machine::{ConnectedNext, EndBlobNext}; use iroh::get::{self, get_data_path, get_missing_range, get_missing_ranges, pathbuf_from_name}; -use iroh::protocol::{GetRequest}; +use iroh::protocol::GetRequest; use iroh::provider::Ticket; use iroh::tokio_util::SeekOptimized; use iroh::{Hash, PeerId}; @@ -266,7 +266,7 @@ mod tests { #[test] fn generate_headers() -> ::std::io::Result<()> { ::safer_ffi::headers::builder() - .to_file("libiroh.h")? + .to_file("iroh.h")? .generate() } From 8864622216974df06cf6883282aaad197ff66be6 Mon Sep 17 00:00:00 2001 From: b5 Date: Thu, 13 Jul 2023 13:27:21 -0400 Subject: [PATCH 12/32] WIP --- Cargo.lock | 119 ++++++++++ Cargo.toml | 5 +- iroh-ffi/.gitignore | 4 - iroh-ffi/Cargo.toml | 16 +- iroh-ffi/src/error.rs | 70 ++++++ iroh-ffi/src/get.rs | 0 iroh-ffi/src/lib.rs | 509 +++++++++++++++++++++--------------------- iroh-ffi/src/mod.rs | 16 ++ iroh-ffi/src/node.rs | 66 ++++++ iroh-ffi/src/util.rs | 160 ++++++------- 10 files changed, 617 insertions(+), 348 deletions(-) delete mode 100644 iroh-ffi/.gitignore create mode 100644 iroh-ffi/src/error.rs create mode 100644 iroh-ffi/src/get.rs create mode 100644 iroh-ffi/src/mod.rs create mode 100644 iroh-ffi/src/node.rs diff --git a/Cargo.lock b/Cargo.lock index c9edee91c9..c0e41655e7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -692,6 +692,16 @@ dependencies = [ "zeroize", ] +[[package]] +name = "ctor" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d2301688392eb071b0bf1a37be05c469d3cc4dbbd95df672fe28ab021e6a096" +dependencies = [ + "quote", + "syn 1.0.109", +] + [[package]] name = "curve25519-dalek" version = "4.0.0-rc.3" @@ -1243,6 +1253,17 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "ghost" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55f62cab8c48c54b8aba6588bd75fd00cdfe8517e79797c3662c5ed0c011d257" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.25", +] + [[package]] name = "gimli" version = "0.27.3" @@ -1606,6 +1627,28 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "inventory" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0eb5160c60ba1e809707918ee329adb99d222888155835c6feedba19f6c3fd4" +dependencies = [ + "ctor", + "ghost", + "inventory-impl", +] + +[[package]] +name = "inventory-impl" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e41b53715c6f0c4be49510bb82dee2c1e51c8586d885abe65396e82ed518548" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "io-lifetimes" version = "1.0.11" @@ -1814,6 +1857,24 @@ dependencies = [ "zeroize", ] +[[package]] +name = "iroh_ffi" +version = "0.1.0" +dependencies = [ + "anyhow", + "blake3", + "data-encoding", + "iroh", + "iroh-net", + "libc", + "multibase", + "num_cpus", + "safer-ffi", + "tempfile", + "tokio", + "tokio-util", +] + [[package]] name = "is-terminal" version = "0.4.9" @@ -1967,6 +2028,21 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "mini_paste" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2499b7bd9834270bf24cfc4dd96be59020ba6fd7f3276b772aee2de66e82b63" +dependencies = [ + "mini_paste-proc_macro", +] + +[[package]] +name = "mini_paste-proc_macro" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c5f1f52e39b728e73af4b454f1b29173d4544607bd395dafe1918fd149db67" + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -2683,6 +2759,12 @@ dependencies = [ "version_check", ] +[[package]] +name = "proc-macro-hack" +version = "0.5.20+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" + [[package]] name = "proc-macro2" version = "1.0.64" @@ -3014,6 +3096,17 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2" +[[package]] +name = "require_unsafe_in_body" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "296446f2214753e33792f632262451e8d037cd9697df15db2fd531bdce5a0138" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "reqwest" version = "0.11.18" @@ -3269,6 +3362,32 @@ version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fe232bdf6be8c8de797b22184ee71118d63780ea42ac85b61d1baa6d3b782ae9" +[[package]] +name = "safer-ffi" +version = "0.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c710243617290d86a49a30564d94d0b646eacf6d7b67035e20d6e8a21f1193" +dependencies = [ + "inventory", + "libc", + "mini_paste", + "proc-macro-hack", + "require_unsafe_in_body", + "safer_ffi-proc_macro", +] + +[[package]] +name = "safer_ffi-proc_macro" +version = "0.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc02dc034daa0944eb133b448f261ef7422ccae768e30f30ce5cdeb4ae4e506c" +dependencies = [ + "proc-macro-hack", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "salsa20" version = "0.10.2" diff --git a/Cargo.toml b/Cargo.toml index 98886580a8..f76980f803 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,11 +1,12 @@ [workspace] members = [ "iroh", - "iroh-net", "iroh-bytes", + "iroh-ffi", "iroh-metrics", + "iroh-net", "iroh/examples/hello-world", - "iroh/examples/collection" + "iroh/examples/collection", ] [profile.optimized-release] diff --git a/iroh-ffi/.gitignore b/iroh-ffi/.gitignore deleted file mode 100644 index d3988d0919..0000000000 --- a/iroh-ffi/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -/target -/Cargo.lock -main -iroh.h \ No newline at end of file diff --git a/iroh-ffi/Cargo.toml b/iroh-ffi/Cargo.toml index 042db5b9b9..b0d5fdaba0 100644 --- a/iroh-ffi/Cargo.toml +++ b/iroh-ffi/Cargo.toml @@ -4,6 +4,11 @@ version = "0.1.0" edition = "2021" publish = false rust-version = "1.64" +readme = "README.md" +description = "IPFS reimagined" +license = "MIT/Apache-2.0" +authors = ["dignifiedquire ", "n0 team"] +repository = "https://github.com/n0-computer/iroh" [profile.release] strip = "symbols" @@ -16,13 +21,16 @@ crate-type = ["staticlib", "cdylib"] c-headers = ["safer-ffi/headers"] [dependencies] -safer-ffi = { version = "0.0.10", features = ["proc_macros"] } +# indicatif = { version = "0.17", features = ["tokio"] } anyhow = "1.0.69" blake3 = "1.3.3" -iroh = { path = "../", default-features = false } -indicatif = { version = "0.17", features = ["tokio"] } data-encoding = { version = "2.3.3" } -multibase = { version = "0.9.1"} +iroh = { path = "../iroh", default-features = false } +iroh-net = { path = "../iroh-net" } libc = "0.2.141" +multibase = { version = "0.9.1"} +num_cpus = { version = "1.15.0", optional = true } +safer-ffi = { version = "0.0.10", features = ["proc_macros"] } tempfile = "3.4" tokio = "1.25.0" +tokio-util = { version = "0.7", features = ["io-util", "io"] } diff --git a/iroh-ffi/src/error.rs b/iroh-ffi/src/error.rs new file mode 100644 index 0000000000..edef374e71 --- /dev/null +++ b/iroh-ffi/src/error.rs @@ -0,0 +1,70 @@ +use crate::error::IrohError; +use safer_ffi::prelude::*; + +impl From for repr_c::Box { + fn from(error: anyhow::Error) -> Self { + Box::new(IrohError { inner: error }).into() + } +} + +const IROH_ERROR_OTHER: u32 = 1; + +#[ffi_export] +#[derive_ReprC(rename = "iroh_error_code")] +#[repr(u32)] +/// Constant values for error codes from iroh_error_t. +pub enum IrohErrorCode { + Other = IROH_ERROR_OTHER, +} + +impl From for IrohErrorCode { + fn from(code: u32) -> Self { + match code { + IROH_ERROR_OTHER => IrohErrorCode::Other, + _ => IrohErrorCode::Other, + } + } +} + +impl From<&IrohError> for IrohErrorCode { + fn from(error: &IrohError) -> Self { + match error { + IrohError::Other(_) => IrohErrorCode::Other, + } + } +} + +#[derive_ReprC(rename = "iroh_error")] +#[repr(opaque)] +/// @class iroh_error_t +/// An opaque struct representing an error. +pub struct IrohError { + inner: anyhow::Error, +} + +#[ffi_export] +/// @memberof ns_error_t +/// Deallocate an ns_error_t. +pub fn iroh_error_free(error: repr_c::Box) { + drop(error) +} + +#[ffi_export] +/// @memberof iroh_error_t +/// Returns an owned string describing the error in greater detail. +/// +/// Caller is responsible for deallocating returned string via iroh_string_free. +pub fn iroh_error_message_get(error: &IrohError) -> char_p::Box { + error + .inner + .to_string() + .try_into() + .unwrap_or_else(|_| char_p::new("Unknown")) +} + +#[ffi_export] +/// @memberof ns_error_t +/// Returns an error code that identifies the error. +pub fn ns_error_code_get(error: &IrohError) -> u32 { + IrohErrorCode::from(&error.inner) as u32 +} diff --git a/iroh-ffi/src/get.rs b/iroh-ffi/src/get.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/iroh-ffi/src/lib.rs b/iroh-ffi/src/lib.rs index 04e78d5193..ebcf275273 100644 --- a/iroh-ffi/src/lib.rs +++ b/iroh-ffi/src/lib.rs @@ -1,279 +1,272 @@ -use std::{path::PathBuf, str::FromStr}; +// use std::{path::PathBuf, str::FromStr}; -use ::safer_ffi::prelude::*; -use anyhow::{Context, Result}; -use indicatif::{HumanBytes, HumanDuration}; -use tokio::io::AsyncWriteExt; -use tokio::runtime::Runtime; +// use ::safer_ffi::prelude::*; +// use anyhow::{Context, Result}; +// use indicatif::{HumanBytes, HumanDuration}; +// use tokio::io::AsyncWriteExt; +// use tokio::runtime::Runtime; -use crate::util::Blake3Cid; -use iroh::blobs::{Blob, Collection}; -use iroh::get::get_response_machine::{ConnectedNext, EndBlobNext}; -use iroh::get::{self, get_data_path, get_missing_range, get_missing_ranges, pathbuf_from_name}; -use iroh::protocol::GetRequest; -use iroh::provider::Ticket; -use iroh::tokio_util::SeekOptimized; -use iroh::{Hash, PeerId}; +// use crate::util::Blake3Cid; +// use iroh::blobs::{Blob, Collection}; +// use iroh::get::get_response_machine::{ConnectedNext, EndBlobNext}; +// use iroh::get::{self, get_data_path, get_missing_range, get_missing_ranges, pathbuf_from_name}; +// use iroh::protocol::GetRequest; +// use iroh::provider::Ticket; +// use iroh::tokio_util::SeekOptimized; +// use iroh::{Hash, PeerId}; -mod util; +// mod util; -const MAX_CONCURRENT_DIALS: u8 = 16; +// const MAX_CONCURRENT_DIALS: u8 = 16; -#[ffi_export] -fn iroh_get( - hash: char_p::Ref<'_>, - peer: char_p::Ref<'_>, - peer_addr: char_p::Ref<'_>, - out_path: char_p::Ref<'_>, -) -> u32 { - let result = std::panic::catch_unwind(|| { - let hash = hash.to_str().parse::().unwrap(); - let peer = peer.to_str().parse::().unwrap(); - let peer_addr = peer_addr.to_str().parse().unwrap(); - let out_path = PathBuf::from_str(out_path.to_str()).unwrap(); - let rt = Runtime::new().unwrap(); +// #[ffi_export] +// fn iroh_get( +// hash: char_p::Ref<'_>, +// peer: char_p::Ref<'_>, +// peer_addr: char_p::Ref<'_>, +// out_path: char_p::Ref<'_>, +// ) -> u32 { +// let result = std::panic::catch_unwind(|| { +// let hash = hash.to_str().parse::().unwrap(); +// let peer = peer.to_str().parse::().unwrap(); +// let peer_addr = peer_addr.to_str().parse().unwrap(); +// let out_path = PathBuf::from_str(out_path.to_str()).unwrap(); +// let rt = Runtime::new().unwrap(); - rt.block_on(get_to_dir( - GetInteractive::Hash { - hash, - opts: get::Options { - peer_id: Some(peer), - addr: peer_addr, - keylog: false, - }, - single: false, - }, - out_path, - )) - .unwrap(); - 0 - }); - if result.is_err() { - eprintln!("error: rust panicked"); - return 1; - } - result.unwrap() -} +// rt.block_on(get_to_dir( +// GetInteractive::Hash { +// hash, +// opts: get::Options { +// peer_id: Some(peer), +// addr: peer_addr, +// keylog: false, +// }, +// single: false, +// }, +// out_path, +// )) +// .unwrap(); +// 0 +// }); +// if result.is_err() { +// eprintln!("error: rust panicked"); +// return 1; +// } +// result.unwrap() +// } -#[ffi_export] -fn iroh_get_ticket(ticket: char_p::Ref<'_>, out_path: char_p::Ref<'_>) -> u32 { - let result = std::panic::catch_unwind(|| { - let ticket = ticket.to_str().parse::().unwrap(); - let out_path = PathBuf::from_str(out_path.to_str()).unwrap(); - let rt = Runtime::new().unwrap(); - rt.block_on(get_to_dir( - GetInteractive::Ticket { - ticket, - keylog: false, - }, - out_path, - )) - .unwrap(); - 0 - }); - if result.is_err() { - eprintln!("error: rust panicked"); - return 1; - } - result.unwrap() -} +// #[ffi_export] +// fn iroh_get_ticket(ticket: char_p::Ref<'_>, out_path: char_p::Ref<'_>) -> u32 { +// let result = std::panic::catch_unwind(|| { +// let ticket = ticket.to_str().parse::().unwrap(); +// let out_path = PathBuf::from_str(out_path.to_str()).unwrap(); +// let rt = Runtime::new().unwrap(); +// rt.block_on(get_to_dir( +// GetInteractive::Ticket { +// ticket, +// keylog: false, +// }, +// out_path, +// )) +// .unwrap(); +// 0 +// }); +// if result.is_err() { +// eprintln!("error: rust panicked"); +// return 1; +// } +// result.unwrap() +// } -#[derive(Debug)] -enum GetInteractive { - Ticket { - ticket: Ticket, - keylog: bool, - }, - Hash { - hash: Hash, - opts: get::Options, - single: bool, - }, -} +// #[derive(Debug)] +// enum GetInteractive { +// Ticket { +// ticket: Ticket, +// keylog: bool, +// }, +// Hash { +// hash: Hash, +// opts: get::Options, +// single: bool, +// }, +// } -impl GetInteractive { - fn hash(&self) -> Hash { - match self { - GetInteractive::Ticket { ticket, .. } => ticket.hash(), - GetInteractive::Hash { hash, .. } => *hash, - } - } +// impl GetInteractive { +// fn hash(&self) -> Hash { +// match self { +// GetInteractive::Ticket { ticket, .. } => ticket.hash(), +// GetInteractive::Hash { hash, .. } => *hash, +// } +// } - fn single(&self) -> bool { - match self { - GetInteractive::Ticket { .. } => false, - GetInteractive::Hash { single, .. } => *single, - } - } -} +// fn single(&self) -> bool { +// match self { +// GetInteractive::Ticket { .. } => false, +// GetInteractive::Hash { single, .. } => *single, +// } +// } +// } -/// Get into a file or directory -async fn get_to_dir(get: GetInteractive, out_dir: PathBuf) -> Result<()> { - let hash = get.hash(); - let single = get.single(); - println!("Fetching: {}", Blake3Cid::new(hash)); - println!("[1/3] Connecting ..."); +// /// Get into a file or directory +// async fn get_to_dir(get: GetInteractive, out_dir: PathBuf) -> Result<()> { +// let hash = get.hash(); +// let single = get.single(); +// println!("Fetching: {}", Blake3Cid::new(hash)); +// println!("[1/3] Connecting ..."); - let temp_dir = out_dir.join(".iroh-tmp"); - let (query, collection) = if single { - let name = Blake3Cid::new(hash).to_string(); - let query = get_missing_range(&get.hash(), name.as_str(), &temp_dir, &out_dir)?; - (query, vec![Blob { hash, name }]) - } else { - let (query, collection) = get_missing_ranges(get.hash(), &out_dir, &temp_dir)?; - ( - query, - collection.map(|x| x.into_inner()).unwrap_or_default(), - ) - }; +// let temp_dir = out_dir.join(".iroh-tmp"); +// let (query, collection) = if single { +// let name = Blake3Cid::new(hash).to_string(); +// let query = get_missing_range(&get.hash(), name.as_str(), &temp_dir, &out_dir)?; +// (query, vec![Blob { hash, name }]) +// } else { +// let (query, collection) = get_missing_ranges(get.hash(), &out_dir, &temp_dir)?; +// ( +// query, +// collection.map(|x| x.into_inner()).unwrap_or_default(), +// ) +// }; - let init_download_progress = |count: u64, missing_bytes: u64| { - println!("[3/3] Downloading ..."); - println!( - " {} file(s) with total transfer size {}", - count, - HumanBytes(missing_bytes) - ); - }; +// let init_download_progress = |count: u64, missing_bytes: u64| { +// println!("[3/3] Downloading ..."); +// println!( +// " {} file(s) with total transfer size {}", +// count, +// HumanBytes(missing_bytes) +// ); +// }; - // collection info, in case we won't get a callback with is_root - let collection_info = if collection.is_empty() { - None - } else { - Some((collection.len() as u64, 0)) - }; +// // collection info, in case we won't get a callback with is_root +// let collection_info = if collection.is_empty() { +// None +// } else { +// Some((collection.len() as u64, 0)) +// }; - let request = GetRequest::new(get.hash(), query).into(); - let response = match get { - GetInteractive::Ticket { ticket, keylog } => { - get::run_ticket(&ticket, request, keylog, MAX_CONCURRENT_DIALS).await? - } - GetInteractive::Hash { opts, .. } => get::run(request, opts).await?, - }; - let connected = response.next().await?; - println!("[2/3] Requesting ..."); - if let Some((count, missing_bytes)) = collection_info { - init_download_progress(count, missing_bytes); - } - let (mut next, collection) = match connected.next().await? { - ConnectedNext::StartRoot(curr) => { - tokio::fs::create_dir_all(&temp_dir) - .await - .context("unable to create directory {temp_dir}")?; - tokio::fs::create_dir_all(&out_dir) - .await - .context("Unable to create directory {out_dir}")?; - let curr = curr.next(); - let (curr, collection_data) = curr.concatenate_into_vec().await?; - let collection = Collection::from_bytes(&collection_data)?; - init_download_progress(collection.total_entries(), collection.total_blobs_size()); - tokio::fs::write(get_data_path(&temp_dir, hash), collection_data).await?; - (curr.next(), collection.into_inner()) - } - ConnectedNext::StartChild(start_child) => { - (EndBlobNext::MoreChildren(start_child), collection) - } - ConnectedNext::Closing(finish) => (EndBlobNext::Closing(finish), collection), - }; - // read all the children - let finishing = loop { - let start = match next { - EndBlobNext::MoreChildren(sc) => sc, - EndBlobNext::Closing(finish) => break finish, - }; - let child_offset = start.child_offset() as usize; - let blob = match collection.get(child_offset) { - Some(blob) => blob, - None => break start.finish(), - }; +// let request = GetRequest::new(get.hash(), query).into(); +// let response = match get { +// GetInteractive::Ticket { ticket, keylog } => { +// get::run_ticket(&ticket, request, keylog, MAX_CONCURRENT_DIALS).await? +// } +// GetInteractive::Hash { opts, .. } => get::run(request, opts).await?, +// }; +// let connected = response.next().await?; +// println!("[2/3] Requesting ..."); +// if let Some((count, missing_bytes)) = collection_info { +// init_download_progress(count, missing_bytes); +// } +// let (mut next, collection) = match connected.next().await? { +// ConnectedNext::StartRoot(curr) => { +// tokio::fs::create_dir_all(&temp_dir) +// .await +// .context("unable to create directory {temp_dir}")?; +// tokio::fs::create_dir_all(&out_dir) +// .await +// .context("Unable to create directory {out_dir}")?; +// let curr = curr.next(); +// let (curr, collection_data) = curr.concatenate_into_vec().await?; +// let collection = Collection::from_bytes(&collection_data)?; +// init_download_progress(collection.total_entries(), collection.total_blobs_size()); +// tokio::fs::write(get_data_path(&temp_dir, hash), collection_data).await?; +// (curr.next(), collection.into_inner()) +// } +// ConnectedNext::StartChild(start_child) => { +// (EndBlobNext::MoreChildren(start_child), collection) +// } +// ConnectedNext::Closing(finish) => (EndBlobNext::Closing(finish), collection), +// }; +// // read all the children +// let finishing = loop { +// let start = match next { +// EndBlobNext::MoreChildren(sc) => sc, +// EndBlobNext::Closing(finish) => break finish, +// }; +// let child_offset = start.child_offset() as usize; +// let blob = match collection.get(child_offset) { +// Some(blob) => blob, +// None => break start.finish(), +// }; - let hash = blob.hash; - let name = &blob.name; - let name = if name.is_empty() { - PathBuf::from(hash.to_string()) - } else { - pathbuf_from_name(name) - }; - // pb.set_message(format!("Receiving '{}'...", name.display())); - // pb.reset(); - let header = start.next(blob.hash); +// let hash = blob.hash; +// let name = &blob.name; +// let name = if name.is_empty() { +// PathBuf::from(hash.to_string()) +// } else { +// pathbuf_from_name(name) +// }; +// // pb.set_message(format!("Receiving '{}'...", name.display())); +// // pb.reset(); +// let header = start.next(blob.hash); - let curr = { - let final_path = out_dir.join(&name); - let tempname = blake3::Hash::from(hash).to_hex(); - let data_path = temp_dir.join(format!("{}.data.part", tempname)); - let outboard_path = temp_dir.join(format!("{}.outboard.part", tempname)); - let data_file = tokio::fs::OpenOptions::new() - .write(true) - .create(true) - .open(&data_path) - .await?; - let mut data_file = SeekOptimized::new(data_file).into(); - let (curr, size) = header.next().await?; - // pb.set_length(size); - let mut outboard_file = if size > 0 { - let outboard_file = tokio::fs::OpenOptions::new() - .write(true) - .create(true) - .open(&outboard_path) - .await?; - let outboard_file = SeekOptimized::new(outboard_file).into(); - Some(outboard_file) - } else { - None - }; - let curr = curr - .write_all_with_outboard(&mut outboard_file, &mut data_file) - .await?; - tokio::fs::create_dir_all( - final_path - .parent() - .context("final path should have parent")?, - ) - .await?; - // Flush the data file first, it is the only thing that matters at this point - data_file.into_inner().shutdown().await?; - // Rename temp file, to target name - // once this is done, the file is considered complete - tokio::fs::rename(data_path, final_path).await?; - if let Some(outboard_file) = outboard_file.take() { - // not sure if we have to do this - outboard_file.into_inner().shutdown().await?; - // delete the outboard file - tokio::fs::remove_file(outboard_path).await?; - } - curr - }; - next = curr.next(); - }; - let stats = finishing.next().await?; - tokio::fs::remove_dir_all(temp_dir).await?; - println!( - "Transferred {} in {}, {}/s", - HumanBytes(stats.bytes_read), - HumanDuration(stats.elapsed), - HumanBytes((stats.bytes_read as f64 / stats.elapsed.as_secs_f64()) as u64) - ); +// let curr = { +// let final_path = out_dir.join(&name); +// let tempname = blake3::Hash::from(hash).to_hex(); +// let data_path = temp_dir.join(format!("{}.data.part", tempname)); +// let outboard_path = temp_dir.join(format!("{}.outboard.part", tempname)); +// let data_file = tokio::fs::OpenOptions::new() +// .write(true) +// .create(true) +// .open(&data_path) +// .await?; +// let mut data_file = SeekOptimized::new(data_file).into(); +// let (curr, size) = header.next().await?; +// // pb.set_length(size); +// let mut outboard_file = if size > 0 { +// let outboard_file = tokio::fs::OpenOptions::new() +// .write(true) +// .create(true) +// .open(&outboard_path) +// .await?; +// let outboard_file = SeekOptimized::new(outboard_file).into(); +// Some(outboard_file) +// } else { +// None +// }; +// let curr = curr +// .write_all_with_outboard(&mut outboard_file, &mut data_file) +// .await?; +// tokio::fs::create_dir_all( +// final_path +// .parent() +// .context("final path should have parent")?, +// ) +// .await?; +// // Flush the data file first, it is the only thing that matters at this point +// data_file.into_inner().shutdown().await?; +// // Rename temp file, to target name +// // once this is done, the file is considered complete +// tokio::fs::rename(data_path, final_path).await?; +// if let Some(outboard_file) = outboard_file.take() { +// // not sure if we have to do this +// outboard_file.into_inner().shutdown().await?; +// // delete the outboard file +// tokio::fs::remove_file(outboard_path).await?; +// } +// curr +// }; +// next = curr.next(); +// }; +// let stats = finishing.next().await?; +// tokio::fs::remove_dir_all(temp_dir).await?; +// println!( +// "Transferred {} in {}, {}/s", +// HumanBytes(stats.bytes_read), +// HumanDuration(stats.elapsed), +// HumanBytes((stats.bytes_read as f64 / stats.elapsed.as_secs_f64()) as u64) +// ); - Ok(()) -} +// Ok(()) +// } -#[cfg(test)] -mod tests { +// #[cfg(test)] +// mod tests { - /// The following test function is necessary for the header generation. - #[::safer_ffi::cfg_headers] - #[test] - fn generate_headers() -> ::std::io::Result<()> { - ::safer_ffi::headers::builder() - .to_file("iroh.h")? - .generate() - } +// /// The following test function is necessary for the header generation. - #[test] - fn get_ticket_test() { - // TODO - // let result = get_ticket(); - // assert_eq!(result, 0); - } -} +// #[test] +// fn get_ticket_test() { +// // TODO +// // let result = get_ticket(); +// // assert_eq!(result, 0); +// } +// } diff --git a/iroh-ffi/src/mod.rs b/iroh-ffi/src/mod.rs new file mode 100644 index 0000000000..c0448de2ac --- /dev/null +++ b/iroh-ffi/src/mod.rs @@ -0,0 +1,16 @@ +mod get; +mod node; + +pub use crate::ffi::get::*; +pub use crate::ffi::node::*; + +#[cfg(feature = "headers")] +pub fn generate_headers() -> std::io::Result<()> { + safer_ffi::headers::builder().to_file("iroh.h")?.generate() +} + +#[ffi_export] +/// Deallocate an Iroh-allocated string. +pub fn iroh_string_free(string: char_p::Box) { + drop(string) +} diff --git a/iroh-ffi/src/node.rs b/iroh-ffi/src/node.rs new file mode 100644 index 0000000000..46a6a5a4ad --- /dev/null +++ b/iroh-ffi/src/node.rs @@ -0,0 +1,66 @@ +use anyhow::Result; +use safer_ffi::prelude::*; +use tokio::Runtime; + +use iroh::{ + node::{Node, DEFAULT_BIND_ADDR}, + provider::Database, +}; +use iroh_net::tls::Keypair; + +#[derive_ReprC(rename = "iroh_node")] +#[repr(opaque)] +/// @class iroh_node_t +pub struct IrohNode { + inner: Node, + async_runtime: Arc, +} + +impl IrohNode { + // pub fn new() -> Result { + // todo!() + // } + + pub fn async_runtime(&self) -> Arc { + self.async_runtime.clone() + } + + pub fn inner(&self) -> &Node { + &self.inner + } + + pub fn inner_mut(&mut self) -> &mut Node { + &mut self.inner + } +} + +#[ffi_export] +/// @memberof iroh_node_t +/// Initialize a iroh_node_t instance. +/// +pub fn iroh_initialize() -> Option> { + let tokio_rt = tokio::runtime::Builder::new_multi_thread() + .thread_name("main-runtime") + .worker_threads(2) + .enable_all() + .build()?; + + let tokio = tokio::runtime::Handle::current(); + let tpc = tokio_util::task::LocalPoolHandle::new(num_cpus::get()); + let rt = iroh::bytes::runtime::Handle::new(tokio, tpc); + + let db = Database::default(); + let keypair = Keypair::generate(); + let node = Node::builder(db) + .bind_addr(DEFAULT_BIND_ADDR) + .keypair(keypair) + .runtime(rt) + .spawn() + .await?; + + repr_c::Box::new(IrohNode { + inner: node, + runtime: tokio_rt, + }) + .ok() +} diff --git a/iroh-ffi/src/util.rs b/iroh-ffi/src/util.rs index a046254902..4c759d94be 100644 --- a/iroh-ffi/src/util.rs +++ b/iroh-ffi/src/util.rs @@ -1,90 +1,90 @@ -//! Utility functions and types. -use std::{fmt, str::FromStr}; +// //! Utility functions and types. +// use std::{fmt, str::FromStr}; -use anyhow::Result; -use iroh::Hash; +// use anyhow::Result; +// use iroh::Hash; -#[repr(transparent)] -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct Blake3Cid(pub Hash); +// #[repr(transparent)] +// #[derive(Debug, Clone, Copy, PartialEq, Eq)] +// pub struct Blake3Cid(pub Hash); -const CID_PREFIX: [u8; 4] = [ - 0x01, // version - 0x55, // raw codec - 0x1e, // hash function, blake3 - 0x20, // hash size, 32 bytes -]; +// const CID_PREFIX: [u8; 4] = [ +// 0x01, // version +// 0x55, // raw codec +// 0x1e, // hash function, blake3 +// 0x20, // hash size, 32 bytes +// ]; -impl Blake3Cid { - pub fn new(hash: Hash) -> Self { - Blake3Cid(hash) - } +// impl Blake3Cid { +// pub fn new(hash: Hash) -> Self { +// Blake3Cid(hash) +// } - // pub fn as_hash(&self) -> &Hash { - // &self.0 - // } +// // pub fn as_hash(&self) -> &Hash { +// // &self.0 +// // } - pub fn as_bytes(&self) -> [u8; 36] { - let hash: [u8; 32] = self.0.as_ref().try_into().unwrap(); - let mut res = [0u8; 36]; - res[0..4].copy_from_slice(&CID_PREFIX); - res[4..36].copy_from_slice(&hash); - res - } +// pub fn as_bytes(&self) -> [u8; 36] { +// let hash: [u8; 32] = self.0.as_ref().try_into().unwrap(); +// let mut res = [0u8; 36]; +// res[0..4].copy_from_slice(&CID_PREFIX); +// res[4..36].copy_from_slice(&hash); +// res +// } - pub fn from_bytes(bytes: &[u8]) -> anyhow::Result { - anyhow::ensure!( - bytes.len() == 36, - "invalid cid length, expected 36, got {}", - bytes.len() - ); - anyhow::ensure!(bytes[0..4] == CID_PREFIX, "invalid cid prefix"); - let mut hash = [0u8; 32]; - hash.copy_from_slice(&bytes[4..36]); - Ok(Blake3Cid(Hash::from(hash))) - } -} +// pub fn from_bytes(bytes: &[u8]) -> anyhow::Result { +// anyhow::ensure!( +// bytes.len() == 36, +// "invalid cid length, expected 36, got {}", +// bytes.len() +// ); +// anyhow::ensure!(bytes[0..4] == CID_PREFIX, "invalid cid prefix"); +// let mut hash = [0u8; 32]; +// hash.copy_from_slice(&bytes[4..36]); +// Ok(Blake3Cid(Hash::from(hash))) +// } +// } -impl fmt::Display for Blake3Cid { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - // result will be 58 bytes plus prefix - let mut res = [b'b'; 59]; - // write the encoded bytes - data_encoding::BASE32_NOPAD.encode_mut(&self.as_bytes(), &mut res[1..]); - // convert to string, this is guaranteed to succeed - let t = std::str::from_utf8_mut(res.as_mut()).unwrap(); - // hack since data_encoding doesn't have BASE32LOWER_NOPAD as a const - t.make_ascii_lowercase(); - // write the str, no allocations - f.write_str(t) - } -} +// impl fmt::Display for Blake3Cid { +// fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { +// // result will be 58 bytes plus prefix +// let mut res = [b'b'; 59]; +// // write the encoded bytes +// data_encoding::BASE32_NOPAD.encode_mut(&self.as_bytes(), &mut res[1..]); +// // convert to string, this is guaranteed to succeed +// let t = std::str::from_utf8_mut(res.as_mut()).unwrap(); +// // hack since data_encoding doesn't have BASE32LOWER_NOPAD as a const +// t.make_ascii_lowercase(); +// // write the str, no allocations +// f.write_str(t) +// } +// } -impl FromStr for Blake3Cid { - type Err = anyhow::Error; +// impl FromStr for Blake3Cid { +// type Err = anyhow::Error; - fn from_str(s: &str) -> Result { - let sb = s.as_bytes(); - if sb.len() == 59 && sb[0] == b'b' { - // this is a base32 encoded cid, we can decode it directly - let mut t = [0u8; 58]; - t.copy_from_slice(&sb[1..]); - // hack since data_encoding doesn't have BASE32LOWER_NOPAD as a const - std::str::from_utf8_mut(t.as_mut()) - .unwrap() - .make_ascii_uppercase(); - // decode the bytes - let mut res = [0u8; 36]; - data_encoding::BASE32_NOPAD - .decode_mut(&t, &mut res) - .map_err(|_e| anyhow::anyhow!("invalid base32"))?; - // convert to cid, this will check the prefix - Self::from_bytes(&res) - } else { - // if we want to support all the weird multibase prefixes, we have no choice - // but to use the multibase crate - let (_base, bytes) = multibase::decode(s)?; - Self::from_bytes(bytes.as_ref()) - } - } -} +// fn from_str(s: &str) -> Result { +// let sb = s.as_bytes(); +// if sb.len() == 59 && sb[0] == b'b' { +// // this is a base32 encoded cid, we can decode it directly +// let mut t = [0u8; 58]; +// t.copy_from_slice(&sb[1..]); +// // hack since data_encoding doesn't have BASE32LOWER_NOPAD as a const +// std::str::from_utf8_mut(t.as_mut()) +// .unwrap() +// .make_ascii_uppercase(); +// // decode the bytes +// let mut res = [0u8; 36]; +// data_encoding::BASE32_NOPAD +// .decode_mut(&t, &mut res) +// .map_err(|_e| anyhow::anyhow!("invalid base32"))?; +// // convert to cid, this will check the prefix +// Self::from_bytes(&res) +// } else { +// // if we want to support all the weird multibase prefixes, we have no choice +// // but to use the multibase crate +// let (_base, bytes) = multibase::decode(s)?; +// Self::from_bytes(bytes.as_ref()) +// } +// } +// } From 15c45a2814eb06fb2c3682e80d95dc0167a71d04 Mon Sep 17 00:00:00 2001 From: b5 Date: Thu, 13 Jul 2023 14:21:00 -0400 Subject: [PATCH 13/32] WIP 2 --- iroh/src/get.rs | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 iroh/src/get.rs diff --git a/iroh/src/get.rs b/iroh/src/get.rs new file mode 100644 index 0000000000..e69de29bb2 From 80cdef8abf54528d6e55fb9e4768e0a5137e6783 Mon Sep 17 00:00:00 2001 From: b5 Date: Thu, 13 Jul 2023 22:01:05 -0400 Subject: [PATCH 14/32] WIP --- Cargo.toml | 3 + iroh-ffi/Cargo.toml | 6 +- iroh-ffi/src/get.rs | 115 +++++++++++++++++++++++++++++++++++++++ iroh-ffi/src/node.rs | 4 -- iroh/src/commands/get.rs | 6 +- iroh/src/dial.rs | 4 +- iroh/src/get.rs | 0 iroh/src/node.rs | 14 +++++ iroh/tests/provide.rs | 12 ++-- 9 files changed, 145 insertions(+), 19 deletions(-) delete mode 100644 iroh/src/get.rs diff --git a/Cargo.toml b/Cargo.toml index f76980f803..a808cade32 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,3 +16,6 @@ debug-assertions = false opt-level = 3 panic = 'abort' incremental = false + +[profile.ffi] +strip = "symbols" \ No newline at end of file diff --git a/iroh-ffi/Cargo.toml b/iroh-ffi/Cargo.toml index b0d5fdaba0..291e7550fc 100644 --- a/iroh-ffi/Cargo.toml +++ b/iroh-ffi/Cargo.toml @@ -3,16 +3,12 @@ name = "iroh_ffi" version = "0.1.0" edition = "2021" publish = false -rust-version = "1.64" readme = "README.md" description = "IPFS reimagined" license = "MIT/Apache-2.0" -authors = ["dignifiedquire ", "n0 team"] +authors = ["n0 team"] repository = "https://github.com/n0-computer/iroh" -[profile.release] -strip = "symbols" - [lib] name="iroh" crate-type = ["staticlib", "cdylib"] diff --git a/iroh-ffi/src/get.rs b/iroh-ffi/src/get.rs index e69de29bb2..6c6967f7d8 100644 --- a/iroh-ffi/src/get.rs +++ b/iroh-ffi/src/get.rs @@ -0,0 +1,115 @@ +use iroh::dial::{dial, Ticket}; + +#[ffi_export] +/// @memberof iroh_node_t +// TODO(b5): optional token arg +fn iroh_get( + iroh_node: &mut IrohNode, + hash: char_p::Ref<'_>, + peer: char_p::Ref<'_>, + peer_addr: char_p::Ref<'_>, + out_path: char_p::Ref<'_>, + callback: extern "C" fn(Option>), +) -> u32 { + let rt = node.async_runtime(); + node.async_runtime().spawn(async move { + let result = async move { + let hash = hash.to_str().parse::()?; + let peer = peer.to_str().parse::()?; + let peer_addr = peer_addr.to_str().parse()?; + let conn = node + .inner() + .clone() + .dial(peer, &[peer_addr], &iroh_bytes::protocol::ALPN); + let out_path = PathBuf::from_str(out_path.to_str()).unwrap(); + get_blob_to_file(conn, hash, None).await + } + .await; + + match result { + Ok() => rt.spawn_blocking(move || callback(None)), + Err(error) => rt.spawn_blocking(move || callback(Some(IrohError::from(error).into()))), + }; + }); +} + +#[ffi_export] +/// @memberof iroh_node_t +/// Get a collection from a peer. +pub fn iroh_get_ticket( + node: &mut IrohNode, + ticket: char_p::Ref<'_>, + out_path: char_p::Ref<'_>, + callback: extern "C" fn(Option>), +) { + let rt = node.async_runtime(); + node.async_runtime().spawn(async move { + let result = async { + let out_path = PathBuf::from_str(out_path.to_str())?; + // let keypair = node.inner(). + let ticket = Ticket::from_str(ticket.to_str())?; + // TODO(b5): use the node endpoint(s) to dial + let conn = node.inner().clone().dial( + ticket.peer(), + ticket.addrs(), + &iroh_bytes::protocol::ALPN, + ); + get_blob_to_file(conn, ticket.hash(), ticket.token(), out_path).await + } + .await; + + match result { + Ok() => rt.spawn_blocking(move || callback(None)), + Err(error) => rt.spawn_blocking(move || callback(Some(IrohError::from(error).into()))), + }; + }); +} + +async fn get_blob_to_file( + conn: quinn::Connection, + hash: Hash, + token: Option, + out_path: PathBuf, +) -> Result<()> { + get_blob_ranges_to_file( + conn, + hash, + token, + RangeSpecSeq::new([RangeSet2::all()]), + out_path, + ) + .await? +} + +// TODO(b5): This currently assumes "all" ranges, needs to be adjusted to honor +// RangeSpecSeq args other than "all" +async fn get_blob_ranges_to_file( + conn: quinn::Connection, + hash: Hash, + token: Option, + ranges: RangeSpecSeq, + out_path: PathBuf, +) -> Result<()> { + let request = Request::Get(GetRequest::new(hash, RangeSpecSeq::new([RangeSet2::all()]))); + let response = fsm::start(conn, request); + let connected = response.next().await?; + + let fsm::ConnectedNext::StartRoot(curr) = connected.next().await? else { + return Ok(None) + }; + let header = curr.next(); + + let path = out_path.clone(); + let mut file = File::create(move || { + std::fs::OpenOptions::new() + .write(true) + .create(true) + .open(&path) + }) + .await?; + + let (curr, _size) = header.next().await?; + let _curr = curr.write_all(&mut file).await?; + file.sync().await?; + Ok(()) +} diff --git a/iroh-ffi/src/node.rs b/iroh-ffi/src/node.rs index 46a6a5a4ad..4a0c4132cf 100644 --- a/iroh-ffi/src/node.rs +++ b/iroh-ffi/src/node.rs @@ -17,10 +17,6 @@ pub struct IrohNode { } impl IrohNode { - // pub fn new() -> Result { - // todo!() - // } - pub fn async_runtime(&self) -> Arc { self.async_runtime.clone() } diff --git a/iroh/src/commands/get.rs b/iroh/src/commands/get.rs index 7d020eb17d..72ac492b7a 100644 --- a/iroh/src/commands/get.rs +++ b/iroh/src/commands/get.rs @@ -67,7 +67,7 @@ impl GetInteractive { let collection_info = Some((1, 0)); let request = self.new_request(query).with_token(self.token.clone()); - let connection = iroh::dial::dial(self.opts).await?; + let connection = iroh::dial::dial(self.opts, &iroh_bytes::protocol::ALPN).await?; let response = fsm::start(connection, request); let connected = response.next().await?; write(format!("{} Requesting ...", style("[2/3]").bold().dim())); @@ -155,7 +155,7 @@ impl GetInteractive { }; let request = self.new_request(query).with_token(self.token.clone()); - let connection = iroh::dial::dial(self.opts).await?; + let connection = iroh::dial::dial(self.opts, &iroh_bytes::protocol::ALPN).await?; let response = fsm::start(connection, request); let connected = response.next().await?; write(format!("{} Requesting ...", style("[2/3]").bold().dim())); @@ -322,7 +322,7 @@ impl GetInteractive { let pb = make_download_pb(); let request = self.new_request(query).with_token(self.token.clone()); - let connection = iroh::dial::dial(self.opts).await?; + let connection = iroh::dial::dial(self.opts, &iroh_bytes::protocol::ALPN).await?; let response = fsm::start(connection, request); let connected = response.next().await?; write(format!("{} Requesting ...", style("[2/3]").bold().dim())); diff --git a/iroh/src/dial.rs b/iroh/src/dial.rs index a4af57a47a..8b693c64ed 100644 --- a/iroh/src/dial.rs +++ b/iroh/src/dial.rs @@ -34,7 +34,7 @@ pub struct Options { /// Note that this will create an entirely new endpoint, so it should be only /// used for short lived connections. If you want to connect to multiple peers, /// it is preferable to create an endpoint and use `connect` on the endpoint. -pub async fn dial(opts: Options) -> anyhow::Result { +pub async fn dial(opts: Options, alpn: &[u8]) -> anyhow::Result { let endpoint = iroh_net::MagicEndpoint::builder() .keypair(opts.keypair) .derp_map(opts.derp_map) @@ -42,7 +42,7 @@ pub async fn dial(opts: Options) -> anyhow::Result { .bind(0) .await?; endpoint - .connect(opts.peer_id, &iroh_bytes::protocol::ALPN, &opts.addrs) + .connect(opts.peer_id, alpn, &opts.addrs) .await .context("failed to connect to provider") } diff --git a/iroh/src/get.rs b/iroh/src/get.rs deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/iroh/src/node.rs b/iroh/src/node.rs index 0784d7103e..d47aa0c6f0 100644 --- a/iroh/src/node.rs +++ b/iroh/src/node.rs @@ -542,6 +542,20 @@ impl Node { self.inner.keypair.public().into() } + /// Dial a given peer + pub async fn dial( + &self, + alpn: &[u8], + peer_id: PeerId, + known_addrs: Vec, + ) -> Result { + self.inner + .endpoint + .connect(peer_id, alpn, &known_addrs) + .await + .context("failed to dial") + } + /// Subscribe to [`Event`]s emitted from the node, informing about connections and /// progress. /// diff --git a/iroh/tests/provide.rs b/iroh/tests/provide.rs index 05c59e1242..898bb2d308 100644 --- a/iroh/tests/provide.rs +++ b/iroh/tests/provide.rs @@ -434,9 +434,10 @@ async fn test_blob_reader_partial() -> Result<()> { let peer_id = node.peer_id(); let timeout = tokio::time::timeout(std::time::Duration::from_secs(10), async move { - let connection = iroh::dial::dial(get_options(peer_id, node_addr)) - .await - .unwrap(); + let connection = + iroh::dial::dial(get_options(peer_id, node_addr), &iroh_bytes::protocol::ALPN) + .await + .unwrap(); let response = fsm::start(connection, GetRequest::all(hash).into()); // connect let connected = response.next().await.unwrap(); @@ -565,7 +566,7 @@ async fn run_custom_get_request( request: AnyGetRequest, collection_parser: C, ) -> anyhow::Result<(Bytes, BTreeMap, Stats)> { - let connection = iroh::dial::dial(opts).await?; + let connection = iroh::dial::dial(opts, &iroh_bytes::protocol::ALPN).await?; let initial = fsm::start(connection, request); use fsm::*; let mut items = BTreeMap::new(); @@ -746,7 +747,8 @@ async fn test_custom_request_blob() { token: None, data: Bytes::from(&b"hello"[..]), }); - let connection = iroh::dial::dial(get_options(peer_id, addrs)).await?; + let connection = + iroh::dial::dial(get_options(peer_id, addrs), &iroh_bytes::protocol::ALPN).await?; let response = fsm::start(connection, request); let connected = response.next().await?; let ConnectedNext::StartRoot(start) = connected.next().await? else { panic!() }; From 92c15adba613c8ceb6cd0fa596cb32ec0e84bc52 Mon Sep 17 00:00:00 2001 From: b5 Date: Fri, 14 Jul 2023 08:42:43 -0400 Subject: [PATCH 15/32] WIP --- iroh-ffi/Cargo.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/iroh-ffi/Cargo.toml b/iroh-ffi/Cargo.toml index 291e7550fc..ad811cd930 100644 --- a/iroh-ffi/Cargo.toml +++ b/iroh-ffi/Cargo.toml @@ -17,7 +17,6 @@ crate-type = ["staticlib", "cdylib"] c-headers = ["safer-ffi/headers"] [dependencies] -# indicatif = { version = "0.17", features = ["tokio"] } anyhow = "1.0.69" blake3 = "1.3.3" data-encoding = { version = "2.3.3" } From d8866b5c79aec26bbd656180987ce13228880ec4 Mon Sep 17 00:00:00 2001 From: b5 Date: Fri, 14 Jul 2023 16:36:44 -0400 Subject: [PATCH 16/32] WIP --- Cargo.lock | 149 +++++++++---- Cargo.toml | 1 + Package.swift | 33 +++ iroh-ffi/Cargo.toml | 13 +- iroh-ffi/src/bin/generate-headers.rs | 3 + iroh-ffi/src/error.rs | 25 ++- iroh-ffi/src/get.rs | 85 +++++--- iroh-ffi/src/lib.rs | 288 ++------------------------ iroh-ffi/src/mod.rs | 16 -- iroh-ffi/src/node.rs | 61 ++++-- iroh-ffi/src/util.rs | 90 -------- iroh/src/node.rs | 4 +- scripts/build-xcframework.sh | 56 +++++ swift/Sources/Iroh/Iroh.swift | 21 ++ swift/Tests/IrohTests/IrohTests.swift | 8 + swift/include/module.modulemap | 4 + 16 files changed, 369 insertions(+), 488 deletions(-) create mode 100644 Package.swift create mode 100644 iroh-ffi/src/bin/generate-headers.rs delete mode 100644 iroh-ffi/src/mod.rs delete mode 100644 iroh-ffi/src/util.rs create mode 100755 scripts/build-xcframework.sh create mode 100644 swift/Sources/Iroh/Iroh.swift create mode 100644 swift/Tests/IrohTests/IrohTests.swift create mode 100644 swift/include/module.modulemap diff --git a/Cargo.lock b/Cargo.lock index c0e41655e7..9415c409fd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1075,6 +1075,35 @@ dependencies = [ "libc", ] +[[package]] +name = "ext-trait" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d772df1c1a777963712fb68e014235e80863d6a91a85c4e06ba2d16243a310e5" +dependencies = [ + "ext-trait-proc_macros", +] + +[[package]] +name = "ext-trait-proc_macros" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ab7934152eaf26aa5aa9f7371408ad5af4c31357073c9e84c3b9d7f11ad639a" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "extension-traits" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a296e5a895621edf9fa8329c83aa1cb69a964643e36cf54d8d7a69b789089537" +dependencies = [ + "ext-trait", +] + [[package]] name = "fallible-iterator" version = "0.2.0" @@ -1865,10 +1894,13 @@ dependencies = [ "blake3", "data-encoding", "iroh", + "iroh-io", "iroh-net", "libc", "multibase", "num_cpus", + "quinn", + "range-collections", "safer-ffi", "tempfile", "tokio", @@ -1974,6 +2006,22 @@ dependencies = [ "libc", ] +[[package]] +name = "macro_rules_attribute" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf0c9b980bf4f3a37fd7b1c066941dd1b1d0152ce6ee6e8fe8c49b9f6810d862" +dependencies = [ + "macro_rules_attribute-proc_macro", + "paste", +] + +[[package]] +name = "macro_rules_attribute-proc_macro" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58093314a45e00c77d5c508f76e77c3396afbbc0d01506e7fae47b018bac2b1d" + [[package]] name = "match_cfg" version = "0.1.0" @@ -2028,21 +2076,6 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" -[[package]] -name = "mini_paste" -version = "0.1.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2499b7bd9834270bf24cfc4dd96be59020ba6fd7f3276b772aee2de66e82b63" -dependencies = [ - "mini_paste-proc_macro", -] - -[[package]] -name = "mini_paste-proc_macro" -version = "0.1.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c5f1f52e39b728e73af4b454f1b29173d4544607bd395dafe1918fd149db67" - [[package]] name = "minimal-lexical" version = "0.2.1" @@ -2716,6 +2749,16 @@ dependencies = [ "yansi", ] +[[package]] +name = "prettyplease" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c8646e95016a7a6c4adea95bafa8a16baab64b583356217f2c85db4a39d9a86" +dependencies = [ + "proc-macro2", + "syn 1.0.109", +] + [[package]] name = "primeorder" version = "0.13.2" @@ -2759,12 +2802,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "proc-macro-hack" -version = "0.5.20+deprecated" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" - [[package]] name = "proc-macro2" version = "1.0.64" @@ -3096,17 +3133,6 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2" -[[package]] -name = "require_unsafe_in_body" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "296446f2214753e33792f632262451e8d037cd9697df15db2fd531bdce5a0138" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "reqwest" version = "0.11.18" @@ -3364,25 +3390,29 @@ checksum = "fe232bdf6be8c8de797b22184ee71118d63780ea42ac85b61d1baa6d3b782ae9" [[package]] name = "safer-ffi" -version = "0.0.10" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40c710243617290d86a49a30564d94d0b646eacf6d7b67035e20d6e8a21f1193" +checksum = "4f47f1d2f33598dab2baa9517fffa1cf722f2e3a30633f2a230f20f9da67c564" dependencies = [ "inventory", "libc", - "mini_paste", - "proc-macro-hack", - "require_unsafe_in_body", - "safer_ffi-proc_macro", + "macro_rules_attribute", + "paste", + "safer_ffi-proc_macros", + "scopeguard", + "uninit", + "unwind_safe", + "with_builtin_macros", ] [[package]] -name = "safer_ffi-proc_macro" -version = "0.0.10" +name = "safer_ffi-proc_macros" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc02dc034daa0944eb133b448f261ef7422ccae768e30f30ce5cdeb4ae4e506c" +checksum = "b08f58cf71a58bda5734758eb20051cdb66c06c9243badbc45092ced1be834df" dependencies = [ - "proc-macro-hack", + "macro_rules_attribute", + "prettyplease", "proc-macro2", "quote", "syn 1.0.109", @@ -4328,6 +4358,15 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" +[[package]] +name = "uninit" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e130f2ed46ca5d8ec13c7ff95836827f92f5f5f37fd2b2bf16f33c408d98bb6" +dependencies = [ + "extension-traits", +] + [[package]] name = "universal-hash" version = "0.5.1" @@ -4344,6 +4383,12 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" +[[package]] +name = "unwind_safe" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0976c77def3f1f75c4ef892a292c31c0bbe9e3d0702c63044d7c76db298171a3" + [[package]] name = "url" version = "2.4.0" @@ -4775,6 +4820,26 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "with_builtin_macros" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a59d55032495429b87f9d69954c6c8602e4d3f3e0a747a12dea6b0b23de685da" +dependencies = [ + "with_builtin_macros-proc_macros", +] + +[[package]] +name = "with_builtin_macros-proc_macros" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15bd7679c15e22924f53aee34d4e448c45b674feb6129689af88593e129f8f42" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "wmi" version = "0.13.1" diff --git a/Cargo.toml b/Cargo.toml index a808cade32..2f68460a33 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,4 +18,5 @@ panic = 'abort' incremental = false [profile.ffi] +inherits = 'release' strip = "symbols" \ No newline at end of file diff --git a/Package.swift b/Package.swift new file mode 100644 index 0000000000..441f19dd95 --- /dev/null +++ b/Package.swift @@ -0,0 +1,33 @@ +// swift-tools-version: 5.8 +// The swift-tools-version declares the minimum version of Swift required to build this package. + +import PackageDescription + +let package = Package( + name: "iroh", + platforms: [ + .iOS(.v13), + ], + products: [ + // Products define the executables and libraries a package produces, + // making them visible to other packages. + .library( + name: "Iroh", + targets: ["Iroh"]), + ], + targets: [ + // Targets are the basic building blocks of a package, defining a module or a test suite. + // Targets can depend on other targets in this package and products from dependencies. + .target( + name: "Iroh", + dependencies: ["LibIroh"], + path: "swift/Sources/Iroh"), + .testTarget( + name: "IrohTests", + dependencies: ["Iroh"], + path: "swift/Tests/IrohTests"), + .binaryTarget( + name: "LibIroh", + path: "target/LibIroh.xcframework"), + ] +) diff --git a/iroh-ffi/Cargo.toml b/iroh-ffi/Cargo.toml index ad811cd930..9b84764449 100644 --- a/iroh-ffi/Cargo.toml +++ b/iroh-ffi/Cargo.toml @@ -10,8 +10,8 @@ authors = ["n0 team"] repository = "https://github.com/n0-computer/iroh" [lib] -name="iroh" -crate-type = ["staticlib", "cdylib"] +# name="iroh" +crate-type = ["rlib", "staticlib", "cdylib"] [features] c-headers = ["safer-ffi/headers"] @@ -20,12 +20,15 @@ c-headers = ["safer-ffi/headers"] anyhow = "1.0.69" blake3 = "1.3.3" data-encoding = { version = "2.3.3" } -iroh = { path = "../iroh", default-features = false } +iroh = { path = "../iroh", default-features = false, features = ["mem-db", "iroh-collection"] } +iroh-io = { version = "0.2.1" } iroh-net = { path = "../iroh-net" } libc = "0.2.141" multibase = { version = "0.9.1"} -num_cpus = { version = "1.15.0", optional = true } -safer-ffi = { version = "0.0.10", features = ["proc_macros"] } +num_cpus = { version = "1.15.0" } +quinn = "0.10" +range-collections = "0.4.0" +safer-ffi = { version = "0.1.0-rc1", features = ["proc_macros"] } tempfile = "3.4" tokio = "1.25.0" tokio-util = { version = "0.7", features = ["io-util", "io"] } diff --git a/iroh-ffi/src/bin/generate-headers.rs b/iroh-ffi/src/bin/generate-headers.rs new file mode 100644 index 0000000000..2c0dee0356 --- /dev/null +++ b/iroh-ffi/src/bin/generate-headers.rs @@ -0,0 +1,3 @@ +fn main() -> std::io::Result<()> { + iroh_ffi::generate_headers() +} diff --git a/iroh-ffi/src/error.rs b/iroh-ffi/src/error.rs index edef374e71..85b98592d8 100644 --- a/iroh-ffi/src/error.rs +++ b/iroh-ffi/src/error.rs @@ -1,9 +1,8 @@ -use crate::error::IrohError; use safer_ffi::prelude::*; -impl From for repr_c::Box { - fn from(error: anyhow::Error) -> Self { - Box::new(IrohError { inner: error }).into() +impl From for repr_c::Box { + fn from(error: IrohError) -> Self { + Box::new(IrohError { inner: error.inner }).into() } } @@ -27,10 +26,8 @@ impl From for IrohErrorCode { } impl From<&IrohError> for IrohErrorCode { - fn from(error: &IrohError) -> Self { - match error { - IrohError::Other(_) => IrohErrorCode::Other, - } + fn from(_error: &IrohError) -> Self { + IrohErrorCode::Other } } @@ -42,6 +39,16 @@ pub struct IrohError { inner: anyhow::Error, } +impl IrohError { + pub fn new(err: anyhow::Error) -> Self { + IrohError { inner: err } + } + + pub fn inner(self) -> anyhow::Error { + self.inner + } +} + #[ffi_export] /// @memberof ns_error_t /// Deallocate an ns_error_t. @@ -66,5 +73,5 @@ pub fn iroh_error_message_get(error: &IrohError) -> char_p::Box { /// @memberof ns_error_t /// Returns an error code that identifies the error. pub fn ns_error_code_get(error: &IrohError) -> u32 { - IrohErrorCode::from(&error.inner) as u32 + IrohErrorCode::from(error) as u32 } diff --git a/iroh-ffi/src/get.rs b/iroh-ffi/src/get.rs index 6c6967f7d8..3d4310ded3 100644 --- a/iroh-ffi/src/get.rs +++ b/iroh-ffi/src/get.rs @@ -1,34 +1,56 @@ -use iroh::dial::{dial, Ticket}; +use std::path::PathBuf; +use std::str::FromStr; + +use anyhow::Result; +use iroh_io::{AsyncSliceWriter, File}; +use range_collections::RangeSet2; +use safer_ffi::prelude::*; + +use iroh::{ + bytes::{ + get::fsm, + protocol::{GetRequest, RangeSpecSeq, Request, RequestToken}, + Hash, + }, + dial::Ticket, + net::tls::PeerId, +}; + +use crate::{error::IrohError, node::IrohNode}; #[ffi_export] /// @memberof iroh_node_t // TODO(b5): optional token arg fn iroh_get( - iroh_node: &mut IrohNode, + node: &mut IrohNode, hash: char_p::Ref<'_>, peer: char_p::Ref<'_>, peer_addr: char_p::Ref<'_>, out_path: char_p::Ref<'_>, callback: extern "C" fn(Option>), -) -> u32 { +) { + let node1 = node.inner().clone(); let rt = node.async_runtime(); - node.async_runtime().spawn(async move { + let hash = hash.to_string(); + let peer = peer.to_string(); + let peer_addr = peer_addr.to_string(); + let out_path = PathBuf::from(out_path.to_string()); + + node.async_runtime().clone().spawn(async move { let result = async move { - let hash = hash.to_str().parse::()?; - let peer = peer.to_str().parse::()?; - let peer_addr = peer_addr.to_str().parse()?; - let conn = node - .inner() - .clone() - .dial(peer, &[peer_addr], &iroh_bytes::protocol::ALPN); - let out_path = PathBuf::from_str(out_path.to_str()).unwrap(); - get_blob_to_file(conn, hash, None).await + let hash = hash.parse::()?; + let peer = peer.parse::()?; + let peer_addr = peer_addr.parse()?; + let conn = node1 + .dial(&iroh::bytes::protocol::ALPN, peer, &vec![peer_addr]) + .await?; + get_blob_to_file(conn, hash, None, out_path).await } .await; match result { - Ok() => rt.spawn_blocking(move || callback(None)), - Err(error) => rt.spawn_blocking(move || callback(Some(IrohError::from(error).into()))), + Ok(()) => rt.spawn_blocking(move || callback(None)), + Err(error) => rt.spawn_blocking(move || callback(Some(IrohError::new(error).into()))), }; }); } @@ -42,25 +64,30 @@ pub fn iroh_get_ticket( out_path: char_p::Ref<'_>, callback: extern "C" fn(Option>), ) { + let ticket = ticket.to_string(); + let out_path = PathBuf::from(out_path.to_string()); + let rt = node.async_runtime(); node.async_runtime().spawn(async move { let result = async { - let out_path = PathBuf::from_str(out_path.to_str())?; - // let keypair = node.inner(). - let ticket = Ticket::from_str(ticket.to_str())?; + let ticket = Ticket::from_str(ticket.as_str())?; // TODO(b5): use the node endpoint(s) to dial - let conn = node.inner().clone().dial( - ticket.peer(), - ticket.addrs(), - &iroh_bytes::protocol::ALPN, - ); - get_blob_to_file(conn, ticket.hash(), ticket.token(), out_path).await + let conn = node + .inner() + .clone() + .dial( + &iroh::bytes::protocol::ALPN, + ticket.peer(), + &ticket.addrs().to_vec(), + ) + .await?; + get_blob_to_file(conn, ticket.hash(), ticket.token().cloned(), out_path).await } .await; match result { - Ok() => rt.spawn_blocking(move || callback(None)), - Err(error) => rt.spawn_blocking(move || callback(Some(IrohError::from(error).into()))), + Ok(()) => rt.spawn_blocking(move || callback(None)), + Err(error) => rt.spawn_blocking(move || callback(Some(IrohError::new(error).into()))), }; }); } @@ -78,7 +105,7 @@ async fn get_blob_to_file( RangeSpecSeq::new([RangeSet2::all()]), out_path, ) - .await? + .await } // TODO(b5): This currently assumes "all" ranges, needs to be adjusted to honor @@ -90,12 +117,12 @@ async fn get_blob_ranges_to_file( ranges: RangeSpecSeq, out_path: PathBuf, ) -> Result<()> { - let request = Request::Get(GetRequest::new(hash, RangeSpecSeq::new([RangeSet2::all()]))); + let request = Request::Get(GetRequest::new(hash, ranges)).with_token(token); let response = fsm::start(conn, request); let connected = response.next().await?; let fsm::ConnectedNext::StartRoot(curr) = connected.next().await? else { - return Ok(None) + return Ok(()) }; let header = curr.next(); diff --git a/iroh-ffi/src/lib.rs b/iroh-ffi/src/lib.rs index ebcf275273..33052dc2f5 100644 --- a/iroh-ffi/src/lib.rs +++ b/iroh-ffi/src/lib.rs @@ -1,272 +1,16 @@ -// use std::{path::PathBuf, str::FromStr}; - -// use ::safer_ffi::prelude::*; -// use anyhow::{Context, Result}; -// use indicatif::{HumanBytes, HumanDuration}; -// use tokio::io::AsyncWriteExt; -// use tokio::runtime::Runtime; - -// use crate::util::Blake3Cid; -// use iroh::blobs::{Blob, Collection}; -// use iroh::get::get_response_machine::{ConnectedNext, EndBlobNext}; -// use iroh::get::{self, get_data_path, get_missing_range, get_missing_ranges, pathbuf_from_name}; -// use iroh::protocol::GetRequest; -// use iroh::provider::Ticket; -// use iroh::tokio_util::SeekOptimized; -// use iroh::{Hash, PeerId}; - -// mod util; - -// const MAX_CONCURRENT_DIALS: u8 = 16; - -// #[ffi_export] -// fn iroh_get( -// hash: char_p::Ref<'_>, -// peer: char_p::Ref<'_>, -// peer_addr: char_p::Ref<'_>, -// out_path: char_p::Ref<'_>, -// ) -> u32 { -// let result = std::panic::catch_unwind(|| { -// let hash = hash.to_str().parse::().unwrap(); -// let peer = peer.to_str().parse::().unwrap(); -// let peer_addr = peer_addr.to_str().parse().unwrap(); -// let out_path = PathBuf::from_str(out_path.to_str()).unwrap(); -// let rt = Runtime::new().unwrap(); - -// rt.block_on(get_to_dir( -// GetInteractive::Hash { -// hash, -// opts: get::Options { -// peer_id: Some(peer), -// addr: peer_addr, -// keylog: false, -// }, -// single: false, -// }, -// out_path, -// )) -// .unwrap(); -// 0 -// }); -// if result.is_err() { -// eprintln!("error: rust panicked"); -// return 1; -// } -// result.unwrap() -// } - -// #[ffi_export] -// fn iroh_get_ticket(ticket: char_p::Ref<'_>, out_path: char_p::Ref<'_>) -> u32 { -// let result = std::panic::catch_unwind(|| { -// let ticket = ticket.to_str().parse::().unwrap(); -// let out_path = PathBuf::from_str(out_path.to_str()).unwrap(); -// let rt = Runtime::new().unwrap(); -// rt.block_on(get_to_dir( -// GetInteractive::Ticket { -// ticket, -// keylog: false, -// }, -// out_path, -// )) -// .unwrap(); -// 0 -// }); -// if result.is_err() { -// eprintln!("error: rust panicked"); -// return 1; -// } -// result.unwrap() -// } - -// #[derive(Debug)] -// enum GetInteractive { -// Ticket { -// ticket: Ticket, -// keylog: bool, -// }, -// Hash { -// hash: Hash, -// opts: get::Options, -// single: bool, -// }, -// } - -// impl GetInteractive { -// fn hash(&self) -> Hash { -// match self { -// GetInteractive::Ticket { ticket, .. } => ticket.hash(), -// GetInteractive::Hash { hash, .. } => *hash, -// } -// } - -// fn single(&self) -> bool { -// match self { -// GetInteractive::Ticket { .. } => false, -// GetInteractive::Hash { single, .. } => *single, -// } -// } -// } - -// /// Get into a file or directory -// async fn get_to_dir(get: GetInteractive, out_dir: PathBuf) -> Result<()> { -// let hash = get.hash(); -// let single = get.single(); -// println!("Fetching: {}", Blake3Cid::new(hash)); -// println!("[1/3] Connecting ..."); - -// let temp_dir = out_dir.join(".iroh-tmp"); -// let (query, collection) = if single { -// let name = Blake3Cid::new(hash).to_string(); -// let query = get_missing_range(&get.hash(), name.as_str(), &temp_dir, &out_dir)?; -// (query, vec![Blob { hash, name }]) -// } else { -// let (query, collection) = get_missing_ranges(get.hash(), &out_dir, &temp_dir)?; -// ( -// query, -// collection.map(|x| x.into_inner()).unwrap_or_default(), -// ) -// }; - -// let init_download_progress = |count: u64, missing_bytes: u64| { -// println!("[3/3] Downloading ..."); -// println!( -// " {} file(s) with total transfer size {}", -// count, -// HumanBytes(missing_bytes) -// ); -// }; - -// // collection info, in case we won't get a callback with is_root -// let collection_info = if collection.is_empty() { -// None -// } else { -// Some((collection.len() as u64, 0)) -// }; - -// let request = GetRequest::new(get.hash(), query).into(); -// let response = match get { -// GetInteractive::Ticket { ticket, keylog } => { -// get::run_ticket(&ticket, request, keylog, MAX_CONCURRENT_DIALS).await? -// } -// GetInteractive::Hash { opts, .. } => get::run(request, opts).await?, -// }; -// let connected = response.next().await?; -// println!("[2/3] Requesting ..."); -// if let Some((count, missing_bytes)) = collection_info { -// init_download_progress(count, missing_bytes); -// } -// let (mut next, collection) = match connected.next().await? { -// ConnectedNext::StartRoot(curr) => { -// tokio::fs::create_dir_all(&temp_dir) -// .await -// .context("unable to create directory {temp_dir}")?; -// tokio::fs::create_dir_all(&out_dir) -// .await -// .context("Unable to create directory {out_dir}")?; -// let curr = curr.next(); -// let (curr, collection_data) = curr.concatenate_into_vec().await?; -// let collection = Collection::from_bytes(&collection_data)?; -// init_download_progress(collection.total_entries(), collection.total_blobs_size()); -// tokio::fs::write(get_data_path(&temp_dir, hash), collection_data).await?; -// (curr.next(), collection.into_inner()) -// } -// ConnectedNext::StartChild(start_child) => { -// (EndBlobNext::MoreChildren(start_child), collection) -// } -// ConnectedNext::Closing(finish) => (EndBlobNext::Closing(finish), collection), -// }; -// // read all the children -// let finishing = loop { -// let start = match next { -// EndBlobNext::MoreChildren(sc) => sc, -// EndBlobNext::Closing(finish) => break finish, -// }; -// let child_offset = start.child_offset() as usize; -// let blob = match collection.get(child_offset) { -// Some(blob) => blob, -// None => break start.finish(), -// }; - -// let hash = blob.hash; -// let name = &blob.name; -// let name = if name.is_empty() { -// PathBuf::from(hash.to_string()) -// } else { -// pathbuf_from_name(name) -// }; -// // pb.set_message(format!("Receiving '{}'...", name.display())); -// // pb.reset(); -// let header = start.next(blob.hash); - -// let curr = { -// let final_path = out_dir.join(&name); -// let tempname = blake3::Hash::from(hash).to_hex(); -// let data_path = temp_dir.join(format!("{}.data.part", tempname)); -// let outboard_path = temp_dir.join(format!("{}.outboard.part", tempname)); -// let data_file = tokio::fs::OpenOptions::new() -// .write(true) -// .create(true) -// .open(&data_path) -// .await?; -// let mut data_file = SeekOptimized::new(data_file).into(); -// let (curr, size) = header.next().await?; -// // pb.set_length(size); -// let mut outboard_file = if size > 0 { -// let outboard_file = tokio::fs::OpenOptions::new() -// .write(true) -// .create(true) -// .open(&outboard_path) -// .await?; -// let outboard_file = SeekOptimized::new(outboard_file).into(); -// Some(outboard_file) -// } else { -// None -// }; -// let curr = curr -// .write_all_with_outboard(&mut outboard_file, &mut data_file) -// .await?; -// tokio::fs::create_dir_all( -// final_path -// .parent() -// .context("final path should have parent")?, -// ) -// .await?; -// // Flush the data file first, it is the only thing that matters at this point -// data_file.into_inner().shutdown().await?; -// // Rename temp file, to target name -// // once this is done, the file is considered complete -// tokio::fs::rename(data_path, final_path).await?; -// if let Some(outboard_file) = outboard_file.take() { -// // not sure if we have to do this -// outboard_file.into_inner().shutdown().await?; -// // delete the outboard file -// tokio::fs::remove_file(outboard_path).await?; -// } -// curr -// }; -// next = curr.next(); -// }; -// let stats = finishing.next().await?; -// tokio::fs::remove_dir_all(temp_dir).await?; -// println!( -// "Transferred {} in {}, {}/s", -// HumanBytes(stats.bytes_read), -// HumanDuration(stats.elapsed), -// HumanBytes((stats.bytes_read as f64 / stats.elapsed.as_secs_f64()) as u64) -// ); - -// Ok(()) -// } - -// #[cfg(test)] -// mod tests { - -// /// The following test function is necessary for the header generation. - -// #[test] -// fn get_ticket_test() { -// // TODO -// // let result = get_ticket(); -// // assert_eq!(result, 0); -// } -// } +use safer_ffi::prelude::*; + +pub mod error; +pub mod get; +pub mod node; + +#[cfg(feature = "c-headers")] +pub fn generate_headers() -> std::io::Result<()> { + safer_ffi::headers::builder().to_file("iroh.h")?.generate() +} + +#[ffi_export] +/// Deallocate an Iroh-allocated string. +pub fn iroh_string_free(string: char_p::Box) { + drop(string) +} diff --git a/iroh-ffi/src/mod.rs b/iroh-ffi/src/mod.rs deleted file mode 100644 index c0448de2ac..0000000000 --- a/iroh-ffi/src/mod.rs +++ /dev/null @@ -1,16 +0,0 @@ -mod get; -mod node; - -pub use crate::ffi::get::*; -pub use crate::ffi::node::*; - -#[cfg(feature = "headers")] -pub fn generate_headers() -> std::io::Result<()> { - safer_ffi::headers::builder().to_file("iroh.h")?.generate() -} - -#[ffi_export] -/// Deallocate an Iroh-allocated string. -pub fn iroh_string_free(string: char_p::Box) { - drop(string) -} diff --git a/iroh-ffi/src/node.rs b/iroh-ffi/src/node.rs index 4a0c4132cf..d3ad7d982f 100644 --- a/iroh-ffi/src/node.rs +++ b/iroh-ffi/src/node.rs @@ -1,31 +1,32 @@ -use anyhow::Result; +use std::sync::Arc; + use safer_ffi::prelude::*; -use tokio::Runtime; +use tokio::runtime::Runtime as TokioRuntime; use iroh::{ + database::mem, + net::tls::Keypair, node::{Node, DEFAULT_BIND_ADDR}, - provider::Database, }; -use iroh_net::tls::Keypair; #[derive_ReprC(rename = "iroh_node")] #[repr(opaque)] /// @class iroh_node_t pub struct IrohNode { - inner: Node, + inner: Node, async_runtime: Arc, } impl IrohNode { - pub fn async_runtime(&self) -> Arc { + pub fn async_runtime(&self) -> Arc { self.async_runtime.clone() } - pub fn inner(&self) -> &Node { + pub fn inner(&self) -> &Node { &self.inner } - pub fn inner_mut(&mut self) -> &mut Node { + pub fn inner_mut(&mut self) -> &mut Node { &mut self.inner } } @@ -39,24 +40,38 @@ pub fn iroh_initialize() -> Option> { .thread_name("main-runtime") .worker_threads(2) .enable_all() - .build()?; + .build() + .ok()?; let tokio = tokio::runtime::Handle::current(); let tpc = tokio_util::task::LocalPoolHandle::new(num_cpus::get()); - let rt = iroh::bytes::runtime::Handle::new(tokio, tpc); + let rt = iroh::bytes::util::runtime::Handle::new(tokio, tpc); - let db = Database::default(); + let db = mem::Database::default(); let keypair = Keypair::generate(); - let node = Node::builder(db) - .bind_addr(DEFAULT_BIND_ADDR) - .keypair(keypair) - .runtime(rt) - .spawn() - .await?; - - repr_c::Box::new(IrohNode { - inner: node, - runtime: tokio_rt, - }) - .ok() + let node = tokio_rt + .block_on(async { + Node::builder(db) + .bind_addr(DEFAULT_BIND_ADDR.into()) + .keypair(keypair) + .runtime(&rt) + .spawn() + .await + }) + .unwrap(); + + Some( + Box::new(IrohNode { + inner: node, + async_runtime: Arc::new(tokio_rt), + }) + .into(), + ) +} + +#[ffi_export] +/// @memberof iroh_node_t +/// Deallocate a ns_noosphere_t instance. +pub fn iroh_free(node: repr_c::Box) { + drop(node) } diff --git a/iroh-ffi/src/util.rs b/iroh-ffi/src/util.rs deleted file mode 100644 index 4c759d94be..0000000000 --- a/iroh-ffi/src/util.rs +++ /dev/null @@ -1,90 +0,0 @@ -// //! Utility functions and types. -// use std::{fmt, str::FromStr}; - -// use anyhow::Result; -// use iroh::Hash; - -// #[repr(transparent)] -// #[derive(Debug, Clone, Copy, PartialEq, Eq)] -// pub struct Blake3Cid(pub Hash); - -// const CID_PREFIX: [u8; 4] = [ -// 0x01, // version -// 0x55, // raw codec -// 0x1e, // hash function, blake3 -// 0x20, // hash size, 32 bytes -// ]; - -// impl Blake3Cid { -// pub fn new(hash: Hash) -> Self { -// Blake3Cid(hash) -// } - -// // pub fn as_hash(&self) -> &Hash { -// // &self.0 -// // } - -// pub fn as_bytes(&self) -> [u8; 36] { -// let hash: [u8; 32] = self.0.as_ref().try_into().unwrap(); -// let mut res = [0u8; 36]; -// res[0..4].copy_from_slice(&CID_PREFIX); -// res[4..36].copy_from_slice(&hash); -// res -// } - -// pub fn from_bytes(bytes: &[u8]) -> anyhow::Result { -// anyhow::ensure!( -// bytes.len() == 36, -// "invalid cid length, expected 36, got {}", -// bytes.len() -// ); -// anyhow::ensure!(bytes[0..4] == CID_PREFIX, "invalid cid prefix"); -// let mut hash = [0u8; 32]; -// hash.copy_from_slice(&bytes[4..36]); -// Ok(Blake3Cid(Hash::from(hash))) -// } -// } - -// impl fmt::Display for Blake3Cid { -// fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { -// // result will be 58 bytes plus prefix -// let mut res = [b'b'; 59]; -// // write the encoded bytes -// data_encoding::BASE32_NOPAD.encode_mut(&self.as_bytes(), &mut res[1..]); -// // convert to string, this is guaranteed to succeed -// let t = std::str::from_utf8_mut(res.as_mut()).unwrap(); -// // hack since data_encoding doesn't have BASE32LOWER_NOPAD as a const -// t.make_ascii_lowercase(); -// // write the str, no allocations -// f.write_str(t) -// } -// } - -// impl FromStr for Blake3Cid { -// type Err = anyhow::Error; - -// fn from_str(s: &str) -> Result { -// let sb = s.as_bytes(); -// if sb.len() == 59 && sb[0] == b'b' { -// // this is a base32 encoded cid, we can decode it directly -// let mut t = [0u8; 58]; -// t.copy_from_slice(&sb[1..]); -// // hack since data_encoding doesn't have BASE32LOWER_NOPAD as a const -// std::str::from_utf8_mut(t.as_mut()) -// .unwrap() -// .make_ascii_uppercase(); -// // decode the bytes -// let mut res = [0u8; 36]; -// data_encoding::BASE32_NOPAD -// .decode_mut(&t, &mut res) -// .map_err(|_e| anyhow::anyhow!("invalid base32"))?; -// // convert to cid, this will check the prefix -// Self::from_bytes(&res) -// } else { -// // if we want to support all the weird multibase prefixes, we have no choice -// // but to use the multibase crate -// let (_base, bytes) = multibase::decode(s)?; -// Self::from_bytes(bytes.as_ref()) -// } -// } -// } diff --git a/iroh/src/node.rs b/iroh/src/node.rs index d47aa0c6f0..45f7a3d73a 100644 --- a/iroh/src/node.rs +++ b/iroh/src/node.rs @@ -547,11 +547,11 @@ impl Node { &self, alpn: &[u8], peer_id: PeerId, - known_addrs: Vec, + known_addrs: &Vec, ) -> Result { self.inner .endpoint - .connect(peer_id, alpn, &known_addrs) + .connect(peer_id, alpn, known_addrs) .await .context("failed to dial") } diff --git a/scripts/build-xcframework.sh b/scripts/build-xcframework.sh new file mode 100755 index 0000000000..66cb3f16cf --- /dev/null +++ b/scripts/build-xcframework.sh @@ -0,0 +1,56 @@ +# export RUSTFLAGS="-C embed-bitcode=yes" +# export CARGO_TARGET_AARCH64_APPLE_IOS_SIM_LINKER="/usr/bin/clang" +# export LIBRARY_PATH="//usr/lib" +export PATH="/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin" +export PATH="$HOME/.cargo/bin:$PATH" + +REPO_ROOT=".." +RUST_FFI_DIR="../iroh-ffi" +OUT_DIR="../build" +RUST_TOOLCHAIN="1.71.0" + +echo "Generate Iroh C header, copy Module map" +mkdir -p "${OUT_DIR}/include" +cargo +${RUST_TOOLCHAIN} test --features c-headers --manifest-path "${RUST_FFI_DIR}/Cargo.toml" -- generate_headers +cp ${RUST_FFI_DIR}/libiroh.h ${OUT_DIR}/include/iroh.h +cp ${REPO_ROOT}/swift/include/module.modulemap ${OUT_DIR}/include/module.modulemap + +echo "Build Iroh Libraries for Apple Platforms" + +targets=( + "aarch64-apple-ios" + "x86_64-apple-ios" + "aarch64-apple-ios-sim" +) + +for target in "${targets[@]}"; do + cargo +`cat ${REPO_ROOT}/rust-toolchain` build --package iroh_ffi --release --target "${target}" --manifest-path "${RUST_FFI_DIR}/Cargo.toml" + mkdir -p "${OUT_DIR}/lib_${target}" + cp "${RUST_FFI_DIR}/target/${target}/release/libiroh.a" "${OUT_DIR}/lib_${target}/libiroh.a" +done + +echo "Run Lipo" +mkdir -p "${OUT_DIR}/lib_ios-simulator-universal" +# cargo +`cat ${REPO_ROOT}/rust-toolchain` lipo \ +# --targets aarch64-apple-ios-sim,x86_64-apple-ios \ +# --manifest-path "${RUST_FFI_DIR}/Cargo.toml" +# cp "${RUST_FFI_DIR}/target/universal/release/libiroh.a" "${OUT_DIR}/lib_ios-simulator-universal/libiroh.a" + +lipo -create \ + "${OUT_DIR}/lib_x86_64-apple-ios/libiroh.a" \ + "${OUT_DIR}/lib_aarch64-apple-ios-sim/libiroh.a" \ + -output "${OUT_DIR}/lib_ios-simulator-universal/libiroh.a" + + +echo "Create XCFramework" + +xcodebuild -create-xcframework \ + -library ${OUT_DIR}/lib_ios-simulator-universal/libiroh.a \ + -headers ${OUT_DIR}/include/ \ + -library ${OUT_DIR}/lib_aarch64-apple-ios/libiroh.a \ + -headers ${OUT_DIR}/include/ \ + -output ${REPO_ROOT}/LibIroh.xcframework BUILD_LIBRARY_FOR_DISTRIBUTION=YES + +zip -r ${REPO_ROOT}/libiroh-xcframework.zip ${REPO_ROOT}/LibIroh.xcframework + +echo "Done" \ No newline at end of file diff --git a/swift/Sources/Iroh/Iroh.swift b/swift/Sources/Iroh/Iroh.swift new file mode 100644 index 0000000000..cecb69ac35 --- /dev/null +++ b/swift/Sources/Iroh/Iroh.swift @@ -0,0 +1,21 @@ +@_exported import Iroh + +import Foundation + +public enum IrohError: Error { + case unexpected(UInt32) +} + +public func irohGet(cid: String, peer: String, peerAddr:String, outPath:String) throws { + let status = iroh_get(cid, peer, peerAddr, outPath) + guard status == errSecSuccess else { + throw IrohError.unexpected(status) + } +} + +public func irohGetTicket(ticket: String, outPath: String) throws { + let status = iroh_get_ticket(ticket, outPath) + guard status == errSecSuccess else { + throw IrohError.unexpected(status) + } +} diff --git a/swift/Tests/IrohTests/IrohTests.swift b/swift/Tests/IrohTests/IrohTests.swift new file mode 100644 index 0000000000..46711ceb8f --- /dev/null +++ b/swift/Tests/IrohTests/IrohTests.swift @@ -0,0 +1,8 @@ +import XCTest +@testable import IrohSwift + +final class IrohSwiftTests: XCTestCase { + func testIrohGet() throws { + irhoGet("", "/tmp") + } +} diff --git a/swift/include/module.modulemap b/swift/include/module.modulemap new file mode 100644 index 0000000000..b72894d657 --- /dev/null +++ b/swift/include/module.modulemap @@ -0,0 +1,4 @@ +module Iroh { + header "iroh.h" + export * +} \ No newline at end of file From ad0e3a693770ab4edd8ef442a266d79ed7355af8 Mon Sep 17 00:00:00 2001 From: b5 Date: Fri, 14 Jul 2023 22:45:08 -0400 Subject: [PATCH 17/32] higher ranked lifetime error --- iroh-ffi/iroh.h | 101 +++++++++++++++++++++++++++ iroh-ffi/src/bin/generate-headers.rs | 13 +++- iroh-ffi/src/error.rs | 8 +-- iroh-ffi/src/get.rs | 91 ++++++++++++------------ iroh-ffi/src/node.rs | 4 -- iroh/src/node.rs | 29 +++++--- 6 files changed, 177 insertions(+), 69 deletions(-) create mode 100644 iroh-ffi/iroh.h diff --git a/iroh-ffi/iroh.h b/iroh-ffi/iroh.h new file mode 100644 index 0000000000..527c8a2d35 --- /dev/null +++ b/iroh-ffi/iroh.h @@ -0,0 +1,101 @@ +/*! \file */ +/******************************************* + * * + * File auto-generated by `::safer_ffi`. * + * * + * Do not manually edit this file. * + * * + *******************************************/ + +#ifndef __RUST_IROH_FFI__ +#define __RUST_IROH_FFI__ +#ifdef __cplusplus +extern "C" { +#endif + + +#include +#include + +/** \brief + * Constant values for error codes from iroh_error_t. + */ +/** \remark Has the same ABI as `uint32_t` **/ +#ifdef DOXYGEN +typedef +#endif +enum iroh_error_code { + /** */ + IROH_ERROR_CODE_OTHER = 1, +} +#ifndef DOXYGEN +; typedef uint32_t +#endif +iroh_error_code_t; + +/** \brief + * @class iroh_error_t + * An opaque struct representing an error. + */ +typedef struct iroh_error iroh_error_t; + +/** \brief + * @memberof ns_error_t + * Deallocate an ns_error_t. + */ +void +iroh_error_free ( + iroh_error_t * error); + +/** \brief + * @memberof iroh_error_t + * Returns an owned string describing the error in greater detail. + * + * Caller is responsible for deallocating returned string via iroh_string_free. + */ +char * +iroh_error_message_get ( + iroh_error_t const * error); + +/** \brief + * @class iroh_node_t + */ +typedef struct iroh_node iroh_node_t; + +/** \brief + * @memberof iroh_node_t + * Deallocate a ns_noosphere_t instance. + */ +void +iroh_free ( + iroh_node_t * node); + +/** \brief + * @memberof iroh_node_t + * Initialize a iroh_node_t instance. + * + */ +iroh_node_t * +iroh_initialize (void); + +/** \brief + * Deallocate an Iroh-allocated string. + */ +void +iroh_string_free ( + char * string); + +/** \brief + * @memberof ns_error_t + * Returns an error code that identifies the error. + */ +uint32_t +ns_error_code_get ( + iroh_error_t const * error); + + +#ifdef __cplusplus +} /* extern \"C\" */ +#endif + +#endif /* __RUST_IROH_FFI__ */ diff --git a/iroh-ffi/src/bin/generate-headers.rs b/iroh-ffi/src/bin/generate-headers.rs index 2c0dee0356..20097ec2dd 100644 --- a/iroh-ffi/src/bin/generate-headers.rs +++ b/iroh-ffi/src/bin/generate-headers.rs @@ -1,3 +1,12 @@ -fn main() -> std::io::Result<()> { - iroh_ffi::generate_headers() +use anyhow::{anyhow, Result}; + +#[cfg(feature = "c-headers")] +fn main() -> Result<()> { + iroh_ffi::generate_headers().map_err(|e| anyhow!(e.to_string()))?; + Ok(()) +} + +#[cfg(not(feature = "c-headers"))] +fn main() -> Result<()> { + Err(anyhow!("Must run with --features c-headers")) } diff --git a/iroh-ffi/src/error.rs b/iroh-ffi/src/error.rs index 85b98592d8..c1031a5b56 100644 --- a/iroh-ffi/src/error.rs +++ b/iroh-ffi/src/error.rs @@ -50,8 +50,8 @@ impl IrohError { } #[ffi_export] -/// @memberof ns_error_t -/// Deallocate an ns_error_t. +/// @memberof iroh_error_t +/// Deallocate an iroh_error_t. pub fn iroh_error_free(error: repr_c::Box) { drop(error) } @@ -70,8 +70,8 @@ pub fn iroh_error_message_get(error: &IrohError) -> char_p::Box { } #[ffi_export] -/// @memberof ns_error_t +/// @memberof iroh_error_t /// Returns an error code that identifies the error. -pub fn ns_error_code_get(error: &IrohError) -> u32 { +pub fn iroh_error_code_get(error: &IrohError) -> u32 { IrohErrorCode::from(error) as u32 } diff --git a/iroh-ffi/src/get.rs b/iroh-ffi/src/get.rs index 3d4310ded3..5e4601ebd0 100644 --- a/iroh-ffi/src/get.rs +++ b/iroh-ffi/src/get.rs @@ -12,75 +12,70 @@ use iroh::{ protocol::{GetRequest, RangeSpecSeq, Request, RequestToken}, Hash, }, - dial::Ticket, - net::tls::PeerId, + dial::{dial, Ticket}, + // net::tls::PeerId, }; use crate::{error::IrohError, node::IrohNode}; -#[ffi_export] -/// @memberof iroh_node_t -// TODO(b5): optional token arg -fn iroh_get( - node: &mut IrohNode, - hash: char_p::Ref<'_>, - peer: char_p::Ref<'_>, - peer_addr: char_p::Ref<'_>, - out_path: char_p::Ref<'_>, - callback: extern "C" fn(Option>), -) { - let node1 = node.inner().clone(); - let rt = node.async_runtime(); - let hash = hash.to_string(); - let peer = peer.to_string(); - let peer_addr = peer_addr.to_string(); - let out_path = PathBuf::from(out_path.to_string()); +// #[ffi_export] +// /// @memberof iroh_node_t +// // TODO(b5): optional token arg +// fn iroh_get( +// node: &IrohNode, +// hash: char_p::Ref<'_>, +// peer: char_p::Ref<'_>, +// peer_addr: char_p::Ref<'_>, +// out_path: char_p::Ref<'_>, +// callback: extern "C" fn(Option>), +// ) { +// let node1 = node.inner(); +// let rt = node.async_runtime(); +// let hash = hash.to_string(); +// let peer = peer.to_string(); +// let peer_addr = peer_addr.to_string(); +// let out_path = PathBuf::from(out_path.to_string()); - node.async_runtime().clone().spawn(async move { - let result = async move { - let hash = hash.parse::()?; - let peer = peer.parse::()?; - let peer_addr = peer_addr.parse()?; - let conn = node1 - .dial(&iroh::bytes::protocol::ALPN, peer, &vec![peer_addr]) - .await?; - get_blob_to_file(conn, hash, None, out_path).await - } - .await; +// rt.spawn(async move { +// let result = async { +// let hash = hash.parse::()?; +// let peer = peer.parse::()?; +// let peer_addr = peer_addr.parse()?; - match result { - Ok(()) => rt.spawn_blocking(move || callback(None)), - Err(error) => rt.spawn_blocking(move || callback(Some(IrohError::new(error).into()))), - }; - }); -} +// let conn = node1 +// .dial(&iroh::bytes::protocol::ALPN, peer, &vec![peer_addr]) +// .await?; +// get_blob_to_file(conn, hash, None, out_path).await +// } +// .await; + +// match result { +// Ok(()) => rt.spawn_blocking(move || callback(None)), +// Err(error) => rt.spawn_blocking(move || callback(Some(IrohError::new(error).into()))), +// }; +// }); +// } #[ffi_export] /// @memberof iroh_node_t /// Get a collection from a peer. pub fn iroh_get_ticket( - node: &mut IrohNode, + node: &IrohNode, ticket: char_p::Ref<'_>, out_path: char_p::Ref<'_>, callback: extern "C" fn(Option>), ) { let ticket = ticket.to_string(); let out_path = PathBuf::from(out_path.to_string()); - + let keypair = node.inner().keypair(); let rt = node.async_runtime(); + node.async_runtime().spawn(async move { let result = async { let ticket = Ticket::from_str(ticket.as_str())?; - // TODO(b5): use the node endpoint(s) to dial - let conn = node - .inner() - .clone() - .dial( - &iroh::bytes::protocol::ALPN, - ticket.peer(), - &ticket.addrs().to_vec(), - ) - .await?; + // TODO(b5): pull DerpMap from node, feed into here: + let opts = ticket.as_get_options(keypair, None); + let conn = dial(opts, &iroh::bytes::protocol::ALPN).await?; get_blob_to_file(conn, ticket.hash(), ticket.token().cloned(), out_path).await } .await; diff --git a/iroh-ffi/src/node.rs b/iroh-ffi/src/node.rs index d3ad7d982f..2124a9ee0d 100644 --- a/iroh-ffi/src/node.rs +++ b/iroh-ffi/src/node.rs @@ -25,10 +25,6 @@ impl IrohNode { pub fn inner(&self) -> &Node { &self.inner } - - pub fn inner_mut(&mut self) -> &mut Node { - &mut self.inner - } } #[ffi_export] diff --git a/iroh/src/node.rs b/iroh/src/node.rs index 45f7a3d73a..dc005456c3 100644 --- a/iroh/src/node.rs +++ b/iroh/src/node.rs @@ -543,17 +543,24 @@ impl Node { } /// Dial a given peer - pub async fn dial( - &self, - alpn: &[u8], - peer_id: PeerId, - known_addrs: &Vec, - ) -> Result { - self.inner - .endpoint - .connect(peer_id, alpn, known_addrs) - .await - .context("failed to dial") + /// TODO(b5): I'd like to use this in iroh_ffi to have the node manage + /// a single endpoint, but I run into lifetime errors when trying to work + /// this way + // pub async fn dial( + // &self, + // alpn: &[u8], + // peer_id: PeerId, + // known_addrs: &Vec, + // ) -> Result { + // self.inner + // .endpoint + // .connect(peer_id, alpn, known_addrs) + // .await + // .context("failed to dial") + // } + + pub fn keypair(&self) -> Keypair { + self.inner.keypair.clone() } /// Subscribe to [`Event`]s emitted from the node, informing about connections and From 7ba860d5b6698e290b79705f295307d6d32cbfb0 Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Sat, 15 Jul 2023 12:31:43 +0200 Subject: [PATCH 18/32] fix runtime execution --- iroh-ffi/src/get.rs | 47 +++++++++++++++++++++++++++----------------- iroh-ffi/src/node.rs | 18 ++++++++--------- 2 files changed, 37 insertions(+), 28 deletions(-) diff --git a/iroh-ffi/src/get.rs b/iroh-ffi/src/get.rs index 5e4601ebd0..212631a171 100644 --- a/iroh-ffi/src/get.rs +++ b/iroh-ffi/src/get.rs @@ -4,7 +4,7 @@ use std::str::FromStr; use anyhow::Result; use iroh_io::{AsyncSliceWriter, File}; use range_collections::RangeSet2; -use safer_ffi::prelude::*; +use safer_ffi::{closure::ArcDynFn1, prelude::*}; use iroh::{ bytes::{ @@ -68,23 +68,34 @@ pub fn iroh_get_ticket( let ticket = ticket.to_string(); let out_path = PathBuf::from(out_path.to_string()); let keypair = node.inner().keypair(); - let rt = node.async_runtime(); - - node.async_runtime().spawn(async move { - let result = async { - let ticket = Ticket::from_str(ticket.as_str())?; - // TODO(b5): pull DerpMap from node, feed into here: - let opts = ticket.as_get_options(keypair, None); - let conn = dial(opts, &iroh::bytes::protocol::ALPN).await?; - get_blob_to_file(conn, ticket.hash(), ticket.token().cloned(), out_path).await - } - .await; - - match result { - Ok(()) => rt.spawn_blocking(move || callback(None)), - Err(error) => rt.spawn_blocking(move || callback(Some(IrohError::new(error).into()))), - }; - }); + let rt = node.async_runtime().clone(); + node.async_runtime() + .local_pool() + .spawn_pinned(move || async move { + let result = async { + let ticket = Ticket::from_str(&ticket)?; + + // TODO(b5): pull DerpMap from node, feed into here: + let opts = ticket.as_get_options(keypair, None); + let conn = dial(opts, &iroh::bytes::protocol::ALPN).await?; + get_blob_to_file(conn, ticket.hash(), ticket.token().cloned(), out_path).await?; + Ok(()) + } + .await; + + let result = as_opt_err(result); + rt.main() + .spawn_blocking(move || callback(result)) + .await + .ok(); + }); +} + +fn as_opt_err(res: Result<()>) -> Option> { + match res { + Ok(()) => None, + Err(err) => Some(IrohError::new(err).into()), + } } async fn get_blob_to_file( diff --git a/iroh-ffi/src/node.rs b/iroh-ffi/src/node.rs index 2124a9ee0d..a4a88605ec 100644 --- a/iroh-ffi/src/node.rs +++ b/iroh-ffi/src/node.rs @@ -1,9 +1,7 @@ -use std::sync::Arc; - use safer_ffi::prelude::*; -use tokio::runtime::Runtime as TokioRuntime; use iroh::{ + bytes::util::runtime::Handle, database::mem, net::tls::Keypair, node::{Node, DEFAULT_BIND_ADDR}, @@ -14,12 +12,12 @@ use iroh::{ /// @class iroh_node_t pub struct IrohNode { inner: Node, - async_runtime: Arc, + async_runtime: Handle, } impl IrohNode { - pub fn async_runtime(&self) -> Arc { - self.async_runtime.clone() + pub fn async_runtime(&self) -> &Handle { + &self.async_runtime } pub fn inner(&self) -> &Node { @@ -39,13 +37,13 @@ pub fn iroh_initialize() -> Option> { .build() .ok()?; - let tokio = tokio::runtime::Handle::current(); let tpc = tokio_util::task::LocalPoolHandle::new(num_cpus::get()); - let rt = iroh::bytes::util::runtime::Handle::new(tokio, tpc); + let rt = iroh::bytes::util::runtime::Handle::new(tokio_rt.handle().clone(), tpc); let db = mem::Database::default(); let keypair = Keypair::generate(); - let node = tokio_rt + let node = rt + .main() .block_on(async { Node::builder(db) .bind_addr(DEFAULT_BIND_ADDR.into()) @@ -59,7 +57,7 @@ pub fn iroh_initialize() -> Option> { Some( Box::new(IrohNode { inner: node, - async_runtime: Arc::new(tokio_rt), + async_runtime: rt, }) .into(), ) From 8591f16be7f42aeb3d385ea372df016dc3a21a02 Mon Sep 17 00:00:00 2001 From: b5 Date: Sat, 15 Jul 2023 14:42:31 -0400 Subject: [PATCH 19/32] cleanups --- iroh-ffi/Cargo.toml | 4 +- iroh-ffi/iroh.h | 101 ----------------------------------- iroh-ffi/src/error.rs | 2 + iroh-ffi/src/get.rs | 2 +- iroh-ffi/src/node.rs | 1 + scripts/build-xcframework.sh | 7 +-- 6 files changed, 10 insertions(+), 107 deletions(-) delete mode 100644 iroh-ffi/iroh.h diff --git a/iroh-ffi/Cargo.toml b/iroh-ffi/Cargo.toml index 9b84764449..dc0686ac35 100644 --- a/iroh-ffi/Cargo.toml +++ b/iroh-ffi/Cargo.toml @@ -10,8 +10,8 @@ authors = ["n0 team"] repository = "https://github.com/n0-computer/iroh" [lib] -# name="iroh" -crate-type = ["rlib", "staticlib", "cdylib"] +name="iroh" +crate-type = ["rlib", "staticlib"] [features] c-headers = ["safer-ffi/headers"] diff --git a/iroh-ffi/iroh.h b/iroh-ffi/iroh.h deleted file mode 100644 index 527c8a2d35..0000000000 --- a/iroh-ffi/iroh.h +++ /dev/null @@ -1,101 +0,0 @@ -/*! \file */ -/******************************************* - * * - * File auto-generated by `::safer_ffi`. * - * * - * Do not manually edit this file. * - * * - *******************************************/ - -#ifndef __RUST_IROH_FFI__ -#define __RUST_IROH_FFI__ -#ifdef __cplusplus -extern "C" { -#endif - - -#include -#include - -/** \brief - * Constant values for error codes from iroh_error_t. - */ -/** \remark Has the same ABI as `uint32_t` **/ -#ifdef DOXYGEN -typedef -#endif -enum iroh_error_code { - /** */ - IROH_ERROR_CODE_OTHER = 1, -} -#ifndef DOXYGEN -; typedef uint32_t -#endif -iroh_error_code_t; - -/** \brief - * @class iroh_error_t - * An opaque struct representing an error. - */ -typedef struct iroh_error iroh_error_t; - -/** \brief - * @memberof ns_error_t - * Deallocate an ns_error_t. - */ -void -iroh_error_free ( - iroh_error_t * error); - -/** \brief - * @memberof iroh_error_t - * Returns an owned string describing the error in greater detail. - * - * Caller is responsible for deallocating returned string via iroh_string_free. - */ -char * -iroh_error_message_get ( - iroh_error_t const * error); - -/** \brief - * @class iroh_node_t - */ -typedef struct iroh_node iroh_node_t; - -/** \brief - * @memberof iroh_node_t - * Deallocate a ns_noosphere_t instance. - */ -void -iroh_free ( - iroh_node_t * node); - -/** \brief - * @memberof iroh_node_t - * Initialize a iroh_node_t instance. - * - */ -iroh_node_t * -iroh_initialize (void); - -/** \brief - * Deallocate an Iroh-allocated string. - */ -void -iroh_string_free ( - char * string); - -/** \brief - * @memberof ns_error_t - * Returns an error code that identifies the error. - */ -uint32_t -ns_error_code_get ( - iroh_error_t const * error); - - -#ifdef __cplusplus -} /* extern \"C\" */ -#endif - -#endif /* __RUST_IROH_FFI__ */ diff --git a/iroh-ffi/src/error.rs b/iroh-ffi/src/error.rs index c1031a5b56..7fe9f89662 100644 --- a/iroh-ffi/src/error.rs +++ b/iroh-ffi/src/error.rs @@ -11,6 +11,7 @@ const IROH_ERROR_OTHER: u32 = 1; #[ffi_export] #[derive_ReprC(rename = "iroh_error_code")] #[repr(u32)] +#[derive(Debug)] /// Constant values for error codes from iroh_error_t. pub enum IrohErrorCode { Other = IROH_ERROR_OTHER, @@ -33,6 +34,7 @@ impl From<&IrohError> for IrohErrorCode { #[derive_ReprC(rename = "iroh_error")] #[repr(opaque)] +#[derive(Debug)] /// @class iroh_error_t /// An opaque struct representing an error. pub struct IrohError { diff --git a/iroh-ffi/src/get.rs b/iroh-ffi/src/get.rs index 212631a171..cfd966f11d 100644 --- a/iroh-ffi/src/get.rs +++ b/iroh-ffi/src/get.rs @@ -4,7 +4,7 @@ use std::str::FromStr; use anyhow::Result; use iroh_io::{AsyncSliceWriter, File}; use range_collections::RangeSet2; -use safer_ffi::{closure::ArcDynFn1, prelude::*}; +use safer_ffi::prelude::*; use iroh::{ bytes::{ diff --git a/iroh-ffi/src/node.rs b/iroh-ffi/src/node.rs index a4a88605ec..4a752c6956 100644 --- a/iroh-ffi/src/node.rs +++ b/iroh-ffi/src/node.rs @@ -9,6 +9,7 @@ use iroh::{ #[derive_ReprC(rename = "iroh_node")] #[repr(opaque)] +#[derive(Debug)] /// @class iroh_node_t pub struct IrohNode { inner: Node, diff --git a/scripts/build-xcframework.sh b/scripts/build-xcframework.sh index 66cb3f16cf..14d0555ee3 100755 --- a/scripts/build-xcframework.sh +++ b/scripts/build-xcframework.sh @@ -7,11 +7,12 @@ export PATH="$HOME/.cargo/bin:$PATH" REPO_ROOT=".." RUST_FFI_DIR="../iroh-ffi" OUT_DIR="../build" -RUST_TOOLCHAIN="1.71.0" +# TODO(b5): explicitly enforce build toolchain for all cargo invocations +# RUST_TOOLCHAIN="1.71.0" echo "Generate Iroh C header, copy Module map" mkdir -p "${OUT_DIR}/include" -cargo +${RUST_TOOLCHAIN} test --features c-headers --manifest-path "${RUST_FFI_DIR}/Cargo.toml" -- generate_headers +cargo run --features c-headers --manifest-path "${RUST_FFI_DIR}/Cargo.toml --bin generate-headers" cp ${RUST_FFI_DIR}/libiroh.h ${OUT_DIR}/include/iroh.h cp ${REPO_ROOT}/swift/include/module.modulemap ${OUT_DIR}/include/module.modulemap @@ -24,7 +25,7 @@ targets=( ) for target in "${targets[@]}"; do - cargo +`cat ${REPO_ROOT}/rust-toolchain` build --package iroh_ffi --release --target "${target}" --manifest-path "${RUST_FFI_DIR}/Cargo.toml" + cargo build --package iroh_ffi --release --target "${target}" --manifest-path "${RUST_FFI_DIR}/Cargo.toml" mkdir -p "${OUT_DIR}/lib_${target}" cp "${RUST_FFI_DIR}/target/${target}/release/libiroh.a" "${OUT_DIR}/lib_${target}/libiroh.a" done From ac98e7d584c066261895c9ce176cb3bfa4a0af78 Mon Sep 17 00:00:00 2001 From: b5 Date: Mon, 17 Jul 2023 10:27:46 -0400 Subject: [PATCH 20/32] WIP --- Package.swift | 1 + iroh-ffi/Cargo.toml | 2 +- scripts/build-xcframework.sh | 42 +++++++++++------- swift/Sources/Iroh/Iroh.swift | 33 +++++++------- swift/Tests/IrohTests/IrohTests.swift | 63 ++++++++++++++++++++++++++- swift/include/module.modulemap | 2 +- 6 files changed, 106 insertions(+), 37 deletions(-) diff --git a/Package.swift b/Package.swift index 441f19dd95..f71261ba72 100644 --- a/Package.swift +++ b/Package.swift @@ -7,6 +7,7 @@ let package = Package( name: "iroh", platforms: [ .iOS(.v13), + .macOS(.v11) ], products: [ // Products define the executables and libraries a package produces, diff --git a/iroh-ffi/Cargo.toml b/iroh-ffi/Cargo.toml index dc0686ac35..d8d4d02e79 100644 --- a/iroh-ffi/Cargo.toml +++ b/iroh-ffi/Cargo.toml @@ -11,7 +11,7 @@ repository = "https://github.com/n0-computer/iroh" [lib] name="iroh" -crate-type = ["rlib", "staticlib"] +crate-type = ["staticlib"] [features] c-headers = ["safer-ffi/headers"] diff --git a/scripts/build-xcframework.sh b/scripts/build-xcframework.sh index 14d0555ee3..705aefd05c 100755 --- a/scripts/build-xcframework.sh +++ b/scripts/build-xcframework.sh @@ -12,22 +12,24 @@ OUT_DIR="../build" echo "Generate Iroh C header, copy Module map" mkdir -p "${OUT_DIR}/include" -cargo run --features c-headers --manifest-path "${RUST_FFI_DIR}/Cargo.toml --bin generate-headers" -cp ${RUST_FFI_DIR}/libiroh.h ${OUT_DIR}/include/iroh.h +cargo run --features c-headers --bin generate-headers +mv iroh.h ${OUT_DIR}/include/iroh.h cp ${REPO_ROOT}/swift/include/module.modulemap ${OUT_DIR}/include/module.modulemap echo "Build Iroh Libraries for Apple Platforms" targets=( - "aarch64-apple-ios" - "x86_64-apple-ios" - "aarch64-apple-ios-sim" + # "aarch64-apple-ios" + # "x86_64-apple-ios" + # "aarch64-apple-ios-sim" + "x86_64-apple-darwin" + "aarch64-apple-darwin" ) for target in "${targets[@]}"; do - cargo build --package iroh_ffi --release --target "${target}" --manifest-path "${RUST_FFI_DIR}/Cargo.toml" - mkdir -p "${OUT_DIR}/lib_${target}" - cp "${RUST_FFI_DIR}/target/${target}/release/libiroh.a" "${OUT_DIR}/lib_${target}/libiroh.a" + cargo build --package iroh_ffi --release --target ${target} + mkdir -p ${OUT_DIR}/lib_${target} + cp "${REPO_ROOT}/target/${target}/release/libiroh_ffi.a" "${OUT_DIR}/lib_${target}/libiroh_ffi.a" done echo "Run Lipo" @@ -38,20 +40,28 @@ mkdir -p "${OUT_DIR}/lib_ios-simulator-universal" # cp "${RUST_FFI_DIR}/target/universal/release/libiroh.a" "${OUT_DIR}/lib_ios-simulator-universal/libiroh.a" lipo -create \ - "${OUT_DIR}/lib_x86_64-apple-ios/libiroh.a" \ - "${OUT_DIR}/lib_aarch64-apple-ios-sim/libiroh.a" \ - -output "${OUT_DIR}/lib_ios-simulator-universal/libiroh.a" + "${OUT_DIR}/lib_x86_64-apple-ios/libiroh_ffi.a" \ + "${OUT_DIR}/lib_aarch64-apple-ios-sim/libiroh_ffi.a" \ + -output "${OUT_DIR}/lib_ios-simulator-universal/libiroh_ffi.a" echo "Create XCFramework" +rm -rf ${REPO_ROOT}/target/LibIroh.xcframework + +# xcodebuild -create-xcframework \ +# -library ${OUT_DIR}/lib_ios-simulator-universal/libiroh_ffi.a \ +# -headers ${OUT_DIR}/include/ \ +# -library ${OUT_DIR}/lib_aarch64-apple-ios/libiroh_ffi.a \ +# -headers ${OUT_DIR}/include/ \ +# -output ${REPO_ROOT}/target/LibIroh.xcframework + xcodebuild -create-xcframework \ - -library ${OUT_DIR}/lib_ios-simulator-universal/libiroh.a \ - -headers ${OUT_DIR}/include/ \ - -library ${OUT_DIR}/lib_aarch64-apple-ios/libiroh.a \ + -library ${OUT_DIR}/lib_aarch64-apple-darwin/libiroh_ffi.a \ -headers ${OUT_DIR}/include/ \ - -output ${REPO_ROOT}/LibIroh.xcframework BUILD_LIBRARY_FOR_DISTRIBUTION=YES + -output ${REPO_ROOT}/target/LibIroh.xcframework -zip -r ${REPO_ROOT}/libiroh-xcframework.zip ${REPO_ROOT}/LibIroh.xcframework +# echo "Zip XCFramework" +# zip -r ${REPO_ROOT}/target/libiroh-xcframework.zip ${REPO_ROOT}/target/LibIroh.xcframework echo "Done" \ No newline at end of file diff --git a/swift/Sources/Iroh/Iroh.swift b/swift/Sources/Iroh/Iroh.swift index cecb69ac35..99c875d8b3 100644 --- a/swift/Sources/Iroh/Iroh.swift +++ b/swift/Sources/Iroh/Iroh.swift @@ -1,21 +1,20 @@ -@_exported import Iroh +@_exported import IrohFFI -import Foundation +// public enum IrohError: Error { +// case unknown(UInt32) +// } -public enum IrohError: Error { - case unexpected(UInt32) -} +// public func irohGet(cid: String, peer: String, peerAddr:String, outPath:String) throws { +// // let status = iroh_get(cid, peer, peerAddr, outPath) +// // guard status == errSecSuccess else { +// // throw IrohError.unexpected(status) +// // } +// } -public func irohGet(cid: String, peer: String, peerAddr:String, outPath:String) throws { - let status = iroh_get(cid, peer, peerAddr, outPath) - guard status == errSecSuccess else { - throw IrohError.unexpected(status) - } -} +// public func irohGetTicket(ticket: String, outPath: String) throws { +// let status = iroh_get_ticket(ticket, outPath) +// guard status == errSecSuccess else { +// throw IrohError.unexpected(status) +// } +// } -public func irohGetTicket(ticket: String, outPath: String) throws { - let status = iroh_get_ticket(ticket, outPath) - guard status == errSecSuccess else { - throw IrohError.unexpected(status) - } -} diff --git a/swift/Tests/IrohTests/IrohTests.swift b/swift/Tests/IrohTests/IrohTests.swift index 46711ceb8f..1059d5748f 100644 --- a/swift/Tests/IrohTests/IrohTests.swift +++ b/swift/Tests/IrohTests/IrohTests.swift @@ -1,8 +1,67 @@ import XCTest -@testable import IrohSwift +import Iroh final class IrohSwiftTests: XCTestCase { func testIrohGet() throws { - irhoGet("", "/tmp") + + } + + func testInitializeNoosphereThenWriteAFileThenSaveThenReadItBack() throws { + // This is a basic integration test to ensure that file writing and + // reading from swift works as intended + let iroh = iroh_initialize() + + // ns_tracing_initialize(NS_NOOSPHERE_LOG_CHATTY.rawValue) + // ns_key_create(noosphere, "bob", nil) + + // let sphere_receipt = ns_sphere_create(noosphere, "bob", nil) + + // let sphere_identity_ptr = ns_sphere_receipt_identity(sphere_receipt, nil) + // let sphere_mnemonic_ptr = ns_sphere_receipt_mnemonic(sphere_receipt, nil) + + // let sphere_identity = String.init(cString: sphere_identity_ptr!) + // let sphere_mnemonic = String.init(cString: sphere_mnemonic_ptr!) + + // print("Sphere identity:", sphere_identity) + // print("Recovery code:", sphere_mnemonic) + + // let sphere = ns_sphere_open(noosphere, sphere_identity_ptr, nil) + + // let file_bytes = "Hello, Subconscious".data(using: .utf8)! + + // file_bytes.withUnsafeBytes({ rawBufferPointer in + // let bufferPointer = rawBufferPointer.bindMemory(to: UInt8.self) + // let pointer = bufferPointer.baseAddress! + // let bodyRaw = slice_ref_uint8( + // ptr: pointer, len: file_bytes.count + // ) + // ns_sphere_content_write(noosphere, sphere, "hello", "text/subtext", bodyRaw, nil, nil) + // }) + + // ns_sphere_save(noosphere, sphere, nil, nil) + + // let file = ns_sphere_content_read_blocking(noosphere, sphere, "/hello", nil) + + // let content_type_values = ns_sphere_file_header_values_read(file, "Content-Type") + // let content_type = String.init(cString: content_type_values.ptr.pointee!) + + // print("Content-Type:", content_type) + + // let contents = ns_sphere_file_contents_read_blocking(noosphere, file, nil) + // let data: Data = .init(bytes: contents.ptr, count: contents.len) + // let subtext = String.init(decoding: data, as: UTF8.self) + + // print("Contents:", subtext) + + // ns_string_array_free(content_type_values) + // ns_bytes_free(contents) + // ns_sphere_file_free(file) + // ns_sphere_free(sphere) + // ns_string_free(sphere_identity_ptr) + // ns_string_free(sphere_mnemonic_ptr) + // ns_sphere_receipt_free(sphere_receipt) + iroh_free(iroh) + + print("fin!") } } diff --git a/swift/include/module.modulemap b/swift/include/module.modulemap index b72894d657..3a97204d8d 100644 --- a/swift/include/module.modulemap +++ b/swift/include/module.modulemap @@ -1,4 +1,4 @@ -module Iroh { +module IrohFFI { header "iroh.h" export * } \ No newline at end of file From 657085ef01d2eb3a2fbdacdd6d24181ac82ffbb8 Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Tue, 18 Jul 2023 15:06:54 +0200 Subject: [PATCH 21/32] refactor: move c-header generation into a test --- Cargo.lock | 42 +++++----- iroh-ffi/Cargo.toml | 7 +- iroh-ffi/README.md | 32 ++++++++ iroh-ffi/iroh.h | 112 +++++++++++++++++++++++++++ iroh-ffi/readme.md | 5 -- iroh-ffi/src/bin/generate-headers.rs | 12 --- iroh-ffi/src/lib.rs | 20 +++-- 7 files changed, 183 insertions(+), 47 deletions(-) create mode 100644 iroh-ffi/README.md create mode 100644 iroh-ffi/iroh.h delete mode 100644 iroh-ffi/readme.md delete mode 100644 iroh-ffi/src/bin/generate-headers.rs diff --git a/Cargo.lock b/Cargo.lock index 9415c409fd..1daf248657 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1789,6 +1789,27 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "iroh-ffi" +version = "0.1.0" +dependencies = [ + "anyhow", + "blake3", + "data-encoding", + "iroh", + "iroh-io", + "iroh-net", + "libc", + "multibase", + "num_cpus", + "quinn", + "range-collections", + "safer-ffi", + "tempfile", + "tokio", + "tokio-util", +] + [[package]] name = "iroh-io" version = "0.2.1" @@ -1886,27 +1907,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "iroh_ffi" -version = "0.1.0" -dependencies = [ - "anyhow", - "blake3", - "data-encoding", - "iroh", - "iroh-io", - "iroh-net", - "libc", - "multibase", - "num_cpus", - "quinn", - "range-collections", - "safer-ffi", - "tempfile", - "tokio", - "tokio-util", -] - [[package]] name = "is-terminal" version = "0.4.9" diff --git a/iroh-ffi/Cargo.toml b/iroh-ffi/Cargo.toml index d8d4d02e79..e451ee8220 100644 --- a/iroh-ffi/Cargo.toml +++ b/iroh-ffi/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "iroh_ffi" +name = "iroh-ffi" version = "0.1.0" edition = "2021" publish = false @@ -10,10 +10,11 @@ authors = ["n0 team"] repository = "https://github.com/n0-computer/iroh" [lib] -name="iroh" -crate-type = ["staticlib"] +name = "iroh" +crate-type = ["cdylib", "staticlib"] [features] +default = [] c-headers = ["safer-ffi/headers"] [dependencies] diff --git a/iroh-ffi/README.md b/iroh-ffi/README.md new file mode 100644 index 0000000000..0527ed3957 --- /dev/null +++ b/iroh-ffi/README.md @@ -0,0 +1,32 @@ +# iroh-ffi + +> C bindings for Iroh + +Running `cargo build --release` will produce a dynamic library and a static library. + +For builds targeting older versions of MacOS, build with with: `MACOSX_DEPLOYMENT_TARGET=10.7 && cargo build --target x86_64-apple-darwin --release`. + +## Development + +### Regenerating `iroh.h`: + +```sh +cargo test build_headers --features c-headers +``` + +# License + +This project is licensed under either of + + * Apache License, Version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or + http://www.apache.org/licenses/LICENSE-2.0) + * MIT license ([LICENSE-MIT](LICENSE-MIT) or + http://opensource.org/licenses/MIT) + +at your option. + +### Contribution + +Unless you explicitly state otherwise, any contribution intentionally submitted +for inclusion in this project by you, as defined in the Apache-2.0 license, +shall be dual licensed as above, without any additional terms or conditions. diff --git a/iroh-ffi/iroh.h b/iroh-ffi/iroh.h new file mode 100644 index 0000000000..80b731c8e2 --- /dev/null +++ b/iroh-ffi/iroh.h @@ -0,0 +1,112 @@ +/*! \file */ +/******************************************* + * * + * File auto-generated by `::safer_ffi`. * + * * + * Do not manually edit this file. * + * * + *******************************************/ + +#ifndef __RUST_IROH_FFI__ +#define __RUST_IROH_FFI__ +#ifdef __cplusplus +extern "C" { +#endif + + +#include +#include + +/** \brief + * Constant values for error codes from iroh_error_t. + */ +/** \remark Has the same ABI as `uint32_t` **/ +#ifdef DOXYGEN +typedef +#endif +enum iroh_error_code { + /** */ + IROH_ERROR_CODE_OTHER = 1, +} +#ifndef DOXYGEN +; typedef uint32_t +#endif +iroh_error_code_t; + +/** \brief + * @class iroh_error_t + * An opaque struct representing an error. + */ +typedef struct iroh_error iroh_error_t; + +/** \brief + * @memberof iroh_error_t + * Returns an error code that identifies the error. + */ +uint32_t +iroh_error_code_get ( + iroh_error_t const * error); + +/** \brief + * @memberof iroh_error_t + * Deallocate an iroh_error_t. + */ +void +iroh_error_free ( + iroh_error_t * error); + +/** \brief + * @memberof iroh_error_t + * Returns an owned string describing the error in greater detail. + * + * Caller is responsible for deallocating returned string via iroh_string_free. + */ +char * +iroh_error_message_get ( + iroh_error_t const * error); + +/** \brief + * @class iroh_node_t + */ +typedef struct iroh_node iroh_node_t; + +/** \brief + * @memberof iroh_node_t + * Deallocate a ns_noosphere_t instance. + */ +void +iroh_free ( + iroh_node_t * node); + +/** \brief + * @memberof iroh_node_t + * Get a collection from a peer. + */ +void +iroh_get_ticket ( + iroh_node_t const * node, + char const * ticket, + char const * out_path, + void (*callback)(iroh_error_t *)); + +/** \brief + * @memberof iroh_node_t + * Initialize a iroh_node_t instance. + * + */ +iroh_node_t * +iroh_initialize (void); + +/** \brief + * Deallocate an Iroh-allocated string. + */ +void +iroh_string_free ( + char * string); + + +#ifdef __cplusplus +} /* extern \"C\" */ +#endif + +#endif /* __RUST_IROH_FFI__ */ diff --git a/iroh-ffi/readme.md b/iroh-ffi/readme.md deleted file mode 100644 index ed1e92c818..0000000000 --- a/iroh-ffi/readme.md +++ /dev/null @@ -1,5 +0,0 @@ -# C bindings for Iroh - -Running `cargo build --release` will produce a dynamic library. - -For builds targeting older versions of MacOS, build with with: `MACOSX_DEPLOYMENT_TARGET=10.7 && cargo build --target x86_64-apple-darwin --release`. \ No newline at end of file diff --git a/iroh-ffi/src/bin/generate-headers.rs b/iroh-ffi/src/bin/generate-headers.rs deleted file mode 100644 index 20097ec2dd..0000000000 --- a/iroh-ffi/src/bin/generate-headers.rs +++ /dev/null @@ -1,12 +0,0 @@ -use anyhow::{anyhow, Result}; - -#[cfg(feature = "c-headers")] -fn main() -> Result<()> { - iroh_ffi::generate_headers().map_err(|e| anyhow!(e.to_string()))?; - Ok(()) -} - -#[cfg(not(feature = "c-headers"))] -fn main() -> Result<()> { - Err(anyhow!("Must run with --features c-headers")) -} diff --git a/iroh-ffi/src/lib.rs b/iroh-ffi/src/lib.rs index 33052dc2f5..5bad1c4926 100644 --- a/iroh-ffi/src/lib.rs +++ b/iroh-ffi/src/lib.rs @@ -4,13 +4,21 @@ pub mod error; pub mod get; pub mod node; -#[cfg(feature = "c-headers")] -pub fn generate_headers() -> std::io::Result<()> { - safer_ffi::headers::builder().to_file("iroh.h")?.generate() -} - #[ffi_export] /// Deallocate an Iroh-allocated string. pub fn iroh_string_free(string: char_p::Box) { - drop(string) + drop(string); +} + +// Generates the headers. +// +// `cargo test build_headers --features c-headers` to build +#[safer_ffi::cfg_headers] +#[test] +fn build_headers() -> std::io::Result<()> { + safer_ffi::headers::builder() + .to_file("iroh.h")? + .generate()?; + + Ok(()) } From b2309ef8bb2e42bf68bf117ca5851369f0af0340 Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Tue, 18 Jul 2023 15:51:49 +0200 Subject: [PATCH 22/32] add iroh.pc generation --- iroh-ffi/build.rs | 35 +++++++++++++++++++++++++++++++++++ iroh-ffi/iroh.pc.in | 11 +++++++++++ 2 files changed, 46 insertions(+) create mode 100644 iroh-ffi/build.rs create mode 100644 iroh-ffi/iroh.pc.in diff --git a/iroh-ffi/build.rs b/iroh-ffi/build.rs new file mode 100644 index 0000000000..cb997c7865 --- /dev/null +++ b/iroh-ffi/build.rs @@ -0,0 +1,35 @@ +use std::path::PathBuf; +use std::{env, fs}; + +fn main() { + let out_path = PathBuf::from(env::var("OUT_DIR").unwrap()); + let target_path = out_path.join("../../.."); + let target_triple = env::var("TARGET").unwrap(); + + // macOS or iOS + let libs_priv = if target_triple.contains("apple") || target_triple.contains("darwin") { + // TODO: verify all these are needed + "-framework SystemConfiguration -framework Security -framework Foundation" + } else { + "" + }; + + let pkg_config = format!( + include_str!("iroh.pc.in"), + name = "iroh", + description = env::var("CARGO_PKG_DESCRIPTION").unwrap(), + url = env::var("CARGO_PKG_HOMEPAGE").unwrap_or_else(|_| "".to_string()), + version = env::var("CARGO_PKG_VERSION").unwrap(), + libs_priv = libs_priv, + prefix = env::var("PREFIX").unwrap_or_else(|_| "/usr/local".to_string()), + libdir = env::var("LIBDIR").unwrap_or_else(|_| "/usr/local/lib".to_string()), + includedir = env::var("INCLUDEDIR").unwrap_or_else(|_| "/usr/local/include".to_string()), + ); + + fs::create_dir_all(target_path.join("pkgconfig")).unwrap(); + fs::write( + target_path.join("pkgconfig").join("iroh.pc"), + pkg_config.as_bytes(), + ) + .unwrap(); +} diff --git a/iroh-ffi/iroh.pc.in b/iroh-ffi/iroh.pc.in new file mode 100644 index 0000000000..3be2531305 --- /dev/null +++ b/iroh-ffi/iroh.pc.in @@ -0,0 +1,11 @@ +prefix={prefix} +libdir={libdir} +includedir={includedir} + +Name: {name} +Description: {description} +URL: {url} +Version: {version} +Cflags: -I${{includedir}} +Libs: -L${{libdir}} -liroh +Libs.private: {libs_priv} From 4625e7be02123a465d45365cb751c1dd7a320fef Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Tue, 18 Jul 2023 15:52:20 +0200 Subject: [PATCH 23/32] make c example program work --- iroh-ffi/c/.gitignore | 2 ++ iroh-ffi/c/Makefile | 6 +++--- iroh-ffi/c/main.c | 46 ++++++++++++++++++++++++++++++++++--------- iroh-ffi/iroh.h | 5 ++--- iroh-ffi/src/get.rs | 30 ++++++++++++---------------- 5 files changed, 57 insertions(+), 32 deletions(-) create mode 100644 iroh-ffi/c/.gitignore diff --git a/iroh-ffi/c/.gitignore b/iroh-ffi/c/.gitignore new file mode 100644 index 0000000000..b7e1509cd9 --- /dev/null +++ b/iroh-ffi/c/.gitignore @@ -0,0 +1,2 @@ +iroh.h +main \ No newline at end of file diff --git a/iroh-ffi/c/Makefile b/iroh-ffi/c/Makefile index cf618124c8..fee6eabeaf 100644 --- a/iroh-ffi/c/Makefile +++ b/iroh-ffi/c/Makefile @@ -1,8 +1,8 @@ build: iroh.h - cargo build --release - $(CC) main.c -o main -L ../target/release -l iroh -l pthread -l dl + cargo build -p iroh-ffi --release + $(CC) main.c -o main -L ../../target/release -l iroh -l pthread -l dl iroh.h: - cargo test --features c-headers -- generate_headers + cargo test -p iroh-ffi --features c-headers -- generate_headers cp ../iroh.h iroh.h diff --git a/iroh-ffi/c/main.c b/iroh-ffi/c/main.c index 4348e909f4..4d63fea6a3 100644 --- a/iroh-ffi/c/main.c +++ b/iroh-ffi/c/main.c @@ -4,13 +4,41 @@ #include "iroh.h" int main (int argc, char const * const argv[]) { - if (argc < 3) { - printf("Usage: %s \n", argv[0]); - return 1; - } + if (argc < 3) { + printf("Usage: %s \n", argv[0]); + return 1; + } + + printf("starting node...\n"); + fflush(stdout); + + iroh_node_t *node = iroh_initialize(); + if (node == NULL) { + printf("failed to start node\n"); + fflush(stdout); + return -1; + } + + printf("node started\n"); + fflush(stdout); + + const char *out_path = argv[1]; + const char *ticket = argv[2]; + iroh_error_t * err = iroh_get_ticket(node, ticket, out_path); + if (err != NULL) { + char * msg = iroh_error_message_get(err); + printf("failed: %s\n", msg); + fflush(stdout); + + iroh_string_free(msg); + iroh_error_free(err); + return -1; + } + + printf("done\n"); + fflush(stdout); + iroh_free(node); + + return EXIT_SUCCESS; +} - const char *out_path = argv[1]; - const char *ticket = argv[2]; - iroh_get_ticket(ticket, out_path); - return EXIT_SUCCESS; -} \ No newline at end of file diff --git a/iroh-ffi/iroh.h b/iroh-ffi/iroh.h index 80b731c8e2..c50b3a1bbb 100644 --- a/iroh-ffi/iroh.h +++ b/iroh-ffi/iroh.h @@ -82,12 +82,11 @@ iroh_free ( * @memberof iroh_node_t * Get a collection from a peer. */ -void +iroh_error_t * iroh_get_ticket ( iroh_node_t const * node, char const * ticket, - char const * out_path, - void (*callback)(iroh_error_t *)); + char const * out_path); /** \brief * @memberof iroh_node_t diff --git a/iroh-ffi/src/get.rs b/iroh-ffi/src/get.rs index cfd966f11d..f669b3d5d1 100644 --- a/iroh-ffi/src/get.rs +++ b/iroh-ffi/src/get.rs @@ -63,32 +63,28 @@ pub fn iroh_get_ticket( node: &IrohNode, ticket: char_p::Ref<'_>, out_path: char_p::Ref<'_>, - callback: extern "C" fn(Option>), -) { +) -> Option> { let ticket = ticket.to_string(); let out_path = PathBuf::from(out_path.to_string()); let keypair = node.inner().keypair(); - let rt = node.async_runtime().clone(); - node.async_runtime() - .local_pool() - .spawn_pinned(move || async move { - let result = async { + + let result: anyhow::Result<_> = node.async_runtime().main().block_on(async move { + node.async_runtime() + .local_pool() + .spawn_pinned(move || async move { let ticket = Ticket::from_str(&ticket)?; // TODO(b5): pull DerpMap from node, feed into here: let opts = ticket.as_get_options(keypair, None); let conn = dial(opts, &iroh::bytes::protocol::ALPN).await?; get_blob_to_file(conn, ticket.hash(), ticket.token().cloned(), out_path).await?; - Ok(()) - } - .await; - - let result = as_opt_err(result); - rt.main() - .spawn_blocking(move || callback(result)) - .await - .ok(); - }); + anyhow::Ok(()) + }) + .await??; + Ok(()) + }); + + as_opt_err(result) } fn as_opt_err(res: Result<()>) -> Option> { From ec01446c5de9281d805608ec17a5ef888a315867 Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Tue, 18 Jul 2023 15:54:40 +0200 Subject: [PATCH 24/32] ffi: adjust Cargo.toml --- iroh-ffi/Cargo.toml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/iroh-ffi/Cargo.toml b/iroh-ffi/Cargo.toml index e451ee8220..820dd583ad 100644 --- a/iroh-ffi/Cargo.toml +++ b/iroh-ffi/Cargo.toml @@ -4,11 +4,14 @@ version = "0.1.0" edition = "2021" publish = false readme = "README.md" -description = "IPFS reimagined" +description = "Bytes. Distributed. In C." license = "MIT/Apache-2.0" authors = ["n0 team"] repository = "https://github.com/n0-computer/iroh" +# Sadly this also needs to be updated in .github/workflows/ci.yml +rust-version = "1.66" + [lib] name = "iroh" crate-type = ["cdylib", "staticlib"] From 58d1a191edfb66cc21c12447167c733381875d37 Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Tue, 18 Jul 2023 18:50:58 +0200 Subject: [PATCH 25/32] fixup examples and dial --- iroh-ffi/src/get.rs | 2 +- iroh/Cargo.toml | 8 ++++++++ iroh/src/commands/get.rs | 6 +++--- iroh/src/dial.rs | 9 +++++++-- iroh/tests/provide.rs | 12 +++++------- 5 files changed, 24 insertions(+), 13 deletions(-) diff --git a/iroh-ffi/src/get.rs b/iroh-ffi/src/get.rs index f669b3d5d1..aa6401d8a7 100644 --- a/iroh-ffi/src/get.rs +++ b/iroh-ffi/src/get.rs @@ -76,7 +76,7 @@ pub fn iroh_get_ticket( // TODO(b5): pull DerpMap from node, feed into here: let opts = ticket.as_get_options(keypair, None); - let conn = dial(opts, &iroh::bytes::protocol::ALPN).await?; + let conn = dial(opts).await?; get_blob_to_file(conn, ticket.hash(), ticket.token().cloned(), out_path).await?; anyhow::Ok(()) }) diff --git a/iroh/Cargo.toml b/iroh/Cargo.toml index c04b21998c..60398c10e7 100644 --- a/iroh/Cargo.toml +++ b/iroh/Cargo.toml @@ -89,3 +89,11 @@ required-features = ["mem-db", "iroh-collection"] [[example]] name = "hello-world" required-features = ["mem-db"] + +[[example]] +name = "dump-blob-stream" +required-features = ["iroh-collection"] + +[[example]] +name = "dump-blob-fsm" +required-features = [] diff --git a/iroh/src/commands/get.rs b/iroh/src/commands/get.rs index 249306abfd..ae9976b73a 100644 --- a/iroh/src/commands/get.rs +++ b/iroh/src/commands/get.rs @@ -67,7 +67,7 @@ impl GetInteractive { let collection_info = Some((1, 0)); let request = self.new_request(query).with_token(self.token.clone()); - let connection = iroh::dial::dial(self.opts, &iroh_bytes::protocol::ALPN).await?; + let connection = iroh::dial::dial(self.opts).await?; let response = fsm::start(connection, request); let connected = response.next().await?; write(format!("{} Requesting ...", style("[2/3]").bold().dim())); @@ -155,7 +155,7 @@ impl GetInteractive { }; let request = self.new_request(query).with_token(self.token.clone()); - let connection = iroh::dial::dial(self.opts, &iroh_bytes::protocol::ALPN).await?; + let connection = iroh::dial::dial(self.opts).await?; let response = fsm::start(connection, request); let connected = response.next().await?; write(format!("{} Requesting ...", style("[2/3]").bold().dim())); @@ -322,7 +322,7 @@ impl GetInteractive { let pb = make_download_pb(); let request = self.new_request(query).with_token(self.token.clone()); - let connection = iroh::dial::dial(self.opts, &iroh_bytes::protocol::ALPN).await?; + let connection = iroh::dial::dial(self.opts).await?; let response = fsm::start(connection, request); let connected = response.next().await?; write(format!("{} Requesting ...", style("[2/3]").bold().dim())); diff --git a/iroh/src/dial.rs b/iroh/src/dial.rs index c29353ee9d..e55e6eb988 100644 --- a/iroh/src/dial.rs +++ b/iroh/src/dial.rs @@ -36,7 +36,7 @@ pub struct Options { /// Note that this will create an entirely new endpoint, so it should be only /// used for short lived connections. If you want to connect to multiple peers, /// it is preferable to create an endpoint and use `connect` on the endpoint. -pub async fn dial(opts: Options, alpn: &[u8]) -> anyhow::Result { +pub async fn dial(opts: Options) -> anyhow::Result { let endpoint = iroh_net::MagicEndpoint::builder() .keypair(opts.keypair) .derp_map(opts.derp_map) @@ -44,7 +44,12 @@ pub async fn dial(opts: Options, alpn: &[u8]) -> anyhow::Result Result<()> { let peer_id = node.peer_id(); let timeout = tokio::time::timeout(std::time::Duration::from_secs(10), async move { - let connection = - iroh::dial::dial(get_options(peer_id, node_addr), &iroh_bytes::protocol::ALPN) - .await - .unwrap(); + let connection = iroh::dial::dial(get_options(peer_id, node_addr)) + .await + .unwrap(); let response = fsm::start(connection, GetRequest::all(hash).into()); // connect let connected = response.next().await.unwrap(); @@ -573,7 +572,7 @@ async fn run_custom_get_request( request: AnyGetRequest, collection_parser: C, ) -> anyhow::Result<(Bytes, BTreeMap, Stats)> { - let connection = iroh::dial::dial(opts, &iroh_bytes::protocol::ALPN).await?; + let connection = iroh::dial::dial(opts).await?; let initial = fsm::start(connection, request); use fsm::*; let mut items = BTreeMap::new(); @@ -754,8 +753,7 @@ async fn test_custom_request_blob() { token: None, data: Bytes::from(&b"hello"[..]), }); - let connection = - iroh::dial::dial(get_options(peer_id, addrs), &iroh_bytes::protocol::ALPN).await?; + let connection = iroh::dial::dial(get_options(peer_id, addrs)).await?; let response = fsm::start(connection, request); let connected = response.next().await?; let ConnectedNext::StartRoot(start) = connected.next().await? else { panic!() }; From a48738376f5e2f9ca932b9384e1c2e05128220e1 Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Tue, 18 Jul 2023 21:57:57 +0200 Subject: [PATCH 26/32] download the full collection into a folder --- Cargo.lock | 1 + iroh-ffi/Cargo.toml | 1 + iroh-ffi/src/get.rs | 110 +++++++++++++++++++++++--------------------- 3 files changed, 60 insertions(+), 52 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 725e8c8c86..c2aeaadf60 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1808,6 +1808,7 @@ version = "0.1.0" dependencies = [ "anyhow", "blake3", + "bytes", "data-encoding", "iroh", "iroh-io", diff --git a/iroh-ffi/Cargo.toml b/iroh-ffi/Cargo.toml index 820dd583ad..0a0c4db055 100644 --- a/iroh-ffi/Cargo.toml +++ b/iroh-ffi/Cargo.toml @@ -23,6 +23,7 @@ c-headers = ["safer-ffi/headers"] [dependencies] anyhow = "1.0.69" blake3 = "1.3.3" +bytes = "1" data-encoding = { version = "2.3.3" } iroh = { path = "../iroh", default-features = false, features = ["mem-db", "iroh-collection"] } iroh-io = { version = "0.2.1" } diff --git a/iroh-ffi/src/get.rs b/iroh-ffi/src/get.rs index aa6401d8a7..92260c48be 100644 --- a/iroh-ffi/src/get.rs +++ b/iroh-ffi/src/get.rs @@ -1,19 +1,18 @@ use std::path::PathBuf; use std::str::FromStr; -use anyhow::Result; -use iroh_io::{AsyncSliceWriter, File}; -use range_collections::RangeSet2; +use anyhow::{bail, ensure, Result}; +use bytes::Bytes; +use iroh_io::AsyncSliceWriter; use safer_ffi::prelude::*; use iroh::{ bytes::{ get::fsm, - protocol::{GetRequest, RangeSpecSeq, Request, RequestToken}, - Hash, + protocol::{AnyGetRequest, GetRequest}, }, + collection::Collection, dial::{dial, Ticket}, - // net::tls::PeerId, }; use crate::{error::IrohError, node::IrohNode}; @@ -75,9 +74,9 @@ pub fn iroh_get_ticket( let ticket = Ticket::from_str(&ticket)?; // TODO(b5): pull DerpMap from node, feed into here: - let opts = ticket.as_get_options(keypair, None); - let conn = dial(opts).await?; - get_blob_to_file(conn, ticket.hash(), ticket.token().cloned(), out_path).await?; + let dial_opts = ticket.as_get_options(keypair, None); + let conn = dial(dial_opts).await?; + get_collection_to_folder(conn, ticket, out_path).await?; anyhow::Ok(()) }) .await??; @@ -94,51 +93,58 @@ fn as_opt_err(res: Result<()>) -> Option> { } } -async fn get_blob_to_file( - conn: quinn::Connection, - hash: Hash, - token: Option, +async fn get_collection_to_folder( + connection: quinn::Connection, + ticket: Ticket, out_path: PathBuf, ) -> Result<()> { - get_blob_ranges_to_file( - conn, - hash, - token, - RangeSpecSeq::new([RangeSet2::all()]), - out_path, - ) - .await -} + use fsm::*; + + ensure!(!out_path.is_file(), "out_path must not be a file"); + tokio::fs::create_dir_all(&out_path).await?; + + let request = + AnyGetRequest::Get(GetRequest::all(ticket.hash())).with_token(ticket.token().cloned()); + + let initial = fsm::start(connection, request); + + let connected = initial.next().await?; + // we assume that the request includes the entire collection + let (mut next, _root, collection) = { + let ConnectedNext::StartRoot(sc) = connected.next().await? else { + bail!("request did not include collection"); + }; + + let (done, data) = sc.next().concatenate_into_vec().await?; + let data = Bytes::from(data); + let collection = Collection::from_bytes(&data)?; + + (done.next(), data, collection) + }; + + // download all the children + let mut blobs = collection.blobs().iter(); + let finishing = loop { + let start = match next { + EndBlobNext::MoreChildren(start) => start, + EndBlobNext::Closing(finishing) => break finishing, + }; + + // get the hash of the next blob, or finish if there are no more + let Some(blob) = blobs.next() else { + break start.finish(); + }; + + let start = start.next(blob.hash); + let file_path = out_path.join(&blob.name); + let mut file = iroh_io::File::create(move || std::fs::File::create(&file_path)).await?; + + let done = start.write_all(&mut file).await?; + file.sync().await?; + + next = done.next(); + }; + let _stats = finishing.next().await?; -// TODO(b5): This currently assumes "all" ranges, needs to be adjusted to honor -// RangeSpecSeq args other than "all" -async fn get_blob_ranges_to_file( - conn: quinn::Connection, - hash: Hash, - token: Option, - ranges: RangeSpecSeq, - out_path: PathBuf, -) -> Result<()> { - let request = Request::Get(GetRequest::new(hash, ranges)).with_token(token); - let response = fsm::start(conn, request); - let connected = response.next().await?; - - let fsm::ConnectedNext::StartRoot(curr) = connected.next().await? else { - return Ok(()) - }; - let header = curr.next(); - - let path = out_path.clone(); - let mut file = File::create(move || { - std::fs::OpenOptions::new() - .write(true) - .create(true) - .open(&path) - }) - .await?; - - let (curr, _size) = header.next().await?; - let _curr = curr.write_all(&mut file).await?; - file.sync().await?; Ok(()) } From 0ed812c66d4cc6eb022a93a9d9f1aa03884b405f Mon Sep 17 00:00:00 2001 From: b5 Date: Tue, 18 Jul 2023 23:52:19 -0400 Subject: [PATCH 27/32] update swift package, build xcframework build script --- Package.swift | 18 +-- iroh-ffi/iroh.h | 111 ------------------ scripts/build-xcframework.sh | 75 ++++++------ .../Iroh.swift => IrohSwift/IrohSwift.swift} | 6 +- .../Tests/IrohSwiftTests/IrohSwiftTests.swift | 18 +++ swift/Tests/IrohTests/IrohTests.swift | 67 ----------- swift/include/module.modulemap | 4 +- 7 files changed, 76 insertions(+), 223 deletions(-) delete mode 100644 iroh-ffi/iroh.h rename swift/Sources/{Iroh/Iroh.swift => IrohSwift/IrohSwift.swift} (87%) create mode 100644 swift/Tests/IrohSwiftTests/IrohSwiftTests.swift delete mode 100644 swift/Tests/IrohTests/IrohTests.swift diff --git a/Package.swift b/Package.swift index f71261ba72..369a01a7cf 100644 --- a/Package.swift +++ b/Package.swift @@ -4,29 +4,29 @@ import PackageDescription let package = Package( - name: "iroh", + name: "IrohSwift", platforms: [ - .iOS(.v13), - .macOS(.v11) + .iOS(.v16), + .macOS(.v13) ], products: [ // Products define the executables and libraries a package produces, // making them visible to other packages. .library( - name: "Iroh", - targets: ["Iroh"]), + name: "IrohSwift", + targets: ["IrohSwift"]), ], targets: [ // Targets are the basic building blocks of a package, defining a module or a test suite. // Targets can depend on other targets in this package and products from dependencies. .target( - name: "Iroh", + name: "IrohSwift", dependencies: ["LibIroh"], - path: "swift/Sources/Iroh"), + path: "swift/Sources/IrohSwift"), .testTarget( name: "IrohTests", - dependencies: ["Iroh"], - path: "swift/Tests/IrohTests"), + dependencies: ["IrohSwift"], + path: "swift/Tests/IrohSwiftTests"), .binaryTarget( name: "LibIroh", path: "target/LibIroh.xcframework"), diff --git a/iroh-ffi/iroh.h b/iroh-ffi/iroh.h deleted file mode 100644 index c50b3a1bbb..0000000000 --- a/iroh-ffi/iroh.h +++ /dev/null @@ -1,111 +0,0 @@ -/*! \file */ -/******************************************* - * * - * File auto-generated by `::safer_ffi`. * - * * - * Do not manually edit this file. * - * * - *******************************************/ - -#ifndef __RUST_IROH_FFI__ -#define __RUST_IROH_FFI__ -#ifdef __cplusplus -extern "C" { -#endif - - -#include -#include - -/** \brief - * Constant values for error codes from iroh_error_t. - */ -/** \remark Has the same ABI as `uint32_t` **/ -#ifdef DOXYGEN -typedef -#endif -enum iroh_error_code { - /** */ - IROH_ERROR_CODE_OTHER = 1, -} -#ifndef DOXYGEN -; typedef uint32_t -#endif -iroh_error_code_t; - -/** \brief - * @class iroh_error_t - * An opaque struct representing an error. - */ -typedef struct iroh_error iroh_error_t; - -/** \brief - * @memberof iroh_error_t - * Returns an error code that identifies the error. - */ -uint32_t -iroh_error_code_get ( - iroh_error_t const * error); - -/** \brief - * @memberof iroh_error_t - * Deallocate an iroh_error_t. - */ -void -iroh_error_free ( - iroh_error_t * error); - -/** \brief - * @memberof iroh_error_t - * Returns an owned string describing the error in greater detail. - * - * Caller is responsible for deallocating returned string via iroh_string_free. - */ -char * -iroh_error_message_get ( - iroh_error_t const * error); - -/** \brief - * @class iroh_node_t - */ -typedef struct iroh_node iroh_node_t; - -/** \brief - * @memberof iroh_node_t - * Deallocate a ns_noosphere_t instance. - */ -void -iroh_free ( - iroh_node_t * node); - -/** \brief - * @memberof iroh_node_t - * Get a collection from a peer. - */ -iroh_error_t * -iroh_get_ticket ( - iroh_node_t const * node, - char const * ticket, - char const * out_path); - -/** \brief - * @memberof iroh_node_t - * Initialize a iroh_node_t instance. - * - */ -iroh_node_t * -iroh_initialize (void); - -/** \brief - * Deallocate an Iroh-allocated string. - */ -void -iroh_string_free ( - char * string); - - -#ifdef __cplusplus -} /* extern \"C\" */ -#endif - -#endif /* __RUST_IROH_FFI__ */ diff --git a/scripts/build-xcframework.sh b/scripts/build-xcframework.sh index 705aefd05c..3820c7ffee 100755 --- a/scripts/build-xcframework.sh +++ b/scripts/build-xcframework.sh @@ -7,61 +7,70 @@ export PATH="$HOME/.cargo/bin:$PATH" REPO_ROOT=".." RUST_FFI_DIR="../iroh-ffi" OUT_DIR="../build" +BUILD_MACOS=false # TODO(b5): explicitly enforce build toolchain for all cargo invocations # RUST_TOOLCHAIN="1.71.0" -echo "Generate Iroh C header, copy Module map" +echo "1. Generate Iroh C header, copy Module map" mkdir -p "${OUT_DIR}/include" -cargo run --features c-headers --bin generate-headers -mv iroh.h ${OUT_DIR}/include/iroh.h +cargo test build_headers --features c-headers +mv ${RUST_FFI_DIR}/iroh.h ${OUT_DIR}/include/iroh.h cp ${REPO_ROOT}/swift/include/module.modulemap ${OUT_DIR}/include/module.modulemap -echo "Build Iroh Libraries for Apple Platforms" +echo "2. Build Iroh Libraries for Apple Platforms" targets=( - # "aarch64-apple-ios" - # "x86_64-apple-ios" - # "aarch64-apple-ios-sim" "x86_64-apple-darwin" "aarch64-apple-darwin" -) + "aarch64-apple-ios" + "x86_64-apple-ios" + "aarch64-apple-ios-sim" +); for target in "${targets[@]}"; do - cargo build --package iroh_ffi --release --target ${target} - mkdir -p ${OUT_DIR}/lib_${target} - cp "${REPO_ROOT}/target/${target}/release/libiroh_ffi.a" "${OUT_DIR}/lib_${target}/libiroh_ffi.a" + echo "compile for ${target}" + cargo build --package iroh-ffi --release --target ${target} + mkdir -p ${OUT_DIR}/${target} + cp "${REPO_ROOT}/target/${target}/release/libiroh.a" "${OUT_DIR}/${target}/libiroh.a" done -echo "Run Lipo" -mkdir -p "${OUT_DIR}/lib_ios-simulator-universal" -# cargo +`cat ${REPO_ROOT}/rust-toolchain` lipo \ -# --targets aarch64-apple-ios-sim,x86_64-apple-ios \ -# --manifest-path "${RUST_FFI_DIR}/Cargo.toml" -# cp "${RUST_FFI_DIR}/target/universal/release/libiroh.a" "${OUT_DIR}/lib_ios-simulator-universal/libiroh.a" +echo "3. Run Lipo" +if [ "$BUILD_MACOS" = true ]; then + mkdir -p "${OUT_DIR}/apple-darwin-universal" + lipo -create \ + "${OUT_DIR}/x86_64-apple-darwin/libiroh.a" \ + "${OUT_DIR}/aarch64-apple-darwin/libiroh.a" \ + -output "${OUT_DIR}/apple-darwin-universal/libiroh.a" +else + mkdir -p "${OUT_DIR}/ios-simulator-universal" + lipo -create \ + "${OUT_DIR}/x86_64-apple-ios/libiroh.a" \ + "${OUT_DIR}/aarch64-apple-ios-sim/libiroh.a" \ + -output "${OUT_DIR}/ios-simulator-universal/libiroh.a" +fi -lipo -create \ - "${OUT_DIR}/lib_x86_64-apple-ios/libiroh_ffi.a" \ - "${OUT_DIR}/lib_aarch64-apple-ios-sim/libiroh_ffi.a" \ - -output "${OUT_DIR}/lib_ios-simulator-universal/libiroh_ffi.a" -echo "Create XCFramework" +echo "4. Create XCFramework" rm -rf ${REPO_ROOT}/target/LibIroh.xcframework -# xcodebuild -create-xcframework \ -# -library ${OUT_DIR}/lib_ios-simulator-universal/libiroh_ffi.a \ -# -headers ${OUT_DIR}/include/ \ -# -library ${OUT_DIR}/lib_aarch64-apple-ios/libiroh_ffi.a \ -# -headers ${OUT_DIR}/include/ \ -# -output ${REPO_ROOT}/target/LibIroh.xcframework +if [ "$BUILD_MACOS" = true ]; then + xcodebuild -create-xcframework \ + -library ${OUT_DIR}/apple-darwin-universal/libiroh.a \ + -headers ${OUT_DIR}/include/ \ + -output ${REPO_ROOT}/target/LibIroh.xcframework +else + xcodebuild -create-xcframework \ + -library ${OUT_DIR}/ios-simulator-universal/libiroh.a \ + -headers ${OUT_DIR}/include/ \ + -library ${OUT_DIR}/aarch64-apple-ios/libiroh.a \ + -headers ${OUT_DIR}/include/ \ + -output ${REPO_ROOT}/target/LibIroh.xcframework +fi -xcodebuild -create-xcframework \ - -library ${OUT_DIR}/lib_aarch64-apple-darwin/libiroh_ffi.a \ - -headers ${OUT_DIR}/include/ \ - -output ${REPO_ROOT}/target/LibIroh.xcframework -# echo "Zip XCFramework" +# echo "5. Zip XCFramework" # zip -r ${REPO_ROOT}/target/libiroh-xcframework.zip ${REPO_ROOT}/target/LibIroh.xcframework echo "Done" \ No newline at end of file diff --git a/swift/Sources/Iroh/Iroh.swift b/swift/Sources/IrohSwift/IrohSwift.swift similarity index 87% rename from swift/Sources/Iroh/Iroh.swift rename to swift/Sources/IrohSwift/IrohSwift.swift index 99c875d8b3..e16c726672 100644 --- a/swift/Sources/Iroh/Iroh.swift +++ b/swift/Sources/IrohSwift/IrohSwift.swift @@ -1,9 +1,13 @@ -@_exported import IrohFFI +@_exported import IrohSwift // public enum IrohError: Error { // case unknown(UInt32) // } +public func irohGetFoobar() { + print("get it") +} + // public func irohGet(cid: String, peer: String, peerAddr:String, outPath:String) throws { // // let status = iroh_get(cid, peer, peerAddr, outPath) // // guard status == errSecSuccess else { diff --git a/swift/Tests/IrohSwiftTests/IrohSwiftTests.swift b/swift/Tests/IrohSwiftTests/IrohSwiftTests.swift new file mode 100644 index 0000000000..7903184d9a --- /dev/null +++ b/swift/Tests/IrohSwiftTests/IrohSwiftTests.swift @@ -0,0 +1,18 @@ +import XCTest +import IrohSwift + +final class IrohSwiftTests: XCTestCase { + func testIrohGet() throws { + + } + + func testInitIroh() throws { + // This is a basic integration test to ensure that file writing and + // reading from swift works as intended + let iroh = iroh_initialize() + + iroh_free(iroh) + + print("fin!") + } +} diff --git a/swift/Tests/IrohTests/IrohTests.swift b/swift/Tests/IrohTests/IrohTests.swift deleted file mode 100644 index 1059d5748f..0000000000 --- a/swift/Tests/IrohTests/IrohTests.swift +++ /dev/null @@ -1,67 +0,0 @@ -import XCTest -import Iroh - -final class IrohSwiftTests: XCTestCase { - func testIrohGet() throws { - - } - - func testInitializeNoosphereThenWriteAFileThenSaveThenReadItBack() throws { - // This is a basic integration test to ensure that file writing and - // reading from swift works as intended - let iroh = iroh_initialize() - - // ns_tracing_initialize(NS_NOOSPHERE_LOG_CHATTY.rawValue) - // ns_key_create(noosphere, "bob", nil) - - // let sphere_receipt = ns_sphere_create(noosphere, "bob", nil) - - // let sphere_identity_ptr = ns_sphere_receipt_identity(sphere_receipt, nil) - // let sphere_mnemonic_ptr = ns_sphere_receipt_mnemonic(sphere_receipt, nil) - - // let sphere_identity = String.init(cString: sphere_identity_ptr!) - // let sphere_mnemonic = String.init(cString: sphere_mnemonic_ptr!) - - // print("Sphere identity:", sphere_identity) - // print("Recovery code:", sphere_mnemonic) - - // let sphere = ns_sphere_open(noosphere, sphere_identity_ptr, nil) - - // let file_bytes = "Hello, Subconscious".data(using: .utf8)! - - // file_bytes.withUnsafeBytes({ rawBufferPointer in - // let bufferPointer = rawBufferPointer.bindMemory(to: UInt8.self) - // let pointer = bufferPointer.baseAddress! - // let bodyRaw = slice_ref_uint8( - // ptr: pointer, len: file_bytes.count - // ) - // ns_sphere_content_write(noosphere, sphere, "hello", "text/subtext", bodyRaw, nil, nil) - // }) - - // ns_sphere_save(noosphere, sphere, nil, nil) - - // let file = ns_sphere_content_read_blocking(noosphere, sphere, "/hello", nil) - - // let content_type_values = ns_sphere_file_header_values_read(file, "Content-Type") - // let content_type = String.init(cString: content_type_values.ptr.pointee!) - - // print("Content-Type:", content_type) - - // let contents = ns_sphere_file_contents_read_blocking(noosphere, file, nil) - // let data: Data = .init(bytes: contents.ptr, count: contents.len) - // let subtext = String.init(decoding: data, as: UTF8.self) - - // print("Contents:", subtext) - - // ns_string_array_free(content_type_values) - // ns_bytes_free(contents) - // ns_sphere_file_free(file) - // ns_sphere_free(sphere) - // ns_string_free(sphere_identity_ptr) - // ns_string_free(sphere_mnemonic_ptr) - // ns_sphere_receipt_free(sphere_receipt) - iroh_free(iroh) - - print("fin!") - } -} diff --git a/swift/include/module.modulemap b/swift/include/module.modulemap index 3a97204d8d..c78606b0f9 100644 --- a/swift/include/module.modulemap +++ b/swift/include/module.modulemap @@ -1,4 +1,4 @@ -module IrohFFI { +module IrohSwift { header "iroh.h" export * -} \ No newline at end of file +} From facc08c54a9812a11d810f93a65c97823e61bdbf Mon Sep 17 00:00:00 2001 From: b5 Date: Thu, 20 Jul 2023 09:47:43 -0400 Subject: [PATCH 28/32] WIP --- .cargo/config.toml | 26 ++++++++++++++++++++++++++ Package.swift | 2 +- iroh-ffi/build.rs | 3 ++- scripts/build-xcframework.sh | 1 + 4 files changed, 30 insertions(+), 2 deletions(-) diff --git a/.cargo/config.toml b/.cargo/config.toml index 018fa32046..605f803434 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,2 +1,28 @@ [build] rustflags = ["-Wmissing_debug_implementations"] + + +[target.x86_64-apple-darwin] +# linker = "/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang" +# linker = "/usr/bin/clang" +linker = "rust-lld" + +[target.aarch64-apple-darwin] +# linker = "/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang" +# linker = "/usr/bin/clang" +linker = "rust-lld" + +[target.aarch64-apple-ios] +# linker = "/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang" +# linker = "/usr/bin/clang" +linker = "rust-lld" + +[target.x86_64-apple-ios] +# linker = "/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang" +# linker = "/usr/bin/clang" +linker = "rust-lld" + +[target.aarch64-apple-ios-sim] +# linker = "/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang" +# linker = "/usr/bin/clang" +linker = "rust-lld" diff --git a/Package.swift b/Package.swift index 369a01a7cf..35fab8bbd4 100644 --- a/Package.swift +++ b/Package.swift @@ -6,7 +6,7 @@ import PackageDescription let package = Package( name: "IrohSwift", platforms: [ - .iOS(.v16), + .iOS(.v15), .macOS(.v13) ], products: [ diff --git a/iroh-ffi/build.rs b/iroh-ffi/build.rs index cb997c7865..047189d37a 100644 --- a/iroh-ffi/build.rs +++ b/iroh-ffi/build.rs @@ -9,7 +9,8 @@ fn main() { // macOS or iOS let libs_priv = if target_triple.contains("apple") || target_triple.contains("darwin") { // TODO: verify all these are needed - "-framework SystemConfiguration -framework Security -framework Foundation" + // "-framework SystemConfiguration -framework Security -framework Foundation" + "-framework SystemConfiguration" } else { "" }; diff --git a/scripts/build-xcframework.sh b/scripts/build-xcframework.sh index 3820c7ffee..d18a7143e2 100755 --- a/scripts/build-xcframework.sh +++ b/scripts/build-xcframework.sh @@ -1,6 +1,7 @@ # export RUSTFLAGS="-C embed-bitcode=yes" # export CARGO_TARGET_AARCH64_APPLE_IOS_SIM_LINKER="/usr/bin/clang" # export LIBRARY_PATH="//usr/lib" +set -eu export PATH="/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin" export PATH="$HOME/.cargo/bin:$PATH" From 376675a8188087de2af1d377182def3412ca5934 Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Fri, 21 Jul 2023 10:30:42 +0200 Subject: [PATCH 29/32] chore(ci): add ffi bindings to CI (#1270) * chore(ci): add ffi bindings to CI * bump * fix diff detection * fix * check in iroh.h * disable ffi doctests * fix * fix output status of compare --- .github/workflows/bindings.yml | 67 ++++++++++++++++++++ .github/workflows/netsim.yml | 3 +- iroh-ffi/iroh.h | 111 +++++++++++++++++++++++++++++++++ 3 files changed, 180 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/bindings.yml create mode 100644 iroh-ffi/iroh.h diff --git a/.github/workflows/bindings.yml b/.github/workflows/bindings.yml new file mode 100644 index 0000000000..68854a1c8b --- /dev/null +++ b/.github/workflows/bindings.yml @@ -0,0 +1,67 @@ +name: FFI-CI + +on: + pull_request: + push: + branches: + - main + +env: + RUST_BACKTRACE: 1 + RUSTFLAGS: -Dwarnings + RUSTDOCFLAGS: -Dwarnings + MSRV: "1.66" + +jobs: + build_and_test_ffi_bindings: + name: Build and test ffi bindings + runs-on: ${{ matrix.name }} + strategy: + fail-fast: false + matrix: + name: [macOS-latest] + rust: [nightly, stable] + steps: + - name: Checkout + uses: actions/checkout@master + with: + submodules: recursive + + - name: Install ${{ matrix.rust }} + uses: dtolnay/rust-toolchain@master + with: + toolchain: ${{ matrix.rust }} + + - name: Create a copy of iroh.h + run: | + cd iroh-ffi + touch iroh.h + cp iroh.h iroh.compare.h + + - name: tests (all features) + run: | + cd iroh-ffi + cargo test --all-features --lib --bins --tests + + - name: tests (no features) + run: | + cd iroh-ffi + cargo test --no-default-features --lib --bins --tests + + - name: tests (default features) + run: | + cd iroh-ffi + cargo test --lib --bins --tests + + - name: Build and test bindings + run: | + cd iroh-ffi + cargo build --release + cd c/ + make build + + - name: Compare iroh.h + run: | + cd iroh-ffi + diff iroh.h iroh.compare.h + cmp --silent iroh.h iroh.compare.h || (echo -e "iroh.h file has changed." & (exit 1)) diff --git a/.github/workflows/netsim.yml b/.github/workflows/netsim.yml index f154f5ed6d..98d4153047 100644 --- a/.github/workflows/netsim.yml +++ b/.github/workflows/netsim.yml @@ -10,7 +10,8 @@ on: env: RUST_BACKTRACE: 1 RUSTFLAGS: -Dwarnings - MSRV: "1.63" + RUSTDOCFLAGS: -Dwarnings + MSRV: "1.66" jobs: netsim: diff --git a/iroh-ffi/iroh.h b/iroh-ffi/iroh.h new file mode 100644 index 0000000000..c50b3a1bbb --- /dev/null +++ b/iroh-ffi/iroh.h @@ -0,0 +1,111 @@ +/*! \file */ +/******************************************* + * * + * File auto-generated by `::safer_ffi`. * + * * + * Do not manually edit this file. * + * * + *******************************************/ + +#ifndef __RUST_IROH_FFI__ +#define __RUST_IROH_FFI__ +#ifdef __cplusplus +extern "C" { +#endif + + +#include +#include + +/** \brief + * Constant values for error codes from iroh_error_t. + */ +/** \remark Has the same ABI as `uint32_t` **/ +#ifdef DOXYGEN +typedef +#endif +enum iroh_error_code { + /** */ + IROH_ERROR_CODE_OTHER = 1, +} +#ifndef DOXYGEN +; typedef uint32_t +#endif +iroh_error_code_t; + +/** \brief + * @class iroh_error_t + * An opaque struct representing an error. + */ +typedef struct iroh_error iroh_error_t; + +/** \brief + * @memberof iroh_error_t + * Returns an error code that identifies the error. + */ +uint32_t +iroh_error_code_get ( + iroh_error_t const * error); + +/** \brief + * @memberof iroh_error_t + * Deallocate an iroh_error_t. + */ +void +iroh_error_free ( + iroh_error_t * error); + +/** \brief + * @memberof iroh_error_t + * Returns an owned string describing the error in greater detail. + * + * Caller is responsible for deallocating returned string via iroh_string_free. + */ +char * +iroh_error_message_get ( + iroh_error_t const * error); + +/** \brief + * @class iroh_node_t + */ +typedef struct iroh_node iroh_node_t; + +/** \brief + * @memberof iroh_node_t + * Deallocate a ns_noosphere_t instance. + */ +void +iroh_free ( + iroh_node_t * node); + +/** \brief + * @memberof iroh_node_t + * Get a collection from a peer. + */ +iroh_error_t * +iroh_get_ticket ( + iroh_node_t const * node, + char const * ticket, + char const * out_path); + +/** \brief + * @memberof iroh_node_t + * Initialize a iroh_node_t instance. + * + */ +iroh_node_t * +iroh_initialize (void); + +/** \brief + * Deallocate an Iroh-allocated string. + */ +void +iroh_string_free ( + char * string); + + +#ifdef __cplusplus +} /* extern \"C\" */ +#endif + +#endif /* __RUST_IROH_FFI__ */ From 5ea389baa72bcc80104966dc8bd3cb7805ff166f Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Mon, 31 Jul 2023 21:22:57 +0200 Subject: [PATCH 30/32] use patched system-configuration --- Cargo.lock | 6 ++---- Cargo.toml | 6 +++++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c2aeaadf60..594a430d52 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3961,8 +3961,7 @@ dependencies = [ [[package]] name = "system-configuration" version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +source = "git+https://github.com/tmpfs/system-configuration-rs?branch=ios-hack#572913bf0c74ed996b2beeac3d61ed681075cbd4" dependencies = [ "bitflags 1.3.2", "core-foundation", @@ -3972,8 +3971,7 @@ dependencies = [ [[package]] name = "system-configuration-sys" version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +source = "git+https://github.com/tmpfs/system-configuration-rs?branch=ios-hack#572913bf0c74ed996b2beeac3d61ed681075cbd4" dependencies = [ "core-foundation-sys", "libc", diff --git a/Cargo.toml b/Cargo.toml index 2c0c940882..b24c69b363 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,4 +17,8 @@ incremental = false [profile.ffi] inherits = 'release' -strip = "symbols" \ No newline at end of file +strip = "symbols" + +[patch.crates-io] +# https://github.com/mullvad/system-configuration-rs/pull/42 +system-configuration = { git = "https://github.com/tmpfs/system-configuration-rs", branch = "ios-hack" } \ No newline at end of file From 39df47730872f435c04297daee577480b2e7539e Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Mon, 31 Jul 2023 21:34:02 +0200 Subject: [PATCH 31/32] cleanup --- .cargo/config.toml | 26 -------------------------- .gitignore | 1 + 2 files changed, 1 insertion(+), 26 deletions(-) diff --git a/.cargo/config.toml b/.cargo/config.toml index 605f803434..018fa32046 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,28 +1,2 @@ [build] rustflags = ["-Wmissing_debug_implementations"] - - -[target.x86_64-apple-darwin] -# linker = "/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang" -# linker = "/usr/bin/clang" -linker = "rust-lld" - -[target.aarch64-apple-darwin] -# linker = "/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang" -# linker = "/usr/bin/clang" -linker = "rust-lld" - -[target.aarch64-apple-ios] -# linker = "/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang" -# linker = "/usr/bin/clang" -linker = "rust-lld" - -[target.x86_64-apple-ios] -# linker = "/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang" -# linker = "/usr/bin/clang" -linker = "rust-lld" - -[target.aarch64-apple-ios-sim] -# linker = "/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/clang" -# linker = "/usr/bin/clang" -linker = "rust-lld" diff --git a/.gitignore b/.gitignore index fe8147e83b..9e422c7cc6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ /target iroh.config.toml +/build From 9ffab35d333b9ba286c77ac08d4bbdd23827199b Mon Sep 17 00:00:00 2001 From: dignifiedquire Date: Mon, 31 Jul 2023 21:47:06 +0200 Subject: [PATCH 32/32] fixup merge --- .github/workflows/netsim.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/netsim.yml b/.github/workflows/netsim.yml index 7924e2d1d2..259dea05ed 100644 --- a/.github/workflows/netsim.yml +++ b/.github/workflows/netsim.yml @@ -10,14 +10,8 @@ on: env: RUST_BACKTRACE: 1 RUSTFLAGS: -Dwarnings -<<<<<<< HEAD RUSTDOCFLAGS: -Dwarnings MSRV: "1.66" -||||||| 5bc9c047 - MSRV: "1.63" -======= - MSRV: "1.66" ->>>>>>> origin/main jobs: netsim: