diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 9f2939618..addf1acf1 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -22,6 +22,8 @@ jobs: toolchain: 1.65.0 override: true components: rustfmt + - name: Add wasm32 target + run: rustup target add wasm32-unknown-unknown - name: Checkout uses: actions/checkout@v2 with: @@ -32,6 +34,18 @@ jobs: run: cargo build --release - name: unit tests run: cargo test -- --nocapture + - name: (MacOS) install LLVM + uses: KyleMayes/install-llvm-action@v2 + if: "${{ matrix.platform == 'macos-latest' }}" + with: + version: "17" + - name: (MacOS) set LLVM as CC + if: "${{ matrix.platform == 'macos-latest' }}" + run: echo "CC=$(pwd)/llvm/bin/clang-17" >> $GITHUB_ENV + - name: build (wasm32) + run: cargo build --target wasm32-unknown-unknown + - name: check (wasm32) + run: cargo check --target wasm32-unknown-unknown test-windows: runs-on: windows-latest steps: diff --git a/Cargo.toml b/Cargo.toml index 1ded686d2..54d68a7e1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,6 @@ test = false doc = false [dependencies] -libc = "0.2" lz4-sys = { path = "lz4-sys", version = "1.10.0" } [dev-dependencies] diff --git a/lz4-sys/build.rs b/lz4-sys/build.rs index c69d25716..025b7083c 100644 --- a/lz4-sys/build.rs +++ b/lz4-sys/build.rs @@ -40,6 +40,14 @@ fn run() -> Result<(), Box> { } } } + let need_wasm_shim = target == "wasm32-unknown-unknown" || target.starts_with("wasm32-wasi"); + + if need_wasm_shim { + println!("cargo:rerun-if-changed=wasm-shim/stdlib.h"); + println!("cargo:rerun-if-changed=wasm-shim/string.h"); + + compiler.include("wasm-shim/"); + } compiler.compile("liblz4.a"); let src = env::current_dir()?.join("liblz4").join("lib"); diff --git a/lz4-sys/src/lib.rs b/lz4-sys/src/lib.rs index 6804f6d2d..559809e19 100644 --- a/lz4-sys/src/lib.rs +++ b/lz4-sys/src/lib.rs @@ -5,7 +5,19 @@ extern crate libc; target_arch = "wasm32", not(any(target_env = "wasi", target_os = "wasi")) )))] -use libc::{c_void, c_char, c_uint, size_t, c_int, c_ulonglong}; +pub use libc::{c_char, c_int, c_uint, c_ulonglong, c_void, size_t}; + +#[cfg(all( + target_arch = "wasm32", + not(any(target_env = "wasi", target_os = "wasi")) +))] +extern crate alloc; + +#[cfg(all( + target_arch = "wasm32", + not(any(target_env = "wasi", target_os = "wasi")) +))] +mod wasm_shim; #[cfg(all( target_arch = "wasm32", @@ -17,14 +29,14 @@ extern crate std; target_arch = "wasm32", not(any(target_env = "wasi", target_os = "wasi")) ))] -use std::os::raw::{c_void, c_char, c_uint, c_int}; +pub use std::os::raw::{c_char, c_int, c_uint, c_ulonglong, c_void}; #[cfg(all( target_arch = "wasm32", not(any(target_env = "wasi", target_os = "wasi")) ))] #[allow(non_camel_case_types)] -type size_t = usize; +pub type size_t = usize; #[derive(Clone, Copy, Debug)] #[repr(C)] diff --git a/lz4-sys/src/wasm_shim.rs b/lz4-sys/src/wasm_shim.rs new file mode 100644 index 000000000..e6bbfb53e --- /dev/null +++ b/lz4-sys/src/wasm_shim.rs @@ -0,0 +1,123 @@ +//! A shim for the libc functions used in lz4-rs that are not available when building for wasm +//! targets. Adapted from the shim present in the [zstd](https://github.com/gyscos/zstd-rs) crate. +//! zstd-rs license here: +//! The MIT License (MIT) +//! Copyright (c) 2016 Alexandre Bury +//! +//! Permission is hereby granted, free of charge, to any person obtaining a copy of this software +//! and associated documentation files (the "Software"), to deal in the Software without +//! restriction, including without limitation the rights to use, copy, modify, merge, publish, +//! distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the +//! Software is furnished to do so, subject to the following conditions: +//! +//! The above copyright notice and this permission notice shall be included in all copies or +//! substantial portions of the Software. +//! +//! THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING +//! BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +//! NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +//! DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +//! OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +use alloc::alloc::{alloc, alloc_zeroed, dealloc, Layout}; +use core::ffi::{c_int, c_void}; + +const USIZE_ALIGN: usize = core::mem::align_of::(); +const USIZE_SIZE: usize = core::mem::size_of::(); + +#[no_mangle] +pub extern "C" fn rust_lz4_wasm_shim_malloc(size: usize) -> *mut c_void { + wasm_shim_alloc::(size) +} + +#[no_mangle] +pub extern "C" fn rust_lz4_wasm_shim_memcmp( + str1: *const c_void, + str2: *const c_void, + n: usize, +) -> i32 { + // Safety: function contracts requires str1 and str2 at least `n`-long. + unsafe { + let str1: &[u8] = core::slice::from_raw_parts(str1 as *const u8, n); + let str2: &[u8] = core::slice::from_raw_parts(str2 as *const u8, n); + match str1.cmp(str2) { + core::cmp::Ordering::Less => -1, + core::cmp::Ordering::Equal => 0, + core::cmp::Ordering::Greater => 1, + } + } +} + +#[no_mangle] +pub extern "C" fn rust_lz4_wasm_shim_calloc(nmemb: usize, size: usize) -> *mut c_void { + // note: calloc expects the allocation to be zeroed + wasm_shim_alloc::(nmemb * size) +} + +#[inline] +fn wasm_shim_alloc(size: usize) -> *mut c_void { + // in order to recover the size upon free, we store the size below the allocation + // special alignment is never requested via the malloc API, + // so it's not stored, and usize-alignment is used + // memory layout: [size] [allocation] + + let full_alloc_size = size + USIZE_SIZE; + + unsafe { + let layout = Layout::from_size_align_unchecked(full_alloc_size, USIZE_ALIGN); + + let ptr = if ZEROED { + alloc_zeroed(layout) + } else { + alloc(layout) + }; + + // SAFETY: ptr is usize-aligned and we've allocated sufficient memory + ptr.cast::().write(full_alloc_size); + + ptr.add(USIZE_SIZE).cast() + } +} + +#[no_mangle] +pub unsafe extern "C" fn rust_lz4_wasm_shim_free(ptr: *mut c_void) { + // the layout for the allocation needs to be recovered for dealloc + // - the size must be recovered from directly below the allocation + // - the alignment will always by USIZE_ALIGN + + let alloc_ptr = ptr.sub(USIZE_SIZE); + // SAFETY: the allocation routines must uphold having a valid usize below the provided pointer + let full_alloc_size = alloc_ptr.cast::().read(); + + let layout = Layout::from_size_align_unchecked(full_alloc_size, USIZE_ALIGN); + dealloc(alloc_ptr.cast(), layout); +} + +#[no_mangle] +pub unsafe extern "C" fn rust_lz4_wasm_shim_memcpy( + dest: *mut c_void, + src: *const c_void, + n: usize, +) -> *mut c_void { + core::ptr::copy_nonoverlapping(src as *const u8, dest as *mut u8, n); + dest +} + +#[no_mangle] +pub unsafe extern "C" fn rust_lz4_wasm_shim_memmove( + dest: *mut c_void, + src: *const c_void, + n: usize, +) -> *mut c_void { + core::ptr::copy(src as *const u8, dest as *mut u8, n); + dest +} + +#[no_mangle] +pub unsafe extern "C" fn rust_lz4_wasm_shim_memset( + dest: *mut c_void, + c: c_int, + n: usize, +) -> *mut c_void { + core::ptr::write_bytes(dest as *mut u8, c as u8, n); + dest +} diff --git a/lz4-sys/wasm-shim/assert.h b/lz4-sys/wasm-shim/assert.h new file mode 100644 index 000000000..682b8e0bc --- /dev/null +++ b/lz4-sys/wasm-shim/assert.h @@ -0,0 +1,6 @@ +#ifndef _ASSERT_H +#define _ASSERT_H + +#define assert(expr) + +#endif // _ASSERT_H diff --git a/lz4-sys/wasm-shim/stdlib.h b/lz4-sys/wasm-shim/stdlib.h new file mode 100644 index 000000000..c8f822b05 --- /dev/null +++ b/lz4-sys/wasm-shim/stdlib.h @@ -0,0 +1,14 @@ +#include + +#ifndef _STDLIB_H +#define _STDLIB_H 1 + +void *rust_lz4_wasm_shim_malloc(size_t size); +void *rust_lz4_wasm_shim_calloc(size_t nmemb, size_t size); +void rust_lz4_wasm_shim_free(void *ptr); + +#define malloc(size) rust_lz4_wasm_shim_malloc(size) +#define calloc(nmemb, size) rust_lz4_wasm_shim_calloc(nmemb, size) +#define free(ptr) rust_lz4_wasm_shim_free(ptr) + +#endif // _STDLIB_H diff --git a/lz4-sys/wasm-shim/string.h b/lz4-sys/wasm-shim/string.h new file mode 100644 index 000000000..83504fe52 --- /dev/null +++ b/lz4-sys/wasm-shim/string.h @@ -0,0 +1,31 @@ +#include + +#ifndef _STRING_H +#define _STRING_H 1 + +int rust_lz4_wasm_shim_memcmp(const void *str1, const void *str2, size_t n); +void *rust_lz4_wasm_shim_memcpy(void *restrict dest, const void *restrict src, size_t n); +void *rust_lz4_wasm_shim_memmove(void *dest, const void *src, size_t n); +void *rust_lz4_wasm_shim_memset(void *dest, int c, size_t n); + +inline int memcmp(const void *str1, const void *str2, size_t n) +{ + return rust_lz4_wasm_shim_memcmp(str1, str2, n); +} + +inline void *memcpy(void *restrict dest, const void *restrict src, size_t n) +{ + return rust_lz4_wasm_shim_memcpy(dest, src, n); +} + +inline void *memmove(void *dest, const void *src, size_t n) +{ + return rust_lz4_wasm_shim_memmove(dest, src, n); +} + +inline void *memset(void *dest, int c, size_t n) +{ + return rust_lz4_wasm_shim_memset(dest, c, n); +} + +#endif // _STRING_H diff --git a/src/lib.rs b/src/lib.rs index 674bf9c30..7667063d1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,3 @@ -extern crate libc; extern crate lz4_sys; pub mod liblz4; @@ -16,21 +15,4 @@ pub use crate::liblz4::BlockMode; pub use crate::liblz4::BlockSize; pub use crate::liblz4::ContentChecksum; -#[cfg(not(all( - target_arch = "wasm32", - not(any(target_env = "wasi", target_os = "wasi")) -)))] -use libc::{c_char, size_t}; - -#[cfg(all( - target_arch = "wasm32", - not(any(target_env = "wasi", target_os = "wasi")) -))] -use std::os::raw::c_char; - -#[cfg(all( - target_arch = "wasm32", - not(any(target_env = "wasi", target_os = "wasi")) -))] -#[allow(non_camel_case_types)] -type size_t = usize; +use lz4_sys::{c_char, size_t};