From ce503f7412713355fa04aeb5e8e2dfc112801f64 Mon Sep 17 00:00:00 2001 From: Mike Waychison Date: Tue, 21 Nov 2023 16:10:38 -0500 Subject: [PATCH 1/5] russimp: Introduce FileSystem loading When loading resources from memory buffers, sometimes the resources in question link to secondary files that must also be loaded. An example of this is the loading of .obj files, which store their materials in a secondary .mtl file. This file can't be passed in using the from_buffer() loading path, resulting in the need to load the files from the host native filesystem using the from_file() loading path. This change implements a rust safe loading path to allowing applications to implement their own virtual filesystem interface with assimp. This is convenient for use in combination with an application specific resource loader. To use, callers simply implement the fs::FileSystem trait, which neccesitates the implementation of open() which returns their own type as a trait object implementing fs::FileOperations. This new fs module handles all the unsafe glue required to interface this safe trait-based interface with the underlying unsafe aiFileIO parts. --- src/fs.rs | 336 +++++++++++++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 1 + src/scene.rs | 24 +++- 3 files changed, 359 insertions(+), 2 deletions(-) create mode 100644 src/fs.rs diff --git a/src/fs.rs b/src/fs.rs new file mode 100644 index 0000000..604ae46 --- /dev/null +++ b/src/fs.rs @@ -0,0 +1,336 @@ +//! The `fs` module contains functionality for interfacing custom resource loading. +//! +//! Implement the FileSystem trait for your custom resource loading, with its open() method returning +//! objects satisfying the FileOperations trait. +use russimp_sys::{aiFile, aiFileIO, aiOrigin, aiReturn, size_t}; +use std::convert::TryInto; +use std::ffi::CStr; +use std::io::SeekFrom; + +/// Implement FileSystem to use custom resource loading using `Scene::from_filesystem()`. +/// +/// Rusty version of the underlying aiFileIO type. +pub trait FileSystem { + fn open(&self, file_path: &str, mode: &str) -> Option>; +} + +/// Implement this for a given resource to support custom resource loading. +/// +/// This trait class is the rusty version of the underlying aiFile type. +pub trait FileOperations { + /// Should return the number of bytes read, or Err if read unsuccessful. + fn read(&mut self, buf: &mut [u8]) -> Result; + /// Should return the number of bytes written, or Err if write unsuccessful. + fn write(&mut self, buf: &[u8]) -> Result; + fn tell(&mut self) -> u64; + fn size(&mut self) -> u64; + fn seek(&mut self, seek_from: SeekFrom) -> Result<(), ()>; + fn flush(&mut self); + fn close(&mut self); +} + +/// This type allows us to generate C stubs for whatever trait object the user supplies. +pub struct FileOperationsWrapper { + ai_file: aiFileIO, + _phantom: std::marker::PhantomData, +} + +impl FileOperationsWrapper { + /// Returns a wrapper that can create an aiFileIO to be used with the assimp C-API. + pub fn new(file_system: &T) -> FileOperationsWrapper { + let trait_obj: &dyn FileSystem = file_system; + let managed_box = Box::new(trait_obj); + let user_data = Box::into_raw(managed_box); + let user_data = user_data as *mut i8; + FileOperationsWrapper { + ai_file: aiFileIO { + OpenProc: Some(FileOperationsWrapper::::io_open), + CloseProc: Some(FileOperationsWrapper::::io_close), + UserData: user_data, + }, + _phantom: Default::default(), + } + } + /// Get the aiFileIO to pass to the C-interface. + pub fn ai_file(&mut self) -> &mut aiFileIO { + &mut self.ai_file + } + /// Implementation for aiFileIO::OpenProc. + unsafe extern "C" fn io_open( + ai_file_io: *mut aiFileIO, + file_path: *const ::std::os::raw::c_char, + mode: *const ::std::os::raw::c_char, + ) -> *mut aiFile { + let file_system = Box::leak(Box::from_raw( + (*ai_file_io).UserData as *mut &mut dyn FileSystem, + )); + + let file_path = CStr::from_ptr(file_path) + .to_str() + .unwrap_or("Invalid UTF-8 Filename"); + let mode = CStr::from_ptr(mode) + .to_str() + .unwrap_or("Invalid UTF-8 Mode"); + let file = match file_system.open(file_path, mode) { + None => return std::ptr::null_mut(), + Some(file) => file, + }; + + // Take the returned file, and double box it here so that it can be converted to a single + // raw pointer that can be stuffed in the UserData. + let double_box = Box::new(file); + let managed_box = Box::into_raw(double_box); // Cleaned up in io_close. + let user_data = managed_box as *mut i8; + let ai_file = aiFile { + ReadProc: Some(Self::io_read), + WriteProc: Some(Self::io_write), + TellProc: Some(Self::io_tell), + FileSizeProc: Some(Self::io_size), + SeekProc: Some(Self::io_seek), + FlushProc: Some(Self::io_flush), + UserData: user_data, + }; + // Lifetime of ai_file is managed by backend assimp library, cleaned up in io_close(). + Box::into_raw(Box::new(ai_file)) + } + + /// Implementation for aiFileIO::CloseProc. + unsafe extern "C" fn io_close(_ai_file_io: *mut aiFileIO, ai_file: *mut aiFile) { + // Given that this is close, we are careful to not leak, but instead drop the file when we + // exit this scope. + let ai_file = Box::from_raw(ai_file); + let mut file: Box> = + Box::from_raw((*ai_file).UserData as *mut Box); + file.close(); + } + /// Turn an aiFile pointer into a the "self" object. + /// + /// Safety: Only safe to call once from within each of the io_* callbacks. This assumes that + /// the loading library has ownership of the aiFile object that was returned by the FileSystem. + /// It is expected to only be called serially on a single thread for the lifetype 'a, which + /// *should* keep access scoped to within the io_* callback. + unsafe fn get_file<'a>(ai_file: *mut aiFile) -> &'a mut Box { + // We return a "leaked" pointer here, using the saved off double box pointer that we + // stuffed in the UserData. We "leak" as we don't acutally want to return ownership, only + // the mutable reference. The box is manually cleaned up as part of io_close. + Box::leak(Box::from_raw( + (*ai_file).UserData as *mut Box, + )) + } + // Implementation for aiFile::ReadProc. + unsafe extern "C" fn io_read( + ai_file: *mut aiFile, + buffer: *mut std::os::raw::c_char, + size: size_t, + count: size_t, + ) -> size_t { + let file = Self::get_file(ai_file); + let mut buffer = + std::slice::from_raw_parts_mut(buffer as *mut u8, (size * count).try_into().unwrap()); + if size == 0 { + panic!("Size 0 is invalid"); + } + if count == 0 { + panic!("Count 0 is invalid"); + } + if size > std::usize::MAX as u64 { + panic!("huge read size not supported"); + } + let size = size as usize; + if size == 1 { + // This looks like a memcpy. + if count > std::usize::MAX as u64 { + panic!("huge read not supported"); + } + let count = count as usize; + + let (buffer, _) = buffer.split_at_mut(count); + match file.read(buffer) { + Ok(size) => size as u64, + Err(_) => std::u64::MAX, + } + } else { + // We have to copy in strides. Implement this by looping for each object and tally the + // count of full objects read. + let mut total: u64 = 0; + for _ in 0..count { + let split = buffer.split_at_mut(size as usize); + buffer = split.1; + let bytes_read = match file.read(split.0) { + Err(_) => break, + Ok(bytes_read) => bytes_read, + }; + if bytes_read != size { + break; + } + total = total + 1; + } + total + } + } + // Implementation for aiFile::WriteProc. + unsafe extern "C" fn io_write( + ai_file: *mut aiFile, + buffer: *const std::os::raw::c_char, + size: size_t, + count: size_t, + ) -> size_t { + let file = Self::get_file(ai_file); + let mut buffer = + std::slice::from_raw_parts(buffer as *mut u8, (size * count).try_into().unwrap()); + if size == 0 { + panic!("Write of size 0"); + } + if count == 0 { + panic!("Write of count 0"); + } + if size > std::usize::MAX as u64 { + panic!("huge write size not supported"); + } + let size = size as usize; + if size == 1 { + if count > std::usize::MAX as u64 { + panic!("huge write not supported"); + } + let count = count as usize; + + let (buffer, _) = buffer.split_at(count); + match file.write(buffer) { + Ok(size) => size as u64, + Err(_) => std::u64::MAX, + } + } else { + // Write in strides. Implement this by looping for each object and tally the + // count of full objects written. + let mut total: u64 = 0; + for _ in 0..count { + let split = buffer.split_at(size as usize); + buffer = split.1; + let bytes_written = match file.write(split.0) { + Err(_) => break, + Ok(bytes_written) => bytes_written, + }; + if bytes_written != size { + break; + } + total = total + 1; + } + total + } + } + // Implementation for aiFile::TellProc. + unsafe extern "C" fn io_tell(ai_file: *mut aiFile) -> size_t { + let file = Self::get_file(ai_file); + file.tell() + } + // Implementation for aiFile::FileSizeProc. + unsafe extern "C" fn io_size(ai_file: *mut aiFile) -> size_t { + let file = Self::get_file(ai_file); + file.size() + } + // Implementation for aiFile::SeekProc. + unsafe extern "C" fn io_seek(ai_file: *mut aiFile, pos: size_t, origin: aiOrigin) -> aiReturn { + let file = Self::get_file(ai_file); + let seek_from = match origin { + russimp_sys::aiOrigin_aiOrigin_SET => SeekFrom::Start(pos), + russimp_sys::aiOrigin_aiOrigin_CUR => SeekFrom::Current(pos as i64), + russimp_sys::aiOrigin_aiOrigin_END => SeekFrom::End(pos as i64), + _ => panic!("Assimp passed invalid origin"), + }; + match file.seek(seek_from) { + Ok(()) => 0, + Err(()) => russimp_sys::aiReturn_aiReturn_FAILURE, + } + } + // Implementation for aiFile::FlushProc. + unsafe extern "C" fn io_flush(ai_file: *mut aiFile) { + let file = Self::get_file(ai_file); + file.flush(); + } +} + +impl Drop for FileOperationsWrapper { + fn drop(&mut self) { + // Re-construct and drop the box that was used for the C-API. + let _managed_box: Box<&dyn FileSystem> = + unsafe { Box::from_raw(self.ai_file.UserData as *mut &dyn FileSystem) }; + } +} + +#[cfg(test)] +mod test { + use crate::scene::PostProcess; + use crate::scene::Scene; + use crate::utils; + use std::fs::File; + use std::io::{SeekFrom, prelude::*}; + + struct MyFileOperations { + file: File, + } + + impl super::FileOperations for MyFileOperations { + fn read(&mut self, buf: &mut [u8]) -> Result { + self.file.read(buf).or_else(|_| Err(())) + } + + fn write(&mut self, _buf: &[u8]) -> Result { + unimplemented!("write support"); + } + + fn tell(&mut self) -> u64 { + self.file.seek(SeekFrom::Current(0)).unwrap_or(0) + } + + fn size(&mut self) -> u64 { + self.file.metadata().expect("Missing metadata").len() + } + + fn seek(&mut self, seek_from: SeekFrom) -> Result<(), ()> { + match self.file.seek(seek_from) { + Ok(_) => Ok(()), + Err(_) => Err(()), + } + } + + fn flush(&mut self) { + // write suppot not implemented. + } + + fn close(&mut self) { + // Nothing to do. + } + } + + struct MyFS {} + + impl super::FileSystem for MyFS { + fn open(&self, file_path: &str, mode: &str) -> Option> { + // We only support reading for this test. + assert_eq!(mode, "rb"); + let file = File::open(file_path).expect("Couldn't open {file_path}"); + Some(Box::new(MyFileOperations{file})) + } + } + + #[test] + fn test_file_operations() { + // Load the cube.obj as it also has to load the cube.mtl through the filesystem to get the + // materials right. + let model_path = utils::get_model("models/OBJ/cube.obj"); + let mut myfs = MyFS {}; + let scene = Scene::from_file_system( + model_path.as_str(), + vec![ + PostProcess::CalculateTangentSpace, + PostProcess::Triangulate, + PostProcess::JoinIdenticalVertices, + PostProcess::SortByPrimitiveType, + ], + &mut myfs, + ).unwrap(); + + assert_eq!(scene.meshes[0].texture_coords.len(), 8); + assert_eq!(scene.materials.len(), 2); + } +} diff --git a/src/lib.rs b/src/lib.rs index ae1ceee..2cb27f7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -25,6 +25,7 @@ pub mod animation; pub mod bone; pub mod camera; pub mod face; +pub mod fs; pub mod light; pub mod material; pub mod mesh; diff --git a/src/scene.rs b/src/scene.rs index 3c43e0b..ff21549 100644 --- a/src/scene.rs +++ b/src/scene.rs @@ -1,7 +1,7 @@ use crate::material::generate_materials; use crate::{ - animation::Animation, camera::Camera, light::Light, material::Material, mesh::Mesh, - metadata::MetaData, node::Node, sys::*, *, + animation::Animation, camera::Camera, fs::{FileSystem, FileOperationsWrapper}, light::Light, material::Material, + mesh::Mesh, metadata::MetaData, node::Node, sys::*, *, }; use std::{ ffi::{CStr, CString}, @@ -430,6 +430,20 @@ impl Scene { result } + pub fn from_file_system(file_path: &str, flags: PostProcessSteps, file_io: &mut T) -> Russult { + let bitwise_flag = flags.into_iter().fold(0, |acc, x| acc | (x as u32)); + let file_path = CString::new(file_path).unwrap(); + + let raw_scene = Scene::get_scene_from_filesystem(file_path, bitwise_flag, file_io); + if raw_scene.is_none() { + return Err(Scene::get_error()); + } + + let result = Scene::new(raw_scene.unwrap()); + Scene::drop_scene(raw_scene); + + result + } pub fn from_buffer(buffer: &[u8], flags: PostProcessSteps, hint: &str) -> Russult { let bitwise_flag = flags.into_iter().fold(0, |acc, x| acc | (x as u32)); let hint = CString::new(hint).unwrap(); @@ -459,6 +473,12 @@ impl Scene { unsafe { aiImportFile(string.as_ptr(), flags).as_ref() } } + #[inline] + fn get_scene_from_filesystem<'a, T: FileSystem>(string: CString, flags: u32, fs: &mut T) -> Option<&'a aiScene> { + let mut file_io = FileOperationsWrapper::new(fs); + unsafe { aiImportFileEx(string.as_ptr(), flags, file_io.ai_file()).as_ref() } + } + #[inline] fn get_scene_from_file_from_memory<'a>( buffer: &[u8], From 86aa13efb13b68b635e6c1e3e961abb7a76f35c5 Mon Sep 17 00:00:00 2001 From: Mike Waychison Date: Fri, 24 Nov 2023 15:49:37 -0500 Subject: [PATCH 2/5] fs: use usize consistently Fixed various issues when building against russimp-sys 2.0. --- src/fs.rs | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/src/fs.rs b/src/fs.rs index 604ae46..4214eba 100644 --- a/src/fs.rs +++ b/src/fs.rs @@ -2,7 +2,7 @@ //! //! Implement the FileSystem trait for your custom resource loading, with its open() method returning //! objects satisfying the FileOperations trait. -use russimp_sys::{aiFile, aiFileIO, aiOrigin, aiReturn, size_t}; +use russimp_sys::{aiFile, aiFileIO, aiOrigin, aiReturn}; use std::convert::TryInto; use std::ffi::CStr; use std::io::SeekFrom; @@ -22,8 +22,8 @@ pub trait FileOperations { fn read(&mut self, buf: &mut [u8]) -> Result; /// Should return the number of bytes written, or Err if write unsuccessful. fn write(&mut self, buf: &[u8]) -> Result; - fn tell(&mut self) -> u64; - fn size(&mut self) -> u64; + fn tell(&mut self) -> usize; + fn size(&mut self) -> usize; fn seek(&mut self, seek_from: SeekFrom) -> Result<(), ()>; fn flush(&mut self); fn close(&mut self); @@ -121,9 +121,9 @@ impl FileOperationsWrapper { unsafe extern "C" fn io_read( ai_file: *mut aiFile, buffer: *mut std::os::raw::c_char, - size: size_t, - count: size_t, - ) -> size_t { + size: usize, + count: usize, + ) -> usize { let file = Self::get_file(ai_file); let mut buffer = std::slice::from_raw_parts_mut(buffer as *mut u8, (size * count).try_into().unwrap()); @@ -133,26 +133,26 @@ impl FileOperationsWrapper { if count == 0 { panic!("Count 0 is invalid"); } - if size > std::usize::MAX as u64 { + if size > std::usize::MAX { panic!("huge read size not supported"); } let size = size as usize; if size == 1 { // This looks like a memcpy. - if count > std::usize::MAX as u64 { + if count > std::usize::MAX { panic!("huge read not supported"); } let count = count as usize; let (buffer, _) = buffer.split_at_mut(count); match file.read(buffer) { - Ok(size) => size as u64, - Err(_) => std::u64::MAX, + Ok(size) => size, + Err(_) => std::usize::MAX, } } else { // We have to copy in strides. Implement this by looping for each object and tally the // count of full objects read. - let mut total: u64 = 0; + let mut total: usize = 0; for _ in 0..count { let split = buffer.split_at_mut(size as usize); buffer = split.1; @@ -172,9 +172,9 @@ impl FileOperationsWrapper { unsafe extern "C" fn io_write( ai_file: *mut aiFile, buffer: *const std::os::raw::c_char, - size: size_t, - count: size_t, - ) -> size_t { + size: usize, + count: usize, + ) -> usize { let file = Self::get_file(ai_file); let mut buffer = std::slice::from_raw_parts(buffer as *mut u8, (size * count).try_into().unwrap()); @@ -184,25 +184,25 @@ impl FileOperationsWrapper { if count == 0 { panic!("Write of count 0"); } - if size > std::usize::MAX as u64 { + if size > std::usize::MAX { panic!("huge write size not supported"); } let size = size as usize; if size == 1 { - if count > std::usize::MAX as u64 { + if count > std::usize::MAX { panic!("huge write not supported"); } let count = count as usize; let (buffer, _) = buffer.split_at(count); match file.write(buffer) { - Ok(size) => size as u64, - Err(_) => std::u64::MAX, + Ok(size) => size, + Err(_) => std::usize::MAX, } } else { // Write in strides. Implement this by looping for each object and tally the // count of full objects written. - let mut total: u64 = 0; + let mut total: usize = 0; for _ in 0..count { let split = buffer.split_at(size as usize); buffer = split.1; @@ -219,20 +219,20 @@ impl FileOperationsWrapper { } } // Implementation for aiFile::TellProc. - unsafe extern "C" fn io_tell(ai_file: *mut aiFile) -> size_t { + unsafe extern "C" fn io_tell(ai_file: *mut aiFile) -> usize { let file = Self::get_file(ai_file); file.tell() } // Implementation for aiFile::FileSizeProc. - unsafe extern "C" fn io_size(ai_file: *mut aiFile) -> size_t { + unsafe extern "C" fn io_size(ai_file: *mut aiFile) -> usize { let file = Self::get_file(ai_file); file.size() } // Implementation for aiFile::SeekProc. - unsafe extern "C" fn io_seek(ai_file: *mut aiFile, pos: size_t, origin: aiOrigin) -> aiReturn { + unsafe extern "C" fn io_seek(ai_file: *mut aiFile, pos: usize, origin: aiOrigin) -> aiReturn { let file = Self::get_file(ai_file); let seek_from = match origin { - russimp_sys::aiOrigin_aiOrigin_SET => SeekFrom::Start(pos), + russimp_sys::aiOrigin_aiOrigin_SET => SeekFrom::Start(pos as u64), russimp_sys::aiOrigin_aiOrigin_CUR => SeekFrom::Current(pos as i64), russimp_sys::aiOrigin_aiOrigin_END => SeekFrom::End(pos as i64), _ => panic!("Assimp passed invalid origin"), From c58341ce34e521df496567707a90fda0df2c4d0b Mon Sep 17 00:00:00 2001 From: Mike Waychison Date: Fri, 24 Nov 2023 15:55:43 -0500 Subject: [PATCH 3/5] fs: more missed usize --- src/fs.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fs.rs b/src/fs.rs index 4214eba..dc6d4aa 100644 --- a/src/fs.rs +++ b/src/fs.rs @@ -278,11 +278,11 @@ mod test { unimplemented!("write support"); } - fn tell(&mut self) -> u64 { + fn tell(&mut self) -> usize { self.file.seek(SeekFrom::Current(0)).unwrap_or(0) } - fn size(&mut self) -> u64 { + fn size(&mut self) -> usize { self.file.metadata().expect("Missing metadata").len() } From 1e90aa25278733126630d2ee6c9e883e82c50886 Mon Sep 17 00:00:00 2001 From: Mike Waychison Date: Sat, 25 Nov 2023 14:34:34 -0500 Subject: [PATCH 4/5] pub(crate) FileOperationsWrapper --- src/fs.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fs.rs b/src/fs.rs index dc6d4aa..419a9dc 100644 --- a/src/fs.rs +++ b/src/fs.rs @@ -30,7 +30,7 @@ pub trait FileOperations { } /// This type allows us to generate C stubs for whatever trait object the user supplies. -pub struct FileOperationsWrapper { +pub(crate) struct FileOperationsWrapper { ai_file: aiFileIO, _phantom: std::marker::PhantomData, } From 35f9ff5c97a6ce14f778303a6f17364a75c209fe Mon Sep 17 00:00:00 2001 From: Mike Waychison Date: Sat, 25 Nov 2023 14:41:57 -0500 Subject: [PATCH 5/5] Use immutable reference for FileSystem Remove unneccesary and wrong mut usage. FileSystem is only used as an immutable reference, usage expects user to use internal mutability if they so desire it, as this is the least restrictive option vs requiring mutable FileSystem. --- src/fs.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fs.rs b/src/fs.rs index 419a9dc..6f7f766 100644 --- a/src/fs.rs +++ b/src/fs.rs @@ -62,7 +62,7 @@ impl FileOperationsWrapper { mode: *const ::std::os::raw::c_char, ) -> *mut aiFile { let file_system = Box::leak(Box::from_raw( - (*ai_file_io).UserData as *mut &mut dyn FileSystem, + (*ai_file_io).UserData as *mut &dyn FileSystem, )); let file_path = CStr::from_ptr(file_path)