diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b3adca1..8d0c4a3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,11 +32,7 @@ jobs: cache-directories: ${{ env.CARGO_TARGET_DIR }} - name: "Check" - run: cargo check --all - - name: "Build (Without Python node as it is build with maturin)" - run: cargo build --all --exclude dora-node-api-python - - name: "Test" - run: cargo test --all --exclude dora-ros2-bridge-python + run: cargo check --all --all-features clippy: name: "Clippy" @@ -48,14 +44,7 @@ jobs: - run: cargo --version --verbose - name: "Clippy" - run: cargo clippy --all - - name: "Clippy (tracing feature)" - run: cargo clippy --all --features tracing - if: false # only the dora-runtime has this feature, but it is currently commented out - - name: "Clippy (metrics feature)" - run: cargo clippy --all --features metrics - if: false # only the dora-runtime has this feature, but it is currently commented out - + run: cargo clippy --all --all-features rustfmt: name: "Formatting" runs-on: ubuntu-latest diff --git a/.github/workflows/pip_release.yml b/.github/workflows/pip_release.yml index 1a5898e..f149f53 100644 --- a/.github/workflows/pip_release.yml +++ b/.github/workflows/pip_release.yml @@ -44,7 +44,7 @@ jobs: uses: PyO3/maturin-action@v1 with: target: ${{ matrix.platform.target }} - args: --release --out dist --find-interpreter + args: --release --out dist --find-interpreter --manifest-path libraries/fastformat/Cargo.toml --all-features sccache: "true" manylinux: auto - name: Upload wheels @@ -73,7 +73,7 @@ jobs: uses: PyO3/maturin-action@v1 with: target: ${{ matrix.platform.target }} - args: --release --out dist --find-interpreter + args: --release --out dist --find-interpreter --manifest-path libraries/fastformat/Cargo.toml --all-features sccache: "true" manylinux: musllinux_1_2 - name: Upload wheels @@ -108,7 +108,7 @@ jobs: target: ${{ matrix.platform.target }} manylinux: auto container: off - args: --release -o dist -i 3.8 + args: --release -o dist -i 3.8 --manifest-path libraries/fastformat/Cargo.toml --all-features - name: Upload wheels uses: actions/upload-artifact@v3 with: @@ -134,7 +134,7 @@ jobs: uses: PyO3/maturin-action@v1 with: target: ${{ matrix.platform.target }} - args: --release --out dist --find-interpreter + args: --release --out dist --find-interpreter --manifest-path libraries/fastformat/Cargo.toml --all-features sccache: "true" - name: Upload wheels uses: actions/upload-artifact@v4 @@ -160,7 +160,7 @@ jobs: uses: PyO3/maturin-action@v1 with: target: ${{ matrix.platform.target }} - args: --release --out dist --find-interpreter + args: --release --out dist --find-interpreter --manifest-path libraries/fastformat/Cargo.toml --all-features sccache: "true" - name: Upload wheels uses: actions/upload-artifact@v4 @@ -176,7 +176,7 @@ jobs: uses: PyO3/maturin-action@v1 with: command: sdist - args: --out dist + args: --out dist --manifest-path libraries/fastformat/Cargo.toml - name: Upload sdist uses: actions/upload-artifact@v4 with: diff --git a/.gitignore b/.gitignore index c8f0442..94325a3 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,7 @@ __pycache__/ .pytest_cache/ *.py[cod] -# C extensions +# C extensionst *.so # Distribution / packaging @@ -70,3 +70,15 @@ docs/_build/ # Pyenv .python-version + +# PyO3 config +.pyo3_config + +# UV +uv.lock + +# Dora +dora-coordinator.txt +dora-daemon.txt + +out/ diff --git a/Cargo.lock b/Cargo.lock index 0b42c06..23dc5b0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,21 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "addr2line" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + [[package]] name = "ahash" version = "0.8.11" @@ -25,6 +40,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "aligned-vec" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4aa90d7ce82d4be67b64039a3d588d38dbcc6736577de4a847025ce5b0c468d1" +dependencies = [ + "serde", +] + [[package]] name = "android-tzdata" version = "0.1.1" @@ -40,6 +64,55 @@ dependencies = [ "libc", ] +[[package]] +name = "anstream" +version = "0.6.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64e15c1ab1f89faffbf04a634d5e1962e9074f2741eef6d97f3c4e322426d526" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1" + +[[package]] +name = "anstyle-parse" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb47de1e80c2b463c735db5b217a0ddc39d612e7ac9e2e96a5aed1f57616c1cb" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d36fc52c7f6c869915e99412912f22093507da8d9e942ceaf66fe4b7c14422a" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bf74e1b6e971609db8ca7a9ce79fd5768ab6ae46441c572e46cf596f59e57f8" +dependencies = [ + "anstyle", + "windows-sys 0.52.0", +] + [[package]] name = "arrow" version = "52.2.0" @@ -88,7 +161,7 @@ dependencies = [ "arrow-schema", "chrono", "half", - "hashbrown", + "hashbrown 0.14.5", "num", ] @@ -181,7 +254,7 @@ dependencies = [ "arrow-schema", "chrono", "half", - "indexmap", + "indexmap 2.5.0", "lexical-core", "num", "serde", @@ -222,6 +295,9 @@ name = "arrow-schema" version = "52.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e972cd1ff4a4ccd22f86d3e53e835c2ed92e0eea6a3e8eadb72b4f1ac802cf8" +dependencies = [ + "serde", +] [[package]] name = "arrow-select" @@ -251,7 +327,18 @@ dependencies = [ "memchr", "num", "regex", - "regex-syntax", + "regex-syntax 0.8.4", +] + +[[package]] +name = "async-trait" +version = "0.1.82" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a27b8a3a6e1a44fa4c8baf1f653e4172e81486d4941f2237e20dc2d0cf4ddff1" +dependencies = [ + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -269,12 +356,56 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +[[package]] +name = "backtrace" +version = "0.3.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cc23269a4f8976d0a4d2e7109211a419fe30e8d88d677cd60b6bc79c5732e0a" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + [[package]] name = "base64" version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "benchmark" +version = "0.1.0" +dependencies = [ + "eyre", + "which 6.0.3", +] + +[[package]] +name = "benchmark-node" +version = "0.1.0" +dependencies = [ + "arrow", + "clap", + "dora-node-api", + "eyre", + "fastformat", + "rand", +] + +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -287,12 +418,30 @@ version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +[[package]] +name = "bitvec" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + [[package]] name = "bumpalo" version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.7.1" @@ -301,9 +450,12 @@ checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50" [[package]] name = "cc" -version = "1.1.7" +version = "1.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26a5c3fd7bfa1ce3897a3a3501d362b2d87b7f2583ebcb4a949ec25911025cbc" +checksum = "e9d013ecb737093c0e86b151a7b837993cf9ec6c502946cfb44bedc392421e0b" +dependencies = [ + "shlex", +] [[package]] name = "cfg-if" @@ -320,9 +472,55 @@ dependencies = [ "android-tzdata", "iana-time-zone", "num-traits", - "windows-targets", + "windows-targets 0.52.6", +] + +[[package]] +name = "clap" +version = "4.5.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e5a21b8495e732f1b3c364c9949b201ca7bae518c502c80256c96ad79eaf6ac" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.5.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cf2dd12af7a047ad9d6da2b6b249759a22a7abc0f474c1dae1777afa4b21a73" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.5.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "501d359d5f3dcaf6ecdeee48833ae73ec6e42723a1e52419c79abf9507eec0a0" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn", ] +[[package]] +name = "clap_lex" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" + +[[package]] +name = "colorchoice" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0" + [[package]] name = "const-random" version = "0.1.18" @@ -345,9 +543,24 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "crossbeam-channel" +version = "0.5.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" [[package]] name = "crunchy" @@ -377,546 +590,1701 @@ dependencies = [ ] [[package]] -name = "equivalent" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" - -[[package]] -name = "eyre" -version = "0.6.12" +name = "dashmap" +version = "5.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cd915d99f24784cdc19fd37ef22b97e3ff0ae756c7e492e9fbfe897d61e2aec" +checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ - "indenter", + "cfg-if", + "hashbrown 0.14.5", + "lock_api", "once_cell", + "parking_lot_core", ] [[package]] -name = "fastformat" -version = "0.1.0" -dependencies = [ - "arrow", - "eyre", - "ndarray", - "pyo3", -] +name = "diatomic-waker" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af873b6853650fb206431c52fa7bbf6917146b70a8a9979d6d141f5d5394086b" [[package]] -name = "flatbuffers" -version = "24.3.25" +name = "dirs" +version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8add37afff2d4ffa83bc748a70b4b1370984f6980768554182424ef71447c35f" +checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059" dependencies = [ - "bitflags 1.3.2", - "rustc_version", + "dirs-sys", ] [[package]] -name = "getrandom" -version = "0.2.15" +name = "dirs-sys" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" dependencies = [ - "cfg-if", "libc", - "wasi", + "redox_users", + "winapi", ] [[package]] -name = "half" -version = "2.4.1" +name = "dora-arrow-convert" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" +checksum = "de3227f41fe84f388d7539484250191b47c878157c10a63fb4440b98138aeb9b" dependencies = [ - "cfg-if", - "crunchy", - "num-traits", + "arrow", + "eyre", ] [[package]] -name = "hashbrown" -version = "0.14.5" +name = "dora-core" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +checksum = "de19282b81e0d276301d4e485ff47857ebe319ed8d1ac9da17743558041464ce" +dependencies = [ + "eyre", + "log", + "once_cell", + "schemars", + "serde", + "serde-with-expand-env", + "serde_json", + "serde_yaml 0.9.34+deprecated", + "tokio", + "tracing", + "uhlc", + "uuid", + "which 5.0.0", +] [[package]] -name = "heck" -version = "0.4.1" +name = "dora-message" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +checksum = "2b9baa6308c128806479dfadaf417e48de29033d80d63c309968ca3184a54287" +dependencies = [ + "aligned-vec", + "arrow-data", + "arrow-schema", + "dora-core", + "eyre", + "log", + "semver", + "serde", + "tokio", + "uuid", +] [[package]] -name = "iana-time-zone" -version = "0.1.60" +name = "dora-node-api" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" +checksum = "85d73f6a4df64fd63da996e25147c6e34067c28d1f3c104c81c4ba97658e959f" dependencies = [ - "android_system_properties", - "core-foundation-sys", - "iana-time-zone-haiku", - "js-sys", - "wasm-bindgen", - "windows-core", + "aligned-vec", + "arrow", + "bincode", + "dora-arrow-convert", + "dora-core", + "dora-message", + "dora-tracing", + "eyre", + "flume", + "futures", + "futures-concurrency", + "futures-timer", + "serde_json", + "serde_yaml 0.8.26", + "shared-memory-server", + "shared_memory_extended", + "tracing", ] [[package]] -name = "iana-time-zone-haiku" -version = "0.1.2" +name = "dora-tracing" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +checksum = "f4963d389df992accf27e94b1100605d07e64fbb9743a7fd451f66541e99e4c4" dependencies = [ - "cc", + "eyre", + "opentelemetry", + "opentelemetry-jaeger", + "tracing", + "tracing-opentelemetry", + "tracing-subscriber", ] [[package]] -name = "indenter" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683" +name = "dummy-opencv-capture" +version = "0.1.0" +dependencies = [ + "arrow", + "fastformat", +] [[package]] -name = "indexmap" -version = "2.3.0" +name = "dyn-clone" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de3fc2e30ba82dd1b3911c8de1ffc143c74a914a14e99514d7637e3099df5ea0" -dependencies = [ - "equivalent", - "hashbrown", -] +checksum = "0d6ef0072f8a535281e4876be788938b528e9a1d43900b82c2569af7da799125" [[package]] -name = "indoc" -version = "2.0.5" +name = "either" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" [[package]] -name = "itoa" -version = "1.0.11" +name = "equivalent" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] -name = "js-sys" -version = "0.3.69" +name = "errno" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" +checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" dependencies = [ - "wasm-bindgen", + "libc", + "windows-sys 0.52.0", ] [[package]] -name = "lazy_static" -version = "1.5.0" +name = "eyre" +version = "0.6.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +checksum = "7cd915d99f24784cdc19fd37ef22b97e3ff0ae756c7e492e9fbfe897d61e2aec" +dependencies = [ + "indenter", + "once_cell", +] [[package]] -name = "lexical-core" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cde5de06e8d4c2faabc400238f9ae1c74d5412d03a7bd067645ccbc47070e46" +name = "fastformat" +version = "0.1.0" dependencies = [ - "lexical-parse-float", - "lexical-parse-integer", - "lexical-util", - "lexical-write-float", - "lexical-write-integer", + "fastformat-converter", + "fastformat-datatypes", + "pyo3", ] [[package]] -name = "lexical-parse-float" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "683b3a5ebd0130b8fb52ba0bdc718cc56815b6a097e28ae5a6997d0ad17dc05f" +name = "fastformat-converter" +version = "0.1.0" dependencies = [ - "lexical-parse-integer", - "lexical-util", - "static_assertions", + "arrow", + "eyre", + "ndarray 0.16.1", + "numpy", +] + +[[package]] +name = "fastformat-datatypes" +version = "0.1.0" +dependencies = [ + "arrow", + "eyre", + "fastformat-converter", + "ndarray 0.16.1", +] + +[[package]] +name = "fastrand" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" +dependencies = [ + "instant", +] + +[[package]] +name = "flatbuffers" +version = "24.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8add37afff2d4ffa83bc748a70b4b1370984f6980768554182424ef71447c35f" +dependencies = [ + "bitflags 1.3.2", + "rustc_version", +] + +[[package]] +name = "flume" +version = "0.10.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1657b4441c3403d9f7b3409e47575237dac27b1b5726df654a6ecbf92f0f7577" +dependencies = [ + "futures-core", + "futures-sink", + "nanorand", + "pin-project", + "spin", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + +[[package]] +name = "futures" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-buffered" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91fa130f3777d0d4b0993653c20bc433026d3290627693c4ed1b18dd237357ab" +dependencies = [ + "diatomic-waker", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "futures-channel" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-concurrency" +version = "7.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b14ac911e85d57c5ea6eef76d7b4d4a3177ecd15f4bea2e61927e9e3823e19f" +dependencies = [ + "bitvec", + "futures-buffered", + "futures-core", + "futures-lite", + "pin-project", + "slab", + "smallvec", +] + +[[package]] +name = "futures-core" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" + +[[package]] +name = "futures-executor" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" + +[[package]] +name = "futures-lite" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49a9d51ce47660b1e808d3c990b4709f2f415d928835a17dfd16991515c46bce" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "memchr", + "parking", + "pin-project-lite", + "waker-fn", +] + +[[package]] +name = "futures-macro" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" + +[[package]] +name = "futures-task" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" + +[[package]] +name = "futures-timer" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" + +[[package]] +name = "futures-util" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "gimli" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" + +[[package]] +name = "half" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" +dependencies = [ + "cfg-if", + "crunchy", + "num-traits", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "home" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + +[[package]] +name = "iana-time-zone" +version = "0.1.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "indenter" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683" + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", +] + +[[package]] +name = "indexmap" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68b900aa2f7301e21c36462b170ee99994de34dff39a4a6a528e80e7376d07e5" +dependencies = [ + "equivalent", + "hashbrown 0.14.5", +] + +[[package]] +name = "indoc" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" + +[[package]] +name = "instant" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "integer-encoding" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + +[[package]] +name = "itoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" + +[[package]] +name = "js-sys" +version = "0.3.70" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "lexical-core" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cde5de06e8d4c2faabc400238f9ae1c74d5412d03a7bd067645ccbc47070e46" +dependencies = [ + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", + "lexical-write-float", + "lexical-write-integer", +] + +[[package]] +name = "lexical-parse-float" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683b3a5ebd0130b8fb52ba0bdc718cc56815b6a097e28ae5a6997d0ad17dc05f" +dependencies = [ + "lexical-parse-integer", + "lexical-util", + "static_assertions", ] [[package]] name = "lexical-parse-integer" version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d0994485ed0c312f6d965766754ea177d07f9c00c9b82a5ee62ed5b47945ee9" +checksum = "6d0994485ed0c312f6d965766754ea177d07f9c00c9b82a5ee62ed5b47945ee9" +dependencies = [ + "lexical-util", + "static_assertions", +] + +[[package]] +name = "lexical-util" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5255b9ff16ff898710eb9eb63cb39248ea8a5bb036bea8085b1a767ff6c4e3fc" +dependencies = [ + "static_assertions", +] + +[[package]] +name = "lexical-write-float" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accabaa1c4581f05a3923d1b4cfd124c329352288b7b9da09e766b0668116862" +dependencies = [ + "lexical-util", + "lexical-write-integer", + "static_assertions", +] + +[[package]] +name = "lexical-write-integer" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1b6f3d1f4422866b68192d62f77bc5c700bee84f3069f2469d7bc8c77852446" +dependencies = [ + "lexical-util", + "static_assertions", +] + +[[package]] +name = "libc" +version = "0.2.158" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439" + +[[package]] +name = "libm" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" + +[[package]] +name = "libredox" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" +dependencies = [ + "bitflags 2.6.0", + "libc", +] + +[[package]] +name = "linked-hash-map" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" + +[[package]] +name = "linux-raw-sys" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" + +[[package]] +name = "lock_api" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" +dependencies = [ + "serde", +] + +[[package]] +name = "matchers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +dependencies = [ + "regex-automata 0.1.10", +] + +[[package]] +name = "matrixmultiply" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9380b911e3e96d10c1f415da0876389aaf1b56759054eeb0de7df940c456ba1a" +dependencies = [ + "autocfg", + "rawpointer", +] + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "memoffset" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" +dependencies = [ + "autocfg", +] + +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + +[[package]] +name = "miniz_oxide" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08" +dependencies = [ + "adler", +] + +[[package]] +name = "mio" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec" +dependencies = [ + "hermit-abi", + "libc", + "wasi", + "windows-sys 0.52.0", +] + +[[package]] +name = "nanorand" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a51313c5820b0b02bd422f4b44776fbf47961755c74ce64afc73bfad10226c3" +dependencies = [ + "getrandom", +] + +[[package]] +name = "ndarray" +version = "0.15.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb12d4e967ec485a5f71c6311fe28158e9d6f4bc4a447b474184d0f91a8fa32" +dependencies = [ + "matrixmultiply", + "num-complex", + "num-integer", + "num-traits", + "rawpointer", +] + +[[package]] +name = "ndarray" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "882ed72dce9365842bf196bdeedf5055305f11fc8c03dee7bb0194a6cad34841" +dependencies = [ + "matrixmultiply", + "num-complex", + "num-integer", + "num-traits", + "portable-atomic", + "portable-atomic-util", + "rawpointer", +] + +[[package]] +name = "nix" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f3790c00a0150112de0f4cd161e3d7fc4b2d8a5542ffc35f099a2562aecb35c" +dependencies = [ + "bitflags 1.3.2", + "cc", + "cfg-if", + "libc", + "memoffset 0.6.5", +] + +[[package]] +name = "nix" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b" +dependencies = [ + "bitflags 1.3.2", + "cfg-if", + "libc", +] + +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "num_cpus" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "numpy" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec170733ca37175f5d75a5bea5911d6ff45d2cd52849ce98b685394e4f2f37f4" +dependencies = [ + "libc", + "ndarray 0.15.6", + "num-complex", + "num-integer", + "num-traits", + "pyo3", + "rustc-hash", +] + +[[package]] +name = "object" +version = "0.36.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "084f1a5821ac4c651660a94a7153d27ac9d8a53736203f58b31945ded098070a" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" + +[[package]] +name = "opentelemetry" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69d6c3d7288a106c0a363e4b0e8d308058d56902adefb16f4936f417ffef086e" +dependencies = [ + "opentelemetry_api", + "opentelemetry_sdk", +] + +[[package]] +name = "opentelemetry-jaeger" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e785d273968748578931e4dc3b4f5ec86b26e09d9e0d66b55adda7fce742f7a" +dependencies = [ + "async-trait", + "futures", + "futures-executor", + "once_cell", + "opentelemetry", + "opentelemetry-semantic-conventions", + "thiserror", + "thrift", + "tokio", +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b02e0230abb0ab6636d18e2ba8fa02903ea63772281340ccac18e0af3ec9eeb" +dependencies = [ + "opentelemetry", +] + +[[package]] +name = "opentelemetry_api" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c24f96e21e7acc813c7a8394ee94978929db2bcc46cf6b5014fc612bf7760c22" +dependencies = [ + "fnv", + "futures-channel", + "futures-util", + "indexmap 1.9.3", + "js-sys", + "once_cell", + "pin-project-lite", + "thiserror", +] + +[[package]] +name = "opentelemetry_sdk" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ca41c4933371b61c2a2f214bf16931499af4ec90543604ec828f7a625c09113" +dependencies = [ + "async-trait", + "crossbeam-channel", + "dashmap", + "fnv", + "futures-channel", + "futures-executor", + "futures-util", + "once_cell", + "opentelemetry_api", + "percent-encoding", + "rand", + "thiserror", + "tokio", + "tokio-stream", +] + +[[package]] +name = "ordered-float" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3305af35278dd29f46fcdd139e0b1fbfae2153f0e5928b39b035542dd31e37b7" +dependencies = [ + "num-traits", +] + +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + +[[package]] +name = "parking" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae" + +[[package]] +name = "parking_lot" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets 0.52.6", +] + +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + +[[package]] +name = "pin-project" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "portable-atomic" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da544ee218f0d287a911e9c99a39a8c9bc8fcad3cb8db5959940044ecfc67265" + +[[package]] +name = "portable-atomic-util" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcdd8420072e66d54a407b3316991fe946ce3ab1083a7f575b2463866624704d" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "pyo3" +version = "0.21.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e00b96a521718e08e03b1a622f01c8a8deb50719335de3f60b3b3950f069d8" +dependencies = [ + "cfg-if", + "indoc", + "libc", + "memoffset 0.9.1", + "parking_lot", + "portable-atomic", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", + "unindent", +] + +[[package]] +name = "pyo3-build-config" +version = "0.21.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7883df5835fafdad87c0d888b266c8ec0f4c9ca48a5bed6bbb592e8dedee1b50" +dependencies = [ + "once_cell", + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.21.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01be5843dc60b916ab4dad1dca6d20b9b4e6ddc8e15f50c47fe6d85f1fb97403" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-macros" +version = "0.21.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77b34069fc0682e11b31dbd10321cbf94808394c56fd996796ce45217dfac53c" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.21.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08260721f32db5e1a5beae69a55553f56b99bd0e1c3e6e0a5e8851a9d0f5a85c" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "pyo3-build-config", + "quote", + "syn", +] + +[[package]] +name = "quote" +version = "1.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "lexical-util", - "static_assertions", + "getrandom", ] [[package]] -name = "lexical-util" -version = "0.8.5" +name = "raw_sync_2" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5255b9ff16ff898710eb9eb63cb39248ea8a5bb036bea8085b1a767ff6c4e3fc" +checksum = "f067b45fa17e31d15636789c2638bd562da5496d498876cf0495df78f7e4fdcb" dependencies = [ - "static_assertions", + "cfg-if", + "libc", + "nix 0.23.2", + "rand", + "winapi", ] [[package]] -name = "lexical-write-float" -version = "0.8.5" +name = "rawpointer" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accabaa1c4581f05a3923d1b4cfd124c329352288b7b9da09e766b0668116862" +checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" + +[[package]] +name = "redox_syscall" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a908a6e00f1fdd0dfd9c0eb08ce85126f6d8bbda50017e74bc4a4b7d4a926a4" dependencies = [ - "lexical-util", - "lexical-write-integer", - "static_assertions", + "bitflags 2.6.0", ] [[package]] -name = "lexical-write-integer" -version = "0.8.5" +name = "redox_users" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1b6f3d1f4422866b68192d62f77bc5c700bee84f3069f2469d7bc8c77852446" +checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" dependencies = [ - "lexical-util", - "static_assertions", + "getrandom", + "libredox", + "thiserror", ] [[package]] -name = "libc" -version = "0.2.155" +name = "regex" +version = "1.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" +checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata 0.4.7", + "regex-syntax 0.8.4", +] [[package]] -name = "libm" -version = "0.2.8" +name = "regex-automata" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +dependencies = [ + "regex-syntax 0.6.29", +] [[package]] -name = "lock_api" -version = "0.4.12" +name = "regex-automata" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" dependencies = [ - "autocfg", - "scopeguard", + "aho-corasick", + "memchr", + "regex-syntax 0.8.4", ] [[package]] -name = "log" -version = "0.4.22" +name = "regex-syntax" +version = "0.6.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] -name = "matrixmultiply" -version = "0.3.9" +name = "regex-syntax" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9380b911e3e96d10c1f415da0876389aaf1b56759054eeb0de7df940c456ba1a" +checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" + +[[package]] +name = "rustc-demangle" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" + +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" dependencies = [ - "autocfg", - "rawpointer", + "semver", ] [[package]] -name = "memchr" -version = "2.7.4" +name = "rustix" +version = "0.38.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +checksum = "3f55e80d50763938498dd5ebb18647174e0c76dc38c5505294bb224624f30f36" +dependencies = [ + "bitflags 2.6.0", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.52.0", +] [[package]] -name = "memoffset" -version = "0.9.1" +name = "ryu" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" + +[[package]] +name = "schemars" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09c024468a378b7e36765cd36702b7a90cc3cba11654f6685c8f233408e89e92" dependencies = [ - "autocfg", + "dyn-clone", + "schemars_derive", + "serde", + "serde_json", ] [[package]] -name = "ndarray" -version = "0.15.6" +name = "schemars_derive" +version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adb12d4e967ec485a5f71c6311fe28158e9d6f4bc4a447b474184d0f91a8fa32" +checksum = "b1eee588578aff73f856ab961cd2f79e36bc45d7ded33a7562adba4667aecc0e" dependencies = [ - "matrixmultiply", - "num-complex", - "num-integer", - "num-traits", - "rawpointer", + "proc-macro2", + "quote", + "serde_derive_internals", + "syn", ] [[package]] -name = "num" -version = "0.4.3" +name = "scopeguard" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "semver" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" dependencies = [ - "num-bigint", - "num-complex", - "num-integer", - "num-iter", - "num-rational", - "num-traits", + "serde", ] [[package]] -name = "num-bigint" -version = "0.4.6" +name = "serde" +version = "1.0.209" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +checksum = "99fce0ffe7310761ca6bf9faf5115afbc19688edd00171d81b1bb1b116c63e09" dependencies = [ - "num-integer", - "num-traits", + "serde_derive", ] [[package]] -name = "num-complex" -version = "0.4.6" +name = "serde-with-expand-env" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +checksum = "888d884a3be3a209308d0b66f1918ff18f60e93db837259e53ea7d8dd14e7e98" dependencies = [ - "num-traits", + "serde", + "shellexpand", ] [[package]] -name = "num-integer" -version = "0.1.46" +name = "serde_derive" +version = "1.0.209" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +checksum = "a5831b979fd7b5439637af1752d535ff49f4860c0f341d1baeb6faf0f4242170" dependencies = [ - "num-traits", + "proc-macro2", + "quote", + "syn", ] [[package]] -name = "num-iter" -version = "0.1.45" +name = "serde_derive_internals" +version = "0.29.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ - "autocfg", - "num-integer", - "num-traits", + "proc-macro2", + "quote", + "syn", ] [[package]] -name = "num-rational" -version = "0.4.2" +name = "serde_json" +version = "1.0.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" dependencies = [ - "num-bigint", - "num-integer", - "num-traits", + "itoa", + "memchr", + "ryu", + "serde", ] [[package]] -name = "num-traits" -version = "0.2.19" +name = "serde_yaml" +version = "0.8.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +checksum = "578a7433b776b56a35785ed5ce9a7e777ac0598aac5a6dd1b4b18a307c7fc71b" dependencies = [ - "autocfg", - "libm", + "indexmap 1.9.3", + "ryu", + "serde", + "yaml-rust", ] [[package]] -name = "once_cell" -version = "1.19.0" +name = "serde_yaml" +version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +dependencies = [ + "indexmap 2.5.0", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] [[package]] -name = "parking_lot" -version = "0.12.3" +name = "sharded-slab" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" dependencies = [ - "lock_api", - "parking_lot_core", + "lazy_static", ] [[package]] -name = "parking_lot_core" -version = "0.9.10" +name = "shared-memory-server" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" +checksum = "c483b5396dedea562f2a86577a5ce1e92840a4f2a4f339a7ff6090ffa0ad6f54" dependencies = [ - "cfg-if", - "libc", - "redox_syscall", - "smallvec", - "windows-targets", + "bincode", + "eyre", + "raw_sync_2", + "serde", + "shared_memory_extended", + "tracing", ] [[package]] -name = "portable-atomic" -version = "1.7.0" +name = "shared_memory_extended" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da544ee218f0d287a911e9c99a39a8c9bc8fcad3cb8db5959940044ecfc67265" +checksum = "004d7ece9a3be64f85471d50967710b0a146144225bed5f0abd0514a3bed086f" +dependencies = [ + "cfg-if", + "libc", + "nix 0.26.4", + "rand", + "win-sys", +] [[package]] -name = "proc-macro2" -version = "1.0.86" +name = "shellexpand" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +checksum = "7ccc8076840c4da029af4f87e4e8daeb0fca6b87bbb02e10cb60b791450e11e4" dependencies = [ - "unicode-ident", + "dirs", ] [[package]] -name = "pyo3" -version = "0.21.2" +name = "shlex" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5e00b96a521718e08e03b1a622f01c8a8deb50719335de3f60b3b3950f069d8" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "signal-hook-registry" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" dependencies = [ - "cfg-if", - "indoc", "libc", - "memoffset", - "parking_lot", - "portable-atomic", - "pyo3-build-config", - "pyo3-ffi", - "pyo3-macros", - "unindent", ] [[package]] -name = "pyo3-build-config" -version = "0.21.2" +name = "slab" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7883df5835fafdad87c0d888b266c8ec0f4c9ca48a5bed6bbb592e8dedee1b50" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" dependencies = [ - "once_cell", - "target-lexicon", + "autocfg", ] [[package]] -name = "pyo3-ffi" -version = "0.21.2" +name = "smallvec" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01be5843dc60b916ab4dad1dca6d20b9b4e6ddc8e15f50c47fe6d85f1fb97403" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" + +[[package]] +name = "socket2" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" dependencies = [ "libc", - "pyo3-build-config", + "windows-sys 0.52.0", ] [[package]] -name = "pyo3-macros" -version = "0.21.2" +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77b34069fc0682e11b31dbd10321cbf94808394c56fd996796ce45217dfac53c" -dependencies = [ - "proc-macro2", - "pyo3-macros-backend", - "quote", - "syn", -] +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] -name = "pyo3-macros-backend" -version = "0.21.2" +name = "syn" +version = "2.0.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08260721f32db5e1a5beae69a55553f56b99bd0e1c3e6e0a5e8851a9d0f5a85c" +checksum = "9f35bcdf61fd8e7be6caf75f429fdca8beb3ed76584befb503b1569faee373ed" dependencies = [ - "heck", "proc-macro2", - "pyo3-build-config", "quote", - "syn", + "unicode-ident", ] [[package]] -name = "quote" -version = "1.0.36" +name = "tap" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" -dependencies = [ - "proc-macro2", -] +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] -name = "rawpointer" -version = "0.2.1" +name = "target-lexicon" +version = "0.12.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" +checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" [[package]] -name = "redox_syscall" -version = "0.5.3" +name = "thiserror" +version = "1.0.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a908a6e00f1fdd0dfd9c0eb08ce85126f6d8bbda50017e74bc4a4b7d4a926a4" +checksum = "c0342370b38b6a11b6cc11d6a805569958d54cfa061a29969c3b5ce2ea405724" dependencies = [ - "bitflags 2.6.0", + "thiserror-impl", ] [[package]] -name = "regex" -version = "1.10.6" +name = "thiserror-impl" +version = "1.0.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619" +checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" dependencies = [ - "aho-corasick", - "memchr", - "regex-automata", - "regex-syntax", + "proc-macro2", + "quote", + "syn", ] [[package]] -name = "regex-automata" -version = "0.4.7" +name = "thread_local" +version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" +checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax", + "cfg-if", + "once_cell", ] [[package]] -name = "regex-syntax" -version = "0.8.4" +name = "threadpool" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" +checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa" +dependencies = [ + "num_cpus", +] [[package]] -name = "rustc_version" -version = "0.4.0" +name = "thrift" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +checksum = "09678c4cdbb4eed72e18b7c2af1329c69825ed16fcbac62d083fc3e2b0590ff0" dependencies = [ - "semver", + "byteorder", + "integer-encoding", + "log", + "ordered-float", + "threadpool", ] [[package]] -name = "ryu" -version = "1.0.18" +name = "tiny-keccak" +version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] [[package]] -name = "scopeguard" -version = "1.2.0" +name = "tokio" +version = "1.40.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +checksum = "e2b070231665d27ad9ec9b8df639893f46727666c6767db40317fbe920a5d998" +dependencies = [ + "backtrace", + "bytes", + "libc", + "mio", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "windows-sys 0.52.0", +] [[package]] -name = "semver" -version = "1.0.23" +name = "tokio-stream" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" +checksum = "4f4e6ce100d0eb49a2734f8c0812bcd324cf357d21810932c5df6b96ef2b86f1" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] [[package]] -name = "serde" -version = "1.0.204" +name = "tracing" +version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ - "serde_derive", + "pin-project-lite", + "tracing-attributes", + "tracing-core", ] [[package]] -name = "serde_derive" -version = "1.0.204" +name = "tracing-attributes" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", @@ -924,53 +2292,82 @@ dependencies = [ ] [[package]] -name = "serde_json" -version = "1.0.122" +name = "tracing-core" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "784b6203951c57ff748476b126ccb5e8e2959a5c19e5c617ab1956be3dbc68da" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" dependencies = [ - "itoa", - "memchr", - "ryu", - "serde", + "once_cell", + "valuable", ] [[package]] -name = "smallvec" -version = "1.13.2" +name = "tracing-log" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" +checksum = "f751112709b4e791d8ce53e32c4ed2d353565a795ce84da2285393f41557bdf2" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] [[package]] -name = "static_assertions" -version = "1.1.0" +name = "tracing-log" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] [[package]] -name = "syn" -version = "2.0.72" +name = "tracing-opentelemetry" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc4b9b9bf2add8093d3f2c0204471e951b2285580335de42f9d2534f3ae7a8af" +checksum = "21ebb87a95ea13271332df069020513ab70bdb5637ca42d6e492dc3bbbad48de" dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", + "once_cell", + "opentelemetry", + "tracing", + "tracing-core", + "tracing-log 0.1.4", + "tracing-subscriber", ] [[package]] -name = "target-lexicon" -version = "0.12.15" +name = "tracing-subscriber" +version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4873307b7c257eddcb50c9bedf158eb669578359fb28428bef438fec8e6ba7c2" +checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log 0.2.0", +] [[package]] -name = "tiny-keccak" -version = "2.0.2" +name = "uhlc" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +checksum = "8d291a7454d390b753ef68df8145da18367e32883ec2fa863959f0aefb915cdb" dependencies = [ - "crunchy", + "hex", + "humantime", + "lazy_static", + "log", + "serde", + "spin", + "uuid", ] [[package]] @@ -985,12 +2382,46 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce" +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "uuid" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314" +dependencies = [ + "getrandom", + "serde", +] + +[[package]] +name = "valuable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" + [[package]] name = "version_check" version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "waker-fn" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "317211a0dc0ceedd78fb2ca9a44aed3d7b9b26f81870d485c07122b4350673b7" + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -999,19 +2430,20 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" +checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5" dependencies = [ "cfg-if", + "once_cell", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" +checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b" dependencies = [ "bumpalo", "log", @@ -1024,9 +2456,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" +checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -1034,9 +2466,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" +checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" dependencies = [ "proc-macro2", "quote", @@ -1047,9 +2479,78 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.92" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" + +[[package]] +name = "which" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bf3ea8596f3a0dd5980b46430f2058dfe2c36a27ccfbb1845d6fbfcd9ba6e14" +dependencies = [ + "either", + "home", + "once_cell", + "rustix", + "windows-sys 0.48.0", +] + +[[package]] +name = "which" +version = "6.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ee928febd44d98f2f459a4a79bd4d928591333a494a10a868418ac1b39cf1f" +dependencies = [ + "either", + "home", + "rustix", + "winsafe", +] + +[[package]] +name = "win-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b7b128a98c1cfa201b09eb49ba285887deb3cbe7466a98850eb1adabb452be5" +dependencies = [ + "windows", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" +checksum = "45296b64204227616fdbf2614cefa4c236b98ee64dfaaaa435207ed99fe7829f" +dependencies = [ + "windows_aarch64_msvc 0.34.0", + "windows_i686_gnu 0.34.0", + "windows_i686_msvc 0.34.0", + "windows_x86_64_gnu 0.34.0", + "windows_x86_64_msvc 0.34.0", +] [[package]] name = "windows-core" @@ -1057,7 +2558,40 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", ] [[package]] @@ -1066,28 +2600,58 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", ] +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_msvc" +version = "0.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17cffbe740121affb56fad0fc0e421804adf0ae00891205213b5cecd30db881d" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_i686_gnu" +version = "0.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2564fde759adb79129d9b4f54be42b32c89970c18ebf93124ca8870a498688ed" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + [[package]] name = "windows_i686_gnu" version = "0.52.6" @@ -1100,36 +2664,103 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_msvc" +version = "0.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cd9d32ba70453522332c14d38814bceeb747d80b3958676007acadd7e166956" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_x86_64_gnu" +version = "0.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfce6deae227ee8d356d19effc141a509cc503dfd1f850622ec4b0f84428e1f4" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_msvc" +version = "0.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d19538ccc21819d01deaf88d6a17eae6596a12e9aafdbb97916fb49896d89de9" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + [[package]] name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winsafe" +version = "0.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d135d17ab770252ad95e9a872d365cf3090e3be864a34ab46f48555993efc904" + +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + +[[package]] +name = "yaml-rust" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" +dependencies = [ + "linked-hash-map", +] + [[package]] name = "zerocopy" version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" dependencies = [ + "byteorder", "zerocopy-derive", ] diff --git a/Cargo.toml b/Cargo.toml index fbf00ca..8dd0273 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,17 +1,28 @@ -[package] -name = "fastformat" +[workspace] +members = [ + "libraries/converter", + "libraries/datatypes", + "libraries/fastformat", + "examples/dummy-opencv-capture", + "examples/benchmark", + "examples/benchmark/node", +] +resolver = "2" + +[workspace.package] version = "0.1.0" +description = "" edition = "2021" +documentation = "" license = "Apache-2.0" -description = "Fast world data converter" +repository = "https://github.com/dora-rs/fastformat/" -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html -[lib] -name = "fastformat" -crate-type = ["cdylib", "lib"] +[workspace.dependencies] +fastformat-datatypes = { path = "libraries/datatypes" } +fastformat-converter = { path = "libraries/converter" } +fastformat = { path = "libraries/fastformat" } -[dependencies] -pyo3 = "0.21.1" -ndarray = "0.15.4" -arrow = "52.1.0" +arrow = "52.2.0" eyre = "0.6.12" +ndarray = "0.16.1" +numpy = "0.21.0" diff --git a/README.md b/README.md index e951c04..59b0163 100644 --- a/README.md +++ b/README.md @@ -21,8 +21,14 @@ Numpy, or Arrow, without unnecessary copies. ## DataTypes - **Image**: (Arrow representation is a **UnionArray**), - - Field "data': UintXArray (e.g [0, 255, 0, 255, 0, 255, ...]) + - Field "data": UintXArray (e.g [0, 255, 0, 255, 0, 255, ...]) - Field "width": Uint32Array (e.g [1280]) - Field "height": Uint32Array (e.g [720]) - Field "encoding": StringArray (e.g ["RGB8"]) - Field "name" (Optional): StringArray (e.g,["image.front_camera"] or [None]) + +- **BBox**: (Arrow representation is a **UnionArray**), + - Field "data": Float32Array (e.g [0.0f32, 1.0f32, ...]) + - Field "confidence": Float32Array (e.g [0.98f32, 0.76f32, ...]) + - Field "label": StringArray (e.g ["cat", "car", ..."]) + - Field "encoding": StringArray (e.g ["XYXY"] or ["XYWH"]) diff --git a/examples/benchmark/Cargo.toml b/examples/benchmark/Cargo.toml new file mode 100644 index 0000000..f65d808 --- /dev/null +++ b/examples/benchmark/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "benchmark" +version.workspace = true +edition.workspace = true +documentation.workspace = true +description.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +which = "6.0.3" +eyre = { workspace = true } diff --git a/examples/benchmark/dataflow_fastformat.yml b/examples/benchmark/dataflow_fastformat.yml new file mode 100644 index 0000000..645c574 --- /dev/null +++ b/examples/benchmark/dataflow_fastformat.yml @@ -0,0 +1,16 @@ +nodes: + - id: sender-fastformat + build: cargo build -p benchmark-node --release + path: ../../target/release/benchmark-node + args: --sender + outputs: + - latency + - throughput + + - id: receiver-fastformat + build: cargo build -p benchmark-node --release + path: ../../target/release/benchmark-node + args: --receiver + inputs: + latency: sender-fastformat/latency + throughput: sender-fastformat/throughput diff --git a/examples/benchmark/dataflow_raw.yml b/examples/benchmark/dataflow_raw.yml new file mode 100644 index 0000000..a9fdb61 --- /dev/null +++ b/examples/benchmark/dataflow_raw.yml @@ -0,0 +1,16 @@ +nodes: + - id: sender-raw + build: cargo build -p benchmark-node --release + path: ../../target/release/benchmark-node + args: --sender --raw + outputs: + - latency + - throughput + + - id: receiver-raw + build: cargo build -p benchmark-node --release + path: ../../target/release/benchmark-node + args: --receiver --raw + inputs: + latency: sender-raw/latency + throughput: sender-raw/throughput diff --git a/examples/benchmark/node/Cargo.toml b/examples/benchmark/node/Cargo.toml new file mode 100644 index 0000000..6ce8ed4 --- /dev/null +++ b/examples/benchmark/node/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "benchmark-node" +version.workspace = true +edition.workspace = true +documentation.workspace = true +description.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +fastformat = { workspace = true, features = ["arrow"] } +arrow = { workspace = true } +eyre = { workspace = true } +clap = { version = "4.5.17", features = ["derive"] } +rand = "0.8.5" +dora-node-api = "0.3.6" diff --git a/examples/benchmark/node/src/main.rs b/examples/benchmark/node/src/main.rs new file mode 100644 index 0000000..2e454b2 --- /dev/null +++ b/examples/benchmark/node/src/main.rs @@ -0,0 +1,214 @@ +use dora_node_api::{dora_core::config::DataId, DoraNode, Event}; +use rand::Rng; + +use std::collections::HashMap; + +use clap::Parser; + +use eyre::Result; + +use std::time::Duration; +use std::time::Instant; + +use fastformat::Image; + +#[derive(Parser, Debug)] +#[command(version, about, long_about = None)] +struct Args { + #[clap(long)] + receiver: bool, + + #[clap(long)] + sender: bool, + + #[clap(long)] + raw: bool, +} + +fn main() -> Result<()> { + let args = Args::parse(); + + let (r, s, raw) = (args.receiver, args.sender, args.raw); + + match (r, s) { + (true, false) => receiver(raw), + (false, true) => sender(raw), + _ => Err(eyre::eyre!( + "Exactly one of --receiver or --sender must be set" + )), + } +} + +fn sender(raw: bool) -> Result<()> { + let latency = DataId::from("latency".to_owned()); + let throughput = DataId::from("throughput".to_owned()); + + let (mut node, _events) = DoraNode::init_from_env()?; + + let sizes: [(u32, u32, u32); 4] = [ + (720, 480, 3), + (1280, 720, 3), + (1920, 1080, 3), + (3840, 2160, 3), + ]; + + let mut data = HashMap::new(); + for (width, height, c) in &sizes { + let size = (width * height * c) as usize; + let vec: Vec = rand::thread_rng() + .sample_iter(rand::distributions::Standard) + .take(size) + .collect(); + + data.insert(size, vec); + } + + // test latency first + for (width, height, c) in &sizes { + let (width, height, c) = (width.clone(), height.clone(), c.clone()); + let size = (width * height * c) as usize; + for _ in 0..300 { + if raw { + let data = data.get(&size).unwrap(); + + node.send_output_raw(latency.clone(), Default::default(), data.len(), |out| { + out.copy_from_slice(&data); + })?; + } else { + let data = data.get(&size).unwrap(); + let image = Image::new_bgr8(data.clone(), width, height, None)?; + let arrow = image.into_arrow()?; + + node.send_output( + latency.clone(), + Default::default(), + arrow::array::UnionArray::from(arrow), + )?; + } + + // sleep a bit to avoid queue buildup + std::thread::sleep(Duration::from_millis(10)); + } + } + + // wait a bit to ensure that all throughput messages reached their target + std::thread::sleep(Duration::from_secs(2)); + + // then throughput with full speed + for (width, height, c) in &sizes { + let (width, height, c) = (width.clone(), height.clone(), c.clone()); + let size = (width * height * c) as usize; + for _ in 0..300 { + if raw { + let data = data.get(&size).unwrap(); + + node.send_output_raw(throughput.clone(), Default::default(), data.len(), |out| { + out.copy_from_slice(&data); + })?; + } else { + let data = data.get(&size).unwrap(); + let image = Image::new_bgr8(data.clone(), width, height, None)?; + let arrow = image.into_arrow()?; + + node.send_output( + throughput.clone(), + Default::default(), + arrow::array::UnionArray::from(arrow), + )?; + } + } + } + + Ok(()) +} + +fn receiver(raw: bool) -> Result<()> { + let (_node, mut events) = DoraNode::init_from_env()?; + + // latency is tested first + let mut latency = true; + + let mut current_size = 0; + let mut n = 0; + let mut start = Instant::now(); + let mut latencies = Vec::new(); + + println!("Latency:"); + + while let Some(event) = events.recv() { + match event { + Event::Input { id, metadata, data } => { + let data_len = match raw { + true => data.len(), + false => { + use arrow::array::Array; + + let image_raw = Image::raw_data(data.0.into_data())?; + let image = Image::view_from_raw_data(&image_raw)?; + + image.data.len() + } + }; + + if data_len != current_size { + if n > 0 { + record_results(start, current_size, n, latencies, latency); + } + current_size = data_len; + n = 0; + start = Instant::now(); + latencies = Vec::new(); + } + + match id.as_str() { + "latency" if latency => {} + "throughput" if latency => { + latency = false; + println!("Throughput:"); + } + "throughput" => {} + other => { + eprintln!("Ignoring unexpected input `{other}`"); + continue; + } + } + + n += 1; + latencies.push( + metadata + .timestamp() + .get_time() + .to_system_time() + .elapsed() + .unwrap_or_default(), + ); + } + Event::InputClosed { id } => { + println!("Input `{id}` was closed"); + } + other => eprintln!("Received unexpected input: {other:?}"), + } + } + + record_results(start, current_size, n, latencies, latency); + + Ok(()) +} + +fn record_results( + start: Instant, + current_size: usize, + n: u32, + latencies: Vec, + latency: bool, +) { + let msg = if latency { + let avg_latency = latencies.iter().sum::() / n; + format!("size {current_size:<#8x}: {avg_latency:?}") + } else { + let duration = start.elapsed(); + let msg_per_sec = n as f64 / duration.as_secs_f64(); + format!("size {current_size:<#8x}: {msg_per_sec:.0} messages per second") + }; + println!("{msg}"); +} diff --git a/examples/benchmark/src/main.rs b/examples/benchmark/src/main.rs new file mode 100644 index 0000000..1134627 --- /dev/null +++ b/examples/benchmark/src/main.rs @@ -0,0 +1,81 @@ +use eyre::{Result, WrapErr}; + +use std::path::Path; + +fn main() -> Result<()> { + let root = Path::new(env!("CARGO_MANIFEST_DIR")); + std::env::set_current_dir(root).wrap_err("failed to set working dir")?; + + let dataflow = Path::new("dataflow_fastformat.yml"); + destroy_dora()?; + spawn_dora()?; + build_dataflow(dataflow)?; + start_dataflow(dataflow)?; + + let dataflow = Path::new("dataflow_raw.yml"); + start_dataflow(dataflow)?; + + Ok(()) +} + +fn destroy_dora() -> eyre::Result<()> { + let dora = which::which("dora").wrap_err("dora not found")?; + let mut cmd = std::process::Command::new(&dora); + + cmd.arg("destroy"); + + if cmd.status().wrap_err("failed to destroy dora")?.success() { + println!("dora destroyed successfully"); + } else { + println!("dora destroy failed"); + } + + Ok(()) +} + +fn spawn_dora() -> eyre::Result<()> { + let dora = which::which("dora").wrap_err("dora not found")?; + let mut cmd = std::process::Command::new(&dora); + + cmd.arg("up"); + + if cmd.status().wrap_err("failed to spawn dora")?.success() { + println!("dora spawned successfully"); + } else { + println!("dora spawn failed"); + } + + Ok(()) +} + +fn build_dataflow(dataflow: &Path) -> eyre::Result<()> { + let dora = which::which("dora").wrap_err("dora not found")?; + let mut cmd = std::process::Command::new(&dora); + + cmd.arg("build"); + cmd.arg("--").arg(dataflow); + + if cmd.status().wrap_err("failed to build dataflow")?.success() { + println!("dataflow built successfully"); + } else { + println!("dataflow build failed"); + } + + Ok(()) +} + +fn start_dataflow(dataflow: &Path) -> eyre::Result<()> { + let dora = which::which("dora").wrap_err("dora not found")?; + let mut cmd = std::process::Command::new(&dora); + + cmd.arg("start"); + cmd.arg("--").arg(dataflow); + + if cmd.status().wrap_err("failed to start dataflow")?.success() { + println!("dataflow executed successfully"); + } else { + println!("dataflow failed"); + } + + Ok(()) +} diff --git a/examples/dummy-opencv-capture/Cargo.toml b/examples/dummy-opencv-capture/Cargo.toml new file mode 100644 index 0000000..9bd193b --- /dev/null +++ b/examples/dummy-opencv-capture/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "dummy-opencv-capture" +version.workspace = true +edition.workspace = true +documentation.workspace = true +description.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +fastformat = { workspace = true, features = ["arrow", "ndarray"] } +arrow = { workspace = true } diff --git a/examples/dummy-opencv-capture/main.rs b/examples/dummy-opencv-capture/src/main.rs similarity index 60% rename from examples/dummy-opencv-capture/main.rs rename to examples/dummy-opencv-capture/src/main.rs index 97b6359..91a403f 100644 --- a/examples/dummy-opencv-capture/main.rs +++ b/examples/dummy-opencv-capture/src/main.rs @@ -1,11 +1,14 @@ extern crate fastformat; -use fastformat::image::Image; +use fastformat::image::{NdarrayImage, NdarrayImageView}; +use fastformat::Image; -fn camera_read() -> ndarray::Array { +use fastformat::ndarray::Ndarray; + +fn camera_read() -> NdarrayImage { // Dummy camera read - let flat_image = vec![0; 27]; + let flat_image = (110..137).collect::>(); println!( "Generate a camera image at address: {:?}", flat_image.as_ptr() @@ -13,23 +16,23 @@ fn camera_read() -> ndarray::Array { let image = Image::new_bgr8(flat_image, 3, 3, None).unwrap(); - return image.bgr8_to_ndarray().unwrap(); + image.into_ndarray().unwrap() } -fn image_show(_frame: ndarray::ArrayView) { +fn image_show(frame: NdarrayImageView) { // Dummy image show - println!("Showing an image."); + println!("{:?}", frame); } -fn send_output(arrow_array: arrow::array::UnionArray) { +fn send_output(arrow_array: arrow::array::ArrayData) { // Dummy send output let image = Image::from_arrow(arrow_array).unwrap(); println!( "Sending an image to dataflow. Image address is: {:?}", - image.as_ptr() + image.data.as_ptr() ); } @@ -37,10 +40,11 @@ fn main() { // Read OpenCV Camera, default is ndarray BGR8 let frame = camera_read(); - let image = Image::bgr8_from_ndarray(frame, Some("camera.left")).unwrap(); + let image = Image::from_ndarray(frame).unwrap(); // Convert to RGB8, apply some filter (Black and White). - let mut frame = image.to_rgb8().unwrap().rgb8_to_ndarray().unwrap(); + let (frame, encoding, name) = image.into_rgb8().unwrap().into_ndarray().unwrap(); + let mut frame = frame.into_u8_ix3().unwrap(); for i in 0..frame.shape()[0] { for j in 0..frame.shape()[1] { @@ -59,10 +63,10 @@ fn main() { } } - let image = Image::rgb8_from_ndarray(frame, Some("camera.left.baw")).unwrap(); + let image = Image::from_ndarray((Ndarray::U8IX3(frame), encoding, name)).unwrap(); // Plot the image, you may only need a ndarray_view - image_show(image.rgb8_to_ndarray_view().unwrap()); + image_show(image.to_ndarray_view().unwrap()); - send_output(image.to_arrow().unwrap()); + send_output(image.into_arrow().unwrap()); } diff --git a/libraries/converter/Cargo.toml b/libraries/converter/Cargo.toml new file mode 100644 index 0000000..9a77680 --- /dev/null +++ b/libraries/converter/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "fastformat-converter" +version.workspace = true +edition.workspace = true +documentation.workspace = true +description.workspace = true +license.workspace = true +repository.workspace = true + +[features] +arrow = ["dep:arrow"] +ndarray = ["dep:ndarray"] +numpy = ["ndarray", "dep:numpy"] +pyarrow = ["arrow"] + +[dependencies] +arrow = { workspace = true, optional = true } +ndarray = { workspace = true, optional = true } +numpy = { workspace = true, optional = true } +eyre = { workspace = true } diff --git a/libraries/converter/src/arrow.rs b/libraries/converter/src/arrow.rs new file mode 100644 index 0000000..89b0ed4 --- /dev/null +++ b/libraries/converter/src/arrow.rs @@ -0,0 +1,399 @@ +use eyre::{Context, OptionExt, Report, Result}; +use std::{collections::HashMap, sync::Arc}; + +pub struct FastFormatArrowRawData { + buffers: HashMap, + offset_buffers: HashMap>, + + array_data: HashMap, +} + +pub struct FastFormatArrowBuilder { + union_children: Vec, + union_fields: Vec<(i8, arrow::datatypes::FieldRef)>, +} + +impl FastFormatArrowRawData { + pub fn new(array_data: arrow::array::ArrayData) -> Result { + use arrow::array::Array; + + let array = arrow::array::UnionArray::from(array_data); + + let mut result = HashMap::new(); + + let (union_fields, _, _, children) = array.into_parts(); + + for (a, b) in union_fields.iter() { + let child = children + .get(a as usize) + .ok_or_eyre(Report::msg( + format!( + "Invalid union array field {}'s index (= {}). Must be >= 0 and correspond to children index in the array", + b, a + ), + ))? + .clone() + .into_data(); + + result.insert(b.name().to_string(), child); + } + + Ok(Self { + buffers: HashMap::new(), + offset_buffers: HashMap::new(), + array_data: result, + }) + } + + pub fn load_primitive( + self, + field: &str, + ) -> Result { + let mut array_data = self.array_data; + let mut buffers = self.buffers; + let offset_buffers = self.offset_buffers; + + let data = array_data.remove(field).ok_or_eyre(Report::msg(format!( + "Invalid field {} for this map of data", + field + )))?; + + let array = arrow::array::PrimitiveArray::::from(data); + let (_, buffer, _) = array.into_parts(); + + buffers.insert(field.to_string(), buffer.into_inner()); + + Ok(Self { + buffers, + offset_buffers, + array_data, + }) + } + + pub fn load_utf(self, field: &str) -> Result { + let mut array_data = self.array_data; + let mut buffers = self.buffers; + let mut offset_buffers = self.offset_buffers; + + let data = array_data.remove(field).ok_or_eyre(Report::msg(format!( + "Invalid field {} for this map of data", + field + )))?; + + let array = arrow::array::StringArray::from(data); + let (offset_buffer, buffer, _) = array.into_parts(); + + buffers.insert(field.to_string(), buffer); + offset_buffers.insert(field.to_string(), offset_buffer); + + Ok(Self { + buffers, + offset_buffers, + array_data, + }) + } + + pub fn utf8_singleton(&self, field: &str) -> Result { + let buffer = self.buffers.get(field).ok_or_eyre(Report::msg(format!( + "Invalid field {} for this map of data", + field + )))?; + + let offset_buffer = self + .offset_buffers + .get(field) + .ok_or_eyre(Report::msg(format!( + "Invalid field {} for this map of data", + field + )))?; + + let slice = buffer.as_slice(); + let mut iterator = offset_buffer.iter(); + iterator.next(); + + let last_offset = iterator.next().cloned().ok_or_eyre(Report::msg(format!( + "No offset associated with field {}", + field + )))? as usize; + + let slice = &slice[0..last_offset]; + + String::from_utf8(slice.to_vec()).wrap_err(Report::msg("Invalid UTF-8 string")) + } + + pub fn utf16_singleton(&self, field: &str) -> Result { + let buffer = self.buffers.get(field).ok_or_eyre(Report::msg(format!( + "Invalid field {} for this map of data", + field + )))?; + + let offset_buffer = self + .offset_buffers + .get(field) + .ok_or_eyre(Report::msg(format!( + "Invalid field {} for this map of data", + field + )))?; + + let slice = buffer.typed_data::(); + let mut iterator = offset_buffer.iter(); + iterator.next(); + + let last_offset = iterator.next().cloned().ok_or_eyre(Report::msg(format!( + "No offset associated with field {}", + field + )))? as usize; + + let slice = &slice[0..last_offset]; + + String::from_utf16(slice).wrap_err(Report::msg("Invalid UTF-16 string")) + } + + pub fn primitive_singleton( + &self, + field: &str, + ) -> Result { + let buffer = self.buffers.get(field).ok_or_eyre(Report::msg(format!( + "Invalid field {} for this map of data", + field + )))?; + + let slice = buffer.typed_data::(); + + Ok(slice[0]) + } + + pub fn utf8_array(&self, field: &str) -> Result> { + let buffer = self.buffers.get(field).ok_or_eyre(Report::msg(format!( + "Invalid field {} for this map of data", + field + )))?; + + let offset_buffer = self + .offset_buffers + .get(field) + .ok_or_eyre(Report::msg(format!( + "Invalid field {} for this map of data", + field + )))?; + + let slice = buffer.as_slice(); + let mut iterator = offset_buffer.iter(); + iterator.next(); + + let mut last_offset = 0; + + iterator + .map(|&offset| { + let offset = offset as usize; + let slice = &slice[last_offset..offset]; + last_offset = offset; + + String::from_utf8(slice.to_vec()) + .wrap_err(Report::msg("Array is not UTF-8 encoded.")) + }) + .collect::>>() + } + + pub fn utf16_array(&self, field: &str) -> Result> { + let buffer = self.buffers.get(field).ok_or_eyre(Report::msg(format!( + "Invalid field {} for this map of data", + field + )))?; + + let offset_buffer = self + .offset_buffers + .get(field) + .ok_or_eyre(Report::msg(format!( + "Invalid field {} for this map of data", + field + )))?; + + let slice = buffer.typed_data::(); + let mut iterator = offset_buffer.iter(); + iterator.next(); + + let mut last_offset = 0; + + iterator + .map(|&offset| { + let offset = offset as usize; + let slice = &slice[last_offset..offset]; + last_offset = offset; + + String::from_utf16(slice).wrap_err(Report::msg("Array is not UTF-16 encoded.")) + }) + .collect::>>() + } + + pub fn primitive_array_view<'a, T: arrow::datatypes::ArrowPrimitiveType>( + &'a self, + field: &str, + ) -> Result<&'a [T::Native]> { + let buffer = self.buffers.get(field).ok_or_eyre(Report::msg(format!( + "Invalid field {} for this map of data", + field + )))?; + + let slice = buffer.typed_data::(); + + Ok(slice) + } + + pub fn primitive_array( + &mut self, + field: &str, + ) -> Result> { + let buffer = self.buffers.remove(field).ok_or_eyre(Report::msg(format!( + "Invalid field {} for this map of data", + field + )))?; + + match buffer.into_vec::() { + Ok(vec) => Ok(vec), + Err(buffer) => { + self.buffers.insert(field.to_string(), buffer); + + Err(Report::msg("Invalid primitive array type. Or the buffer is shared. If you're not sure that the buffer is owned, use primitive_array_view instead.")) + } + } + } +} + +impl FastFormatArrowBuilder { + pub fn new() -> Self { + Self { + union_children: Vec::new(), + union_fields: Vec::new(), + } + } + + pub fn push_primitive_singleton( + self, + field: &str, + value: T::Native, + data_type: arrow::datatypes::DataType, + nullable: bool, + ) -> Self { + let mut union_children = self.union_children; + let mut union_fields = self.union_fields; + + let index = union_children.len(); + + let data = Arc::new(arrow::array::PrimitiveArray::::from_value(value, 1)); + union_children.push(data); + + let field = ( + index as i8, + Arc::new(arrow::datatypes::Field::new(field, data_type, nullable)), + ); + union_fields.push(field); + + Self { + union_children, + union_fields, + } + } + + pub fn push_primitive_array( + self, + field: &str, + value: Vec, + data_type: arrow::datatypes::DataType, + nullable: bool, + ) -> Self { + let mut union_children = self.union_children; + let mut union_fields = self.union_fields; + + let index = union_children.len(); + + let data = Arc::new(arrow::array::PrimitiveArray::::from_iter_values(value)); + union_children.push(data); + + let field = ( + index as i8, + Arc::new(arrow::datatypes::Field::new(field, data_type, nullable)), + ); + union_fields.push(field); + + Self { + union_children, + union_fields, + } + } + + pub fn push_utf_singleton( + self, + field: &str, + value: String, + data_type: arrow::datatypes::DataType, + nullable: bool, + ) -> Self { + let mut union_children = self.union_children; + let mut union_fields = self.union_fields; + + let index = union_children.len(); + + let data = Arc::new(arrow::array::StringArray::from(vec![value])); + union_children.push(data); + + let field = ( + index as i8, + Arc::new(arrow::datatypes::Field::new(field, data_type, nullable)), + ); + union_fields.push(field); + + Self { + union_children, + union_fields, + } + } + + pub fn push_utf_array( + self, + field: &str, + value: Vec, + data_type: arrow::datatypes::DataType, + nullable: bool, + ) -> Self { + let mut union_children = self.union_children; + let mut union_fields = self.union_fields; + + let index = union_children.len(); + + let data = Arc::new(arrow::array::StringArray::from(value)); + union_children.push(data); + + let field = ( + index as i8, + Arc::new(arrow::datatypes::Field::new(field, data_type, nullable)), + ); + union_fields.push(field); + + Self { + union_children, + union_fields, + } + } + + pub fn into_arrow(self) -> Result { + use arrow::array::Array; + + let type_ids = [].into_iter().collect::>(); + let offsets = [].into_iter().collect::>(); + + let union_fields = self + .union_fields + .into_iter() + .collect::(); + + Ok(arrow::array::UnionArray::try_new( + union_fields, + type_ids, + Some(offsets), + self.union_children, + ) + .wrap_err("Failed to create UnionArray with Image data.")? + .into_data()) + } +} diff --git a/libraries/converter/src/lib.rs b/libraries/converter/src/lib.rs new file mode 100644 index 0000000..4070fa8 --- /dev/null +++ b/libraries/converter/src/lib.rs @@ -0,0 +1,5 @@ +#[cfg(feature = "arrow")] +pub mod arrow; + +#[cfg(feature = "ndarray")] +pub mod ndarray; diff --git a/libraries/converter/src/ndarray.rs b/libraries/converter/src/ndarray.rs new file mode 100644 index 0000000..32ba460 --- /dev/null +++ b/libraries/converter/src/ndarray.rs @@ -0,0 +1,79 @@ +use eyre::Result; + +#[derive(Debug)] +pub enum Ndarray { + F32IX1(ndarray::Array), + U8IX2(ndarray::Array), + U8IX3(ndarray::Array), + STRIX1(ndarray::Array), +} + +impl Ndarray { + pub fn as_ptr(&self) -> *const u64 { + match self { + Ndarray::F32IX1(array) => array.as_ptr() as *const u64, + Ndarray::U8IX2(array) => array.as_ptr() as *const u64, + Ndarray::U8IX3(array) => array.as_ptr() as *const u64, + Ndarray::STRIX1(array) => array.as_ptr() as *const u64, + } + } + + pub fn into_u8_ix3(self) -> Result> { + match self { + Ndarray::U8IX3(array) => Ok(array), + _ => Err(eyre::Report::msg("Expected U8IX3")), + } + } + + pub fn into_u8_ix2(self) -> Result> { + match self { + Ndarray::U8IX2(array) => Ok(array), + _ => Err(eyre::Report::msg("Expected U8IX2")), + } + } + + pub fn into_f32_ix1(self) -> Result> { + match self { + Ndarray::F32IX1(array) => Ok(array), + _ => Err(eyre::Report::msg("Expected F32IX1")), + } + } +} + +#[derive(Debug)] +pub enum NdarrayView<'a> { + F32IX1(ndarray::ArrayView<'a, f32, ndarray::Ix1>), + U8IX2(ndarray::ArrayView<'a, u8, ndarray::Ix2>), + U8IX3(ndarray::ArrayView<'a, u8, ndarray::Ix3>), + STRIX1(ndarray::ArrayView<'a, String, ndarray::Ix1>), +} + +impl<'a> NdarrayView<'a> { + pub fn as_ptr(&self) -> *const u64 { + match self { + NdarrayView::F32IX1(array) => array.as_ptr() as *const u64, + NdarrayView::U8IX2(array) => array.as_ptr() as *const u64, + NdarrayView::U8IX3(array) => array.as_ptr() as *const u64, + NdarrayView::STRIX1(array) => array.as_ptr() as *const u64, + } + } +} + +#[derive(Debug)] +pub enum NdarrayViewMut<'a> { + F32IX1(ndarray::ArrayViewMut<'a, f32, ndarray::Ix1>), + U8IX2(ndarray::ArrayViewMut<'a, u8, ndarray::Ix2>), + U8IX3(ndarray::ArrayViewMut<'a, u8, ndarray::Ix3>), + STRIX1(ndarray::ArrayViewMut<'a, String, ndarray::Ix1>), +} + +impl NdarrayViewMut<'_> { + pub fn as_ptr(&self) -> *const u64 { + match self { + NdarrayViewMut::F32IX1(array) => array.as_ptr() as *const u64, + NdarrayViewMut::U8IX2(array) => array.as_ptr() as *const u64, + NdarrayViewMut::U8IX3(array) => array.as_ptr() as *const u64, + NdarrayViewMut::STRIX1(array) => array.as_ptr() as *const u64, + } + } +} diff --git a/libraries/datatypes/Cargo.toml b/libraries/datatypes/Cargo.toml new file mode 100644 index 0000000..76ed651 --- /dev/null +++ b/libraries/datatypes/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "fastformat-datatypes" +version.workspace = true +edition.workspace = true +documentation.workspace = true +description.workspace = true +license.workspace = true +repository.workspace = true + +[features] +arrow = ["fastformat-converter/arrow", "dep:arrow"] +ndarray = ["fastformat-converter/ndarray", "dep:ndarray"] + +[dependencies] +fastformat-converter = { workspace = true } +eyre = { workspace = true } +arrow = { workspace = true, optional = true } +ndarray = { workspace = true, optional = true } diff --git a/libraries/datatypes/src/bbox.rs b/libraries/datatypes/src/bbox.rs new file mode 100644 index 0000000..3c8bd4e --- /dev/null +++ b/libraries/datatypes/src/bbox.rs @@ -0,0 +1,143 @@ +use eyre::{ContextCompat, Result}; + +use encoding::Encoding; + +use std::borrow::Cow; + +mod xywh; +mod xyxy; + +#[cfg(feature = "arrow")] +mod arrow; + +#[cfg(feature = "ndarray")] +mod ndarray; + +mod encoding; + +pub struct BBox<'a> { + pub data: Cow<'a, [f32]>, + pub confidence: Cow<'a, [f32]>, + pub label: Vec, + pub encoding: Encoding, +} + +impl BBox<'_> { + pub fn into_xyxy(self) -> Result { + match self.encoding { + Encoding::XYWH => { + let mut data = self.data; + { + let data = data.to_mut(); + + for i in 0..self.confidence.len() { + let x = data + .get(i * 4) + .wrap_err("Not enough data matching 4 values per box!") + .cloned()?; + let y = data + .get(i * 4 + 1) + .wrap_err("Not enough data matching 4 values per box!") + .cloned()?; + let w = data + .get(i * 4 + 2) + .wrap_err("Not enough data matching 4 values per box!") + .cloned()?; + let h = data + .get(i * 4 + 3) + .wrap_err("Not enough data matching 4 values per box!") + .cloned()?; + + data[i * 4 + 2] = x + w; + data[i * 4 + 3] = y + h; + } + } + + Ok(Self { + data, + confidence: self.confidence, + label: self.label, + encoding: self.encoding, + }) + } + Encoding::XYXY => Ok(self), + } + } + + pub fn into_xywh(self) -> Result { + match self.encoding { + Encoding::XYXY => { + let mut data = self.data; + { + let data = data.to_mut(); + + for i in 0..self.confidence.len() { + let x1 = data + .get(i * 4) + .wrap_err("Not enough data matching 4 values per box!") + .cloned()?; + let y1 = data + .get(i * 4 + 1) + .wrap_err("Not enough data matching 4 values per box!") + .cloned()?; + let x2 = data + .get(i * 4 + 2) + .wrap_err("Not enough data matching 4 values per box!") + .cloned()?; + let y2 = data + .get(i * 4 + 3) + .wrap_err("Not enough data matching 4 values per box!") + .cloned()?; + + data[i * 4 + 2] = x2 - x1; + data[i * 4 + 3] = y2 - y1; + } + } + + Ok(Self { + data, + confidence: self.confidence, + label: self.label, + encoding: self.encoding, + }) + } + Encoding::XYWH => Ok(self), + } + } +} + +mod tests { + #[test] + fn test_xyxy_into_xywh() { + use crate::bbox::BBox; + + let flat_bbox = vec![1.0, 1.0, 2.0, 2.0]; + let confidence = vec![0.98]; + let label = vec!["cat".to_string()]; + + let bbox = BBox::new_xyxy(flat_bbox, confidence, label).unwrap(); + let final_bbox = bbox.into_xywh().unwrap(); + let final_bbox_data = final_bbox.data; + + let expected_bbox = vec![1.0, 1.0, 1.0, 1.0]; + + assert_eq!(expected_bbox, final_bbox_data.into_owned()); + } + + #[test] + fn test_xywh_into_xyxy() { + use crate::bbox::BBox; + + let flat_bbox = vec![1.0, 1.0, 1.0, 1.0]; + let confidence = vec![0.98]; + let label = vec!["cat".to_string()]; + + let bbox = BBox::new_xywh(flat_bbox, confidence, label).unwrap(); + let final_bbox = bbox.into_xyxy().unwrap(); + let final_bbox_data = final_bbox.data; + + let expected_bbox = vec![1.0, 1.0, 2.0, 2.0]; + + assert_eq!(expected_bbox, final_bbox_data.into_owned()); + } +} diff --git a/libraries/datatypes/src/bbox/arrow.rs b/libraries/datatypes/src/bbox/arrow.rs new file mode 100644 index 0000000..c379fe4 --- /dev/null +++ b/libraries/datatypes/src/bbox/arrow.rs @@ -0,0 +1,152 @@ +use std::borrow::Cow; + +use super::{encoding::Encoding, BBox}; +use fastformat_converter::arrow::{FastFormatArrowBuilder, FastFormatArrowRawData}; + +use eyre::Result; + +impl<'a> BBox<'a> { + pub fn raw_data(array_data: arrow::array::ArrayData) -> Result { + use arrow::datatypes::Float32Type; + + let raw_data = FastFormatArrowRawData::new(array_data)? + .load_primitive::("data")? + .load_primitive::("confidence")? + .load_utf("label")? + .load_utf("encoding")?; + + Ok(raw_data) + } + + pub fn from_raw_data(mut raw_data: FastFormatArrowRawData) -> Result { + use arrow::datatypes::Float32Type; + + let data = raw_data.primitive_array::("data")?; + let confidence = raw_data.primitive_array::("confidence")?; + let label = raw_data.utf8_array("label")?; + let encoding = Encoding::from_string(raw_data.utf8_singleton("encoding")?)?; + + Ok(Self { + data: Cow::Owned(data), + confidence: Cow::Owned(confidence), + label, + encoding, + }) + } + + pub fn view_from_raw_data(raw_data: &'a FastFormatArrowRawData) -> Result { + use arrow::datatypes::Float32Type; + + let data = raw_data.primitive_array_view::("data")?; + let confidence = raw_data.primitive_array_view::("confidence")?; + let label = raw_data.utf8_array("label")?; + let encoding = Encoding::from_string(raw_data.utf8_singleton("encoding")?)?; + + Ok(Self { + data: Cow::Borrowed(data), + confidence: Cow::Borrowed(confidence), + label, + encoding, + }) + } + + pub fn from_arrow(array_data: arrow::array::ArrayData) -> Result { + Self::from_raw_data(Self::raw_data(array_data)?) + } + + pub fn into_arrow(self) -> Result { + use arrow::datatypes::{ + DataType::{Float32, Utf8}, + Float32Type, + }; + + let raw_data = FastFormatArrowBuilder::new() + .push_primitive_array::("data", self.data.into_owned(), Float32, false) + .push_primitive_array::( + "confidence", + self.confidence.into_owned(), + Float32, + false, + ) + .push_utf_array("label", self.label, Utf8, false) + .push_utf_singleton("encoding", self.encoding.to_string(), Utf8, false); + + raw_data.into_arrow() + } +} + +mod tests { + #[test] + fn test_arrow_zero_copy_conversion() { + use crate::bbox::BBox; + + let flat_bbox = vec![1.0, 1.0, 2.0, 2.0]; + let original_buffer_address = flat_bbox.as_ptr(); + + let confidence = vec![0.98]; + let label = vec!["cat".to_string()]; + + let xyxy_bbox = BBox::new_xyxy(flat_bbox, confidence, label).unwrap(); + let bbox_buffer_address = xyxy_bbox.data.as_ptr(); + + let arrow_bbox = xyxy_bbox.into_arrow().unwrap(); + + let xyxy_bbox = BBox::from_arrow(arrow_bbox).unwrap(); + let xyxy_bbox_buffer = xyxy_bbox.data.as_ptr(); + + let xywh_bbox = xyxy_bbox.into_xywh().unwrap(); + let xywh_bbox_buffer = xywh_bbox.data.as_ptr(); + + assert_eq!(original_buffer_address, bbox_buffer_address); + assert_eq!(bbox_buffer_address, xyxy_bbox_buffer); + assert_eq!(xyxy_bbox_buffer, xywh_bbox_buffer); + } + + #[test] + fn test_arrow_zero_copy_read_only() { + use crate::bbox::BBox; + + let flat_bbox = vec![1.0, 1.0, 2.0, 2.0]; + let original_buffer_address = flat_bbox.as_ptr(); + + let confidence = vec![0.98]; + let label = vec!["cat".to_string()]; + + let xyxy_bbox = BBox::new_xyxy(flat_bbox, confidence, label).unwrap(); + let bbox_buffer_address = xyxy_bbox.data.as_ptr(); + + let arrow_bbox = xyxy_bbox.into_arrow().unwrap(); + + let raw_data = BBox::raw_data(arrow_bbox).unwrap(); + let xyxy_bbox = BBox::view_from_raw_data(&raw_data).unwrap(); + let xyxy_bbox_buffer = xyxy_bbox.data.as_ptr(); + + assert_eq!(original_buffer_address, bbox_buffer_address); + assert_eq!(bbox_buffer_address, xyxy_bbox_buffer); + } + + #[test] + fn test_arrow_zero_copy_copy_on_write() { + use crate::bbox::BBox; + + let flat_bbox = vec![1.0, 1.0, 2.0, 2.0]; + let original_buffer_address = flat_bbox.as_ptr(); + + let confidence = vec![0.98]; + let label = vec!["cat".to_string()]; + + let xyxy_bbox = BBox::new_xyxy(flat_bbox, confidence, label).unwrap(); + let bbox_buffer_address = xyxy_bbox.data.as_ptr(); + + let arrow_bbox = xyxy_bbox.into_arrow().unwrap(); + + let raw_data = BBox::raw_data(arrow_bbox).unwrap(); + let xyxy_bbox = BBox::view_from_raw_data(&raw_data).unwrap(); + let xywh_bbox = xyxy_bbox.into_xywh().unwrap(); + + let final_bbox_buffer = xywh_bbox.data.as_ptr(); + + assert_eq!(original_buffer_address, bbox_buffer_address); + assert_ne!(bbox_buffer_address, final_bbox_buffer); + } +} diff --git a/libraries/datatypes/src/bbox/encoding.rs b/libraries/datatypes/src/bbox/encoding.rs new file mode 100644 index 0000000..62532de --- /dev/null +++ b/libraries/datatypes/src/bbox/encoding.rs @@ -0,0 +1,29 @@ +use eyre::{Report, Result}; + +use std::fmt::Display; + +#[allow(clippy::upper_case_acronyms)] +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum Encoding { + XYXY, + XYWH, +} + +impl Encoding { + pub fn from_string(encoding: String) -> Result { + match encoding.as_str() { + "XYXY" => Ok(Self::XYXY), + "XYWH" => Ok(Self::XYWH), + _ => Err(Report::msg(format!("Invalid String Encoding {}", encoding))), + } + } +} + +impl Display for Encoding { + fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::result::Result<(), std::fmt::Error> { + match self { + Self::XYXY => write!(fmt, "XYXY"), + Self::XYWH => write!(fmt, "XYWH"), + } + } +} diff --git a/libraries/datatypes/src/bbox/ndarray.rs b/libraries/datatypes/src/bbox/ndarray.rs new file mode 100644 index 0000000..601dec0 --- /dev/null +++ b/libraries/datatypes/src/bbox/ndarray.rs @@ -0,0 +1,97 @@ +use super::{encoding::Encoding, BBox}; +use eyre::{Context, Report, Result}; + +use fastformat_converter::ndarray::{Ndarray, NdarrayView, NdarrayViewMut}; + +pub type NdarrayBBox = (Ndarray, Ndarray, Ndarray, Encoding); +pub type NdarrayBBoxView<'a> = (NdarrayView<'a>, NdarrayView<'a>, NdarrayView<'a>, Encoding); +pub type NdarrayBBoxViewMut<'a> = ( + NdarrayViewMut<'a>, + NdarrayViewMut<'a>, + NdarrayViewMut<'a>, + Encoding, +); + +impl BBox<'_> { + pub fn from_ndarray(ndarray: NdarrayBBox) -> Result { + match ndarray { + ( + Ndarray::F32IX1(data), + Ndarray::F32IX1(confidence), + Ndarray::STRIX1(label), + Encoding::XYXY, + ) => Self::new_xyxy( + data.into_raw_vec_and_offset().0, + confidence.into_raw_vec_and_offset().0, + label.into_raw_vec_and_offset().0, + ), + ( + Ndarray::F32IX1(data), + Ndarray::F32IX1(confidence), + Ndarray::STRIX1(label), + Encoding::XYWH, + ) => Self::new_xywh( + data.into_raw_vec_and_offset().0, + confidence.into_raw_vec_and_offset().0, + label.into_raw_vec_and_offset().0, + ), + _ => Err(Report::msg("Invalid Ndarray type")).context("from_ndarray"), + } + } + + pub fn into_ndarray(self) -> Result { + Ok(( + Ndarray::F32IX1( + ndarray::Array::from_shape_vec(self.data.len(), self.data.into_owned()) + .wrap_err("Failed to reshape data into ndarray")?, + ), + Ndarray::F32IX1( + ndarray::Array::from_shape_vec(self.confidence.len(), self.confidence.into_owned()) + .wrap_err("Failed to reshape confidence into ndarray")?, + ), + Ndarray::STRIX1( + ndarray::Array::from_shape_vec(self.label.len(), self.label) + .wrap_err("Failed to reshape label into ndarray")?, + ), + self.encoding, + )) + } +} + +impl<'a> BBox<'a> { + pub fn to_ndarray_view(&'a self) -> Result> { + Ok(( + NdarrayView::F32IX1( + ndarray::ArrayView::from_shape(self.data.len(), &self.data) + .wrap_err("Failed to reshape data into ndarray")?, + ), + NdarrayView::F32IX1( + ndarray::ArrayView::from_shape(self.confidence.len(), &self.confidence) + .wrap_err("Failed to reshape confidence into ndarray")?, + ), + NdarrayView::STRIX1( + ndarray::ArrayView::from_shape(self.label.len(), &self.label) + .wrap_err("Failed to reshape label into ndarray")?, + ), + self.encoding, + )) + } + + pub fn to_ndarray_view_mut(&'a mut self) -> Result> { + Ok(( + NdarrayViewMut::F32IX1( + ndarray::ArrayViewMut::from_shape(self.data.len(), self.data.to_mut()) + .wrap_err("Failed to reshape data into ndarray")?, + ), + NdarrayViewMut::F32IX1( + ndarray::ArrayViewMut::from_shape(self.confidence.len(), self.confidence.to_mut()) + .wrap_err("Failed to reshape confidence into ndarray")?, + ), + NdarrayViewMut::STRIX1( + ndarray::ArrayViewMut::from_shape(self.label.len(), self.label.as_mut()) + .wrap_err("Failed to reshape label into ndarray")?, + ), + self.encoding, + )) + } +} diff --git a/libraries/datatypes/src/bbox/xywh.rs b/libraries/datatypes/src/bbox/xywh.rs new file mode 100644 index 0000000..0f9c5c9 --- /dev/null +++ b/libraries/datatypes/src/bbox/xywh.rs @@ -0,0 +1,37 @@ +use std::borrow::Cow; + +use super::{encoding::Encoding, BBox}; +use eyre::{Report, Result}; + +impl BBox<'_> { + pub fn new_xywh(data: Vec, confidence: Vec, label: Vec) -> Result { + if confidence.len() != label.len() + || confidence.len() * 4 != data.len() + || label.len() * 4 != data.len() + { + return Err(Report::msg( + "Confidence, Label and Data doesn't match length", + )); + } + + Ok(BBox { + data: Cow::from(data), + confidence: Cow::from(confidence), + label, + encoding: Encoding::XYWH, + }) + } +} + +mod tests { + #[test] + fn test_xywh_creation() { + use crate::bbox::BBox; + + let flat_bbox = vec![1.0, 1.0, 1.0, 1.0]; + let confidence = vec![0.98]; + let label = vec!["cat".to_string()]; + + BBox::new_xywh(flat_bbox, confidence, label).unwrap(); + } +} diff --git a/libraries/datatypes/src/bbox/xyxy.rs b/libraries/datatypes/src/bbox/xyxy.rs new file mode 100644 index 0000000..06a57f1 --- /dev/null +++ b/libraries/datatypes/src/bbox/xyxy.rs @@ -0,0 +1,34 @@ +use std::borrow::Cow; + +use super::{encoding::Encoding, BBox}; +use eyre::{Report, Result}; + +impl BBox<'_> { + pub fn new_xyxy(data: Vec, confidence: Vec, label: Vec) -> Result { + if confidence.len() != label.len() || confidence.len() * 4 != data.len() { + return Err(Report::msg( + "Confidence, Label and Data doesn't match length", + )); + } + + Ok(BBox { + data: Cow::from(data), + confidence: Cow::from(confidence), + label, + encoding: Encoding::XYXY, + }) + } +} + +mod tests { + #[test] + fn test_xyxy_creation() { + use crate::bbox::BBox; + + let flat_bbox = vec![1.0, 1.0, 2.0, 2.0]; + let confidence = vec![0.98]; + let label = vec!["cat".to_string()]; + + BBox::new_xyxy(flat_bbox, confidence, label).unwrap(); + } +} diff --git a/src/image.rs b/libraries/datatypes/src/image.rs similarity index 69% rename from src/image.rs rename to libraries/datatypes/src/image.rs index 9af6e3d..c52749a 100644 --- a/src/image.rs +++ b/libraries/datatypes/src/image.rs @@ -1,38 +1,38 @@ use eyre::{Report, Result}; -use container::DataContainer; -use encoding::Encoding; +use data::ImageData; +pub use encoding::Encoding; mod bgr8; mod gray8; mod rgb8; +#[cfg(feature = "arrow")] mod arrow; -mod container; -mod encoding; -#[derive(Debug)] -pub struct Image { - data: DataContainer, +#[cfg(feature = "ndarray")] +mod ndarray; - width: u32, - height: u32, +#[cfg(feature = "ndarray")] +pub use ndarray::{NdarrayImage, NdarrayImageView, NdarrayImageViewMut}; - encoding: Encoding, +mod data; +pub mod encoding; - name: Option, -} +#[derive(Debug)] +pub struct Image<'a> { + pub data: ImageData<'a>, -impl Image { - pub fn as_ptr(&self) -> *const u8 { - self.data.as_ptr() - } + pub width: u32, + pub height: u32, - pub fn data(&self) -> &DataContainer { - &self.data - } + pub encoding: Encoding, + + pub name: Option, +} - pub fn to_rgb8(self) -> Result { +impl Image<'_> { + pub fn into_rgb8(self) -> Result { match self.encoding { Encoding::BGR8 => { let mut data = self.data.into_u8()?; @@ -40,9 +40,8 @@ impl Image { for i in (0..data.len()).step_by(3) { data.swap(i, i + 2); } - Ok(Image { - data: DataContainer::from_u8(data), + data: ImageData::from_vec_u8(data), width: self.width, height: self.height, encoding: Encoding::RGB8, @@ -54,7 +53,7 @@ impl Image { } } - pub fn to_bgr8(self) -> Result { + pub fn into_bgr8(self) -> Result { match self.encoding { Encoding::RGB8 => { let mut data = self.data.into_u8()?; @@ -64,7 +63,7 @@ impl Image { } Ok(Image { - data: DataContainer::from_u8(data), + data: ImageData::from_vec_u8(data), width: self.width, height: self.height, encoding: Encoding::BGR8, @@ -79,14 +78,14 @@ impl Image { mod tests { #[test] - fn test_rgb8_to_bgr8() { + fn test_rgb8_into_bgr8() { use crate::image::Image; let flat_image = (0..27).collect::>(); let image = Image::new_rgb8(flat_image, 3, 3, Some("camera.test")).unwrap(); - let final_image = image.to_bgr8().unwrap(); - let final_image_data = final_image.data().as_u8().unwrap(); + let final_image = image.into_bgr8().unwrap(); + let final_image_data = final_image.data.as_u8().unwrap(); let expected_image = vec![ 2, 1, 0, 5, 4, 3, 8, 7, 6, 11, 10, 9, 14, 13, 12, 17, 16, 15, 20, 19, 18, 23, 22, 21, @@ -97,14 +96,14 @@ mod tests { } #[test] - fn test_bgr8_to_rgb8() { + fn test_bgr8_into_rgb8() { use crate::image::Image; let flat_image = (0..27).collect::>(); let image = Image::new_bgr8(flat_image, 3, 3, Some("camera.test")).unwrap(); - let final_image = image.to_rgb8().unwrap(); - let final_image_data = final_image.data().as_u8().unwrap(); + let final_image = image.into_rgb8().unwrap(); + let final_image_data = final_image.data.as_u8().unwrap(); let expected_image = vec![ 2, 1, 0, 5, 4, 3, 8, 7, 6, 11, 10, 9, 14, 13, 12, 17, 16, 15, 20, 19, 18, 23, 22, 21, diff --git a/libraries/datatypes/src/image/arrow.rs b/libraries/datatypes/src/image/arrow.rs new file mode 100644 index 0000000..86c33e7 --- /dev/null +++ b/libraries/datatypes/src/image/arrow.rs @@ -0,0 +1,253 @@ +use super::{data::ImageData, encoding::Encoding, Image}; +use eyre::Result; +use fastformat_converter::arrow::{FastFormatArrowBuilder, FastFormatArrowRawData}; + +impl<'a> Image<'a> { + /// Extracts raw data from an Arrow `ArrayData` and converts it to `FastFormatArrowRawData`. + /// + /// This function loads the primitive and UTF fields corresponding to an image's width, height, + /// encoding, and name from the provided `ArrayData`. It determines the image encoding and loads + /// the appropriate pixel data. + /// + /// # Arguments + /// + /// * `array_data` - The Arrow `ArrayData` containing the image metadata and pixel data. + /// + /// # Returns + /// + /// A `Result` containing the `FastFormatArrowRawData` if successful, or an error otherwise. + /// + /// # Errors + /// + /// Returns an error if the expected fields (width, height, encoding, or pixel data) are missing + /// or if the data format is invalid. + pub fn raw_data(array_data: arrow::array::ArrayData) -> Result { + use arrow::datatypes::{UInt32Type, UInt8Type}; + + let raw_data = FastFormatArrowRawData::new(array_data)? + .load_primitive::("width")? + .load_primitive::("height")? + .load_utf("encoding")? + .load_utf("name")?; + + let encoding = Encoding::from_string(raw_data.utf8_singleton("encoding")?)?; + let raw_data = match encoding { + Encoding::RGB8 => raw_data.load_primitive::("data")?, + Encoding::BGR8 => raw_data.load_primitive::("data")?, + Encoding::GRAY8 => raw_data.load_primitive::("data")?, + }; + + Ok(raw_data) + } + + /// Constructs an `Image` object from `FastFormatArrowRawData`. + /// + /// This function parses the width, height, encoding, and name of the image from the + /// provided raw data and loads the pixel data based on the encoding type. + /// + /// # Arguments + /// + /// * `raw_data` - The raw data containing the image metadata and pixel information. + /// + /// # Returns + /// + /// A `Result` containing the constructed `Image` if successful, or an error otherwise. + /// + /// # Errors + /// + /// Returns an error if the raw data is missing fields or contains invalid values. + pub fn from_raw_data(mut raw_data: FastFormatArrowRawData) -> Result { + use arrow::datatypes::{UInt32Type, UInt8Type}; + + let width = raw_data.primitive_singleton::("width")?; + let height = raw_data.primitive_singleton::("height")?; + let encoding = Encoding::from_string(raw_data.utf8_singleton("encoding")?)?; + let name = Some(raw_data.utf8_singleton("name")?).filter(|s| !s.is_empty()); + + let data = match encoding { + Encoding::RGB8 => raw_data.primitive_array::("data")?, + Encoding::BGR8 => raw_data.primitive_array::("data")?, + Encoding::GRAY8 => raw_data.primitive_array::("data")?, + }; + + Ok(Self { + data: ImageData::from_vec_u8(data), + width, + height, + encoding, + name, + }) + } + + /// Creates a read-only view of an `Image` from `FastFormatArrowRawData`. + /// + /// This function provides a zero-copy read-only view of the image data, allowing efficient + /// access to the metadata and pixel data without copying the underlying memory. + /// + /// # Arguments + /// + /// * `raw_data` - A reference to the raw data containing the image metadata and pixel information. + /// + /// # Returns + /// + /// A `Result` containing the `Image` if successful, or an error otherwise. + /// + /// # Errors + /// + /// Returns an error if the raw data is invalid or if required fields are missing. + pub fn view_from_raw_data(raw_data: &'a FastFormatArrowRawData) -> Result { + use arrow::datatypes::{UInt32Type, UInt8Type}; + + let width = raw_data.primitive_singleton::("width")?; + let height = raw_data.primitive_singleton::("height")?; + let encoding = Encoding::from_string(raw_data.utf8_singleton("encoding")?)?; + let name = Some(raw_data.utf8_singleton("name")?).filter(|s| !s.is_empty()); + + let data = match encoding { + Encoding::RGB8 => raw_data.primitive_array_view::("data")?, + Encoding::BGR8 => raw_data.primitive_array_view::("data")?, + Encoding::GRAY8 => raw_data.primitive_array_view::("data")?, + }; + + Ok(Self { + data: ImageData::from_slice_u8(data), + width, + height, + encoding, + name, + }) + } + + /// Converts Arrow `ArrayData` into an `Image`. + /// + /// This function combines the process of extracting raw data and converting it into an + /// `Image` object. + /// + /// # Arguments + /// + /// * `array_data` - The Arrow `ArrayData` containing the image metadata and pixel data. + /// + /// # Returns + /// + /// A `Result` containing the `Image` if successful, or an error otherwise. + pub fn from_arrow(array_data: arrow::array::ArrayData) -> Result { + Self::from_raw_data(Self::raw_data(array_data)?) + } + + /// Converts an `Image` into Arrow `ArrayData`. + /// + /// This function serializes the image metadata and pixel data into Arrow format, allowing + /// the image to be stored or transmitted as Arrow `ArrayData`. + /// + /// # Returns + /// + /// A `Result` containing the serialized `ArrayData` if successful, or an error otherwise. + pub fn into_arrow(self) -> Result { + use arrow::datatypes::{ + DataType::{UInt32, UInt8, Utf8}, + UInt32Type, UInt8Type, + }; + + let raw_data = FastFormatArrowBuilder::new() + .push_primitive_singleton::("width", self.width, UInt32, false) + .push_primitive_singleton::("height", self.height, UInt32, false) + .push_utf_singleton("encoding", self.encoding.to_string(), Utf8, false) + .push_utf_singleton( + "name", + self.name.map_or_else(|| "".to_string(), |s| s), + Utf8, + false, + ); + + let raw_data = match self.encoding { + Encoding::RGB8 => raw_data.push_primitive_array::( + "data", + self.data.into_u8()?, + UInt8, + false, + ), + Encoding::BGR8 => raw_data.push_primitive_array::( + "data", + self.data.into_u8()?, + UInt8, + false, + ), + Encoding::GRAY8 => raw_data.push_primitive_array::( + "data", + self.data.into_u8()?, + UInt8, + false, + ), + }; + + raw_data.into_arrow() + } +} + +mod tests { + #[test] + fn test_arrow_zero_copy_conversion() { + use crate::image::Image; + + let flat_image = vec![0; 27]; + let original_buffer_address = flat_image.as_ptr() as *const u64; + + let bgr8_image = Image::new_bgr8(flat_image, 3, 3, None).unwrap(); + let image_buffer_address = bgr8_image.data.as_ptr(); + + let arrow_image = bgr8_image.into_arrow().unwrap(); + + let bgr8_image = Image::from_arrow(arrow_image).unwrap(); + let bgr8_image_buffer = bgr8_image.data.as_ptr(); + + let rgb8_image = bgr8_image.into_rgb8().unwrap(); + let rgb8_image_buffer = rgb8_image.data.as_ptr(); + + assert_eq!(original_buffer_address, image_buffer_address); + assert_eq!(image_buffer_address, bgr8_image_buffer); + assert_eq!(bgr8_image_buffer, rgb8_image_buffer); + } + + #[test] + fn test_arrow_zero_copy_read_only() { + use crate::image::Image; + + let flat_image = vec![0; 27]; + let original_buffer_address = flat_image.as_ptr() as *const u64; + + let bgr8_image = Image::new_bgr8(flat_image, 3, 3, None).unwrap(); + let image_buffer_address = bgr8_image.data.as_ptr(); + + let arrow_image = bgr8_image.into_arrow().unwrap(); + + let raw_data = Image::raw_data(arrow_image).unwrap(); + let new_image = Image::view_from_raw_data(&raw_data).unwrap(); + + let final_image_buffer = new_image.data.as_ptr(); + + assert_eq!(original_buffer_address, image_buffer_address); + assert_eq!(image_buffer_address, final_image_buffer); + } + + #[test] + fn test_arrow_zero_copy_copy_on_write() { + use crate::image::Image; + + let flat_image = vec![0; 27]; + let original_buffer_address = flat_image.as_ptr() as *const u64; + + let bgr8_image = Image::new_bgr8(flat_image, 3, 3, None).unwrap(); + let image_buffer_address = bgr8_image.data.as_ptr(); + + let arrow_image = bgr8_image.into_arrow().unwrap(); + + let raw_data = Image::raw_data(arrow_image).unwrap(); + let bgr8_image = Image::view_from_raw_data(&raw_data).unwrap(); + let rgb8_image = bgr8_image.into_rgb8().unwrap(); + + let final_image_buffer = rgb8_image.data.as_ptr(); + + assert_eq!(original_buffer_address, image_buffer_address); + assert_ne!(image_buffer_address, final_image_buffer); + } +} diff --git a/libraries/datatypes/src/image/bgr8.rs b/libraries/datatypes/src/image/bgr8.rs new file mode 100644 index 0000000..934c25d --- /dev/null +++ b/libraries/datatypes/src/image/bgr8.rs @@ -0,0 +1,61 @@ +use super::{data::ImageData, encoding::Encoding, Image}; +use eyre::{Report, Result}; + +impl Image<'_> { + /// Creates a new `Image` in BGR8 format. + /// + /// This function constructs a new `Image` object with the given pixel data, width, height, + /// and an optional name. It ensures that the pixel data length matches the expected size + /// for the given width, height, and BGR8 encoding (3 bytes per pixel). + /// + /// # Arguments + /// + /// * `data` - A `Vec` containing the pixel data in BGR8 format. + /// * `width` - The width of the image. + /// * `height` - The height of the image. + /// * `name` - An optional string slice representing the name of the image. + /// + /// # Returns + /// + /// A `Result` containing the constructed `Image` if successful, or an error otherwise. + /// + /// # Errors + /// + /// Returns an error if the length of the pixel data does not match the expected size + /// based on the width, height, and BGR8 encoding. + /// + /// # Example + /// + /// ``` + /// use fastformat_datatypes::image::Image; + /// + /// let data = vec![0; 27]; // 3x3 image with 3 bytes per pixel + /// let image = Image::new_bgr8(data, 3, 3, Some("example")).unwrap(); + /// ``` + pub fn new_bgr8(data: Vec, width: u32, height: u32, name: Option<&str>) -> Result { + if width * height * 3 != data.len() as u32 { + return Err(Report::msg( + "Width, height and BGR8 encoding doesn't match data length.", + )); + } + + Ok(Image { + data: ImageData::from_vec_u8(data), + width, + height, + encoding: Encoding::BGR8, + name: name.map(|s| s.to_string()), + }) + } +} + +mod tests { + #[test] + fn test_bgr8_creation() { + use crate::image::Image; + + let flat_image = vec![0; 27]; + + Image::new_bgr8(flat_image, 3, 3, Some("camera.test")).unwrap(); + } +} diff --git a/libraries/datatypes/src/image/data.rs b/libraries/datatypes/src/image/data.rs new file mode 100644 index 0000000..18e4567 --- /dev/null +++ b/libraries/datatypes/src/image/data.rs @@ -0,0 +1,117 @@ +use eyre::Result; + +use std::borrow::Cow; + +#[derive(Debug)] +pub enum ImageData<'a> { + U8(Cow<'a, [u8]>), + U16(Cow<'a, [u16]>), + F32(Cow<'a, [f32]>), +} + +impl ImageData<'_> { + pub fn len(&self) -> usize { + match self { + Self::U8(data) => data.len(), + Self::U16(data) => data.len(), + Self::F32(data) => data.len(), + } + } + + pub fn as_ptr(&self) -> *const u64 { + match self { + Self::U8(data) => data.as_ptr() as *const u64, + Self::U16(data) => data.as_ptr() as *const u64, + Self::F32(data) => data.as_ptr() as *const u64, + } + } + + pub fn into_u8(self) -> Result> { + match self { + Self::U8(data) => Ok(data.into_owned()), + _ => Err(eyre::Report::msg("Can't convert data to u8")), + } + } + + pub fn into_u16(self) -> Result> { + match self { + Self::U16(data) => Ok(data.into_owned()), + _ => Err(eyre::Report::msg("Can't convert data to u16")), + } + } + + pub fn into_f32(self) -> Result> { + match self { + Self::F32(data) => Ok(data.into_owned()), + _ => Err(eyre::Report::msg("Can't convert data to f32")), + } + } + + pub fn as_u8(&self) -> Result<&[u8]> { + match self { + Self::U8(data) => Ok(data), + _ => Err(eyre::Report::msg("Can't convert data to u8")), + } + } + + pub fn as_u16(&self) -> Result<&[u16]> { + match self { + Self::U16(data) => Ok(data), + _ => Err(eyre::Report::msg("Can't convert data to u16")), + } + } + + pub fn as_f32(&self) -> Result<&[f32]> { + match self { + Self::F32(data) => Ok(data), + _ => Err(eyre::Report::msg("Can't convert data to f32")), + } + } + + pub fn as_mut_u8(&mut self) -> Result<&mut Vec> { + match self { + Self::U8(data) => Ok(data.to_mut()), + _ => Err(eyre::Report::msg("Can't convert data to u8")), + } + } + + pub fn as_mut_u16(&mut self) -> Result<&mut Vec> { + match self { + Self::U16(data) => Ok(data.to_mut()), + _ => Err(eyre::Report::msg("Can't convert data to 16")), + } + } + + pub fn as_mut_f32(&mut self) -> Result<&mut Vec> { + match self { + Self::F32(data) => Ok(data.to_mut()), + _ => Err(eyre::Report::msg("Can't convert data to f32")), + } + } + + pub fn from_vec_u8(data: Vec) -> Self { + Self::U8(Cow::from(data)) + } + + pub fn from_vec_u16(data: Vec) -> Self { + Self::U16(Cow::from(data)) + } + + pub fn from_vec_f32(data: Vec) -> Self { + Self::F32(Cow::from(data)) + } +} + +impl<'a> ImageData<'a> { + pub fn from_slice_u8(data: &'a [u8]) -> Self { + Self::U8(Cow::from(data)) + } + + pub fn from_slice_u16(data: &'a [u16]) -> Self { + Self::U16(Cow::from(data)) + } + + pub fn from_slice_f32(data: &'a [f32]) -> Self { + Self::F32(Cow::from(data)) + } +} diff --git a/src/image/encoding.rs b/libraries/datatypes/src/image/encoding.rs similarity index 94% rename from src/image/encoding.rs rename to libraries/datatypes/src/image/encoding.rs index ff1c22c..52b915d 100644 --- a/src/image/encoding.rs +++ b/libraries/datatypes/src/image/encoding.rs @@ -2,7 +2,7 @@ use eyre::{Report, Result}; use std::fmt::Display; -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, PartialEq)] pub enum Encoding { RGB8, BGR8, diff --git a/libraries/datatypes/src/image/gray8.rs b/libraries/datatypes/src/image/gray8.rs new file mode 100644 index 0000000..1e6c379 --- /dev/null +++ b/libraries/datatypes/src/image/gray8.rs @@ -0,0 +1,59 @@ +use super::{data::ImageData, encoding::Encoding, Image}; +use eyre::{Report, Result}; + +impl Image<'_> { + /// Creates a new `Image` in Gray8 format. + /// + /// This function constructs a new `Image` object with the given pixel data, width, height, + /// and an optional name. It ensures that the pixel data length matches the expected size + /// for the given width and height. + /// + /// # Arguments + /// + /// * `data` - A `Vec` containing the pixel data in Gray8 format. + /// * `width` - The width of the image. + /// * `height` - The height of the image. + /// * `name` - An optional string slice representing the name of the image. + /// + /// # Returns + /// + /// A `Result` containing the constructed `Image` if successful, or an error otherwise. + /// + /// # Errors + /// + /// Returns an error if the length of the pixel data does not match the expected size + /// based on the width and height. + /// + /// # Example + /// + /// ``` + /// use fastformat_datatypes::image::Image; + /// + /// let data = vec![0; 9]; // 3x3 image with 1 byte per pixel + /// let image = Image::new_gray8(data, 3, 3, Some("example")).unwrap(); + /// ``` + pub fn new_gray8(data: Vec, width: u32, height: u32, name: Option<&str>) -> Result { + if data.len() != (width * height) as usize { + return Err(Report::msg("Invalid data data length.")); + } + + Ok(Image { + data: ImageData::from_vec_u8(data), + width, + height, + encoding: Encoding::GRAY8, + name: name.map(|s| s.to_string()), + }) + } +} + +mod test { + #[test] + fn test_gray8_creation() { + use crate::image::Image; + + let flat_image = (1..10).collect::>(); + + Image::new_gray8(flat_image, 3, 3, Some("camera.test")).unwrap(); + } +} diff --git a/libraries/datatypes/src/image/ndarray.rs b/libraries/datatypes/src/image/ndarray.rs new file mode 100644 index 0000000..5db284c --- /dev/null +++ b/libraries/datatypes/src/image/ndarray.rs @@ -0,0 +1,244 @@ +use super::{encoding::Encoding, Image}; +use eyre::{Context, Report, Result}; + +use fastformat_converter::ndarray::{Ndarray, NdarrayView, NdarrayViewMut}; + +pub type NdarrayImage = (Ndarray, Encoding, Option); +pub type NdarrayImageView<'a> = (NdarrayView<'a>, Encoding, Option<&'a str>); +pub type NdarrayImageViewMut<'a> = (NdarrayViewMut<'a>, Encoding, Option<&'a str>); + +impl Image<'_> { + pub fn from_ndarray(ndarray: NdarrayImage) -> Result { + match ndarray { + (Ndarray::U8IX3(array), Encoding::BGR8, name) => { + let width = array.shape()[1] as u32; + let height = array.shape()[0] as u32; + + let (data, _) = array.into_raw_vec_and_offset(); + + Self::new_bgr8(data, width, height, name.as_deref()) + } + (Ndarray::U8IX3(array), Encoding::RGB8, name) => { + let width = array.shape()[1] as u32; + let height = array.shape()[0] as u32; + + let (data, _) = array.into_raw_vec_and_offset(); + + Self::new_rgb8(data, width, height, name.as_deref()) + } + (Ndarray::U8IX2(array), Encoding::GRAY8, name) => { + let width = array.shape()[1] as u32; + let height = array.shape()[0] as u32; + + let (data, _) = array.into_raw_vec_and_offset(); + + Self::new_gray8(data, width, height, name.as_deref()) + } + _ => Err(Report::msg("Invalid Ndarray type")).context("from_ndarray"), + } + } + + pub fn into_ndarray(self) -> Result { + match self.encoding { + Encoding::BGR8 => { + let ndarray = ndarray::Array::from_shape_vec( + (self.height as usize, self.width as usize, 3), + self.data.into_u8()?, + ) + .wrap_err("Failed to reshape data into ndarray: width, height and BGR8 encoding doesn't match data data length."); + + ndarray.map(|array| (Ndarray::U8IX3(array), self.encoding, self.name)) + } + Encoding::RGB8 => { + let ndarray = ndarray::Array::from_shape_vec( + (self.height as usize, self.width as usize, 3), + self.data.into_u8()?, + ) + .wrap_err("Failed to reshape data into ndarray: width, height and RGB8 encoding doesn't match data data length."); + + ndarray.map(|array| (Ndarray::U8IX3(array), self.encoding, self.name)) + } + Encoding::GRAY8 => { + let ndarray = ndarray::Array::from_shape_vec( + (self.height as usize, self.width as usize), + self.data.into_u8()?, + ) + .wrap_err("Failed to reshape data into ndarray: width, height and GRAY8 encoding doesn't match data data length."); + + ndarray.map(|array| (Ndarray::U8IX2(array), self.encoding, self.name)) + } + } + } +} + +impl<'a> Image<'a> { + pub fn to_ndarray_view(&'a self) -> Result<(NdarrayView<'a>, Encoding, Option<&str>)> { + match self.encoding { + Encoding::BGR8 => { + let array = ndarray::ArrayView3::from_shape( + (self.height as usize, self.width as usize, 3), + self.data.as_u8()?, + ) + .wrap_err("Failed to create ndarray view: width, height and BGR8 encoding doesn't match data data length."); + + array.map(|array| { + ( + NdarrayView::U8IX3(array), + self.encoding, + self.name.as_deref(), + ) + }) + } + Encoding::RGB8 => { + let array = ndarray::ArrayView3::from_shape( + (self.height as usize, self.width as usize, 3), + self.data.as_u8()?, + ) + .wrap_err("Failed to create ndarray view: width, height and RGB8 encoding doesn't match data data length."); + + array.map(|array| { + ( + NdarrayView::U8IX3(array), + self.encoding, + self.name.as_deref(), + ) + }) + } + Encoding::GRAY8 => { + let array = ndarray::ArrayView2::from_shape( + (self.height as usize, self.width as usize), + self.data.as_u8()?, + ) + .wrap_err("Failed to create ndarray view: width, height and GRAY8 encoding doesn't match data data length."); + + array.map(|array| { + ( + NdarrayView::U8IX2(array), + self.encoding, + self.name.as_deref(), + ) + }) + } + } + } + + pub fn to_ndarray_view_mut( + &'a mut self, + ) -> Result<(NdarrayViewMut<'a>, Encoding, Option<&str>)> { + match self.encoding { + Encoding::BGR8 => { + let array = ndarray::ArrayViewMut3::from_shape( + (self.height as usize, self.width as usize, 3), + self.data.as_mut_u8()?, + ) + .wrap_err("Failed to create ndarray view: width, height and BGR8 encoding doesn't match data data length."); + + array.map(|array| { + ( + NdarrayViewMut::U8IX3(array), + self.encoding, + self.name.as_deref(), + ) + }) + } + Encoding::RGB8 => { + let array = ndarray::ArrayViewMut3::from_shape( + (self.height as usize, self.width as usize, 3), + self.data.as_mut_u8()?, + ) + .wrap_err("Failed to create ndarray view: width, height and RGB8 encoding doesn't match data data length."); + + array.map(|array| { + ( + NdarrayViewMut::U8IX3(array), + self.encoding, + self.name.as_deref(), + ) + }) + } + Encoding::GRAY8 => { + let array = ndarray::ArrayViewMut2::from_shape( + (self.height as usize, self.width as usize), + self.data.as_mut_u8()?, + ) + .wrap_err("Failed to create ndarray view: width, height and GRAY8 encoding doesn't match data data length."); + + array.map(|array| { + ( + NdarrayViewMut::U8IX2(array), + self.encoding, + self.name.as_deref(), + ) + }) + } + } + } +} + +mod tests { + #[test] + fn test_bgr8_from_ndarray() { + use crate::image::Image; + use fastformat_converter::ndarray::Ndarray; + + let array = Ndarray::U8IX3(ndarray::Array3::::zeros((3, 3, 3))); + + let image = Image::from_ndarray((array, crate::image::Encoding::BGR8, None)).unwrap(); + + assert_eq!(image.encoding, crate::image::Encoding::BGR8) + } + + #[test] + fn test_bgr8_into_ndarray() { + use crate::image::Image; + + let flat_image = vec![0; 27]; + + let image = Image::new_bgr8(flat_image, 3, 3, Some("camera.test")).unwrap(); + + image.into_ndarray().unwrap(); + } + + #[test] + fn test_bgr8_into_ndarray_view() { + use crate::image::Image; + + let flat_image = vec![0; 27]; + + let image = Image::new_bgr8(flat_image, 3, 3, Some("camera.test")).unwrap(); + + image.to_ndarray_view().unwrap(); + } + + #[test] + fn test_bgr8_into_ndarray_view_mut() { + use crate::image::Image; + + let flat_image = vec![0; 27]; + + let mut image = Image::new_bgr8(flat_image, 3, 3, Some("camera.test")).unwrap(); + + image.to_ndarray_view_mut().unwrap(); + } + + #[test] + fn test_bgr8_ndarray_zero_copy_conversion() { + use crate::image::Image; + + let flat_image = vec![0; 27]; + let original_buffer_address = flat_image.as_ptr() as *const u64; + + let bgr8_image = Image::new_bgr8(flat_image, 3, 3, None).unwrap(); + let image_buffer_address = bgr8_image.data.as_ptr(); + + let bgr8_ndarray = bgr8_image.into_ndarray().unwrap(); + let ndarray_buffer_address = bgr8_ndarray.0.as_ptr(); + + let final_image = Image::from_ndarray(bgr8_ndarray).unwrap(); + let final_image_buffer_address = final_image.data.as_ptr(); + + assert_eq!(original_buffer_address, image_buffer_address); + assert_eq!(image_buffer_address, ndarray_buffer_address); + assert_eq!(ndarray_buffer_address, final_image_buffer_address); + } +} diff --git a/libraries/datatypes/src/image/rgb8.rs b/libraries/datatypes/src/image/rgb8.rs new file mode 100644 index 0000000..aa16504 --- /dev/null +++ b/libraries/datatypes/src/image/rgb8.rs @@ -0,0 +1,59 @@ +use super::{data::ImageData, encoding::Encoding, Image}; +use eyre::{Report, Result}; + +impl Image<'_> { + /// Creates a new `Image` in RGB8 format. + /// + /// This function constructs a new `Image` object with the given pixel data, width, height, + /// and an optional name. It ensures that the pixel data length matches the expected size + /// for the given width and height. + /// + /// # Arguments + /// + /// * `data` - A `Vec` containing the pixel data in RGB8 format. + /// * `width` - The width of the image. + /// * `height` - The height of the image. + /// * `name` - An optional string slice representing the name of the image. + /// + /// # Returns + /// + /// A `Result` containing the constructed `Image` if successful, or an error otherwise. + /// + /// # Errors + /// + /// Returns an error if the length of the pixel data does not match the expected size + /// based on the width and height. + /// + /// # Example + /// + /// ``` + /// use fastformat_datatypes::image::Image; + /// + /// let data = vec![0; 27]; // 3x3 image with 3 bytes per pixel + /// let image = Image::new_rgb8(data, 3, 3, Some("example")).unwrap(); + /// ``` + pub fn new_rgb8(data: Vec, width: u32, height: u32, name: Option<&str>) -> Result { + if data.len() != (width * height * 3) as usize { + return Err(Report::msg("Invalid pixel data length.")); + } + + Ok(Image { + data: ImageData::from_vec_u8(data), + width, + height, + encoding: Encoding::RGB8, + name: name.map(|s| s.to_string()), + }) + } +} + +mod tests { + #[test] + fn test_rgb8_creation() { + use crate::image::Image; + + let flat_image = vec![0; 27]; + + Image::new_rgb8(flat_image, 3, 3, Some("camera.test")).unwrap(); + } +} diff --git a/libraries/datatypes/src/lib.rs b/libraries/datatypes/src/lib.rs new file mode 100644 index 0000000..5649d3a --- /dev/null +++ b/libraries/datatypes/src/lib.rs @@ -0,0 +1,2 @@ +pub mod bbox; +pub mod image; diff --git a/libraries/fastformat/Cargo.toml b/libraries/fastformat/Cargo.toml new file mode 100644 index 0000000..6dadecb --- /dev/null +++ b/libraries/fastformat/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "fastformat" +version.workspace = true +edition.workspace = true +documentation.workspace = true +description.workspace = true +license.workspace = true +repository.workspace = true + +[lib] +name = "fastformat" +crate-type = ["cdylib", "lib"] + +[features] +arrow = ["fastformat-datatypes/arrow", "fastformat-converter/arrow"] +ndarray = ["fastformat-datatypes/ndarray", "fastformat-converter/ndarray"] +numpy = ["fastformat-converter/numpy"] +pyarrow = ["fastformat-converter/pyarrow"] + +[dependencies] +fastformat-datatypes = { workspace = true } +fastformat-converter = { workspace = true } + +pyo3 = { version = "0.21", features = ["extension-module"], optional = true } diff --git a/pyproject.toml b/libraries/fastformat/pyproject.toml similarity index 91% rename from pyproject.toml rename to libraries/fastformat/pyproject.toml index 439fdf8..b7d94ea 100644 --- a/pyproject.toml +++ b/libraries/fastformat/pyproject.toml @@ -11,5 +11,9 @@ classifiers = [ "Programming Language :: Python :: Implementation :: PyPy", ] dynamic = ["version"] + [tool.maturin] features = ["pyo3/extension-module"] + +[tool.pyright] +venvPath = ".venv" diff --git a/libraries/fastformat/src/lib.rs b/libraries/fastformat/src/lib.rs new file mode 100644 index 0000000..d7aa2fb --- /dev/null +++ b/libraries/fastformat/src/lib.rs @@ -0,0 +1,8 @@ +pub use fastformat_datatypes::image; +pub use fastformat_datatypes::image::Image; + +#[cfg(feature = "arrow")] +pub use fastformat_converter::arrow; + +#[cfg(feature = "ndarray")] +pub use fastformat_converter::ndarray; diff --git a/src/arrow.rs b/src/arrow.rs deleted file mode 100644 index 9f84456..0000000 --- a/src/arrow.rs +++ /dev/null @@ -1,162 +0,0 @@ -use arrow::datatypes::DataType; - -use std::{collections::HashMap, sync::Arc}; - -use eyre::{ContextCompat, Result}; - -/// Creates a lookup table (`HashMap`) from the fields of a union. -/// -/// This function takes a reference to `arrow::datatypes::UnionFields` and -/// creates a `HashMap` where the field names are the keys (as `String`) and -/// the associated values are the field identifiers (`i8`). -/// -/// # Arguments -/// -/// * `fields` - A reference to the fields of the Arrow union data structure (`arrow::datatypes::UnionFields`). -/// -/// # Returns -/// -/// A `HashMap` with the field names as keys and their identifiers (`i8`) as values. -/// -/// # Example -/// -/// ``` -/// use arrow::datatypes::{Field, DataType, UnionFields}; -/// use std::collections::HashMap; -/// -/// use fastformat::arrow::union_lookup_table; -/// -/// let fields = UnionFields::new( -/// vec![1, 2], -/// vec![ -/// Field::new("field1", DataType::Int32, false), -/// Field::new("field2", DataType::Float64, false), -/// ], -/// ); -/// -/// let lookup_table = union_lookup_table(&fields); -/// -/// assert_eq!(lookup_table.get("field1"), Some(&1)); -/// assert_eq!(lookup_table.get("field2"), Some(&2)); -/// ``` -pub fn union_lookup_table(fields: &arrow::datatypes::UnionFields) -> HashMap { - let mut result = HashMap::new(); - - for field in fields.iter() { - let (a, b) = field; - - result.insert(b.name().to_string(), a); - } - - result -} - -/// Retrieves a column from a `UnionArray` by its field name and downcasts it to the specified type. -/// -/// This function takes a reference to an `arrow::array::UnionArray`, a field name, -/// and a lookup table mapping field names to their identifiers. It retrieves the column -/// corresponding to the field name from the union array and attempts to downcast it to -/// the specified type `T`. -/// -/// # Arguments -/// -/// * `array` - A reference to the `UnionArray` from which to retrieve the column. -/// * `field` - The name of the field whose column is to be retrieved. -/// * `lookup_table` - A reference to a `HashMap` that maps field names (`String`) to their identifiers (`i8`). -/// -/// # Returns -/// -/// A `Result` containing a reference to the column cast to type `T` if successful, or an error otherwise. -/// -/// # Errors -/// -/// Returns an error if the field name is not found in the lookup table, or if the retrieved column -/// cannot be downcast to the specified type `T`. -/// -/// # Example -/// -/// ``` -/// use arrow::array::Array; -/// -/// use fastformat::image::Image; -/// use fastformat::arrow::union_lookup_table; -/// use fastformat::arrow::column_by_name; -/// -/// let data = vec![0; 27]; // 3x3 image with 3 bytes per pixel -/// let image = Image::new_bgr8(data, 3, 3, None).unwrap(); -/// let array = image.to_arrow().unwrap(); -/// -/// let union_fields = match array.data_type() { -/// arrow::datatypes::DataType::Union(fields, ..) => fields, -/// _ => panic!("Unexpected data type for image array") -/// }; -/// -/// let lookup_table = union_lookup_table(&union_fields); -/// -/// let int_column = column_by_name::(&array, "field1", &lookup_table); -/// ``` -pub fn column_by_name<'a, T: 'static>( - array: &'a arrow::array::UnionArray, - field: &'a str, - lookup_table: &'a HashMap, -) -> Result<&'a T> { - let index = lookup_table - .get(field) - .cloned() - .wrap_err(format!("Couldn't get field {} from look_up table", field))?; - - return array - .child(index) - .as_any() - .downcast_ref::() - .wrap_err(format!("Couldn't downcast field {} to type T", field)); -} - -/// Creates a tuple representing a union field with an index and an `Arc`-wrapped `Field`. -/// -/// This function constructs a tuple where the first element is the given index and the second element -/// is an `Arc`-wrapped `Field` constructed using the provided name, data type, and nullability. -/// -/// # Arguments -/// -/// * `index` - An identifier (`i8`) for the union field. -/// * `name` - A string slice representing the name of the field. -/// * `data_type` - The data type of the field (`arrow::datatypes::DataType`). -/// * `nullable` - A boolean indicating whether the field is nullable. -/// -/// # Returns -/// -/// A tuple where the first element is the given index (`i8`) and the second element is an `Arc`-wrapped -/// `Field` constructed from the provided name, data type, and nullability. -/// -/// # Example -/// -/// ``` -/// use arrow::datatypes::{DataType, Field}; -/// use std::sync::Arc; -/// use fastformat::arrow::union_field; -/// -/// let index = 1; -/// let name = "field1"; -/// let data_type = DataType::Int32; -/// let nullable = false; -/// -/// let union_field_tuple = union_field(index, name, data_type, nullable); -/// -/// assert_eq!(union_field_tuple.0, 1); -/// assert_eq!(union_field_tuple.1.name(), "field1"); -/// assert_eq!(union_field_tuple.1.data_type(), &DataType::Int32); -/// assert_eq!(union_field_tuple.1.is_nullable(), false); -/// ``` -/// -pub fn union_field( - index: i8, - name: &str, - data_type: DataType, - nullable: bool, -) -> (i8, Arc) { - ( - index, - Arc::new(arrow::datatypes::Field::new(name, data_type, nullable)), - ) -} diff --git a/src/image/arrow.rs b/src/image/arrow.rs deleted file mode 100644 index 3661588..0000000 --- a/src/image/arrow.rs +++ /dev/null @@ -1,206 +0,0 @@ -use crate::arrow::{column_by_name, union_field, union_lookup_table}; - -use super::{container::DataContainer, encoding::Encoding, Image}; -use eyre::{Context, Report, Result}; - -use std::{collections::HashMap, mem, sync::Arc}; - -impl Image { - unsafe fn arrow_data_to_vec( - array: &arrow::array::UnionArray, - lookup_table: &HashMap, - ) -> Result> { - let arrow = column_by_name::>(array, "data", lookup_table)?; - let ptr = arrow.values().as_ptr(); - let len = arrow.len(); - - Ok(Vec::from_raw_parts(ptr as *mut G, len, len)) - } - - /// Constructs an `Image` from an Arrow `UnionArray`. - /// - /// This function takes an Arrow `UnionArray` and extracts the necessary fields to construct - /// an `Image` object. It validates the data type of the `UnionArray`, builds a lookup table for - /// the fields, retrieves the image properties (width, height, encoding, name), and decodes the - /// pixel data based on the encoding. - /// - /// # Arguments - /// - /// * `array` - A reference to an `arrow::array::UnionArray` that contains the image data. - /// - /// # Returns - /// - /// A `Result` containing the constructed `Image` if successful, or an error otherwise. - /// - /// # Errors - /// - /// Returns an error if the `UnionArray` has an invalid data type, if required fields are missing, - /// or if the pixel data cannot be downcasted to the expected type based on the encoding. - /// - /// # Example - /// - /// ``` - /// use fastformat::image::Image; - /// - /// let data = vec![0; 27]; // 3x3 image with 3 bytes per pixel - /// let image = Image::new_bgr8(data, 3, 3, None).unwrap(); - /// let array = image.to_arrow().unwrap(); - /// - /// let image = Image::from_arrow(array).unwrap(); - /// ``` - pub fn from_arrow(array: arrow::array::UnionArray) -> Result { - use arrow::array::Array; - - let union_fields = match array.data_type() { - arrow::datatypes::DataType::Union(fields, ..) => fields, - _ => { - return Err(Report::msg("UnionArray has invalid data type.")); - } - }; - - let lookup_table = union_lookup_table(union_fields); - - let width = - column_by_name::(&array, "width", &lookup_table)?.value(0); - let height = - column_by_name::(&array, "height", &lookup_table)?.value(0); - let encoding = Encoding::from_string( - column_by_name::(&array, "encoding", &lookup_table)? - .value(0) - .to_string(), - )?; - - let name = column_by_name::(&array, "name", &lookup_table)?; - - let name = if name.is_null(0) { - None - } else { - Some(name.value(0).to_string()) - }; - - unsafe { - let array = mem::ManuallyDrop::new(array); - - let data = match encoding { - Encoding::RGB8 => DataContainer::from_u8(Self::arrow_data_to_vec::< - arrow::datatypes::UInt8Type, - u8, - >(&array, &lookup_table)?), - Encoding::BGR8 => DataContainer::from_u8(Self::arrow_data_to_vec::< - arrow::datatypes::UInt8Type, - u8, - >(&array, &lookup_table)?), - Encoding::GRAY8 => DataContainer::from_u8(Self::arrow_data_to_vec::< - arrow::datatypes::UInt8Type, - u8, - >(&array, &lookup_table)?), - }; - - Ok(Image { - data, - width, - height, - encoding, - name, - }) - } - } - - fn convert_image_details_to_arrow(image: Image) -> Result>> { - let width = Arc::new(arrow::array::UInt32Array::from(vec![image.width; 1])); - let height = Arc::new(arrow::array::UInt32Array::from(vec![image.height; 1])); - - let encoding = Arc::new(arrow::array::StringArray::from(vec![ - image - .encoding - .to_string(); - 1 - ])); - - let name = Arc::new(arrow::array::StringArray::from(vec![image.name.clone(); 1])); - - let data: Arc = match image.encoding { - Encoding::RGB8 => Arc::new(arrow::array::UInt8Array::from(image.data.into_u8()?)), - Encoding::BGR8 => Arc::new(arrow::array::UInt8Array::from(image.data.into_u8()?)), - Encoding::GRAY8 => Arc::new(arrow::array::UInt8Array::from(image.data.into_u8()?)), - }; - - Ok(vec![data, width, height, encoding, name]) - } - - /// Converts an `Image` into an Arrow `UnionArray`. - /// - /// This function takes an `Image` object and converts it into an Arrow `UnionArray` - /// that contains the image properties and pixel data. The conversion handles different - /// image encodings (BGR8, RGB8, GRAY8) and ensures that the resulting `UnionArray` - /// contains all necessary fields. - /// - /// # Arguments - /// - /// * `self` - The `Image` object to be converted. - /// - /// # Returns - /// - /// A `Result` containing the constructed `arrow::array::UnionArray` if successful, or an error otherwise. - /// - /// # Errors - /// - /// Returns an error if the `UnionArray` cannot be created due to issues with the provided data. - /// - /// # Example - /// - /// ``` - /// use fastformat::image::Image; - /// - /// let data = vec![0; 640 * 480 * 3]; - /// let image = Image::new_bgr8(data, 640, 480, None).unwrap(); - /// - /// let arrow_array = image.to_arrow().unwrap(); - /// ``` - pub fn to_arrow(self) -> Result { - let type_ids = [].into_iter().collect::>(); - let offsets = [].into_iter().collect::>(); - - let datatype = match self.encoding { - Encoding::RGB8 => arrow::datatypes::DataType::UInt8, - Encoding::BGR8 => arrow::datatypes::DataType::UInt8, - Encoding::GRAY8 => arrow::datatypes::DataType::UInt8, - }; - - let union_fields = [ - union_field(0, "data", datatype, false), - union_field(1, "width", arrow::datatypes::DataType::UInt32, false), - union_field(2, "height", arrow::datatypes::DataType::UInt32, false), - union_field(3, "encoding", arrow::datatypes::DataType::Utf8, false), - union_field(4, "name", arrow::datatypes::DataType::Utf8, true), - ] - .into_iter() - .collect::(); - - let children = Self::convert_image_details_to_arrow(self)?; - - arrow::array::UnionArray::try_new(union_fields, type_ids, Some(offsets), children) - .wrap_err("Failed to create UnionArray width Image data.") - } -} - -mod tests { - #[test] - fn test_arrow_zero_copy_conversion() { - use crate::image::Image; - - let flat_image = vec![0; 27]; - let original_buffer_address = flat_image.as_ptr(); - - let bgr8_image = Image::new_bgr8(flat_image, 3, 3, None).unwrap(); - let image_buffer_address = bgr8_image.as_ptr(); - - let arrow_image = bgr8_image.to_arrow().unwrap(); - - let new_image = Image::from_arrow(arrow_image).unwrap(); - let final_image_buffer = new_image.as_ptr(); - - assert_eq!(original_buffer_address, image_buffer_address); - assert_eq!(image_buffer_address, final_image_buffer); - } -} diff --git a/src/image/bgr8.rs b/src/image/bgr8.rs deleted file mode 100644 index f9c1f08..0000000 --- a/src/image/bgr8.rs +++ /dev/null @@ -1,282 +0,0 @@ -use super::{container::DataContainer, encoding::Encoding, Image}; -use eyre::{Context, Report, Result}; - -impl Image { - /// Creates a new `Image` in BGR8 format. - /// - /// This function constructs a new `Image` object with the given pixel data, width, height, - /// and an optional name. It ensures that the pixel data length matches the expected size - /// for the given width, height, and BGR8 encoding (3 bytes per pixel). - /// - /// # Arguments - /// - /// * `data` - A `Vec` containing the pixel data in BGR8 format. - /// * `width` - The width of the image. - /// * `height` - The height of the image. - /// * `name` - An optional string slice representing the name of the image. - /// - /// # Returns - /// - /// A `Result` containing the constructed `Image` if successful, or an error otherwise. - /// - /// # Errors - /// - /// Returns an error if the length of the pixel data does not match the expected size - /// based on the width, height, and BGR8 encoding. - /// - /// # Example - /// - /// ``` - /// use fastformat::image::Image; - /// - /// let data = vec![0; 27]; // 3x3 image with 3 bytes per pixel - /// let image = Image::new_bgr8(data, 3, 3, Some("example")).unwrap(); - /// ``` - pub fn new_bgr8(data: Vec, width: u32, height: u32, name: Option<&str>) -> Result { - if width * height * 3 != data.len() as u32 { - return Err(Report::msg( - "Width, height and BGR8 encoding doesn't match data data length.", - )); - } - - Ok(Image { - data: DataContainer::from_u8(data), - width, - height, - encoding: Encoding::BGR8, - name: name.map(|s| s.to_string()), - }) - } - - /// Creates a new `Image` in BGR8 format from an ndarray. - /// - /// This function constructs a new `Image` object from an `ndarray::Array` with shape (height, width, 3). - /// It converts the ndarray into a raw vector and uses it to create the `Image`. - /// - /// # Arguments - /// - /// * `array` - An `ndarray::Array` containing the pixel data in BGR8 format. - /// * `name` - An optional string slice representing the name of the image. - /// - /// # Returns - /// - /// A `Result` containing the constructed `Image` if successful, or an error otherwise. - /// - /// # Errors - /// - /// Returns an error if the ndarray cannot be converted into a valid `Image`. - /// - /// # Example - /// - /// ``` - /// use ndarray::Array3; - /// use fastformat::image::Image; - /// - /// let array = Array3::::zeros((3, 3, 3)); // 3x3 image with 3 channels - /// let image = Image::bgr8_from_ndarray(array, Some("example")).unwrap(); - /// ``` - pub fn bgr8_from_ndarray( - array: ndarray::Array, - name: Option<&str>, - ) -> Result { - let width = array.shape()[1] as u32; - let height = array.shape()[0] as u32; - - let data = array.into_raw_vec(); - - Self::new_bgr8(data, width, height, name) - } - - /// Converts a BGR8 `Image` into an ndarray. - /// - /// This function takes a BGR8 `Image` and converts it into an `ndarray::Array`. - /// The resulting ndarray has shape (height, width, 3). - /// - /// # Arguments - /// - /// * `self` - The `Image` object to be converted. - /// - /// # Returns - /// - /// A `Result` containing the constructed ndarray if successful, or an error otherwise. - /// - /// # Errors - /// - /// Returns an error if the `Image` is not in BGR8 format or if the pixel data cannot be reshaped - /// into the expected ndarray format. - /// - /// # Example - /// - /// ``` - /// use fastformat::image::Image; - /// - /// let data = vec![0; 27]; // 3x3 image with 3 bytes per pixel - /// let image = Image::new_bgr8(data, 3, 3, Some("example")).unwrap(); - /// - /// let ndarray = image.bgr8_to_ndarray().unwrap(); - /// ``` - pub fn bgr8_to_ndarray(self) -> Result> { - match self.encoding { - Encoding::BGR8 => ndarray::Array::from_shape_vec( - (self.height as usize, self.width as usize, 3), - self.data.into_u8()?, - ) - .wrap_err("Failed to reshape data into ndarray: width, height and BGR8 encoding doesn't match data data length."), - _ => Err(Report::msg("Image is not in BGR8 format")), - } - } - - /// Converts a BGR8 `Image` into an ndarray view. - /// - /// This function takes a reference to a BGR8 `Image` and creates an `ndarray::ArrayView` - /// over the pixel data. The resulting view has shape (height, width, 3). - /// - /// # Arguments - /// - /// * `&self` - A reference to the `Image` object to be viewed. - /// - /// # Returns - /// - /// A `Result` containing the ndarray view if successful, or an error otherwise. - /// - /// # Errors - /// - /// Returns an error if the `Image` is not in BGR8 format or if the pixel data cannot be reshaped - /// into the expected ndarray view format. - /// - /// # Example - /// - /// ``` - /// use fastformat::image::Image; - /// - /// let data = vec![0; 27]; // 3x3 image with 3 bytes per pixel - /// let image = Image::new_bgr8(data, 3, 3, Some("example")).unwrap(); - /// - /// let ndarray_view = image.bgr8_to_ndarray_view().unwrap(); - /// ``` - pub fn bgr8_to_ndarray_view(&self) -> Result> { - match self.encoding { - Encoding::BGR8 => ndarray::ArrayView::from_shape( - (self.height as usize, self.width as usize, 3), - self.data.as_u8()?, - ) - .wrap_err("Failed to reshape data into ndarray: width, height and BGR8 encoding doesn't match data data length."), - _ => Err(Report::msg("Image is not in BGR8 format")), - } - } - - /// Converts a mutable BGR8 `Image` into a mutable ndarray view. - /// - /// This function takes a mutable reference to a BGR8 `Image` and creates an `ndarray::ArrayViewMut` - /// over the pixel data. The resulting view has shape (height, width, 3). - /// - /// # Arguments - /// - /// * `&mut self` - A mutable reference to the `Image` object to be viewed. - /// - /// # Returns - /// - /// A `Result` containing the mutable ndarray view if successful, or an error otherwise. - /// - /// # Errors - /// - /// Returns an error if the `Image` is not in BGR8 format or if the pixel data cannot be reshaped - /// into the expected mutable ndarray view format. - /// - /// # Example - /// - /// ``` - /// use fastformat::image::Image; - /// - /// let data = vec![0; 27]; // 3x3 image with 3 bytes per pixel - /// let mut image = Image::new_bgr8(data, 3, 3, Some("example")).unwrap(); - /// - /// let ndarray_view_mut = image.bgr8_to_ndarray_view_mut().unwrap(); - /// ``` - pub fn bgr8_to_ndarray_view_mut(&mut self) -> Result> { - match self.encoding { - Encoding::BGR8 => ndarray::ArrayViewMut::from_shape( - (self.height as usize, self.width as usize, 3), - self.data.as_mut_u8()?, - ) - .wrap_err("Failed to reshape data into ndarray: width, height and BGR8 encoding doesn't match data data length."), - _ => Err(Report::msg("Image is not in BGR8 format")), - } - } -} - -mod tests { - #[test] - fn test_bgr8_creation() { - use crate::image::Image; - - let flat_image = vec![0; 27]; - - Image::new_bgr8(flat_image, 3, 3, Some("camera.test")).unwrap(); - } - - #[test] - fn test_bgr8_from_ndarray() { - use ndarray::Array3; - - use crate::image::Image; - - let array = Array3::::zeros((3, 3, 3)); - - Image::bgr8_from_ndarray(array, Some("camera.test")).unwrap(); - } - - #[test] - fn test_bgr8_to_ndarray() { - use crate::image::Image; - - let flat_image = vec![0; 27]; - - let image = Image::new_bgr8(flat_image, 3, 3, Some("camera.test")).unwrap(); - - image.bgr8_to_ndarray().unwrap(); - } - - #[test] - fn test_bgr8_to_ndarray_view() { - use crate::image::Image; - - let flat_image = vec![0; 27]; - - let image = Image::new_bgr8(flat_image, 3, 3, Some("camera.test")).unwrap(); - - image.bgr8_to_ndarray_view().unwrap(); - } - - #[test] - fn test_bgr8_to_ndarray_view_mut() { - use crate::image::Image; - - let flat_image = vec![0; 27]; - - let mut image = Image::new_bgr8(flat_image, 3, 3, Some("camera.test")).unwrap(); - - image.bgr8_to_ndarray_view_mut().unwrap(); - } - - #[test] - fn test_bgr8_ndarray_zero_copy_conversion() { - use crate::image::Image; - - let flat_image = vec![0; 27]; - let original_buffer_address = flat_image.as_ptr(); - - let bgr8_image = Image::new_bgr8(flat_image, 3, 3, None).unwrap(); - let image_buffer_address = bgr8_image.as_ptr(); - - let bgr8_ndarray = bgr8_image.bgr8_to_ndarray().unwrap(); - let ndarray_buffer_address = bgr8_ndarray.as_ptr(); - - let final_image = Image::bgr8_from_ndarray(bgr8_ndarray, None).unwrap(); - let final_image_buffer_address = final_image.as_ptr(); - - assert_eq!(original_buffer_address, image_buffer_address); - assert_eq!(image_buffer_address, ndarray_buffer_address); - assert_eq!(ndarray_buffer_address, final_image_buffer_address); - } -} diff --git a/src/image/container.rs b/src/image/container.rs deleted file mode 100644 index d05dd91..0000000 --- a/src/image/container.rs +++ /dev/null @@ -1,36 +0,0 @@ -use eyre::Result; - -#[derive(Debug)] -pub enum DataContainer { - U8Data(Vec), -} - -impl DataContainer { - pub fn as_ptr(&self) -> *const u8 { - match self { - Self::U8Data(data) => data.as_ptr(), - } - } - - pub fn into_u8(self) -> Result> { - match self { - Self::U8Data(data) => Ok(data), - } - } - - pub fn as_u8(&self) -> Result<&Vec> { - match self { - Self::U8Data(data) => Ok(data), - } - } - - pub fn as_mut_u8(&mut self) -> Result<&mut Vec> { - match self { - Self::U8Data(data) => Ok(data), - } - } - - pub fn from_u8(data: Vec) -> Self { - Self::U8Data(data) - } -} diff --git a/src/image/gray8.rs b/src/image/gray8.rs deleted file mode 100644 index 9a02230..0000000 --- a/src/image/gray8.rs +++ /dev/null @@ -1,280 +0,0 @@ -use super::{container::DataContainer, encoding::Encoding, Image}; -use eyre::{Context, Report, Result}; - -impl Image { - /// Creates a new `Image` in Gray8 format. - /// - /// This function constructs a new `Image` object with the given pixel data, width, height, - /// and an optional name. It ensures that the pixel data length matches the expected size - /// for the given width and height. - /// - /// # Arguments - /// - /// * `data` - A `Vec` containing the pixel data in Gray8 format. - /// * `width` - The width of the image. - /// * `height` - The height of the image. - /// * `name` - An optional string slice representing the name of the image. - /// - /// # Returns - /// - /// A `Result` containing the constructed `Image` if successful, or an error otherwise. - /// - /// # Errors - /// - /// Returns an error if the length of the pixel data does not match the expected size - /// based on the width and height. - /// - /// # Example - /// - /// ``` - /// use fastformat::image::Image; - /// - /// let data = vec![0; 9]; // 3x3 image with 1 byte per pixel - /// let image = Image::new_gray8(data, 3, 3, Some("example")).unwrap(); - /// ``` - pub fn new_gray8(data: Vec, width: u32, height: u32, name: Option<&str>) -> Result { - if data.len() != (width * height) as usize { - return Err(Report::msg("Invalid data data length.")); - } - - Ok(Image { - data: DataContainer::from_u8(data), - width, - height, - encoding: Encoding::GRAY8, - name: name.map(|s| s.to_string()), - }) - } - - /// Creates a new `Image` in Gray8 format from an ndarray. - /// - /// This function constructs a new `Image` object from an `ndarray::Array` with shape (height, width). - /// It converts the ndarray into a raw vector and uses it to create the `Image`. - /// - /// # Arguments - /// - /// * `array` - An `ndarray::Array` containing the pixel data in Gray8 format. - /// * `name` - An optional string slice representing the name of the image. - /// - /// # Returns - /// - /// A `Result` containing the constructed `Image` if successful, or an error otherwise. - /// - /// # Errors - /// - /// Returns an error if the ndarray cannot be converted into a valid `Image`. - /// - /// # Example - /// - /// ``` - /// use ndarray::Array2; - /// use fastformat::image::Image; - /// - /// let array = Array2::::zeros((3, 3)); // 3x3 image - /// let image = Image::gray8_from_ndarray(array, Some("example")).unwrap(); - /// ``` - pub fn gray8_from_ndarray( - array: ndarray::Array, - name: Option<&str>, - ) -> Result { - let width = array.shape()[1] as u32; - let height = array.shape()[0] as u32; - - let data = array.into_raw_vec(); - - Self::new_gray8(data, width, height, name) - } - - /// Converts a Gray8 `Image` into an ndarray. - /// - /// This function takes a Gray8 `Image` and converts it into an `ndarray::Array`. - /// The resulting ndarray has shape (height, width). - /// - /// # Arguments - /// - /// * `self` - The `Image` object to be converted. - /// - /// # Returns - /// - /// A `Result` containing the constructed ndarray if successful, or an error otherwise. - /// - /// # Errors - /// - /// Returns an error if the `Image` is not in Gray8 format or if the pixel data cannot be reshaped - /// into the expected ndarray format. - /// - /// # Example - /// - /// ``` - /// use fastformat::image::Image; - /// - /// let data = vec![0; 9]; // 3x3 image with 1 byte per pixel - /// let image = Image::new_gray8(data, 3, 3, Some("example")).unwrap(); - /// - /// let ndarray = image.gray8_to_ndarray().unwrap(); - /// ``` - pub fn gray8_to_ndarray(self) -> Result> { - match self.encoding { - Encoding::GRAY8 => ndarray::Array::from_shape_vec( - (self.height as usize, self.width as usize), - self.data.into_u8()?, - ) - .wrap_err("Failed to reshape data into ndarray: width, height and Gray8 encoding doesn't match data data length."), - _ => Err(Report::msg("Image is not in Gray8 format")), - } - } - - /// Converts a Gray8 `Image` into an ndarray view. - /// - /// This function takes a reference to a Gray8 `Image` and creates an `ndarray::ArrayView` - /// over the pixel data. The resulting view has shape (height, width). - /// - /// # Arguments - /// - /// * `&self` - A reference to the `Image` object to be viewed. - /// - /// # Returns - /// - /// A `Result` containing the ndarray view if successful, or an error otherwise. - /// - /// # Errors - /// - /// Returns an error if the `Image` is not in Gray8 format or if the pixel data cannot be reshaped - /// into the expected ndarray view format. - /// - /// # Example - /// - /// ``` - /// use fastformat::image::Image; - /// - /// let data = vec![0; 9]; // 3x3 image with 1 byte per pixel - /// let image = Image::new_gray8(data, 3, 3, Some("example")).unwrap(); - /// - /// let ndarray_view = image.gray8_to_ndarray_view().unwrap(); - /// ``` - pub fn gray8_to_ndarray_view(&self) -> Result> { - match self.encoding { - Encoding::GRAY8 => ndarray::ArrayView::from_shape( - (self.height as usize, self.width as usize), - self.data.as_u8()?, - ) - .wrap_err("Failed to reshape data into ndarray: width, height and Gray8 encoding doesn't match data data length."), - _ => Err(Report::msg("Image is not in Gray8 format")), - } - } - - /// Converts a mutable Gray8 `Image` into a mutable ndarray view. - /// - /// This function takes a mutable reference to a Gray8 `Image` and creates an `ndarray::ArrayViewMut` - /// over the pixel data. The resulting view has shape (height, width). - /// - /// # Arguments - /// - /// * `&mut self` - A mutable reference to the `Image` object to be viewed. - /// - /// # Returns - /// - /// A `Result` containing the mutable ndarray view if successful, or an error otherwise. - /// - /// # Errors - /// - /// Returns an error if the `Image` is not in Gray8 format or if the pixel data cannot be reshaped - /// into the expected mutable ndarray view format. - /// - /// # Example - /// - /// ``` - /// use fastformat::image::Image; - /// - /// let data = vec![0; 9]; // 3x3 image with 1 byte per pixel - /// let mut image = Image::new_gray8(data, 3, 3, Some("example")).unwrap(); - /// - /// let ndarray_view_mut = image.gray8_to_ndarray_view_mut().unwrap(); - /// ``` - pub fn gray8_to_ndarray_view_mut(&mut self) -> Result> { - match self.encoding { - Encoding::GRAY8 => ndarray::ArrayViewMut::from_shape( - (self.height as usize, self.width as usize), - self.data.as_mut_u8()?, - ) - .wrap_err("Failed to reshape data into ndarray: width, height and Gray8 encoding doesn't match data data length."), - _ => Err(Report::msg("Image is not in Gray8 format")), - } - } -} - -mod test { - #[test] - fn test_gray8_creation() { - use crate::image::Image; - - let flat_image = (1..10).collect::>(); - - Image::new_gray8(flat_image, 3, 3, Some("camera.test")).unwrap(); - } - - #[test] - fn test_gray8_from_ndarray() { - use ndarray::Array2; - - use crate::image::Image; - - let array = Array2::::zeros((3, 3)); - - Image::gray8_from_ndarray(array, Some("camera.test")).unwrap(); - } - - #[test] - fn test_gray8_to_ndarray() { - use crate::image::Image; - - let flat_image = (1..10).collect::>(); - - let image = Image::new_gray8(flat_image, 3, 3, Some("camera.test")).unwrap(); - - image.gray8_to_ndarray().unwrap(); - } - - #[test] - fn test_gray8_to_ndarray_view() { - use crate::image::Image; - - let flat_image = (1..10).collect::>(); - - let image = Image::new_gray8(flat_image, 3, 3, Some("camera.test")).unwrap(); - - image.gray8_to_ndarray_view().unwrap(); - } - - #[test] - fn test_gray8_to_ndarray_view_mut() { - use crate::image::Image; - - let flat_image = (1..10).collect::>(); - - let mut image = Image::new_gray8(flat_image, 3, 3, Some("camera.test")).unwrap(); - - image.gray8_to_ndarray_view_mut().unwrap(); - } - - #[test] - fn test_gray8_ndarray_zero_copy_conversion() { - use crate::image::Image; - - let flat_image = (1..10).collect::>(); - let original_buffer_address = flat_image.as_ptr(); - - let gray8_image = Image::new_gray8(flat_image, 3, 3, None).unwrap(); - let image_buffer_address = gray8_image.as_ptr(); - - let gray8_ndarray = gray8_image.gray8_to_ndarray().unwrap(); - let ndarray_buffer_address = gray8_ndarray.as_ptr(); - - let final_image = Image::gray8_from_ndarray(gray8_ndarray, None).unwrap(); - let final_image_buffer_address = final_image.as_ptr(); - - assert_eq!(original_buffer_address, image_buffer_address); - assert_eq!(image_buffer_address, ndarray_buffer_address); - assert_eq!(ndarray_buffer_address, final_image_buffer_address); - } -} diff --git a/src/image/rgb8.rs b/src/image/rgb8.rs deleted file mode 100644 index 8ebfda2..0000000 --- a/src/image/rgb8.rs +++ /dev/null @@ -1,280 +0,0 @@ -use super::{container::DataContainer, encoding::Encoding, Image}; -use eyre::{Context, Report, Result}; - -impl Image { - /// Creates a new `Image` in RGB8 format. - /// - /// This function constructs a new `Image` object with the given pixel data, width, height, - /// and an optional name. It ensures that the pixel data length matches the expected size - /// for the given width and height. - /// - /// # Arguments - /// - /// * `data` - A `Vec` containing the pixel data in RGB8 format. - /// * `width` - The width of the image. - /// * `height` - The height of the image. - /// * `name` - An optional string slice representing the name of the image. - /// - /// # Returns - /// - /// A `Result` containing the constructed `Image` if successful, or an error otherwise. - /// - /// # Errors - /// - /// Returns an error if the length of the pixel data does not match the expected size - /// based on the width and height. - /// - /// # Example - /// - /// ``` - /// use fastformat::image::Image; - /// - /// let data = vec![0; 27]; // 3x3 image with 3 bytes per pixel - /// let image = Image::new_rgb8(data, 3, 3, Some("example")).unwrap(); - /// ``` - pub fn new_rgb8(data: Vec, width: u32, height: u32, name: Option<&str>) -> Result { - if data.len() != (width * height * 3) as usize { - return Err(Report::msg("Invalid pixel data length.")); - } - - Ok(Image { - data: DataContainer::from_u8(data), - width, - height, - encoding: Encoding::RGB8, - name: name.map(|s| s.to_string()), - }) - } - - /// Creates a new `Image` in RGB8 format from an ndarray. - /// - /// This function constructs a new `Image` object from an `ndarray::Array` with shape (height, width, 3). - /// It converts the ndarray into a raw vector and uses it to create the `Image`. - /// - /// # Arguments - /// - /// * `array` - An `ndarray::Array` containing the pixel data in RGB8 format. - /// * `name` - An optional string slice representing the name of the image. - /// - /// # Returns - /// - /// A `Result` containing the constructed `Image` if successful, or an error otherwise. - /// - /// # Errors - /// - /// Returns an error if the ndarray cannot be converted into a valid `Image`. - /// - /// # Example - /// - /// ``` - /// use ndarray::Array3; - /// use fastformat::image::Image; - /// - /// let array = Array3::::zeros((3, 3, 3)); // 3x3 image with 3 bytes per pixel - /// let image = Image::rgb8_from_ndarray(array, Some("example")).unwrap(); - /// ``` - pub fn rgb8_from_ndarray( - array: ndarray::Array, - name: Option<&str>, - ) -> Result { - let width = array.shape()[1] as u32; - let height = array.shape()[0] as u32; - - let data = array.into_raw_vec(); - - Self::new_rgb8(data, width, height, name) - } - - /// Converts an RGB8 `Image` into an ndarray. - /// - /// This function takes an RGB8 `Image` and converts it into an `ndarray::Array`. - /// The resulting ndarray has shape (height, width, 3). - /// - /// # Arguments - /// - /// * `self` - The `Image` object to be converted. - /// - /// # Returns - /// - /// A `Result` containing the constructed ndarray if successful, or an error otherwise. - /// - /// # Errors - /// - /// Returns an error if the `Image` is not in RGB8 format or if the pixel data cannot be reshaped - /// into the expected ndarray format. - /// - /// # Example - /// - /// ``` - /// use fastformat::image::Image; - /// - /// let data = vec![0; 27]; // 3x3 image with 3 bytes per pixel - /// let image = Image::new_rgb8(data, 3, 3, Some("example")).unwrap(); - /// - /// let ndarray = image.rgb8_to_ndarray().unwrap(); - /// ``` - pub fn rgb8_to_ndarray(self) -> Result> { - match self.encoding { - Encoding::RGB8 => ndarray::Array::from_shape_vec( - (self.height as usize, self.width as usize, 3), - self.data.into_u8()?, - ) - .wrap_err("Failed to reshape data into ndarray: width, height and RGB8 encoding doesn't match data data length."), - _ => Err(Report::msg("Image is not in RGB8 format")), - } - } - - /// Converts an RGB8 `Image` into an ndarray view. - /// - /// This function takes a reference to an RGB8 `Image` and creates an `ndarray::ArrayView` - /// over the pixel data. The resulting view has shape (height, width, 3). - /// - /// # Arguments - /// - /// * `&self` - A reference to the `Image` object to be viewed. - /// - /// # Returns - /// - /// A `Result` containing the ndarray view if successful, or an error otherwise. - /// - /// # Errors - /// - /// Returns an error if the `Image` is not in RGB8 format or if the pixel data cannot be reshaped - /// into the expected ndarray view format. - /// - /// # Example - /// - /// ``` - /// use fastformat::image::Image; - /// - /// let data = vec![0; 27]; // 3x3 image with 3 bytes per pixel - /// let image = Image::new_rgb8(data, 3, 3, Some("example")).unwrap(); - /// - /// let ndarray_view = image.rgb8_to_ndarray_view().unwrap(); - /// ``` - pub fn rgb8_to_ndarray_view(&self) -> Result> { - match self.encoding { - Encoding::RGB8 => ndarray::ArrayView::from_shape( - (self.height as usize, self.width as usize, 3), - self.data.as_u8()?, - ) - .wrap_err("Failed to reshape data into ndarray: width, height and RGB8 encoding doesn't match data data length."), - _ => Err(Report::msg("Image is not in RGB8 format")), - } - } - - /// Converts a mutable RGB8 `Image` into a mutable ndarray view. - /// - /// This function takes a mutable reference to an RGB8 `Image` and creates an `ndarray::ArrayViewMut` - /// over the pixel data. The resulting view has shape (height, width, 3). - /// - /// # Arguments - /// - /// * `&mut self` - A mutable reference to the `Image` object to be viewed. - /// - /// # Returns - /// - /// A `Result` containing the mutable ndarray view if successful, or an error otherwise. - /// - /// # Errors - /// - /// Returns an error if the `Image` is not in RGB8 format or if the pixel data cannot be reshaped - /// into the expected mutable ndarray view format. - /// - /// # Example - /// - /// ``` - /// use fastformat::image::Image; - /// - /// let data = vec![0; 27]; // 3x3 image with 3 bytes per pixel - /// let mut image = Image::new_rgb8(data, 3, 3, Some("example")).unwrap(); - /// - /// let ndarray_view_mut = image.rgb8_to_ndarray_view_mut().unwrap(); - /// ``` - pub fn rgb8_to_ndarray_view_mut(&mut self) -> Result> { - match self.encoding { - Encoding::RGB8 => ndarray::ArrayViewMut::from_shape( - (self.height as usize, self.width as usize, 3), - self.data.as_mut_u8()?, - ) - .wrap_err("Failed to reshape data into ndarray: width, height and RGB8 encoding doesn't match data data length."), - _ => Err(Report::msg("Image is not in RGB8 format")), - } - } -} - -mod tests { - #[test] - fn test_rgb8_creation() { - use crate::image::Image; - - let flat_image = vec![0; 27]; - - Image::new_rgb8(flat_image, 3, 3, Some("camera.test")).unwrap(); - } - - #[test] - fn test_rgb8_from_ndarray() { - use ndarray::Array3; - - use crate::image::Image; - - let array = Array3::::zeros((3, 3, 3)); - - Image::rgb8_from_ndarray(array, Some("camera.test")).unwrap(); - } - - #[test] - fn test_rgb8_to_ndarray() { - use crate::image::Image; - - let flat_image = vec![0; 27]; - - let image = Image::new_rgb8(flat_image, 3, 3, Some("camera.test")).unwrap(); - - image.rgb8_to_ndarray().unwrap(); - } - - #[test] - fn test_rgb8_to_ndarray_view() { - use crate::image::Image; - - let flat_image = vec![0; 27]; - - let image = Image::new_rgb8(flat_image, 3, 3, Some("camera.test")).unwrap(); - - image.rgb8_to_ndarray_view().unwrap(); - } - - #[test] - fn test_rgb8_to_ndarray_view_mut() { - use crate::image::Image; - - let flat_image = vec![0; 27]; - - let mut image = Image::new_rgb8(flat_image, 3, 3, Some("camera.test")).unwrap(); - - image.rgb8_to_ndarray_view_mut().unwrap(); - } - - #[test] - fn test_rgb8_ndarray_zero_copy_conversion() { - use crate::image::Image; - - let flat_image = vec![0; 27]; - let original_buffer_address = flat_image.as_ptr(); - - let rgb8_image = Image::new_rgb8(flat_image, 3, 3, None).unwrap(); - let image_buffer_address = rgb8_image.as_ptr(); - - let rgb8_ndarray = rgb8_image.rgb8_to_ndarray().unwrap(); - let ndarray_buffer_address = rgb8_ndarray.as_ptr(); - - let final_image = Image::rgb8_from_ndarray(rgb8_ndarray, None).unwrap(); - let final_image_buffer_address = final_image.as_ptr(); - - assert_eq!(original_buffer_address, image_buffer_address); - assert_eq!(image_buffer_address, ndarray_buffer_address); - assert_eq!(ndarray_buffer_address, final_image_buffer_address); - } -} diff --git a/src/lib.rs b/src/lib.rs deleted file mode 100644 index 7419bd0..0000000 --- a/src/lib.rs +++ /dev/null @@ -1,18 +0,0 @@ -use pyo3::prelude::*; - -pub mod arrow; - -pub mod image; - -/// Formats the sum of two numbers as string. -#[pyfunction] -fn sum_as_string(a: usize, b: usize) -> PyResult { - Ok((a + b).to_string()) -} - -/// A Python module implemented in Rust. -#[pymodule] -fn fastformat(m: &Bound<'_, PyModule>) -> PyResult<()> { - m.add_function(wrap_pyfunction!(sum_as_string, m)?)?; - Ok(()) -}