Skip to content

Commit

Permalink
Auto merge of #122905 - dpaoliello:sync-portable-simd-2024-03-22, r=w…
Browse files Browse the repository at this point in the history
…orkingjubilee

Portable SIMD subtree update

Syncs nightly to the latest changes from rust-lang/portable-simd

r? `@calebzulawski`
  • Loading branch information
bors committed Mar 24, 2024
2 parents 2f090c3 + 9e0ec25 commit 9b8d12c
Show file tree
Hide file tree
Showing 13 changed files with 505 additions and 52 deletions.
5 changes: 5 additions & 0 deletions library/portable-simd/.github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,11 @@ jobs:
- name: Test (release)
run: cargo test --verbose --target=${{ matrix.target }} --release

- name: Generate docs
run: cargo doc --verbose --target=${{ matrix.target }}
env:
RUSTDOCFLAGS: -Dwarnings

wasm-tests:
name: "wasm (firefox, ${{ matrix.name }})"
runs-on: ubuntu-latest
Expand Down
3 changes: 3 additions & 0 deletions library/portable-simd/Cargo.lock
Original file line number Diff line number Diff line change
Expand Up @@ -177,6 +177,9 @@ name = "std_float"
version = "0.1.0"
dependencies = [
"core_simd",
"test_helpers",
"wasm-bindgen",
"wasm-bindgen-test",
]

[[package]]
Expand Down
12 changes: 11 additions & 1 deletion library/portable-simd/crates/core_simd/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,12 @@
simd_ffi,
staged_api,
strict_provenance,
prelude_import,
ptr_metadata
)]
#![cfg_attr(
all(
any(target_arch = "aarch64", target_arch = "arm",),
any(target_arch = "aarch64", target_arch = "arm64ec", target_arch = "arm",),
any(
all(target_feature = "v6", not(target_feature = "mclass")),
all(target_feature = "mclass", target_feature = "dsp"),
Expand All @@ -33,12 +34,21 @@
any(target_arch = "powerpc", target_arch = "powerpc64"),
feature(stdarch_powerpc)
)]
#![cfg_attr(
all(target_arch = "x86_64", target_feature = "avx512f"),
feature(stdarch_x86_avx512)
)]
#![warn(missing_docs, clippy::missing_inline_in_public_items)] // basically all items, really
#![deny(unsafe_op_in_unsafe_fn, clippy::undocumented_unsafe_blocks)]
#![doc(test(attr(deny(warnings))))]
#![allow(internal_features)]
#![unstable(feature = "portable_simd", issue = "86656")]
//! Portable SIMD module.

#[prelude_import]
#[allow(unused_imports)]
use core::prelude::v1::*;

#[path = "mod.rs"]
mod core_simd;
pub use self::core_simd::simd;
6 changes: 6 additions & 0 deletions library/portable-simd/crates/core_simd/src/masks.rs
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ mod sealed {
fn eq(self, other: Self) -> bool;

fn to_usize(self) -> usize;
fn max_unsigned() -> u64;

type Unsigned: SimdElement;

Expand Down Expand Up @@ -78,6 +79,11 @@ macro_rules! impl_element {
self as usize
}

#[inline]
fn max_unsigned() -> u64 {
<$unsigned>::MAX as u64
}

type Unsigned = $unsigned;

const TRUE: Self = -1;
Expand Down
8 changes: 6 additions & 2 deletions library/portable-simd/crates/core_simd/src/swizzle_dyn.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,10 @@ where
#[inline]
pub fn swizzle_dyn(self, idxs: Simd<u8, N>) -> Self {
#![allow(unused_imports, unused_unsafe)]
#[cfg(all(target_arch = "aarch64", target_endian = "little"))]
#[cfg(all(
any(target_arch = "aarch64", target_arch = "arm64ec"),
target_endian = "little"
))]
use core::arch::aarch64::{uint8x8_t, vqtbl1q_u8, vtbl1_u8};
#[cfg(all(
target_arch = "arm",
Expand All @@ -37,6 +40,7 @@ where
#[cfg(all(
any(
target_arch = "aarch64",
target_arch = "arm64ec",
all(target_arch = "arm", target_feature = "v7")
),
target_feature = "neon",
Expand All @@ -48,7 +52,7 @@ where
#[cfg(target_feature = "simd128")]
16 => transize(wasm::i8x16_swizzle, self, idxs),
#[cfg(all(
target_arch = "aarch64",
any(target_arch = "aarch64", target_arch = "arm64ec"),
target_feature = "neon",
target_endian = "little"
))]
Expand Down
244 changes: 244 additions & 0 deletions library/portable-simd/crates/core_simd/src/vector.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
use crate::simd::{
cmp::SimdPartialOrd,
num::SimdUint,
ptr::{SimdConstPtr, SimdMutPtr},
LaneCount, Mask, MaskElement, SupportedLaneCount, Swizzle,
};
Expand Down Expand Up @@ -262,6 +263,7 @@ where
/// # Panics
///
/// Panics if the slice's length is less than the vector's `Simd::N`.
/// Use `load_or_default` for an alternative that does not panic.
///
/// # Example
///
Expand Down Expand Up @@ -315,6 +317,143 @@ where
unsafe { self.store(slice.as_mut_ptr().cast()) }
}

/// Reads contiguous elements from `slice`. Elements are read so long as they're in-bounds for
/// the `slice`. Otherwise, the default value for the element type is returned.
///
/// # Examples
/// ```
/// # #![feature(portable_simd)]
/// # #[cfg(feature = "as_crate")] use core_simd::simd;
/// # #[cfg(not(feature = "as_crate"))] use core::simd;
/// # use simd::Simd;
/// let vec: Vec<i32> = vec![10, 11];
///
/// let result = Simd::<i32, 4>::load_or_default(&vec);
/// assert_eq!(result, Simd::from_array([10, 11, 0, 0]));
/// ```
#[must_use]
#[inline]
pub fn load_or_default(slice: &[T]) -> Self
where
T: Default,
{
Self::load_or(slice, Default::default())
}

/// Reads contiguous elements from `slice`. Elements are read so long as they're in-bounds for
/// the `slice`. Otherwise, the corresponding value from `or` is passed through.
///
/// # Examples
/// ```
/// # #![feature(portable_simd)]
/// # #[cfg(feature = "as_crate")] use core_simd::simd;
/// # #[cfg(not(feature = "as_crate"))] use core::simd;
/// # use simd::Simd;
/// let vec: Vec<i32> = vec![10, 11];
/// let or = Simd::from_array([-5, -4, -3, -2]);
///
/// let result = Simd::load_or(&vec, or);
/// assert_eq!(result, Simd::from_array([10, 11, -3, -2]));
/// ```
#[must_use]
#[inline]
pub fn load_or(slice: &[T], or: Self) -> Self {
Self::load_select(slice, Mask::splat(true), or)
}

/// Reads contiguous elements from `slice`. Each element is read from memory if its
/// corresponding element in `enable` is `true`.
///
/// When the element is disabled or out of bounds for the slice, that memory location
/// is not accessed and the corresponding value from `or` is passed through.
///
/// # Examples
/// ```
/// # #![feature(portable_simd)]
/// # #[cfg(feature = "as_crate")] use core_simd::simd;
/// # #[cfg(not(feature = "as_crate"))] use core::simd;
/// # use simd::{Simd, Mask};
/// let vec: Vec<i32> = vec![10, 11, 12, 13, 14, 15, 16, 17, 18];
/// let enable = Mask::from_array([true, true, false, true]);
/// let or = Simd::from_array([-5, -4, -3, -2]);
///
/// let result = Simd::load_select(&vec, enable, or);
/// assert_eq!(result, Simd::from_array([10, 11, -3, 13]));
/// ```
#[must_use]
#[inline]
pub fn load_select_or_default(slice: &[T], enable: Mask<<T as SimdElement>::Mask, N>) -> Self
where
T: Default,
{
Self::load_select(slice, enable, Default::default())
}

/// Reads contiguous elements from `slice`. Each element is read from memory if its
/// corresponding element in `enable` is `true`.
///
/// When the element is disabled or out of bounds for the slice, that memory location
/// is not accessed and the corresponding value from `or` is passed through.
///
/// # Examples
/// ```
/// # #![feature(portable_simd)]
/// # #[cfg(feature = "as_crate")] use core_simd::simd;
/// # #[cfg(not(feature = "as_crate"))] use core::simd;
/// # use simd::{Simd, Mask};
/// let vec: Vec<i32> = vec![10, 11, 12, 13, 14, 15, 16, 17, 18];
/// let enable = Mask::from_array([true, true, false, true]);
/// let or = Simd::from_array([-5, -4, -3, -2]);
///
/// let result = Simd::load_select(&vec, enable, or);
/// assert_eq!(result, Simd::from_array([10, 11, -3, 13]));
/// ```
#[must_use]
#[inline]
pub fn load_select(
slice: &[T],
mut enable: Mask<<T as SimdElement>::Mask, N>,
or: Self,
) -> Self {
enable &= mask_up_to(slice.len());
// SAFETY: We performed the bounds check by updating the mask. &[T] is properly aligned to
// the element.
unsafe { Self::load_select_ptr(slice.as_ptr(), enable, or) }
}

/// Reads contiguous elements from `slice`. Each element is read from memory if its
/// corresponding element in `enable` is `true`.
///
/// When the element is disabled, that memory location is not accessed and the corresponding
/// value from `or` is passed through.
#[must_use]
#[inline]
pub unsafe fn load_select_unchecked(
slice: &[T],
enable: Mask<<T as SimdElement>::Mask, N>,
or: Self,
) -> Self {
let ptr = slice.as_ptr();
// SAFETY: The safety of reading elements from `slice` is ensured by the caller.
unsafe { Self::load_select_ptr(ptr, enable, or) }
}

/// Reads contiguous elements starting at `ptr`. Each element is read from memory if its
/// corresponding element in `enable` is `true`.
///
/// When the element is disabled, that memory location is not accessed and the corresponding
/// value from `or` is passed through.
#[must_use]
#[inline]
pub unsafe fn load_select_ptr(
ptr: *const T,
enable: Mask<<T as SimdElement>::Mask, N>,
or: Self,
) -> Self {
// SAFETY: The safety of reading elements through `ptr` is ensured by the caller.
unsafe { core::intrinsics::simd::simd_masked_load(enable.to_int(), ptr, or) }
}

/// Reads from potentially discontiguous indices in `slice` to construct a SIMD vector.
/// If an index is out-of-bounds, the element is instead selected from the `or` vector.
///
Expand Down Expand Up @@ -493,6 +632,77 @@ where
unsafe { core::intrinsics::simd::simd_gather(or, source, enable.to_int()) }
}

/// Conditionally write contiguous elements to `slice`. The `enable` mask controls
/// which elements are written, as long as they're in-bounds of the `slice`.
/// If the element is disabled or out of bounds, no memory access to that location
/// is made.
///
/// # Examples
/// ```
/// # #![feature(portable_simd)]
/// # #[cfg(feature = "as_crate")] use core_simd::simd;
/// # #[cfg(not(feature = "as_crate"))] use core::simd;
/// # use simd::{Simd, Mask};
/// let mut arr = [0i32; 4];
/// let write = Simd::from_array([-5, -4, -3, -2]);
/// let enable = Mask::from_array([false, true, true, true]);
///
/// write.store_select(&mut arr[..3], enable);
/// assert_eq!(arr, [0, -4, -3, 0]);
/// ```
#[inline]
pub fn store_select(self, slice: &mut [T], mut enable: Mask<<T as SimdElement>::Mask, N>) {
enable &= mask_up_to(slice.len());
// SAFETY: We performed the bounds check by updating the mask. &[T] is properly aligned to
// the element.
unsafe { self.store_select_ptr(slice.as_mut_ptr(), enable) }
}

/// Conditionally write contiguous elements to `slice`. The `enable` mask controls
/// which elements are written.
///
/// # Safety
///
/// Every enabled element must be in bounds for the `slice`.
///
/// # Examples
/// ```
/// # #![feature(portable_simd)]
/// # #[cfg(feature = "as_crate")] use core_simd::simd;
/// # #[cfg(not(feature = "as_crate"))] use core::simd;
/// # use simd::{Simd, Mask};
/// let mut arr = [0i32; 4];
/// let write = Simd::from_array([-5, -4, -3, -2]);
/// let enable = Mask::from_array([false, true, true, true]);
///
/// unsafe { write.store_select_unchecked(&mut arr, enable) };
/// assert_eq!(arr, [0, -4, -3, -2]);
/// ```
#[inline]
pub unsafe fn store_select_unchecked(
self,
slice: &mut [T],
enable: Mask<<T as SimdElement>::Mask, N>,
) {
let ptr = slice.as_mut_ptr();
// SAFETY: The safety of writing elements in `slice` is ensured by the caller.
unsafe { self.store_select_ptr(ptr, enable) }
}

/// Conditionally write contiguous elements starting from `ptr`.
/// The `enable` mask controls which elements are written.
/// When disabled, the memory location corresponding to that element is not accessed.
///
/// # Safety
///
/// Memory addresses for element are calculated [`pointer::wrapping_offset`] and
/// each enabled element must satisfy the same conditions as [`core::ptr::write`].
#[inline]
pub unsafe fn store_select_ptr(self, ptr: *mut T, enable: Mask<<T as SimdElement>::Mask, N>) {
// SAFETY: The safety of writing elements through `ptr` is ensured by the caller.
unsafe { core::intrinsics::simd::simd_masked_store(enable.to_int(), ptr, self) }
}

/// Writes the values in a SIMD vector to potentially discontiguous indices in `slice`.
/// If an index is out-of-bounds, the write is suppressed without panicking.
/// If two elements in the scattered vector would write to the same index
Expand Down Expand Up @@ -980,3 +1190,37 @@ where
{
type Mask = isize;
}

#[inline]
fn lane_indices<const N: usize>() -> Simd<usize, N>
where
LaneCount<N>: SupportedLaneCount,
{
let mut index = [0; N];
for i in 0..N {
index[i] = i;
}
Simd::from_array(index)
}

#[inline]
fn mask_up_to<M, const N: usize>(len: usize) -> Mask<M, N>
where
LaneCount<N>: SupportedLaneCount,
M: MaskElement,
{
let index = lane_indices::<N>();
let max_value: u64 = M::max_unsigned();
macro_rules! case {
($ty:ty) => {
if N < <$ty>::MAX as usize && max_value as $ty as u64 == max_value {
return index.cast().simd_lt(Simd::splat(len.min(N) as $ty)).cast();
}
};
}
case!(u8);
case!(u16);
case!(u32);
case!(u64);
index.simd_lt(Simd::splat(len)).cast()
}
2 changes: 1 addition & 1 deletion library/portable-simd/crates/core_simd/src/vendor.rs
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ mod x86;
#[cfg(target_arch = "wasm32")]
mod wasm32;

#[cfg(any(target_arch = "aarch64", target_arch = "arm",))]
#[cfg(any(target_arch = "aarch64", target_arch = "arm64ec", target_arch = "arm",))]
mod arm;

#[cfg(any(target_arch = "powerpc", target_arch = "powerpc64"))]
Expand Down
Loading

0 comments on commit 9b8d12c

Please sign in to comment.