From ceead1bda65519cca77a1f946e22c0d68eadc209 Mon Sep 17 00:00:00 2001 From: Maybe Waffle Date: Tue, 16 Apr 2024 01:51:21 +0000 Subject: [PATCH 1/7] Change intrinsic types to use `u32` instead of `T` to match stable reexports --- compiler/rustc_codegen_llvm/src/intrinsic.rs | 27 +++++++---- .../src/interpret/intrinsics.rs | 21 +++++---- .../rustc_hir_analysis/src/check/intrinsic.rs | 14 +++--- library/core/src/intrinsics.rs | 47 +++++++++++++++++++ library/core/src/num/nonzero.rs | 14 +++++- library/core/src/num/uint_macros.rs | 25 ++++++++-- 6 files changed, 117 insertions(+), 31 deletions(-) diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 2bed7c1bd1ccf..ab9f20fdf63c3 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -315,25 +315,32 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { Some((width, signed)) => match name { sym::ctlz | sym::cttz => { let y = self.const_bool(false); - self.call_intrinsic( + let ret = self.call_intrinsic( &format!("llvm.{name}.i{width}"), &[args[0].immediate(), y], - ) + ); + + self.intcast(ret, llret_ty, false) } sym::ctlz_nonzero => { let y = self.const_bool(true); let llvm_name = &format!("llvm.ctlz.i{width}"); - self.call_intrinsic(llvm_name, &[args[0].immediate(), y]) + let ret = self.call_intrinsic(llvm_name, &[args[0].immediate(), y]); + self.intcast(ret, llret_ty, false) } sym::cttz_nonzero => { let y = self.const_bool(true); let llvm_name = &format!("llvm.cttz.i{width}"); - self.call_intrinsic(llvm_name, &[args[0].immediate(), y]) + let ret = self.call_intrinsic(llvm_name, &[args[0].immediate(), y]); + self.intcast(ret, llret_ty, false) + } + sym::ctpop => { + let ret = self.call_intrinsic( + &format!("llvm.ctpop.i{width}"), + &[args[0].immediate()], + ); + self.intcast(ret, llret_ty, false) } - sym::ctpop => self.call_intrinsic( - &format!("llvm.ctpop.i{width}"), - &[args[0].immediate()], - ), sym::bswap => { if width == 8 { args[0].immediate() // byte swap a u8/i8 is just a no-op @@ -355,6 +362,10 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { // rotate = funnel shift with first two args the same let llvm_name = &format!("llvm.fsh{}.i{}", if is_left { 'l' } else { 'r' }, width); + + // llvm expects shift to be the same type as the values, but rust always uses `u32` + let raw_shift = self.intcast(raw_shift, self.val_ty(val), false); + self.call_intrinsic(llvm_name, &[val, val, raw_shift]) } sym::saturating_add | sym::saturating_sub => { diff --git a/compiler/rustc_const_eval/src/interpret/intrinsics.rs b/compiler/rustc_const_eval/src/interpret/intrinsics.rs index 63c709d8aedd6..8a32d7bff1d83 100644 --- a/compiler/rustc_const_eval/src/interpret/intrinsics.rs +++ b/compiler/rustc_const_eval/src/interpret/intrinsics.rs @@ -173,7 +173,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { let ty = instance_args.type_at(0); let layout = self.layout_of(ty)?; let val = self.read_scalar(&args[0])?; - let out_val = self.numeric_intrinsic(intrinsic_name, val, layout)?; + + let out_val = self.numeric_intrinsic(intrinsic_name, val, layout, dest.layout)?; self.write_scalar(out_val, dest)?; } sym::saturating_add | sym::saturating_sub => { @@ -200,12 +201,15 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { sym::rotate_left | sym::rotate_right => { // rotate_left: (X << (S % BW)) | (X >> ((BW - S) % BW)) // rotate_right: (X << ((BW - S) % BW)) | (X >> (S % BW)) - let layout = self.layout_of(instance_args.type_at(0))?; + let layout_val = self.layout_of(instance_args.type_at(0))?; let val = self.read_scalar(&args[0])?; - let val_bits = val.to_bits(layout.size)?; + let val_bits = val.to_bits(layout_val.size)?; + + let layout_raw_shift = self.layout_of(self.tcx.types.u32)?; let raw_shift = self.read_scalar(&args[1])?; - let raw_shift_bits = raw_shift.to_bits(layout.size)?; - let width_bits = u128::from(layout.size.bits()); + let raw_shift_bits = raw_shift.to_bits(layout_raw_shift.size)?; + + let width_bits = u128::from(layout_val.size.bits()); let shift_bits = raw_shift_bits % width_bits; let inv_shift_bits = (width_bits - shift_bits) % width_bits; let result_bits = if intrinsic_name == sym::rotate_left { @@ -213,8 +217,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { } else { (val_bits >> shift_bits) | (val_bits << inv_shift_bits) }; - let truncated_bits = self.truncate(result_bits, layout); - let result = Scalar::from_uint(truncated_bits, layout.size); + let truncated_bits = self.truncate(result_bits, layout_val); + let result = Scalar::from_uint(truncated_bits, layout_val.size); self.write_scalar(result, dest)?; } sym::copy => { @@ -472,6 +476,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { name: Symbol, val: Scalar, layout: TyAndLayout<'tcx>, + ret_layout: TyAndLayout<'tcx>, ) -> InterpResult<'tcx, Scalar> { assert!(layout.ty.is_integral(), "invalid type for numeric intrinsic: {}", layout.ty); let bits = val.to_bits(layout.size)?; @@ -487,7 +492,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { sym::bitreverse => (bits << extra).reverse_bits(), _ => bug!("not a numeric intrinsic: {}", name), }; - Ok(Scalar::from_uint(bits_out, layout.size)) + Ok(Scalar::from_uint(bits_out, ret_layout.size)) } pub fn exact_div( diff --git a/compiler/rustc_hir_analysis/src/check/intrinsic.rs b/compiler/rustc_hir_analysis/src/check/intrinsic.rs index bd64621f07738..d73b157ec23c6 100644 --- a/compiler/rustc_hir_analysis/src/check/intrinsic.rs +++ b/compiler/rustc_hir_analysis/src/check/intrinsic.rs @@ -411,13 +411,11 @@ pub fn check_intrinsic_type( (1, 0, vec![Ty::new_mut_ptr(tcx, param(0)), param(0)], Ty::new_unit(tcx)) } - sym::ctpop - | sym::ctlz - | sym::ctlz_nonzero - | sym::cttz - | sym::cttz_nonzero - | sym::bswap - | sym::bitreverse => (1, 0, vec![param(0)], param(0)), + sym::ctpop | sym::ctlz | sym::ctlz_nonzero | sym::cttz | sym::cttz_nonzero => { + (1, 0, vec![param(0)], tcx.types.u32) + } + + sym::bswap | sym::bitreverse => (1, 0, vec![param(0)], param(0)), sym::three_way_compare => { (1, 0, vec![param(0), param(0)], tcx.ty_ordering_enum(Some(span))) @@ -460,7 +458,7 @@ pub fn check_intrinsic_type( (1, 0, vec![param(0), param(0)], param(0)) } sym::unchecked_shl | sym::unchecked_shr => (2, 0, vec![param(0), param(1)], param(0)), - sym::rotate_left | sym::rotate_right => (1, 0, vec![param(0), param(0)], param(0)), + sym::rotate_left | sym::rotate_right => (1, 0, vec![param(0), tcx.types.u32], param(0)), sym::unchecked_add | sym::unchecked_sub | sym::unchecked_mul => { (1, 0, vec![param(0), param(0)], param(0)) } diff --git a/library/core/src/intrinsics.rs b/library/core/src/intrinsics.rs index 9406efd7ab24a..23e444219c56d 100644 --- a/library/core/src/intrinsics.rs +++ b/library/core/src/intrinsics.rs @@ -1987,6 +1987,13 @@ extern "rust-intrinsic" { /// The stabilized versions of this intrinsic are available on the integer /// primitives via the `count_ones` method. For example, /// [`u32::count_ones`] + #[cfg(not(bootstrap))] + #[rustc_const_stable(feature = "const_ctpop", since = "1.40.0")] + #[rustc_safe_intrinsic] + #[rustc_nounwind] + pub fn ctpop(x: T) -> u32; + + #[cfg(bootstrap)] #[rustc_const_stable(feature = "const_ctpop", since = "1.40.0")] #[rustc_safe_intrinsic] #[rustc_nounwind] @@ -2028,6 +2035,13 @@ extern "rust-intrinsic" { /// let num_leading = ctlz(x); /// assert_eq!(num_leading, 16); /// ``` + #[cfg(not(bootstrap))] + #[rustc_const_stable(feature = "const_ctlz", since = "1.40.0")] + #[rustc_safe_intrinsic] + #[rustc_nounwind] + pub fn ctlz(x: T) -> u32; + + #[cfg(bootstrap)] #[rustc_const_stable(feature = "const_ctlz", since = "1.40.0")] #[rustc_safe_intrinsic] #[rustc_nounwind] @@ -2050,6 +2064,12 @@ extern "rust-intrinsic" { /// let num_leading = unsafe { ctlz_nonzero(x) }; /// assert_eq!(num_leading, 3); /// ``` + #[cfg(not(bootstrap))] + #[rustc_const_stable(feature = "constctlz", since = "1.50.0")] + #[rustc_nounwind] + pub fn ctlz_nonzero(x: T) -> u32; + + #[cfg(bootstrap)] #[rustc_const_stable(feature = "constctlz", since = "1.50.0")] #[rustc_nounwind] pub fn ctlz_nonzero(x: T) -> T; @@ -2090,6 +2110,13 @@ extern "rust-intrinsic" { /// let num_trailing = cttz(x); /// assert_eq!(num_trailing, 16); /// ``` + #[cfg(not(bootstrap))] + #[rustc_const_stable(feature = "const_cttz", since = "1.40.0")] + #[rustc_safe_intrinsic] + #[rustc_nounwind] + pub fn cttz(x: T) -> u32; + + #[cfg(bootstrap)] #[rustc_const_stable(feature = "const_cttz", since = "1.40.0")] #[rustc_safe_intrinsic] #[rustc_nounwind] @@ -2112,6 +2139,12 @@ extern "rust-intrinsic" { /// let num_trailing = unsafe { cttz_nonzero(x) }; /// assert_eq!(num_trailing, 3); /// ``` + #[cfg(not(bootstrap))] + #[rustc_const_stable(feature = "const_cttz_nonzero", since = "1.53.0")] + #[rustc_nounwind] + pub fn cttz_nonzero(x: T) -> u32; + + #[cfg(bootstrap)] #[rustc_const_stable(feature = "const_cttz_nonzero", since = "1.53.0")] #[rustc_nounwind] pub fn cttz_nonzero(x: T) -> T; @@ -2288,6 +2321,13 @@ extern "rust-intrinsic" { /// The stabilized versions of this intrinsic are available on the integer /// primitives via the `rotate_left` method. For example, /// [`u32::rotate_left`] + #[cfg(not(bootstrap))] + #[rustc_const_stable(feature = "const_int_rotate", since = "1.40.0")] + #[rustc_safe_intrinsic] + #[rustc_nounwind] + pub fn rotate_left(x: T, shift: u32) -> T; + + #[cfg(bootstrap)] #[rustc_const_stable(feature = "const_int_rotate", since = "1.40.0")] #[rustc_safe_intrinsic] #[rustc_nounwind] @@ -2303,6 +2343,13 @@ extern "rust-intrinsic" { /// The stabilized versions of this intrinsic are available on the integer /// primitives via the `rotate_right` method. For example, /// [`u32::rotate_right`] + #[cfg(not(bootstrap))] + #[rustc_const_stable(feature = "const_int_rotate", since = "1.40.0")] + #[rustc_safe_intrinsic] + #[rustc_nounwind] + pub fn rotate_right(x: T, shift: u32) -> T; + + #[cfg(bootstrap)] #[rustc_const_stable(feature = "const_int_rotate", since = "1.40.0")] #[rustc_safe_intrinsic] #[rustc_nounwind] diff --git a/library/core/src/num/nonzero.rs b/library/core/src/num/nonzero.rs index 62ea7abf6528a..a6724baa774bc 100644 --- a/library/core/src/num/nonzero.rs +++ b/library/core/src/num/nonzero.rs @@ -528,7 +528,12 @@ macro_rules! nonzero_integer { #[inline] pub const fn leading_zeros(self) -> u32 { // SAFETY: since `self` cannot be zero, it is safe to call `ctlz_nonzero`. - unsafe { intrinsics::ctlz_nonzero(self.get() as $UnsignedPrimitive) as u32 } + unsafe { + #[cfg(not(bootstrap))] + return intrinsics::ctlz_nonzero(self.get() as $UnsignedPrimitive); + #[cfg(bootstrap)] + return intrinsics::ctlz_nonzero(self.get() as $UnsignedPrimitive) as u32; + } } /// Returns the number of trailing zeros in the binary representation @@ -552,7 +557,12 @@ macro_rules! nonzero_integer { #[inline] pub const fn trailing_zeros(self) -> u32 { // SAFETY: since `self` cannot be zero, it is safe to call `cttz_nonzero`. - unsafe { intrinsics::cttz_nonzero(self.get() as $UnsignedPrimitive) as u32 } + unsafe { + #[cfg(not(bootstrap))] + return intrinsics::cttz_nonzero(self.get() as $UnsignedPrimitive); + #[cfg(bootstrap)] + return intrinsics::cttz_nonzero(self.get() as $UnsignedPrimitive) as u32; + } } /// Returns the number of ones in the binary representation of `self`. diff --git a/library/core/src/num/uint_macros.rs b/library/core/src/num/uint_macros.rs index ba6a243041ce1..e82f94ea78b0a 100644 --- a/library/core/src/num/uint_macros.rs +++ b/library/core/src/num/uint_macros.rs @@ -77,7 +77,10 @@ macro_rules! uint_impl { without modifying the original"] #[inline(always)] pub const fn count_ones(self) -> u32 { - intrinsics::ctpop(self as $ActualT) as u32 + #[cfg(not(bootstrap))] + return intrinsics::ctpop(self as $ActualT); + #[cfg(bootstrap)] + return intrinsics::ctpop(self as $ActualT) as u32; } /// Returns the number of zeros in the binary representation of `self`. @@ -119,7 +122,10 @@ macro_rules! uint_impl { without modifying the original"] #[inline(always)] pub const fn leading_zeros(self) -> u32 { - intrinsics::ctlz(self as $ActualT) as u32 + #[cfg(not(bootstrap))] + return intrinsics::ctlz(self as $ActualT); + #[cfg(bootstrap)] + return intrinsics::ctlz(self as $ActualT) as u32; } /// Returns the number of trailing zeros in the binary representation @@ -140,7 +146,10 @@ macro_rules! uint_impl { without modifying the original"] #[inline(always)] pub const fn trailing_zeros(self) -> u32 { - intrinsics::cttz(self) as u32 + #[cfg(not(bootstrap))] + return intrinsics::cttz(self); + #[cfg(bootstrap)] + return intrinsics::cttz(self) as u32; } /// Returns the number of leading ones in the binary representation of `self`. @@ -205,7 +214,10 @@ macro_rules! uint_impl { without modifying the original"] #[inline(always)] pub const fn rotate_left(self, n: u32) -> Self { - intrinsics::rotate_left(self, n as $SelfT) + #[cfg(not(bootstrap))] + return intrinsics::rotate_left(self, n); + #[cfg(bootstrap)] + return intrinsics::rotate_left(self, n as $SelfT); } /// Shifts the bits to the right by a specified amount, `n`, @@ -230,7 +242,10 @@ macro_rules! uint_impl { without modifying the original"] #[inline(always)] pub const fn rotate_right(self, n: u32) -> Self { - intrinsics::rotate_right(self, n as $SelfT) + #[cfg(not(bootstrap))] + return intrinsics::rotate_right(self, n); + #[cfg(bootstrap)] + return intrinsics::rotate_right(self, n as $SelfT); } /// Reverses the byte order of the integer. From e4a854e49534eaa986e156a58696b790e9c61ff6 Mon Sep 17 00:00:00 2001 From: Maybe Waffle Date: Tue, 16 Apr 2024 11:53:15 +0000 Subject: [PATCH 2/7] Miri: adopt to new intrinsic types --- src/tools/miri/src/shims/intrinsics/simd.rs | 2 +- src/tools/miri/tests/fail/intrinsics/ctlz_nonzero.rs | 2 +- src/tools/miri/tests/fail/intrinsics/cttz_nonzero.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/tools/miri/src/shims/intrinsics/simd.rs b/src/tools/miri/src/shims/intrinsics/simd.rs index a2fc4f0f761af..9a0671430d438 100644 --- a/src/tools/miri/src/shims/intrinsics/simd.rs +++ b/src/tools/miri/src/shims/intrinsics/simd.rs @@ -163,7 +163,7 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { } } Op::Numeric(name) => { - this.numeric_intrinsic(name, op.to_scalar(), op.layout)? + this.numeric_intrinsic(name, op.to_scalar(), op.layout, op.layout)? } }; this.write_scalar(val, &dest)?; diff --git a/src/tools/miri/tests/fail/intrinsics/ctlz_nonzero.rs b/src/tools/miri/tests/fail/intrinsics/ctlz_nonzero.rs index c26cd4cadb539..0b34afc6090a8 100644 --- a/src/tools/miri/tests/fail/intrinsics/ctlz_nonzero.rs +++ b/src/tools/miri/tests/fail/intrinsics/ctlz_nonzero.rs @@ -2,7 +2,7 @@ mod rusti { extern "rust-intrinsic" { - pub fn ctlz_nonzero(x: T) -> T; + pub fn ctlz_nonzero(x: T) -> u32; } } diff --git a/src/tools/miri/tests/fail/intrinsics/cttz_nonzero.rs b/src/tools/miri/tests/fail/intrinsics/cttz_nonzero.rs index 25a0501fdd80e..e220411f58544 100644 --- a/src/tools/miri/tests/fail/intrinsics/cttz_nonzero.rs +++ b/src/tools/miri/tests/fail/intrinsics/cttz_nonzero.rs @@ -2,7 +2,7 @@ mod rusti { extern "rust-intrinsic" { - pub fn cttz_nonzero(x: T) -> T; + pub fn cttz_nonzero(x: T) -> u32; } } From 7ce867f55296055d5c456bdc4002ee7048a40049 Mon Sep 17 00:00:00 2001 From: Maybe Waffle Date: Tue, 16 Apr 2024 11:56:21 +0000 Subject: [PATCH 3/7] Add an assertion in const eval --- compiler/rustc_const_eval/src/interpret/intrinsics.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/compiler/rustc_const_eval/src/interpret/intrinsics.rs b/compiler/rustc_const_eval/src/interpret/intrinsics.rs index 8a32d7bff1d83..4d37c3c22cd7c 100644 --- a/compiler/rustc_const_eval/src/interpret/intrinsics.rs +++ b/compiler/rustc_const_eval/src/interpret/intrinsics.rs @@ -488,8 +488,14 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { } sym::ctlz | sym::ctlz_nonzero => u128::from(bits.leading_zeros()) - extra, sym::cttz | sym::cttz_nonzero => u128::from((bits << extra).trailing_zeros()) - extra, - sym::bswap => (bits << extra).swap_bytes(), - sym::bitreverse => (bits << extra).reverse_bits(), + sym::bswap => { + assert_eq!(layout, ret_layout); + (bits << extra).swap_bytes() + } + sym::bitreverse => { + assert_eq!(layout, ret_layout); + (bits << extra).reverse_bits() + } _ => bug!("not a numeric intrinsic: {}", name), }; Ok(Scalar::from_uint(bits_out, ret_layout.size)) From c2046c4b09e04a1ebffbc7465a967267be1b2047 Mon Sep 17 00:00:00 2001 From: Maybe Waffle Date: Tue, 16 Apr 2024 12:35:22 +0000 Subject: [PATCH 4/7] Add codegen tests for changed intrinsics --- tests/codegen/intrinsics/ctlz.rs | 56 +++++++++++++++++++++++++ tests/codegen/intrinsics/ctpop.rs | 31 ++++++++++++++ tests/codegen/intrinsics/rotate_left.rs | 31 ++++++++++++++ 3 files changed, 118 insertions(+) create mode 100644 tests/codegen/intrinsics/ctlz.rs create mode 100644 tests/codegen/intrinsics/ctpop.rs create mode 100644 tests/codegen/intrinsics/rotate_left.rs diff --git a/tests/codegen/intrinsics/ctlz.rs b/tests/codegen/intrinsics/ctlz.rs new file mode 100644 index 0000000000000..0d54d21ce12bb --- /dev/null +++ b/tests/codegen/intrinsics/ctlz.rs @@ -0,0 +1,56 @@ +//@ compile-flags: -C no-prepopulate-passes + +#![crate_type = "lib"] +#![feature(core_intrinsics)] + +use std::intrinsics::{ctlz, ctlz_nonzero}; + +// CHECK-LABEL: @ctlz_u16 +#[no_mangle] +pub unsafe fn ctlz_u16(x: u16) -> u32 { + // CHECK: %[[tmp:.*]] = call i16 @llvm.ctlz.i16(i16 %x, i1 false) + // CHECK: zext i16 %[[tmp]] to i32 + ctlz(x) +} + +// CHECK-LABEL: @ctlz_nzu16 +#[no_mangle] +pub unsafe fn ctlz_nzu16(x: u16) -> u32 { + // CHECK: %[[tmp:.*]] = call i16 @llvm.ctlz.i16(i16 %x, i1 true) + // CHECK: zext i16 %[[tmp]] to i32 + ctlz_nonzero(x) +} + +// CHECK-LABEL: @ctlz_u32 +#[no_mangle] +pub unsafe fn ctlz_u32(x: u32) -> u32 { + // CHECK: call i32 @llvm.ctlz.i32(i32 %x, i1 false) + // CHECK-NOT: zext + // CHECK-NOT: trunc + ctlz(x) +} + +// CHECK-LABEL: @ctlz_nzu32 +#[no_mangle] +pub unsafe fn ctlz_nzu32(x: u32) -> u32 { + // CHECK: call i32 @llvm.ctlz.i32(i32 %x, i1 true) + // CHECK-NOT: zext + // CHECK-NOT: trunc + ctlz_nonzero(x) +} + +// CHECK-LABEL: @ctlz_u64 +#[no_mangle] +pub unsafe fn ctlz_u64(x: u64) -> u32 { + // CHECK: %[[tmp:.*]] = call i64 @llvm.ctlz.i64(i64 %x, i1 false) + // CHECK: trunc i64 %[[tmp]] to i32 + ctlz(x) +} + +// CHECK-LABEL: @ctlz_nzu64 +#[no_mangle] +pub unsafe fn ctlz_nzu64(x: u64) -> u32 { + // CHECK: %[[tmp:.*]] = call i64 @llvm.ctlz.i64(i64 %x, i1 true) + // CHECK: trunc i64 %[[tmp]] to i32 + ctlz_nonzero(x) +} diff --git a/tests/codegen/intrinsics/ctpop.rs b/tests/codegen/intrinsics/ctpop.rs new file mode 100644 index 0000000000000..f4043325de988 --- /dev/null +++ b/tests/codegen/intrinsics/ctpop.rs @@ -0,0 +1,31 @@ +//@ compile-flags: -C no-prepopulate-passes + +#![crate_type = "lib"] +#![feature(core_intrinsics)] + +use std::intrinsics::ctpop; + +// CHECK-LABEL: @ctpop_u16 +#[no_mangle] +pub unsafe fn ctpop_u16(x: u16) -> u32 { + // CHECK: %[[tmp:.*]] = call i16 @llvm.ctpop.i16(i16 %x) + // CHECK: zext i16 %[[tmp]] to i32 + ctpop(x) +} + +// CHECK-LABEL: @ctpop_u32 +#[no_mangle] +pub unsafe fn ctpop_u32(x: u32) -> u32 { + // CHECK: call i32 @llvm.ctpop.i32(i32 %x) + // CHECK-NOT: zext + // CHECK-NOT: trunc + ctpop(x) +} + +// CHECK-LABEL: @ctpop_u64 +#[no_mangle] +pub unsafe fn ctpop_u64(x: u64) -> u32 { + // CHECK: %[[tmp:.*]] = call i64 @llvm.ctpop.i64(i64 %x) + // CHECK: trunc i64 %[[tmp]] to i32 + ctpop(x) +} diff --git a/tests/codegen/intrinsics/rotate_left.rs b/tests/codegen/intrinsics/rotate_left.rs new file mode 100644 index 0000000000000..4f6c5cbaed6aa --- /dev/null +++ b/tests/codegen/intrinsics/rotate_left.rs @@ -0,0 +1,31 @@ +//@ compile-flags: -C no-prepopulate-passes + +#![crate_type = "lib"] +#![feature(core_intrinsics)] + +use std::intrinsics::rotate_left; + +// CHECK-LABEL: @rotate_left_u16 +#[no_mangle] +pub unsafe fn rotate_left_u16(x: u16, shift: u32) -> u16 { + // CHECK: %[[tmp:.*]] = trunc i32 %shift to i16 + // CHECK: call i16 @llvm.fshl.i16(i16 %x, i16 %x, i16 %[[tmp]]) + rotate_left(x, shift) +} + +// CHECK-LABEL: @rotate_left_u32 +#[no_mangle] +pub unsafe fn rotate_left_u32(x: u32, shift: u32) -> u32 { + // CHECK-NOT: trunc + // CHECK-NOT: zext + // CHECK: call i32 @llvm.fshl.i32(i32 %x, i32 %x, i32 %shift) + rotate_left(x, shift) +} + +// CHECK-LABEL: @rotate_left_u64 +#[no_mangle] +pub unsafe fn rotate_left_u64(x: u64, shift: u32) -> u64 { + // CHECK: %[[tmp:.*]] = zext i32 %shift to i64 + // CHECK: call i64 @llvm.fshl.i64(i64 %x, i64 %x, i64 %[[tmp]]) + rotate_left(x, shift) +} From 4b6bbcbaa35e61db72424bbd161c01f50f860bbc Mon Sep 17 00:00:00 2001 From: Maybe Waffle Date: Tue, 16 Apr 2024 12:39:53 +0000 Subject: [PATCH 5/7] Fixup 2 ui tests using changed intrinsics --- tests/ui/intrinsics/bad-intrinsic-monomorphization.rs | 2 +- tests/ui/intrinsics/intrinsics-integer.rs | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/ui/intrinsics/bad-intrinsic-monomorphization.rs b/tests/ui/intrinsics/bad-intrinsic-monomorphization.rs index fa9cbe4400cef..254ac24f0b941 100644 --- a/tests/ui/intrinsics/bad-intrinsic-monomorphization.rs +++ b/tests/ui/intrinsics/bad-intrinsic-monomorphization.rs @@ -16,7 +16,7 @@ use std::intrinsics; #[derive(Copy, Clone)] pub struct Foo(i64); -pub fn test_cttz(v: Foo) -> Foo { +pub fn test_cttz(v: Foo) -> u32 { intrinsics::cttz(v) //~^ ERROR `cttz` intrinsic: expected basic integer type, found `Foo` } diff --git a/tests/ui/intrinsics/intrinsics-integer.rs b/tests/ui/intrinsics/intrinsics-integer.rs index bfd7e4714fef8..7dbc4b8b7cece 100644 --- a/tests/ui/intrinsics/intrinsics-integer.rs +++ b/tests/ui/intrinsics/intrinsics-integer.rs @@ -6,13 +6,13 @@ mod rusti { extern "rust-intrinsic" { #[rustc_safe_intrinsic] - pub fn ctpop(x: T) -> T; + pub fn ctpop(x: T) -> u32; #[rustc_safe_intrinsic] - pub fn ctlz(x: T) -> T; - pub fn ctlz_nonzero(x: T) -> T; + pub fn ctlz(x: T) -> u32; + pub fn ctlz_nonzero(x: T) -> u32; #[rustc_safe_intrinsic] - pub fn cttz(x: T) -> T; - pub fn cttz_nonzero(x: T) -> T; + pub fn cttz(x: T) -> u32; + pub fn cttz_nonzero(x: T) -> u32; #[rustc_safe_intrinsic] pub fn bswap(x: T) -> T; #[rustc_safe_intrinsic] From c0b5cc9003f6464c11ae1c0662c6a7e06f6f5cab Mon Sep 17 00:00:00 2001 From: Maybe Waffle Date: Fri, 19 Apr 2024 18:45:25 +0000 Subject: [PATCH 6/7] Do intrinsic changes in `rustc_codegen_cranelift` --- compiler/rustc_codegen_cranelift/example/example.rs | 2 +- compiler/rustc_codegen_cranelift/example/mini_core.rs | 2 +- compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs | 10 +++++++--- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/compiler/rustc_codegen_cranelift/example/example.rs b/compiler/rustc_codegen_cranelift/example/example.rs index 885e55bc76423..1ef2aa5dd8ea4 100644 --- a/compiler/rustc_codegen_cranelift/example/example.rs +++ b/compiler/rustc_codegen_cranelift/example/example.rs @@ -149,7 +149,7 @@ pub fn array_as_slice(arr: &[u8; 3]) -> &[u8] { arr } -pub unsafe fn use_ctlz_nonzero(a: u16) -> u16 { +pub unsafe fn use_ctlz_nonzero(a: u16) -> u32 { intrinsics::ctlz_nonzero(a) } diff --git a/compiler/rustc_codegen_cranelift/example/mini_core.rs b/compiler/rustc_codegen_cranelift/example/mini_core.rs index e45c16ee280a7..5e535ff62e173 100644 --- a/compiler/rustc_codegen_cranelift/example/mini_core.rs +++ b/compiler/rustc_codegen_cranelift/example/mini_core.rs @@ -627,7 +627,7 @@ pub mod intrinsics { pub fn min_align_of_val(val: *const T) -> usize; pub fn copy(src: *const T, dst: *mut T, count: usize); pub fn transmute(e: T) -> U; - pub fn ctlz_nonzero(x: T) -> T; + pub fn ctlz_nonzero(x: T) -> u32; #[rustc_safe_intrinsic] pub fn needs_drop() -> bool; #[rustc_safe_intrinsic] diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs index 0b213ff826969..79a90507fa2e1 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs @@ -26,6 +26,7 @@ use rustc_span::source_map::Spanned; use rustc_span::symbol::{sym, Symbol}; pub(crate) use self::llvm::codegen_llvm_intrinsic_call; +use crate::cast::clif_intcast; use crate::prelude::*; fn bug_on_incorrect_arg_count(intrinsic: impl std::fmt::Display) -> ! { @@ -627,7 +628,8 @@ fn codegen_regular_intrinsic_call<'tcx>( // FIXME trap on `ctlz_nonzero` with zero arg. let res = fx.bcx.ins().clz(val); - let res = CValue::by_val(res, arg.layout()); + let res = clif_intcast(fx, res, types::I32, false); + let res = CValue::by_val(res, ret.layout()); ret.write_cvalue(fx, res); } sym::cttz | sym::cttz_nonzero => { @@ -636,7 +638,8 @@ fn codegen_regular_intrinsic_call<'tcx>( // FIXME trap on `cttz_nonzero` with zero arg. let res = fx.bcx.ins().ctz(val); - let res = CValue::by_val(res, arg.layout()); + let res = clif_intcast(fx, res, types::I32, false); + let res = CValue::by_val(res, ret.layout()); ret.write_cvalue(fx, res); } sym::ctpop => { @@ -644,7 +647,8 @@ fn codegen_regular_intrinsic_call<'tcx>( let val = arg.load_scalar(fx); let res = fx.bcx.ins().popcnt(val); - let res = CValue::by_val(res, arg.layout()); + let res = clif_intcast(fx, res, types::I32, false); + let res = CValue::by_val(res, ret.layout()); ret.write_cvalue(fx, res); } sym::bitreverse => { From 468179c680a35e47ed49fd22fe029415bc983b19 Mon Sep 17 00:00:00 2001 From: Maybe Waffle Date: Sat, 20 Apr 2024 03:22:54 +0000 Subject: [PATCH 7/7] Fixup `rustc_codegen_gcc` test signature --- compiler/rustc_codegen_gcc/example/example.rs | 7 ++++--- compiler/rustc_codegen_gcc/example/mini_core.rs | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/compiler/rustc_codegen_gcc/example/example.rs b/compiler/rustc_codegen_gcc/example/example.rs index 5878e8548d926..7c21b73b630e8 100644 --- a/compiler/rustc_codegen_gcc/example/example.rs +++ b/compiler/rustc_codegen_gcc/example/example.rs @@ -153,9 +153,10 @@ fn array_as_slice(arr: &[u8; 3]) -> &[u8] { arr } -unsafe fn use_ctlz_nonzero(a: u16) -> u16 { - intrinsics::ctlz_nonzero(a) -} +// FIXME: fix the intrinsic implementation to work with the new ->u32 signature +// unsafe fn use_ctlz_nonzero(a: u16) -> u32 { +// intrinsics::ctlz_nonzero(a) +// } fn ptr_as_usize(ptr: *const u8) -> usize { ptr as usize diff --git a/compiler/rustc_codegen_gcc/example/mini_core.rs b/compiler/rustc_codegen_gcc/example/mini_core.rs index 4665009e191e6..8ffa66a489464 100644 --- a/compiler/rustc_codegen_gcc/example/mini_core.rs +++ b/compiler/rustc_codegen_gcc/example/mini_core.rs @@ -593,7 +593,7 @@ pub mod intrinsics { pub fn min_align_of_val(val: *const T) -> usize; pub fn copy(src: *const T, dst: *mut T, count: usize); pub fn transmute(e: T) -> U; - pub fn ctlz_nonzero(x: T) -> T; + pub fn ctlz_nonzero(x: T) -> u32; #[rustc_safe_intrinsic] pub fn needs_drop() -> bool; #[rustc_safe_intrinsic]