diff --git a/compiler/rustc_codegen_cranelift/src/constant.rs b/compiler/rustc_codegen_cranelift/src/constant.rs index ef72e6efb946b..0d2642299d825 100644 --- a/compiler/rustc_codegen_cranelift/src/constant.rs +++ b/compiler/rustc_codegen_cranelift/src/constant.rs @@ -251,6 +251,11 @@ pub(crate) fn codegen_const_value<'tcx>( .iconst(fx.pointer_type, i64::try_from(end.checked_sub(start).unwrap()).unwrap()); CValue::by_val_pair(ptr, len, layout) } + ConstValue::CustomSlice { data, length } => { + let ptr = pointer_for_allocation(fx, data).get_addr(fx); + let len = fx.bcx.ins().iconst(fx.pointer_type, i64::try_from(length).unwrap()); + CValue::by_val_pair(ptr, len, layout) + } } } diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs index 6937e658ed5ee..bad2bca244c54 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs @@ -720,7 +720,7 @@ fn codegen_regular_intrinsic_call<'tcx>( dest.write_cvalue(fx, val); }; - pref_align_of | needs_drop | type_id | type_name | variant_count, () { + pref_align_of | needs_drop | type_id | type_name | variant_count | validity_invariants_of, () { let const_val = fx.tcx.const_eval_instance(ParamEnv::reveal_all(), instance, None).unwrap(); let val = crate::constant::codegen_const_value( diff --git a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs index 0ed4c3f1d9430..fa567d139497b 100644 --- a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs +++ b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs @@ -106,7 +106,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { | sym::needs_drop | sym::type_id | sym::type_name - | sym::variant_count => { + | sym::variant_count + | sym::validity_invariants_of => { let value = bx .tcx() .const_eval_instance(ty::ParamEnv::reveal_all(), instance, None) diff --git a/compiler/rustc_codegen_ssa/src/mir/operand.rs b/compiler/rustc_codegen_ssa/src/mir/operand.rs index 08be4c0a7b622..ddb88d97b7f9b 100644 --- a/compiler/rustc_codegen_ssa/src/mir/operand.rs +++ b/compiler/rustc_codegen_ssa/src/mir/operand.rs @@ -100,6 +100,22 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> { let b_llval = bx.const_usize((end - start) as u64); OperandValue::Pair(a_llval, b_llval) } + ConstValue::CustomSlice { data, length } => { + let Abi::ScalarPair(a_scalar, _) = layout.abi else { + bug!("from_const: invalid ScalarPair layout: {:#?}", layout); + }; + let a = Scalar::from_pointer( + Pointer::new(bx.tcx().create_memory_alloc(data), Size::ZERO), + &bx.tcx(), + ); + let a_llval = bx.scalar_to_backend( + a, + a_scalar, + bx.scalar_pair_element_backend_type(layout, 0, true), + ); + let b_llval = bx.const_usize(length as u64); + OperandValue::Pair(a_llval, b_llval) + } ConstValue::ByRef { alloc, offset } => { return bx.load_operand(bx.from_const_alloc(layout, alloc, offset)); } diff --git a/compiler/rustc_const_eval/src/interpret/intrinsics.rs b/compiler/rustc_const_eval/src/interpret/intrinsics.rs index e51c51cf45e5e..95d3951160e44 100644 --- a/compiler/rustc_const_eval/src/interpret/intrinsics.rs +++ b/compiler/rustc_const_eval/src/interpret/intrinsics.rs @@ -5,6 +5,7 @@ use std::convert::TryFrom; use rustc_hir::def_id::DefId; +use rustc_hir::lang_items::LangItem; use rustc_middle::mir::{ self, interpret::{ConstValue, GlobalId, InterpResult, Scalar}, @@ -16,6 +17,7 @@ use rustc_middle::ty::subst::SubstsRef; use rustc_middle::ty::{Ty, TyCtxt}; use rustc_span::symbol::{sym, Symbol}; use rustc_target::abi::{Abi, Align, InitKind, Primitive, Size}; +use rustc_target::spec::SanitizerSet; use super::{ util::ensure_monomorphic_enough, CheckInAllocMsg, ImmTy, InterpCx, Machine, OpTy, PlaceTy, @@ -24,6 +26,7 @@ use super::{ mod caller_location; mod type_name; +mod validity_invariants_of; fn numeric_intrinsic(name: Symbol, bits: u128, kind: Primitive) -> Scalar { let size = match kind { @@ -103,6 +106,23 @@ pub(crate) fn eval_nullary_intrinsic<'tcx>( | ty::Tuple(_) | ty::Error(_) => ConstValue::from_machine_usize(0u64, &tcx), }, + sym::validity_invariants_of => { + let msan = tcx.sess.opts.debugging_opts.sanitizer.contains(SanitizerSet::MEMORY); + let disable = tcx.sess.opts.debugging_opts.no_validity_invariant_checks; + + let strictness = if disable { + validity_invariants_of::InvariantStrictness::Disable + } else if msan { + validity_invariants_of::InvariantStrictness::All + } else { + validity_invariants_of::InvariantStrictness::Normal + }; + + ensure_monomorphic_enough(tcx, tp_ty)?; + let (data, length) = + validity_invariants_of::alloc_validity_invariants_of(tcx, tp_ty, strictness); + ConstValue::CustomSlice { data, length } + } other => bug!("`{}` is not a zero arg intrinsic", other), }) } @@ -162,6 +182,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { | sym::needs_drop | sym::type_id | sym::type_name + | sym::validity_invariants_of | sym::variant_count => { let gid = GlobalId { instance, promoted: None }; let ty = match intrinsic_name { @@ -169,6 +190,11 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { sym::needs_drop => self.tcx.types.bool, sym::type_id => self.tcx.types.u64, sym::type_name => self.tcx.mk_static_str(), + sym::validity_invariants_of => { + let item = self.tcx.require_lang_item(LangItem::ValidityInvariant, None); + let ty = self.tcx.type_of(item); + self.tcx.mk_imm_ref(self.tcx.lifetimes.re_static, self.tcx.mk_slice(ty)) + } _ => bug!(), }; let val = diff --git a/compiler/rustc_const_eval/src/interpret/intrinsics/validity_invariants_of.rs b/compiler/rustc_const_eval/src/interpret/intrinsics/validity_invariants_of.rs new file mode 100644 index 0000000000000..4ff1629798fc1 --- /dev/null +++ b/compiler/rustc_const_eval/src/interpret/intrinsics/validity_invariants_of.rs @@ -0,0 +1,171 @@ +use rustc_data_structures::fx::FxHashMap; +use rustc_hir::lang_items::LangItem; +use rustc_middle::mir::interpret::{AllocRange, Allocation, ConstAllocation, Scalar as MirScalar}; +use rustc_middle::mir::Mutability; +use rustc_middle::ty::layout::LayoutCx; +use rustc_middle::ty::{ParamEnv, ParamEnvAnd}; +use rustc_middle::ty::{Ty, TyCtxt}; +use rustc_target::abi::{ + Abi, FieldsShape, HasDataLayout, Integer, Primitive, Scalar, Size, TyAndLayout, Variants, + WrappingRange, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +enum InvariantSize { + U8, + U16, + U32, + U64, + U128, + Pointer, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +struct InvariantKey { + offset: Size, + size: InvariantSize, +} + +// FIXME: Don't add duplicate invariants (maybe use a HashMap?) +fn add_invariants<'tcx>( + tcx: TyCtxt<'tcx>, + ty: Ty<'tcx>, + invs: &mut FxHashMap, + offset: Size, + strictness: InvariantStrictness, +) { + if strictness == InvariantStrictness::Disable { + return; + } + + let x = tcx.layout_of(ParamEnvAnd { param_env: ParamEnv::reveal_all(), value: ty }); + + if let Ok(layout) = x { + if let Abi::Scalar(Scalar::Initialized { value, valid_range }) = layout.layout.abi() { + let size = match value { + Primitive::Int(Integer::I8, _) => InvariantSize::U8, + Primitive::Int(Integer::I16, _) => InvariantSize::U16, + Primitive::Int(Integer::I32, _) => InvariantSize::U32, + Primitive::Int(Integer::I64, _) => InvariantSize::U64, + Primitive::Int(Integer::I128, _) => InvariantSize::U128, + Primitive::F32 => InvariantSize::U32, + Primitive::F64 => InvariantSize::U64, + Primitive::Pointer => InvariantSize::Pointer, + }; + + if !valid_range.is_full_for(value.size(&tcx)) || strictness == InvariantStrictness::All + { + // Pick the first scalar we see, this means NonZeroU8(u8) ends up with only one + // invariant, the stricter one. + let _: Result<_, _> = invs.try_insert(InvariantKey { offset, size }, valid_range); + } + } + + //dbg!(&ty, &layout); + if !matches!(layout.layout.variants(), Variants::Single { .. }) { + // We *don't* want to look for fields inside enums. + return; + } + + let param_env = ParamEnv::reveal_all(); + let layout_cx = LayoutCx { tcx, param_env }; + + match layout.layout.fields() { + FieldsShape::Primitive => {} + FieldsShape::Union(_) => {} + FieldsShape::Array { stride, count } => { + // We may wish to bail out if we're generating too many invariants. + // That would lead to false negatives, though. + for idx in 0..*count { + let off = offset + *stride * idx; + let f = layout.field(&layout_cx, idx as usize); + add_invariants(tcx, f.ty, invs, off, strictness); + } + } + FieldsShape::Arbitrary { offsets, .. } => { + for (idx, &field_offset) in offsets.iter().enumerate() { + let f = layout.field(&layout_cx, idx); + if f.ty == ty { + // Some types contain themselves as fields, such as + // &mut [T] + // Easy solution is to just not recurse then. + } else { + add_invariants(tcx, f.ty, invs, offset + field_offset, strictness); + } + } + } + } + } +} + +fn get_layout_of_invariant<'tcx>(tcx: TyCtxt<'tcx>) -> TyAndLayout<'tcx, Ty<'tcx>> { + let item = tcx.require_lang_item(LangItem::ValidityInvariant, None); + let ty = tcx.type_of(item); + let layout = tcx + .layout_of(ParamEnv::reveal_all().and(ty)) + .expect("invalid layout for ValidityInvariant lang item"); + layout +} + +#[derive(PartialEq, Clone, Copy, Eq)] +pub(crate) enum InvariantStrictness { + Disable, + Normal, + All, +} + +/// Directly returns a `ConstAllocation` containing a list of validity invariants of the given type. +pub(crate) fn alloc_validity_invariants_of<'tcx>( + tcx: TyCtxt<'tcx>, + ty: Ty<'tcx>, + strictness: InvariantStrictness, +) -> (ConstAllocation<'tcx>, usize) { + let mut invs = FxHashMap::default(); + + let layout = tcx.data_layout(); + let validity_invariant = get_layout_of_invariant(tcx); + + add_invariants(tcx, ty, &mut invs, Size::ZERO, strictness); + + let allocation_size = validity_invariant.layout.size() * invs.len() as u64; + let mut alloc = + Allocation::uninit(allocation_size, validity_invariant.layout.align().abi, true).unwrap(); + + let offset_off = validity_invariant.layout.fields().offset(0); + let size_off = validity_invariant.layout.fields().offset(1); + let start_off = validity_invariant.layout.fields().offset(2); + let end_off = validity_invariant.layout.fields().offset(3); + + for (idx, invariant) in invs.iter().enumerate() { + let offset = idx as u64 * validity_invariant.layout.size(); + + let offset_range = AllocRange { start: offset + offset_off, size: layout.pointer_size }; + alloc + .write_scalar( + &tcx, + offset_range, + MirScalar::from_machine_usize(invariant.0.offset.bytes(), &tcx).into(), + ) + .unwrap(); + + let size_range = AllocRange { start: offset + size_off, size: Size::from_bytes(1) }; + alloc + .write_scalar(&tcx, size_range, MirScalar::from_u8(invariant.0.size as u8).into()) + .unwrap(); + + let offset_range = AllocRange { start: offset + start_off, size: Size::from_bytes(16) }; + alloc + .write_scalar(&tcx, offset_range, MirScalar::from_u128(invariant.1.start).into()) + .unwrap(); + + let offset_range = AllocRange { start: offset + end_off, size: Size::from_bytes(16) }; + alloc + .write_scalar(&tcx, offset_range, MirScalar::from_u128(invariant.1.end).into()) + .unwrap(); + } + + // The allocation is not mutable, we just needed write_scalar. + alloc.mutability = Mutability::Not; + + (tcx.intern_const_alloc(alloc), invs.len()) +} diff --git a/compiler/rustc_const_eval/src/interpret/operand.rs b/compiler/rustc_const_eval/src/interpret/operand.rs index 6b05a49575fd9..ebbff75f0a102 100644 --- a/compiler/rustc_const_eval/src/interpret/operand.rs +++ b/compiler/rustc_const_eval/src/interpret/operand.rs @@ -692,6 +692,19 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { self, )) } + ConstValue::CustomSlice { data, length } => { + // We rely on mutability being set correctly in `data` to prevent writes + // where none should happen. + let ptr = Pointer::new( + self.tcx.create_memory_alloc(data), + Size::ZERO, // offset: 0 + ); + Operand::Immediate(Immediate::new_slice( + Scalar::from_pointer(self.global_base_pointer(ptr)?, &*self.tcx), + u64::try_from(length).unwrap(), + self, + )) + } }; Ok(OpTy { op, layout }) } diff --git a/compiler/rustc_hir/src/lang_items.rs b/compiler/rustc_hir/src/lang_items.rs index 9e5d781892487..1f38b8219fe60 100644 --- a/compiler/rustc_hir/src/lang_items.rs +++ b/compiler/rustc_hir/src/lang_items.rs @@ -327,6 +327,7 @@ language_item_table! { Range, sym::Range, range_struct, Target::Struct, GenericRequirement::None; RangeToInclusive, sym::RangeToInclusive, range_to_inclusive_struct, Target::Struct, GenericRequirement::None; RangeTo, sym::RangeTo, range_to_struct, Target::Struct, GenericRequirement::None; + ValidityInvariant, sym::ValidityInvariant, validity_invariant_struct, Target::Struct, GenericRequirement::None; } pub enum GenericRequirement { diff --git a/compiler/rustc_middle/src/mir/interpret/value.rs b/compiler/rustc_middle/src/mir/interpret/value.rs index e80918d5e5d03..610abe4b187a1 100644 --- a/compiler/rustc_middle/src/mir/interpret/value.rs +++ b/compiler/rustc_middle/src/mir/interpret/value.rs @@ -37,6 +37,9 @@ pub enum ConstValue<'tcx> { /// Used only for `&[u8]` and `&str` Slice { data: ConstAllocation<'tcx>, start: usize, end: usize }, + /// Like `Slice`, but for types that aren't 1 byte long. + CustomSlice { data: ConstAllocation<'tcx>, length: usize }, + /// A value not represented/representable by `Scalar` or `Slice` ByRef { /// The backing memory of the value, may contain more memory than needed for just the value @@ -61,6 +64,9 @@ impl<'a, 'tcx> Lift<'tcx> for ConstValue<'a> { ConstValue::ByRef { alloc, offset } => { ConstValue::ByRef { alloc: tcx.lift(alloc)?, offset } } + ConstValue::CustomSlice { data, length } => { + ConstValue::CustomSlice { data: tcx.lift(data)?, length } + } }) } } @@ -69,7 +75,9 @@ impl<'tcx> ConstValue<'tcx> { #[inline] pub fn try_to_scalar(&self) -> Option> { match *self { - ConstValue::ByRef { .. } | ConstValue::Slice { .. } => None, + ConstValue::ByRef { .. } + | ConstValue::Slice { .. } + | ConstValue::CustomSlice { .. } => None, ConstValue::Scalar(val) => Some(val), } } @@ -258,6 +266,11 @@ impl Scalar { Scalar::Int(i.into()) } + #[inline] + pub fn from_u128(i: u128) -> Self { + Scalar::Int(i.into()) + } + #[inline] pub fn from_machine_usize(i: u64, cx: &impl HasDataLayout) -> Self { Self::from_uint(i, cx.data_layout().pointer_size) diff --git a/compiler/rustc_middle/src/mir/pretty.rs b/compiler/rustc_middle/src/mir/pretty.rs index 462c0ada3cf87..d44d2567cc388 100644 --- a/compiler/rustc_middle/src/mir/pretty.rs +++ b/compiler/rustc_middle/src/mir/pretty.rs @@ -451,6 +451,7 @@ impl<'tcx> Visitor<'tcx> for ExtraComments<'tcx> { let fmt_val = |val: &ConstValue<'tcx>| match val { ConstValue::Scalar(s) => format!("Scalar({:?})", s), ConstValue::Slice { .. } => format!("Slice(..)"), + ConstValue::CustomSlice { .. } => format!("CustomSlice(..)"), ConstValue::ByRef { .. } => format!("ByRef(..)"), }; @@ -679,7 +680,9 @@ pub fn write_allocations<'tcx>( ConstValue::Scalar(interpret::Scalar::Int { .. }) => { Either::Left(Either::Right(std::iter::empty())) } - ConstValue::ByRef { alloc, .. } | ConstValue::Slice { data: alloc, .. } => { + ConstValue::ByRef { alloc, .. } + | ConstValue::Slice { data: alloc, .. } + | ConstValue::CustomSlice { data: alloc, .. } => { Either::Right(alloc_ids_from_alloc(alloc)) } } diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs index 441e1f9f6a2b8..ccb591a552256 100644 --- a/compiler/rustc_session/src/options.rs +++ b/compiler/rustc_session/src/options.rs @@ -1380,6 +1380,8 @@ options! { "run LLVM in non-parallel mode (while keeping codegen-units and ThinLTO)"), no_unique_section_names: bool = (false, parse_bool, [TRACKED], "do not use unique names for text and data sections when -Z function-sections is used"), + no_validity_invariant_checks: bool = (false, parse_bool, [TRACKED], + "do not generate any validity invariants in the validity_invariants_of intrinsic"), no_profiler_runtime: bool = (false, parse_no_flag, [TRACKED], "prevent automatic injection of the profiler_builtins crate"), normalize_docs: bool = (false, parse_bool, [TRACKED], diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs index 6daf811e26f14..e0afbccff53e0 100644 --- a/compiler/rustc_span/src/symbol.rs +++ b/compiler/rustc_span/src/symbol.rs @@ -277,6 +277,7 @@ symbols! { TyKind, Unknown, UnsafeArg, + ValidityInvariant, Vec, VecDeque, Wrapper, @@ -1524,6 +1525,7 @@ symbols! { va_list, va_start, val, + validity_invariants_of, values, var, variant_count, diff --git a/compiler/rustc_typeck/src/check/intrinsic.rs b/compiler/rustc_typeck/src/check/intrinsic.rs index 7fe710cf8f4f2..3e73864d7a777 100644 --- a/compiler/rustc_typeck/src/check/intrinsic.rs +++ b/compiler/rustc_typeck/src/check/intrinsic.rs @@ -9,6 +9,7 @@ use crate::require_same_types; use rustc_errors::struct_span_err; use rustc_hir as hir; +use rustc_hir::lang_items::LangItem; use rustc_middle::traits::{ObligationCause, ObligationCauseCode}; use rustc_middle::ty::subst::Subst; use rustc_middle::ty::{self, TyCtxt}; @@ -102,7 +103,8 @@ pub fn intrinsic_operation_unsafety(intrinsic: Symbol) -> hir::Unsafety { | sym::type_name | sym::forget | sym::black_box - | sym::variant_count => hir::Unsafety::Normal, + | sym::variant_count + | sym::validity_invariants_of => hir::Unsafety::Normal, _ => hir::Unsafety::Unsafe, } } @@ -191,6 +193,12 @@ pub fn check_intrinsic_type(tcx: TyCtxt<'_>, it: &hir::ForeignItem<'_>) { sym::needs_drop => (1, Vec::new(), tcx.types.bool), sym::type_name => (1, Vec::new(), tcx.mk_static_str()), + sym::validity_invariants_of => { + let item = tcx.require_lang_item(LangItem::ValidityInvariant, None); + let ty = tcx.type_of(item); + let slice = tcx.mk_imm_ref(tcx.lifetimes.re_static, tcx.mk_slice(ty)); + (1, Vec::new(), slice) + } sym::type_id => (1, Vec::new(), tcx.types.u64), sym::offset | sym::arith_offset => ( 1, diff --git a/library/core/src/intrinsics.rs b/library/core/src/intrinsics.rs index ba837ea9a7898..01f3d419bfa7e 100644 --- a/library/core/src/intrinsics.rs +++ b/library/core/src/intrinsics.rs @@ -1976,6 +1976,13 @@ extern "rust-intrinsic" { /// [`std::hint::black_box`]: crate::hint::black_box #[rustc_const_unstable(feature = "const_black_box", issue = "none")] pub fn black_box(dummy: T) -> T; + + /// Returns a list of invaraints that must be valid in order for T to be valid. + /// + /// This is used internally to allow for runtime assertions inside `MaybeUninit::assume_init` + #[cfg(not(bootstrap))] + #[rustc_const_unstable(feature = "const_intrinsic_validity_invariants_of", issue = "none")] + pub fn validity_invariants_of() -> &'static [Invariant]; } // Some functions are defined here because they accidentally got made @@ -2138,6 +2145,40 @@ pub const unsafe fn copy_nonoverlapping(src: *const T, dst: *mut T, count: us } } +#[derive(Debug, Clone, Copy, PartialEq)] +#[repr(u8)] +#[allow(dead_code)] // https://github.com/rust-lang/rust/issues/85677 +pub enum InvariantSize { + U8 = 0, + U16 = 1, + U32 = 2, + U64 = 3, + U128 = 4, + Pointer = 5, +} + +/// An Invariant is a field and its valid range. The valid range may be full, in which case it +/// should still be branched on, to allow tools like memory sanitizer to verify the memory is not +/// uninit. +// Be sure to keep the fields in order (offset, size, start, end), validity_invariants_of.rs needs +// that. +#[cfg(not(bootstrap))] +#[derive(Debug, Clone, Copy, PartialEq)] +#[lang = "ValidityInvariant"] +pub struct Invariant { + /// The offset in bytes from the start of the struct + pub offset: usize, + /// The size/type of the invariant. + pub size: InvariantSize, + /// The start point of the valid range of this field. Is allowed to be > valid_range_end, see + /// + /// which this follows the semantics of. + pub valid_range_start: u128, + + /// The end point of the range. + pub valid_range_end: u128, +} + /// Copies `count * size_of::()` bytes from `src` to `dst`. The source /// and destination may overlap. /// @@ -2394,3 +2435,71 @@ where { called_in_const.call_once(arg) } + +#[cfg(bootstrap)] +pub const unsafe fn assert_validity_of(_: *const T) -> bool { + true +} + +#[cfg(not(bootstrap))] +/// Asserts that the value at `value` is a valid T. +/// +/// Best effort, can miss some UB, and is UB if the value is invalid. +pub unsafe fn assert_validity_of(value: *const T) -> bool { + // We have to do this, since we call assert_validity_of inside MaybeUninit::assume_init + // and if we had used ptr::read_unaligned, that would be a recursive call. + #[repr(packed)] + struct Unaligned(T); + + // SAFETY: The pointer dereferences here are valid if `value` is valid. + unsafe { + let invariants = validity_invariants_of::(); + for invariant in invariants { + let off = invariant.offset as usize; + let start = invariant.valid_range_start; + let end = invariant.valid_range_end; + + let (value, max): (u128, u128) = match invariant.size { + InvariantSize::U8 => ((*(value.cast::().add(off))).into(), u8::MAX.into()), + InvariantSize::U16 => ( + (*value.cast::().add(off).cast::>()).0.into(), + u16::MAX.into(), + ), + InvariantSize::U32 => ( + (*value.cast::().add(off).cast::>()).0.into(), + u32::MAX.into(), + ), + InvariantSize::U64 => ( + (*value.cast::().add(off).cast::>()).0.into(), + u64::MAX.into(), + ), + InvariantSize::U128 => ( + (*value.cast::().add(off).cast::>()).0.into(), + u128::MAX.into(), + ), + InvariantSize::Pointer => ( + (*value.cast::().add(off).cast::>()).0.addr() as u128, + usize::MAX as u128, + ), + }; + + if start > end { + if !((start..=max).contains(&value) || (0..=end).contains(&value)) { + panic!( + "read value {value} which was not in range 0..={end} or {start}..={max} at offset {off} in type {}", + core::any::type_name::() + ); + } + } else { + if !(start..=end).contains(&value) { + panic!( + "read value {value} which was not in range {start}..={end} at offset {off} in type {}", + core::any::type_name::() + ); + } + } + } + + true + } +} diff --git a/library/core/src/lib.rs b/library/core/src/lib.rs index 093c7d298734a..6e1d304e50cd2 100644 --- a/library/core/src/lib.rs +++ b/library/core/src/lib.rs @@ -156,7 +156,6 @@ #![feature(const_slice_from_ref)] #![feature(const_slice_index)] #![feature(const_is_char_boundary)] -// // Language features: #![feature(abi_unadjusted)] #![feature(allow_internal_unsafe)] diff --git a/library/core/src/mem/maybe_uninit.rs b/library/core/src/mem/maybe_uninit.rs index b4ea536083392..0ceb0b6f41c83 100644 --- a/library/core/src/mem/maybe_uninit.rs +++ b/library/core/src/mem/maybe_uninit.rs @@ -626,11 +626,17 @@ impl MaybeUninit { #[inline(always)] #[rustc_diagnostic_item = "assume_init"] #[track_caller] + #[rustc_allow_const_fn_unstable(const_refs_to_cell)] pub const unsafe fn assume_init(self) -> T { // SAFETY: the caller must guarantee that `self` is initialized. // This also means that `self` must be a `value` variant. unsafe { intrinsics::assert_inhabited::(); + + intrinsics::assert_unsafe_precondition!(intrinsics::assert_validity_of::( + &self as *const MaybeUninit as *const T + )); + ManuallyDrop::into_inner(self.value) } } @@ -795,6 +801,11 @@ impl MaybeUninit { // This also means that `self` must be a `value` variant. unsafe { intrinsics::assert_inhabited::(); + + intrinsics::assert_unsafe_precondition!(intrinsics::assert_validity_of::( + self as *const MaybeUninit as *const T + )); + &*self.as_ptr() } } @@ -912,6 +923,11 @@ impl MaybeUninit { // This also means that `self` must be a `value` variant. unsafe { intrinsics::assert_inhabited::(); + + intrinsics::assert_unsafe_precondition!(intrinsics::assert_validity_of::( + self as *const MaybeUninit as *const T + )); + &mut *self.as_mut_ptr() } } diff --git a/src/test/ui/intrinsics/validity_invariants_of.rs b/src/test/ui/intrinsics/validity_invariants_of.rs new file mode 100644 index 0000000000000..d5156053316ab --- /dev/null +++ b/src/test/ui/intrinsics/validity_invariants_of.rs @@ -0,0 +1,97 @@ +// run-pass +// revisions: disabled normal sanitized +// [disabled]compile-flags: -Zno-validity-invariant-checks +// [sanitized]compile-flags: -Z sanitizer=memory +#![feature(core_intrinsics, const_intrinsic_validity_invariants_of)] + +use std::mem::MaybeUninit; +use std::intrinsics::{Invariant, InvariantSize, validity_invariants_of, assert_validity_of}; +use std::num::NonZeroU16; + +#[repr(C)] +struct MyStruct { + a: NonZeroU16, + b: u8, + c: bool, +} + +const MY_STRUCT_INVARIANTS: &'static [Invariant] = validity_invariants_of::(); +const OPTION_INVARIANTS: &'static [Invariant] = validity_invariants_of::>(); + +fn main() { + let mut invs: Vec = MY_STRUCT_INVARIANTS.to_vec(); + + invs.sort_by_key(|x| x.offset); + + if cfg!(disabled) { + assert_eq!(&invs, &[]); + } else if cfg!(normal) { + assert_eq!(&invs, &[ + Invariant { + offset: 0, + size: InvariantSize::U16, + valid_range_start: 1, + valid_range_end: 65535 + }, + Invariant { + offset: 3, + size: InvariantSize::U8, + valid_range_start: 0, + valid_range_end: 1 + }, + ]); + } else { + assert!(cfg!(sanitized)); + + assert_eq!(&invs, &[ + Invariant { + offset: 0, + size: InvariantSize::U16, + valid_range_start: 1, + valid_range_end: 65535 + }, + Invariant { + offset: 2, + size: InvariantSize::U8, + valid_range_start: 0, + valid_range_end: 255 + }, + Invariant { + offset: 3, + size: InvariantSize::U8, + valid_range_start: 0, + valid_range_end: 1 + }, + ]); + } + + + unsafe { + let v = MyStruct { a: NonZeroU16::new(1).unwrap(), b: 2, c: true }; + assert!(assert_validity_of(&v as *const _)); + } + + if cfg!(sanitized) { + assert_eq!(OPTION_INVARIANTS, &[ + Invariant { + offset: 0, + size: InvariantSize::Pointer, + valid_range_start: 1, + valid_range_end: 0, + }, + ]); + } else { + assert_eq!(OPTION_INVARIANTS, &[]); + } + + + unsafe { + let p = MaybeUninit::>::zeroed().as_ptr(); + assert!(assert_validity_of(p)); + } + + // There's two code paths for generating the data, be sure to test that compile time and + // runtime matches. + assert_eq!(validity_invariants_of::>(), OPTION_INVARIANTS); + assert_eq!(validity_invariants_of::(), MY_STRUCT_INVARIANTS); +} diff --git a/src/test/ui/sanitize/memory.rs b/src/test/ui/sanitize/memory.rs index b53f19a5b01aa..2da871a62fcb9 100644 --- a/src/test/ui/sanitize/memory.rs +++ b/src/test/ui/sanitize/memory.rs @@ -1,7 +1,7 @@ // needs-sanitizer-support // needs-sanitizer-memory // -// compile-flags: -Z sanitizer=memory -Zsanitizer-memory-track-origins -O +// compile-flags: -Z sanitizer=memory -Zno-validity-invariant-checks -Zsanitizer-memory-track-origins -O // // run-fail // error-pattern: MemorySanitizer: use-of-uninitialized-value @@ -10,6 +10,8 @@ // // This test case intentionally limits the usage of the std, // since it will be linked with an uninstrumented version of it. +// +// -Zno-validity-invariant-checks is needed in order to not fail inside the assume_init #![feature(core_intrinsics)] #![feature(start)]