From f38015f470a5fe323cfc71f711f11677dcc80599 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Sat, 3 Aug 2024 11:39:59 -0400 Subject: [PATCH 1/3] codegen: NFC refactoring to use Align type --- src/ccall.cpp | 10 +++++----- src/cgutils.cpp | 41 ++++++++++++++++++++--------------------- src/codegen.cpp | 17 ++++++++++------- src/intrinsics.cpp | 12 ++++++------ 4 files changed, 41 insertions(+), 39 deletions(-) diff --git a/src/ccall.cpp b/src/ccall.cpp index 3c2857608c163..d913ed4e27e7d 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -554,8 +554,8 @@ static Value *julia_to_native( // pass the address of an alloca'd thing, not a box // since those are immutable. Value *slot = emit_static_alloca(ctx, to); - unsigned align = julia_alignment(jlto); - cast(slot)->setAlignment(Align(align)); + Align align(julia_alignment(jlto)); + cast(slot)->setAlignment(align); setName(ctx.emission_context, slot, "native_convert_buffer"); if (!jvinfo.ispointer()) { jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, jvinfo.tbaa); @@ -2230,7 +2230,7 @@ jl_cgval_t function_sig_t::emit_a_ccall( Value *strct = emit_allocobj(ctx, (jl_datatype_t*)rt, true); setName(ctx.emission_context, strct, "ccall_ret_box"); MDNode *tbaa = jl_is_mutable(rt) ? ctx.tbaa().tbaa_mutab : ctx.tbaa().tbaa_immut; - int boxalign = julia_alignment(rt); + Align boxalign(julia_alignment(rt)); // copy the data from the return value to the new struct const DataLayout &DL = ctx.builder.GetInsertBlock()->getModule()->getDataLayout(); auto resultTy = result->getType(); @@ -2240,8 +2240,8 @@ jl_cgval_t function_sig_t::emit_a_ccall( // When this happens, cast through memory. auto slot = emit_static_alloca(ctx, resultTy); setName(ctx.emission_context, slot, "type_pun_slot"); - slot->setAlignment(Align(boxalign)); - ctx.builder.CreateAlignedStore(result, slot, Align(boxalign)); + slot->setAlignment(boxalign); + ctx.builder.CreateAlignedStore(result, slot, boxalign); jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa); emit_memcpy(ctx, strct, ai, slot, ai, rtsz, boxalign, boxalign); } diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 613d7ae719448..297fcb164b112 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -315,7 +315,7 @@ static Value *emit_pointer_from_objref(jl_codectx_t &ctx, Value *V) } static Value *emit_unbox(jl_codectx_t &ctx, Type *to, const jl_cgval_t &x, jl_value_t *jt); -static void emit_unbox_store(jl_codectx_t &ctx, const jl_cgval_t &x, Value* dest, MDNode *tbaa_dest, unsigned alignment, bool isVolatile=false); +static void emit_unbox_store(jl_codectx_t &ctx, const jl_cgval_t &x, Value* dest, MDNode *tbaa_dest, Align alignment, bool isVolatile=false); static bool type_is_permalloc(jl_value_t *typ) { @@ -1006,11 +1006,10 @@ static Value *data_pointer(jl_codectx_t &ctx, const jl_cgval_t &x) } static void emit_memcpy_llvm(jl_codectx_t &ctx, Value *dst, jl_aliasinfo_t const &dst_ai, Value *src, - jl_aliasinfo_t const &src_ai, uint64_t sz, unsigned align_dst, unsigned align_src, bool is_volatile) + jl_aliasinfo_t const &src_ai, uint64_t sz, Align align_dst, Align align_src, bool is_volatile) { if (sz == 0) return; - assert(align_dst && "align must be specified"); #if JL_LLVM_VERSION < 170000 // If the types are small and simple, use load and store directly. // Going through memcpy can cause LLVM (e.g. SROA) to create bitcasts between float and int @@ -1053,7 +1052,7 @@ static void emit_memcpy_llvm(jl_codectx_t &ctx, Value *dst, jl_aliasinfo_t const if (isa(dst) && !dst->hasName()) setName(ctx.emission_context, dst, "memcpy_refined_dst"); auto val = src_ai.decorateInst(ctx.builder.CreateAlignedLoad(directel, src, MaybeAlign(align_src), is_volatile)); - dst_ai.decorateInst(ctx.builder.CreateAlignedStore(val, dst, Align(align_dst), is_volatile)); + dst_ai.decorateInst(ctx.builder.CreateAlignedStore(val, dst, align_dst, is_volatile)); ++SkippedMemcpys; return; } @@ -1072,12 +1071,12 @@ static void emit_memcpy_llvm(jl_codectx_t &ctx, Value *dst, jl_aliasinfo_t const // above problem won't be as serious. auto merged_ai = dst_ai.merge(src_ai); - ctx.builder.CreateMemCpy(dst, Align(align_dst), src, Align(align_src), sz, is_volatile, + ctx.builder.CreateMemCpy(dst, align_dst, src, align_src, sz, is_volatile, merged_ai.tbaa, merged_ai.tbaa_struct, merged_ai.scope, merged_ai.noalias); } static void emit_memcpy_llvm(jl_codectx_t &ctx, Value *dst, jl_aliasinfo_t const &dst_ai, Value *src, - jl_aliasinfo_t const &src_ai, Value *sz, unsigned align_dst, unsigned align_src, bool is_volatile) + jl_aliasinfo_t const &src_ai, Value *sz, Align align_dst, Align align_src, bool is_volatile) { if (auto const_sz = dyn_cast(sz)) { emit_memcpy_llvm(ctx, dst, dst_ai, src, src_ai, const_sz->getZExtValue(), align_dst, align_src, is_volatile); @@ -1086,20 +1085,20 @@ static void emit_memcpy_llvm(jl_codectx_t &ctx, Value *dst, jl_aliasinfo_t const ++EmittedMemcpys; auto merged_ai = dst_ai.merge(src_ai); - ctx.builder.CreateMemCpy(dst, MaybeAlign(align_dst), src, MaybeAlign(align_src), sz, is_volatile, + ctx.builder.CreateMemCpy(dst, align_dst, src, align_src, sz, is_volatile, merged_ai.tbaa, merged_ai.tbaa_struct, merged_ai.scope, merged_ai.noalias); } template static void emit_memcpy(jl_codectx_t &ctx, Value *dst, jl_aliasinfo_t const &dst_ai, Value *src, - jl_aliasinfo_t const &src_ai, T1 &&sz, unsigned align_dst, unsigned align_src, bool is_volatile=false) + jl_aliasinfo_t const &src_ai, T1 &&sz, Align align_dst, Align align_src, bool is_volatile=false) { emit_memcpy_llvm(ctx, dst, dst_ai, src, src_ai, sz, align_dst, align_src, is_volatile); } template static void emit_memcpy(jl_codectx_t &ctx, Value *dst, jl_aliasinfo_t const &dst_ai, const jl_cgval_t &src, - T1 &&sz, unsigned align_dst, unsigned align_src, bool is_volatile=false) + T1 &&sz, Align align_dst, Align align_src, bool is_volatile=false) { auto src_ai = jl_aliasinfo_t::fromTBAA(ctx, src.tbaa); emit_memcpy_llvm(ctx, dst, dst_ai, data_pointer(ctx, src), src_ai, sz, align_dst, align_src, is_volatile); @@ -1999,7 +1998,7 @@ static jl_cgval_t typed_load(jl_codectx_t &ctx, Value *ptr, Value *idx_0based, j else if (!alignment) alignment = julia_alignment(jltype); if (intcast && Order == AtomicOrdering::NotAtomic) { - emit_memcpy(ctx, intcast, jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_stack), data, jl_aliasinfo_t::fromTBAA(ctx, tbaa), nb, alignment, intcast->getAlign().value()); + emit_memcpy(ctx, intcast, jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_stack), data, jl_aliasinfo_t::fromTBAA(ctx, tbaa), nb, Align(alignment), intcast->getAlign()); } else { if (!isboxed && jl_is_genericmemoryref_type(jltype)) { @@ -2176,7 +2175,7 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, } else { assert(Order == AtomicOrdering::NotAtomic && !isboxed && rhs.typ == jltype); - emit_unbox_store(ctx, rhs, ptr, tbaa, alignment); + emit_unbox_store(ctx, rhs, ptr, tbaa, Align(alignment)); } } else if (isswapfield) { @@ -2329,7 +2328,7 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, } else { assert(!isboxed && rhs.typ == jltype); - emit_unbox_store(ctx, rhs, ptr, tbaa, alignment); + emit_unbox_store(ctx, rhs, ptr, tbaa, Align(alignment)); } ctx.builder.CreateBr(DoneBB); instr = load; @@ -2668,7 +2667,7 @@ static jl_cgval_t emit_unionload(jl_codectx_t &ctx, Value *addr, Value *ptindex, if (al > 1) lv->setAlignment(Align(al)); jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa); - emit_memcpy(ctx, lv, ai, addr, ai, fsz, al, al); + emit_memcpy(ctx, lv, ai, addr, ai, fsz, Align(al), Align(al)); addr = lv; } return mark_julia_slot(fsz > 0 ? addr : nullptr, jfty, tindex, tbaa); @@ -3104,11 +3103,11 @@ static Value *emit_genericmemoryowner(jl_codectx_t &ctx, Value *t) static Value *emit_allocobj(jl_codectx_t &ctx, jl_datatype_t *jt, bool fully_initialized); static void init_bits_value(jl_codectx_t &ctx, Value *newv, Value *v, MDNode *tbaa, - unsigned alignment = sizeof(void*)) // min alignment in julia's gc is pointer-aligned + Align alignment = Align(sizeof(void*))) // min alignment in julia's gc is pointer-aligned { jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa); // newv should already be tagged - ai.decorateInst(ctx.builder.CreateAlignedStore(v, newv, Align(alignment))); + ai.decorateInst(ctx.builder.CreateAlignedStore(v, newv, alignment)); } static void init_bits_cgval(jl_codectx_t &ctx, Value *newv, const jl_cgval_t& v, MDNode *tbaa) @@ -3116,7 +3115,7 @@ static void init_bits_cgval(jl_codectx_t &ctx, Value *newv, const jl_cgval_t& v, // newv should already be tagged if (v.ispointer()) { unsigned align = std::max(julia_alignment(v.typ), (unsigned)sizeof(void*)); - emit_memcpy(ctx, newv, jl_aliasinfo_t::fromTBAA(ctx, tbaa), v, jl_datatype_size(v.typ), align, julia_alignment(v.typ)); + emit_memcpy(ctx, newv, jl_aliasinfo_t::fromTBAA(ctx, tbaa), v, jl_datatype_size(v.typ), Align(align), Align(julia_alignment(v.typ))); } else { init_bits_value(ctx, newv, v.V, tbaa); @@ -3582,7 +3581,7 @@ static void emit_unionmove(jl_codectx_t &ctx, Value *dest, MDNode *tbaa_dst, con if (jl_is_pointerfree(typ)) { unsigned alignment = julia_alignment(typ); if (!src.ispointer() || src.constant) { - emit_unbox_store(ctx, src, dest, tbaa_dst, alignment, isVolatile); + emit_unbox_store(ctx, src, dest, tbaa_dst, Align(alignment), isVolatile); } else { Value *src_ptr = data_pointer(ctx, src); @@ -3592,7 +3591,7 @@ static void emit_unionmove(jl_codectx_t &ctx, Value *dest, MDNode *tbaa_dst, con // if (skip) src_ptr = ctx.builder.CreateSelect(skip, dest, src_ptr); auto f = [&] { (void)emit_memcpy(ctx, dest, jl_aliasinfo_t::fromTBAA(ctx, tbaa_dst), src_ptr, - jl_aliasinfo_t::fromTBAA(ctx, src.tbaa), nb, alignment, alignment, isVolatile); + jl_aliasinfo_t::fromTBAA(ctx, src.tbaa), nb, Align(alignment), Align(alignment), isVolatile); return nullptr; }; if (skip) @@ -3627,7 +3626,7 @@ static void emit_unionmove(jl_codectx_t &ctx, Value *dest, MDNode *tbaa_dst, con return; } else { emit_memcpy(ctx, dest, jl_aliasinfo_t::fromTBAA(ctx, tbaa_dst), src_ptr, - jl_aliasinfo_t::fromTBAA(ctx, src.tbaa), nb, alignment, alignment, isVolatile); + jl_aliasinfo_t::fromTBAA(ctx, src.tbaa), nb, Align(alignment), Align(alignment), isVolatile); } } ctx.builder.CreateBr(postBB); @@ -3653,7 +3652,7 @@ static void emit_unionmove(jl_codectx_t &ctx, Value *dest, MDNode *tbaa_dst, con Value *datatype = emit_typeof(ctx, src, false, false); Value *copy_bytes = emit_datatype_size(ctx, datatype); (void)emit_memcpy(ctx, dest, jl_aliasinfo_t::fromTBAA(ctx, tbaa_dst), data_pointer(ctx, src), - jl_aliasinfo_t::fromTBAA(ctx, src.tbaa), copy_bytes, 1, 1, isVolatile); + jl_aliasinfo_t::fromTBAA(ctx, src.tbaa), copy_bytes, Align(1), Align(1), isVolatile); return nullptr; }; if (skip) @@ -4046,7 +4045,7 @@ static jl_cgval_t emit_new_struct(jl_codectx_t &ctx, jl_value_t *ty, size_t narg } else if (init_as_value) { fval = emit_unbox(ctx, fty, fval_info, jtype); } else { - emit_unbox_store(ctx, fval_info, dest, ctx.tbaa().tbaa_stack, jl_field_align(sty, i)); + emit_unbox_store(ctx, fval_info, dest, ctx.tbaa().tbaa_stack, Align(jl_field_align(sty, i))); } } if (init_as_value) { diff --git a/src/codegen.cpp b/src/codegen.cpp index 875beb7c287dc..d8d7a1fd23814 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -5652,7 +5652,7 @@ static jl_cgval_t emit_varinfo(jl_codectx_t &ctx, jl_varinfo_t &vi, jl_sym_t *va else { const DataLayout &DL = jl_Module->getDataLayout(); uint64_t sz = DL.getTypeStoreSize(T); - emit_memcpy(ctx, ssaslot, jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_stack), vi.value, sz, ssaslot->getAlign().value(), varslot->getAlign().value()); + emit_memcpy(ctx, ssaslot, jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_stack), vi.value, sz, ssaslot->getAlign(), varslot->getAlign()); } Value *tindex = NULL; if (vi.pTIndex) @@ -5756,8 +5756,9 @@ static void emit_vi_assignment_unboxed(jl_codectx_t &ctx, jl_varinfo_t &vi, Valu // This check should probably mostly catch the relevant situations. if (vi.value.V != rval_info.V) { Value *copy_bytes = ConstantInt::get(getInt32Ty(ctx.builder.getContext()), jl_datatype_size(vi.value.typ)); + Align alignment(julia_alignment(rval_info.typ)); emit_memcpy(ctx, vi.value.V, jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_stack), rval_info, copy_bytes, - julia_alignment(rval_info.typ), julia_alignment(rval_info.typ), vi.isVolatile); + alignment, alignment, vi.isVolatile); } } else { @@ -6868,8 +6869,9 @@ static void emit_cfunc_invalidate( root1 = ctx.builder.CreateConstInBoundsGEP2_32(get_returnroots_type(ctx, return_roots), root1, 0, 0); ctx.builder.CreateStore(gf_ret, root1); } + Align alignment(julia_alignment(rettype)); emit_memcpy(ctx, &*gf_thunk->arg_begin(), jl_aliasinfo_t::fromTBAA(ctx, nullptr), gf_ret, - jl_aliasinfo_t::fromTBAA(ctx, nullptr), jl_datatype_size(rettype), julia_alignment(rettype), julia_alignment(rettype)); + jl_aliasinfo_t::fromTBAA(ctx, nullptr), jl_datatype_size(rettype), Align(alignment), Align(alignment)); ctx.builder.CreateRetVoid(); break; } @@ -8729,7 +8731,7 @@ static jl_llvm_functions_t jl_cgval_t closure_world = typed_load(ctx, worldaddr, NULL, (jl_value_t*)jl_long_type, nullptr, nullptr, false, AtomicOrdering::NotAtomic, false, ctx.types().alignof_ptr.value()); ctx.world_age_at_entry = closure_world.V; // The tls world in a OC is the world of the closure - emit_unbox_store(ctx, closure_world, world_age_field, ctx.tbaa().tbaa_gcframe, ctx.types().alignof_ptr.value()); + emit_unbox_store(ctx, closure_world, world_age_field, ctx.tbaa().tbaa_gcframe, ctx.types().alignof_ptr); // Load closure env Value *envaddr = ctx.builder.CreateInBoundsGEP( @@ -9272,8 +9274,9 @@ static jl_llvm_functions_t } if (returninfo.cc == jl_returninfo_t::SRet) { assert(jl_is_concrete_type(jlrettype)); + Align alignment(julia_alignment(jlrettype)); emit_memcpy(ctx, sret, jl_aliasinfo_t::fromTBAA(ctx, nullptr), retvalinfo, - jl_datatype_size(jlrettype), julia_alignment(jlrettype), julia_alignment(jlrettype)); + jl_datatype_size(jlrettype), alignment, alignment); } else { // must be jl_returninfo_t::Union emit_unionmove(ctx, sret, nullptr, retvalinfo, /*skip*/isboxed_union); @@ -9511,7 +9514,7 @@ static jl_llvm_functions_t // load of val) if the runtime type of val isn't phiType Value *isvalid = emit_isa_and_defined(ctx, val, phiType); emit_guarded_test(ctx, isvalid, nullptr, [&] { - emit_unbox_store(ctx, update_julia_type(ctx, val, phiType), dest, ctx.tbaa().tbaa_stack, julia_alignment(phiType)); + emit_unbox_store(ctx, update_julia_type(ctx, val, phiType), dest, ctx.tbaa().tbaa_stack, Align(julia_alignment(phiType))); return nullptr; }); } @@ -9538,7 +9541,7 @@ static jl_llvm_functions_t if (VN) V = Constant::getNullValue(ctx.types().T_prjlvalue); if (dest) - emit_unbox_store(ctx, val, dest, ctx.tbaa().tbaa_stack, julia_alignment(val.typ)); + emit_unbox_store(ctx, val, dest, ctx.tbaa().tbaa_stack, Align(julia_alignment(val.typ))); RTindex = ConstantInt::get(getInt8Ty(ctx.builder.getContext()), tindex); } } diff --git a/src/intrinsics.cpp b/src/intrinsics.cpp index ad89abf6ca1a2..23d272ea75e7f 100644 --- a/src/intrinsics.cpp +++ b/src/intrinsics.cpp @@ -405,7 +405,7 @@ static Value *emit_unboxed_coercion(jl_codectx_t &ctx, Type *to, Value *unboxed) } else if (!ty->isIntOrPtrTy() && !ty->isFloatingPointTy()) { assert(DL.getTypeSizeInBits(ty) == DL.getTypeSizeInBits(to)); - AllocaInst *cast = ctx.builder.CreateAlloca(ty); + AllocaInst *cast = emit_static_alloca(ctx, ty); setName(ctx.emission_context, cast, "coercion"); ctx.builder.CreateStore(unboxed, cast); unboxed = ctx.builder.CreateLoad(to, cast); @@ -498,7 +498,7 @@ static Value *emit_unbox(jl_codectx_t &ctx, Type *to, const jl_cgval_t &x, jl_va } // emit code to store a raw value into a destination -static void emit_unbox_store(jl_codectx_t &ctx, const jl_cgval_t &x, Value *dest, MDNode *tbaa_dest, unsigned alignment, bool isVolatile) +static void emit_unbox_store(jl_codectx_t &ctx, const jl_cgval_t &x, Value *dest, MDNode *tbaa_dest, Align alignment, bool isVolatile) { if (x.isghost) { // this can happen when a branch yielding a different type ends @@ -510,7 +510,7 @@ static void emit_unbox_store(jl_codectx_t &ctx, const jl_cgval_t &x, Value *dest if (!x.ispointer()) { // already unboxed, but sometimes need conversion (e.g. f32 -> i32) assert(x.V); Value *unboxed = zext_struct(ctx, x.V); - StoreInst *store = ctx.builder.CreateAlignedStore(unboxed, dest, Align(alignment)); + StoreInst *store = ctx.builder.CreateAlignedStore(unboxed, dest, alignment); store->setVolatile(isVolatile); jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa_dest); ai.decorateInst(store); @@ -518,7 +518,7 @@ static void emit_unbox_store(jl_codectx_t &ctx, const jl_cgval_t &x, Value *dest } Value *src = data_pointer(ctx, x); - emit_memcpy(ctx, dest, jl_aliasinfo_t::fromTBAA(ctx, tbaa_dest), src, jl_aliasinfo_t::fromTBAA(ctx, x.tbaa), jl_datatype_size(x.typ), alignment, julia_alignment(x.typ), isVolatile); + emit_memcpy(ctx, dest, jl_aliasinfo_t::fromTBAA(ctx, tbaa_dest), src, jl_aliasinfo_t::fromTBAA(ctx, x.tbaa), jl_datatype_size(x.typ), Align(alignment), Align(julia_alignment(x.typ)), isVolatile); } static jl_datatype_t *staticeval_bitstype(const jl_cgval_t &targ) @@ -770,7 +770,7 @@ static jl_cgval_t emit_pointerref(jl_codectx_t &ctx, ArrayRef argv) thePtr = ctx.builder.CreateInBoundsGEP(getInt8Ty(ctx.builder.getContext()), thePtr, im1); setName(ctx.emission_context, thePtr, "pointerref_src"); MDNode *tbaa = best_tbaa(ctx.tbaa(), ety); - emit_memcpy(ctx, strct, jl_aliasinfo_t::fromTBAA(ctx, tbaa), thePtr, jl_aliasinfo_t::fromTBAA(ctx, nullptr), size, sizeof(jl_value_t*), align_nb); + emit_memcpy(ctx, strct, jl_aliasinfo_t::fromTBAA(ctx, tbaa), thePtr, jl_aliasinfo_t::fromTBAA(ctx, nullptr), size, Align(sizeof(jl_value_t*)), Align(align_nb)); return mark_julia_type(ctx, strct, true, ety); } else { @@ -847,7 +847,7 @@ static jl_cgval_t emit_pointerset(jl_codectx_t &ctx, ArrayRef argv) setName(ctx.emission_context, im1, "pointerset_offset"); auto gep = ctx.builder.CreateInBoundsGEP(getInt8Ty(ctx.builder.getContext()), thePtr, im1); setName(ctx.emission_context, gep, "pointerset_ptr"); - emit_memcpy(ctx, gep, jl_aliasinfo_t::fromTBAA(ctx, nullptr), x, size, align_nb, julia_alignment(ety)); + emit_memcpy(ctx, gep, jl_aliasinfo_t::fromTBAA(ctx, nullptr), x, size, Align(align_nb), Align(julia_alignment(ety))); } else { bool isboxed; From e1e5a46b002a3979f6b217f1ecd7cede83bf45fd Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Sat, 3 Aug 2024 11:43:12 -0400 Subject: [PATCH 2/3] codegen: update type of x after type-assert Later code likes to see that the type is consistent with the cgval and the unbox. --- src/intrinsics.cpp | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/intrinsics.cpp b/src/intrinsics.cpp index 23d272ea75e7f..4bfe3f184d24b 100644 --- a/src/intrinsics.cpp +++ b/src/intrinsics.cpp @@ -799,7 +799,7 @@ static jl_cgval_t emit_runtime_pointerset(jl_codectx_t &ctx, ArrayRef argv) { const jl_cgval_t &e = argv[0]; - const jl_cgval_t &x = argv[1]; + jl_cgval_t x = argv[1]; const jl_cgval_t &i = argv[2]; const jl_cgval_t &align = argv[3]; @@ -822,6 +822,9 @@ static jl_cgval_t emit_pointerset(jl_codectx_t &ctx, ArrayRef argv) return jl_cgval_t(); } emit_typecheck(ctx, x, ety, "pointerset"); + x = update_julia_type(ctx, x, ety); + if (x.typ == jl_bottom_type) + return jl_cgval_t(); Value *idx = emit_unbox(ctx, ctx.types().T_size, i, (jl_value_t*)jl_long_type); Value *im1 = ctx.builder.CreateSub(idx, ConstantInt::get(ctx.types().T_size, 1)); @@ -992,7 +995,7 @@ static jl_cgval_t emit_atomic_pointerop(jl_codectx_t &ctx, intrinsic f, ArrayRef bool ismodifyfield = f == atomic_pointermodify; const jl_cgval_t undefval; const jl_cgval_t &e = argv[0]; - const jl_cgval_t &x = isreplacefield || ismodifyfield ? argv[2] : argv[1]; + jl_cgval_t x = isreplacefield || ismodifyfield ? argv[2] : argv[1]; const jl_cgval_t &y = isreplacefield || ismodifyfield ? argv[1] : undefval; const jl_cgval_t &ord = isreplacefield || ismodifyfield ? argv[3] : argv[2]; const jl_cgval_t &failord = isreplacefield ? argv[4] : undefval; @@ -1034,8 +1037,12 @@ static jl_cgval_t emit_atomic_pointerop(jl_codectx_t &ctx, intrinsic f, ArrayRef emit_error(ctx, msg); return jl_cgval_t(); } - if (!ismodifyfield) + if (!ismodifyfield) { emit_typecheck(ctx, x, ety, std::string(jl_intrinsic_name((int)f))); + x = update_julia_type(ctx, x, ety); + if (x.typ == jl_bottom_type) + return jl_cgval_t(); + } size_t nb = jl_datatype_size(ety); if ((nb & (nb - 1)) != 0 || nb > MAX_POINTERATOMIC_SIZE) { From 8bfef8f1b60155020abc9bdab8aef9788eb458ba Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 1 Aug 2024 17:11:07 +0000 Subject: [PATCH 3/3] codegen: take gc roots (and alloca alignment) more seriously Due to limitations in the LLVM implementation, we are forced to emit fairly bad code here. But we need to make sure it is still correct with respect to GC rooting. The PR 50833c84d454ef989797e035294ba27b3cca79b7 also changed the meaning of haspadding without changing all of the existing users to use the new equivalent flag (haspadding || !isbitsegal), incurring additional breakage here as well and needing more tests for that. Fixes #54720 --- src/abi_aarch64.cpp | 8 ++-- src/abi_arm.cpp | 2 +- src/abi_ppc64le.cpp | 2 +- src/cgutils.cpp | 97 ++++++++++++++++++++++++++++----------------- src/codegen.cpp | 3 ++ src/datatype.c | 14 +++---- test/atomics.jl | 6 +++ 7 files changed, 83 insertions(+), 49 deletions(-) diff --git a/src/abi_aarch64.cpp b/src/abi_aarch64.cpp index 7c31b6606139a..0a193ee132556 100644 --- a/src/abi_aarch64.cpp +++ b/src/abi_aarch64.cpp @@ -16,7 +16,7 @@ struct ABI_AArch64Layout : AbiLayout { Type *get_llvm_vectype(jl_datatype_t *dt, LLVMContext &ctx) const { // Assume jl_is_datatype(dt) && !jl_is_abstracttype(dt) - // `!dt->name->mutabl && dt->pointerfree && !dt->haspadding && dt->nfields > 0` + // `!dt->name->mutabl && dt->pointerfree && !dt->haspadding && dt->isbitsegal && dt->nfields > 0` if (dt->layout == NULL || jl_is_layout_opaque(dt->layout)) return nullptr; size_t nfields = dt->layout->nfields; @@ -62,7 +62,7 @@ Type *get_llvm_vectype(jl_datatype_t *dt, LLVMContext &ctx) const Type *get_llvm_fptype(jl_datatype_t *dt, LLVMContext &ctx) const { // Assume jl_is_datatype(dt) && !jl_is_abstracttype(dt) - // `!dt->name->mutabl && dt->pointerfree && !dt->haspadding && dt->nfields == 0` + // `!dt->name->mutabl && dt->pointerfree && !dt->haspadding && dt->isbitsegal && dt->nfields == 0` Type *lltype; // Check size first since it's cheaper. switch (jl_datatype_size(dt)) { @@ -88,7 +88,7 @@ Type *get_llvm_fptype(jl_datatype_t *dt, LLVMContext &ctx) const Type *get_llvm_fp_or_vectype(jl_datatype_t *dt, LLVMContext &ctx) const { // Assume jl_is_datatype(dt) && !jl_is_abstracttype(dt) - if (dt->name->mutabl || dt->layout->npointers || dt->layout->flags.haspadding) + if (dt->name->mutabl || dt->layout->npointers || !dt->layout->flags.isbitsegal || dt->layout->flags.haspadding) return nullptr; return dt->layout->nfields ? get_llvm_vectype(dt, ctx) : get_llvm_fptype(dt, ctx); } @@ -184,7 +184,7 @@ Type *isHFAorHVA(jl_datatype_t *dt, size_t &nele, LLVMContext &ctx) const // uniquely addressable members. // Maximum HFA and HVA size is 64 bytes (4 x fp128 or 16bytes vector) size_t dsz = jl_datatype_size(dt); - if (dsz > 64 || !dt->layout || dt->layout->npointers || dt->layout->flags.haspadding) + if (dsz > 64 || !dt->layout || dt->layout->npointers || !dt->layout->flags.isbitsegal || dt->layout->flags.haspadding) return NULL; nele = 0; ElementType eltype; diff --git a/src/abi_arm.cpp b/src/abi_arm.cpp index 68f980d7b40da..8839a37da6e13 100644 --- a/src/abi_arm.cpp +++ b/src/abi_arm.cpp @@ -82,7 +82,7 @@ size_t isLegalHA(jl_datatype_t *dt, Type *&base, LLVMContext &ctx) const if (jl_is_structtype(dt)) { // Fast path checks before descending the type hierarchy // (4 x 128b vector == 64B max size) - if (jl_datatype_size(dt) > 64 || dt->layout->npointers || dt->layout->flags.haspadding) + if (jl_datatype_size(dt) > 64 || dt->layout->npointers || !dt->layout->flags.isbitsegal || dt->layout->flags.haspadding) return 0; base = NULL; diff --git a/src/abi_ppc64le.cpp b/src/abi_ppc64le.cpp index 1f10817cfeeee..f02e1022ddc2d 100644 --- a/src/abi_ppc64le.cpp +++ b/src/abi_ppc64le.cpp @@ -44,7 +44,7 @@ struct ABI_PPC64leLayout : AbiLayout { // count the homogeneous floating aggregate size (saturating at max count of 8) unsigned isHFA(jl_datatype_t *ty, jl_datatype_t **ty0, bool *hva) const { - if (jl_datatype_size(ty) > 128 || ty->layout->npointers || ty->layout->flags.haspadding) + if (jl_datatype_size(ty) > 128 || ty->layout->npointers || !ty->layout->flags.isbitsegal || ty->layout->flags.haspadding) return 9; size_t i, l = ty->layout->nfields; diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 297fcb164b112..08d51f52b613b 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -2119,12 +2119,15 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, FailOrder = AtomicOrdering::Monotonic; unsigned nb = isboxed ? sizeof(void*) : jl_datatype_size(jltype); AllocaInst *intcast = nullptr; + Type *intcast_eltyp = nullptr; + bool tracked_pointers = isboxed || CountTrackedPointers(elty).count > 0; if (!isboxed && Order != AtomicOrdering::NotAtomic && !elty->isIntOrPtrTy()) { + intcast_eltyp = elty; + elty = Type::getIntNTy(ctx.builder.getContext(), 8 * nb); if (!issetfield) { intcast = emit_static_alloca(ctx, elty); setName(ctx.emission_context, intcast, "atomic_store_box"); } - elty = Type::getIntNTy(ctx.builder.getContext(), 8 * nb); } Type *realelty = elty; if (Order != AtomicOrdering::NotAtomic && isa(elty)) { @@ -2133,14 +2136,20 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, elty = Type::getIntNTy(ctx.builder.getContext(), 8 * nb2); } Value *r = nullptr; - if (issetfield || isswapfield || isreplacefield || issetfieldonce) { - if (isboxed) + if (issetfield || isswapfield || isreplacefield || issetfieldonce) { // e.g. !ismodifyfield + assert(isboxed || rhs.typ == jltype); + if (isboxed) { r = boxed(ctx, rhs); - else if (aliasscope || Order != AtomicOrdering::NotAtomic || CountTrackedPointers(realelty).count) { + } + else if (intcast) { + emit_unbox_store(ctx, rhs, intcast, ctx.tbaa().tbaa_stack, intcast->getAlign()); + r = ctx.builder.CreateLoad(realelty, intcast); + } + else if (aliasscope || Order != AtomicOrdering::NotAtomic || tracked_pointers) { r = emit_unbox(ctx, realelty, rhs, jltype); - if (realelty != elty) - r = ctx.builder.CreateZExt(r, elty); } + if (realelty != elty) + r = ctx.builder.CreateZExt(r, elty); } if (isboxed) alignment = sizeof(void*); @@ -2222,7 +2231,14 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, Current->addIncoming(instr, SkipBB); ctx.builder.SetInsertPoint(BB); } - Compare = emit_unbox(ctx, realelty, cmp, jltype); + cmp = update_julia_type(ctx, cmp, jltype); + if (intcast) { + emit_unbox_store(ctx, cmp, intcast, ctx.tbaa().tbaa_stack, intcast->getAlign()); + Compare = ctx.builder.CreateLoad(realelty, intcast); + } + else { + Compare = emit_unbox(ctx, realelty, cmp, jltype); + } if (realelty != elty) Compare = ctx.builder.CreateZExt(Compare, elty); } @@ -2269,16 +2285,17 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, if (realelty != elty) realCompare = ctx.builder.CreateTrunc(realCompare, realelty); if (intcast) { + assert(!isboxed); ctx.builder.CreateStore(realCompare, intcast); - if (maybe_null_if_boxed) - realCompare = ctx.builder.CreateLoad(intcast->getAllocatedType(), intcast); + if (tracked_pointers) + realCompare = ctx.builder.CreateLoad(intcast_eltyp, intcast); } - if (maybe_null_if_boxed) { - Value *first_ptr = isboxed ? Compare : extract_first_ptr(ctx, Compare); - if (first_ptr) - null_load_check(ctx, first_ptr, mod, var); + if (maybe_null_if_boxed && tracked_pointers) { + Value *first_ptr = isboxed ? realCompare : extract_first_ptr(ctx, realCompare); + assert(first_ptr); + null_load_check(ctx, first_ptr, mod, var); } - if (intcast) + if (intcast && !tracked_pointers) oldval = mark_julia_slot(intcast, jltype, NULL, ctx.tbaa().tbaa_stack); else oldval = mark_julia_type(ctx, realCompare, isboxed, jltype); @@ -2286,11 +2303,18 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, if (isboxed) { r = boxed(ctx, rhs); } - else if (Order != AtomicOrdering::NotAtomic || CountTrackedPointers(realelty).count) { + else if (intcast) { + emit_unbox_store(ctx, rhs, intcast, ctx.tbaa().tbaa_stack, intcast->getAlign()); + r = ctx.builder.CreateLoad(realelty, intcast); + if (!tracked_pointers) // oldval is a slot, so put the oldval back + ctx.builder.CreateStore(realCompare, intcast); + } + else if (Order != AtomicOrdering::NotAtomic) { + assert(!tracked_pointers); r = emit_unbox(ctx, realelty, rhs, jltype); - if (realelty != elty) - r = ctx.builder.CreateZExt(r, elty); } + if (realelty != elty) + r = ctx.builder.CreateZExt(r, elty); if (needlock) emit_lockstate_value(ctx, needlock, true); cmp = oldval; @@ -2356,9 +2380,10 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, realinstr = ctx.builder.CreateTrunc(realinstr, realelty); if (intcast) { ctx.builder.CreateStore(realinstr, intcast); + // n.b. this oldval is only used for emit_f_is in this branch, so we know a priori that it does not need a gc-root oldval = mark_julia_slot(intcast, jltype, NULL, ctx.tbaa().tbaa_stack); if (maybe_null_if_boxed) - realinstr = ctx.builder.CreateLoad(intcast->getAllocatedType(), intcast); + realinstr = ctx.builder.CreateLoad(intcast_eltyp, intcast); } else { oldval = mark_julia_type(ctx, realinstr, isboxed, jltype); @@ -2398,20 +2423,23 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, ctx.builder.SetInsertPoint(DoneBB); if (needlock) emit_lockstate_value(ctx, needlock, false); - if (parent != NULL) { + if (parent != NULL && r && tracked_pointers && (!isboxed || !type_is_permalloc(rhs.typ))) { if (isreplacefield || issetfieldonce) { - // TODO: avoid this branch if we aren't making a write barrier BasicBlock *BB = BasicBlock::Create(ctx.builder.getContext(), "xchg_wb", ctx.f); DoneBB = BasicBlock::Create(ctx.builder.getContext(), "done_xchg_wb", ctx.f); ctx.builder.CreateCondBr(Success, BB, DoneBB); ctx.builder.SetInsertPoint(BB); } - if (r) { - if (!isboxed) - emit_write_multibarrier(ctx, parent, r, rhs.typ); - else if (!type_is_permalloc(rhs.typ)) - emit_write_barrier(ctx, parent, r); + if (realelty != elty) + r = ctx.builder.Insert(CastInst::Create(Instruction::Trunc, r, realelty)); + if (intcast) { + ctx.builder.CreateStore(r, intcast); + r = ctx.builder.CreateLoad(intcast_eltyp, intcast); } + if (!isboxed) + emit_write_multibarrier(ctx, parent, r, rhs.typ); + else if (!type_is_permalloc(rhs.typ)) + emit_write_barrier(ctx, parent, r); if (isreplacefield || issetfieldonce) { ctx.builder.CreateBr(DoneBB); ctx.builder.SetInsertPoint(DoneBB); @@ -2430,21 +2458,18 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, instr = ctx.builder.Insert(CastInst::Create(Instruction::Trunc, instr, realelty)); if (intcast) { ctx.builder.CreateStore(instr, intcast); - instr = nullptr; + if (tracked_pointers) + instr = ctx.builder.CreateLoad(intcast_eltyp, intcast); } - if (maybe_null_if_boxed) { - if (intcast) - instr = ctx.builder.CreateLoad(intcast->getAllocatedType(), intcast); + if (maybe_null_if_boxed && tracked_pointers) { Value *first_ptr = isboxed ? instr : extract_first_ptr(ctx, instr); - if (first_ptr) - null_load_check(ctx, first_ptr, mod, var); - if (intcast && !first_ptr) - instr = nullptr; + assert(first_ptr); + null_load_check(ctx, first_ptr, mod, var); } - if (instr) - oldval = mark_julia_type(ctx, instr, isboxed, jltype); - else + if (intcast && !tracked_pointers) oldval = mark_julia_slot(intcast, jltype, NULL, ctx.tbaa().tbaa_stack); + else + oldval = mark_julia_type(ctx, instr, isboxed, jltype); if (isreplacefield) { Success = ctx.builder.CreateZExt(Success, getInt8Ty(ctx.builder.getContext())); const jl_cgval_t argv[2] = {oldval, mark_julia_type(ctx, Success, false, jl_bool_type)}; diff --git a/src/codegen.cpp b/src/codegen.cpp index d8d7a1fd23814..ef18d9bd1b673 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -3374,11 +3374,14 @@ static size_t emit_masked_bits_compare(callback &emit_desc, jl_datatype_t *aty, size_t padding_bytes = 0; size_t nfields = jl_datatype_nfields(aty); size_t total_size = jl_datatype_size(aty); + assert(aty->layout->flags.isbitsegal); for (size_t i = 0; i < nfields; ++i) { size_t offset = jl_field_offset(aty, i); size_t fend = i == nfields - 1 ? total_size : jl_field_offset(aty, i + 1); size_t fsz = jl_field_size(aty, i); jl_datatype_t *fty = (jl_datatype_t*)jl_field_type(aty, i); + assert(jl_is_datatype(fty)); // union fields should never reach here + assert(fty->layout->flags.isbitsegal); if (jl_field_isptr(aty, i) || !fty->layout->flags.haspadding) { // The field has no internal padding data_bytes += fsz; diff --git a/src/datatype.c b/src/datatype.c index e7ee15a63f56e..e88977dd67400 100644 --- a/src/datatype.c +++ b/src/datatype.c @@ -1250,7 +1250,7 @@ JL_DLLEXPORT int jl_atomic_cmpswap_bits(jl_datatype_t *dt, jl_value_t *y /* pre- } else if (nb == 1) { uint8_t *y8 = (uint8_t*)y; - assert(!dt->layout->flags.haspadding); + assert(dt->layout->flags.isbitsegal && !dt->layout->flags.haspadding); if (dt == et) { *y8 = *(uint8_t*)expected; uint8_t z8 = *(uint8_t*)src; @@ -1263,7 +1263,7 @@ JL_DLLEXPORT int jl_atomic_cmpswap_bits(jl_datatype_t *dt, jl_value_t *y /* pre- } else if (nb == 2) { uint16_t *y16 = (uint16_t*)y; - assert(!dt->layout->flags.haspadding); + assert(dt->layout->flags.isbitsegal && !dt->layout->flags.haspadding); if (dt == et) { *y16 = *(uint16_t*)expected; uint16_t z16 = *(uint16_t*)src; @@ -1281,7 +1281,7 @@ JL_DLLEXPORT int jl_atomic_cmpswap_bits(jl_datatype_t *dt, jl_value_t *y /* pre- uint32_t z32 = zext_read32(src, nb); while (1) { success = jl_atomic_cmpswap((_Atomic(uint32_t)*)dst, y32, z32); - if (success || !dt->layout->flags.haspadding || !jl_egal__bits(y, expected, dt)) + if (success || (dt->layout->flags.isbitsegal && !dt->layout->flags.haspadding) || !jl_egal__bits(y, expected, dt)) break; } } @@ -1298,7 +1298,7 @@ JL_DLLEXPORT int jl_atomic_cmpswap_bits(jl_datatype_t *dt, jl_value_t *y /* pre- uint64_t z64 = zext_read64(src, nb); while (1) { success = jl_atomic_cmpswap((_Atomic(uint64_t)*)dst, y64, z64); - if (success || !dt->layout->flags.haspadding || !jl_egal__bits(y, expected, dt)) + if (success || (dt->layout->flags.isbitsegal && !dt->layout->flags.haspadding) || !jl_egal__bits(y, expected, dt)) break; } } @@ -1316,7 +1316,7 @@ JL_DLLEXPORT int jl_atomic_cmpswap_bits(jl_datatype_t *dt, jl_value_t *y /* pre- jl_uint128_t z128 = zext_read128(src, nb); while (1) { success = jl_atomic_cmpswap((_Atomic(jl_uint128_t)*)dst, y128, z128); - if (success || !dt->layout->flags.haspadding || !jl_egal__bits(y, expected, dt)) + if (success || (dt->layout->flags.isbitsegal && !dt->layout->flags.haspadding) || !jl_egal__bits(y, expected, dt)) break; } } @@ -2010,7 +2010,7 @@ inline jl_value_t *modify_bits(jl_value_t *ty, char *p, uint8_t *psel, jl_value_ else { char *px = lock(p, parent, needlock, isatomic); int success = memcmp(px, (char*)r, fsz) == 0; - if (!success && ((jl_datatype_t*)rty)->layout->flags.haspadding) + if (!success && (!((jl_datatype_t*)rty)->layout->flags.isbitsegal || ((jl_datatype_t*)rty)->layout->flags.haspadding)) success = jl_egal__bits((jl_value_t*)px, r, (jl_datatype_t*)rty); if (success) { if (isunion) { @@ -2125,7 +2125,7 @@ inline jl_value_t *replace_bits(jl_value_t *ty, char *p, uint8_t *psel, jl_value success = (rty == jl_typeof(expected)); if (success) { success = memcmp((char*)r, (char*)expected, rsz) == 0; - if (!success && ((jl_datatype_t*)rty)->layout->flags.haspadding) + if (!success && (!((jl_datatype_t*)rty)->layout->flags.isbitsegal || ((jl_datatype_t*)rty)->layout->flags.haspadding)) success = jl_egal__bits(r, expected, (jl_datatype_t*)rty); } *((uint8_t*)r + fsz) = success ? 1 : 0; diff --git a/test/atomics.jl b/test/atomics.jl index 3df9e7d0f63c0..165c8dc4e2dfc 100644 --- a/test/atomics.jl +++ b/test/atomics.jl @@ -129,6 +129,7 @@ test_field_operators(ARefxy{Any}(123_10, 123_20)) test_field_operators(ARefxy{Union{Nothing,Int}}(123_10, nothing)) test_field_operators(ARefxy{Complex{Int32}}(123_10, 123_20)) test_field_operators(ARefxy{Complex{Int128}}(123_10, 123_20)) +test_field_operators(ARefxy{Complex{Real}}(123_10, 123_20)) test_field_operators(ARefxy{PadIntA}(123_10, 123_20)) test_field_operators(ARefxy{PadIntB}(123_10, 123_20)) #FIXME: test_field_operators(ARefxy{Int24}(123_10, 123_20)) @@ -317,6 +318,7 @@ test_field_orderings(ARefxy{Any}(true, false), true, false) test_field_orderings(ARefxy{Union{Nothing,Missing}}(nothing, missing), nothing, missing) test_field_orderings(ARefxy{Union{Nothing,Int}}(nothing, 123_1), nothing, 123_1) test_field_orderings(Complex{Int128}(10, 30), Complex{Int128}(20, 40)) +test_field_orderings(Complex{Real}(10, 30), Complex{Real}(20, 40)) test_field_orderings(10.0, 20.0) test_field_orderings(NaN, Inf) @@ -568,6 +570,7 @@ test_global_operators(Any) test_global_operators(Union{Nothing,Int}) test_global_operators(Complex{Int32}) test_global_operators(Complex{Int128}) +test_global_operators(Complex{Real}) test_global_operators(PadIntA) test_global_operators(PadIntB) #FIXME: test_global_operators(Int24) @@ -691,6 +694,7 @@ test_global_orderings(Any, true, false) test_global_orderings(Union{Nothing,Missing}, nothing, missing) test_global_orderings(Union{Nothing,Int}, nothing, 123_1) test_global_orderings(Complex{Int128}, Complex{Int128}(10, 30), Complex{Int128}(20, 40)) +test_global_orderings(Complex{Real}, Complex{Real}(10, 30), Complex{Real}(20, 40)) test_global_orderings(Float64, 10.0, 20.0) test_global_orderings(Float64, NaN, Inf) @@ -844,6 +848,7 @@ test_memory_operators(Any) test_memory_operators(Union{Nothing,Int}) test_memory_operators(Complex{Int32}) test_memory_operators(Complex{Int128}) +test_memory_operators(Complex{Real}) test_memory_operators(PadIntA) test_memory_operators(PadIntB) #FIXME: test_memory_operators(Int24) @@ -1031,6 +1036,7 @@ test_memory_orderings(Any, true, false) test_memory_orderings(Union{Nothing,Missing}, nothing, missing) test_memory_orderings(Union{Nothing,Int}, nothing, 123_1) test_memory_orderings(Complex{Int128}(10, 30), Complex{Int128}(20, 40)) +test_memory_orderings(Complex{Real}(10, 30), Complex{Real}(20, 40)) test_memory_orderings(10.0, 20.0) test_memory_orderings(NaN, Inf)