diff --git a/base/atomics.jl b/base/atomics.jl index dde981b8c0e6a..e6f3a5654cbf7 100644 --- a/base/atomics.jl +++ b/base/atomics.jl @@ -364,13 +364,13 @@ for typ in atomictypes irt = "$ilt, $ilt*" @eval getindex(x::Atomic{$typ}) = GC.@preserve x llvmcall($""" - %ptr = inttoptr i$WORD_SIZE %0 to $lt* + %ptr = bitcast i8* %0 to $lt* %rv = load atomic $rt %ptr acquire, align $(gc_alignment(typ)) ret $lt %rv """, $typ, Tuple{Ptr{$typ}}, unsafe_convert(Ptr{$typ}, x)) @eval setindex!(x::Atomic{$typ}, v::$typ) = GC.@preserve x llvmcall($""" - %ptr = inttoptr i$WORD_SIZE %0 to $lt* + %ptr = bitcast i8* %0 to $lt* store atomic $lt %1, $lt* %ptr release, align $(gc_alignment(typ)) ret void """, Cvoid, Tuple{Ptr{$typ}, $typ}, unsafe_convert(Ptr{$typ}, x), v) @@ -379,7 +379,7 @@ for typ in atomictypes if typ <: Integer @eval atomic_cas!(x::Atomic{$typ}, cmp::$typ, new::$typ) = GC.@preserve x llvmcall($""" - %ptr = inttoptr i$WORD_SIZE %0 to $lt* + %ptr = bitcast i8* %0 to $lt* %rs = cmpxchg $lt* %ptr, $lt %1, $lt %2 acq_rel acquire %rv = extractvalue { $lt, i1 } %rs, 0 ret $lt %rv @@ -388,7 +388,7 @@ for typ in atomictypes else @eval atomic_cas!(x::Atomic{$typ}, cmp::$typ, new::$typ) = GC.@preserve x llvmcall($""" - %iptr = inttoptr i$WORD_SIZE %0 to $ilt* + %iptr = bitcast i8* %0 to $ilt* %icmp = bitcast $lt %1 to $ilt %inew = bitcast $lt %2 to $ilt %irs = cmpxchg $ilt* %iptr, $ilt %icmp, $ilt %inew acq_rel acquire @@ -411,7 +411,7 @@ for typ in atomictypes if typ <: Integer @eval $fn(x::Atomic{$typ}, v::$typ) = GC.@preserve x llvmcall($""" - %ptr = inttoptr i$WORD_SIZE %0 to $lt* + %ptr = bitcast i8* %0 to $lt* %rv = atomicrmw $rmw $lt* %ptr, $lt %1 acq_rel ret $lt %rv """, $typ, Tuple{Ptr{$typ}, $typ}, unsafe_convert(Ptr{$typ}, x), v) @@ -419,7 +419,7 @@ for typ in atomictypes rmwop === :xchg || continue @eval $fn(x::Atomic{$typ}, v::$typ) = GC.@preserve x llvmcall($""" - %iptr = inttoptr i$WORD_SIZE %0 to $ilt* + %iptr = bitcast i8* %0 to $ilt* %ival = bitcast $lt %1 to $ilt %irv = atomicrmw $rmw $ilt* %iptr, $ilt %ival acq_rel %rv = bitcast $ilt %irv to $lt diff --git a/base/pointer.jl b/base/pointer.jl index 2f67c8a6757cb..494e87996355c 100644 --- a/base/pointer.jl +++ b/base/pointer.jl @@ -313,8 +313,8 @@ isless(x::Ptr{T}, y::Ptr{T}) where {T} = x < y <(x::Ptr, y::Ptr) = UInt(x) < UInt(y) -(x::Ptr, y::Ptr) = UInt(x) - UInt(y) -+(x::Ptr, y::Integer) = oftype(x, add_ptr(UInt(x), (y % UInt) % UInt)) --(x::Ptr, y::Integer) = oftype(x, sub_ptr(UInt(x), (y % UInt) % UInt)) ++(x::Ptr, y::Integer) = add_ptr(x, (y % UInt) % UInt) +-(x::Ptr, y::Integer) = sub_ptr(x, (y % UInt) % UInt) +(x::Integer, y::Ptr) = y + x unsigned(x::Ptr) = UInt(x) diff --git a/base/task.jl b/base/task.jl index ba96d7bca5095..ecc0944352491 100644 --- a/base/task.jl +++ b/base/task.jl @@ -154,8 +154,7 @@ const _state_index = findfirst(==(:_state), fieldnames(Task)) @eval function load_state_acquire(t) # TODO: Replace this by proper atomic operations when available @GC.preserve t llvmcall($(""" - %ptr = inttoptr i$(Sys.WORD_SIZE) %0 to i8* - %rv = load atomic i8, i8* %ptr acquire, align 8 + %rv = load atomic i8, i8* %0 acquire, align 8 ret i8 %rv """), UInt8, Tuple{Ptr{UInt8}}, Ptr{UInt8}(pointer_from_objref(t) + fieldoffset(Task, _state_index))) diff --git a/src/ccall.cpp b/src/ccall.cpp index 0fcc838905004..e35b8e8baa280 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -616,7 +616,7 @@ static void interpret_symbol_arg(jl_codectx_t &ctx, native_sym_arg_t &out, jl_va emit_cpointercheck(ctx, arg1, errmsg); } arg1 = update_julia_type(ctx, arg1, (jl_value_t*)jl_voidpointer_type); - jl_ptr = emit_unbox(ctx, ctx.types().T_size, arg1, (jl_value_t*)jl_voidpointer_type); + jl_ptr = emit_unbox(ctx, ctx.types().T_ptr, arg1, (jl_value_t*)jl_voidpointer_type); } else { out.gcroot = ptr; @@ -696,7 +696,7 @@ static jl_cgval_t emit_cglobal(jl_codectx_t &ctx, jl_value_t **args, size_t narg else { rt = (jl_value_t*)jl_voidpointer_type; } - Type *lrt = ctx.types().T_size; + Type *lrt = ctx.types().T_ptr; assert(lrt == julia_type_to_llvm(ctx, rt)); interpret_symbol_arg(ctx, sym, args[1], /*ccall=*/false, false); @@ -723,7 +723,6 @@ static jl_cgval_t emit_cglobal(jl_codectx_t &ctx, jl_value_t **args, size_t narg } else /*if (ctx.emission_context.imaging) */{ res = runtime_sym_lookup(ctx, cast(getInt8PtrTy(ctx.builder.getContext())), sym.f_lib, NULL, sym.f_name, ctx.f); - res = ctx.builder.CreatePtrToInt(res, lrt); } } @@ -1462,14 +1461,14 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs) (void)isVa; // prevent compiler warning if (is_libjulia_func(jl_value_ptr)) { ++CCALL_STAT(jl_value_ptr); - assert(retboxed ? lrt == ctx.types().T_prjlvalue : lrt == ctx.types().T_size); + assert(retboxed ? lrt == ctx.types().T_prjlvalue : lrt == ctx.types().T_ptr); assert(!isVa && !llvmcall && nccallargs == 1); jl_value_t *tti = jl_svecref(at, 0); Type *largty; bool isboxed; if (jl_is_abstract_ref_type(tti)) { tti = (jl_value_t*)jl_voidpointer_type; - largty = ctx.types().T_size; + largty = ctx.types().T_ptr; isboxed = false; } else { @@ -1482,7 +1481,6 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs) } else { retval = emit_unbox(ctx, largty, argv[0], tti); - retval = emit_inttoptr(ctx, retval, ctx.types().T_pjlvalue); } // retval is now an untracked jl_value_t* if (retboxed) @@ -1561,9 +1559,7 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs) assert(lrt == ctx.types().T_size); assert(!isVa && !llvmcall && nccallargs == 0); JL_GC_POP(); - return mark_or_box_ccall_result(ctx, - ctx.builder.CreatePtrToInt(get_current_ptls(ctx), lrt), - retboxed, rt, unionall, static_rt); + return mark_or_box_ccall_result(ctx, get_current_ptls(ctx), retboxed, rt, unionall, static_rt); } else if (is_libjulia_func(jl_threadid)) { ++CCALL_STAT(jl_threadid); @@ -1683,21 +1679,20 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs) } else if (is_libjulia_func(jl_string_ptr)) { ++CCALL_STAT(jl_string_ptr); - assert(lrt == ctx.types().T_size); + assert(lrt == ctx.types().T_ptr); assert(!isVa && !llvmcall && nccallargs == 1); auto obj = emit_bitcast(ctx, emit_pointer_from_objref(ctx, boxed(ctx, argv[0])), ctx.types().T_pprjlvalue); // The inbounds gep makes it more clear to LLVM that the resulting value is not // a null pointer. auto strp = ctx.builder.CreateConstInBoundsGEP1_32(ctx.types().T_prjlvalue, obj, 1); - strp = ctx.builder.CreatePtrToInt(strp, ctx.types().T_size); setName(ctx.emission_context, strp, "string_ptr"); JL_GC_POP(); return mark_or_box_ccall_result(ctx, strp, retboxed, rt, unionall, static_rt); } else if (is_libjulia_func(jl_symbol_name)) { ++CCALL_STAT(jl_symbol_name); - assert(lrt == ctx.types().T_size); + assert(lrt == ctx.types().T_ppint8); assert(!isVa && !llvmcall && nccallargs == 1); auto obj = emit_bitcast(ctx, emit_pointer_from_objref(ctx, boxed(ctx, argv[0])), ctx.types().T_pprjlvalue); @@ -1705,7 +1700,6 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs) // a null pointer. auto strp = ctx.builder.CreateConstInBoundsGEP1_32( ctx.types().T_prjlvalue, obj, (sizeof(jl_sym_t) + sizeof(void*) - 1) / sizeof(void*)); - strp = ctx.builder.CreatePtrToInt(strp, ctx.types().T_size); setName(ctx.emission_context, strp, "symbol_name"); JL_GC_POP(); return mark_or_box_ccall_result(ctx, strp, retboxed, rt, unionall, static_rt); @@ -1750,14 +1744,12 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs) const jl_cgval_t &dst = argv[0]; const jl_cgval_t &src = argv[1]; const jl_cgval_t &n = argv[2]; - Value *destp = emit_unbox(ctx, ctx.types().T_size, dst, (jl_value_t*)jl_voidpointer_type); + Value *destp = emit_unbox(ctx, ctx.types().T_ptr, dst, (jl_value_t*)jl_voidpointer_type); ctx.builder.CreateMemCpy( - emit_inttoptr(ctx, destp, getInt8PtrTy(ctx.builder.getContext())), + destp, MaybeAlign(1), - emit_inttoptr(ctx, - emit_unbox(ctx, ctx.types().T_size, src, (jl_value_t*)jl_voidpointer_type), - getInt8PtrTy(ctx.builder.getContext())), + emit_unbox(ctx, ctx.types().T_ptr, src, (jl_value_t*)jl_voidpointer_type), MaybeAlign(1), emit_unbox(ctx, ctx.types().T_size, n, (jl_value_t*)jl_ulong_type), false); @@ -1770,11 +1762,11 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs) const jl_cgval_t &dst = argv[0]; const jl_cgval_t &val = argv[1]; const jl_cgval_t &n = argv[2]; - Value *destp = emit_unbox(ctx, ctx.types().T_size, dst, (jl_value_t*)jl_voidpointer_type); + Value *destp = emit_unbox(ctx, ctx.types().T_ptr, dst, (jl_value_t*)jl_voidpointer_type); Value *val32 = emit_unbox(ctx, getInt32Ty(ctx.builder.getContext()), val, (jl_value_t*)jl_uint32_type); Value *val8 = ctx.builder.CreateTrunc(val32, getInt8Ty(ctx.builder.getContext()), "memset_val"); ctx.builder.CreateMemSet( - emit_inttoptr(ctx, destp, getInt8PtrTy(ctx.builder.getContext())), + destp, val8, emit_unbox(ctx, ctx.types().T_size, n, (jl_value_t*)jl_ulong_type), MaybeAlign(1) @@ -1788,14 +1780,12 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs) const jl_cgval_t &dst = argv[0]; const jl_cgval_t &src = argv[1]; const jl_cgval_t &n = argv[2]; - Value *destp = emit_unbox(ctx, ctx.types().T_size, dst, (jl_value_t*)jl_voidpointer_type); + Value *destp = emit_unbox(ctx, ctx.types().T_ptr, dst, (jl_value_t*)jl_voidpointer_type); ctx.builder.CreateMemMove( - emit_inttoptr(ctx, destp, getInt8PtrTy(ctx.builder.getContext())), + destp, MaybeAlign(0), - emit_inttoptr(ctx, - emit_unbox(ctx, ctx.types().T_size, src, (jl_value_t*)jl_voidpointer_type), - getInt8PtrTy(ctx.builder.getContext())), + emit_unbox(ctx, ctx.types().T_ptr, src, (jl_value_t*)jl_voidpointer_type), MaybeAlign(0), emit_unbox(ctx, ctx.types().T_size, n, (jl_value_t*)jl_ulong_type), false); @@ -1999,7 +1989,7 @@ jl_cgval_t function_sig_t::emit_a_ccall( ++LiteralCCalls; null_pointer_check(ctx, symarg.jl_ptr, nullptr); Type *funcptype = PointerType::get(functype, 0); - llvmf = emit_inttoptr(ctx, symarg.jl_ptr, funcptype); + llvmf = ctx.builder.CreateBitCast(symarg.jl_ptr, funcptype); } else if (symarg.fptr != NULL) { ++LiteralCCalls; diff --git a/src/cgutils.cpp b/src/cgutils.cpp index b17b174dd8a8d..8385b5682a0bc 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -690,6 +690,8 @@ static Type *bitstype_to_llvm(jl_value_t *bt, LLVMContext &ctxt, bool llvmcall = return getDoubleTy(ctxt); if (bt == (jl_value_t*)jl_bfloat16_type) return getBFloatTy(ctxt); + if (jl_is_cpointer_type(bt)) + return PointerType::get(getInt8Ty(ctxt), 0); if (jl_is_llvmpointer_type(bt)) { jl_value_t *as_param = jl_tparam1(bt); int as; diff --git a/src/codegen.cpp b/src/codegen.cpp index faa9de2762471..1b218a279bf6b 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -283,6 +283,7 @@ extern void _chkstk(void); // types struct jl_typecache_t { + Type *T_ptr; Type *T_size; Type *T_jlvalue; Type *T_pjlvalue; @@ -304,7 +305,7 @@ struct jl_typecache_t { bool initialized; jl_typecache_t() : - T_jlvalue(nullptr), T_pjlvalue(nullptr), T_prjlvalue(nullptr), + T_ptr(nullptr), T_jlvalue(nullptr), T_pjlvalue(nullptr), T_prjlvalue(nullptr), T_ppjlvalue(nullptr), T_pprjlvalue(nullptr), T_jlgenericmemory(nullptr), T_jlarray(nullptr), T_pjlarray(nullptr), T_jlfunc(nullptr), T_jlfuncparams(nullptr), T_sigatomic(nullptr), T_ppint8(nullptr), @@ -315,6 +316,7 @@ struct jl_typecache_t { return; } initialized = true; + T_ptr = getInt8PtrTy(context); T_ppint8 = PointerType::get(getInt8PtrTy(context), 0); T_sigatomic = Type::getIntNTy(context, sizeof(sig_atomic_t) * 8); T_size = DL.getIntPtrType(context); diff --git a/src/intrinsics.cpp b/src/intrinsics.cpp index 3a98850ddca68..dabba2a363e0e 100644 --- a/src/intrinsics.cpp +++ b/src/intrinsics.cpp @@ -625,10 +625,14 @@ static jl_cgval_t generic_bitcast(jl_codectx_t &ctx, ArrayRef argv) vx = ctx.builder.CreateZExt(vx, llvmt); } else if (vxt->isPointerTy() && !llvmt->isPointerTy()) { vx = ctx.builder.CreatePtrToInt(vx, llvmt); - setName(ctx.emission_context, vx, "bitcast_coercion"); + if (isa(vx) && !vx->hasName()) + // CreatePtrToInt may undo an IntToPtr + setName(ctx.emission_context, vx, "bitcast_coercion"); } else if (!vxt->isPointerTy() && llvmt->isPointerTy()) { vx = emit_inttoptr(ctx, vx, llvmt); - setName(ctx.emission_context, vx, "bitcast_coercion"); + if (isa(vx) && !vx->hasName()) + // emit_inttoptr may undo an PtrToInt + setName(ctx.emission_context, vx, "bitcast_coercion"); } else { vx = emit_bitcast(ctx, vx, llvmt); setName(ctx.emission_context, vx, "bitcast_coercion"); @@ -741,7 +745,8 @@ static jl_cgval_t emit_pointerref(jl_codectx_t &ctx, ArrayRef argv) if (ety == (jl_value_t*)jl_any_type) { Value *thePtr = emit_unbox(ctx, ctx.types().T_pprjlvalue, e, e.typ); - setName(ctx.emission_context, thePtr, "unbox_any_ptr"); + if (isa(thePtr) && !thePtr->hasName()) + setName(ctx.emission_context, thePtr, "unbox_any_ptr"); LoadInst *load = ctx.builder.CreateAlignedLoad(ctx.types().T_prjlvalue, ctx.builder.CreateInBoundsGEP(ctx.types().T_prjlvalue, thePtr, im1), Align(align_nb)); setName(ctx.emission_context, load, "any_unbox"); jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_data); @@ -1270,6 +1275,27 @@ static jl_cgval_t emit_intrinsic(jl_codectx_t &ctx, intrinsic f, jl_value_t **ar ++Emitted_pointerset; assert(nargs == 4); return emit_pointerset(ctx, argv); + + case add_ptr: { + assert(nargs == 2); + if (!jl_is_cpointer_type(argv[0].typ) || argv[1].typ != (jl_value_t*)jl_ulong_type) + return emit_runtime_call(ctx, f, argv, nargs); + Value *ptr = emit_unbox(ctx, ctx.types().T_ptr, argv[0], argv[0].typ); + Value *off = emit_unbox(ctx, ctx.types().T_size, argv[1], argv[1].typ); + Value *ans = ctx.builder.CreateGEP(getInt8Ty(ctx.builder.getContext()), ptr, off); + return mark_julia_type(ctx, ans, false, argv[0].typ); + } + case sub_ptr: { + assert(nargs == 2); + if (!jl_is_cpointer_type(argv[0].typ) || argv[1].typ != (jl_value_t*)jl_ulong_type) + return emit_runtime_call(ctx, f, argv, nargs); + Value *ptr = emit_unbox(ctx, ctx.types().T_ptr, argv[0], argv[0].typ); + Value *off = emit_unbox(ctx, ctx.types().T_size, argv[1], argv[1].typ); + Value *ans = ctx.builder.CreateGEP(getInt8Ty(ctx.builder.getContext()), ptr, + ctx.builder.CreateNeg(off)); + return mark_julia_type(ctx, ans, false, argv[0].typ); + } + case atomic_fence: ++Emitted_atomic_fence; assert(nargs == 1); @@ -1437,26 +1463,6 @@ static Value *emit_untyped_intrinsic(jl_codectx_t &ctx, intrinsic f, ArrayRef