Skip to content

Commit

Permalink
Switch LLVM codegen of Ptr{T} to an actual pointer type.
Browse files Browse the repository at this point in the history
  • Loading branch information
@maleadt
maleadt committed Mar 11, 2024
1 parent 1ba83f0 commit 6f27310
Show file tree
Hide file tree
Showing 8 changed files with 75 additions and 62 deletions.
12 changes: 6 additions & 6 deletions base/atomics.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -364,13 +364,13 @@ for typ in atomictypes
irt = "$ilt, $ilt*"
@eval getindex(x::Atomic{$typ}) =
GC.@preserve x llvmcall($"""
%ptr = inttoptr i$WORD_SIZE %0 to $lt*
%ptr = bitcast i8* %0 to $lt*
%rv = load atomic $rt %ptr acquire, align $(gc_alignment(typ))
ret $lt %rv
""", $typ, Tuple{Ptr{$typ}}, unsafe_convert(Ptr{$typ}, x))
@eval setindex!(x::Atomic{$typ}, v::$typ) =
GC.@preserve x llvmcall($"""
%ptr = inttoptr i$WORD_SIZE %0 to $lt*
%ptr = bitcast i8* %0 to $lt*
store atomic $lt %1, $lt* %ptr release, align $(gc_alignment(typ))
ret void
""", Cvoid, Tuple{Ptr{$typ}, $typ}, unsafe_convert(Ptr{$typ}, x), v)
Expand All @@ -379,7 +379,7 @@ for typ in atomictypes
if typ <: Integer
@eval atomic_cas!(x::Atomic{$typ}, cmp::$typ, new::$typ) =
GC.@preserve x llvmcall($"""
%ptr = inttoptr i$WORD_SIZE %0 to $lt*
%ptr = bitcast i8* %0 to $lt*
%rs = cmpxchg $lt* %ptr, $lt %1, $lt %2 acq_rel acquire
%rv = extractvalue { $lt, i1 } %rs, 0
ret $lt %rv
Expand All @@ -388,7 +388,7 @@ for typ in atomictypes
else
@eval atomic_cas!(x::Atomic{$typ}, cmp::$typ, new::$typ) =
GC.@preserve x llvmcall($"""
%iptr = inttoptr i$WORD_SIZE %0 to $ilt*
%iptr = bitcast i8* %0 to $ilt*
%icmp = bitcast $lt %1 to $ilt
%inew = bitcast $lt %2 to $ilt
%irs = cmpxchg $ilt* %iptr, $ilt %icmp, $ilt %inew acq_rel acquire
Expand All @@ -411,15 +411,15 @@ for typ in atomictypes
if typ <: Integer
@eval $fn(x::Atomic{$typ}, v::$typ) =
GC.@preserve x llvmcall($"""
%ptr = inttoptr i$WORD_SIZE %0 to $lt*
%ptr = bitcast i8* %0 to $lt*
%rv = atomicrmw $rmw $lt* %ptr, $lt %1 acq_rel
ret $lt %rv
""", $typ, Tuple{Ptr{$typ}, $typ}, unsafe_convert(Ptr{$typ}, x), v)
else
rmwop === :xchg || continue
@eval $fn(x::Atomic{$typ}, v::$typ) =
GC.@preserve x llvmcall($"""
%iptr = inttoptr i$WORD_SIZE %0 to $ilt*
%iptr = bitcast i8* %0 to $ilt*
%ival = bitcast $lt %1 to $ilt
%irv = atomicrmw $rmw $ilt* %iptr, $ilt %ival acq_rel
%rv = bitcast $ilt %irv to $lt
Expand Down
4 changes: 2 additions & 2 deletions base/pointer.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -313,8 +313,8 @@ isless(x::Ptr{T}, y::Ptr{T}) where {T} = x < y
<(x::Ptr, y::Ptr) = UInt(x) < UInt(y)
-(x::Ptr, y::Ptr) = UInt(x) - UInt(y)

+(x::Ptr, y::Integer) = oftype(x, add_ptr(UInt(x), (y % UInt) % UInt))
-(x::Ptr, y::Integer) = oftype(x, sub_ptr(UInt(x), (y % UInt) % UInt))
+(x::Ptr, y::Integer) = add_ptr(x, (y % UInt) % UInt)
-(x::Ptr, y::Integer) = sub_ptr(x, (y % UInt) % UInt)
+(x::Integer, y::Ptr) = y + x

unsigned(x::Ptr) = UInt(x)
Expand Down
3 changes: 1 addition & 2 deletions base/task.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -154,8 +154,7 @@ const _state_index = findfirst(==(:_state), fieldnames(Task))
@eval function load_state_acquire(t)
# TODO: Replace this by proper atomic operations when available
@GC.preserve t llvmcall($("""
%ptr = inttoptr i$(Sys.WORD_SIZE) %0 to i8*
%rv = load atomic i8, i8* %ptr acquire, align 8
%rv = load atomic i8, i8* %0 acquire, align 8
ret i8 %rv
"""), UInt8, Tuple{Ptr{UInt8}},
Ptr{UInt8}(pointer_from_objref(t) + fieldoffset(Task, _state_index)))
Expand Down
42 changes: 16 additions & 26 deletions src/ccall.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -616,7 +616,7 @@ static void interpret_symbol_arg(jl_codectx_t &ctx, native_sym_arg_t &out, jl_va
emit_cpointercheck(ctx, arg1, errmsg);
}
arg1 = update_julia_type(ctx, arg1, (jl_value_t*)jl_voidpointer_type);
jl_ptr = emit_unbox(ctx, ctx.types().T_size, arg1, (jl_value_t*)jl_voidpointer_type);
jl_ptr = emit_unbox(ctx, ctx.types().T_ptr, arg1, (jl_value_t*)jl_voidpointer_type);
}
else {
out.gcroot = ptr;
Expand Down Expand Up @@ -696,7 +696,7 @@ static jl_cgval_t emit_cglobal(jl_codectx_t &ctx, jl_value_t **args, size_t narg
else {
rt = (jl_value_t*)jl_voidpointer_type;
}
Type *lrt = ctx.types().T_size;
Type *lrt = ctx.types().T_ptr;
assert(lrt == julia_type_to_llvm(ctx, rt));

interpret_symbol_arg(ctx, sym, args[1], /*ccall=*/false, false);
Expand All @@ -723,7 +723,6 @@ static jl_cgval_t emit_cglobal(jl_codectx_t &ctx, jl_value_t **args, size_t narg
}
else /*if (ctx.emission_context.imaging) */{
res = runtime_sym_lookup(ctx, cast<PointerType>(getInt8PtrTy(ctx.builder.getContext())), sym.f_lib, NULL, sym.f_name, ctx.f);
res = ctx.builder.CreatePtrToInt(res, lrt);
}
}

Expand Down Expand Up @@ -1462,14 +1461,14 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs)
(void)isVa; // prevent compiler warning
if (is_libjulia_func(jl_value_ptr)) {
++CCALL_STAT(jl_value_ptr);
assert(retboxed ? lrt == ctx.types().T_prjlvalue : lrt == ctx.types().T_size);
assert(retboxed ? lrt == ctx.types().T_prjlvalue : lrt == ctx.types().T_ptr);
assert(!isVa && !llvmcall && nccallargs == 1);
jl_value_t *tti = jl_svecref(at, 0);
Type *largty;
bool isboxed;
if (jl_is_abstract_ref_type(tti)) {
tti = (jl_value_t*)jl_voidpointer_type;
largty = ctx.types().T_size;
largty = ctx.types().T_ptr;
isboxed = false;
}
else {
Expand All @@ -1482,7 +1481,6 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs)
}
else {
retval = emit_unbox(ctx, largty, argv[0], tti);
retval = emit_inttoptr(ctx, retval, ctx.types().T_pjlvalue);
}
// retval is now an untracked jl_value_t*
if (retboxed)
Expand Down Expand Up @@ -1561,9 +1559,7 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs)
assert(lrt == ctx.types().T_size);
assert(!isVa && !llvmcall && nccallargs == 0);
JL_GC_POP();
return mark_or_box_ccall_result(ctx,
ctx.builder.CreatePtrToInt(get_current_ptls(ctx), lrt),
retboxed, rt, unionall, static_rt);
return mark_or_box_ccall_result(ctx, get_current_ptls(ctx), retboxed, rt, unionall, static_rt);
}
else if (is_libjulia_func(jl_threadid)) {
++CCALL_STAT(jl_threadid);
Expand Down Expand Up @@ -1683,29 +1679,27 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs)
}
else if (is_libjulia_func(jl_string_ptr)) {
++CCALL_STAT(jl_string_ptr);
assert(lrt == ctx.types().T_size);
assert(lrt == ctx.types().T_ptr);
assert(!isVa && !llvmcall && nccallargs == 1);
auto obj = emit_bitcast(ctx, emit_pointer_from_objref(ctx, boxed(ctx, argv[0])),
ctx.types().T_pprjlvalue);
// The inbounds gep makes it more clear to LLVM that the resulting value is not
// a null pointer.
auto strp = ctx.builder.CreateConstInBoundsGEP1_32(ctx.types().T_prjlvalue, obj, 1);
strp = ctx.builder.CreatePtrToInt(strp, ctx.types().T_size);
setName(ctx.emission_context, strp, "string_ptr");
JL_GC_POP();
return mark_or_box_ccall_result(ctx, strp, retboxed, rt, unionall, static_rt);
}
else if (is_libjulia_func(jl_symbol_name)) {
++CCALL_STAT(jl_symbol_name);
assert(lrt == ctx.types().T_size);
assert(lrt == ctx.types().T_ppint8);
assert(!isVa && !llvmcall && nccallargs == 1);
auto obj = emit_bitcast(ctx, emit_pointer_from_objref(ctx, boxed(ctx, argv[0])),
ctx.types().T_pprjlvalue);
// The inbounds gep makes it more clear to LLVM that the resulting value is not
// a null pointer.
auto strp = ctx.builder.CreateConstInBoundsGEP1_32(
ctx.types().T_prjlvalue, obj, (sizeof(jl_sym_t) + sizeof(void*) - 1) / sizeof(void*));
strp = ctx.builder.CreatePtrToInt(strp, ctx.types().T_size);
setName(ctx.emission_context, strp, "symbol_name");
JL_GC_POP();
return mark_or_box_ccall_result(ctx, strp, retboxed, rt, unionall, static_rt);
Expand Down Expand Up @@ -1750,14 +1744,12 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs)
const jl_cgval_t &dst = argv[0];
const jl_cgval_t &src = argv[1];
const jl_cgval_t &n = argv[2];
Value *destp = emit_unbox(ctx, ctx.types().T_size, dst, (jl_value_t*)jl_voidpointer_type);
Value *destp = emit_unbox(ctx, ctx.types().T_ptr, dst, (jl_value_t*)jl_voidpointer_type);

ctx.builder.CreateMemCpy(
emit_inttoptr(ctx, destp, getInt8PtrTy(ctx.builder.getContext())),
destp,
MaybeAlign(1),
emit_inttoptr(ctx,
emit_unbox(ctx, ctx.types().T_size, src, (jl_value_t*)jl_voidpointer_type),
getInt8PtrTy(ctx.builder.getContext())),
emit_unbox(ctx, ctx.types().T_ptr, src, (jl_value_t*)jl_voidpointer_type),
MaybeAlign(1),
emit_unbox(ctx, ctx.types().T_size, n, (jl_value_t*)jl_ulong_type),
false);
Expand All @@ -1770,11 +1762,11 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs)
const jl_cgval_t &dst = argv[0];
const jl_cgval_t &val = argv[1];
const jl_cgval_t &n = argv[2];
Value *destp = emit_unbox(ctx, ctx.types().T_size, dst, (jl_value_t*)jl_voidpointer_type);
Value *destp = emit_unbox(ctx, ctx.types().T_ptr, dst, (jl_value_t*)jl_voidpointer_type);
Value *val32 = emit_unbox(ctx, getInt32Ty(ctx.builder.getContext()), val, (jl_value_t*)jl_uint32_type);
Value *val8 = ctx.builder.CreateTrunc(val32, getInt8Ty(ctx.builder.getContext()), "memset_val");
ctx.builder.CreateMemSet(
emit_inttoptr(ctx, destp, getInt8PtrTy(ctx.builder.getContext())),
destp,
val8,
emit_unbox(ctx, ctx.types().T_size, n, (jl_value_t*)jl_ulong_type),
MaybeAlign(1)
Expand All @@ -1788,14 +1780,12 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs)
const jl_cgval_t &dst = argv[0];
const jl_cgval_t &src = argv[1];
const jl_cgval_t &n = argv[2];
Value *destp = emit_unbox(ctx, ctx.types().T_size, dst, (jl_value_t*)jl_voidpointer_type);
Value *destp = emit_unbox(ctx, ctx.types().T_ptr, dst, (jl_value_t*)jl_voidpointer_type);

ctx.builder.CreateMemMove(
emit_inttoptr(ctx, destp, getInt8PtrTy(ctx.builder.getContext())),
destp,
MaybeAlign(0),
emit_inttoptr(ctx,
emit_unbox(ctx, ctx.types().T_size, src, (jl_value_t*)jl_voidpointer_type),
getInt8PtrTy(ctx.builder.getContext())),
emit_unbox(ctx, ctx.types().T_ptr, src, (jl_value_t*)jl_voidpointer_type),
MaybeAlign(0),
emit_unbox(ctx, ctx.types().T_size, n, (jl_value_t*)jl_ulong_type),
false);
Expand Down Expand Up @@ -1999,7 +1989,7 @@ jl_cgval_t function_sig_t::emit_a_ccall(
++LiteralCCalls;
null_pointer_check(ctx, symarg.jl_ptr, nullptr);
Type *funcptype = PointerType::get(functype, 0);
llvmf = emit_inttoptr(ctx, symarg.jl_ptr, funcptype);
llvmf = ctx.builder.CreateBitCast(symarg.jl_ptr, funcptype);
}
else if (symarg.fptr != NULL) {
++LiteralCCalls;
Expand Down
2 changes: 2 additions & 0 deletions src/cgutils.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -690,6 +690,8 @@ static Type *bitstype_to_llvm(jl_value_t *bt, LLVMContext &ctxt, bool llvmcall =
return getDoubleTy(ctxt);
if (bt == (jl_value_t*)jl_bfloat16_type)
return getBFloatTy(ctxt);
if (jl_is_cpointer_type(bt))
return PointerType::get(getInt8Ty(ctxt), 0);
if (jl_is_llvmpointer_type(bt)) {
jl_value_t *as_param = jl_tparam1(bt);
int as;
Expand Down
4 changes: 3 additions & 1 deletion src/codegen.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -283,6 +283,7 @@ extern void _chkstk(void);

// types
struct jl_typecache_t {
Type *T_ptr;
Type *T_size;
Type *T_jlvalue;
Type *T_pjlvalue;
Expand All @@ -304,7 +305,7 @@ struct jl_typecache_t {
bool initialized;

jl_typecache_t() :
T_jlvalue(nullptr), T_pjlvalue(nullptr), T_prjlvalue(nullptr),
T_ptr(nullptr), T_jlvalue(nullptr), T_pjlvalue(nullptr), T_prjlvalue(nullptr),
T_ppjlvalue(nullptr), T_pprjlvalue(nullptr),
T_jlgenericmemory(nullptr), T_jlarray(nullptr), T_pjlarray(nullptr),
T_jlfunc(nullptr), T_jlfuncparams(nullptr), T_sigatomic(nullptr), T_ppint8(nullptr),
Expand All @@ -315,6 +316,7 @@ struct jl_typecache_t {
return;
}
initialized = true;
T_ptr = getInt8PtrTy(context);
T_ppint8 = PointerType::get(getInt8PtrTy(context), 0);
T_sigatomic = Type::getIntNTy(context, sizeof(sig_atomic_t) * 8);
T_size = DL.getIntPtrType(context);
Expand Down
52 changes: 29 additions & 23 deletions src/intrinsics.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -625,10 +625,14 @@ static jl_cgval_t generic_bitcast(jl_codectx_t &ctx, ArrayRef<jl_cgval_t> argv)
vx = ctx.builder.CreateZExt(vx, llvmt);
} else if (vxt->isPointerTy() && !llvmt->isPointerTy()) {
vx = ctx.builder.CreatePtrToInt(vx, llvmt);
setName(ctx.emission_context, vx, "bitcast_coercion");
if (isa<Instruction>(vx) && !vx->hasName())
// CreatePtrToInt may undo an IntToPtr
setName(ctx.emission_context, vx, "bitcast_coercion");
} else if (!vxt->isPointerTy() && llvmt->isPointerTy()) {
vx = emit_inttoptr(ctx, vx, llvmt);
setName(ctx.emission_context, vx, "bitcast_coercion");
if (isa<Instruction>(vx) && !vx->hasName())
// emit_inttoptr may undo an PtrToInt
setName(ctx.emission_context, vx, "bitcast_coercion");
} else {
vx = emit_bitcast(ctx, vx, llvmt);
setName(ctx.emission_context, vx, "bitcast_coercion");
Expand Down Expand Up @@ -741,7 +745,8 @@ static jl_cgval_t emit_pointerref(jl_codectx_t &ctx, ArrayRef<jl_cgval_t> argv)

if (ety == (jl_value_t*)jl_any_type) {
Value *thePtr = emit_unbox(ctx, ctx.types().T_pprjlvalue, e, e.typ);
setName(ctx.emission_context, thePtr, "unbox_any_ptr");
if (isa<Instruction>(thePtr) && !thePtr->hasName())
setName(ctx.emission_context, thePtr, "unbox_any_ptr");
LoadInst *load = ctx.builder.CreateAlignedLoad(ctx.types().T_prjlvalue, ctx.builder.CreateInBoundsGEP(ctx.types().T_prjlvalue, thePtr, im1), Align(align_nb));
setName(ctx.emission_context, load, "any_unbox");
jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_data);
Expand Down Expand Up @@ -1270,6 +1275,27 @@ static jl_cgval_t emit_intrinsic(jl_codectx_t &ctx, intrinsic f, jl_value_t **ar
++Emitted_pointerset;
assert(nargs == 4);
return emit_pointerset(ctx, argv);

case add_ptr: {
assert(nargs == 2);
if (!jl_is_cpointer_type(argv[0].typ) || argv[1].typ != (jl_value_t*)jl_ulong_type)
return emit_runtime_call(ctx, f, argv, nargs);
Value *ptr = emit_unbox(ctx, ctx.types().T_ptr, argv[0], argv[0].typ);
Value *off = emit_unbox(ctx, ctx.types().T_size, argv[1], argv[1].typ);
Value *ans = ctx.builder.CreateGEP(getInt8Ty(ctx.builder.getContext()), ptr, off);
return mark_julia_type(ctx, ans, false, argv[0].typ);
}
case sub_ptr: {
assert(nargs == 2);
if (!jl_is_cpointer_type(argv[0].typ) || argv[1].typ != (jl_value_t*)jl_ulong_type)
return emit_runtime_call(ctx, f, argv, nargs);
Value *ptr = emit_unbox(ctx, ctx.types().T_ptr, argv[0], argv[0].typ);
Value *off = emit_unbox(ctx, ctx.types().T_size, argv[1], argv[1].typ);
Value *ans = ctx.builder.CreateGEP(getInt8Ty(ctx.builder.getContext()), ptr,
ctx.builder.CreateNeg(off));
return mark_julia_type(ctx, ans, false, argv[0].typ);
}

case atomic_fence:
++Emitted_atomic_fence;
assert(nargs == 1);
Expand Down Expand Up @@ -1437,26 +1463,6 @@ static Value *emit_untyped_intrinsic(jl_codectx_t &ctx, intrinsic f, ArrayRef<Va
case srem_int: return ctx.builder.CreateSRem(x, y);
case urem_int: return ctx.builder.CreateURem(x, y);

// LLVM will not fold ptrtoint+arithmetic+inttoptr to GEP. The reason for this
// has to do with alias analysis. When adding two integers, either one of them
// could be the pointer base. With getelementptr, it is clear which of the
// operands is the pointer base. We also have this information at the julia
// level. Thus, to not lose information, we need to have a separate intrinsic
// for pointer arithmetic which lowers to getelementptr.
case add_ptr: {
return ctx.builder.CreatePtrToInt(
ctx.builder.CreateGEP(getInt8Ty(ctx.builder.getContext()),
emit_inttoptr(ctx, x, getInt8PtrTy(ctx.builder.getContext())), y), t);

}

case sub_ptr: {
return ctx.builder.CreatePtrToInt(
ctx.builder.CreateGEP(getInt8Ty(ctx.builder.getContext()),
emit_inttoptr(ctx, x, getInt8PtrTy(ctx.builder.getContext())), ctx.builder.CreateNeg(y)), t);

}

case neg_float: return math_builder(ctx)().CreateFNeg(x);
case neg_float_fast: return math_builder(ctx, true)().CreateFNeg(x);
case add_float: return math_builder(ctx)().CreateFAdd(x, y);
Expand Down
18 changes: 16 additions & 2 deletions src/runtime_intrinsics.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1384,10 +1384,8 @@ JL_DLLEXPORT jl_value_t *jl_##name(jl_value_t *a, jl_value_t *b, jl_value_t *c)
un_iintrinsic_fast(LLVMNeg, neg, neg_int, u)
#define add(a,b) a + b
bi_iintrinsic_fast(LLVMAdd, add, add_int, u)
bi_iintrinsic_fast(LLVMAdd, add, add_ptr, u)
#define sub(a,b) a - b
bi_iintrinsic_fast(LLVMSub, sub, sub_int, u)
bi_iintrinsic_fast(LLVMSub, sub, sub_ptr, u)
#define mul(a,b) a * b
bi_iintrinsic_fast(LLVMMul, mul, mul_int, u)
#define div(a,b) a / b
Expand Down Expand Up @@ -1696,3 +1694,19 @@ JL_DLLEXPORT jl_value_t *jl_have_fma(jl_value_t *typ)
else
return jl_false;
}

JL_DLLEXPORT jl_value_t *jl_add_ptr(jl_value_t *ptr, jl_value_t *offset)
{
JL_TYPECHK(add_ptr, pointer, ptr);
JL_TYPECHK(add_ptr, ulong, offset);
long ptrval = jl_unbox_long(ptr) + jl_unbox_long(offset);
return jl_new_bits(jl_typeof(ptr), &ptrval);
}

JL_DLLEXPORT jl_value_t *jl_sub_ptr(jl_value_t *ptr, jl_value_t *offset)
{
JL_TYPECHK(sub_ptr, pointer, ptr);
JL_TYPECHK(sub_ptr, ulong, offset);
long ptrval = jl_unbox_long(ptr) - jl_unbox_long(offset);
return jl_new_bits(jl_typeof(ptr), &ptrval);
}

0 comments on commit 6f27310

Please sign in to comment.