Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

layout optimization internal changes (support pointers inlining/unboxing into parents/codegen) [disabled] #33886

Merged
merged 1 commit into from
Dec 17, 2019
Merged

layout optimization internal changes (support pointers inlining/unboxing into parents/codegen) [disabled] #33886

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 44 additions & 35 deletions base/array.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,7 @@ size(a::Array{<:Any,N}) where {N} = (@_inline_meta; ntuple(M -> size(a, M), Val(

asize_from(a::Array, n) = n > ndims(a) ? () : (arraysize(a,n), asize_from(a, n+1)...)

allocatedinline(::Type{T}) where {T} = (@_pure_meta; ccall(:jl_array_store_unboxed, Cint, (Any,), T) != Cint(0))
allocatedinline(::Type{T}) where {T} = (@_pure_meta; ccall(:jl_stored_inline, Cint, (Any,), T) != Cint(0))

"""
Base.isbitsunion(::Type{T})
Expand All @@ -177,14 +177,20 @@ false
isbitsunion(u::Union) = allocatedinline(u)
isbitsunion(x) = false

function _unsetindex!(A::Array{T}, i::Int) where {T}
@inbounds function _unsetindex!(A::Array{T}, i::Int) where {T}
Copy link
Sponsor Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What does @inbounds on a function do?

Copy link
Sponsor Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Supposed to be @inline.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

See #34127

@boundscheck checkbounds(A, i)
t = @_gc_preserve_begin A
p = Ptr{Ptr{Cvoid}}(pointer(A, i))
if !allocatedinline(T)
t = @_gc_preserve_begin A
p = Ptr{Ptr{Cvoid}}(pointer(A))
unsafe_store!(p, C_NULL, i)
@_gc_preserve_end t
unsafe_store!(p, C_NULL)
elseif T isa DataType
if !datatype_pointerfree(T)
for j = 1:(Core.sizeof(T) ÷ Core.sizeof(Ptr{Cvoid}))
unsafe_store!(p, C_NULL, j)
end
end
end
@_gc_preserve_end t
return A
end

Expand Down Expand Up @@ -255,19 +261,41 @@ the same manner as C.
function unsafe_copyto!(dest::Array{T}, doffs, src::Array{T}, soffs, n) where T
t1 = @_gc_preserve_begin dest
t2 = @_gc_preserve_begin src
if isbitsunion(T)
destp = pointer(dest, doffs)
srcp = pointer(src, soffs)
if !allocatedinline(T)
ccall(:jl_array_ptr_copy, Cvoid, (Any, Ptr{Cvoid}, Any, Ptr{Cvoid}, Int),
dest, destp, src, srcp, n)
elseif isbitstype(T)
ccall(:memmove, Ptr{Cvoid}, (Ptr{Cvoid}, Ptr{Cvoid}, UInt),
pointer(dest, doffs), pointer(src, soffs), n * aligned_sizeof(T))
destp, srcp, n * aligned_sizeof(T))
elseif isbitsunion(T)
ccall(:memmove, Ptr{Cvoid}, (Ptr{Cvoid}, Ptr{Cvoid}, UInt),
destp, srcp, n * aligned_sizeof(T))
# copy selector bytes
ccall(:memmove, Ptr{Cvoid}, (Ptr{Cvoid}, Ptr{Cvoid}, UInt),
ccall(:jl_array_typetagdata, Ptr{UInt8}, (Any,), dest) + doffs - 1,
ccall(:jl_array_typetagdata, Ptr{UInt8}, (Any,), src) + soffs - 1,
n)
elseif allocatedinline(T)
unsafe_copyto!(pointer(dest, doffs), pointer(src, soffs), n)
else
ccall(:jl_array_ptr_copy, Cvoid, (Any, Ptr{Cvoid}, Any, Ptr{Cvoid}, Int),
dest, pointer(dest, doffs), src, pointer(src, soffs), n)
# handle base-case: everything else above was just optimizations
@inbounds if destp < srcp || destp > srcp + n
for i = 1:n
if isassigned(src, soffs + i - 1)
dest[doffs + i - 1] = src[soffs + i - 1]
else
_unsetindex!(dest, doffs + i - 1)
end
end
else
for i = n:-1:1
if isassigned(src, soffs + i - 1)
dest[doffs + i - 1] = src[soffs + i - 1]
else
_unsetindex!(dest, doffs + i - 1)
end
end
end
end
@_gc_preserve_end t2
@_gc_preserve_end t1
Expand Down Expand Up @@ -1566,32 +1594,13 @@ function vcat(arrays::Vector{T}...) where T
n += length(a)
end
arr = Vector{T}(undef, n)
ptr = pointer(arr)
if isbitsunion(T)
selptr = ccall(:jl_array_typetagdata, Ptr{UInt8}, (Any,), arr)
end
elsz = aligned_sizeof(T)
t = @_gc_preserve_begin arr
nd = 1
for a in arrays
na = length(a)
nba = na * elsz
if isbitsunion(T)
ccall(:memcpy, Ptr{Cvoid}, (Ptr{Cvoid}, Ptr{Cvoid}, UInt),
ptr, a, nba)
# copy selector bytes
ccall(:memcpy, Ptr{Cvoid}, (Ptr{Cvoid}, Ptr{Cvoid}, UInt),
selptr, ccall(:jl_array_typetagdata, Ptr{UInt8}, (Any,), a), na)
selptr += na
elseif allocatedinline(T)
ccall(:memcpy, Ptr{Cvoid}, (Ptr{Cvoid}, Ptr{Cvoid}, UInt),
ptr, a, nba)
else
ccall(:jl_array_ptr_copy, Cvoid, (Any, Ptr{Cvoid}, Any, Ptr{Cvoid}, Int),
arr, ptr, a, pointer(a), na)
end
ptr += nba
@assert nd + na <= 1 + length(arr) # Concurrent modification of arrays?
unsafe_copyto!(arr, nd, a, 1, na)
nd += na
end
@_gc_preserve_end t
return arr
end

Expand Down
1 change: 1 addition & 0 deletions base/reflection.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -318,6 +318,7 @@ datatype_fieldtypes(x::DataType) = ccall(:jl_get_fieldtypes, Any, (Any,), x)
struct DataTypeLayout
nfields::UInt32
npointers::UInt32
firstptr::Int32
alignment::UInt32
# alignment : 9;
# haspadding : 1;
Expand Down
2 changes: 1 addition & 1 deletion base/refpointer.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ RefArray(x::AbstractArray{T}, i::Int=1, roots::Nothing=nothing) where {T} = RefA
convert(::Type{Ref{T}}, x::AbstractArray{T}) where {T} = RefArray(x, 1)

function unsafe_convert(P::Type{Ptr{T}}, b::RefArray{T}) where T
if datatype_pointerfree(RefValue{T})
if allocatedinline(T)
p = pointer(b.x, b.i)
elseif isconcretetype(T) && T.mutable
p = pointer_from_objref(b.x[b.i])
Expand Down
2 changes: 1 addition & 1 deletion base/refvalue.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ RefValue(x::T) where {T} = RefValue{T}(x)
isassigned(x::RefValue) = isdefined(x, :x)

function unsafe_convert(P::Type{Ptr{T}}, b::RefValue{T}) where T
if datatype_pointerfree(RefValue{T})
if allocatedinline(T)
p = pointer_from_objref(b)
elseif isconcretetype(T) && T.mutable
p = pointer_from_objref(b.x)
Expand Down
70 changes: 47 additions & 23 deletions src/array.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,6 @@ char *jl_array_typetagdata(jl_array_t *a) JL_NOTSAFEPOINT
return ((char*)jl_array_data(a)) + ((jl_array_ndims(a) == 1 ? (a->maxsize - a->offset) : jl_array_len(a)) * a->elsize) + a->offset;
}

JL_DLLEXPORT int jl_array_store_unboxed(jl_value_t *eltype) JL_NOTSAFEPOINT
{
size_t fsz = 0, al = 0;
return jl_islayout_inline(eltype, &fsz, &al);
}

STATIC_INLINE jl_value_t *jl_array_owner(jl_array_t *a JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT
{
if (a->flags.how == 3) {
Expand All @@ -53,7 +47,7 @@ size_t jl_arr_xtralloc_limit = 0;
#define MAXINTVAL (((size_t)-1)>>1)

static jl_array_t *_new_array_(jl_value_t *atype, uint32_t ndims, size_t *dims,
int isunboxed, int isunion, int elsz)
int isunboxed, int hasptr, int isunion, int elsz)
{
jl_ptls_t ptls = jl_get_ptls_states();
size_t i, tot, nel=1;
Expand Down Expand Up @@ -101,7 +95,7 @@ static jl_array_t *_new_array_(jl_value_t *atype, uint32_t ndims, size_t *dims,
// No allocation or safepoint allowed after this
a->flags.how = 0;
data = (char*)a + doffs;
if ((tot > 0 && !isunboxed) || isunion)
if (tot > 0 && (!isunboxed || hasptr || isunion)) // TODO: check for zeroinit
memset(data, 0, tot);
}
else {
Expand All @@ -113,7 +107,7 @@ static jl_array_t *_new_array_(jl_value_t *atype, uint32_t ndims, size_t *dims,
// No allocation or safepoint allowed after this
a->flags.how = 2;
jl_gc_track_malloced_array(ptls, a);
if (!isunboxed || isunion)
if (tot > 0 && (!isunboxed || hasptr || isunion)) // TODO: check for zeroinit
// need to zero out isbits union array selector bytes to ensure a valid type index
memset(data, 0, tot);
}
Expand All @@ -127,6 +121,7 @@ static jl_array_t *_new_array_(jl_value_t *atype, uint32_t ndims, size_t *dims,
#endif
a->flags.ndims = ndims;
a->flags.ptrarray = !isunboxed;
a->flags.hasptr = hasptr;
a->elsize = elsz;
a->flags.isshared = 0;
a->flags.isaligned = 1;
Expand All @@ -135,9 +130,12 @@ static jl_array_t *_new_array_(jl_value_t *atype, uint32_t ndims, size_t *dims,
a->nrows = nel;
a->maxsize = nel;
}
else if (a->flags.ndims != ndims) {
jl_exceptionf(jl_argumenterror_type, "invalid Array dimensions");
}
else {
size_t *adims = &a->nrows;
for(i=0; i < ndims; i++)
for (i = 0; i < ndims; i++)
adims[i] = dims[i];
}

Expand All @@ -152,6 +150,7 @@ static inline jl_array_t *_new_array(jl_value_t *atype, uint32_t ndims, size_t *
jl_type_error_rt("Array", "element type", (jl_value_t*)jl_type_type, eltype);
int isunboxed = jl_islayout_inline(eltype, &elsz, &al);
int isunion = jl_is_uniontype(eltype);
int hasptr = isunboxed && (jl_is_datatype(eltype) && ((jl_datatype_t*)eltype)->layout->npointers > 0);
if (!isunboxed) {
elsz = sizeof(void*);
al = elsz;
Expand All @@ -160,13 +159,13 @@ static inline jl_array_t *_new_array(jl_value_t *atype, uint32_t ndims, size_t *
elsz = LLT_ALIGN(elsz, al);
}

return _new_array_(atype, ndims, dims, isunboxed, isunion, elsz);
return _new_array_(atype, ndims, dims, isunboxed, hasptr, isunion, elsz);
}

jl_array_t *jl_new_array_for_deserialization(jl_value_t *atype, uint32_t ndims, size_t *dims,
int isunboxed, int isunion, int elsz)
int isunboxed, int hasptr, int isunion, int elsz)
{
return _new_array_(atype, ndims, dims, isunboxed, isunion, elsz);
return _new_array_(atype, ndims, dims, isunboxed, hasptr, isunion, elsz);
}

#ifndef JL_NDEBUG
Expand Down Expand Up @@ -224,10 +223,12 @@ JL_DLLEXPORT jl_array_t *jl_reshape_array(jl_value_t *atype, jl_array_t *data,
"reinterpret from alignment %d bytes to alignment %d bytes not allowed",
(int) oldalign, (int) align);
a->flags.ptrarray = 0;
a->flags.hasptr = data->flags.hasptr;
}
else {
a->elsize = sizeof(void*);
a->flags.ptrarray = 1;
a->flags.hasptr = 0;
}

// if data is itself a shared wrapper,
Expand All @@ -247,6 +248,9 @@ JL_DLLEXPORT jl_array_t *jl_reshape_array(jl_value_t *atype, jl_array_t *data,
a->nrows = l;
a->maxsize = l;
}
else if (a->flags.ndims != ndims) {
jl_exceptionf(jl_argumenterror_type, "invalid Array dimensions");
}
else {
size_t *adims = &a->nrows;
size_t l = 1;
Expand Down Expand Up @@ -281,6 +285,7 @@ JL_DLLEXPORT jl_array_t *jl_string_to_array(jl_value_t *str)
a->flags.isaligned = 0;
a->elsize = 1;
a->flags.ptrarray = 0;
a->flags.hasptr = 0;
jl_array_data_owner(a) = str;
a->flags.how = 3;
a->flags.isshared = 1;
Expand All @@ -300,12 +305,12 @@ JL_DLLEXPORT jl_array_t *jl_ptr_to_array_1d(jl_value_t *atype, void *data,
jl_array_t *a;
jl_value_t *eltype = jl_tparam0(atype);

int isunboxed = jl_array_store_unboxed(eltype);
size_t elsz;
unsigned align;
int isunboxed = jl_stored_inline(eltype);
if (isunboxed && jl_is_uniontype(eltype))
jl_exceptionf(jl_argumenterror_type,
"unsafe_wrap: unspecified layout for union element type");
size_t elsz;
unsigned align;
if (isunboxed) {
elsz = jl_datatype_size(eltype);
align = jl_datatype_align(eltype);
Expand All @@ -328,6 +333,7 @@ JL_DLLEXPORT jl_array_t *jl_ptr_to_array_1d(jl_value_t *atype, void *data,
#endif
a->elsize = LLT_ALIGN(elsz, align);
a->flags.ptrarray = !isunboxed;
a->flags.hasptr = isunboxed && (jl_is_datatype(eltype) && ((jl_datatype_t*)eltype)->layout->npointers > 0);
a->flags.ndims = 1;
a->flags.isshared = 1;
a->flags.isaligned = 0; // TODO: allow passing memalign'd buffers
Expand Down Expand Up @@ -366,12 +372,12 @@ JL_DLLEXPORT jl_array_t *jl_ptr_to_array(jl_value_t *atype, void *data,
return jl_ptr_to_array_1d(atype, data, nel, own_buffer);
jl_value_t *eltype = jl_tparam0(atype);

int isunboxed = jl_array_store_unboxed(eltype);
size_t elsz;
unsigned align;
int isunboxed = jl_stored_inline(eltype);
if (isunboxed && jl_is_uniontype(eltype))
jl_exceptionf(jl_argumenterror_type,
"unsafe_wrap: unspecified layout for union element type");
size_t elsz;
unsigned align;
if (isunboxed) {
elsz = jl_datatype_size(eltype);
align = jl_datatype_align(eltype);
Expand All @@ -394,6 +400,7 @@ JL_DLLEXPORT jl_array_t *jl_ptr_to_array(jl_value_t *atype, void *data,
#endif
a->elsize = LLT_ALIGN(elsz, align);
a->flags.ptrarray = !isunboxed;
a->flags.hasptr = isunboxed && (jl_is_datatype(eltype) && ((jl_datatype_t*)eltype)->layout->npointers > 0);
a->flags.ndims = ndims;
a->offset = 0;
a->flags.isshared = 1;
Expand All @@ -408,6 +415,8 @@ JL_DLLEXPORT jl_array_t *jl_ptr_to_array(jl_value_t *atype, void *data,
}

assert(ndims != 1); // handled above
if (a->flags.ndims != ndims)
jl_exceptionf(jl_argumenterror_type, "invalid Array dimensions");
memcpy(&a->nrows, dims, ndims * sizeof(size_t));
return a;
}
Expand Down Expand Up @@ -559,8 +568,16 @@ JL_DLLEXPORT jl_value_t *jl_arrayref(jl_array_t *a, size_t i)

JL_DLLEXPORT int jl_array_isassigned(jl_array_t *a, size_t i)
{
if (a->flags.ptrarray)
if (a->flags.ptrarray) {
return ((jl_value_t**)jl_array_data(a))[i] != NULL;
}
else if (a->flags.hasptr) {
jl_datatype_t *eltype = (jl_datatype_t*)jl_tparam0(jl_typeof(a));
assert(eltype->layout->first_ptr >= 0);
jl_value_t **slot =
(jl_value_t**)(&((char*)a->data)[i*a->elsize] + eltype->layout->first_ptr);
return *slot != NULL;
}
return 1;
}

Expand All @@ -585,6 +602,8 @@ JL_DLLEXPORT void jl_arrayset(jl_array_t *a JL_ROOTING_ARGUMENT, jl_value_t *rhs
return;
}
jl_assign_bits(&((char*)a->data)[i * a->elsize], rhs);
if (a->flags.hasptr)
jl_gc_multi_wb(jl_array_owner(a), rhs);
}
else {
((jl_value_t**)a->data)[i] = rhs;
Expand All @@ -598,6 +617,11 @@ JL_DLLEXPORT void jl_arrayunset(jl_array_t *a, size_t i)
jl_bounds_error_int((jl_value_t*)a, i + 1);
if (a->flags.ptrarray)
((jl_value_t**)a->data)[i] = NULL;
else if (a->flags.hasptr) {
size_t elsize = a->elsize;
jl_assume(elsize >= sizeof(void*) && elsize % sizeof(void*) == 0);
memset(&((jl_value_t**)a->data)[i], 0, elsize);
}
}

// at this size and bigger, allocate resized array data with malloc
Expand Down Expand Up @@ -814,7 +838,7 @@ STATIC_INLINE void jl_array_grow_at_beg(jl_array_t *a, size_t idx, size_t inc,
#endif
a->nrows = newnrows;
a->data = newdata;
if (a->flags.ptrarray) {
if (a->flags.ptrarray || a->flags.hasptr) { // TODO: check for zeroinit
memset(newdata + idx * elsz, 0, nbinc);
}
else if (isbitsunion) {
Expand Down Expand Up @@ -895,7 +919,7 @@ STATIC_INLINE void jl_array_grow_at_end(jl_array_t *a, size_t idx,
a->length = newnrows;
#endif
a->nrows = newnrows;
if (a->flags.ptrarray) {
if (a->flags.ptrarray || a->flags.hasptr) { // TODO: check for zeroinit
memset(data + idx * elsz, 0, inc * elsz);
}
}
Expand Down Expand Up @@ -1134,7 +1158,7 @@ JL_DLLEXPORT jl_array_t *jl_array_copy(jl_array_t *ary)
int isunion = jl_is_uniontype(jl_tparam0(jl_typeof(ary)));
jl_array_t *new_ary = _new_array_(jl_typeof(ary), jl_array_ndims(ary),
&ary->nrows, !ary->flags.ptrarray,
isunion, elsz);
ary->flags.hasptr, isunion, elsz);
memcpy(new_ary->data, ary->data, len * elsz);
// ensure isbits union arrays copy their selector bytes correctly
if (jl_array_isbitsunion(ary))
Expand Down
Loading