diff --git a/base/idset.jl b/base/idset.jl index 0a4d4275b4231..6e0b7a85a5b23 100644 --- a/base/idset.jl +++ b/base/idset.jl @@ -1,13 +1,13 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -# Like Set, but using IdDict -mutable struct IdSet{T} <: AbstractSet{T} - dict::IdDict{T,Nothing} - - IdSet{T}() where {T} = new(IdDict{T,Nothing}()) - IdSet{T}(s::IdSet{T}) where {T} = new(copy(s.dict)) +mutable struct IdSet{K} <: AbstractSet{K} + list::Memory{Any} + idxs::Union{Memory{UInt8}, Memory{UInt16}, Memory{UInt32}} + count::Int + max::Int # n.b. always <= length(list) + IdSet{T}() where {T} = new(Memory{Any}(undef, 0), Memory{UInt8}(undef, 0), 0, 0) + IdSet{T}(s::IdSet{T}) where {T} = new(copy(s.list), copy(s.idxs), s.count, s.max) end - IdSet{T}(itr) where {T} = union!(IdSet{T}(), itr) IdSet() = IdSet{Any}() @@ -15,22 +15,77 @@ copymutable(s::IdSet) = typeof(s)(s) emptymutable(s::IdSet{T}, ::Type{U}=T) where {T,U} = IdSet{U}() copy(s::IdSet) = typeof(s)(s) -isempty(s::IdSet) = isempty(s.dict) -length(s::IdSet) = length(s.dict) -in(@nospecialize(x), s::IdSet) = haskey(s.dict, x) -push!(s::IdSet, @nospecialize(x)) = (s.dict[x] = nothing; s) -pop!(s::IdSet, @nospecialize(x)) = (pop!(s.dict, x); x) -pop!(s::IdSet, @nospecialize(x), @nospecialize(default)) = (x in s ? pop!(s, x) : default) -delete!(s::IdSet, @nospecialize(x)) = (delete!(s.dict, x); s) +haskey(s::IdSet, @nospecialize(key)) = ccall(:jl_idset_peek_bp, Int, (Any, Any, Any), s.list, s.idxs, key) != -1 +isempty(s::IdSet) = s.count == 0 +length(s::IdSet) = s.count +in(@nospecialize(x), s::IdSet) = haskey(s, x) +function push!(s::IdSet, @nospecialize(x)) + idx = ccall(:jl_idset_peek_bp, Int, (Any, Any, Any), s.list, s.idxs, x) + if idx >= 0 + s.list[idx + 1] = x + else + if s.max < length(s.list) + idx = s.max + @assert !isassigned(s.list, idx + 1) + s.list[idx + 1] = x + s.max = idx + 1 + else + newidx = RefValue{Int}(0) + setfield!(s, :list, ccall(:jl_idset_put_key, Any, (Any, Any, Ptr{Int}), s.list, x, newidx)) + idx = newidx[] + s.max = idx < 0 ? -idx : idx + 1 + end + @assert s.list[s.max] === x + setfield!(s, :idxs, ccall(:jl_idset_put_idx, Any, (Any, Any, Int), s.list, s.idxs, idx)) + s.count += 1 + end + s +end +function _pop!(s::IdSet, @nospecialize(x)) + removed = ccall(:jl_idset_pop, Int, (Any, Any, Any), s.list, s.idxs, x) + if removed != -1 + s.count -= 1 + while s.max > 0 && !isassigned(s.list, s.max) + s.max -= 1 + end + end + removed +end +pop!(s::IdSet, @nospecialize(x)) = _pop!(s, x) == -1 ? throw(KeyError(x)) : x +pop!(s::IdSet, @nospecialize(x), @nospecialize(default)) = _pop!(s, x) == -1 ? default : x +delete!(s::IdSet, @nospecialize(x)) = (_pop!(s, x); s) -sizehint!(s::IdSet, newsz) = (sizehint!(s.dict, newsz); s) -empty!(s::IdSet) = (empty!(s.dict); s) +function sizehint!(s::IdSet, newsz) + # TODO: grow/compact list and perform rehash, if profitable? + # TODO: shrink? + # s.list = resize(s.list, newsz) + # newsz = _tablesz(newsz) + # oldsz = length(s.idxs) + # #grow at least 25% + # if newsz < (oldsz*5)>>2 + # return s + # end + # rehash!(s, newsz) + nothing +end + +function empty!(s::IdSet) + fill!(s.idxs, 0x00) + list = s.list + for i = 1:s.max + _unsetindex!(list, i) + end + s.count = 0 + s.max = 0 + s +end filter!(f, d::IdSet) = unsafe_filter!(f, d) -function iterate(s::IdSet, state...) - y = iterate(s.dict, state...) - y === nothing && return nothing - ((k, _), i) = y - return (k, i) +function iterate(s::IdSet{S}, state=0) where {S} + while true + state += 1 + state > s.max && return nothing + isassigned(s.list, state) && return s.list[state]::S, state + end end diff --git a/src/Makefile b/src/Makefile index 4d105adec0dbb..5fc2bc2eaaa41 100644 --- a/src/Makefile +++ b/src/Makefile @@ -306,7 +306,7 @@ $(BUILDDIR)/julia_flisp.boot: $(addprefix $(SRCDIR)/,jlfrontend.scm flisp/aliase $(BUILDDIR)/codegen-stubs.o $(BUILDDIR)/codegen-stubs.dbg.obj: $(SRCDIR)/intrinsics.h $(BUILDDIR)/aotcompile.o $(BUILDDIR)/aotcompile.dbg.obj: $(SRCDIR)/jitlayers.h $(SRCDIR)/llvm-codegen-shared.h $(SRCDIR)/processor.h $(BUILDDIR)/ast.o $(BUILDDIR)/ast.dbg.obj: $(BUILDDIR)/julia_flisp.boot.inc $(SRCDIR)/flisp/*.h -$(BUILDDIR)/builtins.o $(BUILDDIR)/builtins.dbg.obj: $(SRCDIR)/iddict.c $(SRCDIR)/builtin_proto.h +$(BUILDDIR)/builtins.o $(BUILDDIR)/builtins.dbg.obj: $(SRCDIR)/iddict.c $(SRCDIR)/idset.c $(SRCDIR)/builtin_proto.h $(BUILDDIR)/codegen.o $(BUILDDIR)/codegen.dbg.obj: $(addprefix $(SRCDIR)/,\ intrinsics.cpp jitlayers.h intrinsics.h llvm-codegen-shared.h cgutils.cpp ccall.cpp abi_*.cpp processor.h builtin_proto.h) $(BUILDDIR)/datatype.o $(BUILDDIR)/datatype.dbg.obj: $(SRCDIR)/support/htable.h $(SRCDIR)/support/htable.inc diff --git a/src/builtins.c b/src/builtins.c index 04b45500fddaa..88b91baa19279 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -490,6 +490,7 @@ JL_DLLEXPORT uintptr_t jl_object_id(jl_value_t *v) JL_NOTSAFEPOINT // eq hash table -------------------------------------------------------------- #include "iddict.c" +#include "idset.c" // object model and type primitives ------------------------------------------- diff --git a/src/gc.c b/src/gc.c index bc0361c8ac7a8..38cebf151493d 100644 --- a/src/gc.c +++ b/src/gc.c @@ -3096,7 +3096,8 @@ static void gc_mark_roots(jl_gc_markqueue_t *mq) // constants gc_try_claim_and_push(mq, jl_emptytuple_type, NULL); gc_try_claim_and_push(mq, cmpswap_names, NULL); - gc_try_claim_and_push(mq, jl_global_roots_table, NULL); + gc_try_claim_and_push(mq, jl_global_roots_list, NULL); + gc_try_claim_and_push(mq, jl_global_roots_keyset, NULL); } // find unmarked objects that need to be finalized from the finalizer list "list". diff --git a/src/gf.c b/src/gf.c index 5013706030777..e5249e18d1d68 100644 --- a/src/gf.c +++ b/src/gf.c @@ -110,7 +110,7 @@ static int8_t jl_cachearg_offset(jl_methtable_t *mt) /// ----- Insertion logic for special entries ----- /// -static uint_t speccache_hash(size_t idx, jl_svec_t *data) +static uint_t speccache_hash(size_t idx, jl_value_t *data) { jl_method_instance_t *ml = (jl_method_instance_t*)jl_svecref(data, idx); jl_value_t *sig = ml->specTypes; @@ -119,7 +119,7 @@ static uint_t speccache_hash(size_t idx, jl_svec_t *data) return ((jl_datatype_t*)sig)->hash; } -static int speccache_eq(size_t idx, const void *ty, jl_svec_t *data, uint_t hv) +static int speccache_eq(size_t idx, const void *ty, jl_value_t *data, uint_t hv) { jl_method_instance_t *ml = (jl_method_instance_t*)jl_svecref(data, idx); jl_value_t *sig = ml->specTypes; @@ -139,7 +139,7 @@ static jl_method_instance_t *jl_specializations_get_linfo_(jl_method_t *m JL_PRO jl_value_t *ut = jl_is_unionall(type) ? jl_unwrap_unionall(type) : type; JL_TYPECHK(specializations, datatype, ut); uint_t hv = ((jl_datatype_t*)ut)->hash; - jl_array_t *speckeyset = NULL; + jl_genericmemory_t *speckeyset = NULL; jl_value_t *specializations = NULL; size_t i = -1, cl = 0, lastcl; for (int locked = 0; locked < 2; locked++) { @@ -164,7 +164,7 @@ static jl_method_instance_t *jl_specializations_get_linfo_(jl_method_t *m JL_PRO } cl = jl_svec_len(specializations); if (hv) { - ssize_t idx = jl_smallintset_lookup(speckeyset, speccache_eq, type, (jl_svec_t*)specializations, hv); + ssize_t idx = jl_smallintset_lookup(speckeyset, speccache_eq, type, specializations, hv, 0); if (idx != -1) { jl_method_instance_t *mi = (jl_method_instance_t*)jl_svecref(specializations, idx); if (locked) @@ -210,7 +210,7 @@ static jl_method_instance_t *jl_specializations_get_linfo_(jl_method_t *m JL_PRO jl_atomic_store_release(&m->specializations, specializations); jl_gc_wb(m, specializations); if (hv) - jl_smallintset_insert(&m->speckeyset, (jl_value_t*)m, speccache_hash, 0, (jl_svec_t*)specializations); + jl_smallintset_insert(&m->speckeyset, (jl_value_t*)m, speccache_hash, 0, specializations); } if (hv) { _Atomic(jl_method_instance_t*) *data = (_Atomic(jl_method_instance_t*)*)jl_svec_data(specializations); @@ -242,7 +242,7 @@ static jl_method_instance_t *jl_specializations_get_linfo_(jl_method_t *m JL_PRO assert(jl_svecref(specializations, i) == jl_nothing); jl_svecset(specializations, i, mi); if (hv) - jl_smallintset_insert(&m->speckeyset, (jl_value_t*)m, speccache_hash, i, (jl_svec_t*)specializations); + jl_smallintset_insert(&m->speckeyset, (jl_value_t*)m, speccache_hash, i, specializations); JL_GC_POP(); } JL_UNLOCK(&m->writelock); // may gc diff --git a/src/iddict.c b/src/iddict.c index eaa55657e2517..8d17f172b91fe 100644 --- a/src/iddict.c +++ b/src/iddict.c @@ -194,3 +194,4 @@ size_t jl_eqtable_nextind(jl_genericmemory_t *t, size_t i) #undef hash_size #undef max_probe +#undef h2index diff --git a/src/idset.c b/src/idset.c new file mode 100644 index 0000000000000..b9711ee17f021 --- /dev/null +++ b/src/idset.c @@ -0,0 +1,118 @@ +// This file is a part of Julia. License is MIT: https://julialang.org/license + + +static uint_t idset_hash(size_t idx, jl_value_t *data) +{ + jl_value_t *x = jl_genericmemory_ptr_ref(data, idx); + // x should not be NULL, unless there was concurrent corruption + return x == NULL ? 0 : jl_object_id(x); +} + +static int idset_eq(size_t idx, const void *y, jl_value_t *data, uint_t hv) +{ + jl_value_t *x = jl_genericmemory_ptr_ref(data, idx); + // x should not be NULL, unless there was concurrent corruption + return x == NULL ? 0 : jl_egal(x, (jl_value_t*)y); +} + +jl_genericmemory_t *jl_idset_rehash(jl_genericmemory_t *keys, jl_genericmemory_t *idxs, size_t newsz) +{ + if (newsz == 0) + return idxs; + newsz = next_power_of_two(newsz); + //if (idxs->length == newsz) + // jl_idset_put_idx(keys, idxs, -newsz+1); + //else + return smallintset_rehash(idxs, idset_hash, (jl_value_t*)keys, newsz, 0); +} + +// Return idx if key is in hash, otherwise -1 +// Note: lookup in the IdSet is permitted concurrently, if you avoid deletions, +// and assuming you do use an external lock around all insertions +ssize_t jl_idset_peek_bp(jl_genericmemory_t *keys, jl_genericmemory_t *idxs, jl_value_t *key) JL_NOTSAFEPOINT +{ + uintptr_t hv = jl_object_id(key); + return jl_smallintset_lookup(idxs, idset_eq, key, (jl_value_t*)keys, hv, 0); +} + +jl_value_t *jl_idset_get(jl_genericmemory_t *keys, jl_genericmemory_t *idxs, jl_value_t *key) JL_NOTSAFEPOINT +{ + ssize_t idx = jl_idset_peek_bp(keys, idxs, key); + if (idx == -1) + return NULL; + return jl_genericmemory_ptr_ref(keys, idx); +} + + +static ssize_t idset_compact(jl_genericmemory_t *keys) +{ + // compact keys before rehashing idxs + ssize_t i, j; + ssize_t rehash = 0; + for (i = j = 0; i < keys->length; i++) { + jl_value_t *k = jl_genericmemory_ptr_ref(keys, i); + if (k != NULL) { + if (i != j) { + rehash = 1; + jl_genericmemory_ptr_set(keys, j, k); + jl_genericmemory_ptr_set(keys, i, NULL); + } + j++; + } + } + return rehash ? -j : j; +} + +jl_genericmemory_t *jl_idset_put_key(jl_genericmemory_t *keys, jl_value_t *key, ssize_t *newidx) +{ + ssize_t l = keys->length; + ssize_t i = l; + while (i > 0 && jl_genericmemory_ptr_ref(keys, i - 1) == NULL) + i--; + // i points to the place to insert + *newidx = i; + if (i == l) { + i = idset_compact(keys); + if (i < 0) { + *newidx = i - 1; + i = -i; + } + if (i >= l / 3 * 2) { + size_t nl = l < 4 ? 4 : (l * 3) >> 1; // grow space by 50% if less than 33% free after compacting + jl_genericmemory_t *nk = jl_alloc_genericmemory(jl_memory_any_type, nl); + if (i > 0) + memcpy(nk->ptr, keys->ptr, sizeof(void*) * i); + keys = nk; + } + } + assert(jl_genericmemory_ptr_ref(keys, i) == NULL); + jl_genericmemory_ptr_set(keys, i, key); + return keys; +} + +jl_genericmemory_t *jl_idset_put_idx(jl_genericmemory_t *keys, jl_genericmemory_t *idxs, ssize_t idx) +{ + _Atomic(jl_genericmemory_t*) newidxs = idxs; + JL_GC_PUSH1(&newidxs); + if (idx < 0) { // full rehash + smallintset_empty(idxs); + for (ssize_t i = 0; i < -idx; i++) + if (jl_genericmemory_ptr_ref(keys, i) != NULL) + jl_smallintset_insert(&newidxs, NULL, idset_hash, i, (jl_value_t*)keys); + } + else { + jl_smallintset_insert(&newidxs, NULL, idset_hash, idx, (jl_value_t*)keys); + } + JL_GC_POP(); + return jl_atomic_load_relaxed(&newidxs); +} + +/* returns idx if key is in hash, otherwise -1 */ +ssize_t jl_idset_pop(jl_genericmemory_t *keys, jl_genericmemory_t *idxs, jl_value_t *key) JL_NOTSAFEPOINT +{ + uintptr_t hv = jl_object_id(key); + ssize_t idx = jl_smallintset_lookup(idxs, idset_eq, key, (jl_value_t*)keys, hv, 1); + if (idx != -1) + jl_genericmemory_ptr_set(keys, idx, NULL); + return idx; +} diff --git a/src/init.c b/src/init.c index 925ef0018048f..d4128c8ae9e40 100644 --- a/src/init.c +++ b/src/init.c @@ -861,7 +861,8 @@ static NOINLINE void _finish_julia_init(JL_IMAGE_SEARCH rel, jl_ptls_t ptls, jl_ jl_restore_system_image(jl_options.image_file); } else { jl_init_types(); - jl_global_roots_table = jl_alloc_memory_any(0); + jl_global_roots_list = (jl_genericmemory_t*)jl_an_empty_memory_any; + jl_global_roots_keyset = (jl_genericmemory_t*)jl_an_empty_memory_any; } jl_init_flisp(); diff --git a/src/jl_exported_data.inc b/src/jl_exported_data.inc index 1a843c71f0e1e..e840d2e8a40dc 100644 --- a/src/jl_exported_data.inc +++ b/src/jl_exported_data.inc @@ -73,12 +73,13 @@ XX(jl_genericmemory_type) \ XX(jl_genericmemory_typename) \ XX(jl_memory_uint8_type) \ + XX(jl_memory_uint16_type) \ + XX(jl_memory_uint32_type) \ XX(jl_memory_uint64_type) \ XX(jl_memoryref_any_type) \ XX(jl_genericmemoryref_type) \ XX(jl_genericmemoryref_typename) \ XX(jl_memoryref_uint8_type) \ - XX(jl_memoryref_uint64_type) \ XX(jl_methoderror_type) \ XX(jl_method_instance_type) \ XX(jl_method_match_type) \ diff --git a/src/jltypes.c b/src/jltypes.c index a30d746bbb320..5666319860f6a 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -2977,6 +2977,9 @@ void jl_init_types(void) JL_GC_DISABLED jl_memory_any_type = jl_apply_type3((jl_value_t*)jl_genericmemory_type, (jl_value_t*)jl_not_atomic_sym, (jl_value_t*)jl_any_type, cpumem); jl_memory_uint8_type = jl_apply_type3((jl_value_t*)jl_genericmemory_type, (jl_value_t*)jl_not_atomic_sym, (jl_value_t*)jl_uint8_type, cpumem); + jl_memory_uint16_type = jl_apply_type3((jl_value_t*)jl_genericmemory_type, (jl_value_t*)jl_not_atomic_sym, (jl_value_t*)jl_uint16_type, cpumem); + jl_memory_uint32_type = jl_apply_type3((jl_value_t*)jl_genericmemory_type, (jl_value_t*)jl_not_atomic_sym, (jl_value_t*)jl_uint32_type, cpumem); + jl_memory_uint64_type = jl_apply_type3((jl_value_t*)jl_genericmemory_type, (jl_value_t*)jl_not_atomic_sym, (jl_value_t*)jl_uint64_type, cpumem); jl_memoryref_any_type = jl_apply_type3((jl_value_t*)jl_genericmemoryref_type, (jl_value_t*)jl_not_atomic_sym, (jl_value_t*)jl_any_type, cpumem); jl_memoryref_uint8_type = jl_apply_type3((jl_value_t*)jl_genericmemoryref_type, (jl_value_t*)jl_not_atomic_sym, (jl_value_t*)jl_uint8_type, cpumem); @@ -3004,7 +3007,7 @@ void jl_init_types(void) JL_GC_DISABLED // finish initializing module Core core = jl_core_module; - jl_atomic_store_relaxed(&core->bindingkeyset, (jl_array_t*)jl_an_empty_vec_any); + jl_atomic_store_relaxed(&core->bindingkeyset, (jl_genericmemory_t*)jl_an_empty_memory_any); // export own name, so "using Foo" makes "Foo" itself visible jl_set_const(core, core->name, (jl_value_t*)core); jl_module_public(core, core->name, 1); @@ -3180,7 +3183,7 @@ void jl_init_types(void) JL_GC_DISABLED jl_ulong_type, jl_type_type, jl_any_type, // union(jl_simplevector_type, jl_method_instance_type), - jl_array_type, + jl_genericmemory_type, // union(jl_memory_uint8_type, jl_memory_uint16_type, jl_memory_uint32_type, jl_memory_uint64_type, jl_memory_any_type) jl_string_type, jl_any_type, jl_any_type, diff --git a/src/julia.h b/src/julia.h index d728458f50180..409da556ed957 100644 --- a/src/julia.h +++ b/src/julia.h @@ -320,7 +320,7 @@ typedef struct _jl_method_t { // table of all jl_method_instance_t specializations we have _Atomic(jl_value_t*) specializations; // allocated as [hashable, ..., NULL, linear, ....], or a single item - _Atomic(jl_array_t*) speckeyset; // index lookup by hash into specializations + _Atomic(jl_genericmemory_t*) speckeyset; // index lookup by hash into specializations jl_value_t *slot_syms; // compacted list of slot names (String) jl_value_t *external_mt; // reference to the method table this method is part of, null if part of the internal table @@ -611,7 +611,7 @@ typedef struct _jl_module_t { jl_sym_t *name; struct _jl_module_t *parent; _Atomic(jl_svec_t*) bindings; - _Atomic(jl_array_t*) bindingkeyset; // index lookup by name into bindings + _Atomic(jl_genericmemory_t*) bindingkeyset; // index lookup by name into bindings // hidden fields: arraylist_t usings; // modules with all bindings potentially imported jl_uuid_t build_id; @@ -881,6 +881,9 @@ extern JL_DLLIMPORT jl_value_t *jl_array_int32_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_value_t *jl_array_uint32_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_value_t *jl_array_uint64_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_value_t *jl_memory_uint8_type JL_GLOBALLY_ROOTED; +extern JL_DLLIMPORT jl_value_t *jl_memory_uint16_type JL_GLOBALLY_ROOTED; +extern JL_DLLIMPORT jl_value_t *jl_memory_uint32_type JL_GLOBALLY_ROOTED; +extern JL_DLLIMPORT jl_value_t *jl_memory_uint64_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_value_t *jl_memory_any_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_value_t *jl_memoryref_uint8_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_value_t *jl_memoryref_any_type JL_GLOBALLY_ROOTED; diff --git a/src/julia_internal.h b/src/julia_internal.h index 45b1f893ef71a..da025a900400e 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -805,7 +805,8 @@ JL_DLLEXPORT void jl_binding_deprecation_warning(jl_module_t *m, jl_sym_t *sym, extern jl_array_t *jl_module_init_order JL_GLOBALLY_ROOTED; extern htable_t jl_current_modules JL_GLOBALLY_ROOTED; extern JL_DLLEXPORT jl_module_t *jl_precompile_toplevel_module JL_GLOBALLY_ROOTED; -extern jl_genericmemory_t *jl_global_roots_table JL_GLOBALLY_ROOTED; +extern jl_genericmemory_t *jl_global_roots_list JL_GLOBALLY_ROOTED; +extern jl_genericmemory_t *jl_global_roots_keyset JL_GLOBALLY_ROOTED; JL_DLLEXPORT int jl_is_globally_rooted(jl_value_t *val JL_MAYBE_UNROOTED) JL_NOTSAFEPOINT; JL_DLLEXPORT jl_value_t *jl_as_global_root(jl_value_t *val JL_MAYBE_UNROOTED); @@ -1420,10 +1421,19 @@ void jl_safepoint_resume_thread_mach(jl_ptls_t ptls2, int16_t tid2) JL_NOTSAFEPO // -- smallintset.c -- // -typedef uint_t (*smallintset_hash)(size_t val, jl_svec_t *data); -typedef int (*smallintset_eq)(size_t val, const void *key, jl_svec_t *data, uint_t hv); -ssize_t jl_smallintset_lookup(jl_array_t *cache, smallintset_eq eq, const void *key, jl_svec_t *data, uint_t hv); -void jl_smallintset_insert(_Atomic(jl_array_t*) *pcache, jl_value_t *parent, smallintset_hash hash, size_t val, jl_svec_t *data); +typedef uint_t (*smallintset_hash)(size_t val, jl_value_t *data); +typedef int (*smallintset_eq)(size_t val, const void *key, jl_value_t *data, uint_t hv); +ssize_t jl_smallintset_lookup(jl_genericmemory_t *cache, smallintset_eq eq, const void *key, jl_value_t *data, uint_t hv, int pop); +void jl_smallintset_insert(_Atomic(jl_genericmemory_t*) *pcache, jl_value_t *parent, smallintset_hash hash, size_t val, jl_value_t *data); +jl_genericmemory_t* smallintset_rehash(jl_genericmemory_t* a, smallintset_hash hash, jl_value_t *data, size_t newsz, size_t np); +void smallintset_empty(const jl_genericmemory_t *a) JL_NOTSAFEPOINT; + +JL_DLLEXPORT jl_genericmemory_t *jl_idset_rehash(jl_genericmemory_t *keys, jl_genericmemory_t *idxs, size_t newsz); +JL_DLLEXPORT ssize_t jl_idset_peek_bp(jl_genericmemory_t *keys, jl_genericmemory_t *idxs, jl_value_t *key) JL_NOTSAFEPOINT; +jl_value_t *jl_idset_get(jl_genericmemory_t *keys, jl_genericmemory_t *idxs, jl_value_t *key) JL_NOTSAFEPOINT; +JL_DLLEXPORT jl_genericmemory_t *jl_idset_put_key(jl_genericmemory_t *keys, jl_value_t *key, ssize_t *newidx); +JL_DLLEXPORT jl_genericmemory_t *jl_idset_put_idx(jl_genericmemory_t *keys, jl_genericmemory_t *idxs, ssize_t idx); +JL_DLLEXPORT ssize_t jl_idset_pop(jl_genericmemory_t *keys, jl_genericmemory_t *idxs, jl_value_t *key) JL_NOTSAFEPOINT; // -- typemap.c -- // diff --git a/src/method.c b/src/method.c index 8b89e66c62eb0..9cb7e83d57c1c 100644 --- a/src/method.c +++ b/src/method.c @@ -795,7 +795,7 @@ JL_DLLEXPORT jl_method_t *jl_new_method_uninit(jl_module_t *module) jl_method_t *m = (jl_method_t*)jl_gc_alloc(ct->ptls, sizeof(jl_method_t), jl_method_type); jl_atomic_store_relaxed(&m->specializations, (jl_value_t*)jl_emptysvec); - jl_atomic_store_relaxed(&m->speckeyset, (jl_array_t*)jl_an_empty_vec_any); + jl_atomic_store_relaxed(&m->speckeyset, (jl_genericmemory_t*)jl_an_empty_memory_any); m->sig = NULL; m->slot_syms = NULL; m->roots = NULL; diff --git a/src/module.c b/src/module.c index 04ddb4f4fb3ae..47309c897ae63 100644 --- a/src/module.c +++ b/src/module.c @@ -39,7 +39,7 @@ JL_DLLEXPORT jl_module_t *jl_new_module_(jl_sym_t *name, jl_module_t *parent, ui bitmix(name->hash, parent->hash); JL_MUTEX_INIT(&m->lock, "module->lock"); jl_atomic_store_relaxed(&m->bindings, jl_emptysvec); - jl_atomic_store_relaxed(&m->bindingkeyset, (jl_array_t*)jl_an_empty_vec_any); + jl_atomic_store_relaxed(&m->bindingkeyset, (jl_genericmemory_t*)jl_an_empty_memory_any); arraylist_new(&m->usings, 0); if (jl_core_module && default_names) { JL_GC_PUSH1(&m); @@ -699,14 +699,14 @@ JL_DLLEXPORT int jl_binding_resolved_p(jl_module_t *m, jl_sym_t *var) return b && jl_atomic_load_relaxed(&b->owner) != NULL; } -static uint_t bindingkey_hash(size_t idx, jl_svec_t *data) +static uint_t bindingkey_hash(size_t idx, jl_value_t *data) { jl_binding_t *b = (jl_binding_t*)jl_svecref(data, idx); jl_sym_t *var = b->globalref->name; return var->hash; } -static int bindingkey_eq(size_t idx, const void *var, jl_svec_t *data, uint_t hv) +static int bindingkey_eq(size_t idx, const void *var, jl_value_t *data, uint_t hv) { jl_binding_t *b = (jl_binding_t*)jl_svecref(data, idx); jl_sym_t *name = b->globalref->name; @@ -717,9 +717,9 @@ JL_DLLEXPORT jl_binding_t *jl_get_module_binding(jl_module_t *m, jl_sym_t *var, { uint_t hv = var->hash; for (int locked = 0; ; locked++) { - jl_array_t *bindingkeyset = jl_atomic_load_acquire(&m->bindingkeyset); + jl_genericmemory_t *bindingkeyset = jl_atomic_load_acquire(&m->bindingkeyset); jl_svec_t *bindings = jl_atomic_load_relaxed(&m->bindings); - ssize_t idx = jl_smallintset_lookup(bindingkeyset, bindingkey_eq, var, bindings, hv); // acquire + ssize_t idx = jl_smallintset_lookup(bindingkeyset, bindingkey_eq, var, (jl_value_t*)bindings, hv, 0); // acquire if (idx != -1) { jl_binding_t *b = (jl_binding_t*)jl_svecref(bindings, idx); // relaxed if (locked) @@ -753,7 +753,7 @@ JL_DLLEXPORT jl_binding_t *jl_get_module_binding(jl_module_t *m, jl_sym_t *var, jl_binding_t *b = new_binding(m, var); assert(jl_svecref(bindings, i) == jl_nothing); jl_svecset(bindings, i, b); // relaxed - jl_smallintset_insert(&m->bindingkeyset, (jl_value_t*)m, bindingkey_hash, i, bindings); // release + jl_smallintset_insert(&m->bindingkeyset, (jl_value_t*)m, bindingkey_hash, i, (jl_value_t*)bindings); // release JL_UNLOCK(&m->lock); return b; } diff --git a/src/smallintset.c b/src/smallintset.c index b3af6e45c7834..df67239f79fb5 100644 --- a/src/smallintset.c +++ b/src/smallintset.c @@ -24,91 +24,103 @@ extern "C" { #endif -static inline size_t jl_intref(const jl_array_t *arr, size_t idx) JL_NOTSAFEPOINT +static inline size_t ignore_tombstone(size_t val, size_t tombstone) JL_NOTSAFEPOINT { - jl_value_t *el = jl_tparam0(jl_typeof(arr)); - if (el == (jl_value_t*)jl_uint8_type) - return jl_atomic_load_relaxed(&jl_array_data(arr, _Atomic(uint8_t))[idx]); - else if (el == (jl_value_t*)jl_uint16_type) - return jl_atomic_load_relaxed(&jl_array_data(arr, _Atomic(uint16_t))[idx]); - else if (el == (jl_value_t*)jl_uint32_type) - return jl_atomic_load_relaxed(&jl_array_data(arr, _Atomic(uint32_t))[idx]); + return val == tombstone ? 0 : val; +} +static inline size_t jl_intref(const jl_genericmemory_t *arr, size_t idx) JL_NOTSAFEPOINT +{ + jl_value_t *el = (jl_value_t*)jl_typetagof(arr); + if (el == jl_memory_uint8_type) + return ignore_tombstone(jl_atomic_load_relaxed(&((_Atomic(uint8_t)*)arr->ptr)[idx]), (uint8_t)-1); + else if (el == jl_memory_uint16_type) + return ignore_tombstone(jl_atomic_load_relaxed(&((_Atomic(uint16_t)*)arr->ptr)[idx]), (uint16_t)-1); + else if (el == jl_memory_uint32_type) + return ignore_tombstone(jl_atomic_load_relaxed(&((_Atomic(uint32_t)*)arr->ptr)[idx]), (uint32_t)-1); else abort(); } -static inline size_t jl_intref_acquire(const jl_array_t *arr, size_t idx) JL_NOTSAFEPOINT +static inline size_t acquire_tombstone(size_t val, size_t tombstone) JL_NOTSAFEPOINT { - jl_value_t *el = jl_tparam0(jl_typeof(arr)); - if (el == (jl_value_t*)jl_uint8_type) - return jl_atomic_load_acquire(&jl_array_data(arr, _Atomic(uint8_t))[idx]); - else if (el == (jl_value_t*)jl_uint16_type) - return jl_atomic_load_acquire(&jl_array_data(arr, _Atomic(uint16_t))[idx]); - else if (el == (jl_value_t*)jl_uint32_type) - return jl_atomic_load_acquire(&jl_array_data(arr, _Atomic(uint32_t))[idx]); + return val == tombstone ? (size_t)-1 : val; +} +static inline size_t jl_intref_acquire(const jl_genericmemory_t *arr, size_t idx) JL_NOTSAFEPOINT +{ + jl_value_t *el = (jl_value_t*)jl_typetagof(arr); + if (el == jl_memory_uint8_type) + return acquire_tombstone(jl_atomic_load_acquire(&((_Atomic(uint8_t)*)arr->ptr)[idx]), (uint8_t)-1); + else if (el == jl_memory_uint16_type) + return acquire_tombstone(jl_atomic_load_acquire(&((_Atomic(uint16_t)*)arr->ptr)[idx]), (uint16_t)-1); + else if (el == jl_memory_uint32_type) + return acquire_tombstone(jl_atomic_load_acquire(&((_Atomic(uint32_t)*)arr->ptr)[idx]), (uint32_t)-1); else abort(); } -static inline void jl_intset_release(const jl_array_t *arr, size_t idx, size_t val) JL_NOTSAFEPOINT +static inline void jl_intset_release(const jl_genericmemory_t *arr, size_t idx, size_t val) JL_NOTSAFEPOINT { - jl_value_t *el = jl_tparam0(jl_typeof(arr)); - if (el == (jl_value_t*)jl_uint8_type) - jl_atomic_store_release(&jl_array_data(arr, _Atomic(uint8_t))[idx], val); - else if (el == (jl_value_t*)jl_uint16_type) - jl_atomic_store_release(&jl_array_data(arr, _Atomic(uint16_t))[idx], val); - else if (el == (jl_value_t*)jl_uint32_type) - jl_atomic_store_release(&jl_array_data(arr, _Atomic(uint32_t))[idx], val); + jl_value_t *el = (jl_value_t*)jl_typetagof(arr); + if (el == jl_memory_uint8_type) + jl_atomic_store_release(&((_Atomic(uint8_t)*)arr->ptr)[idx], val); + else if (el == jl_memory_uint16_type) + jl_atomic_store_release(&((_Atomic(uint16_t)*)arr->ptr)[idx], val); + else if (el == jl_memory_uint32_type) + jl_atomic_store_release(&((_Atomic(uint32_t)*)arr->ptr)[idx], val); else abort(); } -static inline size_t jl_max_int(const jl_array_t *arr) +static inline size_t jl_max_int(const jl_genericmemory_t *arr) JL_NOTSAFEPOINT { - jl_value_t *el = jl_tparam0(jl_typeof(arr)); - if (el == (jl_value_t*)jl_uint8_type) + jl_value_t *el = (jl_value_t*)jl_typetagof(arr); + if (el == jl_memory_uint8_type) return 0xFF; - else if (el == (jl_value_t*)jl_uint16_type) + else if (el == jl_memory_uint16_type) return 0xFFFF; - else if (el == (jl_value_t*)jl_uint32_type) + else if (el == jl_memory_uint32_type) return 0xFFFFFFFF; - else if (el == (jl_value_t*)jl_any_type) + else if (el == jl_memory_any_type) return 0; else abort(); } -static jl_array_t *jl_alloc_int_1d(size_t np, size_t len) +void smallintset_empty(const jl_genericmemory_t *a) JL_NOTSAFEPOINT { - jl_value_t *ty; size_t elsize; - if (np < 0xFF) { - ty = jl_array_uint8_type; + jl_value_t *el = (jl_value_t*)jl_typetagof(a); + if (el == jl_memory_uint8_type) elsize = sizeof(uint8_t); - } - else if (np < 0xFFFF) { - static jl_value_t *int16 JL_ALWAYS_LEAFTYPE = NULL; - if (int16 == NULL) - int16 = jl_apply_array_type((jl_value_t*)jl_uint16_type, 1); - ty = int16; + else if (el == jl_memory_uint16_type) elsize = sizeof(uint16_t); - } - else { - assert(np < 0x7FFFFFFF); - static jl_value_t *int32 JL_ALWAYS_LEAFTYPE = NULL; - if (int32 == NULL) - int32 = jl_apply_array_type((jl_value_t*)jl_uint32_type, 1); - ty = int32; + else if (el == jl_memory_uint32_type) elsize = sizeof(uint32_t); - } - jl_array_t *a = jl_alloc_array_1d(ty, len); - memset(jl_array_data(a, char), 0, len * elsize); + else if (el == jl_memory_any_type) + elsize = 0; + else + abort(); + memset(a->ptr, 0, a->length * elsize); +} + +static jl_genericmemory_t *jl_alloc_int_1d(size_t np, size_t len) +{ + jl_value_t *ty; + if (np < 0xFF) + ty = jl_memory_uint8_type; + else if (np < 0xFFFF) + ty = jl_memory_uint16_type; + else + ty = jl_memory_uint32_type; + assert(np < 0x7FFFFFFF); + jl_genericmemory_t *a = jl_alloc_genericmemory(ty, len); + smallintset_empty(a); return a; } -ssize_t jl_smallintset_lookup(jl_array_t *cache, smallintset_eq eq, const void *key, jl_svec_t *data, uint_t hv) +ssize_t jl_smallintset_lookup(jl_genericmemory_t *cache, smallintset_eq eq, const void *key, jl_value_t *data, uint_t hv, int pop) { - size_t sz = jl_array_nrows(cache); + size_t sz = cache->length; if (sz == 0) return -1; JL_GC_PUSH1(&cache); @@ -122,8 +134,10 @@ ssize_t jl_smallintset_lookup(jl_array_t *cache, smallintset_eq eq, const void * JL_GC_POP(); return -1; } - if (eq(val1 - 1, key, data, hv)) { + if (val1 != -1 && eq(val1 - 1, key, data, hv)) { JL_GC_POP(); + if (pop) + jl_intset_release(cache, index, (size_t)-1); // replace with tombstone return val1 - 1; } index = (index + 1) & (sz - 1); @@ -133,9 +147,9 @@ ssize_t jl_smallintset_lookup(jl_array_t *cache, smallintset_eq eq, const void * return -1; } -static int smallintset_insert_(jl_array_t *a, uint_t hv, size_t val1) +static int smallintset_insert_(jl_genericmemory_t *a, uint_t hv, size_t val1) JL_NOTSAFEPOINT { - size_t sz = jl_array_nrows(a); + size_t sz = a->length; if (sz <= 1) return 0; size_t orig, index, iter; @@ -153,16 +167,17 @@ static int smallintset_insert_(jl_array_t *a, uint_t hv, size_t val1) } while (iter <= maxprobe && index != orig); return 0; } +//} -static void smallintset_rehash(_Atomic(jl_array_t*) *pcache, jl_value_t *parent, smallintset_hash hash, jl_svec_t *data, size_t newsz, size_t np); - -void jl_smallintset_insert(_Atomic(jl_array_t*) *pcache, jl_value_t *parent, smallintset_hash hash, size_t val, jl_svec_t *data) +void jl_smallintset_insert(_Atomic(jl_genericmemory_t*) *pcache, jl_value_t *parent, smallintset_hash hash, size_t val, jl_value_t *data) { - jl_array_t *a = jl_atomic_load_relaxed(pcache); - if (val + 1 > jl_max_int(a)) - smallintset_rehash(pcache, parent, hash, data, jl_array_nrows(a), val + 1); + jl_genericmemory_t *a = jl_atomic_load_relaxed(pcache); + if (val + 1 >= jl_max_int(a)) { + a = smallintset_rehash(a, hash, data, a->length, val + 1); + jl_atomic_store_release(pcache, a); + if (parent) jl_gc_wb(parent, a); + } while (1) { - a = jl_atomic_load_relaxed(pcache); if (smallintset_insert_(a, hash(val, data), val + 1)) return; @@ -172,21 +187,22 @@ void jl_smallintset_insert(_Atomic(jl_array_t*) *pcache, jl_value_t *parent, sma /* lots of time rehashing all the keys over and over. */ size_t newsz; a = jl_atomic_load_relaxed(pcache); - size_t sz = jl_array_nrows(a); + size_t sz = a->length; if (sz < HT_N_INLINE) newsz = HT_N_INLINE; else if (sz >= (1 << 19) || (sz <= (1 << 8))) newsz = sz << 1; else newsz = sz << 2; - smallintset_rehash(pcache, parent, hash, data, newsz, 0); + a = smallintset_rehash(a, hash, data, newsz, 0); + jl_atomic_store_release(pcache, a); + if (parent) jl_gc_wb(parent, a); } } -static void smallintset_rehash(_Atomic(jl_array_t*) *pcache, jl_value_t *parent, smallintset_hash hash, jl_svec_t *data, size_t newsz, size_t np) +jl_genericmemory_t* smallintset_rehash(jl_genericmemory_t* a, smallintset_hash hash, jl_value_t *data, size_t newsz, size_t np) { - jl_array_t *a = jl_atomic_load_relaxed(pcache); - size_t sz = jl_array_nrows(a); + size_t sz = a->length; size_t i; for (i = 0; i < sz; i += 1) { size_t val = jl_intref(a, i); @@ -194,7 +210,7 @@ static void smallintset_rehash(_Atomic(jl_array_t*) *pcache, jl_value_t *parent, np = val; } while (1) { - jl_array_t *newa = jl_alloc_int_1d(np, newsz); + jl_genericmemory_t *newa = jl_alloc_int_1d(np + 1, newsz); JL_GC_PUSH1(&newa); for (i = 0; i < sz; i += 1) { size_t val1 = jl_intref(a, i); @@ -205,16 +221,12 @@ static void smallintset_rehash(_Atomic(jl_array_t*) *pcache, jl_value_t *parent, } } JL_GC_POP(); - if (i == sz) { - jl_atomic_store_release(pcache, newa); - jl_gc_wb(parent, newa); - return; - } + if (i == sz) + return newa; newsz <<= 1; } } - #ifdef __cplusplus } #endif diff --git a/src/staticdata.c b/src/staticdata.c index 52a354b4933d7..6f21d0aa4aa2c 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -98,7 +98,7 @@ extern "C" { // TODO: put WeakRefs on the weak_refs list during deserialization // TODO: handle finalizers -#define NUM_TAGS 175 +#define NUM_TAGS 178 // An array of references that need to be restored from the sysimg // This is a manually constructed dual of the gvars array, which would be produced by codegen for Julia code, for C. @@ -201,6 +201,9 @@ jl_value_t **const*const get_tags(void) { INSERT_TAG(jl_genericmemory_type); INSERT_TAG(jl_memory_any_type); INSERT_TAG(jl_memory_uint8_type); + INSERT_TAG(jl_memory_uint16_type); + INSERT_TAG(jl_memory_uint32_type); + INSERT_TAG(jl_memory_uint64_type); INSERT_TAG(jl_genericmemoryref_type); INSERT_TAG(jl_memoryref_any_type); INSERT_TAG(jl_memoryref_uint8_type); @@ -2420,7 +2423,8 @@ static void jl_strip_all_codeinfos(void) // --- entry points --- -jl_genericmemory_t *jl_global_roots_table; +jl_genericmemory_t *jl_global_roots_list; +jl_genericmemory_t *jl_global_roots_keyset; jl_mutex_t global_roots_lock; JL_DLLEXPORT int jl_is_globally_rooted(jl_value_t *val JL_MAYBE_UNROOTED) JL_NOTSAFEPOINT @@ -2449,14 +2453,17 @@ JL_DLLEXPORT jl_value_t *jl_as_global_root(jl_value_t *val JL_MAYBE_UNROOTED) if ((uint64_t)(n+512) < 1024) return jl_box_int64(n); } + // TODO: check table before acquiring lock to reduce writer contention JL_GC_PUSH1(&val); JL_LOCK(&global_roots_lock); - jl_value_t *rval = jl_eqtable_getkey(jl_global_roots_table, val, NULL); + jl_value_t *rval = jl_idset_get(jl_global_roots_list, jl_global_roots_keyset, val); if (rval) { val = rval; } else { - jl_global_roots_table = jl_eqtable_put(jl_global_roots_table, val, jl_nothing, NULL); + ssize_t idx; + jl_global_roots_list = jl_idset_put_key(jl_global_roots_list, val, &idx); + jl_global_roots_keyset = jl_idset_put_idx(jl_global_roots_list, jl_global_roots_keyset, idx); } JL_UNLOCK(&global_roots_lock); JL_GC_POP(); @@ -2597,7 +2604,7 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, jl_docmeta_sym = (jl_sym_t*)jl_get_global((jl_module_t*)docs, jl_symbol("META")); } } - jl_genericmemory_t *global_roots_table = NULL; + jl_genericmemory_t *global_roots_list = NULL; { // step 1: record values (recursively) that need to go in the image size_t i; @@ -2641,15 +2648,17 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, record_gvars(&s, &gvars); record_external_fns(&s, &external_fns); jl_serialize_reachable(&s); - // step 1.3: prune (garbage collect) special weak references from the jl_global_roots_table + // step 1.3: prune (garbage collect) special weak references from the jl_global_roots_list if (worklist == NULL) { - global_roots_table = jl_alloc_memory_any(0); - for (size_t i = 0; i < jl_global_roots_table->length; i += 2) { - jl_value_t *val = jl_genericmemory_ptr_ref(jl_global_roots_table, i); - if (ptrhash_get(&serialization_order, val) != HT_NOTFOUND) - global_roots_table = jl_eqtable_put(global_roots_table, val, jl_nothing, NULL); + global_roots_list = jl_alloc_memory_any(0); + for (size_t i = 0; i < jl_global_roots_list->length; i++) { + jl_value_t *val = jl_genericmemory_ptr_ref(jl_global_roots_list, i); + if (val && ptrhash_get(&serialization_order, val) != HT_NOTFOUND) { + ssize_t idx; + global_roots_list = jl_idset_put_key(global_roots_list, val, &idx); + } } - jl_queue_for_serialization(&s, global_roots_table); + jl_queue_for_serialization(&s, global_roots_list); jl_serialize_reachable(&s); } // step 1.4: prune (garbage collect) some special weak references from @@ -2761,7 +2770,7 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_array_t *mod_array, jl_value_t *tag = *tags[i]; jl_write_value(&s, tag); } - jl_write_value(&s, global_roots_table); + jl_write_value(&s, global_roots_list); jl_write_value(&s, s.ptls->root_task->tls); write_uint32(f, jl_get_gs_ctr()); write_uint(f, jl_atomic_load_acquire(&jl_world_counter)); @@ -3072,7 +3081,7 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl JL_SMALL_TYPEOF(XX) #undef XX export_jl_small_typeof(); - jl_global_roots_table = (jl_genericmemory_t*)jl_read_value(&s); + jl_global_roots_list = (jl_genericmemory_t*)jl_read_value(&s); // set typeof extra-special values now that we have the type set by tags above jl_astaggedvalue(jl_nothing)->header = (uintptr_t)jl_nothing_type | jl_astaggedvalue(jl_nothing)->header; s.ptls->root_task->tls = jl_read_value(&s); @@ -3370,6 +3379,8 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl o->bits.in_image = 1; } arraylist_free(&cleanup_list); + if (!s.incremental && jl_global_roots_list->length > 0) + jl_global_roots_keyset = jl_idset_put_idx(jl_global_roots_list, (jl_genericmemory_t*)jl_an_empty_memory_any, -jl_global_roots_list->length); for (size_t i = 0; i < s.fixup_objs.len; i++) { uintptr_t item = (uintptr_t)s.fixup_objs.items[i]; jl_value_t *obj = (jl_value_t*)(image_base + item); diff --git a/sysimage.mk b/sysimage.mk index 44c5762ec8b81..e6fa54be5f186 100644 --- a/sysimage.mk +++ b/sysimage.mk @@ -26,6 +26,9 @@ COMPILER_SRCS := $(addprefix $(JULIAHOME)/, \ base/docs/core.jl \ base/abstractarray.jl \ base/abstractdict.jl \ + base/abstractset.jl \ + base/iddict.jl \ + base/idset.jl \ base/array.jl \ base/bitarray.jl \ base/bitset.jl \