From ae2aad593f9175f8f835d724b16898e2b91237b0 Mon Sep 17 00:00:00 2001 From: Petr Vana Date: Thu, 24 Feb 2022 16:19:44 +0100 Subject: [PATCH 01/18] Improve performance of 'Dict' --- base/dict.jl | 118 ++++++++++++++------------------ base/set.jl | 17 +++-- base/weakkeydict.jl | 2 +- stdlib/Random/src/generation.jl | 2 +- test/sets.jl | 8 +-- 5 files changed, 69 insertions(+), 78 deletions(-) diff --git a/base/dict.jl b/base/dict.jl index dabdfa5c34773..ab2ae377d8684 100644 --- a/base/dict.jl +++ b/base/dict.jl @@ -76,9 +76,8 @@ Dict{String, Int64} with 2 entries: ``` """ mutable struct Dict{K,V} <: AbstractDict{K,V} - slots::Array{UInt8,1} - keys::Array{K,1} - vals::Array{V,1} + slots::Vector{UInt8} + pairs::Vector{Pair{K,V}} # stored pairs (key::K => value::V) ndel::Int count::Int age::UInt @@ -87,14 +86,13 @@ mutable struct Dict{K,V} <: AbstractDict{K,V} function Dict{K,V}() where V where K n = 16 - new(zeros(UInt8,n), Vector{K}(undef, n), Vector{V}(undef, n), 0, 0, 0, 1, 0) + new(zeros(UInt8,n), Vector{Pair{K,V}}(undef, n), 0, 0, 0, 1, 0) end function Dict{K,V}(d::Dict{K,V}) where V where K - new(copy(d.slots), copy(d.keys), copy(d.vals), d.ndel, d.count, d.age, - d.idxfloor, d.maxprobe) + new(copy(d.slots), copy(d.pairs), d.ndel, d.count, d.age, d.idxfloor, d.maxprobe) end - function Dict{K, V}(slots, keys, vals, ndel, count, age, idxfloor, maxprobe) where {K, V} - new(slots, keys, vals, ndel, count, age, idxfloor, maxprobe) + function Dict{K, V}(slots, pairs, ndel, count, age, idxfloor, maxprobe) where {K, V} + new(slots, pairs, ndel, count, age, idxfloor, maxprobe) end end function Dict{K,V}(kv) where V where K @@ -172,10 +170,9 @@ hashindex(key, sz) = (((hash(key)::UInt % Int) & (sz-1)) + 1)::Int @propagate_inbounds isslotfilled(h::Dict, i::Int) = h.slots[i] == 0x1 @propagate_inbounds isslotmissing(h::Dict, i::Int) = h.slots[i] == 0x2 -@constprop :none function rehash!(h::Dict{K,V}, newsz = length(h.keys)) where V where K +@constprop :none function rehash!(h::Dict{K,V}, newsz = length(h.pairs)) where V where K olds = h.slots - oldk = h.keys - oldv = h.vals + oldp = h.pairs sz = length(olds) newsz = _tablesz(newsz) h.age += 1 @@ -183,32 +180,27 @@ hashindex(key, sz) = (((hash(key)::UInt % Int) & (sz-1)) + 1)::Int if h.count == 0 resize!(h.slots, newsz) fill!(h.slots, 0) - resize!(h.keys, newsz) - resize!(h.vals, newsz) + resize!(h.pairs, newsz) h.ndel = 0 return h end slots = zeros(UInt8,newsz) - keys = Vector{K}(undef, newsz) - vals = Vector{V}(undef, newsz) + pairs = Vector{Pair{K,V}}(undef, newsz) age0 = h.age count = 0 maxprobe = 0 for i = 1:sz @inbounds if olds[i] == 0x1 - k = oldk[i] - v = oldv[i] - index0 = index = hashindex(k, newsz) + index0 = index = hashindex(oldp[i].first, newsz) while slots[index] != 0 index = (index & (newsz-1)) + 1 end probe = (index - index0) & (newsz-1) probe > maxprobe && (maxprobe = probe) slots[index] = 0x1 - keys[index] = k - vals[index] = v + pairs[index] = oldp[i] count += 1 if h.age != age0 @@ -219,8 +211,7 @@ hashindex(key, sz) = (((hash(key)::UInt % Int) & (sz-1)) + 1)::Int end h.slots = slots - h.keys = keys - h.vals = vals + h.pairs = pairs h.count = count h.ndel = 0 h.maxprobe = maxprobe @@ -265,10 +256,8 @@ Dict{String, Int64}() function empty!(h::Dict{K,V}) where V where K fill!(h.slots, 0x0) sz = length(h.slots) - empty!(h.keys) - empty!(h.vals) - resize!(h.keys, sz) - resize!(h.vals, sz) + empty!(h.pairs) + resize!(h.pairs, sz) h.ndel = 0 h.count = 0 h.age += 1 @@ -278,18 +267,21 @@ end # get the index where a key is stored, or -1 if not present function ht_keyindex(h::Dict{K,V}, key) where V where K - sz = length(h.keys) + sz = length(h.pairs) iter = 0 maxprobe = h.maxprobe index = hashindex(key, sz) - keys = h.keys + pairs = h.pairs @inbounds while true if isslotempty(h,index) break end - if !isslotmissing(h,index) && (key === keys[index] || isequal(key,keys[index])) - return index + if !isslotmissing(h,index) + k = pairs[index].first + if (key === k || isequal(key, k)) + return index + end end index = (index & (sz-1)) + 1 @@ -303,12 +295,12 @@ end # and the key would be inserted at pos # This version is for use by setindex! and get! function ht_keyindex2!(h::Dict{K,V}, key) where V where K - sz = length(h.keys) + sz = length(h.pairs) iter = 0 maxprobe = h.maxprobe index = hashindex(key, sz) avail = 0 - keys = h.keys + pairs = h.pairs @inbounds while true if isslotempty(h,index) @@ -324,7 +316,7 @@ function ht_keyindex2!(h::Dict{K,V}, key) where V where K # in case "key" already exists in a later collided slot. avail = -index end - elseif key === keys[index] || isequal(key, keys[index]) + elseif key === pairs[index].first || isequal(key, pairs[index].first) return index end @@ -353,15 +345,14 @@ end @propagate_inbounds function _setindex!(h::Dict, v, key, index) h.slots[index] = 0x1 - h.keys[index] = key - h.vals[index] = v + h.pairs[index] = key => v h.count += 1 h.age += 1 if index < h.idxfloor h.idxfloor = index end - sz = length(h.keys) + sz = length(h.pairs) # Rehash now if necessary if h.ndel >= ((3*sz)>>2) || h.count*3 > sz*2 # > 3/4 deleted or > 2/3 full @@ -384,8 +375,7 @@ function setindex!(h::Dict{K,V}, v0, key::K) where V where K if index > 0 h.age += 1 - @inbounds h.keys[index] = key - @inbounds h.vals[index] = v + @inbounds h.pairs[index] = key => v else @inbounds _setindex!(h, v, key, -index) end @@ -393,18 +383,17 @@ function setindex!(h::Dict{K,V}, v0, key::K) where V where K return h end -function setindex!(h::Dict{K,Any}, v, key::K) where K - @nospecialize v +function setindex!(h::Dict{K,Any}, v::Any, key::K) where K + #@nospecialize v index = ht_keyindex2!(h, key) if index > 0 h.age += 1 - @inbounds h.keys[index] = key - @inbounds h.vals[index] = v + @inbounds h.pairs[index] = key => v else @inbounds _setindex!(h, v, key, -index) end - + return h end @@ -475,7 +464,7 @@ end function get!(default::Callable, h::Dict{K,V}, key::K) where V where K index = ht_keyindex2!(h, key) - index > 0 && return h.vals[index] + index > 0 && return h.pairs[index].second age0 = h.age v = convert(V, default()) @@ -484,18 +473,16 @@ function get!(default::Callable, h::Dict{K,V}, key::K) where V where K end if index > 0 h.age += 1 - @inbounds h.keys[index] = key - @inbounds h.vals[index] = v + @inbounds h.pairs[index] = key => v else @inbounds _setindex!(h, v, key, -index) end return v end - function getindex(h::Dict{K,V}, key) where V where K index = ht_keyindex(h, key) - @inbounds return (index < 0) ? throw(KeyError(key)) : h.vals[index]::V + @inbounds return (index < 0) ? throw(KeyError(key)) : h.pairs[index].second::V end """ @@ -522,7 +509,7 @@ get(collection, key, default) function get(h::Dict{K,V}, key, default) where V where K index = ht_keyindex(h, key) - @inbounds return (index < 0) ? default : h.vals[index]::V + @inbounds return (index < 0) ? default : h.pairs[index].second::V end """ @@ -544,7 +531,7 @@ get(::Function, collection, key) function get(default::Callable, h::Dict{K,V}, key) where V where K index = ht_keyindex(h, key) - @inbounds return (index < 0) ? default() : h.vals[index]::V + @inbounds return (index < 0) ? default() : h.pairs[index].second::V end """ @@ -590,11 +577,11 @@ julia> getkey(D, 'd', 'a') """ function getkey(h::Dict{K,V}, key, default) where V where K index = ht_keyindex(h, key) - @inbounds return (index<0) ? default : h.keys[index]::K + @inbounds return (index<0) ? default : h.pairs[index].first::K end function _pop!(h::Dict, index) - @inbounds val = h.vals[index] + @inbounds val = h.pairs[index].second _delete!(h, index) return val end @@ -636,16 +623,14 @@ end function pop!(h::Dict) isempty(h) && throw(ArgumentError("dict must be non-empty")) idx = skip_deleted_floor!(h) - @inbounds key = h.keys[idx] - @inbounds val = h.vals[idx] + @inbounds pair = h.pairs[idx] _delete!(h, idx) - key => val + pair end function _delete!(h::Dict{K,V}, index) where {K,V} @inbounds h.slots[index] = 0x2 - @inbounds _unsetindex!(h.keys, index) - @inbounds _unsetindex!(h.vals, index) + @inbounds _unsetindex!(h.pairs, index) h.ndel += 1 h.count -= 1 h.age += 1 @@ -700,7 +685,7 @@ function skip_deleted_floor!(h::Dict) idx end -@propagate_inbounds _iterate(t::Dict{K,V}, i) where {K,V} = i == 0 ? nothing : (Pair{K,V}(t.keys[i],t.vals[i]), i == typemax(Int) ? 0 : i+1) +@propagate_inbounds _iterate(t::Dict{K,V}, i) where {K,V} = i == 0 ? nothing : (t.pairs[i], i == typemax(Int) ? 0 : i+1) @propagate_inbounds function iterate(t::Dict) _iterate(t, skip_deleted_floor!(t)) end @@ -713,14 +698,17 @@ length(t::Dict) = t.count i == 0 && return nothing i = skip_deleted(v.dict, i) i == 0 && return nothing - vals = T <: KeySet ? v.dict.keys : v.dict.vals - (@inbounds vals[i], i == typemax(Int) ? 0 : i+1) + if T <: KeySet + (@inbounds v.dict.pairs[i].first, i == typemax(Int) ? 0 : i+1) + else + (@inbounds v.dict.pairs[i].second, i == typemax(Int) ? 0 : i+1) + end end function filter!(pred, h::Dict{K,V}) where {K,V} h.count == 0 && return h @inbounds for i=1:length(h.slots) - if h.slots[i] == 0x01 && !pred(Pair{K,V}(h.keys[i], h.vals[i])) + if h.slots[i] == 0x01 && !pred(h.pairs[i]) _delete!(h, i) end end @@ -735,11 +723,11 @@ end function map!(f, iter::ValueIterator{<:Dict}) dict = iter.dict - vals = dict.vals + pairs = dict.pairs # @inbounds is here so that it gets propagated to isslotfilled - @inbounds for i = dict.idxfloor:lastindex(vals) + @inbounds for i = dict.idxfloor:lastindex(pairs) if isslotfilled(dict, i) - vals[i] = f(vals[i]) + pairs[i] = pairs[i].first => f(pairs[i].second) end end return iter @@ -749,7 +737,7 @@ function mergewith!(combine, d1::Dict{K, V}, d2::AbstractDict) where {K, V} for (k, v) in d2 i = ht_keyindex2!(d1, k) if i > 0 - d1.vals[i] = combine(d1.vals[i], v) + d1.pairs[i] = d1.pairs[i].first => combine(d1.pairs[i].second, v) else if !isequal(k, convert(K, k)) throw(ArgumentError("$(limitrepr(k)) is not a valid key for type $K")) diff --git a/base/set.jl b/base/set.jl index 371799f2e3ff5..47ec407af186c 100644 --- a/base/set.jl +++ b/base/set.jl @@ -14,9 +14,14 @@ Set() = Set{Any}() function Set{T}(s::KeySet{T, <:Dict{T}}) where {T} d = s.dict slots = copy(d.slots) - keys = copy(d.keys) - vals = similar(d.vals, Nothing) - _Set(Dict{T,Nothing}(slots, keys, vals, d.ndel, d.count, d.age, d.idxfloor, d.maxprobe)) + n = length(d.pairs) + pairs = Vector{Pair{T,Nothing}}(undef, n) + for i in 1:n + if isslotfilled(d, i) + pairs[i] = d.pairs[i].first::T => nothing + end + end + _Set(Dict{T,Nothing}(slots, pairs, d.ndel, d.count, d.age, d.idxfloor, d.maxprobe)) end """ @@ -754,14 +759,12 @@ function _replace!(new::Callable, t::Dict{K,V}, A::AbstractDict, count::Int) whe news = Pair{K,V}[] i = skip_deleted_floor!(t) @inbounds while i != 0 - k1, v1 = t.keys[i], t.vals[i] - x1 = Pair{K,V}(k1, v1) + x1 = t.pairs[i] x2 = new(x1) if x1 !== x2 k2, v2 = first(x2), last(x2) if isequal(k1, k2) - t.keys[i] = k2 - t.vals[i] = v2 + t.pairs[i] = k2::K => v2::V t.age += 1 else _delete!(t, i) diff --git a/base/weakkeydict.jl b/base/weakkeydict.jl index 0a9987671ea9b..c27f7afd8326b 100644 --- a/base/weakkeydict.jl +++ b/base/weakkeydict.jl @@ -69,7 +69,7 @@ function _cleanup_locked(h::WeakKeyDict) h.dirty = false idx = skip_deleted_floor!(h.ht) while idx != 0 - if h.ht.keys[idx].value === nothing + if h.ht.pairs[idx].first.value === nothing _delete!(h.ht, idx) end idx = skip_deleted(h.ht, idx + 1) diff --git a/stdlib/Random/src/generation.jl b/stdlib/Random/src/generation.jl index ddbf6dce98bec..dae3fde018373 100644 --- a/stdlib/Random/src/generation.jl +++ b/stdlib/Random/src/generation.jl @@ -434,7 +434,7 @@ end function rand(rng::AbstractRNG, sp::SamplerSimple{<:Dict,<:Sampler}) while true i = rand(rng, sp.data) - Base.isslotfilled(sp[], i) && @inbounds return (sp[].keys[i] => sp[].vals[i]) + Base.isslotfilled(sp[], i) && @inbounds return sp[].pairs[i] end end diff --git a/test/sets.jl b/test/sets.jl index 1999dbf3d020f..dc087f660a889 100644 --- a/test/sets.jl +++ b/test/sets.jl @@ -169,14 +169,14 @@ end # array element s = Set(["a", "b", "c"]) Base.rehash!(s) - k = s.dict.keys + k = s.dict.pairs Base.rehash!(s) - @test length(k) == length(s.dict.keys) + @test length(k) == length(s.dict.pairs) for i in 1:length(k) if isassigned(k, i) - @test k[i] == s.dict.keys[i] + @test k[i] == s.dict.pairs[i].first else - @test !isassigned(s.dict.keys, i) + @test !isassigned(s.dict.pairs, i) end end @test s == Set(["a", "b", "c"]) From 0acaa9aa1c6a2eea2f9728caed1c7df272edcfee Mon Sep 17 00:00:00 2001 From: Petr Vana Date: Thu, 24 Feb 2022 20:04:50 +0100 Subject: [PATCH 02/18] Fix Set --- base/dict.jl | 4 ++-- base/set.jl | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/base/dict.jl b/base/dict.jl index ab2ae377d8684..f33eaf36064d7 100644 --- a/base/dict.jl +++ b/base/dict.jl @@ -383,8 +383,8 @@ function setindex!(h::Dict{K,V}, v0, key::K) where V where K return h end -function setindex!(h::Dict{K,Any}, v::Any, key::K) where K - #@nospecialize v +function setindex!(h::Dict{K,Any}, v, key::K) where K + @nospecialize v index = ht_keyindex2!(h, key) if index > 0 diff --git a/base/set.jl b/base/set.jl index 47ec407af186c..e5db98e488c85 100644 --- a/base/set.jl +++ b/base/set.jl @@ -762,9 +762,8 @@ function _replace!(new::Callable, t::Dict{K,V}, A::AbstractDict, count::Int) whe x1 = t.pairs[i] x2 = new(x1) if x1 !== x2 - k2, v2 = first(x2), last(x2) - if isequal(k1, k2) - t.pairs[i] = k2::K => v2::V + if isequal(x1.first, x2.first) + t.pairs[i] = x2 t.age += 1 else _delete!(t, i) From 830685849d995f0c323668d5e528113b80976c65 Mon Sep 17 00:00:00 2001 From: Petr Vana Date: Thu, 24 Feb 2022 21:20:18 +0100 Subject: [PATCH 03/18] Fix most of the tests --- test/compiler/inference.jl | 2 +- test/sets.jl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/compiler/inference.jl b/test/compiler/inference.jl index 61058f9589f52..050690e841898 100644 --- a/test/compiler/inference.jl +++ b/test/compiler/inference.jl @@ -273,7 +273,7 @@ barTuple2() = fooTuple{tuple(:y)}() # issue #6050 @test Core.Compiler.getfield_tfunc( Dict{Int64,Tuple{UnitRange{Int64},UnitRange{Int64}}}, - Core.Compiler.Const(:vals)) == Array{Tuple{UnitRange{Int64},UnitRange{Int64}},1} + Core.Compiler.Const(:pairs)) == Array{Pair{Int64, Tuple{UnitRange{Int64},UnitRange{Int64}}},1} # assert robustness of `getfield_tfunc` struct GetfieldRobustness diff --git a/test/sets.jl b/test/sets.jl index dc087f660a889..a5250da3cc743 100644 --- a/test/sets.jl +++ b/test/sets.jl @@ -174,7 +174,7 @@ end @test length(k) == length(s.dict.pairs) for i in 1:length(k) if isassigned(k, i) - @test k[i] == s.dict.pairs[i].first + @test k[i].first == s.dict.pairs[i].first else @test !isassigned(s.dict.pairs, i) end From 9a1a6606a3ac102f044d3ede4d1aa26b2454af3d Mon Sep 17 00:00:00 2001 From: Petr Vana Date: Fri, 25 Feb 2022 00:47:30 +0100 Subject: [PATCH 04/18] Use explicite types for pairs --- base/dict.jl | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/base/dict.jl b/base/dict.jl index f33eaf36064d7..9e5e6fe907f4a 100644 --- a/base/dict.jl +++ b/base/dict.jl @@ -316,8 +316,11 @@ function ht_keyindex2!(h::Dict{K,V}, key) where V where K # in case "key" already exists in a later collided slot. avail = -index end - elseif key === pairs[index].first || isequal(key, pairs[index].first) - return index + else + p = pairs[index] + if key === p.first || isequal(key, p.first) + return index + end end index = (index & (sz-1)) + 1 @@ -343,9 +346,9 @@ function ht_keyindex2!(h::Dict{K,V}, key) where V where K return ht_keyindex2!(h, key) end -@propagate_inbounds function _setindex!(h::Dict, v, key, index) +@propagate_inbounds function _setindex!(h::Dict{K,V}, v, key, index) where V where K h.slots[index] = 0x1 - h.pairs[index] = key => v + h.pairs[index] = Pair{K,V}(key, v) h.count += 1 h.age += 1 if index < h.idxfloor @@ -375,7 +378,7 @@ function setindex!(h::Dict{K,V}, v0, key::K) where V where K if index > 0 h.age += 1 - @inbounds h.pairs[index] = key => v + @inbounds h.pairs[index] = Pair{K,V}(key, v) else @inbounds _setindex!(h, v, key, -index) end @@ -389,7 +392,7 @@ function setindex!(h::Dict{K,Any}, v, key::K) where K if index > 0 h.age += 1 - @inbounds h.pairs[index] = key => v + @inbounds h.pairs[index] = Pair{K,Any}(key, v) else @inbounds _setindex!(h, v, key, -index) end @@ -473,7 +476,7 @@ function get!(default::Callable, h::Dict{K,V}, key::K) where V where K end if index > 0 h.age += 1 - @inbounds h.pairs[index] = key => v + @inbounds h.pairs[index] = Pair{K,V}(key, v) else @inbounds _setindex!(h, v, key, -index) end @@ -721,13 +724,13 @@ function reduce(::typeof(merge), items::Vector{<:Dict}) return reduce(merge!, items; init=Dict{K,V}()) end -function map!(f, iter::ValueIterator{<:Dict}) +function map!(f, iter::ValueIterator{<:Dict{K, V}}) where {K, V} dict = iter.dict pairs = dict.pairs # @inbounds is here so that it gets propagated to isslotfilled @inbounds for i = dict.idxfloor:lastindex(pairs) if isslotfilled(dict, i) - pairs[i] = pairs[i].first => f(pairs[i].second) + pairs[i] = Pair{K,V}(pairs[i].first, f(pairs[i].second)) end end return iter @@ -737,7 +740,7 @@ function mergewith!(combine, d1::Dict{K, V}, d2::AbstractDict) where {K, V} for (k, v) in d2 i = ht_keyindex2!(d1, k) if i > 0 - d1.pairs[i] = d1.pairs[i].first => combine(d1.pairs[i].second, v) + d1.pairs[i] = Pair{K,V}(d1.pairs[i].first, combine(d1.pairs[i].second, v)) else if !isequal(k, convert(K, k)) throw(ArgumentError("$(limitrepr(k)) is not a valid key for type $K")) From 82e7a9d9548b527a70951968471aa84fc949f2d0 Mon Sep 17 00:00:00 2001 From: Petr Vana Date: Fri, 25 Feb 2022 08:36:49 +0100 Subject: [PATCH 05/18] Mark broken test of precompilation --- test/precompile.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/precompile.jl b/test/precompile.jl index 411267705622d..7748065cb86b5 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -630,10 +630,10 @@ precompile_test_harness("code caching") do dir Base.invokelatest() do Dict{M.X2,Any}()[M.X2()] = nothing end - @test M.X2 ∈ m.roots + @test_broken M.X2 ∈ m.roots groups = group_roots(m) @test_broken M.X ∈ groups[Mid] # requires caching external compilation results - @test M.X2 ∈ groups[rootid(@__MODULE__)] + @test_broken M.X2 ∈ groups[rootid(@__MODULE__)] @test !isempty(groups[Bid]) minternal = which(M.getelsize, (Vector,)) mi = minternal.specializations[1] From 96d80a52519519b0fc608df005ce8e4eb89e9130 Mon Sep 17 00:00:00 2001 From: Petr Vana Date: Fri, 25 Feb 2022 09:41:24 +0100 Subject: [PATCH 06/18] Reenable broken test --- test/precompile.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/precompile.jl b/test/precompile.jl index dd1286e9f8d75..d39dcd9f7ccb8 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -651,7 +651,7 @@ precompile_test_harness("code caching") do dir Base.invokelatest() do Dict{M.X2,Any}()[M.X2()] = nothing end - @test_broken M.X2 ∈ m.roots + @test M.X2 ∈ m.roots groups = group_roots(m) @test M.X ∈ groups[Mid] # attributed to M @test M.X2 ∈ groups[0] # activate module is not known From f37cc2d7975f1be0230d6eca031ee0596f7ca124 Mon Sep 17 00:00:00 2001 From: Petr Vana Date: Fri, 25 Feb 2022 17:32:00 +0100 Subject: [PATCH 07/18] Use simplified OldDict for testing precompilation --- test/precompile.jl | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/test/precompile.jl b/test/precompile.jl index d39dcd9f7ccb8..56ba3cd0acdba 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -588,19 +588,35 @@ end precompile_test_harness("code caching") do dir Bid = rootid(Base) Cache_module = :Cacheb8321416e8a3e2f1 - # Note: calling setindex!(::Dict{K,V}, ::Any, ::K) adds both compression and codegen roots + # Note: calling store(::DictOld{K,V}, ::Any, ::K) adds both compression and codegen roots write(joinpath(dir, "$Cache_module.jl"), """ module $Cache_module + mutable struct OldDict{K,V} <: AbstractDict{K,V} + keys::Vector{K} + vals::Vector{V} + + function OldDict{K,V}() where V where K + n = 16 + new(Vector{K}(undef, n), Vector{V}(undef, n)) + end + end + + function store(h::OldDict{K,Any}, v, key::K) where K + @nospecialize v + h.keys[1] = key + h.vals[1] = v + end + struct X end struct X2 end @noinline function f(d) @noinline - d[X()] = nothing + store(d, nothing, X()) end @noinline fpush(dest) = push!(dest, X()) function callboth() - f(Dict{X,Any}()) + f(OldDict{X,Any}()) fpush(X[]) nothing end @@ -645,17 +661,17 @@ precompile_test_harness("code caching") do dir end @test hasspec # Test that compilation adds to method roots with appropriate provenance - m = which(setindex!, (Dict{M.X,Any}, Any, M.X)) + m = which(M.store, (M.OldDict{M.X,Any}, Any, M.X)) @test M.X ∈ m.roots # Check that roots added outside of incremental builds get attributed to a moduleid of 0 Base.invokelatest() do - Dict{M.X2,Any}()[M.X2()] = nothing + M.store(M.OldDict{M.X2,Any}(), nothing, M.X2()) end @test M.X2 ∈ m.roots groups = group_roots(m) @test M.X ∈ groups[Mid] # attributed to M @test M.X2 ∈ groups[0] # activate module is not known - @test !isempty(groups[Bid]) + #@test !isempty(groups[Bid]) # Check that internal methods and their roots are accounted appropriately minternal = which(M.getelsize, (Vector,)) mi = minternal.specializations[1] From 25d9bec0251ed851546f36d635a1fb63fba174c4 Mon Sep 17 00:00:00 2001 From: Petr Vana Date: Fri, 25 Feb 2022 18:07:33 +0100 Subject: [PATCH 08/18] Fix precompile test --- test/precompile.jl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/precompile.jl b/test/precompile.jl index 56ba3cd0acdba..f3d5f2e4cbf50 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -586,7 +586,6 @@ end # method root provenance & external code caching precompile_test_harness("code caching") do dir - Bid = rootid(Base) Cache_module = :Cacheb8321416e8a3e2f1 # Note: calling store(::DictOld{K,V}, ::Any, ::K) adds both compression and codegen roots write(joinpath(dir, "$Cache_module.jl"), @@ -671,7 +670,7 @@ precompile_test_harness("code caching") do dir groups = group_roots(m) @test M.X ∈ groups[Mid] # attributed to M @test M.X2 ∈ groups[0] # activate module is not known - #@test !isempty(groups[Bid]) + @test !isempty(groups[Mid]) # Check that internal methods and their roots are accounted appropriately minternal = which(M.getelsize, (Vector,)) mi = minternal.specializations[1] From 8d39aefaafa6846f67c694b0978d1b371348657a Mon Sep 17 00:00:00 2001 From: Petr Vana Date: Fri, 25 Feb 2022 20:27:10 +0100 Subject: [PATCH 09/18] Fix white space --- base/dict.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/dict.jl b/base/dict.jl index 9e5e6fe907f4a..aa78e24631a2e 100644 --- a/base/dict.jl +++ b/base/dict.jl @@ -396,7 +396,7 @@ function setindex!(h::Dict{K,Any}, v, key::K) where K else @inbounds _setindex!(h, v, key, -index) end - + return h end From 08d7d8806cd5d183efbfd74b41570965d20177dc Mon Sep 17 00:00:00 2001 From: Petr Vana Date: Fri, 25 Feb 2022 23:55:48 +0100 Subject: [PATCH 10/18] Apply suggestions from code review Co-authored-by: Simeon Schaub --- base/dict.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/base/dict.jl b/base/dict.jl index 189deee0d7b77..4f1981751a738 100644 --- a/base/dict.jl +++ b/base/dict.jl @@ -318,8 +318,8 @@ function ht_keyindex2!(h::Dict{K,V}, key) where V where K avail = -index end else - p = pairs[index] - if key === p.first || isequal(key, p.first) + k = pairs[index].first + if key === k || isequal(key, k) return index end end @@ -347,7 +347,7 @@ function ht_keyindex2!(h::Dict{K,V}, key) where V where K return ht_keyindex2!(h, key) end -@propagate_inbounds function _setindex!(h::Dict{K,V}, v, key, index) where V where K +@propagate_inbounds function _setindex!(h::Dict{K,V}, v, key, index) where {K, V} h.slots[index] = 0x1 h.pairs[index] = Pair{K,V}(key, v) h.count += 1 From afd7975b7bb8292bb6ac453aeabeb5e8f512b140 Mon Sep 17 00:00:00 2001 From: Petr Vana Date: Sat, 26 Feb 2022 00:09:53 +0100 Subject: [PATCH 11/18] Apply suggestions from code review Co-authored-by: Simeon Schaub --- base/dict.jl | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/base/dict.jl b/base/dict.jl index 4f1981751a738..0f7b02dc65e23 100644 --- a/base/dict.jl +++ b/base/dict.jl @@ -702,11 +702,8 @@ length(t::Dict) = t.count i == 0 && return nothing i = skip_deleted(v.dict, i) i == 0 && return nothing - if T <: KeySet - (@inbounds v.dict.pairs[i].first, i == typemax(Int) ? 0 : i+1) - else - (@inbounds v.dict.pairs[i].second, i == typemax(Int) ? 0 : i+1) - end + p = @inbounds v.dict.pairs[i] + return p[T <: KeySet ? 1 : 2], i == typemax(Int) ? 0 : i+1 end function filter!(pred, h::Dict{K,V}) where {K,V} From e69826f042a531bcd7e020344ed9d896cca037af Mon Sep 17 00:00:00 2001 From: Petr Vana Date: Sat, 26 Feb 2022 00:47:01 +0100 Subject: [PATCH 12/18] Fix performance of Set constructor if both {T,V} are bitstypes --- base/set.jl | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/base/set.jl b/base/set.jl index e5db98e488c85..0fc343e273810 100644 --- a/base/set.jl +++ b/base/set.jl @@ -11,14 +11,18 @@ Set{T}(s::Set{T}) where {T} = _Set(Dict{T,Nothing}(s.dict)) Set{T}(itr) where {T} = union!(Set{T}(), itr) Set() = Set{Any}() -function Set{T}(s::KeySet{T, <:Dict{T}}) where {T} +function Set{T}(s::KeySet{T, <:Dict{T,V}}) where {T,V} d = s.dict slots = copy(d.slots) n = length(d.pairs) - pairs = Vector{Pair{T,Nothing}}(undef, n) - for i in 1:n - if isslotfilled(d, i) - pairs[i] = d.pairs[i].first::T => nothing + if isbitstype(T) && isbitstype(V) + pairs = [Pair{T,Nothing}(d.pairs[i].first, nothing) for i in 1:n] + else + pairs = Vector{Pair{T,Nothing}}(undef, n) + for i in 1:n + if isslotfilled(d, i) + pairs[i] = Pair{T,Nothing}(d.pairs[i].first, nothing) + end end end _Set(Dict{T,Nothing}(slots, pairs, d.ndel, d.count, d.age, d.idxfloor, d.maxprobe)) From 6600b03168ab29be689f888a4c438cd42f5d2690 Mon Sep 17 00:00:00 2001 From: Petr Vana Date: Sat, 26 Feb 2022 01:00:46 +0100 Subject: [PATCH 13/18] Apply suggestions from code review --- base/set.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/base/set.jl b/base/set.jl index 0fc343e273810..c333d380c1d12 100644 --- a/base/set.jl +++ b/base/set.jl @@ -14,10 +14,10 @@ Set() = Set{Any}() function Set{T}(s::KeySet{T, <:Dict{T,V}}) where {T,V} d = s.dict slots = copy(d.slots) - n = length(d.pairs) - if isbitstype(T) && isbitstype(V) - pairs = [Pair{T,Nothing}(d.pairs[i].first, nothing) for i in 1:n] + if isbitstype(Pair{T, V}) + pairs = Pair{T, Nothing}[Pair{T, Nothing}(x.first, nothing) for x in d.pairs] else + n = length(d.pairs) pairs = Vector{Pair{T,Nothing}}(undef, n) for i in 1:n if isslotfilled(d, i) From d789afc52d2dfe40bfd2b1100f7e4d81611cffeb Mon Sep 17 00:00:00 2001 From: Petr Vana Date: Sat, 26 Feb 2022 11:51:45 +0100 Subject: [PATCH 14/18] Add inbounds --- base/set.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/set.jl b/base/set.jl index c333d380c1d12..82ea1d62c985d 100644 --- a/base/set.jl +++ b/base/set.jl @@ -21,7 +21,7 @@ function Set{T}(s::KeySet{T, <:Dict{T,V}}) where {T,V} pairs = Vector{Pair{T,Nothing}}(undef, n) for i in 1:n if isslotfilled(d, i) - pairs[i] = Pair{T,Nothing}(d.pairs[i].first, nothing) + @inbounds pairs[i] = Pair{T,Nothing}(d.pairs[i].first, nothing) end end end From 326798e3375b00b3559fce3d631562ee449843ae Mon Sep 17 00:00:00 2001 From: Petr Vana Date: Thu, 24 Mar 2022 20:30:12 +0100 Subject: [PATCH 15/18] Remove empty lines in test --- test/precompile.jl | 3 --- 1 file changed, 3 deletions(-) diff --git a/test/precompile.jl b/test/precompile.jl index cddfd8edbb836..cd81d58cd7e7a 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -594,19 +594,16 @@ precompile_test_harness("code caching") do dir mutable struct OldDict{K,V} <: AbstractDict{K,V} keys::Vector{K} vals::Vector{V} - function OldDict{K,V}() where V where K n = 16 new(Vector{K}(undef, n), Vector{V}(undef, n)) end end - function store(h::OldDict{K,Any}, v, key::K) where K @nospecialize v h.keys[1] = key h.vals[1] = v end - struct X end struct X2 end @noinline function f(d) From f9b27dcb17e1621fdc22bcfc78894e2efad87b75 Mon Sep 17 00:00:00 2001 From: Petr Vana Date: Thu, 24 Nov 2022 02:07:12 +0100 Subject: [PATCH 16/18] Backward compatibility for keys and vals --- base/deprecated.jl | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/base/deprecated.jl b/base/deprecated.jl index 6953cd600cacd..9a81712336843 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -364,3 +364,28 @@ end end # END 1.9 deprecations + +# BEGIN 1.10 deprecations +struct DepricatedKeyDictAccessor + dict::Dict +end + +struct DepricatedValueDictAccessor + dict::Dict +end + +getindex(a::DepricatedKeyDictAccessor, i::Integer) = a.dict.pairs[i].first +getindex(a::DepricatedValueDictAccessor, i::Integer) = a.dict.pairs[i].second + +function getproperty(d::Dict, s::Symbol) + if s == :keys + depwarn("For Dict, please use dict.pairs[i].first instead of dict.keys[i].", :getproperty, force=true) + return DepricatedKeyDictAccessor(d) + elseif s == :vals + depwarn("For Dict, please use dict.pairs[i].second instead of dict.vals[i].", :getproperty, force=true) + return DepricatedValueDictAccessor(d) + end + return getfield(d, s) +end + +# END 1.10 deprecations From d093e7d95be658a513fba2eb12e01ead739915b6 Mon Sep 17 00:00:00 2001 From: Petr Vana Date: Sun, 27 Nov 2022 12:40:55 +0100 Subject: [PATCH 17/18] Improve backward compatibility --- base/deprecated.jl | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/base/deprecated.jl b/base/deprecated.jl index 9a81712336843..e62cd40ec10b1 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -366,17 +366,32 @@ end # END 1.9 deprecations # BEGIN 1.10 deprecations -struct DepricatedKeyDictAccessor - dict::Dict +struct DepricatedKeyDictAccessor{K,V} + dict::Dict{K,V} end -struct DepricatedValueDictAccessor - dict::Dict +struct DepricatedValueDictAccessor{K,V} + dict::Dict{K,V} end getindex(a::DepricatedKeyDictAccessor, i::Integer) = a.dict.pairs[i].first getindex(a::DepricatedValueDictAccessor, i::Integer) = a.dict.pairs[i].second +function setindex!(a::DepricatedKeyDictAccessor{K,V}, value, i::Integer) where {K,V} + d = a.dict + d.pairs[i] = Pair{K,V}(value, d.pairs[i].second) + a +end + +function setindex!(a::DepricatedValueDictAccessor{K,V}, value, i::Integer) where {K,V} + d = a.dict + d.pairs[i] = Pair{K,V}(d.pairs[i].first, value) + a +end + +length(a::DepricatedKeyDictAccessor) = length(a.dict.pairs) +length(a::DepricatedValueDictAccessor) = length(a.dict.pairs) + function getproperty(d::Dict, s::Symbol) if s == :keys depwarn("For Dict, please use dict.pairs[i].first instead of dict.keys[i].", :getproperty, force=true) From 58414f80c91040d4d0a2a87dd9029639e93f0227 Mon Sep 17 00:00:00 2001 From: Petr Vana Date: Mon, 28 Nov 2022 14:17:33 +0100 Subject: [PATCH 18/18] Improve backward compatibility II --- base/deprecated.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/base/deprecated.jl b/base/deprecated.jl index e62cd40ec10b1..f4dd7a98612ab 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -366,11 +366,11 @@ end # END 1.9 deprecations # BEGIN 1.10 deprecations -struct DepricatedKeyDictAccessor{K,V} +struct DepricatedKeyDictAccessor{K,V} <: AbstractVector{K} dict::Dict{K,V} end -struct DepricatedValueDictAccessor{K,V} +struct DepricatedValueDictAccessor{K,V} <: AbstractVector{V} dict::Dict{K,V} end @@ -389,8 +389,8 @@ function setindex!(a::DepricatedValueDictAccessor{K,V}, value, i::Integer) where a end -length(a::DepricatedKeyDictAccessor) = length(a.dict.pairs) -length(a::DepricatedValueDictAccessor) = length(a.dict.pairs) +size(a::DepricatedKeyDictAccessor) = (length(a.dict.pairs),) +size(a::DepricatedValueDictAccessor) = (length(a.dict.pairs),) function getproperty(d::Dict, s::Symbol) if s == :keys