From 69cea9c2b9cfbb416b3dddd2f59995fe56b3100f Mon Sep 17 00:00:00 2001 From: Aaron Hill Date: Mon, 17 Aug 2020 13:38:37 -0400 Subject: [PATCH 1/6] Query-ify LocalDefId <-> HirId conversion Based on PR #70039 --- src/librustc_middle/arena.rs | 1 + src/librustc_middle/hir/map/collector.rs | 10 +++++++++- src/librustc_middle/hir/map/mod.rs | 13 ++++++++++--- src/librustc_middle/hir/mod.rs | 3 +++ src/librustc_middle/query/mod.rs | 16 ++++++++++++++++ 5 files changed, 39 insertions(+), 4 deletions(-) diff --git a/src/librustc_middle/arena.rs b/src/librustc_middle/arena.rs index f6570cc95d27d..4e6dc8e13d920 100644 --- a/src/librustc_middle/arena.rs +++ b/src/librustc_middle/arena.rs @@ -94,6 +94,7 @@ macro_rules! arena_types { [few] hir_definitions: rustc_hir::definitions::Definitions, [] hir_owner: rustc_middle::hir::Owner<$tcx>, [] hir_owner_nodes: rustc_middle::hir::OwnerNodes<$tcx>, + [] hir_owner_defs: rustc_data_structures::fx::FxHashMap, // Note that this deliberately duplicates items in the `rustc_hir::arena`, // since we need to allocate this type on both the `rustc_hir` arena diff --git a/src/librustc_middle/hir/map/collector.rs b/src/librustc_middle/hir/map/collector.rs index dce06a5f7eeec..9ad65ea950091 100644 --- a/src/librustc_middle/hir/map/collector.rs +++ b/src/librustc_middle/hir/map/collector.rs @@ -133,7 +133,7 @@ impl<'a, 'hir> NodeCollector<'a, 'hir> { hcx, hir_body_nodes, map: (0..definitions.def_index_count()) - .map(|_| HirOwnerData { signature: None, with_bodies: None }) + .map(|_| HirOwnerData { signature: None, with_bodies: None, defs: None }) .collect(), }; collector.insert_entry( @@ -229,6 +229,14 @@ impl<'a, 'hir> NodeCollector<'a, 'hir> { id.local_id, ParentedNode { parent: entry.parent.local_id, node: entry.node }, ); + + // Check if this HirId has a DefId, and insert it in the `defs` map if so. + if let Some(def_id) = self.definitions.opt_hir_id_to_local_def_id(id) { + if data.defs.is_none() { + data.defs = Some(arena.alloc(FxHashMap::default())); + }; + data.defs.as_mut().unwrap().insert(id.local_id, def_id); + } } } diff --git a/src/librustc_middle/hir/map/mod.rs b/src/librustc_middle/hir/map/mod.rs index a6cc7cbc9207c..1896b72959039 100644 --- a/src/librustc_middle/hir/map/mod.rs +++ b/src/librustc_middle/hir/map/mod.rs @@ -4,6 +4,7 @@ use crate::hir::{Owner, OwnerNodes}; use crate::ty::query::Providers; use crate::ty::TyCtxt; use rustc_ast as ast; +use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::svh::Svh; use rustc_hir::def::{DefKind, Res}; use rustc_hir::def_id::{CrateNum, DefId, LocalDefId, CRATE_DEF_INDEX, LOCAL_CRATE}; @@ -89,6 +90,7 @@ fn is_body_owner<'hir>(node: Node<'hir>, hir_id: HirId) -> bool { pub(super) struct HirOwnerData<'hir> { pub(super) signature: Option<&'hir Owner<'hir>>, pub(super) with_bodies: Option<&'hir mut OwnerNodes<'hir>>, + pub(super) defs: Option<&'hir mut FxHashMap>, } pub struct IndexedHir<'hir> { @@ -170,17 +172,22 @@ impl<'hir> Map<'hir> { #[inline] pub fn opt_local_def_id(&self, hir_id: HirId) -> Option { - self.tcx.definitions.opt_hir_id_to_local_def_id(hir_id) + if hir_id.local_id == ItemLocalId::from_u32(0) { + // Every HirId owner has a DefId, so we can just return it directly here + Some(hir_id.owner) + } else { + self.tcx.hir_owner_defs(hir_id.owner).and_then(|map| map.get(&hir_id.local_id).copied()) + } } #[inline] pub fn local_def_id_to_hir_id(&self, def_id: LocalDefId) -> HirId { - self.tcx.definitions.local_def_id_to_hir_id(def_id) + self.tcx.local_def_id_to_hir_id(def_id).unwrap() } #[inline] pub fn opt_local_def_id_to_hir_id(&self, def_id: LocalDefId) -> Option { - self.tcx.definitions.opt_local_def_id_to_hir_id(def_id) + self.tcx.local_def_id_to_hir_id(def_id) } pub fn def_kind(&self, local_def_id: LocalDefId) -> DefKind { diff --git a/src/librustc_middle/hir/mod.rs b/src/librustc_middle/hir/mod.rs index ae3b30217cc4a..488e520659aa9 100644 --- a/src/librustc_middle/hir/mod.rs +++ b/src/librustc_middle/hir/mod.rs @@ -92,5 +92,8 @@ pub fn provide(providers: &mut Providers) { span_bug!(hir.span(hir_id), "fn_arg_names: unexpected item {:?}", id); } }; + providers.local_def_id_to_hir_id = |tcx, id| tcx.definitions.opt_local_def_id_to_hir_id(id); + providers.hir_owner_defs = + |tcx, index| tcx.index_hir(LOCAL_CRATE).map[index].defs.as_ref().map(|defs| &**defs); map::provide(providers); } diff --git a/src/librustc_middle/query/mod.rs b/src/librustc_middle/query/mod.rs index d874edf627472..ee1f0778bfa6d 100644 --- a/src/librustc_middle/query/mod.rs +++ b/src/librustc_middle/query/mod.rs @@ -112,6 +112,22 @@ rustc_queries! { desc { |tcx| "computing the optional const parameter of `{}`", tcx.def_path_str(key.to_def_id()) } } + // Converts a `LocalDefId` to a `HirId`. + // This can be conveniently accessed by `tcx.hir().as_local_hir_id`. + // Avoid calling this query directly. + query local_def_id_to_hir_id(key: LocalDefId) -> Option { + eval_always + desc { "converting a LocalDefId to HirId" } + } + + // A map of the HIR items which also have a `DefId` in addition to their `HirId`. + // This can be conveniently accessed by `tcx.hir().opt_local_def_id`. + // Avoid calling this query directly. + query hir_owner_defs(key: LocalDefId) -> Option<&'tcx FxHashMap> { + eval_always + desc { "getting a LocalDefId map" } + } + /// Records the type of every item. query type_of(key: DefId) -> Ty<'tcx> { desc { |tcx| "computing type of `{}`", tcx.def_path_str(key) } From 26eaae43f9ccdaba57810f714ac629d6ca2e7847 Mon Sep 17 00:00:00 2001 From: Aaron Hill Date: Mon, 17 Aug 2020 14:08:53 -0400 Subject: [PATCH 2/6] bug! on DefPath collision during forcing dep node Now that LocalDefId <-> HirId conversions are query-ified, we should never get to this point. --- src/librustc_middle/dep_graph/mod.rs | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/src/librustc_middle/dep_graph/mod.rs b/src/librustc_middle/dep_graph/mod.rs index 6697524279874..bcc4db9bbf75f 100644 --- a/src/librustc_middle/dep_graph/mod.rs +++ b/src/librustc_middle/dep_graph/mod.rs @@ -122,17 +122,7 @@ impl<'tcx> DepContext for TyCtxt<'tcx> { ); } } else { - // This `DefPath` does not have a - // corresponding `DepNode` (e.g. a - // struct field), and the ` DefPath` - // collided with the `DefPath` of a - // proper item that existed in the - // previous compilation session. - // - // Since the given `DefPath` does not - // denote the item that previously - // existed, we just fail to mark green. - return false; + bug!("DepNode {:?} collided (DefId {:?})", dep_node, def_id); } } else { // If the node does not exist anymore, we From eceb27bb7adae1b8d0e851ecef7804ad748d1432 Mon Sep 17 00:00:00 2001 From: Aaron Hill Date: Tue, 18 Aug 2020 15:50:02 -0400 Subject: [PATCH 3/6] Introduce IndexVecCache and use it for the new HIR queries --- src/librustc_middle/query/mod.rs | 2 + src/librustc_query_system/query/caches.rs | 81 +++++++++++++++++++++++ src/librustc_query_system/query/mod.rs | 1 + 3 files changed, 84 insertions(+) diff --git a/src/librustc_middle/query/mod.rs b/src/librustc_middle/query/mod.rs index ee1f0778bfa6d..e20bc36917618 100644 --- a/src/librustc_middle/query/mod.rs +++ b/src/librustc_middle/query/mod.rs @@ -116,6 +116,7 @@ rustc_queries! { // This can be conveniently accessed by `tcx.hir().as_local_hir_id`. // Avoid calling this query directly. query local_def_id_to_hir_id(key: LocalDefId) -> Option { + storage(IndexVecCacheSelector) eval_always desc { "converting a LocalDefId to HirId" } } @@ -124,6 +125,7 @@ rustc_queries! { // This can be conveniently accessed by `tcx.hir().opt_local_def_id`. // Avoid calling this query directly. query hir_owner_defs(key: LocalDefId) -> Option<&'tcx FxHashMap> { + storage(IndexVecCacheSelector) eval_always desc { "getting a LocalDefId map" } } diff --git a/src/librustc_query_system/query/caches.rs b/src/librustc_query_system/query/caches.rs index 1839e1af45eef..cf5eec9c3c824 100644 --- a/src/librustc_query_system/query/caches.rs +++ b/src/librustc_query_system/query/caches.rs @@ -1,6 +1,7 @@ use crate::dep_graph::DepNodeIndex; use crate::query::plumbing::{QueryLookup, QueryState}; use crate::query::QueryContext; +use rustc_index::vec::{Idx, IndexVec}; use rustc_arena::TypedArena; use rustc_data_structures::fx::FxHashMap; @@ -218,3 +219,83 @@ impl<'tcx, K: Eq + Hash, V: 'tcx> QueryCache for ArenaCache<'tcx, K, V> { f(Box::new(results)) } } + +pub struct IndexVecCacheSelector; + +impl CacheSelector for IndexVecCacheSelector { + type Cache = IndexVecCache; +} + +pub struct IndexVecCache(PhantomData<(K, V)>); + +impl Default for IndexVecCache { + fn default() -> Self { + IndexVecCache(PhantomData) + } +} + +impl QueryStorage for IndexVecCache { + type Value = V; + type Stored = V; + + #[inline] + fn store_nocache(&self, value: Self::Value) -> Self::Stored { + // We have no dedicated storage + value + } +} + +impl QueryCache for IndexVecCache { + type Key = K; + type Sharded = IndexVec>; + + #[inline(always)] + fn lookup( + &self, + state: &QueryState, + key: K, + on_hit: OnHit, + on_miss: OnMiss, + ) -> R + where + OnHit: FnOnce(&V, DepNodeIndex) -> R, + OnMiss: FnOnce(K, QueryLookup<'_, CTX, K, Self::Sharded>) -> R, + { + let mut lookup = state.get_lookup(&key); + let lock = &mut *lookup.lock; + + let result = lock.cache.get(key); + + if let Some(Some(value)) = result { on_hit(&value.1, value.2) } else { on_miss(key, lookup) } + } + + #[inline] + fn complete( + &self, + lock_sharded_storage: &mut Self::Sharded, + key: K, + value: V, + index: DepNodeIndex, + ) -> Self::Stored { + lock_sharded_storage.ensure_contains_elem(key, || None); + lock_sharded_storage[key] = Some((key, value.clone(), index)); + value + } + + + fn iter( + &self, + shards: &Sharded, + get_shard: impl Fn(&mut L) -> &mut Self::Sharded, + f: impl for<'a> FnOnce(Box + 'a>) -> R, + ) -> R { + let mut shards = shards.lock_shards(); + let mut shards: Vec<_> = shards.iter_mut().map(|shard| get_shard(shard)).collect(); + let results = shards.iter_mut().flat_map(|shard| { + shard.iter().flat_map(|v| { + v.as_ref().map(|val| (&val.0, &val.1, val.2)) + }) + }); + f(Box::new(results)) + } +} diff --git a/src/librustc_query_system/query/mod.rs b/src/librustc_query_system/query/mod.rs index 49097725bc9b9..5f842988083e7 100644 --- a/src/librustc_query_system/query/mod.rs +++ b/src/librustc_query_system/query/mod.rs @@ -9,6 +9,7 @@ pub use self::job::{QueryInfo, QueryJob, QueryJobId, QueryJobInfo}; mod caches; pub use self::caches::{ ArenaCacheSelector, CacheSelector, DefaultCacheSelector, QueryCache, QueryStorage, + IndexVecCacheSelector }; mod config; From 438f0d590d3be04d255a9b94f2d632ca6f88886d Mon Sep 17 00:00:00 2001 From: Aaron Hill Date: Tue, 18 Aug 2020 18:33:48 -0400 Subject: [PATCH 4/6] Avoid hashing key in IndexVecCache --- src/librustc_query_system/query/caches.rs | 10 +++--- src/librustc_query_system/query/plumbing.rs | 36 +++++++++++++-------- 2 files changed, 28 insertions(+), 18 deletions(-) diff --git a/src/librustc_query_system/query/caches.rs b/src/librustc_query_system/query/caches.rs index cf5eec9c3c824..bf304e0c71bcb 100644 --- a/src/librustc_query_system/query/caches.rs +++ b/src/librustc_query_system/query/caches.rs @@ -103,10 +103,10 @@ impl QueryCache for DefaultCache { OnHit: FnOnce(&V, DepNodeIndex) -> R, OnMiss: FnOnce(K, QueryLookup<'_, CTX, K, Self::Sharded>) -> R, { - let mut lookup = state.get_lookup(&key); + let (mut lookup, key_hash) = state.get_lookup(&key, true); let lock = &mut *lookup.lock; - let result = lock.cache.raw_entry().from_key_hashed_nocheck(lookup.key_hash, &key); + let result = lock.cache.raw_entry().from_key_hashed_nocheck(key_hash.unwrap(), &key); if let Some((_, value)) = result { on_hit(&value.0, value.1) } else { on_miss(key, lookup) } } @@ -181,10 +181,10 @@ impl<'tcx, K: Eq + Hash, V: 'tcx> QueryCache for ArenaCache<'tcx, K, V> { OnHit: FnOnce(&&'tcx V, DepNodeIndex) -> R, OnMiss: FnOnce(K, QueryLookup<'_, CTX, K, Self::Sharded>) -> R, { - let mut lookup = state.get_lookup(&key); + let (mut lookup, key_hash) = state.get_lookup(&key, true); let lock = &mut *lookup.lock; - let result = lock.cache.raw_entry().from_key_hashed_nocheck(lookup.key_hash, &key); + let result = lock.cache.raw_entry().from_key_hashed_nocheck(key_hash.unwrap(), &key); if let Some((_, value)) = result { on_hit(&&value.0, value.1) @@ -261,7 +261,7 @@ impl QueryCache for IndexVecCache { OnHit: FnOnce(&V, DepNodeIndex) -> R, OnMiss: FnOnce(K, QueryLookup<'_, CTX, K, Self::Sharded>) -> R, { - let mut lookup = state.get_lookup(&key); + let (mut lookup, _) = state.get_lookup(&key, false); let lock = &mut *lookup.lock; let result = lock.cache.get(key); diff --git a/src/librustc_query_system/query/plumbing.rs b/src/librustc_query_system/query/plumbing.rs index ae042cc808126..720f381a21e7b 100644 --- a/src/librustc_query_system/query/plumbing.rs +++ b/src/librustc_query_system/query/plumbing.rs @@ -13,7 +13,7 @@ use crate::query::QueryContext; use rustc_data_structures::cold_path; use rustc_data_structures::fingerprint::Fingerprint; use rustc_data_structures::fx::{FxHashMap, FxHasher}; -use rustc_data_structures::sharded::Sharded; +use rustc_data_structures::sharded::{Sharded, SHARDS}; use rustc_data_structures::sync::{Lock, LockGuard}; use rustc_data_structures::thin_vec::ThinVec; use rustc_errors::{Diagnostic, FatalError}; @@ -55,17 +55,28 @@ impl QueryState { pub(super) fn get_lookup<'tcx>( &'tcx self, key: &C::Key, - ) -> QueryLookup<'tcx, CTX, C::Key, C::Sharded> { - // We compute the key's hash once and then use it for both the - // shard lookup and the hashmap lookup. This relies on the fact - // that both of them use `FxHasher`. - let mut hasher = FxHasher::default(); - key.hash(&mut hasher); - let key_hash = hasher.finish(); - - let shard = self.shards.get_shard_index_by_hash(key_hash); - let lock = self.shards.get_shard_by_index(shard).lock(); - QueryLookup { key_hash, shard, lock } + needs_hash: bool + ) -> (QueryLookup<'tcx, CTX, C::Key, C::Sharded>, Option) { + let key_hash = if needs_hash || SHARDS != 1 { + // We compute the key's hash once and then use it for both the + // shard lookup and the hashmap lookup. This relies on the fact + // that both of them use `FxHasher`. + let mut hasher = FxHasher::default(); + key.hash(&mut hasher); + Some(hasher.finish()) + } else { + None + }; + + let (shard, lock) = if SHARDS == 1 { + (0, self.shards.get_shard_by_index(0).lock()) + } else { + let shard = self.shards.get_shard_index_by_hash(key_hash.unwrap()); + let lock = self.shards.get_shard_by_index(shard).lock(); + (shard, lock) + }; + + (QueryLookup { shard, lock }, key_hash) } } @@ -139,7 +150,6 @@ impl Default for QueryState { /// Values used when checking a query cache which can be reused on a cache-miss to execute the query. pub struct QueryLookup<'tcx, CTX: QueryContext, K, C> { - pub(super) key_hash: u64, shard: usize, pub(super) lock: LockGuard<'tcx, QueryStateShard>, } From 9db3cdd67e85c94146d5ec92cbb763eeac739343 Mon Sep 17 00:00:00 2001 From: Aaron Hill Date: Tue, 18 Aug 2020 18:34:24 -0400 Subject: [PATCH 5/6] Run fmt --- src/librustc_query_system/query/caches.rs | 11 ++++++----- src/librustc_query_system/query/mod.rs | 4 ++-- src/librustc_query_system/query/plumbing.rs | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/librustc_query_system/query/caches.rs b/src/librustc_query_system/query/caches.rs index bf304e0c71bcb..d02a84a6c79a4 100644 --- a/src/librustc_query_system/query/caches.rs +++ b/src/librustc_query_system/query/caches.rs @@ -266,7 +266,11 @@ impl QueryCache for IndexVecCache { let result = lock.cache.get(key); - if let Some(Some(value)) = result { on_hit(&value.1, value.2) } else { on_miss(key, lookup) } + if let Some(Some(value)) = result { + on_hit(&value.1, value.2) + } else { + on_miss(key, lookup) + } } #[inline] @@ -282,7 +286,6 @@ impl QueryCache for IndexVecCache { value } - fn iter( &self, shards: &Sharded, @@ -292,9 +295,7 @@ impl QueryCache for IndexVecCache { let mut shards = shards.lock_shards(); let mut shards: Vec<_> = shards.iter_mut().map(|shard| get_shard(shard)).collect(); let results = shards.iter_mut().flat_map(|shard| { - shard.iter().flat_map(|v| { - v.as_ref().map(|val| (&val.0, &val.1, val.2)) - }) + shard.iter().flat_map(|v| v.as_ref().map(|val| (&val.0, &val.1, val.2))) }); f(Box::new(results)) } diff --git a/src/librustc_query_system/query/mod.rs b/src/librustc_query_system/query/mod.rs index 5f842988083e7..fbdbd1847f01e 100644 --- a/src/librustc_query_system/query/mod.rs +++ b/src/librustc_query_system/query/mod.rs @@ -8,8 +8,8 @@ pub use self::job::{QueryInfo, QueryJob, QueryJobId, QueryJobInfo}; mod caches; pub use self::caches::{ - ArenaCacheSelector, CacheSelector, DefaultCacheSelector, QueryCache, QueryStorage, - IndexVecCacheSelector + ArenaCacheSelector, CacheSelector, DefaultCacheSelector, IndexVecCacheSelector, QueryCache, + QueryStorage, }; mod config; diff --git a/src/librustc_query_system/query/plumbing.rs b/src/librustc_query_system/query/plumbing.rs index 720f381a21e7b..6bbd1ec821b6d 100644 --- a/src/librustc_query_system/query/plumbing.rs +++ b/src/librustc_query_system/query/plumbing.rs @@ -55,7 +55,7 @@ impl QueryState { pub(super) fn get_lookup<'tcx>( &'tcx self, key: &C::Key, - needs_hash: bool + needs_hash: bool, ) -> (QueryLookup<'tcx, CTX, C::Key, C::Sharded>, Option) { let key_hash = if needs_hash || SHARDS != 1 { // We compute the key's hash once and then use it for both the From 9cc2e498f8a777ec7eee876aee44a0467cf61386 Mon Sep 17 00:00:00 2001 From: Aaron Hill Date: Wed, 19 Aug 2020 22:38:29 -0400 Subject: [PATCH 6/6] Avoid hashing key when storing in 'active' --- src/librustc_query_system/query/caches.rs | 9 ++ src/librustc_query_system/query/config.rs | 111 ++++++++++++++++++++ src/librustc_query_system/query/mod.rs | 3 +- src/librustc_query_system/query/plumbing.rs | 59 ++++++----- 4 files changed, 157 insertions(+), 25 deletions(-) diff --git a/src/librustc_query_system/query/caches.rs b/src/librustc_query_system/query/caches.rs index d02a84a6c79a4..a445b454a5ae8 100644 --- a/src/librustc_query_system/query/caches.rs +++ b/src/librustc_query_system/query/caches.rs @@ -1,4 +1,5 @@ use crate::dep_graph::DepNodeIndex; +use crate::query::config::QueryActiveStore; use crate::query::plumbing::{QueryLookup, QueryState}; use crate::query::QueryContext; use rustc_index::vec::{Idx, IndexVec}; @@ -28,6 +29,10 @@ pub trait QueryCache: QueryStorage { type Key: Hash; type Sharded: Default; + fn make_store() -> QueryActiveStore { + QueryActiveStore::Multi(Default::default()) + } + /// Checks if the query is already computed and in the cache. /// It returns the shard index and a lock guard to the shard, /// which will be used if the query is not in the cache and we need @@ -249,6 +254,10 @@ impl QueryCache for IndexVecCache { type Key = K; type Sharded = IndexVec>; + fn make_store() -> QueryActiveStore { + QueryActiveStore::Single(None) + } + #[inline(always)] fn lookup( &self, diff --git a/src/librustc_query_system/query/config.rs b/src/librustc_query_system/query/config.rs index 549056570f9bc..5f24b090d9c12 100644 --- a/src/librustc_query_system/query/config.rs +++ b/src/librustc_query_system/query/config.rs @@ -5,10 +5,12 @@ use crate::dep_graph::SerializedDepNodeIndex; use crate::query::caches::QueryCache; use crate::query::plumbing::CycleError; use crate::query::{QueryContext, QueryState}; +use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::profiling::ProfileCategory; use rustc_data_structures::fingerprint::Fingerprint; use std::borrow::Cow; +use std::collections::hash_map::Entry; use std::fmt::Debug; use std::hash::Hash; @@ -70,6 +72,115 @@ impl QueryVtable { } } +pub enum QueryActiveStore { + Single(Option<(K, V)>), + Multi(FxHashMap), +} + +impl QueryActiveStore { + pub fn is_empty(&self) -> bool { + match self { + QueryActiveStore::Single(val) => val.is_none(), + QueryActiveStore::Multi(val) => val.is_empty(), + } + } + pub fn iter(&self) -> Box + '_> { + match self { + QueryActiveStore::Single(val) => { + Box::new(val.as_ref().map(|entry| (&entry.0, &entry.1)).into_iter()) + } + QueryActiveStore::Multi(val) => Box::new(val.iter()), + } + } + pub fn get_or_insert( + &mut self, + key: K, + present: impl FnOnce(&mut V) -> R, + absent: impl FnOnce() -> (V, R), + ) -> R { + match self { + QueryActiveStore::Single(entry) => { + if let Some((_key, val)) = entry.as_mut() { + present(val) + } else { + let (val, res) = absent(); + *entry = Some((key, val)); + res + } + } + QueryActiveStore::Multi(map) => match map.entry(key) { + Entry::Occupied(mut entry) => present(entry.get_mut()), + Entry::Vacant(entry) => { + let (val, res) = absent(); + entry.insert(val); + res + } + }, + } + } + pub fn insert(&mut self, key: K, val: V) { + match self { + QueryActiveStore::Single(entry) => *entry = Some((key, val)), + QueryActiveStore::Multi(map) => { + map.insert(key, val); + } + } + } + pub fn remove(&mut self, key: &K) -> Option { + match self { + QueryActiveStore::Single(entry) => entry.take().map(|(_k, v)| v), + QueryActiveStore::Multi(map) => map.remove(key), + } + } +} + +/*pub trait QueryActiveStore { + fn is_empty(&self) -> bool; + //fn iter(&self) -> Box + '_>; + fn get_or_insert(&mut self, key: K, present: impl FnOnce(&mut V) -> R, absent: impl FnOnce() -> (V, R)) -> R; + fn remove(&mut self, key: &K); +} + +impl QueryActiveStore for FxHashMap { + fn is_empty(&self) -> bool { FxHashMap::is_empty(self) } + //fn iter(&self) -> Box + '_> { Box::new(FxHashMap::iter(self)) } + fn get_or_insert(&mut self, key: K, present: impl FnOnce(&mut V) -> R, absent: impl FnOnce() -> (V, R)) -> R { + match self.entry(key) { + Entry::Occupied(mut entry) => { + present(entry.get_mut()) + } + Entry::Vacant(entry) => { + let (val, res) = absent(); + entry.insert(val); + res + } + } + } + fn remove(&mut self, key: &K) { + FxHashMap::remove(self, key); + } +} + +impl QueryActiveStore for RefCell> { + fn is_empty(&self) -> bool { self.borrow().is_none() } + /*fn iter(&self) -> Box + '_> { + Box::new(self.borrow().as_ref().map(|val| (&val.0, &val.1)).into_iter()) + }*/ + fn get_or_insert(&mut self, key: K, present: impl FnOnce(&mut V) -> R, absent: impl FnOnce() -> (V, R)) -> R { + let entry = &mut *self.borrow_mut(); + if let Some((_key, val)) = entry { + present(val) + } else { + let (val, res) = absent(); + *entry = Some((key, val)); + res + } + } + fn remove(&mut self, key: &K) { + self.borrow_mut().take(); + } +}*/ + pub trait QueryAccessors: QueryConfig { const ANON: bool; const EVAL_ALWAYS: bool; diff --git a/src/librustc_query_system/query/mod.rs b/src/librustc_query_system/query/mod.rs index fbdbd1847f01e..946881344cf71 100644 --- a/src/librustc_query_system/query/mod.rs +++ b/src/librustc_query_system/query/mod.rs @@ -15,9 +15,10 @@ pub use self::caches::{ mod config; pub use self::config::{QueryAccessors, QueryConfig, QueryDescription}; +use rustc_data_structures::fx::FxHashMap; + use crate::dep_graph::{DepContext, DepGraph}; -use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::stable_hasher::HashStable; use rustc_data_structures::sync::Lock; use rustc_data_structures::thin_vec::ThinVec; diff --git a/src/librustc_query_system/query/plumbing.rs b/src/librustc_query_system/query/plumbing.rs index 6bbd1ec821b6d..b47c2a6711000 100644 --- a/src/librustc_query_system/query/plumbing.rs +++ b/src/librustc_query_system/query/plumbing.rs @@ -5,7 +5,7 @@ use crate::dep_graph::{DepKind, DepNode}; use crate::dep_graph::{DepNodeIndex, SerializedDepNodeIndex}; use crate::query::caches::QueryCache; -use crate::query::config::{QueryDescription, QueryVtable, QueryVtableExt}; +use crate::query::config::{QueryActiveStore, QueryDescription, QueryVtable, QueryVtableExt}; use crate::query::job::{QueryInfo, QueryJob, QueryJobId, QueryJobInfo, QueryShardJobId}; use crate::query::QueryContext; @@ -19,7 +19,6 @@ use rustc_data_structures::thin_vec::ThinVec; use rustc_errors::{Diagnostic, FatalError}; use rustc_span::source_map::DUMMY_SP; use rustc_span::Span; -use std::collections::hash_map::Entry; use std::convert::TryFrom; use std::fmt::Debug; use std::hash::{Hash, Hasher}; @@ -31,18 +30,12 @@ use std::sync::atomic::{AtomicUsize, Ordering}; pub(super) struct QueryStateShard { pub(super) cache: C, - active: FxHashMap>, + active: QueryActiveStore>, /// Used to generate unique ids for active jobs. jobs: u32, } -impl Default for QueryStateShard { - fn default() -> QueryStateShard { - QueryStateShard { cache: Default::default(), active: Default::default(), jobs: 0 } - } -} - pub struct QueryState { cache: C, shards: Sharded>, @@ -100,7 +93,12 @@ impl QueryState { ) -> R { self.cache.iter(&self.shards, |shard| &mut shard.cache, f) } +} +impl QueryState +where + C::Key: Eq + Hash + Clone + Debug, +{ #[inline(always)] pub fn all_inactive(&self) -> bool { let shards = self.shards.lock_shards(); @@ -141,7 +139,11 @@ impl Default for QueryState { fn default() -> QueryState { QueryState { cache: C::default(), - shards: Default::default(), + shards: Sharded::new(|| QueryStateShard { + cache: Default::default(), + active: C::make_store(), + jobs: 0, + }), #[cfg(debug_assertions)] cache_hits: AtomicUsize::new(0), } @@ -193,9 +195,13 @@ where { let lock = &mut *lookup.lock; - let (latch, mut _query_blocked_prof_timer) = match lock.active.entry((*key).clone()) { - Entry::Occupied(mut entry) => { - match entry.get_mut() { + let shard = lookup.shard; + let jobs = &mut lock.jobs; + + let res = lock.active.get_or_insert( + (*key).clone(), + |result| { + match result { QueryResult::Started(job) => { // For parallel queries, we'll block and wait until the query running // in another thread has completed. Record how long we wait in the @@ -207,34 +213,39 @@ where }; // Create the id of the job we're waiting for - let id = QueryJobId::new(job.id, lookup.shard, query.dep_kind); + let id = QueryJobId::new(job.id, shard, query.dep_kind); - (job.latch(id), _query_blocked_prof_timer) + Err((job.latch(id), _query_blocked_prof_timer)) } QueryResult::Poisoned => FatalError.raise(), } - } - Entry::Vacant(entry) => { + }, + || { // No job entry for this query. Return a new one to be started later. // Generate an id unique within this shard. - let id = lock.jobs.checked_add(1).unwrap(); - lock.jobs = id; + let id = jobs.checked_add(1).unwrap(); + *jobs = id; let id = QueryShardJobId(NonZeroU32::new(id).unwrap()); - let global_id = QueryJobId::new(id, lookup.shard, query.dep_kind); + let global_id = QueryJobId::new(id, shard, query.dep_kind); let job = tcx.current_query_job(); let job = QueryJob::new(id, span, job); - entry.insert(QueryResult::Started(job)); + let result = QueryResult::Started(job); let owner = JobOwner { state, id: global_id, key: (*key).clone() }; - return TryGetJob::NotYetStarted(owner); - } - }; + (result, Ok(TryGetJob::NotYetStarted(owner))) + }, + ); mem::drop(lookup.lock); + let (latch, mut _query_blocked_prof_timer) = match res { + Ok(val) => return val, + Err(val) => val, + }; + // If we are single-threaded we know that we have cycle error, // so we just return the error. #[cfg(not(parallel_compiler))]