diff --git a/compiler/rustc_middle/src/query/mod.rs b/compiler/rustc_middle/src/query/mod.rs index 03f3ceb8d1733..bc69f0fd14e6b 100644 --- a/compiler/rustc_middle/src/query/mod.rs +++ b/compiler/rustc_middle/src/query/mod.rs @@ -1066,7 +1066,7 @@ rustc_queries! { "const-evaluating + checking `{}`", key.value.display(tcx) } - cache_on_disk_if { true } + // cache_on_disk_if { true } Cannot cache this anymore } /// Evaluates const items or anonymous constants @@ -1081,7 +1081,7 @@ rustc_queries! { "simplifying constant for the type system `{}`", key.value.display(tcx) } - cache_on_disk_if { true } + // cache_on_disk_if { true } Cannot cache this anymore } /// Evaluate a constant and convert it to a type level constant or @@ -1148,7 +1148,7 @@ rustc_queries! { /// look up the correct symbol name of instances from upstream crates. query symbol_name(key: ty::Instance<'tcx>) -> ty::SymbolName<'tcx> { desc { "computing the symbol for `{}`", key } - cache_on_disk_if { true } + // cache_on_disk_if { true } Cannot cache this anymore } query def_kind(def_id: DefId) -> DefKind { @@ -1284,7 +1284,7 @@ rustc_queries! { query codegen_select_candidate( key: (ty::ParamEnv<'tcx>, ty::TraitRef<'tcx>) ) -> Result<&'tcx ImplSource<'tcx, ()>, CodegenObligationError> { - cache_on_disk_if { true } + // cache_on_disk_if { true } Cannot cache this anymore desc { |tcx| "computing candidate for `{}`", key.1 } } @@ -1894,7 +1894,7 @@ rustc_queries! { } query unused_generic_params(key: ty::InstanceDef<'tcx>) -> UnusedGenericParams { - cache_on_disk_if { key.def_id().is_local() } + // cache_on_disk_if { key.def_id().is_local() } Cannot cache this anymore desc { |tcx| "determining which generic parameters are unused by `{}`", tcx.def_path_str(key.def_id()) diff --git a/compiler/rustc_query_system/src/dep_graph/graph.rs b/compiler/rustc_query_system/src/dep_graph/graph.rs index bc09972185a62..5012c834684e7 100644 --- a/compiler/rustc_query_system/src/dep_graph/graph.rs +++ b/compiler/rustc_query_system/src/dep_graph/graph.rs @@ -133,7 +133,7 @@ impl DepGraph { let colors = DepNodeColorMap::new(prev_graph_node_count); // Instantiate a dependy-less node only once for anonymous queries. - let _green_node_index = current.intern_new_node( + let _green_node_index = current.alloc_new_node( profiler, DepNode { kind: D::DEP_KIND_NULL, hash: current.anon_id_seed.into() }, EdgesVec::new(), @@ -272,6 +272,7 @@ impl DepGraph { D::with_deps(TaskDepsRef::Forbid, op) } + // FIXME(sparse_fps): Document #[inline(always)] pub fn with_task, A: Debug, R>( &self, @@ -287,6 +288,7 @@ impl DepGraph { } } + // FIXME(sparse_fps): Document pub fn with_anon_task, OP, R>( &self, cx: Tcx, @@ -297,7 +299,7 @@ impl DepGraph { OP: FnOnce() -> R, { match self.data() { - Some(data) => data.with_anon_task(cx, dep_kind, op), + Some(data) => data.with_anon_task(cx, dep_kind, true, op), None => (op(), self.next_virtual_depnode_index()), } } @@ -395,61 +397,71 @@ impl DepGraphData { (result, dep_node_index) } + // FIXME(sparse_fps): Document /// Executes something within an "anonymous" task, that is, a task the /// `DepNode` of which is determined by the list of inputs it read from. pub(crate) fn with_anon_task, OP, R>( &self, cx: Tcx, dep_kind: DepKind, + intern: bool, op: OP, ) -> (R, DepNodeIndex) where OP: FnOnce() -> R, { - debug_assert!(!cx.is_eval_always(dep_kind)); - let task_deps = Lock::new(TaskDeps::default()); let result = D::with_deps(TaskDepsRef::Allow(&task_deps), op); let task_deps = task_deps.into_inner(); let task_deps = task_deps.reads; - let dep_node_index = match task_deps.len() { - 0 => { - // Because the dep-node id of anon nodes is computed from the sets of its - // dependencies we already know what the ID of this dependency-less node is - // going to be (i.e. equal to the precomputed - // `SINGLETON_DEPENDENCYLESS_ANON_NODE`). As a consequence we can skip creating - // a `StableHasher` and sending the node through interning. - DepNodeIndex::SINGLETON_DEPENDENCYLESS_ANON_NODE - } - 1 => { - // When there is only one dependency, don't bother creating a node. - task_deps[0] - } - _ => { - // The dep node indices are hashed here instead of hashing the dep nodes of the - // dependencies. These indices may refer to different nodes per session, but this isn't - // a problem here because we that ensure the final dep node hash is per session only by - // combining it with the per session random number `anon_id_seed`. This hash only need - // to map the dependencies to a single value on a per session basis. - let mut hasher = StableHasher::new(); - task_deps.hash(&mut hasher); - - let target_dep_node = DepNode { - kind: dep_kind, - // Fingerprint::combine() is faster than sending Fingerprint - // through the StableHasher (at least as long as StableHasher - // is so slow). - hash: self.current.anon_id_seed.combine(hasher.finish()).into(), - }; - - self.current.intern_new_node( - cx.profiler(), - target_dep_node, - task_deps, - Fingerprint::ZERO, - ) + let dep_node_index = if intern { + // FIXME(sparse_fp): what is this assertion about? + debug_assert!(!cx.is_eval_always(dep_kind)); + + match task_deps.len() { + 0 => { + // Because the dep-node id of anon nodes is computed from the sets of its + // dependencies we already know what the ID of this dependency-less node is + // going to be (i.e. equal to the precomputed + // `SINGLETON_DEPENDENCYLESS_ANON_NODE`). As a consequence we can skip creating + // a `StableHasher` and sending the node through interning. + DepNodeIndex::SINGLETON_DEPENDENCYLESS_ANON_NODE + } + 1 => { + // When there is only one dependency, don't bother creating a node. + task_deps[0] + } + _ => { + // The dep node indices are hashed here instead of hashing the dep nodes of the + // dependencies. These indices may refer to different nodes per session, but this isn't + // a problem here because we that ensure the final dep node hash is per session only by + // combining it with the per session random number `anon_id_seed`. This hash only need + // to map the dependencies to a single value on a per session basis. + let mut hasher = StableHasher::new(); + task_deps.hash(&mut hasher); + dep_kind.hash(&mut hasher); + + let dedup_fingerprint: Fingerprint = hasher.finish(); + + match self + .current + .interned_node_to_index + .lock_shard_by_value(&dedup_fingerprint) + .entry(dedup_fingerprint) + { + Entry::Occupied(entry) => *entry.get(), + Entry::Vacant(entry) => { + let dep_node_index = + self.current.alloc_anon_node(cx.profiler(), dep_kind, task_deps); + entry.insert(dep_node_index); + dep_node_index + } + } + } } + } else { + self.current.alloc_anon_node(cx.profiler(), dep_kind, task_deps) }; (result, dep_node_index) @@ -616,18 +628,20 @@ impl DepGraph { } impl DepGraphData { - #[inline] - fn dep_node_index_of_opt(&self, dep_node: &DepNode) -> Option { - if let Some(prev_index) = self.previous.node_to_index_opt(dep_node) { - self.current.prev_index_to_index.lock()[prev_index] - } else { - self.current.new_node_to_index.lock_shard_by_value(dep_node).get(dep_node).copied() - } - } + // #[inline] + // fn dep_node_index_of_opt(&self, dep_node: &DepNode) -> Option { + // if let Some(prev_index) = self.previous.node_to_index_opt(dep_node) { + // self.current.prev_index_to_index.lock()[prev_index] + // } else { + // self.current.interned_node_to_index.lock_shard_by_value(dep_node).get(dep_node).copied() + // } + // } #[inline] - fn dep_node_exists(&self, dep_node: &DepNode) -> bool { - self.dep_node_index_of_opt(dep_node).is_some() + fn dep_node_exists(&self, _dep_node: &DepNode) -> bool { + // FIXME(sparse_fps): bring back assertions + //self.dep_node_index_of_opt(dep_node).is_some() + false } fn node_color(&self, dep_node: &DepNode) -> Option { @@ -1071,7 +1085,7 @@ rustc_index::newtype_index! { /// first, and `data` second. pub(super) struct CurrentDepGraph { encoder: Steal>, - new_node_to_index: Sharded>, + interned_node_to_index: Sharded>, prev_index_to_index: Lock>>, /// This is used to verify that fingerprints do not change between the creation of a node @@ -1152,7 +1166,7 @@ impl CurrentDepGraph { record_graph, record_stats, )), - new_node_to_index: Sharded::new(|| { + interned_node_to_index: Sharded::new(|| { FxHashMap::with_capacity_and_hasher( new_node_count_estimate / sharded::shards(), Default::default(), @@ -1182,29 +1196,30 @@ impl CurrentDepGraph { /// Writes the node to the current dep-graph and allocates a `DepNodeIndex` for it. /// Assumes that this is a node that has no equivalent in the previous dep-graph. #[inline(always)] - fn intern_new_node( + fn alloc_new_node( &self, profiler: &SelfProfilerRef, key: DepNode, edges: EdgesVec, current_fingerprint: Fingerprint, ) -> DepNodeIndex { - let dep_node_index = match self.new_node_to_index.lock_shard_by_value(&key).entry(key) { - Entry::Occupied(entry) => *entry.get(), - Entry::Vacant(entry) => { - let dep_node_index = - self.encoder.borrow().send(profiler, key, current_fingerprint, edges); - entry.insert(dep_node_index); - dep_node_index - } - }; - + let dep_node_index = self.encoder.borrow().send(profiler, key, current_fingerprint, edges); #[cfg(debug_assertions)] self.record_edge(dep_node_index, key, current_fingerprint); dep_node_index } + #[inline(always)] + fn alloc_anon_node( + &self, + profiler: &SelfProfilerRef, + dep_kind: DepKind, + edges: EdgesVec, + ) -> DepNodeIndex { + self.encoder.borrow().send_anon_node(profiler, dep_kind, edges) + } + fn intern_node( &self, profiler: &SelfProfilerRef, @@ -1262,7 +1277,7 @@ impl CurrentDepGraph { let fingerprint = fingerprint.unwrap_or(Fingerprint::ZERO); // This is a new node: it didn't exist in the previous compilation session. - let dep_node_index = self.intern_new_node(profiler, key, edges, fingerprint); + let dep_node_index = self.alloc_new_node(profiler, key, edges, fingerprint); (dep_node_index, None) } @@ -1274,7 +1289,8 @@ impl CurrentDepGraph { prev_graph: &SerializedDepGraph, prev_index: SerializedDepNodeIndex, ) -> DepNodeIndex { - self.debug_assert_not_in_new_nodes(prev_graph, prev_index); + // FIXME(sparse_fp): restore assertions + // self.debug_assert_not_in_new_nodes(prev_graph, prev_index); let mut prev_index_to_index = self.prev_index_to_index.lock(); @@ -1296,18 +1312,19 @@ impl CurrentDepGraph { } } - #[inline] - fn debug_assert_not_in_new_nodes( - &self, - prev_graph: &SerializedDepGraph, - prev_index: SerializedDepNodeIndex, - ) { - let node = &prev_graph.index_to_node(prev_index); - debug_assert!( - !self.new_node_to_index.lock_shard_by_value(node).contains_key(node), - "node from previous graph present in new node collection" - ); - } + // FIXME(sparse_fp): restore assertions + // #[inline] + // fn debug_assert_not_in_new_nodes( + // &self, + // prev_graph: &SerializedDepGraph, + // prev_index: SerializedDepNodeIndex, + // ) { + // let node = &prev_graph.index_to_node(prev_index); + // debug_assert!( + // !self.interned_node_to_index.lock_shard_by_value(node).contains_key(node), + // "node from previous graph present in new node collection" + // ); + // } } #[derive(Debug, Clone, Copy)] diff --git a/compiler/rustc_query_system/src/dep_graph/serialized.rs b/compiler/rustc_query_system/src/dep_graph/serialized.rs index a70f4138cfb53..a8dd14c2b807f 100644 --- a/compiler/rustc_query_system/src/dep_graph/serialized.rs +++ b/compiler/rustc_query_system/src/dep_graph/serialized.rs @@ -259,8 +259,16 @@ impl SerializedDepGraph { .map(|_| UnhashMap::with_capacity_and_hasher(d.read_u32() as usize, Default::default())) .collect(); + let anon_node_fingerprint: PackedFingerprint = anon_node_fingerprint().into(); + for (idx, node) in nodes.iter_enumerated() { - index[node.kind.as_usize()].insert(node.hash, idx); + // FIXME(sparse_fps): Filter out anon nodes from the reverse index + // by looking for the special DepNode::hash. This + // is a hack, we should not store depnodes and + // fingerprints for these to begin with. + if node.hash != anon_node_fingerprint { + index[node.kind.as_usize()].insert(node.hash, idx); + } } SerializedDepGraph { nodes, fingerprints, edge_list_indices, edge_list_data, index } @@ -311,7 +319,14 @@ impl SerializedNodeHeader { fn new(node_info: &NodeInfo) -> Self { debug_assert_eq!(Self::TOTAL_BITS, Self::LEN_BITS + Self::WIDTH_BITS + Self::KIND_BITS); - let NodeInfo { node, fingerprint, edges } = node_info; + let (node, fingerprint, edges) = match node_info { + NodeInfo::Full(node, fingerprint, edges) => (*node, *fingerprint, edges), + NodeInfo::Anon(kind, edges) => ( + DepNode { kind: *kind, hash: anon_node_fingerprint().into() }, + anon_node_fingerprint(), + edges, + ), + }; let mut head = node.kind.as_inner(); @@ -336,10 +351,10 @@ impl SerializedNodeHeader { #[cfg(debug_assertions)] { let res = Self { bytes, _marker: PhantomData }; - assert_eq!(node_info.fingerprint, res.fingerprint()); - assert_eq!(node_info.node, res.node()); + assert_eq!(node_info.fingerprint(), res.fingerprint()); + assert_eq!(node_info.node(), res.node()); if let Some(len) = res.len() { - assert_eq!(node_info.edges.len(), len); + assert_eq!(node_info.edges().len(), len); } } Self { bytes, _marker: PhantomData } @@ -394,23 +409,45 @@ impl SerializedNodeHeader { } #[derive(Debug)] -struct NodeInfo { - node: DepNode, - fingerprint: Fingerprint, - edges: EdgesVec, +enum NodeInfo { + Full(DepNode, Fingerprint, EdgesVec), + Anon(DepKind, EdgesVec), } impl NodeInfo { + #[cfg(debug_assertions)] + fn fingerprint(&self) -> Fingerprint { + match *self { + NodeInfo::Full(_, fingerprint, _) => fingerprint, + NodeInfo::Anon(_, _) => anon_node_fingerprint(), + } + } + + #[inline] + fn node(&self) -> DepNode { + match *self { + NodeInfo::Full(node, _, _) => node, + NodeInfo::Anon(kind, _) => DepNode { kind, hash: anon_node_fingerprint().into() }, + } + } + + #[inline] + fn edges(&self) -> &EdgesVec { + match self { + NodeInfo::Full(_, _, edges) | NodeInfo::Anon(_, edges) => edges, + } + } + fn encode(&self, e: &mut FileEncoder) { let header = SerializedNodeHeader::::new(self); e.write_array(header.bytes); if header.len().is_none() { - e.emit_usize(self.edges.len()); + e.emit_usize(self.edges().len()); } let bytes_per_index = header.bytes_per_index(); - for node_index in self.edges.iter() { + for node_index in self.edges().iter() { e.write_with(|dest| { *dest = node_index.as_u32().to_le_bytes(); bytes_per_index @@ -455,20 +492,20 @@ impl EncoderState { ) -> DepNodeIndex { let index = DepNodeIndex::new(self.total_node_count); self.total_node_count += 1; - self.kind_stats[node.node.kind.as_usize()] += 1; + self.kind_stats[node.node().kind.as_usize()] += 1; - let edge_count = node.edges.len(); + let edge_count = node.edges().len(); self.total_edge_count += edge_count; if let Some(record_graph) = &record_graph { // Do not ICE when a query is called from within `with_query`. if let Some(record_graph) = &mut record_graph.try_lock() { - record_graph.push(index, node.node, &node.edges); + record_graph.push(index, node.node(), node.edges()); } } if let Some(stats) = &mut self.stats { - let kind = node.node.kind; + let kind = node.node().kind; let stat = stats.entry(kind).or_insert(Stat { kind, node_counter: 0, edge_counter: 0 }); stat.node_counter += 1; @@ -597,7 +634,18 @@ impl GraphEncoder { edges: EdgesVec, ) -> DepNodeIndex { let _prof_timer = profiler.generic_activity("incr_comp_encode_dep_graph"); - let node = NodeInfo { node, fingerprint, edges }; + let node = NodeInfo::Full(node, fingerprint, edges); + self.status.lock().encode_node(&node, &self.record_graph) + } + + pub(crate) fn send_anon_node( + &self, + profiler: &SelfProfilerRef, + dep_kind: DepKind, + edges: EdgesVec, + ) -> DepNodeIndex { + let _prof_timer = profiler.generic_activity("incr_comp_encode_dep_graph"); + let node = NodeInfo::Anon(dep_kind, edges); self.status.lock().encode_node(&node, &self.record_graph) } @@ -606,3 +654,8 @@ impl GraphEncoder { self.status.into_inner().finish(profiler) } } + +#[inline] +pub fn anon_node_fingerprint() -> Fingerprint { + Fingerprint::new(0x01234567_89abcdef, 0xfedcba98_7654321) +} diff --git a/compiler/rustc_query_system/src/query/plumbing.rs b/compiler/rustc_query_system/src/query/plumbing.rs index 41638b38c742d..68a957f247eb2 100644 --- a/compiler/rustc_query_system/src/query/plumbing.rs +++ b/compiler/rustc_query_system/src/query/plumbing.rs @@ -491,7 +491,10 @@ where Q: QueryConfig, Qcx: QueryContext, { - if !query.anon() && !query.eval_always() { + let is_reconstructible = !query.anon() + && qcx.dep_context().dep_kind_info(query.dep_kind()).force_from_dep_node.is_some(); + + if is_reconstructible && !query.eval_always() { // `to_dep_node` is expensive for some `DepKind`s. let dep_node = dep_node_opt.get_or_insert_with(|| query.construct_dep_node(*qcx.dep_context(), &key)); @@ -510,23 +513,26 @@ where let (result, dep_node_index) = qcx.start_query(job_id, query.depth_limit(), Some(&diagnostics), || { - if query.anon() { - return dep_graph_data.with_anon_task(*qcx.dep_context(), query.dep_kind(), || { - query.compute(qcx, key) - }); + if is_reconstructible { + // `to_dep_node` is expensive for some `DepKind`s. + let dep_node = dep_node_opt + .unwrap_or_else(|| query.construct_dep_node(*qcx.dep_context(), &key)); + + dep_graph_data.with_task( + dep_node, + (qcx, query), + key, + |(qcx, query), key| query.compute(qcx, key), + query.hash_result(), + ) + } else { + dep_graph_data.with_anon_task( + *qcx.dep_context(), + query.dep_kind(), + query.anon(), + || query.compute(qcx, key), + ) } - - // `to_dep_node` is expensive for some `DepKind`s. - let dep_node = - dep_node_opt.unwrap_or_else(|| query.construct_dep_node(*qcx.dep_context(), &key)); - - dep_graph_data.with_task( - dep_node, - (qcx, query), - key, - |(qcx, query), key| query.compute(qcx, key), - query.hash_result(), - ) }); prof_timer.finish_with_query_invocation_id(dep_node_index.into());