diff --git a/Cargo.lock b/Cargo.lock index 1b89223c8bef1..508bcaf57d533 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1046,7 +1046,7 @@ dependencies = [ "autocfg", "cfg-if", "crossbeam-utils", - "memoffset", + "memoffset 0.7.1", "scopeguard", ] @@ -2622,6 +2622,15 @@ dependencies = [ "autocfg", ] +[[package]] +name = "memoffset" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1" +dependencies = [ + "autocfg", +] + [[package]] name = "minifier" version = "0.2.2" @@ -4556,6 +4565,7 @@ name = "rustc_query_impl" version = "0.0.0" dependencies = [ "measureme", + "memoffset 0.8.0", "rustc-rayon-core", "rustc_ast", "rustc_data_structures", diff --git a/compiler/rustc_interface/src/passes.rs b/compiler/rustc_interface/src/passes.rs index 81c1d665ef072..043f7e49f5b96 100644 --- a/compiler/rustc_interface/src/passes.rs +++ b/compiler/rustc_interface/src/passes.rs @@ -17,7 +17,7 @@ use rustc_lint::{unerased_lint_store, BufferedEarlyLint, EarlyCheckNode, LintSto use rustc_metadata::creader::CStore; use rustc_middle::arena::Arena; use rustc_middle::dep_graph::DepGraph; -use rustc_middle::ty::query::{ExternProviders, Providers}; +use rustc_middle::ty::query::{ExternProviders, Providers, QuerySystem}; use rustc_middle::ty::{self, GlobalCtxt, RegisteredTools, TyCtxt}; use rustc_mir_build as mir_build; use rustc_parse::{parse_crate_from_file, parse_crate_from_source_str, validate_attr}; @@ -687,9 +687,10 @@ pub fn create_global_ctxt<'tcx>( callback(sess, &mut local_providers, &mut extern_providers); } - let queries = queries.get_or_init(|| { - TcxQueries::new(local_providers, extern_providers, query_result_on_disk_cache) - }); + let queries = + queries.get_or_init(|| TcxQueries::new(&local_providers, query_result_on_disk_cache)); + + let query_system = QuerySystem::new(local_providers, extern_providers); sess.time("setup_global_ctxt", || { gcx_cell.get_or_init(move || { @@ -701,6 +702,7 @@ pub fn create_global_ctxt<'tcx>( untracked, dep_graph, queries.on_disk_cache.as_ref().map(OnDiskCache::as_dyn), + query_system, queries.as_dyn(), rustc_query_impl::query_callbacks(arena), ) diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs index 433c9ab9aa2ef..35280676a0271 100644 --- a/compiler/rustc_middle/src/ty/context.rs +++ b/compiler/rustc_middle/src/ty/context.rs @@ -78,6 +78,8 @@ use std::iter; use std::mem; use std::ops::{Bound, Deref}; +use super::query::QuerySystem; + const TINY_CONST_EVAL_LIMIT: Limit = Limit(20); pub trait OnDiskCache<'tcx>: rustc_data_structures::sync::Sync { @@ -681,6 +683,7 @@ impl<'tcx> TyCtxt<'tcx> { untracked: Untracked, dep_graph: DepGraph, on_disk_cache: Option<&'tcx dyn OnDiskCache<'tcx>>, + query_system: QuerySystem<'tcx>, queries: &'tcx dyn query::QueryEngine<'tcx>, query_kinds: &'tcx [DepKindStruct<'tcx>], ) -> GlobalCtxt<'tcx> { @@ -706,7 +709,7 @@ impl<'tcx> TyCtxt<'tcx> { untracked, on_disk_cache, queries, - query_system: Default::default(), + query_system, query_kinds, ty_rcache: Default::default(), pred_rcache: Default::default(), diff --git a/compiler/rustc_middle/src/ty/query.rs b/compiler/rustc_middle/src/ty/query.rs index 3d9a5075d4ade..cac74944c64d9 100644 --- a/compiler/rustc_middle/src/ty/query.rs +++ b/compiler/rustc_middle/src/ty/query.rs @@ -71,12 +71,24 @@ use std::sync::Arc; pub(crate) use rustc_query_system::query::QueryJobId; use rustc_query_system::query::*; -#[derive(Default)] pub struct QuerySystem<'tcx> { + pub local_providers: Box, + pub extern_providers: Box, pub arenas: QueryArenas<'tcx>, pub caches: QueryCaches<'tcx>, } +impl<'tcx> QuerySystem<'tcx> { + pub fn new(local_providers: Providers, extern_providers: ExternProviders) -> Self { + QuerySystem { + local_providers: Box::new(local_providers), + extern_providers: Box::new(extern_providers), + arenas: Default::default(), + caches: Default::default(), + } + } +} + #[derive(Copy, Clone)] pub struct TyCtxtAt<'tcx> { pub tcx: TyCtxt<'tcx>, diff --git a/compiler/rustc_query_impl/Cargo.toml b/compiler/rustc_query_impl/Cargo.toml index 3e8a88c7e81c5..0233513afaca0 100644 --- a/compiler/rustc_query_impl/Cargo.toml +++ b/compiler/rustc_query_impl/Cargo.toml @@ -7,6 +7,7 @@ edition = "2021" [dependencies] +memoffset = { version = "0.8.0", features = ["unstable_const"] } measureme = "10.0.0" rustc_ast = { path = "../rustc_ast" } rustc_data_structures = { path = "../rustc_data_structures" } diff --git a/compiler/rustc_query_impl/src/lib.rs b/compiler/rustc_query_impl/src/lib.rs index d7708a3bc3f42..fe2ff2345d225 100644 --- a/compiler/rustc_query_impl/src/lib.rs +++ b/compiler/rustc_query_impl/src/lib.rs @@ -3,12 +3,13 @@ #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")] // this shouldn't be necessary, but the check for `&mut _` is too naive and denies returning a function pointer that takes a mut ref #![feature(const_mut_refs)] +#![feature(const_refs_to_cell)] #![feature(min_specialization)] #![feature(never_type)] #![feature(once_cell)] #![feature(rustc_attrs)] #![recursion_limit = "256"] -#![allow(rustc::potential_query_instability)] +#![allow(rustc::potential_query_instability, unused_parens)] #![deny(rustc::untranslatable_diagnostic)] #![deny(rustc::diagnostic_outside_of_impl)] @@ -17,25 +18,33 @@ extern crate rustc_macros; #[macro_use] extern crate rustc_middle; -use rustc_data_structures::sync::AtomicU64; +use memoffset::offset_of; +use rustc_data_structures::fingerprint::Fingerprint; +use rustc_data_structures::stable_hasher::HashStable; use rustc_middle::arena::Arena; use rustc_middle::dep_graph::{self, DepKindStruct}; use rustc_middle::query::Key; +use rustc_middle::ty::query::QueryCaches; use rustc_middle::ty::query::{ query_keys, query_provided, query_provided_to_value, query_storage, query_values, }; -use rustc_middle::ty::query::{ExternProviders, Providers, QueryEngine}; +use rustc_middle::ty::query::{Providers, QueryEngine}; use rustc_middle::ty::TyCtxt; +use rustc_query_system::dep_graph::DepNodeParams; +use rustc_query_system::ich::StableHashingContext; use rustc_span::Span; +use std::fmt; +use std::marker::PhantomData; #[macro_use] mod plumbing; -pub use plumbing::QueryCtxt; +pub use plumbing::{Queries, QueryCtxt}; use rustc_query_system::query::*; #[cfg(parallel_compiler)] pub use rustc_query_system::query::{deadlock, QueryContext}; pub use rustc_query_system::query::QueryConfig; +use rustc_query_system::HandleCycleError; mod on_disk_cache; pub use on_disk_cache::OnDiskCache; @@ -43,11 +52,177 @@ pub use on_disk_cache::OnDiskCache; mod profiling_support; pub use self::profiling_support::alloc_self_profile_query_strings; -rustc_query_append! { define_queries! } +trait Bool { + fn value() -> bool; +} + +struct True; + +impl Bool for True { + fn value() -> bool { + true + } +} +struct False; + +impl Bool for False { + fn value() -> bool { + false + } +} +struct DynamicQuery<'tcx, C: QueryCache> { + name: &'static str, + cache_on_disk: fn(tcx: TyCtxt<'tcx>, key: &C::Key) -> bool, + execute_query: fn(tcx: TyCtxt<'tcx>, k: C::Key) -> C::Value, + compute: fn(tcx: TyCtxt<'tcx>, key: C::Key) -> C::Value, + try_load_from_disk: + fn(qcx: QueryCtxt<'tcx>, idx: &C::Key) -> TryLoadFromDisk, C::Value>, + query_state: usize, + query_cache: usize, + eval_always: bool, + dep_kind: rustc_middle::dep_graph::DepKind, + handle_cycle_error: HandleCycleError, + hash_result: Option, &C::Value) -> Fingerprint>, +} + +struct DynamicConfig<'tcx, C: QueryCache, Anon, DepthLimit, Feedable> { + dynamic: &'tcx DynamicQuery<'tcx, C>, + data: PhantomData<(Anon, DepthLimit, Feedable)>, +} + +impl<'tcx, C: QueryCache, Anon, DepthLimit, Feedable> Copy + for DynamicConfig<'tcx, C, Anon, DepthLimit, Feedable> +{ +} +impl<'tcx, C: QueryCache, Anon, DepthLimit, Feedable> Clone + for DynamicConfig<'tcx, C, Anon, DepthLimit, Feedable> +{ + fn clone(&self) -> Self { + DynamicConfig { dynamic: self.dynamic, data: PhantomData } + } +} + +impl<'tcx, C: QueryCache, Anon, DepthLimit, Feedable> fmt::Debug + for DynamicConfig<'tcx, C, Anon, DepthLimit, Feedable> +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "DynamicConfig<{}>", self.dynamic.name) + } +} + +impl<'tcx, C: QueryCache, Anon: Bool, DepthLimit: Bool, Feedable: Bool> QueryConfig> + for DynamicConfig<'tcx, C, Anon, DepthLimit, Feedable> +where + for<'a> C::Key: HashStable>, +{ + type Key = C::Key; + type Value = C::Value; + type Cache = C; + + #[inline(always)] + fn name(self) -> &'static str { + self.dynamic.name + } + + #[inline(always)] + fn cache_on_disk(self, tcx: TyCtxt<'tcx>, key: &Self::Key) -> bool { + (self.dynamic.cache_on_disk)(tcx, key) + } + + #[inline(always)] + fn query_state<'a>( + self, + tcx: QueryCtxt<'tcx>, + ) -> &'a QueryState + where + QueryCtxt<'tcx>: 'a, + { + unsafe { + &*((&tcx.queries.query_states as *const QueryStates<'tcx> as *const u8) + .offset(self.dynamic.query_state as isize) as *const _) + } + } -impl<'tcx> Queries<'tcx> { - // Force codegen in the dyn-trait transformation in this crate. - pub fn as_dyn(&'tcx self) -> &'tcx dyn QueryEngine<'tcx> { - self + #[inline(always)] + fn query_cache<'a>(self, tcx: QueryCtxt<'tcx>) -> &'a Self::Cache + where + 'tcx: 'a, + { + unsafe { + &*((&tcx.query_system.caches as *const QueryCaches<'tcx> as *const u8) + .offset(self.dynamic.query_cache as isize) as *const _) + } + } + + #[inline(always)] + fn execute_query(self, tcx: TyCtxt<'tcx>, key: Self::Key) -> Self::Value { + (self.dynamic.execute_query)(tcx, key) + } + + #[inline(always)] + fn compute(self, tcx: TyCtxt<'tcx>, key: Self::Key) -> Self::Value { + (self.dynamic.compute)(tcx, key) + } + + #[inline(always)] + fn try_load_from_disk( + self, + qcx: QueryCtxt<'tcx>, + key: &Self::Key, + ) -> rustc_query_system::query::TryLoadFromDisk, Self::Value> { + (self.dynamic.try_load_from_disk)(qcx, key) + } + + #[inline(always)] + fn anon(self) -> bool { + Anon::value() + } + + #[inline(always)] + fn eval_always(self) -> bool { + self.dynamic.eval_always + } + + #[inline(always)] + fn depth_limit(self) -> bool { + DepthLimit::value() + } + + #[inline(always)] + fn feedable(self) -> bool { + Feedable::value() + } + + #[inline(always)] + fn dep_kind(self) -> rustc_middle::dep_graph::DepKind { + self.dynamic.dep_kind + } + + #[inline(always)] + fn handle_cycle_error(self) -> rustc_query_system::HandleCycleError { + self.dynamic.handle_cycle_error + } + + #[inline(always)] + fn hash_result(self) -> rustc_query_system::query::HashResult { + self.dynamic.hash_result } } + +trait QueryToConfig<'tcx>: 'tcx +where + for<'a> ::Key: HashStable>, +{ + type Cache: QueryCache; + type Key: DepNodeParams>; + + type Anon: Bool; + type DepthLimit: Bool; + type Feedable: Bool; + + fn config( + qcx: QueryCtxt<'tcx>, + ) -> DynamicConfig<'tcx, Self::Cache, Self::Anon, Self::DepthLimit, Self::Feedable>; +} + +rustc_query_append! { define_queries! } diff --git a/compiler/rustc_query_impl/src/on_disk_cache.rs b/compiler/rustc_query_impl/src/on_disk_cache.rs index 70c481fb0ee2e..0e7b268f25d72 100644 --- a/compiler/rustc_query_impl/src/on_disk_cache.rs +++ b/compiler/rustc_query_impl/src/on_disk_cache.rs @@ -13,7 +13,7 @@ use rustc_middle::mir::{self, interpret}; use rustc_middle::ty::codec::{RefDecodable, TyDecoder, TyEncoder}; use rustc_middle::ty::{self, Ty, TyCtxt}; use rustc_query_system::dep_graph::DepContext; -use rustc_query_system::query::{QueryCache, QueryContext, QuerySideEffects}; +use rustc_query_system::query::{QueryCache, QuerySideEffects}; use rustc_serialize::{ opaque::{FileEncodeResult, FileEncoder, IntEncodedWithFixedSize, MemDecoder}, Decodable, Decoder, Encodable, Encoder, @@ -1056,24 +1056,24 @@ impl<'a, 'tcx> Encodable> for [u8] { } } -pub fn encode_query_results<'a, 'tcx, CTX, Q>( - tcx: CTX, +pub fn encode_query_results<'a, 'tcx, Q>( + query: Q, + qcx: QueryCtxt<'tcx>, encoder: &mut CacheEncoder<'a, 'tcx>, query_result_index: &mut EncodedDepNodeIndex, ) where - CTX: QueryContext + 'tcx, - Q: super::QueryConfig, + Q: super::QueryConfig>, Q::Value: Encodable>, { - let _timer = tcx - .dep_context() + let _timer = qcx + .tcx .profiler() - .verbose_generic_activity_with_arg("encode_query_results_for", std::any::type_name::()); + .verbose_generic_activity_with_arg("encode_query_results_for", query.name()); - assert!(Q::query_state(tcx).all_inactive()); - let cache = Q::query_cache(tcx); + assert!(query.query_state(qcx).all_inactive()); + let cache = query.query_cache(qcx); cache.iter(&mut |key, value, dep_node| { - if Q::cache_on_disk(*tcx.dep_context(), &key) { + if query.cache_on_disk(qcx.tcx, &key) { let dep_node = SerializedDepNodeIndex::new(dep_node.index()); // Record position of the cache entry. diff --git a/compiler/rustc_query_impl/src/plumbing.rs b/compiler/rustc_query_impl/src/plumbing.rs index a8592bd70862c..0128d56975b5b 100644 --- a/compiler/rustc_query_impl/src/plumbing.rs +++ b/compiler/rustc_query_impl/src/plumbing.rs @@ -4,7 +4,8 @@ use crate::on_disk_cache::{CacheDecoder, CacheEncoder, EncodedDepNodeIndex}; use crate::profiling_support::QueryKeyStringCache; -use crate::{on_disk_cache, Queries}; +use crate::QueryStates; +use crate::{on_disk_cache, QueryToConfig}; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_data_structures::sync::{AtomicU64, Lock}; use rustc_errors::{Diagnostic, Handler}; @@ -12,14 +13,16 @@ use rustc_middle::dep_graph::{ self, DepKind, DepKindStruct, DepNode, DepNodeIndex, SerializedDepNodeIndex, }; use rustc_middle::query::Key; +use rustc_middle::ty::query::QueryEngine; use rustc_middle::ty::tls::{self, ImplicitCtxt}; use rustc_middle::ty::{self, TyCtxt}; use rustc_query_system::dep_graph::{DepNodeParams, HasDepContext}; use rustc_query_system::ich::StableHashingContext; use rustc_query_system::query::{ - force_query, QueryConfig, QueryContext, QueryJobId, QueryMap, QuerySideEffects, QueryStackFrame, + force_query, QueryCache, QueryConfig, QueryContext, QueryJobId, QueryMap, QuerySideEffects, + QueryStackFrame, }; -use rustc_query_system::{LayoutOfDepth, QueryOverflow, Value}; +use rustc_query_system::{LayoutOfDepth, QueryOverflow}; use rustc_serialize::Decodable; use rustc_session::Limit; use rustc_span::def_id::LOCAL_CRATE; @@ -238,51 +241,95 @@ macro_rules! is_eval_always { }; } -macro_rules! depth_limit { - ([]) => {{ - false - }}; - ([(depth_limit) $($rest:tt)*]) => {{ - true - }}; +macro_rules! depth_limit_type { + ([]) => { + False + }; + ([(depth_limit) $($rest:tt)*]) => { + True + }; ([$other:tt $($modifiers:tt)*]) => { - depth_limit!([$($modifiers)*]) + depth_limit_type!([$($modifiers)*]) }; } -macro_rules! feedable { - ([]) => {{ - false - }}; - ([(feedable) $($rest:tt)*]) => {{ - true - }}; +macro_rules! feedable_type { + ([]) => { + False + }; + ([(feedable) $($rest:tt)*]) => { + True + }; ([$other:tt $($modifiers:tt)*]) => { - feedable!([$($modifiers)*]) + feedable_type!([$($modifiers)*]) + }; +} + +macro_rules! is_anon_type { + ([]) => { + False + }; + ([(anon) $($rest:tt)*]) => { + True + }; + ([$other:tt $($modifiers:tt)*]) => { + is_anon_type!([$($modifiers)*]) + }; +} + +macro_rules! if_arena { + ([] $arena:tt $no_arena:tt) => { + $no_arena + }; + ([(arena_cache) $($rest:tt)*] $arena:tt $no_arena:tt) => { + $arena + }; + ([$other:tt $($modifiers:tt)*]$($args:tt)*) => { + if_arena!([$($modifiers)*]$($args)*) + }; +} + +macro_rules! if_extern_provider { + ([] $yes:tt $no:tt) => { + $no + }; + ([(separate_provide_extern) $($rest:tt)*] $yes:tt $no:tt) => { + $yes + }; + ([$other:tt $($modifiers:tt)*]$($args:tt)*) => { + if_extern_provider!([$($modifiers)*]$($args)*) + }; +} + +macro_rules! if_compute_is_local_provider { + ($modifiers:tt $yes:tt $no:tt) => { + if_extern_provider!($modifiers $no ( + if_arena!($modifiers $no $yes) + )) }; } macro_rules! hash_result { - ([]) => {{ - Some(dep_graph::hash_result) + ([][$V:ty]) => {{ + Some(|hcx, result| dep_graph::hash_result(hcx, result)) }}; - ([(no_hash) $($rest:tt)*]) => {{ + ([(no_hash) $($rest:tt)*][$V:ty]) => {{ None }}; - ([$other:tt $($modifiers:tt)*]) => { - hash_result!([$($modifiers)*]) + ([$other:tt $($modifiers:tt)*][$($args:tt)*]) => { + hash_result!([$($modifiers)*][$($args)*]) }; } macro_rules! get_provider { ([][$tcx:expr, $name:ident, $key:expr]) => {{ - $tcx.queries.local_providers.$name + $tcx.query_system.local_providers.$name }}; ([(separate_provide_extern) $($rest:tt)*][$tcx:expr, $name:ident, $key:expr]) => {{ if $key.query_crate_is_local() { - $tcx.queries.local_providers.$name + $tcx.query_system.local_providers.$name } else { - $tcx.queries.extern_providers.$name + $tcx.query_system.extern_providers.$name } }}; ([$other:tt $($modifiers:tt)*][$($args:tt)*]) => { @@ -350,18 +397,17 @@ pub(crate) fn create_query_frame< QueryStackFrame::new(description, span, def_id, def_kind, kind, ty_adt_id, hash) } -fn try_load_from_on_disk_cache<'tcx, Q>(tcx: TyCtxt<'tcx>, dep_node: DepNode) +fn try_load_from_on_disk_cache<'tcx, Q>(query: Q, tcx: TyCtxt<'tcx>, dep_node: DepNode) where Q: QueryConfig>, - Q::Key: DepNodeParams>, { debug_assert!(tcx.dep_graph.is_green(&dep_node)); let key = Q::Key::recover(tcx, &dep_node).unwrap_or_else(|| { panic!("Failed to recover key for {:?} with hash {}", dep_node, dep_node.hash) }); - if Q::cache_on_disk(tcx, &key) { - let _ = Q::execute_query(tcx, key); + if query.cache_on_disk(tcx, &key) { + let _ = query.execute_query(tcx, key); } } @@ -375,11 +421,9 @@ where tcx.on_disk_cache().as_ref()?.try_load_query_result(*tcx, id) } -fn force_from_dep_node<'tcx, Q>(tcx: TyCtxt<'tcx>, dep_node: DepNode) -> bool +fn force_from_dep_node<'tcx, Q>(query: Q, tcx: TyCtxt<'tcx>, dep_node: DepNode) -> bool where Q: QueryConfig>, - Q::Key: DepNodeParams>, - Q::Value: Value, DepKind>, { // We must avoid ever having to call `force_from_dep_node()` for a // `DepNode::codegen_unit`: @@ -403,7 +447,7 @@ where #[cfg(debug_assertions)] let _guard = tracing::span!(tracing::Level::TRACE, stringify!($name), ?key).entered(); let tcx = QueryCtxt::from_tcx(tcx); - force_query::(tcx, key, dep_node); + force_query(query, tcx, key, dep_node); true } else { false @@ -412,8 +456,8 @@ where pub(crate) fn query_callback<'tcx, Q>(is_anon: bool, is_eval_always: bool) -> DepKindStruct<'tcx> where - Q: QueryConfig>, - Q::Key: DepNodeParams>, + Q: QueryToConfig<'tcx>, + for<'a> ::Key: HashStable>, { let fingerprint_style = Q::Key::fingerprint_style(); @@ -431,8 +475,12 @@ where is_anon, is_eval_always, fingerprint_style, - force_from_dep_node: Some(force_from_dep_node::), - try_load_from_on_disk_cache: Some(try_load_from_on_disk_cache::), + force_from_dep_node: Some(|tcx, dep_node| { + force_from_dep_node(Q::config(QueryCtxt::from_tcx(tcx)), tcx, dep_node) + }), + try_load_from_on_disk_cache: Some(|tcx, dep_node| { + try_load_from_on_disk_cache(Q::config(QueryCtxt::from_tcx(tcx)), tcx, dep_node) + }), } } @@ -462,78 +510,85 @@ macro_rules! define_queries { mod queries { use std::marker::PhantomData; - $(pub struct $name<'tcx> { - data: PhantomData<&'tcx ()> - })* + $( + #[derive(Copy, Clone, Default)] + pub struct $name<'tcx> { + data: PhantomData<&'tcx ()> + } + )* } - $(impl<'tcx> QueryConfig> for queries::$name<'tcx> { - type Key = query_keys::$name<'tcx>; - type Value = query_values::$name<'tcx>; - const NAME: &'static str = stringify!($name); + #[allow(nonstandard_style)] + mod dynamic_query { + use super::*; - #[inline] - fn cache_on_disk(tcx: TyCtxt<'tcx>, key: &Self::Key) -> bool { - ::rustc_middle::query::cached::$name(tcx, key) - } + $( + pub(super) fn $name<'tcx>( + _local_provider: fn( + tcx: TyCtxt<'tcx>, key: query_keys::$name<'tcx> + ) -> query_provided::$name<'tcx> + ) -> DynamicQuery<'tcx, query_storage::$name<'tcx>> { + DynamicQuery { + name: stringify!($name), + cache_on_disk: |tcx, key| ::rustc_middle::query::cached::$name(tcx, key), + execute_query: |tcx, key| tcx.$name(key), + compute: if_compute_is_local_provider!( + [$($modifiers)*] + _local_provider + (|tcx, key: query_keys::$name<'tcx>| query_provided_to_value::$name( + tcx, + get_provider!([$($modifiers)*][tcx, $name, key])(tcx, key) + )) + ), + try_load_from_disk: |qcx, key| { + fn try_load_from_disk<'tcx>( + _qcx: QueryCtxt<'tcx>, + _key: &query_keys::$name<'tcx> + ) -> TryLoadFromDisk, query_values::$name<'tcx>> { + should_ever_cache_on_disk!([$($modifiers)*] { + if ::rustc_middle::query::cached::$name(_qcx.tcx, _key) { + Some(|qcx, dep_node| { + let value = $crate::plumbing::try_load_from_disk::>( + qcx, + dep_node + ); + value.map(|value| query_provided_to_value::$name(qcx.tcx, value)) + }) + } else { + None + } + } { + None + }) + } + try_load_from_disk(qcx, key) + }, + query_state: offset_of!(QueryStates<'static>, $name), + query_cache: offset_of!(QueryCaches<'static>, $name), + eval_always: is_eval_always!([$($modifiers)*]), + dep_kind: dep_graph::DepKind::$name, + handle_cycle_error: handle_cycle_error!([$($modifiers)*]), + hash_result: hash_result!([$($modifiers)*][query_values::$name]), + } + } + )* + } + $(impl<'tcx> QueryToConfig<'tcx> for queries::$name<'tcx> { type Cache = query_storage::$name<'tcx>; + type Key = query_keys::$name<'tcx>; - #[inline(always)] - fn query_state<'a>(tcx: QueryCtxt<'tcx>) -> &'a QueryState - where QueryCtxt<'tcx>: 'a - { - &tcx.queries.$name - } + type Anon = is_anon_type!([$($modifiers)*]); + type DepthLimit = depth_limit_type!([$($modifiers)*]); + type Feedable = feedable_type!([$($modifiers)*]); #[inline(always)] - fn query_cache<'a>(tcx: QueryCtxt<'tcx>) -> &'a Self::Cache - where 'tcx:'a - { - &tcx.query_system.caches.$name - } - - fn execute_query(tcx: TyCtxt<'tcx>, key: Self::Key) -> Self::Value { - tcx.$name(key) - } - - #[inline] - #[allow(unused_variables)] - fn compute(qcx: QueryCtxt<'tcx>, key: Self::Key) -> Self::Value { - query_provided_to_value::$name( - qcx.tcx, - get_provider!([$($modifiers)*][qcx, $name, key])(qcx.tcx, key) - ) - } - - #[inline] - fn try_load_from_disk(_qcx: QueryCtxt<'tcx>, _key: &Self::Key) -> rustc_query_system::query::TryLoadFromDisk, Self> { - should_ever_cache_on_disk!([$($modifiers)*] { - if Self::cache_on_disk(_qcx.tcx, _key) { - Some(|qcx: QueryCtxt<'tcx>, dep_node| { - let value = $crate::plumbing::try_load_from_disk::>( - qcx, - dep_node - ); - value.map(|value| query_provided_to_value::$name(qcx.tcx, value)) - }) - } else { - None - } - } { - None - }) + fn config(qcx: QueryCtxt<'tcx>) -> DynamicConfig<'tcx, Self::Cache, Self::Anon, Self::DepthLimit, Self::Feedable> { + DynamicConfig { + dynamic: &qcx.queries.dynamic_queries.$name, + data: PhantomData, + } } - - const ANON: bool = is_anon!([$($modifiers)*]); - const EVAL_ALWAYS: bool = is_eval_always!([$($modifiers)*]); - const DEPTH_LIMIT: bool = depth_limit!([$($modifiers)*]); - const FEEDABLE: bool = feedable!([$($modifiers)*]); - - const DEP_KIND: rustc_middle::dep_graph::DepKind = dep_graph::DepKind::$name; - const HANDLE_CYCLE_ERROR: rustc_query_system::HandleCycleError = handle_cycle_error!([$($modifiers)*]); - - const HASH_RESULT: rustc_query_system::query::HashResult, Self> = hash_result!([$($modifiers)*]); })* #[allow(nonstandard_style)] @@ -607,6 +662,7 @@ macro_rules! define_queries { use $crate::profiling_support::QueryKeyStringCache; use rustc_query_system::query::QueryMap; use rustc_middle::dep_graph::DepKind; + use crate::QueryToConfig; pub(super) const fn dummy_query_struct<'tcx>() -> QueryStruct<'tcx> { fn noop_try_collect_active_jobs(_: QueryCtxt<'_>, _: &mut QueryMap) -> Option<()> { @@ -635,7 +691,7 @@ macro_rules! define_queries { let name = stringify!($name); $crate::plumbing::create_query_frame(tcx, rustc_middle::query::descs::$name, key, kind, name) }; - tcx.queries.$name.try_collect_active_jobs( + tcx.queries.query_states.$name.try_collect_active_jobs( tcx, make_query, qmap, @@ -649,8 +705,13 @@ macro_rules! define_queries { string_cache, ) }, - encode_query_results: expand_if_cached!([$($modifiers)*], |tcx, encoder, query_result_index| - $crate::on_disk_cache::encode_query_results::<_, super::queries::$name<'_>>(tcx, encoder, query_result_index) + encode_query_results: expand_if_cached!([$($modifiers)*], |qcx, encoder, query_result_index| + $crate::on_disk_cache::encode_query_results( + super::queries::$name::config(qcx), + qcx, + encoder, + query_result_index, + ) ), }})* } @@ -661,62 +722,75 @@ macro_rules! define_queries { } } -use crate::{ExternProviders, OnDiskCache, Providers}; +use crate::{OnDiskCache, Providers}; + +pub struct Queries<'tcx> { + pub(crate) query_structs: Vec>, + pub on_disk_cache: Option>, + jobs: AtomicU64, + pub(crate) dynamic_queries: crate::DynamicQueries<'tcx>, + pub(crate) query_states: crate::QueryStates<'tcx>, +} impl<'tcx> Queries<'tcx> { - pub fn new( - local_providers: Providers, - extern_providers: ExternProviders, - on_disk_cache: Option>, - ) -> Self { + pub fn new(local_providers: &Providers, on_disk_cache: Option>) -> Self { use crate::query_structs; Queries { - local_providers: Box::new(local_providers), - extern_providers: Box::new(extern_providers), + dynamic_queries: crate::DynamicQueries::new(&local_providers), query_structs: make_dep_kind_array!(query_structs).to_vec(), on_disk_cache, jobs: AtomicU64::new(1), - ..Queries::default() + query_states: QueryStates::default(), } } + + // Force codegen in the dyn-trait transformation in this crate. + pub fn as_dyn(&'tcx self) -> &'tcx dyn QueryEngine<'tcx> { + self + } + + pub(crate) fn try_collect_active_jobs( + &'tcx self, + tcx: TyCtxt<'tcx>, + ) -> Option> { + let tcx = QueryCtxt { tcx, queries: self }; + let mut jobs = QueryMap::default(); + + for query in &self.query_structs { + (query.try_collect_active_jobs)(tcx, &mut jobs); + } + + Some(jobs) + } } macro_rules! define_queries_struct { ( input: ($(([$($modifiers:tt)*] [$($attr:tt)*] [$name:ident]))*)) => { - #[derive(Default)] - pub struct Queries<'tcx> { - local_providers: Box, - extern_providers: Box, - query_structs: Vec<$crate::plumbing::QueryStruct<'tcx>>, - pub on_disk_cache: Option>, - jobs: AtomicU64, - + #[allow(dead_code)] + struct DynamicQueries<'tcx> { $( - $(#[$attr])* - $name: QueryState< - as QueryConfig>>::Key, - rustc_middle::dep_graph::DepKind, - >, + $name: DynamicQuery<'tcx, query_storage::$name<'tcx>>, )* } - impl<'tcx> Queries<'tcx> { - pub(crate) fn try_collect_active_jobs( - &'tcx self, - tcx: TyCtxt<'tcx>, - ) -> Option> { - let tcx = QueryCtxt { tcx, queries: self }; - let mut jobs = QueryMap::default(); - - for query in &self.query_structs { - (query.try_collect_active_jobs)(tcx, &mut jobs); + impl DynamicQueries<'_> { + fn new(local_providers: &Providers) -> Self { + Self { + $( + $name: dynamic_query::$name(local_providers.$name), + )* } - - Some(jobs) } } + #[derive(Default)] + struct QueryStates<'tcx> { + $( + $name: QueryState, rustc_middle::dep_graph::DepKind>, + )* + } + impl<'tcx> QueryEngine<'tcx> for Queries<'tcx> { fn as_any(&'tcx self) -> &'tcx dyn std::any::Any { let this = unsafe { std::mem::transmute::<&Queries<'_>, &Queries<'_>>(self) }; @@ -735,11 +809,17 @@ macro_rules! define_queries_struct { &'tcx self, tcx: TyCtxt<'tcx>, span: Span, - key: as QueryConfig>>::Key, + key: query_keys::$name<'tcx>, mode: QueryMode, ) -> Option> { let qcx = QueryCtxt { tcx, queries: self }; - get_query::, _, rustc_middle::dep_graph::DepKind>(qcx, span, key, mode) + get_query( + queries::$name::config(qcx), + qcx, + span, + key, + mode + ) })* } }; diff --git a/compiler/rustc_query_system/src/dep_graph/dep_node.rs b/compiler/rustc_query_system/src/dep_graph/dep_node.rs index 9e1ca6ab515d8..ca4aa249ee997 100644 --- a/compiler/rustc_query_system/src/dep_graph/dep_node.rs +++ b/compiler/rustc_query_system/src/dep_graph/dep_node.rs @@ -108,7 +108,9 @@ impl fmt::Debug for DepNode { } } -pub trait DepNodeParams: fmt::Debug + Sized { +pub trait DepNodeParams: + fmt::Debug + Sized + for<'a> HashStable> +{ fn fingerprint_style() -> FingerprintStyle; /// This method turns the parameters of a DepNodeConstructor into an opaque diff --git a/compiler/rustc_query_system/src/dep_graph/mod.rs b/compiler/rustc_query_system/src/dep_graph/mod.rs index 6969f2dbef8b8..ba83b77563165 100644 --- a/compiler/rustc_query_system/src/dep_graph/mod.rs +++ b/compiler/rustc_query_system/src/dep_graph/mod.rs @@ -88,6 +88,15 @@ impl HasDepContext for T { } } +impl HasDepContext for (T, Q) { + type DepKind = T::DepKind; + type DepContext = T::DepContext; + + fn dep_context(&self) -> &Self::DepContext { + self.0.dep_context() + } +} + /// Describes the contents of the fingerprint generated by a given query. #[derive(Debug, PartialEq, Eq, Copy, Clone)] pub enum FingerprintStyle { diff --git a/compiler/rustc_query_system/src/query/caches.rs b/compiler/rustc_query_system/src/query/caches.rs index 4b3cd16c29f25..5f554a54deac1 100644 --- a/compiler/rustc_query_system/src/query/caches.rs +++ b/compiler/rustc_query_system/src/query/caches.rs @@ -16,12 +16,9 @@ pub trait CacheSelector<'tcx, V> { V: Copy; } -pub trait QueryStorage { - type Value: Copy; -} - -pub trait QueryCache: QueryStorage + Sized { +pub trait QueryCache: Sized { type Key: Hash + Eq + Copy + Debug; + type Value: Copy + Debug; /// Checks if the query is already computed and in the cache. /// It returns the shard index and a lock guard to the shard, @@ -55,16 +52,13 @@ impl Default for DefaultCache { } } -impl QueryStorage for DefaultCache { - type Value = V; -} - impl QueryCache for DefaultCache where K: Eq + Hash + Copy + Debug, V: Copy + Debug, { type Key = K; + type Value = V; #[inline(always)] fn lookup(&self, key: &K) -> Option<(V, DepNodeIndex)> { @@ -127,15 +121,12 @@ impl Default for SingleCache { } } -impl QueryStorage for SingleCache { - type Value = V; -} - impl QueryCache for SingleCache where V: Copy + Debug, { type Key = (); + type Value = V; #[inline(always)] fn lookup(&self, _key: &()) -> Option<(V, DepNodeIndex)> { @@ -173,16 +164,13 @@ impl Default for VecCache { } } -impl QueryStorage for VecCache { - type Value = V; -} - impl QueryCache for VecCache where K: Eq + Idx + Copy + Debug, V: Copy + Debug, { type Key = K; + type Value = V; #[inline(always)] fn lookup(&self, key: &K) -> Option<(V, DepNodeIndex)> { diff --git a/compiler/rustc_query_system/src/query/config.rs b/compiler/rustc_query_system/src/query/config.rs index d563738734657..888512d75991c 100644 --- a/compiler/rustc_query_system/src/query/config.rs +++ b/compiler/rustc_query_system/src/query/config.rs @@ -10,14 +10,12 @@ use rustc_data_structures::fingerprint::Fingerprint; use std::fmt::Debug; use std::hash::Hash; -pub type HashResult = - Option, &>::Value) -> Fingerprint>; +pub type HashResult = Option, &V) -> Fingerprint>; -pub type TryLoadFromDisk = - Option Option<>::Value>>; +pub type TryLoadFromDisk = Option Option>; -pub trait QueryConfig { - const NAME: &'static str; +pub trait QueryConfig: Copy { + fn name(self) -> &'static str; // `Key` and `Value` are `Copy` instead of `Clone` to ensure copying them stays cheap, // but it isn't necessary. @@ -27,36 +25,35 @@ pub trait QueryConfig { type Cache: QueryCache; // Don't use this method to access query results, instead use the methods on TyCtxt - fn query_state<'a>(tcx: Qcx) -> &'a QueryState + fn query_state<'a>(self, tcx: Qcx) -> &'a QueryState where Qcx: 'a; // Don't use this method to access query results, instead use the methods on TyCtxt - fn query_cache<'a>(tcx: Qcx) -> &'a Self::Cache + fn query_cache<'a>(self, tcx: Qcx) -> &'a Self::Cache where Qcx: 'a; - fn cache_on_disk(tcx: Qcx::DepContext, key: &Self::Key) -> bool; + fn cache_on_disk(self, tcx: Qcx::DepContext, key: &Self::Key) -> bool; // Don't use this method to compute query results, instead use the methods on TyCtxt - fn execute_query(tcx: Qcx::DepContext, k: Self::Key) -> Self::Value; + fn execute_query(self, tcx: Qcx::DepContext, k: Self::Key) -> Self::Value; - fn compute(tcx: Qcx, key: Self::Key) -> Self::Value; + fn compute(self, tcx: Qcx::DepContext, key: Self::Key) -> Self::Value; - fn try_load_from_disk(qcx: Qcx, idx: &Self::Key) -> TryLoadFromDisk; + fn try_load_from_disk(self, qcx: Qcx, idx: &Self::Key) -> TryLoadFromDisk; - const ANON: bool; - const EVAL_ALWAYS: bool; - const DEPTH_LIMIT: bool; - const FEEDABLE: bool; + fn anon(self) -> bool; + fn eval_always(self) -> bool; + fn depth_limit(self) -> bool; + fn feedable(self) -> bool; - const DEP_KIND: Qcx::DepKind; - const HANDLE_CYCLE_ERROR: HandleCycleError; - - const HASH_RESULT: HashResult; + fn dep_kind(self) -> Qcx::DepKind; + fn handle_cycle_error(self) -> HandleCycleError; + fn hash_result(self) -> HashResult; // Just here for convernience and checking that the key matches the kind, don't override this. - fn construct_dep_node(tcx: Qcx::DepContext, key: &Self::Key) -> DepNode { - DepNode::construct(tcx, Self::DEP_KIND, key) + fn construct_dep_node(self, tcx: Qcx::DepContext, key: &Self::Key) -> DepNode { + DepNode::construct(tcx, self.dep_kind(), key) } } diff --git a/compiler/rustc_query_system/src/query/mod.rs b/compiler/rustc_query_system/src/query/mod.rs index 383c63cd2f8a7..312b0e1688dc9 100644 --- a/compiler/rustc_query_system/src/query/mod.rs +++ b/compiler/rustc_query_system/src/query/mod.rs @@ -8,8 +8,7 @@ pub use self::job::{print_query_stack, QueryInfo, QueryJob, QueryJobId, QueryJob mod caches; pub use self::caches::{ - CacheSelector, DefaultCacheSelector, QueryCache, QueryStorage, SingleCacheSelector, - VecCacheSelector, + CacheSelector, DefaultCacheSelector, QueryCache, SingleCacheSelector, VecCacheSelector, }; mod config; diff --git a/compiler/rustc_query_system/src/query/plumbing.rs b/compiler/rustc_query_system/src/query/plumbing.rs index 586bd38fbb80b..ba96916d9b5c1 100644 --- a/compiler/rustc_query_system/src/query/plumbing.rs +++ b/compiler/rustc_query_system/src/query/plumbing.rs @@ -2,6 +2,7 @@ //! generate the actual methods on tcx which find and execute the provider, //! manage the caches, and so forth. +use crate::dep_graph::HasDepContext; use crate::dep_graph::{DepContext, DepKind, DepNode, DepNodeIndex, DepNodeParams}; use crate::ich::StableHashingContext; use crate::query::caches::QueryCache; @@ -127,7 +128,7 @@ fn mk_cycle( handler: HandleCycleError, ) -> R where - Qcx: QueryContext + crate::query::HasDepContext, + Qcx: QueryContext + HasDepContext, R: std::fmt::Debug + Value, { let error = report_cycle(qcx.dep_context().sess(), &cycle_error); @@ -182,7 +183,7 @@ where key: K, ) -> TryGetJob<'b, K, D> where - Qcx: QueryContext + crate::query::HasDepContext, + Qcx: QueryContext + HasDepContext, { #[cfg(parallel_compiler)] let mut state_lock = state.active.get_shard_by_value(&key).lock(); @@ -349,6 +350,7 @@ where #[inline(never)] fn try_execute_query( + query: Q, qcx: Qcx, span: Span, key: Q::Key, @@ -358,12 +360,12 @@ where Q: QueryConfig, Qcx: QueryContext, { - let state = Q::query_state(qcx); + let state = query.query_state(qcx); match JobOwner::<'_, Q::Key, Qcx::DepKind>::try_start(&qcx, state, span, key) { TryGetJob::NotYetStarted(job) => { - let (result, dep_node_index) = execute_job::(qcx, key, dep_node, job.id); - let cache = Q::query_cache(qcx); - if Q::FEEDABLE { + let (result, dep_node_index) = execute_job(query, qcx, key.clone(), dep_node, job.id); + let cache = query.query_cache(qcx); + if query.feedable() { // We should not compute queries that also got a value via feeding. // This can't happen, as query feeding adds the very dependencies to the fed query // as its feeding query had. So if the fed query is red, so is its feeder, which will @@ -378,12 +380,12 @@ where (result, Some(dep_node_index)) } TryGetJob::Cycle(error) => { - let result = mk_cycle(qcx, error, Q::HANDLE_CYCLE_ERROR); + let result = mk_cycle(qcx, error, query.handle_cycle_error()); (result, None) } #[cfg(parallel_compiler)] TryGetJob::JobCompleted(query_blocked_prof_timer) => { - let Some((v, index)) = Q::query_cache(qcx).lookup(&key) else { + let Some((v, index)) = query.query_cache(qcx).lookup(&key) else { panic!("value must be in cache after waiting") }; @@ -397,6 +399,7 @@ where #[inline(always)] fn execute_job( + query: Q, qcx: Qcx, key: Q::Key, mut dep_node_opt: Option>, @@ -417,14 +420,16 @@ where } let prof_timer = qcx.dep_context().profiler().query_provider(); - let result = qcx.start_query(job_id, Q::DEPTH_LIMIT, None, || Q::compute(qcx, key)); + let result = qcx.start_query(job_id, query.depth_limit(), None, || { + query.compute(*qcx.dep_context(), key) + }); let dep_node_index = dep_graph.next_virtual_depnode_index(); prof_timer.finish_with_query_invocation_id(dep_node_index.into()); // Similarly, fingerprint the result to assert that // it doesn't have anything not considered hashable. if cfg!(debug_assertions) - && let Some(hash_result) = Q::HASH_RESULT + && let Some(hash_result) = query.hash_result() { qcx.dep_context().with_stable_hashing_context(|mut hcx| { hash_result(&mut hcx, &result); @@ -434,15 +439,15 @@ where return (result, dep_node_index); } - if !Q::ANON && !Q::EVAL_ALWAYS { + if !query.anon() && !query.eval_always() { // `to_dep_node` is expensive for some `DepKind`s. let dep_node = - dep_node_opt.get_or_insert_with(|| Q::construct_dep_node(*qcx.dep_context(), &key)); + dep_node_opt.get_or_insert_with(|| query.construct_dep_node(*qcx.dep_context(), &key)); // The diagnostics for this query will be promoted to the current session during // `try_mark_green()`, so we can ignore them here. if let Some(ret) = qcx.start_query(job_id, false, None, || { - try_load_from_disk_and_cache_in_memory::(qcx, &key, &dep_node) + try_load_from_disk_and_cache_in_memory(query, qcx, &key, &dep_node) }) { return ret; } @@ -452,17 +457,24 @@ where let diagnostics = Lock::new(ThinVec::new()); let (result, dep_node_index) = - qcx.start_query(job_id, Q::DEPTH_LIMIT, Some(&diagnostics), || { - if Q::ANON { - return dep_graph - .with_anon_task(*qcx.dep_context(), Q::DEP_KIND, || Q::compute(qcx, key)); + qcx.start_query(job_id, query.depth_limit(), Some(&diagnostics), || { + if query.anon() { + return dep_graph.with_anon_task(*qcx.dep_context(), query.dep_kind(), || { + query.compute(*qcx.dep_context(), key) + }); } // `to_dep_node` is expensive for some `DepKind`s. let dep_node = - dep_node_opt.unwrap_or_else(|| Q::construct_dep_node(*qcx.dep_context(), &key)); - - dep_graph.with_task(dep_node, qcx, key, Q::compute, Q::HASH_RESULT) + dep_node_opt.unwrap_or_else(|| query.construct_dep_node(*qcx.dep_context(), &key)); + + dep_graph.with_task( + dep_node, + (*qcx.dep_context(), query), + key, + |(tcx, query), key| query.compute(tcx, key), + query.hash_result(), + ) }); prof_timer.finish_with_query_invocation_id(dep_node_index.into()); @@ -471,7 +483,7 @@ where let side_effects = QuerySideEffects { diagnostics }; if std::intrinsics::unlikely(!side_effects.is_empty()) { - if Q::ANON { + if query.anon() { qcx.store_side_effects_for_anon_node(dep_node_index, side_effects); } else { qcx.store_side_effects(dep_node_index, side_effects); @@ -483,6 +495,7 @@ where #[inline(always)] fn try_load_from_disk_and_cache_in_memory( + query: Q, qcx: Qcx, key: &Q::Key, dep_node: &DepNode, @@ -501,7 +514,7 @@ where // First we try to load the result from the on-disk cache. // Some things are never cached on disk. - if let Some(try_load_from_disk) = Q::try_load_from_disk(qcx, &key) { + if let Some(try_load_from_disk) = query.try_load_from_disk(qcx, &key) { let prof_timer = qcx.dep_context().profiler().incr_cache_loading(); // The call to `with_query_deserialization` enforces that no new `DepNodes` @@ -535,7 +548,7 @@ where if std::intrinsics::unlikely( try_verify || qcx.dep_context().sess().opts.unstable_opts.incremental_verify_ich, ) { - incremental_verify_ich(*qcx.dep_context(), &result, dep_node, Q::HASH_RESULT); + incremental_verify_ich(*qcx.dep_context(), &result, dep_node, query.hash_result()); } return Some((result, dep_node_index)); @@ -554,7 +567,7 @@ where let prof_timer = qcx.dep_context().profiler().query_provider(); // The dep-graph for this computation is already in-place. - let result = dep_graph.with_ignore(|| Q::compute(qcx, *key)); + let result = dep_graph.with_ignore(|| query.compute(*qcx.dep_context(), *key)); prof_timer.finish_with_query_invocation_id(dep_node_index.into()); @@ -567,7 +580,7 @@ where // // See issue #82920 for an example of a miscompilation that would get turned into // an ICE by this check - incremental_verify_ich(*qcx.dep_context(), &result, dep_node, Q::HASH_RESULT); + incremental_verify_ich(*qcx.dep_context(), &result, dep_node, query.hash_result()); Some((result, dep_node_index)) } @@ -688,19 +701,23 @@ fn incremental_verify_ich_failed(sess: &Session, dep_node: DebugArg<'_>, result: /// /// Note: The optimization is only available during incr. comp. #[inline(never)] -fn ensure_must_run(qcx: Qcx, key: &Q::Key) -> (bool, Option>) +fn ensure_must_run( + query: Q, + qcx: Qcx, + key: &Q::Key, +) -> (bool, Option>) where Q: QueryConfig, Qcx: QueryContext, { - if Q::EVAL_ALWAYS { + if query.eval_always() { return (true, None); } // Ensuring an anonymous query makes no sense - assert!(!Q::ANON); + assert!(!query.anon()); - let dep_node = Q::construct_dep_node(*qcx.dep_context(), key); + let dep_node = query.construct_dep_node(*qcx.dep_context(), key); let dep_graph = qcx.dep_context().dep_graph(); match dep_graph.try_mark_green(qcx, &dep_node) { @@ -728,15 +745,19 @@ pub enum QueryMode { } #[inline(always)] -pub fn get_query(qcx: Qcx, span: Span, key: Q::Key, mode: QueryMode) -> Option +pub fn get_query( + query: Q, + qcx: Qcx, + span: Span, + key: Q::Key, + mode: QueryMode, +) -> Option where - D: DepKind, Q: QueryConfig, - Q::Value: Value, Qcx: QueryContext, { let dep_node = if let QueryMode::Ensure = mode { - let (must_run, dep_node) = ensure_must_run::(qcx, &key); + let (must_run, dep_node) = ensure_must_run(query, qcx, &key); if !must_run { return None; } @@ -746,28 +767,30 @@ where }; let (result, dep_node_index) = - ensure_sufficient_stack(|| try_execute_query::(qcx, span, key, dep_node)); + ensure_sufficient_stack(|| try_execute_query(query, qcx, span, key, dep_node)); if let Some(dep_node_index) = dep_node_index { qcx.dep_context().dep_graph().read_index(dep_node_index) } Some(result) } -pub fn force_query(qcx: Qcx, key: Q::Key, dep_node: DepNode) -where - D: DepKind, +pub fn force_query( + query: Q, + qcx: Qcx, + key: Q::Key, + dep_node: DepNode<::DepKind>, +) where Q: QueryConfig, - Q::Value: Value, Qcx: QueryContext, { // We may be concurrently trying both execute and force a query. // Ensure that only one of them runs the query. - if let Some((_, index)) = Q::query_cache(qcx).lookup(&key) { + if let Some((_, index)) = query.query_cache(qcx).lookup(&key) { qcx.dep_context().profiler().query_cache_hit(index.into()); return; } - debug_assert!(!Q::ANON); + debug_assert!(!query.anon()); - ensure_sufficient_stack(|| try_execute_query::(qcx, DUMMY_SP, key, Some(dep_node))); + ensure_sufficient_stack(|| try_execute_query(query, qcx, DUMMY_SP, key, Some(dep_node))); }