From 2b38c4bdea7acf72b8322660de5a4f86d561a65c Mon Sep 17 00:00:00 2001 From: Niko Matsakis Date: Thu, 21 Jul 2016 12:33:23 -0400 Subject: [PATCH 1/9] Extend DepGraph so it can track "work-products" A work product right now is just a `.o` file. In the future it probably includes other kinds of files, such as `.bc` files saving the unoptimized LLVM IR. However, because WorkProductIds must be independent of DefIds, so that they don't need translation, this system may not be suitable *as is* for storing fine-grained information (such as the MIR for individual defs), as it was originally intended. We will want to refactor some for that. --- src/librustc/dep_graph/dep_node.rs | 21 ++++++ src/librustc/dep_graph/graph.rs | 114 ++++++++++++++++++++++++++--- src/librustc/dep_graph/mod.rs | 2 + 3 files changed, 128 insertions(+), 9 deletions(-) diff --git a/src/librustc/dep_graph/dep_node.rs b/src/librustc/dep_graph/dep_node.rs index 73b96651b05e2..dd7f0286574d0 100644 --- a/src/librustc/dep_graph/dep_node.rs +++ b/src/librustc/dep_graph/dep_node.rs @@ -9,6 +9,7 @@ // except according to those terms. use std::fmt::Debug; +use std::sync::Arc; macro_rules! try_opt { ($e:expr) => ( @@ -45,6 +46,10 @@ pub enum DepNode { // in an extern crate. MetaData(D), + // Represents some artifact that we save to disk. Note that these + // do not have a def-id as part of their identifier. + WorkProduct(Arc), + // Represents different phases in the compiler. CrateReader, CollectLanguageItems, @@ -189,6 +194,11 @@ impl DepNode { TransCrate => Some(TransCrate), TransWriteMetadata => Some(TransWriteMetadata), LinkBinary => Some(LinkBinary), + + // work product names do not need to be mapped, because + // they are always absolute. + WorkProduct(ref id) => Some(WorkProduct(id.clone())), + Hir(ref d) => op(d).map(Hir), MetaData(ref d) => op(d).map(MetaData), CollectItem(ref d) => op(d).map(CollectItem), @@ -229,3 +239,14 @@ impl DepNode { } } } + +/// A "work product" corresponds to a `.o` (or other) file that we +/// save in between runs. These ids do not have a DefId but rather +/// some independent path or string that persists between runs without +/// the need to be mapped or unmapped. (This ensures we can serialize +/// them even in the absence of a tcx.) +#[derive(Clone, Debug, PartialEq, Eq, Hash, RustcEncodable, RustcDecodable)] +pub enum WorkProductId { + PartitionObjectFile(String), // see (*TransPartition) below +} + diff --git a/src/librustc/dep_graph/graph.rs b/src/librustc/dep_graph/graph.rs index 741ad65c29fda..ab7013df33f14 100644 --- a/src/librustc/dep_graph/graph.rs +++ b/src/librustc/dep_graph/graph.rs @@ -9,22 +9,44 @@ // except according to those terms. use hir::def_id::DefId; +use rustc_data_structures::fnv::FnvHashMap; +use std::cell::{Ref, RefCell}; use std::rc::Rc; +use std::sync::Arc; -use super::dep_node::DepNode; +use super::dep_node::{DepNode, WorkProductId}; use super::query::DepGraphQuery; use super::raii; use super::thread::{DepGraphThreadData, DepMessage}; #[derive(Clone)] pub struct DepGraph { - data: Rc + data: Rc +} + +struct DepGraphData { + /// we send messages to the thread to let it build up the dep-graph + /// from the current run + thread: DepGraphThreadData, + + /// when we load, there may be `.o` files, cached mir, or other such + /// things available to us. If we find that they are not dirty, we + /// load the path to the file storing those work-products here into + /// this map. We can later look for and extract that data. + previous_work_products: RefCell, WorkProduct>>, + + /// work-products that we generate in this run + work_products: RefCell, WorkProduct>>, } impl DepGraph { pub fn new(enabled: bool) -> DepGraph { DepGraph { - data: Rc::new(DepGraphThreadData::new(enabled)) + data: Rc::new(DepGraphData { + thread: DepGraphThreadData::new(enabled), + previous_work_products: RefCell::new(FnvHashMap()), + work_products: RefCell::new(FnvHashMap()) + }) } } @@ -32,19 +54,19 @@ impl DepGraph { /// then the other methods on this `DepGraph` will have no net effect. #[inline] pub fn enabled(&self) -> bool { - self.data.enabled() + self.data.thread.enabled() } pub fn query(&self) -> DepGraphQuery { - self.data.query() + self.data.thread.query() } pub fn in_ignore<'graph>(&'graph self) -> raii::IgnoreTask<'graph> { - raii::IgnoreTask::new(&self.data) + raii::IgnoreTask::new(&self.data.thread) } pub fn in_task<'graph>(&'graph self, key: DepNode) -> raii::DepTask<'graph> { - raii::DepTask::new(&self.data, key) + raii::DepTask::new(&self.data.thread, key) } pub fn with_ignore(&self, op: OP) -> R @@ -62,10 +84,84 @@ impl DepGraph { } pub fn read(&self, v: DepNode) { - self.data.enqueue(DepMessage::Read(v)); + self.data.thread.enqueue(DepMessage::Read(v)); } pub fn write(&self, v: DepNode) { - self.data.enqueue(DepMessage::Write(v)); + self.data.thread.enqueue(DepMessage::Write(v)); + } + + /// Indicates that a previous work product exists for `v`. This is + /// invoked during initial start-up based on what nodes are clean + /// (and what files exist in the incr. directory). + pub fn insert_previous_work_product(&self, v: &Arc, data: WorkProduct) { + debug!("insert_previous_work_product({:?}, {:?})", v, data); + self.data.previous_work_products.borrow_mut() + .insert(v.clone(), data); + } + + /// Indicates that we created the given work-product in this run + /// for `v`. This record will be preserved and loaded in the next + /// run. + pub fn insert_work_product(&self, v: &Arc, data: WorkProduct) { + debug!("insert_work_product({:?}, {:?})", v, data); + self.data.work_products.borrow_mut() + .insert(v.clone(), data); } + + /// Check whether a previous work product exists for `v` and, if + /// so, return the path that leads to it. Used to skip doing work. + pub fn previous_work_product(&self, v: &Arc) -> Option { + self.data.previous_work_products.borrow() + .get(v) + .cloned() + } + + /// Access the map of work-products created during this run. Only + /// used during saving of the dep-graph. + pub fn work_products(&self) -> Ref, WorkProduct>> { + self.data.work_products.borrow() + } +} + +/// A "work product" is an intermediate result that we save into the +/// incremental directory for later re-use. The primary example are +/// the object files that we save for each partition at code +/// generation time. +/// +/// Each work product is associated with a dep-node, representing the +/// process that produced the work-product. If that dep-node is found +/// to be dirty when we load up, then we will delete the work-product +/// at load time. If the work-product is found to be clean, the we +/// will keep a record in the `previous_work_products` list. +/// +/// In addition, work products have an associated hash. This hash is +/// an extra hash that can be used to decide if the work-product from +/// a previous compilation can be re-used (in addition to the dirty +/// edges check). +/// +/// As the primary example, consider the object files we generate for +/// each partition. In the first run, we create partitions based on +/// the symbols that need to be compiled. For each partition P, we +/// hash the symbols in P and create a `WorkProduct` record associated +/// with `DepNode::TransPartition(P)`; the hash is the set of symbols +/// in P. +/// +/// The next time we compile, if the `DepNode::TransPartition(P)` is +/// judged to be clean (which means none of the things we read to +/// generate the partition were found to be dirty), it will be loaded +/// into previous work products. We will then regenerate the set of +/// symbols in the partition P and hash them (note that new symbols +/// may be added -- for example, new monomorphizations -- even if +/// nothing in P changed!). We will compare that hash against the +/// previous hash. If it matches up, we can reuse the object file. +#[derive(Clone, Debug, RustcEncodable, RustcDecodable)] +pub struct WorkProduct { + /// extra hash used to decide if work-product is still suitable; + /// note that this is *not* a hash of the work-product itself. + /// See documentation on `WorkProduct` type for an example. + pub input_hash: u64, + + /// filename storing this work-product (found in the incr. comp. directory) + pub file_name: String, } diff --git a/src/librustc/dep_graph/mod.rs b/src/librustc/dep_graph/mod.rs index e65f6bbcf7aab..a499cb10f2325 100644 --- a/src/librustc/dep_graph/mod.rs +++ b/src/librustc/dep_graph/mod.rs @@ -20,7 +20,9 @@ mod visit; pub use self::dep_tracking_map::{DepTrackingMap, DepTrackingMapConfig}; pub use self::dep_node::DepNode; +pub use self::dep_node::WorkProductId; pub use self::graph::DepGraph; +pub use self::graph::WorkProduct; pub use self::query::DepGraphQuery; pub use self::visit::visit_all_items_in_krate; pub use self::raii::DepTask; From ffc13b2f80dfe60895bb415175fa246d7247a33c Mon Sep 17 00:00:00 2001 From: Niko Matsakis Date: Thu, 21 Jul 2016 12:41:29 -0400 Subject: [PATCH 2/9] Store `crate_disambiguator` as an `InternedString` We used to use `Name`, but the session outlives the tokenizer, which means that attempts to read this field after trans has complete otherwise panic. All reads want an `InternedString` anyhow. --- src/librustc/session/mod.rs | 7 +++++-- src/librustc/ty/context.rs | 2 +- src/librustc_driver/driver.rs | 3 ++- src/librustc_incremental/calculate_svh.rs | 8 ++++---- src/librustc_metadata/creader.rs | 2 +- src/librustc_metadata/encoder.rs | 2 +- 6 files changed, 14 insertions(+), 10 deletions(-) diff --git a/src/librustc/session/mod.rs b/src/librustc/session/mod.rs index 5901c42b52582..cee18232ec98a 100644 --- a/src/librustc/session/mod.rs +++ b/src/librustc/session/mod.rs @@ -80,7 +80,7 @@ pub struct Session { // forms a unique global identifier for the crate. It is used to allow // multiple crates with the same name to coexist. See the // trans::back::symbol_names module for more information. - pub crate_disambiguator: Cell, + pub crate_disambiguator: RefCell, pub features: RefCell, /// The maximum recursion limit for potentially infinitely recursive @@ -106,6 +106,9 @@ pub struct Session { } impl Session { + pub fn local_crate_disambiguator(&self) -> token::InternedString { + self.crate_disambiguator.borrow().clone() + } pub fn struct_span_warn<'a, S: Into>(&'a self, sp: S, msg: &str) @@ -438,7 +441,7 @@ pub fn build_session_(sopts: config::Options, plugin_attributes: RefCell::new(Vec::new()), crate_types: RefCell::new(Vec::new()), dependency_formats: RefCell::new(FnvHashMap()), - crate_disambiguator: Cell::new(token::intern("")), + crate_disambiguator: RefCell::new(token::intern("").as_str()), features: RefCell::new(feature_gate::Features::new()), recursion_limit: Cell::new(64), next_node_id: Cell::new(1), diff --git a/src/librustc/ty/context.rs b/src/librustc/ty/context.rs index 56938a7a83858..5444dd9476120 100644 --- a/src/librustc/ty/context.rs +++ b/src/librustc/ty/context.rs @@ -504,7 +504,7 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> { pub fn crate_disambiguator(self, cnum: ast::CrateNum) -> token::InternedString { if cnum == LOCAL_CRATE { - self.sess.crate_disambiguator.get().as_str() + self.sess.local_crate_disambiguator() } else { self.sess.cstore.crate_disambiguator(cnum) } diff --git a/src/librustc_driver/driver.rs b/src/librustc_driver/driver.rs index 9a94cc16bfe8c..e1fb7d05b7268 100644 --- a/src/librustc_driver/driver.rs +++ b/src/librustc_driver/driver.rs @@ -566,7 +566,8 @@ pub fn phase_2_configure_and_expand<'a, F>(sess: &Session, }); *sess.crate_types.borrow_mut() = collect_crate_types(sess, &krate.attrs); - sess.crate_disambiguator.set(token::intern(&compute_crate_disambiguator(sess))); + *sess.crate_disambiguator.borrow_mut() = + token::intern(&compute_crate_disambiguator(sess)).as_str(); time(time_passes, "recursion limit", || { middle::recursion_limit::update_recursion_limit(sess, &krate); diff --git a/src/librustc_incremental/calculate_svh.rs b/src/librustc_incremental/calculate_svh.rs index cbc246ac2a11b..70704f5dec0d2 100644 --- a/src/librustc_incremental/calculate_svh.rs +++ b/src/librustc_incremental/calculate_svh.rs @@ -36,7 +36,7 @@ impl<'a, 'tcx> SvhCalculate for TyCtxt<'a, 'tcx, 'tcx> { // to ensure it is not incorporating implementation artifacts into // the hash that are not otherwise visible.) - let crate_disambiguator = self.sess.crate_disambiguator.get(); + let crate_disambiguator = self.sess.local_crate_disambiguator(); let krate = self.map.krate(); // FIXME: this should use SHA1, not SipHash. SipHash is not built to @@ -47,10 +47,10 @@ impl<'a, 'tcx> SvhCalculate for TyCtxt<'a, 'tcx, 'tcx> { // FIXME(#32753) -- at (*) we `to_le` for endianness, but is // this enough, and does it matter anyway? "crate_disambiguator".hash(&mut state); - crate_disambiguator.as_str().len().to_le().hash(&mut state); // (*) - crate_disambiguator.as_str().hash(&mut state); + crate_disambiguator.len().to_le().hash(&mut state); // (*) + crate_disambiguator.hash(&mut state); - debug!("crate_disambiguator: {:?}", crate_disambiguator.as_str()); + debug!("crate_disambiguator: {:?}", crate_disambiguator); debug!("state: {:?}", state); { diff --git a/src/librustc_metadata/creader.rs b/src/librustc_metadata/creader.rs index d4443c6d09d7c..0b60fc386a7bb 100644 --- a/src/librustc_metadata/creader.rs +++ b/src/librustc_metadata/creader.rs @@ -243,7 +243,7 @@ impl<'a> CrateReader<'a> { // Check for (potential) conflicts with the local crate if self.local_crate_name == crate_name && - self.sess.crate_disambiguator.get().as_str() == disambiguator { + self.sess.local_crate_disambiguator() == disambiguator { span_fatal!(self.sess, span, E0519, "the current crate is indistinguishable from one of its \ dependencies: it has the same crate-name `{}` and was \ diff --git a/src/librustc_metadata/encoder.rs b/src/librustc_metadata/encoder.rs index 7314259423592..c896263de948e 100644 --- a/src/librustc_metadata/encoder.rs +++ b/src/librustc_metadata/encoder.rs @@ -1893,7 +1893,7 @@ fn encode_metadata_inner(rbml_w: &mut Encoder, encode_crate_name(rbml_w, &ecx.link_meta.crate_name); encode_crate_triple(rbml_w, &ecx.tcx.sess.opts.target_triple); encode_hash(rbml_w, &ecx.link_meta.crate_hash); - encode_crate_disambiguator(rbml_w, &ecx.tcx.sess.crate_disambiguator.get().as_str()); + encode_crate_disambiguator(rbml_w, &ecx.tcx.sess.local_crate_disambiguator()); encode_dylib_dependency_formats(rbml_w, &ecx); encode_panic_strategy(rbml_w, &ecx); From cca4804251957646d4840bf33b3b13e3f2b645de Mon Sep 17 00:00:00 2001 From: Niko Matsakis Date: Thu, 21 Jul 2016 12:44:59 -0400 Subject: [PATCH 3/9] Code to save/load the work-products map from disk Work products are deleted if any of their inputs are dirty. --- src/librustc_driver/driver.rs | 11 ++- src/librustc_incremental/lib.rs | 2 + src/librustc_incremental/persist/data.rs | 12 ++- src/librustc_incremental/persist/load.rs | 121 ++++++++++++++++++----- src/librustc_incremental/persist/mod.rs | 1 + src/librustc_incremental/persist/save.rs | 51 +++++++--- src/librustc_incremental/persist/util.rs | 46 ++++++--- 7 files changed, 191 insertions(+), 53 deletions(-) diff --git a/src/librustc_driver/driver.rs b/src/librustc_driver/driver.rs index e1fb7d05b7268..a48ff2533485c 100644 --- a/src/librustc_driver/driver.rs +++ b/src/librustc_driver/driver.rs @@ -88,7 +88,7 @@ pub fn compile_input(sess: &Session, // We need nested scopes here, because the intermediate results can keep // large chunks of memory alive and we want to free them as soon as // possible to keep the peak memory usage low - let (outputs, trans) = { + let (outputs, trans, id) = { let krate = match phase_1_parse_input(sess, cfg, input) { Ok(krate) => krate, Err(mut parse_error) => { @@ -212,11 +212,11 @@ pub fn compile_input(sess: &Session, // Discard interned strings as they are no longer required. token::clear_ident_interner(); - Ok((outputs, trans)) + Ok((outputs, trans, id.clone())) })?? }; - let phase5_result = phase_5_run_llvm_passes(sess, &trans, &outputs); + let phase5_result = phase_5_run_llvm_passes(sess, &id, &trans, &outputs); controller_entry_point!(after_llvm, sess, @@ -1020,6 +1020,7 @@ pub fn phase_4_translate_to_llvm<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, /// Run LLVM itself, producing a bitcode file, assembly file or object file /// as a side effect. pub fn phase_5_run_llvm_passes(sess: &Session, + crate_name: &str, trans: &trans::CrateTranslation, outputs: &OutputFilenames) -> CompileResult { if sess.opts.cg.no_integrated_as { @@ -1041,6 +1042,10 @@ pub fn phase_5_run_llvm_passes(sess: &Session, || write::run_passes(sess, trans, &sess.opts.output_types, outputs)); } + time(sess.time_passes(), + "serialize work products", + move || rustc_incremental::save_work_products(sess, crate_name)); + if sess.err_count() > 0 { Err(sess.err_count()) } else { diff --git a/src/librustc_incremental/lib.rs b/src/librustc_incremental/lib.rs index ed31e0ba51056..352e5979d011e 100644 --- a/src/librustc_incremental/lib.rs +++ b/src/librustc_incremental/lib.rs @@ -19,6 +19,7 @@ html_root_url = "https://doc.rust-lang.org/nightly/")] #![cfg_attr(not(stage0), deny(warnings))] +#![feature(question_mark)] #![feature(rustc_private)] #![feature(staged_api)] @@ -40,3 +41,4 @@ pub use assert_dep_graph::assert_dep_graph; pub use calculate_svh::SvhCalculate; pub use persist::load_dep_graph; pub use persist::save_dep_graph; +pub use persist::save_work_products; diff --git a/src/librustc_incremental/persist/data.rs b/src/librustc_incremental/persist/data.rs index f57ab19a5256e..95e9a16f29bbe 100644 --- a/src/librustc_incremental/persist/data.rs +++ b/src/librustc_incremental/persist/data.rs @@ -10,8 +10,9 @@ //! The data that we will serialize and deserialize. -use rustc::dep_graph::DepNode; +use rustc::dep_graph::{DepNode, WorkProduct, WorkProductId}; use rustc::hir::def_id::DefIndex; +use std::sync::Arc; use super::directory::DefPathIndex; @@ -55,6 +56,15 @@ pub struct SerializedHash { pub hash: u64, } +#[derive(Debug, RustcEncodable, RustcDecodable)] +pub struct SerializedWorkProduct { + /// node that produced the work-product + pub id: Arc, + + /// work-product data itself + pub work_product: WorkProduct, +} + /// Data for use when downstream crates get recompiled. #[derive(Debug, RustcEncodable, RustcDecodable)] pub struct SerializedMetadataHashes { diff --git a/src/librustc_incremental/persist/load.rs b/src/librustc_incremental/persist/load.rs index 0ac1018462ee7..9fef2285aa7eb 100644 --- a/src/librustc_incremental/persist/load.rs +++ b/src/librustc_incremental/persist/load.rs @@ -14,12 +14,13 @@ use rbml::Error; use rbml::opaque::Decoder; use rustc::dep_graph::DepNode; use rustc::hir::def_id::DefId; +use rustc::session::Session; use rustc::ty::TyCtxt; use rustc_data_structures::fnv::FnvHashSet; use rustc_serialize::Decodable as RustcDecodable; use std::io::Read; -use std::fs::File; -use std::path::Path; +use std::fs::{self, File}; +use std::path::{Path}; use super::data::*; use super::directory::*; @@ -38,18 +39,40 @@ type CleanEdges = Vec<(DepNode, DepNode)>; /// actually it doesn't matter all that much.) See `README.md` for /// more general overview. pub fn load_dep_graph<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>) { + if tcx.sess.opts.incremental.is_none() { + return; + } + let _ignore = tcx.dep_graph.in_ignore(); + load_dep_graph_if_exists(tcx); + dirty_clean::check_dirty_clean_annotations(tcx); +} - if let Some(dep_graph) = dep_graph_path(tcx) { - // FIXME(#32754) lock file? - load_dep_graph_if_exists(tcx, &dep_graph); - dirty_clean::check_dirty_clean_annotations(tcx); +fn load_dep_graph_if_exists<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>) { + let dep_graph_path = dep_graph_path(tcx).unwrap(); + let dep_graph_data = match load_data(tcx.sess, &dep_graph_path) { + Some(p) => p, + None => return // no file + }; + + let work_products_path = tcx_work_products_path(tcx).unwrap(); + let work_products_data = match load_data(tcx.sess, &work_products_path) { + Some(p) => p, + None => return // no file + }; + + match decode_dep_graph(tcx, &dep_graph_data, &work_products_data) { + Ok(()) => return, + Err(err) => bug!("decoding error in dep-graph from `{}` and `{}`: {}", + dep_graph_path.display(), + work_products_path.display(), + err), } } -pub fn load_dep_graph_if_exists<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, path: &Path) { +fn load_data(sess: &Session, path: &Path) -> Option> { if !path.exists() { - return; + return None; } let mut data = vec![]; @@ -57,31 +80,32 @@ pub fn load_dep_graph_if_exists<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, path: &Pa File::open(path) .and_then(|mut file| file.read_to_end(&mut data)) { - Ok(_) => { } + Ok(_) => { + Some(data) + } Err(err) => { - tcx.sess.err( + sess.err( &format!("could not load dep-graph from `{}`: {}", path.display(), err)); - return; + None } } - match decode_dep_graph(tcx, &data) { - Ok(dirty) => dirty, - Err(err) => { - bug!("decoding error in dep-graph from `{}`: {}", path.display(), err); - } - } } +/// Decode the dep graph and load the edges/nodes that are still clean +/// into `tcx.dep_graph`. On success, returns a hashset containing all +/// the paths of work-products from clean nodes (any work-products not +/// in this set can be deleted). pub fn decode_dep_graph<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, - data: &[u8]) + dep_graph_data: &[u8], + work_products_data: &[u8]) -> Result<(), Error> { // Deserialize the directory and dep-graph. - let mut decoder = Decoder::new(data, 0); - let directory = try!(DefIdDirectory::decode(&mut decoder)); - let serialized_dep_graph = try!(SerializedDepGraph::decode(&mut decoder)); + let mut dep_graph_decoder = Decoder::new(dep_graph_data, 0); + let directory = try!(DefIdDirectory::decode(&mut dep_graph_decoder)); + let serialized_dep_graph = try!(SerializedDepGraph::decode(&mut dep_graph_decoder)); debug!("decode_dep_graph: directory = {:#?}", directory); debug!("decode_dep_graph: serialized_dep_graph = {:#?}", serialized_dep_graph); @@ -121,12 +145,18 @@ pub fn decode_dep_graph<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, // Add nodes and edges that are not dirty into our main graph. let dep_graph = tcx.dep_graph.clone(); for (source, target) in clean_edges.into_iter().chain(clean_nodes) { - let _task = dep_graph.in_task(target.clone()); - dep_graph.read(source.clone()); - debug!("decode_dep_graph: clean edge: {:?} -> {:?}", source, target); + + let _task = dep_graph.in_task(target); + dep_graph.read(source); } + // Add in work-products that are still clean, and delete those that are + // dirty. + let mut work_product_decoder = Decoder::new(work_products_data, 0); + let work_products = try!(>::decode(&mut work_product_decoder)); + reconcile_work_products(tcx, work_products, &dirty_nodes); + Ok(()) } @@ -141,9 +171,9 @@ fn initial_dirty_nodes<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, match hash.node.map_def(|&i| retraced.def_id(i)) { Some(dep_node) => { let current_hash = hcx.hash(&dep_node).unwrap(); - debug!("initial_dirty_nodes: hash of {:?} is {:?}, was {:?}", - dep_node, current_hash, hash.hash); if current_hash != hash.hash { + debug!("initial_dirty_nodes: {:?} is dirty as hash is {:?}, was {:?}", + dep_node, current_hash, hash.hash); dirty_nodes.insert(dep_node); } } @@ -177,6 +207,8 @@ fn compute_clean_edges(serialized_edges: &[(SerializedEdge)], clean_edges.push((source, target)) } else { // source removed, target must be dirty + debug!("compute_clean_edges: {:?} dirty because {:?} no longer exists", + target, serialized_source); dirty_nodes.insert(target); } } else { @@ -213,3 +245,40 @@ fn compute_clean_edges(serialized_edges: &[(SerializedEdge)], clean_edges } + +/// Go through the list of work-products produced in the previous run. +/// Delete any whose nodes have been found to be dirty or which are +/// otherwise no longer applicable. +fn reconcile_work_products<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, + work_products: Vec, + dirty_nodes: &DirtyNodes) { + debug!("reconcile_work_products({:?})", work_products); + for swp in work_products { + let dep_node = DepNode::WorkProduct(swp.id.clone()); + if dirty_nodes.contains(&dep_node) { + debug!("reconcile_work_products: dep-node for {:?} is dirty", swp); + delete_dirty_work_product(tcx, swp); + } else { + let path = in_incr_comp_dir(tcx.sess, &swp.work_product.file_name).unwrap(); + if path.exists() { + tcx.dep_graph.insert_previous_work_product(&swp.id, swp.work_product); + } else { + debug!("reconcile_work_products: file for {:?} does not exist", swp); + } + } + } +} + +fn delete_dirty_work_product(tcx: TyCtxt, + swp: SerializedWorkProduct) { + debug!("delete_dirty_work_product({:?})", swp); + let path = in_incr_comp_dir(tcx.sess, &swp.work_product.file_name).unwrap(); + match fs::remove_file(&path) { + Ok(()) => { } + Err(err) => { + tcx.sess.warn( + &format!("file-system error deleting outdated file `{}`: {}", + path.display(), err)); + } + } +} diff --git a/src/librustc_incremental/persist/mod.rs b/src/librustc_incremental/persist/mod.rs index 72ccc29c97b63..30e7d7873ecc1 100644 --- a/src/librustc_incremental/persist/mod.rs +++ b/src/librustc_incremental/persist/mod.rs @@ -22,3 +22,4 @@ mod util; pub use self::load::load_dep_graph; pub use self::save::save_dep_graph; +pub use self::save::save_work_products; diff --git a/src/librustc_incremental/persist/save.rs b/src/librustc_incremental/persist/save.rs index 99f4d4f307298..305250d59623c 100644 --- a/src/librustc_incremental/persist/save.rs +++ b/src/librustc_incremental/persist/save.rs @@ -11,6 +11,7 @@ use rbml::opaque::Encoder; use rustc::dep_graph::DepNode; use rustc::middle::cstore::LOCAL_CRATE; +use rustc::session::Session; use rustc::ty::TyCtxt; use rustc_serialize::{Encodable as RustcEncodable}; use std::hash::{Hasher, SipHasher}; @@ -24,19 +25,26 @@ use super::hash::*; use super::util::*; pub fn save_dep_graph<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>) { + debug!("save_dep_graph()"); let _ignore = tcx.dep_graph.in_ignore(); + let sess = tcx.sess; let mut hcx = HashContext::new(tcx); - save_in(&mut hcx, dep_graph_path(tcx), encode_dep_graph); - save_in(&mut hcx, metadata_hash_path(tcx, LOCAL_CRATE), encode_metadata_hashes); + save_in(sess, dep_graph_path(tcx), |e| encode_dep_graph(&mut hcx, e)); + save_in(sess, metadata_hash_path(tcx, LOCAL_CRATE), |e| encode_metadata_hashes(&mut hcx, e)); } -fn save_in<'a, 'tcx, F>(hcx: &mut HashContext<'a, 'tcx>, - opt_path_buf: Option, - encode: F) - where F: FnOnce(&mut HashContext<'a, 'tcx>, &mut Encoder) -> io::Result<()> -{ - let tcx = hcx.tcx; +pub fn save_work_products(sess: &Session, local_crate_name: &str) { + debug!("save_work_products()"); + let _ignore = sess.dep_graph.in_ignore(); + let path = sess_work_products_path(sess, local_crate_name); + save_in(sess, path, |e| encode_work_products(sess, e)); +} +fn save_in(sess: &Session, + opt_path_buf: Option, + encode: F) + where F: FnOnce(&mut Encoder) -> io::Result<()> +{ let path_buf = match opt_path_buf { Some(p) => p, None => return @@ -49,7 +57,7 @@ fn save_in<'a, 'tcx, F>(hcx: &mut HashContext<'a, 'tcx>, match fs::remove_file(&path_buf) { Ok(()) => { } Err(err) => { - tcx.sess.err( + sess.err( &format!("unable to delete old dep-graph at `{}`: {}", path_buf.display(), err)); return; @@ -59,10 +67,10 @@ fn save_in<'a, 'tcx, F>(hcx: &mut HashContext<'a, 'tcx>, // generate the data in a memory buffer let mut wr = Cursor::new(Vec::new()); - match encode(hcx, &mut Encoder::new(&mut wr)) { + match encode(&mut Encoder::new(&mut wr)) { Ok(()) => { } Err(err) => { - tcx.sess.err( + sess.err( &format!("could not encode dep-graph to `{}`: {}", path_buf.display(), err)); return; @@ -77,7 +85,7 @@ fn save_in<'a, 'tcx, F>(hcx: &mut HashContext<'a, 'tcx>, { Ok(_) => { } Err(err) => { - tcx.sess.err( + sess.err( &format!("failed to write dep-graph to `{}`: {}", path_buf.display(), err)); return; @@ -192,3 +200,22 @@ pub fn encode_metadata_hashes<'a, 'tcx>(hcx: &mut HashContext<'a, 'tcx>, Ok(()) } + +pub fn encode_work_products(sess: &Session, + encoder: &mut Encoder) + -> io::Result<()> +{ + let work_products: Vec<_> = + sess.dep_graph.work_products() + .iter() + .map(|(id, work_product)| { + SerializedWorkProduct { + id: id.clone(), + work_product: work_product.clone(), + } + }) + .collect(); + + work_products.encode(encoder) +} + diff --git a/src/librustc_incremental/persist/util.rs b/src/librustc_incremental/persist/util.rs index a77a9607e7734..f1e81fdb266b9 100644 --- a/src/librustc_incremental/persist/util.rs +++ b/src/librustc_incremental/persist/util.rs @@ -9,6 +9,7 @@ // except according to those terms. use rustc::middle::cstore::LOCAL_CRATE; +use rustc::session::Session; use rustc::ty::TyCtxt; use std::fs; @@ -17,33 +18,56 @@ use std::path::{Path, PathBuf}; use syntax::ast; pub fn dep_graph_path(tcx: TyCtxt) -> Option { - path(tcx, LOCAL_CRATE, "local") + tcx_path(tcx, LOCAL_CRATE, "local") } pub fn metadata_hash_path(tcx: TyCtxt, cnum: ast::CrateNum) -> Option { - path(tcx, cnum, "metadata") + tcx_path(tcx, cnum, "metadata") } -fn path(tcx: TyCtxt, cnum: ast::CrateNum, suffix: &str) -> Option { +pub fn tcx_work_products_path(tcx: TyCtxt) -> Option { + let crate_name = tcx.crate_name(LOCAL_CRATE); + sess_work_products_path(tcx.sess, &crate_name) +} + +pub fn sess_work_products_path(sess: &Session, + local_crate_name: &str) + -> Option { + let crate_disambiguator = sess.local_crate_disambiguator(); + path(sess, local_crate_name, &crate_disambiguator, "work-products") +} + +pub fn in_incr_comp_dir(sess: &Session, file_name: &str) -> Option { + sess.opts.incremental.as_ref().map(|incr_dir| incr_dir.join(file_name)) +} + +fn tcx_path(tcx: TyCtxt, + cnum: ast::CrateNum, + middle: &str) + -> Option { + path(tcx.sess, &tcx.crate_name(cnum), &tcx.crate_disambiguator(cnum), middle) +} + +fn path(sess: &Session, + crate_name: &str, + crate_disambiguator: &str, + middle: &str) + -> Option { // For now, just save/load dep-graph from // directory/dep_graph.rbml - tcx.sess.opts.incremental.as_ref().and_then(|incr_dir| { + sess.opts.incremental.as_ref().and_then(|incr_dir| { match create_dir_racy(&incr_dir) { Ok(()) => {} Err(err) => { - tcx.sess.err( + sess.err( &format!("could not create the directory `{}`: {}", incr_dir.display(), err)); return None; } } - let crate_name = tcx.crate_name(cnum); - let crate_disambiguator = tcx.crate_disambiguator(cnum); - let file_name = format!("{}-{}.{}.bin", - crate_name, - crate_disambiguator, - suffix); + let file_name = format!("{}-{}.{}.bin", crate_name, crate_disambiguator, middle); + Some(incr_dir.join(file_name)) }) } From 58d4b8edd319d0f0d76024504cdfc74f89a001b1 Mon Sep 17 00:00:00 2001 From: Niko Matsakis Date: Thu, 21 Jul 2016 12:49:59 -0400 Subject: [PATCH 4/9] Modify trans to skip generating `.o` files This checks the `previous_work_products` data from the dep-graph and tries to simply copy a `.o` file if possible. We also add new work-products into the dep-graph, and create edges to/from the dep-node for a work-product. --- src/librustc_incremental/lib.rs | 2 + src/librustc_incremental/persist/mod.rs | 3 + .../persist/work_product.rs | 61 ++++++++ src/librustc_trans/back/write.rs | 67 +++++++-- src/librustc_trans/base.rs | 137 +++++++++++++----- src/librustc_trans/consts.rs | 2 +- src/librustc_trans/context.rs | 69 +++++++-- src/librustc_trans/glue.rs | 2 +- src/librustc_trans/lib.rs | 14 ++ src/librustc_trans/monomorphize.rs | 2 +- src/librustc_trans/partitioning.rs | 80 +++++++--- src/librustc_trans/trans_item.rs | 30 +++- src/test/run-make/execution-engine/test.rs | 7 +- src/test/run-make/llvm-phase/test.rs | 7 +- 14 files changed, 385 insertions(+), 98 deletions(-) create mode 100644 src/librustc_incremental/persist/work_product.rs diff --git a/src/librustc_incremental/lib.rs b/src/librustc_incremental/lib.rs index 352e5979d011e..0d11b0794feac 100644 --- a/src/librustc_incremental/lib.rs +++ b/src/librustc_incremental/lib.rs @@ -41,4 +41,6 @@ pub use assert_dep_graph::assert_dep_graph; pub use calculate_svh::SvhCalculate; pub use persist::load_dep_graph; pub use persist::save_dep_graph; +pub use persist::save_trans_partition; pub use persist::save_work_products; +pub use persist::in_incr_comp_dir; diff --git a/src/librustc_incremental/persist/mod.rs b/src/librustc_incremental/persist/mod.rs index 30e7d7873ecc1..1157f494ce604 100644 --- a/src/librustc_incremental/persist/mod.rs +++ b/src/librustc_incremental/persist/mod.rs @@ -19,7 +19,10 @@ mod hash; mod load; mod save; mod util; +mod work_product; pub use self::load::load_dep_graph; pub use self::save::save_dep_graph; pub use self::save::save_work_products; +pub use self::work_product::save_trans_partition; +pub use self::util::in_incr_comp_dir; diff --git a/src/librustc_incremental/persist/work_product.rs b/src/librustc_incremental/persist/work_product.rs new file mode 100644 index 0000000000000..01ac3f6c391d9 --- /dev/null +++ b/src/librustc_incremental/persist/work_product.rs @@ -0,0 +1,61 @@ +// Copyright 2012-2015 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +//! This module contains files for saving intermediate work-products. + +use persist::util::*; +use rustc::dep_graph::{WorkProduct, WorkProductId}; +use rustc::session::Session; +use std::fs; +use std::path::Path; +use std::sync::Arc; + +pub fn save_trans_partition(sess: &Session, + partition_name: &str, + partition_hash: u64, + path_to_obj_file: &Path) { + debug!("save_trans_partition({:?},{},{})", + partition_name, + partition_hash, + path_to_obj_file.display()); + if sess.opts.incremental.is_none() { + return; + } + let id = Arc::new(WorkProductId::PartitionObjectFile(partition_name.to_string())); + let file_name = format!("cgu-{}", partition_name); + let path_in_incr_dir = in_incr_comp_dir(sess, &file_name).unwrap(); + + // try to delete the file if it already exists + // + // FIXME(#34955) we can be smarter here -- if we are re-using, no need to do anything + if path_in_incr_dir.exists() { + let _ = fs::remove_file(&path_in_incr_dir); + } + + match + fs::hard_link(path_to_obj_file, &path_in_incr_dir) + .or_else(|_| fs::copy(path_to_obj_file, &path_in_incr_dir).map(|_| ())) + { + Ok(_) => { + let work_product = WorkProduct { + input_hash: partition_hash, + file_name: file_name, + }; + sess.dep_graph.insert_work_product(&id, work_product); + } + Err(err) => { + sess.warn(&format!("error copying object file `{}` \ + to incremental directory as `{}`: {}", + path_to_obj_file.display(), + path_in_incr_dir.display(), + err)); + } + } +} diff --git a/src/librustc_trans/back/write.rs b/src/librustc_trans/back/write.rs index 33cffa8a48013..70925218781bd 100644 --- a/src/librustc_trans/back/write.rs +++ b/src/librustc_trans/back/write.rs @@ -10,13 +10,14 @@ use back::lto; use back::link::{get_linker, remove}; +use rustc_incremental::save_trans_partition; use session::config::{OutputFilenames, Passes, SomePasses, AllPasses}; use session::Session; use session::config::{self, OutputType}; use llvm; use llvm::{ModuleRef, TargetMachineRef, PassManagerRef, DiagnosticInfoRef, ContextRef}; use llvm::SMDiagnosticRef; -use {CrateTranslation, ModuleTranslation}; +use {CrateTranslation, ModuleLlvm, ModuleSource, ModuleTranslation}; use util::common::time; use util::common::path2cstr; use errors::{self, Handler, Level, DiagnosticBuilder}; @@ -26,6 +27,7 @@ use syntax_pos::MultiSpan; use std::collections::HashMap; use std::ffi::{CStr, CString}; use std::fs; +use std::io; use std::path::{Path, PathBuf}; use std::str; use std::sync::{Arc, Mutex}; @@ -422,10 +424,11 @@ unsafe extern "C" fn diagnostic_handler(info: DiagnosticInfoRef, user: *mut c_vo // Unsafe due to LLVM calls. unsafe fn optimize_and_codegen(cgcx: &CodegenContext, mtrans: ModuleTranslation, + mllvm: ModuleLlvm, config: ModuleConfig, output_names: OutputFilenames) { - let llmod = mtrans.llmod; - let llcx = mtrans.llcx; + let llmod = mllvm.llmod; + let llcx = mllvm.llcx; let tm = config.tm; // llcx doesn't outlive this function, so we can put this on the stack. @@ -628,8 +631,14 @@ unsafe fn optimize_and_codegen(cgcx: &CodegenContext, pub fn cleanup_llvm(trans: &CrateTranslation) { for module in trans.modules.iter() { unsafe { - llvm::LLVMDisposeModule(module.llmod); - llvm::LLVMContextDispose(module.llcx); + match module.source { + ModuleSource::Translated(llvm) => { + llvm::LLVMDisposeModule(llvm.llmod); + llvm::LLVMContextDispose(llvm.llcx); + } + ModuleSource::Preexisting(_) => { + } + } } } } @@ -743,6 +752,13 @@ pub fn run_passes(sess: &Session, run_work_multithreaded(sess, work_items, num_workers); } + // If in incr. comp. mode, preserve the `.o` files for potential re-use + for mtrans in trans.modules.iter() { + let path_to_obj = crate_output.temp_path(OutputType::Object, Some(&mtrans.name)); + debug!("wrote module {:?} to {:?}", mtrans.name, path_to_obj); + save_trans_partition(sess, &mtrans.name, mtrans.symbol_name_hash, &path_to_obj); + } + // All codegen is finished. unsafe { llvm::LLVMRustDisposeTargetMachine(tm); @@ -913,13 +929,46 @@ fn build_work_item(sess: &Session, } } +fn link_or_copy, Q: AsRef>(p: P, q: Q) -> io::Result<()> { + let p = p.as_ref(); + let q = q.as_ref(); + if q.exists() { + try!(fs::remove_file(&q)); + } + fs::hard_link(p, q) + .or_else(|_| fs::copy(p, q).map(|_| ())) +} + fn execute_work_item(cgcx: &CodegenContext, work_item: WorkItem) { unsafe { - optimize_and_codegen(cgcx, - work_item.mtrans, - work_item.config, - work_item.output_names); + match work_item.mtrans.source { + ModuleSource::Translated(mllvm) => { + debug!("llvm-optimizing {:?}", work_item.mtrans.name); + optimize_and_codegen(cgcx, + work_item.mtrans, + mllvm, + work_item.config, + work_item.output_names); + } + ModuleSource::Preexisting(ref buf) => { + let obj_out = work_item.output_names.temp_path(OutputType::Object, + Some(&work_item.mtrans.name)); + debug!("copying pre-existing module `{}` from {} to {}", + work_item.mtrans.name, + buf.display(), + obj_out.display()); + match link_or_copy(buf, &obj_out) { + Ok(()) => { } + Err(err) => { + cgcx.handler.err(&format!("unable to copy {} to {}: {}", + buf.display(), + obj_out.display(), + err)); + } + } + } + } } } diff --git a/src/librustc_trans/base.rs b/src/librustc_trans/base.rs index ea8c248d0239f..a77ababaa6355 100644 --- a/src/librustc_trans/base.rs +++ b/src/librustc_trans/base.rs @@ -26,6 +26,8 @@ #![allow(non_camel_case_types)] use super::CrateTranslation; +use super::ModuleLlvm; +use super::ModuleSource; use super::ModuleTranslation; use back::link; @@ -43,6 +45,7 @@ use rustc::ty::adjustment::CustomCoerceUnsized; use rustc::dep_graph::DepNode; use rustc::hir::map as hir_map; use rustc::util::common::time; +use rustc_incremental::in_incr_comp_dir; use rustc::mir::mir_map::MirMap; use rustc_data_structures::graph::OUTGOING; use session::config::{self, NoDebugInfo, FullDebugInfo}; @@ -99,6 +102,7 @@ use std::cell::{Cell, RefCell}; use std::collections::HashMap; use std::ptr; use std::rc::Rc; +use std::path::PathBuf; use std::str; use std::{i8, i16, i32, i64}; use syntax_pos::{Span, DUMMY_SP}; @@ -2133,7 +2137,7 @@ pub fn maybe_create_entry_wrapper(ccx: &CrateContext) { let instance = Instance::mono(ccx.shared(), main_def_id); - if !ccx.codegen_unit().items.contains_key(&TransItem::Fn(instance)) { + if !ccx.codegen_unit().contains_item(&TransItem::Fn(instance)) { // We want to create the wrapper in the same codegen unit as Rust's main // function. return; @@ -2270,7 +2274,7 @@ fn internalize_symbols<'a, 'tcx>(ccxs: &CrateContextList<'a, 'tcx>, // Collect all symbols that need to stay externally visible because they // are referenced via a declaration in some other codegen unit. - for ccx in ccxs.iter() { + for ccx in ccxs.iter_need_trans() { for val in iter_globals(ccx.llmod()).chain(iter_functions(ccx.llmod())) { let linkage = llvm::LLVMGetLinkage(val); // We only care about external declarations (not definitions) @@ -2315,7 +2319,7 @@ fn internalize_symbols<'a, 'tcx>(ccxs: &CrateContextList<'a, 'tcx>, // Examine each external definition. If the definition is not used in // any other compilation unit, and is not reachable from other crates, // then give it internal linkage. - for ccx in ccxs.iter() { + for ccx in ccxs.iter_need_trans() { for val in iter_globals(ccx.llmod()).chain(iter_functions(ccx.llmod())) { let linkage = llvm::LLVMGetLinkage(val); @@ -2362,7 +2366,7 @@ fn create_imps(cx: &CrateContextList) { "\x01__imp_" }; unsafe { - for ccx in cx.iter() { + for ccx in cx.iter_need_trans() { let exported: Vec<_> = iter_globals(ccx.llmod()) .filter(|&val| { llvm::LLVMGetLinkage(val) == @@ -2514,8 +2518,11 @@ pub fn trans_crate<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, let metadata_module = ModuleTranslation { name: "metadata".to_string(), - llcx: shared_ccx.metadata_llcx(), - llmod: shared_ccx.metadata_llmod(), + symbol_name_hash: 0, // we always rebuild metadata, at least for now + source: ModuleSource::Translated(ModuleLlvm { + llcx: shared_ccx.metadata_llcx(), + llmod: shared_ccx.metadata_llmod(), + }), }; let no_builtins = attr::contains_name(&krate.attrs, "no_builtins"); @@ -2525,14 +2532,29 @@ pub fn trans_crate<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, let symbol_map = Rc::new(symbol_map); + let previous_work_products = trans_reuse_previous_work_products(tcx, + &codegen_units, + &symbol_map); + let crate_context_list = CrateContextList::new(&shared_ccx, codegen_units, + previous_work_products, symbol_map.clone()); - let modules = crate_context_list.iter() - .map(|ccx| ModuleTranslation { - name: String::from(&ccx.codegen_unit().name[..]), - llcx: ccx.llcx(), - llmod: ccx.llmod() + let modules: Vec<_> = crate_context_list.iter_all() + .map(|ccx| { + let source = match ccx.previous_work_product() { + Some(buf) => ModuleSource::Preexisting(buf.clone()), + None => ModuleSource::Translated(ModuleLlvm { + llcx: ccx.llcx(), + llmod: ccx.llmod(), + }), + }; + + ModuleTranslation { + name: String::from(ccx.codegen_unit().name()), + symbol_name_hash: ccx.codegen_unit().compute_symbol_name_hash(tcx, &symbol_map), + source: source, + } }) .collect(); @@ -2551,41 +2573,44 @@ pub fn trans_crate<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, } // Instantiate translation items without filling out definitions yet... - for ccx in crate_context_list.iter() { - let trans_items = ccx.codegen_unit() - .items_in_deterministic_order(tcx, &symbol_map); + for ccx in crate_context_list.iter_need_trans() { + let cgu = ccx.codegen_unit(); + let trans_items = cgu.items_in_deterministic_order(tcx, &symbol_map); - for (trans_item, linkage) in trans_items { - trans_item.predefine(&ccx, linkage); - } + tcx.dep_graph.with_task(cgu.work_product_dep_node(), || { + for (trans_item, linkage) in trans_items { + trans_item.predefine(&ccx, linkage); + } + }); } // ... and now that we have everything pre-defined, fill out those definitions. - for ccx in crate_context_list.iter() { - let trans_items = ccx.codegen_unit() - .items_in_deterministic_order(tcx, &symbol_map); - - for (trans_item, _) in trans_items { - trans_item.define(&ccx); - } + for ccx in crate_context_list.iter_need_trans() { + let cgu = ccx.codegen_unit(); + let trans_items = cgu.items_in_deterministic_order(tcx, &symbol_map); + tcx.dep_graph.with_task(cgu.work_product_dep_node(), || { + for (trans_item, _) in trans_items { + trans_item.define(&ccx); + } - // If this codegen unit contains the main function, also create the - // wrapper here - maybe_create_entry_wrapper(&ccx); + // If this codegen unit contains the main function, also create the + // wrapper here + maybe_create_entry_wrapper(&ccx); - // Run replace-all-uses-with for statics that need it - for &(old_g, new_g) in ccx.statics_to_rauw().borrow().iter() { - unsafe { - let bitcast = llvm::LLVMConstPointerCast(new_g, llvm::LLVMTypeOf(old_g)); - llvm::LLVMReplaceAllUsesWith(old_g, bitcast); - llvm::LLVMDeleteGlobal(old_g); + // Run replace-all-uses-with for statics that need it + for &(old_g, new_g) in ccx.statics_to_rauw().borrow().iter() { + unsafe { + let bitcast = llvm::LLVMConstPointerCast(new_g, llvm::LLVMTypeOf(old_g)); + llvm::LLVMReplaceAllUsesWith(old_g, bitcast); + llvm::LLVMDeleteGlobal(old_g); + } } - } - // Finalize debuginfo - if ccx.sess().opts.debuginfo != NoDebugInfo { - debuginfo::finalize(&ccx); - } + // Finalize debuginfo + if ccx.sess().opts.debuginfo != NoDebugInfo { + debuginfo::finalize(&ccx); + } + }); } symbol_names_test::report_symbol_names(&shared_ccx); @@ -2679,6 +2704,38 @@ pub fn trans_crate<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, } } +/// For each CGU, identify if we can reuse an existing object file (or +/// maybe other context). +fn trans_reuse_previous_work_products(tcx: TyCtxt, + codegen_units: &[CodegenUnit], + symbol_map: &SymbolMap) + -> Vec> { + debug!("trans_reuse_previous_work_products()"); + codegen_units + .iter() + .map(|cgu| { + let id = cgu.work_product_id(); + + let hash = cgu.compute_symbol_name_hash(tcx, symbol_map); + + debug!("trans_reuse_previous_work_products: id={:?} hash={}", id, hash); + + if let Some(work_product) = tcx.dep_graph.previous_work_product(&id) { + if work_product.input_hash == hash { + debug!("trans_reuse_previous_work_products: reusing {:?}", work_product); + return Some(in_incr_comp_dir(tcx.sess, &work_product.file_name).unwrap()); + } else { + debug!("trans_reuse_previous_work_products: \ + not reusing {:?} because hash changed to {:?}", + work_product, hash); + } + } + + None + }) + .collect() +} + fn collect_and_partition_translation_items<'a, 'tcx>(scx: &SharedCrateContext<'a, 'tcx>) -> (Vec>, SymbolMap<'tcx>) { let time_passes = scx.sess().time_passes(); @@ -2739,10 +2796,10 @@ fn collect_and_partition_translation_items<'a, 'tcx>(scx: &SharedCrateContext<'a let mut item_to_cgus = HashMap::new(); for cgu in &codegen_units { - for (&trans_item, &linkage) in &cgu.items { + for (&trans_item, &linkage) in cgu.items() { item_to_cgus.entry(trans_item) .or_insert(Vec::new()) - .push((cgu.name.clone(), linkage)); + .push((cgu.name().clone(), linkage)); } } diff --git a/src/librustc_trans/consts.rs b/src/librustc_trans/consts.rs index f662ba75cc6fe..571d2731fb216 100644 --- a/src/librustc_trans/consts.rs +++ b/src/librustc_trans/consts.rs @@ -1029,7 +1029,7 @@ pub fn get_static<'a, 'tcx>(ccx: &CrateContext<'a, 'tcx>, def_id: DefId) assert!(!ccx.external_srcs().borrow().contains_key(&id)); let defined_in_current_codegen_unit = ccx.codegen_unit() - .items + .items() .contains_key(&TransItem::Static(id)); if defined_in_current_codegen_unit { if declare::get_declared_value(ccx, sym).is_none() { diff --git a/src/librustc_trans/context.rs b/src/librustc_trans/context.rs index 88903726d64f7..bf69629821285 100644 --- a/src/librustc_trans/context.rs +++ b/src/librustc_trans/context.rs @@ -40,6 +40,7 @@ use util::nodemap::{NodeMap, NodeSet, DefIdMap, FnvHashMap, FnvHashSet}; use std::ffi::{CStr, CString}; use std::cell::{Cell, RefCell}; +use std::path::PathBuf; use std::marker::PhantomData; use std::ptr; use std::rc::Rc; @@ -95,6 +96,7 @@ pub struct SharedCrateContext<'a, 'tcx: 'a> { pub struct LocalCrateContext<'tcx> { llmod: ModuleRef, llcx: ContextRef, + previous_work_product: Option, tn: TypeNames, // FIXME: This seems to be largely unused. codegen_unit: CodegenUnit<'tcx>, needs_unwind_cleanup_cache: RefCell, bool>>, @@ -198,24 +200,39 @@ pub struct CrateContextList<'a, 'tcx: 'a> { } impl<'a, 'tcx: 'a> CrateContextList<'a, 'tcx> { - pub fn new(shared_ccx: &'a SharedCrateContext<'a, 'tcx>, codegen_units: Vec>, + previous_work_products: Vec>, symbol_map: Rc>) -> CrateContextList<'a, 'tcx> { CrateContextList { shared: shared_ccx, - local_ccxs: codegen_units.into_iter().map(|codegen_unit| { - LocalCrateContext::new(shared_ccx, codegen_unit, symbol_map.clone()) + local_ccxs: codegen_units.into_iter().zip(previous_work_products).map(|(cgu, path)| { + LocalCrateContext::new(shared_ccx, cgu, path, symbol_map.clone()) }).collect() } } - pub fn iter<'b>(&'b self) -> CrateContextIterator<'b, 'tcx> { + /// Iterate over all crate contexts, whether or not they need + /// translation. That is, whether or not a `.o` file is available + /// for re-use from a previous incr. comp.). + pub fn iter_all<'b>(&'b self) -> CrateContextIterator<'b, 'tcx> { CrateContextIterator { shared: self.shared, index: 0, - local_ccxs: &self.local_ccxs[..] + local_ccxs: &self.local_ccxs[..], + filter_to_previous_work_product_unavail: false, + } + } + + /// Iterator over all CCX that need translation (cannot reuse results from + /// previous incr. comp.). + pub fn iter_need_trans<'b>(&'b self) -> CrateContextIterator<'b, 'tcx> { + CrateContextIterator { + shared: self.shared, + index: 0, + local_ccxs: &self.local_ccxs[..], + filter_to_previous_work_product_unavail: true, } } @@ -239,24 +256,38 @@ pub struct CrateContextIterator<'a, 'tcx: 'a> { shared: &'a SharedCrateContext<'a, 'tcx>, local_ccxs: &'a [LocalCrateContext<'tcx>], index: usize, + + /// if true, only return results where `previous_work_product` is none + filter_to_previous_work_product_unavail: bool, } impl<'a, 'tcx> Iterator for CrateContextIterator<'a,'tcx> { type Item = CrateContext<'a, 'tcx>; fn next(&mut self) -> Option> { - if self.index >= self.local_ccxs.len() { - return None; - } + loop { + if self.index >= self.local_ccxs.len() { + return None; + } - let index = self.index; - self.index += 1; + let index = self.index; + self.index += 1; - Some(CrateContext { - shared: self.shared, - index: index, - local_ccxs: self.local_ccxs, - }) + let ccx = CrateContext { + shared: self.shared, + index: index, + local_ccxs: self.local_ccxs, + }; + + if + self.filter_to_previous_work_product_unavail && + ccx.previous_work_product().is_some() + { + continue; + } + + return Some(ccx); + } } } @@ -510,6 +541,7 @@ impl<'b, 'tcx> SharedCrateContext<'b, 'tcx> { impl<'tcx> LocalCrateContext<'tcx> { fn new<'a>(shared: &SharedCrateContext<'a, 'tcx>, codegen_unit: CodegenUnit<'tcx>, + previous_work_product: Option, symbol_map: Rc>) -> LocalCrateContext<'tcx> { unsafe { @@ -521,7 +553,7 @@ impl<'tcx> LocalCrateContext<'tcx> { // crashes if the module identifier is same as other symbols // such as a function name in the module. // 1. http://llvm.org/bugs/show_bug.cgi?id=11479 - let llmod_id = format!("{}.rs", codegen_unit.name); + let llmod_id = format!("{}.rs", codegen_unit.name()); let (llcx, llmod) = create_context_and_module(&shared.tcx.sess, &llmod_id[..]); @@ -535,6 +567,7 @@ impl<'tcx> LocalCrateContext<'tcx> { let local_ccx = LocalCrateContext { llmod: llmod, llcx: llcx, + previous_work_product: previous_work_product, codegen_unit: codegen_unit, tn: TypeNames::new(), needs_unwind_cleanup_cache: RefCell::new(FnvHashMap()), @@ -694,6 +727,10 @@ impl<'b, 'tcx> CrateContext<'b, 'tcx> { self.local().llcx } + pub fn previous_work_product(&self) -> Option<&PathBuf> { + self.local().previous_work_product.as_ref() + } + pub fn codegen_unit(&self) -> &CodegenUnit<'tcx> { &self.local().codegen_unit } diff --git a/src/librustc_trans/glue.rs b/src/librustc_trans/glue.rs index ef7d0ea165d60..6bc48546dfa8c 100644 --- a/src/librustc_trans/glue.rs +++ b/src/librustc_trans/glue.rs @@ -239,7 +239,7 @@ fn get_drop_glue_core<'a, 'tcx>(ccx: &CrateContext<'a, 'tcx>, Falling back to on-demand instantiation.", g, TransItem::DropGlue(g).to_raw_string(), - ccx.codegen_unit().name); + ccx.codegen_unit().name()); ccx.stats().n_fallback_instantiations.set(ccx.stats() .n_fallback_instantiations diff --git a/src/librustc_trans/lib.rs b/src/librustc_trans/lib.rs index fa0a1fdc37523..6ac64d3291084 100644 --- a/src/librustc_trans/lib.rs +++ b/src/librustc_trans/lib.rs @@ -37,6 +37,8 @@ #![feature(unicode)] #![feature(question_mark)] +use std::path::PathBuf; + extern crate arena; extern crate flate; extern crate getopts; @@ -133,6 +135,18 @@ mod value; #[derive(Clone)] pub struct ModuleTranslation { pub name: String, + pub symbol_name_hash: u64, + pub source: ModuleSource, +} + +#[derive(Clone)] +pub enum ModuleSource { + Preexisting(PathBuf), + Translated(ModuleLlvm), +} + +#[derive(Copy, Clone)] +pub struct ModuleLlvm { pub llcx: llvm::ContextRef, pub llmod: llvm::ModuleRef, } diff --git a/src/librustc_trans/monomorphize.rs b/src/librustc_trans/monomorphize.rs index 00c0e91103500..96a05f11bfd11 100644 --- a/src/librustc_trans/monomorphize.rs +++ b/src/librustc_trans/monomorphize.rs @@ -52,7 +52,7 @@ pub fn monomorphic_fn<'a, 'tcx>(ccx: &CrateContext<'a, 'tcx>, debug!("leaving monomorphic fn {:?}", instance); return (val, mono_ty); } else { - assert!(!ccx.codegen_unit().items.contains_key(&TransItem::Fn(instance))); + assert!(!ccx.codegen_unit().contains_item(&TransItem::Fn(instance))); } debug!("monomorphic_fn({:?})", instance); diff --git a/src/librustc_trans/partitioning.rs b/src/librustc_trans/partitioning.rs index 8073359ede87e..12df8bd83708c 100644 --- a/src/librustc_trans/partitioning.rs +++ b/src/librustc_trans/partitioning.rs @@ -119,12 +119,15 @@ use collector::InliningMap; use llvm; use monomorphize; +use rustc::dep_graph::{DepNode, WorkProductId}; use rustc::hir::def_id::DefId; use rustc::hir::map::DefPathData; use rustc::session::config::NUMBERED_CODEGEN_UNIT_MARKER; use rustc::ty::TyCtxt; use rustc::ty::item_path::characteristic_def_id_of_type; use std::cmp::Ordering; +use std::hash::{Hash, Hasher, SipHasher}; +use std::sync::Arc; use symbol_map::SymbolMap; use syntax::ast::NodeId; use syntax::parse::token::{self, InternedString}; @@ -140,11 +143,54 @@ pub enum PartitioningStrategy { } pub struct CodegenUnit<'tcx> { - pub name: InternedString, - pub items: FnvHashMap, llvm::Linkage>, + name: InternedString, + items: FnvHashMap, llvm::Linkage>, } impl<'tcx> CodegenUnit<'tcx> { + pub fn new(name: InternedString, + items: FnvHashMap, llvm::Linkage>) + -> Self { + CodegenUnit { + name: name, + items: items, + } + } + + pub fn empty(name: InternedString) -> Self { + Self::new(name, FnvHashMap()) + } + + pub fn contains_item(&self, item: &TransItem<'tcx>) -> bool { + self.items.contains_key(item) + } + + pub fn name(&self) -> &str { + &self.name + } + + pub fn items(&self) -> &FnvHashMap, llvm::Linkage> { + &self.items + } + + pub fn work_product_id(&self) -> Arc { + Arc::new(WorkProductId::PartitionObjectFile(self.name().to_string())) + } + + pub fn work_product_dep_node(&self) -> DepNode { + DepNode::WorkProduct(self.work_product_id()) + } + + pub fn compute_symbol_name_hash(&self, tcx: TyCtxt, symbol_map: &SymbolMap) -> u64 { + let mut state = SipHasher::new(); + let all_items = self.items_in_deterministic_order(tcx, symbol_map); + for (item, _) in all_items { + let symbol_name = symbol_map.get(item).unwrap(); + symbol_name.hash(&mut state); + } + state.finish() + } + pub fn items_in_deterministic_order(&self, tcx: TyCtxt, symbol_map: &SymbolMap) @@ -277,10 +323,7 @@ fn place_root_translation_items<'a, 'tcx, I>(tcx: TyCtxt<'a, 'tcx, 'tcx>, }; let make_codegen_unit = || { - CodegenUnit { - name: codegen_unit_name.clone(), - items: FnvHashMap(), - } + CodegenUnit::empty(codegen_unit_name.clone()) }; let mut codegen_unit = codegen_units.entry(codegen_unit_name.clone()) @@ -319,10 +362,8 @@ fn place_root_translation_items<'a, 'tcx, I>(tcx: TyCtxt<'a, 'tcx, 'tcx>, if codegen_units.is_empty() { let codegen_unit_name = InternedString::new(FALLBACK_CODEGEN_UNIT); codegen_units.entry(codegen_unit_name.clone()) - .or_insert_with(|| CodegenUnit { - name: codegen_unit_name.clone(), - items: FnvHashMap(), - }); + .or_insert_with(|| CodegenUnit::new(codegen_unit_name.clone(), + FnvHashMap())); } PreInliningPartitioning { @@ -362,10 +403,8 @@ fn merge_codegen_units<'tcx>(initial_partitioning: &mut PreInliningPartitioning< // we reach the target count while codegen_units.len() < target_cgu_count { let index = codegen_units.len(); - codegen_units.push(CodegenUnit { - name: numbered_codegen_unit_name(crate_name, index), - items: FnvHashMap() - }); + codegen_units.push( + CodegenUnit::empty(numbered_codegen_unit_name(crate_name, index))); } } @@ -381,10 +420,8 @@ fn place_inlined_translation_items<'tcx>(initial_partitioning: PreInliningPartit follow_inlining(*root, inlining_map, &mut reachable); } - let mut new_codegen_unit = CodegenUnit { - name: codegen_unit.name.clone(), - items: FnvHashMap(), - }; + let mut new_codegen_unit = + CodegenUnit::empty(codegen_unit.name.clone()); // Add all translation items that are not already there for trans_item in reachable { @@ -560,10 +597,9 @@ fn single_codegen_unit<'a, 'tcx, I>(tcx: TyCtxt<'a, 'tcx, 'tcx>, items.insert(trans_item, linkage); } - CodegenUnit { - name: numbered_codegen_unit_name(&tcx.crate_name[..], 0), - items: items - } + CodegenUnit::new( + numbered_codegen_unit_name(&tcx.crate_name[..], 0), + items) } fn numbered_codegen_unit_name(crate_name: &str, index: usize) -> InternedString { diff --git a/src/librustc_trans/trans_item.rs b/src/librustc_trans/trans_item.rs index fc95d208f32cc..3afedb4909067 100644 --- a/src/librustc_trans/trans_item.rs +++ b/src/librustc_trans/trans_item.rs @@ -23,12 +23,12 @@ use glue::DropGlueKind; use llvm; use monomorphize::{self, Instance}; use inline; +use rustc::dep_graph::DepNode; use rustc::hir; use rustc::hir::map as hir_map; use rustc::hir::def_id::DefId; use rustc::ty::{self, Ty, TyCtxt, TypeFoldable}; use rustc::ty::subst; -use rustc::dep_graph::DepNode; use rustc_const_eval::fatal_const_eval_err; use std::hash::{Hash, Hasher}; use syntax::ast::{self, NodeId}; @@ -68,16 +68,27 @@ impl<'tcx> Hash for TransItem<'tcx> { impl<'a, 'tcx> TransItem<'tcx> { pub fn define(&self, ccx: &CrateContext<'a, 'tcx>) { - debug!("BEGIN IMPLEMENTING '{} ({})' in cgu {}", self.to_string(ccx.tcx()), self.to_raw_string(), - ccx.codegen_unit().name); + ccx.codegen_unit().name()); + + // (*) This code executes in the context of a dep-node for the + // entire CGU. In some cases, we introduce dep-nodes for + // particular items that we are translating (these nodes will + // have read edges coming into the CGU node). These smaller + // nodes are not needed for correctness -- we always + // invalidate an entire CGU at a time -- but they enable + // finer-grained testing, since you can write tests that check + // that the incoming edges to a particular fn are from a + // particular set. self.register_reads(ccx); match *self { TransItem::Static(node_id) => { + let def_id = ccx.tcx().map.local_def_id(node_id); + let _task = ccx.tcx().dep_graph.in_task(DepNode::TransCrateItem(def_id)); // (*) let item = ccx.tcx().map.expect_item(node_id); if let hir::ItemStatic(_, m, ref expr) = item.node { match consts::trans_static(&ccx, m, expr, item.id, &item.attrs) { @@ -93,6 +104,13 @@ impl<'a, 'tcx> TransItem<'tcx> { } } TransItem::Fn(instance) => { + let _task; + + if instance.def.is_local() { + _task = ccx.tcx().dep_graph.in_task( + DepNode::TransCrateItem(instance.def)); // (*) + } + base::trans_instance(&ccx, instance); } TransItem::DropGlue(dg) => { @@ -103,7 +121,7 @@ impl<'a, 'tcx> TransItem<'tcx> { debug!("END IMPLEMENTING '{} ({})' in cgu {}", self.to_string(ccx.tcx()), self.to_raw_string(), - ccx.codegen_unit().name); + ccx.codegen_unit().name()); } /// If necessary, creates a subtask for trans'ing a particular item and registers reads on @@ -152,7 +170,7 @@ impl<'a, 'tcx> TransItem<'tcx> { debug!("BEGIN PREDEFINING '{} ({})' in cgu {}", self.to_string(ccx.tcx()), self.to_raw_string(), - ccx.codegen_unit().name); + ccx.codegen_unit().name()); let symbol_name = ccx.symbol_map() .get_or_compute(ccx.shared(), *self); @@ -174,7 +192,7 @@ impl<'a, 'tcx> TransItem<'tcx> { debug!("END PREDEFINING '{} ({})' in cgu {}", self.to_string(ccx.tcx()), self.to_raw_string(), - ccx.codegen_unit().name); + ccx.codegen_unit().name()); } fn predefine_static(ccx: &CrateContext<'a, 'tcx>, diff --git a/src/test/run-make/execution-engine/test.rs b/src/test/run-make/execution-engine/test.rs index 2e90b51843263..b58295d47f21d 100644 --- a/src/test/run-make/execution-engine/test.rs +++ b/src/test/run-make/execution-engine/test.rs @@ -20,6 +20,7 @@ extern crate rustc_metadata; extern crate rustc_resolve; extern crate rustc_errors; extern crate rustc_errors as errors; +extern crate rustc_trans; #[macro_use] extern crate syntax; use std::ffi::{CStr, CString}; @@ -37,6 +38,7 @@ use rustc::session::build_session; use rustc_driver::{driver, abort_on_err}; use rustc_resolve::MakeGlobMap; use rustc_metadata::cstore::CStore; +use rustc_trans::ModuleSource; use libc::c_void; use rustc_errors::registry::Registry; @@ -261,7 +263,10 @@ fn compile_program(input: &str, sysroot: PathBuf) .filter_map(|(_, p)| p).collect(); assert_eq!(trans.modules.len(), 1); - let llmod = trans.modules[0].llmod; + let llmod = match trans.modules[0].source { + ModuleSource::Preexisting(_) => unimplemented!(), + ModuleSource::Translated(llvm) => llvm.llmod, + }; // Workaround because raw pointers do not impl Send let modp = llmod as usize; diff --git a/src/test/run-make/llvm-phase/test.rs b/src/test/run-make/llvm-phase/test.rs index 402b5ed83556c..19e410fef5388 100644 --- a/src/test/run-make/llvm-phase/test.rs +++ b/src/test/run-make/llvm-phase/test.rs @@ -13,11 +13,13 @@ extern crate rustc; extern crate rustc_driver; extern crate rustc_llvm; +extern crate rustc_trans; #[macro_use] extern crate syntax; extern crate getopts; use rustc_driver::{CompilerCalls, Compilation}; use rustc_driver::driver::CompileController; +use rustc_trans::ModuleSource; use rustc::session::Session; use syntax::codemap::FileLoader; use std::io; @@ -51,7 +53,10 @@ impl<'a> CompilerCalls<'a> for JitCalls { state.session.abort_if_errors(); let trans = state.trans.unwrap(); assert_eq!(trans.modules.len(), 1); - let rs_llmod = trans.modules[0].llmod; + let rs_llmod = match trans.modules[0].source { + ModuleSource::Preexisting(_) => unimplemented!(), + ModuleSource::Translated(llvm) => llvm.llmod, + }; unsafe { rustc_llvm::LLVMDumpModule(rs_llmod) }; }); cc From 08a72d20c9c97ab7b1d3b023d6cc0caddcd12f77 Mon Sep 17 00:00:00 2001 From: Niko Matsakis Date: Thu, 21 Jul 2016 12:50:15 -0400 Subject: [PATCH 5/9] Add a testing mechanism and a simple spike test --- src/librustc_trans/assert_module_sources.rs | 149 ++++++++++++++++++++ src/librustc_trans/base.rs | 3 + src/librustc_trans/lib.rs | 1 + src/libsyntax/feature_gate.rs | 10 ++ src/test/incremental/spike-neg1.rs | 67 +++++++++ src/test/incremental/spike-neg2.rs | 67 +++++++++ src/test/incremental/spike.rs | 63 +++++++++ 7 files changed, 360 insertions(+) create mode 100644 src/librustc_trans/assert_module_sources.rs create mode 100644 src/test/incremental/spike-neg1.rs create mode 100644 src/test/incremental/spike-neg2.rs create mode 100644 src/test/incremental/spike.rs diff --git a/src/librustc_trans/assert_module_sources.rs b/src/librustc_trans/assert_module_sources.rs new file mode 100644 index 0000000000000..e0532e7476f51 --- /dev/null +++ b/src/librustc_trans/assert_module_sources.rs @@ -0,0 +1,149 @@ +// Copyright 2012-2015 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +//! This pass is only used for UNIT TESTS related to incremental +//! compilation. It tests whether a particular `.o` file will be re-used +//! from a previous compilation or whether it must be regenerated. +//! +//! The user adds annotations to the crate of the following form: +//! +//! ``` +//! #![rustc_partition_reused(module="spike", cfg="rpass2")] +//! #![rustc_partition_translated(module="spike-x", cfg="rpass2")] +//! ``` +//! +//! The first indicates (in the cfg `rpass2`) that `spike.o` will be +//! reused, the second that `spike-x.o` will be recreated. If these +//! annotations are inaccurate, errors are reported. +//! +//! The reason that we use `cfg=...` and not `#[cfg_attr]` is so that +//! the HIR doesn't change as a result of the annotations, which might +//! perturb the reuse results. + +use rustc::ty::TyCtxt; +use syntax::ast; +use syntax::attr::AttrMetaMethods; +use syntax::parse::token::InternedString; + +use {ModuleSource, ModuleTranslation}; + +const PARTITION_REUSED: &'static str = "rustc_partition_reused"; +const PARTITION_TRANSLATED: &'static str = "rustc_partition_translated"; + +const MODULE: &'static str = "module"; +const CFG: &'static str = "cfg"; + +#[derive(Debug, PartialEq)] +enum Disposition { Reused, Translated } + +pub fn assert_module_sources<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, + modules: &[ModuleTranslation]) { + let _ignore = tcx.dep_graph.in_ignore(); + + if tcx.sess.opts.incremental.is_none() { + return; + } + + let ams = AssertModuleSource { tcx: tcx, modules: modules }; + for attr in &tcx.map.krate().attrs { + ams.check_attr(attr); + } +} + +struct AssertModuleSource<'a, 'tcx: 'a> { + tcx: TyCtxt<'a, 'tcx, 'tcx>, + modules: &'a [ModuleTranslation], +} + +impl<'a, 'tcx> AssertModuleSource<'a, 'tcx> { + fn check_attr(&self, attr: &ast::Attribute) { + let disposition = if attr.check_name(PARTITION_REUSED) { + Disposition::Reused + } else if attr.check_name(PARTITION_TRANSLATED) { + Disposition::Translated + } else { + return; + }; + + if !self.check_config(attr) { + debug!("check_attr: config does not match, ignoring attr"); + return; + } + + let mname = self.field(attr, MODULE); + let mtrans = self.modules.iter().find(|mtrans| &mtrans.name[..] == &mname[..]); + let mtrans = match mtrans { + Some(m) => m, + None => { + debug!("module name `{}` not found amongst:", mname); + for mtrans in self.modules { + debug!("module named `{}` with disposition {:?}", + mtrans.name, + self.disposition(mtrans)); + } + + self.tcx.sess.span_err( + attr.span, + &format!("no module named `{}`", mname)); + return; + } + }; + + let mtrans_disposition = self.disposition(mtrans); + if disposition != mtrans_disposition { + self.tcx.sess.span_err( + attr.span, + &format!("expected module named `{}` to be {:?} but is {:?}", + mname, + disposition, + mtrans_disposition)); + } + } + + fn disposition(&self, mtrans: &ModuleTranslation) -> Disposition { + match mtrans.source { + ModuleSource::Preexisting(_) => Disposition::Reused, + ModuleSource::Translated(_) => Disposition::Translated, + } + } + + fn field(&self, attr: &ast::Attribute, name: &str) -> InternedString { + for item in attr.meta_item_list().unwrap_or(&[]) { + if item.check_name(name) { + if let Some(value) = item.value_str() { + return value; + } else { + self.tcx.sess.span_fatal( + item.span, + &format!("associated value expected for `{}`", name)); + } + } + } + + self.tcx.sess.span_fatal( + attr.span, + &format!("no field `{}`", name)); + } + + /// Scan for a `cfg="foo"` attribute and check whether we have a + /// cfg flag called `foo`. + fn check_config(&self, attr: &ast::Attribute) -> bool { + let config = &self.tcx.map.krate().config; + let value = self.field(attr, CFG); + debug!("check_config(config={:?}, value={:?})", config, value); + if config.iter().any(|c| c.check_name(&value[..])) { + debug!("check_config: matched"); + return true; + } + debug!("check_config: no match found"); + return false; + } + +} diff --git a/src/librustc_trans/base.rs b/src/librustc_trans/base.rs index a77ababaa6355..69a884431359d 100644 --- a/src/librustc_trans/base.rs +++ b/src/librustc_trans/base.rs @@ -30,6 +30,7 @@ use super::ModuleLlvm; use super::ModuleSource; use super::ModuleTranslation; +use assert_module_sources; use back::link; use back::linker::LinkerInfo; use llvm::{BasicBlockRef, Linkage, ValueRef, Vector, get_param}; @@ -2558,6 +2559,8 @@ pub fn trans_crate<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, }) .collect(); + assert_module_sources::assert_module_sources(tcx, &modules); + // Skip crate items and just output metadata in -Z no-trans mode. if tcx.sess.opts.no_trans { let linker_info = LinkerInfo::new(&shared_ccx, &[]); diff --git a/src/librustc_trans/lib.rs b/src/librustc_trans/lib.rs index 6ac64d3291084..67475081caec7 100644 --- a/src/librustc_trans/lib.rs +++ b/src/librustc_trans/lib.rs @@ -88,6 +88,7 @@ mod macros; mod abi; mod adt; mod asm; +mod assert_module_sources; mod attributes; mod base; mod basic_block; diff --git a/src/libsyntax/feature_gate.rs b/src/libsyntax/feature_gate.rs index 27485ee65fcc0..f80f25a2e776d 100644 --- a/src/libsyntax/feature_gate.rs +++ b/src/libsyntax/feature_gate.rs @@ -481,6 +481,16 @@ pub const KNOWN_ATTRIBUTES: &'static [(&'static str, AttributeType, AttributeGat is just used for rustc unit tests \ and will never be stable", cfg_fn!(rustc_attrs))), + ("rustc_partition_reused", Whitelisted, Gated("rustc_attrs", + "this attribute \ + is just used for rustc unit tests \ + and will never be stable", + cfg_fn!(rustc_attrs))), + ("rustc_partition_translated", Whitelisted, Gated("rustc_attrs", + "this attribute \ + is just used for rustc unit tests \ + and will never be stable", + cfg_fn!(rustc_attrs))), ("rustc_symbol_name", Whitelisted, Gated("rustc_attrs", "internal rustc attributes will never be stable", cfg_fn!(rustc_attrs))), diff --git a/src/test/incremental/spike-neg1.rs b/src/test/incremental/spike-neg1.rs new file mode 100644 index 0000000000000..e84906d12d0ce --- /dev/null +++ b/src/test/incremental/spike-neg1.rs @@ -0,0 +1,67 @@ +// Copyright 2014 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +// A variant of the first "spike" test that serves to test the +// `rustc_partition_reused` and `rustc_partition_translated` tests. +// Here we change and say that the `x` module will be reused (when in +// fact it will not), and then indicate that the test itself +// should-fail (because an error will be reported, and hence the +// revision rpass2 will not compile, despite being named rpass). + +// revisions:rpass1 rpass2 +// should-fail + +#![feature(rustc_attrs)] + +#![rustc_partition_reused(module="spike_neg1", cfg="rpass2")] +#![rustc_partition_reused(module="spike_neg1-x", cfg="rpass2")] // this is wrong! +#![rustc_partition_reused(module="spike_neg1-y", cfg="rpass2")] + +mod x { + pub struct X { + x: u32, y: u32, + } + + #[cfg(rpass1)] + fn make() -> X { + X { x: 22, y: 0 } + } + + #[cfg(rpass2)] + fn make() -> X { + X { x: 11, y: 11 } + } + + #[rustc_dirty(label="TypeckItemBody", cfg="rpass2")] + #[rustc_clean(label="ItemSignature", cfg="rpass2")] + pub fn new() -> X { + make() + } + + #[rustc_clean(label="TypeckItemBody", cfg="rpass2")] + #[rustc_clean(label="ItemSignature", cfg="rpass2")] + pub fn sum(x: &X) -> u32 { + x.x + x.y + } +} + +mod y { + use x; + + #[rustc_clean(label="TypeckItemBody", cfg="rpass2")] + pub fn assert_sum() -> bool { + let x = x::new(); + x::sum(&x) == 22 + } +} + +pub fn main() { + y::assert_sum(); +} diff --git a/src/test/incremental/spike-neg2.rs b/src/test/incremental/spike-neg2.rs new file mode 100644 index 0000000000000..40f4b4f0c44ce --- /dev/null +++ b/src/test/incremental/spike-neg2.rs @@ -0,0 +1,67 @@ +// Copyright 2014 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +// A variant of the first "spike" test that serves to test the +// `rustc_partition_reused` and `rustc_partition_translated` tests. +// Here we change and say that the `y` module will be translated (when +// in fact it will not), and then indicate that the test itself +// should-fail (because an error will be reported, and hence the +// revision rpass2 will not compile, despite being named rpass). + +// revisions:rpass1 rpass2 +// should-fail + +#![feature(rustc_attrs)] + +#![rustc_partition_reused(module="spike_neg2", cfg="rpass2")] +#![rustc_partition_translated(module="spike_neg2-x", cfg="rpass2")] +#![rustc_partition_translated(module="spike_neg2-y", cfg="rpass2")] // this is wrong! + +mod x { + pub struct X { + x: u32, y: u32, + } + + #[cfg(rpass1)] + fn make() -> X { + X { x: 22, y: 0 } + } + + #[cfg(rpass2)] + fn make() -> X { + X { x: 11, y: 11 } + } + + #[rustc_dirty(label="TypeckItemBody", cfg="rpass2")] + #[rustc_clean(label="ItemSignature", cfg="rpass2")] + pub fn new() -> X { + make() + } + + #[rustc_clean(label="TypeckItemBody", cfg="rpass2")] + #[rustc_clean(label="ItemSignature", cfg="rpass2")] + pub fn sum(x: &X) -> u32 { + x.x + x.y + } +} + +mod y { + use x; + + #[rustc_clean(label="TypeckItemBody", cfg="rpass2")] + pub fn assert_sum() -> bool { + let x = x::new(); + x::sum(&x) == 22 + } +} + +pub fn main() { + y::assert_sum(); +} diff --git a/src/test/incremental/spike.rs b/src/test/incremental/spike.rs new file mode 100644 index 0000000000000..68af20d419151 --- /dev/null +++ b/src/test/incremental/spike.rs @@ -0,0 +1,63 @@ +// Copyright 2014 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +// A first "spike" for incremental compilation: here, we change the +// content of the `make` function, and we find that we can reuse the +// `y` module entirely (but not the `x` module). + +// revisions:rpass1 rpass2 + +#![feature(rustc_attrs)] + +#![rustc_partition_reused(module="spike", cfg="rpass2")] +#![rustc_partition_translated(module="spike-x", cfg="rpass2")] +#![rustc_partition_reused(module="spike-y", cfg="rpass2")] + +mod x { + pub struct X { + x: u32, y: u32, + } + + #[cfg(rpass1)] + fn make() -> X { + X { x: 22, y: 0 } + } + + #[cfg(rpass2)] + fn make() -> X { + X { x: 11, y: 11 } + } + + #[rustc_dirty(label="TypeckItemBody", cfg="rpass2")] + #[rustc_clean(label="ItemSignature", cfg="rpass2")] + pub fn new() -> X { + make() + } + + #[rustc_clean(label="TypeckItemBody", cfg="rpass2")] + #[rustc_clean(label="ItemSignature", cfg="rpass2")] + pub fn sum(x: &X) -> u32 { + x.x + x.y + } +} + +mod y { + use x; + + #[rustc_clean(label="TypeckItemBody", cfg="rpass2")] + pub fn assert_sum() -> bool { + let x = x::new(); + x::sum(&x) == 22 + } +} + +pub fn main() { + y::assert_sum(); +} From ceeb158e0a04c3a73a4c5014609020ee628b4c06 Mon Sep 17 00:00:00 2001 From: Niko Matsakis Date: Fri, 22 Jul 2016 10:39:30 -0400 Subject: [PATCH 6/9] Address mw nits --- src/librustc/dep_graph/graph.rs | 10 ++++---- src/librustc/util/fs.rs | 14 +++++++++++ src/librustc_driver/driver.rs | 24 +++++++++---------- src/librustc_incremental/persist/load.rs | 11 +++++---- .../persist/work_product.rs | 6 ++--- src/librustc_trans/back/write.rs | 12 +--------- src/librustc_trans/base.rs | 13 ++++++++-- src/librustc_trans/partitioning.rs | 3 +-- src/librustc_trans/trans_item.rs | 8 ++----- src/test/incremental/spike-neg1.rs | 5 ---- src/test/incremental/spike-neg2.rs | 5 ---- 11 files changed, 54 insertions(+), 57 deletions(-) diff --git a/src/librustc/dep_graph/graph.rs b/src/librustc/dep_graph/graph.rs index ab7013df33f14..8691894b32513 100644 --- a/src/librustc/dep_graph/graph.rs +++ b/src/librustc/dep_graph/graph.rs @@ -25,17 +25,17 @@ pub struct DepGraph { } struct DepGraphData { - /// we send messages to the thread to let it build up the dep-graph - /// from the current run + /// We send messages to the thread to let it build up the dep-graph + /// from the current run. thread: DepGraphThreadData, - /// when we load, there may be `.o` files, cached mir, or other such + /// When we load, there may be `.o` files, cached mir, or other such /// things available to us. If we find that they are not dirty, we /// load the path to the file storing those work-products here into /// this map. We can later look for and extract that data. previous_work_products: RefCell, WorkProduct>>, - /// work-products that we generate in this run + /// Work-products that we generate in this run. work_products: RefCell, WorkProduct>>, } @@ -132,7 +132,7 @@ impl DepGraph { /// Each work product is associated with a dep-node, representing the /// process that produced the work-product. If that dep-node is found /// to be dirty when we load up, then we will delete the work-product -/// at load time. If the work-product is found to be clean, the we +/// at load time. If the work-product is found to be clean, then we /// will keep a record in the `previous_work_products` list. /// /// In addition, work products have an associated hash. This hash is diff --git a/src/librustc/util/fs.rs b/src/librustc/util/fs.rs index 4936e049ef2ee..f4e1c06090e59 100644 --- a/src/librustc/util/fs.rs +++ b/src/librustc/util/fs.rs @@ -10,6 +10,8 @@ use std::path::{self, Path, PathBuf}; use std::ffi::OsString; +use std::fs; +use std::io; // Unfortunately, on windows, it looks like msvcrt.dll is silently translating // verbatim paths under the hood to non-verbatim paths! This manifests itself as @@ -53,3 +55,15 @@ pub fn fix_windows_verbatim_for_gcc(p: &Path) -> PathBuf { _ => p.to_path_buf(), } } + +/// Copy `p` into `q`, preferring to use hard-linking if possible. If +/// `q` already exists, it is removed first. +pub fn link_or_copy, Q: AsRef>(p: P, q: Q) -> io::Result<()> { + let p = p.as_ref(); + let q = q.as_ref(); + if q.exists() { + try!(fs::remove_file(&q)); + } + fs::hard_link(p, q) + .or_else(|_| fs::copy(p, q).map(|_| ())) +} diff --git a/src/librustc_driver/driver.rs b/src/librustc_driver/driver.rs index a48ff2533485c..f172f38b80968 100644 --- a/src/librustc_driver/driver.rs +++ b/src/librustc_driver/driver.rs @@ -88,7 +88,7 @@ pub fn compile_input(sess: &Session, // We need nested scopes here, because the intermediate results can keep // large chunks of memory alive and we want to free them as soon as // possible to keep the peak memory usage low - let (outputs, trans, id) = { + let (outputs, trans, crate_name) = { let krate = match phase_1_parse_input(sess, cfg, input) { Ok(krate) => krate, Err(mut parse_error) => { @@ -113,13 +113,13 @@ pub fn compile_input(sess: &Session, }; let outputs = build_output_filenames(input, outdir, output, &krate.attrs, sess); - let id = link::find_crate_name(Some(sess), &krate.attrs, input); + let crate_name = link::find_crate_name(Some(sess), &krate.attrs, input); let ExpansionResult { expanded_crate, defs, analysis, resolutions, mut hir_forest } = { phase_2_configure_and_expand( - sess, &cstore, krate, &id, addl_plugins, control.make_glob_map, + sess, &cstore, krate, &crate_name, addl_plugins, control.make_glob_map, |expanded_crate| { let mut state = CompileState::state_after_expand( - input, sess, outdir, output, &cstore, expanded_crate, &id, + input, sess, outdir, output, &cstore, expanded_crate, &crate_name, ); controller_entry_point!(after_expand, sess, state, Ok(())); Ok(()) @@ -127,7 +127,7 @@ pub fn compile_input(sess: &Session, )? }; - write_out_deps(sess, &outputs, &id); + write_out_deps(sess, &outputs, &crate_name); let arenas = ty::CtxtArenas::new(); @@ -151,7 +151,7 @@ pub fn compile_input(sess: &Session, &resolutions, &expanded_crate, &hir_map.krate(), - &id), + &crate_name), Ok(())); } @@ -171,7 +171,7 @@ pub fn compile_input(sess: &Session, analysis, resolutions, &arenas, - &id, + &crate_name, |tcx, mir_map, analysis, result| { { // Eventually, we will want to track plugins. @@ -186,7 +186,7 @@ pub fn compile_input(sess: &Session, &analysis, mir_map.as_ref(), tcx, - &id); + &crate_name); (control.after_analysis.callback)(&mut state); if control.after_analysis.stop == Compilation::Stop { @@ -212,11 +212,11 @@ pub fn compile_input(sess: &Session, // Discard interned strings as they are no longer required. token::clear_ident_interner(); - Ok((outputs, trans, id.clone())) + Ok((outputs, trans, crate_name.clone())) })?? }; - let phase5_result = phase_5_run_llvm_passes(sess, &id, &trans, &outputs); + let phase5_result = phase_5_run_llvm_passes(sess, &crate_name, &trans, &outputs); controller_entry_point!(after_llvm, sess, @@ -1069,14 +1069,14 @@ fn escape_dep_filename(filename: &str) -> String { filename.replace(" ", "\\ ") } -fn write_out_deps(sess: &Session, outputs: &OutputFilenames, id: &str) { +fn write_out_deps(sess: &Session, outputs: &OutputFilenames, crate_name: &str) { let mut out_filenames = Vec::new(); for output_type in sess.opts.output_types.keys() { let file = outputs.path(*output_type); match *output_type { OutputType::Exe => { for output in sess.crate_types.borrow().iter() { - let p = link::filename_for_input(sess, *output, id, outputs); + let p = link::filename_for_input(sess, *output, crate_name, outputs); out_filenames.push(p); } } diff --git a/src/librustc_incremental/persist/load.rs b/src/librustc_incremental/persist/load.rs index 9fef2285aa7eb..6b856459aabfe 100644 --- a/src/librustc_incremental/persist/load.rs +++ b/src/librustc_incremental/persist/load.rs @@ -63,10 +63,13 @@ fn load_dep_graph_if_exists<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>) { match decode_dep_graph(tcx, &dep_graph_data, &work_products_data) { Ok(()) => return, - Err(err) => bug!("decoding error in dep-graph from `{}` and `{}`: {}", + Err(err) => { + tcx.sess.warn( + &format!("decoding error in dep-graph from `{}` and `{}`: {}", dep_graph_path.display(), work_products_path.display(), - err), + err)); + } } } @@ -94,9 +97,7 @@ fn load_data(sess: &Session, path: &Path) -> Option> { } /// Decode the dep graph and load the edges/nodes that are still clean -/// into `tcx.dep_graph`. On success, returns a hashset containing all -/// the paths of work-products from clean nodes (any work-products not -/// in this set can be deleted). +/// into `tcx.dep_graph`. pub fn decode_dep_graph<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, dep_graph_data: &[u8], work_products_data: &[u8]) diff --git a/src/librustc_incremental/persist/work_product.rs b/src/librustc_incremental/persist/work_product.rs index 01ac3f6c391d9..7695291bf620b 100644 --- a/src/librustc_incremental/persist/work_product.rs +++ b/src/librustc_incremental/persist/work_product.rs @@ -13,6 +13,7 @@ use persist::util::*; use rustc::dep_graph::{WorkProduct, WorkProductId}; use rustc::session::Session; +use rustc::util::fs::link_or_copy; use std::fs; use std::path::Path; use std::sync::Arc; @@ -39,10 +40,7 @@ pub fn save_trans_partition(sess: &Session, let _ = fs::remove_file(&path_in_incr_dir); } - match - fs::hard_link(path_to_obj_file, &path_in_incr_dir) - .or_else(|_| fs::copy(path_to_obj_file, &path_in_incr_dir).map(|_| ())) - { + match link_or_copy(path_to_obj_file, &path_in_incr_dir) { Ok(_) => { let work_product = WorkProduct { input_hash: partition_hash, diff --git a/src/librustc_trans/back/write.rs b/src/librustc_trans/back/write.rs index 70925218781bd..08d7b531c2f1b 100644 --- a/src/librustc_trans/back/write.rs +++ b/src/librustc_trans/back/write.rs @@ -20,6 +20,7 @@ use llvm::SMDiagnosticRef; use {CrateTranslation, ModuleLlvm, ModuleSource, ModuleTranslation}; use util::common::time; use util::common::path2cstr; +use util::fs::link_or_copy; use errors::{self, Handler, Level, DiagnosticBuilder}; use errors::emitter::Emitter; use syntax_pos::MultiSpan; @@ -27,7 +28,6 @@ use syntax_pos::MultiSpan; use std::collections::HashMap; use std::ffi::{CStr, CString}; use std::fs; -use std::io; use std::path::{Path, PathBuf}; use std::str; use std::sync::{Arc, Mutex}; @@ -929,16 +929,6 @@ fn build_work_item(sess: &Session, } } -fn link_or_copy, Q: AsRef>(p: P, q: Q) -> io::Result<()> { - let p = p.as_ref(); - let q = q.as_ref(); - if q.exists() { - try!(fs::remove_file(&q)); - } - fs::hard_link(p, q) - .or_else(|_| fs::copy(p, q).map(|_| ())) -} - fn execute_work_item(cgcx: &CodegenContext, work_item: WorkItem) { unsafe { diff --git a/src/librustc_trans/base.rs b/src/librustc_trans/base.rs index 69a884431359d..e73c4f41c9412 100644 --- a/src/librustc_trans/base.rs +++ b/src/librustc_trans/base.rs @@ -2262,12 +2262,20 @@ fn write_metadata(cx: &SharedCrateContext, /// Find any symbols that are defined in one compilation unit, but not declared /// in any other compilation unit. Give these symbols internal linkage. -fn internalize_symbols<'a, 'tcx>(ccxs: &CrateContextList<'a, 'tcx>, +fn internalize_symbols<'a, 'tcx>(sess: &Session, + ccxs: &CrateContextList<'a, 'tcx>, symbol_map: &SymbolMap<'tcx>, reachable: &FnvHashSet<&str>) { let scx = ccxs.shared(); let tcx = scx.tcx(); + // In incr. comp. mode, we can't necessarily see all refs since we + // don't generate LLVM IR for reused modules, so skip this + // step. Later we should get smarter. + if sess.opts.debugging_opts.incremental.is_some() { + return; + } + // 'unsafe' because we are holding on to CStr's from the LLVM module within // this block. unsafe { @@ -2682,7 +2690,8 @@ pub fn trans_crate<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, } time(shared_ccx.sess().time_passes(), "internalize symbols", || { - internalize_symbols(&crate_context_list, + internalize_symbols(sess, + &crate_context_list, &symbol_map, &reachable_symbols.iter() .map(|s| &s[..]) diff --git a/src/librustc_trans/partitioning.rs b/src/librustc_trans/partitioning.rs index 12df8bd83708c..32bcbf9f7562b 100644 --- a/src/librustc_trans/partitioning.rs +++ b/src/librustc_trans/partitioning.rs @@ -362,8 +362,7 @@ fn place_root_translation_items<'a, 'tcx, I>(tcx: TyCtxt<'a, 'tcx, 'tcx>, if codegen_units.is_empty() { let codegen_unit_name = InternedString::new(FALLBACK_CODEGEN_UNIT); codegen_units.entry(codegen_unit_name.clone()) - .or_insert_with(|| CodegenUnit::new(codegen_unit_name.clone(), - FnvHashMap())); + .or_insert_with(|| CodegenUnit::empty(codegen_unit_name.clone())); } PreInliningPartitioning { diff --git a/src/librustc_trans/trans_item.rs b/src/librustc_trans/trans_item.rs index 3afedb4909067..fc2758e50f2ce 100644 --- a/src/librustc_trans/trans_item.rs +++ b/src/librustc_trans/trans_item.rs @@ -104,12 +104,8 @@ impl<'a, 'tcx> TransItem<'tcx> { } } TransItem::Fn(instance) => { - let _task; - - if instance.def.is_local() { - _task = ccx.tcx().dep_graph.in_task( - DepNode::TransCrateItem(instance.def)); // (*) - } + let _task = ccx.tcx().dep_graph.in_task( + DepNode::TransCrateItem(instance.def)); // (*) base::trans_instance(&ccx, instance); } diff --git a/src/test/incremental/spike-neg1.rs b/src/test/incremental/spike-neg1.rs index e84906d12d0ce..b00c68a184ed0 100644 --- a/src/test/incremental/spike-neg1.rs +++ b/src/test/incremental/spike-neg1.rs @@ -39,14 +39,10 @@ mod x { X { x: 11, y: 11 } } - #[rustc_dirty(label="TypeckItemBody", cfg="rpass2")] - #[rustc_clean(label="ItemSignature", cfg="rpass2")] pub fn new() -> X { make() } - #[rustc_clean(label="TypeckItemBody", cfg="rpass2")] - #[rustc_clean(label="ItemSignature", cfg="rpass2")] pub fn sum(x: &X) -> u32 { x.x + x.y } @@ -55,7 +51,6 @@ mod x { mod y { use x; - #[rustc_clean(label="TypeckItemBody", cfg="rpass2")] pub fn assert_sum() -> bool { let x = x::new(); x::sum(&x) == 22 diff --git a/src/test/incremental/spike-neg2.rs b/src/test/incremental/spike-neg2.rs index 40f4b4f0c44ce..472d11d7f9028 100644 --- a/src/test/incremental/spike-neg2.rs +++ b/src/test/incremental/spike-neg2.rs @@ -39,14 +39,10 @@ mod x { X { x: 11, y: 11 } } - #[rustc_dirty(label="TypeckItemBody", cfg="rpass2")] - #[rustc_clean(label="ItemSignature", cfg="rpass2")] pub fn new() -> X { make() } - #[rustc_clean(label="TypeckItemBody", cfg="rpass2")] - #[rustc_clean(label="ItemSignature", cfg="rpass2")] pub fn sum(x: &X) -> u32 { x.x + x.y } @@ -55,7 +51,6 @@ mod x { mod y { use x; - #[rustc_clean(label="TypeckItemBody", cfg="rpass2")] pub fn assert_sum() -> bool { let x = x::new(); x::sum(&x) == 22 From 2f9fff21911a3e419b21e56dba145bf0deab6f81 Mon Sep 17 00:00:00 2001 From: Niko Matsakis Date: Mon, 25 Jul 2016 10:51:14 -0400 Subject: [PATCH 7/9] Keep multiple files per work-product In the older version, a `.o` and ` .bc` file were separate work-products. This newer version keeps, for each codegen-unit, a set of files of different kinds. We assume that if any kinds are available then all the kinds we need are available, since the precise set of switches will depend on attributes and command-line switches. Should probably test this: the effect of changing attributes in particular might not be successfully tracked? --- src/librustc/dep_graph/dep_node.rs | 4 +- src/librustc/dep_graph/graph.rs | 7 +- src/librustc/session/config.rs | 23 ++++--- src/librustc_incremental/persist/load.rs | 31 ++++++--- .../persist/work_product.rs | 68 ++++++++++--------- src/librustc_trans/back/write.rs | 56 ++++++++++----- src/librustc_trans/base.rs | 8 +-- src/librustc_trans/context.rs | 15 ++-- src/librustc_trans/lib.rs | 12 +++- src/librustc_trans/partitioning.rs | 7 +- .../rlib_cross_crate/auxiliary/a.rs | 25 +++++++ src/test/incremental/rlib_cross_crate/b.rs | 38 +++++++++++ 12 files changed, 202 insertions(+), 92 deletions(-) create mode 100644 src/test/incremental/rlib_cross_crate/auxiliary/a.rs create mode 100644 src/test/incremental/rlib_cross_crate/b.rs diff --git a/src/librustc/dep_graph/dep_node.rs b/src/librustc/dep_graph/dep_node.rs index dd7f0286574d0..c9247539990a9 100644 --- a/src/librustc/dep_graph/dep_node.rs +++ b/src/librustc/dep_graph/dep_node.rs @@ -246,7 +246,5 @@ impl DepNode { /// the need to be mapped or unmapped. (This ensures we can serialize /// them even in the absence of a tcx.) #[derive(Clone, Debug, PartialEq, Eq, Hash, RustcEncodable, RustcDecodable)] -pub enum WorkProductId { - PartitionObjectFile(String), // see (*TransPartition) below -} +pub struct WorkProductId(pub String); diff --git a/src/librustc/dep_graph/graph.rs b/src/librustc/dep_graph/graph.rs index 8691894b32513..bb027b11b45af 100644 --- a/src/librustc/dep_graph/graph.rs +++ b/src/librustc/dep_graph/graph.rs @@ -10,6 +10,7 @@ use hir::def_id::DefId; use rustc_data_structures::fnv::FnvHashMap; +use session::config::OutputType; use std::cell::{Ref, RefCell}; use std::rc::Rc; use std::sync::Arc; @@ -157,11 +158,11 @@ impl DepGraph { /// previous hash. If it matches up, we can reuse the object file. #[derive(Clone, Debug, RustcEncodable, RustcDecodable)] pub struct WorkProduct { - /// extra hash used to decide if work-product is still suitable; + /// Extra hash used to decide if work-product is still suitable; /// note that this is *not* a hash of the work-product itself. /// See documentation on `WorkProduct` type for an example. pub input_hash: u64, - /// filename storing this work-product (found in the incr. comp. directory) - pub file_name: String, + /// Saved files associated with this CGU + pub saved_files: Vec<(OutputType, String)>, } diff --git a/src/librustc/session/config.rs b/src/librustc/session/config.rs index a0c2416d24cff..690395399efa1 100644 --- a/src/librustc/session/config.rs +++ b/src/librustc/session/config.rs @@ -61,7 +61,7 @@ pub enum DebugInfoLevel { FullDebugInfo, } -#[derive(Clone, Copy, PartialEq, Eq, Hash)] +#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, RustcEncodable, RustcDecodable)] pub enum OutputType { Bitcode, Assembly, @@ -105,6 +105,17 @@ impl OutputType { OutputType::DepInfo => "dep-info", } } + + pub fn extension(&self) -> &'static str { + match *self { + OutputType::Bitcode => "bc", + OutputType::Assembly => "s", + OutputType::LlvmAssembly => "ll", + OutputType::Object => "o", + OutputType::DepInfo => "d", + OutputType::Exe => "", + } + } } #[derive(Clone)] @@ -215,15 +226,7 @@ impl OutputFilenames { flavor: OutputType, codegen_unit_name: Option<&str>) -> PathBuf { - let extension = match flavor { - OutputType::Bitcode => "bc", - OutputType::Assembly => "s", - OutputType::LlvmAssembly => "ll", - OutputType::Object => "o", - OutputType::DepInfo => "d", - OutputType::Exe => "", - }; - + let extension = flavor.extension(); self.temp_path_ext(extension, codegen_unit_name) } diff --git a/src/librustc_incremental/persist/load.rs b/src/librustc_incremental/persist/load.rs index 6b856459aabfe..36b6c79c40f5d 100644 --- a/src/librustc_incremental/persist/load.rs +++ b/src/librustc_incremental/persist/load.rs @@ -260,11 +260,20 @@ fn reconcile_work_products<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, debug!("reconcile_work_products: dep-node for {:?} is dirty", swp); delete_dirty_work_product(tcx, swp); } else { - let path = in_incr_comp_dir(tcx.sess, &swp.work_product.file_name).unwrap(); - if path.exists() { + let all_files_exist = + swp.work_product + .saved_files + .iter() + .all(|&(_, ref file_name)| { + let path = in_incr_comp_dir(tcx.sess, &file_name).unwrap(); + path.exists() + }); + if all_files_exist { + debug!("reconcile_work_products: all files for {:?} exist", swp); tcx.dep_graph.insert_previous_work_product(&swp.id, swp.work_product); } else { - debug!("reconcile_work_products: file for {:?} does not exist", swp); + debug!("reconcile_work_products: some file for {:?} does not exist", swp); + delete_dirty_work_product(tcx, swp); } } } @@ -273,13 +282,15 @@ fn reconcile_work_products<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, fn delete_dirty_work_product(tcx: TyCtxt, swp: SerializedWorkProduct) { debug!("delete_dirty_work_product({:?})", swp); - let path = in_incr_comp_dir(tcx.sess, &swp.work_product.file_name).unwrap(); - match fs::remove_file(&path) { - Ok(()) => { } - Err(err) => { - tcx.sess.warn( - &format!("file-system error deleting outdated file `{}`: {}", - path.display(), err)); + for &(_, ref file_name) in &swp.work_product.saved_files { + let path = in_incr_comp_dir(tcx.sess, file_name).unwrap(); + match fs::remove_file(&path) { + Ok(()) => { } + Err(err) => { + tcx.sess.warn( + &format!("file-system error deleting outdated file `{}`: {}", + path.display(), err)); + } } } } diff --git a/src/librustc_incremental/persist/work_product.rs b/src/librustc_incremental/persist/work_product.rs index 7695291bf620b..c106ea8f26269 100644 --- a/src/librustc_incremental/persist/work_product.rs +++ b/src/librustc_incremental/persist/work_product.rs @@ -13,47 +13,51 @@ use persist::util::*; use rustc::dep_graph::{WorkProduct, WorkProductId}; use rustc::session::Session; +use rustc::session::config::OutputType; use rustc::util::fs::link_or_copy; -use std::fs; -use std::path::Path; +use std::path::PathBuf; use std::sync::Arc; pub fn save_trans_partition(sess: &Session, - partition_name: &str, + cgu_name: &str, partition_hash: u64, - path_to_obj_file: &Path) { - debug!("save_trans_partition({:?},{},{})", - partition_name, + files: &[(OutputType, PathBuf)]) { + debug!("save_trans_partition({:?},{},{:?})", + cgu_name, partition_hash, - path_to_obj_file.display()); + files); if sess.opts.incremental.is_none() { return; } - let id = Arc::new(WorkProductId::PartitionObjectFile(partition_name.to_string())); - let file_name = format!("cgu-{}", partition_name); - let path_in_incr_dir = in_incr_comp_dir(sess, &file_name).unwrap(); + let work_product_id = Arc::new(WorkProductId(cgu_name.to_string())); - // try to delete the file if it already exists - // - // FIXME(#34955) we can be smarter here -- if we are re-using, no need to do anything - if path_in_incr_dir.exists() { - let _ = fs::remove_file(&path_in_incr_dir); - } + let saved_files: Option> = + files.iter() + .map(|&(kind, ref path)| { + let file_name = format!("cgu-{}.{}", cgu_name, kind.extension()); + let path_in_incr_dir = in_incr_comp_dir(sess, &file_name).unwrap(); + match link_or_copy(path, &path_in_incr_dir) { + Ok(_) => Some((kind, file_name)), + Err(err) => { + sess.warn(&format!("error copying object file `{}` \ + to incremental directory as `{}`: {}", + path.display(), + path_in_incr_dir.display(), + err)); + None + } + } + }) + .collect(); + let saved_files = match saved_files { + Some(v) => v, + None => return, + }; - match link_or_copy(path_to_obj_file, &path_in_incr_dir) { - Ok(_) => { - let work_product = WorkProduct { - input_hash: partition_hash, - file_name: file_name, - }; - sess.dep_graph.insert_work_product(&id, work_product); - } - Err(err) => { - sess.warn(&format!("error copying object file `{}` \ - to incremental directory as `{}`: {}", - path_to_obj_file.display(), - path_in_incr_dir.display(), - err)); - } - } + let work_product = WorkProduct { + input_hash: partition_hash, + saved_files: saved_files, + }; + + sess.dep_graph.insert_work_product(&work_product_id, work_product); } diff --git a/src/librustc_trans/back/write.rs b/src/librustc_trans/back/write.rs index 08d7b531c2f1b..4b9d5dd9e8d67 100644 --- a/src/librustc_trans/back/write.rs +++ b/src/librustc_trans/back/write.rs @@ -337,6 +337,8 @@ struct CodegenContext<'a> { remark: Passes, // Worker thread number worker: usize, + // Directory where incremental data is stored (if any) + incremental: Option, } impl<'a> CodegenContext<'a> { @@ -347,6 +349,7 @@ impl<'a> CodegenContext<'a> { plugin_passes: sess.plugin_llvm_passes.borrow().clone(), remark: sess.opts.cg.remark.clone(), worker: 0, + incremental: sess.opts.incremental.clone(), } } } @@ -612,7 +615,7 @@ unsafe fn optimize_and_codegen(cgcx: &CodegenContext, if copy_bc_to_obj { debug!("copying bitcode {:?} to obj {:?}", bc_out, obj_out); - if let Err(e) = fs::copy(&bc_out, &obj_out) { + if let Err(e) = link_or_copy(&bc_out, &obj_out) { cgcx.handler.err(&format!("failed to copy bitcode to object file: {}", e)); } } @@ -754,9 +757,19 @@ pub fn run_passes(sess: &Session, // If in incr. comp. mode, preserve the `.o` files for potential re-use for mtrans in trans.modules.iter() { - let path_to_obj = crate_output.temp_path(OutputType::Object, Some(&mtrans.name)); - debug!("wrote module {:?} to {:?}", mtrans.name, path_to_obj); - save_trans_partition(sess, &mtrans.name, mtrans.symbol_name_hash, &path_to_obj); + let mut files = vec![]; + + if modules_config.emit_obj { + let path = crate_output.temp_path(OutputType::Object, Some(&mtrans.name)); + files.push((OutputType::Object, path)); + } + + if modules_config.emit_bc { + let path = crate_output.temp_path(OutputType::Bitcode, Some(&mtrans.name)); + files.push((OutputType::Bitcode, path)); + } + + save_trans_partition(sess, &mtrans.name, mtrans.symbol_name_hash, &files); } // All codegen is finished. @@ -941,20 +954,24 @@ fn execute_work_item(cgcx: &CodegenContext, work_item.config, work_item.output_names); } - ModuleSource::Preexisting(ref buf) => { - let obj_out = work_item.output_names.temp_path(OutputType::Object, - Some(&work_item.mtrans.name)); - debug!("copying pre-existing module `{}` from {} to {}", - work_item.mtrans.name, - buf.display(), - obj_out.display()); - match link_or_copy(buf, &obj_out) { - Ok(()) => { } - Err(err) => { - cgcx.handler.err(&format!("unable to copy {} to {}: {}", - buf.display(), - obj_out.display(), - err)); + ModuleSource::Preexisting(wp) => { + let incremental = cgcx.incremental.as_ref().unwrap(); + let name = &work_item.mtrans.name; + for (kind, saved_file) in wp.saved_files { + let obj_out = work_item.output_names.temp_path(kind, Some(name)); + let source_file = incremental.join(&saved_file); + debug!("copying pre-existing module `{}` from {:?} to {}", + work_item.mtrans.name, + source_file, + obj_out.display()); + match link_or_copy(&source_file, &obj_out) { + Ok(()) => { } + Err(err) => { + cgcx.handler.err(&format!("unable to copy {} to {}: {}", + source_file.display(), + obj_out.display(), + err)); + } } } } @@ -994,6 +1011,8 @@ fn run_work_multithreaded(sess: &Session, let mut tx = Some(tx); futures.push(rx); + let incremental = sess.opts.incremental.clone(); + thread::Builder::new().name(format!("codegen-{}", i)).spawn(move || { let diag_handler = Handler::with_emitter(true, false, box diag_emitter); @@ -1005,6 +1024,7 @@ fn run_work_multithreaded(sess: &Session, plugin_passes: plugin_passes, remark: remark, worker: i, + incremental: incremental, }; loop { diff --git a/src/librustc_trans/base.rs b/src/librustc_trans/base.rs index e73c4f41c9412..5a19ddff7462a 100644 --- a/src/librustc_trans/base.rs +++ b/src/librustc_trans/base.rs @@ -43,10 +43,9 @@ use rustc::ty::subst::{self, Substs}; use rustc::traits; use rustc::ty::{self, Ty, TyCtxt, TypeFoldable}; use rustc::ty::adjustment::CustomCoerceUnsized; -use rustc::dep_graph::DepNode; +use rustc::dep_graph::{DepNode, WorkProduct}; use rustc::hir::map as hir_map; use rustc::util::common::time; -use rustc_incremental::in_incr_comp_dir; use rustc::mir::mir_map::MirMap; use rustc_data_structures::graph::OUTGOING; use session::config::{self, NoDebugInfo, FullDebugInfo}; @@ -103,7 +102,6 @@ use std::cell::{Cell, RefCell}; use std::collections::HashMap; use std::ptr; use std::rc::Rc; -use std::path::PathBuf; use std::str; use std::{i8, i16, i32, i64}; use syntax_pos::{Span, DUMMY_SP}; @@ -2721,7 +2719,7 @@ pub fn trans_crate<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>, fn trans_reuse_previous_work_products(tcx: TyCtxt, codegen_units: &[CodegenUnit], symbol_map: &SymbolMap) - -> Vec> { + -> Vec> { debug!("trans_reuse_previous_work_products()"); codegen_units .iter() @@ -2735,7 +2733,7 @@ fn trans_reuse_previous_work_products(tcx: TyCtxt, if let Some(work_product) = tcx.dep_graph.previous_work_product(&id) { if work_product.input_hash == hash { debug!("trans_reuse_previous_work_products: reusing {:?}", work_product); - return Some(in_incr_comp_dir(tcx.sess, &work_product.file_name).unwrap()); + return Some(work_product); } else { debug!("trans_reuse_previous_work_products: \ not reusing {:?} because hash changed to {:?}", diff --git a/src/librustc_trans/context.rs b/src/librustc_trans/context.rs index bf69629821285..a8f8474e94070 100644 --- a/src/librustc_trans/context.rs +++ b/src/librustc_trans/context.rs @@ -10,7 +10,7 @@ use llvm; use llvm::{ContextRef, ModuleRef, ValueRef, BuilderRef}; -use rustc::dep_graph::{DepNode, DepTrackingMap, DepTrackingMapConfig}; +use rustc::dep_graph::{DepNode, DepTrackingMap, DepTrackingMapConfig, WorkProduct}; use middle::cstore::LinkMeta; use rustc::hir::def::ExportMap; use rustc::hir::def_id::DefId; @@ -40,7 +40,6 @@ use util::nodemap::{NodeMap, NodeSet, DefIdMap, FnvHashMap, FnvHashSet}; use std::ffi::{CStr, CString}; use std::cell::{Cell, RefCell}; -use std::path::PathBuf; use std::marker::PhantomData; use std::ptr; use std::rc::Rc; @@ -96,7 +95,7 @@ pub struct SharedCrateContext<'a, 'tcx: 'a> { pub struct LocalCrateContext<'tcx> { llmod: ModuleRef, llcx: ContextRef, - previous_work_product: Option, + previous_work_product: Option, tn: TypeNames, // FIXME: This seems to be largely unused. codegen_unit: CodegenUnit<'tcx>, needs_unwind_cleanup_cache: RefCell, bool>>, @@ -202,13 +201,13 @@ pub struct CrateContextList<'a, 'tcx: 'a> { impl<'a, 'tcx: 'a> CrateContextList<'a, 'tcx> { pub fn new(shared_ccx: &'a SharedCrateContext<'a, 'tcx>, codegen_units: Vec>, - previous_work_products: Vec>, + previous_work_products: Vec>, symbol_map: Rc>) -> CrateContextList<'a, 'tcx> { CrateContextList { shared: shared_ccx, - local_ccxs: codegen_units.into_iter().zip(previous_work_products).map(|(cgu, path)| { - LocalCrateContext::new(shared_ccx, cgu, path, symbol_map.clone()) + local_ccxs: codegen_units.into_iter().zip(previous_work_products).map(|(cgu, wp)| { + LocalCrateContext::new(shared_ccx, cgu, wp, symbol_map.clone()) }).collect() } } @@ -541,7 +540,7 @@ impl<'b, 'tcx> SharedCrateContext<'b, 'tcx> { impl<'tcx> LocalCrateContext<'tcx> { fn new<'a>(shared: &SharedCrateContext<'a, 'tcx>, codegen_unit: CodegenUnit<'tcx>, - previous_work_product: Option, + previous_work_product: Option, symbol_map: Rc>) -> LocalCrateContext<'tcx> { unsafe { @@ -727,7 +726,7 @@ impl<'b, 'tcx> CrateContext<'b, 'tcx> { self.local().llcx } - pub fn previous_work_product(&self) -> Option<&PathBuf> { + pub fn previous_work_product(&self) -> Option<&WorkProduct> { self.local().previous_work_product.as_ref() } diff --git a/src/librustc_trans/lib.rs b/src/librustc_trans/lib.rs index 67475081caec7..81a1dbeb7fe74 100644 --- a/src/librustc_trans/lib.rs +++ b/src/librustc_trans/lib.rs @@ -37,7 +37,7 @@ #![feature(unicode)] #![feature(question_mark)] -use std::path::PathBuf; +use rustc::dep_graph::WorkProduct; extern crate arena; extern crate flate; @@ -135,6 +135,11 @@ mod value; #[derive(Clone)] pub struct ModuleTranslation { + /// The name of the module. When the crate may be saved between + /// compilations, incremental compilation requires that name be + /// unique amongst **all** crates. Therefore, it should contain + /// something unique to this crate (e.g., a module path) as well + /// as the crate name and disambiguator. pub name: String, pub symbol_name_hash: u64, pub source: ModuleSource, @@ -142,7 +147,10 @@ pub struct ModuleTranslation { #[derive(Clone)] pub enum ModuleSource { - Preexisting(PathBuf), + /// Copy the `.o` files or whatever from the incr. comp. directory. + Preexisting(WorkProduct), + + /// Rebuild from this LLVM module. Translated(ModuleLlvm), } diff --git a/src/librustc_trans/partitioning.rs b/src/librustc_trans/partitioning.rs index 32bcbf9f7562b..ade6e8abeb32a 100644 --- a/src/librustc_trans/partitioning.rs +++ b/src/librustc_trans/partitioning.rs @@ -143,7 +143,12 @@ pub enum PartitioningStrategy { } pub struct CodegenUnit<'tcx> { + /// A name for this CGU. Incremental compilation requires that + /// name be unique amongst **all** crates. Therefore, it should + /// contain something unique to this crate (e.g., a module path) + /// as well as the crate name and disambiguator. name: InternedString, + items: FnvHashMap, llvm::Linkage>, } @@ -174,7 +179,7 @@ impl<'tcx> CodegenUnit<'tcx> { } pub fn work_product_id(&self) -> Arc { - Arc::new(WorkProductId::PartitionObjectFile(self.name().to_string())) + Arc::new(WorkProductId(self.name().to_string())) } pub fn work_product_dep_node(&self) -> DepNode { diff --git a/src/test/incremental/rlib_cross_crate/auxiliary/a.rs b/src/test/incremental/rlib_cross_crate/auxiliary/a.rs new file mode 100644 index 0000000000000..ff5fd63471449 --- /dev/null +++ b/src/test/incremental/rlib_cross_crate/auxiliary/a.rs @@ -0,0 +1,25 @@ +// Copyright 2014 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +// no-prefer-dynamic + +#![crate_type="rlib"] + +#[cfg(rpass1)] +pub type X = u32; + +#[cfg(rpass2)] +pub type X = i32; + +// this version doesn't actually change anything: +#[cfg(rpass3)] +pub type X = i32; + +pub type Y = char; diff --git a/src/test/incremental/rlib_cross_crate/b.rs b/src/test/incremental/rlib_cross_crate/b.rs new file mode 100644 index 0000000000000..55398370425a3 --- /dev/null +++ b/src/test/incremental/rlib_cross_crate/b.rs @@ -0,0 +1,38 @@ +// Copyright 2014 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +// Same test as `type_alias_cross_crate`, but with +// `no-prefer-dynamic`, ensuring that we test what happens when we +// build rlibs (before we were only testing dylibs, which meant we +// didn't realize we had to preserve a `bc` file as well). + +// aux-build:a.rs +// revisions:rpass1 rpass2 rpass3 +// no-prefer-dynamic + + +#![feature(rustc_attrs)] + +extern crate a; + +#[rustc_dirty(label="TypeckItemBody", cfg="rpass2")] +#[rustc_clean(label="TypeckItemBody", cfg="rpass3")] +pub fn use_X() -> u32 { + let x: a::X = 22; + x as u32 +} + +#[rustc_clean(label="TypeckItemBody", cfg="rpass2")] +#[rustc_clean(label="TypeckItemBody", cfg="rpass3")] +pub fn use_Y() { + let x: a::Y = 'c'; +} + +pub fn main() { } From e359147d1214c6fe8e010118c5df05ca10f0a2b9 Mon Sep 17 00:00:00 2001 From: Niko Matsakis Date: Thu, 28 Jul 2016 15:39:02 -0400 Subject: [PATCH 8/9] hash def-path's better actually we shouldn't even hash nested items at all, but that is addressed in a followup PR --- src/librustc_incremental/calculate_svh.rs | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/librustc_incremental/calculate_svh.rs b/src/librustc_incremental/calculate_svh.rs index 70704f5dec0d2..d433bdea6ecaa 100644 --- a/src/librustc_incremental/calculate_svh.rs +++ b/src/librustc_incremental/calculate_svh.rs @@ -119,6 +119,7 @@ mod svh_visitor { use rustc::ty::TyCtxt; use rustc::hir; use rustc::hir::*; + use rustc::hir::map::DefPath; use rustc::hir::intravisit as visit; use rustc::hir::intravisit::{Visitor, FnKind}; @@ -135,6 +136,15 @@ mod svh_visitor { -> Self { StrictVersionHashVisitor { st: st, tcx: tcx } } + + fn hash_def_path(&mut self, path: &DefPath) { + self.tcx.crate_name(path.krate).hash(self.st); + self.tcx.crate_disambiguator(path.krate).hash(self.st); + for data in &path.data { + data.data.as_interned_str().hash(self.st); + data.disambiguator.hash(self.st); + } + } } // To off-load the bulk of the hash-computation on #[derive(Hash)], @@ -289,9 +299,9 @@ mod svh_visitor { impl<'a, 'tcx> Visitor<'a> for StrictVersionHashVisitor<'a, 'tcx> { fn visit_nested_item(&mut self, item: ItemId) { - debug!("visit_nested_item: {:?} st={:?}", item, self.st); - let def_path = self.tcx.map.def_path_from_id(item.id); - def_path.hash(self.st); + let def_path = self.tcx.map.def_path_from_id(item.id).unwrap(); + debug!("visit_nested_item: def_path={:?} st={:?}", def_path, self.st); + self.hash_def_path(&def_path); } fn visit_variant_data(&mut self, s: &'a VariantData, name: Name, From 42cd5d4ee28a1c1b3bf4f07e27b1ca5a03fd9b02 Mon Sep 17 00:00:00 2001 From: Niko Matsakis Date: Thu, 28 Jul 2016 15:39:34 -0400 Subject: [PATCH 9/9] make it possible to track where hash diverges --- src/librustc_incremental/calculate_svh.rs | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/librustc_incremental/calculate_svh.rs b/src/librustc_incremental/calculate_svh.rs index d433bdea6ecaa..7b1e0d2d0c8b6 100644 --- a/src/librustc_incremental/calculate_svh.rs +++ b/src/librustc_incremental/calculate_svh.rs @@ -306,12 +306,14 @@ mod svh_visitor { fn visit_variant_data(&mut self, s: &'a VariantData, name: Name, g: &'a Generics, _: NodeId, _: Span) { + debug!("visit_variant_data: st={:?}", self.st); SawStructDef(name.as_str()).hash(self.st); visit::walk_generics(self, g); visit::walk_struct_def(self, s) } fn visit_variant(&mut self, v: &'a Variant, g: &'a Generics, item_id: NodeId) { + debug!("visit_variant: st={:?}", self.st); SawVariant.hash(self.st); // walk_variant does not call walk_generics, so do it here. visit::walk_generics(self, g); @@ -333,14 +335,17 @@ mod svh_visitor { // pattern, please move that method up above this comment.) fn visit_name(&mut self, _: Span, name: Name) { + debug!("visit_name: st={:?}", self.st); SawIdent(name.as_str()).hash(self.st); } fn visit_lifetime(&mut self, l: &'a Lifetime) { + debug!("visit_lifetime: st={:?}", self.st); SawLifetime(l.name.as_str()).hash(self.st); } fn visit_lifetime_def(&mut self, l: &'a LifetimeDef) { + debug!("visit_lifetime_def: st={:?}", self.st); SawLifetimeDef(l.lifetime.name.as_str()).hash(self.st); } @@ -350,14 +355,18 @@ mod svh_visitor { // that a change to a crate body will require downstream // crates to be recompiled. fn visit_expr(&mut self, ex: &'a Expr) { + debug!("visit_expr: st={:?}", self.st); SawExpr(saw_expr(&ex.node)).hash(self.st); visit::walk_expr(self, ex) } fn visit_stmt(&mut self, s: &'a Stmt) { + debug!("visit_stmt: st={:?}", self.st); SawStmt(saw_stmt(&s.node)).hash(self.st); visit::walk_stmt(self, s) } fn visit_foreign_item(&mut self, i: &'a ForeignItem) { + debug!("visit_foreign_item: st={:?}", self.st); + // FIXME (#14132) ideally we would incorporate privacy (or // perhaps reachability) somewhere here, so foreign items // that do not leak into downstream crates would not be @@ -367,6 +376,7 @@ mod svh_visitor { fn visit_item(&mut self, i: &'a Item) { debug!("visit_item: {:?} st={:?}", i, self.st); + // FIXME (#14132) ideally would incorporate reachability // analysis somewhere here, so items that never leak into // downstream crates (e.g. via monomorphisation or @@ -375,55 +385,68 @@ mod svh_visitor { } fn visit_mod(&mut self, m: &'a Mod, _s: Span, _n: NodeId) { + debug!("visit_mod: st={:?}", self.st); SawMod.hash(self.st); visit::walk_mod(self, m) } fn visit_decl(&mut self, d: &'a Decl) { + debug!("visit_decl: st={:?}", self.st); SawDecl.hash(self.st); visit::walk_decl(self, d) } fn visit_ty(&mut self, t: &'a Ty) { + debug!("visit_ty: st={:?}", self.st); SawTy.hash(self.st); visit::walk_ty(self, t) } fn visit_generics(&mut self, g: &'a Generics) { + debug!("visit_generics: st={:?}", self.st); SawGenerics.hash(self.st); visit::walk_generics(self, g) } fn visit_fn(&mut self, fk: FnKind<'a>, fd: &'a FnDecl, b: &'a Block, s: Span, _: NodeId) { + debug!("visit_fn: st={:?}", self.st); SawFn.hash(self.st); visit::walk_fn(self, fk, fd, b, s) } fn visit_trait_item(&mut self, ti: &'a TraitItem) { + debug!("visit_trait_item: st={:?}", self.st); SawTraitItem.hash(self.st); visit::walk_trait_item(self, ti) } fn visit_impl_item(&mut self, ii: &'a ImplItem) { + debug!("visit_impl_item: st={:?}", self.st); SawImplItem.hash(self.st); visit::walk_impl_item(self, ii) } fn visit_struct_field(&mut self, s: &'a StructField) { + debug!("visit_struct_field: st={:?}", self.st); SawStructField.hash(self.st); visit::walk_struct_field(self, s) } fn visit_path(&mut self, path: &'a Path, _: ast::NodeId) { + debug!("visit_path: st={:?}", self.st); SawPath.hash(self.st); visit::walk_path(self, path) } fn visit_block(&mut self, b: &'a Block) { + debug!("visit_block: st={:?}", self.st); SawBlock.hash(self.st); visit::walk_block(self, b) } fn visit_pat(&mut self, p: &'a Pat) { + debug!("visit_pat: st={:?}", self.st); SawPat.hash(self.st); visit::walk_pat(self, p) } fn visit_local(&mut self, l: &'a Local) { + debug!("visit_local: st={:?}", self.st); SawLocal.hash(self.st); visit::walk_local(self, l) } fn visit_arm(&mut self, a: &'a Arm) { + debug!("visit_arm: st={:?}", self.st); SawArm.hash(self.st); visit::walk_arm(self, a) } }