diff --git a/src/librustc_data_structures/bitvec.rs b/src/librustc_data_structures/bitvec.rs index 092b406ae9e3e..cb648038c3436 100644 --- a/src/librustc_data_structures/bitvec.rs +++ b/src/librustc_data_structures/bitvec.rs @@ -52,9 +52,8 @@ impl BitVector { pub fn grow(&mut self, num_bits: usize) { let num_words = u64s(num_bits); - let extra_words = self.data.len() - num_words; - if extra_words > 0 { - self.data.extend((0..extra_words).map(|_| 0)); + if self.data.len() < num_words { + self.data.resize(num_words, 0) } } @@ -284,15 +283,27 @@ fn union_two_vecs() { #[test] fn grow() { let mut vec1 = BitVector::new(65); - assert!(vec1.insert(3)); - assert!(!vec1.insert(3)); - assert!(vec1.insert(5)); - assert!(vec1.insert(64)); + for index in 0 .. 65 { + assert!(vec1.insert(index)); + assert!(!vec1.insert(index)); + } vec1.grow(128); - assert!(vec1.contains(3)); - assert!(vec1.contains(5)); - assert!(vec1.contains(64)); - assert!(!vec1.contains(126)); + + // Check if the bits set before growing are still set + for index in 0 .. 65 { + assert!(vec1.contains(index)); + } + + // Check if the new bits are all un-set + for index in 65 .. 128 { + assert!(!vec1.contains(index)); + } + + // Check that we can set all new bits without running out of bounds + for index in 65 .. 128 { + assert!(vec1.insert(index)); + assert!(!vec1.insert(index)); + } } #[test] diff --git a/src/librustc_llvm/lib.rs b/src/librustc_llvm/lib.rs index 12d652bd5cb87..526b6bf68be24 100644 --- a/src/librustc_llvm/lib.rs +++ b/src/librustc_llvm/lib.rs @@ -97,7 +97,7 @@ pub enum Visibility { // DLLExportLinkage, GhostLinkage and LinkOnceODRAutoHideLinkage. // LinkerPrivateLinkage and LinkerPrivateWeakLinkage are not included either; // they've been removed in upstream LLVM commit r203866. -#[derive(Copy, Clone, PartialEq, Eq, Debug)] +#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)] pub enum Linkage { ExternalLinkage = 0, AvailableExternallyLinkage = 1, diff --git a/src/librustc_trans/base.rs b/src/librustc_trans/base.rs index 9190389b722b6..b100282781ece 100644 --- a/src/librustc_trans/base.rs +++ b/src/librustc_trans/base.rs @@ -58,7 +58,6 @@ use attributes; use build::*; use builder::{Builder, noname}; use callee::{Callee, CallArgs, ArgExprs, ArgVals}; -use partitioning; use cleanup::{self, CleanupMethods, DropHint}; use closure; use common::{Block, C_bool, C_bytes_in_context, C_i32, C_int, C_uint, C_integral}; @@ -83,6 +82,7 @@ use machine::{llalign_of_min, llsize_of, llsize_of_real}; use meth; use mir; use monomorphize::{self, Instance}; +use partitioning::{self, PartitioningStrategy, InstantiationMode}; use symbol_names_test; use tvec; use type_::Type; @@ -2934,12 +2934,21 @@ fn collect_translation_items<'a, 'tcx>(ccx: &CrateContext<'a, 'tcx>) { None => TransItemCollectionMode::Lazy }; - let (items, inlining_map) = time(time_passes, "translation item collection", || { + let (items, reference_map) = time(time_passes, "translation item collection", || { collector::collect_crate_translation_items(&ccx, collection_mode) }); + let strategy = if ccx.sess().opts.debugging_opts.incremental.is_some() { + PartitioningStrategy::PerModule + } else { + PartitioningStrategy::FixedUnitCount(ccx.sess().opts.cg.codegen_units) + }; + let codegen_units = time(time_passes, "codegen unit partitioning", || { - partitioning::partition(ccx.tcx(), items.iter().cloned(), &inlining_map) + partitioning::partition(ccx.tcx(), + items.iter().cloned(), + strategy, + &reference_map) }); if ccx.sess().opts.debugging_opts.print_trans_items.is_some() { @@ -2967,17 +2976,18 @@ fn collect_translation_items<'a, 'tcx>(ccx: &CrateContext<'a, 'tcx>) { output.push_str(&cgu_name[..]); let linkage_abbrev = match linkage { - llvm::ExternalLinkage => "External", - llvm::AvailableExternallyLinkage => "Available", - llvm::LinkOnceAnyLinkage => "OnceAny", - llvm::LinkOnceODRLinkage => "OnceODR", - llvm::WeakAnyLinkage => "WeakAny", - llvm::WeakODRLinkage => "WeakODR", - llvm::AppendingLinkage => "Appending", - llvm::InternalLinkage => "Internal", - llvm::PrivateLinkage => "Private", - llvm::ExternalWeakLinkage => "ExternalWeak", - llvm::CommonLinkage => "Common", + InstantiationMode::Def(llvm::ExternalLinkage) => "External", + InstantiationMode::Def(llvm::AvailableExternallyLinkage) => "Available", + InstantiationMode::Def(llvm::LinkOnceAnyLinkage) => "OnceAny", + InstantiationMode::Def(llvm::LinkOnceODRLinkage) => "OnceODR", + InstantiationMode::Def(llvm::WeakAnyLinkage) => "WeakAny", + InstantiationMode::Def(llvm::WeakODRLinkage) => "WeakODR", + InstantiationMode::Def(llvm::AppendingLinkage) => "Appending", + InstantiationMode::Def(llvm::InternalLinkage) => "Internal", + InstantiationMode::Def(llvm::PrivateLinkage) => "Private", + InstantiationMode::Def(llvm::ExternalWeakLinkage) => "ExternalWeak", + InstantiationMode::Def(llvm::CommonLinkage) => "Common", + InstantiationMode::Decl => "Declaration", }; output.push_str("["); diff --git a/src/librustc_trans/collector.rs b/src/librustc_trans/collector.rs index e8437393625db..e8181579911d2 100644 --- a/src/librustc_trans/collector.rs +++ b/src/librustc_trans/collector.rs @@ -188,6 +188,8 @@ //! this is not implemented however: a translation item will be produced //! regardless of whether it is actually needed or not. +use rustc_data_structures::bitvec::BitVector; + use rustc::hir; use rustc::hir::intravisit as hir_visit; @@ -211,7 +213,7 @@ use base::{custom_coerce_unsize_info, llvm_linkage_by_name}; use context::CrateContext; use common::{fulfill_obligation, normalize_and_test_predicates, type_is_sized}; -use glue; +use glue::{self, DropGlueKind}; use llvm; use meth; use monomorphize::{self, Instance}; @@ -227,7 +229,7 @@ pub enum TransItemCollectionMode { #[derive(PartialEq, Eq, Clone, Copy, Debug)] pub enum TransItem<'tcx> { - DropGlue(Ty<'tcx>), + DropGlue(DropGlueKind<'tcx>), Fn(Instance<'tcx>), Static(NodeId) } @@ -252,12 +254,76 @@ impl<'tcx> Hash for TransItem<'tcx> { } } -pub type InliningMap<'tcx> = FnvHashMap, FnvHashSet>>; +/// Maps every translation item to all translation items it references in its +/// body. +pub struct ReferenceMap<'tcx> { + // Maps a source translation item to a range of target translation items. + // The two numbers in the tuple are the start (inclusive) and + // end index (exclusive) within the `targets` and the `inlined` vecs. + index: FnvHashMap, (usize, usize)>, + targets: Vec>, + inlined: BitVector +} + +impl<'tcx> ReferenceMap<'tcx> { + + fn new() -> ReferenceMap<'tcx> { + ReferenceMap { + index: FnvHashMap(), + targets: Vec::new(), + inlined: BitVector::new(64 * 256), + } + } + + fn record_references(&mut self, source: TransItem<'tcx>, targets: I) + where I: Iterator, bool)> + { + assert!(!self.index.contains_key(&source)); + + let start_index = self.targets.len(); + + for (target, inlined) in targets { + let index = self.targets.len(); + self.targets.push(target); + self.inlined.grow(index + 1); + + if inlined { + self.inlined.insert(index); + } + } + + let end_index = self.targets.len(); + self.index.insert(source, (start_index, end_index)); + } + + // Internally iterate over all items referenced by `source` which will be + // made available for inlining. + pub fn with_inlining_candidates(&self, source: TransItem<'tcx>, mut f: F) + where F: FnMut(TransItem<'tcx>) { + if let Some(&(start_index, end_index)) = self.index.get(&source) + { + for index in start_index .. end_index { + if self.inlined.contains(index) { + f(self.targets[index]) + } + } + } + } + + pub fn get_direct_references_from(&self, source: TransItem<'tcx>) -> &[TransItem<'tcx>] + { + if let Some(&(start_index, end_index)) = self.index.get(&source) { + &self.targets[start_index .. end_index] + } else { + &self.targets[0 .. 0] + } + } +} pub fn collect_crate_translation_items<'a, 'tcx>(ccx: &CrateContext<'a, 'tcx>, mode: TransItemCollectionMode) -> (FnvHashSet>, - InliningMap<'tcx>) { + ReferenceMap<'tcx>) { // We are not tracking dependencies of this pass as it has to be re-executed // every time no matter what. ccx.tcx().dep_graph.with_ignore(|| { @@ -266,17 +332,17 @@ pub fn collect_crate_translation_items<'a, 'tcx>(ccx: &CrateContext<'a, 'tcx>, debug!("Building translation item graph, beginning at roots"); let mut visited = FnvHashSet(); let mut recursion_depths = DefIdMap(); - let mut inlining_map = FnvHashMap(); + let mut reference_map = ReferenceMap::new(); for root in roots { collect_items_rec(ccx, root, &mut visited, &mut recursion_depths, - &mut inlining_map); + &mut reference_map); } - (visited, inlining_map) + (visited, reference_map) }) } @@ -307,7 +373,7 @@ fn collect_items_rec<'a, 'tcx: 'a>(ccx: &CrateContext<'a, 'tcx>, starting_point: TransItem<'tcx>, visited: &mut FnvHashSet>, recursion_depths: &mut DefIdMap, - inlining_map: &mut InliningMap<'tcx>) { + reference_map: &mut ReferenceMap<'tcx>) { if !visited.insert(starting_point.clone()) { // We've been here already, no need to search again. return; @@ -326,7 +392,7 @@ fn collect_items_rec<'a, 'tcx: 'a>(ccx: &CrateContext<'a, 'tcx>, let def_id = ccx.tcx().map.local_def_id(node_id); let ty = ccx.tcx().lookup_item_type(def_id).ty; let ty = glue::get_drop_glue_type(ccx, ty); - neighbors.push(TransItem::DropGlue(ty)); + neighbors.push(TransItem::DropGlue(DropGlueKind::Ty(ty))); recursion_depth_reset = None; } TransItem::Fn(instance) => { @@ -351,9 +417,10 @@ fn collect_items_rec<'a, 'tcx: 'a>(ccx: &CrateContext<'a, 'tcx>, } } + record_references(ccx, starting_point, &neighbors[..], reference_map); + for neighbour in neighbors { - record_inlined_use(ccx, starting_point, neighbour, inlining_map); - collect_items_rec(ccx, neighbour, visited, recursion_depths, inlining_map); + collect_items_rec(ccx, neighbour, visited, recursion_depths, reference_map); } if let Some((def_id, depth)) = recursion_depth_reset { @@ -363,16 +430,17 @@ fn collect_items_rec<'a, 'tcx: 'a>(ccx: &CrateContext<'a, 'tcx>, debug!("END collect_items_rec({})", starting_point.to_string(ccx)); } -fn record_inlined_use<'a, 'tcx>(ccx: &CrateContext<'a, 'tcx>, - caller: TransItem<'tcx>, - callee: TransItem<'tcx>, - inlining_map: &mut InliningMap<'tcx>) { - if callee.is_from_extern_crate() || - callee.requests_inline(ccx.tcx()) { - inlining_map.entry(caller) - .or_insert_with(|| FnvHashSet()) - .insert(callee); - } +fn record_references<'a, 'tcx>(ccx: &CrateContext<'a, 'tcx>, + caller: TransItem<'tcx>, + callees: &[TransItem<'tcx>], + reference_map: &mut ReferenceMap<'tcx>) { + let iter = callees.into_iter() + .map(|callee| { + let is_inlining_candidate = callee.is_from_extern_crate() || + callee.requests_inline(ccx.tcx()); + (*callee, is_inlining_candidate) + }); + reference_map.record_references(caller, iter); } fn check_recursion_limit<'a, 'tcx: 'a>(ccx: &CrateContext<'a, 'tcx>, @@ -485,7 +553,7 @@ impl<'a, 'tcx> MirVisitor<'tcx> for MirNeighborCollector<'a, 'tcx> { &ty); let ty = self.ccx.tcx().erase_regions(&ty); let ty = glue::get_drop_glue_type(self.ccx, ty); - self.output.push(TransItem::DropGlue(ty)); + self.output.push(TransItem::DropGlue(DropGlueKind::Ty(ty))); } self.super_lvalue(lvalue, context); @@ -575,9 +643,17 @@ fn can_have_local_instance<'a, 'tcx>(ccx: &CrateContext<'a, 'tcx>, } fn find_drop_glue_neighbors<'a, 'tcx>(ccx: &CrateContext<'a, 'tcx>, - ty: ty::Ty<'tcx>, - output: &mut Vec>) -{ + dg: DropGlueKind<'tcx>, + output: &mut Vec>) { + let ty = match dg { + DropGlueKind::Ty(ty) => ty, + DropGlueKind::TyContents(_) => { + // We already collected the neighbors of this item via the + // DropGlueKind::Ty variant. + return + } + }; + debug!("find_drop_glue_neighbors: {}", type_to_string(ccx, ty)); // Make sure the exchange_free_fn() lang-item gets translated if @@ -634,6 +710,10 @@ fn find_drop_glue_neighbors<'a, 'tcx>(ccx: &CrateContext<'a, 'tcx>, &Substs::empty()); output.push(trans_item); } + + // This type has a Drop implementation, we'll need the contents-only + // version of the glue too. + output.push(TransItem::DropGlue(DropGlueKind::TyContents(ty))); } // Finally add the types of nested values @@ -661,7 +741,7 @@ fn find_drop_glue_neighbors<'a, 'tcx>(ccx: &CrateContext<'a, 'tcx>, let field_type = glue::get_drop_glue_type(ccx, field_type); if glue::type_needs_drop(ccx.tcx(), field_type) { - output.push(TransItem::DropGlue(field_type)); + output.push(TransItem::DropGlue(DropGlueKind::Ty(field_type))); } } } @@ -669,7 +749,7 @@ fn find_drop_glue_neighbors<'a, 'tcx>(ccx: &CrateContext<'a, 'tcx>, for upvar_ty in &substs.upvar_tys { let upvar_ty = glue::get_drop_glue_type(ccx, upvar_ty); if glue::type_needs_drop(ccx.tcx(), upvar_ty) { - output.push(TransItem::DropGlue(upvar_ty)); + output.push(TransItem::DropGlue(DropGlueKind::Ty(upvar_ty))); } } } @@ -677,14 +757,14 @@ fn find_drop_glue_neighbors<'a, 'tcx>(ccx: &CrateContext<'a, 'tcx>, ty::TyArray(inner_type, _) => { let inner_type = glue::get_drop_glue_type(ccx, inner_type); if glue::type_needs_drop(ccx.tcx(), inner_type) { - output.push(TransItem::DropGlue(inner_type)); + output.push(TransItem::DropGlue(DropGlueKind::Ty(inner_type))); } } ty::TyTuple(ref args) => { for arg in args { let arg = glue::get_drop_glue_type(ccx, arg); if glue::type_needs_drop(ccx.tcx(), arg) { - output.push(TransItem::DropGlue(arg)); + output.push(TransItem::DropGlue(DropGlueKind::Ty(arg))); } } } @@ -1000,7 +1080,7 @@ impl<'b, 'a, 'v> hir_visit::Visitor<'v> for RootCollector<'b, 'a, 'v> { self.ccx.tcx().map.local_def_id(item.id))); let ty = glue::get_drop_glue_type(self.ccx, ty); - self.output.push(TransItem::DropGlue(ty)); + self.output.push(TransItem::DropGlue(DropGlueKind::Ty(ty))); } } } @@ -1413,10 +1493,13 @@ impl<'tcx> TransItem<'tcx> { let hir_map = &ccx.tcx().map; return match *self { - TransItem::DropGlue(t) => { + TransItem::DropGlue(dg) => { let mut s = String::with_capacity(32); - s.push_str("drop-glue "); - push_unique_type_name(ccx, t, &mut s); + match dg { + DropGlueKind::Ty(_) => s.push_str("drop-glue "), + DropGlueKind::TyContents(_) => s.push_str("drop-glue-contents "), + }; + push_unique_type_name(ccx, dg.ty(), &mut s); s } TransItem::Fn(instance) => { @@ -1442,8 +1525,8 @@ impl<'tcx> TransItem<'tcx> { fn to_raw_string(&self) -> String { match *self { - TransItem::DropGlue(t) => { - format!("DropGlue({})", t as *const _ as usize) + TransItem::DropGlue(dg) => { + format!("DropGlue({})", dg.ty() as *const _ as usize) } TransItem::Fn(instance) => { format!("Fn({:?}, {})", diff --git a/src/librustc_trans/glue.rs b/src/librustc_trans/glue.rs index 39ea25619eed0..06761a7016add 100644 --- a/src/librustc_trans/glue.rs +++ b/src/librustc_trans/glue.rs @@ -98,7 +98,7 @@ pub fn get_drop_glue_type<'a, 'tcx>(ccx: &CrateContext<'a, 'tcx>, // Even if there is no dtor for t, there might be one deeper down and we // might need to pass in the vtable ptr. if !type_is_sized(tcx, t) { - return t + return ccx.tcx().erase_regions(&t); } // FIXME (#22815): note that type_needs_drop conservatively @@ -121,10 +121,10 @@ pub fn get_drop_glue_type<'a, 'tcx>(ccx: &CrateContext<'a, 'tcx>, if llsize_of_alloc(ccx, llty) == 0 { tcx.types.i8 } else { - t + ccx.tcx().erase_regions(&t) } } - _ => t + _ => ccx.tcx().erase_regions(&t) } } @@ -215,11 +215,11 @@ pub enum DropGlueKind<'tcx> { } impl<'tcx> DropGlueKind<'tcx> { - fn ty(&self) -> Ty<'tcx> { + pub fn ty(&self) -> Ty<'tcx> { match *self { DropGlueKind::Ty(t) | DropGlueKind::TyContents(t) => t } } - fn map_ty(&self, mut f: F) -> DropGlueKind<'tcx> where F: FnMut(Ty<'tcx>) -> Ty<'tcx> + pub fn map_ty(&self, mut f: F) -> DropGlueKind<'tcx> where F: FnMut(Ty<'tcx>) -> Ty<'tcx> { match *self { DropGlueKind::Ty(t) => DropGlueKind::Ty(f(t)), @@ -487,14 +487,13 @@ pub fn size_and_align_of_dst<'blk, 'tcx>(bcx: &BlockAndBuilder<'blk, 'tcx>, fn make_drop_glue<'blk, 'tcx>(bcx: Block<'blk, 'tcx>, v0: ValueRef, g: DropGlueKind<'tcx>) -> Block<'blk, 'tcx> { - let t = g.ty(); - if collector::collecting_debug_information(bcx.ccx()) { bcx.ccx() - .record_translation_item_as_generated(TransItem::DropGlue(bcx.tcx() - .erase_regions(&t))); + .record_translation_item_as_generated(TransItem::DropGlue(g)); } + let t = g.ty(); + let skip_dtor = match g { DropGlueKind::Ty(_) => false, DropGlueKind::TyContents(_) => true }; // NB: v0 is an *alias* of type t here, not a direct value. let _icx = push_ctxt("make_drop_glue"); diff --git a/src/librustc_trans/partitioning.rs b/src/librustc_trans/partitioning.rs index f2198c0e490f6..250292ee684de 100644 --- a/src/librustc_trans/partitioning.rs +++ b/src/librustc_trans/partitioning.rs @@ -116,8 +116,7 @@ //! source-level module, functions from the same module will be available for //! inlining, even when they are not marked #[inline]. -use collector::{InliningMap, TransItem}; -use context::CrateContext; +use collector::{TransItem, ReferenceMap}; use monomorphize; use rustc::hir::def_id::DefId; use rustc::hir::map::DefPathData; @@ -127,9 +126,28 @@ use llvm; use syntax::parse::token::{self, InternedString}; use util::nodemap::{FnvHashMap, FnvHashSet}; +#[derive(Clone, Copy, Eq, PartialEq, Debug)] +pub enum InstantiationMode { + /// This variant indicates that a translation item should be placed in some + /// codegen unit as a definition and with the given linkage. + Def(llvm::Linkage), + + /// This variant indicates that only a declaration of some translation item + /// should be placed in a given codegen unit. + Decl +} + pub struct CodegenUnit<'tcx> { pub name: InternedString, - pub items: FnvHashMap, llvm::Linkage>, + pub items: FnvHashMap, InstantiationMode>, +} + +pub enum PartitioningStrategy { + /// Generate one codegen unit per source-level module. + PerModule, + + /// Partition the whole crate into a fixed number of codegen units. + FixedUnitCount(usize) } // Anything we can't find a proper codegen unit for goes into this. @@ -137,30 +155,47 @@ const FALLBACK_CODEGEN_UNIT: &'static str = "__rustc_fallback_codegen_unit"; pub fn partition<'tcx, I>(tcx: &TyCtxt<'tcx>, trans_items: I, - inlining_map: &InliningMap<'tcx>) + strategy: PartitioningStrategy, + reference_map: &ReferenceMap<'tcx>) -> Vec> where I: Iterator> { // In the first step, we place all regular translation items into their // respective 'home' codegen unit. Regular translation items are all // functions and statics defined in the local crate. - let initial_partitioning = place_root_translation_items(tcx, trans_items); + let mut initial_partitioning = place_root_translation_items(tcx, trans_items); + + // If the partitioning should produce a fixed count of codegen units, merge + // until that count is reached. + if let PartitioningStrategy::FixedUnitCount(count) = strategy { + merge_codegen_units(&mut initial_partitioning, count, &tcx.crate_name[..]); + } // In the next step, we use the inlining map to determine which addtional // translation items have to go into each codegen unit. These additional // translation items can be drop-glue, functions from external crates, and // local functions the definition of which is marked with #[inline]. - place_inlined_translation_items(initial_partitioning, inlining_map) + let post_inlining = place_inlined_translation_items(initial_partitioning, + reference_map); + + // Now we know all *definitions* within all codegen units, thus we can + // easily determine which declarations need to be placed within each one. + let post_declarations = place_declarations(post_inlining, reference_map); + + post_declarations.0 } -struct InitialPartitioning<'tcx> { +struct PreInliningPartitioning<'tcx> { codegen_units: Vec>, roots: FnvHashSet>, } +struct PostInliningPartitioning<'tcx>(Vec>); +struct PostDeclarationsPartitioning<'tcx>(Vec>); + fn place_root_translation_items<'tcx, I>(tcx: &TyCtxt<'tcx>, trans_items: I) - -> InitialPartitioning<'tcx> + -> PreInliningPartitioning<'tcx> where I: Iterator> { let mut roots = FnvHashSet(); @@ -204,12 +239,13 @@ fn place_root_translation_items<'tcx, I>(tcx: &TyCtxt<'tcx>, } }; - codegen_unit.items.insert(trans_item, linkage); + codegen_unit.items.insert(trans_item, + InstantiationMode::Def(linkage)); roots.insert(trans_item); } } - InitialPartitioning { + PreInliningPartitioning { codegen_units: codegen_units.into_iter() .map(|(_, codegen_unit)| codegen_unit) .collect(), @@ -217,61 +253,115 @@ fn place_root_translation_items<'tcx, I>(tcx: &TyCtxt<'tcx>, } } -fn place_inlined_translation_items<'tcx>(initial_partitioning: InitialPartitioning<'tcx>, - inlining_map: &InliningMap<'tcx>) - -> Vec> { - let mut final_partitioning = Vec::new(); +fn merge_codegen_units<'tcx>(initial_partitioning: &mut PreInliningPartitioning<'tcx>, + target_cgu_count: usize, + crate_name: &str) { + if target_cgu_count >= initial_partitioning.codegen_units.len() { + return; + } + + assert!(target_cgu_count >= 1); + let codegen_units = &mut initial_partitioning.codegen_units; + + // Merge the two smallest codegen units until the target size is reached. + // Note that "size" is estimated here rather inaccurately as the number of + // translation items in a given unit. This could be improved on. + while codegen_units.len() > target_cgu_count { + // Sort small cgus to the back + codegen_units.as_mut_slice().sort_by_key(|cgu| -(cgu.items.len() as i64)); + let smallest = codegen_units.pop().unwrap(); + let second_smallest = codegen_units.last_mut().unwrap(); + + for (k, v) in smallest.items.into_iter() { + second_smallest.items.insert(k, v); + } + } + + for (index, cgu) in codegen_units.iter_mut().enumerate() { + cgu.name = token::intern_and_get_ident(&format!("{}.{}", crate_name, index)[..]); + } +} + +fn place_inlined_translation_items<'tcx>(initial_partitioning: PreInliningPartitioning<'tcx>, + reference_map: &ReferenceMap<'tcx>) + -> PostInliningPartitioning<'tcx> { + let mut new_partitioning = Vec::new(); for codegen_unit in &initial_partitioning.codegen_units[..] { // Collect all items that need to be available in this codegen unit let mut reachable = FnvHashSet(); for root in codegen_unit.items.keys() { - follow_inlining(*root, inlining_map, &mut reachable); + follow_inlining(*root, reference_map, &mut reachable); } - let mut final_codegen_unit = CodegenUnit { + let mut new_codegen_unit = CodegenUnit { name: codegen_unit.name.clone(), items: FnvHashMap(), }; // Add all translation items that are not already there for trans_item in reachable { - if let Some(linkage) = codegen_unit.items.get(&trans_item) { + if let Some(instantiation_mode) = codegen_unit.items.get(&trans_item) { // This is a root, just copy it over - final_codegen_unit.items.insert(trans_item, *linkage); + new_codegen_unit.items.insert(trans_item, *instantiation_mode); } else { if initial_partitioning.roots.contains(&trans_item) { // This item will be instantiated in some other codegen unit, // so we just add it here with AvailableExternallyLinkage - final_codegen_unit.items.insert(trans_item, llvm::AvailableExternallyLinkage); + new_codegen_unit.items.insert(trans_item, + InstantiationMode::Def(llvm::AvailableExternallyLinkage)); } else { // We can't be sure if this will also be instantiated // somewhere else, so we add an instance here with // LinkOnceODRLinkage. That way the item can be discarded if // it's not needed (inlined) after all. - final_codegen_unit.items.insert(trans_item, llvm::LinkOnceODRLinkage); + new_codegen_unit.items.insert(trans_item, + InstantiationMode::Def(llvm::LinkOnceODRLinkage)); } } } - final_partitioning.push(final_codegen_unit); + new_partitioning.push(new_codegen_unit); } - return final_partitioning; + return PostInliningPartitioning(new_partitioning); fn follow_inlining<'tcx>(trans_item: TransItem<'tcx>, - inlining_map: &InliningMap<'tcx>, + reference_map: &ReferenceMap<'tcx>, visited: &mut FnvHashSet>) { if !visited.insert(trans_item) { return; } - if let Some(inlined_items) = inlining_map.get(&trans_item) { - for &inlined_item in inlined_items { - follow_inlining(inlined_item, inlining_map, visited); + reference_map.with_inlining_candidates(trans_item, |target| { + follow_inlining(target, reference_map, visited); + }); + } +} + +fn place_declarations<'tcx>(codegen_units: PostInliningPartitioning<'tcx>, + reference_map: &ReferenceMap<'tcx>) + -> PostDeclarationsPartitioning<'tcx> { + let PostInliningPartitioning(mut codegen_units) = codegen_units; + + for codegen_unit in codegen_units.iter_mut() { + let mut declarations = FnvHashSet(); + + for (trans_item, _) in &codegen_unit.items { + for referenced_item in reference_map.get_direct_references_from(*trans_item) { + if !codegen_unit.items.contains_key(referenced_item) { + declarations.insert(*referenced_item); + } } } + + codegen_unit.items + .extend(declarations.iter() + .map(|trans_item| (*trans_item, + InstantiationMode::Decl))); } + + PostDeclarationsPartitioning(codegen_units) } fn characteristic_def_id_of_trans_item<'tcx>(tcx: &TyCtxt<'tcx>, @@ -304,7 +394,7 @@ fn characteristic_def_id_of_trans_item<'tcx>(tcx: &TyCtxt<'tcx>, Some(instance.def) } - TransItem::DropGlue(t) => characteristic_def_id_of_type(t), + TransItem::DropGlue(dg) => characteristic_def_id_of_type(dg.ty()), TransItem::Static(node_id) => Some(tcx.map.local_def_id(node_id)), } } @@ -340,24 +430,3 @@ fn compute_codegen_unit_name<'tcx>(tcx: &TyCtxt<'tcx>, return token::intern_and_get_ident(&mod_path[..]); } - -impl<'tcx> CodegenUnit<'tcx> { - pub fn _dump<'a>(&self, ccx: &CrateContext<'a, 'tcx>) { - println!("CodegenUnit {} (", self.name); - - let mut items: Vec<_> = self.items - .iter() - .map(|(trans_item, inst)| { - format!("{} -- ({:?})", trans_item.to_string(ccx), inst) - }) - .collect(); - - items.as_mut_slice().sort(); - - for s in items { - println!(" {}", s); - } - - println!(")"); - } -} diff --git a/src/test/codegen-units/item-collection/generic-drop-glue.rs b/src/test/codegen-units/item-collection/generic-drop-glue.rs index 476c84044e686..6da8154540574 100644 --- a/src/test/codegen-units/item-collection/generic-drop-glue.rs +++ b/src/test/codegen-units/item-collection/generic-drop-glue.rs @@ -46,19 +46,22 @@ struct NonGenericNoDrop(i32); struct NonGenericWithDrop(i32); //~ TRANS_ITEM drop-glue generic_drop_glue::NonGenericWithDrop[0] +//~ TRANS_ITEM drop-glue-contents generic_drop_glue::NonGenericWithDrop[0] impl Drop for NonGenericWithDrop { + //~ TRANS_ITEM fn generic_drop_glue::{{impl}}[2]::drop[0] fn drop(&mut self) {} -//~ TRANS_ITEM fn generic_drop_glue::{{impl}}[2]::drop[0] } //~ TRANS_ITEM fn generic_drop_glue::main[0] fn main() { //~ TRANS_ITEM drop-glue generic_drop_glue::StructWithDrop[0] + //~ TRANS_ITEM drop-glue-contents generic_drop_glue::StructWithDrop[0] //~ TRANS_ITEM fn generic_drop_glue::{{impl}}[0]::drop[0] let _ = StructWithDrop { x: 0i8, y: 'a' }.x; //~ TRANS_ITEM drop-glue generic_drop_glue::StructWithDrop[0]<&str, generic_drop_glue::NonGenericNoDrop[0]> + //~ TRANS_ITEM drop-glue-contents generic_drop_glue::StructWithDrop[0]<&str, generic_drop_glue::NonGenericNoDrop[0]> //~ TRANS_ITEM fn generic_drop_glue::{{impl}}[0]::drop[0]<&str, generic_drop_glue::NonGenericNoDrop[0]> let _ = StructWithDrop { x: "&str", y: NonGenericNoDrop(0) }.y; @@ -71,6 +74,7 @@ fn main() { let _ = StructNoDrop { x: NonGenericWithDrop(0), y: 0f64 }.y; //~ TRANS_ITEM drop-glue generic_drop_glue::EnumWithDrop[0] + //~ TRANS_ITEM drop-glue-contents generic_drop_glue::EnumWithDrop[0] //~ TRANS_ITEM fn generic_drop_glue::{{impl}}[1]::drop[0] let _ = match EnumWithDrop::A::(0) { EnumWithDrop::A(x) => x, @@ -78,6 +82,7 @@ fn main() { }; //~ TRANS_ITEM drop-glue generic_drop_glue::EnumWithDrop[0] + //~ TRANS_ITEM drop-glue-contents generic_drop_glue::EnumWithDrop[0] //~ TRANS_ITEM fn generic_drop_glue::{{impl}}[1]::drop[0] let _ = match EnumWithDrop::B::(1.0) { EnumWithDrop::A(x) => x, diff --git a/src/test/codegen-units/item-collection/non-generic-drop-glue.rs b/src/test/codegen-units/item-collection/non-generic-drop-glue.rs index bd8b0c605aecf..91be81a0b8996 100644 --- a/src/test/codegen-units/item-collection/non-generic-drop-glue.rs +++ b/src/test/codegen-units/item-collection/non-generic-drop-glue.rs @@ -14,6 +14,7 @@ #![deny(dead_code)] //~ TRANS_ITEM drop-glue non_generic_drop_glue::StructWithDrop[0] +//~ TRANS_ITEM drop-glue-contents non_generic_drop_glue::StructWithDrop[0] struct StructWithDrop { x: i32 } @@ -28,6 +29,7 @@ struct StructNoDrop { } //~ TRANS_ITEM drop-glue non_generic_drop_glue::EnumWithDrop[0] +//~ TRANS_ITEM drop-glue-contents non_generic_drop_glue::EnumWithDrop[0] enum EnumWithDrop { A(i32) } diff --git a/src/test/codegen-units/item-collection/transitive-drop-glue.rs b/src/test/codegen-units/item-collection/transitive-drop-glue.rs index 21bb29199a685..81a7059fe209f 100644 --- a/src/test/codegen-units/item-collection/transitive-drop-glue.rs +++ b/src/test/codegen-units/item-collection/transitive-drop-glue.rs @@ -18,6 +18,7 @@ struct Root(Intermediate); //~ TRANS_ITEM drop-glue transitive_drop_glue::Intermediate[0] struct Intermediate(Leaf); //~ TRANS_ITEM drop-glue transitive_drop_glue::Leaf[0] +//~ TRANS_ITEM drop-glue-contents transitive_drop_glue::Leaf[0] struct Leaf; impl Drop for Leaf { @@ -25,11 +26,8 @@ impl Drop for Leaf { fn drop(&mut self) {} } -//~ TRANS_ITEM drop-glue transitive_drop_glue::Root[0] struct RootGen(IntermediateGen); -//~ TRANS_ITEM drop-glue transitive_drop_glue::Root[0] struct IntermediateGen(LeafGen); -//~ TRANS_ITEM drop-glue transitive_drop_glue::Root[0] struct LeafGen(T); impl Drop for LeafGen { @@ -44,12 +42,14 @@ fn main() { //~ TRANS_ITEM drop-glue transitive_drop_glue::RootGen[0] //~ TRANS_ITEM drop-glue transitive_drop_glue::IntermediateGen[0] //~ TRANS_ITEM drop-glue transitive_drop_glue::LeafGen[0] + //~ TRANS_ITEM drop-glue-contents transitive_drop_glue::LeafGen[0] //~ TRANS_ITEM fn transitive_drop_glue::{{impl}}[1]::drop[0] let _ = RootGen(IntermediateGen(LeafGen(0u32))); //~ TRANS_ITEM drop-glue transitive_drop_glue::RootGen[0] //~ TRANS_ITEM drop-glue transitive_drop_glue::IntermediateGen[0] //~ TRANS_ITEM drop-glue transitive_drop_glue::LeafGen[0] + //~ TRANS_ITEM drop-glue-contents transitive_drop_glue::LeafGen[0] //~ TRANS_ITEM fn transitive_drop_glue::{{impl}}[1]::drop[0] let _ = RootGen(IntermediateGen(LeafGen(0i16))); } diff --git a/src/test/codegen-units/item-collection/tuple-drop-glue.rs b/src/test/codegen-units/item-collection/tuple-drop-glue.rs index 1bc235de88e1f..ef4bc1dca594c 100644 --- a/src/test/codegen-units/item-collection/tuple-drop-glue.rs +++ b/src/test/codegen-units/item-collection/tuple-drop-glue.rs @@ -14,6 +14,7 @@ #![deny(dead_code)] //~ TRANS_ITEM drop-glue tuple_drop_glue::Dropped[0] +//~ TRANS_ITEM drop-glue-contents tuple_drop_glue::Dropped[0] struct Dropped; impl Drop for Dropped { diff --git a/src/test/codegen-units/partitioning/extern-drop-glue.rs b/src/test/codegen-units/partitioning/extern-drop-glue.rs index ddf5f461aef09..be41232ffbab0 100644 --- a/src/test/codegen-units/partitioning/extern-drop-glue.rs +++ b/src/test/codegen-units/partitioning/extern-drop-glue.rs @@ -9,7 +9,7 @@ // except according to those terms. // ignore-tidy-linelength -// compile-flags:-Zprint-trans-items=lazy +// compile-flags:-Zprint-trans-items=lazy -Zincremental=tmp #![allow(dead_code)] #![crate_type="lib"] @@ -18,6 +18,7 @@ extern crate cgu_extern_drop_glue; //~ TRANS_ITEM drop-glue cgu_extern_drop_glue::Struct[0] @@ extern_drop_glue[OnceODR] extern_drop_glue-mod1[OnceODR] +//~ TRANS_ITEM drop-glue-contents cgu_extern_drop_glue::Struct[0] @@ extern_drop_glue[OnceODR] extern_drop_glue-mod1[OnceODR] struct LocalStruct(cgu_extern_drop_glue::Struct); @@ -40,4 +41,3 @@ mod mod1 { let _ = LocalStruct(cgu_extern_drop_glue::Struct(0)); } } - diff --git a/src/test/codegen-units/partitioning/extern-generic.rs b/src/test/codegen-units/partitioning/extern-generic.rs index 71af676b0a970..cd6f4619f9c80 100644 --- a/src/test/codegen-units/partitioning/extern-generic.rs +++ b/src/test/codegen-units/partitioning/extern-generic.rs @@ -9,7 +9,7 @@ // except according to those terms. // ignore-tidy-linelength -// compile-flags:-Zprint-trans-items=eager +// compile-flags:-Zprint-trans-items=eager -Zincremental=tmp #![allow(dead_code)] #![crate_type="lib"] diff --git a/src/test/codegen-units/partitioning/inlining-from-extern-crate.rs b/src/test/codegen-units/partitioning/inlining-from-extern-crate.rs index 469f2c08c39c2..7b92cf3f7cd52 100644 --- a/src/test/codegen-units/partitioning/inlining-from-extern-crate.rs +++ b/src/test/codegen-units/partitioning/inlining-from-extern-crate.rs @@ -9,7 +9,7 @@ // except according to those terms. // ignore-tidy-linelength -// compile-flags:-Zprint-trans-items=lazy +// compile-flags:-Zprint-trans-items=lazy -Zincremental=tmp #![crate_type="lib"] diff --git a/src/test/codegen-units/partitioning/local-drop-glue.rs b/src/test/codegen-units/partitioning/local-drop-glue.rs index 0c500bb64f8db..f3a973fc47851 100644 --- a/src/test/codegen-units/partitioning/local-drop-glue.rs +++ b/src/test/codegen-units/partitioning/local-drop-glue.rs @@ -9,18 +9,19 @@ // except according to those terms. // ignore-tidy-linelength -// compile-flags:-Zprint-trans-items=lazy +// compile-flags:-Zprint-trans-items=lazy -Zincremental=tmp #![allow(dead_code)] #![crate_type="lib"] //~ TRANS_ITEM drop-glue local_drop_glue::Struct[0] @@ local_drop_glue[OnceODR] local_drop_glue-mod1[OnceODR] +//~ TRANS_ITEM drop-glue-contents local_drop_glue::Struct[0] @@ local_drop_glue[OnceODR] local_drop_glue-mod1[OnceODR] struct Struct { _a: u32 } impl Drop for Struct { - //~ TRANS_ITEM fn local_drop_glue::{{impl}}[0]::drop[0] @@ local_drop_glue[WeakODR] + //~ TRANS_ITEM fn local_drop_glue::{{impl}}[0]::drop[0] @@ local_drop_glue[WeakODR] local_drop_glue-mod1[Declaration] fn drop(&mut self) {} } diff --git a/src/test/codegen-units/partitioning/local-generic.rs b/src/test/codegen-units/partitioning/local-generic.rs index 08c8ff0cb2f91..dd18874baa32b 100644 --- a/src/test/codegen-units/partitioning/local-generic.rs +++ b/src/test/codegen-units/partitioning/local-generic.rs @@ -9,7 +9,7 @@ // except according to those terms. // ignore-tidy-linelength -// compile-flags:-Zprint-trans-items=eager +// compile-flags:-Zprint-trans-items=eager -Zincremental=tmp #![allow(dead_code)] #![crate_type="lib"] @@ -17,10 +17,10 @@ // Used in different modules/codegen units but always instantiated in the same // codegen unit. -//~ TRANS_ITEM fn local_generic::generic[0] @@ local_generic.volatile[WeakODR] -//~ TRANS_ITEM fn local_generic::generic[0] @@ local_generic.volatile[WeakODR] -//~ TRANS_ITEM fn local_generic::generic[0] @@ local_generic.volatile[WeakODR] -//~ TRANS_ITEM fn local_generic::generic[0]<&str> @@ local_generic.volatile[WeakODR] +//~ TRANS_ITEM fn local_generic::generic[0] @@ local_generic.volatile[WeakODR] local_generic[Declaration] +//~ TRANS_ITEM fn local_generic::generic[0] @@ local_generic.volatile[WeakODR] local_generic-mod1[Declaration] +//~ TRANS_ITEM fn local_generic::generic[0] @@ local_generic.volatile[WeakODR] local_generic-mod1-mod1[Declaration] +//~ TRANS_ITEM fn local_generic::generic[0]<&str> @@ local_generic.volatile[WeakODR] local_generic-mod2[Declaration] pub fn generic(x: T) -> T { x } //~ TRANS_ITEM fn local_generic::user[0] @@ local_generic[WeakODR] diff --git a/src/test/codegen-units/partitioning/local-inlining.rs b/src/test/codegen-units/partitioning/local-inlining.rs index d2bfa83834665..f493c8ed63f74 100644 --- a/src/test/codegen-units/partitioning/local-inlining.rs +++ b/src/test/codegen-units/partitioning/local-inlining.rs @@ -9,7 +9,7 @@ // except according to those terms. // ignore-tidy-linelength -// compile-flags:-Zprint-trans-items=lazy +// compile-flags:-Zprint-trans-items=lazy -Zincremental=tmp #![allow(dead_code)] #![crate_type="lib"] diff --git a/src/test/codegen-units/partitioning/local-transitive-inlining.rs b/src/test/codegen-units/partitioning/local-transitive-inlining.rs index 2e47dc5c9020f..31d99e08b6530 100644 --- a/src/test/codegen-units/partitioning/local-transitive-inlining.rs +++ b/src/test/codegen-units/partitioning/local-transitive-inlining.rs @@ -9,7 +9,7 @@ // except according to those terms. // ignore-tidy-linelength -// compile-flags:-Zprint-trans-items=lazy +// compile-flags:-Zprint-trans-items=lazy -Zincremental=tmp #![allow(dead_code)] #![crate_type="lib"] diff --git a/src/test/codegen-units/partitioning/methods-are-with-self-type.rs b/src/test/codegen-units/partitioning/methods-are-with-self-type.rs index 51d2d53f24a80..6d04545f93d6e 100644 --- a/src/test/codegen-units/partitioning/methods-are-with-self-type.rs +++ b/src/test/codegen-units/partitioning/methods-are-with-self-type.rs @@ -9,7 +9,7 @@ // except according to those terms. // ignore-tidy-linelength -// compile-flags:-Zprint-trans-items=lazy +// compile-flags:-Zprint-trans-items=lazy -Zincremental=tmp #![allow(dead_code)] @@ -59,19 +59,19 @@ mod type2 { //~ TRANS_ITEM fn methods_are_with_self_type::main[0] fn main() { - //~ TRANS_ITEM fn methods_are_with_self_type::mod1[0]::{{impl}}[1]::method[0] @@ methods_are_with_self_type.volatile[WeakODR] + //~ TRANS_ITEM fn methods_are_with_self_type::mod1[0]::{{impl}}[1]::method[0] @@ methods_are_with_self_type.volatile[WeakODR] methods_are_with_self_type[Declaration] SomeGenericType(0u32, 0u64).method(); - //~ TRANS_ITEM fn methods_are_with_self_type::mod1[0]::{{impl}}[1]::associated_fn[0] @@ methods_are_with_self_type.volatile[WeakODR] + //~ TRANS_ITEM fn methods_are_with_self_type::mod1[0]::{{impl}}[1]::associated_fn[0] @@ methods_are_with_self_type.volatile[WeakODR] methods_are_with_self_type[Declaration] SomeGenericType::associated_fn('c', "&str"); - //~ TRANS_ITEM fn methods_are_with_self_type::{{impl}}[0]::foo[0] @@ methods_are_with_self_type-type1.volatile[WeakODR] + //~ TRANS_ITEM fn methods_are_with_self_type::{{impl}}[0]::foo[0] @@ methods_are_with_self_type-type1.volatile[WeakODR] methods_are_with_self_type[Declaration] type1::Struct.foo(); - //~ TRANS_ITEM fn methods_are_with_self_type::{{impl}}[0]::foo[0] @@ methods_are_with_self_type-type2.volatile[WeakODR] + //~ TRANS_ITEM fn methods_are_with_self_type::{{impl}}[0]::foo[0] @@ methods_are_with_self_type-type2.volatile[WeakODR] methods_are_with_self_type[Declaration] type2::Struct.foo(); - //~ TRANS_ITEM fn methods_are_with_self_type::Trait[0]::default[0] @@ methods_are_with_self_type-type1.volatile[WeakODR] + //~ TRANS_ITEM fn methods_are_with_self_type::Trait[0]::default[0] @@ methods_are_with_self_type-type1.volatile[WeakODR] methods_are_with_self_type[Declaration] type1::Struct.default(); - //~ TRANS_ITEM fn methods_are_with_self_type::Trait[0]::default[0] @@ methods_are_with_self_type-type2.volatile[WeakODR] + //~ TRANS_ITEM fn methods_are_with_self_type::Trait[0]::default[0] @@ methods_are_with_self_type-type2.volatile[WeakODR] methods_are_with_self_type[Declaration] type2::Struct.default(); } diff --git a/src/test/codegen-units/partitioning/regular-modules.rs b/src/test/codegen-units/partitioning/regular-modules.rs index a761cab2e44e4..69c9bd0a12ae4 100644 --- a/src/test/codegen-units/partitioning/regular-modules.rs +++ b/src/test/codegen-units/partitioning/regular-modules.rs @@ -9,7 +9,7 @@ // except according to those terms. // ignore-tidy-linelength -// compile-flags:-Zprint-trans-items=eager +// compile-flags:-Zprint-trans-items=eager -Z incremental=tmp #![allow(dead_code)] #![crate_type="lib"] diff --git a/src/test/codegen-units/partitioning/statics.rs b/src/test/codegen-units/partitioning/statics.rs index ac6a0c55d4f6d..e853ccaff0dd5 100644 --- a/src/test/codegen-units/partitioning/statics.rs +++ b/src/test/codegen-units/partitioning/statics.rs @@ -9,7 +9,7 @@ // except according to those terms. // ignore-tidy-linelength -// compile-flags:-Zprint-trans-items=lazy +// compile-flags:-Zprint-trans-items=lazy -Zincremental=tmp #![crate_type="lib"]