Skip to content

Commit

Permalink
Serialize additional data for procedural macros
Browse files Browse the repository at this point in the history 
Split off from #62855

This PR deerializes the declaration `Span` and attributes for all
procedural macros from their underlying function definitions.
This allows Rustdoc to properly render doc comments
and source links when inlining procedural macros across crates
  • Loading branch information
@Aaron1011
Aaron1011 committed Aug 16, 2019
1 parent bdfd698 commit 73d7719
Show file tree
Hide file tree
Showing 17 changed files with 238 additions and 212 deletions.
8 changes: 8 additions & 0 deletions src/libproc_macro/bridge/client.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -468,6 +468,14 @@ pub enum ProcMacro {
}

impl ProcMacro {
pub fn name(&self) -> &'static str {
match self {
ProcMacro::CustomDerive { trait_name, .. } => trait_name,
ProcMacro::Attr { name, .. } => name,
ProcMacro::Bang { name, ..} => name
}
}

pub const fn custom_derive(
trait_name: &'static str,
attributes: &'static [&'static str],
Expand Down
26 changes: 0 additions & 26 deletions src/librustc/hir/def_id.rs
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
use crate::ty::{self, TyCtxt};
use crate::hir::map::definitions::FIRST_FREE_DEF_INDEX;
use rustc_data_structures::indexed_vec::Idx;
use std::fmt;
use std::u32;
Expand Down Expand Up @@ -102,31 +101,6 @@ newtype_index! {
}
}

impl DefIndex {
// Proc macros from a proc-macro crate have a kind of virtual DefIndex. This
// function maps the index of the macro within the crate (which is also the
// index of the macro in the CrateMetadata::proc_macros array) to the
// corresponding DefIndex.
pub fn from_proc_macro_index(proc_macro_index: usize) -> DefIndex {
// DefIndex for proc macros start from FIRST_FREE_DEF_INDEX,
// because the first FIRST_FREE_DEF_INDEX indexes are reserved
// for internal use.
let def_index = DefIndex::from(
proc_macro_index.checked_add(FIRST_FREE_DEF_INDEX)
.expect("integer overflow adding `proc_macro_index`"));
assert!(def_index != CRATE_DEF_INDEX);
def_index
}

// This function is the reverse of from_proc_macro_index() above.
pub fn to_proc_macro_index(self: DefIndex) -> usize {
self.index().checked_sub(FIRST_FREE_DEF_INDEX)
.unwrap_or_else(|| {
bug!("using local index {:?} as proc-macro index", self)
})
}
}

impl rustc_serialize::UseSpecializedEncodable for DefIndex {}
impl rustc_serialize::UseSpecializedDecodable for DefIndex {}

Expand Down
19 changes: 0 additions & 19 deletions src/librustc/hir/map/definitions.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -411,10 +411,6 @@ impl Definitions {
}

/// Adds a root definition (no parent) and a few other reserved definitions.
///
/// After the initial definitions are created the first `FIRST_FREE_DEF_INDEX` indexes
/// are taken, so the "user" indexes will be allocated starting with `FIRST_FREE_DEF_INDEX`
/// in ascending order.
pub fn create_root_def(&mut self,
crate_name: &str,
crate_disambiguator: CrateDisambiguator)
Expand Down Expand Up @@ -589,19 +585,6 @@ impl DefPathData {
}
}

/// Evaluates to the number of tokens passed to it.
///
/// Logarithmic counting: every one or two recursive expansions, the number of
/// tokens to count is divided by two, instead of being reduced by one.
/// Therefore, the recursion depth is the binary logarithm of the number of
/// tokens to count, and the expanded tree is likewise very small.
macro_rules! count {
() => (0usize);
($one:tt) => (1usize);
($($pairs:tt $_p:tt)*) => (count!($($pairs)*) << 1usize);
($odd:tt $($rest:tt)*) => (count!($($rest)*) | 1usize);
}

// We define the GlobalMetaDataKind enum with this macro because we want to
// make sure that we exhaustively iterate over all variants when registering
// the corresponding DefIndices in the DefTable.
Expand All @@ -614,8 +597,6 @@ macro_rules! define_global_metadata_kind {
$($variant),*
}

pub const FIRST_FREE_DEF_INDEX: usize = 1 + count!($($variant)*);

impl GlobalMetaDataKind {
fn allocate_def_indices(definitions: &mut Definitions) {
$({
Expand Down
91 changes: 25 additions & 66 deletions src/librustc_metadata/creader.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,7 @@

use crate::cstore::{self, CStore, CrateSource, MetadataBlob};
use crate::locator::{self, CratePaths};
use crate::decoder::proc_macro_def_path_table;
use crate::schema::CrateRoot;
use crate::schema::{CrateRoot};
use rustc_data_structures::sync::{Lrc, RwLock, Lock};

use rustc::hir::def_id::CrateNum;
Expand All @@ -26,11 +25,11 @@ use std::{cmp, fs};
use syntax::ast;
use syntax::attr;
use syntax::ext::allocator::{global_allocator_spans, AllocatorKind};
use syntax::ext::base::{SyntaxExtension, SyntaxExtensionKind};
use syntax::symbol::{Symbol, sym};
use syntax::{span_err, span_fatal};
use syntax_pos::{Span, DUMMY_SP};
use log::{debug, info, log_enabled};
use proc_macro::bridge::client::ProcMacro;

pub struct Library {
pub dylib: Option<(PathBuf, PathKind)>,
Expand Down Expand Up @@ -230,24 +229,13 @@ impl<'a> CrateLoader<'a> {

let dependencies: Vec<CrateNum> = cnum_map.iter().cloned().collect();

let proc_macros = crate_root.proc_macro_decls_static.map(|_| {
let raw_proc_macros = crate_root.proc_macro_data.map(|_| {
if self.sess.opts.debugging_opts.dual_proc_macros {
let host_lib = host_lib.unwrap();
self.load_derive_macros(
&host_lib.metadata.get_root(),
host_lib.dylib.map(|p| p.0),
span
)
let host_lib = host_lib.as_ref().unwrap();
self.dlsym_proc_macros(host_lib.dylib.as_ref().map(|p| p.0.clone()),
&host_lib.metadata.get_root(), span)
} else {
self.load_derive_macros(&crate_root, dylib.clone().map(|p| p.0), span)
}
});

let def_path_table = record_time(&self.sess.perf_stats.decode_def_path_tables_time, || {
if let Some(proc_macros) = &proc_macros {
proc_macro_def_path_table(&crate_root, proc_macros)
} else {
crate_root.def_path_table.decode((&metadata, self.sess))
self.dlsym_proc_macros(dylib.clone().map(|p| p.0), &crate_root, span)
}
});

Expand All @@ -260,13 +248,16 @@ impl<'a> CrateLoader<'a> {
.map(|trait_impls| (trait_impls.trait_id, trait_impls.impls))
.collect();

let def_path_table = record_time(&self.sess.perf_stats.decode_def_path_tables_time, || {
crate_root.def_path_table.decode((&metadata, self.sess))
});

let cmeta = cstore::CrateMetadata {
name: crate_root.name,
imported_name: ident,
extern_crate: Lock::new(None),
def_path_table: Lrc::new(def_path_table),
trait_impls,
proc_macros,
root: crate_root,
blob: metadata,
cnum_map,
Expand All @@ -280,7 +271,10 @@ impl<'a> CrateLoader<'a> {
rlib,
rmeta,
},
private_dep
private_dep,
span,
host_lib,
raw_proc_macros
};

let cmeta = Lrc::new(cmeta);
Expand All @@ -300,8 +294,6 @@ impl<'a> CrateLoader<'a> {
// message we emit
let mut proc_macro_locator = locate_ctxt.clone();

// Try to load a proc macro
proc_macro_locator.is_proc_macro = Some(true);

// Load the proc macro crate for the target
let (locator, target_result) = if self.sess.opts.debugging_opts.dual_proc_macros {
Expand Down Expand Up @@ -389,7 +381,7 @@ impl<'a> CrateLoader<'a> {
match result {
(LoadResult::Previous(cnum), None) => {
let data = self.cstore.get_crate_data(cnum);
if data.root.proc_macro_decls_static.is_some() {
if data.root.proc_macro_data.is_some() {
dep_kind = DepKind::UnexportedMacrosOnly;
}
data.dep_kind.with_lock(|data_dep_kind| {
Expand Down Expand Up @@ -482,7 +474,7 @@ impl<'a> CrateLoader<'a> {
dep_kind: DepKind)
-> cstore::CrateNumMap {
debug!("resolving deps of external crate");
if crate_root.proc_macro_decls_static.is_some() {
if crate_root.proc_macro_data.is_some() {
return cstore::CrateNumMap::new();
}

Expand Down Expand Up @@ -574,19 +566,13 @@ impl<'a> CrateLoader<'a> {
}
}

/// Loads custom derive macros.
///
/// Note that this is intentionally similar to how we load plugins today,
/// but also intentionally separate. Plugins are likely always going to be
/// implemented as dynamic libraries, but we have a possible future where
/// custom derive (and other macro-1.1 style features) are implemented via
/// executables and custom IPC.
fn load_derive_macros(&mut self, root: &CrateRoot<'_>, dylib: Option<PathBuf>, span: Span)
-> Vec<(ast::Name, Lrc<SyntaxExtension>)> {
use std::{env, mem};
fn dlsym_proc_macros(&self,
dylib: Option<PathBuf>,
root: &CrateRoot<'_>,
span: Span
) -> &'static [ProcMacro] {
use std::env;
use crate::dynamic_lib::DynamicLibrary;
use proc_macro::bridge::client::ProcMacro;
use syntax::ext::proc_macro::{BangProcMacro, AttrProcMacro, ProcMacroDerive};

let path = match dylib {
Some(dylib) => dylib,
Expand All @@ -608,38 +594,11 @@ impl<'a> CrateLoader<'a> {
*(sym as *const &[ProcMacro])
};

let extensions = decls.iter().map(|&decl| {
let (name, kind, helper_attrs) = match decl {
ProcMacro::CustomDerive { trait_name, attributes, client } => {
let helper_attrs =
attributes.iter().cloned().map(Symbol::intern).collect::<Vec<_>>();
(
trait_name,
SyntaxExtensionKind::Derive(Box::new(ProcMacroDerive {
client, attrs: helper_attrs.clone()
})),
helper_attrs,
)
}
ProcMacro::Attr { name, client } => (
name, SyntaxExtensionKind::Attr(Box::new(AttrProcMacro { client })), Vec::new()
),
ProcMacro::Bang { name, client } => (
name, SyntaxExtensionKind::Bang(Box::new(BangProcMacro { client })), Vec::new()
)
};

(Symbol::intern(name), Lrc::new(SyntaxExtension {
helper_attrs,
..SyntaxExtension::default(kind, root.edition)
}))
}).collect();

// Intentionally leak the dynamic library. We can't ever unload it
// since the library can make things that will live arbitrarily long.
mem::forget(lib);
std::mem::forget(lib);

extensions
decls
}

/// Look for a plugin registrar. Returns library path, crate
Expand Down
17 changes: 14 additions & 3 deletions src/librustc_metadata/cstore.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,9 @@ pub use crate::cstore_impl::{provide, provide_extern};
pub type CrateNumMap = IndexVec<CrateNum, CrateNum>;

pub use rustc_data_structures::sync::MetadataRef;
use crate::creader::Library;
use syntax_pos::Span;
use proc_macro::bridge::client::ProcMacro;

pub struct MetadataBlob(pub MetadataRef);

Expand Down Expand Up @@ -82,11 +85,19 @@ pub struct CrateMetadata {
pub dep_kind: Lock<DepKind>,
pub source: CrateSource,

pub proc_macros: Option<Vec<(ast::Name, Lrc<SyntaxExtension>)>>,

/// Whether or not this crate should be consider a private dependency
/// for purposes of the 'exported_private_dependencies' lint
pub private_dep: bool
pub private_dep: bool,

pub host_lib: Option<Library>,
pub span: Span,

pub raw_proc_macros: Option<&'static [ProcMacro]>,
}

pub struct FullProcMacro {
pub name: ast::Name,
pub ext: Lrc<SyntaxExtension>
}

pub struct CStore {
Expand Down
7 changes: 4 additions & 3 deletions src/librustc_metadata/cstore_impl.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -426,8 +426,8 @@ impl cstore::CStore {

pub fn load_macro_untracked(&self, id: DefId, sess: &Session) -> LoadedMacro {
let data = self.get_crate_data(id.krate);
if let Some(ref proc_macros) = data.proc_macros {
return LoadedMacro::ProcMacro(proc_macros[id.index.to_proc_macro_index()].1.clone());
if data.is_proc_macro_crate() {
return LoadedMacro::ProcMacro(data.get_proc_macro(id.index, sess).ext);
} else if data.name == sym::proc_macro && data.item_name(id.index) == sym::quote {
let client = proc_macro::bridge::client::Client::expand1(proc_macro::quote);
let kind = SyntaxExtensionKind::Bang(Box::new(BangProcMacro { client }));
Expand All @@ -439,7 +439,8 @@ impl cstore::CStore {
}

let def = data.get_macro(id.index);
let macro_full_name = data.def_path(id.index).to_string_friendly(|_| data.imported_name);
let macro_full_name = data.def_path(id.index)
.to_string_friendly(|_| data.imported_name);
let source_name = FileName::Macros(macro_full_name);

let source_file = sess.parse_sess.source_map().new_source_file(source_name, def.body);
Expand Down
Loading

0 comments on commit 73d7719

Please sign in to comment.