Skip to content

Commit

Permalink
Rollup merge of rust-lang#37569 - jseyfried:improve_expansion_perf, r…
Browse files Browse the repository at this point in the history
…=eddyb

macros: improve expansion performance

This PR fixes that regression, further improves performance on recursive, `tt`-heavy workloads, and makes a variety of other improvements to parsing and expansion performance.

Expansion performance improvements:

| Test case      | Run-time | Memory usage |
| -------------- | -------- | ------------ |
| libsyntax      | 8%       | 10%          |
| librustc       | 15%      | 6%           |
| librustc_trans | 30%      | 6%           |
| rust-lang#37074         | 20%      | 15%          |
| rust-lang#34630         | 40%      | 8%           |

r? @eddyb
  • Loading branch information
alexcrichton committed Nov 5, 2016
2 parents 9f882b9 + 51104e5 commit b9cbbe1
Show file tree
Hide file tree
Showing 16 changed files with 353 additions and 448 deletions.
54 changes: 24 additions & 30 deletions src/librustc/hir/lowering.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1208,36 +1208,30 @@ impl<'a> LoweringContext<'a> {
ExprKind::Break(opt_ident) => hir::ExprBreak(self.lower_opt_sp_ident(opt_ident)),
ExprKind::Continue(opt_ident) => hir::ExprAgain(self.lower_opt_sp_ident(opt_ident)),
ExprKind::Ret(ref e) => hir::ExprRet(e.as_ref().map(|x| self.lower_expr(x))),
ExprKind::InlineAsm(InlineAsm {
ref inputs,
ref outputs,
ref asm,
asm_str_style,
ref clobbers,
volatile,
alignstack,
dialect,
expn_id,
}) => hir::ExprInlineAsm(P(hir::InlineAsm {
inputs: inputs.iter().map(|&(ref c, _)| c.clone()).collect(),
outputs: outputs.iter()
.map(|out| {
hir::InlineAsmOutput {
constraint: out.constraint.clone(),
is_rw: out.is_rw,
is_indirect: out.is_indirect,
}
})
.collect(),
asm: asm.clone(),
asm_str_style: asm_str_style,
clobbers: clobbers.clone().into(),
volatile: volatile,
alignstack: alignstack,
dialect: dialect,
expn_id: expn_id,
}), outputs.iter().map(|out| self.lower_expr(&out.expr)).collect(),
inputs.iter().map(|&(_, ref input)| self.lower_expr(input)).collect()),
ExprKind::InlineAsm(ref asm) => {
let hir_asm = hir::InlineAsm {
inputs: asm.inputs.iter().map(|&(ref c, _)| c.clone()).collect(),
outputs: asm.outputs.iter().map(|out| {
hir::InlineAsmOutput {
constraint: out.constraint.clone(),
is_rw: out.is_rw,
is_indirect: out.is_indirect,
}
}).collect(),
asm: asm.asm.clone(),
asm_str_style: asm.asm_str_style,
clobbers: asm.clobbers.clone().into(),
volatile: asm.volatile,
alignstack: asm.alignstack,
dialect: asm.dialect,
expn_id: asm.expn_id,
};
let outputs =
asm.outputs.iter().map(|out| self.lower_expr(&out.expr)).collect();
let inputs =
asm.inputs.iter().map(|&(_, ref input)| self.lower_expr(input)).collect();
hir::ExprInlineAsm(P(hir_asm), outputs, inputs)
}
ExprKind::Struct(ref path, ref fields, ref maybe_expr) => {
hir::ExprStruct(self.lower_path(path),
fields.iter().map(|x| self.lower_field(x)).collect(),
Expand Down
2 changes: 1 addition & 1 deletion src/libsyntax/ast.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1050,7 +1050,7 @@ pub enum ExprKind {
Ret(Option<P<Expr>>),

/// Output of the `asm!()` macro
InlineAsm(InlineAsm),
InlineAsm(P<InlineAsm>),

/// A macro invocation; pre-expansion
Mac(Mac),
Expand Down
4 changes: 3 additions & 1 deletion src/libsyntax/ext/base.rs
Original file line number Diff line number Diff line change
Expand Up @@ -615,7 +615,9 @@ impl<'a> ExtCtxt<'a> {

pub fn new_parser_from_tts(&self, tts: &[tokenstream::TokenTree])
-> parser::Parser<'a> {
parse::tts_to_parser(self.parse_sess, tts.to_vec())
let mut parser = parse::tts_to_parser(self.parse_sess, tts.to_vec());
parser.allow_interpolated_tts = false; // FIXME(jseyfried) `quote!` can't handle these yet
parser
}
pub fn codemap(&self) -> &'a CodeMap { self.parse_sess.codemap() }
pub fn parse_sess(&self) -> &'a parse::ParseSess { self.parse_sess }
Expand Down
54 changes: 32 additions & 22 deletions src/libsyntax/ext/quote.rs
Original file line number Diff line number Diff line change
Expand Up @@ -80,67 +80,71 @@ pub mod rt {

impl ToTokens for ast::Path {
fn to_tokens(&self, _cx: &ExtCtxt) -> Vec<TokenTree> {
vec![TokenTree::Token(DUMMY_SP,
token::Interpolated(token::NtPath(Box::new(self.clone()))))]
let nt = token::NtPath(self.clone());
vec![TokenTree::Token(DUMMY_SP, token::Interpolated(Rc::new(nt)))]
}
}

impl ToTokens for ast::Ty {
fn to_tokens(&self, _cx: &ExtCtxt) -> Vec<TokenTree> {
vec![TokenTree::Token(self.span, token::Interpolated(token::NtTy(P(self.clone()))))]
let nt = token::NtTy(P(self.clone()));
vec![TokenTree::Token(self.span, token::Interpolated(Rc::new(nt)))]
}
}

impl ToTokens for ast::Block {
fn to_tokens(&self, _cx: &ExtCtxt) -> Vec<TokenTree> {
vec![TokenTree::Token(self.span, token::Interpolated(token::NtBlock(P(self.clone()))))]
let nt = token::NtBlock(P(self.clone()));
vec![TokenTree::Token(self.span, token::Interpolated(Rc::new(nt)))]
}
}

impl ToTokens for ast::Generics {
fn to_tokens(&self, _cx: &ExtCtxt) -> Vec<TokenTree> {
vec![TokenTree::Token(DUMMY_SP, token::Interpolated(token::NtGenerics(self.clone())))]
let nt = token::NtGenerics(self.clone());
vec![TokenTree::Token(DUMMY_SP, token::Interpolated(Rc::new(nt)))]
}
}

impl ToTokens for ast::WhereClause {
fn to_tokens(&self, _cx: &ExtCtxt) -> Vec<TokenTree> {
vec![TokenTree::Token(DUMMY_SP,
token::Interpolated(token::NtWhereClause(self.clone())))]
let nt = token::NtWhereClause(self.clone());
vec![TokenTree::Token(DUMMY_SP, token::Interpolated(Rc::new(nt)))]
}
}

impl ToTokens for P<ast::Item> {
fn to_tokens(&self, _cx: &ExtCtxt) -> Vec<TokenTree> {
vec![TokenTree::Token(self.span, token::Interpolated(token::NtItem(self.clone())))]
let nt = token::NtItem(self.clone());
vec![TokenTree::Token(self.span, token::Interpolated(Rc::new(nt)))]
}
}

impl ToTokens for ast::ImplItem {
fn to_tokens(&self, _cx: &ExtCtxt) -> Vec<TokenTree> {
vec![TokenTree::Token(self.span,
token::Interpolated(token::NtImplItem(P(self.clone()))))]
let nt = token::NtImplItem(self.clone());
vec![TokenTree::Token(self.span, token::Interpolated(Rc::new(nt)))]
}
}

impl ToTokens for P<ast::ImplItem> {
fn to_tokens(&self, _cx: &ExtCtxt) -> Vec<TokenTree> {
vec![TokenTree::Token(self.span, token::Interpolated(token::NtImplItem(self.clone())))]
let nt = token::NtImplItem((**self).clone());
vec![TokenTree::Token(self.span, token::Interpolated(Rc::new(nt)))]
}
}

impl ToTokens for ast::TraitItem {
fn to_tokens(&self, _cx: &ExtCtxt) -> Vec<TokenTree> {
vec![TokenTree::Token(self.span,
token::Interpolated(token::NtTraitItem(P(self.clone()))))]
let nt = token::NtTraitItem(self.clone());
vec![TokenTree::Token(self.span, token::Interpolated(Rc::new(nt)))]
}
}

impl ToTokens for ast::Stmt {
fn to_tokens(&self, _cx: &ExtCtxt) -> Vec<TokenTree> {
let mut tts = vec![
TokenTree::Token(self.span, token::Interpolated(token::NtStmt(P(self.clone()))))
];
let nt = token::NtStmt(self.clone());
let mut tts = vec![TokenTree::Token(self.span, token::Interpolated(Rc::new(nt)))];

// Some statements require a trailing semicolon.
if classify::stmt_ends_with_semi(&self.node) {
Expand All @@ -153,31 +157,36 @@ pub mod rt {

impl ToTokens for P<ast::Expr> {
fn to_tokens(&self, _cx: &ExtCtxt) -> Vec<TokenTree> {
vec![TokenTree::Token(self.span, token::Interpolated(token::NtExpr(self.clone())))]
let nt = token::NtExpr(self.clone());
vec![TokenTree::Token(self.span, token::Interpolated(Rc::new(nt)))]
}
}

impl ToTokens for P<ast::Pat> {
fn to_tokens(&self, _cx: &ExtCtxt) -> Vec<TokenTree> {
vec![TokenTree::Token(self.span, token::Interpolated(token::NtPat(self.clone())))]
let nt = token::NtPat(self.clone());
vec![TokenTree::Token(self.span, token::Interpolated(Rc::new(nt)))]
}
}

impl ToTokens for ast::Arm {
fn to_tokens(&self, _cx: &ExtCtxt) -> Vec<TokenTree> {
vec![TokenTree::Token(DUMMY_SP, token::Interpolated(token::NtArm(self.clone())))]
let nt = token::NtArm(self.clone());
vec![TokenTree::Token(DUMMY_SP, token::Interpolated(Rc::new(nt)))]
}
}

impl ToTokens for ast::Arg {
fn to_tokens(&self, _cx: &ExtCtxt) -> Vec<TokenTree> {
vec![TokenTree::Token(DUMMY_SP, token::Interpolated(token::NtArg(self.clone())))]
let nt = token::NtArg(self.clone());
vec![TokenTree::Token(DUMMY_SP, token::Interpolated(Rc::new(nt)))]
}
}

impl ToTokens for P<ast::Block> {
fn to_tokens(&self, _cx: &ExtCtxt) -> Vec<TokenTree> {
vec![TokenTree::Token(DUMMY_SP, token::Interpolated(token::NtBlock(self.clone())))]
let nt = token::NtBlock(self.clone());
vec![TokenTree::Token(DUMMY_SP, token::Interpolated(Rc::new(nt)))]
}
}

Expand All @@ -204,7 +213,8 @@ pub mod rt {

impl ToTokens for P<ast::MetaItem> {
fn to_tokens(&self, _cx: &ExtCtxt) -> Vec<TokenTree> {
vec![TokenTree::Token(DUMMY_SP, token::Interpolated(token::NtMeta(self.clone())))]
let nt = token::NtMeta(self.clone());
vec![TokenTree::Token(DUMMY_SP, token::Interpolated(Rc::new(nt)))]
}
}

Expand Down
61 changes: 32 additions & 29 deletions src/libsyntax/ext/tt/macro_parser.rs
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,6 @@ use parse::token::{DocComment, MatchNt, SubstNt};
use parse::token::{Token, Nonterminal};
use parse::token;
use print::pprust;
use ptr::P;
use tokenstream::{self, TokenTree};
use util::small_vector::SmallVector;

Expand Down Expand Up @@ -198,7 +197,7 @@ pub fn initial_matcher_pos(ms: Vec<TokenTree>, sep: Option<Token>, lo: BytePos)

pub enum NamedMatch {
MatchedSeq(Vec<Rc<NamedMatch>>, syntax_pos::Span),
MatchedNonterminal(Nonterminal)
MatchedNonterminal(Rc<Nonterminal>)
}

pub fn nameize(p_s: &ParseSess, ms: &[TokenTree], res: &[Rc<NamedMatch>])
Expand Down Expand Up @@ -279,17 +278,16 @@ pub fn token_name_eq(t1 : &Token, t2 : &Token) -> bool {
}
}

pub fn parse(sess: &ParseSess, mut rdr: TtReader, ms: &[TokenTree]) -> NamedParseResult {
let mut cur_eis = SmallVector::one(initial_matcher_pos(ms.to_owned(),
None,
rdr.peek().sp.lo));
pub fn parse(sess: &ParseSess, rdr: TtReader, ms: &[TokenTree]) -> NamedParseResult {
let mut parser = Parser::new_with_doc_flag(sess, Box::new(rdr), true);
let mut cur_eis = SmallVector::one(initial_matcher_pos(ms.to_owned(), None, parser.span.lo));

loop {
let mut bb_eis = Vec::new(); // black-box parsed by parser.rs
let mut next_eis = Vec::new(); // or proceed normally
let mut eof_eis = Vec::new();

let TokenAndSpan { tok, sp } = rdr.peek();
let (sp, tok) = (parser.span, parser.token.clone());

/* we append new items to this while we go */
loop {
Expand Down Expand Up @@ -474,23 +472,19 @@ pub fn parse(sess: &ParseSess, mut rdr: TtReader, ms: &[TokenTree]) -> NamedPars
while !next_eis.is_empty() {
cur_eis.push(next_eis.pop().unwrap());
}
rdr.next_token();
parser.bump();
} else /* bb_eis.len() == 1 */ {
rdr.next_tok = {
let mut rust_parser = Parser::new(sess, Box::new(&mut rdr));
let mut ei = bb_eis.pop().unwrap();
if let TokenTree::Token(span, MatchNt(_, ident)) = ei.top_elts.get_tt(ei.idx) {
let match_cur = ei.match_cur;
(&mut ei.matches[match_cur]).push(Rc::new(MatchedNonterminal(
parse_nt(&mut rust_parser, span, &ident.name.as_str()))));
ei.idx += 1;
ei.match_cur += 1;
} else {
unreachable!()
}
cur_eis.push(ei);
Some(TokenAndSpan { tok: rust_parser.token, sp: rust_parser.span })
};
let mut ei = bb_eis.pop().unwrap();
if let TokenTree::Token(span, MatchNt(_, ident)) = ei.top_elts.get_tt(ei.idx) {
let match_cur = ei.match_cur;
(&mut ei.matches[match_cur]).push(Rc::new(MatchedNonterminal(
Rc::new(parse_nt(&mut parser, span, &ident.name.as_str())))));
ei.idx += 1;
ei.match_cur += 1;
} else {
unreachable!()
}
cur_eis.push(ei);
}
}

Expand All @@ -502,10 +496,19 @@ pub fn parse_nt<'a>(p: &mut Parser<'a>, sp: Span, name: &str) -> Nonterminal {
match name {
"tt" => {
p.quote_depth += 1; //but in theory, non-quoted tts might be useful
let res: ::parse::PResult<'a, _> = p.parse_token_tree();
let res = token::NtTT(P(panictry!(res)));
let mut tt = panictry!(p.parse_token_tree());
p.quote_depth -= 1;
return res;
loop {
let nt = match tt {
TokenTree::Token(_, token::Interpolated(ref nt)) => nt.clone(),
_ => break,
};
match *nt {
token::NtTT(ref sub_tt) => tt = sub_tt.clone(),
_ => break,
}
}
return token::NtTT(tt);
}
_ => {}
}
Expand All @@ -521,7 +524,7 @@ pub fn parse_nt<'a>(p: &mut Parser<'a>, sp: Span, name: &str) -> Nonterminal {
},
"block" => token::NtBlock(panictry!(p.parse_block())),
"stmt" => match panictry!(p.parse_stmt()) {
Some(s) => token::NtStmt(P(s)),
Some(s) => token::NtStmt(s),
None => {
p.fatal("expected a statement").emit();
panic!(FatalError);
Expand All @@ -534,7 +537,7 @@ pub fn parse_nt<'a>(p: &mut Parser<'a>, sp: Span, name: &str) -> Nonterminal {
"ident" => match p.token {
token::Ident(sn) => {
p.bump();
token::NtIdent(Box::new(Spanned::<Ident>{node: sn, span: p.span}))
token::NtIdent(Spanned::<Ident>{node: sn, span: p.span})
}
_ => {
let token_str = pprust::token_to_string(&p.token);
Expand All @@ -544,7 +547,7 @@ pub fn parse_nt<'a>(p: &mut Parser<'a>, sp: Span, name: &str) -> Nonterminal {
}
},
"path" => {
token::NtPath(Box::new(panictry!(p.parse_path(PathStyle::Type))))
token::NtPath(panictry!(p.parse_path(PathStyle::Type)))
},
"meta" => token::NtMeta(panictry!(p.parse_meta_item())),
// this is not supposed to happen, since it has been checked
Expand Down
22 changes: 14 additions & 8 deletions src/libsyntax/ext/tt/macro_rules.rs
Original file line number Diff line number Diff line change
Expand Up @@ -236,22 +236,28 @@ pub fn compile(sess: &ParseSess, def: &ast::MacroDef) -> SyntaxExtension {
// Extract the arguments:
let lhses = match **argument_map.get(&lhs_nm).unwrap() {
MatchedSeq(ref s, _) => {
s.iter().map(|m| match **m {
MatchedNonterminal(NtTT(ref tt)) => {
valid &= check_lhs_nt_follows(sess, tt);
(**tt).clone()
s.iter().map(|m| {
if let MatchedNonterminal(ref nt) = **m {
if let NtTT(ref tt) = **nt {
valid &= check_lhs_nt_follows(sess, tt);
return (*tt).clone();
}
}
_ => sess.span_diagnostic.span_bug(def.span, "wrong-structured lhs")
sess.span_diagnostic.span_bug(def.span, "wrong-structured lhs")
}).collect::<Vec<TokenTree>>()
}
_ => sess.span_diagnostic.span_bug(def.span, "wrong-structured lhs")
};

let rhses = match **argument_map.get(&rhs_nm).unwrap() {
MatchedSeq(ref s, _) => {
s.iter().map(|m| match **m {
MatchedNonterminal(NtTT(ref tt)) => (**tt).clone(),
_ => sess.span_diagnostic.span_bug(def.span, "wrong-structured rhs")
s.iter().map(|m| {
if let MatchedNonterminal(ref nt) = **m {
if let NtTT(ref tt) = **nt {
return (*tt).clone();
}
}
sess.span_diagnostic.span_bug(def.span, "wrong-structured lhs")
}).collect()
}
_ => sess.span_diagnostic.span_bug(def.span, "wrong-structured rhs")
Expand Down
Loading

0 comments on commit b9cbbe1

Please sign in to comment.