Merge #8122

8122: Make bare underscore token an Ident rather than Punct in proc-macro r=edwin0cheng a=kevinmehall

In rustc and proc-macro2, a bare `_` token is parsed for procedural macro purposes as `Ident` rather than `Punct` (see rust-lang/rust#48842). This changes rust-analyzer to match rustc's behavior and implementation by handling `_` as an Ident in token trees, but explicitly preventing `$x:ident` from matching it in MBE.

proc macro crate:
```rust
#[proc_macro]
pub fn input(input: proc_macro::TokenStream) -> proc_macro::TokenStream {
    dbg!(input)
}
```

test crate:
```rust
test_proc_macro::input!(_);
```

output (rustc):
```rust
[test-proc-macro/src/lib.rs:10] input = TokenStream [
    Ident {
        ident: "_",
        span: #0 bytes(173..174),
    },
]
```

output (rust-analyzer before this change):
```rust
[test-proc-macro/src/lib.rs:10] input = TokenStream [
    Punct {
        ch: '_',
        spacing: Joint,
        span: 4294967295,
    },
]
```

output (rust-analyzer after this change):
```rust
[test-proc-macro/src/lib.rs:10] input = TokenStream [
    Ident {
        ident: "_",
        span: 4294967295,
    },
]
```


Co-authored-by: Kevin Mehall <km@kevinmehall.net>

crates/mbe/src/expander/matcher.rs

@@ -762,7 +762,7 @@ impl<'a> TtIter<'a> {
     fn expect_separator(&mut self, separator: &Separator, idx: usize) -> bool {
         let mut fork = self.clone();
         let ok = match separator {
-            Separator::Ident(lhs) if idx == 0 => match fork.expect_ident() {
+            Separator::Ident(lhs) if idx == 0 => match fork.expect_ident_or_underscore() {
                 Ok(rhs) => rhs.text == lhs.text,
                 _ => false,
             },
@@ -852,7 +852,7 @@ impl<'a> TtIter<'a> {
         if punct.char != '\'' {
             return Err(());
         }
-        let ident = self.expect_ident()?;
+        let ident = self.expect_ident_or_underscore()?;
 
         Ok(tt::Subtree {
             delimiter: None,

crates/mbe/src/parser.rs

@@ -177,16 +177,8 @@ fn next_op<'a>(first: &tt::TokenTree, src: &mut TtIter<'a>, mode: Mode) -> Resul
                     Op::Repeat { tokens: MetaTemplate(tokens), separator, kind }
                 }
                 tt::TokenTree::Leaf(leaf) => match leaf {
-                    tt::Leaf::Punct(punct) => {
-                        static UNDERSCORE: SmolStr = SmolStr::new_inline("_");
-
-                        if punct.char != '_' {
-                            return Err(ParseError::Expected("_".to_string()));
-                        }
-                        let name = UNDERSCORE.clone();
-                        let kind = eat_fragment_kind(src, mode)?;
-                        let id = punct.id;
-                        Op::Var { name, kind, id }
+                    tt::Leaf::Punct(_) => {
+                        return Err(ParseError::Expected("ident".to_string()));
                     }
                     tt::Leaf::Ident(ident) if ident.text == "crate" => {
                         // We simply produce identifier `$crate` here. And it will be resolved when lowering ast to Path.

crates/mbe/src/subtree_source.rs

@@ -150,6 +150,7 @@ fn convert_ident(ident: &tt::Ident) -> TtToken {
     let kind = match ident.text.as_ref() {
         "true" => T![true],
         "false" => T![false],
+        "_" => UNDERSCORE,
         i if i.starts_with('\'') => LIFETIME_IDENT,
         _ => SyntaxKind::from_keyword(ident.text.as_str()).unwrap_or(IDENT),
     };

crates/mbe/src/syntax_bridge.rs

@@ -350,7 +350,7 @@ trait TokenConvertor {
             return;
         }
 
-        result.push(if k.is_punct() {
+        result.push(if k.is_punct() && k != UNDERSCORE {
             assert_eq!(range.len(), TextSize::of('.'));
             let delim = match k {
                 T!['('] => Some((tt::DelimiterKind::Parenthesis, T![')'])),
@@ -395,7 +395,9 @@ trait TokenConvertor {
                     {
                         tt::Spacing::Alone
                     }
-                    Some(next) if next.kind().is_punct() => tt::Spacing::Joint,
+                    Some(next) if next.kind().is_punct() && next.kind() != UNDERSCORE => {
+                        tt::Spacing::Joint
+                    }
                     _ => tt::Spacing::Alone,
                 };
                 let char = match token.to_char() {
@@ -415,6 +417,7 @@ trait TokenConvertor {
             let leaf: tt::Leaf = match k {
                 T![true] | T![false] => make_leaf!(Ident),
                 IDENT => make_leaf!(Ident),
+                UNDERSCORE => make_leaf!(Ident),
                 k if k.is_keyword() => make_leaf!(Ident),
                 k if k.is_literal() => make_leaf!(Literal),
                 LIFETIME_IDENT => {

crates/mbe/src/tests/expand.rs

@@ -1079,6 +1079,12 @@ macro_rules! q {
     .assert_expand_items(r#"q![_]"#, r#"0"#);
 }
 
+#[test]
+fn test_underscore_lifetime() {
+    parse_macro(r#"macro_rules! q { ($a:lifetime) => {0}; }"#)
+        .assert_expand_items(r#"q!['_]"#, r#"0"#);
+}
+
 #[test]
 fn test_vertical_bar_with_pat() {
     parse_macro(

crates/mbe/src/tests/rule.rs

@@ -12,6 +12,9 @@ fn test_valid_arms() {
     }
 
     check("($i:ident) => ()");
+    check("($(x),*) => ()");
+    check("($(x)_*) => ()");
+    check("($(x)i*) => ()");
     check("($($i:ident)*) => ($_)");
     check("($($true:ident)*) => ($true)");
     check("($($false:ident)*) => ($false)");
@@ -32,6 +35,7 @@ fn test_invalid_arms() {
 
     check("($i) => ($i)", ParseError::UnexpectedToken("bad fragment specifier 1".into()));
     check("($i:) => ($i)", ParseError::UnexpectedToken("bad fragment specifier 1".into()));
+    check("($i:_) => ()", ParseError::UnexpectedToken("bad fragment specifier 1".into()));
 }
 
 fn parse_macro_arm(arm_definition: &str) -> Result<crate::MacroRules, ParseError> {

crates/mbe/src/tt_iter.rs

@@ -49,6 +49,13 @@ impl<'a> TtIter<'a> {
     }
 
     pub(crate) fn expect_ident(&mut self) -> Result<&'a tt::Ident, ()> {
+        match self.expect_leaf()? {
+            tt::Leaf::Ident(it) if it.text != "_" => Ok(it),
+            _ => Err(()),
+        }
+    }
+
+    pub(crate) fn expect_ident_or_underscore(&mut self) -> Result<&'a tt::Ident, ()> {
         match self.expect_leaf()? {
             tt::Leaf::Ident(it) => Ok(it),
             _ => Err(()),

crates/proc_macro_srv/src/rustc_server.rs

@@ -805,5 +805,14 @@ mod tests {
         let t2 = TokenStream::from_str("(a);").unwrap();
         assert_eq!(t2.token_trees.len(), 2);
         assert_eq!(t2.token_trees[0], subtree_paren_a);
+
+        let underscore = TokenStream::from_str("_").unwrap();
+        assert_eq!(
+            underscore.token_trees[0],
+            tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident {
+                text: "_".into(),
+                id: tt::TokenId::unspecified(),
+            }))
+        );
     }
 }