Break tokens before checking if they are 'probably equal'

src/librustc_ast/tokenstream.rs

@@ -21,6 +21,8 @@ use rustc_macros::HashStable_Generic;
 use rustc_span::{Span, DUMMY_SP};
 use smallvec::{smallvec, SmallVec};
 
+use log::debug;
+
 use std::{iter, mem};
 
 /// When the main rust parser encounters a syntax-extension invocation, it
@@ -338,8 +340,71 @@ impl TokenStream {
             true
         }
 
-        let mut t1 = self.trees().filter(semantic_tree);
-        let mut t2 = other.trees().filter(semantic_tree);
+        // When comparing two `TokenStream`s, we ignore the `IsJoint` information.
+        //
+        // However, `rustc_parse::lexer::tokentrees::TokenStreamBuilder` will
+        // use `Token.glue` on adjacent tokens with the proper `IsJoint`.
+        // Since we are ignoreing `IsJoint`, a 'glued' token (e.g. `BinOp(Shr)`)
+        // and its 'split'/'unglued' compoenents (e.g. `Gt, Gt`) are equivalent
+        // when determining if two `TokenStream`s are 'probably equal'.
+        //
+        // Therefore, we use `break_two_token_op` to convert all tokens
+        // to the 'unglued' form (if it exists). This ensures that two
+        // `TokenStream`s which differ only in how their tokens are glued
+        // will be considered 'probably equal', which allows us to keep spans.
+        //
+        // This is important when the original `TokenStream` contained
+        // extra spaces (e.g. `f :: < Vec < _ > > ( ) ;'). These extra spaces
+        // will be omitted when we pretty-print, which can cause the original
+        // and reparsed `TokenStream`s to differ in the assignment of `IsJoint`,
+        // leading to some tokens being 'glued' together in one stream but not
+        // the other. See #68489 for more details.
+        fn break_tokens(tree: TokenTree) -> impl Iterator<Item = TokenTree> {
+            // In almost all cases, we should have either zero or one levels
+            // of 'unglueing'. However, in some unusual cases, we may need
+            // to iterate breaking tokens mutliple times. For example:
+            // '[BinOpEq(Shr)] => [Gt, Ge] -> [Gt, Gt, Eq]'
+            let mut token_trees: SmallVec<[_; 2]>;
+            if let TokenTree::Token(token) = &tree {
+                let mut out = SmallVec::<[_; 2]>::new();
+                out.push(token.clone());
+                // Iterate to fixpoint:
+                // * We start off with 'out' containing our initial token, and `temp` empty
+                // * If we are able to break any tokens in `out`, then `out` will have
+                //   at least one more element than 'temp', so we will try to break tokens
+                //   again.
+                // * If we cannot break any tokens in 'out', we are done
+                loop {
+                    let mut temp = SmallVec::<[_; 2]>::new();
+                    let mut changed = false;
+
+                    for token in out.into_iter() {
+                        if let Some((first, second)) = token.kind.break_two_token_op() {
+                            temp.push(Token::new(first, DUMMY_SP));
+                            temp.push(Token::new(second, DUMMY_SP));
+                            changed = true;
+                        } else {
+                            temp.push(token);
+                        }
+                    }
+                    out = temp;
+                    if !changed {
+                        break;
+                    }
+                }
+                token_trees = out.into_iter().map(|t| TokenTree::Token(t)).collect();
+                if token_trees.len() != 1 {
+                    debug!("break_tokens: broke {:?} to {:?}", tree, token_trees);
+                }
+            } else {
+                token_trees = SmallVec::new();
+                token_trees.push(tree);
+            }
+            token_trees.into_iter()
+        }
+
+        let mut t1 = self.trees().filter(semantic_tree).flat_map(break_tokens);
+        let mut t2 = other.trees().filter(semantic_tree).flat_map(break_tokens);
         for (t1, t2) in t1.by_ref().zip(t2.by_ref()) {
             if !t1.probably_equal_for_proc_macro(&t2) {
                 return false;

src/test/ui/proc-macro/break-token-spans.rs

@@ -0,0 +1,16 @@
+// aux-build:test-macros.rs
+// Regression test for issues #68489 and #70987
+// Tests that we properly break tokens in `probably_equal_for_proc_macro`
+// See #72306
+//
+// Note that the weird spacing in this example is critical
+// for testing the issue.
+
+extern crate test_macros;
+
+#[test_macros::recollect_attr]
+fn repro() {
+    f :: < Vec < _ > > ( ) ; //~ ERROR cannot find
+    let a: Option<Option<u8>>= true; //~ ERROR mismatched
+}
+fn main() {}

src/test/ui/proc-macro/break-token-spans.stderr

@@ -0,0 +1,21 @@
+error[E0425]: cannot find function `f` in this scope
+  --> $DIR/break-token-spans.rs:13:5
+   |
+LL |     f :: < Vec < _ > > ( ) ;
+   |     ^ not found in this scope
+
+error[E0308]: mismatched types
+  --> $DIR/break-token-spans.rs:14:32
+   |
+LL |     let a: Option<Option<u8>>= true;
+   |            ------------------  ^^^^ expected enum `std::option::Option`, found `bool`
+   |            |
+   |            expected due to this
+   |
+   = note: expected enum `std::option::Option<std::option::Option<u8>>`
+              found type `bool`
+
+error: aborting due to 2 previous errors
+
+Some errors have detailed explanations: E0308, E0425.
+For more information about an error, try `rustc --explain E0308`.

src/test/ui/suggestions/issue-61963.rs

@@ -16,6 +16,7 @@ pub struct Qux<T>(T);
 
 #[dom_struct]
 pub struct Foo {
+    //~^ ERROR trait objects without an explicit `dyn` are deprecated [bare_trait_objects]
     qux: Qux<Qux<Baz>>,
     bar: Box<Bar>,
     //~^ ERROR trait objects without an explicit `dyn` are deprecated [bare_trait_objects]

src/test/ui/suggestions/issue-61963.stderr

@@ -1,5 +1,5 @@
 error: trait objects without an explicit `dyn` are deprecated
-  --> $DIR/issue-61963.rs:20:14
+  --> $DIR/issue-61963.rs:21:14
    |
 LL |     bar: Box<Bar>,
    |              ^^^ help: use `dyn`: `dyn Bar`
@@ -10,5 +10,11 @@ note: the lint level is defined here
 LL | #![deny(bare_trait_objects)]
    |         ^^^^^^^^^^^^^^^^^^
 
-error: aborting due to previous error
+error: trait objects without an explicit `dyn` are deprecated
+  --> $DIR/issue-61963.rs:18:1
+   |
+LL | pub struct Foo {
+   | ^^^ help: use `dyn`: `dyn pub`
+
+error: aborting due to 2 previous errors