Support block expressions within macros

When we parse DelimTokenTree's the delimiter's are synthesised when we
ask for the token stream which results in tokens lacking location info.
This removes the hack by adding the actual tokens from the lexer into the
stream.

Author: philberty

gcc/rust/ast/rust-ast-full-test.cc

@@ -4387,14 +4387,6 @@ std::vector<std::unique_ptr<Token> >
 DelimTokenTree::to_token_stream () const
 {
   std::vector<std::unique_ptr<Token> > tokens;
-
-  // simulate presence of delimiters
-  const_TokenPtr left_paren
-    = Rust::Token::make (left_delim_type_tok_token_id (delim_type),
-			 Linemap::unknown_location ());
-  tokens.push_back (
-    std::unique_ptr<Token> (new Token (std::move (left_paren))));
-
   for (const auto &tree : token_trees)
     {
       std::vector<std::unique_ptr<Token> > stream = tree->to_token_stream ();
@@ -4403,14 +4395,7 @@ DelimTokenTree::to_token_stream () const
 		     std::make_move_iterator (stream.end ()));
     }
 
-  const_TokenPtr right_paren
-    = Rust::Token::make (right_delim_type_tok_token_id (delim_type),
-			 Linemap::unknown_location ());
-  tokens.push_back (
-    std::unique_ptr<Token> (new Token (std::move (right_paren))));
-
   tokens.shrink_to_fit ();
-
   return tokens;
 }
 

gcc/rust/ast/rust-ast.h

@@ -778,36 +778,6 @@ class DelimTokenTree : public TokenTree, public AttrInput
   }
 
   DelimType get_delim_type () const { return delim_type; }
-
-  static TokenId left_delim_type_tok_token_id (DelimType delim_type)
-  {
-    switch (delim_type)
-      {
-      case PARENS:
-	return LEFT_PAREN;
-      case SQUARE:
-	return LEFT_SQUARE;
-      case CURLY:
-	return LEFT_CURLY;
-      default:
-	gcc_unreachable ();
-      }
-  }
-
-  static TokenId right_delim_type_tok_token_id (DelimType delim_type)
-  {
-    switch (delim_type)
-      {
-      case PARENS:
-	return RIGHT_PAREN;
-      case SQUARE:
-	return RIGHT_SQUARE;
-      case CURLY:
-	return RIGHT_CURLY;
-      default:
-	gcc_unreachable ();
-      }
-  }
 };
 
 /* Forward decl - definition moved to rust-expr.h as it requires LiteralExpr to

gcc/rust/parse/rust-parse-impl.h

@@ -905,6 +905,9 @@ Parser<ManagedTokenSource>::parse_delim_token_tree ()
 
   // parse actual token tree vector - 0 or more
   std::vector<std::unique_ptr<AST::TokenTree>> token_trees_in_tree;
+  auto delim_open
+    = std::unique_ptr<AST::Token> (new AST::Token (std::move (t)));
+  token_trees_in_tree.push_back (std::move (delim_open));
 
   // repeat loop until finding the matching delimiter
   t = lexer.peek_token ();
@@ -929,6 +932,9 @@ Parser<ManagedTokenSource>::parse_delim_token_tree ()
       // lexer.skip_token();
       t = lexer.peek_token ();
     }
+  auto delim_close
+    = std::unique_ptr<AST::Token> (new AST::Token (std::move (t)));
+  token_trees_in_tree.push_back (std::move (delim_close));
 
   AST::DelimTokenTree token_tree (delim_type, std::move (token_trees_in_tree),
 				  initial_loc);

gcc/testsuite/rust/execute/torture/macros5.rs

@@ -0,0 +1,13 @@
+macro_rules! add {
+    ($a:expr,$b:expr) => {{
+        $a + $b
+    }};
+}
+
+fn test() -> i32 {
+    add!(1, 2)
+}
+
+fn main() -> i32 {
+    test() - 3
+}