Eager expansion for include* #1805 #1865

gcc/rust/ChangeLog:

	* expand/rust-expand-visitor.h:
	remove auto keyword
	* expand/rust-macro-builtins-helpers.cc:
	allow for changing macro invoc types on eager expansions to
	semicoloned macros
	* expand/rust-macro-builtins-helpers.h:
	add default semicoloned argument
	* expand/rust-macro-builtins-include.cc:
	allow for eager expansion for include and include_bytes
	allow for parsing include invocations as items instead of
	expressions, which allows invocations at global scope
	* expand/rust-macro-expand.cc:
	push Expr type for eager invocations

gcc/testsuite/ChangeLog:

	* rust/compile/macros/builtin/include1.rs:
	add basic include test at global scope
	* rust/compile/macros/builtin/include2.rs:
	add basic include test at local scope with expression
	* rust/compile/macros/builtin/include3.rs:
	add eager expansion test at global scope
	* rust/compile/macros/builtin/include4.rs:
	add eager expansion test at local scope with expression
	* rust/compile/macros/builtin/include_bytes.rs:
	add eager expansion test at global scope
	* rust/compile/macros/builtin/include_rs:
	supporting test file with dummy function
	* rust/compile/macros/builtin/include_rs2:
	supporting test file with dummy string
	* rust/compile/macros/builtin/include_str.rs:
	add eager expansion test at global scope
	* rust/execute/torture/builtin_macro_include_bytes.rs:
	clean up old test logic, add permutations for eager expansion
	* rust/execute/torture/builtin_macro_include_str.rs:
	add eager expansion permutations

Author: liamnaddell

gcc/rust/expand/rust-expand-visitor.h

@@ -140,7 +140,7 @@ class ExpandVisitor : public AST::DefaultASTVisitor
 	    it = values.erase (it);
 	    for (auto &node : final_fragment.get_nodes ())
 	      {
-		auto new_node = extractor (node);
+		U new_node = extractor (node);
 		if (new_node != nullptr)
 		  {
 		    it = values.insert (it, std::move (new_node));

gcc/rust/expand/rust-macro-builtins-helpers.cc

@@ -174,7 +174,8 @@ try_expand_many_expr (Parser<MacroInvocLexer> &parser,
 std::unique_ptr<AST::Expr>
 parse_single_string_literal (BuiltinMacro kind,
 			     AST::DelimTokenTree &invoc_token_tree,
-			     location_t invoc_locus, MacroExpander *expander)
+			     location_t invoc_locus, MacroExpander *expander,
+			     bool is_semicoloned)
 {
   MacroInvocLexer lex (invoc_token_tree.to_token_stream ());
   Parser<MacroInvocLexer> parser (lex);
@@ -221,7 +222,7 @@ parse_single_string_literal (BuiltinMacro kind,
 	    AST::MacroInvocData (AST::SimplePath ({AST::SimplePathSegment (
 				   path_str, invoc_locus)}),
 				 std::move (invoc_token_tree)),
-	    {}, invoc_locus, std::move (pending_invocations));
+	    {}, invoc_locus, std::move (pending_invocations), is_semicoloned);
 	}
       else
 	{
@@ -281,4 +282,4 @@ load_file_bytes (location_t invoc_locus, const char *filename)
 
   return buf;
 }
-} // namespace Rust
+} // namespace Rust

gcc/rust/expand/rust-macro-builtins-helpers.h

@@ -74,7 +74,8 @@ try_expand_many_expr (Parser<MacroInvocLexer> &parser,
 std::unique_ptr<AST::Expr>
 parse_single_string_literal (BuiltinMacro kind,
 			     AST::DelimTokenTree &invoc_token_tree,
-			     location_t invoc_locus, MacroExpander *expander);
+			     location_t invoc_locus, MacroExpander *expander,
+			     bool is_semicoloned = false);
 
 // Treat PATH as a path relative to the source file currently being
 // compiled, and return the absolute path for it.

gcc/rust/expand/rust-macro-builtins-include.cc

@@ -40,7 +40,12 @@ MacroBuiltin::include_bytes_handler (location_t invoc_locus,
   if (lit_expr == nullptr)
     return AST::Fragment::create_error ();
 
-  rust_assert (lit_expr->is_literal ());
+  if (!lit_expr->is_literal ())
+    {
+      auto token_tree = invoc.get_delim_tok_tree ();
+      return AST::Fragment ({AST::SingleASTNode (std::move (lit_expr))},
+			    token_tree.to_token_stream ());
+    }
 
   std::string target_filename
     = source_relative_path (lit_expr->as_string (), invoc_locus);
@@ -188,16 +193,36 @@ MacroBuiltin::include_handler (location_t invoc_locus,
 			       AST::MacroInvocData &invoc,
 			       AST::InvocKind semicolon)
 {
+  bool is_semicoloned = semicolon == AST::InvocKind::Semicoloned;
   /* Get target filename from the macro invocation, which is treated as a path
      relative to the include!-ing file (currently being compiled).  */
-  auto lit_expr
+  std::unique_ptr<AST::Expr> lit_expr
     = parse_single_string_literal (BuiltinMacro::Include,
 				   invoc.get_delim_tok_tree (), invoc_locus,
-				   invoc.get_expander ());
+				   invoc.get_expander (), is_semicoloned);
   if (lit_expr == nullptr)
     return AST::Fragment::create_error ();
 
-  rust_assert (lit_expr->is_literal ());
+  if (!lit_expr->is_literal ())
+    {
+      // We have to expand an inner macro eagerly
+      auto token_tree = invoc.get_delim_tok_tree ();
+
+      // parse_single_string_literal returned an AST::MacroInvocation, which
+      // can either be an AST::Item or AST::Expr. Depending on the context the
+      // original macro was invoked in, we will set AST::Item or AST::Expr
+      // appropriately.
+      if (is_semicoloned)
+	{
+	  std::unique_ptr<AST::Item> lit_item = std::unique_ptr<AST::Item> (
+	    static_cast<AST::MacroInvocation *> (lit_expr.release ()));
+	  return AST::Fragment ({AST::SingleASTNode (std::move (lit_item))},
+				token_tree.to_token_stream ());
+	}
+      else
+	return AST::Fragment ({AST::SingleASTNode (std::move (lit_expr))},
+			      token_tree.to_token_stream ());
+    }
 
   std::string filename
     = source_relative_path (lit_expr->as_string (), invoc_locus);
@@ -218,8 +243,14 @@ MacroBuiltin::include_handler (location_t invoc_locus,
 
   Lexer lex (target_filename, std::move (target_file), linemap);
   Parser<Lexer> parser (lex);
+  std::unique_ptr<AST::Expr> parsed_expr = nullptr;
+  std::vector<std::unique_ptr<AST::Item>> parsed_items{};
+
+  if (is_semicoloned)
+    parsed_items = parser.parse_items ();
+  else
+    parsed_expr = parser.parse_expr ();
 
-  auto parsed_items = parser.parse_items ();
   bool has_error = !parser.get_errors ().empty ();
 
   for (const auto &error : parser.get_errors ())
@@ -233,17 +264,22 @@ MacroBuiltin::include_handler (location_t invoc_locus,
     }
 
   std::vector<AST::SingleASTNode> nodes{};
-  for (auto &item : parsed_items)
+  if (is_semicoloned)
+    for (auto &item : parsed_items)
+      {
+	AST::SingleASTNode node (std::move (item));
+	nodes.push_back (node);
+      }
+  else
     {
-      AST::SingleASTNode node (std::move (item));
+      AST::SingleASTNode node (std::move (parsed_expr));
       nodes.push_back (node);
     }
-
   // FIXME: This returns an empty vector of tokens and works fine, but is that
   // the expected behavior? `include` macros are a bit harder to reason about
   // since they include tokens. Furthermore, our lexer has no easy way to return
   // a slice of tokens like the MacroInvocLexer. So it gets even harder to
-  // extrac tokens from here. For now, let's keep it that way and see if it
+  // extract tokens from here. For now, let's keep it that way and see if it
   // eventually breaks, but I don't expect it to cause many issues since the
   // list of tokens is only used when a macro invocation mixes eager
   // macro invocations and already expanded tokens. Think

gcc/rust/expand/rust-macro-expand.cc

@@ -250,7 +250,12 @@ MacroExpander::expand_invoc (AST::MacroInvocation &invoc,
     }
 
   if (invoc.get_kind () == AST::MacroInvocation::InvocKind::Builtin)
-    expand_eager_invocations (invoc);
+    {
+      // Eager expansions are always expressions
+      push_context (ContextType::EXPR);
+      expand_eager_invocations (invoc);
+      pop_context ();
+    }
 
   AST::MacroInvocData &invoc_data = invoc.get_invoc_data ();
 

gcc/testsuite/rust/compile/macros/builtin/include1.rs

@@ -0,0 +1,14 @@
+#![feature(rustc_attrs)]
+
+#[rustc_builtin_macro]
+macro_rules! include {
+    () => {};
+}
+
+include!("include_rs");
+
+fn main() -> i32 {
+    b();
+
+    0
+}

gcc/testsuite/rust/compile/macros/builtin/include2.rs

@@ -0,0 +1,11 @@
+#![feature(rustc_attrs)]
+
+#[rustc_builtin_macro]
+macro_rules! include {
+    () => {};
+}
+
+fn main() -> i32 {
+    let _ = include!("include_rs2");
+    0
+}

gcc/testsuite/rust/compile/macros/builtin/include3.rs

@@ -0,0 +1,19 @@
+#![feature(rustc_attrs)]
+
+#[rustc_builtin_macro]
+macro_rules! include {
+    () => {};
+}
+
+macro_rules! my_file {
+    () => {"include_rs"};
+}
+
+
+include!(my_file!());
+
+fn main() -> i32 {
+    b();
+
+    0
+}

gcc/testsuite/rust/compile/macros/builtin/include4.rs

@@ -0,0 +1,15 @@
+#![feature(rustc_attrs)]
+
+#[rustc_builtin_macro]
+macro_rules! include {
+    () => {};
+}
+
+macro_rules! my_file {
+    () => {"include_rs2"};
+}
+fn main() -> i32 {
+    let _ = include!(my_file!());
+
+    0
+}

gcc/testsuite/rust/compile/macros/builtin/include_bytes.rs

@@ -5,11 +5,19 @@ macro_rules! include_bytes {
     () => {{}};
 }
 
+macro_rules! file1 {
+    () => {"file"};
+}
+
+static MY_FILE: &[u32;16] = include_bytes!(file!());
+
 fn main() {
     let file = "include.txt";
     include_bytes!(file); // { dg-error "argument must be a string literal" "" }
     include_bytes!(); // { dg-error "macro takes 1 argument" "" }
     include_bytes!("foo.txt", "bar.txt"); // { dg-error "macro takes 1 argument" "" }
     include_bytes!("include_bytes.rs"); // ok
     include_bytes!("include_bytes.rs",); // trailing comma ok
+    include_bytes! (file1!());
+    include_bytes! (file1!(),); // trailing comma ok
 }