Eager expansion for include* #1805 #1865

gcc/rust/ChangeLog:

	* expand/rust-expand-visitor.h:
	remove auto keyword
	* expand/rust-macro-builtins-helpers.cc:
	allow for changing macro invoc types on eager expansions to
	semicoloned macros
	* expand/rust-macro-builtins-helpers.h:
	add default semicoloned argument
	* expand/rust-macro-builtins-include.cc:
	allow for eager expansion for include and include_bytes
	allow for parsing include invocations as items instead of
	expressions, which allows invocations at global scope
	* expand/rust-macro-expand.cc:
	push Expr type for eager invocations

gcc/testsuite/ChangeLog:

	* rust/compile/macros/builtin/include1.rs:
	add basic include test at global scope
	* rust/compile/macros/builtin/include2.rs:
	add basic include test at local scope with expression
	* rust/compile/macros/builtin/include3.rs:
	add eager expansion test at global scope
	* rust/compile/macros/builtin/include4.rs:
	add eager expansion test at local scope with expression
	* rust/compile/macros/builtin/include_bytes.rs:
	add eager expansion test at global scope
	* rust/compile/macros/builtin/include_rs:
	supporting test file with dummy function
	* rust/compile/macros/builtin/include_rs2:
	supporting test file with dummy string
	* rust/compile/macros/builtin/include_str.rs:
	add eager expansion test at global scope
	* rust/execute/torture/builtin_macro_include_bytes.rs:
	clean up old test logic, add permutations for eager expansion
	* rust/execute/torture/builtin_macro_include_str.rs:
	add eager expansion permutations

Author: liamnaddell

gcc/rust/expand/rust-expand-visitor.h

@@ -140,7 +140,7 @@ class ExpandVisitor : public AST::DefaultASTVisitor
 	    it = values.erase (it);
 	    for (auto &node : final_fragment.get_nodes ())
 	      {
-		auto new_node = extractor (node);
+		U new_node = extractor (node);
 		if (new_node != nullptr)
 		  {
 		    it = values.insert (it, std::move (new_node));

gcc/rust/expand/rust-macro-builtins-helpers.cc

@@ -174,7 +174,8 @@ try_expand_many_expr (Parser<MacroInvocLexer> &parser,
 std::unique_ptr<AST::Expr>
 parse_single_string_literal (BuiltinMacro kind,
 			     AST::DelimTokenTree &invoc_token_tree,
-			     location_t invoc_locus, MacroExpander *expander)
+			     location_t invoc_locus, MacroExpander *expander,
+			     bool is_semicoloned)
 {
   MacroInvocLexer lex (invoc_token_tree.to_token_stream ());
   Parser<MacroInvocLexer> parser (lex);
@@ -221,7 +222,7 @@ parse_single_string_literal (BuiltinMacro kind,
 	    AST::MacroInvocData (AST::SimplePath ({AST::SimplePathSegment (
 				   path_str, invoc_locus)}),
 				 std::move (invoc_token_tree)),
-	    {}, invoc_locus, std::move (pending_invocations));
+	    {}, invoc_locus, std::move (pending_invocations), is_semicoloned);
 	}
       else
 	{
@@ -281,4 +282,4 @@ load_file_bytes (location_t invoc_locus, const char *filename)
 
   return buf;
 }
-} // namespace Rust
+} // namespace Rust

gcc/rust/expand/rust-macro-builtins-helpers.h

@@ -74,7 +74,8 @@ try_expand_many_expr (Parser<MacroInvocLexer> &parser,
 std::unique_ptr<AST::Expr>
 parse_single_string_literal (BuiltinMacro kind,
 			     AST::DelimTokenTree &invoc_token_tree,
-			     location_t invoc_locus, MacroExpander *expander);
+			     location_t invoc_locus, MacroExpander *expander,
+			     bool is_semicoloned = false);
 
 // Treat PATH as a path relative to the source file currently being
 // compiled, and return the absolute path for it.

gcc/rust/expand/rust-macro-builtins-include.cc

@@ -40,7 +40,12 @@ MacroBuiltin::include_bytes_handler (location_t invoc_locus,
   if (lit_expr == nullptr)
     return AST::Fragment::create_error ();
 
-  rust_assert (lit_expr->is_literal ());
+  if (!lit_expr->is_literal ())
+    {
+      auto token_tree = invoc.get_delim_tok_tree ();
+      return AST::Fragment ({AST::SingleASTNode (std::move (lit_expr))},
+			    token_tree.to_token_stream ());
+    }
 
   std::string target_filename
     = source_relative_path (lit_expr->as_string (), invoc_locus);
@@ -188,16 +193,36 @@ MacroBuiltin::include_handler (location_t invoc_locus,
 			       AST::MacroInvocData &invoc,
 			       AST::InvocKind semicolon)
 {
+  bool is_semicoloned = semicolon == AST::InvocKind::Semicoloned;
   /* Get target filename from the macro invocation, which is treated as a path
      relative to the include!-ing file (currently being compiled).  */
-  auto lit_expr
+  std::unique_ptr<AST::Expr> lit_expr
     = parse_single_string_literal (BuiltinMacro::Include,
 				   invoc.get_delim_tok_tree (), invoc_locus,
-				   invoc.get_expander ());
+				   invoc.get_expander (), is_semicoloned);
   if (lit_expr == nullptr)
     return AST::Fragment::create_error ();
 
-  rust_assert (lit_expr->is_literal ());
+  if (!lit_expr->is_literal ())
+    {
+      // We have to expand an inner macro eagerly
+      auto token_tree = invoc.get_delim_tok_tree ();
+
+      // parse_single_string_literal returned an AST::MacroInvocation, which
+      // can either be an AST::Item or AST::Expr. Depending on the context the
+      // original macro was invoked in, we will set AST::Item or AST::Expr
+      // appropriately.
+      if (is_semicoloned)
+	{
+	  std::unique_ptr<AST::Item> lit_item = std::unique_ptr<AST::Item> (
+	    static_cast<AST::MacroInvocation *> (lit_expr.release ()));
+	  return AST::Fragment ({AST::SingleASTNode (std::move (lit_item))},
+				token_tree.to_token_stream ());
+	}
+      else
+	return AST::Fragment ({AST::SingleASTNode (std::move (lit_expr))},
+			      token_tree.to_token_stream ());
+    }
 
   std::string filename
     = source_relative_path (lit_expr->as_string (), invoc_locus);
@@ -218,8 +243,14 @@ MacroBuiltin::include_handler (location_t invoc_locus,
 
   Lexer lex (target_filename, std::move (target_file), linemap);
   Parser<Lexer> parser (lex);
+  std::unique_ptr<AST::Expr> parsed_expr = nullptr;
+  std::vector<std::unique_ptr<AST::Item>> parsed_items{};
+
+  if (is_semicoloned)
+    parsed_items = parser.parse_items ();
+  else
+    parsed_expr = parser.parse_expr ();
 
-  auto parsed_items = parser.parse_items ();
   bool has_error = !parser.get_errors ().empty ();
 
   for (const auto &error : parser.get_errors ())
@@ -233,17 +264,22 @@ MacroBuiltin::include_handler (location_t invoc_locus,
     }
 
   std::vector<AST::SingleASTNode> nodes{};
-  for (auto &item : parsed_items)
+  if (is_semicoloned)
+    for (auto &item : parsed_items)
+      {
+	AST::SingleASTNode node (std::move (item));
+	nodes.push_back (node);
+      }
+  else
     {
-      AST::SingleASTNode node (std::move (item));
+      AST::SingleASTNode node (std::move (parsed_expr));
       nodes.push_back (node);
     }
-
   // FIXME: This returns an empty vector of tokens and works fine, but is that
   // the expected behavior? `include` macros are a bit harder to reason about
   // since they include tokens. Furthermore, our lexer has no easy way to return
   // a slice of tokens like the MacroInvocLexer. So it gets even harder to
-  // extrac tokens from here. For now, let's keep it that way and see if it
+  // extract tokens from here. For now, let's keep it that way and see if it
   // eventually breaks, but I don't expect it to cause many issues since the
   // list of tokens is only used when a macro invocation mixes eager
   // macro invocations and already expanded tokens. Think

gcc/rust/expand/rust-macro-expand.cc

@@ -250,7 +250,12 @@ MacroExpander::expand_invoc (AST::MacroInvocation &invoc,
     }
 
   if (invoc.get_kind () == AST::MacroInvocation::InvocKind::Builtin)
-    expand_eager_invocations (invoc);
+    {
+      // Eager expansions are always expressions
+      push_context (ContextType::EXPR);
+      expand_eager_invocations (invoc);
+      pop_context ();
+    }
 
   AST::MacroInvocData &invoc_data = invoc.get_invoc_data ();
 

gcc/testsuite/rust/compile/macros/builtin/include1.rs

@@ -0,0 +1,14 @@
+#![feature(rustc_attrs)]
+
+#[rustc_builtin_macro]
+macro_rules! include {
+    () => {};
+}
+
+include!("include_rs");
+
+fn main() -> i32 {
+    b();
+
+    0
+}

gcc/testsuite/rust/compile/macros/builtin/include2.rs

@@ -0,0 +1,11 @@
+#![feature(rustc_attrs)]
+
+#[rustc_builtin_macro]
+macro_rules! include {
+    () => {};
+}
+
+fn main() -> i32 {
+    let _ = include!("include_rs2");
+    0
+}

gcc/testsuite/rust/compile/macros/builtin/include3.rs

@@ -0,0 +1,19 @@
+#![feature(rustc_attrs)]
+
+#[rustc_builtin_macro]
+macro_rules! include {
+    () => {};
+}
+
+macro_rules! my_file {
+    () => {"include_rs"};
+}
+
+
+include!(my_file!());
+
+fn main() -> i32 {
+    b();
+
+    0
+}

gcc/testsuite/rust/compile/macros/builtin/include4.rs

@@ -0,0 +1,15 @@
+#![feature(rustc_attrs)]
+
+#[rustc_builtin_macro]
+macro_rules! include {
+    () => {};
+}
+
+macro_rules! my_file {
+    () => {"include_rs2"};
+}
+fn main() -> i32 {
+    let _ = include!(my_file!());
+
+    0
+}

gcc/testsuite/rust/compile/macros/builtin/include_bytes.rs

@@ -5,11 +5,19 @@ macro_rules! include_bytes {
     () => {{}};
 }
 
+macro_rules! file1 {
+    () => {"file"};
+}
+
+static MY_FILE: &[u32;16] = include_bytes!(file!());
+
 fn main() {
     let file = "include.txt";
     include_bytes!(file); // { dg-error "argument must be a string literal" "" }
     include_bytes!(); // { dg-error "macro takes 1 argument" "" }
     include_bytes!("foo.txt", "bar.txt"); // { dg-error "macro takes 1 argument" "" }
     include_bytes!("include_bytes.rs"); // ok
     include_bytes!("include_bytes.rs",); // trailing comma ok
+    include_bytes! (file1!());
+    include_bytes! (file1!(),); // trailing comma ok
 }

gcc/testsuite/rust/compile/macros/builtin/include_rs

@@ -0,0 +1 @@
+fn b() {}

gcc/testsuite/rust/compile/macros/builtin/include_rs2

@@ -0,0 +1 @@
+"Gccrs is GREAT!"

gcc/testsuite/rust/compile/macros/builtin/include_str.rs

@@ -5,6 +5,12 @@ macro_rules! include_str {
     () => {{}};
 }
 
+macro_rules! my_file {
+    () => {"include.txt"}
+}
+
+static G_STR:[u8;16] = include_str!(my_file!());
+
 fn main() {
     let file = "include.txt";
     include_str!(file); // { dg-error "argument must be a string literal" "" }
@@ -13,4 +19,6 @@ fn main() {
     include_str!("include_str.rs"); // ok
     include_str!("include_str.rs",); // trailing comma ok
     include_str!("invalid_utf8"); // { dg-error "invalid_utf8 was not a valid utf-8 file" "" }
+    include_str!(my_file!());
+    include_str!(my_file!(),);
 }

gcc/testsuite/rust/execute/torture/builtin_macro_include_bytes.rs

@@ -1,11 +1,16 @@
-// { dg-output "104\r*\n33\r*\n1\r*\n" }
+// { dg-output "1\r*\n1\r*\n1\r*\n" }
+
 #![feature(rustc_attrs)]
 
 #[rustc_builtin_macro]
 macro_rules! include_bytes {
     () => {{}};
 }
 
+macro_rules! my_file {
+    () => {"include.txt"};
+}
+
 extern "C" {
     fn printf(s: *const i8, ...);
 }
@@ -17,32 +22,30 @@ fn print_int(value: i32) {
     }
 }
 
-fn main() -> i32 {
-    let bytes = include_bytes!("include.txt");
-
-    print_int(bytes[0] as i32);
-    print_int(bytes[14] as i32);
-
+fn check_bytes(bytes: &[u8; 16]) {
     let the_bytes = b"hello, include!\n";
 
-    let x = bytes[0] == the_bytes[0]
-        && bytes[1] == the_bytes[1]
-        && bytes[2] == the_bytes[2]
-        && bytes[3] == the_bytes[3]
-        && bytes[4] == the_bytes[4]
-        && bytes[5] == the_bytes[5]
-        && bytes[6] == the_bytes[6]
-        && bytes[7] == the_bytes[7]
-        && bytes[8] == the_bytes[8]
-        && bytes[9] == the_bytes[9]
-        && bytes[10] == the_bytes[10]
-        && bytes[11] == the_bytes[11]
-        && bytes[12] == the_bytes[12]
-        && bytes[13] == the_bytes[13]
-        && bytes[14] == the_bytes[14]
-        && bytes[15] == the_bytes[15];
+    let x = true;
+    let mut i = 0;
+
+    // X is true iff bytes == the_bytes
+    while i < 16 {
+        x = x && (bytes[i] == the_bytes[i]);
+        i += 1;
+    }
 
     print_int(x as i32);
+}
+
+fn main() -> i32 {
+    let bytes1: &'static [u8; 16] = include_bytes!("include.txt");
+    check_bytes(bytes1);
+
+    let bytes2: &'static [u8; 16] = include_bytes!(my_file!());
+    check_bytes(bytes2);
+
+    let bytes3 = include_bytes!(my_file!(),);
+    check_bytes(bytes3);
 
     0
 }

gcc/testsuite/rust/execute/torture/builtin_macro_include_str.rs

@@ -1,11 +1,15 @@
-// { dg-output "hello, include!\r*\n" }
+// { dg-output "hello, include!\r*\nhello, include!\r*\nhello, include!\r*\n" }
 #![feature(rustc_attrs)]
 
 #[rustc_builtin_macro]
 macro_rules! include_str {
     () => {{}};
 }
 
+macro_rules! my_file {
+    () => {"include.txt"};
+}
+
 extern "C" {
     fn printf(fmt: *const i8, ...);
 }
@@ -22,7 +26,10 @@ fn print(s: &str) {
 fn main() -> i32 {
     // include_str! (and include_bytes!) allow for an optional trailing comma.
     let my_str = include_str!("include.txt",);
-
+    print(my_str);
+    let my_str = include_str!(my_file!());
+    print(my_str);
+    let my_str = include_str!(my_file!(),);
     print(my_str);
 
     0