Auto merge of #30300 - sanxiyn:syntax-ext, r=nikomatsakis

This reduces iteration time (`make rustc-stage1`) for moved syntax extensions from 11 minutes to 3 minutes on my machine.

Because of the signature change, this is a [breaking-change] for people directly calling `expand_crate`. I think it is rare: from GitHub search, only case I found is [glassful](https://github.com/kmcallister/glassful).

Author: bors

mk/crates.mk

@@ -58,7 +58,7 @@ RUSTC_CRATES := rustc rustc_typeck rustc_mir rustc_borrowck rustc_resolve rustc_
                 rustc_trans rustc_back rustc_llvm rustc_privacy rustc_lint \
                 rustc_data_structures rustc_front rustc_platform_intrinsics \
                 rustc_plugin rustc_metadata
-HOST_CRATES := syntax $(RUSTC_CRATES) rustdoc fmt_macros
+HOST_CRATES := syntax syntax_ext $(RUSTC_CRATES) rustdoc fmt_macros
 TOOLS := compiletest rustdoc rustc rustbook error-index-generator
 
 DEPS_core :=
@@ -86,17 +86,18 @@ DEPS_serialize := std log
 DEPS_term := std log
 DEPS_test := std getopts serialize rbml term native:rust_test_helpers
 
-DEPS_syntax := std term serialize log fmt_macros arena libc rustc_bitflags
+DEPS_syntax := std term serialize log arena libc rustc_bitflags
+DEPS_syntax_ext := syntax fmt_macros
 
-DEPS_rustc := syntax flate arena serialize getopts rbml rustc_front\
+DEPS_rustc := syntax fmt_macros flate arena serialize getopts rbml rustc_front\
               log graphviz rustc_llvm rustc_back rustc_data_structures
 DEPS_rustc_back := std syntax rustc_llvm rustc_front flate log libc
 DEPS_rustc_borrowck := rustc rustc_front log graphviz syntax
 DEPS_rustc_data_structures := std log serialize
 DEPS_rustc_driver := arena flate getopts graphviz libc rustc rustc_back rustc_borrowck \
                      rustc_typeck rustc_mir rustc_resolve log syntax serialize rustc_llvm \
 	             rustc_trans rustc_privacy rustc_lint rustc_front rustc_plugin \
-                     rustc_metadata
+                     rustc_metadata syntax_ext
 DEPS_rustc_front := std syntax log serialize
 DEPS_rustc_lint := rustc log syntax
 DEPS_rustc_llvm := native:rustllvm libc std rustc_bitflags

src/librustc_driver/driver.rs

@@ -54,6 +54,7 @@ use syntax::parse::token;
 use syntax::util::node_count::NodeCounter;
 use syntax::visit;
 use syntax;
+use syntax_ext;
 
 pub fn compile_input(sess: Session,
                      cstore: &CStore,
@@ -563,12 +564,15 @@ pub fn phase_2_configure_and_expand(sess: &Session,
             recursion_limit: sess.recursion_limit.get(),
             trace_mac: sess.opts.debugging_opts.trace_macros,
         };
-        let (ret, macro_names) = syntax::ext::expand::expand_crate(&sess.parse_sess,
-                                                                    cfg,
-                                                                    macros,
-                                                                    syntax_exts,
-                                                                    &mut feature_gated_cfgs,
-                                                                    krate);
+        let mut ecx = syntax::ext::base::ExtCtxt::new(&sess.parse_sess,
+                                                      krate.config.clone(),
+                                                      cfg,
+                                                      &mut feature_gated_cfgs);
+        syntax_ext::register_builtins(&mut ecx.syntax_env);
+        let (ret, macro_names) = syntax::ext::expand::expand_crate(ecx,
+                                                                   macros,
+                                                                   syntax_exts,
+                                                                   krate);
         if cfg!(windows) {
             env::set_var("PATH", &_old_path);
         }

src/librustc_driver/lib.rs

@@ -57,6 +57,7 @@ extern crate rustc_llvm as llvm;
 extern crate log;
 #[macro_use]
 extern crate syntax;
+extern crate syntax_ext;
 
 pub use syntax::diagnostic;
 

src/libsyntax/ext/base.rs

@@ -464,26 +464,6 @@ fn initial_syntax_expander_table<'feat>(ecfg: &expand::ExpansionConfig<'feat>)
 
     let mut syntax_expanders = SyntaxEnv::new();
     syntax_expanders.insert(intern("macro_rules"), MacroRulesTT);
-    syntax_expanders.insert(intern("format_args"),
-                            // format_args uses `unstable` things internally.
-                            NormalTT(Box::new(ext::format::expand_format_args), None, true));
-    syntax_expanders.insert(intern("env"),
-                            builtin_normal_expander(
-                                    ext::env::expand_env));
-    syntax_expanders.insert(intern("option_env"),
-                            builtin_normal_expander(
-                                    ext::env::expand_option_env));
-    syntax_expanders.insert(intern("concat_idents"),
-                            builtin_normal_expander(
-                                    ext::concat_idents::expand_syntax_ext));
-    syntax_expanders.insert(intern("concat"),
-                            builtin_normal_expander(
-                                    ext::concat::expand_syntax_ext));
-    syntax_expanders.insert(intern("log_syntax"),
-                            builtin_normal_expander(
-                                    ext::log_syntax::expand_syntax_ext));
-
-    ext::deriving::register_all(&mut syntax_expanders);
 
     if ecfg.enable_quotes() {
         // Quasi-quoting expanders
@@ -552,15 +532,6 @@ fn initial_syntax_expander_table<'feat>(ecfg: &expand::ExpansionConfig<'feat>)
     syntax_expanders.insert(intern("module_path"),
                             builtin_normal_expander(
                                     ext::source_util::expand_mod));
-    syntax_expanders.insert(intern("asm"),
-                            builtin_normal_expander(
-                                    ext::asm::expand_asm));
-    syntax_expanders.insert(intern("cfg"),
-                            builtin_normal_expander(
-                                    ext::cfg::expand_cfg));
-    syntax_expanders.insert(intern("trace_macros"),
-                            builtin_normal_expander(
-                                    ext::trace_macros::expand_trace_macros));
     syntax_expanders
 }
 

src/libsyntax/ext/expand.rs

@@ -21,7 +21,7 @@ use attr::{AttrMetaMethods, WithAttrs};
 use codemap;
 use codemap::{Span, Spanned, ExpnInfo, NameAndSpan, MacroBang, MacroAttribute};
 use ext::base::*;
-use feature_gate::{self, Features, GatedCfgAttr};
+use feature_gate::{self, Features};
 use fold;
 use fold::*;
 use util::move_map::MoveMap;
@@ -1276,15 +1276,11 @@ impl<'feat> ExpansionConfig<'feat> {
     }
 }
 
-pub fn expand_crate<'feat>(parse_sess: &parse::ParseSess,
-                           cfg: ExpansionConfig<'feat>,
-                           // these are the macros being imported to this crate:
-                           imported_macros: Vec<ast::MacroDef>,
-                           user_exts: Vec<NamedSyntaxExtension>,
-                           feature_gated_cfgs: &mut Vec<GatedCfgAttr>,
-                           c: Crate) -> (Crate, HashSet<Name>) {
-    let mut cx = ExtCtxt::new(parse_sess, c.config.clone(), cfg,
-                              feature_gated_cfgs);
+pub fn expand_crate(mut cx: ExtCtxt,
+                    // these are the macros being imported to this crate:
+                    imported_macros: Vec<ast::MacroDef>,
+                    user_exts: Vec<NamedSyntaxExtension>,
+                    c: Crate) -> (Crate, HashSet<Name>) {
     if std_inject::no_core(&c) {
         cx.crate_root = None;
     } else if std_inject::no_std(&c) {
@@ -1305,7 +1301,7 @@ pub fn expand_crate<'feat>(parse_sess: &parse::ParseSess,
 
         let mut ret = expander.fold_crate(c);
         ret.exported_macros = expander.cx.exported_macros.clone();
-        parse_sess.span_diagnostic.handler().abort_if_errors();
+        cx.parse_sess.span_diagnostic.handler().abort_if_errors();
         ret
     };
     return (ret, cx.syntax_env.names);
@@ -1401,6 +1397,7 @@ mod tests {
     use ast;
     use ast::Name;
     use codemap;
+    use ext::base::ExtCtxt;
     use ext::mtwt;
     use fold::Folder;
     use parse;
@@ -1471,7 +1468,9 @@ mod tests {
             src,
             Vec::new(), &sess);
         // should fail:
-        expand_crate(&sess,test_ecfg(),vec!(),vec!(), &mut vec![], crate_ast);
+        let mut gated_cfgs = vec![];
+        let ecx = ExtCtxt::new(&sess, vec![], test_ecfg(), &mut gated_cfgs);
+        expand_crate(ecx, vec![], vec![], crate_ast);
     }
 
     // make sure that macros can't escape modules
@@ -1484,7 +1483,9 @@ mod tests {
             "<test>".to_string(),
             src,
             Vec::new(), &sess);
-        expand_crate(&sess,test_ecfg(),vec!(),vec!(), &mut vec![], crate_ast);
+        let mut gated_cfgs = vec![];
+        let ecx = ExtCtxt::new(&sess, vec![], test_ecfg(), &mut gated_cfgs);
+        expand_crate(ecx, vec![], vec![], crate_ast);
     }
 
     // macro_use modules should allow macros to escape
@@ -1496,14 +1497,18 @@ mod tests {
             "<test>".to_string(),
             src,
             Vec::new(), &sess);
-        expand_crate(&sess, test_ecfg(), vec!(), vec!(), &mut vec![], crate_ast);
+        let mut gated_cfgs = vec![];
+        let ecx = ExtCtxt::new(&sess, vec![], test_ecfg(), &mut gated_cfgs);
+        expand_crate(ecx, vec![], vec![], crate_ast);
     }
 
     fn expand_crate_str(crate_str: String) -> ast::Crate {
         let ps = parse::ParseSess::new();
         let crate_ast = panictry!(string_to_parser(&ps, crate_str).parse_crate_mod());
         // the cfg argument actually does matter, here...
-        expand_crate(&ps,test_ecfg(),vec!(),vec!(), &mut vec![], crate_ast).0
+        let mut gated_cfgs = vec![];
+        let ecx = ExtCtxt::new(&ps, vec![], test_ecfg(), &mut gated_cfgs);
+        expand_crate(ecx, vec![], vec![], crate_ast).0
     }
 
     // find the pat_ident paths in a crate

src/libsyntax/lib.rs

@@ -37,7 +37,6 @@
 #![feature(str_escape)]
 #![feature(unicode)]
 
-extern crate fmt_macros;
 extern crate serialize;
 extern crate term;
 extern crate libc;
@@ -110,21 +109,12 @@ pub mod print {
 }
 
 pub mod ext {
-    pub mod asm;
     pub mod base;
     pub mod build;
-    pub mod cfg;
-    pub mod concat;
-    pub mod concat_idents;
-    pub mod deriving;
-    pub mod env;
     pub mod expand;
-    pub mod format;
-    pub mod log_syntax;
     pub mod mtwt;
     pub mod quote;
     pub mod source_util;
-    pub mod trace_macros;
 
     pub mod tt {
         pub mod transcribe;

src/libsyntax/ext/asm.rs renamed to src/libsyntax_ext/asm.rs

@@ -13,15 +13,15 @@
  */
 use self::State::*;
 
-use ast;
-use codemap;
-use codemap::Span;
-use ext::base;
-use ext::base::*;
-use feature_gate;
-use parse::token::{intern, InternedString};
-use parse::token;
-use ptr::P;
+use syntax::ast;
+use syntax::codemap;
+use syntax::codemap::Span;
+use syntax::ext::base;
+use syntax::ext::base::*;
+use syntax::feature_gate;
+use syntax::parse::token::{intern, InternedString};
+use syntax::parse::token;
+use syntax::ptr::P;
 use syntax::ast::AsmDialect;
 
 enum State {

src/libsyntax/ext/cfg.rs renamed to src/libsyntax_ext/cfg.rs

@@ -12,15 +12,15 @@
 /// a literal `true` or `false` based on whether the given cfg matches the
 /// current compilation environment.
 
-use ast;
-use codemap::Span;
-use ext::base::*;
-use ext::base;
-use ext::build::AstBuilder;
-use attr;
-use attr::*;
-use parse::token;
-use config::CfgDiagReal;
+use syntax::ast;
+use syntax::codemap::Span;
+use syntax::ext::base::*;
+use syntax::ext::base;
+use syntax::ext::build::AstBuilder;
+use syntax::attr;
+use syntax::attr::*;
+use syntax::parse::token;
+use syntax::config::CfgDiagReal;
 
 pub fn expand_cfg<'cx>(cx: &mut ExtCtxt,
                        sp: Span,

src/libsyntax/ext/concat.rs renamed to src/libsyntax_ext/concat.rs

@@ -8,11 +8,11 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-use ast;
-use codemap;
-use ext::base;
-use ext::build::AstBuilder;
-use parse::token;
+use syntax::ast;
+use syntax::codemap;
+use syntax::ext::base;
+use syntax::ext::build::AstBuilder;
+use syntax::parse::token;
 
 use std::string::String;
 

src/libsyntax/ext/concat_idents.rs renamed to src/libsyntax_ext/concat_idents.rs

@@ -8,14 +8,14 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-use ast::{self, TokenTree};
-use codemap::Span;
-use ext::base::*;
-use ext::base;
-use feature_gate;
-use parse::token;
-use parse::token::str_to_ident;
-use ptr::P;
+use syntax::ast::{self, TokenTree};
+use syntax::codemap::Span;
+use syntax::ext::base::*;
+use syntax::ext::base;
+use syntax::feature_gate;
+use syntax::parse::token;
+use syntax::parse::token::str_to_ident;
+use syntax::ptr::P;
 
 pub fn expand_syntax_ext<'cx>(cx: &mut ExtCtxt, sp: Span, tts: &[TokenTree])
                               -> Box<base::MacResult+'cx> {

src/libsyntax/ext/deriving/bounds.rs renamed to src/libsyntax_ext/deriving/bounds.rs

@@ -8,11 +8,12 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-use ast::MetaItem;
-use codemap::Span;
-use ext::base::{ExtCtxt, Annotatable};
-use ext::deriving::generic::*;
-use ext::deriving::generic::ty::*;
+use deriving::generic::*;
+use deriving::generic::ty::*;
+
+use syntax::ast::MetaItem;
+use syntax::codemap::Span;
+use syntax::ext::base::{ExtCtxt, Annotatable};
 
 pub fn expand_deriving_unsafe_bound(cx: &mut ExtCtxt,
                                     span: Span,

src/libsyntax/ext/deriving/clone.rs renamed to src/libsyntax_ext/deriving/clone.rs

@@ -8,14 +8,15 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-use ast::{MetaItem, Expr};
-use codemap::Span;
-use ext::base::{ExtCtxt, Annotatable};
-use ext::build::AstBuilder;
-use ext::deriving::generic::*;
-use ext::deriving::generic::ty::*;
-use parse::token::InternedString;
-use ptr::P;
+use deriving::generic::*;
+use deriving::generic::ty::*;
+
+use syntax::ast::{MetaItem, Expr};
+use syntax::codemap::Span;
+use syntax::ext::base::{ExtCtxt, Annotatable};
+use syntax::ext::build::AstBuilder;
+use syntax::parse::token::InternedString;
+use syntax::ptr::P;
 
 pub fn expand_deriving_clone(cx: &mut ExtCtxt,
                              span: Span,

src/libsyntax/ext/deriving/cmp/eq.rs renamed to src/libsyntax_ext/deriving/cmp/eq.rs

@@ -8,14 +8,15 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-use ast::{MetaItem, Expr};
-use codemap::Span;
-use ext::base::{ExtCtxt, Annotatable};
-use ext::build::AstBuilder;
-use ext::deriving::generic::*;
-use ext::deriving::generic::ty::*;
-use parse::token::InternedString;
-use ptr::P;
+use deriving::generic::*;
+use deriving::generic::ty::*;
+
+use syntax::ast::{MetaItem, Expr};
+use syntax::codemap::Span;
+use syntax::ext::base::{ExtCtxt, Annotatable};
+use syntax::ext::build::AstBuilder;
+use syntax::parse::token::InternedString;
+use syntax::ptr::P;
 
 pub fn expand_deriving_eq(cx: &mut ExtCtxt,
                           span: Span,