Skip to content

Commit c334526

Browse files
CentrilCentril
authored
Rollup merge of rust-lang#64041 - matklad:token-stream-tt, r=petrochenkov
use TokenStream rather than &[TokenTree] for built-in macros That way, we don't loose the jointness info
2 parents 7d45eaf + 6136495 commit c334526

File tree

21 files changed

+117
-121
lines changed

21 files changed

+117
-121
lines changed

‎src/doc/unstable-book/src/language-features/plugin.md

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -57,12 +57,12 @@ extern crate rustc;
5757
extern crate rustc_driver;
5858
5959
use syntax::parse::token::{self, Token};
60-
use syntax::tokenstream::TokenTree;
60+
use syntax::tokenstream::{TokenTree, TokenStream};
6161
use syntax::ext::base::{ExtCtxt, MacResult, DummyResult, MacEager};
6262
use syntax_pos::Span;
6363
use rustc_driver::plugin::Registry;
6464
65-
fn expand_rn(cx: &mut ExtCtxt, sp: Span, args: &[TokenTree])
65+
fn expand_rn(cx: &mut ExtCtxt, sp: Span, args: TokenStream)
6666
-> Box<dyn MacResult + 'static> {
6767
6868
static NUMERALS: &'static [(&'static str, usize)] = &[
@@ -78,7 +78,7 @@ fn expand_rn(cx: &mut ExtCtxt, sp: Span, args: &[TokenTree])
7878
return DummyResult::any(sp);
7979
}
8080
81-
let text = match args[0] {
81+
let text = match args.into_trees().next().unwrap() {
8282
TokenTree::Token(Token { kind: token::Ident(s, _), .. }) => s.to_string(),
8383
_ => {
8484
cx.span_err(sp, "argument should be a single identifier");

‎src/librustc_metadata/encoder.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1354,7 +1354,7 @@ impl EncodeContext<'tcx> {
13541354
let def_id = self.tcx.hir().local_def_id(macro_def.hir_id);
13551355
Entry {
13561356
kind: EntryKind::MacroDef(self.lazy(MacroDef {
1357-
body: pprust::tokens_to_string(macro_def.body.clone()),
1357+
body: pprust::tts_to_string(macro_def.body.clone()),
13581358
legacy: macro_def.legacy,
13591359
})),
13601360
visibility: self.lazy(ty::Visibility::Public),

‎src/libsyntax/diagnostics/plugin.rs

Lines changed: 23 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ use crate::ext::base::{ExtCtxt, MacEager, MacResult};
66
use crate::parse::token::{self, Token};
77
use crate::ptr::P;
88
use crate::symbol::kw;
9-
use crate::tokenstream::{TokenTree};
9+
use crate::tokenstream::{TokenTree, TokenStream};
1010

1111
use smallvec::smallvec;
1212
use syntax_pos::Span;
@@ -27,12 +27,11 @@ pub type ErrorMap = BTreeMap<Name, ErrorInfo>;
2727

2828
pub fn expand_diagnostic_used<'cx>(ecx: &'cx mut ExtCtxt<'_>,
2929
span: Span,
30-
token_tree: &[TokenTree])
30+
tts: TokenStream)
3131
-> Box<dyn MacResult+'cx> {
32-
let code = match token_tree {
33-
[
34-
TokenTree::Token(Token { kind: token::Ident(code, _), .. })
35-
] => code,
32+
assert_eq!(tts.len(), 1);
33+
let code = match tts.into_trees().next() {
34+
Some(TokenTree::Token(Token { kind: token::Ident(code, _), .. })) => code,
3635
_ => unreachable!()
3736
};
3837

@@ -62,20 +61,21 @@ pub fn expand_diagnostic_used<'cx>(ecx: &'cx mut ExtCtxt<'_>,
6261

6362
pub fn expand_register_diagnostic<'cx>(ecx: &'cx mut ExtCtxt<'_>,
6463
span: Span,
65-
token_tree: &[TokenTree])
64+
tts: TokenStream)
6665
-> Box<dyn MacResult+'cx> {
67-
let (code, description) = match token_tree {
68-
[
69-
TokenTree::Token(Token { kind: token::Ident(code, _), .. })
70-
] => {
71-
(*code, None)
72-
},
73-
[
74-
TokenTree::Token(Token { kind: token::Ident(code, _), .. }),
75-
TokenTree::Token(Token { kind: token::Comma, .. }),
76-
TokenTree::Token(Token { kind: token::Literal(token::Lit { symbol, .. }), ..})
77-
] => {
78-
(*code, Some(*symbol))
66+
assert!(tts.len() == 1 || tts.len() == 3);
67+
let mut cursor = tts.into_trees();
68+
let code = match cursor.next() {
69+
Some(TokenTree::Token(Token { kind: token::Ident(code, _), .. })) => code,
70+
_ => unreachable!()
71+
};
72+
let description = match (cursor.next(), cursor.next()) {
73+
(None, None) => None,
74+
(
75+
Some(TokenTree::Token(Token { kind: token::Comma, .. })),
76+
Some(TokenTree::Token(Token { kind: token::Literal(token::Lit { symbol, .. }), ..}))
77+
) => {
78+
Some(symbol)
7979
},
8080
_ => unreachable!()
8181
};
@@ -121,12 +121,12 @@ pub fn expand_register_diagnostic<'cx>(ecx: &'cx mut ExtCtxt<'_>,
121121

122122
pub fn expand_build_diagnostic_array<'cx>(ecx: &'cx mut ExtCtxt<'_>,
123123
span: Span,
124-
token_tree: &[TokenTree])
124+
tts: TokenStream)
125125
-> Box<dyn MacResult+'cx> {
126-
assert_eq!(token_tree.len(), 3);
127-
let ident = match &token_tree[2] {
126+
assert_eq!(tts.len(), 3);
127+
let ident = match tts.into_trees().nth(2) {
128128
// DIAGNOSTICS ident.
129-
&TokenTree::Token(Token { kind: token::Ident(name, _), span })
129+
Some(TokenTree::Token(Token { kind: token::Ident(name, _), span }))
130130
=> Ident::new(name, span),
131131
_ => unreachable!()
132132
};

‎src/libsyntax/ext/base.rs

Lines changed: 12 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ use crate::parse::token;
1010
use crate::ptr::P;
1111
use crate::symbol::{kw, sym, Ident, Symbol};
1212
use crate::{ThinVec, MACRO_ARGUMENTS};
13-
use crate::tokenstream::{self, TokenStream, TokenTree};
13+
use crate::tokenstream::{self, TokenStream};
1414
use crate::visit::Visitor;
1515

1616
use errors::{DiagnosticBuilder, DiagnosticId};
@@ -235,18 +235,18 @@ pub trait TTMacroExpander {
235235
}
236236

237237
pub type MacroExpanderFn =
238-
for<'cx> fn(&'cx mut ExtCtxt<'_>, Span, &[tokenstream::TokenTree])
238+
for<'cx> fn(&'cx mut ExtCtxt<'_>, Span, TokenStream)
239239
-> Box<dyn MacResult+'cx>;
240240

241241
impl<F> TTMacroExpander for F
242-
where F: for<'cx> Fn(&'cx mut ExtCtxt<'_>, Span, &[tokenstream::TokenTree])
242+
where F: for<'cx> Fn(&'cx mut ExtCtxt<'_>, Span, TokenStream)
243243
-> Box<dyn MacResult+'cx>
244244
{
245245
fn expand<'cx>(
246246
&self,
247247
ecx: &'cx mut ExtCtxt<'_>,
248248
span: Span,
249-
input: TokenStream,
249+
mut input: TokenStream,
250250
) -> Box<dyn MacResult+'cx> {
251251
struct AvoidInterpolatedIdents;
252252

@@ -268,10 +268,8 @@ impl<F> TTMacroExpander for F
268268
mut_visit::noop_visit_mac(mac, self)
269269
}
270270
}
271-
272-
let input: Vec<_> =
273-
input.trees().map(|mut tt| { AvoidInterpolatedIdents.visit_tt(&mut tt); tt }).collect();
274-
(*self)(ecx, span, &input)
271+
AvoidInterpolatedIdents.visit_tts(&mut input);
272+
(*self)(ecx, span, input)
275273
}
276274
}
277275

@@ -677,7 +675,7 @@ impl SyntaxExtension {
677675
}
678676

679677
pub fn dummy_bang(edition: Edition) -> SyntaxExtension {
680-
fn expander<'cx>(_: &'cx mut ExtCtxt<'_>, span: Span, _: &[TokenTree])
678+
fn expander<'cx>(_: &'cx mut ExtCtxt<'_>, span: Span, _: TokenStream)
681679
-> Box<dyn MacResult + 'cx> {
682680
DummyResult::any(span)
683681
}
@@ -811,9 +809,8 @@ impl<'a> ExtCtxt<'a> {
811809
pub fn monotonic_expander<'b>(&'b mut self) -> expand::MacroExpander<'b, 'a> {
812810
expand::MacroExpander::new(self, true)
813811
}
814-
815-
pub fn new_parser_from_tts(&self, tts: &[tokenstream::TokenTree]) -> parser::Parser<'a> {
816-
parse::stream_to_parser(self.parse_sess, tts.iter().cloned().collect(), MACRO_ARGUMENTS)
812+
pub fn new_parser_from_tts(&self, stream: TokenStream) -> parser::Parser<'a> {
813+
parse::stream_to_parser(self.parse_sess, stream, MACRO_ARGUMENTS)
817814
}
818815
pub fn source_map(&self) -> &'a SourceMap { self.parse_sess.source_map() }
819816
pub fn parse_sess(&self) -> &'a parse::ParseSess { self.parse_sess }
@@ -1019,7 +1016,7 @@ pub fn expr_to_string(cx: &mut ExtCtxt<'_>, expr: P<ast::Expr>, err_msg: &str)
10191016
/// done as rarely as possible).
10201017
pub fn check_zero_tts(cx: &ExtCtxt<'_>,
10211018
sp: Span,
1022-
tts: &[tokenstream::TokenTree],
1019+
tts: TokenStream,
10231020
name: &str) {
10241021
if !tts.is_empty() {
10251022
cx.span_err(sp, &format!("{} takes no arguments", name));
@@ -1030,7 +1027,7 @@ pub fn check_zero_tts(cx: &ExtCtxt<'_>,
10301027
/// expect exactly one string literal, or emit an error and return `None`.
10311028
pub fn get_single_str_from_tts(cx: &mut ExtCtxt<'_>,
10321029
sp: Span,
1033-
tts: &[tokenstream::TokenTree],
1030+
tts: TokenStream,
10341031
name: &str)
10351032
-> Option<String> {
10361033
let mut p = cx.new_parser_from_tts(tts);
@@ -1053,7 +1050,7 @@ pub fn get_single_str_from_tts(cx: &mut ExtCtxt<'_>,
10531050
/// parsing error, emit a non-fatal error and return `None`.
10541051
pub fn get_exprs_from_tts(cx: &mut ExtCtxt<'_>,
10551052
sp: Span,
1056-
tts: &[tokenstream::TokenTree]) -> Option<Vec<P<ast::Expr>>> {
1053+
tts: TokenStream) -> Option<Vec<P<ast::Expr>>> {
10571054
let mut p = cx.new_parser_from_tts(tts);
10581055
let mut es = Vec::new();
10591056
while p.token != token::Eof {

‎src/libsyntax/ext/expand.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -701,7 +701,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
701701
path: &Path,
702702
span: Span,
703703
) -> AstFragment {
704-
let mut parser = self.cx.new_parser_from_tts(&toks.into_trees().collect::<Vec<_>>());
704+
let mut parser = self.cx.new_parser_from_tts(toks);
705705
match parser.parse_ast_fragment(kind, false) {
706706
Ok(fragment) => {
707707
parser.ensure_complete_parse(path, kind.name(), span);

‎src/libsyntax/print/pprust.rs

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -356,11 +356,7 @@ pub fn tt_to_string(tt: tokenstream::TokenTree) -> String {
356356
to_string(|s| s.print_tt(tt, false))
357357
}
358358

359-
pub fn tts_to_string(tts: &[tokenstream::TokenTree]) -> String {
360-
tokens_to_string(tts.iter().cloned().collect())
361-
}
362-
363-
pub fn tokens_to_string(tokens: TokenStream) -> String {
359+
pub fn tts_to_string(tokens: TokenStream) -> String {
364360
to_string(|s| s.print_tts(tokens, false))
365361
}
366362

‎src/libsyntax/tokenstream.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -506,7 +506,7 @@ impl Cursor {
506506

507507
impl fmt::Display for TokenStream {
508508
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
509-
f.write_str(&pprust::tokens_to_string(self.clone()))
509+
f.write_str(&pprust::tts_to_string(self.clone()))
510510
}
511511
}
512512

‎src/libsyntax_ext/asm.rs

Lines changed: 10 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -8,13 +8,12 @@ use errors::DiagnosticBuilder;
88

99
use syntax::ast;
1010
use syntax::ext::base::{self, *};
11-
use syntax::parse;
1211
use syntax::parse::token::{self, Token};
1312
use syntax::ptr::P;
1413
use syntax::symbol::{kw, sym, Symbol};
1514
use syntax::ast::AsmDialect;
1615
use syntax_pos::Span;
17-
use syntax::tokenstream;
16+
use syntax::tokenstream::{self, TokenStream};
1817
use syntax::{span_err, struct_span_err};
1918

2019
enum State {
@@ -43,7 +42,7 @@ const OPTIONS: &[Symbol] = &[sym::volatile, sym::alignstack, sym::intel];
4342

4443
pub fn expand_asm<'cx>(cx: &'cx mut ExtCtxt<'_>,
4544
sp: Span,
46-
tts: &[tokenstream::TokenTree])
45+
tts: TokenStream)
4746
-> Box<dyn base::MacResult + 'cx> {
4847
let mut inline_asm = match parse_inline_asm(cx, sp, tts) {
4948
Ok(Some(inline_asm)) => inline_asm,
@@ -71,20 +70,20 @@ pub fn expand_asm<'cx>(cx: &'cx mut ExtCtxt<'_>,
7170
fn parse_inline_asm<'a>(
7271
cx: &mut ExtCtxt<'a>,
7372
sp: Span,
74-
tts: &[tokenstream::TokenTree],
73+
tts: TokenStream,
7574
) -> Result<Option<ast::InlineAsm>, DiagnosticBuilder<'a>> {
7675
// Split the tts before the first colon, to avoid `asm!("x": y)` being
7776
// parsed as `asm!(z)` with `z = "x": y` which is type ascription.
78-
let first_colon = tts.iter()
77+
let first_colon = tts.trees()
7978
.position(|tt| {
80-
match *tt {
79+
match tt {
8180
tokenstream::TokenTree::Token(Token { kind: token::Colon, .. }) |
8281
tokenstream::TokenTree::Token(Token { kind: token::ModSep, .. }) => true,
8382
_ => false,
8483
}
8584
})
8685
.unwrap_or(tts.len());
87-
let mut p = cx.new_parser_from_tts(&tts[first_colon..]);
86+
let mut p = cx.new_parser_from_tts(tts.trees().skip(first_colon).collect());
8887
let mut asm = kw::Invalid;
8988
let mut asm_str_style = None;
9089
let mut outputs = Vec::new();
@@ -110,7 +109,8 @@ fn parse_inline_asm<'a>(
110109
));
111110
}
112111
// Nested parser, stop before the first colon (see above).
113-
let mut p2 = cx.new_parser_from_tts(&tts[..first_colon]);
112+
let mut p2 =
113+
cx.new_parser_from_tts(tts.trees().take(first_colon).collect());
114114

115115
if p2.token == token::Eof {
116116
let mut err =
@@ -129,12 +129,8 @@ fn parse_inline_asm<'a>(
129129
// This is most likely malformed.
130130
if p2.token != token::Eof {
131131
let mut extra_tts = p2.parse_all_token_trees()?;
132-
extra_tts.extend(tts[first_colon..].iter().cloned());
133-
p = parse::stream_to_parser(
134-
cx.parse_sess,
135-
extra_tts.into_iter().collect(),
136-
Some("inline assembly"),
137-
);
132+
extra_tts.extend(tts.trees().skip(first_colon));
133+
p = cx.new_parser_from_tts(extra_tts.into_iter().collect());
138134
}
139135

140136
asm = s;

‎src/libsyntax_ext/assert.rs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ use syntax_pos::{Span, DUMMY_SP};
1313
pub fn expand_assert<'cx>(
1414
cx: &'cx mut ExtCtxt<'_>,
1515
sp: Span,
16-
tts: &[TokenTree],
16+
tts: TokenStream,
1717
) -> Box<dyn MacResult + 'cx> {
1818
let Assert { cond_expr, custom_message } = match parse_assert(cx, sp, tts) {
1919
Ok(assert) => assert,
@@ -59,9 +59,9 @@ struct Assert {
5959
fn parse_assert<'a>(
6060
cx: &mut ExtCtxt<'a>,
6161
sp: Span,
62-
tts: &[TokenTree]
62+
stream: TokenStream
6363
) -> Result<Assert, DiagnosticBuilder<'a>> {
64-
let mut parser = cx.new_parser_from_tts(tts);
64+
let mut parser = cx.new_parser_from_tts(stream);
6565

6666
if parser.token == token::Eof {
6767
let mut err = cx.struct_span_err(sp, "macro requires a boolean expression as an argument");

‎src/libsyntax_ext/cfg.rs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,14 +7,14 @@ use errors::DiagnosticBuilder;
77
use syntax::ast;
88
use syntax::ext::base::{self, *};
99
use syntax::attr;
10-
use syntax::tokenstream;
10+
use syntax::tokenstream::TokenStream;
1111
use syntax::parse::token;
1212
use syntax_pos::Span;
1313

1414
pub fn expand_cfg(
1515
cx: &mut ExtCtxt<'_>,
1616
sp: Span,
17-
tts: &[tokenstream::TokenTree],
17+
tts: TokenStream,
1818
) -> Box<dyn base::MacResult + 'static> {
1919
let sp = cx.with_legacy_ctxt(sp);
2020

@@ -33,7 +33,7 @@ pub fn expand_cfg(
3333
fn parse_cfg<'a>(
3434
cx: &mut ExtCtxt<'a>,
3535
sp: Span,
36-
tts: &[tokenstream::TokenTree],
36+
tts: TokenStream,
3737
) -> Result<ast::MetaItem, DiagnosticBuilder<'a>> {
3838
let mut p = cx.new_parser_from_tts(tts);
3939

‎src/libsyntax_ext/compile_error.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,11 @@
22

33
use syntax::ext::base::{self, *};
44
use syntax_pos::Span;
5-
use syntax::tokenstream;
5+
use syntax::tokenstream::TokenStream;
66

77
pub fn expand_compile_error<'cx>(cx: &'cx mut ExtCtxt<'_>,
88
sp: Span,
9-
tts: &[tokenstream::TokenTree])
9+
tts: TokenStream)
1010
-> Box<dyn base::MacResult + 'cx> {
1111
let var = match get_single_str_from_tts(cx, sp, tts, "compile_error!") {
1212
None => return DummyResult::any(sp),

‎src/libsyntax_ext/concat.rs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,14 @@
11
use syntax::ast;
22
use syntax::ext::base::{self, DummyResult};
33
use syntax::symbol::Symbol;
4-
use syntax::tokenstream;
4+
use syntax::tokenstream::TokenStream;
55

66
use std::string::String;
77

8-
pub fn expand_syntax_ext(
8+
pub fn expand_concat(
99
cx: &mut base::ExtCtxt<'_>,
1010
sp: syntax_pos::Span,
11-
tts: &[tokenstream::TokenTree],
11+
tts: TokenStream,
1212
) -> Box<dyn base::MacResult + 'static> {
1313
let es = match base::get_exprs_from_tts(cx, sp, tts) {
1414
Some(e) => e,

0 commit comments

Comments
 (0)