From c5a561c0abe432006063408dc3371891585319a6 Mon Sep 17 00:00:00 2001 From: David Tolnay Date: Sun, 2 Sep 2018 06:59:02 -0700 Subject: [PATCH 1/3] proc_macro::Group::span_open and span_close Before this addition, every delimited group like (...) [...] {...} has only a single Span that covers the full source location from opening delimiter to closing delimiter. This makes it impossible for a procedural macro to trigger an error pointing to just the opening or closing delimiter. The Rust compiler does not seem to have the same limitation: mod m { type T = } error: expected type, found `}` --> src/main.rs:3:1 | 3 | } | ^ On that same input, a procedural macro would be forced to trigger the error on the last token inside the block, on the entire block, or on the next token after the block, none of which is really what you want for an error like above. This commit adds group.span_open() and group.span_close() which access the Span associated with just the opening delimiter and just the closing delimiter of the group. Relevant to Syn as we implement real error messages for when parsing fails in a procedural macro. --- src/libproc_macro/lib.rs | 43 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/src/libproc_macro/lib.rs b/src/libproc_macro/lib.rs index 8c667d2f87111..8ed7b8f52baba 100644 --- a/src/libproc_macro/lib.rs +++ b/src/libproc_macro/lib.rs @@ -64,7 +64,7 @@ use syntax::errors::DiagnosticBuilder; use syntax::parse::{self, token}; use syntax::symbol::Symbol; use syntax::tokenstream; -use syntax_pos::{Pos, FileName}; +use syntax_pos::{BytePos, Pos, FileName}; /// The main type provided by this crate, representing an abstract stream of /// tokens, or, more specifically, a sequence of token trees. @@ -671,11 +671,52 @@ impl Group { /// Returns the span for the delimiters of this token stream, spanning the /// entire `Group`. + /// + /// ```text + /// pub fn span(&self) -> Span { + /// ^^^^^^^ + /// ``` #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn span(&self) -> Span { self.span } + /// Returns the span pointing to the opening delimiter of this group, or the + /// span of the entire group if this is a None-delimited group. + /// + /// ```text + /// pub fn span_open(&self) -> Span { + /// ^ + /// ``` + #[unstable(feature = "proc_macro_span", issue = "38356")] + pub fn span_open(&self) -> Span { + if self.delimiter == Delimiter::None { + self.span + } else { + let lo = self.span.0.lo(); + let new_hi = BytePos::from_usize(lo.to_usize() + 1); + Span(self.span.0.with_hi(new_hi)) + } + } + + /// Returns the span pointing to the closing delimiter of this group, or the + /// span of the entire group if this is a None-delimited group. + /// + /// ```text + /// pub fn span_close(&self) -> Span { + /// ^ + /// ``` + #[unstable(feature = "proc_macro_span", issue = "38356")] + pub fn span_close(&self) -> Span { + let hi = self.span.0.hi(); + if self.delimiter == Delimiter::None || hi.to_usize() == 0 { + self.span + } else { + let new_lo = BytePos::from_usize(hi.to_usize() - 1); + Span(self.span.0.with_lo(new_lo)) + } + } + /// Configures the span for this `Group`'s delimiters, but not its internal /// tokens. /// From a1dd39e724c2a37f08f534e90ced134d38a49417 Mon Sep 17 00:00:00 2001 From: David Tolnay Date: Sat, 8 Sep 2018 18:07:02 -0700 Subject: [PATCH 2/3] Track distinct spans for open and close delimiter --- src/libproc_macro/lib.rs | 34 +++----- src/libproc_macro/rustc.rs | 4 +- src/librustc/ich/hcx.rs | 12 +++ src/librustc_resolve/macros.rs | 5 +- src/libsyntax/attr/mod.rs | 4 +- src/libsyntax/ext/quote.rs | 22 +++--- src/libsyntax/ext/tt/macro_parser.rs | 16 ++-- src/libsyntax/ext/tt/macro_rules.rs | 25 +++--- src/libsyntax/ext/tt/quoted.rs | 20 ++--- src/libsyntax/ext/tt/transcribe.rs | 14 ++-- src/libsyntax/fold.rs | 11 ++- src/libsyntax/parse/lexer/tokentrees.rs | 6 +- src/libsyntax/parse/mod.rs | 6 +- src/libsyntax/parse/parser.rs | 22 +++--- src/libsyntax/parse/token.rs | 6 +- src/libsyntax/tokenstream.rs | 79 +++++++++++++------ .../auxiliary/procedural_mbe_matching.rs | 3 +- 17 files changed, 163 insertions(+), 126 deletions(-) diff --git a/src/libproc_macro/lib.rs b/src/libproc_macro/lib.rs index 8ed7b8f52baba..d4737052875b6 100644 --- a/src/libproc_macro/lib.rs +++ b/src/libproc_macro/lib.rs @@ -63,8 +63,8 @@ use std::str::FromStr; use syntax::errors::DiagnosticBuilder; use syntax::parse::{self, token}; use syntax::symbol::Symbol; -use syntax::tokenstream; -use syntax_pos::{BytePos, Pos, FileName}; +use syntax::tokenstream::{self, DelimSpan}; +use syntax_pos::{Pos, FileName}; /// The main type provided by this crate, representing an abstract stream of /// tokens, or, more specifically, a sequence of token trees. @@ -609,7 +609,7 @@ impl fmt::Display for TokenTree { pub struct Group { delimiter: Delimiter, stream: TokenStream, - span: Span, + span: DelimSpan, } #[stable(feature = "proc_macro_lib2", since = "1.29.0")] @@ -650,7 +650,7 @@ impl Group { Group { delimiter: delimiter, stream: stream, - span: Span::call_site(), + span: DelimSpan::from_single(Span::call_site().0), } } @@ -678,11 +678,10 @@ impl Group { /// ``` #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn span(&self) -> Span { - self.span + Span(self.span.entire()) } - /// Returns the span pointing to the opening delimiter of this group, or the - /// span of the entire group if this is a None-delimited group. + /// Returns the span pointing to the opening delimiter of this group. /// /// ```text /// pub fn span_open(&self) -> Span { @@ -690,17 +689,10 @@ impl Group { /// ``` #[unstable(feature = "proc_macro_span", issue = "38356")] pub fn span_open(&self) -> Span { - if self.delimiter == Delimiter::None { - self.span - } else { - let lo = self.span.0.lo(); - let new_hi = BytePos::from_usize(lo.to_usize() + 1); - Span(self.span.0.with_hi(new_hi)) - } + Span(self.span.open) } - /// Returns the span pointing to the closing delimiter of this group, or the - /// span of the entire group if this is a None-delimited group. + /// Returns the span pointing to the closing delimiter of this group. /// /// ```text /// pub fn span_close(&self) -> Span { @@ -708,13 +700,7 @@ impl Group { /// ``` #[unstable(feature = "proc_macro_span", issue = "38356")] pub fn span_close(&self) -> Span { - let hi = self.span.0.hi(); - if self.delimiter == Delimiter::None || hi.to_usize() == 0 { - self.span - } else { - let new_lo = BytePos::from_usize(hi.to_usize() - 1); - Span(self.span.0.with_lo(new_lo)) - } + Span(self.span.close) } /// Configures the span for this `Group`'s delimiters, but not its internal @@ -725,7 +711,7 @@ impl Group { /// tokens at the level of the `Group`. #[stable(feature = "proc_macro_lib2", since = "1.29.0")] pub fn set_span(&mut self, span: Span) { - self.span = span; + self.span = DelimSpan::from_single(span.0); } } diff --git a/src/libproc_macro/rustc.rs b/src/libproc_macro/rustc.rs index 21229d3299d7e..3ce02d1afb10b 100644 --- a/src/libproc_macro/rustc.rs +++ b/src/libproc_macro/rustc.rs @@ -64,7 +64,7 @@ impl TokenTree { tokenstream::TokenTree::Delimited(span, delimed) => { let delimiter = Delimiter::from_internal(delimed.delim); let mut g = Group::new(delimiter, ::TokenStream(delimed.tts.into())); - g.set_span(Span(span)); + g.span = span; return g.into(); } }; @@ -192,7 +192,7 @@ impl TokenTree { self::TokenTree::Punct(tt) => (tt.as_char(), tt.spacing(), tt.span()), self::TokenTree::Group(tt) => { return TokenTree::Delimited( - tt.span.0, + tt.span, Delimited { delim: tt.delimiter.to_internal(), tts: tt.stream.0.into(), diff --git a/src/librustc/ich/hcx.rs b/src/librustc/ich/hcx.rs index 371f631737c98..ed4a1e3d72f44 100644 --- a/src/librustc/ich/hcx.rs +++ b/src/librustc/ich/hcx.rs @@ -28,6 +28,7 @@ use syntax::ast; use syntax::source_map::SourceMap; use syntax::ext::hygiene::SyntaxContext; use syntax::symbol::Symbol; +use syntax::tokenstream::DelimSpan; use syntax_pos::{Span, DUMMY_SP}; use syntax_pos::hygiene; @@ -396,6 +397,17 @@ impl<'a> HashStable> for Span { } } +impl<'a> HashStable> for DelimSpan { + fn hash_stable( + &self, + hcx: &mut StableHashingContext<'a>, + hasher: &mut StableHasher, + ) { + self.open.hash_stable(hcx, hasher); + self.close.hash_stable(hcx, hasher); + } +} + pub fn hash_stable_trait_impls<'a, 'gcx, W, R>( hcx: &mut StableHashingContext<'a>, hasher: &mut StableHasher, diff --git a/src/librustc_resolve/macros.rs b/src/librustc_resolve/macros.rs index 879b4ea3fe579..032ec8de2b791 100644 --- a/src/librustc_resolve/macros.rs +++ b/src/librustc_resolve/macros.rs @@ -35,7 +35,7 @@ use syntax::parse::parser::PathStyle; use syntax::parse::token::{self, Token}; use syntax::ptr::P; use syntax::symbol::{Symbol, keywords}; -use syntax::tokenstream::{TokenStream, TokenTree, Delimited}; +use syntax::tokenstream::{TokenStream, TokenTree, Delimited, DelimSpan}; use syntax::util::lev_distance::find_best_match_for_name; use syntax_pos::{Span, DUMMY_SP}; use errors::Applicability; @@ -279,7 +279,8 @@ impl<'a, 'crateloader: 'a> base::Resolver for Resolver<'a, 'crateloader> { tokens.push(TokenTree::Token(path.span, tok).into()); } } - attrs[i].tokens = TokenTree::Delimited(attrs[i].span, Delimited { + let delim_span = DelimSpan::from_single(attrs[i].span); + attrs[i].tokens = TokenTree::Delimited(delim_span, Delimited { delim: token::Paren, tts: TokenStream::concat(tokens).into(), }).into(); diff --git a/src/libsyntax/attr/mod.rs b/src/libsyntax/attr/mod.rs index 19bbbceff5fc0..a980f3ab51584 100644 --- a/src/libsyntax/attr/mod.rs +++ b/src/libsyntax/attr/mod.rs @@ -34,7 +34,7 @@ use parse::token::{self, Token}; use ptr::P; use symbol::Symbol; use ThinVec; -use tokenstream::{TokenStream, TokenTree, Delimited}; +use tokenstream::{TokenStream, TokenTree, Delimited, DelimSpan}; use GLOBALS; use std::iter; @@ -535,7 +535,7 @@ impl MetaItemKind { } tokens.push(item.node.tokens()); } - TokenTree::Delimited(span, Delimited { + TokenTree::Delimited(DelimSpan::from_single(span), Delimited { delim: token::Paren, tts: TokenStream::concat(tokens).into(), }).into() diff --git a/src/libsyntax/ext/quote.rs b/src/libsyntax/ext/quote.rs index 13a139deea4c3..62bc9fae3b59a 100644 --- a/src/libsyntax/ext/quote.rs +++ b/src/libsyntax/ext/quote.rs @@ -10,14 +10,14 @@ use ast::{self, Arg, Arm, Block, Expr, Item, Pat, Stmt, Ty}; use source_map::respan; -use syntax_pos::Span; +use syntax_pos::{Span, DUMMY_SP}; use ext::base::ExtCtxt; use ext::base; use ext::build::AstBuilder; use parse::parser::{Parser, PathStyle}; use parse::token; use ptr::P; -use tokenstream::{TokenStream, TokenTree}; +use tokenstream::{DelimSpan, TokenStream, TokenTree}; /// Quasiquoting works via token trees. /// @@ -36,7 +36,7 @@ pub mod rt { use symbol::Symbol; use ThinVec; - use tokenstream::{self, TokenTree, TokenStream}; + use tokenstream::{self, DelimSpan, TokenTree, TokenStream}; pub use parse::new_parser_from_tts; pub use syntax_pos::{BytePos, Span, DUMMY_SP, FileName}; @@ -245,7 +245,8 @@ pub mod rt { } inner.push(self.tokens.clone()); - r.push(TokenTree::Delimited(self.span, tokenstream::Delimited { + let delim_span = DelimSpan::from_single(self.span); + r.push(TokenTree::Delimited(delim_span, tokenstream::Delimited { delim: token::Bracket, tts: TokenStream::concat(inner).into() })); r @@ -261,7 +262,7 @@ pub mod rt { impl ToTokens for () { fn to_tokens(&self, _cx: &ExtCtxt) -> Vec { - vec![TokenTree::Delimited(DUMMY_SP, tokenstream::Delimited { + vec![TokenTree::Delimited(DelimSpan::dummy(), tokenstream::Delimited { delim: token::Paren, tts: TokenStream::empty().into(), })] @@ -385,13 +386,16 @@ pub fn unflatten(tts: Vec) -> Vec { let mut results = Vec::new(); let mut result = Vec::new(); + let mut open_span = DUMMY_SP; for tree in tts { match tree { - TokenTree::Token(_, token::OpenDelim(..)) => { + TokenTree::Token(span, token::OpenDelim(..)) => { + open_span = span; results.push(::std::mem::replace(&mut result, Vec::new())); } TokenTree::Token(span, token::CloseDelim(delim)) => { - let tree = TokenTree::Delimited(span, Delimited { + let delim_span = DelimSpan::from_pair(open_span, span); + let tree = TokenTree::Delimited(delim_span, Delimited { delim, tts: result.into_iter().map(TokenStream::from).collect::().into(), }); @@ -756,9 +760,9 @@ fn statements_mk_tt(cx: &ExtCtxt, tt: &TokenTree, quoted: bool) -> Vec { - let mut stmts = statements_mk_tt(cx, &delimed.open_tt(span), false); + let mut stmts = statements_mk_tt(cx, &delimed.open_tt(span.open), false); stmts.extend(statements_mk_tts(cx, delimed.stream())); - stmts.extend(statements_mk_tt(cx, &delimed.close_tt(span), false)); + stmts.extend(statements_mk_tt(cx, &delimed.close_tt(span.close), false)); stmts } } diff --git a/src/libsyntax/ext/tt/macro_parser.rs b/src/libsyntax/ext/tt/macro_parser.rs index c962e7fcbb4cf..2ef8da3f6d80c 100644 --- a/src/libsyntax/ext/tt/macro_parser.rs +++ b/src/libsyntax/ext/tt/macro_parser.rs @@ -85,7 +85,7 @@ pub use self::ParseResult::*; use self::TokenTreeOrTokenTreeSlice::*; use ast::Ident; -use syntax_pos::{self, BytePos, Span}; +use syntax_pos::{self, Span}; use errors::FatalError; use ext::tt::quoted::{self, TokenTree}; use parse::{Directory, ParseSess}; @@ -94,7 +94,7 @@ use parse::token::{self, DocComment, Nonterminal, Token}; use print::pprust; use OneVector; use symbol::keywords; -use tokenstream::TokenStream; +use tokenstream::{DelimSpan, TokenStream}; use rustc_data_structures::fx::FxHashMap; use std::collections::hash_map::Entry::{Occupied, Vacant}; @@ -154,7 +154,7 @@ struct MatcherPos<'a> { /// The beginning position in the source that the beginning of this matcher corresponds to. In /// other words, the token in the source at `sp_lo` is matched against the first token of the /// matcher. - sp_lo: BytePos, + sp_lo: Span, /// For each named metavar in the matcher, we keep track of token trees matched against the /// metavar by the black box parser. In particular, there may be more than one match per @@ -285,7 +285,7 @@ fn create_matches(len: usize) -> Vec>> { /// Generate the top-level matcher position in which the "dot" is before the first token of the /// matcher `ms` and we are going to start matching at position `lo` in the source. -fn initial_matcher_pos(ms: &[TokenTree], lo: BytePos) -> MatcherPos { +fn initial_matcher_pos(ms: &[TokenTree], lo: Span) -> MatcherPos { let match_idx_hi = count_names(ms); let matches = create_matches(match_idx_hi); MatcherPos { @@ -332,7 +332,7 @@ fn initial_matcher_pos(ms: &[TokenTree], lo: BytePos) -> MatcherPos { /// token tree it was derived from. #[derive(Debug, Clone)] pub enum NamedMatch { - MatchedSeq(Rc>, syntax_pos::Span), + MatchedSeq(Rc>, DelimSpan), MatchedNonterminal(Rc), } @@ -488,7 +488,7 @@ fn inner_parse_loop<'a>( // Add matches from this repetition to the `matches` of `up` for idx in item.match_lo..item.match_hi { let sub = item.matches[idx].clone(); - let span = span.with_lo(item.sp_lo); + let span = DelimSpan::from_pair(item.sp_lo, span); new_pos.push_match(idx, MatchedSeq(sub, span)); } @@ -556,7 +556,7 @@ fn inner_parse_loop<'a>( match_cur: item.match_cur, match_hi: item.match_cur + seq.num_captures, up: Some(item), - sp_lo: sp.lo(), + sp_lo: sp.open, top_elts: Tt(TokenTree::Sequence(sp, seq)), }))); } @@ -643,7 +643,7 @@ pub fn parse( // // This MatcherPos instance is allocated on the stack. All others -- and // there are frequently *no* others! -- are allocated on the heap. - let mut initial = initial_matcher_pos(ms, parser.span.lo()); + let mut initial = initial_matcher_pos(ms, parser.span); let mut cur_items = smallvec![MatcherPosHandle::Ref(&mut initial)]; let mut next_items = Vec::new(); diff --git a/src/libsyntax/ext/tt/macro_rules.rs b/src/libsyntax/ext/tt/macro_rules.rs index d09127d6b08b7..86247745c4116 100644 --- a/src/libsyntax/ext/tt/macro_rules.rs +++ b/src/libsyntax/ext/tt/macro_rules.rs @@ -25,7 +25,7 @@ use parse::parser::Parser; use parse::token::{self, NtTT}; use parse::token::Token::*; use symbol::Symbol; -use tokenstream::{TokenStream, TokenTree}; +use tokenstream::{DelimSpan, TokenStream, TokenTree}; use rustc_data_structures::fx::FxHashMap; use std::borrow::Cow; @@ -226,7 +226,7 @@ pub fn compile(sess: &ParseSess, features: &Features, def: &ast::Item, edition: // ...quasiquoting this would be nice. // These spans won't matter, anyways let argument_gram = vec![ - quoted::TokenTree::Sequence(DUMMY_SP, Lrc::new(quoted::SequenceRepetition { + quoted::TokenTree::Sequence(DelimSpan::dummy(), Lrc::new(quoted::SequenceRepetition { tts: vec![ quoted::TokenTree::MetaVarDecl(DUMMY_SP, lhs_nm, ast::Ident::from_str("tt")), quoted::TokenTree::Token(DUMMY_SP, token::FatArrow), @@ -237,7 +237,7 @@ pub fn compile(sess: &ParseSess, features: &Features, def: &ast::Item, edition: num_captures: 2, })), // to phase into semicolon-termination instead of semicolon-separation - quoted::TokenTree::Sequence(DUMMY_SP, Lrc::new(quoted::SequenceRepetition { + quoted::TokenTree::Sequence(DelimSpan::dummy(), Lrc::new(quoted::SequenceRepetition { tts: vec![quoted::TokenTree::Token(DUMMY_SP, token::Semi)], separator: None, op: quoted::KleeneOp::ZeroOrMore, @@ -400,7 +400,8 @@ fn check_lhs_no_empty_seq(sess: &ParseSess, tts: &[quoted::TokenTree]) -> bool { _ => false, } }) { - sess.span_diagnostic.span_err(span, "repetition matches empty token tree"); + let sp = span.entire(); + sess.span_diagnostic.span_err(sp, "repetition matches empty token tree"); return false; } if !check_lhs_no_empty_seq(sess, &seq.tts) { @@ -474,12 +475,12 @@ impl FirstSets { } TokenTree::Delimited(span, ref delimited) => { build_recur(sets, &delimited.tts[..]); - first.replace_with(delimited.open_tt(span)); + first.replace_with(delimited.open_tt(span.open)); } TokenTree::Sequence(sp, ref seq_rep) => { let subfirst = build_recur(sets, &seq_rep.tts[..]); - match sets.first.entry(sp) { + match sets.first.entry(sp.entire()) { Entry::Vacant(vac) => { vac.insert(Some(subfirst.clone())); } @@ -499,7 +500,7 @@ impl FirstSets { if let (Some(ref sep), true) = (seq_rep.separator.clone(), subfirst.maybe_empty) { - first.add_one_maybe(TokenTree::Token(sp, sep.clone())); + first.add_one_maybe(TokenTree::Token(sp.entire(), sep.clone())); } // Reverse scan: Sequence comes before `first`. @@ -534,11 +535,11 @@ impl FirstSets { return first; } TokenTree::Delimited(span, ref delimited) => { - first.add_one(delimited.open_tt(span)); + first.add_one(delimited.open_tt(span.open)); return first; } TokenTree::Sequence(sp, ref seq_rep) => { - match self.first.get(&sp) { + match self.first.get(&sp.entire()) { Some(&Some(ref subfirst)) => { // If the sequence contents can be empty, then the first @@ -546,7 +547,7 @@ impl FirstSets { if let (Some(ref sep), true) = (seq_rep.separator.clone(), subfirst.maybe_empty) { - first.add_one_maybe(TokenTree::Token(sp, sep.clone())); + first.add_one_maybe(TokenTree::Token(sp.entire(), sep.clone())); } assert!(first.maybe_empty); @@ -727,7 +728,7 @@ fn check_matcher_core(sess: &ParseSess, } } TokenTree::Delimited(span, ref d) => { - let my_suffix = TokenSet::singleton(d.close_tt(span)); + let my_suffix = TokenSet::singleton(d.close_tt(span.close)); check_matcher_core(sess, features, attrs, first_sets, &d.tts, &my_suffix); // don't track non NT tokens last.replace_with_irrelevant(); @@ -751,7 +752,7 @@ fn check_matcher_core(sess: &ParseSess, let mut new; let my_suffix = if let Some(ref u) = seq_rep.separator { new = suffix_first.clone(); - new.add_one_maybe(TokenTree::Token(sp, u.clone())); + new.add_one_maybe(TokenTree::Token(sp.entire(), u.clone())); &new } else { &suffix_first diff --git a/src/libsyntax/ext/tt/quoted.rs b/src/libsyntax/ext/tt/quoted.rs index cc635a29275fa..74363f3e5f7d9 100644 --- a/src/libsyntax/ext/tt/quoted.rs +++ b/src/libsyntax/ext/tt/quoted.rs @@ -16,7 +16,7 @@ use parse::{token, ParseSess}; use print::pprust; use symbol::keywords; use syntax_pos::{edition::Edition, BytePos, Span}; -use tokenstream; +use tokenstream::{self, DelimSpan}; use {ast, attr}; use rustc_data_structures::sync::Lrc; @@ -90,9 +90,9 @@ pub enum KleeneOp { #[derive(Debug, Clone, PartialEq, RustcEncodable, RustcDecodable)] pub enum TokenTree { Token(Span, token::Token), - Delimited(Span, Lrc), + Delimited(DelimSpan, Lrc), /// A kleene-style repetition sequence - Sequence(Span, Lrc), + Sequence(DelimSpan, Lrc), /// E.g. `$var` MetaVar(Span, ast::Ident), /// E.g. `$var:expr`. This is only used in the left hand side of MBE macros. @@ -137,10 +137,10 @@ impl TokenTree { } (&TokenTree::Delimited(span, ref delimed), _) => { if index == 0 { - return delimed.open_tt(span); + return delimed.open_tt(span.open); } if index == delimed.tts.len() + 1 { - return delimed.close_tt(span); + return delimed.close_tt(span.close); } delimed.tts[index - 1].clone() } @@ -154,9 +154,9 @@ impl TokenTree { match *self { TokenTree::Token(sp, _) | TokenTree::MetaVar(sp, _) - | TokenTree::MetaVarDecl(sp, _, _) - | TokenTree::Delimited(sp, _) - | TokenTree::Sequence(sp, _) => sp, + | TokenTree::MetaVarDecl(sp, _, _) => sp, + TokenTree::Delimited(sp, _) + | TokenTree::Sequence(sp, _) => sp.entire(), } } } @@ -286,7 +286,7 @@ where if delimited.delim != token::Paren { let tok = pprust::token_to_string(&token::OpenDelim(delimited.delim)); let msg = format!("expected `(`, found `{}`", tok); - sess.span_diagnostic.span_err(span, &msg); + sess.span_diagnostic.span_err(span.entire(), &msg); } // Parse the contents of the sequence itself let sequence = parse( @@ -302,7 +302,7 @@ where let (separator, op) = parse_sep_and_kleene_op( trees, - span, + span.entire(), sess, features, attrs, diff --git a/src/libsyntax/ext/tt/transcribe.rs b/src/libsyntax/ext/tt/transcribe.rs index 549e5f00dcec4..2ed469e8e77f8 100644 --- a/src/libsyntax/ext/tt/transcribe.rs +++ b/src/libsyntax/ext/tt/transcribe.rs @@ -16,8 +16,8 @@ use ext::tt::quoted; use fold::noop_fold_tt; use parse::token::{self, Token, NtTT}; use OneVector; -use syntax_pos::{Span, DUMMY_SP}; -use tokenstream::{TokenStream, TokenTree, Delimited}; +use syntax_pos::DUMMY_SP; +use tokenstream::{TokenStream, TokenTree, Delimited, DelimSpan}; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::sync::Lrc; @@ -30,7 +30,7 @@ enum Frame { Delimited { forest: Lrc, idx: usize, - span: Span, + span: DelimSpan, }, Sequence { forest: Lrc, @@ -42,7 +42,7 @@ enum Frame { impl Frame { fn new(tts: Vec) -> Frame { let forest = Lrc::new(quoted::Delimited { delim: token::NoDelim, tts: tts }); - Frame::Delimited { forest: forest, idx: 0, span: DUMMY_SP } + Frame::Delimited { forest: forest, idx: 0, span: DelimSpan::dummy() } } } @@ -123,20 +123,20 @@ pub fn transcribe(cx: &ExtCtxt, &interpolations, &repeats) { LockstepIterSize::Unconstrained => { - cx.span_fatal(sp, /* blame macro writer */ + cx.span_fatal(sp.entire(), /* blame macro writer */ "attempted to repeat an expression \ containing no syntax \ variables matched as repeating at this depth"); } LockstepIterSize::Contradiction(ref msg) => { // FIXME #2887 blame macro invoker instead - cx.span_fatal(sp, &msg[..]); + cx.span_fatal(sp.entire(), &msg[..]); } LockstepIterSize::Constraint(len, _) => { if len == 0 { if seq.op == quoted::KleeneOp::OneOrMore { // FIXME #2887 blame invoker - cx.span_fatal(sp, "this must repeat at least once"); + cx.span_fatal(sp.entire(), "this must repeat at least once"); } } else { repeats.push((0, len)); diff --git a/src/libsyntax/fold.rs b/src/libsyntax/fold.rs index dff408d233977..032393b4f1253 100644 --- a/src/libsyntax/fold.rs +++ b/src/libsyntax/fold.rs @@ -594,10 +594,13 @@ pub fn noop_fold_tt(tt: TokenTree, fld: &mut T) -> TokenTree { match tt { TokenTree::Token(span, tok) => TokenTree::Token(fld.new_span(span), fld.fold_token(tok)), - TokenTree::Delimited(span, delimed) => TokenTree::Delimited(fld.new_span(span), Delimited { - tts: fld.fold_tts(delimed.stream()).into(), - delim: delimed.delim, - }), + TokenTree::Delimited(span, delimed) => TokenTree::Delimited( + DelimSpan::from_pair(fld.new_span(span.open), fld.new_span(span.close)), + Delimited { + tts: fld.fold_tts(delimed.stream()).into(), + delim: delimed.delim, + } + ), } } diff --git a/src/libsyntax/parse/lexer/tokentrees.rs b/src/libsyntax/parse/lexer/tokentrees.rs index e2fd7faf90387..d19748937e108 100644 --- a/src/libsyntax/parse/lexer/tokentrees.rs +++ b/src/libsyntax/parse/lexer/tokentrees.rs @@ -11,7 +11,7 @@ use print::pprust::token_to_string; use parse::lexer::StringReader; use parse::{token, PResult}; -use tokenstream::{Delimited, TokenStream, TokenTree}; +use tokenstream::{Delimited, DelimSpan, TokenStream, TokenTree}; impl<'a> StringReader<'a> { // Parse a stream of tokens into a list of `TokenTree`s, up to an `Eof`. @@ -68,7 +68,7 @@ impl<'a> StringReader<'a> { let tts = self.parse_token_trees_until_close_delim(); // Expand to cover the entire delimited token tree - let span = pre_span.with_hi(self.span.hi()); + let delim_span = DelimSpan::from_pair(pre_span, self.span); match self.token { // Correct delimiter. @@ -119,7 +119,7 @@ impl<'a> StringReader<'a> { _ => {} } - Ok(TokenTree::Delimited(span, Delimited { + Ok(TokenTree::Delimited(delim_span, Delimited { delim, tts: tts.into(), }).into()) diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs index 28d63399b4461..5c6d5816a472b 100644 --- a/src/libsyntax/parse/mod.rs +++ b/src/libsyntax/parse/mod.rs @@ -722,7 +722,7 @@ mod tests { use attr::first_attr_value_str_by_name; use parse; use print::pprust::item_to_string; - use tokenstream::{self, TokenTree}; + use tokenstream::{self, DelimSpan, TokenTree}; use util::parser_testing::string_to_stream; use util::parser_testing::{string_to_expr, string_to_item}; use with_globals; @@ -805,7 +805,7 @@ mod tests { TokenTree::Token(sp(0, 2), token::Ident(Ident::from_str("fn"), false)).into(), TokenTree::Token(sp(3, 4), token::Ident(Ident::from_str("a"), false)).into(), TokenTree::Delimited( - sp(5, 14), + DelimSpan::from_pair(sp(5, 6), sp(13, 14)), tokenstream::Delimited { delim: token::DelimToken::Paren, tts: TokenStream::concat(vec![ @@ -817,7 +817,7 @@ mod tests { ]).into(), }).into(), TokenTree::Delimited( - sp(15, 21), + DelimSpan::from_pair(sp(15, 16), sp(20, 21)), tokenstream::Delimited { delim: token::DelimToken::Brace, tts: TokenStream::concat(vec![ diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs index c741bde7c5f24..b9d4e9fc268d8 100644 --- a/src/libsyntax/parse/parser.rs +++ b/src/libsyntax/parse/parser.rs @@ -54,7 +54,7 @@ use print::pprust; use ptr::P; use parse::PResult; use ThinVec; -use tokenstream::{self, Delimited, ThinTokenStream, TokenTree, TokenStream}; +use tokenstream::{self, Delimited, DelimSpan, ThinTokenStream, TokenTree, TokenStream}; use symbol::{Symbol, keywords}; use std::borrow::Cow; @@ -262,7 +262,7 @@ struct TokenCursor { #[derive(Clone)] struct TokenCursorFrame { delim: token::DelimToken, - span: Span, + span: DelimSpan, open_delim: bool, tree_cursor: tokenstream::Cursor, close_delim: bool, @@ -293,7 +293,7 @@ enum LastToken { } impl TokenCursorFrame { - fn new(sp: Span, delimited: &Delimited) -> Self { + fn new(sp: DelimSpan, delimited: &Delimited) -> Self { TokenCursorFrame { delim: delimited.delim, span: sp, @@ -311,13 +311,13 @@ impl TokenCursor { let tree = if !self.frame.open_delim { self.frame.open_delim = true; Delimited { delim: self.frame.delim, tts: TokenStream::empty().into() } - .open_tt(self.frame.span) + .open_tt(self.frame.span.open) } else if let Some(tree) = self.frame.tree_cursor.next() { tree } else if !self.frame.close_delim { self.frame.close_delim = true; Delimited { delim: self.frame.delim, tts: TokenStream::empty().into() } - .close_tt(self.frame.span) + .close_tt(self.frame.span.close) } else if let Some(frame) = self.stack.pop() { self.frame = frame; continue @@ -361,7 +361,8 @@ impl TokenCursor { num_of_hashes = cmp::max(num_of_hashes, count); } - let body = TokenTree::Delimited(sp, Delimited { + let delim_span = DelimSpan::from_single(sp); + let body = TokenTree::Delimited(delim_span, Delimited { delim: token::Bracket, tts: [TokenTree::Token(sp, token::Ident(ast::Ident::from_str("doc"), false)), TokenTree::Token(sp, token::Eq), @@ -370,7 +371,7 @@ impl TokenCursor { .iter().cloned().collect::().into(), }); - self.stack.push(mem::replace(&mut self.frame, TokenCursorFrame::new(sp, &Delimited { + self.stack.push(mem::replace(&mut self.frame, TokenCursorFrame::new(delim_span, &Delimited { delim: token::NoDelim, tts: if doc_comment_style(&name.as_str()) == AttrStyle::Inner { [TokenTree::Token(sp, token::Pound), TokenTree::Token(sp, token::Not), body] @@ -560,7 +561,7 @@ impl<'a> Parser<'a> { root_module_name: None, expected_tokens: Vec::new(), token_cursor: TokenCursor { - frame: TokenCursorFrame::new(syntax_pos::DUMMY_SP, &Delimited { + frame: TokenCursorFrame::new(DelimSpan::dummy(), &Delimited { delim: token::NoDelim, tts: tokens.into(), }), @@ -1229,7 +1230,8 @@ impl<'a> Parser<'a> { } match self.token_cursor.frame.tree_cursor.look_ahead(dist - 1) { - Some(TokenTree::Token(span, _)) | Some(TokenTree::Delimited(span, _)) => span, + Some(TokenTree::Token(span, _)) => span, + Some(TokenTree::Delimited(span, _)) => span.entire(), None => self.look_ahead_span(dist - 1), } } @@ -2796,7 +2798,7 @@ impl<'a> Parser<'a> { token::OpenDelim(..) => { let frame = mem::replace(&mut self.token_cursor.frame, self.token_cursor.stack.pop().unwrap()); - self.span = frame.span; + self.span = frame.span.entire(); self.bump(); TokenTree::Delimited(frame.span, Delimited { delim: frame.delim, diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs index 1d0c6b5317a38..6e8014284ec40 100644 --- a/src/libsyntax/parse/token.rs +++ b/src/libsyntax/parse/token.rs @@ -23,8 +23,7 @@ use symbol::keywords; use syntax::parse::parse_stream_from_source_str; use syntax_pos::{self, Span, FileName}; use syntax_pos::symbol::{self, Symbol}; -use tokenstream::{TokenStream, TokenTree}; -use tokenstream; +use tokenstream::{self, DelimSpan, TokenStream, TokenTree}; use std::{cmp, fmt}; use std::mem; @@ -825,7 +824,8 @@ fn prepend_attrs(sess: &ParseSess, // that it encompasses more than each token, but it hopefully is "good // enough" for now at least. builder.push(tokenstream::TokenTree::Token(attr.span, Pound)); - builder.push(tokenstream::TokenTree::Delimited(attr.span, tokens)); + let delim_span = DelimSpan::from_single(attr.span); + builder.push(tokenstream::TokenTree::Delimited(delim_span, tokens)); } builder.push(tokens.clone()); Some(builder.build()) diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs index 840ee299bf338..70867f9e42ff5 100644 --- a/src/libsyntax/tokenstream.rs +++ b/src/libsyntax/tokenstream.rs @@ -22,7 +22,7 @@ //! and a borrowed `TokenStream` is sufficient to build an owned `TokenStream` without taking //! ownership of the original. -use syntax_pos::{BytePos, Span, DUMMY_SP}; +use syntax_pos::{BytePos, Mark, Span, DUMMY_SP}; use ext::base; use ext::tt::{macro_parser, quoted}; use parse::Directory; @@ -97,7 +97,7 @@ pub enum TokenTree { /// A single token Token(Span, token::Token), /// A delimited sequence of token trees - Delimited(Span, Delimited), + Delimited(DelimSpan, Delimited), } impl TokenTree { @@ -145,16 +145,16 @@ impl TokenTree { /// Retrieve the TokenTree's span. pub fn span(&self) -> Span { match *self { - TokenTree::Token(sp, _) | TokenTree::Delimited(sp, _) => sp, + TokenTree::Token(sp, _) => sp, + TokenTree::Delimited(sp, _) => sp.entire(), } } /// Modify the `TokenTree`'s span inplace. pub fn set_span(&mut self, span: Span) { match *self { - TokenTree::Token(ref mut sp, _) | TokenTree::Delimited(ref mut sp, _) => { - *sp = span; - } + TokenTree::Token(ref mut sp, _) => *sp = span, + TokenTree::Delimited(ref mut sp, _) => *sp = DelimSpan::from_single(span), } } @@ -192,27 +192,20 @@ impl TokenStream { let mut iter = slice.iter().enumerate().peekable(); while let Some((pos, ts)) = iter.next() { if let Some((_, next)) = iter.peek() { - match (ts, next) { - (TokenStream { - kind: TokenStreamKind::Tree(TokenTree::Token(_, token::Token::Comma)) - }, _) | - (_, TokenStream { - kind: TokenStreamKind::Tree(TokenTree::Token(_, token::Token::Comma)) - }) => {} - (TokenStream { - kind: TokenStreamKind::Tree(TokenTree::Token(sp, _)) - }, _) | - (TokenStream { - kind: TokenStreamKind::Tree(TokenTree::Delimited(sp, _)) - }, _) => { - let sp = sp.shrink_to_hi(); - let comma = TokenStream { - kind: TokenStreamKind::Tree(TokenTree::Token(sp, token::Comma)), - }; - suggestion = Some((pos, comma, sp)); + let sp = match (&ts.kind, &next.kind) { + (TokenStreamKind::Tree(TokenTree::Token(_, token::Token::Comma)), _) | + (_, TokenStreamKind::Tree(TokenTree::Token(_, token::Token::Comma))) => { + continue; } - _ => {} - } + (TokenStreamKind::Tree(TokenTree::Token(sp, _)), _) => *sp, + (TokenStreamKind::Tree(TokenTree::Delimited(sp, _)), _) => sp.entire(), + _ => continue, + }; + let sp = sp.shrink_to_hi(); + let comma = TokenStream { + kind: TokenStreamKind::Tree(TokenTree::Token(sp, token::Comma)), + }; + suggestion = Some((pos, comma, sp)); } } if let Some((pos, comma, sp)) = suggestion { @@ -718,6 +711,40 @@ impl Decodable for ThinTokenStream { } } +#[derive(Debug, Copy, Clone, PartialEq, RustcEncodable, RustcDecodable)] +pub struct DelimSpan { + pub open: Span, + pub close: Span, +} + +impl DelimSpan { + pub fn from_single(sp: Span) -> Self { + DelimSpan { + open: sp, + close: sp, + } + } + + pub fn from_pair(open: Span, close: Span) -> Self { + DelimSpan { open, close } + } + + pub fn dummy() -> Self { + Self::from_single(DUMMY_SP) + } + + pub fn entire(self) -> Span { + self.open.with_hi(self.close.hi()) + } + + pub fn apply_mark(self, mark: Mark) -> Self { + DelimSpan { + open: self.open.apply_mark(mark), + close: self.close.apply_mark(mark), + } + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/test/run-pass-fulldeps/auxiliary/procedural_mbe_matching.rs b/src/test/run-pass-fulldeps/auxiliary/procedural_mbe_matching.rs index f1777745e06c3..f2f14f84923c0 100644 --- a/src/test/run-pass-fulldeps/auxiliary/procedural_mbe_matching.rs +++ b/src/test/run-pass-fulldeps/auxiliary/procedural_mbe_matching.rs @@ -70,7 +70,8 @@ fn expand_mbe_matches(cx: &mut ExtCtxt, _: Span, args: &[TokenTree]) _ => unreachable!(), } }).collect(); - let arm = cx.arm(seq_sp, pats, cx.expr_bool(seq_sp, true)); + let span = seq_sp.entire(); + let arm = cx.arm(span, pats, cx.expr_bool(span, true)); quote_expr!(cx, match $matched_expr { From 57d6ada91d7c2d9af1115ae515b000fda93b3c4e Mon Sep 17 00:00:00 2001 From: David Tolnay Date: Sat, 8 Sep 2018 23:47:42 -0700 Subject: [PATCH 3/3] Rename sp_lo to sp_open --- src/libsyntax/ext/tt/macro_parser.rs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/libsyntax/ext/tt/macro_parser.rs b/src/libsyntax/ext/tt/macro_parser.rs index 2ef8da3f6d80c..941fd6f1000a3 100644 --- a/src/libsyntax/ext/tt/macro_parser.rs +++ b/src/libsyntax/ext/tt/macro_parser.rs @@ -151,10 +151,10 @@ struct MatcherPos<'a> { top_elts: TokenTreeOrTokenTreeSlice<'a>, /// The position of the "dot" in this matcher idx: usize, - /// The beginning position in the source that the beginning of this matcher corresponds to. In - /// other words, the token in the source at `sp_lo` is matched against the first token of the - /// matcher. - sp_lo: Span, + /// The first span of source source that the beginning of this matcher corresponds to. In other + /// words, the token in the source whose span is `sp_open` is matched against the first token of + /// the matcher. + sp_open: Span, /// For each named metavar in the matcher, we keep track of token trees matched against the /// metavar by the black box parser. In particular, there may be more than one match per @@ -284,8 +284,8 @@ fn create_matches(len: usize) -> Vec>> { } /// Generate the top-level matcher position in which the "dot" is before the first token of the -/// matcher `ms` and we are going to start matching at position `lo` in the source. -fn initial_matcher_pos(ms: &[TokenTree], lo: Span) -> MatcherPos { +/// matcher `ms` and we are going to start matching at the span `open` in the source. +fn initial_matcher_pos(ms: &[TokenTree], open: Span) -> MatcherPos { let match_idx_hi = count_names(ms); let matches = create_matches(match_idx_hi); MatcherPos { @@ -293,8 +293,8 @@ fn initial_matcher_pos(ms: &[TokenTree], lo: Span) -> MatcherPos { top_elts: TtSeq(ms), // "elts" is an abbr. for "elements" // The "dot" is before the first token of the matcher idx: 0, - // We start matching with byte `lo` in the source code - sp_lo: lo, + // We start matching at the span `open` in the source code + sp_open: open, // Initialize `matches` to a bunch of empty `Vec`s -- one for each metavar in `top_elts`. // `match_lo` for `top_elts` is 0 and `match_hi` is `matches.len()`. `match_cur` is 0 since @@ -488,7 +488,7 @@ fn inner_parse_loop<'a>( // Add matches from this repetition to the `matches` of `up` for idx in item.match_lo..item.match_hi { let sub = item.matches[idx].clone(); - let span = DelimSpan::from_pair(item.sp_lo, span); + let span = DelimSpan::from_pair(item.sp_open, span); new_pos.push_match(idx, MatchedSeq(sub, span)); } @@ -556,7 +556,7 @@ fn inner_parse_loop<'a>( match_cur: item.match_cur, match_hi: item.match_cur + seq.num_captures, up: Some(item), - sp_lo: sp.open, + sp_open: sp.open, top_elts: Tt(TokenTree::Sequence(sp, seq)), }))); }