Skip to content

Commit f745e5f

Browse files
petrochenkovpetrochenkov
committed
syntax: Remove duplicate span from token::Ident 
1 parent 4c5d773 commit f745e5f

21 files changed

+181
-184
lines changed

‎src/librustc/ich/impls_syntax.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -353,8 +353,8 @@ impl<'a> HashStable<StableHashingContext<'a>> for token::TokenKind {
353353
}
354354
token::Literal(lit) => lit.hash_stable(hcx, hasher),
355355

356-
token::Ident(ident, is_raw) => {
357-
ident.name.hash_stable(hcx, hasher);
356+
token::Ident(name, is_raw) => {
357+
name.hash_stable(hcx, hasher);
358358
is_raw.hash_stable(hcx, hasher);
359359
}
360360
token::Lifetime(name) => name.hash_stable(hcx, hasher),

‎src/librustdoc/html/highlight.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -325,8 +325,8 @@ impl<'a> Classifier<'a> {
325325
}
326326

327327
// Keywords are also included in the identifier set.
328-
token::Ident(ident, is_raw) => {
329-
match ident.name {
328+
token::Ident(name, is_raw) => {
329+
match name {
330330
kw::Ref | kw::Mut if !is_raw => Class::RefKeyWord,
331331

332332
kw::SelfLower | kw::SelfUpper => Class::Self_,

‎src/libsyntax/attr/mod.rs

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -482,19 +482,19 @@ impl MetaItem {
482482
let path = match tokens.next() {
483483
Some(TokenTree::Token(Token { kind: kind @ token::Ident(..), span })) |
484484
Some(TokenTree::Token(Token { kind: kind @ token::ModSep, span })) => 'arm: {
485-
let mut segments = if let token::Ident(ident, _) = kind {
485+
let mut segments = if let token::Ident(name, _) = kind {
486486
if let Some(TokenTree::Token(Token { kind: token::ModSep, .. })) = tokens.peek() {
487487
tokens.next();
488-
vec![PathSegment::from_ident(ident.with_span_pos(span))]
488+
vec![PathSegment::from_ident(Ident::new(name, span))]
489489
} else {
490-
break 'arm Path::from_ident(ident.with_span_pos(span));
490+
break 'arm Path::from_ident(Ident::new(name, span));
491491
}
492492
} else {
493493
vec![PathSegment::path_root(span)]
494494
};
495495
loop {
496-
if let Some(TokenTree::Token(Token { kind: token::Ident(ident, _), span })) = tokens.next() {
497-
segments.push(PathSegment::from_ident(ident.with_span_pos(span)));
496+
if let Some(TokenTree::Token(Token { kind: token::Ident(name, _), span })) = tokens.next() {
497+
segments.push(PathSegment::from_ident(Ident::new(name, span)));
498498
} else {
499499
return None;
500500
}

‎src/libsyntax/diagnostics/plugin.rs

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ pub fn expand_diagnostic_used<'cx>(ecx: &'cx mut ExtCtxt<'_>,
3939
};
4040

4141
ecx.parse_sess.registered_diagnostics.with_lock(|diagnostics| {
42-
match diagnostics.get_mut(&code.name) {
42+
match diagnostics.get_mut(&code) {
4343
// Previously used errors.
4444
Some(&mut ErrorInfo { description: _, use_site: Some(previous_span) }) => {
4545
ecx.struct_span_warn(span, &format!(
@@ -72,10 +72,10 @@ pub fn expand_register_diagnostic<'cx>(ecx: &'cx mut ExtCtxt<'_>,
7272
token_tree.get(1),
7373
token_tree.get(2)
7474
) {
75-
(1, Some(&TokenTree::Token(Token { kind: token::Ident(ref code, _), .. })), None, None) => {
75+
(1, Some(&TokenTree::Token(Token { kind: token::Ident(code, _), .. })), None, None) => {
7676
(code, None)
7777
},
78-
(3, Some(&TokenTree::Token(Token { kind: token::Ident(ref code, _), .. })),
78+
(3, Some(&TokenTree::Token(Token { kind: token::Ident(code, _), .. })),
7979
Some(&TokenTree::Token(Token { kind: token::Comma, .. })),
8080
Some(&TokenTree::Token(Token { kind: token::Literal(token::Lit { symbol, .. }), .. }))) => {
8181
(code, Some(symbol))
@@ -112,7 +112,7 @@ pub fn expand_register_diagnostic<'cx>(ecx: &'cx mut ExtCtxt<'_>,
112112
description,
113113
use_site: None
114114
};
115-
if diagnostics.insert(code.name, info).is_some() {
115+
if diagnostics.insert(code, info).is_some() {
116116
ecx.span_err(span, &format!(
117117
"diagnostic code {} already registered", code
118118
));
@@ -140,13 +140,13 @@ pub fn expand_build_diagnostic_array<'cx>(ecx: &'cx mut ExtCtxt<'_>,
140140
token_tree: &[TokenTree])
141141
-> Box<dyn MacResult+'cx> {
142142
assert_eq!(token_tree.len(), 3);
143-
let (crate_name, name) = match (&token_tree[0], &token_tree[2]) {
143+
let (crate_name, ident) = match (&token_tree[0], &token_tree[2]) {
144144
(
145145
// Crate name.
146-
&TokenTree::Token(Token { kind: token::Ident(ref crate_name, _), .. }),
146+
&TokenTree::Token(Token { kind: token::Ident(crate_name, _), .. }),
147147
// DIAGNOSTICS ident.
148-
&TokenTree::Token(Token { kind: token::Ident(ref name, _), .. })
149-
) => (*&crate_name, name),
148+
&TokenTree::Token(Token { kind: token::Ident(name, _), span })
149+
) => (crate_name, Ident::new(name, span)),
150150
_ => unreachable!()
151151
};
152152

@@ -209,7 +209,7 @@ pub fn expand_build_diagnostic_array<'cx>(ecx: &'cx mut ExtCtxt<'_>,
209209

210210
MacEager::items(smallvec![
211211
P(ast::Item {
212-
ident: *name,
212+
ident,
213213
attrs: Vec::new(),
214214
id: ast::DUMMY_NODE_ID,
215215
node: ast::ItemKind::Const(

‎src/libsyntax/ext/base.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -269,7 +269,7 @@ impl<F> TTMacroExpander for F
269269
if let token::Interpolated(nt) = &token.kind {
270270
if let token::NtIdent(ident, is_raw) = **nt {
271271
*tt = tokenstream::TokenTree::token(ident.span,
272-
token::Ident(ident, is_raw));
272+
token::Ident(ident.name, is_raw));
273273
}
274274
}
275275
}

‎src/libsyntax/ext/tt/macro_parser.rs

Lines changed: 18 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ pub use NamedMatch::*;
7474
pub use ParseResult::*;
7575
use TokenTreeOrTokenTreeSlice::*;
7676

77-
use crate::ast::Ident;
77+
use crate::ast::{Ident, Name};
7878
use crate::ext::tt::quoted::{self, TokenTree};
7979
use crate::parse::{Directory, ParseSess};
8080
use crate::parse::parser::{Parser, PathStyle};
@@ -429,8 +429,8 @@ pub fn parse_failure_msg(tok: TokenKind) -> String {
429429

430430
/// Performs a token equality check, ignoring syntax context (that is, an unhygienic comparison)
431431
fn token_name_eq(t1: &TokenKind, t2: &TokenKind) -> bool {
432-
if let (Some((id1, is_raw1)), Some((id2, is_raw2))) = (t1.ident(), t2.ident()) {
433-
id1.name == id2.name && is_raw1 == is_raw2
432+
if let (Some((name1, is_raw1)), Some((name2, is_raw2))) = (t1.ident_name(), t2.ident_name()) {
433+
name1 == name2 && is_raw1 == is_raw2
434434
} else if let (Some(name1), Some(name2)) = (t1.lifetime_name(), t2.lifetime_name()) {
435435
name1 == name2
436436
} else {
@@ -466,8 +466,7 @@ fn inner_parse_loop<'root, 'tt>(
466466
next_items: &mut Vec<MatcherPosHandle<'root, 'tt>>,
467467
eof_items: &mut SmallVec<[MatcherPosHandle<'root, 'tt>; 1]>,
468468
bb_items: &mut SmallVec<[MatcherPosHandle<'root, 'tt>; 1]>,
469-
token: &TokenKind,
470-
span: syntax_pos::Span,
469+
token: &Token,
471470
) -> ParseResult<()> {
472471
// Pop items from `cur_items` until it is empty.
473472
while let Some(mut item) = cur_items.pop() {
@@ -510,7 +509,7 @@ fn inner_parse_loop<'root, 'tt>(
510509
// Add matches from this repetition to the `matches` of `up`
511510
for idx in item.match_lo..item.match_hi {
512511
let sub = item.matches[idx].clone();
513-
let span = DelimSpan::from_pair(item.sp_open, span);
512+
let span = DelimSpan::from_pair(item.sp_open, token.span);
514513
new_pos.push_match(idx, MatchedSeq(sub, span));
515514
}
516515

@@ -598,7 +597,7 @@ fn inner_parse_loop<'root, 'tt>(
598597
TokenTree::MetaVarDecl(_, _, id) => {
599598
// Built-in nonterminals never start with these tokens,
600599
// so we can eliminate them from consideration.
601-
if may_begin_with(id.name, token) {
600+
if may_begin_with(token, id.name) {
602601
bb_items.push(item);
603602
}
604603
}
@@ -698,7 +697,6 @@ pub fn parse(
698697
&mut eof_items,
699698
&mut bb_items,
700699
&parser.token,
701-
parser.span,
702700
) {
703701
Success(_) => {}
704702
Failure(token, msg) => return Failure(token, msg),
@@ -806,10 +804,9 @@ pub fn parse(
806804

807805
/// The token is an identifier, but not `_`.
808806
/// We prohibit passing `_` to macros expecting `ident` for now.
809-
fn get_macro_ident(token: &TokenKind) -> Option<(Ident, bool)> {
807+
fn get_macro_name(token: &TokenKind) -> Option<(Name, bool)> {
810808
match *token {
811-
token::Ident(ident, is_raw) if ident.name != kw::Underscore =>
812-
Some((ident, is_raw)),
809+
token::Ident(name, is_raw) if name != kw::Underscore => Some((name, is_raw)),
813810
_ => None,
814811
}
815812
}
@@ -818,7 +815,7 @@ fn get_macro_ident(token: &TokenKind) -> Option<(Ident, bool)> {
818815
///
819816
/// Returning `false` is a *stability guarantee* that such a matcher will *never* begin with that
820817
/// token. Be conservative (return true) if not sure.
821-
fn may_begin_with(name: Symbol, token: &TokenKind) -> bool {
818+
fn may_begin_with(token: &Token, name: Name) -> bool {
822819
/// Checks whether the non-terminal may contain a single (non-keyword) identifier.
823820
fn may_be_ident(nt: &token::Nonterminal) -> bool {
824821
match *nt {
@@ -830,14 +827,14 @@ fn may_begin_with(name: Symbol, token: &TokenKind) -> bool {
830827
match name {
831828
sym::expr => token.can_begin_expr(),
832829
sym::ty => token.can_begin_type(),
833-
sym::ident => get_macro_ident(token).is_some(),
830+
sym::ident => get_macro_name(token).is_some(),
834831
sym::literal => token.can_begin_literal_or_bool(),
835-
sym::vis => match *token {
832+
sym::vis => match token.kind {
836833
// The follow-set of :vis + "priv" keyword + interpolated
837834
token::Comma | token::Ident(..) | token::Interpolated(_) => true,
838835
_ => token.can_begin_type(),
839836
},
840-
sym::block => match *token {
837+
sym::block => match token.kind {
841838
token::OpenDelim(token::Brace) => true,
842839
token::Interpolated(ref nt) => match **nt {
843840
token::NtItem(_)
@@ -851,15 +848,15 @@ fn may_begin_with(name: Symbol, token: &TokenKind) -> bool {
851848
},
852849
_ => false,
853850
},
854-
sym::path | sym::meta => match *token {
851+
sym::path | sym::meta => match token.kind {
855852
token::ModSep | token::Ident(..) => true,
856853
token::Interpolated(ref nt) => match **nt {
857854
token::NtPath(_) | token::NtMeta(_) => true,
858855
_ => may_be_ident(&nt),
859856
},
860857
_ => false,
861858
},
862-
sym::pat => match *token {
859+
sym::pat => match token.kind {
863860
token::Ident(..) | // box, ref, mut, and other identifiers (can stricten)
864861
token::OpenDelim(token::Paren) | // tuple pattern
865862
token::OpenDelim(token::Bracket) | // slice pattern
@@ -875,15 +872,15 @@ fn may_begin_with(name: Symbol, token: &TokenKind) -> bool {
875872
token::Interpolated(ref nt) => may_be_ident(nt),
876873
_ => false,
877874
},
878-
sym::lifetime => match *token {
875+
sym::lifetime => match token.kind {
879876
token::Lifetime(_) => true,
880877
token::Interpolated(ref nt) => match **nt {
881878
token::NtLifetime(_) | token::NtTT(_) => true,
882879
_ => false,
883880
},
884881
_ => false,
885882
},
886-
_ => match *token {
883+
_ => match token.kind {
887884
token::CloseDelim(_) => false,
888885
_ => true,
889886
},
@@ -929,10 +926,10 @@ fn parse_nt<'a>(p: &mut Parser<'a>, sp: Span, name: Symbol) -> Nonterminal {
929926
sym::literal => token::NtLiteral(panictry!(p.parse_literal_maybe_minus())),
930927
sym::ty => token::NtTy(panictry!(p.parse_ty())),
931928
// this could be handled like a token, since it is one
932-
sym::ident => if let Some((ident, is_raw)) = get_macro_ident(&p.token) {
929+
sym::ident => if let Some((name, is_raw)) = get_macro_name(&p.token) {
933930
let span = p.span;
934931
p.bump();
935-
token::NtIdent(Ident::new(ident.name, span), is_raw)
932+
token::NtIdent(Ident::new(name, span), is_raw)
936933
} else {
937934
let token_str = pprust::token_to_string(&p.token);
938935
p.fatal(&format!("expected ident, found {}", &token_str)).emit();

‎src/libsyntax/ext/tt/macro_rules.rs

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1046,8 +1046,7 @@ fn is_in_follow(tok: &quoted::TokenTree, frag: &str) -> IsInFollow {
10461046
match tok {
10471047
TokenTree::Token(token) => match token.kind {
10481048
FatArrow | Comma | Eq | BinOp(token::Or) => IsInFollow::Yes,
1049-
Ident(i, false) if i.name == kw::If ||
1050-
i.name == kw::In => IsInFollow::Yes,
1049+
Ident(name, false) if name == kw::If || name == kw::In => IsInFollow::Yes,
10511050
_ => IsInFollow::No(tokens),
10521051
},
10531052
_ => IsInFollow::No(tokens),
@@ -1064,8 +1063,8 @@ fn is_in_follow(tok: &quoted::TokenTree, frag: &str) -> IsInFollow {
10641063
OpenDelim(token::DelimToken::Bracket) |
10651064
Comma | FatArrow | Colon | Eq | Gt | BinOp(token::Shr) | Semi |
10661065
BinOp(token::Or) => IsInFollow::Yes,
1067-
Ident(i, false) if i.name == kw::As ||
1068-
i.name == kw::Where => IsInFollow::Yes,
1066+
Ident(name, false) if name == kw::As ||
1067+
name == kw::Where => IsInFollow::Yes,
10691068
_ => IsInFollow::No(tokens),
10701069
},
10711070
TokenTree::MetaVarDecl(_, _, frag) if frag.name == sym::block =>
@@ -1092,9 +1091,8 @@ fn is_in_follow(tok: &quoted::TokenTree, frag: &str) -> IsInFollow {
10921091
match tok {
10931092
TokenTree::Token(token) => match token.kind {
10941093
Comma => IsInFollow::Yes,
1095-
Ident(i, is_raw) if is_raw || i.name != kw::Priv =>
1096-
IsInFollow::Yes,
1097-
ref tok => if tok.can_begin_type() {
1094+
Ident(name, is_raw) if is_raw || name != kw::Priv => IsInFollow::Yes,
1095+
_ => if token.can_begin_type() {
10981096
IsInFollow::Yes
10991097
} else {
11001098
IsInFollow::No(tokens)

‎src/libsyntax/ext/tt/quoted.rs

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -323,10 +323,9 @@ where
323323
// metavariable that names the crate of the invocation.
324324
Some(tokenstream::TokenTree::Token(token)) if token.is_ident() => {
325325
let (ident, is_raw) = token.ident().unwrap();
326-
let span = token.span.with_lo(span.lo());
326+
let span = ident.span.with_lo(span.lo());
327327
if ident.name == kw::Crate && !is_raw {
328-
let ident = ast::Ident::new(kw::DollarCrate, ident.span);
329-
TokenTree::token(span, token::Ident(ident, is_raw))
328+
TokenTree::token(span, token::Ident(kw::DollarCrate, is_raw))
330329
} else {
331330
TokenTree::MetaVar(span, ident)
332331
}

‎src/libsyntax/mut_visit.rs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -598,7 +598,6 @@ pub fn noop_visit_tts<T: MutVisitor>(TokenStream(tts): &mut TokenStream, vis: &m
598598
// apply ident visitor if it's an ident, apply other visits to interpolated nodes
599599
pub fn noop_visit_token<T: MutVisitor>(t: &mut TokenKind, vis: &mut T) {
600600
match t {
601-
token::Ident(id, _is_raw) => vis.visit_ident(id),
602601
token::Interpolated(nt) => {
603602
let mut nt = Lrc::make_mut(nt);
604603
vis.visit_interpolated(&mut nt);

‎src/libsyntax/parse/diagnostics.rs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -201,12 +201,12 @@ impl<'a> Parser<'a> {
201201
self.span,
202202
&format!("expected identifier, found {}", self.this_token_descr()),
203203
);
204-
if let token::Ident(ident, false) = &self.token.kind {
205-
if ident.is_raw_guess() {
204+
if let token::Ident(name, false) = self.token.kind {
205+
if Ident::new(name, self.span).is_raw_guess() {
206206
err.span_suggestion(
207207
self.span,
208208
"you can escape reserved keywords to use them as identifiers",
209-
format!("r#{}", ident),
209+
format!("r#{}", name),
210210
Applicability::MaybeIncorrect,
211211
);
212212
}

‎src/libsyntax/parse/lexer/mod.rs

Lines changed: 6 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
use crate::ast::{self, Ident};
1+
use crate::ast;
22
use crate::parse::ParseSess;
33
use crate::parse::token::{self, Token, TokenKind};
44
use crate::symbol::{sym, Symbol};
@@ -61,15 +61,6 @@ impl<'a> StringReader<'a> {
6161
(real, raw)
6262
}
6363

64-
fn mk_ident(&self, string: &str) -> Ident {
65-
let mut ident = Ident::from_str(string);
66-
if let Some(span) = self.override_span {
67-
ident.span = span;
68-
}
69-
70-
ident
71-
}
72-
7364
fn unwrap_or_abort(&mut self, res: Result<Token, ()>) -> Token {
7465
match res {
7566
Ok(tok) => tok,
@@ -858,17 +849,17 @@ impl<'a> StringReader<'a> {
858849

859850
return Ok(self.with_str_from(start, |string| {
860851
// FIXME: perform NFKC normalization here. (Issue #2253)
861-
let ident = self.mk_ident(string);
852+
let name = ast::Name::intern(string);
862853

863854
if is_raw_ident {
864855
let span = self.mk_sp(raw_start, self.pos);
865-
if !ident.can_be_raw() {
866-
self.err_span(span, &format!("`{}` cannot be a raw identifier", ident));
856+
if !name.can_be_raw() {
857+
self.err_span(span, &format!("`{}` cannot be a raw identifier", name));
867858
}
868859
self.sess.raw_identifier_spans.borrow_mut().push(span);
869860
}
870861

871-
token::Ident(ident, is_raw_ident)
862+
token::Ident(name, is_raw_ident)
872863
}));
873864
}
874865
}
@@ -1567,12 +1558,11 @@ mod tests {
15671558
&sh,
15681559
"/* my source file */ fn main() { println!(\"zebra\"); }\n"
15691560
.to_string());
1570-
let id = Ident::from_str("fn");
15711561
assert_eq!(string_reader.next_token(), token::Comment);
15721562
assert_eq!(string_reader.next_token(), token::Whitespace);
15731563
let tok1 = string_reader.next_token();
15741564
let tok2 = Token::new(
1575-
token::Ident(id, false),
1565+
token::Ident(Symbol::intern("fn"), false),
15761566
Span::new(BytePos(21), BytePos(23), NO_EXPANSION),
15771567
);
15781568
assert_eq!(tok1.kind, tok2.kind);

0 commit comments

Comments
 (0)