Skip to content

Commit ce545f5

Browse files
bors[bot]tromey
bors[bot]
and
tromey
authored
Merge #518
518: Token strings r=philberty a=tromey This series implements a couple of small changes to improve the handling of strings attached to Token. Co-authored-by: Tom Tromey <[email protected]>
2 parents 40b9e46 + 9bedf77 commit ce545f5

File tree

2 files changed

+37
-34
lines changed

2 files changed

+37
-34
lines changed

‎gcc/rust/lex/rust-lex.cc

+13-13
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
// Copyright (C) 2020 Free Software Foundation, Inc.
1+
// Copyright (C) 2020, 2021 Free Software Foundation, Inc.
22

33
// This file is part of GCC.
44

@@ -1436,7 +1436,7 @@ Lexer::parse_byte_string (Location loc)
14361436

14371437
str.shrink_to_fit ();
14381438

1439-
return Token::make_byte_string (loc, str);
1439+
return Token::make_byte_string (loc, std::move (str));
14401440
}
14411441

14421442
// Parses a raw byte string.
@@ -1509,7 +1509,7 @@ Lexer::parse_raw_byte_string (Location loc)
15091509

15101510
str.shrink_to_fit ();
15111511

1512-
return Token::make_byte_string (loc, str);
1512+
return Token::make_byte_string (loc, std::move (str));
15131513
}
15141514

15151515
// Parses a raw identifier.
@@ -1559,7 +1559,7 @@ Lexer::parse_raw_identifier (Location loc)
15591559
{
15601560
str.shrink_to_fit ();
15611561

1562-
return Token::make_identifier (loc, str);
1562+
return Token::make_identifier (loc, std::move (str));
15631563
}
15641564
}
15651565

@@ -1623,7 +1623,7 @@ Lexer::parse_string (Location loc)
16231623
}
16241624

16251625
str.shrink_to_fit ();
1626-
return Token::make_string (loc, str);
1626+
return Token::make_string (loc, std::move (str));
16271627
}
16281628

16291629
// Parses an identifier or keyword.
@@ -1659,7 +1659,7 @@ Lexer::parse_identifier_or_keyword (Location loc)
16591659

16601660
TokenId keyword = classify_keyword (str);
16611661
if (keyword == IDENTIFIER)
1662-
return Token::make_identifier (loc, str);
1662+
return Token::make_identifier (loc, std::move (str));
16631663
else
16641664
return Token::make (keyword, loc);
16651665
}
@@ -1736,7 +1736,7 @@ Lexer::parse_raw_string (Location loc, int initial_hash_count)
17361736

17371737
str.shrink_to_fit ();
17381738

1739-
return Token::make_string (loc, str);
1739+
return Token::make_string (loc, std::move (str));
17401740
}
17411741

17421742
template <typename IsDigitFunc>
@@ -1797,7 +1797,7 @@ Lexer::parse_non_decimal_int_literal (Location loc, IsDigitFunc is_digit_func,
17971797
: "<insert unknown base>")));
17981798
return nullptr;
17991799
}
1800-
return Token::make_int (loc, existent_str, type_hint);
1800+
return Token::make_int (loc, std::move (existent_str), type_hint);
18011801
}
18021802

18031803
// Parses a hex, binary or octal int literal.
@@ -1889,7 +1889,7 @@ Lexer::parse_decimal_int_or_float (Location loc)
18891889
current_column += length;
18901890

18911891
str.shrink_to_fit ();
1892-
return Token::make_float (loc, str, type_hint);
1892+
return Token::make_float (loc, std::move (str), type_hint);
18931893
}
18941894
else if (current_char == '.' && check_valid_float_dot_end (peek_input (1)))
18951895
{
@@ -1909,7 +1909,7 @@ Lexer::parse_decimal_int_or_float (Location loc)
19091909
current_column += length;
19101910

19111911
str.shrink_to_fit ();
1912-
return Token::make_float (loc, str, CORETYPE_UNKNOWN);
1912+
return Token::make_float (loc, std::move (str), CORETYPE_UNKNOWN);
19131913
}
19141914
else if (current_char == 'E' || current_char == 'e')
19151915
{
@@ -1938,7 +1938,7 @@ Lexer::parse_decimal_int_or_float (Location loc)
19381938
current_column += length;
19391939

19401940
str.shrink_to_fit ();
1941-
return Token::make_float (loc, str, type_hint);
1941+
return Token::make_float (loc, std::move (str), type_hint);
19421942
}
19431943
else
19441944
{
@@ -1952,7 +1952,7 @@ Lexer::parse_decimal_int_or_float (Location loc)
19521952
current_column += length;
19531953

19541954
str.shrink_to_fit ();
1955-
return Token::make_int (loc, str, type_hint);
1955+
return Token::make_int (loc, std::move (str), type_hint);
19561956
}
19571957
}
19581958

@@ -2026,7 +2026,7 @@ Lexer::parse_char_or_lifetime (Location loc)
20262026
current_column += length;
20272027

20282028
str.shrink_to_fit ();
2029-
return Token::make_lifetime (loc, str);
2029+
return Token::make_lifetime (loc, std::move (str));
20302030
}
20312031
else
20322032
{

‎gcc/rust/lex/rust-token.h

+24-21
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
// Copyright (C) 2020 Free Software Foundation, Inc.
1+
// Copyright (C) 2020, 2021 Free Software Foundation, Inc.
22

33
// This file is part of GCC.
44

@@ -249,7 +249,7 @@ class Token
249249
// Token location.
250250
Location locus;
251251
// Associated text (if any) of token.
252-
std::string *str;
252+
std::unique_ptr<std::string> str;
253253
// TODO: maybe remove issues and just store std::string as value?
254254
/* Type hint for token based on lexer data (e.g. type suffix). Does not exist
255255
* for most tokens. */
@@ -262,9 +262,9 @@ class Token
262262
{}
263263

264264
// Token constructor from token id, location, and a string.
265-
Token (TokenId token_id, Location location, const std::string &paramStr)
266-
: token_id (token_id), locus (location), str (new std::string (paramStr)),
267-
type_hint (CORETYPE_UNKNOWN)
265+
Token (TokenId token_id, Location location, std::string &&paramStr)
266+
: token_id (token_id), locus (location),
267+
str (new std::string (std::move (paramStr))), type_hint (CORETYPE_UNKNOWN)
268268
{}
269269

270270
// Token constructor from token id, location, and a char.
@@ -281,10 +281,10 @@ class Token
281281
{}
282282

283283
// Token constructor from token id, location, a string, and type hint.
284-
Token (TokenId token_id, Location location, const std::string &paramStr,
284+
Token (TokenId token_id, Location location, std::string &&paramStr,
285285
PrimitiveCoreType parType)
286-
: token_id (token_id), locus (location), str (new std::string (paramStr)),
287-
type_hint (parType)
286+
: token_id (token_id), locus (location),
287+
str (new std::string (std::move (paramStr))), type_hint (parType)
288288
{}
289289

290290
public:
@@ -298,7 +298,7 @@ class Token
298298
Token (Token &&other) = default;
299299
Token &operator= (Token &&other) = default;
300300

301-
~Token () { delete str; }
301+
~Token () = default;
302302

303303
/* TODO: make_shared (which saves a heap allocation) does not work with the
304304
* private constructor */
@@ -311,34 +311,37 @@ class Token
311311
}
312312

313313
// Makes and returns a new TokenPtr of type IDENTIFIER.
314-
static TokenPtr make_identifier (Location locus, const std::string &str)
314+
static TokenPtr make_identifier (Location locus, std::string &&str)
315315
{
316316
// return std::make_shared<Token> (IDENTIFIER, locus, str);
317-
return TokenPtr (new Token (IDENTIFIER, locus, str));
317+
return TokenPtr (new Token (IDENTIFIER, locus, std::move (str)));
318318
}
319319

320320
// Makes and returns a new TokenPtr of type INT_LITERAL.
321-
static TokenPtr make_int (Location locus, const std::string &str,
321+
static TokenPtr make_int (Location locus, std::string &&str,
322322
PrimitiveCoreType type_hint = CORETYPE_UNKNOWN)
323323
{
324324
// return std::make_shared<Token> (INT_LITERAL, locus, str, type_hint);
325-
return TokenPtr (new Token (INT_LITERAL, locus, str, type_hint));
325+
return TokenPtr (
326+
new Token (INT_LITERAL, locus, std::move (str), type_hint));
326327
}
327328

328329
// Makes and returns a new TokenPtr of type FLOAT_LITERAL.
329-
static TokenPtr make_float (Location locus, const std::string &str,
330+
static TokenPtr make_float (Location locus, std::string &&str,
330331
PrimitiveCoreType type_hint = CORETYPE_UNKNOWN)
331332
{
332333
// return std::make_shared<Token> (FLOAT_LITERAL, locus, str, type_hint);
333-
return TokenPtr (new Token (FLOAT_LITERAL, locus, str, type_hint));
334+
return TokenPtr (
335+
new Token (FLOAT_LITERAL, locus, std::move (str), type_hint));
334336
}
335337

336338
// Makes and returns a new TokenPtr of type STRING_LITERAL.
337-
static TokenPtr make_string (Location locus, const std::string &str)
339+
static TokenPtr make_string (Location locus, std::string &&str)
338340
{
339341
// return std::make_shared<Token> (STRING_LITERAL, locus, str,
340342
// CORETYPE_STR);
341-
return TokenPtr (new Token (STRING_LITERAL, locus, str, CORETYPE_STR));
343+
return TokenPtr (
344+
new Token (STRING_LITERAL, locus, std::move (str), CORETYPE_STR));
342345
}
343346

344347
// Makes and returns a new TokenPtr of type CHAR_LITERAL.
@@ -356,17 +359,17 @@ class Token
356359
}
357360

358361
// Makes and returns a new TokenPtr of type BYTE_STRING_LITERAL (fix).
359-
static TokenPtr make_byte_string (Location locus, const std::string &str)
362+
static TokenPtr make_byte_string (Location locus, std::string &&str)
360363
{
361364
// return std::make_shared<Token> (BYTE_STRING_LITERAL, locus, str);
362-
return TokenPtr (new Token (BYTE_STRING_LITERAL, locus, str));
365+
return TokenPtr (new Token (BYTE_STRING_LITERAL, locus, std::move (str)));
363366
}
364367

365368
// Makes and returns a new TokenPtr of type LIFETIME.
366-
static TokenPtr make_lifetime (Location locus, const std::string &str)
369+
static TokenPtr make_lifetime (Location locus, std::string &&str)
367370
{
368371
// return std::make_shared<Token> (LIFETIME, locus, str);
369-
return TokenPtr (new Token (LIFETIME, locus, str));
372+
return TokenPtr (new Token (LIFETIME, locus, std::move (str)));
370373
}
371374

372375
// Gets id of the token.

0 commit comments

Comments
 (0)