Track the extent of tokens UIDENT and USCONST

msullivan · Apr 26, 2024 · 0779c5e · 0779c5e
1 parent 1ec3894
commit 0779c5e
Show file tree

Hide file tree

Showing 3 changed files with 8 additions and 4 deletions.
diff --git a/examples/scan.c b/examples/scan.c
@@ -18,7 +18,7 @@ const char* tests[] = {
   "SELECT 4790999999999999999999999999999999999999999999999999999999999999999999999999999999999999 * 9999999999999999999999999999999999999999999999999999999999999999999999999999999999999999",
   "WITH upsert AS (INSERT INTO schema_columns (table_id, name, position, data_type, default_value, not_null, first_snapshot_id) VALUES (?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?, ?) ON CONFLICT (table_id, name) WHERE invalidated_at_snapshot_id IS NULL DO UPDATE SET (position, data_type, default_value, not_null) = (EXCLUDED.position, EXCLUDED.data_type, EXCLUDED.default_value, EXCLUDED.not_null) RETURNING id, table_id, name), all_ids AS (SELECT id FROM upsert), cleanup AS (UPDATE schema_columns SET invalidated_at_snapshot_id = ? WHERE table_id = ? AND invalidated_at_snapshot_id IS NULL AND id NOT IN (SELECT id FROM all_ids)) SELECT * FROM upsert",
   "SELECT $foo$test$foo$, '123' FROM \"xy\", abc",
-  "SELECT '/*not a comment*/', E'\\134', B'10', X'12', -Infinity, U&\"d!0061t!+000061\" UESCAPE '!'"
+  "SELECT '/*not a comment*/', E'\\134', B'10', X'12', -Infinity, U&\"d!0061t!+000061\" UESCAPE '!', U&'d!0061t!+000061' UESCAPE '!'"
 };
 
 int main() {

diff --git a/src/pg_query_scan.c b/src/pg_query_scan.c
@@ -93,7 +93,7 @@ PgQueryScanResult pg_query_scan(const char* input)
       output_tokens[i] = malloc(sizeof(PgQuery__ScanToken));
       pg_query__scan_token__init(output_tokens[i]);
       output_tokens[i]->start = yylloc;
-      if (tok == SCONST || tok == BCONST || tok == XCONST || tok == IDENT || tok == C_COMMENT) {
+      if (tok == SCONST || tok == USCONST || tok == BCONST || tok == XCONST || tok == IDENT || tok == UIDENT || tok == C_COMMENT) {
         output_tokens[i]->end = yyextra.yyllocend;
       } else {
         output_tokens[i]->end = yylloc + ((struct yyguts_t*) yyscanner)->yyleng_r;

diff --git a/test/scan_tests.c b/test/scan_tests.c
@@ -46,7 +46,7 @@ const char* const tests[] = {
     "\"xy\" = IDENT, NO_KEYWORD\n"
     ", = ASCII_44, NO_KEYWORD\n"
     "abc = IDENT, NO_KEYWORD\n",
-  "SELECT '/*not a comment*/', E'\\134', B'10', X'12', -Infinity, U&\"d!0061t!+000061\" UESCAPE '!'",
+  "SELECT '/*not a comment*/', E'\\134', B'10', X'12', -Infinity, U&\"d!0061t!+000061\" UESCAPE '!', U&'d!0061t!+000061' UESCAPE '!'",
     "SELECT = SELECT, RESERVED_KEYWORD\n"
     "'/*not a comment*/' = SCONST, NO_KEYWORD\n"
     ", = ASCII_44, NO_KEYWORD\n"
@@ -59,7 +59,11 @@ const char* const tests[] = {
     "- = ASCII_45, NO_KEYWORD\n"
     "Infinity = IDENT, NO_KEYWORD\n"
     ", = ASCII_44, NO_KEYWORD\n"
-    "U = UIDENT, NO_KEYWORD\n"
+    "U&\"d!0061t!+000061\" = UIDENT, NO_KEYWORD\n"
+    "UESCAPE = UESCAPE, UNRESERVED_KEYWORD\n"
+    "'!' = SCONST, NO_KEYWORD\n"
+    ", = ASCII_44, NO_KEYWORD\n"
+    "U&'d!0061t!+000061' = USCONST, NO_KEYWORD\n"
     "UESCAPE = UESCAPE, UNRESERVED_KEYWORD\n"
     "'!' = SCONST, NO_KEYWORD\n",
   "SELECT 'a'/*;*/'b'; SELECT ';';",