Tune tokenizer to real-world token frequencies

Author: RetroDev256

lib/std/zig/tokenizer.zig

@@ -10,55 +10,56 @@ pub const Token = struct {
     };
 
     pub const Keywords = std.ComptimeStringMap(Tag, .{
-        .{ "addrspace", .keyword_addrspace },
-        .{ "align", .keyword_align },
-        .{ "allowzero", .keyword_allowzero },
-        .{ "and", .keyword_and },
-        .{ "anyframe", .keyword_anyframe },
-        .{ "anytype", .keyword_anytype },
-        .{ "asm", .keyword_asm },
-        .{ "async", .keyword_async },
-        .{ "await", .keyword_await },
-        .{ "break", .keyword_break },
-        .{ "callconv", .keyword_callconv },
-        .{ "catch", .keyword_catch },
-        .{ "comptime", .keyword_comptime },
+        // sorted in descending order of frequency for faster lookup
         .{ "const", .keyword_const },
-        .{ "continue", .keyword_continue },
-        .{ "defer", .keyword_defer },
+        .{ "pub", .keyword_pub },
+        .{ "fn", .keyword_fn },
+        .{ "try", .keyword_try },
+        .{ "return", .keyword_return },
+        .{ "callconv", .keyword_callconv },
+        .{ "if", .keyword_if },
         .{ "else", .keyword_else },
+        .{ "while", .keyword_while },
+        .{ "extern", .keyword_extern },
+        .{ "var", .keyword_var },
+        .{ "volatile", .keyword_volatile },
+        .{ "switch", .keyword_switch },
+        .{ "struct", .keyword_struct },
+        .{ "inline", .keyword_inline },
+        .{ "error", .keyword_error },
+        .{ "defer", .keyword_defer },
+        .{ "comptime", .keyword_comptime },
+        .{ "test", .keyword_test },
+        .{ "break", .keyword_break },
+        .{ "for", .keyword_for },
+        .{ "usingnamespace", .keyword_usingnamespace },
+        .{ "and", .keyword_and },
+        .{ "unreachable", .keyword_unreachable },
         .{ "enum", .keyword_enum },
+        .{ "catch", .keyword_catch },
+        .{ "anytype", .keyword_anytype },
+        .{ "orelse", .keyword_orelse },
+        .{ "align", .keyword_align },
+        .{ "continue", .keyword_continue },
+        .{ "packed", .keyword_packed },
+        .{ "or", .keyword_or },
+        .{ "opaque", .keyword_opaque },
+        .{ "union", .keyword_union },
         .{ "errdefer", .keyword_errdefer },
-        .{ "error", .keyword_error },
         .{ "export", .keyword_export },
-        .{ "extern", .keyword_extern },
-        .{ "fn", .keyword_fn },
-        .{ "for", .keyword_for },
-        .{ "if", .keyword_if },
-        .{ "inline", .keyword_inline },
+        .{ "asm", .keyword_asm },
         .{ "noalias", .keyword_noalias },
         .{ "noinline", .keyword_noinline },
-        .{ "nosuspend", .keyword_nosuspend },
-        .{ "opaque", .keyword_opaque },
-        .{ "or", .keyword_or },
-        .{ "orelse", .keyword_orelse },
-        .{ "packed", .keyword_packed },
-        .{ "pub", .keyword_pub },
-        .{ "resume", .keyword_resume },
-        .{ "return", .keyword_return },
+        .{ "addrspace", .keyword_addrspace },
+        .{ "threadlocal", .keyword_threadlocal },
         .{ "linksection", .keyword_linksection },
-        .{ "struct", .keyword_struct },
+        .{ "async", .keyword_async },
         .{ "suspend", .keyword_suspend },
-        .{ "switch", .keyword_switch },
-        .{ "test", .keyword_test },
-        .{ "threadlocal", .keyword_threadlocal },
-        .{ "try", .keyword_try },
-        .{ "union", .keyword_union },
-        .{ "unreachable", .keyword_unreachable },
-        .{ "usingnamespace", .keyword_usingnamespace },
-        .{ "var", .keyword_var },
-        .{ "volatile", .keyword_volatile },
-        .{ "while", .keyword_while },
+        .{ "resume", .keyword_resume },
+        .{ "allowzero", .keyword_allowzero },
+        .{ "anyframe", .keyword_anyframe },
+        .{ "await", .keyword_await },
+        .{ "nosuspend", .keyword_nosuspend },
     });
 
     pub fn getKeyword(bytes: []const u8) ?Tag {
@@ -190,144 +191,144 @@ pub const Token = struct {
         keyword_while,
 
         pub fn lexeme(tag: Tag) ?[]const u8 {
+            // Sorted in descending order of frequency for faster lookup
             return switch (tag) {
-                .invalid,
-                .identifier,
-                .string_literal,
-                .multiline_string_literal_line,
-                .char_literal,
-                .eof,
-                .builtin,
-                .number_literal,
-                .doc_comment,
-                .container_doc_comment,
-                => null,
-
-                .invalid_periodasterisks => ".**",
-                .bang => "!",
-                .pipe => "|",
-                .pipe_pipe => "||",
-                .pipe_equal => "|=",
-                .equal => "=",
-                .equal_equal => "==",
-                .equal_angle_bracket_right => "=>",
-                .bang_equal => "!=",
+                .identifier => null,
+                .period => ".",
+                .comma => ",",
                 .l_paren => "(",
                 .r_paren => ")",
+                .equal => "=",
                 .semicolon => ";",
-                .percent => "%",
-                .percent_equal => "%=",
+                .number_literal => null,
+                .colon => ":",
+                .keyword_const => "const",
                 .l_brace => "{",
                 .r_brace => "}",
+                .builtin => null,
+                .keyword_pub => "pub",
+                .asterisk => "*",
+                .string_literal => null,
                 .l_bracket => "[",
                 .r_bracket => "]",
-                .period => ".",
-                .period_asterisk => ".*",
+                .question_mark => "?",
+                .keyword_fn => "fn",
+                .doc_comment => null,
+                .keyword_try => "try",
+                .equal_angle_bracket_right => "=>",
+                .keyword_return => "return",
+                .ampersand => "&",
+                .keyword_callconv => "callconv",
+                .pipe => "|",
+                .keyword_if => "if",
+                .keyword_else => "else",
+                .keyword_while => "while",
+                .keyword_extern => "extern",
+                .keyword_var => "var",
+                .keyword_volatile => "volatile",
+                .equal_equal => "==",
+                .keyword_switch => "switch",
+                .keyword_struct => "struct",
+                .multiline_string_literal_line => null,
+                .keyword_inline => "inline",
+                .bang => "!",
+                .minus => "-",
+                .keyword_error => "error",
+                .char_literal => null,
+                .plus => "+",
                 .ellipsis2 => "..",
+                .keyword_defer => "defer",
+                .keyword_comptime => "comptime",
+                .bang_equal => "!=",
+                .keyword_test => "test",
+                .period_asterisk => ".*",
+                .keyword_break => "break",
+                .keyword_for => "for",
+                .plus_equal => "+=",
+                .keyword_usingnamespace => "usingnamespace",
+                .keyword_and => "and",
+                .keyword_unreachable => "unreachable",
+                .keyword_enum => "enum",
+                .keyword_catch => "catch",
+                .keyword_anytype => "anytype",
+                .keyword_orelse => "orelse",
+                .plus_plus => "++",
+                .eof => null,
+                .angle_bracket_left => "<",
+                .keyword_align => "align",
+                .angle_bracket_right => ">",
+                .keyword_continue => "continue",
+                .keyword_packed => "packed",
+                .keyword_or => "or",
+                .angle_bracket_angle_bracket_left => "<<",
+                .container_doc_comment => null,
+                .keyword_opaque => "opaque",
+                .angle_bracket_angle_bracket_right => ">>",
+                .slash => "/",
                 .ellipsis3 => "...",
+                .keyword_union => "union",
+                .angle_bracket_right_equal => ">=",
+                .angle_bracket_left_equal => "<=",
+                .keyword_errdefer => "errdefer",
+                .keyword_export => "export",
+                .asterisk_asterisk => "**",
+                .minus_equal => "-=",
+                .keyword_asm => "asm",
+                .pipe_pipe => "||",
+                .percent => "%",
+                .tilde => "~",
                 .caret => "^",
-                .caret_equal => "^=",
-                .plus => "+",
-                .plus_plus => "++",
-                .plus_equal => "+=",
+                .pipe_equal => "|=",
                 .plus_percent => "+%",
-                .plus_percent_equal => "+%=",
-                .plus_pipe => "+|",
-                .plus_pipe_equal => "+|=",
-                .minus => "-",
-                .minus_equal => "-=",
+                .keyword_noalias => "noalias",
                 .minus_percent => "-%",
-                .minus_percent_equal => "-%=",
-                .minus_pipe => "-|",
-                .minus_pipe_equal => "-|=",
-                .asterisk => "*",
-                .asterisk_equal => "*=",
-                .asterisk_asterisk => "**",
+                .caret_equal => "^=",
+                .plus_percent_equal => "+%=",
                 .asterisk_percent => "*%",
-                .asterisk_percent_equal => "*%=",
-                .asterisk_pipe => "*|",
-                .asterisk_pipe_equal => "*|=",
                 .arrow => "->",
-                .colon => ":",
-                .slash => "/",
-                .slash_equal => "/=",
-                .comma => ",",
-                .ampersand => "&",
                 .ampersand_equal => "&=",
-                .question_mark => "?",
-                .angle_bracket_left => "<",
-                .angle_bracket_left_equal => "<=",
-                .angle_bracket_angle_bracket_left => "<<",
-                .angle_bracket_angle_bracket_left_equal => "<<=",
-                .angle_bracket_angle_bracket_left_pipe => "<<|",
-                .angle_bracket_angle_bracket_left_pipe_equal => "<<|=",
-                .angle_bracket_right => ">",
-                .angle_bracket_right_equal => ">=",
-                .angle_bracket_angle_bracket_right => ">>",
                 .angle_bracket_angle_bracket_right_equal => ">>=",
-                .tilde => "~",
-                .keyword_addrspace => "addrspace",
-                .keyword_align => "align",
+                .keyword_async => "async",
+                .asterisk_equal => "*=",
+                .minus_pipe => "-|",
+                .asterisk_percent_equal => "*%=",
+                .keyword_suspend => "suspend",
+                .keyword_resume => "resume",
                 .keyword_allowzero => "allowzero",
-                .keyword_and => "and",
+                .angle_bracket_angle_bracket_left_equal => "<<=",
+                .slash_equal => "/=",
                 .keyword_anyframe => "anyframe",
-                .keyword_anytype => "anytype",
-                .keyword_asm => "asm",
-                .keyword_async => "async",
                 .keyword_await => "await",
-                .keyword_break => "break",
-                .keyword_callconv => "callconv",
-                .keyword_catch => "catch",
-                .keyword_comptime => "comptime",
-                .keyword_const => "const",
-                .keyword_continue => "continue",
-                .keyword_defer => "defer",
-                .keyword_else => "else",
-                .keyword_enum => "enum",
-                .keyword_errdefer => "errdefer",
-                .keyword_error => "error",
-                .keyword_export => "export",
-                .keyword_extern => "extern",
-                .keyword_fn => "fn",
-                .keyword_for => "for",
-                .keyword_if => "if",
-                .keyword_inline => "inline",
-                .keyword_noalias => "noalias",
-                .keyword_noinline => "noinline",
                 .keyword_nosuspend => "nosuspend",
-                .keyword_opaque => "opaque",
-                .keyword_or => "or",
-                .keyword_orelse => "orelse",
-                .keyword_packed => "packed",
-                .keyword_pub => "pub",
-                .keyword_resume => "resume",
-                .keyword_return => "return",
-                .keyword_linksection => "linksection",
-                .keyword_struct => "struct",
-                .keyword_suspend => "suspend",
-                .keyword_switch => "switch",
-                .keyword_test => "test",
+                .minus_percent_equal => "-%=",
+                .plus_pipe => "+|",
+                .keyword_noinline => "noinline",
+                .keyword_addrspace => "addrspace",
                 .keyword_threadlocal => "threadlocal",
-                .keyword_try => "try",
-                .keyword_union => "union",
-                .keyword_unreachable => "unreachable",
-                .keyword_usingnamespace => "usingnamespace",
-                .keyword_var => "var",
-                .keyword_volatile => "volatile",
-                .keyword_while => "while",
+                .plus_pipe_equal => "+|=",
+                .minus_pipe_equal => "-|=",
+                .asterisk_pipe => "*|",
+                .percent_equal => "%=",
+                .angle_bracket_angle_bracket_left_pipe => "<<|",
+                .invalid => null,
+                .keyword_linksection => "linksection",
+                .asterisk_pipe_equal => "*|=",
+                .angle_bracket_angle_bracket_left_pipe_equal => "<<|=",
+                .invalid_periodasterisks => ".**",
             };
         }
 
         pub fn symbol(tag: Tag) []const u8 {
+            // Sorted in descending order of frequency for faster lookup
             return tag.lexeme() orelse switch (tag) {
-                .invalid => "invalid token",
                 .identifier => "an identifier",
+                .number_literal => "a number literal",
+                .builtin => "a builtin function",
                 .string_literal, .multiline_string_literal_line => "a string literal",
+                .doc_comment, .container_doc_comment => "a document comment",
                 .char_literal => "a character literal",
                 .eof => "EOF",
-                .builtin => "a builtin function",
-                .number_literal => "a number literal",
-                .doc_comment, .container_doc_comment => "a document comment",
+                .invalid => "invalid token",
                 else => unreachable,
             };
         }