@@ -5,7 +5,8 @@ use std::iter::Peekable;
5
5
use std:: str:: CharIndices ;
6
6
7
7
pub type Tok < ' input > = Token < & ' input str > ;
8
- pub type Spanned < ' input , Loc > = Result < ( Loc , Tok < ' input > , Loc ) , Error > ;
8
+ pub type SpannedResult < ' input , Loc > = Result < Spanned < ' input , Loc > , Error > ;
9
+ pub type Spanned < ' input , Loc > = ( Loc , Tok < ' input > , Loc ) ;
9
10
10
11
#[ derive( thiserror:: Error , Clone , Debug , PartialEq ) ]
11
12
pub enum Error {
@@ -447,7 +448,7 @@ impl StringLiteral<&str> {
447
448
// -----------------------------------------------------------------------------
448
449
449
450
impl < ' input > Iterator for Lexer < ' input > {
450
- type Item = Spanned < ' input , usize > ;
451
+ type Item = SpannedResult < ' input , usize > ;
451
452
452
453
fn next ( & mut self ) -> Option < Self :: Item > {
453
454
use Token :: * ;
@@ -461,7 +462,7 @@ impl<'input> Iterator for Lexer<'input> {
461
462
// represent a physical character, instead it is a boundary marker.
462
463
if self . query_start ( start) {
463
464
// dbg!("LQuery"); // NOTE: uncomment this for debugging
464
- return self . token2 ( start, start + 1 , LQuery ) ;
465
+ return Some ( Ok ( self . token2 ( start, start + 1 , LQuery ) ) ) ;
465
466
}
466
467
467
468
// Check if we need to emit a `RQuery` token.
@@ -470,7 +471,7 @@ impl<'input> Iterator for Lexer<'input> {
470
471
// represent a physical character, instead it is a boundary marker.
471
472
if let Some ( pos) = self . query_end ( start) {
472
473
// dbg!("RQuery"); // NOTE: uncomment this for debugging
473
- return self . token2 ( pos, pos + 1 , RQuery ) ;
474
+ return Some ( Ok ( self . token2 ( pos, pos + 1 , RQuery ) ) ) ;
474
475
}
475
476
476
477
// Advance the internal iterator and emit the next token, or loop
@@ -479,26 +480,28 @@ impl<'input> Iterator for Lexer<'input> {
479
480
let result = match ch {
480
481
'"' => Some ( self . string_literal ( start) ) ,
481
482
482
- ';' => self . token ( start, SemiColon ) ,
483
- '\n' => self . token ( start, Newline ) ,
484
- '\\' => self . token ( start, Escape ) ,
483
+ ';' => Some ( Ok ( self . token ( start, SemiColon ) ) ) ,
484
+ '\n' => Some ( Ok ( self . token ( start, Newline ) ) ) ,
485
+ '\\' => Some ( Ok ( self . token ( start, Escape ) ) ) ,
485
486
486
- '(' => self . open ( start, LParen ) ,
487
- '[' => self . open ( start, LBracket ) ,
488
- '{' => self . open ( start, LBrace ) ,
489
- '}' => self . close ( start, RBrace ) ,
490
- ']' => self . close ( start, RBracket ) ,
491
- ')' => self . close ( start, RParen ) ,
487
+ '(' => Some ( Ok ( self . open ( start, LParen ) ) ) ,
488
+ '[' => Some ( Ok ( self . open ( start, LBracket ) ) ) ,
489
+ '{' => Some ( Ok ( self . open ( start, LBrace ) ) ) ,
490
+ '}' => Some ( Ok ( self . close ( start, RBrace ) ) ) ,
491
+ ']' => Some ( Ok ( self . close ( start, RBracket ) ) ) ,
492
+ ')' => Some ( Ok ( self . close ( start, RParen ) ) ) ,
492
493
493
- '.' => self . token ( start, Dot ) ,
494
- ':' => self . token ( start, Colon ) ,
495
- ',' => self . token ( start, Comma ) ,
494
+ '.' => Some ( Ok ( self . token ( start, Dot ) ) ) ,
495
+ ':' => Some ( Ok ( self . token ( start, Colon ) ) ) ,
496
+ ',' => Some ( Ok ( self . token ( start, Comma ) ) ) ,
496
497
497
- '_' if self . test_peek ( char:: is_alphabetic) => Some ( self . internal_test ( start) ) ,
498
- '_' => self . token ( start, Underscore ) ,
498
+ '_' if self . test_peek ( char:: is_alphabetic) => {
499
+ Some ( Ok ( self . internal_test ( start) ) )
500
+ }
501
+ '_' => Some ( Ok ( self . token ( start, Underscore ) ) ) ,
499
502
500
503
'!' if self . test_peek ( |ch| ch == '!' || !is_operator ( ch) ) => {
501
- self . token ( start, Bang )
504
+ Some ( Ok ( self . token ( start, Bang ) ) )
502
505
}
503
506
504
507
'#' => {
@@ -510,14 +513,14 @@ impl<'input> Iterator for Lexer<'input> {
510
513
's' if self . test_peek ( |ch| ch == '\'' ) => Some ( self . raw_string_literal ( start) ) ,
511
514
't' if self . test_peek ( |ch| ch == '\'' ) => Some ( self . timestamp_literal ( start) ) ,
512
515
513
- ch if is_ident_start ( ch) => Some ( self . identifier_or_function_call ( start) ) ,
516
+ ch if is_ident_start ( ch) => Some ( Ok ( self . identifier_or_function_call ( start) ) ) ,
514
517
ch if is_digit ( ch) || ( ch == '-' && self . test_peek ( is_digit) ) => {
515
518
Some ( self . numeric_literal ( start) )
516
519
}
517
- ch if is_operator ( ch) => Some ( self . operator ( start) ) ,
520
+ ch if is_operator ( ch) => Some ( Ok ( self . operator ( start) ) ) ,
518
521
ch if ch. is_whitespace ( ) => continue ,
519
522
520
- ch => self . token ( start, InvalidToken ( ch) ) ,
523
+ ch => Some ( Ok ( self . token ( start, InvalidToken ( ch) ) ) ) ,
521
524
} ;
522
525
523
526
// dbg!(&result); // NOTE: uncomment this for debugging
@@ -529,7 +532,7 @@ impl<'input> Iterator for Lexer<'input> {
529
532
// queries.
530
533
} else if let Some ( end) = self . rquery_indices . pop ( ) {
531
534
// dbg!("RQuery"); // NOTE: uncomment this for debugging
532
- return self . token2 ( end, end + 1 , RQuery ) ;
535
+ return Some ( Ok ( self . token2 ( end, end + 1 , RQuery ) ) ) ;
533
536
}
534
537
535
538
return None ;
@@ -542,7 +545,7 @@ impl<'input> Iterator for Lexer<'input> {
542
545
// -----------------------------------------------------------------------------
543
546
544
547
impl < ' input > Lexer < ' input > {
545
- fn open ( & mut self , start : usize , token : Token < & ' input str > ) -> Option < Spanned < ' input , usize > > {
548
+ fn open ( & mut self , start : usize , token : Token < & ' input str > ) -> Spanned < ' input , usize > {
546
549
match & token {
547
550
Token :: LParen => self . open_parens += 1 ,
548
551
Token :: LBracket => self . open_brackets += 1 ,
@@ -553,7 +556,7 @@ impl<'input> Lexer<'input> {
553
556
self . token ( start, token)
554
557
}
555
558
556
- fn close ( & mut self , start : usize , token : Token < & ' input str > ) -> Option < Spanned < ' input , usize > > {
559
+ fn close ( & mut self , start : usize , token : Token < & ' input str > ) -> Spanned < ' input , usize > {
557
560
match & token {
558
561
Token :: RParen => self . open_parens = self . open_parens . saturating_sub ( 1 ) ,
559
562
Token :: RBracket => self . open_brackets = self . open_brackets . saturating_sub ( 1 ) ,
@@ -564,7 +567,7 @@ impl<'input> Lexer<'input> {
564
567
self . token ( start, token)
565
568
}
566
569
567
- fn token ( & mut self , start : usize , token : Token < & ' input str > ) -> Option < Spanned < ' input , usize > > {
570
+ fn token ( & mut self , start : usize , token : Token < & ' input str > ) -> Spanned < ' input , usize > {
568
571
let end = self . next_index ( ) ;
569
572
self . token2 ( start, end, token)
570
573
}
@@ -574,8 +577,8 @@ impl<'input> Lexer<'input> {
574
577
start : usize ,
575
578
end : usize ,
576
579
token : Token < & ' input str > ,
577
- ) -> Option < Spanned < ' input , usize > > {
578
- Some ( Ok ( ( start, token, end) ) )
580
+ ) -> Spanned < ' input , usize > {
581
+ ( start, token, end)
579
582
}
580
583
581
584
fn query_end ( & mut self , start : usize ) -> Option < usize > {
@@ -640,7 +643,7 @@ impl<'input> Lexer<'input> {
640
643
let mut end = 0 ;
641
644
while let Some ( ( pos, ch) ) = chars. next ( ) {
642
645
let take_until_end =
643
- |result : Spanned < ' input , usize > ,
646
+ |result : SpannedResult < ' input , usize > ,
644
647
last_char : & mut Option < char > ,
645
648
end : & mut usize ,
646
649
chars : & mut Peekable < CharIndices < ' input > > | {
@@ -735,7 +738,7 @@ impl<'input> Lexer<'input> {
735
738
while let Some ( ( pos, ch) ) = chars. peek ( ) {
736
739
let pos = * pos;
737
740
738
- let literal_check = |result : Spanned < ' input , usize > , chars : & mut Peekable < CharIndices < ' input > > | match result {
741
+ let literal_check = |result : SpannedResult < ' input , usize > , chars : & mut Peekable < CharIndices < ' input > > | match result {
739
742
Err ( _) => Err ( ( ) ) ,
740
743
Ok ( ( _, _, new) ) => {
741
744
#[ allow( clippy:: while_let_on_iterator) ]
@@ -854,7 +857,7 @@ impl<'input> Lexer<'input> {
854
857
true
855
858
}
856
859
857
- fn string_literal ( & mut self , start : usize ) -> Spanned < ' input , usize > {
860
+ fn string_literal ( & mut self , start : usize ) -> SpannedResult < ' input , usize > {
858
861
let content_start = self . next_index ( ) ;
859
862
860
863
loop {
@@ -876,19 +879,19 @@ impl<'input> Lexer<'input> {
876
879
Err ( Error :: StringLiteral { start } )
877
880
}
878
881
879
- fn regex_literal ( & mut self , start : usize ) -> Spanned < ' input , usize > {
882
+ fn regex_literal ( & mut self , start : usize ) -> SpannedResult < ' input , usize > {
880
883
self . quoted_literal ( start, Token :: RegexLiteral )
881
884
}
882
885
883
- fn raw_string_literal ( & mut self , start : usize ) -> Spanned < ' input , usize > {
886
+ fn raw_string_literal ( & mut self , start : usize ) -> SpannedResult < ' input , usize > {
884
887
self . quoted_literal ( start, |c| Token :: StringLiteral ( StringLiteral :: Raw ( c) ) )
885
888
}
886
889
887
- fn timestamp_literal ( & mut self , start : usize ) -> Spanned < ' input , usize > {
890
+ fn timestamp_literal ( & mut self , start : usize ) -> SpannedResult < ' input , usize > {
888
891
self . quoted_literal ( start, Token :: TimestampLiteral )
889
892
}
890
893
891
- fn numeric_literal ( & mut self , start : usize ) -> Spanned < ' input , usize > {
894
+ fn numeric_literal ( & mut self , start : usize ) -> SpannedResult < ' input , usize > {
892
895
let ( end, int) = self . take_while ( start, |ch| is_digit ( ch) || ch == '_' ) ;
893
896
894
897
match self . peek ( ) {
@@ -928,7 +931,7 @@ impl<'input> Lexer<'input> {
928
931
Token :: ident ( ident)
929
932
} ;
930
933
931
- Ok ( ( start, token, end) )
934
+ ( start, token, end)
932
935
}
933
936
934
937
fn operator ( & mut self , start : usize ) -> Spanned < ' input , usize > {
@@ -941,21 +944,21 @@ impl<'input> Lexer<'input> {
941
944
op => Token :: Operator ( op) ,
942
945
} ;
943
946
944
- Ok ( ( start, token, end) )
947
+ ( start, token, end)
945
948
}
946
949
947
950
fn internal_test ( & mut self , start : usize ) -> Spanned < ' input , usize > {
948
951
self . bump ( ) ;
949
952
let ( end, test) = self . take_while ( start, char:: is_alphabetic) ;
950
953
951
- Ok ( ( start, Token :: InternalTest ( test) , end) )
954
+ ( start, Token :: InternalTest ( test) , end)
952
955
}
953
956
954
957
fn quoted_literal (
955
958
& mut self ,
956
959
start : usize ,
957
960
tok : impl Fn ( & ' input str ) -> Tok < ' input > ,
958
- ) -> Spanned < ' input , usize > {
961
+ ) -> SpannedResult < ' input , usize > {
959
962
self . bump ( ) ;
960
963
let content_start = self . next_index ( ) ;
961
964
@@ -1122,7 +1125,7 @@ mod test {
1122
1125
use super :: * ;
1123
1126
use crate :: lex:: Token :: * ;
1124
1127
1125
- fn lexer ( input : & str ) -> impl Iterator < Item = Spanned < ' _ , usize > > + ' _ {
1128
+ fn lexer ( input : & str ) -> impl Iterator < Item = SpannedResult < ' _ , usize > > + ' _ {
1126
1129
let mut lexer = Lexer :: new ( input) ;
1127
1130
Box :: new ( std:: iter:: from_fn ( move || Some ( lexer. next ( ) ?) ) )
1128
1131
}
0 commit comments