Add line and column number to TokenizerError (#194)

Addresses https://github.com/andygrove/sqlparser-rs/issues/179 for tokenize errors

Author: Dandandan

src/parser.rs

@@ -49,7 +49,10 @@ use IsLateral::*;
 
 impl From<TokenizerError> for ParserError {
     fn from(e: TokenizerError) -> Self {
-        ParserError::TokenizerError(format!("{:?}", e))
+        ParserError::TokenizerError(format!(
+            "{} at Line: {}, Column {}",
+            e.message, e.line, e.col
+        ))
     }
 }
 

src/tokenizer.rs

@@ -217,7 +217,11 @@ impl fmt::Display for Whitespace {
 
 /// Tokenizer error
 #[derive(Debug, PartialEq)]
-pub struct TokenizerError(String);
+pub struct TokenizerError {
+    pub message: String,
+    pub line: u64,
+    pub col: u64,
+}
 
 /// SQL Tokenizer
 pub struct Tokenizer<'a> {
@@ -331,10 +335,10 @@ impl<'a> Tokenizer<'a> {
                     if chars.next() == Some(quote_end) {
                         Ok(Some(Token::make_word(&s, Some(quote_start))))
                     } else {
-                        Err(TokenizerError(format!(
-                            "Expected close delimiter '{}' before EOF.",
-                            quote_end
-                        )))
+                        self.tokenizer_error(
+                            format!("Expected close delimiter '{}' before EOF.", quote_end)
+                                .as_str(),
+                        )
                     }
                 }
                 // numbers
@@ -395,10 +399,7 @@ impl<'a> Tokenizer<'a> {
                     chars.next(); // consume
                     match chars.peek() {
                         Some('=') => self.consume_and_return(chars, Token::Neq),
-                        _ => Err(TokenizerError(format!(
-                            "Tokenizer Error at Line: {}, Col: {}",
-                            self.line, self.col
-                        ))),
+                        _ => self.tokenizer_error("Expected to see '=' after '!' character"),
                     }
                 }
                 '<' => {
@@ -437,6 +438,14 @@ impl<'a> Tokenizer<'a> {
         }
     }
 
+    fn tokenizer_error<R>(&self, message: &str) -> Result<R, TokenizerError> {
+        Err(TokenizerError {
+            message: message.to_string(),
+            col: self.col,
+            line: self.line,
+        })
+    }
+
     /// Tokenize an identifier or keyword, after the first char is already consumed.
     fn tokenize_word(&self, first_char: char, chars: &mut Peekable<Chars<'_>>) -> String {
         let mut s = first_char.to_string();
@@ -471,10 +480,7 @@ impl<'a> Tokenizer<'a> {
                 }
             }
         }
-        Err(TokenizerError(format!(
-            "Unterminated string literal at Line: {}, Col: {}",
-            self.line, self.col
-        )))
+        self.tokenizer_error("Unterminated string literal")
     }
 
     fn tokenize_multiline_comment(
@@ -499,11 +505,7 @@ impl<'a> Tokenizer<'a> {
                         s.push(ch);
                     }
                 }
-                None => {
-                    break Err(TokenizerError(
-                        "Unexpected EOF while in a multi-line comment".to_string(),
-                    ));
-                }
+                None => break self.tokenizer_error("Unexpected EOF while in a multi-line comment"),
             }
         }
     }
@@ -720,9 +722,11 @@ mod tests {
         let mut tokenizer = Tokenizer::new(&dialect, &sql);
         assert_eq!(
             tokenizer.tokenize(),
-            Err(TokenizerError(
-                "Unterminated string literal at Line: 1, Col: 8".to_string()
-            ))
+            Err(TokenizerError {
+                message: "Unterminated string literal".to_string(),
+                line: 1,
+                col: 8
+            })
         );
     }
 
@@ -843,9 +847,11 @@ mod tests {
         let mut tokenizer = Tokenizer::new(&dialect, &sql);
         assert_eq!(
             tokenizer.tokenize(),
-            Err(TokenizerError(
-                "Expected close delimiter '\"' before EOF.".to_string(),
-            ))
+            Err(TokenizerError {
+                message: "Expected close delimiter '\"' before EOF.".to_string(),
+                line: 1,
+                col: 1
+            })
         );
     }