Remove the tokenizer’s pre-processing step.

Author: SimonSapin

src/tokenizer.rs

@@ -13,7 +13,7 @@ use ast::ComponentValue::*;
 
 /// Returns a `Iterator<(ComponentValue, SourceLocation)>`
 pub fn tokenize(input: &str) -> Tokenizer {
-    let input = preprocess(input);
+    let input = input.into_string();
     Tokenizer {
         length: input.len(),
         input: input,
@@ -30,12 +30,6 @@ impl Iterator<Node> for Tokenizer {
 //  ***********  End of public API  ***********
 
 
-#[inline]
-fn preprocess(input: &str) -> String {
-    input.into_string()
-}
-
-
 pub struct Tokenizer {
     input: String,
     length: uint,  // All counted in bytes, not characters