Make the tokenizer borrow rather than own its input.

Author: SimonSapin

src/from_bytes.rs

@@ -8,9 +8,6 @@ use encoding::label::encoding_from_whatwg_label;
 use encoding::all::UTF_8;
 use encoding::{EncodingRef, DecoderTrap, decode};
 
-use tokenizer::{tokenize, Tokenizer};
-use parser::{parse_stylesheet_rules, StylesheetParser};
-
 
 /// Determine the character encoding of a CSS stylesheet and decode it.
 ///
@@ -68,23 +65,3 @@ fn decode_replace(input: &[u8], fallback_encoding: EncodingRef)-> (String, Encod
     let (result, used_encoding) = decode(input, DecoderTrap::Replace, fallback_encoding);
     (result.unwrap(), used_encoding)
 }
-
-
-/// Parse stylesheet from bytes.
-///
-/// * `css_bytes`: A byte string.
-/// * `protocol_encoding`: The encoding label, if any, defined by HTTP or equivalent protocol.
-///     (e.g. via the `charset` parameter of the `Content-Type` header.)
-/// * `environment_encoding`: An optional `Encoding` object for the [environment encoding]
-///     (http://www.w3.org/TR/css-syntax/#environment-encoding), if any.
-///
-/// Returns a 2-tuple of a `Iterator<Result<Rule, SyntaxError>>`
-/// and the `Encoding` object that was used.
-pub fn parse_stylesheet_rules_from_bytes(
-        css_bytes: &[u8], protocol_encoding_label: Option<&str>,
-        environment_encoding: Option<EncodingRef>)
-     -> (StylesheetParser<Tokenizer>, EncodingRef) {
-    let (css_unicode, encoding) = decode_stylesheet_bytes(
-        css_bytes, protocol_encoding_label, environment_encoding);
-    (parse_stylesheet_rules(tokenize(css_unicode.as_slice())), encoding)
-}

src/lib.rs

@@ -22,7 +22,7 @@ pub use parser::{parse_stylesheet_rules, StylesheetParser,
                  parse_rule_list, RuleListParser,
                  parse_declaration_list, DeclarationListParser,
                  parse_one_rule, parse_one_declaration, parse_one_component_value};
-pub use from_bytes::{decode_stylesheet_bytes, parse_stylesheet_rules_from_bytes};
+pub use from_bytes::decode_stylesheet_bytes;
 pub use color::{RGBA, Color};
 pub use nth::parse_nth;
 pub use serializer::{ToCss, CssStringWriter, serialize_identifier, serialize_string};

src/tests.rs

@@ -185,8 +185,9 @@ fn stylesheet_from_bytes() {
             let environment_encoding = get_string(&map, &"environment_encoding".to_string())
                 .and_then(encoding_from_whatwg_label);
 
-            let (mut rules, used_encoding) = parse_stylesheet_rules_from_bytes(
+            let (css_unicode, used_encoding) = decode_stylesheet_bytes(
                 css.as_slice(), protocol_encoding_label, environment_encoding);
+            let mut rules = parse_stylesheet_rules(tokenize(css_unicode.as_slice()));
 
             (rules.collect::<Vec<Result<Rule, SyntaxError>>>(), used_encoding.name().to_string()).to_json()
         };

src/tokenizer.rs

@@ -13,15 +13,14 @@ use ast::ComponentValue::*;
 
 /// Returns a `Iterator<(ComponentValue, SourceLocation)>`
 pub fn tokenize(input: &str) -> Tokenizer {
-    let input = input.into_string();
     Tokenizer {
         length: input.len(),
         input: input,
         position: 0,
     }
 }
 
-impl Iterator<Node> for Tokenizer {
+impl<'a> Iterator<Node> for Tokenizer<'a> {
     #[inline]
     fn next(&mut self) -> Option<Node> { next_component_value(self) }
 }
@@ -30,8 +29,8 @@ impl Iterator<Node> for Tokenizer {
 //  ***********  End of public API  ***********
 
 
-pub struct Tokenizer {
-    input: String,
+pub struct Tokenizer<'a> {
+    input: &'a str,
     length: uint,  // All counted in bytes, not characters
     position: uint,  // All counted in bytes, not characters
 }
@@ -43,7 +42,7 @@ macro_rules! is_match(
 )
 
 
-impl Tokenizer {
+impl<'a> Tokenizer<'a> {
     #[inline]
     fn is_eof(&self) -> bool { self.position >= self.length }