Rewrite ALL THE THINGS!

The tokenizer is now fully incremental, rather than yielding a block/function at a time.

The convention is now to take `&mut Parser` as input for parsing, and only consume as necessary.

Author: SimonSapin

src/ast.rs

@@ -3,8 +3,6 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 use std::fmt;
-use std::slice;
-use std::vec;
 
 
 #[deriving(PartialEq, Show)]
@@ -22,18 +20,18 @@ pub struct SourceLocation {
 }
 
 
-pub type Node = (ComponentValue, SourceLocation);  // TODO this is not a good name
+pub type Node = (Token, SourceLocation);  // TODO this is not a good name
 
 
 #[deriving(PartialEq, Show)]
-pub enum ComponentValue {
+pub enum Token {
     // Preserved tokens.
     Ident(String),
     AtKeyword(String),
     Hash(String),
     IDHash(String),  // Hash that is a valid ID selector.
     QuotedString(String),
-    URL(String),
+    Url(String),
     Delim(char),
     Number(NumericValue),
     Percentage(NumericValue),
@@ -53,15 +51,15 @@ pub enum ComponentValue {
     CDC,  // -->
 
     // Function
-    Function(String, Vec<ComponentValue>),  // name, arguments
+    Function(String),  // name
 
     // Simple block
-    ParenthesisBlock(Vec<ComponentValue>),  // (…)
-    SquareBracketBlock(Vec<ComponentValue>),  // […]
-    CurlyBracketBlock(Vec<Node>),  // {…}
+    ParenthesisBlock,  // (…)
+    SquareBracketBlock,  // […]
+    CurlyBracketBlock,  // {…}
 
     // These are always invalid
-    BadURL,
+    BadUrl,
     BadString,
     CloseParenthesis, // )
     CloseSquareBracket, // ]
@@ -73,22 +71,22 @@ pub enum ComponentValue {
 pub struct Declaration {
     pub location: SourceLocation,
     pub name: String,
-    pub value: Vec<ComponentValue>,
+    pub value: Vec<Token>,
     pub important: bool,
 }
 
 #[deriving(PartialEq)]
 pub struct QualifiedRule {
     pub location: SourceLocation,
-    pub prelude: Vec<ComponentValue>,
+    pub prelude: Vec<Token>,
     pub block: Vec<Node>,
 }
 
 #[deriving(PartialEq)]
 pub struct AtRule {
     pub location: SourceLocation,
     pub name: String,
-    pub prelude: Vec<ComponentValue>,
+    pub prelude: Vec<Token>,
     pub block: Option<Vec<Node>>,
 }
 
@@ -125,52 +123,3 @@ impl fmt::Show for SyntaxError {
         write!(f, "{}:{} {}", self.location.line, self.location.column, self.reason)
     }
 }
-
-
-pub trait SkipWhitespaceIterable<'a> {
-    fn skip_whitespace(self) -> SkipWhitespaceIterator<'a>;
-}
-
-impl<'a> SkipWhitespaceIterable<'a> for &'a [ComponentValue] {
-    fn skip_whitespace(self) -> SkipWhitespaceIterator<'a> {
-        SkipWhitespaceIterator{ iter_with_whitespace: self.iter() }
-    }
-}
-
-#[deriving(Clone)]
-pub struct SkipWhitespaceIterator<'a> {
-    pub iter_with_whitespace: slice::Items<'a, ComponentValue>,
-}
-
-impl<'a> Iterator<&'a ComponentValue> for SkipWhitespaceIterator<'a> {
-    fn next(&mut self) -> Option<&'a ComponentValue> {
-        for component_value in self.iter_with_whitespace {
-            if component_value != &ComponentValue::WhiteSpace { return Some(component_value) }
-        }
-        None
-    }
-}
-
-
-pub trait MoveSkipWhitespaceIterable {
-    fn move_skip_whitespace(self) -> MoveSkipWhitespaceIterator;
-}
-
-impl MoveSkipWhitespaceIterable for Vec<ComponentValue> {
-    fn move_skip_whitespace(self) -> MoveSkipWhitespaceIterator {
-        MoveSkipWhitespaceIterator{ iter_with_whitespace: self.into_iter() }
-    }
-}
-
-pub struct MoveSkipWhitespaceIterator {
-    iter_with_whitespace: vec::MoveItems<ComponentValue>,
-}
-
-impl Iterator<ComponentValue> for MoveSkipWhitespaceIterator {
-    fn next(&mut self) -> Option<ComponentValue> {
-        for component_value in self.iter_with_whitespace {
-            if component_value != ComponentValue::WhiteSpace { return Some(component_value) }
-        }
-        None
-    }
-}

src/color.rs

@@ -8,9 +8,7 @@ use std::num::{Float, FloatMath};
 
 use text_writer::{mod, TextWriter};
 
-use ast::{ComponentValue, SkipWhitespaceIterable};
-use ast::ComponentValue::{Number, Percentage, Function, Ident, Hash, IDHash, Comma};
-use serializer::ToCss;
+use super::{Token, Parser, ToCss};
 
 
 #[deriving(Clone, PartialEq)]
@@ -66,13 +64,18 @@ impl fmt::Show for Color {
 
 /// Return `Err(())` on invalid or unsupported value (not a color).
 impl Color {
-    pub fn parse(component_value: &ComponentValue) -> Result<Color, ()> {
-        match *component_value {
-            Hash(ref value) | IDHash(ref value) => parse_color_hash(value.as_slice()),
-            Ident(ref value) => parse_color_keyword(value.as_slice()),
-            Function(ref name, ref arguments)
-                => parse_color_function(name.as_slice(), arguments.as_slice()),
-            _ => Err(())
+    pub fn parse(input: &mut Parser) -> Result<Color, ()> {
+        match try!(input.next()) {
+            Token::Hash(ref value) | Token::IDHash(ref value) => {
+                parse_color_hash(value.as_slice())
+            }
+            Token::Ident(ref value) => parse_color_keyword(value.as_slice()),
+            Token::Function(ref name) => {
+                input.parse_nested_block().parse_entirely(|arguments| {
+                    parse_color_function(name.as_slice(), arguments)
+                })
+            }
+            token => input.unexpected(token)
         }
     }
 }
@@ -279,70 +282,43 @@ fn parse_color_hash(value: &str) -> Result<Color, ()> {
 
 
 #[inline]
-fn parse_color_function(name: &str, arguments: &[ComponentValue])
-                        -> Result<Color, ()> {
-    let lower_name = name.to_ascii_lower();
-    let lower_name = lower_name.as_slice();
-
+fn parse_color_function(name: &str, arguments: &mut Parser) -> Result<Color, ()> {
     let (is_rgb, has_alpha) =
-        if "rgba" == lower_name { (true, true) }
-        else if "rgb" == lower_name { (true, false) }
-        else if "hsl" == lower_name { (false, false) }
-        else if "hsla" == lower_name { (false, true) }
+        if name.eq_ignore_ascii_case("rgba") { (true, true) }
+        else if name.eq_ignore_ascii_case("rgb") { (true, false) }
+        else if name.eq_ignore_ascii_case("hsl") { (false, false) }
+        else if name.eq_ignore_ascii_case("hsla") { (false, true) }
         else { return Err(()) };
 
-    let mut iter = arguments.skip_whitespace();
-    macro_rules! expect_comma(
-        () => ( match iter.next() { Some(&Comma) => {}, _ => { return Err(()) } } );
-    )
-    macro_rules! expect_percentage(
-        () => ( match iter.next() {
-            Some(&Percentage(ref v)) => v.value,
-            _ => return Err(()),
-        });
-    )
-    macro_rules! expect_integer(
-        () => ( match iter.next() {
-            Some(&Number(ref v)) if v.int_value.is_some() => v.value,
-            _ => return Err(()),
-        });
-    )
-    macro_rules! expect_number(
-        () => ( match iter.next() {
-            Some(&Number(ref v)) => v.value,
-            _ => return Err(()),
-        });
-    )
-
     let red: f32;
     let green: f32;
     let blue: f32;
     if is_rgb {
         // Either integers or percentages, but all the same type.
-        match iter.next() {
-            Some(&Number(ref v)) if v.int_value.is_some() => {
+        match try!(arguments.next()) {
+            Token::Number(ref v) if v.int_value.is_some() => {
                 red = (v.value / 255.) as f32;
-                expect_comma!();
-                green = (expect_integer!() / 255.) as f32;
-                expect_comma!();
-                blue = (expect_integer!() / 255.) as f32;
+                try!(arguments.expect_comma());
+                green = try!(arguments.expect_integer()) as f32 / 255.;
+                try!(arguments.expect_comma());
+                blue = try!(arguments.expect_integer()) as f32 / 255.;
             }
-            Some(&Percentage(ref v)) => {
+            Token::Percentage(ref v) => {
                 red = (v.value / 100.) as f32;
-                expect_comma!();
-                green = (expect_percentage!() / 100.) as f32;
-                expect_comma!();
-                blue = (expect_percentage!() / 100.) as f32;
+                try!(arguments.expect_comma());
+                green = (try!(arguments.expect_percentage()) / 100.) as f32;
+                try!(arguments.expect_comma());
+                blue = (try!(arguments.expect_percentage()) / 100.) as f32;
             }
             _ => return Err(())
         };
     } else {
-        let hue = expect_number!() / 360.;
+        let hue = try!(arguments.expect_number()) / 360.;
         let hue = hue - hue.floor();
-        expect_comma!();
-        let saturation = (expect_percentage!() / 100.).max(0.).min(1.);
-        expect_comma!();
-        let lightness = (expect_percentage!() / 100.).max(0.).min(1.);
+        try!(arguments.expect_comma());
+        let saturation = (try!(arguments.expect_percentage()) / 100.).max(0.).min(1.);
+        try!(arguments.expect_comma());
+        let lightness = (try!(arguments.expect_percentage()) / 100.).max(0.).min(1.);
 
         // http://www.w3.org/TR/css3-color/#hsl-color
         fn hue_to_rgb(m1: f64, m2: f64, mut h: f64) -> f64 {
@@ -363,14 +339,11 @@ fn parse_color_function(name: &str, arguments: &[ComponentValue])
     }
 
     let alpha = if has_alpha {
-        expect_comma!();
-        (expect_number!()).max(0.).min(1.) as f32
+        try!(arguments.expect_comma());
+        (try!(arguments.expect_number())).max(0.).min(1.) as f32
     } else {
         1.
     };
-    if iter.next().is_none() {
-        Ok(Color::RGBA(RGBA { red: red, green: green, blue: blue, alpha: alpha }))
-    } else {
-        Err(())
-    }
+    try!(arguments.expect_exhausted());
+    Ok(Color::RGBA(RGBA { red: red, green: green, blue: blue, alpha: alpha }))
 }

src/from_bytes.rs

@@ -8,6 +8,8 @@ use encoding::label::encoding_from_whatwg_label;
 use encoding::all::UTF_8;
 use encoding::{EncodingRef, DecoderTrap, decode};
 
+use super::{Parser, RuleListParser, QualifiedRuleParser, AtRuleParser};
+
 
 /// Determine the character encoding of a CSS stylesheet and decode it.
 ///
@@ -65,3 +67,31 @@ fn decode_replace(input: &[u8], fallback_encoding: EncodingRef)-> (String, Encod
     let (result, used_encoding) = decode(input, DecoderTrap::Replace, fallback_encoding);
     (result.unwrap(), used_encoding)
 }
+
+
+/// Parse stylesheet from bytes.
+///
+/// * `css_bytes`: A byte string.
+/// * `protocol_encoding`: The encoding label, if any, defined by HTTP or equivalent protocol.
+///     (e.g. via the `charset` parameter of the `Content-Type` header.)
+/// * `environment_encoding`: An optional `Encoding` object for the [environment encoding]
+///     (http://www.w3.org/TR/css-syntax/#environment-encoding), if any.
+///
+/// Returns a 2-tuple of a `Iterator<Result<Rule, SyntaxError>>`
+/// and the `Encoding` object that was used.
+pub fn parse_stylesheet_rules_from_bytes<QP, AP, R, P, T>(
+        css_bytes: &[u8],
+        protocol_encoding_label: Option<&str>,
+        environment_encoding: Option<EncodingRef>,
+        rules_parser: P,
+        parse: |EncodingRef, RuleListParser<R, QP, AP, P>| -> T)
+        -> T
+        where P: QualifiedRuleParser<QP, R> + AtRuleParser<AP, R> {
+    let (css_unicode, encoding) = decode_stylesheet_bytes(
+        css_bytes, protocol_encoding_label, environment_encoding);
+    // FIXME: Remove option dance when unboxed closures permit.
+    let mut rules_parser = Some(rules_parser);
+    Parser::parse_str(css_unicode.as_slice(), |input| {
+        parse(encoding, RuleListParser::new_for_stylesheet(input, rules_parser.take().unwrap()))
+    })
+}

src/lib.rs

@@ -5,7 +5,7 @@
 #![crate_name = "cssparser"]
 #![crate_type = "rlib"]
 
-#![feature(globs, macro_rules)]
+#![feature(globs, macro_rules, if_let, while_let, unsafe_destructor)]
 
 extern crate encoding;
 extern crate text_writer;
@@ -16,18 +16,18 @@ extern crate test;
 #[cfg(test)]
 extern crate serialize;
 
-pub use color::{parse_color_keyword};
-pub use tokenizer::{tokenize, Tokenizer};
-pub use parser::{parse_stylesheet_rules, StylesheetParser,
-                 parse_rule_list, RuleListParser,
-                 parse_declaration_list, DeclarationListParser,
-                 parse_one_rule, parse_one_declaration, parse_one_component_value};
-pub use from_bytes::decode_stylesheet_bytes;
-pub use color::{RGBA, Color};
+pub use tokenizer::{Tokenizer, Token, NumericValue};
+pub use rules_and_declarations::{Priority, parse_important};
+pub use rules_and_declarations::{DeclarationParser, DeclarationListParser, parse_one_declaration};
+pub use rules_and_declarations::{RuleListParser, parse_one_rule};
+pub use rules_and_declarations::{AtRulePrelude, QualifiedRuleParser, AtRuleParser};
+pub use from_bytes::{decode_stylesheet_bytes, parse_stylesheet_rules_from_bytes};
+pub use color::{RGBA, Color, parse_color_keyword};
 pub use nth::parse_nth;
 pub use serializer::{ToCss, CssStringWriter, serialize_identifier, serialize_string};
+pub use parser::{Parser, Delimiter, Delimiters};
 
-pub mod ast;
+mod rules_and_declarations;
 mod tokenizer;
 mod parser;
 mod from_bytes;