Merge pull request #289 from davesnx/support-variable-declaration

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@davesnx/styled-ppx",
-  "version": "0.30.2",
+  "version": "0.30.3",
   "description": "Typed styled components in ReScript",
   "author": "David Sancho <[email protected]>",
   "license": "MIT",

packages/parser/css_lexer.re

@@ -66,6 +66,9 @@ let token_to_string =
   | Parser.DOT => "."
   | Parser.COMMA => ","
   | Parser.ASTERISK => "*"
+  | Parser.FUNCTION(fn) => fn ++ "("
+  | Parser.URL(url) => url ++ "("
+  | Parser.BAD_URL => "bar url"
 ;
 
 let token_to_debug =
@@ -107,6 +110,9 @@ let token_to_debug =
   | Parser.COMMA => "COMMA"
   | Parser.WS => "WS"
   | Parser.ASTERISK => "ASTERISK"
+  | Parser.FUNCTION(fn) => "FUNCTION(" ++ fn ++ ")"
+  | Parser.URL(u) => "URL(" ++ u ++ ")"
+  | Parser.BAD_URL => "BAD_URL"
 ;
 
 let () =
@@ -136,8 +142,6 @@ let whitespace = [%sedlex.regexp? " " | '\t' | newline];
 
 let whitespaces = [%sedlex.regexp? Star(whitespace)];
 
-let hex_digit = [%sedlex.regexp? '0' .. '9' | 'a' .. 'f' | 'A' .. 'F'];
-
 let digit = [%sedlex.regexp? '0' .. '9'];
 
 let non_ascii = [%sedlex.regexp? '\160' .. '\255'];
@@ -163,7 +167,7 @@ let ident_char = [%sedlex.regexp?
   '_' | 'a' .. 'z' | 'A' .. 'Z' | '0' .. '9' | '-' | non_ascii | escape
 ];
 
-let ident = [%sedlex.regexp? (Opt('-'), ident_start, Star(ident_char))];
+let ident = [%sedlex.regexp? (Opt('-'), Opt('-'), ident_start, Star(ident_char))];
 
 let variable_ident_char = [%sedlex.regexp?
   '_' | 'a' .. 'z' | 'A' .. 'Z' | '0' .. '9' | non_ascii | escape | '\''
@@ -215,8 +219,8 @@ let at_rule = [%sedlex.regexp? ("@", ident)];
 let at_media = [%sedlex.regexp? ("@", "media")];
 let at_keyframes = [%sedlex.regexp? ("@", "keyframes")];
 
-let tag = [%sedlex.regexp?
-  "a"
+let is_tag = fun
+  | "a"
   | "abbr"
   | "address"
   | "area"
@@ -332,8 +336,9 @@ let tag = [%sedlex.regexp?
   | "ul"
   | "var"
   | "video"
-  | "wbr"
-];
+  | "wbr" => true
+  | _ => false
+;
 
 let _a = [%sedlex.regexp? 'A' | 'a'];
 let _b = [%sedlex.regexp? 'B' | 'b'];
@@ -399,70 +404,255 @@ let time = [%sedlex.regexp? _s | (_m, _s)];
 
 let frequency = [%sedlex.regexp? (_h, _z) | (_k, _h, _z)];
 
+/* let escape = [%sedlex.regexp? '\\']; */
+let hex_digit = [%sedlex.regexp? digit | 'A' .. 'F' | 'a' .. 'f'];
+/* let non_ascii_code_point = [%sedlex.regexp? Sub(any, '\000' .. '\128')]; */ // greater than \u0080
+let identifier_start_code_point = [%sedlex.regexp?
+  'a' .. 'z' | 'A' .. 'Z' | non_ascii | '_'
+];
+let starts_with_a_valid_escape = [%sedlex.regexp? ('\\', Sub(any, '\n'))];
+let starts_an_identifier = [%sedlex.regexp?
+  ('-', '-' | identifier_start_code_point | starts_with_a_valid_escape) |
+  identifier_start_code_point
+];
+/* Added "'" to identifier to enable Language Variables */
+let identifier_code_point = [%sedlex.regexp?
+  identifier_start_code_point | digit | '-' | "'"
+];
+let non_printable_code_point = [%sedlex.regexp?
+  '\000' .. '\b' | '\011' | '\014' .. '\031' | '\127'
+];
+/* let ident_char = [%sedlex.regexp?
+  '_' | 'a' .. 'z' | 'A' .. 'Z' | '0' .. '9' | '-' | non_ascii | escape
+]; */
+
+/* This module is a copy/paste of Reason_css_lexer in favor of moving everything into css_lexer */
+module Tokenizer = {
+  open Reason_css_lexer;
+  let lexeme = Sedlexing.utf8;
+
+  let (let.ok) = Result.bind;
+
+  let consume_whitespace = buf =>
+    switch%sedlex (buf) {
+    | Star(whitespace) => Parser.WS
+    | _ => Parser.WS
+  };
+
+  let string_of_uchar = char => {
+    let buf = Buffer.create(0);
+    Buffer.add_utf_8_uchar(buf, char);
+    Buffer.contents(buf);
+  };
+
+  let uchar_of_int = n => Uchar.of_int(n) |> string_of_uchar;
+  let is_surrogate = char_code => char_code >= 0xD800 && char_code <= 0xDFFF;
+
+  let check = (f, buf) => {
+    // TODO: why this second int?
+    Sedlexing.mark(buf, 0);
+    let value = f(buf);
+    let _ = Sedlexing.backtrack(buf);
+    value;
+  };
+
+  // https://drafts.csswg.org/css-syntax-3/#consume-an-escaped-code-point
+  let consume_escaped = buf => {
+    switch%sedlex (buf) {
+    // TODO: spec typo? No more than 5?
+    | Rep(hex_digit, 1 .. 6) =>
+      let hex_string = "0x" ++ lexeme(buf);
+      let char_code = int_of_string(hex_string);
+      let char = uchar_of_int(char_code);
+      let _ = consume_whitespace(buf);
+      char_code == 0 || is_surrogate(char_code)
+        ? Error((Uchar.rep, Invalid_code_point)) : Ok(char);
+    | eof => Error((Uchar.rep, Eof))
+    | any => Ok(lexeme(buf))
+    | _ => failwith("unrecheable")
+    };
+  };
+
+  // https://drafts.csswg.org/css-syntax-3/#consume-remnants-of-bad-url
+  let rec consume_remnants_bad_url = buf =>
+    switch%sedlex (buf) {
+    | ")"
+    | eof => ()
+    | escape =>
+      let _ = consume_escaped(buf);
+      consume_remnants_bad_url(buf);
+    | any => consume_remnants_bad_url(buf)
+    | _ => failwith("grr unreachable")
+  };
+
+  // https://drafts.csswg.org/css-syntax-3/#consume-url-token
+  let consume_url = buf => {
+    let _ = consume_whitespace(buf);
+    let rec read = acc => {
+      let when_whitespace = () => {
+        let _ = consume_whitespace(buf);
+        switch%sedlex (buf) {
+        | ')' => Ok(Parser.URL(acc))
+        | eof => Error((URL(acc), Eof))
+        | _ =>
+          consume_remnants_bad_url(buf);
+          Ok(Parser.BAD_URL);
+        };
+      };
+      switch%sedlex (buf) {
+      | ')' => Ok(Parser.URL(acc))
+      | eof => Error((URL(acc), Eof))
+      | whitespace => when_whitespace()
+      | '"'
+      | '\''
+      | '('
+      | non_printable_code_point =>
+        consume_remnants_bad_url(buf);
+        // TODO: location on error
+        Error((BAD_URL, Invalid_code_point));
+      | escape =>
+        switch (consume_escaped(buf)) {
+        | Ok(char) => read(acc ++ char)
+        | Error((_, error)) => Error((BAD_URL, error))
+        }
+      | any => read(acc ++ lexeme(buf))
+      | _ => failwith("please, unreachable")
+      };
+    };
+    read(lexeme(buf));
+  };
+
+  // https://drafts.csswg.org/css-syntax-3/#consume-name
+  let consume_identifier = buf => {
+    let rec read = acc =>
+      switch%sedlex (buf) {
+      | identifier_code_point => read(acc ++ lexeme(buf))
+      | escape =>
+        // TODO: spec, what should happen when fails?
+        let.ok char = consume_escaped(buf);
+        read(acc ++ char);
+      | _ => Ok(acc)
+      };
+    read(lexeme(buf));
+  };
+
+  let handle_consume_identifier =
+  fun
+  | Error((_, error)) => Error((BAD_IDENT, error))
+  | Ok(string) => Ok(string);
+
+  // https://drafts.csswg.org/css-syntax-3/#consume-ident-like-token
+  let consume_ident_like = buf => {
+    let read_url = string => {
+      // TODO: the whitespace trickery here?
+      let _ = consume_whitespace(buf);
+      let is_function =
+        check(buf =>
+          switch%sedlex (buf) {
+          | '\''
+          | '"' => true
+          | _ => false
+          }
+        );
+      is_function(buf) ? Ok(Parser.FUNCTION(string)) : consume_url(buf);
+    };
+
+  let.ok string = consume_identifier(buf) |> handle_consume_identifier;
+  switch%sedlex (buf) {
+    | "(" =>
+      switch (string) {
+        | "url" => read_url(string)
+        | _ => Ok(Parser.FUNCTION(string))
+      }
+    | _ =>
+      is_tag(string) ? Ok(Parser.TAG(string)) : Ok(Parser.IDENT(string))
+    };
+  };
+};
+
+let handle_tokenizer_error = (buf: Sedlexing.t) =>
+  fun
+  | Ok(value) => value
+  | Error((_, msg)) => {
+    let error: string = Reason_css_lexer.show_error(msg);
+    let position = buf.pos;
+    raise @@ LexingError((position, error));
+  }
+;
+
 let skip_whitespace = ref(false);
 
 let rec get_next_token = (buf) => {
   open Parser;
   open Sedlexing;
   switch%sedlex (buf) {
-  | eof => [EOF]
+  | eof => EOF
   | "/*" => discard_comments(buf)
-  | '.' => [DOT]
-  | ';' => [SEMI_COLON]
+  | '.' => DOT
+  | ';' => SEMI_COLON
   | '}' => {
     skip_whitespace.contents = false;
-    [RIGHT_BRACE];
+    RIGHT_BRACE;
   }
   | '{' => {
     skip_whitespace.contents = true;
-    [LEFT_BRACE];
+    LEFT_BRACE;
   }
-  | "::" => [DOUBLE_COLON]
-  | ':' => [COLON]
-  | '(' => [LEFT_PAREN]
-  | ')' => [RIGHT_PAREN]
-  | '[' => [LEFT_BRACKET]
-  | ']' => [RIGHT_BRACKET]
-  | '%' => [PERCENTAGE]
+  | "::" => DOUBLE_COLON
+  | ':' => COLON
+  | '(' => LEFT_PAREN
+  | ')' => RIGHT_PAREN
+  | '[' => LEFT_BRACKET
+  | ']' => RIGHT_BRACKET
+  | '%' => PERCENTAGE
   | '&' => {
     skip_whitespace.contents = false;
-    [AMPERSAND];
+    AMPERSAND;
   }
-  | '*' => [ASTERISK]
-  | ',' => [COMMA]
-  | variable => [VARIABLE(latin1(~skip=2, ~drop=1, buf) |> String.split_on_char('.'))]
-  | operator => [OPERATOR(latin1(buf))]
-  | combinator => [COMBINATOR(latin1(buf))]
-  | string => [STRING(latin1(~skip=1, ~drop=1, buf))]
-  | important => [IMPORTANT]
+  | '*' => ASTERISK
+  | ',' => COMMA
+  | variable => VARIABLE(latin1(~skip=2, ~drop=1, buf) |> String.split_on_char('.'))
+  | operator => OPERATOR(latin1(buf))
+  | combinator => COMBINATOR(latin1(buf))
+  | string => STRING(latin1(~skip=1, ~drop=1, buf))
+  | important => IMPORTANT
   | at_media => {
     skip_whitespace.contents = false;
-    [AT_MEDIA(latin1(~skip=1, buf))]
+    AT_MEDIA(latin1(~skip=1, buf))
   }
   | at_keyframes => {
     skip_whitespace.contents = false;
-    [AT_KEYFRAMES(latin1(~skip=1, buf))]
+    AT_KEYFRAMES(latin1(~skip=1, buf))
   }
   | at_rule => {
     skip_whitespace.contents = false;
-    [AT_RULE(latin1(~skip=1, buf))]
+    AT_RULE(latin1(~skip=1, buf))
   }
   | at_rule_without_body => {
     skip_whitespace.contents = false;
-    [AT_RULE_STATEMENT(latin1(~skip=1, buf))]
+    AT_RULE_STATEMENT(latin1(~skip=1, buf))
   }
   /* NOTE: should be placed above ident, otherwise pattern with
    * '-[0-9a-z]{1,6}' cannot be matched */
-  | (_u, '+', unicode_range) => [UNICODE_RANGE(latin1(buf))]
-  | tag => [TAG(latin1(buf))]
-  | ident => [IDENT(latin1(buf))]
-  | ('#', name) => [HASH(latin1(~skip=1, buf))]
-  | number => [get_dimension(latin1(buf), buf)]
+  | (_u, '+', unicode_range) => UNICODE_RANGE(latin1(buf))
+  | ('#', name) => HASH(latin1(~skip=1, buf))
+  | number => get_dimension(latin1(buf), buf)
   | whitespaces => {
     if (skip_whitespace^) {
       get_next_token(buf);
-    } else { [WS] } }
-  | any => [DELIM(latin1(buf))]
+    } else {
+      WS
+    }
+  }
+  /* -moz-* */
+  | ("-", ident) => Parser.IDENT(latin1(buf))
+  /* --variable */
+  | ("-", "-", ident) => Parser.IDENT(latin1(buf))
+  | identifier_start_code_point => {
+    let _ = Sedlexing.backtrack(buf);
+    Tokenizer.consume_ident_like(buf) |> handle_tokenizer_error(buf);
+  }
+  | any => DELIM(latin1(buf))
   | _ => assert(false)
   };
 }
@@ -485,24 +675,19 @@ and get_dimension = (n, buf) => {
   }
 };
 
-let token_queue = Queue.create();
-
 let queue_next_tokens_with_location = (buf) => {
   let loc_start = Sedlexing.next_loc(buf);
-  let tokens = get_next_token(buf);
+  let token = get_next_token(buf);
   let loc_end = Sedlexing.next_loc(buf);
-  List.iter (t => Queue.add((t, loc_start, loc_end), token_queue), tokens)
+  (token, loc_start, loc_end);
 }
 
 let parse = (ws, buf, parser) => {
   skip_whitespace.contents = ws;
 
   let last_token = ref((Parser.EOF, Lexing.dummy_pos, Lexing.dummy_pos));
   let next_token = () => {
-    if (Queue.is_empty(token_queue)) {
-      queue_next_tokens_with_location(buf);
-    }
-    last_token := Queue.take(token_queue);
+    last_token := queue_next_tokens_with_location(buf);
     last_token^;
   };