From cabc4b81f0046f0308c13303809577ff881727d7 Mon Sep 17 00:00:00 2001 From: Nikos M Date: Tue, 13 Oct 2015 15:07:18 +0300 Subject: [PATCH] v. 2.1.0 (re-upload) --- README.md | 127 ++------ build/codemirror_grammar.js | 418 +++++++++++++++++--------- build/codemirror_grammar.min.js | 2 +- grammar-reference.md | 44 ++- src/main.js | 36 ++- src/state.js | 40 ++- src/tokenizers.js | 287 +++++++++++++----- src/types.js | 55 ++-- test/grammars/css.js | 135 +++------ test/grammars/javascript-recursion.js | 110 +++---- test/grammars/json.js | 39 +-- test/grammars/xml.js | 127 ++------ 12 files changed, 753 insertions(+), 667 deletions(-) diff --git a/README.md b/README.md index 3d4b050..ac1c40a 100644 --- a/README.md +++ b/README.md @@ -70,62 +70,31 @@ var xml_grammar = { // Style model "Style" : { // lang token type -> Editor (style) tag - "comment_block": "comment", - "meta_block": "meta", - "cdata_block": "atom", - "atom": "atom", - "open_tag": "tag", - "close_open_tag": "tag", - "auto_close_open_tag": "tag", - "close_tag": "tag", - "attribute": "attribute", - "id": "attribute", - "number": "number", - "string": "string" + "comment" : "comment", + "meta" : "meta", + "cdata" : "atom", + "atom" : "atom", + "open_tag" : "tag", + "close_open_tag" : "tag", + "auto_close_open_tag" : "tag", + "close_tag" : "tag", + "att" : "attribute", + "id" : "attribute", + "number" : "number", + "string" : "string" }, // // Lexical model "Lex": { + "comment:comment": [""], - "comment_block": { - "type": "comment", - "tokens": [ - // block comments - // start, end delims - [ "" ] - ] - }, + "cdata:block": [""], - "cdata_block": { - "type": "block", - "tokens": [ - // cdata block - // start, end delims - [ "" ] - ] - }, + "meta:block": ["RE::/<\\?[_a-zA-Z][\\w\\._\\-]*/","?>"], - "meta_block": { - "type": "block", - "tokens": [ - // meta block - // start, end delims - [ "RE::/<\\?[_a-zA-Z][\\w\\._\\-]*/", "?>" ] - ] - }, - - // strings - "string": { - "type": "block", - "multiline": false, - "tokens": [ - // if no end given, end is same as start - [ "\"" ], [ "'" ] - ] - }, + "string:block": {"tokens":[[ "\"" ],[ "'" ]], "multiline":false}, - // numbers, in order of matching "number": [ // dec "RE::/[0-9]\\d*/", @@ -140,8 +109,8 @@ var xml_grammar = { "RE::/&[a-zA-Z][a-zA-Z0-9]*;/" ], - // tag attributes - "attribute": "RE::/[_a-zA-Z][_a-zA-Z0-9\\-]*/", + // tag attribute + "att": "RE::/[_a-zA-Z][_a-zA-Z0-9\\-]*/", // tags "open_tag": "RE::/<([_a-zA-Z][_a-zA-Z0-9\\-]*)/", @@ -149,79 +118,47 @@ var xml_grammar = { "auto_close_open_tag": "/>", "close_tag": "RE::/<\\/([_a-zA-Z][_a-zA-Z0-9\\-]*)>/", - // NEW feature - // action tokens to perform complex grammar functionality - // like associated tag matching and unique identifiers - - "ctx_start": { - "context-start": true - }, - - "ctx_end": { - "context-end": true - }, + "text": "RE::/[^<&]+/", + // actions + "ctx_start:action": {"context-start":true}, + "ctx_end:action": {"context-end":true}, // allow to find duplicate xml identifiers, with action tokens - "unique": { + "unique:action": { "unique": ["id", "$1"], "msg": "Duplicate id attribute \"$0\"" }, - // allow to find duplicate xml tag attributes, with action tokens - "unique_att": { + "unique_att:action": { "unique": ["att", "$0"], - "in-context": true, + "in-context":true, "msg": "Duplicate attribute \"$0\"" }, - // allow to match start/end tags, with action tokens - "match": { - "push": "<$1>" - }, - - "matched": { + "match:action": {"push":"<$1>"}, + "matched:action": { "pop": "<$1>", "msg": "Tags \"$0\" and \"$1\" do not match!" }, - - "nomatch": { - "pop": null - } + "nomatch:action": {"pop":null} }, // // Syntax model (optional) "Syntax": { - // NEW feature - // using PEG/BNF-like shorthands, instead of multiple grammar configuration objects - "id_att": "'id' unique_att '=' string unique", - "tag_att": "attribute unique_att '=' (string | number)", + "tag_att": "att unique_att '=' (string | number)", "start_tag": "open_tag match ctx_start (id_att | tag_att)* (close_open_tag | auto_close_open_tag nomatch) ctx_end", - "end_tag": "close_tag matched", - "tags": { - "type": "ngram", - "tokens": [ - ["start_tag"], - ["end_tag"] - ] - }, + "end_tag": "close_tag matched", - "blocks": { - "type": "ngram", - "tokens": [ - ["comment_block"], - ["cdata_block"], - ["meta_block"], - ] - } + "xml": "comment | cdata | meta | start_tag | end_tag | atom | text" }, // what to parse and in what order - "Parser": [ "blocks", "tags", "atom" ] + "Parser": [ ["xml"] ] }; // 2. parse the grammar into a Codemirror syntax-highlight mode diff --git a/build/codemirror_grammar.js b/build/codemirror_grammar.js index c9d8db5..f356d36 100644 --- a/build/codemirror_grammar.js +++ b/build/codemirror_grammar.js @@ -58,9 +58,8 @@ var A_EMPTY = 64, A_INDENT = 128, A_OUTDENT = 256, - A_BLOCKINDENT = 512, - A_CTX_START = 1024, - A_CTX_END = 2048, + A_CTXSTART = 512, + A_CTXEND = 1024, // // pattern types @@ -71,26 +70,27 @@ var // // token types //T_SPACE = 0, - T_ERROR = 4, - T_DEFAULT = 8, - T_SIMPLE = 16, - T_EOL = 17, - T_NONSPACE = 18, - T_EMPTY = 20, - T_BLOCK = 32, - T_ESCBLOCK = 33, - T_COMMENT = 34, - T_EITHER = 64, - T_SEQUENCE = 128, + T_SOF = 4, T_EOF = 8, T_SOL = 16, T_EOL = 32, + T_SOF_OR_SOL = T_SOF|T_SOL, + T_EMPTY = 64, T_NONSPACE = 128, + //T_EMPTY_OR_NONSPACE = T_EMPTY|T_NONSPACE, + //T_NOT_EMPTY_NOR_NONSPACE = ~T_EMPTY_OR_NONSPACE, + T_SIMPLE = 256, + T_BLOCK = 512, + T_ESCBLOCK = 513, + T_COMMENT = 514, + T_EITHER = 1024, + T_SEQUENCE = 2048, T_ALL = T_SEQUENCE, - T_REPEATED = 256, - T_ZEROORONE = 257, - T_ZEROORMORE = 258, - T_ONEORMORE = 259, - T_GROUP = 512, - T_NGRAM = 1024, + T_REPEATED = 4096, + T_ZEROORONE = 4097, + T_ZEROORMORE = 4098, + T_ONEORMORE = 4099, + T_GROUP = 8192, + T_NGRAM = 16384, T_SEQUENCE_OR_NGRAM = T_SEQUENCE | T_NGRAM, - T_ACTION = 2048, + T_ACTION = 32768, + //T_SOF_SOL_EOL_EOF_ACTION = T_SOF_SOL_EOL_EOF|T_ACTION, // // tokenizer types @@ -112,19 +112,6 @@ var SIMPLE: T_SIMPLE, GROUP: T_GROUP, NGRAM: T_NGRAM - }, - - actionTypes = { - ERROR: A_ERROR, - UNIQUE: A_UNIQUE, - PUSH: A_PUSH, - POP: A_POP, - EMPTY: A_EMPTY, - INDENT: A_INDENT, - OUTDENT: A_OUTDENT, - BLOCKINDENT: A_BLOCKINDENT, - CONTEXT_START: A_CTX_START, - CONTEXT_END: A_CTX_END } ; @@ -719,7 +706,7 @@ var Stack = Class({ // // State Class var State = Class({ - constructor: function State( s, unique, with_errors ) { + constructor: function State( s, unique, status ) { var self = this; // this enables unique state "names" // thus forces highlight to update @@ -729,61 +716,57 @@ var State = Class({ { // clone self.line = s.line; - self.column = s.column; - self.indent = s.indent; + //self.indent = s.indent; + self.status = s.status; + self.token = s.token; + self.block = s.block; self.stack = s.stack.clone(); self.queu = s.queu.slice(); self.symb = clone( s.symb, 1 ); self.ctx = s.ctx.slice(); - self.token = s.token; - self.block = s.block; - self.errors = s.errors; self.err = s.err; } else { self.line = s || 0; - self.column = 0; - self.indent = null; + //self.indent = null; + self.status = status || 0; self.token = null; self.block = null; self.stack = new Stack(); self.queu = []; self.symb = {}; self.ctx = []; - self.errors = !!with_errors; - self.err = self.errors ? {} : null; + self.err = self.status&ERRORS ? {} : null; } } ,id: null ,line: 0 - ,column: 0 - ,indent: 0 + //,indent: 0 + ,status: 0 + ,token: null + ,block: null ,stack: null ,queu: null ,symb: null ,ctx: null ,err: null - ,errors: false - ,token: null - ,block: null ,dispose: function( ) { var self = this; self.id = null; self.line = null; - self.column = null; - self.indent = null; + //self.indent = null; + self.status = null; self.token = null; self.block = null; + if ( self.stack ) self.stack.dispose( ); + self.stack = null; + self.queu = null; self.symb = null; self.ctx = null; - self.errors = null; self.err = null; - self.queu = null; - if ( self.stack ) self.stack.dispose( ); - self.stack = null; return self; } @@ -795,7 +778,7 @@ var State = Class({ // make sure to generate a string which will cover most cases where state needs to be updated by the editor ,toString: function() { var self = this; - return ['', self.id, self.line, self.token ? self.token.name : '0', self.block ? self.block.name : '0'].join('_'); + return [self.id, self.line, self.token ? self.token.name : '0', self.block ? self.block.name : '0'].join('_'); } }); @@ -843,8 +826,8 @@ function match_re( stream, eat ) var self = this, p = self.pattern, s = stream.s, m; m = s.slice( stream.pos ).match( p[0] ); if ( !m || m.index > 0 ) return false; - if ( false !== eat ) stream.mov( m[ 0 ].length ); - return [ self.key, m ]; + if ( false !== eat ) stream.mov( m[ p[1]||0 ].length ); + return [ self.key, p[1] > 0 ? m[p[1]] : m ]; } function match_null( stream, eat ) @@ -1042,7 +1025,7 @@ function get_compositematcher( name, tokens, RegExpID, combined, cachedRegexes, } else if ( 1 < l /*combined*/ ) { - l2 = (l>>1) + 1; + l2 = (l>>>1) + 1; // check if tokens can be combined in one regular expression // if they do not contain sub-arrays or regular expressions for (i=0; i<=l2; i++) @@ -1077,7 +1060,7 @@ function get_compositematcher( name, tokens, RegExpID, combined, cachedRegexes, { matcher = get_simplematcher( name, get_combined_re( tmp, combined ), 0, cachedMatchers ); } - else + else if ( array_of_arrays || has_regexs ) { for (i=0; i 1) ? new CompositePattern( name, tmp ) : tmp[0]; + } + else /* strings */ + { + tmp = tmp.sort( by_length ); + for (i=0; i 1) ? new CompositePattern( name, tmp ) : tmp[0]; } } @@ -1135,6 +1128,12 @@ Token = Class({ self.msg = msg || null; self.$msg = null; self.$clone = null; + if ( T_SOF === self.name ) self.name = ''; + else if ( T_SOL === self.name ) self.name = ''; + else if ( T_EOL === self.name ) self.name = ''; + else if ( T_EOF === self.name ) self.name = ''; + else if ( T_EMPTY === self.name ) self.name = ''; + else if ( T_NONSPACE === self.name ) self.name = ''; } // tokenizer/token name/id @@ -1188,41 +1187,50 @@ Token = Class({ var self = this, token = self.token, type = self.type, tokenID = self.name, t = null; - self.$msg = null; + self.$msg = self.msg || null; state.token = null; //self.pos = null; - //lin = state.line; col = stream.pos; - // match EMPTY token - if ( T_EMPTY === type ) + // match SOF (start-of-file) + if ( T_SOF === type ) { - self.status = 0; - //self.pos = [[lin, col], [lin, col]]; - return true; + if ( (T_SOF&state.status) && stream.sol() ) return true; + } + // match SOL (start-of-line) + else if ( T_SOL === type ) + { + if ( stream.sol() ) return true; } - // match EOL ( with possible leading spaces ) + // match EOL (end-of-line) ( with possible leading spaces ) else if ( T_EOL === type ) { stream.spc(); - if ( stream.eol() ) - { - //self.pos = [[lin, col], [lin, stream.pos]]; - return tokenID; - } + if ( stream.eol() ) return tokenID; + } + // match EOF (end-of-file) ( with possible leading spaces ) + /*else if ( T_EOF === type ) + { + stream.spc(); + if ( stream.eol() ) return true; + }*/ + // match EMPTY token + else if ( T_EMPTY === type ) + { + self.status = 0; + return true; } // match non-space else if ( T_NONSPACE === type ) { if ( (self.status&REQUIRED) && stream.spc() && !stream.eol() ) self.status |= ERROR; self.status &= CLEAR_REQUIRED; - //self.pos = [[lin, col], [lin, col]]; } // else match a simple token else if ( t = token.match(stream) ) { state.token = {name:tokenID, value:stream.cur(), token:t[1]}; - //self.pos = [[lin, col], [lin, stream.pos]]; return tokenID; } + if ( self.status && self.$msg ) self.$msg = group_replace( self.$msg, tokenID, true ); return false; } @@ -1234,13 +1242,13 @@ Token = Class({ } ,err: function( state, l1, c1, l2, c2 ) { - var t = this, m, tok = t.name; + var t = this, m, token = t.name; if ( t.$msg ) m = t.$msg; - else if ( t.status&REQUIRED ) m = 'Token "'+tok+'" Expected'; - else m = 'Syntax Error: "'+tok+'"'; - if ( state && state.errors ) + else if ( t.status&REQUIRED ) m = 'Token "'+token+'" Expected'; + else m = 'Syntax Error: "'+token+'"'; + if ( state && (state.status&ERRORS) ) { - state.err[l1+'_'+c1+'_'+l2+'_'+c2+'_'+tok] = [l1, c1, l2, c2, m]; + state.err[l1+'_'+c1+'_'+l2+'_'+c2+'_'+token] = [l1, c1, l2, c2, m]; } return m; } @@ -1272,7 +1280,7 @@ ActionToken = Class(Token, { ,get: function( stream, state ) { var self = this, action_def = self.token || null, action, t, t0, ns, msg, queu = state.queu, symb = state.symb, ctx = state.ctx, token = state.token, - lin, col1, col2, in_ctx, err = state.err, error, emsg, with_errors = state.errors; + lin, col1, col2, in_ctx, err = state.err, error, emsg, with_errors = !!(state.status&ERRORS); self.status = 0; self.$msg = null; @@ -1295,6 +1303,16 @@ ActionToken = Class(Token, { return false; } + else if ( A_CTXSTART === action ) + { + ctx.unshift({symb:{},queu:[]}); + } + + else if ( A_CTXEND === action ) + { + if ( ctx.length ) ctx.shift(); + } + else if ( A_EMPTY === action ) { if ( in_ctx ) @@ -1307,23 +1325,7 @@ ActionToken = Class(Token, { } } - else if ( A_CTX_START === action ) - { - //ns = uuid('ctx'); - ctx.unshift({symb:{},queu:[]}); - } - - else if ( A_CTX_END === action ) - { - if ( ctx.length ) ctx.shift(); - } - - /*else if ( A_BLOCKINDENT === action ) - { - // TODO - } - - else if ( A_INDENT === action ) + /*else if ( A_INDENT === action ) { // TODO } @@ -1485,7 +1487,6 @@ BlockToken = Class(Token, { { found = 1; endBlock = state.block.end; - //self.pos = state.block.pos; alreadyIn = 1; ret = thisBlockInterior; b_s = state.block.s; @@ -1498,7 +1499,6 @@ BlockToken = Class(Token, { else if ( !state.block && (endBlock = startBlock.match(stream)) ) { found = 1; - //self.pos = [[lin, col],[lin, stream.pos]]; b_s = [lin,col]; b_i = [[lin,stream.pos],[lin,stream.pos]]; b_e = [lin,stream.pos]; @@ -1588,22 +1588,19 @@ BlockToken = Class(Token, { if ( ended || (!continueToNextLine && !continueBlock) ) { state.block = null; - //self.pos[1] = [lin, stream.pos]; } else { - //self.pos[1] = [lin, stream.pos]; - //state.block.pos = self.pos; state.block.i = b_i; state.block.e = b_e; state.block._i += b_21; state.block._e = b_3; state.stack.pushAt( stackPos, self.clone( ), '$id', thisBlock ); } - state.token = {name:thisBlock, value:stream.cur(), token:[b_1+b_2+b_21+b_3, b_2+b_21]}; + state.token = {name:thisBlock, value:stream.cur(), token:[b_1+b_2+b_21+b_3, b_2+b_21, b_1, b_3]}; return ret; } - + if ( self.status && self.$msg ) self.$msg = group_replace( self.$msg, thisBlock, true ); return false; } }); @@ -1685,16 +1682,37 @@ CompositeToken = Class(Token, { if ( match_all ) self.status |= REQUIRED; else self.status &= CLEAR_REQUIRED; stackPos = stack.pos(); - token = tokens[ 0 ].clone().req( match_all ); - style = token.get(stream, state); stackId = self.name+'_'+get_id(); + i0 = 0; + do{ + token = tokens[ i0 ].clone().req( match_all ); + style = token.get(stream, state); + i0++; + // handle SOF,SOL tokens here + }while ( i0= n ) + { + if ( token.status&ERROR /*&& token.REQ*/ ) + { + if ( match_all ) self.status |= ERROR; + else self.status &= CLEAR_ERROR; + stream.bck( streamPos ); + } + else if ( match_all && (token.status&REQUIRED) ) + { + self.status |= ERROR; + } + if ( self.status && !self.$msg ) self.$msg = token.err(); + return style; + } if ( false !== style ) { // not empty token - if ( true !== style ) + if ( true !== style) { - for (i=n-1; i>0; i--) + for (i=n-1; i>=i0; i--) stack.pushAt( stackPos+n-i-1, tokens[ i ].clone().req( 1 ), '$id', stackId ); } @@ -1791,8 +1809,28 @@ function parse_peg_bnf_notation( tok, Lex, Syntax ) if ( '0' === token ) { // interpret as empty tokenizer - sequence.push( 0 ); + sequence.push( T_EMPTY ); + } + else if ( '^^' === token ) + { + // interpret as SOF tokenizer + sequence.push( T_SOF ); } + else if ( '^' === token ) + { + // interpret as SOL tokenizer + sequence.push( T_SOL ); + } + else if ( '$' === token ) + { + // interpret as EOL tokenizer + sequence.push( T_EOL ); + } + /*else if ( '$$' === token ) + { + // interpret as EOF tokenizer + sequence.push( T_EOF ); + }*/ else { if ( !Lex[token] && !Syntax[token] ) @@ -2079,10 +2117,22 @@ function get_tokenizer( tokenID, RegExpID, Lex, Syntax, Style, MSG = null; - if ( null === tokenID ) + if ( T_SOF === tokenID ) + { + // SOF Token + return new Token( T_SOF, T_SOF, tokenID, MSG ); + } + + else if ( T_SOL === tokenID ) + { + // SOL Token + return new Token( T_SOL, T_SOL, tokenID, MSG ); + } + + else if ( T_EOL === tokenID || null === tokenID ) { // EOL Token - return new Token( T_EOL, 'EOL', tokenID, MSG ); + return new Token( T_EOL, T_EOL, tokenID, MSG ); } else if ( "" === tokenID ) @@ -2184,12 +2234,6 @@ function get_tokenizer( tokenID, RegExpID, Lex, Syntax, Style, tok.action = [ 'empty', tok.empty, !!tok['in-context'] ]; delete tok.empty; } - else if ( tok[HAS]('block-indent') ) - { - tok.type = "action"; - tok.action = [ 'block-indent', tok['block-indent'], !!tok['in-context'] ]; - delete tok['block-indent']; - } else if ( tok[HAS]('indent') ) { tok.type = "action"; @@ -2291,15 +2335,25 @@ function get_tokenizer( tokenID, RegExpID, Lex, Syntax, Style, if ( T_SIMPLE & type ) { - if ( "" === tok.tokens ) + if ( T_SOF === tok.tokens ) { - // NONSPACE Token - token = new Token( T_NONSPACE, tokenID, tokenID, MSG ); + // SOF Token + token = new Token( T_SOF, tokenID, tokenID, MSG ); + // pre-cache tokenizer to handle recursive calls to same tokenizer + cachedTokens[ tokenID ] = token; + return token; + } + + else if ( T_SOL === tok.tokens ) + { + // SOL Token + token = new Token( T_SOL, tokenID, tokenID, MSG ); // pre-cache tokenizer to handle recursive calls to same tokenizer cachedTokens[ tokenID ] = token; return token; } - else if ( null === tok.tokens ) + + else if ( T_EOL === tok.tokens || null === tok.tokens ) { // EOL Token token = new Token( T_EOL, tokenID, tokenID, MSG ); @@ -2307,6 +2361,16 @@ function get_tokenizer( tokenID, RegExpID, Lex, Syntax, Style, cachedTokens[ tokenID ] = token; return token; } + + else if ( "" === tok.tokens ) + { + // NONSPACE Token + token = new Token( T_NONSPACE, tokenID, tokenID, MSG ); + // pre-cache tokenizer to handle recursive calls to same tokenizer + cachedTokens[ tokenID ] = token; + return token; + } + else if ( false === tok.tokens || 0 === tok.tokens ) { // EMPTY Token @@ -2324,10 +2388,9 @@ function get_tokenizer( tokenID, RegExpID, Lex, Syntax, Style, if ( !tok[HAS]('action') ) { if ( tok[HAS]('error') ) tok.action = [A_ERROR, tok.error, !!tok['in-context']]; - else if ( tok[HAS]('context-start') ) tok.action = [A_CTX_START, tok['context-start'], !!tok['in-context']]; - else if ( tok[HAS]('context-end') ) tok.action = [A_CTX_END, tok['context-end'], !!tok['in-context']]; + else if ( tok[HAS]('context-start') ) tok.action = [A_CTXSTART, tok['context-start'], !!tok['in-context']]; + else if ( tok[HAS]('context-end') ) tok.action = [A_CTXEND, tok['context-end'], !!tok['in-context']]; else if ( tok[HAS]('empty') ) tok.action = [A_EMPTY, tok.empty, !!tok['in-context']]; - else if ( tok[HAS]('block-indent') ) tok.action = [A_BLOCKINDENT, tok['block-indent'], !!tok['in-context']]; else if ( tok[HAS]('indent') ) tok.action = [A_INDENT, tok.indent, !!tok['in-context']]; else if ( tok[HAS]('outdent') ) tok.action = [A_OUTDENT, tok.outdent, !!tok['in-context']]; else if ( tok[HAS]('unique') ) tok.action = [A_UNIQUE, T_STR&get_type(tok.unique)?['_DEFAULT_',tok.unique]:tok.unique, !!tok['in-context']]; @@ -2337,10 +2400,9 @@ function get_tokenizer( tokenID, RegExpID, Lex, Syntax, Style, else { if ( 'error' === tok.action[0] ) tok.action[0] = A_ERROR; - else if ( 'context-start' === tok.action[0] ) tok.action[0] = A_CTX_START; - else if ( 'context-end' === tok.action[0] ) tok.action[0] = A_CTX_END; + else if ( 'context-start' === tok.action[0] ) tok.action[0] = A_CTXSTART; + else if ( 'context-end' === tok.action[0] ) tok.action[0] = A_CTXEND; else if ( 'empty' === tok.action[0] ) tok.action[0] = A_EMPTY; - else if ( 'block-indent' === tok.action[0] ) tok.action[0] = A_BLOCKINDENT; else if ( 'indent' === tok.action[0] ) tok.action[0] = A_INDENT; else if ( 'outdent' === tok.action[0] ) tok.action[0] = A_OUTDENT; else if ( 'unique' === tok.action[0] ) tok.action[0] = A_UNIQUE; @@ -2358,7 +2420,7 @@ function get_tokenizer( tokenID, RegExpID, Lex, Syntax, Style, if ( tok.autocomplete ) get_autocomplete( tok, tokenID, keywords ); // combine by default if possible using word-boundary delimiter - combine = ( 'undefined' === typeof(tok.combine) ) ? "\\b" : tok.combine; + combine = 'undefined' === typeof(tok.combine) ? "\\b" : tok.combine; token = new Token( T_SIMPLE, tokenID, get_compositematcher( tokenID, tok.tokens.slice(), RegExpID, combine, cachedRegexes, cachedMatchers ), MSG @@ -2490,7 +2552,7 @@ function get_autocomplete( tok, type, keywords ) function parse_grammar( grammar ) { var RegExpID, tokens, numTokens, _tokens, - Style, Lex, Syntax, t, tokenID, token, tok, + Style, Lex, Syntax, t, tokenID, token, tok, tt, tID, tT, cachedRegexes, cachedMatchers, cachedTokens, commentTokens, comments, keywords; // grammar is parsed, return it @@ -2515,6 +2577,66 @@ function parse_grammar( grammar ) Style = grammar.Style || { }; + // shorthand token-type annotation in token_ID + for (tt in Lex) + { + if ( !Lex[HAS](tt) ) continue; + tID = tt.split(':'); + if ( tID[1] && trim(tID[1]).length ) + { + tT = trim(tID[1]).toLowerCase(); + } + else + { + tT = null; + } + tID = tID[0]; + if ( tID !== tt ) + { + Lex[tID] = Lex[tt]; delete Lex[tt]; + if ( tT ) + { + if ( T_OBJ & get_type(Lex[tID]) ) + { + if ( !Lex[tID].type ) Lex[tID].type = tT; + } + else + { + Lex[tID] = {type:tT, tokens:Lex[tID]}; + } + } + } + } + for (tt in Syntax) + { + if ( !Syntax[HAS](tt) ) continue; + tID = tt.split(':'); + if ( tID[1] && trim(tID[1]).length ) + { + tT = trim(tID[1]).toLowerCase(); + } + else + { + tT = null; + } + tID = tID[0]; + if ( tID !== tt ) + { + Syntax[tID] = Syntax[tt]; delete Syntax[tt]; + if ( tT ) + { + if ( T_OBJ & get_type(Syntax[tID]) ) + { + if ( !Syntax[tID].type ) Syntax[tID].type = tT; + } + else + { + Syntax[tID] = {type:tT, tokens:Syntax[tID]}; + } + } + } + } + _tokens = grammar.Parser || [ ]; numTokens = _tokens.length; tokens = [ ]; @@ -2626,9 +2748,8 @@ var Parser = Class({ parse_errors = !!(parse_type&ERRORS); parse_tokens = !!(parse_type&TOKENS); - state = new State( 0, 0, parse_errors ); + state = new State( 0, 0, parse_type ); state.parseAll = 1; - if ( parse_tokens ) { linetokens = []; @@ -2664,12 +2785,22 @@ var Parser = Class({ ; stream = parseAll ? stream : Stream._( stream ); + if ( parseAll ) + { + if ( 0 === state.line ) state.status |= T_SOF; + else state.status &= ~T_SOF; + } + else + { + if ( stream.sol() ) state.status |= T_SOF; + else state.status &= ~T_SOF; + } stack = state.stack; - // if EOL tokenizer is left on stack, pop it now - if ( stream.sol() && !stack.isEmpty() && T_EOL === stack.peek().type ) + if ( stream.sol() ) { - stack.pop(); + // if EOL tokenizer is left on stack, pop it now + while( !stack.isEmpty() && T_EOL === stack.peek().type ) stack.pop(); } // check for non-space tokenizer before parsing space @@ -2724,7 +2855,7 @@ var Parser = Class({ // found token (not empty) else if ( true !== type ) { - type = /*T_SPACE === type ? DEFAULT :*/ Style[type] || DEFAULT; + type = Style[type] || DEFAULT; // action token follows, execute action on current token while ( !stack.isEmpty() && T_ACTION === stack.peek().type ) { @@ -2734,11 +2865,9 @@ var Parser = Class({ /*if ( action.status&ERROR ) { // empty the stack - //stack.empty('$id', /*action* /tokenizer.$id); + //stack.empty('$id', tokenizer.$id); // generate error - //type = ERR; //action.err(state, line, pos, line, stream.pos); - return parseAll ? {value:stream.cur(1), type:type} : (stream.upd()&&type); }*/ } return parseAll ? {value:stream.cur(1), type:type} : (stream.upd()&&type); @@ -2775,7 +2904,7 @@ var Parser = Class({ // found token (not empty) else if ( true !== type ) { - type = /*T_SPACE === type ? DEFAULT :*/ Style[type] || DEFAULT; + type = Style[type] || DEFAULT; // action token follows, execute action on current token while ( !stack.isEmpty() && T_ACTION === stack.peek().type ) { @@ -2787,9 +2916,7 @@ var Parser = Class({ // empty the stack //stack.empty('$id', tokenizer.$id); // generate error - //type = ERR; //action.err(state, line, pos, line, stream.pos); - return parseAll ? {value:stream.cur(1), type:type} : (stream.upd()&&type); }*/ } return parseAll ? {value: stream.cur(1), type: type} : (stream.upd()&&type); @@ -2830,12 +2957,13 @@ function get_mode( grammar, DEFAULT ) */ startState: function( ) { - return new State( ); + var state = new State( ); + return state; }, copyState: function( state ) { - state = state.clone( ); state.line++; - return state; + var statec = state.clone( ); + return statec; }, token: function( stream, state ) { diff --git a/build/codemirror_grammar.min.js b/build/codemirror_grammar.min.js index ecd8fe9..0adb24c 100644 --- a/build/codemirror_grammar.min.js +++ b/build/codemirror_grammar.min.js @@ -6,4 +6,4 @@ * Transform a grammar specification in JSON format, into a syntax-highlight parser mode for CodeMirror * https://github.com/foo123/codemirror-grammar * -**/!function(t,e,n){"use strict";var r,o="object"==typeof module&&module.exports,l="function"==typeof define&&define.amd;o?module.exports=(module.$deps=module.$deps||{})[e]=module.$deps[e]||n.call(t,{NODE:module})||1:l&&"function"==typeof require&&"function"==typeof require.specified&&require.specified(e)?define(e,["require","exports","module"],function(e,r,o){return n.call(t,{AMD:o})}):e in t||(t[e]=r=n.call(t,{})||1)&&l&&define(e,[],function(){return r})}(this,"CodeMirrorGrammar",function(t){"use strict";function e(t,e){var n=this,r=n.pattern,o=t.s.charAt(t.pos)||null;return o&&r===o?(!1!==e&&t.mov(1),[n.key,o]):!1}function n(t,e){var n=this,r=n.pattern,o=t.s.charAt(t.pos)||null;return o&&-10?!1:(!1!==e&&t.mov(n[0].length),[r.key,n])}function l(t,e){var n=this;return!1!==e&&t.end(),[n.key,""]}function s(t,e,n,r){var o=Se(e);if(ae===o)return e;if(r[t])return r[t];n=n||0;var l,s=0;return e&&e.isCharList&&(s=1,delete e.isCharList),l=xe&o?new tn(t,e,xe,n):ge===o?new tn(t,e,ge,n):me&o?new tn(t,e,s?ke:me,n):(be|de)&o?new tn(t,e,be,n):e,r[t]=l}function u(t,e,n,r,o,l){if(l[t])return l[t];var i,c,a,p,f,h,m,g=0,k=0,d=1;if(i=Re(e),a=i.length,1===a)m=s(t,Ue(i[0],n,o),0,l);else if(a>1){for(p=(a>>1)+1,c=0;p>=c;c++)f=Se(i[c]),h=Se(i[a-1-c]),(ge!==f||ge!==h)&&(d=0),de&f||de&h?g=1:(Ge(i[c],n)||Ge(i[a-1-c],n))&&(k=1);if(!d||r&&me&Se(r))if(!r||g||k){for(c=0;a>c;c++)i[c]=de&Se(i[c])?u(t+"_"+c,i[c],n,r,o,l):s(t+"_"+c,Ue(i[c],n,o),c,l);m=a>1?new nn(t,i):i[0]}else m=s(t,Ke(i,r),0,l);else i=i.slice().join(""),i.isCharList=1,m=s(t,i,0,l)}return l[t]=m}function i(t,e,n,r,o){if(o[t])return o[t];var l,u,i,c,a,p,f;for(c=[],a=[],l=je(e),u=0,i=l.length;i>u;u++)p=s(t+"_0_"+u,Ue(l[u][0],n,r),u,o),f=l[u].length>1?be!==p.ptype||me!==Se(l[u][1])||Ge(l[u][1],n)?s(t+"_1_"+u,Ue(l[u][1],n,r),u,o):l[u][1]:p,c.push(p),a.push(f);return o[t]=new en(t,[c,a])}function c(t,e,n){var r,o,l,s,u,i,c,a,p,f,h,m;if(i=new String(Ne(t)),i.pos=0,1===i.length)f=""+t,e[f]||(e[f]={type:"simple",tokens:t}),t=f;else{for(r=[],o=[],l="",h=[];i.posu[0]&&(u[0]=0),2>u.length?u.push(u[0]):u[1]=u[1].length?parseInt(u[1],10)||E:E,0>u[1]&&(u[1]=0),p=o.pop(),f=""+p+["{",u[0],",",isFinite(u[1])?u[1]:"","}"].join(""),n[f]||(n[f]={type:"group",match:[u[0],u[1]],tokens:[p]}),o.push(f)}else{if("}"===a)continue;if("["===a){for(s="";i.pos1?(f=""+o.join(" "),n[f]||(n[f]={type:"group",match:"sequence",tokens:o}),r.push(f)):o.length&&r.push(o[0]),o=[]):"("===a?(h.push([o,r,l]),o=[],r=[],l=""):")"===a&&(o.length>1?(f=""+o.join(" "),n[f]||(n[f]={type:"group",match:"sequence",tokens:o}),r.push(f)):o.length&&r.push(o[0]),o=[],r.length>1?(f=""+r.join(" | "),n[f]||(n[f]={type:"group",match:"either",tokens:r})):r.length&&(f=r[0]),r=[],m=h.pop(),o=m[0],r=m[1],l=m[2],p=f,f="("+p+")",n[f]||(n[f]=Me(e[p]||n[p])),o.push(f))}}else l+=a;l.length&&(e[l]||n[l]||(e[l]={type:"simple",tokens:l}),o.push(l)),l="",o.length>1?(f=""+o.join(" "),n[f]||(n[f]={type:"group",match:"sequence",tokens:o}),r.push(f)):o.length&&r.push(o[0]),o=[],r.length>1?(f=""+r.join(" | "),n[f]||(n[f]={type:"group",match:"either",tokens:r}),t=f):r.length&&(t=r[0]),r=[]}return t}function a(t,e,n,r,o,l,s,h,m,g,k){var d,y,v,b,_,x,C,P,L,D,I,Z,te,oe,le,ue,ie,ce=null;if(ie=null,null===t)return new rn(N,"EOL",t,ie);if(""===t)return new rn(F,"NONSPACE",t,ie);if(!1===t||0===t)return new rn(V,"EMPTY",t,ie);if(de&Se(t)&&(ue=t,t="NGRAM_"+ue.join("_"),r[t]||(r[t]={type:"ngram",tokens:ue})),t=""+t,h[t])return h[t];for(n[t]?(d=n[t],we&Se(d)&&(d=n[t]={type:"simple",tokens:d})):d=r[t]?r[t]:t,me&Se(d)&&(d=c(d,n,r),d=n[d]||r[d]);d.extend;)oe=d.extend,le=n[oe]||r[oe],delete d.extend,le&&(we&Se(le)&&(le={type:"simple",tokens:le}),d=Te(le,d));if("undefined"==typeof d.type&&(d[se]("error")?(d.type="action",d.action=["error",d.error,!!d["in-context"]],delete d.error):d[se]("context-start")?(d.type="action",d.action=["context-start",d["context-start"],!!d["in-context"]],delete d["context-start"]):d[se]("context-end")?(d.type="action",d.action=["context-end",d["context-end"],!!d["in-context"]],delete d["context-end"]):d[se]("empty")?(d.type="action",d.action=["empty",d.empty,!!d["in-context"]],delete d.empty):d[se]("block-indent")?(d.type="action",d.action=["block-indent",d["block-indent"],!!d["in-context"]],delete d["block-indent"]):d[se]("indent")?(d.type="action",d.action=["indent",d.indent,!!d["in-context"]],delete d.indent):d[se]("outdent")?(d.type="action",d.action=["outdent",d.outdent,!!d["in-context"]],delete d.outdent):d[se]("unique")?(d.type="action",d.action=["unique",me&Se(d.unique)?["_DEFAULT_",d.unique]:d.unique,!!d["in-context"]],delete d.unique):d[se]("push")?(d.type="action",d.action=["push",d.push,!!d["in-context"]],delete d.push):d[se]("pop")?(d.type="action",d.action=["pop",d.pop,!!d["in-context"]],delete d.pop):d.sequence||d.all?(d.type="group",d.match="sequence",d.tokens=d.sequence||d.all,d.all?delete d.all:delete d.sequence):d.either?(d.type="group",d.match="either",d.tokens=d.either,delete d.either):d.zeroOrMore?(d.type="group",d.match="zeroOrMore",d.tokens=d.zeroOrMore,delete d.zeroOrMore):d.oneOrMore?(d.type="group",d.match="oneOrMore",d.tokens=d.oneOrMore,delete d.oneOrMore):d.zeroOrOne?(d.type="group",d.match="zeroOrOne",d.tokens=d.zeroOrOne,delete d.zeroOrOne):d.comment?(d.type="comment",d.tokens=d.comment,delete d.comment):d.block?(d.type="block",d.tokens=d.block,delete d.block):d["escaped-block"]?(d.type="escaped-block",d.tokens=d["escaped-block"],delete d["escaped-block"]):d.simple?(d.type="simple",d.tokens=d.simple,delete d.simple):d.type="simple"),y=d.type?re[d.type.toUpperCase().replace(ze,"")]:B,ie=d.msg||null,B&y){if(""===d.tokens)return ce=new rn(F,t,t,ie),h[t]=ce,ce;if(null===d.tokens)return ce=new rn(N,t,t,ie),h[t]=ce,ce;if(!1===d.tokens||0===d.tokens)return ce=new rn(V,t,t,ie),h[t]=ce,ce}if(ee&y||(d.tokens=Re(d.tokens)),ee&y)d[se]("action")?"error"===d.action[0]?d.action[0]=w:"context-start"===d.action[0]?d.action[0]=M:"context-end"===d.action[0]?d.action[0]=T:"empty"===d.action[0]?d.action[0]=A:"block-indent"===d.action[0]?d.action[0]=j:"indent"===d.action[0]?d.action[0]=q:"outdent"===d.action[0]?d.action[0]=R:"unique"===d.action[0]?d.action[0]=O:"push"===d.action[0]?d.action[0]=S:"pop"===d.action[0]&&(d.action[0]=$):d[se]("error")?d.action=[w,d.error,!!d["in-context"]]:d[se]("context-start")?d.action=[M,d["context-start"],!!d["in-context"]]:d[se]("context-end")?d.action=[T,d["context-end"],!!d["in-context"]]:d[se]("empty")?d.action=[A,d.empty,!!d["in-context"]]:d[se]("block-indent")?d.action=[j,d["block-indent"],!!d["in-context"]]:d[se]("indent")?d.action=[q,d.indent,!!d["in-context"]]:d[se]("outdent")?d.action=[R,d.outdent,!!d["in-context"]]:d[se]("unique")?d.action=[O,me&Se(d.unique)?["_DEFAULT_",d.unique]:d.unique,!!d["in-context"]]:d[se]("push")?d.action=[S,d.push,!!d["in-context"]]:d[se]("pop")&&(d.action=[$,d.pop,!!d["in-context"]]),b=d.action.slice(),ce=new on(ee,t,b,ie),h[t]=ce;else if(B&y)d.autocomplete&&f(d,t,k),v="undefined"==typeof d.combine?"\\b":d.combine,ce=new rn(B,t,u(t,d.tokens.slice(),e,v,l,s),ie),h[t]=ce;else if(z&y)G&y&&p(d,g),ce=new ln(y,t,i(t,d.tokens.slice(),e,l,s),ie,d.multiline,d.escape,o[t+".inside"]?1:0),h[t]=ce,d.interleave&&m.push(ce.clone());else if(W&y){for(x=d.tokens.slice(),de&Se(d.match)?ce=new sn(H,t,null,ie,d.match[0],d.match[1]):(_=ne[d.match.toUpperCase()],ce=Q===_?new sn(Q,t,null,ie,0,1):Y===_?new sn(Y,t,null,ie,0,E):J===_?new sn(J,t,null,ie,1,E):U&_?new sn(U,t,null,ie):new sn(K,t,null,ie)),h[t]=ce,C=[],D=0,I=x.length;I>D;D++)C=C.concat(a(x[D],e,n,r,o,l,s,h,m,g,k));ce.set(C)}else if(X&y){for(ce=je(d.tokens.slice()).slice(),P=[],D=0,I=ce.length;I>D;D++)P[D]=ce[D].slice(),ce[D]=new sn(X,t+"_NGRAM_"+D,null,ie);for(h[t]=ce,D=0,I=ce.length;I>D;D++){for(L=P[D],C=[],Z=0,te=L.length;te>Z;Z++)C=C.concat(a(L[Z],e,n,r,o,l,s,h,m,g,k));ce[D].set(C)}}return h[t]}function p(t,e){var n,r,o,l,s,u=je(t.tokens.slice());for(l=0,s=u.length;s>l;l++)n=u[l][0],r=u[l].length>1?u[l][1]:u[l][0],o=u[l].length>2?u[l][2]:"",null===r?(e.line=e.line||[],e.line.push(n)):(e.block=e.block||[],e.block.push([n,r,o]))}function f(t,e,n){var r=[].concat(Re(t.tokens)).map(function(t){return{word:t,meta:e}});n.autocomplete=(n.autocomplete||[]).concat(r)}function h(t){var e,n,r,o,l,s,u,i,c,p,f,h,m,g,k,d;if(t.__parsed)return t;for(f={},h={},m={},k={},d={},g=[],t=Me(t),e=t.RegExpID||null,t.RegExpID=null,delete t.RegExpID,s=t.Lex||{},t.Lex=null,delete t.Lex,u=t.Syntax||{},t.Syntax=null,delete t.Syntax,l=t.Style||{},o=t.Parser||[],r=o.length,n=[],i=0;r>i;i++)c=o[i],p=a(c,e,s,u,l,f,h,m,g,k,d)||null,p&&(de&Se(p)?n=n.concat(p):n.push(p));return t.Parser=n,t.cTokens=g,t.Style=l,t.Comments=k,t.Keywords=d,t.Extra=t.Extra||{},t.__parsed=1,t}function m(t,e){var n=new pn(h(t),{DEFAULT:e||g,ERROR:k}),r=function(t,e){return{startState:function(){return new cn},copyState:function(t){return t=t.clone(),t.line++,t},token:function(t,e){return n.getToken(t,e)},indent:function(r,o,l){return n.indent(r,o,l,t,e)},lineComment:n.LC,blockCommentStart:n.BCS,blockCommentEnd:n.BCE,blockCommentContinue:n.BCC,blockCommentLead:n.BCL,electricChars:n.Extra.electricChars||!1,fold:n.Extra.fold||!1}};return r.supportGrammarAnnotations=!1,r.validator=function(t){if(!r.supportGrammarAnnotations||!t||!t.length)return[];var e,o,l=[],s=n.parse(t,y);if(!s)return l;for(e in s)s.hasOwnProperty(e)&&(o=s[e],l.push({message:o[4]||"Syntax Error",severity:"error",from:CodeMirror.Pos(o[0],o[1]),to:CodeMirror.Pos(o[2],o[3])}));return l},r}var g,k,d=1,y=2,v=4,b=8,_=~v,x=~b,C=v|b,E=1/0,w=4,O=8,S=16,$=32,A=64,q=128,R=256,j=512,M=1024,T=2048,P=2,L=4,D=8,B=16,N=17,F=18,V=20,z=32,I=33,G=34,U=64,K=128,Z=K,H=256,Q=257,Y=258,J=259,W=512,X=1024,te=K|X,ee=2048,ne={EITHER:U,ALL:Z,SEQUENCE:K,ZEROORONE:Q,ZEROORMORE:Y,ONEORMORE:J,REPEATED:H},re={ACTION:ee,BLOCK:z,COMMENT:G,ESCAPEDBLOCK:I,SIMPLE:B,GROUP:W,NGRAM:X},oe=void 0,le="prototype",se="hasOwnProperty",ue="propertyIsEnumerable",ie=(Object.keys,Array[le],Object[le]),ce=(Function[le],ie.toString),ae=4,pe=5,fe=6,he=8,me=16,ge=17,ke=18,de=32,ye=64,ve=128,be=256,_e=512,xe=1024,Ce=2048,Ee=4096,we=me|de,Oe={"[object Number]":ae,"[object String]":me,"[object Array]":de,"[object RegExp]":be,"[object Date]":_e,"[object Function]":ve,"[object Object]":ye},Se=function(t){if(null===t)return xe;if(!0===t||!1===t||t instanceof Boolean)return he;if(oe===t)return Ce;var e=Oe[ce.call(t)]||Ee;return ae===e||t instanceof Number?isNaN(t)?fe:isFinite(t)?ae:pe:me===e||t instanceof String?1===t.length?ge:me:de===e||t instanceof Array?de:be===e||t instanceof RegExp?be:_e===e||t instanceof Date?_e:ve===e||t instanceof Function?ve:ye===e?ye:Ee},$e=function(){var t,e,n,r,o,l,s=arguments,u=s.length;for(t=s[0]||{},o=1;u>o;o++)if(e=s[o],ye===Se(e))for(r in e)e[se](r)&&e[ue](r)&&(n=e[r],l=Se(n),t[r]=ae&l?0+n:_e&l?new Date(n):we&l?n.slice():n);return t},Ae=Object.create,qe=function(t,e){var n,r=arguments.length,o="constructor";return 0===r?(t=Object,e={}):1===r?(e=t||{},t=Object):(t=t||Object,e=e||{}),e[se](o)||(e[o]=function(){}),n=e[o],delete e[o],n[le]=$e(Ae(t[le]),e),n[le][o]=n,n},Re=function(t,e){return e||de!==Se(t)?[t]:t},je=function(t,e){return t=Re(t),(e||de!==Se(t[0]))&&(t=[t]),t},Me=function(t,e){var n,r,o,l,s=Se(t),u=0;if(ae===Se(e)?e>0?(u=e,e=!0):e=!1:e=!1!==e,ye===s){r={};for(o in t)t[se](o)&&t[ue](o)&&(n=Se(t[o]),r[o]=ye===n?e?Me(t[o],u>0?u-1:e):t[o]:de===n?e?Me(t[o],u>0?u-1:e):t[o].slice():_e&n?new Date(t[o]):me&n?t[o].slice():ae&n?0+t[o]:t[o])}else if(de===s)for(l=t.length,r=new Array(l),o=0;l>o;o++)n=Se(t[o]),r[o]=ye===n?e?Me(t[o],u>0?u-1:e):t[o]:de===n?e?Me(t[o],u>0?u-1:e):t[o].slice():_e&n?new Date(t[o]):me&n?t[o].slice():ae&n?0+t[o]:t[o];else r=_e&s?new Date(t):me&s?t.slice():ae&s?0+t:t;return r},Te=function(){var t,e,n,r,o,l,s,u,i,c,a=arguments,p=a.length;if(1>p)return null;for(e=Me(a[0]),n=1;p>n;n++)if(t=a[n])for(r in t)if(t[se](r)&&t[ue](r))if(e[se](r)&&e[ue](r)){if(i=Se(e[r]),c=Se(t[r]),ye===i&&ye===c)e[r]=Te(e[r],t[r]);else if(de===i&&de===c){if(s=e[r],u=t[r],l=u.length,!l)continue;if(s.length)for(o=0;l>o;o++)0>s.indexOf(u[o])&&s.push(u[o]);else e[r]=u.slice()}}else e[r]=Me(t[r]);return e},Pe=/([.*+?^${}()|[\]\/\\\-])/g,Le=function(t){return t.replace(Pe,"\\$1")},De=function(t,e,n){var r,o,l,s,u,i=!0===n?0:1;for(me&Se(e)&&(e=[e,e],i=0),o=t.length,u="",r=0;o>r;)l=t.charAt(r),o>r+1&&"$"===l?(s=t.charCodeAt(r+1),36===s?(u+="$",r+=2):s>=48&&57>=s?(u+=e[i+s-48]||"",r+=2):(u+=l,r+=1)):(u+=l,r+=1);return u},Be=/^\s+|\s+$/g,Ne=String[le].trim?function(t){return t.trim()}:function(t){return t.replace(Be,"")},Fe=function(t,e){return e.length-t.length},Ve=/\r\n|\r|\n/g,ze=/[\-_]/g,Ie=/^([\[\]{}()*+?|'"]|\s)/,Ge=function(t,e){return me&Se(e)&&me&Se(t)&&e.length&&e.length<=t.length&&e==t.substr(0,e.length)},Ue=function(t,e,n){if(!t||ae===Se(t))return t;var r,o=e?e.length||0:0;if(o&&e===t.substr(0,o)){var l,s,u,r,i,c=t.substr(o),a=c.charAt(0),p="";for(r=c.length;r--&&(i=c.charAt(r),a!==i);)"i"===i.toLowerCase()&&(p="i");return l=c.substring(1,r),s="^("+l+")",n[s]||(u=new RegExp(s,p),n[s]=u),n[s]}return t},Ke=function(t,e){var n,r="";return me&Se(e)&&(r=e),n=t.sort(Fe).map(Le).join("|"),[new RegExp("^("+n+")"+r),1]},Ze=0,He=function(){return++Ze},Qe=function(t){return[t||"uuid",++Ze,(new Date).getTime()].join("_")},Ye=Math.max,Je=/^[\s\u00a0]+/,We=/[^\s\u00a0]/,Xe=qe({constructor:function(t){var e=this;e._=null,e.s=t?""+t:"",e.start=e.pos=0,e.lCP=e.lCV=0,e.lS=0},_:null,s:"",start:0,pos:0,lCP:0,lCV:0,lS:0,dispose:function(){var t=this;return t._=null,t.s=null,t.start=null,t.pos=null,t.lCP=null,t.lCV=null,t.lS=null,t},toString:function(){return this.s},sol:function(){return 0===this.pos},eol:function(){return this.pos>=this.s.length},end:function(){var t=this;return t.pos=t.s.length,t},mov:function(t){var e=this;return 0>t?e.pos=Ye(0,e.pos-t):e.pos+=t,e},bck:function(t){var e=this;return e.pos=Ye(0,t),e},sft:function(){var t=this;return t.start=t.pos,t},nxt:function(){var t,e=this,n=e.s;return e.posl;++l)s+=" "==t.charAt(l)?n-s%n:1;return s},Xe._=function(t){var e=new Xe;return e._=t,e.s=""+t.string,e.start=t.start,e.pos=t.pos,e.lCP=t.lastColumnPos,e.lCV=t.lastColumnValue,e.lS=t.lineStart,e};var tn,en,nn,rn,on,ln,sn,un=qe({constructor:function(t){this._=t||[]},_:null,dispose:function(){var t=this;return t._=null,t},toString:function(){return this._.slice().reverse().join("\n")},clone:function(){return new un(this._.slice())},isEmpty:function(){return 0>=this._.length},pos:function(){return this._.length},peek:function(t){var e=this,n=e._;if(t=arguments.length?t:-1,n.length){if(0>t&&0<=n.length+t)return n[n.length+t];if(t>=0&&tn;n++)if(r=l[n].match(t,e))return u?[n,r[1]]:r;return!1}}),en=qe(tn,{constructor:function(t,e){var n=this;n.type=D,n.name=t,n.pattern=e,n.pattern[0]=new nn(n.name+"_Start",n.pattern[0],!1)},match:function(t,e){var n,r,o,l,s,u=this,i=u.pattern,c=i[0],a=i[1];return(r=c.match(t,e))?(n=a[r[0]],l=Se(n),s=c.pattern[r[0]].ptype,be===s&&(ae===l?(o=r[1][n+1],n=new tn(u.name+"_End",o,o.length>1?me:ge)):me===l&&(o=De(n,r[1]),n=new tn(u.name+"_End",o,o.length>1?me:ge))),n):!1}}),rn=qe({constructor:function(t,e,n,r){var o=this;o.type=t||B,o.name=e,o.token=n,o.status=0,o.msg=r||null,o.$msg=null,o.$clone=null},name:null,type:null,token:null,status:0,msg:null,$msg:null,$clone:null,$id:null,dispose:function(){var t=this;return t.type=null,t.name=null,t.token=null,t.status=null,t.msg=null,t.$msg=null,t.$clone=null,t.$id=null,t},clone:function(){var t,e,n,r=this,o=r.$clone;if(t=new r.constructor,t.type=r.type,t.name=r.name,t.token=r.token,t.msg=r.msg,o&&o.length)for(e=0,n=o.length;n>e;e++)t[o[e]]=r[o[e]];return t},get:function(t,e){var n=this,r=n.token,o=n.type,l=n.name,s=null;if(n.$msg=null,e.token=null,V===o)return n.status=0,!0;if(N===o){if(t.spc(),t.eol())return l}else if(F===o)n.status&v&&t.spc()&&!t.eol()&&(n.status|=b),n.status&=_;else if(s=r.match(t))return e.token={name:l,value:t.cur(),token:s[1]},l;return!1},req:function(t){var e=this;return t?e.status|=v:e.status&=_,e},err:function(t,e,n,r,o){var l,s=this,u=s.name;return l=s.$msg?s.$msg:s.status&v?'Token "'+u+'" Expected':'Syntax Error: "'+u+'"',t&&t.errors&&(t.err[e+"_"+n+"_"+r+"_"+o+"_"+u]=[e,n,r,o,l]),l},toString:function(){var t=this;return["[","Token: ",t.name,", ",t.token?t.token.toString():null,"]"].join("")}}),on=qe(rn,{constructor:function(t,e,n,r){var o=this;o.type=t||ee,o.name=e,o.token=n,o.status=0,o.msg=r||null,o.$msg=null,o.$clone=null},get:function(t,e){var n,r,o,l,s,u,i,c,a,p,f,h=this,m=h.token||null,g=e.queu,k=e.symb,d=e.ctx,y=e.token,v=e.err,_=e.errors;if(h.status=0,h.$msg=null,m){if(s=h.msg,n=m[0],r=m[1],a=!!m[2],u=e.line,c=t.pos,i=y&&y.value?c-y.value.length:i-1,w===n)return _&&(h.$msg=s?s:"Error",p=u+"_"+i+"_"+u+"_"+c+"_"+h.name,v[p]=[u,i,u,c,h.err()]),h.status|=b,!1;if(A===n)a?d.length&&(d[0].queu.length=0):g.length=0;else if(M===n)d.unshift({symb:{},queu:[]});else if(T===n)d.length&&d.shift();else if(O===n){if(a){if(!d.length)return!0;k=d[0].symb}if(y){if(o=r[1],l=r[0],o=ae===Se(o)?y.token[o]:De(o,y.token,!0),k[se](l)||(k[l]={}),k[l][se](o))return _&&(h.$msg=s?De(s,o,!0):'Duplicate "'+o+'"',f=h.err(),p=k[l][o][0]+"_"+k[l][o][1]+"_"+k[l][o][2]+"_"+k[l][o][3]+"_"+h.name,v[p]=[k[l][o][0],k[l][o][1],k[l][o][2],k[l][o][3],f],p=u+"_"+i+"_"+u+"_"+c+"_"+h.name,v[p]=[u,i,u,c,f]),h.status|=b,!1;k[l][o]=[u,i,u,c]}}else if($===n){if(a){if(!d.length)return!0;g=d[0].queu}if(r){if(y&&(r=ae===Se(r)?y.token[r]:De(r,y.token)),!g.length||r!==g[0][0])return _&&(g.length?(h.$msg=s?De(s,[g[0][0],r],!0):'Tokens do not match "'+g[0][0]+'","'+r+'"',f=h.err(),p=g[0][1]+"_"+g[0][2]+"_"+g[0][3]+"_"+g[0][4]+"_"+h.name,v[p]=[g[0][1],g[0][2],g[0][3],g[0][4],f],p=u+"_"+i+"_"+u+"_"+c+"_"+h.name,v[p]=[u,i,u,c,f]):(h.$msg=s?De(s,["",r],!0):'Token does not match "'+r+'"',f=h.err(),p=u+"_"+i+"_"+u+"_"+c+"_"+h.name,v[p]=[u,i,u,c,f])),g.shift(),h.status|=b,!1;g.shift()}else g.shift()}else if(S===n&&r){if(a){if(!d.length)return!0;g=d[0].queu}y&&(r=ae===Se(r)?y.token[r]:De(r,y.token)),g.unshift([r,u,i,u,c])}}return!0}}),ln=qe(rn,{constructor:function(t,e,n,r,o,l,s){var u=this;u.type=t,u.name=e,u.token=n,u.status=0,u.msg=r||null,u.mline="undefined"==typeof o?1:o,u.esc=l||"\\",u.inter=s,u.$msg=null,u.$clone=["mline","esc","inter"]},inter:0,mline:0,esc:null,get:function(t,e){var n,r,o,l,s,u,i,c,a,p,f,h,m,g,k=this,d=0,y=0,v="",b=k.mline,x=k.token,C=k.name,E=k.type,w=k.inter,O=w?C+".inside":C,S=0,$=I===E,A=k.esc,q="",R="",j="",M="",T=e.stack;if(k.$msg=k.msg||null,m=e.line,g=t.pos,G===E&&(k.status&=_),s=0,e.block&&e.block.name===C?(y=1,n=e.block.end,s=1,u=O,p=e.block.s,h=e.block.i,f=e.block.e,q=e.block._s,R=e.block._i,M=""):!e.block&&(n=x.match(t))&&(y=1,p=[m,g],h=[[m,t.pos],[m,t.pos]],f=[m,t.pos],q=t.cur(),R="",M="",j="",e.block={name:C,end:n,s:p,i:h,e:f,_s:q,_i:R,_e:j},u=C),y){if(o=T.pos(),l=xe===n.type,w){if(s&&l&&t.sol())return k.status&=_,e.current=null,e.block=null,!1;if(!s)return T.pushAt(o,k.clone(),"$id",C),u}if(d=n.match(t),r=b,a=0,d)u=l?O:C,j=t.cur().slice(M.length);else for(c=t.pos;!t.eol();){if(i=t.pos,!($&&S||!n.match(t))){w&&t.pos>i&&i>c?(u=O,t.bck(i),a=1):(u=C,d=1),j=t.cur().slice(M.length);break}v=t.nxt(),M+=v,S=!S&&v===A}return r=b||$&&S,h[1]=[m,i],f=[m,t.pos],d||!r&&!a?e.block=null:(e.block.i=h,e.block.e=f,e.block._i+=M,e.block._e=j,e.stack.pushAt(o,k.clone(),"$id",C)),e.token={name:C,value:t.cur(),token:[q+R+M+j,R+M]},u}return!1}}),sn=qe(rn,{constructor:function(t,e,n,r,o,l){var s=this;s.type=t?t:H,s.name=e||null,s.token=null,s.status=0,s.msg=r||null,s.min=o||0,s.max=l||E,s.found=0,s.$msg=null,s.$clone=["min","max","found"],n&&s.set(n)},min:0,max:1,found:0,set:function(t){return t&&(this.token=Re(t)),this},get:function(t,e){var n,r,o,l,s,u,i,c,a,p,f,h,m,g,k=this,d=k.type,y=k.token,C=y.length;if(k.$msg=k.msg||null,k.status&=x,f=t.pos,l=e.stack,U===d){for(a=0,p=0,k.status|=v,s=[],n=0;C>n;n++){if(r=y[n].clone().req(1),o=r.get(t,e),r.status&v&&(a++,s.push(r.err())),!1!==o)return o;r.status&b&&(p++,t.bck(f))}return a>0?k.status|=v:k.status&=_,C===p&&a>0?k.status|=b:k.status&=x,k.status&&!k.$msg&&s.length&&(k.$msg=s.join(" | ")),!1}if(te&d){if(g=d&K?1:0,g?k.status|=v:k.status&=_,h=l.pos(),r=y[0].clone().req(g),o=r.get(t,e),m=k.name+"_"+He(),!1!==o){if(!0!==o)for(n=C-1;n>0;n--)l.pushAt(h+C-n-1,y[n].clone().req(1),"$id",m);return o}return r.status&b?(g?k.status|=b:k.status&=x,t.bck(f)):g&&r.status&v&&(k.status|=b),k.status&&!k.$msg&&(k.$msg=r.err()),!1}for(a=0,u=k.found,i=k.min,c=k.max,k.status&=_,h=l.pos(),m=k.name+"_"+He(),s=[],n=0;C>n;n++){if(r=y[n].clone().req(1),o=r.get(t,e),!1!==o){if(++u,c>=u)return k.found=u,l.pushAt(h,k.clone(),"$id",m),k.found=0,o;break}r.status&v&&(a++,s.push(r.err())),r.status&b&&t.bck(f)}return i>u?k.status|=v:k.status&=_,u>c||i>u&&a>0?k.status|=b:k.status&=x,k.status&&!k.$msg&&s.length&&(k.$msg=s.join(" | ")),!1}});var an=CodeMirror||{Pass:{toString:function(){return"CodeMirror.Pass"}}};g=null,k="error";{var pn=qe({constructor:function(t,e){var n=this;n.Extra=t.Extra||{},n.LC=t.Comments.line?t.Comments.line[0]:null,n.BCS=t.Comments.block?t.Comments.block[0][0]:null,n.BCE=t.Comments.block?t.Comments.block[0][1]:null,n.BCC=n.BCL=t.Comments.block?t.Comments.block[0][2]:null,n.DEF=e.DEFAULT,n.ERR=t.Style.error||e.ERROR,n.Keywords=t.Keywords.autocomplete||null,n.Tokens=t.Parser||[],n.cTokens=t.cTokens.length?t.cTokens:null,n.Style=t.Style},Extra:null,LC:null,BCS:null,BCE:null,BCL:null,BCC:null,ERR:null,DEF:null,Keywords:null,cTokens:null,Tokens:null,Style:null,dispose:function(){var t=this;return t.Extra=null,t.LC=null,t.BCS=null,t.BCE=null,t.BCL=null,t.BCC=null,t.ERR=null,t.DEF=null,t.Keywords=null,t.cTokens=null,t.Tokens=null,t.Style=null,t},parse:function(t,e){t=t||"";var n,r,o,l,s,u,i,c,a=this,p=t.split(Ve),f=p.length;if(e=e||d,i=!!(e&y),c=!!(e&d),l=new cn(0,0,i),l.parseAll=1,c)for(r=[],n=0;f>n;n++){for(l.line=n,s=new Xe(p[n]),o=[];!s.eol();)o.push(a.getToken(s,l));r.push(o)}else for(n=0;f>n;n++)for(l.line=n,s=new Xe(p[n]);!s.eol();)a.getToken(s,l);return u=c&&i?{tokens:r,errors:l.err}:c?r:l.err,s.dispose(),l.dispose(),u},getToken:function(t,e){var n,r,o,l,s,u,i,c,a=this,p=a.cTokens,f=a.Tokens,h=f.length,m=!!e.parseAll,g=a.Style,k=a.DEF,d=a.ERR;if(t=m?t:Xe._(t),u=e.stack,t.sol()&&!u.isEmpty()&&N===u.peek().type&&u.pop(),(u.isEmpty()||F!==u.peek().type)&&t.spc())return m?{value:t.cur(1),type:k}:t.upd()&&k;for(c=e.line;!u.isEmpty()&&!t.eol();){if(p)for(r=0;rn;n++)if(i=t.pos,l=f[n],o=l.get(t,e),!1!==o){if(!0!==o){for(o=g[o]||k;!u.isEmpty()&&ee===u.peek().type;)s=u.pop(),s.get(t,e);return m?{value:t.cur(1),type:o}:t.upd()&&o}}else if(l.status&C)return u.empty("$id",l.$id),t.nxt(),o=d,l.err(e,c,i,c,t.pos),m?{value:t.cur(1),type:o}:t.upd()&&o;return t.nxt(),m?{value:t.cur(1),type:k}:t.upd()&&k},indent:function(t,e,n,r){var o=(r.indentUnit||4,an.Pass);return o}});t.CodeMirrorGrammar={VERSION:"2.1.0",clone:Me,extend:Te,parse:h,getMode:m}}return t.CodeMirrorGrammar}); \ No newline at end of file +**/!function(t,e,n){"use strict";var r,o="object"==typeof module&&module.exports,s="function"==typeof define&&define.amd;o?module.exports=(module.$deps=module.$deps||{})[e]=module.$deps[e]||n.call(t,{NODE:module})||1:s&&"function"==typeof require&&"function"==typeof require.specified&&require.specified(e)?define(e,["require","exports","module"],function(e,r,o){return n.call(t,{AMD:o})}):e in t||(t[e]=r=n.call(t,{})||1)&&s&&define(e,[],function(){return r})}(this,"CodeMirrorGrammar",function(t){"use strict";function e(t,e){var n=this,r=n.pattern,o=t.s.charAt(t.pos)||null;return o&&r===o?(!1!==e&&t.mov(1),[n.key,o]):!1}function n(t,e){var n=this,r=n.pattern,o=t.s.charAt(t.pos)||null;return o&&-10?!1:(!1!==e&&t.mov(n[o[1]||0].length),[r.key,o[1]>0?n[o[1]]:n])}function s(t,e){var n=this;return!1!==e&&t.end(),[n.key,""]}function l(t,e,n,r){var o=qe(e);if(he===o)return e;if(r[t])return r[t];n=n||0;var s,l=0;return e&&e.isCharList&&(l=1,delete e.isCharList),s=we&o?new rn(t,e,we,n):ye===o?new rn(t,e,ye,n):de&o?new rn(t,e,l?ve:de,n):(xe|be)&o?new rn(t,e,xe,n):e,r[t]=s}function u(t,e,n,r,o,s){if(s[t])return s[t];var i,a,c,p,f,h,m,g=0,k=0,d=1;if(i=Te(e),c=i.length,1===c)m=l(t,He(i[0],n,o),0,s);else if(c>1){for(p=(c>>>1)+1,a=0;p>=a;a++)f=qe(i[a]),h=qe(i[c-1-a]),(ye!==f||ye!==h)&&(d=0),be&f||be&h?g=1:(Ze(i[a],n)||Ze(i[c-1-a],n))&&(k=1);if(!d||r&&de&qe(r))if(!r||g||k)if(g||k){for(a=0;c>a;a++)i[a]=be&qe(i[a])?u(t+"_"+a,i[a],n,r,o,s):l(t+"_"+a,He(i[a],n,o),a,s);m=c>1?new sn(t,i):i[0]}else{for(i=i.sort(Ie),a=0;c>a;a++)i[a]=l(t+"_"+a,He(i[a],n,o),a,s);m=c>1?new sn(t,i):i[0]}else m=l(t,Qe(i,r),0,s);else i=i.slice().join(""),i.isCharList=1,m=l(t,i,0,s)}return s[t]=m}function i(t,e,n,r,o){if(o[t])return o[t];var s,u,i,a,c,p,f;for(a=[],c=[],s=Le(e),u=0,i=s.length;i>u;u++)p=l(t+"_0_"+u,He(s[u][0],n,r),u,o),f=s[u].length>1?xe!==p.ptype||de!==qe(s[u][1])||Ze(s[u][1],n)?l(t+"_1_"+u,He(s[u][1],n,r),u,o):s[u][1]:p,a.push(p),c.push(f);return o[t]=new on(t,[a,c])}function a(t,e,n){var r,o,s,l,u,i,a,c,p,f,h,m;if(i=new String(ze(t)),i.pos=0,1===i.length)f=""+t,e[f]||(e[f]={type:"simple",tokens:t}),t=f;else{for(r=[],o=[],s="",h=[];i.posu[0]&&(u[0]=0),2>u.length?u.push(u[0]):u[1]=u[1].length?parseInt(u[1],10)||E:E,0>u[1]&&(u[1]=0),p=o.pop(),f=""+p+["{",u[0],",",isFinite(u[1])?u[1]:"","}"].join(""),n[f]||(n[f]={type:"group",match:[u[0],u[1]],tokens:[p]}),o.push(f)}else{if("}"===c)continue;if("["===c){for(l="";i.pos1?(f=""+o.join(" "),n[f]||(n[f]={type:"group",match:"sequence",tokens:o}),r.push(f)):o.length&&r.push(o[0]),o=[]):"("===c?(h.push([o,r,s]),o=[],r=[],s=""):")"===c&&(o.length>1?(f=""+o.join(" "),n[f]||(n[f]={type:"group",match:"sequence",tokens:o}),r.push(f)):o.length&&r.push(o[0]),o=[],r.length>1?(f=""+r.join(" | "),n[f]||(n[f]={type:"group",match:"either",tokens:r})):r.length&&(f=r[0]),r=[],m=h.pop(),o=m[0],r=m[1],s=m[2],p=f,f="("+p+")",n[f]||(n[f]=Pe(e[p]||n[p])),o.push(f))}}else s+=c;s.length&&(e[s]||n[s]||(e[s]={type:"simple",tokens:s}),o.push(s)),s="",o.length>1?(f=""+o.join(" "),n[f]||(n[f]={type:"group",match:"sequence",tokens:o}),r.push(f)):o.length&&r.push(o[0]),o=[],r.length>1?(f=""+r.join(" | "),n[f]||(n[f]={type:"group",match:"either",tokens:r}),t=f):r.length&&(t=r[0]),r=[]}return t}function c(t,e,n,r,o,s,l,h,m,g,k){var d,y,v,b,_,C,x,T,L,P,B,V,K,Y,re,ue,ie,ce=null;if(ie=null,D===t)return new ln(D,D,t,ie);if(N===t)return new ln(N,N,t,ie);if(F===t||null===t)return new ln(F,F,t,ie);if(""===t)return new ln(I,"NONSPACE",t,ie);if(!1===t||0===t)return new ln(z,"EMPTY",t,ie);if(be&qe(t)&&(ue=t,t="NGRAM_"+ue.join("_"),r[t]||(r[t]={type:"ngram",tokens:ue})),t=""+t,h[t])return h[t];for(n[t]?(d=n[t],Se&qe(d)&&(d=n[t]={type:"simple",tokens:d})):d=r[t]?r[t]:t,de&qe(d)&&(d=a(d,n,r),d=n[d]||r[d]);d.extend;)Y=d.extend,re=n[Y]||r[Y],delete d.extend,re&&(Se&qe(re)&&(re={type:"simple",tokens:re}),d=De(re,d));if("undefined"==typeof d.type&&(d[ae]("error")?(d.type="action",d.action=["error",d.error,!!d["in-context"]],delete d.error):d[ae]("context-start")?(d.type="action",d.action=["context-start",d["context-start"],!!d["in-context"]],delete d["context-start"]):d[ae]("context-end")?(d.type="action",d.action=["context-end",d["context-end"],!!d["in-context"]],delete d["context-end"]):d[ae]("empty")?(d.type="action",d.action=["empty",d.empty,!!d["in-context"]],delete d.empty):d[ae]("indent")?(d.type="action",d.action=["indent",d.indent,!!d["in-context"]],delete d.indent):d[ae]("outdent")?(d.type="action",d.action=["outdent",d.outdent,!!d["in-context"]],delete d.outdent):d[ae]("unique")?(d.type="action",d.action=["unique",de&qe(d.unique)?["_DEFAULT_",d.unique]:d.unique,!!d["in-context"]],delete d.unique):d[ae]("push")?(d.type="action",d.action=["push",d.push,!!d["in-context"]],delete d.push):d[ae]("pop")?(d.type="action",d.action=["pop",d.pop,!!d["in-context"]],delete d.pop):d.sequence||d.all?(d.type="group",d.match="sequence",d.tokens=d.sequence||d.all,d.all?delete d.all:delete d.sequence):d.either?(d.type="group",d.match="either",d.tokens=d.either,delete d.either):d.zeroOrMore?(d.type="group",d.match="zeroOrMore",d.tokens=d.zeroOrMore,delete d.zeroOrMore):d.oneOrMore?(d.type="group",d.match="oneOrMore",d.tokens=d.oneOrMore,delete d.oneOrMore):d.zeroOrOne?(d.type="group",d.match="zeroOrOne",d.tokens=d.zeroOrOne,delete d.zeroOrOne):d.comment?(d.type="comment",d.tokens=d.comment,delete d.comment):d.block?(d.type="block",d.tokens=d.block,delete d.block):d["escaped-block"]?(d.type="escaped-block",d.tokens=d["escaped-block"],delete d["escaped-block"]):d.simple?(d.type="simple",d.tokens=d.simple,delete d.simple):d.type="simple"),y=d.type?le[d.type.toUpperCase().replace(Ue,"")]:G,ie=d.msg||null,G&y){if(D===d.tokens)return ce=new ln(D,t,t,ie),h[t]=ce,ce;if(N===d.tokens)return ce=new ln(N,t,t,ie),h[t]=ce,ce;if(F===d.tokens||null===d.tokens)return ce=new ln(F,t,t,ie),h[t]=ce,ce;if(""===d.tokens)return ce=new ln(I,t,t,ie),h[t]=ce,ce;if(!1===d.tokens||0===d.tokens)return ce=new ln(z,t,t,ie),h[t]=ce,ce}if(oe&y||(d.tokens=Te(d.tokens)),oe&y)d[ae]("action")?"error"===d.action[0]?d.action[0]=w:"context-start"===d.action[0]?d.action[0]=j:"context-end"===d.action[0]?d.action[0]=M:"empty"===d.action[0]?d.action[0]=A:"indent"===d.action[0]?d.action[0]=q:"outdent"===d.action[0]?d.action[0]=R:"unique"===d.action[0]?d.action[0]=$:"push"===d.action[0]?d.action[0]=O:"pop"===d.action[0]&&(d.action[0]=S):d[ae]("error")?d.action=[w,d.error,!!d["in-context"]]:d[ae]("context-start")?d.action=[j,d["context-start"],!!d["in-context"]]:d[ae]("context-end")?d.action=[M,d["context-end"],!!d["in-context"]]:d[ae]("empty")?d.action=[A,d.empty,!!d["in-context"]]:d[ae]("indent")?d.action=[q,d.indent,!!d["in-context"]]:d[ae]("outdent")?d.action=[R,d.outdent,!!d["in-context"]]:d[ae]("unique")?d.action=[$,de&qe(d.unique)?["_DEFAULT_",d.unique]:d.unique,!!d["in-context"]]:d[ae]("push")?d.action=[O,d.push,!!d["in-context"]]:d[ae]("pop")&&(d.action=[S,d.pop,!!d["in-context"]]),b=d.action.slice(),ce=new un(oe,t,b,ie),h[t]=ce;else if(G&y)d.autocomplete&&f(d,t,k),v="undefined"==typeof d.combine?"\\b":d.combine,ce=new ln(G,t,u(t,d.tokens.slice(),e,v,s,l),ie),h[t]=ce;else if(U&y)Z&y&&p(d,g),ce=new an(y,t,i(t,d.tokens.slice(),e,s,l),ie,d.multiline,d.escape,o[t+".inside"]?1:0),h[t]=ce,d.interleave&&m.push(ce.clone());else if(ee&y){for(C=d.tokens.slice(),be&qe(d.match)?ce=new cn(J,t,null,ie,d.match[0],d.match[1]):(_=se[d.match.toUpperCase()],ce=W===_?new cn(W,t,null,ie,0,1):X===_?new cn(X,t,null,ie,0,E):te===_?new cn(te,t,null,ie,1,E):H&_?new cn(H,t,null,ie):new cn(Q,t,null,ie)),h[t]=ce,x=[],P=0,B=C.length;B>P;P++)x=x.concat(c(C[P],e,n,r,o,s,l,h,m,g,k));ce.set(x)}else if(ne&y){for(ce=Le(d.tokens.slice()).slice(),T=[],P=0,B=ce.length;B>P;P++)T[P]=ce[P].slice(),ce[P]=new cn(ne,t+"_NGRAM_"+P,null,ie);for(h[t]=ce,P=0,B=ce.length;B>P;P++){for(L=T[P],x=[],V=0,K=L.length;K>V;V++)x=x.concat(c(L[V],e,n,r,o,s,l,h,m,g,k));ce[P].set(x)}}return h[t]}function p(t,e){var n,r,o,s,l,u=Le(t.tokens.slice());for(s=0,l=u.length;l>s;s++)n=u[s][0],r=u[s].length>1?u[s][1]:u[s][0],o=u[s].length>2?u[s][2]:"",null===r?(e.line=e.line||[],e.line.push(n)):(e.block=e.block||[],e.block.push([n,r,o]))}function f(t,e,n){var r=[].concat(Te(t.tokens)).map(function(t){return{word:t,meta:e}});n.autocomplete=(n.autocomplete||[]).concat(r)}function h(t){var e,n,r,o,s,l,u,i,a,p,f,h,m,g,k,d,y,v,b;if(t.__parsed)return t;g={},k={},d={},v={},b={},y=[],t=Pe(t),e=t.RegExpID||null,t.RegExpID=null,delete t.RegExpID,l=t.Lex||{},t.Lex=null,delete t.Lex,u=t.Syntax||{},t.Syntax=null,delete t.Syntax,s=t.Style||{};for(f in l)l[ae](f)&&(h=f.split(":"),m=h[1]&&ze(h[1]).length?ze(h[1]).toLowerCase():null,h=h[0],h!==f&&(l[h]=l[f],delete l[f],m&&(_e&qe(l[h])?l[h].type||(l[h].type=m):l[h]={type:m,tokens:l[h]})));for(f in u)u[ae](f)&&(h=f.split(":"),m=h[1]&&ze(h[1]).length?ze(h[1]).toLowerCase():null,h=h[0],h!==f&&(u[h]=u[f],delete u[f],m&&(_e&qe(u[h])?u[h].type||(u[h].type=m):u[h]={type:m,tokens:u[h]})));for(o=t.Parser||[],r=o.length,n=[],i=0;r>i;i++)a=o[i],p=c(a,e,l,u,s,g,k,d,y,v,b)||null,p&&(be&qe(p)?n=n.concat(p):n.push(p));return t.Parser=n,t.cTokens=y,t.Style=s,t.Comments=v,t.Keywords=b,t.Extra=t.Extra||{},t.__parsed=1,t}function m(t,e){var n=new mn(h(t),{DEFAULT:e||g,ERROR:k}),r=function(t,e){return{startState:function(){var t=new fn;return t},copyState:function(t){var e=t.clone();return e},token:function(t,e){return n.getToken(t,e)},indent:function(r,o,s){return n.indent(r,o,s,t,e)},lineComment:n.LC,blockCommentStart:n.BCS,blockCommentEnd:n.BCE,blockCommentContinue:n.BCC,blockCommentLead:n.BCL,electricChars:n.Extra.electricChars||!1,fold:n.Extra.fold||!1}};return r.supportGrammarAnnotations=!1,r.validator=function(t){if(!r.supportGrammarAnnotations||!t||!t.length)return[];var e,o,s=[],l=n.parse(t,y);if(!l)return s;for(e in l)l.hasOwnProperty(e)&&(o=l[e],s.push({message:o[4]||"Syntax Error",severity:"error",from:CodeMirror.Pos(o[0],o[1]),to:CodeMirror.Pos(o[2],o[3])}));return s},r}var g,k,d=1,y=2,v=4,b=8,_=~v,C=~b,x=v|b,E=1/0,w=4,$=8,O=16,S=32,A=64,q=128,R=256,j=512,M=1024,T=2,L=4,P=8,D=4,B=8,N=16,F=32,V=D|N,z=64,I=128,G=256,U=512,K=513,Z=514,H=1024,Q=2048,Y=Q,J=4096,W=4097,X=4098,te=4099,ee=8192,ne=16384,re=Q|ne,oe=32768,se={EITHER:H,ALL:Y,SEQUENCE:Q,ZEROORONE:W,ZEROORMORE:X,ONEORMORE:te,REPEATED:J},le={ACTION:oe,BLOCK:U,COMMENT:Z,ESCAPEDBLOCK:K,SIMPLE:G,GROUP:ee,NGRAM:ne},ue=void 0,ie="prototype",ae="hasOwnProperty",ce="propertyIsEnumerable",pe=(Object.keys,Array[ie],Object[ie]),fe=(Function[ie],pe.toString),he=4,me=5,ge=6,ke=8,de=16,ye=17,ve=18,be=32,_e=64,Ce=128,xe=256,Ee=512,we=1024,$e=2048,Oe=4096,Se=de|be,Ae={"[object Number]":he,"[object String]":de,"[object Array]":be,"[object RegExp]":xe,"[object Date]":Ee,"[object Function]":Ce,"[object Object]":_e},qe=function(t){if(null===t)return we;if(!0===t||!1===t||t instanceof Boolean)return ke;if(ue===t)return $e;var e=Ae[fe.call(t)]||Oe;return he===e||t instanceof Number?isNaN(t)?ge:isFinite(t)?he:me:de===e||t instanceof String?1===t.length?ye:de:be===e||t instanceof Array?be:xe===e||t instanceof RegExp?xe:Ee===e||t instanceof Date?Ee:Ce===e||t instanceof Function?Ce:_e===e?_e:Oe},Re=function(){var t,e,n,r,o,s,l=arguments,u=l.length;for(t=l[0]||{},o=1;u>o;o++)if(e=l[o],_e===qe(e))for(r in e)e[ae](r)&&e[ce](r)&&(n=e[r],s=qe(n),t[r]=he&s?0+n:Ee&s?new Date(n):Se&s?n.slice():n);return t},je=Object.create,Me=function(t,e){var n,r=arguments.length,o="constructor";return 0===r?(t=Object,e={}):1===r?(e=t||{},t=Object):(t=t||Object,e=e||{}),e[ae](o)||(e[o]=function(){}),n=e[o],delete e[o],n[ie]=Re(je(t[ie]),e),n[ie][o]=n,n},Te=function(t,e){return e||be!==qe(t)?[t]:t},Le=function(t,e){return t=Te(t),(e||be!==qe(t[0]))&&(t=[t]),t},Pe=function(t,e){var n,r,o,s,l=qe(t),u=0;if(he===qe(e)?e>0?(u=e,e=!0):e=!1:e=!1!==e,_e===l){r={};for(o in t)t[ae](o)&&t[ce](o)&&(n=qe(t[o]),r[o]=_e===n?e?Pe(t[o],u>0?u-1:e):t[o]:be===n?e?Pe(t[o],u>0?u-1:e):t[o].slice():Ee&n?new Date(t[o]):de&n?t[o].slice():he&n?0+t[o]:t[o])}else if(be===l)for(s=t.length,r=new Array(s),o=0;s>o;o++)n=qe(t[o]),r[o]=_e===n?e?Pe(t[o],u>0?u-1:e):t[o]:be===n?e?Pe(t[o],u>0?u-1:e):t[o].slice():Ee&n?new Date(t[o]):de&n?t[o].slice():he&n?0+t[o]:t[o];else r=Ee&l?new Date(t):de&l?t.slice():he&l?0+t:t;return r},De=function(){var t,e,n,r,o,s,l,u,i,a,c=arguments,p=c.length;if(1>p)return null;for(e=Pe(c[0]),n=1;p>n;n++)if(t=c[n])for(r in t)if(t[ae](r)&&t[ce](r))if(e[ae](r)&&e[ce](r)){if(i=qe(e[r]),a=qe(t[r]),_e===i&&_e===a)e[r]=De(e[r],t[r]);else if(be===i&&be===a){if(l=e[r],u=t[r],s=u.length,!s)continue;if(l.length)for(o=0;s>o;o++)0>l.indexOf(u[o])&&l.push(u[o]);else e[r]=u.slice()}}else e[r]=Pe(t[r]);return e},Be=/([.*+?^${}()|[\]\/\\\-])/g,Ne=function(t){return t.replace(Be,"\\$1")},Fe=function(t,e,n){var r,o,s,l,u,i=!0===n?0:1;for(de&qe(e)&&(e=[e,e],i=0),o=t.length,u="",r=0;o>r;)s=t.charAt(r),o>r+1&&"$"===s?(l=t.charCodeAt(r+1),36===l?(u+="$",r+=2):l>=48&&57>=l?(u+=e[i+l-48]||"",r+=2):(u+=s,r+=1)):(u+=s,r+=1);return u},Ve=/^\s+|\s+$/g,ze=String[ie].trim?function(t){return t.trim()}:function(t){return t.replace(Ve,"")},Ie=function(t,e){return e.length-t.length},Ge=/\r\n|\r|\n/g,Ue=/[\-_]/g,Ke=/^([\[\]{}()*+?|'"]|\s)/,Ze=function(t,e){return de&qe(e)&&de&qe(t)&&e.length&&e.length<=t.length&&e==t.substr(0,e.length)},He=function(t,e,n){if(!t||he===qe(t))return t;var r,o=e?e.length||0:0;if(o&&e===t.substr(0,o)){var s,l,u,r,i,a=t.substr(o),c=a.charAt(0),p="";for(r=a.length;r--&&(i=a.charAt(r),c!==i);)"i"===i.toLowerCase()&&(p="i");return s=a.substring(1,r),l="^("+s+")",n[l]||(u=new RegExp(l,p),n[l]=u),n[l]}return t},Qe=function(t,e){var n,r="";return de&qe(e)&&(r=e),n=t.sort(Ie).map(Ne).join("|"),[new RegExp("^("+n+")"+r),1]},Ye=0,Je=function(){return++Ye},We=function(t){return[t||"uuid",++Ye,(new Date).getTime()].join("_")},Xe=Math.max,tn=/^[\s\u00a0]+/,en=/[^\s\u00a0]/,nn=Me({constructor:function(t){var e=this;e._=null,e.s=t?""+t:"",e.start=e.pos=0,e.lCP=e.lCV=0,e.lS=0},_:null,s:"",start:0,pos:0,lCP:0,lCV:0,lS:0,dispose:function(){var t=this;return t._=null,t.s=null,t.start=null,t.pos=null,t.lCP=null,t.lCV=null,t.lS=null,t},toString:function(){return this.s},sol:function(){return 0===this.pos},eol:function(){return this.pos>=this.s.length},end:function(){var t=this;return t.pos=t.s.length,t},mov:function(t){var e=this;return 0>t?e.pos=Xe(0,e.pos-t):e.pos+=t,e},bck:function(t){var e=this;return e.pos=Xe(0,t),e},sft:function(){var t=this;return t.start=t.pos,t},nxt:function(){var t,e=this,n=e.s;return e.poss;++s)l+=" "==t.charAt(s)?n-l%n:1;return l},nn._=function(t){var e=new nn;return e._=t,e.s=""+t.string,e.start=t.start,e.pos=t.pos,e.lCP=t.lastColumnPos,e.lCV=t.lastColumnValue,e.lS=t.lineStart,e};var rn,on,sn,ln,un,an,cn,pn=Me({constructor:function(t){this._=t||[]},_:null,dispose:function(){var t=this;return t._=null,t},toString:function(){return this._.slice().reverse().join("\n")},clone:function(){return new pn(this._.slice())},isEmpty:function(){return 0>=this._.length},pos:function(){return this._.length},peek:function(t){var e=this,n=e._;if(t=arguments.length?t:-1,n.length){if(0>t&&0<=n.length+t)return n[n.length+t];if(t>=0&&tn;n++)if(r=s[n].match(t,e))return u?[n,r[1]]:r;return!1}}),on=Me(rn,{constructor:function(t,e){var n=this;n.type=P,n.name=t,n.pattern=e,n.pattern[0]=new sn(n.name+"_Start",n.pattern[0],!1)},match:function(t,e){var n,r,o,s,l,u=this,i=u.pattern,a=i[0],c=i[1];return(r=a.match(t,e))?(n=c[r[0]],s=qe(n),l=a.pattern[r[0]].ptype,xe===l&&(he===s?(o=r[1][n+1],n=new rn(u.name+"_End",o,o.length>1?de:ye)):de===s&&(o=Fe(n,r[1]),n=new rn(u.name+"_End",o,o.length>1?de:ye))),n):!1}}),ln=Me({constructor:function(t,e,n,r){var o=this;o.type=t||G,o.name=e,o.token=n,o.status=0,o.msg=r||null,o.$msg=null,o.$clone=null,D===o.name?o.name="":N===o.name?o.name="":F===o.name?o.name="":B===o.name?o.name="":z===o.name?o.name="":I===o.name&&(o.name="")},name:null,type:null,token:null,status:0,msg:null,$msg:null,$clone:null,$id:null,dispose:function(){var t=this;return t.type=null,t.name=null,t.token=null,t.status=null,t.msg=null,t.$msg=null,t.$clone=null,t.$id=null,t},clone:function(){var t,e,n,r=this,o=r.$clone;if(t=new r.constructor,t.type=r.type,t.name=r.name,t.token=r.token,t.msg=r.msg,o&&o.length)for(e=0,n=o.length;n>e;e++)t[o[e]]=r[o[e]];return t},get:function(t,e){var n=this,r=n.token,o=n.type,s=n.name,l=null;if(n.$msg=n.msg||null,e.token=null,D===o){if(D&e.status&&t.sol())return!0}else if(N===o){if(t.sol())return!0}else if(F===o){if(t.spc(),t.eol())return s}else{if(z===o)return n.status=0,!0;if(I===o)n.status&v&&t.spc()&&!t.eol()&&(n.status|=b),n.status&=_;else if(l=r.match(t))return e.token={name:s,value:t.cur(),token:l[1]},s}return n.status&&n.$msg&&(n.$msg=Fe(n.$msg,s,!0)),!1},req:function(t){var e=this;return t?e.status|=v:e.status&=_,e},err:function(t,e,n,r,o){var s,l=this,u=l.name;return s=l.$msg?l.$msg:l.status&v?'Token "'+u+'" Expected':'Syntax Error: "'+u+'"',t&&t.status&y&&(t.err[e+"_"+n+"_"+r+"_"+o+"_"+u]=[e,n,r,o,s]),s},toString:function(){var t=this;return["[","Token: ",t.name,", ",t.token?t.token.toString():null,"]"].join("")}}),un=Me(ln,{constructor:function(t,e,n,r){var o=this;o.type=t||oe,o.name=e,o.token=n,o.status=0,o.msg=r||null,o.$msg=null,o.$clone=null},get:function(t,e){var n,r,o,s,l,u,i,a,c,p,f,h=this,m=h.token||null,g=e.queu,k=e.symb,d=e.ctx,v=e.token,_=e.err,C=!!(e.status&y);if(h.status=0,h.$msg=null,m){if(l=h.msg,n=m[0],r=m[1],c=!!m[2],u=e.line,a=t.pos,i=v&&v.value?a-v.value.length:i-1,w===n)return C&&(h.$msg=l?l:"Error",p=u+"_"+i+"_"+u+"_"+a+"_"+h.name,_[p]=[u,i,u,a,h.err()]),h.status|=b,!1;if(j===n)d.unshift({symb:{},queu:[]});else if(M===n)d.length&&d.shift();else if(A===n)c?d.length&&(d[0].queu.length=0):g.length=0;else if($===n){if(c){if(!d.length)return!0;k=d[0].symb}if(v){if(o=r[1],s=r[0],o=he===qe(o)?v.token[o]:Fe(o,v.token,!0),k[ae](s)||(k[s]={}),k[s][ae](o))return C&&(h.$msg=l?Fe(l,o,!0):'Duplicate "'+o+'"',f=h.err(),p=k[s][o][0]+"_"+k[s][o][1]+"_"+k[s][o][2]+"_"+k[s][o][3]+"_"+h.name,_[p]=[k[s][o][0],k[s][o][1],k[s][o][2],k[s][o][3],f],p=u+"_"+i+"_"+u+"_"+a+"_"+h.name,_[p]=[u,i,u,a,f]),h.status|=b,!1;k[s][o]=[u,i,u,a]}}else if(S===n){if(c){if(!d.length)return!0;g=d[0].queu}if(r){if(v&&(r=he===qe(r)?v.token[r]:Fe(r,v.token)),!g.length||r!==g[0][0])return C&&(g.length?(h.$msg=l?Fe(l,[g[0][0],r],!0):'Tokens do not match "'+g[0][0]+'","'+r+'"',f=h.err(),p=g[0][1]+"_"+g[0][2]+"_"+g[0][3]+"_"+g[0][4]+"_"+h.name,_[p]=[g[0][1],g[0][2],g[0][3],g[0][4],f],p=u+"_"+i+"_"+u+"_"+a+"_"+h.name,_[p]=[u,i,u,a,f]):(h.$msg=l?Fe(l,["",r],!0):'Token does not match "'+r+'"',f=h.err(),p=u+"_"+i+"_"+u+"_"+a+"_"+h.name,_[p]=[u,i,u,a,f])),g.shift(),h.status|=b,!1;g.shift()}else g.shift()}else if(O===n&&r){if(c){if(!d.length)return!0;g=d[0].queu}v&&(r=he===qe(r)?v.token[r]:Fe(r,v.token)),g.unshift([r,u,i,u,a])}}return!0}}),an=Me(ln,{constructor:function(t,e,n,r,o,s,l){var u=this;u.type=t,u.name=e,u.token=n,u.status=0,u.msg=r||null,u.mline="undefined"==typeof o?1:o,u.esc=s||"\\",u.inter=l,u.$msg=null,u.$clone=["mline","esc","inter"]},inter:0,mline:0,esc:null,get:function(t,e){var n,r,o,s,l,u,i,a,c,p,f,h,m,g,k=this,d=0,y=0,v="",b=k.mline,C=k.token,x=k.name,E=k.type,w=k.inter,$=w?x+".inside":x,O=0,S=K===E,A=k.esc,q="",R="",j="",M="",T=e.stack;if(k.$msg=k.msg||null,m=e.line,g=t.pos,Z===E&&(k.status&=_),l=0,e.block&&e.block.name===x?(y=1,n=e.block.end,l=1,u=$,p=e.block.s,h=e.block.i,f=e.block.e,q=e.block._s,R=e.block._i,M=""):!e.block&&(n=C.match(t))&&(y=1,p=[m,g],h=[[m,t.pos],[m,t.pos]],f=[m,t.pos],q=t.cur(),R="",M="",j="",e.block={name:x,end:n,s:p,i:h,e:f,_s:q,_i:R,_e:j},u=x),y){if(o=T.pos(),s=we===n.type,w){if(l&&s&&t.sol())return k.status&=_,e.current=null,e.block=null,!1;if(!l)return T.pushAt(o,k.clone(),"$id",x),u}if(d=n.match(t),r=b,c=0,d)u=s?$:x,j=t.cur().slice(M.length);else for(a=t.pos;!t.eol();){if(i=t.pos,!(S&&O||!n.match(t))){w&&t.pos>i&&i>a?(u=$,t.bck(i),c=1):(u=x,d=1),j=t.cur().slice(M.length);break}v=t.nxt(),M+=v,O=!O&&v===A}return r=b||S&&O,h[1]=[m,i],f=[m,t.pos],d||!r&&!c?e.block=null:(e.block.i=h,e.block.e=f,e.block._i+=M,e.block._e=j,e.stack.pushAt(o,k.clone(),"$id",x)),e.token={name:x,value:t.cur(),token:[q+R+M+j,R+M,q,j]},u}return k.status&&k.$msg&&(k.$msg=Fe(k.$msg,x,!0)),!1}}),cn=Me(ln,{constructor:function(t,e,n,r,o,s){var l=this;l.type=t?t:J,l.name=e||null,l.token=null,l.status=0,l.msg=r||null,l.min=o||0,l.max=s||E,l.found=0,l.$msg=null,l.$clone=["min","max","found"],n&&l.set(n)},min:0,max:1,found:0,set:function(t){return t&&(this.token=Te(t)),this},get:function(t,e){var n,r,o,s,l,u,i,a,c,p,f,h,m,g,k,d=this,y=d.type,x=d.token,E=x.length;if(d.$msg=d.msg||null,d.status&=C,h=t.pos,l=e.stack,H===y){for(p=0,f=0,d.status|=v,u=[],n=0;E>n;n++){if(o=x[n].clone().req(1),s=o.get(t,e),o.status&v&&(p++,u.push(o.err())),!1!==s)return s;o.status&b&&(f++,t.bck(h))}return p>0?d.status|=v:d.status&=_,E===f&&p>0?d.status|=b:d.status&=C,d.status&&!d.$msg&&u.length&&(d.$msg=u.join(" | ")),!1}if(re&y){k=y&Q?1:0,k?d.status|=v:d.status&=_,m=l.pos(),g=d.name+"_"+Je(),r=0;do o=x[r].clone().req(k),s=o.get(t,e),r++;while(E>r&&!0===s&&V&o.type);if(r>=E)return o.status&b?(k?d.status|=b:d.status&=C,t.bck(h)):k&&o.status&v&&(d.status|=b),d.status&&!d.$msg&&(d.$msg=o.err()),s;if(!1!==s){if(!0!==s)for(n=E-1;n>=r;n--)l.pushAt(m+E-n-1,x[n].clone().req(1),"$id",g);return s}return o.status&b?(k?d.status|=b:d.status&=C,t.bck(h)):k&&o.status&v&&(d.status|=b),d.status&&!d.$msg&&(d.$msg=o.err()),!1}for(p=0,i=d.found,a=d.min,c=d.max,d.status&=_,m=l.pos(),g=d.name+"_"+Je(),u=[],n=0;E>n;n++){if(o=x[n].clone().req(1),s=o.get(t,e),!1!==s){if(++i,c>=i)return d.found=i,l.pushAt(m,d.clone(),"$id",g),d.found=0,s;break}o.status&v&&(p++,u.push(o.err())),o.status&b&&t.bck(h)}return a>i?d.status|=v:d.status&=_,i>c||a>i&&p>0?d.status|=b:d.status&=C,d.status&&!d.$msg&&u.length&&(d.$msg=u.join(" | ")),!1}});var hn=CodeMirror||{Pass:{toString:function(){return"CodeMirror.Pass"}}};g=null,k="error";{var mn=Me({constructor:function(t,e){var n=this;n.Extra=t.Extra||{},n.LC=t.Comments.line?t.Comments.line[0]:null,n.BCS=t.Comments.block?t.Comments.block[0][0]:null,n.BCE=t.Comments.block?t.Comments.block[0][1]:null,n.BCC=n.BCL=t.Comments.block?t.Comments.block[0][2]:null,n.DEF=e.DEFAULT,n.ERR=t.Style.error||e.ERROR,n.Keywords=t.Keywords.autocomplete||null,n.Tokens=t.Parser||[],n.cTokens=t.cTokens.length?t.cTokens:null,n.Style=t.Style},Extra:null,LC:null,BCS:null,BCE:null,BCL:null,BCC:null,ERR:null,DEF:null,Keywords:null,cTokens:null,Tokens:null,Style:null,dispose:function(){var t=this;return t.Extra=null,t.LC=null,t.BCS=null,t.BCE=null,t.BCL=null,t.BCC=null,t.ERR=null,t.DEF=null,t.Keywords=null,t.cTokens=null,t.Tokens=null,t.Style=null,t},parse:function(t,e){t=t||"";var n,r,o,s,l,u,i,a,c=this,p=t.split(Ge),f=p.length;if(e=e||d,i=!!(e&y),a=!!(e&d),s=new fn(0,0,e),s.parseAll=1,a)for(r=[],n=0;f>n;n++){for(s.line=n,l=new nn(p[n]),o=[];!l.eol();)o.push(c.getToken(l,s));r.push(o)}else for(n=0;f>n;n++)for(s.line=n,l=new nn(p[n]);!l.eol();)c.getToken(l,s);return u=a&&i?{tokens:r,errors:s.err}:a?r:s.err,l.dispose(),s.dispose(),u},getToken:function(t,e){var n,r,o,s,l,u,i,a,c=this,p=c.cTokens,f=c.Tokens,h=f.length,m=!!e.parseAll,g=c.Style,k=c.DEF,d=c.ERR;if(t=m?t:nn._(t),m?0===e.line?e.status|=D:e.status&=~D:t.sol()?e.status|=D:e.status&=~D,u=e.stack,t.sol())for(;!u.isEmpty()&&F===u.peek().type;)u.pop();if((u.isEmpty()||I!==u.peek().type)&&t.spc())return m?{value:t.cur(1),type:k}:t.upd()&&k;for(a=e.line;!u.isEmpty()&&!t.eol();){if(p)for(r=0;rn;n++)if(i=t.pos,s=f[n],o=s.get(t,e),!1!==o){if(!0!==o){for(o=g[o]||k;!u.isEmpty()&&oe===u.peek().type;)l=u.pop(),l.get(t,e);return m?{value:t.cur(1),type:o}:t.upd()&&o}}else if(s.status&x)return u.empty("$id",s.$id),t.nxt(),o=d,s.err(e,a,i,a,t.pos),m?{value:t.cur(1),type:o}:t.upd()&&o;return t.nxt(),m?{value:t.cur(1),type:k}:t.upd()&&k},indent:function(t,e,n,r){var o=(r.indentUnit||4,hn.Pass);return o}});t.CodeMirrorGrammar={VERSION:"2.1.0",clone:Pe,extend:De,parse:h,getMode:m}}return t.CodeMirrorGrammar}); \ No newline at end of file diff --git a/grammar-reference.md b/grammar-reference.md index 85c3220..7397af5 100644 --- a/grammar-reference.md +++ b/grammar-reference.md @@ -11,6 +11,7 @@ 1. [Simple Tokens](#simple-tokens) 2. [Block Tokens](#block-tokens) 3. [Action Tokens **(new)**](#action-tokens) + 4. [Lex shorthand type annotations **(new,optional)**](#lex-shorthand-type-annotations) * [Syntax Model **(optional)**](#syntax-model) 1. [Syntax PEG/BNF-like notations **(new)**](#syntax-pegbnf-like-notations) * [Parser](#parser) @@ -74,6 +75,7 @@ example: 2. `"msg"` : `token` `error message` (default `token default error message` ) 3. `"tokens"` : `pattern` or `array of patterns` for this token 4. `properties` depending on `token type` (see below) + 5. *optionaly*, token `"type"` can be **annotated inside** the `token_id` ([see below](#lex-shorthand-type-annotations)) * a token type can be `"simple"` (default), `"block"` , `"escaped-block"` , `"comment"` , `"action"` * a token can *extend / reference* another token using the `extend` property; this way 2 tokens that share common configuration but different styles (depending on context) can be defined only once. Examples are tokens for `identifiers` and `properties` While both have similar tokenizers, the styles (and probably other properties) can be different. @@ -208,7 +210,33 @@ An `action` token in a grammar **applies only and directly to the token precedin ``` +####Lex shorthand type annotations +**(new, optional)** +Lexical tokens can annotate their `type` in their `token_id` as `"token_id:token_type"` for *convenience*. + +**Example:** +```javascript + + +"a_token:comment": [["<--", "-->"]] +// is equivalent to => +"a_token": { + "type": "comment", + "tokens": [["<--", "-->"]] +} + + +"a_token:action": {"push":"$1"} +// is equivalent to => +"a_token": { + "type": "action", + "push": "$1" +} + +// and so on.. + +``` ###Syntax model @@ -328,10 +356,18 @@ Specificaly: "tokens": ["t1* t2", "t3"] } -// literal tokens wrapped in quotes (' or ") -// are equivalent to their literal value -// empty literal token (i.e '') matches NON-SPACE production -// zero literal token (i.e 0) matches EMPTY production +// literal tokens wrapped in quotes (' or ") (e.g 'abc') are equivalent to their literal value (i.e abc) + +// empty literal token w/ quotes (i.e '') matches NON-SPACE production (i.e fails if space is encountered) + +// zero 0 literal token w/o quotes matches EMPTY production (i.e succeeds always) + +// ^ literal token w/o quotes matches SOL (i.e start-of-line, any line) +// ^^ literal token w/o quotes matches SOF (i.e start-of-file, start of line, first line of code, NOTE in some cases behaves like ^, above) + +// $ literal token w/o quotes matches EOL (i.e end-of-line, along with any extra space) +// $$ literal token w/o quotes matches EOF (i.e end-of-file, end of line of last line of code) (currently NOT supported) + "t": "t1 '=' t2" // is equivalent to => "t_equal": { diff --git a/src/main.js b/src/main.js index ac1a1d0..a202cd6 100644 --- a/src/main.js +++ b/src/main.js @@ -78,9 +78,8 @@ var Parser = Class({ parse_errors = !!(parse_type&ERRORS); parse_tokens = !!(parse_type&TOKENS); - state = new State( 0, 0, parse_errors ); + state = new State( 0, 0, parse_type ); state.parseAll = 1; - if ( parse_tokens ) { linetokens = []; @@ -116,12 +115,22 @@ var Parser = Class({ ; stream = parseAll ? stream : Stream._( stream ); + if ( parseAll ) + { + if ( 0 === state.line ) state.status |= T_SOF; + else state.status &= ~T_SOF; + } + else + { + if ( stream.sol() ) state.status |= T_SOF; + else state.status &= ~T_SOF; + } stack = state.stack; - // if EOL tokenizer is left on stack, pop it now - if ( stream.sol() && !stack.isEmpty() && T_EOL === stack.peek().type ) + if ( stream.sol() ) { - stack.pop(); + // if EOL tokenizer is left on stack, pop it now + while( !stack.isEmpty() && T_EOL === stack.peek().type ) stack.pop(); } // check for non-space tokenizer before parsing space @@ -176,7 +185,7 @@ var Parser = Class({ // found token (not empty) else if ( true !== type ) { - type = /*T_SPACE === type ? DEFAULT :*/ Style[type] || DEFAULT; + type = Style[type] || DEFAULT; // action token follows, execute action on current token while ( !stack.isEmpty() && T_ACTION === stack.peek().type ) { @@ -186,11 +195,9 @@ var Parser = Class({ /*if ( action.status&ERROR ) { // empty the stack - //stack.empty('$id', /*action* /tokenizer.$id); + //stack.empty('$id', tokenizer.$id); // generate error - //type = ERR; //action.err(state, line, pos, line, stream.pos); - return parseAll ? {value:stream.cur(1), type:type} : (stream.upd()&&type); }*/ } return parseAll ? {value:stream.cur(1), type:type} : (stream.upd()&&type); @@ -227,7 +234,7 @@ var Parser = Class({ // found token (not empty) else if ( true !== type ) { - type = /*T_SPACE === type ? DEFAULT :*/ Style[type] || DEFAULT; + type = Style[type] || DEFAULT; // action token follows, execute action on current token while ( !stack.isEmpty() && T_ACTION === stack.peek().type ) { @@ -239,9 +246,7 @@ var Parser = Class({ // empty the stack //stack.empty('$id', tokenizer.$id); // generate error - //type = ERR; //action.err(state, line, pos, line, stream.pos); - return parseAll ? {value:stream.cur(1), type:type} : (stream.upd()&&type); }*/ } return parseAll ? {value: stream.cur(1), type: type} : (stream.upd()&&type); @@ -282,12 +287,13 @@ function get_mode( grammar, DEFAULT ) */ startState: function( ) { - return new State( ); + var state = new State( ); + return state; }, copyState: function( state ) { - state = state.clone( ); state.line++; - return state; + var statec = state.clone( ); + return statec; }, token: function( stream, state ) { diff --git a/src/state.js b/src/state.js index 1d00286..ff07ec0 100644 --- a/src/state.js +++ b/src/state.js @@ -2,7 +2,7 @@ // // State Class var State = Class({ - constructor: function State( s, unique, with_errors ) { + constructor: function State( s, unique, status ) { var self = this; // this enables unique state "names" // thus forces highlight to update @@ -12,61 +12,57 @@ var State = Class({ { // clone self.line = s.line; - self.column = s.column; - self.indent = s.indent; + //self.indent = s.indent; + self.status = s.status; + self.token = s.token; + self.block = s.block; self.stack = s.stack.clone(); self.queu = s.queu.slice(); self.symb = clone( s.symb, 1 ); self.ctx = s.ctx.slice(); - self.token = s.token; - self.block = s.block; - self.errors = s.errors; self.err = s.err; } else { self.line = s || 0; - self.column = 0; - self.indent = null; + //self.indent = null; + self.status = status || 0; self.token = null; self.block = null; self.stack = new Stack(); self.queu = []; self.symb = {}; self.ctx = []; - self.errors = !!with_errors; - self.err = self.errors ? {} : null; + self.err = self.status&ERRORS ? {} : null; } } ,id: null ,line: 0 - ,column: 0 - ,indent: 0 + //,indent: 0 + ,status: 0 + ,token: null + ,block: null ,stack: null ,queu: null ,symb: null ,ctx: null ,err: null - ,errors: false - ,token: null - ,block: null ,dispose: function( ) { var self = this; self.id = null; self.line = null; - self.column = null; - self.indent = null; + //self.indent = null; + self.status = null; self.token = null; self.block = null; + if ( self.stack ) self.stack.dispose( ); + self.stack = null; + self.queu = null; self.symb = null; self.ctx = null; - self.errors = null; self.err = null; - self.queu = null; - if ( self.stack ) self.stack.dispose( ); - self.stack = null; return self; } @@ -78,6 +74,6 @@ var State = Class({ // make sure to generate a string which will cover most cases where state needs to be updated by the editor ,toString: function() { var self = this; - return ['', self.id, self.line, self.token ? self.token.name : '0', self.block ? self.block.name : '0'].join('_'); + return [self.id, self.line, self.token ? self.token.name : '0', self.block ? self.block.name : '0'].join('_'); } }); diff --git a/src/tokenizers.js b/src/tokenizers.js index d78ea8a..3cf25c4 100644 --- a/src/tokenizers.js +++ b/src/tokenizers.js @@ -43,8 +43,8 @@ function match_re( stream, eat ) var self = this, p = self.pattern, s = stream.s, m; m = s.slice( stream.pos ).match( p[0] ); if ( !m || m.index > 0 ) return false; - if ( false !== eat ) stream.mov( m[ 0 ].length ); - return [ self.key, m ]; + if ( false !== eat ) stream.mov( m[ p[1]||0 ].length ); + return [ self.key, p[1] > 0 ? m[p[1]] : m ]; } function match_null( stream, eat ) @@ -242,7 +242,7 @@ function get_compositematcher( name, tokens, RegExpID, combined, cachedRegexes, } else if ( 1 < l /*combined*/ ) { - l2 = (l>>1) + 1; + l2 = (l>>>1) + 1; // check if tokens can be combined in one regular expression // if they do not contain sub-arrays or regular expressions for (i=0; i<=l2; i++) @@ -277,7 +277,7 @@ function get_compositematcher( name, tokens, RegExpID, combined, cachedRegexes, { matcher = get_simplematcher( name, get_combined_re( tmp, combined ), 0, cachedMatchers ); } - else + else if ( array_of_arrays || has_regexs ) { for (i=0; i 1) ? new CompositePattern( name, tmp ) : tmp[0]; + } + else /* strings */ + { + tmp = tmp.sort( by_length ); + for (i=0; i 1) ? new CompositePattern( name, tmp ) : tmp[0]; } } @@ -335,6 +345,12 @@ Token = Class({ self.msg = msg || null; self.$msg = null; self.$clone = null; + if ( T_SOF === self.name ) self.name = ''; + else if ( T_SOL === self.name ) self.name = ''; + else if ( T_EOL === self.name ) self.name = ''; + else if ( T_EOF === self.name ) self.name = ''; + else if ( T_EMPTY === self.name ) self.name = ''; + else if ( T_NONSPACE === self.name ) self.name = ''; } // tokenizer/token name/id @@ -388,41 +404,50 @@ Token = Class({ var self = this, token = self.token, type = self.type, tokenID = self.name, t = null; - self.$msg = null; + self.$msg = self.msg || null; state.token = null; //self.pos = null; - //lin = state.line; col = stream.pos; - // match EMPTY token - if ( T_EMPTY === type ) + // match SOF (start-of-file) + if ( T_SOF === type ) { - self.status = 0; - //self.pos = [[lin, col], [lin, col]]; - return true; + if ( (T_SOF&state.status) && stream.sol() ) return true; + } + // match SOL (start-of-line) + else if ( T_SOL === type ) + { + if ( stream.sol() ) return true; } - // match EOL ( with possible leading spaces ) + // match EOL (end-of-line) ( with possible leading spaces ) else if ( T_EOL === type ) { stream.spc(); - if ( stream.eol() ) - { - //self.pos = [[lin, col], [lin, stream.pos]]; - return tokenID; - } + if ( stream.eol() ) return tokenID; + } + // match EOF (end-of-file) ( with possible leading spaces ) + /*else if ( T_EOF === type ) + { + stream.spc(); + if ( stream.eol() ) return true; + }*/ + // match EMPTY token + else if ( T_EMPTY === type ) + { + self.status = 0; + return true; } // match non-space else if ( T_NONSPACE === type ) { if ( (self.status&REQUIRED) && stream.spc() && !stream.eol() ) self.status |= ERROR; self.status &= CLEAR_REQUIRED; - //self.pos = [[lin, col], [lin, col]]; } // else match a simple token else if ( t = token.match(stream) ) { state.token = {name:tokenID, value:stream.cur(), token:t[1]}; - //self.pos = [[lin, col], [lin, stream.pos]]; return tokenID; } + if ( self.status && self.$msg ) self.$msg = group_replace( self.$msg, tokenID, true ); return false; } @@ -434,13 +459,13 @@ Token = Class({ } ,err: function( state, l1, c1, l2, c2 ) { - var t = this, m, tok = t.name; + var t = this, m, token = t.name; if ( t.$msg ) m = t.$msg; - else if ( t.status&REQUIRED ) m = 'Token "'+tok+'" Expected'; - else m = 'Syntax Error: "'+tok+'"'; - if ( state && state.errors ) + else if ( t.status&REQUIRED ) m = 'Token "'+token+'" Expected'; + else m = 'Syntax Error: "'+token+'"'; + if ( state && (state.status&ERRORS) ) { - state.err[l1+'_'+c1+'_'+l2+'_'+c2+'_'+tok] = [l1, c1, l2, c2, m]; + state.err[l1+'_'+c1+'_'+l2+'_'+c2+'_'+token] = [l1, c1, l2, c2, m]; } return m; } @@ -472,7 +497,7 @@ ActionToken = Class(Token, { ,get: function( stream, state ) { var self = this, action_def = self.token || null, action, t, t0, ns, msg, queu = state.queu, symb = state.symb, ctx = state.ctx, token = state.token, - lin, col1, col2, in_ctx, err = state.err, error, emsg, with_errors = state.errors; + lin, col1, col2, in_ctx, err = state.err, error, emsg, with_errors = !!(state.status&ERRORS); self.status = 0; self.$msg = null; @@ -495,6 +520,16 @@ ActionToken = Class(Token, { return false; } + else if ( A_CTXSTART === action ) + { + ctx.unshift({symb:{},queu:[]}); + } + + else if ( A_CTXEND === action ) + { + if ( ctx.length ) ctx.shift(); + } + else if ( A_EMPTY === action ) { if ( in_ctx ) @@ -507,23 +542,7 @@ ActionToken = Class(Token, { } } - else if ( A_CTX_START === action ) - { - //ns = uuid('ctx'); - ctx.unshift({symb:{},queu:[]}); - } - - else if ( A_CTX_END === action ) - { - if ( ctx.length ) ctx.shift(); - } - - /*else if ( A_BLOCKINDENT === action ) - { - // TODO - } - - else if ( A_INDENT === action ) + /*else if ( A_INDENT === action ) { // TODO } @@ -685,7 +704,6 @@ BlockToken = Class(Token, { { found = 1; endBlock = state.block.end; - //self.pos = state.block.pos; alreadyIn = 1; ret = thisBlockInterior; b_s = state.block.s; @@ -698,7 +716,6 @@ BlockToken = Class(Token, { else if ( !state.block && (endBlock = startBlock.match(stream)) ) { found = 1; - //self.pos = [[lin, col],[lin, stream.pos]]; b_s = [lin,col]; b_i = [[lin,stream.pos],[lin,stream.pos]]; b_e = [lin,stream.pos]; @@ -788,22 +805,19 @@ BlockToken = Class(Token, { if ( ended || (!continueToNextLine && !continueBlock) ) { state.block = null; - //self.pos[1] = [lin, stream.pos]; } else { - //self.pos[1] = [lin, stream.pos]; - //state.block.pos = self.pos; state.block.i = b_i; state.block.e = b_e; state.block._i += b_21; state.block._e = b_3; state.stack.pushAt( stackPos, self.clone( ), '$id', thisBlock ); } - state.token = {name:thisBlock, value:stream.cur(), token:[b_1+b_2+b_21+b_3, b_2+b_21]}; + state.token = {name:thisBlock, value:stream.cur(), token:[b_1+b_2+b_21+b_3, b_2+b_21, b_1, b_3]}; return ret; } - + if ( self.status && self.$msg ) self.$msg = group_replace( self.$msg, thisBlock, true ); return false; } }); @@ -885,16 +899,37 @@ CompositeToken = Class(Token, { if ( match_all ) self.status |= REQUIRED; else self.status &= CLEAR_REQUIRED; stackPos = stack.pos(); - token = tokens[ 0 ].clone().req( match_all ); - style = token.get(stream, state); stackId = self.name+'_'+get_id(); + i0 = 0; + do{ + token = tokens[ i0 ].clone().req( match_all ); + style = token.get(stream, state); + i0++; + // handle SOF,SOL tokens here + }while ( i0= n ) + { + if ( token.status&ERROR /*&& token.REQ*/ ) + { + if ( match_all ) self.status |= ERROR; + else self.status &= CLEAR_ERROR; + stream.bck( streamPos ); + } + else if ( match_all && (token.status&REQUIRED) ) + { + self.status |= ERROR; + } + if ( self.status && !self.$msg ) self.$msg = token.err(); + return style; + } if ( false !== style ) { // not empty token - if ( true !== style ) + if ( true !== style) { - for (i=n-1; i>0; i--) + for (i=n-1; i>=i0; i--) stack.pushAt( stackPos+n-i-1, tokens[ i ].clone().req( 1 ), '$id', stackId ); } @@ -991,8 +1026,28 @@ function parse_peg_bnf_notation( tok, Lex, Syntax ) if ( '0' === token ) { // interpret as empty tokenizer - sequence.push( 0 ); + sequence.push( T_EMPTY ); + } + else if ( '^^' === token ) + { + // interpret as SOF tokenizer + sequence.push( T_SOF ); + } + else if ( '^' === token ) + { + // interpret as SOL tokenizer + sequence.push( T_SOL ); + } + else if ( '$' === token ) + { + // interpret as EOL tokenizer + sequence.push( T_EOL ); } + /*else if ( '$$' === token ) + { + // interpret as EOF tokenizer + sequence.push( T_EOF ); + }*/ else { if ( !Lex[token] && !Syntax[token] ) @@ -1279,10 +1334,22 @@ function get_tokenizer( tokenID, RegExpID, Lex, Syntax, Style, MSG = null; - if ( null === tokenID ) + if ( T_SOF === tokenID ) + { + // SOF Token + return new Token( T_SOF, T_SOF, tokenID, MSG ); + } + + else if ( T_SOL === tokenID ) + { + // SOL Token + return new Token( T_SOL, T_SOL, tokenID, MSG ); + } + + else if ( T_EOL === tokenID || null === tokenID ) { // EOL Token - return new Token( T_EOL, 'EOL', tokenID, MSG ); + return new Token( T_EOL, T_EOL, tokenID, MSG ); } else if ( "" === tokenID ) @@ -1384,12 +1451,6 @@ function get_tokenizer( tokenID, RegExpID, Lex, Syntax, Style, tok.action = [ 'empty', tok.empty, !!tok['in-context'] ]; delete tok.empty; } - else if ( tok[HAS]('block-indent') ) - { - tok.type = "action"; - tok.action = [ 'block-indent', tok['block-indent'], !!tok['in-context'] ]; - delete tok['block-indent']; - } else if ( tok[HAS]('indent') ) { tok.type = "action"; @@ -1491,15 +1552,25 @@ function get_tokenizer( tokenID, RegExpID, Lex, Syntax, Style, if ( T_SIMPLE & type ) { - if ( "" === tok.tokens ) + if ( T_SOF === tok.tokens ) { - // NONSPACE Token - token = new Token( T_NONSPACE, tokenID, tokenID, MSG ); + // SOF Token + token = new Token( T_SOF, tokenID, tokenID, MSG ); // pre-cache tokenizer to handle recursive calls to same tokenizer cachedTokens[ tokenID ] = token; return token; } - else if ( null === tok.tokens ) + + else if ( T_SOL === tok.tokens ) + { + // SOL Token + token = new Token( T_SOL, tokenID, tokenID, MSG ); + // pre-cache tokenizer to handle recursive calls to same tokenizer + cachedTokens[ tokenID ] = token; + return token; + } + + else if ( T_EOL === tok.tokens || null === tok.tokens ) { // EOL Token token = new Token( T_EOL, tokenID, tokenID, MSG ); @@ -1507,6 +1578,16 @@ function get_tokenizer( tokenID, RegExpID, Lex, Syntax, Style, cachedTokens[ tokenID ] = token; return token; } + + else if ( "" === tok.tokens ) + { + // NONSPACE Token + token = new Token( T_NONSPACE, tokenID, tokenID, MSG ); + // pre-cache tokenizer to handle recursive calls to same tokenizer + cachedTokens[ tokenID ] = token; + return token; + } + else if ( false === tok.tokens || 0 === tok.tokens ) { // EMPTY Token @@ -1524,10 +1605,9 @@ function get_tokenizer( tokenID, RegExpID, Lex, Syntax, Style, if ( !tok[HAS]('action') ) { if ( tok[HAS]('error') ) tok.action = [A_ERROR, tok.error, !!tok['in-context']]; - else if ( tok[HAS]('context-start') ) tok.action = [A_CTX_START, tok['context-start'], !!tok['in-context']]; - else if ( tok[HAS]('context-end') ) tok.action = [A_CTX_END, tok['context-end'], !!tok['in-context']]; + else if ( tok[HAS]('context-start') ) tok.action = [A_CTXSTART, tok['context-start'], !!tok['in-context']]; + else if ( tok[HAS]('context-end') ) tok.action = [A_CTXEND, tok['context-end'], !!tok['in-context']]; else if ( tok[HAS]('empty') ) tok.action = [A_EMPTY, tok.empty, !!tok['in-context']]; - else if ( tok[HAS]('block-indent') ) tok.action = [A_BLOCKINDENT, tok['block-indent'], !!tok['in-context']]; else if ( tok[HAS]('indent') ) tok.action = [A_INDENT, tok.indent, !!tok['in-context']]; else if ( tok[HAS]('outdent') ) tok.action = [A_OUTDENT, tok.outdent, !!tok['in-context']]; else if ( tok[HAS]('unique') ) tok.action = [A_UNIQUE, T_STR&get_type(tok.unique)?['_DEFAULT_',tok.unique]:tok.unique, !!tok['in-context']]; @@ -1537,10 +1617,9 @@ function get_tokenizer( tokenID, RegExpID, Lex, Syntax, Style, else { if ( 'error' === tok.action[0] ) tok.action[0] = A_ERROR; - else if ( 'context-start' === tok.action[0] ) tok.action[0] = A_CTX_START; - else if ( 'context-end' === tok.action[0] ) tok.action[0] = A_CTX_END; + else if ( 'context-start' === tok.action[0] ) tok.action[0] = A_CTXSTART; + else if ( 'context-end' === tok.action[0] ) tok.action[0] = A_CTXEND; else if ( 'empty' === tok.action[0] ) tok.action[0] = A_EMPTY; - else if ( 'block-indent' === tok.action[0] ) tok.action[0] = A_BLOCKINDENT; else if ( 'indent' === tok.action[0] ) tok.action[0] = A_INDENT; else if ( 'outdent' === tok.action[0] ) tok.action[0] = A_OUTDENT; else if ( 'unique' === tok.action[0] ) tok.action[0] = A_UNIQUE; @@ -1558,7 +1637,7 @@ function get_tokenizer( tokenID, RegExpID, Lex, Syntax, Style, if ( tok.autocomplete ) get_autocomplete( tok, tokenID, keywords ); // combine by default if possible using word-boundary delimiter - combine = ( 'undefined' === typeof(tok.combine) ) ? "\\b" : tok.combine; + combine = 'undefined' === typeof(tok.combine) ? "\\b" : tok.combine; token = new Token( T_SIMPLE, tokenID, get_compositematcher( tokenID, tok.tokens.slice(), RegExpID, combine, cachedRegexes, cachedMatchers ), MSG @@ -1690,7 +1769,7 @@ function get_autocomplete( tok, type, keywords ) function parse_grammar( grammar ) { var RegExpID, tokens, numTokens, _tokens, - Style, Lex, Syntax, t, tokenID, token, tok, + Style, Lex, Syntax, t, tokenID, token, tok, tt, tID, tT, cachedRegexes, cachedMatchers, cachedTokens, commentTokens, comments, keywords; // grammar is parsed, return it @@ -1715,6 +1794,66 @@ function parse_grammar( grammar ) Style = grammar.Style || { }; + // shorthand token-type annotation in token_ID + for (tt in Lex) + { + if ( !Lex[HAS](tt) ) continue; + tID = tt.split(':'); + if ( tID[1] && trim(tID[1]).length ) + { + tT = trim(tID[1]).toLowerCase(); + } + else + { + tT = null; + } + tID = tID[0]; + if ( tID !== tt ) + { + Lex[tID] = Lex[tt]; delete Lex[tt]; + if ( tT ) + { + if ( T_OBJ & get_type(Lex[tID]) ) + { + if ( !Lex[tID].type ) Lex[tID].type = tT; + } + else + { + Lex[tID] = {type:tT, tokens:Lex[tID]}; + } + } + } + } + for (tt in Syntax) + { + if ( !Syntax[HAS](tt) ) continue; + tID = tt.split(':'); + if ( tID[1] && trim(tID[1]).length ) + { + tT = trim(tID[1]).toLowerCase(); + } + else + { + tT = null; + } + tID = tID[0]; + if ( tID !== tt ) + { + Syntax[tID] = Syntax[tt]; delete Syntax[tt]; + if ( tT ) + { + if ( T_OBJ & get_type(Syntax[tID]) ) + { + if ( !Syntax[tID].type ) Syntax[tID].type = tT; + } + else + { + Syntax[tID] = {type:tT, tokens:Syntax[tID]}; + } + } + } + } + _tokens = grammar.Parser || [ ]; numTokens = _tokens.length; tokens = [ ]; diff --git a/src/types.js b/src/types.js index 26b59fe..b490f8e 100644 --- a/src/types.js +++ b/src/types.js @@ -24,9 +24,8 @@ var A_EMPTY = 64, A_INDENT = 128, A_OUTDENT = 256, - A_BLOCKINDENT = 512, - A_CTX_START = 1024, - A_CTX_END = 2048, + A_CTXSTART = 512, + A_CTXEND = 1024, // // pattern types @@ -37,26 +36,27 @@ var // // token types //T_SPACE = 0, - T_ERROR = 4, - T_DEFAULT = 8, - T_SIMPLE = 16, - T_EOL = 17, - T_NONSPACE = 18, - T_EMPTY = 20, - T_BLOCK = 32, - T_ESCBLOCK = 33, - T_COMMENT = 34, - T_EITHER = 64, - T_SEQUENCE = 128, + T_SOF = 4, T_EOF = 8, T_SOL = 16, T_EOL = 32, + T_SOF_OR_SOL = T_SOF|T_SOL, + T_EMPTY = 64, T_NONSPACE = 128, + //T_EMPTY_OR_NONSPACE = T_EMPTY|T_NONSPACE, + //T_NOT_EMPTY_NOR_NONSPACE = ~T_EMPTY_OR_NONSPACE, + T_SIMPLE = 256, + T_BLOCK = 512, + T_ESCBLOCK = 513, + T_COMMENT = 514, + T_EITHER = 1024, + T_SEQUENCE = 2048, T_ALL = T_SEQUENCE, - T_REPEATED = 256, - T_ZEROORONE = 257, - T_ZEROORMORE = 258, - T_ONEORMORE = 259, - T_GROUP = 512, - T_NGRAM = 1024, + T_REPEATED = 4096, + T_ZEROORONE = 4097, + T_ZEROORMORE = 4098, + T_ONEORMORE = 4099, + T_GROUP = 8192, + T_NGRAM = 16384, T_SEQUENCE_OR_NGRAM = T_SEQUENCE | T_NGRAM, - T_ACTION = 2048, + T_ACTION = 32768, + //T_SOF_SOL_EOL_EOF_ACTION = T_SOF_SOL_EOL_EOF|T_ACTION, // // tokenizer types @@ -78,18 +78,5 @@ var SIMPLE: T_SIMPLE, GROUP: T_GROUP, NGRAM: T_NGRAM - }, - - actionTypes = { - ERROR: A_ERROR, - UNIQUE: A_UNIQUE, - PUSH: A_PUSH, - POP: A_POP, - EMPTY: A_EMPTY, - INDENT: A_INDENT, - OUTDENT: A_OUTDENT, - BLOCKINDENT: A_BLOCKINDENT, - CONTEXT_START: A_CTX_START, - CONTEXT_END: A_CTX_END } ; diff --git a/test/grammars/css.js b/test/grammars/css.js index 401c895..ff756ba 100644 --- a/test/grammars/css.js +++ b/test/grammars/css.js @@ -9,24 +9,24 @@ var css_grammar = { // // Style model "Style" : { - "comment": "comment", - "@atrule": "def", - "@import": "def", - "@keyframes": "def", - "@media": "def", - "identifier": "variable", - "!important": "builtin", - "CssAtom": "atom", - "url": "atom", - "format": "atom", - "CssProperty": "property", - "HtmlElement": "tag", - "CssID": "builtin", - "CssClass": "qualifier", - "PseudoElement": "string", - "number": "number", - "string": "string", - "text": "string" + "comment" : "comment", + "@atrule" : "def", + "@import" : "def", + "@keyframes" : "def", + "@media" : "def", + "identifier" : "variable", + "!important" : "builtin", + "CssAtom" : "atom", + "url" : "atom", + "format" : "atom", + "CssProperty" : "property", + "HtmlElement" : "tag", + "CssID" : "builtin", + "CssClass" : "qualifier", + "PseudoElement" : "string", + "number" : "number", + "string" : "string", + "text" : "string" }, @@ -35,8 +35,7 @@ var css_grammar = { "Lex" : { // comments - "comment" : { - "type" : "comment", + "comment:comment" : { "interleave": true, "tokens" : [ // block comments @@ -59,8 +58,7 @@ var css_grammar = { ], // strings - "string" : { - "type" : "block", + "string:block" : { "multiline": false, "tokens" : [ // start, end of string (can be the matched regex group ie. 1 ) @@ -99,89 +97,54 @@ var css_grammar = { "!important" : "RE::/!important\\b/i", // css html element - "HtmlElement" : "RE::/[a-z_\\-][a-z0-9_\\-]*/i" + "HtmlElement" : "RE::/[a-z_\\-][a-z0-9_\\-]*/i", + + "match:action" : { + "push": "$0" + }, + "matched1:action" : { + "pop": "{", + "msg": "Token \"$0\" does not match" + }, + "matched2:action" : { + "pop": "(", + "msg": "Token \"$0\" does not match" + } }, // // Syntax model (optional) "Syntax" : { - "urlDeclaration" : { - "type" : "ngram", - "tokens" : [ "url", "(", "string | text", ")" ] - }, + "urlDeclaration:ngram" : "url '(' match (string | text) ')' matched2", - "formatDeclaration" : { - "type" : "ngram", - "tokens" : [ "format", "(", "string | text", ")" ] - }, + "formatDeclaration:ngram" : "format '(' match (string | text) ')' matched2", - "cssSelector" : { - "type" : "group", - "match" : "oneOrMore", - "tokens" : [ "HtmlElement", "CssID", "CssClass", "PseudoElement", "string", ",", "(", ")", "[", "]", "=", "+", "^", ">", "*", "~"] - }, + "cssSelector" : "(HtmlElement | CssID | CssClass | PseudoElement | string | ',' | '(' match | ')' matched2 | '[' | ']' | '=' | '+' | '^' | '>' | '*' | '~')+", - "RHSAssignment" : { - "type" : "group", - "match" : "oneOrMore", - "tokens" : [ "!important", "urlDeclaration", "formatDeclaration", "string", "number", "CssAtom", ",", "(", ")" ] - }, + "RHSAssignment" : "(!important | urlDeclaration | formatDeclaration | string | number | CssAtom | ',' | '(' match | ')' matched2)+", - "cssAssignment" : { - "type" : "group", - "match" : "all", - "tokens" : [ "CssProperty", ":", "RHSAssignment", "semicolon" ] - }, + "cssAssignment" : "CssProperty ':' RHSAssignment ';'*", // syntax grammar (n-gram) for a block of css assignments - "cssBlock" : { - "type" : "n-gram", - "tokens" : [ - [ "number", "{", "cssAssignment*", "}" ], - [ "cssSelector", "{", "cssAssignment*", "}" ] - ] - }, - - "@importDirective" : { - "type" : "n-gram", - "tokens" : [ "@import", "urlDeclaration", ";" ] - }, + "cssBlock:ngram" : [ + [ "number '{' match cssAssignment* '}' matched1" ], + [ "cssSelector '{' match cssAssignment* '}' matched1" ] + ], - "@keyframesDirective" : { - "type" : "n-gram", - "tokens" : [ "@keyframes", "identifier", "{", "cssBlock*", "}" ] - }, + "@importDirective:ngram" : "@import urlDeclaration ';'", - "cssIdentifiers" : { - "type" : "group", - "match": "oneOrMore", - "tokens" : [ "identifier", "number", "string", ",", "(", ")"] - }, + "@keyframesDirective:ngram" : "@keyframes identifier '{' match cssBlock* '}' matched1", - "@mediaDirective" : { - "type" : "n-gram", - "tokens" : [ "@media", "cssIdentifiers", "{", "cssBlock*", "}" ] - }, + "cssIdentifiers" : "(identifier | number | string | ',' | '(' match | ')' matched2)+", - "semicolon" : ";*", + "@mediaDirective:ngram" : "@media cssIdentifiers '{' match cssBlock* '}' matched1", - "atruleLine" : { - "type" : "group", - "match": "all", - "tokens" : [ "cssIdentifiers", "semicolon" ] - }, + "atruleLine" : "cssIdentifiers ';'*", - "atruleBlock" : { - "type" : "group", - "match": "all", - "tokens" : [ "{", "cssAssignments", "}" ] - }, + "atruleBlock" : "'{' match cssAssignments '}' matched1", - "@atruleDirective" : { - "type" : "n-gram", - "tokens" : [ "@atrule", "atruleBlock | atruleLine" ] - } + "@atruleDirective:ngram" : "@atrule (atruleBlock | atruleLine)" }, // what to parse and in what order diff --git a/test/grammars/javascript-recursion.js b/test/grammars/javascript-recursion.js index eb8bcd2..8965c08 100644 --- a/test/grammars/javascript-recursion.js +++ b/test/grammars/javascript-recursion.js @@ -12,16 +12,16 @@ var js_grammar = { // Style model "Style" : { // lang token type -> Editor (style) tag - "comment": "comment", - "atom": "atom", - "keyword": "keyword", - "builtin": "builtin", - "operator": "operator", - "identifier": "variable", - "property": "attribute", - "number": "number", - "string": "string", - "regex": "string-2" + "comment" : "comment", + "atom" : "atom", + "keyword" : "keyword", + "builtin" : "builtin", + "operator" : "operator", + "identifier" : "variable", + "property" : "attribute", + "number" : "number", + "string" : "string", + "regex" : "string-2" }, @@ -30,15 +30,12 @@ var js_grammar = { "Lex" : { // comments - "comment" : { - "type" : "comment", + "comment:comment" : { "interleave": true, "tokens" : [ // line comment - // start, end delims (null matches end-of-line) [ "//", null ], // block comments - // start, end delims [ "/*", "*/" ] ] }, @@ -68,41 +65,11 @@ var js_grammar = { ], // usual strings - "string" : { - "type" : "escaped-block", - "escape" : "\\", - // start, end of string (can be the matched regex group ie. 1 ) - "tokens" : [ "RE::/(['\"])/", 1 ] - }, + "string:escaped-block" : [ "RE::/(['\"])/", 1 ], // literal regular expressions - "regex" : { - "type" : "escaped-block", - "escape" : "\\", - // javascript literal regular expressions can be parsed similar to strings - "tokens" : [ "/", "RE::#/[gimy]{0,4}#" ] - }, + "regex:escaped-block" : [ "/", "RE::#/[gimy]{0,4}#" ], - // operators - "operator" : { - "combine" : true, - "tokens" : [ - "\\", "+", "-", "*", "/", "%", "&", "|", "^", "~", "<", ">" , "!", - "||", "&&", "==", "!=", "<=", ">=", "<>", ">>", "<<", - "===", "!==", "<<<", ">>>" - ] - }, - - // delimiters - "delimiter" : { - "combine" : true, - "tokens" : [ - "(", ")", "[", "]", "{", "}", ",", "=", ";", "?", ":", - "+=", "-=", "*=", "/=", "%=", "&=", "|=", "^=", "++", "--", - ">>=", "<<=", ">>>=" - ] - }, - // atoms "atom" : { // enable autocompletion for these tokens, with their associated token ID @@ -138,21 +105,33 @@ var js_grammar = { ] }, - "ctx_start": { - "context-start": true - }, + "other" : "RE::/\\S+/", + + "ctx_start:action": {"context-start":true}, - "ctx_end": { - "context-end": true + "ctx_end:action": {"context-end":true}, + + "match:action": {"push": "$0"}, + "matched_bra:action": { + "pop": "{", + "msg": "Brackets do not match" + }, + "matched_paren:action": { + "pop": "(", + "msg": "Parentheses do not match" + }, + "matched_paren2:action": { + "pop": "[", + "msg": "Parentheses do not match" }, - "unique": { + "unique:action": { "unique": ["prop", "$0"], "msg": "Duplicate object property \"$0\"", "in-context": true }, - "unique_prop": { + "unique_prop:action": { "unique": ["prop", "$1"], "msg": "Duplicate object property \"$0\"", "in-context": true @@ -170,29 +149,16 @@ var js_grammar = { "literalPropertyValue" : "literalProperty ':' literalValue", // grammar recursion here - "literalObject" : "'{' ctx_start (literalPropertyValue (',' literalPropertyValue)*)? '}' ctx_end", + "literalObject" : "'{' match ctx_start (literalPropertyValue (',' literalPropertyValue)*)? '}' matched_bra ctx_end", // grammar recursion here - "literalArray" : "'[' (literalValue (',' literalValue)*)? ']'", + "literalArray" : "'[' match (literalValue (',' literalValue)*)? ']' matched_paren2", - "literalStatement" : { - "type": "ngram", - "tokens": [ - ["literalObject"], - ["literalArray"] - ] - } + "brackets" : "'{' match | '}' matched_bra | '(' match | ')' matched_paren | '[' match | ']' matched_paren2", + + "js" : "comment | number | string | regex | keyword | operator | atom | literalObject | literalArray | brackets | other" }, // what to parse and in what order - "Parser" : [ - "comment", - "number", - "string", - "regex", - "keyword", - "operator", - "atom", - "literalStatement" - ] + "Parser" : [ ["js"] ] }; diff --git a/test/grammars/json.js b/test/grammars/json.js index 113c2a7..a74ba51 100644 --- a/test/grammars/json.js +++ b/test/grammars/json.js @@ -12,10 +12,10 @@ var json_grammar = { // Style model "Style" : { // lang token type -> Editor (style) tag - "comment": "comment", - "atom": "atom", - "number": "number", - "string": "string" + "comment" : "comment", + "atom" : "atom", + "number" : "number", + "string" : "string" }, @@ -24,8 +24,7 @@ var json_grammar = { "Lex" : { // comments - "comment" : { - "type" : "comment", + "comment:comment" : { "interleave": true, "tokens" : [ // line comment @@ -57,12 +56,7 @@ var json_grammar = { ], // usual strings - "string" : { - "type" : "escaped-block", - "escape" : "\\", - // start, end of string (can be the matched regex group ie. 1 ) - "tokens" : [ "\"", "\"" ] - }, + "string:escaped-block" : [ "\"", "\"" ], // atoms "atom" : { @@ -71,21 +65,21 @@ var json_grammar = { "tokens" : [ "true", "false", "null" ] }, - "ctx_start": { - "context-start": true - }, + "other": "RE::/\\S+/", - "ctx_end": { - "context-end": true - }, + "ctx_start:action": {"context-start":true}, - "unique": { + "ctx_end:action": {"context-end":true}, + + "error:action":{"error":true,"msg":"Invalid JSON"}, + + "unique:action": { "unique": ["prop", "$0"], "msg": "Duplicate object property \"$0\"", "in-context": true }, - "unique_prop": { + "unique_prop:action": { "unique": ["prop", "$1"], "msg": "Duplicate object property \"$0\"", "in-context": true @@ -100,10 +94,7 @@ var json_grammar = { // grammar recursion here "literalValue" : "atom | string | number | literalArray | literalObject", "literalPropertyValue" : "string unique_prop ':' literalValue", - "json" : { - "type" : "ngram", - "tokens" : ["literalValue"] - } + "json:ngram" : ["literalValue | other error"] }, // what to parse and in what order diff --git a/test/grammars/xml.js b/test/grammars/xml.js index 321fcd0..01c7a98 100644 --- a/test/grammars/xml.js +++ b/test/grammars/xml.js @@ -13,62 +13,31 @@ var xml_grammar = { // Style model "Style" : { // lang token type -> Editor (style) tag - "comment_block": "comment", - "meta_block": "meta", - "cdata_block": "atom", - "atom": "atom", - "open_tag": "tag", - "close_open_tag": "tag", - "auto_close_open_tag": "tag", - "close_tag": "tag", - "attribute": "attribute", - "id": "attribute", - "number": "number", - "string": "string" + "comment" : "comment", + "meta" : "meta", + "cdata" : "atom", + "atom" : "atom", + "open_tag" : "tag", + "close_open_tag" : "tag", + "auto_close_open_tag" : "tag", + "close_tag" : "tag", + "att" : "attribute", + "id" : "attribute", + "number" : "number", + "string" : "string" }, // // Lexical model "Lex": { + "comment:comment": [""], - "comment_block": { - "type": "comment", - "tokens": [ - // block comments - // start, end delims - [ "" ] - ] - }, + "cdata:block": [""], - "cdata_block": { - "type": "block", - "tokens": [ - // cdata block - // start, end delims - [ "" ] - ] - }, + "meta:block": ["RE::/<\\?[_a-zA-Z][\\w\\._\\-]*/","?>"], - "meta_block": { - "type": "block", - "tokens": [ - // meta block - // start, end delims - [ "RE::/<\\?[_a-zA-Z][\\w\\._\\-]*/", "?>" ] - ] - }, - - // strings - "string": { - "type": "block", - "multiline": false, - "tokens": [ - // if no end given, end is same as start - [ "\"" ], [ "'" ] - ] - }, + "string:block": {"tokens":[[ "\"" ],[ "'" ]], "multiline":false}, - // numbers, in order of matching "number": [ // dec "RE::/[0-9]\\d*/", @@ -83,8 +52,8 @@ var xml_grammar = { "RE::/&[a-zA-Z][a-zA-Z0-9]*;/" ], - // tag attributes - "attribute": "RE::/[_a-zA-Z][_a-zA-Z0-9\\-]*/", + // tag attribute + "att": "RE::/[_a-zA-Z][_a-zA-Z0-9\\-]*/", // tags "open_tag": "RE::/<([_a-zA-Z][_a-zA-Z0-9\\-]*)/", @@ -92,77 +61,45 @@ var xml_grammar = { "auto_close_open_tag": "/>", "close_tag": "RE::/<\\/([_a-zA-Z][_a-zA-Z0-9\\-]*)>/", - // NEW feature - // action tokens to perform complex grammar functionality - // like associated tag matching and unique identifiers - - "ctx_start": { - "context-start": true - }, - - "ctx_end": { - "context-end": true - }, + "text": "RE::/[^<&]+/", + // actions + "ctx_start:action": {"context-start":true}, + "ctx_end:action": {"context-end":true}, // allow to find duplicate xml identifiers, with action tokens - "unique": { + "unique:action": { "unique": ["id", "$1"], "msg": "Duplicate id attribute \"$0\"" }, - // allow to find duplicate xml tag attributes, with action tokens - "unique_att": { + "unique_att:action": { "unique": ["att", "$0"], - "in-context": true, + "in-context":true, "msg": "Duplicate attribute \"$0\"" }, - // allow to match start/end tags, with action tokens - "match": { - "push": "<$1>" - }, - - "matched": { + "match:action": {"push":"<$1>"}, + "matched:action": { "pop": "<$1>", "msg": "Tags \"$0\" and \"$1\" do not match!" }, - - "nomatch": { - "pop": null - } + "nomatch:action": {"pop":null} }, // // Syntax model (optional) "Syntax": { - // NEW feature - // using PEG/BNF-like shorthands, instead of multiple grammar configuration objects - "id_att": "'id' unique_att '=' string unique", - "tag_att": "attribute unique_att '=' (string | number)", + "tag_att": "att unique_att '=' (string | number)", "start_tag": "open_tag match ctx_start (id_att | tag_att)* (close_open_tag | auto_close_open_tag nomatch) ctx_end", - "end_tag": "close_tag matched", - "tags": { - "type": "ngram", - "tokens": [ - ["start_tag"], - ["end_tag"] - ] - }, + "end_tag": "close_tag matched", - "blocks": { - "type": "ngram", - "tokens": [ - ["comment_block"], - ["cdata_block"], - ["meta_block"], - ] - } + "xml": "comment | cdata | meta | start_tag | end_tag | atom | text" }, // what to parse and in what order - "Parser": [ "blocks", "tags", "atom" ] + "Parser": [ ["xml"] ] };