From 639e5de6503ae22c8b41262271fbed8a92284c53 Mon Sep 17 00:00:00 2001 From: Nathan Date: Sun, 19 Aug 2018 14:07:31 -0400 Subject: [PATCH 1/5] Inheritance for stateful lexers --- moo.js | 75 ++++++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 70 insertions(+), 5 deletions(-) diff --git a/moo.js b/moo.js index 16eb5b9..1b001db 100644 --- a/moo.js +++ b/moo.js @@ -60,7 +60,13 @@ for (var i = 0; i < keys.length; i++) { var key = keys[i] var thing = object[key] - var rules = Array.isArray(thing) ? thing : [thing] + var rules = [].concat(thing) + if (key === 'include') { + for (var j = 0; j < rules.length; j++) { + result.push({include: rules[j]}) + } + continue + } var match = [] rules.forEach(function(rule) { if (isObject(rule)) { @@ -80,6 +86,13 @@ var result = [] for (var i = 0; i < array.length; i++) { var obj = array[i] + if (obj.include) { + var include = [].concat(obj.include) + for (var j = 0; j < include.length; j++) { + result.push({include: include[j]}) + } + continue + } if (!obj.name) { throw new Error('Rule has no name: ' + JSON.stringify(obj)) } @@ -92,6 +105,9 @@ if (!isObject(obj)) { obj = { match: obj } } + if (obj.include) { + throw new Error('Matching rules cannot also include states') + } // nb. error and fallback imply lineBreaks var options = { @@ -127,10 +143,12 @@ return options } + function toRules(spec) { + return Array.isArray(spec) ? arrayToRules(spec) : objectToRules(spec) + } + var defaultErrorRule = ruleOptions('error', {lineBreaks: true, shouldThrow: true}) function compileRules(rules, hasStates) { - rules = Array.isArray(rules) ? arrayToRules(rules) : objectToRules(rules) - var errorRule = null var fast = Object.create(null) var fastAllowed = true @@ -139,6 +157,11 @@ for (var i = 0; i < rules.length; i++) { var options = rules[i] + if (options.include) { + // all valid inclusions are removed by states() preprocessor + throw new Error('Inheritance is not allowed in stateless lexers') + } + if (options.error || options.fallback) { // errorRule can only be set once if (errorRule) { @@ -214,7 +237,7 @@ } function compile(rules) { - var result = compileRules(rules) + var result = compileRules(toRules(rules)) return new Lexer({start: result}, 'start') } @@ -228,13 +251,55 @@ } } function compileStates(states, start) { + var all = states.$all ? toRules(states.$all) : [] + delete states.$all + var keys = Object.getOwnPropertyNames(states) if (!start) start = keys[0] + var ruleMap = Object.create(null) + for (var i = 0; i < keys.length; i++) { + var key = keys[i] + ruleMap[key] = toRules(states[key]).concat(all) + } + do { + var affected = false + for (var i = 0; i < keys.length; i++) { + var key = keys[i] + var rules = ruleMap[key] + for (var j = 0; j < rules.length; j++) { + var rule = rules[j] + if (!rule.include || rule.include === key) continue + var newRules = ruleMap[rule.include] + if (!newRules) { + throw new Error("Cannot include nonexistent state '" + rule.include + "' (in state '" + key + "')") + } + for (var k = 0; k < newRules.length; k++) { + var newRule = newRules[k] + var l = rules.indexOf(newRule) + if (l > j) { + rules.splice(l, 1) + } else if (l !== -1) { + continue + } + rules.splice(j, 0, newRule) + ++j + affected = true + } + } + } + } while (affected) + var map = Object.create(null) for (var i = 0; i < keys.length; i++) { var key = keys[i] - map[key] = compileRules(states[key], true) + var allRules = ruleMap[key] + var matchRules = [] + for (var j = 0; j < allRules.length; j++) { + var rule = allRules[j] + if (!rule.include) matchRules.push(rule) + } + map[key] = compileRules(matchRules, true) } for (var i = 0; i < keys.length; i++) { From fbf376977ce557964c19dd13af48a982561733b6 Mon Sep 17 00:00:00 2001 From: Nathan Date: Sun, 19 Aug 2018 14:59:10 -0400 Subject: [PATCH 2/5] Fix indirect cycles --- moo.js | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/moo.js b/moo.js index 1b001db..c193572 100644 --- a/moo.js +++ b/moo.js @@ -262,14 +262,15 @@ var key = keys[i] ruleMap[key] = toRules(states[key]).concat(all) } - do { - var affected = false - for (var i = 0; i < keys.length; i++) { - var key = keys[i] - var rules = ruleMap[key] - for (var j = 0; j < rules.length; j++) { - var rule = rules[j] - if (!rule.include || rule.include === key) continue + for (var i = 0; i < keys.length; i++) { + var key = keys[i] + var rules = ruleMap[key] + var included = Object.create(null) + for (var j = 0; j < rules.length; j++) { + var rule = rules[j] + if (!rule.include) continue + var splice = [j, 1] + if (rule.include !== key && !included[rule.include]) { var newRules = ruleMap[rule.include] if (!newRules) { throw new Error("Cannot include nonexistent state '" + rule.include + "' (in state '" + key + "')") @@ -277,18 +278,14 @@ for (var k = 0; k < newRules.length; k++) { var newRule = newRules[k] var l = rules.indexOf(newRule) - if (l > j) { - rules.splice(l, 1) - } else if (l !== -1) { - continue - } - rules.splice(j, 0, newRule) - ++j - affected = true + if (l !== -1) continue + splice.push(newRule) } } + rules.splice.apply(rules, splice) + j-- } - } while (affected) + } var map = Object.create(null) for (var i = 0; i < keys.length; i++) { From 143a5fa61fbc1feee2f86af4fbe439bb34af4d44 Mon Sep 17 00:00:00 2001 From: Nathan Date: Sun, 19 Aug 2018 19:12:19 -0400 Subject: [PATCH 3/5] Remove unnecessary code --- moo.js | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/moo.js b/moo.js index c193572..db46a47 100644 --- a/moo.js +++ b/moo.js @@ -290,13 +290,7 @@ var map = Object.create(null) for (var i = 0; i < keys.length; i++) { var key = keys[i] - var allRules = ruleMap[key] - var matchRules = [] - for (var j = 0; j < allRules.length; j++) { - var rule = allRules[j] - if (!rule.include) matchRules.push(rule) - } - map[key] = compileRules(matchRules, true) + map[key] = compileRules(ruleMap[key], true) } for (var i = 0; i < keys.length; i++) { From 9e1b4587231a4973da3f011e4b6c357365aa3c51 Mon Sep 17 00:00:00 2001 From: Nathan Date: Sun, 19 Aug 2018 19:22:18 -0400 Subject: [PATCH 4/5] Fix other indirect cycles --- moo.js | 1 + 1 file changed, 1 insertion(+) diff --git a/moo.js b/moo.js index db46a47..64ffc33 100644 --- a/moo.js +++ b/moo.js @@ -271,6 +271,7 @@ if (!rule.include) continue var splice = [j, 1] if (rule.include !== key && !included[rule.include]) { + included[rule.include] = true var newRules = ruleMap[rule.include] if (!newRules) { throw new Error("Cannot include nonexistent state '" + rule.include + "' (in state '" + key + "')") From 88cbef43a9bc00eed9587a3bd74d2de824943432 Mon Sep 17 00:00:00 2001 From: Nathan Date: Sun, 19 Aug 2018 19:22:31 -0400 Subject: [PATCH 5/5] Clean up --- moo.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/moo.js b/moo.js index 64ffc33..335d5c0 100644 --- a/moo.js +++ b/moo.js @@ -278,8 +278,7 @@ } for (var k = 0; k < newRules.length; k++) { var newRule = newRules[k] - var l = rules.indexOf(newRule) - if (l !== -1) continue + if (rules.indexOf(newRule) !== -1) continue splice.push(newRule) } }