feat: add ret tokenizer for RegExp

README.md

@@ -69,6 +69,7 @@ The AST explorer provides following code parsers:
   - [regexp-tree][]
   - [regexpp][]
   - [regjsparser][]
+  - [ret][]
 - Protocol Buffers:
   - [pbkit][]
 - Scala
@@ -172,6 +173,7 @@ are included so you can prototype your own plugins:
 [regexp-tree]: https://github.com/DmitrySoshnikov/regexp-tree
 [regexpp]: https://github.com/mysticatea/regexpp
 [regjsparser]: https://github.com/jviereck/regjsparser
+[ret]: https://github.com/fent/ret.js
 [php-parser]: https://github.com/glayzzle/php-parser
 [pug]: https://github.com/pugjs/pug
 [glimmer]: https://github.com/glimmerjs/glimmer-vm

website/package.json

@@ -141,6 +141,7 @@
     "remark-frontmatter": "^3.0.0",
     "remark-gfm": "^1.0.0",
     "remark-math": "^4.0.0",
+    "ret": "^0.4.0",
     "san": "^3.9.4",
     "scalameta-parsers": "^4.4.17",
     "seafox": "^1.6.9",

website/src/parsers/regexp/ret.js

@@ -0,0 +1,92 @@
+import defaultParserInterface from '../utils/defaultParserInterface';
+import pkg from 'ret/package.json';
+
+const ID = 'ret';
+
+export default {
+  ...defaultParserInterface,
+
+  id: ID,
+  displayName: ID,
+  version: pkg.version,
+  homepage: pkg.homepage,
+
+  loadParser(callback) {
+    require(['ret'], (ret) => {
+      callback(ret);
+    });
+  },
+
+  parse(ret, code, options={}) {
+    this.types = ret.types;
+
+    // Enforce /regexp/ syntax to match other regexp parsers.
+    const firstSlash = code.indexOf('/');
+    const lastSlash = code.lastIndexOf('/');
+    if (firstSlash !== 0 || lastSlash < 1) {
+      throw new Error('Please wrap the regex pattern by slash `/`, i.e. /foo/');
+    }
+
+    // ret does not do anything with the flags, so just strip them.
+    const pattern = code.slice(firstSlash + 1, lastSlash);
+    return ret(pattern);
+  },
+
+  *forEachProperty(node) {
+    if (node && typeof node === 'object') {
+      for (let prop in node) {
+        if (prop === 'type') {
+          continue;
+        }
+
+        yield {
+          value: node[prop],
+          key: prop,
+          computed: false,
+        };
+      }
+
+      if (node.type === this.types.CHAR) {
+        yield {
+          value: String.fromCharCode(node.value),
+          key: 'char',
+          computed: true,
+        };
+      }
+    }
+  },
+
+  getNodeName(node) {
+    switch (node.type) {
+      case this.types.ROOT:
+        return "ROOT";
+      case this.types.GROUP:
+        return "GROUP";
+      case this.types.POSITION:
+        return "POSITION";
+      case this.types.SET:
+        return "SET";
+      case this.types.RANGE:
+        return "RANGE";
+      case this.types.REPETITION:
+        return "REPETITION";
+      case this.types.REFERENCE:
+        return "REFERENCE";
+      case this.types.CHAR:
+        return "CHAR";
+    }
+  },
+
+  opensByDefault(node, key) {
+    return (
+      key === "options" ||
+      key === "stack" ||
+      node.type === this.types.CHAR ||
+      Array.isArray(node)
+    );
+  },
+
+  getDefaultOptions() {
+    return {};
+  },
+};

website/webpack.config.js

@@ -193,6 +193,7 @@ module.exports = Object.assign({
           path.join(__dirname, 'node_modules', 'regexp-tree'),
           path.join(__dirname, 'node_modules', 'regjsparser'),
           path.join(__dirname, 'node_modules', 'regexpp'),
+          path.join(__dirname, 'node_modules', 'ret'),
           path.join(__dirname, 'node_modules', 'simple-html-tokenizer'),
           path.join(__dirname, 'node_modules', 'symbol-observable', 'es'),
           path.join(__dirname, 'node_modules', 'typescript-eslint-parser'),

website/yarn.lock

@@ -9429,6 +9429,11 @@ restore-cursor@^3.1.0:
     onetime "^5.1.0"
     signal-exit "^3.0.2"
 
+ret@^0.4.0:
+  version "0.4.0"
+  resolved "https://registry.yarnpkg.com/ret/-/ret-0.4.0.tgz#b040b0e9c2e1ba2df3e9af39fc8159857f50ea74"
+  integrity sha512-ZAUfxFXaRPGDk8P2qxoDT/xz4DF+Iaq+pIH07ZMGhIGWo/5zFZRg/mPvKkzxRWTqfqy9M2Hw7Uqv3z3XwE6cpw==
+
 ret@~0.1.10:
   version "0.1.15"
   resolved "https://registry.yarnpkg.com/ret/-/ret-0.1.15.tgz#b8a4825d5bdb1fc3f6f53c2bc33f81388681c7bc"