Merge pull request #282 from hildjj/source-with-inline-map

Add source-with-inline-map

CHANGELOG.md

@@ -10,22 +10,27 @@ Released: TBD
 
 ### Major Changes
 
-- None
+- [#280](https://github.com/peggyjs/peggy/issues/280) Add inline examples to
+  the documentation, from @hildjj
 
 ### Minor Changes
 
-- [TBD] Use commander's new `.conflicts()` to check for mutually-exclusive CLI
-  options, from @hildjj
-- [TBD] `"*"` is now a valid `allowedStartRule`, which means all rules are allowed, from @hildjj
+- [#274](https://github.com/peggyjs/peggy/issues/274) Use commander's new
+  `.conflicts()` to check for mutually-exclusive CLI options, from @hildjj
+- [#274](https://github.com/peggyjs/peggy/issues/274) `"*"` is now a valid `allowedStartRule`, which means all rules are allowed, from @hildjj
 - [#229](https://github.com/peggyjs/peggy/issues/229) new CLI option
   `-S <rule>` or `--start-rule <rule>` to specify the start rule when testing,
   from @hildjj
 - [#236](https://github.com/peggyjs/peggy/issues/236) Website: show line numbers
   in parser input textarea, from @Mingun
+- [#280](https://github.com/peggyjs/peggy/issues/280) new output type
+  `source-with-inline-map`, which generates source text with an inline map,
+  from @hildjj
 
 ### Bug Fixes
 
-- None
+- [#283](https://github.com/peggyjs/peggy/issues/283) Fix incorrect type
+  information for DiagnosticCallback, from @hildjj
 
 2.0.1
 -----

docs/documentation.html

@@ -376,10 +376,13 @@ <h3 id="generating-a-parser-javascript-api">JavaScript API</h3>
   <code>null</code>).</dd>
 
   <dt><code>format</code></dt>
-  <dd>format of the generated parser (<code>"amd"</code>, <code>"bare"</code>,
-  <code>"commonjs"</code>, <code>"es"</code>, <code>"globals"</code>, or
-  <code>"umd"</code>); valid only when <code>output</code> is set to
-  <code>"source"</code> (default: <code>"bare"</code>).</dd>
+  <dd>
+    Format of the generated parser (<code>"amd"</code>, <code>"bare"</code>,
+    <code>"commonjs"</code>, <code>"es"</code>, <code>"globals"</code>, or
+    <code>"umd"</code>); valid only when <code>output</code> is set to
+    <code>"source"</code>, <code>"source-and-map"</code>, or
+    <code>"source-with-inline-map"</code>. (default: <code>"bare"</code>).
+  </dd>
 
   <dt><code>grammarSource</code></dt>
   <dd>any object that represent origin of the input grammar. The CLI will set
@@ -392,17 +395,36 @@ <h3 id="generating-a-parser-javascript-api">JavaScript API</h3>
   <dd>Callback for informational messages. See <a href="#error-reporting">Error Reporting</a></dd>
 
   <dt><code>output</code></dt>
-  <dd><p>If set to <code>"parser"</code>, the method will return generated parser
-  object; if set to <code>"source"</code>, it will return parser source code as
-  a string (default: <code>"parser"</code>).
-  If set to <code>"source-and-map"</code>, it will return a <a href="https://github.com/mozilla/source-map#sourcenode"><code>SourceNode</code></a> object; you can
-  get source code by calling <code>toString()</code> method or source code and mapping by
-  calling <code>toStringWithSourceMap()</code> method, see the <a href="https://github.com/mozilla/source-map#sourcenode"><code>SourceNode</code></a> documentation
-  (default: <code>"parser"</code>)</p>
-  <blockquote>
-  <p><strong>Note</strong>: because of bug <a href="https://github.com/mozilla/source-map/issues/444">source-map/444</a> you should also set <code>grammarSource</code> to
-  a not-empty string if you set this value to <code>"source-and-map"</code></p>
-  </blockquote></dd>
+  <dd><p>A string, one of:</p>
+    <ul>
+      <li><code>"parser"</code> - return generated parser object.</li>
+      <li><code>"source"</code> - return parser source code as a string.</li>
+      <li><code>"source-and-map"</code> - return a
+        <a href="https://github.com/mozilla/source-map#sourcenode"><code>SourceNode</code></a>
+        object; you can get source code by calling <code>toString()</code>
+        method or source code and mapping by calling
+        <code>toStringWithSourceMap()</code> method, see the
+        <a href="https://github.com/mozilla/source-map#sourcenode"><code>SourceNode</code></a>
+        documentation.
+      </li>
+      <li><code>"source-with-inline-map"</code> - return the parser source along
+        with an embedded source map as a <code>data:</code> URI.  This option
+        leads to a larger output string, but is the easiest to integrate with
+        developer tooling.</li>
+    </ul>
+    <p>(default: <code>"parser"</code>)</p>
+    <blockquote>
+      <p><strong>Note</strong>: because of bug <a
+      href="https://github.com/mozilla/source-map/issues/444">source-map/444</a>
+      you should also set <code>grammarSource</code> to a not-empty string if
+      you set this value to <code>"source-and-map"</code> or
+      <code>"source-with-inline-map"</code>.  The path should be relative to
+      the location where the generated parser code will be stored.  For
+      example, if you are generating <code>lib/parser.js</code> from
+      <code>src/parser.peggy</code>, then your options should be:
+      <code>{ grammarSource: "../src/parser.peggy" }</code></p>
+    </blockquote>
+  </dd>
 
   <dt><code>plugins</code></dt>
   <dd>Plugins to use. See the <a href="#plugins-api">Plugins API</a> section.</dd>

lib/compiler/index.js

@@ -12,6 +12,7 @@ const reportUndefinedRules = require("./passes/report-undefined-rules");
 const reportIncorrectPlucking = require("./passes/report-incorrect-plucking");
 const Session = require("./session");
 const visitor = require("./visitor");
+const { base64 } = require("./utils");
 
 function processOptions(options, defaults) {
   const processedOptions = {};
@@ -118,6 +119,20 @@ const compiler = {
       case "source-and-map":
         return ast.code;
 
+      case "source-with-inline-map": {
+        if (typeof TextEncoder === "undefined") {
+          throw new Error("TextEncoder is not supported by this platform");
+        }
+        const sourceMap = ast.code.toStringWithSourceMap();
+        const encoder = new TextEncoder();
+        const b64 = base64(
+          encoder.encode(JSON.stringify(sourceMap.map.toJSON()))
+        );
+        return sourceMap.code + `\
+//\x23 sourceMappingURL=data:application/json;charset=utf-8;base64,${b64}
+`;
+      }
+
       default:
         throw new Error("Invalid output format: " + options.output + ".");
     }

lib/compiler/utils.js

@@ -50,3 +50,42 @@ function regexpClassEscape(s) {
     .replace(/[\u1000-\uFFFF]/g,      ch => "\\u"  + hex(ch));
 }
 exports.regexpClassEscape = regexpClassEscape;
+
+/**
+ * Base64 encode a Uint8Array.  Needed for browser compatibility where
+ * the Buffer class is not available.
+ *
+ * @param {Uint8Array} u8 Bytes to encode
+ * @returns {string} Base64 encoded string
+ */
+function base64(u8) {
+  // Note: btoa has the worst API, and even mentioning Buffer here will
+  // cause rollup to suck it in.
+
+  // See RFC4648, sec. 4.
+  // https://datatracker.ietf.org/doc/html/rfc4648#section-4
+  const A = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+  const rem = u8.length % 3;
+  const len = u8.length - rem;
+  let res = "";
+
+  for (let i = 0; i < len; i += 3) {
+    res += A[u8[i] >> 2];
+    res += A[((u8[i] & 0x3) << 4) | (u8[i + 1] >> 4)];
+    res += A[((u8[i + 1] & 0xf) << 2) | (u8[i + 2] >> 6)];
+    res += A[u8[i + 2] & 0x3f];
+  }
+  if (rem === 1) {
+    res += A[u8[len] >> 2];
+    res += A[(u8[len] & 0x3) << 4];
+    res += "==";
+  } else if (rem === 2) {
+    res += A[u8[len] >> 2];
+    res += A[((u8[len] & 0x3) << 4) | (u8[len + 1] >> 4)];
+    res += A[(u8[len + 1] & 0xf) << 2];
+    res += "=";
+  }
+
+  return res;
+}
+exports.base64 = base64;

lib/peg.d.ts

@@ -765,6 +765,12 @@ export namespace compiler {
     options: SourceBuildOptions<"source">
   ): string;
 
+  function compile(
+    ast: ast.Grammar,
+    stages: Stages,
+    options: SourceBuildOptions<"source-with-inline-map">
+  ): string;
+
   /**
    * Generates a parser source and source map from a specified grammar AST.
    *
@@ -991,50 +997,20 @@ export interface Session {
   ): void;
 }
 
-export interface DiagnosticCallback {
-  /**
-   * Called when compiler reports an error.
-   *
-   * @param stage Stage in which this diagnostic was originated
-   * @param message Main message, which should describe error objectives
-   * @param location If defined, this is location described in the `message`
-   * @param notes Additional messages with context information
-   */
-  error?(
-    stage: Stage,
-    message: string,
-    location?: LocationRange,
-    notes?: DiagnosticNote[]
-  ): void;
-  /**
-   * Called when compiler reports a warning.
-   *
-   * @param stage Stage in which this diagnostic was originated
-   * @param message Main message, which should describe warning objectives
-   * @param location If defined, this is location described in the `message`
-   * @param notes Additional messages with context information
-   */
-  warning?(
-    stage: Stage,
-    message: string,
-    location?: LocationRange,
-    notes?: DiagnosticNote[]
-  ): void;
-  /**
-   * Called when compiler reports an informational message.
-   *
-   * @param stage Stage in which this diagnostic was originated
-   * @param message Main message, which gives information about an event
-   * @param location If defined, this is location described in the `message`
-   * @param notes Additional messages with context information
-   */
-  info?(
-    stage: Stage,
-    message: string,
-    location?: LocationRange,
-    notes?: DiagnosticNote[]
-  ): void;
-}
+/**
+ * Called when compiler reports an error, warning, or info.
+ *
+ * @param stage Stage in which this diagnostic was originated
+ * @param message Main message, which should describe error objectives
+ * @param location If defined, this is location described in the `message`
+ * @param notes Additional messages with context information
+ */
+export type DiagnosticCallback = (
+  stage: Stage,
+  message: string,
+  location?: LocationRange,
+  notes?: DiagnosticNote[]
+) => void;
 
 /**
  * Parser dependencies, is an object which maps variables used to access the
@@ -1087,14 +1063,20 @@ export interface ParserBuildOptions extends BuildOptionsBase {
    * If set to `"parser"`, the method will return generated parser object;
    * if set to `"source"`, it will return parser source code as a string;
    * if set to `"source-and-map"`, it will return a `SourceNode` object
-   * which can give a parser source code as a string and a source map;
+   *   which can give a parser source code as a string and a source map;
+   * if set to `"source-with-inline-map"`, it will return the parser source
+   *   along with an embedded source map as a `data:` URI;
    * (default: `"parser"`)
    */
   output?: "parser";
 }
 
 /** Possible kinds of source output generators. */
-export type SourceOutputs = "source" | "source-and-map";
+export type SourceOutputs =
+  "parser" |
+  "source" |
+  "source-and-map" |
+  "source-with-inline-map";
 
 /** Base options for all source-generating formats. */
 interface SourceOptionsBase<Output extends SourceOutputs>
@@ -1103,7 +1085,9 @@ interface SourceOptionsBase<Output extends SourceOutputs>
    * If set to `"parser"`, the method will return generated parser object;
    * if set to `"source"`, it will return parser source code as a string;
    * if set to `"source-and-map"`, it will return a `SourceNode` object
-   * which can give a parser source code as a string and a source map;
+   *   which can give a parser source code as a string and a source map;
+   * if set to `"source-with-inline-map"`, it will return the parser source
+   *   along with an embedded source map as a `data:` URI;
    * (default: `"parser"`)
    */
   output: Output;
@@ -1197,6 +1181,28 @@ export function generate(
   options: SourceBuildOptions<"source">
 ): string;
 
+/**
+ * Returns the generated source code as a string appended with a source map as
+ * a `data:` URI.
+ *
+ * Note, `options.grammarSource` MUST contain a string that is a path relative
+ * to the location the generated source will be written to, as no further path
+ * processing will be performed.
+ *
+ * @param grammar String in the format described by the meta-grammar in the
+ *        `parser.pegjs` file
+ * @param options Options that allow you to customize returned parser object
+ *
+ * @throws {SyntaxError}  If the grammar contains a syntax error, for example,
+ *         an unclosed brace
+ * @throws {GrammarError} If the grammar contains a semantic error, for
+ *         example, duplicated labels
+ */
+export function generate(
+  grammar: string,
+  options: SourceBuildOptions<"source-with-inline-map">
+): string;
+
 /**
  * Returns the generated source code and its source map as a `SourceNode`
  * object. You can get the generated code and the source map by using a

test/types/peg.test-d.ts

@@ -15,6 +15,35 @@ const src = readFileSync(
   "utf8"
 );
 
+const problems: peggy.Problem[] = [];
+
+function error(
+  stage: peggy.Stage,
+  message: string,
+  location?: peggy.LocationRange,
+  notes?: peggy.DiagnosticNote[]
+): void {
+  problems.push(["error", message, location, notes]);
+}
+
+function info(
+  stage: peggy.Stage,
+  message: string,
+  location?: peggy.LocationRange,
+  notes?: peggy.DiagnosticNote[]
+): void {
+  problems.push(["info", message, location, notes]);
+}
+
+function warning(
+  stage: peggy.Stage,
+  message: string,
+  location?: peggy.LocationRange,
+  notes?: peggy.DiagnosticNote[]
+): void {
+  problems.push(["warning", message, location, notes]);
+}
+
 describe("peg.d.ts", () => {
   it("executes a grammar", () => {
     expectType<string>(src);
@@ -45,7 +74,12 @@ describe("peg.d.ts", () => {
   });
 
   it("takes a valid tracer", () => {
-    const parser = peggy.generate(src, { trace: true });
+    const parser = peggy.generate(src, {
+      trace: true,
+      error,
+      info,
+      warning,
+    });
     expectType<peggy.Parser>(parser);
 
     parser.parse(" /**/ 1\n", {
@@ -81,6 +115,9 @@ describe("peg.d.ts", () => {
 
     const p3 = peggy.generate(src, { output: true as boolean ? "source-and-map" : "source" });
     expectType<string | SourceNode>(p3);
+
+    const p4 = peggy.generate(src, { output: "source-with-inline-map" });
+    expectType<string>(p4);
   });
 
   it("compiles with source map", () => {
@@ -107,6 +144,13 @@ describe("peg.d.ts", () => {
       { output: true as boolean ? "source-and-map" : "source" }
     );
     expectType<string | SourceNode>(p3);
+
+    const p4 = peggy.compiler.compile(
+      ast,
+      peggy.compiler.passes,
+      { output: "source-with-inline-map" }
+    );
+    expectType<string>(p4);
   });
 
   it("creates an AST", () => {
@@ -369,7 +413,12 @@ describe("peg.d.ts", () => {
     expectType<peggy.ast.Grammar>(ast);
     const parser = peggy.compiler.compile(
       ast,
-      peggy.compiler.passes
+      peggy.compiler.passes,
+      {
+        error,
+        info,
+        warning,
+      }
     );
     expectType<peggy.Parser>(parser);
     expectType<peggy.ast.MatchResult | undefined>(ast.rules[0].match);