Add async version for importing libs. Fix issue with grammarLocations…

… in generated parsers

.npmignore

@@ -110,3 +110,5 @@ dist
 test/
 tsconfig.json
 .github/
+.editorconfig
+eslint.config.js

.vscode/settings.json

@@ -1,3 +1,8 @@
 {
-  "eslint.experimental.useFlatConfig": true
-}
+  "eslint.experimental.useFlatConfig": true,
+  "eslint.validate": [
+    "javascript",
+    "peggy",
+    "typescript"
+  ]
+}

README.md

@@ -15,19 +15,43 @@ import peggy from "peggy-tag";
 const parse = peggy`foo = $("f" "o"+)`;
 console.log(parse("foooo")); // "foooo"
 
-const trace = peggy.withOptions({ trace: true });
-const traceParse = trace`num = n:$[0-9]+ { return parseInt(n, 10); }`
-console.log(traceParse("123"));
-// 1:1-1:1 rule.enter num
-// 1:1-1:4 rule.match num
+const traceGrammar = peggy.withOptions({ trace: true });
+const trace = traceGrammar`num = n:$[0-9]+ { return parseInt(n, 10); }`
+console.log(trace("123"));
+// 8:20-8:20 rule.enter num
+// 8:20-8:23 rule.match num
+// 123
+```
+
+If your grammar imports rules from other grammars, you MUST use the async
+functions `withImports` or `withImportsOptions`
+
+```js
+import {withImports, withImportsOptions} from "peggy-tag";
+
+const parse = await withImports`
+import Foo from './test/fixtures/foo.js'
+bar = Foo`;
+console.log(parse("foo")); // "foo"
+
+const traceGrammar = await withImportsOptions({ trace: true });
+const trace = traceGrammar`num = n:$[0-9]+ { return parseInt(n, 10); }`
+console.log(trace("123"));
+// 11:20-11:20 rule.enter num
+// 11:20-11:23 rule.match num
 // 123
 ```
 
 ## Notes:
 
 - This currently is only tested on Node 18+, no browser version yet.
-- This is for non-performance-sensitive code (e.g. prototypes), because the
+- `--experimental-vm-modules` is required for the async versions that
+  allow importing libraries.
+- This is for NON-performance-sensitive code (e.g. prototypes), because the
   parser with be generated every time the template is evaluated.
+- If your parse function's variable name has exactly five letters (like
+  "parse" or "trace"), the column numbers will be correct.  See issue #14
+  for discussion.
 
 [![Tests](https://github.com/peggyjs/peggy-tag/actions/workflows/node.js.yml/badge.svg)](https://github.com/peggyjs/peggy-tag/actions/workflows/node.js.yml)
 [![codecov](https://codecov.io/gh/peggyjs/peggy-tag/branch/main/graph/badge.svg?token=JCB9G04O47)](https://codecov.io/gh/peggyjs/peggy-tag)

eslint.config.js

@@ -1,11 +1,14 @@
 import mod from "@peggyjs/eslint-config/flat/module.js";
+import peggyjs from "@peggyjs/eslint-plugin/lib/flat/recommended.js";
 
 export default [
   {
     ignores: [
       "node_module/**",
       "**/*.d.ts",
+      "test/fixtures/*.js",
     ],
   },
   mod,
+  peggyjs,
 ];

lib/index.js

@@ -1,78 +1,78 @@
+import { callLocation, combine, formatMessage } from "./utils.js";
+import fromMem from "@peggyjs/from-mem";
 import peggy from "peggy";
-import url from "node:url";
 
 /**
+ * @typedef {function(string, peggy.ParserOptions): any} ParseFunction
+ */
+
+/**
+ * Return a function that has the given parse function wrapped with utilities
+ * that set the grammarLocation and format any errors that are thrown.
  *
- * @param {number} depth How deep in the callstack to go?  "2" is usually the
- *   first interesting one.
- * @returns {peggy.GrammarLocation?} Location of the grammar in the enclosing
- *   file.
+ * @param {ParseFunction} parse
+ * @returns {ParseFunction}
  */
-function callLocation(depth) {
-  const old = Error.prepareStackTrace;
-  Error.prepareStackTrace = (_, s) => s;
-  const stack = new Error().stack;
-  Error.prepareStackTrace = old;
+function curryParse(parse) {
+  return function Parse(text, options = {}) {
+    if (!options.grammarSource) {
+      options.grammarSource = callLocation(2, 7);
+    }
+    try {
+      return parse(text, options);
+    } catch (e) {
+      throw formatMessage(e, options.grammarSource, text);
+    }
+  };
+}
 
-  // Not v8, or short-stacked vs. expectations
-  if (!Array.isArray(stack) || (stack.length < depth)) {
-    return null;
+/**
+ * Turn a templated string into a Peggy parsing function.
+ *
+ * @param {peggy.ParserBuildOptions|undefined} opts Grammar generation options.
+ * @param {string[]} strings The string portions of the template.
+ * @param  {any[]} values The interpolated values of the template.
+ * @returns {ParseFunction} The parsing function.
+ */
+function pegWithOptions(opts, strings, values) {
+  const text = combine(strings, values);
+  const grammarSource = callLocation(3);
+  try {
+    const { parse } = peggy.generate(text, {
+      grammarSource,
+      ...opts,
+    });
+    return curryParse(parse);
+  } catch (e) {
+    throw formatMessage(e, grammarSource, text);
   }
-
-  const callsite = stack[depth];
-  const source = callsite.getFileName();
-  const path = source.startsWith("file:") ? url.fileURLToPath(source) : source;
-  return new peggy.GrammarLocation(
-    path,
-    {
-      offset: callsite.getPosition() + 1, // Go past backtick
-      line: callsite.getLineNumber(),
-      column: callsite.getColumnNumber() + 1,  // Go past backtick
-    }
-  );
 }
 
 /**
  * Turn a templated string into a Peggy parsing function.
  *
  * @param {peggy.ParserBuildOptions|undefined} opts Grammar generation options.
  * @param {string[]} strings The string portions of the template.
- * @param  {...any} values The interpolated values of the template.
- * @returns {function(string, peggy.ParserOptions): any} The parsing function.
+ * @param  {any[]} values The interpolated values of the template.
+ * @returns {Promise<ParseFunction>} The parsing function.
  */
-function pegWithOptions(opts, strings, ...values) {
-  let text = "";
-  strings.forEach((string, i) => {
-    text += string + (values[i] || "");
-  });
-  const grammarSource = callLocation(3) || "peggy-tag";
+async function importPegWithOptions(opts, strings, values) {
+  const text = combine(strings, values);
+  const grammarSource = callLocation(3);
   try {
-    const parser = peggy.generate(text, {
+    const src = /** @type {string} */ (peggy.generate(text, {
       grammarSource,
+      format: "es",
+      output: "source-with-inline-map",
       ...opts,
-    }).parse;
-    return (text, options = {}) => {
-      if (!options.grammarSource) {
-        options.grammarSource = "peggy-tag-parser";
-      }
-      try {
-        return parser(text, options);
-      } catch (e) {
-        // @ts-ignore
-        if (typeof e?.format === "function") {
-          // @ts-ignore
-          e.message = e.format([{ source: options.grammarSource, text }]);
-        }
-        throw e;
-      }
-    };
+    }));
+    const { parse } = /** @type {peggy.Parser} */ (await fromMem(src, {
+      filename: grammarSource.source,
+      format: "es",
+    }));
+    return curryParse(parse);
   } catch (e) {
-    // @ts-ignore
-    if (typeof e?.format === "function") {
-      // @ts-ignore
-      e.message = e.format([{ source: grammarSource, text }]);
-    }
-    throw e;
+    throw formatMessage(e, grammarSource, text);
   }
 }
 
@@ -81,33 +81,71 @@ function pegWithOptions(opts, strings, ...values) {
  *
  * @param {string[]} strings The string portions of the template.
  * @param  {...any} values The interpolated values of the template.
- * @returns {function(string, peggy.ParserOptions): any} The parsing function.
+ * @returns {ParseFunction} The parsing function.
  * @example
  *   import peg from "peggy-tag";
- *   const parser = peg`foo = "foo"`;
- *   console.log(parser("foo"));
+ *   const parse = peg`foo = "foo"`;
+ *   console.log(parse("foo"));
  */
 export default function peg(strings, ...values) {
-  return pegWithOptions(undefined, strings, ...values);
+  return pegWithOptions(undefined, strings, values);
 }
 
 /**
  * Create a template string tag with non-default grammar generation options.
  *
  * @param {peggy.ParserBuildOptions|undefined} opts Grammar generation options.
- * @returns {function(string[], ...any): function(string, peggy.ParserOptions): any}
+ * @returns {function(string[], ...any): ParseFunction}
  * @example
- *   import peg from "peggy-tag";
- *   import myPeg = peg.withOptions({trace: true})
+ *   import { withOptions } from "peggy-tag";
+ *   import myPeg = withOptions({trace: true})
  *   const parser = myPeg`foo = "foo"`;
  *   console.log(parser("foo"));
  */
-peg.withOptions = opts => (
+export function withOptions(opts) {
   /**
-   *
    * @param {string[]} strings The string portions of the template.
    * @param  {...any} values The interpolated values of the template.
-   * @returns {function(string, peggy.ParserOptions): any} The parsing function.
+   * @returns {ParseFunction} The parsing function.
    */
-  (strings, ...values) => pegWithOptions(opts, strings, ...values)
-);
+  return (strings, ...values) => pegWithOptions(opts, strings, values);
+}
+peg.withOptions = withOptions;
+
+/**
+ * Create a parse from a string that may include import statements.
+ *
+ * @param {string[]} strings The string portions of the template.
+ * @param  {...any} values The interpolated values of the template.
+ * @returns {Promise<ParseFunction>} The parsing function.
+ * @example
+ *   import { withImports } from "peggy-tag";
+ *   const parse = await withImports`foo = "foo"`;
+ *   console.log(parse("foo"));
+ */
+export function withImports(strings, ...values) {
+  return importPegWithOptions(undefined, strings, values);
+}
+peg.withImports = withImports;
+
+/**
+ * Create a template string tag with non-default grammar generation options,
+ * for grammars that include imports.
+ *
+ * @param {peggy.ParserBuildOptions|undefined} opts Grammar generation options.
+ * @returns {function(string[], ...any): Promise<ParseFunction>}
+ * @example
+ *   import { withImportsOptions } from "peggy-tag";
+ *   import myPeg = peg.withOptions({trace: true})
+ *   const parser = await myPeg`foo = "foo"`;
+ *   console.log(parser("foo"));
+ */
+export function withImportsOptions(opts) {
+  /**
+   * @param {string[]} strings The string portions of the template.
+   * @param  {...any} values The interpolated values of the template.
+   * @returns {Promise<ParseFunction>} The parsing function.
+   */
+  return (strings, ...values) => importPegWithOptions(opts, strings, values);
+}
+peg.withImportsOptions = withImportsOptions;

lib/utils.js

@@ -0,0 +1,79 @@
+import peggy from "peggy";
+import url from "node:url";
+
+/**
+ * Generate a GrammarLocation for one of the functions up the call stack
+ * from here.  0 is not useful, it's always the callLocation function.
+ * 1 is unlikely to be useful, it's the place you are calling callLocation from,
+ * so you presumably know where you are.  2 is the caller of the function you
+ * are in, etc.
+ *
+ * @param {number} depth How deep in the callstack to go?
+ * @param {number} [offset=1] How many characters to add to the location to
+ *   account for the calling apparatus, such as the backtick or the function
+ *   name + paren.
+ * @returns {peggy.GrammarLocation} Location of the grammar in the enclosing
+ *   file.
+ * @see https://v8.dev/docs/stack-trace-api
+ */
+export function callLocation(depth, offset = 1) {
+  const old = Error.prepareStackTrace;
+  Error.prepareStackTrace = (_, s) => s;
+  const stack = /** @type {NodeJS.CallSite[]} */(
+    /** @type {unknown} */(new Error().stack)
+  );
+  Error.prepareStackTrace = old;
+
+  // Not v8, or short-stacked vs. expectations
+  if (!Array.isArray(stack) || (stack.length < depth)) {
+    return new peggy.GrammarLocation(
+      "peggy-tag",
+      {
+        offset: 0,
+        line: 0,
+        column: 0,
+      }
+    );
+  }
+
+  const callsite = stack[depth];
+  const fn = callsite.getFileName();
+  const path = fn?.startsWith("file:") ? url.fileURLToPath(fn) : fn;
+  return new peggy.GrammarLocation(
+    path,
+    {
+      offset: callsite.getPosition() + offset,
+      // These will be 0 if the frame selected is native code, which
+      // we should never be doing in this package.
+      line: callsite.getLineNumber() || 0,
+      column: (callsite.getColumnNumber() || 0) + offset,
+    }
+  );
+}
+
+/**
+ * Combine the parameters from a tagged template literal into a string.
+ *
+ * @param {string[]} strings
+ * @param {any[]} values
+ * @returns {string}
+ */
+export function combine(strings, values) {
+  return strings.reduce((t, s, i) => t + s + String(values[i] ?? ""), "");
+}
+
+/**
+ * If this is a grammar error, reformat the message using the associated
+ * text.
+ *
+ * @param {any} error An error th
+ * @param {any} source
+ * @param {string} text
+ * @returns {Error} Error with reformatted message, if possible
+ */
+export function formatMessage(error, source, text) {
+  if ((typeof error === "object") && (typeof error?.format === "function")) {
+    error.message = error.format([{ source, text }]);
+  }
+  return error;
+}