refactor/fix/feat: new parser (#1258)

Fixes some parsing bugs and improves error messages.

.eslintignore

@@ -6,6 +6,7 @@ src/test/**/output/
 src/func/funcfiftlib.js
 **/grammar.ohm*.ts
 **/grammar.ohm*.js
+src/grammar/next/grammar.ts
 jest.setup.js
 jest.teardown.js
 /docs

.prettierignore

@@ -1,8 +1,9 @@
 /dist
 /src/func/funcfiftlib.js
 /src/func/funcfiftlib.wasm.js
-/src/grammar/grammar.ohm-bundle.d.ts
-/src/grammar/grammar.ohm-bundle.js
+/src/grammar/prev/grammar.ohm-bundle.d.ts
+/src/grammar/prev/grammar.ohm-bundle.js
+src/grammar/next/grammar.ts
 /src/imports/stdlib.ts
 /grammar
 /docs

CHANGELOG.md

@@ -19,6 +19,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Ability to specify a compile-time message opcode expression: PR [#1188](https://github.com/tact-lang/tact/pull/1188)
 - The `VarInt16`, `VarInt32`, `VarUint16`, `VarUint32` integer serialization types: PR [#1186](https://github.com/tact-lang/tact/pull/1186)
 - `unboc`: a standalone CLI utility to expose Tact's TVM disassembler: PR [#1259](https://github.com/tact-lang/tact/pull/1259)
+- Added alternative parser: PR [#1258](https://github.com/tact-lang/tact/pull/1258)
 
 ### Changed
 

cspell.json

@@ -67,6 +67,7 @@
     "Korshakov",
     "Laika",
     "langle",
+    "langtools",
     "Liskov",
     "lparen",
     "lvalue",
@@ -87,6 +88,7 @@
     "Novus",
     "Offchain",
     "Parens",
+    "pgen",
     "pinst",
     "POSIX",
     "postpack",
@@ -117,6 +119,7 @@
     "Topup",
     "Toncoin",
     "Toncoins",
+    "tonstudio",
     "Trunov",
     "typechecker",
     "uintptr",
@@ -154,6 +157,8 @@
     "src/grammar/test/items-asm-funs.tact",
     "src/grammar/test-asm/*.tact",
     "src/grammar/test-failed/funcid-*.tact",
+    "src/grammar/next/grammar.gg",
+    "src/grammar/next/grammar.ts",
     "src/imports/stdlib.ts",
     "/src/test/compilation-failed/const-eval-failed.spec.ts",
     "src/test/e2e-emulated/address.spec.ts",

docs/src/content/docs/book/config.mdx

@@ -248,6 +248,35 @@ If set to `true{:json}`, enables generation of a [getter](/book/contracts#getter
 
 :::
 
+#### `parser` {#options-parser}
+
+`"new"{:json}` by default.
+
+If set to `new{:json}`, Tact will compile with a new language parser.
+
+If set to `old{:json}`, Tact will compile with an old language parser.
+
+```json filename="tact.config.json" {8,14}
+{
+  "projects": [
+    {
+      "name": "some_prefix",
+      "path": "./contract.tact",
+      "output": "./contract_output",
+      "options": {
+        "debug": true
+      }
+    },
+    {
+      "name": "ContractUnderBlueprint",
+      "options": {
+        "debug": true
+      }
+    }
+  ]
+}
+```
+
 #### `experimental` {#options-experimental}
 
 Experimental options that might be removed in the future. Use with caution!

knip.json

@@ -13,6 +13,7 @@
     "src/prettyPrinter.ts",
     "src/error/display-to-json.ts",
     "src/grammar/src-info.ts",
+    "src/grammar/next/grammar.ts",
     ".github/workflows/tact*.yml"
   ],
   "ignoreDependencies": ["@tact-lang/ton-abi"]

package.json

@@ -10,7 +10,9 @@
   "author": "Steve Korshakov <[email protected]>",
   "license": "MIT",
   "scripts": {
-    "gen:grammar": "ohm generateBundles --withTypes src/grammar/*.ohm",
+    "gen:grammar:old": "ohm generateBundles --withTypes src/grammar/prev/*.ohm",
+    "gen:grammar:new": "pgen src/grammar/next/grammar.gg -o src/grammar/next/grammar.ts",
+    "gen:grammar": "yarn gen:grammar:old && yarn gen:grammar:new",
     "gen:pack": "ts-node ./scripts/pack.ts",
     "gen:compiler": "ts-node ./scripts/prepare.ts",
     "gen": "yarn gen:grammar && yarn gen:pack && yarn gen:compiler",
@@ -50,6 +52,7 @@
     "@tact-lang/opcode": "^0.0.16",
     "@ton/core": "0.59.1",
     "@ton/crypto": "^3.2.0",
+    "@tonstudio/parser-runtime": "^0.0.1",
     "blockstore-core": "1.0.5",
     "change-case": "^4.1.2",
     "crc-32": "1.2.2",
@@ -70,6 +73,7 @@
     "@tact-lang/ton-jest": "^0.0.4",
     "@ton/sandbox": "^0.23.0",
     "@ton/test-utils": "^0.4.2",
+    "@tonstudio/pgen": "^0.0.1",
     "@types/glob": "^8.1.0",
     "@types/jest": "^29.5.12",
     "@types/json-bigint": "^1.0.4",

scripts/copy-files.ts

@@ -11,7 +11,7 @@ const cp = async (fromGlob: string, toPath: string) => {
 
 const main = async () => {
     try {
-        await cp("./src/grammar/grammar.ohm*", "./dist/grammar/");
+        await cp("./src/grammar/prev/grammar.ohm*", "./dist/grammar/prev/");
         await cp("./src/func/funcfiftlib.*", "./dist/func/");
     } catch (e) {
         console.error(e);

src/check.ts

@@ -5,6 +5,7 @@ import { getParser } from "./grammar";
 import files from "./imports/stdlib";
 import { createVirtualFileSystem, TactError, VirtualFileSystem } from "./main";
 import { precompile } from "./pipeline/precompile";
+import { defaultParser } from "./grammar/grammar";
 
 export type CheckResultItem = {
     type: "error" | "warning";
@@ -37,7 +38,7 @@ export function check(args: {
     ctx = featureEnable(ctx, "external"); // Enable external messages flag to avoid external-specific errors
 
     const ast = getAstFactory();
-    const parser = getParser(ast);
+    const parser = getParser(ast, defaultParser);
 
     // Execute check
     const items: CheckResultItem[] = [];

src/config/parseConfig.ts

@@ -27,6 +27,10 @@ export const optionsSchema = z
          * Read more: https://docs.tact-lang.org/book/contracts#interfaces
          */
         interfacesGetter: z.boolean().optional(),
+        /**
+         * If set to "new", uses new parser. If set to "old", uses legacy parser. Default is "old".
+         */
+        parser: z.union([z.literal("new"), z.literal("old")]).optional(),
         /**
          * Experimental options that might be removed in the future. Use with caution!
          */

src/generator/writers/resolveFuncType.spec.ts

@@ -5,6 +5,7 @@ import { resolveFuncType } from "./resolveFuncType";
 import { openContext } from "../../grammar/store";
 import { CompilerContext } from "../../context";
 import { getParser } from "../../grammar";
+import { defaultParser } from "../../grammar/grammar";
 
 const primitiveCode = `
 primitive Int;
@@ -52,7 +53,7 @@ describe("resolveFuncType", () => {
             new CompilerContext(),
             [{ code: primitiveCode, path: "<unknown>", origin: "user" }],
             [],
-            getParser(ast),
+            getParser(ast, defaultParser),
         );
         ctx = resolveDescriptors(ctx, ast);
         const wCtx = new WriterContext(ctx, "Contract1");
@@ -121,7 +122,7 @@ describe("resolveFuncType", () => {
             new CompilerContext(),
             [{ code: primitiveCode, path: "<unknown>", origin: "user" }],
             [],
-            getParser(ast),
+            getParser(ast, defaultParser),
         );
         ctx = resolveDescriptors(ctx, ast);
         const wCtx = new WriterContext(ctx, "Contract1");

src/generator/writers/writeExpression.spec.ts

@@ -9,6 +9,7 @@ import { resolveStatements } from "../../types/resolveStatements";
 import { CompilerContext } from "../../context";
 import { getParser } from "../../grammar";
 import { getAstFactory } from "../../grammar/ast";
+import { defaultParser } from "../../grammar/grammar";
 
 const code = `
 
@@ -75,7 +76,7 @@ describe("writeExpression", () => {
             new CompilerContext(),
             [{ code: code, path: "<unknown>", origin: "user" }],
             [],
-            getParser(ast),
+            getParser(ast, defaultParser),
         );
         ctx = resolveDescriptors(ctx, ast);
         ctx = resolveStatements(ctx);

src/generator/writers/writeSerialization.spec.ts

@@ -15,6 +15,7 @@ import { openContext } from "../../grammar/store";
 import { writeAccessors } from "./writeAccessors";
 import { getParser } from "../../grammar";
 import { getAstFactory } from "../../grammar/ast";
+import { defaultParser } from "../../grammar/grammar";
 
 const code = `
 primitive Int;
@@ -64,7 +65,7 @@ describe("writeSerialization", () => {
                 new CompilerContext(),
                 [{ code, path: "<unknown>", origin: "user" }],
                 [],
-                getParser(ast),
+                getParser(ast, defaultParser),
             );
             ctx = resolveDescriptors(ctx, ast);
             ctx = resolveAllocations(ctx);