compiler: introduce ZonGen and make ast-check run it for ZON inputs

Currently, `zig ast-check` fails on ZON files, because it tries to
interpret the file as Zig source code. This commit introduces a new
verification pass, `std.zig.ZonGen`, which applies to an AST in ZON
mode.

Like `AstGen`, this pass also converts the AST into a more helpful
format. Rather than a sequence of instructions like `Zir`, the output
format of `ZonGen` is a new datastructure called `Zoir`. This type is
essentially a simpler form of AST, containing only the information
required for consumers of ZON. It is also far more compact than
`std.zig.Ast`, with the size generally being comparable to the size of
the well-formatted source file.

The emitted `Zoir` is currently not used aside from the `-t` option to
`ast-check` which causes it to be dumped to stdout. However, in future,
it can be used for comptime `@import` of ZON files, as well as for
simpler handling of files like `build.zig.zon`, and even by other parts
of the Zig Standard Library.

Resolves: #22078

Author: mlugg

lib/std/zig.zig

@@ -14,6 +14,8 @@ pub const isPrimitive = primitives.isPrimitive;
 pub const Ast = @import("zig/Ast.zig");
 pub const AstGen = @import("zig/AstGen.zig");
 pub const Zir = @import("zig/Zir.zig");
+pub const Zoir = @import("zig/Zoir.zig");
+pub const ZonGen = @import("zig/ZonGen.zig");
 pub const system = @import("zig/system.zig");
 pub const CrossTarget = @compileError("deprecated; use std.Target.Query");
 pub const BuiltinFn = @import("zig/BuiltinFn.zig");

lib/std/zig/AstGen.zig

@@ -130,6 +130,8 @@ fn appendRefsAssumeCapacity(astgen: *AstGen, refs: []const Zir.Inst.Ref) void {
 }
 
 pub fn generate(gpa: Allocator, tree: Ast) Allocator.Error!Zir {
+    assert(tree.mode == .zig);
+
     var arena = std.heap.ArenaAllocator.init(gpa);
     defer arena.deinit();
 
@@ -11413,83 +11415,7 @@ fn parseStrLit(
 
 fn failWithStrLitError(astgen: *AstGen, err: std.zig.string_literal.Error, token: Ast.TokenIndex, bytes: []const u8, offset: u32) InnerError {
     const raw_string = bytes[offset..];
-    switch (err) {
-        .invalid_escape_character => |bad_index| {
-            return astgen.failOff(
-                token,
-                offset + @as(u32, @intCast(bad_index)),
-                "invalid escape character: '{c}'",
-                .{raw_string[bad_index]},
-            );
-        },
-        .expected_hex_digit => |bad_index| {
-            return astgen.failOff(
-                token,
-                offset + @as(u32, @intCast(bad_index)),
-                "expected hex digit, found '{c}'",
-                .{raw_string[bad_index]},
-            );
-        },
-        .empty_unicode_escape_sequence => |bad_index| {
-            return astgen.failOff(
-                token,
-                offset + @as(u32, @intCast(bad_index)),
-                "empty unicode escape sequence",
-                .{},
-            );
-        },
-        .expected_hex_digit_or_rbrace => |bad_index| {
-            return astgen.failOff(
-                token,
-                offset + @as(u32, @intCast(bad_index)),
-                "expected hex digit or '}}', found '{c}'",
-                .{raw_string[bad_index]},
-            );
-        },
-        .invalid_unicode_codepoint => |bad_index| {
-            return astgen.failOff(
-                token,
-                offset + @as(u32, @intCast(bad_index)),
-                "unicode escape does not correspond to a valid unicode scalar value",
-                .{},
-            );
-        },
-        .expected_lbrace => |bad_index| {
-            return astgen.failOff(
-                token,
-                offset + @as(u32, @intCast(bad_index)),
-                "expected '{{', found '{c}",
-                .{raw_string[bad_index]},
-            );
-        },
-        .expected_rbrace => |bad_index| {
-            return astgen.failOff(
-                token,
-                offset + @as(u32, @intCast(bad_index)),
-                "expected '}}', found '{c}",
-                .{raw_string[bad_index]},
-            );
-        },
-        .expected_single_quote => |bad_index| {
-            return astgen.failOff(
-                token,
-                offset + @as(u32, @intCast(bad_index)),
-                "expected single quote ('), found '{c}",
-                .{raw_string[bad_index]},
-            );
-        },
-        .invalid_character => |bad_index| {
-            return astgen.failOff(
-                token,
-                offset + @as(u32, @intCast(bad_index)),
-                "invalid byte in string or character literal: '{c}'",
-                .{raw_string[bad_index]},
-            );
-        },
-        .empty_char_literal => {
-            return astgen.failOff(token, offset, "empty character literal", .{});
-        },
-    }
+    return err.lower(raw_string, offset, AstGen.failOff, .{ astgen, token });
 }
 
 fn failNode(
@@ -14019,30 +13945,40 @@ fn emitDbgStmtForceCurrentIndex(gz: *GenZir, lc: LineColumn) !void {
 }
 
 fn lowerAstErrors(astgen: *AstGen) !void {
+    const gpa = astgen.gpa;
     const tree = astgen.tree;
     assert(tree.errors.len > 0);
 
-    const gpa = astgen.gpa;
-    const parse_err = tree.errors[0];
-
     var msg: std.ArrayListUnmanaged(u8) = .empty;
     defer msg.deinit(gpa);
 
     var notes: std.ArrayListUnmanaged(u32) = .empty;
     defer notes.deinit(gpa);
 
-    for (tree.errors[1..]) |note| {
-        if (!note.is_note) break;
-
+    var cur_err = tree.errors[0];
+    for (tree.errors[1..]) |err| {
+        if (err.is_note) {
+            try tree.renderError(err, msg.writer(gpa));
+            try notes.append(gpa, try astgen.errNoteTok(err.token, "{s}", .{msg.items}));
+        } else {
+            // Flush error
+            const extra_offset = tree.errorOffset(cur_err);
+            try tree.renderError(cur_err, msg.writer(gpa));
+            try astgen.appendErrorTokNotesOff(cur_err.token, extra_offset, "{s}", .{msg.items}, notes.items);
+            notes.clearRetainingCapacity();
+            cur_err = err;
+
+            // TODO: `Parse` currently does not have good error recovery mechanisms, so the remaining errors could be bogus.
+            // As such, we'll ignore all remaining errors for now. We should improve `Parse` so that we can report all the errors.
+            return;
+        }
         msg.clearRetainingCapacity();
-        try tree.renderError(note, msg.writer(gpa));
-        try notes.append(gpa, try astgen.errNoteTok(note.token, "{s}", .{msg.items}));
     }
 
-    const extra_offset = tree.errorOffset(parse_err);
-    msg.clearRetainingCapacity();
-    try tree.renderError(parse_err, msg.writer(gpa));
-    try astgen.appendErrorTokNotesOff(parse_err.token, extra_offset, "{s}", .{msg.items}, notes.items);
+    // Flush error
+    const extra_offset = tree.errorOffset(cur_err);
+    try tree.renderError(cur_err, msg.writer(gpa));
+    try astgen.appendErrorTokNotesOff(cur_err.token, extra_offset, "{s}", .{msg.items}, notes.items);
 }
 
 const DeclarationName = union(enum) {

lib/std/zig/ErrorBundle.zig

@@ -507,7 +507,7 @@ pub const Wip = struct {
             }
 
             if (item.data.notes != 0) {
-                const notes_start = try eb.reserveNotes(item.data.notes);
+                const notes_start = try eb.reserveNotes(item.data.notesLen(zir));
                 const block = zir.extraData(Zir.Inst.Block, item.data.notes);
                 const body = zir.extra[block.end..][0..block.data.body_len];
                 for (notes_start.., body) |note_i, body_elem| {
@@ -547,6 +547,77 @@ pub const Wip = struct {
         }
     }
 
+    pub fn addZoirErrorMessages(
+        eb: *ErrorBundle.Wip,
+        zoir: std.zig.Zoir,
+        tree: std.zig.Ast,
+        source: [:0]const u8,
+        src_path: []const u8,
+    ) !void {
+        assert(zoir.hasCompileErrors());
+
+        for (zoir.compile_errors) |err| {
+            const err_span: std.zig.Ast.Span = span: {
+                if (err.token == std.zig.Zoir.CompileError.invalid_token) {
+                    break :span tree.nodeToSpan(err.node_or_offset);
+                }
+                const token_start = tree.tokens.items(.start)[err.token];
+                const start = token_start + err.node_or_offset;
+                const end = token_start + @as(u32, @intCast(tree.tokenSlice(err.token).len));
+                break :span .{ .start = start, .end = end, .main = start };
+            };
+            const err_loc = std.zig.findLineColumn(source, err_span.main);
+
+            try eb.addRootErrorMessage(.{
+                .msg = try eb.addString(err.msg.get(zoir)),
+                .src_loc = try eb.addSourceLocation(.{
+                    .src_path = try eb.addString(src_path),
+                    .span_start = err_span.start,
+                    .span_main = err_span.main,
+                    .span_end = err_span.end,
+                    .line = @intCast(err_loc.line),
+                    .column = @intCast(err_loc.column),
+                    .source_line = try eb.addString(err_loc.source_line),
+                }),
+                .notes_len = err.note_count,
+            });
+
+            const notes_start = try eb.reserveNotes(err.note_count);
+            for (notes_start.., err.first_note.., 0..err.note_count) |eb_note_idx, zoir_note_idx, _| {
+                const note = zoir.error_notes[zoir_note_idx];
+                const note_span: std.zig.Ast.Span = span: {
+                    if (note.token == std.zig.Zoir.CompileError.invalid_token) {
+                        break :span tree.nodeToSpan(note.node_or_offset);
+                    }
+                    const token_start = tree.tokens.items(.start)[note.token];
+                    const start = token_start + note.node_or_offset;
+                    const end = token_start + @as(u32, @intCast(tree.tokenSlice(note.token).len));
+                    break :span .{ .start = start, .end = end, .main = start };
+                };
+                const note_loc = std.zig.findLineColumn(source, note_span.main);
+
+                // This line can cause `wip.extra.items` to be resized.
+                const note_index = @intFromEnum(try eb.addErrorMessage(.{
+                    .msg = try eb.addString(note.msg.get(zoir)),
+                    .src_loc = try eb.addSourceLocation(.{
+                        .src_path = try eb.addString(src_path),
+                        .span_start = note_span.start,
+                        .span_main = note_span.main,
+                        .span_end = note_span.end,
+                        .line = @intCast(note_loc.line),
+                        .column = @intCast(note_loc.column),
+                        .source_line = if (note_loc.eql(err_loc))
+                            0
+                        else
+                            try eb.addString(note_loc.source_line),
+                    }),
+                    .notes_len = 0,
+                }));
+                eb.extra.items[eb_note_idx] = note_index;
+            }
+        }
+    }
+
     fn addOtherMessage(wip: *Wip, other: ErrorBundle, msg_index: MessageIndex) !MessageIndex {
         const other_msg = other.getErrorMessage(msg_index);
         const src_loc = try wip.addOtherSourceLocation(other, other_msg.src_loc);