Support parsing and pass-through of comments.

Author: Feni

Sources/Former/Justfile

@@ -37,3 +37,6 @@ dump:
 
 @compile input outfile:
     echo {{input}} | ./zig-out/bin/Former > {{outfile}}.wat && wat2wasm {{outfile}}.wat -o {{outfile}}.wasm
+
+@emit input:
+    echo {{input}} | ./zig-out/bin/Former

Sources/Former/src/parser.zig

@@ -160,7 +160,7 @@ pub const Parser = struct {
         // Comment - including the starting //.
         var start = self.index;
         self.index += 2;
-        self.seek_till("\n");
+        _ = self.seek_till("\n");
         var end = self.index;
         var commentRef = val.createReference(val.AST_COMMENT, 0);
         return ast.AstNode{ .value = commentRef, .loc = ast.Location{ .start = start, .end = end } };
@@ -222,15 +222,17 @@ pub const Parser = struct {
                     ' ', '\t' => self.skip(),
                     // '\n' => self.lex_block(),
                     // '"' => self.lex_string(),
-                    // '/' => {
-                    //     if (self.peek() == '/') {
-                    //         // TODO: Triple slash for doc comments.
-                    //         self.lex_comment();
-                    //     } else {
-                    //         // Division
-                    //         self.kw(val.KW_DIV, 1);
-                    //     }
-                    // },
+                    '/' => {
+                        if (self.peek() == '/') {
+                            // TODO: Triple slash for doc comments.
+                            var token = self.lex_comment();
+                            try self.insert_op(token);
+                        } else {
+                            // Division
+                            var token = self.kw(val.KW_DIV, 1);
+                            try self.insert_op(token);
+                        }
+                    },
                     '+' => {
                         var token = self.kw(val.KW_ADD, 1);
                         try self.insert_op(token);

Sources/Former/src/value.zig

@@ -37,8 +37,8 @@ const TAG_HEADER2: u16 = BASE_TYPE | 0x0002; // [110] 2 byte header. 4 byte data
 const TAG_HEADER3: u16 = BASE_TYPE | 0x0003; // [011] 3 byte header. 3 byte data.
 const TAG_HEADER4: u16 = BASE_TYPE | 0x0004; // [100] 4 byte header. 2 byte data.
 const TAG_HEADER5: u16 = BASE_TYPE | 0x0005; // [101] 5 byte header. 1 byte data.
-const TAG_INLINE_STRING: u16 = BASE_TYPE | 0x006; // [110] Inline string up to 5 ascii chars inline.
-const TAG_INLINE_BITSET: u16 = BASE_TYPE | 0x007; // [111] Inline bitset.
+const TAG_INLINE_STRING: u16 = BASE_TYPE | 0x0006; // [110] Inline string up to 5 ascii chars inline.
+const TAG_INLINE_BITSET: u16 = BASE_TYPE | 0x0007; // [111] Inline bitset.
 
 const TYPE_HEADER0: u64 = @as(u64, TAG_HEADER0) << 48;
 const TYPE_HEADER1: u64 = @as(u64, TAG_HEADER1) << 48;
@@ -77,13 +77,6 @@ pub const AST_STRING: u8 = 0x21; // 0x22 = '"'
 pub const AST_IDENTIFIER: u8 = 0x41; // 0x41 = 'A'
 pub const AST_COMMENT: u8 = 0x27; // 0x27 = '/'
 
-pub fn getTypeTag(val: u64) u64 {
-    return val & MASK_TYPE;
-}
-
-fn isPrimitiveType(comptime pattern: u64, val: u64) bool {
-    return (val & pattern) == pattern;
-}
 
 pub fn getHeader(val: u64) u64 {
     const header = switch (val & MASK_TYPE) {
@@ -109,6 +102,18 @@ pub fn getPayload(val: u64) u64 {
     return payload;
 }
 
+pub fn getHeader2(val: u64) u16 {
+    return @truncate(u16, (val & MASK_HIGH16) >> 32);
+}
+
+pub fn getTypeTag(val: u64) u64 {
+    return val & MASK_TYPE;
+}
+
+fn isPrimitiveType(comptime pattern: u64, val: u64) bool {
+    return (val & pattern) == pattern;
+}
+
 pub fn isNan(val: u64) bool {
     // Any NaN - quiet or signaling - either positive or negative.
     return isPrimitiveType(UNSIGNED_ANY_NAN, val);
@@ -123,6 +128,14 @@ pub fn isInlineString(val: u64) bool {
     return isPrimitiveType(TYPE_INLINE_STRING, val);
 }
 
+pub fn createHeader2(header: u16, payload: u32) u64 {
+    return TYPE_HEADER2 | (@as(u64, header) << 32) | @as(u64, payload);
+}
+
+pub fn createHeader1(header: u8, payload: u40) u64 {
+    return TYPE_HEADER1 | (@as(u64, header) << 40) | @as(u64, payload);
+}
+
 pub fn createInlineString(str: []const u8) u64 {
     // Inline small strings of up to 6 bytes.
     // The representation does reverse the order of bytes.
@@ -141,12 +154,16 @@ pub fn decodeInlineString(val: u64, out: *[8]u8) void {
 
 pub fn createKeyword(opcode: u8, precedence: u16) u64 {
     // Create a left-associative keyword
-    return TYPE_HEADER1 | @as(u64, opcode) << 40 | (@as(u64, precedence));
+    return createHeader1(opcode, precedence);
 }
 
 pub fn createKeywordRight(opcode: u8, precedence: u16) u64 {
     // Create a right-associative keyword. Top bit of precedence header is 1.
-    return TYPE_HEADER1 | @as(u64, opcode) << 40 | (@as(u64, precedence | 0x1000));
+    return createHeader1(opcode, precedence | 0x1000);
+}
+
+pub fn isKeyword(value: u64) bool {
+    return isPrimitiveType(TYPE_HEADER1, value);
 }
 
 pub fn getPrecedence(tok: u64) u16 {
@@ -161,12 +178,8 @@ pub fn isLeftAssociative(tok: u64) bool {
     return ((tok) & 0x1000) == 0;
 }
 
-pub fn createHeader2(header: u16, payload: u32) u64 {
-    return TYPE_HEADER2 | (@as(u64, header) << 32) | @as(u64, payload);
-}
-
-pub fn createReference(header: u16, payload: u32) u64 {
-    return createHeader2(header, payload);
+pub fn createReference(header: u8, payload: u40) u64 {
+    return createHeader1(header, payload);
 }
 
 pub const OP_ADD: u8 = 0;

Sources/Former/src/wasm.zig

@@ -1,5 +1,6 @@
 // Wasm backend.
 const Parser = @import("parser.zig").Parser;
+const Location = @import("ast.zig").Location;
 const val = @import("value.zig");
 const std = @import("std");
 const stdout = std.io.getStdOut().writer();
@@ -18,6 +19,10 @@ const header = \\(module
 const footer = \\))
 ;
 
+fn getLoc(parser: *Parser, loc: Location) []const u8 {
+    return parser.buffer[loc.start..loc.end];
+}
+
 pub fn emit(parser: *Parser) !void {
     var index: usize = 0;
     try stdout.print("{s}\n", .{header});
@@ -26,7 +31,11 @@ pub fn emit(parser: *Parser) !void {
 
         if (val.isNan(tok.value)) {
             const opcode = val.getOpcode(tok.value);
-            try stdout.print("{s}\n", .{operations[opcode]});
+            if(opcode < operations.len) {
+                try stdout.print("{s}\n", .{operations[opcode]});
+            } else {
+                try stdout.print(";; {s}\n", .{ getLoc(parser, tok.loc ) });
+            }
         } else {
             try stdout.print("f64.const {d}\n", .{tok.value});
         }

Tests/Former/build/test.wasm

@@ -1,6 +1,4 @@
 (module
   (func (export "_start") (result f64)
-f64.const 1
-f64.const 1
-f64.add
+;; 9219192793157599232
 ))

Tests/Former/build/test.wat

@@ -43,13 +43,27 @@ const compile = async (code) => {
 // }
 
 
-async function evaluate(code) {
-    compile(code)
+async function evaluate(expression) {
+    compile(expression)
     const wasm_exports = await load('Former/build/test.wasm');
     return wasm_exports._start();
 }
 
 
 test('Basic numeric expressions', async () => {
+    // Ensure mathematical operator precedence.
     expect(await evaluate("1 + 1")).toBe(2);
-});
+    expect(await evaluate("1 + 2 * 4")).toBe(9);
+    expect(await evaluate("1 * 2 + 3")).toBe(5);
+});
+
+test('Parentheses', async () => {
+    // Parentheses take precedence.
+    expect(await evaluate("(1 + 2) * 4")).toBe(12);
+    expect(await evaluate("4 * (2 + 3) + 7")).toBe(27);
+});
+
+test('Comments', async () => {
+    expect(await evaluate("// Hello")).toBe(12);
+    expect(await evaluate("4 * (2 + 3) + 7")).toBe(27);
+});