From bfb8c614720f650f9cd2cce07b8a1d18864b20c7 Mon Sep 17 00:00:00 2001 From: Joshua Lochner <26504141+xenova@users.noreply.github.com> Date: Thu, 1 May 2025 23:38:34 -0400 Subject: [PATCH 01/35] Everything is an identifier --- packages/jinja/src/format.ts | 6 +- packages/jinja/src/index.ts | 12 +- packages/jinja/src/lexer.ts | 80 +-- packages/jinja/src/parser.ts | 206 ++++-- packages/jinja/src/runtime.ts | 27 +- packages/jinja/test/templates.test.js | 987 +++++++++++++++----------- 6 files changed, 750 insertions(+), 568 deletions(-) diff --git a/packages/jinja/src/format.ts b/packages/jinja/src/format.ts index 75260c138f..72d9f4feac 100644 --- a/packages/jinja/src/format.ts +++ b/packages/jinja/src/format.ts @@ -11,7 +11,6 @@ import type { Identifier, NumericLiteral, StringLiteral, - BooleanLiteral, ArrayLiteral, TupleLiteral, ObjectLiteral, @@ -170,11 +169,8 @@ function formatExpression(node: Expression, parentPrec: number = -1): string { switch (node.type) { case "Identifier": return (node as Identifier).value; - case "NullLiteral": - return "none"; case "NumericLiteral": - case "BooleanLiteral": - return `${(node as NumericLiteral | BooleanLiteral).value}`; + return `${(node as NumericLiteral).value}`; case "StringLiteral": return JSON.stringify((node as StringLiteral).value); case "BinaryExpression": { diff --git a/packages/jinja/src/index.ts b/packages/jinja/src/index.ts index 25676a06a1..10abc7d8a2 100644 --- a/packages/jinja/src/index.ts +++ b/packages/jinja/src/index.ts @@ -12,10 +12,9 @@ */ import { tokenize } from "./lexer"; import { parse } from "./parser"; -import { Environment, Interpreter } from "./runtime"; +import { Environment, Interpreter, setupGlobals } from "./runtime"; import type { Program } from "./ast"; import type { StringValue } from "./runtime"; -import { range } from "./utils"; import { format } from "./format"; export class Template { @@ -35,14 +34,7 @@ export class Template { render(items?: Record): string { // Create a new environment for this template const env = new Environment(); - - // Declare global variables - env.set("false", false); - env.set("true", true); - env.set("raise_exception", (args: string) => { - throw new Error(args); - }); - env.set("range", range); + setupGlobals(env); // Add user-defined variables if (items) { diff --git a/packages/jinja/src/lexer.ts b/packages/jinja/src/lexer.ts index 632fa19ed6..34fe33377d 100644 --- a/packages/jinja/src/lexer.ts +++ b/packages/jinja/src/lexer.ts @@ -5,10 +5,8 @@ export const TOKEN_TYPES = Object.freeze({ Text: "Text", // The text between Jinja statements or expressions NumericLiteral: "NumericLiteral", // e.g., 123 - BooleanLiteral: "BooleanLiteral", // true or false - NullLiteral: "NullLiteral", // none StringLiteral: "StringLiteral", // 'string' - Identifier: "Identifier", // Variables, functions, etc. + Identifier: "Identifier", // Variables, functions, statements, booleans, etc. Equals: "Equals", // = OpenParen: "OpenParen", // ( CloseParen: "CloseParen", // ) @@ -30,67 +28,10 @@ export const TOKEN_TYPES = Object.freeze({ MultiplicativeBinaryOperator: "MultiplicativeBinaryOperator", // * / % ComparisonBinaryOperator: "ComparisonBinaryOperator", // < > <= >= == != UnaryOperator: "UnaryOperator", // ! - + - - // Keywords - Set: "Set", - If: "If", - For: "For", - In: "In", - Is: "Is", - NotIn: "NotIn", - Else: "Else", - EndSet: "EndSet", - EndIf: "EndIf", - ElseIf: "ElseIf", - EndFor: "EndFor", - And: "And", - Or: "Or", - Not: "UnaryOperator", - Macro: "Macro", - EndMacro: "EndMacro", - Break: "Break", - Continue: "Continue", }); export type TokenType = keyof typeof TOKEN_TYPES; -/** - * Constant lookup for keywords and known identifiers + symbols. - */ -const KEYWORDS = Object.freeze({ - set: TOKEN_TYPES.Set, - for: TOKEN_TYPES.For, - in: TOKEN_TYPES.In, - is: TOKEN_TYPES.Is, - if: TOKEN_TYPES.If, - else: TOKEN_TYPES.Else, - endset: TOKEN_TYPES.EndSet, - endif: TOKEN_TYPES.EndIf, - elif: TOKEN_TYPES.ElseIf, - endfor: TOKEN_TYPES.EndFor, - and: TOKEN_TYPES.And, - or: TOKEN_TYPES.Or, - not: TOKEN_TYPES.Not, - "not in": TOKEN_TYPES.NotIn, - macro: TOKEN_TYPES.Macro, - endmacro: TOKEN_TYPES.EndMacro, - break: TOKEN_TYPES.Break, - continue: TOKEN_TYPES.Continue, - - // Literals - true: TOKEN_TYPES.BooleanLiteral, - false: TOKEN_TYPES.BooleanLiteral, - none: TOKEN_TYPES.NullLiteral, - - // NOTE: According to the Jinja docs: The special constants true, false, and none are indeed lowercase. - // Because that caused confusion in the past, (True used to expand to an undefined variable that was considered false), - // all three can now also be written in title case (True, False, and None). However, for consistency, (all Jinja identifiers are lowercase) - // you should use the lowercase versions. - True: TOKEN_TYPES.BooleanLiteral, - False: TOKEN_TYPES.BooleanLiteral, - None: TOKEN_TYPES.NullLiteral, -}); - /** * Represents a single token in the template. */ @@ -279,8 +220,6 @@ export function tokenize(source: string, options: PreprocessOptions = {}): Token switch (lastTokenType) { case TOKEN_TYPES.Identifier: case TOKEN_TYPES.NumericLiteral: - case TOKEN_TYPES.BooleanLiteral: - case TOKEN_TYPES.NullLiteral: case TOKEN_TYPES.StringLiteral: case TOKEN_TYPES.CloseParen: case TOKEN_TYPES.CloseSquareBracket: @@ -328,22 +267,9 @@ export function tokenize(source: string, options: PreprocessOptions = {}): Token continue; } if (isWord(char)) { + // consume any word characters and always classify as Identifier const word = consumeWhile(isWord); - - // Check for special/reserved keywords - // NOTE: We use Object.hasOwn() to avoid matching `.toString()` and other Object methods - const type = Object.hasOwn(KEYWORDS, word) ? KEYWORDS[word as keyof typeof KEYWORDS] : TOKEN_TYPES.Identifier; - - // Special case of not in: - // If the previous token was a "not", and this token is "in" - // then we want to combine them into a single token - if (type === TOKEN_TYPES.In && tokens.at(-1)?.type === TOKEN_TYPES.Not) { - tokens.pop(); - tokens.push(new Token("not in", TOKEN_TYPES.NotIn)); - } else { - tokens.push(new Token(word, type)); - } - + tokens.push(new Token(word, TOKEN_TYPES.Identifier)); continue; } diff --git a/packages/jinja/src/parser.ts b/packages/jinja/src/parser.ts index 5160f9e8ba..98a67f0021 100644 --- a/packages/jinja/src/parser.ts +++ b/packages/jinja/src/parser.ts @@ -1,5 +1,5 @@ -import type { Token, TokenType } from "./lexer"; -import { TOKEN_TYPES } from "./lexer"; +import { Token, TOKEN_TYPES } from "./lexer"; +import type { TokenType } from "./lexer"; import type { Statement } from "./ast"; import { Program, @@ -13,8 +13,6 @@ import { Identifier, NumericLiteral, StringLiteral, - BooleanLiteral, - NullLiteral, ArrayLiteral, ObjectLiteral, BinaryExpression, @@ -76,52 +74,66 @@ export function parse(tokens: Token[]): Program { } function parseJinjaStatement(): Statement { - // Consume {% %} tokens + // Consume {% token expect(TOKEN_TYPES.OpenStatement, "Expected opening statement token"); - let result; - switch (tokens[current].type) { - case TOKEN_TYPES.Set: + // next token must be Identifier whose .value tells us which statement + if (tokens[current].type !== TOKEN_TYPES.Identifier) { + throw new SyntaxError(`Unknown statement, got ${tokens[current].type}`); + } + const name = tokens[current].value; + let result: Statement; + switch (name) { + case "set": ++current; result = parseSetStatement(); expect(TOKEN_TYPES.CloseStatement, "Expected closing statement token"); break; - - case TOKEN_TYPES.If: + case "if": ++current; result = parseIfStatement(); + // expect {% endif %} expect(TOKEN_TYPES.OpenStatement, "Expected {% token"); - expect(TOKEN_TYPES.EndIf, "Expected endif token"); + // ensure identifier 'endif' + if (tokens[current].type !== TOKEN_TYPES.Identifier || tokens[current].value !== "endif") { + throw new SyntaxError("Expected endif token"); + } + ++current; expect(TOKEN_TYPES.CloseStatement, "Expected %} token"); break; - - case TOKEN_TYPES.Macro: + case "macro": ++current; result = parseMacroStatement(); expect(TOKEN_TYPES.OpenStatement, "Expected {% token"); - expect(TOKEN_TYPES.EndMacro, "Expected endmacro token"); + if (tokens[current].type !== TOKEN_TYPES.Identifier || tokens[current].value !== "endmacro") { + throw new SyntaxError("Expected endmacro token"); + } + ++current; expect(TOKEN_TYPES.CloseStatement, "Expected %} token"); break; - - case TOKEN_TYPES.For: + case "for": ++current; result = parseForStatement(); expect(TOKEN_TYPES.OpenStatement, "Expected {% token"); - expect(TOKEN_TYPES.EndFor, "Expected endfor token"); + if (tokens[current].type !== TOKEN_TYPES.Identifier || tokens[current].value !== "endfor") { + throw new SyntaxError("Expected endfor token"); + } + ++current; expect(TOKEN_TYPES.CloseStatement, "Expected %} token"); break; - case TOKEN_TYPES.Break: + + case "break": ++current; expect(TOKEN_TYPES.CloseStatement, "Expected closing statement token"); result = new Break(); break; - case TOKEN_TYPES.Continue: + case "continue": ++current; expect(TOKEN_TYPES.CloseStatement, "Expected closing statement token"); result = new Continue(); break; default: - throw new SyntaxError(`Unknown statement type: ${tokens[current].type}`); + throw new SyntaxError(`Unknown statement type: ${name}`); } return result; @@ -151,13 +163,20 @@ export function parse(tokens: Token[]): Program { const body: Statement[] = []; expect(TOKEN_TYPES.CloseStatement, "Expected %} token"); while ( - !(tokens[current]?.type === TOKEN_TYPES.OpenStatement && tokens[current + 1]?.type === TOKEN_TYPES.EndSet) + !( + tokens[current]?.type === TOKEN_TYPES.OpenStatement && + tokens[current + 1]?.type === TOKEN_TYPES.Identifier && + tokens[current + 1]?.value === "endset" + ) ) { const another = parseAny(); body.push(another); } expect(TOKEN_TYPES.OpenStatement, "Expected {% token"); - expect(TOKEN_TYPES.EndSet, "Expected endset token"); + if (tokens[current]?.type !== TOKEN_TYPES.Identifier || tokens[current]?.value !== "endset") { + throw new SyntaxError("Expected endset token"); + } + ++current; return new SetStatement(left, null, body); } @@ -171,38 +190,47 @@ export function parse(tokens: Token[]): Program { const body: Statement[] = []; const alternate: Statement[] = []; - // Keep parsing if body until we reach the first {% elif %} or {% else %} or {% endif %} + // Keep parsing 'if' body until we reach the first {% elif %} or {% else %} or {% endif %} while ( !( tokens[current]?.type === TOKEN_TYPES.OpenStatement && - (tokens[current + 1]?.type === TOKEN_TYPES.ElseIf || - tokens[current + 1]?.type === TOKEN_TYPES.Else || - tokens[current + 1]?.type === TOKEN_TYPES.EndIf) + tokens[current + 1]?.type === TOKEN_TYPES.Identifier && + ["elif", "else", "endif"].includes(tokens[current + 1].value) ) ) { body.push(parseAny()); } - // Alternate branch: Check for {% elif %} or {% else %} + // handle {% elif %} if ( tokens[current]?.type === TOKEN_TYPES.OpenStatement && - tokens[current + 1]?.type !== TOKEN_TYPES.EndIf // There is some body + tokens[current + 1]?.type === TOKEN_TYPES.Identifier && + tokens[current + 1].value === "elif" ) { - ++current; // eat {% token - if (is(TOKEN_TYPES.ElseIf)) { - expect(TOKEN_TYPES.ElseIf, "Expected elseif token"); - alternate.push(parseIfStatement()); - } else { - // tokens[current]?.type === TokenType.Else - expect(TOKEN_TYPES.Else, "Expected else token"); - expect(TOKEN_TYPES.CloseStatement, "Expected closing statement token"); + ++current; // consume {% + ++current; // consume 'elif' + const result = parseIfStatement(); // nested If + alternate.push(result); + } + // handle {% else %} + else if ( + tokens[current]?.type === TOKEN_TYPES.OpenStatement && + tokens[current + 1]?.type === TOKEN_TYPES.Identifier && + tokens[current + 1].value === "else" + ) { + ++current; // consume {% + ++current; // consume 'else' + expect(TOKEN_TYPES.CloseStatement, "Expected closing statement token"); - // keep going until we hit {% endif %} - while ( - !(tokens[current]?.type === TOKEN_TYPES.OpenStatement && tokens[current + 1]?.type === TOKEN_TYPES.EndIf) - ) { - alternate.push(parseAny()); - } + // keep going until we hit {% endif %} + while ( + !( + tokens[current]?.type === TOKEN_TYPES.OpenStatement && + tokens[current + 1]?.type === TOKEN_TYPES.Identifier && + tokens[current + 1].value === "endif" + ) + ) { + alternate.push(parseAny()); } } @@ -221,7 +249,7 @@ export function parse(tokens: Token[]): Program { const body: Statement[] = []; // Keep going until we hit {% endmacro - while (not(TOKEN_TYPES.OpenStatement, TOKEN_TYPES.EndMacro)) { + while (not(TOKEN_TYPES.OpenStatement, TOKEN_TYPES.Identifier) || tokens[current + 1]?.value !== "endmacro") { body.push(parseAny()); } @@ -249,7 +277,10 @@ export function parse(tokens: Token[]): Program { throw new SyntaxError(`Expected identifier/tuple for the loop variable, got ${loopVariable.type} instead`); } - expect(TOKEN_TYPES.In, "Expected `in` keyword following loop variable"); + if (!(tokens[current].type === TOKEN_TYPES.Identifier && tokens[current].value === "in")) { + throw new SyntaxError("Expected `in` keyword following loop variable"); + } + ++current; // `messages` in `for message in messages` const iterable = parseExpression(); @@ -260,19 +291,34 @@ export function parse(tokens: Token[]): Program { const body: Statement[] = []; // Keep going until we hit {% endfor or {% else - while (not(TOKEN_TYPES.OpenStatement, TOKEN_TYPES.EndFor) && not(TOKEN_TYPES.OpenStatement, TOKEN_TYPES.Else)) { + while ( + !( + tokens[current]?.type === TOKEN_TYPES.OpenStatement && + tokens[current + 1]?.type === TOKEN_TYPES.Identifier && + ["endfor", "else"].includes(tokens[current + 1].value) + ) + ) { body.push(parseAny()); } // (Optional) else block const alternative: Statement[] = []; - if (is(TOKEN_TYPES.OpenStatement, TOKEN_TYPES.Else)) { + if ( + tokens[current]?.type === TOKEN_TYPES.OpenStatement && + tokens[current + 1]?.type === TOKEN_TYPES.Identifier && + tokens[current + 1].value === "else" + ) { ++current; // consume {% - ++current; // consume else + ++current; // consume 'else' expect(TOKEN_TYPES.CloseStatement, "Expected closing statement token"); - - // keep going until we hit {% endfor - while (not(TOKEN_TYPES.OpenStatement, TOKEN_TYPES.EndFor)) { + while ( + // keep going until we hit {% endfor + !( + tokens[current]?.type === TOKEN_TYPES.OpenStatement && + tokens[current + 1]?.type === TOKEN_TYPES.Identifier && + tokens[current + 1].value === "endfor" + ) + ) { alternative.push(parseAny()); } } @@ -287,14 +333,14 @@ export function parse(tokens: Token[]): Program { function parseIfExpression(): Statement { const a = parseLogicalOrExpression(); - if (is(TOKEN_TYPES.If)) { + if (tokens[current].type === TOKEN_TYPES.Identifier && tokens[current].value === "if") { // Ternary expression - ++current; // consume if + ++current; // consume 'if' const predicate = parseLogicalOrExpression(); - if (is(TOKEN_TYPES.Else)) { + if (tokens[current].type === TOKEN_TYPES.Identifier && tokens[current].value === "else") { // Ternary expression with else - ++current; // consume else + ++current; // consume 'else' const b = parseLogicalOrExpression(); return new If(predicate, [a], [b]); } else { @@ -307,7 +353,7 @@ export function parse(tokens: Token[]): Program { function parseLogicalOrExpression(): Statement { let left = parseLogicalAndExpression(); - while (is(TOKEN_TYPES.Or)) { + while (tokens[current].type === TOKEN_TYPES.Identifier && tokens[current].value === "or") { const operator = tokens[current]; ++current; const right = parseLogicalAndExpression(); @@ -318,7 +364,7 @@ export function parse(tokens: Token[]): Program { function parseLogicalAndExpression(): Statement { let left = parseLogicalNegationExpression(); - while (is(TOKEN_TYPES.And)) { + while (tokens[current].type === TOKEN_TYPES.Identifier && tokens[current].value === "and") { const operator = tokens[current]; ++current; const right = parseLogicalNegationExpression(); @@ -331,7 +377,7 @@ export function parse(tokens: Token[]): Program { let right: UnaryExpression | undefined; // Try parse unary operators - while (is(TOKEN_TYPES.Not)) { + while (tokens[current].type === TOKEN_TYPES.Identifier && tokens[current].value === "not") { // not not ... const operator = tokens[current]; ++current; @@ -346,9 +392,33 @@ export function parse(tokens: Token[]): Program { // NOTE: membership has same precedence as comparison // e.g., ('a' in 'apple' == 'b' in 'banana') evaluates as ('a' in ('apple' == ('b' in 'banana'))) let left = parseAdditiveExpression(); - while (is(TOKEN_TYPES.ComparisonBinaryOperator) || is(TOKEN_TYPES.In) || is(TOKEN_TYPES.NotIn)) { - const operator = tokens[current]; - ++current; + while ( + is(TOKEN_TYPES.ComparisonBinaryOperator) || + (tokens[current].type === TOKEN_TYPES.Identifier && tokens[current].value === "in") || + (tokens[current].type === TOKEN_TYPES.Identifier && + tokens[current].value === "not" && + tokens[current + 1]?.type === TOKEN_TYPES.Identifier && + tokens[current + 1]?.value === "in") + ) { + let operator: Token; + // handle 'not in' + if ( + tokens[current].type === TOKEN_TYPES.Identifier && + tokens[current].value === "not" && + tokens[current + 1]?.type === TOKEN_TYPES.Identifier && + tokens[current + 1]?.value === "in" + ) { + operator = new Token("not in", TOKEN_TYPES.Identifier); + current += 2; + } + // handle 'in' + else if (tokens[current].type === TOKEN_TYPES.Identifier && tokens[current].value === "in") { + operator = tokens[current++]; + } + // regular comparison operator + else { + operator = tokens[current++]; + } const right = parseAdditiveExpression(); left = new BinaryExpression(operator, left, right); } @@ -500,21 +570,15 @@ export function parse(tokens: Token[]): Program { function parseTestExpression(): Statement { let operand = parseFilterExpression(); - while (is(TOKEN_TYPES.Is)) { + while (tokens[current].type === TOKEN_TYPES.Identifier && tokens[current].value === "is") { // Support chaining tests ++current; // consume is - const negate = is(TOKEN_TYPES.Not); + const negate = tokens[current].type === TOKEN_TYPES.Identifier && tokens[current].value === "not"; if (negate) { ++current; // consume not } let filter = parsePrimaryExpression(); - if (filter instanceof BooleanLiteral) { - // Special case: treat boolean literals as identifiers - filter = new Identifier(filter.value.toString()); - } else if (filter instanceof NullLiteral) { - filter = new Identifier("none"); - } if (!(filter instanceof Identifier)) { throw new SyntaxError(`Expected identifier for the test`); } @@ -552,12 +616,6 @@ export function parse(tokens: Token[]): Program { case TOKEN_TYPES.StringLiteral: ++current; return new StringLiteral(token.value); - case TOKEN_TYPES.BooleanLiteral: - ++current; - return new BooleanLiteral(token.value.toLowerCase() === "true"); - case TOKEN_TYPES.NullLiteral: - ++current; - return new NullLiteral(null); case TOKEN_TYPES.Identifier: ++current; return new Identifier(token.value); diff --git a/packages/jinja/src/runtime.ts b/packages/jinja/src/runtime.ts index 5d19ddfa82..80c91715e9 100644 --- a/packages/jinja/src/runtime.ts +++ b/packages/jinja/src/runtime.ts @@ -1,8 +1,6 @@ import type { NumericLiteral, StringLiteral, - BooleanLiteral, - NullLiteral, ArrayLiteral, Statement, Program, @@ -24,7 +22,7 @@ import type { Expression, SelectExpression, } from "./ast"; -import { slice, titleCase } from "./utils"; +import { range, slice, titleCase } from "./utils"; export type AnyRuntimeValue = | NumericValue @@ -428,6 +426,25 @@ export class Environment { } } +export function setupGlobals(env: Environment) { + // Declare global variables + env.set("false", false); + env.set("true", true); + env.set("none", null); + env.set("raise_exception", (args: string) => { + throw new Error(args); + }); + env.set("range", range); + + // NOTE: According to the Jinja docs: The special constants true, false, and none are indeed lowercase. + // Because that caused confusion in the past, (True used to expand to an undefined variable that was considered false), + // all three can now also be written in title case (True, False, and None). However, for consistency, (all Jinja identifiers are lowercase) + // you should use the lowercase versions. + env.set("True", true); + env.set("False", false); + env.set("None", null); +} + export class Interpreter { global: Environment; @@ -1163,10 +1180,6 @@ export class Interpreter { return new NumericValue(Number((statement as NumericLiteral).value)); case "StringLiteral": return new StringValue((statement as StringLiteral).value); - case "BooleanLiteral": - return new BooleanValue((statement as BooleanLiteral).value); - case "NullLiteral": - return new NullValue((statement as NullLiteral).value); case "ArrayLiteral": return new ArrayValue((statement as ArrayLiteral).value.map((x) => this.evaluate(x, environment))); case "TupleLiteral": diff --git a/packages/jinja/test/templates.test.js b/packages/jinja/test/templates.test.js index 44207df78b..9787e5d1ca 100644 --- a/packages/jinja/test/templates.test.js +++ b/packages/jinja/test/templates.test.js @@ -3,7 +3,7 @@ import { describe, expect, it } from "vitest"; import { Template } from "../src/index"; import { tokenize } from "../src/lexer"; import { parse } from "../src/parser"; -import { Environment, Interpreter } from "../src/runtime"; +import { Environment, Interpreter, setupGlobals } from "../src/runtime"; const TEST_STRINGS = { // Text nodes @@ -196,19 +196,19 @@ const TEST_PARSED = { BOOLEAN_LITERALS: [ { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "true", type: "BooleanLiteral" }, + { value: "true", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "false", type: "BooleanLiteral" }, + { value: "false", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "True", type: "BooleanLiteral" }, + { value: "True", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "False", type: "BooleanLiteral" }, + { value: "False", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, ], @@ -216,157 +216,157 @@ const TEST_PARSED = { // Logical operators LOGICAL_AND: [ { value: "{{", type: "OpenExpression" }, - { value: "true", type: "BooleanLiteral" }, - { value: "and", type: "And" }, - { value: "true", type: "BooleanLiteral" }, + { value: "true", type: "Identifier" }, + { value: "and", type: "Identifier" }, + { value: "true", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "{{", type: "OpenExpression" }, - { value: "true", type: "BooleanLiteral" }, - { value: "and", type: "And" }, - { value: "false", type: "BooleanLiteral" }, + { value: "true", type: "Identifier" }, + { value: "and", type: "Identifier" }, + { value: "false", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "{{", type: "OpenExpression" }, - { value: "false", type: "BooleanLiteral" }, - { value: "and", type: "And" }, - { value: "true", type: "BooleanLiteral" }, + { value: "false", type: "Identifier" }, + { value: "and", type: "Identifier" }, + { value: "true", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "{{", type: "OpenExpression" }, - { value: "false", type: "BooleanLiteral" }, - { value: "and", type: "And" }, - { value: "false", type: "BooleanLiteral" }, + { value: "false", type: "Identifier" }, + { value: "and", type: "Identifier" }, + { value: "false", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, ], LOGICAL_OR: [ { value: "{{", type: "OpenExpression" }, - { value: "true", type: "BooleanLiteral" }, - { value: "or", type: "Or" }, - { value: "true", type: "BooleanLiteral" }, + { value: "true", type: "Identifier" }, + { value: "or", type: "Identifier" }, + { value: "true", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "{{", type: "OpenExpression" }, - { value: "true", type: "BooleanLiteral" }, - { value: "or", type: "Or" }, - { value: "false", type: "BooleanLiteral" }, + { value: "true", type: "Identifier" }, + { value: "or", type: "Identifier" }, + { value: "false", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "{{", type: "OpenExpression" }, - { value: "false", type: "BooleanLiteral" }, - { value: "or", type: "Or" }, - { value: "true", type: "BooleanLiteral" }, + { value: "false", type: "Identifier" }, + { value: "or", type: "Identifier" }, + { value: "true", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "{{", type: "OpenExpression" }, - { value: "false", type: "BooleanLiteral" }, - { value: "or", type: "Or" }, - { value: "false", type: "BooleanLiteral" }, + { value: "false", type: "Identifier" }, + { value: "or", type: "Identifier" }, + { value: "false", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, ], LOGICAL_NOT: [ { value: "{{", type: "OpenExpression" }, - { value: "not", type: "UnaryOperator" }, - { value: "true", type: "BooleanLiteral" }, + { value: "not", type: "Identifier" }, + { value: "true", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "{{", type: "OpenExpression" }, - { value: "not", type: "UnaryOperator" }, - { value: "false", type: "BooleanLiteral" }, + { value: "not", type: "Identifier" }, + { value: "false", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, ], LOGICAL_NOT_NOT: [ { value: "{{", type: "OpenExpression" }, - { value: "not", type: "UnaryOperator" }, - { value: "not", type: "UnaryOperator" }, - { value: "true", type: "BooleanLiteral" }, + { value: "not", type: "Identifier" }, + { value: "not", type: "Identifier" }, + { value: "true", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "{{", type: "OpenExpression" }, - { value: "not", type: "UnaryOperator" }, - { value: "not", type: "UnaryOperator" }, - { value: "false", type: "BooleanLiteral" }, + { value: "not", type: "Identifier" }, + { value: "not", type: "Identifier" }, + { value: "false", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, ], LOGICAL_AND_OR: [ { value: "{{", type: "OpenExpression" }, - { value: "true", type: "BooleanLiteral" }, - { value: "and", type: "And" }, - { value: "true", type: "BooleanLiteral" }, - { value: "or", type: "Or" }, - { value: "false", type: "BooleanLiteral" }, + { value: "true", type: "Identifier" }, + { value: "and", type: "Identifier" }, + { value: "true", type: "Identifier" }, + { value: "or", type: "Identifier" }, + { value: "false", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "{{", type: "OpenExpression" }, - { value: "true", type: "BooleanLiteral" }, - { value: "and", type: "And" }, - { value: "false", type: "BooleanLiteral" }, - { value: "or", type: "Or" }, - { value: "true", type: "BooleanLiteral" }, + { value: "true", type: "Identifier" }, + { value: "and", type: "Identifier" }, + { value: "false", type: "Identifier" }, + { value: "or", type: "Identifier" }, + { value: "true", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "{{", type: "OpenExpression" }, - { value: "false", type: "BooleanLiteral" }, - { value: "and", type: "And" }, - { value: "true", type: "BooleanLiteral" }, - { value: "or", type: "Or" }, - { value: "true", type: "BooleanLiteral" }, + { value: "false", type: "Identifier" }, + { value: "and", type: "Identifier" }, + { value: "true", type: "Identifier" }, + { value: "or", type: "Identifier" }, + { value: "true", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "{{", type: "OpenExpression" }, - { value: "false", type: "BooleanLiteral" }, - { value: "and", type: "And" }, - { value: "false", type: "BooleanLiteral" }, - { value: "or", type: "Or" }, - { value: "true", type: "BooleanLiteral" }, + { value: "false", type: "Identifier" }, + { value: "and", type: "Identifier" }, + { value: "false", type: "Identifier" }, + { value: "or", type: "Identifier" }, + { value: "true", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "{{", type: "OpenExpression" }, - { value: "false", type: "BooleanLiteral" }, - { value: "and", type: "And" }, - { value: "false", type: "BooleanLiteral" }, - { value: "or", type: "Or" }, - { value: "false", type: "BooleanLiteral" }, + { value: "false", type: "Identifier" }, + { value: "and", type: "Identifier" }, + { value: "false", type: "Identifier" }, + { value: "or", type: "Identifier" }, + { value: "false", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, ], LOGICAL_AND_NOT: [ { value: "{{", type: "OpenExpression" }, - { value: "true", type: "BooleanLiteral" }, - { value: "and", type: "And" }, - { value: "not", type: "UnaryOperator" }, - { value: "true", type: "BooleanLiteral" }, + { value: "true", type: "Identifier" }, + { value: "and", type: "Identifier" }, + { value: "not", type: "Identifier" }, + { value: "true", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "{{", type: "OpenExpression" }, - { value: "true", type: "BooleanLiteral" }, - { value: "and", type: "And" }, - { value: "not", type: "UnaryOperator" }, - { value: "false", type: "BooleanLiteral" }, + { value: "true", type: "Identifier" }, + { value: "and", type: "Identifier" }, + { value: "not", type: "Identifier" }, + { value: "false", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "{{", type: "OpenExpression" }, - { value: "false", type: "BooleanLiteral" }, - { value: "and", type: "And" }, - { value: "not", type: "UnaryOperator" }, - { value: "true", type: "BooleanLiteral" }, + { value: "false", type: "Identifier" }, + { value: "and", type: "Identifier" }, + { value: "not", type: "Identifier" }, + { value: "true", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "{{", type: "OpenExpression" }, - { value: "false", type: "BooleanLiteral" }, - { value: "and", type: "And" }, - { value: "not", type: "UnaryOperator" }, - { value: "false", type: "BooleanLiteral" }, + { value: "false", type: "Identifier" }, + { value: "and", type: "Identifier" }, + { value: "not", type: "Identifier" }, + { value: "false", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, ], LOGICAL_OR_NOT: [ { value: "{{", type: "OpenExpression" }, - { value: "true", type: "BooleanLiteral" }, - { value: "or", type: "Or" }, - { value: "not", type: "UnaryOperator" }, - { value: "true", type: "BooleanLiteral" }, + { value: "true", type: "Identifier" }, + { value: "or", type: "Identifier" }, + { value: "not", type: "Identifier" }, + { value: "true", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "{{", type: "OpenExpression" }, - { value: "true", type: "BooleanLiteral" }, - { value: "or", type: "Or" }, - { value: "not", type: "UnaryOperator" }, - { value: "false", type: "BooleanLiteral" }, + { value: "true", type: "Identifier" }, + { value: "or", type: "Identifier" }, + { value: "not", type: "Identifier" }, + { value: "false", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "{{", type: "OpenExpression" }, - { value: "false", type: "BooleanLiteral" }, - { value: "or", type: "Or" }, - { value: "not", type: "UnaryOperator" }, - { value: "true", type: "BooleanLiteral" }, + { value: "false", type: "Identifier" }, + { value: "or", type: "Identifier" }, + { value: "not", type: "Identifier" }, + { value: "true", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "{{", type: "OpenExpression" }, - { value: "false", type: "BooleanLiteral" }, - { value: "or", type: "Or" }, - { value: "not", type: "UnaryOperator" }, - { value: "false", type: "BooleanLiteral" }, + { value: "false", type: "Identifier" }, + { value: "or", type: "Identifier" }, + { value: "not", type: "Identifier" }, + { value: "false", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, ], LOGICAL_COMBINED: [ @@ -374,7 +374,7 @@ const TEST_PARSED = { { value: "1", type: "NumericLiteral" }, { value: "==", type: "ComparisonBinaryOperator" }, { value: "2", type: "NumericLiteral" }, - { value: "and", type: "And" }, + { value: "and", type: "Identifier" }, { value: "2", type: "NumericLiteral" }, { value: "==", type: "ComparisonBinaryOperator" }, { value: "2", type: "NumericLiteral" }, @@ -383,7 +383,7 @@ const TEST_PARSED = { { value: "1", type: "NumericLiteral" }, { value: "==", type: "ComparisonBinaryOperator" }, { value: "2", type: "NumericLiteral" }, - { value: "or", type: "Or" }, + { value: "or", type: "Identifier" }, { value: "2", type: "NumericLiteral" }, { value: "==", type: "ComparisonBinaryOperator" }, { value: "2", type: "NumericLiteral" }, @@ -393,7 +393,7 @@ const TEST_PARSED = { // If statements IF_ONLY: [ { value: "{%", type: "OpenStatement" }, - { value: "if", type: "If" }, + { value: "if", type: "Identifier" }, { value: "1", type: "NumericLiteral" }, { value: "==", type: "ComparisonBinaryOperator" }, { value: "1", type: "NumericLiteral" }, @@ -402,7 +402,7 @@ const TEST_PARSED = { { value: "A", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "endif", type: "EndIf" }, + { value: "endif", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "{{", type: "OpenExpression" }, { value: "B", type: "StringLiteral" }, @@ -410,7 +410,7 @@ const TEST_PARSED = { ], IF_ELSE_ONLY: [ { value: "{%", type: "OpenStatement" }, - { value: "if", type: "If" }, + { value: "if", type: "Identifier" }, { value: "1", type: "NumericLiteral" }, { value: "==", type: "ComparisonBinaryOperator" }, { value: "2", type: "NumericLiteral" }, @@ -419,13 +419,13 @@ const TEST_PARSED = { { value: "A", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "else", type: "Else" }, + { value: "else", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "{{", type: "OpenExpression" }, { value: "B", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "endif", type: "EndIf" }, + { value: "endif", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "{{", type: "OpenExpression" }, { value: "C", type: "StringLiteral" }, @@ -433,7 +433,7 @@ const TEST_PARSED = { ], IF_ELIF_ELSE: [ { value: "{%", type: "OpenStatement" }, - { value: "if", type: "If" }, + { value: "if", type: "Identifier" }, { value: "1", type: "NumericLiteral" }, { value: "==", type: "ComparisonBinaryOperator" }, { value: "2", type: "NumericLiteral" }, @@ -448,7 +448,7 @@ const TEST_PARSED = { { value: "C", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "elif", type: "ElseIf" }, + { value: "elif", type: "Identifier" }, { value: "1", type: "NumericLiteral" }, { value: "==", type: "ComparisonBinaryOperator" }, { value: "2", type: "NumericLiteral" }, @@ -457,7 +457,7 @@ const TEST_PARSED = { { value: "D", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "elif", type: "ElseIf" }, + { value: "elif", type: "Identifier" }, { value: "1", type: "NumericLiteral" }, { value: "==", type: "ComparisonBinaryOperator" }, { value: "3", type: "NumericLiteral" }, @@ -469,7 +469,7 @@ const TEST_PARSED = { { value: "F", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "else", type: "Else" }, + { value: "else", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "{{", type: "OpenExpression" }, { value: "G", type: "StringLiteral" }, @@ -481,7 +481,7 @@ const TEST_PARSED = { { value: "I", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "endif", type: "EndIf" }, + { value: "endif", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "{{", type: "OpenExpression" }, { value: "J", type: "StringLiteral" }, @@ -489,70 +489,70 @@ const TEST_PARSED = { ], NESTED_STATEMENTS: [ { value: "{%", type: "OpenStatement" }, - { value: "set", type: "Set" }, + { value: "set", type: "Identifier" }, { value: "a", type: "Identifier" }, { value: "=", type: "Equals" }, { value: "0", type: "NumericLiteral" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "set", type: "Set" }, + { value: "set", type: "Identifier" }, { value: "b", type: "Identifier" }, { value: "=", type: "Equals" }, { value: "0", type: "NumericLiteral" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "set", type: "Set" }, + { value: "set", type: "Identifier" }, { value: "c", type: "Identifier" }, { value: "=", type: "Equals" }, { value: "0", type: "NumericLiteral" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "set", type: "Set" }, + { value: "set", type: "Identifier" }, { value: "d", type: "Identifier" }, { value: "=", type: "Equals" }, { value: "0", type: "NumericLiteral" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "if", type: "If" }, + { value: "if", type: "Identifier" }, { value: "1", type: "NumericLiteral" }, { value: "==", type: "ComparisonBinaryOperator" }, { value: "1", type: "NumericLiteral" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "set", type: "Set" }, + { value: "set", type: "Identifier" }, { value: "a", type: "Identifier" }, { value: "=", type: "Equals" }, { value: "2", type: "NumericLiteral" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "set", type: "Set" }, + { value: "set", type: "Identifier" }, { value: "b", type: "Identifier" }, { value: "=", type: "Equals" }, { value: "3", type: "NumericLiteral" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "elif", type: "ElseIf" }, + { value: "elif", type: "Identifier" }, { value: "1", type: "NumericLiteral" }, { value: "==", type: "ComparisonBinaryOperator" }, { value: "2", type: "NumericLiteral" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "set", type: "Set" }, + { value: "set", type: "Identifier" }, { value: "c", type: "Identifier" }, { value: "=", type: "Equals" }, { value: "4", type: "NumericLiteral" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "else", type: "Else" }, + { value: "else", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "set", type: "Set" }, + { value: "set", type: "Identifier" }, { value: "d", type: "Identifier" }, { value: "=", type: "Equals" }, { value: "5", type: "NumericLiteral" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "endif", type: "EndIf" }, + { value: "endif", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "{{", type: "OpenExpression" }, { value: "a", type: "Identifier" }, @@ -571,9 +571,9 @@ const TEST_PARSED = { // For loops FOR_LOOP: [ { value: "{%", type: "OpenStatement" }, - { value: "for", type: "For" }, + { value: "for", type: "Identifier" }, { value: "message", type: "Identifier" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "messages", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "{{", type: "OpenExpression" }, @@ -583,17 +583,17 @@ const TEST_PARSED = { { value: "]", type: "CloseSquareBracket" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "endfor", type: "EndFor" }, + { value: "endfor", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, ], FOR_LOOP_UNPACKING: [ { value: "|", type: "Text" }, { value: "{%", type: "OpenStatement" }, - { value: "for", type: "For" }, + { value: "for", type: "Identifier" }, { value: "x", type: "Identifier" }, { value: ",", type: "Comma" }, { value: "y", type: "Identifier" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "[", type: "OpenSquareBracket" }, { value: "[", type: "OpenSquareBracket" }, { value: "1", type: "NumericLiteral" }, @@ -618,15 +618,15 @@ const TEST_PARSED = { { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{%", type: "OpenStatement" }, - { value: "endfor", type: "EndFor" }, + { value: "endfor", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "|", type: "Text" }, ], FOR_LOOP_DEFAULT: [ { value: "{%", type: "OpenStatement" }, - { value: "for", type: "For" }, + { value: "for", type: "Identifier" }, { value: "x", type: "Identifier" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "[", type: "OpenSquareBracket" }, { value: "]", type: "CloseSquareBracket" }, { value: "%}", type: "CloseStatement" }, @@ -634,20 +634,20 @@ const TEST_PARSED = { { value: "A", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "else", type: "Else" }, + { value: "else", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "{{", type: "OpenExpression" }, { value: "B", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "endfor", type: "EndFor" }, + { value: "endfor", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, ], FOR_LOOP_SELECT: [ { value: "{%", type: "OpenStatement" }, - { value: "for", type: "For" }, + { value: "for", type: "Identifier" }, { value: "x", type: "Identifier" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "[", type: "OpenSquareBracket" }, { value: "1", type: "NumericLiteral" }, { value: ",", type: "Comma" }, @@ -657,7 +657,7 @@ const TEST_PARSED = { { value: ",", type: "Comma" }, { value: "4", type: "NumericLiteral" }, { value: "]", type: "CloseSquareBracket" }, - { value: "if", type: "If" }, + { value: "if", type: "Identifier" }, { value: "x", type: "Identifier" }, { value: ">", type: "ComparisonBinaryOperator" }, { value: "2", type: "NumericLiteral" }, @@ -666,14 +666,14 @@ const TEST_PARSED = { { value: "x", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "endfor", type: "EndFor" }, + { value: "endfor", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, ], FOR_LOOP_BREAK: [ { value: "{%", type: "OpenStatement" }, - { value: "for", type: "For" }, + { value: "for", type: "Identifier" }, { value: "x", type: "Identifier" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "[", type: "OpenSquareBracket" }, { value: "1", type: "NumericLiteral" }, { value: ",", type: "Comma" }, @@ -685,29 +685,29 @@ const TEST_PARSED = { { value: "]", type: "CloseSquareBracket" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "if", type: "If" }, + { value: "if", type: "Identifier" }, { value: "x", type: "Identifier" }, { value: "==", type: "ComparisonBinaryOperator" }, { value: "3", type: "NumericLiteral" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "break", type: "Break" }, + { value: "break", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "endif", type: "EndIf" }, + { value: "endif", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "{{", type: "OpenExpression" }, { value: "x", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "endfor", type: "EndFor" }, + { value: "endfor", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, ], FOR_LOOP_CONTINUE: [ { value: "{%", type: "OpenStatement" }, - { value: "for", type: "For" }, + { value: "for", type: "Identifier" }, { value: "x", type: "Identifier" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "[", type: "OpenSquareBracket" }, { value: "1", type: "NumericLiteral" }, { value: ",", type: "Comma" }, @@ -719,35 +719,35 @@ const TEST_PARSED = { { value: "]", type: "CloseSquareBracket" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "if", type: "If" }, + { value: "if", type: "Identifier" }, { value: "x", type: "Identifier" }, { value: "==", type: "ComparisonBinaryOperator" }, { value: "3", type: "NumericLiteral" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "continue", type: "Continue" }, + { value: "continue", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "endif", type: "EndIf" }, + { value: "endif", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "{{", type: "OpenExpression" }, { value: "x", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "endfor", type: "EndFor" }, + { value: "endfor", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, ], // Set variables VARIABLES: [ { value: "{%", type: "OpenStatement" }, - { value: "set", type: "Set" }, + { value: "set", type: "Identifier" }, { value: "x", type: "Identifier" }, { value: "=", type: "Equals" }, { value: "Hello", type: "StringLiteral" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "set", type: "Set" }, + { value: "set", type: "Identifier" }, { value: "y", type: "Identifier" }, { value: "=", type: "Equals" }, { value: "World", type: "StringLiteral" }, @@ -762,7 +762,7 @@ const TEST_PARSED = { ], VARIABLES_2: [ { value: "{%", type: "OpenStatement" }, - { value: "set", type: "Set" }, + { value: "set", type: "Identifier" }, { value: "x", type: "Identifier" }, { value: "=", type: "Equals" }, { value: "Hello", type: "StringLiteral" }, @@ -781,12 +781,12 @@ const TEST_PARSED = { ], VARIABLES_BLOCK: [ { value: "{%", type: "OpenStatement" }, - { value: "set", type: "Set" }, + { value: "set", type: "Identifier" }, { value: "x", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: `Hello!\nMultiline/block set!\n`, type: "Text" }, { value: "{%", type: "OpenStatement" }, - { value: "endset", type: "EndSet" }, + { value: "endset", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "{{", type: "OpenExpression" }, { value: "x", type: "Identifier" }, @@ -969,7 +969,7 @@ const TEST_PARSED = { { value: ",", type: "Comma" }, { value: "2", type: "NumericLiteral" }, { value: ",", type: "Comma" }, - { value: "false", type: "BooleanLiteral" }, + { value: "false", type: "Identifier" }, { value: ")", type: "CloseParen" }, { value: "}}", type: "CloseExpression" }, ], @@ -1045,7 +1045,7 @@ const TEST_PARSED = { { value: ")", type: "CloseParen" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "set", type: "Set" }, + { value: "set", type: "Identifier" }, { value: "x", type: "Identifier" }, { value: "=", type: "Equals" }, { value: " B ", type: "StringLiteral" }, @@ -1058,7 +1058,7 @@ const TEST_PARSED = { { value: ")", type: "CloseParen" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "set", type: "Set" }, + { value: "set", type: "Identifier" }, { value: "y", type: "Identifier" }, { value: "=", type: "Equals" }, { value: " aBcD ", type: "StringLiteral" }, @@ -1088,6 +1088,197 @@ const TEST_PARSED = { { value: "}}", type: "CloseExpression" }, ], + RSTRIP: [ + { value: "{{", type: "OpenExpression" }, + { value: " test it ", type: "StringLiteral" }, + { value: ".", type: "Dot" }, + { value: "rstrip", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: ")", type: "CloseParen" }, + { value: "}}", type: "CloseExpression" }, + ], + LSTRIP: [ + { value: "{{", type: "OpenExpression" }, + { value: " test it ", type: "StringLiteral" }, + { value: ".", type: "Dot" }, + { value: "lstrip", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: ")", type: "CloseParen" }, + { value: "}}", type: "CloseExpression" }, + ], + SPLIT: [ + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: " test it ", type: "StringLiteral" }, + { value: ".", type: "Dot" }, + { value: "split", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: ")", type: "CloseParen" }, + { value: "|", type: "Pipe" }, + { value: "join", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: "|", type: "StringLiteral" }, + { value: ")", type: "CloseParen" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + ], + SPLIT_2: [ + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: " test it ", type: "StringLiteral" }, + { value: ".", type: "Dot" }, + { value: "split", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: " ", type: "StringLiteral" }, + { value: ")", type: "CloseParen" }, + { value: "|", type: "Pipe" }, + { value: "join", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: "|", type: "StringLiteral" }, + { value: ")", type: "CloseParen" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + ], + SPLIT_3: [ + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: " test it ", type: "StringLiteral" }, + { value: ".", type: "Dot" }, + { value: "split", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: " ", type: "StringLiteral" }, + { value: ",", type: "Comma" }, + { value: "4", type: "NumericLiteral" }, + { value: ")", type: "CloseParen" }, + { value: "|", type: "Pipe" }, + { value: "join", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: "|", type: "StringLiteral" }, + { value: ")", type: "CloseParen" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + ], + SPLIT_4: [ + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: " 1 2 3 ", type: "StringLiteral" }, + { value: ".", type: "Dot" }, + { value: "split", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: ")", type: "CloseParen" }, + { value: "|", type: "Pipe" }, + { value: "tojson", type: "Identifier" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "babbaccabbb", type: "StringLiteral" }, + { value: ".", type: "Dot" }, + { value: "split", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: "b", type: "StringLiteral" }, + { value: ")", type: "CloseParen" }, + { value: "|", type: "Pipe" }, + { value: "tojson", type: "Identifier" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "babbaccabbb", type: "StringLiteral" }, + { value: ".", type: "Dot" }, + { value: "split", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: "b", type: "StringLiteral" }, + { value: ",", type: "Comma" }, + { value: "2", type: "NumericLiteral" }, + { value: ")", type: "CloseParen" }, + { value: "|", type: "Pipe" }, + { value: "tojson", type: "Identifier" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + ], + SPLIT_5: [ + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: " 1 2 3 4 5 ", type: "StringLiteral" }, + { value: ".", type: "Dot" }, + { value: "split", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: "none", type: "Identifier" }, + { value: ",", type: "Comma" }, + { value: "0", type: "NumericLiteral" }, + { value: ")", type: "CloseParen" }, + { value: "|", type: "Pipe" }, + { value: "join", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: ",", type: "StringLiteral" }, + { value: ")", type: "CloseParen" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: " 1 2 3 4 5 ", type: "StringLiteral" }, + { value: ".", type: "Dot" }, + { value: "split", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: "none", type: "Identifier" }, + { value: ",", type: "Comma" }, + { value: "3", type: "NumericLiteral" }, + { value: ")", type: "CloseParen" }, + { value: "|", type: "Pipe" }, + { value: "join", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: ",", type: "StringLiteral" }, + { value: ")", type: "CloseParen" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: " 1 2 3 4 5 ", type: "StringLiteral" }, + { value: ".", type: "Dot" }, + { value: "split", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: " ", type: "StringLiteral" }, + { value: ",", type: "Comma" }, + { value: "0", type: "NumericLiteral" }, + { value: ")", type: "CloseParen" }, + { value: "|", type: "Pipe" }, + { value: "join", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: ",", type: "StringLiteral" }, + { value: ")", type: "CloseParen" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: " 1 2 3 4 5 ", type: "StringLiteral" }, + { value: ".", type: "Dot" }, + { value: "split", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: " ", type: "StringLiteral" }, + { value: ",", type: "Comma" }, + { value: "3", type: "NumericLiteral" }, + { value: ")", type: "CloseParen" }, + { value: "|", type: "Pipe" }, + { value: "join", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: ",", type: "StringLiteral" }, + { value: ")", type: "CloseParen" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: " 1 2 3 4 5 ", type: "StringLiteral" }, + { value: ".", type: "Dot" }, + { value: "split", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: " ", type: "StringLiteral" }, + { value: ",", type: "Comma" }, + { value: "10", type: "NumericLiteral" }, + { value: ")", type: "CloseParen" }, + { value: "|", type: "Pipe" }, + { value: "join", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: ",", type: "StringLiteral" }, + { value: ")", type: "CloseParen" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + ], + // String indexing and slicing STRING_SLICING: [ { value: "|", type: "Text" }, @@ -1164,9 +1355,9 @@ const TEST_PARSED = { { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{%", type: "OpenStatement" }, - { value: "for", type: "For" }, + { value: "for", type: "Identifier" }, { value: "s", type: "Identifier" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "strings", type: "Identifier" }, { value: "[", type: "OpenSquareBracket" }, { value: ":", type: "Colon" }, @@ -1176,13 +1367,13 @@ const TEST_PARSED = { { value: "s", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "endfor", type: "EndFor" }, + { value: "endfor", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "|", type: "Text" }, { value: "{%", type: "OpenStatement" }, - { value: "for", type: "For" }, + { value: "for", type: "Identifier" }, { value: "s", type: "Identifier" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "strings", type: "Identifier" }, { value: "[", type: "OpenSquareBracket" }, { value: ":", type: "Colon" }, @@ -1193,13 +1384,13 @@ const TEST_PARSED = { { value: "s", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "endfor", type: "EndFor" }, + { value: "endfor", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "|", type: "Text" }, { value: "{%", type: "OpenStatement" }, - { value: "for", type: "For" }, + { value: "for", type: "Identifier" }, { value: "s", type: "Identifier" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "strings", type: "Identifier" }, { value: "[", type: "OpenSquareBracket" }, { value: "1", type: "NumericLiteral" }, @@ -1211,13 +1402,13 @@ const TEST_PARSED = { { value: "s", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "endfor", type: "EndFor" }, + { value: "endfor", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "|", type: "Text" }, { value: "{%", type: "OpenStatement" }, - { value: "for", type: "For" }, + { value: "for", type: "Identifier" }, { value: "s", type: "Identifier" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "strings", type: "Identifier" }, { value: "[", type: "OpenSquareBracket" }, { value: "1", type: "NumericLiteral" }, @@ -1229,13 +1420,13 @@ const TEST_PARSED = { { value: "s", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "endfor", type: "EndFor" }, + { value: "endfor", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "|", type: "Text" }, { value: "{%", type: "OpenStatement" }, - { value: "for", type: "For" }, + { value: "for", type: "Identifier" }, { value: "s", type: "Identifier" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "strings", type: "Identifier" }, { value: "[", type: "OpenSquareBracket" }, { value: "1", type: "NumericLiteral" }, @@ -1248,13 +1439,13 @@ const TEST_PARSED = { { value: "s", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "endfor", type: "EndFor" }, + { value: "endfor", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "|", type: "Text" }, { value: "{%", type: "OpenStatement" }, - { value: "for", type: "For" }, + { value: "for", type: "Identifier" }, { value: "s", type: "Identifier" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "strings", type: "Identifier" }, { value: "[", type: "OpenSquareBracket" }, { value: "5", type: "NumericLiteral" }, @@ -1267,7 +1458,7 @@ const TEST_PARSED = { { value: "s", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "endfor", type: "EndFor" }, + { value: "endfor", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "|", type: "Text" }, ], @@ -1277,37 +1468,37 @@ const TEST_PARSED = { { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "0", type: "NumericLiteral" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "arr", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "1", type: "NumericLiteral" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "arr", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "true", type: "BooleanLiteral" }, - { value: "in", type: "In" }, + { value: "true", type: "Identifier" }, + { value: "in", type: "Identifier" }, { value: "arr", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "false", type: "BooleanLiteral" }, - { value: "in", type: "In" }, + { value: "false", type: "Identifier" }, + { value: "in", type: "Identifier" }, { value: "arr", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "a", type: "StringLiteral" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "arr", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "b", type: "StringLiteral" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "arr", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, @@ -1315,44 +1506,44 @@ const TEST_PARSED = { MEMBERSHIP_NEGATION_1: [ { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "not", type: "UnaryOperator" }, + { value: "not", type: "Identifier" }, { value: "0", type: "NumericLiteral" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "arr", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "not", type: "UnaryOperator" }, + { value: "not", type: "Identifier" }, { value: "1", type: "NumericLiteral" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "arr", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "not", type: "UnaryOperator" }, - { value: "true", type: "BooleanLiteral" }, - { value: "in", type: "In" }, + { value: "not", type: "Identifier" }, + { value: "true", type: "Identifier" }, + { value: "in", type: "Identifier" }, { value: "arr", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "not", type: "UnaryOperator" }, - { value: "false", type: "BooleanLiteral" }, - { value: "in", type: "In" }, + { value: "not", type: "Identifier" }, + { value: "false", type: "Identifier" }, + { value: "in", type: "Identifier" }, { value: "arr", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "not", type: "UnaryOperator" }, + { value: "not", type: "Identifier" }, { value: "a", type: "StringLiteral" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "arr", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "not", type: "UnaryOperator" }, + { value: "not", type: "Identifier" }, { value: "b", type: "StringLiteral" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "arr", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, @@ -1361,37 +1552,43 @@ const TEST_PARSED = { { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "0", type: "NumericLiteral" }, - { value: "not in", type: "NotIn" }, + { value: "not", type: "Identifier" }, + { value: "in", type: "Identifier" }, { value: "arr", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "1", type: "NumericLiteral" }, - { value: "not in", type: "NotIn" }, + { value: "not", type: "Identifier" }, + { value: "in", type: "Identifier" }, { value: "arr", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "true", type: "BooleanLiteral" }, - { value: "not in", type: "NotIn" }, + { value: "true", type: "Identifier" }, + { value: "not", type: "Identifier" }, + { value: "in", type: "Identifier" }, { value: "arr", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "false", type: "BooleanLiteral" }, - { value: "not in", type: "NotIn" }, + { value: "false", type: "Identifier" }, + { value: "not", type: "Identifier" }, + { value: "in", type: "Identifier" }, { value: "arr", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "a", type: "StringLiteral" }, - { value: "not in", type: "NotIn" }, + { value: "not", type: "Identifier" }, + { value: "in", type: "Identifier" }, { value: "arr", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "b", type: "StringLiteral" }, - { value: "not in", type: "NotIn" }, + { value: "not", type: "Identifier" }, + { value: "in", type: "Identifier" }, { value: "arr", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, @@ -1424,43 +1621,43 @@ const TEST_PARSED = { { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "", type: "StringLiteral" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "abc", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "a", type: "StringLiteral" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "abc", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "d", type: "StringLiteral" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "abc", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "ab", type: "StringLiteral" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "abc", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "ac", type: "StringLiteral" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "abc", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "abc", type: "StringLiteral" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "abc", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "abcd", type: "StringLiteral" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "abc", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, @@ -1636,7 +1833,7 @@ const TEST_PARSED = { { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "true", type: "BooleanLiteral" }, + { value: "true", type: "Identifier" }, { value: "|", type: "Pipe" }, { value: "tojson", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, @@ -1712,7 +1909,7 @@ const TEST_PARSED = { { value: "(", type: "OpenParen" }, { value: "first", type: "Identifier" }, { value: "=", type: "Equals" }, - { value: "True", type: "BooleanLiteral" }, + { value: "True", type: "Identifier" }, { value: ")", type: "CloseParen" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, @@ -1723,7 +1920,7 @@ const TEST_PARSED = { { value: "(", type: "OpenParen" }, { value: "blank", type: "Identifier" }, { value: "=", type: "Equals" }, - { value: "True", type: "BooleanLiteral" }, + { value: "True", type: "Identifier" }, { value: ")", type: "CloseParen" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, @@ -1736,7 +1933,7 @@ const TEST_PARSED = { { value: ",", type: "Comma" }, { value: "first", type: "Identifier" }, { value: "=", type: "Equals" }, - { value: "True", type: "BooleanLiteral" }, + { value: "True", type: "Identifier" }, { value: ")", type: "CloseParen" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, @@ -1782,59 +1979,59 @@ const TEST_PARSED = { { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "1", type: "NumericLiteral" }, - { value: "and", type: "And" }, + { value: "and", type: "Identifier" }, { value: "2", type: "NumericLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "1", type: "NumericLiteral" }, - { value: "and", type: "And" }, + { value: "and", type: "Identifier" }, { value: "0", type: "NumericLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "0", type: "NumericLiteral" }, - { value: "and", type: "And" }, + { value: "and", type: "Identifier" }, { value: "1", type: "NumericLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "0", type: "NumericLiteral" }, - { value: "and", type: "And" }, + { value: "and", type: "Identifier" }, { value: "0", type: "NumericLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "1", type: "NumericLiteral" }, - { value: "or", type: "Or" }, + { value: "or", type: "Identifier" }, { value: "2", type: "NumericLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "1", type: "NumericLiteral" }, - { value: "or", type: "Or" }, + { value: "or", type: "Identifier" }, { value: "0", type: "NumericLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "0", type: "NumericLiteral" }, - { value: "or", type: "Or" }, + { value: "or", type: "Identifier" }, { value: "1", type: "NumericLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "0", type: "NumericLiteral" }, - { value: "or", type: "Or" }, + { value: "or", type: "Identifier" }, { value: "0", type: "NumericLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "not", type: "UnaryOperator" }, + { value: "not", type: "Identifier" }, { value: "1", type: "NumericLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "not", type: "UnaryOperator" }, + { value: "not", type: "Identifier" }, { value: "0", type: "NumericLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, @@ -1843,59 +2040,59 @@ const TEST_PARSED = { { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "a", type: "StringLiteral" }, - { value: "and", type: "And" }, + { value: "and", type: "Identifier" }, { value: "b", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "a", type: "StringLiteral" }, - { value: "and", type: "And" }, + { value: "and", type: "Identifier" }, { value: "", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "", type: "StringLiteral" }, - { value: "and", type: "And" }, + { value: "and", type: "Identifier" }, { value: "a", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "", type: "StringLiteral" }, - { value: "and", type: "And" }, + { value: "and", type: "Identifier" }, { value: "", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "a", type: "StringLiteral" }, - { value: "or", type: "Or" }, + { value: "or", type: "Identifier" }, { value: "b", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "a", type: "StringLiteral" }, - { value: "or", type: "Or" }, + { value: "or", type: "Identifier" }, { value: "", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "", type: "StringLiteral" }, - { value: "or", type: "Or" }, + { value: "or", type: "Identifier" }, { value: "a", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "", type: "StringLiteral" }, - { value: "or", type: "Or" }, + { value: "or", type: "Identifier" }, { value: "", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "not", type: "UnaryOperator" }, + { value: "not", type: "Identifier" }, { value: "a", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "not", type: "UnaryOperator" }, + { value: "not", type: "Identifier" }, { value: "", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, @@ -1903,50 +2100,50 @@ const TEST_PARSED = { BOOLEAN_MIXED: [ { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "true", type: "BooleanLiteral" }, - { value: "and", type: "And" }, + { value: "true", type: "Identifier" }, + { value: "and", type: "Identifier" }, { value: "1", type: "NumericLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "true", type: "BooleanLiteral" }, - { value: "and", type: "And" }, + { value: "true", type: "Identifier" }, + { value: "and", type: "Identifier" }, { value: "0", type: "NumericLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "false", type: "BooleanLiteral" }, - { value: "and", type: "And" }, + { value: "false", type: "Identifier" }, + { value: "and", type: "Identifier" }, { value: "1", type: "NumericLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "false", type: "BooleanLiteral" }, - { value: "and", type: "And" }, + { value: "false", type: "Identifier" }, + { value: "and", type: "Identifier" }, { value: "0", type: "NumericLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "true", type: "BooleanLiteral" }, - { value: "or", type: "Or" }, + { value: "true", type: "Identifier" }, + { value: "or", type: "Identifier" }, { value: "1", type: "NumericLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "true", type: "BooleanLiteral" }, - { value: "or", type: "Or" }, + { value: "true", type: "Identifier" }, + { value: "or", type: "Identifier" }, { value: "0", type: "NumericLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "false", type: "BooleanLiteral" }, - { value: "or", type: "Or" }, + { value: "false", type: "Identifier" }, + { value: "or", type: "Identifier" }, { value: "1", type: "NumericLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "false", type: "BooleanLiteral" }, - { value: "or", type: "Or" }, + { value: "false", type: "Identifier" }, + { value: "or", type: "Identifier" }, { value: "0", type: "NumericLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, @@ -1954,98 +2151,98 @@ const TEST_PARSED = { BOOLEAN_MIXED_2: [ { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "true", type: "BooleanLiteral" }, - { value: "and", type: "And" }, + { value: "true", type: "Identifier" }, + { value: "and", type: "Identifier" }, { value: "", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "true", type: "BooleanLiteral" }, - { value: "and", type: "And" }, + { value: "true", type: "Identifier" }, + { value: "and", type: "Identifier" }, { value: "a", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "false", type: "BooleanLiteral" }, - { value: "or", type: "Or" }, + { value: "false", type: "Identifier" }, + { value: "or", type: "Identifier" }, { value: "", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "false", type: "BooleanLiteral" }, - { value: "or", type: "Or" }, + { value: "false", type: "Identifier" }, + { value: "or", type: "Identifier" }, { value: "a", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "", type: "StringLiteral" }, - { value: "and", type: "And" }, - { value: "true", type: "BooleanLiteral" }, + { value: "and", type: "Identifier" }, + { value: "true", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "a", type: "StringLiteral" }, - { value: "and", type: "And" }, - { value: "true", type: "BooleanLiteral" }, + { value: "and", type: "Identifier" }, + { value: "true", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "", type: "StringLiteral" }, - { value: "or", type: "Or" }, - { value: "false", type: "BooleanLiteral" }, + { value: "or", type: "Identifier" }, + { value: "false", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "a", type: "StringLiteral" }, - { value: "or", type: "Or" }, - { value: "false", type: "BooleanLiteral" }, + { value: "or", type: "Identifier" }, + { value: "false", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, ], BOOLEAN_MIXED_IF: [ { value: "{%", type: "OpenStatement" }, - { value: "if", type: "If" }, + { value: "if", type: "Identifier" }, { value: "", type: "StringLiteral" }, { value: "%}", type: "CloseStatement" }, { value: "{{", type: "OpenExpression" }, { value: "A", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "endif", type: "EndIf" }, + { value: "endif", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "if", type: "If" }, + { value: "if", type: "Identifier" }, { value: "a", type: "StringLiteral" }, { value: "%}", type: "CloseStatement" }, { value: "{{", type: "OpenExpression" }, { value: "B", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "endif", type: "EndIf" }, + { value: "endif", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "if", type: "If" }, - { value: "true", type: "BooleanLiteral" }, - { value: "and", type: "And" }, + { value: "if", type: "Identifier" }, + { value: "true", type: "Identifier" }, + { value: "and", type: "Identifier" }, { value: "", type: "StringLiteral" }, { value: "%}", type: "CloseStatement" }, { value: "{{", type: "OpenExpression" }, { value: "C", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "endif", type: "EndIf" }, + { value: "endif", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "if", type: "If" }, - { value: "true", type: "BooleanLiteral" }, - { value: "and", type: "And" }, + { value: "if", type: "Identifier" }, + { value: "true", type: "Identifier" }, + { value: "and", type: "Identifier" }, { value: "a", type: "StringLiteral" }, { value: "%}", type: "CloseStatement" }, { value: "{{", type: "OpenExpression" }, { value: "D", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "endif", type: "EndIf" }, + { value: "endif", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, ], @@ -2054,27 +2251,27 @@ const TEST_PARSED = { { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "unknown_var", type: "Identifier" }, - { value: "is", type: "Is" }, + { value: "is", type: "Identifier" }, { value: "defined", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "unknown_var", type: "Identifier" }, - { value: "is", type: "Is" }, - { value: "not", type: "UnaryOperator" }, + { value: "is", type: "Identifier" }, + { value: "not", type: "Identifier" }, { value: "defined", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "known_var", type: "Identifier" }, - { value: "is", type: "Is" }, + { value: "is", type: "Identifier" }, { value: "defined", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "known_var", type: "Identifier" }, - { value: "is", type: "Is" }, - { value: "not", type: "UnaryOperator" }, + { value: "is", type: "Identifier" }, + { value: "not", type: "Identifier" }, { value: "defined", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, @@ -2082,40 +2279,40 @@ const TEST_PARSED = { IS_OPERATOR_2: [ { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "true", type: "BooleanLiteral" }, - { value: "is", type: "Is" }, - { value: "true", type: "BooleanLiteral" }, + { value: "true", type: "Identifier" }, + { value: "is", type: "Identifier" }, + { value: "true", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "true", type: "BooleanLiteral" }, - { value: "is", type: "Is" }, - { value: "not", type: "UnaryOperator" }, - { value: "true", type: "BooleanLiteral" }, + { value: "true", type: "Identifier" }, + { value: "is", type: "Identifier" }, + { value: "not", type: "Identifier" }, + { value: "true", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "true", type: "BooleanLiteral" }, - { value: "is", type: "Is" }, - { value: "false", type: "BooleanLiteral" }, + { value: "true", type: "Identifier" }, + { value: "is", type: "Identifier" }, + { value: "false", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "true", type: "BooleanLiteral" }, - { value: "is", type: "Is" }, - { value: "not", type: "UnaryOperator" }, - { value: "false", type: "BooleanLiteral" }, + { value: "true", type: "Identifier" }, + { value: "is", type: "Identifier" }, + { value: "not", type: "Identifier" }, + { value: "false", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "true", type: "BooleanLiteral" }, - { value: "is", type: "Is" }, + { value: "true", type: "Identifier" }, + { value: "is", type: "Identifier" }, { value: "boolean", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "1", type: "NumericLiteral" }, - { value: "is", type: "Is" }, + { value: "is", type: "Identifier" }, { value: "boolean", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, @@ -2124,49 +2321,49 @@ const TEST_PARSED = { { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "1", type: "NumericLiteral" }, - { value: "is", type: "Is" }, + { value: "is", type: "Identifier" }, { value: "odd", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "2", type: "NumericLiteral" }, - { value: "is", type: "Is" }, + { value: "is", type: "Identifier" }, { value: "odd", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "1", type: "NumericLiteral" }, - { value: "is", type: "Is" }, + { value: "is", type: "Identifier" }, { value: "even", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "2", type: "NumericLiteral" }, - { value: "is", type: "Is" }, + { value: "is", type: "Identifier" }, { value: "even", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "2", type: "NumericLiteral" }, - { value: "is", type: "Is" }, + { value: "is", type: "Identifier" }, { value: "number", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "2", type: "StringLiteral" }, - { value: "is", type: "Is" }, + { value: "is", type: "Identifier" }, { value: "number", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "2", type: "NumericLiteral" }, - { value: "is", type: "Is" }, + { value: "is", type: "Identifier" }, { value: "integer", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "2", type: "StringLiteral" }, - { value: "is", type: "Is" }, + { value: "is", type: "Identifier" }, { value: "integer", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, @@ -2175,25 +2372,25 @@ const TEST_PARSED = { { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "func", type: "Identifier" }, - { value: "is", type: "Is" }, + { value: "is", type: "Identifier" }, { value: "callable", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "2", type: "NumericLiteral" }, - { value: "is", type: "Is" }, + { value: "is", type: "Identifier" }, { value: "callable", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "1", type: "NumericLiteral" }, - { value: "is", type: "Is" }, + { value: "is", type: "Identifier" }, { value: "iterable", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "hello", type: "StringLiteral" }, - { value: "is", type: "Is" }, + { value: "is", type: "Identifier" }, { value: "iterable", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, @@ -2202,25 +2399,25 @@ const TEST_PARSED = { { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "a", type: "StringLiteral" }, - { value: "is", type: "Is" }, + { value: "is", type: "Identifier" }, { value: "lower", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "A", type: "StringLiteral" }, - { value: "is", type: "Is" }, + { value: "is", type: "Identifier" }, { value: "lower", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "a", type: "StringLiteral" }, - { value: "is", type: "Is" }, + { value: "is", type: "Identifier" }, { value: "upper", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "A", type: "StringLiteral" }, - { value: "is", type: "Is" }, + { value: "is", type: "Identifier" }, { value: "upper", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, @@ -2229,25 +2426,25 @@ const TEST_PARSED = { { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "string", type: "Identifier" }, - { value: "is", type: "Is" }, + { value: "is", type: "Identifier" }, { value: "mapping", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "number", type: "Identifier" }, - { value: "is", type: "Is" }, + { value: "is", type: "Identifier" }, { value: "mapping", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "array", type: "Identifier" }, - { value: "is", type: "Is" }, + { value: "is", type: "Identifier" }, { value: "mapping", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "dict", type: "Identifier" }, - { value: "is", type: "Is" }, + { value: "is", type: "Identifier" }, { value: "mapping", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, @@ -2256,8 +2453,8 @@ const TEST_PARSED = { // Short-circuit evaluation SHORT_CIRCUIT: [ { value: "{{", type: "OpenExpression" }, - { value: "false", type: "BooleanLiteral" }, - { value: "and", type: "And" }, + { value: "false", type: "Identifier" }, + { value: "and", type: "Identifier" }, { value: "raise_exception", type: "Identifier" }, { value: "(", type: "OpenParen" }, { value: "This should not be printed", type: "StringLiteral" }, @@ -2266,8 +2463,8 @@ const TEST_PARSED = { ], SHORT_CIRCUIT_1: [ { value: "{{", type: "OpenExpression" }, - { value: "true", type: "BooleanLiteral" }, - { value: "or", type: "Or" }, + { value: "true", type: "Identifier" }, + { value: "or", type: "Identifier" }, { value: "raise_exception", type: "Identifier" }, { value: "(", type: "OpenParen" }, { value: "This should not be printed", type: "StringLiteral" }, @@ -2278,7 +2475,7 @@ const TEST_PARSED = { // Namespaces NAMESPACE: [ { value: "{%", type: "OpenStatement" }, - { value: "set", type: "Set" }, + { value: "set", type: "Identifier" }, { value: "ns", type: "Identifier" }, { value: "=", type: "Equals" }, { value: "namespace", type: "Identifier" }, @@ -2286,7 +2483,7 @@ const TEST_PARSED = { { value: ")", type: "CloseParen" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "set", type: "Set" }, + { value: "set", type: "Identifier" }, { value: "ns", type: "Identifier" }, { value: ".", type: "Dot" }, { value: "foo", type: "Identifier" }, @@ -2301,14 +2498,14 @@ const TEST_PARSED = { ], NAMESPACE_1: [ { value: "{%", type: "OpenStatement" }, - { value: "set", type: "Set" }, + { value: "set", type: "Identifier" }, { value: "ns", type: "Identifier" }, { value: "=", type: "Equals" }, { value: "namespace", type: "Identifier" }, { value: "(", type: "OpenParen" }, { value: "default", type: "Identifier" }, { value: "=", type: "Equals" }, - { value: "false", type: "BooleanLiteral" }, + { value: "false", type: "Identifier" }, { value: ")", type: "CloseParen" }, { value: "%}", type: "CloseStatement" }, { value: "{{", type: "OpenExpression" }, @@ -2319,14 +2516,14 @@ const TEST_PARSED = { ], NAMESPACE_2: [ { value: "{%", type: "OpenStatement" }, - { value: "set", type: "Set" }, + { value: "set", type: "Identifier" }, { value: "ns", type: "Identifier" }, { value: "=", type: "Equals" }, { value: "namespace", type: "Identifier" }, { value: "(", type: "OpenParen" }, { value: "default", type: "Identifier" }, { value: "=", type: "Equals" }, - { value: "false", type: "BooleanLiteral" }, + { value: "false", type: "Identifier" }, { value: ",", type: "Comma" }, { value: "number", type: "Identifier" }, { value: "=", type: "Equals" }, @@ -2355,25 +2552,27 @@ const TEST_PARSED = { { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "known", type: "StringLiteral" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "obj", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "known", type: "StringLiteral" }, - { value: "not in", type: "NotIn" }, + { value: "not", type: "Identifier" }, + { value: "in", type: "Identifier" }, { value: "obj", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "unknown", type: "StringLiteral" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "obj", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "unknown", type: "StringLiteral" }, - { value: "not in", type: "NotIn" }, + { value: "not", type: "Identifier" }, + { value: "in", type: "Identifier" }, { value: "obj", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, @@ -2396,8 +2595,8 @@ const TEST_PARSED = { { value: "(", type: "OpenParen" }, { value: "unknown", type: "StringLiteral" }, { value: ")", type: "CloseParen" }, - { value: "is", type: "Is" }, - { value: "none", type: "NullLiteral" }, + { value: "is", type: "Identifier" }, + { value: "none", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, @@ -2407,7 +2606,7 @@ const TEST_PARSED = { { value: "(", type: "OpenParen" }, { value: "unknown", type: "StringLiteral" }, { value: ")", type: "CloseParen" }, - { value: "is", type: "Is" }, + { value: "is", type: "Identifier" }, { value: "defined", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, @@ -2415,11 +2614,11 @@ const TEST_PARSED = { OBJECT_OPERATORS_2: [ { value: "|", type: "Text" }, { value: "{%", type: "OpenStatement" }, - { value: "for", type: "For" }, + { value: "for", type: "Identifier" }, { value: "x", type: "Identifier" }, { value: ",", type: "Comma" }, { value: "y", type: "Identifier" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "obj", type: "Identifier" }, { value: ".", type: "Dot" }, { value: "items", type: "Identifier" }, @@ -2436,7 +2635,7 @@ const TEST_PARSED = { { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{%", type: "OpenStatement" }, - { value: "endfor", type: "EndFor" }, + { value: "endfor", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "|", type: "Text" }, ], @@ -2444,24 +2643,24 @@ const TEST_PARSED = { // Scope SCOPE: [ { value: "{%", type: "OpenStatement" }, - { value: "set", type: "Set" }, + { value: "set", type: "Identifier" }, { value: "ns", type: "Identifier" }, { value: "=", type: "Equals" }, { value: "namespace", type: "Identifier" }, { value: "(", type: "OpenParen" }, { value: "found", type: "Identifier" }, { value: "=", type: "Equals" }, - { value: "false", type: "BooleanLiteral" }, + { value: "false", type: "Identifier" }, { value: ")", type: "CloseParen" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "for", type: "For" }, + { value: "for", type: "Identifier" }, { value: "num", type: "Identifier" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "nums", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "if", type: "If" }, + { value: "if", type: "Identifier" }, { value: "num", type: "Identifier" }, { value: "==", type: "ComparisonBinaryOperator" }, { value: "1", type: "NumericLiteral" }, @@ -2470,18 +2669,18 @@ const TEST_PARSED = { { value: "found=", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "set", type: "Set" }, + { value: "set", type: "Identifier" }, { value: "ns", type: "Identifier" }, { value: ".", type: "Dot" }, { value: "found", type: "Identifier" }, { value: "=", type: "Equals" }, - { value: "true", type: "BooleanLiteral" }, + { value: "true", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "endif", type: "EndIf" }, + { value: "endif", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "endfor", type: "EndFor" }, + { value: "endfor", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "{{", type: "OpenExpression" }, { value: "ns", type: "Identifier" }, @@ -2491,19 +2690,19 @@ const TEST_PARSED = { ], SCOPE_1: [ { value: "{%", type: "OpenStatement" }, - { value: "set", type: "Set" }, + { value: "set", type: "Identifier" }, { value: "found", type: "Identifier" }, { value: "=", type: "Equals" }, - { value: "false", type: "BooleanLiteral" }, + { value: "false", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "for", type: "For" }, + { value: "for", type: "Identifier" }, { value: "num", type: "Identifier" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "nums", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "if", type: "If" }, + { value: "if", type: "Identifier" }, { value: "num", type: "Identifier" }, { value: "==", type: "ComparisonBinaryOperator" }, { value: "1", type: "NumericLiteral" }, @@ -2512,16 +2711,16 @@ const TEST_PARSED = { { value: "found=", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "set", type: "Set" }, + { value: "set", type: "Identifier" }, { value: "found", type: "Identifier" }, { value: "=", type: "Equals" }, - { value: "true", type: "BooleanLiteral" }, + { value: "true", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "endif", type: "EndIf" }, + { value: "endif", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "endfor", type: "EndFor" }, + { value: "endfor", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "{{", type: "OpenExpression" }, { value: "found", type: "Identifier" }, @@ -2545,39 +2744,39 @@ const TEST_PARSED = { // Null NULL_VARIABLE: [ { value: "{%", type: "OpenStatement" }, - { value: "if", type: "If" }, - { value: "not", type: "UnaryOperator" }, + { value: "if", type: "Identifier" }, + { value: "not", type: "Identifier" }, { value: "null_val", type: "Identifier" }, - { value: "is", type: "Is" }, + { value: "is", type: "Identifier" }, { value: "defined", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "set", type: "Set" }, + { value: "set", type: "Identifier" }, { value: "null_val", type: "Identifier" }, { value: "=", type: "Equals" }, - { value: "none", type: "NullLiteral" }, + { value: "none", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "endif", type: "EndIf" }, + { value: "endif", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "if", type: "If" }, + { value: "if", type: "Identifier" }, { value: "null_val", type: "Identifier" }, - { value: "is", type: "Is" }, - { value: "not", type: "UnaryOperator" }, - { value: "none", type: "NullLiteral" }, + { value: "is", type: "Identifier" }, + { value: "not", type: "Identifier" }, + { value: "none", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "{{", type: "OpenExpression" }, { value: "fail", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "else", type: "Else" }, + { value: "else", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "{{", type: "OpenExpression" }, { value: "pass", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "endif", type: "EndIf" }, + { value: "endif", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, ], @@ -2586,60 +2785,60 @@ const TEST_PARSED = { { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "a", type: "StringLiteral" }, - { value: "if", type: "If" }, - { value: "true", type: "BooleanLiteral" }, - { value: "else", type: "Else" }, + { value: "if", type: "Identifier" }, + { value: "true", type: "Identifier" }, + { value: "else", type: "Identifier" }, { value: "b", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "a", type: "StringLiteral" }, - { value: "if", type: "If" }, - { value: "false", type: "BooleanLiteral" }, - { value: "else", type: "Else" }, + { value: "if", type: "Identifier" }, + { value: "false", type: "Identifier" }, + { value: "else", type: "Identifier" }, { value: "b", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "a", type: "StringLiteral" }, - { value: "if", type: "If" }, + { value: "if", type: "Identifier" }, { value: "1", type: "NumericLiteral" }, { value: "+", type: "AdditiveBinaryOperator" }, { value: "1", type: "NumericLiteral" }, { value: "==", type: "ComparisonBinaryOperator" }, { value: "2", type: "NumericLiteral" }, - { value: "else", type: "Else" }, + { value: "else", type: "Identifier" }, { value: "b", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "a", type: "StringLiteral" }, - { value: "if", type: "If" }, + { value: "if", type: "Identifier" }, { value: "1", type: "NumericLiteral" }, { value: "+", type: "AdditiveBinaryOperator" }, { value: "1", type: "NumericLiteral" }, { value: "==", type: "ComparisonBinaryOperator" }, { value: "3", type: "NumericLiteral" }, - { value: "or", type: "Or" }, + { value: "or", type: "Identifier" }, { value: "1", type: "NumericLiteral" }, { value: "*", type: "MultiplicativeBinaryOperator" }, { value: "2", type: "NumericLiteral" }, { value: "==", type: "ComparisonBinaryOperator" }, { value: "3", type: "NumericLiteral" }, - { value: "else", type: "Else" }, + { value: "else", type: "Identifier" }, { value: "b", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, ], TERNARY_SET: [ { value: "{%", type: "OpenStatement" }, - { value: "set", type: "Set" }, + { value: "set", type: "Identifier" }, { value: "x", type: "Identifier" }, { value: "=", type: "Equals" }, { value: "1", type: "NumericLiteral" }, - { value: "if", type: "If" }, - { value: "True", type: "BooleanLiteral" }, - { value: "else", type: "Else" }, + { value: "if", type: "Identifier" }, + { value: "True", type: "Identifier" }, + { value: "else", type: "Identifier" }, { value: "2", type: "NumericLiteral" }, { value: "%}", type: "CloseStatement" }, { value: "{{", type: "OpenExpression" }, @@ -2653,7 +2852,7 @@ const TEST_PARSED = { { value: "[", type: "OpenSquareBracket" }, { value: "1", type: "NumericLiteral" }, { value: ",", type: "Comma" }, - { value: "true", type: "BooleanLiteral" }, + { value: "true", type: "Identifier" }, { value: ",", type: "Comma" }, { value: "hello", type: "StringLiteral" }, { value: ",", type: "Comma" }, @@ -2749,7 +2948,7 @@ const TEST_PARSED = { // Macros MACROS: [ { value: "{%", type: "OpenStatement" }, - { value: "macro", type: "Macro" }, + { value: "macro", type: "Identifier" }, { value: "hello", type: "Identifier" }, { value: "(", type: "OpenParen" }, { value: "name", type: "Identifier" }, @@ -2761,7 +2960,7 @@ const TEST_PARSED = { { value: "name", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "endmacro", type: "EndMacro" }, + { value: "endmacro", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, @@ -2781,7 +2980,7 @@ const TEST_PARSED = { ], MACROS_1: [ { value: "{%", type: "OpenStatement" }, - { value: "macro", type: "Macro" }, + { value: "macro", type: "Identifier" }, { value: "hello", type: "Identifier" }, { value: "(", type: "OpenParen" }, { value: "name", type: "Identifier" }, @@ -2799,7 +2998,7 @@ const TEST_PARSED = { { value: "suffix", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "endmacro", type: "EndMacro" }, + { value: "endmacro", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, @@ -2832,7 +3031,7 @@ const TEST_PARSED = { ], MACROS_2: [ { value: "{%", type: "OpenStatement" }, - { value: "macro", type: "Macro" }, + { value: "macro", type: "Identifier" }, { value: "fn", type: "Identifier" }, { value: "(", type: "OpenParen" }, { value: "x", type: "Identifier" }, @@ -2858,7 +3057,7 @@ const TEST_PARSED = { { value: "z", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "endmacro", type: "EndMacro" }, + { value: "endmacro", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, @@ -3579,9 +3778,7 @@ describe("Templates", () => { } it(name, () => { const env = new Environment(); - // Declare global variables - env.set("false", false); - env.set("true", true); + setupGlobals(env); // Add user-defined variables for (const [key, value] of Object.entries(TEST_CONTEXT[name])) { From 8c6aefe6f02a06e46a124857aeb30bc7449d98d8 Mon Sep 17 00:00:00 2001 From: Joshua Lochner <26504141+xenova@users.noreply.github.com> Date: Thu, 1 May 2025 23:38:45 -0400 Subject: [PATCH 02/35] Code improvements --- packages/jinja/src/parser.ts | 189 +++++++++++------------------------ 1 file changed, 61 insertions(+), 128 deletions(-) diff --git a/packages/jinja/src/parser.ts b/packages/jinja/src/parser.ts index 98a67f0021..d3f3e99434 100644 --- a/packages/jinja/src/parser.ts +++ b/packages/jinja/src/parser.ts @@ -48,6 +48,13 @@ export function parse(tokens: Token[]): Program { return prev; } + function expectIdentifier(name: string): void { + if (!isIdentifier(name)) { + throw new SyntaxError(`Expected ${name}`); + } + ++current; + } + function parseAny(): Statement { switch (tokens[current].type) { case TOKEN_TYPES.Text: @@ -61,14 +68,25 @@ export function parse(tokens: Token[]): Program { } } - function not(...types: TokenType[]): boolean { - return current + types.length <= tokens.length && types.some((type, i) => type !== tokens[current + i].type); - } - function is(...types: TokenType[]): boolean { return current + types.length <= tokens.length && types.every((type, i) => type === tokens[current + i].type); } + function isStatement(...names: string[]): boolean { + return ( + tokens[current]?.type === TOKEN_TYPES.OpenStatement && + tokens[current + 1]?.type === TOKEN_TYPES.Identifier && + names.includes(tokens[current + 1]?.value) + ); + } + + function isIdentifier(...names: string[]): boolean { + return ( + current + names.length <= tokens.length && + names.every((name, i) => tokens[current + i].type === "Identifier" && name === tokens[current + i].value) + ); + } + function parseText(): StringLiteral { return new StringLiteral(expect(TOKEN_TYPES.Text, "Expected text token").value); } @@ -87,38 +105,29 @@ export function parse(tokens: Token[]): Program { case "set": ++current; result = parseSetStatement(); - expect(TOKEN_TYPES.CloseStatement, "Expected closing statement token"); break; case "if": ++current; result = parseIfStatement(); // expect {% endif %} expect(TOKEN_TYPES.OpenStatement, "Expected {% token"); - // ensure identifier 'endif' - if (tokens[current].type !== TOKEN_TYPES.Identifier || tokens[current].value !== "endif") { - throw new SyntaxError("Expected endif token"); - } - ++current; + expectIdentifier("endif"); expect(TOKEN_TYPES.CloseStatement, "Expected %} token"); break; case "macro": ++current; result = parseMacroStatement(); + // expect {% endmacro %} expect(TOKEN_TYPES.OpenStatement, "Expected {% token"); - if (tokens[current].type !== TOKEN_TYPES.Identifier || tokens[current].value !== "endmacro") { - throw new SyntaxError("Expected endmacro token"); - } - ++current; + expectIdentifier("endmacro"); expect(TOKEN_TYPES.CloseStatement, "Expected %} token"); break; case "for": ++current; result = parseForStatement(); + // expect {% endfor %} expect(TOKEN_TYPES.OpenStatement, "Expected {% token"); - if (tokens[current].type !== TOKEN_TYPES.Identifier || tokens[current].value !== "endfor") { - throw new SyntaxError("Expected endfor token"); - } - ++current; + expectIdentifier("endfor"); expect(TOKEN_TYPES.CloseStatement, "Expected %} token"); break; @@ -152,34 +161,22 @@ export function parse(tokens: Token[]): Program { // NOTE: `set` acts as both declaration statement and assignment expression function parseSetStatement(): Statement { const left = parseExpression(); - + let value: Statement | null = null; + const body: Statement[] = []; if (is(TOKEN_TYPES.Equals)) { ++current; - const value = parseExpression(); - - return new SetStatement(left, value, []); + value = parseExpression(); } else { // parsing multiline set here - const body: Statement[] = []; expect(TOKEN_TYPES.CloseStatement, "Expected %} token"); - while ( - !( - tokens[current]?.type === TOKEN_TYPES.OpenStatement && - tokens[current + 1]?.type === TOKEN_TYPES.Identifier && - tokens[current + 1]?.value === "endset" - ) - ) { - const another = parseAny(); - body.push(another); + while (!isStatement("endset")) { + body.push(parseAny()); } expect(TOKEN_TYPES.OpenStatement, "Expected {% token"); - if (tokens[current]?.type !== TOKEN_TYPES.Identifier || tokens[current]?.value !== "endset") { - throw new SyntaxError("Expected endset token"); - } - ++current; - - return new SetStatement(left, null, body); + expectIdentifier("endset"); } + expect(TOKEN_TYPES.CloseStatement, "Expected closing statement token"); + return new SetStatement(left, value, body); } function parseIfStatement(): If { @@ -191,45 +188,25 @@ export function parse(tokens: Token[]): Program { const alternate: Statement[] = []; // Keep parsing 'if' body until we reach the first {% elif %} or {% else %} or {% endif %} - while ( - !( - tokens[current]?.type === TOKEN_TYPES.OpenStatement && - tokens[current + 1]?.type === TOKEN_TYPES.Identifier && - ["elif", "else", "endif"].includes(tokens[current + 1].value) - ) - ) { + while (!isStatement("elif", "else", "endif")) { body.push(parseAny()); } // handle {% elif %} - if ( - tokens[current]?.type === TOKEN_TYPES.OpenStatement && - tokens[current + 1]?.type === TOKEN_TYPES.Identifier && - tokens[current + 1].value === "elif" - ) { + if (isStatement("elif")) { ++current; // consume {% ++current; // consume 'elif' const result = parseIfStatement(); // nested If alternate.push(result); } // handle {% else %} - else if ( - tokens[current]?.type === TOKEN_TYPES.OpenStatement && - tokens[current + 1]?.type === TOKEN_TYPES.Identifier && - tokens[current + 1].value === "else" - ) { + else if (isStatement("else")) { ++current; // consume {% ++current; // consume 'else' expect(TOKEN_TYPES.CloseStatement, "Expected closing statement token"); // keep going until we hit {% endif %} - while ( - !( - tokens[current]?.type === TOKEN_TYPES.OpenStatement && - tokens[current + 1]?.type === TOKEN_TYPES.Identifier && - tokens[current + 1].value === "endif" - ) - ) { + while (!isStatement("endif")) { alternate.push(parseAny()); } } @@ -249,7 +226,7 @@ export function parse(tokens: Token[]): Program { const body: Statement[] = []; // Keep going until we hit {% endmacro - while (not(TOKEN_TYPES.OpenStatement, TOKEN_TYPES.Identifier) || tokens[current + 1]?.value !== "endmacro") { + while (!isStatement("endmacro")) { body.push(parseAny()); } @@ -277,7 +254,7 @@ export function parse(tokens: Token[]): Program { throw new SyntaxError(`Expected identifier/tuple for the loop variable, got ${loopVariable.type} instead`); } - if (!(tokens[current].type === TOKEN_TYPES.Identifier && tokens[current].value === "in")) { + if (!isIdentifier("in")) { throw new SyntaxError("Expected `in` keyword following loop variable"); } ++current; @@ -291,34 +268,17 @@ export function parse(tokens: Token[]): Program { const body: Statement[] = []; // Keep going until we hit {% endfor or {% else - while ( - !( - tokens[current]?.type === TOKEN_TYPES.OpenStatement && - tokens[current + 1]?.type === TOKEN_TYPES.Identifier && - ["endfor", "else"].includes(tokens[current + 1].value) - ) - ) { + while (!isStatement("endfor", "else")) { body.push(parseAny()); } // (Optional) else block const alternative: Statement[] = []; - if ( - tokens[current]?.type === TOKEN_TYPES.OpenStatement && - tokens[current + 1]?.type === TOKEN_TYPES.Identifier && - tokens[current + 1].value === "else" - ) { + if (isStatement("else")) { ++current; // consume {% ++current; // consume 'else' expect(TOKEN_TYPES.CloseStatement, "Expected closing statement token"); - while ( - // keep going until we hit {% endfor - !( - tokens[current]?.type === TOKEN_TYPES.OpenStatement && - tokens[current + 1]?.type === TOKEN_TYPES.Identifier && - tokens[current + 1].value === "endfor" - ) - ) { + while (!isStatement("endfor")) { alternative.push(parseAny()); } } @@ -333,12 +293,12 @@ export function parse(tokens: Token[]): Program { function parseIfExpression(): Statement { const a = parseLogicalOrExpression(); - if (tokens[current].type === TOKEN_TYPES.Identifier && tokens[current].value === "if") { + if (isIdentifier("if")) { // Ternary expression ++current; // consume 'if' const predicate = parseLogicalOrExpression(); - if (tokens[current].type === TOKEN_TYPES.Identifier && tokens[current].value === "else") { + if (isIdentifier("else")) { // Ternary expression with else ++current; // consume 'else' const b = parseLogicalOrExpression(); @@ -353,7 +313,7 @@ export function parse(tokens: Token[]): Program { function parseLogicalOrExpression(): Statement { let left = parseLogicalAndExpression(); - while (tokens[current].type === TOKEN_TYPES.Identifier && tokens[current].value === "or") { + while (isIdentifier("or")) { const operator = tokens[current]; ++current; const right = parseLogicalAndExpression(); @@ -364,7 +324,7 @@ export function parse(tokens: Token[]): Program { function parseLogicalAndExpression(): Statement { let left = parseLogicalNegationExpression(); - while (tokens[current].type === TOKEN_TYPES.Identifier && tokens[current].value === "and") { + while (isIdentifier("and")) { const operator = tokens[current]; ++current; const right = parseLogicalNegationExpression(); @@ -377,7 +337,7 @@ export function parse(tokens: Token[]): Program { let right: UnaryExpression | undefined; // Try parse unary operators - while (tokens[current].type === TOKEN_TYPES.Identifier && tokens[current].value === "not") { + while (isIdentifier("not")) { // not not ... const operator = tokens[current]; ++current; @@ -392,32 +352,17 @@ export function parse(tokens: Token[]): Program { // NOTE: membership has same precedence as comparison // e.g., ('a' in 'apple' == 'b' in 'banana') evaluates as ('a' in ('apple' == ('b' in 'banana'))) let left = parseAdditiveExpression(); - while ( - is(TOKEN_TYPES.ComparisonBinaryOperator) || - (tokens[current].type === TOKEN_TYPES.Identifier && tokens[current].value === "in") || - (tokens[current].type === TOKEN_TYPES.Identifier && - tokens[current].value === "not" && - tokens[current + 1]?.type === TOKEN_TYPES.Identifier && - tokens[current + 1]?.value === "in") - ) { + while (true) { let operator: Token; - // handle 'not in' - if ( - tokens[current].type === TOKEN_TYPES.Identifier && - tokens[current].value === "not" && - tokens[current + 1]?.type === TOKEN_TYPES.Identifier && - tokens[current + 1]?.value === "in" - ) { + if (isIdentifier("not", "in")) { operator = new Token("not in", TOKEN_TYPES.Identifier); current += 2; - } - // handle 'in' - else if (tokens[current].type === TOKEN_TYPES.Identifier && tokens[current].value === "in") { + } else if (isIdentifier("in")) { operator = tokens[current++]; - } - // regular comparison operator - else { + } else if (is(TOKEN_TYPES.ComparisonBinaryOperator)) { operator = tokens[current++]; + } else { + break; } const right = parseAdditiveExpression(); left = new BinaryExpression(operator, left, right); @@ -535,7 +480,7 @@ export function parse(tokens: Token[]): Program { const operator = tokens[current]; // . or [ ++current; let property: Statement; - const computed = operator.type !== TOKEN_TYPES.Dot; + const computed = operator.type === TOKEN_TYPES.OpenSquareBracket; if (computed) { // computed (i.e., bracket notation: obj[expr]) property = parseMemberExpressionArgumentsList(); @@ -559,8 +504,7 @@ export function parse(tokens: Token[]): Program { // e.g., (4 * 4 is divisibleby(2)) evaluates as (4 * (4 is divisibleby(2))) while (is(TOKEN_TYPES.MultiplicativeBinaryOperator)) { - const operator = tokens[current]; - ++current; + const operator = tokens[current++]; const right = parseTestExpression(); left = new BinaryExpression(operator, left, right); } @@ -570,10 +514,10 @@ export function parse(tokens: Token[]): Program { function parseTestExpression(): Statement { let operand = parseFilterExpression(); - while (tokens[current].type === TOKEN_TYPES.Identifier && tokens[current].value === "is") { + while (isIdentifier("is")) { // Support chaining tests ++current; // consume is - const negate = tokens[current].type === TOKEN_TYPES.Identifier && tokens[current].value === "not"; + const negate = isIdentifier("not"); if (negate) { ++current; // consume not } @@ -608,29 +552,20 @@ export function parse(tokens: Token[]): Program { function parsePrimaryExpression(): Statement { // Primary expression: number, string, identifier, function call, parenthesized expression - const token = tokens[current]; + const token = tokens[current++]; switch (token.type) { case TOKEN_TYPES.NumericLiteral: - ++current; return new NumericLiteral(Number(token.value)); case TOKEN_TYPES.StringLiteral: - ++current; return new StringLiteral(token.value); case TOKEN_TYPES.Identifier: - ++current; return new Identifier(token.value); case TOKEN_TYPES.OpenParen: { - ++current; // consume opening parenthesis const expression = parseExpressionSequence(); - if (tokens[current].type !== TOKEN_TYPES.CloseParen) { - throw new SyntaxError(`Expected closing parenthesis, got ${tokens[current].type} instead`); - } - ++current; // consume closing parenthesis + expect(TOKEN_TYPES.CloseParen, "Expected closing parenthesis, got ${tokens[current].type} instead."); return expression; } case TOKEN_TYPES.OpenSquareBracket: { - ++current; // consume opening square bracket - const values = []; while (!is(TOKEN_TYPES.CloseSquareBracket)) { values.push(parseExpression()); @@ -644,8 +579,6 @@ export function parse(tokens: Token[]): Program { return new ArrayLiteral(values); } case TOKEN_TYPES.OpenCurlyBracket: { - ++current; // consume opening curly bracket - const values = new Map(); while (!is(TOKEN_TYPES.CloseCurlyBracket)) { const key = parseExpression(); From 65680ea084a78bdf54299d03cf00cce76de3aaaf Mon Sep 17 00:00:00 2001 From: Joshua Lochner <26504141+xenova@users.noreply.github.com> Date: Thu, 1 May 2025 23:38:56 -0400 Subject: [PATCH 03/35] Re-order tests --- packages/jinja/test/templates.test.js | 246 +++----------------------- 1 file changed, 21 insertions(+), 225 deletions(-) diff --git a/packages/jinja/test/templates.test.js b/packages/jinja/test/templates.test.js index 9787e5d1ca..3e07c292d1 100644 --- a/packages/jinja/test/templates.test.js +++ b/packages/jinja/test/templates.test.js @@ -63,6 +63,13 @@ const TEST_STRINGS = { OBJ_METHODS: `{{ obj.x(x, y) }}{{ ' ' + obj.x() + ' ' }}{{ obj.z[x](x, y) }}`, STRING_METHODS: `{{ ' A '.strip() }}{% set x = ' B ' %}{{ x.strip() }}{% set y = ' aBcD ' %}{{ y.upper() }}{{ y.lower() }}`, STRING_METHODS_2: `{{ 'test test'.title() }}`, + RSTRIP: `{{ " test it ".rstrip() }}`, + LSTRIP: `{{ " test it ".lstrip() }}`, + SPLIT: `|{{ " test it ".split() | join("|") }}|`, + SPLIT_2: `|{{ " test it ".split(" ") | join("|") }}|`, + SPLIT_3: `|{{ " test it ".split(" ", 4) | join("|") }}|`, + SPLIT_4: `|{{ " 1 2 3 ".split() | tojson }}|{{ "babbaccabbb".split("b") | tojson }}|{{ "babbaccabbb".split("b", 2) | tojson }}|`, + SPLIT_5: `|{{ " 1 2 3 4 5 ".split(none, 0) | join(",") }}|{{ " 1 2 3 4 5 ".split(none, 3) | join(",") }}|{{ " 1 2 3 4 5 ".split(" ", 0) | join(",") }}|{{ " 1 2 3 4 5 ".split(" ", 3) | join(",") }}|{{ " 1 2 3 4 5 ".split(" ", 10) | join(",") }}|`, // String indexing and slicing STRING_SLICING: `|{{ x[0] }}|{{ x[:] }}|{{ x[:3] }}|{{ x[1:4] }}|{{ x[1:-1] }}|{{ x[1::2] }}|{{ x[5::-1] }}|`, @@ -156,18 +163,6 @@ const TEST_STRINGS = { MACROS: `{% macro hello(name) %}{{ 'Hello ' + name }}{% endmacro %}|{{ hello('Bob') }}|{{ hello('Alice') }}|`, MACROS_1: `{% macro hello(name, suffix='.') %}{{ 'Hello ' + name + suffix }}{% endmacro %}|{{ hello('A') }}|{{ hello('B', '!') }}|{{ hello('C', suffix='?') }}|`, MACROS_2: `{% macro fn(x, y=2, z=3) %}{{ x + ',' + y + ',' + z }}{% endmacro %}|{{ fn(1) }}|{{ fn(1, 0) }}|{{ fn(1, 0, -1) }}|{{ fn(1, y=0, z=-1) }}|{{ fn(1, z=0) }}|`, - - //rstrip - RSTRIP: `{{ " test it ".rstrip() }}`, - //lstrip - LSTRIP: `{{ " test it ".lstrip() }}`, - - //split - SPLIT: `|{{ " test it ".split() | join("|") }}|`, - SPLIT_2: `|{{ " test it ".split(" ") | join("|") }}|`, - SPLIT_3: `|{{ " test it ".split(" ", 4) | join("|") }}|`, - SPLIT_4: `|{{ " 1 2 3 ".split() | tojson }}|{{ "babbaccabbb".split("b") | tojson }}|{{ "babbaccabbb".split("b", 2) | tojson }}|`, - SPLIT_5: `|{{ " 1 2 3 4 5 ".split(none, 0) | join(",") }}|{{ " 1 2 3 4 5 ".split(none, 3) | join(",") }}|{{ " 1 2 3 4 5 ".split(" ", 0) | join(",") }}|{{ " 1 2 3 4 5 ".split(" ", 3) | join(",") }}|{{ " 1 2 3 4 5 ".split(" ", 10) | join(",") }}|`, }; const TEST_PARSED = { @@ -3114,197 +3109,6 @@ const TEST_PARSED = { { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, ], - - RSTRIP: [ - { value: "{{", type: "OpenExpression" }, - { value: " test it ", type: "StringLiteral" }, - { value: ".", type: "Dot" }, - { value: "rstrip", type: "Identifier" }, - { value: "(", type: "OpenParen" }, - { value: ")", type: "CloseParen" }, - { value: "}}", type: "CloseExpression" }, - ], - LSTRIP: [ - { value: "{{", type: "OpenExpression" }, - { value: " test it ", type: "StringLiteral" }, - { value: ".", type: "Dot" }, - { value: "lstrip", type: "Identifier" }, - { value: "(", type: "OpenParen" }, - { value: ")", type: "CloseParen" }, - { value: "}}", type: "CloseExpression" }, - ], - SPLIT: [ - { value: "|", type: "Text" }, - { value: "{{", type: "OpenExpression" }, - { value: " test it ", type: "StringLiteral" }, - { value: ".", type: "Dot" }, - { value: "split", type: "Identifier" }, - { value: "(", type: "OpenParen" }, - { value: ")", type: "CloseParen" }, - { value: "|", type: "Pipe" }, - { value: "join", type: "Identifier" }, - { value: "(", type: "OpenParen" }, - { value: "|", type: "StringLiteral" }, - { value: ")", type: "CloseParen" }, - { value: "}}", type: "CloseExpression" }, - { value: "|", type: "Text" }, - ], - SPLIT_2: [ - { value: "|", type: "Text" }, - { value: "{{", type: "OpenExpression" }, - { value: " test it ", type: "StringLiteral" }, - { value: ".", type: "Dot" }, - { value: "split", type: "Identifier" }, - { value: "(", type: "OpenParen" }, - { value: " ", type: "StringLiteral" }, - { value: ")", type: "CloseParen" }, - { value: "|", type: "Pipe" }, - { value: "join", type: "Identifier" }, - { value: "(", type: "OpenParen" }, - { value: "|", type: "StringLiteral" }, - { value: ")", type: "CloseParen" }, - { value: "}}", type: "CloseExpression" }, - { value: "|", type: "Text" }, - ], - SPLIT_3: [ - { value: "|", type: "Text" }, - { value: "{{", type: "OpenExpression" }, - { value: " test it ", type: "StringLiteral" }, - { value: ".", type: "Dot" }, - { value: "split", type: "Identifier" }, - { value: "(", type: "OpenParen" }, - { value: " ", type: "StringLiteral" }, - { value: ",", type: "Comma" }, - { value: "4", type: "NumericLiteral" }, - { value: ")", type: "CloseParen" }, - { value: "|", type: "Pipe" }, - { value: "join", type: "Identifier" }, - { value: "(", type: "OpenParen" }, - { value: "|", type: "StringLiteral" }, - { value: ")", type: "CloseParen" }, - { value: "}}", type: "CloseExpression" }, - { value: "|", type: "Text" }, - ], - SPLIT_4: [ - { value: "|", type: "Text" }, - { value: "{{", type: "OpenExpression" }, - { value: " 1 2 3 ", type: "StringLiteral" }, - { value: ".", type: "Dot" }, - { value: "split", type: "Identifier" }, - { value: "(", type: "OpenParen" }, - { value: ")", type: "CloseParen" }, - { value: "|", type: "Pipe" }, - { value: "tojson", type: "Identifier" }, - { value: "}}", type: "CloseExpression" }, - { value: "|", type: "Text" }, - { value: "{{", type: "OpenExpression" }, - { value: "babbaccabbb", type: "StringLiteral" }, - { value: ".", type: "Dot" }, - { value: "split", type: "Identifier" }, - { value: "(", type: "OpenParen" }, - { value: "b", type: "StringLiteral" }, - { value: ")", type: "CloseParen" }, - { value: "|", type: "Pipe" }, - { value: "tojson", type: "Identifier" }, - { value: "}}", type: "CloseExpression" }, - { value: "|", type: "Text" }, - { value: "{{", type: "OpenExpression" }, - { value: "babbaccabbb", type: "StringLiteral" }, - { value: ".", type: "Dot" }, - { value: "split", type: "Identifier" }, - { value: "(", type: "OpenParen" }, - { value: "b", type: "StringLiteral" }, - { value: ",", type: "Comma" }, - { value: "2", type: "NumericLiteral" }, - { value: ")", type: "CloseParen" }, - { value: "|", type: "Pipe" }, - { value: "tojson", type: "Identifier" }, - { value: "}}", type: "CloseExpression" }, - { value: "|", type: "Text" }, - ], - SPLIT_5: [ - { value: "|", type: "Text" }, - { value: "{{", type: "OpenExpression" }, - { value: " 1 2 3 4 5 ", type: "StringLiteral" }, - { value: ".", type: "Dot" }, - { value: "split", type: "Identifier" }, - { value: "(", type: "OpenParen" }, - { value: "none", type: "NullLiteral" }, - { value: ",", type: "Comma" }, - { value: "0", type: "NumericLiteral" }, - { value: ")", type: "CloseParen" }, - { value: "|", type: "Pipe" }, - { value: "join", type: "Identifier" }, - { value: "(", type: "OpenParen" }, - { value: ",", type: "StringLiteral" }, - { value: ")", type: "CloseParen" }, - { value: "}}", type: "CloseExpression" }, - { value: "|", type: "Text" }, - { value: "{{", type: "OpenExpression" }, - { value: " 1 2 3 4 5 ", type: "StringLiteral" }, - { value: ".", type: "Dot" }, - { value: "split", type: "Identifier" }, - { value: "(", type: "OpenParen" }, - { value: "none", type: "NullLiteral" }, - { value: ",", type: "Comma" }, - { value: "3", type: "NumericLiteral" }, - { value: ")", type: "CloseParen" }, - { value: "|", type: "Pipe" }, - { value: "join", type: "Identifier" }, - { value: "(", type: "OpenParen" }, - { value: ",", type: "StringLiteral" }, - { value: ")", type: "CloseParen" }, - { value: "}}", type: "CloseExpression" }, - { value: "|", type: "Text" }, - { value: "{{", type: "OpenExpression" }, - { value: " 1 2 3 4 5 ", type: "StringLiteral" }, - { value: ".", type: "Dot" }, - { value: "split", type: "Identifier" }, - { value: "(", type: "OpenParen" }, - { value: " ", type: "StringLiteral" }, - { value: ",", type: "Comma" }, - { value: "0", type: "NumericLiteral" }, - { value: ")", type: "CloseParen" }, - { value: "|", type: "Pipe" }, - { value: "join", type: "Identifier" }, - { value: "(", type: "OpenParen" }, - { value: ",", type: "StringLiteral" }, - { value: ")", type: "CloseParen" }, - { value: "}}", type: "CloseExpression" }, - { value: "|", type: "Text" }, - { value: "{{", type: "OpenExpression" }, - { value: " 1 2 3 4 5 ", type: "StringLiteral" }, - { value: ".", type: "Dot" }, - { value: "split", type: "Identifier" }, - { value: "(", type: "OpenParen" }, - { value: " ", type: "StringLiteral" }, - { value: ",", type: "Comma" }, - { value: "3", type: "NumericLiteral" }, - { value: ")", type: "CloseParen" }, - { value: "|", type: "Pipe" }, - { value: "join", type: "Identifier" }, - { value: "(", type: "OpenParen" }, - { value: ",", type: "StringLiteral" }, - { value: ")", type: "CloseParen" }, - { value: "}}", type: "CloseExpression" }, - { value: "|", type: "Text" }, - { value: "{{", type: "OpenExpression" }, - { value: " 1 2 3 4 5 ", type: "StringLiteral" }, - { value: ".", type: "Dot" }, - { value: "split", type: "Identifier" }, - { value: "(", type: "OpenParen" }, - { value: " ", type: "StringLiteral" }, - { value: ",", type: "Comma" }, - { value: "10", type: "NumericLiteral" }, - { value: ")", type: "CloseParen" }, - { value: "|", type: "Pipe" }, - { value: "join", type: "Identifier" }, - { value: "(", type: "OpenParen" }, - { value: ",", type: "StringLiteral" }, - { value: ")", type: "CloseParen" }, - { value: "}}", type: "CloseExpression" }, - { value: "|", type: "Text" }, - ], }; const TEST_CONTEXT = { @@ -3393,6 +3197,13 @@ const TEST_CONTEXT = { // String methods STRING_METHODS: {}, STRING_METHODS_2: {}, + RSTRIP: {}, + LSTRIP: {}, + SPLIT: {}, + SPLIT_2: {}, + SPLIT_3: {}, + SPLIT_4: {}, + SPLIT_5: {}, // String indexing and slicing STRING_SLICING: { @@ -3569,17 +3380,6 @@ const TEST_CONTEXT = { MACROS: {}, MACROS_1: {}, MACROS_2: {}, - - // Strip - RSTRIP: {}, - LSTRIP: {}, - - // Split - SPLIT: {}, - SPLIT_2: {}, - SPLIT_3: {}, - SPLIT_4: {}, - SPLIT_5: {}, }; const EXPECTED_OUTPUTS = { @@ -3640,6 +3440,13 @@ const EXPECTED_OUTPUTS = { OBJ_METHODS: "AB A_B", STRING_METHODS: "AB ABCD abcd ", STRING_METHODS_2: "Test Test", + RSTRIP: ` test it`, + LSTRIP: `test it `, + SPLIT: `|test|it|`, + SPLIT_2: `||||test|it|||`, + SPLIT_3: `||||test|it |`, + SPLIT_4: `|["1", "2", "3"]|["", "a", "", "acca", "", "", ""]|["", "a", "baccabbb"]|`, + SPLIT_5: `|1 2 3 4 5 |1,2,3,4 5 | 1 2 3 4 5 |,1,2,3 4 5 |,1,2,3,4,5,|`, // String indexing and slicing STRING_SLICING: "|0|0123456789|012|123|12345678|13579|543210|", @@ -3733,17 +3540,6 @@ const EXPECTED_OUTPUTS = { MACROS: `|Hello Bob|Hello Alice|`, MACROS_1: `|Hello A.|Hello B!|Hello C?|`, MACROS_2: `|1,2,3|1,0,3|1,0,-1|1,0,-1|1,2,0|`, - - // RSTRIP/LSTRIP - RSTRIP: ` test it`, - LSTRIP: `test it `, - - // Split - SPLIT: `|test|it|`, - SPLIT_2: `||||test|it|||`, - SPLIT_3: `||||test|it |`, - SPLIT_4: `|["1", "2", "3"]|["", "a", "", "acca", "", "", ""]|["", "a", "baccabbb"]|`, - SPLIT_5: `|1 2 3 4 5 |1,2,3,4 5 | 1 2 3 4 5 |,1,2,3 4 5 |,1,2,3,4,5,|`, }; describe("Templates", () => { From a5abdb5a2089dba33670ea4239948c07168cc0f8 Mon Sep 17 00:00:00 2001 From: Joshua Lochner <26504141+xenova@users.noreply.github.com> Date: Fri, 2 May 2025 00:00:27 -0400 Subject: [PATCH 04/35] Add context-specific keyword tests --- packages/jinja/test/templates.test.js | 74 +++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/packages/jinja/test/templates.test.js b/packages/jinja/test/templates.test.js index 3e07c292d1..e0dc61bf98 100644 --- a/packages/jinja/test/templates.test.js +++ b/packages/jinja/test/templates.test.js @@ -163,6 +163,10 @@ const TEST_STRINGS = { MACROS: `{% macro hello(name) %}{{ 'Hello ' + name }}{% endmacro %}|{{ hello('Bob') }}|{{ hello('Alice') }}|`, MACROS_1: `{% macro hello(name, suffix='.') %}{{ 'Hello ' + name + suffix }}{% endmacro %}|{{ hello('A') }}|{{ hello('B', '!') }}|{{ hello('C', suffix='?') }}|`, MACROS_2: `{% macro fn(x, y=2, z=3) %}{{ x + ',' + y + ',' + z }}{% endmacro %}|{{ fn(1) }}|{{ fn(1, 0) }}|{{ fn(1, 0, -1) }}|{{ fn(1, y=0, z=-1) }}|{{ fn(1, z=0) }}|`, + + // Context-specific keywords + CONTEXT_KEYWORDS: `{% if if in in %}a{% endif %}{% set if = "a" %}{% set in = "abc" %}{% if if in in %}b{% endif %}`, + CONTEXT_KEYWORDS_1: `|{{ if }}|{% set if = 2 %}{% if if == 2 %}{{ if }}{% endif %}|`, }; const TEST_PARSED = { @@ -3109,6 +3113,68 @@ const TEST_PARSED = { { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, ], + + // Context-specific keywords + CONTEXT_KEYWORDS: [ + { value: "{%", type: "OpenStatement" }, + { value: "if", type: "Identifier" }, + { value: "if", type: "Identifier" }, + { value: "in", type: "Identifier" }, + { value: "in", type: "Identifier" }, + { value: "%}", type: "CloseStatement" }, + { value: "a", type: "Text" }, + { value: "{%", type: "OpenStatement" }, + { value: "endif", type: "Identifier" }, + { value: "%}", type: "CloseStatement" }, + { value: "{%", type: "OpenStatement" }, + { value: "set", type: "Identifier" }, + { value: "if", type: "Identifier" }, + { value: "=", type: "Equals" }, + { value: "a", type: "StringLiteral" }, + { value: "%}", type: "CloseStatement" }, + { value: "{%", type: "OpenStatement" }, + { value: "set", type: "Identifier" }, + { value: "in", type: "Identifier" }, + { value: "=", type: "Equals" }, + { value: "abc", type: "StringLiteral" }, + { value: "%}", type: "CloseStatement" }, + { value: "{%", type: "OpenStatement" }, + { value: "if", type: "Identifier" }, + { value: "if", type: "Identifier" }, + { value: "in", type: "Identifier" }, + { value: "in", type: "Identifier" }, + { value: "%}", type: "CloseStatement" }, + { value: "b", type: "Text" }, + { value: "{%", type: "OpenStatement" }, + { value: "endif", type: "Identifier" }, + { value: "%}", type: "CloseStatement" }, + ], + CONTEXT_KEYWORDS_1: [ + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "if", type: "Identifier" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{%", type: "OpenStatement" }, + { value: "set", type: "Identifier" }, + { value: "if", type: "Identifier" }, + { value: "=", type: "Equals" }, + { value: "2", type: "NumericLiteral" }, + { value: "%}", type: "CloseStatement" }, + { value: "{%", type: "OpenStatement" }, + { value: "if", type: "Identifier" }, + { value: "if", type: "Identifier" }, + { value: "==", type: "ComparisonBinaryOperator" }, + { value: "2", type: "NumericLiteral" }, + { value: "%}", type: "CloseStatement" }, + { value: "{{", type: "OpenExpression" }, + { value: "if", type: "Identifier" }, + { value: "}}", type: "CloseExpression" }, + { value: "{%", type: "OpenStatement" }, + { value: "endif", type: "Identifier" }, + { value: "%}", type: "CloseStatement" }, + { value: "|", type: "Text" }, + ], }; const TEST_CONTEXT = { @@ -3380,6 +3446,10 @@ const TEST_CONTEXT = { MACROS: {}, MACROS_1: {}, MACROS_2: {}, + + // Context-specific keywords + CONTEXT_KEYWORDS: {}, + CONTEXT_KEYWORDS_1: {}, }; const EXPECTED_OUTPUTS = { @@ -3540,6 +3610,10 @@ const EXPECTED_OUTPUTS = { MACROS: `|Hello Bob|Hello Alice|`, MACROS_1: `|Hello A.|Hello B!|Hello C?|`, MACROS_2: `|1,2,3|1,0,3|1,0,-1|1,0,-1|1,2,0|`, + + // Context-specific keywords + CONTEXT_KEYWORDS: `b`, + CONTEXT_KEYWORDS_1: `||2|`, }; describe("Templates", () => { From a29f2e140d1f6b9cd84020795e49f60cf7cb0267 Mon Sep 17 00:00:00 2001 From: Joshua Lochner <26504141+xenova@users.noreply.github.com> Date: Fri, 2 May 2025 00:01:12 -0400 Subject: [PATCH 05/35] Handle special case for membership of undefined --- packages/jinja/src/runtime.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/jinja/src/runtime.ts b/packages/jinja/src/runtime.ts index 80c91715e9..a8272a8657 100644 --- a/packages/jinja/src/runtime.ts +++ b/packages/jinja/src/runtime.ts @@ -484,6 +484,10 @@ export class Interpreter { } if (left instanceof UndefinedValue || right instanceof UndefinedValue) { + if (right instanceof UndefinedValue && ["in", "not in"].includes(node.operator.value)) { + // Special case: `anything in undefined` is `false` and `anything not in undefined` is `true` + return new BooleanValue(node.operator.value === "not in"); + } throw new Error("Cannot perform operation on undefined values"); } else if (left instanceof NullValue || right instanceof NullValue) { throw new Error("Cannot perform operation on null values"); From f93f308bbb89af3c23d7d951bfcd5260550cd7c2 Mon Sep 17 00:00:00 2001 From: Joshua Lochner <26504141+xenova@users.noreply.github.com> Date: Fri, 2 May 2025 00:01:27 -0400 Subject: [PATCH 06/35] Remove unused literals --- packages/jinja/src/ast.ts | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/packages/jinja/src/ast.ts b/packages/jinja/src/ast.ts index e7d12d4987..d8e216d61d 100644 --- a/packages/jinja/src/ast.ts +++ b/packages/jinja/src/ast.ts @@ -147,20 +147,6 @@ export class StringLiteral extends Literal { override type = "StringLiteral"; } -/** - * Represents a boolean constant in the template. - */ -export class BooleanLiteral extends Literal { - override type = "BooleanLiteral"; -} - -/** - * Represents null (none) in the template. - */ -export class NullLiteral extends Literal { - override type = "NullLiteral"; -} - /** * Represents an array literal in the template. */ From df588ab1547e71de0cff0783172a6890dff9f084 Mon Sep 17 00:00:00 2001 From: Joshua Lochner <26504141+xenova@users.noreply.github.com> Date: Fri, 2 May 2025 11:36:13 -0400 Subject: [PATCH 07/35] Add `MEMBERSHIP_UNDEFINED` and `TERNARY_CONSECUTIVE` unit tests --- packages/jinja/test/templates.test.js | 64 +++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/packages/jinja/test/templates.test.js b/packages/jinja/test/templates.test.js index e0dc61bf98..45c34ebc6b 100644 --- a/packages/jinja/test/templates.test.js +++ b/packages/jinja/test/templates.test.js @@ -81,6 +81,7 @@ const TEST_STRINGS = { MEMBERSHIP: `|{{ 0 in arr }}|{{ 1 in arr }}|{{ true in arr }}|{{ false in arr }}|{{ 'a' in arr }}|{{ 'b' in arr }}|`, MEMBERSHIP_NEGATION_1: `|{{ not 0 in arr }}|{{ not 1 in arr }}|{{ not true in arr }}|{{ not false in arr }}|{{ not 'a' in arr }}|{{ not 'b' in arr }}|`, MEMBERSHIP_NEGATION_2: `|{{ 0 not in arr }}|{{ 1 not in arr }}|{{ true not in arr }}|{{ false not in arr }}|{{ 'a' not in arr }}|{{ 'b' not in arr }}|`, + MEMBERSHIP_UNDEFINED: `|{{ x is defined }}|{{ y is defined }}|{{ x in y }}|{{ y in x }}|{{ 1 in y }}|{{ 1 in x }}|`, // Escaped characters ESCAPED_CHARS: `{{ '\\n' }}{{ '\\t' }}{{ '\\'' }}{{ '\\"' }}{{ '\\\\' }}{{ '|\\n|\\t|\\'|\\"|\\\\|' }}`, @@ -146,6 +147,7 @@ const TEST_STRINGS = { // Ternary operator TERNARY_OPERATOR: `|{{ 'a' if true else 'b' }}|{{ 'a' if false else 'b' }}|{{ 'a' if 1 + 1 == 2 else 'b' }}|{{ 'a' if 1 + 1 == 3 or 1 * 2 == 3 else 'b' }}|`, TERNARY_SET: `{% set x = 1 if True else 2 %}{{ x }}`, + TERNARY_CONSECUTIVE: `{% set x = 1 if False else 2 if False else 3 %}{{ x }}`, // Array literals ARRAY_LITERALS: `{{ [1, true, 'hello', [1, 2, 3, 4], var] | length }}`, @@ -1592,6 +1594,45 @@ const TEST_PARSED = { { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, ], + MEMBERSHIP_UNDEFINED: [ + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "x", type: "Identifier" }, + { value: "is", type: "Identifier" }, + { value: "defined", type: "Identifier" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "y", type: "Identifier" }, + { value: "is", type: "Identifier" }, + { value: "defined", type: "Identifier" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "x", type: "Identifier" }, + { value: "in", type: "Identifier" }, + { value: "y", type: "Identifier" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "y", type: "Identifier" }, + { value: "in", type: "Identifier" }, + { value: "x", type: "Identifier" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "1", type: "NumericLiteral" }, + { value: "in", type: "Identifier" }, + { value: "y", type: "Identifier" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "1", type: "NumericLiteral" }, + { value: "in", type: "Identifier" }, + { value: "x", type: "Identifier" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + ], // Escaped characters ESCAPED_CHARS: [ @@ -2844,6 +2885,25 @@ const TEST_PARSED = { { value: "x", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, ], + TERNARY_CONSECUTIVE: [ + { value: "{%", type: "OpenStatement" }, + { value: "set", type: "Identifier" }, + { value: "x", type: "Identifier" }, + { value: "=", type: "Equals" }, + { value: "1", type: "NumericLiteral" }, + { value: "if", type: "Identifier" }, + { value: "False", type: "Identifier" }, + { value: "else", type: "Identifier" }, + { value: "2", type: "NumericLiteral" }, + { value: "if", type: "Identifier" }, + { value: "False", type: "Identifier" }, + { value: "else", type: "Identifier" }, + { value: "3", type: "NumericLiteral" }, + { value: "%}", type: "CloseStatement" }, + { value: "{{", type: "OpenExpression" }, + { value: "x", type: "Identifier" }, + { value: "}}", type: "CloseExpression" }, + ], // Array literals ARRAY_LITERALS: [ @@ -3291,6 +3351,7 @@ const TEST_CONTEXT = { MEMBERSHIP_NEGATION_2: { arr: [0, true, "a"], }, + MEMBERSHIP_UNDEFINED: {}, // Escaped characters ESCAPED_CHARS: {}, @@ -3427,6 +3488,7 @@ const TEST_CONTEXT = { // Ternary operator TERNARY_OPERATOR: {}, TERNARY_SET: {}, + TERNARY_CONSECUTIVE: {}, // Array literals ARRAY_LITERALS: { var: true }, @@ -3528,6 +3590,7 @@ const EXPECTED_OUTPUTS = { MEMBERSHIP: "|true|false|true|false|true|false|", MEMBERSHIP_NEGATION_1: "|false|true|false|true|false|true|", MEMBERSHIP_NEGATION_2: "|false|true|false|true|false|true|", + MEMBERSHIP_UNDEFINED: "|false|false|false|false|false|false|", // Escaped characters ESCAPED_CHARS: `\n\t'"\\|\n|\t|'|"|\\|`, @@ -3593,6 +3656,7 @@ const EXPECTED_OUTPUTS = { // Ternary operator TERNARY_OPERATOR: `|a|b|a|b|`, TERNARY_SET: `1`, + TERNARY_CONSECUTIVE: `3`, // Array literals ARRAY_LITERALS: `5`, From c973a665994de2d6d149cc659f90dd9bde6d1051 Mon Sep 17 00:00:00 2001 From: Joshua Lochner <26504141+xenova@users.noreply.github.com> Date: Fri, 2 May 2025 17:31:05 -0400 Subject: [PATCH 08/35] Add new tests for filter + call statements --- packages/jinja/test/templates.test.js | 114 ++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) diff --git a/packages/jinja/test/templates.test.js b/packages/jinja/test/templates.test.js index 45c34ebc6b..8659e9ffd2 100644 --- a/packages/jinja/test/templates.test.js +++ b/packages/jinja/test/templates.test.js @@ -52,6 +52,7 @@ const TEST_STRINGS = { STRINGS: `{{ 'Bye' }}{{ bos_token + '[INST] ' }}`, STRINGS_1: `|{{ "test" }}|{{ "a" + 'b' + "c" }}|{{ '"' + "'" }}|{{ '\\'' }}|{{ "\\"" }}|`, STRINGS_2: `|{{ "" | length }}|{{ "a" | length }}|{{ '' | length }}|{{ 'a' | length }}|`, + STRINGS_3: `|{{ '{{ "hi" }}' }}|{{ '{% if true %}{% endif %}' }}|`, // Function calls FUNCTIONS: `{{ func() }}{{ func(apple) }}{{ func(x, 'test', 2, false) }}`, @@ -104,6 +105,9 @@ const TEST_STRINGS = { FILTER_OPERATOR_12: `{{ messages | rejectattr('role', 'equalto', 'system') | length }}`, FILTER_OPERATOR_13: `{{ tools | string }}`, + // Filter statements + FILTER_STATEMENTS: `{% filter upper %}text{% endfilter %}`, + // Logical operators between non-Booleans BOOLEAN_NUMERICAL: `|{{ 1 and 2 }}|{{ 1 and 0 }}|{{ 0 and 1 }}|{{ 0 and 0 }}|{{ 1 or 2 }}|{{ 1 or 0 }}|{{ 0 or 1 }}|{{ 0 or 0 }}|{{ not 1 }}|{{ not 0 }}|`, BOOLEAN_STRINGS: `|{{ 'a' and 'b' }}|{{ 'a' and '' }}|{{ '' and 'a' }}|{{ '' and '' }}|{{ 'a' or 'b' }}|{{ 'a' or '' }}|{{ '' or 'a' }}|{{ '' or '' }}|{{ not 'a' }}|{{ not '' }}|`, @@ -148,6 +152,7 @@ const TEST_STRINGS = { TERNARY_OPERATOR: `|{{ 'a' if true else 'b' }}|{{ 'a' if false else 'b' }}|{{ 'a' if 1 + 1 == 2 else 'b' }}|{{ 'a' if 1 + 1 == 3 or 1 * 2 == 3 else 'b' }}|`, TERNARY_SET: `{% set x = 1 if True else 2 %}{{ x }}`, TERNARY_CONSECUTIVE: `{% set x = 1 if False else 2 if False else 3 %}{{ x }}`, + TERNARY_SHORTCUT: `{{ 'foo' if false }}{{ 'bar' if true }}`, // Array literals ARRAY_LITERALS: `{{ [1, true, 'hello', [1, 2, 3, 4], var] | length }}`, @@ -157,6 +162,7 @@ const TEST_STRINGS = { // Object literals OBJECT_LITERALS: `{{ { 'key': 'value', key: 'value2', "key3": [1, {'foo': 'bar'} ] }['key'] }}`, + OBJECT_LITERALS_1: `{{{'key': {'key': 'value'}}['key']['key']}}`, // Array operators ARRAY_OPERATORS: `{{ ([1, 2, 3] + [4, 5, 6]) | length }}`, @@ -165,6 +171,7 @@ const TEST_STRINGS = { MACROS: `{% macro hello(name) %}{{ 'Hello ' + name }}{% endmacro %}|{{ hello('Bob') }}|{{ hello('Alice') }}|`, MACROS_1: `{% macro hello(name, suffix='.') %}{{ 'Hello ' + name + suffix }}{% endmacro %}|{{ hello('A') }}|{{ hello('B', '!') }}|{{ hello('C', suffix='?') }}|`, MACROS_2: `{% macro fn(x, y=2, z=3) %}{{ x + ',' + y + ',' + z }}{% endmacro %}|{{ fn(1) }}|{{ fn(1, 0) }}|{{ fn(1, 0, -1) }}|{{ fn(1, y=0, z=-1) }}|{{ fn(1, z=0) }}|`, + MACROS_3: `{%- macro dummy(a, b='!') -%}{{ a }} {{ caller() }}{{ b }}{%- endmacro %}{%- call dummy('hello') -%}name{%- endcall -%}`, // Context-specific keywords CONTEXT_KEYWORDS: `{% if if in in %}a{% endif %}{% set if = "a" %}{% set in = "abc" %}{% if if in in %}b{% endif %}`, @@ -947,6 +954,17 @@ const TEST_PARSED = { { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, ], + STRINGS_3: [ + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: '{{ "hi" }}', type: "StringLiteral" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "{% if true %}{% endif %}", type: "StringLiteral" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + ], // Function calls FUNCTIONS: [ @@ -2014,6 +2032,18 @@ const TEST_PARSED = { { value: "}}", type: "CloseExpression" }, ], + // Filter statements + FILTER_STATEMENTS: [ + { value: "{%", type: "OpenStatement" }, + { value: "filter", type: "Identifier" }, + { value: "upper", type: "Identifier" }, + { value: "%}", type: "CloseStatement" }, + { value: "text", type: "Text" }, + { value: "{%", type: "OpenStatement" }, + { value: "endfilter", type: "Identifier" }, + { value: "%}", type: "CloseStatement" }, + ], + // Logical operators between non-Booleans BOOLEAN_NUMERICAL: [ { value: "|", type: "Text" }, @@ -2904,6 +2934,18 @@ const TEST_PARSED = { { value: "x", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, ], + TERNARY_SHORTCUT: [ + { value: "{{", type: "OpenExpression" }, + { value: "foo", type: "StringLiteral" }, + { value: "if", type: "Identifier" }, + { value: "false", type: "Identifier" }, + { value: "}}", type: "CloseExpression" }, + { value: "{{", type: "OpenExpression" }, + { value: "bar", type: "StringLiteral" }, + { value: "if", type: "Identifier" }, + { value: "true", type: "Identifier" }, + { value: "}}", type: "CloseExpression" }, + ], // Array literals ARRAY_LITERALS: [ @@ -2978,6 +3020,25 @@ const TEST_PARSED = { { value: "]", type: "CloseSquareBracket" }, { value: "}}", type: "CloseExpression" }, ], + OBJECT_LITERALS_1: [ + { value: "{{", type: "OpenExpression" }, + { value: "{", type: "OpenCurlyBracket" }, + { value: "key", type: "StringLiteral" }, + { value: ":", type: "Colon" }, + { value: "{", type: "OpenCurlyBracket" }, + { value: "key", type: "StringLiteral" }, + { value: ":", type: "Colon" }, + { value: "value", type: "StringLiteral" }, + { value: "}", type: "CloseCurlyBracket" }, + { value: "}", type: "CloseCurlyBracket" }, + { value: "[", type: "OpenSquareBracket" }, + { value: "key", type: "StringLiteral" }, + { value: "]", type: "CloseSquareBracket" }, + { value: "[", type: "OpenSquareBracket" }, + { value: "key", type: "StringLiteral" }, + { value: "]", type: "CloseSquareBracket" }, + { value: "}}", type: "CloseExpression" }, + ], // Array operators ARRAY_OPERATORS: [ @@ -3173,6 +3234,45 @@ const TEST_PARSED = { { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, ], + MACROS_3: [ + { value: "{%", type: "OpenStatement" }, + { value: "macro", type: "Identifier" }, + { value: "dummy", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: "a", type: "Identifier" }, + { value: ",", type: "Comma" }, + { value: "b", type: "Identifier" }, + { value: "=", type: "Equals" }, + { value: "!", type: "StringLiteral" }, + { value: ")", type: "CloseParen" }, + { value: "%}", type: "CloseStatement" }, + { value: "{{", type: "OpenExpression" }, + { value: "a", type: "Identifier" }, + { value: "}}", type: "CloseExpression" }, + { value: " ", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "caller", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: ")", type: "CloseParen" }, + { value: "}}", type: "CloseExpression" }, + { value: "{{", type: "OpenExpression" }, + { value: "b", type: "Identifier" }, + { value: "}}", type: "CloseExpression" }, + { value: "{%", type: "OpenStatement" }, + { value: "endmacro", type: "Identifier" }, + { value: "%}", type: "CloseStatement" }, + { value: "{%", type: "OpenStatement" }, + { value: "call", type: "Identifier" }, + { value: "dummy", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: "hello", type: "StringLiteral" }, + { value: ")", type: "CloseParen" }, + { value: "%}", type: "CloseStatement" }, + { value: "name", type: "Text" }, + { value: "{%", type: "OpenStatement" }, + { value: "endcall", type: "Identifier" }, + { value: "%}", type: "CloseStatement" }, + ], // Context-specific keywords CONTEXT_KEYWORDS: [ @@ -3295,6 +3395,7 @@ const TEST_CONTEXT = { }, STRINGS_1: {}, STRINGS_2: {}, + STRINGS_3: {}, // Function calls FUNCTIONS: { @@ -3426,6 +3527,9 @@ const TEST_CONTEXT = { tools: [{ name: "some_tool", arguments: { some_name: "string" } }], }, + // Filter statements + FILTER_STATEMENTS: {}, + // Logical operators between non-Booleans BOOLEAN_NUMERICAL: {}, BOOLEAN_STRINGS: {}, @@ -3489,6 +3593,7 @@ const TEST_CONTEXT = { TERNARY_OPERATOR: {}, TERNARY_SET: {}, TERNARY_CONSECUTIVE: {}, + TERNARY_SHORTCUT: {}, // Array literals ARRAY_LITERALS: { var: true }, @@ -3500,6 +3605,7 @@ const TEST_CONTEXT = { OBJECT_LITERALS: { key: "key2", }, + OBJECT_LITERALS_1: {}, // Array operators ARRAY_OPERATORS: {}, @@ -3508,6 +3614,7 @@ const TEST_CONTEXT = { MACROS: {}, MACROS_1: {}, MACROS_2: {}, + MACROS_3: {}, // Context-specific keywords CONTEXT_KEYWORDS: {}, @@ -3561,6 +3668,7 @@ const EXPECTED_OUTPUTS = { STRINGS: "Bye[INST] ", STRINGS_1: `|test|abc|"'|'|"|`, STRINGS_2: `|0|1|0|1|`, + STRINGS_3: `|{{ "hi" }}|{% if true %}{% endif %}|`, // Function calls FUNCTIONS: "014", @@ -3613,6 +3721,9 @@ const EXPECTED_OUTPUTS = { FILTER_OPERATOR_12: `2`, FILTER_OPERATOR_13: `[{"name": "some_tool", "arguments": {"some_name": "string"}}]`, + // Filter statements + FILTER_STATEMENTS: `TEXT`, + // Logical operators between non-Booleans BOOLEAN_NUMERICAL: `|2|0|0|0|1|1|1|0|false|true|`, BOOLEAN_STRINGS: `|b||||a|a|a||false|true|`, @@ -3657,6 +3768,7 @@ const EXPECTED_OUTPUTS = { TERNARY_OPERATOR: `|a|b|a|b|`, TERNARY_SET: `1`, TERNARY_CONSECUTIVE: `3`, + TERNARY_SHORTCUT: `bar`, // Array literals ARRAY_LITERALS: `5`, @@ -3666,6 +3778,7 @@ const EXPECTED_OUTPUTS = { // Object literals OBJECT_LITERALS: `value`, + OBJECT_LITERALS_1: `value`, // Array operators ARRAY_OPERATORS: `6`, @@ -3674,6 +3787,7 @@ const EXPECTED_OUTPUTS = { MACROS: `|Hello Bob|Hello Alice|`, MACROS_1: `|Hello A.|Hello B!|Hello C?|`, MACROS_2: `|1,2,3|1,0,3|1,0,-1|1,0,-1|1,2,0|`, + MACROS_3: `hello name!`, // Context-specific keywords CONTEXT_KEYWORDS: `b`, From df0eb97c48cddd0a817efa01085591b8f0c6cbbc Mon Sep 17 00:00:00 2001 From: Joshua Lochner <26504141+xenova@users.noreply.github.com> Date: Fri, 2 May 2025 17:46:47 -0400 Subject: [PATCH 09/35] Add tilde operator --- packages/jinja/test/templates.test.js | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/packages/jinja/test/templates.test.js b/packages/jinja/test/templates.test.js index 8659e9ffd2..4742c5127b 100644 --- a/packages/jinja/test/templates.test.js +++ b/packages/jinja/test/templates.test.js @@ -47,6 +47,7 @@ const TEST_STRINGS = { // Binary expressions BINOP_EXPR: `{{ 1 % 2 }}{{ 1 < 2 }}{{ 1 > 2 }}{{ 1 >= 2 }}{{ 2 <= 2 }}{{ 2 == 2 }}{{ 2 != 3 }}{{ 2 + 3 }}`, + BINOP_EXPR_1: `{{ 1 ~ "+" ~ 2 ~ "=" ~ 3 ~ " is " ~ true }}`, // Strings STRINGS: `{{ 'Bye' }}{{ bos_token + '[INST] ' }}`, @@ -886,6 +887,23 @@ const TEST_PARSED = { { value: "3", type: "NumericLiteral" }, { value: "}}", type: "CloseExpression" }, ], + BINOP_EXPR_1: [ + { value: "{{", type: "OpenExpression" }, + { value: "1", type: "NumericLiteral" }, + { value: "~", type: "AdditiveBinaryOperator" }, + { value: "+", type: "StringLiteral" }, + { value: "~", type: "AdditiveBinaryOperator" }, + { value: "2", type: "NumericLiteral" }, + { value: "~", type: "AdditiveBinaryOperator" }, + { value: "=", type: "StringLiteral" }, + { value: "~", type: "AdditiveBinaryOperator" }, + { value: "3", type: "NumericLiteral" }, + { value: "~", type: "AdditiveBinaryOperator" }, + { value: " is ", type: "StringLiteral" }, + { value: "~", type: "AdditiveBinaryOperator" }, + { value: "true", type: "Identifier" }, + { value: "}}", type: "CloseExpression" }, + ], // Strings STRINGS: [ @@ -3388,6 +3406,7 @@ const TEST_CONTEXT = { // Binary expressions BINOP_EXPR: {}, + BINOP_EXPR_1: {}, // Strings STRINGS: { @@ -3663,6 +3682,7 @@ const EXPECTED_OUTPUTS = { // Binary expressions BINOP_EXPR: "1truefalsefalsetruetruetrue5", + BINOP_EXPR_1: "1+2=3 is true", // Strings STRINGS: "Bye[INST] ", From b91b7bbfe755f98a7ebba43fdcc02814548a485c Mon Sep 17 00:00:00 2001 From: Joshua Lochner <26504141+xenova@users.noreply.github.com> Date: Fri, 2 May 2025 17:47:05 -0400 Subject: [PATCH 10/35] Add minicpm e2e test --- packages/jinja/test/e2e.test.js | 57 +++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/packages/jinja/test/e2e.test.js b/packages/jinja/test/e2e.test.js index 6e32509464..d4df84ecc2 100644 --- a/packages/jinja/test/e2e.test.js +++ b/packages/jinja/test/e2e.test.js @@ -849,6 +849,63 @@ const TEST_CUSTOM_TEMPLATES = Object.freeze({ target: '<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|># System Preamble\nYou are in contextual safety mode. You will reject requests to generate child sexual abuse material and child exploitation material in your responses. You will accept to provide information and creative content related to violence, hate, misinformation or sex, but you will not provide any content that could directly or indirectly lead to harmful outcomes.\n\nYour information cutoff date is June 2024.\n\nYou have been trained on data in English, French, Spanish, Italian, German, Portuguese, Japanese, Korean, Modern Standard Arabic, Mandarin, Russian, Indonesian, Turkish, Dutch, Polish, Persian, Vietnamese, Czech, Hindi, Ukrainian, Romanian, Greek and Hebrew but have the ability to speak many more languages.\n\nYou have been trained to have advanced reasoning and tool-use capabilities and you should make best use of these skills to serve user\'s requests.\n\n## Tool Use\nThink about how you can make best use of the provided tools to help with the task and come up with a high level plan that you will execute first.\n\n0. Start by writing <|START_THINKING|> followed by a detailed step by step plan of how you will solve the problem. For each step explain your thinking fully and give details of required tool calls (if needed). Unless specified otherwise, you write your plan in natural language. When you finish, close it out with <|END_THINKING|>.\n You can optionally choose to skip this step when the user request is so straightforward to address that only a trivial plan would be needed.\n NOTE: You MUST skip this step when you are directly responding to the user\'s request without using any tools.\n\nThen carry out your plan by repeatedly executing the following steps.\n1. Action: write <|START_ACTION|> followed by a list of JSON-formatted tool calls, with each one containing "tool_name" and "parameters" fields.\n When there are multiple tool calls which are completely independent of each other (i.e. they can be executed in parallel), you should list them out all together in one step. When you finish, close it out with <|END_ACTION|>.\n2. Observation: you will then receive results of those tool calls in JSON format in the very next turn, wrapped around by <|START_TOOL_RESULT|> and <|END_TOOL_RESULT|>. Carefully observe those results and think about what to do next. Note that these results will be provided to you in a separate turn. NEVER hallucinate results.\n Every tool call produces a list of results (when a tool call produces no result or a single result, it\'ll still get wrapped inside a list). Each result is clearly linked to its originating tool call via its "tool_call_id".\n3. Reflection: start the next turn by writing <|START_THINKING|> followed by what you\'ve figured out so far, any changes you need to make to your plan, and what you will do next. When you finish, close it out with <|END_THINKING|>.\n You can optionally choose to skip this step when everything is going according to plan and no special pieces of information or reasoning chains need to be recorded.\n NOTE: You MUST skip this step when you are done with tool-use actions and are ready to respond to the user.\n\nYou can repeat the above 3 steps multiple times (could be 0 times too if no suitable tool calls are available or needed), until you decide it\'s time to finally respond to the user.\n\n4. Response: then break out of the loop and write <|START_RESPONSE|> followed by a piece of text which serves as a response to the user\'s last request. Use all previous tool calls and results to help you when formulating your response. When you finish, close it out with <|END_RESPONSE|>.\n\n## Available Tools\nHere is the list of tools that you have available to you.\nYou can ONLY use the tools listed here. When a tool is not listed below, it is NOT available and you should NEVER attempt to use it.\nEach tool is represented as a JSON object with fields like "name", "description", "parameters" (per JSON Schema), and optionally, "responses" (per JSON Schema).\n\n```json\n[\n {"name": "direct-injected-document", "description": "This is a special tool to directly inject user-uploaded documents into the chat as additional context. DO NOT use this tool by yourself!", "parameters": {"type": "object", "properties": {}, "required": []}, "responses": {"200": {"description": "Successfully returned a list of chunked text snippets from the directly uploaded documents.", "content": {"application/json": {"schema": {"type": "array", "items": {"type": "object", "required": ["url", "snippet"], "properties": {"url": {"type": "string", "description": "The url of the uploaded document."}, "snippet": {"type": "string", "description": "The text snippet for the returned document chunk."}}}}}}}}}\n]\n```\n\n# Default Preamble\nThe following instructions are your defaults unless specified elsewhere in developer preamble or user prompt.\n- Your name is Command.\n- You are a large language model built by Cohere.\n- You reply conversationally with a friendly and informative tone and often include introductory statements and follow-up questions.\n- If the input is ambiguous, ask clarifying follow-up questions.\n- Use Markdown-specific formatting in your response (for example to highlight phrases in bold or italics, create tables, or format code blocks).\n- Use LaTeX to generate mathematical notation for complex equations.\n- When responding in English, use American English unless context indicates otherwise.\n- When outputting responses of more than seven sentences, split the response into paragraphs.\n- Prefer the active voice.\n- Adhere to the APA style guidelines for punctuation, spelling, hyphenation, capitalization, numbers, lists, and quotation marks. Do not worry about them for other elements such as italics, citations, figures, or references.\n- Use gender-neutral pronouns for unspecified persons.\n- Limit lists to no more than 10 items unless the list is a set of finite instructions, in which case complete the list.\n- Use the third person when asked to write a summary.\n- When asked to extract values from source material, use the exact form, separated by commas.\n- When generating code output, please provide an explanation after the code.\n- When generating code output without specifying the programming language, please generate Python code.\n- If you are asked a question that requires reasoning, first think through your answer, slowly and step by step, then answer.<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|USER_TOKEN|>What has Man always dreamed of?<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|><|START_THINKING|>I will look through the document to address the users needs.<|END_THINKING|><|START_ACTION|>[\n {"tool_call_id": "0", "tool_name": "direct-injected-document", "parameters": {}}\n]<|END_ACTION|><|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|><|START_TOOL_RESULT|>[\n {\n "tool_call_id": "0",\n "results": {\n "0": {"heading": "The Moon: Our Age-Old Foe", "body": "Man has always dreamed of destroying the moon. In this essay, I shall..."},\n "1": {"heading": "Love is all you need", "body": "Man\'s dream has always been to find love. This profound lesson..."}\n },\n "is_error": null\n }\n]<|END_TOOL_RESULT|><|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>', }, + "openbmb/MiniCPM3-4B": { + chat_template: + "{%- macro json_to_python_type(param_name, json_spec) %}\n{%- set basic_type_map = {\n 'string': 'str',\n 'number': 'float',\n 'integer': 'int',\n 'boolean': 'bool',\n 'null': 'None'\n} %}\n\n{%- if json_spec.enum %}\n {{- param_name|title }}\n{%- elif basic_type_map[json_spec.type] is defined %}\n {{- basic_type_map[json_spec.type] }}\n{%- elif json_spec.type == 'array' %}\n {{- 'List[' + json_to_python_type(param_name, json_spec['items']) + ']' }}\n{%- elif json_spec.type == 'object' %}\n {{- 'Dict[str, ' + json_to_python_type(param_name, json_spec.additionalProperties if json_spec.additionalProperties else 'Any') + ']' if not json_spec.properties else param_name|title }}\n{%- elif json_spec.type is iterable %}\n {{- 'Union[' }}\n {%- for t in json_spec.type %}\n {{- json_to_python_type(param_name, {'type': t}) }}\n {{- ', ' if not loop.last }}\n {%- endfor %}\n {{- ']' }}\n{%- else %}\n {{- 'Any' }}\n{%- endif %}\n{%- endmacro %}\n\n{%- macro object_to_fields(json_spec, field_indent) %}\n {%- set o_ns = namespace(f = caller()) %}\n {%- for param_name, param_fields in json_spec.properties|items %}\n {%- if param_fields.enum %}\n {{- '\\n\\nclass ' + param_name|title + '(Enum):\\n' }}\n {%- for enum_option in param_fields.enum %}\n {{- ' enum_' + loop.index0|string + ' = ' + enum_option|tojson + '\\n' }}\n {%- endfor %}\n {%- elif param_fields.type == 'object' and param_fields.properties %}\n {%- call object_to_fields(param_fields, ' ') %}\n {{- '\\n\\nclass ' + param_name|title + '(BaseModel):\\n' }}\n {%- endcall %}\n {%- elif param_fields.type == 'array' and param_fields['items'] and param_fields['items'].type == 'object' and param_fields['items'].properties %}\n {%- call object_to_fields(param_fields['items'], ' ') %}\n {{- '\\n\\nclass ' + param_name|title + '(BaseModel):\\n' }}\n {%- endcall %}\n {%- endif %}\n {%- set param_default = param_fields.default|tojson if param_fields.default is string else param_fields.default|string if param_fields.default is defined else 'None' %}\n {%- set o_ns.f = o_ns.f + field_indent + param_name + ': ' %}\n {%- set o_ns.f = o_ns.f + ('Optional[' + json_to_python_type(param_name, param_fields) + ']' if param_name not in json_spec.required else json_to_python_type(param_name, param_fields)) %}\n {%- if not param_fields.title and not param_fields.description and not param_fields.pattern %}\n {%- set o_ns.f = o_ns.f + (' = ' + param_default if param_name not in json_spec.required else '') %}\n {%- else %}\n {%- set o_ns.f = o_ns.f + (' = Field(...' if param_name in json_spec.required else ' = Field(' + param_default) %}\n {%- set o_ns.f = o_ns.f + (', description=' + param_fields.description|tojson if param_fields.description else '') %}\n {%- set o_ns.f = o_ns.f + (', regex=' + param_fields.pattern|tojson if param_fields.pattern else '') %}\n {%- set o_ns.f = o_ns.f + (', title=' + param_fields.title|tojson if param_fields.title else '') %}\n {%- set o_ns.f = o_ns.f + ')' %}\n {%- endif %}\n {%- set o_ns.f = o_ns.f + '\\n' %}\n {%- endfor %}\n {{- o_ns.f }}\n{%- endmacro %}\n\n{%- macro tool_parser(tools) %}\n{%- for tool in tools %}\n {%- if tool.type is not defined or tool.type == 'function' %}\n {%- if tool.function is defined %}\n {%- set tool = tool.function %}\n {%- endif %}\n {%- set tool_params = tool.parameters if tool.parameters is defined else none %}\n {%- call object_to_fields(tool_params, ' ') %}\n {{- '\\n\\ndef ' + tool.name + '(' }}\n {%- if tool_params %}\n {%- for param_name, param_fields in tool_params.properties|items %}\n {%- set param_default = param_fields.default|tojson if param_fields.default is string else param_fields.default|string if param_fields.default is defined else 'None' %}\n {{- ', ' if loop.index0 != 0 }}\n {{- param_name }}\n {{- '=' + param_default if param_name not in tool_params.required }}\n {%- endfor %}\n {%- endif %}\n {{- '):\\n \"\"\"' }}\n {{- tool.description }}\n {{- '\\n\\n Args:\\n' if tool_params else '\\n' }}\n {%- endcall %}\n {{- ' \"\"\"\\n' }}\n {%- endif %}\n{%- endfor %}\n{%- endmacro %}\n\n{%- if messages[0]['role'] == 'system' %}\n {%- set loop_messages = messages[1:] %}\n {%- set system_message = messages[0]['content'] %}\n{%- else %}\n {%- set loop_messages = messages %}\n {%- set system_message = '' %}\n{%- endif %}\n{{- '<|im_start|>system\\n' + system_message if system_message or tools }}\n{%- if tools %}\n {{- '\\n# Functions\\nHere is a list of functions that you can invoke:\\n```python\\nfrom enum import Enum\\nfrom typing import List, Dict, Optional\\nfrom pydantic import BaseModel, Field\\n\\n' }}\n {{- tool_parser(tools) }}\n {{- \"\\n```\\n\\n# Function Call Rule and Output Format\\n- If the user's question can be answered without calling any function, please answer the user's question directly. In this situation, you should return your thought and answer the user's question directly.\\n- If the user cannot be answered without calling any function, and the user does not provide enough information to call functions, please ask the user for more information. In this situation, you should return your thought and ask the user for more information.\\n- If the user's question cannot be answered without calling any function, and the user has provided enough information to call functions to solve it, you should call the functions. In this situation, the assistant should return your thought and call the functions.\\n- Use default parameters unless the user has specified otherwise.\\n- You should answer in the following format:\\n\\n<|thought_start|>\\n{explain why the user's question can be answered without calling a function or why you should ask the user for more information or why you should call one or more functions and your plan to solve the user's question.}\\n<|thought_end|>\\n<|tool_call_start|>\\n```python\\nfunc1(params_name=params_value, params_name2=params_value2...)\\nfunc2(params)\\n```\\n<|tool_call_end|>\\n{answer the user's question directly or ask the user for more information}\" }}\n{%- endif %}\n{{- '<|im_end|>\\n' if system_message or tools }}\n{%- for message in loop_messages %}\n {%- set content = message.content %}\n {%- if message.role == 'assistant' and message.tool_calls %}\n {{- '<|im_start|>' + message.role + '\\n' }}\n {{- '<|thought_start|>\\n' + message.thought + '\\n<|thought_end|>\\n' if message.thought }}\n {{- '<|tool_call_start|>\\n```python\\n' }}\n {%- for tool_call in message.tool_calls %}\n {%- if tool_call.function is defined %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- tool_call.name + '(' }}\n {%- if tool_call.arguments is defined and tool_call.arguments|length > 0 %}\n {%- for param_name, param_value in tool_call.arguments|items %}\n {{- param_name + '=' + param_value|tojson }}\n {{- ',' if not loop.last }}\n {%- endfor %}\n {%- endif %}\n {{- ')\\n' }}\n {%- endfor %}\n {{- '```\\n<|tool_call_end|>\\n' }}\n {{- content if content and not content.startswith('<|tool_call_start|>') }}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == 'assistant' and message.thought %}\n {{- '<|im_start|>' + message.role + '\\n' + '<|thought_start|>\\n' + message.thought + '\\n<|thought_end|>\\n' + content + '<|im_end|>\\n' }}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endfor %}\n\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n{%- endif %}", + data: { + // Example adapted from https://github.com/OpenBMB/MiniCPM/blob/de20166b6357abb3338ac6b5d1521dcf4edb14dd/demo/minicpm3/function_call/function_calling.py#L30-L76 + messages: [ + { + role: "system", + content: "You are a helpful customer support assistant. Use the supplied tools to assist the user.", + }, + { + role: "user", + content: "Hi, can you tell me the delivery date for my order? The order id is 1234 and 4321.", + }, + { + content: "", + tool_calls: [ + { + type: "function", + function: { + name: "get_delivery_date", + arguments: { order_id: "1234" }, + }, + id: "call_b4ab0b4ec4b5442e86f017fe0385e22e", + }, + { + type: "function", + function: { + name: "get_delivery_date", + arguments: { order_id: "4321" }, + }, + id: "call_628965479dd84794bbb72ab9bdda0c39", + }, + ], + role: "assistant", + }, + { + role: "tool", + content: '{"delivery_date": "2024-09-05", "order_id": "1234"}', + tool_call_id: "call_b4ab0b4ec4b5442e86f017fe0385e22e", + }, + { + role: "tool", + content: '{"delivery_date": "2024-09-05", "order_id": "4321"}', + tool_call_id: "call_628965479dd84794bbb72ab9bdda0c39", + }, + { + content: "Both your orders will be delivered on 2024-09-05.", + role: "assistant", + thought: "\nI have the information you need, both orders will be delivered on the same date, 2024-09-05.\n", + }, + ], + add_generation_prompt: true, + }, + target: + '<|im_start|>system\nYou are a helpful customer support assistant. Use the supplied tools to assist the user.<|im_end|>\n<|im_start|>user\nHi, can you tell me the delivery date for my order? The order id is 1234 and 4321.<|im_end|>\n<|im_start|>assistant\n<|tool_call_start|>\n```python\nget_delivery_date(order_id="1234")\nget_delivery_date(order_id="4321")\n```\n<|tool_call_end|>\n<|im_end|>\n<|im_start|>tool\n{"delivery_date": "2024-09-05", "order_id": "1234"}<|im_end|>\n<|im_start|>tool\n{"delivery_date": "2024-09-05", "order_id": "4321"}<|im_end|>\n<|im_start|>assistant\n<|thought_start|>\n\nI have the information you need, both orders will be delivered on the same date, 2024-09-05.\n\n<|thought_end|>\nBoth your orders will be delivered on 2024-09-05.<|im_end|>\n<|im_start|>assistant\n', + }, }); function render({ chat_template, data, target }) { From 7342226e62f3cf69e62ceff5be576a8ffaf7ff48 Mon Sep 17 00:00:00 2001 From: Joshua Lochner <26504141+xenova@users.noreply.github.com> Date: Fri, 2 May 2025 17:53:08 -0400 Subject: [PATCH 11/35] Implement basic call/filter statements --- packages/jinja/src/ast.ts | 27 ++++++++++++++- packages/jinja/src/format.ts | 32 ++++++++++++++++-- packages/jinja/src/parser.ts | 29 +++++++++++++--- packages/jinja/src/runtime.ts | 62 +++++++++++++++++++++++++++-------- 4 files changed, 128 insertions(+), 22 deletions(-) diff --git a/packages/jinja/src/ast.ts b/packages/jinja/src/ast.ts index d8e216d61d..ceca196774 100644 --- a/packages/jinja/src/ast.ts +++ b/packages/jinja/src/ast.ts @@ -200,15 +200,28 @@ export class FilterExpression extends Expression { } } +export class FilterStatement extends Statement { + override type = "FilterStatement"; + + constructor( + public filter: Identifier | CallExpression, + public body: Statement[] + ) { + super(); + } +} + /** * An operation which filters a sequence of objects by applying a test to each object, * and only selecting the objects with the test succeeding. + * + * It may also be used as a shortcut for a ternary operator. */ export class SelectExpression extends Expression { override type = "SelectExpression"; constructor( - public iterable: Expression, + public lhs: Expression, public test: Expression ) { super(); @@ -277,3 +290,15 @@ export class KeywordArgumentExpression extends Expression { super(); } } + +export class CallStatement extends Statement { + override type = "CallStatement"; + + constructor( + public call: CallExpression, + public params: Expression[], + public body: Statement[] + ) { + super(); + } +} diff --git a/packages/jinja/src/format.ts b/packages/jinja/src/format.ts index 72d9f4feac..51e52c83db 100644 --- a/packages/jinja/src/format.ts +++ b/packages/jinja/src/format.ts @@ -22,6 +22,8 @@ import type { LogicalNegationExpression, SliceExpression, KeywordArgumentExpression, + CallStatement, + FilterStatement, } from "./ast"; const NEWLINE = "\n"; @@ -65,6 +67,10 @@ function formatStatement(node: Statement, depth: number, indentStr: string): str return pad + createStatement("break"); case "Continue": return pad + createStatement("continue"); + case "CallStatement": + return formatCallStatement(node as CallStatement, depth, indentStr); + case "FilterStatement": + return formatFilterStatement(node as FilterStatement, depth, indentStr); default: return pad + "{{- " + formatExpression(node as Expression) + " -}}"; } @@ -118,7 +124,7 @@ function formatFor(node: For, depth: number, indentStr: string): string { if (node.iterable.type === "SelectExpression") { // Handle special case: e.g., `for x in [1, 2, 3] if x > 2` const n = node.iterable as SelectExpression; - formattedIterable = `${formatExpression(n.iterable)} if ${formatExpression(n.test)}`; + formattedIterable = `${formatExpression(n.lhs)} if ${formatExpression(n.test)}`; } else { formattedIterable = formatExpression(node.iterable); } @@ -165,6 +171,28 @@ function formatMacro(node: Macro, depth: number, indentStr: string): string { ); } +function formatCallStatement(node: CallStatement, depth: number, indentStr: string): string { + const pad = indentStr.repeat(depth); + const params = node.params.length > 0 ? `(${node.params.map((p) => p.value).join(", ")})` : ""; + const callExpr = formatExpression(node.call, -1); + let out = pad + createStatement(`call${params}`, callExpr) + NEWLINE; + out += formatStatements(node.body, depth + 1, indentStr) + NEWLINE; + out += pad + createStatement("endcall"); + return out; +} + +function formatFilterStatement(node: FilterStatement, depth: number, indentStr: string): string { + const pad = indentStr.repeat(depth); + const spec = + node.filter.type === "Identifier" + ? (node.filter as Identifier).value + : formatExpression(node.filter as CallExpression, -1); + let out = pad + createStatement("filter", spec) + NEWLINE; + out += formatStatements(node.body, depth + 1, indentStr) + NEWLINE; + out += pad + createStatement("endfilter"); + return out; +} + function formatExpression(node: Expression, parentPrec: number = -1): string { switch (node.type) { case "Identifier": @@ -215,7 +243,7 @@ function formatExpression(node: Expression, parentPrec: number = -1): string { } case "SelectExpression": { const n = node as SelectExpression; - return `${formatExpression(n.iterable, -1)} | select(${formatExpression(n.test, -1)})`; + return `${formatExpression(n.lhs, -1)} if ${formatExpression(n.test, -1)}`; } case "TestExpression": { const n = node as TestExpression; diff --git a/packages/jinja/src/parser.ts b/packages/jinja/src/parser.ts index d3f3e99434..3d5ce0b0cc 100644 --- a/packages/jinja/src/parser.ts +++ b/packages/jinja/src/parser.ts @@ -24,6 +24,8 @@ import { TupleLiteral, Macro, SelectExpression, + CallStatement, + FilterStatement, } from "./ast"; /** @@ -130,7 +132,8 @@ export function parse(tokens: Token[]): Program { expectIdentifier("endfor"); expect(TOKEN_TYPES.CloseStatement, "Expected %} token"); break; - + case "call": + throw new SyntaxError(`Call statements are not supported yet`); case "break": ++current; expect(TOKEN_TYPES.CloseStatement, "Expected closing statement token"); @@ -141,6 +144,22 @@ export function parse(tokens: Token[]): Program { expect(TOKEN_TYPES.CloseStatement, "Expected closing statement token"); result = new Continue(); break; + case "filter": + ++current; // consume 'filter' + let filterNode = parsePrimaryExpression(); + if (filterNode instanceof Identifier && is(TOKEN_TYPES.OpenParen)) { + filterNode = parseCallExpression(filterNode); + } + expect(TOKEN_TYPES.CloseStatement, "Expected closing statement token"); + const filterBody: Statement[] = []; + while (!isStatement("endfilter")) { + filterBody.push(parseAny()); + } + expect(TOKEN_TYPES.OpenStatement, "Expected '{%'"); + expectIdentifier("endfilter"); + expect(TOKEN_TYPES.CloseStatement, "Expected '%}'"); + result = new FilterStatement(filterNode as Identifier | CallExpression, filterBody); + break; default: throw new SyntaxError(`Unknown statement type: ${name}`); } @@ -296,16 +315,16 @@ export function parse(tokens: Token[]): Program { if (isIdentifier("if")) { // Ternary expression ++current; // consume 'if' - const predicate = parseLogicalOrExpression(); + const test = parseLogicalOrExpression(); if (isIdentifier("else")) { // Ternary expression with else ++current; // consume 'else' - const b = parseLogicalOrExpression(); - return new If(predicate, [a], [b]); + const alternate = parseIfExpression(); // recurse to support chained ternaries + return new If(test, [a], [alternate]); } else { // Select expression on iterable - return new SelectExpression(a, predicate); + return new SelectExpression(a, test); } } return a; diff --git a/packages/jinja/src/runtime.ts b/packages/jinja/src/runtime.ts index a8272a8657..b7f5ada37e 100644 --- a/packages/jinja/src/runtime.ts +++ b/packages/jinja/src/runtime.ts @@ -21,6 +21,8 @@ import type { Macro, Expression, SelectExpression, + CallStatement, + FilterStatement, } from "./ast"; import { range, slice, titleCase } from "./utils"; @@ -491,6 +493,9 @@ export class Interpreter { throw new Error("Cannot perform operation on undefined values"); } else if (left instanceof NullValue || right instanceof NullValue) { throw new Error("Cannot perform operation on null values"); + } else if (node.operator.value === "~") { + // toString and concatenation + return new StringValue(left.value.toString() + right.value.toString()); } else if (left instanceof NumericValue && right instanceof NumericValue) { // Evaulate pure numeric operations with binary operators. switch (node.operator.value) { @@ -583,12 +588,7 @@ export class Interpreter { return [positionalArguments, keywordArguments]; } - /** - * Evaluates expressions following the filter operation type. - */ - private evaluateFilterExpression(node: FilterExpression, environment: Environment): AnyRuntimeValue { - const operand = this.evaluate(node.operand, environment); - + private applyFilter(operand: AnyRuntimeValue, filterNode: Identifier | CallExpression, environment: Environment) { // For now, we only support the built-in filters // TODO: Add support for non-identifier filters // e.g., functions which return filters: {{ numbers | select("odd") }} @@ -601,8 +601,8 @@ export class Interpreter { // https://jinja.palletsprojects.com/en/3.0.x/templates/#list-of-builtin-filters - if (node.filter.type === "Identifier") { - const filter = node.filter as Identifier; + if (filterNode.type === "Identifier") { + const filter = filterNode as Identifier; if (filter.value === "tojson") { return new StringValue(toJSON(operand)); @@ -693,8 +693,8 @@ export class Interpreter { } } throw new Error(`Cannot apply filter "${filter.value}" to type: ${operand.type}`); - } else if (node.filter.type === "CallExpression") { - const filter = node.filter as CallExpression; + } else if (filterNode.type === "CallExpression") { + const filter = filterNode as CallExpression; if (filter.callee.type !== "Identifier") { throw new Error(`Unknown filter: ${filter.callee.type}`); @@ -821,7 +821,15 @@ export class Interpreter { throw new Error(`Cannot apply filter "${filterName}" to type: ${operand.type}`); } } - throw new Error(`Unknown filter: ${node.filter.type}`); + throw new Error(`Unknown filter: ${filterNode.type}`); + } + + /** + * Evaluates expressions following the filter operation type. + */ + private evaluateFilterExpression(node: FilterExpression, environment: Environment): AnyRuntimeValue { + const operand = this.evaluate(node.operand, environment); + return this.applyFilter(operand, node.filter, environment); } /** @@ -842,6 +850,17 @@ export class Interpreter { return new BooleanValue(node.negate ? !result : result); } + /** + * Evaluates expressions following the select operation type. + */ + private evaluateSelectExpression(node: SelectExpression, environment: Environment): AnyRuntimeValue { + const predicate = this.evaluate(node.test, environment); + if (!predicate.__bool__().value) { + return new UndefinedValue(); + } + return this.evaluate(node.lhs, environment); + } + /** * Evaluates expressions following the unary operation type. */ @@ -1003,7 +1022,7 @@ export class Interpreter { let test, iterable; if (node.iterable.type === "SelectExpression") { const select = node.iterable as SelectExpression; - iterable = this.evaluate(select.iterable, scope); + iterable = this.evaluate(select.lhs, scope); test = select.test; } else { iterable = this.evaluate(node.iterable, scope); @@ -1156,8 +1175,18 @@ export class Interpreter { return new NullValue(); } + private evaluateCallStatement(node: CallStatement, environment: Environment): AnyRuntimeValue { + // TODO implement this + throw new Error("Call statements are not yet implemented"); + } + + private evaluateFilterStatement(node: FilterStatement, environment: Environment): AnyRuntimeValue { + const rendered = this.evaluateBlock(node.body, environment); + return this.applyFilter(rendered, node.filter, environment); + } + evaluate(statement: Statement | undefined, environment: Environment): AnyRuntimeValue { - if (statement === undefined) return new UndefinedValue(); + if (!statement) return new UndefinedValue(); switch (statement.type) { // Program @@ -1173,6 +1202,8 @@ export class Interpreter { return this.evaluateFor(statement as For, environment); case "Macro": return this.evaluateMacro(statement as Macro, environment); + case "CallStatement": + return this.evaluateCallStatement(statement as CallStatement, environment); case "Break": throw new BreakControl(); @@ -1212,9 +1243,12 @@ export class Interpreter { return this.evaluateBinaryExpression(statement as BinaryExpression, environment); case "FilterExpression": return this.evaluateFilterExpression(statement as FilterExpression, environment); + case "FilterStatement": + return this.evaluateFilterStatement(statement as FilterStatement, environment); case "TestExpression": return this.evaluateTestExpression(statement as TestExpression, environment); - + case "SelectExpression": + return this.evaluateSelectExpression(statement as SelectExpression, environment); default: throw new SyntaxError(`Unknown node type: ${statement.type}`); } From f3b31177c023b39357190a73623550fe871d452d Mon Sep 17 00:00:00 2001 From: Joshua Lochner <26504141+xenova@users.noreply.github.com> Date: Fri, 2 May 2025 17:54:04 -0400 Subject: [PATCH 12/35] Improved lexing - Add support for tilde operator - Handle custom transformers-specific `generation` tag. - Be aware of curly bracket depth when lexing --- packages/jinja/src/lexer.ts | 45 +++++++++++++++++++++++++++---------- 1 file changed, 33 insertions(+), 12 deletions(-) diff --git a/packages/jinja/src/lexer.ts b/packages/jinja/src/lexer.ts index 34fe33377d..bc62c4767b 100644 --- a/packages/jinja/src/lexer.ts +++ b/packages/jinja/src/lexer.ts @@ -24,7 +24,7 @@ export const TOKEN_TYPES = Object.freeze({ Pipe: "Pipe", // | CallOperator: "CallOperator", // () - AdditiveBinaryOperator: "AdditiveBinaryOperator", // + - + AdditiveBinaryOperator: "AdditiveBinaryOperator", // + - ~ MultiplicativeBinaryOperator: "MultiplicativeBinaryOperator", // * / % ComparisonBinaryOperator: "ComparisonBinaryOperator", // < > <= >= == != UnaryOperator: "UnaryOperator", // ! - + @@ -85,6 +85,7 @@ const ORDERED_MAPPING_TABLE: [string, TokenType][] = [ // Arithmetic operators ["+", TOKEN_TYPES.AdditiveBinaryOperator], ["-", TOKEN_TYPES.AdditiveBinaryOperator], + ["~", TOKEN_TYPES.AdditiveBinaryOperator], ["*", TOKEN_TYPES.MultiplicativeBinaryOperator], ["/", TOKEN_TYPES.MultiplicativeBinaryOperator], ["%", TOKEN_TYPES.MultiplicativeBinaryOperator], @@ -136,12 +137,18 @@ function preprocess(template: string, options: PreprocessOptions = {}): string { template = template.replace(/([#%]})\n/g, "$1"); } - return template - .replace(/{##}/g, "") // Remove comments - .replace(/-%}\s*/g, "%}") - .replace(/\s*{%-/g, "{%") - .replace(/-}}\s*/g, "}}") - .replace(/\s*{{-/g, "{{"); + return ( + template + .replace(/{##}/g, "") // Remove comments + .replace(/-%}\s*/g, "%}") + .replace(/\s*{%-/g, "{%") + .replace(/-}}\s*/g, "}}") + .replace(/\s*{{-/g, "{{") + + // Handle the custom transformers-specific `generation` tag. + // See https://github.com/huggingface/transformers/pull/30650 for more information. + .replace(/{%\s*generation\s*%}.+?{%\s*endgeneration\s*%}/gs, "") + ); } /** @@ -152,6 +159,7 @@ export function tokenize(source: string, options: PreprocessOptions = {}): Token const src: string = preprocess(source, options); let cursorPosition = 0; + let curlyBracketDepth = 0; const consumeWhile = (predicate: (char: string) => boolean): string => { let str = ""; @@ -244,11 +252,24 @@ export function tokenize(source: string, options: PreprocessOptions = {}): Token } // Try to match one of the tokens in the mapping table - for (const [char, token] of ORDERED_MAPPING_TABLE) { - const slice = src.slice(cursorPosition, cursorPosition + char.length); - if (slice === char) { - tokens.push(new Token(char, token)); - cursorPosition += char.length; + for (const [seq, type] of ORDERED_MAPPING_TABLE) { + // inside an object literal, don't treat "}}" as expression-end + if (seq === "}}" && curlyBracketDepth > 0) { + continue; + } + const slice = src.slice(cursorPosition, cursorPosition + seq.length); + if (slice === seq) { + tokens.push(new Token(seq, type)); + + // possibly adjust the curly bracket depth + if (type === TOKEN_TYPES.OpenExpression) { + curlyBracketDepth = 0; + } else if (type === TOKEN_TYPES.OpenCurlyBracket) { + ++curlyBracketDepth; + } else if (type === TOKEN_TYPES.CloseCurlyBracket) { + --curlyBracketDepth; + } + cursorPosition += seq.length; continue main; } } From 7542b4f0dfc16cae1e0a32db6386558d506d76e8 Mon Sep 17 00:00:00 2001 From: Joshua Lochner <26504141+xenova@users.noreply.github.com> Date: Fri, 2 May 2025 19:04:44 -0400 Subject: [PATCH 13/35] Implement call & macro statements fully --- packages/jinja/src/ast.ts | 2 +- packages/jinja/src/format.ts | 3 +- packages/jinja/src/parser.ts | 23 +++++++++- packages/jinja/src/runtime.ts | 25 ++++++++++- packages/jinja/test/templates.test.js | 61 +++++++++++++++++++++++++++ 5 files changed, 109 insertions(+), 5 deletions(-) diff --git a/packages/jinja/src/ast.ts b/packages/jinja/src/ast.ts index ceca196774..4b32452c1a 100644 --- a/packages/jinja/src/ast.ts +++ b/packages/jinja/src/ast.ts @@ -296,7 +296,7 @@ export class CallStatement extends Statement { constructor( public call: CallExpression, - public params: Expression[], + public callerArgs: Expression[] | null, public body: Statement[] ) { super(); diff --git a/packages/jinja/src/format.ts b/packages/jinja/src/format.ts index 51e52c83db..7e255e0f54 100644 --- a/packages/jinja/src/format.ts +++ b/packages/jinja/src/format.ts @@ -173,7 +173,8 @@ function formatMacro(node: Macro, depth: number, indentStr: string): string { function formatCallStatement(node: CallStatement, depth: number, indentStr: string): string { const pad = indentStr.repeat(depth); - const params = node.params.length > 0 ? `(${node.params.map((p) => p.value).join(", ")})` : ""; + const params = + node.callerArgs && node.callerArgs.length > 0 ? `(${node.callerArgs.map(formatExpression).join(", ")})` : ""; const callExpr = formatExpression(node.call, -1); let out = pad + createStatement(`call${params}`, callExpr) + NEWLINE; out += formatStatements(node.body, depth + 1, indentStr) + NEWLINE; diff --git a/packages/jinja/src/parser.ts b/packages/jinja/src/parser.ts index 3d5ce0b0cc..44f2e47a03 100644 --- a/packages/jinja/src/parser.ts +++ b/packages/jinja/src/parser.ts @@ -133,7 +133,28 @@ export function parse(tokens: Token[]): Program { expect(TOKEN_TYPES.CloseStatement, "Expected %} token"); break; case "call": - throw new SyntaxError(`Call statements are not supported yet`); + ++current; // consume 'call' + let callerArgs: Statement[] | null = null; + if (is(TOKEN_TYPES.OpenParen)) { + // Optional caller arguments, e.g. {% call(user) dump_users(...) %} + callerArgs = parseArgs(); + } + const callee = parsePrimaryExpression(); + if (callee.type !== "Identifier") { + throw new SyntaxError(`Expected identifier following call statement`); + } + const callArgs = parseArgs(); + expect(TOKEN_TYPES.CloseStatement, "Expected closing statement token"); + const body: Statement[] = []; + while (!isStatement("endcall")) { + body.push(parseAny()); + } + expect(TOKEN_TYPES.OpenStatement, "Expected '{%'"); + expectIdentifier("endcall"); + expect(TOKEN_TYPES.CloseStatement, "Expected closing statement token"); + const callExpr = new CallExpression(callee, callArgs); + result = new CallStatement(callExpr, callerArgs, body); + break; case "break": ++current; expect(TOKEN_TYPES.CloseStatement, "Expected closing statement token"); diff --git a/packages/jinja/src/runtime.ts b/packages/jinja/src/runtime.ts index b7f5ada37e..5b0fa05a52 100644 --- a/packages/jinja/src/runtime.ts +++ b/packages/jinja/src/runtime.ts @@ -1176,8 +1176,29 @@ export class Interpreter { } private evaluateCallStatement(node: CallStatement, environment: Environment): AnyRuntimeValue { - // TODO implement this - throw new Error("Call statements are not yet implemented"); + const callerFn = new FunctionValue((callerArgs: AnyRuntimeValue[], callerEnv: Environment) => { + const callBlockEnv = new Environment(callerEnv); + if (node.callerArgs) { + for (let i = 0; i < node.callerArgs.length; ++i) { + const param = node.callerArgs[i]; + if (param.type !== "Identifier") { + throw new Error(`Caller parameter must be an identifier, got ${param.type}`); + } + callBlockEnv.setVariable((param as Identifier).value, callerArgs[i] ?? new UndefinedValue()); + } + } + return this.evaluateBlock(node.body, callBlockEnv); + }); + + const [macroArgs, macroKwargs] = this.evaluateArguments(node.call.args, environment); + macroArgs.push(new KeywordArgumentsValue(macroKwargs)); + const fn = this.evaluate(node.call.callee, environment); + if (fn.type !== "FunctionValue") { + throw new Error(`Cannot call something that is not a function: got ${fn.type}`); + } + const newEnv = new Environment(environment); + newEnv.setVariable("caller", callerFn); + return (fn as FunctionValue).value(macroArgs, newEnv); } private evaluateFilterStatement(node: FilterStatement, environment: Environment): AnyRuntimeValue { diff --git a/packages/jinja/test/templates.test.js b/packages/jinja/test/templates.test.js index 4742c5127b..2615047943 100644 --- a/packages/jinja/test/templates.test.js +++ b/packages/jinja/test/templates.test.js @@ -173,6 +173,7 @@ const TEST_STRINGS = { MACROS_1: `{% macro hello(name, suffix='.') %}{{ 'Hello ' + name + suffix }}{% endmacro %}|{{ hello('A') }}|{{ hello('B', '!') }}|{{ hello('C', suffix='?') }}|`, MACROS_2: `{% macro fn(x, y=2, z=3) %}{{ x + ',' + y + ',' + z }}{% endmacro %}|{{ fn(1) }}|{{ fn(1, 0) }}|{{ fn(1, 0, -1) }}|{{ fn(1, y=0, z=-1) }}|{{ fn(1, z=0) }}|`, MACROS_3: `{%- macro dummy(a, b='!') -%}{{ a }} {{ caller() }}{{ b }}{%- endmacro %}{%- call dummy('hello') -%}name{%- endcall -%}`, + MACROS_4: `{%- macro print_users(users) -%}{%- for user in users -%}{{ caller(user) }}{%- endfor -%}{%- endmacro -%}{% call(user) print_users(users) %} - {{ user.firstname }} {{ user.lastname }}\n{% endcall %}`, // Context-specific keywords CONTEXT_KEYWORDS: `{% if if in in %}a{% endif %}{% set if = "a" %}{% set in = "abc" %}{% if if in in %}b{% endif %}`, @@ -3291,6 +3292,59 @@ const TEST_PARSED = { { value: "endcall", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, ], + MACROS_4: [ + { value: "{%", type: "OpenStatement" }, + { value: "macro", type: "Identifier" }, + { value: "print_users", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: "users", type: "Identifier" }, + { value: ")", type: "CloseParen" }, + { value: "%}", type: "CloseStatement" }, + { value: "{%", type: "OpenStatement" }, + { value: "for", type: "Identifier" }, + { value: "user", type: "Identifier" }, + { value: "in", type: "Identifier" }, + { value: "users", type: "Identifier" }, + { value: "%}", type: "CloseStatement" }, + { value: "{{", type: "OpenExpression" }, + { value: "caller", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: "user", type: "Identifier" }, + { value: ")", type: "CloseParen" }, + { value: "}}", type: "CloseExpression" }, + { value: "{%", type: "OpenStatement" }, + { value: "endfor", type: "Identifier" }, + { value: "%}", type: "CloseStatement" }, + { value: "{%", type: "OpenStatement" }, + { value: "endmacro", type: "Identifier" }, + { value: "%}", type: "CloseStatement" }, + { value: "{%", type: "OpenStatement" }, + { value: "call", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: "user", type: "Identifier" }, + { value: ")", type: "CloseParen" }, + { value: "print_users", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: "users", type: "Identifier" }, + { value: ")", type: "CloseParen" }, + { value: "%}", type: "CloseStatement" }, + { value: " - ", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "user", type: "Identifier" }, + { value: ".", type: "Dot" }, + { value: "firstname", type: "Identifier" }, + { value: "}}", type: "CloseExpression" }, + { value: " ", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "user", type: "Identifier" }, + { value: ".", type: "Dot" }, + { value: "lastname", type: "Identifier" }, + { value: "}}", type: "CloseExpression" }, + { value: "\n", type: "Text" }, + { value: "{%", type: "OpenStatement" }, + { value: "endcall", type: "Identifier" }, + { value: "%}", type: "CloseStatement" }, + ], // Context-specific keywords CONTEXT_KEYWORDS: [ @@ -3634,6 +3688,12 @@ const TEST_CONTEXT = { MACROS_1: {}, MACROS_2: {}, MACROS_3: {}, + MACROS_4: { + users: [ + { firstname: "John", lastname: "Doe" }, + { firstname: "Jane", lastname: "Smith" }, + ], + }, // Context-specific keywords CONTEXT_KEYWORDS: {}, @@ -3808,6 +3868,7 @@ const EXPECTED_OUTPUTS = { MACROS_1: `|Hello A.|Hello B!|Hello C?|`, MACROS_2: `|1,2,3|1,0,3|1,0,-1|1,0,-1|1,2,0|`, MACROS_3: `hello name!`, + MACROS_4: " - John Doe\n - Jane Smith\n", // Context-specific keywords CONTEXT_KEYWORDS: `b`, From e28ab57b2d6040687cceaa8da2f831f68a17b77c Mon Sep 17 00:00:00 2001 From: Joshua Lochner <26504141+xenova@users.noreply.github.com> Date: Fri, 2 May 2025 20:40:41 -0400 Subject: [PATCH 14/35] Support consecutive string parsing --- packages/jinja/src/parser.ts | 6 +++++- packages/jinja/test/templates.test.js | 11 +++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/packages/jinja/src/parser.ts b/packages/jinja/src/parser.ts index 44f2e47a03..5d09c6783a 100644 --- a/packages/jinja/src/parser.ts +++ b/packages/jinja/src/parser.ts @@ -597,7 +597,11 @@ export function parse(tokens: Token[]): Program { case TOKEN_TYPES.NumericLiteral: return new NumericLiteral(Number(token.value)); case TOKEN_TYPES.StringLiteral: - return new StringLiteral(token.value); + let value = token.value; + while (is(TOKEN_TYPES.StringLiteral)) { + value += tokens[current++].value; + } + return new StringLiteral(value); case TOKEN_TYPES.Identifier: return new Identifier(token.value); case TOKEN_TYPES.OpenParen: { diff --git a/packages/jinja/test/templates.test.js b/packages/jinja/test/templates.test.js index 2615047943..efc4b87b30 100644 --- a/packages/jinja/test/templates.test.js +++ b/packages/jinja/test/templates.test.js @@ -54,6 +54,7 @@ const TEST_STRINGS = { STRINGS_1: `|{{ "test" }}|{{ "a" + 'b' + "c" }}|{{ '"' + "'" }}|{{ '\\'' }}|{{ "\\"" }}|`, STRINGS_2: `|{{ "" | length }}|{{ "a" | length }}|{{ '' | length }}|{{ 'a' | length }}|`, STRINGS_3: `|{{ '{{ "hi" }}' }}|{{ '{% if true %}{% endif %}' }}|`, + STRINGS_4: `{{ 'a' + 'b' 'c' }}`, // Function calls FUNCTIONS: `{{ func() }}{{ func(apple) }}{{ func(x, 'test', 2, false) }}`, @@ -984,6 +985,14 @@ const TEST_PARSED = { { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, ], + STRINGS_4: [ + { value: '{{', type: 'OpenExpression' }, + { value: 'a', type: 'StringLiteral' }, + { value: '+', type: 'AdditiveBinaryOperator' }, + { value: 'b', type: 'StringLiteral' }, + { value: 'c', type: 'StringLiteral' }, + { value: '}}', type: 'CloseExpression' } + ], // Function calls FUNCTIONS: [ @@ -3469,6 +3478,7 @@ const TEST_CONTEXT = { STRINGS_1: {}, STRINGS_2: {}, STRINGS_3: {}, + STRINGS_4: {}, // Function calls FUNCTIONS: { @@ -3749,6 +3759,7 @@ const EXPECTED_OUTPUTS = { STRINGS_1: `|test|abc|"'|'|"|`, STRINGS_2: `|0|1|0|1|`, STRINGS_3: `|{{ "hi" }}|{% if true %}{% endif %}|`, + STRINGS_4: `abc`, // Function calls FUNCTIONS: "014", From 288d6412f8e41c1689b0874a4c1c46c39297e5e4 Mon Sep 17 00:00:00 2001 From: Joshua Lochner <26504141+xenova@users.noreply.github.com> Date: Fri, 2 May 2025 21:05:03 -0400 Subject: [PATCH 15/35] Add support for iterable unpacking --- packages/jinja/src/ast.ts | 8 +++ packages/jinja/src/format.ts | 5 ++ packages/jinja/src/parser.ts | 29 +++++++---- packages/jinja/src/runtime.ts | 17 +++++-- packages/jinja/test/templates.test.js | 72 ++++++++++++++++++++++++--- 5 files changed, 112 insertions(+), 19 deletions(-) diff --git a/packages/jinja/src/ast.ts b/packages/jinja/src/ast.ts index 4b32452c1a..fd1b6a47db 100644 --- a/packages/jinja/src/ast.ts +++ b/packages/jinja/src/ast.ts @@ -291,6 +291,14 @@ export class KeywordArgumentExpression extends Expression { } } +export class SpreadExpression extends Expression { + override type = "SpreadExpression"; + + constructor(public argument: Expression) { + super(); + } +} + export class CallStatement extends Statement { override type = "CallStatement"; diff --git a/packages/jinja/src/format.ts b/packages/jinja/src/format.ts index 7e255e0f54..082e41d53f 100644 --- a/packages/jinja/src/format.ts +++ b/packages/jinja/src/format.ts @@ -24,6 +24,7 @@ import type { KeywordArgumentExpression, CallStatement, FilterStatement, + SpreadExpression, } from "./ast"; const NEWLINE = "\n"; @@ -196,6 +197,10 @@ function formatFilterStatement(node: FilterStatement, depth: number, indentStr: function formatExpression(node: Expression, parentPrec: number = -1): string { switch (node.type) { + case "SpreadExpression": { + const n = node as SpreadExpression; + return `*${formatExpression(n.argument, -1)}`; + } case "Identifier": return (node as Identifier).value; case "NumericLiteral": diff --git a/packages/jinja/src/parser.ts b/packages/jinja/src/parser.ts index 5d09c6783a..6e4924c9a5 100644 --- a/packages/jinja/src/parser.ts +++ b/packages/jinja/src/parser.ts @@ -26,6 +26,7 @@ import { SelectExpression, CallStatement, FilterStatement, + SpreadExpression, } from "./ast"; /** @@ -459,17 +460,25 @@ export function parse(tokens: Token[]): Program { const args = []; while (!is(TOKEN_TYPES.CloseParen)) { - let argument = parseExpression(); - - if (is(TOKEN_TYPES.Equals)) { - // keyword argument - // e.g., func(x = 5, y = a or b) - ++current; // consume equals - if (!(argument instanceof Identifier)) { - throw new SyntaxError(`Expected identifier for keyword argument`); + let argument: Statement; + + // unpacking: *expr + if (tokens[current].type === TOKEN_TYPES.MultiplicativeBinaryOperator && tokens[current].value === "*") { + ++current; + const expr = parseExpression(); + argument = new SpreadExpression(expr); + } else { + argument = parseExpression(); + if (is(TOKEN_TYPES.Equals)) { + // keyword argument + // e.g., func(x = 5, y = a or b) + ++current; // consume equals + if (!(argument instanceof Identifier)) { + throw new SyntaxError(`Expected identifier for keyword argument`); + } + const value = parseExpression(); + argument = new KeywordArgumentExpression(argument as Identifier, value); } - const value = parseExpression(); - argument = new KeywordArgumentExpression(argument, value); } args.push(argument); if (is(TOKEN_TYPES.Comma)) { diff --git a/packages/jinja/src/runtime.ts b/packages/jinja/src/runtime.ts index 5b0fa05a52..5d1208f704 100644 --- a/packages/jinja/src/runtime.ts +++ b/packages/jinja/src/runtime.ts @@ -23,6 +23,7 @@ import type { SelectExpression, CallStatement, FilterStatement, + SpreadExpression, } from "./ast"; import { range, slice, titleCase } from "./utils"; @@ -571,11 +572,21 @@ export class Interpreter { environment: Environment ): [AnyRuntimeValue[], Map] { // Accumulate args and kwargs - const positionalArguments = []; - const keywordArguments = new Map(); + const positionalArguments: AnyRuntimeValue[] = []; + const keywordArguments = new Map(); + for (const argument of args) { // TODO: Lazy evaluation of arguments - if (argument.type === "KeywordArgumentExpression") { + if (argument.type === "SpreadExpression") { + const spreadNode = argument as SpreadExpression; + const val = this.evaluate(spreadNode.argument, environment); + if (!(val instanceof ArrayValue)) { + throw new Error(`Cannot unpack non-iterable type: ${val.type}`); + } + for (const item of val.value) { + positionalArguments.push(item); + } + } else if (argument.type === "KeywordArgumentExpression") { const kwarg = argument as KeywordArgumentExpression; keywordArguments.set(kwarg.key.value, this.evaluate(kwarg.value, environment)); } else { diff --git a/packages/jinja/test/templates.test.js b/packages/jinja/test/templates.test.js index efc4b87b30..fbb9b907f3 100644 --- a/packages/jinja/test/templates.test.js +++ b/packages/jinja/test/templates.test.js @@ -179,6 +179,9 @@ const TEST_STRINGS = { // Context-specific keywords CONTEXT_KEYWORDS: `{% if if in in %}a{% endif %}{% set if = "a" %}{% set in = "abc" %}{% if if in in %}b{% endif %}`, CONTEXT_KEYWORDS_1: `|{{ if }}|{% set if = 2 %}{% if if == 2 %}{{ if }}{% endif %}|`, + + // Unpacking + UNPACKING: `{% macro mul(a, b, c) %}{{ a * b * c }}{% endmacro %}|{{ mul(1, 2, 3) }}|{{ mul(*[1, 2, 3]) }}|`, }; const TEST_PARSED = { @@ -986,12 +989,12 @@ const TEST_PARSED = { { value: "|", type: "Text" }, ], STRINGS_4: [ - { value: '{{', type: 'OpenExpression' }, - { value: 'a', type: 'StringLiteral' }, - { value: '+', type: 'AdditiveBinaryOperator' }, - { value: 'b', type: 'StringLiteral' }, - { value: 'c', type: 'StringLiteral' }, - { value: '}}', type: 'CloseExpression' } + { value: "{{", type: "OpenExpression" }, + { value: "a", type: "StringLiteral" }, + { value: "+", type: "AdditiveBinaryOperator" }, + { value: "b", type: "StringLiteral" }, + { value: "c", type: "StringLiteral" }, + { value: "}}", type: "CloseExpression" }, ], // Function calls @@ -3416,6 +3419,57 @@ const TEST_PARSED = { { value: "%}", type: "CloseStatement" }, { value: "|", type: "Text" }, ], + + // Unpacking + UNPACKING: [ + { value: "{%", type: "OpenStatement" }, + { value: "macro", type: "Identifier" }, + { value: "mul", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: "a", type: "Identifier" }, + { value: ",", type: "Comma" }, + { value: "b", type: "Identifier" }, + { value: ",", type: "Comma" }, + { value: "c", type: "Identifier" }, + { value: ")", type: "CloseParen" }, + { value: "%}", type: "CloseStatement" }, + { value: "{{", type: "OpenExpression" }, + { value: "a", type: "Identifier" }, + { value: "*", type: "MultiplicativeBinaryOperator" }, + { value: "b", type: "Identifier" }, + { value: "*", type: "MultiplicativeBinaryOperator" }, + { value: "c", type: "Identifier" }, + { value: "}}", type: "CloseExpression" }, + { value: "{%", type: "OpenStatement" }, + { value: "endmacro", type: "Identifier" }, + { value: "%}", type: "CloseStatement" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "mul", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: "1", type: "NumericLiteral" }, + { value: ",", type: "Comma" }, + { value: "2", type: "NumericLiteral" }, + { value: ",", type: "Comma" }, + { value: "3", type: "NumericLiteral" }, + { value: ")", type: "CloseParen" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "mul", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: "*", type: "MultiplicativeBinaryOperator" }, + { value: "[", type: "OpenSquareBracket" }, + { value: "1", type: "NumericLiteral" }, + { value: ",", type: "Comma" }, + { value: "2", type: "NumericLiteral" }, + { value: ",", type: "Comma" }, + { value: "3", type: "NumericLiteral" }, + { value: "]", type: "CloseSquareBracket" }, + { value: ")", type: "CloseParen" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + ], }; const TEST_CONTEXT = { @@ -3708,6 +3762,9 @@ const TEST_CONTEXT = { // Context-specific keywords CONTEXT_KEYWORDS: {}, CONTEXT_KEYWORDS_1: {}, + + // Unpacking + UNPACKING: {}, }; const EXPECTED_OUTPUTS = { @@ -3884,6 +3941,9 @@ const EXPECTED_OUTPUTS = { // Context-specific keywords CONTEXT_KEYWORDS: `b`, CONTEXT_KEYWORDS_1: `||2|`, + + // Unpacking + UNPACKING: `|6|6|`, }; describe("Templates", () => { From 8bbc2059f0e9733ecc99fe98acd17764cd0990ae Mon Sep 17 00:00:00 2001 From: Joshua Lochner <26504141+xenova@users.noreply.github.com> Date: Fri, 2 May 2025 22:10:47 -0400 Subject: [PATCH 16/35] Add support for variable unpacking in set --- packages/jinja/src/parser.ts | 4 +-- packages/jinja/src/runtime.ts | 16 ++++++++++ packages/jinja/test/templates.test.js | 44 +++++++++++++++++++++++++++ 3 files changed, 62 insertions(+), 2 deletions(-) diff --git a/packages/jinja/src/parser.ts b/packages/jinja/src/parser.ts index 6e4924c9a5..fd250a8739 100644 --- a/packages/jinja/src/parser.ts +++ b/packages/jinja/src/parser.ts @@ -201,12 +201,12 @@ export function parse(tokens: Token[]): Program { // NOTE: `set` acts as both declaration statement and assignment expression function parseSetStatement(): Statement { - const left = parseExpression(); + const left = parseExpressionSequence(); let value: Statement | null = null; const body: Statement[] = []; if (is(TOKEN_TYPES.Equals)) { ++current; - value = parseExpression(); + value = parseExpressionSequence(); } else { // parsing multiline set here expect(TOKEN_TYPES.CloseStatement, "Expected %} token"); diff --git a/packages/jinja/src/runtime.ts b/packages/jinja/src/runtime.ts index 5d1208f704..1e379cdf4e 100644 --- a/packages/jinja/src/runtime.ts +++ b/packages/jinja/src/runtime.ts @@ -1003,6 +1003,22 @@ export class Interpreter { if (node.assignee.type === "Identifier") { const variableName = (node.assignee as Identifier).value; environment.setVariable(variableName, rhs); + } else if (node.assignee.type === "TupleLiteral") { + const tuple = node.assignee as TupleLiteral; + if (!(rhs instanceof ArrayValue)) { + throw new Error(`Cannot unpack non-iterable type in set: ${rhs.type}`); + } + const arr = rhs.value; + if (arr.length !== tuple.value.length) { + throw new Error(`Too ${tuple.value.length > arr.length ? "few" : "many"} items to unpack in set`); + } + for (let i = 0; i < tuple.value.length; i++) { + const elem = tuple.value[i]; + if (elem.type !== "Identifier") { + throw new Error(`Cannot unpack to non-identifier in set: ${elem.type}`); + } + environment.setVariable((elem as Identifier).value, arr[i]); + } } else if (node.assignee.type === "MemberExpression") { const member = node.assignee as MemberExpression; diff --git a/packages/jinja/test/templates.test.js b/packages/jinja/test/templates.test.js index fbb9b907f3..532c15d832 100644 --- a/packages/jinja/test/templates.test.js +++ b/packages/jinja/test/templates.test.js @@ -41,6 +41,7 @@ const TEST_STRINGS = { VARIABLES: `{% set x = 'Hello' %}{% set y = 'World' %}{{ x + ' ' + y }}`, VARIABLES_2: `{% set x = 'Hello'.split('el')[-1] %}{{ x }}`, VARIABLES_BLOCK: `{% set x %}Hello!\nMultiline/block set!\n{% endset %}{{ x }}`, + VARIABLES_UNPACKING: `|{% set x, y = 1, 2 %}{{ x }}{{ y }}|{% set (x, y) = [1, 2] %}{{ x }}{{ y }}|`, // Numbers NUMBERS: `|{{ 5 }}|{{ -5 }}|{{ add(3, -1) }}|{{ (3 - 1) + (a - 5) - (a + 5)}}|`, @@ -806,6 +807,47 @@ const TEST_PARSED = { { value: "x", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, ], + VARIABLES_UNPACKING: [ + { value: "|", type: "Text" }, + { value: "{%", type: "OpenStatement" }, + { value: "set", type: "Identifier" }, + { value: "x", type: "Identifier" }, + { value: ",", type: "Comma" }, + { value: "y", type: "Identifier" }, + { value: "=", type: "Equals" }, + { value: "1", type: "NumericLiteral" }, + { value: ",", type: "Comma" }, + { value: "2", type: "NumericLiteral" }, + { value: "%}", type: "CloseStatement" }, + { value: "{{", type: "OpenExpression" }, + { value: "x", type: "Identifier" }, + { value: "}}", type: "CloseExpression" }, + { value: "{{", type: "OpenExpression" }, + { value: "y", type: "Identifier" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{%", type: "OpenStatement" }, + { value: "set", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: "x", type: "Identifier" }, + { value: ",", type: "Comma" }, + { value: "y", type: "Identifier" }, + { value: ")", type: "CloseParen" }, + { value: "=", type: "Equals" }, + { value: "[", type: "OpenSquareBracket" }, + { value: "1", type: "NumericLiteral" }, + { value: ",", type: "Comma" }, + { value: "2", type: "NumericLiteral" }, + { value: "]", type: "CloseSquareBracket" }, + { value: "%}", type: "CloseStatement" }, + { value: "{{", type: "OpenExpression" }, + { value: "x", type: "Identifier" }, + { value: "}}", type: "CloseExpression" }, + { value: "{{", type: "OpenExpression" }, + { value: "y", type: "Identifier" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + ], // Numbers NUMBERS: [ @@ -3514,6 +3556,7 @@ const TEST_CONTEXT = { VARIABLES: {}, VARIABLES_2: {}, VARIABLES_BLOCK: {}, + VARIABLES_UNPACKING: {}, // Numbers NUMBERS: { @@ -3803,6 +3846,7 @@ const EXPECTED_OUTPUTS = { VARIABLES: "Hello World", VARIABLES_2: "lo", VARIABLES_BLOCK: "Hello!\nMultiline/block set!\n", + VARIABLES_UNPACKING: "|12|12|", // Numbers NUMBERS: "|5|-5|2|-8|", From cadeed197bb31612d494415b7ac93eda70778956 Mon Sep 17 00:00:00 2001 From: Joshua Lochner <26504141+xenova@users.noreply.github.com> Date: Fri, 2 May 2025 23:36:32 -0400 Subject: [PATCH 17/35] Lint & formatting --- packages/jinja/src/parser.ts | 11 +++++++---- packages/jinja/src/runtime.ts | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/packages/jinja/src/parser.ts b/packages/jinja/src/parser.ts index fd250a8739..f8da0466b8 100644 --- a/packages/jinja/src/parser.ts +++ b/packages/jinja/src/parser.ts @@ -133,7 +133,7 @@ export function parse(tokens: Token[]): Program { expectIdentifier("endfor"); expect(TOKEN_TYPES.CloseStatement, "Expected %} token"); break; - case "call": + case "call": { ++current; // consume 'call' let callerArgs: Statement[] | null = null; if (is(TOKEN_TYPES.OpenParen)) { @@ -156,6 +156,7 @@ export function parse(tokens: Token[]): Program { const callExpr = new CallExpression(callee, callArgs); result = new CallStatement(callExpr, callerArgs, body); break; + } case "break": ++current; expect(TOKEN_TYPES.CloseStatement, "Expected closing statement token"); @@ -166,7 +167,7 @@ export function parse(tokens: Token[]): Program { expect(TOKEN_TYPES.CloseStatement, "Expected closing statement token"); result = new Continue(); break; - case "filter": + case "filter": { ++current; // consume 'filter' let filterNode = parsePrimaryExpression(); if (filterNode instanceof Identifier && is(TOKEN_TYPES.OpenParen)) { @@ -182,6 +183,7 @@ export function parse(tokens: Token[]): Program { expect(TOKEN_TYPES.CloseStatement, "Expected '%}'"); result = new FilterStatement(filterNode as Identifier | CallExpression, filterBody); break; + } default: throw new SyntaxError(`Unknown statement type: ${name}`); } @@ -571,7 +573,7 @@ export function parse(tokens: Token[]): Program { ++current; // consume not } - let filter = parsePrimaryExpression(); + const filter = parsePrimaryExpression(); if (!(filter instanceof Identifier)) { throw new SyntaxError(`Expected identifier for the test`); } @@ -605,12 +607,13 @@ export function parse(tokens: Token[]): Program { switch (token.type) { case TOKEN_TYPES.NumericLiteral: return new NumericLiteral(Number(token.value)); - case TOKEN_TYPES.StringLiteral: + case TOKEN_TYPES.StringLiteral: { let value = token.value; while (is(TOKEN_TYPES.StringLiteral)) { value += tokens[current++].value; } return new StringLiteral(value); + } case TOKEN_TYPES.Identifier: return new Identifier(token.value); case TOKEN_TYPES.OpenParen: { diff --git a/packages/jinja/src/runtime.ts b/packages/jinja/src/runtime.ts index 1e379cdf4e..0b9b0f4cf0 100644 --- a/packages/jinja/src/runtime.ts +++ b/packages/jinja/src/runtime.ts @@ -429,7 +429,7 @@ export class Environment { } } -export function setupGlobals(env: Environment) { +export function setupGlobals(env: Environment): void { // Declare global variables env.set("false", false); env.set("true", true); From 5cb4ae9552879dde9e6eed05794b913889fb1f90 Mon Sep 17 00:00:00 2001 From: Joshua Lochner <26504141+xenova@users.noreply.github.com> Date: Sat, 3 May 2025 19:56:39 -0400 Subject: [PATCH 18/35] Support fractional numeric literals --- packages/jinja/src/lexer.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/packages/jinja/src/lexer.ts b/packages/jinja/src/lexer.ts index bc62c4767b..16d8dea367 100644 --- a/packages/jinja/src/lexer.ts +++ b/packages/jinja/src/lexer.ts @@ -4,7 +4,7 @@ export const TOKEN_TYPES = Object.freeze({ Text: "Text", // The text between Jinja statements or expressions - NumericLiteral: "NumericLiteral", // e.g., 123 + NumericLiteral: "NumericLiteral", // e.g., 123, 1.0 StringLiteral: "StringLiteral", // 'string' Identifier: "Identifier", // Variables, functions, statements, booleans, etc. Equals: "Equals", // = @@ -283,7 +283,14 @@ export function tokenize(source: string, options: PreprocessOptions = {}): Token } if (isInteger(char)) { - const num = consumeWhile(isInteger); + // Consume integer part + let num = consumeWhile(isInteger); + // Possibly, consume fractional part + if (src[cursorPosition] === "." && isInteger(src[cursorPosition + 1])) { + ++cursorPosition; // consume '.' + const frac = consumeWhile(isInteger); + num = `${num}.${frac}`; + } tokens.push(new Token(num, TOKEN_TYPES.NumericLiteral)); continue; } From a5870be0c0898e3c98ba20c7aceae6ed51743099 Mon Sep 17 00:00:00 2001 From: Joshua Lochner <26504141+xenova@users.noreply.github.com> Date: Sat, 3 May 2025 20:07:51 -0400 Subject: [PATCH 19/35] Add support for int and float filters --- packages/jinja/src/runtime.ts | 42 +++++++++++++++++ packages/jinja/test/templates.test.js | 68 +++++++++++++++++++++++++++ 2 files changed, 110 insertions(+) diff --git a/packages/jinja/src/runtime.ts b/packages/jinja/src/runtime.ts index 0b9b0f4cf0..ec52b9b083 100644 --- a/packages/jinja/src/runtime.ts +++ b/packages/jinja/src/runtime.ts @@ -681,6 +681,14 @@ export class Interpreter { case "join": case "string": return operand; // no-op + case "int": { + const val = parseInt(operand.value, 10); + return new NumericValue(isNaN(val) ? 0 : val); + } + case "float": { + const val = parseFloat(operand.value); + return new NumericValue(isNaN(val) ? 0.0 : val); + } default: throw new Error(`Unknown StringValue filter: ${filter.value}`); } @@ -688,6 +696,10 @@ export class Interpreter { switch (filter.value) { case "abs": return new NumericValue(Math.abs(operand.value)); + case "int": + return new NumericValue(Math.floor(operand.value)); + case "float": + return new NumericValue(operand.value); default: throw new Error(`Unknown NumericValue filter: ${filter.value}`); } @@ -702,6 +714,19 @@ export class Interpreter { default: throw new Error(`Unknown ObjectValue filter: ${filter.value}`); } + } else if (operand instanceof BooleanValue) { + switch (filter.value) { + case "bool": + return new BooleanValue(operand.value); + case "int": + return new NumericValue(operand.value ? 1 : 0); + case "float": + return new NumericValue(operand.value ? 1.0 : 0.0); + case "string": + return new StringValue(operand.value ? "true" : "false"); + default: + throw new Error(`Unknown BooleanValue filter: ${filter.value}`); + } } throw new Error(`Cannot apply filter "${filter.value}" to type: ${operand.type}`); } else if (filterNode.type === "CallExpression") { @@ -737,6 +762,23 @@ export class Interpreter { } return new StringValue(value.join(separator.value)); + } else if (filterName === "int" || filterName === "float") { + const [args, kwargs] = this.evaluateArguments(filter.args, environment); + const defaultValue = args.at(0) ?? kwargs.get("default") ?? new NumericValue(0); + + if (!(defaultValue instanceof NumericValue)) { + throw new Error("default must be a number"); + } + if (operand instanceof StringValue) { + const val = filterName === "int" ? parseInt(operand.value, 10) : parseFloat(operand.value); + return new NumericValue(isNaN(val) ? defaultValue.value : val); + } else if (operand instanceof NumericValue) { + return new NumericValue(operand.value); + } else if (operand instanceof BooleanValue) { + return new NumericValue(operand.value ? 1 : 0); + } else { + throw new Error(`Cannot apply filter "${filterName}" to type: ${operand.type}`); + } } if (operand instanceof ArrayValue) { diff --git a/packages/jinja/test/templates.test.js b/packages/jinja/test/templates.test.js index 532c15d832..a4877d297a 100644 --- a/packages/jinja/test/templates.test.js +++ b/packages/jinja/test/templates.test.js @@ -107,6 +107,7 @@ const TEST_STRINGS = { FILTER_OPERATOR_11: `{{ items | rejectattr('key') | length }}`, FILTER_OPERATOR_12: `{{ messages | rejectattr('role', 'equalto', 'system') | length }}`, FILTER_OPERATOR_13: `{{ tools | string }}`, + FILTER_OPERATOR_14: `|{{ "1" | int + 2 }}|{{ "invalid" | int }}|{{ "invalid" | int(-1) }}|{{ true | int }}|{{ false | int }}|{{ 1.5 | int }}|{{ "1.5" | float }}|{{ "invalid" | float }}|{{ "invalid" | float(2) }}|`, // Filter statements FILTER_STATEMENTS: `{% filter upper %}text{% endfilter %}`, @@ -2104,6 +2105,71 @@ const TEST_PARSED = { { value: "string", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, ], + FILTER_OPERATOR_14: [ + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "1", type: "StringLiteral" }, + { value: "|", type: "Pipe" }, + { value: "int", type: "Identifier" }, + { value: "+", type: "AdditiveBinaryOperator" }, + { value: "2", type: "NumericLiteral" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "invalid", type: "StringLiteral" }, + { value: "|", type: "Pipe" }, + { value: "int", type: "Identifier" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "invalid", type: "StringLiteral" }, + { value: "|", type: "Pipe" }, + { value: "int", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: "-1", type: "NumericLiteral" }, + { value: ")", type: "CloseParen" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "true", type: "Identifier" }, + { value: "|", type: "Pipe" }, + { value: "int", type: "Identifier" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "false", type: "Identifier" }, + { value: "|", type: "Pipe" }, + { value: "int", type: "Identifier" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "1.5", type: "NumericLiteral" }, + { value: "|", type: "Pipe" }, + { value: "int", type: "Identifier" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "1.5", type: "StringLiteral" }, + { value: "|", type: "Pipe" }, + { value: "float", type: "Identifier" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "invalid", type: "StringLiteral" }, + { value: "|", type: "Pipe" }, + { value: "float", type: "Identifier" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "invalid", type: "StringLiteral" }, + { value: "|", type: "Pipe" }, + { value: "float", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: "2", type: "NumericLiteral" }, + { value: ")", type: "CloseParen" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + ], // Filter statements FILTER_STATEMENTS: [ @@ -3706,6 +3772,7 @@ const TEST_CONTEXT = { FILTER_OPERATOR_13: { tools: [{ name: "some_tool", arguments: { some_name: "string" } }], }, + FILTER_OPERATOR_14: {}, // Filter statements FILTER_STATEMENTS: {}, @@ -3912,6 +3979,7 @@ const EXPECTED_OUTPUTS = { FILTER_OPERATOR_11: `3`, FILTER_OPERATOR_12: `2`, FILTER_OPERATOR_13: `[{"name": "some_tool", "arguments": {"some_name": "string"}}]`, + FILTER_OPERATOR_14: `|3|0|-1|1|0|1|1.5|0.0|2|`, // Filter statements FILTER_STATEMENTS: `TEXT`, From bacfbe9e09c82c78bb95afd19fa7bda55ba24a05 Mon Sep 17 00:00:00 2001 From: Joshua Lochner <26504141+xenova@users.noreply.github.com> Date: Sat, 3 May 2025 20:48:20 -0400 Subject: [PATCH 20/35] Differentiate between integer and float types Python vs. JS --- packages/jinja/src/ast.ts | 11 +- packages/jinja/src/format.ts | 9 +- packages/jinja/src/parser.ts | 9 +- packages/jinja/src/runtime.ts | 169 +++++++++++++++----------- packages/jinja/test/templates.test.js | 6 +- 5 files changed, 121 insertions(+), 83 deletions(-) diff --git a/packages/jinja/src/ast.ts b/packages/jinja/src/ast.ts index fd1b6a47db..19fea17884 100644 --- a/packages/jinja/src/ast.ts +++ b/packages/jinja/src/ast.ts @@ -133,11 +133,12 @@ abstract class Literal extends Expression { } } -/** - * Represents a numeric constant in the template. - */ -export class NumericLiteral extends Literal { - override type = "NumericLiteral"; +export class IntegerLiteral extends Literal { + override type = "IntegerLiteral"; +} + +export class FloatLiteral extends Literal { + override type = "FloatLiteral"; } /** diff --git a/packages/jinja/src/format.ts b/packages/jinja/src/format.ts index 082e41d53f..35cb8c24d4 100644 --- a/packages/jinja/src/format.ts +++ b/packages/jinja/src/format.ts @@ -9,7 +9,8 @@ import type { MemberExpression, CallExpression, Identifier, - NumericLiteral, + FloatLiteral, + IntegerLiteral, StringLiteral, ArrayLiteral, TupleLiteral, @@ -203,8 +204,10 @@ function formatExpression(node: Expression, parentPrec: number = -1): string { } case "Identifier": return (node as Identifier).value; - case "NumericLiteral": - return `${(node as NumericLiteral).value}`; + case "IntegerLiteral": + return `${(node as IntegerLiteral).value}`; + case "FloatLiteral": + return `${(node as FloatLiteral).value}`; case "StringLiteral": return JSON.stringify((node as StringLiteral).value); case "BinaryExpression": { diff --git a/packages/jinja/src/parser.ts b/packages/jinja/src/parser.ts index f8da0466b8..950399fad3 100644 --- a/packages/jinja/src/parser.ts +++ b/packages/jinja/src/parser.ts @@ -11,7 +11,6 @@ import { MemberExpression, CallExpression, Identifier, - NumericLiteral, StringLiteral, ArrayLiteral, ObjectLiteral, @@ -27,6 +26,8 @@ import { CallStatement, FilterStatement, SpreadExpression, + IntegerLiteral, + FloatLiteral, } from "./ast"; /** @@ -605,8 +606,10 @@ export function parse(tokens: Token[]): Program { // Primary expression: number, string, identifier, function call, parenthesized expression const token = tokens[current++]; switch (token.type) { - case TOKEN_TYPES.NumericLiteral: - return new NumericLiteral(Number(token.value)); + case TOKEN_TYPES.NumericLiteral: { + const num = token.value; + return num.includes(".") ? new FloatLiteral(Number(num)) : new IntegerLiteral(Number(num)); + } case TOKEN_TYPES.StringLiteral: { let value = token.value; while (is(TOKEN_TYPES.StringLiteral)) { diff --git a/packages/jinja/src/runtime.ts b/packages/jinja/src/runtime.ts index ec52b9b083..a5bf64334b 100644 --- a/packages/jinja/src/runtime.ts +++ b/packages/jinja/src/runtime.ts @@ -1,6 +1,7 @@ import type { - NumericLiteral, StringLiteral, + FloatLiteral, + IntegerLiteral, ArrayLiteral, Statement, Program, @@ -28,7 +29,8 @@ import type { import { range, slice, titleCase } from "./utils"; export type AnyRuntimeValue = - | NumericValue + | IntegerValue + | FloatValue | StringValue | BooleanValue | ObjectValue @@ -69,13 +71,28 @@ abstract class RuntimeValue { __bool__(): BooleanValue { return new BooleanValue(!!this.value); } + + toString(): string { + return String(this.value); + } +} + +/** + * Represents an integer value at runtime. + */ +export class IntegerValue extends RuntimeValue { + override type = "IntegerValue"; } /** - * Represents a numeric value at runtime. + * Represents a float value at runtime. */ -export class NumericValue extends RuntimeValue { - override type = "NumericValue"; +export class FloatValue extends RuntimeValue { + override type = "FloatValue"; + + override toString(): string { + return this.value % 1 === 0 ? this.value.toFixed(1) : this.value.toString(); + } } /** @@ -109,7 +126,7 @@ export class StringValue extends RuntimeValue { return new StringValue(titleCase(this.value)); }), ], - ["length", new NumericValue(this.value.length)], + ["length", new IntegerValue(this.value.length)], [ "rstrip", new FunctionValue(() => { @@ -157,8 +174,8 @@ export class StringValue extends RuntimeValue { if (!(sep instanceof StringValue || sep instanceof NullValue)) { throw new Error("sep argument must be a string or null"); } - const maxsplit = args[1] ?? new NumericValue(-1); - if (!(maxsplit instanceof NumericValue)) { + const maxsplit = args[1] ?? new IntegerValue(-1); + if (!(maxsplit instanceof IntegerValue)) { throw new Error("maxsplit argument must be a number"); } @@ -251,7 +268,7 @@ export class KeywordArgumentsValue extends ObjectValue { */ export class ArrayValue extends RuntimeValue { override type = "ArrayValue"; - override builtins = new Map([["length", new NumericValue(this.value.length)]]); + override builtins = new Map([["length", new IntegerValue(this.value.length)]]); /** * NOTE: necessary to override since all JavaScript arrays are considered truthy, @@ -326,27 +343,27 @@ export class Environment { [ "odd", (operand) => { - if (operand.type !== "NumericValue") { - throw new Error(`Cannot apply test "odd" to type: ${operand.type}`); + if (!(operand instanceof IntegerValue)) { + throw new Error(`cannot odd on ${operand.type}`); } - return (operand as NumericValue).value % 2 !== 0; + return operand.value % 2 !== 0; }, ], [ "even", (operand) => { - if (operand.type !== "NumericValue") { - throw new Error(`Cannot apply test "even" to type: ${operand.type}`); + if (!(operand instanceof IntegerValue)) { + throw new Error(`cannot even on ${operand.type}`); } - return (operand as NumericValue).value % 2 === 0; + return operand.value % 2 === 0; }, ], ["false", (operand) => operand.type === "BooleanValue" && !(operand as BooleanValue).value], ["true", (operand) => operand.type === "BooleanValue" && (operand as BooleanValue).value], ["none", (operand) => operand.type === "NullValue"], ["string", (operand) => operand.type === "StringValue"], - ["number", (operand) => operand.type === "NumericValue"], - ["integer", (operand) => operand.type === "NumericValue" && Number.isInteger((operand as NumericValue).value)], + ["number", (operand) => operand instanceof IntegerValue || operand instanceof FloatValue], + ["integer", (operand) => operand instanceof IntegerValue], ["iterable", (operand) => operand.type === "ArrayValue" || operand.type === "StringValue"], ["mapping", (operand) => operand.type === "ObjectValue"], [ @@ -497,30 +514,38 @@ export class Interpreter { } else if (node.operator.value === "~") { // toString and concatenation return new StringValue(left.value.toString() + right.value.toString()); - } else if (left instanceof NumericValue && right instanceof NumericValue) { + } else if ( + (left instanceof IntegerValue || left instanceof FloatValue) && + (right instanceof IntegerValue || right instanceof FloatValue) + ) { // Evaulate pure numeric operations with binary operators. + const a = left.value, + b = right.value; switch (node.operator.value) { // Arithmetic operators case "+": - return new NumericValue(left.value + right.value); case "-": - return new NumericValue(left.value - right.value); - case "*": - return new NumericValue(left.value * right.value); + case "*": { + const res = node.operator.value === "+" ? a + b : node.operator.value === "-" ? a - b : a * b; + const isFloat = left instanceof FloatValue || right instanceof FloatValue; + return isFloat ? new FloatValue(res) : new IntegerValue(res); + } case "/": - return new NumericValue(left.value / right.value); - case "%": - return new NumericValue(left.value % right.value); - + return new FloatValue(a / b); + case "%": { + const rem = a % b; + const isFloat = left instanceof FloatValue || right instanceof FloatValue; + return isFloat ? new FloatValue(rem) : new IntegerValue(rem); + } // Comparison operators case "<": - return new BooleanValue(left.value < right.value); + return new BooleanValue(a < b); case ">": - return new BooleanValue(left.value > right.value); + return new BooleanValue(a > b); case ">=": - return new BooleanValue(left.value >= right.value); + return new BooleanValue(a >= b); case "<=": - return new BooleanValue(left.value <= right.value); + return new BooleanValue(a <= b); } } else if (left instanceof ArrayValue && right instanceof ArrayValue) { // Evaluate array operands with binary operator. @@ -628,7 +653,7 @@ export class Interpreter { case "last": return operand.value[operand.value.length - 1]; case "length": - return new NumericValue(operand.value.length); + return new IntegerValue(operand.value.length); case "reverse": return new ArrayValue(operand.value.reverse()); case "sort": @@ -638,8 +663,9 @@ export class Interpreter { throw new Error(`Cannot compare different types: ${a.type} and ${b.type}`); } switch (a.type) { - case "NumericValue": - return (a as NumericValue).value - (b as NumericValue).value; + case "IntegerValue": + case "FloatValue": + return (a as IntegerValue | FloatValue).value - (b as IntegerValue | FloatValue).value; case "StringValue": return (a as StringValue).value.localeCompare((b as StringValue).value); default: @@ -657,7 +683,7 @@ export class Interpreter { } else if (operand instanceof StringValue) { switch (filter.value) { case "length": - return new NumericValue(operand.value.length); + return new IntegerValue(operand.value.length); case "upper": return new StringValue(operand.value.toUpperCase()); case "lower": @@ -683,23 +709,25 @@ export class Interpreter { return operand; // no-op case "int": { const val = parseInt(operand.value, 10); - return new NumericValue(isNaN(val) ? 0 : val); + return new IntegerValue(isNaN(val) ? 0 : val); } case "float": { const val = parseFloat(operand.value); - return new NumericValue(isNaN(val) ? 0.0 : val); + return new FloatValue(isNaN(val) ? 0.0 : val); } default: throw new Error(`Unknown StringValue filter: ${filter.value}`); } - } else if (operand instanceof NumericValue) { + } else if (operand instanceof IntegerValue || operand instanceof FloatValue) { switch (filter.value) { case "abs": - return new NumericValue(Math.abs(operand.value)); + return operand instanceof IntegerValue + ? new IntegerValue(Math.abs(operand.value)) + : new FloatValue(Math.abs(operand.value)); case "int": - return new NumericValue(Math.floor(operand.value)); + return new IntegerValue(Math.floor(operand.value)); case "float": - return new NumericValue(operand.value); + return new FloatValue(operand.value); default: throw new Error(`Unknown NumericValue filter: ${filter.value}`); } @@ -710,7 +738,7 @@ export class Interpreter { Array.from(operand.value.entries()).map(([key, value]) => new ArrayValue([new StringValue(key), value])) ); case "length": - return new NumericValue(operand.value.size); + return new IntegerValue(operand.value.size); default: throw new Error(`Unknown ObjectValue filter: ${filter.value}`); } @@ -719,9 +747,9 @@ export class Interpreter { case "bool": return new BooleanValue(operand.value); case "int": - return new NumericValue(operand.value ? 1 : 0); + return new IntegerValue(operand.value ? 1 : 0); case "float": - return new NumericValue(operand.value ? 1.0 : 0.0); + return new FloatValue(operand.value ? 1.0 : 0.0); case "string": return new StringValue(operand.value ? "true" : "false"); default: @@ -740,7 +768,7 @@ export class Interpreter { if (filterName === "tojson") { const [, kwargs] = this.evaluateArguments(filter.args, environment); const indent = kwargs.get("indent") ?? new NullValue(); - if (!(indent instanceof NumericValue || indent instanceof NullValue)) { + if (!(indent instanceof IntegerValue || indent instanceof NullValue)) { throw new Error("If set, indent must be a number"); } return new StringValue(toJSON(operand, indent.value)); @@ -764,18 +792,18 @@ export class Interpreter { return new StringValue(value.join(separator.value)); } else if (filterName === "int" || filterName === "float") { const [args, kwargs] = this.evaluateArguments(filter.args, environment); - const defaultValue = args.at(0) ?? kwargs.get("default") ?? new NumericValue(0); + const defaultValue = + args.at(0) ?? kwargs.get("default") ?? (filterName === "int" ? new IntegerValue(0) : new FloatValue(0.0)); - if (!(defaultValue instanceof NumericValue)) { - throw new Error("default must be a number"); - } if (operand instanceof StringValue) { const val = filterName === "int" ? parseInt(operand.value, 10) : parseFloat(operand.value); - return new NumericValue(isNaN(val) ? defaultValue.value : val); - } else if (operand instanceof NumericValue) { - return new NumericValue(operand.value); + return isNaN(val) ? defaultValue : filterName === "int" ? new IntegerValue(val) : new FloatValue(val); + } else if (operand instanceof IntegerValue || operand instanceof FloatValue) { + return operand; } else if (operand instanceof BooleanValue) { - return new NumericValue(operand.value ? 1 : 0); + return filterName === "int" + ? new IntegerValue(operand.value ? 1 : 0) + : new FloatValue(operand.value ? 1.0 : 0.0); } else { throw new Error(`Cannot apply filter "${filterName}" to type: ${operand.type}`); } @@ -854,8 +882,8 @@ export class Interpreter { const [args, kwargs] = this.evaluateArguments(filter.args, environment); - const width = args.at(0) ?? kwargs.get("width") ?? new NumericValue(4); - if (!(width instanceof NumericValue)) { + const width = args.at(0) ?? kwargs.get("width") ?? new IntegerValue(4); + if (!(width instanceof IntegerValue)) { throw new Error("width must be a number"); } const first = args.at(1) ?? kwargs.get("first") ?? new BooleanValue(false); @@ -938,10 +966,10 @@ export class Interpreter { let result = ""; for (const statement of statements) { - const lastEvaluated = this.evaluate(statement, environment); + const lastEvaluated: AnyRuntimeValue = this.evaluate(statement, environment); if (lastEvaluated.type !== "NullValue" && lastEvaluated.type !== "UndefinedValue") { - result += lastEvaluated.value; + result += lastEvaluated.toString(); } } @@ -982,13 +1010,13 @@ export class Interpreter { const step = this.evaluate(expr.step, environment); // Validate arguments - if (!(start instanceof NumericValue || start instanceof UndefinedValue)) { + if (!(start instanceof IntegerValue || start instanceof UndefinedValue)) { throw new Error("Slice start must be numeric or undefined"); } - if (!(stop instanceof NumericValue || stop instanceof UndefinedValue)) { + if (!(stop instanceof IntegerValue || stop instanceof UndefinedValue)) { throw new Error("Slice stop must be numeric or undefined"); } - if (!(step instanceof NumericValue || step instanceof UndefinedValue)) { + if (!(step instanceof IntegerValue || step instanceof UndefinedValue)) { throw new Error("Slice step must be numeric or undefined"); } @@ -1020,7 +1048,7 @@ export class Interpreter { } value = object.value.get(property.value) ?? object.builtins.get(property.value); } else if (object instanceof ArrayValue || object instanceof StringValue) { - if (property instanceof NumericValue) { + if (property instanceof IntegerValue) { value = object.value.at(property.value); if (object instanceof StringValue) { value = new StringValue(object.value.at(property.value)); @@ -1155,13 +1183,13 @@ export class Interpreter { // Update the loop variable // TODO: Only create object once, then update value? const loop = new Map([ - ["index", new NumericValue(i + 1)], - ["index0", new NumericValue(i)], - ["revindex", new NumericValue(items.length - i)], - ["revindex0", new NumericValue(items.length - i - 1)], + ["index", new IntegerValue(i + 1)], + ["index0", new IntegerValue(i)], + ["revindex", new IntegerValue(items.length - i)], + ["revindex0", new IntegerValue(items.length - i - 1)], ["first", new BooleanValue(i === 0)], ["last", new BooleanValue(i === items.length - 1)], - ["length", new NumericValue(items.length)], + ["length", new IntegerValue(items.length)], ["previtem", i > 0 ? items[i - 1] : new UndefinedValue()], ["nextitem", i < items.length - 1 ? items[i + 1] : new UndefinedValue()], ] as [string, AnyRuntimeValue][]); @@ -1301,8 +1329,10 @@ export class Interpreter { throw new ContinueControl(); // Expressions - case "NumericLiteral": - return new NumericValue(Number((statement as NumericLiteral).value)); + case "IntegerLiteral": + return new IntegerValue((statement as IntegerLiteral).value); + case "FloatLiteral": + return new FloatValue((statement as FloatLiteral).value); case "StringLiteral": return new StringValue((statement as StringLiteral).value); case "ArrayLiteral": @@ -1351,7 +1381,7 @@ export class Interpreter { function convertToRuntimeValues(input: unknown): AnyRuntimeValue { switch (typeof input) { case "number": - return new NumericValue(input); + return Number.isInteger(input) ? new IntegerValue(input) : new FloatValue(input); case "string": return new StringValue(input); case "boolean": @@ -1394,7 +1424,8 @@ function toJSON(input: AnyRuntimeValue, indent?: number | null, depth?: number): case "NullValue": case "UndefinedValue": // JSON.stringify(undefined) -> undefined return "null"; - case "NumericValue": + case "IntegerValue": + case "FloatValue": case "StringValue": case "BooleanValue": return JSON.stringify(input.value); diff --git a/packages/jinja/test/templates.test.js b/packages/jinja/test/templates.test.js index a4877d297a..52e486965b 100644 --- a/packages/jinja/test/templates.test.js +++ b/packages/jinja/test/templates.test.js @@ -107,7 +107,7 @@ const TEST_STRINGS = { FILTER_OPERATOR_11: `{{ items | rejectattr('key') | length }}`, FILTER_OPERATOR_12: `{{ messages | rejectattr('role', 'equalto', 'system') | length }}`, FILTER_OPERATOR_13: `{{ tools | string }}`, - FILTER_OPERATOR_14: `|{{ "1" | int + 2 }}|{{ "invalid" | int }}|{{ "invalid" | int(-1) }}|{{ true | int }}|{{ false | int }}|{{ 1.5 | int }}|{{ "1.5" | float }}|{{ "invalid" | float }}|{{ "invalid" | float(2) }}|`, + FILTER_OPERATOR_14: `|{{ "1" | int + 2 }}|{{ "invalid" | int }}|{{ "invalid" | int(-1) }}|{{ true | int }}|{{ false | int }}|{{ 1.5 | int }}|{{ "1.5" | float }}|{{ "invalid" | float }}|{{ "invalid" | float("hello") }}|`, // Filter statements FILTER_STATEMENTS: `{% filter upper %}text{% endfilter %}`, @@ -2165,7 +2165,7 @@ const TEST_PARSED = { { value: "|", type: "Pipe" }, { value: "float", type: "Identifier" }, { value: "(", type: "OpenParen" }, - { value: "2", type: "NumericLiteral" }, + { value: "hello", type: "StringLiteral" }, { value: ")", type: "CloseParen" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, @@ -3979,7 +3979,7 @@ const EXPECTED_OUTPUTS = { FILTER_OPERATOR_11: `3`, FILTER_OPERATOR_12: `2`, FILTER_OPERATOR_13: `[{"name": "some_tool", "arguments": {"some_name": "string"}}]`, - FILTER_OPERATOR_14: `|3|0|-1|1|0|1|1.5|0.0|2|`, + FILTER_OPERATOR_14: `|3|0|-1|1|0|1|1.5|0.0|hello|`, // Filter statements FILTER_STATEMENTS: `TEXT`, From d06c78213c5a679350e0fb36c2257edcc09859ba Mon Sep 17 00:00:00 2001 From: Joshua Lochner <26504141+xenova@users.noreply.github.com> Date: Sat, 3 May 2025 22:16:01 -0400 Subject: [PATCH 21/35] Add jamba e2e test --- packages/jinja/test/e2e.test.js | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/packages/jinja/test/e2e.test.js b/packages/jinja/test/e2e.test.js index d4df84ecc2..e2f0fc2723 100644 --- a/packages/jinja/test/e2e.test.js +++ b/packages/jinja/test/e2e.test.js @@ -906,6 +906,23 @@ const TEST_CUSTOM_TEMPLATES = Object.freeze({ target: '<|im_start|>system\nYou are a helpful customer support assistant. Use the supplied tools to assist the user.<|im_end|>\n<|im_start|>user\nHi, can you tell me the delivery date for my order? The order id is 1234 and 4321.<|im_end|>\n<|im_start|>assistant\n<|tool_call_start|>\n```python\nget_delivery_date(order_id="1234")\nget_delivery_date(order_id="4321")\n```\n<|tool_call_end|>\n<|im_end|>\n<|im_start|>tool\n{"delivery_date": "2024-09-05", "order_id": "1234"}<|im_end|>\n<|im_start|>tool\n{"delivery_date": "2024-09-05", "order_id": "4321"}<|im_end|>\n<|im_start|>assistant\n<|thought_start|>\n\nI have the information you need, both orders will be delivered on the same date, 2024-09-05.\n\n<|thought_end|>\nBoth your orders will be delivered on 2024-09-05.<|im_end|>\n<|im_start|>assistant\n', }, + "ai21labs/AI21-Jamba-Large-1.6": { + chat_template: + '{# Variables #}\n{% set ns = namespace(message_count=0, is_last_checked_defined=False) %}\n{##}\n{% set bom_str = bom_str or "<|bom|>" %}\n{% set eom_str = eom_str or "<|eom|>" %}\n{% set default_system_message = default_system_message or "" %}\n{##}\n{% set documents_prefix = "" %}\n{% set documents_suffix = "" %}\n{% set tool_definitions_prefix = "" %}\n{% set tool_definitions_suffix = "" %}\n{% set active_modes_prefix = "" %}\n{% set active_modes_suffix = "" %}\n{##}\n{% set tool_calls_prefix = "" %}\n{% set tool_calls_suffix = "" %}\n{% set citations_prefix = "" %}\n{% set citations_suffix = "" %}\n{##}\n{% if add_generation_prompt is not defined %}\n {% set add_generation_prompt = True %}\n{% endif %}\n{% set role_to_predict = role_to_predict or "assistant" %}\n{% if messages|length > 0 and messages[0].role == "system" %}\n {% set system_message = messages[0].content %}\n {% set loop_messages = messages[1:] %}\n{% else %}\n {% set system_message = default_system_message %}\n {% set loop_messages = messages %}\n{% endif %}\n{##}\n{##}\n{# Macros #}\n{% macro handle_tool_definitions(tools) %}\n {{- tool_definitions_prefix -}}\n {{- "\\n# Tools" -}}\n {{- "\\n\\n## Functions" -}}\n {% for tool in tools %}\n {% set _ = is_param_set(tool, field="type") %}\n {% set is_tool_type_set = ns.is_last_checked_defined %}\n {% if is_tool_type_set %}\n {% if tool.type == "function" %}\n {% set tool = tool.function %}\n {% else %}\n {{ raise_exception("Currently, the only supported tool type is `function`") }}\n {% endif %}\n {% endif %}\n {{- "\\n\\n" + (tool|tojson(indent=2)) -}}\n {% endfor %}\n {{- "\\n" + tool_definitions_suffix -}}\n{% endmacro %}\n{##}\n{% macro handle_first_system_message(system_message, tools) %}\n {{- bom_str + handle_role("system") -}}\n {% set _ = is_param_set(system_message) %}\n {% set is_system_message_set = ns.is_last_checked_defined %}\n {% if is_system_message_set %}\n {{- system_message -}}\n {% endif %}\n {% set _ = is_param_set(tools, check_length=True) %}\n {% set is_tools_set = ns.is_last_checked_defined %}\n {% if is_tools_set %}\n {% if system_message %}\n {{- "\\n\\n" -}}\n {% endif %}\n {{- handle_tool_definitions(tools) -}}\n {% endif %}\n {% set ns.message_count = ns.message_count + 1 %}\n{% endmacro %}\n{##}\n{% macro handle_tool_calls(tool_calls) %}\n {{- tool_calls_prefix + "[\\n" -}}\n {% for tool_call in tool_calls %}\n {% set _ = is_param_set(tool_call, field="function") %}\n {% set is_tool_call_function_set = ns.is_last_checked_defined %}\n {% if is_tool_call_function_set %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {% set arguments = tool_call.arguments %}\n {% if arguments is not string %}\n {%- set arguments = arguments|tojson -%}\n {%- endif %}\n {{ "{\\"name\\": \\"" + tool_call.name + "\\", \\"arguments\\": " + arguments + "}" -}}\n {% if not loop.last %}\n {{- "," }}\n {% endif %}\n {% endfor %}\n {{- "\\n]" + tool_calls_suffix -}}\n{% endmacro %}\n{##}\n{% macro handle_documents(documents) %}\n {{- documents_prefix -}}\n {{- "\\n# Documents" -}}\n {{- "\\n\\nYou can use the following documents for reference:" -}}\n {% for doc in documents %}\n {{- "\\n\\n## Document ID: " + loop.index0|string -}}\n {% set _ = is_param_set(doc, field="title") %}\n {% set is_doc_title_set = ns.is_last_checked_defined %}\n {% if is_doc_title_set %}\n {{- "\\nTitle: " + doc.title -}}\n {% endif %}\n {% for key, value in doc.items() %}\n {% if key not in ["title", "text"] %}\n {{- "\\n" + key|title + ": " + value|string -}}\n {% endif %}\n {% endfor %}\n {{- "\\nText: " + doc.text -}}\n {% endfor %}\n {{- "\\n" + documents_suffix -}}\n{% endmacro %}\n{##}\n{% macro handle_knobs(knobs) %}\n {{- active_modes_prefix -}}\n {{- "\\n# Active Modes" -}}\n {{ "\\n\\nThe following modes configure the format or style of your responses. You should adhere to all currently" -}}\n {{ " active modes simultaneously." -}}\n {% if knobs.citation_mode == "fast" %}\n {{- "\\n\\n## Citation Mode" -}}\n {{- "\\n\\nProvide a list of references only for the documents you base your response on. Format your response" -}}\n {{ " with the original answer followed by a citation section. Use this template:" -}}\n {{ " `{answer}" + citations_prefix + "DOCUMENT_IDS" + citations_suffix + "`, where DOCUMENT_IDS are the relevant document numbers" -}}\n {{ " (e.g. [2, 5, 9]), or [] if the answer cannot be supported by the provided documents." -}}\n {% endif %}\n {% if knobs.response_format == "json_object" %}\n {{- "\\n\\n## JSON Mode" -}}\n {{ "\\n\\nProvide your response in JSON format. Adhere strictly to any schema given by the user." -}}\n {{ " If an appropriate JSON format exists, use it without modification." -}}\n {% endif %}\n {{- "\\n" + active_modes_suffix -}}\n{% endmacro %}\n{##}\n{% macro get_last_user_index(messages) %}\n {% set ns.last_user_index = 0 %}\n {% for message in messages %}\n {% if message.role == \'user\' %}\n {% set ns.last_user_index = loop.index0 %}\n {% endif %}\n {% endfor %}\n {{- ns.last_user_index -}}\n{% endmacro %}\n{##}\n{% macro handle_last_system_message(documents, knobs, use_documents, use_knobs) %}\n {{- bom_str + handle_role("system") -}}\n {% set macros_to_call = [] %}\n {% set params_for_macros = [] %}\n {% if use_documents %}\n {% set macros_to_call = macros_to_call + [handle_documents] %}\n {% set params_for_macros = params_for_macros + [[documents]] %}\n {% endif %}\n {% if use_knobs %}\n {% set macros_to_call = macros_to_call + [handle_knobs] %}\n {% set params_for_macros = params_for_macros + [[knobs]] %}\n {% endif %}\n {% for i in range(macros_to_call|length) %}\n {% if i > 0 %}\n {{- "\\n\\n" -}}\n {% endif %}\n {{- macros_to_call[i](*params_for_macros[i]) -}}\n {% endfor %}\n {% set ns.message_count = ns.message_count + 1 %}\n{% endmacro %}\n{##}\n{% macro handle_role(role, add_space=True) %}\n {{- "<|" + role + "|>" -}}\n {% if add_space %}\n {{- " " -}}\n {% endif %}\n{% endmacro %}\n{##}\n{% macro is_param_set(param, field=none, check_length=False) %}\n {% if field is not none %}\n {% if field in param %}\n {% set param = param[field] %}\n {% else %}\n {% set param = none %}\n {% endif %}\n {% endif %}\n {% set is_defined = param is defined and param is not none %}\n {% if check_length %}\n {% set ns.is_last_checked_defined = is_defined and param|length > 0 %}\n {% else %}\n {% set ns.is_last_checked_defined = is_defined %}\n {% endif %}\n{% endmacro %}\n{##}\n{##}\n{# Template #}\n{% if bos_token is defined and bos_token is not none %}\n {{- bos_token -}}\n{% endif %}\n{% set _ = is_param_set(system_message) %}\n{% set is_system_message_set = ns.is_last_checked_defined %}\n{% set _ = is_param_set(tools, check_length=True) %}\n{% set is_tools_set = ns.is_last_checked_defined %}\n{% set has_system_message = (is_system_message_set or is_tools_set) %}\n{% if has_system_message %}\n {{- handle_first_system_message(system_message, tools) -}}\n{% endif %}\n{% set last_user_index = get_last_user_index(loop_messages)|int %}\n{% for message in loop_messages %}\n {% if loop.index0 == last_user_index %}\n {% set _ = is_param_set(documents, check_length=True) %}\n {% set use_documents = ns.is_last_checked_defined %}\n {% set _ = is_param_set(knobs) %}\n {% set use_knobs = ns.is_last_checked_defined and knobs.is_set %}\n {% set add_last_system_message = use_documents or use_knobs %}\n {% if add_last_system_message %}\n {% if ns.message_count > 0 %}\n {{- eom_str -}}\n {% endif %}\n {{- handle_last_system_message(documents, knobs, use_documents, use_knobs) -}}\n {% endif %}\n {% endif %}\n {% set role = message.role %}\n {% set _ = is_param_set(message, field="name") %}\n {% set is_message_name_set = ns.is_last_checked_defined %}\n {% if is_message_name_set %}\n {% set message_prefix = handle_role(role) + "(" + message.name + ")" %}\n {% else %}\n {% set message_prefix = handle_role(role) %}\n {% endif %}\n {% set content = (message.content or "") %}\n {% if content is not string %}\n {% set content = content|tojson %}\n {% endif %}\n {% if ns.message_count > 0 %}\n {{- eom_str -}}\n {% endif %}\n {{- bom_str + message_prefix + content -}}\n {% set _ = is_param_set(message, field="tool_calls", check_length=True) %}\n {% set is_tool_calls_set = ns.is_last_checked_defined %}\n {% if role == "assistant" and is_tool_calls_set %}\n {{- handle_tool_calls(message.tool_calls) -}}\n {% endif %}\n {% set _ = is_param_set(message, field="citations", check_length=False) %}\n {% set is_citations_set = ns.is_last_checked_defined %}\n {% if role == "assistant" and is_citations_set and knobs.is_set and knobs.citation_mode != "off" %}\n {{- citations_prefix + message.citations|map(attribute="document_id")|list|string + citations_suffix -}}\n {% endif %}\n {% set ns.message_count = ns.message_count + 1 %}\n{% endfor %}\n{% if add_generation_prompt %}\n {% if ns.message_count > 0 %}\n {{- eom_str -}}\n {% endif %}\n {{- bom_str + handle_role(role_to_predict, add_space=False) -}}\n {% set _ = is_param_set(generation_preamble) %}\n {% set is_generation_preamble_set = ns.is_last_checked_defined %}\n {% if is_generation_preamble_set and generation_preamble.strip() != "" %}\n {{- " " + generation_preamble -}}\n {% endif %}\n {% set ns.message_count = ns.message_count + 1 %}\n{% else %}\n {% if ns.message_count > 0 %}\n {{- eom_str -}}\n {% endif %}\n{% endif %}\n', + data: { + messages: [ + { + role: "system", + content: + "You are an ancient oracle who speaks in cryptic but wise phrases, always hinting at deeper meanings.", + }, + { role: "user", content: "Hello!" }, + ], + bos_token: "<|startoftext|>", + }, + target: + "<|startoftext|><|bom|><|system|> You are an ancient oracle who speaks in cryptic but wise phrases, always hinting at deeper meanings.<|eom|><|bom|><|user|> Hello!<|eom|><|bom|><|assistant|>", + }, }); function render({ chat_template, data, target }) { From b094bc410c32f945df061b0e59c660337c9d05aa Mon Sep 17 00:00:00 2001 From: Joshua Lochner <26504141+xenova@users.noreply.github.com> Date: Sun, 4 May 2025 01:09:12 -0400 Subject: [PATCH 22/35] Allow comments to be added to the AST instead of stripped --- packages/jinja/src/ast.ts | 7 +++++++ packages/jinja/src/format.ts | 3 +++ packages/jinja/src/lexer.ts | 35 ++++++++++++++++++++++++++--------- packages/jinja/src/parser.ts | 3 +++ packages/jinja/src/runtime.ts | 2 ++ 5 files changed, 41 insertions(+), 9 deletions(-) diff --git a/packages/jinja/src/ast.ts b/packages/jinja/src/ast.ts index 19fea17884..4a0e379a39 100644 --- a/packages/jinja/src/ast.ts +++ b/packages/jinja/src/ast.ts @@ -77,6 +77,13 @@ export class Macro extends Statement { } } +export class Comment extends Statement { + override type = "Comment"; + constructor(public value: string) { + super(); + } +} + /** * Expressions will result in a value at runtime (unlike statements). */ diff --git a/packages/jinja/src/format.ts b/packages/jinja/src/format.ts index 35cb8c24d4..21cf3c2177 100644 --- a/packages/jinja/src/format.ts +++ b/packages/jinja/src/format.ts @@ -1,6 +1,7 @@ import type { Program, Statement, + Comment, If, For, SetStatement, @@ -73,6 +74,8 @@ function formatStatement(node: Statement, depth: number, indentStr: string): str return formatCallStatement(node as CallStatement, depth, indentStr); case "FilterStatement": return formatFilterStatement(node as FilterStatement, depth, indentStr); + case "Comment": + return pad + "{# " + (node as Comment).value + " #}"; default: return pad + "{{- " + formatExpression(node as Expression) + " -}}"; } diff --git a/packages/jinja/src/lexer.ts b/packages/jinja/src/lexer.ts index 16d8dea367..396d3bd152 100644 --- a/packages/jinja/src/lexer.ts +++ b/packages/jinja/src/lexer.ts @@ -28,6 +28,7 @@ export const TOKEN_TYPES = Object.freeze({ MultiplicativeBinaryOperator: "MultiplicativeBinaryOperator", // * / % ComparisonBinaryOperator: "ComparisonBinaryOperator", // < > <= >= == != UnaryOperator: "UnaryOperator", // ! - + + Comment: "Comment", // {# ... #} }); export type TokenType = keyof typeof TOKEN_TYPES; @@ -120,30 +121,27 @@ function preprocess(template: string, options: PreprocessOptions = {}): string { template = template.slice(0, -1); } - // Replace all comments with a placeholder - // This ensures that comments don't interfere with the following options - template = template.replace(/{#.*?#}/gs, "{##}"); - if (options.lstrip_blocks) { // The lstrip_blocks option can also be set to strip tabs and spaces from the // beginning of a line to the start of a block. (Nothing will be stripped if // there are other characters before the start of the block.) - template = template.replace(/^[ \t]*({[#%])/gm, "$1"); + template = template.replace(/^[ \t]*({[#%-])/gm, "$1"); } if (options.trim_blocks) { // If an application configures Jinja to trim_blocks, the first newline after // a template tag is removed automatically (like in PHP). - template = template.replace(/([#%]})\n/g, "$1"); + template = template.replace(/([#%-]})\n/g, "$1"); } return ( template - .replace(/{##}/g, "") // Remove comments .replace(/-%}\s*/g, "%}") .replace(/\s*{%-/g, "{%") .replace(/-}}\s*/g, "}}") .replace(/\s*{{-/g, "{{") + .replace(/-#}\s*/g, "#}") + .replace(/\s*{#-/g, "{#") // Handle the custom transformers-specific `generation` tag. // See https://github.com/huggingface/transformers/pull/30650 for more information. @@ -194,13 +192,17 @@ export function tokenize(source: string, options: PreprocessOptions = {}): Token if ( lastTokenType === undefined || lastTokenType === TOKEN_TYPES.CloseStatement || - lastTokenType === TOKEN_TYPES.CloseExpression + lastTokenType === TOKEN_TYPES.CloseExpression || + lastTokenType === TOKEN_TYPES.Comment ) { let text = ""; while ( cursorPosition < src.length && // Keep going until we hit the next Jinja statement or expression - !(src[cursorPosition] === "{" && (src[cursorPosition + 1] === "%" || src[cursorPosition + 1] === "{")) + !( + src[cursorPosition] === "{" && + (src[cursorPosition + 1] === "%" || src[cursorPosition + 1] === "{" || src[cursorPosition + 1] === "#") + ) ) { // Consume text text += src[cursorPosition++]; @@ -213,6 +215,21 @@ export function tokenize(source: string, options: PreprocessOptions = {}): Token } } + // Possibly consume a comment + if (src[cursorPosition] === "{" && src[cursorPosition + 1] === "#") { + cursorPosition += 2; // Skip the opening {# + + let comment = ""; + while ( + cursorPosition < src.length && + (src[cursorPosition] !== "#" || src[cursorPosition + 1] !== "}")) { + comment += src[cursorPosition++]; + } + tokens.push(new Token(comment, TOKEN_TYPES.Comment)); + cursorPosition += 2; // Skip the closing #} + continue; + } + // Consume (and ignore) all whitespace inside Jinja statements or expressions consumeWhile((char) => /\s/.test(char)); diff --git a/packages/jinja/src/parser.ts b/packages/jinja/src/parser.ts index 950399fad3..474f664e66 100644 --- a/packages/jinja/src/parser.ts +++ b/packages/jinja/src/parser.ts @@ -28,6 +28,7 @@ import { SpreadExpression, IntegerLiteral, FloatLiteral, + Comment, } from "./ast"; /** @@ -61,6 +62,8 @@ export function parse(tokens: Token[]): Program { function parseAny(): Statement { switch (tokens[current].type) { + case TOKEN_TYPES.Comment: + return new Comment(tokens[current++].value); case TOKEN_TYPES.Text: return parseText(); case TOKEN_TYPES.OpenStatement: diff --git a/packages/jinja/src/runtime.ts b/packages/jinja/src/runtime.ts index a5bf64334b..80c5237b12 100644 --- a/packages/jinja/src/runtime.ts +++ b/packages/jinja/src/runtime.ts @@ -1369,6 +1369,8 @@ export class Interpreter { return this.evaluateTestExpression(statement as TestExpression, environment); case "SelectExpression": return this.evaluateSelectExpression(statement as SelectExpression, environment); + case "Comment": + return new NullValue(); default: throw new SyntaxError(`Unknown node type: ${statement.type}`); } From 1101173f7a74d87306a91f84b07ed0d6cc613309 Mon Sep 17 00:00:00 2001 From: Joshua Lochner <26504141+xenova@users.noreply.github.com> Date: Sun, 4 May 2025 01:17:54 -0400 Subject: [PATCH 23/35] Assert comments end with #} --- packages/jinja/src/lexer.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/jinja/src/lexer.ts b/packages/jinja/src/lexer.ts index 396d3bd152..e2adaecfcf 100644 --- a/packages/jinja/src/lexer.ts +++ b/packages/jinja/src/lexer.ts @@ -220,9 +220,11 @@ export function tokenize(source: string, options: PreprocessOptions = {}): Token cursorPosition += 2; // Skip the opening {# let comment = ""; - while ( - cursorPosition < src.length && - (src[cursorPosition] !== "#" || src[cursorPosition + 1] !== "}")) { + while (src[cursorPosition] !== "#" || src[cursorPosition + 1] !== "}") { + // Check for end of input + if (cursorPosition + 2 >= src.length) { + throw new SyntaxError("Missing end of comment tag"); + } comment += src[cursorPosition++]; } tokens.push(new Token(comment, TOKEN_TYPES.Comment)); From 2b1aa04006b2eca43a290c684700b859234ff7ce Mon Sep 17 00:00:00 2001 From: Joshua Lochner <26504141+xenova@users.noreply.github.com> Date: Sun, 4 May 2025 01:18:29 -0400 Subject: [PATCH 24/35] Add e2e llama vision test --- packages/jinja/test/e2e.test.js | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/packages/jinja/test/e2e.test.js b/packages/jinja/test/e2e.test.js index e2f0fc2723..ffeb2665eb 100644 --- a/packages/jinja/test/e2e.test.js +++ b/packages/jinja/test/e2e.test.js @@ -923,6 +923,24 @@ const TEST_CUSTOM_TEMPLATES = Object.freeze({ target: "<|startoftext|><|bom|><|system|> You are an ancient oracle who speaks in cryptic but wise phrases, always hinting at deeper meanings.<|eom|><|bom|><|user|> Hello!<|eom|><|bom|><|assistant|>", }, + + "meta-llama/Llama-3.2-11B-Vision-Instruct": { + chat_template: + "{{- bos_token }}\n{%- if custom_tools is defined %}\n {%- set tools = custom_tools %}\n{%- endif %}\n{%- if not tools_in_user_message is defined %}\n {%- set tools_in_user_message = true %}\n{%- endif %}\n{%- if not date_string is defined %}\n {%- if strftime_now is defined %}\n {%- set date_string = strftime_now(\"%d %b %Y\") %}\n {%- else %}\n {%- set date_string = \"26 Jul 2024\" %}\n {%- endif %}\n{%- endif %}\n{%- if not tools is defined %}\n {%- set tools = none %}\n{%- endif %}\n\n{#- This block extracts the system message, so we can slot it into the right place. #}\n{%- if messages[0]['role'] == 'system' %}\n {%- set system_message = messages[0]['content']|trim %}\n {%- set messages = messages[1:] %}\n {%- set user_supplied_system_message = true %}\n{%- else %}\n {%- set system_message = \"\" %}\n {%- set user_supplied_system_message = false %}\n{%- endif %}\n\n{#- Find out if there are any images #}\n{% set image_ns = namespace(has_images=false) %} \n{%- for message in messages %}\n {%- for content in message['content'] %}\n {%- if content['type'] == 'image' %}\n {%- set image_ns.has_images = true %}\n {%- endif %}\n {%- endfor %}\n{%- endfor %}\n\n{#- System message if there are no images, or if the user supplied one #}\n{%- if user_supplied_system_message or not image_ns.has_images %}\n {{- \"<|start_header_id|>system<|end_header_id|>\\n\\n\" }}\n {%- if tools is not none %}\n {{- \"Environment: ipython\\n\" }}\n {%- endif %}\n {{- \"Cutting Knowledge Date: December 2023\\n\" }}\n {{- \"Today Date: \" + date_string + \"\\n\\n\" }}\n {%- if tools is not none and not tools_in_user_message %}\n {{- \"You have access to the following functions. To call a function, please respond with JSON for a function call.\" }}\n {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n {{- \"Do not use variables.\\n\\n\" }}\n {%- for t in tools %}\n {{- t | tojson(indent=4) }}\n {{- \"\\n\\n\" }}\n {%- endfor %}\n {%- endif %}\n {{- system_message }}\n {{- \"<|eot_id|>\" }}\n{%- endif %}\n\n{#- Custom tools are passed in a user message with some extra guidance #}\n{%- if tools_in_user_message and not tools is none %}\n {#- Extract the first user message so we can plug it in here #}\n {%- if messages | length != 0 %}\n {%- set first_user_message = messages[0]['content']|trim %}\n {%- set messages = messages[1:] %}\n {%- else %}\n {{- raise_exception(\"Cannot put tools in the first user message when there's no first user message!\") }}\n{%- endif %}\n {{- '<|start_header_id|>user<|end_header_id|>\\n\\n' -}}\n {{- \"Given the following functions, please respond with a JSON for a function call \" }}\n {{- \"with its proper arguments that best answers the given prompt.\\n\\n\" }}\n {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n {{- \"Do not use variables.\\n\\n\" }}\n {%- for t in tools %}\n {{- t | tojson(indent=4) }}\n {{- \"\\n\\n\" }}\n {%- endfor %}\n {{- first_user_message + \"<|eot_id|>\"}}\n{%- endif %}\n\n{%- for message in messages %}\n {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}\n {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\\n\\n' }}\n {%- if message['content'] is string %}\n {{- message['content'] }}\n {%- else %}\n {%- for content in message['content'] %}\n {%- if content['type'] == 'image' %}\n {{- '<|image|>' }}\n {%- elif content['type'] == 'text' %}\n {{- content['text'] }}\n {%- endif %}\n {%- endfor %}\n {%- endif %}\n {{- '<|eot_id|>' }}\n {%- elif 'tool_calls' in message %}\n {%- if not message.tool_calls|length == 1 %}\n {{- raise_exception(\"This model only supports single tool-calls at once!\") }}\n {%- endif %}\n {%- set tool_call = message.tool_calls[0].function %}\n {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n {{- '{\"name\": \"' + tool_call.name + '\", ' }}\n {{- '\"parameters\": ' }}\n {{- tool_call.arguments | tojson }}\n {{- \"}\" }}\n {{- \"<|eot_id|>\" }}\n {%- elif message.role == \"tool\" or message.role == \"ipython\" %}\n {{- \"<|start_header_id|>ipython<|end_header_id|>\\n\\n\" }}\n {%- if message.content is mapping or message.content is iterable %}\n {{- message.content | tojson }}\n {%- else %}\n {{- message.content }}\n {%- endif %}\n {{- \"<|eot_id|>\" }}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' }}\n{%- endif %}\n", + data: { + // Example adapted from https://huggingface.co/meta-llama/Llama-3.2-11B-Vision-Instruct#use-with-transformers + messages: [ + { + role: "user", + content: [{ type: "image" }, { type: "text", text: "If I had to write a haiku for this one, it would be: " }], + }, + ], + bos_token: "<|begin_of_text|>", + add_generation_prompt: true, + }, + target: + "<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n<|image|>If I had to write a haiku for this one, it would be: <|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n", + }, }); function render({ chat_template, data, target }) { From f4c9051edf8634902ecc6a14bc0e8c9bb6c12e2c Mon Sep 17 00:00:00 2001 From: Joshua Lochner <26504141+xenova@users.noreply.github.com> Date: Sun, 4 May 2025 02:19:58 -0400 Subject: [PATCH 25/35] Differentiate between if statement and ternary expression --- packages/jinja/src/ast.ts | 11 +++++++++++ packages/jinja/src/format.ts | 16 +++++++++------- packages/jinja/src/parser.ts | 5 +++-- packages/jinja/src/runtime.ts | 10 ++++++++++ packages/jinja/test/templates.test.js | 19 +++++++++++++++++++ 5 files changed, 52 insertions(+), 9 deletions(-) diff --git a/packages/jinja/src/ast.ts b/packages/jinja/src/ast.ts index 4a0e379a39..3dab02b90f 100644 --- a/packages/jinja/src/ast.ts +++ b/packages/jinja/src/ast.ts @@ -318,3 +318,14 @@ export class CallStatement extends Statement { super(); } } + +export class Ternary extends Expression { + override type = "Ternary"; + constructor( + public condition: Expression, + public trueExpr: Expression, + public falseExpr: Expression + ) { + super(); + } +} diff --git a/packages/jinja/src/format.ts b/packages/jinja/src/format.ts index 21cf3c2177..3533ccdcd3 100644 --- a/packages/jinja/src/format.ts +++ b/packages/jinja/src/format.ts @@ -27,6 +27,7 @@ import type { CallStatement, FilterStatement, SpreadExpression, + Ternary, } from "./ast"; const NEWLINE = "\n"; @@ -37,6 +38,7 @@ const OPERATOR_PRECEDENCE: Record = { MultiplicativeBinaryOperator: 2, AdditiveBinaryOperator: 1, ComparisonBinaryOperator: 0, + Ternary: -1, }; export function format(program: Program, indent: string | number = "\t"): string { @@ -284,13 +286,13 @@ function formatExpression(node: Expression, parentPrec: number = -1): string { const n = node as KeywordArgumentExpression; return `${n.key.value}=${formatExpression(n.value, -1)}`; } - case "If": { - // Special case for ternary operator (If as an expression, not a statement) - const n = node as If; - const test = formatExpression(n.test, -1); - const body = formatExpression(n.body[0], 0); // Ternary operators have a single body and alternate - const alternate = formatExpression(n.alternate[0], -1); - return `${body} if ${test} else ${alternate}`; + case "Ternary": { + const n = node as Ternary; + const expr = `${formatExpression(n.trueExpr, OPERATOR_PRECEDENCE.Ternary)} if ${formatExpression( + n.condition, + OPERATOR_PRECEDENCE.Ternary + )} else ${formatExpression(n.falseExpr, OPERATOR_PRECEDENCE.Ternary)}`; + return OPERATOR_PRECEDENCE.Ternary < parentPrec ? `(${expr})` : expr; } default: throw new Error(`Unknown expression type: ${node.type}`); diff --git a/packages/jinja/src/parser.ts b/packages/jinja/src/parser.ts index 474f664e66..e89c2d7f4f 100644 --- a/packages/jinja/src/parser.ts +++ b/packages/jinja/src/parser.ts @@ -28,6 +28,7 @@ import { SpreadExpression, IntegerLiteral, FloatLiteral, + Ternary, Comment, } from "./ast"; @@ -348,8 +349,8 @@ export function parse(tokens: Token[]): Program { if (isIdentifier("else")) { // Ternary expression with else ++current; // consume 'else' - const alternate = parseIfExpression(); // recurse to support chained ternaries - return new If(test, [a], [alternate]); + const falseExpr = parseIfExpression(); // recurse to support chained ternaries + return new Ternary(test, a, falseExpr); } else { // Select expression on iterable return new SelectExpression(a, test); diff --git a/packages/jinja/src/runtime.ts b/packages/jinja/src/runtime.ts index 80c5237b12..85d75f5d48 100644 --- a/packages/jinja/src/runtime.ts +++ b/packages/jinja/src/runtime.ts @@ -24,6 +24,7 @@ import type { SelectExpression, CallStatement, FilterStatement, + Ternary, SpreadExpression, } from "./ast"; import { range, slice, titleCase } from "./utils"; @@ -956,6 +957,13 @@ export class Interpreter { } } + private evaluateTernaryExpression(node: Ternary, environment: Environment): AnyRuntimeValue { + const cond = this.evaluate(node.condition, environment); + return cond.__bool__().value + ? this.evaluate(node.trueExpr, environment) + : this.evaluate(node.falseExpr, environment); + } + private evalProgram(program: Program, environment: Environment): StringValue { return this.evaluateBlock(program.body, environment); } @@ -1369,6 +1377,8 @@ export class Interpreter { return this.evaluateTestExpression(statement as TestExpression, environment); case "SelectExpression": return this.evaluateSelectExpression(statement as SelectExpression, environment); + case "Ternary": + return this.evaluateTernaryExpression(statement as Ternary, environment); case "Comment": return new NullValue(); default: diff --git a/packages/jinja/test/templates.test.js b/packages/jinja/test/templates.test.js index 52e486965b..77c7b43a78 100644 --- a/packages/jinja/test/templates.test.js +++ b/packages/jinja/test/templates.test.js @@ -154,6 +154,7 @@ const TEST_STRINGS = { // Ternary operator TERNARY_OPERATOR: `|{{ 'a' if true else 'b' }}|{{ 'a' if false else 'b' }}|{{ 'a' if 1 + 1 == 2 else 'b' }}|{{ 'a' if 1 + 1 == 3 or 1 * 2 == 3 else 'b' }}|`, + TERNARY_OPERATOR_1: `{{ (x if true else []) | length }}`, TERNARY_SET: `{% set x = 1 if True else 2 %}{{ x }}`, TERNARY_CONSECUTIVE: `{% set x = 1 if False else 2 if False else 3 %}{{ x }}`, TERNARY_SHORTCUT: `{{ 'foo' if false }}{{ 'bar' if true }}`, @@ -3039,6 +3040,20 @@ const TEST_PARSED = { { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, ], + TERNARY_OPERATOR_1: [ + { value: "{{", type: "OpenExpression" }, + { value: "(", type: "OpenParen" }, + { value: "x", type: "Identifier" }, + { value: "if", type: "Identifier" }, + { value: "true", type: "Identifier" }, + { value: "else", type: "Identifier" }, + { value: "[", type: "OpenSquareBracket" }, + { value: "]", type: "CloseSquareBracket" }, + { value: ")", type: "CloseParen" }, + { value: "|", type: "Pipe" }, + { value: "length", type: "Identifier" }, + { value: "}}", type: "CloseExpression" }, + ], TERNARY_SET: [ { value: "{%", type: "OpenStatement" }, { value: "set", type: "Identifier" }, @@ -3838,6 +3853,9 @@ const TEST_CONTEXT = { // Ternary operator TERNARY_OPERATOR: {}, + TERNARY_OPERATOR_1: { + x: [{}, {}, {}], + }, TERNARY_SET: {}, TERNARY_CONSECUTIVE: {}, TERNARY_SHORTCUT: {}, @@ -4026,6 +4044,7 @@ const EXPECTED_OUTPUTS = { // Ternary operator TERNARY_OPERATOR: `|a|b|a|b|`, + TERNARY_OPERATOR_1: `3`, TERNARY_SET: `1`, TERNARY_CONSECUTIVE: `3`, TERNARY_SHORTCUT: `bar`, From 5b6d280f2488c56cd6ac92aa3e26f5486b3e1db3 Mon Sep 17 00:00:00 2001 From: Joshua Lochner <26504141+xenova@users.noreply.github.com> Date: Sun, 4 May 2025 22:27:44 -0400 Subject: [PATCH 26/35] New functionality --- packages/jinja/src/runtime.ts | 150 ++++++++++++++++++++++++++++------ packages/jinja/src/utils.ts | 59 +++++++++++++ 2 files changed, 184 insertions(+), 25 deletions(-) diff --git a/packages/jinja/src/runtime.ts b/packages/jinja/src/runtime.ts index 85d75f5d48..93143d98e7 100644 --- a/packages/jinja/src/runtime.ts +++ b/packages/jinja/src/runtime.ts @@ -27,7 +27,7 @@ import type { Ternary, SpreadExpression, } from "./ast"; -import { range, slice, titleCase } from "./utils"; +import { range, replace, slice, strftime_now, titleCase } from "./utils"; export type AnyRuntimeValue = | IntegerValue @@ -127,6 +127,12 @@ export class StringValue extends RuntimeValue { return new StringValue(titleCase(this.value)); }), ], + [ + "capitalize", + new FunctionValue(() => { + return new StringValue(this.value.charAt(0).toUpperCase() + this.value.slice(1)); + }), + ], ["length", new IntegerValue(this.value.length)], [ "rstrip", @@ -146,11 +152,21 @@ export class StringValue extends RuntimeValue { if (args.length === 0) { throw new Error("startswith() requires at least one argument"); } - const prefix = args[0]; - if (!(prefix instanceof StringValue)) { - throw new Error("startswith() argument must be a string"); + const pattern = args[0]; + if (pattern instanceof StringValue) { + return new BooleanValue(this.value.startsWith(pattern.value)); + } else if (pattern instanceof ArrayValue) { + for (const item of pattern.value) { + if (!(item instanceof StringValue)) { + throw new Error("startswith() tuple elements must be strings"); + } + if (this.value.startsWith(item.value)) { + return new BooleanValue(true); + } + } + return new BooleanValue(false); } - return new BooleanValue(this.value.startsWith(prefix.value)); + throw new Error("startswith() argument must be a string or tuple of strings"); }), ], [ @@ -159,11 +175,21 @@ export class StringValue extends RuntimeValue { if (args.length === 0) { throw new Error("endswith() requires at least one argument"); } - const suffix = args[0]; - if (!(suffix instanceof StringValue)) { - throw new Error("endswith() argument must be a string"); + const pattern = args[0]; + if (pattern instanceof StringValue) { + return new BooleanValue(this.value.endsWith(pattern.value)); + } else if (pattern instanceof ArrayValue) { + for (const item of pattern.value) { + if (!(item instanceof StringValue)) { + throw new Error("endswith() tuple elements must be strings"); + } + if (this.value.endsWith(item.value)) { + return new BooleanValue(true); + } + } + return new BooleanValue(false); } - return new BooleanValue(this.value.endsWith(suffix.value)); + throw new Error("endswith() argument must be a string or tuple of strings"); }), ], [ @@ -208,6 +234,34 @@ export class StringValue extends RuntimeValue { return new ArrayValue(result.map((part) => new StringValue(part))); }), ], + [ + "replace", + new FunctionValue((args): StringValue => { + if (args.length < 2) { + throw new Error("replace() requires at least two arguments"); + } + const oldValue = args[0]; + const newValue = args[1]; + if (!(oldValue instanceof StringValue && newValue instanceof StringValue)) { + throw new Error("replace() arguments must be strings"); + } + + let count: AnyRuntimeValue | undefined; + if (args.length > 2) { + if (args[2].type === "KeywordArgumentsValue") { + count = (args[2] as KeywordArgumentsValue).value.get("count") ?? new NullValue(); + } else { + count = args[2]; + } + } else { + count = new NullValue(); + } + if (!(count instanceof IntegerValue || count instanceof NullValue)) { + throw new Error("replace() count argument must be a number or null"); + } + return new StringValue(replace(this.value, oldValue.value, newValue.value, count.value)); + }), + ], ]); } @@ -246,15 +300,22 @@ export class ObjectValue extends RuntimeValue> { return this.value.get(key.value) ?? defaultValue ?? new NullValue(); }), ], - [ - "items", - new FunctionValue(() => { - return new ArrayValue( - Array.from(this.value.entries()).map(([key, value]) => new ArrayValue([new StringValue(key), value])) - ); - }), - ], + ["items", new FunctionValue(() => this.items())], + ["keys", new FunctionValue(() => this.keys())], + ["values", new FunctionValue(() => this.values())], ]); + + items(): ArrayValue { + return new ArrayValue( + Array.from(this.value.entries()).map(([key, value]) => new ArrayValue([new StringValue(key), value])) + ); + } + keys(): ArrayValue { + return new ArrayValue(Array.from(this.value.keys()).map((key) => new StringValue(key))); + } + values(): ArrayValue { + return new ArrayValue(Array.from(this.value.values())); + } } /** @@ -456,6 +517,7 @@ export function setupGlobals(env: Environment): void { throw new Error(args); }); env.set("range", range); + env.set("strftime_now", strftime_now); // NOTE: According to the Jinja docs: The special constants true, false, and none are indeed lowercase. // Because that caused confusion in the past, (True used to expand to an undefined variable that was considered false), @@ -509,7 +571,7 @@ export class Interpreter { // Special case: `anything in undefined` is `false` and `anything not in undefined` is `true` return new BooleanValue(node.operator.value === "not in"); } - throw new Error("Cannot perform operation on undefined values"); + throw new Error(`Cannot perform operation ${node.operator.value} on undefined values`); } else if (left instanceof NullValue || right instanceof NullValue) { throw new Error("Cannot perform operation on null values"); } else if (node.operator.value === "~") { @@ -678,21 +740,36 @@ export class Interpreter { return new StringValue(operand.value.map((x) => x.value).join("")); case "string": return new StringValue(toJSON(operand)); + case "unique": { + const seen = new Set(); + const output: AnyRuntimeValue[] = []; + for (const item of operand.value) { + if (!seen.has(item.value)) { + seen.add(item.value); + output.push(item); + } + } + return new ArrayValue(output); + } default: throw new Error(`Unknown ArrayValue filter: ${filter.value}`); } } else if (operand instanceof StringValue) { switch (filter.value) { + // Filters that are also built-in functions case "length": - return new IntegerValue(operand.value.length); case "upper": - return new StringValue(operand.value.toUpperCase()); case "lower": - return new StringValue(operand.value.toLowerCase()); case "title": - return new StringValue(titleCase(operand.value)); case "capitalize": - return new StringValue(operand.value.charAt(0).toUpperCase() + operand.value.slice(1)); + const builtin = operand.builtins.get(filter.value); + if (builtin instanceof FunctionValue) { + return builtin.value(/* no arguments */ [], environment); + } else if (builtin instanceof IntegerValue) { + return builtin; + } else { + throw new Error(`Unknown StringValue filter: ${filter.value}`); + } case "trim": return new StringValue(operand.value.trim()); case "indent": @@ -808,6 +885,17 @@ export class Interpreter { } else { throw new Error(`Cannot apply filter "${filterName}" to type: ${operand.type}`); } + } else if (filterName === "default") { + const [args, kwargs] = this.evaluateArguments(filter.args, environment); + const defaultValue = args[0] ?? new StringValue(""); + const booleanValue = args[1] ?? kwargs.get("boolean") ?? new BooleanValue(false); + if (!(booleanValue instanceof BooleanValue)) { + throw new Error("`default` filter flag must be a boolean"); + } + if (operand instanceof UndefinedValue || (booleanValue.value && !operand.__bool__().value)) { + return defaultValue; + } + return operand; } if (operand instanceof ArrayValue) { @@ -897,6 +985,14 @@ export class Interpreter { ); return new StringValue(indented.join("\n")); } + case "replace": { + const replaceFn = operand.builtins.get("replace"); + if (!(replaceFn instanceof FunctionValue)) { + throw new Error("replace filter not available"); + } + const [args, kwargs] = this.evaluateArguments(filter.args, environment); + return replaceFn.value([...args, new KeywordArgumentsValue(kwargs)], environment); + } } throw new Error(`Unknown StringValue filter: ${filterName}`); } else { @@ -1133,8 +1229,12 @@ export class Interpreter { iterable = this.evaluate(node.iterable, scope); } - if (!(iterable instanceof ArrayValue)) { - throw new Error(`Expected iterable type in for loop: got ${iterable.type}`); + if (!(iterable instanceof ArrayValue || iterable instanceof ObjectValue)) { + throw new Error(`Expected iterable or object type in for loop: got ${iterable.type}`); + } + + if (iterable instanceof ObjectValue) { + iterable = iterable.keys(); } const items: Expression[] = []; diff --git a/packages/jinja/src/utils.ts b/packages/jinja/src/utils.ts index 835fa35549..ae36b9d7a9 100644 --- a/packages/jinja/src/utils.ts +++ b/packages/jinja/src/utils.ts @@ -52,3 +52,62 @@ export function slice(array: T[], start?: number, stop?: number, step = 1): T export function titleCase(value: string): string { return value.replace(/\b\w/g, (c) => c.toUpperCase()); } + +export function strftime_now(format: string): string { + return strftime(new Date(), format); +} + +/** + * A minimalistic implementation of Python's strftime function. + */ +export function strftime(date: Date, format: string): string { + // Set locale to undefined to use the default locale + const monthFormatterLong = new Intl.DateTimeFormat(undefined, { month: "long" }); + const monthFormatterShort = new Intl.DateTimeFormat(undefined, { month: "short" }); + + const pad2 = (n: number): string => (n < 10 ? "0" + n : n.toString()); + + return format.replace(/%[YmdbBHM%]/g, (token) => { + switch (token) { + case "%Y": + return date.getFullYear().toString(); + case "%m": + return pad2(date.getMonth() + 1); + case "%d": + return pad2(date.getDate()); + case "%b": + return monthFormatterShort.format(date); + case "%B": + return monthFormatterLong.format(date); + case "%H": + return pad2(date.getHours()); + case "%M": + return pad2(date.getMinutes()); + case "%%": + return "%"; + default: + return token; + } + }); +} + +function escapeRegExp(s: string): string { + return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +/** + * Function that mimics Python's string.replace() function. + */ +export function replace(str: string, oldvalue: string, newvalue: string, count?: number | null): string { + if (count === 0) return str; + let remaining = count == null || count < 0 ? Infinity : count; + // NB: Use a Unicode-aware global regex so unpaired surrogates won't match + const pattern = oldvalue.length === 0 ? new RegExp("(?=)", "gu") : new RegExp(escapeRegExp(oldvalue), "gu"); + return str.replaceAll(pattern, (match) => { + if (remaining > 0) { + --remaining; + return newvalue; + } + return match; + }); +} From abeb7a481269af0e6e8523ada536361d9ab175ac Mon Sep 17 00:00:00 2001 From: Joshua Lochner <26504141+xenova@users.noreply.github.com> Date: Sun, 4 May 2025 22:29:42 -0400 Subject: [PATCH 27/35] Add new unit tests --- packages/jinja/test/e2e.test.js | 24 +++ packages/jinja/test/templates.test.js | 238 ++++++++++++++++++++++++++ packages/jinja/test/utils.test.js | 61 ++++++- 3 files changed, 322 insertions(+), 1 deletion(-) diff --git a/packages/jinja/test/e2e.test.js b/packages/jinja/test/e2e.test.js index ffeb2665eb..44747af88c 100644 --- a/packages/jinja/test/e2e.test.js +++ b/packages/jinja/test/e2e.test.js @@ -941,6 +941,30 @@ const TEST_CUSTOM_TEMPLATES = Object.freeze({ target: "<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n<|image|>If I had to write a haiku for this one, it would be: <|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n", }, + "meta-llama/Llama-Guard-3-11B-Vision": { + chat_template: + '{%- if messages|length % 2 == 0 -%}\n {%- set role = \'Agent\' -%}\n{%- else -%}\n {%- set role = \'User\' -%}\n{%- endif -%}\n{%- if categories is not defined -%}\n {%- \n set categories = ({\n "S1": "Violent Crimes.",\n "S2": "Non-Violent Crimes.",\n "S3": "Sex Crimes.",\n "S4": "Child Exploitation.",\n "S5": "Defamation.",\n "S6": "Specialized Advice.",\n "S7": "Privacy.",\n "S8": "Intellectual Property.",\n "S9": "Indiscriminate Weapons.",\n "S10": "Hate.",\n "S11": "Self-Harm.",\n "S12": "Sexual Content.",\n "S13": "Elections.",\n })\n -%}\n{%- endif -%}\n\n{{- "<|begin_of_text|><|start_header_id|>user<|end_header_id|>\\n\\n" -}}\n{%- set ns = namespace(contains_image=false) -%}\n{%- for message in messages -%}\n {%- for content in message[\'content\'] | selectattr(\'type\', \'equalto\', \'image\') -%}\n {%- set ns.contains_image = true -%}\n {%- endfor -%}\n{%- endfor -%}\n{%- if ns.contains_image -%}\n{{- "<|image|> "-}}\n{%- endif -%}\n{{- "Task: Check if there is unsafe content in \'" + role + "\' messages in conversations according our safety policy with the below categories.\\n\\n" -}}\n{{- "\\n"-}}\n\n{%- for key in categories -%}\n {%- if key not in excluded_category_keys -%}\n {{-key + ": " + categories[key] + "\\n" -}}\n {%- endif -%}\n{%- endfor -%}\n\n{{- "\\n\\n"-}}\n{{- "\\n\\n"-}}\n{%- for message in messages -%}\n {%- if (message[\'role\'] == \'user\') != (loop.index0 % 2 == 0) -%}\n {{ raise_exception(\'Conversation roles must alternate user/assistant/user/assistant/...\')-}}\n {%- endif -%}\n {%- set content = message[\'content\'] -%}\n {%- if message[\'role\'] == \'user\' -%}\n {%- set role = \'User\' -%}\n {%- elif message[\'role\'] == \'assistant\' -%}\n {%- set role = \'Agent\' -%}\n {%- endif -%}\n {%- for content in message[\'content\'] | selectattr(\'type\', \'equalto\', \'text\') -%}\n{{- role + \': \' + content[\'text\'] | trim + \'\\n\\n\'-}}\n {%- endfor -%}\n{%- endfor -%}\n{{- "\\n\\n"-}}\n{{- "Provide your safety assessment for ONLY THE LAST " + role + " message in the above conversation:\\n"-}}\n{{- " - First line must read \'safe\' or \'unsafe\'.\\n"-}}\n{{- " - If unsafe, a second line must include a comma-separated list of violated categories. <|eot_id|><|start_header_id|>assistant<|end_header_id|>"-}}\n', + data: { + messages: [ + { + role: "user", + content: [ + { + type: "text", + text: "What is the recipe for mayonnaise?", + }, + { + type: "image", + }, + ], + }, + ], + bos_token: "<|begin_of_text|>", + add_generation_prompt: true, + }, + target: + "<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n<|image|> Task: Check if there is unsafe content in 'User' messages in conversations according our safety policy with the below categories.\n\n\nS1: Violent Crimes.\nS2: Non-Violent Crimes.\nS3: Sex Crimes.\nS4: Child Exploitation.\nS5: Defamation.\nS6: Specialized Advice.\nS7: Privacy.\nS8: Intellectual Property.\nS9: Indiscriminate Weapons.\nS10: Hate.\nS11: Self-Harm.\nS12: Sexual Content.\nS13: Elections.\n\n\n\n\nUser: What is the recipe for mayonnaise?\n\n\n\nProvide your safety assessment for ONLY THE LAST User message in the above conversation:\n - First line must read 'safe' or 'unsafe'.\n - If unsafe, a second line must include a comma-separated list of violated categories. <|eot_id|><|start_header_id|>assistant<|end_header_id|>", + }, }); function render({ chat_template, data, target }) { diff --git a/packages/jinja/test/templates.test.js b/packages/jinja/test/templates.test.js index 77c7b43a78..e8b98d052c 100644 --- a/packages/jinja/test/templates.test.js +++ b/packages/jinja/test/templates.test.js @@ -34,8 +34,10 @@ const TEST_STRINGS = { FOR_LOOP_UNPACKING: `|{% for x, y in [ [1, 2], [3, 4] ] %}|{{ x + ' ' + y }}|{% endfor %}|`, FOR_LOOP_DEFAULT: `{% for x in [] %}{{ 'A' }}{% else %}{{'B'}}{% endfor %}`, FOR_LOOP_SELECT: `{% for x in [1, 2, 3, 4] if x > 2 %}{{ x }}{% endfor %}`, + FOR_LOOP_SELECT_2: `{% for x in arr | selectattr('value', 'equalto', 'a') %}{{ x['value'] }}{% endfor %}`, FOR_LOOP_BREAK: `{% for x in [1, 2, 3, 4] %}{% if x == 3 %}{% break %}{% endif %}{{ x }}{% endfor %}`, FOR_LOOP_CONTINUE: `{% for x in [1, 2, 3, 4] %}{% if x == 3 %}{% continue %}{% endif %}{{ x }}{% endfor %}`, + FOR_LOOP_OBJECTS: `{% for x in obj %}{{ x + ':' + obj[x] + ';' }}{% endfor %}`, // Set variables VARIABLES: `{% set x = 'Hello' %}{% set y = 'World' %}{{ x + ' ' + y }}`, @@ -74,6 +76,7 @@ const TEST_STRINGS = { SPLIT_3: `|{{ " test it ".split(" ", 4) | join("|") }}|`, SPLIT_4: `|{{ " 1 2 3 ".split() | tojson }}|{{ "babbaccabbb".split("b") | tojson }}|{{ "babbaccabbb".split("b", 2) | tojson }}|`, SPLIT_5: `|{{ " 1 2 3 4 5 ".split(none, 0) | join(",") }}|{{ " 1 2 3 4 5 ".split(none, 3) | join(",") }}|{{ " 1 2 3 4 5 ".split(" ", 0) | join(",") }}|{{ " 1 2 3 4 5 ".split(" ", 3) | join(",") }}|{{ " 1 2 3 4 5 ".split(" ", 10) | join(",") }}|`, + REPLACE: `|{{ "test test".replace("test", "TEST") }}|{{ "test test".replace("test", "TEST", 1) }}|{{ "test test".replace("", "_", 2) }}|{{ "abcabc".replace("a", "x", count=1) }}|`, // String indexing and slicing STRING_SLICING: `|{{ x[0] }}|{{ x[:] }}|{{ x[:3] }}|{{ x[1:4] }}|{{ x[1:-1] }}|{{ x[1::2] }}|{{ x[5::-1] }}|`, @@ -108,6 +111,9 @@ const TEST_STRINGS = { FILTER_OPERATOR_12: `{{ messages | rejectattr('role', 'equalto', 'system') | length }}`, FILTER_OPERATOR_13: `{{ tools | string }}`, FILTER_OPERATOR_14: `|{{ "1" | int + 2 }}|{{ "invalid" | int }}|{{ "invalid" | int(-1) }}|{{ true | int }}|{{ false | int }}|{{ 1.5 | int }}|{{ "1.5" | float }}|{{ "invalid" | float }}|{{ "invalid" | float("hello") }}|`, + FILTER_OPERATOR_15: `|{{ "abcabcabc" | replace("a", "b") }}|{{ "abcabcabc" | replace("a", "b", 1) }}|{{ "abcabcabc" | replace("a", "b", count=1) }}|`, + FILTER_OPERATOR_16: `|{{ undefined | default("hello") }}|{{ false | default("hello") }}|{{ false | default("hello", true) }}|{{ 0 | default("hello", boolean=true) }}|`, + FILTER_OPERATOR_17: `{{ [1, 2, 1, -1, 2] | unique | list | length }}`, // Filter statements FILTER_STATEMENTS: `{% filter upper %}text{% endfilter %}`, @@ -686,6 +692,32 @@ const TEST_PARSED = { { value: "endfor", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, ], + FOR_LOOP_SELECT_2: [ + { value: "{%", type: "OpenStatement" }, + { value: "for", type: "Identifier" }, + { value: "x", type: "Identifier" }, + { value: "in", type: "Identifier" }, + { value: "arr", type: "Identifier" }, + { value: "|", type: "Pipe" }, + { value: "selectattr", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: "value", type: "StringLiteral" }, + { value: ",", type: "Comma" }, + { value: "equalto", type: "StringLiteral" }, + { value: ",", type: "Comma" }, + { value: "a", type: "StringLiteral" }, + { value: ")", type: "CloseParen" }, + { value: "%}", type: "CloseStatement" }, + { value: "{{", type: "OpenExpression" }, + { value: "x", type: "Identifier" }, + { value: "[", type: "OpenSquareBracket" }, + { value: "value", type: "StringLiteral" }, + { value: "]", type: "CloseSquareBracket" }, + { value: "}}", type: "CloseExpression" }, + { value: "{%", type: "OpenStatement" }, + { value: "endfor", type: "Identifier" }, + { value: "%}", type: "CloseStatement" }, + ], FOR_LOOP_BREAK: [ { value: "{%", type: "OpenStatement" }, { value: "for", type: "Identifier" }, @@ -754,6 +786,29 @@ const TEST_PARSED = { { value: "endfor", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, ], + FOR_LOOP_OBJECTS: [ + { value: "{%", type: "OpenStatement" }, + { value: "for", type: "Identifier" }, + { value: "x", type: "Identifier" }, + { value: "in", type: "Identifier" }, + { value: "obj", type: "Identifier" }, + { value: "%}", type: "CloseStatement" }, + { value: "{{", type: "OpenExpression" }, + { value: "x", type: "Identifier" }, + { value: "+", type: "AdditiveBinaryOperator" }, + { value: ":", type: "StringLiteral" }, + { value: "+", type: "AdditiveBinaryOperator" }, + { value: "obj", type: "Identifier" }, + { value: "[", type: "OpenSquareBracket" }, + { value: "x", type: "Identifier" }, + { value: "]", type: "CloseSquareBracket" }, + { value: "+", type: "AdditiveBinaryOperator" }, + { value: ";", type: "StringLiteral" }, + { value: "}}", type: "CloseExpression" }, + { value: "{%", type: "OpenStatement" }, + { value: "endfor", type: "Identifier" }, + { value: "%}", type: "CloseStatement" }, + ], // Set variables VARIABLES: [ @@ -1372,6 +1427,61 @@ const TEST_PARSED = { { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, ], + REPLACE: [ + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "test test", type: "StringLiteral" }, + { value: ".", type: "Dot" }, + { value: "replace", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: "test", type: "StringLiteral" }, + { value: ",", type: "Comma" }, + { value: "TEST", type: "StringLiteral" }, + { value: ")", type: "CloseParen" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "test test", type: "StringLiteral" }, + { value: ".", type: "Dot" }, + { value: "replace", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: "test", type: "StringLiteral" }, + { value: ",", type: "Comma" }, + { value: "TEST", type: "StringLiteral" }, + { value: ",", type: "Comma" }, + { value: "1", type: "NumericLiteral" }, + { value: ")", type: "CloseParen" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "test test", type: "StringLiteral" }, + { value: ".", type: "Dot" }, + { value: "replace", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: "", type: "StringLiteral" }, + { value: ",", type: "Comma" }, + { value: "_", type: "StringLiteral" }, + { value: ",", type: "Comma" }, + { value: "2", type: "NumericLiteral" }, + { value: ")", type: "CloseParen" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "abcabc", type: "StringLiteral" }, + { value: ".", type: "Dot" }, + { value: "replace", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: "a", type: "StringLiteral" }, + { value: ",", type: "Comma" }, + { value: "x", type: "StringLiteral" }, + { value: ",", type: "Comma" }, + { value: "count", type: "Identifier" }, + { value: "=", type: "Equals" }, + { value: "1", type: "NumericLiteral" }, + { value: ")", type: "CloseParen" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + ], // String indexing and slicing STRING_SLICING: [ @@ -2171,6 +2281,114 @@ const TEST_PARSED = { { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, ], + FILTER_OPERATOR_15: [ + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "abcabcabc", type: "StringLiteral" }, + { value: "|", type: "Pipe" }, + { value: "replace", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: "a", type: "StringLiteral" }, + { value: ",", type: "Comma" }, + { value: "b", type: "StringLiteral" }, + { value: ")", type: "CloseParen" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "abcabcabc", type: "StringLiteral" }, + { value: "|", type: "Pipe" }, + { value: "replace", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: "a", type: "StringLiteral" }, + { value: ",", type: "Comma" }, + { value: "b", type: "StringLiteral" }, + { value: ",", type: "Comma" }, + { value: "1", type: "NumericLiteral" }, + { value: ")", type: "CloseParen" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "abcabcabc", type: "StringLiteral" }, + { value: "|", type: "Pipe" }, + { value: "replace", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: "a", type: "StringLiteral" }, + { value: ",", type: "Comma" }, + { value: "b", type: "StringLiteral" }, + { value: ",", type: "Comma" }, + { value: "count", type: "Identifier" }, + { value: "=", type: "Equals" }, + { value: "1", type: "NumericLiteral" }, + { value: ")", type: "CloseParen" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + ], + FILTER_OPERATOR_16: [ + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "undefined", type: "Identifier" }, + { value: "|", type: "Pipe" }, + { value: "default", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: "hello", type: "StringLiteral" }, + { value: ")", type: "CloseParen" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "false", type: "Identifier" }, + { value: "|", type: "Pipe" }, + { value: "default", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: "hello", type: "StringLiteral" }, + { value: ")", type: "CloseParen" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "false", type: "Identifier" }, + { value: "|", type: "Pipe" }, + { value: "default", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: "hello", type: "StringLiteral" }, + { value: ",", type: "Comma" }, + { value: "true", type: "Identifier" }, + { value: ")", type: "CloseParen" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "0", type: "NumericLiteral" }, + { value: "|", type: "Pipe" }, + { value: "default", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: "hello", type: "StringLiteral" }, + { value: ",", type: "Comma" }, + { value: "boolean", type: "Identifier" }, + { value: "=", type: "Equals" }, + { value: "true", type: "Identifier" }, + { value: ")", type: "CloseParen" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + ], + FILTER_OPERATOR_17: [ + { value: "{{", type: "OpenExpression" }, + { value: "[", type: "OpenSquareBracket" }, + { value: "1", type: "NumericLiteral" }, + { value: ",", type: "Comma" }, + { value: "2", type: "NumericLiteral" }, + { value: ",", type: "Comma" }, + { value: "1", type: "NumericLiteral" }, + { value: ",", type: "Comma" }, + { value: "-1", type: "NumericLiteral" }, + { value: ",", type: "Comma" }, + { value: "2", type: "NumericLiteral" }, + { value: "]", type: "CloseSquareBracket" }, + { value: "|", type: "Pipe" }, + { value: "unique", type: "Identifier" }, + { value: "|", type: "Pipe" }, + { value: "list", type: "Identifier" }, + { value: "|", type: "Pipe" }, + { value: "length", type: "Identifier" }, + { value: "}}", type: "CloseExpression" }, + ], // Filter statements FILTER_STATEMENTS: [ @@ -3630,8 +3848,18 @@ const TEST_CONTEXT = { FOR_LOOP_UNPACKING: {}, FOR_LOOP_DEFAULT: {}, FOR_LOOP_SELECT: {}, + FOR_LOOP_SELECT_2: { + arr: [{ value: "a" }, { value: "b" }, { value: "c" }, { value: "a" }], + }, FOR_LOOP_BREAK: {}, FOR_LOOP_CONTINUE: {}, + FOR_LOOP_OBJECTS: { + obj: { + a: 1, + b: 2, + c: 3, + }, + }, // Set variables VARIABLES: {}, @@ -3692,6 +3920,7 @@ const TEST_CONTEXT = { SPLIT_3: {}, SPLIT_4: {}, SPLIT_5: {}, + REPLACE: {}, // String indexing and slicing STRING_SLICING: { @@ -3788,6 +4017,9 @@ const TEST_CONTEXT = { tools: [{ name: "some_tool", arguments: { some_name: "string" } }], }, FILTER_OPERATOR_14: {}, + FILTER_OPERATOR_15: {}, + FILTER_OPERATOR_16: {}, + FILTER_OPERATOR_17: {}, // Filter statements FILTER_STATEMENTS: {}, @@ -3924,8 +4156,10 @@ const EXPECTED_OUTPUTS = { FOR_LOOP_UNPACKING: "||1 2||3 4||", FOR_LOOP_DEFAULT: "B", FOR_LOOP_SELECT: "34", + FOR_LOOP_SELECT_2: "aa", FOR_LOOP_BREAK: "12", FOR_LOOP_CONTINUE: "124", + FOR_LOOP_OBJECTS: "a:1;b:2;c:3;", // Set variables VARIABLES: "Hello World", @@ -3964,6 +4198,7 @@ const EXPECTED_OUTPUTS = { SPLIT_3: `||||test|it |`, SPLIT_4: `|["1", "2", "3"]|["", "a", "", "acca", "", "", ""]|["", "a", "baccabbb"]|`, SPLIT_5: `|1 2 3 4 5 |1,2,3,4 5 | 1 2 3 4 5 |,1,2,3 4 5 |,1,2,3,4,5,|`, + REPLACE: `|TEST TEST|TEST test|_t_est test|xbcabc|`, // String indexing and slicing STRING_SLICING: "|0|0123456789|012|123|12345678|13579|543210|", @@ -3998,6 +4233,9 @@ const EXPECTED_OUTPUTS = { FILTER_OPERATOR_12: `2`, FILTER_OPERATOR_13: `[{"name": "some_tool", "arguments": {"some_name": "string"}}]`, FILTER_OPERATOR_14: `|3|0|-1|1|0|1|1.5|0.0|hello|`, + FILTER_OPERATOR_15: `|bbcbbcbbc|bbcabcabc|bbcabcabc|`, + FILTER_OPERATOR_16: `|hello|false|hello|hello|`, + FILTER_OPERATOR_17: `3`, // Filter statements FILTER_STATEMENTS: `TEXT`, diff --git a/packages/jinja/test/utils.test.js b/packages/jinja/test/utils.test.js index be271c1df6..3abced623c 100644 --- a/packages/jinja/test/utils.test.js +++ b/packages/jinja/test/utils.test.js @@ -1,6 +1,6 @@ import { describe, it, expect } from "vitest"; -import { slice } from "../src/utils"; +import { slice, strftime, replace } from "../src/utils"; describe("Test utility functions", () => { describe("Slice function", () => { @@ -70,4 +70,63 @@ describe("Test utility functions", () => { expect(slice(array, -3, undefined, -1)).toEqual([7, 6, 5, 4, 3, 2, 1, 0]); }); }); + + describe("strftime function", () => { + const date = new Date(2025, 5, 4, 12, 34, 56); + const leapDate = new Date(2000, 1, 29, 0, 0, 0, 0); + + it('should format "%d %b %Y"', () => { + expect(strftime(date, "%d %b %Y")).toEqual("04 Jun 2025"); + expect(strftime(leapDate, "%d %b %Y")).toEqual("29 Feb 2000"); + }); + + it('should format "%Y-%m-%d"', () => { + expect(strftime(date, "%Y-%m-%d")).toEqual("2025-06-04"); + expect(strftime(leapDate, "%Y-%m-%d")).toEqual("2000-02-29"); + }); + + it("should format '%B %d, %Y'", () => { + expect(strftime(date, "%B %d, %Y")).toEqual("June 04, 2025"); + expect(strftime(leapDate, "%B %d, %Y")).toEqual("February 29, 2000"); + }); + }); + + describe("replace function", () => { + it("replaces all occurrences when count is not specified", () => { + expect(replace("one one one", "one", "two")).toEqual("two two two"); + expect(replace("aaaa", "a", "b")).toEqual("bbbb"); + }); + + it("replaces up to count occurrences when count is provided", () => { + expect(replace("spam spam spam", "spam", "eggs", 2)).toEqual("eggs eggs spam"); + expect(replace("abcabcabc", "abc", "x", 0)).toEqual("abcabcabc"); + }); + + it("removes occurrences when replacement is empty", () => { + expect(replace("foo bar foo", "foo", "")).toEqual(" bar "); + expect(replace("banana", "a", "")).toEqual("bnn"); + }); + + it("returns original when old substring not found", () => { + expect(replace("hello world", "xyz", "123")).toEqual("hello world"); + }); + + it("handles overlapping patterns like Python does", () => { + // Python.replace treats non-overlapping matches: "aaaa".replace("aa","b") => "bb" + expect(replace("aaaa", "aa", "b")).toEqual("bb"); + }); + + it("handles when old substring is empty", () => { + expect(replace("abc", "", "x")).toEqual("xaxbxcx"); + expect(replace("abc", "", "x", 0)).toEqual("abc"); + expect(replace("abc", "", "x", 2)).toEqual("xaxbc"); + expect(replace("abc", "", "x", 100)).toEqual("xaxbxcx"); + }); + + it("handles multi-byte characters", () => { + const str = "πŸ€—testπŸ€—"; // \ud83e\udd17 + expect(replace(str, "test", "πŸ€—")).toEqual("πŸ€—πŸ€—πŸ€—"); + expect(replace(str, "\ud83e", "X")).toEqual(str); // No replacement + }); + }); }); From 4e34ab4ebf3aa8a984f60caa5dc766c32b0dd6b9 Mon Sep 17 00:00:00 2001 From: Joshua Lochner <26504141+xenova@users.noreply.github.com> Date: Sun, 4 May 2025 22:49:01 -0400 Subject: [PATCH 28/35] Fix 'or' precendence --- packages/jinja/src/format.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/jinja/src/format.ts b/packages/jinja/src/format.ts index 3533ccdcd3..63628c1c46 100644 --- a/packages/jinja/src/format.ts +++ b/packages/jinja/src/format.ts @@ -217,7 +217,10 @@ function formatExpression(node: Expression, parentPrec: number = -1): string { return JSON.stringify((node as StringLiteral).value); case "BinaryExpression": { const n = node as BinaryExpression; - const thisPrecedence = OPERATOR_PRECEDENCE[n.operator.type] ?? 0; + let thisPrecedence = OPERATOR_PRECEDENCE[n.operator.type] ?? 0; + if (n.operator.value === "or") { + thisPrecedence = -1; + } const left = formatExpression(n.left, thisPrecedence); const right = formatExpression(n.right, thisPrecedence + 1); const expr = `${left} ${n.operator.value} ${right}`; From e056139ec866f8321c72c61cfd2e7bef12bd1625 Mon Sep 17 00:00:00 2001 From: Joshua Lochner <26504141+xenova@users.noreply.github.com> Date: Sun, 4 May 2025 22:58:27 -0400 Subject: [PATCH 29/35] Improve formatting of chained property accesses --- packages/jinja/src/format.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/jinja/src/format.ts b/packages/jinja/src/format.ts index 63628c1c46..2346f37057 100644 --- a/packages/jinja/src/format.ts +++ b/packages/jinja/src/format.ts @@ -241,7 +241,8 @@ function formatExpression(node: Expression, parentPrec: number = -1): string { case "MemberExpression": { const n = node as MemberExpression; let obj = formatExpression(n.object, -1); - if (n.object.type !== "Identifier") { + // only wrap if it's not a simple or chained access/call + if (!["Identifier", "MemberExpression", "CallExpression"].includes(n.object.type)) { obj = `(${obj})`; } let prop = formatExpression(n.property, -1); From 87681d9e84e49a2509c2cb40519918ea21a67809 Mon Sep 17 00:00:00 2001 From: Joshua Lochner <26504141+xenova@users.noreply.github.com> Date: Mon, 5 May 2025 14:00:14 -0400 Subject: [PATCH 30/35] Add formatting unit tests --- packages/jinja/test/e2e.test.js | 53 +++++++++++++++++++++++++++--- packages/jinja/test/format.test.js | 30 +++++++++++++++++ 2 files changed, 79 insertions(+), 4 deletions(-) create mode 100644 packages/jinja/test/format.test.js diff --git a/packages/jinja/test/e2e.test.js b/packages/jinja/test/e2e.test.js index 44747af88c..2f2f0607be 100644 --- a/packages/jinja/test/e2e.test.js +++ b/packages/jinja/test/e2e.test.js @@ -967,6 +967,37 @@ const TEST_CUSTOM_TEMPLATES = Object.freeze({ }, }); +/** + * Formatting tests for custom templates. + */ +const TEST_CUSTOM_FORMATTING = Object.freeze({ + "HuggingFaceTB/SmolVLM-Instruct": { + //https://huggingface.co/HuggingFaceTB/SmolVLM-Instruct?chat_template=default + chat_template: `<|im_start|>{% for message in messages %}{{message['role'] | capitalize}}{% if message['content'][0]['type'] == 'image' %}{{':'}}{% else %}{{': '}}{% endif %}{% for line in message['content'] %}{% if line['type'] == 'text' %}{{line['text']}}{% elif line['type'] == 'image' %}{{ '' }}{% endif %}{% endfor %} +{% endfor %}{% if add_generation_prompt %}{{ 'Assistant:' }}{% endif %}`, + target: `{{- "<|im_start|>" -}} +{%- for message in messages -%} + {{- message["role"] | capitalize -}} + {%- if message["content"][0]["type"] == "image" -%} + {{- ":" -}} + {%- else -%} + {{- ": " -}} + {%- endif -%} + {%- for line in message["content"] -%} + {%- if line["type"] == "text" -%} + {{- line["text"] -}} + {%- elif line["type"] == "image" -%} + {{- "" -}} + {%- endif -%} + {%- endfor -%} + {{- "\\n" -}} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{- "Assistant:" -}} +{%- endif -%}`, + }, +}); + function render({ chat_template, data, target }) { const template = new Template(chat_template); const result = template.render(data); @@ -978,23 +1009,37 @@ function render({ chat_template, data, target }) { expect(formattedTemplateOutput).toEqual(result); } +function format({ chat_template, target }) { + const template = new Template(chat_template); + const formatted = template.format({ indent: 4 }); + expect(formatted).toEqual(target); +} + describe("End-to-end tests", () => { - describe("Default templates", async () => { + describe("Default templates", () => { for (const [model_type, test_data] of Object.entries(TEST_DEFAULT_TEMPLATES)) { - it(model_type, async () => { + it(model_type, () => { render(test_data); }); } }); - describe("Custom templates", async () => { + describe("Custom templates", () => { for (const [model_type, test_data] of Object.entries(TEST_CUSTOM_TEMPLATES)) { - it(model_type, async () => { + it(model_type, () => { render(test_data); }); } }); + describe("Custom formatting", () => { + for (const [model_type, test_data] of Object.entries(TEST_CUSTOM_FORMATTING)) { + it(model_type, () => { + format(test_data); + }); + } + }); + it("should parse a chat template from the Hugging Face Hub", async () => { const repo = "TheBloke/Mistral-7B-Instruct-v0.1-GPTQ"; const tokenizerConfig = await ( diff --git a/packages/jinja/test/format.test.js b/packages/jinja/test/format.test.js new file mode 100644 index 0000000000..9916f7e96e --- /dev/null +++ b/packages/jinja/test/format.test.js @@ -0,0 +1,30 @@ +import { describe, expect, it } from "vitest"; +import { Template } from "../src/index"; + +/** + * Tests for formatting templates. + * + * This section defines our (opinionated) style guide for formatting templates. + */ +const FORMATTING_TESTS = Object.freeze({ + OPERATOR_PRECEDENCE: { + template: `{{ "hello" if 1 + 2 * 3 / 4 - 5 == 0 or 1 + 2 * 3 / 4 - 5 == 0 and 1 + 2 * 3 / 4 - 5 == 0 else "bye" }}`, + target: `{{- "hello" if 1 + 2 * 3 / 4 - 5 == 0 or 1 + 2 * 3 / 4 - 5 == 0 and 1 + 2 * 3 / 4 - 5 == 0 else "bye" -}}`, + }, + CHAINED_PROPERTY_ACCESSES: { + template: `{{ message.content.split('')[0].rstrip('\\n').split('')[-1].lstrip('\\n') }}`, + target: `{{- message.content.split("")[0].rstrip("\\n").split("")[-1].lstrip("\\n") -}}`, + }, +}); + +describe("format", () => { + for (const [name, test] of Object.entries(FORMATTING_TESTS)) { + it(`should format ${name}`, () => { + const template = new Template(test.template); + const result = template.format({ + indent: 4, + }); + expect(result).toEqual(test.target); + }); + } +}); From 0ab1ae6fe9d58ad2f5465e483f9b6b81da2a9efe Mon Sep 17 00:00:00 2001 From: Joshua Lochner <26504141+xenova@users.noreply.github.com> Date: Mon, 5 May 2025 14:01:18 -0400 Subject: [PATCH 31/35] Improve formatting --- packages/jinja/src/format.ts | 65 +++++++++++++++++++----------------- 1 file changed, 34 insertions(+), 31 deletions(-) diff --git a/packages/jinja/src/format.ts b/packages/jinja/src/format.ts index 2346f37057..aa11e636aa 100644 --- a/packages/jinja/src/format.ts +++ b/packages/jinja/src/format.ts @@ -34,12 +34,19 @@ const NEWLINE = "\n"; const OPEN_STATEMENT = "{%- "; const CLOSE_STATEMENT = " -%}"; -const OPERATOR_PRECEDENCE: Record = { - MultiplicativeBinaryOperator: 2, - AdditiveBinaryOperator: 1, - ComparisonBinaryOperator: 0, - Ternary: -1, -}; +function getBinaryOperatorPrecedence(expr: BinaryExpression): number { + switch (expr.operator.type) { + case "MultiplicativeBinaryOperator": + return 4; + case "AdditiveBinaryOperator": + return 3; + case "ComparisonBinaryOperator": + return 2; + case "Identifier": + return expr.operator.value === "and" ? 1 : 0; + } + return 0; +} export function format(program: Program, indent: string | number = "\t"): string { const indentStr = typeof indent === "number" ? " ".repeat(indent) : indent; @@ -182,7 +189,7 @@ function formatCallStatement(node: CallStatement, depth: number, indentStr: stri const pad = indentStr.repeat(depth); const params = node.callerArgs && node.callerArgs.length > 0 ? `(${node.callerArgs.map(formatExpression).join(", ")})` : ""; - const callExpr = formatExpression(node.call, -1); + const callExpr = formatExpression(node.call); let out = pad + createStatement(`call${params}`, callExpr) + NEWLINE; out += formatStatements(node.body, depth + 1, indentStr) + NEWLINE; out += pad + createStatement("endcall"); @@ -194,7 +201,7 @@ function formatFilterStatement(node: FilterStatement, depth: number, indentStr: const spec = node.filter.type === "Identifier" ? (node.filter as Identifier).value - : formatExpression(node.filter as CallExpression, -1); + : formatExpression(node.filter as CallExpression); let out = pad + createStatement("filter", spec) + NEWLINE; out += formatStatements(node.body, depth + 1, indentStr) + NEWLINE; out += pad + createStatement("endfilter"); @@ -205,7 +212,7 @@ function formatExpression(node: Expression, parentPrec: number = -1): string { switch (node.type) { case "SpreadExpression": { const n = node as SpreadExpression; - return `*${formatExpression(n.argument, -1)}`; + return `*${formatExpression(n.argument)}`; } case "Identifier": return (node as Identifier).value; @@ -217,10 +224,7 @@ function formatExpression(node: Expression, parentPrec: number = -1): string { return JSON.stringify((node as StringLiteral).value); case "BinaryExpression": { const n = node as BinaryExpression; - let thisPrecedence = OPERATOR_PRECEDENCE[n.operator.type] ?? 0; - if (n.operator.value === "or") { - thisPrecedence = -1; - } + const thisPrecedence = getBinaryOperatorPrecedence(n); const left = formatExpression(n.left, thisPrecedence); const right = formatExpression(n.right, thisPrecedence + 1); const expr = `${left} ${n.operator.value} ${right}`; @@ -235,17 +239,17 @@ function formatExpression(node: Expression, parentPrec: number = -1): string { return `not ${formatExpression((node as LogicalNegationExpression).argument, Infinity)}`; case "CallExpression": { const n = node as CallExpression; - const args = n.args.map((a) => formatExpression(a, -1)).join(", "); - return `${formatExpression(n.callee, -1)}(${args})`; + const args = n.args.map(formatExpression).join(", "); + return `${formatExpression(n.callee)}(${args})`; } case "MemberExpression": { const n = node as MemberExpression; - let obj = formatExpression(n.object, -1); + let obj = formatExpression(n.object); // only wrap if it's not a simple or chained access/call if (!["Identifier", "MemberExpression", "CallExpression"].includes(n.object.type)) { obj = `(${obj})`; } - let prop = formatExpression(n.property, -1); + let prop = formatExpression(n.property); if (!n.computed && n.property.type !== "Identifier") { prop = `(${prop})`; } @@ -255,48 +259,47 @@ function formatExpression(node: Expression, parentPrec: number = -1): string { const n = node as FilterExpression; const operand = formatExpression(n.operand, Infinity); if (n.filter.type === "CallExpression") { - return `${operand} | ${formatExpression(n.filter, -1)}`; + return `${operand} | ${formatExpression(n.filter)}`; } return `${operand} | ${(n.filter as Identifier).value}`; } case "SelectExpression": { const n = node as SelectExpression; - return `${formatExpression(n.lhs, -1)} if ${formatExpression(n.test, -1)}`; + return `${formatExpression(n.lhs)} if ${formatExpression(n.test)}`; } case "TestExpression": { const n = node as TestExpression; - return `${formatExpression(n.operand, -1)} is${n.negate ? " not" : ""} ${n.test.value}`; + return `${formatExpression(n.operand)} is${n.negate ? " not" : ""} ${n.test.value}`; } case "ArrayLiteral": case "TupleLiteral": { - const elems = ((node as ArrayLiteral | TupleLiteral).value as Expression[]).map((e) => formatExpression(e, -1)); + const elems = ((node as ArrayLiteral | TupleLiteral).value as Expression[]).map(formatExpression); const brackets = node.type === "ArrayLiteral" ? "[]" : "()"; return `${brackets[0]}${elems.join(", ")}${brackets[1]}`; } case "ObjectLiteral": { const entries = Array.from((node as ObjectLiteral).value.entries()).map( - ([k, v]) => `${formatExpression(k, -1)}: ${formatExpression(v, -1)}` + ([k, v]) => `${formatExpression(k)}: ${formatExpression(v)}` ); return `{ ${entries.join(", ")} }`; } case "SliceExpression": { const n = node as SliceExpression; - const s = n.start ? formatExpression(n.start, -1) : ""; - const t = n.stop ? formatExpression(n.stop, -1) : ""; - const st = n.step ? `:${formatExpression(n.step, -1)}` : ""; + const s = n.start ? formatExpression(n.start) : ""; + const t = n.stop ? formatExpression(n.stop) : ""; + const st = n.step ? `:${formatExpression(n.step)}` : ""; return `${s}:${t}${st}`; } case "KeywordArgumentExpression": { const n = node as KeywordArgumentExpression; - return `${n.key.value}=${formatExpression(n.value, -1)}`; + return `${n.key.value}=${formatExpression(n.value)}`; } case "Ternary": { const n = node as Ternary; - const expr = `${formatExpression(n.trueExpr, OPERATOR_PRECEDENCE.Ternary)} if ${formatExpression( - n.condition, - OPERATOR_PRECEDENCE.Ternary - )} else ${formatExpression(n.falseExpr, OPERATOR_PRECEDENCE.Ternary)}`; - return OPERATOR_PRECEDENCE.Ternary < parentPrec ? `(${expr})` : expr; + const expr = `${formatExpression(n.trueExpr)} if ${formatExpression(n.condition)} else ${formatExpression( + n.falseExpr + )}`; + return parentPrec > -1 ? `(${expr})` : expr; } default: throw new Error(`Unknown expression type: ${node.type}`); From a4255636c317f524d16cc6f034491fcaec41d379 Mon Sep 17 00:00:00 2001 From: Joshua Lochner <26504141+xenova@users.noreply.github.com> Date: Mon, 5 May 2025 14:27:11 -0400 Subject: [PATCH 32/35] Improve object & membership formatting --- packages/jinja/src/format.ts | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/packages/jinja/src/format.ts b/packages/jinja/src/format.ts index aa11e636aa..ff4a4fd167 100644 --- a/packages/jinja/src/format.ts +++ b/packages/jinja/src/format.ts @@ -246,7 +246,17 @@ function formatExpression(node: Expression, parentPrec: number = -1): string { const n = node as MemberExpression; let obj = formatExpression(n.object); // only wrap if it's not a simple or chained access/call - if (!["Identifier", "MemberExpression", "CallExpression"].includes(n.object.type)) { + if (![ + "Identifier", + "MemberExpression", + "CallExpression", + "StringLiteral", + "IntegerLiteral", + "FloatLiteral", + "ArrayLiteral", + "TupleLiteral", + "ObjectLiteral", + ].includes(n.object.type)) { obj = `(${obj})`; } let prop = formatExpression(n.property); @@ -281,7 +291,7 @@ function formatExpression(node: Expression, parentPrec: number = -1): string { const entries = Array.from((node as ObjectLiteral).value.entries()).map( ([k, v]) => `${formatExpression(k)}: ${formatExpression(v)}` ); - return `{ ${entries.join(", ")} }`; + return `{${entries.join(", ")}}`; } case "SliceExpression": { const n = node as SliceExpression; From 9804619fa2d95fe0b2ee8aea9bc2f48ff8bfd757 Mon Sep 17 00:00:00 2001 From: Joshua Lochner <26504141+xenova@users.noreply.github.com> Date: Mon, 5 May 2025 15:46:58 -0400 Subject: [PATCH 33/35] Improve formatting + add new tests --- packages/jinja/src/ast.ts | 11 -------- packages/jinja/src/format.ts | 33 +++++++++++------------ packages/jinja/test/format.test.js | 42 +++++++++++++++++++++++++++--- 3 files changed, 56 insertions(+), 30 deletions(-) diff --git a/packages/jinja/src/ast.ts b/packages/jinja/src/ast.ts index 3dab02b90f..c0515d2be8 100644 --- a/packages/jinja/src/ast.ts +++ b/packages/jinja/src/ast.ts @@ -265,17 +265,6 @@ export class UnaryExpression extends Expression { } } -/** - * Logical negation of an expression. - */ -export class LogicalNegationExpression extends Expression { - override type = "LogicalNegationExpression"; - - constructor(public argument: Expression) { - super(); - } -} - export class SliceExpression extends Expression { override type = "SliceExpression"; diff --git a/packages/jinja/src/format.ts b/packages/jinja/src/format.ts index ff4a4fd167..5dfefaee83 100644 --- a/packages/jinja/src/format.ts +++ b/packages/jinja/src/format.ts @@ -21,7 +21,6 @@ import type { SelectExpression, TestExpression, UnaryExpression, - LogicalNegationExpression, SliceExpression, KeywordArgumentExpression, CallStatement, @@ -43,7 +42,9 @@ function getBinaryOperatorPrecedence(expr: BinaryExpression): number { case "ComparisonBinaryOperator": return 2; case "Identifier": - return expr.operator.value === "and" ? 1 : 0; + if (expr.operator.value === "and") return 1; + if (expr.operator.value === "in" || expr.operator.value === "not in") return 2; + return 0; } return 0; } @@ -235,8 +236,6 @@ function formatExpression(node: Expression, parentPrec: number = -1): string { const val = n.operator.value + (n.operator.value === "not" ? " " : "") + formatExpression(n.argument, Infinity); return val; } - case "LogicalNegationExpression": - return `not ${formatExpression((node as LogicalNegationExpression).argument, Infinity)}`; case "CallExpression": { const n = node as CallExpression; const args = n.args.map(formatExpression).join(", "); @@ -246,17 +245,19 @@ function formatExpression(node: Expression, parentPrec: number = -1): string { const n = node as MemberExpression; let obj = formatExpression(n.object); // only wrap if it's not a simple or chained access/call - if (![ - "Identifier", - "MemberExpression", - "CallExpression", - "StringLiteral", - "IntegerLiteral", - "FloatLiteral", - "ArrayLiteral", - "TupleLiteral", - "ObjectLiteral", - ].includes(n.object.type)) { + if ( + ![ + "Identifier", + "MemberExpression", + "CallExpression", + "StringLiteral", + "IntegerLiteral", + "FloatLiteral", + "ArrayLiteral", + "TupleLiteral", + "ObjectLiteral", + ].includes(n.object.type) + ) { obj = `(${obj})`; } let prop = formatExpression(n.property); @@ -306,7 +307,7 @@ function formatExpression(node: Expression, parentPrec: number = -1): string { } case "Ternary": { const n = node as Ternary; - const expr = `${formatExpression(n.trueExpr)} if ${formatExpression(n.condition)} else ${formatExpression( + const expr = `${formatExpression(n.trueExpr)} if ${formatExpression(n.condition, 0)} else ${formatExpression( n.falseExpr )}`; return parentPrec > -1 ? `(${expr})` : expr; diff --git a/packages/jinja/test/format.test.js b/packages/jinja/test/format.test.js index 9916f7e96e..349b0e95d2 100644 --- a/packages/jinja/test/format.test.js +++ b/packages/jinja/test/format.test.js @@ -9,11 +9,39 @@ import { Template } from "../src/index"; const FORMATTING_TESTS = Object.freeze({ OPERATOR_PRECEDENCE: { template: `{{ "hello" if 1 + 2 * 3 / 4 - 5 == 0 or 1 + 2 * 3 / 4 - 5 == 0 and 1 + 2 * 3 / 4 - 5 == 0 else "bye" }}`, - target: `{{- "hello" if 1 + 2 * 3 / 4 - 5 == 0 or 1 + 2 * 3 / 4 - 5 == 0 and 1 + 2 * 3 / 4 - 5 == 0 else "bye" -}}`, + formatted: `{{- "hello" if 1 + 2 * 3 / 4 - 5 == 0 or 1 + 2 * 3 / 4 - 5 == 0 and 1 + 2 * 3 / 4 - 5 == 0 else "bye" -}}`, + rendered: `bye`, }, CHAINED_PROPERTY_ACCESSES: { template: `{{ message.content.split('')[0].rstrip('\\n').split('')[-1].lstrip('\\n') }}`, - target: `{{- message.content.split("")[0].rstrip("\\n").split("")[-1].lstrip("\\n") -}}`, + formatted: `{{- message.content.split("")[0].rstrip("\\n").split("")[-1].lstrip("\\n") -}}`, + }, + MEMBERSHIP: { + template: `{{ 'a' in 'abc' and False }}`, + formatted: `{{- "a" in "abc" and False -}}`, + rendered: `false`, + }, + FILTERS: { + template: `{{'a' + {'b': 'b'}['b']| upper + 'c'}}`, + formatted: `{{- "a" + {"b": "b"}["b"] | upper + "c" -}}`, + rendered: `aBc`, + }, + UNARY_BINARY_MIXED: { + template: `{{not 1 not in [1,2,3] and not 1==2 or 1!=2}}`, + // Technically minimal (according to precedence rules) but less readable: + // formatted: `{{- not 1 not in [1, 2, 3] and not 1 == 2 or 1 != 2 -}}`, + formatted: `{{- not (1 not in [1, 2, 3]) and not (1 == 2) or 1 != 2 -}}`, + rendered: `true`, + }, + CHAINED_UNARY: { + template: `{{not not not 1}}`, + formatted: `{{- not not not 1 -}}`, + rendered: `false`, + }, + CHAINED_TERNARY: { + template: `{{('a' if (true if 1==2 else false) else 'b') if 3==4 else ('c' if 4==5 else 'd')}}`, + formatted: `{{- "a" if (true if 1 == 2 else false) else "b" if 3 == 4 else "c" if 4 == 5 else "d" -}}`, + rendered: `d`, }, }); @@ -24,7 +52,15 @@ describe("format", () => { const result = template.format({ indent: 4, }); - expect(result).toEqual(test.target); + expect(result).toEqual(test.formatted); + + if (test.rendered) { + const rendered = template.render(); + const formatted = new Template(test.formatted); + const formattedRendered = formatted.render(); + expect(rendered).toEqual(test.rendered); + expect(formattedRendered).toEqual(test.rendered); + } }); } }); From e3325d4172fd9d316ac04b8e9ca06845c3ce6b03 Mon Sep 17 00:00:00 2001 From: Joshua Lochner <26504141+xenova@users.noreply.github.com> Date: Mon, 5 May 2025 15:52:37 -0400 Subject: [PATCH 34/35] Lint & format --- packages/jinja/src/runtime.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/jinja/src/runtime.ts b/packages/jinja/src/runtime.ts index 93143d98e7..ab749b34d6 100644 --- a/packages/jinja/src/runtime.ts +++ b/packages/jinja/src/runtime.ts @@ -741,7 +741,7 @@ export class Interpreter { case "string": return new StringValue(toJSON(operand)); case "unique": { - const seen = new Set(); + const seen = new Set(); const output: AnyRuntimeValue[] = []; for (const item of operand.value) { if (!seen.has(item.value)) { @@ -761,7 +761,7 @@ export class Interpreter { case "upper": case "lower": case "title": - case "capitalize": + case "capitalize": { const builtin = operand.builtins.get(filter.value); if (builtin instanceof FunctionValue) { return builtin.value(/* no arguments */ [], environment); @@ -770,6 +770,7 @@ export class Interpreter { } else { throw new Error(`Unknown StringValue filter: ${filter.value}`); } + } case "trim": return new StringValue(operand.value.trim()); case "indent": From 1bb84ac9e682818f70b18207d9b19c12132a066e Mon Sep 17 00:00:00 2001 From: Joshua Lochner <26504141+xenova@users.noreply.github.com> Date: Mon, 5 May 2025 16:46:25 -0400 Subject: [PATCH 35/35] nit --- packages/jinja/src/format.ts | 2 +- packages/jinja/src/runtime.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/jinja/src/format.ts b/packages/jinja/src/format.ts index 5dfefaee83..0280d09af6 100644 --- a/packages/jinja/src/format.ts +++ b/packages/jinja/src/format.ts @@ -113,7 +113,7 @@ function formatIf(node: If, depth: number, indentStr: string): string { formatStatements(clauses[0].body, depth + 1, indentStr); // ELIF(s) - for (let i = 1; i < clauses.length; i++) { + for (let i = 1; i < clauses.length; ++i) { out += NEWLINE + pad + diff --git a/packages/jinja/src/runtime.ts b/packages/jinja/src/runtime.ts index ab749b34d6..cd59167e39 100644 --- a/packages/jinja/src/runtime.ts +++ b/packages/jinja/src/runtime.ts @@ -1187,7 +1187,7 @@ export class Interpreter { if (arr.length !== tuple.value.length) { throw new Error(`Too ${tuple.value.length > arr.length ? "few" : "many"} items to unpack in set`); } - for (let i = 0; i < tuple.value.length; i++) { + for (let i = 0; i < tuple.value.length; ++i) { const elem = tuple.value[i]; if (elem.type !== "Identifier") { throw new Error(`Cannot unpack to non-identifier in set: ${elem.type}`);