diff --git a/packages/jinja/src/ast.ts b/packages/jinja/src/ast.ts index e7d12d498..c0515d2be 100644 --- a/packages/jinja/src/ast.ts +++ b/packages/jinja/src/ast.ts @@ -77,6 +77,13 @@ export class Macro extends Statement { } } +export class Comment extends Statement { + override type = "Comment"; + constructor(public value: string) { + super(); + } +} + /** * Expressions will result in a value at runtime (unlike statements). */ @@ -133,11 +140,12 @@ abstract class Literal extends Expression { } } -/** - * Represents a numeric constant in the template. - */ -export class NumericLiteral extends Literal { - override type = "NumericLiteral"; +export class IntegerLiteral extends Literal { + override type = "IntegerLiteral"; +} + +export class FloatLiteral extends Literal { + override type = "FloatLiteral"; } /** @@ -147,20 +155,6 @@ export class StringLiteral extends Literal { override type = "StringLiteral"; } -/** - * Represents a boolean constant in the template. - */ -export class BooleanLiteral extends Literal { - override type = "BooleanLiteral"; -} - -/** - * Represents null (none) in the template. - */ -export class NullLiteral extends Literal { - override type = "NullLiteral"; -} - /** * Represents an array literal in the template. */ @@ -214,15 +208,28 @@ export class FilterExpression extends Expression { } } +export class FilterStatement extends Statement { + override type = "FilterStatement"; + + constructor( + public filter: Identifier | CallExpression, + public body: Statement[] + ) { + super(); + } +} + /** * An operation which filters a sequence of objects by applying a test to each object, * and only selecting the objects with the test succeeding. + * + * It may also be used as a shortcut for a ternary operator. */ export class SelectExpression extends Expression { override type = "SelectExpression"; constructor( - public iterable: Expression, + public lhs: Expression, public test: Expression ) { super(); @@ -258,17 +265,6 @@ export class UnaryExpression extends Expression { } } -/** - * Logical negation of an expression. - */ -export class LogicalNegationExpression extends Expression { - override type = "LogicalNegationExpression"; - - constructor(public argument: Expression) { - super(); - } -} - export class SliceExpression extends Expression { override type = "SliceExpression"; @@ -291,3 +287,34 @@ export class KeywordArgumentExpression extends Expression { super(); } } + +export class SpreadExpression extends Expression { + override type = "SpreadExpression"; + + constructor(public argument: Expression) { + super(); + } +} + +export class CallStatement extends Statement { + override type = "CallStatement"; + + constructor( + public call: CallExpression, + public callerArgs: Expression[] | null, + public body: Statement[] + ) { + super(); + } +} + +export class Ternary extends Expression { + override type = "Ternary"; + constructor( + public condition: Expression, + public trueExpr: Expression, + public falseExpr: Expression + ) { + super(); + } +} diff --git a/packages/jinja/src/format.ts b/packages/jinja/src/format.ts index 75260c138..0280d09af 100644 --- a/packages/jinja/src/format.ts +++ b/packages/jinja/src/format.ts @@ -1,6 +1,7 @@ import type { Program, Statement, + Comment, If, For, SetStatement, @@ -9,9 +10,9 @@ import type { MemberExpression, CallExpression, Identifier, - NumericLiteral, + FloatLiteral, + IntegerLiteral, StringLiteral, - BooleanLiteral, ArrayLiteral, TupleLiteral, ObjectLiteral, @@ -20,20 +21,33 @@ import type { SelectExpression, TestExpression, UnaryExpression, - LogicalNegationExpression, SliceExpression, KeywordArgumentExpression, + CallStatement, + FilterStatement, + SpreadExpression, + Ternary, } from "./ast"; const NEWLINE = "\n"; const OPEN_STATEMENT = "{%- "; const CLOSE_STATEMENT = " -%}"; -const OPERATOR_PRECEDENCE: Record = { - MultiplicativeBinaryOperator: 2, - AdditiveBinaryOperator: 1, - ComparisonBinaryOperator: 0, -}; +function getBinaryOperatorPrecedence(expr: BinaryExpression): number { + switch (expr.operator.type) { + case "MultiplicativeBinaryOperator": + return 4; + case "AdditiveBinaryOperator": + return 3; + case "ComparisonBinaryOperator": + return 2; + case "Identifier": + if (expr.operator.value === "and") return 1; + if (expr.operator.value === "in" || expr.operator.value === "not in") return 2; + return 0; + } + return 0; +} export function format(program: Program, indent: string | number = "\t"): string { const indentStr = typeof indent === "number" ? " ".repeat(indent) : indent; @@ -66,6 +80,12 @@ function formatStatement(node: Statement, depth: number, indentStr: string): str return pad + createStatement("break"); case "Continue": return pad + createStatement("continue"); + case "CallStatement": + return formatCallStatement(node as CallStatement, depth, indentStr); + case "FilterStatement": + return formatFilterStatement(node as FilterStatement, depth, indentStr); + case "Comment": + return pad + "{# " + (node as Comment).value + " #}"; default: return pad + "{{- " + formatExpression(node as Expression) + " -}}"; } @@ -93,7 +113,7 @@ function formatIf(node: If, depth: number, indentStr: string): string { formatStatements(clauses[0].body, depth + 1, indentStr); // ELIF(s) - for (let i = 1; i < clauses.length; i++) { + for (let i = 1; i < clauses.length; ++i) { out += NEWLINE + pad + @@ -119,7 +139,7 @@ function formatFor(node: For, depth: number, indentStr: string): string { if (node.iterable.type === "SelectExpression") { // Handle special case: e.g., `for x in [1, 2, 3] if x > 2` const n = node.iterable as SelectExpression; - formattedIterable = `${formatExpression(n.iterable)} if ${formatExpression(n.test)}`; + formattedIterable = `${formatExpression(n.lhs)} if ${formatExpression(n.test)}`; } else { formattedIterable = formatExpression(node.iterable); } @@ -166,20 +186,46 @@ function formatMacro(node: Macro, depth: number, indentStr: string): string { ); } +function formatCallStatement(node: CallStatement, depth: number, indentStr: string): string { + const pad = indentStr.repeat(depth); + const params = + node.callerArgs && node.callerArgs.length > 0 ? `(${node.callerArgs.map(formatExpression).join(", ")})` : ""; + const callExpr = formatExpression(node.call); + let out = pad + createStatement(`call${params}`, callExpr) + NEWLINE; + out += formatStatements(node.body, depth + 1, indentStr) + NEWLINE; + out += pad + createStatement("endcall"); + return out; +} + +function formatFilterStatement(node: FilterStatement, depth: number, indentStr: string): string { + const pad = indentStr.repeat(depth); + const spec = + node.filter.type === "Identifier" + ? (node.filter as Identifier).value + : formatExpression(node.filter as CallExpression); + let out = pad + createStatement("filter", spec) + NEWLINE; + out += formatStatements(node.body, depth + 1, indentStr) + NEWLINE; + out += pad + createStatement("endfilter"); + return out; +} + function formatExpression(node: Expression, parentPrec: number = -1): string { switch (node.type) { + case "SpreadExpression": { + const n = node as SpreadExpression; + return `*${formatExpression(n.argument)}`; + } case "Identifier": return (node as Identifier).value; - case "NullLiteral": - return "none"; - case "NumericLiteral": - case "BooleanLiteral": - return `${(node as NumericLiteral | BooleanLiteral).value}`; + case "IntegerLiteral": + return `${(node as IntegerLiteral).value}`; + case "FloatLiteral": + return `${(node as FloatLiteral).value}`; case "StringLiteral": return JSON.stringify((node as StringLiteral).value); case "BinaryExpression": { const n = node as BinaryExpression; - const thisPrecedence = OPERATOR_PRECEDENCE[n.operator.type] ?? 0; + const thisPrecedence = getBinaryOperatorPrecedence(n); const left = formatExpression(n.left, thisPrecedence); const right = formatExpression(n.right, thisPrecedence + 1); const expr = `${left} ${n.operator.value} ${right}`; @@ -190,20 +236,31 @@ function formatExpression(node: Expression, parentPrec: number = -1): string { const val = n.operator.value + (n.operator.value === "not" ? " " : "") + formatExpression(n.argument, Infinity); return val; } - case "LogicalNegationExpression": - return `not ${formatExpression((node as LogicalNegationExpression).argument, Infinity)}`; case "CallExpression": { const n = node as CallExpression; - const args = n.args.map((a) => formatExpression(a, -1)).join(", "); - return `${formatExpression(n.callee, -1)}(${args})`; + const args = n.args.map(formatExpression).join(", "); + return `${formatExpression(n.callee)}(${args})`; } case "MemberExpression": { const n = node as MemberExpression; - let obj = formatExpression(n.object, -1); - if (n.object.type !== "Identifier") { + let obj = formatExpression(n.object); + // only wrap if it's not a simple or chained access/call + if ( + ![ + "Identifier", + "MemberExpression", + "CallExpression", + "StringLiteral", + "IntegerLiteral", + "FloatLiteral", + "ArrayLiteral", + "TupleLiteral", + "ObjectLiteral", + ].includes(n.object.type) + ) { obj = `(${obj})`; } - let prop = formatExpression(n.property, -1); + let prop = formatExpression(n.property); if (!n.computed && n.property.type !== "Identifier") { prop = `(${prop})`; } @@ -213,48 +270,47 @@ function formatExpression(node: Expression, parentPrec: number = -1): string { const n = node as FilterExpression; const operand = formatExpression(n.operand, Infinity); if (n.filter.type === "CallExpression") { - return `${operand} | ${formatExpression(n.filter, -1)}`; + return `${operand} | ${formatExpression(n.filter)}`; } return `${operand} | ${(n.filter as Identifier).value}`; } case "SelectExpression": { const n = node as SelectExpression; - return `${formatExpression(n.iterable, -1)} | select(${formatExpression(n.test, -1)})`; + return `${formatExpression(n.lhs)} if ${formatExpression(n.test)}`; } case "TestExpression": { const n = node as TestExpression; - return `${formatExpression(n.operand, -1)} is${n.negate ? " not" : ""} ${n.test.value}`; + return `${formatExpression(n.operand)} is${n.negate ? " not" : ""} ${n.test.value}`; } case "ArrayLiteral": case "TupleLiteral": { - const elems = ((node as ArrayLiteral | TupleLiteral).value as Expression[]).map((e) => formatExpression(e, -1)); + const elems = ((node as ArrayLiteral | TupleLiteral).value as Expression[]).map(formatExpression); const brackets = node.type === "ArrayLiteral" ? "[]" : "()"; return `${brackets[0]}${elems.join(", ")}${brackets[1]}`; } case "ObjectLiteral": { const entries = Array.from((node as ObjectLiteral).value.entries()).map( - ([k, v]) => `${formatExpression(k, -1)}: ${formatExpression(v, -1)}` + ([k, v]) => `${formatExpression(k)}: ${formatExpression(v)}` ); - return `{ ${entries.join(", ")} }`; + return `{${entries.join(", ")}}`; } case "SliceExpression": { const n = node as SliceExpression; - const s = n.start ? formatExpression(n.start, -1) : ""; - const t = n.stop ? formatExpression(n.stop, -1) : ""; - const st = n.step ? `:${formatExpression(n.step, -1)}` : ""; + const s = n.start ? formatExpression(n.start) : ""; + const t = n.stop ? formatExpression(n.stop) : ""; + const st = n.step ? `:${formatExpression(n.step)}` : ""; return `${s}:${t}${st}`; } case "KeywordArgumentExpression": { const n = node as KeywordArgumentExpression; - return `${n.key.value}=${formatExpression(n.value, -1)}`; + return `${n.key.value}=${formatExpression(n.value)}`; } - case "If": { - // Special case for ternary operator (If as an expression, not a statement) - const n = node as If; - const test = formatExpression(n.test, -1); - const body = formatExpression(n.body[0], 0); // Ternary operators have a single body and alternate - const alternate = formatExpression(n.alternate[0], -1); - return `${body} if ${test} else ${alternate}`; + case "Ternary": { + const n = node as Ternary; + const expr = `${formatExpression(n.trueExpr)} if ${formatExpression(n.condition, 0)} else ${formatExpression( + n.falseExpr + )}`; + return parentPrec > -1 ? `(${expr})` : expr; } default: throw new Error(`Unknown expression type: ${node.type}`); diff --git a/packages/jinja/src/index.ts b/packages/jinja/src/index.ts index 25676a06a..10abc7d8a 100644 --- a/packages/jinja/src/index.ts +++ b/packages/jinja/src/index.ts @@ -12,10 +12,9 @@ */ import { tokenize } from "./lexer"; import { parse } from "./parser"; -import { Environment, Interpreter } from "./runtime"; +import { Environment, Interpreter, setupGlobals } from "./runtime"; import type { Program } from "./ast"; import type { StringValue } from "./runtime"; -import { range } from "./utils"; import { format } from "./format"; export class Template { @@ -35,14 +34,7 @@ export class Template { render(items?: Record): string { // Create a new environment for this template const env = new Environment(); - - // Declare global variables - env.set("false", false); - env.set("true", true); - env.set("raise_exception", (args: string) => { - throw new Error(args); - }); - env.set("range", range); + setupGlobals(env); // Add user-defined variables if (items) { diff --git a/packages/jinja/src/lexer.ts b/packages/jinja/src/lexer.ts index 632fa19ed..e2adaecfc 100644 --- a/packages/jinja/src/lexer.ts +++ b/packages/jinja/src/lexer.ts @@ -4,11 +4,9 @@ export const TOKEN_TYPES = Object.freeze({ Text: "Text", // The text between Jinja statements or expressions - NumericLiteral: "NumericLiteral", // e.g., 123 - BooleanLiteral: "BooleanLiteral", // true or false - NullLiteral: "NullLiteral", // none + NumericLiteral: "NumericLiteral", // e.g., 123, 1.0 StringLiteral: "StringLiteral", // 'string' - Identifier: "Identifier", // Variables, functions, etc. + Identifier: "Identifier", // Variables, functions, statements, booleans, etc. Equals: "Equals", // = OpenParen: "OpenParen", // ( CloseParen: "CloseParen", // ) @@ -26,71 +24,15 @@ export const TOKEN_TYPES = Object.freeze({ Pipe: "Pipe", // | CallOperator: "CallOperator", // () - AdditiveBinaryOperator: "AdditiveBinaryOperator", // + - + AdditiveBinaryOperator: "AdditiveBinaryOperator", // + - ~ MultiplicativeBinaryOperator: "MultiplicativeBinaryOperator", // * / % ComparisonBinaryOperator: "ComparisonBinaryOperator", // < > <= >= == != UnaryOperator: "UnaryOperator", // ! - + - - // Keywords - Set: "Set", - If: "If", - For: "For", - In: "In", - Is: "Is", - NotIn: "NotIn", - Else: "Else", - EndSet: "EndSet", - EndIf: "EndIf", - ElseIf: "ElseIf", - EndFor: "EndFor", - And: "And", - Or: "Or", - Not: "UnaryOperator", - Macro: "Macro", - EndMacro: "EndMacro", - Break: "Break", - Continue: "Continue", + Comment: "Comment", // {# ... #} }); export type TokenType = keyof typeof TOKEN_TYPES; -/** - * Constant lookup for keywords and known identifiers + symbols. - */ -const KEYWORDS = Object.freeze({ - set: TOKEN_TYPES.Set, - for: TOKEN_TYPES.For, - in: TOKEN_TYPES.In, - is: TOKEN_TYPES.Is, - if: TOKEN_TYPES.If, - else: TOKEN_TYPES.Else, - endset: TOKEN_TYPES.EndSet, - endif: TOKEN_TYPES.EndIf, - elif: TOKEN_TYPES.ElseIf, - endfor: TOKEN_TYPES.EndFor, - and: TOKEN_TYPES.And, - or: TOKEN_TYPES.Or, - not: TOKEN_TYPES.Not, - "not in": TOKEN_TYPES.NotIn, - macro: TOKEN_TYPES.Macro, - endmacro: TOKEN_TYPES.EndMacro, - break: TOKEN_TYPES.Break, - continue: TOKEN_TYPES.Continue, - - // Literals - true: TOKEN_TYPES.BooleanLiteral, - false: TOKEN_TYPES.BooleanLiteral, - none: TOKEN_TYPES.NullLiteral, - - // NOTE: According to the Jinja docs: The special constants true, false, and none are indeed lowercase. - // Because that caused confusion in the past, (True used to expand to an undefined variable that was considered false), - // all three can now also be written in title case (True, False, and None). However, for consistency, (all Jinja identifiers are lowercase) - // you should use the lowercase versions. - True: TOKEN_TYPES.BooleanLiteral, - False: TOKEN_TYPES.BooleanLiteral, - None: TOKEN_TYPES.NullLiteral, -}); - /** * Represents a single token in the template. */ @@ -144,6 +86,7 @@ const ORDERED_MAPPING_TABLE: [string, TokenType][] = [ // Arithmetic operators ["+", TOKEN_TYPES.AdditiveBinaryOperator], ["-", TOKEN_TYPES.AdditiveBinaryOperator], + ["~", TOKEN_TYPES.AdditiveBinaryOperator], ["*", TOKEN_TYPES.MultiplicativeBinaryOperator], ["/", TOKEN_TYPES.MultiplicativeBinaryOperator], ["%", TOKEN_TYPES.MultiplicativeBinaryOperator], @@ -178,29 +121,32 @@ function preprocess(template: string, options: PreprocessOptions = {}): string { template = template.slice(0, -1); } - // Replace all comments with a placeholder - // This ensures that comments don't interfere with the following options - template = template.replace(/{#.*?#}/gs, "{##}"); - if (options.lstrip_blocks) { // The lstrip_blocks option can also be set to strip tabs and spaces from the // beginning of a line to the start of a block. (Nothing will be stripped if // there are other characters before the start of the block.) - template = template.replace(/^[ \t]*({[#%])/gm, "$1"); + template = template.replace(/^[ \t]*({[#%-])/gm, "$1"); } if (options.trim_blocks) { // If an application configures Jinja to trim_blocks, the first newline after // a template tag is removed automatically (like in PHP). - template = template.replace(/([#%]})\n/g, "$1"); + template = template.replace(/([#%-]})\n/g, "$1"); } - return template - .replace(/{##}/g, "") // Remove comments - .replace(/-%}\s*/g, "%}") - .replace(/\s*{%-/g, "{%") - .replace(/-}}\s*/g, "}}") - .replace(/\s*{{-/g, "{{"); + return ( + template + .replace(/-%}\s*/g, "%}") + .replace(/\s*{%-/g, "{%") + .replace(/-}}\s*/g, "}}") + .replace(/\s*{{-/g, "{{") + .replace(/-#}\s*/g, "#}") + .replace(/\s*{#-/g, "{#") + + // Handle the custom transformers-specific `generation` tag. + // See https://github.com/huggingface/transformers/pull/30650 for more information. + .replace(/{%\s*generation\s*%}.+?{%\s*endgeneration\s*%}/gs, "") + ); } /** @@ -211,6 +157,7 @@ export function tokenize(source: string, options: PreprocessOptions = {}): Token const src: string = preprocess(source, options); let cursorPosition = 0; + let curlyBracketDepth = 0; const consumeWhile = (predicate: (char: string) => boolean): string => { let str = ""; @@ -245,13 +192,17 @@ export function tokenize(source: string, options: PreprocessOptions = {}): Token if ( lastTokenType === undefined || lastTokenType === TOKEN_TYPES.CloseStatement || - lastTokenType === TOKEN_TYPES.CloseExpression + lastTokenType === TOKEN_TYPES.CloseExpression || + lastTokenType === TOKEN_TYPES.Comment ) { let text = ""; while ( cursorPosition < src.length && // Keep going until we hit the next Jinja statement or expression - !(src[cursorPosition] === "{" && (src[cursorPosition + 1] === "%" || src[cursorPosition + 1] === "{")) + !( + src[cursorPosition] === "{" && + (src[cursorPosition + 1] === "%" || src[cursorPosition + 1] === "{" || src[cursorPosition + 1] === "#") + ) ) { // Consume text text += src[cursorPosition++]; @@ -264,6 +215,23 @@ export function tokenize(source: string, options: PreprocessOptions = {}): Token } } + // Possibly consume a comment + if (src[cursorPosition] === "{" && src[cursorPosition + 1] === "#") { + cursorPosition += 2; // Skip the opening {# + + let comment = ""; + while (src[cursorPosition] !== "#" || src[cursorPosition + 1] !== "}") { + // Check for end of input + if (cursorPosition + 2 >= src.length) { + throw new SyntaxError("Missing end of comment tag"); + } + comment += src[cursorPosition++]; + } + tokens.push(new Token(comment, TOKEN_TYPES.Comment)); + cursorPosition += 2; // Skip the closing #} + continue; + } + // Consume (and ignore) all whitespace inside Jinja statements or expressions consumeWhile((char) => /\s/.test(char)); @@ -279,8 +247,6 @@ export function tokenize(source: string, options: PreprocessOptions = {}): Token switch (lastTokenType) { case TOKEN_TYPES.Identifier: case TOKEN_TYPES.NumericLiteral: - case TOKEN_TYPES.BooleanLiteral: - case TOKEN_TYPES.NullLiteral: case TOKEN_TYPES.StringLiteral: case TOKEN_TYPES.CloseParen: case TOKEN_TYPES.CloseSquareBracket: @@ -305,11 +271,24 @@ export function tokenize(source: string, options: PreprocessOptions = {}): Token } // Try to match one of the tokens in the mapping table - for (const [char, token] of ORDERED_MAPPING_TABLE) { - const slice = src.slice(cursorPosition, cursorPosition + char.length); - if (slice === char) { - tokens.push(new Token(char, token)); - cursorPosition += char.length; + for (const [seq, type] of ORDERED_MAPPING_TABLE) { + // inside an object literal, don't treat "}}" as expression-end + if (seq === "}}" && curlyBracketDepth > 0) { + continue; + } + const slice = src.slice(cursorPosition, cursorPosition + seq.length); + if (slice === seq) { + tokens.push(new Token(seq, type)); + + // possibly adjust the curly bracket depth + if (type === TOKEN_TYPES.OpenExpression) { + curlyBracketDepth = 0; + } else if (type === TOKEN_TYPES.OpenCurlyBracket) { + ++curlyBracketDepth; + } else if (type === TOKEN_TYPES.CloseCurlyBracket) { + --curlyBracketDepth; + } + cursorPosition += seq.length; continue main; } } @@ -323,27 +302,21 @@ export function tokenize(source: string, options: PreprocessOptions = {}): Token } if (isInteger(char)) { - const num = consumeWhile(isInteger); + // Consume integer part + let num = consumeWhile(isInteger); + // Possibly, consume fractional part + if (src[cursorPosition] === "." && isInteger(src[cursorPosition + 1])) { + ++cursorPosition; // consume '.' + const frac = consumeWhile(isInteger); + num = `${num}.${frac}`; + } tokens.push(new Token(num, TOKEN_TYPES.NumericLiteral)); continue; } if (isWord(char)) { + // consume any word characters and always classify as Identifier const word = consumeWhile(isWord); - - // Check for special/reserved keywords - // NOTE: We use Object.hasOwn() to avoid matching `.toString()` and other Object methods - const type = Object.hasOwn(KEYWORDS, word) ? KEYWORDS[word as keyof typeof KEYWORDS] : TOKEN_TYPES.Identifier; - - // Special case of not in: - // If the previous token was a "not", and this token is "in" - // then we want to combine them into a single token - if (type === TOKEN_TYPES.In && tokens.at(-1)?.type === TOKEN_TYPES.Not) { - tokens.pop(); - tokens.push(new Token("not in", TOKEN_TYPES.NotIn)); - } else { - tokens.push(new Token(word, type)); - } - + tokens.push(new Token(word, TOKEN_TYPES.Identifier)); continue; } diff --git a/packages/jinja/src/parser.ts b/packages/jinja/src/parser.ts index 5160f9e8b..e89c2d7f4 100644 --- a/packages/jinja/src/parser.ts +++ b/packages/jinja/src/parser.ts @@ -1,5 +1,5 @@ -import type { Token, TokenType } from "./lexer"; -import { TOKEN_TYPES } from "./lexer"; +import { Token, TOKEN_TYPES } from "./lexer"; +import type { TokenType } from "./lexer"; import type { Statement } from "./ast"; import { Program, @@ -11,10 +11,7 @@ import { MemberExpression, CallExpression, Identifier, - NumericLiteral, StringLiteral, - BooleanLiteral, - NullLiteral, ArrayLiteral, ObjectLiteral, BinaryExpression, @@ -26,6 +23,13 @@ import { TupleLiteral, Macro, SelectExpression, + CallStatement, + FilterStatement, + SpreadExpression, + IntegerLiteral, + FloatLiteral, + Ternary, + Comment, } from "./ast"; /** @@ -50,8 +54,17 @@ export function parse(tokens: Token[]): Program { return prev; } + function expectIdentifier(name: string): void { + if (!isIdentifier(name)) { + throw new SyntaxError(`Expected ${name}`); + } + ++current; + } + function parseAny(): Statement { switch (tokens[current].type) { + case TOKEN_TYPES.Comment: + return new Comment(tokens[current++].value); case TOKEN_TYPES.Text: return parseText(); case TOKEN_TYPES.OpenStatement: @@ -63,65 +76,121 @@ export function parse(tokens: Token[]): Program { } } - function not(...types: TokenType[]): boolean { - return current + types.length <= tokens.length && types.some((type, i) => type !== tokens[current + i].type); - } - function is(...types: TokenType[]): boolean { return current + types.length <= tokens.length && types.every((type, i) => type === tokens[current + i].type); } + function isStatement(...names: string[]): boolean { + return ( + tokens[current]?.type === TOKEN_TYPES.OpenStatement && + tokens[current + 1]?.type === TOKEN_TYPES.Identifier && + names.includes(tokens[current + 1]?.value) + ); + } + + function isIdentifier(...names: string[]): boolean { + return ( + current + names.length <= tokens.length && + names.every((name, i) => tokens[current + i].type === "Identifier" && name === tokens[current + i].value) + ); + } + function parseText(): StringLiteral { return new StringLiteral(expect(TOKEN_TYPES.Text, "Expected text token").value); } function parseJinjaStatement(): Statement { - // Consume {% %} tokens + // Consume {% token expect(TOKEN_TYPES.OpenStatement, "Expected opening statement token"); - let result; - switch (tokens[current].type) { - case TOKEN_TYPES.Set: + // next token must be Identifier whose .value tells us which statement + if (tokens[current].type !== TOKEN_TYPES.Identifier) { + throw new SyntaxError(`Unknown statement, got ${tokens[current].type}`); + } + const name = tokens[current].value; + let result: Statement; + switch (name) { + case "set": ++current; result = parseSetStatement(); - expect(TOKEN_TYPES.CloseStatement, "Expected closing statement token"); break; - - case TOKEN_TYPES.If: + case "if": ++current; result = parseIfStatement(); + // expect {% endif %} expect(TOKEN_TYPES.OpenStatement, "Expected {% token"); - expect(TOKEN_TYPES.EndIf, "Expected endif token"); + expectIdentifier("endif"); expect(TOKEN_TYPES.CloseStatement, "Expected %} token"); break; - - case TOKEN_TYPES.Macro: + case "macro": ++current; result = parseMacroStatement(); + // expect {% endmacro %} expect(TOKEN_TYPES.OpenStatement, "Expected {% token"); - expect(TOKEN_TYPES.EndMacro, "Expected endmacro token"); + expectIdentifier("endmacro"); expect(TOKEN_TYPES.CloseStatement, "Expected %} token"); break; - - case TOKEN_TYPES.For: + case "for": ++current; result = parseForStatement(); + // expect {% endfor %} expect(TOKEN_TYPES.OpenStatement, "Expected {% token"); - expect(TOKEN_TYPES.EndFor, "Expected endfor token"); + expectIdentifier("endfor"); expect(TOKEN_TYPES.CloseStatement, "Expected %} token"); break; - case TOKEN_TYPES.Break: + case "call": { + ++current; // consume 'call' + let callerArgs: Statement[] | null = null; + if (is(TOKEN_TYPES.OpenParen)) { + // Optional caller arguments, e.g. {% call(user) dump_users(...) %} + callerArgs = parseArgs(); + } + const callee = parsePrimaryExpression(); + if (callee.type !== "Identifier") { + throw new SyntaxError(`Expected identifier following call statement`); + } + const callArgs = parseArgs(); + expect(TOKEN_TYPES.CloseStatement, "Expected closing statement token"); + const body: Statement[] = []; + while (!isStatement("endcall")) { + body.push(parseAny()); + } + expect(TOKEN_TYPES.OpenStatement, "Expected '{%'"); + expectIdentifier("endcall"); + expect(TOKEN_TYPES.CloseStatement, "Expected closing statement token"); + const callExpr = new CallExpression(callee, callArgs); + result = new CallStatement(callExpr, callerArgs, body); + break; + } + case "break": ++current; expect(TOKEN_TYPES.CloseStatement, "Expected closing statement token"); result = new Break(); break; - case TOKEN_TYPES.Continue: + case "continue": ++current; expect(TOKEN_TYPES.CloseStatement, "Expected closing statement token"); result = new Continue(); break; + case "filter": { + ++current; // consume 'filter' + let filterNode = parsePrimaryExpression(); + if (filterNode instanceof Identifier && is(TOKEN_TYPES.OpenParen)) { + filterNode = parseCallExpression(filterNode); + } + expect(TOKEN_TYPES.CloseStatement, "Expected closing statement token"); + const filterBody: Statement[] = []; + while (!isStatement("endfilter")) { + filterBody.push(parseAny()); + } + expect(TOKEN_TYPES.OpenStatement, "Expected '{%'"); + expectIdentifier("endfilter"); + expect(TOKEN_TYPES.CloseStatement, "Expected '%}'"); + result = new FilterStatement(filterNode as Identifier | CallExpression, filterBody); + break; + } default: - throw new SyntaxError(`Unknown statement type: ${tokens[current].type}`); + throw new SyntaxError(`Unknown statement type: ${name}`); } return result; @@ -139,28 +208,23 @@ export function parse(tokens: Token[]): Program { // NOTE: `set` acts as both declaration statement and assignment expression function parseSetStatement(): Statement { - const left = parseExpression(); - + const left = parseExpressionSequence(); + let value: Statement | null = null; + const body: Statement[] = []; if (is(TOKEN_TYPES.Equals)) { ++current; - const value = parseExpression(); - - return new SetStatement(left, value, []); + value = parseExpressionSequence(); } else { // parsing multiline set here - const body: Statement[] = []; expect(TOKEN_TYPES.CloseStatement, "Expected %} token"); - while ( - !(tokens[current]?.type === TOKEN_TYPES.OpenStatement && tokens[current + 1]?.type === TOKEN_TYPES.EndSet) - ) { - const another = parseAny(); - body.push(another); + while (!isStatement("endset")) { + body.push(parseAny()); } expect(TOKEN_TYPES.OpenStatement, "Expected {% token"); - expect(TOKEN_TYPES.EndSet, "Expected endset token"); - - return new SetStatement(left, null, body); + expectIdentifier("endset"); } + expect(TOKEN_TYPES.CloseStatement, "Expected closing statement token"); + return new SetStatement(left, value, body); } function parseIfStatement(): If { @@ -171,38 +235,27 @@ export function parse(tokens: Token[]): Program { const body: Statement[] = []; const alternate: Statement[] = []; - // Keep parsing if body until we reach the first {% elif %} or {% else %} or {% endif %} - while ( - !( - tokens[current]?.type === TOKEN_TYPES.OpenStatement && - (tokens[current + 1]?.type === TOKEN_TYPES.ElseIf || - tokens[current + 1]?.type === TOKEN_TYPES.Else || - tokens[current + 1]?.type === TOKEN_TYPES.EndIf) - ) - ) { + // Keep parsing 'if' body until we reach the first {% elif %} or {% else %} or {% endif %} + while (!isStatement("elif", "else", "endif")) { body.push(parseAny()); } - // Alternate branch: Check for {% elif %} or {% else %} - if ( - tokens[current]?.type === TOKEN_TYPES.OpenStatement && - tokens[current + 1]?.type !== TOKEN_TYPES.EndIf // There is some body - ) { - ++current; // eat {% token - if (is(TOKEN_TYPES.ElseIf)) { - expect(TOKEN_TYPES.ElseIf, "Expected elseif token"); - alternate.push(parseIfStatement()); - } else { - // tokens[current]?.type === TokenType.Else - expect(TOKEN_TYPES.Else, "Expected else token"); - expect(TOKEN_TYPES.CloseStatement, "Expected closing statement token"); + // handle {% elif %} + if (isStatement("elif")) { + ++current; // consume {% + ++current; // consume 'elif' + const result = parseIfStatement(); // nested If + alternate.push(result); + } + // handle {% else %} + else if (isStatement("else")) { + ++current; // consume {% + ++current; // consume 'else' + expect(TOKEN_TYPES.CloseStatement, "Expected closing statement token"); - // keep going until we hit {% endif %} - while ( - !(tokens[current]?.type === TOKEN_TYPES.OpenStatement && tokens[current + 1]?.type === TOKEN_TYPES.EndIf) - ) { - alternate.push(parseAny()); - } + // keep going until we hit {% endif %} + while (!isStatement("endif")) { + alternate.push(parseAny()); } } @@ -221,7 +274,7 @@ export function parse(tokens: Token[]): Program { const body: Statement[] = []; // Keep going until we hit {% endmacro - while (not(TOKEN_TYPES.OpenStatement, TOKEN_TYPES.EndMacro)) { + while (!isStatement("endmacro")) { body.push(parseAny()); } @@ -249,7 +302,10 @@ export function parse(tokens: Token[]): Program { throw new SyntaxError(`Expected identifier/tuple for the loop variable, got ${loopVariable.type} instead`); } - expect(TOKEN_TYPES.In, "Expected `in` keyword following loop variable"); + if (!isIdentifier("in")) { + throw new SyntaxError("Expected `in` keyword following loop variable"); + } + ++current; // `messages` in `for message in messages` const iterable = parseExpression(); @@ -260,19 +316,17 @@ export function parse(tokens: Token[]): Program { const body: Statement[] = []; // Keep going until we hit {% endfor or {% else - while (not(TOKEN_TYPES.OpenStatement, TOKEN_TYPES.EndFor) && not(TOKEN_TYPES.OpenStatement, TOKEN_TYPES.Else)) { + while (!isStatement("endfor", "else")) { body.push(parseAny()); } // (Optional) else block const alternative: Statement[] = []; - if (is(TOKEN_TYPES.OpenStatement, TOKEN_TYPES.Else)) { + if (isStatement("else")) { ++current; // consume {% - ++current; // consume else + ++current; // consume 'else' expect(TOKEN_TYPES.CloseStatement, "Expected closing statement token"); - - // keep going until we hit {% endfor - while (not(TOKEN_TYPES.OpenStatement, TOKEN_TYPES.EndFor)) { + while (!isStatement("endfor")) { alternative.push(parseAny()); } } @@ -287,19 +341,19 @@ export function parse(tokens: Token[]): Program { function parseIfExpression(): Statement { const a = parseLogicalOrExpression(); - if (is(TOKEN_TYPES.If)) { + if (isIdentifier("if")) { // Ternary expression - ++current; // consume if - const predicate = parseLogicalOrExpression(); + ++current; // consume 'if' + const test = parseLogicalOrExpression(); - if (is(TOKEN_TYPES.Else)) { + if (isIdentifier("else")) { // Ternary expression with else - ++current; // consume else - const b = parseLogicalOrExpression(); - return new If(predicate, [a], [b]); + ++current; // consume 'else' + const falseExpr = parseIfExpression(); // recurse to support chained ternaries + return new Ternary(test, a, falseExpr); } else { // Select expression on iterable - return new SelectExpression(a, predicate); + return new SelectExpression(a, test); } } return a; @@ -307,7 +361,7 @@ export function parse(tokens: Token[]): Program { function parseLogicalOrExpression(): Statement { let left = parseLogicalAndExpression(); - while (is(TOKEN_TYPES.Or)) { + while (isIdentifier("or")) { const operator = tokens[current]; ++current; const right = parseLogicalAndExpression(); @@ -318,7 +372,7 @@ export function parse(tokens: Token[]): Program { function parseLogicalAndExpression(): Statement { let left = parseLogicalNegationExpression(); - while (is(TOKEN_TYPES.And)) { + while (isIdentifier("and")) { const operator = tokens[current]; ++current; const right = parseLogicalNegationExpression(); @@ -331,7 +385,7 @@ export function parse(tokens: Token[]): Program { let right: UnaryExpression | undefined; // Try parse unary operators - while (is(TOKEN_TYPES.Not)) { + while (isIdentifier("not")) { // not not ... const operator = tokens[current]; ++current; @@ -346,9 +400,18 @@ export function parse(tokens: Token[]): Program { // NOTE: membership has same precedence as comparison // e.g., ('a' in 'apple' == 'b' in 'banana') evaluates as ('a' in ('apple' == ('b' in 'banana'))) let left = parseAdditiveExpression(); - while (is(TOKEN_TYPES.ComparisonBinaryOperator) || is(TOKEN_TYPES.In) || is(TOKEN_TYPES.NotIn)) { - const operator = tokens[current]; - ++current; + while (true) { + let operator: Token; + if (isIdentifier("not", "in")) { + operator = new Token("not in", TOKEN_TYPES.Identifier); + current += 2; + } else if (isIdentifier("in")) { + operator = tokens[current++]; + } else if (is(TOKEN_TYPES.ComparisonBinaryOperator)) { + operator = tokens[current++]; + } else { + break; + } const right = parseAdditiveExpression(); left = new BinaryExpression(operator, left, right); } @@ -404,17 +467,25 @@ export function parse(tokens: Token[]): Program { const args = []; while (!is(TOKEN_TYPES.CloseParen)) { - let argument = parseExpression(); - - if (is(TOKEN_TYPES.Equals)) { - // keyword argument - // e.g., func(x = 5, y = a or b) - ++current; // consume equals - if (!(argument instanceof Identifier)) { - throw new SyntaxError(`Expected identifier for keyword argument`); + let argument: Statement; + + // unpacking: *expr + if (tokens[current].type === TOKEN_TYPES.MultiplicativeBinaryOperator && tokens[current].value === "*") { + ++current; + const expr = parseExpression(); + argument = new SpreadExpression(expr); + } else { + argument = parseExpression(); + if (is(TOKEN_TYPES.Equals)) { + // keyword argument + // e.g., func(x = 5, y = a or b) + ++current; // consume equals + if (!(argument instanceof Identifier)) { + throw new SyntaxError(`Expected identifier for keyword argument`); + } + const value = parseExpression(); + argument = new KeywordArgumentExpression(argument as Identifier, value); } - const value = parseExpression(); - argument = new KeywordArgumentExpression(argument, value); } args.push(argument); if (is(TOKEN_TYPES.Comma)) { @@ -465,7 +536,7 @@ export function parse(tokens: Token[]): Program { const operator = tokens[current]; // . or [ ++current; let property: Statement; - const computed = operator.type !== TOKEN_TYPES.Dot; + const computed = operator.type === TOKEN_TYPES.OpenSquareBracket; if (computed) { // computed (i.e., bracket notation: obj[expr]) property = parseMemberExpressionArgumentsList(); @@ -489,8 +560,7 @@ export function parse(tokens: Token[]): Program { // e.g., (4 * 4 is divisibleby(2)) evaluates as (4 * (4 is divisibleby(2))) while (is(TOKEN_TYPES.MultiplicativeBinaryOperator)) { - const operator = tokens[current]; - ++current; + const operator = tokens[current++]; const right = parseTestExpression(); left = new BinaryExpression(operator, left, right); } @@ -500,21 +570,15 @@ export function parse(tokens: Token[]): Program { function parseTestExpression(): Statement { let operand = parseFilterExpression(); - while (is(TOKEN_TYPES.Is)) { + while (isIdentifier("is")) { // Support chaining tests ++current; // consume is - const negate = is(TOKEN_TYPES.Not); + const negate = isIdentifier("not"); if (negate) { ++current; // consume not } - let filter = parsePrimaryExpression(); - if (filter instanceof BooleanLiteral) { - // Special case: treat boolean literals as identifiers - filter = new Identifier(filter.value.toString()); - } else if (filter instanceof NullLiteral) { - filter = new Identifier("none"); - } + const filter = parsePrimaryExpression(); if (!(filter instanceof Identifier)) { throw new SyntaxError(`Expected identifier for the test`); } @@ -544,35 +608,27 @@ export function parse(tokens: Token[]): Program { function parsePrimaryExpression(): Statement { // Primary expression: number, string, identifier, function call, parenthesized expression - const token = tokens[current]; + const token = tokens[current++]; switch (token.type) { - case TOKEN_TYPES.NumericLiteral: - ++current; - return new NumericLiteral(Number(token.value)); - case TOKEN_TYPES.StringLiteral: - ++current; - return new StringLiteral(token.value); - case TOKEN_TYPES.BooleanLiteral: - ++current; - return new BooleanLiteral(token.value.toLowerCase() === "true"); - case TOKEN_TYPES.NullLiteral: - ++current; - return new NullLiteral(null); + case TOKEN_TYPES.NumericLiteral: { + const num = token.value; + return num.includes(".") ? new FloatLiteral(Number(num)) : new IntegerLiteral(Number(num)); + } + case TOKEN_TYPES.StringLiteral: { + let value = token.value; + while (is(TOKEN_TYPES.StringLiteral)) { + value += tokens[current++].value; + } + return new StringLiteral(value); + } case TOKEN_TYPES.Identifier: - ++current; return new Identifier(token.value); case TOKEN_TYPES.OpenParen: { - ++current; // consume opening parenthesis const expression = parseExpressionSequence(); - if (tokens[current].type !== TOKEN_TYPES.CloseParen) { - throw new SyntaxError(`Expected closing parenthesis, got ${tokens[current].type} instead`); - } - ++current; // consume closing parenthesis + expect(TOKEN_TYPES.CloseParen, "Expected closing parenthesis, got ${tokens[current].type} instead."); return expression; } case TOKEN_TYPES.OpenSquareBracket: { - ++current; // consume opening square bracket - const values = []; while (!is(TOKEN_TYPES.CloseSquareBracket)) { values.push(parseExpression()); @@ -586,8 +642,6 @@ export function parse(tokens: Token[]): Program { return new ArrayLiteral(values); } case TOKEN_TYPES.OpenCurlyBracket: { - ++current; // consume opening curly bracket - const values = new Map(); while (!is(TOKEN_TYPES.CloseCurlyBracket)) { const key = parseExpression(); diff --git a/packages/jinja/src/runtime.ts b/packages/jinja/src/runtime.ts index 5d19ddfa8..cd59167e3 100644 --- a/packages/jinja/src/runtime.ts +++ b/packages/jinja/src/runtime.ts @@ -1,8 +1,7 @@ import type { - NumericLiteral, StringLiteral, - BooleanLiteral, - NullLiteral, + FloatLiteral, + IntegerLiteral, ArrayLiteral, Statement, Program, @@ -23,11 +22,16 @@ import type { Macro, Expression, SelectExpression, + CallStatement, + FilterStatement, + Ternary, + SpreadExpression, } from "./ast"; -import { slice, titleCase } from "./utils"; +import { range, replace, slice, strftime_now, titleCase } from "./utils"; export type AnyRuntimeValue = - | NumericValue + | IntegerValue + | FloatValue | StringValue | BooleanValue | ObjectValue @@ -68,13 +72,28 @@ abstract class RuntimeValue { __bool__(): BooleanValue { return new BooleanValue(!!this.value); } + + toString(): string { + return String(this.value); + } } /** - * Represents a numeric value at runtime. + * Represents an integer value at runtime. */ -export class NumericValue extends RuntimeValue { - override type = "NumericValue"; +export class IntegerValue extends RuntimeValue { + override type = "IntegerValue"; +} + +/** + * Represents a float value at runtime. + */ +export class FloatValue extends RuntimeValue { + override type = "FloatValue"; + + override toString(): string { + return this.value % 1 === 0 ? this.value.toFixed(1) : this.value.toString(); + } } /** @@ -108,7 +127,13 @@ export class StringValue extends RuntimeValue { return new StringValue(titleCase(this.value)); }), ], - ["length", new NumericValue(this.value.length)], + [ + "capitalize", + new FunctionValue(() => { + return new StringValue(this.value.charAt(0).toUpperCase() + this.value.slice(1)); + }), + ], + ["length", new IntegerValue(this.value.length)], [ "rstrip", new FunctionValue(() => { @@ -127,11 +152,21 @@ export class StringValue extends RuntimeValue { if (args.length === 0) { throw new Error("startswith() requires at least one argument"); } - const prefix = args[0]; - if (!(prefix instanceof StringValue)) { - throw new Error("startswith() argument must be a string"); + const pattern = args[0]; + if (pattern instanceof StringValue) { + return new BooleanValue(this.value.startsWith(pattern.value)); + } else if (pattern instanceof ArrayValue) { + for (const item of pattern.value) { + if (!(item instanceof StringValue)) { + throw new Error("startswith() tuple elements must be strings"); + } + if (this.value.startsWith(item.value)) { + return new BooleanValue(true); + } + } + return new BooleanValue(false); } - return new BooleanValue(this.value.startsWith(prefix.value)); + throw new Error("startswith() argument must be a string or tuple of strings"); }), ], [ @@ -140,11 +175,21 @@ export class StringValue extends RuntimeValue { if (args.length === 0) { throw new Error("endswith() requires at least one argument"); } - const suffix = args[0]; - if (!(suffix instanceof StringValue)) { - throw new Error("endswith() argument must be a string"); + const pattern = args[0]; + if (pattern instanceof StringValue) { + return new BooleanValue(this.value.endsWith(pattern.value)); + } else if (pattern instanceof ArrayValue) { + for (const item of pattern.value) { + if (!(item instanceof StringValue)) { + throw new Error("endswith() tuple elements must be strings"); + } + if (this.value.endsWith(item.value)) { + return new BooleanValue(true); + } + } + return new BooleanValue(false); } - return new BooleanValue(this.value.endsWith(suffix.value)); + throw new Error("endswith() argument must be a string or tuple of strings"); }), ], [ @@ -156,8 +201,8 @@ export class StringValue extends RuntimeValue { if (!(sep instanceof StringValue || sep instanceof NullValue)) { throw new Error("sep argument must be a string or null"); } - const maxsplit = args[1] ?? new NumericValue(-1); - if (!(maxsplit instanceof NumericValue)) { + const maxsplit = args[1] ?? new IntegerValue(-1); + if (!(maxsplit instanceof IntegerValue)) { throw new Error("maxsplit argument must be a number"); } @@ -189,6 +234,34 @@ export class StringValue extends RuntimeValue { return new ArrayValue(result.map((part) => new StringValue(part))); }), ], + [ + "replace", + new FunctionValue((args): StringValue => { + if (args.length < 2) { + throw new Error("replace() requires at least two arguments"); + } + const oldValue = args[0]; + const newValue = args[1]; + if (!(oldValue instanceof StringValue && newValue instanceof StringValue)) { + throw new Error("replace() arguments must be strings"); + } + + let count: AnyRuntimeValue | undefined; + if (args.length > 2) { + if (args[2].type === "KeywordArgumentsValue") { + count = (args[2] as KeywordArgumentsValue).value.get("count") ?? new NullValue(); + } else { + count = args[2]; + } + } else { + count = new NullValue(); + } + if (!(count instanceof IntegerValue || count instanceof NullValue)) { + throw new Error("replace() count argument must be a number or null"); + } + return new StringValue(replace(this.value, oldValue.value, newValue.value, count.value)); + }), + ], ]); } @@ -227,15 +300,22 @@ export class ObjectValue extends RuntimeValue> { return this.value.get(key.value) ?? defaultValue ?? new NullValue(); }), ], - [ - "items", - new FunctionValue(() => { - return new ArrayValue( - Array.from(this.value.entries()).map(([key, value]) => new ArrayValue([new StringValue(key), value])) - ); - }), - ], + ["items", new FunctionValue(() => this.items())], + ["keys", new FunctionValue(() => this.keys())], + ["values", new FunctionValue(() => this.values())], ]); + + items(): ArrayValue { + return new ArrayValue( + Array.from(this.value.entries()).map(([key, value]) => new ArrayValue([new StringValue(key), value])) + ); + } + keys(): ArrayValue { + return new ArrayValue(Array.from(this.value.keys()).map((key) => new StringValue(key))); + } + values(): ArrayValue { + return new ArrayValue(Array.from(this.value.values())); + } } /** @@ -250,7 +330,7 @@ export class KeywordArgumentsValue extends ObjectValue { */ export class ArrayValue extends RuntimeValue { override type = "ArrayValue"; - override builtins = new Map([["length", new NumericValue(this.value.length)]]); + override builtins = new Map([["length", new IntegerValue(this.value.length)]]); /** * NOTE: necessary to override since all JavaScript arrays are considered truthy, @@ -325,27 +405,27 @@ export class Environment { [ "odd", (operand) => { - if (operand.type !== "NumericValue") { - throw new Error(`Cannot apply test "odd" to type: ${operand.type}`); + if (!(operand instanceof IntegerValue)) { + throw new Error(`cannot odd on ${operand.type}`); } - return (operand as NumericValue).value % 2 !== 0; + return operand.value % 2 !== 0; }, ], [ "even", (operand) => { - if (operand.type !== "NumericValue") { - throw new Error(`Cannot apply test "even" to type: ${operand.type}`); + if (!(operand instanceof IntegerValue)) { + throw new Error(`cannot even on ${operand.type}`); } - return (operand as NumericValue).value % 2 === 0; + return operand.value % 2 === 0; }, ], ["false", (operand) => operand.type === "BooleanValue" && !(operand as BooleanValue).value], ["true", (operand) => operand.type === "BooleanValue" && (operand as BooleanValue).value], ["none", (operand) => operand.type === "NullValue"], ["string", (operand) => operand.type === "StringValue"], - ["number", (operand) => operand.type === "NumericValue"], - ["integer", (operand) => operand.type === "NumericValue" && Number.isInteger((operand as NumericValue).value)], + ["number", (operand) => operand instanceof IntegerValue || operand instanceof FloatValue], + ["integer", (operand) => operand instanceof IntegerValue], ["iterable", (operand) => operand.type === "ArrayValue" || operand.type === "StringValue"], ["mapping", (operand) => operand.type === "ObjectValue"], [ @@ -428,6 +508,26 @@ export class Environment { } } +export function setupGlobals(env: Environment): void { + // Declare global variables + env.set("false", false); + env.set("true", true); + env.set("none", null); + env.set("raise_exception", (args: string) => { + throw new Error(args); + }); + env.set("range", range); + env.set("strftime_now", strftime_now); + + // NOTE: According to the Jinja docs: The special constants true, false, and none are indeed lowercase. + // Because that caused confusion in the past, (True used to expand to an undefined variable that was considered false), + // all three can now also be written in title case (True, False, and None). However, for consistency, (all Jinja identifiers are lowercase) + // you should use the lowercase versions. + env.set("True", true); + env.set("False", false); + env.set("None", null); +} + export class Interpreter { global: Environment; @@ -467,33 +567,48 @@ export class Interpreter { } if (left instanceof UndefinedValue || right instanceof UndefinedValue) { - throw new Error("Cannot perform operation on undefined values"); + if (right instanceof UndefinedValue && ["in", "not in"].includes(node.operator.value)) { + // Special case: `anything in undefined` is `false` and `anything not in undefined` is `true` + return new BooleanValue(node.operator.value === "not in"); + } + throw new Error(`Cannot perform operation ${node.operator.value} on undefined values`); } else if (left instanceof NullValue || right instanceof NullValue) { throw new Error("Cannot perform operation on null values"); - } else if (left instanceof NumericValue && right instanceof NumericValue) { + } else if (node.operator.value === "~") { + // toString and concatenation + return new StringValue(left.value.toString() + right.value.toString()); + } else if ( + (left instanceof IntegerValue || left instanceof FloatValue) && + (right instanceof IntegerValue || right instanceof FloatValue) + ) { // Evaulate pure numeric operations with binary operators. + const a = left.value, + b = right.value; switch (node.operator.value) { // Arithmetic operators case "+": - return new NumericValue(left.value + right.value); case "-": - return new NumericValue(left.value - right.value); - case "*": - return new NumericValue(left.value * right.value); + case "*": { + const res = node.operator.value === "+" ? a + b : node.operator.value === "-" ? a - b : a * b; + const isFloat = left instanceof FloatValue || right instanceof FloatValue; + return isFloat ? new FloatValue(res) : new IntegerValue(res); + } case "/": - return new NumericValue(left.value / right.value); - case "%": - return new NumericValue(left.value % right.value); - + return new FloatValue(a / b); + case "%": { + const rem = a % b; + const isFloat = left instanceof FloatValue || right instanceof FloatValue; + return isFloat ? new FloatValue(rem) : new IntegerValue(rem); + } // Comparison operators case "<": - return new BooleanValue(left.value < right.value); + return new BooleanValue(a < b); case ">": - return new BooleanValue(left.value > right.value); + return new BooleanValue(a > b); case ">=": - return new BooleanValue(left.value >= right.value); + return new BooleanValue(a >= b); case "<=": - return new BooleanValue(left.value <= right.value); + return new BooleanValue(a <= b); } } else if (left instanceof ArrayValue && right instanceof ArrayValue) { // Evaluate array operands with binary operator. @@ -545,11 +660,21 @@ export class Interpreter { environment: Environment ): [AnyRuntimeValue[], Map] { // Accumulate args and kwargs - const positionalArguments = []; - const keywordArguments = new Map(); + const positionalArguments: AnyRuntimeValue[] = []; + const keywordArguments = new Map(); + for (const argument of args) { // TODO: Lazy evaluation of arguments - if (argument.type === "KeywordArgumentExpression") { + if (argument.type === "SpreadExpression") { + const spreadNode = argument as SpreadExpression; + const val = this.evaluate(spreadNode.argument, environment); + if (!(val instanceof ArrayValue)) { + throw new Error(`Cannot unpack non-iterable type: ${val.type}`); + } + for (const item of val.value) { + positionalArguments.push(item); + } + } else if (argument.type === "KeywordArgumentExpression") { const kwarg = argument as KeywordArgumentExpression; keywordArguments.set(kwarg.key.value, this.evaluate(kwarg.value, environment)); } else { @@ -562,12 +687,7 @@ export class Interpreter { return [positionalArguments, keywordArguments]; } - /** - * Evaluates expressions following the filter operation type. - */ - private evaluateFilterExpression(node: FilterExpression, environment: Environment): AnyRuntimeValue { - const operand = this.evaluate(node.operand, environment); - + private applyFilter(operand: AnyRuntimeValue, filterNode: Identifier | CallExpression, environment: Environment) { // For now, we only support the built-in filters // TODO: Add support for non-identifier filters // e.g., functions which return filters: {{ numbers | select("odd") }} @@ -580,8 +700,8 @@ export class Interpreter { // https://jinja.palletsprojects.com/en/3.0.x/templates/#list-of-builtin-filters - if (node.filter.type === "Identifier") { - const filter = node.filter as Identifier; + if (filterNode.type === "Identifier") { + const filter = filterNode as Identifier; if (filter.value === "tojson") { return new StringValue(toJSON(operand)); @@ -596,7 +716,7 @@ export class Interpreter { case "last": return operand.value[operand.value.length - 1]; case "length": - return new NumericValue(operand.value.length); + return new IntegerValue(operand.value.length); case "reverse": return new ArrayValue(operand.value.reverse()); case "sort": @@ -606,8 +726,9 @@ export class Interpreter { throw new Error(`Cannot compare different types: ${a.type} and ${b.type}`); } switch (a.type) { - case "NumericValue": - return (a as NumericValue).value - (b as NumericValue).value; + case "IntegerValue": + case "FloatValue": + return (a as IntegerValue | FloatValue).value - (b as IntegerValue | FloatValue).value; case "StringValue": return (a as StringValue).value.localeCompare((b as StringValue).value); default: @@ -619,21 +740,37 @@ export class Interpreter { return new StringValue(operand.value.map((x) => x.value).join("")); case "string": return new StringValue(toJSON(operand)); + case "unique": { + const seen = new Set(); + const output: AnyRuntimeValue[] = []; + for (const item of operand.value) { + if (!seen.has(item.value)) { + seen.add(item.value); + output.push(item); + } + } + return new ArrayValue(output); + } default: throw new Error(`Unknown ArrayValue filter: ${filter.value}`); } } else if (operand instanceof StringValue) { switch (filter.value) { + // Filters that are also built-in functions case "length": - return new NumericValue(operand.value.length); case "upper": - return new StringValue(operand.value.toUpperCase()); case "lower": - return new StringValue(operand.value.toLowerCase()); case "title": - return new StringValue(titleCase(operand.value)); - case "capitalize": - return new StringValue(operand.value.charAt(0).toUpperCase() + operand.value.slice(1)); + case "capitalize": { + const builtin = operand.builtins.get(filter.value); + if (builtin instanceof FunctionValue) { + return builtin.value(/* no arguments */ [], environment); + } else if (builtin instanceof IntegerValue) { + return builtin; + } else { + throw new Error(`Unknown StringValue filter: ${filter.value}`); + } + } case "trim": return new StringValue(operand.value.trim()); case "indent": @@ -649,13 +786,27 @@ export class Interpreter { case "join": case "string": return operand; // no-op + case "int": { + const val = parseInt(operand.value, 10); + return new IntegerValue(isNaN(val) ? 0 : val); + } + case "float": { + const val = parseFloat(operand.value); + return new FloatValue(isNaN(val) ? 0.0 : val); + } default: throw new Error(`Unknown StringValue filter: ${filter.value}`); } - } else if (operand instanceof NumericValue) { + } else if (operand instanceof IntegerValue || operand instanceof FloatValue) { switch (filter.value) { case "abs": - return new NumericValue(Math.abs(operand.value)); + return operand instanceof IntegerValue + ? new IntegerValue(Math.abs(operand.value)) + : new FloatValue(Math.abs(operand.value)); + case "int": + return new IntegerValue(Math.floor(operand.value)); + case "float": + return new FloatValue(operand.value); default: throw new Error(`Unknown NumericValue filter: ${filter.value}`); } @@ -666,14 +817,27 @@ export class Interpreter { Array.from(operand.value.entries()).map(([key, value]) => new ArrayValue([new StringValue(key), value])) ); case "length": - return new NumericValue(operand.value.size); + return new IntegerValue(operand.value.size); default: throw new Error(`Unknown ObjectValue filter: ${filter.value}`); } + } else if (operand instanceof BooleanValue) { + switch (filter.value) { + case "bool": + return new BooleanValue(operand.value); + case "int": + return new IntegerValue(operand.value ? 1 : 0); + case "float": + return new FloatValue(operand.value ? 1.0 : 0.0); + case "string": + return new StringValue(operand.value ? "true" : "false"); + default: + throw new Error(`Unknown BooleanValue filter: ${filter.value}`); + } } throw new Error(`Cannot apply filter "${filter.value}" to type: ${operand.type}`); - } else if (node.filter.type === "CallExpression") { - const filter = node.filter as CallExpression; + } else if (filterNode.type === "CallExpression") { + const filter = filterNode as CallExpression; if (filter.callee.type !== "Identifier") { throw new Error(`Unknown filter: ${filter.callee.type}`); @@ -683,7 +847,7 @@ export class Interpreter { if (filterName === "tojson") { const [, kwargs] = this.evaluateArguments(filter.args, environment); const indent = kwargs.get("indent") ?? new NullValue(); - if (!(indent instanceof NumericValue || indent instanceof NullValue)) { + if (!(indent instanceof IntegerValue || indent instanceof NullValue)) { throw new Error("If set, indent must be a number"); } return new StringValue(toJSON(operand, indent.value)); @@ -705,6 +869,34 @@ export class Interpreter { } return new StringValue(value.join(separator.value)); + } else if (filterName === "int" || filterName === "float") { + const [args, kwargs] = this.evaluateArguments(filter.args, environment); + const defaultValue = + args.at(0) ?? kwargs.get("default") ?? (filterName === "int" ? new IntegerValue(0) : new FloatValue(0.0)); + + if (operand instanceof StringValue) { + const val = filterName === "int" ? parseInt(operand.value, 10) : parseFloat(operand.value); + return isNaN(val) ? defaultValue : filterName === "int" ? new IntegerValue(val) : new FloatValue(val); + } else if (operand instanceof IntegerValue || operand instanceof FloatValue) { + return operand; + } else if (operand instanceof BooleanValue) { + return filterName === "int" + ? new IntegerValue(operand.value ? 1 : 0) + : new FloatValue(operand.value ? 1.0 : 0.0); + } else { + throw new Error(`Cannot apply filter "${filterName}" to type: ${operand.type}`); + } + } else if (filterName === "default") { + const [args, kwargs] = this.evaluateArguments(filter.args, environment); + const defaultValue = args[0] ?? new StringValue(""); + const booleanValue = args[1] ?? kwargs.get("boolean") ?? new BooleanValue(false); + if (!(booleanValue instanceof BooleanValue)) { + throw new Error("`default` filter flag must be a boolean"); + } + if (operand instanceof UndefinedValue || (booleanValue.value && !operand.__bool__().value)) { + return defaultValue; + } + return operand; } if (operand instanceof ArrayValue) { @@ -780,8 +972,8 @@ export class Interpreter { const [args, kwargs] = this.evaluateArguments(filter.args, environment); - const width = args.at(0) ?? kwargs.get("width") ?? new NumericValue(4); - if (!(width instanceof NumericValue)) { + const width = args.at(0) ?? kwargs.get("width") ?? new IntegerValue(4); + if (!(width instanceof IntegerValue)) { throw new Error("width must be a number"); } const first = args.at(1) ?? kwargs.get("first") ?? new BooleanValue(false); @@ -794,13 +986,29 @@ export class Interpreter { ); return new StringValue(indented.join("\n")); } + case "replace": { + const replaceFn = operand.builtins.get("replace"); + if (!(replaceFn instanceof FunctionValue)) { + throw new Error("replace filter not available"); + } + const [args, kwargs] = this.evaluateArguments(filter.args, environment); + return replaceFn.value([...args, new KeywordArgumentsValue(kwargs)], environment); + } } throw new Error(`Unknown StringValue filter: ${filterName}`); } else { throw new Error(`Cannot apply filter "${filterName}" to type: ${operand.type}`); } } - throw new Error(`Unknown filter: ${node.filter.type}`); + throw new Error(`Unknown filter: ${filterNode.type}`); + } + + /** + * Evaluates expressions following the filter operation type. + */ + private evaluateFilterExpression(node: FilterExpression, environment: Environment): AnyRuntimeValue { + const operand = this.evaluate(node.operand, environment); + return this.applyFilter(operand, node.filter, environment); } /** @@ -821,6 +1029,17 @@ export class Interpreter { return new BooleanValue(node.negate ? !result : result); } + /** + * Evaluates expressions following the select operation type. + */ + private evaluateSelectExpression(node: SelectExpression, environment: Environment): AnyRuntimeValue { + const predicate = this.evaluate(node.test, environment); + if (!predicate.__bool__().value) { + return new UndefinedValue(); + } + return this.evaluate(node.lhs, environment); + } + /** * Evaluates expressions following the unary operation type. */ @@ -835,6 +1054,13 @@ export class Interpreter { } } + private evaluateTernaryExpression(node: Ternary, environment: Environment): AnyRuntimeValue { + const cond = this.evaluate(node.condition, environment); + return cond.__bool__().value + ? this.evaluate(node.trueExpr, environment) + : this.evaluate(node.falseExpr, environment); + } + private evalProgram(program: Program, environment: Environment): StringValue { return this.evaluateBlock(program.body, environment); } @@ -845,10 +1071,10 @@ export class Interpreter { let result = ""; for (const statement of statements) { - const lastEvaluated = this.evaluate(statement, environment); + const lastEvaluated: AnyRuntimeValue = this.evaluate(statement, environment); if (lastEvaluated.type !== "NullValue" && lastEvaluated.type !== "UndefinedValue") { - result += lastEvaluated.value; + result += lastEvaluated.toString(); } } @@ -889,13 +1115,13 @@ export class Interpreter { const step = this.evaluate(expr.step, environment); // Validate arguments - if (!(start instanceof NumericValue || start instanceof UndefinedValue)) { + if (!(start instanceof IntegerValue || start instanceof UndefinedValue)) { throw new Error("Slice start must be numeric or undefined"); } - if (!(stop instanceof NumericValue || stop instanceof UndefinedValue)) { + if (!(stop instanceof IntegerValue || stop instanceof UndefinedValue)) { throw new Error("Slice stop must be numeric or undefined"); } - if (!(step instanceof NumericValue || step instanceof UndefinedValue)) { + if (!(step instanceof IntegerValue || step instanceof UndefinedValue)) { throw new Error("Slice step must be numeric or undefined"); } @@ -927,7 +1153,7 @@ export class Interpreter { } value = object.value.get(property.value) ?? object.builtins.get(property.value); } else if (object instanceof ArrayValue || object instanceof StringValue) { - if (property instanceof NumericValue) { + if (property instanceof IntegerValue) { value = object.value.at(property.value); if (object instanceof StringValue) { value = new StringValue(object.value.at(property.value)); @@ -952,6 +1178,22 @@ export class Interpreter { if (node.assignee.type === "Identifier") { const variableName = (node.assignee as Identifier).value; environment.setVariable(variableName, rhs); + } else if (node.assignee.type === "TupleLiteral") { + const tuple = node.assignee as TupleLiteral; + if (!(rhs instanceof ArrayValue)) { + throw new Error(`Cannot unpack non-iterable type in set: ${rhs.type}`); + } + const arr = rhs.value; + if (arr.length !== tuple.value.length) { + throw new Error(`Too ${tuple.value.length > arr.length ? "few" : "many"} items to unpack in set`); + } + for (let i = 0; i < tuple.value.length; ++i) { + const elem = tuple.value[i]; + if (elem.type !== "Identifier") { + throw new Error(`Cannot unpack to non-identifier in set: ${elem.type}`); + } + environment.setVariable((elem as Identifier).value, arr[i]); + } } else if (node.assignee.type === "MemberExpression") { const member = node.assignee as MemberExpression; @@ -982,14 +1224,18 @@ export class Interpreter { let test, iterable; if (node.iterable.type === "SelectExpression") { const select = node.iterable as SelectExpression; - iterable = this.evaluate(select.iterable, scope); + iterable = this.evaluate(select.lhs, scope); test = select.test; } else { iterable = this.evaluate(node.iterable, scope); } - if (!(iterable instanceof ArrayValue)) { - throw new Error(`Expected iterable type in for loop: got ${iterable.type}`); + if (!(iterable instanceof ArrayValue || iterable instanceof ObjectValue)) { + throw new Error(`Expected iterable or object type in for loop: got ${iterable.type}`); + } + + if (iterable instanceof ObjectValue) { + iterable = iterable.keys(); } const items: Expression[] = []; @@ -1046,13 +1292,13 @@ export class Interpreter { // Update the loop variable // TODO: Only create object once, then update value? const loop = new Map([ - ["index", new NumericValue(i + 1)], - ["index0", new NumericValue(i)], - ["revindex", new NumericValue(items.length - i)], - ["revindex0", new NumericValue(items.length - i - 1)], + ["index", new IntegerValue(i + 1)], + ["index0", new IntegerValue(i)], + ["revindex", new IntegerValue(items.length - i)], + ["revindex0", new IntegerValue(items.length - i - 1)], ["first", new BooleanValue(i === 0)], ["last", new BooleanValue(i === items.length - 1)], - ["length", new NumericValue(items.length)], + ["length", new IntegerValue(items.length)], ["previtem", i > 0 ? items[i - 1] : new UndefinedValue()], ["nextitem", i < items.length - 1 ? items[i + 1] : new UndefinedValue()], ] as [string, AnyRuntimeValue][]); @@ -1135,8 +1381,39 @@ export class Interpreter { return new NullValue(); } + private evaluateCallStatement(node: CallStatement, environment: Environment): AnyRuntimeValue { + const callerFn = new FunctionValue((callerArgs: AnyRuntimeValue[], callerEnv: Environment) => { + const callBlockEnv = new Environment(callerEnv); + if (node.callerArgs) { + for (let i = 0; i < node.callerArgs.length; ++i) { + const param = node.callerArgs[i]; + if (param.type !== "Identifier") { + throw new Error(`Caller parameter must be an identifier, got ${param.type}`); + } + callBlockEnv.setVariable((param as Identifier).value, callerArgs[i] ?? new UndefinedValue()); + } + } + return this.evaluateBlock(node.body, callBlockEnv); + }); + + const [macroArgs, macroKwargs] = this.evaluateArguments(node.call.args, environment); + macroArgs.push(new KeywordArgumentsValue(macroKwargs)); + const fn = this.evaluate(node.call.callee, environment); + if (fn.type !== "FunctionValue") { + throw new Error(`Cannot call something that is not a function: got ${fn.type}`); + } + const newEnv = new Environment(environment); + newEnv.setVariable("caller", callerFn); + return (fn as FunctionValue).value(macroArgs, newEnv); + } + + private evaluateFilterStatement(node: FilterStatement, environment: Environment): AnyRuntimeValue { + const rendered = this.evaluateBlock(node.body, environment); + return this.applyFilter(rendered, node.filter, environment); + } + evaluate(statement: Statement | undefined, environment: Environment): AnyRuntimeValue { - if (statement === undefined) return new UndefinedValue(); + if (!statement) return new UndefinedValue(); switch (statement.type) { // Program @@ -1152,6 +1429,8 @@ export class Interpreter { return this.evaluateFor(statement as For, environment); case "Macro": return this.evaluateMacro(statement as Macro, environment); + case "CallStatement": + return this.evaluateCallStatement(statement as CallStatement, environment); case "Break": throw new BreakControl(); @@ -1159,14 +1438,12 @@ export class Interpreter { throw new ContinueControl(); // Expressions - case "NumericLiteral": - return new NumericValue(Number((statement as NumericLiteral).value)); + case "IntegerLiteral": + return new IntegerValue((statement as IntegerLiteral).value); + case "FloatLiteral": + return new FloatValue((statement as FloatLiteral).value); case "StringLiteral": return new StringValue((statement as StringLiteral).value); - case "BooleanLiteral": - return new BooleanValue((statement as BooleanLiteral).value); - case "NullLiteral": - return new NullValue((statement as NullLiteral).value); case "ArrayLiteral": return new ArrayValue((statement as ArrayLiteral).value.map((x) => this.evaluate(x, environment))); case "TupleLiteral": @@ -1195,9 +1472,16 @@ export class Interpreter { return this.evaluateBinaryExpression(statement as BinaryExpression, environment); case "FilterExpression": return this.evaluateFilterExpression(statement as FilterExpression, environment); + case "FilterStatement": + return this.evaluateFilterStatement(statement as FilterStatement, environment); case "TestExpression": return this.evaluateTestExpression(statement as TestExpression, environment); - + case "SelectExpression": + return this.evaluateSelectExpression(statement as SelectExpression, environment); + case "Ternary": + return this.evaluateTernaryExpression(statement as Ternary, environment); + case "Comment": + return new NullValue(); default: throw new SyntaxError(`Unknown node type: ${statement.type}`); } @@ -1210,7 +1494,7 @@ export class Interpreter { function convertToRuntimeValues(input: unknown): AnyRuntimeValue { switch (typeof input) { case "number": - return new NumericValue(input); + return Number.isInteger(input) ? new IntegerValue(input) : new FloatValue(input); case "string": return new StringValue(input); case "boolean": @@ -1253,7 +1537,8 @@ function toJSON(input: AnyRuntimeValue, indent?: number | null, depth?: number): case "NullValue": case "UndefinedValue": // JSON.stringify(undefined) -> undefined return "null"; - case "NumericValue": + case "IntegerValue": + case "FloatValue": case "StringValue": case "BooleanValue": return JSON.stringify(input.value); diff --git a/packages/jinja/src/utils.ts b/packages/jinja/src/utils.ts index 835fa3554..ae36b9d7a 100644 --- a/packages/jinja/src/utils.ts +++ b/packages/jinja/src/utils.ts @@ -52,3 +52,62 @@ export function slice(array: T[], start?: number, stop?: number, step = 1): T export function titleCase(value: string): string { return value.replace(/\b\w/g, (c) => c.toUpperCase()); } + +export function strftime_now(format: string): string { + return strftime(new Date(), format); +} + +/** + * A minimalistic implementation of Python's strftime function. + */ +export function strftime(date: Date, format: string): string { + // Set locale to undefined to use the default locale + const monthFormatterLong = new Intl.DateTimeFormat(undefined, { month: "long" }); + const monthFormatterShort = new Intl.DateTimeFormat(undefined, { month: "short" }); + + const pad2 = (n: number): string => (n < 10 ? "0" + n : n.toString()); + + return format.replace(/%[YmdbBHM%]/g, (token) => { + switch (token) { + case "%Y": + return date.getFullYear().toString(); + case "%m": + return pad2(date.getMonth() + 1); + case "%d": + return pad2(date.getDate()); + case "%b": + return monthFormatterShort.format(date); + case "%B": + return monthFormatterLong.format(date); + case "%H": + return pad2(date.getHours()); + case "%M": + return pad2(date.getMinutes()); + case "%%": + return "%"; + default: + return token; + } + }); +} + +function escapeRegExp(s: string): string { + return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +/** + * Function that mimics Python's string.replace() function. + */ +export function replace(str: string, oldvalue: string, newvalue: string, count?: number | null): string { + if (count === 0) return str; + let remaining = count == null || count < 0 ? Infinity : count; + // NB: Use a Unicode-aware global regex so unpaired surrogates won't match + const pattern = oldvalue.length === 0 ? new RegExp("(?=)", "gu") : new RegExp(escapeRegExp(oldvalue), "gu"); + return str.replaceAll(pattern, (match) => { + if (remaining > 0) { + --remaining; + return newvalue; + } + return match; + }); +} diff --git a/packages/jinja/test/e2e.test.js b/packages/jinja/test/e2e.test.js index 6e3250946..2f2f0607b 100644 --- a/packages/jinja/test/e2e.test.js +++ b/packages/jinja/test/e2e.test.js @@ -849,6 +849,153 @@ const TEST_CUSTOM_TEMPLATES = Object.freeze({ target: '<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|># System Preamble\nYou are in contextual safety mode. You will reject requests to generate child sexual abuse material and child exploitation material in your responses. You will accept to provide information and creative content related to violence, hate, misinformation or sex, but you will not provide any content that could directly or indirectly lead to harmful outcomes.\n\nYour information cutoff date is June 2024.\n\nYou have been trained on data in English, French, Spanish, Italian, German, Portuguese, Japanese, Korean, Modern Standard Arabic, Mandarin, Russian, Indonesian, Turkish, Dutch, Polish, Persian, Vietnamese, Czech, Hindi, Ukrainian, Romanian, Greek and Hebrew but have the ability to speak many more languages.\n\nYou have been trained to have advanced reasoning and tool-use capabilities and you should make best use of these skills to serve user\'s requests.\n\n## Tool Use\nThink about how you can make best use of the provided tools to help with the task and come up with a high level plan that you will execute first.\n\n0. Start by writing <|START_THINKING|> followed by a detailed step by step plan of how you will solve the problem. For each step explain your thinking fully and give details of required tool calls (if needed). Unless specified otherwise, you write your plan in natural language. When you finish, close it out with <|END_THINKING|>.\n You can optionally choose to skip this step when the user request is so straightforward to address that only a trivial plan would be needed.\n NOTE: You MUST skip this step when you are directly responding to the user\'s request without using any tools.\n\nThen carry out your plan by repeatedly executing the following steps.\n1. Action: write <|START_ACTION|> followed by a list of JSON-formatted tool calls, with each one containing "tool_name" and "parameters" fields.\n When there are multiple tool calls which are completely independent of each other (i.e. they can be executed in parallel), you should list them out all together in one step. When you finish, close it out with <|END_ACTION|>.\n2. Observation: you will then receive results of those tool calls in JSON format in the very next turn, wrapped around by <|START_TOOL_RESULT|> and <|END_TOOL_RESULT|>. Carefully observe those results and think about what to do next. Note that these results will be provided to you in a separate turn. NEVER hallucinate results.\n Every tool call produces a list of results (when a tool call produces no result or a single result, it\'ll still get wrapped inside a list). Each result is clearly linked to its originating tool call via its "tool_call_id".\n3. Reflection: start the next turn by writing <|START_THINKING|> followed by what you\'ve figured out so far, any changes you need to make to your plan, and what you will do next. When you finish, close it out with <|END_THINKING|>.\n You can optionally choose to skip this step when everything is going according to plan and no special pieces of information or reasoning chains need to be recorded.\n NOTE: You MUST skip this step when you are done with tool-use actions and are ready to respond to the user.\n\nYou can repeat the above 3 steps multiple times (could be 0 times too if no suitable tool calls are available or needed), until you decide it\'s time to finally respond to the user.\n\n4. Response: then break out of the loop and write <|START_RESPONSE|> followed by a piece of text which serves as a response to the user\'s last request. Use all previous tool calls and results to help you when formulating your response. When you finish, close it out with <|END_RESPONSE|>.\n\n## Available Tools\nHere is the list of tools that you have available to you.\nYou can ONLY use the tools listed here. When a tool is not listed below, it is NOT available and you should NEVER attempt to use it.\nEach tool is represented as a JSON object with fields like "name", "description", "parameters" (per JSON Schema), and optionally, "responses" (per JSON Schema).\n\n```json\n[\n {"name": "direct-injected-document", "description": "This is a special tool to directly inject user-uploaded documents into the chat as additional context. DO NOT use this tool by yourself!", "parameters": {"type": "object", "properties": {}, "required": []}, "responses": {"200": {"description": "Successfully returned a list of chunked text snippets from the directly uploaded documents.", "content": {"application/json": {"schema": {"type": "array", "items": {"type": "object", "required": ["url", "snippet"], "properties": {"url": {"type": "string", "description": "The url of the uploaded document."}, "snippet": {"type": "string", "description": "The text snippet for the returned document chunk."}}}}}}}}}\n]\n```\n\n# Default Preamble\nThe following instructions are your defaults unless specified elsewhere in developer preamble or user prompt.\n- Your name is Command.\n- You are a large language model built by Cohere.\n- You reply conversationally with a friendly and informative tone and often include introductory statements and follow-up questions.\n- If the input is ambiguous, ask clarifying follow-up questions.\n- Use Markdown-specific formatting in your response (for example to highlight phrases in bold or italics, create tables, or format code blocks).\n- Use LaTeX to generate mathematical notation for complex equations.\n- When responding in English, use American English unless context indicates otherwise.\n- When outputting responses of more than seven sentences, split the response into paragraphs.\n- Prefer the active voice.\n- Adhere to the APA style guidelines for punctuation, spelling, hyphenation, capitalization, numbers, lists, and quotation marks. Do not worry about them for other elements such as italics, citations, figures, or references.\n- Use gender-neutral pronouns for unspecified persons.\n- Limit lists to no more than 10 items unless the list is a set of finite instructions, in which case complete the list.\n- Use the third person when asked to write a summary.\n- When asked to extract values from source material, use the exact form, separated by commas.\n- When generating code output, please provide an explanation after the code.\n- When generating code output without specifying the programming language, please generate Python code.\n- If you are asked a question that requires reasoning, first think through your answer, slowly and step by step, then answer.<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|USER_TOKEN|>What has Man always dreamed of?<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|><|START_THINKING|>I will look through the document to address the users needs.<|END_THINKING|><|START_ACTION|>[\n {"tool_call_id": "0", "tool_name": "direct-injected-document", "parameters": {}}\n]<|END_ACTION|><|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|><|START_TOOL_RESULT|>[\n {\n "tool_call_id": "0",\n "results": {\n "0": {"heading": "The Moon: Our Age-Old Foe", "body": "Man has always dreamed of destroying the moon. In this essay, I shall..."},\n "1": {"heading": "Love is all you need", "body": "Man\'s dream has always been to find love. This profound lesson..."}\n },\n "is_error": null\n }\n]<|END_TOOL_RESULT|><|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>', }, + "openbmb/MiniCPM3-4B": { + chat_template: + "{%- macro json_to_python_type(param_name, json_spec) %}\n{%- set basic_type_map = {\n 'string': 'str',\n 'number': 'float',\n 'integer': 'int',\n 'boolean': 'bool',\n 'null': 'None'\n} %}\n\n{%- if json_spec.enum %}\n {{- param_name|title }}\n{%- elif basic_type_map[json_spec.type] is defined %}\n {{- basic_type_map[json_spec.type] }}\n{%- elif json_spec.type == 'array' %}\n {{- 'List[' + json_to_python_type(param_name, json_spec['items']) + ']' }}\n{%- elif json_spec.type == 'object' %}\n {{- 'Dict[str, ' + json_to_python_type(param_name, json_spec.additionalProperties if json_spec.additionalProperties else 'Any') + ']' if not json_spec.properties else param_name|title }}\n{%- elif json_spec.type is iterable %}\n {{- 'Union[' }}\n {%- for t in json_spec.type %}\n {{- json_to_python_type(param_name, {'type': t}) }}\n {{- ', ' if not loop.last }}\n {%- endfor %}\n {{- ']' }}\n{%- else %}\n {{- 'Any' }}\n{%- endif %}\n{%- endmacro %}\n\n{%- macro object_to_fields(json_spec, field_indent) %}\n {%- set o_ns = namespace(f = caller()) %}\n {%- for param_name, param_fields in json_spec.properties|items %}\n {%- if param_fields.enum %}\n {{- '\\n\\nclass ' + param_name|title + '(Enum):\\n' }}\n {%- for enum_option in param_fields.enum %}\n {{- ' enum_' + loop.index0|string + ' = ' + enum_option|tojson + '\\n' }}\n {%- endfor %}\n {%- elif param_fields.type == 'object' and param_fields.properties %}\n {%- call object_to_fields(param_fields, ' ') %}\n {{- '\\n\\nclass ' + param_name|title + '(BaseModel):\\n' }}\n {%- endcall %}\n {%- elif param_fields.type == 'array' and param_fields['items'] and param_fields['items'].type == 'object' and param_fields['items'].properties %}\n {%- call object_to_fields(param_fields['items'], ' ') %}\n {{- '\\n\\nclass ' + param_name|title + '(BaseModel):\\n' }}\n {%- endcall %}\n {%- endif %}\n {%- set param_default = param_fields.default|tojson if param_fields.default is string else param_fields.default|string if param_fields.default is defined else 'None' %}\n {%- set o_ns.f = o_ns.f + field_indent + param_name + ': ' %}\n {%- set o_ns.f = o_ns.f + ('Optional[' + json_to_python_type(param_name, param_fields) + ']' if param_name not in json_spec.required else json_to_python_type(param_name, param_fields)) %}\n {%- if not param_fields.title and not param_fields.description and not param_fields.pattern %}\n {%- set o_ns.f = o_ns.f + (' = ' + param_default if param_name not in json_spec.required else '') %}\n {%- else %}\n {%- set o_ns.f = o_ns.f + (' = Field(...' if param_name in json_spec.required else ' = Field(' + param_default) %}\n {%- set o_ns.f = o_ns.f + (', description=' + param_fields.description|tojson if param_fields.description else '') %}\n {%- set o_ns.f = o_ns.f + (', regex=' + param_fields.pattern|tojson if param_fields.pattern else '') %}\n {%- set o_ns.f = o_ns.f + (', title=' + param_fields.title|tojson if param_fields.title else '') %}\n {%- set o_ns.f = o_ns.f + ')' %}\n {%- endif %}\n {%- set o_ns.f = o_ns.f + '\\n' %}\n {%- endfor %}\n {{- o_ns.f }}\n{%- endmacro %}\n\n{%- macro tool_parser(tools) %}\n{%- for tool in tools %}\n {%- if tool.type is not defined or tool.type == 'function' %}\n {%- if tool.function is defined %}\n {%- set tool = tool.function %}\n {%- endif %}\n {%- set tool_params = tool.parameters if tool.parameters is defined else none %}\n {%- call object_to_fields(tool_params, ' ') %}\n {{- '\\n\\ndef ' + tool.name + '(' }}\n {%- if tool_params %}\n {%- for param_name, param_fields in tool_params.properties|items %}\n {%- set param_default = param_fields.default|tojson if param_fields.default is string else param_fields.default|string if param_fields.default is defined else 'None' %}\n {{- ', ' if loop.index0 != 0 }}\n {{- param_name }}\n {{- '=' + param_default if param_name not in tool_params.required }}\n {%- endfor %}\n {%- endif %}\n {{- '):\\n \"\"\"' }}\n {{- tool.description }}\n {{- '\\n\\n Args:\\n' if tool_params else '\\n' }}\n {%- endcall %}\n {{- ' \"\"\"\\n' }}\n {%- endif %}\n{%- endfor %}\n{%- endmacro %}\n\n{%- if messages[0]['role'] == 'system' %}\n {%- set loop_messages = messages[1:] %}\n {%- set system_message = messages[0]['content'] %}\n{%- else %}\n {%- set loop_messages = messages %}\n {%- set system_message = '' %}\n{%- endif %}\n{{- '<|im_start|>system\\n' + system_message if system_message or tools }}\n{%- if tools %}\n {{- '\\n# Functions\\nHere is a list of functions that you can invoke:\\n```python\\nfrom enum import Enum\\nfrom typing import List, Dict, Optional\\nfrom pydantic import BaseModel, Field\\n\\n' }}\n {{- tool_parser(tools) }}\n {{- \"\\n```\\n\\n# Function Call Rule and Output Format\\n- If the user's question can be answered without calling any function, please answer the user's question directly. In this situation, you should return your thought and answer the user's question directly.\\n- If the user cannot be answered without calling any function, and the user does not provide enough information to call functions, please ask the user for more information. In this situation, you should return your thought and ask the user for more information.\\n- If the user's question cannot be answered without calling any function, and the user has provided enough information to call functions to solve it, you should call the functions. In this situation, the assistant should return your thought and call the functions.\\n- Use default parameters unless the user has specified otherwise.\\n- You should answer in the following format:\\n\\n<|thought_start|>\\n{explain why the user's question can be answered without calling a function or why you should ask the user for more information or why you should call one or more functions and your plan to solve the user's question.}\\n<|thought_end|>\\n<|tool_call_start|>\\n```python\\nfunc1(params_name=params_value, params_name2=params_value2...)\\nfunc2(params)\\n```\\n<|tool_call_end|>\\n{answer the user's question directly or ask the user for more information}\" }}\n{%- endif %}\n{{- '<|im_end|>\\n' if system_message or tools }}\n{%- for message in loop_messages %}\n {%- set content = message.content %}\n {%- if message.role == 'assistant' and message.tool_calls %}\n {{- '<|im_start|>' + message.role + '\\n' }}\n {{- '<|thought_start|>\\n' + message.thought + '\\n<|thought_end|>\\n' if message.thought }}\n {{- '<|tool_call_start|>\\n```python\\n' }}\n {%- for tool_call in message.tool_calls %}\n {%- if tool_call.function is defined %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- tool_call.name + '(' }}\n {%- if tool_call.arguments is defined and tool_call.arguments|length > 0 %}\n {%- for param_name, param_value in tool_call.arguments|items %}\n {{- param_name + '=' + param_value|tojson }}\n {{- ',' if not loop.last }}\n {%- endfor %}\n {%- endif %}\n {{- ')\\n' }}\n {%- endfor %}\n {{- '```\\n<|tool_call_end|>\\n' }}\n {{- content if content and not content.startswith('<|tool_call_start|>') }}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == 'assistant' and message.thought %}\n {{- '<|im_start|>' + message.role + '\\n' + '<|thought_start|>\\n' + message.thought + '\\n<|thought_end|>\\n' + content + '<|im_end|>\\n' }}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endfor %}\n\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n{%- endif %}", + data: { + // Example adapted from https://github.com/OpenBMB/MiniCPM/blob/de20166b6357abb3338ac6b5d1521dcf4edb14dd/demo/minicpm3/function_call/function_calling.py#L30-L76 + messages: [ + { + role: "system", + content: "You are a helpful customer support assistant. Use the supplied tools to assist the user.", + }, + { + role: "user", + content: "Hi, can you tell me the delivery date for my order? The order id is 1234 and 4321.", + }, + { + content: "", + tool_calls: [ + { + type: "function", + function: { + name: "get_delivery_date", + arguments: { order_id: "1234" }, + }, + id: "call_b4ab0b4ec4b5442e86f017fe0385e22e", + }, + { + type: "function", + function: { + name: "get_delivery_date", + arguments: { order_id: "4321" }, + }, + id: "call_628965479dd84794bbb72ab9bdda0c39", + }, + ], + role: "assistant", + }, + { + role: "tool", + content: '{"delivery_date": "2024-09-05", "order_id": "1234"}', + tool_call_id: "call_b4ab0b4ec4b5442e86f017fe0385e22e", + }, + { + role: "tool", + content: '{"delivery_date": "2024-09-05", "order_id": "4321"}', + tool_call_id: "call_628965479dd84794bbb72ab9bdda0c39", + }, + { + content: "Both your orders will be delivered on 2024-09-05.", + role: "assistant", + thought: "\nI have the information you need, both orders will be delivered on the same date, 2024-09-05.\n", + }, + ], + add_generation_prompt: true, + }, + target: + '<|im_start|>system\nYou are a helpful customer support assistant. Use the supplied tools to assist the user.<|im_end|>\n<|im_start|>user\nHi, can you tell me the delivery date for my order? The order id is 1234 and 4321.<|im_end|>\n<|im_start|>assistant\n<|tool_call_start|>\n```python\nget_delivery_date(order_id="1234")\nget_delivery_date(order_id="4321")\n```\n<|tool_call_end|>\n<|im_end|>\n<|im_start|>tool\n{"delivery_date": "2024-09-05", "order_id": "1234"}<|im_end|>\n<|im_start|>tool\n{"delivery_date": "2024-09-05", "order_id": "4321"}<|im_end|>\n<|im_start|>assistant\n<|thought_start|>\n\nI have the information you need, both orders will be delivered on the same date, 2024-09-05.\n\n<|thought_end|>\nBoth your orders will be delivered on 2024-09-05.<|im_end|>\n<|im_start|>assistant\n', + }, + "ai21labs/AI21-Jamba-Large-1.6": { + chat_template: + '{# Variables #}\n{% set ns = namespace(message_count=0, is_last_checked_defined=False) %}\n{##}\n{% set bom_str = bom_str or "<|bom|>" %}\n{% set eom_str = eom_str or "<|eom|>" %}\n{% set default_system_message = default_system_message or "" %}\n{##}\n{% set documents_prefix = "" %}\n{% set documents_suffix = "" %}\n{% set tool_definitions_prefix = "" %}\n{% set tool_definitions_suffix = "" %}\n{% set active_modes_prefix = "" %}\n{% set active_modes_suffix = "" %}\n{##}\n{% set tool_calls_prefix = "" %}\n{% set tool_calls_suffix = "" %}\n{% set citations_prefix = "" %}\n{% set citations_suffix = "" %}\n{##}\n{% if add_generation_prompt is not defined %}\n {% set add_generation_prompt = True %}\n{% endif %}\n{% set role_to_predict = role_to_predict or "assistant" %}\n{% if messages|length > 0 and messages[0].role == "system" %}\n {% set system_message = messages[0].content %}\n {% set loop_messages = messages[1:] %}\n{% else %}\n {% set system_message = default_system_message %}\n {% set loop_messages = messages %}\n{% endif %}\n{##}\n{##}\n{# Macros #}\n{% macro handle_tool_definitions(tools) %}\n {{- tool_definitions_prefix -}}\n {{- "\\n# Tools" -}}\n {{- "\\n\\n## Functions" -}}\n {% for tool in tools %}\n {% set _ = is_param_set(tool, field="type") %}\n {% set is_tool_type_set = ns.is_last_checked_defined %}\n {% if is_tool_type_set %}\n {% if tool.type == "function" %}\n {% set tool = tool.function %}\n {% else %}\n {{ raise_exception("Currently, the only supported tool type is `function`") }}\n {% endif %}\n {% endif %}\n {{- "\\n\\n" + (tool|tojson(indent=2)) -}}\n {% endfor %}\n {{- "\\n" + tool_definitions_suffix -}}\n{% endmacro %}\n{##}\n{% macro handle_first_system_message(system_message, tools) %}\n {{- bom_str + handle_role("system") -}}\n {% set _ = is_param_set(system_message) %}\n {% set is_system_message_set = ns.is_last_checked_defined %}\n {% if is_system_message_set %}\n {{- system_message -}}\n {% endif %}\n {% set _ = is_param_set(tools, check_length=True) %}\n {% set is_tools_set = ns.is_last_checked_defined %}\n {% if is_tools_set %}\n {% if system_message %}\n {{- "\\n\\n" -}}\n {% endif %}\n {{- handle_tool_definitions(tools) -}}\n {% endif %}\n {% set ns.message_count = ns.message_count + 1 %}\n{% endmacro %}\n{##}\n{% macro handle_tool_calls(tool_calls) %}\n {{- tool_calls_prefix + "[\\n" -}}\n {% for tool_call in tool_calls %}\n {% set _ = is_param_set(tool_call, field="function") %}\n {% set is_tool_call_function_set = ns.is_last_checked_defined %}\n {% if is_tool_call_function_set %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {% set arguments = tool_call.arguments %}\n {% if arguments is not string %}\n {%- set arguments = arguments|tojson -%}\n {%- endif %}\n {{ "{\\"name\\": \\"" + tool_call.name + "\\", \\"arguments\\": " + arguments + "}" -}}\n {% if not loop.last %}\n {{- "," }}\n {% endif %}\n {% endfor %}\n {{- "\\n]" + tool_calls_suffix -}}\n{% endmacro %}\n{##}\n{% macro handle_documents(documents) %}\n {{- documents_prefix -}}\n {{- "\\n# Documents" -}}\n {{- "\\n\\nYou can use the following documents for reference:" -}}\n {% for doc in documents %}\n {{- "\\n\\n## Document ID: " + loop.index0|string -}}\n {% set _ = is_param_set(doc, field="title") %}\n {% set is_doc_title_set = ns.is_last_checked_defined %}\n {% if is_doc_title_set %}\n {{- "\\nTitle: " + doc.title -}}\n {% endif %}\n {% for key, value in doc.items() %}\n {% if key not in ["title", "text"] %}\n {{- "\\n" + key|title + ": " + value|string -}}\n {% endif %}\n {% endfor %}\n {{- "\\nText: " + doc.text -}}\n {% endfor %}\n {{- "\\n" + documents_suffix -}}\n{% endmacro %}\n{##}\n{% macro handle_knobs(knobs) %}\n {{- active_modes_prefix -}}\n {{- "\\n# Active Modes" -}}\n {{ "\\n\\nThe following modes configure the format or style of your responses. You should adhere to all currently" -}}\n {{ " active modes simultaneously." -}}\n {% if knobs.citation_mode == "fast" %}\n {{- "\\n\\n## Citation Mode" -}}\n {{- "\\n\\nProvide a list of references only for the documents you base your response on. Format your response" -}}\n {{ " with the original answer followed by a citation section. Use this template:" -}}\n {{ " `{answer}" + citations_prefix + "DOCUMENT_IDS" + citations_suffix + "`, where DOCUMENT_IDS are the relevant document numbers" -}}\n {{ " (e.g. [2, 5, 9]), or [] if the answer cannot be supported by the provided documents." -}}\n {% endif %}\n {% if knobs.response_format == "json_object" %}\n {{- "\\n\\n## JSON Mode" -}}\n {{ "\\n\\nProvide your response in JSON format. Adhere strictly to any schema given by the user." -}}\n {{ " If an appropriate JSON format exists, use it without modification." -}}\n {% endif %}\n {{- "\\n" + active_modes_suffix -}}\n{% endmacro %}\n{##}\n{% macro get_last_user_index(messages) %}\n {% set ns.last_user_index = 0 %}\n {% for message in messages %}\n {% if message.role == \'user\' %}\n {% set ns.last_user_index = loop.index0 %}\n {% endif %}\n {% endfor %}\n {{- ns.last_user_index -}}\n{% endmacro %}\n{##}\n{% macro handle_last_system_message(documents, knobs, use_documents, use_knobs) %}\n {{- bom_str + handle_role("system") -}}\n {% set macros_to_call = [] %}\n {% set params_for_macros = [] %}\n {% if use_documents %}\n {% set macros_to_call = macros_to_call + [handle_documents] %}\n {% set params_for_macros = params_for_macros + [[documents]] %}\n {% endif %}\n {% if use_knobs %}\n {% set macros_to_call = macros_to_call + [handle_knobs] %}\n {% set params_for_macros = params_for_macros + [[knobs]] %}\n {% endif %}\n {% for i in range(macros_to_call|length) %}\n {% if i > 0 %}\n {{- "\\n\\n" -}}\n {% endif %}\n {{- macros_to_call[i](*params_for_macros[i]) -}}\n {% endfor %}\n {% set ns.message_count = ns.message_count + 1 %}\n{% endmacro %}\n{##}\n{% macro handle_role(role, add_space=True) %}\n {{- "<|" + role + "|>" -}}\n {% if add_space %}\n {{- " " -}}\n {% endif %}\n{% endmacro %}\n{##}\n{% macro is_param_set(param, field=none, check_length=False) %}\n {% if field is not none %}\n {% if field in param %}\n {% set param = param[field] %}\n {% else %}\n {% set param = none %}\n {% endif %}\n {% endif %}\n {% set is_defined = param is defined and param is not none %}\n {% if check_length %}\n {% set ns.is_last_checked_defined = is_defined and param|length > 0 %}\n {% else %}\n {% set ns.is_last_checked_defined = is_defined %}\n {% endif %}\n{% endmacro %}\n{##}\n{##}\n{# Template #}\n{% if bos_token is defined and bos_token is not none %}\n {{- bos_token -}}\n{% endif %}\n{% set _ = is_param_set(system_message) %}\n{% set is_system_message_set = ns.is_last_checked_defined %}\n{% set _ = is_param_set(tools, check_length=True) %}\n{% set is_tools_set = ns.is_last_checked_defined %}\n{% set has_system_message = (is_system_message_set or is_tools_set) %}\n{% if has_system_message %}\n {{- handle_first_system_message(system_message, tools) -}}\n{% endif %}\n{% set last_user_index = get_last_user_index(loop_messages)|int %}\n{% for message in loop_messages %}\n {% if loop.index0 == last_user_index %}\n {% set _ = is_param_set(documents, check_length=True) %}\n {% set use_documents = ns.is_last_checked_defined %}\n {% set _ = is_param_set(knobs) %}\n {% set use_knobs = ns.is_last_checked_defined and knobs.is_set %}\n {% set add_last_system_message = use_documents or use_knobs %}\n {% if add_last_system_message %}\n {% if ns.message_count > 0 %}\n {{- eom_str -}}\n {% endif %}\n {{- handle_last_system_message(documents, knobs, use_documents, use_knobs) -}}\n {% endif %}\n {% endif %}\n {% set role = message.role %}\n {% set _ = is_param_set(message, field="name") %}\n {% set is_message_name_set = ns.is_last_checked_defined %}\n {% if is_message_name_set %}\n {% set message_prefix = handle_role(role) + "(" + message.name + ")" %}\n {% else %}\n {% set message_prefix = handle_role(role) %}\n {% endif %}\n {% set content = (message.content or "") %}\n {% if content is not string %}\n {% set content = content|tojson %}\n {% endif %}\n {% if ns.message_count > 0 %}\n {{- eom_str -}}\n {% endif %}\n {{- bom_str + message_prefix + content -}}\n {% set _ = is_param_set(message, field="tool_calls", check_length=True) %}\n {% set is_tool_calls_set = ns.is_last_checked_defined %}\n {% if role == "assistant" and is_tool_calls_set %}\n {{- handle_tool_calls(message.tool_calls) -}}\n {% endif %}\n {% set _ = is_param_set(message, field="citations", check_length=False) %}\n {% set is_citations_set = ns.is_last_checked_defined %}\n {% if role == "assistant" and is_citations_set and knobs.is_set and knobs.citation_mode != "off" %}\n {{- citations_prefix + message.citations|map(attribute="document_id")|list|string + citations_suffix -}}\n {% endif %}\n {% set ns.message_count = ns.message_count + 1 %}\n{% endfor %}\n{% if add_generation_prompt %}\n {% if ns.message_count > 0 %}\n {{- eom_str -}}\n {% endif %}\n {{- bom_str + handle_role(role_to_predict, add_space=False) -}}\n {% set _ = is_param_set(generation_preamble) %}\n {% set is_generation_preamble_set = ns.is_last_checked_defined %}\n {% if is_generation_preamble_set and generation_preamble.strip() != "" %}\n {{- " " + generation_preamble -}}\n {% endif %}\n {% set ns.message_count = ns.message_count + 1 %}\n{% else %}\n {% if ns.message_count > 0 %}\n {{- eom_str -}}\n {% endif %}\n{% endif %}\n', + data: { + messages: [ + { + role: "system", + content: + "You are an ancient oracle who speaks in cryptic but wise phrases, always hinting at deeper meanings.", + }, + { role: "user", content: "Hello!" }, + ], + bos_token: "<|startoftext|>", + }, + target: + "<|startoftext|><|bom|><|system|> You are an ancient oracle who speaks in cryptic but wise phrases, always hinting at deeper meanings.<|eom|><|bom|><|user|> Hello!<|eom|><|bom|><|assistant|>", + }, + + "meta-llama/Llama-3.2-11B-Vision-Instruct": { + chat_template: + "{{- bos_token }}\n{%- if custom_tools is defined %}\n {%- set tools = custom_tools %}\n{%- endif %}\n{%- if not tools_in_user_message is defined %}\n {%- set tools_in_user_message = true %}\n{%- endif %}\n{%- if not date_string is defined %}\n {%- if strftime_now is defined %}\n {%- set date_string = strftime_now(\"%d %b %Y\") %}\n {%- else %}\n {%- set date_string = \"26 Jul 2024\" %}\n {%- endif %}\n{%- endif %}\n{%- if not tools is defined %}\n {%- set tools = none %}\n{%- endif %}\n\n{#- This block extracts the system message, so we can slot it into the right place. #}\n{%- if messages[0]['role'] == 'system' %}\n {%- set system_message = messages[0]['content']|trim %}\n {%- set messages = messages[1:] %}\n {%- set user_supplied_system_message = true %}\n{%- else %}\n {%- set system_message = \"\" %}\n {%- set user_supplied_system_message = false %}\n{%- endif %}\n\n{#- Find out if there are any images #}\n{% set image_ns = namespace(has_images=false) %} \n{%- for message in messages %}\n {%- for content in message['content'] %}\n {%- if content['type'] == 'image' %}\n {%- set image_ns.has_images = true %}\n {%- endif %}\n {%- endfor %}\n{%- endfor %}\n\n{#- System message if there are no images, or if the user supplied one #}\n{%- if user_supplied_system_message or not image_ns.has_images %}\n {{- \"<|start_header_id|>system<|end_header_id|>\\n\\n\" }}\n {%- if tools is not none %}\n {{- \"Environment: ipython\\n\" }}\n {%- endif %}\n {{- \"Cutting Knowledge Date: December 2023\\n\" }}\n {{- \"Today Date: \" + date_string + \"\\n\\n\" }}\n {%- if tools is not none and not tools_in_user_message %}\n {{- \"You have access to the following functions. To call a function, please respond with JSON for a function call.\" }}\n {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n {{- \"Do not use variables.\\n\\n\" }}\n {%- for t in tools %}\n {{- t | tojson(indent=4) }}\n {{- \"\\n\\n\" }}\n {%- endfor %}\n {%- endif %}\n {{- system_message }}\n {{- \"<|eot_id|>\" }}\n{%- endif %}\n\n{#- Custom tools are passed in a user message with some extra guidance #}\n{%- if tools_in_user_message and not tools is none %}\n {#- Extract the first user message so we can plug it in here #}\n {%- if messages | length != 0 %}\n {%- set first_user_message = messages[0]['content']|trim %}\n {%- set messages = messages[1:] %}\n {%- else %}\n {{- raise_exception(\"Cannot put tools in the first user message when there's no first user message!\") }}\n{%- endif %}\n {{- '<|start_header_id|>user<|end_header_id|>\\n\\n' -}}\n {{- \"Given the following functions, please respond with a JSON for a function call \" }}\n {{- \"with its proper arguments that best answers the given prompt.\\n\\n\" }}\n {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n {{- \"Do not use variables.\\n\\n\" }}\n {%- for t in tools %}\n {{- t | tojson(indent=4) }}\n {{- \"\\n\\n\" }}\n {%- endfor %}\n {{- first_user_message + \"<|eot_id|>\"}}\n{%- endif %}\n\n{%- for message in messages %}\n {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}\n {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\\n\\n' }}\n {%- if message['content'] is string %}\n {{- message['content'] }}\n {%- else %}\n {%- for content in message['content'] %}\n {%- if content['type'] == 'image' %}\n {{- '<|image|>' }}\n {%- elif content['type'] == 'text' %}\n {{- content['text'] }}\n {%- endif %}\n {%- endfor %}\n {%- endif %}\n {{- '<|eot_id|>' }}\n {%- elif 'tool_calls' in message %}\n {%- if not message.tool_calls|length == 1 %}\n {{- raise_exception(\"This model only supports single tool-calls at once!\") }}\n {%- endif %}\n {%- set tool_call = message.tool_calls[0].function %}\n {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n {{- '{\"name\": \"' + tool_call.name + '\", ' }}\n {{- '\"parameters\": ' }}\n {{- tool_call.arguments | tojson }}\n {{- \"}\" }}\n {{- \"<|eot_id|>\" }}\n {%- elif message.role == \"tool\" or message.role == \"ipython\" %}\n {{- \"<|start_header_id|>ipython<|end_header_id|>\\n\\n\" }}\n {%- if message.content is mapping or message.content is iterable %}\n {{- message.content | tojson }}\n {%- else %}\n {{- message.content }}\n {%- endif %}\n {{- \"<|eot_id|>\" }}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' }}\n{%- endif %}\n", + data: { + // Example adapted from https://huggingface.co/meta-llama/Llama-3.2-11B-Vision-Instruct#use-with-transformers + messages: [ + { + role: "user", + content: [{ type: "image" }, { type: "text", text: "If I had to write a haiku for this one, it would be: " }], + }, + ], + bos_token: "<|begin_of_text|>", + add_generation_prompt: true, + }, + target: + "<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n<|image|>If I had to write a haiku for this one, it would be: <|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n", + }, + "meta-llama/Llama-Guard-3-11B-Vision": { + chat_template: + '{%- if messages|length % 2 == 0 -%}\n {%- set role = \'Agent\' -%}\n{%- else -%}\n {%- set role = \'User\' -%}\n{%- endif -%}\n{%- if categories is not defined -%}\n {%- \n set categories = ({\n "S1": "Violent Crimes.",\n "S2": "Non-Violent Crimes.",\n "S3": "Sex Crimes.",\n "S4": "Child Exploitation.",\n "S5": "Defamation.",\n "S6": "Specialized Advice.",\n "S7": "Privacy.",\n "S8": "Intellectual Property.",\n "S9": "Indiscriminate Weapons.",\n "S10": "Hate.",\n "S11": "Self-Harm.",\n "S12": "Sexual Content.",\n "S13": "Elections.",\n })\n -%}\n{%- endif -%}\n\n{{- "<|begin_of_text|><|start_header_id|>user<|end_header_id|>\\n\\n" -}}\n{%- set ns = namespace(contains_image=false) -%}\n{%- for message in messages -%}\n {%- for content in message[\'content\'] | selectattr(\'type\', \'equalto\', \'image\') -%}\n {%- set ns.contains_image = true -%}\n {%- endfor -%}\n{%- endfor -%}\n{%- if ns.contains_image -%}\n{{- "<|image|> "-}}\n{%- endif -%}\n{{- "Task: Check if there is unsafe content in \'" + role + "\' messages in conversations according our safety policy with the below categories.\\n\\n" -}}\n{{- "\\n"-}}\n\n{%- for key in categories -%}\n {%- if key not in excluded_category_keys -%}\n {{-key + ": " + categories[key] + "\\n" -}}\n {%- endif -%}\n{%- endfor -%}\n\n{{- "\\n\\n"-}}\n{{- "\\n\\n"-}}\n{%- for message in messages -%}\n {%- if (message[\'role\'] == \'user\') != (loop.index0 % 2 == 0) -%}\n {{ raise_exception(\'Conversation roles must alternate user/assistant/user/assistant/...\')-}}\n {%- endif -%}\n {%- set content = message[\'content\'] -%}\n {%- if message[\'role\'] == \'user\' -%}\n {%- set role = \'User\' -%}\n {%- elif message[\'role\'] == \'assistant\' -%}\n {%- set role = \'Agent\' -%}\n {%- endif -%}\n {%- for content in message[\'content\'] | selectattr(\'type\', \'equalto\', \'text\') -%}\n{{- role + \': \' + content[\'text\'] | trim + \'\\n\\n\'-}}\n {%- endfor -%}\n{%- endfor -%}\n{{- "\\n\\n"-}}\n{{- "Provide your safety assessment for ONLY THE LAST " + role + " message in the above conversation:\\n"-}}\n{{- " - First line must read \'safe\' or \'unsafe\'.\\n"-}}\n{{- " - If unsafe, a second line must include a comma-separated list of violated categories. <|eot_id|><|start_header_id|>assistant<|end_header_id|>"-}}\n', + data: { + messages: [ + { + role: "user", + content: [ + { + type: "text", + text: "What is the recipe for mayonnaise?", + }, + { + type: "image", + }, + ], + }, + ], + bos_token: "<|begin_of_text|>", + add_generation_prompt: true, + }, + target: + "<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n<|image|> Task: Check if there is unsafe content in 'User' messages in conversations according our safety policy with the below categories.\n\n\nS1: Violent Crimes.\nS2: Non-Violent Crimes.\nS3: Sex Crimes.\nS4: Child Exploitation.\nS5: Defamation.\nS6: Specialized Advice.\nS7: Privacy.\nS8: Intellectual Property.\nS9: Indiscriminate Weapons.\nS10: Hate.\nS11: Self-Harm.\nS12: Sexual Content.\nS13: Elections.\n\n\n\n\nUser: What is the recipe for mayonnaise?\n\n\n\nProvide your safety assessment for ONLY THE LAST User message in the above conversation:\n - First line must read 'safe' or 'unsafe'.\n - If unsafe, a second line must include a comma-separated list of violated categories. <|eot_id|><|start_header_id|>assistant<|end_header_id|>", + }, +}); + +/** + * Formatting tests for custom templates. + */ +const TEST_CUSTOM_FORMATTING = Object.freeze({ + "HuggingFaceTB/SmolVLM-Instruct": { + //https://huggingface.co/HuggingFaceTB/SmolVLM-Instruct?chat_template=default + chat_template: `<|im_start|>{% for message in messages %}{{message['role'] | capitalize}}{% if message['content'][0]['type'] == 'image' %}{{':'}}{% else %}{{': '}}{% endif %}{% for line in message['content'] %}{% if line['type'] == 'text' %}{{line['text']}}{% elif line['type'] == 'image' %}{{ '' }}{% endif %}{% endfor %} +{% endfor %}{% if add_generation_prompt %}{{ 'Assistant:' }}{% endif %}`, + target: `{{- "<|im_start|>" -}} +{%- for message in messages -%} + {{- message["role"] | capitalize -}} + {%- if message["content"][0]["type"] == "image" -%} + {{- ":" -}} + {%- else -%} + {{- ": " -}} + {%- endif -%} + {%- for line in message["content"] -%} + {%- if line["type"] == "text" -%} + {{- line["text"] -}} + {%- elif line["type"] == "image" -%} + {{- "" -}} + {%- endif -%} + {%- endfor -%} + {{- "\\n" -}} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{- "Assistant:" -}} +{%- endif -%}`, + }, }); function render({ chat_template, data, target }) { @@ -862,23 +1009,37 @@ function render({ chat_template, data, target }) { expect(formattedTemplateOutput).toEqual(result); } +function format({ chat_template, target }) { + const template = new Template(chat_template); + const formatted = template.format({ indent: 4 }); + expect(formatted).toEqual(target); +} + describe("End-to-end tests", () => { - describe("Default templates", async () => { + describe("Default templates", () => { for (const [model_type, test_data] of Object.entries(TEST_DEFAULT_TEMPLATES)) { - it(model_type, async () => { + it(model_type, () => { render(test_data); }); } }); - describe("Custom templates", async () => { + describe("Custom templates", () => { for (const [model_type, test_data] of Object.entries(TEST_CUSTOM_TEMPLATES)) { - it(model_type, async () => { + it(model_type, () => { render(test_data); }); } }); + describe("Custom formatting", () => { + for (const [model_type, test_data] of Object.entries(TEST_CUSTOM_FORMATTING)) { + it(model_type, () => { + format(test_data); + }); + } + }); + it("should parse a chat template from the Hugging Face Hub", async () => { const repo = "TheBloke/Mistral-7B-Instruct-v0.1-GPTQ"; const tokenizerConfig = await ( diff --git a/packages/jinja/test/format.test.js b/packages/jinja/test/format.test.js new file mode 100644 index 000000000..349b0e95d --- /dev/null +++ b/packages/jinja/test/format.test.js @@ -0,0 +1,66 @@ +import { describe, expect, it } from "vitest"; +import { Template } from "../src/index"; + +/** + * Tests for formatting templates. + * + * This section defines our (opinionated) style guide for formatting templates. + */ +const FORMATTING_TESTS = Object.freeze({ + OPERATOR_PRECEDENCE: { + template: `{{ "hello" if 1 + 2 * 3 / 4 - 5 == 0 or 1 + 2 * 3 / 4 - 5 == 0 and 1 + 2 * 3 / 4 - 5 == 0 else "bye" }}`, + formatted: `{{- "hello" if 1 + 2 * 3 / 4 - 5 == 0 or 1 + 2 * 3 / 4 - 5 == 0 and 1 + 2 * 3 / 4 - 5 == 0 else "bye" -}}`, + rendered: `bye`, + }, + CHAINED_PROPERTY_ACCESSES: { + template: `{{ message.content.split('')[0].rstrip('\\n').split('')[-1].lstrip('\\n') }}`, + formatted: `{{- message.content.split("")[0].rstrip("\\n").split("")[-1].lstrip("\\n") -}}`, + }, + MEMBERSHIP: { + template: `{{ 'a' in 'abc' and False }}`, + formatted: `{{- "a" in "abc" and False -}}`, + rendered: `false`, + }, + FILTERS: { + template: `{{'a' + {'b': 'b'}['b']| upper + 'c'}}`, + formatted: `{{- "a" + {"b": "b"}["b"] | upper + "c" -}}`, + rendered: `aBc`, + }, + UNARY_BINARY_MIXED: { + template: `{{not 1 not in [1,2,3] and not 1==2 or 1!=2}}`, + // Technically minimal (according to precedence rules) but less readable: + // formatted: `{{- not 1 not in [1, 2, 3] and not 1 == 2 or 1 != 2 -}}`, + formatted: `{{- not (1 not in [1, 2, 3]) and not (1 == 2) or 1 != 2 -}}`, + rendered: `true`, + }, + CHAINED_UNARY: { + template: `{{not not not 1}}`, + formatted: `{{- not not not 1 -}}`, + rendered: `false`, + }, + CHAINED_TERNARY: { + template: `{{('a' if (true if 1==2 else false) else 'b') if 3==4 else ('c' if 4==5 else 'd')}}`, + formatted: `{{- "a" if (true if 1 == 2 else false) else "b" if 3 == 4 else "c" if 4 == 5 else "d" -}}`, + rendered: `d`, + }, +}); + +describe("format", () => { + for (const [name, test] of Object.entries(FORMATTING_TESTS)) { + it(`should format ${name}`, () => { + const template = new Template(test.template); + const result = template.format({ + indent: 4, + }); + expect(result).toEqual(test.formatted); + + if (test.rendered) { + const rendered = template.render(); + const formatted = new Template(test.formatted); + const formattedRendered = formatted.render(); + expect(rendered).toEqual(test.rendered); + expect(formattedRendered).toEqual(test.rendered); + } + }); + } +}); diff --git a/packages/jinja/test/templates.test.js b/packages/jinja/test/templates.test.js index 44207df78..e8b98d052 100644 --- a/packages/jinja/test/templates.test.js +++ b/packages/jinja/test/templates.test.js @@ -3,7 +3,7 @@ import { describe, expect, it } from "vitest"; import { Template } from "../src/index"; import { tokenize } from "../src/lexer"; import { parse } from "../src/parser"; -import { Environment, Interpreter } from "../src/runtime"; +import { Environment, Interpreter, setupGlobals } from "../src/runtime"; const TEST_STRINGS = { // Text nodes @@ -34,24 +34,30 @@ const TEST_STRINGS = { FOR_LOOP_UNPACKING: `|{% for x, y in [ [1, 2], [3, 4] ] %}|{{ x + ' ' + y }}|{% endfor %}|`, FOR_LOOP_DEFAULT: `{% for x in [] %}{{ 'A' }}{% else %}{{'B'}}{% endfor %}`, FOR_LOOP_SELECT: `{% for x in [1, 2, 3, 4] if x > 2 %}{{ x }}{% endfor %}`, + FOR_LOOP_SELECT_2: `{% for x in arr | selectattr('value', 'equalto', 'a') %}{{ x['value'] }}{% endfor %}`, FOR_LOOP_BREAK: `{% for x in [1, 2, 3, 4] %}{% if x == 3 %}{% break %}{% endif %}{{ x }}{% endfor %}`, FOR_LOOP_CONTINUE: `{% for x in [1, 2, 3, 4] %}{% if x == 3 %}{% continue %}{% endif %}{{ x }}{% endfor %}`, + FOR_LOOP_OBJECTS: `{% for x in obj %}{{ x + ':' + obj[x] + ';' }}{% endfor %}`, // Set variables VARIABLES: `{% set x = 'Hello' %}{% set y = 'World' %}{{ x + ' ' + y }}`, VARIABLES_2: `{% set x = 'Hello'.split('el')[-1] %}{{ x }}`, VARIABLES_BLOCK: `{% set x %}Hello!\nMultiline/block set!\n{% endset %}{{ x }}`, + VARIABLES_UNPACKING: `|{% set x, y = 1, 2 %}{{ x }}{{ y }}|{% set (x, y) = [1, 2] %}{{ x }}{{ y }}|`, // Numbers NUMBERS: `|{{ 5 }}|{{ -5 }}|{{ add(3, -1) }}|{{ (3 - 1) + (a - 5) - (a + 5)}}|`, // Binary expressions BINOP_EXPR: `{{ 1 % 2 }}{{ 1 < 2 }}{{ 1 > 2 }}{{ 1 >= 2 }}{{ 2 <= 2 }}{{ 2 == 2 }}{{ 2 != 3 }}{{ 2 + 3 }}`, + BINOP_EXPR_1: `{{ 1 ~ "+" ~ 2 ~ "=" ~ 3 ~ " is " ~ true }}`, // Strings STRINGS: `{{ 'Bye' }}{{ bos_token + '[INST] ' }}`, STRINGS_1: `|{{ "test" }}|{{ "a" + 'b' + "c" }}|{{ '"' + "'" }}|{{ '\\'' }}|{{ "\\"" }}|`, STRINGS_2: `|{{ "" | length }}|{{ "a" | length }}|{{ '' | length }}|{{ 'a' | length }}|`, + STRINGS_3: `|{{ '{{ "hi" }}' }}|{{ '{% if true %}{% endif %}' }}|`, + STRINGS_4: `{{ 'a' + 'b' 'c' }}`, // Function calls FUNCTIONS: `{{ func() }}{{ func(apple) }}{{ func(x, 'test', 2, false) }}`, @@ -63,6 +69,14 @@ const TEST_STRINGS = { OBJ_METHODS: `{{ obj.x(x, y) }}{{ ' ' + obj.x() + ' ' }}{{ obj.z[x](x, y) }}`, STRING_METHODS: `{{ ' A '.strip() }}{% set x = ' B ' %}{{ x.strip() }}{% set y = ' aBcD ' %}{{ y.upper() }}{{ y.lower() }}`, STRING_METHODS_2: `{{ 'test test'.title() }}`, + RSTRIP: `{{ " test it ".rstrip() }}`, + LSTRIP: `{{ " test it ".lstrip() }}`, + SPLIT: `|{{ " test it ".split() | join("|") }}|`, + SPLIT_2: `|{{ " test it ".split(" ") | join("|") }}|`, + SPLIT_3: `|{{ " test it ".split(" ", 4) | join("|") }}|`, + SPLIT_4: `|{{ " 1 2 3 ".split() | tojson }}|{{ "babbaccabbb".split("b") | tojson }}|{{ "babbaccabbb".split("b", 2) | tojson }}|`, + SPLIT_5: `|{{ " 1 2 3 4 5 ".split(none, 0) | join(",") }}|{{ " 1 2 3 4 5 ".split(none, 3) | join(",") }}|{{ " 1 2 3 4 5 ".split(" ", 0) | join(",") }}|{{ " 1 2 3 4 5 ".split(" ", 3) | join(",") }}|{{ " 1 2 3 4 5 ".split(" ", 10) | join(",") }}|`, + REPLACE: `|{{ "test test".replace("test", "TEST") }}|{{ "test test".replace("test", "TEST", 1) }}|{{ "test test".replace("", "_", 2) }}|{{ "abcabc".replace("a", "x", count=1) }}|`, // String indexing and slicing STRING_SLICING: `|{{ x[0] }}|{{ x[:] }}|{{ x[:3] }}|{{ x[1:4] }}|{{ x[1:-1] }}|{{ x[1::2] }}|{{ x[5::-1] }}|`, @@ -74,6 +88,7 @@ const TEST_STRINGS = { MEMBERSHIP: `|{{ 0 in arr }}|{{ 1 in arr }}|{{ true in arr }}|{{ false in arr }}|{{ 'a' in arr }}|{{ 'b' in arr }}|`, MEMBERSHIP_NEGATION_1: `|{{ not 0 in arr }}|{{ not 1 in arr }}|{{ not true in arr }}|{{ not false in arr }}|{{ not 'a' in arr }}|{{ not 'b' in arr }}|`, MEMBERSHIP_NEGATION_2: `|{{ 0 not in arr }}|{{ 1 not in arr }}|{{ true not in arr }}|{{ false not in arr }}|{{ 'a' not in arr }}|{{ 'b' not in arr }}|`, + MEMBERSHIP_UNDEFINED: `|{{ x is defined }}|{{ y is defined }}|{{ x in y }}|{{ y in x }}|{{ 1 in y }}|{{ 1 in x }}|`, // Escaped characters ESCAPED_CHARS: `{{ '\\n' }}{{ '\\t' }}{{ '\\'' }}{{ '\\"' }}{{ '\\\\' }}{{ '|\\n|\\t|\\'|\\"|\\\\|' }}`, @@ -95,6 +110,13 @@ const TEST_STRINGS = { FILTER_OPERATOR_11: `{{ items | rejectattr('key') | length }}`, FILTER_OPERATOR_12: `{{ messages | rejectattr('role', 'equalto', 'system') | length }}`, FILTER_OPERATOR_13: `{{ tools | string }}`, + FILTER_OPERATOR_14: `|{{ "1" | int + 2 }}|{{ "invalid" | int }}|{{ "invalid" | int(-1) }}|{{ true | int }}|{{ false | int }}|{{ 1.5 | int }}|{{ "1.5" | float }}|{{ "invalid" | float }}|{{ "invalid" | float("hello") }}|`, + FILTER_OPERATOR_15: `|{{ "abcabcabc" | replace("a", "b") }}|{{ "abcabcabc" | replace("a", "b", 1) }}|{{ "abcabcabc" | replace("a", "b", count=1) }}|`, + FILTER_OPERATOR_16: `|{{ undefined | default("hello") }}|{{ false | default("hello") }}|{{ false | default("hello", true) }}|{{ 0 | default("hello", boolean=true) }}|`, + FILTER_OPERATOR_17: `{{ [1, 2, 1, -1, 2] | unique | list | length }}`, + + // Filter statements + FILTER_STATEMENTS: `{% filter upper %}text{% endfilter %}`, // Logical operators between non-Booleans BOOLEAN_NUMERICAL: `|{{ 1 and 2 }}|{{ 1 and 0 }}|{{ 0 and 1 }}|{{ 0 and 0 }}|{{ 1 or 2 }}|{{ 1 or 0 }}|{{ 0 or 1 }}|{{ 0 or 0 }}|{{ not 1 }}|{{ not 0 }}|`, @@ -138,7 +160,10 @@ const TEST_STRINGS = { // Ternary operator TERNARY_OPERATOR: `|{{ 'a' if true else 'b' }}|{{ 'a' if false else 'b' }}|{{ 'a' if 1 + 1 == 2 else 'b' }}|{{ 'a' if 1 + 1 == 3 or 1 * 2 == 3 else 'b' }}|`, + TERNARY_OPERATOR_1: `{{ (x if true else []) | length }}`, TERNARY_SET: `{% set x = 1 if True else 2 %}{{ x }}`, + TERNARY_CONSECUTIVE: `{% set x = 1 if False else 2 if False else 3 %}{{ x }}`, + TERNARY_SHORTCUT: `{{ 'foo' if false }}{{ 'bar' if true }}`, // Array literals ARRAY_LITERALS: `{{ [1, true, 'hello', [1, 2, 3, 4], var] | length }}`, @@ -148,6 +173,7 @@ const TEST_STRINGS = { // Object literals OBJECT_LITERALS: `{{ { 'key': 'value', key: 'value2', "key3": [1, {'foo': 'bar'} ] }['key'] }}`, + OBJECT_LITERALS_1: `{{{'key': {'key': 'value'}}['key']['key']}}`, // Array operators ARRAY_OPERATORS: `{{ ([1, 2, 3] + [4, 5, 6]) | length }}`, @@ -156,18 +182,15 @@ const TEST_STRINGS = { MACROS: `{% macro hello(name) %}{{ 'Hello ' + name }}{% endmacro %}|{{ hello('Bob') }}|{{ hello('Alice') }}|`, MACROS_1: `{% macro hello(name, suffix='.') %}{{ 'Hello ' + name + suffix }}{% endmacro %}|{{ hello('A') }}|{{ hello('B', '!') }}|{{ hello('C', suffix='?') }}|`, MACROS_2: `{% macro fn(x, y=2, z=3) %}{{ x + ',' + y + ',' + z }}{% endmacro %}|{{ fn(1) }}|{{ fn(1, 0) }}|{{ fn(1, 0, -1) }}|{{ fn(1, y=0, z=-1) }}|{{ fn(1, z=0) }}|`, + MACROS_3: `{%- macro dummy(a, b='!') -%}{{ a }} {{ caller() }}{{ b }}{%- endmacro %}{%- call dummy('hello') -%}name{%- endcall -%}`, + MACROS_4: `{%- macro print_users(users) -%}{%- for user in users -%}{{ caller(user) }}{%- endfor -%}{%- endmacro -%}{% call(user) print_users(users) %} - {{ user.firstname }} {{ user.lastname }}\n{% endcall %}`, - //rstrip - RSTRIP: `{{ " test it ".rstrip() }}`, - //lstrip - LSTRIP: `{{ " test it ".lstrip() }}`, + // Context-specific keywords + CONTEXT_KEYWORDS: `{% if if in in %}a{% endif %}{% set if = "a" %}{% set in = "abc" %}{% if if in in %}b{% endif %}`, + CONTEXT_KEYWORDS_1: `|{{ if }}|{% set if = 2 %}{% if if == 2 %}{{ if }}{% endif %}|`, - //split - SPLIT: `|{{ " test it ".split() | join("|") }}|`, - SPLIT_2: `|{{ " test it ".split(" ") | join("|") }}|`, - SPLIT_3: `|{{ " test it ".split(" ", 4) | join("|") }}|`, - SPLIT_4: `|{{ " 1 2 3 ".split() | tojson }}|{{ "babbaccabbb".split("b") | tojson }}|{{ "babbaccabbb".split("b", 2) | tojson }}|`, - SPLIT_5: `|{{ " 1 2 3 4 5 ".split(none, 0) | join(",") }}|{{ " 1 2 3 4 5 ".split(none, 3) | join(",") }}|{{ " 1 2 3 4 5 ".split(" ", 0) | join(",") }}|{{ " 1 2 3 4 5 ".split(" ", 3) | join(",") }}|{{ " 1 2 3 4 5 ".split(" ", 10) | join(",") }}|`, + // Unpacking + UNPACKING: `{% macro mul(a, b, c) %}{{ a * b * c }}{% endmacro %}|{{ mul(1, 2, 3) }}|{{ mul(*[1, 2, 3]) }}|`, }; const TEST_PARSED = { @@ -196,19 +219,19 @@ const TEST_PARSED = { BOOLEAN_LITERALS: [ { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "true", type: "BooleanLiteral" }, + { value: "true", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "false", type: "BooleanLiteral" }, + { value: "false", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "True", type: "BooleanLiteral" }, + { value: "True", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "False", type: "BooleanLiteral" }, + { value: "False", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, ], @@ -216,157 +239,157 @@ const TEST_PARSED = { // Logical operators LOGICAL_AND: [ { value: "{{", type: "OpenExpression" }, - { value: "true", type: "BooleanLiteral" }, - { value: "and", type: "And" }, - { value: "true", type: "BooleanLiteral" }, + { value: "true", type: "Identifier" }, + { value: "and", type: "Identifier" }, + { value: "true", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "{{", type: "OpenExpression" }, - { value: "true", type: "BooleanLiteral" }, - { value: "and", type: "And" }, - { value: "false", type: "BooleanLiteral" }, + { value: "true", type: "Identifier" }, + { value: "and", type: "Identifier" }, + { value: "false", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "{{", type: "OpenExpression" }, - { value: "false", type: "BooleanLiteral" }, - { value: "and", type: "And" }, - { value: "true", type: "BooleanLiteral" }, + { value: "false", type: "Identifier" }, + { value: "and", type: "Identifier" }, + { value: "true", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "{{", type: "OpenExpression" }, - { value: "false", type: "BooleanLiteral" }, - { value: "and", type: "And" }, - { value: "false", type: "BooleanLiteral" }, + { value: "false", type: "Identifier" }, + { value: "and", type: "Identifier" }, + { value: "false", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, ], LOGICAL_OR: [ { value: "{{", type: "OpenExpression" }, - { value: "true", type: "BooleanLiteral" }, - { value: "or", type: "Or" }, - { value: "true", type: "BooleanLiteral" }, + { value: "true", type: "Identifier" }, + { value: "or", type: "Identifier" }, + { value: "true", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "{{", type: "OpenExpression" }, - { value: "true", type: "BooleanLiteral" }, - { value: "or", type: "Or" }, - { value: "false", type: "BooleanLiteral" }, + { value: "true", type: "Identifier" }, + { value: "or", type: "Identifier" }, + { value: "false", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "{{", type: "OpenExpression" }, - { value: "false", type: "BooleanLiteral" }, - { value: "or", type: "Or" }, - { value: "true", type: "BooleanLiteral" }, + { value: "false", type: "Identifier" }, + { value: "or", type: "Identifier" }, + { value: "true", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "{{", type: "OpenExpression" }, - { value: "false", type: "BooleanLiteral" }, - { value: "or", type: "Or" }, - { value: "false", type: "BooleanLiteral" }, + { value: "false", type: "Identifier" }, + { value: "or", type: "Identifier" }, + { value: "false", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, ], LOGICAL_NOT: [ { value: "{{", type: "OpenExpression" }, - { value: "not", type: "UnaryOperator" }, - { value: "true", type: "BooleanLiteral" }, + { value: "not", type: "Identifier" }, + { value: "true", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "{{", type: "OpenExpression" }, - { value: "not", type: "UnaryOperator" }, - { value: "false", type: "BooleanLiteral" }, + { value: "not", type: "Identifier" }, + { value: "false", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, ], LOGICAL_NOT_NOT: [ { value: "{{", type: "OpenExpression" }, - { value: "not", type: "UnaryOperator" }, - { value: "not", type: "UnaryOperator" }, - { value: "true", type: "BooleanLiteral" }, + { value: "not", type: "Identifier" }, + { value: "not", type: "Identifier" }, + { value: "true", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "{{", type: "OpenExpression" }, - { value: "not", type: "UnaryOperator" }, - { value: "not", type: "UnaryOperator" }, - { value: "false", type: "BooleanLiteral" }, + { value: "not", type: "Identifier" }, + { value: "not", type: "Identifier" }, + { value: "false", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, ], LOGICAL_AND_OR: [ { value: "{{", type: "OpenExpression" }, - { value: "true", type: "BooleanLiteral" }, - { value: "and", type: "And" }, - { value: "true", type: "BooleanLiteral" }, - { value: "or", type: "Or" }, - { value: "false", type: "BooleanLiteral" }, + { value: "true", type: "Identifier" }, + { value: "and", type: "Identifier" }, + { value: "true", type: "Identifier" }, + { value: "or", type: "Identifier" }, + { value: "false", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "{{", type: "OpenExpression" }, - { value: "true", type: "BooleanLiteral" }, - { value: "and", type: "And" }, - { value: "false", type: "BooleanLiteral" }, - { value: "or", type: "Or" }, - { value: "true", type: "BooleanLiteral" }, + { value: "true", type: "Identifier" }, + { value: "and", type: "Identifier" }, + { value: "false", type: "Identifier" }, + { value: "or", type: "Identifier" }, + { value: "true", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "{{", type: "OpenExpression" }, - { value: "false", type: "BooleanLiteral" }, - { value: "and", type: "And" }, - { value: "true", type: "BooleanLiteral" }, - { value: "or", type: "Or" }, - { value: "true", type: "BooleanLiteral" }, + { value: "false", type: "Identifier" }, + { value: "and", type: "Identifier" }, + { value: "true", type: "Identifier" }, + { value: "or", type: "Identifier" }, + { value: "true", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "{{", type: "OpenExpression" }, - { value: "false", type: "BooleanLiteral" }, - { value: "and", type: "And" }, - { value: "false", type: "BooleanLiteral" }, - { value: "or", type: "Or" }, - { value: "true", type: "BooleanLiteral" }, + { value: "false", type: "Identifier" }, + { value: "and", type: "Identifier" }, + { value: "false", type: "Identifier" }, + { value: "or", type: "Identifier" }, + { value: "true", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "{{", type: "OpenExpression" }, - { value: "false", type: "BooleanLiteral" }, - { value: "and", type: "And" }, - { value: "false", type: "BooleanLiteral" }, - { value: "or", type: "Or" }, - { value: "false", type: "BooleanLiteral" }, + { value: "false", type: "Identifier" }, + { value: "and", type: "Identifier" }, + { value: "false", type: "Identifier" }, + { value: "or", type: "Identifier" }, + { value: "false", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, ], LOGICAL_AND_NOT: [ { value: "{{", type: "OpenExpression" }, - { value: "true", type: "BooleanLiteral" }, - { value: "and", type: "And" }, - { value: "not", type: "UnaryOperator" }, - { value: "true", type: "BooleanLiteral" }, + { value: "true", type: "Identifier" }, + { value: "and", type: "Identifier" }, + { value: "not", type: "Identifier" }, + { value: "true", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "{{", type: "OpenExpression" }, - { value: "true", type: "BooleanLiteral" }, - { value: "and", type: "And" }, - { value: "not", type: "UnaryOperator" }, - { value: "false", type: "BooleanLiteral" }, + { value: "true", type: "Identifier" }, + { value: "and", type: "Identifier" }, + { value: "not", type: "Identifier" }, + { value: "false", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "{{", type: "OpenExpression" }, - { value: "false", type: "BooleanLiteral" }, - { value: "and", type: "And" }, - { value: "not", type: "UnaryOperator" }, - { value: "true", type: "BooleanLiteral" }, + { value: "false", type: "Identifier" }, + { value: "and", type: "Identifier" }, + { value: "not", type: "Identifier" }, + { value: "true", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "{{", type: "OpenExpression" }, - { value: "false", type: "BooleanLiteral" }, - { value: "and", type: "And" }, - { value: "not", type: "UnaryOperator" }, - { value: "false", type: "BooleanLiteral" }, + { value: "false", type: "Identifier" }, + { value: "and", type: "Identifier" }, + { value: "not", type: "Identifier" }, + { value: "false", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, ], LOGICAL_OR_NOT: [ { value: "{{", type: "OpenExpression" }, - { value: "true", type: "BooleanLiteral" }, - { value: "or", type: "Or" }, - { value: "not", type: "UnaryOperator" }, - { value: "true", type: "BooleanLiteral" }, + { value: "true", type: "Identifier" }, + { value: "or", type: "Identifier" }, + { value: "not", type: "Identifier" }, + { value: "true", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "{{", type: "OpenExpression" }, - { value: "true", type: "BooleanLiteral" }, - { value: "or", type: "Or" }, - { value: "not", type: "UnaryOperator" }, - { value: "false", type: "BooleanLiteral" }, + { value: "true", type: "Identifier" }, + { value: "or", type: "Identifier" }, + { value: "not", type: "Identifier" }, + { value: "false", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "{{", type: "OpenExpression" }, - { value: "false", type: "BooleanLiteral" }, - { value: "or", type: "Or" }, - { value: "not", type: "UnaryOperator" }, - { value: "true", type: "BooleanLiteral" }, + { value: "false", type: "Identifier" }, + { value: "or", type: "Identifier" }, + { value: "not", type: "Identifier" }, + { value: "true", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "{{", type: "OpenExpression" }, - { value: "false", type: "BooleanLiteral" }, - { value: "or", type: "Or" }, - { value: "not", type: "UnaryOperator" }, - { value: "false", type: "BooleanLiteral" }, + { value: "false", type: "Identifier" }, + { value: "or", type: "Identifier" }, + { value: "not", type: "Identifier" }, + { value: "false", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, ], LOGICAL_COMBINED: [ @@ -374,7 +397,7 @@ const TEST_PARSED = { { value: "1", type: "NumericLiteral" }, { value: "==", type: "ComparisonBinaryOperator" }, { value: "2", type: "NumericLiteral" }, - { value: "and", type: "And" }, + { value: "and", type: "Identifier" }, { value: "2", type: "NumericLiteral" }, { value: "==", type: "ComparisonBinaryOperator" }, { value: "2", type: "NumericLiteral" }, @@ -383,7 +406,7 @@ const TEST_PARSED = { { value: "1", type: "NumericLiteral" }, { value: "==", type: "ComparisonBinaryOperator" }, { value: "2", type: "NumericLiteral" }, - { value: "or", type: "Or" }, + { value: "or", type: "Identifier" }, { value: "2", type: "NumericLiteral" }, { value: "==", type: "ComparisonBinaryOperator" }, { value: "2", type: "NumericLiteral" }, @@ -393,7 +416,7 @@ const TEST_PARSED = { // If statements IF_ONLY: [ { value: "{%", type: "OpenStatement" }, - { value: "if", type: "If" }, + { value: "if", type: "Identifier" }, { value: "1", type: "NumericLiteral" }, { value: "==", type: "ComparisonBinaryOperator" }, { value: "1", type: "NumericLiteral" }, @@ -402,7 +425,7 @@ const TEST_PARSED = { { value: "A", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "endif", type: "EndIf" }, + { value: "endif", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "{{", type: "OpenExpression" }, { value: "B", type: "StringLiteral" }, @@ -410,7 +433,7 @@ const TEST_PARSED = { ], IF_ELSE_ONLY: [ { value: "{%", type: "OpenStatement" }, - { value: "if", type: "If" }, + { value: "if", type: "Identifier" }, { value: "1", type: "NumericLiteral" }, { value: "==", type: "ComparisonBinaryOperator" }, { value: "2", type: "NumericLiteral" }, @@ -419,13 +442,13 @@ const TEST_PARSED = { { value: "A", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "else", type: "Else" }, + { value: "else", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "{{", type: "OpenExpression" }, { value: "B", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "endif", type: "EndIf" }, + { value: "endif", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "{{", type: "OpenExpression" }, { value: "C", type: "StringLiteral" }, @@ -433,7 +456,7 @@ const TEST_PARSED = { ], IF_ELIF_ELSE: [ { value: "{%", type: "OpenStatement" }, - { value: "if", type: "If" }, + { value: "if", type: "Identifier" }, { value: "1", type: "NumericLiteral" }, { value: "==", type: "ComparisonBinaryOperator" }, { value: "2", type: "NumericLiteral" }, @@ -448,7 +471,7 @@ const TEST_PARSED = { { value: "C", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "elif", type: "ElseIf" }, + { value: "elif", type: "Identifier" }, { value: "1", type: "NumericLiteral" }, { value: "==", type: "ComparisonBinaryOperator" }, { value: "2", type: "NumericLiteral" }, @@ -457,7 +480,7 @@ const TEST_PARSED = { { value: "D", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "elif", type: "ElseIf" }, + { value: "elif", type: "Identifier" }, { value: "1", type: "NumericLiteral" }, { value: "==", type: "ComparisonBinaryOperator" }, { value: "3", type: "NumericLiteral" }, @@ -469,7 +492,7 @@ const TEST_PARSED = { { value: "F", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "else", type: "Else" }, + { value: "else", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "{{", type: "OpenExpression" }, { value: "G", type: "StringLiteral" }, @@ -481,7 +504,7 @@ const TEST_PARSED = { { value: "I", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "endif", type: "EndIf" }, + { value: "endif", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "{{", type: "OpenExpression" }, { value: "J", type: "StringLiteral" }, @@ -489,70 +512,70 @@ const TEST_PARSED = { ], NESTED_STATEMENTS: [ { value: "{%", type: "OpenStatement" }, - { value: "set", type: "Set" }, + { value: "set", type: "Identifier" }, { value: "a", type: "Identifier" }, { value: "=", type: "Equals" }, { value: "0", type: "NumericLiteral" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "set", type: "Set" }, + { value: "set", type: "Identifier" }, { value: "b", type: "Identifier" }, { value: "=", type: "Equals" }, { value: "0", type: "NumericLiteral" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "set", type: "Set" }, + { value: "set", type: "Identifier" }, { value: "c", type: "Identifier" }, { value: "=", type: "Equals" }, { value: "0", type: "NumericLiteral" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "set", type: "Set" }, + { value: "set", type: "Identifier" }, { value: "d", type: "Identifier" }, { value: "=", type: "Equals" }, { value: "0", type: "NumericLiteral" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "if", type: "If" }, + { value: "if", type: "Identifier" }, { value: "1", type: "NumericLiteral" }, { value: "==", type: "ComparisonBinaryOperator" }, { value: "1", type: "NumericLiteral" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "set", type: "Set" }, + { value: "set", type: "Identifier" }, { value: "a", type: "Identifier" }, { value: "=", type: "Equals" }, { value: "2", type: "NumericLiteral" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "set", type: "Set" }, + { value: "set", type: "Identifier" }, { value: "b", type: "Identifier" }, { value: "=", type: "Equals" }, { value: "3", type: "NumericLiteral" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "elif", type: "ElseIf" }, + { value: "elif", type: "Identifier" }, { value: "1", type: "NumericLiteral" }, { value: "==", type: "ComparisonBinaryOperator" }, { value: "2", type: "NumericLiteral" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "set", type: "Set" }, + { value: "set", type: "Identifier" }, { value: "c", type: "Identifier" }, { value: "=", type: "Equals" }, { value: "4", type: "NumericLiteral" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "else", type: "Else" }, + { value: "else", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "set", type: "Set" }, + { value: "set", type: "Identifier" }, { value: "d", type: "Identifier" }, { value: "=", type: "Equals" }, { value: "5", type: "NumericLiteral" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "endif", type: "EndIf" }, + { value: "endif", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "{{", type: "OpenExpression" }, { value: "a", type: "Identifier" }, @@ -571,9 +594,9 @@ const TEST_PARSED = { // For loops FOR_LOOP: [ { value: "{%", type: "OpenStatement" }, - { value: "for", type: "For" }, + { value: "for", type: "Identifier" }, { value: "message", type: "Identifier" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "messages", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "{{", type: "OpenExpression" }, @@ -583,17 +606,17 @@ const TEST_PARSED = { { value: "]", type: "CloseSquareBracket" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "endfor", type: "EndFor" }, + { value: "endfor", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, ], FOR_LOOP_UNPACKING: [ { value: "|", type: "Text" }, { value: "{%", type: "OpenStatement" }, - { value: "for", type: "For" }, + { value: "for", type: "Identifier" }, { value: "x", type: "Identifier" }, { value: ",", type: "Comma" }, { value: "y", type: "Identifier" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "[", type: "OpenSquareBracket" }, { value: "[", type: "OpenSquareBracket" }, { value: "1", type: "NumericLiteral" }, @@ -618,15 +641,15 @@ const TEST_PARSED = { { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{%", type: "OpenStatement" }, - { value: "endfor", type: "EndFor" }, + { value: "endfor", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "|", type: "Text" }, ], FOR_LOOP_DEFAULT: [ { value: "{%", type: "OpenStatement" }, - { value: "for", type: "For" }, + { value: "for", type: "Identifier" }, { value: "x", type: "Identifier" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "[", type: "OpenSquareBracket" }, { value: "]", type: "CloseSquareBracket" }, { value: "%}", type: "CloseStatement" }, @@ -634,20 +657,20 @@ const TEST_PARSED = { { value: "A", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "else", type: "Else" }, + { value: "else", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "{{", type: "OpenExpression" }, { value: "B", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "endfor", type: "EndFor" }, + { value: "endfor", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, ], FOR_LOOP_SELECT: [ { value: "{%", type: "OpenStatement" }, - { value: "for", type: "For" }, + { value: "for", type: "Identifier" }, { value: "x", type: "Identifier" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "[", type: "OpenSquareBracket" }, { value: "1", type: "NumericLiteral" }, { value: ",", type: "Comma" }, @@ -657,7 +680,7 @@ const TEST_PARSED = { { value: ",", type: "Comma" }, { value: "4", type: "NumericLiteral" }, { value: "]", type: "CloseSquareBracket" }, - { value: "if", type: "If" }, + { value: "if", type: "Identifier" }, { value: "x", type: "Identifier" }, { value: ">", type: "ComparisonBinaryOperator" }, { value: "2", type: "NumericLiteral" }, @@ -666,14 +689,40 @@ const TEST_PARSED = { { value: "x", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "endfor", type: "EndFor" }, + { value: "endfor", type: "Identifier" }, + { value: "%}", type: "CloseStatement" }, + ], + FOR_LOOP_SELECT_2: [ + { value: "{%", type: "OpenStatement" }, + { value: "for", type: "Identifier" }, + { value: "x", type: "Identifier" }, + { value: "in", type: "Identifier" }, + { value: "arr", type: "Identifier" }, + { value: "|", type: "Pipe" }, + { value: "selectattr", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: "value", type: "StringLiteral" }, + { value: ",", type: "Comma" }, + { value: "equalto", type: "StringLiteral" }, + { value: ",", type: "Comma" }, + { value: "a", type: "StringLiteral" }, + { value: ")", type: "CloseParen" }, + { value: "%}", type: "CloseStatement" }, + { value: "{{", type: "OpenExpression" }, + { value: "x", type: "Identifier" }, + { value: "[", type: "OpenSquareBracket" }, + { value: "value", type: "StringLiteral" }, + { value: "]", type: "CloseSquareBracket" }, + { value: "}}", type: "CloseExpression" }, + { value: "{%", type: "OpenStatement" }, + { value: "endfor", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, ], FOR_LOOP_BREAK: [ { value: "{%", type: "OpenStatement" }, - { value: "for", type: "For" }, + { value: "for", type: "Identifier" }, { value: "x", type: "Identifier" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "[", type: "OpenSquareBracket" }, { value: "1", type: "NumericLiteral" }, { value: ",", type: "Comma" }, @@ -685,29 +734,29 @@ const TEST_PARSED = { { value: "]", type: "CloseSquareBracket" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "if", type: "If" }, + { value: "if", type: "Identifier" }, { value: "x", type: "Identifier" }, { value: "==", type: "ComparisonBinaryOperator" }, { value: "3", type: "NumericLiteral" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "break", type: "Break" }, + { value: "break", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "endif", type: "EndIf" }, + { value: "endif", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "{{", type: "OpenExpression" }, { value: "x", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "endfor", type: "EndFor" }, + { value: "endfor", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, ], FOR_LOOP_CONTINUE: [ { value: "{%", type: "OpenStatement" }, - { value: "for", type: "For" }, + { value: "for", type: "Identifier" }, { value: "x", type: "Identifier" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "[", type: "OpenSquareBracket" }, { value: "1", type: "NumericLiteral" }, { value: ",", type: "Comma" }, @@ -719,35 +768,58 @@ const TEST_PARSED = { { value: "]", type: "CloseSquareBracket" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "if", type: "If" }, + { value: "if", type: "Identifier" }, { value: "x", type: "Identifier" }, { value: "==", type: "ComparisonBinaryOperator" }, { value: "3", type: "NumericLiteral" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "continue", type: "Continue" }, + { value: "continue", type: "Identifier" }, + { value: "%}", type: "CloseStatement" }, + { value: "{%", type: "OpenStatement" }, + { value: "endif", type: "Identifier" }, + { value: "%}", type: "CloseStatement" }, + { value: "{{", type: "OpenExpression" }, + { value: "x", type: "Identifier" }, + { value: "}}", type: "CloseExpression" }, + { value: "{%", type: "OpenStatement" }, + { value: "endfor", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, + ], + FOR_LOOP_OBJECTS: [ { value: "{%", type: "OpenStatement" }, - { value: "endif", type: "EndIf" }, + { value: "for", type: "Identifier" }, + { value: "x", type: "Identifier" }, + { value: "in", type: "Identifier" }, + { value: "obj", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "{{", type: "OpenExpression" }, { value: "x", type: "Identifier" }, + { value: "+", type: "AdditiveBinaryOperator" }, + { value: ":", type: "StringLiteral" }, + { value: "+", type: "AdditiveBinaryOperator" }, + { value: "obj", type: "Identifier" }, + { value: "[", type: "OpenSquareBracket" }, + { value: "x", type: "Identifier" }, + { value: "]", type: "CloseSquareBracket" }, + { value: "+", type: "AdditiveBinaryOperator" }, + { value: ";", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "endfor", type: "EndFor" }, + { value: "endfor", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, ], // Set variables VARIABLES: [ { value: "{%", type: "OpenStatement" }, - { value: "set", type: "Set" }, + { value: "set", type: "Identifier" }, { value: "x", type: "Identifier" }, { value: "=", type: "Equals" }, { value: "Hello", type: "StringLiteral" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "set", type: "Set" }, + { value: "set", type: "Identifier" }, { value: "y", type: "Identifier" }, { value: "=", type: "Equals" }, { value: "World", type: "StringLiteral" }, @@ -762,7 +834,7 @@ const TEST_PARSED = { ], VARIABLES_2: [ { value: "{%", type: "OpenStatement" }, - { value: "set", type: "Set" }, + { value: "set", type: "Identifier" }, { value: "x", type: "Identifier" }, { value: "=", type: "Equals" }, { value: "Hello", type: "StringLiteral" }, @@ -781,16 +853,57 @@ const TEST_PARSED = { ], VARIABLES_BLOCK: [ { value: "{%", type: "OpenStatement" }, - { value: "set", type: "Set" }, + { value: "set", type: "Identifier" }, { value: "x", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: `Hello!\nMultiline/block set!\n`, type: "Text" }, { value: "{%", type: "OpenStatement" }, - { value: "endset", type: "EndSet" }, + { value: "endset", type: "Identifier" }, + { value: "%}", type: "CloseStatement" }, + { value: "{{", type: "OpenExpression" }, + { value: "x", type: "Identifier" }, + { value: "}}", type: "CloseExpression" }, + ], + VARIABLES_UNPACKING: [ + { value: "|", type: "Text" }, + { value: "{%", type: "OpenStatement" }, + { value: "set", type: "Identifier" }, + { value: "x", type: "Identifier" }, + { value: ",", type: "Comma" }, + { value: "y", type: "Identifier" }, + { value: "=", type: "Equals" }, + { value: "1", type: "NumericLiteral" }, + { value: ",", type: "Comma" }, + { value: "2", type: "NumericLiteral" }, + { value: "%}", type: "CloseStatement" }, + { value: "{{", type: "OpenExpression" }, + { value: "x", type: "Identifier" }, + { value: "}}", type: "CloseExpression" }, + { value: "{{", type: "OpenExpression" }, + { value: "y", type: "Identifier" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{%", type: "OpenStatement" }, + { value: "set", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: "x", type: "Identifier" }, + { value: ",", type: "Comma" }, + { value: "y", type: "Identifier" }, + { value: ")", type: "CloseParen" }, + { value: "=", type: "Equals" }, + { value: "[", type: "OpenSquareBracket" }, + { value: "1", type: "NumericLiteral" }, + { value: ",", type: "Comma" }, + { value: "2", type: "NumericLiteral" }, + { value: "]", type: "CloseSquareBracket" }, { value: "%}", type: "CloseStatement" }, { value: "{{", type: "OpenExpression" }, { value: "x", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, + { value: "{{", type: "OpenExpression" }, + { value: "y", type: "Identifier" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, ], // Numbers @@ -878,6 +991,23 @@ const TEST_PARSED = { { value: "3", type: "NumericLiteral" }, { value: "}}", type: "CloseExpression" }, ], + BINOP_EXPR_1: [ + { value: "{{", type: "OpenExpression" }, + { value: "1", type: "NumericLiteral" }, + { value: "~", type: "AdditiveBinaryOperator" }, + { value: "+", type: "StringLiteral" }, + { value: "~", type: "AdditiveBinaryOperator" }, + { value: "2", type: "NumericLiteral" }, + { value: "~", type: "AdditiveBinaryOperator" }, + { value: "=", type: "StringLiteral" }, + { value: "~", type: "AdditiveBinaryOperator" }, + { value: "3", type: "NumericLiteral" }, + { value: "~", type: "AdditiveBinaryOperator" }, + { value: " is ", type: "StringLiteral" }, + { value: "~", type: "AdditiveBinaryOperator" }, + { value: "true", type: "Identifier" }, + { value: "}}", type: "CloseExpression" }, + ], // Strings STRINGS: [ @@ -946,6 +1076,25 @@ const TEST_PARSED = { { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, ], + STRINGS_3: [ + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: '{{ "hi" }}', type: "StringLiteral" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "{% if true %}{% endif %}", type: "StringLiteral" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + ], + STRINGS_4: [ + { value: "{{", type: "OpenExpression" }, + { value: "a", type: "StringLiteral" }, + { value: "+", type: "AdditiveBinaryOperator" }, + { value: "b", type: "StringLiteral" }, + { value: "c", type: "StringLiteral" }, + { value: "}}", type: "CloseExpression" }, + ], // Function calls FUNCTIONS: [ @@ -969,7 +1118,7 @@ const TEST_PARSED = { { value: ",", type: "Comma" }, { value: "2", type: "NumericLiteral" }, { value: ",", type: "Comma" }, - { value: "false", type: "BooleanLiteral" }, + { value: "false", type: "Identifier" }, { value: ")", type: "CloseParen" }, { value: "}}", type: "CloseExpression" }, ], @@ -1045,7 +1194,7 @@ const TEST_PARSED = { { value: ")", type: "CloseParen" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "set", type: "Set" }, + { value: "set", type: "Identifier" }, { value: "x", type: "Identifier" }, { value: "=", type: "Equals" }, { value: " B ", type: "StringLiteral" }, @@ -1058,7 +1207,7 @@ const TEST_PARSED = { { value: ")", type: "CloseParen" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "set", type: "Set" }, + { value: "set", type: "Identifier" }, { value: "y", type: "Identifier" }, { value: "=", type: "Equals" }, { value: " aBcD ", type: "StringLiteral" }, @@ -1088,85 +1237,331 @@ const TEST_PARSED = { { value: "}}", type: "CloseExpression" }, ], - // String indexing and slicing - STRING_SLICING: [ - { value: "|", type: "Text" }, + RSTRIP: [ { value: "{{", type: "OpenExpression" }, - { value: "x", type: "Identifier" }, - { value: "[", type: "OpenSquareBracket" }, - { value: "0", type: "NumericLiteral" }, - { value: "]", type: "CloseSquareBracket" }, + { value: " test it ", type: "StringLiteral" }, + { value: ".", type: "Dot" }, + { value: "rstrip", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: ")", type: "CloseParen" }, + { value: "}}", type: "CloseExpression" }, + ], + LSTRIP: [ + { value: "{{", type: "OpenExpression" }, + { value: " test it ", type: "StringLiteral" }, + { value: ".", type: "Dot" }, + { value: "lstrip", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: ")", type: "CloseParen" }, { value: "}}", type: "CloseExpression" }, + ], + SPLIT: [ { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "x", type: "Identifier" }, - { value: "[", type: "OpenSquareBracket" }, - { value: ":", type: "Colon" }, - { value: "]", type: "CloseSquareBracket" }, + { value: " test it ", type: "StringLiteral" }, + { value: ".", type: "Dot" }, + { value: "split", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: ")", type: "CloseParen" }, + { value: "|", type: "Pipe" }, + { value: "join", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: "|", type: "StringLiteral" }, + { value: ")", type: "CloseParen" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, + ], + SPLIT_2: [ + { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "x", type: "Identifier" }, - { value: "[", type: "OpenSquareBracket" }, - { value: ":", type: "Colon" }, - { value: "3", type: "NumericLiteral" }, - { value: "]", type: "CloseSquareBracket" }, + { value: " test it ", type: "StringLiteral" }, + { value: ".", type: "Dot" }, + { value: "split", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: " ", type: "StringLiteral" }, + { value: ")", type: "CloseParen" }, + { value: "|", type: "Pipe" }, + { value: "join", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: "|", type: "StringLiteral" }, + { value: ")", type: "CloseParen" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, + ], + SPLIT_3: [ + { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "x", type: "Identifier" }, - { value: "[", type: "OpenSquareBracket" }, - { value: "1", type: "NumericLiteral" }, - { value: ":", type: "Colon" }, + { value: " test it ", type: "StringLiteral" }, + { value: ".", type: "Dot" }, + { value: "split", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: " ", type: "StringLiteral" }, + { value: ",", type: "Comma" }, { value: "4", type: "NumericLiteral" }, - { value: "]", type: "CloseSquareBracket" }, + { value: ")", type: "CloseParen" }, + { value: "|", type: "Pipe" }, + { value: "join", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: "|", type: "StringLiteral" }, + { value: ")", type: "CloseParen" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, + ], + SPLIT_4: [ + { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "x", type: "Identifier" }, - { value: "[", type: "OpenSquareBracket" }, - { value: "1", type: "NumericLiteral" }, - { value: ":", type: "Colon" }, - { value: "-1", type: "NumericLiteral" }, - { value: "]", type: "CloseSquareBracket" }, + { value: " 1 2 3 ", type: "StringLiteral" }, + { value: ".", type: "Dot" }, + { value: "split", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: ")", type: "CloseParen" }, + { value: "|", type: "Pipe" }, + { value: "tojson", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "x", type: "Identifier" }, - { value: "[", type: "OpenSquareBracket" }, - { value: "1", type: "NumericLiteral" }, - { value: ":", type: "Colon" }, - { value: ":", type: "Colon" }, - { value: "2", type: "NumericLiteral" }, - { value: "]", type: "CloseSquareBracket" }, + { value: "babbaccabbb", type: "StringLiteral" }, + { value: ".", type: "Dot" }, + { value: "split", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: "b", type: "StringLiteral" }, + { value: ")", type: "CloseParen" }, + { value: "|", type: "Pipe" }, + { value: "tojson", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "x", type: "Identifier" }, - { value: "[", type: "OpenSquareBracket" }, - { value: "5", type: "NumericLiteral" }, - { value: ":", type: "Colon" }, - { value: ":", type: "Colon" }, - { value: "-1", type: "NumericLiteral" }, - { value: "]", type: "CloseSquareBracket" }, + { value: "babbaccabbb", type: "StringLiteral" }, + { value: ".", type: "Dot" }, + { value: "split", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: "b", type: "StringLiteral" }, + { value: ",", type: "Comma" }, + { value: "2", type: "NumericLiteral" }, + { value: ")", type: "CloseParen" }, + { value: "|", type: "Pipe" }, + { value: "tojson", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, ], - - // Array indexing and slicing - ARRAY_SLICING: [ + SPLIT_5: [ { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "strings", type: "Identifier" }, - { value: "[", type: "OpenSquareBracket" }, + { value: " 1 2 3 4 5 ", type: "StringLiteral" }, + { value: ".", type: "Dot" }, + { value: "split", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: "none", type: "Identifier" }, + { value: ",", type: "Comma" }, { value: "0", type: "NumericLiteral" }, - { value: "]", type: "CloseSquareBracket" }, + { value: ")", type: "CloseParen" }, + { value: "|", type: "Pipe" }, + { value: "join", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: ",", type: "StringLiteral" }, + { value: ")", type: "CloseParen" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: " 1 2 3 4 5 ", type: "StringLiteral" }, + { value: ".", type: "Dot" }, + { value: "split", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: "none", type: "Identifier" }, + { value: ",", type: "Comma" }, + { value: "3", type: "NumericLiteral" }, + { value: ")", type: "CloseParen" }, + { value: "|", type: "Pipe" }, + { value: "join", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: ",", type: "StringLiteral" }, + { value: ")", type: "CloseParen" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: " 1 2 3 4 5 ", type: "StringLiteral" }, + { value: ".", type: "Dot" }, + { value: "split", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: " ", type: "StringLiteral" }, + { value: ",", type: "Comma" }, + { value: "0", type: "NumericLiteral" }, + { value: ")", type: "CloseParen" }, + { value: "|", type: "Pipe" }, + { value: "join", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: ",", type: "StringLiteral" }, + { value: ")", type: "CloseParen" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: " 1 2 3 4 5 ", type: "StringLiteral" }, + { value: ".", type: "Dot" }, + { value: "split", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: " ", type: "StringLiteral" }, + { value: ",", type: "Comma" }, + { value: "3", type: "NumericLiteral" }, + { value: ")", type: "CloseParen" }, + { value: "|", type: "Pipe" }, + { value: "join", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: ",", type: "StringLiteral" }, + { value: ")", type: "CloseParen" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: " 1 2 3 4 5 ", type: "StringLiteral" }, + { value: ".", type: "Dot" }, + { value: "split", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: " ", type: "StringLiteral" }, + { value: ",", type: "Comma" }, + { value: "10", type: "NumericLiteral" }, + { value: ")", type: "CloseParen" }, + { value: "|", type: "Pipe" }, + { value: "join", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: ",", type: "StringLiteral" }, + { value: ")", type: "CloseParen" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + ], + REPLACE: [ + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "test test", type: "StringLiteral" }, + { value: ".", type: "Dot" }, + { value: "replace", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: "test", type: "StringLiteral" }, + { value: ",", type: "Comma" }, + { value: "TEST", type: "StringLiteral" }, + { value: ")", type: "CloseParen" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "test test", type: "StringLiteral" }, + { value: ".", type: "Dot" }, + { value: "replace", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: "test", type: "StringLiteral" }, + { value: ",", type: "Comma" }, + { value: "TEST", type: "StringLiteral" }, + { value: ",", type: "Comma" }, + { value: "1", type: "NumericLiteral" }, + { value: ")", type: "CloseParen" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "test test", type: "StringLiteral" }, + { value: ".", type: "Dot" }, + { value: "replace", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: "", type: "StringLiteral" }, + { value: ",", type: "Comma" }, + { value: "_", type: "StringLiteral" }, + { value: ",", type: "Comma" }, + { value: "2", type: "NumericLiteral" }, + { value: ")", type: "CloseParen" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "abcabc", type: "StringLiteral" }, + { value: ".", type: "Dot" }, + { value: "replace", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: "a", type: "StringLiteral" }, + { value: ",", type: "Comma" }, + { value: "x", type: "StringLiteral" }, + { value: ",", type: "Comma" }, + { value: "count", type: "Identifier" }, + { value: "=", type: "Equals" }, + { value: "1", type: "NumericLiteral" }, + { value: ")", type: "CloseParen" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + ], + + // String indexing and slicing + STRING_SLICING: [ + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "x", type: "Identifier" }, + { value: "[", type: "OpenSquareBracket" }, + { value: "0", type: "NumericLiteral" }, + { value: "]", type: "CloseSquareBracket" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "x", type: "Identifier" }, + { value: "[", type: "OpenSquareBracket" }, + { value: ":", type: "Colon" }, + { value: "]", type: "CloseSquareBracket" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "x", type: "Identifier" }, + { value: "[", type: "OpenSquareBracket" }, + { value: ":", type: "Colon" }, + { value: "3", type: "NumericLiteral" }, + { value: "]", type: "CloseSquareBracket" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "x", type: "Identifier" }, + { value: "[", type: "OpenSquareBracket" }, + { value: "1", type: "NumericLiteral" }, + { value: ":", type: "Colon" }, + { value: "4", type: "NumericLiteral" }, + { value: "]", type: "CloseSquareBracket" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "x", type: "Identifier" }, + { value: "[", type: "OpenSquareBracket" }, + { value: "1", type: "NumericLiteral" }, + { value: ":", type: "Colon" }, + { value: "-1", type: "NumericLiteral" }, + { value: "]", type: "CloseSquareBracket" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "x", type: "Identifier" }, + { value: "[", type: "OpenSquareBracket" }, + { value: "1", type: "NumericLiteral" }, + { value: ":", type: "Colon" }, + { value: ":", type: "Colon" }, + { value: "2", type: "NumericLiteral" }, + { value: "]", type: "CloseSquareBracket" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "x", type: "Identifier" }, + { value: "[", type: "OpenSquareBracket" }, + { value: "5", type: "NumericLiteral" }, + { value: ":", type: "Colon" }, + { value: ":", type: "Colon" }, + { value: "-1", type: "NumericLiteral" }, + { value: "]", type: "CloseSquareBracket" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + ], + + // Array indexing and slicing + ARRAY_SLICING: [ + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "strings", type: "Identifier" }, + { value: "[", type: "OpenSquareBracket" }, + { value: "0", type: "NumericLiteral" }, + { value: "]", type: "CloseSquareBracket" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{%", type: "OpenStatement" }, - { value: "for", type: "For" }, + { value: "for", type: "Identifier" }, { value: "s", type: "Identifier" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "strings", type: "Identifier" }, { value: "[", type: "OpenSquareBracket" }, { value: ":", type: "Colon" }, @@ -1176,13 +1571,13 @@ const TEST_PARSED = { { value: "s", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "endfor", type: "EndFor" }, + { value: "endfor", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "|", type: "Text" }, { value: "{%", type: "OpenStatement" }, - { value: "for", type: "For" }, + { value: "for", type: "Identifier" }, { value: "s", type: "Identifier" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "strings", type: "Identifier" }, { value: "[", type: "OpenSquareBracket" }, { value: ":", type: "Colon" }, @@ -1193,13 +1588,13 @@ const TEST_PARSED = { { value: "s", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "endfor", type: "EndFor" }, + { value: "endfor", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "|", type: "Text" }, { value: "{%", type: "OpenStatement" }, - { value: "for", type: "For" }, + { value: "for", type: "Identifier" }, { value: "s", type: "Identifier" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "strings", type: "Identifier" }, { value: "[", type: "OpenSquareBracket" }, { value: "1", type: "NumericLiteral" }, @@ -1211,13 +1606,13 @@ const TEST_PARSED = { { value: "s", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "endfor", type: "EndFor" }, + { value: "endfor", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "|", type: "Text" }, { value: "{%", type: "OpenStatement" }, - { value: "for", type: "For" }, + { value: "for", type: "Identifier" }, { value: "s", type: "Identifier" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "strings", type: "Identifier" }, { value: "[", type: "OpenSquareBracket" }, { value: "1", type: "NumericLiteral" }, @@ -1229,13 +1624,13 @@ const TEST_PARSED = { { value: "s", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "endfor", type: "EndFor" }, + { value: "endfor", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "|", type: "Text" }, { value: "{%", type: "OpenStatement" }, - { value: "for", type: "For" }, + { value: "for", type: "Identifier" }, { value: "s", type: "Identifier" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "strings", type: "Identifier" }, { value: "[", type: "OpenSquareBracket" }, { value: "1", type: "NumericLiteral" }, @@ -1248,13 +1643,13 @@ const TEST_PARSED = { { value: "s", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "endfor", type: "EndFor" }, + { value: "endfor", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "|", type: "Text" }, { value: "{%", type: "OpenStatement" }, - { value: "for", type: "For" }, + { value: "for", type: "Identifier" }, { value: "s", type: "Identifier" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "strings", type: "Identifier" }, { value: "[", type: "OpenSquareBracket" }, { value: "5", type: "NumericLiteral" }, @@ -1267,7 +1662,7 @@ const TEST_PARSED = { { value: "s", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "endfor", type: "EndFor" }, + { value: "endfor", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "|", type: "Text" }, ], @@ -1277,37 +1672,37 @@ const TEST_PARSED = { { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "0", type: "NumericLiteral" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "arr", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "1", type: "NumericLiteral" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "arr", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "true", type: "BooleanLiteral" }, - { value: "in", type: "In" }, + { value: "true", type: "Identifier" }, + { value: "in", type: "Identifier" }, { value: "arr", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "false", type: "BooleanLiteral" }, - { value: "in", type: "In" }, + { value: "false", type: "Identifier" }, + { value: "in", type: "Identifier" }, { value: "arr", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "a", type: "StringLiteral" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "arr", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "b", type: "StringLiteral" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "arr", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, @@ -1315,44 +1710,44 @@ const TEST_PARSED = { MEMBERSHIP_NEGATION_1: [ { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "not", type: "UnaryOperator" }, + { value: "not", type: "Identifier" }, { value: "0", type: "NumericLiteral" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "arr", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "not", type: "UnaryOperator" }, + { value: "not", type: "Identifier" }, { value: "1", type: "NumericLiteral" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "arr", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "not", type: "UnaryOperator" }, - { value: "true", type: "BooleanLiteral" }, - { value: "in", type: "In" }, + { value: "not", type: "Identifier" }, + { value: "true", type: "Identifier" }, + { value: "in", type: "Identifier" }, { value: "arr", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "not", type: "UnaryOperator" }, - { value: "false", type: "BooleanLiteral" }, - { value: "in", type: "In" }, + { value: "not", type: "Identifier" }, + { value: "false", type: "Identifier" }, + { value: "in", type: "Identifier" }, { value: "arr", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "not", type: "UnaryOperator" }, + { value: "not", type: "Identifier" }, { value: "a", type: "StringLiteral" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "arr", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "not", type: "UnaryOperator" }, + { value: "not", type: "Identifier" }, { value: "b", type: "StringLiteral" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "arr", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, @@ -1361,41 +1756,86 @@ const TEST_PARSED = { { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "0", type: "NumericLiteral" }, - { value: "not in", type: "NotIn" }, + { value: "not", type: "Identifier" }, + { value: "in", type: "Identifier" }, { value: "arr", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "1", type: "NumericLiteral" }, - { value: "not in", type: "NotIn" }, + { value: "not", type: "Identifier" }, + { value: "in", type: "Identifier" }, { value: "arr", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "true", type: "BooleanLiteral" }, - { value: "not in", type: "NotIn" }, + { value: "true", type: "Identifier" }, + { value: "not", type: "Identifier" }, + { value: "in", type: "Identifier" }, { value: "arr", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "false", type: "BooleanLiteral" }, - { value: "not in", type: "NotIn" }, + { value: "false", type: "Identifier" }, + { value: "not", type: "Identifier" }, + { value: "in", type: "Identifier" }, { value: "arr", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "a", type: "StringLiteral" }, - { value: "not in", type: "NotIn" }, + { value: "not", type: "Identifier" }, + { value: "in", type: "Identifier" }, { value: "arr", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "b", type: "StringLiteral" }, - { value: "not in", type: "NotIn" }, + { value: "not", type: "Identifier" }, + { value: "in", type: "Identifier" }, { value: "arr", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, ], + MEMBERSHIP_UNDEFINED: [ + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "x", type: "Identifier" }, + { value: "is", type: "Identifier" }, + { value: "defined", type: "Identifier" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "y", type: "Identifier" }, + { value: "is", type: "Identifier" }, + { value: "defined", type: "Identifier" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "x", type: "Identifier" }, + { value: "in", type: "Identifier" }, + { value: "y", type: "Identifier" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "y", type: "Identifier" }, + { value: "in", type: "Identifier" }, + { value: "x", type: "Identifier" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "1", type: "NumericLiteral" }, + { value: "in", type: "Identifier" }, + { value: "y", type: "Identifier" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "1", type: "NumericLiteral" }, + { value: "in", type: "Identifier" }, + { value: "x", type: "Identifier" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + ], // Escaped characters ESCAPED_CHARS: [ @@ -1424,43 +1864,43 @@ const TEST_PARSED = { { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "", type: "StringLiteral" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "abc", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "a", type: "StringLiteral" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "abc", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "d", type: "StringLiteral" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "abc", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "ab", type: "StringLiteral" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "abc", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "ac", type: "StringLiteral" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "abc", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "abc", type: "StringLiteral" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "abc", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "abcd", type: "StringLiteral" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "abc", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, @@ -1636,7 +2076,7 @@ const TEST_PARSED = { { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "true", type: "BooleanLiteral" }, + { value: "true", type: "Identifier" }, { value: "|", type: "Pipe" }, { value: "tojson", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, @@ -1712,7 +2152,7 @@ const TEST_PARSED = { { value: "(", type: "OpenParen" }, { value: "first", type: "Identifier" }, { value: "=", type: "Equals" }, - { value: "True", type: "BooleanLiteral" }, + { value: "True", type: "Identifier" }, { value: ")", type: "CloseParen" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, @@ -1723,7 +2163,7 @@ const TEST_PARSED = { { value: "(", type: "OpenParen" }, { value: "blank", type: "Identifier" }, { value: "=", type: "Equals" }, - { value: "True", type: "BooleanLiteral" }, + { value: "True", type: "Identifier" }, { value: ")", type: "CloseParen" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, @@ -1736,7 +2176,7 @@ const TEST_PARSED = { { value: ",", type: "Comma" }, { value: "first", type: "Identifier" }, { value: "=", type: "Equals" }, - { value: "True", type: "BooleanLiteral" }, + { value: "True", type: "Identifier" }, { value: ")", type: "CloseParen" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, @@ -1776,66 +2216,251 @@ const TEST_PARSED = { { value: "string", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, ], - - // Logical operators between non-Booleans - BOOLEAN_NUMERICAL: [ + FILTER_OPERATOR_14: [ { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "1", type: "NumericLiteral" }, - { value: "and", type: "And" }, + { value: "1", type: "StringLiteral" }, + { value: "|", type: "Pipe" }, + { value: "int", type: "Identifier" }, + { value: "+", type: "AdditiveBinaryOperator" }, { value: "2", type: "NumericLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "1", type: "NumericLiteral" }, - { value: "and", type: "And" }, - { value: "0", type: "NumericLiteral" }, - { value: "}}", type: "CloseExpression" }, - { value: "|", type: "Text" }, - { value: "{{", type: "OpenExpression" }, - { value: "0", type: "NumericLiteral" }, - { value: "and", type: "And" }, - { value: "1", type: "NumericLiteral" }, + { value: "invalid", type: "StringLiteral" }, + { value: "|", type: "Pipe" }, + { value: "int", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "0", type: "NumericLiteral" }, - { value: "and", type: "And" }, - { value: "0", type: "NumericLiteral" }, + { value: "invalid", type: "StringLiteral" }, + { value: "|", type: "Pipe" }, + { value: "int", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: "-1", type: "NumericLiteral" }, + { value: ")", type: "CloseParen" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "1", type: "NumericLiteral" }, - { value: "or", type: "Or" }, - { value: "2", type: "NumericLiteral" }, + { value: "true", type: "Identifier" }, + { value: "|", type: "Pipe" }, + { value: "int", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "1", type: "NumericLiteral" }, - { value: "or", type: "Or" }, - { value: "0", type: "NumericLiteral" }, + { value: "false", type: "Identifier" }, + { value: "|", type: "Pipe" }, + { value: "int", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "0", type: "NumericLiteral" }, - { value: "or", type: "Or" }, - { value: "1", type: "NumericLiteral" }, + { value: "1.5", type: "NumericLiteral" }, + { value: "|", type: "Pipe" }, + { value: "int", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "0", type: "NumericLiteral" }, - { value: "or", type: "Or" }, - { value: "0", type: "NumericLiteral" }, + { value: "1.5", type: "StringLiteral" }, + { value: "|", type: "Pipe" }, + { value: "float", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "not", type: "UnaryOperator" }, - { value: "1", type: "NumericLiteral" }, + { value: "invalid", type: "StringLiteral" }, + { value: "|", type: "Pipe" }, + { value: "float", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "not", type: "UnaryOperator" }, - { value: "0", type: "NumericLiteral" }, + { value: "invalid", type: "StringLiteral" }, + { value: "|", type: "Pipe" }, + { value: "float", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: "hello", type: "StringLiteral" }, + { value: ")", type: "CloseParen" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + ], + FILTER_OPERATOR_15: [ + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "abcabcabc", type: "StringLiteral" }, + { value: "|", type: "Pipe" }, + { value: "replace", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: "a", type: "StringLiteral" }, + { value: ",", type: "Comma" }, + { value: "b", type: "StringLiteral" }, + { value: ")", type: "CloseParen" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "abcabcabc", type: "StringLiteral" }, + { value: "|", type: "Pipe" }, + { value: "replace", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: "a", type: "StringLiteral" }, + { value: ",", type: "Comma" }, + { value: "b", type: "StringLiteral" }, + { value: ",", type: "Comma" }, + { value: "1", type: "NumericLiteral" }, + { value: ")", type: "CloseParen" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "abcabcabc", type: "StringLiteral" }, + { value: "|", type: "Pipe" }, + { value: "replace", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: "a", type: "StringLiteral" }, + { value: ",", type: "Comma" }, + { value: "b", type: "StringLiteral" }, + { value: ",", type: "Comma" }, + { value: "count", type: "Identifier" }, + { value: "=", type: "Equals" }, + { value: "1", type: "NumericLiteral" }, + { value: ")", type: "CloseParen" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + ], + FILTER_OPERATOR_16: [ + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "undefined", type: "Identifier" }, + { value: "|", type: "Pipe" }, + { value: "default", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: "hello", type: "StringLiteral" }, + { value: ")", type: "CloseParen" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "false", type: "Identifier" }, + { value: "|", type: "Pipe" }, + { value: "default", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: "hello", type: "StringLiteral" }, + { value: ")", type: "CloseParen" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "false", type: "Identifier" }, + { value: "|", type: "Pipe" }, + { value: "default", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: "hello", type: "StringLiteral" }, + { value: ",", type: "Comma" }, + { value: "true", type: "Identifier" }, + { value: ")", type: "CloseParen" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "0", type: "NumericLiteral" }, + { value: "|", type: "Pipe" }, + { value: "default", type: "Identifier" }, + { value: "(", type: "OpenParen" }, + { value: "hello", type: "StringLiteral" }, + { value: ",", type: "Comma" }, + { value: "boolean", type: "Identifier" }, + { value: "=", type: "Equals" }, + { value: "true", type: "Identifier" }, + { value: ")", type: "CloseParen" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + ], + FILTER_OPERATOR_17: [ + { value: "{{", type: "OpenExpression" }, + { value: "[", type: "OpenSquareBracket" }, + { value: "1", type: "NumericLiteral" }, + { value: ",", type: "Comma" }, + { value: "2", type: "NumericLiteral" }, + { value: ",", type: "Comma" }, + { value: "1", type: "NumericLiteral" }, + { value: ",", type: "Comma" }, + { value: "-1", type: "NumericLiteral" }, + { value: ",", type: "Comma" }, + { value: "2", type: "NumericLiteral" }, + { value: "]", type: "CloseSquareBracket" }, + { value: "|", type: "Pipe" }, + { value: "unique", type: "Identifier" }, + { value: "|", type: "Pipe" }, + { value: "list", type: "Identifier" }, + { value: "|", type: "Pipe" }, + { value: "length", type: "Identifier" }, + { value: "}}", type: "CloseExpression" }, + ], + + // Filter statements + FILTER_STATEMENTS: [ + { value: "{%", type: "OpenStatement" }, + { value: "filter", type: "Identifier" }, + { value: "upper", type: "Identifier" }, + { value: "%}", type: "CloseStatement" }, + { value: "text", type: "Text" }, + { value: "{%", type: "OpenStatement" }, + { value: "endfilter", type: "Identifier" }, + { value: "%}", type: "CloseStatement" }, + ], + + // Logical operators between non-Booleans + BOOLEAN_NUMERICAL: [ + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "1", type: "NumericLiteral" }, + { value: "and", type: "Identifier" }, + { value: "2", type: "NumericLiteral" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "1", type: "NumericLiteral" }, + { value: "and", type: "Identifier" }, + { value: "0", type: "NumericLiteral" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "0", type: "NumericLiteral" }, + { value: "and", type: "Identifier" }, + { value: "1", type: "NumericLiteral" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "0", type: "NumericLiteral" }, + { value: "and", type: "Identifier" }, + { value: "0", type: "NumericLiteral" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "1", type: "NumericLiteral" }, + { value: "or", type: "Identifier" }, + { value: "2", type: "NumericLiteral" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "1", type: "NumericLiteral" }, + { value: "or", type: "Identifier" }, + { value: "0", type: "NumericLiteral" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "0", type: "NumericLiteral" }, + { value: "or", type: "Identifier" }, + { value: "1", type: "NumericLiteral" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "0", type: "NumericLiteral" }, + { value: "or", type: "Identifier" }, + { value: "0", type: "NumericLiteral" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "not", type: "Identifier" }, + { value: "1", type: "NumericLiteral" }, + { value: "}}", type: "CloseExpression" }, + { value: "|", type: "Text" }, + { value: "{{", type: "OpenExpression" }, + { value: "not", type: "Identifier" }, + { value: "0", type: "NumericLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, ], @@ -1843,59 +2468,59 @@ const TEST_PARSED = { { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "a", type: "StringLiteral" }, - { value: "and", type: "And" }, + { value: "and", type: "Identifier" }, { value: "b", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "a", type: "StringLiteral" }, - { value: "and", type: "And" }, + { value: "and", type: "Identifier" }, { value: "", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "", type: "StringLiteral" }, - { value: "and", type: "And" }, + { value: "and", type: "Identifier" }, { value: "a", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "", type: "StringLiteral" }, - { value: "and", type: "And" }, + { value: "and", type: "Identifier" }, { value: "", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "a", type: "StringLiteral" }, - { value: "or", type: "Or" }, + { value: "or", type: "Identifier" }, { value: "b", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "a", type: "StringLiteral" }, - { value: "or", type: "Or" }, + { value: "or", type: "Identifier" }, { value: "", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "", type: "StringLiteral" }, - { value: "or", type: "Or" }, + { value: "or", type: "Identifier" }, { value: "a", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "", type: "StringLiteral" }, - { value: "or", type: "Or" }, + { value: "or", type: "Identifier" }, { value: "", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "not", type: "UnaryOperator" }, + { value: "not", type: "Identifier" }, { value: "a", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "not", type: "UnaryOperator" }, + { value: "not", type: "Identifier" }, { value: "", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, @@ -1903,50 +2528,50 @@ const TEST_PARSED = { BOOLEAN_MIXED: [ { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "true", type: "BooleanLiteral" }, - { value: "and", type: "And" }, + { value: "true", type: "Identifier" }, + { value: "and", type: "Identifier" }, { value: "1", type: "NumericLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "true", type: "BooleanLiteral" }, - { value: "and", type: "And" }, + { value: "true", type: "Identifier" }, + { value: "and", type: "Identifier" }, { value: "0", type: "NumericLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "false", type: "BooleanLiteral" }, - { value: "and", type: "And" }, + { value: "false", type: "Identifier" }, + { value: "and", type: "Identifier" }, { value: "1", type: "NumericLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "false", type: "BooleanLiteral" }, - { value: "and", type: "And" }, + { value: "false", type: "Identifier" }, + { value: "and", type: "Identifier" }, { value: "0", type: "NumericLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "true", type: "BooleanLiteral" }, - { value: "or", type: "Or" }, + { value: "true", type: "Identifier" }, + { value: "or", type: "Identifier" }, { value: "1", type: "NumericLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "true", type: "BooleanLiteral" }, - { value: "or", type: "Or" }, + { value: "true", type: "Identifier" }, + { value: "or", type: "Identifier" }, { value: "0", type: "NumericLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "false", type: "BooleanLiteral" }, - { value: "or", type: "Or" }, + { value: "false", type: "Identifier" }, + { value: "or", type: "Identifier" }, { value: "1", type: "NumericLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "false", type: "BooleanLiteral" }, - { value: "or", type: "Or" }, + { value: "false", type: "Identifier" }, + { value: "or", type: "Identifier" }, { value: "0", type: "NumericLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, @@ -1954,98 +2579,98 @@ const TEST_PARSED = { BOOLEAN_MIXED_2: [ { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "true", type: "BooleanLiteral" }, - { value: "and", type: "And" }, + { value: "true", type: "Identifier" }, + { value: "and", type: "Identifier" }, { value: "", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "true", type: "BooleanLiteral" }, - { value: "and", type: "And" }, + { value: "true", type: "Identifier" }, + { value: "and", type: "Identifier" }, { value: "a", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "false", type: "BooleanLiteral" }, - { value: "or", type: "Or" }, + { value: "false", type: "Identifier" }, + { value: "or", type: "Identifier" }, { value: "", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "false", type: "BooleanLiteral" }, - { value: "or", type: "Or" }, + { value: "false", type: "Identifier" }, + { value: "or", type: "Identifier" }, { value: "a", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "", type: "StringLiteral" }, - { value: "and", type: "And" }, - { value: "true", type: "BooleanLiteral" }, + { value: "and", type: "Identifier" }, + { value: "true", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "a", type: "StringLiteral" }, - { value: "and", type: "And" }, - { value: "true", type: "BooleanLiteral" }, + { value: "and", type: "Identifier" }, + { value: "true", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "", type: "StringLiteral" }, - { value: "or", type: "Or" }, - { value: "false", type: "BooleanLiteral" }, + { value: "or", type: "Identifier" }, + { value: "false", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "a", type: "StringLiteral" }, - { value: "or", type: "Or" }, - { value: "false", type: "BooleanLiteral" }, + { value: "or", type: "Identifier" }, + { value: "false", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, ], BOOLEAN_MIXED_IF: [ { value: "{%", type: "OpenStatement" }, - { value: "if", type: "If" }, + { value: "if", type: "Identifier" }, { value: "", type: "StringLiteral" }, { value: "%}", type: "CloseStatement" }, { value: "{{", type: "OpenExpression" }, { value: "A", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "endif", type: "EndIf" }, + { value: "endif", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "if", type: "If" }, + { value: "if", type: "Identifier" }, { value: "a", type: "StringLiteral" }, { value: "%}", type: "CloseStatement" }, { value: "{{", type: "OpenExpression" }, { value: "B", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "endif", type: "EndIf" }, + { value: "endif", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "if", type: "If" }, - { value: "true", type: "BooleanLiteral" }, - { value: "and", type: "And" }, + { value: "if", type: "Identifier" }, + { value: "true", type: "Identifier" }, + { value: "and", type: "Identifier" }, { value: "", type: "StringLiteral" }, { value: "%}", type: "CloseStatement" }, { value: "{{", type: "OpenExpression" }, { value: "C", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "endif", type: "EndIf" }, + { value: "endif", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "if", type: "If" }, - { value: "true", type: "BooleanLiteral" }, - { value: "and", type: "And" }, + { value: "if", type: "Identifier" }, + { value: "true", type: "Identifier" }, + { value: "and", type: "Identifier" }, { value: "a", type: "StringLiteral" }, { value: "%}", type: "CloseStatement" }, { value: "{{", type: "OpenExpression" }, { value: "D", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "endif", type: "EndIf" }, + { value: "endif", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, ], @@ -2054,27 +2679,27 @@ const TEST_PARSED = { { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "unknown_var", type: "Identifier" }, - { value: "is", type: "Is" }, + { value: "is", type: "Identifier" }, { value: "defined", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "unknown_var", type: "Identifier" }, - { value: "is", type: "Is" }, - { value: "not", type: "UnaryOperator" }, + { value: "is", type: "Identifier" }, + { value: "not", type: "Identifier" }, { value: "defined", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "known_var", type: "Identifier" }, - { value: "is", type: "Is" }, + { value: "is", type: "Identifier" }, { value: "defined", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "known_var", type: "Identifier" }, - { value: "is", type: "Is" }, - { value: "not", type: "UnaryOperator" }, + { value: "is", type: "Identifier" }, + { value: "not", type: "Identifier" }, { value: "defined", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, @@ -2082,40 +2707,40 @@ const TEST_PARSED = { IS_OPERATOR_2: [ { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "true", type: "BooleanLiteral" }, - { value: "is", type: "Is" }, - { value: "true", type: "BooleanLiteral" }, + { value: "true", type: "Identifier" }, + { value: "is", type: "Identifier" }, + { value: "true", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "true", type: "BooleanLiteral" }, - { value: "is", type: "Is" }, - { value: "not", type: "UnaryOperator" }, - { value: "true", type: "BooleanLiteral" }, + { value: "true", type: "Identifier" }, + { value: "is", type: "Identifier" }, + { value: "not", type: "Identifier" }, + { value: "true", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "true", type: "BooleanLiteral" }, - { value: "is", type: "Is" }, - { value: "false", type: "BooleanLiteral" }, + { value: "true", type: "Identifier" }, + { value: "is", type: "Identifier" }, + { value: "false", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "true", type: "BooleanLiteral" }, - { value: "is", type: "Is" }, - { value: "not", type: "UnaryOperator" }, - { value: "false", type: "BooleanLiteral" }, + { value: "true", type: "Identifier" }, + { value: "is", type: "Identifier" }, + { value: "not", type: "Identifier" }, + { value: "false", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "true", type: "BooleanLiteral" }, - { value: "is", type: "Is" }, + { value: "true", type: "Identifier" }, + { value: "is", type: "Identifier" }, { value: "boolean", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "1", type: "NumericLiteral" }, - { value: "is", type: "Is" }, + { value: "is", type: "Identifier" }, { value: "boolean", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, @@ -2124,49 +2749,49 @@ const TEST_PARSED = { { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "1", type: "NumericLiteral" }, - { value: "is", type: "Is" }, + { value: "is", type: "Identifier" }, { value: "odd", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "2", type: "NumericLiteral" }, - { value: "is", type: "Is" }, + { value: "is", type: "Identifier" }, { value: "odd", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "1", type: "NumericLiteral" }, - { value: "is", type: "Is" }, + { value: "is", type: "Identifier" }, { value: "even", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "2", type: "NumericLiteral" }, - { value: "is", type: "Is" }, + { value: "is", type: "Identifier" }, { value: "even", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "2", type: "NumericLiteral" }, - { value: "is", type: "Is" }, + { value: "is", type: "Identifier" }, { value: "number", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "2", type: "StringLiteral" }, - { value: "is", type: "Is" }, + { value: "is", type: "Identifier" }, { value: "number", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "2", type: "NumericLiteral" }, - { value: "is", type: "Is" }, + { value: "is", type: "Identifier" }, { value: "integer", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "2", type: "StringLiteral" }, - { value: "is", type: "Is" }, + { value: "is", type: "Identifier" }, { value: "integer", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, @@ -2175,25 +2800,25 @@ const TEST_PARSED = { { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "func", type: "Identifier" }, - { value: "is", type: "Is" }, + { value: "is", type: "Identifier" }, { value: "callable", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "2", type: "NumericLiteral" }, - { value: "is", type: "Is" }, + { value: "is", type: "Identifier" }, { value: "callable", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "1", type: "NumericLiteral" }, - { value: "is", type: "Is" }, + { value: "is", type: "Identifier" }, { value: "iterable", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "hello", type: "StringLiteral" }, - { value: "is", type: "Is" }, + { value: "is", type: "Identifier" }, { value: "iterable", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, @@ -2202,25 +2827,25 @@ const TEST_PARSED = { { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "a", type: "StringLiteral" }, - { value: "is", type: "Is" }, + { value: "is", type: "Identifier" }, { value: "lower", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "A", type: "StringLiteral" }, - { value: "is", type: "Is" }, + { value: "is", type: "Identifier" }, { value: "lower", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "a", type: "StringLiteral" }, - { value: "is", type: "Is" }, + { value: "is", type: "Identifier" }, { value: "upper", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "A", type: "StringLiteral" }, - { value: "is", type: "Is" }, + { value: "is", type: "Identifier" }, { value: "upper", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, @@ -2229,25 +2854,25 @@ const TEST_PARSED = { { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "string", type: "Identifier" }, - { value: "is", type: "Is" }, + { value: "is", type: "Identifier" }, { value: "mapping", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "number", type: "Identifier" }, - { value: "is", type: "Is" }, + { value: "is", type: "Identifier" }, { value: "mapping", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "array", type: "Identifier" }, - { value: "is", type: "Is" }, + { value: "is", type: "Identifier" }, { value: "mapping", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "dict", type: "Identifier" }, - { value: "is", type: "Is" }, + { value: "is", type: "Identifier" }, { value: "mapping", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, @@ -2256,8 +2881,8 @@ const TEST_PARSED = { // Short-circuit evaluation SHORT_CIRCUIT: [ { value: "{{", type: "OpenExpression" }, - { value: "false", type: "BooleanLiteral" }, - { value: "and", type: "And" }, + { value: "false", type: "Identifier" }, + { value: "and", type: "Identifier" }, { value: "raise_exception", type: "Identifier" }, { value: "(", type: "OpenParen" }, { value: "This should not be printed", type: "StringLiteral" }, @@ -2266,8 +2891,8 @@ const TEST_PARSED = { ], SHORT_CIRCUIT_1: [ { value: "{{", type: "OpenExpression" }, - { value: "true", type: "BooleanLiteral" }, - { value: "or", type: "Or" }, + { value: "true", type: "Identifier" }, + { value: "or", type: "Identifier" }, { value: "raise_exception", type: "Identifier" }, { value: "(", type: "OpenParen" }, { value: "This should not be printed", type: "StringLiteral" }, @@ -2278,7 +2903,7 @@ const TEST_PARSED = { // Namespaces NAMESPACE: [ { value: "{%", type: "OpenStatement" }, - { value: "set", type: "Set" }, + { value: "set", type: "Identifier" }, { value: "ns", type: "Identifier" }, { value: "=", type: "Equals" }, { value: "namespace", type: "Identifier" }, @@ -2286,7 +2911,7 @@ const TEST_PARSED = { { value: ")", type: "CloseParen" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "set", type: "Set" }, + { value: "set", type: "Identifier" }, { value: "ns", type: "Identifier" }, { value: ".", type: "Dot" }, { value: "foo", type: "Identifier" }, @@ -2301,14 +2926,14 @@ const TEST_PARSED = { ], NAMESPACE_1: [ { value: "{%", type: "OpenStatement" }, - { value: "set", type: "Set" }, + { value: "set", type: "Identifier" }, { value: "ns", type: "Identifier" }, { value: "=", type: "Equals" }, { value: "namespace", type: "Identifier" }, { value: "(", type: "OpenParen" }, { value: "default", type: "Identifier" }, { value: "=", type: "Equals" }, - { value: "false", type: "BooleanLiteral" }, + { value: "false", type: "Identifier" }, { value: ")", type: "CloseParen" }, { value: "%}", type: "CloseStatement" }, { value: "{{", type: "OpenExpression" }, @@ -2319,14 +2944,14 @@ const TEST_PARSED = { ], NAMESPACE_2: [ { value: "{%", type: "OpenStatement" }, - { value: "set", type: "Set" }, + { value: "set", type: "Identifier" }, { value: "ns", type: "Identifier" }, { value: "=", type: "Equals" }, { value: "namespace", type: "Identifier" }, { value: "(", type: "OpenParen" }, { value: "default", type: "Identifier" }, { value: "=", type: "Equals" }, - { value: "false", type: "BooleanLiteral" }, + { value: "false", type: "Identifier" }, { value: ",", type: "Comma" }, { value: "number", type: "Identifier" }, { value: "=", type: "Equals" }, @@ -2355,25 +2980,27 @@ const TEST_PARSED = { { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "known", type: "StringLiteral" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "obj", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "known", type: "StringLiteral" }, - { value: "not in", type: "NotIn" }, + { value: "not", type: "Identifier" }, + { value: "in", type: "Identifier" }, { value: "obj", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "unknown", type: "StringLiteral" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "obj", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "unknown", type: "StringLiteral" }, - { value: "not in", type: "NotIn" }, + { value: "not", type: "Identifier" }, + { value: "in", type: "Identifier" }, { value: "obj", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, @@ -2396,8 +3023,8 @@ const TEST_PARSED = { { value: "(", type: "OpenParen" }, { value: "unknown", type: "StringLiteral" }, { value: ")", type: "CloseParen" }, - { value: "is", type: "Is" }, - { value: "none", type: "NullLiteral" }, + { value: "is", type: "Identifier" }, + { value: "none", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, @@ -2407,7 +3034,7 @@ const TEST_PARSED = { { value: "(", type: "OpenParen" }, { value: "unknown", type: "StringLiteral" }, { value: ")", type: "CloseParen" }, - { value: "is", type: "Is" }, + { value: "is", type: "Identifier" }, { value: "defined", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, @@ -2415,11 +3042,11 @@ const TEST_PARSED = { OBJECT_OPERATORS_2: [ { value: "|", type: "Text" }, { value: "{%", type: "OpenStatement" }, - { value: "for", type: "For" }, + { value: "for", type: "Identifier" }, { value: "x", type: "Identifier" }, { value: ",", type: "Comma" }, { value: "y", type: "Identifier" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "obj", type: "Identifier" }, { value: ".", type: "Dot" }, { value: "items", type: "Identifier" }, @@ -2436,7 +3063,7 @@ const TEST_PARSED = { { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{%", type: "OpenStatement" }, - { value: "endfor", type: "EndFor" }, + { value: "endfor", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "|", type: "Text" }, ], @@ -2444,24 +3071,24 @@ const TEST_PARSED = { // Scope SCOPE: [ { value: "{%", type: "OpenStatement" }, - { value: "set", type: "Set" }, + { value: "set", type: "Identifier" }, { value: "ns", type: "Identifier" }, { value: "=", type: "Equals" }, { value: "namespace", type: "Identifier" }, { value: "(", type: "OpenParen" }, { value: "found", type: "Identifier" }, { value: "=", type: "Equals" }, - { value: "false", type: "BooleanLiteral" }, + { value: "false", type: "Identifier" }, { value: ")", type: "CloseParen" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "for", type: "For" }, + { value: "for", type: "Identifier" }, { value: "num", type: "Identifier" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "nums", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "if", type: "If" }, + { value: "if", type: "Identifier" }, { value: "num", type: "Identifier" }, { value: "==", type: "ComparisonBinaryOperator" }, { value: "1", type: "NumericLiteral" }, @@ -2470,18 +3097,18 @@ const TEST_PARSED = { { value: "found=", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "set", type: "Set" }, + { value: "set", type: "Identifier" }, { value: "ns", type: "Identifier" }, { value: ".", type: "Dot" }, { value: "found", type: "Identifier" }, { value: "=", type: "Equals" }, - { value: "true", type: "BooleanLiteral" }, + { value: "true", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "endif", type: "EndIf" }, + { value: "endif", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "endfor", type: "EndFor" }, + { value: "endfor", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "{{", type: "OpenExpression" }, { value: "ns", type: "Identifier" }, @@ -2491,19 +3118,19 @@ const TEST_PARSED = { ], SCOPE_1: [ { value: "{%", type: "OpenStatement" }, - { value: "set", type: "Set" }, + { value: "set", type: "Identifier" }, { value: "found", type: "Identifier" }, { value: "=", type: "Equals" }, - { value: "false", type: "BooleanLiteral" }, + { value: "false", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "for", type: "For" }, + { value: "for", type: "Identifier" }, { value: "num", type: "Identifier" }, - { value: "in", type: "In" }, + { value: "in", type: "Identifier" }, { value: "nums", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "if", type: "If" }, + { value: "if", type: "Identifier" }, { value: "num", type: "Identifier" }, { value: "==", type: "ComparisonBinaryOperator" }, { value: "1", type: "NumericLiteral" }, @@ -2512,16 +3139,16 @@ const TEST_PARSED = { { value: "found=", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "set", type: "Set" }, + { value: "set", type: "Identifier" }, { value: "found", type: "Identifier" }, { value: "=", type: "Equals" }, - { value: "true", type: "BooleanLiteral" }, + { value: "true", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "endif", type: "EndIf" }, + { value: "endif", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "endfor", type: "EndFor" }, + { value: "endfor", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "{{", type: "OpenExpression" }, { value: "found", type: "Identifier" }, @@ -2545,39 +3172,39 @@ const TEST_PARSED = { // Null NULL_VARIABLE: [ { value: "{%", type: "OpenStatement" }, - { value: "if", type: "If" }, - { value: "not", type: "UnaryOperator" }, + { value: "if", type: "Identifier" }, + { value: "not", type: "Identifier" }, { value: "null_val", type: "Identifier" }, - { value: "is", type: "Is" }, + { value: "is", type: "Identifier" }, { value: "defined", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "set", type: "Set" }, + { value: "set", type: "Identifier" }, { value: "null_val", type: "Identifier" }, { value: "=", type: "Equals" }, - { value: "none", type: "NullLiteral" }, + { value: "none", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "endif", type: "EndIf" }, + { value: "endif", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "{%", type: "OpenStatement" }, - { value: "if", type: "If" }, + { value: "if", type: "Identifier" }, { value: "null_val", type: "Identifier" }, - { value: "is", type: "Is" }, - { value: "not", type: "UnaryOperator" }, - { value: "none", type: "NullLiteral" }, + { value: "is", type: "Identifier" }, + { value: "not", type: "Identifier" }, + { value: "none", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "{{", type: "OpenExpression" }, { value: "fail", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "else", type: "Else" }, + { value: "else", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "{{", type: "OpenExpression" }, { value: "pass", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "endif", type: "EndIf" }, + { value: "endif", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, ], @@ -2586,66 +3213,111 @@ const TEST_PARSED = { { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "a", type: "StringLiteral" }, - { value: "if", type: "If" }, - { value: "true", type: "BooleanLiteral" }, - { value: "else", type: "Else" }, + { value: "if", type: "Identifier" }, + { value: "true", type: "Identifier" }, + { value: "else", type: "Identifier" }, { value: "b", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "a", type: "StringLiteral" }, - { value: "if", type: "If" }, - { value: "false", type: "BooleanLiteral" }, - { value: "else", type: "Else" }, + { value: "if", type: "Identifier" }, + { value: "false", type: "Identifier" }, + { value: "else", type: "Identifier" }, { value: "b", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "a", type: "StringLiteral" }, - { value: "if", type: "If" }, + { value: "if", type: "Identifier" }, { value: "1", type: "NumericLiteral" }, { value: "+", type: "AdditiveBinaryOperator" }, { value: "1", type: "NumericLiteral" }, { value: "==", type: "ComparisonBinaryOperator" }, { value: "2", type: "NumericLiteral" }, - { value: "else", type: "Else" }, + { value: "else", type: "Identifier" }, { value: "b", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, { value: "a", type: "StringLiteral" }, - { value: "if", type: "If" }, + { value: "if", type: "Identifier" }, { value: "1", type: "NumericLiteral" }, { value: "+", type: "AdditiveBinaryOperator" }, { value: "1", type: "NumericLiteral" }, { value: "==", type: "ComparisonBinaryOperator" }, { value: "3", type: "NumericLiteral" }, - { value: "or", type: "Or" }, + { value: "or", type: "Identifier" }, { value: "1", type: "NumericLiteral" }, { value: "*", type: "MultiplicativeBinaryOperator" }, { value: "2", type: "NumericLiteral" }, { value: "==", type: "ComparisonBinaryOperator" }, { value: "3", type: "NumericLiteral" }, - { value: "else", type: "Else" }, + { value: "else", type: "Identifier" }, { value: "b", type: "StringLiteral" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, ], + TERNARY_OPERATOR_1: [ + { value: "{{", type: "OpenExpression" }, + { value: "(", type: "OpenParen" }, + { value: "x", type: "Identifier" }, + { value: "if", type: "Identifier" }, + { value: "true", type: "Identifier" }, + { value: "else", type: "Identifier" }, + { value: "[", type: "OpenSquareBracket" }, + { value: "]", type: "CloseSquareBracket" }, + { value: ")", type: "CloseParen" }, + { value: "|", type: "Pipe" }, + { value: "length", type: "Identifier" }, + { value: "}}", type: "CloseExpression" }, + ], TERNARY_SET: [ { value: "{%", type: "OpenStatement" }, - { value: "set", type: "Set" }, + { value: "set", type: "Identifier" }, + { value: "x", type: "Identifier" }, + { value: "=", type: "Equals" }, + { value: "1", type: "NumericLiteral" }, + { value: "if", type: "Identifier" }, + { value: "True", type: "Identifier" }, + { value: "else", type: "Identifier" }, + { value: "2", type: "NumericLiteral" }, + { value: "%}", type: "CloseStatement" }, + { value: "{{", type: "OpenExpression" }, + { value: "x", type: "Identifier" }, + { value: "}}", type: "CloseExpression" }, + ], + TERNARY_CONSECUTIVE: [ + { value: "{%", type: "OpenStatement" }, + { value: "set", type: "Identifier" }, { value: "x", type: "Identifier" }, { value: "=", type: "Equals" }, { value: "1", type: "NumericLiteral" }, - { value: "if", type: "If" }, - { value: "True", type: "BooleanLiteral" }, - { value: "else", type: "Else" }, + { value: "if", type: "Identifier" }, + { value: "False", type: "Identifier" }, + { value: "else", type: "Identifier" }, { value: "2", type: "NumericLiteral" }, + { value: "if", type: "Identifier" }, + { value: "False", type: "Identifier" }, + { value: "else", type: "Identifier" }, + { value: "3", type: "NumericLiteral" }, { value: "%}", type: "CloseStatement" }, { value: "{{", type: "OpenExpression" }, { value: "x", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, ], + TERNARY_SHORTCUT: [ + { value: "{{", type: "OpenExpression" }, + { value: "foo", type: "StringLiteral" }, + { value: "if", type: "Identifier" }, + { value: "false", type: "Identifier" }, + { value: "}}", type: "CloseExpression" }, + { value: "{{", type: "OpenExpression" }, + { value: "bar", type: "StringLiteral" }, + { value: "if", type: "Identifier" }, + { value: "true", type: "Identifier" }, + { value: "}}", type: "CloseExpression" }, + ], // Array literals ARRAY_LITERALS: [ @@ -2653,7 +3325,7 @@ const TEST_PARSED = { { value: "[", type: "OpenSquareBracket" }, { value: "1", type: "NumericLiteral" }, { value: ",", type: "Comma" }, - { value: "true", type: "BooleanLiteral" }, + { value: "true", type: "Identifier" }, { value: ",", type: "Comma" }, { value: "hello", type: "StringLiteral" }, { value: ",", type: "Comma" }, @@ -2720,6 +3392,25 @@ const TEST_PARSED = { { value: "]", type: "CloseSquareBracket" }, { value: "}}", type: "CloseExpression" }, ], + OBJECT_LITERALS_1: [ + { value: "{{", type: "OpenExpression" }, + { value: "{", type: "OpenCurlyBracket" }, + { value: "key", type: "StringLiteral" }, + { value: ":", type: "Colon" }, + { value: "{", type: "OpenCurlyBracket" }, + { value: "key", type: "StringLiteral" }, + { value: ":", type: "Colon" }, + { value: "value", type: "StringLiteral" }, + { value: "}", type: "CloseCurlyBracket" }, + { value: "}", type: "CloseCurlyBracket" }, + { value: "[", type: "OpenSquareBracket" }, + { value: "key", type: "StringLiteral" }, + { value: "]", type: "CloseSquareBracket" }, + { value: "[", type: "OpenSquareBracket" }, + { value: "key", type: "StringLiteral" }, + { value: "]", type: "CloseSquareBracket" }, + { value: "}}", type: "CloseExpression" }, + ], // Array operators ARRAY_OPERATORS: [ @@ -2749,7 +3440,7 @@ const TEST_PARSED = { // Macros MACROS: [ { value: "{%", type: "OpenStatement" }, - { value: "macro", type: "Macro" }, + { value: "macro", type: "Identifier" }, { value: "hello", type: "Identifier" }, { value: "(", type: "OpenParen" }, { value: "name", type: "Identifier" }, @@ -2761,7 +3452,7 @@ const TEST_PARSED = { { value: "name", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "endmacro", type: "EndMacro" }, + { value: "endmacro", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, @@ -2781,7 +3472,7 @@ const TEST_PARSED = { ], MACROS_1: [ { value: "{%", type: "OpenStatement" }, - { value: "macro", type: "Macro" }, + { value: "macro", type: "Identifier" }, { value: "hello", type: "Identifier" }, { value: "(", type: "OpenParen" }, { value: "name", type: "Identifier" }, @@ -2799,7 +3490,7 @@ const TEST_PARSED = { { value: "suffix", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "endmacro", type: "EndMacro" }, + { value: "endmacro", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, @@ -2832,7 +3523,7 @@ const TEST_PARSED = { ], MACROS_2: [ { value: "{%", type: "OpenStatement" }, - { value: "macro", type: "Macro" }, + { value: "macro", type: "Identifier" }, { value: "fn", type: "Identifier" }, { value: "(", type: "OpenParen" }, { value: "x", type: "Identifier" }, @@ -2858,7 +3549,7 @@ const TEST_PARSED = { { value: "z", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "{%", type: "OpenStatement" }, - { value: "endmacro", type: "EndMacro" }, + { value: "endmacro", type: "Identifier" }, { value: "%}", type: "CloseStatement" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, @@ -2915,193 +3606,207 @@ const TEST_PARSED = { { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, ], - - RSTRIP: [ - { value: "{{", type: "OpenExpression" }, - { value: " test it ", type: "StringLiteral" }, - { value: ".", type: "Dot" }, - { value: "rstrip", type: "Identifier" }, + MACROS_3: [ + { value: "{%", type: "OpenStatement" }, + { value: "macro", type: "Identifier" }, + { value: "dummy", type: "Identifier" }, { value: "(", type: "OpenParen" }, + { value: "a", type: "Identifier" }, + { value: ",", type: "Comma" }, + { value: "b", type: "Identifier" }, + { value: "=", type: "Equals" }, + { value: "!", type: "StringLiteral" }, { value: ")", type: "CloseParen" }, - { value: "}}", type: "CloseExpression" }, - ], - LSTRIP: [ + { value: "%}", type: "CloseStatement" }, { value: "{{", type: "OpenExpression" }, - { value: " test it ", type: "StringLiteral" }, - { value: ".", type: "Dot" }, - { value: "lstrip", type: "Identifier" }, - { value: "(", type: "OpenParen" }, - { value: ")", type: "CloseParen" }, + { value: "a", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, - ], - SPLIT: [ - { value: "|", type: "Text" }, + { value: " ", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: " test it ", type: "StringLiteral" }, - { value: ".", type: "Dot" }, - { value: "split", type: "Identifier" }, - { value: "(", type: "OpenParen" }, - { value: ")", type: "CloseParen" }, - { value: "|", type: "Pipe" }, - { value: "join", type: "Identifier" }, + { value: "caller", type: "Identifier" }, { value: "(", type: "OpenParen" }, - { value: "|", type: "StringLiteral" }, { value: ")", type: "CloseParen" }, { value: "}}", type: "CloseExpression" }, - { value: "|", type: "Text" }, - ], - SPLIT_2: [ - { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: " test it ", type: "StringLiteral" }, - { value: ".", type: "Dot" }, - { value: "split", type: "Identifier" }, + { value: "b", type: "Identifier" }, + { value: "}}", type: "CloseExpression" }, + { value: "{%", type: "OpenStatement" }, + { value: "endmacro", type: "Identifier" }, + { value: "%}", type: "CloseStatement" }, + { value: "{%", type: "OpenStatement" }, + { value: "call", type: "Identifier" }, + { value: "dummy", type: "Identifier" }, { value: "(", type: "OpenParen" }, - { value: " ", type: "StringLiteral" }, + { value: "hello", type: "StringLiteral" }, { value: ")", type: "CloseParen" }, - { value: "|", type: "Pipe" }, - { value: "join", type: "Identifier" }, + { value: "%}", type: "CloseStatement" }, + { value: "name", type: "Text" }, + { value: "{%", type: "OpenStatement" }, + { value: "endcall", type: "Identifier" }, + { value: "%}", type: "CloseStatement" }, + ], + MACROS_4: [ + { value: "{%", type: "OpenStatement" }, + { value: "macro", type: "Identifier" }, + { value: "print_users", type: "Identifier" }, { value: "(", type: "OpenParen" }, - { value: "|", type: "StringLiteral" }, + { value: "users", type: "Identifier" }, { value: ")", type: "CloseParen" }, - { value: "}}", type: "CloseExpression" }, - { value: "|", type: "Text" }, - ], - SPLIT_3: [ - { value: "|", type: "Text" }, + { value: "%}", type: "CloseStatement" }, + { value: "{%", type: "OpenStatement" }, + { value: "for", type: "Identifier" }, + { value: "user", type: "Identifier" }, + { value: "in", type: "Identifier" }, + { value: "users", type: "Identifier" }, + { value: "%}", type: "CloseStatement" }, { value: "{{", type: "OpenExpression" }, - { value: " test it ", type: "StringLiteral" }, - { value: ".", type: "Dot" }, - { value: "split", type: "Identifier" }, + { value: "caller", type: "Identifier" }, { value: "(", type: "OpenParen" }, - { value: " ", type: "StringLiteral" }, - { value: ",", type: "Comma" }, - { value: "4", type: "NumericLiteral" }, + { value: "user", type: "Identifier" }, { value: ")", type: "CloseParen" }, - { value: "|", type: "Pipe" }, - { value: "join", type: "Identifier" }, + { value: "}}", type: "CloseExpression" }, + { value: "{%", type: "OpenStatement" }, + { value: "endfor", type: "Identifier" }, + { value: "%}", type: "CloseStatement" }, + { value: "{%", type: "OpenStatement" }, + { value: "endmacro", type: "Identifier" }, + { value: "%}", type: "CloseStatement" }, + { value: "{%", type: "OpenStatement" }, + { value: "call", type: "Identifier" }, { value: "(", type: "OpenParen" }, - { value: "|", type: "StringLiteral" }, + { value: "user", type: "Identifier" }, { value: ")", type: "CloseParen" }, - { value: "}}", type: "CloseExpression" }, - { value: "|", type: "Text" }, - ], - SPLIT_4: [ - { value: "|", type: "Text" }, - { value: "{{", type: "OpenExpression" }, - { value: " 1 2 3 ", type: "StringLiteral" }, - { value: ".", type: "Dot" }, - { value: "split", type: "Identifier" }, + { value: "print_users", type: "Identifier" }, { value: "(", type: "OpenParen" }, + { value: "users", type: "Identifier" }, { value: ")", type: "CloseParen" }, - { value: "|", type: "Pipe" }, - { value: "tojson", type: "Identifier" }, - { value: "}}", type: "CloseExpression" }, - { value: "|", type: "Text" }, + { value: "%}", type: "CloseStatement" }, + { value: " - ", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "babbaccabbb", type: "StringLiteral" }, + { value: "user", type: "Identifier" }, { value: ".", type: "Dot" }, - { value: "split", type: "Identifier" }, - { value: "(", type: "OpenParen" }, - { value: "b", type: "StringLiteral" }, - { value: ")", type: "CloseParen" }, - { value: "|", type: "Pipe" }, - { value: "tojson", type: "Identifier" }, + { value: "firstname", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, - { value: "|", type: "Text" }, + { value: " ", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: "babbaccabbb", type: "StringLiteral" }, + { value: "user", type: "Identifier" }, { value: ".", type: "Dot" }, - { value: "split", type: "Identifier" }, - { value: "(", type: "OpenParen" }, - { value: "b", type: "StringLiteral" }, - { value: ",", type: "Comma" }, - { value: "2", type: "NumericLiteral" }, - { value: ")", type: "CloseParen" }, - { value: "|", type: "Pipe" }, - { value: "tojson", type: "Identifier" }, + { value: "lastname", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, - { value: "|", type: "Text" }, + { value: "\n", type: "Text" }, + { value: "{%", type: "OpenStatement" }, + { value: "endcall", type: "Identifier" }, + { value: "%}", type: "CloseStatement" }, ], - SPLIT_5: [ + + // Context-specific keywords + CONTEXT_KEYWORDS: [ + { value: "{%", type: "OpenStatement" }, + { value: "if", type: "Identifier" }, + { value: "if", type: "Identifier" }, + { value: "in", type: "Identifier" }, + { value: "in", type: "Identifier" }, + { value: "%}", type: "CloseStatement" }, + { value: "a", type: "Text" }, + { value: "{%", type: "OpenStatement" }, + { value: "endif", type: "Identifier" }, + { value: "%}", type: "CloseStatement" }, + { value: "{%", type: "OpenStatement" }, + { value: "set", type: "Identifier" }, + { value: "if", type: "Identifier" }, + { value: "=", type: "Equals" }, + { value: "a", type: "StringLiteral" }, + { value: "%}", type: "CloseStatement" }, + { value: "{%", type: "OpenStatement" }, + { value: "set", type: "Identifier" }, + { value: "in", type: "Identifier" }, + { value: "=", type: "Equals" }, + { value: "abc", type: "StringLiteral" }, + { value: "%}", type: "CloseStatement" }, + { value: "{%", type: "OpenStatement" }, + { value: "if", type: "Identifier" }, + { value: "if", type: "Identifier" }, + { value: "in", type: "Identifier" }, + { value: "in", type: "Identifier" }, + { value: "%}", type: "CloseStatement" }, + { value: "b", type: "Text" }, + { value: "{%", type: "OpenStatement" }, + { value: "endif", type: "Identifier" }, + { value: "%}", type: "CloseStatement" }, + ], + CONTEXT_KEYWORDS_1: [ { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: " 1 2 3 4 5 ", type: "StringLiteral" }, - { value: ".", type: "Dot" }, - { value: "split", type: "Identifier" }, - { value: "(", type: "OpenParen" }, - { value: "none", type: "NullLiteral" }, - { value: ",", type: "Comma" }, - { value: "0", type: "NumericLiteral" }, - { value: ")", type: "CloseParen" }, - { value: "|", type: "Pipe" }, - { value: "join", type: "Identifier" }, - { value: "(", type: "OpenParen" }, - { value: ",", type: "StringLiteral" }, - { value: ")", type: "CloseParen" }, + { value: "if", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, + { value: "{%", type: "OpenStatement" }, + { value: "set", type: "Identifier" }, + { value: "if", type: "Identifier" }, + { value: "=", type: "Equals" }, + { value: "2", type: "NumericLiteral" }, + { value: "%}", type: "CloseStatement" }, + { value: "{%", type: "OpenStatement" }, + { value: "if", type: "Identifier" }, + { value: "if", type: "Identifier" }, + { value: "==", type: "ComparisonBinaryOperator" }, + { value: "2", type: "NumericLiteral" }, + { value: "%}", type: "CloseStatement" }, { value: "{{", type: "OpenExpression" }, - { value: " 1 2 3 4 5 ", type: "StringLiteral" }, - { value: ".", type: "Dot" }, - { value: "split", type: "Identifier" }, - { value: "(", type: "OpenParen" }, - { value: "none", type: "NullLiteral" }, - { value: ",", type: "Comma" }, - { value: "3", type: "NumericLiteral" }, - { value: ")", type: "CloseParen" }, - { value: "|", type: "Pipe" }, - { value: "join", type: "Identifier" }, - { value: "(", type: "OpenParen" }, - { value: ",", type: "StringLiteral" }, - { value: ")", type: "CloseParen" }, + { value: "if", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, + { value: "{%", type: "OpenStatement" }, + { value: "endif", type: "Identifier" }, + { value: "%}", type: "CloseStatement" }, { value: "|", type: "Text" }, - { value: "{{", type: "OpenExpression" }, - { value: " 1 2 3 4 5 ", type: "StringLiteral" }, - { value: ".", type: "Dot" }, - { value: "split", type: "Identifier" }, + ], + + // Unpacking + UNPACKING: [ + { value: "{%", type: "OpenStatement" }, + { value: "macro", type: "Identifier" }, + { value: "mul", type: "Identifier" }, { value: "(", type: "OpenParen" }, - { value: " ", type: "StringLiteral" }, + { value: "a", type: "Identifier" }, { value: ",", type: "Comma" }, - { value: "0", type: "NumericLiteral" }, - { value: ")", type: "CloseParen" }, - { value: "|", type: "Pipe" }, - { value: "join", type: "Identifier" }, - { value: "(", type: "OpenParen" }, - { value: ",", type: "StringLiteral" }, + { value: "b", type: "Identifier" }, + { value: ",", type: "Comma" }, + { value: "c", type: "Identifier" }, { value: ")", type: "CloseParen" }, + { value: "%}", type: "CloseStatement" }, + { value: "{{", type: "OpenExpression" }, + { value: "a", type: "Identifier" }, + { value: "*", type: "MultiplicativeBinaryOperator" }, + { value: "b", type: "Identifier" }, + { value: "*", type: "MultiplicativeBinaryOperator" }, + { value: "c", type: "Identifier" }, { value: "}}", type: "CloseExpression" }, + { value: "{%", type: "OpenStatement" }, + { value: "endmacro", type: "Identifier" }, + { value: "%}", type: "CloseStatement" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: " 1 2 3 4 5 ", type: "StringLiteral" }, - { value: ".", type: "Dot" }, - { value: "split", type: "Identifier" }, + { value: "mul", type: "Identifier" }, { value: "(", type: "OpenParen" }, - { value: " ", type: "StringLiteral" }, + { value: "1", type: "NumericLiteral" }, + { value: ",", type: "Comma" }, + { value: "2", type: "NumericLiteral" }, { value: ",", type: "Comma" }, { value: "3", type: "NumericLiteral" }, { value: ")", type: "CloseParen" }, - { value: "|", type: "Pipe" }, - { value: "join", type: "Identifier" }, - { value: "(", type: "OpenParen" }, - { value: ",", type: "StringLiteral" }, - { value: ")", type: "CloseParen" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, { value: "{{", type: "OpenExpression" }, - { value: " 1 2 3 4 5 ", type: "StringLiteral" }, - { value: ".", type: "Dot" }, - { value: "split", type: "Identifier" }, + { value: "mul", type: "Identifier" }, { value: "(", type: "OpenParen" }, - { value: " ", type: "StringLiteral" }, + { value: "*", type: "MultiplicativeBinaryOperator" }, + { value: "[", type: "OpenSquareBracket" }, + { value: "1", type: "NumericLiteral" }, { value: ",", type: "Comma" }, - { value: "10", type: "NumericLiteral" }, - { value: ")", type: "CloseParen" }, - { value: "|", type: "Pipe" }, - { value: "join", type: "Identifier" }, - { value: "(", type: "OpenParen" }, - { value: ",", type: "StringLiteral" }, + { value: "2", type: "NumericLiteral" }, + { value: ",", type: "Comma" }, + { value: "3", type: "NumericLiteral" }, + { value: "]", type: "CloseSquareBracket" }, { value: ")", type: "CloseParen" }, { value: "}}", type: "CloseExpression" }, { value: "|", type: "Text" }, @@ -3143,13 +3848,24 @@ const TEST_CONTEXT = { FOR_LOOP_UNPACKING: {}, FOR_LOOP_DEFAULT: {}, FOR_LOOP_SELECT: {}, + FOR_LOOP_SELECT_2: { + arr: [{ value: "a" }, { value: "b" }, { value: "c" }, { value: "a" }], + }, FOR_LOOP_BREAK: {}, FOR_LOOP_CONTINUE: {}, + FOR_LOOP_OBJECTS: { + obj: { + a: 1, + b: 2, + c: 3, + }, + }, // Set variables VARIABLES: {}, VARIABLES_2: {}, VARIABLES_BLOCK: {}, + VARIABLES_UNPACKING: {}, // Numbers NUMBERS: { @@ -3159,6 +3875,7 @@ const TEST_CONTEXT = { // Binary expressions BINOP_EXPR: {}, + BINOP_EXPR_1: {}, // Strings STRINGS: { @@ -3166,6 +3883,8 @@ const TEST_CONTEXT = { }, STRINGS_1: {}, STRINGS_2: {}, + STRINGS_3: {}, + STRINGS_4: {}, // Function calls FUNCTIONS: { @@ -3194,6 +3913,14 @@ const TEST_CONTEXT = { // String methods STRING_METHODS: {}, STRING_METHODS_2: {}, + RSTRIP: {}, + LSTRIP: {}, + SPLIT: {}, + SPLIT_2: {}, + SPLIT_3: {}, + SPLIT_4: {}, + SPLIT_5: {}, + REPLACE: {}, // String indexing and slicing STRING_SLICING: { @@ -3215,6 +3942,7 @@ const TEST_CONTEXT = { MEMBERSHIP_NEGATION_2: { arr: [0, true, "a"], }, + MEMBERSHIP_UNDEFINED: {}, // Escaped characters ESCAPED_CHARS: {}, @@ -3288,6 +4016,13 @@ const TEST_CONTEXT = { FILTER_OPERATOR_13: { tools: [{ name: "some_tool", arguments: { some_name: "string" } }], }, + FILTER_OPERATOR_14: {}, + FILTER_OPERATOR_15: {}, + FILTER_OPERATOR_16: {}, + FILTER_OPERATOR_17: {}, + + // Filter statements + FILTER_STATEMENTS: {}, // Logical operators between non-Booleans BOOLEAN_NUMERICAL: {}, @@ -3350,7 +4085,12 @@ const TEST_CONTEXT = { // Ternary operator TERNARY_OPERATOR: {}, + TERNARY_OPERATOR_1: { + x: [{}, {}, {}], + }, TERNARY_SET: {}, + TERNARY_CONSECUTIVE: {}, + TERNARY_SHORTCUT: {}, // Array literals ARRAY_LITERALS: { var: true }, @@ -3362,6 +4102,7 @@ const TEST_CONTEXT = { OBJECT_LITERALS: { key: "key2", }, + OBJECT_LITERALS_1: {}, // Array operators ARRAY_OPERATORS: {}, @@ -3370,17 +4111,20 @@ const TEST_CONTEXT = { MACROS: {}, MACROS_1: {}, MACROS_2: {}, + MACROS_3: {}, + MACROS_4: { + users: [ + { firstname: "John", lastname: "Doe" }, + { firstname: "Jane", lastname: "Smith" }, + ], + }, - // Strip - RSTRIP: {}, - LSTRIP: {}, + // Context-specific keywords + CONTEXT_KEYWORDS: {}, + CONTEXT_KEYWORDS_1: {}, - // Split - SPLIT: {}, - SPLIT_2: {}, - SPLIT_3: {}, - SPLIT_4: {}, - SPLIT_5: {}, + // Unpacking + UNPACKING: {}, }; const EXPECTED_OUTPUTS = { @@ -3412,24 +4156,30 @@ const EXPECTED_OUTPUTS = { FOR_LOOP_UNPACKING: "||1 2||3 4||", FOR_LOOP_DEFAULT: "B", FOR_LOOP_SELECT: "34", + FOR_LOOP_SELECT_2: "aa", FOR_LOOP_BREAK: "12", FOR_LOOP_CONTINUE: "124", + FOR_LOOP_OBJECTS: "a:1;b:2;c:3;", // Set variables VARIABLES: "Hello World", VARIABLES_2: "lo", VARIABLES_BLOCK: "Hello!\nMultiline/block set!\n", + VARIABLES_UNPACKING: "|12|12|", // Numbers NUMBERS: "|5|-5|2|-8|", // Binary expressions BINOP_EXPR: "1truefalsefalsetruetruetrue5", + BINOP_EXPR_1: "1+2=3 is true", // Strings STRINGS: "Bye[INST] ", STRINGS_1: `|test|abc|"'|'|"|`, STRINGS_2: `|0|1|0|1|`, + STRINGS_3: `|{{ "hi" }}|{% if true %}{% endif %}|`, + STRINGS_4: `abc`, // Function calls FUNCTIONS: "014", @@ -3441,6 +4191,14 @@ const EXPECTED_OUTPUTS = { OBJ_METHODS: "AB A_B", STRING_METHODS: "AB ABCD abcd ", STRING_METHODS_2: "Test Test", + RSTRIP: ` test it`, + LSTRIP: `test it `, + SPLIT: `|test|it|`, + SPLIT_2: `||||test|it|||`, + SPLIT_3: `||||test|it |`, + SPLIT_4: `|["1", "2", "3"]|["", "a", "", "acca", "", "", ""]|["", "a", "baccabbb"]|`, + SPLIT_5: `|1 2 3 4 5 |1,2,3,4 5 | 1 2 3 4 5 |,1,2,3 4 5 |,1,2,3,4,5,|`, + REPLACE: `|TEST TEST|TEST test|_t_est test|xbcabc|`, // String indexing and slicing STRING_SLICING: "|0|0123456789|012|123|12345678|13579|543210|", @@ -3452,6 +4210,7 @@ const EXPECTED_OUTPUTS = { MEMBERSHIP: "|true|false|true|false|true|false|", MEMBERSHIP_NEGATION_1: "|false|true|false|true|false|true|", MEMBERSHIP_NEGATION_2: "|false|true|false|true|false|true|", + MEMBERSHIP_UNDEFINED: "|false|false|false|false|false|false|", // Escaped characters ESCAPED_CHARS: `\n\t'"\\|\n|\t|'|"|\\|`, @@ -3473,6 +4232,13 @@ const EXPECTED_OUTPUTS = { FILTER_OPERATOR_11: `3`, FILTER_OPERATOR_12: `2`, FILTER_OPERATOR_13: `[{"name": "some_tool", "arguments": {"some_name": "string"}}]`, + FILTER_OPERATOR_14: `|3|0|-1|1|0|1|1.5|0.0|hello|`, + FILTER_OPERATOR_15: `|bbcbbcbbc|bbcabcabc|bbcabcabc|`, + FILTER_OPERATOR_16: `|hello|false|hello|hello|`, + FILTER_OPERATOR_17: `3`, + + // Filter statements + FILTER_STATEMENTS: `TEXT`, // Logical operators between non-Booleans BOOLEAN_NUMERICAL: `|2|0|0|0|1|1|1|0|false|true|`, @@ -3516,7 +4282,10 @@ const EXPECTED_OUTPUTS = { // Ternary operator TERNARY_OPERATOR: `|a|b|a|b|`, + TERNARY_OPERATOR_1: `3`, TERNARY_SET: `1`, + TERNARY_CONSECUTIVE: `3`, + TERNARY_SHORTCUT: `bar`, // Array literals ARRAY_LITERALS: `5`, @@ -3526,6 +4295,7 @@ const EXPECTED_OUTPUTS = { // Object literals OBJECT_LITERALS: `value`, + OBJECT_LITERALS_1: `value`, // Array operators ARRAY_OPERATORS: `6`, @@ -3534,17 +4304,15 @@ const EXPECTED_OUTPUTS = { MACROS: `|Hello Bob|Hello Alice|`, MACROS_1: `|Hello A.|Hello B!|Hello C?|`, MACROS_2: `|1,2,3|1,0,3|1,0,-1|1,0,-1|1,2,0|`, + MACROS_3: `hello name!`, + MACROS_4: " - John Doe\n - Jane Smith\n", - // RSTRIP/LSTRIP - RSTRIP: ` test it`, - LSTRIP: `test it `, + // Context-specific keywords + CONTEXT_KEYWORDS: `b`, + CONTEXT_KEYWORDS_1: `||2|`, - // Split - SPLIT: `|test|it|`, - SPLIT_2: `||||test|it|||`, - SPLIT_3: `||||test|it |`, - SPLIT_4: `|["1", "2", "3"]|["", "a", "", "acca", "", "", ""]|["", "a", "baccabbb"]|`, - SPLIT_5: `|1 2 3 4 5 |1,2,3,4 5 | 1 2 3 4 5 |,1,2,3 4 5 |,1,2,3,4,5,|`, + // Unpacking + UNPACKING: `|6|6|`, }; describe("Templates", () => { @@ -3579,9 +4347,7 @@ describe("Templates", () => { } it(name, () => { const env = new Environment(); - // Declare global variables - env.set("false", false); - env.set("true", true); + setupGlobals(env); // Add user-defined variables for (const [key, value] of Object.entries(TEST_CONTEXT[name])) { diff --git a/packages/jinja/test/utils.test.js b/packages/jinja/test/utils.test.js index be271c1df..3abced623 100644 --- a/packages/jinja/test/utils.test.js +++ b/packages/jinja/test/utils.test.js @@ -1,6 +1,6 @@ import { describe, it, expect } from "vitest"; -import { slice } from "../src/utils"; +import { slice, strftime, replace } from "../src/utils"; describe("Test utility functions", () => { describe("Slice function", () => { @@ -70,4 +70,63 @@ describe("Test utility functions", () => { expect(slice(array, -3, undefined, -1)).toEqual([7, 6, 5, 4, 3, 2, 1, 0]); }); }); + + describe("strftime function", () => { + const date = new Date(2025, 5, 4, 12, 34, 56); + const leapDate = new Date(2000, 1, 29, 0, 0, 0, 0); + + it('should format "%d %b %Y"', () => { + expect(strftime(date, "%d %b %Y")).toEqual("04 Jun 2025"); + expect(strftime(leapDate, "%d %b %Y")).toEqual("29 Feb 2000"); + }); + + it('should format "%Y-%m-%d"', () => { + expect(strftime(date, "%Y-%m-%d")).toEqual("2025-06-04"); + expect(strftime(leapDate, "%Y-%m-%d")).toEqual("2000-02-29"); + }); + + it("should format '%B %d, %Y'", () => { + expect(strftime(date, "%B %d, %Y")).toEqual("June 04, 2025"); + expect(strftime(leapDate, "%B %d, %Y")).toEqual("February 29, 2000"); + }); + }); + + describe("replace function", () => { + it("replaces all occurrences when count is not specified", () => { + expect(replace("one one one", "one", "two")).toEqual("two two two"); + expect(replace("aaaa", "a", "b")).toEqual("bbbb"); + }); + + it("replaces up to count occurrences when count is provided", () => { + expect(replace("spam spam spam", "spam", "eggs", 2)).toEqual("eggs eggs spam"); + expect(replace("abcabcabc", "abc", "x", 0)).toEqual("abcabcabc"); + }); + + it("removes occurrences when replacement is empty", () => { + expect(replace("foo bar foo", "foo", "")).toEqual(" bar "); + expect(replace("banana", "a", "")).toEqual("bnn"); + }); + + it("returns original when old substring not found", () => { + expect(replace("hello world", "xyz", "123")).toEqual("hello world"); + }); + + it("handles overlapping patterns like Python does", () => { + // Python.replace treats non-overlapping matches: "aaaa".replace("aa","b") => "bb" + expect(replace("aaaa", "aa", "b")).toEqual("bb"); + }); + + it("handles when old substring is empty", () => { + expect(replace("abc", "", "x")).toEqual("xaxbxcx"); + expect(replace("abc", "", "x", 0)).toEqual("abc"); + expect(replace("abc", "", "x", 2)).toEqual("xaxbc"); + expect(replace("abc", "", "x", 100)).toEqual("xaxbxcx"); + }); + + it("handles multi-byte characters", () => { + const str = "πŸ€—testπŸ€—"; // \ud83e\udd17 + expect(replace(str, "test", "πŸ€—")).toEqual("πŸ€—πŸ€—πŸ€—"); + expect(replace(str, "\ud83e", "X")).toEqual(str); // No replacement + }); + }); });