feat: Distribute exact-match lucene variable references (#2987)

## Summary

This PR enhances support for variables in lucene by rewriting `Field:"$var"` to `(Field:"A" OR Field"B")` (when A and B are selected for $var. This preserves exact-match semantics. The basic expansion `Field:("A" OR "B")` is a substring condition `Field ILIKE "%A%" OR Field ILIKE "%B%".

The quoted form was chosen for exact-match semantics because `Field:"A"` is exact match. `Field:$var` continues using the basic expansion, matching `Field:A` substring semantics.

## How

The rewrite is done through the following process:

1. Tokenize the lucene input string using the existing macro/variable tokenizer. This turns the text into a stream of Variable/Macro/Text tokens.
2. Replace all Variable type tokens with unique placeholders `__hdx_sentinel_N` in a sentinelString, and track the locations of the placeholders within the sentinelString. We do this to ensure that the sentinelString can be parsed as valid lucene (${var} reference are not valid single lucene terms).
3. Parse the sentinelString as lucene
4. Rewrite the terms representing Field:"Value" terms (from the AST), where Value is a tracked placeholder string in the sentinelString.

The transform is a lucene --> lucene transformation, so the resulting lucene still goes through the existing lucene --> SQL transpiler and inherits all of its optimizations.

### Screenshots or video

Some examples:

<img width="1546" height="906" alt="Screenshot 2026-08-24 at 3 02 01 PM" src="https://github.com/user-attachments/assets/42ff6723-8f20-4d8b-a140-8a3805390f45" />
<img width="1554" height="887" alt="Screenshot 2026-08-24 at 3 03 07 PM" src="https://github.com/user-attachments/assets/cdbcda38-4158-4985-8e8d-a64e06152a85" />
<img width="1551" height="866" alt="Screenshot 2026-08-24 at 3 02 52 PM" src="https://github.com/user-attachments/assets/93ae488e-2bf5-484f-85a2-9ce018e1a686" />
<img width="1548" height="921" alt="Screenshot 2026-08-24 at 3 02 34 PM" src="https://github.com/user-attachments/assets/0144e937-c926-4461-8ce6-df57dc338952" />
<img width="1547" height="923" alt="Screenshot 2026-08-24 at 3 02 15 PM" src="https://github.com/user-attachments/assets/f0d862f2-957e-443f-8d7a-9a24e34660b0" />


### How to test on Vercel preview

- Create a dashboard and add some variables via the Edit Filters and Variables button
- Create a chart and reference lucene variables. Inspect the SQL generated for various conditions

### References



- Linear Issue: Closes HDX-5156
- Related PRs:

Author: pulpdrew

.changeset/lucene-variable-field-distribution.md

@@ -0,0 +1,6 @@
+---
+'@hyperdx/app': patch
+'@hyperdx/common-utils': patch
+---
+
+feat: Distribute exact-match lucene variable references

packages/app/src/components/SQLEditor/__tests__/variableCompletions.test.ts

@@ -95,13 +95,11 @@ describe('buildLuceneVariableSuggestions', () => {
     // No braced or explicit-format forms either — in a Lucene input the bare
     // reference already renders in the lucene format.
     expect(buildLuceneVariableSuggestions([SERVICE])).toEqual([
-      {
-        value: '$service',
-        label: '$service',
-        description:
-          'The selected values of service. Expands to: ("api" OR "web")',
-      },
+      expect.objectContaining({ value: '$service', label: '$service' }),
     ]);
+    expect(buildLuceneVariableSuggestions([SERVICE])[0].description).toContain(
+      'Expands to: ("api" OR "web")',
+    );
   });
 
   it('previews the empty selection as the term that drops out', () => {
@@ -131,6 +129,14 @@ describe('expandLuceneVariablesForEnglishDisplay', () => {
     );
   });
 
+  it('expands a quoted reference to an exact match per value', () => {
+    // Quoting is how an author opts into exact matching, so the English
+    // summary has to show that shape rather than the substring one above.
+    expect(expand('ServiceName:"$service"', [SERVICE])).toBe(
+      '(ServiceName:"api" OR ServiceName:"web")',
+    );
+  });
+
   it('leaves an unselected variable as written', () => {
     // `("")` reads as `'ServiceName' is <blank>` once serialized to English,
     // which is worse than naming the placeholder that has no value yet.
@@ -148,6 +154,15 @@ describe('expandLuceneVariablesForEnglishDisplay', () => {
     ).toBe('ServiceName:("api" OR "web") AND Env:$env');
   });
 
+  it('leaves a half-typed format as written', () => {
+    // `${service:l}` is a keystroke on the way to `${service:lucene}`, and
+    // expanding it throws. This runs on every keystroke, so it has to be
+    // survivable rather than take the input down with it.
+    expect(expand('ServiceName:${service:l}', [SERVICE])).toBe(
+      'ServiceName:${service:l}',
+    );
+  });
+
   it('leaves unknown references and the variable macros alone', () => {
     expect(
       expand('$nope AND $__filter(ServiceName, $service)', [SERVICE]),

packages/app/src/components/SQLEditor/variableCompletions.tsx

@@ -5,7 +5,7 @@ import {
 } from '@hyperdx/common-utils/dist/macros';
 import { ChartVariable } from '@hyperdx/common-utils/dist/types';
 import {
-  substituteVariables,
+  substituteWithContext,
   VARIABLE_FORMATS,
   VariableFormat,
 } from '@hyperdx/common-utils/dist/variables';
@@ -17,17 +17,22 @@ const VARIABLE_FORMAT_DESCRIPTIONS: Record<VariableFormat, string> = {
   sqlstring: "Quoted and comma-separated, escaped for SQL. e.g. 'a', 'b', 'c'",
   csv: 'Comma-separated and unquoted. Not SQL-escaped. e.g. a,b,c',
   regex: 'A regex alternation. Regex escaped. e.g. (a|b|c)',
-  lucene: 'An OR of quoted terms, for Lucene inputs. e.g. ("a" OR "b" OR "c")',
+  lucene:
+    'An OR of quoted terms, for Lucene inputs. e.g. ("a" OR "b" OR "c"). Quote the reference (field:"$var") for exact-match behavior. Leave unquoted (field:$var) for substring matching.',
 };
 
-/** What `snippet` expands to against the variable's current selection. */
-function describeVariableExpansion(
+/** What `snippet` expands to in SQL against the variable's current selection. */
+function describeSqlVariableExpansion(
   snippet: string,
   variable: ChartVariable,
 ): string | undefined {
   let expansion: string;
   try {
-    expansion = substituteVariables(snippet, [variable]);
+    expansion = substituteWithContext(snippet, {
+      variables: [variable],
+      defaultFormat: 'sqlstring',
+      inputLanguage: 'sql',
+    });
   } catch {
     return undefined;
   }
@@ -77,7 +82,7 @@ function referenceCompletions(variable: ChartVariable): SQLCompletion[] {
 
   /** A static description and an expansion preview given the current selection */
   const help = (snippet: string, description: string) => {
-    const expansion = describeVariableExpansion(snippet, variable);
+    const expansion = describeSqlVariableExpansion(snippet, variable);
     return expansion ? completionInfo(description, expansion) : description;
   };
 
@@ -153,9 +158,10 @@ export type LuceneVariableSuggestion = {
 
 /** Expand references the way a Lucene expression is expanded at query time. */
 const substituteLucene = (text: string, variables: ChartVariable[]) =>
-  substituteVariables(text, variables, {
+  substituteWithContext(text, {
+    variables,
     defaultFormat: 'lucene',
-    disableMacros: true,
+    inputLanguage: 'lucene',
   });
 
 /**
@@ -174,7 +180,7 @@ export function buildLuceneVariableSuggestions(
     return {
       value: reference,
       label: reference,
-      description: `The selected values of ${variable.name}. Expands to: ${expansion}`,
+      description: `The selected values of ${variable.name}. Expands to: ${expansion} by default, or (Field:"value1" OR Field:"value2") when quoted like Field:"$${variable.name}".`,
     };
   });
 }
@@ -188,6 +194,10 @@ export function buildLuceneVariableSuggestions(
  * `("")`, which the English serializer reads as `'field' is <blank>` even
  * though that form filters nothing; leaving the reference as written is the
  * honest rendering of "no value chosen yet".
+ *
+ * Expansion can throw on a reference that is well-formed but not yet valid —
+ * `${name:l}` is a keystroke on the way to `${name:lucene}` — and this runs on
+ * every keystroke, so a failure falls back to the text as written.
  */
 export function expandLuceneVariablesForEnglishDisplay(
   text: string,
@@ -196,7 +206,12 @@ export function expandLuceneVariablesForEnglishDisplay(
   const selected = (variables ?? []).filter(
     variable => variable.values.length > 0,
   );
-  return selected.length > 0 ? substituteLucene(text, selected) : text;
+  if (selected.length === 0) return text;
+  try {
+    return substituteLucene(text, selected);
+  } catch {
+    return text;
+  }
 }
 
 /** Context providing in-scope dashboard variables for descendant inputs. */

packages/common-utils/src/__tests__/queryParser.test.ts

@@ -3,6 +3,8 @@ import { ClickhouseClient } from '@/clickhouse/node';
 import { getMetadata } from '@/core/metadata';
 import {
   CustomSchemaSQLSerializerV2,
+  decodeSpecialTokensToSource,
+  encodeSpecialTokens,
   genEnglishExplanation,
   parseKvItemsCastExpression,
   parseKvItemsExpression,
@@ -20,6 +22,24 @@ afterAll(() => {
   jest.restoreAllMocks();
 });
 
+describe('special token encoding', () => {
+  it('decodeSpecialTokensToSource is a lossless inverse of encodeSpecialTokens', () => {
+    const queries = [
+      'Url:http://example.com',
+      'Url:https://example.com/path',
+      'Host:localhost:3000',
+      'Body:path\\\\to\\\\file',
+      'foo\\:bar:baz',
+      'Url:http://localhost:8080 AND Body:a\\\\b AND foo\\:bar:x',
+    ];
+    for (const query of queries) {
+      expect(decodeSpecialTokensToSource(encodeSpecialTokens(query))).toBe(
+        query,
+      );
+    }
+  });
+});
+
 describe('CustomSchemaSQLSerializerV2 - json', () => {
   const metadata = getMetadata(
     new ClickhouseClient({ host: 'http://localhost:8123' }),
@@ -177,6 +197,43 @@ describe('CustomSchemaSQLSerializerV2 - json', () => {
       sql: "(((ServiceName ILIKE '%foo bar baz%')))",
       english: '(ServiceName contains "foo bar baz")',
     },
+    // The shapes the `lucene` variable format expands a field-scoped reference
+    // into. Distributing the field is what makes each value an exact match:
+    // the grouped `ServiceName:("a" OR "b")` above is a substring match.
+    {
+      lucene: '(ServiceName:"a" OR ServiceName:"b")',
+      sql: "(((ServiceName = 'a') OR (ServiceName = 'b')))",
+      english: "('ServiceName' is a OR 'ServiceName' is b)",
+    },
+    {
+      lucene: '(ServiceName:"a")',
+      sql: "(((ServiceName = 'a')))",
+      english: "('ServiceName' is a)",
+    },
+    {
+      // A Map key distributes to exact equality per value AND keeps the
+      // per-term index hint. Contrast the grouped form above, which is a
+      // substring match.
+      lucene:
+        '(LogAttributes.error.message:"a" OR LogAttributes.error.message:"b")',
+      sql: "(((`LogAttributes`['error.message'] = 'a' AND indexHint(mapContains(`LogAttributes`, 'error.message'))) OR (`LogAttributes`['error.message'] = 'b' AND indexHint(mapContains(`LogAttributes`, 'error.message')))))",
+      english:
+        "('LogAttributes.error.message' is a OR 'LogAttributes.error.message' is b)",
+    },
+    {
+      // Same for a JSON path: exact equality per value, where the grouped form
+      // above compiles to ILIKE.
+      lucene:
+        '(ResourceAttributesJSON.error.message:"a" OR ResourceAttributesJSON.error.message:"b")',
+      sql: "(((toString(`ResourceAttributesJSON`.`error`.`message`) = 'a') OR (toString(`ResourceAttributesJSON`.`error`.`message`) = 'b')))",
+      english:
+        "('ResourceAttributesJSON.error.message' is a OR 'ResourceAttributesJSON.error.message' is b)",
+    },
+    {
+      lucene: 'NOT (ServiceName:"a" OR ServiceName:"b")',
+      sql: "(NOT ((ServiceName = 'a') OR (ServiceName = 'b')))",
+      english: "NOT ('ServiceName' is a OR 'ServiceName' is b)",
+    },
     {
       lucene: 'ServiceName:(abc def)',
       sql: "(((ServiceName ILIKE '%abc%') AND (ServiceName ILIKE '%def%')))",
@@ -503,12 +560,42 @@ describe('CustomSchemaSQLSerializerV2 - json', () => {
     ['ServiceName:("")', '(((1=1)))'],
     ['("")', '(((1=1)))'],
     ['ServiceName:""', "((ServiceName = ''))"],
+    // A Map key drops out the same way a plain column does.
+    ['LogAttributes.error.message:("")', '(((1=1)))'],
   ])('renders the empty lucene term %s as %s', async (lucene, expected) => {
     expect(await new SearchQueryBuilder(lucene, serializer).build()).toBe(
       expected,
     );
   });
 
+  // Two empty-selection shapes that do NOT cleanly drop out. Pinned as-is so a
+  // fix shows up here as a deliberate change rather than a surprise; see the
+  // note on each.
+  it.each([
+    [
+      // A JSON path renders as a match-anything ILIKE rather than `1=1`. Whether
+      // this really matches every row depends on what `toString` yields for an
+      // absent path — if that is NULL, rows missing the attribute are filtered
+      // out while nothing is selected.
+      'ResourceAttributesJSON.error.message:("")',
+      "(((toString(`ResourceAttributesJSON`.`error`.`message`) ILIKE '%%')))",
+    ],
+    [
+      // `NOT (1=1)` matches NOTHING. A negated reference — `-ServiceName:$svc`
+      // or `-ServiceName:"$svc"` — therefore empties the tile until a value is
+      // selected, which is the opposite of the no-op the empty state intends.
+      '-ServiceName:("")',
+      '(NOT ((1=1)))',
+    ],
+  ])(
+    'renders the empty lucene term %s as %s, which is not a no-op',
+    async (lucene, expected) => {
+      expect(await new SearchQueryBuilder(lucene, serializer).build()).toBe(
+        expected,
+      );
+    },
+  );
+
   it('correctly searches multi-column implicit field', async () => {
     const serializer = new CustomSchemaSQLSerializerV2({
       metadata,