feat: enable filters for json columns (#1087)

Closes HDX-1965

Author: knudtty

.changeset/nervous-guests-help.md

@@ -0,0 +1,6 @@
+---
+"@hyperdx/common-utils": patch
+"@hyperdx/app": patch
+---
+
+feat: enable filters for json columns

packages/app/src/components/DBRowJsonViewer.tsx

@@ -154,20 +154,10 @@ export function DBRowJsonViewer({
   const getLineActions = useCallback<GetLineActions>(
     ({ keyPath, value }) => {
       const actions: LineAction[] = [];
-      let fieldPath = mergePath(keyPath);
+      const fieldPath = mergePath(keyPath, jsonColumns);
       const isJsonColumn =
         keyPath.length > 0 && jsonColumns?.includes(keyPath[0]);
 
-      if (isJsonColumn) {
-        fieldPath = keyPath.join('.');
-        if (keyPath.length > 1) {
-          fieldPath = `${keyPath[0]}.${keyPath
-            .slice(1)
-            .map(k => `\`${k}\``)
-            .join('.')}`;
-        }
-      }
-
       // Add to Filters action (strings only)
       // FIXME: TOTAL HACK To disallow adding timestamp to filters
       if (

packages/app/src/components/DBSearchPageFilters.tsx

@@ -17,10 +17,15 @@ import {
   Tooltip,
   UnstyledButton,
 } from '@mantine/core';
+import { notifications } from '@mantine/notifications';
 import { IconSearch } from '@tabler/icons-react';
 
 import { useExplainQuery } from '@/hooks/useExplainQuery';
-import { useAllFields, useGetKeyValues } from '@/hooks/useMetadata';
+import {
+  useAllFields,
+  useGetKeyValues,
+  useJsonColumns,
+} from '@/hooks/useMetadata';
 import useResizable from '@/hooks/useResizable';
 import { getMetadata } from '@/metadata';
 import { FilterStateHook, usePinnedFilters } from '@/searchFilters';
@@ -406,11 +411,26 @@ const DBSearchPageFiltersComponent = ({
   const { data: countData } = useExplainQuery(chartConfig);
   const numRows: number = countData?.[0]?.rows ?? 0;
 
-  const { data, isLoading } = useAllFields({
+  const { data: jsonColumns } = useJsonColumns({
+    databaseName: chartConfig.from.databaseName,
+    tableName: chartConfig.from.tableName,
+    connectionId: chartConfig.connection,
+  });
+  const { data, isLoading, error } = useAllFields({
     databaseName: chartConfig.from.databaseName,
     tableName: chartConfig.from.tableName,
     connectionId: chartConfig.connection,
   });
+  useEffect(() => {
+    if (error) {
+      notifications.show({
+        color: 'red',
+        title: error?.name,
+        message: error?.message,
+        autoClose: 5000,
+      });
+    }
+  }, [error]);
 
   const [showMoreFields, setShowMoreFields] = useState(false);
 
@@ -431,7 +451,7 @@ const DBSearchPageFiltersComponent = ({
         // todo: add number type with sliders :D
       )
       .map(({ path, type }) => {
-        return { type, path: mergePath(path) };
+        return { type, path: mergePath(path, jsonColumns ?? []) };
       })
       .filter(
         field =>
@@ -445,9 +465,8 @@ const DBSearchPageFiltersComponent = ({
         path =>
           !['body', 'timestamp', '_hdx_body'].includes(path.toLowerCase()),
       );
-
     return strings;
-  }, [data, filterState, showMoreFields]);
+  }, [data, jsonColumns, filterState, showMoreFields]);
 
   // Special case for live tail
   const [dateRange, setDateRange] = useState<[Date, Date]>(

packages/app/src/utils.ts

@@ -646,12 +646,21 @@ export const legacyMetricNameToNameAndDataType = (metricName?: string) => {
 };
 
 // Date formatting
-export const mergePath = (path: string[]) => {
+export const mergePath = (path: string[], jsonColumns: string[] = []) => {
   const [key, ...rest] = path;
   if (rest.length === 0) {
     return key;
   }
-  return `${key}['${rest.join("']['")}']`;
+  return jsonColumns.includes(key)
+    ? `${key}.${rest
+        .map(v =>
+          v
+            .split('.')
+            .map(v => (v.startsWith('`') && v.endsWith('`') ? v : `\`${v}\``))
+            .join('.'),
+        )
+        .join('.')}`
+    : `${key}['${rest.join("']['")}']`;
 };
 
 export const _useTry = <T>(fn: () => T): [null | Error | unknown, null | T] => {

packages/common-utils/src/metadata.ts

@@ -16,6 +16,7 @@ import type { ChartConfig, ChartConfigWithDateRange, TSource } from '@/types';
 // If filters initially are taking too long to load, decrease this number.
 // Between 1e6 - 5e6 is a good range.
 export const DEFAULT_METADATA_MAX_ROWS_TO_READ = 3e6;
+const DEFAULT_MAX_KEYS = 1000;
 
 export class MetadataCache {
   private cache = new Map<string, any>();
@@ -219,7 +220,7 @@ export class Metadata {
     databaseName,
     tableName,
     column,
-    maxKeys = 1000,
+    maxKeys = DEFAULT_MAX_KEYS,
     connectionId,
     metricName,
   }: {
@@ -304,6 +305,72 @@ export class Metadata {
     });
   }
 
+  async getJSONKeys({
+    column,
+    maxKeys = DEFAULT_MAX_KEYS,
+    databaseName,
+    tableName,
+    connectionId,
+    metricName,
+  }: {
+    column: string;
+    maxKeys?: number;
+  } & TableConnection) {
+    const cacheKey = metricName
+      ? `${databaseName}.${tableName}.${column}.${metricName}.keys`
+      : `${databaseName}.${tableName}.${column}.keys`;
+
+    return this.cache.getOrFetch<{ key: string; chType: string }[]>(
+      cacheKey,
+      async () => {
+        const where = metricName
+          ? chSql`WHERE MetricName=${{ String: metricName }}`
+          : '';
+        const sql = chSql`WITH all_paths AS
+        (
+            SELECT DISTINCT JSONDynamicPathsWithTypes(${{ Identifier: column }}) as paths
+            FROM ${tableExpr({ database: databaseName, table: tableName })} ${where}
+            LIMIT ${{ Int32: maxKeys }}
+            SETTINGS timeout_overflow_mode = 'break', max_execution_time = 2
+        )
+        SELECT groupUniqArrayMap(paths) as pathMap
+        FROM all_paths;`;
+
+        const keys = await this.clickhouseClient
+          .query<'JSON'>({
+            query: sql.sql,
+            query_params: sql.params,
+            connectionId,
+            clickhouse_settings: {
+              max_rows_to_read: String(DEFAULT_METADATA_MAX_ROWS_TO_READ),
+              read_overflow_mode: 'break',
+              ...this.clickhouseSettings,
+            },
+          })
+          .then(res => res.json<{ pathMap: Record<string, string[]> }>())
+          .then(d => {
+            const keys: { key: string; chType: string }[] = [];
+            for (const [key, typeArr] of Object.entries(d.data[0].pathMap)) {
+              if (key || !typeArr || !Array.isArray(typeArr)) {
+                throw new Error(
+                  `Error fetching keys for filters (key: ${key}, typeArr: ${typeArr})`,
+                );
+              }
+              keys.push({
+                key: key
+                  .split('.')
+                  .map(v => `\`${v}\``)
+                  .join('.'),
+                chType: typeArr[0],
+              });
+            }
+            return keys;
+          });
+        return keys;
+      },
+    );
+  }
+
   async getMapValues({
     databaseName,
     tableName,
@@ -391,10 +458,30 @@ export class Metadata {
       });
     }
 
-    const mapColumns = filterColumnMetaByType(columns, [JSDataType.Map]) ?? [];
+    const mapColumns =
+      filterColumnMetaByType(columns, [JSDataType.Map, JSDataType.JSON]) ?? [];
 
     await Promise.all(
       mapColumns.map(async column => {
+        if (convertCHDataTypeToJSType(column.type) === JSDataType.JSON) {
+          const paths = await this.getJSONKeys({
+            databaseName,
+            tableName,
+            column: column.name,
+            connectionId,
+            metricName,
+          });
+
+          for (const path of paths) {
+            fields.push({
+              path: [column.name, path.key],
+              type: path.chType,
+              jsType: convertCHDataTypeToJSType(path.chType),
+            });
+          }
+          return;
+        }
+
         const keys = await this.getMapKeys({
           databaseName,
           tableName,