feat: 🎸 Refactor session recording filters (#2981)

This commit has a few notable changes:

1. A simple attempt at establishing a pattern for using SELECT statements. This will be useful in the future for joins/subquery support.
2. Expanded usage of searching to be able to pass instead of just a string, an object that lets a consumer choose which columns to search on.
3. Decouple our session recording filters from being loaded from the route model hook which is a departure from our convention. This has the benefit of actually letting the page first load and having the filters load async so the page doesn't wait for the filters to finish. This is also much faster to query for each individual filter option from SQLite rather than returning all data to filter in application code.

Author: ZedLi

addons/api/addon/handlers/sqlite-handler.js

@@ -49,7 +49,13 @@ export default class SqliteHandler {
         const schema = store.modelFor(type);
         const serializer = store.serializerFor(type);
 
-        let { page, pageSize, query: queryObj, ...remainingQuery } = query;
+        let {
+          page,
+          pageSize,
+          select,
+          query: queryObj,
+          ...remainingQuery
+        } = query;
         let payload,
           listToken,
           writeToDbPromise,
@@ -119,7 +125,7 @@ export default class SqliteHandler {
         const { sql, parameters } = generateSQLExpressions(type, queryObj, {
           page,
           pageSize,
-          select: ['data'],
+          select: select ?? [{ field: 'data' }],
         });
 
         const rows = await this.sqlite.fetchResource({
@@ -129,7 +135,7 @@ export default class SqliteHandler {
 
         const { sql: countSql, parameters: countParams } =
           generateSQLExpressions(type, queryObj, {
-            select: ['count(*) as total'],
+            select: [{ field: '*', isCount: true, alias: 'total' }],
           });
         const count = await this.sqlite.fetchResource({
           sql: countSql,

addons/api/addon/services/db.js

@@ -0,0 +1,44 @@
+/**
+ * Copyright (c) HashiCorp, Inc.
+ * SPDX-License-Identifier: BUSL-1.1
+ */
+
+import Service, { service } from '@ember/service';
+import { modelMapping } from 'api/services/sqlite';
+import { assert } from '@ember/debug';
+import { generateSQLExpressions } from '../utils/sqlite-query';
+
+/**
+ * Database service for querying Boundary resources from storage.
+ * Provides a high-level interface for executing queries against the local database.
+ */
+export default class DbService extends Service {
+  @service sqlite;
+
+  /**
+   * Query resources from the database by type.
+   *
+   * @param {string} type - The resource type (e.g., 'scope', 'user', 'target').
+   *                        Must be a supported model type defined in modelMapping.
+   * @param {Object} query - Query configuration object
+   * @returns {Promise<Object>} Promise resolving to query results from sqlite service
+   * @throws {Error} Assertion error if resource type is not supported
+   */
+  query(type, query) {
+    const supportedModels = Object.keys(modelMapping);
+    assert('Resource type is not supported.', supportedModels.includes(type));
+
+    let { page, pageSize, select, query: queryObj } = query;
+
+    const { sql, parameters } = generateSQLExpressions(type, queryObj, {
+      page,
+      pageSize,
+      select,
+    });
+
+    return this.sqlite.fetchResource({
+      sql,
+      parameters,
+    });
+  }
+}

addons/api/addon/services/sqlite.js

@@ -100,6 +100,8 @@ export const modelMapping = {
     target_name: 'create_time_values.target.name',
     target_scope_id: 'create_time_values.target.scope.id',
     target_scope_name: 'create_time_values.target.scope.name',
+    target_scope_parent_scope_id:
+      'create_time_values.target.scope.parent_scope_id',
     created_time: 'created_time',
   },
   session: {
@@ -114,21 +116,6 @@ export const modelMapping = {
   },
 };
 
-// A list of tables that we support searching using FTS5 in SQLite.
-export const searchTables = new Set([
-  'target',
-  'alias',
-  'group',
-  'role',
-  'user',
-  'credential-store',
-  'scope',
-  'auth-method',
-  'host-catalog',
-  'session-recording',
-  'session',
-]);
-
 export default class SqliteDbService extends Service {
   // =attributes
 

addons/api/addon/utils/sqlite-query.js

@@ -4,9 +4,9 @@
  */
 
 import { modelMapping } from 'api/services/sqlite';
-import { searchTables } from 'api/services/sqlite';
 import { typeOf } from '@ember/utils';
 import { underscore } from '@ember/string';
+import { assert } from '@ember/debug';
 
 /**
  * Takes a POJO representing a filter query and builds a SQL query.
@@ -35,17 +35,15 @@ export function generateSQLExpressions(
   addFilterConditions({ filters, parameters, conditions });
   addSearchConditions({ search, resource, tableName, parameters, conditions });
 
+  const selectClause = constructSelectClause(select, tableName);
   const orderByClause = constructOrderByClause(resource, sort);
-
   const whereClause = conditions.length ? `WHERE ${and(conditions)}` : '';
 
   const paginationClause = page && pageSize ? `LIMIT ? OFFSET ?` : '';
   if (paginationClause) {
     parameters.push(pageSize, (page - 1) * pageSize);
   }
 
-  const selectClause = `SELECT ${select ? select.join(', ') : '*'} FROM "${tableName}"`;
-
   return {
     // Replace any empty newlines or leading whitespace on each line to be consistent with formatting
     // This is mainly to help us read and test the generated SQL as it has no effect on the actual SQL execution.
@@ -142,32 +140,83 @@ function addSearchConditions({
   parameters,
   conditions,
 }) {
-  if (!search) {
+  if (!search || !modelMapping[resource]) {
     return;
   }
 
-  if (searchTables.has(resource)) {
-    // Use the special prefix indicator "*" for full-text search
-    parameters.push(`"${search}"*`);
-    // Use a subquery to match against the FTS table with rowids as SQLite is
-    // much more efficient with FTS queries when using rowids or MATCH (or both).
-    // We could have also used a join here but a subquery is simpler.
-    conditions.push(
-      `rowid IN (SELECT rowid FROM ${tableName}_fts WHERE ${tableName}_fts MATCH ?)`,
+  // Use the special prefix indicator "*" for full-text search
+  if (typeOf(search) === 'object') {
+    if (!search?.text) {
+      return;
+    }
+
+    parameters.push(
+      or(search.fields.map((field) => `${field}:"${search.text}"*`)),
     );
-    return;
+  } else {
+    parameters.push(`"${search}"*`);
   }
 
-  const fields = Object.keys(modelMapping[resource]);
-  const searchConditions = parenthetical(
-    or(
-      fields.map((field) => {
-        parameters.push(`%${search}%`);
-        return `${field}${OPERATORS['contains']}`;
-      }),
-    ),
+  // Use a subquery to match against the FTS table with rowids as SQLite is
+  // much more efficient with FTS queries when using rowids or MATCH (or both).
+  // We could have also used a join here but a subquery is simpler.
+  conditions.push(
+    `rowid IN (SELECT rowid FROM ${tableName}_fts WHERE ${tableName}_fts MATCH ?)`,
   );
-  conditions.push(searchConditions);
+}
+
+function constructSelectClause(select = [{ field: '*' }], tableName) {
+  const distinctColumns = select.filter(({ isDistinct }) => isDistinct);
+  const nonDistinctColumns = select.filter(({ isDistinct }) => !isDistinct);
+
+  const buildColumnExpression = ({ field, isCount, alias, isDistinct }) => {
+    let column = field;
+
+    // If there's just distinct with nothing else, this will be handled separately
+    // and we will just return the column back as is
+    if (isCount && isDistinct) {
+      column = `count(DISTINCT ${column})`;
+    } else if (isCount) {
+      column = `count(${column})`;
+    }
+
+    if (alias) {
+      column = `${column} as ${alias}`;
+    }
+
+    return column;
+  };
+
+  let selectColumns;
+
+  if (distinctColumns.length > 0) {
+    // Check if any columns also have COUNT (or other aggregate function in the future)
+    const hasAggregateDistinct = distinctColumns.some((col) => col.isCount);
+
+    if (hasAggregateDistinct) {
+      selectColumns = distinctColumns.map(buildColumnExpression).join(', ');
+
+      // Add back in the non distinct columns
+      if (nonDistinctColumns.length > 0) {
+        const regularPart = nonDistinctColumns
+          .map(buildColumnExpression)
+          .join(', ');
+        selectColumns = `${selectColumns}, ${regularPart}`;
+      }
+    } else {
+      assert(
+        'Can not combine non-distincts with multi column distincts',
+        nonDistinctColumns.length === 0,
+      );
+
+      // Only do multi column DISTINCT with no aggregate functions
+      selectColumns = `DISTINCT ${distinctColumns.map(buildColumnExpression).join(', ')}`;
+    }
+  } else {
+    selectColumns = select.map(buildColumnExpression).join(', ');
+  }
+
+  return `SELECT ${selectColumns} FROM "${tableName}"`;
 }
 
 function constructOrderByClause(resource, sort) {

addons/api/addon/workers/utils/schema.js

@@ -352,6 +352,7 @@ CREATE TABLE IF NOT EXISTS session_recording (
     target_name TEXT,
     target_scope_id TEXT,
     target_scope_name TEXT,
+    target_scope_parent_scope_id TEXT,
     created_time TEXT NOT NULL,
     data TEXT NOT NULL
 );
@@ -371,21 +372,22 @@ CREATE VIRTUAL TABLE IF NOT EXISTS session_recording_fts USING fts5(
     target_name,
     target_scope_id,
     target_scope_name,
+    target_scope_parent_scope_id,
     created_time,
     content='',
 );
 
 CREATE TRIGGER IF NOT EXISTS session_recording_ai AFTER INSERT ON session_recording BEGIN
     INSERT INTO session_recording_fts(
-        rowid, id, type, state, start_time, end_time, duration, scope_id, user_id, user_name, target_id, target_name, target_scope_id, target_scope_name, created_time
+        rowid, id, type, state, start_time, end_time, duration, scope_id, user_id, user_name, target_id, target_name, target_scope_id, target_scope_name, target_scope_parent_scope_id, created_time
     ) VALUES (
-        new.rowid, new.id, new.type, new.state, new.start_time, new.end_time, new.duration, new.scope_id, new.user_id, new.user_name, new.target_id, new.target_name, new.target_scope_id, new.target_scope_name, new.created_time
+        new.rowid, new.id, new.type, new.state, new.start_time, new.end_time, new.duration, new.scope_id, new.user_id, new.user_name, new.target_id, new.target_name, new.target_scope_id, new.target_scope_name, new.target_scope_parent_scope_id, new.created_time
     );
 END;
 
 CREATE TRIGGER IF NOT EXISTS session_recording_ad AFTER DELETE ON session_recording BEGIN
-    INSERT INTO session_recording_fts(session_recording_fts, rowid, id, type, state, start_time, end_time, duration, scope_id, user_id, user_name, target_id, target_name, target_scope_id, target_scope_name, created_time)
-    VALUES('delete', old.rowid, old.id, old.type, old.state, old.start_time, old.end_time, old.duration, old.scope_id, old.user_id, old.user_name, old.target_id, old.target_name, old.target_scope_id, old.target_scope_name, old.created_time);
+    INSERT INTO session_recording_fts(session_recording_fts, rowid, id, type, state, start_time, end_time, duration, scope_id, user_id, user_name, target_id, target_name, target_scope_id, target_scope_name, target_scope_parent_scope_id, created_time)
+    VALUES('delete', old.rowid, old.id, old.type, old.state, old.start_time, old.end_time, old.duration, old.scope_id, old.user_id, old.user_name, old.target_id, old.target_name, old.target_scope_id, old.target_scope_name, old.target_scope_parent_scope_id, old.created_time);
 END;`;
 
 const createSessionTables = `

addons/api/app/services/db.js

@@ -0,0 +1,6 @@
+/**
+ * Copyright (c) HashiCorp, Inc.
+ * SPDX-License-Identifier: BUSL-1.1
+ */
+
+export { default } from 'api/services/db';

addons/api/tests/unit/services/db-test.js

@@ -0,0 +1,19 @@
+/**
+ * Copyright (c) HashiCorp, Inc.
+ * SPDX-License-Identifier: BUSL-1.1
+ */
+
+import { module, test } from 'qunit';
+import { setupTest } from 'dummy/tests/helpers';
+
+module('Unit | Service | db', function (hooks) {
+  setupTest(hooks);
+
+  test('query throws assertion error for unsupported resource type', function (assert) {
+    let service = this.owner.lookup('service:db');
+
+    assert.throws(() => {
+      service.query('unsupported-type', { query: {} });
+    }, /Resource type is not supported/);
+  });
+});

addons/api/tests/unit/services/sqlite-test.js

@@ -6,11 +6,10 @@
 import { module, test } from 'qunit';
 import { setupTest } from 'dummy/tests/helpers';
 import { setupSqlite } from 'api/test-support/helpers/sqlite';
-import { modelMapping, searchTables } from 'api/services/sqlite';
+import { modelMapping } from 'api/services/sqlite';
 import { underscore } from '@ember/string';
 
 const supportedModels = Object.keys(modelMapping);
-const supportedFtsTables = [...searchTables];
 
 module('Unit | Service | sqlite', function (hooks) {
   setupTest(hooks);
@@ -38,7 +37,7 @@ module('Unit | Service | sqlite', function (hooks) {
 
   test.each(
     'Mapping matches fts table columns',
-    supportedFtsTables,
+    supportedModels,
     async function (assert, resource) {
       const service = this.owner.lookup('service:sqlite');