feat: Create data cube index tables in a schema. (uwdata#519)

* feat!: Create data cube index tables in a schema.

* rename to data_cube_schema and implement corresponding python part

* docs: Fix docs typo.

---------

Co-authored-by: Dominik Moritz <[email protected]>

Author: jheer

dev/index.html

@@ -160,7 +160,7 @@
   }
 
   function setIndex() {
-    vg.coordinator().dataCubeIndexer.enabled(indexToggle.checked);
+    vg.coordinator().dataCubeIndexer.enabled = indexToggle.checked;
   }
 
   function reload() {

docs/api/core/coordinator.md

@@ -20,7 +20,7 @@ Create a new Mosaic Coordinator to manage all database communication for clients
 * _logger_: The logger to use, defaults to `console`.
 * _cache_: Boolean flag to enable/disable query caching (default `true`).
 * _consolidate_ Boolean flag to enable/disable query consolidation (default `true`).
-* _indexes_: Data cube indexer options object. The _enabled_ flag (default `true`) determines if data cube indexes should be used when possible. The _temp_ flag (default `true`) controls if temporary tables should be created for data cube indexes.
+* _indexes_: Data cube indexer options object. The _enabled_ flag (default `true`) determines if data cube indexes should be used when possible. The _schema_ option (default `'mosaic'`) indicates the database schema in which data cube index tables should be created.
 
 ## databaseConnector
 

packages/core/src/Coordinator.js

@@ -1,7 +1,9 @@
 import { socketConnector } from './connectors/socket.js';
 import { DataCubeIndexer } from './DataCubeIndexer.js';
+import { MosaicClient } from './MosaicClient.js';
 import { QueryManager, Priority } from './QueryManager.js';
 import { queryFieldInfo } from './util/field-info.js';
+import { QueryResult } from './util/query-result.js';
 import { voidLogger } from './util/void-logger.js';
 
 /**
@@ -24,6 +26,10 @@ export function coordinator(instance) {
   return _instance;
 }
 
+/**
+ * @typedef {import('@uwdata/mosaic-sql').Query | string} QueryType
+ */
+
 /**
  * A Mosaic Coordinator manages all database communication for clients and
  * handles selection updates. The Coordinator also performs optimizations
@@ -34,7 +40,8 @@ export function coordinator(instance) {
  * @param {*} [options.manager] The query manager to use.
  * @param {boolean} [options.cache=true] Boolean flag to enable/disable query caching.
  * @param {boolean} [options.consolidate=true] Boolean flag to enable/disable query consolidation.
- * @param {object} [options.indexes] Data cube indexer options.
+ * @param {import('./DataCubeIndexer.js').DataCubeIndexerOptions} [options.indexes]
+ *  Data cube indexer options.
  */
 export class Coordinator {
   constructor(db = socketConnector(), {
@@ -48,10 +55,10 @@ export class Coordinator {
     this.manager = manager;
     this.manager.cache(cache);
     this.manager.consolidate(consolidate);
-    this.dataCubeIndexer = new DataCubeIndexer(this, indexes);
-    this.logger(logger);
     this.databaseConnector(db);
+    this.logger(logger);
     this.clear();
+    this.dataCubeIndexer = new DataCubeIndexer(this, indexes);
   }
 
   /**
@@ -98,7 +105,7 @@ export class Coordinator {
   /**
    * Cancel previosuly submitted query requests. These queries will be
    * canceled if they are queued but have not yet been submitted.
-   * @param {import('./util/query-result.js').QueryResult[]} requests An array
+   * @param {QueryResult[]} requests An array
    *  of query result objects, such as those returned by the `query` method.
    */
   cancel(requests) {
@@ -107,29 +114,30 @@ export class Coordinator {
 
   /**
    * Issue a query for which no result (return value) is needed.
-   * @param {import('@uwdata/mosaic-sql').Query | string} query The query.
+   * @param {QueryType | QueryType[]} query The query or an array of queries.
+   *  Each query should be either a Query builder object or a SQL string.
    * @param {object} [options] An options object.
    * @param {number} [options.priority] The query priority, defaults to
    *  `Priority.Normal`.
-   * @returns {import('./util/query-result.js').QueryResult} A query result
+   * @returns {QueryResult} A query result
    *  promise.
    */
   exec(query, { priority = Priority.Normal } = {}) {
-    query = Array.isArray(query) ? query.join(';\n') : query;
+    query = Array.isArray(query) ? query.filter(x => x).join(';\n') : query;
     return this.manager.request({ type: 'exec', query }, priority);
   }
 
   /**
    * Issue a query to the backing database. The submitted query may be
    * consolidate with other queries and its results may be cached.
-   * @param {import('@uwdata/mosaic-sql').Query | string} query The query.
+   * @param {QueryType} query The query as either a Query builder object
+   *  or a SQL string.
    * @param {object} [options] An options object.
    * @param {'arrow' | 'json'} [options.type] The query result format type.
    * @param {boolean} [options.cache=true] If true, cache the query result.
    * @param {number} [options.priority] The query priority, defaults to
    *  `Priority.Normal`.
-   * @returns {import('./util/query-result.js').QueryResult} A query result
-   *  promise.
+   * @returns {QueryResult} A query result promise.
    */
   query(query, {
     type = 'arrow',
@@ -143,11 +151,11 @@ export class Coordinator {
   /**
    * Issue a query to prefetch data for later use. The query result is cached
    * for efficient future access.
-   * @param {import('@uwdata/mosaic-sql').Query | string} query The query.
+   * @param {QueryType} query The query as either a Query builder object
+   *  or a SQL string.
    * @param {object} [options] An options object.
    * @param {'arrow' | 'json'} [options.type] The query result format type.
-   * @returns {import('./util/query-result.js').QueryResult} A query result
-   *  promise.
+   * @returns {QueryResult} A query result promise.
    */
   prefetch(query, options = {}) {
     return this.query(query, { ...options, cache: true, priority: Priority.Low });
@@ -159,7 +167,7 @@ export class Coordinator {
    * @param {string} name The name of the bundle.
    * @param {[string | {sql: string},  {alias: string}]} queries The queries to save into the bundle.
    * @param {number} priority Request priority.
-   * @returns
+   * @returns {QueryResult} A query result promise.
    */
   createBundle(name, queries, priority = Priority.Low) {
     const options = { name, queries: queries.map(q => typeof q == 'string' ? {sql: q} : q) };
@@ -170,7 +178,7 @@ export class Coordinator {
    * Load a bundle into the cache.
    * @param {string} name The name of the bundle.
    * @param {number} priority Request priority.
-   * @returns
+   * @returns {QueryResult} A query result promise.
    */
   loadBundle(name, priority = Priority.High) {
     const options = { name };
@@ -182,8 +190,8 @@ export class Coordinator {
   /**
    * Update client data by submitting the given query and returning the
    * data (or error) to the client.
-   * @param {import('./MosaicClient.js').MosaicClient} client A Mosaic client.
-   * @param {import('@uwdata/mosaic-sql').Query | string} query The data query.
+   * @param {MosaicClient} client A Mosaic client.
+   * @param {QueryType} query The data query.
    * @param {number} [priority] The query priority.
    * @returns {Promise} A Promise that resolves upon completion of the update.
    */
@@ -201,10 +209,8 @@ export class Coordinator {
    * Issue a query request for a client. If the query is null or undefined,
    * the client is simply updated. Otherwise `updateClient` is called. As a
    * side effect, this method clears the current data cube indexer state.
-   * @param {import('./MosaicClient.js').MosaicClient} client The client
-   *  to update.
-   * @param {import('@uwdata/mosaic-sql').Query | string | null} [query]
-   *  The query to issue.
+   * @param {MosaicClient} client The client to update.
+   * @param {QueryType | null} [query] The query to issue.
    */
   requestQuery(client, query) {
     this.dataCubeIndexer.clear();
@@ -215,8 +221,7 @@ export class Coordinator {
 
   /**
    * Connect a client to the coordinator.
-   * @param {import('./MosaicClient.js').MosaicClient} client The Mosaic
-   *  client to connect.
+   * @param {MosaicClient} client The Mosaic client to connect.
    */
   async connect(client) {
     const { clients } = this;
@@ -247,8 +252,7 @@ export class Coordinator {
 
   /**
    * Disconnect a client from the coordinator.
-   * @param {import('./MosaicClient.js').MosaicClient} client The Mosaic
-   *  client to disconnect.
+   * @param {MosaicClient} client The Mosaic client to disconnect.
    */
   disconnect(client) {
     const { clients, filterGroups } = this;
@@ -267,8 +271,8 @@ export class Coordinator {
  * Connect a selection-client pair to the coordinator to process updates.
  * @param {Coordinator} mc The Mosaic coordinator.
  * @param {import('./Selection.js').Selection} selection A selection.
- * @param {import('./MosaicClient.js').MosaicClient} client A Mosiac
- *  client that is filtered by the given selection.
+ * @param {MosaicClient} client A Mosiac client that is filtered by the
+ *  given selection.
  */
 function connectSelection(mc, selection, client) {
   if (!selection) return;

packages/core/src/DataCubeIndexer.js

@@ -6,55 +6,112 @@ import { fnv_hash } from './util/hash.js';
 
 const Skip = { skip: true, result: null };
 
+/**
+ * @typedef {object} DataCubeIndexerOptions
+ * @property {string} [schema] Database schema (namespace) in which to write
+ *  data cube index tables (default 'mosaic').
+ * @property {boolean} [options.enabled=true] Flag to enable or disable the
+ *  indexer. This setting can later be updated via the `enabled` method.
+ */
+
 /**
  * Build and query optimized indices ("data cubes") for fast computation of
  * groupby aggregate queries over compatible client queries and selections.
  * A data cube contains pre-aggregated data for a Mosaic client, subdivided
  * by possible query values from an active selection clause. These cubes are
  * realized as as database tables that can be queried for rapid updates.
+ *
  * Compatible client queries must consist of only groupby dimensions and
  * supported aggregate functions. Compatible selections must contain an active
  * clause that exposes metadata for an interval or point value predicate.
+ *
+ * Data cube index tables are written to a dedicated schema (namespace) that
+ * can be set using the *schema* constructor option. This schema acts as a
+ * persistent cache, and index tables may be used across sessions. The
+ * `dropIndexTables` method issues a query to remove *all* tables within
+ * this schema. This may be needed if the original tables have updated data,
+ * but should be used with care.
  */
 export class DataCubeIndexer {
   /**
    * Create a new data cube index table manager.
    * @param {import('./Coordinator.js').Coordinator} coordinator A Mosaic coordinator.
-   * @param {object} [options] Indexer options.
-   * @param {boolean} [options.enabled=true] Flag to enable/disable indexer.
-   * @param {boolean} [options.temp=true] Flag to indicate if generated data
-   *  cube index tables should be temporary tables.
+   * @param {DataCubeIndexerOptions} [options] Data cube indexer options.
    */
   constructor(coordinator, {
-    enabled = true,
-    temp = true
+    schema = 'mosaic',
+    enabled = true
   } = {}) {
     /** @type {Map<import('./MosaicClient.js').MosaicClient, DataCubeInfo | Skip | null>} */
     this.indexes = new Map();
     this.active = null;
-    this.temp = temp;
     this.mc = coordinator;
+    this._schema = schema;
     this._enabled = enabled;
   }
 
   /**
-   * Set the enabled state of this indexer. If false, any cached state is
+   * Set the enabled state of this indexer. If false, any local state is
    * cleared and subsequent index calls will return null until re-enabled.
-   * @param {boolean} state The enabled state.
+   * This method has no effect on any index tables already in the database.
+   * @param {boolean} [state] The enabled state to set.
    */
-  enabled(state) {
-    if (state === undefined) {
-      return this._enabled;
-    } else if (this._enabled !== state) {
+  set enabled(state) {
+    if (this._enabled !== state) {
       if (!state) this.clear();
       this._enabled = state;
     }
   }
 
+  /**
+   * Get the enabled state of this indexer.
+   * @returns {boolean} The current enabled state.
+   */
+  get enabled() {
+    return this._enabled;
+  }
+
+  /**
+   * Set the database schema used by this indexer. Upon changes, any local
+   * state is cleared. This method does _not_ drop any existing data cube
+   * tables, use `dropIndexTables` before changing the schema to also remove
+   * existing index tables in the database.
+   * @param {string} [schema] The schema name to set.
+   */
+  set schema(schema) {
+    if (this._schema !== schema) {
+      this.clear();
+      this._schema = schema;
+    }
+  }
+
+  /**
+   * Get the database schema used by this indexer.
+   * @returns {string} The current schema name.
+   */
+  get schema() {
+    return this._schema;
+  }
+
+  /**
+   * Issues a query through the coordinator to drop the current index table
+   * schema. *All* tables in the schema will be removed and local state is
+   * cleared. Call this method if the underlying base tables have been updated,
+   * causing derived index tables to become stale and inaccurate. Use this
+   * method with care! Once dropped, the schema will be repopulated by future
+   * data cube indexer requests.
+   * @returns A query result promise.
+   */
+  dropIndexTables() {
+    this.clear();
+    return this.mc.exec(`DROP SCHEMA IF EXISTS "${this.schema}" CASCADE`);
+  }
+
   /**
    * Clear the cache of data cube index table entries for the current active
    * selection clause. This method does _not_ drop any existing data cube
-   * tables.
+   * tables. Use `dropIndexTables` to remove existing index tables from the
+   * database.
    */
   clear() {
     this.indexes.clear();
@@ -77,9 +134,9 @@ export class DataCubeIndexer {
    */
   index(client, selection, activeClause) {
     // if not enabled, do nothing
-    if (!this._enabled) return null;
+    if (!this.enabled) return null;
 
-    const { indexes, mc, temp } = this;
+    const { indexes, mc, schema } = this;
     const { source } = activeClause;
 
     // if there is no clause source to track, do nothing
@@ -125,8 +182,11 @@ export class DataCubeIndexer {
     } else {
       // generate data cube index table
       const filter = selection.remove(source).predicate(client);
-      info = dataCubeInfo(client.query(filter), active, indexCols);
-      info.result = mc.exec(create(info.table, info.create, { temp }));
+      info = dataCubeInfo(client.query(filter), active, indexCols, schema);
+      info.result = mc.exec([
+        `CREATE SCHEMA IF NOT EXISTS ${schema}`,
+        create(info.table, info.create, { temp: false })
+      ]);
       info.result.catch(e => mc.logger().error(e));
     }
 
@@ -223,7 +283,7 @@ function binInterval(scale, pixelSize, bin) {
  * @param {*} indexCols Data cube index column definitions.
  * @returns {DataCubeInfo}
  */
-function dataCubeInfo(clientQuery, active, indexCols) {
+function dataCubeInfo(clientQuery, active, indexCols, schema) {
   const { dims, aggr, aux } = indexCols;
   const { columns } = active;
 
@@ -246,7 +306,7 @@ function dataCubeInfo(clientQuery, active, indexCols) {
   // generate creation query string and hash id
   const create = query.toString();
   const id = (fnv_hash(create) >>> 0).toString(16);
-  const table = `cube_index_${id}`;
+  const table = `${schema}.cube_${id}`;
 
   // generate data cube select query
   const select = Query
@@ -255,7 +315,7 @@ function dataCubeInfo(clientQuery, active, indexCols) {
     .groupby(dims)
     .orderby(order);
 
-  return new DataCubeInfo({ table, create, active, select });
+  return new DataCubeInfo({ id, table, create, active, select });
 }
 
 /**