diff --git a/packages/core/src/QueryManager.js b/packages/core/src/QueryManager.js index 9badd816..ba9792d6 100644 --- a/packages/core/src/QueryManager.js +++ b/packages/core/src/QueryManager.js @@ -38,7 +38,8 @@ export class QueryManager { async submit(request, result) { try { const { query, type, cache = false, record = true, options } = request; - const sql = query ? `${query}` : null; + const params = []; + let sql = query ? query.toString(params) : null; // update recorders if (record) { @@ -58,9 +59,9 @@ export class QueryManager { // issue query, potentially cache result const t0 = performance.now(); if (this._logQueries) { - this._logger.debug('Query', { type, sql, ...options }); + this._logger.debug('Query', { type, sql, params, ...options }); } - const data = await this.db.query({ type, sql, ...options }); + const data = await this.db.query({ type, sql, params: params.length ? params : undefined, ...options }); if (cache) this.clientCache.set(sql, data); this._logger.debug(`Request: ${(performance.now() - t0).toFixed(1)}`); result.fulfill(data); diff --git a/packages/duckdb/bin/run-server.js b/packages/duckdb/bin/run-server.js index 417c104d..3f91d3ca 100755 --- a/packages/duckdb/bin/run-server.js +++ b/packages/duckdb/bin/run-server.js @@ -2,6 +2,6 @@ import { DuckDB, dataServer } from '../src/index.js'; // the database to connect to, default is main memory -const dbPath = process.argv[2] || ':memory:'; +const dbPath = process.argv[2] || './test.db'; dataServer(new DuckDB(dbPath), { rest: true, socket: true }); diff --git a/packages/duckdb/src/Cache.js b/packages/duckdb/src/Cache.js index 3867efe3..15781604 100644 --- a/packages/duckdb/src/Cache.js +++ b/packages/duckdb/src/Cache.js @@ -5,8 +5,10 @@ import path from 'node:path'; const DEFAULT_CACHE_DIR = '.cache'; const DEFAULT_TTL = 1000 * 60 * 60 * 24 * 7; // 7 days -export function cacheKey(hashable, type) { - return createHash('sha256').update(hashable).digest('hex') + '.' + type; +export function cacheKey(hashable, type, params) { + const hash = createHash('sha256').update(hashable); + if (params) hash.update(params.join(",")); + return hash.digest('hex') + '.' + type; } class CacheEntry { diff --git a/packages/duckdb/src/DuckDB.js b/packages/duckdb/src/DuckDB.js index 1d109a62..df038791 100644 --- a/packages/duckdb/src/DuckDB.js +++ b/packages/duckdb/src/DuckDB.js @@ -20,6 +20,7 @@ export class DuckDB { this.db = new duckdb.Database(path, config); this.con = this.db.connect(); this.exec(initStatements); + this.preparedStatements = new Map; } close() { @@ -35,7 +36,13 @@ export class DuckDB { } prepare(sql) { - return new DuckDBStatement(this.con.prepare(sql)); + let statement = this.preparedStatements.get(sql); + if (statement) { + return statement + } + statement = new DuckDBStatement(this.con.prepare(sql)); + this.preparedStatements.set(sql, statement); + return statement; } exec(sql) { @@ -114,7 +121,7 @@ export class DuckDBStatement { arrowBuffer(params) { return new Promise((resolve, reject) => { - this.con.arrowIPCAll(...params, (err, result) => { + this.statement.arrowIPCAll(...params, (err, result) => { if (err) { reject(err); } else { diff --git a/packages/duckdb/src/data-server.js b/packages/duckdb/src/data-server.js index 2f2e81ac..a5f2d42a 100644 --- a/packages/duckdb/src/data-server.js +++ b/packages/duckdb/src/data-server.js @@ -72,14 +72,14 @@ export function queryHandler(db, queryCache) { // retrieve query result async function retrieve(query, get) { - const { sql, type, persist } = query; - const key = cacheKey(sql, type); + const { sql, type, persist, params } = query; + const key = cacheKey(sql, type, params); let result = queryCache?.get(key); if (result) { console.log('CACHE HIT'); } else { - result = await get(sql); + result = await get(params ?? sql); if (persist) { queryCache?.set(key, result, { persist }); } @@ -102,8 +102,13 @@ export function queryHandler(db, queryCache) { } try { - const { sql, type = 'json' } = query; - console.log(`> ${type.toUpperCase()}${sql ? ' ' + sql : ''}`); + const { sql, type = 'json', params } = query; + console.log(`> ${type.toUpperCase()}${sql ? ' ' + sql : ''}`, params); + + let statement; + if (params) { + statement = db.prepare(sql); + } // process query and return result switch (type) { @@ -112,13 +117,19 @@ export function queryHandler(db, queryCache) { await db.exec(sql); res.done(); break; + case 'prepare': + // Prepare the query for later execution + await db.prepare(sql); + res.done(); + break; case 'arrow': // Apache Arrow response format - res.arrow(await retrieve(query, sql => db.arrowBuffer(sql))); + console.log(statement, params) + res.arrow(await retrieve(query, statement?.arrowBuffer.bind(statement) ?? db.arrowBuffer.bind(db))); break; case 'json': // JSON response format - res.json(await retrieve(query, sql => db.query(sql))); + res.json(await retrieve(query, statement?.query.bind(statement) ?? db.query.bind(db))); break; case 'create-bundle': // Create a named bundle of precomputed resources diff --git a/packages/duckdb/test/duckdb-test.js b/packages/duckdb/test/duckdb-test.js index 08050866..9985fa87 100644 --- a/packages/duckdb/test/duckdb-test.js +++ b/packages/duckdb/test/duckdb-test.js @@ -36,4 +36,24 @@ describe('DuckDB', () => { await db.exec('DROP TABLE json'); }); }); + + describe('prepare', () => { + it('can run a prepared statement', async () => { + const statement = db.prepare('SELECT ?+? AS foo'); + const res0 = await statement.query([1,2]); + assert.deepEqual(res0, [{foo: 3}]); + + const res1 = await statement.query([2,3]); + assert.deepEqual(res1, [{foo: 5}]); + }); + + it('can run a prepared arrow statement', async () => { + const statement = db.prepare('SELECT ?+? AS foo'); + const res0 = await statement.arrowBuffer([1,2]); + assert.deepEqual(res0, [{foo: 3}]); + + const res1 = await statement.arrowBuffer([2,3]); + assert.deepEqual(res1, [{foo: 5}]); + }); + }); }); diff --git a/packages/sql/src/Query.js b/packages/sql/src/Query.js index fa3abd7e..a0b82e41 100644 --- a/packages/sql/src/Query.js +++ b/packages/sql/src/Query.js @@ -411,7 +411,7 @@ export class Query { return q; } - toString() { + toString(params = []) { const { with: cte, select, distinct, from, sample, where, groupby, having, window, qualify, orderby, limit, offset @@ -421,7 +421,7 @@ export class Query { // WITH if (cte.length) { - const list = cte.map(({ as, query })=> `"${as}" AS (${query})`); + const list = cte.map(({ as, query })=> `"${as.toString(params)}" AS (${query})`); sql.push(`WITH ${list.join(', ')}`); } @@ -444,7 +444,7 @@ export class Query { // WHERE if (where.length) { - const clauses = where.map(String).filter(x => x).join(' AND '); + const clauses = where.map(c => c.toString(params)).filter(x => x).join(' AND '); if (clauses) sql.push(`WHERE ${clauses}`); } @@ -463,13 +463,13 @@ export class Query { // HAVING if (having.length) { - const clauses = having.map(String).filter(x => x).join(' AND '); + const clauses = having.map(c => c.toString(params)).filter(x => x).join(' AND '); if (clauses) sql.push(`HAVING ${clauses}`); } // WINDOW if (window.length) { - const windows = window.map(({ as, expr }) => `"${as}" AS (${expr})`); + const windows = window.map(({ as, expr }) => `"${as.toString(params)}" AS (${expr})`); sql.push(`WINDOW ${windows.join(', ')}`); } diff --git a/packages/sql/src/expression.js b/packages/sql/src/expression.js index c84bbdba..1002db43 100644 --- a/packages/sql/src/expression.js +++ b/packages/sql/src/expression.js @@ -37,11 +37,13 @@ export class SQLExpression { * @param {string[]} [columns=[]] The column dependencies * @param {object} [props] Additional properties for this expression. */ - constructor(parts, columns, props) { + constructor(parts, columns, queryParams, props) { this._expr = Array.isArray(parts) ? parts : [parts]; this._deps = columns || []; this.annotate(props); + this.params = queryParams; + const params = this._expr.filter(part => isParamLike(part)); if (params.length > 0) { /** @type {ParamLike[]} */ @@ -108,9 +110,13 @@ export class SQLExpression { * Generate a SQL code string corresponding to this expression. * @returns {string} A SQL code string. */ - toString() { + toString(params) { + console.log(this?.params) + if (params) { + params.push(...(this?.params ?? [])); + } return this._expr - .map(p => isParamLike(p) && !isSQLExpression(p) ? literalToSQL(p.value) : p) + .map(p => isParamLike(p) && !isSQLExpression(p) ? literalToSQL(p.value, params) : p) .join(''); } @@ -134,7 +140,7 @@ function update(expr, callbacks) { } } -export function parseSQL(strings, exprs) { +export function parseSQL(strings, exprs, params) { const spans = [strings[0]]; const cols = new Set; const n = exprs.length; @@ -146,7 +152,7 @@ export function parseSQL(strings, exprs) { if (Array.isArray(e?.columns)) { e.columns.forEach(col => cols.add(col)); } - spans[k] += typeof e === 'string' ? e : literalToSQL(e); + spans[k] += typeof e === 'string' ? e : literalToSQL(e, params); } const s = strings[++i]; if (isParamLike(spans[k])) { @@ -165,6 +171,7 @@ export function parseSQL(strings, exprs) { * references), or parameterized values. */ export function sql(strings, ...exprs) { - const { spans, cols } = parseSQL(strings, exprs); - return new SQLExpression(spans, cols); + const params = []; + const { spans, cols } = parseSQL(strings, exprs, params); + return new SQLExpression(spans, cols, params); } diff --git a/packages/sql/src/literal.js b/packages/sql/src/literal.js index 4a6ae61b..1e34a859 100644 --- a/packages/sql/src/literal.js +++ b/packages/sql/src/literal.js @@ -2,5 +2,5 @@ import { literalToSQL } from './to-sql.js'; export const literal = value => ({ value, - toString: () => literalToSQL(value) + toString: (params) => literalToSQL(value, params) }); diff --git a/packages/sql/src/load/create.js b/packages/sql/src/load/create.js index e00bf696..1478c5af 100644 --- a/packages/sql/src/load/create.js +++ b/packages/sql/src/load/create.js @@ -1,6 +1,6 @@ export function create(name, query, { replace = false, - temp = true, + temp = false, view = false } = {}) { return 'CREATE' diff --git a/packages/sql/src/operators.js b/packages/sql/src/operators.js index dda6d9b7..3df05b8d 100644 --- a/packages/sql/src/operators.js +++ b/packages/sql/src/operators.js @@ -47,6 +47,7 @@ function rangeOp(op, a, range, exclusive) { const expr = !range ? sql`` : exclusive ? sql`${prefix}(${range[0]} <= ${a} AND ${a} < ${range[1]})` : sql`(${a} ${op} ${range[0]} AND ${range[1]})`; + expr.params = [...(range[0].params ?? []), ... (range[1].params ?? [])]; return expr.annotate({ op, visit, field: a, range }); } diff --git a/packages/sql/src/to-sql.js b/packages/sql/src/to-sql.js index fd2d742f..7e812b16 100644 --- a/packages/sql/src/to-sql.js +++ b/packages/sql/src/to-sql.js @@ -22,14 +22,29 @@ export function toSQL(value) { * @param {*} value The literal value. * @returns {string} A SQL string. */ -export function literalToSQL(value) { +export function literalToSQL(value, params) { switch (typeof value) { case 'boolean': + if (params) { + params.push(value); + return '?'; + } return value ? 'TRUE' : 'FALSE'; case 'string': + if (params) { + params.push(value); + return '?'; + } return `'${value.replace(`'`, `''`)}'`; case 'number': - return Number.isFinite(value) ? String(value) : 'NULL'; + if (Number.isFinite(value)) { + if (params) { + params.push(value); + return '?'; + } + return String(value) + } + return 'NULL'; default: if (value == null) { return 'NULL'; @@ -46,6 +61,9 @@ export function literalToSQL(value) { return `'${value.source}'`; } else { // otherwise rely on string coercion + if (params && value.toSQL) { + return value.toSQL(params) + } return String(value); } } diff --git a/packages/sql/test/literal-test.js b/packages/sql/test/literal-test.js index 45398016..2a7e1370 100644 --- a/packages/sql/test/literal-test.js +++ b/packages/sql/test/literal-test.js @@ -62,4 +62,11 @@ describe('literal', () => { assert.strictEqual(stringWithQuotes.value, `don't`); assert.strictEqual(String(stringWithQuotes), `'don''t'`); }); + it(`supports toSQL`, () => { + const numberExpr = literal(1); + + const params = []; + assert.strictEqual(numberExpr.toSQL(params), `?`); + assert.deepStrictEqual(params, [1]); + }) }); diff --git a/packages/sql/test/query-test.js b/packages/sql/test/query-test.js index 2fc90c29..2c6a07a7 100644 --- a/packages/sql/test/query-test.js +++ b/packages/sql/test/query-test.js @@ -290,6 +290,23 @@ describe('Query', () => { .toString(), query ); + + assert.strictEqual(gt(bar, 50).toSQL([]), `"bar" > ?`); + + const params = []; + assert.strictEqual( + Query + .select(foo) + .from('data') + .where(gt(bar, 50), lt(bar, 100)) + .toSQL(params), + [ + 'SELECT "foo"', + 'FROM "data"', + 'WHERE ("bar" > ?) AND ("bar" < ?)' + ].join(' ') + ); + assert.deepStrictEqual(params, [50, 100]); }); it('selects ordered rows', () => {