From c3617647cbac2c6537ce04aeee77b6f558428ae5 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Wed, 15 Feb 2023 06:57:16 +0000 Subject: [PATCH 01/34] indexeddb: improve allDocs() perf with skip & key ranges --- .../pouchdb-adapter-indexeddb/src/allDocs.js | 214 ++++++++++-------- .../pouchdb-adapter-indexeddb/src/bulkDocs.js | 15 +- .../pouchdb-adapter-indexeddb/src/get.js | 6 +- .../src/getAttachment.js | 6 +- .../pouchdb-adapter-indexeddb/src/index.js | 13 +- .../pouchdb-adapter-indexeddb/src/setup.js | 3 +- 6 files changed, 143 insertions(+), 114 deletions(-) diff --git a/packages/node_modules/pouchdb-adapter-indexeddb/src/allDocs.js b/packages/node_modules/pouchdb-adapter-indexeddb/src/allDocs.js index d8ae99b1c4..86c919616b 100644 --- a/packages/node_modules/pouchdb-adapter-indexeddb/src/allDocs.js +++ b/packages/node_modules/pouchdb-adapter-indexeddb/src/allDocs.js @@ -28,31 +28,16 @@ function allDocsKeys(keys, docStore, allDocsInner) { function createKeyRange(start, end, inclusiveEnd, key, descending) { try { - if (start && end) { - if (descending) { - return IDBKeyRange.bound(end, start, !inclusiveEnd, false); - } else { - return IDBKeyRange.bound(start, end, false, !inclusiveEnd); - } - } else if (start) { - if (descending) { - return IDBKeyRange.upperBound(start); - } else { - return IDBKeyRange.lowerBound(start); - } - } else if (end) { - if (descending) { - return IDBKeyRange.lowerBound(end, !inclusiveEnd); - } else { - return IDBKeyRange.upperBound(end, !inclusiveEnd); - } - } else if (key) { - return IDBKeyRange.only(key); + if (key) { + return IDBKeyRange.only([0, key]); + } else if (descending) { + return IDBKeyRange.bound(end, start, !inclusiveEnd, false); + } else { + return IDBKeyRange.bound(start, end, false, !inclusiveEnd); } } catch (e) { return {error: e}; } - return null; } function handleKeyRangeError(opts, metadata, err, callback) { @@ -95,31 +80,46 @@ export default function (txn, metadata, opts, callback) { var results = []; var processing = []; - var start = 'startkey' in opts ? opts.startkey : false; - var end = 'endkey' in opts ? opts.endkey : false; var key = 'key' in opts ? opts.key : false; var keys = 'keys' in opts ? opts.keys : false; var skip = opts.skip || 0; - var limit = typeof opts.limit === 'number' ? opts.limit : -1; + var limit = typeof opts.limit === 'number' ? opts.limit : undefined; var inclusiveEnd = opts.inclusive_end !== false; var descending = 'descending' in opts && opts.descending ? 'prev' : null; - - var keyRange; - if (!keys) { - keyRange = createKeyRange(start, end, inclusiveEnd, key, descending); - if (keyRange && keyRange.error) { - return handleKeyRangeError(opts, metadata, keyRange.error, callback); - } - } + var start = 'startkey' in opts ? opts.startkey : (descending ? '\uffff' : ''); + var end = 'endkey' in opts ? opts.endkey : (descending ? '' : '\uffff'); var docStore = txn.txn.objectStore(DOC_STORE); - txn.txn.oncomplete = onTxnComplete; - if (keys) { - return allDocsKeys(opts.keys, docStore, allDocsInner); + txn.txn.oncomplete = onTxnComplete; + const allDocsInner = doc => { + if (doc.error) { + return results.push(doc); + } + + const row = { id:doc.id, key:doc.id, value:{ rev:doc.rev } }; + + if (doc.deleted) { + row.value.deleted = true; + row.doc = null; + } else if (opts.include_docs) { + include_doc(row, doc); + } + + results.push(row); + }; + return allDocsKeys(keys, docStore, allDocsInner); } + let keyRange = createKeyRange([0, start], [0, end], inclusiveEnd, key, descending); + if (keyRange.error) { + return handleKeyRangeError(opts, metadata, keyRange.error, callback); + } + + // txn.oncomplete must be set AFTER key-range-error is generated + txn.txn.oncomplete = onTxnComplete; + function include_doc(row, doc) { var docData = doc.revs[doc.rev].data; @@ -139,79 +139,99 @@ export default function (txn, metadata, opts, callback) { } } - function allDocsInner(doc) { - if (doc.error && keys) { - // key was not found with "keys" requests - results.push(doc); - return true; - } - - var row = { - id: doc.id, - key: doc.id, - value: { - rev: doc.rev - } + function onTxnComplete() { + const returnVal = { + total_rows: metadata.doc_count, + offset: 0, + rows: results }; + /* istanbul ignore if */ + if (opts.update_seq) { + returnVal.update_seq = metadata.seq; + } - var deleted = doc.deleted; - if (deleted) { - if (keys) { - results.push(row); - row.value.deleted = true; - row.doc = null; - } - } else if (skip-- <= 0) { - results.push(row); - if (opts.include_docs) { - include_doc(row, doc); - } - if (--limit === 0) { - return false; - } + if (processing.length) { + Promise.all(processing).then(function () { + callback(null, returnVal); + }); + } else { + callback(null, returnVal); } - return true; } - function onTxnComplete() { - Promise.all(processing).then(function () { - var returnVal = { - total_rows: metadata.doc_count, - offset: 0, - rows: results - }; + const notDeletedDocsIndex = docStore.index('___not_deleted'); - /* istanbul ignore if */ - if (opts.update_seq) { - returnVal.update_seq = metadata.seq; + if (skip) { + notDeletedDocsIndex.openKeyCursor(keyRange, descending || 'next').onsuccess = (e) => { + const cursor = e.target.result; + if (!cursor) { + return txn.txn.commit(); } - callback(null, returnVal); - }); - } - - var cursor = descending ? - docStore.openCursor(keyRange, descending) : - docStore.openCursor(keyRange); - cursor.onsuccess = function (e) { + if (skip) { + cursor.advance(skip); + skip = 0; + return; + } - var doc = e.target.result && e.target.result.value; + const lastSkipped = cursor.key; + if (lastSkipped === undefined) { + // no results + return txn.txn.commit(); + } - // Happens if opts does not have limit, - // because cursor will end normally then, - // when all docs are retrieved. - // Would not be needed, if getAll() optimization was used like in #6059 - if (!doc) { return; } + // At this point, existing keyRange bounds are already compound keys. + if (descending) { + keyRange = createKeyRange(lastSkipped, keyRange.lower, inclusiveEnd, key, descending); + } else { + keyRange = createKeyRange(lastSkipped, keyRange.upper, inclusiveEnd, key, descending); + } + if (keyRange.error) { + txn.txn.abort(); + return handleKeyRangeError(opts, metadata, keyRange.error, callback); + } - // Skip local docs - if (/^_local/.test(doc.id)) { - return e.target.result.continue(); - } + fetchResults(); + }; + } else { + fetchResults(); + } - var continueCursor = allDocsInner(doc); - if (continueCursor) { - e.target.result.continue(); + function fetchResults() { + // REVIEW: there is a risk here with getting all results into memory - if they have multiple + // revs, then we risk loading loads of extra data which is then discarded. This could be + // reduced with batching. + // REVIEW: this also loads a lot of unused data when include_docs is false. The fastest and + // most pragmatic approach here may be batching... + if (descending && limit) { + // getAll() does not support descending, so this can either be implemented with a cursor, or + // by calling getAll(), iterating from the top, and discarding. It is currently using the + // latter approach, but may need thought and optimisation + notDeletedDocsIndex.getAll(keyRange).onsuccess = (e) => { + const values = e.target.result; + for (let i=values.length-1; i>=0 && limit--; --i) { + const doc = values[i]; + const row = { id:doc.id, key:doc.id, value:{ rev:doc.rev } }; + if (opts.include_docs) { + include_doc(row, doc); + } + results[values.length - i - 1] = row; + } + return txn.txn.commit(); + }; + } else { + notDeletedDocsIndex.getAll(keyRange, limit).onsuccess = (e) => { + const values = e.target.result; + for (let i=0; i Date: Fri, 17 Feb 2023 13:16:18 +0000 Subject: [PATCH 02/34] Rename index --- .../node_modules/pouchdb-adapter-indexeddb/src/allDocs.js | 8 ++++---- .../node_modules/pouchdb-adapter-indexeddb/src/setup.js | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/node_modules/pouchdb-adapter-indexeddb/src/allDocs.js b/packages/node_modules/pouchdb-adapter-indexeddb/src/allDocs.js index 86c919616b..c76579f602 100644 --- a/packages/node_modules/pouchdb-adapter-indexeddb/src/allDocs.js +++ b/packages/node_modules/pouchdb-adapter-indexeddb/src/allDocs.js @@ -159,10 +159,10 @@ export default function (txn, metadata, opts, callback) { } } - const notDeletedDocsIndex = docStore.index('___not_deleted'); + const dbIndex = docStore.index('_isDeleted_id'); if (skip) { - notDeletedDocsIndex.openKeyCursor(keyRange, descending || 'next').onsuccess = (e) => { + dbIndex.openKeyCursor(keyRange, descending || 'next').onsuccess = (e) => { const cursor = e.target.result; if (!cursor) { return txn.txn.commit(); @@ -207,7 +207,7 @@ export default function (txn, metadata, opts, callback) { // getAll() does not support descending, so this can either be implemented with a cursor, or // by calling getAll(), iterating from the top, and discarding. It is currently using the // latter approach, but may need thought and optimisation - notDeletedDocsIndex.getAll(keyRange).onsuccess = (e) => { + dbIndex.getAll(keyRange).onsuccess = (e) => { const values = e.target.result; for (let i=values.length-1; i>=0 && limit--; --i) { const doc = values[i]; @@ -220,7 +220,7 @@ export default function (txn, metadata, opts, callback) { return txn.txn.commit(); }; } else { - notDeletedDocsIndex.getAll(keyRange, limit).onsuccess = (e) => { + dbIndex.getAll(keyRange, limit).onsuccess = (e) => { const values = e.target.result; for (let i=0; i Date: Wed, 11 Oct 2023 13:09:04 +0000 Subject: [PATCH 03/34] fix eol-last --- .../node_modules/pouchdb-adapter-indexeddb/src/getAttachment.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/node_modules/pouchdb-adapter-indexeddb/src/getAttachment.js b/packages/node_modules/pouchdb-adapter-indexeddb/src/getAttachment.js index 798cc9be86..74ed6906f1 100644 --- a/packages/node_modules/pouchdb-adapter-indexeddb/src/getAttachment.js +++ b/packages/node_modules/pouchdb-adapter-indexeddb/src/getAttachment.js @@ -17,4 +17,4 @@ export default function getAttachment(txn, docId, attachId, attachment, opts, cb cb(null, btoa(binString)); }); } -} \ No newline at end of file +} From 42cc8531e64fca8276c744caa6428000e74d5d05 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Wed, 11 Oct 2023 14:19:58 +0000 Subject: [PATCH 04/34] add migration --- .../pouchdb-adapter-indexeddb/src/setup.js | 42 ++++++++++++++++--- 1 file changed, 37 insertions(+), 5 deletions(-) diff --git a/packages/node_modules/pouchdb-adapter-indexeddb/src/setup.js b/packages/node_modules/pouchdb-adapter-indexeddb/src/setup.js index 28ce9a3a24..9ed8863a2d 100644 --- a/packages/node_modules/pouchdb-adapter-indexeddb/src/setup.js +++ b/packages/node_modules/pouchdb-adapter-indexeddb/src/setup.js @@ -1,6 +1,7 @@ 'use strict'; import { uuid } from 'pouchdb-utils'; +import { isLocalId } from 'pouchdb-adapter-utils'; import { META_STORE, DOC_STORE, rawIndexFields, naturalIndexName, correctIndexFields } from './util'; @@ -8,7 +9,7 @@ import { META_STORE, DOC_STORE, rawIndexFields, naturalIndexName, correctIndexFi // Core PouchDB schema version. Increment this if we, as a library, want to make // schema changes in indexeddb. See upgradePouchDbSchema() // -var POUCHDB_IDB_VERSION = 1; +var POUCHDB_IDB_VERSION = 2; // // Functions that manage a combinate indexeddb version, by combining the current @@ -91,17 +92,21 @@ function maintainNativeIndexes(openReq, reject) { }; } -function upgradePouchDbSchema(db, pouchdbVersion) { +function upgradePouchDbSchema(db, tx, pouchdbVersion) { if (pouchdbVersion < 1) { var docStore = db.createObjectStore(DOC_STORE, {keyPath : 'id'}); docStore.createIndex('seq', 'seq', {unique: true}); - docStore.createIndex('_isDeleted_id', [ 'deleted', 'id' ], {unique: true}); db.createObjectStore(META_STORE, {keyPath: 'id'}); } + if (pouchdbVersion < 2) { + const docStore = tx.objectStore(DOC_STORE); + docStore.createIndex('_isDeleted_id', [ 'deleted', 'id' ], {unique: true}); + } + // Declare more PouchDB schema changes here - // if (pouchdbVersion < 2) { .. } + // if (pouchdbVersion < 3) { .. } } function openDatabase(openDatabases, api, opts, resolve, reject) { @@ -123,11 +128,38 @@ function openDatabase(openDatabases, api, opts, resolve, reject) { throw new Error('Database was deleted while open'); } + const tx = e.target.transaction; var db = e.target.result; var pouchdbVersion = getPouchDbVersion(e.oldVersion); - upgradePouchDbSchema(db, pouchdbVersion); + upgradePouchDbSchema(db, tx, pouchdbVersion); maintainNativeIndexes(openReq, reject); + + if (pouchdbVersion < 2) { + const docStore = openReq.transaction.objectStore(DOC_STORE); + const metaStore = openReq.transaction.objectStore(META_STORE); + + const allDocsReq = docStore.openCursor(); + allDocsReq.onsuccess = event => { + const cursor = event.target.result; + if (!cursor) { + return; + } + + const doc = cursor.value; + + if (!isLocalId(doc.id)) { + return cursor.continue(); + } + + // Move _local/ docs to the META_STORE + metaStore.put(doc).onsuccess = () => { + cursor.delete(doc).onsuccess = () => { + cursor.continue(); + }; + }; + }; + } }; openReq.onblocked = function (e) { From dd6ed4c7b8a688d3dc6c8f47bee40ffb1ef15473 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Mon, 16 Oct 2023 09:56:48 +0000 Subject: [PATCH 05/34] Use isLocalId() --- .../node_modules/pouchdb-adapter-indexeddb/src/bulkDocs.js | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/node_modules/pouchdb-adapter-indexeddb/src/bulkDocs.js b/packages/node_modules/pouchdb-adapter-indexeddb/src/bulkDocs.js index c535fcf09d..e5289eaa92 100644 --- a/packages/node_modules/pouchdb-adapter-indexeddb/src/bulkDocs.js +++ b/packages/node_modules/pouchdb-adapter-indexeddb/src/bulkDocs.js @@ -13,7 +13,7 @@ import { binaryStringToBlobOrBuffer as binStringToBlobOrBuffer } from 'pouchdb-binary-utils'; -import { parseDoc } from 'pouchdb-adapter-utils'; +import { isLocalId, parseDoc } from 'pouchdb-adapter-utils'; import { binaryMd5 as md5 } from 'pouchdb-md5'; import { winningRev as calculateWinningRev, merge, compactTree } from 'pouchdb-merge'; @@ -70,8 +70,7 @@ export default function (api, req, opts, metadata, dbOpts, idbChanges, callback) } docs.forEach(function (doc) { - const isLocal = doc.id.startsWith('_local/'); - const docStore = isLocal ? META_STORE : DOC_STORE; + const docStore = isLocalId(doc.id) ? META_STORE : DOC_STORE; txn.objectStore(docStore).get(doc.id).onsuccess = readDone; }); } @@ -197,7 +196,7 @@ export default function (api, req, opts, metadata, dbOpts, idbChanges, callback) var winningRev = calculateWinningRev(doc); // rev of new doc for attachments and to return it var writtenRev = doc.rev; - var isLocal = doc.id.startsWith('_local/'); + const isLocal = isLocalId(doc.id); var theDoc = doc.revs[winningRev].data; From 8d72af6f386c2c702ba58ab684809b99f4731d93 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Mon, 11 Dec 2023 14:57:31 +0000 Subject: [PATCH 06/34] rename META_STORE to META_LOCAL_STORE --- .../pouchdb-adapter-indexeddb/src/bulkDocs.js | 16 ++++++++-------- .../pouchdb-adapter-indexeddb/src/get.js | 4 ++-- .../pouchdb-adapter-indexeddb/src/index.js | 6 +++--- .../pouchdb-adapter-indexeddb/src/setup.js | 18 +++++++++--------- .../pouchdb-adapter-indexeddb/src/util.js | 4 ++-- 5 files changed, 24 insertions(+), 24 deletions(-) diff --git a/packages/node_modules/pouchdb-adapter-indexeddb/src/bulkDocs.js b/packages/node_modules/pouchdb-adapter-indexeddb/src/bulkDocs.js index 0c66e371fe..69e10b0a5a 100644 --- a/packages/node_modules/pouchdb-adapter-indexeddb/src/bulkDocs.js +++ b/packages/node_modules/pouchdb-adapter-indexeddb/src/bulkDocs.js @@ -19,7 +19,7 @@ import { isLocalId, parseDoc } from 'pouchdb-adapter-utils'; import { binaryMd5 as md5 } from 'pouchdb-md5'; import { winningRev as calculateWinningRev, merge, compactTree } from 'pouchdb-merge'; -import { DOC_STORE, META_STORE, idbError } from './util'; +import { DOC_STORE, META_LOCAL_STORE, idbError } from './util'; import { rewrite } from './rewrite'; @@ -62,7 +62,7 @@ export default function (api, req, opts, metadata, dbOpts, idbChanges, callback) } docs.forEach(function (doc) { - const docStore = isLocalId(doc.id) ? META_STORE : DOC_STORE; + const docStore = isLocalId(doc.id) ? META_LOCAL_STORE : DOC_STORE; txn.objectStore(docStore).get(doc.id).onsuccess = readDone; }); } @@ -284,7 +284,7 @@ export default function (api, req, opts, metadata, dbOpts, idbChanges, callback) // Local documents have different revision handling if (isLocal && doc.deleted) { - txn.objectStore(META_STORE).delete(doc.id).onsuccess = function () { + txn.objectStore(META_LOCAL_STORE).delete(doc.id).onsuccess = function () { results[i] = { ok: true, id: doc.id, @@ -295,7 +295,7 @@ export default function (api, req, opts, metadata, dbOpts, idbChanges, callback) return; } - const docStore = isLocal ? META_STORE : DOC_STORE; + const docStore = isLocal ? META_LOCAL_STORE : DOC_STORE; txn.objectStore(docStore).put(doc).onsuccess = function () { results[i] = { ok: true, @@ -308,7 +308,7 @@ export default function (api, req, opts, metadata, dbOpts, idbChanges, callback) function updateSeq(i) { if (i === lastWriteIndex) { - txn.objectStore(META_STORE).put(metadata); + txn.objectStore(META_LOCAL_STORE).put(metadata); } } @@ -396,11 +396,11 @@ export default function (api, req, opts, metadata, dbOpts, idbChanges, callback) } preProcessAttachments().then(function () { - // REVIEW: We _could_ check doc ids here, and only open the META_STORE if all docs are local. + // REVIEW: We _could_ check doc ids here, and only open the META_LOCAL_STORE if all docs are local. // This will marginally slow things down for non-local docs, as they always cause updates to the - // META_STORE. It seems pragmatic to keep the code simple and optimise for calls to bulkDocs() + // META_LOCAL_STORE. It seems pragmatic to keep the code simple and optimise for calls to bulkDocs() // which include non-local docs. - api._openTransactionSafely([DOC_STORE, META_STORE], 'readwrite', function (err, _txn) { + api._openTransactionSafely([DOC_STORE, META_LOCAL_STORE], 'readwrite', function (err, _txn) { if (err) { return callback(err); } diff --git a/packages/node_modules/pouchdb-adapter-indexeddb/src/get.js b/packages/node_modules/pouchdb-adapter-indexeddb/src/get.js index 1e0d8c658a..eb10fc19f0 100644 --- a/packages/node_modules/pouchdb-adapter-indexeddb/src/get.js +++ b/packages/node_modules/pouchdb-adapter-indexeddb/src/get.js @@ -2,7 +2,7 @@ import { createError, MISSING_DOC } from 'pouchdb-errors'; -import { DOC_STORE, META_STORE } from './util'; +import { DOC_STORE, META_LOCAL_STORE } from './util'; import { latest as getLatest } from 'pouchdb-merge'; @@ -11,7 +11,7 @@ export default function (txn, id, opts, callback) { return callback(txn.error); } - const docStore = id.startsWith('_local/') ? META_STORE : DOC_STORE; + const docStore = id.startsWith('_local/') ? META_LOCAL_STORE : DOC_STORE; txn.txn.objectStore(docStore).get(id).onsuccess = function (e) { var doc = e.target.result; diff --git a/packages/node_modules/pouchdb-adapter-indexeddb/src/index.js b/packages/node_modules/pouchdb-adapter-indexeddb/src/index.js index 3c770e8e67..b3d76cadc4 100644 --- a/packages/node_modules/pouchdb-adapter-indexeddb/src/index.js +++ b/packages/node_modules/pouchdb-adapter-indexeddb/src/index.js @@ -16,7 +16,7 @@ import destroy from './destroy'; import {query, viewCleanup} from './find'; import purge from './purge'; -import { DOC_STORE, META_STORE } from './util'; +import { DOC_STORE, META_LOCAL_STORE } from './util'; var ADAPTER_NAME = 'indexeddb'; @@ -107,7 +107,7 @@ function IndexeddbPouch(dbOpts, callback) { return info(metadata, cb); }); - api._get = $t(get, [META_STORE, DOC_STORE]); // TODO can be more selective with which store we're opening here + api._get = $t(get, [META_LOCAL_STORE, DOC_STORE]); // TODO can be more selective with which store we're opening here api._bulkDocs = $(function (_, req, opts, callback) { bulkDocs(api, req, opts, metadata, dbOpts, idbChanges, callback); @@ -123,7 +123,7 @@ function IndexeddbPouch(dbOpts, callback) { changes(txn, idbChanges, api, dbOpts, opts); }, [DOC_STORE]); - api._getRevisionTree = $t(getRevisionTree, [META_STORE, DOC_STORE]); // TODO as per get(), we could check and only open correct store here + api._getRevisionTree = $t(getRevisionTree, [META_LOCAL_STORE, DOC_STORE]); // TODO as per get(), we could check and only open correct store here api._doCompaction = $t(doCompaction, [DOC_STORE], 'readwrite'); api._customFindAbstractMapper = { diff --git a/packages/node_modules/pouchdb-adapter-indexeddb/src/setup.js b/packages/node_modules/pouchdb-adapter-indexeddb/src/setup.js index 7163bdb40b..e53006f9d3 100644 --- a/packages/node_modules/pouchdb-adapter-indexeddb/src/setup.js +++ b/packages/node_modules/pouchdb-adapter-indexeddb/src/setup.js @@ -5,7 +5,7 @@ import { isLocalId } from 'pouchdb-adapter-utils'; import { checkBlobSupport } from 'pouchdb-adapter-utils'; -import { META_STORE, DOC_STORE, rawIndexFields, naturalIndexName, correctIndexFields } from './util'; +import { META_LOCAL_STORE, DOC_STORE, rawIndexFields, naturalIndexName, correctIndexFields } from './util'; // // Core PouchDB schema version. Increment this if we, as a library, want to make @@ -99,7 +99,7 @@ function upgradePouchDbSchema(db, tx, pouchdbVersion) { var docStore = db.createObjectStore(DOC_STORE, {keyPath : 'id'}); docStore.createIndex('seq', 'seq', {unique: true}); - db.createObjectStore(META_STORE, {keyPath: 'id'}); + db.createObjectStore(META_LOCAL_STORE, {keyPath: 'id'}); } if (pouchdbVersion < 2) { @@ -139,7 +139,7 @@ function openDatabase(openDatabases, api, opts, resolve, reject) { if (pouchdbVersion < 2) { const docStore = openReq.transaction.objectStore(DOC_STORE); - const metaStore = openReq.transaction.objectStore(META_STORE); + const metaStore = openReq.transaction.objectStore(META_LOCAL_STORE); const allDocsReq = docStore.openCursor(); allDocsReq.onsuccess = event => { @@ -154,7 +154,7 @@ function openDatabase(openDatabases, api, opts, resolve, reject) { return cursor.continue(); } - // Move _local/ docs to the META_STORE + // Move _local/ docs to the META_LOCAL_STORE metaStore.put(doc).onsuccess = () => { cursor.delete(doc).onsuccess = () => { cursor.continue(); @@ -201,15 +201,15 @@ function openDatabase(openDatabases, api, opts, resolve, reject) { } }; - var metadata = {id: META_STORE}; - var txn = idb.transaction([META_STORE], 'readwrite'); + var metadata = {id: META_LOCAL_STORE}; + var txn = idb.transaction([META_LOCAL_STORE], 'readwrite'); txn.oncomplete = function () { resolve({idb, metadata}); }; - var metaStore = txn.objectStore(META_STORE); - metaStore.get(META_STORE).onsuccess = function (e) { + var metaStore = txn.objectStore(META_LOCAL_STORE); + metaStore.get(META_LOCAL_STORE).onsuccess = function (e) { metadata = e.target.result || metadata; var changed = false; @@ -233,7 +233,7 @@ function openDatabase(openDatabases, api, opts, resolve, reject) { const createBlobDoc = blob => ({ id:'blob-support', blob }); - checkBlobSupport(txn, META_STORE, createBlobDoc).then(blobSupport => { + checkBlobSupport(txn, META_LOCAL_STORE, createBlobDoc).then(blobSupport => { // Unfortunate that we have to track this in both the metadata and on // api, but sometimes we have access to one, sometimes the other (and // sometimes both). We could change function signatures in index.js diff --git a/packages/node_modules/pouchdb-adapter-indexeddb/src/util.js b/packages/node_modules/pouchdb-adapter-indexeddb/src/util.js index 213743dd26..3363a37f5e 100644 --- a/packages/node_modules/pouchdb-adapter-indexeddb/src/util.js +++ b/packages/node_modules/pouchdb-adapter-indexeddb/src/util.js @@ -9,7 +9,7 @@ import { import { sanitise } from './rewrite'; var DOC_STORE = 'docs'; -var META_STORE = 'meta'; +var META_LOCAL_STORE = 'meta'; function idbError(callback) { return function (evt) { @@ -110,7 +110,7 @@ function correctIndexFields(fields) { export { DOC_STORE, - META_STORE, + META_LOCAL_STORE, idbError, processAttachment, rawIndexFields, From 7f3e9800d5c96bbae2f4f9bc178670451fb8a3b0 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Mon, 11 Dec 2023 15:00:57 +0000 Subject: [PATCH 07/34] get(): use isLocalId() --- packages/node_modules/pouchdb-adapter-indexeddb/src/get.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/node_modules/pouchdb-adapter-indexeddb/src/get.js b/packages/node_modules/pouchdb-adapter-indexeddb/src/get.js index eb10fc19f0..267772585f 100644 --- a/packages/node_modules/pouchdb-adapter-indexeddb/src/get.js +++ b/packages/node_modules/pouchdb-adapter-indexeddb/src/get.js @@ -4,6 +4,7 @@ import { createError, MISSING_DOC } from 'pouchdb-errors'; import { DOC_STORE, META_LOCAL_STORE } from './util'; +import { isLocalId } from 'pouchdb-adapter-utils'; import { latest as getLatest } from 'pouchdb-merge'; export default function (txn, id, opts, callback) { @@ -11,7 +12,7 @@ export default function (txn, id, opts, callback) { return callback(txn.error); } - const docStore = id.startsWith('_local/') ? META_LOCAL_STORE : DOC_STORE; + const docStore = isLocalId(id) ? META_LOCAL_STORE : DOC_STORE; txn.txn.objectStore(docStore).get(id).onsuccess = function (e) { var doc = e.target.result; From 8b95c7fb97a8b84e9f7c0207c0d1dea9a731284e Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Mon, 11 Dec 2023 15:10:27 +0000 Subject: [PATCH 08/34] Implement getLocal() --- .../pouchdb-adapter-indexeddb/src/get.js | 43 +------------------ .../pouchdb-adapter-indexeddb/src/getLocal.js | 5 +++ .../pouchdb-adapter-indexeddb/src/index.js | 4 +- .../pouchdb-adapter-indexeddb/src/util.js | 42 +++++++++++++++++- 4 files changed, 50 insertions(+), 44 deletions(-) create mode 100644 packages/node_modules/pouchdb-adapter-indexeddb/src/getLocal.js diff --git a/packages/node_modules/pouchdb-adapter-indexeddb/src/get.js b/packages/node_modules/pouchdb-adapter-indexeddb/src/get.js index 267772585f..2cba8639d6 100644 --- a/packages/node_modules/pouchdb-adapter-indexeddb/src/get.js +++ b/packages/node_modules/pouchdb-adapter-indexeddb/src/get.js @@ -1,44 +1,5 @@ 'use strict'; -import { createError, MISSING_DOC } from 'pouchdb-errors'; +import { getDocFn, DOC_STORE } from './util'; -import { DOC_STORE, META_LOCAL_STORE } from './util'; - -import { isLocalId } from 'pouchdb-adapter-utils'; -import { latest as getLatest } from 'pouchdb-merge'; - -export default function (txn, id, opts, callback) { - if (txn.error) { - return callback(txn.error); - } - - const docStore = isLocalId(id) ? META_LOCAL_STORE : DOC_STORE; - - txn.txn.objectStore(docStore).get(id).onsuccess = function (e) { - var doc = e.target.result; - var rev; - if (!opts.rev) { - rev = (doc && doc.rev); - } else { - rev = opts.latest ? getLatest(opts.rev, doc) : opts.rev; - } - - if (!doc || (doc.deleted && !opts.rev) || !(rev in doc.revs)) { - callback(createError(MISSING_DOC, 'missing')); - return; - } - - var result = doc.revs[rev].data; - result._id = doc.id; - result._rev = rev; - - // WARNING: expecting possible old format - // TODO: why are we passing the transaction in the context? - // It's not clear we ever thread these txns usefully - callback(null, { - doc: result, - metadata: doc, - ctx: txn - }); - }; -} +export default getDocFn(DOC_STORE); diff --git a/packages/node_modules/pouchdb-adapter-indexeddb/src/getLocal.js b/packages/node_modules/pouchdb-adapter-indexeddb/src/getLocal.js new file mode 100644 index 0000000000..5cd28aa4d8 --- /dev/null +++ b/packages/node_modules/pouchdb-adapter-indexeddb/src/getLocal.js @@ -0,0 +1,5 @@ +'use strict'; + +import { getDocFn, META_LOCAL_STORE } from './util'; + +export default getDocFn(META_LOCAL_STORE); diff --git a/packages/node_modules/pouchdb-adapter-indexeddb/src/index.js b/packages/node_modules/pouchdb-adapter-indexeddb/src/index.js index b3d76cadc4..f23fd0e5f2 100644 --- a/packages/node_modules/pouchdb-adapter-indexeddb/src/index.js +++ b/packages/node_modules/pouchdb-adapter-indexeddb/src/index.js @@ -6,6 +6,7 @@ import setup from './setup'; // API implementations import info from './info'; import get from './get'; +import getLocal from './getLocal'; import getAttachment from './getAttachment'; import bulkDocs from './bulkDocs'; import allDocs from './allDocs'; @@ -107,7 +108,8 @@ function IndexeddbPouch(dbOpts, callback) { return info(metadata, cb); }); - api._get = $t(get, [META_LOCAL_STORE, DOC_STORE]); // TODO can be more selective with which store we're opening here + api._get = $t(get, [DOC_STORE]); + api._getLocal = $t(getLocal, [META_LOCAL_STORE]); api._bulkDocs = $(function (_, req, opts, callback) { bulkDocs(api, req, opts, metadata, dbOpts, idbChanges, callback); diff --git a/packages/node_modules/pouchdb-adapter-indexeddb/src/util.js b/packages/node_modules/pouchdb-adapter-indexeddb/src/util.js index 3363a37f5e..2b4e70ed38 100644 --- a/packages/node_modules/pouchdb-adapter-indexeddb/src/util.js +++ b/packages/node_modules/pouchdb-adapter-indexeddb/src/util.js @@ -1,12 +1,13 @@ 'use strict'; -import { createError, IDB_ERROR } from 'pouchdb-errors'; +import { createError, IDB_ERROR, MISSING_DOC } from 'pouchdb-errors'; import { base64StringToBlobOrBuffer as b64StringToBluffer, btoa, readAsBinaryString, } from 'pouchdb-binary-utils'; import { sanitise } from './rewrite'; +import { latest as getLatest } from 'pouchdb-merge'; var DOC_STORE = 'docs'; var META_LOCAL_STORE = 'meta'; @@ -108,6 +109,42 @@ function correctIndexFields(fields) { ); } +function getDocFn(docStore) { + return function (txn, id, opts, callback) { + if (txn.error) { + return callback(txn.error); + } + + txn.txn.objectStore(docStore).get(id).onsuccess = function (e) { + var doc = e.target.result; + var rev; + if (!opts.rev) { + rev = (doc && doc.rev); + } else { + rev = opts.latest ? getLatest(opts.rev, doc) : opts.rev; + } + + if (!doc || (doc.deleted && !opts.rev) || !(rev in doc.revs)) { + callback(createError(MISSING_DOC, 'missing')); + return; + } + + var result = doc.revs[rev].data; + result._id = doc.id; + result._rev = rev; + + // WARNING: expecting possible old format + // TODO: why are we passing the transaction in the context? + // It's not clear we ever thread these txns usefully + callback(null, { + doc: result, + metadata: doc, + ctx: txn + }); + }; + }; +} + export { DOC_STORE, META_LOCAL_STORE, @@ -116,5 +153,6 @@ export { rawIndexFields, isPartialFilterView, naturalIndexName, - correctIndexFields + correctIndexFields, + getDocFn, }; From 7d468daf6efdc369c8adf8fa4c68c7e01d1e6fd3 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Mon, 11 Dec 2023 15:14:25 +0000 Subject: [PATCH 09/34] Revert "Implement getLocal()" This reverts commit 8b95c7fb97a8b84e9f7c0207c0d1dea9a731284e. --- .../pouchdb-adapter-indexeddb/src/get.js | 43 ++++++++++++++++++- .../pouchdb-adapter-indexeddb/src/getLocal.js | 5 --- .../pouchdb-adapter-indexeddb/src/index.js | 4 +- .../pouchdb-adapter-indexeddb/src/util.js | 42 +----------------- 4 files changed, 44 insertions(+), 50 deletions(-) delete mode 100644 packages/node_modules/pouchdb-adapter-indexeddb/src/getLocal.js diff --git a/packages/node_modules/pouchdb-adapter-indexeddb/src/get.js b/packages/node_modules/pouchdb-adapter-indexeddb/src/get.js index 2cba8639d6..267772585f 100644 --- a/packages/node_modules/pouchdb-adapter-indexeddb/src/get.js +++ b/packages/node_modules/pouchdb-adapter-indexeddb/src/get.js @@ -1,5 +1,44 @@ 'use strict'; -import { getDocFn, DOC_STORE } from './util'; +import { createError, MISSING_DOC } from 'pouchdb-errors'; -export default getDocFn(DOC_STORE); +import { DOC_STORE, META_LOCAL_STORE } from './util'; + +import { isLocalId } from 'pouchdb-adapter-utils'; +import { latest as getLatest } from 'pouchdb-merge'; + +export default function (txn, id, opts, callback) { + if (txn.error) { + return callback(txn.error); + } + + const docStore = isLocalId(id) ? META_LOCAL_STORE : DOC_STORE; + + txn.txn.objectStore(docStore).get(id).onsuccess = function (e) { + var doc = e.target.result; + var rev; + if (!opts.rev) { + rev = (doc && doc.rev); + } else { + rev = opts.latest ? getLatest(opts.rev, doc) : opts.rev; + } + + if (!doc || (doc.deleted && !opts.rev) || !(rev in doc.revs)) { + callback(createError(MISSING_DOC, 'missing')); + return; + } + + var result = doc.revs[rev].data; + result._id = doc.id; + result._rev = rev; + + // WARNING: expecting possible old format + // TODO: why are we passing the transaction in the context? + // It's not clear we ever thread these txns usefully + callback(null, { + doc: result, + metadata: doc, + ctx: txn + }); + }; +} diff --git a/packages/node_modules/pouchdb-adapter-indexeddb/src/getLocal.js b/packages/node_modules/pouchdb-adapter-indexeddb/src/getLocal.js deleted file mode 100644 index 5cd28aa4d8..0000000000 --- a/packages/node_modules/pouchdb-adapter-indexeddb/src/getLocal.js +++ /dev/null @@ -1,5 +0,0 @@ -'use strict'; - -import { getDocFn, META_LOCAL_STORE } from './util'; - -export default getDocFn(META_LOCAL_STORE); diff --git a/packages/node_modules/pouchdb-adapter-indexeddb/src/index.js b/packages/node_modules/pouchdb-adapter-indexeddb/src/index.js index f23fd0e5f2..b3d76cadc4 100644 --- a/packages/node_modules/pouchdb-adapter-indexeddb/src/index.js +++ b/packages/node_modules/pouchdb-adapter-indexeddb/src/index.js @@ -6,7 +6,6 @@ import setup from './setup'; // API implementations import info from './info'; import get from './get'; -import getLocal from './getLocal'; import getAttachment from './getAttachment'; import bulkDocs from './bulkDocs'; import allDocs from './allDocs'; @@ -108,8 +107,7 @@ function IndexeddbPouch(dbOpts, callback) { return info(metadata, cb); }); - api._get = $t(get, [DOC_STORE]); - api._getLocal = $t(getLocal, [META_LOCAL_STORE]); + api._get = $t(get, [META_LOCAL_STORE, DOC_STORE]); // TODO can be more selective with which store we're opening here api._bulkDocs = $(function (_, req, opts, callback) { bulkDocs(api, req, opts, metadata, dbOpts, idbChanges, callback); diff --git a/packages/node_modules/pouchdb-adapter-indexeddb/src/util.js b/packages/node_modules/pouchdb-adapter-indexeddb/src/util.js index 2b4e70ed38..3363a37f5e 100644 --- a/packages/node_modules/pouchdb-adapter-indexeddb/src/util.js +++ b/packages/node_modules/pouchdb-adapter-indexeddb/src/util.js @@ -1,13 +1,12 @@ 'use strict'; -import { createError, IDB_ERROR, MISSING_DOC } from 'pouchdb-errors'; +import { createError, IDB_ERROR } from 'pouchdb-errors'; import { base64StringToBlobOrBuffer as b64StringToBluffer, btoa, readAsBinaryString, } from 'pouchdb-binary-utils'; import { sanitise } from './rewrite'; -import { latest as getLatest } from 'pouchdb-merge'; var DOC_STORE = 'docs'; var META_LOCAL_STORE = 'meta'; @@ -109,42 +108,6 @@ function correctIndexFields(fields) { ); } -function getDocFn(docStore) { - return function (txn, id, opts, callback) { - if (txn.error) { - return callback(txn.error); - } - - txn.txn.objectStore(docStore).get(id).onsuccess = function (e) { - var doc = e.target.result; - var rev; - if (!opts.rev) { - rev = (doc && doc.rev); - } else { - rev = opts.latest ? getLatest(opts.rev, doc) : opts.rev; - } - - if (!doc || (doc.deleted && !opts.rev) || !(rev in doc.revs)) { - callback(createError(MISSING_DOC, 'missing')); - return; - } - - var result = doc.revs[rev].data; - result._id = doc.id; - result._rev = rev; - - // WARNING: expecting possible old format - // TODO: why are we passing the transaction in the context? - // It's not clear we ever thread these txns usefully - callback(null, { - doc: result, - metadata: doc, - ctx: txn - }); - }; - }; -} - export { DOC_STORE, META_LOCAL_STORE, @@ -153,6 +116,5 @@ export { rawIndexFields, isPartialFilterView, naturalIndexName, - correctIndexFields, - getDocFn, + correctIndexFields }; From e2ea6eea812cb40e7c3f43f65ee1a4699327682f Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Mon, 11 Dec 2023 15:22:17 +0000 Subject: [PATCH 10/34] Implement getLocal() --- .../pouchdb-adapter-indexeddb/src/get.js | 7 ++--- .../pouchdb-adapter-indexeddb/src/getLocal.js | 26 +++++++++++++++++++ .../pouchdb-adapter-indexeddb/src/index.js | 4 ++- 3 files changed, 31 insertions(+), 6 deletions(-) create mode 100644 packages/node_modules/pouchdb-adapter-indexeddb/src/getLocal.js diff --git a/packages/node_modules/pouchdb-adapter-indexeddb/src/get.js b/packages/node_modules/pouchdb-adapter-indexeddb/src/get.js index 267772585f..f3e5ac6de1 100644 --- a/packages/node_modules/pouchdb-adapter-indexeddb/src/get.js +++ b/packages/node_modules/pouchdb-adapter-indexeddb/src/get.js @@ -2,9 +2,8 @@ import { createError, MISSING_DOC } from 'pouchdb-errors'; -import { DOC_STORE, META_LOCAL_STORE } from './util'; +import { DOC_STORE } from './util'; -import { isLocalId } from 'pouchdb-adapter-utils'; import { latest as getLatest } from 'pouchdb-merge'; export default function (txn, id, opts, callback) { @@ -12,9 +11,7 @@ export default function (txn, id, opts, callback) { return callback(txn.error); } - const docStore = isLocalId(id) ? META_LOCAL_STORE : DOC_STORE; - - txn.txn.objectStore(docStore).get(id).onsuccess = function (e) { + txn.txn.objectStore(DOC_STORE).get(id).onsuccess = function (e) { var doc = e.target.result; var rev; if (!opts.rev) { diff --git a/packages/node_modules/pouchdb-adapter-indexeddb/src/getLocal.js b/packages/node_modules/pouchdb-adapter-indexeddb/src/getLocal.js new file mode 100644 index 0000000000..d651ee6f1d --- /dev/null +++ b/packages/node_modules/pouchdb-adapter-indexeddb/src/getLocal.js @@ -0,0 +1,26 @@ +'use strict'; + +import { createError, MISSING_DOC } from 'pouchdb-errors'; + +import { META_LOCAL_STORE } from './util'; + +export default function (txn, id, callback) { + if (txn.error) { + return callback(txn.error); + } + + txn.txn.objectStore(META_LOCAL_STORE).get(id).onsuccess = function (e) { + const doc = e.target.result; + + if (!doc) { + callback(createError(MISSING_DOC, 'missing')); + return; + } + + const result = doc.revs[doc.rev].data; + result._id = doc.id; + result._rev = doc.rev; + + callback(null, result); + }; +} diff --git a/packages/node_modules/pouchdb-adapter-indexeddb/src/index.js b/packages/node_modules/pouchdb-adapter-indexeddb/src/index.js index b3d76cadc4..f23fd0e5f2 100644 --- a/packages/node_modules/pouchdb-adapter-indexeddb/src/index.js +++ b/packages/node_modules/pouchdb-adapter-indexeddb/src/index.js @@ -6,6 +6,7 @@ import setup from './setup'; // API implementations import info from './info'; import get from './get'; +import getLocal from './getLocal'; import getAttachment from './getAttachment'; import bulkDocs from './bulkDocs'; import allDocs from './allDocs'; @@ -107,7 +108,8 @@ function IndexeddbPouch(dbOpts, callback) { return info(metadata, cb); }); - api._get = $t(get, [META_LOCAL_STORE, DOC_STORE]); // TODO can be more selective with which store we're opening here + api._get = $t(get, [DOC_STORE]); + api._getLocal = $t(getLocal, [META_LOCAL_STORE]); api._bulkDocs = $(function (_, req, opts, callback) { bulkDocs(api, req, opts, metadata, dbOpts, idbChanges, callback); From 21878223ae0cdc2b5f0b8d7f4595a865ad35bd95 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Sun, 31 Dec 2023 08:15:42 +0000 Subject: [PATCH 11/34] mocha v10: fix failure handling Fixes error: An error occurred while bailing: TypeError: Cannot read property 'forEach' of undefined --- bin/test-browser.js | 10 ++++++---- tests/integration/webrunner.js | 3 ++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/bin/test-browser.js b/bin/test-browser.js index eeadce56c8..2927611217 100755 --- a/bin/test-browser.js +++ b/bin/test-browser.js @@ -90,7 +90,8 @@ class RemoteRunner { try { var additionalProps = ['pass', 'fail', 'pending'].indexOf(event.name) === -1 ? {} : { slow: event.obj.slow ? function () { return event.obj.slow; } : function () { return 60; }, - fullTitle: event.obj.fullTitle ? function () { return event.obj.fullTitle; } : undefined + fullTitle: event.obj.fullTitle ? function () { return event.obj.fullTitle; } : undefined, + titlePath: event.obj.titlePath ? function () { return event.obj.titlePath; } : undefined, }; var obj = Object.assign({}, event.obj, additionalProps); @@ -123,9 +124,10 @@ class RemoteRunner { handleFailed() { if (bail) { try { - this.handlers['end'].forEach(function (handler) { - handler(); - }); + const triggerHandler = handler => handler(); + this.onceHandlers.get('end').forEach(triggerHandler); + this.onceHandlers.delete('end'); + this.handlers.get('end').forEach(triggerHandler); } catch (e) { console.log('An error occurred while bailing:', e); } finally { diff --git a/tests/integration/webrunner.js b/tests/integration/webrunner.js index 4720b5e4b2..1577c9663a 100644 --- a/tests/integration/webrunner.js +++ b/tests/integration/webrunner.js @@ -21,7 +21,8 @@ title: obj.title, duration: obj.duration, slow: typeof obj.slow === 'function' ? obj.slow() : undefined, - fullTitle: typeof obj.fullTitle === 'function' ? obj.fullTitle() : undefined + fullTitle: typeof obj.fullTitle === 'function' ? obj.fullTitle() : undefined, + titlePath: typeof obj.titlePath === 'function' ? obj.titlePath() : undefined, }, err: err && { actual: err.actual, From d50645d2394723318fc48ed061f5e6364df03d08 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Sun, 31 Dec 2023 08:15:42 +0000 Subject: [PATCH 12/34] mocha v10: fix failure handling Fixes error: An error occurred while bailing: TypeError: Cannot read property 'forEach' of undefined And subsequent error: An error occurred while bailing: TypeError: test.titlePath is not a function --- bin/test-browser.js | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/bin/test-browser.js b/bin/test-browser.js index 2927611217..5d32585abc 100755 --- a/bin/test-browser.js +++ b/bin/test-browser.js @@ -86,6 +86,15 @@ class RemoteRunner { this.handlers.get(name).push(handler); } + triggerHandlers(eventName, handlerArgs) { + const triggerHandler = handler => handler.apply(null, handlerArgs); + + this.onceHandlers.get(eventName).forEach(triggerHandler); + this.onceHandlers.delete(eventName); + + this.handlers.get(eventName).forEach(triggerHandler); + } + async handleEvent(event) { try { var additionalProps = ['pass', 'fail', 'pending'].indexOf(event.name) === -1 ? {} : { @@ -95,12 +104,7 @@ class RemoteRunner { }; var obj = Object.assign({}, event.obj, additionalProps); - const triggerHandler = handler => handler(obj, event.err); - - this.onceHandlers.get(event.name).forEach(triggerHandler); - this.onceHandlers.delete(event.name); - - this.handlers.get(event.name).forEach(triggerHandler); + this.triggerHandlers(event.name, [ obj, event.err ]); switch (event.name) { case 'fail': this.handleFailed(); break; @@ -124,10 +128,7 @@ class RemoteRunner { handleFailed() { if (bail) { try { - const triggerHandler = handler => handler(); - this.onceHandlers.get('end').forEach(triggerHandler); - this.onceHandlers.delete('end'); - this.handlers.get('end').forEach(triggerHandler); + this.triggerHandlers('end'); } catch (e) { console.log('An error occurred while bailing:', e); } finally { From 85fbca82d14fd7df7e8fe97dd5222217b15a42a0 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Sun, 31 Dec 2023 09:04:55 +0000 Subject: [PATCH 13/34] indexeddb._getLocal(): process attachments --- .../pouchdb-adapter-indexeddb/src/getLocal.js | 19 ++++++++++++++++--- .../pouchdb-adapter-indexeddb/src/index.js | 4 +++- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/packages/node_modules/pouchdb-adapter-indexeddb/src/getLocal.js b/packages/node_modules/pouchdb-adapter-indexeddb/src/getLocal.js index d651ee6f1d..8973959de7 100644 --- a/packages/node_modules/pouchdb-adapter-indexeddb/src/getLocal.js +++ b/packages/node_modules/pouchdb-adapter-indexeddb/src/getLocal.js @@ -2,9 +2,12 @@ import { createError, MISSING_DOC } from 'pouchdb-errors'; -import { META_LOCAL_STORE } from './util'; +import { META_LOCAL_STORE, processAttachment } from './util'; -export default function (txn, id, callback) { +// _getLocal() doesn't know if opts.binary is set or not, so assume it's not. +const BINARY_ATTACHMENTS = false; + +export default function (txn, id, api, callback) { if (txn.error) { return callback(txn.error); } @@ -21,6 +24,16 @@ export default function (txn, id, callback) { result._id = doc.id; result._rev = doc.rev; - callback(null, result); + if (result._attachments) { + const processing = []; + for (var name in result._attachments) { + processing.push(processAttachment(name, doc, result, BINARY_ATTACHMENTS, api.blobSupport)); + } + Promise.all(processing) + .then(() => callback(null, result)) + .catch(callback); + } else { + callback(null, result); + } }; } diff --git a/packages/node_modules/pouchdb-adapter-indexeddb/src/index.js b/packages/node_modules/pouchdb-adapter-indexeddb/src/index.js index f23fd0e5f2..5d6c8ec83b 100644 --- a/packages/node_modules/pouchdb-adapter-indexeddb/src/index.js +++ b/packages/node_modules/pouchdb-adapter-indexeddb/src/index.js @@ -109,7 +109,9 @@ function IndexeddbPouch(dbOpts, callback) { }); api._get = $t(get, [DOC_STORE]); - api._getLocal = $t(getLocal, [META_LOCAL_STORE]); + api._getLocal = $t(function (txn, id, callback) { + return getLocal(txn, id, api, callback); + }, [META_LOCAL_STORE]); api._bulkDocs = $(function (_, req, opts, callback) { bulkDocs(api, req, opts, metadata, dbOpts, idbChanges, callback); From 02a837c61cf9dee4853d2291201be1589ec213c3 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Sun, 24 Mar 2024 11:53:37 +0000 Subject: [PATCH 14/34] more comment --- packages/node_modules/pouchdb-adapter-indexeddb/src/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/node_modules/pouchdb-adapter-indexeddb/src/index.js b/packages/node_modules/pouchdb-adapter-indexeddb/src/index.js index 5d6c8ec83b..9eb070f1f1 100644 --- a/packages/node_modules/pouchdb-adapter-indexeddb/src/index.js +++ b/packages/node_modules/pouchdb-adapter-indexeddb/src/index.js @@ -127,7 +127,7 @@ function IndexeddbPouch(dbOpts, callback) { changes(txn, idbChanges, api, dbOpts, opts); }, [DOC_STORE]); - api._getRevisionTree = $t(getRevisionTree, [META_LOCAL_STORE, DOC_STORE]); // TODO as per get(), we could check and only open correct store here + api._getRevisionTree = $t(getRevisionTree, [META_LOCAL_STORE, DOC_STORE]); // TODO as per get(), we could check and only open correct store here // TODO TODO may only be relevant for non-local docs anyway api._doCompaction = $t(doCompaction, [DOC_STORE], 'readwrite'); api._customFindAbstractMapper = { From 9d518e823da1233a7c01268db4fbafa52af0ac26 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Mon, 25 Mar 2024 09:23:39 +0000 Subject: [PATCH 15/34] Resolve REVIEW comment --- .../node_modules/pouchdb-adapter-indexeddb/src/bulkDocs.js | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/node_modules/pouchdb-adapter-indexeddb/src/bulkDocs.js b/packages/node_modules/pouchdb-adapter-indexeddb/src/bulkDocs.js index cabe634f97..ee2ee1a865 100644 --- a/packages/node_modules/pouchdb-adapter-indexeddb/src/bulkDocs.js +++ b/packages/node_modules/pouchdb-adapter-indexeddb/src/bulkDocs.js @@ -396,10 +396,9 @@ export default function (api, req, opts, metadata, dbOpts, idbChanges, callback) } preProcessAttachments().then(function () { - // REVIEW: We _could_ check doc ids here, and only open the META_LOCAL_STORE if all docs are local. - // This will marginally slow things down for non-local docs, as they always cause updates to the - // META_LOCAL_STORE. It seems pragmatic to keep the code simple and optimise for calls to bulkDocs() - // which include non-local docs. + // We _could_ check doc ids here, and skip opening DOC_STORE if all docs are local. + // This may marginally slow things down for local docs. It seems pragmatic to keep + // the code simple and optimise for calls to bulkDocs() which include non-local docs. api._openTransactionSafely([DOC_STORE, META_LOCAL_STORE], 'readwrite', function (err, _txn) { if (err) { return callback(err); From 81ed8d51446560ff642a8b632c533cc5062af183 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Mon, 25 Mar 2024 09:24:50 +0000 Subject: [PATCH 16/34] getRevisionTree(): only open DOC_STORE --- packages/node_modules/pouchdb-adapter-indexeddb/src/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/node_modules/pouchdb-adapter-indexeddb/src/index.js b/packages/node_modules/pouchdb-adapter-indexeddb/src/index.js index 9eb070f1f1..8aecbf2083 100644 --- a/packages/node_modules/pouchdb-adapter-indexeddb/src/index.js +++ b/packages/node_modules/pouchdb-adapter-indexeddb/src/index.js @@ -127,7 +127,7 @@ function IndexeddbPouch(dbOpts, callback) { changes(txn, idbChanges, api, dbOpts, opts); }, [DOC_STORE]); - api._getRevisionTree = $t(getRevisionTree, [META_LOCAL_STORE, DOC_STORE]); // TODO as per get(), we could check and only open correct store here // TODO TODO may only be relevant for non-local docs anyway + api._getRevisionTree = $t(getRevisionTree, [DOC_STORE]); api._doCompaction = $t(doCompaction, [DOC_STORE], 'readwrite'); api._customFindAbstractMapper = { From 06c2b2654b33be8e8744aa8c4521649755d63a3a Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Mon, 25 Mar 2024 11:42:44 +0000 Subject: [PATCH 17/34] fetchResults() batch --- .../pouchdb-adapter-indexeddb/src/allDocs.js | 103 ++++++++++-------- 1 file changed, 60 insertions(+), 43 deletions(-) diff --git a/packages/node_modules/pouchdb-adapter-indexeddb/src/allDocs.js b/packages/node_modules/pouchdb-adapter-indexeddb/src/allDocs.js index 0ab9fb70eb..f66535b44f 100644 --- a/packages/node_modules/pouchdb-adapter-indexeddb/src/allDocs.js +++ b/packages/node_modules/pouchdb-adapter-indexeddb/src/allDocs.js @@ -26,14 +26,14 @@ function allDocsKeys(keys, docStore, allDocsInner) { }); } -function createKeyRange(start, end, inclusiveEnd, key, descending) { +function createKeyRange(start, end, inclusiveEnd, key, descending, inclusiveStart=true) { try { if (key) { return IDBKeyRange.only([0, key]); } else if (descending) { - return IDBKeyRange.bound(end, start, !inclusiveEnd, false); + return IDBKeyRange.bound(end, start, !inclusiveEnd, !inclusiveStart); } else { - return IDBKeyRange.bound(start, end, false, !inclusiveEnd); + return IDBKeyRange.bound(start, end, !inclusiveStart, !inclusiveEnd); } } catch (e) { return {error: e}; @@ -162,31 +162,45 @@ export default function (txn, metadata, opts, callback) { const dbIndex = docStore.index('_isDeleted_id'); - if (skip) { + if (!skip && !limit) { + fetchResults(); + } else { + let firstKey; + let limitKey = limit > 0; + dbIndex.openKeyCursor(keyRange, descending || 'next').onsuccess = (e) => { const cursor = e.target.result; - if (!cursor) { - return txn.txn.commit(); - } if (skip) { + if (!cursor) { return txn.txn.commit(); } cursor.advance(skip); skip = 0; return; } - const lastSkipped = cursor.key; - if (lastSkipped === undefined) { - // no results - return txn.txn.commit(); + if (firstKey === undefined) { + firstKey = cursor && cursor.key; + if (!firstKey) { return txn.txn.commit(); } } - // At this point, existing keyRange bounds are already compound keys. - if (descending) { - keyRange = createKeyRange(lastSkipped, keyRange.lower, inclusiveEnd, key, descending); - } else { - keyRange = createKeyRange(lastSkipped, keyRange.upper, inclusiveEnd, key, descending); + if (limit) { + if (limit > 1 && cursor) { + cursor.advance(limit - 1); + limit = undefined; + return; + } + limit = undefined; + } + + + if (limitKey) { + limitKey = cursor && cursor.key; + } + if (!limitKey) { + limitKey = descending ? keyRange.lower : keyRange.upper; } + + keyRange = createKeyRange(firstKey, limitKey, inclusiveEnd, key, descending); if (keyRange.error) { txn.txn.abort(); return handleKeyRangeError(opts, metadata, keyRange.error, callback); @@ -194,42 +208,45 @@ export default function (txn, metadata, opts, callback) { fetchResults(); }; - } else { - fetchResults(); } function fetchResults() { - // REVIEW: there is a risk here with getting all results into memory - if they have multiple - // revs, then we risk loading loads of extra data which is then discarded. This could be - // reduced with batching. - // REVIEW: this also loads a lot of unused data when include_docs is false. The fastest and - // most pragmatic approach here may be batching... - if (descending && limit) { - // getAll() does not support descending, so this can either be implemented with a cursor, or - // by calling getAll(), iterating from the top, and discarding. It is currently using the - // latter approach, but may need thought and optimisation - dbIndex.getAll(keyRange).onsuccess = (e) => { - const values = e.target.result; - for (let i=values.length-1; i>=0 && limit--; --i) { - const doc = values[i]; + // There is a risk here with getting all results into memory - if they have multiple + // revs, then we risk loading loads of extra data which is then discarded. This is + // reduced by batching. This also loads unused data when include_docs is false. + + // TODO the tests which cover this will only actually test it if batch size < 5ish + const batchSize = 100; + + fetchNextBatch(); + + function fetchNextBatch() { + dbIndex.getAll(keyRange, batchSize).onsuccess = (e) => { + const batch = e.target.result; + for (let i=0; i { - const values = e.target.result; - for (let i=0; i= batchSize) { + const lastSeenKey = [0, batch[batch.length-1].id]; + const startKey = descending ? keyRange.upper : lastSeenKey; + const endKey = descending ? lastSeenKey : keyRange.upper; + if (startKey[1] !== endKey[1]) { + // TODO unclear why + const _incEnd = descending ? false : inclusiveEnd; + const _incStart = descending ? true : false; + // TODO move inclusiveStart arg to be more consistent + keyRange = createKeyRange(startKey, endKey, _incEnd, key, descending, _incStart); + return fetchNextBatch(); } - results[i] = row; + } + if (descending) { + results.reverse(); } return txn.txn.commit(); }; From b9992fc6ae91dc8513802e17d1e81b4434315cbb Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Mon, 25 Mar 2024 16:58:35 +0000 Subject: [PATCH 18/34] Add a big test --- tests/integration/test.all_docs.js | 113 +++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) diff --git a/tests/integration/test.all_docs.js b/tests/integration/test.all_docs.js index a04431b90c..49cf89b6dc 100644 --- a/tests/integration/test.all_docs.js +++ b/tests/integration/test.all_docs.js @@ -460,6 +460,119 @@ adapters.forEach(function (adapter) { }); + it('test total_rows with a variety of criteria * 100', function (done) { + var db = new PouchDB(dbs.name); + + const docs = []; + for (let i=0; i<1000; ++i) docs.push({ _id:i.toString().padStart(5, '0') }); + + db.bulkDocs({docs}).then(function (res) { + const deletes = []; + for(let i=300; i<400; ++i) { + docs[i]._deleted = true; + docs[i]._rev = res[i].rev; + deletes.push(db.remove(docs[i])); + } + for(let i=700; i<800; ++i) { + docs[i]._deleted = true; + docs[i]._rev = res[i].rev; + deletes.push(db.remove(docs[i])); + } + return Promise.all(deletes); + }).then(function (deleted) { + deleted.should.have.length(200); + return db.allDocs(); + }).then(function (res) { + res.rows.should.have.length(800, 'correctly return rows'); + res.total_rows.should.equal(800, 'correctly return total_rows'); + return db.allDocs({startkey : '00500'}); + }).then(function (res) { + res.rows.should.have.length(400, 'correctly return rows'); + res.total_rows.should.equal(800, 'correctly return total_rows'); + return db.allDocs({startkey : '00500', skip : 200, limit : 1000}); + }).then(function (res) { + res.rows.should.have.length(200, 'correctly return rows'); + res.total_rows.should.equal(800, 'correctly return total_rows'); + return db.allDocs({startkey : '00500', limit : 0}); + }).then(function (res) { + res.rows.should.have + .length(0, 'correctly return rows, startkey w/ limit=0'); + res.total_rows.should.equal(800, 'correctly return total_rows'); + return db.allDocs({keys : ['00500'], limit : 0}); + }).then(function (res) { + res.rows.should.have + .length(0, 'correctly return rows, keys w/ limit=0'); + res.total_rows.should.equal(800, 'correctly return total_rows'); + return db.allDocs({limit : 0}); + }).then(function (res) { + res.rows.should.have.length(0, 'correctly return rows, limit=0'); + res.total_rows.should.equal(800, 'correctly return total_rows'); + return db.allDocs({startkey : '00500', descending : true, skip : 1}); + }).then(function (res) { + res.rows.should.have.length(400, 'correctly return rows'); + res.total_rows.should.equal(800, 'correctly return total_rows'); + return db.allDocs({startkey : '00500', endkey : 'z'}); + }).then(function (res) { + res.rows.should.have.length(400, 'correctly return rows'); + res.total_rows.should.equal(800, 'correctly return total_rows'); + return db.allDocs({startkey : '00500', endkey : '00500'}); + }).then(function (res) { + res.rows.should.have.length(1, 'correctly return rows'); + res.total_rows.should.equal(800, 'correctly return total_rows'); + return db.allDocs({startkey : '00500', endkey : '00400'}); + }).then(function (res) { + res.rows.should.have.length(0, 'correctly return rows'); + res.total_rows.should.equal(800, 'correctly return total_rows'); + return db.allDocs({startkey : '00599', endkey : '00400', descending : true}); + }).then(function (res) { + res.rows.should.have.length(200, 'correctly return rows'); + res.total_rows.should.equal(800, 'correctly return total_rows'); + return db.allDocs({startkey : '00300', endkey : '00799', descending : false}); + }).then(function (res) { + res.rows.should.have.length(300, 'correctly return rows'); + res.total_rows.should.equal(800, 'correctly return total_rows'); + return db.allDocs({startkey : '00799', endkey : '00300', descending : true}); + }).then(function (res) { + res.rows.should.have.length(300, 'correctly return rows'); + res.total_rows.should.equal(800, 'correctly return total_rows'); + return db.allDocs({startkey : '', endkey : '00000'}); + }).then(function (res) { + res.rows.should.have.length(1, 'correctly return rows'); + res.total_rows.should.equal(800, 'correctly return total_rows'); + return db.allDocs({keys : ['00000', '00100', '00300']}); + }).then(function (res) { + res.rows.should.have.length(3, 'correctly return rows'); + res.total_rows.should.equal(800, 'correctly return total_rows'); + return db.allDocs({keys : ['00000', '00100', '00000', '00200', '00100', '00100']}); + }).then(function (res) { + res.rows.should.have.length(6, 'correctly return rows'); + res.rows.map(function (row) { return row.key; }).should.deep.equal( + ['00000', '00100', '00000', '00200', '00100', '00100']); + res.total_rows.should.equal(800, 'correctly return total_rows'); + return db.allDocs({keys : []}); + }).then(function (res) { + res.rows.should.have.length(0, 'correctly return rows'); + res.total_rows.should.equal(800, 'correctly return total_rows'); + return db.allDocs({keys : ['00700']}); + }).then(function (res) { + res.rows.should.have.length(1, 'correctly return rows'); + res.total_rows.should.equal(800, 'correctly return total_rows'); + return db.allDocs({key : '00300'}); + }).then(function (res) { + res.rows.should.have.length(0, 'correctly return rows'); + res.total_rows.should.equal(800, 'correctly return total_rows'); + return db.allDocs({key : '00200'}); + }).then(function (res) { + res.rows.should.have.length(1, 'correctly return rows'); + res.total_rows.should.equal(800, 'correctly return total_rows'); + return db.allDocs({key : 'z'}); + }).then(function (res) { + res.rows.should.have.length(0, 'correctly return rows'); + res.total_rows.should.equal(800, 'correctly return total_rows'); + done(); + }, done); + }); + it('test total_rows with both skip and limit', function (done) { var db = new PouchDB(dbs.name); var docs = { From 32ab14aa9100e42c03cbcb2326224f44230f3190 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Mon, 25 Mar 2024 17:08:38 +0000 Subject: [PATCH 19/34] add some more to the test --- tests/integration/test.all_docs.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/integration/test.all_docs.js b/tests/integration/test.all_docs.js index 49cf89b6dc..cd8e156b4b 100644 --- a/tests/integration/test.all_docs.js +++ b/tests/integration/test.all_docs.js @@ -527,7 +527,15 @@ adapters.forEach(function (adapter) { }).then(function (res) { res.rows.should.have.length(200, 'correctly return rows'); res.total_rows.should.equal(800, 'correctly return total_rows'); + return db.allDocs({startkey:'00599', endkey:'00400', descending:true, inclusive_end:false }); + }).then(function (res) { + res.rows.should.have.length(199, 'correctly return rows'); + res.total_rows.should.equal(800, 'correctly return total_rows'); return db.allDocs({startkey : '00300', endkey : '00799', descending : false}); + }).then(function (res) { + res.rows.should.have.length(300, 'correctly return rows'); + res.total_rows.should.equal(800, 'correctly return total_rows'); + return db.allDocs({startkey:'00300', endkey:'00799', descending:false, inclusive_end:false }); }).then(function (res) { res.rows.should.have.length(300, 'correctly return rows'); res.total_rows.should.equal(800, 'correctly return total_rows'); From 9f3399c1cd860f9cba579aea7d373dbd0244036f Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Mon, 25 Mar 2024 17:14:09 +0000 Subject: [PATCH 20/34] lint --- .../pouchdb-adapter-indexeddb/src/allDocs.js | 21 +++++++++---------- tests/integration/test.all_docs.js | 8 ++++--- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/packages/node_modules/pouchdb-adapter-indexeddb/src/allDocs.js b/packages/node_modules/pouchdb-adapter-indexeddb/src/allDocs.js index f66535b44f..3e97a0fcc6 100644 --- a/packages/node_modules/pouchdb-adapter-indexeddb/src/allDocs.js +++ b/packages/node_modules/pouchdb-adapter-indexeddb/src/allDocs.js @@ -26,7 +26,7 @@ function allDocsKeys(keys, docStore, allDocsInner) { }); } -function createKeyRange(start, end, inclusiveEnd, key, descending, inclusiveStart=true) { +function createKeyRange(start, end, inclusiveStart, inclusiveEnd, key, descending) { try { if (key) { return IDBKeyRange.only([0, key]); @@ -112,7 +112,7 @@ export default function (txn, metadata, opts, callback) { return allDocsKeys(keys, docStore, allDocsInner); } - let keyRange = createKeyRange([0, start], [0, end], inclusiveEnd, key, descending); + let keyRange = createKeyRange([0, start], [0, end], true, inclusiveEnd, key, descending); if (keyRange.error) { return handleKeyRangeError(opts, metadata, keyRange.error, callback); } @@ -200,7 +200,7 @@ export default function (txn, metadata, opts, callback) { limitKey = descending ? keyRange.lower : keyRange.upper; } - keyRange = createKeyRange(firstKey, limitKey, inclusiveEnd, key, descending); + keyRange = createKeyRange(firstKey, limitKey, true, inclusiveEnd, key, descending); if (keyRange.error) { txn.txn.abort(); return handleKeyRangeError(opts, metadata, keyRange.error, callback); @@ -214,8 +214,9 @@ export default function (txn, metadata, opts, callback) { // There is a risk here with getting all results into memory - if they have multiple // revs, then we risk loading loads of extra data which is then discarded. This is // reduced by batching. This also loads unused data when include_docs is false. - - // TODO the tests which cover this will only actually test it if batch size < 5ish + // + // Current batch size is quite arbitrary, but seems like (1) more than a typical + // result size, and (2) not so big it's likely to cause issues. const batchSize = 100; fetchNextBatch(); @@ -233,15 +234,13 @@ export default function (txn, metadata, opts, callback) { } if (batch.length >= batchSize) { - const lastSeenKey = [0, batch[batch.length-1].id]; + const lastSeenKey = [ 0, batch[batch.length-1].id ]; const startKey = descending ? keyRange.upper : lastSeenKey; const endKey = descending ? lastSeenKey : keyRange.upper; if (startKey[1] !== endKey[1]) { - // TODO unclear why - const _incEnd = descending ? false : inclusiveEnd; - const _incStart = descending ? true : false; - // TODO move inclusiveStart arg to be more consistent - keyRange = createKeyRange(startKey, endKey, _incEnd, key, descending, _incStart); + const incEnd = descending ? false : inclusiveEnd; + const incStart = descending ? true : false; + keyRange = createKeyRange(startKey, endKey, incStart, incEnd, key, descending); return fetchNextBatch(); } } diff --git a/tests/integration/test.all_docs.js b/tests/integration/test.all_docs.js index cd8e156b4b..846a2b42fb 100644 --- a/tests/integration/test.all_docs.js +++ b/tests/integration/test.all_docs.js @@ -464,16 +464,18 @@ adapters.forEach(function (adapter) { var db = new PouchDB(dbs.name); const docs = []; - for (let i=0; i<1000; ++i) docs.push({ _id:i.toString().padStart(5, '0') }); + for (let i=0; i<1000; ++i) { + docs.push({ _id:i.toString().padStart(5, '0') }); + } db.bulkDocs({docs}).then(function (res) { const deletes = []; - for(let i=300; i<400; ++i) { + for (let i=300; i<400; ++i) { docs[i]._deleted = true; docs[i]._rev = res[i].rev; deletes.push(db.remove(docs[i])); } - for(let i=700; i<800; ++i) { + for (let i=700; i<800; ++i) { docs[i]._deleted = true; docs[i]._rev = res[i].rev; deletes.push(db.remove(docs[i])); From f86dd92cb7761e70f9cf0901de07fb1b8dd7a9a0 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Wed, 27 Mar 2024 11:44:36 +0000 Subject: [PATCH 21/34] getLocal(): remove use strict & attachment support --- .../pouchdb-adapter-indexeddb/src/getLocal.js | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/packages/node_modules/pouchdb-adapter-indexeddb/src/getLocal.js b/packages/node_modules/pouchdb-adapter-indexeddb/src/getLocal.js index 8973959de7..254db6daea 100644 --- a/packages/node_modules/pouchdb-adapter-indexeddb/src/getLocal.js +++ b/packages/node_modules/pouchdb-adapter-indexeddb/src/getLocal.js @@ -1,5 +1,3 @@ -'use strict'; - import { createError, MISSING_DOC } from 'pouchdb-errors'; import { META_LOCAL_STORE, processAttachment } from './util'; @@ -24,16 +22,6 @@ export default function (txn, id, api, callback) { result._id = doc.id; result._rev = doc.rev; - if (result._attachments) { - const processing = []; - for (var name in result._attachments) { - processing.push(processAttachment(name, doc, result, BINARY_ATTACHMENTS, api.blobSupport)); - } - Promise.all(processing) - .then(() => callback(null, result)) - .catch(callback); - } else { - callback(null, result); - } + callback(null, result); }; } From f8895989243f9ac33febd803d738ce2780a7f365 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Wed, 27 Mar 2024 11:49:09 +0000 Subject: [PATCH 22/34] lint --- .../node_modules/pouchdb-adapter-indexeddb/src/getLocal.js | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/node_modules/pouchdb-adapter-indexeddb/src/getLocal.js b/packages/node_modules/pouchdb-adapter-indexeddb/src/getLocal.js index 254db6daea..60afb2c68f 100644 --- a/packages/node_modules/pouchdb-adapter-indexeddb/src/getLocal.js +++ b/packages/node_modules/pouchdb-adapter-indexeddb/src/getLocal.js @@ -1,9 +1,6 @@ import { createError, MISSING_DOC } from 'pouchdb-errors'; -import { META_LOCAL_STORE, processAttachment } from './util'; - -// _getLocal() doesn't know if opts.binary is set or not, so assume it's not. -const BINARY_ATTACHMENTS = false; +import { META_LOCAL_STORE } from './util'; export default function (txn, id, api, callback) { if (txn.error) { From dd62ea2e3d73f376b3cc70509c03b42a4d4e9648 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Wed, 27 Mar 2024 16:44:00 +0000 Subject: [PATCH 23/34] Revert "getLocal(): remove use strict & attachment support" This reverts commit f86dd92cb7761e70f9cf0901de07fb1b8dd7a9a0. --- .../pouchdb-adapter-indexeddb/src/getLocal.js | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/packages/node_modules/pouchdb-adapter-indexeddb/src/getLocal.js b/packages/node_modules/pouchdb-adapter-indexeddb/src/getLocal.js index 60afb2c68f..5b6d28e1fb 100644 --- a/packages/node_modules/pouchdb-adapter-indexeddb/src/getLocal.js +++ b/packages/node_modules/pouchdb-adapter-indexeddb/src/getLocal.js @@ -1,3 +1,5 @@ +'use strict'; + import { createError, MISSING_DOC } from 'pouchdb-errors'; import { META_LOCAL_STORE } from './util'; @@ -19,6 +21,16 @@ export default function (txn, id, api, callback) { result._id = doc.id; result._rev = doc.rev; - callback(null, result); + if (result._attachments) { + const processing = []; + for (var name in result._attachments) { + processing.push(processAttachment(name, doc, result, BINARY_ATTACHMENTS, api.blobSupport)); + } + Promise.all(processing) + .then(() => callback(null, result)) + .catch(callback); + } else { + callback(null, result); + } }; } From 213d8c482b3269925ce3d59a56bb3ebbf8adf44c Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Wed, 27 Mar 2024 16:45:22 +0000 Subject: [PATCH 24/34] Revert "lint" This reverts commit f8895989243f9ac33febd803d738ce2780a7f365. --- .../node_modules/pouchdb-adapter-indexeddb/src/getLocal.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/node_modules/pouchdb-adapter-indexeddb/src/getLocal.js b/packages/node_modules/pouchdb-adapter-indexeddb/src/getLocal.js index 5b6d28e1fb..8973959de7 100644 --- a/packages/node_modules/pouchdb-adapter-indexeddb/src/getLocal.js +++ b/packages/node_modules/pouchdb-adapter-indexeddb/src/getLocal.js @@ -2,7 +2,10 @@ import { createError, MISSING_DOC } from 'pouchdb-errors'; -import { META_LOCAL_STORE } from './util'; +import { META_LOCAL_STORE, processAttachment } from './util'; + +// _getLocal() doesn't know if opts.binary is set or not, so assume it's not. +const BINARY_ATTACHMENTS = false; export default function (txn, id, api, callback) { if (txn.error) { From aed6f98be068f5f3f4557708864ce928a3dd1876 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 28 Mar 2024 05:29:42 +0000 Subject: [PATCH 25/34] remove use strict --- packages/node_modules/pouchdb-adapter-indexeddb/src/getLocal.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/node_modules/pouchdb-adapter-indexeddb/src/getLocal.js b/packages/node_modules/pouchdb-adapter-indexeddb/src/getLocal.js index 8973959de7..46c920d4c0 100644 --- a/packages/node_modules/pouchdb-adapter-indexeddb/src/getLocal.js +++ b/packages/node_modules/pouchdb-adapter-indexeddb/src/getLocal.js @@ -1,5 +1,3 @@ -'use strict'; - import { createError, MISSING_DOC } from 'pouchdb-errors'; import { META_LOCAL_STORE, processAttachment } from './util'; From 56a17f4ec7cb3ecb5e18c955af8ba822d9dae23f Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Thu, 28 Mar 2024 05:30:08 +0000 Subject: [PATCH 26/34] use const --- packages/node_modules/pouchdb-adapter-indexeddb/src/getLocal.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/node_modules/pouchdb-adapter-indexeddb/src/getLocal.js b/packages/node_modules/pouchdb-adapter-indexeddb/src/getLocal.js index 46c920d4c0..0f9f2a02cd 100644 --- a/packages/node_modules/pouchdb-adapter-indexeddb/src/getLocal.js +++ b/packages/node_modules/pouchdb-adapter-indexeddb/src/getLocal.js @@ -24,7 +24,7 @@ export default function (txn, id, api, callback) { if (result._attachments) { const processing = []; - for (var name in result._attachments) { + for (const name in result._attachments) { processing.push(processAttachment(name, doc, result, BINARY_ATTACHMENTS, api.blobSupport)); } Promise.all(processing) From f9a5ba341fbd58b490ddb5207cf320baf443c82f Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Wed, 3 Apr 2024 12:37:36 +0000 Subject: [PATCH 27/34] delete seq index; don't create other index --- .../pouchdb-adapter-indexeddb/src/setup.js | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/node_modules/pouchdb-adapter-indexeddb/src/setup.js b/packages/node_modules/pouchdb-adapter-indexeddb/src/setup.js index e53006f9d3..609792e2ad 100644 --- a/packages/node_modules/pouchdb-adapter-indexeddb/src/setup.js +++ b/packages/node_modules/pouchdb-adapter-indexeddb/src/setup.js @@ -94,7 +94,7 @@ function maintainNativeIndexes(openReq, reject) { }; } -function upgradePouchDbSchema(db, tx, pouchdbVersion) { +function upgradePouchDbSchema(dbName, db, tx, pouchdbVersion) { if (pouchdbVersion < 1) { var docStore = db.createObjectStore(DOC_STORE, {keyPath : 'id'}); docStore.createIndex('seq', 'seq', {unique: true}); @@ -103,8 +103,12 @@ function upgradePouchDbSchema(db, tx, pouchdbVersion) { } if (pouchdbVersion < 2) { - const docStore = tx.objectStore(DOC_STORE); - docStore.createIndex('_isDeleted_id', [ 'deleted', 'id' ], {unique: true}); + if (dbName.includes('-mrview-')) { + docStore.deleteIndex('seq'); + } else { + const docStore = tx.objectStore(DOC_STORE); + docStore.createIndex('_isDeleted_id', [ 'deleted', 'id' ], {unique: true}); + } } // Declare more PouchDB schema changes here @@ -134,7 +138,7 @@ function openDatabase(openDatabases, api, opts, resolve, reject) { var db = e.target.result; var pouchdbVersion = getPouchDbVersion(e.oldVersion); - upgradePouchDbSchema(db, tx, pouchdbVersion); + upgradePouchDbSchema(opts.name, db, tx, pouchdbVersion); maintainNativeIndexes(openReq, reject); if (pouchdbVersion < 2) { From cb90ef3363bda13f0bc6daef142de5470b6817d2 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Wed, 3 Apr 2024 13:04:13 +0000 Subject: [PATCH 28/34] don't delete seq - seems to be required somewhere --- packages/node_modules/pouchdb-adapter-indexeddb/src/setup.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/node_modules/pouchdb-adapter-indexeddb/src/setup.js b/packages/node_modules/pouchdb-adapter-indexeddb/src/setup.js index 609792e2ad..a0f93ee24a 100644 --- a/packages/node_modules/pouchdb-adapter-indexeddb/src/setup.js +++ b/packages/node_modules/pouchdb-adapter-indexeddb/src/setup.js @@ -104,7 +104,7 @@ function upgradePouchDbSchema(dbName, db, tx, pouchdbVersion) { if (pouchdbVersion < 2) { if (dbName.includes('-mrview-')) { - docStore.deleteIndex('seq'); + //docStore.deleteIndex('seq'); } else { const docStore = tx.objectStore(DOC_STORE); docStore.createIndex('_isDeleted_id', [ 'deleted', 'id' ], {unique: true}); From 179cc49741c4e6ac928a0e5d894cad31729b8615 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Wed, 3 Apr 2024 15:18:01 +0000 Subject: [PATCH 29/34] delete seq; maintain _isDeleted_id --- .../node_modules/pouchdb-adapter-indexeddb/src/setup.js | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/node_modules/pouchdb-adapter-indexeddb/src/setup.js b/packages/node_modules/pouchdb-adapter-indexeddb/src/setup.js index a0f93ee24a..706c1a061a 100644 --- a/packages/node_modules/pouchdb-adapter-indexeddb/src/setup.js +++ b/packages/node_modules/pouchdb-adapter-indexeddb/src/setup.js @@ -103,11 +103,10 @@ function upgradePouchDbSchema(dbName, db, tx, pouchdbVersion) { } if (pouchdbVersion < 2) { + const docStore = tx.objectStore(DOC_STORE); + docStore.createIndex('_isDeleted_id', [ 'deleted', 'id' ], {unique: true}); if (dbName.includes('-mrview-')) { - //docStore.deleteIndex('seq'); - } else { - const docStore = tx.objectStore(DOC_STORE); - docStore.createIndex('_isDeleted_id', [ 'deleted', 'id' ], {unique: true}); + docStore.deleteIndex('seq'); } } From 8c0b977d515036a685a0fe5e76816158eb5d3d86 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Fri, 5 Apr 2024 07:57:24 +0000 Subject: [PATCH 30/34] allDocs: don't recurse --- .../pouchdb-adapter-indexeddb/src/allDocs.js | 61 ++++++++++--------- 1 file changed, 33 insertions(+), 28 deletions(-) diff --git a/packages/node_modules/pouchdb-adapter-indexeddb/src/allDocs.js b/packages/node_modules/pouchdb-adapter-indexeddb/src/allDocs.js index 3e97a0fcc6..ae6b0910ef 100644 --- a/packages/node_modules/pouchdb-adapter-indexeddb/src/allDocs.js +++ b/packages/node_modules/pouchdb-adapter-indexeddb/src/allDocs.js @@ -210,7 +210,7 @@ export default function (txn, metadata, opts, callback) { }; } - function fetchResults() { + async function fetchResults() { // There is a risk here with getting all results into memory - if they have multiple // revs, then we risk loading loads of extra data which is then discarded. This is // reduced by batching. This also loads unused data when include_docs is false. @@ -219,36 +219,41 @@ export default function (txn, metadata, opts, callback) { // result size, and (2) not so big it's likely to cause issues. const batchSize = 100; - fetchNextBatch(); - - function fetchNextBatch() { - dbIndex.getAll(keyRange, batchSize).onsuccess = (e) => { - const batch = e.target.result; - for (let i=0; i { + dbIndex.getAll(kr, batchSize).onsuccess = (e) => { + const batch = e.target.result; + for (let i=0; i= batchSize) { - const lastSeenKey = [ 0, batch[batch.length-1].id ]; - const startKey = descending ? keyRange.upper : lastSeenKey; - const endKey = descending ? lastSeenKey : keyRange.upper; - if (startKey[1] !== endKey[1]) { - const incEnd = descending ? false : inclusiveEnd; - const incStart = descending ? true : false; - keyRange = createKeyRange(startKey, endKey, incStart, incEnd, key, descending); - return fetchNextBatch(); + if (batch.length >= batchSize) { + const lastSeenKey = [ 0, batch[batch.length-1].id ]; + const startKey = descending ? kr.upper : lastSeenKey; + const endKey = descending ? lastSeenKey : kr.upper; + if (startKey[1] !== endKey[1]) { + const incEnd = descending ? false : inclusiveEnd; + const incStart = descending ? true : false; + return resolve(createKeyRange(startKey, endKey, incStart, incEnd, key, descending)); + } } - } - if (descending) { - results.reverse(); - } - return txn.txn.commit(); - }; + return resolve(); + }; + }); } } } From 063c725bbe0217ab6e229ea2d42d2a0729cf6eca Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Fri, 5 Apr 2024 08:47:24 +0000 Subject: [PATCH 31/34] rename deleted,id db index --- .../node_modules/pouchdb-adapter-indexeddb/src/allDocs.js | 2 +- packages/node_modules/pouchdb-adapter-indexeddb/src/setup.js | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/node_modules/pouchdb-adapter-indexeddb/src/allDocs.js b/packages/node_modules/pouchdb-adapter-indexeddb/src/allDocs.js index ae6b0910ef..589b07664d 100644 --- a/packages/node_modules/pouchdb-adapter-indexeddb/src/allDocs.js +++ b/packages/node_modules/pouchdb-adapter-indexeddb/src/allDocs.js @@ -160,7 +160,7 @@ export default function (txn, metadata, opts, callback) { } } - const dbIndex = docStore.index('_isDeleted_id'); + const dbIndex = docStore.index('deleted,id'); if (!skip && !limit) { fetchResults(); diff --git a/packages/node_modules/pouchdb-adapter-indexeddb/src/setup.js b/packages/node_modules/pouchdb-adapter-indexeddb/src/setup.js index 706c1a061a..413b4d6e14 100644 --- a/packages/node_modules/pouchdb-adapter-indexeddb/src/setup.js +++ b/packages/node_modules/pouchdb-adapter-indexeddb/src/setup.js @@ -72,7 +72,7 @@ function maintainNativeIndexes(openReq, reject) { var expectedIndexNames = Object.keys(expectedIndexes); // Delete any indexes that aren't system indexes or expected - var systemIndexNames = ['seq','_isDeleted_id']; + var systemIndexNames = ['seq', 'deleted,id']; existingIndexNames.forEach(function (index) { if (systemIndexNames.indexOf(index) === -1 && expectedIndexNames.indexOf(index) === -1) { docStore.deleteIndex(index); @@ -104,7 +104,7 @@ function upgradePouchDbSchema(dbName, db, tx, pouchdbVersion) { if (pouchdbVersion < 2) { const docStore = tx.objectStore(DOC_STORE); - docStore.createIndex('_isDeleted_id', [ 'deleted', 'id' ], {unique: true}); + docStore.createIndex('deleted,id', [ 'deleted', 'id' ], {unique: true}); if (dbName.includes('-mrview-')) { docStore.deleteIndex('seq'); } From 569557d874d46b450016ed7ce9cf2fca8781d502 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Mon, 8 Apr 2024 11:58:37 +0000 Subject: [PATCH 32/34] new test: make more robust in CI --- tests/integration/test.all_docs.js | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/tests/integration/test.all_docs.js b/tests/integration/test.all_docs.js index 846a2b42fb..a8f3c09d53 100644 --- a/tests/integration/test.all_docs.js +++ b/tests/integration/test.all_docs.js @@ -473,16 +473,34 @@ adapters.forEach(function (adapter) { for (let i=300; i<400; ++i) { docs[i]._deleted = true; docs[i]._rev = res[i].rev; - deletes.push(db.remove(docs[i])); + deletes.push(docs[i]); } for (let i=700; i<800; ++i) { docs[i]._deleted = true; docs[i]._rev = res[i].rev; - deletes.push(db.remove(docs[i])); + deletes.push(docs[i]); } - return Promise.all(deletes); - }).then(function (deleted) { - deleted.should.have.length(200); + if (adapter === 'http') { + return testUtils.getServerType().then(serverType => { + if (serverType === 'pouchdb-express-router') { + // Workaround for https://github.com/pouchdb/pouchdb-server/issues/476 + return deletes.reduce( + (chain, doc) => chain.then(() => db.remove(doc)), + Promise.resolve(), + ); + } + return Promise.all(deletes.map(doc => db.remove(doc))) + .then(function (deleted) { + deleted.should.have.length(200); + }); + }); + } else { + return Promise.all(deletes.map(doc => db.remove(doc))) + .then(function (deleted) { + deleted.should.have.length(200); + }); + } + }).then(function () { return db.allDocs(); }).then(function (res) { res.rows.should.have.length(800, 'correctly return rows'); From 14124225ff78678c0d4854c1e69ef38e85a7d8b1 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Tue, 9 Apr 2024 07:07:15 +0000 Subject: [PATCH 33/34] Update/add tests for local doc attachments --- .../src/getAttachment.js | 7 ++ tests/integration/test.attachments.js | 80 ++++++++++++++++++- 2 files changed, 84 insertions(+), 3 deletions(-) diff --git a/packages/node_modules/pouchdb-adapter-indexeddb/src/getAttachment.js b/packages/node_modules/pouchdb-adapter-indexeddb/src/getAttachment.js index b3d07789e1..73cde6d0a2 100644 --- a/packages/node_modules/pouchdb-adapter-indexeddb/src/getAttachment.js +++ b/packages/node_modules/pouchdb-adapter-indexeddb/src/getAttachment.js @@ -1,5 +1,7 @@ 'use strict'; +import { isLocalId } from 'pouchdb-adapter-utils'; +import { createError, MISSING_DOC } from 'pouchdb-errors'; import { base64StringToBlobOrBuffer as b64StringToBluffer, btoa, @@ -7,6 +9,11 @@ import { } from 'pouchdb-binary-utils'; export default function getAttachment(docId, attachId, attachment, opts, cb) { + if (isLocalId(docId)) { + cb(createError(MISSING_DOC, 'missing')); + return; + } + const doc = opts.metadata; const data = doc.attachments[attachment.digest].data; diff --git a/tests/integration/test.attachments.js b/tests/integration/test.attachments.js index e87efbc2df..981ef6e284 100644 --- a/tests/integration/test.attachments.js +++ b/tests/integration/test.attachments.js @@ -1,5 +1,7 @@ 'use strict'; +const should = require('chai').should(); + var adapters = ['local', 'http']; var repl_adapters = [ ['local', 'http'], @@ -2050,12 +2052,84 @@ adapters.forEach(function (adapter) { } catch (caughtErr) { err = caughtErr; } + should.not.exist(res); + + if (adapter === 'local') { + err.message.should.equal('missing'); + // TODO indexeddb errors should probably have .reason set + if (db.adapter !== 'indexeddb') { + err.reason.should.equal('missing'); + } + } else if (adapter === 'http') { + const serverType = await testUtils.getServerType(); + if (serverType === 'couchdb') { + err.status.should.equal(400); + const body = await err.json(); + body.reason.should.equal('_local documents do not accept attachments.'); + } else if (serverType === 'pouchdb-express-router' || serverType === 'express-pouchdb') { + err.status.should.equal(404); + const body = await err.json(); + body.reason.should.equal('missing'); + } else { + throw new Error(`No handling for server type: '${serverType}'`); + } + } else { + throw new Error(`No handling for adapter: '${adapter}'`); + } + }); + + it('Test getAttachment for _local doc - should not return non-existent attachment', async () => { + const db = new PouchDB(dbs.name); + await db.put(binAttDocLocal); + + let res, err; + try { + res = await db.getAttachment('_local/bin_doc', 'not-real.txt'); + } catch (caughtErr) { + err = caughtErr; + } + should.not.exist(res); if (adapter === 'local') { - if (db.adapter === 'indexeddb') { - const data = await testUtils.readBlobPromise(res); - data.should.equal('This is a base64 encoded text', 'correct data'); + err.message.should.equal('missing'); + // TODO indexeddb errors should probably have .reason set + if (db.adapter !== 'indexeddb') { + err.reason.should.equal('missing'); + } + } else if (adapter === 'http') { + const serverType = await testUtils.getServerType(); + if (serverType === 'couchdb') { + err.status.should.equal(400); + const body = await err.json(); + body.reason.should.equal('_local documents do not accept attachments.'); + } else if (serverType === 'pouchdb-express-router' || serverType === 'express-pouchdb') { + err.status.should.equal(404); + const body = await err.json(); + body.reason.should.equal('missing'); } else { + throw new Error(`No handling for server type: '${serverType}'`); + } + } else { + throw new Error(`No handling for adapter: '${adapter}'`); + } + }); + + it('Test getAttachment for _local doc - should not return attachment on non-existent doc', async () => { + const db = new PouchDB(dbs.name); + await db.put(binAttDocLocal); + + let res, err; + try { + res = await db.getAttachment('_local/not_a_doc', 'not-real.txt'); + } catch (caughtErr) { + err = caughtErr; + } + should.not.exist(res); + + if (adapter === 'local') { + err.message.should.equal('missing'); + // TODO indexeddb errors should probably have .reason set + if (db.adapter !== 'indexeddb') { err.reason.should.equal('missing'); } } else if (adapter === 'http') { From d8d9f4217c05de93ced9bb7dcb7a5bcc374e5033 Mon Sep 17 00:00:00 2001 From: alxndrsn Date: Tue, 9 Apr 2024 10:30:40 +0000 Subject: [PATCH 34/34] update github issue link --- tests/integration/test.all_docs.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test.all_docs.js b/tests/integration/test.all_docs.js index a8f3c09d53..469751afa0 100644 --- a/tests/integration/test.all_docs.js +++ b/tests/integration/test.all_docs.js @@ -483,7 +483,7 @@ adapters.forEach(function (adapter) { if (adapter === 'http') { return testUtils.getServerType().then(serverType => { if (serverType === 'pouchdb-express-router') { - // Workaround for https://github.com/pouchdb/pouchdb-server/issues/476 + // Workaround for https://github.com/pouchdb/pouchdb-express-router/issues/18 return deletes.reduce( (chain, doc) => chain.then(() => db.remove(doc)), Promise.resolve(),