From 55ba2bade7dd86473e6fee7159a089823a43c4f5 Mon Sep 17 00:00:00 2001 From: m5r Date: Mon, 14 Oct 2024 19:00:13 +0200 Subject: [PATCH 01/20] build image for nouveau # Conflicts: # couchdb/Dockerfile --- couchdb-nouveau/Dockerfile | 19 +++++++++++++ couchdb-nouveau/nouveau.yaml | 27 +++++++++++++++++++ couchdb/10-docker-default.ini | 4 +++ couchdb/Dockerfile | 2 +- nginx/Dockerfile | 2 +- .../cht-couchdb-single-node.yml.template | 15 +++++++++++ scripts/build/versions.js | 2 +- 7 files changed, 68 insertions(+), 3 deletions(-) create mode 100644 couchdb-nouveau/Dockerfile create mode 100644 couchdb-nouveau/nouveau.yaml diff --git a/couchdb-nouveau/Dockerfile b/couchdb-nouveau/Dockerfile new file mode 100644 index 00000000000..14277ddee04 --- /dev/null +++ b/couchdb-nouveau/Dockerfile @@ -0,0 +1,19 @@ +FROM couchdb:3.4.1-nouveau AS base + +# temporary fix https://github.com/apache/couchdb/issues/5262 +RUN apt-get update && apt-get install -y wget unzip +RUN wget https://github.com/user-attachments/files/17186496/nouveau-1.0-SNAPSHOT-4299acf4.jar.zip -O /tmp/nouveau-4299acf4.jar.zip +RUN unzip /tmp/nouveau-4299acf4.jar.zip -d /tmp + +FROM couchdb:3.4.1-nouveau + +COPY --chown=nouveau:nouveau --from=base /tmp/nouveau-1.0-SNAPSHOT-4299acf4.jar /opt/nouveau/lib/nouveau-1.0-SNAPSHOT.jar +COPY --chown=nouveau:nouveau nouveau.yaml /opt/nouveau/etc/nouveau.yaml + +VOLUME /data/nouveau + +# 5987: Nouveau App +# 5989: Nouveau Admin +EXPOSE 5987 5989 + +LABEL Authors="MEDIC SRE TEAM" diff --git a/couchdb-nouveau/nouveau.yaml b/couchdb-nouveau/nouveau.yaml new file mode 100644 index 00000000000..351bec8c0f3 --- /dev/null +++ b/couchdb-nouveau/nouveau.yaml @@ -0,0 +1,27 @@ +maxIndexesOpen: 3000 +commitIntervalSeconds: 30 +idleSeconds: 60 +rootDir: ./data/nouveau + +logging: + level: INFO + +server: + applicationConnectors: + - type: http + bindHost: 0.0.0.0 + port: 5987 + useDateHeader: false + adminConnectors: + - type: http + bindHost: 0.0.0.0 + port: 5989 + useDateHeader: false + gzip: + includedMethods: + - GET + - POST + requestLog: + appenders: + - type: console + target: stderr diff --git a/couchdb/10-docker-default.ini b/couchdb/10-docker-default.ini index 8c924a92d54..2a976ac0b11 100644 --- a/couchdb/10-docker-default.ini +++ b/couchdb/10-docker-default.ini @@ -42,3 +42,7 @@ n=1 [attachments] compressible_types = text/*, application/javascript, application/json, application/xml compression_level = 8 + +[nouveau] +enable = true +url = http://nouveau:5987 diff --git a/couchdb/Dockerfile b/couchdb/Dockerfile index 2afba060d56..1e4f0586f0d 100644 --- a/couchdb/Dockerfile +++ b/couchdb/Dockerfile @@ -1,4 +1,4 @@ -FROM couchdb:3.4.2 as base_couchdb_build +FROM couchdb:3.4.1 AS base_couchdb_build COPY --chown=couchdb:couchdb 10-docker-default.ini /opt/couchdb/etc/default.d/ COPY --chown=couchdb:couchdb vm.args /opt/couchdb/etc/ diff --git a/nginx/Dockerfile b/nginx/Dockerfile index 1c55842a50e..289eeb0af22 100644 --- a/nginx/Dockerfile +++ b/nginx/Dockerfile @@ -1,5 +1,5 @@ # base build -FROM nginx:1.25.1-alpine as base_nginx +FROM nginx:1.25.1-alpine AS base_nginx RUN apk add --update --no-cache \ curl \ socat \ diff --git a/scripts/build/cht-couchdb-single-node.yml.template b/scripts/build/cht-couchdb-single-node.yml.template index 458b31f0e8c..921720ba968 100644 --- a/scripts/build/cht-couchdb-single-node.yml.template +++ b/scripts/build/cht-couchdb-single-node.yml.template @@ -20,6 +20,21 @@ services: networks: cht-net: + nouveau: + image: {{{ repo }}}/cht-couchdb-nouveau:{{ tag }} + volumes: + - ${COUCHDB_NOUVEAU_DATA:-./srv_nouveau}:/data/nouveau + restart: always + depends_on: + - couchdb + logging: + driver: "local" + options: + max-size: "${LOG_MAX_SIZE:-50m}" + max-file: "${LOG_MAX_FILES:-20}" + networks: + cht-net: + volumes: cht-credentials: diff --git a/scripts/build/versions.js b/scripts/build/versions.js index a1ed5be17eb..b67b8680edc 100644 --- a/scripts/build/versions.js +++ b/scripts/build/versions.js @@ -61,5 +61,5 @@ module.exports = { getRepo, escapeBranchName, SERVICES: ['api', 'sentinel'], - INFRASTRUCTURE: ['couchdb', 'haproxy', 'haproxy-healthcheck', 'nginx'], + INFRASTRUCTURE: ['couchdb', 'couchdb-nouveau', 'haproxy', 'haproxy-healthcheck', 'nginx'], }; From 653c77c4255748fb39d35d5af1145fa2446f6d02 Mon Sep 17 00:00:00 2001 From: m5r Date: Mon, 14 Oct 2024 19:10:16 +0200 Subject: [PATCH 02/20] nouveau-backed fulltext search index --- ddocs/medic-db/medic-nouveau/_id | 1 + .../nouveau/contacts_by_freetext/index.js | 47 +++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 ddocs/medic-db/medic-nouveau/_id create mode 100644 ddocs/medic-db/medic-nouveau/nouveau/contacts_by_freetext/index.js diff --git a/ddocs/medic-db/medic-nouveau/_id b/ddocs/medic-db/medic-nouveau/_id new file mode 100644 index 00000000000..3585031d0eb --- /dev/null +++ b/ddocs/medic-db/medic-nouveau/_id @@ -0,0 +1 @@ +_design/medic-nouveau diff --git a/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_freetext/index.js b/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_freetext/index.js new file mode 100644 index 00000000000..907aec546ce --- /dev/null +++ b/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_freetext/index.js @@ -0,0 +1,47 @@ +function(doc) { + const skip = [ '_id', '_rev', 'type', 'refid', 'geolocation' ]; + let toIndex = ''; + + const types = [ 'district_hospital', 'health_center', 'clinic', 'person' ]; + let idx; + if (doc.type === 'contact') { + idx = types.indexOf(doc.contact_type); + if (idx === -1) { + idx = doc.contact_type; + } + } else { + idx = types.indexOf(doc.type); + } + + const isContactDoc = idx !== -1; + if (isContactDoc) { + Object.keys(doc).forEach(function(key) { + const value = doc[key]; + if (!key || !value) { + return; + } + + key = key.toLowerCase(); + if (skip.indexOf(key) !== -1 || /_date$/.test(key)) { + return; + } + + if (typeof value === 'string') { + toIndex += ' ' + value; + } + + /*if (typeof value === 'number') { + index('double', key, value, { store: true }); + }*/ + + /*if (typeof value === 'string') { + index('text', key, value, { store: true }); + }*/ + }); + + toIndex = toIndex.trim(); + if (toIndex) { + index('text', 'default', toIndex, { store: true }); + } + } +} From 76c34b859dd9581cb4b0e90beb7345d19fa86c4e Mon Sep 17 00:00:00 2001 From: m5r Date: Thu, 7 Nov 2024 22:08:30 +0100 Subject: [PATCH 03/20] couchdb & nouveau: 3.4.1 => 3.4.2 --- couchdb-nouveau/Dockerfile | 10 +--------- couchdb/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/couchdb-nouveau/Dockerfile b/couchdb-nouveau/Dockerfile index 14277ddee04..712463369ea 100644 --- a/couchdb-nouveau/Dockerfile +++ b/couchdb-nouveau/Dockerfile @@ -1,13 +1,5 @@ -FROM couchdb:3.4.1-nouveau AS base +FROM couchdb:3.4.2-nouveau -# temporary fix https://github.com/apache/couchdb/issues/5262 -RUN apt-get update && apt-get install -y wget unzip -RUN wget https://github.com/user-attachments/files/17186496/nouveau-1.0-SNAPSHOT-4299acf4.jar.zip -O /tmp/nouveau-4299acf4.jar.zip -RUN unzip /tmp/nouveau-4299acf4.jar.zip -d /tmp - -FROM couchdb:3.4.1-nouveau - -COPY --chown=nouveau:nouveau --from=base /tmp/nouveau-1.0-SNAPSHOT-4299acf4.jar /opt/nouveau/lib/nouveau-1.0-SNAPSHOT.jar COPY --chown=nouveau:nouveau nouveau.yaml /opt/nouveau/etc/nouveau.yaml VOLUME /data/nouveau diff --git a/couchdb/Dockerfile b/couchdb/Dockerfile index 1e4f0586f0d..97435c8528f 100644 --- a/couchdb/Dockerfile +++ b/couchdb/Dockerfile @@ -1,4 +1,4 @@ -FROM couchdb:3.4.1 AS base_couchdb_build +FROM couchdb:3.4.2 AS base_couchdb_build COPY --chown=couchdb:couchdb 10-docker-default.ini /opt/couchdb/etc/default.d/ COPY --chown=couchdb:couchdb vm.args /opt/couchdb/etc/ From 6502ea1b799f490cb16d67dad3dee178a540b056 Mon Sep 17 00:00:00 2001 From: m5r Date: Thu, 7 Nov 2024 22:13:56 +0100 Subject: [PATCH 04/20] `contacts_by_type_freetext` & `reports_by_freetext` nouveau-style --- .../nouveau/contacts_by_freetext/index.js | 20 ++++---- .../contacts_by_type_freetext/index.js | 49 ++++++++++++++++++ .../nouveau/reports_by_freetext/index.js | 51 +++++++++++++++++++ 3 files changed, 111 insertions(+), 9 deletions(-) create mode 100644 ddocs/medic-db/medic-nouveau/nouveau/contacts_by_type_freetext/index.js create mode 100644 ddocs/medic-db/medic-nouveau/nouveau/reports_by_freetext/index.js diff --git a/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_freetext/index.js b/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_freetext/index.js index 907aec546ce..ca881675740 100644 --- a/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_freetext/index.js +++ b/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_freetext/index.js @@ -1,8 +1,8 @@ -function(doc) { - const skip = [ '_id', '_rev', 'type', 'refid', 'geolocation' ]; +function (doc) { + const skip = ['_id', '_rev', 'type', 'refid', 'geolocation']; let toIndex = ''; - const types = [ 'district_hospital', 'health_center', 'clinic', 'person' ]; + const types = ['district_hospital', 'health_center', 'clinic', 'person']; let idx; if (doc.type === 'contact') { idx = types.indexOf(doc.contact_type); @@ -15,7 +15,7 @@ function(doc) { const isContactDoc = idx !== -1; if (isContactDoc) { - Object.keys(doc).forEach(function(key) { + Object.keys(doc).forEach(function (key) { const value = doc[key]; if (!key || !value) { return; @@ -28,14 +28,16 @@ function(doc) { if (typeof value === 'string') { toIndex += ' ' + value; + // index('text', key, value, { store: true }); } - /*if (typeof value === 'number') { - index('double', key, value, { store: true }); - }*/ + if (typeof value === 'number') { + // index('double', key, value, { store: true }); + } - /*if (typeof value === 'string') { - index('text', key, value, { store: true }); + /*const fieldNameRegex = /^\$?[a-zA-Z][a-zA-Z0-9_]*$/g + if (fieldNameRegex.test(key)) { + console.log(`key "${key}" doesn't pass regex`); }*/ }); diff --git a/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_type_freetext/index.js b/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_type_freetext/index.js new file mode 100644 index 00000000000..0ef89e84be7 --- /dev/null +++ b/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_type_freetext/index.js @@ -0,0 +1,49 @@ +function (doc) { + var skip = ['_id', '_rev', 'type', 'refid', 'geolocation']; + let toIndex = ''; + + var types = ['district_hospital', 'health_center', 'clinic', 'person']; + var idx; + var type; + if (doc.type === 'contact') { + type = doc.contact_type; + idx = types.indexOf(type); + if (idx === -1) { + idx = type; + } + } else { + type = doc.type; + idx = types.indexOf(type); + } + if (idx !== -1) { + Object.keys(doc).forEach(function (key) { + var value = doc[key]; + if (!key || !value) { + return; + } + key = key.toLowerCase(); + if (skip.indexOf(key) !== -1 || /_date$/.test(key)) { + return; + } + + if (typeof value === 'string') { + toIndex += ' ' + value; + // index('text', key, value, { store: true }); + } + + if (typeof value === 'number') { + // index('double', key, value, { store: true }); + } + + /*const fieldNameRegex = /^\$?[a-zA-Z][a-zA-Z0-9_]*$/g + if (fieldNameRegex.test(key)) { + console.log(`key "${key}" doesn't pass regex`); + }*/ + }); + } + + toIndex = toIndex.trim(); + if (toIndex) { + index('text', 'default', toIndex, { store: true }); + } +} diff --git a/ddocs/medic-db/medic-nouveau/nouveau/reports_by_freetext/index.js b/ddocs/medic-db/medic-nouveau/nouveau/reports_by_freetext/index.js new file mode 100644 index 00000000000..ca3dd7cbe97 --- /dev/null +++ b/ddocs/medic-db/medic-nouveau/nouveau/reports_by_freetext/index.js @@ -0,0 +1,51 @@ +function (doc) { + var skip = ['_id', '_rev', 'type', 'refid', 'content']; + let toIndex = ''; + + var emitField = function (key, value) { + if (!key || !value) { + return; + } + key = key.toLowerCase(); + if (skip.indexOf(key) !== -1 || /_date$/.test(key)) { + return; + } + + if (typeof value === 'string') { + toIndex += ' ' + value; + // index('text', key, value, { store: true }); + } + + if (typeof value === 'number') { + // index('double', key, value, { store: true }); + } + + const fieldNameRegex = /^\$?[a-zA-Z][a-zA-Z0-9_]*$/g + if (fieldNameRegex.test(key)) { + console.log(`key "${key}" doesn't pass regex`); + } + }; + + if (doc.type === 'data_record' && doc.form) { + Object.keys(doc).forEach(function (key) { + emitField(key, doc[key]); + }); + if (doc.fields) { + Object.keys(doc.fields).forEach(function (key) { + emitField(key, doc.fields[key]); + }); + } + if (doc.contact && doc.contact._id) { + // index('text', 'contact', doc.contact._id.toLowerCase(), { store: true }); + /*const fieldNameRegex = /^\$?[a-zA-Z][a-zA-Z0-9_]*$/g + if (fieldNameRegex.test('contact')) { + console.log(`key "contact" doesn't pass regex`); + }*/ + } + } + + toIndex = toIndex.trim(); + if (toIndex) { + index('text', 'default', toIndex, { store: true }); + } +} From e6c0a590e5481b528f1c64d9ebc77bf7445a3359 Mon Sep 17 00:00:00 2001 From: m5r Date: Mon, 18 Nov 2024 19:46:34 +0100 Subject: [PATCH 05/20] =?UTF-8?q?remove=20freetext=20views=20=F0=9F=91=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../views/contacts_by_freetext/map.js | 52 ------------------ .../views/contacts_by_type_freetext/map.js | 54 ------------------- .../views/reports_by_freetext/map.js | 46 ---------------- 3 files changed, 152 deletions(-) delete mode 100644 ddocs/medic-db/medic-client/views/contacts_by_freetext/map.js delete mode 100644 ddocs/medic-db/medic-client/views/contacts_by_type_freetext/map.js delete mode 100644 ddocs/medic-db/medic-client/views/reports_by_freetext/map.js diff --git a/ddocs/medic-db/medic-client/views/contacts_by_freetext/map.js b/ddocs/medic-db/medic-client/views/contacts_by_freetext/map.js deleted file mode 100644 index 86e0d9cbe20..00000000000 --- a/ddocs/medic-db/medic-client/views/contacts_by_freetext/map.js +++ /dev/null @@ -1,52 +0,0 @@ -function(doc) { - var skip = [ '_id', '_rev', 'type', 'refid', 'geolocation' ]; - - var usedKeys = []; - var emitMaybe = function(key, value) { - if (usedKeys.indexOf(key) === -1 && // Not already used - key.length > 2 // Not too short - ) { - usedKeys.push(key); - emit([key], value); - } - }; - - var emitField = function(key, value, order) { - if (!key || !value) { - return; - } - key = key.toLowerCase(); - if (skip.indexOf(key) !== -1 || /_date$/.test(key)) { - return; - } - if (typeof value === 'string') { - value = value.toLowerCase(); - value.split(/\s+/).forEach(function(word) { - emitMaybe(word, order); - }); - } - if (typeof value === 'number' || typeof value === 'string') { - emitMaybe(key + ':' + value, order); - } - }; - - var types = [ 'district_hospital', 'health_center', 'clinic', 'person' ]; - var idx; - if (doc.type === 'contact') { - idx = types.indexOf(doc.contact_type); - if (idx === -1) { - idx = doc.contact_type; - } - } else { - idx = types.indexOf(doc.type); - } - - if (idx !== -1) { - var dead = !!doc.date_of_death; - var muted = !!doc.muted; - var order = dead + ' ' + muted + ' ' + idx + ' ' + (doc.name && doc.name.toLowerCase()); - Object.keys(doc).forEach(function(key) { - emitField(key, doc[key], order); - }); - } -} diff --git a/ddocs/medic-db/medic-client/views/contacts_by_type_freetext/map.js b/ddocs/medic-db/medic-client/views/contacts_by_type_freetext/map.js deleted file mode 100644 index 85015e29f9d..00000000000 --- a/ddocs/medic-db/medic-client/views/contacts_by_type_freetext/map.js +++ /dev/null @@ -1,54 +0,0 @@ -function(doc) { - var skip = [ '_id', '_rev', 'type', 'refid', 'geolocation' ]; - - var usedKeys = []; - var emitMaybe = function(type, key, value) { - if (usedKeys.indexOf(key) === -1 && // Not already used - key.length > 2 // Not too short - ) { - usedKeys.push(key); - emit([ type, key ], value); - } - }; - - var emitField = function(type, key, value, order) { - if (!key || !value) { - return; - } - key = key.toLowerCase(); - if (skip.indexOf(key) !== -1 || /_date$/.test(key)) { - return; - } - if (typeof value === 'string') { - value = value.toLowerCase(); - value.split(/\s+/).forEach(function(word) { - emitMaybe(type, word, order); - }); - } - if (typeof value === 'number' || typeof value === 'string') { - emitMaybe(type, key + ':' + value, order); - } - }; - - var types = [ 'district_hospital', 'health_center', 'clinic', 'person' ]; - var idx; - var type; - if (doc.type === 'contact') { - type = doc.contact_type; - idx = types.indexOf(type); - if (idx === -1) { - idx = type; - } - } else { - type = doc.type; - idx = types.indexOf(type); - } - if (idx !== -1) { - var dead = !!doc.date_of_death; - var muted = !!doc.muted; - var order = dead + ' ' + muted + ' ' + idx + ' ' + (doc.name && doc.name.toLowerCase()); - Object.keys(doc).forEach(function(key) { - emitField(type, key, doc[key], order); - }); - } -} diff --git a/ddocs/medic-db/medic-client/views/reports_by_freetext/map.js b/ddocs/medic-db/medic-client/views/reports_by_freetext/map.js deleted file mode 100644 index 41550efee4c..00000000000 --- a/ddocs/medic-db/medic-client/views/reports_by_freetext/map.js +++ /dev/null @@ -1,46 +0,0 @@ -function(doc) { - var skip = [ '_id', '_rev', 'type', 'refid', 'content' ]; - - var usedKeys = []; - var emitMaybe = function(key, value) { - if (usedKeys.indexOf(key) === -1 && // Not already used - key.length > 2 // Not too short - ) { - usedKeys.push(key); - emit([key], value); - } - }; - - var emitField = function(key, value, reportedDate) { - if (!key || !value) { - return; - } - key = key.toLowerCase(); - if (skip.indexOf(key) !== -1 || /_date$/.test(key)) { - return; - } - if (typeof value === 'string') { - value = value.toLowerCase(); - value.split(/\s+/).forEach(function(word) { - emitMaybe(word, reportedDate); - }); - } - if (typeof value === 'number' || typeof value === 'string') { - emitMaybe(key + ':' + value, reportedDate); - } - }; - - if (doc.type === 'data_record' && doc.form) { - Object.keys(doc).forEach(function(key) { - emitField(key, doc[key], doc.reported_date); - }); - if (doc.fields) { - Object.keys(doc.fields).forEach(function(key) { - emitField(key, doc.fields[key], doc.reported_date); - }); - } - if (doc.contact && doc.contact._id) { - emitMaybe('contact:' + doc.contact._id.toLowerCase(), doc.reported_date); - } - } -} From 950e3c56ef0e23e15ba7d38677c12b06a651a2af Mon Sep 17 00:00:00 2001 From: m5r Date: Wed, 20 Nov 2024 16:13:12 +0100 Subject: [PATCH 06/20] all 3 indexes indexing properly --- .../nouveau/contacts_by_freetext/index.js | 10 ------- .../contacts_by_type_freetext/index.js | 10 ------- .../nouveau/reports_by_freetext/index.js | 27 +++++-------------- 3 files changed, 6 insertions(+), 41 deletions(-) diff --git a/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_freetext/index.js b/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_freetext/index.js index ca881675740..d987d3c7f9e 100644 --- a/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_freetext/index.js +++ b/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_freetext/index.js @@ -28,17 +28,7 @@ function (doc) { if (typeof value === 'string') { toIndex += ' ' + value; - // index('text', key, value, { store: true }); } - - if (typeof value === 'number') { - // index('double', key, value, { store: true }); - } - - /*const fieldNameRegex = /^\$?[a-zA-Z][a-zA-Z0-9_]*$/g - if (fieldNameRegex.test(key)) { - console.log(`key "${key}" doesn't pass regex`); - }*/ }); toIndex = toIndex.trim(); diff --git a/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_type_freetext/index.js b/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_type_freetext/index.js index 0ef89e84be7..c95cfe4bd01 100644 --- a/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_type_freetext/index.js +++ b/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_type_freetext/index.js @@ -28,17 +28,7 @@ function (doc) { if (typeof value === 'string') { toIndex += ' ' + value; - // index('text', key, value, { store: true }); } - - if (typeof value === 'number') { - // index('double', key, value, { store: true }); - } - - /*const fieldNameRegex = /^\$?[a-zA-Z][a-zA-Z0-9_]*$/g - if (fieldNameRegex.test(key)) { - console.log(`key "${key}" doesn't pass regex`); - }*/ }); } diff --git a/ddocs/medic-db/medic-nouveau/nouveau/reports_by_freetext/index.js b/ddocs/medic-db/medic-nouveau/nouveau/reports_by_freetext/index.js index ca3dd7cbe97..56b810d5f4e 100644 --- a/ddocs/medic-db/medic-nouveau/nouveau/reports_by_freetext/index.js +++ b/ddocs/medic-db/medic-nouveau/nouveau/reports_by_freetext/index.js @@ -13,16 +13,6 @@ function (doc) { if (typeof value === 'string') { toIndex += ' ' + value; - // index('text', key, value, { store: true }); - } - - if (typeof value === 'number') { - // index('double', key, value, { store: true }); - } - - const fieldNameRegex = /^\$?[a-zA-Z][a-zA-Z0-9_]*$/g - if (fieldNameRegex.test(key)) { - console.log(`key "${key}" doesn't pass regex`); } }; @@ -35,17 +25,12 @@ function (doc) { emitField(key, doc.fields[key]); }); } - if (doc.contact && doc.contact._id) { - // index('text', 'contact', doc.contact._id.toLowerCase(), { store: true }); - /*const fieldNameRegex = /^\$?[a-zA-Z][a-zA-Z0-9_]*$/g - if (fieldNameRegex.test('contact')) { - console.log(`key "contact" doesn't pass regex`); - }*/ - } - } - toIndex = toIndex.trim(); - if (toIndex) { - index('text', 'default', toIndex, { store: true }); + toIndex = toIndex.trim(); + if (toIndex) { + index('text', 'default', toIndex, { store: true }); + } else { + log(`******* empty toIndex "${toIndex}" "${doc._id}"`); + } } } From b9b64fba61d1f9723fac57a1696112c1b4502386 Mon Sep 17 00:00:00 2001 From: m5r Date: Wed, 20 Nov 2024 17:03:11 +0100 Subject: [PATCH 07/20] debug problematic keys --- .../medic-nouveau/nouveau/contacts_by_freetext/index.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_freetext/index.js b/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_freetext/index.js index d987d3c7f9e..63716595d8c 100644 --- a/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_freetext/index.js +++ b/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_freetext/index.js @@ -21,6 +21,11 @@ function (doc) { return; } + const fieldNameRegex = /^\$?[a-zA-Z][a-zA-Z0-9_]*$/g + if (!fieldNameRegex.test(key)) { + log(`key "${key}" doesn't pass regex - "${doc.id}"`); + } + key = key.toLowerCase(); if (skip.indexOf(key) !== -1 || /_date$/.test(key)) { return; @@ -28,6 +33,7 @@ function (doc) { if (typeof value === 'string') { toIndex += ' ' + value; + index('text', key, value, { store: true }); } }); From 90a15995daf283421ca78833d70c1dbf3f9a3c84 Mon Sep 17 00:00:00 2001 From: m5r Date: Mon, 25 Nov 2024 20:39:47 +0100 Subject: [PATCH 08/20] debug problematic keys with numbers... --- .../medic-nouveau/nouveau/contacts_by_freetext/index.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_freetext/index.js b/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_freetext/index.js index 63716595d8c..c7fd389b409 100644 --- a/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_freetext/index.js +++ b/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_freetext/index.js @@ -35,6 +35,10 @@ function (doc) { toIndex += ' ' + value; index('text', key, value, { store: true }); } + + if (typeof value === 'number') { + index('double', key, value, { store: true }); + } }); toIndex = toIndex.trim(); From 9fa0db76624a43e2493ae8da162efb43a3108914 Mon Sep 17 00:00:00 2001 From: m5r Date: Wed, 27 Nov 2024 11:26:48 +0100 Subject: [PATCH 09/20] ok contacts_by_freetext done --- .../nouveau/contacts_by_freetext/index.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_freetext/index.js b/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_freetext/index.js index c7fd389b409..03300c60043 100644 --- a/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_freetext/index.js +++ b/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_freetext/index.js @@ -21,16 +21,16 @@ function (doc) { return; } + key = key.toLowerCase().trim(); + if (skip.indexOf(key) !== -1 || /_date$/.test(key)) { + return; + } + const fieldNameRegex = /^\$?[a-zA-Z][a-zA-Z0-9_]*$/g if (!fieldNameRegex.test(key)) { log(`key "${key}" doesn't pass regex - "${doc.id}"`); } - key = key.toLowerCase(); - if (skip.indexOf(key) !== -1 || /_date$/.test(key)) { - return; - } - if (typeof value === 'string') { toIndex += ' ' + value; index('text', key, value, { store: true }); From d507f20b976f67701f195a42e1c9618eec28634f Mon Sep 17 00:00:00 2001 From: m5r Date: Wed, 27 Nov 2024 11:43:44 +0100 Subject: [PATCH 10/20] ok contacts_by_type_freetext --- .../nouveau/contacts_by_type_freetext/index.js | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_type_freetext/index.js b/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_type_freetext/index.js index c95cfe4bd01..4d8033ed831 100644 --- a/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_type_freetext/index.js +++ b/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_type_freetext/index.js @@ -21,13 +21,24 @@ function (doc) { if (!key || !value) { return; } - key = key.toLowerCase(); + + key = key.toLowerCase().trim(); if (skip.indexOf(key) !== -1 || /_date$/.test(key)) { return; } + const fieldNameRegex = /^\$?[a-zA-Z][a-zA-Z0-9_]*$/g + if (!fieldNameRegex.test(key)) { + log(`key "${key}" doesn't pass regex - "${doc.id}"`); + } + if (typeof value === 'string') { toIndex += ' ' + value; + index('text', key, value, { store: true }); + } + + if (typeof value === 'number') { + index('double', key, value, { store: true }); } }); } From 123cb7b8c5d0d939c13bfff4468084c80083ca55 Mon Sep 17 00:00:00 2001 From: m5r Date: Wed, 27 Nov 2024 15:11:57 +0100 Subject: [PATCH 11/20] ok reports_by_freetext, excluding `_attachments` --- .../nouveau/reports_by_freetext/index.js | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/ddocs/medic-db/medic-nouveau/nouveau/reports_by_freetext/index.js b/ddocs/medic-db/medic-nouveau/nouveau/reports_by_freetext/index.js index 56b810d5f4e..99f5eb7038c 100644 --- a/ddocs/medic-db/medic-nouveau/nouveau/reports_by_freetext/index.js +++ b/ddocs/medic-db/medic-nouveau/nouveau/reports_by_freetext/index.js @@ -1,18 +1,29 @@ function (doc) { - var skip = ['_id', '_rev', 'type', 'refid', 'content']; - let toIndex = ''; + var skip = ['_id', '_rev', 'type', 'refid', 'content', '_attachments']; + var toIndex = ''; var emitField = function (key, value) { if (!key || !value) { return; } - key = key.toLowerCase(); + + key = key.toLowerCase().trim(); if (skip.indexOf(key) !== -1 || /_date$/.test(key)) { return; } + var fieldNameRegex = /^\$?[a-zA-Z][a-zA-Z0-9_]*$/g + if (!fieldNameRegex.test(key)) { + log(`key "${key}" doesn't pass regex - "${doc._id}"`); + } + if (typeof value === 'string') { toIndex += ' ' + value; + index('text', key, value, { store: true }); + } + + if (typeof value === 'number') { + index('double', key, value, { store: true }); } }; From ae8f4c8512ac63ff7f85da9d03d0fe4975b914aa Mon Sep 17 00:00:00 2001 From: m5r Date: Wed, 27 Nov 2024 15:43:10 +0100 Subject: [PATCH 12/20] get rid of debugging logs --- .../nouveau/contacts_by_freetext/index.js | 17 ++++++----------- .../contacts_by_type_freetext/index.js | 19 ++++++++----------- .../nouveau/reports_by_freetext/index.js | 7 ------- 3 files changed, 14 insertions(+), 29 deletions(-) diff --git a/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_freetext/index.js b/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_freetext/index.js index 03300c60043..a322659b626 100644 --- a/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_freetext/index.js +++ b/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_freetext/index.js @@ -1,9 +1,9 @@ function (doc) { - const skip = ['_id', '_rev', 'type', 'refid', 'geolocation']; - let toIndex = ''; + var skip = ['_id', '_rev', 'type', 'refid', 'geolocation']; + var toIndex = ''; - const types = ['district_hospital', 'health_center', 'clinic', 'person']; - let idx; + var types = ['district_hospital', 'health_center', 'clinic', 'person']; + var idx; if (doc.type === 'contact') { idx = types.indexOf(doc.contact_type); if (idx === -1) { @@ -13,10 +13,10 @@ function (doc) { idx = types.indexOf(doc.type); } - const isContactDoc = idx !== -1; + var isContactDoc = idx !== -1; if (isContactDoc) { Object.keys(doc).forEach(function (key) { - const value = doc[key]; + var value = doc[key]; if (!key || !value) { return; } @@ -26,11 +26,6 @@ function (doc) { return; } - const fieldNameRegex = /^\$?[a-zA-Z][a-zA-Z0-9_]*$/g - if (!fieldNameRegex.test(key)) { - log(`key "${key}" doesn't pass regex - "${doc.id}"`); - } - if (typeof value === 'string') { toIndex += ' ' + value; index('text', key, value, { store: true }); diff --git a/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_type_freetext/index.js b/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_type_freetext/index.js index 4d8033ed831..777dd188485 100644 --- a/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_type_freetext/index.js +++ b/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_type_freetext/index.js @@ -1,6 +1,6 @@ function (doc) { var skip = ['_id', '_rev', 'type', 'refid', 'geolocation']; - let toIndex = ''; + var toIndex = ''; var types = ['district_hospital', 'health_center', 'clinic', 'person']; var idx; @@ -15,7 +15,9 @@ function (doc) { type = doc.type; idx = types.indexOf(type); } - if (idx !== -1) { + + var isContactDoc = idx !== -1; + if (isContactDoc) { Object.keys(doc).forEach(function (key) { var value = doc[key]; if (!key || !value) { @@ -27,11 +29,6 @@ function (doc) { return; } - const fieldNameRegex = /^\$?[a-zA-Z][a-zA-Z0-9_]*$/g - if (!fieldNameRegex.test(key)) { - log(`key "${key}" doesn't pass regex - "${doc.id}"`); - } - if (typeof value === 'string') { toIndex += ' ' + value; index('text', key, value, { store: true }); @@ -41,10 +38,10 @@ function (doc) { index('double', key, value, { store: true }); } }); - } - toIndex = toIndex.trim(); - if (toIndex) { - index('text', 'default', toIndex, { store: true }); + toIndex = toIndex.trim(); + if (toIndex) { + index('text', 'default', toIndex, { store: true }); + } } } diff --git a/ddocs/medic-db/medic-nouveau/nouveau/reports_by_freetext/index.js b/ddocs/medic-db/medic-nouveau/nouveau/reports_by_freetext/index.js index 99f5eb7038c..3e901ea1592 100644 --- a/ddocs/medic-db/medic-nouveau/nouveau/reports_by_freetext/index.js +++ b/ddocs/medic-db/medic-nouveau/nouveau/reports_by_freetext/index.js @@ -12,11 +12,6 @@ function (doc) { return; } - var fieldNameRegex = /^\$?[a-zA-Z][a-zA-Z0-9_]*$/g - if (!fieldNameRegex.test(key)) { - log(`key "${key}" doesn't pass regex - "${doc._id}"`); - } - if (typeof value === 'string') { toIndex += ' ' + value; index('text', key, value, { store: true }); @@ -40,8 +35,6 @@ function (doc) { toIndex = toIndex.trim(); if (toIndex) { index('text', 'default', toIndex, { store: true }); - } else { - log(`******* empty toIndex "${toIndex}" "${doc._id}"`); } } } From 6fbb492d915f82c0d9ae7a61654515cc84756343 Mon Sep 17 00:00:00 2001 From: m5r Date: Wed, 27 Nov 2024 17:27:01 +0100 Subject: [PATCH 13/20] got ordering to work with the following request parameters: `curl -H "Content-Type: application/json" -X POST "http://localhost:5984/medic/_design/medic-nouveau/_nouveau/contacts_by_freetext" -d "{\"q\":\"name:asha\",\"sort\":\"-cht_sort_order\",\"limit\":50}"` --- .../medic-nouveau/nouveau/contacts_by_freetext/index.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_freetext/index.js b/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_freetext/index.js index a322659b626..1b30c2405ce 100644 --- a/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_freetext/index.js +++ b/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_freetext/index.js @@ -36,6 +36,11 @@ function (doc) { } }); + var dead = !!doc.date_of_death; + var muted = !!doc.muted; + var order = dead + ' ' + muted + ' ' + idx + ' ' + (doc.name && doc.name.toLowerCase()); + index('string', 'cht_sort_order', order, { store: false }); + toIndex = toIndex.trim(); if (toIndex) { index('text', 'default', toIndex, { store: true }); From ce98950b715e831ae8e17045f755ee4b27e59c30 Mon Sep 17 00:00:00 2001 From: m5r Date: Thu, 28 Nov 2024 11:54:55 +0100 Subject: [PATCH 14/20] `?q={search}+AND+cht_contact_type:district_hospital` --- .../nouveau/contacts_by_freetext/index.js | 10 ++-- .../contacts_by_type_freetext/index.js | 47 ------------------- 2 files changed, 7 insertions(+), 50 deletions(-) delete mode 100644 ddocs/medic-db/medic-nouveau/nouveau/contacts_by_type_freetext/index.js diff --git a/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_freetext/index.js b/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_freetext/index.js index 1b30c2405ce..2502f61245b 100644 --- a/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_freetext/index.js +++ b/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_freetext/index.js @@ -4,13 +4,16 @@ function (doc) { var types = ['district_hospital', 'health_center', 'clinic', 'person']; var idx; + var type; if (doc.type === 'contact') { - idx = types.indexOf(doc.contact_type); + type = doc.contact_type; + idx = types.indexOf(type); if (idx === -1) { - idx = doc.contact_type; + idx = type; } } else { - idx = types.indexOf(doc.type); + type = doc.type; + idx = types.indexOf(type); } var isContactDoc = idx !== -1; @@ -40,6 +43,7 @@ function (doc) { var muted = !!doc.muted; var order = dead + ' ' + muted + ' ' + idx + ' ' + (doc.name && doc.name.toLowerCase()); index('string', 'cht_sort_order', order, { store: false }); + index('text', 'cht_contact_type', type, { store: false }); toIndex = toIndex.trim(); if (toIndex) { diff --git a/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_type_freetext/index.js b/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_type_freetext/index.js deleted file mode 100644 index 777dd188485..00000000000 --- a/ddocs/medic-db/medic-nouveau/nouveau/contacts_by_type_freetext/index.js +++ /dev/null @@ -1,47 +0,0 @@ -function (doc) { - var skip = ['_id', '_rev', 'type', 'refid', 'geolocation']; - var toIndex = ''; - - var types = ['district_hospital', 'health_center', 'clinic', 'person']; - var idx; - var type; - if (doc.type === 'contact') { - type = doc.contact_type; - idx = types.indexOf(type); - if (idx === -1) { - idx = type; - } - } else { - type = doc.type; - idx = types.indexOf(type); - } - - var isContactDoc = idx !== -1; - if (isContactDoc) { - Object.keys(doc).forEach(function (key) { - var value = doc[key]; - if (!key || !value) { - return; - } - - key = key.toLowerCase().trim(); - if (skip.indexOf(key) !== -1 || /_date$/.test(key)) { - return; - } - - if (typeof value === 'string') { - toIndex += ' ' + value; - index('text', key, value, { store: true }); - } - - if (typeof value === 'number') { - index('double', key, value, { store: true }); - } - }); - - toIndex = toIndex.trim(); - if (toIndex) { - index('text', 'default', toIndex, { store: true }); - } - } -} From f333b28877fa436061ab472dccb8e54ec7ce3fcf Mon Sep 17 00:00:00 2001 From: m5r Date: Mon, 2 Dec 2024 17:17:55 +0100 Subject: [PATCH 15/20] patch eslint couchdb plugin --- patches/eslint-plugin-couchdb+0.2.0.patch | 27 +++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 patches/eslint-plugin-couchdb+0.2.0.patch diff --git a/patches/eslint-plugin-couchdb+0.2.0.patch b/patches/eslint-plugin-couchdb+0.2.0.patch new file mode 100644 index 00000000000..03488adbb4c --- /dev/null +++ b/patches/eslint-plugin-couchdb+0.2.0.patch @@ -0,0 +1,27 @@ +diff --git a/node_modules/eslint-plugin-couchdb/lib/index.js b/node_modules/eslint-plugin-couchdb/lib/index.js +index 3399a8f..418f58b 100644 +--- a/node_modules/eslint-plugin-couchdb/lib/index.js ++++ b/node_modules/eslint-plugin-couchdb/lib/index.js +@@ -18,12 +18,14 @@ var front = function(functions, jsonObject) { + } + + // https://docs.couchdb.org/en/stable/query-server/javascript.html# ++var indexFront = front(['index', 'isArray', 'log', 'require', 'sum', 'toJSON'], true); + var mapFront = front(['emit', 'isArray', 'log', 'require', 'sum', 'toJSON'], true); + var reduceFront = front(['isArray', 'log', 'sum', 'toJSON'], true); + var vdoFront = front(['isArray', 'log', 'require', 'sum', 'toJSON'], true); + var showFront = front(['isArray', 'log', 'provides', 'registerType', 'require', 'sum', 'toJSON'], true); + var listFront = front(['getRow', 'isArray', 'log', 'provides', 'registerType', 'require', 'send', 'start', 'sum', 'toJSON'], true); + ++var indexTest = /nouveau\/[^\/]+\/index\.js$/; + var mapTest = /views\/[^\/]+\/map\.js$/; + var reduceTest = /views\/[^\/]+\/reduce\.js$/; + var vdoTest = /validate_doc_update.js$/; +@@ -31,6 +33,7 @@ var showTest = /shows\/.*.js$/; + var listTest = /lists\/.*.js$/; + + var FN_TYPES = [ ++ {test: indexTest, front: indexFront}, + {test: mapTest, front: mapFront}, + {test: reduceTest, front: reduceFront}, + {test: vdoTest, front: vdoFront}, From 8d19d322d084da737e39c288e7ea235302e9fd9f Mon Sep 17 00:00:00 2001 From: m5r Date: Mon, 2 Dec 2024 17:39:00 +0100 Subject: [PATCH 16/20] skip freetext views unit tests --- webapp/tests/mocha/unit/views/contacts_by_freetext.spec.js | 2 +- webapp/tests/mocha/unit/views/contacts_by_type_freetext.spec.js | 2 +- webapp/tests/mocha/unit/views/reports_by_freetext.spec.js | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/webapp/tests/mocha/unit/views/contacts_by_freetext.spec.js b/webapp/tests/mocha/unit/views/contacts_by_freetext.spec.js index 6d5d5e3f1eb..9aba6f9b506 100644 --- a/webapp/tests/mocha/unit/views/contacts_by_freetext.spec.js +++ b/webapp/tests/mocha/unit/views/contacts_by_freetext.spec.js @@ -36,7 +36,7 @@ const nonAsciiDoc = { reported_date: 1496068842996 }; -describe('contacts_by_freetext view', () => { +describe.skip('contacts_by_freetext view', () => { it('indexes doc name', () => { // given diff --git a/webapp/tests/mocha/unit/views/contacts_by_type_freetext.spec.js b/webapp/tests/mocha/unit/views/contacts_by_type_freetext.spec.js index 4384080ba94..977d933c278 100644 --- a/webapp/tests/mocha/unit/views/contacts_by_type_freetext.spec.js +++ b/webapp/tests/mocha/unit/views/contacts_by_type_freetext.spec.js @@ -53,7 +53,7 @@ const configurableHierarchyDoc = { let map; -describe('contacts_by_type_freetext view', () => { +describe.skip('contacts_by_type_freetext view', () => { beforeEach(() => map = utils.loadView('medic-db', 'medic-client', 'contacts_by_type_freetext')); diff --git a/webapp/tests/mocha/unit/views/reports_by_freetext.spec.js b/webapp/tests/mocha/unit/views/reports_by_freetext.spec.js index 49cdaa1b0fc..8c93f02c26f 100644 --- a/webapp/tests/mocha/unit/views/reports_by_freetext.spec.js +++ b/webapp/tests/mocha/unit/views/reports_by_freetext.spec.js @@ -112,7 +112,7 @@ const doc = { ] }; -describe('reports_by_freetext view', () => { +describe.skip('reports_by_freetext view', () => { it('indexes doc name', () => { // given From 31d274deffe28fb6750822b9b8fdcafbabc4e165 Mon Sep 17 00:00:00 2001 From: m5r Date: Mon, 2 Dec 2024 17:50:46 +0100 Subject: [PATCH 17/20] fix monitoring unit test --- api/tests/mocha/services/monitoring.spec.js | 1 + 1 file changed, 1 insertion(+) diff --git a/api/tests/mocha/services/monitoring.spec.js b/api/tests/mocha/services/monitoring.spec.js index 9b35e821d79..efc9f38fa0c 100644 --- a/api/tests/mocha/services/monitoring.spec.js +++ b/api/tests/mocha/services/monitoring.spec.js @@ -69,6 +69,7 @@ const VIEW_INDEXES_BY_DB = { 'medic-admin', 'medic-client', 'medic-conflicts', + 'medic-nouveau', 'medic-scripts', 'medic-sms', ], From 0d4862538b08d2021b0a7ab86d05cf51fb358127 Mon Sep 17 00:00:00 2001 From: m5r Date: Mon, 2 Dec 2024 18:12:58 +0100 Subject: [PATCH 18/20] fix monitoring unit test --- api/tests/mocha/services/monitoring.spec.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/tests/mocha/services/monitoring.spec.js b/api/tests/mocha/services/monitoring.spec.js index efc9f38fa0c..31c369e02ab 100644 --- a/api/tests/mocha/services/monitoring.spec.js +++ b/api/tests/mocha/services/monitoring.spec.js @@ -69,7 +69,6 @@ const VIEW_INDEXES_BY_DB = { 'medic-admin', 'medic-client', 'medic-conflicts', - 'medic-nouveau', 'medic-scripts', 'medic-sms', ], @@ -251,6 +250,7 @@ const getExpectedViewIndexes = (dbName) => { const getCurrentDdocNames = (db) => getBundledDdocs(db) .then(ddocs => ddocs + .filter(ddoc => !!ddoc.views) .map(ddoc => ddoc._id) .map(ddocId => ddocId.split('/')[1])); From a713e34afd41aacc1af1765a95609ed22b216d99 Mon Sep 17 00:00:00 2001 From: m5r Date: Thu, 12 Dec 2024 16:03:01 +0100 Subject: [PATCH 19/20] add nouveau service to compose file for couchdb cluster --- scripts/build/cht-couchdb-cluster.yml.template | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/scripts/build/cht-couchdb-cluster.yml.template b/scripts/build/cht-couchdb-cluster.yml.template index 0cb08e22b68..48af139660a 100644 --- a/scripts/build/cht-couchdb-cluster.yml.template +++ b/scripts/build/cht-couchdb-cluster.yml.template @@ -63,6 +63,23 @@ services: networks: cht-net: + nouveau: + image: {{{ repo }}}/cht-couchdb-nouveau:{{ tag }} + volumes: + - ${COUCHDB_NOUVEAU_DATA:-./srv_nouveau}:/data/nouveau + restart: always + depends_on: + - couchdb-1.local + - couchdb-2.local + - couchdb-3.local + logging: + driver: "local" + options: + max-size: "${LOG_MAX_SIZE:-50m}" + max-file: "${LOG_MAX_FILES:-20}" + networks: + cht-net: + volumes: cht-credentials: From db53828bb59759802e3d2408bd4198c226312046 Mon Sep 17 00:00:00 2001 From: Mokhtar Date: Mon, 16 Dec 2024 09:45:32 +0100 Subject: [PATCH 20/20] feat(#9690): monitor nouveau (#9700) --- api/src/services/monitoring.js | 50 +++++- api/tests/mocha/services/monitoring.spec.js | 143 +++++++++++++++--- .../api/controllers/monitoring.spec.js | 27 +++- tests/utils/index.js | 1 + 4 files changed, 195 insertions(+), 26 deletions(-) diff --git a/api/src/services/monitoring.js b/api/src/services/monitoring.js index 462b2d9d524..5d69c9d41d8 100644 --- a/api/src/services/monitoring.js +++ b/api/src/services/monitoring.js @@ -27,6 +27,15 @@ const VIEW_INDEXES_TO_MONITOR = { users: ['users'], }; +const NOUVEAU_INDEXES_TO_MONITOR = { + medic: { + 'medic-nouveau': [ + 'contacts_by_freetext', + 'reports_by_freetext', + ], + }, +}; + const MESSAGE_QUEUE_STATUS_KEYS = ['due', 'scheduled', 'muted', 'failed', 'delivered']; const fromEntries = (keys, value) => { // "shim" of Object.fromEntries @@ -115,7 +124,7 @@ const getFragmentation = ({ sizes }, viewIndexInfos) => { return totalFile / totalActive; }; -const mapDbInfo = (dbInfo, viewIndexInfos) => { +const mapDbInfo = (dbInfo, viewIndexInfos, nouveauIndexInfos) => { return { name: dbInfo.db_name || '', update_sequence: getSequenceNumber(dbInfo.update_seq), @@ -131,8 +140,13 @@ const mapDbInfo = (dbInfo, viewIndexInfos) => { sizes: { active: defaultNumber(viewIndexInfo.view_index?.sizes?.active), file: defaultNumber(viewIndexInfo.view_index?.sizes?.file), - } - })) + }, + })), + nouveau_indexes: nouveauIndexInfos?.map(nouveauIndexInfo => ({ + name: nouveauIndexInfo.name || '', + num_docs: defaultNumber(nouveauIndexInfo.search_index.num_docs), + disk_size: defaultNumber(nouveauIndexInfo.search_index.disk_size), + })), }; }; @@ -167,11 +181,37 @@ const fetchViewIndexInfosForDb = (db) => Promise.all( const fetchAllViewIndexInfos = () => Promise.all(Object.keys(VIEW_INDEXES_TO_MONITOR).map(fetchViewIndexInfosForDb)); +const fetchNouveauIndexInfo = (db, designDoc, indexName) => request + .get({ + url: `${environment.serverUrl}/${db}/_design/${designDoc}/_nouveau_info/${indexName}`, + json: true + }) + .catch(err => { + logger.error('Error fetching nouveau index info: %o', err); + return null; + }); + +const fetchNouveauIndexInfosForDdoc = (db, ddoc) => NOUVEAU_INDEXES_TO_MONITOR[db][ddoc].map( + indexName => fetchNouveauIndexInfo(DBS_TO_MONITOR[db], ddoc, indexName), +); + +const fetchNouveauIndexInfosForDb = (db) => Promise.all(Object.keys(NOUVEAU_INDEXES_TO_MONITOR[db]).flatMap( + ddoc => fetchNouveauIndexInfosForDdoc(db, ddoc), +)).then((nouveauIndexInfos) => nouveauIndexInfos.filter(info => info)); + +const fetchAllNouveauIndexInfos = () => Promise.all( + Object.keys(NOUVEAU_INDEXES_TO_MONITOR).map(fetchNouveauIndexInfosForDb), +); + const getDbInfos = async () => { - const [dbInfos, viewIndexInfos] = await Promise.all([fetchDbsInfo(), fetchAllViewIndexInfos()]); + const [dbInfos, viewIndexInfos, nouveauIndexInfos] = await Promise.all([ + fetchDbsInfo(), + fetchAllViewIndexInfos(), + fetchAllNouveauIndexInfos(), + ]); const result = {}; Object.keys(DBS_TO_MONITOR).forEach((dbKey, i) => { - result[dbKey] = mapDbInfo(dbInfos[i], viewIndexInfos[i]); + result[dbKey] = mapDbInfo(dbInfos[i], viewIndexInfos[i], nouveauIndexInfos[i]); }); return result; }; diff --git a/api/tests/mocha/services/monitoring.spec.js b/api/tests/mocha/services/monitoring.spec.js index 31c369e02ab..92fef3f45cd 100644 --- a/api/tests/mocha/services/monitoring.spec.js +++ b/api/tests/mocha/services/monitoring.spec.js @@ -161,6 +161,35 @@ const VIEW_INDEX_INFO_BY_DESIGN = { } }; +const NOUVEAU_DDOCS_BY_DB = { + [environment.db]: ['medic-nouveau'], +}; + +const NOUVEAU_INDEX_INFO_BY_DDOC = { + 'medic-nouveau': { + reports_by_freetext: { + name: '_design/medic-nouveau/reports_by_freetext', + search_index: { + update_seq: 1956891, + purge_seq: 0, + num_docs: 183741, + disk_size: 157258510, + signature: 'cfd67cbb4800308021b6547bcf21cbf99b9476186b5251f317b221225714c5d3', + }, + }, + contacts_by_freetext: { + name: '_design/medic-nouveau/contacts_by_freetext', + search_index: { + update_seq: 1956891, + purge_seq: 0, + num_docs: 207734, + disk_size: 76815351, + signature: '46de1dfc576838494f798264571dc59658db7ea164915dd459a7752c31591ae6', + }, + }, + }, +}; + const setUpMocks = () => { sinon.stub(deployInfo, 'get').resolves({ version: '5.3.2' }); sinon.stub(request, 'get') @@ -173,6 +202,16 @@ const setUpMocks = () => { .resolves(VIEW_INDEX_INFO_BY_DESIGN[designDoc]); }); }); + Object.keys(NOUVEAU_DDOCS_BY_DB).forEach(dbName => { + NOUVEAU_DDOCS_BY_DB[dbName].forEach(designDoc => { + Object.keys(NOUVEAU_INDEX_INFO_BY_DDOC[designDoc]).forEach(indexName => { + request.get + .withArgs( + sinon.match({ url: `${environment.serverUrl}/${dbName}/_design/${designDoc}/_nouveau_info/${indexName}` }), + ).resolves(NOUVEAU_INDEX_INFO_BY_DDOC[designDoc][indexName]); + }); + }); + }); sinon.stub(request, 'post').withArgs(sinon.match({ url: `${environment.serverUrl}/_dbs_info` })) .resolves(dbInfos); sinon.stub(db.sentinel, 'get').withArgs('_local/transitions-seq') @@ -284,9 +323,21 @@ describe('Monitoring service', () => { update_sequence: 100, sizes: { active: 600, - file: 700 + file: 700, }, - view_indexes: getExpectedViewIndexes(environment.db) + view_indexes: getExpectedViewIndexes(environment.db), + nouveau_indexes: [ + { + disk_size: 76815351, + name: '_design/medic-nouveau/contacts_by_freetext', + num_docs: 207734, + }, + { + disk_size: 157258510, + name: '_design/medic-nouveau/reports_by_freetext', + num_docs: 183741, + }, + ], }, sentinel: { doc_count: 30, @@ -298,7 +349,8 @@ describe('Monitoring service', () => { active: 500, file: 500 }, - view_indexes: getExpectedViewIndexes(`${environment.db}-sentinel`) + view_indexes: getExpectedViewIndexes(`${environment.db}-sentinel`), + nouveau_indexes: undefined, }, users: { doc_count: 50, @@ -310,7 +362,8 @@ describe('Monitoring service', () => { active: 500, file: 501 }, - view_indexes: getExpectedViewIndexes('_users') + view_indexes: getExpectedViewIndexes('_users'), + nouveau_indexes: undefined, }, usersmeta: { doc_count: 40, @@ -322,7 +375,8 @@ describe('Monitoring service', () => { active: 500, file: 5000 }, - view_indexes: getExpectedViewIndexes(`${environment.db}-users-meta`) + view_indexes: getExpectedViewIndexes(`${environment.db}-users-meta`), + nouveau_indexes: undefined, } }); chai.expect(actual.messaging).to.deep.equal({ @@ -348,6 +402,14 @@ describe('Monitoring service', () => { [{ json: true, url: `${environment.serverUrl}/${environment.db}/_design/medic-admin/_info` }], [{ json: true, url: `${environment.serverUrl}/${environment.db}/_design/medic-client/_info` }], [{ json: true, url: `${environment.serverUrl}/${environment.db}/_design/medic-conflicts/_info` }], + [{ + json: true, + url: `${environment.serverUrl}/${environment.db}/_design/medic-nouveau/_nouveau_info/contacts_by_freetext`, + }], + [{ + json: true, + url: `${environment.serverUrl}/${environment.db}/_design/medic-nouveau/_nouveau_info/reports_by_freetext`, + }], [{ json: true, url: `${environment.serverUrl}/${environment.db}/_design/medic-scripts/_info` }], [{ json: true, url: `${environment.serverUrl}/${environment.db}/_design/medic-sms/_info` }], [{ json: true, url: `${environment.serverUrl}/${environment.db}-sentinel/_design/sentinel/_info` }], @@ -395,7 +457,19 @@ describe('Monitoring service', () => { active: 600, file: 700 }, - view_indexes: getExpectedViewIndexes(environment.db) + view_indexes: getExpectedViewIndexes(environment.db), + nouveau_indexes: [ + { + disk_size: 76815351, + name: '_design/medic-nouveau/contacts_by_freetext', + num_docs: 207734, + }, + { + disk_size: 157258510, + name: '_design/medic-nouveau/reports_by_freetext', + num_docs: 183741, + }, + ], }, sentinel: { doc_count: 30, @@ -407,7 +481,8 @@ describe('Monitoring service', () => { active: 500, file: 500 }, - view_indexes: getExpectedViewIndexes(`${environment.db}-sentinel`) + view_indexes: getExpectedViewIndexes(`${environment.db}-sentinel`), + nouveau_indexes: undefined, }, users: { doc_count: 50, @@ -419,7 +494,8 @@ describe('Monitoring service', () => { active: 500, file: 501 }, - view_indexes: getExpectedViewIndexes('_users') + view_indexes: getExpectedViewIndexes('_users'), + nouveau_indexes: undefined, }, usersmeta: { doc_count: 40, @@ -431,7 +507,8 @@ describe('Monitoring service', () => { active: 500, file: 5000 }, - view_indexes: getExpectedViewIndexes(`${environment.db}-users-meta`) + view_indexes: getExpectedViewIndexes(`${environment.db}-users-meta`), + nouveau_indexes: undefined, } }); chai.expect(actual.messaging).to.deep.equal({ @@ -484,6 +561,14 @@ describe('Monitoring service', () => { [{ json: true, url: `${environment.serverUrl}/${environment.db}/_design/medic-admin/_info` }], [{ json: true, url: `${environment.serverUrl}/${environment.db}/_design/medic-client/_info` }], [{ json: true, url: `${environment.serverUrl}/${environment.db}/_design/medic-conflicts/_info` }], + [{ + json: true, + url: `${environment.serverUrl}/${environment.db}/_design/medic-nouveau/_nouveau_info/contacts_by_freetext`, + }], + [{ + json: true, + url: `${environment.serverUrl}/${environment.db}/_design/medic-nouveau/_nouveau_info/reports_by_freetext`, + }], [{ json: true, url: `${environment.serverUrl}/${environment.db}/_design/medic-scripts/_info` }], [{ json: true, url: `${environment.serverUrl}/${environment.db}/_design/medic-sms/_info` }], [{ json: true, url: `${environment.serverUrl}/${environment.db}-sentinel/_design/sentinel/_info` }], @@ -532,7 +617,8 @@ describe('Monitoring service', () => { active: -1, file: -1 }, - view_indexes: [] + view_indexes: [], + nouveau_indexes: [], }, sentinel: { doc_count: -1, @@ -544,7 +630,8 @@ describe('Monitoring service', () => { active: -1, file: -1 }, - view_indexes: [] + view_indexes: [], + nouveau_indexes: undefined, }, users: { doc_count: -1, @@ -556,7 +643,8 @@ describe('Monitoring service', () => { active: -1, file: -1 }, - view_indexes: [] + view_indexes: [], + nouveau_indexes: undefined, }, usersmeta: { doc_count: -1, @@ -568,7 +656,8 @@ describe('Monitoring service', () => { active: -1, file: -1 }, - view_indexes: [] + view_indexes: [], + nouveau_indexes: undefined, } }); chai.expect(actual.messaging).to.deep.equal({ @@ -592,6 +681,14 @@ describe('Monitoring service', () => { [{ json: true, url: `${environment.serverUrl}/${environment.db}/_design/medic-admin/_info` }], [{ json: true, url: `${environment.serverUrl}/${environment.db}/_design/medic-client/_info` }], [{ json: true, url: `${environment.serverUrl}/${environment.db}/_design/medic-conflicts/_info` }], + [{ + json: true, + url: `${environment.serverUrl}/${environment.db}/_design/medic-nouveau/_nouveau_info/contacts_by_freetext`, + }], + [{ + json: true, + url: `${environment.serverUrl}/${environment.db}/_design/medic-nouveau/_nouveau_info/reports_by_freetext`, + }], [{ json: true, url: `${environment.serverUrl}/${environment.db}/_design/medic-scripts/_info` }], [{ json: true, url: `${environment.serverUrl}/${environment.db}/_design/medic-sms/_info` }], [{ json: true, url: `${environment.serverUrl}/${environment.db}-sentinel/_design/sentinel/_info` }], @@ -628,7 +725,8 @@ describe('Monitoring service', () => { active: -1, file: -1 }, - view_indexes: [] + view_indexes: [], + nouveau_indexes: [], }, sentinel: { doc_count: -1, @@ -640,7 +738,8 @@ describe('Monitoring service', () => { active: -1, file: -1 }, - view_indexes: [] + view_indexes: [], + nouveau_indexes: undefined, }, users: { doc_count: -1, @@ -652,7 +751,8 @@ describe('Monitoring service', () => { active: -1, file: -1 }, - view_indexes: [] + view_indexes: [], + nouveau_indexes: undefined, }, usersmeta: { doc_count: -1, @@ -664,7 +764,8 @@ describe('Monitoring service', () => { active: -1, file: -1 }, - view_indexes: [] + view_indexes: [], + nouveau_indexes: undefined, } }); chai.expect(actual.messaging).to.deep.equal({ @@ -715,6 +816,14 @@ describe('Monitoring service', () => { [{ json: true, url: `${environment.serverUrl}/${environment.db}/_design/medic-admin/_info` }], [{ json: true, url: `${environment.serverUrl}/${environment.db}/_design/medic-client/_info` }], [{ json: true, url: `${environment.serverUrl}/${environment.db}/_design/medic-conflicts/_info` }], + [{ + json: true, + url: `${environment.serverUrl}/${environment.db}/_design/medic-nouveau/_nouveau_info/contacts_by_freetext`, + }], + [{ + json: true, + url: `${environment.serverUrl}/${environment.db}/_design/medic-nouveau/_nouveau_info/reports_by_freetext`, + }], [{ json: true, url: `${environment.serverUrl}/${environment.db}/_design/medic-scripts/_info` }], [{ json: true, url: `${environment.serverUrl}/${environment.db}/_design/medic-sms/_info` }], [{ json: true, url: `${environment.serverUrl}/${environment.db}-sentinel/_design/sentinel/_info` }], diff --git a/tests/integration/api/controllers/monitoring.spec.js b/tests/integration/api/controllers/monitoring.spec.js index 8fc24483f09..01bfd1a2af8 100644 --- a/tests/integration/api/controllers/monitoring.spec.js +++ b/tests/integration/api/controllers/monitoring.spec.js @@ -5,7 +5,7 @@ const utils = require('@utils'); const sentinelUtils = require('@utils/sentinel'); const VIEW_INDEXES_BY_DB = { - ['medic-test']: [ + 'medic-test': [ 'medic', 'medic-admin', 'medic-client', @@ -13,11 +13,17 @@ const VIEW_INDEXES_BY_DB = { 'medic-scripts', 'medic-sms', ], - ['medic-test-sentinel']: ['sentinel'], - ['medic-test-users-meta']: ['users-meta'], + 'medic-test-sentinel': ['sentinel'], + 'medic-test-users-meta': ['users-meta'], _users: ['users'], }; +const NOUVEAU_INDEXES_BY_DB = { + ['medic-test']: { + 'medic-nouveau': ['contacts_by_freetext', 'reports_by_freetext'], + }, +}; + const getAppVersion = async () => { const deployInfo = await utils.request({ path: '/api/deploy-info' }); return deployInfo.version; @@ -37,7 +43,18 @@ const getExpectedViewIndexes = (db) => { })); }; -const INDETERMINATE_FIELDS = ['current', 'uptime', 'date', 'fragmentation', 'node', 'sizes']; +const getExpectedNouveauIndexes = (db) => { + if (!NOUVEAU_INDEXES_BY_DB[db]) { + return; + } + + const ddocs = Object.entries(NOUVEAU_INDEXES_BY_DB[db]); + return ddocs.flatMap(([ddocName, indexes]) => indexes.map(indexName => ({ + name: `_design/${ddocName}/${indexName}`, + }))); +}; + +const INDETERMINATE_FIELDS = ['current', 'uptime', 'date', 'fragmentation', 'node', 'sizes', 'disk_size', 'num_docs']; const assertCouchDbDataSizeFields = (couchData) => { chai.expect(couchData.fragmentation).to.be.gte(0); @@ -86,6 +103,7 @@ describe('monitoring', () => { doc_count: medicInfo.doc_count, doc_del_count: medicInfo.doc_del_count, view_indexes: getExpectedViewIndexes('medic-test'), + nouveau_indexes: getExpectedNouveauIndexes('medic-test'), }, sentinel: { name: 'medic-test-sentinel', @@ -164,6 +182,7 @@ describe('monitoring', () => { doc_count: medicInfo.doc_count, doc_del_count: medicInfo.doc_del_count, view_indexes: getExpectedViewIndexes('medic-test'), + nouveau_indexes: getExpectedNouveauIndexes('medic-test'), }, sentinel: { name: 'medic-test-sentinel', diff --git a/tests/utils/index.js b/tests/utils/index.js index 89e1c5bda1c..250ae826f0e 100644 --- a/tests/utils/index.js +++ b/tests/utils/index.js @@ -1163,6 +1163,7 @@ const startServices = async () => { env.DB1_DATA = makeTempDir('ci-dbdata'); env.DB2_DATA = makeTempDir('ci-dbdata'); env.DB3_DATA = makeTempDir('ci-dbdata'); + env.COUCHDB_NOUVEAU_DATA = makeTempDir('ci-nouveaudata'); await dockerComposeCmd('up -d'); const services = await dockerComposeCmd('ps -q');