From edfc33425aecb2077ac831f3a2cba6c08c805651 Mon Sep 17 00:00:00 2001 From: Joseph Scheuhammer Date: Fri, 29 Jun 2018 16:18:29 -0400 Subject: [PATCH 01/32] GPII-3138: Modified snapset loading to remove existing snapsets, then reload them. First checkin: Replaced loadData.sh with deleteAndLoadSnapsets.js. This is a partial reimplementation of the load script and currently does only: 1. Find the current snapset Prefs Safes in the data base. 2. Find the Prefs Safes associated GPII Keys 3. Delete the above records from the data base Still to come: loading the current snapsets as before (and polishing of code). --- loadData.sh => deleteAndLoadSnapSets.sh | 0 deleteAndLoadSnapsets.js | 148 ++++++++++++++++++++++++ httpTests.js | 138 ++++++++++++++++++++++ 3 files changed, 286 insertions(+) rename loadData.sh => deleteAndLoadSnapSets.sh (100%) create mode 100644 deleteAndLoadSnapsets.js create mode 100644 httpTests.js diff --git a/loadData.sh b/deleteAndLoadSnapSets.sh similarity index 100% rename from loadData.sh rename to deleteAndLoadSnapSets.sh diff --git a/deleteAndLoadSnapsets.js b/deleteAndLoadSnapsets.js new file mode 100644 index 0000000..02ec826 --- /dev/null +++ b/deleteAndLoadSnapsets.js @@ -0,0 +1,148 @@ +/*! +Copyright 2018 OCAD University + +Licensed under the New BSD license. You may not use this file except in +compliance with this License. + +You may obtain a copy of the License at +https://github.com/GPII/universal/blob/master/LICENSE.txt +*/ + +// This script modifies the preferences data base: +// 1. Finds all the Prefs Safes of type "snapset" (prefsSafesType = "snapset"), +// 2. Finds all the GPII Keys associated with each snapset Prefs Safe +// 3. Deletes the found Prefs Safes and associated GPII Keys +// 4. Loads the latest Prefs Safes and GPII Keys from %universal/build/dbData +// 5. Loads the latest credentials, clients, and views from %universal/testData/dbData +// +// A sample command that runs this script: +// node deleteAndLoadSnapsets.js COUCHDBURL + +"use strict"; + +var http = require('http'), + fluid = require("../universal/node_modules/infusion"); + +var gpii = fluid.registerNamespace("gpii"); +fluid.registerNamespace("gpii.dataLoader"); + +var dbLoader = gpii.dataLoader; + +debugger; +dbLoader.couchDBURL = process.argv[2]; +if (gpii.dataLoader.couchDBURL === undefined) { + console.log ("COUCHDB_URL environment variable must be defined"); + process.exit(1); +} +console.log("COUCHDB_URL: '" + dbLoader.couchDBURL + "'"); +dbLoader.prefsSafesViewUrl = dbLoader.couchDBURL + "/_design/views/_view/findSnapsetPrefsSafes"; +dbLoader.gpiiKeyViewUrl = dbLoader.couchDBURL + "/_design/views/_view/findGpiiKeysByPrefsSafeId"; +dbLoader.docsToRemove=[]; + +/** + * Find the Prefs Safes of type "snapSet", and ??? + * @param {Object} response - The response from the data base containing the + * snapSet Prefs Safes as a JSON string. + */ +dbLoader.processSnapsets = function (response) { + var snapSetsString = ""; + response.setEncoding("utf8"); + response.on("data", function (chunk) { + snapSetsString += chunk; + }); + response.on("end", function () { + dbLoader.snapSets = JSON.parse(snapSetsString); +// debugger; + dbLoader.snapSets.rows.forEach(function (aSnapset) { + aSnapset.value._deleted = "true"; +// debugger; + console.log(aSnapset.value._rev); + dbLoader.docsToRemove.push(aSnapset.value); + }); + dbLoader.addGpiiKeysAndBulkDelete(dbLoader.snapSets.rows, dbLoader.docsToRemove); + }); +}; + +/** + * Find the GPII key records associated with the given snapset Prefs Safes, + * mark them for deletion, and then request that all be deleted in bulk. + * @param {Array} snapSets - The snapsets of interest. + * @param {Array} docsToRemove - Array to append the GPII key records to delete + */ +dbLoader.addGpiiKeysAndBulkDelete = function (snapSets, docsToRemove) { + snapSets.forEach(function (aSnapset) { +// console.log("addGpiiKeysAndBulkDelete: " + JSON.stringify(aSnapset.value, null, 2)); + + var gpiiKeyId = aSnapset.value._id; + var gpiiKeyViewUrl = dbLoader.gpiiKeyViewUrl + "?key=%22" + gpiiKeyId + "%22"; + console.log("addGpiiKeysAndBulkDelete: gpiiKeyViewUrl is " + gpiiKeyViewUrl); + var getGpiiKeysRequest = http.request(gpiiKeyViewUrl, function (resp) { + var respString = ""; + resp.setEncoding("utf8"); + resp.on("data", function (chunk) { + respString += chunk; + }); + resp.on("end", function () { + console.log("addGpiiKeysAndBulkDelete: respString is " + respString); + var gpiiKeyRecords = JSON.parse(respString); + gpiiKeyRecords.rows.forEach(function (record) { + record.value._deleted = true; + docsToRemove.push(record.value); + }); + console.log("addGpiiKeysToRemove onEnd: " + JSON.stringify (docsToRemove, null, 2)); + }); + }); + getGpiiKeysRequest.on("error", function (e) { + console.log("Finding snapsets GPII Keys error: " + e.message); + }); + getGpiiKeysRequest.end(); + }); + dbLoader.doBatchDelete(docsToRemove); +}; + +/** + * Delete the snapset Prefs Safes and their associated GPII Keys. + * @param {Array} docsToRemove - Array of records to delete. + */ +dbLoader.doBatchDelete = function (docsToRemove) { + var batchDeleteOptions = { + hostname: "localhost", + port: 5984, + path: "/gpii/_bulk_docs", + method: "POST", + headers: { + "Accept": "application/json", + "Content-Length": 0, // filled in below + "Content-Type": "application/json" + } + }; + var batchPostData = JSON.stringify({"docs": docsToRemove}); + batchDeleteOptions.headers["Content-Length"] = Buffer.byteLength(batchPostData); + console.log("Batch Delete snapsets: " + batchPostData); + console.log("Batch Delete snapsets: " + JSON.stringify(batchDeleteOptions, null, 2)); + var batchDeleteReq = http.request(batchDeleteOptions, function (res) { + console.log("STATUS: " + res.statusCode); + console.log("HEADERS: " + JSON.stringify(res.headers, null, 2)); + res.setEncoding('utf8'); + res.on('data', (chunk) => { + console.log("BODY: " + chunk); + }); + res.on('end', function () { + console.log('Batch Delete snapsets: No more data in response.'); + }); + }); + batchDeleteReq.on('error', function (e) { + console.error("Error deleting snapset Prefs Safes and their GPII Keys: " + e.message); + }); + batchDeleteReq.write(batchPostData); + batchDeleteReq.end(); +}; + +debugger; +dbLoader.snapSetsRequest = http.request(dbLoader.prefsSafesViewUrl, dbLoader.processSnapsets); +dbLoader.snapSetsRequest.on("error", function (e) { + console.log("Finding snapsets Prefs Safes error: " + e.message); +}); +dbLoader.snapSetsRequest.end(); +debugger; +var x = 5; diff --git a/httpTests.js b/httpTests.js new file mode 100644 index 0000000..e5efa15 --- /dev/null +++ b/httpTests.js @@ -0,0 +1,138 @@ +var http = require('http'); +var querystring = require("querystring"); + +var baseOptions = { + hostname: "localhost", + port: 5984, + method: "GET" +}; + +// var snapSetViewOptions = baseOptions; snapSetViewOptions.path = /gpii/_design/views/_view/findSnapsetPrefsSafes" + +var snapSetsViewOptions = { + hostname: "localhost", + port: 5984, + path: "/gpii/_design/views/_view/findSnapsetPrefsSafes", + method: "GET" +}; + +var globalToRemoveArray = []; + +var snapSetsString=""; +var snapSetReq = http.request(snapSetsViewOptions, function (response) { + response.setEncoding("utf8"); + + response.on("data", function (chunk) { + snapSetsString += chunk; + }); + + response.on("end", function () { +// console.log(snapSetsString); + var snapSets = JSON.parse(snapSetsString); +// debugger; + for (var i=0; i < snapSets.rows.length; i++) { + var aSnapSet = snapSets.rows[i]; +// console.log("[" + aSnapSet.id + "," + aSnapSet.value._rev + "]"); + } + globalToRemoveArray.push(snapSets.rows[0].value); + globalToRemoveArray.push(snapSets.rows[5].value); + globalToRemoveArray[0]._deleted = true; +// globalToRemoveArray[1]._deletion = true; + console.log(JSON.stringify(globalToRemoveArray[0], null, 2)); + addGpiiKeysToRemove(globalToRemoveArray); + console.log("FOO!"); + }); +}); +snapSetReq.end(); + +getOmarOptions = { + hostname: "localhost", + port: 5984, + path: "/gpii/omar", + method: "GET" +}; + +var deleteString = ""; +var deleteRequest = http.request(getOmarOptions, function (response) { + console.log("STATUS: " + response.statusCode); + response.setEncoding("utf8"); + + response.on("data", function (chunk) { + deleteString += chunk; + }); + + response.on("end", function () { + console.log(deleteString); + }); +}); +deleteRequest.end(); + + +getGpiiKeysOptions = { + hostname: "localhost", + port: 5984, + path: "/gpii/_design/views/_view/findGpiiKeysByPrefsSafeId", + method: "GET" + +}; +function addGpiiKeysToRemove (anArray) { + var gpiiKeyId = anArray[0]._id; + console.log("addGpiiKeysToRemove: _id is " + gpiiKeyId); + getGpiiKeysOptions.path += ("?key=%22" + gpiiKeyId + "%22"); + console.log("addGpiiKeysToRemove: query is " + getGpiiKeysOptions.path); + var getGpiiKeyReq = http.request(getGpiiKeysOptions, function (resp) { + var respString = ""; + resp.setEncoding("utf8"); + resp.on("data", function (chunk) { + respString += chunk; + }); + resp.on("end", function () { + var gpiiKeyRecords = JSON.parse(respString); + gpiiKeyRecords.rows.forEach(function (record) { + record.value._deleted = true; + anArray.push(record.value); + }); + console.log("addGpiiKeysToRemove onEnd: " + JSON.stringify (anArray, null, 2)); + doBatchDelete(anArray); + }); + }); + getGpiiKeyReq.end(); +} + +batchDeleteOptions = { + hostname: "localhost", + port: 5984, + path: "/gpii/_bulk_docs", + method: "POST", +// method: "DELETE", + headers: { + "Accept": "application/json", + "Content-Length": 0, // fill in later + "Content-Type": "application/json" + } +}; +var batchPostData = ""; +function doBatchDelete (anArray) { + var theDocs = {}; + theDocs["docs"] = anArray; + batchPostData = JSON.stringify(theDocs); + batchDeleteOptions.headers["Content-Length"] = Buffer.byteLength(batchPostData); + console.log("doBatchDelete: " + batchPostData); + console.log("doBatchDelete: " + JSON.stringify(batchDeleteOptions, null, 2)); + var batchDeleteReq = http.request(batchDeleteOptions, function (res) { + console.log("STATUS: " + res.statusCode); + console.log("HEADERS: " + JSON.stringify(res.headers, null, 2)); + res.setEncoding('utf8'); + res.on('data', (chunk) => { + console.log("BODY: " + chunk); + }); + res.on('end', function () { + console.log('No more data in response.'); + }); + }); + batchDeleteReq.on('error', function (e) { + console.error("problem with batchDeleteReq: " + e.message); + }); + batchDeleteReq.write(batchPostData); + batchDeleteReq.end(); +} \ No newline at end of file From 760b7957aeacfd14408e1af317be34c57dc3190e Mon Sep 17 00:00:00 2001 From: Joseph Scheuhammer Date: Fri, 29 Jun 2018 16:43:00 -0400 Subject: [PATCH 02/32] GPII-3138: Script to update snapsets. Fixed an errant true that should not have been a string. --- deleteAndLoadSnapsets.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deleteAndLoadSnapsets.js b/deleteAndLoadSnapsets.js index 02ec826..fdb7ab5 100644 --- a/deleteAndLoadSnapsets.js +++ b/deleteAndLoadSnapsets.js @@ -54,7 +54,7 @@ dbLoader.processSnapsets = function (response) { dbLoader.snapSets = JSON.parse(snapSetsString); // debugger; dbLoader.snapSets.rows.forEach(function (aSnapset) { - aSnapset.value._deleted = "true"; + aSnapset.value._deleted = true; // debugger; console.log(aSnapset.value._rev); dbLoader.docsToRemove.push(aSnapset.value); From 35831410e47c29afb7082eede34c9ee7e31be4e9 Mon Sep 17 00:00:00 2001 From: Joseph Scheuhammer Date: Wed, 4 Jul 2018 11:29:14 -0400 Subject: [PATCH 03/32] GPII-3138: Script to update snapsets. Added more logging to see why the call to doBatchDelete() happens before doBatchDelete() is called. The bug is that only the snapset Prefs Safes are deleted. Their GPII Keys are added to the 'docsToRemove' array after that deletion. I'm not sure why, but suspect it has to do with some kind of asynchrony in the http responses. Also, I have not found the condition to test when all the GPII Key records have been added. --- deleteAndLoadSnapsets.js | 43 ++++++++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/deleteAndLoadSnapsets.js b/deleteAndLoadSnapsets.js index fdb7ab5..cbb60f2 100644 --- a/deleteAndLoadSnapsets.js +++ b/deleteAndLoadSnapsets.js @@ -25,16 +25,17 @@ var http = require('http'), var gpii = fluid.registerNamespace("gpii"); fluid.registerNamespace("gpii.dataLoader"); +fluid.setLogging(fluid.logLevel.INFO) var dbLoader = gpii.dataLoader; debugger; dbLoader.couchDBURL = process.argv[2]; if (gpii.dataLoader.couchDBURL === undefined) { - console.log ("COUCHDB_URL environment variable must be defined"); + fluid.log ("COUCHDB_URL environment variable must be defined"); process.exit(1); } -console.log("COUCHDB_URL: '" + dbLoader.couchDBURL + "'"); +fluid.log("COUCHDB_URL: '" + dbLoader.couchDBURL + "'"); dbLoader.prefsSafesViewUrl = dbLoader.couchDBURL + "/_design/views/_view/findSnapsetPrefsSafes"; dbLoader.gpiiKeyViewUrl = dbLoader.couchDBURL + "/_design/views/_view/findGpiiKeysByPrefsSafeId"; dbLoader.docsToRemove=[]; @@ -56,10 +57,12 @@ dbLoader.processSnapsets = function (response) { dbLoader.snapSets.rows.forEach(function (aSnapset) { aSnapset.value._deleted = true; // debugger; - console.log(aSnapset.value._rev); + fluid.log(aSnapset.value._rev); dbLoader.docsToRemove.push(aSnapset.value); }); dbLoader.addGpiiKeysAndBulkDelete(dbLoader.snapSets.rows, dbLoader.docsToRemove); + fluid.log("Back from dbLoader.addGpiiKeysAndBulkDelete(), docsToRemove is: " + batchPostData); + var batchPostData = JSON.stringify({"docs": dbLoader.docsToRemove}); }); }; @@ -70,12 +73,14 @@ dbLoader.processSnapsets = function (response) { * @param {Array} docsToRemove - Array to append the GPII key records to delete */ dbLoader.addGpiiKeysAndBulkDelete = function (snapSets, docsToRemove) { + debugger; snapSets.forEach(function (aSnapset) { -// console.log("addGpiiKeysAndBulkDelete: " + JSON.stringify(aSnapset.value, null, 2)); +// fluid.log("addGpiiKeysAndBulkDelete: " + JSON.stringify(aSnapset.value, null, 2)); var gpiiKeyId = aSnapset.value._id; var gpiiKeyViewUrl = dbLoader.gpiiKeyViewUrl + "?key=%22" + gpiiKeyId + "%22"; - console.log("addGpiiKeysAndBulkDelete: gpiiKeyViewUrl is " + gpiiKeyViewUrl); + fluid.log("addGpiiKeysAndBulkDelete: gpiiKeyViewUrl is " + gpiiKeyViewUrl); + // This request doesn't end until after line 105 executes (too late) var getGpiiKeysRequest = http.request(gpiiKeyViewUrl, function (resp) { var respString = ""; resp.setEncoding("utf8"); @@ -83,19 +88,23 @@ dbLoader.addGpiiKeysAndBulkDelete = function (snapSets, docsToRemove) { respString += chunk; }); resp.on("end", function () { - console.log("addGpiiKeysAndBulkDelete: respString is " + respString); + debugger; + fluid.log("addGpiiKeysAndBulkDelete: respString is " + respString); var gpiiKeyRecords = JSON.parse(respString); gpiiKeyRecords.rows.forEach(function (record) { record.value._deleted = true; docsToRemove.push(record.value); + fluid.log("[" + gpiiKeyId + ", " + docsToRemove.length + "] addGpiiKeysToRemove onEnd: " + JSON.stringify (docsToRemove[docsToRemove.length-1], null, 2)); }); - console.log("addGpiiKeysToRemove onEnd: " + JSON.stringify (docsToRemove, null, 2)); - }); + }); + debugger; + var x = 5; }); getGpiiKeysRequest.on("error", function (e) { - console.log("Finding snapsets GPII Keys error: " + e.message); + fluid.log("Finding snapsets GPII Keys error: " + e.message); }); getGpiiKeysRequest.end(); + var x = 5; }); dbLoader.doBatchDelete(docsToRemove); }; @@ -118,21 +127,21 @@ dbLoader.doBatchDelete = function (docsToRemove) { }; var batchPostData = JSON.stringify({"docs": docsToRemove}); batchDeleteOptions.headers["Content-Length"] = Buffer.byteLength(batchPostData); - console.log("Batch Delete snapsets: " + batchPostData); - console.log("Batch Delete snapsets: " + JSON.stringify(batchDeleteOptions, null, 2)); + fluid.log("Batch Delete snapsets: " + batchPostData); + fluid.log("Batch Delete snapsets: " + JSON.stringify(batchDeleteOptions, null, 2)); var batchDeleteReq = http.request(batchDeleteOptions, function (res) { - console.log("STATUS: " + res.statusCode); - console.log("HEADERS: " + JSON.stringify(res.headers, null, 2)); + fluid.log("STATUS: " + res.statusCode); + fluid.log("HEADERS: " + JSON.stringify(res.headers, null, 2)); res.setEncoding('utf8'); res.on('data', (chunk) => { - console.log("BODY: " + chunk); + fluid.log("BODY: " + chunk); }); res.on('end', function () { - console.log('Batch Delete snapsets: No more data in response.'); + fluid.log('Batch Delete snapsets: No more data in response.'); }); }); batchDeleteReq.on('error', function (e) { - console.error("Error deleting snapset Prefs Safes and their GPII Keys: " + e.message); + fluid.error("Error deleting snapset Prefs Safes and their GPII Keys: " + e.message); }); batchDeleteReq.write(batchPostData); batchDeleteReq.end(); @@ -141,7 +150,7 @@ dbLoader.doBatchDelete = function (docsToRemove) { debugger; dbLoader.snapSetsRequest = http.request(dbLoader.prefsSafesViewUrl, dbLoader.processSnapsets); dbLoader.snapSetsRequest.on("error", function (e) { - console.log("Finding snapsets Prefs Safes error: " + e.message); + fluid.log("Finding snapsets Prefs Safes error: " + e.message); }); dbLoader.snapSetsRequest.end(); debugger; From f2374d455f3e1bff0a2f6a571dc3be6f5c371c54 Mon Sep 17 00:00:00 2001 From: Joseph Scheuhammer Date: Wed, 4 Jul 2018 16:51:38 -0400 Subject: [PATCH 04/32] GPII-3138: Script to update snapsets in the database. This version uses a DELETE url to delete each GPII Key as it is fetched using the findGpiiKeysByPrefsSafeId view with the Prefs Safe ID. Then the snapset Prefs Safes are deleted in bulk. --- deleteAndLoadSnapsets.js | 52 +++++++++++++++++++++++++++++----------- 1 file changed, 38 insertions(+), 14 deletions(-) diff --git a/deleteAndLoadSnapsets.js b/deleteAndLoadSnapsets.js index cbb60f2..6134236 100644 --- a/deleteAndLoadSnapsets.js +++ b/deleteAndLoadSnapsets.js @@ -53,16 +53,13 @@ dbLoader.processSnapsets = function (response) { }); response.on("end", function () { dbLoader.snapSets = JSON.parse(snapSetsString); -// debugger; dbLoader.snapSets.rows.forEach(function (aSnapset) { aSnapset.value._deleted = true; -// debugger; fluid.log(aSnapset.value._rev); dbLoader.docsToRemove.push(aSnapset.value); }); dbLoader.addGpiiKeysAndBulkDelete(dbLoader.snapSets.rows, dbLoader.docsToRemove); - fluid.log("Back from dbLoader.addGpiiKeysAndBulkDelete(), docsToRemove is: " + batchPostData); - var batchPostData = JSON.stringify({"docs": dbLoader.docsToRemove}); + fluid.log("Back from dbLoader.addGpiiKeysAndBulkDelete(), docsToRemove is: " + JSON.stringify({"docs": dbLoader.docsToRemove})); }); }; @@ -75,40 +72,67 @@ dbLoader.processSnapsets = function (response) { dbLoader.addGpiiKeysAndBulkDelete = function (snapSets, docsToRemove) { debugger; snapSets.forEach(function (aSnapset) { -// fluid.log("addGpiiKeysAndBulkDelete: " + JSON.stringify(aSnapset.value, null, 2)); - var gpiiKeyId = aSnapset.value._id; var gpiiKeyViewUrl = dbLoader.gpiiKeyViewUrl + "?key=%22" + gpiiKeyId + "%22"; fluid.log("addGpiiKeysAndBulkDelete: gpiiKeyViewUrl is " + gpiiKeyViewUrl); - // This request doesn't end until after line 105 executes (too late) var getGpiiKeysRequest = http.request(gpiiKeyViewUrl, function (resp) { var respString = ""; resp.setEncoding("utf8"); resp.on("data", function (chunk) { respString += chunk; }); + // This response doesn't end until after the call to doBatchDelete() + // below (too late) resp.on("end", function () { debugger; fluid.log("addGpiiKeysAndBulkDelete: respString is " + respString); var gpiiKeyRecords = JSON.parse(respString); gpiiKeyRecords.rows.forEach(function (record) { - record.value._deleted = true; - docsToRemove.push(record.value); - fluid.log("[" + gpiiKeyId + ", " + docsToRemove.length + "] addGpiiKeysToRemove onEnd: " + JSON.stringify (docsToRemove[docsToRemove.length-1], null, 2)); + dbLoader.deleteGpiiKey(record.value); }); }); - debugger; - var x = 5; }); getGpiiKeysRequest.on("error", function (e) { fluid.log("Finding snapsets GPII Keys error: " + e.message); }); getGpiiKeysRequest.end(); - var x = 5; }); dbLoader.doBatchDelete(docsToRemove); }; +/** + * Delete the given GPII Key record. + * @param {Object} gpiiKey - The key to delete. + */ +dbLoader.deleteGpiiKey = function (gpiiKey) { + var deleteOptions = { + hostname: "localhost", + port: 5984, + path: "", // filled in below. + method: "DELETE", + headers: { + "Accept": "application/json" + } + }; + deleteOptions.path = "/gpii/" + gpiiKey._id + "?rev=" + gpiiKey._rev; + fluid.log("deleteGpiiKey(): path is '" + deleteOptions.path + "'"); + var deleteGpiiKeyRequest = http.request(deleteOptions, function (res) { + fluid.log("STATUS: " + res.statusCode); + fluid.log("HEADERS: " + JSON.stringify(res.headers, null, 2)); + res.setEncoding('utf8'); + res.on("data", (chunk) => { + fluid.log("BODY: " + chunk); + }); + res.on("end", function () { + fluid.log("GPII Key deleted(?)"); + }); + }); + deleteGpiiKeyRequest.on("error", function (e) { + fluid.log("Finding snapsets GPII Keys error: " + e.message); + }); + deleteGpiiKeyRequest.end(); +}; + /** * Delete the snapset Prefs Safes and their associated GPII Keys. * @param {Array} docsToRemove - Array of records to delete. @@ -128,7 +152,7 @@ dbLoader.doBatchDelete = function (docsToRemove) { var batchPostData = JSON.stringify({"docs": docsToRemove}); batchDeleteOptions.headers["Content-Length"] = Buffer.byteLength(batchPostData); fluid.log("Batch Delete snapsets: " + batchPostData); - fluid.log("Batch Delete snapsets: " + JSON.stringify(batchDeleteOptions, null, 2)); + console.log("Batch Delete snapsets: " + JSON.stringify(batchDeleteOptions, null, 2)); var batchDeleteReq = http.request(batchDeleteOptions, function (res) { fluid.log("STATUS: " + res.statusCode); fluid.log("HEADERS: " + JSON.stringify(res.headers, null, 2)); From c1a494f615ec42c6014f01394040283966c607ae Mon Sep 17 00:00:00 2001 From: Joseph Scheuhammer Date: Fri, 6 Jul 2018 10:23:31 -0400 Subject: [PATCH 05/32] GPII-3138: Script to update snapsets in the database. Cleaned up code. --- deleteAndLoadSnapsets.js | 41 ++++++++++++++++++---------------------- 1 file changed, 18 insertions(+), 23 deletions(-) diff --git a/deleteAndLoadSnapsets.js b/deleteAndLoadSnapsets.js index 6134236..71c53ec 100644 --- a/deleteAndLoadSnapsets.js +++ b/deleteAndLoadSnapsets.js @@ -12,26 +12,22 @@ https://github.com/GPII/universal/blob/master/LICENSE.txt // 1. Finds all the Prefs Safes of type "snapset" (prefsSafesType = "snapset"), // 2. Finds all the GPII Keys associated with each snapset Prefs Safe // 3. Deletes the found Prefs Safes and associated GPII Keys -// 4. Loads the latest Prefs Safes and GPII Keys from %universal/build/dbData -// 5. Loads the latest credentials, clients, and views from %universal/testData/dbData // // A sample command that runs this script: -// node deleteAndLoadSnapsets.js COUCHDBURL +// node deleteAndLoadSnapsets.js $COUCHDBURL "use strict"; var http = require('http'), - fluid = require("../universal/node_modules/infusion"); + fluid = require("infusion"); var gpii = fluid.registerNamespace("gpii"); fluid.registerNamespace("gpii.dataLoader"); fluid.setLogging(fluid.logLevel.INFO) var dbLoader = gpii.dataLoader; - -debugger; dbLoader.couchDBURL = process.argv[2]; -if (gpii.dataLoader.couchDBURL === undefined) { +if (!fluid.isValue(dbLoader.couchDBURL)) { fluid.log ("COUCHDB_URL environment variable must be defined"); process.exit(1); } @@ -41,7 +37,8 @@ dbLoader.gpiiKeyViewUrl = dbLoader.couchDBURL + "/_design/views/_view/findGpiiKe dbLoader.docsToRemove=[]; /** - * Find the Prefs Safes of type "snapSet", and ??? + * Find the Prefs Safes of type "snapSet", add them to an array of records to + * remove, then use them to find their associated GPII Key(s). * @param {Object} response - The response from the data base containing the * snapSet Prefs Safes as a JSON string. */ @@ -65,13 +62,12 @@ dbLoader.processSnapsets = function (response) { /** * Find the GPII key records associated with the given snapset Prefs Safes, - * mark them for deletion, and then request that all be deleted in bulk. + * delete each one as found, and then batch delete all the snapset Prefs Safes. * @param {Array} snapSets - The snapsets of interest. - * @param {Array} docsToRemove - Array to append the GPII key records to delete + * @param {Array} docsToRemove - Array of records to delete. */ dbLoader.addGpiiKeysAndBulkDelete = function (snapSets, docsToRemove) { - debugger; - snapSets.forEach(function (aSnapset) { + fluid.each (snapSets, function (aSnapset) { var gpiiKeyId = aSnapset.value._id; var gpiiKeyViewUrl = dbLoader.gpiiKeyViewUrl + "?key=%22" + gpiiKeyId + "%22"; fluid.log("addGpiiKeysAndBulkDelete: gpiiKeyViewUrl is " + gpiiKeyViewUrl); @@ -81,10 +77,12 @@ dbLoader.addGpiiKeysAndBulkDelete = function (snapSets, docsToRemove) { resp.on("data", function (chunk) { respString += chunk; }); - // This response doesn't end until after the call to doBatchDelete() - // below (too late) + // This response "end" event doesn't finish until after the call to + // doBatchDelete() below, which is too late to add the GPII Key to + // docsToRemove and have them removed. + // Note - if there is a way to determine the final "end" event, then + // it could call doBatchDelete(). resp.on("end", function () { - debugger; fluid.log("addGpiiKeysAndBulkDelete: respString is " + respString); var gpiiKeyRecords = JSON.parse(respString); gpiiKeyRecords.rows.forEach(function (record) { @@ -93,7 +91,7 @@ dbLoader.addGpiiKeysAndBulkDelete = function (snapSets, docsToRemove) { }); }); getGpiiKeysRequest.on("error", function (e) { - fluid.log("Finding snapsets GPII Keys error: " + e.message); + fluid.log("Error finding snapsets' associated GPII Keys: " + e.message); }); getGpiiKeysRequest.end(); }); @@ -124,11 +122,11 @@ dbLoader.deleteGpiiKey = function (gpiiKey) { fluid.log("BODY: " + chunk); }); res.on("end", function () { - fluid.log("GPII Key deleted(?)"); + fluid.log("GPII Key deleted"); }); }); deleteGpiiKeyRequest.on("error", function (e) { - fluid.log("Finding snapsets GPII Keys error: " + e.message); + fluid.log("Error finding GPII Key: " + e.message); }); deleteGpiiKeyRequest.end(); }; @@ -165,17 +163,14 @@ dbLoader.doBatchDelete = function (docsToRemove) { }); }); batchDeleteReq.on('error', function (e) { - fluid.error("Error deleting snapset Prefs Safes and their GPII Keys: " + e.message); + fluid.error("Error deleting snapset Prefs Safes: " + e.message); }); batchDeleteReq.write(batchPostData); batchDeleteReq.end(); }; -debugger; dbLoader.snapSetsRequest = http.request(dbLoader.prefsSafesViewUrl, dbLoader.processSnapsets); dbLoader.snapSetsRequest.on("error", function (e) { - fluid.log("Finding snapsets Prefs Safes error: " + e.message); + fluid.log("Error finding snapsets Prefs Safes: " + e.message); }); dbLoader.snapSetsRequest.end(); -debugger; -var x = 5; From a8a161f881e5dff2fdbf7abed2f00e27e92b9f1c Mon Sep 17 00:00:00 2001 From: Joseph Scheuhammer Date: Fri, 6 Jul 2018 10:27:18 -0400 Subject: [PATCH 06/32] GPII-3138: Script to update snapsets in the database Changed name of script as it only deletes the old snapset Prefs Safes and their associated GPII Key records. The bash script still loads the latest snapsets into the database. --- deleteAndLoadSnapsets.js => deleteSnapsets.js | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename deleteAndLoadSnapsets.js => deleteSnapsets.js (100%) diff --git a/deleteAndLoadSnapsets.js b/deleteSnapsets.js similarity index 100% rename from deleteAndLoadSnapsets.js rename to deleteSnapsets.js From 98e70d36ec6d8ae51610d521e02a79cddb4c2750 Mon Sep 17 00:00:00 2001 From: Joseph Scheuhammer Date: Fri, 6 Jul 2018 14:42:19 -0400 Subject: [PATCH 07/32] GPII-3138: Update snapsets in the database - Cleaned up logging in deleteSnapsets.js, - Modified Dockerfile re: new name of script "deleteAndLoadSnapsets.sh", - Modified deleteAndLoadSnapsets.sh regarding NODE_PATH for node to use --- Dockerfile | 4 ++-- ...oadSnapSets.sh => deleteAndLoadSnapsets.sh | 2 ++ deleteSnapsets.js | 20 +------------------ 3 files changed, 5 insertions(+), 21 deletions(-) rename deleteAndLoadSnapSets.sh => deleteAndLoadSnapsets.sh (91%) diff --git a/Dockerfile b/Dockerfile index cecd711..bc6260f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,6 +13,6 @@ RUN apk add --no-cache curl git && \ node scripts/convertPrefs.js testData/preferences/ build/dbData/ && \ apk del git -COPY loadData.sh /usr/local/bin +COPY deleteAndLoadSnapsets.sh /usr/local/bin/ -CMD ["/usr/local/bin/loadData.sh"] +CMD ["/usr/local/bin/deleteAndLoadSnapsets.sh"] diff --git a/deleteAndLoadSnapSets.sh b/deleteAndLoadSnapsets.sh similarity index 91% rename from deleteAndLoadSnapSets.sh rename to deleteAndLoadSnapsets.sh index 0c521f7..1311f76 100755 --- a/deleteAndLoadSnapSets.sh +++ b/deleteAndLoadSnapsets.sh @@ -2,6 +2,7 @@ STATIC_DATA_DIR=${STATIC_DATA_DIR:-/home/node/universal/testData/dbData} BUILD_DATA_DIR=${BUILD_DATA_DIR:-/home/node/universal/build/dbData} +NODE_PATH=${NODE_PATH:-/home/node/universal} log() { echo "$(date +'%Y-%m-%d %H:%M:%S') - $1" @@ -45,5 +46,6 @@ if ! curl -fsS -X PUT "$COUCHDB_URL"; then fi # Submit data +node $NODE_PATH/scripts/deleteSnapsets.js $COUCHDB_URL loadData $STATIC_DATA_DIR loadData $BUILD_DATA_DIR diff --git a/deleteSnapsets.js b/deleteSnapsets.js index 71c53ec..7fac5eb 100644 --- a/deleteSnapsets.js +++ b/deleteSnapsets.js @@ -52,11 +52,9 @@ dbLoader.processSnapsets = function (response) { dbLoader.snapSets = JSON.parse(snapSetsString); dbLoader.snapSets.rows.forEach(function (aSnapset) { aSnapset.value._deleted = true; - fluid.log(aSnapset.value._rev); dbLoader.docsToRemove.push(aSnapset.value); }); dbLoader.addGpiiKeysAndBulkDelete(dbLoader.snapSets.rows, dbLoader.docsToRemove); - fluid.log("Back from dbLoader.addGpiiKeysAndBulkDelete(), docsToRemove is: " + JSON.stringify({"docs": dbLoader.docsToRemove})); }); }; @@ -70,7 +68,6 @@ dbLoader.addGpiiKeysAndBulkDelete = function (snapSets, docsToRemove) { fluid.each (snapSets, function (aSnapset) { var gpiiKeyId = aSnapset.value._id; var gpiiKeyViewUrl = dbLoader.gpiiKeyViewUrl + "?key=%22" + gpiiKeyId + "%22"; - fluid.log("addGpiiKeysAndBulkDelete: gpiiKeyViewUrl is " + gpiiKeyViewUrl); var getGpiiKeysRequest = http.request(gpiiKeyViewUrl, function (resp) { var respString = ""; resp.setEncoding("utf8"); @@ -83,7 +80,6 @@ dbLoader.addGpiiKeysAndBulkDelete = function (snapSets, docsToRemove) { // Note - if there is a way to determine the final "end" event, then // it could call doBatchDelete(). resp.on("end", function () { - fluid.log("addGpiiKeysAndBulkDelete: respString is " + respString); var gpiiKeyRecords = JSON.parse(respString); gpiiKeyRecords.rows.forEach(function (record) { dbLoader.deleteGpiiKey(record.value); @@ -113,17 +109,9 @@ dbLoader.deleteGpiiKey = function (gpiiKey) { } }; deleteOptions.path = "/gpii/" + gpiiKey._id + "?rev=" + gpiiKey._rev; - fluid.log("deleteGpiiKey(): path is '" + deleteOptions.path + "'"); var deleteGpiiKeyRequest = http.request(deleteOptions, function (res) { fluid.log("STATUS: " + res.statusCode); fluid.log("HEADERS: " + JSON.stringify(res.headers, null, 2)); - res.setEncoding('utf8'); - res.on("data", (chunk) => { - fluid.log("BODY: " + chunk); - }); - res.on("end", function () { - fluid.log("GPII Key deleted"); - }); }); deleteGpiiKeyRequest.on("error", function (e) { fluid.log("Error finding GPII Key: " + e.message); @@ -149,17 +137,11 @@ dbLoader.doBatchDelete = function (docsToRemove) { }; var batchPostData = JSON.stringify({"docs": docsToRemove}); batchDeleteOptions.headers["Content-Length"] = Buffer.byteLength(batchPostData); - fluid.log("Batch Delete snapsets: " + batchPostData); - console.log("Batch Delete snapsets: " + JSON.stringify(batchDeleteOptions, null, 2)); var batchDeleteReq = http.request(batchDeleteOptions, function (res) { fluid.log("STATUS: " + res.statusCode); fluid.log("HEADERS: " + JSON.stringify(res.headers, null, 2)); - res.setEncoding('utf8'); - res.on('data', (chunk) => { - fluid.log("BODY: " + chunk); - }); res.on('end', function () { - fluid.log('Batch Delete snapsets: No more data in response.'); + fluid.log('Batch deletion of snapsets'); }); }); batchDeleteReq.on('error', function (e) { From d042026512fee5f9e72c1fe98f1155e8229f1600 Mon Sep 17 00:00:00 2001 From: Joseph Scheuhammer Date: Tue, 10 Jul 2018 10:32:46 -0400 Subject: [PATCH 08/32] GPII-3138: Update snapsets in the database Moved the delete script to universal. --- deleteSnapsets.js | 158 ---------------------------------------------- 1 file changed, 158 deletions(-) delete mode 100644 deleteSnapsets.js diff --git a/deleteSnapsets.js b/deleteSnapsets.js deleted file mode 100644 index 7fac5eb..0000000 --- a/deleteSnapsets.js +++ /dev/null @@ -1,158 +0,0 @@ -/*! -Copyright 2018 OCAD University - -Licensed under the New BSD license. You may not use this file except in -compliance with this License. - -You may obtain a copy of the License at -https://github.com/GPII/universal/blob/master/LICENSE.txt -*/ - -// This script modifies the preferences data base: -// 1. Finds all the Prefs Safes of type "snapset" (prefsSafesType = "snapset"), -// 2. Finds all the GPII Keys associated with each snapset Prefs Safe -// 3. Deletes the found Prefs Safes and associated GPII Keys -// -// A sample command that runs this script: -// node deleteAndLoadSnapsets.js $COUCHDBURL - -"use strict"; - -var http = require('http'), - fluid = require("infusion"); - -var gpii = fluid.registerNamespace("gpii"); -fluid.registerNamespace("gpii.dataLoader"); -fluid.setLogging(fluid.logLevel.INFO) - -var dbLoader = gpii.dataLoader; -dbLoader.couchDBURL = process.argv[2]; -if (!fluid.isValue(dbLoader.couchDBURL)) { - fluid.log ("COUCHDB_URL environment variable must be defined"); - process.exit(1); -} -fluid.log("COUCHDB_URL: '" + dbLoader.couchDBURL + "'"); -dbLoader.prefsSafesViewUrl = dbLoader.couchDBURL + "/_design/views/_view/findSnapsetPrefsSafes"; -dbLoader.gpiiKeyViewUrl = dbLoader.couchDBURL + "/_design/views/_view/findGpiiKeysByPrefsSafeId"; -dbLoader.docsToRemove=[]; - -/** - * Find the Prefs Safes of type "snapSet", add them to an array of records to - * remove, then use them to find their associated GPII Key(s). - * @param {Object} response - The response from the data base containing the - * snapSet Prefs Safes as a JSON string. - */ -dbLoader.processSnapsets = function (response) { - var snapSetsString = ""; - response.setEncoding("utf8"); - response.on("data", function (chunk) { - snapSetsString += chunk; - }); - response.on("end", function () { - dbLoader.snapSets = JSON.parse(snapSetsString); - dbLoader.snapSets.rows.forEach(function (aSnapset) { - aSnapset.value._deleted = true; - dbLoader.docsToRemove.push(aSnapset.value); - }); - dbLoader.addGpiiKeysAndBulkDelete(dbLoader.snapSets.rows, dbLoader.docsToRemove); - }); -}; - -/** - * Find the GPII key records associated with the given snapset Prefs Safes, - * delete each one as found, and then batch delete all the snapset Prefs Safes. - * @param {Array} snapSets - The snapsets of interest. - * @param {Array} docsToRemove - Array of records to delete. - */ -dbLoader.addGpiiKeysAndBulkDelete = function (snapSets, docsToRemove) { - fluid.each (snapSets, function (aSnapset) { - var gpiiKeyId = aSnapset.value._id; - var gpiiKeyViewUrl = dbLoader.gpiiKeyViewUrl + "?key=%22" + gpiiKeyId + "%22"; - var getGpiiKeysRequest = http.request(gpiiKeyViewUrl, function (resp) { - var respString = ""; - resp.setEncoding("utf8"); - resp.on("data", function (chunk) { - respString += chunk; - }); - // This response "end" event doesn't finish until after the call to - // doBatchDelete() below, which is too late to add the GPII Key to - // docsToRemove and have them removed. - // Note - if there is a way to determine the final "end" event, then - // it could call doBatchDelete(). - resp.on("end", function () { - var gpiiKeyRecords = JSON.parse(respString); - gpiiKeyRecords.rows.forEach(function (record) { - dbLoader.deleteGpiiKey(record.value); - }); - }); - }); - getGpiiKeysRequest.on("error", function (e) { - fluid.log("Error finding snapsets' associated GPII Keys: " + e.message); - }); - getGpiiKeysRequest.end(); - }); - dbLoader.doBatchDelete(docsToRemove); -}; - -/** - * Delete the given GPII Key record. - * @param {Object} gpiiKey - The key to delete. - */ -dbLoader.deleteGpiiKey = function (gpiiKey) { - var deleteOptions = { - hostname: "localhost", - port: 5984, - path: "", // filled in below. - method: "DELETE", - headers: { - "Accept": "application/json" - } - }; - deleteOptions.path = "/gpii/" + gpiiKey._id + "?rev=" + gpiiKey._rev; - var deleteGpiiKeyRequest = http.request(deleteOptions, function (res) { - fluid.log("STATUS: " + res.statusCode); - fluid.log("HEADERS: " + JSON.stringify(res.headers, null, 2)); - }); - deleteGpiiKeyRequest.on("error", function (e) { - fluid.log("Error finding GPII Key: " + e.message); - }); - deleteGpiiKeyRequest.end(); -}; - -/** - * Delete the snapset Prefs Safes and their associated GPII Keys. - * @param {Array} docsToRemove - Array of records to delete. - */ -dbLoader.doBatchDelete = function (docsToRemove) { - var batchDeleteOptions = { - hostname: "localhost", - port: 5984, - path: "/gpii/_bulk_docs", - method: "POST", - headers: { - "Accept": "application/json", - "Content-Length": 0, // filled in below - "Content-Type": "application/json" - } - }; - var batchPostData = JSON.stringify({"docs": docsToRemove}); - batchDeleteOptions.headers["Content-Length"] = Buffer.byteLength(batchPostData); - var batchDeleteReq = http.request(batchDeleteOptions, function (res) { - fluid.log("STATUS: " + res.statusCode); - fluid.log("HEADERS: " + JSON.stringify(res.headers, null, 2)); - res.on('end', function () { - fluid.log('Batch deletion of snapsets'); - }); - }); - batchDeleteReq.on('error', function (e) { - fluid.error("Error deleting snapset Prefs Safes: " + e.message); - }); - batchDeleteReq.write(batchPostData); - batchDeleteReq.end(); -}; - -dbLoader.snapSetsRequest = http.request(dbLoader.prefsSafesViewUrl, dbLoader.processSnapsets); -dbLoader.snapSetsRequest.on("error", function (e) { - fluid.log("Error finding snapsets Prefs Safes: " + e.message); -}); -dbLoader.snapSetsRequest.end(); From 96bddc6b3574e2b7be18e07e5598d070524571ba Mon Sep 17 00:00:00 2001 From: Joseph Scheuhammer Date: Tue, 10 Jul 2018 12:58:02 -0400 Subject: [PATCH 09/32] GPII-3138: Update snapsets in the database Added more log messages. --- deleteAndLoadSnapsets.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/deleteAndLoadSnapsets.sh b/deleteAndLoadSnapsets.sh index 1311f76..58c8468 100755 --- a/deleteAndLoadSnapsets.sh +++ b/deleteAndLoadSnapsets.sh @@ -32,6 +32,10 @@ if [ -z "$COUCHDB_URL" ]; then fi log "Starting" +log "Clear index: $CLEAR_INDEX" +log "Static: $STATIC_DATA_DIR" +log "Build: $BUILD_DATA_DIR" +log "Node path: $NODE_PATH" if [ ! -z "$CLEAR_INDEX" ]; then log "Deleting database at $COUCHDB_URL" From 7a290b25cc2aaa6336ceaf46de5e2d5282dff7b5 Mon Sep 17 00:00:00 2001 From: Joseph Scheuhammer Date: Tue, 10 Jul 2018 15:50:14 -0400 Subject: [PATCH 10/32] GPII-3138: Update snapsets in the database Modified the way that universal is set up within the Docker image in order that the latest version of universal is used. This is done by cloning from github when the image is run, not when the image is built. Also, this means that the preferences are converted to Prefs Safes and GPII Keys at "run-time", not "build-time". --- Dockerfile | 11 +---------- deleteAndLoadSnapsets.sh | 13 +++++++++++++ 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index bc6260f..18302d8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,16 +2,7 @@ FROM node:8-alpine WORKDIR /home/node -RUN apk add --no-cache curl git && \ - git clone https://github.com/GPII/universal.git && \ - cd universal && \ - rm -f package-lock.json && \ - npm install json5 && \ - npm install fs && \ - npm install rimraf && \ - npm install mkdirp && \ - node scripts/convertPrefs.js testData/preferences/ build/dbData/ && \ - apk del git +RUN apk add --no-cache curl git COPY deleteAndLoadSnapsets.sh /usr/local/bin/ diff --git a/deleteAndLoadSnapsets.sh b/deleteAndLoadSnapsets.sh index 58c8468..e98ffac 100755 --- a/deleteAndLoadSnapsets.sh +++ b/deleteAndLoadSnapsets.sh @@ -36,7 +36,20 @@ log "Clear index: $CLEAR_INDEX" log "Static: $STATIC_DATA_DIR" log "Build: $BUILD_DATA_DIR" log "Node path: $NODE_PATH" +log "Working directory: `pwd`" +# Set up universal +git clone https://github.com/GPII/universal.git +cd universal +rm -f package-lock.json +npm install json5 +npm install fs +npm install rimraf +npm install mkdirp +node scripts/convertPrefs.js testData/preferences/ build/dbData/ +cd - + +# Initialize (possibly clear) data base if [ ! -z "$CLEAR_INDEX" ]; then log "Deleting database at $COUCHDB_URL" if ! curl -fsS -X DELETE "$COUCHDB_URL"; then From 01908db6399b32e9e2bdaa9e90298d1481673cfb Mon Sep 17 00:00:00 2001 From: Joseph Scheuhammer Date: Wed, 11 Jul 2018 09:38:07 -0400 Subject: [PATCH 11/32] GPII-3138: Update snapsets in the database Removed test/investigative script. --- httpTests.js | 138 --------------------------------------------------- 1 file changed, 138 deletions(-) delete mode 100644 httpTests.js diff --git a/httpTests.js b/httpTests.js deleted file mode 100644 index e5efa15..0000000 --- a/httpTests.js +++ /dev/null @@ -1,138 +0,0 @@ -var http = require('http'); -var querystring = require("querystring"); - -var baseOptions = { - hostname: "localhost", - port: 5984, - method: "GET" -}; - -// var snapSetViewOptions = baseOptions; snapSetViewOptions.path = /gpii/_design/views/_view/findSnapsetPrefsSafes" - -var snapSetsViewOptions = { - hostname: "localhost", - port: 5984, - path: "/gpii/_design/views/_view/findSnapsetPrefsSafes", - method: "GET" -}; - -var globalToRemoveArray = []; - -var snapSetsString=""; -var snapSetReq = http.request(snapSetsViewOptions, function (response) { - response.setEncoding("utf8"); - - response.on("data", function (chunk) { - snapSetsString += chunk; - }); - - response.on("end", function () { -// console.log(snapSetsString); - var snapSets = JSON.parse(snapSetsString); -// debugger; - for (var i=0; i < snapSets.rows.length; i++) { - var aSnapSet = snapSets.rows[i]; -// console.log("[" + aSnapSet.id + "," + aSnapSet.value._rev + "]"); - } - globalToRemoveArray.push(snapSets.rows[0].value); - globalToRemoveArray.push(snapSets.rows[5].value); - globalToRemoveArray[0]._deleted = true; -// globalToRemoveArray[1]._deletion = true; - console.log(JSON.stringify(globalToRemoveArray[0], null, 2)); - addGpiiKeysToRemove(globalToRemoveArray); - console.log("FOO!"); - }); -}); -snapSetReq.end(); - -getOmarOptions = { - hostname: "localhost", - port: 5984, - path: "/gpii/omar", - method: "GET" -}; - -var deleteString = ""; -var deleteRequest = http.request(getOmarOptions, function (response) { - console.log("STATUS: " + response.statusCode); - response.setEncoding("utf8"); - - response.on("data", function (chunk) { - deleteString += chunk; - }); - - response.on("end", function () { - console.log(deleteString); - }); -}); -deleteRequest.end(); - - -getGpiiKeysOptions = { - hostname: "localhost", - port: 5984, - path: "/gpii/_design/views/_view/findGpiiKeysByPrefsSafeId", - method: "GET" - -}; -function addGpiiKeysToRemove (anArray) { - var gpiiKeyId = anArray[0]._id; - console.log("addGpiiKeysToRemove: _id is " + gpiiKeyId); - getGpiiKeysOptions.path += ("?key=%22" + gpiiKeyId + "%22"); - console.log("addGpiiKeysToRemove: query is " + getGpiiKeysOptions.path); - var getGpiiKeyReq = http.request(getGpiiKeysOptions, function (resp) { - var respString = ""; - resp.setEncoding("utf8"); - resp.on("data", function (chunk) { - respString += chunk; - }); - resp.on("end", function () { - var gpiiKeyRecords = JSON.parse(respString); - gpiiKeyRecords.rows.forEach(function (record) { - record.value._deleted = true; - anArray.push(record.value); - }); - console.log("addGpiiKeysToRemove onEnd: " + JSON.stringify (anArray, null, 2)); - doBatchDelete(anArray); - }); - }); - getGpiiKeyReq.end(); -} - -batchDeleteOptions = { - hostname: "localhost", - port: 5984, - path: "/gpii/_bulk_docs", - method: "POST", -// method: "DELETE", - headers: { - "Accept": "application/json", - "Content-Length": 0, // fill in later - "Content-Type": "application/json" - } -}; -var batchPostData = ""; -function doBatchDelete (anArray) { - var theDocs = {}; - theDocs["docs"] = anArray; - batchPostData = JSON.stringify(theDocs); - batchDeleteOptions.headers["Content-Length"] = Buffer.byteLength(batchPostData); - console.log("doBatchDelete: " + batchPostData); - console.log("doBatchDelete: " + JSON.stringify(batchDeleteOptions, null, 2)); - var batchDeleteReq = http.request(batchDeleteOptions, function (res) { - console.log("STATUS: " + res.statusCode); - console.log("HEADERS: " + JSON.stringify(res.headers, null, 2)); - res.setEncoding('utf8'); - res.on('data', (chunk) => { - console.log("BODY: " + chunk); - }); - res.on('end', function () { - console.log('No more data in response.'); - }); - }); - batchDeleteReq.on('error', function (e) { - console.error("problem with batchDeleteReq: " + e.message); - }); - batchDeleteReq.write(batchPostData); - batchDeleteReq.end(); -} \ No newline at end of file From 372e9c43375805c6ff4e97eb9d29e5ddfa128be0 Mon Sep 17 00:00:00 2001 From: Joseph Scheuhammer Date: Wed, 11 Jul 2018 15:11:53 -0400 Subject: [PATCH 12/32] GPII-3138: Update snapsets in the database Updated README.md --- README.md | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index cfefebe..e43d757 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,12 @@ # CouchDB Data Loader -Builds a [sidecar container](http://blog.kubernetes.io/2015/06/the-distributed-system-toolkit-patterns.html) with the CouchDB data from the GPII/universal repository baked in and a mechanism for loading them into a CouchDB database. +Builds a [sidecar container](http://blog.kubernetes.io/2015/06/the-distributed-system-toolkit-patterns.html) that contains the `git` command and a shell script for setting up a CouchDB database. When the docker image is run, this sequence is executed: +1. Clones the latest version of [GPII universal](https://github.com/gpii/universal/), +1. Converts the preferences in universal into snapset Prefs Safes and GPII Keys, +1. Creates a CouchDB database if none exits, +1. Optionally clears an existing database of all its records, +1. Deletes any snapsets currently in the database, +1. Loads the latest snapsets created at the second step into the database. ## Building @@ -12,6 +18,7 @@ Builds a [sidecar container](http://blog.kubernetes.io/2015/06/the-distributed-s - `CLEAR_INDEX`: If defined, the database at $COUCHDB_URL will be deleted and recreated. (optional) - `STATIC_DATA_DIR`: The directory where the static data to be loaded into CouchDB resides. (optional) - `BUILD_DATA_DIR`: The directory where the data built from a npm step resides. (optional) +- `NODE_PATH`: Universal's root directory. (optional) The use of environment variables for data directories is useful if you want to mount the database data using a Docker volume and point the data loader at it. @@ -31,5 +38,5 @@ $ docker run -d -p 8081:8081 --name preferences --link couchdb -e NODE_ENV=gpii. Loading couchdb data from a different location (e.g. /home/vagrant/sync/universal/testData/dbData for static data directory and /home/vagrant/sync/universal/build/dbData for build data directory): ``` -$ docker run --name dataloader --link couchdb -v /home/vagrant/sync/universal/testData/dbData:/static_data -e STATIC_DATA_DIR=/static_data -v /home/vagrant/sync/universal/build/dbData:/build_data -e BUILD_DATA_DIR=/build_data -e COUCHDB_URL=http://couchdb:5984/gpii -e CLEAR_INDEX=1 gpii/gpii-dataloader +$ docker run --name dataloader --link couchdb -v /home/vagrant/sync/universal/testData/dbData:/static_data -e STATIC_DATA_DIR=/static_data -v /home/vagrant/sync/universal/build/dbData:/build_data -e BUILD_DATA_DIR=/build_data -e COUCHDB_URL=http://couchdb:5984/gpii [-e CLEAR_INDEX=1] -v /home/vagrant/sync/universal:/universal -e NODE_PATH=/universal gpii/gpii-dataloader ``` From 81185745f73fe4abaab02dfbc5e7d5f78f5f4e28 Mon Sep 17 00:00:00 2001 From: Joseph Scheuhammer Date: Mon, 16 Jul 2018 15:53:56 -0400 Subject: [PATCH 13/32] GPII-3138: Update snapsets in the database Modified in response to Cindy and Tyler's comments. --- README.md | 12 +++++++++--- deleteAndLoadSnapsets.sh | 10 +++++----- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index e43d757..3acd90b 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,6 @@ Builds a [sidecar container](http://blog.kubernetes.io/2015/06/the-distributed-s - `CLEAR_INDEX`: If defined, the database at $COUCHDB_URL will be deleted and recreated. (optional) - `STATIC_DATA_DIR`: The directory where the static data to be loaded into CouchDB resides. (optional) - `BUILD_DATA_DIR`: The directory where the data built from a npm step resides. (optional) -- `NODE_PATH`: Universal's root directory. (optional) The use of environment variables for data directories is useful if you want to mount the database data using a Docker volume and point the data loader at it. @@ -35,8 +34,15 @@ $ docker run -d -p 8081:8081 --name preferences --link couchdb -e NODE_ENV=gpii. ``` -Loading couchdb data from a different location (e.g. /home/vagrant/sync/universal/testData/dbData for static data directory and /home/vagrant/sync/universal/build/dbData for build data directory): +Below are two versions of loading couchdb data from a different location (e.g. /home/vagrant/sync/universal/testData/dbData for static data directory and /home/vagrant/sync/universal/build/dbData for build data directory). The first version has the optional `CLEAR_INDEX` set to erase and reset the database prior to other database changes: ``` -$ docker run --name dataloader --link couchdb -v /home/vagrant/sync/universal/testData/dbData:/static_data -e STATIC_DATA_DIR=/static_data -v /home/vagrant/sync/universal/build/dbData:/build_data -e BUILD_DATA_DIR=/build_data -e COUCHDB_URL=http://couchdb:5984/gpii [-e CLEAR_INDEX=1] -v /home/vagrant/sync/universal:/universal -e NODE_PATH=/universal gpii/gpii-dataloader +$ docker run --name dataloader --link couchdb -v /home/vagrant/sync/universal/testData/dbData:/static_data -e STATIC_DATA_DIR=/static_data -v /home/vagrant/sync/universal/build/dbData:/build_data -e BUILD_DATA_DIR=/build_data -e COUCHDB_URL=http://couchdb:5984/gpii -e CLEAR_INDEX=1 gpii/gpii-dataloader ``` + +The second version has `CLEAR_INDEX` set to nothing such that any existing database is left intact prior to subsequent changes to it (e.g., deleting the snapsets): + +``` +$ docker run --name dataloader --link couchdb -v /home/vagrant/sync/universal/testData/dbData:/static_data -e STATIC_DATA_DIR=/static_data -v /home/vagrant/sync/universal/build/dbData:/build_data -e BUILD_DATA_DIR=/build_data -e COUCHDB_URL=http://couchdb:5984/gpii -e CLEAR_INDEX= gpii/gpii-dataloader +``` +``` \ No newline at end of file diff --git a/deleteAndLoadSnapsets.sh b/deleteAndLoadSnapsets.sh index e98ffac..7dfee22 100755 --- a/deleteAndLoadSnapsets.sh +++ b/deleteAndLoadSnapsets.sh @@ -2,7 +2,6 @@ STATIC_DATA_DIR=${STATIC_DATA_DIR:-/home/node/universal/testData/dbData} BUILD_DATA_DIR=${BUILD_DATA_DIR:-/home/node/universal/build/dbData} -NODE_PATH=${NODE_PATH:-/home/node/universal} log() { echo "$(date +'%Y-%m-%d %H:%M:%S') - $1" @@ -39,15 +38,16 @@ log "Node path: $NODE_PATH" log "Working directory: `pwd`" # Set up universal -git clone https://github.com/GPII/universal.git +git clone --depth 1 https://github.com/GPII/universal.git cd universal -rm -f package-lock.json + npm install json5 npm install fs npm install rimraf npm install mkdirp +npm install infusion +rm -f package-lock.json node scripts/convertPrefs.js testData/preferences/ build/dbData/ -cd - # Initialize (possibly clear) data base if [ ! -z "$CLEAR_INDEX" ]; then @@ -63,6 +63,6 @@ if ! curl -fsS -X PUT "$COUCHDB_URL"; then fi # Submit data -node $NODE_PATH/scripts/deleteSnapsets.js $COUCHDB_URL +node scripts/deleteSnapsets.js $COUCHDB_URL loadData $STATIC_DATA_DIR loadData $BUILD_DATA_DIR From d42e2a50a3488ae60f42dd033c439edb226632cf Mon Sep 17 00:00:00 2001 From: Joseph Scheuhammer Date: Fri, 20 Jul 2018 14:00:15 -0400 Subject: [PATCH 14/32] GPII-3138: Update snapsets in the database Removed old log message regarding NODE_PATH. --- deleteAndLoadSnapsets.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/deleteAndLoadSnapsets.sh b/deleteAndLoadSnapsets.sh index 7dfee22..49d1e77 100755 --- a/deleteAndLoadSnapsets.sh +++ b/deleteAndLoadSnapsets.sh @@ -34,7 +34,6 @@ log "Starting" log "Clear index: $CLEAR_INDEX" log "Static: $STATIC_DATA_DIR" log "Build: $BUILD_DATA_DIR" -log "Node path: $NODE_PATH" log "Working directory: `pwd`" # Set up universal From a730327bf942df6f70491d4e92573a16f9e82c15 Mon Sep 17 00:00:00 2001 From: Joseph Scheuhammer Date: Mon, 23 Jul 2018 10:52:47 -0400 Subject: [PATCH 15/32] GPII-3138: Update snapsets in the database Modified bash script in light of changes to universal's "deleteAndLoadSnapsets.js" node script. --- deleteAndLoadSnapsets.sh | 22 +--------------------- 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/deleteAndLoadSnapsets.sh b/deleteAndLoadSnapsets.sh index 49d1e77..7ff73ba 100755 --- a/deleteAndLoadSnapsets.sh +++ b/deleteAndLoadSnapsets.sh @@ -7,24 +7,6 @@ log() { echo "$(date +'%Y-%m-%d %H:%M:%S') - $1" } -loadData() { - log "Loading data from $1" - - for file in $1/*.json; do - log "Submitting $file" - - curl -H 'Content-Type: application/json' -X POST "$COUCHDB_URL/_bulk_docs" -d @- < Date: Tue, 24 Jul 2018 13:42:52 -0400 Subject: [PATCH 16/32] GPII-3138: Update snapsets in the database. Bash script now checks that the static and build data folders exist and contain data. If not, the script exits with an error message. --- deleteAndLoadSnapsets.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/deleteAndLoadSnapsets.sh b/deleteAndLoadSnapsets.sh index 7ff73ba..0419ed2 100755 --- a/deleteAndLoadSnapsets.sh +++ b/deleteAndLoadSnapsets.sh @@ -7,12 +7,24 @@ log() { echo "$(date +'%Y-%m-%d %H:%M:%S') - $1" } +# Verify variables if [ -z "$COUCHDB_URL" ]; then echo "COUCHDB_URL environment variable must be defined" exit 1 fi +if [ ! -d "$STATIC_DATA_DIR" -o ! "$(ls -A $STATIC_DATA_DIR/*.json)" ]; then + echo "STATIC_DATA_DIR ($STATIC_DATA_DIR) must exist and contain data" + exit 1 +fi + +if [ ! -d "$BUILD_DATA_DIR" -o ! "$(ls -A $BUILD_DATA_DIR/*.json)" ]; then + echo "BUILD_DATA_DIR ($BUILD_DATA_DIR) must exist and contain data" + exit 1 +fi + log "Starting" +log "CouchDB: $COUCHDB_URL" log "Clear index: $CLEAR_INDEX" log "Static: $STATIC_DATA_DIR" log "Build: $BUILD_DATA_DIR" From 933d03fbc09aff72a82910a088639979b41d3ae6 Mon Sep 17 00:00:00 2001 From: Joseph Scheuhammer Date: Mon, 30 Jul 2018 15:37:22 -0400 Subject: [PATCH 17/32] GPII-3138: Docker image that updates snapset in the database Modified in light of changes to universal where both "user" and "snapset" Prefs Safes are created. The gpii-dataloader docker image updates only "snapset" Prefs Safes. --- deleteAndLoadSnapsets.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deleteAndLoadSnapsets.sh b/deleteAndLoadSnapsets.sh index 0419ed2..07497a2 100755 --- a/deleteAndLoadSnapsets.sh +++ b/deleteAndLoadSnapsets.sh @@ -1,7 +1,7 @@ #!/bin/sh STATIC_DATA_DIR=${STATIC_DATA_DIR:-/home/node/universal/testData/dbData} -BUILD_DATA_DIR=${BUILD_DATA_DIR:-/home/node/universal/build/dbData} +BUILD_DATA_DIR=${BUILD_DATA_DIR:-/home/node/universal/build/dbData/snapset} log() { echo "$(date +'%Y-%m-%d %H:%M:%S') - $1" @@ -40,7 +40,7 @@ npm install rimraf npm install mkdirp npm install infusion rm -f package-lock.json -node scripts/convertPrefs.js testData/preferences/ build/dbData/ +node scripts/convertPrefs.js testData/preferences/ build/dbData/snapset snapset # Initialize (possibly clear) data base if [ ! -z "$CLEAR_INDEX" ]; then From 0ba06e48a133e7d7ffc5b86bee95b646b5ccd8af Mon Sep 17 00:00:00 2001 From: Joseph Scheuhammer Date: Tue, 31 Jul 2018 10:16:53 -0400 Subject: [PATCH 18/32] GPII-3138: Docker image that updates snapset in the database Removed sensitive information from CouchDB url when logged to the console. --- deleteAndLoadSnapsets.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/deleteAndLoadSnapsets.sh b/deleteAndLoadSnapsets.sh index 07497a2..c670604 100755 --- a/deleteAndLoadSnapsets.sh +++ b/deleteAndLoadSnapsets.sh @@ -23,8 +23,10 @@ if [ ! -d "$BUILD_DATA_DIR" -o ! "$(ls -A $BUILD_DATA_DIR/*.json)" ]; then exit 1 fi +COUCHDB_URL_SANITIZED=`echo "$COUCHDB_URL" | sed -e 's,\(://\)[^/]*\(@\),\1\2,g'` + log "Starting" -log "CouchDB: $COUCHDB_URL" +log "CouchDB: $COUCHDB_URL_SANITIZED" log "Clear index: $CLEAR_INDEX" log "Static: $STATIC_DATA_DIR" log "Build: $BUILD_DATA_DIR" @@ -44,13 +46,13 @@ node scripts/convertPrefs.js testData/preferences/ build/dbData/snapset snapset # Initialize (possibly clear) data base if [ ! -z "$CLEAR_INDEX" ]; then - log "Deleting database at $COUCHDB_URL" + log "Deleting database at $COUCHDB_URL_SANITIZED" if ! curl -fsS -X DELETE "$COUCHDB_URL"; then log "Error deleting database" fi fi -log "Creating database at $COUCHDB_URL" +log "Creating database at $COUCHDB_URL_SANITIZED" if ! curl -fsS -X PUT "$COUCHDB_URL"; then log "Database already exists at $COUCHDB_URL" fi From 9bb3f46a907cb19399eedba1a75e9c0fa24c8973 Mon Sep 17 00:00:00 2001 From: Joseph Scheuhammer Date: Tue, 31 Jul 2018 16:37:06 -0400 Subject: [PATCH 19/32] GPII-3138: Docker image that updates snapset in the database Updated to handle demo 'user' Prefs Safes. --- deleteAndLoadSnapsets.sh | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/deleteAndLoadSnapsets.sh b/deleteAndLoadSnapsets.sh index c670604..b810ba6 100755 --- a/deleteAndLoadSnapsets.sh +++ b/deleteAndLoadSnapsets.sh @@ -2,6 +2,7 @@ STATIC_DATA_DIR=${STATIC_DATA_DIR:-/home/node/universal/testData/dbData} BUILD_DATA_DIR=${BUILD_DATA_DIR:-/home/node/universal/build/dbData/snapset} +BUILD_DEMOUSER_DIR=${BUILD_DEMOUSER_DIR:-/home/node/universal/build/dbData/demouser} log() { echo "$(date +'%Y-%m-%d %H:%M:%S') - $1" @@ -23,6 +24,11 @@ if [ ! -d "$BUILD_DATA_DIR" -o ! "$(ls -A $BUILD_DATA_DIR/*.json)" ]; then exit 1 fi +if [ ! -d "$BUILD_DEMOUSER_DIR" -o ! "$(ls -A $BUILD_DEMOUSER_DIR/*.json)" ]; then + echo "BUILD_DATA_DIR ($BUILD_DEMOUSER_DIR) must exist and contain data" + exit 1 +fi + COUCHDB_URL_SANITIZED=`echo "$COUCHDB_URL" | sed -e 's,\(://\)[^/]*\(@\),\1\2,g'` log "Starting" @@ -30,6 +36,7 @@ log "CouchDB: $COUCHDB_URL_SANITIZED" log "Clear index: $CLEAR_INDEX" log "Static: $STATIC_DATA_DIR" log "Build: $BUILD_DATA_DIR" +log "Demo User: $BUILD_DEMOUSER_DIR" log "Working directory: `pwd`" # Set up universal @@ -42,7 +49,8 @@ npm install rimraf npm install mkdirp npm install infusion rm -f package-lock.json -node scripts/convertPrefs.js testData/preferences/ build/dbData/snapset snapset +node scripts/convertPrefs.js testData/preferences/ build/dbData/snapset/ snapset +node scripts/convertPrefs.js testData/preferences/demoUserPrefs/ build/dbData/demouser/ user # Initialize (possibly clear) data base if [ ! -z "$CLEAR_INDEX" ]; then @@ -58,4 +66,4 @@ if ! curl -fsS -X PUT "$COUCHDB_URL"; then fi # Submit data -node scripts/deleteAndLoadSnapsets.js $COUCHDB_URL $STATIC_DATA_DIR $BUILD_DATA_DIR +node scripts/deleteAndLoadSnapsets.js $COUCHDB_URL $STATIC_DATA_DIR $BUILD_DATA_DIR $BUILD_DEMOUSER_DIR From 8c60cd82f64cf38c8d773992bbb2a7cd3fa04700 Mon Sep 17 00:00:00 2001 From: Joseph Scheuhammer Date: Wed, 1 Aug 2018 11:04:35 -0400 Subject: [PATCH 20/32] GPII-3138: Docker image that updates snapset in the database Updated README.md based on changes due to demo user preferences. --- README.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 3acd90b..3c219e2 100644 --- a/README.md +++ b/README.md @@ -2,11 +2,13 @@ Builds a [sidecar container](http://blog.kubernetes.io/2015/06/the-distributed-system-toolkit-patterns.html) that contains the `git` command and a shell script for setting up a CouchDB database. When the docker image is run, this sequence is executed: 1. Clones the latest version of [GPII universal](https://github.com/gpii/universal/), -1. Converts the preferences in universal into snapset Prefs Safes and GPII Keys, +1. Converts the preferences in universal into `snapset` Prefs Safes and GPII Keys, +1. Converts the demo user preferences into `user` Prefs Safes and GPII Keys, 1. Creates a CouchDB database if none exits, 1. Optionally clears an existing database of all its records, 1. Deletes any snapsets currently in the database, 1. Loads the latest snapsets created at the second step into the database. +1. Loads the latest demo user preferences created at the third step into the database. ## Building @@ -17,7 +19,8 @@ Builds a [sidecar container](http://blog.kubernetes.io/2015/06/the-distributed-s - `COUCHDB_URL`: URL of the CouchDB database. (required) - `CLEAR_INDEX`: If defined, the database at $COUCHDB_URL will be deleted and recreated. (optional) - `STATIC_DATA_DIR`: The directory where the static data to be loaded into CouchDB resides. (optional) -- `BUILD_DATA_DIR`: The directory where the data built from a npm step resides. (optional) +- `BUILD_DATA_DIR`: The directory where the data built from the conversion step resides. (optional) +- `BUILD_DEMOUSER_DIR`: The directory where data built from converting the demo user preferences resides. (optional) The use of environment variables for data directories is useful if you want to mount the database data using a Docker volume and point the data loader at it. @@ -37,12 +40,11 @@ $ docker run -d -p 8081:8081 --name preferences --link couchdb -e NODE_ENV=gpii. Below are two versions of loading couchdb data from a different location (e.g. /home/vagrant/sync/universal/testData/dbData for static data directory and /home/vagrant/sync/universal/build/dbData for build data directory). The first version has the optional `CLEAR_INDEX` set to erase and reset the database prior to other database changes: ``` -$ docker run --name dataloader --link couchdb -v /home/vagrant/sync/universal/testData/dbData:/static_data -e STATIC_DATA_DIR=/static_data -v /home/vagrant/sync/universal/build/dbData:/build_data -e BUILD_DATA_DIR=/build_data -e COUCHDB_URL=http://couchdb:5984/gpii -e CLEAR_INDEX=1 gpii/gpii-dataloader +$ docker run --name dataloader --link couchdb -v /home/vagrant/sync/universal/testData/dbData:/static_data -e STATIC_DATA_DIR=/static_data -v /home/vagrant/sync/universal/build/dbData:/build_data -e BUILD_DATA_DIR=/build_data -v /home/vagrant/sync/universal/build/dbData/demouser -e BUILD_DEMOUSER_DIR=/build_data/demouser -e COUCHDB_URL=http://couchdb:5984/gpii -e CLEAR_INDEX=1 gpii/gpii-dataloader ``` The second version has `CLEAR_INDEX` set to nothing such that any existing database is left intact prior to subsequent changes to it (e.g., deleting the snapsets): ``` -$ docker run --name dataloader --link couchdb -v /home/vagrant/sync/universal/testData/dbData:/static_data -e STATIC_DATA_DIR=/static_data -v /home/vagrant/sync/universal/build/dbData:/build_data -e BUILD_DATA_DIR=/build_data -e COUCHDB_URL=http://couchdb:5984/gpii -e CLEAR_INDEX= gpii/gpii-dataloader -``` +$ docker run --name dataloader --link couchdb -v /home/vagrant/sync/universal/testData/dbData:/static_data -e STATIC_DATA_DIR=/static_data -v /home/vagrant/sync/universal/build/dbData:/build_data -e BUILD_DATA_DIR=/build_data -v /home/vagrant/sync/universal/build/dbData/demouser -e BUILD_DEMOUSER_DIR=/build_data/demouser -e COUCHDB_URL=http://couchdb:5984/gpii -e CLEAR_INDEX= gpii/gpii-dataloader ``` \ No newline at end of file From ab35aa166be577d2787d01f080b1cd793440bb15 Mon Sep 17 00:00:00 2001 From: Joseph Scheuhammer Date: Sun, 5 Aug 2018 15:51:17 -0400 Subject: [PATCH 21/32] GPII-3138: Docker image that updates snapset in the database Modified to not bail if the provided static, build, or demo user preferences directories are non-existent or do not contain data. Each directory is checked individually and if invalid, the script uses the version of the directory within the cloned universal repository. --- deleteAndLoadSnapsets.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/deleteAndLoadSnapsets.sh b/deleteAndLoadSnapsets.sh index b810ba6..fcfa946 100755 --- a/deleteAndLoadSnapsets.sh +++ b/deleteAndLoadSnapsets.sh @@ -15,18 +15,18 @@ if [ -z "$COUCHDB_URL" ]; then fi if [ ! -d "$STATIC_DATA_DIR" -o ! "$(ls -A $STATIC_DATA_DIR/*.json)" ]; then - echo "STATIC_DATA_DIR ($STATIC_DATA_DIR) must exist and contain data" - exit 1 + echo "STATIC_DATA_DIR ($STATIC_DATA_DIR) does not exist or does not contain data, using universal's 'testData/dbData' as the default" + STATIC_DATA_DIR=./testData/dbData fi if [ ! -d "$BUILD_DATA_DIR" -o ! "$(ls -A $BUILD_DATA_DIR/*.json)" ]; then - echo "BUILD_DATA_DIR ($BUILD_DATA_DIR) must exist and contain data" - exit 1 + echo "BUILD_DATA_DIR ($BUILD_DATA_DIR) does not exist or does not contain data, using universal's 'build/dbData/snapset' as the default" + BUILD_DATA_DIR=./build/dbData/snapset fi if [ ! -d "$BUILD_DEMOUSER_DIR" -o ! "$(ls -A $BUILD_DEMOUSER_DIR/*.json)" ]; then - echo "BUILD_DATA_DIR ($BUILD_DEMOUSER_DIR) must exist and contain data" - exit 1 + echo "BUILD_DEMOUSER_DIR ($BUILD_DEMOUSER_DIR) does not exist or does not contain data, using universal's 'build/dbData/demouser' as the default" + BUILD_DEMOUSER_DIR=./build/dbData/demouser fi COUCHDB_URL_SANITIZED=`echo "$COUCHDB_URL" | sed -e 's,\(://\)[^/]*\(@\),\1\2,g'` From 0034b4db386fcd37170f5cd85cc209ed584b3ea3 Mon Sep 17 00:00:00 2001 From: Joseph Scheuhammer Date: Tue, 7 Aug 2018 15:36:44 -0400 Subject: [PATCH 22/32] GPII-3138: Docker image that updates snapset in the database Modified to use curl command within bash to load just the static data. Loading static data using the "deleteAndLoadSnapsets.js" node script results in a permissions error, whereas using bash/curl does not. But, also, the node script is for modifying snapsets, not "updating" the static data. --- deleteAndLoadSnapsets.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/deleteAndLoadSnapsets.sh b/deleteAndLoadSnapsets.sh index fcfa946..fb23ee3 100755 --- a/deleteAndLoadSnapsets.sh +++ b/deleteAndLoadSnapsets.sh @@ -8,6 +8,24 @@ log() { echo "$(date +'%Y-%m-%d %H:%M:%S') - $1" } +loadStaticData() { + log "Loading static data from $STATIC_DATA_DIR" + + for file in $STATIC_DATA_DIR/*.json; do + log "Submitting $file" + + curl -H 'Content-Type: application/json' -X POST "$COUCHDB_URL/_bulk_docs" -d @- < Date: Tue, 7 Aug 2018 16:02:15 -0400 Subject: [PATCH 23/32] GPII-3138: Docker image that updates snapset in the database Followup to previous commit: removed STATIC_DATA_DIR argument from call to deleteAndLoadSnapsets.js node script. --- deleteAndLoadSnapsets.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deleteAndLoadSnapsets.sh b/deleteAndLoadSnapsets.sh index fb23ee3..eee5776 100755 --- a/deleteAndLoadSnapsets.sh +++ b/deleteAndLoadSnapsets.sh @@ -85,4 +85,4 @@ fi # Submit data loadStaticData -node scripts/deleteAndLoadSnapsets.js $COUCHDB_URL $STATIC_DATA_DIR $BUILD_DATA_DIR $BUILD_DEMOUSER_DIR +node scripts/deleteAndLoadSnapsets.js $COUCHDB_URL $BUILD_DATA_DIR $BUILD_DEMOUSER_DIR From c9cf927305452bd60a2b1edd3455e5f9775e3082 Mon Sep 17 00:00:00 2001 From: Joseph Scheuhammer Date: Mon, 10 Sep 2018 16:55:13 -0400 Subject: [PATCH 24/32] GPII-3138: Docker image that updates snapsets in the database Moved loading/updating of static data to universal's deleteAndLoadSnapsets.js script. --- deleteAndLoadSnapsets.sh | 21 +-------------------- 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/deleteAndLoadSnapsets.sh b/deleteAndLoadSnapsets.sh index 23649d6..5634a9f 100755 --- a/deleteAndLoadSnapsets.sh +++ b/deleteAndLoadSnapsets.sh @@ -18,24 +18,6 @@ warm_indices(){ log "Finished warming indices..." } -loadStaticData() { - log "Loading static data from $STATIC_DATA_DIR" - - for file in $STATIC_DATA_DIR/*.json; do - log "Submitting $file" - - curl -H 'Content-Type: application/json' -X POST "$COUCHDB_URL/_bulk_docs" -d @- < Date: Wed, 12 Sep 2018 10:51:41 -0400 Subject: [PATCH 25/32] GPII-3138: Docker image that updates snapsets in the database Bash script loads universal from klown's GPII-3138 branch. NB: This needs to change to gpii master universal branch when merged. --- deleteAndLoadSnapsets.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deleteAndLoadSnapsets.sh b/deleteAndLoadSnapsets.sh index 5634a9f..0f52187 100755 --- a/deleteAndLoadSnapsets.sh +++ b/deleteAndLoadSnapsets.sh @@ -50,7 +50,7 @@ log "Demo User: $BUILD_DEMOUSER_DIR" log "Working directory: `pwd`" # Set up universal -git clone --depth 1 https://github.com/GPII/universal.git +git clone --depth 1 --branch GPII-3138 https://github.com/klown/universal.git cd universal npm install json5 From c14879c1ca5da65e8b5b1a942933ac3e4008fee9 Mon Sep 17 00:00:00 2001 From: Joseph Scheuhammer Date: Wed, 12 Sep 2018 13:53:00 -0400 Subject: [PATCH 26/32] GPII-3138: Docker image that updates snapsets in the database Removed lines that had to do with the 20 empty user preference, and "Roy" and "Quint", as they are handled elsewhere. --- deleteAndLoadSnapsets.sh | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/deleteAndLoadSnapsets.sh b/deleteAndLoadSnapsets.sh index 0f52187..e8e7a56 100755 --- a/deleteAndLoadSnapsets.sh +++ b/deleteAndLoadSnapsets.sh @@ -2,7 +2,6 @@ STATIC_DATA_DIR=${STATIC_DATA_DIR:-/home/node/universal/testData/dbData} BUILD_DATA_DIR=${BUILD_DATA_DIR:-/home/node/universal/build/dbData/snapset} -BUILD_DEMOUSER_DIR=${BUILD_DEMOUSER_DIR:-/home/node/universal/build/dbData/demouser} log() { echo "$(date +'%Y-%m-%d %H:%M:%S') - $1" @@ -34,11 +33,6 @@ if [ ! -d "$BUILD_DATA_DIR" -o ! "$(ls -A $BUILD_DATA_DIR/*.json)" ]; then BUILD_DATA_DIR=./build/dbData/snapset fi -if [ ! -d "$BUILD_DEMOUSER_DIR" -o ! "$(ls -A $BUILD_DEMOUSER_DIR/*.json)" ]; then - echo "BUILD_DEMOUSER_DIR ($BUILD_DEMOUSER_DIR) does not exist or does not contain data, using universal's 'build/dbData/demouser' as the default" - BUILD_DEMOUSER_DIR=./build/dbData/demouser -fi - COUCHDB_URL_SANITIZED=`echo "$COUCHDB_URL" | sed -e 's,\(://\)[^/]*\(@\),\1\2,g'` log "Starting" @@ -46,7 +40,6 @@ log "CouchDB: $COUCHDB_URL_SANITIZED" log "Clear index: $CLEAR_INDEX" log "Static: $STATIC_DATA_DIR" log "Build: $BUILD_DATA_DIR" -log "Demo User: $BUILD_DEMOUSER_DIR" log "Working directory: `pwd`" # Set up universal @@ -60,7 +53,6 @@ npm install mkdirp npm install infusion rm -f package-lock.json node scripts/convertPrefs.js testData/preferences/ build/dbData/snapset/ snapset -node scripts/convertPrefs.js testData/preferences/demoUserPrefs/ build/dbData/demouser/ user # Initialize (possibly clear) data base if [ ! -z "$CLEAR_INDEX" ]; then @@ -76,7 +68,7 @@ if ! curl -fsS -X PUT "$COUCHDB_URL"; then fi # Submit data -node scripts/deleteAndLoadSnapsets.js $COUCHDB_URL $STATIC_DATA_DIR $BUILD_DATA_DIR $BUILD_DEMOUSER_DIR +node scripts/deleteAndLoadSnapsets.js $COUCHDB_URL $STATIC_DATA_DIR $BUILD_DATA_DIR # Warm Data warm_indices From b7bea343821f75131aa6f0ae1b385f35dd930fb4 Mon Sep 17 00:00:00 2001 From: Joseph Scheuhammer Date: Tue, 18 Sep 2018 14:23:55 -0400 Subject: [PATCH 27/32] GPII-3138: Docker image that updates snapsets in the database Fixed a case of logging sensitive information from the CouchDB URL (username and password). --- deleteAndLoadSnapsets.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deleteAndLoadSnapsets.sh b/deleteAndLoadSnapsets.sh index e8e7a56..6d912d8 100755 --- a/deleteAndLoadSnapsets.sh +++ b/deleteAndLoadSnapsets.sh @@ -64,7 +64,7 @@ fi log "Creating database at $COUCHDB_URL_SANITIZED" if ! curl -fsS -X PUT "$COUCHDB_URL"; then - log "Database already exists at $COUCHDB_URL" + log "Database already exists at $COUCHDB_URL_SANITIZED" fi # Submit data From 9b4b3f1322d48daac6c61739bcc57dec211fc095 Mon Sep 17 00:00:00 2001 From: Joseph Scheuhammer Date: Thu, 20 Sep 2018 14:08:10 -0400 Subject: [PATCH 28/32] GPII-3138: Docker image that updates snapsets in the database Modified to exit early with a non-zero status if the deleteAndLoadSnapsets.js fails. --- deleteAndLoadSnapsets.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/deleteAndLoadSnapsets.sh b/deleteAndLoadSnapsets.sh index 6d912d8..91b9386 100755 --- a/deleteAndLoadSnapsets.sh +++ b/deleteAndLoadSnapsets.sh @@ -69,6 +69,11 @@ fi # Submit data node scripts/deleteAndLoadSnapsets.js $COUCHDB_URL $STATIC_DATA_DIR $BUILD_DATA_DIR +err=$? +if [ $err ]; then + log "deleteAndLoadSnapsets.js failed with $err, exiting" + exit $err +fi # Warm Data warm_indices From 0ec50b3c2a3b2bafa22821f348178dbafadcde52 Mon Sep 17 00:00:00 2001 From: Joseph Scheuhammer Date: Thu, 20 Sep 2018 14:08:10 -0400 Subject: [PATCH 29/32] GPII-3138: Docker image that updates snapsets in the database Modified to exit early with a non-zero status if the deleteAndLoadSnapsets.js fails. --- deleteAndLoadSnapsets.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/deleteAndLoadSnapsets.sh b/deleteAndLoadSnapsets.sh index 6d912d8..f47cab3 100755 --- a/deleteAndLoadSnapsets.sh +++ b/deleteAndLoadSnapsets.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -ex STATIC_DATA_DIR=${STATIC_DATA_DIR:-/home/node/universal/testData/dbData} BUILD_DATA_DIR=${BUILD_DATA_DIR:-/home/node/universal/build/dbData/snapset} @@ -69,6 +69,11 @@ fi # Submit data node scripts/deleteAndLoadSnapsets.js $COUCHDB_URL $STATIC_DATA_DIR $BUILD_DATA_DIR +err=$? +if [ $err != 0 ]; then + log "deleteAndLoadSnapsets.js failed with $err, exiting" + exit $err +fi # Warm Data warm_indices From a1479326a10f8f1f24741b09d35ed43c8307e8bc Mon Sep 17 00:00:00 2001 From: Joseph Scheuhammer Date: Thu, 20 Sep 2018 14:57:05 -0400 Subject: [PATCH 30/32] GPII-3138: Docker image that updates snapsets in the database Removed extra logging. --- deleteAndLoadSnapsets.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deleteAndLoadSnapsets.sh b/deleteAndLoadSnapsets.sh index f47cab3..187bdb1 100755 --- a/deleteAndLoadSnapsets.sh +++ b/deleteAndLoadSnapsets.sh @@ -1,4 +1,4 @@ -#!/bin/sh -ex +#!/bin/sh STATIC_DATA_DIR=${STATIC_DATA_DIR:-/home/node/universal/testData/dbData} BUILD_DATA_DIR=${BUILD_DATA_DIR:-/home/node/universal/build/dbData/snapset} From ab4d265c30c24f00e8572cbc4876b31f7b5b2687 Mon Sep 17 00:00:00 2001 From: Joseph Scheuhammer Date: Thu, 27 Sep 2018 13:55:17 -0400 Subject: [PATCH 31/32] GPII-3138: Docker image that updates snapsets in the database Updated the README.md to remove references to uploading "user" preferences (PrefsSafes and their GPII Keys) as the dataloader is no longer responsible for that. --- README.md | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 3c219e2..828d196 100644 --- a/README.md +++ b/README.md @@ -3,12 +3,11 @@ Builds a [sidecar container](http://blog.kubernetes.io/2015/06/the-distributed-system-toolkit-patterns.html) that contains the `git` command and a shell script for setting up a CouchDB database. When the docker image is run, this sequence is executed: 1. Clones the latest version of [GPII universal](https://github.com/gpii/universal/), 1. Converts the preferences in universal into `snapset` Prefs Safes and GPII Keys, -1. Converts the demo user preferences into `user` Prefs Safes and GPII Keys, 1. Creates a CouchDB database if none exits, 1. Optionally clears an existing database of all its records, +1. Updates the database with respect to its `design/views` document, as required, 1. Deletes any snapsets currently in the database, 1. Loads the latest snapsets created at the second step into the database. -1. Loads the latest demo user preferences created at the third step into the database. ## Building @@ -20,7 +19,6 @@ Builds a [sidecar container](http://blog.kubernetes.io/2015/06/the-distributed-s - `CLEAR_INDEX`: If defined, the database at $COUCHDB_URL will be deleted and recreated. (optional) - `STATIC_DATA_DIR`: The directory where the static data to be loaded into CouchDB resides. (optional) - `BUILD_DATA_DIR`: The directory where the data built from the conversion step resides. (optional) -- `BUILD_DEMOUSER_DIR`: The directory where data built from converting the demo user preferences resides. (optional) The use of environment variables for data directories is useful if you want to mount the database data using a Docker volume and point the data loader at it. @@ -40,11 +38,11 @@ $ docker run -d -p 8081:8081 --name preferences --link couchdb -e NODE_ENV=gpii. Below are two versions of loading couchdb data from a different location (e.g. /home/vagrant/sync/universal/testData/dbData for static data directory and /home/vagrant/sync/universal/build/dbData for build data directory). The first version has the optional `CLEAR_INDEX` set to erase and reset the database prior to other database changes: ``` -$ docker run --name dataloader --link couchdb -v /home/vagrant/sync/universal/testData/dbData:/static_data -e STATIC_DATA_DIR=/static_data -v /home/vagrant/sync/universal/build/dbData:/build_data -e BUILD_DATA_DIR=/build_data -v /home/vagrant/sync/universal/build/dbData/demouser -e BUILD_DEMOUSER_DIR=/build_data/demouser -e COUCHDB_URL=http://couchdb:5984/gpii -e CLEAR_INDEX=1 gpii/gpii-dataloader +$ docker run --name dataloader --link couchdb -v /home/vagrant/sync/universal/testData/dbData:/static_data -e STATIC_DATA_DIR=/static_data -v /home/vagrant/sync/universal/build/dbData:/build_data -e BUILD_DATA_DIR=/build_data -e COUCHDB_URL=http://couchdb:5984/gpii -e CLEAR_INDEX=1 gpii/gpii-dataloader ``` The second version has `CLEAR_INDEX` set to nothing such that any existing database is left intact prior to subsequent changes to it (e.g., deleting the snapsets): ``` -$ docker run --name dataloader --link couchdb -v /home/vagrant/sync/universal/testData/dbData:/static_data -e STATIC_DATA_DIR=/static_data -v /home/vagrant/sync/universal/build/dbData:/build_data -e BUILD_DATA_DIR=/build_data -v /home/vagrant/sync/universal/build/dbData/demouser -e BUILD_DEMOUSER_DIR=/build_data/demouser -e COUCHDB_URL=http://couchdb:5984/gpii -e CLEAR_INDEX= gpii/gpii-dataloader -``` \ No newline at end of file +$ docker run --name dataloader --link couchdb -v /home/vagrant/sync/universal/testData/dbData:/static_data -e STATIC_DATA_DIR=/static_data -v /home/vagrant/sync/universal/build/dbData:/build_data -e BUILD_DATA_DIR=/build_data -e COUCHDB_URL=http://couchdb:5984/gpii -e CLEAR_INDEX= gpii/gpii-dataloader +``` From 2680cd3e51d820b626f5536d2a729f0d6eb4c1c0 Mon Sep 17 00:00:00 2001 From: Joseph Scheuhammer Date: Wed, 17 Oct 2018 13:04:53 -0400 Subject: [PATCH 32/32] GPII-3138: Docker image that updates snapsets in the database Since universal dataloader pull was merged into universal master, updated to reference that repository. --- deleteAndLoadSnapsets.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deleteAndLoadSnapsets.sh b/deleteAndLoadSnapsets.sh index 187bdb1..5c77246 100755 --- a/deleteAndLoadSnapsets.sh +++ b/deleteAndLoadSnapsets.sh @@ -43,7 +43,7 @@ log "Build: $BUILD_DATA_DIR" log "Working directory: `pwd`" # Set up universal -git clone --depth 1 --branch GPII-3138 https://github.com/klown/universal.git +git clone --depth 1 https://github.com/GPII/universal.git cd universal npm install json5