diff --git a/documentation/DataLoader.md b/documentation/DataLoader.md new file mode 100644 index 000000000..3e8830d69 --- /dev/null +++ b/documentation/DataLoader.md @@ -0,0 +1,82 @@ +## Data Loaders + +Data loaders are used for setting up GPII production environment by loading authorization and/or preferences test data into CouchDB. See [Data Loader issue ticket](https://issues.gpii.net/browse/GPII-1987) for details. + +### Preferences Data Loader + +**Script Location:** `scripts/dataLoader-prefs.js` + +**Infusion Component:** `gpii.dataLoader.prefsDataLoader` + +**Component Source Code:** `gpii/node_modules/dataLoader/src/prefsDataLoader.js` + +Preferences Data Loader reads all JSON files from the given directory. Each JSON file is loaded as a individual CouchDB document with its file name as the corresponding document id. + +To start the data loading of the preferences data, run the following command in the `universal` directory: + +``` +node scripts/dataLoader-prefs.js +``` + +#### Component Options + +| Option | Type | Description | Default | +| ----------------- | ---------- | ----------- | ------- | +| `dbName` | String | Required. The name of the CouchDB database to be created for loading preferences data. If the database already exists, delete it and recreate. | preferences | +| `dataPath` | String | Required. The directory where all preference JSON files reside. | %universal/testData/preferences/ | +| `couchDbUrl` | String | Required. The URL to the CouchDB. | http://localhost:5984 | + +#### Supported Events + +| Event | Description | Parameters | Parameters Description | +| ----- | ----------- | ---------- | ---------------------- | +| `onDataLoaded` | Fires when all preference data has been loaded into the CouchDB. | None | | +| `onDataLoadedError` | Fires when an error occurs at loading preference data. | errorMsg | The error message. | + +### Authorization Data Loader + +**Script Location:** `scripts/dataLoader-auth.js` + +**Infusion Component:** `gpii.dataLoader.authDataLoader` + +**Component Source Code:** `gpii/node_modules/dataLoader/src/authDataLoader.js` + +Authorization Data Loader reads given JSON files from the file system and load them into CouchDB as they are. + +To start the data loading of the authorization data, run the following command in the `universal` directory: + +``` +node scripts/dataLoader-auth.js +``` + +#### Component Options + +| Option | Type | Description | Default | +| ----------------- | ---------- | ----------- | ------- | +| `dbName` | String | Required. The name of the CouchDB database to be created for loading authorization data. If the database already exists, delete it and recreate. | auth | +| `dataFile` | Array | Required. An array of JSON data files to be loaded. See the section below for an example of a JSON data file.| ["%universal/testData/security/TestOAuth2DataStore.json", "%universal/gpii/node_modules/gpii-oauth2/gpii-oauth2-datastore/dbViews/views.json"] | +| `couchDbUrl` | String | Required. The URL to the CouchDB. | `http://localhost:5984` | + +**Example of A JSON Data File** +``` +[ + { + "_id": "user-1", + "type": "user", + "name": "chromehc", + "defaultGpiiToken": "review3_chrome_high_contrast" + }, { + "_id": "user-2", + "type": "user", + "name": "ma1", + "defaultGpiiToken": "review3_ma1" + } +] +``` + +#### Supported Events + +| Event | Description | Parameters | Parameters Description | +| ----- | ----------- | ---------- | ---------------------- | +| `onDataLoaded` | Fires when all preference data has been loaded into the CouchDB. | None | | +| `onDataLoadedError` | Fires when an error occurs at loading preference data. | errorMsg | The error message. | diff --git a/gpii/node_modules/dataLoader/README.md b/gpii/node_modules/dataLoader/README.md new file mode 100644 index 000000000..5a6f13d50 --- /dev/null +++ b/gpii/node_modules/dataLoader/README.md @@ -0,0 +1,10 @@ +Data Loader +=========== + +Data Loaders to load given data or JSON files into CouchDB. + +--- + +### Dependencies + +[infusion](https://github.com/fluid-project/infusion) framework. diff --git a/gpii/node_modules/dataLoader/index.js b/gpii/node_modules/dataLoader/index.js new file mode 100644 index 000000000..fe6057087 --- /dev/null +++ b/gpii/node_modules/dataLoader/index.js @@ -0,0 +1,9 @@ +"use strict"; + +var fluid = require("infusion"); + +fluid.module.register("dataLoader", __dirname, require); + +require("./src/authDataLoader.js"); +require("./src/dataLoader.js"); +require("./src/prefsDataLoader.js"); diff --git a/gpii/node_modules/dataLoader/src/authDataLoader.js b/gpii/node_modules/dataLoader/src/authDataLoader.js new file mode 100644 index 000000000..138abc2e9 --- /dev/null +++ b/gpii/node_modules/dataLoader/src/authDataLoader.js @@ -0,0 +1,48 @@ +/*! +Copyright 2017 OCAD University + +Licensed under the New BSD license. You may not use this file except in +compliance with this License. + +The research leading to these results has received funding from the European Union's +Seventh Framework Programme (FP7/2007-2013) under grant agreement no. 289016. + +You may obtain a copy of the License at +https://github.com/GPII/universal/blob/master/LICENSE.txt +*/ + +"use strict"; + +var fluid = require("infusion"), + gpii = fluid.registerNamespace("gpii"); + +require("./dataLoader.js"); + +fluid.defaults("gpii.dataLoader.authDataLoader", { + gradeNames: ["gpii.dataLoader", "gpii.dataLoader.reporter"], + databases: { + expander: { + funcName: "gpii.dataLoader.authDataLoader.constructAuthData", + args: ["{that}.options.dbName", "{that}.options.dataFile"] + } + }, + listeners: { + "onCreate.load": { + listener: "gpii.dataLoader.performLoad", + args: ["{that}.options.dbName", "{that}"] + } + } +}); + +/** + * Construct the value of `options.databases` that to be accepted by `gpii.dataLoader` (See dataLoader.js). + * + * @param dbName {String} The database name; + * @param dataFile {Array} An array of data paths to files to be loaded into the database. + */ +gpii.dataLoader.authDataLoader.constructAuthData = function (dbName, dataFile) { + var togo = {}; + fluid.set(togo, [dbName, "dataFile"], dataFile); + + return togo; +}; diff --git a/gpii/node_modules/dataLoader/src/dataLoader.js b/gpii/node_modules/dataLoader/src/dataLoader.js new file mode 100644 index 000000000..cd12eac9f --- /dev/null +++ b/gpii/node_modules/dataLoader/src/dataLoader.js @@ -0,0 +1,311 @@ +/*! +Copyright 2016-2017 OCAD University + +Licensed under the New BSD license. You may not use this file except in +compliance with this License. + +The research leading to these results has received funding from the European Union's +Seventh Framework Programme (FP7/2007-2013) under grant agreement no. 289016. + +You may obtain a copy of the License at +https://github.com/GPII/universal/blob/master/LICENSE.txt +*/ + +// The underlying data loader implementation shared by ./authDataLoader.js and ./prefsDataLoader.js +// to load GPII test data into CouchDB. +// Steps to load data: +// 1. Check the existence of the database; +// 2. If the database already exists, delete and recreate the database to ensure a clean data load; otherwise, create the database; +// 3. Load data in bulk using CouchDB /_bulk_docs API - http://docs.couchdb.org/en/2.0.0/api/database/bulk-api.html#db-bulk-docs + +"use strict"; + +var fluid = require("infusion"), + gpii = fluid.registerNamespace("gpii"), + fs = require("fs"); + +fluid.require("kettle", require); + +// Data Source used to interact with CouchDB +fluid.defaults("gpii.dataLoader.dataSource", { + gradeNames: ["kettle.dataSource.URL"], + readOnlyGrade: "gpii.dataLoader.dataSource", + termMap: { + couchDbUrl: "noencode:%couchDbUrl", + dbName: "%dbName" + } +}); + +fluid.defaults("gpii.dataLoader.dataSource.writable", { + gradeNames: ["gpii.dataLoader.dataSource", "kettle.dataSource.URL.writable"], + writable: true +}); + +// Data loader +fluid.defaults("gpii.dataLoader", { + gradeNames: ["fluid.component"], + // Accepted formats: + // dbName1: { + // dataFile: ["pathToDataFile1", "pathToDataFile1"...] + // }, + // dbName2: { + // data: [{ + // "_id": {String}, + // [Other object key-value pairs] + // }, { + // ... + // }...] + // } + databases: { // Supplied by integrators + }, + couchDbUrl: null, // Supplied by integrators + components: { + getDbDataSource: { + type: "gpii.dataLoader.dataSource", + options: { + url: "%couchDbUrl/%dbName" + } + }, + deleteDbDataSource: { + type: "gpii.dataLoader.dataSource", + options: { + url: "%couchDbUrl/%dbName", + method: "DELETE" + } + }, + createDbDataSource: { + type: "gpii.dataLoader.dataSource", + options: { + url: "%couchDbUrl/%dbName", + method: "PUT" + } + }, + loadDataSource: { + type: "gpii.dataLoader.dataSource.writable", + options: { + url: "%couchDbUrl/%dbName/_bulk_docs", + writeMethod: "POST" + } + } + }, + invokers: { + load: { + funcName: "gpii.dataLoader.load", + args: ["{that}"] + } + }, + events: { + onDataLoaded: null, + onDataLoadedError: null + } +}); + +/** + * The main load function that processes the data loading for databases defined in `that.options.databases` one by one. + * The structure of `that.options.databases`: + * { + * dbName1: [data], + * dbName2: [data] + * ... + * } + * The format of [data] is in the structure allowed by CouchDB /_bulk_docs API (http://docs.couchdb.org/en/2.0.0/api/database/bulk-api.html#db-bulk-docs) + * + * @param that {Component} An instance of "gpii.dataLoader" + * @return {Promise} The promise of the data loading process. + */ +gpii.dataLoader.load = function (that) { + var promiseTogo = fluid.promise(); + var loadDataPromises = []; + + // Validate and extract data from data files + var dataFileResults = gpii.dataLoader.processDataFiles(that.options.databases); + + if (dataFileResults.errors.length > 0) { + promiseTogo.reject("Data file(s) not found in the file system: " + dataFileResults.errors.join()); + } else { + var databases = dataFileResults.databases; + // Process databases one by one + fluid.each(databases, function (dbData, dbName) { + var directModel = { + couchDbUrl: that.options.couchDbUrl, + dbName: dbName + }; + + // Delete and recreate the database if it already exists; otherwise, create the database + var prepareDbPromise = gpii.dataLoader.prepareDB(that.getDbDataSource, that.createDbDataSource, that.deleteDbDataSource, directModel); + + prepareDbPromise.then(function () { + // load data + if (dbData) { + loadDataPromises.push(gpii.dataLoader.loadData(that.loadDataSource, dbData, directModel)); + } + + var loadDataPromise = fluid.promise.sequence(loadDataPromises); + fluid.promise.follow(loadDataPromise, promiseTogo); + }); + }); + } + + return promiseTogo; +}; + +/** + * Loop thru each database, to ensure the existence of data files in database.dataFile option. If files exist, + * extract data from data files, concat with data from database.data, and return the concatenated data for each database. + * Otherwise, report errors on non-existent files. + * + * @param databases {Object} The same structure as options.databases of the "gpii.dataLoader" component; + * + * @return {Object} In a structure of {errors: [{String}, ...], databases: [{dbName1: [{data}]}, ...]}. + */ +gpii.dataLoader.processDataFiles = function (databases) { + var errors = []; + // Structure: {database1: {data: []}, ...} + var allData = {}; + + fluid.each(databases, function (database, dbName) { + var data = []; + if (database.dataFile) { + // Loop thru data files to check their existence. + // If doesn't exist, report error. Otherwise, extract the data + fluid.each(database.dataFile, function (oneDataFile) { + var actualPath = fluid.module.resolvePath(oneDataFile); + if (!fs.existsSync(actualPath)) { + errors.push(actualPath); + } else { + data = data.concat(fluid.makeArray(require(actualPath))); + } + }); + } + + // Concat the extracted data from data files with the provided input at database.data + if (database.data) { + data = data.concat(fluid.makeArray(database.data)); + } + + if (data.length > 0) { + fluid.set(allData, [dbName], data); + } + }); + + return { + errors: errors, + databases: allData + }; +}; + +/** + * To create a database. + * + * @param createDbDataSource {DataSource} The data source for creating a database; + * @param directModel {Object} The direct model expressing the "coordinates" of the model to be fetched + * + * @return {Promise} A promise with the response of creating the database. + */ +gpii.dataLoader.createDb = function (createDbDataSource, directModel) { + return createDbDataSource.get(directModel); +}; + +/** + * Prepare the database before loading the data. If the database already exists, delete and recreate it; otherwise, create the database. + * + * @param getDbDataSource {DataSource} The data source for checking the existence of a database; + * @param createDbDataSource {DataSource} The data source for creating a database; + * @param deleteDbDataSource {DataSource} The data source for deleting a database; + * @param directModel {Object} The direct model expressing the "coordinates" of the model to be fetched + * + * @return {Promise} A promise with the response of preparing the database. + */ +gpii.dataLoader.prepareDB = function (getDbDataSource, createDbDataSource, deleteDbDataSource, directModel) { + var promiseTogo = fluid.promise(); + var getDbPromise = getDbDataSource.get(directModel); + + getDbPromise.then(function () { + // The database already exists, delete and recreate to ensure a clean database for loading data + var deleteDbPromise = deleteDbDataSource.get(directModel); + deleteDbPromise.then(function () { + var createDbPromise = gpii.dataLoader.createDb(createDbDataSource, directModel); + fluid.promise.follow(createDbPromise, promiseTogo); + }); + }, function () { + // The database does not exist, create it + var createDbPromise = gpii.dataLoader.createDb(createDbDataSource, directModel); + fluid.promise.follow(createDbPromise, promiseTogo); + }); + + return promiseTogo; +}; + +/** + * Load the data into a database. + * + * @param loadDataSource {DataSource} The data source for loading the data; + * @param data {DataSource} The data to be loaded; + * @param directModel {Object} The direct model expressing the "coordinates" of the model to be fetched + * + * @return {Promise} A promise with the response of loading the data. + */ +gpii.dataLoader.loadData = function (loadDataSource, data, directModel) { + if (!data) { + return fluid.promise(); + } + + // Convert to couchDB accepted doc format for using /_bulk_docs end point + data = { + "docs": data + }; + + return loadDataSource.set(directModel, data); +}; + +/** + * The API function to trigger the loading function to load data. + * @param dbName {String} The database name; + * @param loadFunc {Function} The load() invoker defined for an instance of `gpii.dataLoader` component (See dataLoader.js); + * @return {None} The success or fail result of the loading is reported on the console. + */ +gpii.dataLoader.performLoad = function (dbName, loaderComponent) { + var promise = loaderComponent.load(); + promise.then(function () { + loaderComponent.events.onDataLoaded.fire(); + }, function (err) { + loaderComponent.events.onDataLoadedError.fire(err); + }); +}; + +// A base grade to report the data loading results to the console. +// This grade should be used as a base grade for auth and prefs data loaders +// where the db name is defined as well as proper events. +fluid.defaults("gpii.dataLoader.reporter", { + gradeNames: ["fluid.component"], + dbName: "dbName", // Must be provided + events: { + onDataLoaded: null, + onDataLoadedError: null + }, + listeners: { + "onDataLoaded.report": { + listener: "console.log", + args: { + expander: { + funcName: "fluid.stringTemplate", + args: ["The %dbName data has been loaded successfully.", { + dbName: "{that}.options.dbName" + }] + } + } + }, + "onDataLoadedError.report": { + listener: "console.log", + args: { + expander: { + funcName: "fluid.stringTemplate", + args: ["Error at loading the %dbName data. Error details: %err", { + dbName: "{that}.options.dbName", + err: "{arguments}.0" + }] + } + } + } + } +}); diff --git a/gpii/node_modules/dataLoader/src/prefsDataLoader.js b/gpii/node_modules/dataLoader/src/prefsDataLoader.js new file mode 100644 index 000000000..47b36bd55 --- /dev/null +++ b/gpii/node_modules/dataLoader/src/prefsDataLoader.js @@ -0,0 +1,195 @@ +/*! +Copyright 2016-2017 OCAD University + +Licensed under the New BSD license. You may not use this file except in +compliance with this License. + +The research leading to these results has received funding from the European Union's +Seventh Framework Programme (FP7/2007-2013) under grant agreement no. 289016. + +You may obtain a copy of the License at +https://github.com/GPII/universal/blob/master/LICENSE.txt +*/ + +/** + * The components required by ../prefsDataLoader.js + */ + +"use strict"; + +var fluid = require("infusion"), + gpii = fluid.registerNamespace("gpii"), + fs = require("fs"); + +fluid.setLogging(true); + +require("./dataLoader.js"); + +/** + * Preferences Data Loader: to load the preferences (json files) from a specific path into CouchDB + * This component is composed by two sub-components: + * 1. dataConverter: to read thru the preferences data and construct the json file contents + * into the data structure that can be accepted by CouchDB /_bulk_docs API + * 2. loader: take the data structure created by dataConverter and load into CouchDB. Using + * "gpii.dataLoader" as the underlying grade for loading. + */ +fluid.defaults("gpii.dataLoader.prefsDataLoader", { + gradeNames: ["gpii.dataLoader.reporter"], + + // Below is the list of options for configing the preferences data loader + dbName: "preferences", + dataPath: "%universal/testData/preferences/", + couchDbUrl: "http://localhost:5984", + + components: { + dataConverter: { + type: "gpii.dataLoader.prefsDataLoader.dataConverter", + createOnEvent: "onConvertData", + options: { + listeners: { + "onPrefsDataStructureConstructed.escalate": { + listener: "{prefsDataLoader}.events.onCreateLoader.fire", + args: ["{arguments}.0"] + } + } + } + }, + loader: { + type: "gpii.dataLoader", + createOnEvent: "onCreateLoader", + options: { + data: "{arguments}.0", + databases: { + expander: { + funcName: "gpii.dataLoader.prefsDataLoader.constructDBOption", + args: ["{prefsDataLoader}.options.dbName", "{that}.options.data"] + } + }, + events: { + onDataLoaded: "{prefsDataLoader}.events.onDataLoaded", + onDataLoadedError: "{prefsDataLoader}.events.onDataLoadedError" + }, + listeners: { + "onCreate.escalate": "{prefsDataLoader}.events.onLoaderReady.fire" + } + } + } + }, + events: { + // fired to create the subcomponent "dataConverter" after ensuring the given options are valid + onConvertData: null, + // fired to create the subcomponent "loader" once the data is converted and ready to load + onCreateLoader: null, + // fired to trigger the load process when "loader" is ready + onLoaderReady: null, + // fired after the data is successfully loaded + onDataLoaded: null, + // fired with an error message if the load fails + onDataLoadedError: null + }, + listeners: { + "onCreate.validateInputs": { + listener: "gpii.dataLoader.prefsDataLoader.validateInputs", + args: ["{that}.options.dataPath", "{that}.events.onConvertData", "{that}.events.onDataLoadedError"] + }, + "onLoaderReady.load": { + listener: "gpii.dataLoader.performLoad", + args: ["{that}.options.dbName", "{that}.loader"] + } + }, + distributeOptions: { + "couchDbUrl": { + source: "{that}.options.couchDbUrl", + target: "{that > loader}.options.couchDbUrl" + }, + "dataPath": { + source: "{that}.options.dataPath", + target: "{that > dataConverter}.options.dataPath" + } + } +}); + +/** + * Construct the value of `options.databases` that to be accepted by `gpii.dataLoader` (See dataLoader.js). + * + * @param dbName {String} The database name; + * @param data {Object} The data to be loaded into the database. + */ +gpii.dataLoader.prefsDataLoader.validateInputs = function (dataPath, successEvt, errorEvt) { + var actualPath = fluid.module.resolvePath(dataPath); + if (fs.existsSync(actualPath)) { + successEvt.fire(); + } else { + errorEvt.fire("The provided data path does not exist: " + actualPath); + } +}; + +/** + * Construct the value of `options.databases` that to be accepted by `gpii.dataLoader` (See dataLoader.js). + * + * @param dbName {String} The database name; + * @param data {Object} The data to be loaded into the database. + */ +gpii.dataLoader.prefsDataLoader.constructDBOption = function (dbName, data) { + var togo = {}; + fluid.set(togo, dbName + ".data", data); + + return togo; +}; + +/* + * Converts the physical preferences data files into the structure that can be imported into CouchDB + */ +fluid.defaults("gpii.dataLoader.prefsDataLoader.dataConverter", { + gradeNames: ["fluid.component"], + dataPath: "%universal/testData/preferences/", + events: { + onPrefsDataStructureConstructed: null + }, + listeners: { + "onCreate.constructPrefsDataStructure": { + listener: "gpii.dataLoader.prefsDataLoader.dataConverter.constructPrefsDataStructure", + args: ["{that}.options.dataPath", "{that}.events.onPrefsDataStructureConstructed"] + } + } +}); + +/** + * Read each preferences data from physical json file and construct them into a data structure that is accepted by + * the CouchDB /_bulk_docs API (http://docs.couchdb.org/en/2.0.0/api/database/bulk-api.html#db-bulk-docs) + * + * @param dataPath {String} The data path to where the preferences files are located; + * @param onPrefsDataStructureConstructedEvent {Event} fires when the data structure is constructed. The event is fired with + * an argument that is the constructed data structure. + * @return {Event} fires the data ready event with an argument of the constructed data structure. + */ +gpii.dataLoader.prefsDataLoader.dataConverter.constructPrefsDataStructure = function (dataPath, onPrefsDataStructureConstructedEvent) { + // Pad the dataPath with an ending "/" + dataPath = dataPath.substr(dataPath.length - 1) === "/" ? dataPath : dataPath + "/"; + + var prefsDataStructure = []; + var actualDataPath = fluid.module.resolvePath(dataPath); + var prefsDataList = fs.readdirSync(actualDataPath); + + fluid.each(prefsDataList, function (filename) { + var fileExtension = filename.slice(-5); + // Only json files contain preferences data + if (fileExtension === ".json") { + var gpiiToken = filename.substring(0, filename.length - 5); + var fullPath = actualDataPath + filename; + + var data = JSON.parse(fs.readFileSync(fullPath, "utf8")); + + // The actual preferences data needs to be the value of a field named "value" when being imported into CouchDB + var prefsData = {}; + fluid.set(prefsData, "value", data); + + var finalData = fluid.extend(prefsData, { + "_id": gpiiToken + }); + prefsDataStructure.push(finalData); + } + }); + + onPrefsDataStructureConstructedEvent.fire(prefsDataStructure); +}; diff --git a/gpii/node_modules/dataLoader/test/AuthDataLoaderTests.js b/gpii/node_modules/dataLoader/test/AuthDataLoaderTests.js new file mode 100644 index 000000000..50d445b2f --- /dev/null +++ b/gpii/node_modules/dataLoader/test/AuthDataLoaderTests.js @@ -0,0 +1,151 @@ +/*! + * GPII Auth Data Loader Tests + * + * Copyright 2017 OCAD University + * + * Licensed under the New BSD license. You may not use this file except in + * compliance with this License. + * + * You may obtain a copy of the License at + * https://github.com/GPII/universal/blob/master/LICENSE.txt + */ + +"use strict"; + +var fluid = require("infusion"), + gpii = fluid.registerNamespace("gpii"); + +require("gpii-pouchdb"); +gpii.pouch.loadTestingSupport(); + +require("dataLoader"); +require("./DataLoaderTestsUtils.js"); + +//*********** 1. The successful data loading test case ***********// +fluid.defaults("gpii.tests.authDataLoader.success", { + gradeNames: ["gpii.dataLoader.authDataLoader"], + dbName: "auth", + dataFile: [ + "%universal/testData/security/TestOAuth2DataStore.json", + "%universal/gpii/node_modules/gpii-oauth2/gpii-oauth2-datastore/dbViews/views.json" + ], + couchDbUrl: "http://localhost:1234" +}); + +// This component must be name as "testEnvironment" because of the reason described at ./DataLoaderTestsUtils.js line 19 +fluid.defaults("gpii.tests.authDataLoader.success.testEnvironment", { + gradeNames: ["gpii.tests.dataLoader.testEnvironment"], + dataLoaderGrade: "gpii.tests.authDataLoader.success", + events: { + onFixturesConstructed: { + events: { + onDataLoaded: "onDataLoaded" + } + } + } +}); + +fluid.defaults("gpii.tests.authTestCaseHolder.success", { + gradeNames: ["gpii.tests.dataLoader.baseTestCaseHolder"], + expected: { + total: { + doc_count: 17 + } + }, + rawModules: [{ + name: "Testing Auth Data Loader - success", + tests: [{ + name: "Testing the successful loading", + type: "test", + sequence: [{ + func: "{totalRequest}.send" + }, { + listener: "gpii.tests.dataLoader.checkResponse", + event: "{totalRequest}.events.onComplete", + args: ["The total number of records should be as expected", "{totalRequest}.nativeResponse", "{arguments}.0", 200, "{testCaseHolder}.options.expected.total"] + }] + }] + }], + components: { + totalRequest: { + type: "kettle.test.request.http", + options: { + path: "/auth", + port: 1234, + method: "GET" + } + } + } +}); + +fluid.defaults("gpii.tests.authDataLoaderTests.success", { + gradeNames: ["gpii.tests.authDataLoader.success.testEnvironment"], + components: { + testCaseHolder: { + type: "gpii.tests.authTestCaseHolder.success" + } + } +}); + +//*********** 2. The error data loading test case ***********// +fluid.defaults("gpii.tests.authDataLoader.error", { + gradeNames: ["gpii.dataLoader.authDataLoader"], + dbName: "auth", + dataFile: [ + "%universal/tests/nonExistent.json", + "%universal/gpii/node_modules/gpii-oauth2/gpii-oauth2-datastore/dbViews/views.json" + ], + couchDbUrl: "http://localhost:1234" +}); + +// This component must be name as "testEnvironment" because of the reason described at ./DataLoaderTestsUtils.js line 19 +fluid.defaults("gpii.tests.authDataLoader.error.testEnvironment", { + gradeNames: ["gpii.tests.dataLoader.testEnvironment"], + dataLoaderGrade: "gpii.tests.authDataLoader.error" +}); + +fluid.defaults("gpii.tests.authTestCaseHolder.error", { + gradeNames: ["gpii.tests.dataLoader.baseTestCaseHolder"], + rawModules: [{ + name: "Testing Auth Data Loader - error", + tests: [{ + name: "Testing the non-existence of data files", + type: "test", + sequence: [{ + event: "{testEnvironment}.events.onDataLoadedError", + listener: "gpii.tests.dataLoader.checkError", + args: ["{arguments}.0", "Data file\\(s\\) not found in the file system\:.*nonExistent.json"] + }, { + func: "{verifyDbRequest}.send" + }, { + listener: "jqUnit.assertEquals", + event: "{verifyDbRequest}.events.onComplete", + args: ["The total number of records should be as expected", "{verifyDbRequest}.nativeResponse.statusCode", 404] + }] + }] + }], + components: { + verifyDbRequest: { + type: "kettle.test.request.http", + options: { + path: "/auth", + port: 1234, + method: "GET" + } + } + } +}); + +fluid.defaults("gpii.tests.authDataLoaderTests.error", { + gradeNames: ["gpii.tests.authDataLoader.error.testEnvironment"], + components: { + testCaseHolder: { + type: "gpii.tests.authTestCaseHolder.error" + } + } +}); + +fluid.test.runTests([ + "gpii.tests.authDataLoaderTests.success", + "gpii.tests.authDataLoaderTests.error" +]); diff --git a/gpii/node_modules/dataLoader/test/DataLoaderTests.js b/gpii/node_modules/dataLoader/test/DataLoaderTests.js new file mode 100644 index 000000000..fd39ca0f1 --- /dev/null +++ b/gpii/node_modules/dataLoader/test/DataLoaderTests.js @@ -0,0 +1,187 @@ +/*! + * GPII Preferences Data Loader Tests + * + * Copyright 2016-2017 OCAD University + * + * Licensed under the New BSD license. You may not use this file except in + * compliance with this License. + * + * You may obtain a copy of the License at + * https://github.com/GPII/universal/blob/master/LICENSE.txt + */ + +"use strict"; + +var fluid = require("infusion"), + jqUnit = fluid.require("node-jqunit", require, "jqUnit"), + gpii = fluid.registerNamespace("gpii"); + +require("dataLoader"); +require("./DataLoaderTestsUtils.js"); + +fluid.registerNamespace("gpii.tests.dataLoader"); + +fluid.defaults("gpii.tests.dataLoader.dataConverter", { + gradeNames: ["gpii.dataLoader.prefsDataLoader.dataConverter"], + dataPath: "data/", + expected: [{ + carla: { + "flat": { + "contexts": { + "gpii-default": { + "name": "Default preferences", + "preferences": { + "http://registry.gpii.net/applications/com.texthelp.readWriteGold": { + "ApplicationSettings.AppBar.optToolbarLargeIcons.$t": true, + "ApplicationSettings.Speech.optSAPI5Speed.$t": 50, + "ApplicationSettings.Speech.optAutoUseScreenReading.$t": false + } + } + } + } + } + } + }, { + sammy: { + "flat": { + "contexts": { + "gpii-default": { + "name": "Default preferences", + "preferences": { + "http://registry.gpii.net/common/fontSize": 24, + "http://registry.gpii.net/common/foregroundColor": "white", + "http://registry.gpii.net/common/backgroundColor": "black", + "http://registry.gpii.net/common/fontFaceFontName": ["Comic Sans"], + "http://registry.gpii.net/common/fontFaceGenericFontFace": "sans serif", + "http://registry.gpii.net/common/magnification": 2.0, + "http://registry.gpii.net/common/tracking": ["mouse"], + "http://registry.gpii.net/common/invertColours": true + } + } + } + } + } + }] +}); + +jqUnit.test("Test gpii.dataLoader.processDataFiles", function () { + var testData = { + case1: { + db1: { + dataFile: [ + "%dataLoader/test/data/prefs/carla.json", + "%dataLoader/test/data/views-test.json" + ], + data: { + "test1": "test1" + } + }, + db2: { + data: { + "test2": "test2" + } + } + }, + case2: { + db1: { + dataFile: [ + "%dataLoader/test/data/nonExistent1.json", + "%dataLoader/test/data/prefs/sammy.json" + ], + data: { + "test1": "test1" + } + }, + db2: { + dataFile: [ + "%dataLoader/test/data/nonExistent2.json" + ], + data: { + "test2": "test2" + } + } + } + }; + + var expected = { + case1: { + errors: [], + databases: { + db1: [{ + "flat": { + "contexts": { + "gpii-default": { + "name": "Default preferences", + "preferences": { + "http://registry.gpii.net/applications/com.texthelp.readWriteGold": { + "ApplicationSettings.AppBar.optToolbarLargeIcons.$t": true, + "ApplicationSettings.Speech.optSAPI5Speed.$t": 50, + "ApplicationSettings.Speech.optAutoUseScreenReading.$t": false + } + } + } + } + } + }, { + "_id": "_design/views", + "views": { + "findUserByName": { + "map": "function(doc) {if (doc.type === 'user') emit(doc.name, doc)}" + } + } + }, { + "test1": "test1" + }], + db2: [{ + "test2": "test2" + }] + } + }, + case2: { + // The errors array contains the regex patterns to match actual returned errors + // since the actual errors are full paths of files that vary when this test runs + // on different machines. + errors: [ + ".*test\/data\/nonExistent1.json", + ".*test\/data\/nonExistent2.json" + ], + databases: { + db1: [{ + "flat": { + "contexts": { + "gpii-default": { + "name": "Default preferences", + "preferences": { + "http://registry.gpii.net/common/fontSize": 24, + "http://registry.gpii.net/common/foregroundColor": "white", + "http://registry.gpii.net/common/backgroundColor": "black", + "http://registry.gpii.net/common/fontFaceFontName": ["Comic Sans"], + "http://registry.gpii.net/common/fontFaceGenericFontFace": "sans serif", + "http://registry.gpii.net/common/magnification": 2.0, + "http://registry.gpii.net/common/tracking": ["mouse"], + "http://registry.gpii.net/common/invertColours": true + } + } + } + } + }, { + "test1": "test1" + }], + db2: [{ + "test2": "test2" + }] + } + } + }; + + fluid.each(testData, function (data, caseName) { + var result = gpii.dataLoader.processDataFiles(data); + + // Verify result.data + jqUnit.assertDeepEq("The returned result is expected", expected[caseName].databases, result.databases); + + // Verify result.errors + var isErrorMatch = gpii.tests.dataLoader.verifyMatches(result.errors, expected[caseName].errors); + jqUnit.assertTrue("All expected errors are reported", isErrorMatch); + }); +}); diff --git a/gpii/node_modules/dataLoader/test/DataLoaderTestsUtils.js b/gpii/node_modules/dataLoader/test/DataLoaderTestsUtils.js new file mode 100644 index 000000000..270dca0ae --- /dev/null +++ b/gpii/node_modules/dataLoader/test/DataLoaderTestsUtils.js @@ -0,0 +1,110 @@ +/** +GPII Data Loader Tests Utils + +Copyright 2017 OCAD University + +Licensed under the New BSD license. You may not use this file except in +compliance with this License. + +You may obtain a copy of the License at +https://github.com/GPII/universal/blob/master/LICENSE.txt +*/ + +"use strict"; + +var fluid = require("infusion"), + jqUnit = fluid.require("node-jqunit", require, "jqUnit"), + gpii = fluid.registerNamespace("gpii"); + +// The base testEnvironment grade used by all successful and failed test cases. +// This component must be name as "testEnvironment" because its base grade "gpii.test.pouch.environment" +// looks up this compnent name for port and events. +// See: https://github.com/GPII/gpii-pouchdb/blob/master/src/test/environment.js#L55 +fluid.defaults("gpii.tests.dataLoader.testEnvironment", { + gradeNames: ["gpii.test.pouch.environment"], + port: 1234, + // Must be supplied with a "gpii.dataLoader.dataLoader" grade to ensure what's defined in "dataLoader" sub-component makes sense + dataLoaderGrade: null, + distributeOptions: { + source: "{that}.options.dataLoaderGrade", + target: "{that > dataLoader}.options.gradeNames" + }, + components: { + dataLoader: { + type: "fluid.component", + createOnEvent: "onHarnessReady", + options: { + listeners: { + "onDataLoaded.escalate": "{testEnvironment}.events.onDataLoaded.fire", + "onDataLoadedError.escalate": "{testEnvironment}.events.onDataLoadedError.fire" + } + } + } + }, + events: { + onDataLoaded: null, + onDataLoadedError: null + } +}); + +fluid.defaults("gpii.tests.dataLoader.baseTestCaseHolder", { + gradeNames: ["gpii.test.express.caseHolder"], + sequenceEnd: [{ + func: "{gpii.test.pouch.environment}.events.onCleanup.fire" + }, { + event: "{gpii.test.pouch.environment}.events.onCleanupComplete", + listener: "fluid.log", + args: ["Database cleanup complete"] + }] +}); + +/** + * The utility function that uses regex to compare whether a string or each string in an given array + * match the regex pattern provided in the expected string or array. + * @param toCompare {String|Array} A string or an array of strings to be compared + * @param expectedMatches {String|Array} A string or an array of regex patterns to be matched + * @return {Boolean} true if all strings in the toCompare match regex patterns in expectedMatches in order, otherwise, return false. + */ +gpii.tests.dataLoader.verifyMatches = function (toCompare, expectedMatches) { + toCompare = fluid.makeArray(toCompare); + expectedMatches = fluid.makeArray(expectedMatches); + + var matched = []; + fluid.each(toCompare, function (one, i) { + var matchResult = one.match(expectedMatches[i]); + if (matchResult && matchResult.length > 0) { + matched.push(one); + } + }); + + return toCompare.length === matched.length; +}; + +/** + * The kettle testing assert function to verify the response of a kettle request + * @param msg {String} A message to be displayed by jqUnit assertion + * @param response {Object} A http response + * @param body {Object} The response carried by the `onComplete` event of a kettle request + * @param expectedStatus {Number} The expected http status code + * @param expected {Object} The expected response data + * @return None + */ +gpii.tests.dataLoader.checkResponse = function (msg, response, body, expectedStatus, expected) { + expectedStatus = expectedStatus ? expectedStatus : 200; + var bodyData = JSON.parse(body); + + gpii.test.express.helpers.isSaneResponse(response, body, expectedStatus); + jqUnit.assertLeftHand(msg, expected, bodyData); +}; + +/** + * The testing assert function to verify an error message + * @param msg {String} A message to verify + * @param expectedMatchedError {String} A regex match pattern to match the msg against. Using regex + * is to avoid the verification of the real machine path that is different when a test runs on different machines. + * @return None + */ +gpii.tests.dataLoader.checkError = function (msg, expectedMatchedError) { + var isErrorMatch = gpii.tests.dataLoader.verifyMatches(msg, expectedMatchedError); + jqUnit.assertTrue("The expected error is reported", isErrorMatch); +}; diff --git a/gpii/node_modules/dataLoader/test/PrefsDataLoaderTests.js b/gpii/node_modules/dataLoader/test/PrefsDataLoaderTests.js new file mode 100644 index 000000000..42b8f5ddb --- /dev/null +++ b/gpii/node_modules/dataLoader/test/PrefsDataLoaderTests.js @@ -0,0 +1,211 @@ +/*! + * GPII Preferences Data Loader Tests + * + * Copyright 2016-2017 OCAD University + * + * Licensed under the New BSD license. You may not use this file except in + * compliance with this License. + * + * You may obtain a copy of the License at + * https://github.com/GPII/universal/blob/master/LICENSE.txt + */ + +"use strict"; + +var fluid = require("infusion"), + jqUnit = fluid.require("node-jqunit", require, "jqUnit"), + gpii = fluid.registerNamespace("gpii"); + +require("gpii-pouchdb"); +gpii.pouch.loadTestingSupport(); + +require("dataLoader"); +require("./DataLoaderTestsUtils.js"); + +fluid.registerNamespace("gpii.tests.prefsDataLoader"); + +//*********** Test gpii.dataLoader.prefsDataLoader.dataConverter ***********// +fluid.defaults("gpii.tests.prefsDataLoader.dataConverter", { + gradeNames: ["gpii.dataLoader.prefsDataLoader.dataConverter"], + dataPath: "%dataLoader/test/data/prefs/", + expected: [{ + "_id": "carla", + "value": { + "flat": { + "contexts": { + "gpii-default": { + "name": "Default preferences", + "preferences": { + "http://registry.gpii.net/applications/com.texthelp.readWriteGold": { + "ApplicationSettings.AppBar.optToolbarLargeIcons.$t": true, + "ApplicationSettings.Speech.optSAPI5Speed.$t": 50, + "ApplicationSettings.Speech.optAutoUseScreenReading.$t": false + } + } + } + } + } + } + }, { + "_id": "sammy", + "value": { + "flat": { + "contexts": { + "gpii-default": { + "name": "Default preferences", + "preferences": { + "http://registry.gpii.net/common/fontSize": 24, + "http://registry.gpii.net/common/foregroundColor": "white", + "http://registry.gpii.net/common/backgroundColor": "black", + "http://registry.gpii.net/common/fontFaceFontName": ["Comic Sans"], + "http://registry.gpii.net/common/fontFaceGenericFontFace": "sans serif", + "http://registry.gpii.net/common/magnification": 2.0, + "http://registry.gpii.net/common/tracking": ["mouse"], + "http://registry.gpii.net/common/invertColours": true + } + } + } + } + } + }] +}); + +jqUnit.asyncTest("Test gpii.dataLoader.prefsDataLoader.dataConverter", function () { + gpii.tests.prefsDataLoader.dataConverter({ + listeners: { + "onPrefsDataStructureConstructed.assert": { + listener: "jqUnit.assertDeepEq", + args: ["The converted data structure is expected", "{that}.options.expected", "{arguments}.0"] + }, + "onPrefsDataStructureConstructed.runTests": { + listener: "jqUnit.start", + priority: "last" + } + } + }); +}); + +//*********** Test gpii.dataLoader.prefsDataLoader ***********// + +//*********** 1. The successful data loading test case ***********// +fluid.defaults("gpii.tests.prefsDataLoader.success", { + gradeNames: ["gpii.dataLoader.prefsDataLoader"], + dbName: "preferences", + dataPath: "%dataLoader/test/data/prefs", + couchDbUrl: "http://localhost:1234" +}); + +// This component must be name as "testEnvironment" because of the reason described at ./DataLoaderTestsUtils.js line 19 +fluid.defaults("gpii.tests.prefsDataLoader.success.testEnvironment", { + gradeNames: ["gpii.tests.dataLoader.testEnvironment"], + dataLoaderGrade: "gpii.tests.prefsDataLoader.success", + events: { + onFixturesConstructed: { + events: { + onDataLoaded: "onDataLoaded" + } + } + } +}); + +fluid.defaults("gpii.tests.prefsTestCaseHolder.success", { + gradeNames: ["gpii.tests.dataLoader.baseTestCaseHolder"], + expected: { + total: { + doc_count: 2 + } + }, + rawModules: [{ + name: "Testing Prefs Data Loader - success", + tests: [{ + name: "Testing a successful preferences data loading process", + type: "test", + sequence: [{ + func: "{totalRequest}.send" + }, { + listener: "gpii.tests.dataLoader.checkResponse", + event: "{totalRequest}.events.onComplete", + args: ["The total number of records should be as expected", "{totalRequest}.nativeResponse", "{arguments}.0", 200, "{testCaseHolder}.options.expected.total"] + }] + }] + }], + components: { + totalRequest: { + type: "kettle.test.request.http", + options: { + path: "/preferences", + port: 1234, + method: "GET" + } + } + } +}); + +fluid.defaults("gpii.tests.prefsDataLoaderTests.success", { + gradeNames: ["gpii.tests.prefsDataLoader.success.testEnvironment"], + components: { + testCaseHolder: { + type: "gpii.tests.prefsTestCaseHolder.success" + } + } +}); + +//*********** 2. The error data loading test case ***********// +fluid.defaults("gpii.tests.prefsDataLoader.error", { + gradeNames: ["gpii.dataLoader.prefsDataLoader"], + dbName: "preferences", + dataPath: "%universal/nonExistent/", + couchDbUrl: "http://localhost:1234" +}); + +// This component must be name as "testEnvironment" because of the reason described at ./DataLoaderTestsUtils.js line 19 +fluid.defaults("gpii.tests.prefsDataLoader.error.testEnvironment", { + gradeNames: ["gpii.tests.dataLoader.testEnvironment"], + dataLoaderGrade: "gpii.tests.prefsDataLoader.error" +}); + +fluid.defaults("gpii.tests.prefsTestCaseHolder.error", { + gradeNames: ["gpii.tests.dataLoader.baseTestCaseHolder"], + rawModules: [{ + name: "Testing Preferences Data Loader - error", + tests: [{ + name: "Testing the non-existence of a data path", + type: "test", + sequence: [{ + event: "{testEnvironment}.events.onDataLoadedError", + listener: "gpii.tests.dataLoader.checkError", + args: ["{arguments}.0", "The provided data path does not exist\:.*\/nonExistent\/"] + }, { + func: "{verifyDbRequest}.send" + }, { + listener: "jqUnit.assertEquals", + event: "{verifyDbRequest}.events.onComplete", + args: ["The total number of records should be as expected", "{verifyDbRequest}.nativeResponse.statusCode", 404] + }] + }] + }], + components: { + verifyDbRequest: { + type: "kettle.test.request.http", + options: { + path: "/auth", + port: 1234, + method: "GET" + } + } + } +}); + +fluid.defaults("gpii.tests.prefsDataLoaderTests.error", { + gradeNames: ["gpii.tests.prefsDataLoader.error.testEnvironment"], + components: { + testCaseHolder: { + type: "gpii.tests.prefsTestCaseHolder.error" + } + } +}); + +fluid.test.runTests([ + "gpii.tests.prefsDataLoaderTests.success", + "gpii.tests.prefsDataLoaderTests.error" +]); diff --git a/gpii/node_modules/dataLoader/test/data/prefs/carla.json b/gpii/node_modules/dataLoader/test/data/prefs/carla.json new file mode 100644 index 000000000..3bc9c517f --- /dev/null +++ b/gpii/node_modules/dataLoader/test/data/prefs/carla.json @@ -0,0 +1,16 @@ +{ + "flat": { + "contexts": { + "gpii-default": { + "name": "Default preferences", + "preferences": { + "http://registry.gpii.net/applications/com.texthelp.readWriteGold": { + "ApplicationSettings.AppBar.optToolbarLargeIcons.$t": true, + "ApplicationSettings.Speech.optSAPI5Speed.$t": 50, + "ApplicationSettings.Speech.optAutoUseScreenReading.$t": false + } + } + } + } + } +} diff --git a/gpii/node_modules/dataLoader/test/data/prefs/franklin.md b/gpii/node_modules/dataLoader/test/data/prefs/franklin.md new file mode 100644 index 000000000..18886de46 --- /dev/null +++ b/gpii/node_modules/dataLoader/test/data/prefs/franklin.md @@ -0,0 +1,17 @@ +# chris.json + +## Weekly Integration Tests: Preference Sets + +This preference set is used for the +[weekly integration tests](http://wiki.gpii.net/w/Weekly_Integration_Test_Plan) and is manually tested on a regular basis. The expected system behaviour on login is described here. + +It has also been used in the [third review of Cloud4all](https://github.com/GPII/universal/blob/master/testData/preferences/review3/review3-preferences.md). + +## Details +`franklin.json`: preference set with a combination of common and application-specific terms for the following settings: + * Read&Write Gold settings: + * App bar width, + * show text on the app bar, + * optional tool bar shows large icons and text, + * SAPI5 speed set to 50, + * Rule-Based Matchmaker. diff --git a/gpii/node_modules/dataLoader/test/data/prefs/sammy.json b/gpii/node_modules/dataLoader/test/data/prefs/sammy.json new file mode 100644 index 000000000..459103b01 --- /dev/null +++ b/gpii/node_modules/dataLoader/test/data/prefs/sammy.json @@ -0,0 +1,19 @@ +{ + "flat": { + "contexts": { + "gpii-default": { + "name": "Default preferences", + "preferences": { + "http://registry.gpii.net/common/fontSize": 24, + "http://registry.gpii.net/common/foregroundColor": "white", + "http://registry.gpii.net/common/backgroundColor": "black", + "http://registry.gpii.net/common/fontFaceFontName": ["Comic Sans"], + "http://registry.gpii.net/common/fontFaceGenericFontFace": "sans serif", + "http://registry.gpii.net/common/magnification": 2.0, + "http://registry.gpii.net/common/tracking": ["mouse"], + "http://registry.gpii.net/common/invertColours": true + } + } + } + } +} \ No newline at end of file diff --git a/gpii/node_modules/dataLoader/test/data/unused/david.json b/gpii/node_modules/dataLoader/test/data/unused/david.json new file mode 100644 index 000000000..92125aefe --- /dev/null +++ b/gpii/node_modules/dataLoader/test/data/unused/david.json @@ -0,0 +1,33 @@ +{ + "flat": { + "contexts": { + "gpii-default": { + "name": "Default preferences", + "preferences": { + "http://registry.gpii.net/applications/com.texthelp.readWriteGold": { + "ApplicationSettings.AppBar.optToolbarIconSet.$t": "Professional", + "ApplicationSettings.AppBar.optToolbarButtonGroupNameCurrent.$t": "Reading Features", + "ApplicationSettings.AppBar.DocType.$t": "1", + "ApplicationSettings.AppBar.Width.$t": 788, + "ApplicationSettings.AppBar.ShowText.$t": true, + "ApplicationSettings.AppBar.optToolbarShowText.$t": true, + "ApplicationSettings.AppBar.LargeIcons.$t": true, + "ApplicationSettings.AppBar.optToolbarLargeIcons.$t": true, + "ApplicationSettings.Speech.optSAPI5Pitch.$t": 33, + "ApplicationSettings.Speech.optSAPI5Speed.$t": 31, + "ApplicationSettings.Speech.optSAPI5Volume.$t": 90, + "ApplicationSettings.Speech.optSAPI5PauseBetweenWords.$t": 115, + "ApplicationSettings.Speech.optAutoUseScreenReading.$t": false, + "ApplicationSettings.Speech.optSAPI5Voice.$t": "ScanSoft UK Indian Sangeeta", + "ApplicationSettings.Speech.WebHighlighting.$t": true, + "ApplicationSettings.Translation.ToLanguage.$t": "hi", + "ApplicationSettings.Speech.optSAPI5SpeechHighlightContext.$t": 0, + "ApplicationSettings.Scanning.ScanDestination.$t": "Word", + "ApplicationSettings.Scanning.ScanToFile.$t": true, + "ApplicationSettings.Spelling.SpellAsIType.$t": false + } + } + } + } + } +} \ No newline at end of file diff --git a/gpii/node_modules/dataLoader/test/data/views-test.json b/gpii/node_modules/dataLoader/test/data/views-test.json new file mode 100644 index 000000000..e314ed60f --- /dev/null +++ b/gpii/node_modules/dataLoader/test/data/views-test.json @@ -0,0 +1,10 @@ +[ + { + "_id": "_design/views", + "views": { + "findUserByName": { + "map": "function(doc) {if (doc.type === 'user') emit(doc.name, doc)}" + } + } + } +] diff --git a/gpii/node_modules/gpii-oauth2/gpii-oauth2-datastore/test/DbDataStoreTestsUtils.js b/gpii/node_modules/gpii-oauth2/gpii-oauth2-datastore/test/DbDataStoreTestsUtils.js index 85e0971d4..1883ab02e 100644 --- a/gpii/node_modules/gpii-oauth2/gpii-oauth2-datastore/test/DbDataStoreTestsUtils.js +++ b/gpii/node_modules/gpii-oauth2/gpii-oauth2-datastore/test/DbDataStoreTestsUtils.js @@ -28,7 +28,10 @@ fluid.defaults("gpii.tests.dbDataStore.environment", { "%gpii-oauth2/gpii-oauth2-datastore/test/data/gpiiAuthTestData.json", "%gpii-oauth2/gpii-oauth2-datastore/dbViews/views.json" ] - } + }, + _users: {}, + _replicator: {}, + pouch__all_dbs__: {} } }, components: { diff --git a/gpii/node_modules/pouchManager/configs/gpii.pouchManager.config.base.json b/gpii/node_modules/pouchManager/configs/gpii.pouchManager.config.base.json index 251505a47..f937142e5 100644 --- a/gpii/node_modules/pouchManager/configs/gpii.pouchManager.config.base.json +++ b/gpii/node_modules/pouchManager/configs/gpii.pouchManager.config.base.json @@ -11,7 +11,10 @@ "%universal/testData/security/TestOAuth2DataStore.json", "%gpii-oauth2/gpii-oauth2-datastore/dbViews/views.json" ] - } + }, + "_users": {}, + "_replicator": {}, + "pouch__all_dbs__": {} } } }, diff --git a/gpii/node_modules/pouchManager/test/pouchManagerTests.js b/gpii/node_modules/pouchManager/test/pouchManagerTests.js index 8bb1e600a..ce5f3a814 100644 --- a/gpii/node_modules/pouchManager/test/pouchManagerTests.js +++ b/gpii/node_modules/pouchManager/test/pouchManagerTests.js @@ -38,7 +38,10 @@ fluid.defaults("gpii.tests.pouchManager", { data: [ "%universal/gpii/node_modules/pouchManager/test/data/oauth2TestData.json" ] - } + }, + _users: {}, + _replicator: {}, + pouch__all_dbs__: {} } } }); diff --git a/gpii/node_modules/testing/src/PouchTestCaseHolder.js b/gpii/node_modules/testing/src/PouchTestCaseHolder.js index e7ed7b8c9..dd3a2c2a4 100644 --- a/gpii/node_modules/testing/src/PouchTestCaseHolder.js +++ b/gpii/node_modules/testing/src/PouchTestCaseHolder.js @@ -45,7 +45,10 @@ fluid.defaults("gpii.test.pouch.pouchTestCaseHolder", { "%universal/tests/platform/cloud/OAuth2AcceptanceDataStore.json", "%gpii-oauth2/gpii-oauth2-datastore/dbViews/views.json" ] - } + }, + _users: {}, + _replicator: {}, + pouch__all_dbs__: {} } }, events: { diff --git a/package.json b/package.json index c77856a04..d0badaedc 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ "bugs": "http://issues.gpii.net/browse/GPII", "homepage": "http://gpii.net/", "dependencies": { - "infusion": "3.0.0-dev.20170131T153243Z.6aab53a", + "infusion": "3.0.0-dev.20170322T234120Z.278de35", "node-jqunit": "1.1.4", "ini": "git://github.com/GPII/ini.git#be8a04aa22f5ad9321ebcbba7740314a53bc8ab2", "node-uuid": "1.4.7", @@ -25,7 +25,7 @@ "glob": "7.1.1", "write-file-atomic": "1.3.1", "json5": "0.5.1", - "gpii-pouchdb": "git://github.com/the-t-in-rtf/gpii-pouchdb#3bf92dbb99570154b880e6254601b532861b1ee4", + "gpii-pouchdb": "1.0.2", "fluid-resolve": "1.2.0", "rimraf": "2.5.4", "mkdirp": "0.5.1" diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 000000000..b1a63db4c --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,41 @@ +This directory contains these scripts: + +* **browserifyTestDependency.js**: To browserify test dependencies required for running some web tests. +* **dataLoader-auth.js**: To load authorization test data into CouchDB. +* **dataLoader-prefs.js**: To load preferences test data into CouchDB. + +## Browserify Test Dependencies: + +**Script name:** `browserifyTestDependency.js` + +This script runs as a npm postinstall step to browserify some node modules that are required for running a few web tests. The creation of this script is to work around the issue that some of these node modules are only pulled in as `devDependencies`(see [GPII package.json](../package.json)), which means the browserifying should only occurs when GPII runs in the development mode. This script ensures the present of these node modules before proceeding with the browserifying. It prevents the failure of running `npm install` when setting up GPII in the production mode. + +To start this script, run the following command in the `universal` directory: + +``` +node scripts/browserifyTestDependency.js +``` + +## Data Loaders + +See [Data loader documentation](../documentation/DataLoader.md) to understand how to config data loaders. + +## Preferences Data Loader + +**Script name:** `dataLoader-prefs.js` + +To load the preferences data into CouchDB, run the following command in the `universal` directory: + +``` +node scripts/dataLoader-prefs.js +``` + +### Authorization Data Loader + +**Script name:** `dataLoader-auth.js` + +To load the authorization data into CouchDB, run the following command in the `universal` directory: + +``` +node scripts/dataLoader-auth.js +``` diff --git a/scripts/dataLoader-auth.js b/scripts/dataLoader-auth.js new file mode 100644 index 000000000..c2e21f5cd --- /dev/null +++ b/scripts/dataLoader-auth.js @@ -0,0 +1,34 @@ +/*! +Copyright 2016-2017 OCAD University + +Licensed under the New BSD license. You may not use this file except in +compliance with this License. + +The research leading to these results has received funding from the European Union's +Seventh Framework Programme (FP7/2007-2013) under grant agreement no. 289016. + +You may obtain a copy of the License at +https://github.com/GPII/universal/blob/master/LICENSE.txt +*/ + +/** + * The data loader to load GPII authorization server data into CouchDB. + * Before running this script, make sure CouchDB is started and ready for handling + * database operation requests. + */ + +"use strict"; + +var fluid = require("infusion"), + gpii = fluid.registerNamespace("gpii"); + +require("../gpii/node_modules/dataLoader/index.js"); + +gpii.dataLoader.authDataLoader({ + dbName: "auth", + dataFile: [ + "%universal/testData/security/TestOAuth2DataStore.json", + "%universal/gpii/node_modules/gpii-oauth2/gpii-oauth2-datastore/dbViews/views.json" + ], + couchDbUrl: "http://localhost:5984" +}); diff --git a/scripts/dataLoader-prefs.js b/scripts/dataLoader-prefs.js new file mode 100644 index 000000000..77119ad07 --- /dev/null +++ b/scripts/dataLoader-prefs.js @@ -0,0 +1,31 @@ +/*! +Copyright 2016-2017 OCAD University + +Licensed under the New BSD license. You may not use this file except in +compliance with this License. + +The research leading to these results has received funding from the European Union's +Seventh Framework Programme (FP7/2007-2013) under grant agreement no. 289016. + +You may obtain a copy of the License at +https://github.com/GPII/universal/blob/master/LICENSE.txt +*/ + +/** + * The data loader to load GPII preferences data into CouchDB. + * Before running this script, make sure CouchDB is started and ready for handling + * database operation requests. + */ + +"use strict"; + +var fluid = require("infusion"), + gpii = fluid.registerNamespace("gpii"); + +require("../gpii/node_modules/dataLoader/index.js"); + +gpii.dataLoader.prefsDataLoader({ + dbName: "preferences", + dataPath: "%universal/testData/preferences/", + couchDbUrl: "http://localhost:5984" +}); diff --git a/tests/all-tests.js b/tests/all-tests.js index d284ca49d..fe644b828 100644 --- a/tests/all-tests.js +++ b/tests/all-tests.js @@ -51,6 +51,9 @@ var testIncludes = [ "./UntrustedBrowserChannelTests.js", "./UntrustedDevelopmentTests.js", "./UntrustedUserLogonStateChangeTests.js", + "../gpii/node_modules/dataLoader/test/AuthDataLoaderTests.js", + "../gpii/node_modules/dataLoader/test/DataLoaderTests.js", + "../gpii/node_modules/dataLoader/test/PrefsDataLoaderTests.js", "../gpii/node_modules/flowManager/test/SaveTests.js", "../gpii/node_modules/flowManager/test/UpdateTests.js", "../gpii/node_modules/flowManager/test/BrowserChannelTests.js", diff --git a/vagrant-configs/provisioning/couchdb-vars.yml b/vagrant-configs/provisioning/couchdb-vars.yml index 1f678a4fe..2cf380ea1 100644 --- a/vagrant-configs/provisioning/couchdb-vars.yml +++ b/vagrant-configs/provisioning/couchdb-vars.yml @@ -4,8 +4,6 @@ # and 'couchdb_admin_password' couchdb_databases: - - preferences - - auth couchdb_delete_existing_databases: true diff --git a/vagrant-configs/provisioning/deploy.sh b/vagrant-configs/provisioning/deploy.sh index 205db0f1a..3f9cdfdaa 100644 --- a/vagrant-configs/provisioning/deploy.sh +++ b/vagrant-configs/provisioning/deploy.sh @@ -14,7 +14,8 @@ sudo PYTHONUNBUFFERED=1 ansible-playbook /home/vagrant/sync/vagrant-configs/prov # Install Flow Manager sudo PYTHONUNBUFFERED=1 ansible-playbook /home/vagrant/sync/vagrant-configs/provisioning/flow-manager.yml --tags="install,configure,deploy" -# Add additional CouchDB data -kanso upload /home/vagrant/sync/testData/security/TestOAuth2DataStore.json http://localhost:5984/auth -kanso upload /home/vagrant/sync/gpii/node_modules/gpii-oauth2/gpii-oauth2-datastore/dbViews/views.json http://localhost:5984/auth +# Load Preferences Test Data into CouchDB +node /home/vagrant/sync/scripts/dataLoader-prefs.js +# Load Authorization Test Data into CouchDB +node /home/vagrant/sync/scripts/dataLoader-auth.js diff --git a/vagrant-configs/provisioning/preferences-server.yml b/vagrant-configs/provisioning/preferences-server.yml index 159e157f9..be8861029 100644 --- a/vagrant-configs/provisioning/preferences-server.yml +++ b/vagrant-configs/provisioning/preferences-server.yml @@ -10,5 +10,3 @@ - facts - nodejs - preferences-server - - preferences-server-data-loader - diff --git a/vagrant-configs/provisioning/requirements.yml b/vagrant-configs/provisioning/requirements.yml index 31fce8e40..271ccc7bc 100644 --- a/vagrant-configs/provisioning/requirements.yml +++ b/vagrant-configs/provisioning/requirements.yml @@ -10,8 +10,5 @@ - src: https://github.com/gpii-ops/ansible-preferences-server name: preferences-server -- src: https://github.com/gpii-ops/ansible-preferences-server-data-loader - name: preferences-server-data-loader - - src: https://github.com/gpii-ops/ansible-flow-manager name: flow-manager