diff --git a/.gitignore b/.gitignore index 944cd989..00085d97 100644 --- a/.gitignore +++ b/.gitignore @@ -246,4 +246,5 @@ web/templates/manifest.json .jj/ # deprecated modules -lib/utils/1m_dep.js \ No newline at end of file +lib/utils/1m_dep.js +script/google_api_key.json diff --git a/lib/AppContext.js b/lib/AppContext.js index 9ab1c52a..697cfffa 100644 --- a/lib/AppContext.js +++ b/lib/AppContext.js @@ -5,13 +5,16 @@ import { DataHarmonizer } from '.'; import { findLocalesForLangcodes } from './utils/i18n'; import { Template, getTemplatePathInScope } from '../lib/utils/templates'; -import { wait, isEmpty, pascalToLowerWithSpaces } from '../lib/utils/general'; +import { wait, isEmpty} from '../lib/utils/general'; import { invert, removeNumericKeys, consolidate } from '../lib/utils/objects'; import { createDataHarmonizerContainer, createDataHarmonizerTab } from '../web'; import { getExportFormats } from 'schemas'; import { range as arrayRange } from 'lodash'; import { deepMerge } from '../lib/utils/objects'; import Validator from './Validator'; +import i18next from 'i18next'; + +import {SchemaEditor} from './SchemaEditor'; class AppConfig { constructor(template_path = null) { @@ -19,42 +22,318 @@ class AppConfig { this.template_path = template_path; } } + export default class AppContext { - //schema_tree = {}; dhs = {}; current_data_harmonizer_name = null; currentSelection = null; template = null; export_formats = {}; - oneToManyAppContext = null; field_settings = {}; + dependent_rows = new Map(); + constructor(appConfig = new AppConfig(getTemplatePathInScope())) { this.appConfig = appConfig; - this.bindEventListeners(); + console.log("this should only be done once") + // THIS WAS $(document), but that was causing haywire issues with Handsontable!!!! + // Each tab was requiring a double click to fully render its table!!! + $('#data-harmonizer-tabs').on('dhTabChange', (event, class_name) => { + //if (this.getCurrentDataHarmonizer() !== class_name) { + //this.tabChange(class_name); + //} + }); } - // Method to bind event listeners - bindEventListeners() { - $(document).on('dhTabChange', (event, data) => { - console.info( - 'dhTabChange', - this.getCurrentDataHarmonizer().template_name, - '->', - data.specName - ); - const { specName } = data; - this.setCurrentDataHarmonizer(specName); - // Should trigger Toolbar to refresh sections - $(document).trigger('dhCurrentChange', { - dh: this.getCurrentDataHarmonizer(), - }); - }); + getSchemaRef() {return this.template?.default?.schema} + + async tabChange(class_name) { + console.info(`Tab change: ${this.getCurrentDataHarmonizer().template_name} -> ${class_name}`); + if (this.getCurrentDataHarmonizer() !== class_name) { + this.setCurrentDataHarmonizer(class_name); + // Trigger Toolbar to refresh sections and tab content ONLY IF not already there. + let dh = this.getCurrentDataHarmonizer(); + + // Toolbar.js updateGUI() also does these. Merge? + this.toolbar.setupSectionMenu(dh); + this.toolbar.setupJumpToModal(dh); + this.toolbar.setupFillModal(dh); + + // Changing tabs clears out validation for that tab. + dh.clearValidationResults(); + this.refreshTabDisplay(); + } + + }; + + /** + * Accessed by both a tabChange() but also by a #report_select_type change + */ + refreshTabDisplay() { + + let dh = this.getCurrentDataHarmonizer(); + let class_name = dh.template_name; + let report_type = $('#report_select_type').val(); + + //console.log("REFRESHTABDISPLAY", class_name, this.schemaEditor, report_type) + + // Display Focus path only if there is some path > 1 element. + $("#record-hierarchy-div").toggle(Object.keys(this.dhs).length > 1); + + // Record hierarchy only shown for "Records by 1-many key" + $('#record-hierarchy').toggle(report_type === ""); + + // The Schema tab always displays a full list, even with "record(s) by selected key" option set in menu. + if (this.schemaEditor !== undefined && class_name === 'Schema') { + // Report type override + report_type = 'all'; + } + + // A Schema Editor interface item that is visible only if Slot tab is showing. + $('#slot_report_type, #slot_report_type option') + .toggle(this.schemaEditor !== undefined && class_name === 'Slot'); + + // Save, deselect, and then set cursor, Otherwise selected cell gets + // set to same column but first row after filter!!! + let cursor = dh.hot.getSelected(); + dh.hot.deselectCell(); + + // Schema editor SCHEMA tab should never be filtered. + // ONLY DEPENDENT TABS/TABLES SHOULD BE FILTERED. + // OR MORE SIMPLY WHEN FILTERING WE ALWAYS PRESERVE FOCUSED NODE + // - BUT WE DON'T WANT EVENT TRIGGERED + switch (report_type) { + case 'all': + this.tabFilter(dh, {}); // gets rid of any filters + this.slotTypeFilter(dh); // Done via hidden fields. no constraint (no keys) + break; + + case 'slot': + //this.slotTypeFilter(dh, ['slot']); + //dh.hot.getPlugin('multiColumnSorting').sort() ; //{ column: 3, sortOrder: 'asc' } + break; + + case 'attribute': + this.slotTypeFilter(dh, ['attribute']); + break; + + case 'slot_usage': + this.slotTypeFilter(dh, ['slot','slot_usage']); + // UNFINISHED CODE + columnSorting.sort({ + column: dh.slot_name_column, + sortOrder: 'asc' + //compareFunctionFactory: function(sortOrder, columnMeta) { + // return function(value, nextValue) { + // return 0; // Custom compare function for the first column (don't sort) + // } + //} + }); + + break; + + // Records by selected key case + case '': + default: + let dependent_report = this.dependent_rows.get(class_name); + this.tabFilter(dh, dependent_report.fkey_vals); // filter by selected keys. + } + + // DON'T RESTORE CURSOR UNTIL WE KNOW THAT IT IS POINTING TO SAME KEY AS BEFORE FILTER? + // Refreshes dependent record list. + if (cursor) { + // Adding custom event type to selectCell() call, but so far not using this. + cursor[0].push('afterFiltering'); + // Now this may not select row related to original selection after filter? + // Suggests converting cursor into unique_key fields, for reselection. + dh.hot.selectCell(...cursor[0], ''); + // Note this will cause a "double event" of selecting same cell twice. + + } + + //columnsorting cannot be disabled. If it is, below code will error. + const columnSorting = dh.hot.getPlugin('multiColumnSorting'); + // Unable to get original multiColumnSorting.initConfig via + // columnSorting.getSortConfig() ?!!!! + // WARNING - this clears out initialConfig TOO?!?! + //columnSorting.clearSort(); // Reset sort so filter matches raw fields + columnSorting.sort(dh.defaultMultiColumnSortConfig); // possibly null + + dh.render(); // Otherwise cells() implements meta and class stuff that isn't reflected + + } + + /** filterByKeys() SHOULD NOT BE APPLIED TO A schema's ROOT (top level) tab. + * E.g. for SCHEMA EDITOR, on "Schema" TAB - we never want to filter selection + * there unless there's a way of releasing that filter. + */ + async filterByKeys_UNUSED_UNUSED(dh, key_vals = {}) { + + /* For Slot tab query, only foreign key at moment is schema_id. + * However, if user has selected a table in Table/Class tab, we want + * filter on class_name field. + schema_id + match to foreign keys: + For every slot_id, class_id, name found, also include slot_id, name. + alt: find slot_id, name, and class_id = key or NULL + I.e. allow NULL to apply just to class_id field. + */ + /* + if (class_name === 'Slot') { + + let column = dh.slot_name_to_column[key_name]; + filtersPlugin.addCondition(column, 'eq', [value]); // + // Get selected table/class, if any: + const class_column = this.slot_name_to_column['class_name']; + const class_dh = this.context.dhs['Class']; + const focused_class_col = class_dh.slot_name_to_column['name']; + const focused_class_row = class_dh.current_selection[0]; + const focused_class_name = (focused_class_row > -1) ? class_dh.hot.getDataAtCell(focused_class_row, focused_class_col) : ''; + // If user has clicked on a table, use that focus to constrain Field list + if (focused_class_name > '') { + filtersPlugin.addCondition(class_column, 'eq', [focused_class_name], 'disjunction'); + //filtersPlugin.addCondition(class_column, 'empty',[], 'disjunction'); + } + // With no table selected, only show rows that DONT have a table/class mentioned + else { + filtersPlugin.addCondition(class_column, 'empty',[]); + } + } + */ + + + + // Special call to initialize highlight of schema library slots having 'slot_type' = 'field'. + // Assumes crudFindAllRowsByKeyVals() will let in slots/fields with null values in order to + // also bring in slots not associated with a class. (no class_name). + +// let multiColumnSorting = dh.hot.getPlugin('multiColumnSorting'); + //const columnSorting = dh.hot.getPlugin('columnSorting'); + //columnSorting.clearSort(); // Reset sort so filter matches raw fields + + const mode = $('#report_select_type').val(); + + switch (mode) { - $(document).on('dhCurrentSelectionChange', (event, data) => { - const { currentSelection } = data; - this.currentSelection = currentSelection; + // Just a list of schema fields: + // sort by slot_group if any, then rank if any, then alphabetically + // Slots and attributes are fully editable. + case 'slot': + + // Table-only fields. Sort as above. + // NOTE: This shows attributes for ALL tables in schema. + // One issue - possible naming collision with schema.slot name will cause schema slot attributes to overwrite slot.attributes slot values? + case 'attribute': + this.slotTypeFilter(dh, [mode]); +// filtersPlugin.addCondition(dh.slot_type_column, 'eq', [mode]); //slot_type +// filtersPlugin.filter(); + +/* + multiColumnSorting.sort([ + {column: 5, sortOrder: 'asc'}, // slot_group + {column: 4, sortOrder: 'asc'}, // rank + {column: 1, sortOrder: 'asc'}, // slot.name + ]); +*/ + break; + + /**********************************************************/ + // Show all slot_usage fields. For each, show schema.slot if available. + case 'slot_usage': + + this.slotTypeFilter(dh, ['slot','slot_usage']); + + + break; + + default: + //Show all types of field: + this.slotTypeFilter(dh, ['slot','slot_usage','attribute']); + } + dh.render(); // Otherwise cells() implements meta and class stuff that isn't reflected + + } + + // Filtering has unpredictable results in terms of rendering, and in + // conjunction with sorting, so using hiddenrowsplugin instead. + tabFilter(dh, key_vals) { + dh.hot.suspendExecution(); // Recommended in handsontable example. + let filtersPlugin = dh.hot.getPlugin('filters'); + filtersPlugin.clearConditions(); + + const hiddenRowsPlugin = dh.hot.getPlugin('hiddenRows'); + var hidden = []; + var shown = []; + + for (let row = 0; row < dh.hot.countSourceRows(); row++) { + let found = true; + Object.entries(key_vals).forEach(([key_name, value]) => { + let column = dh.slot_name_to_column[key_name]; + if (dh.hot.getDataAtCell(row, column) !== value) + found = false; + }) + if (found) + shown.push(row); + else + hidden.push(row) + } + + hiddenRowsPlugin.showRows(shown); + hiddenRowsPlugin.hideRows(hidden); + dh.hot.resumeExecution(); + dh.render(); // Otherwise chunks of old html left with old visible rows + }; + +/* + tabFilter(dh, key_vals) { + dh.hot.suspendExecution(); // Recommended in handsontable example. + let filtersPlugin = dh.hot.getPlugin('filters'); + filtersPlugin.clearConditions(); + + Object.entries(key_vals).forEach(([key_name, value]) => { + let column = dh.slot_name_to_column[key_name]; + //console.log('filter on', class_name, key_name, column, value); //foreign_key, + if (column !== undefined) { + // See https://handsontable.com/docs/javascript-data-grid/api/filters/ + filtersPlugin.addCondition(column, 'eq', [value]); // + } + else + console.log(`ERROR: unable to find filter column "${key_name}" in "${dh.template_name}" table. Check DH_linkML unique_key_slots for this class`); }); + + filtersPlugin.filter(); + dh.hot.resumeExecution(); + }; +*/ + + // Controlling shown or hidden + slotTypeFilter(dh, show_type = null) { + //dh.suspendExecution(); + // See https://handsontable.com/docs/javascript-data-grid/api/core/#suspendexecution + const hiddenRowsPlugin = dh.hot.getPlugin('hiddenRows'); + var hidden = []; + var shown = []; + + for (let row = 0; row < dh.hot.countSourceRows(); row++){ + if (show_type) { + let slot_type = dh.hot.getSourceDataAtCell(row, this.slot_type_column); + if (show_type.includes(slot_type)) { + shown.push(row) + } + else { + hidden.push(row) + } + } + else { // default is to show all fields + shown.push(row) + } + + } + hiddenRowsPlugin.showRows(shown); + hiddenRowsPlugin.hideRows(hidden); + //console.log("Showhide",shown,hidden) + //dh.hot.resumeExecution(); + dh.render(); // Otherwise chunks of old html left with old visible rows } /* @@ -80,9 +359,8 @@ export default class AppContext { const [schema_path, template_name] = this.appConfig.template_path.split('/'); - //alert('template:' + schema_path + ':' + template_path + ':' + this.appConfig.template_path) - this.template = await Template.create(schema_path, { forced_schema }); + this.template = await Template.create(schema_path, { forced_schema }); if (locale !== null) { this.template.updateLocale(locale); } @@ -92,6 +370,9 @@ export default class AppContext { ? {} : await getExportFormats(schema_path); + // Disable export if no export.js content + $('#export-to-dropdown-item').toggleClass("disabled", isEmpty(this.export_formats)); + const schema = locale !== null ? this.template.localized.schema @@ -109,8 +390,32 @@ export default class AppContext { } } - // Merges any existing dataharmonizer instances with the ones newly created. + // Ensure dynamic Enums are added so picklist source / range references are + // detected and pulldown/multiselect menus are crafted; and validation works + this.schemaEditor?.initMenus(); + + + // For all schemas, implement menu inheritance if any. Inheritance assembles + // .inherits[enum1 name, enum2 name, ... , this enum], merging each in order + // such that permissible value labels, descriptions and mappings can be + // overwritten. See https://linkml.io/linkml-model/latest/docs/inherits/ + Object.entries(schema.enums || {}).forEach(([enum_name, enum_obj]) => { + if (enum_obj.inherits) { + const values = {}; + enum_obj.inherits.forEach(inherit_enum_name => { + Object.assign(values, schema.enums[inherit_enum_name].permissible_values); + }); + // Order is important here - given enum_obj must be last. + enum_obj.permissible_values = {...values, ...enum_obj.permissible_values}; + } + }); + this.dhs = this.makeDHsFromRelations(schema, template_name); + this.schemaEditor?.refreshMenus(); // requires classes Class, Enum to be built. + // this.currentDataHarmonizer is now set. + this.crudGetDependentRows(this.current_data_harmonizer_name); + this.crudUpdateRecordPath(); + this.refreshTabDisplay(); return this; } @@ -119,11 +424,19 @@ export default class AppContext { * Creates Data Harmonizers from the schema tree. * FUTURE: Make this a 2 pass thing. First pass creates every dataharmonizer * data structure; 2nd pass initializes handsontable part. + * This has HTML construction going on inside it. This should be separated + * out into a rendering function. + * * @param {Object} schema_tree - The schema tree from which to create Data Harmonizers. * @returns {Object} An object mapping the class names to their respective Data Harmonizer objects. */ makeDHsFromRelations(schema, template_name) { - let data_harmonizers = {}; + let data_harmonizers = {}; // Tacitly ordered attribute object + + // If using DH_LinkML schema, this is the Schema Editor schema! + if (schema.name === 'DH_LinkML') { // referenced elsewhere as dh.context.schemaEditor + this.schemaEditor = new SchemaEditor(schema, this); + }; Object.entries(schema.classes) // Container class is only used in input and output file coordination. @@ -135,17 +448,34 @@ export default class AppContext { .forEach((obj, index) => { if (obj.length > 0) { const [class_name, class_obj] = obj; - - // If it shares a key with another class which is its parent, this DH must be a child - const is_child = this.crudGetParents(class_name); - + // Move createDataHarmonizerTab() to HTML utilities, and move prep there. + // Tooltip lists any parents of this class whose keys must be satisfied. + const is_child = !(class_name == template_name); + let tooltip = Object.keys(this.relations[class_name]?.parent) + .map((parent_name) => schema.classes[parent_name].title || schema.classes[parent_name].name) + .join(', '); + if (tooltip.length) { + tooltip = `${i18next.t('tooltip-require-selection')}: ${tooltip}`; //Requires key selection from x, y, ... + } const dhId = `data-harmonizer-grid-${index}`; const tab_label = class_obj.title ? class_obj.title : class_name; - const dhTab = createDataHarmonizerTab(dhId, tab_label, index === 0); - dhTab.addEventListener('click', () => { - $(document).trigger('dhTabChange', { - specName: class_name, - }); + const dhTab = createDataHarmonizerTab(dhId, tab_label, class_name, tooltip, index === 0); + const self = this; + + // Future: Change createDataHarmonizerTab() to return jquery dom element id? + dhTab.addEventListener('click', (event) => { + // Or try mouseup if issues with .filtersPlugin and bootstrap timing??? + // Disabled tabs do not triger dhTabChange. + if (event.srcElement.classList.contains("disabled")) + return false; + + // When using handsontable filter(), a timing thing causes + // HandsonTable not to render header row(s) and left sidebar + // column(s) on tab change unless we do a timed delay on + // tabChange() call. + setTimeout(() => { this.tabChange(class_name)}, 200); + //this.tabChange(class_name); + return false; }); // Each selected DataHarmonizer is rendered here. @@ -161,7 +491,8 @@ export default class AppContext { hot_override_settings: { minRows: is_child ? 0 : 10, minSpareRows: 0, - height: is_child ? '50vh' : '75vh', + //minHeight: '200px', // nonsensical if no rows exist + height: '75vh', // TODO: Workaround, possibly due to too small section column on child tables // colWidths: is_child ? 256 : undefined, colWidths: 200, @@ -175,44 +506,20 @@ export default class AppContext { // based on LinkML class this.useTemplate(handsOnDH, class_name); - handsOnDH.createHot(); + handsOnDH.createHot(); // Instance without data. - /* FUTURE: separate dh from all .context schema/template data/functions + // FUTURE: try reorganization to separate dh from all .context + // schema/template data/functions - const dh = { - template: schema.classes[class_name] + // If this class_name is a range of a Container attribute, + // then create an easy reference to container name + if (class_name != 'Container' && 'Container' in schema.classes) { + let found_name = Object.values(schema.classes.Container.attributes) + .filter((v) => v.range && v.range === class_name)?.[0]?.name; + if (found_name) + handsOnDH.container_name = found_name; } - - // Set up the data structure (.template,.slots,.slot_names etc - // based on LinkML class - this.useTemplate(dh, class_name); - - // Each selected DataHarmonizer is rendered here. - // NOTE: this may be running twice?. - // in 1-M, different DataHarmonizers have fewer rows to start with - // Child tables are displayed differently so override the default - // HoT settings. - const dhSubroot = createDataHarmonizerContainer(dhId, index === 0); - const handsOnDH = new DataHarmonizer(dhSubroot, dh, { - loadingScreenRoot: document.body, - template_name: class_name, - schema: schema, // assign during constructor so validator can init on it. - hot_override_settings: { - minRows: is_child ? 0 : 10, - minSpareRows: 0, - height: is_child ? '50vh' : '75vh', - // TODO: Workaround, possibly due to too small section column on child tables - // colWidths: is_child ? 256 : undefined, - colWidths: 200, - }, - }); - - //Object.assign(dh, handsOnDH); - handsOnDH['template'] = schema.classes[class_name]; - handsOnDH.validator.useTargetClass(class_name); - data_harmonizers[class_name] = handsOnDH; - handsOnDH.createHot(); -*/ + //console.log("CONTAINER LOOKUP", handsOnDH.container_name); if (is_child) { /* Initialization of each child table is to hide all rows until @@ -240,62 +547,82 @@ export default class AppContext { useTemplate(dh, template_name) { dh.template_name = template_name; dh.sections = []; // This will hold template's new data including table sections. - // let self = this; const sectionIndex = new Map(); - // Gets LinkML SchemaView() of given template - // problem: multiclass schemas means this template doesn't reflect the underlying structure anymore. - // const specification = this.schema.classes[template_name]; - // Class contains inferred attributes ... for now pop back into slots - - let attributes = Object.entries(this.template.default.schema.classes) - .filter(([cls_key]) => cls_key === dh.template_name) - .reduce((acc, [, spec]) => { - return { - ...acc, - ...spec.attributes, - }; - }, {}); - - /* Lookup each column in terms table. A term looks like: - is_a: "core field", - title: "history/fire", - slot_uri: "MIXS:0001086" - comments: (3) ['Expected value: date', 'Occurrence: 1', 'This field is used uniquely in: soil'] - description: "Historical and/or physical evidence of fire" - examples: [{…}], - multivalued: false, - range: "date", - ... - */ + // A class has slots list used just to include Schema slots, and a slot_usage dict which provides extended attributes to the schema slots. But both these are "compiled" by either tabular_to_json.py or linkml.py SchemaView() call into the class's attributes dictionary. + const raw_attributes = this.template.default.schema.classes[dh.template_name].attributes; + //console.log("Attributes(", template_name,")", raw_attributes); + + // Attributes need to be organized by slot group, and then rank within those slot groups, before processing below. + + // NMDC created slots having same name as slot_groups, and giving each a rank, to control ordering. + + /* Need to determine order of slot_usage items, in order to sort attributes in same order. + * 1st guess of order comes by first encounter in attributes. Slot_usage doesn't necessarily + * mention order of every slot. However it does indicate tailoring content such as ordering + * to given class. + * Mixins get wovent into attribute slots, e.g. + * "mixins": [ + * "DhMultiviewCommonColumnsMixin", + * "SampIdNewTermsMixin" + * ], + */ + + let sections = {}; + let count = 0; + for (let name in raw_attributes) { + let slot_group = raw_attributes[name].slot_group; + if (!slot_group) { + slot_group = 'Field'; //Make a default slot_group to gather any unnamed ones. + raw_attributes[name].slot_group = slot_group; + } + if (!sections[slot_group]) { + // Templates can declare a slot_group as a slot in order to control ranking + // May want to qualify this as NMDC did with slot "is_a":"dh_section" + // ISSUE: a spec can mention a slot_group as a Title of a slot! + if (this.template.default.schema.slots[slot_group]?.rank) { + + sections[slot_group] = this.template.default.schema.slots[slot_group].rank; + } + else { + // OTHERWISE First encountered slot_group gets priority + count ++; + sections[slot_group] = count; + } + } + } + + const attributes = Object.fromEntries(Object.entries(raw_attributes).sort((a, b) => sections[a[1].slot_group] - sections[b[1].slot_group] || a[1].rank - b[1].rank)); - for (let name in attributes) { - // ISSUE: a template's slot definition via SchemaView() currently - // doesn't get any_of or exact_mapping constructs. So we start - // with slot, then add class's slots reference. - let field = deepMerge( + for (name in attributes) { + //console.log("SLOT", name, attributes[name], this.template.current.schema.slots[name]) + /** ISSUE: tabular_to_schema.py is not compiling locale fields into class attributes. It + * only implements SchemaView() inheritance. So deepMerge goes and gets Schema level + * slot definition which has locale stuff in it. + * FUTURE: simplify to just language overlay. + */ + + let slot = deepMerge( attributes[name], this.template.current.schema.slots[name] ); - - //let field = attributes[name]; + + //let slot = attributes[name]; let section_title = null; - if ('slot_group' in field) { - // We have a field positioned within a section (or hierarchy) - section_title = field.slot_group; + if ('slot_group' in slot) { + // We have a slot positioned within a section (or hierarchy) + section_title = slot.slot_group; } else { - if ('is_a' in field) { - section_title = field.is_a; + if ('is_a' in slot) { + section_title = slot.is_a; } else { - section_title = 'Generic'; - console.warn("Template field doesn't have section: ", name); + section_title = 'Field'; + console.warn("Template slot doesn't have section: ", name); } } - // We have a field positioned within a section (or hierarchy) - if (!sectionIndex.has(section_title)) { sectionIndex.set(section_title, sectionIndex.size); let section_parts = { @@ -306,164 +633,133 @@ export default class AppContext { if (section) { Object.assign(section_parts, section); } - dh.sections.push(section_parts); } let section = dh.sections[sectionIndex.get(section_title)]; - let new_field = Object.assign(structuredClone(field), { section_title }); // shallow copy + let new_slot = Object.assign(structuredClone(slot), { section_title }); // shallow copy // Some specs don't add plain english title, so fill that with name // for display. - if (!('title' in new_field)) { - new_field.title = new_field.name; + if (!('title' in new_slot)) { + new_slot.title = new_slot.name; } - // Default field type xsd:token allows all strings that don't have - // newlines or tabs - new_field.datatype = null; - + /** Handsontable columns have a "cell type" which may trigger datepicker + * or boolean editing controls and date/numeric etc. validation. The + * basic default datatype is "text", but date, time, select, dropdown, + * checkbox etc are options, but we have only implemented some of them. + * See https://handsontable.com/docs/javascript-data-grid/cell-type/ + */ + new_slot.datatype = null; + + /** + * For LinkML we have a default slot type of xsd:string, but + * xsd:token, which strips whitespace before or after text, and strips + * extra space within text. xsd:string allows whitespace anywhere. + * + * A LinkML slot can have multiple values in its range. Each value can + * be a fundamental LinkML data type, a Class, or an Enum reference. + * See https://linkml.io/linkml/schemas/slots.html#ranges + * We hold each vetted range in range_array, and + */ let range_array = []; - // Multiple ranges allowed. For now just accepting enumerations - if ('any_of' in new_field) { - for (let item of new_field.any_of) { - if ( - item.range in this.template.default.schema.enums || - item.range in this.template.default.schema.types || - pascalToLowerWithSpaces(item.range) in - this.template.default.schema.enums || - pascalToLowerWithSpaces(item.range) in - this.template.default.schema.types - // || Object.keys(this.template.current.schema.enums).some(k => k.indexOf('geo_loc_name') !== '-1') || - // Object.keys(this.template.current.schema.types).some(k => k.indexOf('geo_loc_name') !== '-1') - ) { - range_array.push(item.range); + // Range expressed as logic on given array: / all_of / any_of / exactly_one_of / none_of + for (let range_type of ['all_of','any_of','exactly_one_of','none_of']) { + if (range_type in new_slot) { + for (let item of new_slot[range_type]) { + if ( + item.range in this.template.default.schema.enums || + item.range in this.template.default.schema.types || + item.range in this.template.default.schema.classes // Possibility of collisions here w enums and types. + ) { + range_array.push(item.range); + } } } - } else { - range_array.push(new_field.range); } + // Idea is that schema slot will only have range XOR any_of etc. construct, not both. + if (range_array.length == 0) + range_array.push(new_slot.range); + // Parse slot's range(s) for (let range of range_array) { if (range === undefined) { - console.warn('field has no range', new_field.title); + console.warn(`ERROR: Template ${template_name} slot ${new_slot.name} has no range.`); + continue; } - // Range is a datatype? - const types = this.template.default.schema.types; - if (range in types || pascalToLowerWithSpaces(range) in types) { - const range_obj = - typeof types[range] !== 'undefined' - ? types[range] - : types[pascalToLowerWithSpaces(range)]; + // Here range is a datatype + if (range in this.template.default.schema.types) { /* LinkML typically translates "string" to "uri":"xsd:string" - // but this is problematic because that allows newlines which - // break spreadsheet saving of items in tsv/csv format. Use - // xsd:token to block newlines and tabs, and clean out leading - // and trailing space. xsd:normalizedString allows lead and trai - // FUTURE: figure out how to accomodate newlines? - */ + * but this is problematic because that allows newlines which + * break spreadsheet saving of items in tsv/csv format. Use + * xsd:token to block newlines and tabs, and clean out leading + * and trailing whitespace. + * Note: xsd:normalizedString allows lead and trailing whitespace. + */ switch (range) { case 'WhitespaceMinimizedString': - new_field.datatype = 'xsd:token'; + new_slot.datatype = 'xsd:token'; break; case 'string': - new_field.datatype = 'string'; + new_slot.datatype = 'string'; break; case 'Provenance': - new_field.datatype = 'Provenance'; + new_slot.datatype = 'Provenance'; break; default: - new_field.datatype = range_obj.uri; - // e.g. 'time' and 'datetime' -> xsd:dateTime'; 'date' -> xsd:date + new_slot.datatype = this.template.default.schema.types[range].uri; + // e.g. 'time' and 'datetime' -> xsd:dateTime'; 'date' -> xsd:date } - } else { - // If range is an enumeration ... - const enums = this.template.current.schema.enums; - if (range in enums || pascalToLowerWithSpaces(range) in enums) { - const range_obj = - typeof enums[range] !== 'undefined' - ? enums[range] - : enums[pascalToLowerWithSpaces(range)]; - - if (!('sources' in new_field)) { - new_field.sources = []; - } - if (!('flatVocabulary' in new_field)) { - new_field.flatVocabulary = []; - } - if (!('flatVocabularyLCase' in new_field)) { - new_field.flatVocabularyLCase = []; - } - if (!('permissible_values' in new_field)) { - new_field.permissible_values = {}; - } + continue; + } + + // If range is an enumeration ... + if (range in this.template.current.schema.enums) { + new_slot.sources??= []; + new_slot.sources.push(range); + continue; + } - new_field.permissible_values[range] = range_obj.permissible_values; - - new_field.sources.push(range); - // This calculates for each categorical field in schema.yaml a - // flat list of allowed values (indented to represent hierarchy) - let flatVocab = this.stringifyNestedVocabulary( - range_obj.permissible_values - ); - new_field.flatVocabulary.push(...flatVocab); - // Lowercase version used for easy lookup/validation - new_field.flatVocabularyLCase.push( - ...flatVocab.map((val) => val.trim().toLowerCase()) - ); - } else { - // If range is a class ... - // multiple => 1-many complex object - if (range in this.template.current.schema.classes) { - if (range == 'quantity value') { - /* LinkML model for quantity value, along lines of https://schema.org/QuantitativeValue, e.g. xsd:decimal + unit - PROBLEM: There are a variety of quantity values specified, some allowing units - which would need to go in a second column unless validated as text within column. - - description: >- - A simple quantity, e.g. 2cm - attributes: - verbatim: - description: >- - Unnormalized atomic string representation, should in syntax {number} {unit} - has unit: - description: >- - The unit of the quantity - slot_uri: qudt:unit - has numeric value: - description: >- - The number part of the quantity - range: - double - class_uri: qudt:QuantityValue - mappings: - - schema:QuantityValue - */ - } - } - } - } // End range parsing + // If range is a class ... ??? + if (range in this.template.current.schema.classes) { + continue; + } + + /* FUTURE: LinkML model for quantity value, along lines of + * https://schema.org/QuantitativeValue, e.g. xsd:decimal + unit + * PROBLEM: There are a variety of quantity values specified, some + * allowing units which would need to go in a second column unless + * validated as text within column. + */ + + console.warn(`ERROR: Template "${template_name}" slot "${new_slot.name}" range "${range}" is not recognized as a data type, Class or Enum.`); + } + + if (new_slot.sources?.length > 0) { + // sets slots flatVocabulary etc. based on sources enums. + this.setSlotRangeLookup(new_slot); } // Provide default datatype if no other selected - if (!new_field.datatype) { - new_field.datatype = 'xsd:token'; + if (!new_slot.datatype) { + new_slot.datatype = 'xsd:token'; } - // field.todos is used to store some date tests that haven't been + // new_field.todos is used to store some date tests that haven't been // implemented as rules yet. - if (new_field.datatype == 'xsd:date' && new_field.todos) { + if (new_slot.datatype == 'xsd:date' && new_slot.todos) { // Have to feed any min/max date comparison back into min max value fields - for (const test of new_field.todos) { + for (const test of new_slot.todos) { if (test.substr(0, 2) == '>=') { - new_field.minimum_value = test.substr(2); + new_slot.minimum_value = test.substr(2); } if (test.substr(0, 2) == '<=') { - new_field.maximum_value = test.substr(2); + new_slot.maximum_value = test.substr(2); } } } @@ -471,86 +767,144 @@ export default class AppContext { /* Older DH enables mappings of one template field to one or more export format fields */ - this.setExportField(new_field, true); + this.setExportField(new_slot, true); /* https://linkml.io/linkml-model/docs/structured_pattern/ https://github.com/linkml/linkml/issues/674 Look up its parts in "settings", and assemble a regular - expression for them to compile into "pattern" field. + expression for them to compile into "pattern" attribute. This augments basic datatype validation structured_pattern: syntax: "{float} {unit.length}" interpolated: true ## all {...}s are replaced using settings partial_match: false */ - if ('structured_pattern' in new_field) { - switch (new_field.structured_pattern.syntax) { + if ('structured_pattern' in new_slot) { + switch (new_slot.structured_pattern.syntax) { case '{UPPER_CASE}': case '{lower_case}': case '{Title_Case}': - new_field.capitalize = true; + new_slot.capitalize = true; } - // TO DO: Do conversion here into pattern field. + // TO DO: Do conversion here into pattern attribute. } // pattern is supposed to be exlusive to string_serialization - if ('pattern' in field && field.pattern.length) { + if (slot.pattern?.length) { // Trap invalid regex - // Issue with NMDC MIxS "current land use" field pattern: "[ ....(all sorts of things) ]" syntax. + // Issue with NMDC MIxS "current land use" slot pattern: "[ ....(all sorts of things) ]" syntax. try { - new_field.pattern = new RegExp(field.pattern); + new_slot.pattern = new RegExp(slot.pattern); } catch (err) { console.warn( - `TEMPLATE ERROR: Check the regular expression syntax for "${new_field.title}".` + `TEMPLATE ERROR: Check the regular expression syntax for "${new_slot.name}".` ); console.error(err); // Allow anything until regex fixed. - new_field.pattern = new RegExp(/.*/); + new_slot.pattern = new RegExp(/.*/); } } if (this.field_settings[name]) { - Object.assign(new_field, this.field_settings[name]); + Object.assign(new_slot, this.field_settings[name]); } - if (field.annotations) { - new_field.annotations = field.annotations; + if (slot.annotations) { + new_slot.annotations = slot.annotations; } - section.children.push(new_field); + section.children.push(new_slot); } // End slot processing loop + this.setDHSlotLookups(dh); + + }; + + /** A set of lookup functions is compiled for each DH template / class. + */ + setDHSlotLookups(dh) { + // Sections and their children are sorted by .rank parameter if available dh.sections.sort((a, b) => a.rank - b.rank); - // Sort kids in each section + // Sort kids in each section by rank given in LinkML class's slot_usage. for (let ptr in this.sections) { dh.sections[ptr]['children'].sort((a, b) => a.rank - b.rank); } - // Easy lookup arrays. + // LOOKUP FUNCTIONS (as arrays) // Array of slot objects, with each index being a column in Handsontable. - // LOOKUP: table (Handsontable) column to slot object. - dh.slots = Array.prototype.concat.apply( - [], - dh.sections.map((section) => section.children) - ); + dh.slots = dh.sections.map((section) => section.children).flat(1); // LOOKUP: column # to name dh.slot_names = dh.slots.map((slot) => slot.name); - + // LOOKUP: column # to title + dh.slot_titles = dh.slots.map((slot) => slot.title); + // LOOKUP: slot name to column # - // Replaces dh.getColumnIndexByFieldName() unless loose name/title match needed // Note of course that duplicate names yield farthest name index. dh.slot_name_to_column = {}; Object.entries(dh.slots).forEach( - ([index, slot]) => (dh.slot_name_to_column[slot.name] = index) + ([index, slot]) => (dh.slot_name_to_column[slot.name] = parseInt(index)) ); + + // LOOKUP: slot title to column # + dh.slot_title_to_column = {}; + Object.entries(dh.slots).forEach( + ([index, slot]) => (dh.slot_title_to_column[slot.title] = parseInt(index)) + ); + } + + /** Compile a slot's enumeration lookup (used in single and multivalued + * pulldowns) + * + * Special case: if slot_name == "range" and template name = "Slot" then + * loaded templates are parts of the schema editor and we need this slot + * to provide a menu (enumeration) of all possible ranges that LinkML + * would allow, namely, the list of loaded Types, Classes, and + * Enumerations. For now, Types are the ones uploaded with the schema, + * later they will be editable. Classes and Enumerations come directly + * from the loaded Class and Enum tabs. As new items are added + * or removed from Class and Enum tabs, this schema_editor schema's + * "range" slot's .sources and permissible_values object and related + * objects must be refreshed. + */ + setSlotRangeLookup(slot) { + + for (let range of slot.sources) { + const permissible_values = this.template.current.schema.enums[range].permissible_values; + + slot.permissible_values??= {}; // Assigns new object only if undefined. + slot.permissible_values[range] = permissible_values; + + // For convenience, precalculates flat list of permissible values for use + // in getContainerData. + slot.merged_permissible_values??= {}; + Object.assign(slot.merged_permissible_values, permissible_values); + + // This calculates for each categorical slot range in schema.yaml a + // flat list of allowed values (indented to represent hierarchy) + slot.flatVocabulary??= []; + const flatVocab = this.stringifyNestedVocabulary(permissible_values); + slot.flatVocabulary.push(...flatVocab); + + // Lowercase version used for easy lookup/validation in case where + // provider data might not have the capitalization right. + slot.flatVocabularyLCase??= []; + slot.flatVocabularyLCase.push( + ...flatVocab.map((val) => val.trim().toLowerCase()) + ); + + } + } /** - * Recursively flatten vocabulary into an array of strings, with each + * Recursively flatten vocabulary into an array of strings, with each * string's level of depth in the vocabulary being indicated by leading * spaces. + * ISSUE August, 2025: It seems Handsontable css no longer indents + * display of choice.text leading spaces. Remedied in DataHarmonizer.js + * enableMultiSelection() * FUTURE possible functionality: * Both validation and display of picklist item becomes conditional on * other picklist item if "depends on" indicated in picklist item/branch. @@ -562,17 +916,19 @@ export default class AppContext { let stack = []; for (const pointer in vocab_list) { let choice = vocab_list[pointer]; + this.setExportField(choice, false); // Move elsewhere? + + /* PHASING THIS OUT - depth/level now handled in DataHarmonizer.js + * updateSources(slot). let level = 0; if ('is_a' in choice) { level = stack.indexOf(choice.is_a) + 1; - stack.splice(level + 1, 1000, choice.text); + stack.splice(level + 1, 1000, choice.text); // 1000 -> all subsequent text. } else { stack = [choice.text]; } - - this.setExportField(choice, false); - - ret.push(' '.repeat(level) + choice.text); + */ + ret.push(choice.text); } return ret; } @@ -822,34 +1178,38 @@ export default class AppContext { * Paraphrased example OUTPUT relations.[class_name]: { - parent: { - [foreign_class]: { - [slot_name]: [foreign_slot_name], - [slot_name_2]: [foreign_slot_name_2]... - } - }, - child: { - [foreign_class]: { - [slot_name]: [foreign_slot_name], - [slot_name_2]: [foreign_slot_name_2]... - } - }, - dependents: {[class_name]:[class dh],...}, - dependent_slots: { - [slot_name]: { - [foreign_class]: {foreign_slot_name]}... - }, - unique_key_slots: { - [slot_name]: {[unique_key_name]:true, ... - } + parent: {[foreign_class]: {[slot_name]: [foreign_slot_name]...}...}, + child: {[dependent_class]: {[slot_name]: dependent_slot_name...}...}, + dependent_slots: {[slot_name]: { [dependent_class]: dependent_slot_name}...}}, + target_slots: {[foreign_slot_name]: {[class_name]: slot_name}}; + unique_key_slots: {[slot_name]: {[unique_key_name]:true, ... } + + // Added on + dependents: {[dependent_name]:[dependent dh],...}, } */ crudGetRelations(schema) { let relations = {}; + + // First pass establishes basic attributes so 2nd pass references to + // .parent or .child don't matter, i.e. classes can be processed in + // any order. + Object.entries(schema.classes).forEach(([class_name, class_obj]) => { + if (class_name != 'Container' && class_name != 'dh_interface') { + relations[class_name] = { + parent: {}, + child: {}, + dependent_slots: {}, + foreign_key_count: 0, + target_slots: {}, + unique_key_slots: {} + }; + } + }); + Object.entries(schema.classes).forEach(([class_name, class_obj]) => { if (class_name != 'Container' && class_name != 'dh_interface') { - relations[class_name] = {}; Object.entries(class_obj.attributes ?? {}).forEach( ([slot_name, attribute]) => { @@ -865,53 +1225,48 @@ export default class AppContext { foreign_slot_name = key; foreign_class = attribute.range; } else { - console.log( - 'Class', - class_name, - 'has slot', - slot_name, - 'foreign key', - attribute.annotations?.foreign_key?.value, - 'but no target class information in key or slot range.' - ); + console.log(`Class ${class_name} has slot ${slot_name} and foreign key ${attribute.annotations?.foreign_key?.value} but no target class information in key or slot range.`); return; } - Object.assign(relations[class_name], { - parent: { [foreign_class]: { [slot_name]: foreign_slot_name } }, - }); - // And reverse relation - Object.assign(relations[foreign_class], { - child: { [class_name]: { [foreign_slot_name]: slot_name } }, - }); - // And dependent slots via foreign key relations. - Object.assign(relations[class_name], { - dependent_slots: { - [slot_name]: { - parent: foreign_class, - slot: foreign_slot_name, - }, - }, - }); + + relations[class_name].parent[foreign_class] ??= {}; + relations[class_name].parent[foreign_class][slot_name] = foreign_slot_name; + + // And dependent slots within this class via other's foreign keys + // Only one dependent can depend on a slot. + relations[class_name].dependent_slots[slot_name] = { + foreign_class: foreign_class, + foreign_slot: foreign_slot_name + }; + + // And reverse relations: this is another table's child (dependent), its slot -> this slot. + relations[foreign_class].child[class_name] ??= {}; + relations[foreign_class].child[class_name][foreign_slot_name] = slot_name; + + // And list of local slots that connect to targets of dependent slot foreign keys. + // Another table's target slot_name is this table's slot. + // Note: more than one foreign key can point to target slot. + relations[foreign_class].target_slots[foreign_slot_name] ??= {}; + relations[foreign_class].target_slots[foreign_slot_name][class_name] = slot_name; } } ); + relations[class_name].foreign_key_count = Object.keys(relations[class_name].dependent_slots).length; + // Now do unique keys in class_obj which might not be mentioned in // foreign_class relationships. Object.entries(class_obj.unique_keys ?? {}).forEach( ([key_name, key_obj]) => { - Object.entries(key_obj.unique_key_slots ?? {}).forEach( - ([index, slot_name]) => { - Object.assign(relations[class_name], { - unique_key_slots: { [slot_name]: { [key_name]: true } }, - }); - } - ); + for (const slot_name of key_obj.unique_key_slots) { + relations[class_name].unique_key_slots[slot_name] = key_name; + } } ); + + } }); - return relations; } @@ -924,12 +1279,18 @@ export default class AppContext { } /** Retrieves ORDERED DICT of tables that ultimately have given template as - * a foreign key, IN ORDER. Whether it is to enact cascading visibility, update or + * a foreign key, IN ORDER. Whether to enact cascading visibility, update or * deletion events, this provides the order in which to trigger changes. * Issue is that intermediate tables need to be checked in order due to * dependencies by foreign keys. If a table depended on an intermediate that * hadn't been refreshed, we'd get a wrong display. * Using Map to ensure order. + * + * FUTURE: There may be a situation where a dependent is appendend onto the + * stack as a child of some parent, but another class earlier in stack also + * depends on it. That earlier class should be moved to be after freshly + * appended one?! + * * @param {String} class_names initial value * @param {Array} class_names array of relations * @return {Object} class_names where each descendent class_name is mentioned. @@ -939,32 +1300,105 @@ export default class AppContext { return stack; } const children = this.crudGetChildren(class_name); - if (!children) { - // catches "undefined" - return stack; + if (isEmpty(children)) { + return stack; // catches undefined case } // Add each child to end of stack. let next_name = ''; for (const dependent_name in children) { - if (next_name == '') next_name = dependent_name; + if (next_name == '') + next_name = dependent_name; stack.set(dependent_name, true); } return this.crudGetDependents(next_name, ptr + 1, stack); } + /** + * In Display section (top of page), show cookiecrumb path from root to current + * template that user has made a selection in. A complexity is that a + * class's keys may be composed of slots from have two (or more) separate + * parents, each with parent(s) etc. + * + */ + crudUpdateRecordPath() { + const schema = this.template.current.schema; + const current_template_name = this.getCurrentDataHarmonizer().template_name; + let class_name = current_template_name; // current tab + let hierarchy_text = []; + let stack = [class_name]; + let done = {}; + while (stack.length >0) { + class_name = stack.pop(); // pop class_name + if (class_name in done) + continue; + const class_label = schema.classes[class_name].title || schema.classes[class_name].name; + const dependent = this.dependent_rows.get(class_name); + let key_string = ''; + let tooltip = ''; + Object.entries(dependent.key_vals).reverse().forEach(([key, value]) => { + if (!(key in dependent.fkey_vals)) { + //value = '...'; + //continue; // Don't repeat value + //value = ''; + //else + if (value === null) + value = '_'; + key_string = value + (key_string ? ', ' : '') + key_string; + tooltip = key + (tooltip ? ', ' : '') + tooltip; + } + }); + let tooltip_text = `${tooltip} `; + done[class_name] = `${class_label}: ${key_string}${tooltip_text} `; + + // SEVERAL parents are possible - need them all displayed. + let parents = Object.keys(this.relations[class_name].parent); + if (!isEmpty(parents)) + stack.push(...parents); + } + //Need to assemble hierarchy text in same order as dhs order. + for (class_name in this.template.default.schema.classes) { + if (class_name in done) + hierarchy_text.push(done[class_name]); + } + $("#record-hierarchy").html(hierarchy_text.join(' / ')); + } + + crudCalculateDependentKeys(class_name) { + this.crudGetDependentRows(class_name); + this.crudUpdateRecordPath(); + let class_dependents = this.relations[class_name].dependents; + for (let [dependent_name] of class_dependents.entries()) { + this.setDHTabStatus(dependent_name); + const dh = this.dhs[dependent_name]; + }; + }; + /* For given class, refresh view of all dependent tables that have a direct * or indirect foreign key relationship to given class. * Performance might show up as an issue later if lots of long dependent * tables to re-render. - * FUTURE: show enabled/disabled dependent tab based on whether parent - * table has focused primary key? */ + /* crudFilterDependentViews(class_name) { - for (var [dependent_name, obj] of this.relations[class_name].dependents) { - this.crudFilterByForeignKey(dependent_name); - } - } + // This recalculates rows to show. + let class_dependents = this.relations[class_name].dependents; + let log_processed_dhs=[]; + for (let [dependent_name] of class_dependents.entries()) { + let dependent_report = this.dependent_rows.get(dependent_name); + this.setDHTabStatus(dependent_name); + if (dependent_name != class_name) { // No need to redo class/ tab that user is currently on + let dh = this.dhs[dependent_name]; + dh.filterByKeys(dh, dependent_name, dependent_report.fkey_vals); + //this.crudFilterDependentRows(dependent_name, dependent_report); + } + }; + + //console.log('crudFilterDependentViews',class_name,' > ', log_processed_dhs); + + this.crudUpdateRecordPath(); + } + */ /* For given class name (template), show only rows that conform to that * class's foreign key-value constraints. * (Could make a touch more efficient by skipping if we can tell that a @@ -972,150 +1406,65 @@ export default class AppContext { * POSSIBLE ISSUE: if given class table has one parent table where no * primary key selection has been made! */ - crudFilterByForeignKey(class_name) { - const hotInstance = this.dhs[class_name]?.hot; + crudFilterDependentRows(class_name, dependent_report) { + alert("this shouldn't be running") + const dh = this.dhs[class_name]; + const hotInstance = dh?.hot; // This can happen when one DH is rendered but not other dependents yet. if (!hotInstance) return; - const hiddenRowsPlugin = hotInstance.getPlugin('hiddenRows'); - - // Ensure all rows are visible - const oldHiddenRows = hiddenRowsPlugin.getHiddenRows(); - // arrayRange() makes [0, ... n] - let rowsToHide = arrayRange(0, hotInstance.countSourceRows()); - - //Fetch key-values to match - const parents = this.crudGetParents(class_name); - let [required_selections, errors] = this.crudGetForeignKeyValues(parents); - - // FUTURE: may be able to specialize to hide display based on particular - // parent(s) keys? - if (errors.length == 0) { - rowsToHide = rowsToHide.filter((row) => { - for (const [slot_name, value] of Object.entries(required_selections)) { - const col = this.dhs[class_name].getColumnIndexByFieldName(slot_name); - const cellValue = hotInstance.getDataAtCell(row, col); - // Null value test in case where parent or subordinate field is null. - if (cellValue != value.value) return true; - } - }); + + // Changing shown rows means we should get rid of any existing selections. + // And not assume user has clicked on anything. + hotInstance.deselectCell(); + dh.current_selection = [null,null,null,null]; + + // this class's foreign keys if any, are partially or fully satisfied. + if (dependent_report.fkey_status > 0) { + + hotInstance.suspendExecution(); // See https://handsontable.com/docs/javascript-data-grid/api/core/#suspendexecution + + const hiddenRowsPlugin = hotInstance.getPlugin('hiddenRows'); + + // Ensure all rows are visible + const oldHiddenRows = hiddenRowsPlugin.getHiddenRows(); + + // arrayRange() makes [0, ... n] + let rowsToHide = arrayRange(0, hotInstance.countSourceRows()); + rowsToHide = rowsToHide.filter((row) => !dependent_report.rows?.includes(row)); + + // Make more efficient using Set() difference comparison? + //if (rowsToHide != oldHiddenRows) { + //Future: make more efficient by switching state of show/hide deltas? + hiddenRowsPlugin.showRows(oldHiddenRows); + hiddenRowsPlugin.hideRows(rowsToHide); // Hide the calculated rows + + hotInstance.resumeExecution(); } - // Make more efficient using Set() difference comparison? - //if (rowsToHide != oldHiddenRows) { - //Future: make more efficient by switching state of show/hide deltas? - hiddenRowsPlugin.showRows(oldHiddenRows); - hiddenRowsPlugin.hideRows(rowsToHide); // Hide the calculated rows - hotInstance.render(); // Render the table to apply changes - //} - } - /** - * Here we deal with adding rows that need foreign keys. Locate each foreign - * key and fetch its focused value, and copy into new records below. - * If missing key value(s) encountered, prompt user to focus appropriate - * tab(s) row and try again. This requires that every parent be synchronized - * with its parent, etc. So more simple to handle this by a top-down foreign - * key synchronization, rather than bottom-up then top-down messaging. - * - * OUTPUT: - * @param {Object} required_selections: Dictionary of [slot_name]:[value]. - * @param {String} errors: If user hasn't made a row selection in one of the parent foreign key, this is returned in a report. - */ - crudGetForeignKeyValues(parents) { - let required_selections = {}; - let errors = ''; - Object.entries(parents).forEach(([parent_name, parent]) => { - Object.entries(parent).forEach(([slot_name, foreign_slot_name]) => { - const value = this.crudGetSelectedTableSlotValue( - parent_name, - foreign_slot_name - ); - if (value == null) { - errors += `\n- ${parent_name}.${foreign_slot_name}: _____`; - } else { - required_selections[slot_name] = { - value: value, - parent: parent_name, - slot: foreign_slot_name, - }; - } - }); - }); - return [required_selections, errors]; } - crudGetSelectedTableSlotValue(class_name, slot_name) { - const dh = this.dhs[class_name]; - // getSelected() returns array with each row being a selection range - // e.g. [[startRow, startCol, endRow, endCol],...]. - const selected = dh.hot.getSelected(); - if (selected) { - const row = selected[0][0]; // Get first selection's row. - const col = dh.getColumnIndexByFieldName(slot_name); - const value = dh.hot.getDataAtCell(row, col); - if (value && value.length > 0) return value; - } - return null; // DH doesn't return null for cell, so this flags no selection. + /** Activates/deactivates tab based on satisfaction of table's foreign keys. + * fkey_status > 0 means parent table(s) slots used as keys by this table + * are filled in so we can show content for this table, so activate tab. + * Otherwise tab is deactivated. + */ + setDHTabStatus(class_name) { + // Presumes dependent_rows has been updated: + const domId = Object.keys(this.dhs).indexOf(class_name); + const state = this.dependent_rows.get(class_name).fkey_status; + $('#tab-data-harmonizer-grid-' + domId) + .toggleClass('disabled',state === 0) + .parent().toggleClass('disabled',state === 0); } /** - * OBSOLETE. The crudGetDependentChanges() now handles this. - * TEST FOR CHANGE TO DEPENDENT TABLE FOREIGN KEY(S) - * CASE 1: A shift from a complete foreign key to: - * A) Another complete combination. - * 1) duplicate? - * a) MERGE? - * 2) unique: - * Prompt to change all underlying records? - * B) Deleted part - * BLOCK. Suggest deleting row. - * C) Cleared out. - * Block - same as B). - - * - changed key (part) -> prompt user to use "right click: Change primary key option" + * Returns a count and row #s and slot keys of each dependent table's records + * that are connected to particular foreign key(s) in root class_name table + * WHERE: * - * CASE 2: Shift from incomplete foreign key to complete one: - * - This doesn't necessitate any action except say for a refresh - * of dependent table in case where we allow dependent to have - * "loose cannon" foreign key references. - * ISSUE IS THIS WOULD HAVE TO CASCADE KEY SENSITIVITY. - * WHERE DOES KEY SENSITIVITY RESPONSIBILITY END? - * BEST TO RECURSIVELY CHECK DEPENDENCIES - * @param {String} class_name name of root template/class to check - * @param {Integer} row of parent template to check for primary key values. - * @param {Array} changes to primary key values on that row???? - * @param {Boolean} update indicating whether to make changes to data. - - crudGetDependentUpdates(class_name, row, update=false) { - let change_log = ''; - for (const [dependent_name, obj] of this.relations[class_name].dependents) { - const parents = this.crudGetParents(dependent_name); - // Checks to see which selection area of each parent is active (usually - // just one parent but sometimes a table is dependent on two others) - // GETS RECORDS THAT MATCH EXISTING KEYS, for which new KEY(S) pertain. - let [required_selections, errors] = this.crudGetForeignKeyValues(parents); - // Ignore tables with errors as these indicate dependent tables with - // incomplete foreign keys. - // Note: user can "rescue" visibility of dependent records that already - // exist with final key value(s). - if (!errors) - for (const [slot_name, value] of Object.entries(required_selections)) { - change_log = change_log + `\n - ${dependent_name} record(s) with ${slot_name}: ${value.value}`; - } - } - return change_log; - } */ - - /** - * Returns a count of each dependent table's records that are connected to - * particular foreign key(s) in class_name table WHERE: - * - * 1) Dependent's primary key is held in a single root table - * record such that deleting or updating that root record necessarily - * triggers deletes or updates to underlying tables. - * - * 2) Dependent's primary key is held in more than one root table record - * no action in this case (except alert) since operations on remaining - * records should determine dependents fates. + * 1) Dependent's primary key is held in one or more root/parent tables. + * Function records which keys were changed so that subsequent delete or update + * work can be done. * * crudGetDependentChanges is a 2 pass call. First pass does search through * dependents list, mapping each to either: @@ -1123,7 +1472,7 @@ export default class AppContext { * primary key is found * b) null in case where no primary key matched. * - * Final step executes given action on dependent_mapping. + * Final step executes given action on dependent_rowsping. * * a) Update is simpler case where only cascading update of changed root * table slot(s) is needed. @@ -1140,122 +1489,341 @@ export default class AppContext { * - but this is determined from the dependent's parent/foreign_keys * perspective. * + * CASE 2: Shift from incomplete foreign key to complete one: + * - This doesn't necessitate any action except say for a refresh + * of dependent table in case where we allow dependent to have + * "loose cannon" foreign key references. + * ISSUE IS THIS WOULD HAVE TO CASCADE KEY SENSITIVITY. + * WHERE DOES KEY SENSITIVITY RESPONSIBILITY END? + * * TO DO: If dependent depends on more than one parent does this affect our * returned affected list? Assuming not for now. - * + * The delete or update in root table is row specific and happens in calling + * routine via user confirmation. * Note: this report is done regardless of visibility of rows/columns to user. + * + * Builds a list of dependents that have root-related records + * dependent_rows: { + * [dependent_name]: { + * slots: {} // just like start_name, these are values of all slots that other tables depend on. + * [parent]: { + * fkey_vals: {[slot_name]: value, ... }, + * key_vals: {[slot_name]: value, ... }, + * fkey_status: 0-2, + * changed_slots: {[slot_name]: value, ... }, + * count: [# records in dependent matching this combo], + * rows: [row # of affected row, ...] + * } + * } * - * INPUT - * @class_name {String} name of root table to begin looking for cascading effect of foreign key changes. - * @do_action {String} either default false, or "delete" to signal delete of found records, or "update" to trigger update on primary/foreign key of found records. + * For a given DH table, 1st user-selected row is used to get values for all non-foreign key and target slots. + * + * @param {String} start_name name of root table to begin looking for cascading effect of foreign key changes. + * @param {String} class_name name of class (DataHarmonizer instance) to start from. + * @param {Boolean} skipable: yes = don't do deep dive if no keys have changed for class name. Only for update case. + * @param {Object} changes dictionary pertaining to a particular dh row's slots. + * @return {Object} this.dependent_rows updated table of given start_name class's foreign and other keys and their current and changed values */ - crudGetDependentChanges(class_name, do_action = false) { - let change_report = ''; - let found_dependent_rows = false; - // Contains a list of dependents that have root-related records - let dependent_map = new Map(); - - for (const [dependent_name, obj] of this.relations[class_name].dependents) { - const ddh = this.dhs[dependent_name]; - const parents = this.crudGetParents(dependent_name); - Object.entries(parents).forEach(([parent_name, parent]) => { - // Assemble the list of a parent's slots that are foreign keys of this - // dependent. Then see if there is more than one parent record which - // has those key values. If so, no need to delete anything in - // dependent. - const pdh = this.dhs[parent_name]; - const parent_row = pdh.hot.getSelected()?.[0]?.[0]; - // parent-to-child linked key-values - const [p_KeyVals, d_KeyVals] = this.crudGetKeyVals( - parent_name, - dependent_name, - parent_row - ); - // Search below starts with row 0. - let found_row = this.crudFindByKeyVals(pdh, p_KeyVals); - // Looking for another record besides parent being deleted. - if (found_row == parent_row) - found_row = this.crudFindByKeyVals(pdh, p_KeyVals, parent_row + 1); - // If at least one other row has the keyVals_p, then no need to delete - // dependent records. - if (found_row == null) { - let found_rows = []; - // Now find any DEPENDENT table rows that have to be deleted. - found_row = this.crudFindByKeyVals(ddh, d_KeyVals); - while (found_row != null) { - // https://handsontable.com/docs/javascript-data-grid/api/core/#alter - // Prepped for alter 'remove rows' [[1,1],[4,1], ...] structure, where each - found_rows.push([found_row, 1]); - found_row = this.crudFindByKeyVals(ddh, d_KeyVals, found_row + 1); + crudGetDependentRows(start_name, skippable=false, changes = {}) { + // Changes only apply to start_name class; all dependents should "see" them + // only via dependent_rows foreign key values. + let [key_vals, fkey_vals, fkey_status] = + this.crudGetDependentKeyVals(start_name, changes); + + // Changes may include any key slots (not just foreign keys), but other + // slots are ignored. + let changed_keys = Object.fromEntries(Object.entries(key_vals) + .filter(([slot_name, value]) => changes.hasOwnProperty(slot_name)) + .map(([slot_name, value]) => ([slot_name, changes[slot_name].value])) + ); + + this.dependent_rows.set(start_name, { + fkey_vals: fkey_vals, + fkey_status: fkey_status, + key_vals: key_vals, + key_changed_vals: changed_keys + }); + + // For each dependent + this.relations[start_name]?.dependents?.keys().forEach((class_name) => { + // 1) Assemble the needed key_vals for this start class and its + // dependents (via foreign keys). + let [key_vals, fkey_vals, fkey_status] = + this.crudGetDependentKeyVals(class_name); + let dependent_rows_obj = { + fkey_vals: fkey_vals, + fkey_status: fkey_status, + key_vals: key_vals, + key_changed_vals: {} + }; + + let changed_dep_keys = {}; + + Object.entries(this.relations[class_name].dependent_slots) + .forEach(([slot_name, key_mapping]) => { + const parent_changes = this.dependent_rows.get(key_mapping.foreign_class)['key_changed_vals']; + // For all dependents, changes if any come from parent table(s). + // Parent field names are different though - via relations! + let parent_slot_name = key_mapping.foreign_slot; + if (parent_changes.hasOwnProperty(parent_slot_name)) { + changed_dep_keys[slot_name] = parent_changes[parent_slot_name]; + } + }) + dependent_rows_obj['key_changed_vals'] = changed_dep_keys; + + //if (changes && slot_name in key_vals) { + // changed_key_vals[slot_name] = changes[slot_name].value; + //} + /* The skippable situation at any level, true case used only for update + // situation. As done in getChangeReport() we need to test if key_status + // = 0, i.e. no foreign key satisfied. If so we shouldn't be so emptiness of + // the list of foreign keys target slots ^ changed_slots + + if (skippable && key_status == 0) + return + */ + + if (fkey_status > 0) { + // Issue: special case: Field / slot table has slots with no class name, + // as well as class name = one in fkey_vals. Allow empty key fields - + // leave that to validation to catch. + let rows = this.crudFindAllRowsByKeyVals(class_name, fkey_vals); + if (rows.length) { + + dependent_rows_obj['count'] = rows.length; + dependent_rows_obj['rows'] = rows; + } + } + + this.dependent_rows.set(class_name, dependent_rows_obj); + + }) + + } + + /** + * Function to get a class_name's unique and targeted slots and their values. + * 1) Get slots that need to be filled to satisfy + * - this table's foreign keys + * - dependent table foreign keys. (queried for) + * 2) Get values for those slots from table(s) that class_name depends on, + * or from root class user selection, or from query + * + * fkey_status (relative to parents, not dependents): + * 0: full foreign key not satisfied + * 1: foreign key partly satified... + * 2: no foreign keys (ergo, satisfied) + * 3: foreign keys satisfied, + * + * OLD: + * 1: no foreign keys (ergo, satisfied) + * 2: foreign keys satisfied, + * + * // Not implemented: remaining unique key slots unsatisfied + * // 3: foreign keys satisfied, no other unique key slots. + * // 4: all foreign + other unique key slots satisfied. + * + * @param {String} class_name to return key_vals dictionary for + * @param {Object} dependent_rows dictionary of dependents to get values from + * @param {Object} changes in root (holds current vs new slot values) + * @return {Object} key_vals by slot_name of existing key values + * @return {Object} changed_key_vals by slot_name of key values that are changing (and need to be updated) + * @return {Boolean} key_status = integer + */ + crudGetDependentKeyVals(class_name, changes) { + const class_dh = this.dhs[class_name]; + let fkey_vals = {}; + let key_vals = {}; + let fkey_status = 3; // Assume fks satisfied + + if (class_dh) { // FUTURE: avoid this case altogether. + + // Get value for each slot that has a foreign key pointing to another table + let dependent_slots = this.relations[class_name].dependent_slots; + + // If no foreign keys, then key satisfied + if (this.relations[class_name].foreign_key_count === 0) { + fkey_status = 2; + } + else { + // Here one or more foreign (dependent) key slots exist. + let found_fkey = false; + for (let [slot_name, slot_link] of Object.entries(dependent_slots)) { + let dependent_f_rows = this.dependent_rows.get(slot_link.foreign_class); + if (!dependent_f_rows) { + console.log(`No dependent foreign row links for ${class_name}:${slot_name} slot on dependent slots: `, dependent_slots); + break; } - if (do_action == 'delete') { - ddh.hot.alter('remove_row', found_rows); - } else if (do_action == 'update') { - //... - } else if (found_rows.length > 0) { - found_dependent_rows = true; - let key_vals = Object.entries(d_KeyVals) - .join('\n') - .replace(',', ': '); - change_report += `\n - ${found_rows.length} ${dependent_name} records with key:\n ${key_vals}`; + // If foreign table fkey_status == 0 then user shouldn't be seeing + // any of its records, and so this dependent table should be hidden + // too. + if (dependent_f_rows.fkey_status == 0) { + fkey_status = 0; + break; // Exit loop, final fkey_status set. + } + + let foreign_slots = dependent_f_rows.key_vals; + let foreign_value = foreign_slots[slot_link.foreign_slot]; + if (foreign_value === undefined || foreign_value == '') { + foreign_value = null; + } + fkey_vals[slot_name] = foreign_value; + key_vals[slot_name] = foreign_value; + + if (fkey_vals[slot_name] === null) { + fkey_status = 0; + } + else { + found_fkey = true; } } - }); - } - return do_action || found_dependent_rows == false ? false : change_report; + // Signal partially complete fkey. + if (fkey_status === 0 && found_fkey) + fkey_status = 1; + } + + // Here a field/slot is under user control rather than a foreign_key, so + // get value from table's focused selection. Merges unique_key_slot & + // target_slot list for lookup of values (i.e. no overwrites). + let search_slots = Object.keys(Object.assign({}, + this.relations[class_name].unique_key_slots, + this.relations[class_name].target_slots + )); + for (let slot_name of search_slots) { + if (!(slot_name in dependent_slots)) { // FK slot already inherited. + // VERIFY IF THIS HANDLES ALL SCENARIOS + // If there is a user-selected row, this likely is used to get non-foreign key values. + const class_row = class_dh.hot.getSelected()?.[0]?.[0]; //1st row in selection range. + key_vals[slot_name] = this.crudGetDHCellValue(class_dh, slot_name, class_row); + } + } + + }; + + return [key_vals, fkey_vals, fkey_status]; + } + + crudGetDHCellValue(dh, slot_name, row) { + const col = dh.slot_name_to_column[slot_name]; + return dh.hot.getDataAtCell(row, col); } - /* construct two lookup tables for given parent and dependent tables so - * we can quickly lookup their respective foreign key slots and values. + /** + * Used in dh.addRows(). Locate each foreign key and fetch its focused value, + * and copy into new records below. + * If missing key value(s) encountered, prompt user to focus appropriate + * tab(s) row and try again. This requires that every parent be synchronized + * with its parent, etc. + * + * OUTPUT: + * @param {Object} required_selections: Dictionary of [slot_name]:[value]. + * @param {String} errors: If user hasn't made a row selection in one of the parent foreign key, this is returned in a report. */ - crudGetKeyVals(class_name, dependent_name, row) { - const pdh = this.dhs[class_name]; - const ddh = this.dhs[dependent_name]; - let linked_slots = this.relations[class_name].child[dependent_name]; - let class_slots = {}; - let child_slots = {}; - Object.entries(linked_slots).forEach(([class_slot, child_slot]) => { - let col = pdh.slot_name_to_column[class_slot]; - let value = pdh.hot.getDataAtCell(row, col); - class_slots[class_slot] = value; - col = pdh.slot_name_to_column[child_slot]; - child_slots[child_slot] = value; // linked so must be the same. + crudGetForeignKeyValues(class_foreign_keys) { + let required_selections = {}; + let errors = ''; + Object.entries(class_foreign_keys).forEach(([parent_name, parent]) => { + Object.entries(parent).forEach(([slot_name, foreign_slot_name]) => { + const value = this.crudGetSelectedTableSlotValue( + parent_name, + foreign_slot_name + ); + if (value == null) { + const dh = this.dhs[parent_name]; + const parent_title = this.template.current.schema.classes[parent_name].title; + const foreign_slot_title = dh.slots[dh.slot_name_to_column[foreign_slot_name]].title; + errors += `\n- ${parent_title}.${foreign_slot_title}: _____`; + } else { + required_selections[slot_name] = { + value: value, + parent: parent_name, + slot: foreign_slot_name, + }; + } + }); }); + return [required_selections, errors]; + } - return [class_slots, child_slots]; + /** + * Get value of given class's slot where user has focused on. If no focus, + * return null indicating so. + * @param {string} class_name + * @param {string} slot_name + * @return {String} value of class slot at focused row, or null + */ + crudGetSelectedTableSlotValue(class_name, slot_name) { + const dh = this.dhs[class_name]; + // getSelected() returns array with each row being a selection range + // e.g. [[startRow, startCol, endRow, endCol],...]. + const selected = dh.hot.getSelected(); + if (selected) { + const row = selected[0][0]; // Get first selection's row. + const col = dh.slot_name_to_column[slot_name]; + const value = dh.hot.getDataAtCell(row, col); + if (value && value.length > 0) return value; + } + return null; // DH doesn't return null for cell, so this flags no selection. } /* Looks for 2nd instance of row containing keyVals, returns that row or * null if not found. PROBABLY NOT USEFUL since our work on changing or * adding unique_key fields tests for keys BEFORE user change, i.e. just * finding one existing key is enough to halt operation. - */ + crudFindDuplicate(dh, keyVals) { - let found_row = this.crudFindByKeyVals(dh, keyVals); + let found_row = this.crudFindRowByKeyVals(dh, keyVals); // Looking for another record besides parent being deleted. - return this.crudFindByKeyVals(dh, keyVals, found_row + 1); + if (found_row === false) + return false; + return this.crudFindRowByKeyVals(dh, keyVals, found_row + 1); } + */ - /* Looks for row (starting from 0 by default) containing given keyVals. + /** + * Search for key_vals combo in given dh handsontable. + * @param {Object} dh DataHarmonizer instance + * @param {Object} key_vals a dictionary of slots and their values + * @return {Array} of rows, if any, each being an integer. + */ + crudFindAllRowsByKeyVals(class_name, key_vals) { + const dh = this.dhs[class_name]; + let rows = []; + let row = this.crudFindRowByKeyVals(dh, key_vals); + while (row !== false) { + rows.push(row); + row = this.crudFindRowByKeyVals(dh, key_vals, row+1); + } + return rows; + } + + /** Looks for row (starting from 0 by default) containing given key_vals. * returns null if not found. + * FUTURE, allow value to be an array of permitted values. * ISSUE: This isn't using indexes on tables, so not very efficient. + * @param {Object} dh Dataharmonizer instance + * @param {Object} keyv_als slot & value combination to search for. + * @param {Integer} row to start search after. + * @return {Integer} row that key_vals were found on - OR false if none found. */ - crudFindByKeyVals(dh, keyVals, row = 0) { - const total_rows = dh.hot.countSourceRows(); // .countSourceRows(); + crudFindRowByKeyVals(dh, key_vals, row = 0) { + const total_rows = dh.hot.countSourceRows(); while (row < total_rows) { - // .every() causes return on first break - if ( - Object.entries(keyVals).every(([slot_name, value]) => { - const col = dh.getColumnIndexByFieldName(slot_name); - return value == dh.hot.getDataAtCell(row, col); + if ( // .every() causes return on first break with row + Object.entries(key_vals).every(([slot_name, value]) => { + const col_value = dh.hot.getDataAtCell(row, dh.slot_name_to_column[slot_name]); + // Allow empty values by HandsonTable? But adding "col_value === null" triggers infinite loop? + return value == col_value; //|| (slot_name === 'class_id' && col_value === '') }) - ) + ) { break; - else row++; + } + row++; } if (row == total_rows) // Nothing found. - row = null; + row = false; + return row; } @@ -1265,8 +1833,10 @@ export default class AppContext { Object.entries(this.crudGetParents(class_name)).forEach( ([parent_name, parent]) => { Object.entries(parent).forEach(([slot_name, foreign_name]) => { - let col = dh.getColumnIndexByFieldName(slot_name); - dh.updateColumnSettings(col, { readOnly: true }); + let col = dh.slot_name_to_column[slot_name]; + if (dh.slots[col].required) { + dh.updateColumnSettings(col, { readOnly: true }); + } }); } ); @@ -1275,28 +1845,23 @@ export default class AppContext { /** Determine if given dh row has a COMPLETE given unique_key. This includes * a key which might have previously empty slot value(s) that now have changed * user-entered value(s). Also determine if that key is CHANGED as a result - * of user input. If not, can ignore. + * of user input. + * + * Called once in dh.beforeChange() * * @param {Object} dh instance * @param {Integer} row * @param {String} key_name to examine * @param {Object} changes for a row with slot_name as key. * @param {Object} dictionary of slots and their new values. - * @param {String} textual report of any slots which were changed. - * - * @return {Boolean} found boolean true if complete keyVals - * @return {Object} keyVals set of slots and their values from template or change - * @return {String} change_log report of changed slot values + * @return {Boolean} found boolean true if complete keyVals + * @return {Object} keyVals set of slots and their values from template or change + * @return {String} change_log report of changed slot values */ - crudHasCompleteUniqueKey( - dh, - row, - key_name, - changes, - keyVals = {}, - change_log = '' - ) { + crudHasCompleteUniqueKey(dh, row, key_name, changes) { const key_obj = dh.template.unique_keys[key_name]; + let keyVals = {}; + let change_log = ''; let changed = false; let complete = Object.entries(key_obj.unique_key_slots) // If every slot has a value, then the whole key is complete. @@ -1311,7 +1876,7 @@ export default class AppContext { return !!change.value; } // Not a changed slot. - let col = dh.getColumnIndexByFieldName(slot_name); + let col = dh.slot_name_to_column[slot_name]; let value = dh.hot.getDataAtCell(row, col); keyVals[key_name] = value; return !!value; diff --git a/lib/DataHarmonizer.js b/lib/DataHarmonizer.js index aa53d8f0..5e2c40ce 100644 --- a/lib/DataHarmonizer.js +++ b/lib/DataHarmonizer.js @@ -2,39 +2,26 @@ import '@selectize/selectize'; import Handsontable from 'handsontable'; import SheetClip from 'sheetclip'; import $ from 'jquery'; -import 'jquery-ui-bundle'; -import 'jquery-ui/dist/themes/base/jquery-ui.css'; +import YAML from 'yaml';//#{ parse, stringify } from 'yaml' +//import {strOptions} from 'yaml/types' +//strOptions.fold.lineWidth = 0; // Prevents yaml long strings from wrapping. +//strOptions.fold.defaultStringType = 'QUOTE_DOUBLE' +//YAML.scalarOptions.str.defaultType = 'QUOTE_SINGLE' +//scalarOptions.str.defaultType = 'QUOTE_SINGLE' import i18next from 'i18next'; -import { utils as XlsxUtils, read as xlsxRead } from 'xlsx/xlsx.js'; import { renderContent, urlToClickableAnchor } from './utils/content'; -import { readFileAsync, updateSheetRange } from '../lib/utils/files'; -//import { findSlotNamesForClass } from '../lib/utils/templates'; -import { - isValidHeaderRow, - rowIsEmpty, - wait, - stripDiv, - isEmptyUnitVal, - pascalToLowerWithSpaces, -} from '../lib/utils/general'; -import { invert, deepMerge, looseMatchInObject } from '../lib/utils/objects'; +import { isEmpty, rowIsEmpty, isEmptyUnitVal } from '../lib/utils/general'; import { changeCase, - dataArrayToObject, - dataObjectToArray, - fieldUnitBinTest, formatMultivaluedValue, JSON_SCHEMA_FORMAT, - KEEP_ORIGINAL, MULTIVALUED_DELIMITER, - parseMultivaluedValue, - titleOverText, + parseMultivaluedValue } from './utils/fields'; -import { - checkProvenance, +import { checkProvenance, itemCompare, validateValAgainstVocab, validateValsAgainstVocab, @@ -44,10 +31,7 @@ import 'handsontable/dist/handsontable.full.css'; import './data-harmonizer.css'; import '@selectize/selectize/dist/css/selectize.bootstrap4.css'; -import specifyHeadersModal from './specifyHeadersModal.html'; -import unmappedHeadersModal from './unmappedHeadersModal.html'; -import fieldDescriptionsModal from './fieldDescriptionsModal.html'; - +import contentModals from './contentModals.html'; import HelpSidebar from './HelpSidebar'; import Validator from './Validator'; @@ -65,6 +49,8 @@ import { multiKeyValueListRenderer, } from './editors'; +Handsontable.renderers.registerRenderer('multiKeyValueListRenderer', multiKeyValueListRenderer); + Handsontable.cellTypes.registerCellType('key-value-list', { editor: KeyValueListEditor, validator: keyValueListValidator, @@ -134,7 +120,7 @@ class DataHarmonizer { this.dateFormat = options.dateFormat || 'yyyy-MM-dd'; this.datetimeFormat = options.datetimeFormat || 'yyyy-MM-dd hh:mm aa'; this.timeFormat = options.timeFormat || 'hh:mm aa'; - this.dateExportBehavior = options.dateExportBehavior || JSON_SCHEMA_FORMAT; + this.dateExportBehavior = options.dateExportBehavior || 'JSON_SCHEMA_FORMAT'; this.validator = new Validator(this.schema, MULTIVALUED_DELIMITER, { dateFormat: this.dateFormat, datetimeFormat: this.datetimeFormat, @@ -146,10 +132,6 @@ class DataHarmonizer { if (this.helpSidebarOptions.enabled) { const opts = Object.assign({}, this.helpSidebarOptions); opts.onToggle = (open) => { - // always do a HOT rerender on toggle in addition to anything client-specified - if (this.hot) { - this.render(); - } if (typeof this.helpSidebarOptions.onToggle === 'function') { this.helpSidebarOptions.onToggle(open); } @@ -157,91 +139,27 @@ class DataHarmonizer { this.helpSidebar = new HelpSidebar(this.root, opts); } - $(this.modalsRoot).append(specifyHeadersModal); - $(this.modalsRoot).append(unmappedHeadersModal); - $(this.modalsRoot).append(fieldDescriptionsModal); - - // Reset specify header modal values when the modal is closed - $('#specify-headers-modal').on('hidden.bs.modal', () => { - $('#specify-headers-err-msg').hide(); - $('#specify-headers-confirm-btn').unbind(); - }); + $(this.modalsRoot).append(contentModals); - // Field descriptions. TODO: Need to account for dynamically rendered cells. - $(this.root).on('dblclick', '.secondary-header-cell', (e) => { + // Field descriptions. + // ISSUE: conflict with single click on Handsontable column header + // triggering selection of whole column. + $(this.root).on('click', '.secondary-header-text', (e) => { //dblclick // NOTE: innerText is no longer a stable reference due to i18n // Ensure hitting currentTarget instead of child // with innerText so we can guarantee the reference field. - const field_reference = e.currentTarget.getAttribute('data-ref'); + const field_reference = $(e.currentTarget).parents('.secondary-header-cell').attr('data-ref'); const field = this.slots.find((field) => field.title === field_reference); $('#field-description-text').html(this.getComment(field)); $('#field-description-modal').modal('show'); }); + } render() { this.hot.render(); } - /** - * Open file specified by user. - * Only opens `xlsx`, `xlsx`, `csv` and `tsv` files. Will launch the specify - * headers modal if the file's headers do not match the grid's headers. - * @param {File} file User file. - * @param {Object} xlsx SheetJS variable. - * @return {Promise<>} Resolves after loading data or launching specify headers - * modal. - */ - async openFile(file) { - // methods for finding the correspondence between ranges and their containers - // in the 1-M code, tables correspond to entities that are ranges of their containers - // TODO: might need to be generalized in a future iteration, if ranges overlap - const container_for_range = (container, maybe_range) => { - Object.entries(container.attributes).forEach( - ([container_class, attributes]) => { - if (attributes.range === maybe_range) { - return container_class; - } - } - ); - return maybe_range; - }; - - // const range_for_container = (container, container_class) => - // container.attributes[container_class].range; - - let list_data; - try { - let contentBuffer = await readFileAsync(file); - - if (file.type === 'application/json') { - let jsonData; - try { - jsonData = JSON.parse(contentBuffer.text); - } catch (error) { - throw new Error('Invalid JSON data', error); - } - - const container_class = container_for_range( - this.context.template.default.schema.classes.Container, - this.template_name - ); - - if (container_class) { - let dataObjects = jsonData.Container[container_class]; - list_data = this.loadDataObjects(dataObjects); - this.hot.loadData(list_data); - } - } else { - // assume tabular data if not a JSON datatype - list_data = this.loadSpreadsheetData(contentBuffer.binary); - this.hot.loadData(list_data); - } - } catch (err) { - console.error(err); - } - } - // Called by toolbox.js validate() async validate() { // const data = this.getTrimmedData(); @@ -259,14 +177,6 @@ class DataHarmonizer { this.hot.render(); } - newHotFile() { - this.context.runBehindLoadingScreen( - function () { - this.createHot(); - }.bind(this) - ); - } - /** * Create a blank instance of Handsontable. * @param {Object} template. @@ -281,174 +191,45 @@ class DataHarmonizer { } this.hot = new Handsontable(this.hotRoot, { - licenseKey: 'non-commercial-and-evaluation', + licenseKey: 'non-commercial-and-evaluation' }); - if (this.slots) { - /* This seems to be redundant now with newer Handsontable code. - this.clipboardCache = ''; - this.clipboardCoordCache = { - 'CopyPaste.copy': {}, - 'CopyPaste.cut': {}, - 'CopyPaste.paste': {}, - 'Action:CopyPaste': '', - }; // { startCol, startRow, endCol, endRow } - this.sheetclip = new SheetClip(); - - const hot_copy_paste_settings = { - afterCopy: function (changes, coords) { - self.clipboardCache = self.sheetclip.stringify(changes); - self.clipboardCoordCache['CopyPaste.copy'] = coords[0]; - self.clipboardCoordCache['Action:CopyPaste'] = 'copy'; - }, - afterCut: function (changes, coords) { - self.clipboardCache = self.sheetclip.stringify(changes); - self.clipboardCoordCache['CopyPaste.cut'] = coords[0]; - self.clipboardCoordCache['Action:CopyPaste'] = 'cut'; - }, - afterPaste: function (changes, coords) { - // we want to be sure that our cache is up to date, even if someone pastes data from another source than our tables. - self.clipboardCache = self.sheetclip.stringify(changes); - self.clipboardCoordCache['CopyPaste.paste'] = coords[0]; - self.clipboardCoordCache['Action:CopyPaste'] = 'paste'; - }, - - contextMenu: [ - 'copy', - 'cut', - { - key: 'paste', - name: 'Paste', - disabled: function () { - return self.clipboardCache.length === 0; - }, - callback: function () { - //var plugin = this.getPlugin('copyPaste'); - - //this.listen(); - // BUG: It seems like extra lf is added by sheetclip, causing - // empty last row to be added and pasted. Exception is pasting - // of single empty cell - //if (self.clipboardCache.length > 0) - // self.clipboardCache = self.clipboardCache.slice(0, -1); - //plugin.paste(self.clipboardCache); - }, - }, - 'remove_row', - 'row_above', - 'row_below', - ], + // This catches case where user dblclicks on column label, and just wants + // to see help info. Without this check, column is selected first, which + // blows away row selection. + Handsontable.hooks.add('beforeOnCellMouseDown', function(event, coords, element) { + // Check if a column header was clicked + if (coords.row < 0 && event.srcElement.classList.contains('secondary-header-text')) { + event.stopImmediatePropagation(); // Prevent default behavior and bubbling + } + }, this.hot); - }; - */ + if (this.slots) { const hot_settings = { //...hot_copy_paste_settings, data: data, // Enables true reset - nestedHeaders: this.getNestedHeaders(), + nestedHeaders: this.getNestedHeaders(), // Provides section / column rows. autoColumnSize: true, // Enable automatic column size calculation columns: this.getColumns(), colHeaders: true, rowHeaders: true, + renderallrows: false, + manualRowMove: true, copyPaste: true, - contextMenu: [ - { - key: 'remove_row', - name: 'Remove row', - callback: function () { - // Enables removal of a row and all dependent table rows. - // If there are 1-many cascading deletes, verify if that's ok. - let selection = self.hot.getSelected()[0][0]; - let warning = self.context.crudGetDependentChanges( - self.template_name - ); - if (!warning) { - self.hot.alter('remove_row', selection); - return true; - } - // Some cascading deletes to confirm here. - if ( - confirm( - 'WARNING: If you proceed, this will include deletion of one\n or more dependent records:\n' + - warning - ) - ) { - // User has seen the warning and has confirmed ok to proceed. - if (warning) { - // "True" parameter triggers deletions - self.context.crudGetDependentChanges( - self.template_name, - 'delete' - ); - self.hot.alter('remove_row', selection); - } - } - }, - }, - { - key: 'row_above', - name: 'Insert row above', - callback: function (action, selection, event) { - // Ensuring that rows inserted into foreign-key dependent tables - // take in the appropriate focused foreign-key values on creation. - self.addRows('insert_row_above', 1, self.hot.getSelected()[0][0]); - }, - }, - { - key: 'row_below', - name: 'Insert row below', - callback: function () { - // As above. - self.addRows( - 'insert_row_above', - 1, - parseInt(self.hot.getSelected()[0][0]) + 1 - ); - }, - }, - /*, - { - key: 'change_key', - name: 'Change key field', - // Disable on menu for dependent fields and fields that aren't - // in a unique_key - hidden: function () { - //let col = self.getSelectedRangeLast().to.col; - let col = self.current_selection[1]; - if (col == -1) // row selected here. - return true; - let slot = self.slots[col]; - let relations = self.context.relations[self.template_name]; - if (relations.dependent_slots?.[slot.name]) - return true; - // Hide if not a primary key field - return (!relations.unique_key_slots?.[slot.name]) - }, - // grid_coords = [0:{ start:{row:_,col:_}, end:{row:_,col:_} },... ] - callback: function (action, selection) { - let col = selection[0].start.col; - let slot = self.slots[col]; - let dependent_slots = self.context.relations[self.template_name].dependent_slots; - if (dependent_slots && slot.name in dependent_slots) { - let link = dependent_slots[slot.name]; - alert(`The [${slot.name}] key value in row ${parseInt(selection[0].start.row) + 1} can only be changed by editing the related parent ${link.parent} table [${link.slot}] record field.`); - return false; - } - let entry = prompt("Enter new value for this primary key field, or press Cancel"); - alert("Coming soon...") - - }, - }, - */ - ], outsideClickDeselects: false, // for maintaining selection between tabs manualColumnResize: true, //colWidths: [100], //Just fixes first column width - minRows: 100, - minSpareRows: 100, + minRows: 5, + minSpareRows: 0, width: '100%', // Future: For empty dependent tables, tailor to minimize height. height: '75vh', + fixedRowsTop: 0, + manualRowResize: true, + // define your custom query method, e.g. queryMethod: searchMatchCriteria + search: {}, fixedColumnsLeft: 1, // Future: enable control of this. + manualColumnFreeze: true, hiddenColumns: { copyPasteEnabled: true, indicators: true, @@ -463,6 +244,155 @@ class DataHarmonizer { invalidCellClassName: '', licenseKey: 'non-commercial-and-evaluation', + // observeChanges: true, // TEST THIS https://forum.handsontable.com/t/observechange-performance-considerations/3054 + + // TESTING May 15, 2025 + // Handsontable note: Directly replicating the full filter UI within the cell's right-click context menu is not a standard feature and would require significant custom development. + dropdownMenu: ['filter_by_condition', 'filter_by_value', 'filter_action_bar'], + + // https://handsontable.com/docs/javascript-data-grid/context-menu/ + contextMenu: { + items: { + remove_row: { + name: 'Remove row', + callback() { + // Enables removal of a row and all dependent table rows. + // If there are 1-many cascading deletes, verify if that's ok. + let selection = self.hot.getSelected()[0][0]; + let [change_report, change_message] = self.getChangeReport(self.template_name); + if (!change_message.length) { + self.hot.alter('remove_row', selection); + return true; + } + /* + * For deletes: (For now, ignore duplicate root key case: If + * encountering foreign key involving root_class slot, test if that has + * > 1 row. If so, delete ok without examining other dependents.) + */ + + // Some cascading deletes to confirm here. + if ( + confirm( + 'WARNING: If you proceed, this will include deletion of one\n or more dependent records, and this cannot be undone:\n' + + change_message + ) + ) { + // User has seen the warning and has confirmed ok to proceed. + for (let [dependent_name, dependent_obj] of Object.entries(change_report)) { + if (dependent_obj.rows?.length) { + let dh_changes = dependent_obj.rows.map(x => [x,1]); + self.context.dhs[dependent_name].hot.alter('remove_row', dh_changes); + } + }; + self.hot.alter('remove_row', selection); + } + }, + }, + row_above: { + name: 'Insert row above', + callback (action, selection, event) { + // Ensuring that rows inserted into foreign-key dependent tables + // take in the appropriate focused foreign-key values on creation. + self.addRows('insert_row_above', 1, self.hot.getSelected()[0][0]); + }, + }, + row_below: { + name: 'Insert row below', + callback() { + // As above. self.hot.toPhysicalRow() + self.addRows( + 'insert_row_above', + 1, parseInt(self.hot.getSelected()[0][0]) + 1 + ); + }, + }, + 'hsep1': '---------', + load_schema: { + name: 'Load LinkML schema.yaml', + hidden() { + return self.template_name != 'Schema'; + }, + callback() {$('#schema_upload').click();} + }, + save_schema: { + name: 'Save as LinkML schema.yaml', + hidden() { + return self.template_name != 'Schema'; + }, + callback() {self.context.schemaEditor.saveSchema(self)} + }, + // FUTURE Implementation + // Issue is that this doesn't freeze the column user is on. It + // freezes the first unfrozen column to left. + // Possibly switching to reference columns by name might fix. + /* + freeze_column: { + + name: 'freeze column' + }, + unfreeze_column: { + name: 'unfreeze column' + }, + */ + + /* Issue: Handsontable doesn't provide filter menu via cell + * like it does on column header. Only predefined filters + * can be triggered here. + filter_by_condition etc.: { + name: 'Filter by condition', + } + */ + + reset_filter: { + name: 'Reset filter', + disabled() { + const filtersPlugin = self.hot.getPlugin('filters'); + const filters = filtersPlugin.exportConditions(); + return (filters.length === 0); + }, + callback() { + const filtersPlugin = self.hot.getPlugin('filters'); + filtersPlugin.clearConditions(); + filtersPlugin.filter(); + } + }, + 'hsep3': '---------', + translations: { + name: 'Translations', + hidden() { + const schema = self.context.dhs.Schema; + // Hide if not in schema editor. + if (schema?.schema.name !== "DH_LinkML") return true; + }, + disabled() { + const schema = self.context.dhs.Schema; + // Hide if not in schema editor. + if (schema?.schema.name !== "DH_LinkML") return true; + // Hide if not translation fields. + if (!(self.template_name in self.context.schemaEditor.TRANSLATABLE)) return true; + // Hide if no locales + const current_row = schema.current_selection[0]; + if (current_row === null || current_row === undefined || current_row < 0) + return false; + const locales = schema.hot.getCellMeta(current_row, 0).locales; + return !locales; + }, + callback() {self.context.schemaEditor.translationForm(self)} + } + } + }, + // FIXING ISSUE WHERE HIGHLIGHTED FIELDS AREN'T IN GOOD SHAPE AFTER SORTING. + afterColumnSort: function (currentSortConfig, destinationSortConfigs) { + // if (self.schema.name === 'DH_LinkML' && self.template_name === 'Slot') { + // // Somehow on first call the this.context.dhs doesnt exist yet, so passing self. + // self.schemaSlotClassView(self); + // } + }, + + afterFilter: function (filters) { + // column: 0, conditions: ('cancogen_covid-19', "eq", [], operation: "conjunction" + + }, //beforePaste: //afterPaste: @@ -479,13 +409,21 @@ class DataHarmonizer { * null values. * beforeChange source: https://handsontable.com/docs/8.1.0/tutorial-using-callbacks.html#page-source-definition * + * Note: on fresh load of data - including empty dataset with default + * # rows, this event can be triggered "silently" on update of fields likedata + * empty provenance field. * @param {Array} grid_changes array of [row, column, old value (incl. undefined), new value (incl. null)]. - * @param {String} action can be CopyPaste.paste, ... + * @param {String} action can be CopyPaste.paste, multiselect_change, ... thisChange */ beforeChange: function (grid_changes, action) { - // Ignore addition of new records to table. - if (action == 'add_row') return; + // Ignore addition of new records to table. + //ISSUE: prevalidate fires on some things? + // "multiselect_change" occurs at moment when user clicks on column header. + if (['add_row','upload','prevalidate','multiselect','batch_updates'].includes(action)) + return; // Allow whatever was going to happen. if (!grid_changes) return; // Is this ever the case? + console.log('beforeChange', grid_changes, action); + /* TRICKY CASE: for tables depending on some other table for their * primary keys, grid_changes might involve "CopyPaste.paste" action @@ -494,19 +432,26 @@ class DataHarmonizer { * change list in this case to include insertion of primary key(s). */ - console.log('beforeChange', grid_changes, action); - /* Cut & paste can include 2 or more * changes in different columns on a row, or on multiple rows, so we * need to validate 1 row at a time. Row_changes data structure enables * this. */ let row_changes = self.getRowChanges(grid_changes); + console.log("row_changes", row_changes); // Validate each row of changes for (const row in row_changes) { let changes = row_changes[row]; + /* TEST for change in Schema in_language + * If new languages added or deleted, prompt to confirm. + */ + if (self.context.schemaEditor && self.template_name === 'Schema' + && 'locales' in changes) { + return self.context.schemaEditor.setLocales(changes); + }; + /* TEST FOR DUPLICATE IN IDENTIFIER SLOT * Change request cancelled (sudden death) if user tries to change * an identifier (unique key) value but this would result in a @@ -519,10 +464,10 @@ class DataHarmonizer { // Look for collision with an existing row's column value. // Change hasn't been implemented yet so we won't run into // change's row. - let search_row = self.context.crudFindByKeyVals(self, { + let search_row = self.context.crudFindRowByKeyVals(self, { [slot_name]: change.value, }); - if (search_row && search_row != row) { + if (search_row !== false && search_row != row) { alert( `Skipping change on row ${ parseInt(row) + 1 @@ -579,22 +524,18 @@ class DataHarmonizer { * "unique_key_slots": [ * "sample_collector_sample_id" * ], + * + * ISSUE: currently can set a key field to blank. PROBABLY block that. */ // Examine each unique_key for (let key_name in self.template.unique_keys) { - // Determine if key has a full set of values (found=true) + // Determine if key has a full set of values (complete=true) let [complete, changed, keyVals, change_log] = - self.context.crudHasCompleteUniqueKey( - self, - row, - key_name, - changes - ); - //let someEmptyKey = Object.entries(keyVals).some(([slot_name, value]) => !value); + self.context.crudHasCompleteUniqueKey(self, row, key_name, changes); if (complete && changed) { // Here we have complete unique_keys entry to test for duplication - let duplicate_row = self.context.crudFindByKeyVals( + let duplicate_row = self.context.crudFindRowByKeyVals( self, keyVals ); @@ -613,34 +554,47 @@ class DataHarmonizer { } } - // TEST FOR CHANGE TO DEPENDENT TABLE FOREIGN KEY(S) - // IF PRIMARY KEY CHANGE ACCEPTED BELOW AFTER PROMPT, THEN IMPLEMENT - // DATA CHANGE IN afterChange call...??? ISSUE is that technically - // the underlying table data would be getting changed before the - // change to this table's primary keys. + // UPDATE CASE: TEST FOR CHANGE TO DEPENDENT TABLE FOREIGN KEY(S) + // NOTE: if primary key change accepted below after prompt, then + // data change involked here is to dependent table data, and this + // happens in advance of root table key changes. + const change_prelude = `Your key change on ${self.template_name} + row ${parseInt(row) + 1} would also change existing dependent + table records, and this cannot be undone. Do you want to continue? Check:\n`; + + let [change_report, change_message] = self.getChangeReport(self.template_name, true, changes); + + //console.log("change report", change_report, change_message); + //console.log("relations", self.context.relations) + // confirm() presents user with report containing notice of subordinate changes + // that need to be acted on. + if (!change_message || confirm(change_prelude + change_message)) { + if (change_message) { + + // User has seen the warning and has confirmed ok to proceed. + for (let [dependent_name, dependent_obj] of Object.entries(change_report)) { + // Changes to current dh are done below. + if (dependent_name != self.template_name) { + if (dependent_obj.rows?.length) { + let dependent_dh = self.context.dhs[dependent_name]; + self.hot.suspendExecution(); // Advantageous to do this? + dependent_dh.hot.batchRender(() => { + for (let dep_row in dependent_obj.rows) { + Object.entries(dependent_obj.key_changed_vals).forEach(([dep_slot, dep_value]) => { + let dep_col = dependent_dh.slot_name_to_column[dep_slot]; + // While setDataAtCell triggers this beforeUpdate() again, 'batch_updates' event is trapped without further ado above. + dependent_dh.hot.setDataAtCell(dep_row, dep_col, dep_value, 'batch_updates'); + }) + } + }); // End of hot batch render + self.hot.resumeExecution(); + } + } + }; + }; // End of update action on dependent table keys, but not this dh table. - // TEST IF CHANGE IS TO A DEPENDENT TABLE's Foreign key. If not, IGNORE - let warning = self.context.crudGetDependentChanges( - self.template_name - ); - // confirm(warning) presents user with report containing warning text. - if ( - !warning || - confirm( - `Your key change on ${self.template_name} row ${ - parseInt(row) + 1 - } would also change existing dependent table records! Do you want to continue? Check:\n` + - warning - ) - ) { - if (warning) { - // "True" parameter triggers updates - self.context.crudGetDependentChanges( - self.template_name, - 'update' - ); - } } else return false; + } // end of row in row_change let triggered_changes = []; @@ -659,34 +613,106 @@ class DataHarmonizer { // If a change is carried out in the grid but user doesn't // change selection, e.g. just edits 1 cell content, then // an afterSelection isn't triggered. THEREFORE we have to - // trigger a crudFilterDependentViews() call. + // trigger a crudCalculateDependentKeys() call here. + // action e.g. edit, updateData, CopyPaste.paste afterChange: function (grid_changes, action) { - // e.g. action=edit. - // FUTURE: trigger only if change to primary key fields. - console.log('afterChange', grid_changes, action); - //let row_changes = self.getRowChanges(grid_changes); - //Ignore if partial key left by change. For full dependent key, - // Check changes against any dependent key slot. If match to at least - // one slot, then trigger crudFilterDependentViews. - self.context.crudFilterDependentViews(self.template_name); - }, + if (['upload','updateData','batch_updates'].includes(action)) { + // This is being called for every cell change in an 'upload' + return; + } - afterSelection: (row, column, row2, column2) => { - /* - * Test if user-focused row has changed. If so, then possibly - * foreign key values that make up some dependent table's record(s) - * have changed, so that table's view should be refreshed. Dependent - * tables determine what parent foreign key values they need to filter view by. - * This event is not involved in row content change. - * - * FUTURE: - */ - if (self.current_selection[0] != row) { - //console.log("afterSelection",self.template_name,self.context.relations) - self.context.crudFilterDependentViews(self.template_name); + // Trigger only if change to some key slot child needs. + if (grid_changes) { + let row_changes = self.getRowChanges(grid_changes); + console.log("afterChange()", row_changes, self.hasRowKeyChange(self.template_name, row_changes)) + if (self.hasRowKeyChange(self.template_name, row_changes)) { + self.context.crudCalculateDependentKeys(self.template_name); + // If in schema editor mode, update or insert to name field (a + // key field) of class, slot or enum (should cause update in + // compiled enums and slot flatVocabularies. + + if (self.context.schemaEditor) { + switch (self.template_name) { + case 'Schema': + // For a schema name change (the only key in schema table), + // update ALL related SchemaEditor Menus (classes, slots ...) + // for that template. + self.context.schemaEditor.refreshMenus(); + break; + + case 'Slot': + // A change in a Field/Slot table's row's schema + self.context.schemaEditor.refreshMenus(['SchemaSlotMenu']); + break; + + case 'Type': + case 'Class': + case 'Enum': + self.context.schemaEditor.refreshMenus([`Schema${self.template_name}Menu`]); + } + } + } } + }, + + /* + * Test if user-focused row has changed. If so, then possibly + * foreign key values that make up some dependent table's record(s) + * have changed, so that table's view should be refreshed. Dependent + * tables determine what parent foreign key values they need to filter view by. + * This event is not directly involved in row content change. + * + * As well, refresh schema menus if selected schema has changed. + * col can == -1 for entire row selection; row can == -1 or -2 for + * header row selection + */ + // https://handsontable.com/docs/javascript-data-grid/api/hooks/#afterselection + afterSelectionEnd: (row, column, row2, column2, selectionLayerLevel, action) => { + + //console.log("afterSelectionEnd",self.schema, row, column, row2, column2, selectionLayerLevel) + // This is getting triggered twice? Trap and exit + if (row === self.current_selection[0] + && column === self.current_selection[1] + && row2 === self.current_selection[2] + && column2 === self.current_selection[3] + ) { + console.log('Double event: afterSelectionEnd', row, column, row2, column2, selectionLayerLevel, action) + return false + } + // ORDER IS IMPORTANT HERE. Must calculate row_change, column_change, + // and then set current_selection so crudCalculateDependentKeys() can get + // the right possibly NEW schema_id !!! + // self.current_selection[0] === null || + // self.current_selection[1] === null || + const row_change = !(self.current_selection[0] === row); + const column_change = !(self.current_selection[1] === column); self.current_selection = [row, column, row2, column2]; + if (column < 0) column = 0; + if (row < 0) row = 0; + + // FUTURE Efficiency: test for change in dependent key value across + // rows, if none, skip this. + if (row_change) { // primary_key slot cell value change case handled in afterChange() event above. + self.context.crudCalculateDependentKeys(self.template_name); + + if (self.context.schemaEditor) { // Schema Editor specific function + if (self.template_name === 'Schema') { + self.context.schemaEditor.refreshMenus(); + } + if (self.template_name === 'Class' || self.template_name === 'Slot') { + // Have to update Slot > SlotUsage > slot_group menu for given Class. + // (Currently not having Schema Class itself control slot_group.) + // Find slot menu field and update source controlled vocab. + // ISSUE: Menu won't work/validate if multiple classes are displayed. + //const class_name = self.hot.getDataAtCell( + // row, self.slot_name_to_column['name'] + //); + self.context.schemaEditor.refreshMenus(['SchemaSlotMenu','SchemaSlotGroupMenu']); + } + } + } + // - See if sidebar info is required for top column click. if (this.helpSidebar) { if (column > -1) { @@ -695,8 +721,9 @@ class DataHarmonizer { self.helpSidebar.setContent(helpContent); } else self.helpSidebar.close(); } - return false; + return true; }, + // Bit of a hackey way to RESTORE classes to secondary headers. They are // removed by Handsontable when re-rendering main table. afterGetColHeader: function (column, TH, headerlev) { @@ -720,6 +747,7 @@ class DataHarmonizer { } } }, + afterRenderer: (TD, row, col) => { if (Object.prototype.hasOwnProperty.call(self.invalid_cells, row)) { if ( @@ -731,10 +759,22 @@ class DataHarmonizer { } } }, - ...this.hot_override_settings, + ...this.hot_override_settings, // custom overrides from DH setup. }; + // DEFAULT multiColumnSorting settings which are overriden in initTab() + hot_settings.multiColumnSorting = { + sortEmptyCells: true, // false = empty rows at end of table regardless of sort + indicator: true, // true = header indicator + headerAction: true, // true = header double click sort + } + + // Within the SchemaEditor template context, this adds settings based on + // particular tabs, e.g. Slot Editor "Slot". It modifies hot_settings above; + this.context.schemaEditor?.initTab(this, this.template_name, hot_settings); + this.hot.updateSettings(hot_settings); + this.enableMultiSelection(); } else { console.warn( @@ -742,25 +782,127 @@ class DataHarmonizer { ); } } + + + // Handsontable search extension for future use (fixed their example). + // DH search defaults to any substring match (See hot_settings above). + // Function below provides string exact match (incl. case sensitivity) + searchMatchCriteria (queryStr, value, metadata) { + return queryStr.toString() === value?.toString(); + } + + clearValidationResults() { + $('#next-error-button,#no-error-button').hide(); + this.invalid_cells = {}; + } + + + getLocales() { + if (this.template_name === 'Schema' && this.schema.name === 'DH_LinkML') { + // Locales are stored in 1st column of selected row of Schema class table. + const schema_metadata = this.hot.getCellMeta(this.current_selection[0], 0); + if (!('locales' in schema_metadata)) + schema_metadata.locales = {}; + return schema_metadata.locales + } + else { + // locales stored in a given DH's schema. + // Read-only, no opportunity to create them here. + return this.schema.extensions?.locales?.value; + } + } - /* Create row_change to validate cell changes, row by row. It is a + /** + * Returns a dependent_report of changed table rows, as well as an affected + * key slots summary. Given changes object is user's sot edits, if any, on + * a row. + * + * The skippable = true parameter enables this report to exit quickly rather + * than doing a deep recursive dive. This is for the case where we are + * testing an update action on class_name, but if no dependents are affected + * then we don't have to do a dive into examining each of its dependents. + * Both view and delete actions require deep dive so are not skippable. + * + * Note, this can be triggered per row where say an empty data provenance field + * is updated silently. + * + * @param {String} class_name name of class (DataHarmonizer instance) to start from. + * @param {Boolean} skippable: yes = don't do deep dive if no keys have changed for class name. + * @param {Object} changes dictionary pertaining to a particular dh row's slots. + * @returns {Object} dependent_report dictionary of class_names and the key slot related data and change-affected rows + * @returns {String} change_message echoes description of entailed changes in dependent tables. + */ + getChangeReport(class_name, skippable=false, changes = {}) { + let dependent_report = {}; + let change_message = ''; + + // If skippable, determine if changes apply to any target slots. + // If no pertinent key changes, return an empty report + if (skippable && ! this.hasRowKeyChange(class_name, {0:changes})) { //dummy row_changes row + return [dependent_report, change_message]; + } + + // Copy and generate messaging for cascading classes and their related rows report + this.context.crudGetDependentRows(class_name, skippable, changes); + for (let [dependent_name] of this.context.relations[class_name].dependents.entries()) { + + let dependent_rep = this.context.dependent_rows.get(dependent_name); + console.log("report for ", dependent_name, dependent_rep) + dependent_report[dependent_name] = dependent_rep; + + if (class_name != dependent_name && dependent_rep.count) { + let fkey_vals = Object.entries(dependent_rep.fkey_vals) + .filter(([x, y]) => y != null) + .join('\n ') + .replaceAll(',', ': '); + change_message += `\n - ${dependent_rep.count} ${dependent_name} record(s) with key:\n ${fkey_vals}`; + } + } + return [dependent_report, change_message]; + } + + + hasRowKeyChange(class_name, changes) { + // OBSOLETE: If a change affects a slot that is a target of other tables, return true. + // UPDATED: using .unique_key_slots rather than .target_slots + const class_relations = this.context.relations[class_name]; + for (const key in class_relations.unique_key_slots) { //, ...class_relations.target_slots] + for (const [row, change] of Object.entries(changes)) { + //console.log("CHANGE",class_name, "change", change,"KEY", key, row) + if (key in change) { + return true; + } + } + } + return false; + } + + /* Return row_change to validate cell changes, row by row. + It is a * tacitly ordered dict of row #s. * {[row]: {[slot_name]: {old_value: _, value: _} } } */ + // GRID CHANGES NOW REFERENCES slot_name aka column name getRowChanges(grid_changes) { - let row_change = {}; + let row_changes = {}; for (const change of grid_changes) { + // FUTURE switch to this when implementing Handsontable col.data=[slot_name] + // let slot = this.slots[this.slot_name_to_column[change[1]]] let slot = this.slots[change[1]]; + if (!slot) + continue; // Case possibly happens on cut and paste or undo function. + let row = change[0]; // A user entry of same value as existing one is // still counted as a change, so blocking that case here. // Also blocking any falsy equality - null & '' & undefined if (change[2] !== change[3] || !change[2] != !change[3]) { - if (!(row in row_change)) { - row_change[row] = {}; + if (!(row in row_changes)) { + row_changes[row] = {}; } - row_change[change[0]][slot.name] = { + + row_changes[row][slot.name] = { slot: slot, col: change[1], old_value: change[2], @@ -768,7 +910,7 @@ class DataHarmonizer { }; } } - return row_change; + return row_changes; } updateColumnSettings(columnIndex, options) { @@ -789,12 +931,14 @@ class DataHarmonizer { */ addRows(row_where, numRows, startRowIndex = false) { numRows = parseInt(numRows); // Coming from form string input. - // Get the starting row index where the new rows will be added - if (startRowIndex === false) startRowIndex = this.hot.countSourceRows(); - + // Get the VISUAL (not source) starting row index where the new rows will be added + if (startRowIndex === false) { + // Count visual row because hot.alter below modifies visual rows not source rows + startRowIndex = this.hot.countRows(); //countSourceRows() + } let parents = this.context.crudGetParents(this.template_name); // If this has no foreign key parent table(s) then go ahead and add x rows. - if (!parents) { + if (isEmpty(parents)) { // Insert the new rows below the last existing row this.hot.alter(row_where, startRowIndex, numRows); return; @@ -806,13 +950,16 @@ class DataHarmonizer { this.context.crudGetForeignKeyValues(parents); if (errors) { // Prompt user to select appropriate parent table row(s) first. + // FRINGE ISSUE: When user selects row/cell on subordinate table, then mouse-clicks on disabled tab, and then add-row, they get popup pertaining to disabled tab table. $('#empty-parent-key-modal-info').html(errors); $('#empty-parent-key-modal').modal('show'); return; } + // NEW ROWS ARE ADDED HERE. this.hot.alter(row_where, startRowIndex, numRows); - + this.hot.selectCell(startRowIndex, 0); + this.hot.scrollViewportTo({ row: startRowIndex}); // Populate new rows with selected foreign key value(s) // "add_row" is critical so we can ignore that in before_change event. // Otherwise a dependent view with "this change will impact ..." @@ -820,22 +967,13 @@ class DataHarmonizer { this.hot.batchRender(() => { for (let row = startRowIndex; row < startRowIndex + numRows; row++) { Object.entries(required_selections).forEach(([slot_name, value]) => { - const col = this.getColumnIndexByFieldName(slot_name); - this.hot.setDataAtCell(row, col, value.value, 'add_row'); + const col = this.slot_name_to_column[slot_name]; + this.hot.setDataAtCell(row, parseInt(col), value.value, 'add_row'); }); } }); } - getColumnIndexByFieldName(slot_name) { - for (let i = 0; i < this.slots.length; i++) { - if (looseMatchInObject(['name'])(this.slots[i])(slot_name)) { - return i; - } - } - return -1; - } - /** * Hides the columns at the specified indexes within the Handsontable instance. * @@ -897,6 +1035,7 @@ class DataHarmonizer { } } + // FUTURE: use Handsontable filter instead. changeRowVisibility(id) { // Grid becomes sluggish if viewport outside visible grid upon re-rendering this.hot.scrollViewportTo(0, 1); @@ -926,23 +1065,29 @@ class DataHarmonizer { this.render(); } + // Hide all rows filterAll(dh) { // Access the Handsontable instance's filter plugin const filtersPlugin = dh.hot.getPlugin('filters'); + if (filtersPlugin.isEnabled()) { - // Clear any existing filters - filtersPlugin.clearConditions(); + // Clear any existing filters + filtersPlugin.clearConditions(); - // Add a filter condition that no row will satisfy - // For example, set a condition on the first column to check if the value - // equals a non-existent value - filtersPlugin.addCondition(0, 'eq', '###NO_ROW_MATCH###'); + // Add a filter condition that no row will satisfy + // For example, set a condition on the first column to check if the value + // equals a non-existent value + filtersPlugin.addCondition(0, 'eq', ['###NO_ROW_MATCH###']); - // Apply the filter to hide all rows - filtersPlugin.filter(); + dh.hot.suspendExecution(); + // Apply the filter to hide all rows + filtersPlugin.filter(); + dh.hot.resumeExecution(); + } } filterAllEmpty(dh) { + dh.hot.suspendExecution(); const filtersPlugin = dh.hot.getPlugin('filters'); // Clear any existing filters @@ -954,15 +1099,20 @@ class DataHarmonizer { }); filtersPlugin.filter(); + dh.hot.resumeExecution(); } + + + // Ensuring popup hyperlinks occur for any URL in free-text. renderSemanticID(curieOrURI, as_markup = false) { if (curieOrURI) { if (curieOrURI.toLowerCase().startsWith('http')) { if (as_markup) return `[${curieOrURI}](${curieOrURI})`; return `${curieOrURI}`; - } else if (curieOrURI.includes(':')) { + } + else if (curieOrURI.includes(':')) { const [prefix, reference] = curieOrURI.split(':', 2); if (prefix && reference && prefix in this.schema.prefixes) { // Lookup curie @@ -1000,7 +1150,7 @@ class DataHarmonizer { padding:10px; font-size:1.5rem; } - + .coding_name {font-size: .7rem} table th {font-weight: bold; text-align: left; font-size:1.3rem;} table th.label {font-weight:bold; width: 25%} table th.description {width: 20%} @@ -1011,28 +1161,40 @@ class DataHarmonizer { table td.label {font-weight:bold;} ul { padding: 0; } + span.required {font-weight:normal;background-color:yellow} + span.recommended {font-weight:normal;background-color:plum} `; if (mystyle != null) { style = mystyle; } + let enum_list = {}; // Build list of enums actually used in this template let row_html = ''; + for (const section of this.sections) { row_html += ` -

${ +

${ section.title || section.name - }

+ } `; + for (const slot of section.children) { + if (slot.sources) { + for (let source of slot.sources) + enum_list[source] = true; + } const slot_dict = this.getCommentDict(slot); const slot_uri = this.renderSemanticID(slot_dict.slot_uri); row_html += ''; if (this.columnHelpEntries.includes('column')) { - row_html += `${slot_dict.title}
${slot_uri}`; + row_html += ` + ${slot_dict.title}
+ (${slot_dict.name})
+ ${slot_uri} ${slot.required ? ' required ':''} ${slot.recommended ? ' recommended ':''}`; } if (this.columnHelpEntries.includes('description')) { row_html += `${slot_dict.description}`; @@ -1050,33 +1212,37 @@ class DataHarmonizer { } } - // Note this may include more enumerations than exist in a given template. - let enum_html = ``; - console.log(this.schema.enums); + // Only include enumerations that exist in a given template. + let enum_html = ''; for (const key of Object.keys(this.schema.enums).sort()) { - const enumeration = this.schema.enums[key]; - let title = - enumeration.title != enumeration.name ? enumeration.title : ''; - enum_html += ` - ${enumeration.name} - "${title}"
${this.renderSemanticID( - enumeration.enum_uri - )} - `; - - for (const item_key in enumeration.permissible_values) { - const item = enumeration.permissible_values[item_key]; - let text = item.text == item_key ? '' : item.text; - let title = !item.title || item.title == item_key ? '' : item.title; - enum_html += ` - ${this.renderSemanticID(item.meaning)} - ${item_key} - ${text} - ${title} + if (key in enum_list) { + const enumeration = this.schema.enums[key]; + enum_html += ` + ${enumeration.title ? enumeration.title + '
(' + enumeration.name + ')' : enumeration.name} ${this.renderSemanticID( + enumeration.enum_uri)} `; + + for (const item_key in enumeration.permissible_values) { + const item = enumeration.permissible_values[item_key]; + let title = !item.title || item.title == item_key ? '' : item.title; + enum_html += ` + ${this.renderSemanticID(item.meaning)} + ${item_key} + ${title} + ${item.description || ''} + `; + } } } + if (enum_html) + enum_html = ` + ${i18next.t('help-semantic_uri')} + ${i18next.t('help-sidebar__code')} + ${i18next.t('help-sidebar__title')} + ${i18next.t('help-sidebar__description')} + ` + enum_html; + var win = window.open( '', 'Reference', @@ -1140,10 +1306,11 @@ class DataHarmonizer {
+
- + ${enum_html} @@ -1203,55 +1370,10 @@ class DataHarmonizer { parseInt(column), true ); + //Ensures field is positioned on left side of screen. this.hot.scrollViewportTo(row, column); - } - /***************************** PRIVATE functions *************************/ - - /** - * Load data into table as an array of objects. The keys of each object are - * field names and the values are the cell values. - * - * @param {Array} data table data - */ - loadDataObjects(data, locale = null) { - if (typeof data === 'object' && !Array.isArray(data) && data !== null) { - // An object was provided, so try to pick out the grid data from - // one of it's slots. - const inferredIndexSlot = this.getInferredIndexSlot(); - - if (inferredIndexSlot) { - data = data[inferredIndexSlot]; - } else { - const dataKeys = Object.keys(data); - if (dataKeys.length === 1) { - data = data[dataKeys[0]]; - } - } - } - - if (!Array.isArray(data)) { - console.warn('Unable to get grid data from input'); - return; - } - let listData = []; - data.forEach((row) => { - const dataArray = dataObjectToArray(row, this.slots, { - serializedDateFormat: this.dateExportBehavior, - dateFormat: this.dateFormat, - datetimeFormat: this.datetimeFormat, - timeFormat: this.timeFormat, - translationMap: - locale !== null - ? invert(i18next.getResourceBundle(locale, 'translation')) - : undefined, - }); - listData.push(dataArray); - }); - const matrixFieldChange = this.matrixFieldChangeRules(listData); - - return matrixFieldChange; } /** @@ -1261,120 +1383,66 @@ class DataHarmonizer { * In cases where headers need to be defined or are incomplete, a modal is triggered * for user specification. * + * Trying to offer flexibility that user can load a schema class from a + * spreadsheet that may have several tabs with variations on tab name as + * class name or title or as a Container attribute name. That doesn't mean + * the sheet can be re-saved back into its source file though. + * + * This runs separately for each tab in schema. HOWEVER - all tabs are + * sharing a dialogue about whether tabular data lines up with specification. + * + * WARNING: making this async messes with loading order, leading to some tabs + * not having data? + * @param {string|Buffer} data - The binary string or buffer of the spreadsheet data. * @returns {Array>|null} A matrix representing the processed spreadsheet data. * Returns `null` if headers need to be specified by the user. * @throws Will throw an error if the given data cannot be read as a workbook. */ - loadSpreadsheetData(data) { - const workbook = xlsxRead(data, { - type: 'binary', - raw: true, - cellDates: true, // Ensures date formatted as YYYY-MM-DD dates - dateNF: 'yyyy-mm-dd', //'yyyy/mm/dd;@' - }); - - const worksheet = workbook.Sheets[this.template_name] - ? updateSheetRange(workbook.Sheets[this.template_name]) - : updateSheetRange(workbook.Sheets[workbook.SheetNames[0]]); - const matrix = XlsxUtils.sheet_to_json(worksheet, { - header: 1, - raw: false, - range: 0, - }); - const headerRowData = this.compareMatrixHeadersToGrid(matrix); - if (headerRowData > 0) { - return this.matrixFieldChangeRules(matrix.slice(headerRowData)); - } else { - this.launchSpecifyHeadersModal(matrix); - return null; + getSpreadsheetName(workbook) { + + // Defaults to 1st tab if current DH instance doesn't match sheet tab. + // Excel xls, xlsx worksheet tabs are plural container attribute names + // rather than template_name or template title. + const sheet_names = workbook.SheetNames; + + // Default is to try processing first tab if no tab name match + // If loading a tsv or csv file, default will be something like "Sheet1". + let sheet_name = sheet_names[0]; + // template.name is language-invariant so most reliable + if (sheet_names.includes(this.template.name)) + sheet_name = this.template.name; + // This is a Language dependent option: + else if (this.template.title && sheet_names.includes(this.template.title)) { + sheet_name = this.template.title; + } + // If loading tsv or csv, don't try matching sheet name to Container + else if (sheet_names.length > 1 && 'Container' in this.schema.classes) { + // A given Container attribute [e.g. CanCOGeNCovid19Data] may have a + // .range that matches this template name. If so, find the container + // attribute name (usually plural) as spreadsheet tab name. + const match = Object.entries(this.schema.classes['Container'].attributes) + // cls_key = name = usually plural container attribute name + .find(([cls_key, { name, range }]) => range == this.template.name); + if (match) { + sheet_name = match[0]; + } } - } - - /** - * Ask user to specify row in matrix containing secondary headers before load. - * Calls `alertOfUnmappedHeaders` if necessary. - * @param {Array} matrix Data that user must specify headers for. - * @param {Object} hot Handsontable instance of grid. - * @param {Object} data See `data.js`. - */ - launchSpecifyHeadersModal(matrix) { - let flatHeaders = this.getFlatHeaders(); - const self = this; - if (flatHeaders) { - $('#field-mapping').prepend( - ''.repeat(flatHeaders[1].length + 1) - ); - $('#expected-headers-tr').html( - '' - ); - $('#actual-headers-tr').html( - '' - ); - flatHeaders[1].forEach(function (item, i) { - if (item != matrix[1][i]) { - $('#field-mapping col').get(i + 1).style.backgroundColor = 'orange'; - } - }); - - $('#specify-headers-modal').modal('show'); - $('#specify-headers-confirm-btn').click(() => { - const specifiedHeaderRow = parseInt($('#specify-headers-input').val()); - if (!isValidHeaderRow(matrix, specifiedHeaderRow)) { - $('#specify-headers-err-msg').show(); - } else { - // Try to load data again using User specified header row - const mappedMatrixObj = self.mapMatrixToGrid( - matrix, - specifiedHeaderRow - 1 - ); - $('#specify-headers-modal').modal('hide'); - self.context.runBehindLoadingScreen(() => { - self.hot.loadData( - self.matrixFieldChangeRules(mappedMatrixObj.matrix.slice(2)) - ); - if (mappedMatrixObj.unmappedHeaders.length) { - const unmappedHeaderDivs = mappedMatrixObj.unmappedHeaders.map( - (header) => `
  • ${header}
    • ` - ); - $('#unmapped-headers-list').html(unmappedHeaderDivs); - $('#unmapped-headers-modal').modal('show'); - } - return true; // Removes runBehindLoadingScreen() opacity. - }); - } - }); + //console.log("WORKBOOK", workbook.Sheets, sheet_name) + if (!(sheet_name in workbook.Sheets)) { + return false; } - } + return sheet_name; - /** - * Determine if first or second row of a matrix has the same headers as the - * grid's secondary (2nd row) headers. If neither, return false. - * @param {Array>} matrix - * @param {Object} data See `data.js`. - * @return {Integer} row that data starts on, or false if no exact header row - * recognized. - */ - compareMatrixHeadersToGrid(matrix) { - const expectedSecondRow = this.getFlatHeaders()[1]; - const actualFirstRow = matrix[0]; - const actualSecondRow = matrix[1]; - if (JSON.stringify(expectedSecondRow) === JSON.stringify(actualFirstRow)) { - return 1; - } - if (JSON.stringify(expectedSecondRow) === JSON.stringify(actualSecondRow)) { - return 2; - } - return false; } + /***************************** PRIVATE functions *************************/ + /** * Construct a dictionary of source field names pointing to column index + * USED IN GRDI export.js + * * @param {Object} fields A flat version of data.js. * @return {Map} Dictionary of all fields. */ @@ -1388,6 +1456,8 @@ class DataHarmonizer { /** * Construct a dictionary of source field TITLES pointing to column index + * USED IN GRDI export.js + * * @param {Object} fields A flat version of data.js. * @return {Map} Dictionary of all fields. */ @@ -1399,87 +1469,6 @@ class DataHarmonizer { return map; } - /** - * Create a matrix containing the grid's headers. Empty strings are used to - * indicate merged cells. - * @return {Array>} Grid headers. - */ - getFlatHeaders() { - const rows = [[], []]; - - for (const parent of this.sections) { - let min_cols = parent.children.length; - if (min_cols < 1) { - // Close current dialog and switch to error message - //$('specify-headers-modal').modal('hide'); - //$('#unmapped-headers-modal').modal('hide'); - const errMsg = `The template for the loaded file has a configuration error:
    - ${parent.title}
    - This is a field that has no parent, or a section that has no fields.`; - $('#unmapped-headers-list').html(errMsg); - $('#unmapped-headers-modal').modal('show'); - - return false; - } - rows[0].push(parent.title); - // pad remainder of first row columns with empty values - if (min_cols > 1) { - rows[0].push(...Array(min_cols - 1).fill('')); - } - // Now add 2nd row child titles - rows[1].push(...parent.children.map((child) => child.title)); - } - return rows; - } - - /** - * Map matrix columns to grid columns. - * Currently assumes mapped columns will have the same label, but allows them - * to be in a different order. If the matrix is missing a column, a blank - * column is used. - * @param {Array>} matrix - * @param {Number} matrixHeaderRow Row containing matrix's column labels. - * @return {MappedMatrixObj} Mapped matrix and details. - */ - mapMatrixToGrid(matrix, matrixHeaderRow) { - const expectedHeaders = this.getFlatHeaders(); - const expectedSecondaryHeaders = expectedHeaders[1]; - const actualSecondaryHeaders = matrix[matrixHeaderRow]; - - // Map current column indices to their indices in matrix to map - const headerMap = {}; - const unmappedHeaders = []; - for (const [i, expectedVal] of expectedSecondaryHeaders.entries()) { - headerMap[i] = actualSecondaryHeaders.findIndex((actualVal) => { - return actualVal === expectedVal; - }); - if (headerMap[i] === -1) { - unmappedHeaders.push(expectedVal); - } - } - - const dataRows = matrix.slice(matrixHeaderRow + 1); - const mappedDataRows = []; - // Iterate over non-header-rows in matrix to map - for (const i of dataRows.keys()) { - mappedDataRows[i] = []; - // Iterate over columns in current validator version - for (const j of expectedSecondaryHeaders.keys()) { - // -1 means the matrix to map does not have this column - if (headerMap[j] === -1) { - mappedDataRows[i][j] = ''; - } else { - mappedDataRows[i][j] = dataRows[i][headerMap[j]]; - } - } - } - - return { - matrix: [...expectedHeaders, ...mappedDataRows], - unmappedHeaders: unmappedHeaders, - }; - } - /** * Create a matrix containing the nested headers supplied to Handsontable. * These headers are HTML strings, with useful selectors for the primary and @@ -1506,11 +1495,10 @@ class DataHarmonizer { } /** - * Create an array of cell properties specifying data type for all grid columns. - * AVOID EMPLOYING VALIDATION LOGIC HERE -- HANDSONTABLE'S VALIDATION - * PERFORMANCE IS AWFUL. WE MAKE OUR OWN IN `VALIDATE_GRID`. - * - * REEXAMINE ABOVE ASSUMPTION - IT MAY BE BECAUSE OF set cell value issue that is now solved. + * Create an array of cell properties specifying data type for all grid + * columns. In past avoided handsontable validation here but this should be + * reexamined. IT MAY BE that using batch() operations and not setDataAtCell, + * that this is now solved. * * @param {Object} data See TABLE. * @return {Array} Cell properties for each grid column. @@ -1518,86 +1506,73 @@ class DataHarmonizer { getColumns() { let ret = []; - const fields = this.slots; - // NOTE: this single loop lets us assume that columns are produced in the same indexical order as the fields // Translating between columns and fields may be necessary as one can contain more information than the other. - for (let field of fields) { + for (let slot of this.slots) { let col = {}; - if (field.required) { - col.required = field.required; + if (slot.required) { + col.required = slot.required; } - if (field.recommended) { - col.recommended = field.recommended; + if (slot.recommended) { + col.recommended = slot.recommended; } - col.name = field.name; - + col.name = slot.name; + // FUTURE: Implement this - it allows columns to be referenced by name, + // but affects a number of handsontable calls, e.g. event + // grid_changes[1] is slot name instead of integer. + // col.data = slot.name; + + /* A field with .sources indicates that one or more enumerations + * are involved, and so it should use a select or multiselect + * pulldown menu. + * Both have to handle indentation of choices where enumeration + * has "is_a" parents. + * Note that single select pulldown list handled by enablemultiselection() + * has div class="handsontableEditor listbox handsontable". + */ col.source = null; - - if (field.sources) { - const options = field.sources.flatMap((source) => { - if (field.permissible_values[source]) - return Object.values(field.permissible_values[source]).reduce( - (acc, item) => { - acc.push({ - label: titleOverText(item), - value: titleOverText(item), - _id: item.text, - }); - return acc; - }, - [] - ); - else { - alert( - 'Schema Error: Slot mentions enumeration ' + - source + - ' but this was not found in enumeration dictionary, or it has no selections' - ); - return []; - } - }); - - col.source = options; - if (field.multivalued === true) { - col.editor = 'text'; - col.renderer = multiKeyValueListRenderer(field); - col.meta = { - datatype: 'multivalued', // metadata - }; + if (slot.sources) { + col.source = this.updateSources(slot); + if (slot.multivalued === true) { + col.type = 'autocomplete'; + // Prevents 'autocomplete' renderer which is regular pulldown from + // showing as well: + col.editor = 'text'; + col.renderer = 'multiKeyValueListRenderer'; + col.meta = {datatype: 'multivalued'} // metadata } else { + // Autocomplete allows filter as you type selection but source expecting just a single word. + //col.type = 'autocomplete'; // Issue is HandsonTable 13.0.1 doesn't provide indentation. col.type = 'key-value-list'; - if ( - !field.sources.includes('NullValueMenu') || - field.sources.length > 1 - ) { - col.trimDropdown = false; // Allow expansion of pulldown past field width - } } } - if (field.datatype == 'xsd:date') { - col.type = 'dh.date'; - col.allowInvalid = true; // making a difference? - col.flatpickrConfig = { - dateFormat: this.dateFormat, //yyyy-MM-dd - enableTime: false, - }; - } else if (field.datatype == 'xsd:dateTime') { - col.type = 'dh.datetime'; - col.flatpickrConfig = { - dateFormat: this.datetimeFormat, - }; - } else if (field.datatype == 'xsd:time') { - col.type = 'dh.time'; - col.flatpickrConfig = { - dateFormat: this.timeFormat, - }; + switch (slot.datatype) { + case 'xsd:date': + col.type = 'dh.date'; + col.allowInvalid = true; // making a difference? + col.flatpickrConfig = { + dateFormat: this.dateFormat, //yyyy-MM-dd + enableTime: false, + }; + break; + case 'xsd:dateTime': + col.type = 'dh.datetime'; + col.flatpickrConfig = { + dateFormat: this.datetimeFormat, + }; + break; + case 'xsd:time': + col.type = 'dh.time'; + col.flatpickrConfig = { + dateFormat: this.timeFormat, + }; + break; } - if (typeof field.getColumn === 'function') { - col = field.getColumn(this, col); + if (typeof slot.getColumn === 'function') { + col = slot.getColumn(this, col); } ret.push(col); @@ -1606,8 +1581,52 @@ class DataHarmonizer { return ret; } + /** Function for initializing Handsontable dropdown and key-value-list element + * source. Call to refresh a given col specification with dynamic content. + * Note that Dataharmonizer ".sources" => Column ".source" + */ + updateSources(slot) { + + let options = []; + + slot.sources.forEach((source) => { + let stack = []; + + if (!(source in slot.permissible_values)) { + alert(`Schema Programming Error: Slot range references enumeration ${source} but this was not found in enumeration dictionary`); + } + // This case catches empty menus, which merits an error in end-user DH but is ok in Schema Editor. + else if ((this.schema.name !== "DH_LinkML") && !slot.permissible_values[source]) { + alert(`Schema Programming Error: Slot range enumeration's ${source} has no selections!`); + } + else + Object.values(slot.permissible_values[source] || {}).forEach( + (permissible_value) => { + let level = 0; + const code = permissible_value.text; + if ('is_a' in permissible_value) { + level = stack.indexOf(permissible_value.is_a) + 1; + stack.splice(level + 1, 1000, code); + } else { + stack = [code]; + } + + options.push({ + depth: level, + label: permissible_value.title || code, + value: code, + _id: code, // Used by picklist renderer. + }); + + } + ); + }); + //console.log("picklist for ", slot.name, options, slot.multivalued) + return options; + }; + /** - * Enable multiselection on select rows. + * Enable multiselection input control on cells which have options and a multivalued metadata type. * Indentation workaround: multiples of " " double space before label are * taken to be indentation levels. * @param {Object} hot Handonstable grid instance. @@ -1620,67 +1639,99 @@ class DataHarmonizer { const dh = this.hot; this.hot.updateSettings({ afterBeginEditing: function (row, col) { - if ( - self.slots[col].flatVocabulary && - self.slots[col].multivalued === true - ) { + let meta = dh.getColumnMeta(col); + if (meta.source && meta.meta?.datatype === 'multivalued') { + + /* const value = dh.getDataAtCell(row, col); const selections = parseMultivaluedValue(value); const formattedValue = formatMultivaluedValue(selections); - // Cleanup of empty values that can occur with leading/trailing or double ";" + // Cleanup of empty values that can occur with user editing in popup leading/trailing or double ";" if (value !== formattedValue) { - dh.setDataAtCell(row, col, formattedValue, 'thisChange'); + dh.setDataAtCell(row, col, formattedValue, 'prevalidate'); } let content = ''; - if (self.slots[col].sources) { - self.slots[col].sources.forEach((source) => { - Object.values(self.slots[col].permissible_values[source]).forEach( - (permissible_value) => { - const field = permissible_value.text; - const field_trim = field.trim(); - let selected = selections.includes(field_trim) - ? 'selected="selected"' - : ''; - content += ``; - } - ); - }); - } + Object.values(meta.source).forEach(choice => { + const {label, value} = choice; // unpacking + let selected = selections.includes(value) + ? 'selected="selected"' + : ''; + content += ``; + }); $('#field-description-text').html( - ` - ${self.slots[col].title} - + `${self.slots[col].title} ` ); - $('#field-description-modal').modal('show'); - $('#field-description-text .multiselect') + */ + $('#multiselect-text').html( + `${self.slots[col].title} + ` + ); + + $('#multiselect-text .multiselect') .selectize({ maxItems: null, + searchField: ["_id","label"], // for autocomplete to work + //selectOnTab: false, + options: meta.source, + delimiter: MULTIVALUED_DELIMITER, + items: dh.getDataAtCell(row, col)?.split(MULTIVALUED_DELIMITER) || [], + + hideSelected: false, + onChange: function(value) { + this.focus(); // required for cursor + }, + /* + onBlur: function(e, dest) {alert('blurring')}, // works + onDelete: function(values) { // works + return confirm(values = array of deleted items); + }, + */ render: { + // This is the label shown for the selected item in the popup + // input control. item: function (data, escape) { - return `
    ` + escape(data.text) + `
    `; + return '
    ' + escape(data.label) + '
    '; }, + // This is the option in popup input control option list. option: (data, escape) => { - const value = data.value.trim(); - let indentation = 12 + data.text.trim().search(/\S/) * 8; // pixels - return `
    ${escape(data.text)}
    `; + //const value = data.value.trim(); + // Depth is coming from col.source created by updateSources(slot) + let indentation = 'selectDepth_' + (parseInt(data.depth) + 1); + return `
    ${escape(data.label)}
    `; }, + }, }) // must be rendered when html is visible + // See https://selectize.dev/docs/events + /*.on('destroy', ...) not working because .destroy() never fired. + + // Problem: change fires 'beforeChange' on each setDataAtCell .on('change', function () { let newValCsv = formatMultivaluedValue( $('#field-description-text .multiselect').val() ); - dh.setDataAtCell(row, col, newValCsv, 'thisChange'); + dh.setDataAtCell(row, col, newValCsv, 'multiselect_add'); }); - // Saves users a click: - $('#field-description-text .multiselect')[0].selectize.focus(); + */ + + // HACK TO GET A SINGLE beforeChange event on "OK" of .selectize + // so that validation happens only once on updated multiselect items. + // This is key to saving selected content. + $('#multiselect-modal button[data-dismiss]').off().on('click', function () { + let newValCsv = formatMultivaluedValue( + $('#multiselect-text .multiselect').val() + ); + dh.setDataAtCell(row, col, newValCsv, 'multiselect_change'); + }) + + $('#multiselect-modal').modal('show'); + // Automatically opens menu below input box + $('#multiselect-text .multiselect')[0].selectize.focus(); } }, }); @@ -1733,14 +1784,14 @@ class DataHarmonizer { if (this.columnHelpEntries.includes('column')) { ret += `

    ${i18next.t( 'help-sidebar__column' - )}: ${field.title || field.name}

    `; + )}: ${slot_dict.title}
    (${slot_dict.name})

    `; } // Requires markup treatment of URLS. const slot_uri = this.renderSemanticID(field.slot_uri); if (field.slot_uri && this.columnHelpEntries.includes('slot_uri')) { ret += `

    ${i18next.t( - 'help-sidebar__slot_uri' + 'help-semantic_uri' )}: ${slot_uri}

    `; } @@ -1772,17 +1823,7 @@ class DataHarmonizer { } getCommentDict(field) { - /* HACK: due to nested arrays, this needs to keep track of when comments - * and examples need to be overriden. This might be happening elsewhere in - * the codebase (particularly with slices/...). Our version of deepMerge - * forces array overrides, considered only with dictionaries as objects. - ISSUE: THIS SHOULDN'T BE NECESSARY WITH NEW CODE!!! - ENUMERATIONS AREN'T GETTING TRANSLATED? - - let field = deepMerge( //uses getCommentDict(input_field) - deepMerge(input_field, this.context.template.default.schema.slots[input_field.name]), - this.context.template.localized.schema.slots[input_field.name] - ); + /* ENUMERATIONS AREN'T GETTING TRANSLATED? */ let guide = { title: field.title, @@ -1796,7 +1837,7 @@ class DataHarmonizer { }; let guidance = []; - if (field.comments && field.comments.length) { + if (field.comments && field.comments.length > 0) { guidance = guidance.concat(field.comments); } if (field.pattern) { @@ -1844,20 +1885,6 @@ class DataHarmonizer { ); } guide.menus = '
    • ' + menus.join('
    • ') + '
    '; - /* - // List null value menu items directly - for (const [, item] of Object.entries(field.sources)) { - // List null value menu items directly - if (item === 'NullValueMenu') { - let null_values = Object.keys( - this.schema.enums[item].permissible_values - ); - sources.push(item + ': (' + null_values.join('; ') + ')'); - } else { - sources.push(item); - } - } - */ guide.sources = '
    • ' + field.sources.join('
    • ') + '
    '; } @@ -1867,12 +1894,13 @@ class DataHarmonizer { guide.guidance = guidance .map(function (paragraph) { - return '

    ' + paragraph + '

    '; + return paragraph + '
    '; }) .join('\n'); // Makes full URIs that aren't in markup into - if (guide.guidance) guide.guidance = urlToClickableAnchor(guide.guidance); + if (guide.guidance) + guide.guidance = urlToClickableAnchor(guide.guidance); if (field.examples && field.examples.length) { let examples = []; @@ -1901,9 +1929,10 @@ class DataHarmonizer { } /** + * Used in /web/templates/ script export.js files * Get grid data without trailing blank rows. * @return {Array>} Grid data without trailing blank rows. - */ + */ getTrimmedData() { const rowStart = 0; const rowEnd = this.hot.countRows() - this.hot.countEmptyRows(true) - 1; @@ -1912,101 +1941,9 @@ class DataHarmonizer { return this.hot.getData(rowStart, colStart, rowEnd, colEnd); } - /** - * Get grid data as a list of objects. The keys of each object are the - * field names and the values are parsed according to the field's datatype. - * If parsing fails the original string value is used instead. This behavior - * can be changed using the `options` argument. - * - * The list of objects can optionally be wrapped in a top-level object based - * on the `indexSlot` option. - * - * @param {Object} options An object with any of the following keys: - * - parseFailureBehavior: `KEEP_ORIGINAL` (default) | `REMOVE` | `THROW_ERROR` - * controls how values which are not parsable according to the column's datatype are - * handled. By default the original string value is retained. If this option is `REMOVE` - * the unparsable value is removed entirely. If this option is `THROW_ERROR`, an error - * is thrown when the first unparsable value is encountered. - * - indexSlot: boolean | string. If a string is provided the output will be an object - * with one key. The key is provided string. The value of the key is the array of objects - * representing the grid data. If `false` is provided (the default), the output will - * be the array of objects representing the grid data. If `true` is provided, an index - * slot will be inferred from the schema and the current template by identifying - * its a tree_root (https://w3id.org/linkml/tree_root) class and inspecting the ranges of - * attributes. If that inference fails, `fallbackIndexSlot` will be used instead. - * - fallbackIndexSlot: A string that will be used as the index slot name if indexSlot is - * `true` and the inference process fails to identify a unique index slot candidate. - * @return {(Object|Array)} - */ - getDataObjects(options = {}) { - const { parseFailureBehavior, indexSlot, fallbackIndexSlot } = { - parseFailureBehavior: KEEP_ORIGINAL, - indexSlot: false, - fallbackIndexSlot: 'rows', - ...options, - }; - const listData = this.hot.getData(); - const arrayOfObjects = listData - .filter((row) => !rowIsEmpty(row)) - .map((row) => - dataArrayToObject(row, this.slots, { - dateBehavior: this.dateExportBehavior, - parseFailureBehavior, - dateFormat: this.dateFormat, - datetimeFormat: this.datetimeFormat, - timeFormat: this.timeFormat, - }) - ); - if (typeof indexSlot === 'string') { - return { - [indexSlot]: arrayOfObjects, - }; - } else if (indexSlot === true) { - let inferredIndexSlot = this.getInferredIndexSlot(); - if (!inferredIndexSlot) { - inferredIndexSlot = fallbackIndexSlot; - } - return { - [inferredIndexSlot]: arrayOfObjects, - }; - } else { - return arrayOfObjects; - } - } - - getInferredIndexSlot() { - if (this.template_name) { - return this.template_name; - } - - const classes = Object.values(this.schema.classes); - const treeRootClass = classes.find((cls) => cls.tree_root); - if (!treeRootClass) { - console.warn( - `While getting inferred index slot, could not find tree_root class.` - ); - return; - } - const treeRootAttrs = Object.values(treeRootClass.attributes); - const index_attrs = treeRootAttrs.filter( - (attr) => attr.range === this.template_name - ); - if (!index_attrs || index_attrs.length !== 1) { - console.warn( - `While getting inferred index slot, could not find single slot with range ${this.template_name} on tree_root class ${treeRootClass.name}.` - ); - return; - } - return index_attrs[0].name; - } - - /** - * - * From export_utils.js - * - */ /** + * Used in /web/templates/ script export.js files * Modifies exportHeaders map of fields so that each field contains an array * of one or more source fields by name that are used to compose it. * This code works on exportHeaders as either a Map or an array of @@ -2053,10 +1990,7 @@ class DataHarmonizer { } let sources = exportHeaders.get(target.field); if (!sources) { - console.warn( - 'Malformed export.js exportHeader field:', - target.field - ); + console.warn('Malformed export.js exportHeader field:', target.field); } // If given field isn't already mapped, add it. if (sources.indexOf(field.name) == -1) { @@ -2186,6 +2120,8 @@ class DataHarmonizer { * val. * @return {string} `valToMatch` if condition is satisfied; empty str * otherwise. + * + * USED IN GRDI EXPORT */ getIfThenField( headerNameToCheck, @@ -2211,6 +2147,8 @@ class DataHarmonizer { * val. * @return {string} Intersection of user-inputted values in * `headerNameToCheck` and vals in `matchedValsSet`. + * + * USED IN GRDI EXPORT */ getMatchedValsField( headerNameToCheck, @@ -2237,6 +2175,8 @@ class DataHarmonizer { * @param {Object} sourceFieldNameMap `getFieldNameMap` return * val. * @return {string} First non-null val in `headerNamesToCheck`. + * + * USED 1ce in GRDI EXPORT */ getFirstNonNullField(headerNamesToCheck, inputRow, sourceFieldNameMap) { const nullValsSet = new Set( @@ -2352,6 +2292,12 @@ class DataHarmonizer { const row = change[0]; const col = change[1]; const field = fields[col]; + //console.log("fieldChangeRules", change, triggered_changes) + + if (!field) { + console.log("Cut and paste action, no field yet?", change, col); + return; + } // Test field against capitalization change. if (field.capitalize && change[3] && change[3].length > 0) { @@ -2411,7 +2357,6 @@ class DataHarmonizer { // On "[field] unit" here, where next column is [field]_bin. if (field.name === prevName + '_unit') { if (prevName + '_bin' === nextName) { - console.log('on', prevName + '_unit'); // trigger reevaluation of bin from field matrixRow[col - 1] = this.hot.getDataAtCell(row, col - 1); matrixRow[col] = change[3]; @@ -2446,8 +2391,8 @@ class DataHarmonizer { /** * Modify matrix data for grid according to specified rules. - * This is useful when calling `hot.loadData`, as cell changes from said method - * are not recognized by `afterChange`. + * This is useful when calling `hot.loadData`, as cell changes from said + * method are not recognized by `afterChange`. * @param {Array>} matrix Data meant for grid. * @return {Array>} Modified matrix. */ @@ -2488,9 +2433,9 @@ class DataHarmonizer { } } // - else if (fieldUnitBinTest(this.slots, col)) { + else if (this.fieldUnitBinTest(this.slots, col)) { // 2 specifies bin offset - // Matrix operations have 0 for binOffset + // matrix operations have 0 for binOffset this.binChangeTest(matrix, col, this.slots, 2, triggered_changes); } } @@ -2501,10 +2446,22 @@ class DataHarmonizer { matrix[change[0]][change[1]] = change[3]; } }); - return matrix; } + /** + * Test to see if col's field is followed by [field unit],[field bin] fields + * @param {Object} fields See `data.js`. + * @param {Integer} column of numeric field. + */ + fieldUnitBinTest(fields, col) { + return ( + fields.length > col + 2 && + fields[col + 1].name == fields[col].name + '_unit' && + fields[col + 2].name == fields[col].name + '_bin' + ); + } + /** * Adjust given dateString date to match year or month granularity given by * dateGranularity parameter. If month unit required but not supplied, then @@ -2635,8 +2592,24 @@ class DataHarmonizer { * `{0: {0: 'Required cells cannot be empty'}}` */ getInvalidCells(data) { - //const fieldNames = this.slots.map((field) => field.name); - return this.validator.validate(data, this.slot_names); // fieldNames); + // Each Row needs to be mapped to visible row OR REMOVED? + return this.validator.validate(data, this.slot_names); + /* + * ISSUE is that validate is on raw data, but we want errors reported on + * visual rows that user can navigate to in their current requested view. + * HOWEVER toVisualRow() doesn't take into account hidden rows or sorting it appears. + * So we should probably revise validator.validate to go backwards from visual to raw data? + + const errors = this.validator.validate(data, this.slot_names); + let visual_errors = {}; + for (const key in errors) { + let new_index = this.hot.toVisualRow(parseInt(key)); + if (new_index) + visual_errors[new_index] = errors[key]; + }; + console.log("errors", errors, visual_errors); + return visual_errors; + */ } /** @@ -2665,7 +2638,8 @@ class DataHarmonizer { const field = this.slots[col]; const datatype = field.datatype; - if (cellVal && datatype === 'xsd:token') { + if (cellVal && datatype === 'xsd:token' && typeof cellVal === 'string') { + // console.log("valdation", cellVal) const minimized = cellVal .replace(whitespace_minimized_re, ' ') .trim(); @@ -2673,7 +2647,7 @@ class DataHarmonizer { if (minimized !== cellVal) { cellVal = minimized; data[row][col] = cellVal; - cellChanges.push([row, col, minimized, 'thisChange']); + cellChanges.push([row, col, minimized, 'prevalidation']); } } @@ -2695,13 +2669,13 @@ class DataHarmonizer { } if (update) { data[row][col] = update; - cellChanges.push([row, col, update, 'thisChange']); + cellChanges.push([row, col, update, 'prevalidation']); } } else { const [, update] = validateValAgainstVocab(cellVal, field); if (update) { data[row][col] = update; - cellChanges.push([row, col, update, 'thisChange']); + cellChanges.push([row, col, update, 'prevalidation']); } } } @@ -2763,71 +2737,6 @@ class DataHarmonizer { return true; } - fullData(handsontableInstance = null) { - if (handsontableInstance === null) { - handsontableInstance = this.hot; - } - let obj = handsontableInstance.getSourceData(); - const headerSize = this.getFlatHeaders()[1].length; // fields, not sections; - - // Create an immutable unfilled array structure filled with `null` - const unfilledArrayStruct = Object.freeze(new Array(headerSize).fill(null)); - - // Map the original objects to the new array structure - const array = obj.map((object) => { - // Create a new copy of the unfilled array structure - let unfilledArrayCopy = [...unfilledArrayStruct]; - - // Populate the new array with values from the original object - Object.entries(object).forEach(([key, value]) => { - unfilledArrayCopy[key] = value; - }); - - return unfilledArrayCopy; - }); - - // const fullHotData = handsontableInstance.getData(); - const fullHotData = array; - return fullHotData; - } - - toJSON() { - const handsontableInstance = this.hot; - const tableData = this.fullData(handsontableInstance); - const columnHeaders = handsontableInstance.getColHeader().map(stripDiv); // TODO: use fields() or this.getFlatHeaders()[1]; - - function createStruct(row) { - const structInstance = {}; - // iterate over the columns in a row - for (let i = 0; i < row.length; i++) { - const columnHeader = columnHeaders[i]; - - // Optional type checking (adjust data types as needed) - if (typeof columnHeader === 'string') { - structInstance[columnHeader] = row[i]; - } else if (typeof columnHeader === 'number') { - structInstance[columnHeader] = Number(row[i]); // Convert to number - } else { - // Handle other data types if needed - structInstance[columnHeader] = row[i]; - } - } - - return structInstance; - } - - const arrayOfStructs = []; - - // Loop through each row and create a struct using the function - for (const row of tableData) { - // remove empty rows - if (!row.every((val) => val === null)) { - arrayOfStructs.push(createStruct(row)); - } - } - - return arrayOfStructs; - } } export default DataHarmonizer; diff --git a/lib/FieldMapper.js b/lib/FieldMapper.js new file mode 100644 index 00000000..1512f10e --- /dev/null +++ b/lib/FieldMapper.js @@ -0,0 +1,642 @@ + +/**************** DataHarmonizer User Profiles stored in Browser ************** + * This script provides a FieldMapper class which manipulates a modal report + * of all schema single template or 1m template field matches to some incomming + * dataset. It is designed to be called repeatedly, receiving each tab/table's + * headers in the appendFieldMappingModal() call, and adding to a table mapping + * report. Users have drag/drop access to the incomming headers to align them + * with the schema's headers, if need be. They are not forced to specify all + * field mappings, some can be left empty and so no data will be loaded for + * them. The report returns the mapping information which can be acted upon. + * + * This interface is simple in looks and capability - it only handles changes + * in ordering and renaming of fields, such that one could upgrade a schema and + * discover that if existing data files are read into it directly then + * columns of info are missing. + * + * Note that https://pypi.org/project/linkml-map/ can effect a comparable + * mapping specification transform on a data file on the command line. + * + * Multiple mapping profiles (named by a user) for each schema () to some file + * can exist in browser's long term storage for each user. A browser + * localStorage 'dataharmonizer_settings' key holds these and other DH settings + * (in future, settings like favoured language, default schema and template). + * + * Not handled: odd case of data having duplicate column names. + * Incomming data file might not have section headers, so don't make + * code dependent on them. + * + * localStorage "DataHarmonizer" stores all objects via a YAML string. + * - DataHarmonizer.schema = {} object holding schema name keys. Current + * loaded schema name key will be added if working on field-mappings. + * - .schema[schema_name]: (unlikely for schema name to change) + * - [profile_name] (user defined; useful to include version) + * - schema_version: [schema_version] (optional) + * // FUTURE - file: [file1, file2, etc...] (pertinent data file names) + * - tables: list of schema classes (tables) that need 1+ mappings. + * - tables[table name (schema class)]: + * .map: [{'from': ... , 'to': ...}, ...] + */ + +import $ from 'jquery'; +import {readBrowserDHSettings, saveBrowserDHSettings, clearDH} from './Toolbar'; +import YAML from 'yaml'; +import { updateSheetRange } from '../lib/utils/files'; +import { dataObjectToArray } from '../lib/utils/fields'; +import { utils as XlsxUtils } from 'xlsx/xlsx.js'; + +// A call like fm = new FieldMapper().bind(this), provides caller environment +// variables +export class FieldMapper { + constructor(context, file, data) { + + // References AppContext as created via Toolbar.js + this.file = file; + this.file.ext = this.file.name.split('.').pop() + this.context = context; + this.data = data; // Used to map columns on specific DH's on save. + // For jquery $(button) access to FieldMapper functions below. + self = this; + // The html content of the visual mapping table in the form: + this.field_mapping_html = ''; + + + /********************** Initialize form elements *************************/ + // Indicates uploaded data file name in field mapping modal. + $('#field-mapping-data-file-name').text(this.file.name); + $('#field-mapping-display').text(''); + + // Populate schema mapping profiles. Keep first option instructions/prompt. + $('#field-mapping-select option:gt(0)').remove(); + const dh_settings = readBrowserDHSettings(); + const schema = this.context.getSchemaRef(); + dh_settings.schema ??= {}; + if (!(schema.name in dh_settings.schema)) + dh_settings.schema[schema.name] = {}; + Object.entries(dh_settings.schema[schema.name]).forEach(([name, value]) => { + const newOption = new Option(name, name); + $('#field-mapping-select').append(newOption); + }); + $('#field-mapping-delete-btn').prop('disabled', true); + + // Provide default value for new mapping profile name + // This is a suggested name, but users can change it. + //$('#field-mapping-name').val(`${schema.name}_${schema.version} to ${this.file.name}`); + + /************************* Field Mapping Modal Events ********************* + * Since FieldMapper instances are created each time a user loads a data + * file and that has a field missmatch, the following code is run + * repeatedly although the DOM html already exists with previous instances + * established events. Consequently we have to turn off the previously made + * events and redo them. FUTURE: just recreate the modal HTML? + */ + + /** + * If switching to a new profile when on the field-mapping form: + * - load it in text area + * - copy it to field-mapping-name + * - implement it in combined drag & drop table list of data file fields. + */ + $('#field-mapping-select').off('change').on('change', function(e) { + const profile_name = $('#field-mapping-select').val(); + $('#field-mapping-delete-btn').prop('disabled', profile_name == ''); + if (profile_name.length > 0) { + const [dh_settings, profile] = self.getProfile(profile_name); + self.renderFieldMappingProfile(profile_name, profile); + } + }); + + $("#field-mapping-concise-checkbox").off('change').on('change', function(){ + self.conciseFieldMappingModal($(this).is(':checked')); + }); + + // "Forgets" selected user-specified profile by erasing it in localStorage. + // This doesn't affect maping rendered in form. + $('#field-mapping-delete-btn').off('click').on('click', function() { + const profile_name = $('#field-mapping-select').val(); + if (profile_name > '') { // we have something to delete. + const dh_settings = readBrowserDHSettings(); + const schema = self.context.getSchemaRef(); + if (dh_settings.schema[schema.name]?.[profile_name]) + delete(dh_settings.schema[schema.name][profile_name]); + saveBrowserDHSettings(dh_settings); + $(`#field-mapping-select option[value="${profile_name}"]`).remove(); + $('#field-mapping-display').text(''); + $('#field-mapping-name').val('') + $('#field-mapping-delete-btn').prop('disabled', true); // Disable delete button until next selection. + } + + }); + + $('#field-mapping-save-btn').off('click').on('click', function (){ + const profile_name = $('#field-mapping-name').val().trim(); // trim whitespace + $('#field-mapping-name').val(profile_name) + if (profile_name) { + self.updateProfileMapping(profile_name); // profile object updated in-situ. + const [dh_settings, profile] = self.getProfile(profile_name); + self.renderFieldMappingProfile(profile_name, profile); + } + else { + alert("Please provide a name for this mapping profile.") + } + }); + + // A reset button redoes the mapping content back to previous copy. + $('#field-mapping-reset-btn').off('click').on('click', function() { + self.renderFieldMappingModal(); + }); + + $('#field-mapping-confirm-btn').off('click').on('click', function() { + self.loadMappedData(); + }); + } + + /** Indicates user is finished with field mapping adjustments, so try + * loading the appropriate table(s) of the file again. Get mapping straight + * from user drag-drop HTML table. Guaranteed that mappings are in + * consecutive DH template/class sections. + * + * Currently we let people get off this screen without pressing "Mapping + * complete". Need to block that. Data for those tables won't be loaded. Add + * a cancel button? If so does it cancel whole load? + */ + loadMappedData() { + + const mappings = this.getProfileMapping(); + + for (const [template_name, dh] of Object.entries(this.context.dhs)) { + + if (!(this.data[template_name] && this.data[template_name].data)) + continue; + + const data_table = this.data[template_name].data; + const new_table = []; + const slot_to_data_col = this.data[template_name].slot_to_data_col_matches + const data_field_to_col = this.data[dh.template.name].header_lookup; + // Do mapping (in map_obj.map ) if any needed + const map_obj = mappings?.tables[template_name]; + + // Process each row of data + Object.entries(data_table).forEach(([row_ptr, row]) => { + const new_row = []; + + // Make full dh row, and add as many exact matches from incomming data + // as possible + dh.slot_names.forEach((slot_name, index) => { + let value = null; + if (index in slot_to_data_col) { + value = row[slot_to_data_col[index]]; + } + new_row.push(value); + }) + + if (map_obj) { + // Might not be any rules for a table that is missing some fields but + // user has not supplied any mappings. + // Overwrite any (empty) fields with user-defined column mapping. + Object.entries(map_obj?.map || {}).forEach(([ptr, mapping]) => { + const col_from = data_field_to_col[mapping.from]; + const col_to = dh.slot_title_to_column[mapping.to]; + new_row[col_to] = row[col_from]; + }); + } + + // if JSON, then examine some slot's datatype fields and overwrite with + // new converted values. + // Determine if this is still needed. + if (this.file.ext === 'json') { + new_table[row_ptr] = dataObjectToArray(new_row, dh, { + serializedDateFormat: this.dateExportBehavior, + dateFormat: this.dateFormat,// Probably NULL! + datetimeFormat: this.datetimeFormat, // Probably NULL! + timeFormat: this.timeFormat,// Probably NULL! + }); + } + else + new_table.push(new_row); + + }) + + clearDH(dh); // Does away with dh settings. + // Load mapping table into dh tab. + dh.hot.loadData( dh.matrixFieldChangeRules(new_table) ); + + }; + + $('#field-mapping-modal').modal('hide'); + + } + + /** + * Aligns Schema template fields and their names with incoming data. Presents + * a two column report of fields, which are sortable on the input file side. + * + * This script is called repeatedly such that successive calls add sections + * to the modal that are dedicated to each template where field mismatches occur. + * + * All JSON datasets should be satisfied via name. Tabular data might have + * locale headers. + * + * file_name can also be used as a key to access browser localStorage + * predefined mapping to a schema. + * + * @param {Array} matrix Data that user must specify headers for. + * @param {Integer} header_row which is 1 more than actual row (natural number). + * @param {String} file_name name of data file user selected for loading. + */ + appendFieldMappingModal(dh) { + + const data_fields = this.data[dh.template.name].header; + + // Lookup dictionary + const data_field_to_col = Object.fromEntries( + data_fields.map((item, index) => {return [item, index]}) + ); + this.data[dh.template.name].header_lookup = data_field_to_col; + + console.log("data_field_to_col", data_field_to_col); + + // FUTURE: PROTECT AGAINST DUPLICATE FIELD NAMES IN DATA FILE. + // Preliminary scan for all matches via ordered slot_names array + let slot_matches = new Array(dh.slot_names.length).fill(false); + let data_matches = new Array(data_fields.length).fill(false); + //let found_by_title = false; + dh.slot_names.forEach((slot_name, index) => { + if (slot_name in data_field_to_col) { // JSON data matches on slot.name + //console.log("FOUND NAME", slot_name, data_field_to_col[slot_name]) + const data_index = data_field_to_col[slot_name]; + slot_matches[index] = data_index; + data_matches[data_index] = index; + } + else { + // TRYING TITLE as well (default language) and if found flag this. + let title = dh.slots[dh.slot_name_to_column[slot_name]].title; + if (title in data_field_to_col) { // Tabular tsv,csv,xls,xlsx matching + //console.log("FOUND TITLE", title, data_field_to_rcol[title]) + const data_index = data_field_to_col[title] + slot_matches[index] = data_index; + data_matches[data_index] = index; + //found_by_title = true; + } + } + }); + // Store exact matches for each template + this.data[dh.template.name].slot_to_data_col_matches = slot_matches; + //this.data[dh.template.name].data_matches = data_matches; + + // Display template/tab/class (i.e. this call to extend content is + // dedicated to that content. + let html = ` +     + + + + `; + + let slot_row = 0; // IMPORTANT - whether matched by .name or .title + // Loop through each section + let done_data_row = {}; + Object.entries(dh.sections) + .forEach(([i, section]) => { + + // Adding template and section to make easier to search and replace content. + html += ` + + + + + `; + + let old_match_data_row = null; + Object.entries(section.children).forEach(([section_slot_row, slot]) => { + + if (slot_matches[slot_row] !== false && slot_matches[slot_row] >= 0) { + const data_row = slot_matches[slot_row]; + old_match_data_row = data_row; + // HERE slots are always displayed by their title (for multilingual sanity?) + const ordering = (slot_row != data_row) ? `${data_row}` : data_row + html += ` + + + + `; + + } + else { + + // Do slot side's mismatched item. + html += ` + + + + `; + } + + // If we don't have a match, then we have to ensure all the + // mismatched field headers for both tables are provided until + // the next match. (If the data table had columns whose order is + // completely different, then semantically the template section + // slots will have unrelated fields near them, c'est la vie.) + + // Possibility that next data row(s) are not a match so squeeze them + // in here. + // Add each data_matches' entry between last match and next one. + let data_row = (old_match_data_row || -1) + 1; + while (data_row < data_matches.length + && (data_matches[data_row] === false + && ! done_data_row[data_row]) + ){ + done_data_row[data_row] = true; // Best way to handle this case? + const ordering = slot_row != data_row ? 'reordered' : '' + html += ` + + + + `; + data_row += 1; + } + + slot_row += 1; + }); + + html += ` + + `; + + }); + + this.field_mapping_html += html; + + } + + // Update field mapping form with user's selected dh_settings.schema[schema_name][profile_name] + renderFieldMappingProfile(profile_name, profile) { + $('#field-mapping-name').val(profile_name); + + $('#field-mapping-display').text(YAML.stringify({[profile_name]: profile})); + if (!$(`#field-mapping-select option[value="${profile_name}"]`).length) { + const newOption = new Option(profile_name, profile_name); + $('#field-mapping-select').append(newOption) //.select(); + } + $('#field-mapping-delete-btn').prop('disabled', false); + + this.renderFieldMappingModal(profile_name); + } + + /** Updates browser localStorage dataharmonizer_settings.schema.[schema name][profile name] + * based on form's displayed html resulting from user's drag moves of data fields. + * + */ + updateProfileMapping(profile_name) { + let [dh_settings] = this.getProfile(profile_name); + const profile = this.getProfileMapping(); // Its not changing profile in place. + const schema = this.context.getSchemaRef(); + dh_settings.schema[schema.name][profile_name] = profile; + saveBrowserDHSettings(dh_settings); + } + + // Retrieves browser memory dataharmonizer_settings + // .schema[schema name][profile] data structure and initializes if not yet + // accessed. + getProfile(profile_name) { + const dh_settings = readBrowserDHSettings(); + const schema = this.context.getSchemaRef(); + if (!('schema' in dh_settings)) + dh_settings.schema = {}; + if (!(schema.name in dh_settings.schema)) + dh_settings.schema[schema.name] = {}; + if (!(profile_name in dh_settings.schema[schema.name])) { + dh_settings.schema[schema.name][profile_name] = { + schema_version: schema.version, + tables: {} + } + } + return [dh_settings, dh_settings.schema[schema.name][profile_name]]; + }; + + /** + * Build mappings for all of a schema's table(s) based on visual table layout + * of sections and user's drag/drop of data fields. Only later might a + * profile name be associated with this. Outputs a "mapping" dictionary + * containing a "tables" dictionary, with each mapping.tables[table_name].map + * containing a "from" field lable and "to" field label. + */ + getProfileMapping() { + + function get_label (nmstr) { + const i = nmstr.indexOf(' ')+1; + return (i ? nmstr.slice(i) : '') + }; + // In each tbody[data-table] section that has a template_name, look for + // any tr which has a "field-mismatch" attribute. That tr first td will + // have slotname of schema/template. + // Each + const schema = this.context.getSchemaRef(); + let mapping = { + schema_version: schema.version, + tables: {} + }; + + $('table#field-mapping-table tbody[data-table]').each((t_index, tbody) => { + + // A given schema class often has multiple tbody, each for a data-table section + const table_name = $(tbody).attr('data-table'); + + mapping.tables[table_name]??= {}; // init empty value if it doesn't exist. + $(tbody).find('td:first-child.field-mismatch').each((index, slot_field) => { + // Retrieve labels, get past column id) prefix; + const data_label = get_label($(slot_field).next('td').text()); + if (data_label.length > 0) {// target has a mapping in it. + let row_map = { + 'to': get_label($(slot_field).text()), + 'from': data_label + }; + const table_section = $(tbody).attr('data-section') || ''; + if (table_section) + row_map['section'] = table_section; + + // Adds table if it isn't present. + mapping.tables[table_name].map??= []; // establishes array if not there. + mapping.tables[table_name].map.push(row_map); + } + }) + // The .map attribute exists only if 1+ mappings. + + }); + return mapping; + + } + + initFieldMappingModal() { //file_name, field_mapping_html) { + // Indicates uploaded data file name in field mapping modal. + $('#field-mapping-data-file-name').text(this.file_name); + $('#field-mapping-display').text(''); + // Store html so user can reset to it. + //this.field_mapping_html = field_mapping_html; + // Populate schema mapping profiles. Keep first option instructions/prompt. + $('#field-mapping-select options').slice(1).remove(); + const dh_settings = readBrowserDHSettings(); + const schema = this.context.getSchemaRef(); + if (!dh_settings.schema) + dh_settings.schema = {}; + if (!(schema.name in dh_settings.schema)) + dh_settings.schema[schema.name] = {}; + Object.entries(dh_settings.schema[schema.name]).forEach(([name, value]) => { + const newOption = new Option(name, name); + $('#field-mapping-select').append(newOption); + }); + + // Provide default value for new mapping profile name + // This is a suggested name, but users can change it. + $('#field-mapping-delete-btn').prop('disabled', true); + //$('#field-mapping-name').val(`${schema.name}_${schema.version} to ${file_name}`); + } + + + /** + * Assumes this.field_mapping_html holds html that can be reset to. + * profile_name if any, points to browser-in-memory mapping profile + * to implement. Currently this is done in a crude way via textual + * search and replace on the html rather than, say, data-slot-name="..." + */ + renderFieldMappingModal(profile_name = null) { + // Clears out previous mapping report + $('#field-mapping-table tbody').remove(); + $('#field-mapping-table thead').after(this.field_mapping_html); + + // Sets the checkbox status to hide matched fields via CSS. + this.conciseFieldMappingModal($("#field-mapping-concise-checkbox").is(':checked')); + + // A bit crude, but we recreate the mapping by moving td + // content around. We don't care what the schema source or + // data field target rows are (they could have changed across + // data file versions. Just the names count. + if (profile_name) { + this.applyFieldMapping(profile_name); + } + + // To enable drag-drop on newly-created elements in field-mapping-modal + // FUTURE: limit drag to within each dh table. + $(".draggable-mapping-item").draggable({ + containment: "#field-mapping-table", // but scroll inside field-mapping-container + helper: 'clone', + cursor: 'grab', + axis: "y", + drag: function(event, ui) { // scroll: true doesn't work on cloned div. + var $scrollContainer = $("#field-mapping-container"); + var containerOffset = $scrollContainer.offset(); + var containerHeight = $scrollContainer.height(); + var draggableTop = ui.offset.top; + var scrollAmount = 20; // Adjust scroll speed + // Scroll down + if (draggableTop + ui.helper.outerHeight() > containerOffset.top + containerHeight) { + $scrollContainer.scrollTop($scrollContainer.scrollTop() + scrollAmount); + } + // Scroll up + else if (draggableTop < containerOffset.top) { + $scrollContainer.scrollTop($scrollContainer.scrollTop() - scrollAmount); + } + } + }); + + $(".draggable-mapping-item").droppable({ + accept: ".draggable-mapping-item", // Only accept elements with this class + //activeClass: "ui-state-active", + hoverClass: "ui-state-hover", + drop: function(event, ui) { + const source_text = ui.draggable[0].innerText; + ui.draggable[0].innerText = event.target.innerText; + event.target.innerText = source_text; + $(this).css("background-color", "lightskyblue"); + ui.draggable.css("background-color", "lightblue"); + } + }); + } + + // User has dragged one field down to the row of another within a table. + // Switch the labels of the selected fields. + // WARNING: THIS TEXT MATCHING ALGORITHM IS VERY SENSITIVE TO spaces etc. + // in field label HTML display + applyFieldMapping(profile_name) { + const [dh_settings, profile] = this.getProfile(profile_name); + + Object.entries(profile.tables || {}).forEach(([table_name, table_obj]) => { + + const mismatched_rows = $(`table#field-mapping-table tbody[data-table="${table_name}"] > tr.field-mismatch`); + // Doing table by table; otherwise identical field names in different + // tables lead to garbled rule implementation. + + Object.entries(table_obj?.map || {}).forEach(([index, mapping]) => { + // Find mapping.from text. mapping.from td will initially have empty + // data field td. First fetch the 2nd data file field value + const schema_slot_td = $(mismatched_rows).find('td:first-child') + .filter(function() {return $(this).text().endsWith(') ' + mapping.to)}); + const schema_data_td = $(mismatched_rows).find('td:eq(1)') + .filter(function() {return $(this).text().endsWith(') ' + mapping.from)}); + + // Now do the switch of values as mapping dictates: + const source_data_td_text = $(schema_data_td).text(); + $(schema_data_td).text(''); // Clear out old data td side. + $(schema_slot_td).next('td').text(source_data_td_text); + + }); + }); + } + + // UI checkbox allows mapping form to be collapsed to just the rows that + // have mismatch information. + conciseFieldMappingModal(hide_flag) { + const elements = $("table#field-mapping-table tr.field-match"); + const sections = $("table#field-mapping-table tbody.field-mapping-section:not(:has(td.field-mismatch))"); + if (hide_flag) { + elements.hide(); + sections.hide(); + } + else { + elements.show(); + sections.show(); + } + } + + // Modal show is outside of renderFieldMappingModal() above because "reset" + // button is on form itself, and it needs to render just table content of + // form without redoing library component. + async show() { + $('#field-mapping-modal').modal('show'); + return false; + } + +} + + +/** + * Determine if first or second row of a matrix has the same headers as the + * grid's secondary (2nd row) headers. If neither, return false. + * Usually first row is section headers in sparse array, and 2nd row is field + * TITLES (locale/language sensitive). + * @param {Array>} matrix + * @param {Object} data See `data.js`. + * @return ({Integer}, {Integer}, {Integer}) row that data starts on, total schema rows for table, and count of exact matches, or false if no exact header row recognized. + */ +export function findHeaderRow(dh, matrix) { + const schemaRow = dh.slot_titles; + const first_row_matches = schemaRow.filter(element => matrix[0].includes(element)).length; + const second_row_matches = schemaRow.filter(element => matrix[1].includes(element)).length; + + let row = false; + let count = 0; + if (first_row_matches && first_row_matches > second_row_matches) { + row = 1; + count = first_row_matches; + } + // Odd case if 2nd row and 1st row counts actually match. + if (second_row_matches && second_row_matches >= first_row_matches) { + row = 2; + count = second_row_matches; + } + + return [row, count]; + }; + + +export default FieldMapper; \ No newline at end of file diff --git a/lib/Footer.js b/lib/Footer.js index e739ba5b..2a72f399 100644 --- a/lib/Footer.js +++ b/lib/Footer.js @@ -14,8 +14,9 @@ const TEMPLATE = ` value="1" />
    - more rows at the bottom. + row(s)
    + `; @@ -23,7 +24,6 @@ class Footer { constructor(root, context) { this.root = $(root); this.root.append(TEMPLATE); - this.root.find('.add-rows-button').on('click', () => { const numRows = this.root.find('.add-rows-input').val(); context.getCurrentDataHarmonizer().addRows('insert_row_below', numRows); diff --git a/lib/HelpSidebar.css b/lib/HelpSidebar.css index b1acd021..01e51fe2 100644 --- a/lib/HelpSidebar.css +++ b/lib/HelpSidebar.css @@ -5,7 +5,8 @@ background-color: white; transition-property: right; transition-timing-function: ease; - border-left: 1px solid #ddd; + border:0; + /*border-left: 1px solid #ddd;*/ } .help-sidebar__toggler { @@ -13,9 +14,11 @@ top: 0; transform: translateX(-100%); z-index: 200; - border: 1px solid #ddd; + border: 0px; + border-radius:5px; border-right-width: 0; - background-color: white; + background-color: #007bff; + color: white; } .help-sidebar__content { diff --git a/lib/SchemaEditor.js b/lib/SchemaEditor.js new file mode 100644 index 00000000..dd4b224f --- /dev/null +++ b/lib/SchemaEditor.js @@ -0,0 +1,1541 @@ +import $ from 'jquery'; +import {deleteEmptyKeyVals, setJSON} from '../lib/utils/general'; +import YAML from 'yaml'; + +export class SchemaEditor { + + /** DH Tab maps to schema.extensions.locales.value[locale][part][key1][part2][key2] + * The key names link between locales hierarchy part dictionary keys and the + * DH row slots that user is focused on. + * tab: [schema_part, key1, [part2, key2], [attribute_list] + */ + TRANSLATABLE = { + 'Schema': [null, null, null, null, ['description']], + 'Class': ['classes', 'name', null, null, ['title','description']], + 'Slot': ['slots', 'name', null, null, ['title','description','comments','examples']], + 'Enum': ['enums', 'name', null, null, ['title','description']], + 'SlotUsage': ['classes', 'class_id', 'slot_usage', 'slot_id', ['title','description','comments','examples']], + 'PermissibleValue': ['enums', 'enum_id', 'permissible_values', 'text', ['title','description']] + } + + // These are SchemaEditor dynamically handled menu items: A schema's classes, data types, slots, and slot groups vary. + // In contrast, the SchemaSlotTypeMenu is constant across all schemas, and so is not in this list. + SCHEMAMENUS = ['SchemaDataTypeMenu','SchemaMenu','SchemaClassMenu','SchemaSlotMenu','SchemaSlotGroupMenu','SchemaEnumMenu']; + + constructor(schema, context) { + this.schema = schema; + this.context = context; + } + + + initMenus() { + this.SCHEMAMENUS.forEach((item) => { + if (!(item in this.schema.enums)) { + // Permissible_values are computed later when a particular schema is loaded. + this.schema.enums[item] = {name: item}; + } + }); + } + + /** Schema editor functionality: This function refreshes both the given + * enumeration menu (or default list of them), and ALSO updates the slot + * select/multiselect lists that reference them, for editing purposes. + * + * 1) Regenerate given menus + * CASE A: No schema loaded. Revert to dh.schema for source of types, + * classes, enums. + * CASE B: A schema has been loaded. In addition to dh.schema data types, + * Use Class & Enum template hot data. + * + * 2) Find any DH class / tab's slot that mentions changed enums, and + * regenerate its "cached" flatVocabulary etc. + * + * @param {Array} enums list of special enumeration menus. Default list + * covers all menus that schema editor might have changed dynamically. + */ + refreshMenus(enums = this.SCHEMAMENUS) { + + // If an instance of schema editor has been created. + if (this.context.schemaEditor) { + + const schema = this.schema; + for (const enum_name of enums) { + // Initialize TypeMenu, ClassMenu and EnumMenu + + let user_schema_name = this.getSchemaEditorSelectedSchema(); + // Here in Schema Editor, user hasn't selected a particular schema. + if (!user_schema_name) + user_schema_name = 'DH_LinkML'; + + const permissible_values = {}; + + switch (enum_name) { + + // A list of all loaded schemas by name (not sensitive to selected schema) + case 'SchemaMenu': + const schema_dh = this.context.dhs['Schema']; + const schema_name_ptr = schema_dh.slot_name_to_column['name']; + for (let row=0; row < schema_dh.hot.countSourceRows(); row ++) { + const slot_group_text = schema_dh.hot.getSourceDataAtCell(row, schema_name_ptr); + if (slot_group_text && !(slot_group_text in permissible_values)) { + permissible_values[slot_group_text] = {text: slot_group_text, title: slot_group_text}; + } + }; + break; + + case 'SchemaDataTypeMenu': + + // This is list of basic LinkML types imported via DH_LinkML spec + // (Including a few - provenance and WhitespaceMinimizedString - + // that DH offers). + Object.entries(schema.types).forEach(([data_type, type_obj]) => { + permissible_values[data_type] = { + name: type_obj.name, + text: data_type, + description: type_obj.description || '' + }; + }); + + // FUTURE: Extend above types with menu of user-defined types. + break; + + case 'SchemaClassMenu': + // Get the classes from the Classes tab, filtered for schema_name + // selected in Schema tab. + this.getDynamicMenu(schema, schema.classes, 'Class', user_schema_name, permissible_values); + break; + + case 'SchemaEnumMenu': + // Get the enums from the Enums tab, filtered for schema + // selected in Schema tab. + this.getDynamicMenu(schema, schema.enums, 'Enum', user_schema_name, permissible_values); + break; + + case 'SchemaSlotMenu': + // Get the enums from the Enums tab, filtered for schema + // selected in Schema tab. + this.getDynamicMenu(schema, schema.slots, 'Slot', user_schema_name, permissible_values); + break; + + case 'SchemaSlotGroupMenu': + /** Fabricate list of pertinent SchemaSlotGroupMenu. This will be + * all distinct slot_group entries mentioned in all schema classes, + * or will be filtered by user's selected Table/class tab class. + * slot_group values and make menu for it. + * FUTURE: perhaps separate tab for managing possible slot_group + */ + + const filter = { + schema_id: this.getSchemaEditorSelectedSchema(), + //slot_type: 'slot_usage' // FUTURE, allow ['slot_usage','attribute'] + } + + // If user selected a class name on Table tab, then Field tab will + // focus on just that class, and so menu should filter down to rows + // that only mention selected class. + const class_name = this.getSchemaEditorSelectedClass(); + if (class_name > '') { + filter.class_name = class_name; + } + + const slot_rows = this.context.crudFindAllRowsByKeyVals('Slot', filter); + const slot_dh = this.context.dhs['Slot']; + const slot_group_ptr = slot_dh.slot_name_to_column['slot_group']; + + // Inefficient insofar as every row examined, with many repeated + // slot_groups encountered. + // Slot group textual name is being used as key in permissible_values object. + // FUTURE: organize field group menu by Slot tab Display filters + // rather than Table tab selection. + // care of situation where users are comparing classes class for situation where + for (let row of slot_rows) { + const slot_group_text = slot_dh.hot.getDataAtCell(row, slot_group_ptr); + if (slot_group_text && !(slot_group_text in permissible_values)) { + permissible_values[slot_group_text] = {text: slot_group_text, title: slot_group_text}; + } + }; + + //console.log("DYNAMIC", class_name, filter, slot_rows, permissible_values, slot_dh.slots[slot_group_ptr]) + break; + + } + + // ISSUE: Handsontable multiselect elements behaving differently from pulldowns. + // CANT dynamically reprogram dropdown single selects + // ONLY reprogrammable for multiselects. + // Reset menu's permissible_values to latest. + if (! (enum_name in schema.enums)) { + //console.log("Adding enumeration", enum_name); + schema.enums[enum_name] = {name: enum_name}; + + } + schema.enums[enum_name].permissible_values = permissible_values; + + } + + // Then trigger update for any slot that has given menu in range. + // Requires scanning each dh_template's slots for one that mentions + // an enums enum, and updating each one's flatVocabulary if found. + for (let tab_dh of Object.values(this.context.dhs)) { + const Cols = tab_dh.columns; + let change = false; + for (let slot_obj of Object.values(tab_dh.slots)) { + for (let slot_enum_name of Object.values(slot_obj.sources || {})) { + // Found a regenerated enum from above so recalculate slot lookups + if (enums.includes(slot_enum_name)) { + this.context.setSlotRangeLookup(slot_obj); + if (slot_obj.sources) { + //FUTURE: tab_dh.hot.propToCol(slot_obj.name) after Handsontable col.data=[slot_name] implemented + const meta = tab_dh.hot.getColumnMeta(tab_dh.slot_name_to_column[slot_obj.name]); + meta.source = tab_dh.updateSources(slot_obj); + } + break; + } + } + } + } + } + } + + getSchemaEditorSelectedSchema() { + const dh = this.context.dhs['Schema']; + return dh.hot.getDataAtCell(dh.current_selection[0], dh.slot_name_to_column['name']); + } + + getSchemaEditorSelectedClass() { + const dh = this.context.dhs['Class']; + return dh.hot.getDataAtCell(dh.current_selection[0], dh.slot_name_to_column['name']); + } + + + /** For generating permissible_values for SchemaSlotTypeMenu, SchemaClassMenu, + * SchemaEnumMenu menus from schema editor schema or user schema. + */ + getDynamicMenu(schema, schema_focus, template_name, user_schema_name, permissible_values) { + // When does this case ever happen? + if (user_schema_name === 'DH_LinkML') { + for (let focus_name of Object.keys(schema_focus)) { + permissible_values[focus_name] = { + name: focus_name, + text: focus_name, + description: schema_focus[focus_name].description + } + } + } + else { + let focus_dh = this.context.dhs[template_name]; + let name_col = focus_dh.slot_name_to_column['name']; + let description_col = focus_dh.slot_name_to_column['description']; + for (let row in this.context.crudFindAllRowsByKeyVals(template_name, {schema_id: user_schema_name})) { + let focus_name = focus_dh.hot.getDataAtCell(row, name_col); + if (focus_name) {//Ignore empty class_name field + permissible_values[focus_name] = { + name: focus_name, + text: focus_name, + description: focus_dh.hot.getDataAtCell(row, description_col) + } + } + } + } + } + + translationForm(dh) { + + const schema = dh.context.dhs.Schema; + // Each schema_editor schema has locales object stored in its first + // row cell metadata. Issue: if a schema has lost focus, and instead + // all schemas are selected ... + const schema_row = schema.current_selection[0]; + if (schema_row < 0) { + alert("In order to see the translation form, first select a row with a schema that has locales.") + return false; + } + const locales = schema.hot.getCellMeta(schema_row, 0).locales; + if (!locales) { + alert("In order to see the translation form, first select a row with a schema that has locales.") + return false; + } + + let locale_map = dh.schema.enums?.LanguagesMenu?.permissible_values || {}; + + const locale_field = schema.slot_name_to_column['in_language']; + const language_code = schema.hot.getDataAtCell(schema_row, locale_field); + const default_language = language_code in locale_map ? locale_map[language_code].title : language_code; + + // Translation table form for all selected rows. + const [schema_part, key_name, sub_part, sub_part_key_name, text_columns] = this.TRANSLATABLE[dh.template_name]; + + let translate_rows = ''; + + // Provide translation forms for user selected range of rows + for (let row = dh.current_selection[0]; row <= dh.current_selection[2]; row++) { + + // 1st content row of table shows english or default translation. + let default_row_text = ''; + let translatable = ''; + let column_count = 0; + for (var column_name of text_columns) { + column_count ++; + let col = dh.slot_name_to_column[column_name]; + // Tabular slot_usage may inherit empty values. + let text = dh.hot.getSourceDataAtCell(dh.hot.toPhysicalRow(row), col) || ''; + default_row_text += `    `; + translatable += text + '\n'; + } + + // Key for class, slot, enum: + const key = dh.hot.getDataAtCell(row, dh.slot_name_to_column[key_name], 'lookup'); + let key2 = null; + if (sub_part_key_name) { + key2 = dh.hot.getDataAtCell(row, dh.slot_name_to_column[sub_part_key_name], 'lookup'); + if (!key2) { + console.log("key2",key2, "lookup from", this.TRANSLATABLE[dh.template_name]); + alert("unable to get key2 from lookup of:" + sub_part_key_name) + } + } + + if (key) { + translate_rows += ``; + } + + translate_rows += `${default_row_text}`; + + // DISPLAY locale for each schema in_language menu item + for (const [locale, locale_schema] of Object.entries(locales)) { + let translate_cells = ''; + let translate_text = ''; + let path = ''; + for (let column_name of text_columns) { + // If items are in a component of class, like slot_usage or permissible_values + // schema_part='enums', id='enum_id', 'permissible_values', 'name', + // Translations can be sparse/incomplete + let value = null; + if (sub_part) { + // Sparse locale files might not have particular fields. + value = locale_schema[schema_part]?.[key]?.[sub_part]?.[key2]?.[column_name] || ''; + path = `${locale}.${schema_part}.${key}.${sub_part}.${key2}.${column_name}`; + } + else if (schema_part) { + value = locale_schema[schema_part]?.[key]?.[column_name] || ''; + path = `${locale}.${schema_part}.${key}.${column_name}`; + } + else { // There should always be a locale_schema + value = locale_schema?.[column_name] || ''; + path = `${locale}.${column_name}`; + } + + if (!!value && Array.isArray(value) && value.length > 0) { + // Some inputs are array of [{value: ..., description: ...} etc. + if (typeof value[0] === 'object') + value = Object.values(value).map((x) => x.value).join(';'); + else + value = value.join(';') + } + + translate_cells += ``; + } + // Because origin is different, we can't bring google + // translate results directly into an iframe. + let translate = ``; + const trans_language = locale_map[locale].title; + translate_rows += `${translate_cells}`; + } + }; + + $('#translate-modal-content').html( + `
    +
    ${i18next.t('help-picklists')}

    ${i18next.t('help-picklists')}
    Expected second row ' + - flatHeaders[1].join('') + - 'Imported second row ' + - matrix[1].join('') + - '
    ${dh.template_name}
    ${section.title}
    ${slot_row}) ${slot.title}${ordering}) ${data_fields[data_row]}
    ${slot_row}) ${slot.title}
     ${data_row}) ${data_fields[data_row]}
    ${text}
    ${key}${key2 ? ' /' + key2 : ''}
    ${default_language}
    ${trans_language}${translate}
    + + + + + + + + + ${translate_rows} + +
    locale${this.TRANSLATABLE[dh.template_name][4].join('')}translate
    +
    ` + ); + $('#translate-modal').modal('show'); + } + + /** + * Enacts change to a classes' locales with appropriate creation or + * deletion. + */ + setLocales(changes) { + const dh_schema = this.context.dhs.Schema; + + let old_langs = changes.locales.old_value + ? new Set(changes.locales.old_value.split(';')) + : new Set(); + let new_langs = changes.locales.value + ? new Set(changes.locales.value.split(';')) + : new Set(); + let deleted = Array.from(old_langs.difference(new_langs).keys()); + let created = Array.from(new_langs.difference(old_langs).keys()); + + // If old language has been dropped or a new one added, prompt user: + if (deleted.length || created.length) { + let locale_map = this.schema.enums?.LanguagesMenu?.permissible_values || {}; + let deleted_titles = deleted.map((item) => locale_map[item]?.title || item); + let created_titles = created.map((item) => locale_map[item]?.title || item); + + let message = `Please confirm that you would like to: \n\n`; + if (deleted.length) { + message += `DELETE A LOCALE AND ALL ITS TRANSLATIONS for: ${deleted_titles.join('; ')}\n\n`; + } + if (created.length) { + message += `ADD LOCALE(s): ${created_titles.join('; ')}`; + } + let proceed = confirm(message); + if (!proceed) return false; + + const locales = dh_schema.getLocales(); + for (const locale of deleted) { + delete locales[locale]; + } + // An empty locale branch. + for (const locale of created) { + locales[locale] = {}; + } + } + return true; + }; + + /* Currently functionality exclusive to Schema Editor tabs: + *Empty table render will still trigger .cells () for all row 0 columns + */ + initTab (dh, class_name, hot_settings) { + + // + // At moment we have to rely on handsontable .cells() call in + // createHot() hot_settings to apply styling to slot report, so + // add quick lookup parameters to help that. + dh.schema_name_column = dh.slot_name_to_column['schema_id']; + dh.slot_class_name_column = dh.slot_name_to_column['class_name']; + dh.slot_name_column = dh.slot_name_to_column['name']; + dh.slot_rank_column = dh.slot_name_to_column['rank']; + dh.slot_type_column = dh.slot_name_to_column['slot_type']; + dh.slot_group_column = dh.slot_name_to_column['slot_group']; + + switch (class_name) { + case 'Slot': + this.initSlotTab (dh, hot_settings); + break; + case 'SlotGroup': + case 'Slot': + this.initSlotGroupTab (dh, hot_settings); + break; + } + }; + + initSlotTab (dh, hot_settings) { + + const slot_table_attribute_column = ['rank','slot_group','inlined','inlined_as_list'].map((x) => dh.slot_name_to_column[x]); + + // See https://forum.handsontable.com/t/how-to-unhide-columns-after-hiding-them/5086/6 + hot_settings.contextMenu.items['hidden_columns_hide'] = {}; + hot_settings.contextMenu.items['hidden_columns_show'] = {}; + // Could be turning off/on based on expert user + hot_settings.hiddenColumns = { + // set columns that are hidden by default + columns: slot_table_attribute_column, + indicators: false + } + + hot_settings.fixedColumnsLeft = 4; // Freeze both schema and slot name. + + // function(row, col, prop) has prop == column name if implemented; otherwise = col # + // In some report views certain kinds of row are readOnly, e.g. Schema + // Editor schema slots if looking at a class's slot_usage slots. + // Issue: https://forum.handsontable.com/t/gh-6274-best-place-to-set-cell-meta-data/4710 + // We can't lookup existing .getCellMeta() without causing stack overflow. + // ISSUE: We have to disable sorting for 'Slot' table because + // separate reporting controls are at work. + + // ASSUMES VISUAL alphabetical order with schema fields at top + const slot_editable_keys = [dh.slot_type_column, dh.slot_class_name_column, dh.slot_name_column]; + + // ISSUE: user clicking on "toggle expert user mode" doesn't visually + // take effect until after dh.render(), so cellProp.readOnly doesn't + // work right away. + hot_settings.cells = function(row, col) { + let cellProp = {}; + let read_only = false; + + // slot, slot_usage, attribute + let slot_type = dh.hot.getSourceDataAtCell(row, dh.slot_type_column); + cellProp.className = 'tabFieldTd ' + slot_type; + let visual_row = dh.hot.toVisualRow(row); + + if (col in [dh.schema_name_column]) { // 0th column usually. + read_only = true; + } + + if (slot_type === 'slot' && !dh.context.expert_user) { + read_only = true; + } + + if (slot_type === 'slot_usage') { + if (slot_editable_keys.includes(col)) { + read_only = false; + } + + // INHERIT read-only from slot fields. + // If previous row has type 'slot' and same 'name' + else + if (read_only === false) { + // Source data or visual data? + let found = false; + const this_slot_name = dh.hot.getDataAtCell(visual_row, dh.slot_name_column); + const this_schema = dh.hot.getDataAtCell(visual_row, dh.schema_name_column); + while (visual_row > 0 && this_slot_name === dh.hot.getDataAtCell(visual_row-1, dh.slot_name_column) + && this_schema === dh.hot.getDataAtCell(visual_row-1, dh.schema_name_column)) { + visual_row += -1; + found = true; + } + if (found) { + const prev_slot_type = dh.hot.getDataAtCell(visual_row, dh.slot_type_column); + const prev_value = dh.hot.getDataAtCell(visual_row, col); + if (prev_value && prev_slot_type === 'slot') { + read_only = true; + cellProp.className += ' inherited' + } + } + } + } + + /* Handsontable assigns .htDimmed to any cell with .readOnly = true + * see https://handsontable.com/docs/javascript-data-grid/disabled-cells/ + */ + cellProp.readOnly = read_only; + + return cellProp; + } + + hot_settings.multiColumnSorting = { + // let the end user sort data by clicking on the column name (set by default) + headerAction: false, + // don't sort empty cells – move rows that contain empty cells to the bottom (set by default) + sortEmptyCells: false, + // enable the sort order icon that appears next to the column name (set by default) + indicator: false, + } + + // Somehow putting this into multiColumnSorting.initConfig {} doesn't + // enable memory of it, as though init config is being wiped out. + dh.defaultMultiColumnSortConfig = [ + { + column: dh.slot_name_column, // slot.name + sortOrder: 'asc' + }, + /* + { + column: 1, // slot type + sortOrder: 'asc' + }, + { + column: 3, // schema + sortOrder: 'asc' + } + */ + ]; + } + + initSlotGroupTab (dh, hot_settings) { + + hot_settings.multiColumnSorting = { + // let the end user sort data by clicking on the column name (set by default) + headerAction: false, + // don't sort empty cells – move rows that contain empty cells to the bottom (set by default) + sortEmptyCells: false, + // enable the sort order icon that appears next to the column name (set by default) + indicator: false, + } + + //hot_settings.filters = false; + //hot_settings.dropDownMenu = false; + hot_settings.fixedColumnsLeft = 0; + //hot_settings.filters = false; // WHY NOT WORKING? Causes error as though + // filters were enabled on SlotGroup tab. + + // Somehow putting this into multiColumnSorting.initConfig {} doesn't + // enable memory of it, as though init config is being wiped out. + dh.defaultMultiColumnSortConfig = [ + {column: dh.slot_name_column, sortOrder: 'asc'}, + {column: dh.slot_class_name_column, sortOrder: 'asc'}, + {column: dh.schema_name_column, sortOrder: 'asc'} + ]; + + } + + // The opposite of loadSchemaYAML! + saveSchema () { + /* + if (!this.context.schemaEditor) { + alert('This option is only available while in the DataHarmonizer schema editor.'); + return false; + } + */ + + // User-focused row gives top-level schema info: + let dh_schema = this.context.dhs.Schema; + let schema_focus_row = dh_schema.current_selection[0]; + let schema_name = dh_schema.hot.getDataAtCell(schema_focus_row, 0); + if (schema_name === null) { + alert("The currently selected schema needs to be named before saving. If you have named your schema please make sure it is selected before saving"); + return; + } + let save_prompt = `Provide a name for the ${schema_name} schema YAML file. This will save the following schema parts:\n`; + + let [save_report, confirm_message] = dh_schema.getChangeReport('Schema'); + + // prompt() user for schema file name. + let file_name = prompt(save_prompt + confirm_message, 'schema.yaml'); + + if (!file_name) + return false; + + /** Provide defaults here in ordered object prototype so that saved object + * is consistent. At class and slot level ordered object prototypes are + * also used, but empty values are cleared out at bottom of save script. + */ + let new_schema = new Map([ + ['id', ''], + ['name', ''], + ['description', ''], + ['version', ''], + ['in_language', 'en'], + ['default_prefix', ''], + ['imports', ['linkml:types']], + ['prefixes', {}], + ['classes', new Map()], + ['slots', new Map()], + ['enums', {}], + ['types', { + WhitespaceMinimizedString: { + name: 'WhitespaceMinimizedString', + typeof: 'string', + description: 'A string that has all whitespace trimmed off of beginning and end, and all internal whitespace segments reduced to single spaces. Whitespace includes #x9 (tab), #xA (linefeed), and #xD (carriage return).', + base: 'str', + uri: 'xsd:token' + }, + Provenance: { + name: 'Provenance', + typeof: 'string', + description: 'A field containing a DataHarmonizer versioning marker. It is issued by DataHarmonizer when validation is applied to a given row of data.', + base: 'str', + uri: 'xsd:token' + } + }], + ['settings', {}], + ['extensions', {}] + ]); + + // Loop through loaded DH schema and all its dependent child tabs. + let components = ['Schema', ... Object.keys(this.context.relations['Schema'].child)]; + for (let [ptr, tab_name] of components.entries()) { + // For Schema, key slot is 'name'; for all other tables it is + // 'schema_id' which has a foreign key relationship to schema + let schema_key_slot = (tab_name === 'Schema') ? 'name' : 'schema_id'; + let rows = this.context.crudFindAllRowsByKeyVals(tab_name, {[schema_key_slot]: schema_name}) + let dependent_dh = this.context.dhs[tab_name]; + + // Schema | Prefix | Class | UniqueKey | Slot | Annotation | Enum | PermissibleValue | Setting | Extension + for (let dep_row of rows) { + // Convert row slots into an object for easier reference. + let record = {}; + for (let [dep_col, dep_slot] of Object.entries(dependent_dh.slots)) { + // 'row_update' attribute may avoid triggering handsontable events + let value = dependent_dh.hot.getDataAtCell(dep_row, dep_col, 'reading'); + if (value !== undefined && value !== '') { //.length > 0 // took out !!value - was skipping numbers. + // YAML: Quotes need to be stripped from boolean, Integer and decimal values + // Expect that this datatype is the first any_of range item. + // ALL multiselect values are converted to appropriate array or + // key/value pairs as detailed below. + record[dep_slot.name] = setJSON(value, dep_slot.datatype); + } + } + + // Do appropriate constructions per schema component + let target_obj = null; + switch (tab_name) { + case 'Schema': + //console.log("SCHEMA",tab_name, {... record}) + this.copyAttributes(tab_name, record, new_schema, + ['id','name','description','version','in_language','default_prefix']); + + // TODO Ensure each Schema.locales entry exists under Container.extensions.locales... + + break; + + case 'Prefix': + new_schema.get('prefixes')[record.prefix] = record.reference; + break; + + case 'Class': // Added in order + target_obj = this.getClass(new_schema, record.name); + // ALL MULTISELECT ';' delimited fields get converted back into lists. + if (record.see_also) + record.see_also = this.getArrayFromDelimited(record.see_also); + if (record.tree_root) + record.tree_root = this.setToBoolean(record.tree_root); + + this.copyAttributes(tab_name, record, target_obj, + ['name','title','description','version','class_uri','is_a','tree_root','see_also'] + ); + + break; + + case 'UniqueKey': + let class_record = this.getClass(new_schema, record.class_name); + if (!class_record.get('unique_keys')) + class_record.set('unique_keys', {}); + + target_obj = class_record.get('unique_keys')[record.name] = { + unique_key_slots: record.unique_key_slots.split(';') // array of slot_names + } + this.copyAttributes(tab_name, record, target_obj, + ['description','notes']); + break; + + case 'Slot': + if (record.name) { + + let slot_name = record.name; + let su_class_obj = null; + if (['slot_usage','attribute'].includes(record.slot_type)) { + // Error case if no record.class_name. + su_class_obj = this.getClass(new_schema, record.class_name); + } + switch (record.slot_type) { + + // slot_usage and attribute cases are connected to a class + case 'slot_usage': + su_class_obj.get('slots').push(slot_name); + target_obj = su_class_obj.get('slot_usage')[slot_name] ??= { + name: slot_name, + rank: Object.keys(su_class_obj.get('slot_usage')).length + 1 + }; + break; + + case 'attribute': + // See https://linkml.io/linkml/intro/tutorial02.html for Container objects. + // plural attributes + target_obj = su_class_obj.get('attributes')[slot_name] ??= { + name: slot_name, + rank: Object.keys(su_class_obj.get('attributes')).length + 1 + }; + break; + + // Defined as a Schema slot in case where slot_type is empty: + case 'slot': + default: + target_obj = this.getSlot(new_schema, slot_name); + //target_obj = new_schema.get('slots')[slot_name] ??= {name: slot_name}; + break; + } + + let ranges = record.range?.split(';') || []; + if (ranges.length > 1) { + //if (ranges.length == 1) + //target_obj.range = record.range; + //record.any_of = {}; + //else { // more than one range here + //array of { range: ...} + //target_obj.any_of = ranges.map(x => {return {range: x}}); + record.any_of = ranges.map((x) => {return {'range': x}}); //{return {range: x}}); + record.range = ''; + //} + }; + + if (record.aliases) + record.aliases = this.getArrayFromDelimited(record.aliases); + if (record.todos) + record.todos = this.getArrayFromDelimited(record.todos); + if (record.exact_mappings) + record.exact_mappings = this.getArrayFromDelimited(record.exact_mappings); + if (record.comments) + record.comments = this.getArrayFromDelimited(record.comments); + if (record.examples) + record.examples = this.getArrayFromDelimited(record.examples, 'value'); + // Simplifying https://linkml.io/linkml-model/latest/docs/UnitOfMeasure/ to just ucum_unit. + if (record.unit) + record.unit = {ucum_code: record.unit}; + if (record.structured_pattern) { + //const reg_string = record.structured_pattern; + //console.log('structure', reg_string) + record.structured_pattern = { + 'syntax': record.structured_pattern, + 'partial_match': false, + 'interpolated': true + }; + } + // target_obj .name, .rank, .range, .any_of are handled above. + this.copyAttributes(tab_name, record, target_obj, ['name','slot_group','inlined','inlined_as_list','slot_uri','title','range','any_of','unit','required','recommended','description','aliases','identifier','multivalued','minimum_value','maximum_value','minimum_cardinality','maximum_cardinality','pattern','structured_pattern','todos', 'equals_expression','exact_mappings','comments','examples','version','notes']); + + //if (slot_name== 'passage_number') + // console.log('passage_number', record.minimum_value, target_obj) + } + break; + + case 'Annotation': + + // If slot type is more specific then switch target to appropriate reference. + switch (record.annotation_type) { + case 'schema': + target_obj = new_schema; + break + + case 'class': + target_obj = this.getClass(new_schema, record.class_name); + break; + + case 'slot': + target_obj = new_schema.get('slots').get(record.slot_name); + console.log('annotation', target_obj, record.annotation_type, record.slot_name, new_schema) + break; + + case 'slot_usage': + target_obj = this.getClass(new_schema, record.class_name); + target_obj = target_obj.get('slot_usage')[record.slot_name] ??= {}; + break; + + case 'attribute': + target_obj = this.getClass(new_schema, record.class_name); + target_obj = target_obj.get('attributes')[record.slot_name] ??= {}; + break; + } + + // And we're just adding annotations[record.name] onto given target_obj: + if (typeof target_obj === 'map') { + if (!target_obj.has('annotations')) + target_obj.set('annotations', {}); + target_obj = target_obj.get('annotations'); + } + else { + if (!('annotations' in target_obj)) + target_obj['annotations'] = {}; + target_obj = target_obj.annotations; + } + + target_obj[record.name] = { + key: record.name, // convert name to 'key' + value: record.value + } + + //FUTURE: ADD MENU FOR COMMON ANNOTATIONS LIKE 'foreign_key'? Provide help info that way. + + break; + + case 'Enum': + let enum_obj = new_schema.get('enums')[record.name] ??= {}; + this.copyAttributes(tab_name, record, enum_obj, ['name','title','enum_uri','description']); + break; + + case 'PermissibleValue': // LOOP?????? 'text shouldn't be overwritten. + let permissible_values = new_schema.get('enums')[record.enum_id].permissible_values ??= {}; + target_obj = permissible_values[record.text] ??= {}; + if (record.exact_mappings) { + record.exact_mappings = this.getArrayFromDelimited(record.exact_mappings); + } + this.copyAttributes(tab_name, record, target_obj, ['text','title','description','meaning', 'is_a','exact_mappings','notes']); + break; + + case 'EnumSource': + + // Required field so error situation if it isn't .includes or .minus: + if (record.criteria) { + + let enum_target_obj = new_schema.get('enums')[record.enum_id] ??= {}; + enum_target_obj = enum_target_obj[record.criteria] ??= []; + + if (record.source_nodes) { + record.source_nodes = this.getArrayFromDelimited(record.source_nodes); + } + + if (record.relationship_types) { + record.relationship_types = this.getArrayFromDelimited(record.relationship_types); + } + // The .includes and .minus attributes hold arrays of specifications. + let target_ptr = enum_target_obj.push({}); + console.log(target_ptr, enum_target_obj) + enum_target_obj = enum_target_obj[target_ptr-1]; + + this.copyAttributes(tab_name, record, enum_target_obj, ['source_ontology','is_direct','source_nodes','include_self','relationship_types']); + } + break; + + case 'Setting': + new_schema.get('settings')[record.name] = record.value; + break; + + case 'Type': + // Coming soon, saving all custom/loaded data types. + // Issue: Keep LinkML imported types uncompiled? + break; + } + } + }; + + console.table("SAVING SCHEMA", new_schema); + + // Get rid of empty values.// Remove all class and slot attributes that + // have empty values "", {}, []. + new_schema.get('classes').forEach((attr_map) => { + deleteEmptyKeyVals(attr_map); + }); + + console.log("SLOTS", new_schema.get('slots')) + new_schema.get('slots').forEach((attr_map) => { + deleteEmptyKeyVals(attr_map); + }); + + let metadata = dh_schema.hot.getCellMeta(schema_focus_row, 0); + if (metadata.locales) { + console.log("Got Locales", metadata.locales) + new_schema.set('extensions', {locales: {tag: 'locales', value: metadata.locales}}); + } + + //https://www.yaml.info/learn/quote.html + const a = document.createElement("a"); + //Save JSON version - except this doesn't yet have schemaView() processing: + //a.href = URL.createObjectURL(new Blob([JSON.stringify(schema, null, 2)], { + // quotingType: '"' + //YAML.scalarOptions.str.defaultType = 'PLAIN'; + a.href = URL.createObjectURL(new Blob([YAML.stringify(new_schema, + {singleQuote: true, + lineWidth: 0, + customTags: ['timestamp'] + } + )], {type: 'text/plain'})); + + //strOptions.fold.lineWidth = 80; // Prevents yaml long strings from wrapping. + //strOptions.fold.defaultStringType = 'QUOTE_DOUBLE' + + + //quotingType: '"' + a.setAttribute("download", file_name); + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + + + return true; + } + + getArrayFromDelimited(value, filter_attribute = null) { + if (!value || Array.isArray(value)) + return value; // Error case actually. + return value.split(';') + .map((item) => filter_attribute ? {[filter_attribute]: item} : item) + } + + /** + * Target object gets added/updated the given attribute_list fields, in order. + * + */ + copyAttributes(class_name, record, target, attribute_list) { + for (let [ptr, attr_name] of Object.entries(attribute_list)) { + if (attr_name in record) { //No need to create/save empty values + if (target instanceof Map) {// Required for Map, preserves order. + target.set(attr_name, record[attr_name]); + } + else { + if (!target || !record) { + console.log(`Error: Saving ${class_name}, missing parameters:`, record, target, attribute_list) + alert(`Software Error: Saving ${class_name} ${attr_name}: no target or record`) + } + else { + target[attr_name] = record[attr_name]; + } + } + } + } + }; + + /** + * Components of a schema are set up as Maps with all attributes detailed, + * so order of attributes is preserved. Empty components get removed at end + * of processing with the deleteEmptyKeyVals() call. + */ + getClass(schema, name) { + if (!schema.get('classes').has(name)) { + schema.get('classes').set(name, new Map([ + ['name', ''], + ['title', ''], + ['description', ''], + ['version', ''], + ['class_uri', ''], + ['is_a', ''], + ['tree_root', ''], + ['see_also', []], + ['unique_keys', {}], + ['slots', []], + ['slot_usage', {}], + ['attributes', {}] + ]) ); + } + return schema.get('classes').get(name); + }; + + getSlot(schema, name) { + if (!schema.get('slots').has(name)) { + schema.get('slots').set(name, new Map([ + ['name', ''], + ['rank', ''], + ['slot_group', ''], + ['inlined', ''], + ['inlined_as_list', ''], + ['slot_uri', ''], + ['title', ''], + ['range', ''], + ['any_of', ''], + ['unit', {}], + ['required', ''], + ['recommended', ''], + ['description', ''], + ['aliases', ''], + ['identifier', ''], + ['multivalued', ''], + ['minimum_value', ''], + ['maximum_value', ''], + ['minimum_cardinality', ''], + ['maximum_cardinality', ''], + ['pattern', ''], + ['structured_pattern', {}], + ['todos', ''], + ['equals_expression', ''], + ['exact_mappings', []], + ['comments', ''], + ['examples', ''], + ['version', ''], + ['notes', ''], + ['attributes', {}] + ]) ); + } + return schema.get('slots').get(name); + }; + + /***************************** LOAD & SAVE SCHEMAS **************************/ + + // Classes & slots (tables & fields) in loaded schema editor schema guide what can be imported. + // See https://handsontable.com/docs/javascript-data-grid/api/core/#updatedata + // Note: setDataAtCell triggers: beforeUpdateData, afterUpdateData, afterChange + loadSchemaYAML(text) { + // Critical to ensure focus click work gets data loaded before timing + // reaction in response to loading data / data sets. + let dh_schema = this.context.dhs.Schema; + dh_schema.hot.suspendExecution(); + + let schema = null; + try { + schema = YAML.parse(text); + if (schema === null) + throw new SyntaxError('Schema .yaml file could not be parsed. Did you select a .json file instead?') + } + catch ({ name, message }) { + alert(`Unable to open schema.yaml file. ${name}: ${message}`); + return false; + } + + let schema_name = schema.name; // Using this as the identifier field for schema (but not .id uri) + let loaded_schema_name = schema.name; // In case of loading 2nd version of a given schema. + + let dh_uk = this.context.dhs.UniqueKey; + let dh_slot = this.context.dhs.Slot; + let dh_pv = this.context.dhs.PermissibleValue; + let dh_annotation = this.context.dhs.Annotation; + /** Since user has selected one row/place to load schema at, the Schema table itself + * is handled differently from all the subordinate tables. + * + * If user already has a schema loaded by same name, then: + * - If user is focused on row having same schema, then overwrite (reload) it. + * - If user is on empty row then load the schema as [schema]_X or schema_[version] + * This enables versions of a given schema to be loaded and compared. + * - Otherwise let user know to select an empty row. + * + * FUTURE: simplify to having new Schema added to next available row from top. + */ + let rows = this.context.crudFindAllRowsByKeyVals('Schema', {'name': schema_name}); + let focus_cell = dh_schema.hot.getSelected(); // Might not be one if user just accessed loadSchema by menu + let focus_row = 0; + if (focus_cell) { + focus_row = parseInt(focus_cell[0][0]); // can be -1 row + if (focus_row < 0) + focus_row = 0; + } + + // Find an empty row + if (!focus_cell) { + for (focus_row = 0; focus_row < dh_schema.hot.countRows(); focus_row ++) { + if (dh_schema.hot.isEmptyRow(focus_row)) { + break; + } + } + // here we have focus_row be next available empty row, or new row # at + // bottom of full table. + dh_schema.hot.selectCell(focus_row, 0); + } + + + let focused_schema = dh_schema.hot.getDataAtCell(focus_row, 0) || ''; + let reload = false; + if (rows.length > 0) { + // RELOAD: If focused row is where schema_name is, then consider this a reload + if (rows[0] == focus_row) { + reload = true; + } + else { + // Empty row so load schema here under a [schema_x] name + if (!focused_schema) { + let base_name = schema.name + '_'; + if (schema.version) { + base_name = base_name + schema.version + '_'; + } + let ptr = 1; + while (this.context.crudFindAllRowsByKeyVals('Schema', {'name': base_name + ptr}).length) { + ptr++; + } + loaded_schema_name = base_name + ptr; + } + // Some other schema is loaded in this row, so don't toast that. + else { + return alert("This schema row is occupied. Select an empty schema row to upload to."); + } + } + } + if (focused_schema.length) { + return alert("This schema row is occupied. Select an empty schema row to upload to."); + } + + // If user has requested Schema reload, then delete all existing rows in + // all tables subordinate to Schema that have given schema_name as their + // schema_id key. Possible to improve efficiency via delta insert/update? + // (+Prefix) | Class (+UniqueKey) | Slot (+SlotUsage) | Enum (+PermissableValues) + if (reload === true) { + for (let class_name of Object.keys(this.context.relations['Schema'].child)) { + this.deleteRowsByKeys(class_name, {'schema_id': schema_name}); + } + } + + // Now fill in all of Schema simple attribute slots via uploaded schema slots. + // Allowing setDataAtCell here since performance impact is low. + for (let [dep_col, dep_slot] of Object.entries(dh_schema.slots)) { + if (dep_slot.name in schema) { + let value = null; + // List of schema slot value exceptions to handle: + switch (dep_slot.name) { + // Name change can occur with v.1.2.3_X suffix + case 'name': + value = loaded_schema_name; + break; + case 'see_also': + value = this.getDelimitedString(schema.see_also); + break; + case 'imports': + value = this.getDelimitedString(schema.imports); + break; + + default: + value = schema[dep_slot.name] ??= ''; + } + dh_schema.hot.setDataAtCell(focus_row, parseInt(dep_col), value, 'upload'); + } + } + + /* As well, "schema.extensions", may contain a locale. If so, we add + * right-click functionality on textual cells to enable editing of this + * content, and the local extension is saved. + * schema.extensions?.locales?.value contains {[locale]:[schema] ...} + */ + const locales = schema.extensions?.locales?.value; + if (locales) { + dh_schema.hot.setCellMeta(focus_row, 0, 'locales', locales); + const locale_list = Object.keys(locales).join(';'); + console.log("locales", locales, locale_list) + dh_schema.hot.setDataAtCell(focus_row, dh_schema.slot_name_to_column['locales'], locale_list, 'upload') + } + + // For each DH instance, tables contains the current table of data for that instance. + // For efficiency in loading a new schema, we add to end of each existing table. + let tables = {}; + for (let class_name of Object.keys(this.context.relations['Schema'].child)) { + const dh_table = this.context.dhs[class_name]; + // Doing console.log(hot.getData()) only returns visible rows. + // getSourceData() returns source (visible and hidden) + // https://jsfiddle.net/handsoncode/71y9axdj/ + tables[dh_table.template_name] = dh_table.hot.getSourceData(); + + // Need to RELEASE FILTER? + //const filtersPlugin = dh_table.hot.getPlugin('filters'); + //filtersPlugin.clearConditions(); + //filtersPlugin.filter(); + } + + this.checkForAnnotations(tables, loaded_schema_name, null, null, 'class', schema); + + // Technical notes: Handsontable appears to get overloaded by inserting data via + // setDataAtCell() after loading of subsequent schemas. + // Now using getData() and setData() as these avoid slowness or crashes + // involved in adding data row by row via setDataAtCell(). Seems that + // events start getting triggered into a downward spiral after a certain + // size of table reached. + // 1) Tried using Handsontable dh.hot.alter() to add rows, but this ERRORS + // with "Assertion failed: Expecting an unsigned number." if alter() is + // surrounded by "suspendRender()". Found alter() appears not to be needed + // since Row added automatically via setDataAtCell(). + + // 2nd pass, now start building up table records from core Schema prefixes, + // enums, slots, classes, settings, extensions entries: + let conversion = { + prefixes: 'Prefix', + enums: 'Enum', // Done before slots and classes so slot.range and + //class.slot_usage range can include them. + slots: 'Slot', // Done before classes because class.slot_usage and + // class.attributes add items AFTER main inheritable + // slots. FUTURE: ENSURE ORDERING ??? + classes: 'Class', + settings: 'Setting', + extensions: 'Extension' + }; + + for (let [schema_part, class_name] of Object.entries(conversion)) { + + let dh = this.context.dhs[class_name]; + + // Cycle through parts of uploaded schema's corresponding prefixes / + // classes / slots / enums + // value may be a string or an object in its own right. + + for (let [item_name, value] of Object.entries(schema[schema_part] || {})) { + + // Do appropriate constructions per schema component + switch (class_name) { + //case 'Schema': //done above + // break; + + case 'Prefix': + this.addRowRecord(dh, tables, { + schema_id: loaded_schema_name, + prefix: item_name, + reference: value // In this case value is a string + }); + break; + + case 'Setting': + this.addRowRecord(dh, tables, { + schema_id: loaded_schema_name, + name: item_name, + value: value // In this case value is a string + }); + break; + + case 'Extension': + // FUTURE: make this read-only? + // Each locale entry gets copied to the Extension table/class in a shallow way + // But also gets copied to the schema locales table held in first cell + // See "if (locales)" condition above. + // FUTURE: revise this so Extension's cell metadata holds it? + this.addRowRecord(dh, tables, { + schema_id: loaded_schema_name, + name: item_name, + value: value // In this case value is a string or object. It won't be renderable via DH + }); + + break; + + case 'Enum': + let enum_id = value.name; + this.addRowRecord(dh, tables, { + schema_id: loaded_schema_name, + name: enum_id, + title: value.title, + description: value.description, + enum_uri: value.enum_uri + }); + // If enumeration has permissible values, add them to dh_permissible_value table. + if (value.permissible_values) { + for (let [key_name, obj] of Object.entries(value.permissible_values)) { + this.addRowRecord(dh_pv, tables, { + schema_id: loaded_schema_name, + enum_id: enum_id, + text: key_name, + title: obj.title, + description: obj.description, + meaning: obj.meaning, + exact_mappings: this.getDelimitedString(obj.exact_mappings), + is_a: obj.is_a, + notes: obj.notes // ??= '' + }); + }; + } + // Handling the arrays of downloadable / cacheable enumeration inclusion and excluded sources. + if (value.includes) + this.setEnumSource(tables, loaded_schema_name, enum_id, value.includes, 'includes'); + if (value.minus) + this.setEnumSource(tables, loaded_schema_name, enum_id, value.minus, 'minus'); + + break; + + // Slot table is LinkML "slot_definition". This same datatype is + // referenced by class.slot_usage and class.annotation, so those + // entries are added here. + case 'Slot': + // Setting up empty class name as empty string since human edits to + // create new generic slots will create same. + let slot_name = value.name; + this.addSlotRecord(dh, tables, loaded_schema_name, '', 'slot', slot_name, value); + this.checkForAnnotations(tables, loaded_schema_name, null, slot_name, 'slot', value); + break; + + case 'Class': + let class_name = value.name; + this.addRowRecord(dh, tables, { + schema_id: loaded_schema_name, + name: class_name, + title: value.title, + description: value.description, + version: value.version, + class_uri: value.class_uri, + is_a: value.is_a, + tree_root: this.getBoolean(value.tree_root), // Not needed? + see_also: this.getDelimitedString(value.see_also) + }); + + this.checkForAnnotations(tables, loaded_schema_name, class_name, null, 'class', value); // i.e. class.annotations = ... + + // FUTURE: could ensure the unique_key_slots are marked required here. + if (value.unique_keys) { + for (let [key_name, obj] of Object.entries(value.unique_keys)) { + this.addRowRecord(dh_uk, tables, { + schema_id: loaded_schema_name, + class_name: class_name, + name: key_name, + unique_key_slots: this.getDelimitedString(obj.unique_key_slots), + description: obj.description, + notes: obj.notes // ??= '' + }); + }; + }; + // class.slot_usage holds slot_definitions which are overrides on slots of slot_type 'slot' + if (value.slot_usage) { + // pass class_id as value.name into this?!!!!!!!e + for (let [slot_name, obj] of Object.entries(value.slot_usage)) { + this.addSlotRecord(dh_slot, tables, loaded_schema_name, class_name, 'slot_usage', slot_name, obj); + }; + } + // class.attributes holds slot_definitions which are custom (not related to schema slots) + + + // IGNORE attributes FOR CONTAINER? + if (value.attributes) { + for (let [slot_name, obj] of Object.entries(value.attributes)) { + this.addSlotRecord(dh_slot, tables, loaded_schema_name, class_name, 'attribute', slot_name, obj); + // dh, tables, schema_name, class_name, slot_type, slot_key, slot_obj + }; + } + + break; + } + }; + + }; + + // Get all of the DH instances loaded. + for (let class_name of Object.keys(this.context.relations['Schema'].child)) { + let dh = this.context.dhs[class_name]; + // AVOID: dh.hot.loadData(...); INNEFICIENT + dh.hot.updateSettings({data:Object.values(tables[class_name])}); + } + + // New data type, class & enumeration items need to be reflected in DH + // SCHEMAEDITOR menus. Done each time a schema is uploaded or focused on. + this.context.schemaEditor.refreshMenus(); + this.context.crudCalculateDependentKeys(dh_schema.template_name); + + dh_schema.hot.resumeExecution(); + }; + + setEnumSource(tables, loaded_schema_name, enum_id, source_array, criteria) { + for (let source of source_array) { + this.addRowRecord(this.dhs.EnumSource, tables, { + schema_id: loaded_schema_name, + enum_id: enum_id, + criteria: criteria, + source_ontology: source.source_ontology, + is_direct: source.is_direct, + source_nodes: this.getDelimitedString(source.source_nodes), + include_self: source.include_self, + relationship_types: this.getDelimitedString(source.relationship_types) + }); + } + } + + /** + * Annotations are currently possible on schema, class, slot, slot_usage and attribute. + * source_obj often doesn't have schema_name, class_name, slot_name so they are parameters. + * + */ + checkForAnnotations(tables, schema_name, class_name, slot_name, annotation_type, source_obj) { + // For now DH annotations only apply to slots, slot_usages, and class.attributes + if (source_obj.annotations) { + let dh_annotation = this.context.dhs.Annotation; + let base_record = { + schema_id: schema_name, + annotation_type: annotation_type, + } + switch (annotation_type) { + case 'schema': + break; + case 'class': + base_record.class_name = class_name; + break; + case 'slot': + base_record.slot_name = slot_name; + break; + case 'slot_usage': + case 'attribute': + base_record.class_name = class_name; + base_record.slot_name = slot_name; + break; + } + for (let [tag, obj] of Object.entries(source_obj.annotations)) { + let record = Object.assign(base_record, { + annotation_type: annotation_type, + name: tag, // FUTURE tag: ... + value: obj.value + }); + // NORMALLY name: would be key: but current problem is that header in + // schema_editor tables is using [text] as slot_group for a field + // yields that if that [text] is a slot name, then title is being + // looked up as a SLOT rather than as an enumeration - because DH code + // doesn’t have enumeration yet... + this.addRowRecord(dh_annotation, tables, record); + }; + } + } + + /** The slot object, and the Class.slot_usage object (which gets to enhance + * add attributes to a slot but not override existing slot attributes) are + * identical in potential attributes, so construct the same object for both + * + * slot_obj may contain annotations, in which case they get added to + * annotations table + */ + addSlotRecord(dh, tables, schema_name, class_name, slot_type, slot_key, slot_obj) { + + let slot_record = { + schema_id: schema_name, + slot_type: slot_type, // slot or slot_usage or annotation + name: slot_key, + + // For slots associated with table by slot_usage or annotation + // Not necessarily a slot_obj.class_name since class_name is a key onto slot_obj + class_name: class_name, + rank: slot_obj.rank, + slot_group: slot_obj.slot_group || '', + inlined: this.getBoolean(slot_obj.inlined), + inlined_as_list: this.getBoolean(slot_obj.inlined_as_list), + + slot_uri: slot_obj.slot_uri, + title: slot_obj.title, + range: slot_obj.range || this.getDelimitedString(slot_obj.any_of, 'range'), + unit: slot_obj.unit?.ucum_code || '', // See https://linkml.io/linkml-model/latest/docs/UnitOfMeasure/ + required: this.getBoolean(slot_obj.required), + recommended: this.getBoolean(slot_obj.recommended), + description: slot_obj.description, + aliases: slot_obj.aliases, + identifier: this.getBoolean(slot_obj.identifier), + multivalued: this.getBoolean(slot_obj.multivalued), + minimum_value: slot_obj.minimum_value, + maximum_value: slot_obj.maximum_value, + minimum_cardinality: slot_obj.minimum_cardinality, + maximum_cardinality: slot_obj.maximum_cardinality, + pattern: slot_obj.pattern, + //NOTE that structured_pattern's partial_match and interpolated parameters are ignored. + structured_pattern: slot_obj.structured_pattern?.syntax || '', + equals_expression: slot_obj.equals_expression, + todos: this.getDelimitedString(slot_obj.todos), + exact_mappings: this.getDelimitedString(slot_obj.exact_mappings), + comments: this.getDelimitedString(slot_obj.comments), + examples: this.getDelimitedString(slot_obj.examples, 'value'), + version: slot_obj.version, + notes: slot_obj.notes + }; + this.addRowRecord(dh, tables, slot_record); + + // Slot type can be 'slot' or 'slot_usage' or 'attribute' here. + if (slot_type === 'slot_usage') + this.checkForAnnotations(tables, schema_name, class_name, slot_key, 'slot_usage', slot_obj); + else if (slot_type === 'attribute') + this.checkForAnnotations(tables, schema_name, class_name, slot_key, 'attribute', slot_obj); + }; + + /** + * Returns value as is if it isn't an array, but if it is, returns it as + * semi-colon delimited list. + * @param {Array or String} to convert into semi-colon delimited list. + * @param {String} filter_attribute: A some lists contain objects + */ + getDelimitedString(value, filter_attribute = null) { + if (Array.isArray(value)) { + if (filter_attribute) { + return value.filter((item) => filter_attribute in item) + .map((obj) => obj[filter_attribute]) + .join(';'); + } + return value.join(';'); + } + return value; + } + + + /** Incomming data has booleans as json true/false; convert to handsontable TRUE / FALSE + * Return string so validation works on that (validateValAgainstVocab() where picklist + * is boolean) + */ + getBoolean(value) { + if (value === undefined) + return value; // Allow default / empty-value to be passed along. + return(!!value).toString().toUpperCase(); + }; + + setToBoolean(value) { + return value?.toLowerCase?.() === 'true'; + } + + deleteRowsByKeys(class_name, keys) { + let dh = this.context.dhs[class_name]; + let rows = dh.context.crudFindAllRowsByKeyVals(class_name, keys); + //if (!rows) + // continue; + let dh_changes = rows.map(x => [x,1]); + //if (dh_changes.length) + dh.hot.alter('remove_row', dh_changes); + }; + + /** Insert new row for corresponding table item in uploaded schema. + + */ + addRowRecord(dh, tables, record) { + + let target_record = new Array(dh.slots.length).fill(null); + for (let [slot_name, value] of Object.entries(record)) { + if (slot_name in dh.slot_name_to_column) { + target_record[dh.slot_name_to_column[slot_name]] = value; + } + else + console.error(`Error: Upload of ${dh.template_name} table mentions key:value of (${slot_name}:${value}) but Schema model doesn't include this key`) + } + tables[dh.template_name].push(target_record); + }; + + +// END of SchemaEditor Class +} + +export default SchemaEditor; \ No newline at end of file diff --git a/lib/Toolbar.js b/lib/Toolbar.js index 1f8fd9cc..8a1fe8d6 100644 --- a/lib/Toolbar.js +++ b/lib/Toolbar.js @@ -8,31 +8,31 @@ */ import $ from 'jquery'; +import 'jquery-ui-bundle'; +import 'jquery-ui/dist/themes/base/jquery-ui.css'; + import 'bootstrap/js/dist/carousel'; import 'bootstrap/js/dist/dropdown'; import 'bootstrap/js/dist/modal'; import '@selectize/selectize'; import '@selectize/selectize/dist/css/selectize.bootstrap4.css'; - +const nestedProperty = require('nested-property'); +import {isEmpty, deleteEmptyKeyVals, isEmptyUnitVal, setJSON} from '../lib/utils/general'; +import YAML from 'yaml'; import { exportFile, exportJsonFile, readFileAsync, - createWorkbookFromJSON, + updateSheetRange, exportWorkbook, - importJsonFile, - prependToSheet, + modifySheetRow, } from '../lib/utils/files'; -import { nullValuesToString, isEmptyUnitVal } from '../lib/utils/general'; -import { MULTIVALUED_DELIMITER, titleOverText } from '../lib/utils/fields'; -import { takeKeys, invert } from '../lib/utils/objects'; -import { - findBestLocaleMatch, - // templatePathForSchemaURI, - rangeToContainerClass, - LocaleNotSupportedError, -} from '../lib/utils/templates'; +import { MULTIVALUED_DELIMITER} from '../lib/utils/fields'; +import { takeKeys, invert } from '../lib/utils/objects'; +import { findBestLocaleMatch, LocaleNotSupportedError } from '../lib/utils/templates'; +import { utils as XlsxUtils, read as xlsxRead} from 'xlsx/xlsx.js'; +import { FieldMapper, findHeaderRow } from '../lib/FieldMapper'; import { findLocalesForLangcodes, interface_translation } from './utils/i18n'; import i18next from 'i18next'; @@ -43,9 +43,11 @@ import { menu } from 'schemas'; import pkg from '../package.json'; const VERSION = pkg.version; +const SCHEMA_EDITOR_EXPERT_TABS = '#tab-bar-Prefix, #tab-bar-UniqueKey, #tab-bar-Annotation, #tab-bar-EnumSource, #tab-bar-Extension'; class Toolbar { constructor(root, context, options = {}) { + this.file_extensions = ['xlsx', 'xls', 'tsv', 'csv', 'json']; this.context = context; this.root = root; this.menu = menu; @@ -54,10 +56,13 @@ class Toolbar { this.getExportFormats = options.getExportFormats || this._defaultGetExportFormats; this.getLanguages = options.getLanguages || this._defaultGetLanguages; - + context.toolbar = this; + // Memory of field mapping form content so it can be reset: $(this.root).append(template); + this.initialize(options); this.bindEvents(); + } initialize(options) { @@ -68,6 +73,34 @@ class Toolbar { this.$selectTemplate = $('#select-template'); $('#version-dropdown-item').text('version ' + VERSION); + this.initializeLocale(); + + // Defer loading the Getting Started content until it is used for the first + // time. In ES bundles this dynamic import results in a separate output chunk, + // which reduces the initial load size. There is no equivalent in UMD bundles, + // in which case the content gets inlined. + $('#getting-started-modal').on('show.bs.modal', async function () { + const modalContainer = $('#getting-started-carousel-container'); + if (!modalContainer.html()) { + const { getGettingStartedMarkup } = await import( + './toolbarGettingStarted' + ); + modalContainer.html(getGettingStartedMarkup()); + } + }); + + // Select menu for available templates. If the `templatePath` option was + // provided attempt to use that one. If not the first item in the menu + // will be used. + this.updateTemplateOptions(options.templatePath); + + // Load default (or URL specified) template + this.loadSelectedTemplate(); + + } + + initializeLocale() { + const translationSelect = $('#select-translation-localization').empty(); for (const { nativeName, langcode } of Object.values( findLocalesForLangcodes(Object.keys(interface_translation)) @@ -143,17 +176,13 @@ class Toolbar { }; // Main function logic to iterate over the data + // EFFICIENCY: move ${this.context.getCurrentDataHarmonizer().hot + // OUT OF LOOP. return dataHarmonizerData.map((row, row_index) => { return row.map((value, index) => { - if (value != null && value != '' && isTranslatable[index]) { - console.warn( - `translating ${value} at ${index} (${this.context - .getCurrentDataHarmonizer() - .hot.getDataAtCell(row_index, index)})` - ); - } // Only translate if the value exists and it's marked as translatable if (value && isTranslatable[index]) { + console.warn(`translating ${value} at ${index}`); // Check if the value is multivalued (contains the delimiter) if (value.includes(MULTIVALUED_DELIMITER)) { return translateMultivalued(value); // Translate multivalued string @@ -214,63 +243,66 @@ class Toolbar { console.warn(`locale not supported ${i18next.language}`); } }); - - // Defer loading the Getting Started content until it is used for the first - // time. In ES bundles this dynamic import results in a separate output chunk, - // which reduces the initial load size. There is no equivalent in UMD bundles, - // in which case the content gets inlined. - $('#getting-started-modal').on('show.bs.modal', async function () { - const modalContainer = $('#getting-started-carousel-container'); - if (!modalContainer.html()) { - const { getGettingStartedMarkup } = await import( - './toolbarGettingStarted' - ); - modalContainer.html(getGettingStartedMarkup()); - } - }); - - // Select menu for available templates. If the `templatePath` option was - // provided attempt to use that one. If not the first item in the menu - // will be used. - this.updateTemplateOptions(); - - if (options.templatePath) { - const [schema_folder, template_name] = options.templatePath.split('/'); - //for (const [schema_name, schema_obj] of Object.entries(this.menu)) { - for (const schema_obj of Object.values(this.menu)) { - if ( - 'folder' in schema_obj && - schema_obj['folder'] == schema_folder && - template_name in schema_obj['templates'] - ) { - this.$selectTemplate.val(options.templatePath); - } - } - } - this.loadSelectedTemplate(); } - // Should fire only once per screen load/reload. bindEvents() { + const self = this; + $('#getting-started-modal').on( 'show.bs.modal', this.loadGettingStartedModalContent ); + // TEMPLATE LOADING $('#select-template-load').on('click', () => this.loadSelectedTemplate()); - // $('#view-template-drafts').on('change', () => this.updateTemplateOptions()); $('#upload-template-input').on('change', this.uploadTemplate.bind(this)); + + // SCHEMA EDITOR + // Load a DH schema.yaml file into this slot. + // Prompt user for schema file name. + $('#schema_expert').on('click', (event) => { + let expert_user = $('#schema_expert').is(':checked'); + this.context.expert_user = expert_user; + $(SCHEMA_EDITOR_EXPERT_TABS).toggle(expert_user); + }); + + $('#schema_upload').on('change', (event) => { + const reader = new FileReader(); + reader.addEventListener('load', (event2) => { + if (reader.error) { + alert("Schema file was not found" + reader.error.code); + return false; + } + else { + self.context.schemaEditor.loadSchemaYAML(event2.target.result); + } + // reader.readyState will be 1 here = LOADING + }); + // reader.readyState should be 0 here. + // files[0] has .name, .lastModified integer and .lastModifiedDate + if (event.target.files[0]) { + reader.readAsText(event.target.files[0]); + } + $("#schema_upload").val(''); + + }) + $('#load-schema-editor-button').on('click', (event) => { + this.$selectTemplate.val('schema_editor/Schema').change(); + this.loadSelectedTemplate(); + }); + + $('#save-template-button').on('click', (event) => { + this.context.getCurrentDataHarmonizer().saveSchema(); + }); + + // DATA FILE $('#new-dropdown-item, #clear-data-confirm-btn').on( 'click', this.createNewFile.bind(this) ); - $('#open-file-input').on('change', this.openFile.bind(this)); + $('#open-file-input').on('change', this.openDataFile.bind(this)); $('#save-as-dropdown-item').on('click', () => this.showSaveAsModal()); - $('#file-ext-save-as-select').on( - 'change', - this.toggleJsonOptions.bind(this) - ); $('#save-as-json-use-index').on('change', this.toggleJsonIndexKey); - $('#save-as-confirm-btn').on('click', this.saveFile.bind(this)); + $('#save-as-confirm-btn').on('click', this.saveDataFile.bind(this)); $('#save-as-modal').on('hidden.bs.modal', this.resetSaveAsModal); $('#export-to-confirm-btn').on('click', this.exportFile.bind(this)); $('#export-to-format-select').on( @@ -311,7 +343,37 @@ class Toolbar { $('#next-error-button').on('click', this.locateNextError.bind(this)); $('#validate-btn').on('click', this.validate.bind(this)); $('#help_reference').on('click', this.showReference.bind(this)); - } + + // Future: move to SchemaEditor + $('#translation-save').on('click', this.translationUpdate.bind(this)); + $('#translation-form-button').on('click', (event) => { + this.context.getCurrentDataHarmonizer().translationForm(); + }); + + $('#search-field').on('keyup', (event) => { + // https://handsontable.com/docs/javascript-data-grid/searching-values/ + let dh = this.context.getCurrentDataHarmonizer(); + const search = dh.hot.getPlugin('search'); + dh.queryResult = search.query(event.target.value); // Array of (row, col, text) + dh.hot.render(); + $('#previous-search-button, #next-search-button').toggle(dh.queryResult.length>0) + }); + + $('#previous-search-button, #next-search-button') + .hide() + .on('click', this.searchNavigation.bind(this)); + + // Slot tab will pay attention to selected schema by + // default, but it is possible to select from all schemas, all tables, and/or slot groups. + // See appcontext tabChange() for actions after initialization; + $('#report_select_type').on('change', (event) => { + let dh = this.context.getCurrentDataHarmonizer(); + dh.context.refreshTabDisplay(); + }); + + }; + + /************************** Top Level Menu Commands ************************/ async loadGettingStartedModalContent() { const modalContainer = $('#getting-started-carousel-container'); @@ -321,25 +383,67 @@ class Toolbar { ); modalContainer.html(getGettingStartedMarkup()); } - } - - updateTemplateOptions() { + }; + + // Move forward or backwards through search results. + // Don't use jquery td.current etc. selectors - non-existent if not visible. + searchNavigation(event) { + let direction = $(event.target).is('#next-search-button') ? 1 : -1; + let dh = this.context.getCurrentDataHarmonizer(); + if (!dh.queryResult?.length) return; + let current = dh.current_selection; + let ptr; + if (current[0] === null) + ptr = 0; // If no current item, focus on first result. + else { + ptr = dh.queryResult.findIndex((e) => e.row == current[0] && e.col == current[1]); + if (ptr === undefined) { + ptr = 0; // Also start at beginning. + } + else { + ptr = ptr + direction; + if (ptr < 0) + ptr = dh.queryResult.length-1; + else if (ptr >= dh.queryResult.length) + ptr = 0; + } + } + let item = dh.queryResult[ptr]; + dh.scrollTo(item.row, item.col); + }; + + // Load Toolbar dialogue with all menu.json display:true template names + // and paths. If given template_path is among them, then ensure that one + // is selected (otherwise default is to load first one in menu). + updateTemplateOptions(template_path = null) { this.$selectTemplate.empty(); + let found_schema_editor = false; + // There can be a case where no templates are available because display + // == false for all of them. for (const [schema_name, schema_obj] of Object.entries(this.menu)) { // malformed entry if no 'templates' attribute + const schema_folder = schema_obj.folder; const templates = schema_obj['templates'] || {}; for (const [template_name, template_obj] of Object.entries(templates)) { - let path = schema_obj.folder + '/' + template_name; - let label = - Object.keys(templates).length == 1 - ? template_name - : schema_obj.folder + '/' + template_name; - if (template_obj.display) { + const path = schema_folder + '/' + template_name; + if (path == 'schema_editor/Schema') { + found_schema_editor = true; + } + const label = Object.keys(templates).length == 1 + ? template_name : path; + if (template_obj.display === true) { this.$selectTemplate.append(new Option(label, path)); + if (path == template_path) { + this.$selectTemplate.val(path); + } } } } - } + + // Hide Schema Editor options if no schema_editor/Schema on menu. + $('#load-schema-editor-button, #schema-editor-menu').toggle(found_schema_editor); + + }; async uploadTemplate() { const dh = this.context.getCurrentDataHarmonizer(); @@ -351,153 +455,335 @@ class Toolbar { 'Please upload a template schema.json file' ); } else { - dh.invalid_cells = {}; await this.loadSelectedTemplate(file); $('#upload-template-input')[0].value = ''; - this.hideValidationResultButtons(); + dh.clearValidationResults(); dh.current_selection = [null, null, null, null]; } - } + }; + // Future: create clearTable() option that clears dependent tables too. createNewFile(e) { - const dh = this.context.getCurrentDataHarmonizer(); + const dh = Object.values(this.context.dhs)[0]; // First dh. const isNotEmpty = dh.hot.countRows() - dh.hot.countEmptyRows(); if (e.target.id === 'new-dropdown-item' && isNotEmpty) { $('#clear-data-warning-modal').modal('show'); } else { + for (let [dependent_name] of this.context.relations[dh.template_name].dependents.entries()) { + clearDH(this.context.dhs[dependent_name]); + } + clearDH(dh); $('#file_name_display').text(''); - this.hideValidationResultButtons(); - dh.newHotFile(); } - } + }; + // INITIALIZATIN: Triggers only for loaded top-level template, rather than + // any dependent. restartInterface(myContext) { - //console.log('myContext',myContext); myContext.addTranslationResources(myContext.template); this.updateGUI( myContext.getCurrentDataHarmonizer(), myContext.getTemplateName() ); + let hot = myContext.getCurrentDataHarmonizer().hot; + // By selecting first cell on reload, we trigger deactivation of dependent + // tables that have incomplete keys wrt top-level class. + hot.selectCell(0, 0); + hot.deselectCell(); // prevent accidental edits. + console.log('restartInterface with selectCell(0,0)'); } - async openFile() { + /** + * User clicks on "Data -> Open" menu item" + */ + async openDataFile() { const file = $('#open-file-input')[0].files[0]; if (!file) { // Happens when user cancels open action. return; } + const file_parts = file.name.split('.'); const ext = file.name.split('.').pop(); - const acceptedExts = ['xlsx', 'xls', 'tsv', 'csv', 'json']; - if (!acceptedExts.includes(ext)) { + if (!this.file_extensions.includes(ext)) { this.showError( 'open', `Only ${acceptedExts.join(', ')} files are supported` ); - } else { - if (ext === 'json') { - $('#open-file-input')[0].value = ''; // enables reload of file by same name. - // JSON is the only format that contains reference to schema to - // utilize by "schema" URI, as well as "in_language" locale, and - // its Container class contains template spec itself. - // It may have several dhs, one for each Container class mentioned. - const contentBuffer = await readFileAsync(file); - //alert("opening " + file.name) - let jsonData; - try { - jsonData = JSON.parse(contentBuffer.text); - } catch (error) { - throw new Error('Invalid JSON data', error); - } + return; + } - const { schema_uri, in_language = null } = importJsonFile(jsonData); - const translationSelect = $('#select-translation-localization'); - const previous_language = i18next.language; - // ensure is localized in the same language as the file - if (in_language !== previous_language) { - i18next.changeLanguage( - in_language === 'en' ? 'default' : in_language - ); - translationSelect.val( - i18next.language === 'en' ? 'default' : in_language - ); - // .trigger('input'); - $(document).localize(); - } + // enables reload of file by same name (otherwise browser ignores) and + // enable save by it too. + $('#open-file-input')[0].value = ''; + $('#base-name-save-as-input').val(file_parts[0]); + $('#file-ext-save-as-select').val(ext); - // If we're loading a JSON file, we have to match its schema_uri to a - // schema. Check loaded schema, but if not there, lookup in menu. - // Then if provided, load schema in_language locale. - // Future: automate detection of json file schema via menu datastructure. - // In which case template_path below has to be changed to match. + console.log('reload 2: openDataFile'); - // Currently loaded schema_uri: this.context.template.default.schema.id - if (schema_uri != this.context.template.default.schema.id) { - alert( - `The current json file's schema "${schema_uri}" is required, but one must select this template from menu first, if available. Online retrieval of schemas is not yet available.` - ); - return false; - } - const locale = in_language; - - console.log('reload 2: openfile'); - const template_path = this.context.appConfig.template_path; - // e.g. canada_covid19/CanCOGeN_Covid-19 - await this.context - .reload(template_path, locale) - .then((context) => { - if (!jsonData.Container) { - alert( - 'Error: JSON data file does not have Container dictionary.' - ); - } else { - // The data is possibly *sparse*. loadDataObjects() fills in missing values. - for (const dh in context.dhs) { - // Gets name of container object holding the given dh class. - const container_class = rangeToContainerClass( - context.template.default.schema.classes.Container, - dh - ); - - const list_data = context.dhs[dh].loadDataObjects( - jsonData.Container[container_class], - locale - ); - - console.log('here', container_class, list_data); // - - if (list_data) context.dhs[dh].hot.loadData(list_data); - else - alert( - 'Unable to fetch table data from json file ' + - template_path + - ' for ' + - dh - ); - } - } + let loaded = false; + if (ext === 'json') { + loaded = this.openJSONDataFile(file) + this.restartInterface(this.context); // To handle possible locale change + } + + // Here we're loading tsv, csv, (just one template/class) + // OR xls, xlsx (which can have a schema with multiple classes) + else { + loaded = this.openTabularDataFile(file); + } - return context; - }) - .then(this.restartInterface.bind(this)); - } else { - for (const dh of Object.values(this.context.dhs)) { - dh.invalid_cells = {}; - await this.context.runBehindLoadingScreen(dh.openFile.bind(dh), [ - file, - ]); - dh.current_selection = [null, null, null, null]; - } + if (loaded) + $('#file_name_display').text(file.name); + } + + /** + * JSON and .xls/.xlsx are formats that contains schema URI that data pertains + * to. For JSON this can have the extra step of requesting users to load the + * schema if it isn't already loaded. + * As well it has the "in_language" locale code, and its Container class holds + * attributes object that lists data for one or more templates / Classes. + * THIS IS GETTING A SCHEMA RELOAD JUST BECAUSE OF A POTENTIAL LANGUAGE CHANGE + * AS PROVIDED IN JSON DATA FILE in_language="...". + { + "schema": "https://example.com/folder", + "version": "x.y.z", + "in_language": "en", + "container": {(class table name): [{row 1 object}, {row 2 object}, ...], ...} + } + */ + async openJSONDataFile(file) { + + const contentBuffer = await readFileAsync(file); + let jsonData; + try { + jsonData = JSON.parse(contentBuffer.text); + } + catch (error) { + throw new Error('Invalid JSON data', error); + return false; + } + + // Validate whether data file has minimum necessary Schema components. + // Future: change .schema label to .schema_uri + const metaDataFields = ['schema', 'version', 'in_language', 'container']; + const matches = metaDataFields.filter(element => !(element in jsonData)).toString(); + if (matches) { + alert(`The JSON data file's metadata is not compatible for loading! Missing fields: ${matches}`); + return false; + }; + + /** + * If a JSON file is being loaded, it's schema uri must match its schema + * uri to the current loaded schema. Check loaded schema, but if not there, + * lookup in menu. Then if provided, load schema in_language locale. + * + * Future: automate detection of json file schema via menu datastructure. + * In which case template_path below has to be changed to match. + */ + const schema = this.context.getSchemaRef(); + if (jsonData.schema != schema.id) { + alert(`The data schema template file "${jsonData.schema}" needs to be loaded first before this data file can be loaded. \n\nSelect the template, either by selecting it in the template menu, or by using the "Upload Template" option.`); + return false; + }; + + // Does interface language switch to match data content + this.check_locale(jsonData.in_language); + + const template_path = this.context.appConfig.template_path; + // e.g. canada_covid19/CanCOGeN_Covid-19 + await this.context.reload(template_path, jsonData.in_language) // an AppContext.js call + + // First construct a set of data tables from input file's perspective + const raw_data = {}; // data[table]{header:..., data:...} + const messages = []; + + /** + * The JSON file should have one or more tables in it that match to loaded + * schema. The data is usually *sparse*, so we have to read ALL keys per + * row to determine if they are new, and recreate data as + */ + for (const [template_name, dh] of Object.entries(this.context.dhs)) { + + const data = jsonData.container[dh.container_name]; + if (!data) { + messages.push(`The LinkML schema "${schema.name}" template "${template_name}" has NO corresponding table in the uploaded data file. The data file's tabs or tables are: "${Object.keys(jsonData.container)}". Consequently, no data will be loaded. Either a new table has been added to schema, or the wrong schema is loaded.\n`); + continue; + } + + /** + * The JSON array of row objects sparse, so only keys with values are + * included. Search through a table's row objects to accumulate a full + * list of their keys. The col_index provides a ROUGH idea of which + * column to claim for a given key, though a sparse array may vary from + * row to row depending on required/optional fields. + */ + const header = []; + Object.entries(data).forEach(([row_index, row]) => { + Object.entries(row).forEach(([key, col_index]) => { + if (!(header.includes(key))) { + header.splice(col_index, 0, key); + } + }); + }); - $('#file_name_display').text(file.name); - $('#open-file-input')[0].value = ''; // enables reload of file by same name. - this.hideValidationResultButtons(); + // JSON data SHOULD USE slot_names BUT some test files have slot_titles. + const matches = header.filter((element, index) => dh.slot_names.includes(element) || dh.slot_titles.includes(element)).length; + // raw_data only contains templates that match incomming data tables. + raw_data[template_name] = { + header: header, + data: data, + matches: matches, + table_name: dh.container_name } + + }; + // Future: make a generic modal for this: + if (messages.length) { + alert(messages.join('\n')) } + + this.loadTabularData(file, raw_data); + return true; } - async saveFile() { + /** + * FUTURE: read in "Schema" tab in the workbook to get the schema, version, + * and locale information. Currently its assumed that loaded data should fit + * pre-loaded schema. + */ + async openTabularDataFile(file) { + let workbook; + const messages = []; + + try { + let contentBuffer = await readFileAsync(file); + workbook = xlsxRead(contentBuffer.binary, { + type: 'binary', + raw: true, + cellDates: true, // Ensures date formatted as YYYY-MM-DD dates + dateNF: 'yyyy-mm-dd', //'yyyy/mm/dd;@' + }); + } + catch (error) { + throw new Error('Invalid spreadsheet data', error); + return false; + } + + // First construct a set of data tables from input file's perspective + const raw_data = {}; // data[table]{header:..., data:...} + + for (const [template_name, dh] of Object.entries(this.context.dhs)) { + + let sheet_name = dh.getSpreadsheetName(workbook); // relevant to given dh. + if (!sheet_name) { + messages.push(`The LinkML schema "${this.context.getSchemaRef().name}" template "${template_name}" has NO corresponding table in the uploaded data file. The data file's tabs or tables are: "${workbook.SheetNames}". Consequently, no data will be loaded. Either a new table has been added to schema, or the wrong schema is loaded.\n`); + continue; + } + + const worksheet = updateSheetRange(workbook.Sheets[sheet_name]); + // Load data pretending that header row is first row, but it is usually + // on 2nd row. + // 1 = first row contains header and output is array of arrays. + // 0 = ditto, and output is objects; + // [array of strings] = these become headers, and sheet data starts at row 0 + const matrix = XlsxUtils.sheet_to_json(worksheet, {header: 1, raw: false, range: 0}); + + // Calculate REAL headerRow - usually it is 2nd row of tsv/csv/xlsx data): + const [header_row, matches] = findHeaderRow(dh, matrix); + + // A DH template/tab had NO fields matching to respective data table! + // It is possibly a new table in schema and so will be allowed in as + // an empty table. + if (header_row < 1 || matches === 0) { + messages.push(`The LinkML schema "${this.context.getSchemaRef().name}" template "${template_name}" has NO header row OR NO matching fields with the corresponding data file tab or table. This DH tab table will be empty.\n`); + continue; + } + + // raw_data only contains templates which were matched to incomming + // data tables. Data starts at calculated header row. + raw_data[template_name] = { + header: matrix[header_row-1], // an array of [{field_name: value}...] + data: matrix.slice(header_row), + matches: matches, + table_name: sheet_name + } + + }; + + // Future: make a generic modal for this: + if (messages.length) { + alert(messages.join('\n')) + } + + this.loadTabularData(file, raw_data); + return true; + } + + /* Here raw_data might not have right field headers, so it is tested against + * loaded schema, and any missmatches trigger popup display of fields for + * user to review and map. Tables without mismatches are loaded immediately. + * Tables with missmatches are loaded after users field map (if any) is + * established. + */ + loadTabularData(file, raw_data) { + + const fieldMapper = new FieldMapper(this.context, file, raw_data); + + /** + * Examine data file for content appropriate for each tab in DH, loading + * in order provided by schema. No need to add this dh to fieldMapper if: + * + * 1) all data fields have been matched to template. In this scenario + * template might have some new fields which will be empty on load. + * 2) Some data fields haven't matched, but all the matches complete + * the template. + * + * Field content requirements are managed by validation system elsewhere. + */ + Object.entries(this.context.dhs).forEach(([template_name, dh]) => { + let data = raw_data[template_name]; + // 29, 34 total, 173 fields total. + //console.log("MATCHES",data.matches,data.header.length,dh.slots.length); + if (data && (data.matches != data.header.length) && (data.matches != dh.slots.length)) { + // Have user try to correct any missing mappings in a class/template. + fieldMapper.appendFieldMappingModal(dh); + } + }); + + if (fieldMapper.field_mapping_html) { + // The specify headers modal displays incongruities in ALL DH tab field + // mapping. It is appended to when problem spotted in mapping below. + // FieldMapper will hot.loadData tables from revised mapping. + fieldMapper.renderFieldMappingModal(); + this.context.runBehindLoadingScreen(fieldMapper.show); + } + else { + // No issues with field mapping so load all the data. + fieldMapper.loadMappedData(); + } + + } + + // Localize the DOM app interface to same language as the data file. + // Requires translation file to have given data file locale. + check_locale(language = 'en') { + if (language !== i18next.language) { + i18next.changeLanguage(language === 'en' ? 'default' : language); + const translationSelect = $('#select-translation-localization'); + translationSelect.val(i18next.language === 'en' ? 'default' : language); + $(document).localize(); + } + } + + /***************************** SAVE DATA FILES ****************************/ + + // Save one tab, or 1-many tabs of data in format given by form controls. + // FUTURE: Switch to file system API so can use file browser to select + // target of save. See + // https://code.tutsplus.com/how-to-save-a-file-with-javascript--cms-41105t + async saveDataFile() { const baseName = $('#base-name-save-as-input').val(); const ext = $('#file-ext-save-as-select').val(); const lang = $('#select-translation-localization').val(); @@ -505,182 +791,248 @@ class Toolbar { $('#save-as-err-msg').text('Specify a file name'); return; } - const MultiEntityJSON = Object.values(this.context.dhs).reduce( - (acc, dh) => { - return Object.assign(acc, { [dh.template_name]: dh.toJSON() }); - }, - {} - ); - const schema_container = - this.context.template.default.schema.classes.Container; - - // default schema is guaranteed to feature the Container - const Container = Object.entries(schema_container.attributes).reduce( - (acc, [cls_key, { name, range }]) => { - if (typeof range !== 'undefined' && this.context.dhs[range]) { - //Or test range against this.context.appConfig.template_path.split('/')[1] - const processedClass = { - [name]: MultiEntityJSON[range] - .map((obj) => nullValuesToString(obj)) - .map((entry) => { - // translation: if available, use title over text given a non-default localization for export - // TODO?: check if current lang is equal to current schema lang? - const fields = this.context.dhs[range].slots; - const findField = (colKey) => - fields.filter((field) => field.title === colKey)[0]; - entry = Object.fromEntries( - Object.entries(entry).map(([k, v]) => { - const field = findField(k); - let nv = v; - if (field.sources && !isEmptyUnitVal(v)) { - const merged_permissible_values = field.sources.reduce( - (acc, source) => { - return Object.assign( - acc, - field.permissible_values[source] - ); - }, - {} - ); - if (field.multivalued === true) { - nv = v - .split(MULTIVALUED_DELIMITER) - .map((_v) => { - if (!(_v in merged_permissible_values)) - console.warn( - `${_v} not in merged_permissible_values ${Object.keys( - merged_permissible_values - )}` - ); - return _v in merged_permissible_values - ? titleOverText(merged_permissible_values[_v]) - : _v; - }) - .join(MULTIVALUED_DELIMITER); - } else { - if (!(v in merged_permissible_values)) - console.warn( - `${v} not in merged_permissible_values ${Object.keys( - merged_permissible_values - )}` - ); - nv = - v in merged_permissible_values - ? titleOverText(merged_permissible_values[v]) - : v; - } - } - return [k, nv]; - }) - ); - return entry; - }), - }; - - return Object.assign(acc, processedClass); - } else { - console.warn('Container entry has no range:', cls_key); - return acc; - } - }, - {} - ); + const container_datasets = this.getContainerData(ext); + + if (ext === 'json') { + this.saveJSONDataFile(container_datasets, lang, baseName, ext); + } + + // Here we process .xlsx, .xls, .tsv, .csv + else { + this.saveTabularDataFile(container_datasets, baseName, ext); + } + $('#save-as-modal').modal('hide'); + } + + async saveJSONDataFile(container_datasets, lang, baseName, ext) { const JSONFormat = { schema: this.context.template.schema.id, - location: this.context.template.location, version: this.context.template.schema.version, - in_language: lang === 'default' ? 'en' : lang, - Container, + container: container_datasets }; - if (ext === 'json') { - const filterFunctionTemplate = - (condCallback = () => true, keyCallback = (id) => id) => - (obj) => - Object.keys(obj).reduce((acc, itemKey) => { - return condCallback(obj, acc, itemKey) - ? { - ...acc, - [keyCallback(itemKey)]: obj[itemKey], - } - : acc; - }, {}); + if (lang !== 'default') { + JSONFormat.in_language = lang; + } - const filterEmptyKeys = filterFunctionTemplate( - (obj, acc, itemKey) => - itemKey in obj && !(itemKey in acc) && obj[itemKey] != '', - (id) => id - ); + // Clear out empty values in each Container attribute's content (an + // array of given LinkML class's objects) + for (const class_name in JSONFormat.container) { + for (const row in JSONFormat.container[class_name]) { + const obj = JSONFormat.container[class_name][row]; + deleteEmptyKeyVals(obj); + }; + } - const processEntryKeys = (lst) => lst.map(filterEmptyKeys); + await this.context.runBehindLoadingScreen(exportJsonFile, [ + JSONFormat, + baseName, + ext, + ]); + } - for (let concept in JSONFormat.Container) { - JSONFormat.Container[concept] = processEntryKeys( - JSONFormat.Container[concept] - ); - } + async saveTabularDataFile(container_datasets, baseName, ext) { - await this.context.runBehindLoadingScreen(exportJsonFile, [ - JSONFormat, - baseName, - ext, - ]); - } else { - const sectionCoordinatesByClass = Object.values(this.context.dhs).reduce( - (acc, dh) => { - const sectionTiles = dh.sections.map((s) => s.title); - const takeSections = takeKeys(sectionTiles); - return Object.assign(acc, { - [rangeToContainerClass(schema_container, dh.template_name)]: invert( - takeSections(dh.getColumnCoordinates()) - ), - }); - }, - {} - ); + const container_table_names = + this.context.template.default.schema.classes.Container.attributes; - const columnCoordinatesByClass = Object.values(this.context.dhs).reduce( - (acc, dh) => { - const columnIndexCoordinates = dh.slots.reduce( - (acc, field, i) => Object.assign(acc, { [field.name]: i }), - {} - ); - return Object.assign(acc, { - [rangeToContainerClass(schema_container, dh.template_name)]: invert( - columnIndexCoordinates - ), - }); - }, - {} - ); + const sectionCoordinatesByClass = Object.values(this.context.dhs).reduce( + (acc, dh) => { + const sectionTiles = dh.sections.map((s) => s.title); + const takeSections = takeKeys(sectionTiles); + return Object.assign(acc, { + [dh.container_name]: invert( + takeSections(dh.getColumnCoordinates()) + ), + }); + }, + {} + ); - let MultiEntityWorkbook = createWorkbookFromJSON(JSONFormat.Container); - MultiEntityWorkbook.SheetNames.forEach((sheetName) => { - if (!(JSONFormat.Container[sheetName].length > 0)) { - prependToSheet( - MultiEntityWorkbook, - sheetName, - columnCoordinatesByClass[sheetName] - ); - } - prependToSheet( + // The workbook is set up with Container tab names. + let MultiEntityWorkbook = XlsxUtils.book_new(); + + // Loop through each sheet in the DH combined container datasets + for (const container_name in container_datasets) { + // Convert data to an array of arrays for xlsx + const xlsx_table = XlsxUtils.json_to_sheet(container_datasets[container_name]); + // Add the worksheet to the workbook + XlsxUtils.book_append_sheet(MultiEntityWorkbook, xlsx_table, container_name); + // Behaviour different for xlsx vs tsv files etc? + if (!(container_datasets[container_name].length > 0)) { + modifySheetRow( MultiEntityWorkbook, - sheetName, - sectionCoordinatesByClass[sheetName] + container_name, + this.context.dhs[container_table_names[container_name].range].slot_names, + -1 // Signals to insert a row ); - }); - - await this.context.runBehindLoadingScreen(exportWorkbook, [ + } + // 1 row of section headers + modifySheetRow( MultiEntityWorkbook, - baseName, - ext, - ]); + container_name, + sectionCoordinatesByClass[container_name], + -1 + ); } - $('#save-as-modal').modal('hide'); + + // Add Schema tab for exell outputs + if (ext == 'xlsx' || ext == 'xls') { + const schema = this.context.template.default.schema; + const container_table_list = Object.keys(schema.classes?.Container?.attributes || {}).join(';'); + const xlsx_table = [ + ["id", "name", "version", "in_language", "container"], + [schema.id, schema.name, schema.version, schema.in_language, container_table_list] + ]; + const ws = XlsxUtils.aoa_to_sheet(xlsx_table); + XlsxUtils.book_append_sheet(MultiEntityWorkbook, ws, 'Schema'); + } + + // Saving one file here for xls, xlsx, or multiple files for tsv/csv + await this.context.runBehindLoadingScreen(exportWorkbook, [ + MultiEntityWorkbook, + baseName, + ext, + ]); } + /** + * Returns dictionary of pertinent tab classes and their tabular row data. + * Container's attribute names are usually plural forms of class names, + * but may be more arbitrary. They are used directly in JSON and xls/xlsx + * output. + * + * Look through a schema Container's attributes and for each one, if it is + * a loaded DH template/tab, return an array of rows where each is a DH + * instance. + * + * @param {String} ext - file type, which influences controls how column + * headers should be named. + */ + getContainerData(ext) { + + let Container = {}; + + // A schema's defined Container class where details of which classes to + // include in one or one-to-many tabular data saved format are contained. + const container_attributes = + this.context.template.default.schema.classes.Container.attributes; + + // Loop that processes container tables + for (const [attribute_name, attribute] of Object.entries(container_attributes)) { + + if (!attribute.range || !this.context.dhs[attribute.range]) { + console.warn(`Schema Container attribute "${attribute_name}" has no .range pointing to schema class.`); + continue; + } + + const dh = this.context.dhs[attribute.range] + const table = []; + + // Loop through each row of DH tab data and create a structure that + // includes all slot keys. row is an object with indexes 0, 1 ... for each column + + for (const row of dh.hot.getSourceData()) { + // Ignore empty rows + if (isEmpty(row)) // || row.every((val) => val === null) + continue; + + // row_dict for JSON is minimalist dictionary - only filled fields + // All other formats include full tabular column list + // Make this an ordered dict via map object? + let row_dict = {}; + if (ext !== 'json') { + Object.values(dh.slots).forEach((slot) => {row_dict[slot.title] = ''}); + } + + // Iterate over the columns in a row + Object.entries(row).forEach(([index, value]) => { + const slot_name = dh.slot_names[index]; + const slot = dh.slots[dh.slot_name_to_column[slot_name]]; + let new_value = value; + if (!isEmptyUnitVal(value)) { // 'undefined' / null / ''; + + // If a slot has sources it means one of its ranges is a picklist. + if (slot.sources) { + // Case where we should save array for json. + if (slot.multivalued === true) { + new_value = value.split(MULTIVALUED_DELIMITER) // An array + .map((_value) => { + // AppContext.js slot.merged_permissible_values is dictionary + // of enum permissible value entries. + return this.getValueByNameOrTitle(_value, ext, slot.merged_permissible_values); + }); + // Convert new_value to a string unless json and > 1 value + // in which case keep it as array. + if (ext !== 'json' || new_value.length < 2 ) { + new_value = new_value.join(MULTIVALUED_DELIMITER); + } + } + else { + new_value = this.getValueByNameOrTitle(value, ext, slot.merged_permissible_values); + } + } + + /** A slot might have a menu selection such as a null value above. + * But it might also be a number like age, or boolean. HANDLE + * conversion of numbers and booleans by slot datatype. + */ + new_value = setJSON(new_value, slot.datatype); + + } + + // Currently only JSON format stores via native slot.name keys + // For tabular data, this could be made an option in the future. + row_dict[ext === 'json' ? slot_name : slot.title] = new_value; + + }) + // Append to table. + table.push(row_dict); + + } // End of DH data row processing. + + Container[attribute_name] = table; + + }; + + return Container + } + + /** + * Value is looked up as enumeration text or title if possible. + * value_list is merged schema.enums[x].permissible_values structure. + */ + getValueByNameOrTitle (value, file_extension, value_list) { + + if (isEmpty(value_list)) + return value; + + if (!(value in value_list)) + console.warn(`${value} not in slot.sources`, value_list); + return value; + + if (file_extension === 'json') + return value; // Why this case? + + if (value_list[value].title) + return value_list[value].title; + + return value_list[value].text; + + } + +/* +export function titleOverText(enm) { + try { + return typeof enm.title !== 'undefined' ? enm.title : enm.text; + } catch (e) { + console.error(e, enm); + } +} +*/ + showSaveAsModal() { const dh = this.context.getCurrentDataHarmonizer(); if (!$.isEmptyObject(dh.invalid_cells)) { @@ -690,25 +1042,16 @@ class Toolbar { } } - toggleJsonOptions(evt) { - if (evt.target.value === 'json') { - $('#save-as-json-options').removeClass('d-none'); - const inferredIndexSlot = this.context - .getCurrentDataHarmonizer() - .getInferredIndexSlot(); - $('#save-as-json-index-key').val(inferredIndexSlot); - } else { - $('#save-as-json-options').addClass('d-none'); - } - } - toggleJsonIndexKey(evt) { $('#save-as-json-index-key').prop('disabled', !evt.target.checked); } resetSaveAsModal() { $('#save-as-err-msg').text(''); - $('#base-name-save-as-input').val(''); + // Questionable: To prevent quirk of javascript inability to save file by same + // name twice we have to clear out field and then put its contents back in. + //const baseName = $('#base-name-save-as-input').val(); + //$('#base-name-save-as-input').val('').val(baseName); } async exportFile() { @@ -806,6 +1149,13 @@ class Toolbar { let focus_col = dh.current_selection[1]; const all_rows = Object.keys(dh.invalid_cells); + + if (all_rows.length == 0) { + // No error to find. + // Issue: user may have moved dh into hiding invalid rows. + return; + } + const error1_row = all_rows[0]; if (focus_row === null) { focus_row = error1_row; @@ -838,6 +1188,7 @@ class Toolbar { async validate() { const dh = this.context.getCurrentDataHarmonizer(); + //await dh.validate();//dh.validate().bind(dh); await this.context.runBehindLoadingScreen(dh.validate.bind(dh)); if (Object.keys(dh.invalid_cells).length > 0) { @@ -849,10 +1200,26 @@ class Toolbar { } } - showReference() { - for (const dh in this.context.dhs) { - this.context.dhs[dh].renderReference(); + /** Take translation modal edited rows and update this schema's locale + * extension. + */ + translationUpdate() { + const schema = this.context.dhs.Schema; + const locales = schema.hot.getCellMeta(schema.current_selection[0], 0).locales; + + for (let input of $("#translate-modal-content textarea")) { + let path = $(input).data('path'); + let value = $(input).val(); + // path e.g. fr.enums.HostAgeUnitMenu.permissible_values.year.description + nestedProperty.set(locales, path, value); } + + $('#translate-modal').modal('hide'); + } + + showReference() { + const dh = this.context.getCurrentDataHarmonizer(); + dh.renderReference(); // Prevents another popup on repeated click if user focuses away from // previous popup. return false; @@ -863,8 +1230,9 @@ class Toolbar { $(`#${prefix}-error-modal`).modal('show'); } - hideValidationResultButtons() { - $('#next-error-button,#no-error-button').hide(); + hideValidationResults() { + const dh = this.context.getCurrentDataHarmonizer(); + dh.clearValidationResults(); } // LoadSelectedTemplate either comes from @@ -896,6 +1264,13 @@ class Toolbar { try { const contentBuffer = await readFileAsync(file); schema = JSON.parse(contentBuffer.text); + + if (schema.Container) { + alert("This doesn't appear to be a DataHarmonizer schema! Cancelling upload.") + return null; + } + + // Phasing this out in favour of Container. template_name = Object.keys(schema.classes).find( (e) => schema.classes[e].is_a === 'dh_interface' ); @@ -926,39 +1301,63 @@ class Toolbar { const loadFromMenu = async () => { const template_path = this.$selectTemplate.val(); + if (!template_path) { + alert("Error: loadFromFile() was unable to load a default schema because no schemas or templates exist or have a display:true setting (in /web/templates/menu.json)"); + return false; + } const [schema_folder, template_name] = template_path.split('/'); const schema = await this.getSchema(schema_folder, template_name); - return { template_path, schema }; }; + // If given a file, load that, otherwise select first menu item let loadResult = file ? await loadFromFile(file) : await loadFromMenu(); if (!loadResult) return; let { template_path, schema } = loadResult; - // RELOAD THE INTERFACE BY INTERACTING WITH THE CONTEXT + // Error checking: if menu lists a class name that no longer exists in LinkML spec + // then alert user (developer) of issue, and return without change. + const [schema_folder, template_name] = template_path.split('/'); + + if (!(template_name in schema.classes)) { + alert(`Warning: The requested schema template ${template_path} could not be loaded. If present in the menu.json file, its entry likely needs to be updated by a DataHarmonizer schema maintainer.`); + return; + } + // RELOAD THE INTERFACE BY INTERACTING WITH THE CONTEXT console.log('reload 3: loadSelectedTemplate'); this.context - .reload(template_path, null, file ? schema : null) + .reload(template_path, null, schema) .then(this.restartInterface.bind(this)); // SETUP MODAL EVENTS + // moved to AppContext.js onDHTabChange event handler. + /* $(document).on('dhCurrentChange', (event, extraData) => { - this.setupSectionMenu(extraData.dh); + const dh = extraData.dh; + const class_name = dh.template_name; + this.setupSectionMenu(dh); // Jump to modal as well - this.setupJumpToModal(extraData.dh); - this.setupFillModal(extraData.dh); + this.setupJumpToModal(dh); + this.setupFillModal(dh); + dh.clearValidationResults(); + let dependent_report = dh.context.dependent_rows.get(class_name); + // await dh.filterByKeys(dh, class_name, dependent_report.fkey_vals); + //dh.hot.render(); // Required to update picklist choices ???? }); - + */ // INTERNATIONALIZE THE INTERFACE // Interface manually controlled via language pulldown. //i18next.changeLanguage('default'); $(document).localize(); + + + // ISSUE: dhs aren't setup yet, due to asynchronous .reload() ??? + // Or it is that dh.context is getting overwritten too late. } updateGUI(dh, template_name) { - $('#template_name_display').text(template_name); + //$('#template_name_display').text(template_name); $('#file_name_display').text(''); const selectExportFormat = $('#export-to-format-select'); selectExportFormat.children(':not(:first)').remove(); @@ -989,21 +1388,29 @@ class Toolbar { const helpSop = $('#help_sop'); const template_classes = this.context.template.schema.classes[template_name]; - if (template_classes && template_classes.see_also) { - helpSop.attr('href', template_classes.see_also).show(); + if (template_classes?.see_also) { + if (Array.isArray(template_classes.see_also)) + helpSop.attr('href', template_classes.see_also[0]).show(); + else + helpSop.attr('href', template_classes.see_also).show(); } else { helpSop.hide(); } + let schema_editor = this.context.getCurrentDataHarmonizer().schema.name === 'DH_LinkML'; + let expert_user = $('#schema_expert').is(':checked'); + $('#schema-editor-menu').toggle(schema_editor); + $(SCHEMA_EDITOR_EXPERT_TABS).toggle(expert_user); + $('#slot_report_control').hide(); + this.setupJumpToModal(dh); this.setupSectionMenu(dh); this.setupFillModal(dh); - this.hideValidationResultButtons(); + dh.clearValidationResults(); } setupFillModal(dh) { const fillColumnInput = $('#fill-column-input').empty(); - // Initialize the selectize input field for column selection fillColumnInput.selectize({ valueField: 'title', @@ -1044,6 +1451,7 @@ class Toolbar { }); } + // Note: HandsonTable won't advance to column if table has empty rows; no workaround to this. setupJumpToModal(dh) { const columnCoordinates = dh.getColumnCoordinates(); @@ -1054,7 +1462,7 @@ class Toolbar { }); // Set up the modal opening event to clear and refresh the options dynamically - $('#jump-to-modal').on('shown.bs.modal', () => { + $('#jump-to-modal').off().on('shown.bs.modal', () => { const selectizeInstance = jumpToInput[0].selectize; // Ensure the selectize input is fully cleared before loading new options @@ -1075,13 +1483,13 @@ class Toolbar { }); // Set up the change event for handling the jump-to functionality - $('#jump-to-input').on('change', (e) => { + $('#jump-to-input').off().on('change', (e) => { if (!e.target.value) return; // If no value is selected, do nothing const columnX = columnCoordinates[e.target.value]; - // Scroll to the selected column position - dh.scrollTo(0, columnX); + //dh.scrollTo(0, columnX); + dh.hot.selectColumns(columnX, columnX) // Hide the modal after the selection is made $('#jump-to-modal').modal('hide'); @@ -1119,6 +1527,44 @@ class Toolbar { }, }; } + +} + + +/** + * Clears out data from a dataharmonizer instance. preserve_settings = + * boolean flag which keeps settings. The "Data > New" menu option clears + * these out. Settings may be reset as well, which may be good + * for reloading of data from a data file? + */ +export function clearDH(dh, preserve_settings = false) { + dh.clearValidationResults(); + dh.current_selection = [null, null, null, null]; + if (preserve_settings) + dh.hot.clear(); // only clears the data + else + dh.hot.updateSettings({data : []}); // Clears all settings in the table as well as data. +}; + +/** + * Using browser "localStorage" to have settings that persist with user. + */ +export function readBrowserDHSettings() { + let dh_settings_yaml = localStorage.getItem('dataharmonizer_settings'); + if (!dh_settings_yaml) { + // Provide basic shell of all DH browser-based settings here + return {'schema' : {}} + } + return YAML.parse(dh_settings_yaml); +} + +export function saveBrowserDHSettings(dh_settings) { + localStorage.setItem('dataharmonizer_settings', YAML.stringify(dh_settings)); +} + +// Option to clear out localStorage 'dataharmonizer_settings' +export function deleteBrowserDHSettings() { + localStorage.removeItem('dataharmonizer_settings'); } export default Toolbar; diff --git a/lib/Validator.js b/lib/Validator.js index 1b61650e..29cc9143 100644 --- a/lib/Validator.js +++ b/lib/Validator.js @@ -264,7 +264,12 @@ class Validator { let splitValues; if (slotDefinition.multivalued) { - splitValues = value.split(this.#multivaluedDelimiter); + if (Array.isArray(value)) {// Handsontable seems to allow arrays for cell values. + splitValues = value; + console.log(`Multivalued slot ${slotDefinition.name} cell data is an array; should it be loaded as a delimited string? See arraygetValidatorForSlot()`, value); + } + else + splitValues = value.split(this.#multivaluedDelimiter); if ( slotDefinition.minimum_cardinality !== undefined && splitValues.length < slotDefinition.minimum_cardinality diff --git a/lib/contentModals.html b/lib/contentModals.html new file mode 100644 index 00000000..a802b3c2 --- /dev/null +++ b/lib/contentModals.html @@ -0,0 +1,308 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lib/data-harmonizer.css b/lib/data-harmonizer.css index befaccd1..3b2c5c6a 100644 --- a/lib/data-harmonizer.css +++ b/lib/data-harmonizer.css @@ -4,18 +4,100 @@ z-index: 1000; } -#specify-headers-comparison-row { - overflow-x: auto; -} - -#specify-headers-err-msg { - display: none; +.secondary-header-text:hover { + background-color: lightblue; } #field-mapping-container { - overflow-x: scroll; + overflow-y: scroll; padding-bottom: 1rem; scrollbar-width: 1rem; + max-height: 500px; +} + +table#field-mapping-table { + border-spacing: 2px; + border-collapse: separate; + width: 100%; + height:auto; + text-align: left; +} +table#field-mapping-table th { + background-color: lightgray; + position: sticky; + top: 0; + padding: 3px 7px; + z-index: 1; +} +tbody.field-mapping-template tr { + background-color:#30D0E0; + font-size: 1.3rem; + font-weight: bold; +} + +table#field-mapping-table tr td { + min-height:30px; +} + +table#field-mapping-table .hidden { + display:none; +} + +#field-mapping-profiles { + background-color:lightgray; + margin-top: 4px; + padding-bottom:5px; +} + +.reordered { + background-color: yellow; +} +td.field-mismatch { + border-radius: 10px; + padding-left: 8px; + background-color: orange; +} + +tbody.field-mapping-section > tr:first-child td { + background-color:#46D0EC; + font-size: 1.1rem; + font-weight: bold; +} +/* +tbody.field-mapping-section { + display: none; +} +tbody.field-mapping-section:has(td.field-mismatch) { + display: revert; +} + +tbody.field-mapping-section:has(td.field-mismatch) > tr:first-child { + background-color:orange !important; +} +*/ +td.draggable-mapping-item { + min-width: 150px; + padding: 3px 8px; + cursor: grab !important; + +} +.ui-draggable-dragging { + background-color: lightblue; + border: 1px solid lightbrown; + opacity: 0.8; + cursor: move !important; + height: 25px; +} +/* +.ui-state-active { + background-color: #ffe0b2; + border-color: orange; +} +*/ + +.ui-state-hover { + background-color: #c8e6c9; /* Lighter green for hover state */ + border-color: green; } .handsontable.listbox th, @@ -23,21 +105,210 @@ white-space: pre; } -#field-mapping { - white-space: nowrap; +/* Styling for current open user picklist editor */ +div.handsontableEditor div .wtSpreader { + width:auto; +} + +div.handsontableEditor {} +div.handsontableEditor.menu-above { + background-color:lightgray; + position:absolute; } -#field-mapping col { - border-left: 2px solid black; +/* TESTING FOR WIDER DROPDOWN MENU WIDTH */ +.handsontable.listbox .ht_master table { + width: auto; } -#field-mapping col:first-child { - background: #ff0; + +.handsontable.listbox { + white-space: pre !important; +} +.handsontable.listbox td { + border-radius: 3px; + border: 1px solid silver; + background-color: #ddd; } -#field-mapping col:nth-child(2n + 3) { - background: #ccc; +.handsontable.listbox td:hover { + background-color: lightblue !important; +} +.handsontable.listbox td.current.highlight { + background-color: lightblue !important; +} + + +.handsontable span.colHeader.columnSorting::before { +/* SVG up down icon, convert to URL encoded for background-images below. + + + + +*/ + +background-image: url("data:image/svg+xml,%3Csvg%20width%3D%2224%22%20height%3D%2224%22%20viewBox%3D%220%200%2024%2024%22%20fill%3D%22none%22%20xmlns%3D%22http%3A%2F%2Fwww.w3.org%2F2000%2Fsvg%22%3E%0A%20%20%3Cpath%20d%3D%22M6%209.65685L7.41421%2011.0711L11.6569%206.82843L15.8995%2011.0711L17.3137%209.65685L11.6569%204L6%209.65685Z%22%0A%20%20%20%20fill%3D%22currentColor%22%2F%3E%0A%20%20%3Cpath%0A%20%20%20%20d%3D%22M6%2014.4433L7.41421%2013.0291L11.6569%2017.2717L15.8995%2013.0291L17.3137%2014.4433L11.6569%2020.1001L6%2014.4433Z%22%0A%20%20%20%20fill%3D%22currentColor%22%2F%3E%0A%20%20%3C%2Fsvg%3E"); + background-size:1.1rem; + right:-10px; + padding-right:10px; + padding-bottom:15px; + } -#field-mapping tr td, -#field-mapping tr th { - padding: 3px; +.handsontable span.colHeader.columnSorting.ascending::before { + /* arrow up */ + background-image: url("data:image/svg+xml,%3Csvg%20width%3D%2224%22%20height%3D%2224%22%20viewBox%3D%220%200%2024%2024%22%20fill%3D%22none%22%20xmlns%3D%22http%3A%2F%2Fwww.w3.org%2F2000%2Fsvg%22%3E%0A%20%20%3Cpath%20d%3D%22M6%209.65685L7.41421%2011.0711L11.6569%206.82843L15.8995%2011.0711L17.3137%209.65685L11.6569%204L6%209.65685Z%22%0A%20%20%20%20fill%3D%22currentColor%22%2F%3E%0A%20%20%3C%2Fsvg%3E"); + top:6px; } + +.handsontable span.colHeader.columnSorting.descending::before { + background-image: url("data:image/svg+xml,%3Csvg%20width%3D%2224%22%20height%3D%2224%22%20viewBox%3D%220%200%2024%2024%22%20fill%3D%22none%22%20xmlns%3D%22http%3A%2F%2Fwww.w3.org%2F2000%2Fsvg%22%3E%0A%20%20%3Cpath%0A%20%20%20%20d%3D%22M6%2014.4433L7.41421%2013.0291L11.6569%2017.2717L15.8995%2013.0291L17.3137%2014.4433L11.6569%2020.1001L6%2014.4433Z%22%0A%20%20%20%20fill%3D%22currentColor%22%2F%3E%0A%3C%2Fsvg%3E"); + top:13px; +} + +/* Removes arrows from section header row as well as any disabled sort columns */ +.handsontable span.colHeader.columnSorting.indicatorDisabled::before { + background-size:0 !important; +} + + +.secondary-header-text { + cursor:pointer !important; + z-index: 10; + margin-right:10px; + margin-top:5px; +} + +.data-harmonizer-grid .secondary-header-cell:hover { + cursor: pointer; +} +.data-harmonizer-grid td.invalid-cell { + background-color: #ffcccb !important; +} +.data-harmonizer-grid td.empty-invalid-cell { + background-color: #ff91a4 !important; +} +.data-harmonizer-grid .htAutocompleteArrow { + color: gray; +} +.data-harmonizer-grid th { + text-align: left; +} +.data-harmonizer-grid th.required { + background-color: yellow !important; +} +.data-harmonizer-grid th.recommended { + background-color: plum !important; +} + + +.handsontable .ht__manualRowMove.after-selection--rows tbody th.ht__highlight, .handsontable.ht__manualRowMove.after-selection--rows tbody th.ht__highlight { + cursor: move; + cursor: -moz-grab; + cursor: -webkit-grab; + cursor: grab; +} +/*.modal.show */ +#translate-modal .modal-dialog { + width:95%; + max-width: 100% !important; + margin:1rem !important; +} +#translate-modal table { + min-width: 50%; + margin-bottom:15px; +} +#translate-modal table th { + background-color:#EEE; + padding:5px; +} +#translate-modal table td { + padding:5px; +} +#translate-modal table th.locale { + width:60px; +} + +#translate-modal table tr td, table tr th { + vertical-align: top; +} +#translate-modal table tr.translate td { + background-color:#EFE; + margin-top:15px; +} +tr.translation-input td textarea { + min-height:3lh; + max-height:10lh; + min-width:200px; + field-sizing: content; +} + +tr.translate_key td { + border-top:1px solid gray; + font-weight:bold; +} + +.handsontable td.tabFieldTd.htDimmed { + /* height:20px !important; + overflow-y:hidden !important; */ + white-space: nowrap; /* Prevent text from wrapping */ + text-overflow: ellipsis; /* Add ellipsis for truncated text */ +} + +.handsontable td.tabFieldTd.htDimmed.current { + white-space: normal; /* Prevent text from wrapping */ + text-overflow: unset; /* Add ellipsis for truncated text */ +} + +/* Handsontable assigns .htDimmed to any cell with .readOnly = true */ +td.tabFieldTd.slot { + border-top:2px solid green; + background-color: #EFE; +} + +td.tabFieldTd.attribute { + border-top:2px solid green; + background-color: lightyellow; +} + +td.tabFieldTd.slot_usage.inherited::before { + position:relative; + content: "as above"; + color: grey; + border:1px solid grey; + background-color: #edffed; + border-radius: 3px; + top:1px; + padding: 2px 5px; +} + +/* Not sure how to display frozen column right side in header area +.data-harmonizer-grid th.overlayEdge { + border-right: 2px solid black !important; +} +*/ + +/* Handsontable class. Marks right side of frozen columns */ +.ht_clone_left { + border-right: 2px solid black !important; +} + +/* Handsontable class. Enables clipped display of frozen column identifier fields */ +.ht_clone_inline_start td { + white-space: nowrap; + text-overflow: ellipsis; +} + +.coding_name {font-size: .8rem} /* Used where slot.name is shown */ + +td.hide { + display:none; +} + +.selectDepth_0 {padding-left:0px !important} +.selectDepth_1 {padding-left:20px !important} +.selectDepth_2 {padding-left:40px !important} +.selectDepth_3 {padding-left:60px !important} +.selectDepth_4 {padding-left:80px !important} +.selectDepth_5 {padding-left:100px !important} \ No newline at end of file diff --git a/lib/editors/FlatpickrEditor.js b/lib/editors/FlatpickrEditor.js index b3f601b9..2175238e 100644 --- a/lib/editors/FlatpickrEditor.js +++ b/lib/editors/FlatpickrEditor.js @@ -18,7 +18,7 @@ class FlatpickrEditor extends Handsontable.editors.TextEditor { init() { super.init(); - this.instance.addHook('afterDestroy', () => { + this.instance?.addHook('afterDestroy', () => { this.parentDestroyed = true; this.destroyElements(); }); diff --git a/lib/editors/KeyValueEditor.js b/lib/editors/KeyValueEditor.js index f38fc1f1..18423116 100644 --- a/lib/editors/KeyValueEditor.js +++ b/lib/editors/KeyValueEditor.js @@ -1,5 +1,6 @@ +import $ from 'jquery'; import Handsontable from 'handsontable'; -import { MULTIVALUED_DELIMITER, titleOverText } from '../utils/fields'; +import { MULTIVALUED_DELIMITER} from '../utils/fields'; import { isEmptyUnitVal } from '../utils/general'; // Derived from: https://jsfiddle.net/handsoncode/f0b41jug/ @@ -13,6 +14,7 @@ export default class KeyValueListEditor extends Handsontable.editors /** * Prepares the editor instance by setting up various options for Handsontable. * + * It appears this instance hangs around for the life of the handsontable! * @param {number} row The row index of the cell being edited. * @param {number} col The column index of the cell being edited. * @param {string} prop The property name or column index of the cell being edited. @@ -20,26 +22,61 @@ export default class KeyValueListEditor extends Handsontable.editors * @param {any} value The value of the cell. * @param {object} cellProperties The properties of the cell. */ + prepare(row, col, prop, td, value, cellProperties) { super.prepare(row, col, prop, td, value, cellProperties); + let self = this; + this.MENU_HEIGHT = 200; + + function filter(event) { + const text = event.srcElement.value.toLowerCase();// word typed so far. + const hide = []; + const show = []; + let count = 0; + self.htOptions.data.forEach((row, index) => { + count += 1; + if (row.label.toLowerCase().includes(text)) // && count < 13 + show.push(index); + else + hide.push(index); + }); + self.hiddenRowsPlugin.showRows(show); + self.hiddenRowsPlugin.hideRows(hide); + self.dropdownHotInstance.render(); + } + + // Setting for pulldown menu display + + // Adding dynamic filter. DOM element textarea.handsontableInput. This is + // relative to event's TEXTAREA since there are other textareas around + // Done as an onkeyup() since its reset each time user visits a table cell. + this.TEXTAREA.onkeyup = filter; + Object.assign(this.htOptions, { licenseKey: 'non-commercial-and-evaluation', data: this.cellProperties.source, - columns: [ - { - data: '_id', - }, - { - data: 'label', - }, - ], - hiddenColumns: { - columns: [1], + rowHeaders: false, + colWidths: 250, + height: this.MENU_HEIGHT, + columns: [{data: '_id'},{data: 'label'}], + hiddenColumns: {columns: [0]}, + hiddenRows: {rows: []}, + /* + renderer: function(instance, td, row, col, prop, value, cellProperties) { + // This custom renderer controls the appearance of each item in the dropdown list + //Handsontable.renderers.TextRenderer.apply(this, arguments); + console.log(instance, td, row, col, prop, value) + td.innerHTML = `⭐ ${value}`; // Add custom HTML, icons, etc. + }, + */ + cells: function(row, col) { + var cellProp = {}; + // cellProperties.prop = 12 (column), cellProperties.row = the row. + cellProp.className = 'selectDepth_' + (cellProperties.source[row]?.depth || '0'); + return cellProp }, - colWidths: 150, beforeValueRender(value, { row, instance }) { - if (instance) { - // const _id = instance.getDataAtRowProp(row, '_id'); + if (instance) { // i.e. an instance that has data: 'label' above? const label = instance.getDataAtRowProp(row, 'label'); return label; } @@ -51,12 +88,36 @@ export default class KeyValueListEditor extends Handsontable.editors this.htOptions.cells = cellProperties.keyValueListCells; } - // HACK: these two lines were in the original code provided, yet appear to result in an error in our code. - // I've commented them out, they don't appear necessary otherwise. The beforeValueRender function works just fine. - // if (this.htEditor) { - // this.htEditor.destroy(); - // } - // this.htEditor = new Handsontable(this.htContainer, this.htOptions); + } + + // Done once each time user clicks on cell and menu is displayed. + focus() { + + super.focus(); + + // Helpers for autocomplete .filter() show/hide of rows: + this.dropdownHotInstance = this.hot.getActiveEditor().htEditor; + this.hiddenRowsPlugin = this.dropdownHotInstance.getPlugin('hiddenRows'); + + // This section is to fix a handsontable bug where autocomplete/ dropdown + // menus don't display if at bottom of a menu. + this.menu = this.TEXTAREA.nextElementSibling; + const menu_height = $(this.menu).height(); + + this.input_top = $(this.TEXTAREA).offset().top; + this.input_height = $(this.TEXTAREA).height(); + const dh_height = $('#data-harmonizer-grid').height(); + const dh_top = $('#data-harmonizer-grid').offset().top; + // Flip menu to top of input area if it would otherwise extend below DH table area. + this.menu_above = (dh_height + dh_top) < (this.input_top + menu_height); + // This is the starting height of the menu. + if (this.menu_above) + $('div.handsontableEditor').css('top', '-' + menu_height + 'px'); + else + $('div.handsontableEditor').css('top', '0px'); + // Imposes table.htCoreposition: fixed in data-harmonizer.css : + $(this.menu).toggleClass('menu-above', this.menu_above); + } /** @@ -101,9 +162,12 @@ export default class KeyValueListEditor extends Handsontable.editors * @param {function} callback A callback function to execute with the result of validation. */ export const keyValueListValidator = function (value, callback) { + // Used AFTER user makes selection in menu. However the DH "Validate" button + // uses other validation. + let valueToValidate = value; - if (valueToValidate === null || valueToValidate === void 0) { + if (valueToValidate === null || valueToValidate === void 0) { // === void 0 ~= undefined valueToValidate = ''; } @@ -112,6 +176,7 @@ export const keyValueListValidator = function (value, callback) { } else { callback(this.source.find(({ _id }) => _id === value) ? true : false); } + }; /** @@ -126,53 +191,41 @@ export const keyValueListValidator = function (value, callback) { * @param {object} cellProperties The properties of the cell. */ export const keyValueListRenderer = function ( - hot, - TD, - row, - col, - prop, - value, - cellProperties -) { + hot, TD, row, col, prop, value, cellProperties) { // Call the autocomplete renderer to ensure default styles and behavior are applied + // RENDERER text that is shown is not validated directly. Handsontable.renderers .getRenderer('autocomplete') .apply(this, [hot, TD, row, col, prop, value, cellProperties]); - const item = cellProperties.source.find(({ _id }) => _id === value); - if (item) { - // Use the label as the display value but keep the _id as the stored value - const label = item.label; - TD.innerHTML = `
    ▼
    ${label}`; // This directly sets what is displayed in the cell - } + const item = cellProperties.source.find(_x => _x._id === value); + TD.innerHTML = `
    ▼
    ${item?.label || value || ''}`; + }; -export const multiKeyValueListRenderer = (field) => { - const merged_permissible_values = field.sources.reduce((acc, source) => { - return Object.assign(acc, field.permissible_values[source]); - }, {}); - return function (hot, TD, row, col, prop, value, cellProperties) { +export const multiKeyValueListRenderer = function (hot, TD, row, col, prop, value, cellProperties) { // Call the autocomplete renderer to ensure default styles and behavior are applied Handsontable.renderers .getRenderer('autocomplete') .apply(this, [hot, TD, row, col, prop, value, cellProperties]); + let label = ''; + // Since multiple values, we must compose the labels for the resulting display. if (!isEmptyUnitVal(value)) { - const label = value - .split(MULTIVALUED_DELIMITER) - .map((key) => { - if (!(key in merged_permissible_values)) - console.warn( - `${key} not in merged_permissible_values ${Object.keys( - merged_permissible_values - )}` - ); - return key in merged_permissible_values - ? titleOverText(merged_permissible_values[key]) - : key; - }) - .join(MULTIVALUED_DELIMITER); - TD.innerHTML = `
    ▼
    ${label}`; // This directly sets what is displayed in the cell + label = value + .split(MULTIVALUED_DELIMITER) + .map((value_item) => { + const choice = cellProperties.source.find(({ _id }) => _id === value_item); + //if (!(choice)) + // console.warn(`"${value_item}" is not in permissible_values for "${cellProperties.name}" slot.`); + return choice ? choice.label : value_item; + }) + .join(MULTIVALUED_DELIMITER); } + + // This directly sets what is displayed in the cell on render() + // Uses the label as the display value but keep the _id as the stored value + TD.innerHTML = `
    ▼
    ${label}`; + //} }; -}; + diff --git a/lib/fieldDescriptionsModal.html b/lib/fieldDescriptionsModal.html deleted file mode 100644 index 539d5b5d..00000000 --- a/lib/fieldDescriptionsModal.html +++ /dev/null @@ -1,24 +0,0 @@ - diff --git a/lib/index.js b/lib/index.js index 367f7ecb..d5dc1e3e 100644 --- a/lib/index.js +++ b/lib/index.js @@ -2,4 +2,4 @@ export { default as Footer } from './Footer'; export { default as Toolbar } from './Toolbar'; export { default as DataHarmonizer } from './DataHarmonizer'; export { default as AppContext } from './AppContext'; -export { DATE_OBJECT, JSON_SCHEMA_FORMAT, INPUT_FORMAT } from './utils/fields'; +/*export { DATE_OBJECT, JSON_SCHEMA_FORMAT, INPUT_FORMAT } from './utils/fields';*/ diff --git a/lib/specifyHeadersModal.html b/lib/specifyHeadersModal.html deleted file mode 100644 index a2b1cf50..00000000 --- a/lib/specifyHeadersModal.html +++ /dev/null @@ -1,62 +0,0 @@ - diff --git a/lib/toolbar.html b/lib/toolbar.html index 317ab25e..fdfe9f1c 100644 --- a/lib/toolbar.html +++ b/lib/toolbar.html @@ -1,5 +1,23 @@
    +
    -
    -
    -
    - Template -
    -
    -
    -
    -
    -
    - -
    - -
    -
    - Language -
    -
    -
    - -
    -
    -
    @@ -261,7 +326,7 @@
    -
    +
    +
    + +
    + +
    +
    + +
    +
    + +
    +
    + + +
    +
    + +
    +
    + Display + + +
    + + + + + +
    + +
    .json
    -
    @@ -851,227 +977,112 @@
    - -