diff --git a/client/components/CandidateTests/CandidateTestPlanRun/queries.js b/client/components/CandidateTests/CandidateTestPlanRun/queries.js index fa056e72d..5c5a066e3 100644 --- a/client/components/CandidateTests/CandidateTestPlanRun/queries.js +++ b/client/components/CandidateTests/CandidateTestPlanRun/queries.js @@ -132,8 +132,8 @@ export const CANDIDATE_REPORTS_QUERY = gql` unexpectedBehaviors { id text - otherUnexpectedBehaviorText } + unexpectedBehaviorNote } } draftTestPlanRuns { diff --git a/client/components/Reports/SummarizeTestPlanReport.jsx b/client/components/Reports/SummarizeTestPlanReport.jsx index 24e6d191c..d9745d528 100644 --- a/client/components/Reports/SummarizeTestPlanReport.jsx +++ b/client/components/Reports/SummarizeTestPlanReport.jsx @@ -274,11 +274,10 @@ SummarizeTestPlanReport.propTypes = { unexpectedBehaviors: PropTypes.arrayOf( PropTypes.shape({ id: PropTypes.string.isRequired, - text: PropTypes.string.isRequired, - otherUnexpectedBehaviorText: - PropTypes.string + text: PropTypes.string.isRequired }).isRequired - ).isRequired + ).isRequired, + unexpectedBehaviorNote: PropTypes.string }).isRequired ).isRequired }).isRequired diff --git a/client/components/Reports/TestPlanResultsTable.jsx b/client/components/Reports/TestPlanResultsTable.jsx index 9966ac8ec..3ab1a842c 100644 --- a/client/components/Reports/TestPlanResultsTable.jsx +++ b/client/components/Reports/TestPlanResultsTable.jsx @@ -116,6 +116,20 @@ const TestPlanResultsTable = ({ test, testResult, tableClassName = '' }) => { ) )} + {scenarioResult.unexpectedBehaviorNote ? ( +
+ Explanation:  + + " + { + scenarioResult.unexpectedBehaviorNote + } + " + +
+ ) : ( + '' + )} ) : ( 'None' diff --git a/client/components/Reports/queries.js b/client/components/Reports/queries.js index f3116d627..cb43b9e41 100644 --- a/client/components/Reports/queries.js +++ b/client/components/Reports/queries.js @@ -119,8 +119,8 @@ export const REPORT_PAGE_QUERY = gql` unexpectedBehaviors { id text - otherUnexpectedBehaviorText } + unexpectedBehaviorNote } } draftTestPlanRuns { diff --git a/client/components/ReviewConflicts/ReviewConflicts.jsx b/client/components/ReviewConflicts/ReviewConflicts.jsx index e62bd928f..7c4b49774 100644 --- a/client/components/ReviewConflicts/ReviewConflicts.jsx +++ b/client/components/ReviewConflicts/ReviewConflicts.jsx @@ -91,18 +91,23 @@ const ReviewConflicts = ({ const { testPlanRun, scenarioResult } = result; let resultFormatted; if (scenarioResult.unexpectedBehaviors.length) { - resultFormatted = scenarioResult.unexpectedBehaviors - .map(({ otherUnexpectedBehaviorText, text }) => { - return `"${otherUnexpectedBehaviorText ?? text}"`; - }) - .join(' and '); + resultFormatted = + 'the unexpected behavior ' + + scenarioResult.unexpectedBehaviors + .map(({ text }) => `"${text.toLowerCase()}"`) + .join(' and '); } else { resultFormatted = 'no unexpected behavior'; } + let noteFormatted = scenarioResult.unexpectedBehaviorNote + ? ` with the explanation ` + + `"${scenarioResult.unexpectedBehaviorNote}"` + : ''; return (
  • Tester {testPlanRun.tester.username} recorded output " - {scenarioResult.output}" and noted {resultFormatted}. + {scenarioResult.output}" and noted  + {resultFormatted + noteFormatted}.
  • ); }); @@ -160,11 +165,10 @@ ReviewConflicts.propTypes = { output: PropTypes.string.isRequired, unexpectedBehaviors: PropTypes.arrayOf( PropTypes.shape({ - text: PropTypes.string.isRequired, - otherUnexpectedBehaviorText: - PropTypes.string + text: PropTypes.string.isRequired }) - ).isRequired + ).isRequired, + unexpectedBehaviorNote: PropTypes.string }), assertionResult: PropTypes.shape({ passed: PropTypes.bool.isRequired, diff --git a/client/components/TestPlanUpdater/queries.js b/client/components/TestPlanUpdater/queries.js index a6d92d169..916249cf2 100644 --- a/client/components/TestPlanUpdater/queries.js +++ b/client/components/TestPlanUpdater/queries.js @@ -83,8 +83,8 @@ export const VERSION_QUERY = gql` } unexpectedBehaviors { id - otherUnexpectedBehaviorText } + unexpectedBehaviorNote } } } diff --git a/client/components/TestRenderer/index.jsx b/client/components/TestRenderer/index.jsx index 7cf911128..f82873f22 100644 --- a/client/components/TestRenderer/index.jsx +++ b/client/components/TestRenderer/index.jsx @@ -309,8 +309,10 @@ const TestRenderer = ({ output, assertionResults, unexpectedBehaviors, + unexpectedBehaviorNote, highlightRequired = false, // atOutput - unexpectedBehaviorHighlightRequired = false + unexpectedBehaviorHighlightRequired = false, + expandUnexpected = false } = scenarioResults[i]; if (output) commands[i].atOutput.value = output; @@ -344,33 +346,34 @@ const TestRenderer = ({ * 5 = OTHER */ const unexpectedBehavior = unexpectedBehaviors[k]; - if (unexpectedBehavior.id === 'EXCESSIVELY_VERBOSE') + if (unexpectedBehavior === 'EXCESSIVELY_VERBOSE') commands[i].unexpected.behaviors[0].checked = true; - if (unexpectedBehavior.id === 'UNEXPECTED_CURSOR_POSITION') + if (unexpectedBehavior === 'UNEXPECTED_CURSOR_POSITION') commands[i].unexpected.behaviors[1].checked = true; - if (unexpectedBehavior.id === 'SLUGGISH') + if (unexpectedBehavior === 'SLUGGISH') commands[i].unexpected.behaviors[2].checked = true; - if (unexpectedBehavior.id === 'AT_CRASHED') + if (unexpectedBehavior === 'AT_CRASHED') commands[i].unexpected.behaviors[3].checked = true; - if (unexpectedBehavior.id === 'BROWSER_CRASHED') + if (unexpectedBehavior === 'BROWSER_CRASHED') commands[i].unexpected.behaviors[4].checked = true; - if (unexpectedBehavior.id === 'OTHER') { + if (unexpectedBehavior === 'OTHER') { commands[i].unexpected.behaviors[5].checked = true; - commands[i].unexpected.behaviors[5].more.value = - unexpectedBehavior.otherUnexpectedBehaviorText; - commands[ - i - ].unexpected.behaviors[5].more.highlightRequired = - unexpectedBehavior.highlightRequired; } } - } else if (unexpectedBehaviors) + } else if (!expandUnexpected) { // but not populated commands[i].unexpected.hasUnexpected = 'doesNotHaveUnexpected'; - else commands[i].unexpected.hasUnexpected = 'notSet'; + } else if (expandUnexpected) { + commands[i].unexpected.hasUnexpected = 'hasUnexpected'; + commands[i].unexpected.expand = true; + } else commands[i].unexpected.hasUnexpected = 'notSet'; commands[i].unexpected.highlightRequired = unexpectedBehaviorHighlightRequired; + + commands[i].unexpected.note = { + value: unexpectedBehaviorNote ?? '' + }; } return { ...state, commands, currentUserAction: 'validateResults' }; @@ -450,12 +453,6 @@ const TestRenderer = ({ const unexpectedError = item.unexpected.highlightRequired; if (unexpectedError) return true; - const { behaviors } = item.unexpected; - const uncheckedBehaviorsMoreError = behaviors.some(item => { - if (item.more) return item.more.highlightRequired; - return false; - }); - if (uncheckedBehaviorsMoreError) return true; return false; }); } @@ -678,6 +675,23 @@ const TestRenderer = ({ ) )} + {!details.unexpectedBehaviors + .note.length ? ( + '' + ) : ( +
  • + Explanation:  + + " + { + details + .unexpectedBehaviors + .note + } + " + +
  • + )} @@ -1112,7 +1126,6 @@ const TestRenderer = ({ checked, focus, description, - more, change } = option; return ( @@ -1126,7 +1139,7 @@ const TestRenderer = ({ description } id={`${description}-${commandIndex}`} - className={`undesirable-${commandIndex}`} + className={`unexpected-${commandIndex}`} tabIndex={ optionIndex === 0 @@ -1137,10 +1150,10 @@ const TestRenderer = ({ isSubmitted && focus } - defaultChecked={ + checked={ checked } - onClick={e => + onChange={e => change( e .target @@ -1156,67 +1169,76 @@ const TestRenderer = ({ }
    - {more && ( -
    - - - more.change( - e - .target - .value - ) - } - disabled={ - !checked - } - /> -
    - )} ); } )} +
    + + + unexpectedBehaviors.failChoice.note.change( + e.target.value + ) + } + /> +
    diff --git a/client/components/TestRun/index.jsx b/client/components/TestRun/index.jsx index 2844aa7d9..fb1887dab 100644 --- a/client/components/TestRun/index.jsx +++ b/client/components/TestRun/index.jsx @@ -329,7 +329,8 @@ const TestRun = () => { const remapScenarioResults = ( rendererState, scenarioResults, - captureHighlightRequired = false + captureHighlightRequired = false, + save = false ) => { let newScenarioResults = []; if (!rendererState || !scenarioResults) { @@ -379,6 +380,7 @@ const TestRun = () => { const { hasUnexpected, behaviors, highlightRequired } = unexpected; if (hasUnexpected === 'hasUnexpected') { unexpectedBehaviors = []; + if (!save) scenarioResult.expandUnexpected = true; /** * 0 = EXCESSIVELY_VERBOSE * 1 = UNEXPECTED_CURSOR_POSITION @@ -391,38 +393,21 @@ const TestRun = () => { const behavior = behaviors[i]; if (behavior.checked) { if (i === 0) - unexpectedBehaviors.push({ - id: 'EXCESSIVELY_VERBOSE' - }); + unexpectedBehaviors.push('EXCESSIVELY_VERBOSE'); if (i === 1) - unexpectedBehaviors.push({ - id: 'UNEXPECTED_CURSOR_POSITION' - }); - if (i === 2) - unexpectedBehaviors.push({ id: 'SLUGGISH' }); - if (i === 3) - unexpectedBehaviors.push({ id: 'AT_CRASHED' }); - if (i === 4) - unexpectedBehaviors.push({ id: 'BROWSER_CRASHED' }); - if (i === 5) { - const moreResult = { - id: 'OTHER', - otherUnexpectedBehaviorText: behavior.more.value - }; unexpectedBehaviors.push( - captureHighlightRequired - ? { - ...moreResult, - highlightRequired: - behavior.more.highlightRequired - } - : moreResult + 'UNEXPECTED_CURSOR_POSITION' ); - } + if (i === 2) unexpectedBehaviors.push('SLUGGISH'); + if (i === 3) unexpectedBehaviors.push('AT_CRASHED'); + if (i === 4) + unexpectedBehaviors.push('BROWSER_CRASHED'); + if (i === 5) unexpectedBehaviors.push('OTHER'); } } - } else if (hasUnexpected === 'doesNotHaveUnexpected') + } else if (hasUnexpected === 'doesNotHaveUnexpected') { unexpectedBehaviors = []; + } // re-assign scenario result due to read only values scenarioResult.output = atOutput.value ? atOutput.value : null; @@ -432,6 +417,8 @@ const TestRun = () => { scenarioResult.unexpectedBehaviors = unexpectedBehaviors ? [...unexpectedBehaviors] : null; + scenarioResult.unexpectedBehaviorNote = + unexpected.note.value === '' ? null : unexpected.note.value; if (captureHighlightRequired) scenarioResult.unexpectedBehaviorHighlightRequired = highlightRequired; @@ -447,8 +434,23 @@ const TestRun = () => { !rendererState || !testResult.scenarioResults || rendererState.commands.length !== testResult.scenarioResults.length - ) - return testResult; + ) { + // Mapping unexpected behaviors to expected TestRenderer downstream format + const scenarioResults = testResult.scenarioResults.map( + scenarioResult => { + return { + ...scenarioResult, + unexpectedBehaviors: scenarioResult.unexpectedBehaviors + ? scenarioResult.unexpectedBehaviors.map( + behavior => behavior.id + ) + : scenarioResult.unexpectedBehaviors + }; + } + ); + + return { ...testResult, scenarioResults }; + } const scenarioResults = remapScenarioResults( rendererState, @@ -477,7 +479,8 @@ const TestRun = () => { const scenarioResults = remapScenarioResults( testRunStateRef.current || recentTestRunStateRef.current, currentTest.testResult?.scenarioResults, - false + false, + true ); await handleSaveOrSubmitTestResultAction( diff --git a/client/components/TestRun/queries.js b/client/components/TestRun/queries.js index ef557d992..473e4d5ad 100644 --- a/client/components/TestRun/queries.js +++ b/client/components/TestRun/queries.js @@ -35,8 +35,8 @@ export const TEST_RUN_PAGE_QUERY = gql` unexpectedBehaviors { id text - otherUnexpectedBehaviorText } + unexpectedBehaviorNote } } testPlanReport { @@ -71,8 +71,8 @@ export const TEST_RUN_PAGE_QUERY = gql` output unexpectedBehaviors { text - otherUnexpectedBehaviorText } + unexpectedBehaviorNote } assertionResult { passed @@ -180,8 +180,8 @@ export const TEST_RUN_PAGE_ANON_QUERY = gql` output unexpectedBehaviors { text - otherUnexpectedBehaviorText } + unexpectedBehaviorNote } assertionResult { passed diff --git a/client/resources/.eslintrc b/client/resources/.eslintrc new file mode 100644 index 000000000..6a13c542e --- /dev/null +++ b/client/resources/.eslintrc @@ -0,0 +1,24 @@ +{ + "env": { + "browser": true, + "es6": true, + "node": true + }, + "extends": [ + "eslint:recommended", + "plugin:prettier/recommended" + ], + "globals": { + "Atomics": "readonly", + "SharedArrayBuffer": "readonly" + }, + "parserOptions": { + "ecmaVersion": 2018, + "sourceType": "module" + }, + "rules": { + "no-dupe-keys": 0, + "no-undef": 0, + "no-unused-vars": 0 + } +} diff --git a/client/resources/.prettierrc b/client/resources/.prettierrc new file mode 100644 index 000000000..b8742b063 --- /dev/null +++ b/client/resources/.prettierrc @@ -0,0 +1,7 @@ +{ + "singleQuote": true, + "printWidth": 100, + "arrowParens": "avoid", + "tabWidth": 2, + "trailingComma": "es5" +} diff --git a/client/resources/aria-at-harness.mjs b/client/resources/aria-at-harness.mjs index 40a6cdfbb..6e99eabb6 100644 --- a/client/resources/aria-at-harness.mjs +++ b/client/resources/aria-at-harness.mjs @@ -1,7 +1,16 @@ -import {element, fragment, property, attribute, className, style, focus, render} from "./vrender.mjs"; -import {userCloseWindow, userOpenWindow, WhitespaceStyleMap} from "./aria-at-test-run.mjs"; -import {TestRunExport, TestRunInputOutput} from "./aria-at-test-io-format.mjs"; -import {TestWindow} from "./aria-at-test-window.mjs"; +import { + element, + fragment, + property, + attribute, + className, + style, + focus, + render, +} from './vrender.mjs'; +import { userCloseWindow, userOpenWindow, WhitespaceStyleMap } from './aria-at-test-run.mjs'; +import { TestRunExport, TestRunInputOutput } from './aria-at-test-io-format.mjs'; +import { TestWindow } from './aria-at-test-window.mjs'; const PAGE_STYLES = ` table { @@ -63,7 +72,7 @@ const PAGE_STYLES = ` let testRunIO = new TestRunInputOutput(); testRunIO.setTitleInputFromTitle(document.title); testRunIO.setUnexpectedInputFromBuiltin(); -testRunIO.setScriptsInputFromMap(typeof scripts === "object" ? scripts : {}); +testRunIO.setScriptsInputFromMap(typeof scripts === 'object' ? scripts : {}); /** * @param {SupportJSON} newSupport @@ -71,7 +80,9 @@ testRunIO.setScriptsInputFromMap(typeof scripts === "object" ? scripts : {}); */ export function initialize(newSupport, newCommandsData) { testRunIO.setSupportInputFromJSON(newSupport); - testRunIO.setConfigInputFromQueryParamsAndSupport(Array.from(new URL(document.location).searchParams)); + testRunIO.setConfigInputFromQueryParamsAndSupport( + Array.from(new URL(document.location).searchParams) + ); testRunIO.setKeysInputFromBuiltinAndConfig(); testRunIO.setCommandsInputFromJSONAndConfigKeys(newCommandsData); } @@ -81,7 +92,7 @@ export function initialize(newSupport, newCommandsData) { */ export function verifyATBehavior(atBehavior) { if (testRunIO.behaviorInput !== null) { - throw new Error("Test files should only contain one verifyATBehavior call."); + throw new Error('Test files should only contain one verifyATBehavior call.'); } testRunIO.setBehaviorInputFromJSONAndCommandsConfigKeysTitleUnexpected(atBehavior); @@ -91,13 +102,16 @@ export async function loadCollectedTestAsync(testRoot, testFileName) { const collectedTestResponse = await fetch(`${testRoot}/${testFileName}`); const collectedTestJson = await collectedTestResponse.json(); await testRunIO.setInputsFromCollectedTestAsync(collectedTestJson, testRoot); - testRunIO.setConfigInputFromQueryParamsAndSupport([['at', collectedTestJson.target.at.key], ...Array.from(new URL(document.location).searchParams)]); + testRunIO.setConfigInputFromQueryParamsAndSupport([ + ['at', collectedTestJson.target.at.key], + ...Array.from(new URL(document.location).searchParams), + ]); displayInstructionsForBehaviorTest(); } export function displayTestPageAndInstructions(testPage) { - if (document.readyState !== "complete") { + if (document.readyState !== 'complete') { window.setTimeout(() => { displayTestPageAndInstructions(testPage); }, 100); @@ -106,8 +120,8 @@ export function displayTestPageAndInstructions(testPage) { testRunIO.setPageUriInputFromPageUri(testPage); - document.querySelector("html").setAttribute("lang", "en"); - var style = document.createElement("style"); + document.querySelector('html').setAttribute('lang', 'en'); + var style = document.createElement('style'); style.innerHTML = PAGE_STYLES; document.head.appendChild(style); @@ -152,20 +166,20 @@ function displayInstructionsForBehaviorTest() { if (window.parent && window.parent.postMessage) { // results can be submitted by parent posting a message to the // iFrame with a data.type property of 'submit' - window.addEventListener("message", function (message) { - if (!validateMessage(message, "submit")) return; + window.addEventListener('message', function (message) { + if (!validateMessage(message, 'submit')) return; app.hooks.submit(); }); // send message to parent that test has loaded window.parent.postMessage( { - type: "loaded", + type: 'loaded', data: { testPageUri: windowManager.pageUri, }, }, - "*" + '*' ); } } @@ -174,7 +188,7 @@ function validateMessage(message, type) { if (window.location.origin !== message.origin) { return false; } - if (!message.data || typeof message.data !== "object") { + if (!message.data || typeof message.data !== 'object') { return false; } if (message.data.type !== type) { @@ -191,10 +205,10 @@ function postResults(resultsJSON) { if (window.parent && window.parent.postMessage) { window.parent.postMessage( { - type: "results", + type: 'results', data: resultsJSON, }, - "*" + '*' ); } } @@ -203,61 +217,61 @@ function bind(fn, ...args) { return (...moreArgs) => fn(...args, ...moreArgs); } -const a = bind(element, "a"); -const br = bind(element, "br"); -const button = bind(element, "button"); -const div = bind(element, "div"); -const em = bind(element, "em"); -const fieldset = bind(element, "fieldset"); -const h1 = bind(element, "h1"); -const h2 = bind(element, "h2"); -const h3 = bind(element, "h3"); -const hr = bind(element, "hr"); -const input = bind(element, "input"); -const label = bind(element, "label"); -const legend = bind(element, "legend"); -const li = bind(element, "li"); -const ol = bind(element, "ol"); -const p = bind(element, "p"); -const script = bind(element, "script"); -const section = bind(element, "section"); -const span = bind(element, "span"); -const table = bind(element, "table"); -const td = bind(element, "td"); -const textarea = bind(element, "textarea"); -const th = bind(element, "th"); -const tr = bind(element, "tr"); -const ul = bind(element, "ul"); - -const forInput = bind(attribute, "for"); -const href = bind(attribute, "href"); -const id = bind(attribute, "id"); -const name = bind(attribute, "name"); -const tabIndex = bind(attribute, "tabindex"); -const textContent = bind(attribute, "textContent"); -const type = bind(attribute, "type"); - -const value = bind(property, "value"); -const checked = bind(property, "checked"); -const disabled = bind(property, "disabled"); +const a = bind(element, 'a'); +const br = bind(element, 'br'); +const button = bind(element, 'button'); +const div = bind(element, 'div'); +const em = bind(element, 'em'); +const fieldset = bind(element, 'fieldset'); +const h1 = bind(element, 'h1'); +const h2 = bind(element, 'h2'); +const h3 = bind(element, 'h3'); +const hr = bind(element, 'hr'); +const input = bind(element, 'input'); +const label = bind(element, 'label'); +const legend = bind(element, 'legend'); +const li = bind(element, 'li'); +const ol = bind(element, 'ol'); +const p = bind(element, 'p'); +const script = bind(element, 'script'); +const section = bind(element, 'section'); +const span = bind(element, 'span'); +const table = bind(element, 'table'); +const td = bind(element, 'td'); +const textarea = bind(element, 'textarea'); +const th = bind(element, 'th'); +const tr = bind(element, 'tr'); +const ul = bind(element, 'ul'); + +const forInput = bind(attribute, 'for'); +const href = bind(attribute, 'href'); +const id = bind(attribute, 'id'); +const name = bind(attribute, 'name'); +const tabIndex = bind(attribute, 'tabindex'); +const textContent = bind(attribute, 'textContent'); +const type = bind(attribute, 'type'); + +const value = bind(property, 'value'); +const checked = bind(property, 'checked'); +const disabled = bind(property, 'disabled'); /** @type {(cb: (ev: MouseEvent) => void) => any} */ -const onclick = bind(property, "onclick"); +const onclick = bind(property, 'onclick'); /** @type {(cb: (ev: InputEvent) => void) => any} */ -const onchange = bind(property, "onchange"); +const onchange = bind(property, 'onchange'); /** @type {(cb: (ev: KeyboardEvent) => void) => any} */ -const onkeydown = bind(property, "onkeydown"); +const onkeydown = bind(property, 'onkeydown'); /** * @param {Description} value */ function rich(value) { - if (typeof value === "string") { + if (typeof value === 'string') { return value; } else if (Array.isArray(value)) { return fragment(...value.map(rich)); } else { - if ("whitespace" in value) { + if ('whitespace' in value) { if (value.whitespace === WhitespaceStyleMap.LINE_BREAK) { return br(); } @@ -265,9 +279,9 @@ function rich(value) { } return (value.href ? a.bind(null, href(value.href)) : span)( className([ - value.offScreen ? "off-screen" : "", - value.required ? "required" : "", - value.highlightRequired ? "highlight-required" : "", + value.offScreen ? 'off-screen' : '', + value.required ? 'required' : '', + value.highlightRequired ? 'highlight-required' : '', ]), rich(value.description) ); @@ -279,22 +293,28 @@ function rich(value) { */ function renderVirtualTestPage(doc) { return fragment( - "instructions" in doc + 'instructions' in doc ? div( section( - id("errors"), - style({display: doc.errors && doc.errors.visible ? "block" : "none"}), + id('errors'), + style({ display: doc.errors && doc.errors.visible ? 'block' : 'none' }), h2(doc.errors ? doc.errors.header : ''), - ul(...(doc.errors && doc.errors.errors ? doc.errors.errors.map(error => li(error)) : [])), + ul( + ...(doc.errors && doc.errors.errors ? doc.errors.errors.map(error => li(error)) : []) + ), hr() ), - section(id("instructions"), renderVirtualInstructionDocument(doc.instructions)), - section(id("record-results")) + section(id('instructions'), renderVirtualInstructionDocument(doc.instructions)), + section(id('record-results')) ) : null, - "results" in doc ? renderVirtualResultsTable(doc.results) : null, + 'results' in doc ? renderVirtualResultsTable(doc.results) : null, doc.resultsJSON - ? script(type("text/json"), id("__ariaatharness__results__"), textContent(JSON.stringify(doc.resultsJSON))) + ? script( + type('text/json'), + id('__ariaatharness__results__'), + textContent(JSON.stringify(doc.resultsJSON)) + ) : null ); } @@ -332,7 +352,7 @@ function renderVirtualInstructionDocument(doc) { /** * @param {InstructionDocumentResultsHeader} param0 */ - function resultHeader({header, description}) { + function resultHeader({ header, description }) { return fragment(h2(rich(header)), p(rich(description))); } @@ -348,7 +368,9 @@ function renderVirtualInstructionDocument(doc) { textarea( value(command.atOutput.value), focus(command.atOutput.focus), - onchange(ev => command.atOutput.change(/** @type {HTMLInputElement} */ (ev.currentTarget).value)) + onchange(ev => + command.atOutput.change(/** @type {HTMLInputElement} */ (ev.currentTarget).value) + ) ) ), table( @@ -371,24 +393,38 @@ function renderVirtualInstructionDocument(doc) { return fieldset( id(`cmd-${commandIndex}-problem`), rich(unexpected.description), - div(radioChoice(`problem-${commandIndex}-true`, `problem-${commandIndex}`, unexpected.passChoice)), - div(radioChoice(`problem-${commandIndex}-false`, `problem-${commandIndex}`, unexpected.failChoice)), + div( + radioChoice( + `problem-${commandIndex}-true`, + `problem-${commandIndex}`, + unexpected.passChoice + ) + ), + div( + radioChoice( + `problem-${commandIndex}-false`, + `problem-${commandIndex}`, + unexpected.failChoice + ) + ), fieldset( - className(["problem-select"]), + className(['problem-select']), id(`cmd-${commandIndex}-problem-checkboxes`), legend(rich(unexpected.failChoice.options.header)), ...unexpected.failChoice.options.options.map(failOption => fragment( input( - type("checkbox"), + type('checkbox'), value(failOption.description), id(`${failOption.description}-${commandIndex}`), - className([`undesirable-${commandIndex}`]), - tabIndex(failOption.tabbable ? "0" : "-1"), + className([`unexpected-${commandIndex}`]), + tabIndex(failOption.tabbable ? '0' : '-1'), disabled(!failOption.enabled), checked(failOption.checked), focus(failOption.focus), - onchange(ev => failOption.change(/** @type {HTMLInputElement} */ (ev.currentTarget).checked)), + onchange(ev => + failOption.change(/** @type {HTMLInputElement} */ (ev.currentTarget).checked) + ), onkeydown(ev => { if (failOption.keydown(ev.key)) { ev.stopPropagation(); @@ -396,22 +432,29 @@ function renderVirtualInstructionDocument(doc) { } }) ), - label(forInput(`${failOption.description}-${commandIndex}`), rich(failOption.description)), - br(), - failOption.more - ? div( - label(forInput(`${failOption.description}-${commandIndex}-input`), rich(failOption.more.description)), - input( - type("text"), - id(`${failOption.description}-${commandIndex}-input`), - name(`${failOption.description}-${commandIndex}-input`), - className(["undesirable-other-input"]), - disabled(!failOption.more.enabled), - value(failOption.more.value), - onchange(ev => failOption.more.change(/** @type {HTMLInputElement} */ (ev.currentTarget).value)) - ) + label( + forInput(`${failOption.description}-${commandIndex}`), + rich(failOption.description) + ), + br() + ) + ), + fragment( + div( + label(forInput('unexpected-behavior-note'), rich('Add an explanation')), + input( + type('text'), + id('unexpected-behavior-note'), + name('unexpected-behavior-note'), + className(['unexpected-behavior-note']), + value(unexpected.failChoice.note.value), + disabled(!unexpected.failChoice.note.enabled), + onchange(ev => + unexpected.failChoice.note.change( + /** @type {HTMLInputElement} */ (ev.currentTarget).value ) - : fragment() + ) + ) ) ) ) @@ -428,13 +471,17 @@ function renderVirtualInstructionDocument(doc) { td(rich(assertion.description)), td( ...[assertion.passChoice].map(choice => - radioChoice(`pass-${commandIndex}-${assertionIndex}`, `result-${commandIndex}-${assertionIndex}`, choice) + radioChoice( + `pass-${commandIndex}-${assertionIndex}`, + `result-${commandIndex}-${assertionIndex}`, + choice + ) ) ), td( ...assertion.failChoices.map((choice, failIndex) => radioChoice( - `${failIndex === 0 ? "missing" : "fail"}-${commandIndex}-${assertionIndex}`, + `${failIndex === 0 ? 'missing' : 'fail'}-${commandIndex}-${assertionIndex}`, `result-${commandIndex}-${assertionIndex}`, choice ) @@ -451,7 +498,7 @@ function renderVirtualInstructionDocument(doc) { function radioChoice(idKey, nameKey, choice) { return fragment( input( - type("radio"), + type('radio'), id(idKey), name(nameKey), checked(choice.checked), @@ -466,7 +513,12 @@ function renderVirtualInstructionDocument(doc) { * @param {InstructionDocumentInstructionsInstructions} param0 * @returns */ - function instructCommands({header, instructions, strongInstructions: boldInstructions, commands}) { + function instructCommands({ + header, + instructions, + strongInstructions: boldInstructions, + commands, + }) { return fragment( h2(rich(header)), ol( @@ -480,9 +532,9 @@ function renderVirtualInstructionDocument(doc) { /** * @param {InstructionDocumentInstructions} param0 */ - function instructionHeader({header, description}) { + function instructionHeader({ header, description }) { return fragment( - h1(id("behavior-header"), tabIndex("0"), focus(header.focus), rich(header.header)), + h1(id('behavior-header'), tabIndex('0'), focus(header.focus), rich(header.header)), p(rich(description)) ); } @@ -490,8 +542,12 @@ function renderVirtualInstructionDocument(doc) { /** * @param {InstructionDocumentInstructionsAssertions} param0 */ - function instructAssertions({header, description, assertions}) { - return fragment(h2(rich(header)), p(rich(description)), ol(...map(assertions, compose(li, em, rich)))); + function instructAssertions({ header, description, assertions }) { + return fragment( + h2(rich(header)), + p(rich(description)), + ol(...map(assertions, compose(li, em, rich))) + ); } } @@ -501,12 +557,18 @@ function renderVirtualInstructionDocument(doc) { function renderVirtualResultsTable(results) { return fragment( h1(rich(results.header)), - h2(id("overallstatus"), rich(results.status.header)), + h2(id('overallstatus'), rich(results.status.header)), table( - (({description, support, details}) => tr(th(description), th(support), th(details)))(results.table.headers), + (({ description, support, details }) => tr(th(description), th(support), th(details)))( + results.table.headers + ), results.table.commands.map( - ({description, support, details: {output, passingAssertions, failingAssertions, unexpectedBehaviors}}) => + ({ + description, + support, + details: { output, passingAssertions, failingAssertions, unexpectedBehaviors }, + }) => fragment( tr( td(rich(description)), @@ -527,9 +589,20 @@ function renderVirtualResultsTable(results) { * @param {object} list * @param {Description} list.description * @param {Description[]} list.items + * @param {String} [list.note] */ - function commandDetailsList({description, items}) { - return div(description, ul(...items.map(description => li(rich(description))))); + function commandDetailsList({ + description, + items, + note: { value: noteValue = '', description: noteDescription } = {}, + }) { + return div( + description, + ul( + ...items.map(description => li(rich(description))), + noteValue.length ? li(rich(noteDescription), ' ', em(noteValue)) : fragment() + ) + ); } } diff --git a/client/resources/aria-at-test-io-format.mjs b/client/resources/aria-at-test-io-format.mjs index fc789f803..19be71e3a 100644 --- a/client/resources/aria-at-test-io-format.mjs +++ b/client/resources/aria-at-test-io-format.mjs @@ -9,15 +9,15 @@ import { AssertionResultMap, UserActionMap, CommonResultMap, -} from "./aria-at-test-run.mjs"; -import * as keysModule from "./keys.mjs"; +} from './aria-at-test-run.mjs'; +import * as keysModule from './keys.mjs'; const UNEXPECTED_BEHAVIORS = [ - "Output is excessively verbose, e.g., includes redundant and/or irrelevant speech", - "Reading cursor position changed in an unexpected manner", - "Screen reader became extremely sluggish", - "Screen reader crashed", - "Browser crashed", + 'Output is excessively verbose, e.g., includes redundant and/or irrelevant speech', + 'Reading cursor position changed in an unexpected manner', + 'Screen reader became extremely sluggish', + 'Screen reader crashed', + 'Browser crashed', ]; /** Depends on ConfigInput. */ @@ -56,25 +56,25 @@ class KeysInput { if (this._value.modeInstructions[atMode]) { return this._value.modeInstructions[atMode]; } - return ""; + return ''; } /** * @param {object} data * @param {ConfigInput} data.configInput */ - static fromBuiltinAndConfig({configInput}) { + static fromBuiltinAndConfig({ configInput }) { const keys = keysModule; const atKey = configInput.at().key; invariant( - ["jaws", "nvda", "voiceover_macos"].includes(atKey), + ['jaws', 'nvda', 'voiceover_macos'].includes(atKey), '%s is one of "jaws", "nvda", or "voiceover_macos"', atKey ); return new KeysInput({ - origin: "resources/keys.mjs", + origin: 'resources/keys.mjs', keys, at: atKey, modeInstructions: { @@ -130,7 +130,7 @@ class SupportInput { */ findAT(atKey) { const lowercaseATKey = atKey.toLowerCase(); - return this._value.ats.find(({key}) => key === lowercaseATKey); + return this._value.ats.find(({ key }) => key === lowercaseATKey); } /** @@ -145,7 +145,7 @@ class SupportInput { */ static fromCollectedTest(collectedTest) { return new SupportInput({ - ats: [{key: collectedTest.target.at.key, name: collectedTest.target.at.name}], + ats: [{ key: collectedTest.target.at.key, name: collectedTest.target.at.name }], applies_to: {}, examples: [], }); @@ -180,7 +180,9 @@ class CommandsInput { const assistiveTech = this._value.at; if (!this._value.commands[task]) { - throw new Error(`Task "${task}" does not exist, please add to at-commands or correct your spelling.`); + throw new Error( + `Task "${task}" does not exist, please add to at-commands or correct your spelling.` + ); } else if (!this._value.commands[task][atMode]) { throw new Error( `Mode "${atMode}" instructions for task "${task}" does not exist, please add to at-commands or correct your spelling.` @@ -192,10 +194,10 @@ class CommandsInput { for (let c of commandsData) { let innerCommands = []; - let commandSequence = c[0].split(","); + let commandSequence = c[0].split(','); for (let command of commandSequence) { command = this._keysInput.keysForCommand(command); - if (typeof command === "undefined") { + if (typeof command === 'undefined') { throw new Error( `Key instruction identifier "${c}" for AT "${assistiveTech.name}", mode "${atMode}", task "${task}" is not an available identified. Update you commands.json file to the correct identifier or add your identifier to resources/keys.mjs.` ); @@ -205,7 +207,7 @@ class CommandsInput { command = furtherInstruction ? `${command} ${furtherInstruction}` : command; innerCommands.push(command); } - commands.push(innerCommands.join(", then ")); + commands.push(innerCommands.join(', then ')); } return commands; @@ -217,8 +219,8 @@ class CommandsInput { * @param {ConfigInput} data.configInput * @param {KeysInput} data.keysInput */ - static fromJSONAndConfigKeys(json, {configInput, keysInput}) { - return new CommandsInput({commands: json, at: configInput.at()}, keysInput); + static fromJSONAndConfigKeys(json, { configInput, keysInput }) { + return new CommandsInput({ commands: json, at: configInput.at() }, keysInput); } /** @@ -226,13 +228,14 @@ class CommandsInput { * @param {object} data * @param {KeysInput} data.keysInput */ - static fromCollectedTestKeys(collectedTest, {keysInput}) { + static fromCollectedTestKeys(collectedTest, { keysInput }) { return new CommandsInput( { commands: { [collectedTest.info.task]: { [collectedTest.target.mode]: { - [collectedTest.target.at.key]: collectedTest.commands.map(({id, extraInstruction}) => + [collectedTest.target.at + .key]: collectedTest.commands.map(({ id, extraInstruction }) => extraInstruction ? [id, extraInstruction] : [id] ), }, @@ -291,17 +294,17 @@ class ConfigInput { * @param {object} data * @param {SupportInput} data.supportInput */ - static fromQueryParamsAndSupport(queryParams, {supportInput}) { + static fromQueryParamsAndSupport(queryParams, { supportInput }) { const errors = []; let at = supportInput.defaultAT(); let displaySubmitButton = true; let renderResultsAfterSubmit = true; - let resultFormat = "SubmitResultsJSON"; + let resultFormat = 'SubmitResultsJSON'; let resultJSON = null; for (const [key, value] of queryParams) { - if (key === "at") { + if (key === 'at') { const requestedAT = value; const knownAt = supportInput.findAT(requestedAT); if (knownAt) { @@ -311,17 +314,19 @@ class ConfigInput { `Harness does not have commands for the requested assistive technology ('${requestedAT}'), showing commands for assistive technology '${at.name}' instead. To test '${requestedAT}', please contribute command mappings to this project.` ); } - } else if (key === "showResults") { + } else if (key === 'showResults') { displaySubmitButton = decodeBooleanParam(value, displaySubmitButton); - } else if (key === "showSubmitButton") { + } else if (key === 'showSubmitButton') { renderResultsAfterSubmit = decodeBooleanParam(value, renderResultsAfterSubmit); - } else if (key === "resultFormat") { - if (value !== "SubmitResultsJSON" && value !== "TestResultJSON") { - errors.push(`resultFormat can be 'SubmitResultsJSON' or 'TestResultJSON'. '${value}' is not supported.`); + } else if (key === 'resultFormat') { + if (value !== 'SubmitResultsJSON' && value !== 'TestResultJSON') { + errors.push( + `resultFormat can be 'SubmitResultsJSON' or 'TestResultJSON'. '${value}' is not supported.` + ); continue; } resultFormat = value; - } else if (key === "resultJSON") { + } else if (key === 'resultJSON') { try { resultJSON = JSON.parse(value); } catch (error) { @@ -330,7 +335,7 @@ class ConfigInput { } } - if (resultJSON && resultFormat !== "TestResultJSON") { + if (resultJSON && resultFormat !== 'TestResultJSON') { errors.push(`resultJSON requires resultFormat to be set to 'TestResultJSON'.`); resultJSON = null; } @@ -349,9 +354,9 @@ class ConfigInput { * @returns {boolean} */ function decodeBooleanParam(param, defaultValue) { - if (param === "true") { + if (param === 'true') { return true; - } else if (param === "false") { + } else if (param === 'false') { return false; } return defaultValue; @@ -380,7 +385,7 @@ class ScriptsInput { * @param {SetupScripts} scripts */ static fromScriptsMap(scripts) { - return new ScriptsInput({scripts}); + return new ScriptsInput({ scripts }); } /** @@ -388,7 +393,7 @@ class ScriptsInput { * @private */ static scriptsFromSource(script) { - return {[script.name]: new Function("testPageDocument", script.source)}; + return { [script.name]: new Function('testPageDocument', script.source) }; } /** @@ -409,11 +414,13 @@ class ScriptsInput { return await Promise.race([ new Promise(resolve => { window.scriptsJsonpLoaded = resolve; - const scriptTag = document.createElement("script"); + const scriptTag = document.createElement('script'); scriptTag.src = script.jsonpPath; document.body.appendChild(scriptTag); }), - new Promise((_, reject) => setTimeout(() => reject(new Error("Loading scripts timeout error")), 10000)), + new Promise((_, reject) => + setTimeout(() => reject(new Error('Loading scripts timeout error')), 10000) + ), ]); } @@ -421,20 +428,26 @@ class ScriptsInput { * @param {AriaATFile.CollectedTest} collectedAsync * @param {string} dataUrl url to directory where CollectedTest was loaded from */ - static async fromCollectedTestAsync({target: {setupScript}}, dataUrl) { + static async fromCollectedTestAsync({ target: { setupScript } }, dataUrl) { if (!setupScript) { - return new ScriptsInput({scripts: {}}); + return new ScriptsInput({ scripts: {} }); } try { - return new ScriptsInput({scripts: ScriptsInput.scriptsFromSource(setupScript)}); + return new ScriptsInput({ scripts: ScriptsInput.scriptsFromSource(setupScript) }); } catch (error) { try { - return new ScriptsInput({scripts: await ScriptsInput.scriptsFromModuleAsync(setupScript, dataUrl)}); + return new ScriptsInput({ + scripts: await ScriptsInput.scriptsFromModuleAsync(setupScript, dataUrl), + }); } catch (error2) { try { - return new ScriptsInput({scripts: await ScriptsInput.scriptsFromJsonpAsync(setupScript, dataUrl)}); + return new ScriptsInput({ + scripts: await ScriptsInput.scriptsFromJsonpAsync(setupScript, dataUrl), + }); } catch (error3) { - throw new Error([error, error2, error3].map(error => error.stack || error.message).join("\n\n")); + throw new Error( + [error, error2, error3].map(error => error.stack || error.message).join('\n\n') + ); } } } @@ -460,8 +473,8 @@ class UnexpectedInput { static fromBuiltin() { return new UnexpectedInput({ behaviors: [ - ...UNEXPECTED_BEHAVIORS.map(description => ({description, requireExplanation: false})), - {description: "Other", requireExplanation: true}, + ...UNEXPECTED_BEHAVIORS.map(description => ({ description, requireExplanation: false })), + { description: 'Other', requireExplanation: true }, ], }); } @@ -521,7 +534,7 @@ class BehaviorInput { */ static fromJSONCommandsConfigKeysTitleUnexpected( json, - {commandsInput, configInput, keysInput, titleInput, unexpectedInput} + { commandsInput, configInput, keysInput, titleInput, unexpectedInput } ) { const mode = Array.isArray(json.mode) ? json.mode[0] : json.mode; const at = configInput.at(); @@ -541,12 +554,13 @@ class BehaviorInput { priority: Number(assertionTuple[0]), assertion: assertionTuple[1], })), - additionalAssertions: (json.additional_assertions ? json.additional_assertions[at.key] || [] : []).map( - assertionTuple => ({ - priority: Number(assertionTuple[0]), - assertion: assertionTuple[1], - }) - ), + additionalAssertions: (json.additional_assertions + ? json.additional_assertions[at.key] || [] + : [] + ).map(assertionTuple => ({ + priority: Number(assertionTuple[0]), + assertion: assertionTuple[1], + })), unexpectedBehaviors: unexpectedInput.behaviors(), }, }); @@ -560,8 +574,8 @@ class BehaviorInput { * @param {UnexpectedInput} data.unexpectedInput */ static fromCollectedTestCommandsKeysUnexpected( - {info, target, instructions, assertions}, - {commandsInput, keysInput, unexpectedInput} + { info, target, instructions, assertions }, + { commandsInput, keysInput, unexpectedInput } ) { return new BehaviorInput({ behavior: { @@ -574,7 +588,7 @@ class BehaviorInput { setupScriptDescription: target.setupScript ? target.setupScript.description : '', setupTestPage: target.setupScript ? target.setupScript.name : undefined, commands: commandsInput.getCommands(info.task, target.mode), - assertions: assertions.map(({priority, expectation: assertion}) => ({ + assertions: assertions.map(({ priority, expectation: assertion }) => ({ priority, assertion, })), @@ -604,7 +618,7 @@ class PageUriInput { * @param {string} pageUri */ static fromPageUri(pageUri) { - return new PageUriInput({pageUri}); + return new PageUriInput({ pageUri }); } } @@ -639,35 +653,35 @@ export class TestRunInputOutput { setBehaviorInputFromJSONAndCommandsConfigKeysTitleUnexpected(behaviorJSON) { invariant( this.commandsInput !== null, - "Call %s or %s before calling %s.", + 'Call %s or %s before calling %s.', this.setCommandsInput.name, this.setCommandsInputFromJSONAndConfigKeys.name, this.setBehaviorInputFromJSONAndCommandsConfigKeysTitleUnexpected.name ); invariant( this.configInput !== null, - "Call %s or %s before calling %s.", + 'Call %s or %s before calling %s.', this.setConfigInput.name, this.setConfigInputFromQueryParamsAndSupport.name, this.setBehaviorInputFromJSONAndCommandsConfigKeysTitleUnexpected.name ); invariant( this.keysInput !== null, - "Call %s or %s before calling %s.", + 'Call %s or %s before calling %s.', this.setKeysInput.name, this.setKeysInputFromBuiltinAndConfig.name, this.setBehaviorInputFromJSONAndCommandsConfigKeysTitleUnexpected.name ); invariant( this.titleInput !== null, - "Call %s or %s before calling %s.", + 'Call %s or %s before calling %s.', this.setTitleInput.name, this.setTitleInputFromTitle.name, this.setBehaviorInputFromJSONAndCommandsConfigKeysTitleUnexpected.name ); invariant( this.unexpectedInput !== null, - "Call %s or %s before calling %s.", + 'Call %s or %s before calling %s.', this.setUnexpectedInput.name, this.setUnexpectedInputFromBuiltin.name, this.setBehaviorInputFromJSONAndCommandsConfigKeysTitleUnexpected.name @@ -697,7 +711,7 @@ export class TestRunInputOutput { const unexpectedInput = UnexpectedInput.fromBuiltin(); const keysInput = KeysInput.fromCollectedTest(collectedTest); - const commandsInput = CommandsInput.fromCollectedTestKeys(collectedTest, {keysInput}); + const commandsInput = CommandsInput.fromCollectedTestKeys(collectedTest, { keysInput }); const behaviorInput = BehaviorInput.fromCollectedTestCommandsKeysUnexpected(collectedTest, { commandsInput, keysInput, @@ -724,14 +738,14 @@ export class TestRunInputOutput { setCommandsInputFromJSONAndConfigKeys(commandsJSON) { invariant( this.configInput !== null, - "Call %s or %s before calling %s.", + 'Call %s or %s before calling %s.', this.setConfigInput.name, this.setConfigInputFromQueryParamsAndSupport.name, this.setCommandsInputFromJSONAndConfigKeys.name ); invariant( this.keysInput !== null, - "Call %s or %s before calling %s.", + 'Call %s or %s before calling %s.', this.setKeysInput.name, this.setKeysInputFromBuiltinAndConfig.name, this.setCommandsInputFromJSONAndConfigKeys.name @@ -754,7 +768,7 @@ export class TestRunInputOutput { setConfigInputFromQueryParamsAndSupport(queryParams) { invariant( this.supportInput !== null, - "Call %s or %s before calling %s.", + 'Call %s or %s before calling %s.', this.setSupportInput.name, this.setSupportInputFromJSON.name, this.setConfigInputFromQueryParamsAndSupport.name @@ -775,13 +789,13 @@ export class TestRunInputOutput { setKeysInputFromBuiltinAndConfig() { invariant( this.configInput !== null, - "Call %s or %s before calling %s.", + 'Call %s or %s before calling %s.', this.setConfigInput.name, this.setConfigInputFromQueryParamsAndSupport.name, this.setCommandsInputFromJSONAndConfigKeys.name ); - this.setKeysInput(KeysInput.fromBuiltinAndConfig({configInput: this.configInput})); + this.setKeysInput(KeysInput.fromBuiltinAndConfig({ configInput: this.configInput })); } /** @param {PageUriInput} pageUriInput */ @@ -837,20 +851,24 @@ export class TestRunInputOutput { testRunState() { invariant( this.behaviorInput !== null, - "Call %s or %s before calling %s.", + 'Call %s or %s before calling %s.', this.setBehaviorInput.name, this.setBehaviorInputFromJSONAndCommandsConfigKeysTitleUnexpected.name, this.testRunState.name ); invariant( this.configInput !== null, - "Call %s or %s before calling %s.", + 'Call %s or %s before calling %s.', this.setConfigInput.name, this.setConfigInputFromQueryParamsAndSupport.name, this.testRunState.name ); - const errors = [...this.behaviorInput.errors, ...this.commandsInput.errors, ...this.configInput.errors]; + const errors = [ + ...this.behaviorInput.errors, + ...this.commandsInput.errors, + ...this.configInput.errors, + ]; const test = this.behaviorInput.behavior(); const config = this.configInput; @@ -861,7 +879,7 @@ export class TestRunInputOutput { task: test.task, mode: test.mode, modeInstructions: test.modeInstructions, - userInstructions: test.specificUserInstruction.split("|"), + userInstructions: test.specificUserInstruction.split('|'), setupScriptDescription: test.setupScriptDescription, }, config: { @@ -874,36 +892,39 @@ export class TestRunInputOutput { enabled: true, }, commands: test.commands.map( - command => - /** @type {import("./aria-at-test-run.mjs").TestRunCommand} */ ({ - description: command, - atOutput: { - highlightRequired: false, - value: "", - }, - assertions: test.assertions.map(assertion => ({ - description: assertion.assertion, - highlightRequired: false, - priority: assertion.priority, - result: CommonResultMap.NOT_SET, - })), - additionalAssertions: test.additionalAssertions.map(assertion => ({ - description: assertion.assertion, - highlightRequired: false, - priority: assertion.priority, - result: CommonResultMap.NOT_SET, + command => /** @type {import("./aria-at-test-run.mjs").TestRunCommand} */ ({ + description: command, + atOutput: { + highlightRequired: false, + value: '', + }, + assertions: test.assertions.map(assertion => ({ + description: assertion.assertion, + highlightRequired: false, + priority: assertion.priority, + result: CommonResultMap.NOT_SET, + })), + additionalAssertions: test.additionalAssertions.map(assertion => ({ + description: assertion.assertion, + highlightRequired: false, + priority: assertion.priority, + result: CommonResultMap.NOT_SET, + })), + unexpected: { + highlightRequired: false, + hasUnexpected: HasUnexpectedBehaviorMap.NOT_SET, + tabbedBehavior: 0, + behaviors: test.unexpectedBehaviors.map(({ description, requireExplanation }) => ({ + description, + checked: false, + requireExplanation, })), - unexpected: { + note: { highlightRequired: false, - hasUnexpected: HasUnexpectedBehaviorMap.NOT_SET, - tabbedBehavior: 0, - behaviors: test.unexpectedBehaviors.map(({description, requireExplanation}) => ({ - description, - checked: false, - more: requireExplanation ? {highlightRequired: false, value: ""} : null, - })), + value: '', }, - }) + }, + }) ), }; @@ -917,21 +938,21 @@ export class TestRunInputOutput { testWindowOptions() { invariant( this.behaviorInput !== null, - "Call %s or %s before calling %s.", + 'Call %s or %s before calling %s.', this.setBehaviorInput.name, this.setBehaviorInputFromJSONAndCommandsConfigKeysTitleUnexpected.name, this.testWindowOptions.name ); invariant( this.pageUriInput !== null, - "Call %s or %s before calling %s.", + 'Call %s or %s before calling %s.', this.setPageUriInput.name, this.setPageUriInputFromPageUri.name, this.testWindowOptions.name ); invariant( this.scriptsInput !== null, - "Call %s or %s before calling %s.", + 'Call %s or %s before calling %s.', this.setScriptsInput.name, this.setScriptsInputFromMap.name, this.testWindowOptions.name @@ -951,7 +972,7 @@ export class TestRunInputOutput { submitResultsJSON(state) { invariant( this.behaviorInput !== null, - "Call %s or %s before calling %s.", + 'Call %s or %s before calling %s.', this.setBehaviorInput.name, this.setBehaviorInputFromJSONAndCommandsConfigKeysTitleUnexpected.name, this.submitResultsJSON.name @@ -966,28 +987,40 @@ export class TestRunInputOutput { specific_user_instruction: behavior.specificUserInstruction, summary: { 1: { - pass: countAssertions(({priority, result}) => priority === 1 && result === CommonResultMap.PASS), - fail: countAssertions(({priority, result}) => priority === 1 && result !== CommonResultMap.PASS), + pass: countAssertions( + ({ priority, result }) => priority === 1 && result === CommonResultMap.PASS + ), + fail: countAssertions( + ({ priority, result }) => priority === 1 && result !== CommonResultMap.PASS + ), }, 2: { - pass: countAssertions(({priority, result}) => priority === 2 && result === CommonResultMap.PASS), - fail: countAssertions(({priority, result}) => priority === 2 && result !== CommonResultMap.PASS), + pass: countAssertions( + ({ priority, result }) => priority === 2 && result === CommonResultMap.PASS + ), + fail: countAssertions( + ({ priority, result }) => priority === 2 && result !== CommonResultMap.PASS + ), }, - unexpectedCount: countUnexpectedBehaviors(({checked}) => checked), + unexpectedCount: countUnexpectedBehaviors(({ checked }) => checked), }, commands: state.commands.map(command => ({ command: command.description, output: command.atOutput.value, support: commandSupport(command), - assertions: [...command.assertions, ...command.additionalAssertions].map(assertionToAssertion), + assertions: [...command.assertions, ...command.additionalAssertions].map( + assertionToAssertion + ), unexpected_behaviors: command.unexpected.behaviors - .filter(({checked}) => checked) - .map(({description, more}) => (more ? more.value : description)), + .filter(({ checked }) => checked) + .map(({ description }) => description), })), }; /** @type {SubmitResultStatusJSON} */ - const status = state.commands.map(commandSupport).some(support => support === CommandSupportJSONMap.FAILING) + const status = state.commands + .map(commandSupport) + .some(support => support === CommandSupportJSONMap.FAILING) ? StatusJSONMap.FAIL : StatusJSONMap.PASS; @@ -999,10 +1032,13 @@ export class TestRunInputOutput { function commandSupport(command) { const allAssertions = [...command.assertions, ...command.additionalAssertions]; - return allAssertions.some(({priority, result}) => priority === 1 && result !== CommonResultMap.PASS) || - command.unexpected.behaviors.some(({checked}) => checked) + return allAssertions.some( + ({ priority, result }) => priority === 1 && result !== CommonResultMap.PASS + ) || command.unexpected.behaviors.some(({ checked }) => checked) ? CommandSupportJSONMap.FAILING - : allAssertions.some(({priority, result}) => priority === 2 && result !== CommonResultMap.PASS) + : allAssertions.some( + ({ priority, result }) => priority === 2 && result !== CommonResultMap.PASS + ) ? CommandSupportJSONMap.ALL_REQUIRED : CommandSupportJSONMap.FULL; } @@ -1013,7 +1049,8 @@ export class TestRunInputOutput { */ function countAssertions(filter) { return state.commands.reduce( - (carry, command) => carry + [...command.assertions, ...command.additionalAssertions].filter(filter).length, + (carry, command) => + carry + [...command.assertions, ...command.additionalAssertions].filter(filter).length, 0 ); } @@ -1023,7 +1060,10 @@ export class TestRunInputOutput { * @returns {number} */ function countUnexpectedBehaviors(filter) { - return state.commands.reduce((carry, command) => carry + command.unexpected.behaviors.filter(filter).length, 0); + return state.commands.reduce( + (carry, command) => carry + command.unexpected.behaviors.filter(filter).length, + 0 + ); } /** @@ -1073,15 +1113,15 @@ export class TestRunInputOutput { output: command.atOutput.value, assertionResults: command.assertions.map(assertion => ({ assertion: { - priority: assertion.priority === 1 ? "REQUIRED" : "OPTIONAL", + priority: assertion.priority === 1 ? 'REQUIRED' : 'OPTIONAL', text: assertion.description, }, - passed: assertion.result === "pass", + passed: assertion.result === 'pass', failedReason: - assertion.result === "failIncorrect" - ? "INCORRECT_OUTPUT" - : assertion.result === "failMissing" - ? "NO_OUTPUT" + assertion.result === 'failIncorrect' + ? 'INCORRECT_OUTPUT' + : assertion.result === 'failMissing' + ? 'NO_OUTPUT' : null, })), unexpectedBehaviors: command.unexpected.behaviors @@ -1089,11 +1129,11 @@ export class TestRunInputOutput { behavior.checked ? { text: behavior.description, - otherUnexpectedBehaviorText: behavior.more ? behavior.more.value : null, } : null ) .filter(Boolean), + unexpectedBehaviorNote: command.unexpected.note.value || null, })), }; } @@ -1106,7 +1146,7 @@ export class TestRunInputOutput { // If ConfigInput is available and resultFormat is TestResultJSON return result in that format. if (this.configInput !== null) { const resultFormat = this.configInput.resultFormat(); - if (resultFormat === "TestResultJSON") { + if (resultFormat === 'TestResultJSON') { return this.testResultJSON(state); } } @@ -1128,25 +1168,28 @@ export class TestRunInputOutput { const scenarioResult = testResult.scenarioResults[commandIndex]; return { ...command, - atOutput: {highlightRequired: false, value: scenarioResult.output}, + atOutput: { highlightRequired: false, value: scenarioResult.output }, assertions: command.assertions.map((assertion, assertionIndex) => { const assertionResult = scenarioResult.assertionResults[assertionIndex]; return { ...assertion, highlightRequired: false, result: assertionResult.passed - ? "pass" - : assertionResult.failedReason === "INCORRECT_OUTPUT" - ? "failIncorrect" - : assertionResult.failedReason === "NO_OUTPUT" - ? "failMissing" - : "notSet", + ? 'pass' + : assertionResult.failedReason === 'INCORRECT_OUTPUT' + ? 'failIncorrect' + : assertionResult.failedReason === 'NO_OUTPUT' + ? 'failMissing' + : 'notSet', }; }), unexpected: { ...command.unexpected, highlightRequired: false, - hasUnexpected: scenarioResult.unexpectedBehaviors.length > 0 ? "hasUnexpected" : "doesNotHaveUnexpected", + hasUnexpected: + scenarioResult.unexpectedBehaviors.length > 0 + ? 'hasUnexpected' + : 'doesNotHaveUnexpected', tabbedBehavior: 0, behaviors: command.unexpected.behaviors.map(behavior => { const behaviorResult = scenarioResult.unexpectedBehaviors.find( @@ -1155,14 +1198,12 @@ export class TestRunInputOutput { return { ...behavior, checked: behaviorResult ? true : false, - more: behavior.more - ? { - highlightRequired: false, - value: behaviorResult ? behaviorResult.otherUnexpectedBehaviorText : "", - } - : behavior.more, }; }), + note: { + highlightRequired: false, + value: scenarioResult.unexpectedBehaviorNote || '', + }, }, }; }), @@ -1178,7 +1219,7 @@ export class TestRunExport extends TestRun { /** * @param {TestRunOptions & TestRunExportOptions} options */ - constructor({resultsJSON, ...parentOptions}) { + constructor({ resultsJSON, ...parentOptions }) { super(parentOptions); this.resultsJSON = resultsJSON; @@ -1186,7 +1227,7 @@ export class TestRunExport extends TestRun { testPageAndResults() { const testPage = this.testPage(); - if ("results" in testPage) { + if ('results' in testPage) { return { ...testPage, resultsJSON: this.resultsJSON(this.state), @@ -1195,7 +1236,9 @@ export class TestRunExport extends TestRun { return { ...testPage, resultsJSON: - this.state.currentUserAction === UserActionMap.CLOSE_TEST_WINDOW ? this.resultsJSON(this.state) : null, + this.state.currentUserAction === UserActionMap.CLOSE_TEST_WINDOW + ? this.resultsJSON(this.state) + : null, }; } } @@ -1215,7 +1258,7 @@ export class TestRunExport extends TestRun { */ const AssertionPassJSONMap = createEnumMap({ - GOOD_OUTPUT: "Good Output", + GOOD_OUTPUT: 'Good Output', }); /** @@ -1234,9 +1277,9 @@ const AssertionPassJSONMap = createEnumMap({ */ const AssertionFailJSONMap = createEnumMap({ - NO_OUTPUT: "No Output", - INCORRECT_OUTPUT: "Incorrect Output", - NO_SUPPORT: "No Support", + NO_OUTPUT: 'No Output', + INCORRECT_OUTPUT: 'Incorrect Output', + NO_SUPPORT: 'No Support', }); /** @typedef {SubmitResultDetailsCommandsAssertionsPass | SubmitResultDetailsCommandsAssertionsFail} SubmitResultAssertionsJSON */ @@ -1253,9 +1296,9 @@ const AssertionFailJSONMap = createEnumMap({ */ const CommandSupportJSONMap = createEnumMap({ - FULL: "FULL", - FAILING: "FAILING", - ALL_REQUIRED: "ALL REQUIRED", + FULL: 'FULL', + FAILING: 'FAILING', + ALL_REQUIRED: 'ALL REQUIRED', }); /** @@ -1267,8 +1310,8 @@ const CommandSupportJSONMap = createEnumMap({ */ const StatusJSONMap = createEnumMap({ - PASS: "PASS", - FAIL: "FAIL", + PASS: 'PASS', + FAIL: 'FAIL', }); /** @@ -1280,7 +1323,7 @@ const StatusJSONMap = createEnumMap({ function invariant(test, message, ...args) { if (!test) { let index = 0; - throw new Error(message.replace(/%%|%\w/g, match => (match[0] !== "%%" ? args[index++] : "%"))); + throw new Error(message.replace(/%%|%\w/g, match => (match[0] !== '%%' ? args[index++] : '%'))); } } diff --git a/client/resources/aria-at-test-run.mjs b/client/resources/aria-at-test-run.mjs index e2dd9f04f..f9e1dd117 100644 --- a/client/resources/aria-at-test-run.mjs +++ b/client/resources/aria-at-test-run.mjs @@ -4,7 +4,7 @@ export class TestRun { * @param {Partial} [param0.hooks] * @param {TestRunState} param0.state */ - constructor({hooks, state}) { + constructor({ hooks, state }) { /** @type {TestRunState} */ this.state = state; @@ -19,7 +19,7 @@ export class TestRun { setCommandAssertion: bindDispatch(userChangeCommandAssertion), setCommandHasUnexpectedBehavior: bindDispatch(userChangeCommandHasUnexpectedBehavior), setCommandUnexpectedBehavior: bindDispatch(userChangeCommandUnexpectedBehavior), - setCommandUnexpectedBehaviorMore: bindDispatch(userChangeCommandUnexpectedBehaviorMore), + setCommandUnexpectedBehaviorNote: bindDispatch(userChangeCommandUnexpectedBehaviorNote), setCommandOutput: bindDispatch(userChangeCommandOutput), submit: () => submitResult(this), ...hooks, @@ -76,7 +76,7 @@ export function createEnumMap(map) { } export const WhitespaceStyleMap = createEnumMap({ - LINE_BREAK: "lineBreak", + LINE_BREAK: 'lineBreak', }); function bind(fn, ...args) { @@ -99,13 +99,15 @@ export function instructionDocument(resultState, hooks) { // As a hack, special case mode instructions for VoiceOver for macOS until we // support modeless tests. ToDo: remove this when resolving issue #194 const modePhrase = - resultState.config.at.name === "VoiceOver for macOS" - ? "Describe " + resultState.config.at.name === 'VoiceOver for macOS' + ? 'Describe ' : `With ${resultState.config.at.name} in ${mode} mode, describe `; - const commands = resultState.commands.map(({description}) => description); - const assertions = resultState.commands[0].assertions.map(({description}) => description); - const additionalAssertions = resultState.commands[0].additionalAssertions.map(({description}) => description); + const commands = resultState.commands.map(({ description }) => description); + const assertions = resultState.commands[0].assertions.map(({ description }) => description); + const additionalAssertions = resultState.commands[0].additionalAssertions.map( + ({ description }) => description + ); let firstRequired = true; function focusFirstRequired() { @@ -119,7 +121,7 @@ export function instructionDocument(resultState, hooks) { return { errors: { visible: resultState.errors && resultState.errors.length > 0 ? true : false, - header: "Test cannot be performed due to error(s)!", + header: 'Test cannot be performed due to error(s)!', errors: resultState.errors || [], }, instructions: { @@ -129,13 +131,13 @@ export function instructionDocument(resultState, hooks) { }, description: `${modePhrase} how ${resultState.config.at.name} behaves when performing task "${lastInstruction}"`, instructions: { - header: "Test instructions", + header: 'Test instructions', instructions: [ [ `Restore default settings for ${resultState.config.at.name}. For help, read `, { - href: "https://github.com/w3c/aria-at/wiki/Configuring-Screen-Readers-for-Testing", - description: "Configuring Screen Readers for Testing", + href: 'https://github.com/w3c/aria-at/wiki/Configuring-Screen-Readers-for-Testing', + description: 'Configuring Screen Readers for Testing', }, `.`, ], @@ -148,26 +150,26 @@ export function instructionDocument(resultState, hooks) { }, }, assertions: { - header: "Success Criteria", + header: 'Success Criteria', description: `To pass this test, ${resultState.config.at.name} needs to meet all the following assertions when each specified command is executed:`, assertions, }, openTestPage: { - button: "Open Test Page", + button: 'Open Test Page', enabled: resultState.openTest.enabled, click: hooks.openTestPage, }, }, results: { header: { - header: "Record Results", + header: 'Record Results', description: `${resultState.info.description}`, }, commands: commands.map(commandResult), }, submit: resultState.config.displaySubmitButton ? { - button: "Submit Results", + button: 'Submit Results', click: hooks.submit, } : null, @@ -185,7 +187,7 @@ export function instructionDocument(resultState, hooks) { ...partialChoice, checked: resultAssertion.result === resultValue, focus: - resultState.currentUserAction === "validateResults" && + resultState.currentUserAction === 'validateResults' && resultAssertion.highlightRequired && focusFirstRequired(), }; @@ -207,20 +209,20 @@ export function instructionDocument(resultState, hooks) { { required: true, highlightRequired: resultStateCommand.atOutput.highlightRequired, - description: "(required)", + description: '(required)', }, ], value: resultStateCommand.atOutput.value, focus: - resultState.currentUserAction === "validateResults" && + resultState.currentUserAction === 'validateResults' && resultStateCommand.atOutput.highlightRequired && focusFirstRequired(), - change: atOutput => hooks.setCommandOutput({commandIndex, atOutput}), + change: atOutput => hooks.setCommandOutput({ commandIndex, atOutput }), }, assertionsHeader: { - descriptionHeader: "Assertion", - passHeader: "Success case", - failHeader: "Failure cases", + descriptionHeader: 'Assertion', + passHeader: 'Success case', + failHeader: 'Failure cases', }, assertions: [ ...assertions.map(bind(assertionResult, commandIndex)), @@ -228,18 +230,20 @@ export function instructionDocument(resultState, hooks) { ], unexpectedBehaviors: { description: [ - "Were there additional undesirable behaviors?", + 'Were there additional unexpected behaviors?', { required: true, highlightRequired: resultStateCommand.unexpected.highlightRequired, - description: "(required)", + description: '(required)', }, ], passChoice: { - label: "No, there were no additional undesirable behaviors.", - checked: resultUnexpectedBehavior.hasUnexpected === HasUnexpectedBehaviorMap.DOES_NOT_HAVE_UNEXPECTED, + label: 'No, there were no additional unexpected behaviors.', + checked: + resultUnexpectedBehavior.hasUnexpected === + HasUnexpectedBehaviorMap.DOES_NOT_HAVE_UNEXPECTED, focus: - resultState.currentUserAction === "validateResults" && + resultState.currentUserAction === 'validateResults' && resultUnexpectedBehavior.highlightRequired && resultUnexpectedBehavior.hasUnexpected === HasUnexpectedBehaviorMap.NOT_SET && focusFirstRequired(), @@ -250,10 +254,12 @@ export function instructionDocument(resultState, hooks) { }), }, failChoice: { - label: "Yes, there were additional undesirable behaviors", - checked: resultUnexpectedBehavior.hasUnexpected === HasUnexpectedBehaviorMap.HAS_UNEXPECTED, + label: 'Yes, there were additional unexpected behaviors.', + checked: + resultUnexpectedBehavior.hasUnexpected === HasUnexpectedBehaviorMap.HAS_UNEXPECTED || + resultUnexpectedBehavior.expand, focus: - resultState.currentUserAction === "validateResults" && + resultState.currentUserAction === 'validateResults' && resultUnexpectedBehavior.highlightRequired && resultUnexpectedBehavior.hasUnexpected === HasUnexpectedBehaviorMap.NOT_SET && focusFirstRequired(), @@ -263,54 +269,73 @@ export function instructionDocument(resultState, hooks) { hasUnexpected: HasUnexpectedBehaviorMap.HAS_UNEXPECTED, }), options: { - header: "Undesirable behaviors", + header: 'Unexpected behaviors', options: resultUnexpectedBehavior.behaviors.map((behavior, unexpectedIndex) => { return { description: behavior.description, - enabled: resultUnexpectedBehavior.hasUnexpected === HasUnexpectedBehaviorMap.HAS_UNEXPECTED, + enabled: + resultUnexpectedBehavior.hasUnexpected === + HasUnexpectedBehaviorMap.HAS_UNEXPECTED, tabbable: resultUnexpectedBehavior.tabbedBehavior === unexpectedIndex, checked: behavior.checked, focus: - typeof resultState.currentUserAction === "object" && - resultState.currentUserAction.action === UserObjectActionMap.FOCUS_UNDESIRABLE + typeof resultState.currentUserAction === 'object' && + resultState.currentUserAction.action === UserObjectActionMap.FOCUS_UNEXPECTED ? resultState.currentUserAction.commandIndex === commandIndex && resultUnexpectedBehavior.tabbedBehavior === unexpectedIndex : resultState.currentUserAction === UserActionMap.VALIDATE_RESULTS && - resultUnexpectedBehavior.hasUnexpected === HasUnexpectedBehaviorMap.HAS_UNEXPECTED && - resultUnexpectedBehavior.behaviors.every(({checked}) => !checked) && + resultUnexpectedBehavior.hasUnexpected === + HasUnexpectedBehaviorMap.HAS_UNEXPECTED && + resultUnexpectedBehavior.behaviors.every(({ checked }) => !checked) && focusFirstRequired(), - change: checked => hooks.setCommandUnexpectedBehavior({commandIndex, unexpectedIndex, checked}), + change: checked => + hooks.setCommandUnexpectedBehavior({ commandIndex, unexpectedIndex, checked }), keydown: key => { const increment = keyToFocusIncrement(key); if (increment) { - hooks.focusCommandUnexpectedBehavior({commandIndex, unexpectedIndex, increment}); + hooks.focusCommandUnexpectedBehavior({ + commandIndex, + unexpectedIndex, + increment, + }); return true; } return false; }, - more: behavior.more - ? { - description: /** @type {Description[]} */ ([ - `If "other" selected, explain`, - { - required: true, - highlightRequired: behavior.more.highlightRequired, - description: "(required)", - }, - ]), - enabled: behavior.checked, - value: behavior.more.value, - focus: - resultState.currentUserAction === "validateResults" && - behavior.more.highlightRequired && - focusFirstRequired(), - change: value => - hooks.setCommandUnexpectedBehaviorMore({commandIndex, unexpectedIndex, more: value}), - } - : null, }; }), }, + note: { + description: /** @type {Description[]} */ ([ + `Add an explanation`, + { + required: resultUnexpectedBehavior.behaviors.some( + ({ checked, requireExplanation }) => requireExplanation && checked + ), + highlightRequired: + resultState.currentUserAction === 'validateResults' && + resultUnexpectedBehavior.behaviors.some( + ({ checked, requireExplanation }) => requireExplanation && checked + ), + description: resultUnexpectedBehavior.behaviors.some( + ({ checked, requireExplanation }) => requireExplanation && checked + ) + ? ' (required)' + : ' (not required)', + }, + ]), + enabled: + resultUnexpectedBehavior.hasUnexpected === HasUnexpectedBehaviorMap.HAS_UNEXPECTED && + resultUnexpectedBehavior.behaviors.some(({ checked }) => checked), + value: resultUnexpectedBehavior.note.value, + focus: + resultState.currentUserAction === 'validateResults' && + resultUnexpectedBehavior.behaviors.some( + ({ checked, requireExplanation }) => requireExplanation && checked + ) && + focusFirstRequired(), + change: value => hooks.setCommandUnexpectedBehaviorNote({ commandIndex, note: value }), + }, }, }, }; @@ -329,7 +354,7 @@ export function instructionDocument(resultState, hooks) { { required: true, highlightRequired: resultAssertion.highlightRequired, - description: "(required: mark output)", + description: '(required: mark output)', }, ], passChoice: assertionChoice(resultAssertion, CommonResultMap.PASS, { @@ -337,10 +362,11 @@ export function instructionDocument(resultState, hooks) { `Good Output `, { offScreen: true, - description: "for assertion", + description: 'for assertion', }, ], - click: () => hooks.setCommandAssertion({commandIndex, assertionIndex, result: CommonResultMap.PASS}), + click: () => + hooks.setCommandAssertion({ commandIndex, assertionIndex, result: CommonResultMap.PASS }), }), failChoices: [ assertionChoice(resultAssertion, AssertionResultMap.FAIL_MISSING, { @@ -348,22 +374,30 @@ export function instructionDocument(resultState, hooks) { `No Output `, { offScreen: true, - description: "for assertion", + description: 'for assertion', }, ], click: () => - hooks.setCommandAssertion({commandIndex, assertionIndex, result: AssertionResultMap.FAIL_MISSING}), + hooks.setCommandAssertion({ + commandIndex, + assertionIndex, + result: AssertionResultMap.FAIL_MISSING, + }), }), assertionChoice(resultAssertion, AssertionResultMap.FAIL_INCORRECT, { label: [ `Incorrect Output `, { offScreen: true, - description: "for assertion", + description: 'for assertion', }, ], click: () => - hooks.setCommandAssertion({commandIndex, assertionIndex, result: AssertionResultMap.FAIL_INCORRECT}), + hooks.setCommandAssertion({ + commandIndex, + assertionIndex, + result: AssertionResultMap.FAIL_INCORRECT, + }), }), ], }); @@ -375,18 +409,19 @@ export function instructionDocument(resultState, hooks) { * @param {number} assertionIndex */ function additionalAssertionResult(commandIndex, assertion, assertionIndex) { - const resultAdditionalAssertion = resultState.commands[commandIndex].additionalAssertions[assertionIndex]; + const resultAdditionalAssertion = + resultState.commands[commandIndex].additionalAssertions[assertionIndex]; return /** @type {InstructionDocumentResultsCommandsAssertion} */ ({ description: [ assertion, { required: true, highlightRequired: resultAdditionalAssertion.highlightRequired, - description: "(required: mark support)", + description: '(required: mark support)', }, ], passChoice: assertionChoice(resultAdditionalAssertion, AdditionalAssertionResultMap.PASS, { - label: ["Good Support ", {offScreen: true, description: "for assertion"}], + label: ['Good Support ', { offScreen: true, description: 'for assertion' }], click: () => hooks.setCommandAdditionalAssertion({ commandIndex, @@ -396,7 +431,7 @@ export function instructionDocument(resultState, hooks) { }), failChoices: [ assertionChoice(resultAdditionalAssertion, AdditionalAssertionResultMap.FAIL_SUPPORT, { - label: ["No Support ", {offScreen: true, description: "for assertion"}], + label: ['No Support ', { offScreen: true, description: 'for assertion' }], click: () => hooks.setCommandAdditionalAssertion({ commandIndex, @@ -414,13 +449,13 @@ export function instructionDocument(resultState, hooks) { */ export const UserActionMap = createEnumMap({ - LOAD_PAGE: "loadPage", - OPEN_TEST_WINDOW: "openTestWindow", - CLOSE_TEST_WINDOW: "closeTestWindow", - VALIDATE_RESULTS: "validateResults", - CHANGE_TEXT: "changeText", - CHANGE_SELECTION: "changeSelection", - SHOW_RESULTS: "showResults", + LOAD_PAGE: 'loadPage', + OPEN_TEST_WINDOW: 'openTestWindow', + CLOSE_TEST_WINDOW: 'closeTestWindow', + VALIDATE_RESULTS: 'validateResults', + CHANGE_TEXT: 'changeText', + CHANGE_SELECTION: 'changeSelection', + SHOW_RESULTS: 'showResults', }); /** @@ -428,7 +463,7 @@ export const UserActionMap = createEnumMap({ */ export const UserObjectActionMap = createEnumMap({ - FOCUS_UNDESIRABLE: "focusUndesirable", + FOCUS_UNEXPECTED: 'focusUnexpected', }); /** @@ -440,14 +475,14 @@ export const UserObjectActionMap = createEnumMap({ */ export const HasUnexpectedBehaviorMap = createEnumMap({ - NOT_SET: "notSet", - HAS_UNEXPECTED: "hasUnexpected", - DOES_NOT_HAVE_UNEXPECTED: "doesNotHaveUnexpected", + NOT_SET: 'notSet', + HAS_UNEXPECTED: 'hasUnexpected', + DOES_NOT_HAVE_UNEXPECTED: 'doesNotHaveUnexpected', }); export const CommonResultMap = createEnumMap({ - NOT_SET: "notSet", - PASS: "pass", + NOT_SET: 'notSet', + PASS: 'pass', }); /** @@ -456,7 +491,7 @@ export const CommonResultMap = createEnumMap({ export const AdditionalAssertionResultMap = createEnumMap({ ...CommonResultMap, - FAIL_SUPPORT: "failSupport", + FAIL_SUPPORT: 'failSupport', }); /** @@ -465,8 +500,8 @@ export const AdditionalAssertionResultMap = createEnumMap({ export const AssertionResultMap = createEnumMap({ ...CommonResultMap, - FAIL_MISSING: "failMissing", - FAIL_INCORRECT: "failIncorrect", + FAIL_MISSING: 'failMissing', + FAIL_INCORRECT: 'failIncorrect', }); /** @@ -475,7 +510,7 @@ export const AssertionResultMap = createEnumMap({ * @param {string} props.atOutput * @returns {(state: TestRunState) => TestRunState} */ -export function userChangeCommandOutput({commandIndex, atOutput}) { +export function userChangeCommandOutput({ commandIndex, atOutput }) { return function (state) { return { ...state, @@ -502,7 +537,7 @@ export function userChangeCommandOutput({commandIndex, atOutput}) { * @param {AssertionResult} props.result * @returns {(state: TestRunState) => TestRunState} */ -export function userChangeCommandAssertion({commandIndex, assertionIndex, result}) { +export function userChangeCommandAssertion({ commandIndex, assertionIndex, result }) { return function (state) { return { ...state, @@ -513,7 +548,7 @@ export function userChangeCommandAssertion({commandIndex, assertionIndex, result : { ...command, assertions: command.assertions.map((assertion, assertionI) => - assertionI !== assertionIndex ? assertion : {...assertion, result} + assertionI !== assertionIndex ? assertion : { ...assertion, result } ), } ), @@ -528,7 +563,11 @@ export function userChangeCommandAssertion({commandIndex, assertionIndex, result * @param {AdditionalAssertionResult} props.result * @returns {(state: TestRunState) => TestRunState} */ -export function userChangeCommandAdditionalAssertion({commandIndex, additionalAssertionIndex, result}) { +export function userChangeCommandAdditionalAssertion({ + commandIndex, + additionalAssertionIndex, + result, +}) { return function (state) { return { ...state, @@ -539,7 +578,7 @@ export function userChangeCommandAdditionalAssertion({commandIndex, additionalAs : { ...command, additionalAssertions: command.additionalAssertions.map((assertion, assertionI) => - assertionI !== additionalAssertionIndex ? assertion : {...assertion, result} + assertionI !== additionalAssertionIndex ? assertion : { ...assertion, result } ), } ), @@ -553,7 +592,7 @@ export function userChangeCommandAdditionalAssertion({commandIndex, additionalAs * @param {HasUnexpectedBehavior} props.hasUnexpected * @returns {(state: TestRunState) => TestRunState} */ -export function userChangeCommandHasUnexpectedBehavior({commandIndex, hasUnexpected}) { +export function userChangeCommandHasUnexpectedBehavior({ commandIndex, hasUnexpected }) { return function (state) { return { ...state, @@ -565,13 +604,17 @@ export function userChangeCommandHasUnexpectedBehavior({commandIndex, hasUnexpec ...command, unexpected: { ...command.unexpected, + expand: hasUnexpected === HasUnexpectedBehaviorMap.HAS_UNEXPECTED, hasUnexpected: hasUnexpected, tabbedBehavior: hasUnexpected === HasUnexpectedBehaviorMap.HAS_UNEXPECTED ? 0 : -1, behaviors: command.unexpected.behaviors.map(behavior => ({ ...behavior, checked: false, - more: behavior.more ? {...behavior.more, value: ""} : null, })), + note: { + ...command.unexpected.note, + value: '', + }, }, } ), @@ -586,7 +629,7 @@ export function userChangeCommandHasUnexpectedBehavior({commandIndex, hasUnexpec * @param {boolean} props.checked * @returns {(state: TestRunState) => TestRunState} */ -export function userChangeCommandUnexpectedBehavior({commandIndex, unexpectedIndex, checked}) { +export function userChangeCommandUnexpectedBehavior({ commandIndex, unexpectedIndex, checked }) { return function (state) { return { ...state, @@ -616,11 +659,10 @@ export function userChangeCommandUnexpectedBehavior({commandIndex, unexpectedInd /** * @param {object} props * @param {number} props.commandIndex - * @param {number} props.unexpectedIndex - * @param {string} props.more + * @param {string} props.note * @returns {(state: TestRunState) => TestRunState} */ -export function userChangeCommandUnexpectedBehaviorMore({commandIndex, unexpectedIndex, more}) { +export function userChangeCommandUnexpectedBehaviorNote({ commandIndex, note }) { return function (state) { return { ...state, @@ -632,17 +674,10 @@ export function userChangeCommandUnexpectedBehaviorMore({commandIndex, unexpecte ...command, unexpected: { ...command.unexpected, - behaviors: command.unexpected.behaviors.map((unexpected, unexpectedI) => - unexpectedI !== unexpectedIndex - ? unexpected - : /** @type {TestRunUnexpectedBehavior} */ ({ - ...unexpected, - more: { - ...unexpected.more, - value: more, - }, - }) - ), + note: { + ...command.unexpected.note, + value: note, + }, }, }) ), @@ -656,17 +691,17 @@ export function userChangeCommandUnexpectedBehaviorMore({commandIndex, unexpecte */ function keyToFocusIncrement(key) { switch (key) { - case "Up": - case "ArrowUp": - case "Left": - case "ArrowLeft": - return "previous"; - - case "Down": - case "ArrowDown": - case "Right": - case "ArrowRight": - return "next"; + case 'Up': + case 'ArrowUp': + case 'Left': + case 'ArrowLeft': + return 'previous'; + + case 'Down': + case 'ArrowDown': + case 'Right': + case 'ArrowRight': + return 'next'; } } @@ -709,7 +744,10 @@ function submitResult(app) { export function userShowResults() { return function (/** @type {TestRunState} */ state) { - return /** @type {TestRunState} */ ({...state, currentUserAction: UserActionMap.SHOW_RESULTS}); + return /** @type {TestRunState} */ ({ + ...state, + currentUserAction: UserActionMap.SHOW_RESULTS, + }); }; } @@ -720,15 +758,22 @@ export function userShowResults() { function isSomeFieldRequired(state) { return state.commands.some( command => - command.atOutput.value.trim() === "" || + command.atOutput.value.trim() === '' || command.assertions.some(assertion => assertion.result === CommonResultMap.NOT_SET) || - command.additionalAssertions.some(assertion => assertion.result === CommonResultMap.NOT_SET) || + command.additionalAssertions.some( + assertion => assertion.result === CommonResultMap.NOT_SET + ) || command.unexpected.hasUnexpected === HasUnexpectedBehaviorMap.NOT_SET || (command.unexpected.hasUnexpected === HasUnexpectedBehaviorMap.HAS_UNEXPECTED && - (command.unexpected.behaviors.every(({checked}) => !checked) || - command.unexpected.behaviors.some( - behavior => behavior.checked && behavior.more && behavior.more.value.trim() === "" - ))) + (command.unexpected.behaviors.every(({ checked }) => !checked) || + command.unexpected.behaviors.some(behavior => { + return ( + behavior.checked && + behavior.requireExplanation && + command.unexpected.note && + command.unexpected.note.value.trim() === '' + ); + }))) ); } @@ -741,79 +786,86 @@ function resultsTableDocument(state) { header: state.info.description, status: { header: [ - "Test result: ", + 'Test result: ', state.commands.some( - ({assertions, additionalAssertions, unexpected}) => + ({ assertions, additionalAssertions, unexpected }) => [...assertions, ...additionalAssertions].some( - ({priority, result}) => priority === 1 && result !== CommonResultMap.PASS - ) || unexpected.behaviors.some(({checked}) => checked) + ({ priority, result }) => priority === 1 && result !== CommonResultMap.PASS + ) || unexpected.behaviors.some(({ checked }) => checked) ) - ? "FAIL" - : "PASS", + ? 'FAIL' + : 'PASS', ], }, table: { headers: { - description: "Command", - support: "Support", - details: "Details", + description: 'Command', + support: 'Support', + details: 'Details', }, commands: state.commands.map(command => { const allAssertions = [...command.assertions, ...command.additionalAssertions]; - let passingAssertions = ["No passing assertions."]; - let failingAssertions = ["No failing assertions."]; - let unexpectedBehaviors = ["No unexpect behaviors."]; + let passingAssertions = ['No passing assertions.']; + let failingAssertions = ['No failing assertions.']; + let unexpectedBehaviors = ['No unexpect behaviors.']; - if (allAssertions.some(({result}) => result === CommonResultMap.PASS)) { + if (allAssertions.some(({ result }) => result === CommonResultMap.PASS)) { passingAssertions = allAssertions - .filter(({result}) => result === CommonResultMap.PASS) - .map(({description}) => description); + .filter(({ result }) => result === CommonResultMap.PASS) + .map(({ description }) => description); } - if (allAssertions.some(({result}) => result !== CommonResultMap.PASS)) { + if (allAssertions.some(({ result }) => result !== CommonResultMap.PASS)) { failingAssertions = allAssertions - .filter(({result}) => result !== CommonResultMap.PASS) - .map(({description}) => description); + .filter(({ result }) => result !== CommonResultMap.PASS) + .map(({ description }) => description); } - if (command.unexpected.behaviors.some(({checked}) => checked)) { + if (command.unexpected.behaviors.some(({ checked }) => checked)) { unexpectedBehaviors = command.unexpected.behaviors - .filter(({checked}) => checked) - .map(({description, more}) => (more ? more.value : description)); + .filter(({ checked }) => checked) + .map(({ description }) => description); } return { description: command.description, support: - allAssertions.some(({priority, result}) => priority === 1 && result !== CommonResultMap.PASS) || - command.unexpected.behaviors.some(({checked}) => checked) - ? "FAILING" - : allAssertions.some(({priority, result}) => priority === 2 && result !== CommonResultMap.PASS) - ? "ALL_REQUIRED" - : "FULL", + allAssertions.some( + ({ priority, result }) => priority === 1 && result !== CommonResultMap.PASS + ) || command.unexpected.behaviors.some(({ checked }) => checked) + ? 'FAILING' + : allAssertions.some( + ({ priority, result }) => priority === 2 && result !== CommonResultMap.PASS + ) + ? 'ALL_REQUIRED' + : 'FULL', details: { output: /** @type {Description} */ [ - "output:", - /** @type {DescriptionWhitespace} */ ({whitespace: WhitespaceStyleMap.LINE_BREAK}), - " ", - ...command.atOutput.value - .split(/(\r\n|\r|\n)/g) - .map(output => - /\r\n|\r|\n/.test(output) - ? /** @type {DescriptionWhitespace} */ ({whitespace: WhitespaceStyleMap.LINE_BREAK}) - : output - ), + 'output:', + /** @type {DescriptionWhitespace} */ ({ whitespace: WhitespaceStyleMap.LINE_BREAK }), + ' ', + ...command.atOutput.value.split(/(\r\n|\r|\n)/g).map(output => + /\r\n|\r|\n/.test(output) + ? /** @type {DescriptionWhitespace} */ ({ + whitespace: WhitespaceStyleMap.LINE_BREAK, + }) + : output + ), ], passingAssertions: { - description: "Passing Assertions:", + description: 'Passing Assertions:', items: passingAssertions, }, failingAssertions: { - description: "Failing Assertions:", + description: 'Failing Assertions:', items: failingAssertions, }, unexpectedBehaviors: { - description: "Unexpected Behavior", + description: 'Unexpected Behavior:', items: unexpectedBehaviors, + note: { + description: 'Explanation:', + value: command.unexpected.note.value, + }, }, }, }; @@ -827,7 +879,7 @@ export function userOpenWindow() { /** @type {TestRunState} */ ({ ...state, currentUserAction: UserActionMap.OPEN_TEST_WINDOW, - openTest: {...state.openTest, enabled: false}, + openTest: { ...state.openTest, enabled: false }, }); } @@ -836,7 +888,7 @@ export function userCloseWindow() { /** @type {TestRunState} */ ({ ...state, currentUserAction: UserActionMap.CLOSE_TEST_WINDOW, - openTest: {...state.openTest, enabled: true}, + openTest: { ...state.openTest, enabled: true }, }); } @@ -847,23 +899,25 @@ export function userCloseWindow() { * @param {TestRunFocusIncrement} props.increment * @returns {(state: TestRunState) => TestRunState} */ -export function userFocusCommandUnexpectedBehavior({commandIndex, unexpectedIndex, increment}) { +export function userFocusCommandUnexpectedBehavior({ commandIndex, unexpectedIndex, increment }) { return function (state) { const unexpectedLength = state.commands[commandIndex].unexpected.behaviors.length; - const incrementValue = increment === "next" ? 1 : -1; - const newUnexpectedIndex = (unexpectedIndex + incrementValue + unexpectedLength) % unexpectedLength; + const incrementValue = increment === 'next' ? 1 : -1; + const newUnexpectedIndex = + (unexpectedIndex + incrementValue + unexpectedLength) % unexpectedLength; return { ...state, currentUserAction: { - action: UserObjectActionMap.FOCUS_UNDESIRABLE, + action: UserObjectActionMap.FOCUS_UNEXPECTED, commandIndex, unexpectedIndex: newUnexpectedIndex, }, commands: state.commands.map((command, commandI) => { const tabbed = command.unexpected.tabbedBehavior; const unexpectedLength = command.unexpected.behaviors.length; - const newTabbed = (tabbed + (increment === "next" ? 1 : -1) + unexpectedLength) % unexpectedLength; + const newTabbed = + (tabbed + (increment === 'next' ? 1 : -1) + unexpectedLength) % unexpectedLength; return commandI !== commandIndex ? command : { @@ -906,18 +960,15 @@ export function userValidateState() { highlightRequired: command.unexpected.hasUnexpected === HasUnexpectedBehaviorMap.NOT_SET || (command.unexpected.hasUnexpected === HasUnexpectedBehaviorMap.HAS_UNEXPECTED && - command.unexpected.behaviors.every(({checked}) => !checked)), - behaviors: command.unexpected.behaviors.map(unexpected => { - return unexpected.more - ? { - ...unexpected, - more: { - ...unexpected.more, - highlightRequired: unexpected.checked && !unexpected.more.value.trim(), - }, - } - : unexpected; - }), + command.unexpected.behaviors.every(({ checked }) => !checked)), + note: { + ...command.unexpected.note, + highlightRequired: + command.unexpected.note.value.trim() === '' && + command.unexpected.behaviors.some( + ({ checked, requireExplanation }) => requireExplanation && checked + ), + }, }, }; }), @@ -1127,13 +1178,13 @@ export function userValidateState() { * @property {(options: {commandIndex: number, hasUnexpected: HasUnexpectedBehavior}) => void } setCommandHasUnexpectedBehavior * @property {(options: {commandIndex: number, atOutput: string}) => void} setCommandOutput * @property {(options: {commandIndex: number, unexpectedIndex: number, checked}) => void } setCommandUnexpectedBehavior - * @property {(options: {commandIndex: number, unexpectedIndex: number, more: string}) => void } setCommandUnexpectedBehaviorMore + * @property {(options: {commandIndex: number, unexpectedIndex: number, more: string}) => void } setCommandUnexpectedBehaviorNote * @property {() => void} submit */ /** * @typedef UserActionFocusUnexpected - * @property {typeof UserObjectActionMap["FOCUS_UNDESIRABLE"]} action + * @property {typeof UserObjectActionMap["FOCUS_UNEXPECTED"]} action * @property {number} commandIndex * @property {number} unexpectedIndex */ diff --git a/client/resources/at-commands.mjs b/client/resources/at-commands.mjs index 4550de934..898bcc39e 100644 --- a/client/resources/at-commands.mjs +++ b/client/resources/at-commands.mjs @@ -21,13 +21,13 @@ export class commandsAPI { * } * } */ -constructor(commands, support) { + constructor(commands, support) { if (!commands) { - throw new Error("You must initialize commandsAPI with a commands data object"); + throw new Error('You must initialize commandsAPI with a commands data object'); } if (!support) { - throw new Error("You must initialize commandsAPI with a support data object"); + throw new Error('You must initialize commandsAPI with a support data object'); } this.AT_COMMAND_MAP = commands; @@ -36,19 +36,18 @@ constructor(commands, support) { reading: { jaws: `Verify the Virtual Cursor is active by pressing ${keys.ALT_DELETE}. If it is not, exit Forms Mode to activate the Virtual Cursor by pressing ${keys.ESC}.`, nvda: `Insure NVDA is in browse mode by pressing ${keys.ESC}. Note: This command has no effect if NVDA is already in browse mode.`, - voiceover_macos: `Toggle Quick Nav ON by pressing the ${keys.LEFT} and ${keys.RIGHT} keys at the same time.` + voiceover_macos: `Toggle Quick Nav ON by pressing the ${keys.LEFT} and ${keys.RIGHT} keys at the same time.`, }, interaction: { jaws: `Verify the PC Cursor is active by pressing ${keys.ALT_DELETE}. If it is not, turn off the Virtual Cursor by pressing ${keys.INS_Z}.`, nvda: `If NVDA did not make the focus mode sound when the test page loaded, press ${keys.INS_SPACE} to turn focus mode on.`, - voiceover_macos: `Toggle Quick Nav OFF by pressing the ${keys.LEFT} and ${keys.RIGHT} keys at the same time.` - } + voiceover_macos: `Toggle Quick Nav OFF by pressing the ${keys.LEFT} and ${keys.RIGHT} keys at the same time.`, + }, }; this.support = support; } - /** * Get AT-specific instruction * @param {string} mode - The mode of the screen reader, "reading" or "interaction" @@ -58,10 +57,13 @@ constructor(commands, support) { */ getATCommands(mode, task, assistiveTech) { if (!this.AT_COMMAND_MAP[task]) { - throw new Error(`Task "${task}" does not exist, please add to at-commands or correct your spelling.`); - } - else if (!this.AT_COMMAND_MAP[task][mode]) { - throw new Error(`Mode "${mode}" instructions for task "${task}" does not exist, please add to at-commands or correct your spelling.`); + throw new Error( + `Task "${task}" does not exist, please add to at-commands or correct your spelling.` + ); + } else if (!this.AT_COMMAND_MAP[task][mode]) { + throw new Error( + `Mode "${mode}" instructions for task "${task}" does not exist, please add to at-commands or correct your spelling.` + ); } let commandsData = this.AT_COMMAND_MAP[task][mode][assistiveTech.key] || []; @@ -73,14 +75,16 @@ constructor(commands, support) { for (let command of commandSequence) { command = keys[command]; if (typeof command === 'undefined') { - throw new Error(`Key instruction identifier "${c}" for AT "${assistiveTech.name}", mode "${mode}", task "${task}" is not an available identified. Update you commands.json file to the correct identifier or add your identifier to resources/keys.mjs.`); + throw new Error( + `Key instruction identifier "${c}" for AT "${assistiveTech.name}", mode "${mode}", task "${task}" is not an available identified. Update you commands.json file to the correct identifier or add your identifier to resources/keys.mjs.` + ); } let furtherInstruction = c[1]; command = furtherInstruction ? `${command} ${furtherInstruction}` : command; innerCommands.push(command); } - commands.push(innerCommands.join(", then ")); + commands.push(innerCommands.join(', then ')); } return commands; diff --git a/client/resources/types/aria-at-test-result.js b/client/resources/types/aria-at-test-result.js index ff989c6a5..be8cd78ac 100644 --- a/client/resources/types/aria-at-test-result.js +++ b/client/resources/types/aria-at-test-result.js @@ -34,5 +34,5 @@ * @property {object[]} scenarioResults[].unexpectedBehaviors * @property {string} scenarioResults[].unexpectedBehaviors[].id * @property {string} scenarioResults[].unexpectedBehaviors[].text - * @property {string | null} [scenarioResults[].unexpectedBehaviors[].otherUnexpectedBehaviorText] + * @property {string | null} [scenarioResults[].unexpectedBehaviorNote] */ diff --git a/client/resources/types/aria-at-test-run.js b/client/resources/types/aria-at-test-run.js index cdeceb90e..c12c3b142 100644 --- a/client/resources/types/aria-at-test-run.js +++ b/client/resources/types/aria-at-test-run.js @@ -16,12 +16,12 @@ */ /** - * @typedef {"focusUndesirable"} AriaATTestRun.UserActionObjectName + * @typedef {"focusUnexpected"} AriaATTestRun.UserActionObjectName */ /** * @typedef AriaATTestRun.UserActionFocusUnexpected - * @property {"focusUndesirable"} action + * @property {"focusUnexpected"} action * @property {number} commandIndex * @property {number} unexpectedIndex */ diff --git a/server/graphql-schema.js b/server/graphql-schema.js index db548ae41..edbfcca13 100644 --- a/server/graphql-schema.js +++ b/server/graphql-schema.js @@ -553,6 +553,10 @@ const graphqlSchema = gql` Submitted test results require this field to be filled in. """ unexpectedBehaviors: [UnexpectedBehavior] + """ + Optional note provided by the tester explaining the unexpected behavior. + """ + unexpectedBehaviorNote: String } """ @@ -574,7 +578,11 @@ const graphqlSchema = gql` """ See ScenarioResult type for more information. """ - unexpectedBehaviors: [UnexpectedBehaviorInput] + unexpectedBehaviors: [UnexpectedBehaviorId] + """ + See ScenarioResult type for more information. + """ + unexpectedBehaviorNote: String } # TODO: figure out if this type can be removed and NO_OUTPUT can become an @@ -650,26 +658,6 @@ const graphqlSchema = gql` Human-readable sentence describing the failure. """ text: String! - """ - One of the unexpected behaviors is "other", which means the user must - provide text explaining what occurred. For all other unexpected - behaviors this field can be ignored. - """ - otherUnexpectedBehaviorText: String - } - - """ - Minimal plain representation of an UnexpectedBehavior. - """ - input UnexpectedBehaviorInput { - """ - See UnexpectedBehavior for more information. - """ - id: UnexpectedBehaviorId! - """ - See UnexpectedBehavior for more information. - """ - otherUnexpectedBehaviorText: String } """ diff --git a/server/migrations/20220726035632-unexpectedBehaviorNote.js b/server/migrations/20220726035632-unexpectedBehaviorNote.js new file mode 100644 index 000000000..96cda6c9f --- /dev/null +++ b/server/migrations/20220726035632-unexpectedBehaviorNote.js @@ -0,0 +1,77 @@ +const { omit } = require('lodash'); +const { TestPlanRun } = require('../models'); + +module.exports = { + up: async () => { + const testPlanRuns = await TestPlanRun.findAll(); + await Promise.all( + testPlanRuns.map(testPlanRun => { + const newTestResults = testPlanRun.testResults.map( + testResult => ({ + ...testResult, + scenarioResults: testResult.scenarioResults.map( + scenarioResult => { + const note = + scenarioResult.unexpectedBehaviors?.find( + each => + !!each.otherUnexpectedBehaviorText + )?.otherUnexpectedBehaviorText ?? null; + return { + ...scenarioResult, + unexpectedBehaviors: + scenarioResult.unexpectedBehaviors?.map( + unexpectedBehavior => + unexpectedBehavior.id + ), + unexpectedBehaviorNote: note + }; + } + ) + }) + ); + testPlanRun.testResults = newTestResults; + return testPlanRun.save(); + }) + ); + }, + + down: async () => { + const testPlanRuns = await TestPlanRun.findAll(); + await Promise.all( + testPlanRuns.map(testPlanRun => { + const newTestResults = testPlanRun.testResults.map( + testResult => ({ + ...testResult, + scenarioResults: testResult.scenarioResults.map( + scenarioResult => { + return omit( + { + ...scenarioResult, + unexpectedBehaviors: + scenarioResult.unexpectedBehaviors?.map( + unexpectedBehaviorId => { + return unexpectedBehaviorId !== + 'OTHER' + ? { + id: unexpectedBehaviorId + } + : { + id: 'OTHER', + otherUnexpectedBehaviorText: + scenarioResult.unexpectedBehaviorNote + }; + } + ) + }, + ['unexpectedBehaviorNote'] + ); + } + ) + }) + ); + testPlanRun.testResults = newTestResults; + return testPlanRun.save(); + }) + ); + } +}; diff --git a/server/resolvers/TestPlanReport/conflictsResolver.js b/server/resolvers/TestPlanReport/conflictsResolver.js index da5ea42e9..723f0daf3 100644 --- a/server/resolvers/TestPlanReport/conflictsResolver.js +++ b/server/resolvers/TestPlanReport/conflictsResolver.js @@ -62,7 +62,8 @@ const conflictsResolver = async testPlanReport => { for (let i = 0; i < testResults[0].scenarioResults.length; i += 1) { const scenarioResultComparisons = testResults.map(testResult => { - // Note that output is not considered + // Note that the output and unexpectedBehaviorNote are not + // considered return pick(testResult.scenarioResults[i], [ 'unexpectedBehaviors' ]); diff --git a/server/resolvers/TestPlanRun/testResultsResolver.js b/server/resolvers/TestPlanRun/testResultsResolver.js index e23dd492a..bc99ae70f 100644 --- a/server/resolvers/TestPlanRun/testResultsResolver.js +++ b/server/resolvers/TestPlanRun/testResultsResolver.js @@ -32,12 +32,14 @@ const testResultsResolver = testPlanRun => { }) ), unexpectedBehaviors: scenarioResult.unexpectedBehaviors?.map( - unexpectedBehavior => ({ - ...unexpectedBehavior, - text: unexpectedBehaviorsJson.find( - each => each.id === unexpectedBehavior.id - ).text - }) + unexpectedBehaviorId => { + return { + id: unexpectedBehaviorId, + text: unexpectedBehaviorsJson.find( + each => each.id === unexpectedBehaviorId + ).text + }; + } ) })) }; diff --git a/server/resolvers/TestPlanRunOperations/createTestResultSkeleton.js b/server/resolvers/TestPlanRunOperations/createTestResultSkeleton.js index ff8df7081..c0d6de12d 100644 --- a/server/resolvers/TestPlanRunOperations/createTestResultSkeleton.js +++ b/server/resolvers/TestPlanRunOperations/createTestResultSkeleton.js @@ -41,7 +41,8 @@ const createTestResultSkeleton = ({ failedReason: null }; }), - unexpectedBehaviors: null + unexpectedBehaviors: null, + unexpectedBehaviorNote: null }; }) }; diff --git a/server/resolvers/TestResultOperations/saveTestResultCommon.js b/server/resolvers/TestResultOperations/saveTestResultCommon.js index 932ab7e41..13fa8d982 100644 --- a/server/resolvers/TestResultOperations/saveTestResultCommon.js +++ b/server/resolvers/TestResultOperations/saveTestResultCommon.js @@ -32,11 +32,10 @@ const saveTestResultCommon = async ({ throw new AuthenticationError(); } - // The populateData function will populate associations of JSON-based - // models, but not Sequelize-based models. This is why the - // convertTestResultToInput function is needed to make testResultPopulated - // equivalent to testPlanRun.testResults. const oldTestResults = testPlanRun.testResults; + // testResultPopulated is a TestResult type and has populated scenario, + // test, assertion etc. fields. It should just be a TestResultInput type for + // saving in the database. See graphql-schema.js for more info. const oldTestResult = convertTestResultToInput(testResultPopulated); const newTestResult = deepCustomMerge(oldTestResult, input, { @@ -51,7 +50,7 @@ const saveTestResultCommon = async ({ ], { pickKeys: ['id', 'testId', 'scenarioId', 'assertionId'], - excludeKeys: ['unexpectedBehaviors'] + excludeKeys: ['unexpectedBehaviors', 'unexpectedBehaviorNote'] } ); if (isCorrupted) { @@ -99,23 +98,31 @@ const assertTestResultIsValid = newTestResult => { } }; - const checkUnexpectedBehavior = unexpectedBehavior => { - if ( - (!!unexpectedBehavior.otherUnexpectedBehaviorText && - unexpectedBehavior.id !== 'OTHER') || - (!unexpectedBehavior.otherUnexpectedBehaviorText && - unexpectedBehavior.id === 'OTHER') - ) { + const checkUnexpectedBehavior = ( + unexpectedBehavior, + unexpectedBehaviorNote + ) => { + if (!unexpectedBehaviorNote && unexpectedBehavior.id === 'OTHER') { failed = true; } }; const checkScenarioResult = scenarioResult => { - if (!scenarioResult.output || !scenarioResult.unexpectedBehaviors) { + if ( + !scenarioResult.output || + !scenarioResult.unexpectedBehaviors || + (scenarioResult.unexpectedBehaviorNote && + !scenarioResult.unexpectedBehaviors.length) + ) { failed = true; } scenarioResult.assertionResults.forEach(checkAssertionResult); - scenarioResult.unexpectedBehaviors?.forEach(checkUnexpectedBehavior); + scenarioResult.unexpectedBehaviors?.forEach(unexpectedBehavior => { + checkUnexpectedBehavior( + unexpectedBehavior, + scenarioResult.unexpectedBehaviorNote + ); + }); }; newTestResult.scenarioResults.forEach(checkScenarioResult); diff --git a/server/scripts/populate-test-data/populateFakeTestResults.js b/server/scripts/populate-test-data/populateFakeTestResults.js index 0278cf0c0..9d66bc39d 100644 --- a/server/scripts/populate-test-data/populateFakeTestResults.js +++ b/server/scripts/populate-test-data/populateFakeTestResults.js @@ -198,7 +198,8 @@ const getFake = async ({ passed: true }) ), - unexpectedBehaviors: [] + unexpectedBehaviors: [], + unexpectedBehaviorNote: null })) }); @@ -220,22 +221,20 @@ const getFake = async ({ 'NO_OUTPUT'; break; case 'failingDueToUnexpectedBehaviors': - testResult.scenarioResults[0].unexpectedBehaviors.push({ - id: 'OTHER', - otherUnexpectedBehaviorText: 'Seeded other unexpected behavior' - }); + testResult.scenarioResults[0].unexpectedBehaviors.push('OTHER'); + testResult.scenarioResults[0].unexpectedBehaviorNote = + 'Seeded other unexpected behavior'; break; case 'failingDueToMultiple': testResult.scenarioResults[0].assertionResults[0].passed = false; testResult.scenarioResults[0].assertionResults[0].failedReason = 'INCORRECT_OUTPUT'; - testResult.scenarioResults[0].unexpectedBehaviors.push({ - id: 'EXCESSIVELY_VERBOSE' - }); - testResult.scenarioResults[0].unexpectedBehaviors.push({ - id: 'OTHER', - otherUnexpectedBehaviorText: 'Seeded other unexpected behavior' - }); + testResult.scenarioResults[0].unexpectedBehaviors.push( + 'EXCESSIVELY_VERBOSE' + ); + testResult.scenarioResults[0].unexpectedBehaviors.push('OTHER'); + testResult.scenarioResults[0].unexpectedBehaviorNote = + 'Seeded other unexpected behavior'; break; default: throw new Error(); diff --git a/server/tests/integration/graphql.test.js b/server/tests/integration/graphql.test.js index 18e18810c..ca5b57a87 100644 --- a/server/tests/integration/graphql.test.js +++ b/server/tests/integration/graphql.test.js @@ -337,8 +337,8 @@ describe('graphql', () => { __typename id text - otherUnexpectedBehaviorText } + unexpectedBehaviorNote } } testResultsLength @@ -755,8 +755,8 @@ const getMutationInputs = async () => { } unexpectedBehaviors { id - otherUnexpectedBehaviorText } + unexpectedBehaviorNote } } diff --git a/server/tests/integration/test-queue.test.js b/server/tests/integration/test-queue.test.js index f7278aed0..a8f669e78 100644 --- a/server/tests/integration/test-queue.test.js +++ b/server/tests/integration/test-queue.test.js @@ -474,8 +474,8 @@ describe('test queue', () => { output unexpectedBehaviors { text - otherUnexpectedBehaviorText } + unexpectedBehaviorNote } assertionResult { passed @@ -511,6 +511,7 @@ describe('test queue', () => { }, "scenarioResult": { "output": "automatically seeded sample output", + "unexpectedBehaviorNote": null, "unexpectedBehaviors": [], }, "testPlanRun": { @@ -527,6 +528,7 @@ describe('test queue', () => { }, "scenarioResult": { "output": "automatically seeded sample output", + "unexpectedBehaviorNote": null, "unexpectedBehaviors": [], }, "testPlanRun": { @@ -563,6 +565,7 @@ describe('test queue', () => { }, "scenarioResult": { "output": "automatically seeded sample output", + "unexpectedBehaviorNote": null, "unexpectedBehaviors": [], }, "testPlanRun": { @@ -579,6 +582,7 @@ describe('test queue', () => { }, "scenarioResult": { "output": "automatically seeded sample output", + "unexpectedBehaviorNote": null, "unexpectedBehaviors": [], }, "testPlanRun": { @@ -612,6 +616,7 @@ describe('test queue', () => { "assertionResult": null, "scenarioResult": { "output": "automatically seeded sample output", + "unexpectedBehaviorNote": null, "unexpectedBehaviors": [], }, "testPlanRun": { @@ -625,9 +630,9 @@ describe('test queue', () => { "assertionResult": null, "scenarioResult": { "output": "automatically seeded sample output", + "unexpectedBehaviorNote": "Seeded other unexpected behavior", "unexpectedBehaviors": [ { - "otherUnexpectedBehaviorText": "Seeded other unexpected behavior", "text": "Other", }, ], @@ -661,6 +666,7 @@ describe('test queue', () => { "assertionResult": null, "scenarioResult": { "output": "automatically seeded sample output", + "unexpectedBehaviorNote": null, "unexpectedBehaviors": [], }, "testPlanRun": { @@ -674,13 +680,12 @@ describe('test queue', () => { "assertionResult": null, "scenarioResult": { "output": "automatically seeded sample output", + "unexpectedBehaviorNote": "Seeded other unexpected behavior", "unexpectedBehaviors": [ { - "otherUnexpectedBehaviorText": null, "text": "Output is excessively verbose, e.g., includes redundant and/or irrelevant speech", }, { - "otherUnexpectedBehaviorText": "Seeded other unexpected behavior", "text": "Other", }, ], @@ -717,6 +722,7 @@ describe('test queue', () => { }, "scenarioResult": { "output": "automatically seeded sample output", + "unexpectedBehaviorNote": null, "unexpectedBehaviors": [], }, "testPlanRun": { @@ -733,13 +739,12 @@ describe('test queue', () => { }, "scenarioResult": { "output": "automatically seeded sample output", + "unexpectedBehaviorNote": "Seeded other unexpected behavior", "unexpectedBehaviors": [ { - "otherUnexpectedBehaviorText": null, "text": "Output is excessively verbose, e.g., includes redundant and/or irrelevant speech", }, { - "otherUnexpectedBehaviorText": "Seeded other unexpected behavior", "text": "Other", }, ],