Skip to content

Commit 8ea4def

Browse files
howard-ehoward-e
authored
fix: Include result of undesirable behaviors assertions in summary of failing assertions (#1317)
* Ignore implicitly checked MAY assertion in shouldShowFailingAssertionsSummary boolean * Include failed undesirable behaviors in failing assertions summary table * Correct getMetrics function properly count undesirable behaviors as failures as it is derived elsewhere * Update snapshots
1 parent 518efb1 commit 8ea4def

File tree

7 files changed

+176
-81
lines changed

7 files changed

+176
-81
lines changed

‎client/components/TestRun/TestNavigator.jsx

+5-1
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,11 @@ const TestNavigator = ({
3434
);
3535

3636
const shouldShowFailingAssertionsSummary = useMemo(() => {
37-
return isVendor && testPlanReport.metrics.assertionsFailedCount > 0;
37+
return (
38+
isVendor &&
39+
(testPlanReport.metrics.mustAssertionsFailedCount > 0 ||
40+
testPlanReport.metrics.shouldAssertionsFailedCount > 0)
41+
);
3842
}, [isVendor, testPlanReport]);
3943

4044
return (

‎client/hooks/useFailingAssertions.js

+54-20
Original file line numberDiff line numberDiff line change
@@ -9,27 +9,61 @@ export const useFailingAssertions = testPlanReport => {
99
return testPlanReport.finalizedTestResults.flatMap(
1010
(testResult, testIndex) => {
1111
return testResult.scenarioResults.flatMap(scenarioResult => {
12-
return (
13-
scenarioResult.assertionResults
14-
.filter(assertionResult => !assertionResult.passed)
15-
// We only want to show MUST and SHOULD assertions
16-
.filter(
17-
assertionResult =>
18-
assertionResult.assertion.priority !== 'MAY' &&
19-
assertionResult.assertion.priority !== 'EXCLUDE'
20-
)
21-
.map(assertionResult => ({
22-
testResultId: testResult.id,
23-
testIndex,
24-
testTitle: testResult.test.title,
25-
scenarioCommands: scenarioResult.scenario.commands
26-
.map(cmd => cmd.text)
27-
.join(', '),
28-
assertionText: assertionResult.assertion.text,
29-
priority: assertionResult.assertion.priority,
30-
output: scenarioResult.output
31-
}))
12+
const commonResult = {
13+
testResultId: testResult.id,
14+
testIndex,
15+
testTitle: testResult.test.title,
16+
scenarioCommands: scenarioResult.scenario.commands
17+
.map((cmd, index) => {
18+
if (index === scenarioResult.scenario.commands.length - 1) {
19+
return cmd.text;
20+
}
21+
// Prevent instances of duplicated setting in brackets.
22+
// eg. Down Arrow (virtual cursor active) then Down Arrow (virtual cursor active)
23+
//
24+
// Expectation is Down Arrow then Down Arrow (virtual cursor active), because the setting will always be
25+
// the same for the listed key combination.
26+
//
27+
// Some revision of how that key combination + setting is rendered may be useful
28+
return cmd.text.split(' (')[0];
29+
})
30+
.join(' then ')
31+
};
32+
33+
const assertionResults = scenarioResult.assertionResults
34+
.filter(assertionResult => !assertionResult.passed)
35+
// We only want to show MUST and SHOULD assertions
36+
.filter(
37+
assertionResult =>
38+
assertionResult.assertion.priority !== 'MAY' &&
39+
assertionResult.assertion.priority !== 'EXCLUDE'
40+
)
41+
.map(assertionResult => ({
42+
...commonResult,
43+
assertionText: assertionResult.assertion.text,
44+
priority: assertionResult.assertion.priority,
45+
output: scenarioResult.output
46+
}));
47+
48+
const unexpectedResults = scenarioResult.unexpectedBehaviors.map(
49+
unexpectedBehavior => ({
50+
...commonResult,
51+
assertionText:
52+
unexpectedBehavior.impact.toLowerCase() === 'moderate'
53+
? 'Other behaviors that create moderate negative impacts are not exhibited'
54+
: unexpectedBehavior.impact.toLowerCase() === 'severe'
55+
? 'Other behaviors that create severe negative impacts are not exhibited'
56+
: 'N/A',
57+
priority:
58+
unexpectedBehavior.impact.toLowerCase() === 'moderate'
59+
? 'SHOULD'
60+
: unexpectedBehavior.impact.toLowerCase() === 'severe'
61+
? 'MUST'
62+
: 'N/A',
63+
output: unexpectedBehavior.text
64+
})
3265
);
66+
return [...assertionResults, ...unexpectedResults];
3367
});
3468
}
3569
);

‎client/tests/e2e/snapshots/saved/_candidate-review.html

+3-3
Original file line numberDiff line numberDiff line change
@@ -167,7 +167,7 @@ <h3>
167167
style="justify-content: center"
168168
><i
169169
><b>3 assertions</b> failed across
170-
<b>2 tests</b> run with <b>1 browser</b></i
170+
<b>3 tests</b> run with <b>1 browser</b></i
171171
></span
172172
>
173173
</td>
@@ -241,7 +241,7 @@ <h3>
241241
style="justify-content: center"
242242
><i
243243
><b>3 assertions</b> failed across
244-
<b>2 tests</b> run with <b>1 browser</b></i
244+
<b>3 tests</b> run with <b>1 browser</b></i
245245
></span
246246
>
247247
</td>
@@ -317,7 +317,7 @@ <h3>
317317
style="justify-content: center"
318318
><i
319319
><b>3 assertions</b> failed across
320-
<b>2 tests</b> run with <b>1 browser</b></i
320+
<b>3 tests</b> run with <b>1 browser</b></i
321321
></span
322322
>
323323
</td>

‎client/tests/e2e/snapshots/saved/_candidate-test-plan_24_1#summary.html

+16-1
Original file line numberDiff line numberDiff line change
@@ -305,7 +305,7 @@ <h1 id="failing-assertions-heading">
305305
<div
306306
class="failing-assertions-summary-table-container">
307307
<p>
308-
2 assertions failed across 26 commands in 2 tests.
308+
3 assertions failed across 26 commands in 3 tests.
309309
</p>
310310
<div class="table-responsive">
311311
<table
@@ -344,6 +344,21 @@ <h1 id="failing-assertions-heading">
344344
<td>Role 'button' is conveyed</td>
345345
<td>automatically seeded sample output</td>
346346
</tr>
347+
<tr>
348+
<td>
349+
<a href="/candidate-test-plan/24/1#5"
350+
>Close a modal dialog using a button in
351+
reading mode</a
352+
>
353+
</td>
354+
<td>Space</td>
355+
<td>SHOULD</td>
356+
<td>
357+
Other behaviors that create moderate
358+
negative impacts are not exhibited
359+
</td>
360+
<td>Other</td>
361+
</tr>
347362
</tbody>
348363
</table>
349364
</div>

‎client/tests/e2e/snapshots/saved/_report_67_targets_20.html

+33
Original file line numberDiff line numberDiff line change
@@ -370,6 +370,39 @@ <h2>Results for NVDA 2023.3.3 and Chrome</h2>
370370
<h2 id="failing-assertions-heading">
371371
Summary of Failing Assertions (0 must, 1 should)
372372
</h2>
373+
<p>1 assertions failed across 28 commands in 1 tests.</p>
374+
<div class="table-responsive">
375+
<table
376+
aria-labelledby="failing-assertions-heading"
377+
class="table table-bordered">
378+
<thead>
379+
<tr>
380+
<th>Test Name</th>
381+
<th>Command</th>
382+
<th>Assertion Priority</th>
383+
<th>Assertion</th>
384+
<th>NVDA Response</th>
385+
</tr>
386+
</thead>
387+
<tbody>
388+
<tr>
389+
<td>
390+
<a
391+
href="/report/67/targets/20#result-ODhiYeyIxMiI6MjB9TRkYW"
392+
>Navigate backwards to a mixed checkbox in reading mode</a
393+
>
394+
</td>
395+
<td>Shift+X</td>
396+
<td>SHOULD</td>
397+
<td>
398+
Other behaviors that create moderate negative impacts are
399+
not exhibited
400+
</td>
401+
<td>Other</td>
402+
</tr>
403+
</tbody>
404+
</table>
405+
</div>
373406
<div class="test-result-heading">
374407
<h2 id="result-NjAyYeyIxMiI6MjB92VjN2" tabindex="-1">
375408
Test 1: Navigate forwards to a mixed checkbox in reading

‎server/migrations/20250206165746-updateMetrics.js

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
'use strict';
2+
3+
const { recalculateMetrics, dumpTable } = require('./utils');
4+
5+
/** @type {import('sequelize-cli').Migration} */
6+
module.exports = {
7+
async up(queryInterface) {
8+
return queryInterface.sequelize.transaction(async transaction => {
9+
await dumpTable('TestPlanReport');
10+
await recalculateMetrics(queryInterface, transaction);
11+
});
12+
},
13+
14+
async down() {
15+
// Restore dumped TestPlanReport table if needed
16+
}
17+
};

‎shared/getMetrics.js

+48-56
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,11 @@ const countTests = ({
1111
}) => {
1212
const countScenarioResult = scenarioResult => {
1313
return (
14-
scenarioResult?.assertionResults?.every(
14+
(scenarioResult?.assertionResults?.every(
1515
assertionResult => assertionResult.passed
16-
) || 0
16+
) &&
17+
scenarioResult.unexpectedBehaviors.length === 0) ||
18+
0
1719
);
1820
};
1921

@@ -33,6 +35,24 @@ const countTests = ({
3335
return countScenarioResult(scenarioResult);
3436
};
3537

38+
const countAvailableData = (
39+
countScenarioResult,
40+
{ testPlanReport, testResult, scenarioResult }
41+
) => {
42+
const countTestResult = testResult => {
43+
return sum(testResult?.scenarioResults?.map(countScenarioResult) || []);
44+
};
45+
const countTestPlanReport = testPlanReport => {
46+
return sum(
47+
testPlanReport?.finalizedTestResults?.map(countTestResult) || []
48+
);
49+
};
50+
51+
if (testPlanReport) return countTestPlanReport(testPlanReport);
52+
if (testResult) return countTestResult(testResult);
53+
return countScenarioResult(scenarioResult);
54+
};
55+
3656
const countAssertions = ({
3757
testPlanReport, // Choose one to provide
3858
testResult, // Choose one to provide
@@ -77,47 +97,33 @@ const countAssertions = ({
7797
if (passedOnly) return all.filter(each => each.passed).length;
7898
return all.length;
7999
};
80-
const countTestResult = testResult => {
81-
return sum(testResult?.scenarioResults?.map(countScenarioResult) || []);
82-
};
83-
const countTestPlanReport = testPlanReport => {
84-
return sum(
85-
testPlanReport?.finalizedTestResults?.map(countTestResult) || []
86-
);
87-
};
88-
89-
if (testPlanReport) return countTestPlanReport(testPlanReport);
90-
if (testResult) return countTestResult(testResult);
91-
return countScenarioResult(scenarioResult);
100+
return countAvailableData(countScenarioResult, {
101+
testPlanReport,
102+
testResult,
103+
scenarioResult
104+
});
92105
};
93106

94107
const countUnexpectedBehaviors = ({
95-
scenarioResult, // Choose one to provide
108+
testPlanReport, // Choose one to provide
96109
testResult, // Choose one to provide
97-
testPlanReport // Choose one to provide
110+
scenarioResult // Choose one to provide
98111
}) => {
99112
const countScenarioResult = scenarioResult => {
100113
return scenarioResult?.unexpectedBehaviors?.length || 0;
101114
};
102-
const countTestResult = testResult => {
103-
return sum(testResult?.scenarioResults?.map(countScenarioResult) || []);
104-
};
105-
const countTestPlanReport = testPlanReport => {
106-
return sum(
107-
testPlanReport?.finalizedTestResults?.map(countTestResult) || []
108-
);
109-
};
110-
111-
if (testPlanReport) return countTestPlanReport(testPlanReport);
112-
if (testResult) return countTestResult(testResult);
113-
return countScenarioResult(scenarioResult);
115+
return countAvailableData(countScenarioResult, {
116+
testPlanReport,
117+
testResult,
118+
scenarioResult
119+
});
114120
};
115121

116122
const countUnexpectedBehaviorsImpact = (
117123
{
118-
scenarioResult, // Choose one to provide
124+
testPlanReport, // Choose one to provide
119125
testResult, // Choose one to provide
120-
testPlanReport // Choose one to provide
126+
scenarioResult // Choose one to provide
121127
},
122128
impact
123129
) => {
@@ -126,40 +132,26 @@ const countUnexpectedBehaviorsImpact = (
126132
? 1
127133
: 0;
128134
};
129-
const countTestResult = testResult => {
130-
return sum(testResult?.scenarioResults?.map(countScenarioResult) || []);
131-
};
132-
const countTestPlanReport = testPlanReport => {
133-
return sum(
134-
testPlanReport?.finalizedTestResults?.map(countTestResult) || []
135-
);
136-
};
137-
138-
if (testPlanReport) return countTestPlanReport(testPlanReport);
139-
if (testResult) return countTestResult(testResult);
140-
return countScenarioResult(scenarioResult);
135+
return countAvailableData(countScenarioResult, {
136+
testPlanReport,
137+
testResult,
138+
scenarioResult
139+
});
141140
};
142141

143142
const countCommands = ({
144-
scenarioResult, // Choose one to provide
143+
testPlanReport, // Choose one to provide
145144
testResult, // Choose one to provide
146-
testPlanReport // Choose one to provide
145+
scenarioResult // Choose one to provide
147146
}) => {
148147
const countScenarioResult = scenarioResult => {
149148
return scenarioResult?.scenario?.commands?.length ? 1 : 0;
150149
};
151-
const countTestResult = testResult => {
152-
return sum(testResult?.scenarioResults?.map(countScenarioResult) || []);
153-
};
154-
const countTestPlanReport = testPlanReport => {
155-
return sum(
156-
testPlanReport?.finalizedTestResults?.map(countTestResult) || []
157-
);
158-
};
159-
160-
if (testPlanReport) return countTestPlanReport(testPlanReport);
161-
if (testResult) return countTestResult(testResult);
162-
return countScenarioResult(scenarioResult);
150+
return countAvailableData(countScenarioResult, {
151+
testPlanReport,
152+
testResult,
153+
scenarioResult
154+
});
163155
};
164156

165157
const calculateAssertionPriorityCounts = (result, priority) => {

0 commit comments

Comments
 (0)