Skip to content

Commit 51b6097

Browse files
committed
Enhance test runner to detect and prioritize incomplete test runs
This commit adds incomplete test detection to perl_test_runner.pl to help identify high-impact fix opportunities where tests crash before completion. ## Problem Previously, when a test file crashed (e.g., pat_rt_report.t planning 2514 tests but crashing at test 193), the runner would only report: - status: 'fail' - ok_count: 193 This didn't highlight that 2,321 tests were BLOCKED and never ran, making it hard to prioritize high-impact fixes. ## Solution 1. **Detect incomplete runs:** Parse '1..N' plan and compare to actual tests run 2. **Calculate blocked tests:** incomplete_tests = planned - actual_tests_run 3. **Count as failures:** Add incomplete_tests to not_ok_count for visibility 4. **Mark status:** Set status='incomplete' for special handling 5. **Prioritized display:** New section shows top incomplete files by blocked count ## Features Added ### Enhanced parse_tap_output(): - Tracks planned_tests from '1..N' line - Tracks actual_tests_run from ok/not ok count - Calculates incomplete_tests (blocked tests that never ran) - Detects 'Looks like you planned N tests but ran M' message - Adds incomplete_tests to not_ok_count for pass rate visibility - Sets status='incomplete' when tests are blocked ### New print_incomplete_opportunities(): - Shows "HIGH-PRIORITY OPPORTUNITIES" section - Lists incomplete tests sorted by blocked test count (descending) - Displays: file name, blocked count, actual/planned tests - Shows first error for each incomplete test - Calculates total blocked tests across all files ### Updated print_summary(): - Calls print_incomplete_opportunities() after main summary - Shows incomplete test count in file summary - Pass rate now reflects blocked tests as failures ## Example Output ``` TEST SUMMARY: Total files: 9 Passed: 7 Failed: 0 Errors: 0 Timeouts: 0 Incomplete: 2 Total tests: 502 OK: 435 Not OK: 67 # Includes 62 blocked tests! Pass rate: 86.7% 🎯 HIGH-PRIORITY OPPORTUNITIES (Incomplete Test Runs): (Tests that crashed/failed before completion - fixing these unlocks many tests at once) Total incomplete tests: 2 files, 62 blocked tests Top 2 incomplete test files (by blocked test count): 1. lex.t Blocked: 61 tests (68/129 ran) Error: (?{...}) code blocks in regex not implemented... 2. term.t Blocked: 1 tests (6/7 ran) ``` ## Impact This enhancement helps prioritize fixes by highlighting: - Which test files have the most blocked tests - What errors caused tests to stop running - How many tests could be unlocked by fixing each issue For example, the $^R fix unlocked 2,266 tests in pat_rt_report.t by fixing a single crash at test 193. This feature makes such opportunities visible. ## Files Modified - dev/tools/perl_test_runner.pl: Added incomplete detection and display
1 parent da6f9c5 commit 51b6097

File tree

1 file changed

+94
-4
lines changed

1 file changed

+94
-4
lines changed

dev/tools/perl_test_runner.pl

100644100755
Lines changed: 94 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@
4444
# Global state
4545
my %results;
4646
my %summary = (
47-
pass => 0, fail => 0, error => 0, timeout => 0,
47+
pass => 0, fail => 0, error => 0, timeout => 0, incomplete => 0,
4848
total_ok => 0, total_not_ok => 0, total_tests => 0,
4949
total_skipped => 0, total_todo => 0
5050
);
@@ -212,7 +212,7 @@ sub process_test_result {
212212

213213
# Print result
214214
my %status_chars = (
215-
pass => '', fail => '', error => '!', timeout => 'T'
215+
pass => '', fail => '', error => '!', timeout => 'T', incomplete => 'I'
216216
);
217217
my $char = $status_chars{$result->{status}} || '?';
218218

@@ -297,6 +297,7 @@ sub run_single_test {
297297
return {
298298
status => 'timeout',
299299
ok_count => 0, not_ok_count => 0, total_tests => 0,
300+
planned_tests => 0, actual_tests_run => 0, incomplete_tests => 0,
300301
skip_count => 0, todo_count => 0,
301302
errors => ['Test timed out'], missing_features => []
302303
};
@@ -352,6 +353,8 @@ sub parse_tap_output {
352353
my ($ok_count, $not_ok_count, $total_tests) = (0, 0, 0);
353354
my ($skip_count, $todo_count) = (0, 0);
354355
my (@errors, @missing_features);
356+
my $planned_tests = 0; # From 1..N line
357+
my $actual_tests_run = 0; # Actual ok + not ok count
355358

356359
# Parse TAP output
357360
for my $line (@lines) {
@@ -360,23 +363,32 @@ sub parse_tap_output {
360363

361364
# Test plan
362365
if ($line =~ /^1\.\.(\d+)/) {
366+
$planned_tests = $1;
363367
$total_tests = $1;
364368
next;
365369
}
366370

367371
# Test results
368372
if ($line =~ /^ok\s+\d+/) {
369373
$ok_count++;
374+
$actual_tests_run++;
370375
$skip_count++ if $line =~ /#\s*skip/i;
371376
$todo_count++ if $line =~ /#\s*todo/i;
372377
next;
373378
}
374379

375380
if ($line =~ /^not ok\s+\d+/) {
376381
$not_ok_count++;
382+
$actual_tests_run++;
377383
next;
378384
}
379385

386+
# Detect "Looks like you planned N tests but ran M" message
387+
if ($line =~ /looks like you planned (\d+) tests but ran (\d+)/i) {
388+
$planned_tests = $1 unless $planned_tests;
389+
$actual_tests_run = $2 unless $actual_tests_run;
390+
}
391+
380392
# Look for errors and missing features
381393
my $line_lower = lc($line);
382394
if ($line_lower =~ /error|fatal|died|exception|abort|not implemented|unimplemented|unsupported|syntax error|compilation failed|can't locate|undefined subroutine|bareword not allowed/) {
@@ -394,12 +406,29 @@ sub parse_tap_output {
394406
}
395407
}
396408

409+
# Calculate actual tests run if not already set
410+
$actual_tests_run = $ok_count + $not_ok_count unless $actual_tests_run;
411+
412+
# Calculate incomplete tests (tests that were blocked/never ran)
413+
my $incomplete_tests = 0;
414+
if ($planned_tests > 0 && $actual_tests_run < $planned_tests) {
415+
$incomplete_tests = $planned_tests - $actual_tests_run;
416+
}
417+
397418
# If no test plan found, use count
398419
$total_tests = $ok_count + $not_ok_count if $total_tests == 0;
399420

421+
# Add incomplete tests to not_ok_count for visibility in pass rate
422+
# These are tests that were planned but never ran due to crashes
423+
$not_ok_count += $incomplete_tests;
424+
$total_tests = $planned_tests if $planned_tests > $total_tests;
425+
400426
# Determine status
401427
my $status;
402-
if ($ok_count == 0 && $not_ok_count == 0) {
428+
if ($incomplete_tests > 0) {
429+
# Incomplete run is a special high-priority case
430+
$status = 'incomplete';
431+
} elsif ($ok_count == 0 && $not_ok_count == 0) {
403432
$status = $exit_code == 0 ? 'pass' : 'error';
404433
} elsif ($not_ok_count == 0 && $ok_count > 0) {
405434
$status = 'pass';
@@ -418,6 +447,9 @@ sub parse_tap_output {
418447
ok_count => $ok_count,
419448
not_ok_count => $not_ok_count,
420449
total_tests => $total_tests,
450+
planned_tests => $planned_tests,
451+
actual_tests_run => $actual_tests_run,
452+
incomplete_tests => $incomplete_tests,
421453
skip_count => $skip_count,
422454
todo_count => $todo_count,
423455
errors => \@errors,
@@ -428,11 +460,12 @@ sub parse_tap_output {
428460

429461
sub print_summary {
430462
print "\nTEST SUMMARY:\n";
431-
printf " Total files: %d\n", $summary{pass} + $summary{fail} + $summary{error} + $summary{timeout};
463+
printf " Total files: %d\n", $summary{pass} + $summary{fail} + $summary{error} + $summary{timeout} + $summary{incomplete};
432464
printf " Passed: %d\n", $summary{pass};
433465
printf " Failed: %d\n", $summary{fail};
434466
printf " Errors: %d\n", $summary{error};
435467
printf " Timeouts: %d\n", $summary{timeout};
468+
printf " Incomplete: %d\n", $summary{incomplete};
436469
print "\n";
437470
printf " Total tests: %d\n", $summary{total_tests};
438471
printf " OK: %d\n", $summary{total_ok};
@@ -444,6 +477,63 @@ sub print_summary {
444477
my $pass_rate = ($summary{total_ok} / $summary{total_tests}) * 100;
445478
printf " Pass rate: %.1f%%\n", $pass_rate;
446479
}
480+
481+
# Show incomplete test opportunities
482+
print_incomplete_opportunities();
483+
}
484+
485+
sub print_incomplete_opportunities {
486+
# Find all incomplete tests and sort by blocked test count
487+
my @incomplete_tests;
488+
for my $file (keys %results) {
489+
my $result = $results{$file};
490+
if ($result->{status} eq 'incomplete' && $result->{incomplete_tests} > 0) {
491+
push @incomplete_tests, {
492+
file => $file,
493+
incomplete => $result->{incomplete_tests},
494+
planned => $result->{planned_tests},
495+
actual => $result->{actual_tests_run},
496+
errors => $result->{errors},
497+
};
498+
}
499+
}
500+
501+
return unless @incomplete_tests;
502+
503+
# Sort by blocked test count (descending)
504+
@incomplete_tests = sort { $b->{incomplete} <=> $a->{incomplete} } @incomplete_tests;
505+
506+
print "\n🎯 HIGH-PRIORITY OPPORTUNITIES (Incomplete Test Runs):\n";
507+
print "(Tests that crashed/failed before completion - fixing these unlocks many tests at once)\n\n";
508+
509+
my $total_blocked = 0;
510+
for my $test (@incomplete_tests) {
511+
$total_blocked += $test->{incomplete};
512+
}
513+
514+
printf " Total incomplete tests: %d files, %d blocked tests\n\n", scalar(@incomplete_tests), $total_blocked;
515+
516+
# Show top 10 or all if fewer
517+
my $show_count = @incomplete_tests > 10 ? 10 : scalar(@incomplete_tests);
518+
print " Top $show_count incomplete test files (by blocked test count):\n\n";
519+
520+
for my $i (0 .. $show_count - 1) {
521+
my $test = $incomplete_tests[$i];
522+
printf " %2d. %-40s Blocked: %4d tests (%d/%d ran)\n",
523+
$i + 1,
524+
$test->{file},
525+
$test->{incomplete},
526+
$test->{actual},
527+
$test->{planned};
528+
529+
# Show first error if available
530+
if ($test->{errors} && @{$test->{errors}} > 0) {
531+
my $error = $test->{errors}[0];
532+
# Truncate long errors
533+
$error = substr($error, 0, 80) . "..." if length($error) > 80;
534+
print " Error: $error\n";
535+
}
536+
}
447537
}
448538

449539
sub print_feature_impact {

0 commit comments

Comments
 (0)