Add ability to not error when no tests are run (#2650)

Can be useful when disabling tests in an automated fashion and you don't want the test to fail when all tests are disabled. --------- Signed-off-by: Brentley Jones <[email protected]>
bazelbuild · Feb 11, 2025 · be89110 · be89110
1 parent a71e87b
commit be89110
Show file tree

Hide file tree

Showing 3 changed files with 77 additions and 23 deletions.
diff --git a/apple/testing/default_runner/ios_xctestrun_runner.template.sh b/apple/testing/default_runner/ios_xctestrun_runner.template.sh
@@ -534,32 +534,34 @@ if [[ "$test_exit_code" -ne 0 ]]; then
   exit "$test_exit_code"
 fi
 
-parallel_testing_enabled=false
-if grep -q "-parallel-testing-enabled YES" "$testlog"; then
-  parallel_testing_enabled=true
-fi
-
-# Fail when bundle executes nothing
-no_tests_ran=false
-if [[ $parallel_testing_enabled == true ]]; then
-  # When executing tests in parallel, test start markers are absent when no
-  # tests are run.
-  test_execution_count=$(grep -c -e "Test suite '.*' started*" "$testlog")
-  if [[ "$test_execution_count" == "0" ]]; then
-    no_tests_ran=true
+if [[ "${ERROR_ON_NO_TESTS_RAN:-1}" == "1" ]]; then
+  parallel_testing_enabled=false
+  if grep -q "-parallel-testing-enabled YES" "$testlog"; then
+    parallel_testing_enabled=true
   fi
-else
-  # Assume the final 'Executed N tests' or 'Executed 1 test' is the
-  # total execution count for the test bundle.
-  test_target_execution_count=$(grep -e "Executed [[:digit:]]\{1,\} tests*," "$testlog" | tail -n1)
-  if echo "$test_target_execution_count" | grep -q -e "Executed 0 tests, with 0 failures"; then
-    no_tests_ran=true
+
+  # Fail when bundle executes nothing
+  no_tests_ran=false
+  if [[ $parallel_testing_enabled == true ]]; then
+    # When executing tests in parallel, test start markers are absent when no
+    # tests are run.
+    test_execution_count=$(grep -c -e "Test suite '.*' started*" "$testlog")
+    if [[ "$test_execution_count" == "0" ]]; then
+      no_tests_ran=true
+    fi
+  else
+    # Assume the final 'Executed N tests' or 'Executed 1 test' is the
+    # total execution count for the test bundle.
+    test_target_execution_count=$(grep -e "Executed [[:digit:]]\{1,\} tests*," "$testlog" | tail -n1)
+    if echo "$test_target_execution_count" | grep -q -e "Executed 0 tests, with 0 failures"; then
+      no_tests_ran=true
+    fi
   fi
-fi
 
-if [[ $no_tests_ran == true ]]; then
-  echo "error: no tests were executed, is the test bundle empty?" >&2
-  exit 1
+  if [[ $no_tests_ran == true ]]; then
+    echo "error: no tests were executed, is the test bundle empty?" >&2
+    exit 1
+  fi
 fi
 
 # When tests crash after they have reportedly completed, XCTest marks them as

diff --git a/test/ios_xctestrun_runner_ui_test.sh b/test/ios_xctestrun_runner_ui_test.sh
@@ -377,6 +377,33 @@ function test_ios_ui_test_with_filter() {
   expect_log "Executed 1 test, with 0 failures"
 }
 
+function test_ios_ui_test_with_filter_no_tests_ran_fail() {
+  create_sim_runners
+  create_ios_app
+  create_ios_ui_tests
+  ! do_ios_test --test_filter=PassingUITest/testInvalid //ios:PassingUITest|| fail "should fail"
+
+  expect_log "Test Suite 'PassingUITest.xctest' passed"
+  expect_log "Test Suite 'Selected tests' passed"
+  expect_log "Executed 0 tests, with 0 failures"
+  expect_log "error: no tests were executed, is the test bundle empty?"
+}
+
+function test_ios_ui_test_with_filter_no_tests_ran_pass() {
+  create_sim_runners
+  create_ios_app
+  create_ios_ui_tests
+  do_ios_test \
+    --test_env=ERROR_ON_NO_TESTS_RAN=0 \
+    --test_filter=PassingUITest/testInvalid \
+    //ios:PassingUITest \
+    || fail "should pass"
+
+  expect_log "Test Suite 'PassingUITest.xctest' passed"
+  expect_log "Test Suite 'Selected tests' passed"
+  expect_log "Executed 0 tests, with 0 failures"
+}
+
 function test_ios_ui_test_underscore_with_filter() {
   create_sim_runners
   create_ios_app

diff --git a/test/ios_xctestrun_runner_unit_test.sh b/test/ios_xctestrun_runner_unit_test.sh
@@ -696,6 +696,31 @@ function test_ios_unit_test_with_filter() {
   expect_log "Executed 1 test, with 0 failures"
 }
 
+function test_ios_unit_test_with_filter_no_tests_ran_fail() {
+  create_sim_runners
+  create_ios_unit_tests
+  ! do_ios_test --test_filter=PassingUnitTest/testInvalid //ios:PassingUnitTest || fail "should fail"
+
+  expect_log "Test Suite 'PassingUnitTest' passed"
+  expect_log "Test Suite 'PassingUnitTest.xctest' passed"
+  expect_log "Executed 0 tests, with 0 failures"
+  expect_log "error: no tests were executed, is the test bundle empty?"
+}
+
+function test_ios_unit_test_with_filter_no_tests_ran_pass() {
+  create_sim_runners
+  create_ios_unit_tests
+  do_ios_test \
+    --test_env=ERROR_ON_NO_TESTS_RAN=0 \
+    --test_filter=PassingUnitTest/testInvalid \
+    //ios:PassingUnitTest \
+    || fail "should pass"
+
+  expect_log "Test Suite 'PassingUnitTest' passed"
+  expect_log "Test Suite 'PassingUnitTest.xctest' passed"
+  expect_log "Executed 0 tests, with 0 failures"
+}
+
 function test_ios_unit_test_with_multi_filter() {
   create_sim_runners
   create_ios_unit_tests