qualcomm-linux · Milosz Wasilewski (mwasilew) · Aug 28, 2026 · Aug 28, 2026 · Aug 28, 2026 · Aug 28, 2026
@@ -15,6 +15,13 @@ inputs:
   summary_title:
     description: "Title for the summary part of the report"
     required: false
+  known_failures_file:
+    description: >
+      Path to the YAML file listing the known failures of the tested build
+      variant. Relative to the workspace. Missing file means no known failure.
+      See .github/known-failures/README.md for the file format.
+    required: false
+    default: ""
 outputs:
   artifact_id:
     description: "ID of the uploaded artifact"
@@ -43,6 +50,7 @@ runs:
           INPUTS_PREFIX: ${{ inputs.prefix }}
           INPUTS_SUMMARY_FILE_NAME: ${{ inputs.summary_file_name }}
           INPUTS_SUMMARY_TITLE: ${{ inputs.summary_title }}
+          INPUTS_KNOWN_FAILURES_FILE: ${{ inputs.known_failures_file }}
         run: |
           set -x
           # When a failed LAVA job is re-run, GitHub keeps the original failed
@@ -70,6 +78,9 @@ runs:
           do
               JOB_ID=$(cat "$TESTJOB" | jq ".id")
               JOB_URL="https://lava.infra.foundries.io/results/$JOB_ID"
+              # the job log renders every line as <code id="L<n>">, so a test
+              # result can be linked straight to the line it was logged on
+              JOB_LOG_URL="https://lava.infra.foundries.io/scheduler/job/$JOB_ID"
               JOB_DETAILS=$(curl -s "https://lava.infra.foundries.io/api/v0.2/jobs/$JOB_ID/")
               JOB_HEALTH=$(echo "$JOB_DETAILS" | jq -r ".health")
               JOB_STATE=$(echo "$JOB_DETAILS" | jq -r ".state")
@@ -89,7 +100,12 @@ runs:
                           SUITE_ID=$(echo "$SUITE" | jq -r ".id")
                           SUITE_TESTS=$(curl -s "https://lava.infra.foundries.io/api/v0.2/jobs/$JOB_ID/suites/$SUITE_ID/tests/")
                           if [ "$SUITE_NAME" != "lava" ]; then
-                              echo "$SUITE_TESTS" | jq ".results[] | {(.name): {result: .result, url: .resource_uri}} | to_entries | .[]"
+                              # link to the log line the result was reported on
+                              # rather than to the API resource of the test case
+                              echo "$SUITE_TESTS" | jq --arg log "$JOB_LOG_URL" '.results[]
+                                  | {(.name): {result: .result,
+                                               url: (if .start_log_line then "\($log)#L\(.start_log_line)" else $log end)}}
+                                  | to_entries | .[]'
                           fi
                       fi
                   else
@@ -99,7 +115,9 @@ runs:
                           TEST_RESULT_BOOT="fail"
                       fi
                       TEST_NAME="boot"
-                      TEST_URL="${JOB_URL}"
+                      # a boot job has no test case to point at, so link to the
+                      # log itself, which is what a boot failure is read from
+                      TEST_URL="${JOB_LOG_URL}"
                       TEST_RESULT_OBJ=$(jq --arg url "$TEST_URL" --arg result "$TEST_RESULT_BOOT" -n -c '$ARGS.named')
                       jq -n -c --arg key "$TEST_NAME" --argjson value "$TEST_RESULT_OBJ" '$ARGS.named'
                   fi
@@ -108,21 +126,75 @@ runs:
           # combine entries from boot jobs and pre-merge jobs
           done | jq -s -c 'reduce .[] as $i ({}; .[$i.key] = ((.[$i.key] // {}) + $i.value))')
           echo "$INPUT"
+
+          # Load the list of known failures of this build variant. Every build
+          # variant has its own list, see .github/known-failures/README.md.
+          # A missing or empty file simply means "no known failure".
+          # An entry is either a bare test name or a mapping with a "test" and an
+          # optional "comment" holding the issue it is tracked in. Both forms are
+          # normalised to {device: {test name: comment}}.
+          KNOWN_FAILURES="{}"
+          if [ -n "${INPUTS_KNOWN_FAILURES_FILE}" ]; then
+              if [ -f "${GITHUB_WORKSPACE}/${INPUTS_KNOWN_FAILURES_FILE}" ]; then
+                  KNOWN_FAILURES_RAW=$(yq -o=json -I=0 "${GITHUB_WORKSPACE}/${INPUTS_KNOWN_FAILURES_FILE}" | jq -c '. // {}')
+                  KNOWN_FAILURES=$(echo "${KNOWN_FAILURES_RAW}" | jq -c '
+                    with_entries(.value |= ((. // []) | map(
+                      if type == "string" then {key: ., value: ""}
+                      elif type == "object" and (.test | type) == "string" then {key: .test, value: (.comment // "" | tostring)}
+                      else empty
+                      end) | from_entries))')
+                  # entries that match neither form are dropped by the mapping above
+                  RAW_COUNT=$(echo "${KNOWN_FAILURES_RAW}" | jq '[.[] | (. // []) | length] | add // 0')
+                  KNOWN_COUNT=$(echo "${KNOWN_FAILURES}" | jq '[.[] | length] | add // 0')
+                  if [ "${RAW_COUNT}" != "${KNOWN_COUNT}" ]; then
+                      echo "::warning::${INPUTS_KNOWN_FAILURES_FILE}: ignored $((RAW_COUNT - KNOWN_COUNT)) malformed entry/entries"
+                  fi
+              else
+                  echo "::warning::known failures file ${INPUTS_KNOWN_FAILURES_FILE} not found"
+              fi
+          fi
+          echo "${KNOWN_FAILURES}"
+
+          # Turn the LAVA result of every test into the status that is reported.
+          # A failure that is on the known failures list is reported as "known
+          # failure" and counted as a pass. A test that passes while it is on
+          # the list is reported as "unexpected pass" and counted as a failure,
+          # so that stale entries get noticed and removed from the list. The
+          # comment of the entry is carried over so it can be reported too.
+          INPUT=$(echo "$INPUT" | jq -c --argjson known "${KNOWN_FAILURES}" '
+            with_entries(
+              # a device specific entry overrides the one listed for all devices
+              (($known["*"] // {}) + ($known[.key] // {})) as $known_tests
+              | .value |= with_entries(
+                  .key as $test
+                  | ($known_tests | has($test)) as $is_known
+                  | .value.status = (
+                      if .value.result == "fail" and $is_known then "known failure"
+                      elif .value.result == "pass" and $is_known then "unexpected pass"
+                      else .value.result
+                      end)
+                  | if $is_known then .value.comment = $known_tests[$test] else . end)
+            )')
+          echo "$INPUT"
+
           DEVICES=$(echo "$INPUT" | jq -r 'keys[]' | sort)
 
           RESULTS=$(echo "$INPUT" | jq -r '.[] | keys[]?' | sort -u)
 
           # Print collapsible box
           echo "<details>" >> "${INPUTS_SUMMARY_FILE_NAME}"
           # Print section summary
-          TOTAL_PASS=$(echo "${INPUT}" | jq '[.. | objects | select(.result? == "pass")] | length')
-          TOTAL_FAIL=$(echo "${INPUT}" | jq '[.. | objects | select(.result? == "fail")] | length')
+          TOTAL_PASS=$(echo "${INPUT}" | jq '[.. | objects | select(.status? == "pass")] | length')
+          TOTAL_KNOWN_FAIL=$(echo "${INPUT}" | jq '[.. | objects | select(.status? == "known failure")] | length')
+          # an unexpected pass counts as a failure: the known failures list is
+          # out of date and has to be updated
+          TOTAL_FAIL=$(echo "${INPUT}" | jq '[.. | objects | select(.status? == "fail" or .status? == "unexpected pass")] | length')
           TOTAL=$(echo "${INPUT}" | jq '[.. | objects | select(.result)] | length')
           SUMMARY_TITLE=""
           if [ "${INPUTS_SUMMARY_TITLE}" != "" ]; then
             SUMMARY_TITLE="${INPUTS_SUMMARY_TITLE}<br/>"
           fi
-          echo "<summary>${SUMMARY_TITLE}Pass: $TOTAL_PASS | Fail: $TOTAL_FAIL | Total: $TOTAL </summary>" >> "${INPUTS_SUMMARY_FILE_NAME}"
+          echo "<summary>${SUMMARY_TITLE}Pass: $TOTAL_PASS | Known failures: $TOTAL_KNOWN_FAIL | Fail: $TOTAL_FAIL | Total: $TOTAL </summary>" >> "${INPUTS_SUMMARY_FILE_NAME}"
           echo "" >> "${INPUTS_SUMMARY_FILE_NAME}"
           echo "" >> "${INPUTS_SUMMARY_FILE_NAME}"
 
@@ -151,10 +223,12 @@ runs:
             printf "| %s |" "$R" >> "${INPUTS_SUMMARY_FILE_NAME}"
 
             for D in $DEVICES; do
-              VALUE=$(echo "$INPUT" | jq -r --arg d "$D" --arg r "$R" '.[$d][$r].result // ""')
+              VALUE=$(echo "$INPUT" | jq -r --arg d "$D" --arg r "$R" '.[$d][$r].status // ""')
               URL=$(echo "$INPUT" | jq -r --arg d "$D" --arg r "$R" '.[$d][$r].url // ""')
               CHECKMARK=":white_check_mark:"
               if [ "${VALUE}" = "fail" ]; then CHECKMARK=":x:"; fi
+              if [ "${VALUE}" = "unexpected pass" ]; then CHECKMARK=":x:"; fi
+              if [ "${VALUE}" = "known failure" ]; then CHECKMARK=":ballot_box_with_check:"; fi
               if [ "${VALUE}" = "skip" ]; then CHECKMARK=":warning:"; fi
               if [ -z "${VALUE}" ]; then CHECKMARK=":no_entry_sign:"; fi
               printf " %s [%s](%s) |" "$CHECKMARK" "$VALUE" "$URL"
@@ -164,6 +238,24 @@ runs:
           done
           echo "" >> "${INPUTS_SUMMARY_FILE_NAME}"
           echo "</details>" >> "${INPUTS_SUMMARY_FILE_NAME}"
+
+          # Print the entries of the known failures list that were applied,
+          # together with the issue each one is annotated with
+          KNOWN_ROWS=$(echo "$INPUT" | jq -r '
+            to_entries[] | .key as $device | .value | to_entries[]
+            | select(.value.status == "known failure" or .value.status == "unexpected pass")
+            | "| \($device) | \(.key) | \(.value.status) | \(.value.comment // "") |"' | sort)
+          if [ -n "${KNOWN_ROWS}" ]; then
+            echo "<details>" >> "${INPUTS_SUMMARY_FILE_NAME}"
+            echo "<summary>Known failures ($TOTAL_KNOWN_FAIL)</summary>" >> "${INPUTS_SUMMARY_FILE_NAME}"
+            echo "" >> "${INPUTS_SUMMARY_FILE_NAME}"
+            echo "| Device | Test | Status | Comment |" >> "${INPUTS_SUMMARY_FILE_NAME}"
+            echo "| ---- | ---- | ---- | ---- |" >> "${INPUTS_SUMMARY_FILE_NAME}"
+            echo "${KNOWN_ROWS}" >> "${INPUTS_SUMMARY_FILE_NAME}"
+            echo "" >> "${INPUTS_SUMMARY_FILE_NAME}"
+            echo "</details>" >> "${INPUTS_SUMMARY_FILE_NAME}"
+          fi
+
           echo "<details>" >> "${INPUTS_SUMMARY_FILE_NAME}"
           echo "<summary>All jobs summary</summary>" >> "${INPUTS_SUMMARY_FILE_NAME}"
           echo "" >> "${INPUTS_SUMMARY_FILE_NAME}"

@@ -0,0 +1,139 @@
+# Known test failures
+
+Each build variant tested by `.github/workflows/test.yml` has its own list of
+known failures in this directory. The file name is
+`<distro_name><distro_suffix>.yaml`, i.e. the same value that is used to name
+the test job summary:
+
+| Build variant               | File                               |
+| --------------------------- | ---------------------------------- |
+| nodistro                    | `nodistro.yaml`                    |
+| qcom-distro                 | `qcom-distro.yaml`                 |
+| qcom-distro_linux-qcom-6.18 | `qcom-distro_linux-qcom-6.18.yaml` |
+
+## Format
+
+The file is a YAML mapping of LAVA device type to the list of tests that are
+expected to fail on that device. Every entry carries the test name and a
+`comment` pointing at the issue the failure is tracked in:
+
+```yaml
+qcm6490-idp:
+  - test: some_failing_test
+    comment: https://github.com/qualcomm-linux/meta-qcom/issues/1234
+  - test: another_failing_test
+    comment: "waiting for the firmware uprev, issue #1235"
+
+# "*" applies to every device tested in this variant
+"*":
+  - test: test_failing_everywhere
+    comment: https://github.com/qualcomm-linux/meta-qcom/issues/1236
+```
+
+`comment` is free text. A link to the reported issue is what makes the list
+reviewable, so add one for every entry. Quote a comment that contains a `#`,
+otherwise YAML treats the rest of the line as a comment of its own. A bare test
+name is also accepted for an entry that has nothing to say:
+
+```yaml
+qcm6490-idp:
+  - some_failing_test
+```
+
+The test name is the name reported by LAVA, i.e. the value shown in the first
+column of the test job summary table. The device type is the LAVA
+`requested_device_type`, i.e. the column header of that table. When a test is
+listed both for a device and under `"*"`, the device entry and its comment win.
+
+An empty file (or a file containing only comments) means that no failure is
+known for that variant.
+
+## How the list is used
+
+Two consumers apply the list.
+
+### The test job summary
+
+`.github/actions/test-job-summary` applies the list when it renders the summary
+table of a build variant:
+
+| Result in LAVA | Listed as known failure | Reported as                             |
+| -------------- | ----------------------- | --------------------------------------- |
+| `fail`         | no                      | :x: `fail`                              |
+| `fail`         | yes                     | :ballot_box_with_check: `known failure` |
+| `pass`         | no                      | :white_check_mark: `pass`               |
+| `pass`         | yes                     | :x: `unexpected pass`                   |
+
+A known failure is counted like a pass, so it does not add to the failure
+count. A test that passes while it is listed as a known failure is counted as
+a failure - that is the signal to remove the entry from this list.
+
+Every entry that was applied is listed with its comment in a "Known failures"
+section below the table.
+
+The summary also reports the result of the boot test under the name `boot`, so
+`boot` can be listed here like any other test name.
+
+### The "Test Results" check
+
+`.github/workflows/publish-results.yml` publishes the JUnit XML files that LAVA
+produced for every test job. Before they are published,
+`.github/scripts/apply-known-failures.py` rewrites them in place: a known
+failure becomes a skipped test, so it no longer fails the check, and a test
+that passes while it is listed becomes a failure. The comment of the entry is
+appended to the message of the rewritten test, so the issue is one click away
+in the check report. The script maps a result file to a variant and a device
+through the file name, which lava-test-plans builds as
+`<prefix>-<variant>-<device>-<job>.yaml`.
+
+The XML files only contain the tests that ran inside a LAVA job, so a `boot`
+entry has no effect on this check.
+
+## Which lists are used
+
+Both consumers apply the lists of the pull request under test, not the ones
+already merged, so a pull request that fixes a listed failure removes the entry
+in the same change, and a pull request that hits a new one can list it right
+away.
+
+The test chain of a pull request runs on the `workflow_run` event, so its own
+checkout is the base branch. It therefore checks this directory out a second
+time from the branch or fork the pull request is built from, into
+`known-failures-pr/`, with a sparse checkout that fetches nothing else. Those
+lists are only ever read as data: the scripts applying them, and everything
+else the chain runs, still come from the base branch.
+
+Before they are applied to the "Test Results" check the lists are checked with
+`--validate --syntax-only`, i.e. for their syntax alone - which variants exist
+is a property of the base branch, and it is `known-failures.yml`, running on
+the pull request itself, that checks the lists against them. A list that does
+not parse, or a fork that is no longer reachable, falls back to the lists of
+the base branch. The run log says which lists were applied.
+
+A push, a nightly build and a manual run have no pull request to take lists
+from and simply use the ones of their own checkout.
+
+## Validation
+
+`.github/workflows/known-failures.yml` validates the lists on every change to
+this directory, to the script, or to the test workflow. It rejects:
+
+* a file that is not valid YAML, or that does not follow the format above,
+* a test listed twice for the same device,
+* a file whose name is not one of the build variants tested by
+  `.github/workflows/test.yml`, and a variant that has no file at all,
+* a device that the variant does not test.
+
+The last two matter because such an entry is not an error at test time, it is
+simply never applied - the list would look like it suppresses a failure while
+the check stays red. An entry without a comment is reported as a warning.
+
+Run the same check locally with:
+
+```shell
+python3 .github/scripts/apply-known-failures.py --validate
+```
+
+Add `--syntax-only` to check the format of the lists without checking them
+against the build variants of this branch. That is what the test chain uses on
+the lists it takes from a pull request.
@@ -0,0 +1,16 @@
+---
+# Known test failures for the "nodistro" build variant.
+#
+# Mapping of LAVA device type to the list of test names that are expected to
+# fail. Use "*" as the device type for a failure seen on every device. Annotate
+# every entry with the issue it is tracked in, so that the list stays reviewable.
+# See README.md in this directory for details.
+#
+# Example:
+#
+# qcm6490-idp:
+#   - test: some_failing_test
+#     comment: https://github.com/qualcomm-linux/meta-qcom/issues/1234
+# "*":
+#   - test: test_failing_everywhere
+#     comment: "flaky since the 6.18 kernel uprev, issue #1235"
@@ -0,0 +1,16 @@
+---
+# Known test failures for the "qcom-distro" build variant.
+#
+# Mapping of LAVA device type to the list of test names that are expected to
+# fail. Use "*" as the device type for a failure seen on every device. Annotate
+# every entry with the issue it is tracked in, so that the list stays reviewable.
+# See README.md in this directory for details.
+#
+# Example:
+#
+# qcm6490-idp:
+#   - test: some_failing_test
+#     comment: https://github.com/qualcomm-linux/meta-qcom/issues/1234
+# "*":
+#   - test: test_failing_everywhere
+#     comment: "flaky since the 6.18 kernel uprev, issue #1235"
@@ -0,0 +1,16 @@
+---
+# Known test failures for the "qcom-distro_linux-qcom-6.18" build variant.
+#
+# Mapping of LAVA device type to the list of test names that are expected to
+# fail. Use "*" as the device type for a failure seen on every device. Annotate
+# every entry with the issue it is tracked in, so that the list stays reviewable.
+# See README.md in this directory for details.
+#
+# Example:
+#
+# qcm6490-idp:
+#   - test: some_failing_test
+#     comment: https://github.com/qualcomm-linux/meta-qcom/issues/1234
+# "*":
+#   - test: test_failing_everywhere
+#     comment: "flaky since the 6.18 kernel uprev, issue #1235"