TEST: Produce JSON logs from pytest runs (#2107)

* produce JSON logs from pytest runs

* make json reports triggerable with cli argument

* Update ci.yml

* Update ci.yml

* fix repeated argument on windows

* avoid adding json-related arguments when not needed

* generate json reports during CI runs

* remove accidental spaces

* fix incorrect file name checking

* remove json report from non-nightly job

---------

Co-authored-by: Ian Faust <[email protected]>

.ci/pipeline/build-and-test-lnx.yml

@@ -55,7 +55,7 @@ steps:
       . /usr/share/miniconda/etc/profile.d/conda.sh
       conda activate CB
       cd ..
-      ./s/conda-recipe/run_test.sh
+      ./s/conda-recipe/run_test.sh --json-report
     displayName: "Sklearnex testing"
   - script: |
       . /usr/share/miniconda/etc/profile.d/conda.sh

.ci/pipeline/build-and-test-win.yml

@@ -51,7 +51,7 @@ steps:
   - script: |
       call activate CB
       cd ..
-      call s\conda-recipe\run_test.bat s\
+      call s\conda-recipe\run_test.bat s\ --json-report
     displayName: 'Sklearnex testing'
   - script: |
       call activate CB

.github/workflows/ci.yml

@@ -245,7 +245,7 @@ jobs:
           call .\.github\scripts\activate_components.bat ${{ steps.set-env.outputs.DPCFLAG }}
           set PYTHON=python
           cd ..
-          call scikit-learn-intelex\conda-recipe\run_test.bat scikit-learn-intelex\
+          call scikit-learn-intelex\conda-recipe\run_test.bat scikit-learn-intelex\ --json-report
       - name: Sklearn testing
         shell: cmd
         run: |

.gitignore

@@ -30,3 +30,6 @@ record*
 
 # example .res files
 tests/_results*
+
+# json reports from pytest
+.pytest_reports/*

conda-recipe/run_test.bat

@@ -23,10 +23,31 @@ IF NOT DEFINED PYTHON (set "PYTHON=python")
 
 %PYTHON% -c "from sklearnex import patch_sklearn; patch_sklearn()" || set exitcode=1
 
-%PYTHON% -m pytest --verbose -s %1tests || set exitcode=1
+rem Note: execute with argument --json-report as second argument
+rem in order to produce a JSON report under folder '.pytest_reports'.
+set with_json_report=0
+if "%~2"=="--json-report" (
+    set with_json_report=1
+    mkdir .pytest_reports
+    del /q .pytest_reports\*.json
+)
+
+if "%with_json_report%"=="1" (
+    %PYTHON% -m pytest --verbose -s %1tests --json-report --json-report-file=.pytest_reports\legacy_report.json || set exitcode=1
+    pytest --verbose --pyargs daal4py --json-report --json-report-file=.pytest_reports\daal4py_report.json || set exitcode=1
+    pytest --verbose --pyargs sklearnex --json-report --json-report-file=.pytest_reports\sklearnex_report.json || set exitcode=1
+    pytest --verbose --pyargs onedal --json-report --json-report-file=.pytest_reports\onedal_report.json || set exitcode=1
+    pytest --verbose %1.ci\scripts\test_global_patch.py --json-report --json-report-file=.pytest_reports\global_patching_report.json || set exitcode=1
+    if NOT EXIST .pytest_reports\legacy_report.json (
+        echo "Error: JSON report files failed to be produced."
+        set exitcode=1
+    )
+) else (
+    %PYTHON% -m pytest --verbose -s %1tests || set exitcode=1
+    pytest --verbose --pyargs daal4py || set exitcode=1
+    pytest --verbose --pyargs sklearnex || set exitcode=1
+    pytest --verbose --pyargs onedal || set exitcode=1
+    pytest --verbose %1.ci\scripts\test_global_patch.py || set exitcode=1
+)
 
-pytest --verbose --pyargs daal4py || set exitcode=1
-pytest --verbose --pyargs sklearnex || set exitcode=1
-pytest --verbose --pyargs onedal || set exitcode=1
-pytest --verbose %1.ci\scripts\test_global_patch.py || set exitcode=1
 EXIT /B %exitcode%

conda-recipe/run_test.sh

@@ -36,28 +36,46 @@ if [ -z "${PYTHON}" ]; then
     export PYTHON=python
 fi
 
+# Note: execute with argument --json-report in order to produce
+# a JSON report under folder '.pytest_reports'. Other arguments
+# will also be forwarded to pytest.
+with_json_report=0
+if [[ "$*" == *"--json-report"* ]]; then
+    echo "Will produce JSON report of tests"
+    with_json_report=1
+    mkdir -p .pytest_reports
+    if [[ ! -z "$(ls .pytest_reports)" ]]; then
+        rm .pytest_reports/*.json
+    fi
+fi
+function json_report_name {
+    if [[ "${with_json_report}" == "1" ]]; then
+        printf -- "--json-report-file=.pytest_reports/$1_report.json"
+    fi
+}
+
 ${PYTHON} -c "from sklearnex import patch_sklearn; patch_sklearn()"
 return_code=$(($return_code + $?))
 
-pytest --verbose -s ${sklex_root}/tests
+pytest --verbose -s ${sklex_root}/tests $@ $(json_report_name legacy)
 return_code=$(($return_code + $?))
 
-pytest --verbose --pyargs daal4py
+pytest --verbose --pyargs daal4py $@ $(json_report_name daal4py)
 return_code=$(($return_code + $?))
 
-pytest --verbose --pyargs sklearnex
+pytest --verbose --pyargs sklearnex $@ $(json_report_name sklearnex)
 return_code=$(($return_code + $?))
 
-pytest --verbose --pyargs onedal
+pytest --verbose --pyargs onedal $@ $(json_report_name onedal)
 return_code=$(($return_code + $?))
 
-pytest --verbose -s ${sklex_root}/.ci/scripts/test_global_patch.py
+pytest --verbose -s ${sklex_root}/.ci/scripts/test_global_patch.py $@ $(json_report_name global_patching)
 return_code=$(($return_code + $?))
 
 echo "NO_DIST=$NO_DIST"
 if [[ ! $NO_DIST ]]; then
     mpirun --version
-    mpirun -n 4 pytest --verbose -s ${sklex_root}/tests/test*spmd*.py
+    mpirun -n 4 pytest --verbose -s ${sklex_root}/tests/test*spmd*.py $@ $(json_report_name mpi_legacy)
     return_code=$(($return_code + $?))
 fi
 

requirements-test.txt

@@ -1,5 +1,6 @@
 pytest==7.4.4 ; python_version <= '3.10'
 pytest==8.3.3 ; python_version >= '3.11'
+pytest-json-report==1.5.0
 numpy>=1.19.5 ; python_version <= '3.9'
 numpy>=1.21.6 ; python_version == '3.10'
 numpy>=1.23.5 ; python_version == '3.11'