From 98010c54e1cc7f207b00426f043bf74220d2ea10 Mon Sep 17 00:00:00 2001 From: david-cortes-intel Date: Mon, 21 Oct 2024 15:13:24 +0200 Subject: [PATCH] TEST: Produce JSON logs from pytest runs (#2107) * produce JSON logs from pytest runs * make json reports triggerable with cli argument * Update ci.yml * Update ci.yml * fix repeated argument on windows * avoid adding json-related arguments when not needed * generate json reports during CI runs * remove accidental spaces * fix incorrect file name checking * remove json report from non-nightly job --------- Co-authored-by: Ian Faust --- .ci/pipeline/build-and-test-lnx.yml | 2 +- .ci/pipeline/build-and-test-win.yml | 2 +- .github/workflows/ci.yml | 2 +- .gitignore | 3 +++ conda-recipe/run_test.bat | 31 ++++++++++++++++++++++++----- conda-recipe/run_test.sh | 30 ++++++++++++++++++++++------ requirements-test.txt | 1 + 7 files changed, 57 insertions(+), 14 deletions(-) diff --git a/.ci/pipeline/build-and-test-lnx.yml b/.ci/pipeline/build-and-test-lnx.yml index 60fbfbded3..b2440ae435 100644 --- a/.ci/pipeline/build-and-test-lnx.yml +++ b/.ci/pipeline/build-and-test-lnx.yml @@ -55,7 +55,7 @@ steps: . /usr/share/miniconda/etc/profile.d/conda.sh conda activate CB cd .. - ./s/conda-recipe/run_test.sh + ./s/conda-recipe/run_test.sh --json-report displayName: "Sklearnex testing" - script: | . /usr/share/miniconda/etc/profile.d/conda.sh diff --git a/.ci/pipeline/build-and-test-win.yml b/.ci/pipeline/build-and-test-win.yml index c09beed84d..87df9c661d 100644 --- a/.ci/pipeline/build-and-test-win.yml +++ b/.ci/pipeline/build-and-test-win.yml @@ -51,7 +51,7 @@ steps: - script: | call activate CB cd .. - call s\conda-recipe\run_test.bat s\ + call s\conda-recipe\run_test.bat s\ --json-report displayName: 'Sklearnex testing' - script: | call activate CB diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d00c366c27..b0c2eea641 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -245,7 +245,7 @@ jobs: call .\.github\scripts\activate_components.bat ${{ steps.set-env.outputs.DPCFLAG }} set PYTHON=python cd .. - call scikit-learn-intelex\conda-recipe\run_test.bat scikit-learn-intelex\ + call scikit-learn-intelex\conda-recipe\run_test.bat scikit-learn-intelex\ --json-report - name: Sklearn testing shell: cmd run: | diff --git a/.gitignore b/.gitignore index 38bc1e03a1..24e062e5ad 100755 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,6 @@ record* # example .res files tests/_results* + +# json reports from pytest +.pytest_reports/* diff --git a/conda-recipe/run_test.bat b/conda-recipe/run_test.bat index a9e27b0c91..9306bc1daf 100644 --- a/conda-recipe/run_test.bat +++ b/conda-recipe/run_test.bat @@ -23,10 +23,31 @@ IF NOT DEFINED PYTHON (set "PYTHON=python") %PYTHON% -c "from sklearnex import patch_sklearn; patch_sklearn()" || set exitcode=1 -%PYTHON% -m pytest --verbose -s %1tests || set exitcode=1 +rem Note: execute with argument --json-report as second argument +rem in order to produce a JSON report under folder '.pytest_reports'. +set with_json_report=0 +if "%~2"=="--json-report" ( + set with_json_report=1 + mkdir .pytest_reports + del /q .pytest_reports\*.json +) + +if "%with_json_report%"=="1" ( + %PYTHON% -m pytest --verbose -s %1tests --json-report --json-report-file=.pytest_reports\legacy_report.json || set exitcode=1 + pytest --verbose --pyargs daal4py --json-report --json-report-file=.pytest_reports\daal4py_report.json || set exitcode=1 + pytest --verbose --pyargs sklearnex --json-report --json-report-file=.pytest_reports\sklearnex_report.json || set exitcode=1 + pytest --verbose --pyargs onedal --json-report --json-report-file=.pytest_reports\onedal_report.json || set exitcode=1 + pytest --verbose %1.ci\scripts\test_global_patch.py --json-report --json-report-file=.pytest_reports\global_patching_report.json || set exitcode=1 + if NOT EXIST .pytest_reports\legacy_report.json ( + echo "Error: JSON report files failed to be produced." + set exitcode=1 + ) +) else ( + %PYTHON% -m pytest --verbose -s %1tests || set exitcode=1 + pytest --verbose --pyargs daal4py || set exitcode=1 + pytest --verbose --pyargs sklearnex || set exitcode=1 + pytest --verbose --pyargs onedal || set exitcode=1 + pytest --verbose %1.ci\scripts\test_global_patch.py || set exitcode=1 +) -pytest --verbose --pyargs daal4py || set exitcode=1 -pytest --verbose --pyargs sklearnex || set exitcode=1 -pytest --verbose --pyargs onedal || set exitcode=1 -pytest --verbose %1.ci\scripts\test_global_patch.py || set exitcode=1 EXIT /B %exitcode% diff --git a/conda-recipe/run_test.sh b/conda-recipe/run_test.sh index af8df1a887..3ed73b3f84 100755 --- a/conda-recipe/run_test.sh +++ b/conda-recipe/run_test.sh @@ -36,28 +36,46 @@ if [ -z "${PYTHON}" ]; then export PYTHON=python fi +# Note: execute with argument --json-report in order to produce +# a JSON report under folder '.pytest_reports'. Other arguments +# will also be forwarded to pytest. +with_json_report=0 +if [[ "$*" == *"--json-report"* ]]; then + echo "Will produce JSON report of tests" + with_json_report=1 + mkdir -p .pytest_reports + if [[ ! -z "$(ls .pytest_reports)" ]]; then + rm .pytest_reports/*.json + fi +fi +function json_report_name { + if [[ "${with_json_report}" == "1" ]]; then + printf -- "--json-report-file=.pytest_reports/$1_report.json" + fi +} + ${PYTHON} -c "from sklearnex import patch_sklearn; patch_sklearn()" return_code=$(($return_code + $?)) -pytest --verbose -s ${sklex_root}/tests +pytest --verbose -s ${sklex_root}/tests $@ $(json_report_name legacy) return_code=$(($return_code + $?)) -pytest --verbose --pyargs daal4py +pytest --verbose --pyargs daal4py $@ $(json_report_name daal4py) return_code=$(($return_code + $?)) -pytest --verbose --pyargs sklearnex +pytest --verbose --pyargs sklearnex $@ $(json_report_name sklearnex) return_code=$(($return_code + $?)) -pytest --verbose --pyargs onedal +pytest --verbose --pyargs onedal $@ $(json_report_name onedal) return_code=$(($return_code + $?)) -pytest --verbose -s ${sklex_root}/.ci/scripts/test_global_patch.py +pytest --verbose -s ${sklex_root}/.ci/scripts/test_global_patch.py $@ $(json_report_name global_patching) return_code=$(($return_code + $?)) echo "NO_DIST=$NO_DIST" if [[ ! $NO_DIST ]]; then mpirun --version - mpirun -n 4 pytest --verbose -s ${sklex_root}/tests/test*spmd*.py + mpirun -n 4 pytest --verbose -s ${sklex_root}/tests/test*spmd*.py $@ $(json_report_name mpi_legacy) return_code=$(($return_code + $?)) fi diff --git a/requirements-test.txt b/requirements-test.txt index ba35d2d28c..2c264f4778 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,5 +1,6 @@ pytest==7.4.4 ; python_version <= '3.10' pytest==8.3.3 ; python_version >= '3.11' +pytest-json-report==1.5.0 numpy>=1.19.5 ; python_version <= '3.9' numpy>=1.21.6 ; python_version == '3.10' numpy>=1.23.5 ; python_version == '3.11'