[CI, enhancement] add pytest-cov for testing (#2104)

* Update requirements-test.txt * Update run_test.sh * Update ci.yml * Update run_test.sh * Update run_test.sh * Update run_test.sh * Update run_test.sh * Update run_test.sh * Update run_test.sh * Update run_test.sh * forgot to add the coveragerc file * add whitespace * Update run_test.sh * Update .coveragerc * Update test_basic_statistics.py * formatting * Update test_basic_statistics.py * Update ci.yml * Update run_test.sh * Update ci.yml * Update run_test.sh * Update test_basic_statistics.py * Update test_incremental_basic_statistics.py * Update run_test.sh * Update ci.yml * Update run_test.sh * Update test_basic_statistics.py * formatting * Update test_incremental_basic_statistics.py * Update test_incremental_basic_statistics.py * Update test_basic_statistics.py * Update test_incremental_basic_statistics.py * Update test_basic_statistics.py * Update test_incremental_basic_statistics.py * Update test_incremental_basic_statistics.py * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update run_test.sh * Update .coveragerc * Update run_test.sh * switch coverage to be coveragerc location * missing change * udpate CI * Update ci.yml * Update run_test.sh * Update run_sklearn_tests.py * Update run_sklearn_tests.py * Update ci.yml * Update run_test.sh * Update run_sklearn_tests.py * Update run_sklearn_tests.py * Update run_sklearn_tests.py * Update run_sklearn_tests.py * Update run_sklearn_tests.py * Update run_sklearn_tests.py * Update run_sklearn_tests.py * Update ci.yml * Update run_test.sh * fix isort * formatting * Update .coveragerc * create viable pathing * add first gpu coverage notes * test coverage removal for gpu * added whitespace * try to fix the bs tests from report * fix mistake * make appending simpler * Update .ci/scripts/run_sklearn_tests.py Co-authored-by: Andreas Huber <[email protected]> * Update run_sklearn_tests.py * formatting * Update .coveragerc * Update ci.yml * Update run_test.bat * Update ci.yml * this will break CI, work in progress: * attempt at fixing dpctl check * make consistent * remove $ * fix env setting windows * further iterations * further fixes * test with a string * further attempts at fixing it * attack at root cause * formatting error: * windows annoyance * windows annoyance * windows annoyance * test * whitespace * Update ci.yml * Update run_sklearn_tests.py * Update run_sklearn_tests.py * Update run_sklearn_tests.py * Update ci.yml * Update run_sklearn_tests.py * Update ci.yml * Update run_test.bat * Update run_test.sh * Update requirements-test.txt * Update ci.yml * Update ci.yml * Update run_sklearn_tests.py * formatting * Update ci.yml * Update ci.yml * Update ci.yml * making change for basic_stats * forgotten spmd changes * Update run_test.sh * Update .coveragerc * extract adjacent changes requested * remove comments from LogReg * remove unneeded code * forgotten change * fix save_attributes * Update test_incremental_basic_statistics_spmd.py * Update test_incremental_basic_statistics_spmd.py * Update test_basic_statistics_spmd.py * Update run_test.bat * Update run_test.bat * Update run_test.sh * Update run_test.sh * Update run_test.bat * fixes requested * Update .coveragerc * Update run_test.sh * Update run_test.sh * Update run_test.sh * attempt at centralizing windows * fix for whitespace * fix for whitespace * fix for -- * make changes suggested * add check to run_test.sh to match run_test.bat * make codefactor suggestion * make a better exit value * last suggested changes --------- Co-authored-by: Andreas Huber <[email protected]>
uxlfoundation · Nov 19, 2024 · cb2c701 · cb2c701
1 parent 5f071fc
commit cb2c701
Show file tree

Hide file tree

Showing 14 changed files with 248 additions and 210 deletions.
diff --git a/.ci/scripts/run_sklearn_tests.py b/.ci/scripts/run_sklearn_tests.py
@@ -37,16 +37,26 @@
     )
     args = parser.parse_args()
 
-    os.chdir(os.path.dirname(sklearn.__file__))
+    sklearn_file_dir = os.path.dirname(sklearn.__file__)
+    os.chdir(sklearn_file_dir)
 
     if os.environ["SELECTED_TESTS"] == "all":
         os.environ["SELECTED_TESTS"] = ""
 
     pytest_args = (
         "--verbose --durations=100 --durations-min=0.01 "
-        f"--rootdir={os.path.dirname(sklearn.__file__)} "
+        f"--rootdir={sklearn_file_dir} "
         f'{os.environ["DESELECTED_TESTS"]} {os.environ["SELECTED_TESTS"]}'.split(" ")
     )
+
+    if rc := os.getenv("COVERAGE_RCFILE"):
+        pytest_args += (
+            "--cov=onedal",
+            "--cov=sklearnex",
+            f"--cov-config={rc}",
+            "--cov-report=",
+        )
+
     while "" in pytest_args:
         pytest_args.remove("")
 
@@ -55,4 +65,5 @@
             return_code = pytest.main(pytest_args)
     else:
         return_code = pytest.main(pytest_args)
+
     sys.exit(int(return_code))
diff --git a/.coveragerc b/.coveragerc
@@ -0,0 +1,40 @@
+#===============================================================================
+# Copyright 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#===============================================================================
+[run]
+branch = True
+source = 
+    sklearnex
+    onedal
+parallel = True
+omit =
+    */onedal/tests/*
+    */onedal/basic_statistics/tests/utils.py
+    */sklearnex/conftest.py
+    */sklearnex/tests/*
+    */sklearnex/preview/*
+    */sklearnex/spmd/*
+[paths]
+sklearnex =
+    sklearnex/
+    */sklearnex
+onedal =
+    onedal/
+    */onedal
+#Uncomment lines below for report generation when an Intel GPU is unavailable
+#[report]
+#exclude_also =
+#    def _onedal_gpu.*
+#    coverage: gpu_start(?s:.)*coverage: gpu_stop
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -93,6 +93,8 @@ jobs:
           DPCFLAG="0"
           if [ $(echo ${{ matrix.PYTHON_VERSION }} | grep '${{ env.DPCTL_PY_VERSIONS }}') ]; then DPCFLAG=''; fi
           echo "DPCFLAG=${DPCFLAG}" >> "$GITHUB_OUTPUT"
+          # enable coverage report generation
+          echo "COVERAGE_RCFILE=$(readlink -f .coveragerc)" >> "$GITHUB_ENV"
       - name: apt-get
         run: sudo apt-get update && sudo apt-get install -y clang-format
       - name: dpcpp installation
@@ -127,14 +129,24 @@ jobs:
         run: |
           source venv/bin/activate
           source .github/scripts/activate_components.sh
+          export COVERAGE_FILE=$(pwd)/.coverage.sklearnex
           cd .ci
           ../conda-recipe/run_test.sh
       - name: Sklearn testing
         run: |
           source venv/bin/activate
           source .github/scripts/activate_components.sh
+          export COVERAGE_FILE=$(pwd)/.coverage.sklearn
           if [ "${{ steps.set-env.outputs.DPCFLAG }}" == "" ]; then export CPU=cpu; fi
           bash .ci/scripts/run_sklearn_tests.sh $CPU
+      - name: Archive coverage data
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage_lnx_Py${{ matrix.PYTHON_VERSION }}_${{ matrix.SKLEARN_VERSION }}
+          path: |
+             .coverage.sklearn
+             .coverage.sklearnex
+          include-hidden-files: true
       - name: Sklearn testing [preview]
         run: |
           source venv/bin/activate
@@ -199,15 +211,13 @@ jobs:
           echo "Unzip complete"
       - name: Set Environment Variables
         id: set-env
-        shell: bash
+        shell: cmd
         run: |
-          echo "C:\msys64\usr\bin;C:\Program Files (x86)\Common Files\Intel\Shared Libraries\bin;" >> $GITHUB_PATH
-          # Disable SPMD testing
-          echo "NO_DIST=1" >> "$GITHUB_ENV"
-          # Determine dpctl support
-          DPCFLAG="0"
-          if [ $(echo ${{ matrix.PYTHON_VERSION }} | grep '${{ env.DPCTL_PY_VERSIONS }}') ]; then DPCFLAG=''; fi
-          echo "DPCFLAG=${DPCFLAG}" >> "$GITHUB_OUTPUT"
+          echo C:\msys64\usr\bin;>> %GITHUB_PATH%
+          echo NO_DIST=YES>> %GITHUB_ENV%
+          set DPCTL_TEMP="${{ env.DPCTL_PY_VERSIONS }}"
+          if not %DPCTL_TEMP:${{ matrix.PYTHON_VERSION }}=%==%DPCTL_TEMP% (echo DPCFLAG=>> %GITHUB_OUTPUT%) else (echo DPCFLAG="0">> %GITHUB_OUTPUT%)
+          echo COVERAGE_RCFILE=%cd%\.coveragerc>> %GITHUB_ENV%
       - name: Download Intel OpenCL CPU Runtime artifact
         if: ${{ steps.set-env.outputs.DPCFLAG == '' }}
         uses: actions/download-artifact@v4
@@ -261,15 +271,25 @@ jobs:
           call .\venv\Scripts\activate.bat
           call .\.github\scripts\activate_components.bat ${{ steps.set-env.outputs.DPCFLAG }}
           set PYTHON=python
+          set COVERAGE_FILE=%cd%\.coverage.sklearnex
           cd ..
           call scikit-learn-intelex\conda-recipe\run_test.bat scikit-learn-intelex\
       - name: Sklearn testing
         shell: cmd
         run: |
           call .\venv\Scripts\activate.bat
           call .\.github\scripts\activate_components.bat ${{ steps.set-env.outputs.DPCFLAG }}
+          set COVERAGE_FILE=%cd%\.coverage.sklearn
           if "${{ steps.set-env.outputs.DPCFLAG }}"=="" set CPU=cpu
           bash .ci/scripts/run_sklearn_tests.sh %CPU%
+      - name: Archive coverage data
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage_win_Py${{ matrix.PYTHON_VERSION }}_${{ matrix.SKLEARN_VERSION }}
+          path: |
+             .coverage.sklearn
+             .coverage.sklearnex
+          include-hidden-files: true
       - name: Sklearn testing [preview]
         shell: cmd
         run: |

diff --git a/conda-recipe/run_test.bat b/conda-recipe/run_test.bat
@@ -28,54 +28,46 @@ if "%PYTHON%"=="python" (
     set NO_DIST=1
 )
 
+
+
 %PYTHON% -c "from sklearnex import patch_sklearn; patch_sklearn()" || set exitcode=1
 
+set "PYTEST_ARGS= "
+
+IF DEFINED COVERAGE_RCFILE (set "PYTEST_ARGS=--cov=onedal --cov=sklearnex --cov-config=%COVERAGE_RCFILE% --cov-append --cov-report= %PYTEST_ARGS%")
+
 rem Note: execute with argument --json-report as second argument
 rem in order to produce a JSON report under folder '.pytest_reports'.
-set with_json_report=0
 if "%~2"=="--json-report" (
-    set with_json_report=1
+    set "PYTEST_ARGS=--json-report --json-report-file=.pytest_reports\FILENAME.json %PYTEST_ARGS%"
+    echo %PYTEST_ARGS%
     mkdir .pytest_reports
     del /q .pytest_reports\*.json
 )
 
 echo "NO_DIST=%NO_DIST%"
-echo "with_json_report=%with_json_report%"
 setlocal enabledelayedexpansion
-if "%with_json_report%"=="1" (
-    pytest --verbose -s "%1tests" --json-report --json-report-file=.pytest_reports\legacy_report.json || set exitcode=1
-    pytest --verbose --pyargs daal4py --json-report --json-report-file=.pytest_reports\daal4py_report.json || set exitcode=1
-    pytest --verbose --pyargs sklearnex --json-report --json-report-file=.pytest_reports\sklearnex_report.json || set exitcode=1
-    pytest --verbose --pyargs onedal --json-report --json-report-file=.pytest_reports\onedal_report.json || set exitcode=1
-    pytest --verbose "%1.ci\scripts\test_global_patch.py" --json-report --json-report-file=.pytest_reports\global_patching_report.json || set exitcode=1
-    if NOT "%NO_DIST%"=="1" (
-        %PYTHON% "%1tests\helper_mpi_tests.py"^
-            pytest -k spmd --with-mpi --verbose -s --pyargs sklearnex^
-            --json-report --json-report-file=.pytest_reports\sklearnex_spmd.json
-        if !errorlevel! NEQ 0 (
-            set exitcode=1
-        )
-        %PYTHON% "%1tests\helper_mpi_tests.py"^
-            pytest --with-mpi --verbose -s "%1tests\test_daal4py_spmd_examples.py"^
-            --json-report --json-report-file=.pytest_reports\mpi_legacy.json
-        if !errorlevel! NEQ 0 (
-            set exitcode=1
-        )
+pytest --verbose -s "%1tests" %PYTEST_ARGS:FILENAME=legacy_report% || set exitcode=1
+pytest --verbose --pyargs daal4py %PYTEST_ARGS:FILENAME=daal4py_report% || set exitcode=1
+pytest --verbose --pyargs sklearnex %PYTEST_ARGS:FILENAME=sklearnex_report% || set exitcode=1
+pytest --verbose --pyargs onedal %PYTEST_ARGS:FILENAME=onedal_report% || set exitcode=1
+pytest --verbose "%1.ci\scripts\test_global_patch.py" %PYTEST_ARGS:FILENAME=global_patching_report% || set exitcode=1
+if NOT "%NO_DIST%"=="1" (
+    %PYTHON% "%1tests\helper_mpi_tests.py"^
+        pytest -k spmd --with-mpi --verbose -s --pyargs sklearnex %PYTEST_ARGS:FILENAME=sklearnex_spmd%
+    if !errorlevel! NEQ 0 (
+        set exitcode=1
     )
+    %PYTHON% "%1tests\helper_mpi_tests.py"^
+        pytest --with-mpi --verbose -s "%1tests\test_daal4py_spmd_examples.py" %PYTEST_ARGS:FILENAME=mpi_legacy%
+    if !errorlevel! NEQ 0 (
+        set exitcode=1
+    )
+)
+if "%~2"=="--json-report" (
     if NOT EXIST .pytest_reports\legacy_report.json (
         echo "Error: JSON report files failed to be produced."
         set exitcode=1
     )
-) else (
-    pytest --verbose -s "%1tests" || set exitcode=1
-    pytest --verbose --pyargs daal4py || set exitcode=1
-    pytest --verbose --pyargs sklearnex || set exitcode=1
-    pytest --verbose --pyargs onedal || set exitcode=1
-    pytest --verbose "%1.ci\scripts\test_global_patch.py" || set exitcode=1
-    if NOT "%NO_DIST%"=="1" (
-        %PYTHON% -m pytest -k spmd --with-mpi --verbose --pyargs sklearnex || set exitcode=1
-        %PYTHON% -m pytest --verbose -s "%1tests\test_daal4py_spmd_examples.py" || set exitcode=1
-    )
 )
-
 EXIT /B %exitcode%
diff --git a/conda-recipe/run_test.sh b/conda-recipe/run_test.sh
@@ -48,28 +48,34 @@ if [[ "$*" == *"--json-report"* ]]; then
         rm .pytest_reports/*.json
     fi
 fi
-function json_report_name {
+
+function generate_pytest_args {
+    declare -a ARGS=()
     if [[ "${with_json_report}" == "1" ]]; then
-        printf -- "--json-report-file=.pytest_reports/$1_report.json"
+        ARGS+=("--json-report-file=.pytest_reports/$1_report.json")
+    fi
+    if [ -n "${COVERAGE_RCFILE}" ]; then
+        ARGS+=(--cov=onedal --cov=sklearnex --cov-config="${COVERAGE_RCFILE}" --cov-append --cov-report=)
     fi
+    printf -- "${ARGS[*]}"
 }
 
 ${PYTHON} -c "from sklearnex import patch_sklearn; patch_sklearn()"
 return_code=$(($return_code + $?))
 
-pytest --verbose -s "${sklex_root}/tests" $@ $(json_report_name legacy)
+pytest --verbose -s "${sklex_root}/tests" $@ $(generate_pytest_args legacy)
 return_code=$(($return_code + $?))
 
-pytest --verbose --pyargs daal4py $@ $(json_report_name daal4py)
+pytest --verbose --pyargs daal4py $@ $(generate_pytest_args daal4py)
 return_code=$(($return_code + $?))
 
-pytest --verbose --pyargs sklearnex $@ $(json_report_name sklearnex)
+pytest --verbose --pyargs sklearnex $@ $(generate_pytest_args sklearnex)
 return_code=$(($return_code + $?))
 
-pytest --verbose --pyargs onedal $@ $(json_report_name onedal)
+pytest --verbose --pyargs onedal $@ $(generate_pytest_args onedal)
 return_code=$(($return_code + $?))
 
-pytest --verbose -s "${sklex_root}/.ci/scripts/test_global_patch.py" $@ $(json_report_name global_patching)
+pytest --verbose -s "${sklex_root}/.ci/scripts/test_global_patch.py" $@ $(generate_pytest_args global_patching)
 return_code=$(($return_code + $?))
 
 echo "NO_DIST=$NO_DIST"
@@ -84,11 +90,16 @@ if [[ ! $NO_DIST ]]; then
         export EXTRA_MPI_ARGS="-n 4"
     fi
     mpirun ${EXTRA_MPI_ARGS} python "${sklex_root}/tests/helper_mpi_tests.py" \
-        pytest -k spmd --with-mpi --verbose --pyargs sklearnex $@ $(json_report_name sklearnex_spmd)
+        pytest -k spmd --with-mpi --verbose --pyargs sklearnex $@ $(generate_pytest_args sklearnex_spmd)
     return_code=$(($return_code + $?))
     mpirun ${EXTRA_MPI_ARGS} python "${sklex_root}/tests/helper_mpi_tests.py" \
-        pytest --verbose -s "${sklex_root}/tests/test_daal4py_spmd_examples.py" $@ $(json_report_name mpi_legacy)
+        pytest --verbose -s "${sklex_root}/tests/test_daal4py_spmd_examples.py" $@ $(generate_pytest_args mpi_legacy)
     return_code=$(($return_code + $?))
 fi
 
+if [[ "$*" == *"--json-report"* ]] && ! [ -f .pytest_reports/legacy_report.json ]; then
+    echo "Error: JSON report files failed to be produced."
+    return_code=$(($return_code + 1))
+fi
+
 exit $return_code