Merge branch 'intel:main' into dev/sklearnex_assert_all_finite

uxlfoundation · Nov 20, 2024 · a571a4e · a571a4e
2 parents 32c565d + 6e0fc2a
commit a571a4e
Show file tree

Hide file tree

Showing 17 changed files with 261 additions and 217 deletions.
diff --git a/.ci/scripts/run_sklearn_tests.py b/.ci/scripts/run_sklearn_tests.py
@@ -37,16 +37,26 @@
     )
     args = parser.parse_args()
 
-    os.chdir(os.path.dirname(sklearn.__file__))
+    sklearn_file_dir = os.path.dirname(sklearn.__file__)
+    os.chdir(sklearn_file_dir)
 
     if os.environ["SELECTED_TESTS"] == "all":
         os.environ["SELECTED_TESTS"] = ""
 
     pytest_args = (
         "--verbose --durations=100 --durations-min=0.01 "
-        f"--rootdir={os.path.dirname(sklearn.__file__)} "
+        f"--rootdir={sklearn_file_dir} "
         f'{os.environ["DESELECTED_TESTS"]} {os.environ["SELECTED_TESTS"]}'.split(" ")
     )
+
+    if rc := os.getenv("COVERAGE_RCFILE"):
+        pytest_args += (
+            "--cov=onedal",
+            "--cov=sklearnex",
+            f"--cov-config={rc}",
+            "--cov-report=",
+        )
+
     while "" in pytest_args:
         pytest_args.remove("")
 
@@ -55,4 +65,5 @@
             return_code = pytest.main(pytest_args)
     else:
         return_code = pytest.main(pytest_args)
+
     sys.exit(int(return_code))
diff --git a/.coveragerc b/.coveragerc
@@ -0,0 +1,40 @@
+#===============================================================================
+# Copyright 2024 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#===============================================================================
+[run]
+branch = True
+source = 
+    sklearnex
+    onedal
+parallel = True
+omit =
+    */onedal/tests/*
+    */onedal/basic_statistics/tests/utils.py
+    */sklearnex/conftest.py
+    */sklearnex/tests/*
+    */sklearnex/preview/*
+    */sklearnex/spmd/*
+[paths]
+sklearnex =
+    sklearnex/
+    */sklearnex
+onedal =
+    onedal/
+    */onedal
+#Uncomment lines below for report generation when an Intel GPU is unavailable
+#[report]
+#exclude_also =
+#    def _onedal_gpu.*
+#    coverage: gpu_start(?s:.)*coverage: gpu_stop
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -4,9 +4,9 @@
 .circleci/                    @napetrov @Alexsandruss @homksei @ahuber21 @ethanglaser
 
 # Docs
-*.md                          @maria-Petrova @napetrov @emmwalsh @Alexsandruss @samir-nasibli @icfaust
-doc/                          @maria-Petrova @napetrov @emmwalsh @Alexsandruss @samir-nasibli @icfaust
-requirements-doc.txt          @emmwalsh @Alexsandruss
+*.md                          @maria-Petrova @napetrov @Alexsandruss @samir-nasibli @icfaust @david-cortes-intel
+doc/                          @maria-Petrova @napetrov @Alexsandruss @samir-nasibli @icfaust @david-cortes-intel
+requirements-doc.txt          @Alexsandruss @samir-nasibli @icfaust @david-cortes-intel
 
 # DPC++
 onedal/                       @Alexsandruss @samir-nasibli @icfaust

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -93,6 +93,8 @@ jobs:
           DPCFLAG="0"
           if [ $(echo ${{ matrix.PYTHON_VERSION }} | grep '${{ env.DPCTL_PY_VERSIONS }}') ]; then DPCFLAG=''; fi
           echo "DPCFLAG=${DPCFLAG}" >> "$GITHUB_OUTPUT"
+          # enable coverage report generation
+          echo "COVERAGE_RCFILE=$(readlink -f .coveragerc)" >> "$GITHUB_ENV"
       - name: apt-get
         run: sudo apt-get update && sudo apt-get install -y clang-format
       - name: dpcpp installation
@@ -127,14 +129,24 @@ jobs:
         run: |
           source venv/bin/activate
           source .github/scripts/activate_components.sh
+          export COVERAGE_FILE=$(pwd)/.coverage.sklearnex
           cd .ci
           ../conda-recipe/run_test.sh
       - name: Sklearn testing
         run: |
           source venv/bin/activate
           source .github/scripts/activate_components.sh
+          export COVERAGE_FILE=$(pwd)/.coverage.sklearn
           if [ "${{ steps.set-env.outputs.DPCFLAG }}" == "" ]; then export CPU=cpu; fi
           bash .ci/scripts/run_sklearn_tests.sh $CPU
+      - name: Archive coverage data
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage_lnx_Py${{ matrix.PYTHON_VERSION }}_${{ matrix.SKLEARN_VERSION }}
+          path: |
+             .coverage.sklearn
+             .coverage.sklearnex
+          include-hidden-files: true
       - name: Sklearn testing [preview]
         run: |
           source venv/bin/activate
@@ -199,15 +211,13 @@ jobs:
           echo "Unzip complete"
       - name: Set Environment Variables
         id: set-env
-        shell: bash
+        shell: cmd
         run: |
-          echo "C:\msys64\usr\bin;C:\Program Files (x86)\Common Files\Intel\Shared Libraries\bin;" >> $GITHUB_PATH
-          # Disable SPMD testing
-          echo "NO_DIST=1" >> "$GITHUB_ENV"
-          # Determine dpctl support
-          DPCFLAG="0"
-          if [ $(echo ${{ matrix.PYTHON_VERSION }} | grep '${{ env.DPCTL_PY_VERSIONS }}') ]; then DPCFLAG=''; fi
-          echo "DPCFLAG=${DPCFLAG}" >> "$GITHUB_OUTPUT"
+          echo C:\msys64\usr\bin;>> %GITHUB_PATH%
+          echo NO_DIST=YES>> %GITHUB_ENV%
+          set DPCTL_TEMP="${{ env.DPCTL_PY_VERSIONS }}"
+          if not %DPCTL_TEMP:${{ matrix.PYTHON_VERSION }}=%==%DPCTL_TEMP% (echo DPCFLAG=>> %GITHUB_OUTPUT%) else (echo DPCFLAG="0">> %GITHUB_OUTPUT%)
+          echo COVERAGE_RCFILE=%cd%\.coveragerc>> %GITHUB_ENV%
       - name: Download Intel OpenCL CPU Runtime artifact
         if: ${{ steps.set-env.outputs.DPCFLAG == '' }}
         uses: actions/download-artifact@v4
@@ -261,15 +271,25 @@ jobs:
           call .\venv\Scripts\activate.bat
           call .\.github\scripts\activate_components.bat ${{ steps.set-env.outputs.DPCFLAG }}
           set PYTHON=python
+          set COVERAGE_FILE=%cd%\.coverage.sklearnex
           cd ..
           call scikit-learn-intelex\conda-recipe\run_test.bat scikit-learn-intelex\
       - name: Sklearn testing
         shell: cmd
         run: |
           call .\venv\Scripts\activate.bat
           call .\.github\scripts\activate_components.bat ${{ steps.set-env.outputs.DPCFLAG }}
+          set COVERAGE_FILE=%cd%\.coverage.sklearn
           if "${{ steps.set-env.outputs.DPCFLAG }}"=="" set CPU=cpu
           bash .ci/scripts/run_sklearn_tests.sh %CPU%
+      - name: Archive coverage data
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage_win_Py${{ matrix.PYTHON_VERSION }}_${{ matrix.SKLEARN_VERSION }}
+          path: |
+             .coverage.sklearn
+             .coverage.sklearnex
+          include-hidden-files: true
       - name: Sklearn testing [preview]
         shell: cmd
         run: |

diff --git a/INSTALL.md b/INSTALL.md
@@ -161,17 +161,20 @@ The build-process (using setup.py) happens in 4 stages:
 * [Intel® oneAPI Data Analytics Library (oneDAL)](https://github.com/oneapi-src/oneDAL) version 2021.1 or later
   * You can use the pre-built `dal-devel` conda package from conda-forge channel
 * MPI (optional, needed for distributed mode)
-  * You can use the pre-built `impi-devel` conda package from conda-forge channel
+  * You can use the pre-built `impi_rt` and `impi-devel` conda packages from conda-forge channel
 * A DPC++ compiler (optional, needed for DPC++ interfaces)
 
 ### Configure the Build with Environment Variables
 * ``SKLEARNEX_VERSION``: sets the package version
 * ``DALROOT``: sets the oneAPI Data Analytics Library path
+* ``MPIROOT``: sets the path to the MPI library that will be used for distributed mode support. Not used when using `NO_DIST=1`
 * ``NO_DIST``: set to '1', 'yes' or alike to build without support for distributed mode
 * ``NO_STREAM``: set to '1', 'yes' or alike to build without support for streaming mode
 * ``NO_DPC``: set to '1', 'yes' or alike to build without support of oneDAL DPC++ interfaces
 * ``OFF_ONEDAL_IFACE``: set to '1' to build without the support of oneDAL interfaces
 
+**Note:** in order to use distributed mode, `mpi4py` is also required, and needs to be built with the same MPI backend as scikit-learn-intelex.
+
 ### Build Intel(R) Extension for Scikit-learn
 
 - To install the package:

diff --git a/conda-recipe/run_test.bat b/conda-recipe/run_test.bat
@@ -28,54 +28,46 @@ if "%PYTHON%"=="python" (
     set NO_DIST=1
 )
 
+
+
 %PYTHON% -c "from sklearnex import patch_sklearn; patch_sklearn()" || set exitcode=1
 
+set "PYTEST_ARGS= "
+
+IF DEFINED COVERAGE_RCFILE (set "PYTEST_ARGS=--cov=onedal --cov=sklearnex --cov-config=%COVERAGE_RCFILE% --cov-append --cov-report= %PYTEST_ARGS%")
+
 rem Note: execute with argument --json-report as second argument
 rem in order to produce a JSON report under folder '.pytest_reports'.
-set with_json_report=0
 if "%~2"=="--json-report" (
-    set with_json_report=1
+    set "PYTEST_ARGS=--json-report --json-report-file=.pytest_reports\FILENAME.json %PYTEST_ARGS%"
+    echo %PYTEST_ARGS%
     mkdir .pytest_reports
     del /q .pytest_reports\*.json
 )
 
 echo "NO_DIST=%NO_DIST%"
-echo "with_json_report=%with_json_report%"
 setlocal enabledelayedexpansion
-if "%with_json_report%"=="1" (
-    pytest --verbose -s "%1tests" --json-report --json-report-file=.pytest_reports\legacy_report.json || set exitcode=1
-    pytest --verbose --pyargs daal4py --json-report --json-report-file=.pytest_reports\daal4py_report.json || set exitcode=1
-    pytest --verbose --pyargs sklearnex --json-report --json-report-file=.pytest_reports\sklearnex_report.json || set exitcode=1
-    pytest --verbose --pyargs onedal --json-report --json-report-file=.pytest_reports\onedal_report.json || set exitcode=1
-    pytest --verbose "%1.ci\scripts\test_global_patch.py" --json-report --json-report-file=.pytest_reports\global_patching_report.json || set exitcode=1
-    if NOT "%NO_DIST%"=="1" (
-        %PYTHON% "%1tests\helper_mpi_tests.py"^
-            pytest -k spmd --with-mpi --verbose -s --pyargs sklearnex^
-            --json-report --json-report-file=.pytest_reports\sklearnex_spmd.json
-        if !errorlevel! NEQ 0 (
-            set exitcode=1
-        )
-        %PYTHON% "%1tests\helper_mpi_tests.py"^
-            pytest --with-mpi --verbose -s "%1tests\test_daal4py_spmd_examples.py"^
-            --json-report --json-report-file=.pytest_reports\mpi_legacy.json
-        if !errorlevel! NEQ 0 (
-            set exitcode=1
-        )
+pytest --verbose -s "%1tests" %PYTEST_ARGS:FILENAME=legacy_report% || set exitcode=1
+pytest --verbose --pyargs daal4py %PYTEST_ARGS:FILENAME=daal4py_report% || set exitcode=1
+pytest --verbose --pyargs sklearnex %PYTEST_ARGS:FILENAME=sklearnex_report% || set exitcode=1
+pytest --verbose --pyargs onedal %PYTEST_ARGS:FILENAME=onedal_report% || set exitcode=1
+pytest --verbose "%1.ci\scripts\test_global_patch.py" %PYTEST_ARGS:FILENAME=global_patching_report% || set exitcode=1
+if NOT "%NO_DIST%"=="1" (
+    %PYTHON% "%1tests\helper_mpi_tests.py"^
+        pytest -k spmd --with-mpi --verbose -s --pyargs sklearnex %PYTEST_ARGS:FILENAME=sklearnex_spmd%
+    if !errorlevel! NEQ 0 (
+        set exitcode=1
     )
+    %PYTHON% "%1tests\helper_mpi_tests.py"^
+        pytest --with-mpi --verbose -s "%1tests\test_daal4py_spmd_examples.py" %PYTEST_ARGS:FILENAME=mpi_legacy%
+    if !errorlevel! NEQ 0 (
+        set exitcode=1
+    )
+)
+if "%~2"=="--json-report" (
     if NOT EXIST .pytest_reports\legacy_report.json (
         echo "Error: JSON report files failed to be produced."
         set exitcode=1
     )
-) else (
-    pytest --verbose -s "%1tests" || set exitcode=1
-    pytest --verbose --pyargs daal4py || set exitcode=1
-    pytest --verbose --pyargs sklearnex || set exitcode=1
-    pytest --verbose --pyargs onedal || set exitcode=1
-    pytest --verbose "%1.ci\scripts\test_global_patch.py" || set exitcode=1
-    if NOT "%NO_DIST%"=="1" (
-        %PYTHON% -m pytest -k spmd --with-mpi --verbose --pyargs sklearnex || set exitcode=1
-        %PYTHON% -m pytest --verbose -s "%1tests\test_daal4py_spmd_examples.py" || set exitcode=1
-    )
 )
-
 EXIT /B %exitcode%
diff --git a/conda-recipe/run_test.sh b/conda-recipe/run_test.sh
@@ -48,28 +48,34 @@ if [[ "$*" == *"--json-report"* ]]; then
         rm .pytest_reports/*.json
     fi
 fi
-function json_report_name {
+
+function generate_pytest_args {
+    declare -a ARGS=()
     if [[ "${with_json_report}" == "1" ]]; then
-        printf -- "--json-report-file=.pytest_reports/$1_report.json"
+        ARGS+=("--json-report-file=.pytest_reports/$1_report.json")
+    fi
+    if [ -n "${COVERAGE_RCFILE}" ]; then
+        ARGS+=(--cov=onedal --cov=sklearnex --cov-config="${COVERAGE_RCFILE}" --cov-append --cov-report=)
     fi
+    printf -- "${ARGS[*]}"
 }
 
 ${PYTHON} -c "from sklearnex import patch_sklearn; patch_sklearn()"
 return_code=$(($return_code + $?))
 
-pytest --verbose -s "${sklex_root}/tests" $@ $(json_report_name legacy)
+pytest --verbose -s "${sklex_root}/tests" $@ $(generate_pytest_args legacy)
 return_code=$(($return_code + $?))
 
-pytest --verbose --pyargs daal4py $@ $(json_report_name daal4py)
+pytest --verbose --pyargs daal4py $@ $(generate_pytest_args daal4py)
 return_code=$(($return_code + $?))
 
-pytest --verbose --pyargs sklearnex $@ $(json_report_name sklearnex)
+pytest --verbose --pyargs sklearnex $@ $(generate_pytest_args sklearnex)
 return_code=$(($return_code + $?))
 
-pytest --verbose --pyargs onedal $@ $(json_report_name onedal)
+pytest --verbose --pyargs onedal $@ $(generate_pytest_args onedal)
 return_code=$(($return_code + $?))
 
-pytest --verbose -s "${sklex_root}/.ci/scripts/test_global_patch.py" $@ $(json_report_name global_patching)
+pytest --verbose -s "${sklex_root}/.ci/scripts/test_global_patch.py" $@ $(generate_pytest_args global_patching)
 return_code=$(($return_code + $?))
 
 echo "NO_DIST=$NO_DIST"
@@ -84,11 +90,16 @@ if [[ ! $NO_DIST ]]; then
         export EXTRA_MPI_ARGS="-n 4"
     fi
     mpirun ${EXTRA_MPI_ARGS} python "${sklex_root}/tests/helper_mpi_tests.py" \
-        pytest -k spmd --with-mpi --verbose --pyargs sklearnex $@ $(json_report_name sklearnex_spmd)
+        pytest -k spmd --with-mpi --verbose --pyargs sklearnex $@ $(generate_pytest_args sklearnex_spmd)
     return_code=$(($return_code + $?))
     mpirun ${EXTRA_MPI_ARGS} python "${sklex_root}/tests/helper_mpi_tests.py" \
-        pytest --verbose -s "${sklex_root}/tests/test_daal4py_spmd_examples.py" $@ $(json_report_name mpi_legacy)
+        pytest --verbose -s "${sklex_root}/tests/test_daal4py_spmd_examples.py" $@ $(generate_pytest_args mpi_legacy)
     return_code=$(($return_code + $?))
 fi
 
+if [[ "$*" == *"--json-report"* ]] && ! [ -f .pytest_reports/legacy_report.json ]; then
+    echo "Error: JSON report files failed to be produced."
+    return_code=$(($return_code + 1))
+fi
+
 exit $return_code
diff --git a/daal4py/sklearn/utils/validation.py b/daal4py/sklearn/utils/validation.py
@@ -98,9 +98,12 @@ def _assert_all_finite(
     )
     _dal_ready = _patching_status.and_conditions(
         [
-            (X.ndim in [1, 2], "X has not 1 or 2 dimensions."),
-            (not np.any(np.equal(X.shape, 0)), "X shape contains 0."),
-            (dt in [np.float32, np.float64], "X dtype is not float32 or float64."),
+            (X.ndim in [1, 2], f"Input {input_name} does not have 1 or 2 dimensions."),
+            (not np.any(np.equal(X.shape, 0)), f"Input {input_name} shape contains a 0."),
+            (
+                dt in [np.float32, np.float64],
+                f"Input {input_name} dtype is not float32 or float64.",
+            ),
         ]
     )
     _patching_status.write_log()