MAINT: Implement option to oneDAL's lib path to sklearnex's RPATH (#2170

)

* add onedal lib to rpath

* make rpath addition optional

* change argument name to abs-rpath

* take only double-dash argument

INSTALL.md

@@ -210,6 +210,13 @@ The build-process (using setup.py) happens in 4 stages:
    python setup.py build
    ```
 
+**Note:** the `setup.py` file will accept an optional argument `--abs-rpath` on linux (for all of `build`/`install`/`develop`/etc.) which will make it add the absolute path to oneDAL's shared objects (.so files) to the rpath of the scikit-learn-intelex extension's shared object files in order to load them automatically. This is not necessary when installing from pip or conda, but can be helpful for development purposes when using a from-source build of oneDAL that resides in a custom folder, as it won't assume that oneDAL's files will be found under default system paths. Example:
+
+```shell
+python setup.py build_ext --inplace --force --abs-rpath
+python setup.py build --abs-rpath
+```
+
 Where: 
 
 * Keys `--single-version-externally-managed` and `--no-deps` are required to not download daal4py after the installation of Intel(R) Extension for Scikit-learn. 

scripts/CMakeLists.txt

@@ -26,6 +26,9 @@ set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS OFF)
 
+option(ADD_ONEDAL_RPATH "Adds oneDAL's file paths to the RPATH here" OFF)
+message(STATUS "ADD_ONEDAL_RPATH:" ${ADD_ONEDAL_RPATH})
+
 if(WIN32)
     # hint CMake to get python from PYTHON env. variable if defined
     if(DEFINED ENV{PYTHON})
@@ -40,7 +43,11 @@ elseif(UNIX)
         set(CMAKE_INSTALL_RPATH "@loader_path/../../../")
     else()
         set(ONEDAL_PY_LINK_OPTIONS "-Wl,-z,noexecstack,-z,relro,-z,now,-fstack-protector-strong,-fno-strict-overflow,-fno-delete-null-pointer-checks,-fwrapv")
-        set(CMAKE_INSTALL_RPATH "$ORIGIN/../../../")
+        if(ADD_ONEDAL_RPATH)
+            set(CMAKE_INSTALL_RPATH "${oneDAL_LIBRARY_DIR}:$ORIGIN/../../../")
+        else()
+            set(CMAKE_INSTALL_RPATH "$ORIGIN/../../../")
+        endif()
     endif()
     set(WARNING_FLAGS "-Winit-self")
 else()

scripts/build_backend.py

@@ -43,7 +43,12 @@
 
 
 def custom_build_cmake_clib(
-    iface, cxx=None, onedal_major_binary_version=1, no_dist=True, use_parameters_lib=True
+    iface,
+    cxx=None,
+    onedal_major_binary_version=1,
+    no_dist=True,
+    use_parameters_lib=True,
+    use_abs_rpath=False,
 ):
     import pybind11
 
@@ -122,6 +127,9 @@ def custom_build_cmake_clib(
             "-DMPI_LIBS=" + MPI_LIBS,
         ]
 
+    if use_abs_rpath:
+        cmake_args += ["-DADD_ONEDAL_RPATH=ON"]
+
     cpu_count = multiprocessing.cpu_count()
     # limit parallel cmake jobs if memory size is insufficient
     # TODO: add on all platforms

setup.py

@@ -41,6 +41,12 @@
 from scripts.package_helpers import get_packages_with_tests
 from scripts.version import get_onedal_shared_libs, get_onedal_version
 
+USE_ABS_RPATH = False
+ARG_ABS_RPATH = "--abs-rpath"
+if ARG_ABS_RPATH in sys.argv:
+    USE_ABS_RPATH = True
+    sys.argv = [arg for arg in sys.argv if arg != ARG_ABS_RPATH]
+
 IS_WIN = False
 IS_MAC = False
 IS_LIN = False
@@ -298,7 +304,9 @@ def get_build_options():
         ela.append("-s")
     if IS_LIN:
         ela.append("-fPIC")
-        ela.append("-Wl,-rpath,$ORIGIN/../../../")
+        ela.append(
+            f"-Wl,-rpath,{(daal_lib_dir + ':') if USE_ABS_RPATH else ''}$ORIGIN/../../../"
+        )
     return eca, ela, include_dir_plat
 
 
@@ -416,6 +424,7 @@ def run(self):
                 onedal_major_binary_version=ONEDAL_MAJOR_BINARY_VERSION,
                 no_dist=no_dist,
                 use_parameters_lib=use_parameters_lib,
+                use_abs_rpath=USE_ABS_RPATH,
             )
         if dpcpp:
             if is_onedal_iface:
@@ -424,13 +433,15 @@ def run(self):
                     onedal_major_binary_version=ONEDAL_MAJOR_BINARY_VERSION,
                     no_dist=no_dist,
                     use_parameters_lib=use_parameters_lib,
+                    use_abs_rpath=USE_ABS_RPATH,
                 )
                 if build_distribute:
                     build_backend.custom_build_cmake_clib(
                         iface="spmd_dpc",
                         onedal_major_binary_version=ONEDAL_MAJOR_BINARY_VERSION,
                         no_dist=no_dist,
                         use_parameters_lib=use_parameters_lib,
+                        use_abs_rpath=USE_ABS_RPATH,
                     )
 
     def post_build(self):