Merge branch 'EESSI:main' into mpi_injection

Author: agimenog

.github/workflows/tests_eessi_extend_module.yml

@@ -86,6 +86,19 @@ jobs:
           # check some specific ones
           check_env_var "EASYBUILD_INSTALLPATH" "$EESSI_SOFTWARE_PATH"
           check_env_var "EASYBUILD_UMASK" "022"
+          # Check some things that are EasyBuild 5+ (available in all EESSI versions so should exist)
+          check_env_var "EASYBUILD_STRICT_RPATH_SANITY_CHECK" "1"
+          check_env_var "EASYBUILD_CUDA_SANITY_CHECK_ERROR_ON_FAILED_CHECKS" "1"
+          check_env_var "EASYBUILD_FAIL_ON_MOD_FILES_GCCCORE" "1"
+          check_env_var "EASYBUILD_LOCAL_VAR_NAMING_CHECK" "error"
+          # Check some EESSI version specific settings
+          if [[ "${{ matrix.EESSI_VERSION }}" != "2023.06" ]]; then
+            check_env_var "EASYBUILD_PREFER_PYTHON_SEARCH_PATH" "EBPYTHONPREFIXES"
+            check_env_var "EASYBUILD_MODULE_SEARCH_PATH_HEADERS" "include_paths"
+            check_env_var "EASYBUILD_SEARCH_PATH_CPP_HEADERS" "include_paths"
+          fi
+          # verify the configuration is actually valid
+          eb --show-config
           # unload and check the environment is clean again
           module unload EESSI-extend
           check_disallowed_env_prefix EASYBUILD_
@@ -97,6 +110,8 @@ jobs:
           # check some specific envvars
           check_env_var "EASYBUILD_INSTALLPATH" "$EESSI_SITE_SOFTWARE_PATH"
           check_env_var "EASYBUILD_UMASK" "022"
+          # verify the configuration is actually valid
+          eb --show-config
           # unload and check the environment is clean again
           module unload EESSI-extend
           check_disallowed_env_prefix EASYBUILD_
@@ -110,6 +125,8 @@ jobs:
           check_env_var "EASYBUILD_INSTALLPATH" "$EXPECTED_INSTALLATION_PATH"
           check_env_var "EASYBUILD_UMASK" "002"
           check_env_var "EASYBUILD_GROUP_WRITABLE_INSTALLDIR" "1"
+          # verify the configuration is actually valid
+          eb --show-config
           # unload and check the environment is clean again
           module unload EESSI-extend
           check_disallowed_env_prefix EASYBUILD_
@@ -123,6 +140,8 @@ jobs:
           export EXPECTED_INSTALLATION_PATH="$MY_INSTALLATION_PATH/$USER/versions/$EESSI_VERSION/software/linux/$EESSI_SOFTWARE_SUBDIR"
           check_env_var "EASYBUILD_INSTALLPATH" "$EXPECTED_INSTALLATION_PATH"
           check_env_var "EASYBUILD_UMASK" "077"
+          # verify the configuration is actually valid
+          eb --show-config
           # unload and check the environment is clean again
           module unload EESSI-extend
           check_disallowed_env_prefix EASYBUILD_

.github/workflows/tests_link_nvidia_host_libraries.yml

@@ -19,7 +19,18 @@ jobs:
 
     - name: Initialize EESSI
       uses: eessi/github-action-eessi@v3
-      
+
+    - name: Test function nvidia_gpu_available before setup of libraries
+      run: |
+         source scripts/utils.sh
+         if nvidia_gpu_available; then
+             echo "Error: Found NVIDIA libraries before the mock libraries were set up."
+             exit 1
+         else
+             echo "NVIDIA libraries were not found."
+             echo "Proceeding to setting up the mock NVIDIA libraries."
+         fi
+
     - name: Setup mock NVIDIA libraries
       run: |
         # Run the script to create mock libraries
@@ -48,6 +59,16 @@ jobs:
         echo "Updating PATH"
         echo "PATH=/tmp/nvidia-bin:$PATH" >> $GITHUB_ENV
 
+    - name: Test nvidia_gpu_available after setup of mock libraries
+      run: |
+        source scripts/utils.sh
+        if nvidia_gpu_available; then
+             echo "mock NVIDIA libraries and nvidia-smi were set up"
+         else
+             echo "Error: mock nvidia-smi is not available."
+             exit 1
+         fi
+
     - name: Test LD_PRELOAD mode
       run: |
         echo ">>> Testing LD_PRELOAD mode"

.github/workflows/tests_scripts.yml

@@ -160,6 +160,6 @@ jobs:
           export EESSI_VERSION_OVERRIDE="${{matrix.EESSI_VERSION}}"
 
           out="${PWD}/eb-${EB_VERSION}.out"
-          ./eessi_container.sh --access rw --mode run --verbose /software-layer-scripts/run_in_compat_layer_env.sh ls 2>&1 | tee ${out}
+          ./eessi_container.sh --repository software.eessi.io,access=rw,mount=bind --mode run --verbose /software-layer-scripts/run_in_compat_layer_env.sh ls 2>&1 | tee ${out}
           echo $(grep "SINGULARITY_BIND" ${out})
           grep "SINGULARITY_BIND" ${out} | grep "software.eessi.io" || (echo "software.eessi.io did not seem to be bind mounted!" && exit 1)

EESSI-extend-easybuild.eb

@@ -45,10 +45,16 @@ description = """
 toolchain = SYSTEM
 
 # All the dependencies we filter in EESSI
-local_deps_to_filter = "Autoconf,Automake,Autotools,binutils,bzip2,DBus,flex,gettext,gperf,help2man,intltool,libreadline,libtool,M4,makeinfo,ncurses,ParMETIS,util-linux,XZ,zlib"
+local_deps_to_filter = "binutils,bzip2,DBus,flex,gettext,gperf,help2man,intltool,libreadline,makeinfo,ncurses,ParMETIS,util-linux,XZ,zlib"
+
+# filter extra dependencies based on CPU family
 local_arch_specific_deps_to_filter = {'aarch64': ',Yasm', 'riscv64': ',Yasm', 'x86_64': ''}
 local_deps_to_filter += local_arch_specific_deps_to_filter[ARCH]
 
+# only filter Autotools & co for EESSI 2023.06
+if version == '2023.06':
+    local_deps_to_filter += ",Autoconf,Automake,Autotools,libtool,M4"
+
 # Set the universal EasyBuild variables
 modextravars = {
     'EASYBUILD_FILTER_DEPS': local_deps_to_filter,
@@ -70,7 +76,7 @@ modextravars = {
 # EASYBUILD_INSTALLPATH=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR}
 # EASYBUILD_SOURCEPATH=${WORKDIR}/easybuild/sources:${EESSI_SOURCEPATH}
 #
-# And also some optional ones based on the kind of installation
+# And also some optional ones based on the installation mode
 # EASYBUILD_SET_GID_BIT
 # EASYBUILD_GROUP_WRITABLE_INSTALLDIR
 # EASYBUILD_UMASK
@@ -98,21 +104,26 @@ if (eessi_accelerator_target ~= nil) then
   end
 end
 
+-- Some environment variables affect behaviour, let's gather them once
+local eessi_cvmfs_install = os.getenv("EESSI_CVMFS_INSTALL") ~= nil
+local eessi_site_install = os.getenv("EESSI_SITE_INSTALL") ~= nil
+local eessi_project_install = os.getenv("EESSI_PROJECT_INSTALL") ~= nil
+local eessi_user_install = os.getenv("EESSI_USER_INSTALL") ~= nil
+
 -- Use an installation prefix that we _should_ have write access to
-if (os.getenv("EESSI_CVMFS_INSTALL") ~= nil) then
+if eessi_cvmfs_install then
   -- Make sure no other EESSI install environment variables are set
-  if ((os.getenv("EESSI_SITE_INSTALL") ~= nil) or (os.getenv("EESSI_PROJECT_INSTALL") ~= nil) or (os.getenv("EESSI_USER_INSTALL") ~= nil)) then
+  if (eessi_site_install or eessi_project_install or eessi_user_install) then
     LmodError("You cannot use EESSI_CVMFS_INSTALL in combination with any other EESSI_*_INSTALL environment variables")
   end
-  eessi_cvmfs_install = true
   easybuild_installpath = os.getenv("EESSI_SOFTWARE_PATH")
   -- enforce accelerator subdirectory usage for CVMFS installs (only if an accelerator install is requested)
   if (eessi_accelerator_target ~= nil) and (cuda_compute_capability ~= nil) and (os.getenv("EESSI_ACCELERATOR_INSTALL") ~= nil) then
       easybuild_installpath = pathJoin(easybuild_installpath, eessi_accelerator_target)
   end
-elseif (os.getenv("EESSI_SITE_INSTALL") ~= nil) then
+elseif eessi_site_install then
   -- Make sure no other EESSI install environment variables are set
-  if ((os.getenv("EESSI_PROJECT_INSTALL") ~= nil) or (os.getenv("EESSI_USER_INSTALL") ~= nil)) then
+  if (eessi_project_install or eessi_user_install) then
     LmodError("You cannot use EESSI_SITE_INSTALL in combination with any other EESSI_*_INSTALL environment variables")
   end
   easybuild_installpath = os.getenv("EESSI_SITE_SOFTWARE_PATH")
@@ -122,35 +133,35 @@ elseif (os.getenv("EESSI_SITE_INSTALL") ~= nil) then
   end
 else
   -- Deal with user and project installs
-  project_install = os.getenv("EESSI_PROJECT_INSTALL")
+  project_install_dir = os.getenv("EESSI_PROJECT_INSTALL")
   project_modulepath = nil
-  if (project_install ~= nil) then
+  if eessi_project_install then
     -- Check the folder exists
-    if not isDir(project_install) then
-      LmodError("The location of EESSI_PROJECT_INSTALL (" .. project_install .. ") does not exist or is not a folder")
+    if not isDir(project_install_dir) then
+      LmodError("The location of EESSI_PROJECT_INSTALL (" .. project_install_dir .. ") does not exist or is not a folder")
     end
     if (mode() == "load") then
-      LmodMessage("Configuring for use of EESSI_PROJECT_INSTALL under " .. project_install)
+      LmodMessage("Configuring for use of EESSI_PROJECT_INSTALL under " .. project_install_dir)
     end
-    easybuild_installpath = string.gsub(os.getenv("EESSI_SOFTWARE_PATH"), os.getenv("EESSI_CVMFS_REPO"), project_install)
+    easybuild_installpath = string.gsub(os.getenv("EESSI_SOFTWARE_PATH"), os.getenv("EESSI_CVMFS_REPO"), project_install_dir)
     project_modulepath = pathJoin(easybuild_installpath, 'modules', 'all')
   end
-  user_install = os.getenv("EESSI_USER_INSTALL")
+  user_install_dir = os.getenv("EESSI_USER_INSTALL")
   user_modulepath = nil
-  if (user_install ~= nil) then
+  if eessi_user_install then
     -- Check the folder exists
-    if not isDir(user_install) then
-      LmodError("The location of EESSI_USER_INSTALL (" .. user_install .. ") does not exist or is not a folder")
+    if not isDir(user_install_dir) then
+      LmodError("The location of EESSI_USER_INSTALL (" .. user_install_dir .. ") does not exist or is not a folder")
     end
-  elseif (user_install == nil) and (project_install == nil) then
+  elseif (user_install_dir == nil) and (project_install_dir == nil) then
     -- No need to check for existence when we use a HOME subdir
-    user_install = pathJoin(os.getenv("HOME"), "eessi")
+    user_install_dir = pathJoin(os.getenv("HOME"), "eessi")
   end
-  if (user_install ~= nil) then
+  if (user_install_dir ~= nil) then
     if (mode() == "load") then
-      LmodMessage("Configuring for use of EESSI_USER_INSTALL under " .. user_install)
+      LmodMessage("Configuring for use of EESSI_USER_INSTALL under " .. user_install_dir)
     end
-    easybuild_installpath = string.gsub(os.getenv("EESSI_SOFTWARE_PATH"), os.getenv("EESSI_CVMFS_REPO"), user_install)
+    easybuild_installpath = string.gsub(os.getenv("EESSI_SOFTWARE_PATH"), os.getenv("EESSI_CVMFS_REPO"), user_install_dir)
     user_modulepath = pathJoin(easybuild_installpath, 'modules', 'all')
   end
 end
@@ -196,10 +207,33 @@ elseif (project_modulepath ~= nil) then
   -- configure MODULEPATH
   prepend_path("MODULEPATH", project_modulepath)
 end
+
 -- Make sure EasyBuild itself is loaded
+-- need to also handle the unload behaviour where the version is defined only before we unload
+easybuild_version = os.getenv("EBVERSIONEASYBUILD")
 if not ( isloaded("EasyBuild") ) then
     load(latest("EasyBuild"))
 end
+easybuild_version = os.getenv("EBVERSIONEASYBUILD") or easybuild_version
+eessi_version = os.getenv("EESSI_VERSION") or "2023.06"
+
+-- Set environment variables that are EasyBuild version specific
+-- Do unload unconditionally, so that even if EB versions were switched in the meantime, this gets unset
+-- This avoids issues where EESSI-extend is first loaded with EB => 5.1 (which set these vars)
+-- but then EB is swapped for a version < 5.1 and then EESSI-extend is unloaded (which would not unset
+-- these vars if we did it conditional on the EB version)
+if mode() == "unload" or mode() == "dependencyCk" or convertToCanonical(easybuild_version) >= convertToCanonical("5.1") then
+  setenv ("EASYBUILD_STRICT_RPATH_SANITY_CHECK", "1")
+  setenv ("EASYBUILD_CUDA_SANITY_CHECK_ERROR_ON_FAILED_CHECKS", "1")
+  setenv ("EASYBUILD_FAIL_ON_MOD_FILES_GCCCORE", "1")
+  setenv ("EASYBUILD_LOCAL_VAR_NAMING_CHECK", "error")
+  -- Set environment variables that are EESSI version specific
+  if convertToCanonical(eessi_version) > convertToCanonical("2023.06") then
+    setenv ("EASYBUILD_PREFER_PYTHON_SEARCH_PATH", "EBPYTHONPREFIXES")
+    setenv ("EASYBUILD_MODULE_SEARCH_PATH_HEADERS", "include_paths")
+    setenv ("EASYBUILD_SEARCH_PATH_CPP_HEADERS", "include_paths")
+  end
+end
 """
 
 moduleclass = 'devel'