From be90f89e1b5c556ef978f0796e25fda09e01ab42 Mon Sep 17 00:00:00 2001 From: Ivan Grigorik Date: Thu, 25 Dec 2025 19:13:42 -0600 Subject: [PATCH 01/12] fix CUDA_HOME env variable --- base/cmake/Modules/KokkosPythonKokkos.cmake | 177 ++++++++++++++++++-- 1 file changed, 166 insertions(+), 11 deletions(-) diff --git a/base/cmake/Modules/KokkosPythonKokkos.cmake b/base/cmake/Modules/KokkosPythonKokkos.cmake index 4b76390e..42177c85 100644 --- a/base/cmake/Modules/KokkosPythonKokkos.cmake +++ b/base/cmake/Modules/KokkosPythonKokkos.cmake @@ -84,26 +84,181 @@ IF(_INTERNAL_KOKKOS) SET(Threads_FOUND OFF) SET(CUDA_FOUND OFF) - IF(NOT Kokkos_ENABLE_THREADS) - FIND_PACKAGE(OpenMP QUIET) - ENDIF() + FIND_PACKAGE(OpenMP QUIET) - IF(NOT DEFINED Kokkos_ENABLE_THREADS AND NOT OpenMP_FOUND) + # Only check for Threads if OpenMP was not found and not explicitly disabled + IF(NOT OpenMP_FOUND AND NOT DEFINED Kokkos_ENABLE_OPENMP) FIND_PACKAGE(Threads QUIET) + ELSE() + SET(Threads_FOUND OFF) ENDIF() - IF(NOT DEFINED Kokkos_ENABLE_CUDA) - FIND_PACKAGE(CUDA QUIET) + # Only enable CUDA if EXPLICITLY requested + # Check both ENABLE_CUDA and Kokkos_ENABLE_CUDA + SET(_CUDA_EXPLICITLY_ENABLED OFF) + + # Check if explicitly enabled + IF((DEFINED ENABLE_CUDA AND ENABLE_CUDA) OR (DEFINED Kokkos_ENABLE_CUDA AND Kokkos_ENABLE_CUDA)) + SET(_CUDA_EXPLICITLY_ENABLED ON) + ENDIF() + + # Only search for CUDA if explicitly enabled + IF(_CUDA_EXPLICITLY_ENABLED AND NOT DEFINED Kokkos_ENABLE_CUDA) + # Use modern CUDAToolkit package instead of deprecated CUDA module + # First, search for CUDA in common installation directories + FILE(GLOB CUDA_SEARCH_PATHS + "/usr/local/cuda-*" + "/usr/local/cuda" + "/opt/cuda-*" + "/opt/cuda" + ) + + # Add common CUDA paths to CMAKE_PREFIX_PATH to help find complete installations + IF(CUDA_SEARCH_PATHS) + LIST(SORT CUDA_SEARCH_PATHS ORDER DESCENDING) # Prefer newer versions + FOREACH(CUDA_PATH ${CUDA_SEARCH_PATHS}) + IF(EXISTS "${CUDA_PATH}/bin/nvcc" AND EXISTS "${CUDA_PATH}/include/cuda_runtime.h") + LIST(APPEND CMAKE_PREFIX_PATH "${CUDA_PATH}") + ENDIF() + ENDFOREACH() + ENDIF() + + FIND_PACKAGE(CUDAToolkit QUIET) + IF(CUDAToolkit_FOUND) + SET(CUDA_FOUND ON) + + # Set CUDA paths for the build system + GET_FILENAME_COMPONENT(CUDA_TOOLKIT_ROOT "${CUDAToolkit_BIN_DIR}" DIRECTORY) + + # Verify we have cuda_runtime.h in the include directory + IF(NOT EXISTS "${CUDAToolkit_INCLUDE_DIRS}/cuda_runtime.h") + + # Try to find a complete CUDA installation manually + FOREACH(CUDA_PATH ${CUDA_SEARCH_PATHS}) + IF(EXISTS "${CUDA_PATH}/include/cuda_runtime.h") + SET(CUDA_TOOLKIT_ROOT "${CUDA_PATH}") + SET(CUDAToolkit_BIN_DIR "${CUDA_PATH}/bin") + SET(CUDAToolkit_INCLUDE_DIRS "${CUDA_PATH}/include") + SET(CUDAToolkit_NVCC_EXECUTABLE "${CUDA_PATH}/bin/nvcc") + BREAK() + ENDIF() + ENDFOREACH() + ENDIF() + + # Set environment variables for the build + SET(ENV{CUDA_HOME} "${CUDA_TOOLKIT_ROOT}") + SET(ENV{CUDACXX} "${CUDAToolkit_NVCC_EXECUTABLE}") + + # Set CMAKE_CUDA_COMPILER if not already set + IF(NOT CMAKE_CUDA_COMPILER) + SET(CMAKE_CUDA_COMPILER "${CUDAToolkit_NVCC_EXECUTABLE}" CACHE FILEPATH "CUDA compiler" FORCE) + ENDIF() + + # Enable CUDA language now that we have the correct compiler + INCLUDE(CheckLanguage) + CHECK_LANGUAGE(CUDA) + IF(CMAKE_CUDA_COMPILER) + ENABLE_LANGUAGE(CUDA) + ENDIF() + + # Set default CUDA architectures if not specified + IF(NOT DEFINED CMAKE_CUDA_ARCHITECTURES) + SET(CMAKE_CUDA_ARCHITECTURES "70;75;80" CACHE STRING "CUDA architectures" FORCE) + ENDIF() + + # Pass CUDA paths to Kokkos + SET(Kokkos_CUDA_DIR "${CUDA_TOOLKIT_ROOT}" CACHE PATH "CUDA installation directory" FORCE) + + # Add CUDA include directories to the include path + INCLUDE_DIRECTORIES(SYSTEM ${CUDAToolkit_INCLUDE_DIRS}) + ELSE() + SET(CUDA_FOUND OFF) + ENDIF() + ELSEIF(_CUDA_EXPLICITLY_ENABLED) + # CUDA explicitly enabled - ensure we have it + # Search for CUDA in common installation directories + FILE(GLOB CUDA_SEARCH_PATHS + "/usr/local/cuda-*" + "/usr/local/cuda" + "/opt/cuda-*" + "/opt/cuda" + ) + + # Add common CUDA paths to CMAKE_PREFIX_PATH + IF(CUDA_SEARCH_PATHS) + LIST(SORT CUDA_SEARCH_PATHS ORDER DESCENDING) + FOREACH(CUDA_PATH ${CUDA_SEARCH_PATHS}) + IF(EXISTS "${CUDA_PATH}/bin/nvcc" AND EXISTS "${CUDA_PATH}/include/cuda_runtime.h") + LIST(APPEND CMAKE_PREFIX_PATH "${CUDA_PATH}") + ENDIF() + ENDFOREACH() + ENDIF() + + FIND_PACKAGE(CUDAToolkit REQUIRED) + SET(CUDA_FOUND ON) + + # Set CUDA paths for the build system + GET_FILENAME_COMPONENT(CUDA_TOOLKIT_ROOT "${CUDAToolkit_BIN_DIR}" DIRECTORY) + + # Verify we have cuda_runtime.h + IF(NOT EXISTS "${CUDAToolkit_INCLUDE_DIRS}/cuda_runtime.h") + # Try to find complete installation + FOREACH(CUDA_PATH ${CUDA_SEARCH_PATHS}) + IF(EXISTS "${CUDA_PATH}/include/cuda_runtime.h") + SET(CUDA_TOOLKIT_ROOT "${CUDA_PATH}") + SET(CUDAToolkit_BIN_DIR "${CUDA_PATH}/bin") + SET(CUDAToolkit_INCLUDE_DIRS "${CUDA_PATH}/include") + SET(CUDAToolkit_NVCC_EXECUTABLE "${CUDA_PATH}/bin/nvcc") + BREAK() + ENDIF() + ENDFOREACH() + ENDIF() + + # Set environment variables + SET(ENV{CUDA_HOME} "${CUDA_TOOLKIT_ROOT}") + SET(ENV{CUDACXX} "${CUDAToolkit_NVCC_EXECUTABLE}") + + IF(NOT CMAKE_CUDA_COMPILER) + SET(CMAKE_CUDA_COMPILER "${CUDAToolkit_NVCC_EXECUTABLE}" CACHE FILEPATH "CUDA compiler" FORCE) + ENDIF() + + # Enable CUDA language now that we have the correct compiler + INCLUDE(CheckLanguage) + CHECK_LANGUAGE(CUDA) + IF(CMAKE_CUDA_COMPILER) + ENABLE_LANGUAGE(CUDA) + ENDIF() + + IF(NOT DEFINED CMAKE_CUDA_ARCHITECTURES) + SET(CMAKE_CUDA_ARCHITECTURES "70;75;80" CACHE STRING "CUDA architectures" FORCE) + ENDIF() + + SET(Kokkos_CUDA_DIR "${CUDA_TOOLKIT_ROOT}" CACHE PATH "CUDA installation directory" FORCE) + INCLUDE_DIRECTORIES(SYSTEM ${CUDAToolkit_INCLUDE_DIRS}) + ELSE() + # CUDA not explicitly enabled or not found + SET(CUDA_FOUND OFF) ENDIF() ADD_OPTION(ENABLE_SERIAL "Enable Serial backend when building Kokkos submodule" ON) ADD_OPTION(ENABLE_OPENMP "Enable OpenMP when building Kokkos submodule" ${OpenMP_FOUND}) - ADD_OPTION(ENABLE_THREADS "Enable Pthreads when building Kokkos submodule" ${Threads_FOUND}) - ADD_OPTION(ENABLE_CUDA "Enable CUDA when building Kokkos submodule" ${CUDA_FOUND}) + # Only enable Threads if OpenMP is not available + IF(OpenMP_FOUND) + ADD_OPTION(ENABLE_THREADS "Enable Pthreads when building Kokkos submodule" OFF) + ELSE() + ADD_OPTION(ENABLE_THREADS "Enable Pthreads when building Kokkos submodule" ${Threads_FOUND}) + ENDIF() + # CUDA must be explicitly enabled - default to OFF + ADD_OPTION(ENABLE_CUDA "Enable CUDA when building Kokkos submodule" OFF) + + # if OpenMP is enabled, ensure Threads is disabled (Kokkos doesn't allow both) + IF(ENABLE_OPENMP) + SET(ENABLE_THREADS OFF) + SET(Kokkos_ENABLE_THREADS OFF) + ENDIF() - # if OpenMP defaulted to ON but Kokkos_ENABLE_THREADS was explicitly set, - # disable OpenMP defaulting to ON - IF(ENABLE_OPENMP AND Kokkos_ENABLE_THREADS) + # if Threads was explicitly enabled, disable OpenMP + IF(Kokkos_ENABLE_THREADS) SET(ENABLE_OPENMP OFF) SET(Kokkos_ENABLE_OPENMP OFF) ENDIF() From 1dc7d8791a81520eaa736d7c9950781f0176bbc9 Mon Sep 17 00:00:00 2001 From: Ivan Grigorik Date: Thu, 25 Dec 2025 19:40:32 -0600 Subject: [PATCH 02/12] fix openMP and Threads build --- base/cmake/Modules/KokkosPythonKokkos.cmake | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/base/cmake/Modules/KokkosPythonKokkos.cmake b/base/cmake/Modules/KokkosPythonKokkos.cmake index 42177c85..4a23bdb8 100644 --- a/base/cmake/Modules/KokkosPythonKokkos.cmake +++ b/base/cmake/Modules/KokkosPythonKokkos.cmake @@ -84,7 +84,10 @@ IF(_INTERNAL_KOKKOS) SET(Threads_FOUND OFF) SET(CUDA_FOUND OFF) - FIND_PACKAGE(OpenMP QUIET) + # Only search for OpenMP if not explicitly disabled and Threads not explicitly enabled + IF(NOT DEFINED Kokkos_ENABLE_THREADS OR NOT Kokkos_ENABLE_THREADS) + FIND_PACKAGE(OpenMP QUIET) + ENDIF() # Only check for Threads if OpenMP was not found and not explicitly disabled IF(NOT OpenMP_FOUND AND NOT DEFINED Kokkos_ENABLE_OPENMP) @@ -289,6 +292,21 @@ IF(_INTERNAL_KOKKOS) SET(ENABLE_CUDA ${Kokkos_ENABLE_CUDA}) ENDIF() + # Enforce mutual exclusion AFTER syncing from command-line options + # Kokkos doesn't allow both OpenMP and Threads to be enabled + IF(ENABLE_OPENMP AND ENABLE_THREADS) + # Prefer the explicitly set option, or OpenMP if both are explicit + IF(DEFINED Kokkos_ENABLE_THREADS AND NOT DEFINED Kokkos_ENABLE_OPENMP) + SET(ENABLE_OPENMP OFF) + SET(Kokkos_ENABLE_OPENMP OFF CACHE BOOL "Build Kokkos submodule with OpenMP support" FORCE) + MESSAGE(STATUS "Disabling OpenMP because Threads was explicitly enabled") + ELSE() + SET(ENABLE_THREADS OFF) + SET(Kokkos_ENABLE_THREADS OFF CACHE BOOL "Build Kokkos submodule with Pthread support" FORCE) + MESSAGE(STATUS "Disabling Threads because OpenMP is enabled") + ENDIF() + ENDIF() + # define the kokkos option as default and/or get it to display IF(ENABLE_SERIAL) ADD_OPTION(Kokkos_ENABLE_SERIAL "Build Kokkos submodule with serial support" ON) From a3e1b7c97d1679d06ab306d19ec6f807df05f488 Mon Sep 17 00:00:00 2001 From: Ivan Grigorik Date: Thu, 25 Dec 2025 19:41:47 -0600 Subject: [PATCH 03/12] rename `build` from base to `build base` --- .github/workflows/base-linux-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/base-linux-ci.yml b/.github/workflows/base-linux-ci.yml index 3a03c690..f18ae2c8 100644 --- a/.github/workflows/base-linux-ci.yml +++ b/.github/workflows/base-linux-ci.yml @@ -60,7 +60,7 @@ jobs: -DPython3_EXECUTABLE=$(which python) \ ${{github.workspace}}/base - - name: Build + - name: Build base run: cmake --build ${{github.workspace}}/base/build --target all --parallel 2 && sudo cmake --build ${{github.workspace}}/base/build --target install --parallel 2 @@ -118,7 +118,7 @@ jobs: /tmp/kokkos-source && cmake --build /tmp/kokkos-build --target all --parallel 2 && sudo cmake --build /tmp/kokkos-build --target install --parallel 2 - - name: Build + - name: Build base run: | CMAKE_BUILD_PARALLEL_LEVEL=2 PYKOKKOS_BASE_SETUP_ARGS="-DENABLE_WERROR=ON -DENABLE_MEMORY_TRAITS=ON -DENABLE_LAYOUTS=ON -DENABLE_VIEW_RANKS=2 -DCMAKE_CXX_STANDARD=17 -DKokkos_ENABLE_THREADS=ON -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DBUILD_SHARED_LIBS=OFF" \ python -m pip install -v --user --no-deps . @@ -178,7 +178,7 @@ jobs: -DPython3_EXECUTABLE=$(which python) \ ${{github.workspace}} - - name: Build + - name: Build base run: cmake --build ${{github.workspace}}/build --target all --parallel 2 && sudo cmake --build ${{github.workspace}}/build --target install --parallel 2 From b0227eaf0cf7b3c4edd3fde2bacf69921ef71a3f Mon Sep 17 00:00:00 2001 From: Ivan Grigorik Date: Mon, 29 Dec 2025 12:00:54 -0600 Subject: [PATCH 04/12] remove duplicated CMakeLists code --- base/cmake/Modules/KokkosPythonKokkos.cmake | 90 +++------------------ 1 file changed, 10 insertions(+), 80 deletions(-) diff --git a/base/cmake/Modules/KokkosPythonKokkos.cmake b/base/cmake/Modules/KokkosPythonKokkos.cmake index 4a23bdb8..6d2455e5 100644 --- a/base/cmake/Modules/KokkosPythonKokkos.cmake +++ b/base/cmake/Modules/KokkosPythonKokkos.cmake @@ -97,7 +97,6 @@ IF(_INTERNAL_KOKKOS) ENDIF() # Only enable CUDA if EXPLICITLY requested - # Check both ENABLE_CUDA and Kokkos_ENABLE_CUDA SET(_CUDA_EXPLICITLY_ENABLED OFF) # Check if explicitly enabled @@ -106,9 +105,7 @@ IF(_INTERNAL_KOKKOS) ENDIF() # Only search for CUDA if explicitly enabled - IF(_CUDA_EXPLICITLY_ENABLED AND NOT DEFINED Kokkos_ENABLE_CUDA) - # Use modern CUDAToolkit package instead of deprecated CUDA module - # First, search for CUDA in common installation directories + IF(_CUDA_EXPLICITLY_ENABLED) FILE(GLOB CUDA_SEARCH_PATHS "/usr/local/cuda-*" "/usr/local/cuda" @@ -126,7 +123,13 @@ IF(_INTERNAL_KOKKOS) ENDFOREACH() ENDIF() - FIND_PACKAGE(CUDAToolkit QUIET) + # Find CUDA toolkit (REQUIRED if Kokkos_ENABLE_CUDA already defined, otherwise QUIET) + IF(DEFINED Kokkos_ENABLE_CUDA) + FIND_PACKAGE(CUDAToolkit REQUIRED) + ELSE() + FIND_PACKAGE(CUDAToolkit QUIET) + ENDIF() + IF(CUDAToolkit_FOUND) SET(CUDA_FOUND ON) @@ -135,8 +138,6 @@ IF(_INTERNAL_KOKKOS) # Verify we have cuda_runtime.h in the include directory IF(NOT EXISTS "${CUDAToolkit_INCLUDE_DIRS}/cuda_runtime.h") - - # Try to find a complete CUDA installation manually FOREACH(CUDA_PATH ${CUDA_SEARCH_PATHS}) IF(EXISTS "${CUDA_PATH}/include/cuda_runtime.h") SET(CUDA_TOOLKIT_ROOT "${CUDA_PATH}") @@ -148,11 +149,9 @@ IF(_INTERNAL_KOKKOS) ENDFOREACH() ENDIF() - # Set environment variables for the build + # Set up env for this build SET(ENV{CUDA_HOME} "${CUDA_TOOLKIT_ROOT}") SET(ENV{CUDACXX} "${CUDAToolkit_NVCC_EXECUTABLE}") - - # Set CMAKE_CUDA_COMPILER if not already set IF(NOT CMAKE_CUDA_COMPILER) SET(CMAKE_CUDA_COMPILER "${CUDAToolkit_NVCC_EXECUTABLE}" CACHE FILEPATH "CUDA compiler" FORCE) ENDIF() @@ -164,82 +163,13 @@ IF(_INTERNAL_KOKKOS) ENABLE_LANGUAGE(CUDA) ENDIF() - # Set default CUDA architectures if not specified - IF(NOT DEFINED CMAKE_CUDA_ARCHITECTURES) - SET(CMAKE_CUDA_ARCHITECTURES "70;75;80" CACHE STRING "CUDA architectures" FORCE) - ENDIF() - - # Pass CUDA paths to Kokkos SET(Kokkos_CUDA_DIR "${CUDA_TOOLKIT_ROOT}" CACHE PATH "CUDA installation directory" FORCE) - - # Add CUDA include directories to the include path INCLUDE_DIRECTORIES(SYSTEM ${CUDAToolkit_INCLUDE_DIRS}) ELSE() SET(CUDA_FOUND OFF) ENDIF() - ELSEIF(_CUDA_EXPLICITLY_ENABLED) - # CUDA explicitly enabled - ensure we have it - # Search for CUDA in common installation directories - FILE(GLOB CUDA_SEARCH_PATHS - "/usr/local/cuda-*" - "/usr/local/cuda" - "/opt/cuda-*" - "/opt/cuda" - ) - - # Add common CUDA paths to CMAKE_PREFIX_PATH - IF(CUDA_SEARCH_PATHS) - LIST(SORT CUDA_SEARCH_PATHS ORDER DESCENDING) - FOREACH(CUDA_PATH ${CUDA_SEARCH_PATHS}) - IF(EXISTS "${CUDA_PATH}/bin/nvcc" AND EXISTS "${CUDA_PATH}/include/cuda_runtime.h") - LIST(APPEND CMAKE_PREFIX_PATH "${CUDA_PATH}") - ENDIF() - ENDFOREACH() - ENDIF() - - FIND_PACKAGE(CUDAToolkit REQUIRED) - SET(CUDA_FOUND ON) - - # Set CUDA paths for the build system - GET_FILENAME_COMPONENT(CUDA_TOOLKIT_ROOT "${CUDAToolkit_BIN_DIR}" DIRECTORY) - - # Verify we have cuda_runtime.h - IF(NOT EXISTS "${CUDAToolkit_INCLUDE_DIRS}/cuda_runtime.h") - # Try to find complete installation - FOREACH(CUDA_PATH ${CUDA_SEARCH_PATHS}) - IF(EXISTS "${CUDA_PATH}/include/cuda_runtime.h") - SET(CUDA_TOOLKIT_ROOT "${CUDA_PATH}") - SET(CUDAToolkit_BIN_DIR "${CUDA_PATH}/bin") - SET(CUDAToolkit_INCLUDE_DIRS "${CUDA_PATH}/include") - SET(CUDAToolkit_NVCC_EXECUTABLE "${CUDA_PATH}/bin/nvcc") - BREAK() - ENDIF() - ENDFOREACH() - ENDIF() - - # Set environment variables - SET(ENV{CUDA_HOME} "${CUDA_TOOLKIT_ROOT}") - SET(ENV{CUDACXX} "${CUDAToolkit_NVCC_EXECUTABLE}") - - IF(NOT CMAKE_CUDA_COMPILER) - SET(CMAKE_CUDA_COMPILER "${CUDAToolkit_NVCC_EXECUTABLE}" CACHE FILEPATH "CUDA compiler" FORCE) - ENDIF() - - # Enable CUDA language now that we have the correct compiler - INCLUDE(CheckLanguage) - CHECK_LANGUAGE(CUDA) - IF(CMAKE_CUDA_COMPILER) - ENABLE_LANGUAGE(CUDA) - ENDIF() - - IF(NOT DEFINED CMAKE_CUDA_ARCHITECTURES) - SET(CMAKE_CUDA_ARCHITECTURES "70;75;80" CACHE STRING "CUDA architectures" FORCE) - ENDIF() - - SET(Kokkos_CUDA_DIR "${CUDA_TOOLKIT_ROOT}" CACHE PATH "CUDA installation directory" FORCE) - INCLUDE_DIRECTORIES(SYSTEM ${CUDAToolkit_INCLUDE_DIRS}) ELSE() - # CUDA not explicitly enabled or not found + # if CUDA not explicitly enabled SET(CUDA_FOUND OFF) ENDIF() From 8e7e10453b312addc996ec7ddc33d00a03f0c2fc Mon Sep 17 00:00:00 2001 From: Ivan Grigorik Date: Mon, 29 Dec 2025 12:05:36 -0600 Subject: [PATCH 05/12] add appropriate warning if both Threads and OpenMP spaces enabled --- base/cmake/Modules/KokkosPythonKokkos.cmake | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/base/cmake/Modules/KokkosPythonKokkos.cmake b/base/cmake/Modules/KokkosPythonKokkos.cmake index 6d2455e5..7c918aba 100644 --- a/base/cmake/Modules/KokkosPythonKokkos.cmake +++ b/base/cmake/Modules/KokkosPythonKokkos.cmake @@ -222,18 +222,18 @@ IF(_INTERNAL_KOKKOS) SET(ENABLE_CUDA ${Kokkos_ENABLE_CUDA}) ENDIF() - # Enforce mutual exclusion AFTER syncing from command-line options - # Kokkos doesn't allow both OpenMP and Threads to be enabled + # Kokkos doesn't allow both OpenMP and Threads - enforce mutual exclusion IF(ENABLE_OPENMP AND ENABLE_THREADS) - # Prefer the explicitly set option, or OpenMP if both are explicit - IF(DEFINED Kokkos_ENABLE_THREADS AND NOT DEFINED Kokkos_ENABLE_OPENMP) + IF(Kokkos_ENABLE_OPENMP AND Kokkos_ENABLE_THREADS) + MESSAGE(FATAL_ERROR "Cannot enable both OpenMP and Threads execution spaces.") + ELSEIF(Kokkos_ENABLE_THREADS) SET(ENABLE_OPENMP OFF) - SET(Kokkos_ENABLE_OPENMP OFF CACHE BOOL "Build Kokkos submodule with OpenMP support" FORCE) - MESSAGE(STATUS "Disabling OpenMP because Threads was explicitly enabled") + SET(Kokkos_ENABLE_OPENMP OFF CACHE BOOL "" FORCE) + MESSAGE(STATUS "Disabling auto-detected OpenMP (Threads explicitly enabled)") ELSE() SET(ENABLE_THREADS OFF) - SET(Kokkos_ENABLE_THREADS OFF CACHE BOOL "Build Kokkos submodule with Pthread support" FORCE) - MESSAGE(STATUS "Disabling Threads because OpenMP is enabled") + SET(Kokkos_ENABLE_THREADS OFF CACHE BOOL "" FORCE) + MESSAGE(STATUS "Disabling Threads (OpenMP enabled)") ENDIF() ENDIF() From de9effbd2c6dbe0100b1ab77511ca4e1e3d52456 Mon Sep 17 00:00:00 2001 From: Ivan Grigorik Date: Thu, 1 Jan 2026 15:57:27 -0600 Subject: [PATCH 06/12] add OpenMP checking only when requested --- base/cmake/Modules/KokkosPythonKokkos.cmake | 29 +++------------------ 1 file changed, 4 insertions(+), 25 deletions(-) diff --git a/base/cmake/Modules/KokkosPythonKokkos.cmake b/base/cmake/Modules/KokkosPythonKokkos.cmake index 7c918aba..1bf09b2b 100644 --- a/base/cmake/Modules/KokkosPythonKokkos.cmake +++ b/base/cmake/Modules/KokkosPythonKokkos.cmake @@ -84,16 +84,12 @@ IF(_INTERNAL_KOKKOS) SET(Threads_FOUND OFF) SET(CUDA_FOUND OFF) - # Only search for OpenMP if not explicitly disabled and Threads not explicitly enabled - IF(NOT DEFINED Kokkos_ENABLE_THREADS OR NOT Kokkos_ENABLE_THREADS) + # Enable OpenMP if explicitly requested + IF((DEFINED ENABLE_OPENMP AND ENABLE_OPENMP) OR (DEFINED Kokkos_ENABLE_OPENMP AND Kokkos_ENABLE_OPENMP)) FIND_PACKAGE(OpenMP QUIET) - ENDIF() - - # Only check for Threads if OpenMP was not found and not explicitly disabled - IF(NOT OpenMP_FOUND AND NOT DEFINED Kokkos_ENABLE_OPENMP) + # If Threads is explicitly requested, find it + ELSEIF((DEFINED ENABLE_THREADS AND ENABLE_THREADS) OR (DEFINED Kokkos_ENABLE_THREADS AND Kokkos_ENABLE_THREADS)) FIND_PACKAGE(Threads QUIET) - ELSE() - SET(Threads_FOUND OFF) ENDIF() # Only enable CUDA if EXPLICITLY requested @@ -165,12 +161,7 @@ IF(_INTERNAL_KOKKOS) SET(Kokkos_CUDA_DIR "${CUDA_TOOLKIT_ROOT}" CACHE PATH "CUDA installation directory" FORCE) INCLUDE_DIRECTORIES(SYSTEM ${CUDAToolkit_INCLUDE_DIRS}) - ELSE() - SET(CUDA_FOUND OFF) ENDIF() - ELSE() - # if CUDA not explicitly enabled - SET(CUDA_FOUND OFF) ENDIF() ADD_OPTION(ENABLE_SERIAL "Enable Serial backend when building Kokkos submodule" ON) @@ -184,18 +175,6 @@ IF(_INTERNAL_KOKKOS) # CUDA must be explicitly enabled - default to OFF ADD_OPTION(ENABLE_CUDA "Enable CUDA when building Kokkos submodule" OFF) - # if OpenMP is enabled, ensure Threads is disabled (Kokkos doesn't allow both) - IF(ENABLE_OPENMP) - SET(ENABLE_THREADS OFF) - SET(Kokkos_ENABLE_THREADS OFF) - ENDIF() - - # if Threads was explicitly enabled, disable OpenMP - IF(Kokkos_ENABLE_THREADS) - SET(ENABLE_OPENMP OFF) - SET(Kokkos_ENABLE_OPENMP OFF) - ENDIF() - # always disable pthread backend since pthreads are not supported on Windows IF(WIN32) SET(ENABLE_THREADS OFF) From d864e490ca8080c526b7714472915a6be198136b Mon Sep 17 00:00:00 2001 From: Ivan Grigorik Date: Thu, 1 Jan 2026 16:12:07 -0600 Subject: [PATCH 07/12] refresh CUDA detection --- base/cmake/Modules/KokkosPythonKokkos.cmake | 77 ++++----------------- base/environment.yml | 1 - 2 files changed, 15 insertions(+), 63 deletions(-) diff --git a/base/cmake/Modules/KokkosPythonKokkos.cmake b/base/cmake/Modules/KokkosPythonKokkos.cmake index 1bf09b2b..a4f4bfef 100644 --- a/base/cmake/Modules/KokkosPythonKokkos.cmake +++ b/base/cmake/Modules/KokkosPythonKokkos.cmake @@ -92,75 +92,28 @@ IF(_INTERNAL_KOKKOS) FIND_PACKAGE(Threads QUIET) ENDIF() - # Only enable CUDA if EXPLICITLY requested - SET(_CUDA_EXPLICITLY_ENABLED OFF) - - # Check if explicitly enabled - IF((DEFINED ENABLE_CUDA AND ENABLE_CUDA) OR (DEFINED Kokkos_ENABLE_CUDA AND Kokkos_ENABLE_CUDA)) - SET(_CUDA_EXPLICITLY_ENABLED ON) - ENDIF() - # Only search for CUDA if explicitly enabled - IF(_CUDA_EXPLICITLY_ENABLED) - FILE(GLOB CUDA_SEARCH_PATHS - "/usr/local/cuda-*" - "/usr/local/cuda" - "/opt/cuda-*" - "/opt/cuda" - ) - - # Add common CUDA paths to CMAKE_PREFIX_PATH to help find complete installations - IF(CUDA_SEARCH_PATHS) - LIST(SORT CUDA_SEARCH_PATHS ORDER DESCENDING) # Prefer newer versions - FOREACH(CUDA_PATH ${CUDA_SEARCH_PATHS}) - IF(EXISTS "${CUDA_PATH}/bin/nvcc" AND EXISTS "${CUDA_PATH}/include/cuda_runtime.h") - LIST(APPEND CMAKE_PREFIX_PATH "${CUDA_PATH}") - ENDIF() - ENDFOREACH() - ENDIF() - - # Find CUDA toolkit (REQUIRED if Kokkos_ENABLE_CUDA already defined, otherwise QUIET) + IF((DEFINED ENABLE_CUDA AND ENABLE_CUDA) OR (DEFINED Kokkos_ENABLE_CUDA AND Kokkos_ENABLE_CUDA)) + # Find CUDA (REQUIRED if Kokkos_ENABLE_CUDA already defined, otherwise QUIET) IF(DEFINED Kokkos_ENABLE_CUDA) - FIND_PACKAGE(CUDAToolkit REQUIRED) + FIND_PACKAGE(CUDA REQUIRED) ELSE() - FIND_PACKAGE(CUDAToolkit QUIET) + FIND_PACKAGE(CUDA QUIET) ENDIF() - IF(CUDAToolkit_FOUND) - SET(CUDA_FOUND ON) - - # Set CUDA paths for the build system - GET_FILENAME_COMPONENT(CUDA_TOOLKIT_ROOT "${CUDAToolkit_BIN_DIR}" DIRECTORY) - - # Verify we have cuda_runtime.h in the include directory - IF(NOT EXISTS "${CUDAToolkit_INCLUDE_DIRS}/cuda_runtime.h") - FOREACH(CUDA_PATH ${CUDA_SEARCH_PATHS}) - IF(EXISTS "${CUDA_PATH}/include/cuda_runtime.h") - SET(CUDA_TOOLKIT_ROOT "${CUDA_PATH}") - SET(CUDAToolkit_BIN_DIR "${CUDA_PATH}/bin") - SET(CUDAToolkit_INCLUDE_DIRS "${CUDA_PATH}/include") - SET(CUDAToolkit_NVCC_EXECUTABLE "${CUDA_PATH}/bin/nvcc") - BREAK() - ENDIF() - ENDFOREACH() - ENDIF() - - # Set up env for this build - SET(ENV{CUDA_HOME} "${CUDA_TOOLKIT_ROOT}") - SET(ENV{CUDACXX} "${CUDAToolkit_NVCC_EXECUTABLE}") - IF(NOT CMAKE_CUDA_COMPILER) - SET(CMAKE_CUDA_COMPILER "${CUDAToolkit_NVCC_EXECUTABLE}" CACHE FILEPATH "CUDA compiler" FORCE) - ENDIF() + IF(CUDA_FOUND) + # set up CUDA include directories + FOREACH(INCLUDE_DIR ${CUDA_INCLUDE_DIRS}) + LIST(APPEND CMAKE_CXX_FLAGS "-I${INCLUDE_DIR}") + LIST(APPEND CMAKE_CUDA_FLAGS "-I${INCLUDE_DIR}") + ENDFOREACH() + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}" CACHE STRING "Flags used by the C++ compiler" FORCE) + SET(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS}" CACHE STRING "Flags used by the CUDA compiler" FORCE) - # Enable CUDA language now that we have the correct compiler - INCLUDE(CheckLanguage) - CHECK_LANGUAGE(CUDA) - IF(CMAKE_CUDA_COMPILER) - ENABLE_LANGUAGE(CUDA) - ENDIF() + ENABLE_LANGUAGE(CUDA) + INCLUDE_DIRECTORIES(SYSTEM ${CUDA_INCLUDE_DIRS}) - SET(Kokkos_CUDA_DIR "${CUDA_TOOLKIT_ROOT}" CACHE PATH "CUDA installation directory" FORCE) - INCLUDE_DIRECTORIES(SYSTEM ${CUDAToolkit_INCLUDE_DIRS}) + SET(Kokkos_CUDA_DIR "${CUDA_TOOLKIT_ROOT_DIR}" CACHE PATH "CUDA installation directory" FORCE) ENDIF() ENDIF() diff --git a/base/environment.yml b/base/environment.yml index a909ddd2..35950e5e 100644 --- a/base/environment.yml +++ b/base/environment.yml @@ -7,4 +7,3 @@ dependencies: - numpy - gcc=11.4.0 - gxx=11.4.0 - From 2fd562d96e3d806f97b79d441044687cc0781feb Mon Sep 17 00:00:00 2001 From: Ivan Grigorik Date: Thu, 1 Jan 2026 16:20:06 -0600 Subject: [PATCH 08/12] fix cuda_runtime.h warning --- base/cmake/Modules/KokkosPythonKokkos.cmake | 24 +++++++++------------ 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/base/cmake/Modules/KokkosPythonKokkos.cmake b/base/cmake/Modules/KokkosPythonKokkos.cmake index a4f4bfef..43bf7875 100644 --- a/base/cmake/Modules/KokkosPythonKokkos.cmake +++ b/base/cmake/Modules/KokkosPythonKokkos.cmake @@ -94,27 +94,23 @@ IF(_INTERNAL_KOKKOS) # Only search for CUDA if explicitly enabled IF((DEFINED ENABLE_CUDA AND ENABLE_CUDA) OR (DEFINED Kokkos_ENABLE_CUDA AND Kokkos_ENABLE_CUDA)) - # Find CUDA (REQUIRED if Kokkos_ENABLE_CUDA already defined, otherwise QUIET) - IF(DEFINED Kokkos_ENABLE_CUDA) - FIND_PACKAGE(CUDA REQUIRED) - ELSE() - FIND_PACKAGE(CUDA QUIET) - ENDIF() - - IF(CUDA_FOUND) - # set up CUDA include directories - FOREACH(INCLUDE_DIR ${CUDA_INCLUDE_DIRS}) + FIND_PACKAGE(CUDAToolkit QUIET) + IF(CUDAToolkit_FOUND) + FOREACH(INCLUDE_DIR ${CUDAToolkit_INCLUDE_DIRS}) LIST(APPEND CMAKE_CXX_FLAGS "-I${INCLUDE_DIR}") LIST(APPEND CMAKE_CUDA_FLAGS "-I${INCLUDE_DIR}") ENDFOREACH() SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}" CACHE STRING "Flags used by the C++ compiler" FORCE) SET(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS}" CACHE STRING "Flags used by the CUDA compiler" FORCE) + ENDIF() - ENABLE_LANGUAGE(CUDA) - INCLUDE_DIRECTORIES(SYSTEM ${CUDA_INCLUDE_DIRS}) - - SET(Kokkos_CUDA_DIR "${CUDA_TOOLKIT_ROOT_DIR}" CACHE PATH "CUDA installation directory" FORCE) + ENABLE_LANGUAGE(CUDA) + IF(CUDAToolkit_FOUND) + INCLUDE_DIRECTORIES(SYSTEM ${CUDAToolkit_INCLUDE_DIRS}) + GET_FILENAME_COMPONENT(CUDA_TOOLKIT_ROOT "${CUDAToolkit_BIN_DIR}" DIRECTORY) + SET(Kokkos_CUDA_DIR "${CUDA_TOOLKIT_ROOT}" CACHE PATH "CUDA installation directory" FORCE) ENDIF() + SET(CUDA_FOUND ON) ENDIF() ADD_OPTION(ENABLE_SERIAL "Enable Serial backend when building Kokkos submodule" ON) From 471d7b1a77d6ebcc33a5f57d4c4cd3a809ad3148 Mon Sep 17 00:00:00 2001 From: Ivan Grigorik Date: Thu, 1 Jan 2026 16:22:44 -0600 Subject: [PATCH 09/12] add indirect OpenMP/Therads enabling --- base/cmake/Modules/KokkosPythonKokkos.cmake | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/base/cmake/Modules/KokkosPythonKokkos.cmake b/base/cmake/Modules/KokkosPythonKokkos.cmake index 43bf7875..b54ccb78 100644 --- a/base/cmake/Modules/KokkosPythonKokkos.cmake +++ b/base/cmake/Modules/KokkosPythonKokkos.cmake @@ -90,6 +90,12 @@ IF(_INTERNAL_KOKKOS) # If Threads is explicitly requested, find it ELSEIF((DEFINED ENABLE_THREADS AND ENABLE_THREADS) OR (DEFINED Kokkos_ENABLE_THREADS AND Kokkos_ENABLE_THREADS)) FIND_PACKAGE(Threads QUIET) + ELSE() + # if none requested - check what's available and prefer OpenMP + FIND_PACKAGE(OpenMP QUIET) + IF(NOT OpenMP_FOUND) + FIND_PACKAGE(Threads QUIET) + ENDIF() ENDIF() # Only search for CUDA if explicitly enabled @@ -114,7 +120,12 @@ IF(_INTERNAL_KOKKOS) ENDIF() ADD_OPTION(ENABLE_SERIAL "Enable Serial backend when building Kokkos submodule" ON) - ADD_OPTION(ENABLE_OPENMP "Enable OpenMP when building Kokkos submodule" ${OpenMP_FOUND}) + # If OpenMP was explicitly requested, use that; otherwise use auto-detection result + IF((DEFINED ENABLE_OPENMP AND ENABLE_OPENMP) OR (DEFINED Kokkos_ENABLE_OPENMP AND Kokkos_ENABLE_OPENMP)) + ADD_OPTION(ENABLE_OPENMP "Enable OpenMP when building Kokkos submodule" ON) + ELSE() + ADD_OPTION(ENABLE_OPENMP "Enable OpenMP when building Kokkos submodule" ${OpenMP_FOUND}) + ENDIF() # Only enable Threads if OpenMP is not available IF(OpenMP_FOUND) ADD_OPTION(ENABLE_THREADS "Enable Pthreads when building Kokkos submodule" OFF) From e3424e37b15fe2bf956ccb91378c7725a50f3394 Mon Sep 17 00:00:00 2001 From: Ivan Grigorik Date: Thu, 1 Jan 2026 16:24:14 -0600 Subject: [PATCH 10/12] remove repetetive logic --- base/cmake/Modules/KokkosPythonKokkos.cmake | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/base/cmake/Modules/KokkosPythonKokkos.cmake b/base/cmake/Modules/KokkosPythonKokkos.cmake index b54ccb78..d72621c0 100644 --- a/base/cmake/Modules/KokkosPythonKokkos.cmake +++ b/base/cmake/Modules/KokkosPythonKokkos.cmake @@ -161,21 +161,6 @@ IF(_INTERNAL_KOKKOS) SET(ENABLE_CUDA ${Kokkos_ENABLE_CUDA}) ENDIF() - # Kokkos doesn't allow both OpenMP and Threads - enforce mutual exclusion - IF(ENABLE_OPENMP AND ENABLE_THREADS) - IF(Kokkos_ENABLE_OPENMP AND Kokkos_ENABLE_THREADS) - MESSAGE(FATAL_ERROR "Cannot enable both OpenMP and Threads execution spaces.") - ELSEIF(Kokkos_ENABLE_THREADS) - SET(ENABLE_OPENMP OFF) - SET(Kokkos_ENABLE_OPENMP OFF CACHE BOOL "" FORCE) - MESSAGE(STATUS "Disabling auto-detected OpenMP (Threads explicitly enabled)") - ELSE() - SET(ENABLE_THREADS OFF) - SET(Kokkos_ENABLE_THREADS OFF CACHE BOOL "" FORCE) - MESSAGE(STATUS "Disabling Threads (OpenMP enabled)") - ENDIF() - ENDIF() - # define the kokkos option as default and/or get it to display IF(ENABLE_SERIAL) ADD_OPTION(Kokkos_ENABLE_SERIAL "Build Kokkos submodule with serial support" ON) From 58bd9c9e22184c18960f74d11af2a751c8fae941 Mon Sep 17 00:00:00 2001 From: Ivan Grigorik Date: Thu, 1 Jan 2026 16:51:11 -0600 Subject: [PATCH 11/12] change comma-separated list to space-separated --- base/cmake/Modules/KokkosPythonKokkos.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/base/cmake/Modules/KokkosPythonKokkos.cmake b/base/cmake/Modules/KokkosPythonKokkos.cmake index d72621c0..5fab690f 100644 --- a/base/cmake/Modules/KokkosPythonKokkos.cmake +++ b/base/cmake/Modules/KokkosPythonKokkos.cmake @@ -103,8 +103,8 @@ IF(_INTERNAL_KOKKOS) FIND_PACKAGE(CUDAToolkit QUIET) IF(CUDAToolkit_FOUND) FOREACH(INCLUDE_DIR ${CUDAToolkit_INCLUDE_DIRS}) - LIST(APPEND CMAKE_CXX_FLAGS "-I${INCLUDE_DIR}") - LIST(APPEND CMAKE_CUDA_FLAGS "-I${INCLUDE_DIR}") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -I${INCLUDE_DIR}") + SET(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -I${INCLUDE_DIR}") ENDFOREACH() SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}" CACHE STRING "Flags used by the C++ compiler" FORCE) SET(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS}" CACHE STRING "Flags used by the CUDA compiler" FORCE) From 0c2dbc20a6095023b3195d99e90ec726f1771135 Mon Sep 17 00:00:00 2001 From: Ivan Grigorik Date: Thu, 1 Jan 2026 16:54:02 -0600 Subject: [PATCH 12/12] synch up with HIP --- base/cmake/Modules/KokkosPythonKokkos.cmake | 26 ++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/base/cmake/Modules/KokkosPythonKokkos.cmake b/base/cmake/Modules/KokkosPythonKokkos.cmake index 5fab690f..5c616d42 100644 --- a/base/cmake/Modules/KokkosPythonKokkos.cmake +++ b/base/cmake/Modules/KokkosPythonKokkos.cmake @@ -83,6 +83,7 @@ IF(_INTERNAL_KOKKOS) SET(OpenMP_FOUND OFF) SET(Threads_FOUND OFF) SET(CUDA_FOUND OFF) + SET(HIP_FOUND OFF) # Enable OpenMP if explicitly requested IF((DEFINED ENABLE_OPENMP AND ENABLE_OPENMP) OR (DEFINED Kokkos_ENABLE_OPENMP AND Kokkos_ENABLE_OPENMP)) @@ -119,6 +120,17 @@ IF(_INTERNAL_KOKKOS) SET(CUDA_FOUND ON) ENDIF() + # search for HIP if explicitly enabled + IF((DEFINED ENABLE_HIP AND ENABLE_HIP) OR (DEFINED Kokkos_ENABLE_HIP AND Kokkos_ENABLE_HIP)) + INCLUDE(CheckLanguage) + CHECK_LANGUAGE(HIP) + + IF(CMAKE_HIP_COMPILER) + ENABLE_LANGUAGE(HIP) + SET(HIP_FOUND ON) + ENDIF() + ENDIF() + ADD_OPTION(ENABLE_SERIAL "Enable Serial backend when building Kokkos submodule" ON) # If OpenMP was explicitly requested, use that; otherwise use auto-detection result IF((DEFINED ENABLE_OPENMP AND ENABLE_OPENMP) OR (DEFINED Kokkos_ENABLE_OPENMP AND Kokkos_ENABLE_OPENMP)) @@ -134,6 +146,8 @@ IF(_INTERNAL_KOKKOS) ENDIF() # CUDA must be explicitly enabled - default to OFF ADD_OPTION(ENABLE_CUDA "Enable CUDA when building Kokkos submodule" OFF) + # HIP must be explicitly enabled - default to OFF + ADD_OPTION(ENABLE_HIP "Enable HIP when building Kokkos submodule" OFF) # always disable pthread backend since pthreads are not supported on Windows IF(WIN32) @@ -161,28 +175,34 @@ IF(_INTERNAL_KOKKOS) SET(ENABLE_CUDA ${Kokkos_ENABLE_CUDA}) ENDIF() + # make sure this pykokkos-base option is synced to Kokkos option + IF(DEFINED Kokkos_ENABLE_HIP) + SET(ENABLE_HIP ${Kokkos_ENABLE_HIP}) + ENDIF() + # define the kokkos option as default and/or get it to display IF(ENABLE_SERIAL) ADD_OPTION(Kokkos_ENABLE_SERIAL "Build Kokkos submodule with serial support" ON) ENDIF() - # define the kokkos option as default and/or get it to display IF(ENABLE_OPENMP) ADD_OPTION(Kokkos_ENABLE_OPENMP "Build Kokkos submodule with OpenMP support" ON) ENDIF() - # define the kokkos option as default and/or get it to display IF(ENABLE_THREADS) ADD_OPTION(Kokkos_ENABLE_THREADS "Build Kokkos submodule with Pthread support" ON) ENDIF() - # define the kokkos option as default and/or get it to display IF(ENABLE_CUDA) ADD_OPTION(Kokkos_ENABLE_CUDA "Build Kokkos submodule with CUDA support" ON) ADD_OPTION(Kokkos_ENABLE_CUDA_UVM "Build Kokkos submodule with CUDA UVM support" ON) ADD_OPTION(Kokkos_ENABLE_CUDA_LAMBDA "Build Kokkos submodule with CUDA lambda support" ON) ENDIF() + IF(ENABLE_HIP) + ADD_OPTION(Kokkos_ENABLE_HIP "Build Kokkos submodule with HIP support" ON) + ENDIF() + # Check if we should use submodule or FetchContent IF(EXISTS ${PROJECT_SOURCE_DIR}/external/kokkos/CMakeLists.txt) # Use git submodule