diff --git a/.github/workflows/base-cancelling.yml b/.github/workflows/base-cancelling.yml new file mode 100644 index 00000000..ab3c3b47 --- /dev/null +++ b/.github/workflows/base-cancelling.yml @@ -0,0 +1,24 @@ +name: "[Base] Cancel Builds on Update" +on: + workflow_run: + workflows: ['[Base] Linux CI', '[Base] Python Package'] + types: ['requested'] + +jobs: + cancel-duplicate-workflow-runs: + name: "Cancel duplicate workflow runs" + strategy: + matrix: + os: [ubuntu-latest, ubuntu-22.04] + + runs-on: ${{matrix.os}} + steps: + - uses: potiuk/cancel-workflow-runs@master + name: "Cancel duplicate workflow runs" + with: + cancelMode: duplicates + cancelFutureDuplicates: true + token: ${{ secrets.GITHUB_TOKEN }} + sourceRunId: ${{ github.event.workflow_run.id }} + notifyPRCancel: true + skipEventTypes: '["schedule"]' diff --git a/.github/workflows/base-linux-ci.yml b/.github/workflows/base-linux-ci.yml new file mode 100644 index 00000000..74ca3646 --- /dev/null +++ b/.github/workflows/base-linux-ci.yml @@ -0,0 +1,201 @@ +name: "[Base] Linux CI" + +on: + push: + branches: [ main, develop ] + pull_request: + branches: [ main, develop ] + paths: + - 'base/**' + - '.github/workflows/base-linux-ci.yml' + +env: + BUILD_TYPE: Release + +jobs: + cmake-build: + strategy: + matrix: + python-version: [3.11, 3.12, 3.13] + kokkos-branch: ['release-candidate-4.7.01'] + os: [ubuntu-latest, ubuntu-22.04] + + runs-on: ${{matrix.os}} + steps: + - uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Update container + run: + sudo apt-get update && + sudo apt-get install -y build-essential gcc g++ ninja-build && + python -m pip install --upgrade pip && + python -m pip install 'cmake==3.22.0' + + - name: Install Kokkos + run: + git clone -b ${{ matrix.kokkos-branch }} https://github.com/kokkos/kokkos.git /tmp/kokkos-source && + cmake -B /tmp/kokkos-build + -DKokkos_ENABLE_SERIAL=ON + -DKokkos_ENABLE_OPENMP=OFF + -DBUILD_SHARED_LIBS=ON + /tmp/kokkos-source && + cmake --build /tmp/kokkos-build --target all --parallel 2 && + sudo cmake --build /tmp/kokkos-build --target install --parallel 2 + + - name: Configure CMake + run: + python -m pip install -r base/requirements.txt && + python -m pip install pytest && + cmake -B ${{github.workspace}}/base/build + -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} + -DENABLE_LAYOUTS=OFF + -DENABLE_MEMORY_TRAITS=OFF + -DENABLE_EXAMPLES=ON + -DENABLE_WERROR=ON + -DENABLE_PRECOMPILED_HEADERS=ON + -DPython3_EXECUTABLE=$(which python) + ${{github.workspace}}/base + + - name: Build + run: + cmake --build ${{github.workspace}}/base/build --target all --parallel 2 && + sudo cmake --build ${{github.workspace}}/base/build --target install --parallel 2 + + - name: Import Test + working-directory: ${{github.workspace}}/base/build + run: + mkdir -p scratch && + cd scratch && + export PYTHONPATH=/usr/local/lib/python${{ matrix.python-version }}/site-packages:${PYTHONPATH} && + DEBUG_OUTPUT=ON python -c "import kokkos; print(kokkos.__file__)" + + - name: Test + working-directory: ${{github.workspace}}/base/build + run: + export PYTHONPATH=${PWD}:/usr/local/lib/python${{ matrix.python-version }}/site-packages:${PYTHONPATH} && + ls && + pytest . && + python ./ex-numpy.py + + pip-build: + strategy: + matrix: + python-version: [3.11, 3.12, 3.13] + kokkos-branch: ['release-candidate-4.7.01'] + os: [ubuntu-latest, ubuntu-22.04] + + runs-on: ${{matrix.os}} + defaults: + run: + working-directory: base + steps: + - uses: actions/checkout@v2 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Update container + run: + sudo apt-get update && + sudo apt-get install -y build-essential gcc g++ ninja-build && + python -m pip install --upgrade pip && + python -m pip install 'cmake==3.22.0' + + - name: Install Kokkos + run: + git clone -b ${{ matrix.kokkos-branch }} https://github.com/kokkos/kokkos.git /tmp/kokkos-source && + cmake -B /tmp/kokkos-build + -DKokkos_ENABLE_SERIAL=ON + -DKokkos_ENABLE_THREADS=ON + -DCMAKE_CXX_STANDARD=20 + -DCMAKE_POSITION_INDEPENDENT_CODE=ON + /tmp/kokkos-source && + cmake --build /tmp/kokkos-build --target all --parallel 2 && + sudo cmake --build /tmp/kokkos-build --target install --parallel 2 + + - name: Build + run: + python -m pip install -r requirements.txt && + python -m pip install pytest && + CMAKE_BUILD_PARALLEL_LEVEL=2 PYKOKKOS_BASE_SETUP_ARGS="-DENABLE_WERROR=ON -DENABLE_MEMORY_TRAITS=ON -DENABLE_LAYOUTS=ON -DENABLE_VIEW_RANKS=2 -DCMAKE_CXX_STANDARD=17 -DKokkos_ENABLE_THREADS=ON -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DBUILD_SHARED_LIBS=OFF" + python -m pip install -v --user --no-deps . + + - name: Import Test + run: + mkdir -p scratch && + cd scratch && + DEBUG_OUTPUT=ON python -c "import kokkos; print(kokkos.__file__)" + + - name: Test + working-directory: base/scratch + run: + python -m kokkos.test + + subproject: + strategy: + matrix: + python-version: [3.11, 3.12, 3.13] + os: [ubuntu-latest, ubuntu-22.04] + + runs-on: ${{matrix.os}} + steps: + - uses: actions/checkout@v3 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Update container + run: + sudo apt-get update && + sudo apt-get install -y build-essential gcc g++ ninja-build && + python -m pip install --upgrade pip && + python -m pip install 'cmake==3.22.0' + + - name: Generate CMakeLists.txt + run: | + echo -e "cmake_minimum_required(VERSION 3.22.0 FATAL_ERROR)\nproject(pykokkos-base-as-subproject LANGUAGES CXX)\nadd_subdirectory(base)" > ${{github.workspace}}/CMakeLists.txt + cat ${{github.workspace}}/CMakeLists.txt + + - name: Configure CMake + run: + python -m pip install -r base/requirements.txt && + python -m pip install pytest && + cmake -B ${{github.workspace}}/build + -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} + -DENABLE_LAYOUTS=OFF + -DENABLE_MEMORY_TRAITS=OFF + -DENABLE_VIEW_RANKS=1 + -DENABLE_EXAMPLES=ON + -DENABLE_WERROR=ON + -DENABLE_SERIAL=ON + -DENABLE_OPENMP=ON + -DBUILD_SHARED_LIBS=ON + -DPYTHON_EXECUTABLE=$(which python) + -DPython3_EXECUTABLE=$(which python) + ${{github.workspace}} + + - name: Build + run: + cmake --build ${{github.workspace}}/build --target all --parallel 2 && + sudo cmake --build ${{github.workspace}}/build --target install --parallel 2 + + - name: Setup PYTHONPATH + run: + echo "PYTHONPATH=/usr/local/lib/python${{ matrix.python-version }}/site-packages:${PYTHONPATH}" >> $GITHUB_ENV + + - name: Import Test + run: + DEBUG_OUTPUT=ON python -c "import kokkos; print(kokkos.__file__)" + + - name: Test + run: + python -m kokkos.test diff --git a/.github/workflows/base-python-package.yml b/.github/workflows/base-python-package.yml new file mode 100644 index 00000000..4a8ced88 --- /dev/null +++ b/.github/workflows/base-python-package.yml @@ -0,0 +1,89 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: "[Base] Python Package" + +on: + push: + branches: [ main, develop ] + pull_request: + branches: [ main, develop ] + paths: + - 'base/**' + - '.github/workflows/base-python-package.yml' + +jobs: + linting: + strategy: + matrix: + python-version: [3.11, 3.12, 3.13] + os: [ubuntu-latest, ubuntu-22.04] + + runs-on: ${{matrix.os}} + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install flake8 + if [ -f base/requirements.txt ]; then pip install -r base/requirements.txt; fi + - name: Lint with flake8 + working-directory: base + run: | + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # flake8 options are defined in setup.cfg + flake8 . --count --statistics + + formatting: + strategy: + matrix: + python-version: [3.13] + os: [ubuntu-latest, ubuntu-22.04] + + runs-on: ${{matrix.os}} + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install black + if [ -f base/requirements.txt ]; then pip install -r base/requirements.txt; fi + # Add LLVM key and install clang-16 + sudo apt-get update + sudo apt-get install -y software-properties-common wget gnupg + wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc + DISTRIB_CODENAME=$(lsb_release -sc) + echo "deb http://apt.llvm.org/${DISTRIB_CODENAME}/ llvm-toolchain-${DISTRIB_CODENAME}-16 main" | sudo tee /etc/apt/sources.list.d/llvm-16.list + if sudo apt-get update 2>&1 | grep -q "llvm-toolchain-${DISTRIB_CODENAME}-16"; then + echo "LLVM repository added successfully" + fi + if sudo apt-get install -y clang-format-16 2>/dev/null; then + echo "Installed clang-format-16" + echo "CLANG_FORMAT=clang-format-16" >> $GITHUB_ENV + fi + - name: black format + working-directory: base + run: | + black --diff --check . + - name: clang-format + working-directory: base + run: | + set +e + FILES=$(find include src examples -type f | egrep '\.hpp$|\.cpp$|\.cpp\.in$') + FORMAT_OUT=$($CLANG_FORMAT -output-replacements-xml ${FILES}) + RET=$(echo ${FORMAT_OUT} | grep -c ' /etc/ld.so.conf.d/llvm.conf && ldconfig' + sh 'ccache --zero-stats' + sh 'rm -rf build /tmp/kokkos-build /tmp/kokkos-source' + sh 'git status && git submodule update --init external' + sh 'git clone -b develop https://github.com/kokkos/kokkos.git /tmp/kokkos-source' + sh '''cmake -B /tmp/kokkos-build \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_CXX_COMPILER=hipcc \ + -DCMAKE_CXX_STANDARD=14 \ + -DKokkos_ENABLE_SERIAL=ON \ + -DKokkos_ENABLE_HIP=ON \ + -DBUILD_SHARED_LIBS=ON \ + /tmp/kokkos-source''' + sh 'cmake --build /tmp/kokkos-build --target all --parallel ${BUILD_JOBS} -- VERBOSE=1' + sh 'cmake --build /tmp/kokkos-build --target install --parallel ${BUILD_JOBS}' + sh 'python3 -m pip install -r requirements.txt' + sh 'python3 -m pip install pytest' + sh '''cmake -B build \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_UNITY_BUILD=OFF \ + -DENABLE_LAYOUTS=ON \ + -DENABLE_MEMORY_TRAITS=OFF \ + -DENABLE_EXAMPLES=ON \ + -DENABLE_WERROR=ON \ + -DENABLE_CTP=ON \ + -DPython3_EXECUTABLE=$(which python3)''' + sh 'cmake --build build --target all --parallel ${BUILD_JOBS} -- VERBOSE=1' + sh 'cmake --build build --target analyze-pykokkos-base-compile-time | tee build/${STAGE_NAME}-pykokkos-base-compile-time.txt' + sh 'cmake --build build --target install --parallel ${BUILD_JOBS}' + sh 'cd build && PYTHONPATH=${PWD} python3 $(which pytest) -s .' + sh 'cd build && PYTHONPATH=${PWD} python3 ./ex-numpy.py' + sh 'cd /tmp && DEBUG_OUTPUT=ON python3 -c "import kokkos; print(kokkos.__file__)"' + } + post { + always { + sh 'ls build/*-compile-time.*' + archiveArtifacts 'build/*-compile-time.*' + } + } + } + stage('CUDA') { + agent { + dockerfile { + filename 'Dockerfile.nvcc' + dir 'scripts/docker' + additionalBuildArgs '' + label 'nvidia-docker && volta' + args '-v /tmp/ccache.kokkos:/tmp/ccache --env NVIDIA_VISIBLE_DEVICES=$NVIDIA_VISIBLE_DEVICES' + } + } + environment { + OMP_NUM_THREADS = 8 + OMP_PLACES = 'threads' + OMP_PROC_BIND = 'spread' + PYTHONPATH = '/usr/local/lib/python3.10/site-packages' + } + steps { + sh 'ccache --zero-stats' + sh 'rm -rf build /tmp/kokkos-build /tmp/kokkos-source' + sh 'git status && git submodule update --init external' + sh 'git clone -b develop https://github.com/kokkos/kokkos.git /tmp/kokkos-source' + sh '''cmake -B /tmp/kokkos-build \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_CXX_COMPILER=g++ \ + -DCMAKE_CXX_STANDARD=14 \ + -DKokkos_ENABLE_SERIAL=ON \ + -DKokkos_ENABLE_CUDA=ON \ + -DKokkos_ENABLE_CUDA_UVM=OFF \ + -DKokkos_ENABLE_CUDA_LAMBDA=ON \ + -DBUILD_SHARED_LIBS=ON \ + /tmp/kokkos-source''' + sh 'cmake --build /tmp/kokkos-build --target all --parallel ${BUILD_JOBS}' + sh 'cmake --build /tmp/kokkos-build --target install --parallel ${BUILD_JOBS}' + sh 'python3 -m pip install -r requirements.txt' + sh 'python3 -m pip install pytest' + sh '''cmake -B build \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_UNITY_BUILD=OFF \ + -DENABLE_LAYOUTS=ON \ + -DENABLE_MEMORY_TRAITS=OFF \ + -DENABLE_VIEW_RANKS=2 \ + -DENABLE_EXAMPLES=ON \ + -DENABLE_WERROR=OFF \ + -DENABLE_CTP=ON \ + -DENABLE_TIMING=ON \ + -DPython3_EXECUTABLE=$(which python3)''' + sh 'cmake --build build --target all --parallel $((${BUILD_JOBS}/2)) -- VERBOSE=1' + sh 'cmake --build build --target analyze-pykokkos-base-compile-time | tee build/${STAGE_NAME}-pykokkos-base-compile-time.txt' + sh 'cmake --build build --target install --parallel ${BUILD_JOBS}' + sh 'cd build && mv nvcc-compile-time.csv ${STAGE_NAME}-nvcc-compile-time.csv && cat ${STAGE_NAME}-nvcc-compile-time.csv' + sh 'cd build && PYTHONPATH=${PWD} python3 $(which pytest) -s .' + sh 'cd build && PYTHONPATH=${PWD} python3 ./ex-numpy.py' + sh 'cd /tmp && DEBUG_OUTPUT=ON python3 -c "import kokkos; print(kokkos.__file__)"' + } + post { + always { + sh 'ls build/*-compile-time.*' + archiveArtifacts 'build/*-compile-time.*' + } + } + } + stage('CUDA-UVM') { + agent { + dockerfile { + filename 'Dockerfile.nvcc' + dir 'scripts/docker' + additionalBuildArgs '' + label 'nvidia-docker && volta' + args '-v /tmp/ccache.kokkos:/tmp/ccache --env NVIDIA_VISIBLE_DEVICES=$NVIDIA_VISIBLE_DEVICES' + } + } + environment { + OMP_NUM_THREADS = 8 + OMP_PLACES = 'threads' + OMP_PROC_BIND = 'spread' + PYTHONPATH = '/usr/local/lib/python3.10/site-packages' + } + steps { + sh 'ccache --zero-stats' + sh 'rm -rf build /tmp/kokkos-build /tmp/kokkos-source' + sh 'git status && git submodule update --init external' + sh 'git clone -b develop https://github.com/kokkos/kokkos.git /tmp/kokkos-source' + sh '''cmake -B /tmp/kokkos-build \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_CXX_COMPILER=g++ \ + -DCMAKE_CXX_STANDARD=14 \ + -DKokkos_ENABLE_SERIAL=ON \ + -DKokkos_ENABLE_CUDA=ON \ + -DKokkos_ENABLE_CUDA_UVM=ON \ + -DKokkos_ENABLE_CUDA_LAMBDA=ON \ + -DBUILD_SHARED_LIBS=ON \ + /tmp/kokkos-source''' + sh 'cmake --build /tmp/kokkos-build --target all --parallel ${BUILD_JOBS}' + sh 'cmake --build /tmp/kokkos-build --target install --parallel ${BUILD_JOBS}' + sh 'python3 -m pip install -r requirements.txt' + sh 'python3 -m pip install pytest' + sh '''cmake -B build \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_UNITY_BUILD=OFF \ + -DENABLE_VIEW_RANKS=2 \ + -DENABLE_LAYOUTS=ON \ + -DENABLE_MEMORY_TRAITS=OFF \ + -DENABLE_EXAMPLES=ON \ + -DENABLE_TIMING=ON \ + -DENABLE_WERROR=OFF \ + -DENABLE_CTP=ON \ + -DPython3_EXECUTABLE=$(which python3)''' + sh 'cmake --build build --target all --parallel $((${BUILD_JOBS}/2)) -- VERBOSE=1' + sh 'cmake --build build --target analyze-pykokkos-base-compile-time | tee build/${STAGE_NAME}-pykokkos-base-compile-time.txt' + sh 'cmake --build build --target install --parallel ${BUILD_JOBS}' + sh 'cd build && mv nvcc-compile-time.csv ${STAGE_NAME}-nvcc-compile-time.csv && cat ${STAGE_NAME}-nvcc-compile-time.csv' + sh 'cd build && PYTHONPATH=${PWD} python3 $(which pytest) -s .' + sh 'cd build && PYTHONPATH=${PWD} python3 ./ex-numpy.py' + sh 'cd /tmp && DEBUG_OUTPUT=ON python3 -c "import kokkos; print(kokkos.__file__)"' + } + post { + always { + sh 'ls build/*-compile-time.*' + archiveArtifacts 'build/*-compile-time.*' + } + } + } + stage('OpenMPTarget') { + agent { + dockerfile { + filename 'Dockerfile.hipcc' + dir 'scripts/docker' + label 'rocm-docker && vega && AMD_Radeon_Instinct_MI60' + args '-v /tmp/ccache.kokkos:/tmp/ccache --device=/dev/kfd --device=/dev/dri --security-opt seccomp=unconfined --group-add video --env HIP_VISIBLE_DEVICES=$HIP_VISIBLE_DEVICES' + } + } + environment { + OMP_NUM_THREADS = 8 + OMP_PLACES = 'threads' + OMP_PROC_BIND = 'spread' + PYTHONPATH = '/usr/local/lib/python3.10/site-packages' + } + steps { + sh 'echo "/opt/rocm/llvm/lib" > /etc/ld.so.conf.d/llvm.conf && ldconfig' + sh 'ccache --zero-stats' + sh 'rm -rf build /tmp/kokkos-build /tmp/kokkos-source' + sh 'git clone -b develop https://github.com/kokkos/kokkos.git /tmp/kokkos-source' + sh '''cmake -B /tmp/kokkos-build \ + -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_CXX_COMPILER=clang++ \ + -DCMAKE_CXX_STANDARD=17 \ + -DKokkos_ENABLE_SERIAL=ON \ + -DKokkos_ENABLE_OPENMPTARGET=ON \ + -DKokkos_ARCH_VEGA906=ON \ + -DBUILD_SHARED_LIBS=ON \ + /tmp/kokkos-source''' + sh 'cmake --build /tmp/kokkos-build --target all --parallel ${BUILD_JOBS}' + sh 'cmake --build /tmp/kokkos-build --target install --parallel ${BUILD_JOBS}' + sh 'python3 -m pip install -r requirements.txt' + sh 'python3 -m pip install pytest' + sh '''cmake -B build \ + -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_UNITY_BUILD=OFF \ + -DENABLE_LAYOUTS=ON \ + -DENABLE_MEMORY_TRAITS=OFF \ + -DENABLE_EXAMPLES=ON \ + -DENABLE_WERROR=OFF \ + -DENABLE_CTP=ON \ + -DPython3_EXECUTABLE=$(which python3)''' + sh 'cmake --build build --target all --parallel ${BUILD_JOBS} -- VERBOSE=1' + sh 'cmake --build build --target analyze-pykokkos-base-compile-time | tee build/${STAGE_NAME}-pykokkos-base-compile-time.txt' + sh 'cmake --build build --target install --parallel ${BUILD_JOBS}' + sh 'cd build && PYTHONPATH=${PWD} python3 $(which pytest) -s .' + sh 'cd build && PYTHONPATH=${PWD} python3 ./ex-numpy.py' + sh 'cd /tmp && DEBUG_OUTPUT=ON python3 -c "import kokkos; print(kokkos.__file__)"' + } + post { + always { + sh 'ls build/*-compile-time.*' + archiveArtifacts 'build/*-compile-time.*' + } + } + } + } + } + } +} diff --git a/base/CMakeLists.txt b/base/CMakeLists.txt new file mode 100644 index 00000000..5c93c309 --- /dev/null +++ b/base/CMakeLists.txt @@ -0,0 +1,218 @@ +CMAKE_MINIMUM_REQUIRED(VERSION 3.16 FATAL_ERROR) + +SET(CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/cmake/Modules ${CMAKE_MODULE_PATH}) + + +INCLUDE(KokkosPythonSetup) +INCLUDE(FetchContent) + +PROJECT( + pykokkos-base + LANGUAGES C CXX + VERSION ${pykokkos-base_VERSION}) + +IF("${CMAKE_SOURCE_DIR}" STREQUAL "${PROJECT_SOURCE_DIR}") + SET(PYKOKKOS_BASE_MAIN_PROJECT ON) +ELSE() + SET(PYKOKKOS_BASE_MAIN_PROJECT OFF) +ENDIF() + +INCLUDE(KokkosPythonUtilities) # miscellaneous macros and functions + +# ensure always PIC +SET(CMAKE_POSITION_INDEPENDENT_CODE ON) + +ADD_OPTION(BUILD_SHARED_LIBS "Build shared libraries" ON) +# force to release if not specified +IF("${CMAKE_BUILD_TYPE}" STREQUAL "") + SET(CMAKE_BUILD_TYPE Release CACHE STRING "Build type" FORCE) +ENDIF() + +# Check if pybind11 target already exists (e.g., when used as a subproject) +if(NOT TARGET pybind11::module) + FIND_PACKAGE(pybind11 CONFIG) + if (pybind11_FOUND) + MESSAGE(STATUS "Found pybind11: ${pybind11_DIR} (version \"${pybind11_VERSION}\")") + else() + MESSAGE(STATUS "Fetching pybind11") + FETCHCONTENT_DECLARE( + pybind11 + URL https://github.com/pybind/pybind11/archive/refs/heads/v3.0.zip + URL_HASH SHA256=1816ea10bf7de362fb7864b131cc90bed6fc393feaf7e27b0ed5a03987bedd10 + ) + FETCHCONTENT_MAKEAVAILABLE(pybind11) + endif() +else() + MESSAGE(STATUS "Using existing pybind11 target") +endif() + +INCLUDE(KokkosPythonKokkos) # find external Kokkos or add submodule +INCLUDE(KokkosPythonFormat) # format target +INCLUDE(KokkosPythonCompilers) # compiler identification +INCLUDE(KokkosPythonOptions) # cache options and various variable settings +INCLUDE(KokkosPythonPackages) # Python Interp +INCLUDE(KokkosPythonBuildOptions) # build-options interface library + +IF(ENABLE_THIN_LTO) + SET(CMAKE_INTERPROCEDURAL_OPTIMIZATION ON) +ENDIF() + +SET(libpykokkos_SOURCES + ${CMAKE_CURRENT_LIST_DIR}/src/libpykokkos.cpp + ${CMAKE_CURRENT_LIST_DIR}/src/backend_version.cpp + ${CMAKE_CURRENT_LIST_DIR}/src/enumeration.cpp + ${CMAKE_CURRENT_LIST_DIR}/src/available.cpp + ${CMAKE_CURRENT_LIST_DIR}/src/common.cpp + ${CMAKE_CURRENT_LIST_DIR}/src/tools.cpp + ${CMAKE_CURRENT_LIST_DIR}/src/execution_spaces.cpp + ${CMAKE_CURRENT_LIST_DIR}/src/complex_dtypes.cpp) + +SET(libpykokkos_HEADERS + ${CMAKE_CURRENT_LIST_DIR}/include/libpykokkos.hpp + ${CMAKE_CURRENT_LIST_DIR}/include/deep_copy.hpp + ${CMAKE_CURRENT_LIST_DIR}/include/concepts.hpp + ${CMAKE_CURRENT_LIST_DIR}/include/defines.hpp + ${CMAKE_CURRENT_LIST_DIR}/include/common.hpp + ${CMAKE_CURRENT_LIST_DIR}/include/traits.hpp + ${CMAKE_CURRENT_LIST_DIR}/include/views.hpp + ${CMAKE_CURRENT_LIST_DIR}/include/fwd.hpp + ${CMAKE_CURRENT_LIST_DIR}/include/execution_spaces.hpp) + +ADD_LIBRARY(libpykokkos-core OBJECT + ${libpykokkos_SOURCES} + ${libpykokkos_HEADERS}) + +TARGET_LINK_LIBRARIES(libpykokkos-core PUBLIC + pybind11::pybind11 + Kokkos::kokkos + libpykokkos::precompiled-headers + libpykokkos::build-options) + +PYBIND11_ADD_MODULE(libpykokkos MODULE NO_EXTRAS + $) + +ADD_SUBDIRECTORY(src) + +# link to kokkos and the custom build properties +TARGET_LINK_LIBRARIES(libpykokkos PRIVATE + pybind11::pybind11 + Kokkos::kokkos + libpykokkos::precompiled-headers + libpykokkos::build-options) + +IF(SKBUILD) + SET(Kokkos_INSTALL_PYTHONDIR ${CMAKE_INSTALL_PREFIX}) + SET(Kokkos_INSTALL_LIBDIR ${CMAKE_INSTALL_PREFIX}/kokkos) +ELSE() + SET(Kokkos_INSTALL_PYTHONDIR ${Python3_SITEARCH}/kokkos) + SET(Kokkos_INSTALL_LIBDIR ${Python3_SITEARCH}/kokkos) +ENDIF() + +# figure out if we can install to Python3_SITEARCH +EXECUTE_PROCESS( + COMMAND ${CMAKE_COMMAND} -E touch ${Python3_SITEARCH}/.__kokkos__init__.py + WORKING_DIRECTORY ${PROJECT_BINARY_DIR} + ERROR_VARIABLE ERR_MSG + RESULT_VARIABLE ERR_CODE) + +ADD_FEATURE(Python3_SITEARCH "Python site-packages directory") + +IF(ERR_CODE AND NOT SKBUILD) + # get the python directory name, e.g. 'python3.6' from + # '/opt/local/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6' + get_filename_component(PYDIR "${Python3_STDLIB}" NAME) + # Should not be CMAKE_INSTALL_LIBDIR! Python won't look in a lib64 folder + set(Kokkos_INSTALL_PYTHONDIR lib/${PYDIR}/site-packages/kokkos) + set(Kokkos_INSTALL_LIBDIR lib/${PYDIR}/site-packages/kokkos) +ENDIF() + +EXECUTE_PROCESS( + COMMAND ${CMAKE_COMMAND} -E rm -f ${Python3_SITEARCH}/.__kokkos__init__.py + WORKING_DIRECTORY ${PROJECT_BINARY_DIR}) + +# location where kokkos libraries are/will be installed +IF(ENABLE_INTERNAL_KOKKOS OR NOT Kokkos_ROOT) + SET(_Kokkos_LIBDIR "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}") +ELSEIF(Kokkos_ROOT) + STRING(REGEX REPLACE "/cmake/.*" "" _Kokkos_LIBDIR "${Kokkos_ROOT}") +ENDIF() + +# absolute path to libpykokkos install +SET(Kokkos_INSTALL_FULL_PYTHONDIR ${Kokkos_INSTALL_LIBDIR}) +IF(NOT IS_ABSOLUTE "${Kokkos_INSTALL_FULL_PYTHONDIR}") + SET(Kokkos_INSTALL_FULL_PYTHONDIR ${CMAKE_INSTALL_PREFIX}/${Kokkos_INSTALL_FULL_PYTHONDIR}) +ENDIF() + +# relative path from libpykokkos install directory to library install directory +FILE(RELATIVE_PATH LIB_RELPATH "${Kokkos_INSTALL_FULL_PYTHONDIR}" "${_Kokkos_LIBDIR}") + +# set the output path to /kokkos so one +# can test the python import from the build directory +# Really, only LIBRARY_* is needed for Unix but Windows +# builds are weird so just setting all of them +SET_TARGET_PROPERTIES(libpykokkos PROPERTIES + LIBRARY_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/kokkos + ARCHIVE_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/kokkos + RUNTIME_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/kokkos + PDB_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/kokkos) + +# configure the rpath: :: +IF(APPLE) + SET_TARGET_PROPERTIES(libpykokkos PROPERTIES + MACOSX_RPATH "@loader_path/${LIB_RELPATH}:@loader_path:${_Kokkos_LIBDIR}:${CMAKE_INSTALL_RPATH}") +ELSEIF(UNIX) + SET_TARGET_PROPERTIES(libpykokkos PROPERTIES + INSTALL_RPATH "\$ORIGIN/${LIB_RELPATH}:\$ORIGIN:${_Kokkos_LIBDIR}:${CMAKE_INSTALL_RPATH}") +ENDIF() + +CONFIGURE_FILE(${PROJECT_SOURCE_DIR}/pytest.ini + ${PROJECT_BINARY_DIR}/pytest.ini COPYONLY) + +IF(NOT SKBUILD) + CONFIGURE_FILE(${PROJECT_SOURCE_DIR}/setup.cfg + ${PROJECT_BINARY_DIR}/setup.cfg COPYONLY) +ENDIF() + +INSTALL(TARGETS libpykokkos + DESTINATION ${Kokkos_INSTALL_LIBDIR}) + +INSTALL(FILES ${PROJECT_BINARY_DIR}/kokkos/__init__.py + DESTINATION ${Kokkos_INSTALL_PYTHONDIR}) + +INSTALL(FILES ${PROJECT_BINARY_DIR}/pytest.ini + DESTINATION ${Kokkos_INSTALL_PYTHONDIR}) + +# glob any python package files +FILE(GLOB_RECURSE PYPACKAGE_FILES ${CMAKE_CURRENT_LIST_DIR}/kokkos/*.py*) +FOREACH(_FILE ${PYPACKAGE_FILES}) + # make it a relative path + STRING(REPLACE "${CMAKE_CURRENT_LIST_DIR}/" "" _OUT_NAME "${_FILE}") + # get the directory of the relative path + GET_FILENAME_COMPONENT(_OUT_PATH "${_OUT_NAME}" DIRECTORY) + # get the name without the extension + GET_FILENAME_COMPONENT(_OUT_NAME "${_OUT_NAME}" NAME_WE) + # target file for configure + SET(_OUT_FILE ${PROJECT_BINARY_DIR}/${_OUT_PATH}/${_OUT_NAME}.py) + # put version, python interpreter, etc. in the file for reference + CONFIGURE_FILE(${_FILE} ${_OUT_FILE} @ONLY) + # patch duplicated subfolder + STRING(REPLACE "kokkos/kokkos" "kokkos" _OUT_PATH "${Kokkos_INSTALL_PYTHONDIR}/${_OUT_PATH}") + # install to the correct folder structure + INSTALL(FILES ${_OUT_FILE} DESTINATION ${_OUT_PATH}) +ENDFOREACH() + +# build the examples, not designed to be built stand-alone +IF(ENABLE_EXAMPLES) + ADD_SUBDIRECTORY(examples) +ENDIF() + +IF(NOT ENABLE_QUIET) + PRINT_FEATURES() +ENDIF() + +IF("CUDA" IN_LIST Kokkos_DEVICES AND CMAKE_CXX_COMPILER_ID MATCHES "GNU") + IF(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 8.0 AND + CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9.0) + MESSAGE(AUTHOR_WARNING "\nNVCC + GCC 8.x + PyBind11 has known compiler errors related to pybind11::detail::collect_arguments overloads\n") + ENDIF() +ENDIF() diff --git a/base/LICENSE b/base/LICENSE new file mode 100644 index 00000000..e64fccc9 --- /dev/null +++ b/base/LICENSE @@ -0,0 +1,43 @@ +//@HEADER +// ************************************************************************ +// +// Kokkos v. 4.7.1 +// Copyright (2025) National Technology & Engineering +// Solutions of Sandia, LLC (NTESS). +// +// Under the terms of Contract DE-NA0003525 with NTESS, +// the U.S. Government retains certain rights in this software. +// +// Kokkos is licensed under 3-clause BSD terms of use: +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the Corporation nor the names of the +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY NTESS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NTESS OR THE +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Questions? Contact Christian R. Trott (crtrott@sandia.gov) +// +// ************************************************************************ +//@HEADER diff --git a/base/MANIFEST.in b/base/MANIFEST.in new file mode 100644 index 00000000..b4af9670 --- /dev/null +++ b/base/MANIFEST.in @@ -0,0 +1,40 @@ + +# direct includes + +include CMakeLists.txt +include VERSION +include LICENSE +include README.md +include CHANGES.md +include CONTRIBUTING.md +include CODE_OF_CONDUCT.md +include requirements.txt +include pytest.ini + +# recursive includes + +recursive-include cmake * +recursive-include kokkos * +recursive-include examples * +recursive-include external * +recursive-include include * +recursive-include src * +recursive-include tests * +recursive-include scripts * +recursive-include .requirements * + +# global exclude + +global-exclude *.pyc +global-exclude *.git* +global-exclude *__pycache__* +global-exclude kokkos/libpykokkos.* +global-exclude kokkos/__init__.py + +# recursive excludes + +recursive-exclude kokkos.egg-info * +recursive-exclude _skbuild * +recursive-exclude build * +recursive-exclude dist * +recursive-exclude kokkos/__pycache__ * diff --git a/base/README.md b/base/README.md new file mode 100644 index 00000000..b9a6d584 --- /dev/null +++ b/base/README.md @@ -0,0 +1,476 @@ +# pykokkos-base +[![linux-ci](https://github.com/kokkos/pykokkos-base/actions/workflows/linux-ci.yml/badge.svg)](https://github.com/kokkos/pykokkos-base/actions/workflows/linux-ci.yml) +[![python-package](https://github.com/kokkos/pykokkos-base/actions/workflows/python-package.yml/badge.svg)](https://github.com/kokkos/pykokkos-base/actions/workflows/python-package.yml) + +> Additional Documentation can be found in [Wiki](https://github.com/kokkos/pykokkos-base/wiki) + +## Overview + +This package contains the minimal set of bindings for [Kokkos](https://github.com/kokkos/kokkos) +interoperability with Python: + +- Free-standing function bindings + - `Kokkos::initialize(...)` + - `Kokkos::finalize()` + - `Kokkos::is_initialized()` + - `Kokkos::deep_copy(...)` + - `Kokkos::create_mirror(...)` + - `Kokkos::create_mirror_view(...)` + - `Kokkos::Tools::profileLibraryLoaded()` + - `Kokkos::Tools::pushRegion(...)` + - `Kokkos::Tools::popRegion()` + - `Kokkos::Tools::createProfileSection(...)` + - `Kokkos::Tools::destroyProfileSection(...)` + - `Kokkos::Tools::startSection(...)` + - `Kokkos::Tools::stopSection(...)` + - `Kokkos::Tools::markEvent(...)` + - `Kokkos::Tools::declareMetadata(...)` + - `Kokkos::Tools::Experimental::set_<...>_callback(...)` +- Data structures + - `Kokkos::View<...>` + - `Kokkos::DynRankView<...>` + - `Kokkos_Profiling_KokkosPDeviceInfo` + - `Kokkos_Profiling_SpaceHandle` + +By importing this package in Python, you can pass the supported Kokkos Views and DynRankViews +from C++ to Python and vice-versa. Furthermore, in Python, these bindings provide interoperability +with numpy and cupy arrays: + +```python +import kokkos +import numpy as np + +view = kokkos.array([2, 2], dtype=kokkos.double, space=kokkos.CudaUVMSpace, + layout=kokkos.LayoutRight, trait=kokkos.RandomAccess, + dynamic=False) + +arr = np.array(view, copy=False) +``` + +## Writing Kokkos in Python + +In order to write native Kokkos in Python, see [pykokkos](https://github.com/kokkos/pykokkos). + +## Installation + +You can install this package via CMake or Python's `setup.py`. The important cmake options are: + +- `ENABLE_VIEW_RANKS` (integer) +- `ENABLE_LAYOUTS` (bool) +- `ENABLE_MEMORY_TRAITS` (bool) +- `ENABLE_INTERNAL_KOKKOS` (bool) + +By default, CMake will enable the layouts and memory traits options if the Kokkos installation was not +built with CUDA support. +If Kokkos was built with CUDA support, `ENABLE_MEMORY_TRAITS` will be disabled by default due to unreasonable +compilation times (> 1 hour). +The `ENABLE_VIEW_RANKS` option (defaults to a value of 4) is the max number of ranks for +`Kokkos::View<...>` that can be returned to Python. For example, value of 4 means that +views of data type `T*`, `T**`, `T***`, and `T****` can be returned to python but +`T*****` and higher cannot. Increasing this value up to 7 can dramatically increase the length +of time required to compile the bindings. + +Installation command with specific arguments looks like this: + +``` +PYKOKKOS_BASE_SETUP_ARGS="-DKokkos_ENABLE_THREADS=OFF -DKokkos_ENABLE_OPENMP=ON -DENABLE_CUDA=ON -DENABLE_LAYOUTS=ON -DENABLE_MEMORY_TRAITS=OFF -DENABLE_VIEW_RANKS=3" pip install ./ --verbose +``` + +### Kokkos Installation + +If the `ENABLE_INTERNAL_KOKKOS` option is not specified the first time CMake is run, CMake will try to +find an existing Kokkos installation. If no existing installation is found, it will build and install +Kokkos from a submodule. When Kokkos is added as a submodule, you can configure the submodule +as you would normally configure Kokkos. However, due to some general awkwardness configuring cmake +from `setup.py` (especially via `pip install`), CMake tries to "automatically" configure +reasonable default CMake settings for the Kokkos submodule. + +Here are the steps when Kokkos is added as a submodule: + +- Set `BUILD_SHARED_LIBS=ON` +- Set `Kokkos_ENABLE_SERIAL=ON` +- `find_package(OpenMP)` + - Was OpenMP found? + - **YES**: set `Kokkos_ENABLE_OPENMP=ON` + - **NO**: `find_package(Threads)` + - Was Threads found? + - **YES**: set `Kokkos_ENABLE_THREADS=ON` (if not Windows) +- `find_package(CUDA)` + - Was CUDA found? + - **YES**: set: + - `Kokkos_ENABLE_CUDA=ON` + - `Kokkos_ENABLE_CUDA_UVM=ON` + - `Kokkos_ENABLE_CUDA_LAMBDA=ON` + +### Configuring Options via CMake + +```console +cmake -DENABLE_LAYOUTS=ON -DENABLE_MEMORY_TRAITS=OFF /path/to/source +``` + +### Configuring Options via `setup.py` + +There are three ways to configure the options: + +1. Via the Python argparse options `--enable-