refactor(deps): organize test and dev dependencies using dependency groups (#995)

Author: cpcloud

.github/workflows/build-wheel.yml

@@ -227,7 +227,7 @@ jobs:
 
       - name: Build cuda.bindings Cython tests
         run: |
-          pip install $(ls ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}/*.whl)[test]
+          pip install ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}/*.whl --group ./cuda_bindings/pyproject.toml:test
           pushd ${{ env.CUDA_BINDINGS_CYTHON_TESTS_DIR }}
           bash build_tests.sh
           popd
@@ -241,7 +241,7 @@ jobs:
 
       - name: Build cuda.core Cython tests
         run: |
-          pip install $(ls ${{ env.CUDA_CORE_ARTIFACTS_DIR }}/*.whl)[test]
+          pip install ${{ env.CUDA_CORE_ARTIFACTS_DIR }}/*.whl --group ./cuda_core/pyproject.toml:test
           pushd ${{ env.CUDA_CORE_CYTHON_TESTS_DIR }}
           bash build_tests.sh
           popd

.github/workflows/test-wheel-linux.yml

@@ -284,7 +284,7 @@ jobs:
         run: |
           set -euo pipefail
           pushd cuda_pathfinder
-          pip install --only-binary=:all: -v ".[test_nvidia_wheels_cu${TEST_CUDA_MAJOR},test_nvidia_wheels_host]"
+          pip install --only-binary=:all: -v . --group "test_nvidia_wheels_cu${TEST_CUDA_MAJOR}" --group test_nvidia_wheels_host
           pip list
           popd
 

.github/workflows/test-wheel-windows.yml

@@ -257,7 +257,7 @@ jobs:
         shell: bash --noprofile --norc -xeuo pipefail {0}
         run: |
           pushd cuda_pathfinder
-          pip install --only-binary=:all: -v ".[test_nvidia_wheels_cu${TEST_CUDA_MAJOR},test_nvidia_wheels_host]"
+          pip install --only-binary=:all: -v . --group "test_nvidia_wheels_cu${TEST_CUDA_MAJOR}" --group test_nvidia_wheels_host
           pip list
           popd
 

ci/tools/run-tests

@@ -24,7 +24,7 @@ test_module=${1}
 # (it is a direct dependency of bindings, and a transitive dependency of core)
 pushd ./cuda_pathfinder
 echo "Installing pathfinder wheel"
-pip install $(ls *.whl)[test]
+pip install ./*.whl --group test
 popd
 
 if [[ "${test_module}" == "pathfinder" ]]; then
@@ -38,15 +38,13 @@ if [[ "${test_module}" == "pathfinder" ]]; then
   echo "Number of \"INFO test_\" lines: $line_count"
   popd
 elif [[ "${test_module}" == "bindings" ]]; then
-  pushd "${CUDA_BINDINGS_ARTIFACTS_DIR}"
   echo "Installing bindings wheel"
+  pushd ./cuda_bindings
   if [[ "${LOCAL_CTK}" == 1 ]]; then
-    pip install $(ls *.whl)[test]
+    pip install "${CUDA_BINDINGS_ARTIFACTS_DIR}"/*.whl --group test
   else
-    pip install $(ls *.whl)[all,test]
+    pip install $(ls "${CUDA_BINDINGS_ARTIFACTS_DIR}"/*.whl)[all] --group test
   fi
-  popd
-  pushd ./cuda_bindings
   echo "Running bindings tests"
   ${SANITIZER_CMD} pytest -rxXs -v tests/
   if [[ "${SKIP_CYTHON_TEST}" == 0 ]]; then
@@ -57,33 +55,29 @@ elif [[ "${test_module}" == "core" ]]; then
   # If build/test majors match: cuda.bindings is installed in the previous step.
   # If mismatch: cuda.bindings is installed from the backport branch.
   if [[ "${SKIP_CUDA_BINDINGS_TEST}" == 1 ]]; then
-    pushd "${CUDA_BINDINGS_ARTIFACTS_DIR}"
     echo "Installing bindings wheel"
     if [[ "${LOCAL_CTK}" == 1 ]]; then
-      pip install *.whl
+      pip install "${CUDA_BINDINGS_ARTIFACTS_DIR}"/*.whl
     else
-      pip install $(ls *.whl)[all]
+      pip install $(ls "${CUDA_BINDINGS_ARTIFACTS_DIR}"/*.whl)[all]
     fi
-    popd
   fi
   TEST_CUDA_MAJOR="$(cut -d '.' -f 1 <<< ${CUDA_VER})"
-  pushd "${CUDA_CORE_ARTIFACTS_DIR}"
   echo "Installing core wheel"
 
   FREE_THREADING=""
   if python -c 'import sys; assert not sys._is_gil_enabled()' 2> /dev/null; then
     FREE_THREADING+="-ft"
   fi
 
+  pushd ./cuda_core
   if [[ "${LOCAL_CTK}" == 1 ]]; then
     # We already installed cuda-bindings, and all CTK components exist locally,
     # so just install the test dependencies.
-    pip install $(ls *.whl)["test-cu${TEST_CUDA_MAJOR}${FREE_THREADING}"]
+    pip install "${CUDA_CORE_ARTIFACTS_DIR}"/*.whl --group "test-cu${TEST_CUDA_MAJOR}${FREE_THREADING}"
   else
-    pip install $(ls *.whl)["cu${TEST_CUDA_MAJOR}","test-cu${TEST_CUDA_MAJOR}${FREE_THREADING}"]
+    pip install $(ls "${CUDA_CORE_ARTIFACTS_DIR}"/*.whl)["cu${TEST_CUDA_MAJOR}"] --group "test-cu${TEST_CUDA_MAJOR}${FREE_THREADING}"
   fi
-  popd
-  pushd ./cuda_core
   echo "Running core tests"
   ${SANITIZER_CMD} pytest -rxXs -v tests/
   # Currently our CI always installs the latest bindings (from either major version).

cuda_bindings/pyproject.toml

@@ -1,6 +1,5 @@
 # SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE
-
 [build-system]
 requires = ["setuptools>=77.0.0", "cython>=3.1,<3.2", "pyclibrary>=0.1.7"]
 build-backend = "setuptools.build_meta"
@@ -36,6 +35,7 @@ all = [
     "cuda-toolkit[cufile]==13.*; sys_platform == 'linux'",
 ]
 
+[dependency-groups]
 test = [
     "cython>=3.1,<3.2",
     "setuptools>=77.0.0",

cuda_core/pyproject.toml

@@ -49,10 +49,9 @@ dependencies = [
 cu11 = ["cuda-bindings[all]==11.8.*"]
 cu12 = ["cuda-bindings[all]==12.*"]
 cu13 = ["cuda-bindings[all]==13.*"]
-# TODO: these should all be in development dependencies; optional dependencies
-# are for features exposed to *users*, not a dumping ground for all tooling
-# needed to build and test the project
-test = ["cython>=3.1", "setuptools", "pytest>=6.2.4"]
+
+[dependency-groups]
+test = ["cython>=3.0", "setuptools", "pytest>=6.2.4"]
 test-cu11 = ["cuda-core[test]", "cupy-cuda11x; python_version < '3.14'", "cuda-toolkit[cudart]==11.*"]  # runtime headers needed by CuPy
 test-cu12 = ["cuda-core[test]", "cupy-cuda12x; python_version < '3.14'", "cuda-toolkit[cudart]==12.*"]  # runtime headers needed by CuPy
 test-cu13 = ["cuda-core[test]", "cupy-cuda13x; python_version < '3.14'", "cuda-toolkit[cudart]==13.*"]  # runtime headers needed by CuPy

cuda_pathfinder/pyproject.toml

@@ -10,7 +10,7 @@ requires-python = ">=3.9"
 dynamic = ["version", "readme"]
 dependencies = []
 
-[project.optional-dependencies]
+[dependency-groups]
 test = [
     "pytest>=6.2.4",
 ]

scripts/run_tests.sh

@@ -158,7 +158,7 @@ ensure_installed() {
   fi
 
   if [[ "${INSTALL_MODE}" == "force" ]]; then
-    pip install -e .[test]
+    pip install -e . --group test
     return 0
   fi
 
@@ -177,7 +177,7 @@ sys.exit(0 if str(p).startswith(str(sub)) else 3)
 PY
   rc=$?
   if [[ $rc -ne 0 ]]; then
-    pip install -e .[test]
+    pip install -e . --group test
   fi
 }
 

toolshed/collect_site_packages_so_files.sh

@@ -17,12 +17,12 @@ fresh_venv() {
 cd cuda_pathfinder/
 fresh_venv ../TmpCp12Venv
 set -x
-pip install --only-binary=:all: -e .[test,test_nvidia_wheels_cu12,test_nvidia_wheels_host]
+pip install --only-binary=:all: -e . --group test --group test_nvidia_wheels_cu12 --group test_nvidia_wheels_host
 set +x
 deactivate
 fresh_venv ../TmpCp13Venv
 set -x
-pip install --only-binary=:all: -e .[test,test_nvidia_wheels_cu13,test_nvidia_wheels_host]
+pip install --only-binary=:all: -e . --group test --group test_nvidia_wheels_cu13 --group test_nvidia_wheels_host
 set +x
 deactivate
 cd ..