build(deps): bump the python group across 1 directory with 68 updates #4213
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: ci | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| concurrency: | |
| # Cancels runs from previous pushes in a PR. | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} | |
| cancel-in-progress: true | |
| permissions: {} | |
| env: | |
| # Disable default package installs. Each job should explicitly install a group. | |
| UV_NO_SYNC: "1" | |
| jobs: | |
| compute-ros-pin: | |
| # Extracts the ros-dev image digest pinned in docker/ros-dev-pin/Dockerfile | |
| # so downstream jobs can reference it from their `container:` field. | |
| # That will be updated by Dependabot, and keep CI tests stable. | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read # For checkout | |
| outputs: | |
| digest: ${{ steps.read.outputs.digest }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v7 | |
| - id: read | |
| run: | | |
| DIGEST=$(grep -oE 'sha256:[a-f0-9]{64}' docker/ros-dev-pin/Dockerfile) | |
| if [ -z "$DIGEST" ]; then | |
| echo "::error::No sha256 digest found in docker/ros-dev-pin/Dockerfile" | |
| exit 1 | |
| fi | |
| echo "digest=$DIGEST" >> "$GITHUB_OUTPUT" | |
| lint: | |
| timeout-minutes: 10 | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read # For checkout | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v7 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v8.2.0 | |
| with: | |
| enable-cache: true | |
| - name: Install lint dependencies | |
| run: uv sync --only-group lint --frozen | |
| - name: Mypy | |
| run: uv run mypy | |
| - name: Run pre-commit | |
| uses: pre-commit/action@v3.0.1 | |
| rust: | |
| timeout-minutes: 20 | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read # For checkout | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v7 | |
| - name: Find Cargo workspace roots | |
| id: find-crates | |
| run: | | |
| crates=$(find . -name Cargo.lock \ | |
| -not -path './target/*' \ | |
| -not -path './.venv/*' \ | |
| -not -path './node_modules/*' \ | |
| -not -path './.git/*' \ | |
| -printf '%h\n' \ | |
| | sed 's|^\./||' \ | |
| | sort) | |
| echo "Found Rust workspace roots:" | |
| echo "$crates" | |
| { | |
| echo "crates<<EOF" | |
| echo "$crates" | |
| echo "EOF" | |
| } >> "$GITHUB_OUTPUT" | |
| - name: Install Rust toolchain | |
| uses: dtolnay/rust-toolchain@stable | |
| with: | |
| components: rustfmt, clippy | |
| - name: Cache cargo build | |
| uses: Swatinem/rust-cache@v2 | |
| with: | |
| workspaces: ${{ steps.find-crates.outputs.crates }} | |
| - name: cargo fmt | |
| env: | |
| CRATES: ${{ steps.find-crates.outputs.crates }} | |
| run: | | |
| for crate in $CRATES; do | |
| (cd "$crate" && cargo fmt --all -- --check) | |
| done | |
| - name: cargo clippy | |
| env: | |
| CRATES: ${{ steps.find-crates.outputs.crates }} | |
| run: | | |
| for crate in $CRATES; do | |
| (cd "$crate" && cargo clippy --workspace --all-targets --all-features --locked -- -D warnings) | |
| done | |
| - name: cargo test | |
| env: | |
| CRATES: ${{ steps.find-crates.outputs.crates }} | |
| run: | | |
| for crate in $CRATES; do | |
| (cd "$crate" && cargo test --workspace --all-features --locked) | |
| done | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v8.2.0 | |
| with: | |
| enable-cache: true | |
| - name: Install dependency for pyaudio | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y portaudio19-dev | |
| - name: Build and test PyO3 bindings | |
| run: | | |
| uv sync --group tests --frozen | |
| source .venv/bin/activate | |
| maturin develop -m dimos/mapping/ray_tracing/rust/Cargo.toml | |
| maturin develop -m dimos/navigation/nav_3d/mls_planner/rust/Cargo.toml | |
| python -c "import dimos_voxel_ray_tracing, dimos_mls_planner" | |
| python -m pytest -c /dev/null --rootdir . --noconftest --import-mode=importlib -p no:cacheprovider -v \ | |
| dimos/mapping/ray_tracing/test_voxel_map.py \ | |
| dimos/mapping/ray_tracing/test_transformer.py \ | |
| dimos/navigation/nav_3d/mls_planner/test_mls_planner.py \ | |
| dimos/navigation/nav_3d/mls_planner/test_transformer.py | |
| md-babel: | |
| timeout-minutes: 15 | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read # For checkout | |
| steps: | |
| - uses: actions/checkout@v7 | |
| - name: Fetch LFS data needed by doc code blocks | |
| run: git lfs pull --include="data/.lfs/go2_bigoffice.db.tar.gz,data/.lfs/unitree_go2_bigoffice_map.pickle.tar.gz" | |
| # Docs decode JPEG from SQLite via PyTurboJPEG; pyaudio needs portaudio. | |
| - name: Install system dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y libturbojpeg portaudio19-dev | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v8.2.0 | |
| with: | |
| enable-cache: true | |
| - uses: actions/setup-node@v6 | |
| with: | |
| node-version: 'lts/*' | |
| - name: Install Python dependencies | |
| run: uv sync --group tests --frozen | |
| - name: Restore Hugging Face model cache | |
| id: hf-cache | |
| uses: actions/cache/restore@v6 | |
| with: | |
| path: ~/.cache/huggingface | |
| # The docs determine which models get downloaded, so any docs | |
| # change may introduce a new model: go online and re-save the | |
| # cache then. restore-keys still seeds the new cache with the | |
| # previously cached models, so only genuinely new models are | |
| # downloaded. | |
| key: hf-models-${{ hashFiles('docs/**/*.md') }} | |
| restore-keys: | | |
| hf-models- | |
| - name: Prefetch Hugging Face models | |
| if: steps.hf-cache.outputs.cache-hit != 'true' | |
| # Anonymous downloads from shared GitHub runner IPs get rate-limited | |
| # by huggingface.co (HTTP 429), so retry with backoff. Models already | |
| # in the restored cache are not re-downloaded. | |
| run: | | |
| for i in 1 2 3 4 5; do | |
| uv run python -c "from transformers import CLIPModel, CLIPProcessor; m = 'openai/clip-vit-base-patch32'; CLIPModel.from_pretrained(m); CLIPProcessor.from_pretrained(m)" && exit 0 | |
| echo "Hugging Face download failed (likely rate-limited); retry $i in $((i * 30))s" | |
| sleep $((i * 30)) | |
| done | |
| echo "::error::Could not download Hugging Face models after 5 attempts" | |
| exit 1 | |
| - name: Save Hugging Face model cache | |
| # Save right after the download (not in a post step) so a failure in | |
| # the doc blocks below doesn't throw away the downloaded models. | |
| if: steps.hf-cache.outputs.cache-hit != 'true' | |
| uses: actions/cache/save@v6 | |
| with: | |
| path: ~/.cache/huggingface | |
| key: hf-models-${{ hashFiles('docs/**/*.md') }} | |
| - name: Run Hugging Face offline on exact cache hit | |
| # Zero huggingface.co requests when every model is already cached: | |
| # immune to rate limiting. | |
| if: steps.hf-cache.outputs.cache-hit == 'true' | |
| run: echo "HF_HUB_OFFLINE=1" >> "$GITHUB_ENV" | |
| - name: Execute documentation code blocks | |
| run: ./bin/run-doc-codeblocks --ci --no-cache | |
| docs-validate: | |
| timeout-minutes: 5 | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read # For checkout | |
| steps: | |
| - uses: actions/checkout@v7 | |
| with: | |
| lfs: false | |
| sparse-checkout: | | |
| docs | |
| - name: Pull docs LFS assets only | |
| run: git lfs pull --include="docs/assets/**" --exclude="" | |
| - uses: actions/setup-node@v6 | |
| with: | |
| node-version: '20' | |
| - name: Install docs dependencies | |
| working-directory: docs | |
| run: npm ci | |
| # Temporarily disabled — Mintlify validation blocking CI | |
| # - name: Validate Mintlify docs | |
| # working-directory: docs | |
| # run: npm run validate | |
| tests: | |
| timeout-minutes: 20 | |
| strategy: | |
| matrix: | |
| pyver: ['3.10', '3.11', '3.12', '3.13', '3.14'] | |
| os: ["ubuntu-latest"] | |
| experimental: [false] | |
| include: | |
| - os: "ubuntu-latest" | |
| pyver: "3.14t" | |
| experimental: false | |
| - os: "ubuntu-24.04-arm" | |
| pyver: "3.14" | |
| experimental: false | |
| fail-fast: true | |
| runs-on: ${{ matrix.os }} | |
| continue-on-error: ${{ matrix.experimental }} | |
| permissions: | |
| contents: read # For checkout | |
| id-token: write # For codecov-action's OIDC upload | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v7 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v8.2.0 | |
| with: | |
| enable-cache: true | |
| - name: Setup Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| allow-prereleases: true | |
| python-version: ${{ matrix.pyver }} | |
| - name: Install dependency for pyaudio (Ubuntu) | |
| if: startsWith(matrix.os, 'ubuntu') | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y portaudio19-dev | |
| - name: Install dependency for pyaudio (macOS) | |
| if: startsWith(matrix.os, 'macos') | |
| run: brew install portaudio | |
| - name: Cap accidental LFS downloads at 1 MiB | |
| run: | | |
| sudo mv /usr/bin/git-lfs /usr/bin/git-lfs.orig | |
| sudo install -m 0755 bin/git-lfs-guard /usr/bin/git-lfs | |
| git config --global --unset-all filter.lfs.process || true | |
| - name: Set PYTHON_GIL=0 for free-threading builds | |
| if: ${{ endsWith(matrix.pyver, 't') }} | |
| run: echo "PYTHON_GIL=0" >> $GITHUB_ENV | |
| - name: Install dependencies | |
| run: uv sync --group tests --frozen | |
| - name: Run tests | |
| run: uv run pytest --numprocesses=3 --cov=dimos/ --junitxml=junit.xml -m 'not (self_hosted or mujoco or self_hosted_large)' | |
| - name: Re-run the failing tests with maximum verbosity | |
| if: failure() | |
| env: | |
| COLOR: yes | |
| run: >- # `exit 1` makes sure that the job remains red with flaky runs | |
| uv run pytest --no-cov -vvvvv --lf -m 'not (self_hosted or mujoco or self_hosted_large)' && exit 1 | |
| shell: bash | |
| - name: Turn coverage into xml | |
| run: uv run python -m coverage xml | |
| - name: Upload coverage | |
| uses: codecov/codecov-action@v7 | |
| with: | |
| disable_search: true | |
| fail_ci_if_error: true | |
| files: ./coverage.xml | |
| flags: OS-${{ matrix.os }},Py-${{ matrix.pyver }} | |
| use_oidc: true | |
| # Install the CLI from PyPI (pinned) instead of cli.codecov.io: | |
| # the GPG key for the default download path is served from a dead | |
| # keybase.io URL, failing every upload. See | |
| # codecov/codecov-action#1955 and #2398. | |
| use_pypi: true | |
| version: "11.2.8" | |
| - name: Upload test results to Codecov | |
| if: ${{ !cancelled() }} | |
| uses: codecov/codecov-action@v7 | |
| with: | |
| report_type: test_results | |
| use_oidc: true | |
| # See the "Upload coverage" step above. | |
| use_pypi: true | |
| version: "11.2.8" | |
| cmu-nav-natives-marker: | |
| # Check whether the modules-for-this-config marker cache | |
| # already exists. Used to skip cmu-nav-natives without using the cachix environment. | |
| if: | | |
| github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read # For checkout | |
| outputs: | |
| cache-hit: ${{ steps.marker.outputs.cache-hit }} | |
| key: ${{ steps.cache-key.outputs.key }} | |
| steps: | |
| - name: Checkout files used for cache key | |
| uses: actions/checkout@v7 | |
| with: | |
| sparse-checkout: | | |
| bin/build-cmu-nav-natives | |
| dimos/navigation/cmu_nav/modules | |
| sparse-checkout-cone-mode: false | |
| - name: Compute cache key | |
| # Recomputing the key in cmu-nav-natives after build breaks the key. | |
| # Save here so it can be reused verbatim. | |
| id: cache-key | |
| run: echo "key=cmu-nav-natives-${{ hashFiles('bin/build-cmu-nav-natives', 'dimos/navigation/cmu_nav/modules/**') }}" >> "$GITHUB_OUTPUT" | |
| - name: Look up marker cache | |
| id: marker | |
| uses: actions/cache/restore@v6 | |
| with: | |
| path: /tmp/cmu-nav-natives-marker | |
| key: ${{ steps.cache-key.outputs.key }} | |
| lookup-only: true | |
| cmu-nav-natives: | |
| # ⚠️ MUST NEVER RUN ON EXTERNAL PRs ⚠️ | |
| # This job has write access to CACHIX_AUTH_TOKEN. Any workflow change that | |
| # reaches a `run:` step here could exfiltrate the token, letting an attacker | |
| # push poisoned binaries to the Cachix cache — which subsequent CI runs and | |
| # every dev with the substituter configured would unwittingly install. The | |
| # `if:` guard below is the only thing keeping fork PRs out — DO NOT REMOVE. | |
| if: | | |
| !cancelled() && | |
| (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository) && | |
| needs.cmu-nav-natives-marker.outputs.cache-hit != 'true' | |
| needs: cmu-nav-natives-marker | |
| timeout-minutes: 300 | |
| environment: cachix | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read # For checkout | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v7 | |
| - name: Free runner disk space | |
| uses: BRAINSia/free-disk-space@v2.1.3 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v8.2.0 | |
| with: | |
| enable-cache: true | |
| - name: Install minimal dimos for module discovery | |
| run: uv sync --no-default-groups --frozen | |
| - name: Install Nix (with Cachix substituter) | |
| env: | |
| INPUT_EXTRA_NIX_CONFIG: | | |
| extra-substituters = https://dimensionalos.cachix.org | |
| extra-trusted-public-keys = dimensionalos.cachix.org-1:20ynj6TjpoD3qTxkdNoeHtgs2G2pNvgAq1EQYLTHJXI= | |
| INPUT_SET_AS_TRUSTED_USER: "true" | |
| run: bash docker/ros/install-nix.sh | |
| - name: Cache Nix fetcher + eval cache | |
| # ~/.cache/nix holds the flake fetcher cache (the github tarballs of | |
| # each `nix build github:…` ref) and the eval cache (compiled flake | |
| # outputs). ~/.local/state/nix holds profile/gcroot bookkeeping. | |
| # Restoring these skips the ~20s/module evaluation cost, which is | |
| # paid every cold run otherwise. /nix/store is intentionally NOT | |
| # cached here — that's the slow heavy part that Cachix handles. | |
| uses: actions/cache@v6 | |
| with: | |
| path: | | |
| ~/.cache/nix | |
| ~/.local/state/nix | |
| key: nix-fetcher-build-${{ runner.os }}-${{ hashFiles('bin/build-cmu-nav-natives', 'dimos/navigation/cmu_nav/modules/**/*.py') }} | |
| restore-keys: nix-fetcher-build-${{ runner.os }}- | |
| - name: Authenticate Cachix | |
| uses: cachix/cachix-action@v17 | |
| with: | |
| name: dimensionalos | |
| authToken: ${{ secrets.CACHIX_AUTH_TOKEN }} | |
| - name: Build cmu_nav native modules | |
| env: | |
| BUILD_WORKERS: "2" | |
| run: uv run bin/build-cmu-nav-natives | |
| - name: Write marker file | |
| run: install -D /dev/null /tmp/cmu-nav-natives-marker/ok | |
| - name: Save marker cache | |
| # Done as a final step (rather than via actions/cache's post-step) so | |
| # a build failure doesn't poison the cache with a marker for a config | |
| # we never actually pushed. | |
| uses: actions/cache/save@v6 | |
| with: | |
| path: /tmp/cmu-nav-natives-marker | |
| key: ${{ needs.cmu-nav-natives-marker.outputs.key }} | |
| self-hosted-tests: | |
| # Skip on PRs from forks which would expose the self-hosted runner to untrusted code from external contributors. | |
| if: | | |
| !cancelled() && | |
| (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository) && | |
| contains(fromJSON('["success", "skipped"]'), needs.cmu-nav-natives.result) | |
| needs: [compute-ros-pin, cmu-nav-natives] | |
| env: | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| ALIBABA_API_KEY: ${{ secrets.ALIBABA_API_KEY }} | |
| timeout-minutes: 45 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - os: Linux | |
| # GitHub Actions only honours `container:` on Linux runners. | |
| container: | |
| image: ghcr.io/dimensionalos/ros-dev@${{ needs.compute-ros-pin.outputs.digest }} | |
| # Persist Nix state on the self-hosted runner's filesystem so | |
| # the ~3 GB module closures survive between jobs. | |
| volumes: | |
| - /var/cache/dimos-nix:/nix | |
| - /var/cache/dimos-root-cache:/root/.cache | |
| markers: "self_hosted or skipif_no_ros" | |
| experimental: false | |
| - os: macOS | |
| container: null # run on host — `container:` is Linux-only | |
| markers: "self_hosted" | |
| experimental: true | |
| runs-on: | |
| - self-hosted | |
| - ${{ matrix.os }} | |
| - base | |
| continue-on-error: ${{ matrix.experimental }} | |
| permissions: | |
| contents: read # For checkout | |
| packages: read # For pulling the ros-dev container from ghcr.io | |
| id-token: write # For codecov-action's OIDC upload | |
| container: ${{ matrix.container }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v7 | |
| with: | |
| clean: false | |
| # If we ever allow external PRs on custom runner, persisting credentials | |
| # could be abused by attackers. | |
| persist-credentials: false | |
| - name: Fix permissions | |
| run: | | |
| git config --global --add safe.directory '*' | |
| git clean -ffdx | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v8.2.0 | |
| with: | |
| enable-cache: true | |
| - name: Install codecov CLI | |
| # Self-hosted runners have no `pip` on PATH, so codecov-action's | |
| # use_pypi can't work here; pre-install the pinned CLI with uv and | |
| # point the action at it via `binary`. See codecov/codecov-action#1955 | |
| # and #2398. | |
| run: | | |
| uv venv .codecov-venv | |
| uv pip install --python .codecov-venv/bin/python codecov-cli==11.2.8 | |
| - name: Install dependencies | |
| run: uv sync --group tests-self-hosted --frozen | |
| - name: Build C++ extensions in-place | |
| run: uv run python setup.py build_ext --inplace | |
| - name: Source ROS environment | |
| # The uv venv is sealed (include-system-site-packages = false), so | |
| # `import rclpy` / `ament_index_python` would fail. Sourcing the ROS | |
| # setup script and exporting PYTHONPATH/AMENT_PREFIX_PATH/etc into | |
| # GITHUB_ENV makes them importable from `uv run`. | |
| if: matrix.os == 'Linux' | |
| shell: bash | |
| run: | | |
| source /opt/ros/${ROS_DISTRO}/setup.bash | |
| { | |
| echo "PYTHONPATH=$PYTHONPATH" | |
| echo "AMENT_PREFIX_PATH=$AMENT_PREFIX_PATH" | |
| echo "CMAKE_PREFIX_PATH=$CMAKE_PREFIX_PATH" | |
| echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH" | |
| echo "ROS_DISTRO=$ROS_DISTRO" | |
| echo "ROS_VERSION=$ROS_VERSION" | |
| echo "ROS_PYTHON_VERSION=$ROS_PYTHON_VERSION" | |
| } >> "$GITHUB_ENV" | |
| - name: Install Nix (with Cachix substituter) | |
| if: contains(matrix.markers, 'skipif_no_ros') | |
| # /nix is bind-mounted from the self-hosted runner's host filesystem | |
| # (see this job's matrix container.volumes). On warm runs Nix is | |
| # already installed; install-nix.sh's `type -p nix` check will exit | |
| # early, but only if the profile bin is on PATH — prepend it before | |
| # calling. If /nix is empty (first job on a fresh runner), nix isn't | |
| # on PATH and the install proceeds normally. After install-nix.sh, | |
| # we unconditionally (re-)write /etc/nix/nix.conf: on warm runs | |
| # install-nix.sh exited before it would have written the config, | |
| # which leaves `experimental-features = nix-command flakes` absent | |
| # and the build dying with "experimental Nix feature 'nix-command' | |
| # is disabled". Writing it here keeps cold and warm runs identical. | |
| run: | | |
| export PATH="/nix/var/nix/profiles/default/bin:$PATH" | |
| bash docker/ros/install-nix.sh | |
| sudo mkdir -p /etc/nix | |
| sudo tee /etc/nix/nix.conf > /dev/null <<'EOF' | |
| experimental-features = nix-command flakes | |
| extra-substituters = https://dimensionalos.cachix.org | |
| extra-trusted-public-keys = dimensionalos.cachix.org-1:20ynj6TjpoD3qTxkdNoeHtgs2G2pNvgAq1EQYLTHJXI= | |
| always-allow-substitutes = true | |
| build-users-group = | |
| max-jobs = auto | |
| EOF | |
| echo "/nix/var/nix/profiles/default/bin" >> "$GITHUB_PATH" | |
| - name: Fetch cmu_nav native modules from Cachix | |
| if: contains(matrix.markers, 'skipif_no_ros') | |
| run: uv run bin/build-cmu-nav-natives | |
| - name: Run tests | |
| run: uv run pytest --cov=dimos/ --junitxml=junit.xml -m '(${{ matrix.markers }}) and not mujoco' | |
| - name: Re-run the failing tests with maximum verbosity | |
| if: failure() | |
| env: | |
| COLOR: yes | |
| run: >- # `exit 1` makes sure that the job remains red with flaky runs | |
| uv run pytest --no-cov -vvvvv --lf -m '(${{ matrix.markers }}) and not mujoco' && exit 1 | |
| shell: bash | |
| - name: Turn coverage into xml | |
| run: uv run python -m coverage xml | |
| - name: Upload coverage | |
| uses: codecov/codecov-action@v7 | |
| with: | |
| disable_search: true | |
| fail_ci_if_error: true | |
| files: ./coverage.xml | |
| flags: SelfHosted-${{ matrix.os }} | |
| use_oidc: true | |
| # Use the pinned CLI from the "Install codecov CLI" step (no pip | |
| # on self-hosted runners, and the default cli.codecov.io download | |
| # has a broken GPG key fetch). See codecov/codecov-action#1955. | |
| binary: .codecov-venv/bin/codecovcli | |
| - name: Upload test results to Codecov | |
| if: ${{ !cancelled() }} | |
| uses: codecov/codecov-action@v7 | |
| with: | |
| report_type: test_results | |
| use_oidc: true | |
| # See the "Upload coverage" step above. | |
| binary: .codecov-venv/bin/codecovcli | |
| - name: Check disk space | |
| if: failure() | |
| run: | | |
| df -h | |
| - name: Garbage-collect the persistent Nix store | |
| # /nix is bind-mounted from the runner host's /var/cache/dimos-nix | |
| # (see this job's container.volumes) so module closures survive between | |
| # jobs. But every run also imports a fresh `…-source` closure of the | |
| # tree — including the large data/.lfs recordings — and nothing ever | |
| # reclaims them, so the store grows unbounded until jobs fail with | |
| # "No space left on device". GC after each run keeps the last few days | |
| # (module closures fetched from Cachix stay warm; stale source closures | |
| # are freed). Runs on the Linux entry, the only one that mounts /nix and | |
| # puts nix on PATH (Install Nix step above). | |
| if: ${{ always() && contains(matrix.markers, 'skipif_no_ros') }} | |
| run: | | |
| df -h /nix || true | |
| nix-collect-garbage --delete-older-than 3d | |
| df -h /nix || true | |
| self-hosted-large-tests: | |
| # Skip on PRs from forks which would expose the self-hosted runner to untrusted code from external contributors. | |
| if: | | |
| github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository | |
| env: | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| ALIBABA_API_KEY: ${{ secrets.ALIBABA_API_KEY }} | |
| DIMSIM_RENDER: gpu | |
| timeout-minutes: 30 | |
| runs-on: [self-hosted, Linux, large] | |
| permissions: | |
| contents: read # For checkout | |
| packages: read # For pulling the ros-dev container from ghcr.io | |
| id-token: write # For codecov-action's OIDC upload | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v7 | |
| with: | |
| clean: false | |
| # If we ever allow external PRs on custom runner, persisting credentials | |
| # could be abused by attackers. | |
| persist-credentials: false | |
| - name: Fix permissions | |
| run: | | |
| sudo git config --global --add safe.directory '*' | |
| sudo git clean -ffdx | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v8.2.0 | |
| with: | |
| enable-cache: true | |
| - name: Install codecov CLI | |
| # Self-hosted runners have no `pip` on PATH, so codecov-action's | |
| # use_pypi can't work here; pre-install the pinned CLI with uv and | |
| # point the action at it via `binary`. See codecov/codecov-action#1955 | |
| # and #2398. | |
| run: | | |
| uv venv .codecov-venv | |
| uv pip install --python .codecov-venv/bin/python codecov-cli==11.2.8 | |
| - name: Install apt dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y \ | |
| `# Compiler toolchain for the in-tree pybind11 C++ extension.` \ | |
| build-essential \ | |
| `# Runtime lib for opencv-python (cv2 loads libGL.so.1 on import).` \ | |
| libgl1 \ | |
| `# Runtime lib for PyTurboJPEG (fast JPEG encode/decode).` \ | |
| libturbojpeg \ | |
| `# Dev package for PyAudio (built from source against PortAudio).` \ | |
| portaudio19-dev \ | |
| `# Chromium runtime libs required by Playwright (used by dimsim browser tests).` \ | |
| libatk1.0-0 libatk-bridge2.0-0 libcairo2 libcups2 libgbm1 \ | |
| libpango-1.0-0 libxcomposite1 libxdamage1 libxkbcommon0 libxrandr2 \ | |
| `# Rerun viewer pulls in X11 keyboard bindings when DISPLAY is set.` \ | |
| libxkbcommon-x11-0 | |
| - name: Install dependencies | |
| run: uv sync --group tests-self-hosted --frozen | |
| - name: Build C++ extensions in-place | |
| run: uv run python setup.py build_ext --inplace | |
| - name: Run tests | |
| # The runner must have an Xorg server with NVIDIA GPU access running | |
| # on DISPLAY=:0 — dimsim's headless Chromium needs real WebGL for the | |
| # heavier scenes (`apt`). Software X via xvfb is not enough. | |
| env: | |
| DISPLAY: ":0" | |
| run: uv run pytest --cov=dimos/ --junitxml=junit.xml -m self_hosted_large | |
| - name: Re-run the failing tests with maximum verbosity | |
| if: failure() | |
| env: | |
| COLOR: yes | |
| DISPLAY: ":0" | |
| run: >- # `exit 1` makes sure that the job remains red with flaky runs | |
| uv run pytest --no-cov -vvvvv --lf -m self_hosted_large && exit 1 | |
| shell: bash | |
| - name: Turn coverage into xml | |
| run: uv run python -m coverage xml | |
| - name: Upload coverage | |
| uses: codecov/codecov-action@v7 | |
| with: | |
| disable_search: true | |
| fail_ci_if_error: true | |
| files: ./coverage.xml | |
| flags: SelfHosted-Large | |
| use_oidc: true | |
| # Use the pinned CLI from the "Install codecov CLI" step (no pip | |
| # on self-hosted runners, and the default cli.codecov.io download | |
| # has a broken GPG key fetch). See codecov/codecov-action#1955. | |
| binary: .codecov-venv/bin/codecovcli | |
| - name: Upload test results to Codecov | |
| if: ${{ !cancelled() }} | |
| uses: codecov/codecov-action@v7 | |
| with: | |
| report_type: test_results | |
| use_oidc: true | |
| # See the "Upload coverage" step above. | |
| binary: .codecov-venv/bin/codecovcli | |
| - name: Check disk space | |
| if: failure() | |
| run: | | |
| df -h | |
| # Cross-job fail-fast: GitHub Actions only fail-fasts within a matrix, | |
| # not across sibling jobs. This watcher fires the moment `tests` fails | |
| # and cancels the whole workflow run. | |
| fail-fast: | |
| if: failure() | |
| needs: [tests] | |
| runs-on: ubuntu-latest | |
| permissions: | |
| actions: write # For `gh run cancel` | |
| steps: | |
| - name: Cancel workflow run | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: gh run cancel ${{ github.run_id }} --repo ${{ github.repository }} | |
| ci-complete: # This is used for branch protection. | |
| if: always() | |
| needs: | |
| - lint | |
| - rust | |
| - md-babel | |
| - docs-validate | |
| - tests | |
| - cmu-nav-natives | |
| - self-hosted-tests | |
| #- self-hosted-large-tests | |
| runs-on: ubuntu-latest | |
| permissions: | |
| id-token: write # For codecov-action's OIDC notify | |
| steps: | |
| - name: Decide whether the needed jobs succeeded or failed | |
| uses: re-actors/alls-green@release/v1 | |
| with: | |
| allowed-skips: cmu-nav-natives, self-hosted-tests, self-hosted-large-tests | |
| jobs: ${{ toJSON(needs) }} | |
| - name: Trigger Codecov notifications | |
| uses: codecov/codecov-action@v7 | |
| with: | |
| run_command: send-notifications | |
| use_oidc: true | |
| fail_ci_if_error: true | |
| # Install the CLI from PyPI (pinned) instead of cli.codecov.io: | |
| # the GPG key for the default download path is served from a dead | |
| # keybase.io URL, failing every upload. See | |
| # codecov/codecov-action#1955 and #2398. | |
| use_pypi: true | |
| version: "11.2.8" |