ci.yml

name: ci
on:
  push:
    branches: [main]
  pull_request:

concurrency:
  # Cancels runs from previous pushes in a PR.
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
  cancel-in-progress: true

permissions: {}

env:
  # Disable default package installs. Each job should explicitly install a group.
  UV_NO_SYNC: "1"

jobs:
  lint:
    timeout-minutes: 10
    runs-on: ubuntu-latest
    permissions:
      contents: read  # For checkout

    steps:
      - name: Checkout
        uses: actions/checkout@v6
      - name: Install uv
        uses: astral-sh/setup-uv@v6
        with:
          enable-cache: true
      - name: Install lint dependencies
        run: uv sync --only-group lint --frozen
      - name: Mypy
        run: uv run mypy
      - name: Run pre-commit
        uses: pre-commit/action@v3.0.1

  md-babel:
    timeout-minutes: 10
    runs-on: ubuntu-latest
    permissions:
      contents: read  # For checkout

    steps:
      - uses: actions/checkout@v6
        with:
          lfs: true
      # Docs decode JPEG from SQLite via PyTurboJPEG; pyaudio needs portaudio.
      - name: Install system dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y libturbojpeg portaudio19-dev
      - name: Install uv
        uses: astral-sh/setup-uv@v6
        with:
          enable-cache: true
      - uses: actions/setup-node@v4
        with:
          node-version: 'lts/*'
      - name: Install Python dependencies
        run: uv sync --group tests --frozen
      - name: Execute documentation code blocks
        run: ./bin/run-doc-codeblocks --ci --no-cache

  tests:
    timeout-minutes: 20
    strategy:
      matrix:
        pyver: ['3.10', '3.11', '3.12', '3.13', '3.14']
        os: [ubuntu]
        experimental: [false]
        include:
          - os: ubuntu
            pyver: "3.14t"
            experimental: false
      fail-fast: true
    runs-on: ${{ matrix.os }}-latest
    continue-on-error: ${{ matrix.experimental }}
    permissions:
      contents: read  # For checkout
      id-token: write  # For codecov-action's OIDC upload

    steps:
      - name: Checkout
        uses: actions/checkout@v6
      - name: Install uv
        uses: astral-sh/setup-uv@v6
        with:
          enable-cache: true
      - name: Setup Python
        uses: actions/setup-python@v6
        with:
          allow-prereleases: true
          python-version: ${{ matrix.pyver }}
      - name: Install dependency for pyaudio (Ubuntu)
        if: matrix.os == 'ubuntu'
        run: |
          sudo apt-get update
          sudo apt-get install -y portaudio19-dev
      - name: Install dependency for pyaudio (macOS)
        if: matrix.os == 'macos'
        run: brew install portaudio
      - name: Remove git LFS to avoid accidental large downloads
        run: sudo rm -f /usr/bin/git-lfs
      - name: Set PYTHON_GIL=0 for free-threading builds
        if: ${{ endsWith(matrix.pyver, 't') }}
        run: echo "PYTHON_GIL=0" >> $GITHUB_ENV
      - name: Install dependencies
        run: uv sync --group tests --frozen
      - name: Run tests
        run: uv run pytest --numprocesses=3 --cov=dimos/ --junitxml=junit.xml -m 'not (tool or self_hosted or mujoco or dimsim)'
      - name: Re-run the failing tests with maximum verbosity
        if: failure()
        env:
          COLOR: yes
        run: >-  # `exit 1` makes sure that the job remains red with flaky runs
          uv run pytest --no-cov -vvvvv --lf  -m 'not (tool or self_hosted or mujoco or dimsim)' && exit 1
        shell: bash
      - name: Turn coverage into xml
        run: uv run python -m coverage xml
      - name: Upload coverage
        uses: codecov/codecov-action@v6
        with:
          disable_search: true
          fail_ci_if_error: true
          files: ./coverage.xml
          flags: OS-${{ matrix.os }},Py-${{ matrix.pyver }}
          use_oidc: true
      - name: Upload test results to Codecov
        if: ${{ !cancelled() }}
        uses: codecov/codecov-action@v6
        with:
          report_type: test_results
          use_oidc: true

  self-hosted-tests:
    # Skip on PRs from forks which would expose the self-hosted runner to untrusted code from external contributors.
    if: |
      github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository
    env:
      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
      ALIBABA_API_KEY: ${{ secrets.ALIBABA_API_KEY }}
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        include:
          - runner: [self-hosted, Linux]
            # GitHub Actions only honours `container:` on Linux runners.
            container:
              image: ghcr.io/dimensionalos/ros-dev:dev
            markers: "self_hosted or skipif_no_ros"
            experimental: false
          - runner: [self-hosted, macos, arm64]
            container: null  # run on host — `container:` is Linux-only
            markers: "self_hosted"
            experimental: true
    runs-on: ${{ matrix.runner }}
    continue-on-error: ${{ matrix.experimental }}
    permissions:
      contents: read  # For checkout
      packages: read  # For pulling the ros-dev container from ghcr.io
      id-token: write  # For codecov-action's OIDC upload
    container: ${{ matrix.container }}

    steps:
      - uses: actions/checkout@v5
        with:
          clean: false
          # If we ever allow external PRs on custom runner, persisting credentials
          # could be abused by attackers.
          persist-credentials: false
      - name: Fix permissions
        run: |
          git config --global --add safe.directory '*'
          git clean -ffdx
      - name: Install uv
        uses: astral-sh/setup-uv@v6
        with:
          enable-cache: true
      - name: Install dependencies
        run: uv sync --group tests-self-hosted --frozen
      - name: Build C++ extensions in-place
        run: uv run python setup.py build_ext --inplace
      - name: Source ROS environment
        # The uv venv is sealed (include-system-site-packages = false), so
        # `import rclpy` / `ament_index_python` would fail. Sourcing the ROS
        # setup script and exporting PYTHONPATH/AMENT_PREFIX_PATH/etc into
        # GITHUB_ENV makes them importable from `uv run`.
        if: contains(toJSON(matrix.runner), 'Linux')
        shell: bash
        run: |
          source /opt/ros/humble/setup.bash
          {
            echo "PYTHONPATH=$PYTHONPATH"
            echo "AMENT_PREFIX_PATH=$AMENT_PREFIX_PATH"
            echo "CMAKE_PREFIX_PATH=$CMAKE_PREFIX_PATH"
            echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH"
            echo "ROS_DISTRO=$ROS_DISTRO"
            echo "ROS_VERSION=$ROS_VERSION"
            echo "ROS_PYTHON_VERSION=$ROS_PYTHON_VERSION"
          } >> "$GITHUB_ENV"
      - name: Run tests
        run: uv run pytest --cov=dimos/ --junitxml=junit.xml -m '(${{ matrix.markers }}) and not (tool or mujoco)'
      - name: Re-run the failing tests with maximum verbosity
        if: failure()
        env:
          COLOR: yes
        run: >-  # `exit 1` makes sure that the job remains red with flaky runs
          uv run pytest --no-cov -vvvvv --lf  -m '(${{ matrix.markers }}) and not (tool or mujoco)' && exit 1
        shell: bash
      - name: Turn coverage into xml
        run: uv run python -m coverage xml
      - name: Upload coverage
        uses: codecov/codecov-action@v6
        with:
          disable_search: true
          fail_ci_if_error: true
          files: ./coverage.xml
          flags: SelfHosted-${{ matrix.runner[1] }}
          use_oidc: true
      - name: Upload test results to Codecov
        if: ${{ !cancelled() }}
        uses: codecov/codecov-action@v6
        with:
          report_type: test_results
          use_oidc: true
      - name: Check disk space
        if: failure()
        run: |
          df -h

  # Cross-job fail-fast: GitHub Actions only fail-fasts within a matrix,
  # not across sibling jobs. This watcher fires the moment `tests` fails
  # and cancels the whole workflow run.
  fail-fast:
    if: failure()
    needs: [tests]
    runs-on: ubuntu-latest
    permissions:
      actions: write  # For `gh run cancel`
    steps:
      - name: Cancel workflow run
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: gh run cancel ${{ github.run_id }} --repo ${{ github.repository }}

  ci-complete:  # This is used for branch protection.
    if: always()

    needs:
    - lint
    - md-babel
    - tests
    - self-hosted-tests

    runs-on: ubuntu-latest
    permissions:
      id-token: write  # For codecov-action's OIDC notify

    steps:
    - name: Decide whether the needed jobs succeeded or failed
      uses: re-actors/alls-green@release/v1
      with:
        allowed-skips: self-hosted-tests
        jobs: ${{ toJSON(needs) }}
    - name: Trigger Codecov notifications
      uses: codecov/codecov-action@v6
      with:
        run_command: send-notifications
        use_oidc: true
        fail_ci_if_error: true