cpp_server_build_test_release.yml

name: C++ Server Build, Test, and Release 🚀

on:
  push:
    branches: ["main"]
    tags:
      - v*
  pull_request:
  merge_group:
  workflow_dispatch:
    inputs:
      enable_signing:
        description: 'Enable MSI signing with SignPath (for testing)'
        required: false
        default: false
        type: boolean

permissions:
  contents: write
  actions: read  # Required for SignPath to read workflow/job details

env:
  LEMONADE_DISABLE_SYSTEMD_JOURNAL: "1"

jobs:
  # ========================================================================
  # BUILD JOBS - Run on rai-160-sdk workers
  # ========================================================================

  build-lemonade-server-installer:
    name: Build Lemonade Server Installer
    runs-on: windows-latest
    outputs:
      unsigned-artifact-id: ${{ steps.upload-unsigned-msi.outputs.artifact-id }}
    steps:
      - uses: actions/checkout@v4
        with:
          clean: true
          fetch-depth: 0

      - name: Install CMake if not available
        shell: PowerShell
        run: |
          # Check if CMake is already installed
          $cmakeInstalled = Get-Command cmake -ErrorAction SilentlyContinue

          if (-not $cmakeInstalled) {
              Write-Host "CMake not found, installing..." -ForegroundColor Yellow

              # Download CMake installer
              $cmakeVersion = "3.28.1"
              $cmakeUrl = "https://github.com/Kitware/CMake/releases/download/v$cmakeVersion/cmake-$cmakeVersion-windows-x86_64.msi"
              $cmakeInstaller = "cmake-installer.msi"

              Invoke-WebRequest -Uri $cmakeUrl -OutFile $cmakeInstaller

              # Install CMake silently
              Start-Process msiexec.exe -ArgumentList "/i $cmakeInstaller /quiet /norestart" -Wait

              # Add CMake to PATH for this session AND future steps
              $cmakePath = "C:\Program Files\CMake\bin"
              $env:PATH = "$cmakePath;$env:PATH"

              # Persist to GITHUB_PATH for future steps
              echo $cmakePath >> $env:GITHUB_PATH

              # Verify installation
              cmake --version
              if ($LASTEXITCODE -ne 0) {
                  Write-Host "ERROR: CMake installation failed!" -ForegroundColor Red
                  exit 1
              }

              Write-Host "CMake installed successfully and added to PATH!" -ForegroundColor Green
          } else {
              Write-Host "CMake is already installed:" -ForegroundColor Green
              cmake --version
          }

      - name: Install WiX Toolset 5.0.2 (CLI)
        shell: PowerShell
        run: |
          $ErrorActionPreference = "Stop"

          Write-Host "Downloading WiX Toolset 5.0.2 CLI..." -ForegroundColor Cyan
          $wixUri = "https://github.com/wixtoolset/wix/releases/download/v5.0.2/wix-cli-x64.msi"
          $msiPath = "$env:RUNNER_TEMP\wix-cli-x64.msi"

          Invoke-WebRequest -UseBasicParsing -Uri $wixUri -OutFile $msiPath

          Write-Host "Installing WiX Toolset 5.0.2 CLI..." -ForegroundColor Cyan
          $p = Start-Process "msiexec.exe" -ArgumentList @("/i", "`"$msiPath`"", "/qn", "/norestart") -PassThru -Wait
          if ($p.ExitCode -ne 0) {
            Write-Host "WiX installer exited with code $($p.ExitCode)" -ForegroundColor Red
            exit $p.ExitCode
          }

      - name: Verify WiX installation
        shell: PowerShell
        run: |
          $ErrorActionPreference = "Stop"

          # WiX CLI MSI does not always add itself to PATH in non-interactive installs,
          # so we locate it explicitly and then update PATH for subsequent steps.
          $wixDirs = @(
            "C:\Program Files\WiX Toolset v5.0\bin",
            "C:\Program Files (x86)\WiX Toolset v5.0\bin"
          )

          $wixExe = $null
          foreach ($dir in $wixDirs) {
            if (Test-Path (Join-Path $dir "wix.exe")) {
              $wixExe = Join-Path $dir "wix.exe"
              break
            }
          }

          if (-not $wixExe) {
            Write-Host "ERROR: wix.exe not found after installation." -ForegroundColor Red
            Get-ChildItem -Recurse "C:\Program Files" -Filter wix.exe -ErrorAction SilentlyContinue | Select-Object -First 20 | Format-List FullName
            exit 1
          }

          $wixDir = Split-Path $wixExe -Parent

          # Persist wix.exe directory to PATH for all subsequent steps
          "$wixDir" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append

          Write-Host "Using WiX from: $wixExe" -ForegroundColor Green
          & $wixExe --version

      - name: Build C++ Server with CMake
        shell: PowerShell
        run: |
          $ErrorActionPreference = "Stop"

          Write-Host "Building lemonade-router and lemonade-server..." -ForegroundColor Cyan

          # Create build directory
          if (Test-Path "build") {
              Remove-Item -Recurse -Force "build"
          }

          # Configure with preset
          cmake --preset windows
          if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }

          # Build
          cmake --build build --config Release
          if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }

          # Verify binaries exist
          if (-not (Test-Path "build\Release\lemonade-router.exe")) {
              Write-Host "ERROR: lemonade-router.exe not found!" -ForegroundColor Red
              exit 1
          }

          if (-not (Test-Path "build\Release\lemonade-server.exe")) {
              Write-Host "ERROR: lemonade-server.exe not found!" -ForegroundColor Red
              exit 1
          }

          if (-not (Test-Path "build\Release\lemonade-tray.exe")) {
              Write-Host "ERROR: lemonade-tray.exe not found!" -ForegroundColor Red
              exit 1
          }

          Write-Host "C++ build successful!" -ForegroundColor Green

      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: '20'

      - name: Build Electron App
        shell: PowerShell
        run: |
          $ErrorActionPreference = "Stop"

          Write-Host "Building Electron app via CMake target..." -ForegroundColor Cyan

          # Build using CMake electron-app target (single source of truth)
          cmake --build --preset windows --target electron-app
          if ($LASTEXITCODE -ne 0) {
              Write-Host "ERROR: Electron app build failed!" -ForegroundColor Red
              exit $LASTEXITCODE
          }

          # Verify the build output exists
          if (-not (Test-Path "build\app\win-unpacked\Lemonade.exe")) {
              Write-Host "ERROR: Electron app executable not found!" -ForegroundColor Red
              exit 1
          }

          Write-Host "Electron app build successful!" -ForegroundColor Green

      - name: Build Web App
        shell: PowerShell
        run: |
          $ErrorActionPreference = "Stop"

          Write-Host "Building Web app..." -ForegroundColor Cyan

          cd src\web-app

          # Install dependencies
          Write-Host "Installing npm dependencies..." -ForegroundColor Yellow
          npm install
          if ($LASTEXITCODE -ne 0) {
              Write-Host "ERROR: npm install failed!" -ForegroundColor Red
              exit $LASTEXITCODE
          }

          # Build the Web app
          Write-Host "Building Web app..." -ForegroundColor Yellow
          npm run build
          if ($LASTEXITCODE -ne 0) {
              Write-Host "ERROR: Web app build failed!" -ForegroundColor Red
              exit $LASTEXITCODE
          }

          # Verify the build output exists
          if (-not (Test-Path "dist\renderer\index.html")) {
              Write-Host "ERROR: Web app build output not found!" -ForegroundColor Red
              exit 1
          }

          Write-Host "Web app build successful!" -ForegroundColor Green

          # Copy web app output to the location expected by CMake/WiX installer
          Write-Host "Copying web app to build\resources\web-app..." -ForegroundColor Yellow
          $targetDir = "..\..\build\resources\web-app"
          if (-not (Test-Path "..\..\build\resources")) {
              New-Item -ItemType Directory -Force -Path "..\..\build\resources" | Out-Null
          }
          if (Test-Path $targetDir) {
              Remove-Item -Recurse -Force $targetDir
          }
          Copy-Item -Path "dist\renderer" -Destination $targetDir -Recurse -Force

          # Verify the copy succeeded
          if (-not (Test-Path "..\..\build\resources\web-app\index.html")) {
              Write-Host "ERROR: Failed to copy web app to build directory!" -ForegroundColor Red
              exit 1
          }

          Write-Host "Web app copied to build directory successfully!" -ForegroundColor Green

      - name: Build the Lemonade Server Installer
        shell: PowerShell
        run: |
          $ErrorActionPreference = "Stop"

          # Run the build installer script
          .\src\cpp\build_installer.ps1

          # Verify installers were created
          if (-not (Test-Path "lemonade-server-minimal.msi")) {
              Write-Host "ERROR: Minimal installer not created!" -ForegroundColor Red
              exit 1
          }

          if (-not (Test-Path "lemonade.msi")) {
              Write-Host "ERROR: Full installer not created!" -ForegroundColor Red
              exit 1
          }

          Write-Host "Installers created successfully!" -ForegroundColor Green

      - name: Upload Lemonade Server Installers
        id: upload-unsigned-msi
        uses: actions/upload-artifact@v4
        with:
          name: Lemonade_Server_MSI
          path: |
            lemonade-server-minimal.msi
            lemonade.msi
          retention-days: 7

  sign-msi-installers:
    name: Sign MSI Installers with SignPath
    runs-on: windows-latest
    needs: build-lemonade-server-installer
    # Sign on tag pushes (releases) or when manually enabled via workflow_dispatch
    if: startsWith(github.ref, 'refs/tags/v') || inputs.enable_signing == true
    steps:
      - name: Sign MSI Installers with SignPath
        id: sign-msi
        uses: signpath/github-action-submit-signing-request@v2
        with:
          api-token: '${{ secrets.SIGNPATH_API_TOKEN }}'
          organization-id: '8103545b-7814-4edc-86d6-a91dc2a2291b'
          project-slug: 'lemonade'
          signing-policy-slug: 'release-signing'
          github-artifact-id: '${{ needs.build-lemonade-server-installer.outputs.unsigned-artifact-id }}'
          wait-for-completion: true
          wait-for-completion-timeout-in-seconds: 3600
          output-artifact-directory: 'signed-msi'
          parameters: |
            version: "${{ startsWith(github.ref, 'refs/tags/') && github.ref_name || 'test' }}"

      - name: Verify Signed MSI Files
        shell: PowerShell
        run: |
          Write-Host "Verifying signed MSI files..." -ForegroundColor Cyan

          if (-not (Test-Path "signed-msi\lemonade-server-minimal.msi")) {
              Write-Host "ERROR: Signed lemonade-server-minimal.msi not found!" -ForegroundColor Red
              exit 1
          }

          if (-not (Test-Path "signed-msi\lemonade.msi")) {
              Write-Host "ERROR: Signed lemonade.msi not found!" -ForegroundColor Red
              exit 1
          }

          Write-Host "Signed MSI files verified!" -ForegroundColor Green
          Get-ChildItem -Path "signed-msi" -Recurse | Format-Table Name, Length

      - name: Upload Signed MSI Installers
        uses: actions/upload-artifact@v4
        with:
          name: Lemonade_Server_MSI_Signed
          path: |
            signed-msi/lemonade-server-minimal.msi
            signed-msi/lemonade.msi
          retention-days: 7

  build-lemonade-deb:
    name: Build Lemonade .deb Package
    runs-on: ubuntu-latest
    outputs:
      version: ${{ steps.get_version.outputs.version }}
    steps:
      - uses: actions/checkout@v4
        with:
          clean: true
          fetch-depth: 0

      - name: Get version from CMakeLists.txt
        id: get_version
        uses: ./.github/actions/get-version

      - name: Build Linux .deb package
        uses: ./.github/actions/build-linux-deb

      - name: Upload .deb package
        uses: actions/upload-artifact@v4
        with:
          name: lemonade-deb
          path: build/lemonade-server_${{ env.LEMONADE_VERSION }}_amd64.deb
          retention-days: 7

  build-lemonade-rpm:
    name: Build Lemonade .rpm Package
    runs-on: ubuntu-latest
    container:
      image: fedora:latest
    outputs:
      version: ${{ steps.get_version.outputs.version }}
    steps:
      - uses: actions/checkout@v4
        with:
          clean: true
          fetch-depth: 0

      - name: Install RPM packaging tools
        shell: bash
        run: |
          set -e
          dnf install -y rpm-build

      - name: Get version from CMakeLists.txt
        id: get_version
        uses: ./.github/actions/get-version

      - name: Build Linux .rpm package
        shell: bash
        run: |
          set -e

          echo "Running setup.sh to configure build environment..."
          bash setup.sh

          echo "Building lemonade-router and lemonade-server for Fedora..."
          cmake --build --preset default

          RPM_FILE="lemonade-server-${LEMONADE_VERSION}.x86_64.rpm"

          cd build

          echo "Creating .rpm package with CPack..."
          cpack -G RPM -V

          if [ ! -f "$RPM_FILE" ]; then
              echo "ERROR: .rpm package not created!"
              echo "Contents of build directory:"
              ls -lR .
              exit 1
          fi

          echo "Package information:"
          rpm -qip "$RPM_FILE"

      - name: Upload .rpm package
        uses: actions/upload-artifact@v4
        with:
          name: lemonade-rpm
          path: build/lemonade-server-${{ env.LEMONADE_VERSION }}.x86_64.rpm
          retention-days: 7

  build-lemonade-macos-dmg:
    name: Build Lemonade macOS .dmg (with Electron App)
    runs-on: macos-latest
    outputs:
      version: ${{ steps.get_version.outputs.version }}
      has_signing: ${{ steps.check_signing.outputs.has_signing }}
    steps:
      - uses: actions/checkout@v4
        with:
          clean: true
          fetch-depth: 0

      - name: Get version from CMakeLists.txt
        id: get_version
        uses: ./.github/actions/get-version

      - name: Check signing secrets
        id: check_signing
        shell: bash
        env:
          APP_CONNECT_KEY: ${{ secrets.MACOS_APP_CONNECT_KEY_GERAMY }}
        run: |
          if [ -n "$APP_CONNECT_KEY" ]; then
            echo "has_signing=true" >> $GITHUB_OUTPUT
            echo "Signing secrets available - will build signed .pkg"
          else
            echo "has_signing=false" >> $GITHUB_OUTPUT
            echo "No signing secrets - will build and test locally"
          fi

      - name: Setup macOS Keychain
        if: steps.check_signing.outputs.has_signing == 'true'
        uses: ./.github/actions/setup-macos-keychain
        with:
          dev-signing-key: ${{ secrets.MACOS_DEV_SIGNING_IDENTITY_KEY }}
          inst-signing-key: ${{ secrets.MACOS_INST_SIGNING_IDENTITY_KEY }}
          certificate-password: ${{ secrets.APPLE_CERTIFICATE_PASSWORD }}
          api-key: ${{ secrets.MACOS_APP_CONNECT_KEY_GERAMY }}
          api-key-id: '3WFZZ8F948'
          api-issuer-id: '2e545619-8206-4d14-9ba9-ef23eff841b2'

      - name: Build macOS .dmg
        uses: ./.github/actions/build-macos-dmg
        with:
          include-electron: 'true'
          skip-packaging: ${{ steps.check_signing.outputs.has_signing != 'true' }}

      - name: Upload .pkg package
        if: steps.check_signing.outputs.has_signing == 'true'
        uses: actions/upload-artifact@v4
        with:
          name: lemonade-macos-pkg
          path: build/*.pkg
          retention-days: 7

      # ---- Unsigned local install + test path ----
      - name: Install binaries locally (unsigned path)
        if: steps.check_signing.outputs.has_signing != 'true'
        shell: bash
        run: |
          set -e
          echo "Installing binaries locally (no signing)..."

          # Copy binaries to /usr/local/bin
          sudo cp build/lemonade-router /usr/local/bin/
          sudo cp build/lemonade-server /usr/local/bin/
          sudo chmod 755 /usr/local/bin/lemonade-router
          sudo chmod 755 /usr/local/bin/lemonade-server

          # Copy resources
          sudo mkdir -p "/Library/Application Support/Lemonade/resources"
          if [ -d "build/resources" ]; then
            sudo cp -R build/resources/* "/Library/Application Support/Lemonade/resources/" 2>/dev/null || true
          fi

          # Run the postinst script for directory setup
          echo "Running post-install script..."
          sudo bash src/cpp/postinst-full-mac "" "/"

          # Verify installation
          echo "Verifying installation..."
          /usr/local/bin/lemonade-server --version
          /usr/local/bin/lemonade-router --version

          echo "Local installation complete!"

      - name: Stop services after local install (unsigned path)
        if: steps.check_signing.outputs.has_signing != 'true'
        shell: bash
        run: |
          echo "Stopping services started by postinst..."
          sudo launchctl bootout system/com.lemonade.server 2>/dev/null || true
          CURRENT_UID=$(id -u)
          launchctl bootout gui/"$CURRENT_UID"/com.lemonade.tray 2>/dev/null || true
          sleep 2

          # Clean up PID and lock files
          sudo rm -f /tmp/lemonade-router.pid
          sudo rm -f /tmp/lemonade_Router.lock
          sudo rm -f /tmp/lemonade_Server.lock
          sudo rm -f /tmp/lemonade_Tray.lock
          sudo rm -f /tmp/lemonade*.pid /tmp/lemonade*.lock

          echo "Services stopped and cleaned up."

      - name: Setup Python and virtual environment (unsigned path)
        if: steps.check_signing.outputs.has_signing != 'true'
        uses: ./.github/actions/setup-venv
        with:
          venv-name: '.venv'
          python-version: '3.10'
          requirements-file: 'test/requirements.txt'

      - name: Run CLI tests (unsigned path)
        if: steps.check_signing.outputs.has_signing != 'true'
        shell: bash
        env:
          LEMONADE_CI_MODE: "True"
          PYTHONIOENCODING: utf-8
          GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
          HF_HOME: ${{ github.workspace }}/hf-cache
        run: |
          set -e
          echo "Running CLI tests..."
          .venv/bin/python test/server_cli.py --server-binary /usr/local/bin/lemonade-server
          .venv/bin/python test/server_cli.py --server-binary /usr/local/bin/lemonade-server --ephemeral
          .venv/bin/python test/server_cli.py --server-binary /usr/local/bin/lemonade-server --listen-all
          .venv/bin/python test/server_cli.py --server-binary /usr/local/bin/lemonade-server --api-key
          echo "CLI tests PASSED!"

      - name: Run endpoint tests (unsigned path)
        if: steps.check_signing.outputs.has_signing != 'true'
        shell: bash
        env:
          LEMONADE_CI_MODE: "True"
          PYTHONIOENCODING: utf-8
          GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
          HF_HOME: ${{ github.workspace }}/hf-cache
        run: |
          set -e
          echo "Running endpoint tests..."
          .venv/bin/python test/server_endpoints.py --server-binary /usr/local/bin/lemonade-server
          .venv/bin/python test/server_endpoints.py --server-binary /usr/local/bin/lemonade-server --server-per-test
          echo "Endpoint tests PASSED!"

      - name: Run Ollama API tests (unsigned path)
        if: steps.check_signing.outputs.has_signing != 'true'
        shell: bash
        env:
          LEMONADE_CI_MODE: "True"
          PYTHONIOENCODING: utf-8
          GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
          HF_HOME: ${{ github.workspace }}/hf-cache
        run: |
          set -e
          echo "Running Ollama API tests..."
          .venv/bin/python test/test_ollama.py --server-binary /usr/local/bin/lemonade-server
          echo "Ollama API tests PASSED!"

      - name: Cleanup keychain
        if: always() && steps.check_signing.outputs.has_signing == 'true'
        shell: bash
        run: |
          if [ -n "$SIGNING_KEYCHAIN_PATH" ] && [ -f "$SIGNING_KEYCHAIN_PATH" ]; then
            echo "Cleaning up temporary keychain..."
            security delete-keychain "$SIGNING_KEYCHAIN_PATH" 2>/dev/null || true
          fi

  build-lemonade-appimage:
    name: Build Lemonade AppImage
    runs-on: ubuntu-latest
    outputs:
      version: ${{ steps.get_version.outputs.version }}
    steps:
      - uses: actions/checkout@v4
        with:
          clean: true
          fetch-depth: 0

      - name: Get version from CMakeLists.txt
        id: get_version
        uses: ./.github/actions/get-version

      - name: Run setup.sh to configure environment
        shell: bash
        run: |
          set -e
          echo "Running setup.sh to configure build environment..."
          bash setup.sh
          echo "Build environment configured successfully!"

      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version: '20'

      - name: Build AppImage
        shell: bash
        run: |
          set -e

          echo "Building Lemonade AppImage..."
          cmake --build --preset default --target appimage

          # Verify AppImage was created
          APPIMAGE_FILE="build/app-appimage/Lemonade-${LEMONADE_VERSION}-x86_64.AppImage"
          if [ ! -f "$APPIMAGE_FILE" ]; then
              echo "ERROR: AppImage not created!"
              echo "Contents of build/app-appimage directory:"
              ls -lh build/app-appimage/ || echo "Directory does not exist"
              exit 1
          fi

          echo "AppImage created successfully!"
          ls -lh "$APPIMAGE_FILE"

          # Verify it's executable
          chmod +x "$APPIMAGE_FILE"
          "$APPIMAGE_FILE" --appimage-help | head -5

      - name: Upload AppImage artifact
        uses: actions/upload-artifact@v4
        with:
          name: lemonade-appimage
          path: build/app-appimage/Lemonade-${{ env.LEMONADE_VERSION }}-x86_64.AppImage
          retention-days: 7


  # ========================================================================
  # TEST JOBS - Inference tests on self-hosted runners
  # ========================================================================

  test-exe-inference:
    name: Test .exe - ${{ matrix.name }}
    runs-on: ${{ matrix.runner }}
    needs: build-lemonade-server-installer
    # Skip inference tests when signing is enabled (tag pushes or manual workflow_dispatch)
    if: ${{ !startsWith(github.ref, 'refs/tags/') && inputs.enable_signing != true }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - name: llamacpp
            script: server_llm.py
            extra_args: "--wrapped-server llamacpp"
            backends: "vulkan rocm"
            runner: [rai300_400, Windows]
          - name: ryzenai
            script: server_llm.py
            extra_args: "--wrapped-server ryzenai"
            backends: "cpu hybrid npu"
            runner: [rai300_400, Windows]
          - name: flm
            script: server_llm.py
            extra_args: "--wrapped-server flm"
            backends: "npu"
            runner: [rai300_400, Windows]
          - name: whisper
            script: server_whisper.py
            extra_args: "--wrapped-server whispercpp"
            backends: "cpu npu"
            runner: [rai300_400, Windows]
          - name: flm-whisper
            script: server_whisper.py
            extra_args: "--wrapped-server flm"
            backends: "npu"
            runner: [rai300_400, Windows]
          - name: stable-diffusion
            script: server_sd.py
            extra_args: ""
            backends: "cpu"
            runner: [rai300_400, Windows]
          - name: text-to-speech
            script: server_tts.py
            extra_args: ""
            backends: ""
            runner: [rai300_400, Windows]
          - name: stable-diffusion (stx-halo)
            script: server_sd.py
            extra_args: ""
            backends: "rocm"
            runner: [stx-halo, Windows]
    env:
      LEMONADE_CI_MODE: "True"
      LEMONADE_CACHE_DIR: ".\\ci-cache"
      PYTHONIOENCODING: utf-8
      GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
    steps:
      - uses: actions/checkout@v4

      - name: Cleanup processes
        uses: ./.github/actions/cleanup-processes-windows

      - name: Set environment variables
        shell: PowerShell
        run: |
          $cwd = (Get-Item .\).FullName
          echo "HF_HOME=$cwd\hf-cache" >> $Env:GITHUB_ENV
          echo "LEMONADE_INSTALL_PATH=$cwd\lemonade_server_install" >> $Env:GITHUB_ENV

      - name: Install and Verify Lemonade Server
        uses: ./.github/actions/install-lemonade-server-msi
        with:
          install-path: ${{ env.LEMONADE_INSTALL_PATH }}

      - name: Setup Python and virtual environment
        uses: ./.github/actions/setup-venv
        with:
          venv-name: '.venv'
          python-version: '3.10'
          requirements-file: 'test/requirements.txt'

      - name: Run tests
        shell: PowerShell
        env:
          HF_HOME: ${{ env.HF_HOME }}
        run: |
          $ErrorActionPreference = "Stop"
          $venvPython = ".\.venv\Scripts\python.exe"
          $serverExe = Join-Path $env:LEMONADE_INSTALL_PATH "bin\lemonade-server.exe"
          $extraArgs = "${{ matrix.extra_args }}" -split " " | Where-Object { $_ }
          $backends = "${{ matrix.backends }}" -split " " | Where-Object { $_ }

          if ($backends.Count -eq 0) {
            Write-Host "Running test/${{ matrix.script }} ${{ matrix.extra_args }}" -ForegroundColor Cyan
            & $venvPython test/${{ matrix.script }} @extraArgs --server-binary $serverExe
            if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
          } else {
            foreach ($backend in $backends) {
              Write-Host "Running test/${{ matrix.script }} ${{ matrix.extra_args }} --backend $backend" -ForegroundColor Cyan
              & $venvPython test/${{ matrix.script }} @extraArgs --backend $backend --server-binary $serverExe
              if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
            }
          }

      - name: Cleanup
        if: always()
        uses: ./.github/actions/cleanup-processes-windows

  test-deb-inference:
    name: Test .deb - ${{ matrix.name }}
    runs-on: [rai300_400, Linux]
    needs: build-lemonade-deb
    # Skip inference tests when signing is enabled (tag pushes or manual workflow_dispatch)
    if: ${{ !startsWith(github.ref, 'refs/tags/') && inputs.enable_signing != true }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - name: llamacpp
            script: server_llm.py
            extra_args: "--wrapped-server llamacpp"
            backends: "vulkan rocm"
          - name: stable-diffusion
            script: server_sd.py
            extra_args: ""
            backends: "cpu rocm"
          - name: whisper
            script: server_whisper.py
            extra_args: "--wrapped-server whispercpp"
            backends: "cpu vulkan"
          - name: text-to-speech
            script: server_tts.py
            extra_args: ""
            backends: ""
    env:
      LEMONADE_CI_MODE: "True"
      PYTHONIOENCODING: utf-8
      GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
      LEMONADE_VERSION: ${{ needs.build-lemonade-deb.outputs.version }}
    steps:
      - uses: actions/checkout@v4

      - name: Cleanup processes
        uses: ./.github/actions/cleanup-processes-linux

      - name: Set HF_HOME environment variable
        run: echo "HF_HOME=$PWD/hf-cache" >> $GITHUB_ENV

      - name: Install Lemonade (.deb)
        uses: ./.github/actions/install-lemonade-deb
        with:
          version: ${{ env.LEMONADE_VERSION }}

      - name: Setup Python and virtual environment
        uses: ./.github/actions/setup-venv
        with:
          venv-name: '.venv'
          python-version: '3.10'
          requirements-file: 'test/requirements.txt'

      - name: Run tests
        env:
          HF_HOME: ${{ env.HF_HOME }}
        run: |
          set -e
          if [ -z "${{ matrix.backends }}" ]; then
            echo "Running test/${{ matrix.script }} ${{ matrix.extra_args }}"
            .venv/bin/python test/${{ matrix.script }} ${{ matrix.extra_args }} --server-binary lemonade-server
          else
            for backend in ${{ matrix.backends }}; do
              echo "Running test/${{ matrix.script }} ${{ matrix.extra_args }} --backend $backend"
              .venv/bin/python test/${{ matrix.script }} ${{ matrix.extra_args }} --backend $backend --server-binary lemonade-server
            done
          fi

      - name: Print server logs on failure
        if: failure()
        run: |
          # Resolve runtime dir: $XDG_RUNTIME_DIR/lemonade when available, else /tmp
          LEMON_RUNTIME_DIR="${XDG_RUNTIME_DIR:+${XDG_RUNTIME_DIR}/lemonade}"
          LEMON_RUNTIME_DIR="${LEMON_RUNTIME_DIR:-/tmp}"

          echo "=== Server Log (if exists) ==="
          LOG_FILE="${LEMON_RUNTIME_DIR}/lemonade-server.log"
          # Also check /tmp as a safety net when XDG was active but log went to fallback
          if [ ! -f "$LOG_FILE" ]; then LOG_FILE="/tmp/lemonade-server.log"; fi
          if [ -f "$LOG_FILE" ]; then
            echo "Last 100 lines of ${LOG_FILE}:"
            tail -100 "$LOG_FILE"
          else
            echo "Log file not found (checked ${LEMON_RUNTIME_DIR} and /tmp)"
          fi

          echo ""
          echo "=== Router PID file (if exists) ==="
          PID_FILE="${LEMON_RUNTIME_DIR}/lemonade-router.pid"
          if [ ! -f "$PID_FILE" ]; then PID_FILE="/tmp/lemonade-router.pid"; fi
          if [ -f "$PID_FILE" ]; then
            cat "$PID_FILE"
          else
            echo "PID file not found"
          fi

          echo ""
          echo "=== Lock file status ==="
          ls -la "${LEMON_RUNTIME_DIR}"/lemonade*.lock 2>/dev/null || true
          ls -la /tmp/lemonade*.lock 2>/dev/null || echo "No lock files found"

      - name: Cleanup
        if: always()
        uses: ./.github/actions/cleanup-processes-linux

  test-rpm-package:
    name: Test .rpm - Fedora
    runs-on: ubuntu-latest
    needs: build-lemonade-rpm
    container:
      image: fedora:latest
    env:
      LEMONADE_VERSION: ${{ needs.build-lemonade-rpm.outputs.version }}
    steps:
      - name: Download Lemonade .rpm Package
        uses: actions/download-artifact@v4
        with:
          name: lemonade-rpm
          path: .

      - name: Install and verify Lemonade (.rpm)
        shell: bash
        run: |
          set -e

          RPM_FILE="lemonade-server-${LEMONADE_VERSION}.x86_64.rpm"

          if [ ! -f "$RPM_FILE" ]; then
              echo "ERROR: .rpm file not found: $RPM_FILE"
              ls -la *.rpm 2>/dev/null || echo "No .rpm files found in current directory"
              exit 1
          fi

          dnf install -y shadow-utils "$RPM_FILE"

          echo "Installed package information:"
          rpm -qi lemonade-server

          echo "Installed file list:"
          rpm -ql lemonade-server | sort

          test -f /etc/lemonade/lemonade.conf
          test -f /opt/bin/lemonade-server
          test -f /opt/bin/lemonade-router

          lemonade-server --version
          lemonade-router --version

  test-dmg-inference:
    name: Test .dmg - llamacpp (metal)
    runs-on: macos-latest
    needs: build-lemonade-macos-dmg
    # Skip inference tests when signing is enabled (tag pushes or manual workflow_dispatch)
    # Also skip when no signing secrets (tests already ran inline in build job)
    if: ${{ needs.build-lemonade-macos-dmg.outputs.has_signing == 'true' && !startsWith(github.ref, 'refs/tags/') && inputs.enable_signing != true }}
    env:
      LEMONADE_CI_MODE: "True"
      PYTHONIOENCODING: utf-8
      GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
      LEMONADE_VERSION: ${{ needs.build-lemonade-macos-dmg.outputs.version }}
    steps:
      - uses: actions/checkout@v4

      - name: Set HF_HOME environment variable
        run: echo "HF_HOME=$PWD/hf-cache" >> $GITHUB_ENV

      - name: Install Lemonade Server (.pkg)
        uses: ./.github/actions/install-lemonade-server-dmg
        with:
          version: ${{ env.LEMONADE_VERSION }}

      - name: Setup Python and virtual environment
        uses: ./.github/actions/setup-venv
        with:
          venv-name: '.venv'
          python-version: '3.10'
          requirements-file: 'test/requirements.txt'

      - name: Test llamacpp (metal)
        env:
          HF_HOME: ${{ env.HF_HOME }}
        run: |
          set -e
          .venv/bin/python test/server_llm.py --wrapped-server llamacpp --backend metal --server-binary /usr/local/bin/lemonade-server

      - name: Print server logs on failure
        if: failure()
        run: |
          echo "=== Server Log (if exists) ==="
          if [ -f /var/log/lemonade/lemonade-server.out.log ]; then
            echo "Last 100 lines of lemonade-server.out.log:"
            tail -100 /var/log/lemonade/lemonade-server.out.log
          fi
          if [ -f /var/log/lemonade/lemonade-server.err.log ]; then
            echo "Last 100 lines of lemonade-server.err.log:"
            tail -100 /var/log/lemonade/lemonade-server.err.log
          fi

          echo ""
          echo "=== Temp logs ==="
          for f in /tmp/lemonade*.log; do
            if [ -f "$f" ]; then
              echo "--- $f ---"
              tail -50 "$f"
            fi
          done 2>/dev/null || echo "No temp log files found"

          echo ""
          echo "=== Lock file status ==="
          ls -la /tmp/lemonade*.lock 2>/dev/null || echo "No lock files found"

  # ========================================================================
  # CLI AND ENDPOINTS TESTS - Run on GitHub-hosted runners (no GPU needed)
  # ========================================================================

  test-cli-endpoints:
    name: Test ${{ matrix.test_type }} (${{ matrix.os }})
    runs-on: ${{ matrix.os }}
    needs:
      - build-lemonade-server-installer
      - build-lemonade-deb
      - build-lemonade-macos-dmg
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, windows-latest, macos-latest]
        test_type: [cli, endpoints, system-info, ollama, llamacpp-system]
        exclude:
          - os: macos-latest
            test_type: system-info
    env:
      LEMONADE_CI_MODE: "True"
      PYTHONIOENCODING: utf-8
      GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
      LEMONADE_VERSION: ${{ needs.build-lemonade-deb.outputs.version }}
    steps:
      - uses: actions/checkout@v4

      # ---- Linux Setup ----
      - name: Download .deb package
        if: runner.os == 'Linux'
        uses: actions/download-artifact@v4
        with:
          name: lemonade-deb
          path: .

      - name: Install Lemonade (.deb)
        if: runner.os == 'Linux'
        shell: bash
        run: |
          DEB_FILE="lemonade-server_${{ env.LEMONADE_VERSION }}_amd64.deb"
          echo "Installing .deb package: $DEB_FILE"
          sudo apt install ./"$DEB_FILE"

          # Verify binaries installed
          lemonade-server --version

          # Verify the systemd service started automatically
          sudo systemctl status lemonade-server

      - name: Set environment (Linux)
        if: runner.os == 'Linux'
        shell: bash
        run: |
          echo "HF_HOME=$PWD/hf-cache" >> $GITHUB_ENV
          echo "VENV_PYTHON=.venv/bin/python" >> $GITHUB_ENV
          echo "SERVER_BINARY=lemonade-server" >> $GITHUB_ENV

      # ---- Windows Setup ----
      - name: Setup (Windows)
        if: runner.os == 'Windows'
        shell: pwsh
        run: |
          $cwd = (Get-Item .).FullName
          echo "HF_HOME=$cwd\hf-cache" >> $Env:GITHUB_ENV
          echo "LEMONADE_INSTALL_PATH=$cwd\lemonade_server_install" >> $Env:GITHUB_ENV

      - name: Install Lemonade Server (Windows)
        if: runner.os == 'Windows'
        uses: ./.github/actions/install-lemonade-server-msi
        with:
          install-path: ${{ env.LEMONADE_INSTALL_PATH }}

      - name: Set paths (Windows)
        if: runner.os == 'Windows'
        shell: pwsh
        run: |
          echo "VENV_PYTHON=.venv/Scripts/python.exe" >> $Env:GITHUB_ENV
          echo "SERVER_BINARY=$Env:LEMONADE_INSTALL_PATH\bin\lemonade-server.exe" >> $Env:GITHUB_ENV

      # ---- macOS Setup (only when signed .pkg is available) ----
      - name: Download .pkg package
        if: runner.os == 'macOS' && needs.build-lemonade-macos-dmg.outputs.has_signing == 'true'
        uses: actions/download-artifact@v4
        with:
          name: lemonade-macos-pkg
          path: .

      - name: Install Lemonade Server (.pkg)
        id: install-pkg
        if: runner.os == 'macOS' && needs.build-lemonade-macos-dmg.outputs.has_signing == 'true'
        uses: ./.github/actions/install-lemonade-server-dmg
        with:
          version: ${{ env.LEMONADE_VERSION }}
          download-artifact: 'false'

      - name: Set environment (macOS)
        if: runner.os == 'macOS' && needs.build-lemonade-macos-dmg.outputs.has_signing == 'true'
        shell: bash
        run: |
          echo "HF_HOME=$PWD/hf-cache" >> $GITHUB_ENV
          echo "VENV_PYTHON=.venv/bin/python" >> $GITHUB_ENV
          echo "SERVER_BINARY=${{ steps.install-pkg.outputs.bin-path }}/lemonade-server" >> $GITHUB_ENV

      # ---- Common Setup ----
      - name: Setup Python and virtual environment
        uses: ./.github/actions/setup-venv
        with:
          venv-name: '.venv'
          python-version: '3.10'
          requirements-file: 'test/requirements.txt'

      # ---- Run Tests ----
      - name: Stop systemd service before tests (Linux)
        if: runner.os == 'Linux'
        shell: bash
        run: |
          echo "Stopping lemonade-server systemd service..."
          sudo systemctl stop lemonade-server || true
          sudo systemctl status lemonade-server || true

      - name: Stop LaunchDaemon before tests (macOS)
        if: runner.os == 'macOS' && needs.build-lemonade-macos-dmg.outputs.has_signing == 'true'
        shell: bash
        run: |
          echo "Stopping lemonade-server LaunchDaemon..."
          sudo launchctl bootout system/com.lemonade.server 2>/dev/null || true
          CURRENT_UID=$(id -u)
          launchctl bootout gui/"$CURRENT_UID"/com.lemonade.tray 2>/dev/null || true
          sleep 2

          # Clean up PID and lock files left behind by killed processes
          echo "Cleaning up stale PID and lock files..."
          sudo rm -f /tmp/lemonade-router.pid
          sudo rm -f /tmp/lemonade_Router.lock
          sudo rm -f /tmp/lemonade_Server.lock
          sudo rm -f /tmp/lemonade_Tray.lock
          sudo rm -f /tmp/lemonade*.pid /tmp/lemonade*.lock

          echo "LaunchDaemon stopped and cleaned up."

      - name: Run ${{ matrix.test_type }} tests
        if: ${{ !(runner.os == 'macOS' && needs.build-lemonade-macos-dmg.outputs.has_signing != 'true') }}
        shell: bash
        env:
          HF_HOME: ${{ env.HF_HOME }}
        run: |
          set -e  # Exit on error

          if [ "${{ matrix.test_type }}" = "cli" ]; then
              echo "Running CLI tests..."
              $VENV_PYTHON test/server_cli.py --server-binary "$SERVER_BINARY"
              $VENV_PYTHON test/server_cli.py --server-binary "$SERVER_BINARY" --ephemeral
              $VENV_PYTHON test/server_cli.py --server-binary "$SERVER_BINARY" --listen-all
              $VENV_PYTHON test/server_cli.py --server-binary "$SERVER_BINARY" --api-key
          elif [ "${{ matrix.test_type }}" = "endpoints" ]; then
              echo "Running endpoint tests..."
              $VENV_PYTHON test/server_endpoints.py --server-binary "$SERVER_BINARY"
              $VENV_PYTHON test/server_endpoints.py --server-binary "$SERVER_BINARY" --server-per-test
          elif [ "${{ matrix.test_type }}" = "system-info" ]; then
              echo "Running system-info mock hardware tests..."
              # Unset LEMONADE_CI_MODE so the server uses mock cache files
              unset LEMONADE_CI_MODE
              $VENV_PYTHON test/server_system_info.py --server-binary "$SERVER_BINARY"
          elif [ "${{ matrix.test_type }}" = "ollama" ]; then
              echo "Running Ollama API tests..."
              $VENV_PYTHON test/test_ollama.py --server-binary "$SERVER_BINARY"
          elif [ "${{ matrix.test_type }}" = "llamacpp-system" ]; then
              echo "Running LlamaCpp System Backend tests..."
              $VENV_PYTHON test/test_llamacpp_system_backend.py --server-binary "$SERVER_BINARY"
          fi

          echo "${{ matrix.test_type }} tests PASSED!"

  # ========================================================================
  # RELEASE JOB - Add artifacts to GitHub release
  # ========================================================================

  release:
    name: Create GitHub Release
    runs-on: ubuntu-latest
    needs:
      - sign-msi-installers
      - build-lemonade-deb
      - build-lemonade-rpm
      - build-lemonade-macos-dmg
      - build-lemonade-appimage
      - test-cli-endpoints
      - test-rpm-package
    if: startsWith(github.ref, 'refs/tags/v')
    env:
      LEMONADE_VERSION: ${{ needs.build-lemonade-deb.outputs.version }}
    steps:
      - name: Checkout for release notes action
        uses: actions/checkout@v4
        with:
          sparse-checkout: .github

      - name: Download Signed Lemonade Server Installer (Windows)
        uses: actions/download-artifact@v4
        with:
          name: Lemonade_Server_MSI_Signed
          path: .

      - name: Download Lemonade .deb Package
        uses: actions/download-artifact@v4
        with:
          name: lemonade-deb
          path: .

      - name: Download Lemonade .rpm Package
        uses: actions/download-artifact@v4
        with:
          name: lemonade-rpm
          path: .

      - name: Download Lemonade macOS .pkg Package
        uses: actions/download-artifact@v4
        with:
          name: lemonade-macos-pkg
          path: .

      - name: Download Lemonade AppImage (Linux)
        uses: actions/download-artifact@v4
        with:
          name: lemonade-appimage
          path: .

      - name: Verify release artifacts
        run: |
          echo "Release artifacts:"
          ls -lh lemonade-server-minimal.msi
          ls -lh lemonade.msi
          ls -lh lemonade-server_${LEMONADE_VERSION}_amd64.deb
          ls -lh lemonade-server-${LEMONADE_VERSION}.x86_64.rpm
          ls -lh *.pkg
          ls -lh Lemonade-${LEMONADE_VERSION}-x86_64.AppImage

      - name: Generate release notes
        id: release-notes
        uses: ./.github/actions/generate-release-notes
        with:
          version: ${{ env.LEMONADE_VERSION }}
          repo: ${{ github.repository }}
          github_token: ${{ secrets.GITHUB_TOKEN }}

      - name: Create Release
        uses: softprops/action-gh-release@v2
        with:
          name: ${{ github.ref_name }}
          body_path: ${{ steps.release-notes.outputs.release_notes_file }}
          files: |
            lemonade-server-minimal.msi
            lemonade.msi
            lemonade-server_${{ env.LEMONADE_VERSION }}_amd64.deb
            lemonade-server-${{ env.LEMONADE_VERSION }}.x86_64.rpm
            *.pkg
            Lemonade-${{ env.LEMONADE_VERSION }}-x86_64.AppImage
          fail_on_unmatched_files: true