Add support for ROCm channels (For llama.cpp and stable diffusion)

.github/workflows/cpp_server_build_test_release.yml

@@ -1622,6 +1622,163 @@ jobs:
         with:
           artifact-name: server-logs-apikey-windows-latest
 
+  # ========================================================================
+  # TEST ROCM STABLE CHANNEL - Verify stable channel on Windows hosted runner
+  # ========================================================================
+
+  test-rocm-stable-channel:
+    name: Test ROCm Stable Channel (Windows)
+    runs-on: windows-latest
+    needs:
+      - build-lemonade-server-installer
+    env:
+      LEMONADE_CI_MODE: "True"
+      PYTHONIOENCODING: utf-8
+      GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup (Windows)
+        shell: powershell
+        run: |
+          $cwd = (Get-Item .).FullName
+          echo "HF_HOME=$cwd\hf-cache" >> $Env:GITHUB_ENV
+          echo "LEMONADE_INSTALL_PATH=$cwd\lemonade_server_install" >> $Env:GITHUB_ENV
+
+      - name: Install Lemonade Server (Windows)
+        uses: ./.github/actions/install-lemonade-server-msi
+        with:
+          install-path: ${{ env.LEMONADE_INSTALL_PATH }}
+
+      - name: Set paths (Windows)
+        shell: powershell
+        run: |
+          echo "VENV_PYTHON=.venv/Scripts/python.exe" >> $Env:GITHUB_ENV
+          echo "SERVER_BINARY=$Env:LEMONADE_INSTALL_PATH\bin\lemonade-server.exe" >> $Env:GITHUB_ENV
+
+      - name: Setup Python and virtual environment
+        uses: ./.github/actions/setup-venv
+        with:
+          venv-name: '.venv'
+          python-version: '3.10'
+          requirements-file: 'test/requirements.txt'
+
+      - name: Set ROCm channel to stable
+        shell: bash
+        run: |
+          set -e
+          echo "Setting rocm_channel to stable in config.json"
+          "$VENV_PYTHON" -c "
+          import json
+          import os
+          from pathlib import Path
+
+          cache_dir = Path.home() / '.cache' / 'lemonade'
+          config_file = cache_dir / 'config.json'
+
+          # Create cache dir if needed
+          cache_dir.mkdir(parents=True, exist_ok=True)
+
+          # Load or create config
+          if config_file.exists():
+              with open(config_file, 'r') as f:
+                  config = json.load(f)
+          else:
+              config = {}
+
+          # Set rocm_channel to stable
+          config['rocm_channel'] = 'stable'
+
+          # Save config
+          with open(config_file, 'w') as f:
+              json.dump(config, f, indent=2)
+
+          print(f'Config updated: rocm_channel = stable')
+          "
+
+      - name: Verify ROCm stable channel configuration
+        shell: bash
+        run: |
+          set -e
+          echo "Verifying rocm_channel setting..."
+          "$VENV_PYTHON" -c "
+          import json
+          from pathlib import Path
+
+          config_file = Path.home() / '.cache' / 'lemonade' / 'config.json'
+
+          if not config_file.exists():
+              print('ERROR: config.json not found')
+              exit(1)
+
+          with open(config_file, 'r') as f:
+              config = json.load(f)
+
+          channel = config.get('rocm_channel', 'NOT_SET')
+          print(f'rocm_channel = {channel}')
+
+          if channel != 'stable':
+              print(f'ERROR: Expected rocm_channel=stable, got {channel}')
+              exit(1)
+
+          print('SUCCESS: rocm_channel is set to stable')
+          "
+
+      - name: Test channel switching with lemonade CLI
+        shell: bash
+        run: |
+          set -e
+          echo "Setting rocm_channel to stable via lemonade CLI..."
+
+          # Use lemonade config set to change the channel
+          "$SERVER_BINARY" config set rocm_channel=stable
+
+          echo "Verifying config was updated..."
+          "$VENV_PYTHON" -c "
+          import json
+          from pathlib import Path
+
+          config_file = Path.home() / '.cache' / 'lemonade' / 'config.json'
+
+          if not config_file.exists():
+              print('ERROR: config.json not found')
+              exit(1)
+
+          with open(config_file, 'r') as f:
+              config = json.load(f)
+
+          channel = config.get('rocm_channel', 'NOT_SET')
+          print(f'rocm_channel = {channel}')
+
+          if channel != 'stable':
+              print(f'ERROR: Expected rocm_channel=stable, got {channel}')
+              exit(1)
+
+          print('SUCCESS: rocm_channel set to stable via CLI')
+          "
+
+      - name: Test recipes command shows rocm backend
+        shell: bash
+        run: |
+          set -e
+          echo "Testing recipes command..."
+          "$SERVER_BINARY" recipes | tee recipes_output.txt
+
+          # Check that rocm backend is listed (not rocm-stable or rocm-preview)
+          if grep -q "rocm-stable" recipes_output.txt || grep -q "rocm-preview" recipes_output.txt; then
+              echo "ERROR: Found rocm-stable or rocm-preview in recipes output (should only show 'rocm')"
+              cat recipes_output.txt
+              exit 1
+          fi
+
+          echo "SUCCESS: Recipes output looks correct"
+
+      - name: Capture and upload server logs
+        if: always()
+        uses: ./.github/actions/capture-server-logs
+        with:
+          artifact-name: server-logs-rocm-stable-channel
+
   # ========================================================================
   # RELEASE JOB - Add artifacts to GitHub release
   # ========================================================================

.github/workflows/validate_llamacpp.yml

@@ -32,6 +32,7 @@ jobs:
     outputs:
       llamacpp_release: ${{ steps.llamacpp.outputs.release }}
       llamacpp_rocm_release: ${{ steps.llamacpp_rocm.outputs.release }}
+      llamacpp_rocm_preview_release: ${{ steps.llamacpp_rocm_preview.outputs.release }}
     steps:
       - name: Get latest llama.cpp release
         id: llamacpp
@@ -51,6 +52,15 @@ jobs:
           echo "Latest llamacpp-rocm release: $RELEASE"
           echo "release=$RELEASE" >> "$GITHUB_OUTPUT"
 
+      - name: Get latest llamacpp-rocm-preview release
+        id: llamacpp_rocm_preview
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          RELEASE=$(gh api repos/lemonade-sdk/llama.cpp/releases/latest --jq '.tag_name')
+          echo "Latest llamacpp-rocm-preview release: $RELEASE"
+          echo "release=$RELEASE" >> "$GITHUB_OUTPUT"
+
   # ========================================================================
   # Build binaries with updated backend versions
   # ========================================================================
@@ -71,6 +81,7 @@ jobs:
         env:
           LLAMACPP_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_release }}
           LLAMACPP_ROCM_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_rocm_release }}
+          LLAMACPP_ROCM_PREVIEW_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_rocm_preview_release }}
         run: |
           $ErrorActionPreference = "Stop"
           $path = "src/cpp/resources/backend_versions.json"
@@ -85,10 +96,22 @@ jobs:
           }
 
           $rocm = $env:LLAMACPP_ROCM_RELEASE
-          Write-Host "Updating llamacpp rocm to $rocm" -ForegroundColor Cyan
-          $old = $data.llamacpp.rocm
-          $data.llamacpp.rocm = $rocm
-          Write-Host "  llamacpp.rocm: $old -> $rocm"
+          Write-Host "Updating llamacpp rocm-nightly to $rocm" -ForegroundColor Cyan
+          $old = $data.llamacpp.'rocm-nightly'
+          $data.llamacpp.'rocm-nightly' = $rocm
+          Write-Host "  llamacpp.rocm-nightly: $old -> $rocm"
+
+          $rocmStable = $env:LLAMACPP_RELEASE
+          Write-Host "Updating llamacpp rocm-stable to $rocmStable" -ForegroundColor Cyan
+          $old = $data.llamacpp.'rocm-stable'
+          $data.llamacpp.'rocm-stable' = $rocmStable
+          Write-Host "  llamacpp.rocm-stable: $old -> $rocmStable"
+
+          $rocmPreview = $env:LLAMACPP_ROCM_PREVIEW_RELEASE
+          Write-Host "Updating llamacpp rocm-preview to $rocmPreview" -ForegroundColor Cyan
+          $old = $data.llamacpp.'rocm-preview'
+          $data.llamacpp.'rocm-preview' = $rocmPreview
+          Write-Host "  llamacpp.rocm-preview: $old -> $rocmPreview"
 
           $data | ConvertTo-Json -Depth 10 | Set-Content $path -Encoding UTF8
 
@@ -121,14 +144,21 @@ jobs:
   # Validate backends on self-hosted runner
   # ========================================================================
   validate:
-    name: Validate ${{ matrix.backend }}
+    name: Validate ${{ matrix.backend }}${{ matrix.channel && format(' ({0})', matrix.channel) || '' }}
     needs: [get-latest-releases, build]
     if: always() && needs.build.result == 'success'
     runs-on: [self-hosted, Windows, 128gb]
     strategy:
       fail-fast: false
       matrix:
-        backend: [vulkan, rocm]
+        include:
+          - backend: vulkan
+          - backend: rocm
+            channel: stable
+          - backend: rocm
+            channel: preview
+          - backend: rocm
+            channel: nightly
     steps:
       - uses: actions/checkout@v5
         with:
@@ -168,8 +198,10 @@ jobs:
           $ErrorActionPreference = "Stop"
           $lemondExe = (Resolve-Path "build\Release\lemond.exe").Path
           $backend = "${{ matrix.backend }}"
-          $logsDir = "server-logs-$backend"
-          $cacheDir = Join-Path $PWD "ci-cache-$backend"
+          $channel = "${{ matrix.channel }}"
+          $label = if ($channel) { "$backend-$channel" } else { $backend }
+          $logsDir = "server-logs-$label"
+          $cacheDir = Join-Path $PWD "ci-cache-$label"
           $venvPython = ".\.venv\Scripts\python.exe"
 
           New-Item -ItemType Directory -Force -Path $logsDir | Out-Null
@@ -192,9 +224,13 @@ jobs:
             $validationArgs = @(
               "test/validate_llamacpp.py",
               "--backend", $backend,
-              "--output", "llamacpp_validation_$backend.json",
+              "--output", "llamacpp_validation_$label.json",
               "--logs-dir", $logsDir
             )
+            if ($channel) {
+              $validationArgs += "--channel"
+              $validationArgs += $channel
+            }
             if ("${{ env.LITE_MODE }}" -eq "true") {
               $validationArgs += "--lite"
             }
@@ -215,16 +251,16 @@ jobs:
         if: always()
         uses: actions/upload-artifact@v7
         with:
-          name: validation-results-${{ matrix.backend }}
-          path: llamacpp_validation_${{ matrix.backend }}.json
+          name: validation-results-${{ matrix.channel && format('{0}-{1}', matrix.backend, matrix.channel) || matrix.backend }}
+          path: llamacpp_validation_${{ matrix.channel && format('{0}-{1}', matrix.backend, matrix.channel) || matrix.backend }}.json
           retention-days: 30
 
       - name: Upload server logs
         if: always()
         uses: actions/upload-artifact@v7
         with:
-          name: server-logs-${{ matrix.backend }}
-          path: server-logs-${{ matrix.backend }}/
+          name: server-logs-${{ matrix.channel && format('{0}-{1}', matrix.backend, matrix.channel) || matrix.backend }}
+          path: server-logs-${{ matrix.channel && format('{0}-{1}', matrix.backend, matrix.channel) || matrix.backend }}/
           retention-days: 30
           if-no-files-found: ignore
 
@@ -256,48 +292,56 @@ jobs:
         env:
           LLAMACPP_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_release }}
           LLAMACPP_ROCM_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_rocm_release }}
+          LLAMACPP_ROCM_PREVIEW_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_rocm_preview_release }}
         run: |
           python3 -c "
           import json, os
           path = 'src/cpp/resources/backend_versions.json'
           with open(path, 'r') as f:
               data = json.load(f)
           llamacpp = os.environ['LLAMACPP_RELEASE']
-          rocm = os.environ['LLAMACPP_ROCM_RELEASE']
+          rocm_preview = os.environ['LLAMACPP_ROCM_PREVIEW_RELEASE']
+          rocm_nightly = os.environ['LLAMACPP_ROCM_RELEASE']
           data['llamacpp']['vulkan'] = llamacpp
           data['llamacpp']['cpu'] = llamacpp
           data['llamacpp']['metal'] = llamacpp
-          data['llamacpp']['rocm'] = rocm
+          data['llamacpp']['rocm-stable'] = llamacpp
+          data['llamacpp']['rocm-nightly'] = rocm_nightly
+          data['llamacpp']['rocm-preview'] = rocm_preview
           with open(path, 'w') as f:
               json.dump(data, f, indent=2)
               f.write('\n')
-          print(f'Updated: vulkan/cpu/metal={llamacpp}, rocm={rocm}')
+          print(f'Updated: vulkan/cpu/metal/rocm-stable={llamacpp}, rocm-nightly={rocm_nightly}, rocm-preview={rocm_preview}')
           "
 
       - name: Generate PR body
         shell: bash
         env:
           LLAMACPP_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_release }}
           LLAMACPP_ROCM_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_rocm_release }}
+          LLAMACPP_ROCM_PREVIEW_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_rocm_preview_release }}
         run: |
           python3 << 'PYEOF'
           import json, os
 
           body_lines = []
           llamacpp = os.environ.get("LLAMACPP_RELEASE", "unknown")
-          rocm = os.environ.get("LLAMACPP_ROCM_RELEASE", "unknown")
+          rocm_nightly = os.environ.get("LLAMACPP_ROCM_RELEASE", "unknown")
+          rocm_preview = os.environ.get("LLAMACPP_ROCM_PREVIEW_RELEASE", "unknown")
 
           body_lines.append("## Auto-update llama.cpp backends")
           body_lines.append("")
-          body_lines.append(f"- **llama.cpp** (vulkan/cpu/metal): `{llamacpp}`")
-          body_lines.append(f"- **llamacpp-rocm**: `{rocm}`")
+          body_lines.append(f"- **llama.cpp** (vulkan/cpu/metal/rocm-stable): `{llamacpp}`")
+          body_lines.append(f"- **llamacpp-rocm** (rocm-nightly): `{rocm_nightly}`")
+          body_lines.append(f"- **llamacpp-rocm-preview**: `{rocm_preview}`")
           body_lines.append("")
           body_lines.append("## Validation Results")
           body_lines.append("")
           body_lines.append("| Model | Result | Response | input_tokens | output_tokens | time_to_first_token | tokens_per_second |")
           body_lines.append("|-------|--------|----------|--------------|---------------|---------------------|-------------------|")
 
-          for backend in ["vulkan", "rocm"]:
+          for backend in ["vulkan", "rocm-stable", "rocm-preview", "rocm-nightly"]:
+              # Labels match the artifact names produced by the validate job
               results_file = f"llamacpp_validation_{backend}.json"
               if not os.path.isfile(results_file):
                   body_lines.append(f"| _{backend}: no results_ | | | | | | |")
@@ -334,8 +378,9 @@ jobs:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           LLAMACPP_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_release }}
           LLAMACPP_ROCM_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_rocm_release }}
+          LLAMACPP_ROCM_PREVIEW_RELEASE: ${{ needs.get-latest-releases.outputs.llamacpp_rocm_preview_release }}
         run: |
-          BRANCH="auto/llamacpp-update-${LLAMACPP_RELEASE}-${LLAMACPP_ROCM_RELEASE}"
+          BRANCH="auto/llamacpp-update-${LLAMACPP_RELEASE}-${LLAMACPP_ROCM_RELEASE}-${LLAMACPP_ROCM_PREVIEW_RELEASE}"
 
           # Check if a PR already exists for this update
           EXISTING_PR=$(gh pr list --head "$BRANCH" --json number --jq '.[0].number' 2>/dev/null || echo "")
@@ -355,7 +400,7 @@ jobs:
 
           git checkout -b "$BRANCH"
           git add src/cpp/resources/backend_versions.json
-          git commit -m "Update llama.cpp to ${LLAMACPP_RELEASE}, rocm to ${LLAMACPP_ROCM_RELEASE}"
+          git commit -m "Update llama.cpp to ${LLAMACPP_RELEASE}, rocm-nightly to ${LLAMACPP_ROCM_RELEASE}, rocm-preview to ${LLAMACPP_ROCM_PREVIEW_RELEASE}"
           git push origin "$BRANCH"
 
           gh pr create \