Merge pull request #1 from kaiser-data/tannaz_gpt2_shakespeare

Version of presentation day

Author: kaiser-data

.github/workflows/deploy-backend.yml

@@ -0,0 +1,19 @@
+name: Deploy Backend
+
+on:
+  push:
+    branches: [main, tannaz_gpt2_shakespeare]
+    paths:
+      - 'docker/backend/**'
+      - 'k8s/03-backend.yaml'
+  workflow_dispatch:  # Allow manual trigger
+
+jobs:
+  deploy:
+    uses: ./.github/workflows/deploy-image.yml
+    with:
+      image_name: ml-backend
+      docker_context: docker/backend
+      deployment_name: ml-backend
+      container_name: backend
+    secrets: inherit

.github/workflows/deploy-frontend.yml

@@ -0,0 +1,19 @@
+name: Deploy Frontend
+
+on:
+  push:
+    branches: [main, tannaz_gpt2_shakespeare]
+    paths:
+      - 'docker/frontend/**'
+      - 'k8s/04-frontend.yaml'
+  workflow_dispatch:  # Allow manual trigger
+
+jobs:
+  deploy:
+    uses: ./.github/workflows/deploy-image.yml
+    with:
+      image_name: ml-frontend
+      docker_context: docker/frontend
+      deployment_name: ml-frontend
+      container_name: frontend
+    secrets: inherit

.github/workflows/deploy-image.yml

@@ -0,0 +1,75 @@
+name: Build and Deploy Image
+
+on:
+  workflow_call:
+    inputs:
+      image_name:
+        required: true
+        type: string
+      docker_context:
+        required: true
+        type: string
+      deployment_name:
+        required: true
+        type: string
+      container_name:
+        required: true
+        type: string
+
+env:
+  ACR_NAME: mlplatformacrwqckxk
+
+jobs:
+  build-and-deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Generate version tag
+        id: version
+        run: echo "tag=v$(date +%Y%m%d)-${GITHUB_SHA::7}" >> $GITHUB_OUTPUT
+
+      - name: Login to ACR
+        uses: azure/docker-login@v1
+        with:
+          login-server: ${{ env.ACR_NAME }}.azurecr.io
+          username: ${{ secrets.AZURE_ACR_USERNAME }}
+          password: ${{ secrets.AZURE_ACR_PASSWORD }}
+
+      - name: Build and push
+        run: |
+          docker build \
+            -t ${{ env.ACR_NAME }}.azurecr.io/${{ inputs.image_name }}:${{ steps.version.outputs.tag }} \
+            -t ${{ env.ACR_NAME }}.azurecr.io/${{ inputs.image_name }}:latest \
+            ${{ inputs.docker_context }}
+          docker push ${{ env.ACR_NAME }}.azurecr.io/${{ inputs.image_name }}:${{ steps.version.outputs.tag }}
+          docker push ${{ env.ACR_NAME }}.azurecr.io/${{ inputs.image_name }}:latest
+
+      - name: Deploy to K8s
+        uses: appleboy/ssh-action@v1.0.0
+        with:
+          host: ${{ secrets.K8S_CONTROL_PLANE_IP }}
+          username: azureuser
+          key: ${{ secrets.K8S_SSH_KEY }}
+          script: |
+            kubectl set image deployment/${{ inputs.deployment_name }} \
+              -n ml-platform \
+              ${{ inputs.container_name }}=${{ env.ACR_NAME }}.azurecr.io/${{ inputs.image_name }}:${{ steps.version.outputs.tag }}
+
+            # Check if target node is available (GPU deployments may fail if VM is off)
+            if kubectl rollout status deployment/${{ inputs.deployment_name }} -n ml-platform --timeout=120s 2>/dev/null; then
+              echo "✅ Deployment rolled out successfully"
+            else
+              echo "⚠️ Rollout pending - target node may be offline. Image updated, will deploy when node is ready."
+              kubectl get deployment/${{ inputs.deployment_name }} -n ml-platform -o wide
+            fi
+
+      - name: Deployment summary
+        run: |
+          echo "### Deployment Complete! :rocket:" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "| Property | Value |" >> $GITHUB_STEP_SUMMARY
+          echo "|----------|-------|" >> $GITHUB_STEP_SUMMARY
+          echo "| Image | \`${{ env.ACR_NAME }}.azurecr.io/${{ inputs.image_name }}:${{ steps.version.outputs.tag }}\` |" >> $GITHUB_STEP_SUMMARY
+          echo "| Deployment | \`${{ inputs.deployment_name }}\` |" >> $GITHUB_STEP_SUMMARY
+          echo "| Commit | \`${GITHUB_SHA::7}\` |" >> $GITHUB_STEP_SUMMARY

.github/workflows/deploy-jupyter.yml

@@ -0,0 +1,19 @@
+name: Deploy JupyterLab
+
+on:
+  push:
+    branches: [main, tannaz_gpt2_shakespeare]
+    paths:
+      - 'docker/jupyterlab/**'
+      - 'k8s/05-jupyterlab.yaml'
+  workflow_dispatch:  # Allow manual trigger
+
+jobs:
+  deploy:
+    uses: ./.github/workflows/deploy-image.yml
+    with:
+      image_name: ml-jupyter
+      docker_context: docker/jupyterlab
+      deployment_name: ml-jupyterlab
+      container_name: jupyterlab
+    secrets: inherit

docker/backend/Dockerfile

@@ -2,17 +2,21 @@ FROM python:3.12-slim
 
 WORKDIR /app
 
-RUN apt-get update && apt-get install -y curl && \
+RUN apt-get update && apt-get install -y curl gnupg lsb-release && \
+    # Install kubectl
     curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" && \
     install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl && \
     rm kubectl && \
+    # Install Azure CLI
+    curl -sL https://aka.ms/InstallAzureCLIDeb | bash && \
     apt-get clean && rm -rf /var/lib/apt/lists/*
 
 COPY requirements.txt .
 
 RUN pip install --no-cache-dir torch==2.2.0 --index-url https://download.pytorch.org/whl/cpu && \
     pip install --no-cache-dir -r requirements.txt
 
+# Cache buster: v6 with Swagger root_path fix
 COPY app.py .
 
 EXPOSE 8000

docker/backend/app.py

@@ -15,7 +15,10 @@
 from transformers import pipeline
 import torch
 
-app = FastAPI(title="ML Platform API")
+app = FastAPI(
+    title="ML Platform API",
+    root_path=os.getenv("ROOT_PATH", "/api")  # For ingress path-based routing
+)
 
 app.add_middleware(
     CORSMiddleware,
@@ -28,6 +31,46 @@
 UPLOAD_DIR = Path(os.getenv("UPLOAD_DIR", "/mnt/uploads"))
 UPLOAD_DIR.mkdir(parents=True, exist_ok=True)
 
+# Azure VM auto-start config
+AZURE_RESOURCE_GROUP = os.getenv("AZURE_RESOURCE_GROUP", "rg-mlplatform")
+AZURE_GPU_VM_NAME = os.getenv("AZURE_GPU_VM_NAME", "vm-mlplatform-gpu")
+
+# Azure Service Principal credentials
+AZURE_CLIENT_ID = os.getenv("AZURE_CLIENT_ID")
+AZURE_CLIENT_SECRET = os.getenv("AZURE_CLIENT_SECRET")
+AZURE_TENANT_ID = os.getenv("AZURE_TENANT_ID")
+
+_azure_logged_in = False
+
+def ensure_azure_login():
+    """Ensure Azure CLI is logged in using Service Principal."""
+    global _azure_logged_in
+    if _azure_logged_in:
+        return True
+
+    if not all([AZURE_CLIENT_ID, AZURE_CLIENT_SECRET, AZURE_TENANT_ID]):
+        print("Azure credentials not configured")
+        return False
+
+    try:
+        result = subprocess.run(
+            ["az", "login", "--service-principal",
+             "-u", AZURE_CLIENT_ID,
+             "-p", AZURE_CLIENT_SECRET,
+             "--tenant", AZURE_TENANT_ID],
+            capture_output=True, text=True, timeout=30
+        )
+        if result.returncode == 0:
+            _azure_logged_in = True
+            print("Azure CLI login successful")
+            return True
+        else:
+            print(f"Azure login failed: {result.stderr}")
+            return False
+    except Exception as e:
+        print(f"Azure login error: {e}")
+        return False
+
 _model_cache = {}
 
 BASE_MODELS = {
@@ -72,16 +115,54 @@ def load_model(model_id: str):
         return None
 
 def check_gpu_worker() -> dict:
+    """Check if GPU worker node is ready in K8s."""
     try:
         result = subprocess.run(
-            ["kubectl", "get", "nodes", "-l", "nvidia.com/gpu.present=true", "-o", "name"],
+            ["kubectl", "get", "nodes", "-l", "nvidia.com/gpu.present=true",
+             "-o", "jsonpath={.items[0].status.conditions[?(@.type=='Ready')].status}"],
             capture_output=True, text=True, timeout=5
         )
-        has_gpu = bool(result.stdout.strip())
-        return {"available": has_gpu, "method": "k8s"}
+        is_ready = result.stdout.strip() == "True"
+        return {"available": is_ready, "method": "k8s"}
     except:
         return {"available": False, "method": "none"}
 
+def start_gpu_vm() -> dict:
+    """Start GPU VM using Azure CLI with Service Principal auth."""
+    # Ensure we're logged in to Azure
+    if not ensure_azure_login():
+        return {"status": "error", "message": "Azure authentication failed - credentials not configured"}
+
+    try:
+        # Check if VM is already running
+        status_result = subprocess.run(
+            ["az", "vm", "get-instance-view",
+             "--resource-group", AZURE_RESOURCE_GROUP,
+             "--name", AZURE_GPU_VM_NAME,
+             "--query", "instanceView.statuses[1].displayStatus", "-o", "tsv"],
+            capture_output=True, text=True, timeout=30
+        )
+        current_status = status_result.stdout.strip()
+
+        if current_status == "VM running":
+            return {"status": "already_running", "message": "GPU VM is already running"}
+
+        # Start the VM
+        start_result = subprocess.run(
+            ["az", "vm", "start",
+             "--resource-group", AZURE_RESOURCE_GROUP,
+             "--name", AZURE_GPU_VM_NAME,
+             "--no-wait"],
+            capture_output=True, text=True, timeout=60
+        )
+
+        if start_result.returncode == 0:
+            return {"status": "starting", "message": "GPU VM start initiated"}
+        else:
+            return {"status": "error", "message": start_result.stderr}
+    except Exception as e:
+        return {"status": "error", "message": str(e)}
+
 def scan_models_dir() -> dict:
     finetuned = {}
     if MODELS_DIR.exists():
@@ -116,7 +197,7 @@ def create_training_job(job_id: str, config: dict) -> dict:
         "kind": "Job",
         "metadata": {
             "name": job_id,
-            "namespace": "default"
+            "namespace": "ml-platform"
         },
         "spec": {
             "ttlSecondsAfterFinished": 3600,
@@ -125,7 +206,7 @@ def create_training_job(job_id: str, config: dict) -> dict:
                     "restartPolicy": "Never",
                     "containers": [{
                         "name": "trainer",
-                        "image": "mlplatformacr2024.azurecr.io/ml-jupyterlab:v1",
+                        "image": "mlplatformacrwqckxk.azurecr.io/ml-jupyter:latest",
                         "command": ["python3", "-c", f"""
 import os
 from transformers import AutoTokenizer, AutoModelForCausalLM, TrainingArguments, Trainer, TextDataset, DataCollatorForLanguageModeling
@@ -190,7 +271,10 @@ def create_training_job(job_id: str, config: dict) -> dict:
                     ],
                     "nodeSelector": {
                         "nvidia.com/gpu.present": "true"
-                    }
+                    },
+                    "imagePullSecrets": [
+                        {"name": "acr-secret"}
+                    ]
                 }
             }
         }
@@ -229,6 +313,32 @@ def health():
 def gpu_status():
     return check_gpu_worker()
 
+@app.post("/gpu/start")
+def gpu_start():
+    """Manually start the GPU VM."""
+    return start_gpu_vm()
+
+@app.post("/gpu/stop")
+def gpu_stop():
+    """Stop the GPU VM to save costs."""
+    if not ensure_azure_login():
+        return {"status": "error", "message": "Azure authentication failed"}
+
+    try:
+        result = subprocess.run(
+            ["az", "vm", "deallocate",
+             "--resource-group", AZURE_RESOURCE_GROUP,
+             "--name", AZURE_GPU_VM_NAME,
+             "--no-wait"],
+            capture_output=True, text=True, timeout=60
+        )
+        if result.returncode == 0:
+            return {"status": "stopping", "message": "GPU VM stop initiated"}
+        else:
+            return {"status": "error", "message": result.stderr}
+    except Exception as e:
+        return {"status": "error", "message": str(e)}
+
 @app.get("/models")
 def list_models():
     models = []
@@ -395,22 +505,21 @@ async def submit_training_job(
         "learning_rate": learning_rate
     }
 
-    if gpu["available"]:
-        k8s_result = create_training_job(job_id, config)
-        return {
-            "job_id": job_id,
-            "status": "submitted" if k8s_result["success"] else "failed",
-            "gpu_available": True,
-            "config": config,
-            "output_path": f"/mnt/models/{model_name.lower()}",
-            "k8s_result": k8s_result
-        }
-    else:
-        return {
-            "job_id": job_id,
-            "status": "pending",
-            "gpu_available": False,
-            "config": config,
-            "output_path": f"/mnt/models/{model_name.lower()}",
-            "message": "GPU not available - job queued"
-        }
+    # Auto-start GPU VM if not available
+    vm_start_result = None
+    if not gpu["available"]:
+        vm_start_result = start_gpu_vm()
+
+    # Create job - K8s will schedule it when GPU node is ready
+    k8s_result = create_training_job(job_id, config)
+
+    return {
+        "job_id": job_id,
+        "status": "submitted" if k8s_result["success"] else "failed",
+        "gpu_available": gpu["available"],
+        "gpu_vm_start": vm_start_result,
+        "config": config,
+        "output_path": f"/mnt/models/{model_name.lower()}",
+        "k8s_result": k8s_result,
+        "message": "Job submitted. GPU VM starting..." if vm_start_result else None
+    }