guide.txt

## Running a HuggingFace model Deepseek-v2-lite with Torch example 

# Apply a PVC creation
kubectl apply -f hf-cache-pvc.yaml
# Check for the PV info
kubectl describe pvc huggingface-cache-pvc
# Apply for running Deepseekv2-lite
kubectl apply -f deepseek-v2-lite-persistent-compatible.yaml
# Check the pod state
kubectl describe pod/deepseek-v2-lite-test-hbggb 
# Monitor the job initalization
kubectl logs -f job.batch/deepseek-v2-lite-test
# Check the downloaded files withing the container
kubectl exec job.batch/deepseek-v2-lite-test -- du -sh /hf-cache/hub
# Or interactivelly
kubectl exec -it job.batch/deepseek-v2-lite-test -- bash

## Running a NIM inference service example

# Creating keys
# kubectl create secret generic <name-api-key>   --from-literal="<key string>" 
# Checking secrets
kubectl get secret
# Apply a PVC creation
kubectl apply -f nim-pvc.yaml
# Start service
kubectl apply -f llama-nemotron-inference-self.yaml
# Expose endpoint
kubectl apply -f service.yaml
# Test inference service with a multi-modal request
curl http://10.150.45.115:8000/v1/chat/completions   -H "Content-Type: application/json"   -d '{
    "model": "nvidia/llama-3.1-nemotron-nano-vl-8b-v1",
    "messages": [
      {
        "role": "user",
        "content": [
          {"type": "text", "text": "What is in this image?"},
          {
            "type": "image_url",
            "image_url": {
              "url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
            }
          }
        ]
      }
    ],
    "max_tokens": 200,
    "temperature": 0.4
  }'