add modepath util and cache

Signed-off-by: pandyamarut <[email protected]>

src/engine.py

@@ -5,10 +5,14 @@
 import asyncio
 import aiohttp
 import os
+from utils import modelpaths
 
 class SGlangEngine:
-    def __init__(self, model=os.getenv('MODEL_NAME'), host="0.0.0.0", port=30000):
-        self.model = model
+    def __init__(self, model=None, host="0.0.0.0", port=30000):
+        if os.getenv("RUNPOD_HUGGINGFACE_MODEL"):
+            self.model = modelpaths()[0]
+        else:
+            self.model = model or os.getenv("MODEL_NAME")    
         self.host = host
         self.port = port
         self.base_url = f"http://{host}:{port}"

src/utils.py

@@ -1,5 +1,5 @@
 import json
-
+import os 
 
 def format_chunk(chunk):
     chunk = chunk.strip()
@@ -23,4 +23,22 @@ def process_response(response):
     for line in response.iter_lines():
         if line:
             decoded_line = line.decode('utf-8')
-            yield format_chunk(decoded_line)
+            yield format_chunk(decoded_line)
+
+def topath(raw: str) -> str:
+    raw = raw.strip()
+    if ":" in raw:
+            model, branch = raw.rsplit(":", maxsplit=1)
+    else:
+            model, branch = raw, "main"
+    if "/" not in model:
+        raise ValueError(f"invalid model: expected one in the form user/model[:path], but got {model}")
+    user, model = model.rsplit("/", maxsplit=1)
+    return os.path.join("/runpod", "cache", "models", user, model, branch)
+
+
+def modelpaths() -> list[str]:
+    raw = os.environ.get("RUNPOD_HUGGINGFACE_MODEL")
+    if not raw:
+        return []
+    return [topath(m) for m in raw.split(",")]