Offline engine inference

[vervitK]

sglang.utils import stream_and_merge, async_stream_and_merge
import sglang as sgl
import asyncio

def main():
# Initialize the LLM engine with only supported parameters
llm = sgl.Engine(
model_path="meta-llama/Llama-3.2-1B",
disable_cuda_graph=True
)

# Create a simple prompt
prompt = "What is the capital of France?"

# Define the generation function using sglang's decorator
@sgl.function
def generate(s):
    s += prompt
    s += sgl.gen("response", max_tokens=100)
    return s

# Execute and print the result
try:
    result = generate()
    print("\nPrompt:", prompt)
    print("\nResponse:", result["response"])
except Exception as e:
    print(f"An error occurred: {e}")

if name == "main":

main()

(.venv) root@ip-10-4-52-157:/home/ubuntu/vervit_sglang_test# python3 test.py
INFO 02-11 07:06:00 init.py:190] Automatically detected platform cuda.
Traceback (most recent call last):
File "/home/ubuntu/vervit_sglang_test/test.py", line 61, in
main()
File "/home/ubuntu/vervit_sglang_test/test.py", line 36, in main
llm = sgl.Engine(
File "/home/ubuntu/ravindra/inference-service/.venv/lib/python3.10/site-packages/sglang/api.py", line 45, in Engine
return Engine(*args, **kwargs)
File "/home/ubuntu/ravindra/inference-service/.venv/lib/python3.10/site-packages/sglang/srt/entrypoints/engine.py", line 99, in init
server_args = ServerArgs(**kwargs)
TypeError: ServerArgs.init() got an unexpected keyword argument 'backend'
(.venv) root@ip-10-4-52-157:/home/ubuntu/vervit_sglang_test# python3 test.py
INFO 02-11 07:07:03 init.py:190] Automatically detected platform cuda.
INFO 02-11 07:07:12 init.py:190] Automatically detected platform cuda.
INFO 02-11 07:07:12 init.py:190] Automatically detected platform cuda.
Loading safetensors checkpoint shards: 0% Completed | 0/1 [00:00<?, ?it/s]
Loading safetensors checkpoint shards: 100% Completed | 1/1 [00:00<00:00, 2.79it/s]
Loading safetensors checkpoint shards: 100% Completed | 1/1 [00:00<00:00, 2.78it/s]

An error occurred: Please specify a backend
(.venv) root@ip-10-4-52-157:/home/ubuntu/vervit_sglang_test#