bug fixes and final updates

Author: salhanyf

src/llm-examples/ollama/README.md

@@ -38,7 +38,7 @@ Before starting, ensure you have [access](https://nag-devops.github.io/speed-hpc
 
 * Open a new terminal window and paste the ssh command to connect to the speed node the server is running on. The command will look like:
     ```shell
-    ssh -L XXXXX:localhost:XXXXX <ENCSusername>@speed.encs.concordia.ca -t ssh speed-XX
+    ssh -L XXXXX:speed-XX:XXXXX <ENCSusername>@speed.encs.concordia.ca -t ssh speed-XX
     ```
 
 * Navigate to ollama directory and do a sanity check
@@ -47,8 +47,25 @@ Before starting, ensure you have [access](https://nag-devops.github.io/speed-hpc
     ollama -v
     ```
 
-* Pull a specific model and run it interactively (optional).
+* Run the `run_ollama.sh` script, replace speed-XX with the name of the node the server is running on 
+    ```shell
+    sbatch -w speed-XX run_ollama.sh
+    ```
+
+    The script will:
+    - Request required resources
+    - Set environment variables
+    - Pull a model to run (in this case it's llama3.2)
+    - Create a python environment to run `ollama_demo.py`
+    - Run `ollama_demo.py` which interact with the model 
+
+Optional:
+1. Check if the server is running, replace XXXXX with the port number
+```shell
+curl http://localhost:XXXXX/api/tags
+```
+
+2. Run a model with a prompt
 ```shell
-ollama pull llama3.2
-echo "What is today" | ollama run llama3.2
-```
+curl -sS http://localhost:56781/api/generate -H "Content-Type: application/json" -d '{"model": "llama3.2","prompt": "why is the sky blue?","stream": false}' | jq -r '.response'
+```

src/llm-examples/ollama/ollama_demo.py

@@ -9,7 +9,7 @@
 
 client = ollama.Client(host=ollama_host)
 response = client.chat(
-    model='llama3.1',
+    model='llama3.2',
     messages=[{
         'role': 'user',
         'content': (
@@ -20,4 +20,4 @@
 )
 
 print(f"[Client connected to {ollama_host}]")
-print(response["message"]["content"])
+print(response["message"]["content"])

src/llm-examples/ollama/run_ollama.sh

@@ -1,9 +1,16 @@
 #!/encs/bin/tcsh
 
+#SBATCH --job-name=ollama-client
+#SBATCH --mem=50G
+#SBATCH --ntasks=1
+#SBATCH --cpus-per-task=4
+#SBATCH --mail-type=ALL
+#SBATCH --output=ollama-%J.out
+
 set ODIR = /speed-scratch/$USER/ollama
 setenv PATH /speed-scratch/$USER/ollama/bin:$PATH
 setenv OLLAMA_MODELS $ODIR/models
-setenv OLLAMA_HOST "`cat $ODIR/.ollama_host`"
+setenv OLLAMA_HOST `cat /speed-scratch/$USER/ollama/.ollama_host`
 
 # Sanity check
 ollama -v
@@ -13,14 +20,21 @@ ollama pull llama3.2
 
 # Create a python environment
 setenv ENV_DIR /speed-scratch/$USER/envs/python-env
-mkdir -p $ENV_DIR/{tmp,pkgs,cache}
 
-setenv TMP $ENV_DIR/tmp
-setenv TMPDIR $ENV_DIR/tmp
-setenv PIP_CACHE_DIR $ENV_DIR/cache
+if ( ! -d $ENV_DIR ) then
+    echo "Creating python environment..."
+    mkdir -p $ENV_DIR/{tmp,pkgs,cache}
+
+    setenv TMP $ENV_DIR/tmp
+    setenv TMPDIR $ENV_DIR/tmp
+    setenv PIP_CACHE_DIR $ENV_DIR/cache
+
+    python3 -m venv $ENV_DIR
+else
+    echo "Python environment already exists."
+endif
 
-python3 -m venv $ENV_DIR
 source $ENV_DIR/bin/activate.csh
 pip install -U pip ollama
 
-python ollama_demo.py
+python ollama_demo.py

src/llm-examples/ollama/start_ollama.sh

@@ -24,10 +24,7 @@ setenv PATH $ODIR/bin:$PATH
 setenv OLLAMA_MODELS $ODIR/models
 mkdir -p $OLLAMA_MODELS
 
-# pull a specific model
-ollama pull llama3.2
-
-# Get an available port for the server
+# Ollama by default listens on 127.0.0.1:11434, the port however can be overwritten
 set PORT = `python -c 'import socket,sys; s=socket.socket(); s.bind(("",0)); print(s.getsockname()[1]); s.close()'`
 setenv OLLAMA_HOST 127.0.0.1:$PORT
 echo "http://localhost:$PORT" >! ${ODIR}/.ollama_host
@@ -42,7 +39,7 @@ echo "===================================================="
 echo "To connect from your laptop, open a new terminal and run:"
 
 echo ""
-echo " ssh -L ${PORT}:localhost:${PORT} ${USER}@speed.encs.concordia.ca -t ssh $NODE"
+echo " ssh -L ${PORT}:${NODE}:${PORT} ${USER}@speed.encs.concordia.ca -t ssh $NODE"
 echo ""
 echo "Once connected, set your environment variables:"
 echo "  setenv PATH ${ODIR}/bin:$PATH"
@@ -52,4 +49,4 @@ echo "===================================================="
 echo ""
 
 # Start server
-srun ollama serve
+srun ollama serve