Fix in docker compose functionality for v1-plugin (#185)

Signed-off-by: PatrykWo <[email protected]>

Author: PatrykWo

.cd/Dockerfile.rhel.tenc.pytorch.vllm

@@ -61,7 +61,7 @@ RUN mkdir -p $VLLM_PATH2 && \
 # Install additional Python packages
 RUN pip3 install datasets pandas
 
-# Copy utility scripts and configuration /// to be enabled later PWolsza
+# Copy utility scripts and configuration
 RUN mkdir -p /root/scripts/
 COPY templates /root/scripts/templates/
 COPY entrypoints /root/scripts/entrypoints/

.cd/Dockerfile.ubuntu.pytorch.vllm

@@ -12,8 +12,8 @@ ARG REPO_TYPE=habanalabs
 FROM ${DOCKER_URL}/${VERSION}/${BASE_NAME}/${REPO_TYPE}/pytorch-installer-${PT_VERSION}:${REVISION}
 
 # Parameterize commit/branch for vllm-project & vllm-gaudi checkout
-ARG VLLM_GAUDI_COMMIT=v0.10.1
-ARG VLLM_PROJECT_COMMIT=v0.10.1
+ARG VLLM_GAUDI_COMMIT=v0.10.2_next
+ARG VLLM_PROJECT_COMMIT=v0.10.2
 
 ENV OMPI_MCA_btl_vader_single_copy_mechanism=none
 
@@ -38,18 +38,16 @@ RUN mkdir -p $VLLM_PATH && \
     git fetch upstream --tags || true && \
     git checkout ${VLLM_PROJECT_COMMIT} && \
     bash -c "pip install -r <(sed '/^[torch]/d' requirements/build.txt)" && \
-    VLLM_TARGET_DEVICE=empty pip install --no-build-isolation -e .
+    VLLM_TARGET_DEVICE=empty pip install --no-build-isolation .
 
 # Clone the vllm-gaudi repository and install inside the container
 
 RUN mkdir -p $VLLM_PATH2 && \
     git clone https://github.com/vllm-project/vllm-gaudi.git $VLLM_PATH2 && \
     cd $VLLM_PATH2 && \
-    git checkout ${VLLM_GAUDI_COMMIT} && \
-    VLLM_TARGET_DEVICE=hpu && pip install -v -e $VLLM_PATH2 --no-build-isolation
-
-    # to be enabled later PWolsza
-    # pip install -v -e $VLLM_PATH2/tests/vllm_test_utils
+# Comment: enable if vllm-gaudi release version is used otherwise main
+    git checkout ${VLLM_GAUDI_COMMIT} && \ 
+    VLLM_TARGET_DEVICE=hpu && pip install -v $VLLM_PATH2 --no-build-isolation
 
 # Install additional Python packages
 RUN pip install datasets && \

.cd/server/settings_vllm.csv

@@ -1,5 +1,5 @@
 MODEL,TENSOR_PARALLEL_SIZE,MAX_MODEL_LEN,TOTAL_GPU_MEM,UNAVAILABLE_MEM_ABS,MODEL_MEM_FROM_CONFIG,MODEL_DTYPE,QUANT_DTYPE,MODEL_MEM,PROFILER_MEM_OVERHEAD,APPROX_MEM_PER_GRAPH_MB,fsdpa,GPU_FREE_MEM_TARGET,BLOCK_SIZE,VLLM_PROMPT_BS_BUCKET_MIN,VLLM_PROMPT_BS_BUCKET_STEP,VLLM_DECODE_BS_BUCKET_MIN,VLLM_DECODE_BS_BUCKET_STEP,VLLM_PROMPT_SEQ_BUCKET_MIN,VLLM_PROMPT_SEQ_BUCKET_STEP,VLLM_DECODE_BLOCK_BUCKET_MIN,VLLM_DECODE_BLOCK_BUCKET_STEP,MAX_NUM_PREFILL_SEQS,NUM_HIDDEN_LAYERS,HIDDEN_SIZE,NUM_KEY_VALUE_HEADS,NUM_ATTENTION_HEADS,CACHE_DTYPE_BYTES,LIMIT_MODEL_LEN,PT_HPU_LAZY_MODE,VLLM_DELAYED_SAMPLING,VLLM_SKIP_WARMUP,EXPERIMENTAL_WEIGHT_SHARING,VLLM_EXPONENTIAL_BUCKETING
-meta-llama/Llama-3.1-8B-Instruct,1,4352,128,2,16060522496,2,2,14.95752716,5.5,10,1,1,128,1,32,1,32,128,256,128,256,16,32,4096,8,32,2,131072,1,TRUE,FALSE,0,FALSE
+meta-llama/Llama-3.1-8B-Instruct,1,4352,128,2,16060522496,2,2,14.95752716,5.5,10,1,1,128,1,32,1,32,128,256,128,256,16,32,4096,8,32,2,131072,1,TRUE,FALSE,0,TRUE
 meta-llama/Llama-3.1-70B-Instruct,4,4352,512,2,1.41107E+11,2,2,131.4165192,5.5,10,1,1,128,1,32,1,32,128,256,128,256,16,80,8192,8,64,2,131072,1,TRUE,FALSE,0,FALSE
 meta-llama/Llama-3.3-70B-Instruct,4,4352,512,2,1.41107E+11,2,2,131.4165192,5.5,10,1,1,128,1,32,1,32,128,256,128,256,16,80,8192,8,64,2,131072,1,TRUE,FALSE,0,FALSE
 meta-llama/Llama-3.2-1B-Instruct,1,4352,128,2,2471645608,2,2,2.301899351,5.5,10,1,1,128,1,32,1,32,128,256,128,256,16,16,2048,8,32,2,131072,1,TRUE,FALSE,0,FALSE

.cd/templates/template_vllm_server.sh

@@ -10,9 +10,6 @@ vllm serve $MODEL \
         --download_dir $HF_HOME \
         --max-model-len $MAX_MODEL_LEN \
         --gpu-memory-utilization $GPU_MEM_UTILIZATION \
-        --use-padding-aware-scheduling \
         --max-num-seqs $MAX_NUM_SEQS \
-        --max-num-prefill-seqs $MAX_NUM_PREFILL_SEQS \
-        --num-scheduler-steps 1 \
         --disable-log-requests \
 2>&1 | tee -a  logs/vllm_server.log