epp-deployment.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: {{ include "gateway-api-inference-extension.name" . }}
  namespace: {{ .Release.Namespace }}
  labels:
    {{- include "gateway-api-inference-extension.labels" . | nindent 4 }}
spec:
  {{- if .Values.inferenceExtension.enableLeaderElection }}
  replicas: 3
  {{- else }}
  replicas: 1
  {{- end }}
  strategy:
    # The current recommended EPP deployment pattern is to have a single active replica. This ensures 
    # optimal performance of the stateful operations such prefix cache aware scorer.
    # The Recreate strategy the old replica is killed immediately, and allow the new replica(s) to 
    # quickly take over. This is particularly important in the high availability set up with leader
    # election, as the rolling update strategy would prevent the old leader being killed because 
    # otherwise the maxUnavailable would be 100%.
    type: Recreate
  selector:
    matchLabels:
      {{- include "gateway-api-inference-extension.selectorLabels" . | nindent 6 }}
  template:
    metadata:
      labels:
        {{- include "gateway-api-inference-extension.selectorLabels" . | nindent 8 }}
    spec:
      serviceAccountName: {{ include "gateway-api-inference-extension.name" . }}
      # Conservatively, this timeout should mirror the longest grace period of the pods within the pool
      terminationGracePeriodSeconds: 130
      containers:
      - name: epp
        image: {{ .Values.inferenceExtension.image.hub }}/{{ .Values.inferenceExtension.image.name }}:{{ .Values.inferenceExtension.image.tag }}
        imagePullPolicy: {{ .Values.inferenceExtension.image.pullPolicy | default "Always" }}
        args:
        - --pool-name
        - {{ .Release.Name }}
        {{- if ne .Values.inferencePool.apiVersion "inference.networking.k8s.io" }}
        - --pool-group
        - "{{ (split "/" .Values.inferencePool.apiVersion)._0 }}"
        {{- end }}
        - --zap-encoder
        - "json"
        - --zap-log-level
        - "info"
        - --config-file
        - "/config/{{ .Values.inferenceExtension.pluginsConfigFile }}"
        {{- if eq (.Values.inferencePool.modelServerType | default "vllm") "triton-tensorrt-llm" }}
        - --total-queued-requests-metric
        - "nv_trt_llm_request_metrics{request_type=waiting}"
        - --kv-cache-usage-percentage-metric
        - "nv_trt_llm_kv_cache_block_metrics{kv_cache_block_type=fraction}"
        - --lora-info-metric
        - "" # Set an empty metric to disable LoRA metric scraping as they are not supported by Triton yet.
        {{- end }}
        {{- if .Values.inferenceExtension.enableLeaderElection }}
        - --ha-enable-leader-election
        {{- end }}
        # Pass additional flags via the inferenceExtension.flags field in values.yaml.
        {{- range .Values.inferenceExtension.flags }}
        - "--{{ .name }}"
        - "{{ .value }}"
        {{- end }}
        ports:
        - name: grpc
          containerPort: 9002
        - name: grpc-health
          containerPort: 9003
        - name: metrics
          containerPort: 9090
        {{- if .Values.inferenceExtension.extraContainerPorts }}
        {{- toYaml .Values.inferenceExtension.extraContainerPorts | nindent 8 }}
        {{- end }}
        livenessProbe:
          {{- if .Values.inferenceExtension.enableLeaderElection }}
          grpc:
            port: 9003
            service: liveness
          {{- else }}
          grpc:
            port: 9003
            service: inference-extension
          {{- end }}
          initialDelaySeconds: 5
          periodSeconds: 10
        readinessProbe:
          {{- if .Values.inferenceExtension.enableLeaderElection }}
          grpc:
            port: 9003
            service: readiness
          {{- else }}
          grpc:
            port: 9003
            service: inference-extension
          {{- end }}
          periodSeconds: 2

        env:
        - name: NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        {{- if .Values.inferenceExtension.env }}
        {{- toYaml .Values.inferenceExtension.env | nindent 8 }}
        {{- end }}
        volumeMounts:
        - name: plugins-config-volume
          mountPath: "/config"
      volumes:
      - name: plugins-config-volume
        configMap:
          name: {{ include "gateway-api-inference-extension.name" . }}
      {{- if .Values.inferenceExtension.affinity }}
      affinity:
        {{- toYaml .Values.inferenceExtension.affinity | nindent 8 }}
      {{- end }}
      {{- if .Values.inferenceExtension.tolerations }}
      tolerations:
        {{- toYaml .Values.inferenceExtension.tolerations | nindent 8 }}
      {{- end }}