From d988120157c8956d7ec98c283837d6006e602af6 Mon Sep 17 00:00:00 2001 From: Dmitry Kondratyev Date: Fri, 3 Oct 2025 13:36:41 -0400 Subject: [PATCH 01/17] add parameters to allow loading envoy config from external file --- helm/supersonic/templates/envoy/configmaps.yaml | 12 ++++++++++++ helm/supersonic/values.yaml | 5 +++++ 2 files changed, 17 insertions(+) diff --git a/helm/supersonic/templates/envoy/configmaps.yaml b/helm/supersonic/templates/envoy/configmaps.yaml index 06e80815..904d0e2d 100644 --- a/helm/supersonic/templates/envoy/configmaps.yaml +++ b/helm/supersonic/templates/envoy/configmaps.yaml @@ -1,4 +1,5 @@ {{- /* Define the Envoy configuration in YAML at the top level */}} +{{- /* (it will be used only if envoy.config.load_external_config is false) */}} {{- define "envoy.configuration.yaml" }} {{- with . }} admin: @@ -223,7 +224,18 @@ metadata: app.kubernetes.io/component: envoy data: envoy.yaml: |- +{{- if .Values.envoy.load_external_config }} + {{- if .Values.envoy.external_config_path }} + {{- /* Load configuration from external file */}} + {{ $.Files.Get .Values.envoy.external_config_path }} + {{- else }} + {{- /* Error if load_external_config is true but external_config_path is empty */}} + {{- fail "envoy.external_config_path must be specified when envoy.load_external_config is true" }} + {{- end }} +{{- else }} +{{- /* Use dynamically generated configuration */}} {{ include "envoy.configuration.yaml" $envoyContext | indent 4 }} +{{- end }} --- {{- if .Values.envoy.rate_limiter.prometheus_based.enabled }} diff --git a/helm/supersonic/values.yaml b/helm/supersonic/values.yaml index 6ac4ef07..877908b5 100644 --- a/helm/supersonic/values.yaml +++ b/helm/supersonic/values.yaml @@ -125,6 +125,11 @@ envoy: # -- Arguments for Envoy args: ["--config-path", "/etc/envoy/envoy.yaml", "--log-level", "info", "--log-path", "/dev/stdout"] + # -- If true, load Envoy configuration from an external file instead of generating it dynamically + load_external_config: false + # -- Path to the external Envoy configuration file + external_config_path: "" + # -- Resource requests and limits for Envoy Proxy. # Note: an Envoy Proxy with too many connections might run out of CPU resources: From dd8b111dc5d5482e7f241d3cb1ecef4e4946ccff Mon Sep 17 00:00:00 2001 From: GitHub Actions Date: Fri, 3 Oct 2025 17:37:00 +0000 Subject: [PATCH 02/17] Update JSON schema --- helm/supersonic/values.schema.json | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/helm/supersonic/values.schema.json b/helm/supersonic/values.schema.json index 089419e7..7873aa9a 100644 --- a/helm/supersonic/values.schema.json +++ b/helm/supersonic/values.schema.json @@ -252,6 +252,12 @@ "type": "string" } }, + "load_external_config": { + "type": "boolean" + }, + "external_config_path": { + "type": "string" + }, "resources": { "type": "object", "properties": { @@ -450,10 +456,12 @@ "args", "auth", "enabled", + "external_config_path", "grpc_route_timeout", "image", "ingress", "loadBalancerPolicy", + "load_external_config", "nodeSelector", "rate_limiter", "replicas", From d0ead7da1ad0ec4559f80522f2304c570eda7c03 Mon Sep 17 00:00:00 2001 From: GitHub Actions Date: Fri, 3 Oct 2025 17:37:12 +0000 Subject: [PATCH 03/17] Update helm docs --- docs/.values-table.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/.values-table.md b/docs/.values-table.md index b9082b9b..ed54abce 100644 --- a/docs/.values-table.md +++ b/docs/.values-table.md @@ -27,6 +27,8 @@ | envoy.replicas | int | `1` | Number of Envoy Proxy pods in Deployment | | envoy.image | string | `"envoyproxy/envoy:v1.30.9"` | Envoy Proxy Docker image | | envoy.args | list | `["--config-path","/etc/envoy/envoy.yaml","--log-level","info","--log-path","/dev/stdout"]` | Arguments for Envoy | +| envoy.load_external_config | bool | `false` | If true, load Envoy configuration from an external file instead of generating it dynamically | +| envoy.external_config_path | string | `""` | Path to the external Envoy configuration file | | envoy.resources | object | `{"limits":{"cpu":8,"memory":"4G"},"requests":{"cpu":1,"memory":"2G"}}` | Resource requests and limits for Envoy Proxy. Note: an Envoy Proxy with too many connections might run out of CPU | | envoy.annotations | object | `{}` | Annotations for Envoy pods | | envoy.nodeSelector | object | `{}` | Node selector for Envoy pods | From 4ff985205e21f0ba960aeb7340ebb3d2257b3556 Mon Sep 17 00:00:00 2001 From: Dmitry Kondratyev Date: Fri, 3 Oct 2025 14:02:58 -0400 Subject: [PATCH 04/17] add a test CI workflow --- .github/workflows/ci-external-config.yaml | 118 ++++++++++++++++++++++ tests/envoy-config-test.yaml | 57 +++++++++++ tests/values-external-envoy-config.yaml | 42 ++++++++ 3 files changed, 217 insertions(+) create mode 100644 .github/workflows/ci-external-config.yaml create mode 100644 tests/envoy-config-test.yaml create mode 100644 tests/values-external-envoy-config.yaml diff --git a/.github/workflows/ci-external-config.yaml b/.github/workflows/ci-external-config.yaml new file mode 100644 index 00000000..474d3e2e --- /dev/null +++ b/.github/workflows/ci-external-config.yaml @@ -0,0 +1,118 @@ +name: ci [external-config] + +on: + push: + branches: + - "**" + pull_request: + branches: + - "main" + +jobs: + deploy-at-github: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Set up Kubernetes cluster with Kind + uses: helm/kind-action@v1.6.0 + with: + cluster_name: gh-k8s-cluster + + - name: Set up Helm + uses: azure/setup-helm@v3 + with: + version: v3.12.0 + + - name: Create CMS namespace + run: | + kubectl create namespace cms + + - name: Install Prometheus Operator CRDs + run: | + helm repo add prometheus-community https://prometheus-community.github.io/helm-charts + helm repo update + kubectl create namespace monitoring + helm install prometheus-operator prometheus-community/kube-prometheus-stack --namespace monitoring --set prometheusOperator.createCustomResource=false --set defaultRules.create=false --set alertmanager.enabled=false --set prometheus.enabled=false --set grafana.enabled=false + + - name: Install KEDA Autoscaler + run: | + helm repo add kedacore https://kedacore.github.io/charts + helm repo update + kubectl create namespace keda + helm install keda kedacore/keda --namespace keda + + - name: Mount CVMFS + run: | + kubectl create namespace cvmfs-csi + helm install -n cvmfs-csi cvmfs-csi oci://registry.cern.ch/kubernetes/charts/cvmfs-csi --values cvmfs/values-cvmfs-csi.yaml + kubectl apply -f cvmfs/cvmfs-storageclass.yaml -n cvmfs-csi + + - name: Deploy Helm chart with external Envoy config + run: | + helm repo add grafana https://grafana.github.io/helm-charts + helm repo add opentelemetry https://open-telemetry.github.io/opentelemetry-helm-charts + helm dependency build ./helm/supersonic + helm upgrade --install supersonic ./helm/supersonic \ + --values tests/values-external-envoy-config.yaml -n cms + + - name: Validate Envoy ConfigMap contains external config + run: | + echo "Checking that Envoy ConfigMap contains external configuration..." + ENVOY_CONFIG=$(kubectl get configmap supersonic-envoy-config -n cms -o jsonpath="{.data.envoy\.yaml}") + + echo "✓ Success!" + + - name: CVMFS Mount ready + run: | + kubectl wait --for condition=Ready pod --all -n cvmfs-csi --timeout 120s + + - name: Envoy proxy ready + run: | + kubectl wait --for condition=Ready pod -l app.kubernetes.io/component=envoy --timeout 120s -n cms + + - name: Prometheus ready + run: | + kubectl wait --for condition=Ready pod -l app.kubernetes.io/name=prometheus --timeout 120s -n cms + kubectl get svc,pod -l app.kubernetes.io/name=prometheus -n cms + + - name: Grafana ready + run: | + kubectl wait --for condition=Ready pod -l app.kubernetes.io/name=grafana --timeout 120s -n cms + + - name: OpenTelemetry Collector ready + run: | + kubectl wait --for condition=Ready pod -l app.kubernetes.io/name=opentelemetry-collector --timeout 120s -n cms + + - name: Tempo ready + run: | + kubectl wait --for condition=Ready pod -l app.kubernetes.io/name=tempo --timeout 300s -n cms + + - name: Autoscaler ready + run: | + kubectl wait --for condition=AbleToScale hpa -l app.kubernetes.io/component=keda --timeout 180s -n cms + kubectl wait --for condition=Ready so -l app.kubernetes.io/component=keda --timeout 180s -n cms + + - name: Triton server ready + run: | + kubectl describe pod -l app.kubernetes.io/component=triton -n cms + kubectl wait --for condition=Ready pod -l app.kubernetes.io/component=triton --timeout 500s -n cms + + - name: Validate Deployment + run: | + kubectl get all -n cms + + - name: Run Perf Analyzer Job + run: | + kubectl apply -f tests/perf-analyzer-job-ci.yaml + kubectl wait --for=condition=complete job/perf-analyzer-job -n cms --timeout=300s || \ + (echo "Perf-analyzer job did not complete in time or failed." && exit 1) + + POD_NAME=$(kubectl get pods -n cms -l job-name=perf-analyzer-job -o jsonpath="{.items[0].metadata.name}") + echo "========== Perf Analyzer Logs ==========" + kubectl logs -n cms "$POD_NAME" + echo "========================================" + + - name: Cleanup + run: kind delete cluster --name gh-k8s-cluster diff --git a/tests/envoy-config-test.yaml b/tests/envoy-config-test.yaml new file mode 100644 index 00000000..4bdbc2c5 --- /dev/null +++ b/tests/envoy-config-test.yaml @@ -0,0 +1,57 @@ +# Test external Envoy configuration file for CI/CD testing +# This is a minimal configuration to validate external config loading works + +admin: + access_log_path: /tmp/admin_access.log + address: + socket_address: + address: 0.0.0.0 + port_value: 9901 + +static_resources: + listeners: + - name: listener_grpc + address: + socket_address: + address: 0.0.0.0 + port_value: 8001 + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + generate_request_id: true + stat_prefix: ingress_grpc + codec_type: AUTO + route_config: + name: local_route_grpc + virtual_hosts: + - name: backend_grpc + domains: ["*"] + routes: + - match: + prefix: "/" + route: + cluster: triton_grpc_service + timeout: 0s + http_filters: + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + http2_protocol_options: {} + + clusters: + - name: triton_grpc_service + connect_timeout: 0.25s + type: STRICT_DNS + lb_policy: ROUND_ROBIN + http2_protocol_options: {} + load_assignment: + cluster_name: triton_grpc_service + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: supersonic-triton + port_value: 8001 diff --git a/tests/values-external-envoy-config.yaml b/tests/values-external-envoy-config.yaml new file mode 100644 index 00000000..561c1694 --- /dev/null +++ b/tests/values-external-envoy-config.yaml @@ -0,0 +1,42 @@ +# Test values for external Envoy configuration testing +# This file is used by CI to validate external config loading + +# Enable Envoy with external configuration +envoy: + enabled: true + # Test external config loading + load_external_config: true + external_config_path: "tests/envoy-config-test.yaml" + replicas: 1 + image: "envoyproxy/envoy:v1.30.9" + +# Enable minimal Triton service for testing +triton: + replicas: 1 + image: "nvcr.io/nvidia/tritonserver:24.12-py3-min" + command: ["/bin/sh", "-c"] + args: + - | + /opt/tritonserver/bin/tritonserver \ + --model-repository=/tmp/ \ + --log-verbose=0 \ + --exit-timeout-secs=60 + +# Disable optional components for faster testing +prometheus: + enabled: false + +grafana: + enabled: false + +keda: + enabled: false + +tempo: + enabled: false + +opentelemetry-collector: + enabled: false + +metricsCollector: + enabled: false From 9fb19d3df15428df7b135308322d0d1c6d0adce5 Mon Sep 17 00:00:00 2001 From: Dmitry Kondratyev Date: Fri, 3 Oct 2025 14:07:21 -0400 Subject: [PATCH 05/17] don't check for disabled components in CI --- .github/workflows/ci-external-config.yaml | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/.github/workflows/ci-external-config.yaml b/.github/workflows/ci-external-config.yaml index 474d3e2e..532fc386 100644 --- a/.github/workflows/ci-external-config.yaml +++ b/.github/workflows/ci-external-config.yaml @@ -72,28 +72,6 @@ jobs: run: | kubectl wait --for condition=Ready pod -l app.kubernetes.io/component=envoy --timeout 120s -n cms - - name: Prometheus ready - run: | - kubectl wait --for condition=Ready pod -l app.kubernetes.io/name=prometheus --timeout 120s -n cms - kubectl get svc,pod -l app.kubernetes.io/name=prometheus -n cms - - - name: Grafana ready - run: | - kubectl wait --for condition=Ready pod -l app.kubernetes.io/name=grafana --timeout 120s -n cms - - - name: OpenTelemetry Collector ready - run: | - kubectl wait --for condition=Ready pod -l app.kubernetes.io/name=opentelemetry-collector --timeout 120s -n cms - - - name: Tempo ready - run: | - kubectl wait --for condition=Ready pod -l app.kubernetes.io/name=tempo --timeout 300s -n cms - - - name: Autoscaler ready - run: | - kubectl wait --for condition=AbleToScale hpa -l app.kubernetes.io/component=keda --timeout 180s -n cms - kubectl wait --for condition=Ready so -l app.kubernetes.io/component=keda --timeout 180s -n cms - - name: Triton server ready run: | kubectl describe pod -l app.kubernetes.io/component=triton -n cms From c439d481c000ac184b86562c349b8098a331b808 Mon Sep 17 00:00:00 2001 From: Dmitry Kondratyev Date: Fri, 3 Oct 2025 14:18:17 -0400 Subject: [PATCH 06/17] switch image in CI workflow --- tests/values-external-envoy-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/values-external-envoy-config.yaml b/tests/values-external-envoy-config.yaml index 561c1694..582e812e 100644 --- a/tests/values-external-envoy-config.yaml +++ b/tests/values-external-envoy-config.yaml @@ -13,7 +13,7 @@ envoy: # Enable minimal Triton service for testing triton: replicas: 1 - image: "nvcr.io/nvidia/tritonserver:24.12-py3-min" + image: "fastml/triton-torchgeo:22.07-py3-geometric" command: ["/bin/sh", "-c"] args: - | From 5f0dfda682e6aed81c35ddd49bdaddc4b874ba70 Mon Sep 17 00:00:00 2001 From: Dmitry Kondratyev Date: Fri, 3 Oct 2025 14:52:31 -0400 Subject: [PATCH 07/17] fix CI values file --- tests/values-external-envoy-config.yaml | 33 ++++++++++++++----------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/tests/values-external-envoy-config.yaml b/tests/values-external-envoy-config.yaml index 582e812e..a9b07743 100644 --- a/tests/values-external-envoy-config.yaml +++ b/tests/values-external-envoy-config.yaml @@ -1,28 +1,33 @@ # Test values for external Envoy configuration testing -# This file is used by CI to validate external config loading -# Enable Envoy with external configuration -envoy: - enabled: true - # Test external config loading - load_external_config: true - external_config_path: "tests/envoy-config-test.yaml" - replicas: 1 - image: "envoyproxy/envoy:v1.30.9" - -# Enable minimal Triton service for testing triton: replicas: 1 - image: "fastml/triton-torchgeo:22.07-py3-geometric" + image: fastml/triton-torchgeo:22.07-py3-geometric # works for CMSSW run3 command: ["/bin/sh", "-c"] args: - | /opt/tritonserver/bin/tritonserver \ - --model-repository=/tmp/ \ + --model-repository=/cvmfs/cms.cern.ch/el9_amd64_gcc12/cms/cmssw/CMSSW_14_1_0_pre7/external/el9_amd64_gcc12/data/RecoBTag/Combined/data/models/ \ --log-verbose=0 \ + --disable-auto-complete-config \ --exit-timeout-secs=60 + resources: + limits: { cpu: 1, memory: 3Gi} + requests: { cpu: 1, memory: 1Gi} + modelRepository: + enabled: true + storageType: cvmfs-pvc + mountPath: /cvmfs + readinessProbe: + reset: true + +envoy: + enabled: true + load_external_config: true + external_config_path: "tests/envoy-config-test.yaml" + replicas: 1 + image: "envoyproxy/envoy:v1.30.9" -# Disable optional components for faster testing prometheus: enabled: false From b1fd1370157369788ada767c228ccdfdeed747ab Mon Sep 17 00:00:00 2001 From: Dmitry Kondratyev Date: Fri, 3 Oct 2025 15:57:25 -0400 Subject: [PATCH 08/17] load values from ConfigMap rather than bare YAML file --- .github/workflows/ci-external-config.yaml | 27 +++- .../templates/envoy/configmaps.yaml | 28 +++- helm/supersonic/values.yaml | 12 +- tests/envoy-config-test.yaml | 126 +++++++++++------- tests/values-external-envoy-config.yaml | 8 +- 5 files changed, 135 insertions(+), 66 deletions(-) diff --git a/.github/workflows/ci-external-config.yaml b/.github/workflows/ci-external-config.yaml index 532fc386..4c2d07e7 100644 --- a/.github/workflows/ci-external-config.yaml +++ b/.github/workflows/ci-external-config.yaml @@ -49,6 +49,10 @@ jobs: helm install -n cvmfs-csi cvmfs-csi oci://registry.cern.ch/kubernetes/charts/cvmfs-csi --values cvmfs/values-cvmfs-csi.yaml kubectl apply -f cvmfs/cvmfs-storageclass.yaml -n cvmfs-csi + - name: Create external Envoy ConfigMap + run: | + kubectl apply -f tests/envoy-config-test.yaml -n cms + - name: Deploy Helm chart with external Envoy config run: | helm repo add grafana https://grafana.github.io/helm-charts @@ -62,7 +66,28 @@ jobs: echo "Checking that Envoy ConfigMap contains external configuration..." ENVOY_CONFIG=$(kubectl get configmap supersonic-envoy-config -n cms -o jsonpath="{.data.envoy\.yaml}") - echo "✓ Success!" + # Check for key indicators that external config was loaded + if echo "$ENVOY_CONFIG" | grep -q "cms_local_rate_limiter"; then + echo "✓ External config loaded: Found rate limiter configuration" + else + echo "✗ External config test failed: Rate limiter not found in config" + exit 1 + fi + + if echo "$ENVOY_CONFIG" | grep -q "local_ratelimit"; then + echo "✓ External config loaded: Found local rate limit filter" + else + echo "✗ External config test failed: Local rate limit filter not found" + exit 1 + fi + + # Verify config doesn't contain dynamic generation markers + if echo "$ENVOY_CONFIG" | grep -q ".Values.envoy"; then + echo "✗ External config test failed: Dynamic template markers found in config" + exit 1 + fi + + echo "✓ All external config validation checks passed" - name: CVMFS Mount ready run: | diff --git a/helm/supersonic/templates/envoy/configmaps.yaml b/helm/supersonic/templates/envoy/configmaps.yaml index 904d0e2d..d9fdc8da 100644 --- a/helm/supersonic/templates/envoy/configmaps.yaml +++ b/helm/supersonic/templates/envoy/configmaps.yaml @@ -224,13 +224,29 @@ metadata: app.kubernetes.io/component: envoy data: envoy.yaml: |- -{{- if .Values.envoy.load_external_config }} - {{- if .Values.envoy.external_config_path }} - {{- /* Load configuration from external file */}} - {{ $.Files.Get .Values.envoy.external_config_path }} +{{- if .Values.envoy.external_config.load_from_configmap }} + {{- if .Values.envoy.external_config.configmap_name }} + {{- /* Load configuration from external ConfigMap */}} + {{- $configmapName := .Values.envoy.external_config.configmap_name }} + {{- $dataKey := .Values.envoy.external_config.configmap_key | default "envoy.yaml" }} + {{- $externalConfig := (lookup "v1" "ConfigMap" .Release.Namespace $configmapName) }} + {{- if $externalConfig }} + {{- if $externalConfig.data }} + {{- if hasKey $externalConfig.data $dataKey }} + {{- /* Use the data from the external ConfigMap */}} + {{- printf "%s" (get $externalConfig.data $dataKey) }} {{- else }} - {{- /* Error if load_external_config is true but external_config_path is empty */}} - {{- fail "envoy.external_config_path must be specified when envoy.load_external_config is true" }} + {{- fail (printf "Expected key '%s' not found in ConfigMap '%s/%s'" $dataKey .Release.Namespace $configmapName) }} + {{- end }} + {{- else }} + {{- fail (printf "No data found in ConfigMap '%s/%s'" .Release.Namespace $configmapName) }} + {{- end }} + {{- else }} + {{- fail (printf "External ConfigMap '%s/%s' not found. Please ensure the ConfigMap exists." .Release.Namespace $configmapName) }} + {{- end }} + {{- else }} + {{- /* Error if load_from_configmap is true but configmap_name is empty */}} + {{- fail "envoy.external_config.configmap_name must be specified when envoy.external_config.load_from_configmap is true" }} {{- end }} {{- else }} {{- /* Use dynamically generated configuration */}} diff --git a/helm/supersonic/values.yaml b/helm/supersonic/values.yaml index 877908b5..ca3e2266 100644 --- a/helm/supersonic/values.yaml +++ b/helm/supersonic/values.yaml @@ -125,10 +125,14 @@ envoy: # -- Arguments for Envoy args: ["--config-path", "/etc/envoy/envoy.yaml", "--log-level", "info", "--log-path", "/dev/stdout"] - # -- If true, load Envoy configuration from an external file instead of generating it dynamically - load_external_config: false - # -- Path to the external Envoy configuration file - external_config_path: "" + # -- External Envoy configuration settings + external_config: + # -- If true, load Envoy configuration from an external ConfigMap instead of generating it dynamically + load_from_configmap: false + # -- Name of the external ConfigMap containing the Envoy configuration + configmap_name: "" + # -- Key name in the external ConfigMap (defaults to "envoy.yaml") + configmap_key: "" # -- Resource requests and limits for Envoy Proxy. # Note: an Envoy Proxy with too many connections might run out of CPU diff --git a/tests/envoy-config-test.yaml b/tests/envoy-config-test.yaml index 4bdbc2c5..3b56daac 100644 --- a/tests/envoy-config-test.yaml +++ b/tests/envoy-config-test.yaml @@ -1,57 +1,81 @@ -# Test external Envoy configuration file for CI/CD testing -# This is a minimal configuration to validate external config loading works +# External ConfigMap containing Envoy configuration for CI testing +# This ConfigMap will be used to test loading external Envoy configuration -admin: - access_log_path: /tmp/admin_access.log - address: - socket_address: - address: 0.0.0.0 - port_value: 9901 - -static_resources: - listeners: - - name: listener_grpc +apiVersion: v1 +kind: ConfigMap +metadata: + name: external-envoy-config + namespace: cms # Will be created in CI + labels: + app.kubernetes.io/name: envoy-external-config + app.kubernetes.io/instance: test +data: + envoy.yaml: | + # External Envoy configuration for CI testing + admin: + access_log_path: /tmp/admin_access.log address: socket_address: address: 0.0.0.0 - port_value: 8001 - filter_chains: - - filters: - - name: envoy.filters.network.http_connection_manager + port_value: 9901 + + static_resources: + listeners: + - name: listener_grpc + address: + socket_address: + address: 0.0.0.0 + port_value: 8001 + + # Enable listener-level rate limiting for testing + listener_filters: + - name: envoy.filters.listener.local_ratelimit typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager - generate_request_id: true - stat_prefix: ingress_grpc - codec_type: AUTO - route_config: - name: local_route_grpc - virtual_hosts: - - name: backend_grpc - domains: ["*"] - routes: - - match: - prefix: "/" - route: - cluster: triton_grpc_service - timeout: 0s - http_filters: - - name: envoy.filters.http.router - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router - http2_protocol_options: {} + "@type": type.googleapis.com/envoy.extensions.filters.listener.local_ratelimit.v3.LocalRateLimit + stat_prefix: cms_local_rate_limiter + token_bucket: + max_tokens: 5 + tokens_per_fill: 1 + fill_interval: 12s + + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + generate_request_id: true + stat_prefix: ingress_grpc + codec_type: AUTO + route_config: + name: local_route_grpc + virtual_hosts: + - name: backend_grpc + domains: ["*"] + routes: + - match: + prefix: "/" + route: + cluster: triton_grpc_service + timeout: 0s + + http_filters: + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + http2_protocol_options: {} - clusters: - - name: triton_grpc_service - connect_timeout: 0.25s - type: STRICT_DNS - lb_policy: ROUND_ROBIN - http2_protocol_options: {} - load_assignment: - cluster_name: triton_grpc_service - endpoints: - - lb_endpoints: - - endpoint: - address: - socket_address: - address: supersonic-triton - port_value: 8001 + clusters: + - name: triton_grpc_service + connect_timeout: 0.25s + type: STRICT_DNS + lb_policy: LEAST_REQUEST + http2_protocol_options: {} + load_assignment: + cluster_name: triton_grpc_service + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: supersonic-triton + port_value: 8001 \ No newline at end of file diff --git a/tests/values-external-envoy-config.yaml b/tests/values-external-envoy-config.yaml index a9b07743..a01d92e9 100644 --- a/tests/values-external-envoy-config.yaml +++ b/tests/values-external-envoy-config.yaml @@ -23,10 +23,10 @@ triton: envoy: enabled: true - load_external_config: true - external_config_path: "tests/envoy-config-test.yaml" - replicas: 1 - image: "envoyproxy/envoy:v1.30.9" + external_config: + load_from_configmap: true + configmap_name: "external-envoy-config" + configmap_key: "envoy.yaml" # Default key name prometheus: enabled: false From 328eb5b2c9489e60abeb0b68812509af9ee58795 Mon Sep 17 00:00:00 2001 From: GitHub Actions Date: Fri, 3 Oct 2025 19:57:39 +0000 Subject: [PATCH 09/17] Update JSON schema --- helm/supersonic/values.schema.json | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/helm/supersonic/values.schema.json b/helm/supersonic/values.schema.json index 7873aa9a..993c0792 100644 --- a/helm/supersonic/values.schema.json +++ b/helm/supersonic/values.schema.json @@ -252,11 +252,24 @@ "type": "string" } }, - "load_external_config": { - "type": "boolean" - }, - "external_config_path": { - "type": "string" + "external_config": { + "type": "object", + "properties": { + "load_from_configmap": { + "type": "boolean" + }, + "configmap_name": { + "type": "string" + }, + "configmap_key": { + "type": "string" + } + }, + "required": [ + "configmap_key", + "configmap_name", + "load_from_configmap" + ] }, "resources": { "type": "object", @@ -456,12 +469,11 @@ "args", "auth", "enabled", - "external_config_path", + "external_config", "grpc_route_timeout", "image", "ingress", "loadBalancerPolicy", - "load_external_config", "nodeSelector", "rate_limiter", "replicas", From 306f89821e91890694f19231868fc4f9c192b6d3 Mon Sep 17 00:00:00 2001 From: GitHub Actions Date: Fri, 3 Oct 2025 19:57:57 +0000 Subject: [PATCH 10/17] Update helm docs --- docs/.values-table.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/.values-table.md b/docs/.values-table.md index ed54abce..78b6fe85 100644 --- a/docs/.values-table.md +++ b/docs/.values-table.md @@ -27,8 +27,10 @@ | envoy.replicas | int | `1` | Number of Envoy Proxy pods in Deployment | | envoy.image | string | `"envoyproxy/envoy:v1.30.9"` | Envoy Proxy Docker image | | envoy.args | list | `["--config-path","/etc/envoy/envoy.yaml","--log-level","info","--log-path","/dev/stdout"]` | Arguments for Envoy | -| envoy.load_external_config | bool | `false` | If true, load Envoy configuration from an external file instead of generating it dynamically | -| envoy.external_config_path | string | `""` | Path to the external Envoy configuration file | +| envoy.external_config | object | `{"configmap_key":"","configmap_name":"","load_from_configmap":false}` | External Envoy configuration settings | +| envoy.external_config.load_from_configmap | bool | `false` | If true, load Envoy configuration from an external ConfigMap instead of generating it dynamically | +| envoy.external_config.configmap_name | string | `""` | Name of the external ConfigMap containing the Envoy configuration | +| envoy.external_config.configmap_key | string | `""` | Key name in the external ConfigMap (defaults to "envoy.yaml") | | envoy.resources | object | `{"limits":{"cpu":8,"memory":"4G"},"requests":{"cpu":1,"memory":"2G"}}` | Resource requests and limits for Envoy Proxy. Note: an Envoy Proxy with too many connections might run out of CPU | | envoy.annotations | object | `{}` | Annotations for Envoy pods | | envoy.nodeSelector | object | `{}` | Node selector for Envoy pods | From 94b332850e45e0a95665643a6654c5614a2fe6e1 Mon Sep 17 00:00:00 2001 From: Dmitry Kondratyev Date: Fri, 3 Oct 2025 16:05:51 -0400 Subject: [PATCH 11/17] cleanup test configs --- tests/envoy-config-test.yaml | 8 +------- tests/values-external-envoy-config.yaml | 4 ++-- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/tests/envoy-config-test.yaml b/tests/envoy-config-test.yaml index 3b56daac..00ef0dac 100644 --- a/tests/envoy-config-test.yaml +++ b/tests/envoy-config-test.yaml @@ -1,14 +1,8 @@ -# External ConfigMap containing Envoy configuration for CI testing -# This ConfigMap will be used to test loading external Envoy configuration - apiVersion: v1 kind: ConfigMap metadata: name: external-envoy-config - namespace: cms # Will be created in CI - labels: - app.kubernetes.io/name: envoy-external-config - app.kubernetes.io/instance: test + namespace: cms data: envoy.yaml: | # External Envoy configuration for CI testing diff --git a/tests/values-external-envoy-config.yaml b/tests/values-external-envoy-config.yaml index a01d92e9..bcff6586 100644 --- a/tests/values-external-envoy-config.yaml +++ b/tests/values-external-envoy-config.yaml @@ -25,8 +25,8 @@ envoy: enabled: true external_config: load_from_configmap: true - configmap_name: "external-envoy-config" - configmap_key: "envoy.yaml" # Default key name + configmap_name: external-envoy-config + configmap_key: envoy.yaml prometheus: enabled: false From 9331e11dea71054d6ada8f3396c7bd657827205c Mon Sep 17 00:00:00 2001 From: Dmitry Kondratyev Date: Fri, 3 Oct 2025 16:09:32 -0400 Subject: [PATCH 12/17] cleanup CI workflow --- .github/workflows/ci-external-config.yaml | 28 ----------------------- 1 file changed, 28 deletions(-) diff --git a/.github/workflows/ci-external-config.yaml b/.github/workflows/ci-external-config.yaml index 4c2d07e7..39eaed48 100644 --- a/.github/workflows/ci-external-config.yaml +++ b/.github/workflows/ci-external-config.yaml @@ -61,34 +61,6 @@ jobs: helm upgrade --install supersonic ./helm/supersonic \ --values tests/values-external-envoy-config.yaml -n cms - - name: Validate Envoy ConfigMap contains external config - run: | - echo "Checking that Envoy ConfigMap contains external configuration..." - ENVOY_CONFIG=$(kubectl get configmap supersonic-envoy-config -n cms -o jsonpath="{.data.envoy\.yaml}") - - # Check for key indicators that external config was loaded - if echo "$ENVOY_CONFIG" | grep -q "cms_local_rate_limiter"; then - echo "✓ External config loaded: Found rate limiter configuration" - else - echo "✗ External config test failed: Rate limiter not found in config" - exit 1 - fi - - if echo "$ENVOY_CONFIG" | grep -q "local_ratelimit"; then - echo "✓ External config loaded: Found local rate limit filter" - else - echo "✗ External config test failed: Local rate limit filter not found" - exit 1 - fi - - # Verify config doesn't contain dynamic generation markers - if echo "$ENVOY_CONFIG" | grep -q ".Values.envoy"; then - echo "✗ External config test failed: Dynamic template markers found in config" - exit 1 - fi - - echo "✓ All external config validation checks passed" - - name: CVMFS Mount ready run: | kubectl wait --for condition=Ready pod --all -n cvmfs-csi --timeout 120s From 192aa6a77ea4509a46036d46de1d3bafbedcd4a6 Mon Sep 17 00:00:00 2001 From: Dmitry Kondratyev Date: Fri, 3 Oct 2025 16:19:34 -0400 Subject: [PATCH 13/17] update envoy conifg --- tests/envoy-config-test.yaml | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/tests/envoy-config-test.yaml b/tests/envoy-config-test.yaml index 00ef0dac..28784c8e 100644 --- a/tests/envoy-config-test.yaml +++ b/tests/envoy-config-test.yaml @@ -21,17 +21,6 @@ data: address: 0.0.0.0 port_value: 8001 - # Enable listener-level rate limiting for testing - listener_filters: - - name: envoy.filters.listener.local_ratelimit - typed_config: - "@type": type.googleapis.com/envoy.extensions.filters.listener.local_ratelimit.v3.LocalRateLimit - stat_prefix: cms_local_rate_limiter - token_bucket: - max_tokens: 5 - tokens_per_fill: 1 - fill_interval: 12s - filter_chains: - filters: - name: envoy.filters.network.http_connection_manager @@ -71,5 +60,5 @@ data: - endpoint: address: socket_address: - address: supersonic-triton + address: supersonic-dev-triton port_value: 8001 \ No newline at end of file From 23d03b538fd0e613eb5efd6f1dfd724f03ff1239 Mon Sep 17 00:00:00 2001 From: Dmitry Kondratyev Date: Fri, 3 Oct 2025 16:22:48 -0400 Subject: [PATCH 14/17] update envoy config --- tests/envoy-config-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/envoy-config-test.yaml b/tests/envoy-config-test.yaml index 28784c8e..793e249e 100644 --- a/tests/envoy-config-test.yaml +++ b/tests/envoy-config-test.yaml @@ -60,5 +60,5 @@ data: - endpoint: address: socket_address: - address: supersonic-dev-triton + address: supersonic-triton port_value: 8001 \ No newline at end of file From 4f71d54c1b51191bebdf1b17b7db66dc362a0aa7 Mon Sep 17 00:00:00 2001 From: Dmitry Kondratyev Date: Fri, 3 Oct 2025 16:26:01 -0400 Subject: [PATCH 15/17] add if-else statements around original envoy configs --- helm/supersonic/templates/envoy/configmaps.yaml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/helm/supersonic/templates/envoy/configmaps.yaml b/helm/supersonic/templates/envoy/configmaps.yaml index d9fdc8da..d78b78fe 100644 --- a/helm/supersonic/templates/envoy/configmaps.yaml +++ b/helm/supersonic/templates/envoy/configmaps.yaml @@ -1,5 +1,5 @@ {{- /* Define the Envoy configuration in YAML at the top level */}} -{{- /* (it will be used only if envoy.config.load_external_config is false) */}} +{{- /* (it will be used only if envoy.external_config.load_from_configmap is false) */}} {{- define "envoy.configuration.yaml" }} {{- with . }} admin: @@ -183,6 +183,8 @@ static_resources: {{- /* Begin iterating over servers */}} {{- if .Values.envoy.enabled }} +{{- if not .Values.envoy.external_config.load_from_configmap }} +{{- /* Only prepare dynamic configuration context when not using external config */}} {{- /* Define variables for ports */}} {{- $tritonGrpcPort := "" -}} {{- $envoyGrpcPort := "" -}} @@ -213,6 +215,10 @@ static_resources: "tritonName" $tritonName "root" . }} +{{- else }} +{{- /* Create empty context for external config case to avoid undefined variable errors */}} +{{- $envoyContext := dict }} +{{- end }} apiVersion: v1 kind: ConfigMap @@ -254,7 +260,7 @@ data: {{- end }} --- -{{- if .Values.envoy.rate_limiter.prometheus_based.enabled }} +{{- if and .Values.envoy.rate_limiter.prometheus_based.enabled (not .Values.envoy.external_config.load_from_configmap) }} {{- /* Create a ConfigMap for the Lua filter */}} apiVersion: v1 kind: ConfigMap From 958e8d2f0e7377ea179692b2ad82e858f5860048 Mon Sep 17 00:00:00 2001 From: Dmitry Kondratyev Date: Fri, 3 Oct 2025 16:32:25 -0400 Subject: [PATCH 16/17] fix undefined variable --- helm/supersonic/templates/envoy/configmaps.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/helm/supersonic/templates/envoy/configmaps.yaml b/helm/supersonic/templates/envoy/configmaps.yaml index d78b78fe..2788e9b2 100644 --- a/helm/supersonic/templates/envoy/configmaps.yaml +++ b/helm/supersonic/templates/envoy/configmaps.yaml @@ -183,6 +183,9 @@ static_resources: {{- /* Begin iterating over servers */}} {{- if .Values.envoy.enabled }} +{{- /* Initialize envoyContext variable */}} +{{- $envoyContext := dict }} + {{- if not .Values.envoy.external_config.load_from_configmap }} {{- /* Only prepare dynamic configuration context when not using external config */}} {{- /* Define variables for ports */}} @@ -205,7 +208,7 @@ static_resources: {{- $tritonName := include "supersonic.tritonName" . }} {{- /* Create a context to pass to the template */}} -{{- $envoyContext := dict +{{- $envoyContext = dict "envoyAdminPort" $envoyAdminPort "envoyGrpcPort" $envoyGrpcPort "tritonGrpcPort" $tritonGrpcPort @@ -215,9 +218,6 @@ static_resources: "tritonName" $tritonName "root" . }} -{{- else }} -{{- /* Create empty context for external config case to avoid undefined variable errors */}} -{{- $envoyContext := dict }} {{- end }} apiVersion: v1 From 12a426219d7a2088ecb24ec96fed107d737192a0 Mon Sep 17 00:00:00 2001 From: Dmitry Kondratyev Date: Fri, 3 Oct 2025 16:36:58 -0400 Subject: [PATCH 17/17] fix indentation --- helm/supersonic/templates/envoy/configmaps.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm/supersonic/templates/envoy/configmaps.yaml b/helm/supersonic/templates/envoy/configmaps.yaml index 2788e9b2..63b2132e 100644 --- a/helm/supersonic/templates/envoy/configmaps.yaml +++ b/helm/supersonic/templates/envoy/configmaps.yaml @@ -240,7 +240,7 @@ data: {{- if $externalConfig.data }} {{- if hasKey $externalConfig.data $dataKey }} {{- /* Use the data from the external ConfigMap */}} - {{- printf "%s" (get $externalConfig.data $dataKey) }} + {{- (get $externalConfig.data $dataKey) | nindent 4 }} {{- else }} {{- fail (printf "Expected key '%s' not found in ConfigMap '%s/%s'" $dataKey .Release.Namespace $configmapName) }} {{- end }}