Add telemetry support

This commit enables the pulpcore API metrics collection. closes #1006 Co-authored-by: André "decko" de Brito <[email protected]>
pulp · Jul 25, 2023 · d69e413 · d69e413
1 parent d21b05d
commit d69e413
Show file tree

Hide file tree

Showing 21 changed files with 587 additions and 40 deletions.
diff --git a/.ci/scripts/telemetry.sh b/.ci/scripts/telemetry.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -eu
+
+DEPLOYMENT_NAME="example-pulp-api"
+
+
+echo "Validating otel-collector-config file ..."
+kubectl exec -c otel-collector-sidecar deployment/$DEPLOYMENT_NAME -- /otelcol validate --config=/etc/otelcol-contrib/otel-collector-config.yaml
+
+echo "Verifying if metrics service is available ..."
+HTTP_STATUS=$(kubectl exec deployment/$DEPLOYMENT_NAME -- curl -sw "%{http_code}" -o /dev/null localhost:8889/metrics)
+if [[ "$HTTP_STATUS" != 200 ]]; then
+  echo "error: ${HTTP_STATUS}"
+  exit 1
+fi
+
+echo "Verifying if \"http_server\" string is found in metrics endpoint ..."
+kubectl exec deployment/$DEPLOYMENT_NAME -- curl -s localhost:8889/metrics | grep http_server  &>/dev/null
+
+echo "Telemetry ok"
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -229,8 +229,9 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - INGRESS_TYPE: ingress
-          - INGRESS_TYPE: nodeport
+          - COMPONENT_TYPE: ingress
+          - COMPONENT_TYPE: nodeport
+          - COMPONENT_TYPE: telemetry
     steps:
       - uses: actions/checkout@v2
         with:
@@ -255,7 +256,7 @@ jobs:
           echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/.ci/assets/httpie/" >> $GITHUB_ENV
           echo "CI_TEST=true" >> $GITHUB_ENV
           echo "TEST=pulp" >> $GITHUB_ENV
-          echo "INGRESS_TYPE=${{ matrix.INGRESS_TYPE }}" >> $GITHUB_ENV
+          echo "COMPONENT_TYPE=${{ matrix.COMPONENT_TYPE }}" >> $GITHUB_ENV
         shell: bash
       - name: Updating registries configuration
         run: |
@@ -313,8 +314,10 @@ jobs:
           kubectl get namespace
           kubectl config set-context --current --namespace=pulp-operator-system
           kubectl apply -f .ci/assets/kubernetes/pulp-admin-password.secret.yaml
-          if [[ "$INGRESS_TYPE" == "ingress" ]]; then
+          if [[ "$COMPONENT_TYPE" == "ingress" ]]; then
             kubectl apply -f config/samples/simple.ingress.yaml
+          elif [[ "$COMPONENT_TYPE" == "telemetry" ]]; then
+            kubectl apply -f config/samples/telemetry.yaml
           else
             kubectl apply -f config/samples/simple.yaml
           fi
@@ -327,9 +330,13 @@ jobs:
         run: .ci/scripts/kubelinter.sh
       - name: Test all components
         run: |
+          export INGRESS_TYPE=$COMPONENT_TYPE
           git clone --depth=1 https://github.com/pulp/pulp_ansible.git
           git clone --depth=1 https://github.com/pulp/pulp_container.git
           .ci/scripts/pulp_tests.sh -m
+          if [[ "$COMPONENT_TYPE" == "telemetry" ]]; then
+            .ci/scripts/telemetry.sh
+          fi
         shell: bash
         env:
           PY_COLORS: '1'

diff --git a/.github/workflows/scripts/show_logs.sh b/.github/workflows/scripts/show_logs.sh
@@ -31,6 +31,7 @@ kubectl describe pods -l app.kubernetes.io/component=api
 echo ::endgroup::
 
 echo ::group::PULP_API_LOGS
+kubectl logs --timestamps -cinit-container -l app.kubernetes.io/component=api --tail=10000
 kubectl logs --timestamps -l app.kubernetes.io/component=api --tail=10000
 echo ::endgroup::
 

diff --git a/CHANGES/1006.feature b/CHANGES/1006.feature
@@ -0,0 +1 @@
+Added the OpenTelemetry support as sidecar container for pulp-api pods.
diff --git a/apis/repo-manager.pulpproject.org/v1beta2/pulp_types.go b/apis/repo-manager.pulpproject.org/v1beta2/pulp_types.go
@@ -551,6 +551,10 @@ type PulpSpec struct {
 	// +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors={"urn:alm:descriptor:com.tectonic.ui:advanced"}
 	NoLog bool `json:"no_log,omitempty"`
 
+	// Telemetry defines the OpenTelemetry configuration
+	// +kubebuilder:validation:Optional
+	Telemetry Telemetry `json:"telemetry,omitempty"`
+
 	ResourceManager ResourceManager `json:"resource_manager,omitempty"`
 }
 
@@ -1032,6 +1036,28 @@ type ResourceManager struct {
 	ResourceRequirements *corev1.ResourceRequirements `json:"resource_requirements,omitempty"`
 }
 
+// Telemetry defines the configuration for OpenTelemetry used by Pulp
+type Telemetry struct {
+
+	// Enable Pulp Telemetry
+	// Default: false
+	// +kubebuilder:default:=false
+	// +kubebuilder:validation:Optional
+	// +nullable
+	Enabled bool `json:"enabled,omitempty"`
+
+	// Defines the protocol used by the instrumentator to comunicate with the collector
+	// Default: http/protobuf
+	// +kubebuilder:default:="http/protobuf"
+	ExporterOtlpProtocol string `json:"exporter_otlp_protocol,omitempty"`
+
+	// Defines the image to be used as collector
+	OpenTelemetryCollectorImage string `json:"otel_collector_image,omitempty"`
+
+	// The image version for opentelemetry-collector image. Default: \"latest\"
+	OpenTelemetryCollectorImageVersion string `json:"otel_collector_image_version,omitempty"`
+}
+
 // PulpStatus defines the observed state of Pulp
 type PulpStatus struct {
 	//+operator-sdk:csv:customresourcedefinitions:type=status,xDescriptors={"urn:alm:descriptor:io.kubernetes.conditions"}
@@ -1054,6 +1080,8 @@ type PulpStatus struct {
 	AdminPasswordSecret string `json:"admin_password_secret,omitempty"`
 	// Name of the secret with the parameters to connect to an external Redis cluster
 	ExternalCacheSecret string `json:"external_cache_secret,omitempty"`
+	// Pulp metrics collection enabled
+	TelemetryEnabled bool `json:"telemetry_enabled,omitempty"`
 
 	// [DEPRECATED] Temporarily adding to keep compatibility with ansible version.
 	StoragePersistentVolumeClaim       string `json:"storagePersistentVolumeClaim,omitempty"`

diff --git a/apis/repo-manager.pulpproject.org/v1beta2/zz_generated.deepcopy.go b/apis/repo-manager.pulpproject.org/v1beta2/zz_generated.deepcopy.go
diff --git a/bundle/manifests/pulp-operator.clusterserviceversion.yaml b/bundle/manifests/pulp-operator.clusterserviceversion.yaml
@@ -324,7 +324,7 @@ metadata:
     capabilities: Full Lifecycle
     categories: Integration & Delivery
     containerImage: quay.io/pulp/pulp-operator:devel
-    createdAt: "2023-07-17T17:24:35Z"
+    createdAt: "2023-07-21T14:53:03Z"
     description: Pulp is a platform for managing repositories of software packages
       and making them available to a large number of consumers.
     operators.operatorframework.io/builder: operator-sdk-v1.29.0

diff --git a/bundle/manifests/repo-manager.pulpproject.org_pulps.yaml b/bundle/manifests/repo-manager.pulpproject.org_pulps.yaml
@@ -25159,6 +25159,27 @@ spec:
                 - Azure
                 - azure
                 type: string
+              telemetry:
+                description: Telemetry defines the OpenTelemetry configuration
+                properties:
+                  enabled:
+                    default: false
+                    description: 'Enable Pulp Telemetry Default: false'
+                    nullable: true
+                    type: boolean
+                  exporter_otlp_protocol:
+                    default: http/protobuf
+                    description: 'Defines the protocol used by the instrumentator
+                      to comunicate with the collector Default: http/protobuf'
+                    type: string
+                  otel_collector_image:
+                    description: Defines the image to be used as collector
+                    type: string
+                  otel_collector_image_version:
+                    description: 'The image version for opentelemetry-collector image.
+                      Default: \"latest\"'
+                    type: string
+                type: object
               tolerations:
                 description: '[DEPRECATED] Temporarily adding to keep compatibility
                   with ansible version. Node tolerations for the Pulp pods.'
@@ -27621,6 +27642,9 @@ spec:
                 type: string
               storageType:
                 type: string
+              telemetry_enabled:
+                description: Pulp metrics collection enabled
+                type: boolean
               upgradedPostgresVersion:
                 type: string
               webURL:

diff --git a/config/crd/bases/repo-manager.pulpproject.org_pulps.yaml b/config/crd/bases/repo-manager.pulpproject.org_pulps.yaml
@@ -25160,6 +25160,27 @@ spec:
                 - Azure
                 - azure
                 type: string
+              telemetry:
+                description: Telemetry defines the OpenTelemetry configuration
+                properties:
+                  enabled:
+                    default: false
+                    description: 'Enable Pulp Telemetry Default: false'
+                    nullable: true
+                    type: boolean
+                  exporter_otlp_protocol:
+                    default: http/protobuf
+                    description: 'Defines the protocol used by the instrumentator
+                      to comunicate with the collector Default: http/protobuf'
+                    type: string
+                  otel_collector_image:
+                    description: Defines the image to be used as collector
+                    type: string
+                  otel_collector_image_version:
+                    description: 'The image version for opentelemetry-collector image.
+                      Default: \"latest\"'
+                    type: string
+                type: object
               tolerations:
                 description: '[DEPRECATED] Temporarily adding to keep compatibility
                   with ansible version. Node tolerations for the Pulp pods.'
@@ -27622,6 +27643,9 @@ spec:
                 type: string
               storageType:
                 type: string
+              telemetry_enabled:
+                description: Pulp metrics collection enabled
+                type: boolean
               upgradedPostgresVersion:
                 type: string
               webURL:

diff --git a/config/samples/telemetry.yaml b/config/samples/telemetry.yaml
@@ -0,0 +1,30 @@
+apiVersion: repo-manager.pulpproject.org/v1beta2
+kind: Pulp
+metadata:
+  name: example-pulp
+spec:
+  telemetry:
+    enabled: true
+  image_version: stable
+  image_web_version: stable
+  api:
+    replicas: 1
+  content:
+    replicas: 1
+  worker:
+    replicas: 1
+  web:
+    replicas: 1
+  database:
+    postgres_storage_class: standard
+  file_storage_access_mode: "ReadWriteOnce"
+  file_storage_size: "2Gi"
+  file_storage_storage_class: standard
+  ingress_type: nodeport
+  pulp_settings:
+    api_root: "/pulp/"
+    allowed_export_paths:
+      - /tmp
+    allowed_import_paths:
+      - /tmp
+    analytics: false
diff --git a/controllers/deployment.go b/controllers/deployment.go
@@ -553,6 +553,60 @@ func (DeploymentAPICommon) Deploy(resources any) client.Object {
 		image = "quay.io/pulp/pulp-minimal:stable"
 	}
 
+	initContainers := []corev1.Container{
+		{
+			Name:    "init-container",
+			Image:   image,
+			Env:     envVars,
+			Command: []string{"/bin/sh"},
+			Args: []string{
+				"-c",
+				`mkdir -p /var/lib/pulp/{media,assets,tmp}
+				/usr/bin/wait_on_postgres.py
+				/usr/local/bin/pulpcore-manager migrate --noinput
+				ADMIN_PASSWORD_FILE=/etc/pulp/pulp-admin-password
+				if [[ -f "$ADMIN_PASSWORD_FILE" ]]; then
+				   echo "pulp admin can be initialized."
+				   PULP_ADMIN_PASSWORD=$(cat $ADMIN_PASSWORD_FILE)
+				fi
+				if [ -n "${PULP_ADMIN_PASSWORD}" ]; then
+					/usr/local/bin/pulpcore-manager reset-admin-password --password "${PULP_ADMIN_PASSWORD}"
+				fi`,
+			},
+			VolumeMounts: volumeMounts,
+		},
+	}
+
+	containers := []corev1.Container{
+		{
+			Name:            "api",
+			Image:           image,
+			ImagePullPolicy: corev1.PullPolicy(pulp.Spec.ImagePullPolicy),
+			Command:         []string{"/bin/sh"},
+			Args: []string{
+				"-c",
+				`exec gunicorn --bind '[::]:24817' pulpcore.app.wsgi:application --name pulp-api --timeout "${PULP_GUNICORN_TIMEOUT}" --workers "${PULP_API_WORKERS}"`,
+			},
+			Env: envVars,
+			Ports: []corev1.ContainerPort{{
+				ContainerPort: 24817,
+				Protocol:      "TCP",
+			}},
+			LivenessProbe:  livenessProbe,
+			ReadinessProbe: readinessProbe,
+			Resources:      resourceRequirements,
+			VolumeMounts:   volumeMounts,
+		},
+	}
+
+	podAnnotations := map[string]string{
+		"kubectl.kubernetes.io/default-container": "api",
+	}
+
+	if pulp.Spec.Telemetry.Enabled {
+		containers, volumes = telemetryConfig(resources, envVars, containers, volumes)
+	}
+
 	// deployment definition
 	dep := &appsv1.Deployment{
 		ObjectMeta: metav1.ObjectMeta{
@@ -581,7 +635,8 @@ func (DeploymentAPICommon) Deploy(resources any) client.Object {
 			},
 			Template: corev1.PodTemplateSpec{
 				ObjectMeta: metav1.ObjectMeta{
-					Labels: ls,
+					Labels:      ls,
+					Annotations: podAnnotations,
 				},
 				Spec: corev1.PodSpec{
 					Affinity:                  affinity,
@@ -591,22 +646,8 @@ func (DeploymentAPICommon) Deploy(resources any) client.Object {
 					Volumes:                   volumes,
 					ServiceAccountName:        pulp.Name,
 					TopologySpreadConstraints: topologySpreadConstraint,
-					Containers: []corev1.Container{{
-						Name:            "api",
-						Image:           image,
-						ImagePullPolicy: corev1.PullPolicy(pulp.Spec.ImagePullPolicy),
-						Args:            []string{"pulp-api"},
-						Env:             envVars,
-						Ports: []corev1.ContainerPort{{
-							ContainerPort: 24817,
-							Protocol:      "TCP",
-						}},
-						LivenessProbe:  livenessProbe,
-						ReadinessProbe: readinessProbe,
-						Resources:      resourceRequirements,
-						VolumeMounts:   volumeMounts,
-					}},
-
+					InitContainers:            initContainers,
+					Containers:                containers,
 					/* the following configs are not defined on pulp-operator (ansible version)  */
 					/* but i'll keep it here just in case we can manage to make deepequal usable */
 					RestartPolicy:                 restartPolicy,