Feat: New PR to add data transformation using JQ

config/config.go

@@ -16,6 +16,7 @@ package config
 import (
 	"os"
 
+	"github.com/prometheus-community/json_exporter/transformers"
 	pconfig "github.com/prometheus/common/config"
 	"go.yaml.in/yaml/v2"
 )
@@ -31,6 +32,7 @@ type Metric struct {
 	Help            string
 	Values          map[string]string
 	AllowMissingKey bool `yaml:"allow_missing_key,omitempty"`
+	Transformations []transformers.TransformationConfig
 }
 
 type ScrapeType string

examples/config.yml

@@ -73,3 +73,29 @@ modules:
     #   content: |
     #     {"time_diff": "{{ duration `95` }}","anotherVar": "{{ .myVal | first }}"}
     #   templatize: true
+  transform:
+    metrics:
+    - name: origin
+      transformations:
+        - type: jq
+          query: |-
+            .result[] | .name as $poolName | .id as $poolId | .origins[] | (  .name as $name | .healthy as $endpointHealth | {endpointName: $name, endpointHealthy: .healthy, poolName: $poolName, address:.address, poolId:$poolId, address:.address} )
+      help: Health of each origin in the pool
+      path: '{ [*] }'
+      type: object
+      labels:
+        pool_id: '{.poolId}'
+        pool_name: '{.poolName}'
+        address: '{.address}'
+        endpoint_name: '{.endpointName}'
+      values:
+        health: '{.endpointHealthy}'  # Extract only the `healthy` field
+    - name: pool
+      type: object
+      help: Health of the pools
+      path: '{.result[*]}'
+      labels:
+        pool_name: '{.name}'
+        pool_id: '{.id}'
+      values:
+        health: '{.healthy}'

examples/prometheus.yml

@@ -30,3 +30,19 @@ scrape_configs:
       - target_label: __address__
         ## Location of the json exporter's real <hostname>:<port>
         replacement: host.docker.internal:7979 # equivalent to "localhost:7979"
+
+  ## Gather metrics from transform-data JSON source using the 'transform' module
+  - job_name: json_transform
+    metrics_path: /probe
+    params:
+      module: [transform]  # Use the 'transform' module
+    static_configs:
+      - targets:
+          - http://localhost:8000/examples/transform-data.json
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: host.docker.internal:7979  # json_exporter instance

examples/transform-data.json

@@ -0,0 +1,45 @@
+{
+  "result": [
+    {
+      "name": "pool1",
+      "id": "1",
+      "healthy": true,
+      "origins": [
+        {
+          "name": "origin1",
+          "address": "127.0.0.1",
+          "healthy": true
+        },
+        {
+          "name": "origin2",
+          "address": "192.168.1.1",
+          "healthy": false
+        }
+      ]
+    },
+    {
+      "name": "pool2",
+      "id": "2",
+      "healthy": true,
+      "origins": [
+        {
+          "name": "origin3",
+          "address": "10.0.0.1",
+          "healthy": true
+        }
+      ]
+    },
+    {
+      "name": "pool3",
+      "id": "3",
+      "healthy": false,
+      "origins": [
+        {
+          "name": "origin4",
+          "address": "10.0.0.1",
+          "healthy": false
+        }
+      ]
+    }
+  ]
+}

exporter/collector.go

@@ -20,6 +20,7 @@ import (
 	"time"
 
 	"github.com/prometheus-community/json_exporter/config"
+	"github.com/prometheus-community/json_exporter/transformers"
 	"github.com/prometheus/client_golang/prometheus"
 	"k8s.io/client-go/util/jsonpath"
 )
@@ -39,6 +40,7 @@ type JSONMetric struct {
 	ValueType              prometheus.ValueType
 	EpochTimestampJSONPath string
 	AllowMissingKey        bool
+	Transformers           []transformers.Transformer
 }
 
 func (mc JSONMetricCollector) Describe(ch chan<- *prometheus.Desc) {
@@ -49,11 +51,21 @@ func (mc JSONMetricCollector) Describe(ch chan<- *prometheus.Desc) {
 
 func (mc JSONMetricCollector) Collect(ch chan<- prometheus.Metric) {
 	for _, m := range mc.JSONMetrics {
+
+		jsonData := mc.Data
+		for _, transformer := range m.Transformers {
+			transformedData, err := transformer.Transform(jsonData)
+			if err != nil {
+				mc.Logger.Error("msg", "Transformation failed", "err", err)
+				continue
+			}
+			jsonData = transformedData
+		}
 		switch m.Type {
 		case config.ValueScrape:
-			value, missing, err := extractValue(mc.Logger, mc.Data, m.KeyJSONPath, false, m.AllowMissingKey)
+			value, missing, err := extractValue(mc.Logger, jsonData, m.KeyJSONPath, false, m.AllowMissingKey)
 			if err != nil {
-				mc.Logger.Error("Failed to extract value for metric", "path", m.KeyJSONPath, "err", err, "metric", m.Desc)
+				mc.Logger.Error("Failed to extract value for metric", "path", m.KeyJSONPath, "err", err, "metric", m.Desc, "data", jsonData)
 				continue
 			}
 			if missing {
@@ -65,18 +77,18 @@ func (mc JSONMetricCollector) Collect(ch chan<- prometheus.Metric) {
 					m.Desc,
 					m.ValueType,
 					floatValue,
-					extractLabels(mc.Logger, mc.Data, m.LabelsJSONPaths)...,
+					extractLabels(mc.Logger, jsonData, m.LabelsJSONPaths)...,
 				)
-				ch <- timestampMetric(mc.Logger, m, mc.Data, metric)
+				ch <- timestampMetric(mc.Logger, m, jsonData, metric)
 			} else {
-				mc.Logger.Error("Failed to convert extracted value to float64", "path", m.KeyJSONPath, "value", value, "err", err, "metric", m.Desc)
+				mc.Logger.Error("Failed to convert extracted value to float64", "path", m.KeyJSONPath, "value", value, "err", err, "metric", m.Desc, "data", jsonData)
 				continue
 			}
 
 		case config.ObjectScrape:
-			values, missing, err := extractValue(mc.Logger, mc.Data, m.KeyJSONPath, true, m.AllowMissingKey)
+			values, missing, err := extractValue(mc.Logger, jsonData, m.KeyJSONPath, true, m.AllowMissingKey)
 			if err != nil {
-				mc.Logger.Error("Failed to extract json objects for metric", "err", err, "metric", m.Desc)
+				mc.Logger.Error("Failed to extract json objects for metric", "err", err, "metric", m.Desc, "data", jsonData)
 				continue
 			}
 			if missing {

exporter/util.go

@@ -28,6 +28,7 @@ import (
 
 	"github.com/Masterminds/sprig/v3"
 	"github.com/prometheus-community/json_exporter/config"
+	"github.com/prometheus-community/json_exporter/transformers"
 	"github.com/prometheus/client_golang/prometheus"
 	pconfig "github.com/prometheus/common/config"
 )
@@ -79,6 +80,14 @@ func CreateMetricsList(c config.Module) ([]JSONMetric, error) {
 		valueType prometheus.ValueType
 	)
 	for _, metric := range c.Metrics {
+		metricTransfomers := []transformers.Transformer{}
+		for _, tConfig := range metric.Transformations {
+			transformer, err := transformers.NewTransformer(tConfig) // Use the package reference here
+			if err != nil {
+				return nil, err
+			}
+			metricTransfomers = append(metricTransfomers, transformer)
+		}
 		switch metric.ValueType {
 		case config.ValueTypeGauge:
 			valueType = prometheus.GaugeValue
@@ -107,6 +116,7 @@ func CreateMetricsList(c config.Module) ([]JSONMetric, error) {
 				ValueType:              valueType,
 				EpochTimestampJSONPath: metric.EpochTimestamp,
 				AllowMissingKey:        metric.AllowMissingKey,
+				Transformers:           metricTransfomers, // Add transformers here
 			}
 			metrics = append(metrics, jsonMetric)
 		case config.ObjectScrape:
@@ -131,6 +141,7 @@ func CreateMetricsList(c config.Module) ([]JSONMetric, error) {
 					ValueType:              valueType,
 					EpochTimestampJSONPath: metric.EpochTimestamp,
 					AllowMissingKey:        metric.AllowMissingKey,
+					Transformers:           metricTransfomers, // Add transformers here
 				}
 				metrics = append(metrics, jsonMetric)
 			}

go.mod

@@ -23,6 +23,8 @@ require (
 	github.com/golang-jwt/jwt/v5 v5.3.1 // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/huandu/xstrings v1.5.0 // indirect
+	github.com/itchyny/gojq v0.12.19 // indirect
+	github.com/itchyny/timefmt-go v0.1.8 // indirect
 	github.com/jpillora/backoff v1.0.0 // indirect
 	github.com/mdlayher/socket v0.6.0 // indirect
 	github.com/mdlayher/vsock v1.3.0 // indirect

go.sum

@@ -29,6 +29,10 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/huandu/xstrings v1.5.0 h1:2ag3IFq9ZDANvthTwTiqSSZLjDc+BedvHPAp5tJy2TI=
 github.com/huandu/xstrings v1.5.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
+github.com/itchyny/gojq v0.12.19 h1:ttXA0XCLEMoaLOz5lSeFOZ6u6Q3QxmG46vfgI4O0DEs=
+github.com/itchyny/gojq v0.12.19/go.mod h1:5galtVPDywX8SPSOrqjGxkBeDhSxEW1gSxoy7tn1iZY=
+github.com/itchyny/timefmt-go v0.1.8 h1:1YEo1JvfXeAHKdjelbYr/uCuhkybaHCeTkH8Bo791OI=
+github.com/itchyny/timefmt-go v0.1.8/go.mod h1:5E46Q+zj7vbTgWY8o5YkMeYb4I6GeWLFnetPy5oBrAI=
 github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=
 github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
 github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=

transformers/jq_transformer.go

@@ -0,0 +1,68 @@
+// Copyright 2020 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package transformers
+
+import (
+	"encoding/json"
+	"github.com/itchyny/gojq"
+)
+
+// JQTransformer struct for jq transformation
+type JQTransformer struct {
+	Query string
+}
+
+// NewJQTransformer creates a new JQTransformer with a given query
+func NewJQTransformer(query string) JQTransformer {
+	return JQTransformer{Query: query}
+}
+
+// Transform applies the jq filter transformation to the input data
+func (jq JQTransformer) Transform(data []byte) ([]byte, error) {
+	return applyJQFilter(data, jq.Query)
+}
+
+// applyJQFilter uses gojq to apply a jq transformation to the input data
+func applyJQFilter(jsonData []byte, jqQuery string) ([]byte, error) {
+	var input interface{}
+	if err := json.Unmarshal(jsonData, &input); err != nil {
+		return nil, err
+	}
+
+	query, err := gojq.Parse(jqQuery)
+	if err != nil {
+		return nil, err
+	}
+
+	iter := query.Run(input)
+	var results []interface{}
+	for {
+		v, ok := iter.Next()
+		if !ok {
+			break
+		}
+		if err, ok := v.(error); ok {
+			return nil, err
+		}
+		results = append(results, v)
+	}
+
+	// Convert the transformed result back to JSON []byte
+	transformedJSON, err := json.Marshal(results)
+	if err != nil {
+		return nil, err
+	}
+
+	return transformedJSON, nil
+}

transformers/transformers.go

@@ -0,0 +1,34 @@
+// Copyright 2020 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package transformers
+
+import "fmt"
+
+type Transformer interface {
+	Transform(data []byte) ([]byte, error)
+}
+
+type TransformationConfig struct {
+	Type  string
+	Query string
+}
+
+func NewTransformer(config TransformationConfig) (Transformer, error) {
+	switch config.Type {
+	case "jq":
+		return NewJQTransformer(config.Query), nil
+	default:
+		return nil, fmt.Errorf("unsupported transformer type: %s", config.Type)
+	}
+}