From 39d4284fd2322e8d49cc1208575a52062edb0fe9 Mon Sep 17 00:00:00 2001 From: Vaktas <49776740+Ekkisleif@users.noreply.github.com> Date: Mon, 27 Jul 2026 17:32:42 +0200 Subject: [PATCH 1/3] Add code from MattK360:transformations Signed-off-by: Vaktas --- config/config.go | 2 + examples/config.yml | 28 ++++++++ examples/prometheus.yml | 16 +++++ examples/transform-data.json | 45 ++++++++++++ exporter/collector.go | 26 +++++-- exporter/util.go | 11 +++ transformers/jq_transformer.go | 68 ++++++++++++++++++ transformers/transformers.go | 34 +++++++++ transformers/transformers_test.go | 114 ++++++++++++++++++++++++++++++ 9 files changed, 337 insertions(+), 7 deletions(-) create mode 100644 examples/transform-data.json create mode 100644 transformers/jq_transformer.go create mode 100644 transformers/transformers.go create mode 100644 transformers/transformers_test.go diff --git a/config/config.go b/config/config.go index 25861daa..301a0ed4 100644 --- a/config/config.go +++ b/config/config.go @@ -16,6 +16,7 @@ package config import ( "os" + "github.com/prometheus-community/json_exporter/transformers" pconfig "github.com/prometheus/common/config" "go.yaml.in/yaml/v2" ) @@ -31,6 +32,7 @@ type Metric struct { Help string Values map[string]string AllowMissingKey bool `yaml:"allow_missing_key,omitempty"` + Transformations []transformers.TransformationConfig } type ScrapeType string diff --git a/examples/config.yml b/examples/config.yml index b70599c1..3017915b 100644 --- a/examples/config.yml +++ b/examples/config.yml @@ -73,3 +73,31 @@ modules: # content: | # {"time_diff": "{{ duration `95` }}","anotherVar": "{{ .myVal | first }}"} # templatize: true + transform: + metrics: + - name: origin + transformations: + - type: jq + query: |- + .result[] | .name as $poolName | .id as $poolId | .origins[] | ( .name as $name | .healthy as $endpointHealth | {endpointName: $name, endpointHealthy: .healthy, poolName: $poolName, address:.address, poolId:$poolId, address:.address} ) + help: Health of each origin in the pool + path: '{ [*] }' + type: object + labels: + pool_id: '{.poolId}' + pool_name: '{.poolName}' + address: '{.address}' + endpoint_name: '{.endpointName}' + values: + health: '{.endpointHealthy}' # Extract only the `healthy` field + + + - name: pool + type: object + help: Health of the pools + path: '{.result[*]}' + labels: + pool_name: '{.name}' + pool_id: '{.id}' + values: + health: '{.healthy}' diff --git a/examples/prometheus.yml b/examples/prometheus.yml index 28f6f6d1..b3f3d04d 100644 --- a/examples/prometheus.yml +++ b/examples/prometheus.yml @@ -30,3 +30,19 @@ scrape_configs: - target_label: __address__ ## Location of the json exporter's real : replacement: host.docker.internal:7979 # equivalent to "localhost:7979" + + ## Gather metrics from transform-data JSON source using the 'transform' module + - job_name: json_transform + metrics_path: /probe + params: + module: [transform] # Use the 'transform' module + static_configs: + - targets: + - http://localhost:8000/examples/transform-data.json + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: host.docker.internal:7979 # json_exporter instance diff --git a/examples/transform-data.json b/examples/transform-data.json new file mode 100644 index 00000000..68d9b33b --- /dev/null +++ b/examples/transform-data.json @@ -0,0 +1,45 @@ +{ + "result": [ + { + "name": "pool1", + "id": "1", + "healthy": true, + "origins": [ + { + "name": "origin1", + "address": "127.0.0.1", + "healthy": true + }, + { + "name": "origin2", + "address": "192.168.1.1", + "healthy": false + } + ] + }, + { + "name": "pool2", + "id": "2", + "healthy": true, + "origins": [ + { + "name": "origin3", + "address": "10.0.0.1", + "healthy": true + } + ] + }, + { + "name": "pool3", + "id": "3", + "healthy": false, + "origins": [ + { + "name": "origin4", + "address": "10.0.0.1", + "healthy": false + } + ] + } + ] +} \ No newline at end of file diff --git a/exporter/collector.go b/exporter/collector.go index 9fdd0c3f..16a0df64 100644 --- a/exporter/collector.go +++ b/exporter/collector.go @@ -20,6 +20,7 @@ import ( "time" "github.com/prometheus-community/json_exporter/config" + "github.com/prometheus-community/json_exporter/transformers" "github.com/prometheus/client_golang/prometheus" "k8s.io/client-go/util/jsonpath" ) @@ -39,6 +40,7 @@ type JSONMetric struct { ValueType prometheus.ValueType EpochTimestampJSONPath string AllowMissingKey bool + Transformers []transformers.Transformer } func (mc JSONMetricCollector) Describe(ch chan<- *prometheus.Desc) { @@ -49,11 +51,21 @@ func (mc JSONMetricCollector) Describe(ch chan<- *prometheus.Desc) { func (mc JSONMetricCollector) Collect(ch chan<- prometheus.Metric) { for _, m := range mc.JSONMetrics { + + jsonData := mc.Data + for _, transformer := range m.Transformers { + transformedData, err := transformer.Transform(jsonData) + if err != nil { + level.Error(mc.Logger).Log("msg", "Transformation failed", "err", err) + continue + } + jsonData = transformedData + } switch m.Type { case config.ValueScrape: - value, missing, err := extractValue(mc.Logger, mc.Data, m.KeyJSONPath, false, m.AllowMissingKey) + value, missing, err := extractValue(mc.Logger, jsonData, m.KeyJSONPath, false, m.AllowMissingKey) if err != nil { - mc.Logger.Error("Failed to extract value for metric", "path", m.KeyJSONPath, "err", err, "metric", m.Desc) + mc.Logger.Error("Failed to extract value for metric", "path", m.KeyJSONPath, "err", err, "metric", m.Desc, "data", jsonData) continue } if missing { @@ -65,18 +77,18 @@ func (mc JSONMetricCollector) Collect(ch chan<- prometheus.Metric) { m.Desc, m.ValueType, floatValue, - extractLabels(mc.Logger, mc.Data, m.LabelsJSONPaths)..., + extractLabels(mc.Logger, jsonData, m.LabelsJSONPaths)..., ) - ch <- timestampMetric(mc.Logger, m, mc.Data, metric) + ch <- timestampMetric(mc.Logger, m, jsonData, metric) } else { - mc.Logger.Error("Failed to convert extracted value to float64", "path", m.KeyJSONPath, "value", value, "err", err, "metric", m.Desc) + mc.Logger.Error("Failed to convert extracted value to float64", "path", m.KeyJSONPath, "value", value, "err", err, "metric", m.Desc, "data", jsonData) continue } case config.ObjectScrape: - values, missing, err := extractValue(mc.Logger, mc.Data, m.KeyJSONPath, true, m.AllowMissingKey) + values, missing, err := extractValue(mc.Logger, jsonData, m.KeyJSONPath, true, m.AllowMissingKey) if err != nil { - mc.Logger.Error("Failed to extract json objects for metric", "err", err, "metric", m.Desc) + mc.Logger.Error("Failed to extract json objects for metric", "err", err, "metric", m.Desc, "data", jsonData) continue } if missing { diff --git a/exporter/util.go b/exporter/util.go index ffdf5fd3..c8b0f0c3 100644 --- a/exporter/util.go +++ b/exporter/util.go @@ -28,6 +28,7 @@ import ( "github.com/Masterminds/sprig/v3" "github.com/prometheus-community/json_exporter/config" + "github.com/prometheus-community/json_exporter/transformers" "github.com/prometheus/client_golang/prometheus" pconfig "github.com/prometheus/common/config" ) @@ -79,6 +80,14 @@ func CreateMetricsList(c config.Module) ([]JSONMetric, error) { valueType prometheus.ValueType ) for _, metric := range c.Metrics { + metricTransfomers := []transformers.Transformer{} + for _, tConfig := range metric.Transformations { + transformer, err := transformers.NewTransformer(tConfig) // Use the package reference here + if err != nil { + return nil, err + } + metricTransfomers = append(metricTransfomers, transformer) + } switch metric.ValueType { case config.ValueTypeGauge: valueType = prometheus.GaugeValue @@ -107,6 +116,7 @@ func CreateMetricsList(c config.Module) ([]JSONMetric, error) { ValueType: valueType, EpochTimestampJSONPath: metric.EpochTimestamp, AllowMissingKey: metric.AllowMissingKey, + Transformers: metricTransfomers, // Add transformers here } metrics = append(metrics, jsonMetric) case config.ObjectScrape: @@ -131,6 +141,7 @@ func CreateMetricsList(c config.Module) ([]JSONMetric, error) { ValueType: valueType, EpochTimestampJSONPath: metric.EpochTimestamp, AllowMissingKey: metric.AllowMissingKey, + Transformers: metricTransfomers, // Add transformers here } metrics = append(metrics, jsonMetric) } diff --git a/transformers/jq_transformer.go b/transformers/jq_transformer.go new file mode 100644 index 00000000..cb358038 --- /dev/null +++ b/transformers/jq_transformer.go @@ -0,0 +1,68 @@ +// Copyright 2020 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package transformers + +import ( + "encoding/json" + "github.com/itchyny/gojq" +) + +// JQTransformer struct for jq transformation +type JQTransformer struct { + Query string +} + +// NewJQTransformer creates a new JQTransformer with a given query +func NewJQTransformer(query string) JQTransformer { + return JQTransformer{Query: query} +} + +// Transform applies the jq filter transformation to the input data +func (jq JQTransformer) Transform(data []byte) ([]byte, error) { + return applyJQFilter(data, jq.Query) +} + +// applyJQFilter uses gojq to apply a jq transformation to the input data +func applyJQFilter(jsonData []byte, jqQuery string) ([]byte, error) { + var input interface{} + if err := json.Unmarshal(jsonData, &input); err != nil { + return nil, err + } + + query, err := gojq.Parse(jqQuery) + if err != nil { + return nil, err + } + + iter := query.Run(input) + var results []interface{} + for { + v, ok := iter.Next() + if !ok { + break + } + if err, ok := v.(error); ok { + return nil, err + } + results = append(results, v) + } + + // Convert the transformed result back to JSON []byte + transformedJSON, err := json.Marshal(results) + if err != nil { + return nil, err + } + + return transformedJSON, nil +} \ No newline at end of file diff --git a/transformers/transformers.go b/transformers/transformers.go new file mode 100644 index 00000000..ec6a9ef2 --- /dev/null +++ b/transformers/transformers.go @@ -0,0 +1,34 @@ +// Copyright 2020 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package transformers + +import "fmt" + +type Transformer interface { + Transform(data []byte) ([]byte, error) +} + +type TransformationConfig struct { + Type string + Query string +} + +func NewTransformer(config TransformationConfig) (Transformer, error) { + switch config.Type { + case "jq": + return NewJQTransformer(config.Query), nil + default: + return nil, fmt.Errorf("unsupported transformer type: %s", config.Type) + } +} \ No newline at end of file diff --git a/transformers/transformers_test.go b/transformers/transformers_test.go new file mode 100644 index 00000000..e32ab0a2 --- /dev/null +++ b/transformers/transformers_test.go @@ -0,0 +1,114 @@ +// Copyright 2020 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package transformers + +import "testing" + +func TestNewTransformerFactory(t *testing.T) { + // Define test cases for multiple jq transformations + tests := []struct { + Config TransformationConfig + Input string + ExpectedOutput string + ShouldSucceed bool + }{ + { + Config: TransformationConfig{ + Type: "jq", + Query: `.result[] | select(.name == "pool1")`, + }, + Input: `{ + "result": [ + {"name":"pool1","origins":[{"name":"origin1","healthy":true},{"name":"origin2","healthy":false}]}, + {"name":"pool2","origins":[{"name":"origin3","healthy":true}]} + ] + }`, + ExpectedOutput: `[{"name":"pool1","origins":[{"healthy":true,"name":"origin1"},{"healthy":false,"name":"origin2"}]}]`, + ShouldSucceed: true, + }, + { + Config: TransformationConfig{ + Type: "jq", + Query: `.result[] | select(has("origins")) | .origins[] | select(.healthy == true)`, + }, + Input: `{ + "result": [ + {"name":"pool1","origins":[{"name":"origin1","healthy":true},{"name":"origin2","healthy":false}]}, + {"name":"pool2","origins":[{"name":"origin3","healthy":true}]} + ] + }`, + ExpectedOutput: `[{"healthy":true,"name":"origin1"},{"healthy":true,"name":"origin3"}]`, + ShouldSucceed: true, + }, + { + Config: TransformationConfig{ + Type: "jq", + Query: `.result[] | .name as $poolName | .id as $poolId | .origins[] | {endpoint_name: .name, endpoint_health: .healthy, pool_name: $poolName, address: .address, pool_id: $poolId}`, + }, + Input: `{"result":[{"name":"pool1","id":"1","origins":[{"name":"origin1","healthy":true, "address":"127.0.0.1"}]}]}`, + ExpectedOutput: `[{"address":"127.0.0.1","endpoint_health":true,"endpoint_name":"origin1","pool_id":"1","pool_name":"pool1"}]`, + ShouldSucceed: true, + }, + { + Config: TransformationConfig{ + Type: "jq", + Query: `.result[] | select(.name == "pool2")`, + }, + Input: `{"result":[{"name":"pool1","id":"1","origins":[{"name":"origin1","healthy":true}]},{"name":"pool2","id":"2","origins":[{"name":"origin2","healthy":true}]}]}`, + ExpectedOutput: `[{"id":"2","name":"pool2","origins":[{"healthy":true,"name":"origin2"}]}]`, + ShouldSucceed: true, + }, + { + Config: TransformationConfig{ + Type: "jq", + Query: `.result[] | select(.name == "pool2")`, + }, + Input: `{"result":[{"name":"pool1","id":"1","origins":[{"name":"origin1","healthy":true}]}]}`, + ExpectedOutput: `null`, + ShouldSucceed: true, + }, + { + Config: TransformationConfig{ + Type: "jq", + Query: `.result[] | .origins[]`, + }, + Input: `{"result":[{"name":"pool1"}]}`, + ExpectedOutput: ``, + ShouldSucceed: false, + }, + } + + // Loop through each test case + for i, test := range tests { + // Create the transformer using NewTransformer + transformer, err := NewTransformer(test.Config) + if err != nil && test.ShouldSucceed { + t.Fatalf("Failed to create transformer %d: %s", i, err) + } + + // Apply the transformation + output, err := transformer.Transform([]byte(test.Input)) + if err != nil && test.ShouldSucceed { + t.Fatalf("Transformation %d failed: %s", i, err) + } + + // Compare the actual output with the expected output + if string(output) != test.ExpectedOutput { + t.Fatalf("Transformation %d failed. Expected: %s, Got: %s", i, test.ExpectedOutput, string(output)) + } + + // Log the successful transformation + t.Logf("Transformation %d succeeded. Output: %s", i, string(output)) + } +} \ No newline at end of file From 97f31c5fc636e32a77ceee7a850f20ca5bf973e1 Mon Sep 17 00:00:00 2001 From: Vaktas Date: Mon, 27 Jul 2026 21:50:22 +0200 Subject: [PATCH 2/3] Fix logging and check the code is working Signed-off-by: Vaktas --- exporter/collector.go | 2 +- go.mod | 2 ++ go.sum | 4 ++++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/exporter/collector.go b/exporter/collector.go index 16a0df64..099e32f7 100644 --- a/exporter/collector.go +++ b/exporter/collector.go @@ -56,7 +56,7 @@ func (mc JSONMetricCollector) Collect(ch chan<- prometheus.Metric) { for _, transformer := range m.Transformers { transformedData, err := transformer.Transform(jsonData) if err != nil { - level.Error(mc.Logger).Log("msg", "Transformation failed", "err", err) + mc.Logger.Error("msg", "Transformation failed", "err", err) continue } jsonData = transformedData diff --git a/go.mod b/go.mod index a0267410..1c87ff46 100644 --- a/go.mod +++ b/go.mod @@ -23,6 +23,8 @@ require ( github.com/golang-jwt/jwt/v5 v5.3.1 // indirect github.com/google/uuid v1.6.0 // indirect github.com/huandu/xstrings v1.5.0 // indirect + github.com/itchyny/gojq v0.12.19 // indirect + github.com/itchyny/timefmt-go v0.1.8 // indirect github.com/jpillora/backoff v1.0.0 // indirect github.com/mdlayher/socket v0.6.0 // indirect github.com/mdlayher/vsock v1.3.0 // indirect diff --git a/go.sum b/go.sum index 4cbfecac..07cb3cbd 100644 --- a/go.sum +++ b/go.sum @@ -29,6 +29,10 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/huandu/xstrings v1.5.0 h1:2ag3IFq9ZDANvthTwTiqSSZLjDc+BedvHPAp5tJy2TI= github.com/huandu/xstrings v1.5.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= +github.com/itchyny/gojq v0.12.19 h1:ttXA0XCLEMoaLOz5lSeFOZ6u6Q3QxmG46vfgI4O0DEs= +github.com/itchyny/gojq v0.12.19/go.mod h1:5galtVPDywX8SPSOrqjGxkBeDhSxEW1gSxoy7tn1iZY= +github.com/itchyny/timefmt-go v0.1.8 h1:1YEo1JvfXeAHKdjelbYr/uCuhkybaHCeTkH8Bo791OI= +github.com/itchyny/timefmt-go v0.1.8/go.mod h1:5E46Q+zj7vbTgWY8o5YkMeYb4I6GeWLFnetPy5oBrAI= github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= From 2e2e86b8fd3c30414d25807b56952d85d2bb2a14 Mon Sep 17 00:00:00 2001 From: Vaktas <49776740+Ekkisleif@users.noreply.github.com> Date: Fri, 31 Jul 2026 14:54:13 +0200 Subject: [PATCH 3/3] Remove whitespaces on the example Signed-off-by: Vaktas <49776740+Ekkisleif@users.noreply.github.com> Signed-off-by: Vaktas --- examples/config.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/examples/config.yml b/examples/config.yml index 3017915b..c4f8b299 100644 --- a/examples/config.yml +++ b/examples/config.yml @@ -90,8 +90,6 @@ modules: endpoint_name: '{.endpointName}' values: health: '{.endpointHealthy}' # Extract only the `healthy` field - - - name: pool type: object help: Health of the pools