Skip to content

Commit c445c4e

Browse files
wzshimingwzshiming
authored
Merge pull request kubernetes-sigs#1228 from wzshiming/clean/prometheus
[kwokctl] Structured Prometheus configuration
2 parents b170654 + 0bd05a0 commit c445c4e

15 files changed

+668
-695
lines changed

‎pkg/kwokctl/components/config/prometheus/types.go

+71
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
/*
2+
Copyright 2024 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
// Package prometheus copy from https://github.com/prometheus/prometheus/blob/919648cafc2c07ed5c1d5dd657b8080bee331aaf/config/config.go#L243
18+
package prometheus
19+
20+
type GlobalConfig struct {
21+
ScrapeInterval string `json:"scrape_interval,omitempty"`
22+
ScrapeTimeout string `json:"scrape_timeout,omitempty"`
23+
EvaluationInterval string `json:"evaluation_interval,omitempty"`
24+
}
25+
26+
type MetricsDiscovery struct {
27+
Scheme string `json:"scheme,omitempty"`
28+
Host string `json:"host,omitempty"`
29+
Path string `json:"path,omitempty"`
30+
}
31+
32+
type Metric struct {
33+
Scheme string `json:"scheme,omitempty"`
34+
Path string `json:"metrics_path,omitempty"`
35+
CertPath string `json:"cert_file,omitempty"`
36+
KeyPath string `json:"key_file,omitempty"`
37+
InsecureSkipVerify bool `json:"insecure_skip_verify,omitempty"`
38+
Host string `json:"host,omitempty"`
39+
}
40+
41+
type ScrapeConfig struct {
42+
JobName string `json:"job_name"`
43+
HttpSdConfigs []HTTPSDConfig `json:"http_sd_configs,omitempty"`
44+
Scheme string `json:"scheme,omitempty"`
45+
HonorTimestamps bool `json:"honor_timestamps,omitempty"`
46+
MetricsPath string `json:"metrics_path,omitempty"`
47+
FollowRedirects bool `json:"follow_redirects,omitempty"`
48+
EnableHttp2 bool `json:"enable_http2,omitempty"`
49+
TLSConfig *TLSConfig `json:"tls_config,omitempty"`
50+
StaticConfigs []StaticConfig `json:"static_configs,omitempty"`
51+
}
52+
53+
type HTTPSDConfig struct {
54+
URL string `json:"url"`
55+
}
56+
57+
type TLSConfig struct {
58+
CertFile string `json:"cert_file"`
59+
KeyFile string `json:"key_file"`
60+
InsecureSkipVerify bool `json:"insecure_skip_verify,omitempty"`
61+
}
62+
63+
type StaticConfig struct {
64+
Targets []string `json:"targets"`
65+
}
66+
67+
// Config is the top-level configuration for Prometheus's config files.
68+
type Config struct {
69+
GlobalConfig GlobalConfig `json:"global"`
70+
ScrapeConfigs []*ScrapeConfig `json:"scrape_configs,omitempty"`
71+
}

‎pkg/kwokctl/components/prometheus_config.go

+65-16
Original file line numberDiff line numberDiff line change
@@ -17,30 +17,79 @@ limitations under the License.
1717
package components
1818

1919
import (
20-
"bytes"
2120
"fmt"
22-
"text/template"
23-
24-
"github.com/Masterminds/sprig/v3"
21+
"net/url"
2522

2623
"sigs.k8s.io/kwok/pkg/apis/internalversion"
27-
28-
_ "embed"
24+
"sigs.k8s.io/kwok/pkg/kwokctl/components/config/prometheus"
25+
"sigs.k8s.io/kwok/pkg/utils/yaml"
2926
)
3027

31-
//go:embed prometheus_config.yaml.tpl
32-
var prometheusYamlTpl string
33-
34-
var prometheusYamlTemplate = template.Must(template.New("prometheus_config").Funcs(sprig.TxtFuncMap()).Parse(prometheusYamlTpl))
35-
36-
// BuildPrometheus builds the prometheus yaml content.
28+
// BuildPrometheus builds the Prometheus configuration.
3729
func BuildPrometheus(conf BuildPrometheusConfig) (string, error) {
38-
buf := bytes.NewBuffer(nil)
39-
err := prometheusYamlTemplate.Execute(buf, conf)
30+
config := prometheus.Config{
31+
GlobalConfig: prometheus.GlobalConfig{
32+
ScrapeInterval: "15s",
33+
ScrapeTimeout: "10s",
34+
EvaluationInterval: "15s",
35+
},
36+
ScrapeConfigs: convertToScrapeConfigs(conf.Components),
37+
}
38+
39+
configJSON, err := yaml.Marshal(config)
4040
if err != nil {
41-
return "", fmt.Errorf("build prometheus error: %w", err)
41+
return "", fmt.Errorf("build prometheus config error: %w", err)
42+
}
43+
return string(configJSON), nil
44+
}
45+
46+
// convertToScrapeConfigs converts internalversion.Component to prometheus.ScrapeConfig.
47+
func convertToScrapeConfigs(components []internalversion.Component) []*prometheus.ScrapeConfig {
48+
var scrapeConfigs []*prometheus.ScrapeConfig
49+
for _, c := range components {
50+
if md := c.MetricsDiscovery; md != nil {
51+
scrapeConfig := &prometheus.ScrapeConfig{}
52+
scrapeConfig.JobName = fmt.Sprintf("%s-metrics-discovery", c.Name)
53+
u := url.URL{
54+
Scheme: md.Scheme,
55+
Host: md.Host,
56+
Path: md.Path,
57+
}
58+
scrapeConfig.HttpSdConfigs = []prometheus.HTTPSDConfig{
59+
{
60+
URL: u.String(),
61+
},
62+
}
63+
scrapeConfigs = append(scrapeConfigs, scrapeConfig)
64+
}
65+
66+
if m := c.Metric; m != nil {
67+
scrapeConfig := &prometheus.ScrapeConfig{}
68+
scrapeConfig.JobName = c.Name
69+
scrapeConfig.Scheme = m.Scheme
70+
scrapeConfig.MetricsPath = m.Path
71+
scrapeConfig.HonorTimestamps = true
72+
scrapeConfig.EnableHttp2 = true
73+
scrapeConfig.FollowRedirects = true
74+
75+
if scrapeConfig.Scheme == "https" {
76+
scrapeConfig.TLSConfig = &prometheus.TLSConfig{}
77+
scrapeConfig.TLSConfig.CertFile = m.CertPath
78+
scrapeConfig.TLSConfig.KeyFile = m.KeyPath
79+
scrapeConfig.TLSConfig.InsecureSkipVerify = true
80+
}
81+
82+
scrapeConfig.StaticConfigs = []prometheus.StaticConfig{
83+
{
84+
Targets: []string{
85+
m.Host,
86+
},
87+
},
88+
}
89+
scrapeConfigs = append(scrapeConfigs, scrapeConfig)
90+
}
4291
}
43-
return buf.String(), nil
92+
return scrapeConfigs
4493
}
4594

4695
// BuildPrometheusConfig is the configuration for building the prometheus config

‎pkg/kwokctl/components/prometheus_config.yaml.tpl

-39
This file was deleted.

‎test/e2e/kwokctl/dryrun/testdata/binary/create_cluster_with_extra.txt

+39-48
Original file line numberDiff line numberDiff line change
@@ -13,85 +13,76 @@ mkdir -p <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/etcd
1313
# Download https://github.com/prometheus/prometheus/releases/download/v2.53.0/prometheus-2.53.0.<OS>-<ARCH>.<TAR> and extract prometheus to <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/bin/prometheus
1414
cat <<EOF ><ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/prometheus.yaml
1515
global:
16+
evaluation_interval: 15s
1617
scrape_interval: 15s
1718
scrape_timeout: 10s
18-
evaluation_interval: 15s
19-
alerting:
20-
alertmanagers:
21-
- follow_redirects: true
22-
enable_http2: true
23-
scheme: http
24-
timeout: 10s
25-
api_version: v2
26-
static_configs:
27-
- targets: []
2819
scrape_configs:
29-
- job_name: "etcd"
30-
scheme: http
20+
- enable_http2: true
21+
follow_redirects: true
3122
honor_timestamps: true
23+
job_name: etcd
3224
metrics_path: /metrics
33-
follow_redirects: true
34-
enable_http2: true
25+
scheme: http
3526
static_configs:
3627
- targets:
3728
- 127.0.0.1:32765
38-
- job_name: "kube-apiserver"
39-
scheme: https
29+
- enable_http2: true
30+
follow_redirects: true
4031
honor_timestamps: true
32+
job_name: kube-apiserver
4133
metrics_path: /metrics
42-
follow_redirects: true
43-
enable_http2: true
44-
tls_config:
45-
cert_file: "<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.crt"
46-
key_file: "<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.key"
47-
insecure_skip_verify: true
34+
scheme: https
4835
static_configs:
4936
- targets:
5037
- 127.0.0.1:32764
51-
- job_name: "kube-controller-manager"
52-
scheme: https
53-
honor_timestamps: true
54-
metrics_path: /metrics
55-
follow_redirects: true
56-
enable_http2: true
5738
tls_config:
58-
cert_file: "<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.crt"
59-
key_file: "<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.key"
39+
cert_file: <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.crt
6040
insecure_skip_verify: true
41+
key_file: <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.key
42+
- enable_http2: true
43+
follow_redirects: true
44+
honor_timestamps: true
45+
job_name: kube-controller-manager
46+
metrics_path: /metrics
47+
scheme: https
6148
static_configs:
6249
- targets:
6350
- 127.0.0.1:32762
64-
- job_name: "kube-scheduler"
65-
scheme: https
66-
honor_timestamps: true
67-
metrics_path: /metrics
68-
follow_redirects: true
69-
enable_http2: true
7051
tls_config:
71-
cert_file: "<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.crt"
72-
key_file: "<ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.key"
52+
cert_file: <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.crt
7353
insecure_skip_verify: true
54+
key_file: <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.key
55+
- enable_http2: true
56+
follow_redirects: true
57+
honor_timestamps: true
58+
job_name: kube-scheduler
59+
metrics_path: /metrics
60+
scheme: https
7461
static_configs:
7562
- targets:
7663
- 127.0.0.1:32761
77-
- job_name: "kwok-controller-metrics-discovery"
78-
http_sd_configs:
64+
tls_config:
65+
cert_file: <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.crt
66+
insecure_skip_verify: true
67+
key_file: <ROOT_DIR>/workdir/clusters/<CLUSTER_NAME>/pki/admin.key
68+
- http_sd_configs:
7969
- url: http://127.0.0.1:32763/discovery/prometheus
80-
- job_name: "kwok-controller"
81-
scheme: http
70+
job_name: kwok-controller-metrics-discovery
71+
- enable_http2: true
72+
follow_redirects: true
8273
honor_timestamps: true
74+
job_name: kwok-controller
8375
metrics_path: /metrics
84-
follow_redirects: true
85-
enable_http2: true
76+
scheme: http
8677
static_configs:
8778
- targets:
8879
- 127.0.0.1:32763
89-
- job_name: "prometheus"
90-
scheme: http
80+
- enable_http2: true
81+
follow_redirects: true
9182
honor_timestamps: true
83+
job_name: prometheus
9284
metrics_path: /metrics
93-
follow_redirects: true
94-
enable_http2: true
85+
scheme: http
9586
static_configs:
9687
- targets:
9788
- 127.0.0.1:9090

0 commit comments

Comments
 (0)