diff --git a/cmd/tools/generate/counter.go b/cmd/tools/generate/counter.go index d7faa957d..6dbe9b829 100644 --- a/cmd/tools/generate/counter.go +++ b/cmd/tools/generate/counter.go @@ -2,2183 +2,97 @@ package generate import ( "encoding/json" - "errors" "fmt" - "io" + "github.com/netapp/harvest/v2/cmd/tools" + "github.com/netapp/harvest/v2/cmd/tools/grafana" + "github.com/netapp/harvest/v2/third_party/tidwall/gjson" "log" - "log/slog" - "maps" - "net/http" - "net/http/httputil" - "net/url" "os" - "os/exec" "path/filepath" - "regexp" - "slices" - "sort" "strings" - "text/template" - "time" - - "github.com/goccy/go-yaml" - "github.com/netapp/harvest/v2/cmd/collectors/keyperf" - "github.com/netapp/harvest/v2/cmd/collectors/statperf" - "github.com/netapp/harvest/v2/cmd/poller/plugin" - "github.com/netapp/harvest/v2/cmd/tools/grafana" - "github.com/netapp/harvest/v2/cmd/tools/rest" - "github.com/netapp/harvest/v2/cmd/tools/rest/clirequestbuilder" - template2 "github.com/netapp/harvest/v2/cmd/tools/template" - "github.com/netapp/harvest/v2/pkg/api/ontapi/zapi" - "github.com/netapp/harvest/v2/pkg/auth" - "github.com/netapp/harvest/v2/pkg/conf" - "github.com/netapp/harvest/v2/pkg/logging" - "github.com/netapp/harvest/v2/pkg/requests" - "github.com/netapp/harvest/v2/pkg/set" - template3 "github.com/netapp/harvest/v2/pkg/template" - "github.com/netapp/harvest/v2/pkg/tree" - "github.com/netapp/harvest/v2/pkg/tree/node" - tw "github.com/netapp/harvest/v2/third_party/olekukonko/tablewriter" - "github.com/netapp/harvest/v2/third_party/tidwall/gjson" ) const ( - keyPerfAPI = "KeyPerf" SgVersion = "11.6.0" CiscoVersion = "9.3.12" ) -var ( - replacer = strings.NewReplacer("\n", "", ":", "") - objectSwaggerMap = map[string]string{ - "aggr": "xc_aggregate", - "environment_sensor": "sensors", - "fcp": "fc_port", - "flexcache": "volume", - "lif": "ip_interface", - "namespace": "nvme_namespace", - "net_port": "xc_broadcast_domain", - "ontaps3": "xc_s3_bucket", - "security_ssh": "cluster_ssh_server", - "svm_cifs": "cifs_service", - "svm_nfs": "nfs_service", - "volume": "xc_volume", - } - swaggerBytes []byte - excludePerfTemplates = map[string]struct{}{ - "volume_node.yaml": {}, // Similar metrics node_volume_* are generated via KeyPerf volume.yaml - "workload_detail.yaml": {}, - "workload_detail_volume.yaml": {}, - } - excludeRestPerfTemplates = map[string]struct{}{ - "volume.yaml": {}, // Volume performance metrics now collected via KeyPerf - } - excludeCounters = map[string]struct{}{ - "latency_histogram": {}, - "nfs4_latency_hist": {}, - "nfs41_latency_hist": {}, - "nfsv3_latency_hist": {}, - "read_latency_hist": {}, - "read_latency_histogram": {}, - "total.latency_histogram": {}, - "write_latency_hist": {}, - "write_latency_histogram": {}, - } - // Excludes these Rest gaps from logs - excludeLogRestCounters = []string{ - "external_service_op_", - "fabricpool_average_latency", - "fabricpool_get_throughput_bytes", - "fabricpool_put_throughput_bytes", - "fabricpool_stats", - "fabricpool_throughput_ops", - "iw_", - "netstat_", - "nvmf_rdma_port_", - "nvmf_tcp_port_", - "ontaps3_svm_", - "smb2_", - } - // Special handling perf objects - specialPerfObjects = map[string]bool{ - "node_nfs": true, - "svm_nfs": true, - } - - knownDescriptionGaps = map[string]struct{}{ - "availability_zone_space_available": {}, - "availability_zone_space_physical_used": {}, - "availability_zone_space_physical_used_percent": {}, - "availability_zone_space_size": {}, - "ontaps3_object_count": {}, - "security_certificate_expiry_time": {}, - "storage_unit_space_efficiency_ratio": {}, - "storage_unit_space_size": {}, - "storage_unit_space_used": {}, - "volume_capacity_tier_footprint": {}, - "volume_capacity_tier_footprint_percent": {}, - "volume_num_compress_attempts": {}, - "volume_num_compress_fail": {}, - "volume_performance_tier_footprint": {}, - "volume_performance_tier_footprint_percent": {}, - } - - knownMappingGaps = map[string]struct{}{ - "aggr_snapshot_inode_used_percent": {}, - "aggr_space_reserved": {}, - "flexcache_": {}, - "fpolicy_": {}, - "quota_disk_used_pct_threshold": {}, - "rw_ctx_": {}, - "security_audit_destination_port": {}, - "storage_unit_": {}, - "wafl_reads_from_pmem": {}, - "node_volume_nfs_": {}, - "nvm_mirror_": {}, - "volume_nfs_": {}, - "svm_vol_nfs": {}, - } - - knownMappingGapsSG = map[string]struct{}{ - "storagegrid_node_cpu_utilization_percentage": {}, - "storagegrid_private_load_balancer_storage_request_body_bytes_bucket": {}, - "storagegrid_private_load_balancer_storage_request_count": {}, - "storagegrid_private_load_balancer_storage_request_time": {}, - "storagegrid_private_load_balancer_storage_rx_bytes": {}, - "storagegrid_private_load_balancer_storage_tx_bytes": {}, - } - - excludeDocumentedRestMetrics = []string{ - "aggr_hybrid_disk_count", - "availability_zone_", - "cifs_session_idle_duration", - "cluster_space_available", - "cluster_software", - "ems_events", - "ethernet_switch_port_", - "export_rule_labels", - "fcvi_", - "flashpool_", - "health_", - "igroup_labels", - "iw_", - "mav_request_", - "mediator_labels", - "metrocluster_", - "ndmp_session", - "net_connection_labels", - "nfs_clients_idle_duration", - "nfs_diag_", - "node_cifs_", - "nvme_lif_", - "nvmf_", - "ontaps3_svm_", - "path_", - "qtree_", - "smb2_", - "snapshot_labels", - "snapshot_restore_size", - "snapshot_create_time", - "snapshot_volume_violation_count", - "snapshot_volume_violation_total_size", - "storage_unit_", - "svm_cifs_", - "svm_ontaps3_svm_", - "svm_vscan_", - "token_", - "volume_top_clients", - "volume_top_files", - "vscan_", - } - - excludeDocumentedZapiMetrics = []string{ - "ems_events", - "external_service_", - "fabricpool_", - "flexcache_", - "fpolicy_svm_failedop_notifications", - "netstat_", - "nvm_mirror_", - "quota_disk_used_pct_threshold", - "snapshot_volume_violation_count", - "snapshot_volume_violation_total_size", - } - - // Exclude extra metrics for REST - excludeNotDocumentedRestMetrics = []string{ - "ALERTS", - "cluster_space_", - "flexcache_", - "hist_", - "igroup_", - "storage_unit_", - "volume_aggr_labels", - "volume_arw_status", - } - - // Exclude extra metrics for ZAPI - excludeNotDocumentedZapiMetrics = []string{ - "ALERTS", - "hist_", - "security_", - "svm_ldap", - "volume_aggr_labels", - } - - // include StatPerf Templates - includeStatPerfTemplates = map[string]struct{}{ - "flexcache.yaml": {}, - "system_node.yaml": {}, - } -) - -var metricsPanelMap = make(map[string]PanelData) var panelKeyMap = make(map[string]bool) - -type Counters struct { - C []Counter `yaml:"counters"` -} - -type CounterMetaData struct { - Date string - OntapVersion string - SGVersion string - CiscoVersion string +var opts = &tools.Options{ + Loglevel: 2, + Image: "harvest:latest", } -type CounterTemplate struct { - Counters []Counter - CounterMetaData CounterMetaData -} - -type MetricDef struct { - API string `yaml:"API"` - Endpoint string `yaml:"Endpoint"` - ONTAPCounter string `yaml:"ONTAPCounter"` - CiscoCounter string `yaml:"CiscoCounter"` - SGCounter string `yaml:"SGCounter"` - Template string `yaml:"Template"` - Unit string `yaml:"Unit"` - Type string `yaml:"Type"` - BaseCounter string `yaml:"BaseCounter"` -} - -type PanelDef struct { - Dashboard string `yaml:"Dashboard"` - Row string `yaml:"Row"` - Type string `yaml:"Type"` - Panel string `yaml:"Panel"` - PanelLink string `yaml:"PanelLink"` -} - -type PanelData struct { - Panels []PanelDef -} - -func (m MetricDef) TableRow() string { - switch { - case strings.Contains(m.Template, "perf"): - unitTypeBase := `
Unit: ` + m.Unit + - `
Type: ` + m.Type + - `
Base: ` + m.BaseCounter - return fmt.Sprintf("| %s | `%s` | `%s`%s | %s |", - m.API, m.Endpoint, m.ONTAPCounter, unitTypeBase, m.Template) - case m.Unit != "": - unit := `
Unit: ` + m.Unit - return fmt.Sprintf("| %s | `%s` | `%s`%s | %s | ", - m.API, m.Endpoint, m.ONTAPCounter, unit, m.Template) - case strings.Contains(m.Template, "ciscorest"): - return fmt.Sprintf("| %s | `%s` | `%s` | %s |", m.API, m.Endpoint, m.CiscoCounter, m.Template) - case strings.Contains(m.Template, "storagegrid"): - return fmt.Sprintf("| %s | `%s` | `%s` | %s |", m.API, m.Endpoint, m.SGCounter, m.Template) - default: - return fmt.Sprintf("| %s | `%s` | `%s` | %s |", m.API, m.Endpoint, m.ONTAPCounter, m.Template) - } -} - -func (p PanelDef) DashboardTableRow() string { - return fmt.Sprintf("| %s | %s | %s | [%s](/%s) |", p.Dashboard, p.Row, p.Type, p.Panel, p.PanelLink) -} - -// [Top $TopResources Average Disk Utilization Per Aggregate](GRAFANA_HOST/d/cdot-aggregate/ontap3a-aggregate?orgId=1&viewPanel=63) -// [p.Panel](GRAFANA_HOST/p.PanelLink) - -type Counter struct { - Object string `yaml:"-"` - Name string `yaml:"Name"` - Description string `yaml:"Description"` - APIs []MetricDef `yaml:"APIs"` - Panels []PanelDef `yaml:"Panels"` - Labels []string `yaml:"Labels"` -} - -func (c Counter) Header() string { - return ` -| API | Endpoint | Metric | Template | -|--------|----------|--------|---------|` -} - -func (c Counter) PanelHeader() string { - return ` -| Dashboard | Row | Type | Panel | -|--------|----------|--------|--------|` -} - -func (c Counter) HasAPIs() bool { - return len(c.APIs) > 0 -} - -func (c Counter) HasPanels() bool { - return len(c.Panels) > 0 -} - -// readSwaggerJSON downloads poller swagger and convert to json format -func readSwaggerJSON() []byte { - var f []byte - path, err := downloadSwaggerForPoller(opts.Poller) - if err != nil { - log.Fatal("failed to download swagger:", err) - return nil - } - cmd := fmt.Sprintf("dasel -f %s -r yaml -w json", path) - f, err = exec.Command("bash", "-c", cmd).Output() - if err != nil { - log.Fatal("Failed to execute command:", cmd, err) - return nil - } - return f +func generateCounterTemplate(metricsPanelMap map[string]tools.PanelData) (map[string]tools.Counter, map[string]tools.Counter) { + sgCounters := tools.GenerateCounters("", make(map[string]tools.Counter), "storagegrid", metricsPanelMap) + tools.GenerateStorageGridCounterTemplate(sgCounters, SgVersion) + ciscoCounters := tools.GenerateCounters("", make(map[string]tools.Counter), "cisco", metricsPanelMap) + tools.GenerateCiscoSwitchCounterTemplate(ciscoCounters, CiscoVersion) + return sgCounters, ciscoCounters } -func downloadSwaggerForPoller(pName string) (string, error) { - var ( - poller *conf.Poller - err error - addr string - shouldDownload = true - swagTime time.Time - ) - if poller, addr, err = rest.GetPollerAndAddr(pName); err != nil { - return "", err +// generateMetadataFiles generates JSON metadata files for MCP server consumption +func generateMetadataFiles(ontapCounters, sgCounters, ciscoCounters map[string]tools.Counter) { + metadataDir := "mcp/metadata" + if err := os.MkdirAll(metadataDir, 0750); err != nil { + fmt.Printf("Error creating metadata directory: %v\n", err) + return } - tmp := os.TempDir() - swaggerPath := filepath.Join(tmp, addr+"-swagger.yaml") - fileInfo, err := os.Stat(swaggerPath) - - if os.IsNotExist(err) { - fmt.Printf("%s does not exist downloading\n", swaggerPath) + // Generate ONTAP metadata + ontapMetadata := extractMetricDescriptions(ontapCounters) + ontapPath := filepath.Join(metadataDir, "ontap_metrics.json") + if err := writeMetadataFile(ontapPath, ontapMetadata); err != nil { + fmt.Printf("Error writing ONTAP metadata: %v\n", err) } else { - swagTime = fileInfo.ModTime() - twoWeeksAgo := swagTime.Local().AddDate(0, 0, -14) - if swagTime.Before(twoWeeksAgo) { - fmt.Printf("%s is more than two weeks old, re-download", swaggerPath) - } else { - shouldDownload = false - } - } - - if shouldDownload { - swaggerURL := "https://" + addr + "/docs/api/swagger.yaml" - bytesDownloaded, err := downloadSwagger(poller, swaggerPath, swaggerURL, false) - if err != nil { - fmt.Printf("error downloading swagger %s\n", err) - if bytesDownloaded == 0 { - // if the tmp file exists, remove it since it is empty - _ = os.Remove(swaggerPath) - } - return "", err - } - fmt.Printf("downloaded %d bytes from %s\n", bytesDownloaded, swaggerURL) - } - - fmt.Printf("Using downloaded file %s with timestamp %s\n", swaggerPath, swagTime) - return swaggerPath, nil -} - -func downloadSwagger(poller *conf.Poller, path string, urlStr string, verbose bool) (int64, error) { - out, err := os.Create(path) - if err != nil { - return 0, fmt.Errorf("unable to create %s to save swagger.yaml", path) - } - defer func(out *os.File) { _ = out.Close() }(out) - request, err := requests.New("GET", urlStr, nil) - if err != nil { - return 0, err - } - - timeout, _ := time.ParseDuration(rest.DefaultTimeout) - credentials := auth.NewCredentials(poller, slog.Default()) - transport, err := credentials.Transport(request, poller) - if err != nil { - return 0, err - } - httpclient := &http.Client{Transport: transport, Timeout: timeout} - - if verbose { - requestOut, _ := httputil.DumpRequestOut(request, false) - fmt.Printf("REQUEST: %s\n%s\n", urlStr, requestOut) - } - response, err := httpclient.Do(request) - if err != nil { - return 0, err - } - //goland:noinspection GoUnhandledErrorResult - defer response.Body.Close() - - if verbose { - debugResp, _ := httputil.DumpResponse(response, false) - fmt.Printf("RESPONSE: \n%s", debugResp) - } - if response.StatusCode != http.StatusOK { - return 0, fmt.Errorf("error making request. server response statusCode=[%d]", response.StatusCode) - } - n, err := io.Copy(out, response.Body) - if err != nil { - return 0, fmt.Errorf("error while downloading %s err=%w", urlStr, err) - } - return n, nil -} - -// searchDescriptionSwagger returns ontap counter description from swagger -func searchDescriptionSwagger(objName string, ontapCounterName string) string { - val, ok := objectSwaggerMap[objName] - if ok { - objName = val - } - searchQuery := strings.Join([]string{"definitions", objName, "properties"}, ".") - cs := strings.Split(ontapCounterName, ".") - for i, c := range cs { - if i < len(cs)-1 { - searchQuery = strings.Join([]string{searchQuery, c, "properties"}, ".") - } else { - searchQuery = strings.Join([]string{searchQuery, c, "description"}, ".") - } - } - t := gjson.GetBytes(swaggerBytes, searchQuery) - return updateDescription(t.ClonedString()) -} - -// processRestCounters parse rest and restperf templates -func processRestCounters(dir string, client *rest.Client) map[string]Counter { - - restPerfCounters := visitRestTemplates(filepath.Join(dir, "conf", "restperf"), client, func(path string, client *rest.Client) map[string]Counter { - if _, ok := excludePerfTemplates[filepath.Base(path)]; ok { - return nil - } - if _, ok := excludeRestPerfTemplates[filepath.Base(path)]; ok { - return nil - } - return processRestPerfCounters(path, client) - }) - - restCounters := visitRestTemplates(filepath.Join(dir, "conf", "rest"), client, func(path string, client *rest.Client) map[string]Counter { // revive:disable-line:unused-parameter - return processRestConfigCounters(path, "REST") - }) - - keyPerfCounters := visitRestTemplates(filepath.Join(dir, "conf", "keyperf"), client, func(path string, client *rest.Client) map[string]Counter { // revive:disable-line:unused-parameter - return processRestConfigCounters(path, keyPerfAPI) - }) - - statPerfCounters := visitRestTemplates(filepath.Join(dir, "conf", "statperf"), client, func(path string, client *rest.Client) map[string]Counter { - if _, ok := includeStatPerfTemplates[filepath.Base(path)]; ok { - return processStatPerfCounters(path, client) - } - return nil - }) - maps.Copy(restCounters, restPerfCounters) - - keyPerfKeys := slices.Sorted(maps.Keys(keyPerfCounters)) - for _, k := range keyPerfKeys { - if strings.Contains(k, "timestamp") || strings.Contains(k, "labels") { - continue - } - v := keyPerfCounters[k] - if v1, ok := restCounters[k]; !ok { - restCounters[k] = v - } else { - v1.APIs = append(v1.APIs, v.APIs...) - restCounters[k] = v1 - } - } - - statPerfKeys := slices.Sorted(maps.Keys(statPerfCounters)) - for _, k := range statPerfKeys { - if strings.Contains(k, "labels") { - continue - } - v := statPerfCounters[k] - if v1, ok := restCounters[k]; !ok { - restCounters[k] = v - } else { - v1.APIs = append(v1.APIs, v.APIs...) - restCounters[k] = v1 - } - } - return restCounters -} - -// processZapiCounters parse zapi and zapiperf templates -func processZapiCounters(dir string, client *zapi.Client) map[string]Counter { - zapiCounters := visitZapiTemplates(filepath.Join(dir, "conf", "zapi", "cdot"), client, func(path string, client *zapi.Client) map[string]Counter { // revive:disable-line:unused-parameter - return processZapiConfigCounters(path) - }) - zapiPerfCounters := visitZapiTemplates(filepath.Join(dir, "conf", "zapiperf", "cdot"), client, func(path string, client *zapi.Client) map[string]Counter { - if _, ok := excludePerfTemplates[filepath.Base(path)]; ok { - return nil - } - return processZAPIPerfCounters(path, client) - }) - - maps.Copy(zapiCounters, zapiPerfCounters) - return zapiCounters -} - -// parseZapiCounters parse zapi template counters -func parseZapiCounters(elem *node.Node, path []string, object string, zc map[string]string) { - - name := elem.GetNameS() - newPath := path - - if elem.GetNameS() != "" { - newPath = append(newPath, name) - } - - if elem.GetContentS() != "" { - v, k := handleZapiCounter(newPath, elem.GetContentS(), object) - if k != "" { - zc[k] = v - } - } - - for _, child := range elem.GetChildren() { - parseZapiCounters(child, newPath, object, zc) + fmt.Printf("ONTAP metadata file generated at %s with %d metrics\n", ontapPath, len(ontapMetadata)) } -} -// handleZapiCounter returns zapi ontap and display counter name -func handleZapiCounter(path []string, content string, object string) (string, string) { - var ( - name, display, key string - splitValues, fullPath []string - ) - - splitValues = strings.Split(content, "=>") - if len(splitValues) == 1 { - name = content + // Generate StorageGrid metadata + sgMetadata := extractMetricDescriptions(sgCounters) + sgPath := filepath.Join(metadataDir, "storagegrid_metrics.json") + if err := writeMetadataFile(sgPath, sgMetadata); err != nil { + fmt.Printf("Error writing StorageGrid metadata: %v\n", err) } else { - name = splitValues[0] - display = strings.TrimSpace(splitValues[1]) - } - - name = strings.TrimSpace(strings.TrimLeft(name, "^")) - fullPath = path - fullPath = append(fullPath, name) - key = strings.Join(fullPath, ".") - if display == "" { - display = template3.ParseZAPIDisplay(object, fullPath) - } - - if content[0] != '^' { - return key, object + "_" + display - } - - return "", "" -} - -// processRestConfigCounters process Rest config templates -func processRestConfigCounters(path string, api string) map[string]Counter { - var ( - counters = make(map[string]Counter) - isInstanceLabels bool - ) - var metricLabels []string - var labels []string - t, err := tree.ImportYaml(path) - if t == nil || err != nil { - fmt.Printf("Unable to import template file %s. File is invalid or empty err=%s\n", path, err) - return nil - } - - model, err := template2.ReadTemplate(path) - if err != nil { - fmt.Printf("Unable to import template file %s. File is invalid or empty err=%s\n", path, err) - return nil - } - noExtraMetrics := len(model.MultiplierMetrics) == 0 && len(model.PluginMetrics) == 0 - templateCounters := t.GetChildS("counters") - if model.ExportData == "false" && noExtraMetrics { - return nil - } - - if templateCounters != nil { - metricLabels, labels, isInstanceLabels = getAllExportedLabels(t, templateCounters.GetAllChildContentS()) - processCounters(templateCounters.GetAllChildContentS(), &model, path, model.Query, counters, metricLabels, api) - if isInstanceLabels { - // This is for object_labels metrics - harvestName := model.Object + "_" + "labels" - description := "This metric provides information about " + model.Name - counters[harvestName] = Counter{ - Name: harvestName, - Description: description, - APIs: []MetricDef{ - { - API: "REST", - Endpoint: model.Query, - Template: path, - ONTAPCounter: "Harvest generated", - }, - }, - Panels: metricsPanelMap[harvestName].Panels, - Labels: labels, - } - } - } - - endpoints := t.GetChildS("endpoints") - if endpoints != nil { - for _, endpoint := range endpoints.GetChildren() { - var query string - for _, line := range endpoint.GetChildren() { - if line.GetNameS() == "query" { - query = line.GetContentS() - } - if line.GetNameS() == "counters" { - processCounters(line.GetAllChildContentS(), &model, path, query, counters, metricLabels, api) - } - } - } - } - - // If the template has any PluginMetrics, add them - for _, metric := range model.PluginMetrics { - co := Counter{ - Object: model.Object, - Name: model.Object + "_" + metric.Name, - APIs: []MetricDef{ - { - API: api, - Endpoint: model.Query, - Template: path, - ONTAPCounter: metric.Source, - }, - }, - Panels: metricsPanelMap[model.Object+"_"+metric.Name].Panels, - } - if model.ExportData != "false" { - counters[co.Name] = co - } - } - - if api == keyPerfAPI { - // handling for templates with common object names - if specialPerfObjects[model.Object] { - return specialHandlingPerfCounters(counters, model) - } - } - - return counters -} - -func processCounters(counterContents []string, model *template2.Model, path, query string, counters map[string]Counter, metricLabels []string, api string) { - var ( - staticCounterDef keyperf.ObjectCounters - err error - defLocation string - ) - if api == keyPerfAPI { - logger := logging.Get() - // CLI conf/keyperf/9.15.0/aggr.yaml - // CI ../../conf/keyperf/9.15.0/volume.yaml - defLocation = filepath.Join(filepath.Dir(filepath.Dir(path)), "static_counter_definitions.yaml") - - staticCounterDef, err = keyperf.LoadStaticCounterDefinitions(model.Object, defLocation, logger) - if err != nil { - fmt.Printf("Failed to load static counter definitions=%s\n", err) - } + fmt.Printf("StorageGrid metadata file generated at %s with %d metrics\n", sgPath, len(sgMetadata)) } - for _, c := range counterContents { - if c == "" { - continue - } - var co Counter - name, display, m, _ := template3.ParseMetric(c) - if _, ok := excludeCounters[name]; ok { - continue - } - description := searchDescriptionSwagger(model.Object, name) - harvestName := model.Object + "_" + display - if m == "float" { - if api == keyPerfAPI { - var ( - unit string - counterType string - denominator string - ) - switch { - case strings.Contains(name, "latency"): - counterType = "average" - unit = "microsec" - denominator = model.Object + "_" + strings.Replace(name, "latency", "iops", 1) - case strings.Contains(name, "iops"): - counterType = "rate" - unit = "per_sec" - case strings.Contains(name, "throughput"): - counterType = "rate" - unit = "b_per_sec" - case strings.Contains(name, "timestamp"): - counterType = "delta" - unit = "sec" - default: - // look up metric in staticCounterDef - if counterDef, exists := staticCounterDef.CounterDefinitions[name]; exists { - counterType = counterDef.Type - unit = counterDef.BaseCounter - } - } - - co = Counter{ - Object: model.Object, - Name: harvestName, - Description: description, - APIs: []MetricDef{ - { - API: api, - Endpoint: model.Query, - Template: path, - ONTAPCounter: name, - Unit: unit, - Type: counterType, - BaseCounter: denominator, - }, - }, - Panels: metricsPanelMap[harvestName].Panels, - Labels: metricLabels, - } - } else { - co = Counter{ - Object: model.Object, - Name: harvestName, - Description: description, - APIs: []MetricDef{ - { - API: api, - Endpoint: query, - Template: path, - ONTAPCounter: name, - }, - }, - Panels: metricsPanelMap[harvestName].Panels, - Labels: metricLabels, - } - } - counters[harvestName] = co - - // If the template has any MultiplierMetrics, add them - for _, metric := range model.MultiplierMetrics { - mc := co - addAggregatedCounter(&mc, metric, harvestName, display) - counters[mc.Name] = mc - } - } + // Generate Cisco metadata + ciscoMetadata := extractMetricDescriptions(ciscoCounters) + ciscoPath := filepath.Join(metadataDir, "cisco_metrics.json") + if err := writeMetadataFile(ciscoPath, ciscoMetadata); err != nil { + fmt.Printf("Error writing Cisco metadata: %v\n", err) + } else { + fmt.Printf("Cisco metadata file generated at %s with %d metrics\n", ciscoPath, len(ciscoMetadata)) } } -// processZAPIPerfCounters process ZapiPerf counters -func processZAPIPerfCounters(path string, client *zapi.Client) map[string]Counter { - var ( - counters = make(map[string]Counter) - request, response *node.Node - zapiUnitMap = make(map[string]string) - zapiTypeMap = make(map[string]string) - zapiDescMap = make(map[string]string) - zapiBaseCounterMap = make(map[string]string) - zapiDeprecateCounterMap = make(map[string]string) - ) - t, err := tree.ImportYaml(path) - if t == nil || err != nil { - fmt.Printf("Unable to import template file %s. File is invalid or empty\n", path) - return nil - } - model, err := template2.ReadTemplate(path) - if err != nil { - fmt.Printf("Unable to import template file %s. File is invalid or empty err=%s\n", path, err) - return nil - } - - noExtraMetrics := len(model.MultiplierMetrics) == 0 && len(model.PluginMetrics) == 0 - templateCounters := t.GetChildS("counters") - override := t.GetChildS("override") - - if model.ExportData == "false" && noExtraMetrics { - return nil - } - - if templateCounters == nil { - return nil - } - - // build request - request = node.NewXMLS("perf-object-counter-list-info") - request.NewChildS("objectname", model.Query) - - if err = client.BuildRequest(request); err != nil { - fmt.Printf("error while building request %+v\n", err) - return nil - } - - if response, err = client.Invoke(""); err != nil { - fmt.Printf("error while invoking api %+v\n", err) - return nil - } - - // fetch counter elements - if elems := response.GetChildS("counters"); elems != nil && len(elems.GetChildren()) != 0 { - for _, counter := range elems.GetChildren() { - name := counter.GetChildContentS("name") - if name == "" { - continue - } - ty := counter.GetChildContentS("properties") - if override != nil { - oty := override.GetChildContentS(name) - if oty != "" { - ty = oty - } - } - - zapiUnitMap[name] = counter.GetChildContentS("unit") - zapiDescMap[name] = updateDescription(counter.GetChildContentS("desc")) - zapiTypeMap[name] = ty - zapiBaseCounterMap[name] = counter.GetChildContentS("base-counter") - - if counter.GetChildContentS("is-deprecated") == "true" { - if r := counter.GetChildContentS("replaced-by"); r != "" { - zapiDeprecateCounterMap[name] = r - } - } - } - } - - metricLabels, labels, isInstanceLabels := getAllExportedLabels(t, templateCounters.GetAllChildContentS()) - if isInstanceLabels { - // This is for object_labels metrics - harvestName := model.Object + "_" + "labels" - description := "This metric provides information about " + model.Name - counters[harvestName] = Counter{ - Name: harvestName, - Description: description, - APIs: []MetricDef{ - { - API: "ZAPI", - Endpoint: model.Query, - Template: path, - ONTAPCounter: "Harvest generated", - }, - }, - Panels: metricsPanelMap[harvestName].Panels, - Labels: labels, - } - } - for _, c := range templateCounters.GetAllChildContentS() { - if c != "" { - name, display, m, _ := template3.ParseMetric(c) - if after, ok := strings.CutPrefix(display, model.Object); ok { - display = after - display = strings.TrimPrefix(display, "_") - } - harvestName := model.Object + "_" + display - if m == "float" { - if _, ok := excludeCounters[name]; ok { - continue - } - if zapiTypeMap[name] != "string" { - description := zapiDescMap[name] - if strings.Contains(path, "volume.yaml") && model.Object == "volume" { - if description != "" { - description += " " - } - description += "(Note: This is applicable only for ONTAP 9.9 and below. Harvest uses KeyPerf collector for ONTAP 9.10 onwards.)" - } - co := Counter{ - Object: model.Object, - Name: harvestName, - Description: description, - APIs: []MetricDef{ - { - API: "ZapiPerf", - Endpoint: "perf-object-get-instances" + " " + model.Query, - Template: path, - ONTAPCounter: name, - Unit: zapiUnitMap[name], - Type: zapiTypeMap[name], - BaseCounter: zapiBaseCounterMap[name], - }, - }, - Panels: metricsPanelMap[harvestName].Panels, - Labels: metricLabels, - } - if model.ExportData != "false" { - counters[harvestName] = co - } - - // handle deprecate counters - if rName, ok := zapiDeprecateCounterMap[name]; ok { - hName := model.Object + "_" + rName - ro := Counter{ - Object: model.Object, - Name: hName, - Description: zapiDescMap[rName], - APIs: []MetricDef{ - { - API: "ZapiPerf", - Endpoint: "perf-object-get-instances" + " " + model.Query, - Template: path, - ONTAPCounter: rName, - Unit: zapiUnitMap[rName], - Type: zapiTypeMap[rName], - BaseCounter: zapiBaseCounterMap[rName], - }, - }, - Panels: metricsPanelMap[hName].Panels, - } - if model.ExportData != "false" { - counters[hName] = ro - } - } - - // If the template has any MultiplierMetrics, add them - for _, metric := range model.MultiplierMetrics { - mc := co - addAggregatedCounter(&mc, metric, harvestName, display) - counters[mc.Name] = mc - } - } - } - } - } - - // If the template has any PluginMetrics, add them - for _, metric := range model.PluginMetrics { - co := Counter{ - Object: model.Object, - Name: model.Object + "_" + metric.Name, - APIs: []MetricDef{ - { - API: "ZapiPerf", - Endpoint: model.Query, - Template: path, - ONTAPCounter: metric.Source, - }, - }, - Panels: metricsPanelMap[model.Object+"_"+metric.Name].Panels, +// extractMetricDescriptions extracts just the name->description mapping +func extractMetricDescriptions(counters map[string]tools.Counter) map[string]string { + metadata := make(map[string]string) + for _, counter := range counters { + // Only include counters with descriptions + if counter.Description != "" { + metadata[counter.Name] = counter.Description } - counters[co.Name] = co - } - // handling for templates with common object names - if specialPerfObjects[model.Object] { - return specialHandlingPerfCounters(counters, model) } - return counters + return metadata } -func processZapiConfigCounters(path string) map[string]Counter { - var ( - counters = make(map[string]Counter) - isInstanceLabels bool - ) - var metricLabels []string - var labels []string - t, err := tree.ImportYaml(path) - if t == nil || err != nil { - fmt.Printf("Unable to import template file %s. File is invalid or empty\n", path) - return nil - } - model, err := template2.ReadTemplate(path) +// writeMetadataFile writes metadata to a JSON file +func writeMetadataFile(path string, metadata map[string]string) error { + file, err := os.Create(path) if err != nil { - fmt.Printf("Unable to import template file %s. File is invalid or empty err=%s\n", path, err) - return nil - } - noExtraMetrics := len(model.MultiplierMetrics) == 0 && len(model.PluginMetrics) == 0 - templateCounters := t.GetChildS("counters") - if model.ExportData == "false" && noExtraMetrics { - return nil - } - if templateCounters == nil { - return nil - } - - zc := make(map[string]string) - metricLabels, labels, isInstanceLabels = getAllExportedLabels(t, templateCounters.GetAllChildContentS()) - if isInstanceLabels { - // This is for object_labels metrics - harvestName := model.Object + "_" + "labels" - description := "This metric provides information about " + model.Name - counters[harvestName] = Counter{ - Name: harvestName, - Description: description, - APIs: []MetricDef{ - { - API: "ZAPI", - Endpoint: model.Query, - Template: path, - ONTAPCounter: "Harvest generated", - }, - }, - Panels: metricsPanelMap[harvestName].Panels, - Labels: labels, - } - } - for _, c := range templateCounters.GetChildren() { - parseZapiCounters(c, []string{}, model.Object, zc) + return err } - - for k, v := range zc { - if _, ok := excludeCounters[k]; ok { - continue - } - co := Counter{ - Object: model.Object, - Name: k, - APIs: []MetricDef{ - { - API: "ZAPI", - Endpoint: model.Query, - Template: path, - ONTAPCounter: v, - }, - }, - Panels: metricsPanelMap[k].Panels, - Labels: metricLabels, - } - if model.ExportData != "false" { - counters[k] = co - } - - // If the template has any MultiplierMetrics, add them - for _, metric := range model.MultiplierMetrics { - mc := co - addAggregatedCounter(&mc, metric, co.Name, model.Object) - counters[mc.Name] = mc - } - } - - // If the template has any PluginMetrics, add them - for _, metric := range model.PluginMetrics { - co := Counter{ - Object: model.Object, - Name: model.Object + "_" + metric.Name, - APIs: []MetricDef{ - { - API: "ZAPI", - Endpoint: model.Query, - Template: path, - ONTAPCounter: metric.Source, - }, - }, - Panels: metricsPanelMap[model.Object+"_"+metric.Name].Panels, - } - counters[co.Name] = co - } - return counters -} - -func visitRestTemplates(dir string, client *rest.Client, eachTemp func(path string, client *rest.Client) map[string]Counter) map[string]Counter { - result := make(map[string]Counter) - err := filepath.Walk(dir, func(path string, _ os.FileInfo, err error) error { - if err != nil { - log.Fatal("failed to read directory:", err) - } - ext := filepath.Ext(path) - if ext != ".yaml" { - return nil - } - if strings.HasSuffix(path, "default.yaml") || strings.HasSuffix(path, "static_counter_definitions.yaml") { - return nil - } - r := eachTemp(path, client) - maps.Copy(result, r) - return nil - }) - - if err != nil { - log.Fatal("failed to read template:", err) - return nil - } - return result -} - -func visitZapiTemplates(dir string, client *zapi.Client, eachTemp func(path string, client *zapi.Client) map[string]Counter) map[string]Counter { - result := make(map[string]Counter) - err := filepath.Walk(dir, func(path string, _ os.FileInfo, err error) error { - if err != nil { - log.Fatal("failed to read directory:", err) - } - ext := filepath.Ext(path) - if ext != ".yaml" { - return nil - } - - r := eachTemp(path, client) - maps.Copy(result, r) - return nil - }) - - if err != nil { - log.Fatal("failed to read template:", err) - return nil - } - return result -} - -func updateDescription(description string) string { - s := replacer.Replace(description) - return s -} - -func generateCounterTemplate() (map[string]Counter, map[string]Counter) { - sgCounters := generateCounters("", make(map[string]Counter), "storagegrid") - generateStorageGridCounterTemplate(sgCounters, SgVersion) - ciscoCounters := generateCounters("", make(map[string]Counter), "cisco") - generateCiscoSwitchCounterTemplate(ciscoCounters, CiscoVersion) - return sgCounters, ciscoCounters -} - -// generateMetadataFiles generates JSON metadata files for MCP server consumption -func generateMetadataFiles(ontapCounters, sgCounters, ciscoCounters map[string]Counter) { - metadataDir := "mcp/metadata" - if err := os.MkdirAll(metadataDir, 0750); err != nil { - fmt.Printf("Error creating metadata directory: %v\n", err) - return - } - - // Generate ONTAP metadata - ontapMetadata := extractMetricDescriptions(ontapCounters) - ontapPath := filepath.Join(metadataDir, "ontap_metrics.json") - if err := writeMetadataFile(ontapPath, ontapMetadata); err != nil { - fmt.Printf("Error writing ONTAP metadata: %v\n", err) - } else { - fmt.Printf("ONTAP metadata file generated at %s with %d metrics\n", ontapPath, len(ontapMetadata)) - } - - // Generate StorageGrid metadata - sgMetadata := extractMetricDescriptions(sgCounters) - sgPath := filepath.Join(metadataDir, "storagegrid_metrics.json") - if err := writeMetadataFile(sgPath, sgMetadata); err != nil { - fmt.Printf("Error writing StorageGrid metadata: %v\n", err) - } else { - fmt.Printf("StorageGrid metadata file generated at %s with %d metrics\n", sgPath, len(sgMetadata)) - } - - // Generate Cisco metadata - ciscoMetadata := extractMetricDescriptions(ciscoCounters) - ciscoPath := filepath.Join(metadataDir, "cisco_metrics.json") - if err := writeMetadataFile(ciscoPath, ciscoMetadata); err != nil { - fmt.Printf("Error writing Cisco metadata: %v\n", err) - } else { - fmt.Printf("Cisco metadata file generated at %s with %d metrics\n", ciscoPath, len(ciscoMetadata)) - } -} - -// extractMetricDescriptions extracts just the name->description mapping -func extractMetricDescriptions(counters map[string]Counter) map[string]string { - metadata := make(map[string]string) - for _, counter := range counters { - // Only include counters with descriptions - if counter.Description != "" { - metadata[counter.Name] = counter.Description - } - } - return metadata -} - -// writeMetadataFile writes metadata to a JSON file -func writeMetadataFile(path string, metadata map[string]string) error { - file, err := os.Create(path) - if err != nil { - return err - } - defer file.Close() + defer file.Close() encoder := json.NewEncoder(file) encoder.SetIndent("", " ") return encoder.Encode(metadata) } -func generateOntapCounterTemplate(counters map[string]Counter, version string) { - targetPath := "docs/ontap-metrics.md" - t, err := template.New("counter.tmpl").ParseFiles("cmd/tools/generate/counter.tmpl") - if err != nil { - panic(err) - } - out, err := os.Create(targetPath) - if err != nil { - panic(err) - } - - keys := make([]string, 0, len(counters)) - for k := range counters { - keys = append(keys, k) - } - sort.Strings(keys) - values := make([]Counter, 0, len(keys)) - - table := tw.NewWriter(os.Stdout) - table.SetBorder(false) - table.SetAutoFormatHeaders(false) - table.SetAutoWrapText(false) - table.SetHeader([]string{"Missing", "Counter", "APIs", "Endpoint", "ONTAPCounter", "Template"}) - - for _, k := range keys { - if k == "" { - continue - } - counter := counters[k] - - if counter.Description == "" { - for _, def := range counter.APIs { - if _, ok := knownDescriptionGaps[counter.Name]; !ok { - appendRow(table, "Description", counter, def) - } - } - } - values = append(values, counter) - } - - for _, k := range keys { - if k == "" { - continue - } - counter := counters[k] - - // Print such counters which are missing Rest mapping - if len(counter.APIs) == 1 { - if counter.APIs[0].API == "ZAPI" { - isPrint := true - for _, substring := range excludeLogRestCounters { - if strings.HasPrefix(counter.Name, substring) { - isPrint = false - break - } - } - // missing Rest Mapping - if isPrint { - for _, def := range counter.APIs { - hasPrefix := false - for prefix := range knownMappingGaps { - if strings.HasPrefix(counter.Name, prefix) { - hasPrefix = true - break - } - } - if !hasPrefix { - appendRow(table, "REST", counter, def) - } - } - } - } - } - - for _, def := range counter.APIs { - if def.ONTAPCounter == "" { - for _, def := range counter.APIs { - hasPrefix := false - for prefix := range knownMappingGaps { - if strings.HasPrefix(counter.Name, prefix) { - hasPrefix = true - break - } - } - if !hasPrefix { - appendRow(table, "Mapping", counter, def) - } - } - } - } - } - table.Render() - c := CounterTemplate{ - Counters: values, - CounterMetaData: CounterMetaData{ - Date: time.Now().Format("2006-Jan-02"), - OntapVersion: version, - }, - } - - err = t.Execute(out, c) - if err != nil { - panic(err) - } - fmt.Printf("Harvest metric documentation generated at %s \n", targetPath) - - if table.NumLines() > 0 { - slog.Error("Issues found: Please refer to the table above") - os.Exit(1) - } -} - -func generateStorageGridCounterTemplate(counters map[string]Counter, version string) { - targetPath := "docs/storagegrid-metrics.md" - t, err := template.New("storagegrid_counter.tmpl").ParseFiles("cmd/tools/generate/storagegrid_counter.tmpl") - if err != nil { - panic(err) - } - out, err := os.Create(targetPath) - if err != nil { - panic(err) - } - - keys := make([]string, 0, len(counters)) - for k := range counters { - keys = append(keys, k) - } - sort.Strings(keys) - values := make([]Counter, 0, len(keys)) - - table := tw.NewWriter(os.Stdout) - table.SetBorder(false) - table.SetAutoFormatHeaders(false) - table.SetAutoWrapText(false) - table.SetHeader([]string{"Missing", "Counter", "APIs", "Endpoint", "SGCounter", "Template"}) - - for _, k := range keys { - if k == "" { - continue - } - counter := counters[k] - if !strings.HasPrefix(counter.Name, "storagegrid_") { - continue - } - - if _, ok := knownMappingGapsSG[k]; !ok { - if counter.Description == "" { - appendRow(table, "Description", counter, MetricDef{API: ""}) - } - } - - values = append(values, counter) - } - - table.Render() - c := CounterTemplate{ - Counters: values, - CounterMetaData: CounterMetaData{ - Date: time.Now().Format("2006-Jan-02"), - SGVersion: version, - }, - } - - err = t.Execute(out, c) - if err != nil { - panic(err) - } - fmt.Printf("Harvest metric documentation generated at %s \n", targetPath) - - if table.NumLines() > 0 { - log.Fatalf("Issues found: refer table above") - } -} - -func generateCiscoSwitchCounterTemplate(counters map[string]Counter, version string) { - targetPath := "docs/cisco-switch-metrics.md" - t, err := template.New("cisco_counter.tmpl").ParseFiles("cmd/tools/generate/cisco_counter.tmpl") - if err != nil { - panic(err) - } - out, err := os.Create(targetPath) - if err != nil { - panic(err) - } - - keys := make([]string, 0, len(counters)) - for k := range counters { - keys = append(keys, k) - } - sort.Strings(keys) - values := make([]Counter, 0, len(keys)) - - table := tw.NewWriter(os.Stdout) - table.SetBorder(false) - table.SetAutoFormatHeaders(false) - table.SetAutoWrapText(false) - table.SetHeader([]string{"Missing", "Counter", "APIs", "Endpoint", "CiscoCounter", "Template"}) - - for _, k := range keys { - if k == "" { - continue - } - counter := counters[k] - if !strings.HasPrefix(counter.Name, "cisco_") { - continue - } - - if counter.Description == "" { - appendRow(table, "Description", counter, MetricDef{API: ""}) - } - - values = append(values, counter) - } - - table.Render() - c := CounterTemplate{ - Counters: values, - CounterMetaData: CounterMetaData{ - Date: time.Now().Format("2006-Jan-02"), - CiscoVersion: version, - }, - } - - err = t.Execute(out, c) - if err != nil { - panic(err) - } - fmt.Printf("Harvest metric documentation generated at %s \n", targetPath) - - if table.NumLines() > 0 { - log.Fatalf("Issues found: refer table above") - } -} - -func appendRow(table *tw.Table, missing string, counter Counter, def MetricDef) { - if def.API != "" { - table.Append([]string{missing, counter.Name, def.API, def.Endpoint, def.ONTAPCounter, def.Template}) - } else { - table.Append([]string{missing, counter.Name}) - } -} - -// Regex to match NFS version and operation -var reRemove = regexp.MustCompile(`NFSv\d+\.\d+`) - -func mergeCounters(restCounters map[string]Counter, zapiCounters map[string]Counter) map[string]Counter { - // handle special counters - restKeys := slices.Sorted(maps.Keys(restCounters)) - for _, k := range restKeys { - v := restCounters[k] - hashIndex := strings.Index(k, "#") - if hashIndex != -1 { - if v1, ok := restCounters[v.Name]; !ok { - v.Description = reRemove.ReplaceAllString(v.Description, "") - // Remove extra spaces from the description - v.Description = strings.Join(strings.Fields(v.Description), " ") - restCounters[v.Name] = v - } else { - v1.APIs = append(v1.APIs, v.APIs...) - restCounters[v.Name] = v1 - } - delete(restCounters, k) - } - } - - zapiKeys := slices.Sorted(maps.Keys(zapiCounters)) - for _, k := range zapiKeys { - v := zapiCounters[k] - hashIndex := strings.Index(k, "#") - if hashIndex != -1 { - if v1, ok := zapiCounters[v.Name]; !ok { - v.Description = reRemove.ReplaceAllString(v.Description, "") - // Remove extra spaces from the description - v.Description = strings.Join(strings.Fields(v.Description), " ") - zapiCounters[v.Name] = v - } else { - v1.APIs = append(v1.APIs, v.APIs...) - zapiCounters[v.Name] = v1 - } - delete(zapiCounters, k) - } - } - - // special keys are deleted hence sort again - zapiKeys = slices.Sorted(maps.Keys(zapiCounters)) - for _, k := range zapiKeys { - v := zapiCounters[k] - if v1, ok := restCounters[k]; ok { - v1.APIs = append(v1.APIs, v.APIs...) - restCounters[k] = v1 - } else { - zapiDef := v.APIs[0] - if zapiDef.ONTAPCounter == "instance_name" || zapiDef.ONTAPCounter == "instance_uuid" { - continue - } - co := Counter{ - Object: v.Object, - Name: v.Name, - Description: v.Description, - APIs: []MetricDef{zapiDef}, - Panels: v.Panels, - } - restCounters[v.Name] = co - } - } - return restCounters -} - -func processRestPerfCounters(path string, client *rest.Client) map[string]Counter { - var ( - records []gjson.Result - counterSchema gjson.Result - counters = make(map[string]Counter) - ) - t, err := tree.ImportYaml(path) - if t == nil || err != nil { - fmt.Printf("Unable to import template file %s. File is invalid or empty\n", path) - return nil - } - model, err := template2.ReadTemplate(path) - if err != nil { - fmt.Printf("Unable to import template file %s. File is invalid or empty err=%s\n", path, err) - return nil - } - noExtraMetrics := len(model.MultiplierMetrics) == 0 && len(model.PluginMetrics) == 0 - templateCounters := t.GetChildS("counters") - override := t.GetChildS("override") - if model.ExportData == "false" && noExtraMetrics { - return nil - } - if templateCounters == nil { - return nil - } - counterMap := make(map[string]string) - counterMapNoPrefix := make(map[string]string) - metricLabels, labels, isInstanceLabels := getAllExportedLabels(t, templateCounters.GetAllChildContentS()) - if isInstanceLabels { - description := "This metric provides information about " + model.Name - // This is for object_labels metrics - harvestName := model.Object + "_" + "labels" - counters[harvestName] = Counter{ - Name: harvestName, - Description: description, - APIs: []MetricDef{ - { - API: "RestPerf", - Endpoint: model.Query, - Template: path, - ONTAPCounter: "Harvest generated", - }, - }, - Panels: metricsPanelMap[harvestName].Panels, - Labels: labels, - } - } - for _, c := range templateCounters.GetAllChildContentS() { - if c != "" { - name, display, m, _ := template3.ParseMetric(c) - if m == "float" { - counterMap[name] = model.Object + "_" + display - counterMapNoPrefix[name] = display - } - } - } - href := rest.NewHrefBuilder(). - APIPath(model.Query). - Build() - records, err = rest.FetchAll(client, href) - if err != nil { - fmt.Printf("error while invoking api %+v\n", err) - return nil - } - - firstRecord := records[0] - if firstRecord.Exists() { - counterSchema = firstRecord.Get("counter_schemas") - } else { - return nil - } - counterSchema.ForEach(func(_, r gjson.Result) bool { - if !r.IsObject() { - return true - } - ontapCounterName := r.Get("name").ClonedString() - if _, ok := excludeCounters[ontapCounterName]; ok { - return true - } - - description := r.Get("description").ClonedString() - ty := r.Get("type").ClonedString() - if override != nil { - oty := override.GetChildContentS(ontapCounterName) - if oty != "" { - ty = oty - } - } - if v, ok := counterMap[ontapCounterName]; ok { - if ty == "string" { - return true - } - c := Counter{ - Object: model.Object, - Name: v, - Description: description, - APIs: []MetricDef{ - { - API: "RestPerf", - Endpoint: model.Query, - Template: path, - ONTAPCounter: ontapCounterName, - Unit: r.Get("unit").ClonedString(), - Type: ty, - BaseCounter: r.Get("denominator.name").ClonedString(), - }, - }, - Panels: metricsPanelMap[v].Panels, - Labels: metricLabels, - } - if model.ExportData != "false" { - counters[c.Name] = c - } - - // If the template has any MultiplierMetrics, add them - for _, metric := range model.MultiplierMetrics { - mc := c - addAggregatedCounter(&mc, metric, v, counterMapNoPrefix[ontapCounterName]) - counters[mc.Name] = mc - } - } - return true - }) - - // If the template has any PluginMetrics, add them - for _, metric := range model.PluginMetrics { - co := Counter{ - Object: model.Object, - Name: model.Object + "_" + metric.Name, - APIs: []MetricDef{ - { - API: "RestPerf", - Endpoint: model.Query, - Template: path, - ONTAPCounter: metric.Source, - }, - }, - Panels: metricsPanelMap[model.Object+"_"+metric.Name].Panels, - } - counters[co.Name] = co - } - // handling for templates with common object names/metric name - if specialPerfObjects[model.Object] { - return specialHandlingPerfCounters(counters, model) - } - return counters -} - -func processStatPerfCounters(path string, client *rest.Client) map[string]Counter { - var ( - records []gjson.Result - counters = make(map[string]Counter) - cliCommand []byte - ) - t, err := tree.ImportYaml(path) - if t == nil || err != nil { - fmt.Printf("Unable to import template file %s. File is invalid or empty\n", path) - return nil - } - model, err := template2.ReadTemplate(path) - if err != nil { - fmt.Printf("Unable to import template file %s. File is invalid or empty err=%s\n", path, err) - return nil - } - noExtraMetrics := len(model.MultiplierMetrics) == 0 && len(model.PluginMetrics) == 0 - templateCounters := t.GetChildS("counters") - override := t.GetChildS("override") - if model.ExportData == "false" && noExtraMetrics { - return nil - } - if templateCounters == nil { - return nil - } - counterMap := make(map[string]string) - counterMapNoPrefix := make(map[string]string) - metricLabels, labels, isInstanceLabels := getAllExportedLabels(t, templateCounters.GetAllChildContentS()) - if isInstanceLabels { - description := "This metric provides information about " + model.Name - // This is for object_labels metrics - harvestName := model.Object + "_" + "labels" - counters[harvestName] = Counter{ - Name: harvestName, - Description: description, - APIs: []MetricDef{ - { - API: "StatPerf", - Endpoint: model.Query, - Template: path, - ONTAPCounter: "Harvest generated", - }, - }, - Panels: metricsPanelMap[harvestName].Panels, - Labels: labels, - } - } - for _, c := range templateCounters.GetAllChildContentS() { - if c != "" { - name, display, m, _ := template3.ParseMetric(c) - if m == "float" { - counterMap[name] = model.Object + "_" + display - counterMapNoPrefix[name] = display - } - } - } - cliCommand, err = clirequestbuilder.New(). - BaseSet(statperf.GetCounterInstanceBaseSet()). - Query("statistics catalog counter show"). - Object(model.Query). - Fields([]string{"counter", "base-counter", "properties", "type", "is-deprecated", "replaced-by", "unit", "description"}). - Build() - if err != nil { - fmt.Printf("error while build clicommand %+v\n", err) - return nil - } - records, err = rest.FetchPost(client, "api/private/cli", cliCommand) - if err != nil { - fmt.Printf("error while invoking api %+v\n", err) - return nil - } - - firstRecord := records[0] - fr := firstRecord.ClonedString() - if fr == "" { - fmt.Printf("no data found for query %s template %s", model.Query, path) - return nil - } - - s := &statperf.StatPerf{} - pCounters, err := s.ParseCounters(fr) - if err != nil { - fmt.Printf("error while parsing records for query %s template %s: %+v\n", model.Query, path, err) - return nil - } - - for _, p := range pCounters { - ontapCounterName := statperf.NormalizeCounterValue(p.Name) - description := statperf.NormalizeCounterValue(p.Description) - ty := p.Type - if override != nil { - oty := override.GetChildContentS(ontapCounterName) - if oty != "" { - ty = oty - } - } - if v, ok := counterMap[ontapCounterName]; ok { - if ty == "string" { - continue - } - c := Counter{ - Object: model.Object, - Name: v, - Description: description, - APIs: []MetricDef{ - { - API: "StatPerf", - Endpoint: model.Query, - Template: path, - ONTAPCounter: ontapCounterName, - Unit: statperf.NormalizeCounterValue(p.Unit), - Type: statperf.NormalizeCounterValue(ty), - BaseCounter: statperf.NormalizeCounterValue(p.BaseCounter), - }, - }, - Panels: metricsPanelMap[v].Panels, - Labels: metricLabels, - } - if model.ExportData != "false" { - counters[c.Name] = c - } - - // If the template has any MultiplierMetrics, add them - for _, metric := range model.MultiplierMetrics { - mc := c - addAggregatedCounter(&mc, metric, v, counterMapNoPrefix[ontapCounterName]) - counters[mc.Name] = mc - } - } - } - - // If the template has any PluginMetrics, add them - for _, metric := range model.PluginMetrics { - co := Counter{ - Object: model.Object, - Name: model.Object + "_" + metric.Name, - APIs: []MetricDef{ - { - API: "StatPerf", - Endpoint: model.Query, - Template: path, - ONTAPCounter: metric.Source, - }, - }, - Panels: metricsPanelMap[model.Object+"_"+metric.Name].Panels, - } - counters[co.Name] = co - } - return counters -} - -func specialHandlingPerfCounters(counters map[string]Counter, model template2.Model) map[string]Counter { - // handling for templates with common object names - modifiedCounters := make(map[string]Counter) - for originalKey, value := range counters { - modifiedKey := model.Name + "#" + originalKey - modifiedCounters[modifiedKey] = value - } - return modifiedCounters -} - -func addAggregatedCounter(c *Counter, metric plugin.DerivedMetric, withPrefix string, noPrefix string) { - if !strings.HasSuffix(c.Description, ".") { - c.Description += "." - } - - if metric.IsMax { - c.Name = metric.Name + "_" + noPrefix - c.Description = fmt.Sprintf("%s %s is the maximum of [%s](#%s) for label `%s`.", - c.Description, c.Name, withPrefix, withPrefix, metric.Source) - } else { - c.Name = metric.Name + "_" + c.Name - if metric.HasCustomName { - c.Name = metric.Source + "_" + noPrefix - } - c.Description = fmt.Sprintf("%s %s is [%s](#%s) aggregated by `%s`.", - c.Description, c.Name, withPrefix, withPrefix, metric.Name) - } - c.Panels = metricsPanelMap[c.Name].Panels -} - -func processExternalCounters(dir string, counters map[string]Counter) map[string]Counter { - dat, err := os.ReadFile(filepath.Join(dir, "cmd", "tools", "generate", "counter.yaml")) - if err != nil { - fmt.Printf("error while reading file %v", err) - return nil - } - var c Counters - - err = yaml.Unmarshal(dat, &c) - if err != nil { - fmt.Printf("error while parsing file %v", err) - return nil - } - for _, v := range c.C { - if v1, ok := counters[v.Name]; !ok { - v.Panels = metricsPanelMap[v.Name].Panels - counters[v.Name] = v - } else { - if v.Description != "" { - v1.Description = v.Description - } - for _, m := range v.APIs { - indices := findAPI(v1.APIs, m) - if len(indices) == 0 { - v1.APIs = append(v1.APIs, m) - } else { - for _, index := range indices { - r := &v1.APIs[index] - if m.ONTAPCounter != "" { - r.ONTAPCounter = m.ONTAPCounter - } - if m.Template != "" { - r.Template = m.Template - } - if m.Endpoint != "" { - r.Endpoint = m.Endpoint - } - if m.Type != "" { - r.Type = m.Type - } - if m.Unit != "" { - r.Unit = m.Unit - } - if m.BaseCounter != "" { - r.BaseCounter = m.BaseCounter - } - } - } - } - counters[v.Name] = v1 - } - } - return counters -} - -func generateCounters(dir string, counters map[string]Counter, collectorName string) map[string]Counter { - dat, err := os.ReadFile(filepath.Join(dir, "cmd", "tools", "generate", collectorName+"_counter.yaml")) - if err != nil { - fmt.Printf("error while reading file %v", err) - return nil - } - var c Counters - - err = yaml.Unmarshal(dat, &c) - if err != nil { - fmt.Printf("error while parsing file %v", err) - return nil - } - - for k, m := range metricsPanelMap { - if !strings.HasPrefix(k, collectorName) { - continue - } - if _, ok := counters[k]; !ok { - counters[k] = Counter{Name: k, Panels: m.Panels} - } - } - - for _, v := range c.C { - if v1, ok := counters[v.Name]; !ok { - v.Panels = metricsPanelMap[v.Name].Panels - counters[v.Name] = v - } else { - if v.Description != "" { - v1.Description = v.Description - } - if len(v.APIs) > 0 { - v1.APIs = v.APIs - } - counters[v.Name] = v1 - } - } - return counters -} - -func findAPI(apis []MetricDef, other MetricDef) []int { - var indices []int - for i, a := range apis { - if a.API == other.API { - indices = append(indices, i) - } - } - return indices -} - -func fetchAndCategorizePrometheusMetrics(promURL string) (map[string]bool, map[string]bool, error) { - urlStr := promURL + "/api/v1/series?match[]={datacenter!=\"\"}" - - u, err := url.Parse(urlStr) - if err != nil { - return nil, nil, fmt.Errorf("failed to parse URL: %w", err) - } - - resp, err := http.Get(u.String()) - if err != nil { - return nil, nil, fmt.Errorf("failed to fetch metrics from Prometheus: %w", err) - } - //goland:noinspection GoUnhandledErrorResult - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return nil, nil, fmt.Errorf("unexpected status code from Prometheus: %d", resp.StatusCode) - } - - var result struct { - Status string `json:"status"` - Data []map[string]string `json:"data"` - } - if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { - return nil, nil, fmt.Errorf("failed to decode Prometheus response: %w", err) - } - if result.Status != "success" { - return nil, nil, fmt.Errorf("unexpected status from Prometheus: %s", result.Status) - } - - // Categorize metrics - restMetrics := make(map[string]bool) - zapiMetrics := make(map[string]bool) - for _, series := range result.Data { - metricName := series["__name__"] - switch series["datacenter"] { - case "REST": - restMetrics[metricName] = true - case "ZAPI": - zapiMetrics[metricName] = true - case "keyperf": - restMetrics[metricName] = true - } - } - - return restMetrics, zapiMetrics, nil -} -func validateMetrics(documentedRest, documentedZapi map[string]Counter, prometheusRest, prometheusZapi map[string]bool) error { - var documentedButMissingRestMetrics []string - var notDocumentedRestMetrics []string - var documentedButMissingZapiMetrics []string - var notDocumentedZapiMetrics []string - - // Helper function to check if a REST metric should be excluded - shouldExcludeRest := func(metric string, apis []MetricDef) bool { - for _, c := range excludeDocumentedRestMetrics { - if strings.Contains(metric, c) { - return true - } - } - - for _, api := range apis { - if api.API == "ZAPI" || api.API == "ZapiPerf" { - return true - } - } - - return false - } - - // Helper function to check if a ZAPI metric should be excluded - shouldExcludeZapi := func(metric string, apis []MetricDef) bool { - for _, prefix := range excludeDocumentedZapiMetrics { - if strings.Contains(metric, prefix) { - return true - } - } - - for _, api := range apis { - if api.API == "REST" || api.API == "RestPerf" { - return true - } - } - return false - } - - // Helper function to check if an extra REST metric should be excluded - shouldExcludeExtraRest := func(metric string, set *set.Set) bool { - - for _, c := range excludeNotDocumentedRestMetrics { - if strings.Contains(metric, c) { - return true - } - } - - var isRestObject bool - for o := range set.Iter() { - if strings.HasPrefix(metric, o) { - isRestObject = true - } - } - return !isRestObject - } - - // Helper function to check if an extra ZAPI metric should be excluded - shouldExcludeExtraZapi := func(metric string) bool { - - for _, c := range excludeNotDocumentedZapiMetrics { - if strings.Contains(metric, c) { - return true - } - } - - return false - } - - restObjects := set.New() - - for metric, counter := range documentedRest { - if counter.Object != "" { - restObjects.Add(counter.Object) - } - if !prometheusRest[metric] && !shouldExcludeRest(metric, counter.APIs) { - documentedButMissingRestMetrics = append(documentedButMissingRestMetrics, metric) - } - } - - for metric := range prometheusRest { - if _, ok := documentedRest[metric]; !ok && !shouldExcludeExtraRest(metric, restObjects) { - notDocumentedRestMetrics = append(notDocumentedRestMetrics, metric) - } - } - - for metric, counter := range documentedZapi { - if !prometheusZapi[metric] && !shouldExcludeZapi(metric, counter.APIs) { - documentedButMissingZapiMetrics = append(documentedButMissingZapiMetrics, metric) - } - } - - for metric := range prometheusZapi { - if _, ok := documentedZapi[metric]; !ok && !shouldExcludeExtraZapi(metric) { - notDocumentedZapiMetrics = append(notDocumentedZapiMetrics, metric) - } - } - - // Sort the slices - sort.Strings(documentedButMissingRestMetrics) - sort.Strings(notDocumentedRestMetrics) - sort.Strings(documentedButMissingZapiMetrics) - sort.Strings(notDocumentedZapiMetrics) - - if len(documentedButMissingRestMetrics) > 0 || len(notDocumentedRestMetrics) > 0 || len(documentedButMissingZapiMetrics) > 0 || len(notDocumentedZapiMetrics) > 0 { - errorMessage := "Validation failed:\n" - if len(documentedButMissingRestMetrics) > 0 { - errorMessage += fmt.Sprintf("Missing Rest metrics in Prometheus but documented: %v\n", documentedButMissingRestMetrics) - } - if len(notDocumentedRestMetrics) > 0 { - errorMessage += fmt.Sprintf("Extra Rest metrics in Prometheus but not documented: %v\n", notDocumentedRestMetrics) - } - if len(documentedButMissingZapiMetrics) > 0 { - errorMessage += fmt.Sprintf("Missing Zapi metrics in Prometheus but documented: %v\n", documentedButMissingZapiMetrics) - } - if len(notDocumentedZapiMetrics) > 0 { - errorMessage += fmt.Sprintf("Extra Zapi metrics in Prometheus but not documented: %v\n", notDocumentedZapiMetrics) - } - return errors.New(errorMessage) - } - - return nil -} - -func categorizeCounters(counters map[string]Counter) (map[string]Counter, map[string]Counter) { - restCounters := make(map[string]Counter) - zapiCounters := make(map[string]Counter) - - for _, counter := range counters { - for _, api := range counter.APIs { - switch api.API { - case "REST": - restCounters[counter.Name] = counter - case "RestPerf": - restCounters[counter.Name] = counter - case "ZAPI": - zapiCounters[counter.Name] = counter - case "ZapiPerf": - zapiCounters[counter.Name] = counter - case "KeyPerf": - restCounters[counter.Name] = counter - } - } - } - - return restCounters, zapiCounters -} - -func getAllExportedLabels(t *node.Node, counterContents []string) ([]string, []string, bool) { - metricLabels := make([]string, 0) - labels := make([]string, 0) - isInstanceLabels := false - eData := true - if exportData := t.GetChildS("export_data"); exportData != nil { - if exportData.GetContentS() == "false" { - eData = false - } - } - if exportOptions := t.GetChildS("export_options"); exportOptions != nil { - if iAllLabels := exportOptions.GetChildS("include_all_labels"); iAllLabels != nil { - if iAllLabels.GetContentS() == "true" { - for _, c := range counterContents { - if c == "" { - continue - } - if _, display, m, _ := template3.ParseMetric(c); m == "key" || m == "label" { - metricLabels = append(metricLabels, display) - } - } - return metricLabels, metricLabels, false - } - } - - if iKeys := exportOptions.GetChildS("instance_keys"); iKeys != nil { - metricLabels = append(metricLabels, iKeys.GetAllChildContentS()...) - } - if iLabels := exportOptions.GetChildS("instance_labels"); iLabels != nil { - labels = append(labels, iLabels.GetAllChildContentS()...) - isInstanceLabels = eData - } - } - return metricLabels, append(labels, metricLabels...), isInstanceLabels -} - -func visitDashboard(dirs []string, eachDash func(data []byte)) { +func visitDashboard(dirs []string, metricsPanelMap map[string]tools.PanelData, eachDash func(data []byte, metricsPanelMap map[string]tools.PanelData)) { for _, dir := range dirs { err := filepath.Walk(dir, func(path string, _ os.FileInfo, err error) error { if err != nil { @@ -2192,7 +106,7 @@ func visitDashboard(dirs []string, eachDash func(data []byte)) { if err != nil { log.Fatalf("failed to read dashboards path=%s err=%v", path, err) } - eachDash(data) + eachDash(data, metricsPanelMap) return nil }) if err != nil { @@ -2201,7 +115,7 @@ func visitDashboard(dirs []string, eachDash func(data []byte)) { } } -func visitExpressions(data []byte) { +func visitExpressions(data []byte, metricsPanelMap map[string]tools.PanelData) { // collect all expressions expressions := make([]grafana.ExprP, 0) dashboard := gjson.GetBytes(data, "title").String() @@ -2237,7 +151,7 @@ func visitExpressions(data []byte) { key := dashboard + expr.RowTitle + expr.Kind + expr.PanelTitle + link + expr.PanelID if !panelKeyMap[m+key] { panelKeyMap[m+key] = true - metricsPanelMap[m] = PanelData{Panels: append(metricsPanelMap[m].Panels, PanelDef{Dashboard: dashboard, Row: expr.RowTitle, Type: expr.Kind, Panel: expr.PanelTitle, PanelLink: link + expr.PanelID})} + metricsPanelMap[m] = tools.PanelData{Panels: append(metricsPanelMap[m].Panels, tools.PanelDef{Dashboard: dashboard, Row: expr.RowTitle, Type: expr.Kind, Panel: expr.PanelTitle, PanelLink: link + expr.PanelID})} } } } diff --git a/cmd/tools/generate/counter.yaml b/cmd/tools/generate/counter.yaml index 3e5a24bea..41d9605fd 100644 --- a/cmd/tools/generate/counter.yaml +++ b/cmd/tools/generate/counter.yaml @@ -2332,8 +2332,155 @@ counters: - Name: lun_block_size Description: Represents the block size being used + - Name: audit_log + Description: Captures the operations such as create, update, and delete attempts on volumes via REST or ONTAP CLI commands + APIs: + - API: REST + Endpoint: NA + ONTAPCounter: Harvest generated + Template: conf/rest/9.12.0/audit_log.yaml + + - Name: change_log + Description: Detect and track changes related to the creation, modification, and deletion of an object of Node, SVM and Volume + APIs: + - API: REST + Endpoint: NA + ONTAPCounter: Harvest generated + Template: conf/rest/9.12.0/node.yaml + - API: REST + Endpoint: NA + ONTAPCounter: Harvest generated + Template: conf/rest/9.10.0/svm.yaml + - API: REST + Endpoint: NA + ONTAPCounter: Harvest generated + Template: conf/rest/9.14.0/volume.yaml + + - Name: poller_cpu_percent + Description: Tracks the percentage of cpu usage of concurrent collectors running. + APIs: + - API: REST + Endpoint: NA + ONTAPCounter: Harvest generated + Template: NA + + - Name: fcp_util_percent + Description: Represent the FCP utilization percentage + APIs: + - API: RestPerf + Endpoint: NA + ONTAPCounter: Harvest generated + Template: conf/restperf/9.12.0/fcp.yaml + + - Name: nfs_diag_storePool_LockAlloc + Description: Represent the current number of lock objects allocated + APIs: + - API: RestPerf + Endpoint: NA + ONTAPCounter: Harvest generated + Template: conf/restperf/9.12.0/nfsv4_pool.yaml + + - Name: nfs_diag_storePool_LockMax + Description: Represent the maximum number of lock objects + APIs: + - API: RestPerf + Endpoint: NA + ONTAPCounter: Harvest generated + Template: conf/restperf/9.12.0/nfsv4_pool.yaml + + - Name: security_account_activediruser + Description: Represent the Active directory user in security account + APIs: + - API: Rest + Endpoint: NA + ONTAPCounter: Harvest generated + Template: conf/rest/9.12.0/security_account.yaml + - Name: security_account_certificateuser + Description: Represent the Certificate user in security account + APIs: + - API: Rest + Endpoint: NA + ONTAPCounter: Harvest generated + Template: conf/rest/9.12.0/security_account.yaml + - Name: security_account_ldapuser + Description: Represent the LDAP user in security account + APIs: + - API: Rest + Endpoint: NA + ONTAPCounter: Harvest generated + Template: conf/rest/9.12.0/security_account.yaml + - Name: security_account_localuser + Description: Represent the Local user in security account + APIs: + - API: Rest + Endpoint: NA + ONTAPCounter: Harvest generated + Template: conf/rest/9.12.0/security_account.yaml + - Name: security_account_samluser + Description: Represent the SAML user in security account + APIs: + - API: Rest + Endpoint: NA + ONTAPCounter: Harvest generated + Template: conf/rest/9.12.0/security_account.yaml + + - Name: security_audit_destination_status + Description: Represent the security audit protocol in security audit destinations + APIs: + - API: Rest + Endpoint: NA + ONTAPCounter: Harvest generated + Template: conf/rest/9.12.0/security_audit_dest.yaml + + - Name: svm_ldap_encrypted + Description: This metric indicates a LDAP session security has been sealed + APIs: + - API: REST + Endpoint: NA + ONTAPCounter: Harvest generated + Template: conf/rest/9.10.0/svm.yaml + - API: ZAPI + Endpoint: NA + ONTAPCounter: Harvest generated + Template: conf/zapi/cdot/9.8.0/svm.yaml + - Name: svm_ldap_signed + Description: This metric indicates a LDAP session security has been signed + APIs: + - API: REST + Endpoint: NA + ONTAPCounter: Harvest generated + Template: conf/rest/9.10.0/svm.yaml + - API: ZAPI + Endpoint: NA + ONTAPCounter: Harvest generated + Template: conf/zapi/cdot/9.8.0/svm.yaml + + - Name: volume_arw_status + Description: Represent the cluster level ARW status + APIs: + - API: Rest + Endpoint: NA + ONTAPCounter: Harvest generated + Template: conf/rest/9.14.0/volume.yaml + + - Name: ethernet_switch_port_new_status + Description: Represent the status of the ethernet switch port + APIs: + - API: Rest + Endpoint: NA + ONTAPCounter: Harvest generated + Template: conf/rest/9.8.0/ethernet_switch_port.yaml + + - Name: vscan_server_disconnected + Description: Represent the disconnected vscan servers to the vscan pool + APIs: + - API: Rest + Endpoint: NA + ONTAPCounter: Harvest generated + Template: conf/rest/9.12.0/vscan.yaml + - # ASAr2 cluster metrics + # ASAr2 cluster metricsF - Name: cluster_space_available Description: Available space across the cluster. APIs: diff --git a/cmd/tools/generate/generate.go b/cmd/tools/generate/generate.go index be1646dcc..842634f6b 100644 --- a/cmd/tools/generate/generate.go +++ b/cmd/tools/generate/generate.go @@ -3,10 +3,8 @@ package generate import ( "errors" "fmt" + "github.com/netapp/harvest/v2/cmd/tools" "github.com/netapp/harvest/v2/cmd/tools/grafana" - "github.com/netapp/harvest/v2/cmd/tools/rest" - "github.com/netapp/harvest/v2/pkg/api/ontapi/zapi" - "github.com/netapp/harvest/v2/pkg/auth" "github.com/netapp/harvest/v2/pkg/color" "github.com/netapp/harvest/v2/pkg/conf" "github.com/netapp/harvest/v2/third_party/tidwall/gjson" @@ -14,7 +12,6 @@ import ( "github.com/spf13/cobra" "io" "log" - "log/slog" "os" "path/filepath" "regexp" @@ -22,7 +19,6 @@ import ( "strconv" "strings" "text/template" - "time" ) type PollerInfo struct { @@ -59,29 +55,8 @@ type PromTemplate struct { PromPort int } -type options struct { - Poller string - loglevel int - image string - filesdPath string - showPorts bool - outputPath string - certDir string - promPort int - grafanaPort int - mounts []string - configPath string - confPath string - promURL string -} - var metricRe = regexp.MustCompile(`(\w+)\{`) -var opts = &options{ - loglevel: 2, - image: "harvest:latest", -} - var Cmd = &cobra.Command{ Use: "generate", Short: "Generate Harvest related files", @@ -138,8 +113,9 @@ func doDockerCompose(cmd *cobra.Command, _ []string) { func doGenerateMetrics(cmd *cobra.Command, _ []string) { addRootOptions(cmd) // reset metricsPanelMap map - metricsPanelMap = make(map[string]PanelData) + metricsPanelMap := make(map[string]tools.PanelData) panelKeyMap = make(map[string]bool) + visitDashboard( []string{ "grafana/dashboards/asar2", @@ -147,19 +123,19 @@ func doGenerateMetrics(cmd *cobra.Command, _ []string) { "grafana/dashboards/cmode-details", "grafana/dashboards/cisco", "grafana/dashboards/storagegrid", - }, - func(data []byte) { - visitExpressions(data) + }, metricsPanelMap, + func(data []byte, metricsPanelMap map[string]tools.PanelData) { + visitExpressions(data, metricsPanelMap) }) - counters, cluster := BuildMetrics("", "", opts.Poller) - generateOntapCounterTemplate(counters, cluster.Version) - sgCounters, ciscoCounters := generateCounterTemplate() + counters, cluster := tools.BuildMetrics("", "", opts.Poller, opts, metricsPanelMap) + tools.GenerateOntapCounterTemplate(counters, cluster.Version) + sgCounters, ciscoCounters := generateCounterTemplate(metricsPanelMap) generateMetadataFiles(counters, sgCounters, ciscoCounters) } func doDescription(cmd *cobra.Command, _ []string) { addRootOptions(cmd) - counters, _ := BuildMetrics("", "", opts.Poller) + counters, _ := tools.BuildMetrics("", "", opts.Poller, opts, make(map[string]tools.PanelData)) grafana.VisitDashboards( []string{"grafana/dashboards/cmode"}, func(path string, data []byte) { @@ -168,8 +144,8 @@ func doDescription(cmd *cobra.Command, _ []string) { } func addRootOptions(cmd *cobra.Command) { - opts.configPath = conf.ConfigPath(cmd.Root().PersistentFlags().Lookup("config").Value.String()) - opts.confPath = cmd.Root().PersistentFlags().Lookup("confpath").Value.String() + opts.ConfigPath = conf.ConfigPath(cmd.Root().PersistentFlags().Lookup("config").Value.String()) + opts.ConfPath = cmd.Root().PersistentFlags().Lookup("confpath").Value.String() } const ( @@ -193,15 +169,15 @@ func generateDocker(kind int) { pollerTemplate = PollerTemplate{} promTemplate := PromTemplate{ - opts.grafanaPort, - opts.promPort, + opts.GrafanaPort, + opts.PromPort, } - _, err := conf.LoadHarvestConfig(opts.configPath) + _, err := conf.LoadHarvestConfig(opts.ConfigPath) if err != nil { - logErrAndExit(err) + tools.LogErrAndExit(err) } - configFilePath = asComposePath(opts.configPath) - certDirPath = asComposePath(opts.certDir) + configFilePath = asComposePath(opts.ConfigPath) + certDirPath = asComposePath(opts.CertDir) filesd := make([]string, 0, len(conf.Config.PollersOrdered)) for _, pollerName := range conf.Config.PollersOrdered { @@ -215,10 +191,10 @@ func generateDocker(kind int) { PollerName: pollerName, ConfigFile: configFilePath, Port: port, - LogLevel: opts.loglevel, - Image: opts.image, + LogLevel: opts.Loglevel, + Image: opts.Image, ContainerName: normalizeContainerNames("poller_" + pollerName), - ShowPorts: opts.showPorts, + ShowPorts: opts.ShowPorts, IsFull: kind == full, CertDir: certDirPath, Mounts: makeMounts(pollerName), @@ -229,14 +205,14 @@ func generateDocker(kind int) { t, err := template.New("docker-compose.tmpl").ParseFiles("container/onePollerPerContainer/docker-compose.tmpl") if err != nil { - logErrAndExit(err) + tools.LogErrAndExit(err) } color.DetectConsole("") - out, err = os.Create(opts.outputPath) + out, err = os.Create(opts.OutputPath) if err != nil { - logErrAndExit(err) + tools.LogErrAndExit(err) } if kind == harvest { @@ -248,17 +224,17 @@ func generateDocker(kind int) { if s := strings.Split(httpsd, ":"); len(s) == 2 { adminPort, err = strconv.Atoi(s[1]) if err != nil { - logErrAndExit(errors.New("invalid httpsd listen configuration. Valid configuration are <>:PORT or :PORT")) + tools.LogErrAndExit(errors.New("invalid httpsd listen configuration. Valid configuration are <>:PORT or :PORT")) } } else { - logErrAndExit(errors.New("invalid httpsd listen configuration. Valid configuration are <>:PORT or :PORT")) + tools.LogErrAndExit(errors.New("invalid httpsd listen configuration. Valid configuration are <>:PORT or :PORT")) } pollerTemplate.Admin = AdminInfo{ ServiceName: "admin", ConfigFile: configFilePath, Port: adminPort, - Image: opts.image, + Image: opts.Image, ContainerName: "admin", Enabled: true, CertDir: certDirPath, @@ -267,33 +243,33 @@ func generateDocker(kind int) { } else { pt, err := template.New("prom-stack.tmpl").ParseFiles("prom-stack.tmpl") if err != nil { - logErrAndExit(err) + tools.LogErrAndExit(err) } promStackOut, err := os.Create("prom-stack.yml") if err != nil { - logErrAndExit(err) + tools.LogErrAndExit(err) } err = pt.Execute(promStackOut, promTemplate) if err != nil { - logErrAndExit(err) + tools.LogErrAndExit(err) } } err = t.Execute(out, pollerTemplate) if err != nil { - logErrAndExit(err) + tools.LogErrAndExit(err) } - f, err := os.Create(opts.filesdPath) + f, err := os.Create(opts.FilesdPath) if err != nil { - logErrAndExit(err) + tools.LogErrAndExit(err) } defer silentClose(f) for _, line := range filesd { _, _ = fmt.Fprintln(f, line) } - _, _ = fmt.Fprintf(os.Stderr, "Wrote file_sd targets to %s\n", opts.filesdPath) + _, _ = fmt.Fprintf(os.Stderr, "Wrote file_sd targets to %s\n", opts.FilesdPath) if os.Getenv("HARVEST_DOCKER") != "" { srcFolder := "/opt/harvest" @@ -301,32 +277,32 @@ func generateDocker(kind int) { err = copyFiles(srcFolder, destFolder) if err != nil { - logErrAndExit(err) + tools.LogErrAndExit(err) } } if kind == harvest { _, _ = fmt.Fprint(os.Stderr, "Start containers with:\n"+ - color.Colorize("docker compose -f "+opts.outputPath+" up -d --remove-orphans\n", color.Green)) + color.Colorize("docker compose -f "+opts.OutputPath+" up -d --remove-orphans\n", color.Green)) } if kind == full { _, _ = fmt.Fprint(os.Stderr, "Start containers with:\n"+ - color.Colorize("docker compose -f prom-stack.yml -f "+opts.outputPath+" up -d --remove-orphans\n", color.Green)) + color.Colorize("docker compose -f prom-stack.yml -f "+opts.OutputPath+" up -d --remove-orphans\n", color.Green)) } } // setup mount(s) for the confpath and any CLI-passed mounts func makeMounts(pollerName string) []string { - var mounts = opts.mounts + var mounts = opts.Mounts p, err := conf.PollerNamed(pollerName) if err != nil { - logErrAndExit(err) + tools.LogErrAndExit(err) } - confPath := opts.confPath + confPath := opts.ConfPath if confPath == "conf" { confPath = p.ConfPath } @@ -489,27 +465,22 @@ func asComposePath(path string) string { return "./" + path } -func logErrAndExit(err error) { - fmt.Printf("%v\n", err) - os.Exit(1) -} - func silentClose(body io.ReadCloser) { _ = body.Close() } func generateSystemd() { var adminService string - _, err := conf.LoadHarvestConfig(opts.configPath) + _, err := conf.LoadHarvestConfig(opts.ConfigPath) if err != nil { - logErrAndExit(err) + tools.LogErrAndExit(err) } if conf.Config.Pollers == nil { return } t, err := template.New("target.tmpl").ParseFiles("service/contrib/target.tmpl") if err != nil { - logErrAndExit(err) + tools.LogErrAndExit(err) } color.DetectConsole("") println("Save the following to " + color.Colorize("/etc/systemd/system/harvest.target", color.Green) + @@ -519,7 +490,7 @@ func generateSystemd() { println("and " + color.Colorize("cp "+harvestAdminService+" /etc/systemd/system/", color.Green)) } println("and then run " + color.Colorize("systemctl daemon-reload", color.Green)) - writeAdminSystemd(opts.configPath) + writeAdminSystemd(opts.ConfigPath) pollers := make([]string, 0) unixPollers := make([]string, 0) @@ -551,7 +522,7 @@ func generateSystemd() { PollersOrdered: pollers, }) if err != nil { - logErrAndExit(err) + tools.LogErrAndExit(err) } } @@ -561,11 +532,11 @@ func writeAdminSystemd(configFp string) { } t, err := template.New("httpsd.tmpl").ParseFiles("service/contrib/httpsd.tmpl") if err != nil { - logErrAndExit(err) + tools.LogErrAndExit(err) } f, err := os.Create(harvestAdminService) if err != nil { - logErrAndExit(err) + tools.LogErrAndExit(err) } defer silentClose(f) configAbsPath, err := filepath.Abs(configFp) @@ -576,68 +547,7 @@ func writeAdminSystemd(configFp string) { println(color.Colorize("✓", color.Green) + " HTTP SD file: " + harvestAdminService + " created") } -func BuildMetrics(dir, configPath, pollerName string) (map[string]Counter, conf.Remote) { - var ( - poller *conf.Poller - err error - restClient *rest.Client - zapiClient *zapi.Client - harvestYmlPath string - ) - - if opts.configPath != "" { - harvestYmlPath = filepath.Join(dir, opts.configPath) - } else { - harvestYmlPath = filepath.Join(dir, configPath) - } - _, err = conf.LoadHarvestConfig(harvestYmlPath) - if err != nil { - logErrAndExit(err) - } - - if poller, _, err = rest.GetPollerAndAddr(pollerName); err != nil { - logErrAndExit(err) - } - - timeout, _ := time.ParseDuration(rest.DefaultTimeout) - credentials := auth.NewCredentials(poller, slog.Default()) - if restClient, err = rest.New(poller, timeout, credentials); err != nil { - fmt.Printf("error creating new client %+v\n", err) - os.Exit(1) - } - if err = restClient.Init(2, conf.Remote{}); err != nil { - fmt.Printf("error init rest client %+v\n", err) - os.Exit(1) - } - - if zapiClient, err = zapi.New(poller, credentials); err != nil { - fmt.Printf("error creating new client %+v\n", err) - os.Exit(1) - } - - swaggerBytes = readSwaggerJSON() - restCounters := processRestCounters(dir, restClient) - zapiCounters := processZapiCounters(dir, zapiClient) - counters := mergeCounters(restCounters, zapiCounters) - counters = processExternalCounters(dir, counters) - - if opts.promURL != "" { - prometheusRest, prometheusZapi, err := fetchAndCategorizePrometheusMetrics(opts.promURL) - if err != nil { - logErrAndExit(err) - } - - documentedRest, documentedZapi := categorizeCounters(counters) - - if err := validateMetrics(documentedRest, documentedZapi, prometheusRest, prometheusZapi); err != nil { - logErrAndExit(err) - } - } - - return counters, restClient.Remote() -} - -func generateDescription(dPath string, data []byte, counters map[string]Counter) { +func generateDescription(dPath string, data []byte, counters map[string]tools.Counter) { var err error dashPath := grafana.ShortPath(dPath) panelDescriptionMap := make(map[string]string) @@ -720,20 +630,20 @@ func init() { flag.StringVarP(&opts.Poller, "poller", "p", "dc1", "name of poller, e.g. 10.193.48.154") _ = descCmd.MarkPersistentFlagRequired("poller") - dFlags.IntVarP(&opts.loglevel, "loglevel", "l", 2, + dFlags.IntVarP(&opts.Loglevel, "loglevel", "l", 2, "logging level (0=trace, 1=debug, 2=info, 3=warning, 4=error, 5=critical)", ) - dFlags.StringVar(&opts.image, "image", "ghcr.io/netapp/harvest:latest", "Harvest image. Use rahulguptajss/harvest:latest to pull from Docker Hub") - dFlags.StringVar(&opts.certDir, "certdir", "./cert", "Harvest certificate dir path") - dFlags.StringVarP(&opts.outputPath, "output", "o", "", "Output file path. ") - dFlags.BoolVarP(&opts.showPorts, "port", "p", true, "Expose poller ports to host machine") + dFlags.StringVar(&opts.Image, "image", "ghcr.io/netapp/harvest:latest", "Harvest image. Use rahulguptajss/harvest:latest to pull from Docker Hub") + dFlags.StringVar(&opts.CertDir, "certdir", "./cert", "Harvest certificate dir path") + dFlags.StringVarP(&opts.OutputPath, "output", "o", "", "Output file path. ") + dFlags.BoolVarP(&opts.ShowPorts, "port", "p", true, "Expose poller ports to host machine") _ = dockerCmd.MarkPersistentFlagRequired("output") - dFlags.StringSliceVar(&opts.mounts, "volume", []string{}, "Additional volume mounts to include in compose file") + dFlags.StringSliceVar(&opts.Mounts, "volume", []string{}, "Additional volume mounts to include in compose file") - fFlags.StringVar(&opts.filesdPath, "filesdpath", "container/prometheus/harvest_targets.yml", + fFlags.StringVar(&opts.FilesdPath, "filesdpath", "container/prometheus/harvest_targets.yml", "Prometheus file_sd target path. Written when the --output is set") - fFlags.IntVar(&opts.promPort, "promPort", 9090, "Prometheus Port") - fFlags.IntVar(&opts.grafanaPort, "grafanaPort", 3000, "Grafana Port") + fFlags.IntVar(&opts.PromPort, "promPort", 9090, "Prometheus Port") + fFlags.IntVar(&opts.GrafanaPort, "grafanaPort", 3000, "Grafana Port") - metricCmd.PersistentFlags().StringVar(&opts.promURL, "prom-url", "", "Prometheus URL for CI validation") + metricCmd.PersistentFlags().StringVar(&opts.PromURL, "prom-url", "", "Prometheus URL for CI validation") } diff --git a/cmd/tools/generate/storagegrid_counter.yaml b/cmd/tools/generate/storagegrid_counter.yaml index e879d042d..a0a7a00e8 100644 --- a/cmd/tools/generate/storagegrid_counter.yaml +++ b/cmd/tools/generate/storagegrid_counter.yaml @@ -230,3 +230,68 @@ counters: Endpoint: grid/accounts-cache SGCounter: policy.quotaObjectBytes Template: conf/storagegrid/11.6.0/tenant.yaml + + - Name: bucket_bytes + Description: Represent the bytes read/write in the bucket + APIs: + - API: REST + Endpoint: NA + ONTAPCounter: Harvest generated + Template: conf/storagegrid/11.6.0/tenant.yaml + - Name: bucket_objects + Description: Represent the number of object count in the bucket + APIs: + - API: REST + Endpoint: NA + ONTAPCounter: Harvest generated + Template: conf/storagegrid/11.6.0/tenant.yaml + - Name: bucket_labels + Description: Represent the detail of bucket + APIs: + - API: REST + Endpoint: NA + ONTAPCounter: Harvest generated + Template: conf/storagegrid/11.6.0/tenant.yaml + - Name: bucket_quota_bytes + Description: Represent the bytes read/write in quota of the bucket + APIs: + - API: REST + Endpoint: NA + ONTAPCounter: Harvest generated + Template: conf/storagegrid/11.6.0/tenant.yaml + + - Name: tenant_labels + Description: Represent the detail of tenant + APIs: + - API: REST + Endpoint: NA + ONTAPCounter: Harvest generated + Template: conf/storagegrid/11.6.0/tenant.yaml + - Name: tenant_logical_quota + Description: Represent the logical quota size in tenant + APIs: + - API: REST + Endpoint: NA + ONTAPCounter: Harvest generated + Template: conf/storagegrid/11.6.0/tenant.yaml + - Name: tenant_logical_used + Description: Represent the logical data used in tenant + APIs: + - API: REST + Endpoint: NA + ONTAPCounter: Harvest generated + Template: conf/storagegrid/11.6.0/tenant.yaml + - Name: tenant_objects + Description: Represent the number of object count in the tenant + APIs: + - API: REST + Endpoint: NA + ONTAPCounter: Harvest generated + Template: conf/storagegrid/11.6.0/tenant.yaml + - Name: tenant_used_percent + Description: Represent the data used percent in tenant + APIs: + - API: REST + Endpoint: NA + ONTAPCounter: Harvest generated + Template: conf/storagegrid/11.6.0/tenant.yaml diff --git a/cmd/tools/grafana/metrics.go b/cmd/tools/grafana/metrics.go index 10405173d..d5735c0b1 100644 --- a/cmd/tools/grafana/metrics.go +++ b/cmd/tools/grafana/metrics.go @@ -2,9 +2,12 @@ package grafana import ( "fmt" + "github.com/netapp/harvest/v2/cmd/tools" + tw "github.com/netapp/harvest/v2/third_party/olekukonko/tablewriter" "github.com/netapp/harvest/v2/third_party/tidwall/gjson" "github.com/spf13/cobra" "log" + "maps" "os" "path/filepath" "regexp" @@ -35,8 +38,15 @@ type Expression struct { func doMetrics(_ *cobra.Command, _ []string) { adjustOptions() validateImport() + // reset metricsPanelMap map + metricsPanelMap := make(map[string]tools.PanelData) + counters, _ := tools.BuildMetrics("", "", "", nil, metricsPanelMap) + sgCounters := tools.GenerateCounters("", make(map[string]tools.Counter), "storagegrid", metricsPanelMap) + ciscoCounters := tools.GenerateCounters("", make(map[string]tools.Counter), "cisco", metricsPanelMap) + maps.Copy(counters, sgCounters) + maps.Copy(counters, ciscoCounters) VisitDashboards([]string{opts.dir}, func(path string, data []byte) { - visitExpressionsAndQueries(path, data) + visitExpressionsAndQueries(path, data, counters) }) } @@ -50,7 +60,8 @@ type ExprP struct { vars []string } -func visitExpressionsAndQueries(path string, data []byte) { +func visitExpressionsAndQueries(path string, data []byte, restCounters map[string]tools.Counter) { + var templateNames string // collect all expressions expressions := make([]ExprP, 0) gjson.GetBytes(data, "panels").ForEach(func(key, value gjson.Result) bool { @@ -92,11 +103,34 @@ func visitExpressionsAndQueries(path string, data []byte) { } } - fmt.Printf("%s\n", ShortPath(path)) + fmt.Printf("Dashboard Name: %s\n", ShortPath(path)) metrics := setToList(metricsSeen) + table := tw.NewWriter(os.Stdout) + table.SetBorder(false) + table.SetAutoFormatHeaders(false) + table.SetAutoWrapText(false) + table.SetHeader([]string{"Metric", "Template Path"}) + for _, metric := range metrics { - fmt.Printf("- %s\n", metric) + var pathSlice []string + apis := restCounters[metric].APIs + if len(apis) == 0 { + if strings.Contains(metric, "hist") { + templateNames = "not documented" + } else { + fmt.Printf("template not found for metric: %s\n", metric) + } + } else { + for _, api := range apis { + pathSlice = append(pathSlice, api.Template) + } + slices.Sort(pathSlice) + templateNames = strings.Join(pathSlice, " ") + } + + table.Append([]string{metric, templateNames}) } + table.Render() fmt.Println() } diff --git a/cmd/tools/util.go b/cmd/tools/util.go new file mode 100644 index 000000000..da8fc221f --- /dev/null +++ b/cmd/tools/util.go @@ -0,0 +1,2258 @@ +package tools + +import ( + "encoding/json" + "errors" + "fmt" + "github.com/goccy/go-yaml" + "github.com/netapp/harvest/v2/cmd/collectors/keyperf" + "github.com/netapp/harvest/v2/cmd/collectors/statperf" + "github.com/netapp/harvest/v2/cmd/poller/plugin" + "github.com/netapp/harvest/v2/cmd/tools/rest" + "github.com/netapp/harvest/v2/cmd/tools/rest/clirequestbuilder" + template2 "github.com/netapp/harvest/v2/cmd/tools/template" + "github.com/netapp/harvest/v2/pkg/api/ontapi/zapi" + "github.com/netapp/harvest/v2/pkg/auth" + "github.com/netapp/harvest/v2/pkg/conf" + "github.com/netapp/harvest/v2/pkg/logging" + "github.com/netapp/harvest/v2/pkg/requests" + "github.com/netapp/harvest/v2/pkg/set" + template3 "github.com/netapp/harvest/v2/pkg/template" + "github.com/netapp/harvest/v2/pkg/tree" + "github.com/netapp/harvest/v2/pkg/tree/node" + tw "github.com/netapp/harvest/v2/third_party/olekukonko/tablewriter" + "github.com/netapp/harvest/v2/third_party/tidwall/gjson" + "io" + "log" + "log/slog" + "maps" + "net/http" + "net/http/httputil" + "net/url" + "os" + "os/exec" + "path/filepath" + "regexp" + "slices" + "sort" + "strings" + "text/template" + "time" +) + +const ( + keyPerfAPI = "KeyPerf" +) + +type Options struct { + Poller string + Loglevel int + Image string + FilesdPath string + ShowPorts bool + OutputPath string + CertDir string + PromPort int + GrafanaPort int + Mounts []string + ConfigPath string + ConfPath string + PromURL string +} + +type CounterMetaData struct { + Date string + OntapVersion string + SGVersion string + CiscoVersion string +} + +type CounterTemplate struct { + Counters []Counter + CounterMetaData CounterMetaData +} + +type Counters struct { + C []Counter `yaml:"counters"` +} + +type Counter struct { + Object string `yaml:"-"` + Name string `yaml:"Name"` + Description string `yaml:"Description"` + APIs []MetricDef `yaml:"APIs"` + Panels []PanelDef `yaml:"Panels"` + Labels []string `yaml:"Labels"` +} + +type MetricDef struct { + API string `yaml:"API"` + Endpoint string `yaml:"Endpoint"` + ONTAPCounter string `yaml:"ONTAPCounter"` + CiscoCounter string `yaml:"CiscoCounter"` + SGCounter string `yaml:"SGCounter"` + Template string `yaml:"Template"` + Unit string `yaml:"Unit"` + Type string `yaml:"Type"` + BaseCounter string `yaml:"BaseCounter"` +} + +type PanelDef struct { + Dashboard string `yaml:"Dashboard"` + Row string `yaml:"Row"` + Type string `yaml:"Type"` + Panel string `yaml:"Panel"` + PanelLink string `yaml:"PanelLink"` +} + +type PanelData struct { + Panels []PanelDef +} + +// Regex to match NFS version and operation +var reRemove = regexp.MustCompile(`NFSv\d+\.\d+`) + +var ( + replacer = strings.NewReplacer("\n", "", ":", "") + objectSwaggerMap = map[string]string{ + "aggr": "xc_aggregate", + "environment_sensor": "sensors", + "fcp": "fc_port", + "flexcache": "volume", + "lif": "ip_interface", + "namespace": "nvme_namespace", + "net_port": "xc_broadcast_domain", + "ontaps3": "xc_s3_bucket", + "security_ssh": "cluster_ssh_server", + "svm_cifs": "cifs_service", + "svm_nfs": "nfs_service", + "volume": "xc_volume", + } + swaggerBytes []byte + excludePerfTemplates = map[string]struct{}{ + "volume_node.yaml": {}, // Similar metrics node_volume_* are generated via KeyPerf volume.yaml + "workload_detail.yaml": {}, + "workload_detail_volume.yaml": {}, + } + excludeRestPerfTemplates = map[string]struct{}{ + "volume.yaml": {}, // Volume performance metrics now collected via KeyPerf + } + excludeCounters = map[string]struct{}{ + "latency_histogram": {}, + "nfs4_latency_hist": {}, + "nfs41_latency_hist": {}, + "nfsv3_latency_hist": {}, + "read_latency_hist": {}, + "read_latency_histogram": {}, + "total.latency_histogram": {}, + "write_latency_hist": {}, + "write_latency_histogram": {}, + } + + // Special handling perf objects + specialPerfObjects = map[string]bool{ + "node_nfs": true, + "svm_nfs": true, + } + + excludeDocumentedRestMetrics = []string{ + "audit_log", + "aggr_hybrid_disk_count", + "availability_zone_", + "change_log", + "cifs_session_idle_duration", + "cluster_space_available", + "cluster_software", + "ems_events", + "ethernet_switch_port_", + "export_rule_labels", + "fcp_util_percent", + "fcvi_", + "flashpool_", + "health_", + "igroup_labels", + "iw_", + "mav_request_", + "mediator_labels", + "metrocluster_", + "ndmp_session", + "net_connection_labels", + "nfs_clients_idle_duration", + "nfs_diag_", + "node_cifs_", + "nvme_lif_", + "nvmf_", + "ontaps3_svm_", + "path_", + "poller_cpu_percent", + "qtree_", + "smb2_", + "snapshot_labels", + "snapshot_restore_size", + "snapshot_create_time", + "snapshot_volume_violation_count", + "snapshot_volume_violation_total_size", + "storage_unit_", + "svm_cifs_", + "svm_ontaps3_svm_", + "svm_vscan_", + "token_", + "volume_top_clients", + "volume_top_files", + "vscan_", + } + + excludeDocumentedZapiMetrics = []string{ + "ems_events", + "external_service_", + "fabricpool_", + "flexcache_", + "fpolicy_svm_failedop_notifications", + "netstat_", + "nvm_mirror_", + "quota_disk_used_pct_threshold", + "snapshot_volume_violation_count", + "snapshot_volume_violation_total_size", + } + + // Exclude extra metrics for REST + excludeNotDocumentedRestMetrics = []string{ + "ALERTS", + "cluster_space_", + "flexcache_", + "hist_", + "igroup_", + "storage_unit_", + "volume_aggr_labels", + "volume_arw_status", + } + + // Exclude extra metrics for ZAPI + excludeNotDocumentedZapiMetrics = []string{ + "ALERTS", + "hist_", + "security_", + "svm_ldap", + "volume_aggr_labels", + } + + // include StatPerf Templates + includeStatPerfTemplates = map[string]struct{}{ + "flexcache.yaml": {}, + "system_node.yaml": {}, + } + + // Excludes these Rest gaps from logs + excludeLogRestCounters = []string{ + "external_service_op_", + "fabricpool_average_latency", + "fabricpool_get_throughput_bytes", + "fabricpool_put_throughput_bytes", + "fabricpool_stats", + "fabricpool_throughput_ops", + "iw_", + "netstat_", + "nvmf_rdma_port_", + "nvmf_tcp_port_", + "ontaps3_svm_", + "smb2_", + } + + knownDescriptionGaps = map[string]struct{}{ + "availability_zone_space_available": {}, + "availability_zone_space_physical_used": {}, + "availability_zone_space_physical_used_percent": {}, + "availability_zone_space_size": {}, + "ontaps3_object_count": {}, + "security_certificate_expiry_time": {}, + "storage_unit_space_efficiency_ratio": {}, + "storage_unit_space_size": {}, + "storage_unit_space_used": {}, + "volume_capacity_tier_footprint": {}, + "volume_capacity_tier_footprint_percent": {}, + "volume_num_compress_attempts": {}, + "volume_num_compress_fail": {}, + "volume_performance_tier_footprint": {}, + "volume_performance_tier_footprint_percent": {}, + } + + knownMappingGaps = map[string]struct{}{ + "aggr_snapshot_inode_used_percent": {}, + "aggr_space_reserved": {}, + "flexcache_": {}, + "fpolicy_": {}, + "quota_disk_used_pct_threshold": {}, + "rw_ctx_": {}, + "security_audit_destination_port": {}, + "storage_unit_": {}, + "wafl_reads_from_pmem": {}, + "node_volume_nfs_": {}, + "nvm_mirror_": {}, + "volume_nfs_": {}, + "svm_vol_nfs": {}, + } + + knownMappingGapsSG = map[string]struct{}{ + "storagegrid_node_cpu_utilization_percentage": {}, + "storagegrid_private_load_balancer_storage_request_body_bytes_bucket": {}, + "storagegrid_private_load_balancer_storage_request_count": {}, + "storagegrid_private_load_balancer_storage_request_time": {}, + "storagegrid_private_load_balancer_storage_rx_bytes": {}, + "storagegrid_private_load_balancer_storage_tx_bytes": {}, + } +) + +func BuildMetrics(dir, configPath, pollerName string, opts *Options, metricsPanelMap map[string]PanelData) (map[string]Counter, conf.Remote) { + var ( + poller *conf.Poller + err error + restClient *rest.Client + zapiClient *zapi.Client + harvestYmlPath string + ) + + if opts == nil { + restCounters := ProcessRestCounters(dir, nil, metricsPanelMap) + zapiCounters := ProcessZapiCounters(dir, nil, metricsPanelMap) + counters := MergeCounters(restCounters, zapiCounters) + counters = ProcessExternalCounters(dir, counters, metricsPanelMap) + return counters, conf.Remote{} + } + + if opts.ConfigPath != "" { + harvestYmlPath = filepath.Join(dir, opts.ConfigPath) + } else { + harvestYmlPath = filepath.Join(dir, configPath) + } + _, err = conf.LoadHarvestConfig(harvestYmlPath) + if err != nil { + LogErrAndExit(err) + } + + if poller, _, err = rest.GetPollerAndAddr(pollerName); err != nil { + LogErrAndExit(err) + } + + timeout, _ := time.ParseDuration(rest.DefaultTimeout) + credentials := auth.NewCredentials(poller, slog.Default()) + if restClient, err = rest.New(poller, timeout, credentials); err != nil { + fmt.Printf("error creating new client %+v\n", err) + os.Exit(1) + } + if err = restClient.Init(2, conf.Remote{}); err != nil { + fmt.Printf("error init rest client %+v\n", err) + os.Exit(1) + } + + if zapiClient, err = zapi.New(poller, credentials); err != nil { + fmt.Printf("error creating new client %+v\n", err) + os.Exit(1) + } + + swaggerBytes = readSwaggerJSON(opts) + restCounters := ProcessRestCounters(dir, restClient, metricsPanelMap) + zapiCounters := ProcessZapiCounters(dir, zapiClient, metricsPanelMap) + counters := MergeCounters(restCounters, zapiCounters) + counters = ProcessExternalCounters(dir, counters, metricsPanelMap) + + if opts.PromURL != "" { + prometheusRest, prometheusZapi, err := FetchAndCategorizePrometheusMetrics(opts.PromURL) + if err != nil { + LogErrAndExit(err) + } + + documentedRest, documentedZapi := CategorizeCounters(counters) + + if err := ValidateMetrics(documentedRest, documentedZapi, prometheusRest, prometheusZapi); err != nil { + LogErrAndExit(err) + } + } + + return counters, restClient.Remote() +} + +func GenerateCounters(dir string, counters map[string]Counter, collectorName string, metricsPanelMap map[string]PanelData) map[string]Counter { + dat, err := os.ReadFile(filepath.Join(dir, "cmd", "tools", "generate", collectorName+"_counter.yaml")) + if err != nil { + fmt.Printf("error while reading file %v", err) + return nil + } + var c Counters + + err = yaml.Unmarshal(dat, &c) + if err != nil { + fmt.Printf("error while parsing file %v", err) + return nil + } + + for k, m := range metricsPanelMap { + if !strings.HasPrefix(k, collectorName) { + continue + } + if _, ok := counters[k]; !ok { + counters[k] = Counter{Name: k, Panels: m.Panels} + } + } + + for _, v := range c.C { + if v1, ok := counters[v.Name]; !ok { + v.Panels = metricsPanelMap[v.Name].Panels + counters[v.Name] = v + } else { + if v.Description != "" { + v1.Description = v.Description + } + if len(v.APIs) > 0 { + v1.APIs = v.APIs + } + counters[v.Name] = v1 + } + } + return counters +} + +// ProcessRestCounters parse rest and restperf templates +func ProcessRestCounters(dir string, client *rest.Client, metricsPanelMap map[string]PanelData) map[string]Counter { + + restPerfCounters := visitRestTemplates(filepath.Join(dir, "conf", "restperf"), client, func(path string, client *rest.Client) map[string]Counter { + if _, ok := excludePerfTemplates[filepath.Base(path)]; ok { + return nil + } + if _, ok := excludeRestPerfTemplates[filepath.Base(path)]; ok { + return nil + } + return processRestPerfCounters(path, client, metricsPanelMap) + }) + + restCounters := visitRestTemplates(filepath.Join(dir, "conf", "rest"), client, func(path string, client *rest.Client) map[string]Counter { // revive:disable-line:unused-parameter + return processRestConfigCounters(path, "REST", metricsPanelMap) + }) + + keyPerfCounters := visitRestTemplates(filepath.Join(dir, "conf", "keyperf"), client, func(path string, client *rest.Client) map[string]Counter { // revive:disable-line:unused-parameter + return processRestConfigCounters(path, keyPerfAPI, metricsPanelMap) + }) + + statPerfCounters := visitRestTemplates(filepath.Join(dir, "conf", "statperf"), client, func(path string, client *rest.Client) map[string]Counter { + if _, ok := includeStatPerfTemplates[filepath.Base(path)]; ok { + return processStatPerfCounters(path, client, metricsPanelMap) + } + return nil + }) + maps.Copy(restCounters, restPerfCounters) + + keyPerfKeys := slices.Sorted(maps.Keys(keyPerfCounters)) + for _, k := range keyPerfKeys { + if strings.Contains(k, "timestamp") || strings.Contains(k, "labels") { + continue + } + v := keyPerfCounters[k] + if v1, ok := restCounters[k]; !ok { + restCounters[k] = v + } else { + v1.APIs = append(v1.APIs, v.APIs...) + restCounters[k] = v1 + } + } + + statPerfKeys := slices.Sorted(maps.Keys(statPerfCounters)) + for _, k := range statPerfKeys { + if strings.Contains(k, "labels") { + continue + } + v := statPerfCounters[k] + if v1, ok := restCounters[k]; !ok { + restCounters[k] = v + } else { + v1.APIs = append(v1.APIs, v.APIs...) + restCounters[k] = v1 + } + } + return restCounters +} + +// ProcessZapiCounters parse zapi and zapiperf templates +func ProcessZapiCounters(dir string, client *zapi.Client, metricsPanelMap map[string]PanelData) map[string]Counter { + zapiCounters := visitZapiTemplates(filepath.Join(dir, "conf", "zapi", "cdot"), client, func(path string, client *zapi.Client) map[string]Counter { // revive:disable-line:unused-parameter + return processZapiConfigCounters(path, metricsPanelMap) + }) + zapiPerfCounters := visitZapiTemplates(filepath.Join(dir, "conf", "zapiperf", "cdot"), client, func(path string, client *zapi.Client) map[string]Counter { + if _, ok := excludePerfTemplates[filepath.Base(path)]; ok { + return nil + } + return processZAPIPerfCounters(path, client, metricsPanelMap) + }) + + maps.Copy(zapiCounters, zapiPerfCounters) + return zapiCounters +} + +func MergeCounters(restCounters map[string]Counter, zapiCounters map[string]Counter) map[string]Counter { + // handle special counters + restKeys := slices.Sorted(maps.Keys(restCounters)) + for _, k := range restKeys { + v := restCounters[k] + hashIndex := strings.Index(k, "#") + if hashIndex != -1 { + if v1, ok := restCounters[v.Name]; !ok { + v.Description = reRemove.ReplaceAllString(v.Description, "") + // Remove extra spaces from the description + v.Description = strings.Join(strings.Fields(v.Description), " ") + restCounters[v.Name] = v + } else { + v1.APIs = append(v1.APIs, v.APIs...) + restCounters[v.Name] = v1 + } + delete(restCounters, k) + } + } + + zapiKeys := slices.Sorted(maps.Keys(zapiCounters)) + for _, k := range zapiKeys { + v := zapiCounters[k] + hashIndex := strings.Index(k, "#") + if hashIndex != -1 { + if v1, ok := zapiCounters[v.Name]; !ok { + v.Description = reRemove.ReplaceAllString(v.Description, "") + // Remove extra spaces from the description + v.Description = strings.Join(strings.Fields(v.Description), " ") + zapiCounters[v.Name] = v + } else { + v1.APIs = append(v1.APIs, v.APIs...) + zapiCounters[v.Name] = v1 + } + delete(zapiCounters, k) + } + } + + // special keys are deleted hence sort again + zapiKeys = slices.Sorted(maps.Keys(zapiCounters)) + for _, k := range zapiKeys { + v := zapiCounters[k] + if v1, ok := restCounters[k]; ok { + v1.APIs = append(v1.APIs, v.APIs...) + restCounters[k] = v1 + } else { + zapiDef := v.APIs[0] + if zapiDef.ONTAPCounter == "instance_name" || zapiDef.ONTAPCounter == "instance_uuid" { + continue + } + co := Counter{ + Object: v.Object, + Name: v.Name, + Description: v.Description, + APIs: []MetricDef{zapiDef}, + Panels: v.Panels, + } + restCounters[v.Name] = co + } + } + return restCounters +} + +func ProcessExternalCounters(dir string, counters map[string]Counter, metricsPanelMap map[string]PanelData) map[string]Counter { + dat, err := os.ReadFile(filepath.Join(dir, "cmd", "tools", "generate", "counter.yaml")) + if err != nil { + fmt.Printf("error while reading file %v", err) + return nil + } + var c Counters + + err = yaml.Unmarshal(dat, &c) + if err != nil { + fmt.Printf("error while parsing file %v", err) + return nil + } + for _, v := range c.C { + if v1, ok := counters[v.Name]; !ok { + v.Panels = metricsPanelMap[v.Name].Panels + counters[v.Name] = v + } else { + if v.Description != "" { + v1.Description = v.Description + } + for _, m := range v.APIs { + indices := findAPI(v1.APIs, m) + if len(indices) == 0 { + v1.APIs = append(v1.APIs, m) + } else { + for _, index := range indices { + r := &v1.APIs[index] + if m.ONTAPCounter != "" { + r.ONTAPCounter = m.ONTAPCounter + } + if m.Template != "" { + r.Template = m.Template + } + if m.Endpoint != "" { + r.Endpoint = m.Endpoint + } + if m.Type != "" { + r.Type = m.Type + } + if m.Unit != "" { + r.Unit = m.Unit + } + if m.BaseCounter != "" { + r.BaseCounter = m.BaseCounter + } + } + } + } + counters[v.Name] = v1 + } + } + return counters +} + +func visitRestTemplates(dir string, client *rest.Client, eachTemp func(path string, client *rest.Client) map[string]Counter) map[string]Counter { + result := make(map[string]Counter) + err := filepath.Walk(dir, func(path string, _ os.FileInfo, err error) error { + if err != nil { + log.Fatal("failed to read directory:", err) + } + ext := filepath.Ext(path) + if ext != ".yaml" { + return nil + } + if strings.HasSuffix(path, "default.yaml") || strings.HasSuffix(path, "static_counter_definitions.yaml") { + return nil + } + r := eachTemp(path, client) + maps.Copy(result, r) + return nil + }) + + if err != nil { + log.Fatal("failed to read template:", err) + return nil + } + return result +} + +func visitZapiTemplates(dir string, client *zapi.Client, eachTemp func(path string, client *zapi.Client) map[string]Counter) map[string]Counter { + result := make(map[string]Counter) + err := filepath.Walk(dir, func(path string, _ os.FileInfo, err error) error { + if err != nil { + log.Fatal("failed to read directory:", err) + } + ext := filepath.Ext(path) + if ext != ".yaml" { + return nil + } + + r := eachTemp(path, client) + maps.Copy(result, r) + return nil + }) + + if err != nil { + log.Fatal("failed to read template:", err) + return nil + } + return result +} + +// processRestConfigCounters process Rest config templates +func processRestConfigCounters(path string, api string, metricsPanelMap map[string]PanelData) map[string]Counter { + var ( + counters = make(map[string]Counter) + isInstanceLabels bool + ) + var metricLabels []string + var labels []string + t, err := tree.ImportYaml(path) + if t == nil || err != nil { + fmt.Printf("Unable to import template file %s. File is invalid or empty err=%s\n", path, err) + return nil + } + + model, err := template2.ReadTemplate(path) + if err != nil { + fmt.Printf("Unable to import template file %s. File is invalid or empty err=%s\n", path, err) + return nil + } + noExtraMetrics := len(model.MultiplierMetrics) == 0 && len(model.PluginMetrics) == 0 + templateCounters := t.GetChildS("counters") + if model.ExportData == "false" && noExtraMetrics { + return nil + } + + if templateCounters != nil { + metricLabels, labels, isInstanceLabels = getAllExportedLabels(t, templateCounters.GetAllChildContentS()) + processCounters(templateCounters.GetAllChildContentS(), &model, path, model.Query, counters, metricLabels, api, metricsPanelMap) + if isInstanceLabels { + // This is for object_labels metrics + harvestName := model.Object + "_" + "labels" + description := "This metric provides information about " + model.Name + counters[harvestName] = Counter{ + Name: harvestName, + Description: description, + APIs: []MetricDef{ + { + API: "REST", + Endpoint: model.Query, + Template: path, + ONTAPCounter: "Harvest generated", + }, + }, + Panels: metricsPanelMap[harvestName].Panels, + Labels: labels, + } + } + } + + endpoints := t.GetChildS("endpoints") + if endpoints != nil { + for _, endpoint := range endpoints.GetChildren() { + var query string + for _, line := range endpoint.GetChildren() { + if line.GetNameS() == "query" { + query = line.GetContentS() + } + if line.GetNameS() == "counters" { + processCounters(line.GetAllChildContentS(), &model, path, query, counters, metricLabels, api, metricsPanelMap) + } + } + } + } + + // If the template has any PluginMetrics, add them + for _, metric := range model.PluginMetrics { + co := Counter{ + Object: model.Object, + Name: model.Object + "_" + metric.Name, + APIs: []MetricDef{ + { + API: api, + Endpoint: model.Query, + Template: path, + ONTAPCounter: metric.Source, + }, + }, + Panels: metricsPanelMap[model.Object+"_"+metric.Name].Panels, + } + if model.ExportData != "false" { + counters[co.Name] = co + } + } + + if api == keyPerfAPI { + // handling for templates with common object names + if specialPerfObjects[model.Object] { + return specialHandlingPerfCounters(counters, model) + } + } + + return counters +} + +func processCounters(counterContents []string, model *template2.Model, path, query string, counters map[string]Counter, metricLabels []string, api string, metricsPanelMap map[string]PanelData) { + var ( + staticCounterDef keyperf.ObjectCounters + err error + defLocation string + ) + if api == keyPerfAPI { + logger := logging.Get() + // CLI conf/keyperf/9.15.0/aggr.yaml + // CI ../../conf/keyperf/9.15.0/volume.yaml + defLocation = filepath.Join(filepath.Dir(filepath.Dir(path)), "static_counter_definitions.yaml") + + staticCounterDef, err = keyperf.LoadStaticCounterDefinitions(model.Object, defLocation, logger) + if err != nil { + fmt.Printf("Failed to load static counter definitions=%s\n", err) + } + } + + for _, c := range counterContents { + if c == "" { + continue + } + var co Counter + name, display, m, _ := template3.ParseMetric(c) + if _, ok := excludeCounters[name]; ok { + continue + } + description := searchDescriptionSwagger(model.Object, name) + harvestName := model.Object + "_" + display + if m == "float" { + if api == keyPerfAPI { + var ( + unit string + counterType string + denominator string + ) + switch { + case strings.Contains(name, "latency"): + counterType = "average" + unit = "microsec" + denominator = model.Object + "_" + strings.Replace(name, "latency", "iops", 1) + case strings.Contains(name, "iops"): + counterType = "rate" + unit = "per_sec" + case strings.Contains(name, "throughput"): + counterType = "rate" + unit = "b_per_sec" + case strings.Contains(name, "timestamp"): + counterType = "delta" + unit = "sec" + default: + // look up metric in staticCounterDef + if counterDef, exists := staticCounterDef.CounterDefinitions[name]; exists { + counterType = counterDef.Type + unit = counterDef.BaseCounter + } + } + + co = Counter{ + Object: model.Object, + Name: harvestName, + Description: description, + APIs: []MetricDef{ + { + API: api, + Endpoint: model.Query, + Template: path, + ONTAPCounter: name, + Unit: unit, + Type: counterType, + BaseCounter: denominator, + }, + }, + Panels: metricsPanelMap[harvestName].Panels, + Labels: metricLabels, + } + } else { + co = Counter{ + Object: model.Object, + Name: harvestName, + Description: description, + APIs: []MetricDef{ + { + API: api, + Endpoint: query, + Template: path, + ONTAPCounter: name, + }, + }, + Panels: metricsPanelMap[harvestName].Panels, + Labels: metricLabels, + } + } + counters[harvestName] = co + + // If the template has any MultiplierMetrics, add them + for _, metric := range model.MultiplierMetrics { + mc := co + addAggregatedCounter(&mc, metric, harvestName, display, metricsPanelMap) + counters[mc.Name] = mc + } + } + } +} + +// processZAPIPerfCounters process ZapiPerf counters +func processZAPIPerfCounters(path string, client *zapi.Client, metricsPanelMap map[string]PanelData) map[string]Counter { + var ( + counters = make(map[string]Counter) + request, response *node.Node + zapiUnitMap = make(map[string]string) + zapiTypeMap = make(map[string]string) + zapiDescMap = make(map[string]string) + zapiBaseCounterMap = make(map[string]string) + zapiDeprecateCounterMap = make(map[string]string) + ) + t, err := tree.ImportYaml(path) + if t == nil || err != nil { + fmt.Printf("Unable to import template file %s. File is invalid or empty\n", path) + return nil + } + model, err := template2.ReadTemplate(path) + if err != nil { + fmt.Printf("Unable to import template file %s. File is invalid or empty err=%s\n", path, err) + return nil + } + + noExtraMetrics := len(model.MultiplierMetrics) == 0 && len(model.PluginMetrics) == 0 + templateCounters := t.GetChildS("counters") + override := t.GetChildS("override") + + if model.ExportData == "false" && noExtraMetrics { + return nil + } + + if templateCounters == nil { + return nil + } + + if client != nil { + // build request + request = node.NewXMLS("perf-object-counter-list-info") + request.NewChildS("objectname", model.Query) + + if err = client.BuildRequest(request); err != nil { + fmt.Printf("error while building request %+v\n", err) + return nil + } + + if response, err = client.Invoke(""); err != nil { + fmt.Printf("error while invoking api %+v\n", err) + return nil + } + // fetch counter elements + if elems := response.GetChildS("counters"); elems != nil && len(elems.GetChildren()) != 0 { + for _, counter := range elems.GetChildren() { + name := counter.GetChildContentS("name") + if name == "" { + continue + } + ty := counter.GetChildContentS("properties") + if override != nil { + oty := override.GetChildContentS(name) + if oty != "" { + ty = oty + } + } + + zapiUnitMap[name] = counter.GetChildContentS("unit") + zapiDescMap[name] = updateDescription(counter.GetChildContentS("desc")) + zapiTypeMap[name] = ty + zapiBaseCounterMap[name] = counter.GetChildContentS("base-counter") + + if counter.GetChildContentS("is-deprecated") == "true" { + if r := counter.GetChildContentS("replaced-by"); r != "" { + zapiDeprecateCounterMap[name] = r + } + } + } + } + } + + metricLabels, labels, isInstanceLabels := getAllExportedLabels(t, templateCounters.GetAllChildContentS()) + if isInstanceLabels { + // This is for object_labels metrics + harvestName := model.Object + "_" + "labels" + description := "This metric provides information about " + model.Name + counters[harvestName] = Counter{ + Name: harvestName, + Description: description, + APIs: []MetricDef{ + { + API: "ZAPI", + Endpoint: model.Query, + Template: path, + ONTAPCounter: "Harvest generated", + }, + }, + Panels: metricsPanelMap[harvestName].Panels, + Labels: labels, + } + } + for _, c := range templateCounters.GetAllChildContentS() { + if c != "" { + name, display, m, _ := template3.ParseMetric(c) + if after, ok := strings.CutPrefix(display, model.Object); ok { + display = after + display = strings.TrimPrefix(display, "_") + } + harvestName := model.Object + "_" + display + if m == "float" { + if _, ok := excludeCounters[name]; ok { + continue + } + if zapiTypeMap[name] != "string" { + description := zapiDescMap[name] + if strings.Contains(path, "volume.yaml") && model.Object == "volume" { + if description != "" { + description += " " + } + description += "(Note: This is applicable only for ONTAP 9.9 and below. Harvest uses KeyPerf collector for ONTAP 9.10 onwards.)" + } + co := Counter{ + Object: model.Object, + Name: harvestName, + Description: description, + APIs: []MetricDef{ + { + API: "ZapiPerf", + Endpoint: "perf-object-get-instances" + " " + model.Query, + Template: path, + ONTAPCounter: name, + Unit: zapiUnitMap[name], + Type: zapiTypeMap[name], + BaseCounter: zapiBaseCounterMap[name], + }, + }, + Panels: metricsPanelMap[harvestName].Panels, + Labels: metricLabels, + } + if model.ExportData != "false" { + counters[harvestName] = co + } + + // handle deprecate counters + if rName, ok := zapiDeprecateCounterMap[name]; ok { + hName := model.Object + "_" + rName + ro := Counter{ + Object: model.Object, + Name: hName, + Description: zapiDescMap[rName], + APIs: []MetricDef{ + { + API: "ZapiPerf", + Endpoint: "perf-object-get-instances" + " " + model.Query, + Template: path, + ONTAPCounter: rName, + Unit: zapiUnitMap[rName], + Type: zapiTypeMap[rName], + BaseCounter: zapiBaseCounterMap[rName], + }, + }, + Panels: metricsPanelMap[hName].Panels, + } + if model.ExportData != "false" { + counters[hName] = ro + } + } + + // If the template has any MultiplierMetrics, add them + for _, metric := range model.MultiplierMetrics { + mc := co + addAggregatedCounter(&mc, metric, harvestName, display, metricsPanelMap) + counters[mc.Name] = mc + } + } + } + } + } + + // If the template has any PluginMetrics, add them + for _, metric := range model.PluginMetrics { + co := Counter{ + Object: model.Object, + Name: model.Object + "_" + metric.Name, + APIs: []MetricDef{ + { + API: "ZapiPerf", + Endpoint: model.Query, + Template: path, + ONTAPCounter: metric.Source, + }, + }, + Panels: metricsPanelMap[model.Object+"_"+metric.Name].Panels, + } + counters[co.Name] = co + } + // handling for templates with common object names + if specialPerfObjects[model.Object] { + return specialHandlingPerfCounters(counters, model) + } + return counters +} + +func processZapiConfigCounters(path string, metricsPanelMap map[string]PanelData) map[string]Counter { + var ( + counters = make(map[string]Counter) + isInstanceLabels bool + ) + var metricLabels []string + var labels []string + t, err := tree.ImportYaml(path) + if t == nil || err != nil { + fmt.Printf("Unable to import template file %s. File is invalid or empty\n", path) + return nil + } + model, err := template2.ReadTemplate(path) + if err != nil { + fmt.Printf("Unable to import template file %s. File is invalid or empty err=%s\n", path, err) + return nil + } + noExtraMetrics := len(model.MultiplierMetrics) == 0 && len(model.PluginMetrics) == 0 + templateCounters := t.GetChildS("counters") + if model.ExportData == "false" && noExtraMetrics { + return nil + } + if templateCounters == nil { + return nil + } + + zc := make(map[string]string) + metricLabels, labels, isInstanceLabels = getAllExportedLabels(t, templateCounters.GetAllChildContentS()) + if isInstanceLabels { + // This is for object_labels metrics + harvestName := model.Object + "_" + "labels" + description := "This metric provides information about " + model.Name + counters[harvestName] = Counter{ + Name: harvestName, + Description: description, + APIs: []MetricDef{ + { + API: "ZAPI", + Endpoint: model.Query, + Template: path, + ONTAPCounter: "Harvest generated", + }, + }, + Panels: metricsPanelMap[harvestName].Panels, + Labels: labels, + } + } + for _, c := range templateCounters.GetChildren() { + parseZapiCounters(c, []string{}, model.Object, zc) + } + + for k, v := range zc { + if _, ok := excludeCounters[k]; ok { + continue + } + co := Counter{ + Object: model.Object, + Name: k, + APIs: []MetricDef{ + { + API: "ZAPI", + Endpoint: model.Query, + Template: path, + ONTAPCounter: v, + }, + }, + Panels: metricsPanelMap[k].Panels, + Labels: metricLabels, + } + if model.ExportData != "false" { + counters[k] = co + } + + // If the template has any MultiplierMetrics, add them + for _, metric := range model.MultiplierMetrics { + mc := co + addAggregatedCounter(&mc, metric, co.Name, model.Object, metricsPanelMap) + counters[mc.Name] = mc + } + } + + // If the template has any PluginMetrics, add them + for _, metric := range model.PluginMetrics { + co := Counter{ + Object: model.Object, + Name: model.Object + "_" + metric.Name, + APIs: []MetricDef{ + { + API: "ZAPI", + Endpoint: model.Query, + Template: path, + ONTAPCounter: metric.Source, + }, + }, + Panels: metricsPanelMap[model.Object+"_"+metric.Name].Panels, + } + counters[co.Name] = co + } + return counters +} + +func processRestPerfCounters(path string, client *rest.Client, metricsPanelMap map[string]PanelData) map[string]Counter { + var ( + records []gjson.Result + counterSchema gjson.Result + counters = make(map[string]Counter) + ) + t, err := tree.ImportYaml(path) + if t == nil || err != nil { + fmt.Printf("Unable to import template file %s. File is invalid or empty\n", path) + return nil + } + model, err := template2.ReadTemplate(path) + if err != nil { + fmt.Printf("Unable to import template file %s. File is invalid or empty err=%s\n", path, err) + return nil + } + noExtraMetrics := len(model.MultiplierMetrics) == 0 && len(model.PluginMetrics) == 0 + templateCounters := t.GetChildS("counters") + override := t.GetChildS("override") + if model.ExportData == "false" && noExtraMetrics { + return nil + } + if templateCounters == nil { + return nil + } + counterMap := make(map[string]string) + counterMapNoPrefix := make(map[string]string) + metricLabels, labels, isInstanceLabels := getAllExportedLabels(t, templateCounters.GetAllChildContentS()) + if isInstanceLabels { + description := "This metric provides information about " + model.Name + // This is for object_labels metrics + harvestName := model.Object + "_" + "labels" + counters[harvestName] = Counter{ + Name: harvestName, + Description: description, + APIs: []MetricDef{ + { + API: "RestPerf", + Endpoint: model.Query, + Template: path, + ONTAPCounter: "Harvest generated", + }, + }, + Panels: metricsPanelMap[harvestName].Panels, + Labels: labels, + } + } + for _, c := range templateCounters.GetAllChildContentS() { + if c != "" { + name, display, m, _ := template3.ParseMetric(c) + if m == "float" { + counterMap[name] = model.Object + "_" + display + counterMapNoPrefix[name] = display + } + } + } + + if client != nil { + href := rest.NewHrefBuilder(). + APIPath(model.Query). + Build() + records, err = rest.FetchAll(client, href) + if err != nil { + fmt.Printf("error while invoking api %+v\n", err) + return nil + } + firstRecord := records[0] + if firstRecord.Exists() { + counterSchema = firstRecord.Get("counter_schemas") + } else { + return nil + } + counterSchema.ForEach(func(_, r gjson.Result) bool { + if !r.IsObject() { + return true + } + ontapCounterName := r.Get("name").ClonedString() + if _, ok := excludeCounters[ontapCounterName]; ok { + return true + } + + description := r.Get("description").ClonedString() + ty := r.Get("type").ClonedString() + if override != nil { + oty := override.GetChildContentS(ontapCounterName) + if oty != "" { + ty = oty + } + } + if v, ok := counterMap[ontapCounterName]; ok { + if ty == "string" { + return true + } + c := Counter{ + Object: model.Object, + Name: v, + Description: description, + APIs: []MetricDef{ + { + API: "RestPerf", + Endpoint: model.Query, + Template: path, + ONTAPCounter: ontapCounterName, + Unit: r.Get("unit").ClonedString(), + Type: ty, + BaseCounter: r.Get("denominator.name").ClonedString(), + }, + }, + Panels: metricsPanelMap[v].Panels, + Labels: metricLabels, + } + if model.ExportData != "false" { + counters[c.Name] = c + } + + // If the template has any MultiplierMetrics, add them + for _, metric := range model.MultiplierMetrics { + mc := c + addAggregatedCounter(&mc, metric, v, counterMapNoPrefix[ontapCounterName], metricsPanelMap) + counters[mc.Name] = mc + } + } + return true + }) + } else { + // + for ontapName, counterName := range counterMap { + c := Counter{ + Object: model.Object, + Name: counterName, + APIs: []MetricDef{ + { + API: "RestPerf", + Endpoint: model.Query, + Template: path, + ONTAPCounter: ontapName, + }, + }, + Panels: metricsPanelMap[counterName].Panels, + Labels: metricLabels, + } + if model.ExportData != "false" { + counters[c.Name] = c + } + + // If the template has any MultiplierMetrics, add them + for _, metric := range model.MultiplierMetrics { + mc := c + addAggregatedCounter(&mc, metric, counterName, counterMapNoPrefix[ontapName], metricsPanelMap) + counters[mc.Name] = mc + } + } + } + + // If the template has any PluginMetrics, add them + for _, metric := range model.PluginMetrics { + co := Counter{ + Object: model.Object, + Name: model.Object + "_" + metric.Name, + APIs: []MetricDef{ + { + API: "RestPerf", + Endpoint: model.Query, + Template: path, + ONTAPCounter: metric.Source, + }, + }, + Panels: metricsPanelMap[model.Object+"_"+metric.Name].Panels, + } + counters[co.Name] = co + } + // handling for templates with common object names/metric name + if specialPerfObjects[model.Object] { + return specialHandlingPerfCounters(counters, model) + } + return counters +} + +func processStatPerfCounters(path string, client *rest.Client, metricsPanelMap map[string]PanelData) map[string]Counter { + var ( + records []gjson.Result + counters = make(map[string]Counter) + cliCommand []byte + ) + t, err := tree.ImportYaml(path) + if t == nil || err != nil { + fmt.Printf("Unable to import template file %s. File is invalid or empty\n", path) + return nil + } + model, err := template2.ReadTemplate(path) + if err != nil { + fmt.Printf("Unable to import template file %s. File is invalid or empty err=%s\n", path, err) + return nil + } + noExtraMetrics := len(model.MultiplierMetrics) == 0 && len(model.PluginMetrics) == 0 + templateCounters := t.GetChildS("counters") + override := t.GetChildS("override") + if model.ExportData == "false" && noExtraMetrics { + return nil + } + if templateCounters == nil { + return nil + } + counterMap := make(map[string]string) + counterMapNoPrefix := make(map[string]string) + metricLabels, labels, isInstanceLabels := getAllExportedLabels(t, templateCounters.GetAllChildContentS()) + if isInstanceLabels { + description := "This metric provides information about " + model.Name + // This is for object_labels metrics + harvestName := model.Object + "_" + "labels" + counters[harvestName] = Counter{ + Name: harvestName, + Description: description, + APIs: []MetricDef{ + { + API: "StatPerf", + Endpoint: model.Query, + Template: path, + ONTAPCounter: "Harvest generated", + }, + }, + Panels: metricsPanelMap[harvestName].Panels, + Labels: labels, + } + } + for _, c := range templateCounters.GetAllChildContentS() { + if c != "" { + name, display, m, _ := template3.ParseMetric(c) + if m == "float" { + counterMap[name] = model.Object + "_" + display + counterMapNoPrefix[name] = display + } + } + } + + if client != nil { + cliCommand, err = clirequestbuilder.New(). + BaseSet(statperf.GetCounterInstanceBaseSet()). + Query("statistics catalog counter show"). + Object(model.Query). + Fields([]string{"counter", "base-counter", "properties", "type", "is-deprecated", "replaced-by", "unit", "description"}). + Build() + if err != nil { + fmt.Printf("error while build clicommand %+v\n", err) + return nil + } + records, err = rest.FetchPost(client, "api/private/cli", cliCommand) + if err != nil { + fmt.Printf("error while invoking api %+v\n", err) + return nil + } + firstRecord := records[0] + fr := firstRecord.ClonedString() + if fr == "" { + fmt.Printf("no data found for query %s template %s", model.Query, path) + return nil + } + s := &statperf.StatPerf{} + pCounters, err := s.ParseCounters(fr) + if err != nil { + fmt.Printf("error while parsing records for query %s template %s: %+v\n", model.Query, path, err) + return nil + } + for _, p := range pCounters { + ontapCounterName := statperf.NormalizeCounterValue(p.Name) + description := statperf.NormalizeCounterValue(p.Description) + ty := p.Type + if override != nil { + oty := override.GetChildContentS(ontapCounterName) + if oty != "" { + ty = oty + } + } + if v, ok := counterMap[ontapCounterName]; ok { + if ty == "string" { + continue + } + c := Counter{ + Object: model.Object, + Name: v, + Description: description, + APIs: []MetricDef{ + { + API: "StatPerf", + Endpoint: model.Query, + Template: path, + ONTAPCounter: ontapCounterName, + Unit: statperf.NormalizeCounterValue(p.Unit), + Type: statperf.NormalizeCounterValue(ty), + BaseCounter: statperf.NormalizeCounterValue(p.BaseCounter), + }, + }, + Panels: metricsPanelMap[v].Panels, + Labels: metricLabels, + } + if model.ExportData != "false" { + counters[c.Name] = c + } + + // If the template has any MultiplierMetrics, add them + for _, metric := range model.MultiplierMetrics { + mc := c + addAggregatedCounter(&mc, metric, v, counterMapNoPrefix[ontapCounterName], metricsPanelMap) + counters[mc.Name] = mc + } + } + } + } else { + for ontapName, counterName := range counterMap { + c := Counter{ + Object: model.Object, + Name: counterName, + APIs: []MetricDef{ + { + API: "StatPerf", + Endpoint: model.Query, + Template: path, + ONTAPCounter: ontapName, + }, + }, + Panels: metricsPanelMap[counterName].Panels, + Labels: metricLabels, + } + if model.ExportData != "false" { + counters[c.Name] = c + } + + // If the template has any MultiplierMetrics, add them + for _, metric := range model.MultiplierMetrics { + mc := c + addAggregatedCounter(&mc, metric, counterName, counterMapNoPrefix[ontapName], metricsPanelMap) + counters[mc.Name] = mc + } + } + } + + // If the template has any PluginMetrics, add them + for _, metric := range model.PluginMetrics { + co := Counter{ + Object: model.Object, + Name: model.Object + "_" + metric.Name, + APIs: []MetricDef{ + { + API: "StatPerf", + Endpoint: model.Query, + Template: path, + ONTAPCounter: metric.Source, + }, + }, + Panels: metricsPanelMap[model.Object+"_"+metric.Name].Panels, + } + counters[co.Name] = co + } + return counters +} + +func specialHandlingPerfCounters(counters map[string]Counter, model template2.Model) map[string]Counter { + // handling for templates with common object names + modifiedCounters := make(map[string]Counter) + for originalKey, value := range counters { + modifiedKey := model.Name + "#" + originalKey + modifiedCounters[modifiedKey] = value + } + return modifiedCounters +} + +func addAggregatedCounter(c *Counter, metric plugin.DerivedMetric, withPrefix string, noPrefix string, metricsPanelMap map[string]PanelData) { + if !strings.HasSuffix(c.Description, ".") { + c.Description += "." + } + + if metric.IsMax { + c.Name = metric.Name + "_" + noPrefix + c.Description = fmt.Sprintf("%s %s is the maximum of [%s](#%s) for label `%s`.", + c.Description, c.Name, withPrefix, withPrefix, metric.Source) + } else { + c.Name = metric.Name + "_" + c.Name + if metric.HasCustomName { + c.Name = metric.Source + "_" + noPrefix + } + c.Description = fmt.Sprintf("%s %s is [%s](#%s) aggregated by `%s`.", + c.Description, c.Name, withPrefix, withPrefix, metric.Name) + } + c.Panels = metricsPanelMap[c.Name].Panels +} + +func findAPI(apis []MetricDef, other MetricDef) []int { + var indices []int + for i, a := range apis { + if a.API == other.API { + indices = append(indices, i) + } + } + return indices +} + +func FetchAndCategorizePrometheusMetrics(promURL string) (map[string]bool, map[string]bool, error) { + urlStr := promURL + "/api/v1/series?match[]={datacenter!=\"\"}" + + u, err := url.Parse(urlStr) + if err != nil { + return nil, nil, fmt.Errorf("failed to parse URL: %w", err) + } + + resp, err := http.Get(u.String()) + if err != nil { + return nil, nil, fmt.Errorf("failed to fetch metrics from Prometheus: %w", err) + } + //goland:noinspection GoUnhandledErrorResult + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, nil, fmt.Errorf("unexpected status code from Prometheus: %d", resp.StatusCode) + } + + var result struct { + Status string `json:"status"` + Data []map[string]string `json:"data"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, nil, fmt.Errorf("failed to decode Prometheus response: %w", err) + } + if result.Status != "success" { + return nil, nil, fmt.Errorf("unexpected status from Prometheus: %s", result.Status) + } + + // Categorize metrics + restMetrics := make(map[string]bool) + zapiMetrics := make(map[string]bool) + for _, series := range result.Data { + metricName := series["__name__"] + switch series["datacenter"] { + case "REST": + restMetrics[metricName] = true + case "ZAPI": + zapiMetrics[metricName] = true + case "keyperf": + restMetrics[metricName] = true + } + } + + return restMetrics, zapiMetrics, nil +} +func ValidateMetrics(documentedRest, documentedZapi map[string]Counter, prometheusRest, prometheusZapi map[string]bool) error { + var documentedButMissingRestMetrics []string + var notDocumentedRestMetrics []string + var documentedButMissingZapiMetrics []string + var notDocumentedZapiMetrics []string + + // Helper function to check if a REST metric should be excluded + shouldExcludeRest := func(metric string, apis []MetricDef) bool { + for _, c := range excludeDocumentedRestMetrics { + if strings.Contains(metric, c) { + return true + } + } + + for _, api := range apis { + if api.API == "ZAPI" || api.API == "ZapiPerf" { + return true + } + } + + return false + } + + // Helper function to check if a ZAPI metric should be excluded + shouldExcludeZapi := func(metric string, apis []MetricDef) bool { + for _, prefix := range excludeDocumentedZapiMetrics { + if strings.Contains(metric, prefix) { + return true + } + } + + for _, api := range apis { + if api.API == "REST" || api.API == "RestPerf" { + return true + } + } + return false + } + + // Helper function to check if an extra REST metric should be excluded + shouldExcludeExtraRest := func(metric string, set *set.Set) bool { + + for _, c := range excludeNotDocumentedRestMetrics { + if strings.Contains(metric, c) { + return true + } + } + + var isRestObject bool + for o := range set.Iter() { + if strings.HasPrefix(metric, o) { + isRestObject = true + } + } + return !isRestObject + } + + // Helper function to check if an extra ZAPI metric should be excluded + shouldExcludeExtraZapi := func(metric string) bool { + + for _, c := range excludeNotDocumentedZapiMetrics { + if strings.Contains(metric, c) { + return true + } + } + + return false + } + + restObjects := set.New() + + for metric, counter := range documentedRest { + if counter.Object != "" { + restObjects.Add(counter.Object) + } + if !prometheusRest[metric] && !shouldExcludeRest(metric, counter.APIs) { + documentedButMissingRestMetrics = append(documentedButMissingRestMetrics, metric) + } + } + + for metric := range prometheusRest { + if _, ok := documentedRest[metric]; !ok && !shouldExcludeExtraRest(metric, restObjects) { + notDocumentedRestMetrics = append(notDocumentedRestMetrics, metric) + } + } + + for metric, counter := range documentedZapi { + if !prometheusZapi[metric] && !shouldExcludeZapi(metric, counter.APIs) { + documentedButMissingZapiMetrics = append(documentedButMissingZapiMetrics, metric) + } + } + + for metric := range prometheusZapi { + if _, ok := documentedZapi[metric]; !ok && !shouldExcludeExtraZapi(metric) { + notDocumentedZapiMetrics = append(notDocumentedZapiMetrics, metric) + } + } + + // Sort the slices + sort.Strings(documentedButMissingRestMetrics) + sort.Strings(notDocumentedRestMetrics) + sort.Strings(documentedButMissingZapiMetrics) + sort.Strings(notDocumentedZapiMetrics) + + if len(documentedButMissingRestMetrics) > 0 || len(notDocumentedRestMetrics) > 0 || len(documentedButMissingZapiMetrics) > 0 || len(notDocumentedZapiMetrics) > 0 { + errorMessage := "Validation failed:\n" + if len(documentedButMissingRestMetrics) > 0 { + errorMessage += fmt.Sprintf("Missing Rest metrics in Prometheus but documented: %v\n", documentedButMissingRestMetrics) + } + if len(notDocumentedRestMetrics) > 0 { + errorMessage += fmt.Sprintf("Extra Rest metrics in Prometheus but not documented: %v\n", notDocumentedRestMetrics) + } + if len(documentedButMissingZapiMetrics) > 0 { + errorMessage += fmt.Sprintf("Missing Zapi metrics in Prometheus but documented: %v\n", documentedButMissingZapiMetrics) + } + if len(notDocumentedZapiMetrics) > 0 { + errorMessage += fmt.Sprintf("Extra Zapi metrics in Prometheus but not documented: %v\n", notDocumentedZapiMetrics) + } + return errors.New(errorMessage) + } + + return nil +} + +func CategorizeCounters(counters map[string]Counter) (map[string]Counter, map[string]Counter) { + restCounters := make(map[string]Counter) + zapiCounters := make(map[string]Counter) + + for _, counter := range counters { + for _, api := range counter.APIs { + switch api.API { + case "REST": + restCounters[counter.Name] = counter + case "RestPerf": + restCounters[counter.Name] = counter + case "ZAPI": + zapiCounters[counter.Name] = counter + case "ZapiPerf": + zapiCounters[counter.Name] = counter + case "KeyPerf": + restCounters[counter.Name] = counter + } + } + } + + return restCounters, zapiCounters +} + +func getAllExportedLabels(t *node.Node, counterContents []string) ([]string, []string, bool) { + metricLabels := make([]string, 0) + labels := make([]string, 0) + isInstanceLabels := false + eData := true + if exportData := t.GetChildS("export_data"); exportData != nil { + if exportData.GetContentS() == "false" { + eData = false + } + } + if exportOptions := t.GetChildS("export_options"); exportOptions != nil { + if iAllLabels := exportOptions.GetChildS("include_all_labels"); iAllLabels != nil { + if iAllLabels.GetContentS() == "true" { + for _, c := range counterContents { + if c == "" { + continue + } + if _, display, m, _ := template3.ParseMetric(c); m == "key" || m == "label" { + metricLabels = append(metricLabels, display) + } + } + return metricLabels, metricLabels, false + } + } + + if iKeys := exportOptions.GetChildS("instance_keys"); iKeys != nil { + metricLabels = append(metricLabels, iKeys.GetAllChildContentS()...) + } + if iLabels := exportOptions.GetChildS("instance_labels"); iLabels != nil { + labels = append(labels, iLabels.GetAllChildContentS()...) + isInstanceLabels = eData + } + } + return metricLabels, append(labels, metricLabels...), isInstanceLabels +} + +// searchDescriptionSwagger returns ontap counter description from swagger +func searchDescriptionSwagger(objName string, ontapCounterName string) string { + val, ok := objectSwaggerMap[objName] + if ok { + objName = val + } + searchQuery := strings.Join([]string{"definitions", objName, "properties"}, ".") + cs := strings.Split(ontapCounterName, ".") + for i, c := range cs { + if i < len(cs)-1 { + searchQuery = strings.Join([]string{searchQuery, c, "properties"}, ".") + } else { + searchQuery = strings.Join([]string{searchQuery, c, "description"}, ".") + } + } + t := gjson.GetBytes(swaggerBytes, searchQuery) + return updateDescription(t.ClonedString()) +} + +// parseZapiCounters parse zapi template counters +func parseZapiCounters(elem *node.Node, path []string, object string, zc map[string]string) { + + name := elem.GetNameS() + newPath := path + + if elem.GetNameS() != "" { + newPath = append(newPath, name) + } + + if elem.GetContentS() != "" { + v, k := handleZapiCounter(newPath, elem.GetContentS(), object) + if k != "" { + zc[k] = v + } + } + + for _, child := range elem.GetChildren() { + parseZapiCounters(child, newPath, object, zc) + } +} + +// handleZapiCounter returns zapi ontap and display counter name +func handleZapiCounter(path []string, content string, object string) (string, string) { + var ( + name, display, key string + splitValues, fullPath []string + ) + + splitValues = strings.Split(content, "=>") + if len(splitValues) == 1 { + name = content + } else { + name = splitValues[0] + display = strings.TrimSpace(splitValues[1]) + } + + name = strings.TrimSpace(strings.TrimLeft(name, "^")) + fullPath = path + fullPath = append(fullPath, name) + key = strings.Join(fullPath, ".") + if display == "" { + display = template3.ParseZAPIDisplay(object, fullPath) + } + + if content[0] != '^' { + return key, object + "_" + display + } + + return "", "" +} + +func updateDescription(description string) string { + s := replacer.Replace(description) + return s +} + +// [Top $TopResources Average Disk Utilization Per Aggregate](GRAFANA_HOST/d/cdot-aggregate/ontap3a-aggregate?orgId=1&viewPanel=63) +// [p.Panel](GRAFANA_HOST/p.PanelLink) + +func (c Counter) Header() string { + return ` +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------|` +} + +func (c Counter) PanelHeader() string { + return ` +| Dashboard | Row | Type | Panel | +|--------|----------|--------|--------|` +} + +func (c Counter) HasAPIs() bool { + return len(c.APIs) > 0 +} + +func (c Counter) HasPanels() bool { + return len(c.Panels) > 0 +} + +func (m MetricDef) TableRow() string { + switch { + case strings.Contains(m.Template, "perf"): + unitTypeBase := `
Unit: ` + m.Unit + + `
Type: ` + m.Type + + `
Base: ` + m.BaseCounter + return fmt.Sprintf("| %s | `%s` | `%s`%s | %s |", + m.API, m.Endpoint, m.ONTAPCounter, unitTypeBase, m.Template) + case m.Unit != "": + unit := `
Unit: ` + m.Unit + return fmt.Sprintf("| %s | `%s` | `%s`%s | %s | ", + m.API, m.Endpoint, m.ONTAPCounter, unit, m.Template) + case strings.Contains(m.Template, "ciscorest"): + return fmt.Sprintf("| %s | `%s` | `%s` | %s |", m.API, m.Endpoint, m.CiscoCounter, m.Template) + case strings.Contains(m.Template, "storagegrid"): + return fmt.Sprintf("| %s | `%s` | `%s` | %s |", m.API, m.Endpoint, m.SGCounter, m.Template) + default: + return fmt.Sprintf("| %s | `%s` | `%s` | %s |", m.API, m.Endpoint, m.ONTAPCounter, m.Template) + } +} + +func (p PanelDef) DashboardTableRow() string { + return fmt.Sprintf("| %s | %s | %s | [%s](/%s) |", p.Dashboard, p.Row, p.Type, p.Panel, p.PanelLink) +} + +// readSwaggerJSON downloads poller swagger and convert to json format +func readSwaggerJSON(opts *Options) []byte { + var f []byte + path, err := downloadSwaggerForPoller(opts.Poller) + if err != nil { + log.Fatal("failed to download swagger:", err) + return nil + } + cmd := fmt.Sprintf("dasel -f %s -r yaml -w json", path) + f, err = exec.Command("bash", "-c", cmd).Output() + if err != nil { + log.Fatal("Failed to execute command:", cmd, err) + return nil + } + return f +} + +func downloadSwaggerForPoller(pName string) (string, error) { + var ( + poller *conf.Poller + err error + addr string + shouldDownload = true + swagTime time.Time + ) + + if poller, addr, err = rest.GetPollerAndAddr(pName); err != nil { + return "", err + } + + tmp := os.TempDir() + swaggerPath := filepath.Join(tmp, addr+"-swagger.yaml") + fileInfo, err := os.Stat(swaggerPath) + + if os.IsNotExist(err) { + fmt.Printf("%s does not exist downloading\n", swaggerPath) + } else { + swagTime = fileInfo.ModTime() + twoWeeksAgo := swagTime.Local().AddDate(0, 0, -14) + if swagTime.Before(twoWeeksAgo) { + fmt.Printf("%s is more than two weeks old, re-download", swaggerPath) + } else { + shouldDownload = false + } + } + + if shouldDownload { + swaggerURL := "https://" + addr + "/docs/api/swagger.yaml" + bytesDownloaded, err := downloadSwagger(poller, swaggerPath, swaggerURL, false) + if err != nil { + fmt.Printf("error downloading swagger %s\n", err) + if bytesDownloaded == 0 { + // if the tmp file exists, remove it since it is empty + _ = os.Remove(swaggerPath) + } + return "", err + } + fmt.Printf("downloaded %d bytes from %s\n", bytesDownloaded, swaggerURL) + } + + fmt.Printf("Using downloaded file %s with timestamp %s\n", swaggerPath, swagTime) + return swaggerPath, nil +} + +func downloadSwagger(poller *conf.Poller, path string, urlStr string, verbose bool) (int64, error) { + out, err := os.Create(path) + if err != nil { + return 0, fmt.Errorf("unable to create %s to save swagger.yaml", path) + } + defer func(out *os.File) { _ = out.Close() }(out) + request, err := requests.New("GET", urlStr, nil) + if err != nil { + return 0, err + } + + timeout, _ := time.ParseDuration(rest.DefaultTimeout) + credentials := auth.NewCredentials(poller, slog.Default()) + transport, err := credentials.Transport(request, poller) + if err != nil { + return 0, err + } + httpclient := &http.Client{Transport: transport, Timeout: timeout} + + if verbose { + requestOut, _ := httputil.DumpRequestOut(request, false) + fmt.Printf("REQUEST: %s\n%s\n", urlStr, requestOut) + } + response, err := httpclient.Do(request) + if err != nil { + return 0, err + } + //goland:noinspection GoUnhandledErrorResult + defer response.Body.Close() + + if verbose { + debugResp, _ := httputil.DumpResponse(response, false) + fmt.Printf("RESPONSE: \n%s", debugResp) + } + if response.StatusCode != http.StatusOK { + return 0, fmt.Errorf("error making request. server response statusCode=[%d]", response.StatusCode) + } + n, err := io.Copy(out, response.Body) + if err != nil { + return 0, fmt.Errorf("error while downloading %s err=%w", urlStr, err) + } + return n, nil +} + +func LogErrAndExit(err error) { + fmt.Printf("%v\n", err) + os.Exit(1) +} + +func GenerateOntapCounterTemplate(counters map[string]Counter, version string) { + targetPath := "docs/ontap-metrics.md" + t, err := template.New("counter.tmpl").ParseFiles("cmd/tools/generate/counter.tmpl") + if err != nil { + panic(err) + } + out, err := os.Create(targetPath) + if err != nil { + panic(err) + } + + keys := make([]string, 0, len(counters)) + for k := range counters { + keys = append(keys, k) + } + sort.Strings(keys) + values := make([]Counter, 0, len(keys)) + + table := tw.NewWriter(os.Stdout) + table.SetBorder(false) + table.SetAutoFormatHeaders(false) + table.SetAutoWrapText(false) + table.SetHeader([]string{"Missing", "Counter", "APIs", "Endpoint", "ONTAPCounter", "Template"}) + + for _, k := range keys { + if k == "" { + continue + } + counter := counters[k] + + if counter.Description == "" { + for _, def := range counter.APIs { + if _, ok := knownDescriptionGaps[counter.Name]; !ok { + appendRow(table, "Description", counter, def) + } + } + } + values = append(values, counter) + } + + for _, k := range keys { + if k == "" { + continue + } + counter := counters[k] + + // Print such counters which are missing Rest mapping + if len(counter.APIs) == 1 { + if counter.APIs[0].API == "ZAPI" { + isPrint := true + for _, substring := range excludeLogRestCounters { + if strings.HasPrefix(counter.Name, substring) { + isPrint = false + break + } + } + // missing Rest Mapping + if isPrint { + for _, def := range counter.APIs { + hasPrefix := false + for prefix := range knownMappingGaps { + if strings.HasPrefix(counter.Name, prefix) { + hasPrefix = true + break + } + } + if !hasPrefix { + appendRow(table, "REST", counter, def) + } + } + } + } + } + + for _, def := range counter.APIs { + if def.ONTAPCounter == "" { + for _, def := range counter.APIs { + hasPrefix := false + for prefix := range knownMappingGaps { + if strings.HasPrefix(counter.Name, prefix) { + hasPrefix = true + break + } + } + if !hasPrefix { + appendRow(table, "Mapping", counter, def) + } + } + } + } + } + table.Render() + c := CounterTemplate{ + Counters: values, + CounterMetaData: CounterMetaData{ + Date: time.Now().Format("2006-Jan-02"), + OntapVersion: version, + }, + } + + err = t.Execute(out, c) + if err != nil { + panic(err) + } + fmt.Printf("Harvest metric documentation generated at %s \n", targetPath) + + if table.NumLines() > 0 { + slog.Error("Issues found: Please refer to the table above") + os.Exit(1) + } +} + +func GenerateStorageGridCounterTemplate(counters map[string]Counter, version string) { + targetPath := "docs/storagegrid-metrics.md" + t, err := template.New("storagegrid_counter.tmpl").ParseFiles("cmd/tools/generate/storagegrid_counter.tmpl") + if err != nil { + panic(err) + } + out, err := os.Create(targetPath) + if err != nil { + panic(err) + } + + keys := make([]string, 0, len(counters)) + for k := range counters { + keys = append(keys, k) + } + sort.Strings(keys) + values := make([]Counter, 0, len(keys)) + + table := tw.NewWriter(os.Stdout) + table.SetBorder(false) + table.SetAutoFormatHeaders(false) + table.SetAutoWrapText(false) + table.SetHeader([]string{"Missing", "Counter", "APIs", "Endpoint", "SGCounter", "Template"}) + + for _, k := range keys { + if k == "" { + continue + } + counter := counters[k] + if !strings.HasPrefix(counter.Name, "storagegrid_") { + continue + } + + if _, ok := knownMappingGapsSG[k]; !ok { + if counter.Description == "" { + appendRow(table, "Description", counter, MetricDef{API: ""}) + } + } + + values = append(values, counter) + } + + table.Render() + c := CounterTemplate{ + Counters: values, + CounterMetaData: CounterMetaData{ + Date: time.Now().Format("2006-Jan-02"), + SGVersion: version, + }, + } + + err = t.Execute(out, c) + if err != nil { + panic(err) + } + fmt.Printf("Harvest metric documentation generated at %s \n", targetPath) + + if table.NumLines() > 0 { + log.Fatalf("Issues found: refer table above") + } +} + +func GenerateCiscoSwitchCounterTemplate(counters map[string]Counter, version string) { + targetPath := "docs/cisco-switch-metrics.md" + t, err := template.New("cisco_counter.tmpl").ParseFiles("cmd/tools/generate/cisco_counter.tmpl") + if err != nil { + panic(err) + } + out, err := os.Create(targetPath) + if err != nil { + panic(err) + } + + keys := make([]string, 0, len(counters)) + for k := range counters { + keys = append(keys, k) + } + sort.Strings(keys) + values := make([]Counter, 0, len(keys)) + + table := tw.NewWriter(os.Stdout) + table.SetBorder(false) + table.SetAutoFormatHeaders(false) + table.SetAutoWrapText(false) + table.SetHeader([]string{"Missing", "Counter", "APIs", "Endpoint", "CiscoCounter", "Template"}) + + for _, k := range keys { + if k == "" { + continue + } + counter := counters[k] + if !strings.HasPrefix(counter.Name, "cisco_") { + continue + } + + if counter.Description == "" { + appendRow(table, "Description", counter, MetricDef{API: ""}) + } + + values = append(values, counter) + } + + table.Render() + c := CounterTemplate{ + Counters: values, + CounterMetaData: CounterMetaData{ + Date: time.Now().Format("2006-Jan-02"), + CiscoVersion: version, + }, + } + + err = t.Execute(out, c) + if err != nil { + panic(err) + } + fmt.Printf("Harvest metric documentation generated at %s \n", targetPath) + + if table.NumLines() > 0 { + log.Fatalf("Issues found: refer table above") + } +} + +func appendRow(table *tw.Table, missing string, counter Counter, def MetricDef) { + if def.API != "" { + table.Append([]string{missing, counter.Name, def.API, def.Endpoint, def.ONTAPCounter, def.Template}) + } else { + table.Append([]string{missing, counter.Name}) + } +} diff --git a/docs/ontap-metrics.md b/docs/ontap-metrics.md index 935ac690e..afa4f232b 100644 --- a/docs/ontap-metrics.md +++ b/docs/ontap-metrics.md @@ -7,7 +7,7 @@ These can be generated on demand by running `bin/harvest grafana metrics`. See - More information about ONTAP REST performance counters can be found [here](https://docs.netapp.com/us-en/ontap-pcmap-9121/index.html). ``` -Creation Date : 2025-Nov-21 +Creation Date : 2025-Dec-08 ONTAP Version: 9.16.1 ``` @@ -1684,6 +1684,24 @@ Performance metric for write I/O operations. +### audit_log + +Captures the operations such as create, update, and delete attempts on volumes via REST or ONTAP CLI commands + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| REST | `NA` | `Harvest generated` | conf/rest/9.12.0/audit_log.yaml | + +The `audit_log` metric is visualized in the following Grafana dashboards: + +/// html | div.grafana-table +| Dashboard | Row | Type | Panel | +|--------|----------|--------|--------| +| ONTAP: AuditLog | Highlights | table | [Volume Changes](/d/cdot-auditlog/ontap3a-auditlog?orgId=1&viewPanel=295) | +/// + + + ### availability_zone_space_available @@ -1724,6 +1742,37 @@ Performance metric for write I/O operations. +### change_log + +Detect and track changes related to the creation, modification, and deletion of an object of Node, SVM and Volume + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| REST | `NA` | `Harvest generated` | conf/rest/9.12.0/node.yaml | +| REST | `NA` | `Harvest generated` | conf/rest/9.10.0/svm.yaml | +| REST | `NA` | `Harvest generated` | conf/rest/9.14.0/volume.yaml | + +The `change_log` metric is visualized in the following Grafana dashboards: + +/// html | div.grafana-table +| Dashboard | Row | Type | Panel | +|--------|----------|--------|--------| +| ONTAP: Changelog Monitor | Node Changes | stat | [Create](/d/cdot-changelog-monitor/ontap3a-changelog monitor?orgId=1&viewPanel=285) | +| ONTAP: Changelog Monitor | Node Changes | stat | [Update](/d/cdot-changelog-monitor/ontap3a-changelog monitor?orgId=1&viewPanel=291) | +| ONTAP: Changelog Monitor | Node Changes | stat | [Delete](/d/cdot-changelog-monitor/ontap3a-changelog monitor?orgId=1&viewPanel=286) | +| ONTAP: Changelog Monitor | Node Changes | table | [Node Changes ](/d/cdot-changelog-monitor/ontap3a-changelog monitor?orgId=1&viewPanel=288) | +| ONTAP: Changelog Monitor | SVM Changes | stat | [Create](/d/cdot-changelog-monitor/ontap3a-changelog monitor?orgId=1&viewPanel=292) | +| ONTAP: Changelog Monitor | SVM Changes | stat | [Update](/d/cdot-changelog-monitor/ontap3a-changelog monitor?orgId=1&viewPanel=298) | +| ONTAP: Changelog Monitor | SVM Changes | stat | [Delete](/d/cdot-changelog-monitor/ontap3a-changelog monitor?orgId=1&viewPanel=300) | +| ONTAP: Changelog Monitor | SVM Changes | table | [SVM Changes ](/d/cdot-changelog-monitor/ontap3a-changelog monitor?orgId=1&viewPanel=301) | +| ONTAP: Changelog Monitor | Volume Changes | stat | [Create](/d/cdot-changelog-monitor/ontap3a-changelog monitor?orgId=1&viewPanel=299) | +| ONTAP: Changelog Monitor | Volume Changes | stat | [Update](/d/cdot-changelog-monitor/ontap3a-changelog monitor?orgId=1&viewPanel=284) | +| ONTAP: Changelog Monitor | Volume Changes | stat | [Delete](/d/cdot-changelog-monitor/ontap3a-changelog monitor?orgId=1&viewPanel=293) | +| ONTAP: Changelog Monitor | Volume Changes | table | [Volume Changes ](/d/cdot-changelog-monitor/ontap3a-changelog monitor?orgId=1&viewPanel=295) | +/// + + + ### cifs_session_connection_count A counter used to track requests that are sent to the volumes to the node. @@ -2823,6 +2872,27 @@ The `environment_sensor_threshold_value` metric is visualized in the following G +### ethernet_switch_port_new_status + +Represent the status of the ethernet switch port + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| Rest | `NA` | `Harvest generated` | conf/rest/9.8.0/ethernet_switch_port.yaml | + +The `ethernet_switch_port_new_status` metric is visualized in the following Grafana dashboards: + +/// html | div.grafana-table +| Dashboard | Row | Type | Panel | +|--------|----------|--------|--------| +| ONTAP: Switch | Highlights | table | [Switch Details](/d/cdot-switch/ontap3a-switch?orgId=1&viewPanel=5) | +| ONTAP: Switch | Interfaces | stat | [Down (Last 24h)](/d/cdot-switch/ontap3a-switch?orgId=1&viewPanel=37) | +| ONTAP: Switch | Interfaces | table | [Down (Last 24h)](/d/cdot-switch/ontap3a-switch?orgId=1&viewPanel=39) | +| ONTAP: Switch | Interfaces | timeseries | [Down (Last 24h)](/d/cdot-switch/ontap3a-switch?orgId=1&viewPanel=40) | +/// + + + ### ethernet_switch_port_receive_discards Total number of discarded packets. @@ -4173,6 +4243,25 @@ Total number of FCP operations +### fcp_util_percent + +Represent the FCP utilization percentage + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| RestPerf | `NA` | `Harvest generated`
Unit:
Type:
Base: | conf/restperf/9.12.0/fcp.yaml | + +The `fcp_util_percent` metric is visualized in the following Grafana dashboards: + +/// html | div.grafana-table +| Dashboard | Row | Type | Panel | +|--------|----------|--------|--------| +| ONTAP: Network | FibreChannel | table | [FC ports](/d/cdot-network/ontap3a-network?orgId=1&viewPanel=71) | +| ONTAP: Node | Network Layer | timeseries | [Top $TopResources FC Ports by Utilization %](/d/cdot-node/ontap3a-node?orgId=1&viewPanel=110) | +/// + + + ### fcp_write_data Amount of data written to the storage system @@ -8114,6 +8203,86 @@ The `net_route_labels` metric is visualized in the following Grafana dashboards: +### netstat_bytes_recvd + +Number of bytes received by a TCP connection + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| ZapiPerf | `perf-object-get-instances netstat` | `bytes_recvd`
Unit: none
Type: raw
Base: | conf/zapiperf/cdot/9.8.0/netstat.yaml | + + + +### netstat_bytes_sent + +Number of bytes sent by a TCP connection + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| ZapiPerf | `perf-object-get-instances netstat` | `bytes_sent`
Unit: none
Type: raw
Base: | conf/zapiperf/cdot/9.8.0/netstat.yaml | + + + +### netstat_cong_win + +Congestion window of a TCP connection + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| ZapiPerf | `perf-object-get-instances netstat` | `cong_win`
Unit: none
Type: raw
Base: | conf/zapiperf/cdot/9.8.0/netstat.yaml | + + + +### netstat_cong_win_th + +Congestion window threshold of a TCP connection + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| ZapiPerf | `perf-object-get-instances netstat` | `cong_win_th`
Unit: none
Type: raw
Base: | conf/zapiperf/cdot/9.8.0/netstat.yaml | + + + +### netstat_ooorcv_pkts + +Number of out-of-order packets received by this TCP connection + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| ZapiPerf | `perf-object-get-instances netstat` | `ooorcv_pkts`
Unit: none
Type: raw
Base: | conf/zapiperf/cdot/9.8.0/netstat.yaml | + + + +### netstat_recv_window + +Receive window size of a TCP connection + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| ZapiPerf | `perf-object-get-instances netstat` | `recv_window`
Unit: none
Type: raw
Base: | conf/zapiperf/cdot/9.8.0/netstat.yaml | + + + +### netstat_rexmit_pkts + +Number of packets retransmitted by this TCP connection + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| ZapiPerf | `perf-object-get-instances netstat` | `rexmit_pkts`
Unit: none
Type: raw
Base: | conf/zapiperf/cdot/9.8.0/netstat.yaml | + + + +### netstat_send_window + +Send window size of a TCP connection + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| ZapiPerf | `perf-object-get-instances netstat` | `send_window`
Unit: none
Type: raw
Base: | conf/zapiperf/cdot/9.8.0/netstat.yaml | + + + ### nfs_clients_idle_duration Specifies an ISO-8601 format of date and time to retrieve the idle time duration in hours, minutes, and seconds format. @@ -8466,6 +8635,44 @@ The `nfs_diag_storePool_LayoutStateMax` metric is visualized in the following Gr +### nfs_diag_storePool_LockAlloc + +Represent the current number of lock objects allocated + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| RestPerf | `NA` | `Harvest generated`
Unit:
Type:
Base: | conf/restperf/9.12.0/nfsv4_pool.yaml | + +The `nfs_diag_storePool_LockAlloc` metric is visualized in the following Grafana dashboards: + +/// html | div.grafana-table +| Dashboard | Row | Type | Panel | +|--------|----------|--------|--------| +| ONTAP: NFSv4 StorePool Monitors | Allocations over 50% | timeseries | [Allocations over 50%](/d/cdot-nfsv4-storepool-monitors/ontap3a-nfsv4 storepool monitors?orgId=1&viewPanel=53) | +| ONTAP: NFS Troubleshooting | Highlights | timeseries | [All nodes with 1% or more allocations in $Datacenter](/d/cdot-nfs-troubleshooting/ontap3a-nfs troubleshooting?orgId=1&viewPanel=2) | +/// + + + +### nfs_diag_storePool_LockMax + +Represent the maximum number of lock objects + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| RestPerf | `NA` | `Harvest generated`
Unit:
Type:
Base: | conf/restperf/9.12.0/nfsv4_pool.yaml | + +The `nfs_diag_storePool_LockMax` metric is visualized in the following Grafana dashboards: + +/// html | div.grafana-table +| Dashboard | Row | Type | Panel | +|--------|----------|--------|--------| +| ONTAP: NFSv4 StorePool Monitors | Allocations over 50% | timeseries | [Allocations over 50%](/d/cdot-nfsv4-storepool-monitors/ontap3a-nfsv4 storepool monitors?orgId=1&viewPanel=53) | +| ONTAP: NFS Troubleshooting | Highlights | timeseries | [All nodes with 1% or more allocations in $Datacenter](/d/cdot-nfs-troubleshooting/ontap3a-nfs troubleshooting?orgId=1&viewPanel=2) | +/// + + + ### nfs_diag_storePool_LockStateAlloc Current number of lock state objects allocated. @@ -15374,6 +15581,24 @@ Tracks the number of concurrent collectors running. +### poller_cpu_percent + +Tracks the percentage of cpu usage of concurrent collectors running. + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| REST | `NA` | `Harvest generated` | NA | + +The `poller_cpu_percent` metric is visualized in the following Grafana dashboards: + +/// html | div.grafana-table +| Dashboard | Row | Type | Panel | +|--------|----------|--------|--------| +| Harvest Metadata | Highlights | timeseries | [% CPU Used](/d/cdot-metadata/harvest metadata?orgId=1&viewPanel=183) | +/// + + + ### poller_memory Tracks the memory usage of the poller process, including Resident Set Size (RSS), swap memory, and Virtual Memory Size (VMS). @@ -16703,6 +16928,44 @@ The `rw_ctx_qos_rewinds` metric is visualized in the following Grafana dashboard +### security_account_activediruser + +Represent the Active directory user in security account + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| Rest | `NA` | `Harvest generated` | conf/rest/9.12.0/security_account.yaml | + +The `security_account_activediruser` metric is visualized in the following Grafana dashboards: + +/// html | div.grafana-table +| Dashboard | Row | Type | Panel | +|--------|----------|--------|--------| +| ONTAP: Security | Highlights | stat | [AD/LDAP](/d/cdot-security/ontap3a-security?orgId=1&viewPanel=192) | +| ONTAP: Security | Cluster Compliance | table | [Cluster Compliance](/d/cdot-security/ontap3a-security?orgId=1&viewPanel=219) | +/// + + + +### security_account_certificateuser + +Represent the Certificate user in security account + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| Rest | `NA` | `Harvest generated` | conf/rest/9.12.0/security_account.yaml | + +The `security_account_certificateuser` metric is visualized in the following Grafana dashboards: + +/// html | div.grafana-table +| Dashboard | Row | Type | Panel | +|--------|----------|--------|--------| +| ONTAP: Security | Highlights | stat | [Certificate](/d/cdot-security/ontap3a-security?orgId=1&viewPanel=190) | +| ONTAP: Security | Cluster Compliance | table | [Cluster Compliance](/d/cdot-security/ontap3a-security?orgId=1&viewPanel=219) | +/// + + + ### security_account_labels This metric provides information about SecurityAccount @@ -16724,6 +16987,63 @@ The `security_account_labels` metric is visualized in the following Grafana dash +### security_account_ldapuser + +Represent the LDAP user in security account + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| Rest | `NA` | `Harvest generated` | conf/rest/9.12.0/security_account.yaml | + +The `security_account_ldapuser` metric is visualized in the following Grafana dashboards: + +/// html | div.grafana-table +| Dashboard | Row | Type | Panel | +|--------|----------|--------|--------| +| ONTAP: Security | Highlights | stat | [AD/LDAP](/d/cdot-security/ontap3a-security?orgId=1&viewPanel=192) | +| ONTAP: Security | Cluster Compliance | table | [Cluster Compliance](/d/cdot-security/ontap3a-security?orgId=1&viewPanel=219) | +/// + + + +### security_account_localuser + +Represent the Local user in security account + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| Rest | `NA` | `Harvest generated` | conf/rest/9.12.0/security_account.yaml | + +The `security_account_localuser` metric is visualized in the following Grafana dashboards: + +/// html | div.grafana-table +| Dashboard | Row | Type | Panel | +|--------|----------|--------|--------| +| ONTAP: Security | Highlights | stat | [Local](/d/cdot-security/ontap3a-security?orgId=1&viewPanel=194) | +| ONTAP: Security | Cluster Compliance | table | [Cluster Compliance](/d/cdot-security/ontap3a-security?orgId=1&viewPanel=219) | +/// + + + +### security_account_samluser + +Represent the SAML user in security account + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| Rest | `NA` | `Harvest generated` | conf/rest/9.12.0/security_account.yaml | + +The `security_account_samluser` metric is visualized in the following Grafana dashboards: + +/// html | div.grafana-table +| Dashboard | Row | Type | Panel | +|--------|----------|--------|--------| +| ONTAP: Security | Highlights | stat | [SAML](/d/cdot-security/ontap3a-security?orgId=1&viewPanel=158) | +| ONTAP: Security | Cluster Compliance | table | [Cluster Compliance](/d/cdot-security/ontap3a-security?orgId=1&viewPanel=219) | +/// + + + ### security_audit_destination_port The destination port used to forward the message. @@ -16734,6 +17054,26 @@ The destination port used to forward the message. +### security_audit_destination_status + +Represent the security audit protocol in security audit destinations + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| Rest | `NA` | `Harvest generated` | conf/rest/9.12.0/security_audit_dest.yaml | + +The `security_audit_destination_status` metric is visualized in the following Grafana dashboards: + +/// html | div.grafana-table +| Dashboard | Row | Type | Panel | +|--------|----------|--------|--------| +| ONTAP: Security | Highlights | stat | [Cluster Compliant %](/d/cdot-security/ontap3a-security?orgId=1&viewPanel=214) | +| ONTAP: Security | Highlights | piechart | [Cluster Compliant](/d/cdot-security/ontap3a-security?orgId=1&viewPanel=215) | +| ONTAP: Security | Cluster Compliance | table | [Cluster Compliance](/d/cdot-security/ontap3a-security?orgId=1&viewPanel=219) | +/// + + + ### security_certificate_expiry_time @@ -18808,6 +19148,44 @@ The `svm_labels` metric is visualized in the following Grafana dashboards: +### svm_ldap_encrypted + +This metric indicates a LDAP session security has been sealed + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| REST | `NA` | `Harvest generated` | conf/rest/9.10.0/svm.yaml | +| ZAPI | `NA` | `Harvest generated` | conf/zapi/cdot/9.8.0/svm.yaml | + +The `svm_ldap_encrypted` metric is visualized in the following Grafana dashboards: + +/// html | div.grafana-table +| Dashboard | Row | Type | Panel | +|--------|----------|--------|--------| +| ONTAP: Security | SVM Compliance | table | [SVM Compliance](/d/cdot-security/ontap3a-security?orgId=1&viewPanel=225) | +/// + + + +### svm_ldap_signed + +This metric indicates a LDAP session security has been signed + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| REST | `NA` | `Harvest generated` | conf/rest/9.10.0/svm.yaml | +| ZAPI | `NA` | `Harvest generated` | conf/zapi/cdot/9.8.0/svm.yaml | + +The `svm_ldap_signed` metric is visualized in the following Grafana dashboards: + +/// html | div.grafana-table +| Dashboard | Row | Type | Panel | +|--------|----------|--------|--------| +| ONTAP: Security | SVM Compliance | table | [SVM Compliance](/d/cdot-security/ontap3a-security?orgId=1&viewPanel=225) | +/// + + + ### svm_new_status This metric indicates a value of 1 if the SVM state is online (indicating the SVM is operational) and a value of 0 for any other state. @@ -21714,6 +22092,24 @@ The `volume_analytics_dir_subdir_count` metric is visualized in the following Gr +### volume_arw_status + +Represent the cluster level ARW status + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| Rest | `NA` | `Harvest generated` | conf/rest/9.14.0/volume.yaml | + +The `volume_arw_status` metric is visualized in the following Grafana dashboards: + +/// html | div.grafana-table +| Dashboard | Row | Type | Panel | +|--------|----------|--------|--------| +| ONTAP: Security | Cluster Compliance | table | [Cluster Compliance](/d/cdot-security/ontap3a-security?orgId=1&viewPanel=219) | +/// + + + ### volume_autosize_grow_threshold_percent Used space threshold which triggers autogrow. When the size-used is greater than this percent of size-total, the volume will be grown. The computed value is rounded down. The default value of this element varies from 85% to 98%, depending on the volume size. It is an error for the grow threshold to be less than or equal to the shrink threshold. @@ -23734,6 +24130,24 @@ The `vscan_scanner_stats_pct_network_used` metric is visualized in the following +### vscan_server_disconnected + +Represent the disconnected vscan servers to the vscan pool + +| API | Endpoint | Metric | Template | +|--------|----------|--------|---------| +| Rest | `NA` | `Harvest generated` | conf/rest/9.12.0/vscan.yaml | + +The `vscan_server_disconnected` metric is visualized in the following Grafana dashboards: + +/// html | div.grafana-table +| Dashboard | Row | Type | Panel | +|--------|----------|--------|--------| +| ONTAP: Vscan | Vscan Server | table | [Disconnected Vscan Servers in Cluster](/d/cdot-vscan/ontap3a-vscan?orgId=1&viewPanel=599) | +/// + + + ### wafl_avg_msg_latency Average turnaround time for WAFL messages in milliseconds. diff --git a/grafana/dashboards/asar2/overview.json b/grafana/dashboards/asar2/overview.json index ae4027d5a..91df0f3cc 100644 --- a/grafana/dashboards/asar2/overview.json +++ b/grafana/dashboards/asar2/overview.json @@ -438,71 +438,6 @@ "title": "SCSI", "type": "stat" }, - { - "cacheTimeout": null, - "datasource": "${DS_PROMETHEUS}", - "description": "Number of nvme.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "noValue": "0", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "rgb(21, 118, 171)", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 4, - "x": 20, - "y": 1 - }, - "id": 8, - "links": [], - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "center", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "8.1.8", - "targets": [ - { - "expr": "count(nvme_labels{cluster=~\"$Cluster\",datacenter=~\"$Datacenter\"})", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "NVMe", - "type": "stat" - }, { "collapsed": true, "datasource": "${DS_PROMETHEUS}", diff --git a/grafana/dashboards/cmode/node.json b/grafana/dashboards/cmode/node.json index a860ab11e..1cd13c2a5 100644 --- a/grafana/dashboards/cmode/node.json +++ b/grafana/dashboards/cmode/node.json @@ -3920,13 +3920,6 @@ "legendFormat": "Established Sessions", "refId": "B" }, - { - "exemplar": false, - "expr": "sum(node_cifs_signed_sessions{cluster=~\"$Cluster\",datacenter=~\"$Datacenter\",node=~\"$Node\"})", - "interval": "", - "legendFormat": "Signed Sessions", - "refId": "C" - }, { "exemplar": false, "expr": "sum(node_cifs_open_files{cluster=~\"$Cluster\",datacenter=~\"$Datacenter\",node=~\"$Node\"})", diff --git a/integration/test/alert_rule_test.go b/integration/test/alert_rule_test.go index 03f286997..6e6ad5db5 100644 --- a/integration/test/alert_rule_test.go +++ b/integration/test/alert_rule_test.go @@ -3,7 +3,7 @@ package main import ( "fmt" "github.com/Netapp/harvest-automation/test/cmds" - "github.com/netapp/harvest/v2/cmd/tools/generate" + "github.com/netapp/harvest/v2/cmd/tools" "github.com/netapp/harvest/v2/pkg/template" "github.com/netapp/harvest/v2/pkg/tree" "github.com/netapp/harvest/v2/pkg/tree/node" @@ -31,9 +31,9 @@ var exceptionMetrics = []string{ func TestAlertRules(t *testing.T) { cmds.SkipIfMissing(t, cmds.Regression) - metrics, _ := generate.BuildMetrics("../..", "integration/test/harvest.yml", "dc1") + metrics, _ := tools.BuildMetrics("../..", "integration/test/harvest.yml", "dc1", nil, make(map[string]tools.PanelData)) for pluginMetric, pluginLabels := range pluginGeneratedMetric { - metrics[pluginMetric] = generate.Counter{Name: pluginMetric, Labels: pluginLabels} + metrics[pluginMetric] = tools.Counter{Name: pluginMetric, Labels: pluginLabels} } alertRules := GetAllAlertRules("../../container/prometheus/", "alert_rules.yml", false) diff --git a/mcp/metadata/ontap_metrics.json b/mcp/metadata/ontap_metrics.json index c494bdeb6..06a838ead 100644 --- a/mcp/metadata/ontap_metrics.json +++ b/mcp/metadata/ontap_metrics.json @@ -104,6 +104,8 @@ "aggr_write_data": "Performance metric for write I/O operations.", "aggr_write_latency": "Performance metric for write I/O operations.", "aggr_write_ops": "Performance metric for write I/O operations.", + "audit_log": "Captures the operations such as create, update, and delete attempts on volumes via REST or ONTAP CLI commands", + "change_log": "Detect and track changes related to the creation, modification, and deletion of an object of Node, SVM and Volume", "cifs_session_connection_count": "A counter used to track requests that are sent to the volumes to the node.", "cifs_session_idle_duration": "Specifies an ISO-8601 format of date and time used to retrieve the idle time duration in hours, minutes, and seconds format.", "cifs_session_labels": "This metric provides information about CIFSSession", @@ -180,6 +182,7 @@ "environment_sensor_power": "Power consumed by a node in Watts.", "environment_sensor_status": "This metric indicates a value of 1 if the sensor threshold state is normal (indicating the sensor is operating within normal parameters) and a value of 0 for any other state.", "environment_sensor_threshold_value": "Provides the sensor reading.", + "ethernet_switch_port_new_status": "Represent the status of the ethernet switch port", "ethernet_switch_port_receive_discards": "Total number of discarded packets.", "ethernet_switch_port_receive_errors": "Number of packet errors.", "ethernet_switch_port_receive_packets": "Total packet count.", @@ -264,6 +267,7 @@ "fcp_threshold_full": "Number of times the total number of outstanding commands on the port exceeds the threshold supported by this port.", "fcp_total_data": "Amount of FCP traffic to and from the storage system", "fcp_total_ops": "Total number of FCP operations", + "fcp_util_percent": "Represent the FCP utilization percentage", "fcp_write_data": "Amount of data written to the storage system", "fcp_write_ops": "Number of write operations", "fcvi_firmware_invalid_crc_count": "Firmware reported invalid CRC count", @@ -499,6 +503,14 @@ "net_port_mtu": "Maximum transmission unit, largest packet size on this network", "net_port_status": "This metric indicates a value of 1 if the port state is up and a value of 0 for any other state.", "net_route_labels": "This metric provides information about NetRoute", + "netstat_bytes_recvd": "Number of bytes received by a TCP connection", + "netstat_bytes_sent": "Number of bytes sent by a TCP connection", + "netstat_cong_win": "Congestion window of a TCP connection", + "netstat_cong_win_th": "Congestion window threshold of a TCP connection", + "netstat_ooorcv_pkts": "Number of out-of-order packets received by this TCP connection", + "netstat_recv_window": "Receive window size of a TCP connection", + "netstat_rexmit_pkts": "Number of packets retransmitted by this TCP connection", + "netstat_send_window": "Send window size of a TCP connection", "nfs_clients_idle_duration": "Specifies an ISO-8601 format of date and time to retrieve the idle time duration in hours, minutes, and seconds format.", "nfs_diag_storePool_ByteLockAlloc": "Current number of byte range lock objects allocated.", "nfs_diag_storePool_ByteLockMax": "Maximum number of byte range lock objects.", @@ -516,6 +528,8 @@ "nfs_diag_storePool_LayoutMax": "Maximum number of layout objects.", "nfs_diag_storePool_LayoutStateAlloc": "Current number of layout state objects allocated.", "nfs_diag_storePool_LayoutStateMax": "Maximum number of layout state objects.", + "nfs_diag_storePool_LockAlloc": "Represent the current number of lock objects allocated", + "nfs_diag_storePool_LockMax": "Represent the maximum number of lock objects", "nfs_diag_storePool_LockStateAlloc": "Current number of lock state objects allocated.", "nfs_diag_storePool_LockStateMax": "Maximum number of lock state objects.", "nfs_diag_storePool_OpenAlloc": "Current number of share objects allocated.", @@ -1000,6 +1014,7 @@ "plex_disk_user_write_latency": "Average latency per block in microseconds for user write operations. plex_disk_user_write_latency is [disk_user_write_latency](#disk_user_write_latency) aggregated by `plex`.", "plex_disk_user_writes": "Number of disk write operations initiated each second for storing data or metadata associated with user requests. plex_disk_user_writes is [disk_user_writes](#disk_user_writes) aggregated by `plex`.", "poller_concurrent_collectors": "Tracks the number of concurrent collectors running.", + "poller_cpu_percent": "Tracks the percentage of cpu usage of concurrent collectors running.", "poller_memory": "Tracks the memory usage of the poller process, including Resident Set Size (RSS), swap memory, and Virtual Memory Size (VMS).", "poller_memory_percent": "Indicates the percentage of memory used by the poller process relative to the total available memory.", "poller_status": "Indicates the operational status of the poller process, where 1 means operational and 0 means not operational.", @@ -1079,8 +1094,14 @@ "rw_ctx_nfs_rewinds": "Array of number of rewinds for NFS ops based on their reasons.", "rw_ctx_qos_flowcontrol": "The number of times QoS limiting has enabled stream flowcontrol.", "rw_ctx_qos_rewinds": "The number of restarts after a rewind because of QoS limiting.", + "security_account_activediruser": "Represent the Active directory user in security account", + "security_account_certificateuser": "Represent the Certificate user in security account", "security_account_labels": "This metric provides information about SecurityAccount", + "security_account_ldapuser": "Represent the LDAP user in security account", + "security_account_localuser": "Represent the Local user in security account", + "security_account_samluser": "Represent the SAML user in security account", "security_audit_destination_port": "The destination port used to forward the message.", + "security_audit_destination_status": "Represent the security audit protocol in security audit destinations", "security_certificate_labels": "This metric provides information about SecurityCert", "security_labels": "This metric provides information about Security", "security_login_labels": "This metric provides information about SecurityLogin", @@ -1195,6 +1216,8 @@ "svm_cifs_write_latency": "Average latency for CIFS write operations", "svm_cifs_write_ops": "Total number of CIFS write operations", "svm_labels": "This metric provides information about SVM", + "svm_ldap_encrypted": "This metric indicates a LDAP session security has been sealed", + "svm_ldap_signed": "This metric indicates a LDAP session security has been signed", "svm_new_status": "This metric indicates a value of 1 if the SVM state is online (indicating the SVM is operational) and a value of 0 for any other state.", "svm_nfs_access_avg_latency": "Average latency of Access procedure requests. The counter keeps track of the average response time of Access requests.", "svm_nfs_access_total": "Total number of Access procedure requests. It is the total number of access success and access error requests.", @@ -1388,6 +1411,7 @@ "volume_analytics_dir_bytes_used": "The actual number of bytes used on disk by this file.", "volume_analytics_dir_file_count": "Number of files in a directory.", "volume_analytics_dir_subdir_count": "Number of sub directories in a directory.", + "volume_arw_status": "Represent the cluster level ARW status", "volume_autosize_grow_threshold_percent": "Used space threshold which triggers autogrow. When the size-used is greater than this percent of size-total, the volume will be grown. The computed value is rounded down. The default value of this element varies from 85% to 98%, depending on the volume size. It is an error for the grow threshold to be less than or equal to the shrink threshold.", "volume_autosize_maximum_size": "The maximum size (in bytes) to which the volume would be grown automatically. The default value is 20% greater than the volume size. It is an error for the maximum volume size to be less than the current volume size. It is also an error for the maximum size to be less than or equal to the minimum size.", "volume_autosize_minimum_size": "Minimum size in bytes up to which the volume shrinks automatically. This size cannot be greater than or equal to the maximum size of volume.", @@ -1486,6 +1510,7 @@ "vscan_scanner_stats_pct_cpu_used": "Percentage CPU utilization on scanner calculated over the last 15 seconds.", "vscan_scanner_stats_pct_mem_used": "Percentage RAM utilization on scanner calculated over the last 15 seconds.", "vscan_scanner_stats_pct_network_used": "Percentage network utilization on scanner calculated for the last 15 seconds.", + "vscan_server_disconnected": "Represent the disconnected vscan servers to the vscan pool", "wafl_avg_msg_latency": "Average turnaround time for WAFL messages in milliseconds.", "wafl_avg_non_wafl_msg_latency": "Average turnaround time for non-WAFL messages in milliseconds.", "wafl_avg_repl_msg_latency": "Average turnaround time for replication WAFL messages in milliseconds.", diff --git a/mcp/metadata/storagegrid_metrics.json b/mcp/metadata/storagegrid_metrics.json index 9ee751dfa..76f1ecae4 100644 --- a/mcp/metadata/storagegrid_metrics.json +++ b/mcp/metadata/storagegrid_metrics.json @@ -1,4 +1,8 @@ { + "bucket_bytes": "Represent the bytes read/write in the bucket", + "bucket_labels": "Represent the detail of bucket", + "bucket_objects": "Represent the number of object count in the bucket", + "bucket_quota_bytes": "Represent the bytes read/write in quota of the bucket", "storagegrid_content_buckets_and_containers": "Total number of S3 buckets and Swift containers", "storagegrid_content_objects": "Total number of S3 and Swift objects (excluding empty objects)", "storagegrid_ilm_awaiting_client_objects": "Total number of objects on this node awaiting ILM evaluation because of client operation (for example, ingest)", @@ -21,5 +25,10 @@ "storagegrid_storage_utilization_total_space_bytes": "Total storage space available in bytes", "storagegrid_storage_utilization_usable_space_bytes": "The total amount of object storage space remaining", "storagegrid_tenant_usage_data_bytes": "The logical size of all objects for the tenant", - "storagegrid_tenant_usage_quota_bytes": "The maximum amount of logical space available for the tenant's object. If a quota metric is not provided, an unlimited amount of space is available" + "storagegrid_tenant_usage_quota_bytes": "The maximum amount of logical space available for the tenant's object. If a quota metric is not provided, an unlimited amount of space is available", + "tenant_labels": "Represent the detail of tenant", + "tenant_logical_quota": "Represent the logical quota size in tenant", + "tenant_logical_used": "Represent the logical data used in tenant", + "tenant_objects": "Represent the number of object count in the tenant", + "tenant_used_percent": "Represent the data used percent in tenant" }