Skip to content

feat: add metrics for obmysql full collect #3197

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
BugsGuru merged 3 commits into from
Jan 5, 2026
Merged

feat: add metrics for obmysql full collect #3197

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e443049
feat: add metrics for obmysql full collect
littleniannian Jan 5, 2026
88b75c3
fix: update ApMetricClientIP to ApMetricUserClientIP
littleniannian Jan 5, 2026
d4fde64
fix: change MetricNameUserClientIp to MetricNameUserClientIP
littleniannian Jan 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions sqle/locale/active.en.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ ApMetaMySQLProcesslist = "Processlist"
ApMetaMySQLSchemaMeta = "Database schema metadata"
ApMetaMySQLTopSQL = "MySQL TOP SQL"
ApMetaObForOracleTopSQL = "OceanBase For Oracle TOP SQL"
ApMetaOceanBaseForMySQLFullCollect = "Full Collect"
ApMetaOracleTopSQL = "Oracle TOP SQL"
ApMetaPerformanceCollect = "Performance metrics"
ApMetaPerformanceCollectTips = "Performance metric collection will incur significant performance overhead, so please enable it with caution. Once enabled, the system will continuously collect performance data (such as QPS, number of connections, etc.) for this data source and generate performance trend charts, which will be displayed on the Performance Insights page."
Expand All @@ -48,6 +49,7 @@ ApMetricNameBufferReadAvg = "Average logical read count"
ApMetricNameCPUTimeAvg = "Average CPU time (μs)"
ApMetricNameCPUTimeTotal = "Total CPU time (s)"
ApMetricNameChecksum = "checksum"
ApMetricUserClientIP = "Client IP"
ApMetricNameCounter = "Execution count"
ApMetricNameCounterMoreThan = "Occurrence count > "
ApMetricNameDBUser = "User"
Expand Down Expand Up @@ -87,9 +89,11 @@ ApMetricNameQueryTimeTotal = "Total execution time(s)"
ApMetricNameQueryTimeTotalMS = "Total execution time(ms)"
ApMetricNameRowExaminedAvg = "Average examined rows"
ApMetricNameRowExaminedAvgMoreThan = "Average examined rows > "
ApMetricNameRequestTime = "Request Time"
ApMetricNameRowsAffectedAvg = "Average affected rows"
ApMetricNameRowsAffectedMax = "Max affected rows"
ApMetricNameTable = "table name"
ApMetricNameTenantName = "Tenant Name"
ApMetricNameTransactionStarted = "transaction started"
ApMetricNameTrxWaitStarted = "transaction wait started"
ApMetricNameUserIOWaitTimeTotal = "I/O wait time (s)"
Expand Down
4 changes: 4 additions & 0 deletions sqle/locale/active.zh.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ ApMetaMySQLProcesslist = "processlist 列表"
ApMetaMySQLSchemaMeta = "库表元数据"
ApMetaMySQLTopSQL = "MySQL TOP SQL"
ApMetaObForOracleTopSQL = "OceanBase For Oracle TOP SQL"
ApMetaOceanBaseForMySQLFullCollect = "全量采集"
ApMetaOracleTopSQL = "Oracle TOP SQL"
ApMetaPerformanceCollect = "数据源性能指标"
ApMetaPerformanceCollectTips = "性能指标采集将产生较大性能开销,请谨慎开启。开启后,系统将持续采集该数据源的性能数据(如QPS、连接数等)，并生成性能趋势图表，体现在性能洞察页面。"
Expand All @@ -48,6 +49,7 @@ ApMetricNameBufferReadAvg = "平均逻辑读次数"
ApMetricNameCPUTimeAvg = "平均 CPU 时间(μs)"
ApMetricNameCPUTimeTotal = "CPU时间占用(s)"
ApMetricNameChecksum = "校验和"
ApMetricUserClientIP = "客户端IP"
ApMetricNameCounter = "执行次数"
ApMetricNameCounterMoreThan = "出现次数 > "
ApMetricNameDBUser = "用户"
Expand Down Expand Up @@ -87,9 +89,11 @@ ApMetricNameQueryTimeTotal = "总执行时间(s)"
ApMetricNameQueryTimeTotalMS = "总执行时间(ms)"
ApMetricNameRowExaminedAvg = "平均扫描行数"
ApMetricNameRowExaminedAvgMoreThan = "平均扫描行数 > "
ApMetricNameRequestTime = "请求时间"
ApMetricNameRowsAffectedAvg = "平均影响行数"
ApMetricNameRowsAffectedMax = "最大影响行数"
ApMetricNameTable = "表名"
ApMetricNameTenantName = "租户名称"
ApMetricNameTransactionStarted = "持有锁事务开始时间"
ApMetricNameTrxWaitStarted = "等待锁事务开始时间"
ApMetricNameUserIOWaitTimeTotal = "I/O等待时间(s)"
Expand Down
72 changes: 38 additions & 34 deletions sqle/locale/message_zh.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -322,6 +322,7 @@ var (
ApMetricNameGrantedLockSql = &i18n.Message{ID: "ApMetricNameGrantedLockSql", Other: "持有锁SQL"}
ApMetricNameWaitingLockSql = &i18n.Message{ID: "ApMetricNameWaitingLockSql", Other: "等待锁SQL"}
ApMetricNameDBUser = &i18n.Message{ID: "ApMetricNameDBUser", Other: "用户"}
ApMetricUserClientIP = &i18n.Message{ID: "ApMetricUserClientIP", Other: "客户端IP"}
ApMetricNameHost = &i18n.Message{ID: "ApMetricNameHost", Other: "主机"}
ApMetricNameMetaName = &i18n.Message{ID: "ApMetricNameMetaName", Other: "对象名称"}
ApMetricNameMetaType = &i18n.Message{ID: "ApMetricNameMetaType", Other: "对象类型"}
Expand Down Expand Up @@ -362,40 +363,43 @@ var (
ApMetricNameQueryTimeAvgMoreThan = &i18n.Message{ID: "ApMetricNameQueryTimeAvgMoreThan", Other: "平均执行时间(s) > "}
ApMetricNameRowExaminedAvgMoreThan = &i18n.Message{ID: "ApMetricNameRowExaminedAvgMoreThan", Other: "平均扫描行数 > "}

ApMetricNameInstance = &i18n.Message{ID: "ApMetricNameInstance", Other: "节点地址"}
ApMetricNameMemMax = &i18n.Message{ID: "ApMetricNameMemMax", Other: "使用的最大内存空间"}
ApMetricNameDiskMax = &i18n.Message{ID: "ApMetricNameDiskMax", Other: "使用的最大硬盘空间"}

ApMetaCustom = &i18n.Message{ID: "ApMetaCustom", Other: "自定义"}
ApMetaMySQLSchemaMeta = &i18n.Message{ID: "ApMetaMySQLSchemaMeta", Other: "库表元数据"}
ApMetaMySQLProcesslist = &i18n.Message{ID: "ApMetaMySQLProcesslist", Other: "processlist 列表"}
ApMetaAliRdsMySQLSlowLog = &i18n.Message{ID: "ApMetaAliRdsMySQLSlowLog", Other: "阿里RDS MySQL慢日志"}
ApMetaAliRdsMySQLAuditLog = &i18n.Message{ID: "ApMetaAliRdsMySQLAuditLog", Other: "阿里RDS MySQL审计日志"}
ApMetaBaiduRdsMySQLSlowLog = &i18n.Message{ID: "ApMetaBaiduRdsMySQLSlowLog", Other: "百度云RDS MySQL慢日志"}
ApMetaHuaweiRdsMySQLSlowLog = &i18n.Message{ID: "ApMetaHuaweiRdsMySQLSlowLog", Other: "华为云RDS MySQL慢日志"}
ApMetaOracleTopSQL = &i18n.Message{ID: "ApMetaOracleTopSQL", Other: "Oracle TOP SQL"}
ApMetaAllAppExtract = &i18n.Message{ID: "ApMetaAllAppExtract", Other: "应用程序SQL抓取"}
ApMetaTiDBAuditLog = &i18n.Message{ID: "ApMetaTiDBAuditLog", Other: "TiDB审计日志"}
ApMetaSlowLog = &i18n.Message{ID: "ApMetaSlowLog", Other: "慢日志"}
ApMetaMDBSlowLog = &i18n.Message{ID: "ApMetaMDBSlowLog", Other: "慢日志（监控库）"}
ApMetaTopSQL = &i18n.Message{ID: "ApMetaTopSQL", Other: "Top SQL"}
ApMetaDB2TopSQL = &i18n.Message{ID: "ApMetaDB2TopSQL", Other: "DB2 Top SQL"}
ApMetaSchemaMeta = &i18n.Message{ID: "ApMetaSchemaMeta", Other: "库表元数据"}
ApMetaDistributedLock = &i18n.Message{ID: "ApMetaDistributedLock", Other: "分布式锁"}
ApMetaDmTopSQL = &i18n.Message{ID: "ApMetaDmTopSQL", Other: "DM TOP SQL"}
ApMetaObForOracleTopSQL = &i18n.Message{ID: "ApMetaObForOracleTopSQL", Other: "OceanBase For Oracle TOP SQL"}
ApMetaPostgreSQLTopSQL = &i18n.Message{ID: "ApMetaPostgreSQLTopSQL", Other: "TOP SQL"}
ApMetaGoldenDBTopSQL = &i18n.Message{ID: "ApMetaGoldenDBTopSQL", Other: "GoldenDB TOP SQL"}
ApMetaTiDBTopSQL = &i18n.Message{ID: "ApMetaTiDBTopSQL", Other: "TiDB TOP SQL"}
ApMetaMySQLTopSQL = &i18n.Message{ID: "ApMetaMySQLTopSQL", Other: "MySQL TOP SQL"}
ApMetricQueryTimeAvg = &i18n.Message{ID: "ApMetricQueryTimeAvg", Other: "平均查询时间(s)"}
ApMetricRowExaminedAvg = &i18n.Message{ID: "ApMetricRowExaminedAvg", Other: "平均扫描行数"}
ApMetaPerformanceCollect = &i18n.Message{ID: "ApMetaPerformanceCollect", Other: "数据源性能指标"}
ApMetaPerformanceCollectTips = &i18n.Message{ID: "ApMetaPerformanceCollectTips", Other: "性能指标采集将产生较大性能开销,请谨慎开启。开启后,系统将持续采集该数据源的性能数据(如QPS、连接数等)，并生成性能趋势图表，体现在性能洞察页面。"}
ApMetaCollectTime = &i18n.Message{ID: "ApMetaCollectTime", Other: "采集时间"}
ApMetaThreadsConnected = &i18n.Message{ID: "ApMetaThreadsConnected", Other: "线程数"}
ApMetaQPS = &i18n.Message{ID: "ApMetaQueries", Other: "QPS"}
ApMetricNameFullTableScanCount = &i18n.Message{ID: "ApMetricNameFullTableScanCount", Other: "全表扫描次数"}
ApMetricNameInstance = &i18n.Message{ID: "ApMetricNameInstance", Other: "节点地址"}
ApMetricNameMemMax = &i18n.Message{ID: "ApMetricNameMemMax", Other: "使用的最大内存空间"}
ApMetricNameDiskMax = &i18n.Message{ID: "ApMetricNameDiskMax", Other: "使用的最大硬盘空间"}
ApMetricNameTenantName = &i18n.Message{ID: "ApMetricNameTenantName", Other: "租户名称"}
ApMetricNameRequestTime = &i18n.Message{ID: "ApMetricNameRequestTime", Other: "请求时间"}

ApMetaCustom = &i18n.Message{ID: "ApMetaCustom", Other: "自定义"}
ApMetaMySQLSchemaMeta = &i18n.Message{ID: "ApMetaMySQLSchemaMeta", Other: "库表元数据"}
ApMetaMySQLProcesslist = &i18n.Message{ID: "ApMetaMySQLProcesslist", Other: "processlist 列表"}
ApMetaAliRdsMySQLSlowLog = &i18n.Message{ID: "ApMetaAliRdsMySQLSlowLog", Other: "阿里RDS MySQL慢日志"}
ApMetaAliRdsMySQLAuditLog = &i18n.Message{ID: "ApMetaAliRdsMySQLAuditLog", Other: "阿里RDS MySQL审计日志"}
ApMetaBaiduRdsMySQLSlowLog = &i18n.Message{ID: "ApMetaBaiduRdsMySQLSlowLog", Other: "百度云RDS MySQL慢日志"}
ApMetaHuaweiRdsMySQLSlowLog = &i18n.Message{ID: "ApMetaHuaweiRdsMySQLSlowLog", Other: "华为云RDS MySQL慢日志"}
ApMetaOracleTopSQL = &i18n.Message{ID: "ApMetaOracleTopSQL", Other: "Oracle TOP SQL"}
ApMetaAllAppExtract = &i18n.Message{ID: "ApMetaAllAppExtract", Other: "应用程序SQL抓取"}
ApMetaTiDBAuditLog = &i18n.Message{ID: "ApMetaTiDBAuditLog", Other: "TiDB审计日志"}
ApMetaSlowLog = &i18n.Message{ID: "ApMetaSlowLog", Other: "慢日志"}
ApMetaMDBSlowLog = &i18n.Message{ID: "ApMetaMDBSlowLog", Other: "慢日志（监控库）"}
ApMetaTopSQL = &i18n.Message{ID: "ApMetaTopSQL", Other: "Top SQL"}
ApMetaDB2TopSQL = &i18n.Message{ID: "ApMetaDB2TopSQL", Other: "DB2 Top SQL"}
ApMetaSchemaMeta = &i18n.Message{ID: "ApMetaSchemaMeta", Other: "库表元数据"}
ApMetaDistributedLock = &i18n.Message{ID: "ApMetaDistributedLock", Other: "分布式锁"}
ApMetaDmTopSQL = &i18n.Message{ID: "ApMetaDmTopSQL", Other: "DM TOP SQL"}
ApMetaObForOracleTopSQL = &i18n.Message{ID: "ApMetaObForOracleTopSQL", Other: "OceanBase For Oracle TOP SQL"}
ApMetaOceanBaseForMySQLFullCollect = &i18n.Message{ID: "ApMetaOceanBaseForMySQLFullCollect", Other: "全量采集"}
ApMetaPostgreSQLTopSQL = &i18n.Message{ID: "ApMetaPostgreSQLTopSQL", Other: "TOP SQL"}
ApMetaGoldenDBTopSQL = &i18n.Message{ID: "ApMetaGoldenDBTopSQL", Other: "GoldenDB TOP SQL"}
ApMetaTiDBTopSQL = &i18n.Message{ID: "ApMetaTiDBTopSQL", Other: "TiDB TOP SQL"}
ApMetaMySQLTopSQL = &i18n.Message{ID: "ApMetaMySQLTopSQL", Other: "MySQL TOP SQL"}
ApMetricQueryTimeAvg = &i18n.Message{ID: "ApMetricQueryTimeAvg", Other: "平均查询时间(s)"}
ApMetricRowExaminedAvg = &i18n.Message{ID: "ApMetricRowExaminedAvg", Other: "平均扫描行数"}
ApMetaPerformanceCollect = &i18n.Message{ID: "ApMetaPerformanceCollect", Other: "数据源性能指标"}
ApMetaPerformanceCollectTips = &i18n.Message{ID: "ApMetaPerformanceCollectTips", Other: "性能指标采集将产生较大性能开销,请谨慎开启。开启后,系统将持续采集该数据源的性能数据(如QPS、连接数等)，并生成性能趋势图表，体现在性能洞察页面。"}
ApMetaCollectTime = &i18n.Message{ID: "ApMetaCollectTime", Other: "采集时间"}
ApMetaThreadsConnected = &i18n.Message{ID: "ApMetaThreadsConnected", Other: "线程数"}
ApMetaQPS = &i18n.Message{ID: "ApMetaQueries", Other: "QPS"}
ApMetricNameFullTableScanCount = &i18n.Message{ID: "ApMetricNameFullTableScanCount", Other: "全表扫描次数"}

ApPriorityHigh = &i18n.Message{ID: "ApPriorityHigh", Other: "高优先级"}

Expand Down
12 changes: 12 additions & 0 deletions sqle/model/instance_audit_plan.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,18 @@ func (s *Storage) GetLatestStartTimeAuditPlanSQLV2(sourceId uint, typ string) (s
return info.StartTime, err
}

// HasSQLManageRecords 检查是否存在指定 source_id 和 source 的 SQL 记录
func (s *Storage) HasSQLManageRecords(sourceId string, source string) (bool, error) {
info := struct {
Count int64 `gorm:"column:cnt"`
}{}
err := s.db.Raw(`SELECT COUNT(*) AS cnt FROM sql_manage_records WHERE source_id = ? AND source = ? AND deleted_at IS NULL`, sourceId, source).Scan(&info).Error
if err != nil {
return false, err
}
return info.Count > 0, nil
}

// 此表对于来源是扫描任务的相关sql, 目前仅在采集和审核时会更新, 如有其他场景更新此表, 需要考虑更新后会触发审核影响
// 如有其他sql业务相关字段补充, 可新增至SQLManageRecordProcess中
type SQLManageRecord struct {
Expand Down
9 changes: 9 additions & 0 deletions sqle/server/auditplan/metrics.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,10 @@ const MetricNameMemMax = "mem_max"
const MetricNameDiskMax = "disk_max"
const MetricNameFullTableScanCount = "full_table_scan_count"

const MetricNameUserClientIP = "user_client_ip"
const MetricNameTenantName = "tenant_name"
const MetricNameRequestTime = "request_time"

var ALLMetric = map[string]MetricType{
MetricNameCounter: MetricTypeInt, // MySQL slow log
MetricNameLastReceiveTimestamp: MetricTypeString, // MySQL slow log
Expand All @@ -92,6 +96,7 @@ var ALLMetric = map[string]MetricType{
MetricNameMetaName: MetricTypeString, // MySQL schema meta
MetricNameMetaType: MetricTypeString, // MySQL schema meta
MetricNameRecordDeleted: MetricTypeBool, // MySQL schema meta
MetricNameHost: MetricTypeString, // OB MySQL Full Collect

MetricNameQueryTimeTotal: MetricTypeFloat, // DB2 TOP SQL | OB Oracle TOP SQL
MetricNameCPUTimeAvg: MetricTypeFloat, // DB2 TOP SQL | OB MySQL TOP SQL
Expand Down Expand Up @@ -131,6 +136,10 @@ var ALLMetric = map[string]MetricType{
MetricNameDiskMax: MetricTypeInt,
MetricNameInstance: MetricTypeString,
MetricNameFullTableScanCount: MetricTypeInt,

MetricNameUserClientIP: MetricTypeString,
MetricNameTenantName: MetricTypeString,
MetricNameRequestTime: MetricTypeString,
}

func LoadMetrics(info map[string]interface{}, metrics []string) Metrics {
Expand Down
Loading