Skip to content

Commit 546668b

Browse files
lukasz-steclukasz-stec
committed
Add Metadata.getMetrics
The goal is to expose in QueryStats, per catalog, connector-specific metrics like metastore api call stats.
1 parent 40dca75 commit 546668b

File tree

21 files changed

+168
-9
lines changed

21 files changed

+168
-9
lines changed

‎client/trino-jdbc/src/test/java/io/trino/jdbc/TestTrinoDatabaseMetaData.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2204,8 +2204,9 @@ private void assertMetadataCalls(
22042204
});
22052205

22062206
actualMetadataCallsCount = actualMetadataCallsCount.stream()
2207-
// Every query involves beginQuery and cleanupQuery, so ignore them.
2208-
.filter(method -> !"ConnectorMetadata.beginQuery".equals(method) && !"ConnectorMetadata.cleanupQuery".equals(method))
2207+
// Every query involves beginQuery, cleanupQuery and getMetrics, so ignore them.
2208+
.filter(method -> !"ConnectorMetadata.beginQuery".equals(method) && !"ConnectorMetadata.cleanupQuery".equals(method)
2209+
&& !"ConnectorMetadata.getMetrics".equals(method))
22092210
.collect(toImmutableMultiset());
22102211

22112212
assertMultisetsEqual(actualMetadataCallsCount, expectedMetadataCallsCount);

‎core/trino-main/src/main/java/io/trino/dispatcher/FailedDispatchQuery.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -336,6 +336,7 @@ private static QueryStats immediateFailureQueryStats()
336336
DataSize.ofBytes(0),
337337
ImmutableList.of(),
338338
DynamicFiltersStats.EMPTY,
339+
ImmutableMap.of(),
339340
ImmutableList.of(),
340341
ImmutableList.of());
341342
}

‎core/trino-main/src/main/java/io/trino/execution/QueryStateMachine.java‎

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
import io.trino.execution.StateMachine.StateChangeListener;
3737
import io.trino.execution.querystats.PlanOptimizersStatsCollector;
3838
import io.trino.execution.warnings.WarningCollector;
39+
import io.trino.metadata.CatalogInfo;
3940
import io.trino.metadata.Metadata;
4041
import io.trino.operator.BlockedReason;
4142
import io.trino.operator.OperatorStats;
@@ -182,6 +183,7 @@ public class QueryStateMachine
182183
private final AtomicReference<Optional<Output>> output = new AtomicReference<>(Optional.empty());
183184
private final AtomicReference<List<TableInfo>> referencedTables = new AtomicReference<>(ImmutableList.of());
184185
private final AtomicReference<List<RoutineInfo>> routines = new AtomicReference<>(ImmutableList.of());
186+
private final AtomicReference<Map<String, Metrics>> catalogMetadataMetrics = new AtomicReference<>(ImmutableMap.of());
185187
private final StateMachine<Optional<QueryInfo>> finalQueryInfo;
186188

187189
private final WarningCollector warningCollector;
@@ -386,6 +388,26 @@ static QueryStateMachine beginWithTicker(
386388
return queryStateMachine;
387389
}
388390

391+
private void collectCatalogMetadataMetrics()
392+
{
393+
if (session.getTransactionId().filter(transactionManager::transactionExists).isEmpty()) {
394+
// The metrics collection depends on active transaction as the metrics
395+
// are stored in the transactional ConnectorMetadata, but the collection can be
396+
// run after the query has failed e.g., via cancel.
397+
return;
398+
}
399+
400+
ImmutableMap.Builder<String, Metrics> catalogMetadataMetrics = ImmutableMap.builder();
401+
for (CatalogInfo activeCatalog : metadata.listActiveCatalogs(session)) {
402+
Metrics metrics = metadata.getMetrics(session, activeCatalog.catalogName());
403+
if (!metrics.getMetrics().isEmpty()) {
404+
catalogMetadataMetrics.put(activeCatalog.catalogName(), metrics);
405+
}
406+
}
407+
408+
this.catalogMetadataMetrics.set(catalogMetadataMetrics.buildOrThrow());
409+
}
410+
389411
public QueryId getQueryId()
390412
{
391413
return queryId;
@@ -936,7 +958,7 @@ private QueryStats getQueryStats(Optional<StagesInfo> stages)
936958
stageGcStatistics.build(),
937959

938960
getDynamicFiltersStats(),
939-
961+
catalogMetadataMetrics.get(),
940962
operatorStatsSummary.build(),
941963
planOptimizersStatsCollector.getTopRuleStats());
942964
}
@@ -1156,6 +1178,7 @@ public boolean transitionToFinishing()
11561178
{
11571179
queryStateTimer.beginFinishing();
11581180

1181+
collectCatalogMetadataMetrics();
11591182
if (!queryState.setIf(FINISHING, currentState -> currentState != FINISHING && !currentState.isDone())) {
11601183
return false;
11611184
}
@@ -1225,10 +1248,11 @@ public boolean transitionToFailed(Throwable throwable)
12251248
return transitionToFailed(throwable, true);
12261249
}
12271250

1228-
private boolean transitionToFailed(Throwable throwable, boolean log)
1251+
private synchronized boolean transitionToFailed(Throwable throwable, boolean log)
12291252
{
12301253
queryStateTimer.endQuery();
12311254

1255+
collectCatalogMetadataMetrics();
12321256
// NOTE: The failure cause must be set before triggering the state change, so
12331257
// listeners can observe the exception. This is safe because the failure cause
12341258
// can only be observed if the transition to FAILED is successful.
@@ -1549,6 +1573,7 @@ private static QueryStats pruneQueryStats(QueryStats queryStats)
15491573
queryStats.getFailedPhysicalWrittenDataSize(),
15501574
queryStats.getStageGcStatistics(),
15511575
queryStats.getDynamicFiltersStats(),
1576+
ImmutableMap.of(),
15521577
ImmutableList.of(), // Remove the operator summaries as OperatorInfo (especially DirectExchangeClientStatus) can hold onto a large amount of memory
15531578
ImmutableList.of());
15541579
}

‎core/trino-main/src/main/java/io/trino/execution/QueryStats.java‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,10 +24,12 @@
2424
import io.trino.operator.TableWriterOperator;
2525
import io.trino.spi.eventlistener.QueryPlanOptimizerStatistics;
2626
import io.trino.spi.eventlistener.StageGcStatistics;
27+
import io.trino.spi.metrics.Metrics;
2728
import jakarta.annotation.Nullable;
2829

2930
import java.time.Instant;
3031
import java.util.List;
32+
import java.util.Map;
3133
import java.util.OptionalDouble;
3234
import java.util.Set;
3335

@@ -127,6 +129,7 @@ public class QueryStats
127129

128130
private final DynamicFiltersStats dynamicFiltersStats;
129131

132+
private final Map<String, Metrics> catalogMetadataMetrics;
130133
private final List<OperatorStats> operatorSummaries;
131134
private final List<QueryPlanOptimizerStatistics> optimizerRulesSummaries;
132135

@@ -218,7 +221,7 @@ public QueryStats(
218221
@JsonProperty("stageGcStatistics") List<StageGcStatistics> stageGcStatistics,
219222

220223
@JsonProperty("dynamicFiltersStats") DynamicFiltersStats dynamicFiltersStats,
221-
224+
@JsonProperty("catalogMetadataMetrics") Map<String, Metrics> catalogMetadataMetrics,
222225
@JsonProperty("operatorSummaries") List<OperatorStats> operatorSummaries,
223226
@JsonProperty("optimizerRulesSummaries") List<QueryPlanOptimizerStatistics> optimizerRulesSummaries)
224227
{
@@ -324,6 +327,7 @@ public QueryStats(
324327

325328
this.dynamicFiltersStats = requireNonNull(dynamicFiltersStats, "dynamicFiltersStats is null");
326329

330+
this.catalogMetadataMetrics = requireNonNull(catalogMetadataMetrics, "catalogMetadataMetrics is null");
327331
requireNonNull(operatorSummaries, "operatorSummaries is null");
328332
this.operatorSummaries = operatorSummaries.stream().map(OperatorStats::pruneDigests).collect(toImmutableList());
329333

@@ -770,6 +774,12 @@ public DynamicFiltersStats getDynamicFiltersStats()
770774
return dynamicFiltersStats;
771775
}
772776

777+
@JsonProperty
778+
public Map<String, Metrics> getCatalogMetadataMetrics()
779+
{
780+
return catalogMetadataMetrics;
781+
}
782+
773783
@JsonProperty
774784
public List<OperatorStats> getOperatorSummaries()
775785
{

‎core/trino-main/src/main/java/io/trino/metadata/Metadata.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@
6767
import io.trino.spi.function.FunctionMetadata;
6868
import io.trino.spi.function.LanguageFunction;
6969
import io.trino.spi.function.OperatorType;
70+
import io.trino.spi.metrics.Metrics;
7071
import io.trino.spi.predicate.TupleDomain;
7172
import io.trino.spi.security.FunctionAuthorization;
7273
import io.trino.spi.security.GrantInfo;
@@ -147,6 +148,11 @@ Optional<TableExecuteHandle> getTableHandleForExecute(
147148

148149
Optional<Object> getInfo(Session session, TableHandle handle);
149150

151+
/**
152+
* Return connector-specific, metadata operations metrics for the given session.
153+
*/
154+
Metrics getMetrics(Session session, String catalogName);
155+
150156
CatalogSchemaTableName getTableName(Session session, TableHandle tableHandle);
151157

152158
/**
@@ -483,6 +489,8 @@ Optional<ConnectorOutputMetadata> finishRefreshMaterializedView(
483489
*/
484490
List<CatalogInfo> listCatalogs(Session session);
485491

492+
List<CatalogInfo> listActiveCatalogs(Session session);
493+
486494
/**
487495
* Get the names that match the specified table prefix (never null).
488496
*/

‎core/trino-main/src/main/java/io/trino/metadata/MetadataManager.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,7 @@
104104
import io.trino.spi.function.OperatorType;
105105
import io.trino.spi.function.SchemaFunctionName;
106106
import io.trino.spi.function.Signature;
107+
import io.trino.spi.metrics.Metrics;
107108
import io.trino.spi.predicate.TupleDomain;
108109
import io.trino.spi.security.FunctionAuthorization;
109110
import io.trino.spi.security.GrantInfo;
@@ -456,6 +457,14 @@ public Optional<Object> getInfo(Session session, TableHandle handle)
456457
return metadata.getInfo(connectorSession, handle.connectorHandle());
457458
}
458459

460+
@Override
461+
public Metrics getMetrics(Session session, String catalogName)
462+
{
463+
return transactionManager.getRequiredCatalogMetadata(session.getRequiredTransactionId(), catalogName)
464+
.getMetadata(session)
465+
.getMetrics(session.toConnectorSession());
466+
}
467+
459468
@Override
460469
public CatalogSchemaTableName getTableName(Session session, TableHandle tableHandle)
461470
{
@@ -1386,6 +1395,12 @@ public List<CatalogInfo> listCatalogs(Session session)
13861395
return transactionManager.getCatalogs(session.getRequiredTransactionId());
13871396
}
13881397

1398+
@Override
1399+
public List<CatalogInfo> listActiveCatalogs(Session session)
1400+
{
1401+
return transactionManager.getActiveCatalogs(session.getRequiredTransactionId());
1402+
}
1403+
13891404
@Override
13901405
public List<QualifiedObjectName> listViews(Session session, QualifiedTablePrefix prefix)
13911406
{

‎core/trino-main/src/main/java/io/trino/tracing/TracingConnectorMetadata.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,7 @@
7979
import io.trino.spi.function.LanguageFunction;
8080
import io.trino.spi.function.SchemaFunctionName;
8181
import io.trino.spi.function.table.ConnectorTableFunctionHandle;
82+
import io.trino.spi.metrics.Metrics;
8283
import io.trino.spi.predicate.TupleDomain;
8384
import io.trino.spi.security.GrantInfo;
8485
import io.trino.spi.security.Privilege;
@@ -252,6 +253,15 @@ public Optional<Object> getInfo(ConnectorSession session, ConnectorTableHandle t
252253
}
253254
}
254255

256+
@Override
257+
public Metrics getMetrics(ConnectorSession session)
258+
{
259+
Span span = startSpan("getMetrics");
260+
try (var _ = scopedSpan(span)) {
261+
return delegate.getMetrics(session);
262+
}
263+
}
264+
255265
@Override
256266
public List<SchemaTableName> listTables(ConnectorSession session, Optional<String> schemaName)
257267
{

‎core/trino-main/src/main/java/io/trino/tracing/TracingMetadata.java‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,7 @@
9797
import io.trino.spi.function.FunctionMetadata;
9898
import io.trino.spi.function.LanguageFunction;
9999
import io.trino.spi.function.OperatorType;
100+
import io.trino.spi.metrics.Metrics;
100101
import io.trino.spi.predicate.TupleDomain;
101102
import io.trino.spi.security.FunctionAuthorization;
102103
import io.trino.spi.security.GrantInfo;
@@ -284,6 +285,15 @@ public Optional<Object> getInfo(Session session, TableHandle handle)
284285
}
285286
}
286287

288+
@Override
289+
public Metrics getMetrics(Session session, String catalogName)
290+
{
291+
Span span = startSpan("getMetrics", catalogName);
292+
try (var _ = scopedSpan(span)) {
293+
return delegate.getMetrics(session, catalogName);
294+
}
295+
}
296+
287297
@Override
288298
public CatalogSchemaTableName getTableName(Session session, TableHandle tableHandle)
289299
{
@@ -868,6 +878,15 @@ public List<CatalogInfo> listCatalogs(Session session)
868878
}
869879
}
870880

881+
@Override
882+
public List<CatalogInfo> listActiveCatalogs(Session session)
883+
{
884+
Span span = startSpan("listActiveCatalogs");
885+
try (var _ = scopedSpan(span)) {
886+
return delegate.listActiveCatalogs(session);
887+
}
888+
}
889+
871890
@Override
872891
public List<QualifiedObjectName> listViews(Session session, QualifiedTablePrefix prefix)
873892
{

‎core/trino-main/src/test/java/io/trino/execution/MockManagedQueryExecution.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -266,6 +266,7 @@ public QueryInfo getFullQueryInfo()
266266

267267
ImmutableList.of(),
268268
DynamicFiltersStats.EMPTY,
269+
ImmutableMap.of(),
269270
ImmutableList.of(),
270271
ImmutableList.of()),
271272
Optional.empty(),

‎core/trino-main/src/test/java/io/trino/execution/TestQueryStats.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
package io.trino.execution;
1515

1616
import com.google.common.collect.ImmutableList;
17+
import com.google.common.collect.ImmutableMap;
1718
import com.google.common.collect.ImmutableSet;
1819
import io.airlift.json.JsonCodec;
1920
import io.airlift.units.DataSize;
@@ -270,7 +271,7 @@ public class TestQueryStats
270271
107)),
271272

272273
DynamicFiltersStats.EMPTY,
273-
274+
ImmutableMap.of(),
274275
operatorSummaries,
275276
optimizerRulesSummaries);
276277

0 commit comments

Comments
 (0)