perf(metrics): tune histogram metric query

.changeset/rotten-boxes-lick.md

@@ -0,0 +1,6 @@
+---
+"@hyperdx/common-utils": patch
+"@hyperdx/api": patch
+---
+
+Tune the histogram metric query memory usage.

packages/api/src/clickhouse/__tests__/__snapshots__/renderChartConfig.test.ts.snap

@@ -178,64 +178,64 @@ Array [
 exports[`renderChartConfig Query Metrics two_timestamps_bounded histogram (p25) 1`] = `
 Array [
   Object {
+    "Rate": 0,
     "__hdx_time_bucket": "2022-01-05T00:00:00Z",
-    "sum(toFloat64OrNull(toString(Rate)))": 0,
   },
   Object {
+    "Rate": 10,
     "__hdx_time_bucket": "2022-01-05T00:01:00Z",
-    "sum(toFloat64OrNull(toString(Rate)))": 10,
   },
 ]
 `;
 
 exports[`renderChartConfig Query Metrics two_timestamps_bounded histogram (p50) 1`] = `
 Array [
   Object {
+    "Rate": 0,
     "__hdx_time_bucket": "2022-01-05T00:00:00Z",
-    "sum(toFloat64OrNull(toString(Rate)))": 0,
   },
   Object {
+    "Rate": 20,
     "__hdx_time_bucket": "2022-01-05T00:01:00Z",
-    "sum(toFloat64OrNull(toString(Rate)))": 20,
   },
 ]
 `;
 
 exports[`renderChartConfig Query Metrics two_timestamps_bounded histogram (p90) 1`] = `
 Array [
   Object {
+    "Rate": 0,
     "__hdx_time_bucket": "2022-01-05T00:00:00Z",
-    "sum(toFloat64OrNull(toString(Rate)))": 0,
   },
   Object {
+    "Rate": 30,
     "__hdx_time_bucket": "2022-01-05T00:01:00Z",
-    "sum(toFloat64OrNull(toString(Rate)))": 30,
   },
 ]
 `;
 
 exports[`renderChartConfig Query Metrics two_timestamps_lower_bound_inf histogram (p50) 1`] = `
 Array [
   Object {
+    "Rate": 0,
     "__hdx_time_bucket": "2022-01-05T00:00:00Z",
-    "sum(toFloat64OrNull(toString(Rate)))": 0,
   },
   Object {
+    "Rate": 1,
     "__hdx_time_bucket": "2022-01-05T00:01:00Z",
-    "sum(toFloat64OrNull(toString(Rate)))": 1,
   },
 ]
 `;
 
 exports[`renderChartConfig Query Metrics two_timestamps_upper_bound_inf histogram (p50) 1`] = `
 Array [
   Object {
+    "Rate": 0,
     "__hdx_time_bucket": "2022-01-05T00:00:00Z",
-    "sum(toFloat64OrNull(toString(Rate)))": 0,
   },
   Object {
+    "Rate": 30,
     "__hdx_time_bucket": "2022-01-05T00:01:00Z",
-    "sum(toFloat64OrNull(toString(Rate)))": 30,
   },
 ]
 `;

packages/api/src/clickhouse/__tests__/renderChartConfig.test.ts

@@ -347,11 +347,15 @@ describe('renderChartConfig', () => {
           BucketCounts: [0, 0, 0],
           ExplicitBounds: [10, 30],
           TimeUnix: new Date(now),
+          Count: 0,
+          Sum: 0,
         },
         {
           BucketCounts: [10, 10, 10],
           ExplicitBounds: [10, 30],
           TimeUnix: new Date(now + ms('1m')),
+          Count: 30,
+          Sum: 550,
         },
       ].map(point => ({
         MetricName: 'test.two_timestamps_lower_bound',
@@ -367,11 +371,15 @@ describe('renderChartConfig', () => {
           BucketCounts: [0, 0, 0],
           ExplicitBounds: [1, 30],
           TimeUnix: new Date(now),
+          Count: 0,
+          Sum: 0,
         },
         {
           BucketCounts: [10, 0, 0],
           ExplicitBounds: [1, 30],
           TimeUnix: new Date(now + ms('1m')),
+          Count: 10,
+          Sum: 5,
         },
       ].map(point => ({
         MetricName: 'test.two_timestamps_lower_bound_inf',
@@ -387,11 +395,15 @@ describe('renderChartConfig', () => {
           BucketCounts: [0, 0, 0],
           ExplicitBounds: [0, 30],
           TimeUnix: new Date(now),
+          Count: 0,
+          Sum: 0,
         },
         {
           BucketCounts: [0, 0, 10],
           ExplicitBounds: [0, 30],
           TimeUnix: new Date(now + ms('1m')),
+          Count: 10,
+          Sum: 350,
         },
       ].map(point => ({
         MetricName: 'test.two_timestamps_upper_bound_inf',
@@ -407,16 +419,22 @@ describe('renderChartConfig', () => {
           BucketCounts: [5, 5, 5],
           ExplicitBounds: [10, 30],
           TimeUnix: new Date(now),
+          Count: 15,
+          Sum: 225,
         },
         {
           BucketCounts: [0, 0, 0],
           ExplicitBounds: [10, 30],
           TimeUnix: new Date(now + ms('1m')),
+          Count: 0,
+          Sum: 0,
         },
         {
           BucketCounts: [10, 10, 10],
           ExplicitBounds: [10, 30],
           TimeUnix: new Date(now + ms('2m')),
+          Count: 30,
+          Sum: 550,
         },
       ].map(point => ({
         MetricName: 'test.three_timestamps_bounded',

packages/common-utils/src/__tests__/__snapshots__/renderChartConfig.test.ts.snap

@@ -36,39 +36,44 @@ exports[`renderChartConfig should generate sql for a single gauge metric 1`] = `
 `;
 
 exports[`renderChartConfig should generate sql for a single histogram metric 1`] = `
-"WITH HistRate AS (
+"WITH Points AS (
           SELECT
-            *,
+
+            last_value(HistogramTable.Count) AS Count,
+            last_value(HistogramTable.BucketCounts) AS BucketCounts,
+            last_value(HistogramTable.ExplicitBounds) AS ExplicitBounds,
+            last_value(HistogramTable.Sum) AS Sum,
+            last_value(HistogramTable.AggregationTemporality) AS AggregationTemporality,
             cityHash64(mapConcat(ScopeAttributes, ResourceAttributes, Attributes)) AS AttributesHash,
-            length(BucketCounts) as CountLength,
-            any(BucketCounts) OVER (ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING) AS PrevBucketCounts,
-            any(CountLength) OVER (ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING) AS PrevCountLength,
-            any(AttributesHash) OVER (ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING) AS PrevAttributesHash,
-            IF(AggregationTemporality = 1,
-               BucketCounts,
-               IF(AttributesHash = PrevAttributesHash AND CountLength = PrevCountLength,
-                  arrayMap((prev, curr) -> IF(curr < prev, curr, toUInt64(toInt64(curr) - toInt64(prev))), PrevBucketCounts, BucketCounts),
-                  BucketCounts)) as BucketRates
-          FROM default.otel_metrics_histogram
+            toUInt64( 0.5 * Count) AS Rank,
+            arrayCumSum(BucketCounts) as CumCounts,
+            arrayFirstIndex(x -> if(x > Rank, 1, 0), CumCounts) AS BucketLowIdx,
+            CASE
+              WHEN Count = 0 THEN 0
+              WHEN Count = 1 THEN Sum
+              WHEN BucketLowIdx = 1 THEN ExplicitBounds[BucketLowIdx]
+              WHEN BucketLowIdx = length(BucketCounts) THEN ExplicitBounds[length(ExplicitBounds)]
+              ELSE
+                ExplicitBounds[BucketLowIdx - 1] +
+                  ((Rank - ExplicitBounds[BucketLowIdx - 1]) / (BucketCounts[BucketLowIdx])) * (ExplicitBounds[BucketLowIdx] - ExplicitBounds[BucketLowIdx - 1])
+            END AS Point
+          FROM default.otel_metrics_histogram HistogramTable
           WHERE (TimeUnix >= fromUnixTimestamp64Milli(1739318400000) AND TimeUnix <= fromUnixTimestamp64Milli(1765670400000)) AND ((MetricName = 'http.server.duration'))
-          ORDER BY Attributes, TimeUnix ASC
-          ),RawHist AS (
-            SELECT
-              *,
-              toUInt64( 0.5 * arraySum(BucketRates)) AS Rank,
-              arrayCumSum(BucketRates) as CumRates,
-              arrayFirstIndex(x -> if(x > Rank, 1, 0), CumRates) AS BucketLowIdx,
-              IF(BucketLowIdx = length(BucketRates),
-                arrayElement(ExplicitBounds, length(ExplicitBounds)),
-                IF(BucketLowIdx > 1,
-                  arrayElement(ExplicitBounds, BucketLowIdx - 1) + (arrayElement(ExplicitBounds, BucketLowIdx) - arrayElement(ExplicitBounds, BucketLowIdx - 1)) *
-                    IF(arrayElement(CumRates, BucketLowIdx) > arrayElement(CumRates, BucketLowIdx - 1),
-                      (Rank - arrayElement(CumRates, BucketLowIdx - 1)) / (arrayElement(CumRates, BucketLowIdx) - arrayElement(CumRates, BucketLowIdx - 1)), 0),
-                  arrayElement(ExplicitBounds, BucketLowIdx)
-                )) as Rate
-            FROM HistRate) SELECT sum(
-      toFloat64OrNull(toString(Rate))
-    ) FROM RawHist WHERE (TimeUnix >= fromUnixTimestamp64Milli(1739318400000) AND TimeUnix <= fromUnixTimestamp64Milli(1765670400000)) LIMIT 10"
+          GROUP BY AttributesHash
+          ORDER BY AttributesHash ASC
+          LIMIT 10
+          SETTINGS short_circuit_function_evaluation = 'enable'
+        ),Rates AS (
+        SELECT
+          \`__hdx_time_bucket\`,
+          any(Point) OVER (ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING) AS PrevPoint,
+          any(AttributesHash) OVER (ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING) AS PrevAttributesHash,
+          IF(AggregationTemporality = 2 AND AttributesHash = PrevAttributesHash AND PrevPoint <= Point,
+            Point - PrevPoint,
+            Point
+          ) AS Rate
+        FROM Points
+        ) SELECT Rate, \`__hdx_time_bucket\` FROM Rates"
 `;
 
 exports[`renderChartConfig should generate sql for a single sum metric 1`] = `