fix(explorer): alert on Tempo API access failures (#1155)

* fix(explorer): alert on Tempo API access failures

* fix(explorer): exclude 401 responses from alerts

Author: decofe

apps/explorer/src/lib/query-retry.ts

@@ -0,0 +1,11 @@
+const NON_RETRYABLE_HTTP_STATUS = /(?:status:\s*|\b)(402|403|429)\b/i
+
+export function shouldRetryQuery(
+	failureCount: number,
+	error: unknown,
+): boolean {
+	if (failureCount >= 2) return false
+
+	const message = error instanceof Error ? error.message : String(error)
+	return !NON_RETRYABLE_HTTP_STATUS.test(message)
+}

apps/explorer/src/lib/server/tempo-api.ts

@@ -1,7 +1,52 @@
 import type * as cadent from 'cadent'
+import * as Sentry from '@sentry/cloudflare'
 import { hc } from 'hono/client'
 import { serverEnv, tempoApiUrl } from './env.ts'
 
+const ALERTABLE_STATUSES = new Set([402, 403, 429])
+const REPORT_THROTTLE_MS = 60_000
+const lastReportedAt = new Map<string, number>()
+
+export function isAlertableTempoApiStatus(status: number): boolean {
+	return ALERTABLE_STATUSES.has(status) || status >= 500
+}
+
+function reportTempoApiResponse(response: Response, method: string): void {
+	if (!isAlertableTempoApiStatus(response.status)) return
+
+	const url = new URL(response.url)
+	const key = `${response.status}:${method}:${url.pathname}`
+	const now = Date.now()
+	const previous = lastReportedAt.get(key) ?? 0
+
+	console.error('[tempo-api] upstream request failed', {
+		method,
+		path: url.pathname,
+		status: response.status,
+	})
+
+	if (now - previous < REPORT_THROTTLE_MS) return
+	lastReportedAt.set(key, now)
+
+	Sentry.captureMessage(`Tempo API returned ${response.status}`, {
+		level: 'error',
+		tags: {
+			component: 'tempo-api-client',
+			method,
+			path: url.pathname,
+			status: String(response.status),
+		},
+	})
+}
+
+const instrumentedFetch: typeof fetch = async (input, init) => {
+	const response = await fetch(input, init)
+	const method =
+		init?.method ?? (input instanceof Request ? input.method : 'GET')
+	reportTempoApiResponse(response, method)
+	return response
+}
+
 /**
  * Typed client for the Tempo API. Server-side only.
  *
@@ -17,6 +62,7 @@ import { serverEnv, tempoApiUrl } from './env.ts'
  * (scopes: `data:read`, `indexer:query`).
  */
 export const api = hc<cadent.App.App>(tempoApiUrl, {
+	fetch: instrumentedFetch,
 	headers: serverEnv.TEMPO_API_KEY
 		? { 'tempo-api-key': serverEnv.TEMPO_API_KEY }
 		: undefined,

apps/explorer/src/router.tsx

@@ -11,6 +11,7 @@ import {
 	ProfileEvents,
 } from '#lib/profiling'
 import { initSentry } from '#lib/sentry'
+import { shouldRetryQuery } from '#lib/query-retry'
 import { routeTree } from '#routeTree.gen.ts'
 
 const queryStartTimes = new WeakMap<object, number>()
@@ -20,6 +21,7 @@ export const getRouter = () => {
 	const queryClient: QueryClient = new QueryClient({
 		defaultOptions: {
 			queries: {
+				retry: shouldRetryQuery,
 				staleTime: 60 * 1_000, // needed for SSR - prevents refetch on hydration
 				gcTime: 1_000 * 60 * 60 * 24, // 24 hours
 				queryKeyHashFn: hashFn,

apps/explorer/test/query-retry.node.test.ts

@@ -0,0 +1,18 @@
+import { describe, expect, it } from 'vitest'
+import { shouldRetryQuery } from '../src/lib/query-retry'
+
+describe('shouldRetryQuery', () => {
+	it.each([
+		'402 Payment Required',
+		'Status: 403',
+		'Request failed with status: 429',
+	])('does not retry non-retryable upstream errors: %s', (message) => {
+		expect(shouldRetryQuery(0, new Error(message))).toBe(false)
+	})
+
+	it('retries other failures at most twice', () => {
+		expect(shouldRetryQuery(0, new Error('upstream unavailable'))).toBe(true)
+		expect(shouldRetryQuery(1, new Error('upstream unavailable'))).toBe(true)
+		expect(shouldRetryQuery(2, new Error('upstream unavailable'))).toBe(false)
+	})
+})

apps/explorer/test/tempo-api-status.node.test.ts

@@ -0,0 +1,12 @@
+import { describe, expect, it } from 'vitest'
+import { isAlertableTempoApiStatus } from '../src/lib/server/tempo-api'
+
+describe('isAlertableTempoApiStatus', () => {
+	it.each([402, 403, 429, 500, 503])('alerts for status %i', (status) => {
+		expect(isAlertableTempoApiStatus(status)).toBe(true)
+	})
+
+	it.each([200, 400, 401, 404])('does not alert for status %i', (status) => {
+		expect(isAlertableTempoApiStatus(status)).toBe(false)
+	})
+})