feat: add custom ingestion key via INGESTION_API_KEY (#1112)

Closes HDX-2283

This adds an environment variable `INGESTION_API_KEY` that can be set by the user. This apiKey will be valid and accepted by the Otel collector. It is in addition to the autogenerated apiKey and will not show in the team settings apiKey section.

Author: knudtty

.changeset/chatty-poets-camp.md

@@ -0,0 +1,5 @@
+---
+"@hyperdx/api": patch
+---
+
+feat: INGESTION_API_KEY allows for environment variable defined api key

docker/hyperdx/entry.local.base.sh

@@ -18,6 +18,7 @@ export MONGO_URI="mongodb://db:27017/hyperdx"
 export OPAMP_SERVER_URL="http://127.0.0.1:${OPAMP_PORT}"
 export CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT="${CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT:-ch-server:9363}"
 
+export HYPERDX_IMAGE=$([[ "${IS_LOCAL_APP_MODE}" == "DANGEROUSLY_is_local_app_mode💀" ]] && echo "all-in-one" || echo "all-in-one-noauth")
 export EXPRESS_SESSION_SECRET="hyperdx is cool 👋"
 # IS_LOCAL_APP_MODE should be set by the calling script
 # Default to dangerous mode if not set

docker/hyperdx/entry.prod.sh

@@ -2,6 +2,7 @@
 
 export FRONTEND_URL="${FRONTEND_URL:-${HYPERDX_APP_URL:-http://localhost}:${HYPERDX_APP_PORT:-8080}}"
 export OPAMP_PORT=${HYPERDX_OPAMP_PORT:-4320}
+export HYPERDX_IMAGE="hyperdx"
 
 # Set to "REQUIRED_AUTH" to enforce API authentication.
 # ⚠️ Do not change this value !!!!

packages/api/.env.development

@@ -23,3 +23,5 @@ NODE_OPTIONS="--max-http-header-size=131072"
 ENABLE_SWAGGER=true
 DEFAULT_CONNECTIONS=[{"name":"Local ClickHouse","host":"http://localhost:8123","username":"default","password":""}]
 DEFAULT_SOURCES=[{"from":{"databaseName":"default","tableName":"otel_logs"},"kind":"log","timestampValueExpression":"TimestampTime","name":"Logs","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","traceSourceId":"Traces","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":"otel_traces"},"kind":"trace","timestampValueExpression":"Timestamp","name":"Traces","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"SpanName","serviceNameExpression":"ServiceName","bodyExpression":"SpanName","eventAttributesExpression":"SpanAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,StatusCode,round(Duration/1e6),SpanName","traceIdExpression":"TraceId","spanIdExpression":"SpanId","durationExpression":"Duration","durationPrecision":9,"parentSpanIdExpression":"ParentSpanId","spanNameExpression":"SpanName","spanKindExpression":"SpanKind","statusCodeExpression":"StatusCode","statusMessageExpression":"StatusMessage","connection":"Local ClickHouse","logSourceId":"Logs","sessionSourceId":"Sessions","metricSourceId":"Metrics"},{"from":{"databaseName":"default","tableName":""},"kind":"metric","timestampValueExpression":"TimeUnix","name":"Metrics","resourceAttributesExpression":"ResourceAttributes","metricTables":{"gauge":"otel_metrics_gauge","histogram":"otel_metrics_histogram","sum":"otel_metrics_sum","_id":"682586a8b1f81924e628e808","id":"682586a8b1f81924e628e808"},"connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","sessionSourceId":"Sessions"},{"from":{"databaseName":"default","tableName":"hyperdx_sessions"},"kind":"session","timestampValueExpression":"TimestampTime","name":"Sessions","displayedTimestampValueExpression":"Timestamp","implicitColumnExpression":"Body","serviceNameExpression":"ServiceName","bodyExpression":"Body","eventAttributesExpression":"LogAttributes","resourceAttributesExpression":"ResourceAttributes","defaultTableSelectExpression":"Timestamp,ServiceName,SeverityText,Body","severityTextExpression":"SeverityText","traceIdExpression":"TraceId","spanIdExpression":"SpanId","connection":"Local ClickHouse","logSourceId":"Logs","traceSourceId":"Traces","metricSourceId":"Metrics"}]
+INGESTION_API_KEY="super-secure-ingestion-api-key"
+HYPERDX_API_KEY=$INGESTION_API_KEY

packages/api/src/config.ts

@@ -15,6 +15,11 @@ export const EXPRESS_SESSION_SECRET = (env.EXPRESS_SESSION_SECRET ||
   DEFAULT_EXPRESS_SESSION) as string;
 export const FRONTEND_URL = (env.FRONTEND_URL ||
   DEFAULT_FRONTEND_URL) as string;
+const HYPERDX_IMAGE = env.HYPERDX_IMAGE;
+export const IS_APP_IMAGE = HYPERDX_IMAGE === 'hyperdx';
+export const IS_ALL_IN_ONE_IMAGE = HYPERDX_IMAGE === 'all-in-one-auth';
+export const IS_LOCAL_IMAGE = HYPERDX_IMAGE === 'all-in-one-noauth';
+export const INGESTION_API_KEY = env.INGESTION_API_KEY ?? '';
 export const HYPERDX_API_KEY = env.HYPERDX_API_KEY as string;
 export const HYPERDX_LOG_LEVEL = env.HYPERDX_LOG_LEVEL as string;
 export const IS_CI = NODE_ENV === 'test';

packages/api/src/opamp/controllers/opampController.ts

@@ -116,6 +116,14 @@ type CollectorConfig = {
 
 export const buildOtelCollectorConfig = (teams: ITeam[]): CollectorConfig => {
   const apiKeys = teams.filter(team => team.apiKey).map(team => team.apiKey);
+
+  if (config.IS_ALL_IN_ONE_IMAGE || config.IS_LOCAL_APP_MODE || config.IS_DEV) {
+    // Only allow INGESTION_API_KEY for dev or all-in-one images for security reasons
+    if (config.INGESTION_API_KEY) {
+      apiKeys.push(config.INGESTION_API_KEY);
+    }
+  }
+
   const collectorAuthenticationEnforced =
     teams[0]?.collectorAuthenticationEnforced;