Merge pull request #1829 from balena-io/loki

Loki
balena-io · Nov 1, 2024 · 2e75d93 · 2e75d93
2 parents 822ebc7 + 8d414cd
commit 2e75d93
Show file tree

Hide file tree

Showing 3 changed files with 25 additions and 0 deletions.
diff --git a/config/confd/templates/env.tmpl b/config/confd/templates/env.tmpl
@@ -68,6 +68,9 @@ LOKI_QUERY_HOST={{getenv "LOKI_QUERY_HOST"}}
 LOKI_INGESTER_HOST={{getenv "LOKI_INGESTER_HOST"}}
 LOKI_QUERY_HTTP_PORT={{getenv "LOKI_QUERY_HTTP_PORT"}}
 LOKI_INGESTER_GRPC_PORT={{getenv "LOKI_INGESTER_GRPC_PORT"}}
+LOKI_HISTORY_GZIP={{getenv "LOKI_HISTORY_GZIP"}}
+LOKI_GRPC_SEND_GZIP={{getenv "LOKI_GRPC_SEND_GZIP"}}
+LOKI_GRPC_RECEIVE_COMPRESSION_LEVEL={{getenv "LOKI_GRPC_RECEIVE_COMPRESSION_LEVEL"}}
 LOKI_READ_PCT={{getenv "LOKI_READ_PCT"}}
 LOKI_WRITE_PCT={{getenv "LOKI_WRITE_PCT"}}
 LOGS_READ_STREAM_FLUSH_INTERVAL={{getenv "LOGS_READ_STREAM_FLUSH_INTERVAL"}}

diff --git a/src/features/device-logs/lib/backends/loki.ts b/src/features/device-logs/lib/backends/loki.ts
@@ -1,5 +1,6 @@
 import _ from 'lodash';
 import { EventEmitter } from 'events';
+import { compressionAlgorithms } from '@grpc/grpc-js';
 
 import loki from 'loki-grpc-client';
 import type { types } from '@balena/pinejs';
@@ -9,6 +10,9 @@ import {
 	LOKI_QUERY_HTTP_PORT,
 	LOKI_INGESTER_HOST,
 	LOKI_INGESTER_GRPC_PORT,
+	LOKI_HISTORY_GZIP,
+	LOKI_GRPC_SEND_GZIP,
+	LOKI_GRPC_RECEIVE_COMPRESSION_LEVEL,
 } from '../../../../lib/config.js';
 import type {
 	DeviceLog,
@@ -124,13 +128,24 @@ export class LokiBackend implements DeviceLogsBackend {
 
 	constructor() {
 		this.subscriptions = new EventEmitter();
+		const compressionAlgorithm = LOKI_GRPC_SEND_GZIP
+			? compressionAlgorithms.gzip
+			: compressionAlgorithms.identity;
 		this.querier = new loki.QuerierClient(
 			lokiIngesterAddress,
 			loki.createInsecureCredentials(),
+			{
+				'grpc.default_compression_algorithm': compressionAlgorithm,
+				'grpc.default_compression_level': LOKI_GRPC_RECEIVE_COMPRESSION_LEVEL,
+			},
 		);
 		this.pusher = new loki.PusherClient(
 			lokiIngesterAddress,
 			loki.createInsecureCredentials(),
+			{
+				'grpc.default_compression_algorithm': compressionAlgorithm,
+				'grpc.default_compression_level': LOKI_GRPC_RECEIVE_COMPRESSION_LEVEL,
+			},
 		);
 		this.tailCalls = new Map();
 		this.push = backoff(
@@ -174,6 +189,7 @@ export class LokiBackend implements DeviceLogsBackend {
 				since: '30d',
 			},
 			json: true,
+			gzip: LOKI_HISTORY_GZIP,
 		});
 
 		const logs = (

diff --git a/src/lib/config.ts b/src/lib/config.ts
@@ -302,6 +302,12 @@ export const LOKI_INGESTER_HOST = optionalVar('LOKI_INGESTER_HOST', LOKI_HOST);
 export const LOKI_QUERY_HOST = optionalVar('LOKI_QUERY_HOST', LOKI_HOST);
 export const LOKI_INGESTER_GRPC_PORT = intVar('LOKI_INGESTER_GRPC_PORT', 9095);
 export const LOKI_QUERY_HTTP_PORT = intVar('LOKI_QUERY_HTTP_PORT', 3100);
+export const LOKI_HISTORY_GZIP = boolVar('LOKI_HISTORY_GZIP', true);
+export const LOKI_GRPC_SEND_GZIP = boolVar('LOKI_GRPC_SEND_GZIP', true);
+export const LOKI_GRPC_RECEIVE_COMPRESSION_LEVEL = intVar(
+	'LOKI_GRPC_RECEIVE_COMPRESSION_LEVEL',
+	2,
+);
 // control the percent of logs written to Loki while scaling up
 export const LOKI_WRITE_PCT = intVar('LOKI_WRITE_PCT', 0);
 /**