Enable compression when storing/tailing loki device logs

Change-type: patch
balena-io · Nov 1, 2024 · c7600cd · c7600cd
1 parent 27b60b9
commit c7600cd
Show file tree

Hide file tree

Showing 2 changed files with 19 additions and 0 deletions.
diff --git a/src/features/device-logs/lib/backends/loki.ts b/src/features/device-logs/lib/backends/loki.ts
@@ -1,5 +1,6 @@
 import _ from 'lodash';
 import { EventEmitter } from 'events';
+import { compressionAlgorithms } from '@grpc/grpc-js';
 
 import loki from 'loki-grpc-client';
 import type { types } from '@balena/pinejs';
@@ -10,6 +11,8 @@ import {
 	LOKI_INGESTER_HOST,
 	LOKI_INGESTER_GRPC_PORT,
 	LOKI_HISTORY_GZIP,
+	LOKI_GRPC_SEND_GZIP,
+	LOKI_GRPC_RECEIVE_COMPRESSION_LEVEL,
 } from '../../../../lib/config.js';
 import type {
 	DeviceLog,
@@ -125,13 +128,24 @@ export class LokiBackend implements DeviceLogsBackend {
 
 	constructor() {
 		this.subscriptions = new EventEmitter();
+		const compressionAlgorithm = LOKI_GRPC_SEND_GZIP
+			? compressionAlgorithms.gzip
+			: compressionAlgorithms.identity;
 		this.querier = new loki.QuerierClient(
 			lokiIngesterAddress,
 			loki.createInsecureCredentials(),
+			{
+				'grpc.default_compression_algorithm': compressionAlgorithm,
+				'grpc.default_compression_level': LOKI_GRPC_RECEIVE_COMPRESSION_LEVEL,
+			},
 		);
 		this.pusher = new loki.PusherClient(
 			lokiIngesterAddress,
 			loki.createInsecureCredentials(),
+			{
+				'grpc.default_compression_algorithm': compressionAlgorithm,
+				'grpc.default_compression_level': LOKI_GRPC_RECEIVE_COMPRESSION_LEVEL,
+			},
 		);
 		this.tailCalls = new Map();
 		this.push = backoff(

diff --git a/src/lib/config.ts b/src/lib/config.ts
@@ -303,6 +303,11 @@ export const LOKI_QUERY_HOST = optionalVar('LOKI_QUERY_HOST', LOKI_HOST);
 export const LOKI_INGESTER_GRPC_PORT = intVar('LOKI_INGESTER_GRPC_PORT', 9095);
 export const LOKI_QUERY_HTTP_PORT = intVar('LOKI_QUERY_HTTP_PORT', 3100);
 export const LOKI_HISTORY_GZIP = boolVar('LOKI_HISTORY_GZIP', true);
+export const LOKI_GRPC_SEND_GZIP = boolVar('LOKI_GRPC_SEND_GZIP', true);
+export const LOKI_GRPC_RECEIVE_COMPRESSION_LEVEL = intVar(
+	'LOKI_GRPC_RECEIVE_COMPRESSION_LEVEL',
+	2,
+);
 // control the percent of logs written to Loki while scaling up
 export const LOKI_WRITE_PCT = intVar('LOKI_WRITE_PCT', 0);
 /**