diff --git a/src/features/device-logs/lib/backends/loki.ts b/src/features/device-logs/lib/backends/loki.ts index e09c532f3..9a4bdd9fb 100644 --- a/src/features/device-logs/lib/backends/loki.ts +++ b/src/features/device-logs/lib/backends/loki.ts @@ -1,5 +1,6 @@ import _ from 'lodash'; import { EventEmitter } from 'events'; +import { compressionAlgorithms } from '@grpc/grpc-js'; import loki from 'loki-grpc-client'; import type { types } from '@balena/pinejs'; @@ -10,6 +11,8 @@ import { LOKI_INGESTER_HOST, LOKI_INGESTER_GRPC_PORT, LOKI_HISTORY_GZIP, + LOKI_GRPC_SEND_GZIP, + LOKI_GRPC_RECEIVE_COMPRESSION_LEVEL, } from '../../../../lib/config.js'; import type { DeviceLog, @@ -125,13 +128,24 @@ export class LokiBackend implements DeviceLogsBackend { constructor() { this.subscriptions = new EventEmitter(); + const compressionAlgorithm = LOKI_GRPC_SEND_GZIP + ? compressionAlgorithms.gzip + : compressionAlgorithms.identity; this.querier = new loki.QuerierClient( lokiIngesterAddress, loki.createInsecureCredentials(), + { + 'grpc.default_compression_algorithm': compressionAlgorithm, + 'grpc.default_compression_level': LOKI_GRPC_RECEIVE_COMPRESSION_LEVEL, + }, ); this.pusher = new loki.PusherClient( lokiIngesterAddress, loki.createInsecureCredentials(), + { + 'grpc.default_compression_algorithm': compressionAlgorithm, + 'grpc.default_compression_level': LOKI_GRPC_RECEIVE_COMPRESSION_LEVEL, + }, ); this.tailCalls = new Map(); this.push = backoff( diff --git a/src/lib/config.ts b/src/lib/config.ts index 1e4be3381..088cad468 100644 --- a/src/lib/config.ts +++ b/src/lib/config.ts @@ -303,6 +303,11 @@ export const LOKI_QUERY_HOST = optionalVar('LOKI_QUERY_HOST', LOKI_HOST); export const LOKI_INGESTER_GRPC_PORT = intVar('LOKI_INGESTER_GRPC_PORT', 9095); export const LOKI_QUERY_HTTP_PORT = intVar('LOKI_QUERY_HTTP_PORT', 3100); export const LOKI_HISTORY_GZIP = boolVar('LOKI_HISTORY_GZIP', true); +export const LOKI_GRPC_SEND_GZIP = boolVar('LOKI_GRPC_SEND_GZIP', true); +export const LOKI_GRPC_RECEIVE_COMPRESSION_LEVEL = intVar( + 'LOKI_GRPC_RECEIVE_COMPRESSION_LEVEL', + 2, +); // control the percent of logs written to Loki while scaling up export const LOKI_WRITE_PCT = intVar('LOKI_WRITE_PCT', 0); /**