From 81794681754585b45b02ce2da7fd03922c5fbf55 Mon Sep 17 00:00:00 2001 From: Jason Rasmussen Date: Thu, 17 Oct 2024 17:43:23 -0400 Subject: [PATCH] refactor(server): telemetry env --- server/src/app.module.ts | 5 +-- server/src/bin/sync-sql.ts | 6 ++- server/src/interfaces/config.interface.ts | 8 ++++ .../repositories/config.repository.spec.ts | 39 ++++++++++++++++++- server/src/repositories/config.repository.ts | 29 +++++++++++++- server/src/repositories/metric.repository.ts | 16 ++++---- server/src/utils/instrumentation.ts | 33 +++------------- .../repositories/config.repository.mock.ts | 15 +++++++ 8 files changed, 108 insertions(+), 43 deletions(-) diff --git a/server/src/app.module.ts b/server/src/app.module.ts index 43aefbd0f035d..fd921150fd564 100644 --- a/server/src/app.module.ts +++ b/server/src/app.module.ts @@ -23,7 +23,6 @@ import { repositories } from 'src/repositories'; import { ConfigRepository } from 'src/repositories/config.repository'; import { services } from 'src/services'; import { DatabaseService } from 'src/services/database.service'; -import { otelConfig } from 'src/utils/instrumentation'; const common = [...services, ...repositories]; @@ -37,14 +36,14 @@ const middleware = [ ]; const configRepository = new ConfigRepository(); -const { bull } = configRepository.getEnv(); +const { bull, otel } = configRepository.getEnv(); const imports = [ BullModule.forRoot(bull.config), BullModule.registerQueue(...bull.queues), ClsModule.forRoot(clsConfig), ConfigModule.forRoot(immichAppConfig), - OpenTelemetryModule.forRoot(otelConfig), + OpenTelemetryModule.forRoot(otel), TypeOrmModule.forRootAsync({ inject: [ModuleRef], useFactory: (moduleRef: ModuleRef) => { diff --git a/server/src/bin/sync-sql.ts b/server/src/bin/sync-sql.ts index 92c3cc11032ce..e4f11cc6928a9 100644 --- a/server/src/bin/sync-sql.ts +++ b/server/src/bin/sync-sql.ts @@ -14,8 +14,8 @@ import { entities } from 'src/entities'; import { ILoggerRepository } from 'src/interfaces/logger.interface'; import { repositories } from 'src/repositories'; import { AccessRepository } from 'src/repositories/access.repository'; +import { ConfigRepository } from 'src/repositories/config.repository'; import { AuthService } from 'src/services/auth.service'; -import { otelConfig } from 'src/utils/instrumentation'; import { Logger } from 'typeorm'; export class SqlLogger implements Logger { @@ -74,6 +74,8 @@ class SqlGenerator { await rm(this.options.targetDir, { force: true, recursive: true }); await mkdir(this.options.targetDir); + const { otel } = new ConfigRepository().getEnv(); + const moduleFixture = await Test.createTestingModule({ imports: [ TypeOrmModule.forRoot({ @@ -84,7 +86,7 @@ class SqlGenerator { logger: this.sqlLogger, }), TypeOrmModule.forFeature(entities), - OpenTelemetryModule.forRoot(otelConfig), + OpenTelemetryModule.forRoot(otel), ], providers: [...repositories, AuthService, SchedulerRegistry], }).compile(); diff --git a/server/src/interfaces/config.interface.ts b/server/src/interfaces/config.interface.ts index 9870b86d10d6c..4391909df7131 100644 --- a/server/src/interfaces/config.interface.ts +++ b/server/src/interfaces/config.interface.ts @@ -1,6 +1,7 @@ import { RegisterQueueOptions } from '@nestjs/bullmq'; import { QueueOptions } from 'bullmq'; import { RedisOptions } from 'ioredis'; +import { OpenTelemetryModuleOptions } from 'nestjs-otel/lib/interfaces'; import { ImmichEnvironment, ImmichWorker, LogLevel } from 'src/enum'; import { VectorExtension } from 'src/interfaces/database.interface'; @@ -54,6 +55,8 @@ export interface EnvData { trustedProxies: string[]; }; + otel: OpenTelemetryModuleOptions; + resourcePaths: { lockFile: string; geodata: { @@ -74,6 +77,11 @@ export interface EnvData { telemetry: { apiPort: number; microservicesPort: number; + enabled: boolean; + apiMetrics: boolean; + hostMetrics: boolean; + repoMetrics: boolean; + jobMetrics: boolean; }; storage: { diff --git a/server/src/repositories/config.repository.spec.ts b/server/src/repositories/config.repository.spec.ts index 36b7b48062746..84da211182793 100644 --- a/server/src/repositories/config.repository.spec.ts +++ b/server/src/repositories/config.repository.spec.ts @@ -12,6 +12,11 @@ const resetEnv = () => { 'IMMICH_TRUSTED_PROXIES', 'IMMICH_API_METRICS_PORT', 'IMMICH_MICROSERVICES_METRICS_PORT', + 'IMMICH_METRICS', + 'IMMICH_API_METRICS', + 'IMMICH_HOST_METRICS', + 'IMMICH_IO_METRICS', + 'IMMICH_JOB_METRICS', 'DB_URL', 'DB_HOSTNAME', @@ -200,11 +205,16 @@ describe('getEnv', () => { }); describe('telemetry', () => { - it('should return default ports', () => { + it('should have default values', () => { const { telemetry } = getEnv(); expect(telemetry).toEqual({ apiPort: 8081, microservicesPort: 8082, + enabled: false, + apiMetrics: false, + hostMetrics: false, + jobMetrics: false, + repoMetrics: false, }); }); @@ -212,10 +222,35 @@ describe('getEnv', () => { process.env.IMMICH_API_METRICS_PORT = '2001'; process.env.IMMICH_MICROSERVICES_METRICS_PORT = '2002'; const { telemetry } = getEnv(); - expect(telemetry).toEqual({ + expect(telemetry).toMatchObject({ apiPort: 2001, microservicesPort: 2002, }); }); + + it('should run with telemetry enabled', () => { + process.env.IMMICH_METRICS = 'true'; + const { telemetry } = getEnv(); + expect(telemetry).toMatchObject({ + enabled: true, + apiMetrics: true, + hostMetrics: true, + jobMetrics: true, + repoMetrics: true, + }); + }); + + it('should run with telemetry enabled and jobs disabled', () => { + process.env.IMMICH_METRICS = 'true'; + process.env.IMMICH_JOB_METRICS = 'false'; + const { telemetry } = getEnv(); + expect(telemetry).toMatchObject({ + enabled: true, + apiMetrics: true, + hostMetrics: true, + jobMetrics: false, + repoMetrics: true, + }); + }); }); }); diff --git a/server/src/repositories/config.repository.ts b/server/src/repositories/config.repository.ts index 8b511fba5ae2c..44b8c7b605e47 100644 --- a/server/src/repositories/config.repository.ts +++ b/server/src/repositories/config.repository.ts @@ -1,6 +1,6 @@ import { Injectable } from '@nestjs/common'; import { join } from 'node:path'; -import { citiesFile } from 'src/constants'; +import { citiesFile, excludePaths } from 'src/constants'; import { ImmichEnvironment, ImmichWorker, LogLevel } from 'src/enum'; import { EnvData, IConfigRepository } from 'src/interfaces/config.interface'; import { DatabaseExtension } from 'src/interfaces/database.interface'; @@ -30,6 +30,8 @@ const asSet = (value: string | undefined, defaults: ImmichWorker[]) => { return new Set(values.length === 0 ? defaults : (values as ImmichWorker[])); }; +const parseBoolean = (value: string | undefined, defaultValue: boolean) => (value ? value === 'true' : defaultValue); + const getEnv = (): EnvData => { const included = asSet(process.env.IMMICH_WORKERS_INCLUDE, [ImmichWorker.API, ImmichWorker.MICROSERVICES]); const excluded = asSet(process.env.IMMICH_WORKERS_EXCLUDE, []); @@ -66,6 +68,16 @@ const getEnv = (): EnvData => { } } + const globalEnabled = parseBoolean(process.env.IMMICH_METRICS, false); + const hostMetrics = parseBoolean(process.env.IMMICH_HOST_METRICS, globalEnabled); + const apiMetrics = parseBoolean(process.env.IMMICH_API_METRICS, globalEnabled); + const repoMetrics = parseBoolean(process.env.IMMICH_IO_METRICS, globalEnabled); + const jobMetrics = parseBoolean(process.env.IMMICH_JOB_METRICS, globalEnabled); + const telemetryEnabled = globalEnabled || hostMetrics || apiMetrics || repoMetrics || jobMetrics; + if (!telemetryEnabled && process.env.OTEL_SDK_DISABLED === undefined) { + process.env.OTEL_SDK_DISABLED = 'true'; + } + return { host: process.env.IMMICH_HOST, port: Number(process.env.IMMICH_PORT) || 2283, @@ -124,6 +136,16 @@ const getEnv = (): EnvData => { .filter(Boolean), }, + otel: { + metrics: { + hostMetrics, + apiMetrics: { + enable: apiMetrics, + ignoreRoutes: excludePaths, + }, + }, + }, + redis: redisConfig, resourcePaths: { @@ -148,6 +170,11 @@ const getEnv = (): EnvData => { telemetry: { apiPort: Number(process.env.IMMICH_API_METRICS_PORT || '') || 8081, microservicesPort: Number(process.env.IMMICH_MICROSERVICES_METRICS_PORT || '') || 8082, + enabled: telemetryEnabled, + hostMetrics, + apiMetrics, + repoMetrics, + jobMetrics, }, workers, diff --git a/server/src/repositories/metric.repository.ts b/server/src/repositories/metric.repository.ts index 5948e92fa67f0..b59bcf9ed1070 100644 --- a/server/src/repositories/metric.repository.ts +++ b/server/src/repositories/metric.repository.ts @@ -1,11 +1,12 @@ -import { Injectable } from '@nestjs/common'; +import { Inject, Injectable } from '@nestjs/common'; import { MetricOptions } from '@opentelemetry/api'; import { MetricService } from 'nestjs-otel'; +import { IConfigRepository } from 'src/interfaces/config.interface'; import { IMetricGroupRepository, IMetricRepository, MetricGroupOptions } from 'src/interfaces/metric.interface'; -import { apiMetrics, hostMetrics, jobMetrics, repoMetrics } from 'src/utils/instrumentation'; class MetricGroupRepository implements IMetricGroupRepository { private enabled = false; + constructor(private metricService: MetricService) {} addToCounter(name: string, value: number, options?: MetricOptions): void { @@ -39,10 +40,11 @@ export class MetricRepository implements IMetricRepository { jobs: MetricGroupRepository; repo: MetricGroupRepository; - constructor(metricService: MetricService) { - this.api = new MetricGroupRepository(metricService).configure({ enabled: apiMetrics }); - this.host = new MetricGroupRepository(metricService).configure({ enabled: hostMetrics }); - this.jobs = new MetricGroupRepository(metricService).configure({ enabled: jobMetrics }); - this.repo = new MetricGroupRepository(metricService).configure({ enabled: repoMetrics }); + constructor(metricService: MetricService, @Inject(IConfigRepository) configRepository: IConfigRepository) { + const { telemetry } = configRepository.getEnv(); + this.api = new MetricGroupRepository(metricService).configure({ enabled: telemetry.apiMetrics }); + this.host = new MetricGroupRepository(metricService).configure({ enabled: telemetry.hostMetrics }); + this.jobs = new MetricGroupRepository(metricService).configure({ enabled: telemetry.jobMetrics }); + this.repo = new MetricGroupRepository(metricService).configure({ enabled: telemetry.repoMetrics }); } } diff --git a/server/src/utils/instrumentation.ts b/server/src/utils/instrumentation.ts index 484ba5901cf09..bd522f27b2b1f 100644 --- a/server/src/utils/instrumentation.ts +++ b/server/src/utils/instrumentation.ts @@ -7,32 +7,19 @@ import { PgInstrumentation } from '@opentelemetry/instrumentation-pg'; import { NodeSDK, contextBase, metrics, resources } from '@opentelemetry/sdk-node'; import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions'; import { snakeCase, startCase } from 'lodash'; -import { OpenTelemetryModuleOptions } from 'nestjs-otel/lib/interfaces'; import { copyMetadataFromFunctionToFunction } from 'nestjs-otel/lib/opentelemetry.utils'; import { performance } from 'node:perf_hooks'; -import { excludePaths, serverVersion } from 'src/constants'; +import { serverVersion } from 'src/constants'; import { DecorateAll } from 'src/decorators'; - -let metricsEnabled = process.env.IMMICH_METRICS === 'true'; -export const hostMetrics = - process.env.IMMICH_HOST_METRICS == null ? metricsEnabled : process.env.IMMICH_HOST_METRICS === 'true'; -export const apiMetrics = - process.env.IMMICH_API_METRICS == null ? metricsEnabled : process.env.IMMICH_API_METRICS === 'true'; -export const repoMetrics = - process.env.IMMICH_IO_METRICS == null ? metricsEnabled : process.env.IMMICH_IO_METRICS === 'true'; -export const jobMetrics = - process.env.IMMICH_JOB_METRICS == null ? metricsEnabled : process.env.IMMICH_JOB_METRICS === 'true'; - -metricsEnabled ||= hostMetrics || apiMetrics || repoMetrics || jobMetrics; -if (!metricsEnabled && process.env.OTEL_SDK_DISABLED === undefined) { - process.env.OTEL_SDK_DISABLED = 'true'; -} +import { ConfigRepository } from 'src/repositories/config.repository'; const aggregation = new metrics.ExplicitBucketHistogramAggregation( [0.1, 0.25, 0.5, 0.75, 1, 2.5, 5, 7.5, 10, 25, 50, 75, 100, 250, 500, 750, 1000, 2500, 5000, 7500, 10_000], true, ); +const { telemetry } = new ConfigRepository().getEnv(); + let otelSingleton: NodeSDK | undefined; export const otelStart = (port: number) => { @@ -64,23 +51,13 @@ export const otelShutdown = async () => { } }; -export const otelConfig: OpenTelemetryModuleOptions = { - metrics: { - hostMetrics, - apiMetrics: { - enable: apiMetrics, - ignoreRoutes: excludePaths, - }, - }, -}; - function ExecutionTimeHistogram({ description, unit = 'ms', valueType = contextBase.ValueType.DOUBLE, }: contextBase.MetricOptions = {}) { return (target: any, propertyKey: string | symbol, descriptor: PropertyDescriptor) => { - if (!repoMetrics || process.env.OTEL_SDK_DISABLED) { + if (!telemetry.repoMetrics || process.env.OTEL_SDK_DISABLED) { return; } diff --git a/server/test/repositories/config.repository.mock.ts b/server/test/repositories/config.repository.mock.ts index c7da917cc4565..bb3cfcebb956c 100644 --- a/server/test/repositories/config.repository.mock.ts +++ b/server/test/repositories/config.repository.mock.ts @@ -35,6 +35,16 @@ const envData: EnvData = { trustedProxies: [], }, + otel: { + metrics: { + hostMetrics: false, + apiMetrics: { + enable: false, + ignoreRoutes: [], + }, + }, + }, + redis: { host: 'redis', port: 6379, @@ -63,6 +73,11 @@ const envData: EnvData = { telemetry: { apiPort: 8081, microservicesPort: 8082, + enabled: false, + hostMetrics: false, + apiMetrics: false, + jobMetrics: false, + repoMetrics: false, }, workers: [ImmichWorker.API, ImmichWorker.MICROSERVICES],