helpy4/immich
1
0
Fork 0
mirror of https://github.com/immich-app/immich synced 2025-11-14 17:36:12 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

refactor(server): telemetry env variables (#13705)

Browse source 
refactor(server)!: telemetry env variables

Co-authored-by: Mert <101130780+mertalev@users.noreply.github.com>
This commit is contained in:
Daniel Dietzler 2024-10-24 23:07:32 +02:00 • committed by GitHub
parent bc06863d28
commit 151ba9f1d9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 67 additions and 79 deletions
Download patch file Download diff file Expand all files Collapse all files

8
server/src/enum.ts
View file

@ -363,3 +363,11 @@ export enum ImmichWorker {
API = 'api',
MICROSERVICES = 'microservices',
}
export enum ImmichTelemetry {
HOST = 'host',
API = 'api',
IO = 'io',
REPO = 'repo',
JOB = 'job',
}

8
server/src/interfaces/config.interface.ts
View file

@ -2,7 +2,7 @@ import { RegisterQueueOptions } from '@nestjs/bullmq';
import { QueueOptions } from 'bullmq';
import { RedisOptions } from 'ioredis';
import { OpenTelemetryModuleOptions } from 'nestjs-otel/lib/interfaces';
import { ImmichEnvironment, ImmichWorker, LogLevel } from 'src/enum';
import { ImmichEnvironment, ImmichTelemetry, ImmichWorker, LogLevel } from 'src/enum';
import { VectorExtension } from 'src/interfaces/database.interface';
export const IConfigRepository = 'IConfigRepository';
@ -77,11 +77,7 @@ export interface EnvData {
telemetry: {
apiPort: number;
microservicesPort: number;
enabled: boolean;
apiMetrics: boolean;
hostMetrics: boolean;
repoMetrics: boolean;
jobMetrics: boolean;
metrics: Set<ImmichTelemetry>;
};
storage: {

45
server/src/repositories/config.repository.spec.ts
View file

@ -1,3 +1,4 @@
import { ImmichTelemetry } from 'src/enum';
import { clearEnvCache, ConfigRepository } from 'src/repositories/config.repository';
const getEnv = () => {
@ -12,11 +13,8 @@ const resetEnv = () => {
'IMMICH_TRUSTED_PROXIES',
'IMMICH_API_METRICS_PORT',
'IMMICH_MICROSERVICES_METRICS_PORT',
'IMMICH_METRICS',
'IMMICH_API_METRICS',
'IMMICH_HOST_METRICS',
'IMMICH_IO_METRICS',
'IMMICH_JOB_METRICS',
'IMMICH_TELEMETRY_INCLUDE',
'IMMICH_TELEMETRY_EXCLUDE',
'DB_URL',
'DB_HOSTNAME',
@ -210,11 +208,7 @@ describe('getEnv', () => {
expect(telemetry).toEqual({
apiPort: 8081,
microservicesPort: 8082,
enabled: false,
apiMetrics: false,
hostMetrics: false,
jobMetrics: false,
repoMetrics: false,
metrics: new Set([]),
});
});
@ -225,32 +219,29 @@ describe('getEnv', () => {
expect(telemetry).toMatchObject({
apiPort: 2001,
microservicesPort: 2002,
metrics: expect.any(Set),
});
});
it('should run with telemetry enabled', () => {
process.env.IMMICH_METRICS = 'true';
process.env.IMMICH_TELEMETRY_INCLUDE = 'all';
const { telemetry } = getEnv();
expect(telemetry).toMatchObject({
enabled: true,
apiMetrics: true,
hostMetrics: true,
jobMetrics: true,
repoMetrics: true,
});
expect(telemetry.metrics).toEqual(new Set(Object.values(ImmichTelemetry)));
});
it('should run with telemetry enabled and jobs disabled', () => {
process.env.IMMICH_METRICS = 'true';
process.env.IMMICH_JOB_METRICS = 'false';
process.env.IMMICH_TELEMETRY_INCLUDE = 'all';
process.env.IMMICH_TELEMETRY_EXCLUDE = 'job';
const { telemetry } = getEnv();
expect(telemetry).toMatchObject({
enabled: true,
apiMetrics: true,
hostMetrics: true,
jobMetrics: false,
repoMetrics: true,
});
expect(telemetry.metrics).toEqual(
new Set([ImmichTelemetry.API, ImmichTelemetry.HOST, ImmichTelemetry.IO, ImmichTelemetry.REPO]),
);
});
it('should run with specific telemetry metrics', () => {
process.env.IMMICH_TELEMETRY_INCLUDE = 'io, host, api';
const { telemetry } = getEnv();
expect(telemetry.metrics).toEqual(new Set([ImmichTelemetry.API, ImmichTelemetry.HOST, ImmichTelemetry.IO]));
});
});
});

43
server/src/repositories/config.repository.ts
View file

@ -2,7 +2,7 @@ import { Injectable } from '@nestjs/common';
import { join } from 'node:path';
import { citiesFile, excludePaths } from 'src/constants';
import { Telemetry } from 'src/decorators';
import { ImmichEnvironment, ImmichWorker, LogLevel } from 'src/enum';
import { ImmichEnvironment, ImmichTelemetry, ImmichWorker, LogLevel } from 'src/enum';
import { EnvData, IConfigRepository } from 'src/interfaces/config.interface';
import { DatabaseExtension } from 'src/interfaces/database.interface';
import { QueueName } from 'src/interfaces/job.interface';
@ -25,18 +25,17 @@ const stagingKeys = {
};
const WORKER_TYPES = new Set(Object.values(ImmichWorker));
const TELEMETRY_TYPES = new Set(Object.values(ImmichTelemetry));
const asSet = (value: string | undefined, defaults: ImmichWorker[]) => {
const asSet = <T>(value: string | undefined, defaults: T[]) => {
const values = (value || '').replaceAll(/\s/g, '').split(',').filter(Boolean);
return new Set(values.length === 0 ? defaults : (values as ImmichWorker[]));
return new Set(values.length === 0 ? defaults : (values as T[]));
};
const parseBoolean = (value: string | undefined, defaultValue: boolean) => (value ? value === 'true' : defaultValue);
const getEnv = (): EnvData => {
const included = asSet(process.env.IMMICH_WORKERS_INCLUDE, [ImmichWorker.API, ImmichWorker.MICROSERVICES]);
const excluded = asSet(process.env.IMMICH_WORKERS_EXCLUDE, []);
const workers = [...setDifference(included, excluded)];
const includedWorkers = asSet(process.env.IMMICH_WORKERS_INCLUDE, [ImmichWorker.API, ImmichWorker.MICROSERVICES]);
const excludedWorkers = asSet(process.env.IMMICH_WORKERS_EXCLUDE, []);
const workers = [...setDifference(includedWorkers, excludedWorkers)];
for (const worker of workers) {
if (!WORKER_TYPES.has(worker)) {
throw new Error(`Invalid worker(s) found: ${workers.join(',')}`);
@ -69,12 +68,18 @@ const getEnv = (): EnvData => {
}
}
const globalEnabled = parseBoolean(process.env.IMMICH_METRICS, false);
const hostMetrics = parseBoolean(process.env.IMMICH_HOST_METRICS, globalEnabled);
const apiMetrics = parseBoolean(process.env.IMMICH_API_METRICS, globalEnabled);
const repoMetrics = parseBoolean(process.env.IMMICH_IO_METRICS, globalEnabled);
const jobMetrics = parseBoolean(process.env.IMMICH_JOB_METRICS, globalEnabled);
const telemetryEnabled = globalEnabled || hostMetrics || apiMetrics || repoMetrics || jobMetrics;
const includedTelemetries =
process.env.IMMICH_TELEMETRY_INCLUDE === 'all'
? new Set(Object.values(ImmichTelemetry))
: asSet<ImmichTelemetry>(process.env.IMMICH_TELEMETRY_INCLUDE, []);
const excludedTelemetries = asSet<ImmichTelemetry>(process.env.IMMICH_TELEMETRY_EXCLUDE, []);
const telemetries = setDifference(includedTelemetries, excludedTelemetries);
for (const telemetry of telemetries) {
if (!TELEMETRY_TYPES.has(telemetry)) {
throw new Error(`Invalid telemetry found: ${telemetry}`);
}
}
return {
host: process.env.IMMICH_HOST,
@ -136,9 +141,9 @@ const getEnv = (): EnvData => {
otel: {
metrics: {
hostMetrics,
hostMetrics: telemetries.has(ImmichTelemetry.HOST),
apiMetrics: {
enable: apiMetrics,
enable: telemetries.has(ImmichTelemetry.API),
ignoreRoutes: excludePaths,
},
},
@ -168,11 +173,7 @@ const getEnv = (): EnvData => {
telemetry: {
apiPort: Number(process.env.IMMICH_API_METRICS_PORT || '') || 8081,
microservicesPort: Number(process.env.IMMICH_MICROSERVICES_METRICS_PORT || '') || 8082,
enabled: telemetryEnabled,
hostMetrics,
apiMetrics,
repoMetrics,
jobMetrics,
metrics: telemetries,
},
workers,

15
server/src/repositories/telemetry.repository.ts
View file

@ -14,7 +14,7 @@ import { snakeCase, startCase } from 'lodash';
import { MetricService } from 'nestjs-otel';
import { copyMetadataFromFunctionToFunction } from 'nestjs-otel/lib/opentelemetry.utils';
import { serverVersion } from 'src/constants';
import { MetadataKey } from 'src/enum';
import { ImmichTelemetry, MetadataKey } from 'src/enum';
import { IConfigRepository } from 'src/interfaces/config.interface';
import { ILoggerRepository } from 'src/interfaces/logger.interface';
import { IMetricGroupRepository, ITelemetryRepository, MetricGroupOptions } from 'src/interfaces/telemetry.interface';
@ -99,17 +99,18 @@ export class TelemetryRepository implements ITelemetryRepository {
@Inject(ILoggerRepository) private logger: ILoggerRepository,
) {
const { telemetry } = this.configRepository.getEnv();
const { apiMetrics, hostMetrics, jobMetrics, repoMetrics } = telemetry;
const { metrics } = telemetry;
this.api = new MetricGroupRepository(metricService).configure({ enabled: apiMetrics });
this.host = new MetricGroupRepository(metricService).configure({ enabled: hostMetrics });
this.jobs = new MetricGroupRepository(metricService).configure({ enabled: jobMetrics });
this.repo = new MetricGroupRepository(metricService).configure({ enabled: repoMetrics });
this.api = new MetricGroupRepository(metricService).configure({ enabled: metrics.has(ImmichTelemetry.API) });
this.host = new MetricGroupRepository(metricService).configure({ enabled: metrics.has(ImmichTelemetry.HOST) });
this.jobs = new MetricGroupRepository(metricService).configure({ enabled: metrics.has(ImmichTelemetry.JOB) });
this.repo = new MetricGroupRepository(metricService).configure({ enabled: metrics.has(ImmichTelemetry.REPO) });
}
setup({ repositories }: { repositories: ClassConstructor<unknown>[] }) {
const { telemetry } = this.configRepository.getEnv();
if (!telemetry.enabled || !telemetry.repoMetrics) {
const { metrics } = telemetry;
if (!metrics.has(ImmichTelemetry.REPO)) {
return;
}

2
server/src/workers/api.ts
View file

@ -20,7 +20,7 @@ async function bootstrap() {
process.title = 'immich-api';
const { telemetry, network } = new ConfigRepository().getEnv();
if (telemetry.enabled) {
if (telemetry.metrics.size > 0) {
bootstrapTelemetry(telemetry.apiPort);
}

2
server/src/workers/microservices.ts
View file

@ -11,7 +11,7 @@ import { isStartUpError } from 'src/services/storage.service';
export async function bootstrap() {
const { telemetry } = new ConfigRepository().getEnv();
if (telemetry.enabled) {
if (telemetry.metrics.size > 0) {
bootstrapTelemetry(telemetry.microservicesPort);
}