feat: microservices be gone (immich-app#9551)

* feat: microservices be gone and api is a worker now too * chore: remove very old startup scripts, surely nobody is using these anymore, right? right?....
mmomjian · May 17, 2024 · 85aca2b · 85aca2b
1 parent ff52300
commit 85aca2b
Show file tree

Hide file tree

Showing 12 changed files with 204 additions and 188 deletions.
diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml
@@ -4,52 +4,36 @@
 
 name: immich-dev
 
-x-server-build: &server-common
-  image: immich-server-dev:latest
-  build:
-    context: ../
-    dockerfile: server/Dockerfile
-    target: dev
-  restart: always
-  volumes:
-    - ../server:/usr/src/app
-    - ../open-api:/usr/src/open-api
-    - ${UPLOAD_LOCATION}/photos:/usr/src/app/upload
-    - ${UPLOAD_LOCATION}/photos/upload:/usr/src/app/upload/upload
-    - /usr/src/app/node_modules
-    - /etc/localtime:/etc/localtime:ro
-  env_file:
-    - .env
-  ulimits:
-    nofile:
-      soft: 1048576
-      hard: 1048576
-
 services:
   immich-server:
     container_name: immich_server
-    command: ['/usr/src/app/bin/immich-dev', 'immich']
-    <<: *server-common
+    command: ['/usr/src/app/bin/immich-dev']
+    image: immich-server-dev:latest
+    build:
+      context: ../
+      dockerfile: server/Dockerfile
+      target: dev
+    restart: always
+    volumes:
+      - ../server:/usr/src/app
+      - ../open-api:/usr/src/open-api
+      - ${UPLOAD_LOCATION}/photos:/usr/src/app/upload
+      - ${UPLOAD_LOCATION}/photos/upload:/usr/src/app/upload/upload
+      - /usr/src/app/node_modules
+      - /etc/localtime:/etc/localtime:ro
+    env_file:
+      - .env
+    ulimits:
+      nofile:
+        soft: 1048576
+        hard: 1048576
     ports:
       - 3001:3001
       - 9230:9230
     depends_on:
       - redis
       - database
 
-  immich-microservices:
-    container_name: immich_microservices
-    command: ['/usr/src/app/bin/immich-dev', 'microservices']
-    <<: *server-common
-    # extends:
-    #   file: hwaccel.transcoding.yml
-    #   service: cpu # set to one of [nvenc, quicksync, rkmpp, vaapi, vaapi-wsl] for accelerated transcoding
-    ports:
-      - 9231:9230
-    depends_on:
-      - database
-      - immich-server
-
   immich-web:
     container_name: immich_web
     image: immich-web-dev:latest

diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml
@@ -1,40 +1,24 @@
 name: immich-prod
 
-x-server-build: &server-common
-  image: immich-server:latest
-  build:
-    context: ../
-    dockerfile: server/Dockerfile
-  volumes:
-    - ${UPLOAD_LOCATION}/photos:/usr/src/app/upload
-    - /etc/localtime:/etc/localtime:ro
-  env_file:
-    - .env
-  restart: always
-
 services:
   immich-server:
     container_name: immich_server
-    command: ['start.sh', 'immich']
-    <<: *server-common
+    image: immich-server:latest
+    build:
+      context: ../
+      dockerfile: server/Dockerfile
+    volumes:
+      - ${UPLOAD_LOCATION}/photos:/usr/src/app/upload
+      - /etc/localtime:/etc/localtime:ro
+    env_file:
+      - .env
+    restart: always
     ports:
       - 2283:3001
     depends_on:
       - redis
       - database
 
-  immich-microservices:
-    container_name: immich_microservices
-    command: ['start.sh', 'microservices']
-    <<: *server-common
-    # extends:
-    #   file: hwaccel.transcoding.yml
-    #   service: cpu # set to one of [nvenc, quicksync, rkmpp, vaapi, vaapi-wsl] for accelerated transcoding
-    depends_on:
-      - redis
-      - database
-      - immich-server
-
   immich-machine-learning:
     container_name: immich_machine_learning
     image: immich-machine-learning:latest

diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
@@ -12,7 +12,6 @@ services:
   immich-server:
     container_name: immich_server
     image: ghcr.io/immich-app/immich-server:${IMMICH_VERSION:-release}
-    command: ['start.sh', 'immich']
     volumes:
       - ${UPLOAD_LOCATION}:/usr/src/app/upload
       - /etc/localtime:/etc/localtime:ro
@@ -25,23 +24,6 @@ services:
       - database
     restart: always
 
-  immich-microservices:
-    container_name: immich_microservices
-    image: ghcr.io/immich-app/immich-server:${IMMICH_VERSION:-release}
-    # extends: # uncomment this section for hardware acceleration - see https://immich.app/docs/features/hardware-transcoding
-    #   file: hwaccel.transcoding.yml
-    #   service: cpu # set to one of [nvenc, quicksync, rkmpp, vaapi, vaapi-wsl] for accelerated transcoding
-    command: ['start.sh', 'microservices']
-    volumes:
-      - ${UPLOAD_LOCATION}:/usr/src/app/upload
-      - /etc/localtime:/etc/localtime:ro
-    env_file:
-      - .env
-    depends_on:
-      - redis
-      - database
-    restart: always
-
   immich-machine-learning:
     container_name: immich_machine_learning
     # For hardware acceleration, add one of -[armnn, cuda, openvino] to the image tag.

diff --git a/e2e/docker-compose.yml b/e2e/docker-compose.yml
@@ -2,38 +2,30 @@ version: '3.8'
 
 name: immich-e2e
 
-x-server-build: &server-common
-  image: immich-server:latest
-  build:
-    context: ../
-    dockerfile: server/Dockerfile
-  environment:
-    - DB_HOSTNAME=database
-    - DB_USERNAME=postgres
-    - DB_PASSWORD=postgres
-    - DB_DATABASE_NAME=immich
-    - IMMICH_MACHINE_LEARNING_ENABLED=false
-    - IMMICH_METRICS=true
-  volumes:
-    - upload:/usr/src/app/upload
-    - ./test-assets:/test-assets
-  depends_on:
-    - redis
-    - database
-
 services:
   immich-server:
     container_name: immich-e2e-server
-    command: ['./start.sh', 'immich']
-    <<: *server-common
+    command: ['./start.sh']
+    image: immich-server:latest
+    build:
+      context: ../
+      dockerfile: server/Dockerfile
+    environment:
+      - DB_HOSTNAME=database
+      - DB_USERNAME=postgres
+      - DB_PASSWORD=postgres
+      - DB_DATABASE_NAME=immich
+      - IMMICH_MACHINE_LEARNING_ENABLED=false
+      - IMMICH_METRICS=true
+    volumes:
+      - upload:/usr/src/app/upload
+      - ./test-assets:/test-assets
+    depends_on:
+      - redis
+      - database
     ports:
       - 2283:3001
 
-  immich-microservices:
-    container_name: immich-e2e-microservices
-    command: ['./start.sh', 'microservices']
-    <<: *server-common
-
   redis:
     image: redis:6.2-alpine@sha256:84882e87b54734154586e5f8abd4dce69fe7311315e2fc6d67c29614c8de2672
 

diff --git a/server/Dockerfile b/server/Dockerfile
@@ -61,3 +61,4 @@ ENV PATH="${PATH}:/usr/src/app/bin"
 VOLUME /usr/src/app/upload
 EXPOSE 3001
 ENTRYPOINT ["tini", "--", "/bin/bash"]
+CMD ["start.sh"]
diff --git a/server/src/main.ts b/server/src/main.ts
@@ -1,67 +1,8 @@
-import { NestFactory } from '@nestjs/core';
-import { NestExpressApplication } from '@nestjs/platform-express';
-import { json } from 'body-parser';
-import cookieParser from 'cookie-parser';
 import { CommandFactory } from 'nest-commander';
-import { existsSync } from 'node:fs';
 import { Worker } from 'node:worker_threads';
-import sirv from 'sirv';
-import { ApiModule, ImmichAdminModule } from 'src/app.module';
+import { ImmichAdminModule } from 'src/app.module';
 import { LogLevel } from 'src/config';
-import { WEB_ROOT, envName, excludePaths, isDev, serverVersion } from 'src/constants';
-import { ILoggerRepository } from 'src/interfaces/logger.interface';
-import { WebSocketAdapter } from 'src/middleware/websocket.adapter';
-import { ApiService } from 'src/services/api.service';
-import { otelSDK } from 'src/utils/instrumentation';
-import { useSwagger } from 'src/utils/misc';
-
-const host = process.env.HOST;
-
-async function bootstrapApi() {
-  otelSDK.start();
-
-  const port = Number(process.env.SERVER_PORT) || 3001;
-  const app = await NestFactory.create<NestExpressApplication>(ApiModule, { bufferLogs: true });
-  const logger = await app.resolve(ILoggerRepository);
-
-  logger.setAppName('ImmichServer');
-  logger.setContext('ImmichServer');
-  app.useLogger(logger);
-  app.set('trust proxy', ['loopback', 'linklocal', 'uniquelocal']);
-  app.set('etag', 'strong');
-  app.use(cookieParser());
-  app.use(json({ limit: '10mb' }));
-  if (isDev) {
-    app.enableCors();
-  }
-  app.useWebSocketAdapter(new WebSocketAdapter(app));
-  useSwagger(app, isDev);
-
-  app.setGlobalPrefix('api', { exclude: excludePaths });
-  if (existsSync(WEB_ROOT)) {
-    // copied from https://github.com/sveltejs/kit/blob/679b5989fe62e3964b9a73b712d7b41831aa1f07/packages/adapter-node/src/handler.js#L46
-    // provides serving of precompressed assets and caching of immutable assets
-    app.use(
-      sirv(WEB_ROOT, {
-        etag: true,
-        gzip: true,
-        brotli: true,
-        setHeaders: (res, pathname) => {
-          if (pathname.startsWith(`/_app/immutable`) && res.statusCode === 200) {
-            res.setHeader('cache-control', 'public,max-age=31536000,immutable');
-          }
-        },
-      }),
-    );
-  }
-  app.use(app.get(ApiService).ssr(excludePaths));
-
-  const server = await (host ? app.listen(port, host) : app.listen(port));
-  server.requestTimeout = 30 * 60 * 1000;
-
-  logger.log(`Immich Server is listening on ${await app.getUrl()} [v${serverVersion}] [${envName}] `);
-}
-
+import { getWorkers } from 'src/utils/workers';
 const immichApp = process.argv[2] || process.env.IMMICH_APP;
 
 if (process.argv[2] === immichApp) {
@@ -73,35 +14,35 @@ async function bootstrapImmichAdmin() {
   await CommandFactory.run(ImmichAdminModule);
 }
 
-function bootstrapMicroservicesWorker() {
-  const worker = new Worker('./dist/workers/microservices.js');
+function bootstrapWorker(name: string) {
+  console.log(`Starting ${name} worker`);
+  const worker = new Worker(`./dist/workers/${name}.js`);
   worker.on('exit', (exitCode) => {
     if (exitCode !== 0) {
-      console.error(`Microservices worker exited with code ${exitCode}`);
+      console.error(`${name} worker exited with code ${exitCode}`);
       process.exit(exitCode);
     }
   });
 }
 
 function bootstrap() {
   switch (immichApp) {
+    case 'immich-admin': {
+      process.title = 'immich_admin_cli';
+      return bootstrapImmichAdmin();
+    }
     case 'immich': {
       process.title = 'immich_server';
-      if (process.env.INTERNAL_MICROSERVICES === 'true') {
-        bootstrapMicroservicesWorker();
-      }
-      return bootstrapApi();
+      return bootstrapWorker('api');
     }
     case 'microservices': {
       process.title = 'immich_microservices';
-      return bootstrapMicroservicesWorker();
-    }
-    case 'immich-admin': {
-      process.title = 'immich_admin_cli';
-      return bootstrapImmichAdmin();
+      return bootstrapWorker('microservices');
     }
     default: {
-      throw new Error(`Invalid app name: ${immichApp}. Expected one of immich|microservices|immich-admin`);
+      for (const worker of getWorkers()) {
+        bootstrapWorker(worker);
+      }
     }
   }
 }

diff --git a/server/src/utils/workers.spec.ts b/server/src/utils/workers.spec.ts
@@ -0,0 +1,49 @@
+import { getWorkers } from 'src/utils/workers';
+
+describe('getWorkers', () => {
+  beforeEach(() => {
+    process.env.IMMICH_WORKERS_INCLUDE = '';
+    process.env.IMMICH_WORKERS_EXCLUDE = '';
+  });
+
+  it('should return default workers', () => {
+    expect(getWorkers()).toEqual(['api', 'microservices']);
+  });
+
+  it('should return included workers', () => {
+    process.env.IMMICH_WORKERS_INCLUDE = 'api';
+    expect(getWorkers()).toEqual(['api']);
+  });
+
+  it('should excluded workers from defaults', () => {
+    process.env.IMMICH_WORKERS_EXCLUDE = 'api';
+    expect(getWorkers()).toEqual(['microservices']);
+  });
+
+  it('should exclude workers from include list', () => {
+    process.env.IMMICH_WORKERS_INCLUDE = 'api,microservices,randomservice';
+    process.env.IMMICH_WORKERS_EXCLUDE = 'randomservice,microservices';
+    expect(getWorkers()).toEqual(['api']);
+  });
+
+  it('should remove whitespace from included workers before parsing', () => {
+    process.env.IMMICH_WORKERS_INCLUDE = 'api, microservices';
+    expect(getWorkers()).toEqual(['api', 'microservices']);
+  });
+
+  it('should remove whitespace from excluded workers before parsing', () => {
+    process.env.IMMICH_WORKERS_EXCLUDE = 'api, microservices';
+    expect(getWorkers()).toEqual([]);
+  });
+
+  it('should remove whitespace from included and excluded workers before parsing', () => {
+    process.env.IMMICH_WORKERS_INCLUDE = 'api, microservices, randomservice,randomservice2';
+    process.env.IMMICH_WORKERS_EXCLUDE = 'randomservice,microservices, randomservice2';
+    expect(getWorkers()).toEqual(['api']);
+  });
+
+  it('should throw error for invalid workers', () => {
+    process.env.IMMICH_WORKERS_INCLUDE = 'api,microservices,randomservice';
+    expect(getWorkers).toThrowError('Invalid worker(s) found: api,microservices,randomservice');
+  });
+});