Add support for custom PostgreSQL configuration #1024

.github/workflows/ci-docker.yml

@@ -16,5 +16,8 @@ jobs:
       - name: Build and run the Docker Compose stack
         run: docker compose up -d
 
+      - name: DEBUG postgres conf
+        run: docker compose run db psql --username=scancodeio scancodeio -c "TABLE pg_file_settings;"
+
       - name: Run tests
         run: docker compose run -e FROM_DOCKER_IMAGE=1 web python ./manage.py test --verbosity=2 --noinput

CHANGELOG.rst

@@ -7,6 +7,9 @@ v33.0.0 (unreleased)
 - Upgrade Django to version 5.0 and drop support for Python 3.8 and 3.9
   https://github.com/nexB/scancode.io/issues/1020
 
+- Add support for custom PostgreSQL configuration file in the docker compose context.
+  https://github.com/nexB/scancode.io/issues/1024
+
 - Refactor run_scancode to not fail on scan errors happening at the resource level,
   such as a timeout. Project error message are created instead.
   https://github.com/nexB/scancode.io/issues/1018

Makefile

@@ -34,6 +34,9 @@ ENV_FILE=.env
 SCANCODEIO_DB_NAME=scancodeio
 SCANCODEIO_DB_USER=scancodeio
 SCANCODEIO_DB_PASSWORD=scancodeio
+DOCKER_COMPOSE=docker compose -f docker-compose.yml
+DOCKER_EXEC=${DOCKER_COMPOSE} exec
+DB_CONTAINER_NAME=db
 POSTGRES_INITDB_ARGS=--encoding=UTF-8 --lc-collate=en_US.UTF-8 --lc-ctype=en_US.UTF-8
 DATE=$(shell date +"%Y-%m-%d_%H%M")
 
@@ -158,4 +161,13 @@ offline-package: docker-images
 	@mkdir -p dist/
 	@tar -cf dist/scancodeio-offline-package-`git describe --tags`.tar build/
 
-.PHONY: virtualenv conf dev envfile install check bandit valid isort check-deploy clean migrate postgresdb sqlitedb backupdb run test docs bump publish docker-images offline-package
+bash:
+	${DOCKER_EXEC} web bash
+
+shell:
+	${DOCKER_EXEC} web ./manage.py shell
+
+psql:
+	${DOCKER_EXEC} ${DB_CONTAINER_NAME} psql --username=${SCANCODEIO_DB_USER} postgres
+
+.PHONY: virtualenv conf dev envfile install check bandit valid isort check-deploy clean migrate postgresdb sqlitedb backupdb run test docs bump publish docker-images offline-package bash shell psql

docker-compose.yml

@@ -7,6 +7,7 @@ services:
       - docker.env
     volumes:
       - db_data:/var/lib/postgresql/data/
+#      - ./etc/postgresql/postgresql.conf:/var/lib/postgresql/data/postgresql.conf
     shm_size: "1gb"
     restart: always
 

docs/faq.rst

@@ -137,3 +137,19 @@ How can I trigger a pipeline scan from a CI/CD, such as Jenkins, TeamCity or Azu
 ------------------------------------------------------------------------------------------
 
 You can use the :ref:`rest_api` to automate your project or pipeline management.
+
+Optimizing Database Performance
+-------------------------------
+
+To enhance the performance of the database, it is highly recommended to leverage
+tools like PGTune.
+PGTune is designed to generate an optimized configuration for PostgreSQL,
+tailored to the maximum performance achievable on a specific hardware configuration.
+
+1. Utilize `PGTune <https://pgtune.leopard.in.ua/>`_  to generate a configuration
+file optimized for PostgreSQL based on your hardware specifications.
+
+2. Replace the default configuration, at ``./etc/postgresql/postgresql.conf``,
+with your custom configuration.
+
+3. Start, or restart, the Docker containers.