diff --git a/.gitignore b/.gitignore index 189f9046..95560ae5 100644 --- a/.gitignore +++ b/.gitignore @@ -177,6 +177,7 @@ dist pgdata/ minio_data/ dump/ +minio-dump/ *.csv *.sql \ No newline at end of file diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index a78e1b0b..642821f7 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -64,6 +64,9 @@ services: - type: bind source: ./dump target: /tmp/dump + - type: bind + source: ./minio-dump + target: /tmp/minio-dump minio: container_name: interapp-minio diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 4feaf13c..94b569b1 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -42,7 +42,7 @@ services: context: ./interapp-backend dockerfile: scheduler/prod.Dockerfile env_file: - - ./interapp-backend/.env.development + - ./interapp-backend/.env.production networks: - interapp-network depends_on: @@ -52,6 +52,7 @@ services: condition: service_healthy volumes: - ./dump:/tmp/dump + - ./minio-dump:/tmp/minio-dump minio: container_name: interapp-minio diff --git a/interapp-backend/scheduler/dev.Dockerfile b/interapp-backend/scheduler/dev.Dockerfile index 3785c337..fbd235aa 100644 --- a/interapp-backend/scheduler/dev.Dockerfile +++ b/interapp-backend/scheduler/dev.Dockerfile @@ -15,6 +15,19 @@ RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg m # update and install postgresql-client-16, tzdata RUN apt-get update && apt-get install -y --fix-missing postgresql-client-16 tzdata && apt-get clean +RUN ARCH=$(case "$(uname -m)" in "x86_64") echo "amd64";; "ppc64le") echo "ppc64le";; *) echo "Unsupported architecture"; exit 1;; esac) && \ + DOWNLOAD_URL=$(case "$ARCH" in "amd64") echo "https://dl.min.io/client/mc/release/linux-amd64/mc";; "ppc64le") echo "https://dl.min.io/client/mc/release/linux-ppc64le/mc";; *) echo "Unsupported architecture"; exit 1;; esac) && \ + # Install wget to download MinIO client + apt-get update && apt-get install -y wget && \ + # Download MinIO client binary + wget -O /usr/local/bin/mc "$DOWNLOAD_URL" && \ + # Make MinIO client binary executable + chmod +x /usr/local/bin/mc && \ + # Clean up + apt-get clean && rm -rf /var/lib/apt/lists/* && \ + # Output success message + echo "MinIO client installed successfully." + ENV TZ=Asia/Singapore RUN ln -snf /usr/share/zoneinfo/"$TZ" /etc/localtime && echo "$TZ" > /etc/timezone RUN apt-get clean diff --git a/interapp-backend/scheduler/prod.Dockerfile b/interapp-backend/scheduler/prod.Dockerfile index ce249c53..99f72104 100644 --- a/interapp-backend/scheduler/prod.Dockerfile +++ b/interapp-backend/scheduler/prod.Dockerfile @@ -15,6 +15,19 @@ RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg m # update and install postgresql-client-16, tzdata RUN apt-get update && apt-get install -y --fix-missing postgresql-client-16 tzdata && apt-get clean +RUN ARCH=$(case "$(uname -m)" in "x86_64") echo "amd64";; "ppc64le") echo "ppc64le";; *) echo "Unsupported architecture"; exit 1;; esac) && \ + DOWNLOAD_URL=$(case "$ARCH" in "amd64") echo "https://dl.min.io/client/mc/release/linux-amd64/mc";; "ppc64le") echo "https://dl.min.io/client/mc/release/linux-ppc64le/mc";; *) echo "Unsupported architecture"; exit 1;; esac) && \ + # Install wget to download MinIO client + apt-get update && apt-get install -y wget && \ + # Download MinIO client binary + wget -O /usr/local/bin/mc "$DOWNLOAD_URL" && \ + # Make MinIO client binary executable + chmod +x /usr/local/bin/mc && \ + # Clean up + apt-get clean && rm -rf /var/lib/apt/lists/* && \ + # Output success message + echo "MinIO client installed successfully." + ENV TZ=Asia/Singapore RUN ln -snf /usr/share/zoneinfo/"$TZ" /etc/localtime && echo "$TZ" > /etc/timezone RUN apt-get clean diff --git a/interapp-backend/scheduler/scheduler.ts b/interapp-backend/scheduler/scheduler.ts index 057ef3f9..19e4f905 100644 --- a/interapp-backend/scheduler/scheduler.ts +++ b/interapp-backend/scheduler/scheduler.ts @@ -155,7 +155,7 @@ schedule('0 0 0 */1 * *', async () => { await $`find ${path} -type f -mtime +7 -exec rm {} +`; const d = new Date(); - const fmted = `interapp_${d.toLocaleDateString('en-GB').replace(/\//g, '_')}`; + const fmted = `interapp_db_${d.toLocaleDateString('en-GB').replace(/\//g, '_')}`; const newFile = `${path}/${fmted}.sql`; await $`touch ${newFile}`; @@ -163,3 +163,41 @@ schedule('0 0 0 */1 * *', async () => { console.info('db snapshot taken at location: ', newFile); }); + +// Simplify environment variable handling and improve error messaging +const envVars = { + MINIO_ENDPOINT: process.env.MINIO_ENDPOINT, + MINIO_ADDRESS: process.env.MINIO_ADDRESS, + MINIO_ROOT_USER: process.env.MINIO_ROOT_USER, + MINIO_ROOT_PASSWORD: process.env.MINIO_ROOT_PASSWORD, + MINIO_BUCKETNAME: process.env.MINIO_BUCKETNAME, +}; +const missingEnv = Object.entries(envVars) + .filter(([, value]) => !value) + .map(([key]) => key); +if (missingEnv.length > 0) { + throw new Error(`Missing required environment variables: ${missingEnv.join(', ')}`); +} +const { MINIO_ENDPOINT, MINIO_ADDRESS, MINIO_ROOT_USER, MINIO_ROOT_PASSWORD, MINIO_BUCKETNAME } = + envVars; +const minioURL = `http://${MINIO_ENDPOINT}${MINIO_ADDRESS}`; +const minioAccessKey = MINIO_ROOT_USER; +const minioSecretKey = MINIO_ROOT_PASSWORD; +const minioBucketName = MINIO_BUCKETNAME; +const minioAliasName = 'minio'; +const minioBackupTask = schedule( + '0 0 0 */1 * *', + async () => { + const path = '/tmp/minio-dump'; + if (!existsSync(path)) mkdirSync(path); + const d = new Date(); + const fmted = `interapp_minio_${d.toLocaleDateString('en-GB').replace(/\//g, '_')}`; + const newFile = `${path}/${fmted}.tar.gz`; + await $`mc mirror ${minioAliasName}/${minioBucketName} /tmp/minio-dump/temp`; + await $`cd /tmp && tar -cvf ${newFile} minio-dump/temp`; + await $`rm -rf /tmp/minio-dump/temp`; + }, + { scheduled: false }, +); +await $`mc alias set ${minioAliasName} ${minioURL} ${minioAccessKey} ${minioSecretKey}`; +minioBackupTask.start();