From f00d24c661a33215aea4059fa4864f1f54577653 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Fri, 15 Sep 2023 17:48:25 -0400 Subject: [PATCH 1/6] [2.x] Support postgres --- db/knex_init_db.js | 5 +-- package-lock.json | 90 ++++++++++++++++++++++++++++++++++++++++++++++ package.json | 1 + server/database.js | 54 +++++++++++++++++++++++++--- 4 files changed, 144 insertions(+), 6 deletions(-) diff --git a/db/knex_init_db.js b/db/knex_init_db.js index 2a27fa1d7b..744153ca3f 100644 --- a/db/knex_init_db.js +++ b/db/knex_init_db.js @@ -5,10 +5,11 @@ const { log } = require("../src/util"); * ⚠️⚠️⚠️⚠️⚠️⚠️ DO NOT ADD ANYTHING HERE! * IF YOU NEED TO ADD FIELDS, ADD IT TO ./db/knex_migrations * See ./db/knex_migrations/README.md for more information + * @param {string} dbType "mariadb" or "postgres" * @returns {Promise} */ -async function createTables() { - log.info("mariadb", "Creating basic tables for MariaDB"); +async function createTables(dbType) { + log.info(dbType, "Creating basic tables"); const knex = R.knex; // TODO: Should check later if it is really the final patch sql file. diff --git a/package-lock.json b/package-lock.json index 4475c5f855..4b548ea319 100644 --- a/package-lock.json +++ b/package-lock.json @@ -86,6 +86,7 @@ "@fortawesome/vue-fontawesome": "~3.0.0-5", "@popperjs/core": "~2.10.2", "@types/bootstrap": "~5.1.9", + "@types/pg": "^8.10.2", "@vitejs/plugin-legacy": "~4.1.0", "@vitejs/plugin-vue": "~4.2.3", "@vue/compiler-sfc": "~3.3.4", @@ -5694,6 +5695,74 @@ "integrity": "sha512-Gj7cI7z+98M282Tqmp2K5EIsoouUEzbBJhQQzDE3jSIRk6r9gsz0oUokqIUR4u1R3dMHo0pDHM7sNOHyhulypw==", "dev": true }, + "node_modules/@types/pg": { + "version": "8.10.2", + "resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.10.2.tgz", + "integrity": "sha512-MKFs9P6nJ+LAeHLU3V0cODEOgyThJ3OAnmOlsZsxux6sfQs3HRXR5bBn7xG5DjckEFhTAxsXi7k7cd0pCMxpJw==", + "dev": true, + "dependencies": { + "@types/node": "*", + "pg-protocol": "*", + "pg-types": "^4.0.1" + } + }, + "node_modules/@types/pg/node_modules/pg-types": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-4.0.1.tgz", + "integrity": "sha512-hRCSDuLII9/LE3smys1hRHcu5QGcLs9ggT7I/TCs0IE+2Eesxi9+9RWAAwZ0yaGjxoWICF/YHLOEjydGujoJ+g==", + "dev": true, + "dependencies": { + "pg-int8": "1.0.1", + "pg-numeric": "1.0.2", + "postgres-array": "~3.0.1", + "postgres-bytea": "~3.0.0", + "postgres-date": "~2.0.1", + "postgres-interval": "^3.0.0", + "postgres-range": "^1.1.1" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/@types/pg/node_modules/postgres-array": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-3.0.2.tgz", + "integrity": "sha512-6faShkdFugNQCLwucjPcY5ARoW1SlbnrZjmGl0IrrqewpvxvhSLHimCVzqeuULCbG0fQv7Dtk1yDbG3xv7Veog==", + "dev": true, + "engines": { + "node": ">=12" + } + }, + "node_modules/@types/pg/node_modules/postgres-bytea": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-3.0.0.tgz", + "integrity": "sha512-CNd4jim9RFPkObHSjVHlVrxoVQXz7quwNFpz7RY1okNNme49+sVyiTvTRobiLV548Hx/hb1BG+iE7h9493WzFw==", + "dev": true, + "dependencies": { + "obuf": "~1.1.2" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/@types/pg/node_modules/postgres-date": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-2.0.1.tgz", + "integrity": "sha512-YtMKdsDt5Ojv1wQRvUhnyDJNSr2dGIC96mQVKz7xufp07nfuFONzdaowrMHjlAzY6GDLd4f+LUHHAAM1h4MdUw==", + "dev": true, + "engines": { + "node": ">=12" + } + }, + "node_modules/@types/pg/node_modules/postgres-interval": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-3.0.0.tgz", + "integrity": "sha512-BSNDnbyZCXSxgA+1f5UU2GmwhoI0aU5yMxRGO8CdFEcY2BQF9xm/7MqKnYoM1nJDk8nONNWDk9WeSmePFhQdlw==", + "dev": true, + "engines": { + "node": ">=12" + } + }, "node_modules/@types/qs": { "version": "6.9.7", "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.7.tgz", @@ -14833,6 +14902,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/obuf": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/obuf/-/obuf-1.1.2.tgz", + "integrity": "sha512-PX1wu0AmAdPqOL1mWhqmlOd8kOIZQwGZw6rh7uby9fTc5lhaOWFLX3I6R1hrF9k3zUY40e6igsLGkDXK92LJNg==", + "dev": true + }, "node_modules/oidc-token-hash": { "version": "5.0.3", "resolved": "https://registry.npmjs.org/oidc-token-hash/-/oidc-token-hash-5.0.3.tgz", @@ -15213,6 +15288,15 @@ "node": ">=4.0.0" } }, + "node_modules/pg-numeric": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/pg-numeric/-/pg-numeric-1.0.2.tgz", + "integrity": "sha512-BM/Thnrw5jm2kKLE5uJkXqqExRUY/toLHda65XgFTBTFYZyopbKjBe29Ii3RbkvlsMoFwD+tHeGaCjjv0gHlyw==", + "dev": true, + "engines": { + "node": ">=4" + } + }, "node_modules/pg-pool": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.6.0.tgz", @@ -15495,6 +15579,12 @@ "node": ">=0.10.0" } }, + "node_modules/postgres-range": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/postgres-range/-/postgres-range-1.1.3.tgz", + "integrity": "sha512-VdlZoocy5lCP0c/t66xAfclglEapXPCIVhqqJRncYpvbCgImF0w67aPKfbqUMr72tO2k5q0TdTZwCLjPTI6C9g==", + "dev": true + }, "node_modules/prelude-ls": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", diff --git a/package.json b/package.json index 4814da6a7a..3e86a7ebcb 100644 --- a/package.json +++ b/package.json @@ -151,6 +151,7 @@ "@fortawesome/vue-fontawesome": "~3.0.0-5", "@popperjs/core": "~2.10.2", "@types/bootstrap": "~5.1.9", + "@types/pg": "^8.10.2", "@vitejs/plugin-legacy": "~4.1.0", "@vitejs/plugin-vue": "~4.2.3", "@vue/compiler-sfc": "~3.3.4", diff --git a/server/database.js b/server/database.js index 09d63aa323..bb7ab01608 100644 --- a/server/database.js +++ b/server/database.js @@ -6,6 +6,7 @@ const knex = require("knex"); const path = require("path"); const { EmbeddedMariaDB } = require("./embedded-mariadb"); const mysql = require("mysql2/promise"); +const pg = require("pg"); /** * Database & App Data Folder @@ -164,6 +165,18 @@ class Database { fs.writeFileSync(path.join(Database.dataDir, "db-config.json"), JSON.stringify(dbConfig, null, 4)); } + /** + * Validate a database name + * @param {string} dbName + * @throws {Error} If the database name is invalid + * @returns {void} + */ + static validateDBName(dbName) { + if (!/^\w+$/.test(dbName)) { + throw Error("Invalid database name. A database name can only consist of letters, numbers and underscores"); + } + } + /** * Connect to the database * @param {boolean} testMode Should the connection be started in test mode? @@ -195,7 +208,6 @@ class Database { log.info("db", `Database Type: ${dbConfig.type}`); if (dbConfig.type === "sqlite") { - if (! fs.existsSync(Database.sqlitePath)) { log.info("server", "Copying Database"); fs.copyFileSync(Database.templatePath, Database.sqlitePath); @@ -220,9 +232,7 @@ class Database { } }; } else if (dbConfig.type === "mariadb") { - if (!/^\w+$/.test(dbConfig.dbName)) { - throw Error("Invalid database name. A database name can only consist of letters, numbers and underscores"); - } + validateDBName(dbConfig.dbName); const connection = await mysql.createConnection({ host: dbConfig.hostname, @@ -259,6 +269,24 @@ class Database { }, pool: mariadbPoolConfig, }; + } else if (dbConfig.type === "postgres") { + validateDBName(dbConfig.dbName); + + const clientConfig = { + host: dbConfig.hostname, + port: dbConfig.port, + user: dbConfig.username, + password: dbConfig.password, + database: dbConfig.dbName, + }; + const client = new pg.Client(clientConfig); + await connection.execute("CREATE DATABASE IF NOT EXISTS " + dbConfig.dbName); + await client.end(); + + config = { + client: "pg", + connection: clientConfig, + }; } else { throw new Error("Unknown Database type: " + dbConfig.type); } @@ -291,6 +319,8 @@ class Database { await this.initSQLite(testMode, noLog); } else if (dbConfig.type.endsWith("mariadb")) { await this.initMariaDB(); + } else if (dbConfig.type === "postgres") { + await this.initPostgres(); } } @@ -340,6 +370,22 @@ class Database { } } + /** + * Initialize PostgresDB + * @returns {Promise} + */ + static async initPostgres() { + log.debug("db", "Checking if PostgresDB database exists..."); + + let hasTable = await R.hasTable("docker_host"); + if (!hasTable) { + const { createTables } = require("../db/knex_init_db"); + await createTables(); + } else { + log.debug("db", "PostgresDB database already exists"); + } + } + /** * Patch the database * @returns {void} From 6ffa4b707b9cba1f94432225bafeb2d5de807695 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Fri, 15 Sep 2023 17:59:19 -0400 Subject: [PATCH 2/6] Update query for pg --- server/database.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/database.js b/server/database.js index bb7ab01608..889933227b 100644 --- a/server/database.js +++ b/server/database.js @@ -707,8 +707,10 @@ class Database { static sqlHourOffset() { if (Database.dbConfig.type === "sqlite") { return "DATETIME('now', ? || ' hours')"; - } else { + } else if (Database.dbConfig.type.endsWith("mariadb")) { return "DATE_ADD(NOW(), INTERVAL ? HOUR)"; + } else if (Database.dbConfig.type === "postgres") { + return "NOW() + INTERVAL '? HOUR'"; } } From 6a2e5202e2f1feace07d16c7db45751c54e87af5 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Fri, 15 Sep 2023 18:01:15 -0400 Subject: [PATCH 3/6] Fix errors --- server/database.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/database.js b/server/database.js index 889933227b..57e93d6955 100644 --- a/server/database.js +++ b/server/database.js @@ -232,7 +232,7 @@ class Database { } }; } else if (dbConfig.type === "mariadb") { - validateDBName(dbConfig.dbName); + this.validateDBName(dbConfig.dbName); const connection = await mysql.createConnection({ host: dbConfig.hostname, @@ -270,7 +270,7 @@ class Database { pool: mariadbPoolConfig, }; } else if (dbConfig.type === "postgres") { - validateDBName(dbConfig.dbName); + this.validateDBName(dbConfig.dbName); const clientConfig = { host: dbConfig.hostname, @@ -280,7 +280,7 @@ class Database { database: dbConfig.dbName, }; const client = new pg.Client(clientConfig); - await connection.execute("CREATE DATABASE IF NOT EXISTS " + dbConfig.dbName); + await client.execute("CREATE DATABASE IF NOT EXISTS " + dbConfig.dbName); await client.end(); config = { From e83c8b99ac3c576bf25c34f87b3190c32f5a9266 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Fri, 15 Sep 2023 18:03:58 -0400 Subject: [PATCH 4/6] Fix jsdoc errors --- server/database.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/database.js b/server/database.js index 57e93d6955..8842fbe998 100644 --- a/server/database.js +++ b/server/database.js @@ -167,7 +167,7 @@ class Database { /** * Validate a database name - * @param {string} dbName + * @param {string} dbName Database name to validate * @throws {Error} If the database name is invalid * @returns {void} */ @@ -703,6 +703,7 @@ class Database { /** * @returns {string} Get the SQL for the current time plus a number of hours + * @throws {Error} If the database type is unknown */ static sqlHourOffset() { if (Database.dbConfig.type === "sqlite") { @@ -712,6 +713,7 @@ class Database { } else if (Database.dbConfig.type === "postgres") { return "NOW() + INTERVAL '? HOUR'"; } + throw new Error("Unknown database type: " + Database.dbConfig.type); } } From cc9f37e971149e228078781c4285e2ad1f3624cc Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Fri, 15 Sep 2023 18:14:56 -0400 Subject: [PATCH 5/6] Update jsdoc `type` type --- server/database.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/database.js b/server/database.js index 8842fbe998..070637806a 100644 --- a/server/database.js +++ b/server/database.js @@ -138,7 +138,7 @@ class Database { * Read the database config * @throws {Error} If the config is invalid * @typedef {string|undefined} envString - * @returns {{type: "sqlite"} | {type:envString, hostname:envString, port:envString, database:envString, username:envString, password:envString}} Database config + * @returns {{type: "sqlite" | "mariadb" | "embedded-mariadb" | "postgres"} | {type:envString, hostname:envString, port:envString, database:envString, username:envString, password:envString}} Database config */ static readDBConfig() { let dbConfig; @@ -158,7 +158,7 @@ class Database { /** * @typedef {string|undefined} envString - * @param {{type: "sqlite"} | {type:envString, hostname:envString, port:envString, database:envString, username:envString, password:envString}} dbConfig the database configuration that should be written + * @param {{type: "sqlite" | "mariadb" | "embedded-mariadb" | "postgres"} | {type:envString, hostname:envString, port:envString, database:envString, username:envString, password:envString}} dbConfig the database configuration that should be written * @returns {void} */ static writeDBConfig(dbConfig) { From 6065940269c6190e29d0d02ca228bacba5adea24 Mon Sep 17 00:00:00 2001 From: Chongyi Zheng Date: Sat, 16 Sep 2023 01:30:30 -0400 Subject: [PATCH 6/6] Fix comments regarding typing --- db/knex_init_db.js | 2 +- server/database.js | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/db/knex_init_db.js b/db/knex_init_db.js index 744153ca3f..c97cbf8e11 100644 --- a/db/knex_init_db.js +++ b/db/knex_init_db.js @@ -5,7 +5,7 @@ const { log } = require("../src/util"); * ⚠️⚠️⚠️⚠️⚠️⚠️ DO NOT ADD ANYTHING HERE! * IF YOU NEED TO ADD FIELDS, ADD IT TO ./db/knex_migrations * See ./db/knex_migrations/README.md for more information - * @param {string} dbType "mariadb" or "postgres" + * @param {"mariadb"|"postgres"} dbType database type, should be either "mariadb" or "postgres" * @returns {Promise} */ async function createTables(dbType) { diff --git a/server/database.js b/server/database.js index 070637806a..8842fbe998 100644 --- a/server/database.js +++ b/server/database.js @@ -138,7 +138,7 @@ class Database { * Read the database config * @throws {Error} If the config is invalid * @typedef {string|undefined} envString - * @returns {{type: "sqlite" | "mariadb" | "embedded-mariadb" | "postgres"} | {type:envString, hostname:envString, port:envString, database:envString, username:envString, password:envString}} Database config + * @returns {{type: "sqlite"} | {type:envString, hostname:envString, port:envString, database:envString, username:envString, password:envString}} Database config */ static readDBConfig() { let dbConfig; @@ -158,7 +158,7 @@ class Database { /** * @typedef {string|undefined} envString - * @param {{type: "sqlite" | "mariadb" | "embedded-mariadb" | "postgres"} | {type:envString, hostname:envString, port:envString, database:envString, username:envString, password:envString}} dbConfig the database configuration that should be written + * @param {{type: "sqlite"} | {type:envString, hostname:envString, port:envString, database:envString, username:envString, password:envString}} dbConfig the database configuration that should be written * @returns {void} */ static writeDBConfig(dbConfig) {