From e2b6428980e1b97b3e8e39dc5b166a92f72e2303 Mon Sep 17 00:00:00 2001 From: Louis Lam Date: Tue, 21 Mar 2023 16:45:46 +0800 Subject: [PATCH 01/16] Create UptimeCalculator --- server/uptime-calculator.js | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 server/uptime-calculator.js diff --git a/server/uptime-calculator.js b/server/uptime-calculator.js new file mode 100644 index 0000000000..685352abae --- /dev/null +++ b/server/uptime-calculator.js @@ -0,0 +1,7 @@ +class UptimeCalculator { + +} + +module.exports = { + UptimeCalculator +}; From 4a195b5e6435b52456f611e56ae37276871c74cb Mon Sep 17 00:00:00 2001 From: Louis Lam Date: Sat, 12 Aug 2023 22:02:26 +0800 Subject: [PATCH 02/16] WIP --- .eslintrc.js | 3 +- package.json | 5 +- server/uptime-calculator.js | 195 ++++++++++++++++++++ server/util-server.js | 2 +- test/backend-test-entry.js | 20 ++ test/backend-test/test-uptime-calculator.js | 181 ++++++++++++++++++ 6 files changed, 403 insertions(+), 3 deletions(-) create mode 100644 test/backend-test-entry.js create mode 100644 test/backend-test/test-uptime-calculator.js diff --git a/.eslintrc.js b/.eslintrc.js index bfe0404a6b..0f19793aa4 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -1,6 +1,7 @@ module.exports = { ignorePatterns: [ - "test/*", + "test/*.js", + "test/cypress", "server/modules/apicache/*", "src/util.js" ], diff --git a/package.json b/package.json index 38ea7249fa..2ad2b50bc4 100644 --- a/package.json +++ b/package.json @@ -24,8 +24,11 @@ "start-server": "node server/server.js", "start-server-dev": "cross-env NODE_ENV=development node server/server.js", "build": "vite build --config ./config/vite.config.js", - "test": "node test/prepare-test-server.js && npm run jest-backend", + "test": "node test/prepare-test-server.js && npm run test-backend", "test-with-build": "npm run build && npm test", + "test-backend": "node test/backend-test-entry.js && npm run jest-backend", + "test-backend:14": "cross-env TEST_BACKEND=1 NODE_OPTIONS=\"--experimental-abortcontroller --no-warnings\" node--test test/backend-test", + "test-backend:18": "cross-env TEST_BACKEND=1 node --test test/backend-test", "jest-backend": "cross-env TEST_BACKEND=1 jest --runInBand --detectOpenHandles --forceExit --config=./config/jest-backend.config.js", "tsc": "tsc", "vite-preview-dist": "vite preview --host --config ./config/vite.config.js", diff --git a/server/uptime-calculator.js b/server/uptime-calculator.js index 685352abae..7451e2400d 100644 --- a/server/uptime-calculator.js +++ b/server/uptime-calculator.js @@ -1,5 +1,200 @@ +const dayjs = require("dayjs"); +const { UP, MAINTENANCE, DOWN, PENDING } = require("../src/util"); + +/** + * Calculates the uptime of a monitor. + */ class UptimeCalculator { + /** + * For testing purposes, we can set the current date to a specific date. + * @type {dayjs.Dayjs} + */ + static currentDate = null; + + /** + * Recent 24-hour uptime, each item is a 1-minute interval + * Key: {number} DivisionKey + */ + uptimeDataList = { + + }; + + /** + * Daily uptime data, + * Key: {number} DailyKey + */ + dailyUptimeDataList = { + + }; + + lastDailyUptimeData = null; + + /** + * + */ + constructor() { + if (process.env.TEST_BACKEND) { + // Override the getCurrentDate() method to return a specific date + // Only for testing + this.getCurrentDate = () => { + if (UptimeCalculator.currentDate) { + return UptimeCalculator.currentDate; + } else { + return dayjs.utc(); + } + }; + } + } + + /** + * TODO + */ + init() { + } + + /** + * @param {number} status status + * @param {dayjs.Dayjs} date The heartbeat date + * @returns {dayjs.Dayjs} date + * @throws {Error} Invalid status + */ + update(status) { + let date = this.getCurrentDate(); + let flatStatus = this.flatStatus(status); + let divisionKey = this.getDivisionKey(date); + let dailyKey = this.getDailyKey(divisionKey); + + if (flatStatus === UP) { + this.uptimeDataList[divisionKey].uptime += 1; + this.dailyUptimeDataList[dailyKey].uptime += 1; + } else { + this.uptimeDataList[divisionKey].downtime += 1; + this.dailyUptimeDataList[dailyKey].downtime += 1; + } + + this.lastDailyUptimeData = this.dailyUptimeDataList[dailyKey]; + + this.clear(); + return date; + } + + /** + * @param {dayjs.Dayjs} date The heartbeat date + * @returns {number} division + */ + getDivisionKey(date) { + // Convert the current date to the nearest minute (e.g. 2021-01-01 12:34:56 -> 2021-01-01 12:34:00) + date = date.startOf("minute"); + + // Convert to timestamp in second + let divisionKey = date.unix(); + + if (! (divisionKey in this.uptimeDataList)) { + this.uptimeDataList[divisionKey] = { + uptime: 0, + downtime: 0, + }; + } + + return divisionKey; + } + + /** + * Convert timestamp to daily key + * @param {number} timestamp Timestamp + * @returns {number} dailyKey + */ + getDailyKey(timestamp) { + let date = dayjs.unix(timestamp); + + // Convert the date to the nearest day (e.g. 2021-01-01 12:34:56 -> 2021-01-01 00:00:00) + // Considering if the user keep changing could affect the calculation, so use UTC time to avoid this problem. + date = date.utc().startOf("day"); + let dailyKey = date.unix(); + + if (!this.dailyUptimeDataList[dailyKey]) { + this.dailyUptimeDataList[dailyKey] = { + uptime: 0, + downtime: 0, + }; + } + + return dailyKey; + } + + /** + * Flat status to UP or DOWN + * @param {number} status + * @returns {number} + * @throws {Error} Invalid status + */ + flatStatus(status) { + switch (status) { + case UP: + case MAINTENANCE: + return UP; + case DOWN: + case PENDING: + return DOWN; + } + throw new Error("Invalid status"); + } + + /** + * + */ + get24HourUptime() { + let dailyKey = this.getDailyKey(this.getCurrentDate().unix()); + let dailyUptimeData = this.dailyUptimeDataList[dailyKey]; + + // No data in last 24 hours, it could be a new monitor or the interval is larger than 24 hours + // Try to use previous data, if no previous data, return 0 + if (dailyUptimeData.uptime === 0 && dailyUptimeData.downtime === 0) { + if (this.lastDailyUptimeData) { + dailyUptimeData = this.lastDailyUptimeData; + } else { + return 0; + } + } + + return dailyUptimeData.uptime / (dailyUptimeData.uptime + dailyUptimeData.downtime); + } + + /** + * + */ + get7DayUptime() { + + } + + /** + * + */ + get30DayUptime() { + + } + + /** + * + */ + get1YearUptime() { + + } + + /** + * + */ + getCurrentDate() { + return dayjs.utc(); + } + + /** + * + */ + clear() { + + } } module.exports = { diff --git a/server/util-server.js b/server/util-server.js index 44500865e3..64c4a0e1cb 100644 --- a/server/util-server.js +++ b/server/util-server.js @@ -854,7 +854,7 @@ exports.doubleCheckPassword = async (socket, currentPassword) => { exports.startUnitTest = async () => { console.log("Starting unit test..."); const npm = /^win/.test(process.platform) ? "npm.cmd" : "npm"; - const child = childProcess.spawn(npm, [ "run", "jest-backend" ]); + const child = childProcess.spawn(npm, [ "run", "test-backend" ]); child.stdout.on("data", (data) => { console.log(data.toString()); diff --git a/test/backend-test-entry.js b/test/backend-test-entry.js new file mode 100644 index 0000000000..7cc8d734fa --- /dev/null +++ b/test/backend-test-entry.js @@ -0,0 +1,20 @@ +// Check Node.js version +const semver = require("semver"); +const childProcess = require("child_process"); + +const nodeVersion = process.versions.node; +console.log("Node.js version: " + nodeVersion); + + + +// Node.js version >= 18 +if (semver.satisfies(nodeVersion, ">= 18")) { + console.log("Use the native test runner: `node --test`"); + childProcess.execSync("npm run test-backend:18", { stdio: "inherit" }); +} else { + // 14 - 16 here + console.log("Use `test` package: `node--test`") + childProcess.execSync("npm run test-backend:14", { stdio: "inherit" }); +} + + diff --git a/test/backend-test/test-uptime-calculator.js b/test/backend-test/test-uptime-calculator.js new file mode 100644 index 0000000000..3febfeb667 --- /dev/null +++ b/test/backend-test/test-uptime-calculator.js @@ -0,0 +1,181 @@ +const { test } = require("node:test"); +const assert = require("node:assert"); +const { UptimeCalculator } = require("../../server/uptime-calculator"); +const dayjs = require("dayjs"); +const { UP, DOWN, PENDING, MAINTENANCE } = require("../../src/util"); +dayjs.extend(require("dayjs/plugin/utc")); +dayjs.extend(require("../../server/modules/dayjs/plugin/timezone")); +dayjs.extend(require("dayjs/plugin/customParseFormat")); + +test("Test Uptime Calculator - custom date", (t) => { + let c1 = new UptimeCalculator(); + + // Test custom date + UptimeCalculator.currentDate = dayjs.utc("2021-01-01T00:00:00.000Z"); + assert.strictEqual(c1.getCurrentDate().unix(), dayjs.utc("2021-01-01T00:00:00.000Z").unix()); +}); + +test("Test update - UP", (t) => { + UptimeCalculator.currentDate = dayjs.utc("2023-08-12 20:46:59"); + let c2 = new UptimeCalculator(); + let date = c2.update(UP); + assert.strictEqual(date.unix(), dayjs.utc("2023-08-12 20:46:59").unix()); +}); + +test("Test update - MAINTENANCE", (t) => { + UptimeCalculator.currentDate = dayjs.utc("2023-08-12 20:47:20"); + let c2 = new UptimeCalculator(); + let date = c2.update(MAINTENANCE); + assert.strictEqual(date.unix(), dayjs.utc("2023-08-12 20:47:20").unix()); +}); + +test("Test update - DOWN", (t) => { + UptimeCalculator.currentDate = dayjs.utc("2023-08-12 20:47:20"); + let c2 = new UptimeCalculator(); + let date = c2.update(DOWN); + assert.strictEqual(date.unix(), dayjs.utc("2023-08-12 20:47:20").unix()); +}); + +test("Test update - PENDING", (t) => { + UptimeCalculator.currentDate = dayjs.utc("2023-08-12 20:47:20"); + let c2 = new UptimeCalculator(); + let date = c2.update(PENDING); + assert.strictEqual(date.unix(), dayjs.utc("2023-08-12 20:47:20").unix()); +}); + +test("Test flatStatus", (t) => { + let c2 = new UptimeCalculator(); + assert.strictEqual(c2.flatStatus(UP), UP); + assert.strictEqual(c2.flatStatus(MAINTENANCE), UP); + assert.strictEqual(c2.flatStatus(DOWN), DOWN); + assert.strictEqual(c2.flatStatus(PENDING), DOWN); +}); + +test("Test update - getDivisionKey", (t) => { + let c2 = new UptimeCalculator(); + let divisionKey = c2.getDivisionKey(dayjs.utc("2023-08-12 20:46:00")); + assert.strictEqual(divisionKey, dayjs.utc("2023-08-12 20:46:00").unix()); + + // Edge case 1 + c2 = new UptimeCalculator(); + divisionKey = c2.getDivisionKey(dayjs.utc("2023-08-12 20:46:01")); + assert.strictEqual(divisionKey, dayjs.utc("2023-08-12 20:46:00").unix()); + + // Edge case 2 + c2 = new UptimeCalculator(); + divisionKey = c2.getDivisionKey(dayjs.utc("2023-08-12 20:46:59")); + assert.strictEqual(divisionKey, dayjs.utc("2023-08-12 20:46:00").unix()); +}); + +test("Test update - getDailyKey", (t) => { + let c2 = new UptimeCalculator(); + let dailyKey = c2.getDailyKey(dayjs.utc("2023-08-12 20:46:00").unix()); + assert.strictEqual(dailyKey, dayjs.utc("2023-08-12").unix()); + + c2 = new UptimeCalculator(); + dailyKey = c2.getDailyKey(dayjs.utc("2023-08-12 23:45:30").unix()); + assert.strictEqual(dailyKey, dayjs.utc("2023-08-12").unix()); + + // Edge case 1 + c2 = new UptimeCalculator(); + dailyKey = c2.getDailyKey(dayjs.utc("2023-08-12 23:59:59").unix()); + assert.strictEqual(dailyKey, dayjs.utc("2023-08-12").unix()); + + // Edge case 2 + c2 = new UptimeCalculator(); + dailyKey = c2.getDailyKey(dayjs.utc("2023-08-12 00:00:00").unix()); + assert.strictEqual(dailyKey, dayjs.utc("2023-08-12").unix()); +}); + +test("Test update - lastDailyUptimeData", (t) => { + let c2 = new UptimeCalculator(); + c2.update(UP); + assert.strictEqual(c2.lastDailyUptimeData.uptime, 1); +}); + +test("Test update - get24HourUptime", (t) => { + UptimeCalculator.currentDate = dayjs.utc("2023-08-12 20:46:59"); + + // No data + let c2 = new UptimeCalculator(); + let uptime = c2.get24HourUptime(); + assert.strictEqual(uptime, 0); + + // 1 Up + c2 = new UptimeCalculator(); + c2.update(UP); + uptime = c2.get24HourUptime(); + assert.strictEqual(uptime, 1); + + // 2 Up + c2 = new UptimeCalculator(); + c2.update(UP); + c2.update(UP); + uptime = c2.get24HourUptime(); + assert.strictEqual(uptime, 1); + + // 3 Up + c2 = new UptimeCalculator(); + c2.update(UP); + c2.update(UP); + c2.update(UP); + uptime = c2.get24HourUptime(); + assert.strictEqual(uptime, 1); + + // 1 MAINTENANCE + c2 = new UptimeCalculator(); + c2.update(MAINTENANCE); + uptime = c2.get24HourUptime(); + assert.strictEqual(uptime, 1); + + // 1 PENDING + c2 = new UptimeCalculator(); + c2.update(PENDING); + uptime = c2.get24HourUptime(); + assert.strictEqual(uptime, 0); + + // 1 DOWN + c2 = new UptimeCalculator(); + c2.update(DOWN); + uptime = c2.get24HourUptime(); + assert.strictEqual(uptime, 0); + + // 2 DOWN + c2 = new UptimeCalculator(); + c2.update(DOWN); + c2.update(DOWN); + uptime = c2.get24HourUptime(); + assert.strictEqual(uptime, 0); + + // 1 DOWN, 1 UP + c2 = new UptimeCalculator(); + c2.update(DOWN); + c2.update(UP); + uptime = c2.get24HourUptime(); + assert.strictEqual(uptime, 0.5); + + // 1 UP, 1 DOWN + c2 = new UptimeCalculator(); + c2.update(UP); + c2.update(DOWN); + uptime = c2.get24HourUptime(); + assert.strictEqual(uptime, 0.5); + + // Add 24 hours + c2 = new UptimeCalculator(); + c2.update(UP); + uptime = c2.get24HourUptime(); + assert.strictEqual(uptime, 1); + UptimeCalculator.currentDate = UptimeCalculator.currentDate.add(24, "hour"); + + // After 24 hours, even if there is no data, the uptime should be still 100% + uptime = c2.get24HourUptime(); + assert.strictEqual(uptime, 1); + + // Add more 24 hours (48 hours) + UptimeCalculator.currentDate = UptimeCalculator.currentDate.add(24, "hour"); + + // After 24 hours, even if there is no data, the uptime should be still 100% + uptime = c2.get24HourUptime(); + assert.strictEqual(uptime, 1); +}); From 26a17d5ce3a7b932937fe9aa15d0fc1c0c702c31 Mon Sep 17 00:00:00 2001 From: Louis Lam Date: Sat, 12 Aug 2023 23:19:09 +0800 Subject: [PATCH 03/16] WIP --- server/uptime-calculator.js | 40 ++++++++- test/backend-test/test-uptime-calculator.js | 97 ++++++++++++++++++++- 2 files changed, 130 insertions(+), 7 deletions(-) diff --git a/server/uptime-calculator.js b/server/uptime-calculator.js index 7451e2400d..06633a0fff 100644 --- a/server/uptime-calculator.js +++ b/server/uptime-calculator.js @@ -161,25 +161,59 @@ class UptimeCalculator { return dailyUptimeData.uptime / (dailyUptimeData.uptime + dailyUptimeData.downtime); } + /** + * @param day + */ + getUptime(day) { + let dailyKey = this.getDailyKey(this.getCurrentDate().unix()); + + let total = { + uptime: 0, + downtime: 0, + }; + + for (let i = 0; i < day; i++) { + let dailyUptimeData = this.dailyUptimeDataList[dailyKey]; + + if (dailyUptimeData) { + total.uptime += dailyUptimeData.uptime; + total.downtime += dailyUptimeData.downtime; + } + + // Previous day + dailyKey -= 86400; + } + + if (total.uptime === 0 && total.downtime === 0) { + if (this.lastDailyUptimeData) { + total = this.lastDailyUptimeData; + } else { + return 0; + } + } + + return total.uptime / (total.uptime + total.downtime); + } + /** * */ get7DayUptime() { - + return this.getUptime(7); } /** * */ get30DayUptime() { - + return this.getUptime(30); } /** * */ get1YearUptime() { - + return this.getUptime(365); } /** diff --git a/test/backend-test/test-uptime-calculator.js b/test/backend-test/test-uptime-calculator.js index 3febfeb667..ca5c00ffcb 100644 --- a/test/backend-test/test-uptime-calculator.js +++ b/test/backend-test/test-uptime-calculator.js @@ -164,18 +164,107 @@ test("Test update - get24HourUptime", (t) => { // Add 24 hours c2 = new UptimeCalculator(); c2.update(UP); + c2.update(UP); + c2.update(UP); + c2.update(UP); + c2.update(DOWN); uptime = c2.get24HourUptime(); - assert.strictEqual(uptime, 1); + assert.strictEqual(uptime, 0.8); UptimeCalculator.currentDate = UptimeCalculator.currentDate.add(24, "hour"); - // After 24 hours, even if there is no data, the uptime should be still 100% + // After 24 hours, even if there is no data, the uptime should be still 80% uptime = c2.get24HourUptime(); - assert.strictEqual(uptime, 1); + assert.strictEqual(uptime, 0.8); // Add more 24 hours (48 hours) UptimeCalculator.currentDate = UptimeCalculator.currentDate.add(24, "hour"); - // After 24 hours, even if there is no data, the uptime should be still 100% + // After 48 hours, even if there is no data, the uptime should be still 80% uptime = c2.get24HourUptime(); + assert.strictEqual(uptime, 0.8); +}); + +test("Test update - get7DayUptime", (t) => { + UptimeCalculator.currentDate = dayjs.utc("2023-08-12 20:46:59"); + + // No data + let c2 = new UptimeCalculator(); + let uptime = c2.get7DayUptime(); + assert.strictEqual(uptime, 0); + + // 1 Up + c2 = new UptimeCalculator(); + c2.update(UP); + uptime = c2.get7DayUptime(); + assert.strictEqual(uptime, 1); + + // 2 Up + c2 = new UptimeCalculator(); + c2.update(UP); + c2.update(UP); + uptime = c2.get7DayUptime(); + assert.strictEqual(uptime, 1); + + // 3 Up + c2 = new UptimeCalculator(); + c2.update(UP); + c2.update(UP); + c2.update(UP); + uptime = c2.get7DayUptime(); assert.strictEqual(uptime, 1); + + // 1 MAINTENANCE + c2 = new UptimeCalculator(); + c2.update(MAINTENANCE); + uptime = c2.get7DayUptime(); + assert.strictEqual(uptime, 1); + + // 1 PENDING + c2 = new UptimeCalculator(); + c2.update(PENDING); + uptime = c2.get7DayUptime(); + assert.strictEqual(uptime, 0); + + // 1 DOWN + c2 = new UptimeCalculator(); + c2.update(DOWN); + uptime = c2.get7DayUptime(); + assert.strictEqual(uptime, 0); + + // 2 DOWN + c2 = new UptimeCalculator(); + c2.update(DOWN); + c2.update(DOWN); + uptime = c2.get7DayUptime(); + assert.strictEqual(uptime, 0); + + // 1 DOWN, 1 UP + c2 = new UptimeCalculator(); + c2.update(DOWN); + c2.update(UP); + uptime = c2.get7DayUptime(); + assert.strictEqual(uptime, 0.5); + + // 1 UP, 1 DOWN + c2 = new UptimeCalculator(); + c2.update(UP); + c2.update(DOWN); + uptime = c2.get7DayUptime(); + assert.strictEqual(uptime, 0.5); + + // Add 7 days + c2 = new UptimeCalculator(); + c2.update(UP); + c2.update(UP); + c2.update(UP); + c2.update(UP); + c2.update(DOWN); + uptime = c2.get7DayUptime(); + assert.strictEqual(uptime, 0.8); + UptimeCalculator.currentDate = UptimeCalculator.currentDate.add(7, "day"); + + // After 7 days, even if there is no data, the uptime should be still 80% + uptime = c2.get7DayUptime(); + assert.strictEqual(uptime, 0.8); + }); From 4733524dd49a8584f8cd8b1d8ad97d5178b0779d Mon Sep 17 00:00:00 2001 From: Louis Lam Date: Sun, 13 Aug 2023 03:14:04 +0800 Subject: [PATCH 04/16] WIP --- server/uptime-calculator.js | 46 ++++---- test/backend-test/test-uptime-calculator.js | 117 +++++++++++++++++++- 2 files changed, 137 insertions(+), 26 deletions(-) diff --git a/server/uptime-calculator.js b/server/uptime-calculator.js index 06633a0fff..730032b9b8 100644 --- a/server/uptime-calculator.js +++ b/server/uptime-calculator.js @@ -12,6 +12,8 @@ class UptimeCalculator { */ static currentDate = null; + monitorID; + /** * Recent 24-hour uptime, each item is a 1-minute interval * Key: {number} DivisionKey @@ -48,14 +50,14 @@ class UptimeCalculator { } /** - * TODO + * @param {number} monitorID */ - init() { + async init(monitorID) { + this.monitorID = monitorID; } /** * @param {number} status status - * @param {dayjs.Dayjs} date The heartbeat date * @returns {dayjs.Dayjs} date * @throws {Error} Invalid status */ @@ -66,11 +68,11 @@ class UptimeCalculator { let dailyKey = this.getDailyKey(divisionKey); if (flatStatus === UP) { - this.uptimeDataList[divisionKey].uptime += 1; - this.dailyUptimeDataList[dailyKey].uptime += 1; + this.uptimeDataList[divisionKey].up += 1; + this.dailyUptimeDataList[dailyKey].up += 1; } else { - this.uptimeDataList[divisionKey].downtime += 1; - this.dailyUptimeDataList[dailyKey].downtime += 1; + this.uptimeDataList[divisionKey].down += 1; + this.dailyUptimeDataList[dailyKey].down += 1; } this.lastDailyUptimeData = this.dailyUptimeDataList[dailyKey]; @@ -92,8 +94,8 @@ class UptimeCalculator { if (! (divisionKey in this.uptimeDataList)) { this.uptimeDataList[divisionKey] = { - uptime: 0, - downtime: 0, + up: 0, + down: 0, }; } @@ -115,8 +117,8 @@ class UptimeCalculator { if (!this.dailyUptimeDataList[dailyKey]) { this.dailyUptimeDataList[dailyKey] = { - uptime: 0, - downtime: 0, + up: 0, + down: 0, }; } @@ -150,7 +152,7 @@ class UptimeCalculator { // No data in last 24 hours, it could be a new monitor or the interval is larger than 24 hours // Try to use previous data, if no previous data, return 0 - if (dailyUptimeData.uptime === 0 && dailyUptimeData.downtime === 0) { + if (dailyUptimeData.up === 0 && dailyUptimeData.down === 0) { if (this.lastDailyUptimeData) { dailyUptimeData = this.lastDailyUptimeData; } else { @@ -158,7 +160,7 @@ class UptimeCalculator { } } - return dailyUptimeData.uptime / (dailyUptimeData.uptime + dailyUptimeData.downtime); + return dailyUptimeData.up / (dailyUptimeData.up + dailyUptimeData.down); } /** @@ -168,23 +170,23 @@ class UptimeCalculator { let dailyKey = this.getDailyKey(this.getCurrentDate().unix()); let total = { - uptime: 0, - downtime: 0, + up: 0, + down: 0, }; for (let i = 0; i < day; i++) { let dailyUptimeData = this.dailyUptimeDataList[dailyKey]; if (dailyUptimeData) { - total.uptime += dailyUptimeData.uptime; - total.downtime += dailyUptimeData.downtime; + total.up += dailyUptimeData.up; + total.down += dailyUptimeData.down; } // Previous day dailyKey -= 86400; } - if (total.uptime === 0 && total.downtime === 0) { + if (total.up === 0 && total.down === 0) { if (this.lastDailyUptimeData) { total = this.lastDailyUptimeData; } else { @@ -192,7 +194,7 @@ class UptimeCalculator { } } - return total.uptime / (total.uptime + total.downtime); + return total.up / (total.up + total.down); } /** @@ -224,10 +226,14 @@ class UptimeCalculator { } /** - * + * TODO */ clear() { + // Clear data older than 24 hours + + // Clear data older than 1 year + // https://stackoverflow.com/a/6630869/1097815 } } diff --git a/test/backend-test/test-uptime-calculator.js b/test/backend-test/test-uptime-calculator.js index ca5c00ffcb..a1cb0dc5b2 100644 --- a/test/backend-test/test-uptime-calculator.js +++ b/test/backend-test/test-uptime-calculator.js @@ -51,7 +51,7 @@ test("Test flatStatus", (t) => { assert.strictEqual(c2.flatStatus(PENDING), DOWN); }); -test("Test update - getDivisionKey", (t) => { +test("Test getDivisionKey", (t) => { let c2 = new UptimeCalculator(); let divisionKey = c2.getDivisionKey(dayjs.utc("2023-08-12 20:46:00")); assert.strictEqual(divisionKey, dayjs.utc("2023-08-12 20:46:00").unix()); @@ -67,7 +67,7 @@ test("Test update - getDivisionKey", (t) => { assert.strictEqual(divisionKey, dayjs.utc("2023-08-12 20:46:00").unix()); }); -test("Test update - getDailyKey", (t) => { +test("Test getDailyKey", (t) => { let c2 = new UptimeCalculator(); let dailyKey = c2.getDailyKey(dayjs.utc("2023-08-12 20:46:00").unix()); assert.strictEqual(dailyKey, dayjs.utc("2023-08-12").unix()); @@ -87,13 +87,13 @@ test("Test update - getDailyKey", (t) => { assert.strictEqual(dailyKey, dayjs.utc("2023-08-12").unix()); }); -test("Test update - lastDailyUptimeData", (t) => { +test("Test lastDailyUptimeData", (t) => { let c2 = new UptimeCalculator(); c2.update(UP); - assert.strictEqual(c2.lastDailyUptimeData.uptime, 1); + assert.strictEqual(c2.lastDailyUptimeData.up, 1); }); -test("Test update - get24HourUptime", (t) => { +test("Test get24HourUptime", (t) => { UptimeCalculator.currentDate = dayjs.utc("2023-08-12 20:46:59"); // No data @@ -184,7 +184,7 @@ test("Test update - get24HourUptime", (t) => { assert.strictEqual(uptime, 0.8); }); -test("Test update - get7DayUptime", (t) => { +test("Test get7DayUptime", (t) => { UptimeCalculator.currentDate = dayjs.utc("2023-08-12 20:46:59"); // No data @@ -268,3 +268,108 @@ test("Test update - get7DayUptime", (t) => { assert.strictEqual(uptime, 0.8); }); + +test("Test get30DayUptime (1 check per day)", (t) => { + + let c2 = new UptimeCalculator(); + let uptime = c2.get7DayUptime(); + assert.strictEqual(uptime, 0); + + UptimeCalculator.currentDate = dayjs.utc("2023-08-12 20:46:59"); + let up = 0; + let down = 0; + let flip = true; + for (let i = 0; i < 30; i++) { + UptimeCalculator.currentDate = UptimeCalculator.currentDate.add(1, "day"); + + if (flip) { + c2.update(UP); + up++; + } else { + c2.update(DOWN); + down++; + } + + uptime = c2.get30DayUptime(); + assert.strictEqual(uptime, up / (up + down)); + + flip = !flip; + } + + // Last 7 days + // Down, Up, Down, Up, Down, Up, Down + // So 3 UP + assert.strictEqual(c2.get7DayUptime(), 3 / 7); +}); + +test("Test get1YearUptime (1 check per day)", (t) => { + + let c2 = new UptimeCalculator(); + let uptime = c2.get7DayUptime(); + assert.strictEqual(uptime, 0); + + UptimeCalculator.currentDate = dayjs.utc("2023-08-12 20:46:59"); + let up = 0; + let down = 0; + let flip = true; + for (let i = 0; i < 365; i++) { + UptimeCalculator.currentDate = UptimeCalculator.currentDate.add(1, "day"); + + if (flip) { + c2.update(UP); + up++; + } else { + c2.update(DOWN); + down++; + } + + uptime = c2.get30DayUptime(); + flip = !flip; + } + + assert.strictEqual(c2.get1YearUptime(), 183 / 365); + assert.strictEqual(c2.get30DayUptime(), 15 / 30); + assert.strictEqual(c2.get7DayUptime(), 4 / 7); +}); + +test("Worst case", async (t) => { + + let c = new UptimeCalculator(); + let up = 0; + let down = 0; + let interval = 20; + + await t.test("Prepare data", async () => { + UptimeCalculator.currentDate = dayjs.utc("2023-08-12 20:46:59"); + + // Simulate 1s interval for a year + for (let i = 0; i < 365 * 24 * 60 * 60; i += interval) { + UptimeCalculator.currentDate = UptimeCalculator.currentDate.add(interval, "second"); + + //Randomly UP, DOWN, MAINTENANCE, PENDING + let rand = Math.random(); + if (rand < 0.25) { + c.update(UP); + up++; + } else if (rand < 0.5) { + c.update(DOWN); + down++; + } else if (rand < 0.75) { + c.update(MAINTENANCE); + up++; + } else { + c.update(PENDING); + down++; + } + + } + + assert.strictEqual(Object.keys(c.uptimeDataList).length, 1440); + assert.strictEqual(Object.keys(c.dailyUptimeDataList).length, 365); + }); + + await t.test("get1YearUptime()", async () => { + assert.strictEqual(c.get1YearUptime(), up / (up + down)); + }); + +}); From e98a38d9dda36db1b5f66ab4cdfa3f08d3d1ccbc Mon Sep 17 00:00:00 2001 From: Louis Lam Date: Tue, 15 Aug 2023 02:20:47 +0800 Subject: [PATCH 05/16] WIP --- server/uptime-calculator.js | 28 ++++++---- server/utils/array-with-key.js | 61 +++++++++++++++++++++ server/utils/limit-queue.js | 33 +++++++++++ test/backend-test/test-uptime-calculator.js | 27 +++++++-- 4 files changed, 135 insertions(+), 14 deletions(-) create mode 100644 server/utils/array-with-key.js create mode 100644 server/utils/limit-queue.js diff --git a/server/uptime-calculator.js b/server/uptime-calculator.js index 730032b9b8..ebe5ffaafc 100644 --- a/server/uptime-calculator.js +++ b/server/uptime-calculator.js @@ -1,5 +1,7 @@ const dayjs = require("dayjs"); const { UP, MAINTENANCE, DOWN, PENDING } = require("../src/util"); +const { LimitQueue } = require("./utils/limit-queue"); +const { log } = require("../src/util"); /** * Calculates the uptime of a monitor. @@ -18,17 +20,13 @@ class UptimeCalculator { * Recent 24-hour uptime, each item is a 1-minute interval * Key: {number} DivisionKey */ - uptimeDataList = { - - }; + uptimeDataList = new LimitQueue(24 * 60); /** * Daily uptime data, * Key: {number} DailyKey */ - dailyUptimeDataList = { - - }; + dailyUptimeDataList = new LimitQueue(365); lastDailyUptimeData = null; @@ -93,10 +91,15 @@ class UptimeCalculator { let divisionKey = date.unix(); if (! (divisionKey in this.uptimeDataList)) { - this.uptimeDataList[divisionKey] = { + let last = this.uptimeDataList.getLastKey(); + if (last && last > divisionKey) { + log.warn("uptime-calc", "The system time has been changed? The uptime data may be inaccurate."); + } + + this.uptimeDataList.push(divisionKey, { up: 0, down: 0, - }; + }); } return divisionKey; @@ -116,10 +119,15 @@ class UptimeCalculator { let dailyKey = date.unix(); if (!this.dailyUptimeDataList[dailyKey]) { - this.dailyUptimeDataList[dailyKey] = { + let last = this.dailyUptimeDataList.getLastKey(); + if (last && last > dailyKey) { + log.warn("uptime-calc", "The system time has been changed? The uptime data may be inaccurate."); + } + + this.dailyUptimeDataList.push(dailyKey, { up: 0, down: 0, - }; + }); } return dailyKey; diff --git a/server/utils/array-with-key.js b/server/utils/array-with-key.js new file mode 100644 index 0000000000..1b6954da00 --- /dev/null +++ b/server/utils/array-with-key.js @@ -0,0 +1,61 @@ +/** + * An object that can be used as an array with a key + * Like PHP's array + */ +class ArrayWithKey { + __stack = []; + + /** + * + */ + constructor() { + + } + + /** + * @param key + * @param value + */ + push(key, value) { + this[key] = value; + this.__stack.push(key); + } + + /** + * + */ + pop() { + let key = this.__stack.pop(); + let prop = this[key]; + delete this[key]; + return prop; + } + + /** + * + */ + getLastKey() { + return this.__stack[this.__stack.length - 1]; + } + + /** + * + */ + shift() { + let key = this.__stack.shift(); + let prop = this[key]; + delete this[key]; + return prop; + } + + /** + * + */ + length() { + return this.__stack.length; + } +} + +module.exports = { + ArrayWithKey +}; diff --git a/server/utils/limit-queue.js b/server/utils/limit-queue.js new file mode 100644 index 0000000000..d30e3c221b --- /dev/null +++ b/server/utils/limit-queue.js @@ -0,0 +1,33 @@ +const { ArrayWithKey } = require("./array-with-key"); + +/** + * Limit Queue + * The first element will be removed when the length exceeds the limit + */ +class LimitQueue extends ArrayWithKey { + + __limit; + + /** + * @param {number} limit + */ + constructor(limit) { + super(); + this.__limit = limit; + } + + /** + * @inheritDoc + */ + push(key, value) { + super.push(key, value); + if (this.length() > this.__limit) { + this.shift(); + } + } + +} + +module.exports = { + LimitQueue +}; diff --git a/test/backend-test/test-uptime-calculator.js b/test/backend-test/test-uptime-calculator.js index a1cb0dc5b2..c53f71f866 100644 --- a/test/backend-test/test-uptime-calculator.js +++ b/test/backend-test/test-uptime-calculator.js @@ -270,12 +270,12 @@ test("Test get7DayUptime", (t) => { }); test("Test get30DayUptime (1 check per day)", (t) => { + UptimeCalculator.currentDate = dayjs.utc("2023-08-12 20:46:59"); let c2 = new UptimeCalculator(); let uptime = c2.get7DayUptime(); assert.strictEqual(uptime, 0); - UptimeCalculator.currentDate = dayjs.utc("2023-08-12 20:46:59"); let up = 0; let down = 0; let flip = true; @@ -303,12 +303,12 @@ test("Test get30DayUptime (1 check per day)", (t) => { }); test("Test get1YearUptime (1 check per day)", (t) => { + UptimeCalculator.currentDate = dayjs.utc("2023-08-12 20:46:59"); let c2 = new UptimeCalculator(); let uptime = c2.get7DayUptime(); assert.strictEqual(uptime, 0); - UptimeCalculator.currentDate = dayjs.utc("2023-08-12 20:46:59"); let up = 0; let down = 0; let flip = true; @@ -332,7 +332,24 @@ test("Test get1YearUptime (1 check per day)", (t) => { assert.strictEqual(c2.get7DayUptime(), 4 / 7); }); +/** + * Code from here: https://stackoverflow.com/a/64550489/1097815 + */ +function memoryUsage() { + const formatMemoryUsage = (data) => `${Math.round(data / 1024 / 1024 * 100) / 100} MB`; + const memoryData = process.memoryUsage(); + + const memoryUsage = { + rss: `${formatMemoryUsage(memoryData.rss)} -> Resident Set Size - total memory allocated for the process execution`, + heapTotal: `${formatMemoryUsage(memoryData.heapTotal)} -> total size of the allocated heap`, + heapUsed: `${formatMemoryUsage(memoryData.heapUsed)} -> actual memory used during the execution`, + external: `${formatMemoryUsage(memoryData.external)} -> V8 external memory`, + }; + return memoryUsage; +} + test("Worst case", async (t) => { + console.log("Memory usage before preparation", memoryUsage()); let c = new UptimeCalculator(); let up = 0; @@ -364,8 +381,10 @@ test("Worst case", async (t) => { } - assert.strictEqual(Object.keys(c.uptimeDataList).length, 1440); - assert.strictEqual(Object.keys(c.dailyUptimeDataList).length, 365); + console.log("Memory usage before preparation", memoryUsage()); + + assert.strictEqual(c.uptimeDataList.length(), 1440); + assert.strictEqual(c.dailyUptimeDataList.length(), 365); }); await t.test("get1YearUptime()", async () => { From 4b65955e6b8dbc959d7e81fee0b106c65254a67d Mon Sep 17 00:00:00 2001 From: Louis Lam Date: Tue, 15 Aug 2023 16:56:49 +0800 Subject: [PATCH 06/16] WIP --- server/uptime-calculator.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/uptime-calculator.js b/server/uptime-calculator.js index ebe5ffaafc..231152836f 100644 --- a/server/uptime-calculator.js +++ b/server/uptime-calculator.js @@ -8,6 +8,8 @@ const { log } = require("../src/util"); */ class UptimeCalculator { + flushInterval = 60 * 1000; + /** * For testing purposes, we can set the current date to a specific date. * @type {dayjs.Dayjs} @@ -52,6 +54,8 @@ class UptimeCalculator { */ async init(monitorID) { this.monitorID = monitorID; + + // Object.assign(new Foo, { a: 1 }) } /** From f182b25f4d1bd6149133eb9f064c59776aca0039 Mon Sep 17 00:00:00 2001 From: Louis Lam Date: Wed, 16 Aug 2023 16:12:37 +0800 Subject: [PATCH 07/16] WIP --- .../2023-08-16-0000-create-uptime.js | 31 +++++++++++++++++++ db/knex_migrations/README.md | 2 -- server/uptime-calculator.js | 17 +++++++--- server/utils/array-with-key.js | 7 +++-- server/utils/limit-queue.js | 6 +++- test/backend-test/test-uptime-calculator.js | 22 +++++++++---- 6 files changed, 69 insertions(+), 16 deletions(-) create mode 100644 db/knex_migrations/2023-08-16-0000-create-uptime.js diff --git a/db/knex_migrations/2023-08-16-0000-create-uptime.js b/db/knex_migrations/2023-08-16-0000-create-uptime.js new file mode 100644 index 0000000000..faf458158b --- /dev/null +++ b/db/knex_migrations/2023-08-16-0000-create-uptime.js @@ -0,0 +1,31 @@ +exports.up = function (knex) { + return knex.schema + .createTable("aggregate_minutely", function (table) { + table.increments("id"); + table.integer("monitor_id").unsigned().notNullable() + .references("id").inTable("monitor") + .onDelete("CASCADE") + .onUpdate("CASCADE"); + table.integer("timestamp").notNullable(); + table.integer("ping").notNullable(); + table.smallint("up").notNullable(); + table.smallint("down").notNullable(); + }) + .createTable("aggregate_daily", function (table) { + table.increments("id"); + table.integer("monitor_id").unsigned().notNullable() + .references("id").inTable("monitor") + .onDelete("CASCADE") + .onUpdate("CASCADE"); + table.integer("timestamp").notNullable(); + table.integer("ping").notNullable(); + table.smallint("up").notNullable(); + table.smallint("down").notNullable(); + }); +}; + +exports.down = function (knex) { + return knex.schema + .dropTable("aggregate_minutely") + .dropTable("aggregate_daily"); +}; diff --git a/db/knex_migrations/README.md b/db/knex_migrations/README.md index 8aae8a665d..95dbcdc54f 100644 --- a/db/knex_migrations/README.md +++ b/db/knex_migrations/README.md @@ -9,8 +9,6 @@ https://knexjs.org/guide/migrations.html#knexfile-in-other-languages ## Template -Filename: YYYYMMDDHHMMSS_name.js - ```js exports.up = function(knex) { diff --git a/server/uptime-calculator.js b/server/uptime-calculator.js index 231152836f..e37b321ea9 100644 --- a/server/uptime-calculator.js +++ b/server/uptime-calculator.js @@ -60,10 +60,11 @@ class UptimeCalculator { /** * @param {number} status status + * @param {number} ping * @returns {dayjs.Dayjs} date * @throws {Error} Invalid status */ - update(status) { + update(status, ping = 0) { let date = this.getCurrentDate(); let flatStatus = this.flatStatus(status); let divisionKey = this.getDivisionKey(date); @@ -77,6 +78,14 @@ class UptimeCalculator { this.dailyUptimeDataList[dailyKey].down += 1; } + // Add avg ping + let count = this.uptimeDataList[divisionKey].up + this.uptimeDataList[divisionKey].down; + this.uptimeDataList[divisionKey].ping = (this.uptimeDataList[divisionKey].ping * (count - 1) + ping) / count; + + // Add avg ping (daily) + count = this.dailyUptimeDataList[dailyKey].up + this.dailyUptimeDataList[dailyKey].down; + this.dailyUptimeDataList[dailyKey].ping = (this.dailyUptimeDataList[dailyKey].ping * (count - 1) + ping) / count; + this.lastDailyUptimeData = this.dailyUptimeDataList[dailyKey]; this.clear(); @@ -103,6 +112,7 @@ class UptimeCalculator { this.uptimeDataList.push(divisionKey, { up: 0, down: 0, + ping: 0, }); } @@ -131,6 +141,7 @@ class UptimeCalculator { this.dailyUptimeDataList.push(dailyKey, { up: 0, down: 0, + ping: 0, }); } @@ -241,10 +252,6 @@ class UptimeCalculator { * TODO */ clear() { - // Clear data older than 24 hours - - // Clear data older than 1 year - // https://stackoverflow.com/a/6630869/1097815 } } diff --git a/server/utils/array-with-key.js b/server/utils/array-with-key.js index 1b6954da00..ffa0da41df 100644 --- a/server/utils/array-with-key.js +++ b/server/utils/array-with-key.js @@ -43,9 +43,12 @@ class ArrayWithKey { */ shift() { let key = this.__stack.shift(); - let prop = this[key]; + let value = this[key]; delete this[key]; - return prop; + return { + key, + value, + }; } /** diff --git a/server/utils/limit-queue.js b/server/utils/limit-queue.js index d30e3c221b..a4744f4a36 100644 --- a/server/utils/limit-queue.js +++ b/server/utils/limit-queue.js @@ -7,6 +7,7 @@ const { ArrayWithKey } = require("./array-with-key"); class LimitQueue extends ArrayWithKey { __limit; + __onExceed = null; /** * @param {number} limit @@ -22,7 +23,10 @@ class LimitQueue extends ArrayWithKey { push(key, value) { super.push(key, value); if (this.length() > this.__limit) { - this.shift(); + let item = this.shift(); + if (this.__onExceed) { + this.__onExceed(item); + } } } diff --git a/test/backend-test/test-uptime-calculator.js b/test/backend-test/test-uptime-calculator.js index c53f71f866..1d6d291008 100644 --- a/test/backend-test/test-uptime-calculator.js +++ b/test/backend-test/test-uptime-calculator.js @@ -359,6 +359,9 @@ test("Worst case", async (t) => { await t.test("Prepare data", async () => { UptimeCalculator.currentDate = dayjs.utc("2023-08-12 20:46:59"); + // Since 2023-08-12 will be out of 365 range, it starts from 2023-08-13 actually + let actualStartDate = dayjs.utc("2023-08-13 00:00:00").unix(); + // Simulate 1s interval for a year for (let i = 0; i < 365 * 24 * 60 * 60; i += interval) { UptimeCalculator.currentDate = UptimeCalculator.currentDate.add(interval, "second"); @@ -367,20 +370,27 @@ test("Worst case", async (t) => { let rand = Math.random(); if (rand < 0.25) { c.update(UP); - up++; + if (UptimeCalculator.currentDate.unix() > actualStartDate) { + up++; + } } else if (rand < 0.5) { c.update(DOWN); - down++; + if (UptimeCalculator.currentDate.unix() > actualStartDate) { + down++; + } } else if (rand < 0.75) { c.update(MAINTENANCE); - up++; + if (UptimeCalculator.currentDate.unix() > actualStartDate) { + up++; + } } else { c.update(PENDING); - down++; + if (UptimeCalculator.currentDate.unix() > actualStartDate) { + down++; + } } - } - + console.log("Final Date: ", UptimeCalculator.currentDate.format("YYYY-MM-DD HH:mm:ss")); console.log("Memory usage before preparation", memoryUsage()); assert.strictEqual(c.uptimeDataList.length(), 1440); From 8fed5e3716b3a363cc92133a445afeea4b2b7101 Mon Sep 17 00:00:00 2001 From: Louis Lam Date: Thu, 17 Aug 2023 22:03:18 +0800 Subject: [PATCH 08/16] WIP --- .../2023-08-16-0000-create-uptime.js | 24 ++-- package-lock.json | 105 ++++++++++++++ package.json | 1 + server/uptime-calculator.js | 136 ++++++++++++------ server/utils/array-with-key.js | 15 ++ src/components/NotificationDialog.vue | 2 +- test/backend-test/test-uptime-calculator.js | 94 ++++++------ 7 files changed, 277 insertions(+), 100 deletions(-) diff --git a/db/knex_migrations/2023-08-16-0000-create-uptime.js b/db/knex_migrations/2023-08-16-0000-create-uptime.js index faf458158b..5113fcd41d 100644 --- a/db/knex_migrations/2023-08-16-0000-create-uptime.js +++ b/db/knex_migrations/2023-08-16-0000-create-uptime.js @@ -1,24 +1,32 @@ exports.up = function (knex) { return knex.schema - .createTable("aggregate_minutely", function (table) { + .createTable("stat_minutely", function (table) { table.increments("id"); + table.comment("This table contains the minutely aggregate statistics for each monitor"); table.integer("monitor_id").unsigned().notNullable() .references("id").inTable("monitor") .onDelete("CASCADE") .onUpdate("CASCADE"); - table.integer("timestamp").notNullable(); - table.integer("ping").notNullable(); + table.integer("timestamp") + .notNullable() + .unique() + .comment("Unix timestamp rounded down to the nearest minute"); + table.float("ping").notNullable().comment("Average ping in milliseconds"); table.smallint("up").notNullable(); table.smallint("down").notNullable(); }) - .createTable("aggregate_daily", function (table) { + .createTable("stat_daily", function (table) { table.increments("id"); + table.comment("This table contains the daily aggregate statistics for each monitor"); table.integer("monitor_id").unsigned().notNullable() .references("id").inTable("monitor") .onDelete("CASCADE") .onUpdate("CASCADE"); - table.integer("timestamp").notNullable(); - table.integer("ping").notNullable(); + table.integer("timestamp") + .notNullable() + .unique() + .comment("Unix timestamp rounded down to the nearest day"); + table.float("ping").notNullable().comment("Average ping in milliseconds"); table.smallint("up").notNullable(); table.smallint("down").notNullable(); }); @@ -26,6 +34,6 @@ exports.up = function (knex) { exports.down = function (knex) { return knex.schema - .dropTable("aggregate_minutely") - .dropTable("aggregate_daily"); + .dropTable("stat_minutely") + .dropTable("stat_daily"); }; diff --git a/package-lock.json b/package-lock.json index 7db707f4e9..21e2bc32eb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -119,6 +119,7 @@ "stylelint": "^15.10.1", "stylelint-config-standard": "~25.0.0", "terser": "~5.15.0", + "test": "^3.3.0", "timezones-list": "~3.0.1", "typescript": "~4.4.4", "v-pagination-3": "~0.1.7", @@ -5981,6 +5982,18 @@ "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==" }, + "node_modules/abort-controller": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "dev": true, + "dependencies": { + "event-target-shim": "^5.0.0" + }, + "engines": { + "node": ">=6.5" + } + }, "node_modules/accepts": { "version": "1.3.8", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", @@ -9377,6 +9390,15 @@ "node": ">= 0.6" } }, + "node_modules/event-target-shim": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", + "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", + "dev": true, + "engines": { + "node": ">=6" + } + }, "node_modules/event-to-promise": { "version": "0.7.0", "resolved": "https://registry.npmjs.org/event-to-promise/-/event-to-promise-0.7.0.tgz", @@ -15529,6 +15551,15 @@ "node": ">=6" } }, + "node_modules/process": { + "version": "0.11.10", + "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", + "integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==", + "dev": true, + "engines": { + "node": ">= 0.6.0" + } + }, "node_modules/process-nextick-args": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", @@ -17297,6 +17328,23 @@ "node": ">=8" } }, + "node_modules/string.prototype.replaceall": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/string.prototype.replaceall/-/string.prototype.replaceall-1.0.7.tgz", + "integrity": "sha512-xB2WV2GlSCSJT5dMGdhdH1noMPiAB91guiepwTYyWY9/0Vq/TZ7RPmnOSUGAEvry08QIK7EMr28aAii+9jC6kw==", + "dev": true, + "dependencies": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.4", + "es-abstract": "^1.20.4", + "get-intrinsic": "^1.1.3", + "has-symbols": "^1.0.3", + "is-regex": "^1.1.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/string.prototype.trim": { "version": "1.2.7", "resolved": "https://registry.npmjs.org/string.prototype.trim/-/string.prototype.trim-1.2.7.tgz", @@ -17825,6 +17873,23 @@ "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", "dev": true }, + "node_modules/test": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/test/-/test-3.3.0.tgz", + "integrity": "sha512-JKlEohxDIJRjwBH/+BrTcAPHljBALrAHw3Zs99RqZlaC605f6BggqXhxkdqZThbSHgaYPwpNJlf9bTSWkb/1rA==", + "dev": true, + "dependencies": { + "minimist": "^1.2.6", + "readable-stream": "^4.3.0", + "string.prototype.replaceall": "^1.0.6" + }, + "bin": { + "node--test": "bin/node--test.js", + "node--test-name-pattern": "bin/node--test-name-pattern.js", + "node--test-only": "bin/node--test-only.js", + "test": "bin/node-core-test.js" + } + }, "node_modules/test-exclude": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-6.0.0.tgz", @@ -17839,6 +17904,46 @@ "node": ">=8" } }, + "node_modules/test/node_modules/buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, + "node_modules/test/node_modules/readable-stream": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.4.2.tgz", + "integrity": "sha512-Lk/fICSyIhodxy1IDK2HazkeGjSmezAWX2egdtJnYhtzKEsBPJowlI6F6LPb5tqIQILrMbx22S5o3GuJavPusA==", + "dev": true, + "dependencies": { + "abort-controller": "^3.0.0", + "buffer": "^6.0.3", + "events": "^3.3.0", + "process": "^0.11.10", + "string_decoder": "^1.3.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, "node_modules/text-table": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", diff --git a/package.json b/package.json index 2ad2b50bc4..0c6da357c4 100644 --- a/package.json +++ b/package.json @@ -184,6 +184,7 @@ "stylelint": "^15.10.1", "stylelint-config-standard": "~25.0.0", "terser": "~5.15.0", + "test": "~3.3.0", "timezones-list": "~3.0.1", "typescript": "~4.4.4", "v-pagination-3": "~0.1.7", diff --git a/server/uptime-calculator.js b/server/uptime-calculator.js index e37b321ea9..5c3cb2a873 100644 --- a/server/uptime-calculator.js +++ b/server/uptime-calculator.js @@ -2,6 +2,7 @@ const dayjs = require("dayjs"); const { UP, MAINTENANCE, DOWN, PENDING } = require("../src/util"); const { LimitQueue } = require("./utils/limit-queue"); const { log } = require("../src/util"); +const { R } = require("redbean-node"); /** * Calculates the uptime of a monitor. @@ -31,6 +32,7 @@ class UptimeCalculator { dailyUptimeDataList = new LimitQueue(365); lastDailyUptimeData = null; + lastUptimeData = null; /** * @@ -67,7 +69,7 @@ class UptimeCalculator { update(status, ping = 0) { let date = this.getCurrentDate(); let flatStatus = this.flatStatus(status); - let divisionKey = this.getDivisionKey(date); + let divisionKey = this.getMinutelyKey(date); let dailyKey = this.getDailyKey(divisionKey); if (flatStatus === UP) { @@ -86,17 +88,45 @@ class UptimeCalculator { count = this.dailyUptimeDataList[dailyKey].up + this.dailyUptimeDataList[dailyKey].down; this.dailyUptimeDataList[dailyKey].ping = (this.dailyUptimeDataList[dailyKey].ping * (count - 1) + ping) / count; - this.lastDailyUptimeData = this.dailyUptimeDataList[dailyKey]; + if (this.dailyUptimeDataList[dailyKey] != this.lastDailyUptimeData) { + + this.lastDailyUptimeData = + this.lastDailyUptimeData.monitor_id = this.monitorID; + this.lastDailyUptimeData.timestamp = dailyKey; + this.lastDailyUptimeData = this.dailyUptimeDataList[dailyKey]; + } + + if (this.uptimeDataList[divisionKey] != this.lastUptimeData) { + this.lastUptimeData = this.uptimeDataList[divisionKey]; + } - this.clear(); return date; } + /** + * Get the daily stat bean + * @param {number} timestamp milliseconds + * @returns {Promise} stat_daily bean + */ + async getDailyStatBean(timestamp) { + let bean = await R.findOne("stat_daily", " monitor_id = ? AND timestamp = ?", [ + this.monitorID, + timestamp, + ]); + + if (!bean) { + bean = R.dispense("stat_daily"); + bean.monitor_id = this.monitorID; + bean.timestamp = timestamp; + } + return bean; + } + /** * @param {dayjs.Dayjs} date The heartbeat date - * @returns {number} division + * @returns {number} Timestamp */ - getDivisionKey(date) { + getMinutelyKey(date) { // Convert the current date to the nearest minute (e.g. 2021-01-01 12:34:56 -> 2021-01-01 12:34:00) date = date.startOf("minute"); @@ -122,7 +152,7 @@ class UptimeCalculator { /** * Convert timestamp to daily key * @param {number} timestamp Timestamp - * @returns {number} dailyKey + * @returns {number} Timestamp */ getDailyKey(timestamp) { let date = dayjs.unix(timestamp); @@ -167,93 +197,105 @@ class UptimeCalculator { } /** - * + * @param {number} num + * @param {string} type "day" | "minute" */ - get24HourUptime() { - let dailyKey = this.getDailyKey(this.getCurrentDate().unix()); - let dailyUptimeData = this.dailyUptimeDataList[dailyKey]; - - // No data in last 24 hours, it could be a new monitor or the interval is larger than 24 hours - // Try to use previous data, if no previous data, return 0 - if (dailyUptimeData.up === 0 && dailyUptimeData.down === 0) { - if (this.lastDailyUptimeData) { - dailyUptimeData = this.lastDailyUptimeData; - } else { - return 0; + getData(num, type = "day") { + let key; + + if (type === "day") { + key = this.getDailyKey(this.getCurrentDate().unix()); + } else { + if (num > 24 * 60) { + throw new Error("The maximum number of minutes is 1440"); } + key = this.getMinutelyKey(this.getCurrentDate()); } - return dailyUptimeData.up / (dailyUptimeData.up + dailyUptimeData.down); - } - - /** - * @param day - */ - getUptime(day) { - let dailyKey = this.getDailyKey(this.getCurrentDate().unix()); - let total = { up: 0, down: 0, }; - for (let i = 0; i < day; i++) { - let dailyUptimeData = this.dailyUptimeDataList[dailyKey]; + let totalPing = 0; + + for (let i = 0; i < num; i++) { + let data; + + if (type === "day") { + data = this.dailyUptimeDataList[key]; + } else { + data = this.uptimeDataList[key]; + } - if (dailyUptimeData) { - total.up += dailyUptimeData.up; - total.down += dailyUptimeData.down; + if (data) { + total.up += data.up; + total.down += data.down; + totalPing += data.ping; } // Previous day - dailyKey -= 86400; + if (type === "day") { + key -= 86400; + } else { + key -= 60; + } } if (total.up === 0 && total.down === 0) { - if (this.lastDailyUptimeData) { + if (type === "day" && this.lastDailyUptimeData) { total = this.lastDailyUptimeData; + } else if (type === "minute" && this.lastUptimeData) { + total = this.lastUptimeData; } else { - return 0; + return { + uptime: 0, + avgPing: 0, + }; } } - return total.up / (total.up + total.down); + return { + uptime: total.up / (total.up + total.down), + avgPing: totalPing / total.up, + }; } /** * */ - get7DayUptime() { - return this.getUptime(7); + get24Hour() { + return this.getData(24, "minute"); } /** * */ - get30DayUptime() { - return this.getUptime(30); + get7Day() { + return this.getData(7); } /** * */ - get1YearUptime() { - return this.getUptime(365); + get30Day() { + return this.getData(30); } /** * */ - getCurrentDate() { - return dayjs.utc(); + get1Year() { + return this.getData(365); } /** - * TODO + * */ - clear() { - // https://stackoverflow.com/a/6630869/1097815 + getCurrentDate() { + return dayjs.utc(); } + } module.exports = { diff --git a/server/utils/array-with-key.js b/server/utils/array-with-key.js index ffa0da41df..847bc24f74 100644 --- a/server/utils/array-with-key.js +++ b/server/utils/array-with-key.js @@ -35,6 +35,9 @@ class ArrayWithKey { * */ getLastKey() { + if (this.__stack.length === 0) { + return null; + } return this.__stack[this.__stack.length - 1]; } @@ -57,6 +60,18 @@ class ArrayWithKey { length() { return this.__stack.length; } + + /** + * Get the last element + * @returns {*|null} The last element, or null if the array is empty + */ + last() { + let key = this.getLastKey(); + if (key === null) { + return null; + } + return this[key]; + } } module.exports = { diff --git a/src/components/NotificationDialog.vue b/src/components/NotificationDialog.vue index 170a1d4cf5..e2024aaebd 100644 --- a/src/components/NotificationDialog.vue +++ b/src/components/NotificationDialog.vue @@ -199,7 +199,7 @@ export default { }, }, - watch: { + watch: { "notification.type"(to, from) { let oldName; if (from) { diff --git a/test/backend-test/test-uptime-calculator.js b/test/backend-test/test-uptime-calculator.js index 1d6d291008..f447b93f9c 100644 --- a/test/backend-test/test-uptime-calculator.js +++ b/test/backend-test/test-uptime-calculator.js @@ -1,4 +1,13 @@ -const { test } = require("node:test"); +const semver = require("semver"); +let test; +const nodeVersion = process.versions.node; +// Node.js version >= 18 +if (semver.satisfies(nodeVersion, ">= 18")) { + test = require("node:test"); +} else { + test = require("test"); +} + const assert = require("node:assert"); const { UptimeCalculator } = require("../../server/uptime-calculator"); const dayjs = require("dayjs"); @@ -51,19 +60,19 @@ test("Test flatStatus", (t) => { assert.strictEqual(c2.flatStatus(PENDING), DOWN); }); -test("Test getDivisionKey", (t) => { +test("Test getMinutelyKey", (t) => { let c2 = new UptimeCalculator(); - let divisionKey = c2.getDivisionKey(dayjs.utc("2023-08-12 20:46:00")); + let divisionKey = c2.getMinutelyKey(dayjs.utc("2023-08-12 20:46:00")); assert.strictEqual(divisionKey, dayjs.utc("2023-08-12 20:46:00").unix()); // Edge case 1 c2 = new UptimeCalculator(); - divisionKey = c2.getDivisionKey(dayjs.utc("2023-08-12 20:46:01")); + divisionKey = c2.getMinutelyKey(dayjs.utc("2023-08-12 20:46:01")); assert.strictEqual(divisionKey, dayjs.utc("2023-08-12 20:46:00").unix()); // Edge case 2 c2 = new UptimeCalculator(); - divisionKey = c2.getDivisionKey(dayjs.utc("2023-08-12 20:46:59")); + divisionKey = c2.getMinutelyKey(dayjs.utc("2023-08-12 20:46:59")); assert.strictEqual(divisionKey, dayjs.utc("2023-08-12 20:46:00").unix()); }); @@ -98,20 +107,21 @@ test("Test get24HourUptime", (t) => { // No data let c2 = new UptimeCalculator(); - let uptime = c2.get24HourUptime(); - assert.strictEqual(uptime, 0); + let data = c2.get24Hour(); + assert.strictEqual(data.uptime, 0); + assert.strictEqual(data.avgPing, 0); // 1 Up c2 = new UptimeCalculator(); c2.update(UP); - uptime = c2.get24HourUptime(); + let uptime = c2.get24Hour().uptime; assert.strictEqual(uptime, 1); // 2 Up c2 = new UptimeCalculator(); c2.update(UP); c2.update(UP); - uptime = c2.get24HourUptime(); + uptime = c2.get24Hour().uptime; assert.strictEqual(uptime, 1); // 3 Up @@ -119,46 +129,46 @@ test("Test get24HourUptime", (t) => { c2.update(UP); c2.update(UP); c2.update(UP); - uptime = c2.get24HourUptime(); + uptime = c2.get24Hour().uptime; assert.strictEqual(uptime, 1); // 1 MAINTENANCE c2 = new UptimeCalculator(); c2.update(MAINTENANCE); - uptime = c2.get24HourUptime(); + uptime = c2.get24Hour().uptime; assert.strictEqual(uptime, 1); // 1 PENDING c2 = new UptimeCalculator(); c2.update(PENDING); - uptime = c2.get24HourUptime(); + uptime = c2.get24Hour().uptime; assert.strictEqual(uptime, 0); // 1 DOWN c2 = new UptimeCalculator(); c2.update(DOWN); - uptime = c2.get24HourUptime(); + uptime = c2.get24Hour().uptime; assert.strictEqual(uptime, 0); // 2 DOWN c2 = new UptimeCalculator(); c2.update(DOWN); c2.update(DOWN); - uptime = c2.get24HourUptime(); + uptime = c2.get24Hour().uptime; assert.strictEqual(uptime, 0); // 1 DOWN, 1 UP c2 = new UptimeCalculator(); c2.update(DOWN); c2.update(UP); - uptime = c2.get24HourUptime(); + uptime = c2.get24Hour().uptime; assert.strictEqual(uptime, 0.5); // 1 UP, 1 DOWN c2 = new UptimeCalculator(); c2.update(UP); c2.update(DOWN); - uptime = c2.get24HourUptime(); + uptime = c2.get24Hour().uptime; assert.strictEqual(uptime, 0.5); // Add 24 hours @@ -168,19 +178,19 @@ test("Test get24HourUptime", (t) => { c2.update(UP); c2.update(UP); c2.update(DOWN); - uptime = c2.get24HourUptime(); + uptime = c2.get24Hour().uptime; assert.strictEqual(uptime, 0.8); UptimeCalculator.currentDate = UptimeCalculator.currentDate.add(24, "hour"); // After 24 hours, even if there is no data, the uptime should be still 80% - uptime = c2.get24HourUptime(); + uptime = c2.get24Hour().uptime; assert.strictEqual(uptime, 0.8); // Add more 24 hours (48 hours) UptimeCalculator.currentDate = UptimeCalculator.currentDate.add(24, "hour"); // After 48 hours, even if there is no data, the uptime should be still 80% - uptime = c2.get24HourUptime(); + uptime = c2.get24Hour().uptime; assert.strictEqual(uptime, 0.8); }); @@ -189,20 +199,20 @@ test("Test get7DayUptime", (t) => { // No data let c2 = new UptimeCalculator(); - let uptime = c2.get7DayUptime(); + let uptime = c2.get7Day().uptime; assert.strictEqual(uptime, 0); // 1 Up c2 = new UptimeCalculator(); c2.update(UP); - uptime = c2.get7DayUptime(); + uptime = c2.get7Day().uptime; assert.strictEqual(uptime, 1); // 2 Up c2 = new UptimeCalculator(); c2.update(UP); c2.update(UP); - uptime = c2.get7DayUptime(); + uptime = c2.get7Day().uptime; assert.strictEqual(uptime, 1); // 3 Up @@ -210,46 +220,46 @@ test("Test get7DayUptime", (t) => { c2.update(UP); c2.update(UP); c2.update(UP); - uptime = c2.get7DayUptime(); + uptime = c2.get7Day().uptime; assert.strictEqual(uptime, 1); // 1 MAINTENANCE c2 = new UptimeCalculator(); c2.update(MAINTENANCE); - uptime = c2.get7DayUptime(); + uptime = c2.get7Day().uptime; assert.strictEqual(uptime, 1); // 1 PENDING c2 = new UptimeCalculator(); c2.update(PENDING); - uptime = c2.get7DayUptime(); + uptime = c2.get7Day().uptime; assert.strictEqual(uptime, 0); // 1 DOWN c2 = new UptimeCalculator(); c2.update(DOWN); - uptime = c2.get7DayUptime(); + uptime = c2.get7Day().uptime; assert.strictEqual(uptime, 0); // 2 DOWN c2 = new UptimeCalculator(); c2.update(DOWN); c2.update(DOWN); - uptime = c2.get7DayUptime(); + uptime = c2.get7Day().uptime; assert.strictEqual(uptime, 0); // 1 DOWN, 1 UP c2 = new UptimeCalculator(); c2.update(DOWN); c2.update(UP); - uptime = c2.get7DayUptime(); + uptime = c2.get7Day().uptime; assert.strictEqual(uptime, 0.5); // 1 UP, 1 DOWN c2 = new UptimeCalculator(); c2.update(UP); c2.update(DOWN); - uptime = c2.get7DayUptime(); + uptime = c2.get7Day().uptime; assert.strictEqual(uptime, 0.5); // Add 7 days @@ -259,12 +269,12 @@ test("Test get7DayUptime", (t) => { c2.update(UP); c2.update(UP); c2.update(DOWN); - uptime = c2.get7DayUptime(); + uptime = c2.get7Day().uptime; assert.strictEqual(uptime, 0.8); UptimeCalculator.currentDate = UptimeCalculator.currentDate.add(7, "day"); // After 7 days, even if there is no data, the uptime should be still 80% - uptime = c2.get7DayUptime(); + uptime = c2.get7Day().uptime; assert.strictEqual(uptime, 0.8); }); @@ -273,7 +283,7 @@ test("Test get30DayUptime (1 check per day)", (t) => { UptimeCalculator.currentDate = dayjs.utc("2023-08-12 20:46:59"); let c2 = new UptimeCalculator(); - let uptime = c2.get7DayUptime(); + let uptime = c2.get30Day().uptime; assert.strictEqual(uptime, 0); let up = 0; @@ -290,7 +300,7 @@ test("Test get30DayUptime (1 check per day)", (t) => { down++; } - uptime = c2.get30DayUptime(); + uptime = c2.get30Day().uptime; assert.strictEqual(uptime, up / (up + down)); flip = !flip; @@ -299,37 +309,33 @@ test("Test get30DayUptime (1 check per day)", (t) => { // Last 7 days // Down, Up, Down, Up, Down, Up, Down // So 3 UP - assert.strictEqual(c2.get7DayUptime(), 3 / 7); + assert.strictEqual(c2.get7Day().uptime, 3 / 7); }); test("Test get1YearUptime (1 check per day)", (t) => { UptimeCalculator.currentDate = dayjs.utc("2023-08-12 20:46:59"); let c2 = new UptimeCalculator(); - let uptime = c2.get7DayUptime(); + let uptime = c2.get1Year().uptime; assert.strictEqual(uptime, 0); - let up = 0; - let down = 0; let flip = true; for (let i = 0; i < 365; i++) { UptimeCalculator.currentDate = UptimeCalculator.currentDate.add(1, "day"); if (flip) { c2.update(UP); - up++; } else { c2.update(DOWN); - down++; } - uptime = c2.get30DayUptime(); + uptime = c2.get30Day().time; flip = !flip; } - assert.strictEqual(c2.get1YearUptime(), 183 / 365); - assert.strictEqual(c2.get30DayUptime(), 15 / 30); - assert.strictEqual(c2.get7DayUptime(), 4 / 7); + assert.strictEqual(c2.get1Year().uptime, 183 / 365); + assert.strictEqual(c2.get30Day().uptime, 15 / 30); + assert.strictEqual(c2.get7Day().uptime, 4 / 7); }); /** @@ -398,7 +404,7 @@ test("Worst case", async (t) => { }); await t.test("get1YearUptime()", async () => { - assert.strictEqual(c.get1YearUptime(), up / (up + down)); + assert.strictEqual(c.get1Year().uptime, up / (up + down)); }); }); From c43ec41d2800fd04f14c9cceaf4798d36b62dec2 Mon Sep 17 00:00:00 2001 From: Louis Lam Date: Fri, 18 Aug 2023 04:18:20 +0800 Subject: [PATCH 09/16] WIP --- .../2023-08-18-0301-heartbeat.js | 19 +++ server/model/monitor.js | 12 +- server/uptime-calculator.js | 102 +++++++++++--- test/backend-test/test-uptime-calculator.js | 126 +++++++++--------- 4 files changed, 170 insertions(+), 89 deletions(-) create mode 100644 db/knex_migrations/2023-08-18-0301-heartbeat.js diff --git a/db/knex_migrations/2023-08-18-0301-heartbeat.js b/db/knex_migrations/2023-08-18-0301-heartbeat.js new file mode 100644 index 0000000000..6b391e7fff --- /dev/null +++ b/db/knex_migrations/2023-08-18-0301-heartbeat.js @@ -0,0 +1,19 @@ +exports.up = function (knex) { + // Add new column heartbeat.end_time + return knex.schema + .alterTable("heartbeat", function (table) { + table.timestamp("end_time").nullable().defaultTo(null); + + // Change time's datatype to timestamp + table.timestamp("time").alter(); + }); + +}; + +exports.down = function (knex) { + // Rename heartbeat.start_time to heartbeat.time + return knex.schema + .alterTable("heartbeat", function (table) { + table.dropColumn("end_time"); + }); +}; diff --git a/server/model/monitor.js b/server/model/monitor.js index 577a55cdbc..b907874145 100644 --- a/server/model/monitor.js +++ b/server/model/monitor.js @@ -23,6 +23,7 @@ const Gamedig = require("gamedig"); const jsonata = require("jsonata"); const jwt = require("jsonwebtoken"); const Database = require("../database"); +const { UptimeCalculator } = require("../uptime-calculator"); /** * status: @@ -346,13 +347,6 @@ class Monitor extends BeanModel { bean.status = flipStatus(bean.status); } - // Duration - if (!isFirstBeat) { - bean.duration = dayjs(bean.time).diff(dayjs(previousBeat.time), "second"); - } else { - bean.duration = 0; - } - try { if (await Monitor.isUnderMaintenance(this.id)) { bean.msg = "Monitor under maintenance"; @@ -976,6 +970,10 @@ class Monitor extends BeanModel { io.to(this.user_id).emit("heartbeat", bean.toJSON()); Monitor.sendStats(io, this.id, this.user_id); + let uptimeCalculator = await UptimeCalculator.getUptimeCalculator(this.id); + let endTimeDayjs = await uptimeCalculator.update(bean.status, parseFloat(bean.ping)); + bean.end_time = R.isoDateTimeMillis(endTimeDayjs); + log.debug("monitor", `[${this.name}] Store`); await R.store(bean); diff --git a/server/uptime-calculator.js b/server/uptime-calculator.js index 5c3cb2a873..7431bd1df3 100644 --- a/server/uptime-calculator.js +++ b/server/uptime-calculator.js @@ -9,7 +9,7 @@ const { R } = require("redbean-node"); */ class UptimeCalculator { - flushInterval = 60 * 1000; + static list = {}; /** * For testing purposes, we can set the current date to a specific date. @@ -23,7 +23,7 @@ class UptimeCalculator { * Recent 24-hour uptime, each item is a 1-minute interval * Key: {number} DivisionKey */ - uptimeDataList = new LimitQueue(24 * 60); + minutelyUptimeDataList = new LimitQueue(24 * 60); /** * Daily uptime data, @@ -34,6 +34,28 @@ class UptimeCalculator { lastDailyUptimeData = null; lastUptimeData = null; + lastDailyStatBean = null; + lastMinutelyStatBean = null; + + /** + * @param monitorID + * @returns {Promise} + */ + static async getUptimeCalculator(monitorID) { + if (!UptimeCalculator.list[monitorID]) { + UptimeCalculator.list[monitorID] = new UptimeCalculator(); + await UptimeCalculator.list[monitorID].init(monitorID); + } + return UptimeCalculator.list[monitorID]; + } + + /** + * @param monitorID + */ + static async remove(monitorID) { + delete UptimeCalculator.list[monitorID]; + } + /** * */ @@ -66,38 +88,49 @@ class UptimeCalculator { * @returns {dayjs.Dayjs} date * @throws {Error} Invalid status */ - update(status, ping = 0) { + async update(status, ping = 0) { let date = this.getCurrentDate(); let flatStatus = this.flatStatus(status); let divisionKey = this.getMinutelyKey(date); let dailyKey = this.getDailyKey(divisionKey); if (flatStatus === UP) { - this.uptimeDataList[divisionKey].up += 1; + this.minutelyUptimeDataList[divisionKey].up += 1; this.dailyUptimeDataList[dailyKey].up += 1; } else { - this.uptimeDataList[divisionKey].down += 1; + this.minutelyUptimeDataList[divisionKey].down += 1; this.dailyUptimeDataList[dailyKey].down += 1; } // Add avg ping - let count = this.uptimeDataList[divisionKey].up + this.uptimeDataList[divisionKey].down; - this.uptimeDataList[divisionKey].ping = (this.uptimeDataList[divisionKey].ping * (count - 1) + ping) / count; + let count = this.minutelyUptimeDataList[divisionKey].up + this.minutelyUptimeDataList[divisionKey].down; + this.minutelyUptimeDataList[divisionKey].ping = (this.minutelyUptimeDataList[divisionKey].ping * (count - 1) + ping) / count; // Add avg ping (daily) count = this.dailyUptimeDataList[dailyKey].up + this.dailyUptimeDataList[dailyKey].down; this.dailyUptimeDataList[dailyKey].ping = (this.dailyUptimeDataList[dailyKey].ping * (count - 1) + ping) / count; - if (this.dailyUptimeDataList[dailyKey] != this.lastDailyUptimeData) { - - this.lastDailyUptimeData = - this.lastDailyUptimeData.monitor_id = this.monitorID; - this.lastDailyUptimeData.timestamp = dailyKey; + if (this.dailyUptimeDataList[dailyKey] !== this.lastDailyUptimeData) { this.lastDailyUptimeData = this.dailyUptimeDataList[dailyKey]; } - if (this.uptimeDataList[divisionKey] != this.lastUptimeData) { - this.lastUptimeData = this.uptimeDataList[divisionKey]; + if (this.minutelyUptimeDataList[divisionKey] !== this.lastUptimeData) { + this.lastUptimeData = this.minutelyUptimeDataList[divisionKey]; + } + + // Update database + if (!process.env.TEST_BACKEND) { + let dailyStatBean = await this.getDailyStatBean(dailyKey); + dailyStatBean.up = this.dailyUptimeDataList[dailyKey].up; + dailyStatBean.down = this.dailyUptimeDataList[dailyKey].down; + dailyStatBean.ping = this.dailyUptimeDataList[dailyKey].ping; + await R.store(dailyStatBean); + + let minutelyStatBean = await this.getMinutelyStatBean(divisionKey); + minutelyStatBean.up = this.minutelyUptimeDataList[divisionKey].up; + minutelyStatBean.down = this.minutelyUptimeDataList[divisionKey].down; + minutelyStatBean.ping = this.minutelyUptimeDataList[divisionKey].ping; + await R.store(minutelyStatBean); } return date; @@ -109,6 +142,10 @@ class UptimeCalculator { * @returns {Promise} stat_daily bean */ async getDailyStatBean(timestamp) { + if (this.lastDailyStatBean && this.lastDailyStatBean.timestamp === timestamp) { + return this.lastDailyStatBean; + } + let bean = await R.findOne("stat_daily", " monitor_id = ? AND timestamp = ?", [ this.monitorID, timestamp, @@ -119,7 +156,34 @@ class UptimeCalculator { bean.monitor_id = this.monitorID; bean.timestamp = timestamp; } - return bean; + + this.lastDailyStatBean = bean; + return this.lastDailyStatBean; + } + + /** + * Get the minutely stat bean + * @param {number} timestamp milliseconds + * @returns {Promise} stat_minutely bean + */ + async getMinutelyStatBean(timestamp) { + if (this.lastMinutelyStatBean && this.lastMinutelyStatBean.timestamp === timestamp) { + return this.lastMinutelyStatBean; + } + + let bean = await R.findOne("stat_minutely", " monitor_id = ? AND timestamp = ?", [ + this.monitorID, + timestamp, + ]); + + if (!bean) { + bean = R.dispense("stat_minutely"); + bean.monitor_id = this.monitorID; + bean.timestamp = timestamp; + } + + this.lastMinutelyStatBean = bean; + return this.lastMinutelyStatBean; } /** @@ -133,13 +197,13 @@ class UptimeCalculator { // Convert to timestamp in second let divisionKey = date.unix(); - if (! (divisionKey in this.uptimeDataList)) { - let last = this.uptimeDataList.getLastKey(); + if (! (divisionKey in this.minutelyUptimeDataList)) { + let last = this.minutelyUptimeDataList.getLastKey(); if (last && last > divisionKey) { log.warn("uptime-calc", "The system time has been changed? The uptime data may be inaccurate."); } - this.uptimeDataList.push(divisionKey, { + this.minutelyUptimeDataList.push(divisionKey, { up: 0, down: 0, ping: 0, @@ -225,7 +289,7 @@ class UptimeCalculator { if (type === "day") { data = this.dailyUptimeDataList[key]; } else { - data = this.uptimeDataList[key]; + data = this.minutelyUptimeDataList[key]; } if (data) { diff --git a/test/backend-test/test-uptime-calculator.js b/test/backend-test/test-uptime-calculator.js index f447b93f9c..b17d517b78 100644 --- a/test/backend-test/test-uptime-calculator.js +++ b/test/backend-test/test-uptime-calculator.js @@ -16,7 +16,7 @@ dayjs.extend(require("dayjs/plugin/utc")); dayjs.extend(require("../../server/modules/dayjs/plugin/timezone")); dayjs.extend(require("dayjs/plugin/customParseFormat")); -test("Test Uptime Calculator - custom date", (t) => { +test("Test Uptime Calculator - custom date", async (t) => { let c1 = new UptimeCalculator(); // Test custom date @@ -24,35 +24,35 @@ test("Test Uptime Calculator - custom date", (t) => { assert.strictEqual(c1.getCurrentDate().unix(), dayjs.utc("2021-01-01T00:00:00.000Z").unix()); }); -test("Test update - UP", (t) => { +test("Test update - UP", async (t) => { UptimeCalculator.currentDate = dayjs.utc("2023-08-12 20:46:59"); let c2 = new UptimeCalculator(); - let date = c2.update(UP); + let date = await c2.update(UP); assert.strictEqual(date.unix(), dayjs.utc("2023-08-12 20:46:59").unix()); }); -test("Test update - MAINTENANCE", (t) => { +test("Test update - MAINTENANCE", async (t) => { UptimeCalculator.currentDate = dayjs.utc("2023-08-12 20:47:20"); let c2 = new UptimeCalculator(); - let date = c2.update(MAINTENANCE); + let date = await c2.update(MAINTENANCE); assert.strictEqual(date.unix(), dayjs.utc("2023-08-12 20:47:20").unix()); }); -test("Test update - DOWN", (t) => { +test("Test update - DOWN", async (t) => { UptimeCalculator.currentDate = dayjs.utc("2023-08-12 20:47:20"); let c2 = new UptimeCalculator(); - let date = c2.update(DOWN); + let date = await c2.update(DOWN); assert.strictEqual(date.unix(), dayjs.utc("2023-08-12 20:47:20").unix()); }); -test("Test update - PENDING", (t) => { +test("Test update - PENDING", async (t) => { UptimeCalculator.currentDate = dayjs.utc("2023-08-12 20:47:20"); let c2 = new UptimeCalculator(); - let date = c2.update(PENDING); + let date = await c2.update(PENDING); assert.strictEqual(date.unix(), dayjs.utc("2023-08-12 20:47:20").unix()); }); -test("Test flatStatus", (t) => { +test("Test flatStatus", async (t) => { let c2 = new UptimeCalculator(); assert.strictEqual(c2.flatStatus(UP), UP); assert.strictEqual(c2.flatStatus(MAINTENANCE), UP); @@ -60,7 +60,7 @@ test("Test flatStatus", (t) => { assert.strictEqual(c2.flatStatus(PENDING), DOWN); }); -test("Test getMinutelyKey", (t) => { +test("Test getMinutelyKey", async (t) => { let c2 = new UptimeCalculator(); let divisionKey = c2.getMinutelyKey(dayjs.utc("2023-08-12 20:46:00")); assert.strictEqual(divisionKey, dayjs.utc("2023-08-12 20:46:00").unix()); @@ -76,7 +76,7 @@ test("Test getMinutelyKey", (t) => { assert.strictEqual(divisionKey, dayjs.utc("2023-08-12 20:46:00").unix()); }); -test("Test getDailyKey", (t) => { +test("Test getDailyKey", async (t) => { let c2 = new UptimeCalculator(); let dailyKey = c2.getDailyKey(dayjs.utc("2023-08-12 20:46:00").unix()); assert.strictEqual(dailyKey, dayjs.utc("2023-08-12").unix()); @@ -96,13 +96,13 @@ test("Test getDailyKey", (t) => { assert.strictEqual(dailyKey, dayjs.utc("2023-08-12").unix()); }); -test("Test lastDailyUptimeData", (t) => { +test("Test lastDailyUptimeData", async (t) => { let c2 = new UptimeCalculator(); - c2.update(UP); + await c2.update(UP); assert.strictEqual(c2.lastDailyUptimeData.up, 1); }); -test("Test get24HourUptime", (t) => { +test("Test get24HourUptime", async (t) => { UptimeCalculator.currentDate = dayjs.utc("2023-08-12 20:46:59"); // No data @@ -113,71 +113,71 @@ test("Test get24HourUptime", (t) => { // 1 Up c2 = new UptimeCalculator(); - c2.update(UP); + await c2.update(UP); let uptime = c2.get24Hour().uptime; assert.strictEqual(uptime, 1); // 2 Up c2 = new UptimeCalculator(); - c2.update(UP); - c2.update(UP); + await c2.update(UP); + await c2.update(UP); uptime = c2.get24Hour().uptime; assert.strictEqual(uptime, 1); // 3 Up c2 = new UptimeCalculator(); - c2.update(UP); - c2.update(UP); - c2.update(UP); + await c2.update(UP); + await c2.update(UP); + await c2.update(UP); uptime = c2.get24Hour().uptime; assert.strictEqual(uptime, 1); // 1 MAINTENANCE c2 = new UptimeCalculator(); - c2.update(MAINTENANCE); + await c2.update(MAINTENANCE); uptime = c2.get24Hour().uptime; assert.strictEqual(uptime, 1); // 1 PENDING c2 = new UptimeCalculator(); - c2.update(PENDING); + await c2.update(PENDING); uptime = c2.get24Hour().uptime; assert.strictEqual(uptime, 0); // 1 DOWN c2 = new UptimeCalculator(); - c2.update(DOWN); + await c2.update(DOWN); uptime = c2.get24Hour().uptime; assert.strictEqual(uptime, 0); // 2 DOWN c2 = new UptimeCalculator(); - c2.update(DOWN); - c2.update(DOWN); + await c2.update(DOWN); + await c2.update(DOWN); uptime = c2.get24Hour().uptime; assert.strictEqual(uptime, 0); // 1 DOWN, 1 UP c2 = new UptimeCalculator(); - c2.update(DOWN); - c2.update(UP); + await c2.update(DOWN); + await c2.update(UP); uptime = c2.get24Hour().uptime; assert.strictEqual(uptime, 0.5); // 1 UP, 1 DOWN c2 = new UptimeCalculator(); - c2.update(UP); - c2.update(DOWN); + await c2.update(UP); + await c2.update(DOWN); uptime = c2.get24Hour().uptime; assert.strictEqual(uptime, 0.5); // Add 24 hours c2 = new UptimeCalculator(); - c2.update(UP); - c2.update(UP); - c2.update(UP); - c2.update(UP); - c2.update(DOWN); + await c2.update(UP); + await c2.update(UP); + await c2.update(UP); + await c2.update(UP); + await c2.update(DOWN); uptime = c2.get24Hour().uptime; assert.strictEqual(uptime, 0.8); UptimeCalculator.currentDate = UptimeCalculator.currentDate.add(24, "hour"); @@ -194,7 +194,7 @@ test("Test get24HourUptime", (t) => { assert.strictEqual(uptime, 0.8); }); -test("Test get7DayUptime", (t) => { +test("Test get7DayUptime", async (t) => { UptimeCalculator.currentDate = dayjs.utc("2023-08-12 20:46:59"); // No data @@ -204,71 +204,71 @@ test("Test get7DayUptime", (t) => { // 1 Up c2 = new UptimeCalculator(); - c2.update(UP); + await c2.update(UP); uptime = c2.get7Day().uptime; assert.strictEqual(uptime, 1); // 2 Up c2 = new UptimeCalculator(); - c2.update(UP); - c2.update(UP); + await c2.update(UP); + await c2.update(UP); uptime = c2.get7Day().uptime; assert.strictEqual(uptime, 1); // 3 Up c2 = new UptimeCalculator(); - c2.update(UP); - c2.update(UP); - c2.update(UP); + await c2.update(UP); + await c2.update(UP); + await c2.update(UP); uptime = c2.get7Day().uptime; assert.strictEqual(uptime, 1); // 1 MAINTENANCE c2 = new UptimeCalculator(); - c2.update(MAINTENANCE); + await c2.update(MAINTENANCE); uptime = c2.get7Day().uptime; assert.strictEqual(uptime, 1); // 1 PENDING c2 = new UptimeCalculator(); - c2.update(PENDING); + await c2.update(PENDING); uptime = c2.get7Day().uptime; assert.strictEqual(uptime, 0); // 1 DOWN c2 = new UptimeCalculator(); - c2.update(DOWN); + await c2.update(DOWN); uptime = c2.get7Day().uptime; assert.strictEqual(uptime, 0); // 2 DOWN c2 = new UptimeCalculator(); - c2.update(DOWN); - c2.update(DOWN); + await c2.update(DOWN); + await c2.update(DOWN); uptime = c2.get7Day().uptime; assert.strictEqual(uptime, 0); // 1 DOWN, 1 UP c2 = new UptimeCalculator(); - c2.update(DOWN); - c2.update(UP); + await c2.update(DOWN); + await c2.update(UP); uptime = c2.get7Day().uptime; assert.strictEqual(uptime, 0.5); // 1 UP, 1 DOWN c2 = new UptimeCalculator(); - c2.update(UP); - c2.update(DOWN); + await c2.update(UP); + await c2.update(DOWN); uptime = c2.get7Day().uptime; assert.strictEqual(uptime, 0.5); // Add 7 days c2 = new UptimeCalculator(); - c2.update(UP); - c2.update(UP); - c2.update(UP); - c2.update(UP); - c2.update(DOWN); + await c2.update(UP); + await c2.update(UP); + await c2.update(UP); + await c2.update(UP); + await c2.update(DOWN); uptime = c2.get7Day().uptime; assert.strictEqual(uptime, 0.8); UptimeCalculator.currentDate = UptimeCalculator.currentDate.add(7, "day"); @@ -279,7 +279,7 @@ test("Test get7DayUptime", (t) => { }); -test("Test get30DayUptime (1 check per day)", (t) => { +test("Test get30DayUptime (1 check per day)", async (t) => { UptimeCalculator.currentDate = dayjs.utc("2023-08-12 20:46:59"); let c2 = new UptimeCalculator(); @@ -293,10 +293,10 @@ test("Test get30DayUptime (1 check per day)", (t) => { UptimeCalculator.currentDate = UptimeCalculator.currentDate.add(1, "day"); if (flip) { - c2.update(UP); + await c2.update(UP); up++; } else { - c2.update(DOWN); + await c2.update(DOWN); down++; } @@ -312,7 +312,7 @@ test("Test get30DayUptime (1 check per day)", (t) => { assert.strictEqual(c2.get7Day().uptime, 3 / 7); }); -test("Test get1YearUptime (1 check per day)", (t) => { +test("Test get1YearUptime (1 check per day)", async (t) => { UptimeCalculator.currentDate = dayjs.utc("2023-08-12 20:46:59"); let c2 = new UptimeCalculator(); @@ -324,9 +324,9 @@ test("Test get1YearUptime (1 check per day)", (t) => { UptimeCalculator.currentDate = UptimeCalculator.currentDate.add(1, "day"); if (flip) { - c2.update(UP); + await c2.update(UP); } else { - c2.update(DOWN); + await c2.update(DOWN); } uptime = c2.get30Day().time; @@ -399,7 +399,7 @@ test("Worst case", async (t) => { console.log("Final Date: ", UptimeCalculator.currentDate.format("YYYY-MM-DD HH:mm:ss")); console.log("Memory usage before preparation", memoryUsage()); - assert.strictEqual(c.uptimeDataList.length(), 1440); + assert.strictEqual(c.minutelyUptimeDataList.length(), 1440); assert.strictEqual(c.dailyUptimeDataList.length(), 365); }); From 4c0e6657c4763e76d72b93e43f8ecd7f4771aa77 Mon Sep 17 00:00:00 2001 From: Louis Lam Date: Fri, 18 Aug 2023 17:14:11 +0800 Subject: [PATCH 10/16] WIP --- .../2023-08-16-0000-create-uptime.js | 6 +- db/knex_migrations/README.md | 4 +- server/database.js | 16 +- server/jobs/clear-old-data.js | 4 +- server/model/monitor.js | 142 ++---------------- server/routers/api-router.js | 3 +- server/uptime-cache-list.js | 51 ------- server/uptime-calculator.js | 20 ++- 8 files changed, 50 insertions(+), 196 deletions(-) delete mode 100644 server/uptime-cache-list.js diff --git a/db/knex_migrations/2023-08-16-0000-create-uptime.js b/db/knex_migrations/2023-08-16-0000-create-uptime.js index 5113fcd41d..ab899311c9 100644 --- a/db/knex_migrations/2023-08-16-0000-create-uptime.js +++ b/db/knex_migrations/2023-08-16-0000-create-uptime.js @@ -9,11 +9,12 @@ exports.up = function (knex) { .onUpdate("CASCADE"); table.integer("timestamp") .notNullable() - .unique() .comment("Unix timestamp rounded down to the nearest minute"); table.float("ping").notNullable().comment("Average ping in milliseconds"); table.smallint("up").notNullable(); table.smallint("down").notNullable(); + + table.unique([ "monitor_id", "timestamp" ]); }) .createTable("stat_daily", function (table) { table.increments("id"); @@ -24,11 +25,12 @@ exports.up = function (knex) { .onUpdate("CASCADE"); table.integer("timestamp") .notNullable() - .unique() .comment("Unix timestamp rounded down to the nearest day"); table.float("ping").notNullable().comment("Average ping in milliseconds"); table.smallint("up").notNullable(); table.smallint("down").notNullable(); + + table.unique([ "monitor_id", "timestamp" ]); }); }; diff --git a/db/knex_migrations/README.md b/db/knex_migrations/README.md index 95dbcdc54f..4bebe34821 100644 --- a/db/knex_migrations/README.md +++ b/db/knex_migrations/README.md @@ -4,8 +4,8 @@ https://knexjs.org/guide/migrations.html#knexfile-in-other-languages ## Basic rules - All tables must have a primary key named `id` -- Filename format: `YYYY-MM-DD-HHMM-patch-name.js` -- Avoid native SQL syntax, use knex methods, because Uptime Kuma supports multiple databases +- Filename format: `YYYY-MM-DD-HHMM-patch-name.js` +- Avoid native SQL syntax, use knex methods, because Uptime Kuma supports SQLite and MariaDB. ## Template diff --git a/server/database.js b/server/database.js index c2617a0da2..e098039634 100644 --- a/server/database.js +++ b/server/database.js @@ -183,6 +183,12 @@ class Database { let config = {}; + let mariadbPoolConfig = { + afterCreate: function (conn, done) { + conn.query("SET time_zone = \"+00:00\";", (_) => { }); + } + }; + log.info("db", `Database Type: ${dbConfig.type}`); if (dbConfig.type === "sqlite") { @@ -233,7 +239,8 @@ class Database { user: dbConfig.username, password: dbConfig.password, database: dbConfig.dbName, - } + }, + pool: mariadbPoolConfig, }; } else if (dbConfig.type === "embedded-mariadb") { let embeddedMariaDB = EmbeddedMariaDB.getInstance(); @@ -245,7 +252,8 @@ class Database { socketPath: embeddedMariaDB.socketPath, user: "node", database: "kuma", - } + }, + pool: mariadbPoolConfig, }; } else { throw new Error("Unknown Database type: " + dbConfig.type); @@ -603,7 +611,9 @@ class Database { log.info("db", "Closing the database"); // Flush WAL to main database - await R.exec("PRAGMA wal_checkpoint(TRUNCATE)"); + if (Database.dbConfig.type === "sqlite") { + await R.exec("PRAGMA wal_checkpoint(TRUNCATE)"); + } while (true) { Database.noReject = true; diff --git a/server/jobs/clear-old-data.js b/server/jobs/clear-old-data.js index 91677f078c..248a4d409d 100644 --- a/server/jobs/clear-old-data.js +++ b/server/jobs/clear-old-data.js @@ -43,7 +43,9 @@ const clearOldData = async () => { [ parsedPeriod * -24 ] ); - await R.exec("PRAGMA optimize;"); + if (Database.dbConfig.type === "sqlite") { + await R.exec("PRAGMA optimize;"); + } } catch (e) { log.error("clearOldData", `Failed to clear old data: ${e.message}`); } diff --git a/server/model/monitor.js b/server/model/monitor.js index b907874145..ccd1f99727 100644 --- a/server/model/monitor.js +++ b/server/model/monitor.js @@ -18,7 +18,6 @@ const apicache = require("../modules/apicache"); const { UptimeKumaServer } = require("../uptime-kuma-server"); const { CacheableDnsHttpAgent } = require("../cacheable-dns-http-agent"); const { DockerHost } = require("../docker"); -const { UptimeCacheList } = require("../uptime-cache-list"); const Gamedig = require("gamedig"); const jsonata = require("jsonata"); const jwt = require("jsonwebtoken"); @@ -966,7 +965,7 @@ class Monitor extends BeanModel { } log.debug("monitor", `[${this.name}] Send to socket`); - UptimeCacheList.clearCache(this.id); + io.to(this.user_id).emit("heartbeat", bean.toJSON()); Monitor.sendStats(io, this.id, this.user_id); @@ -1147,44 +1146,28 @@ class Monitor extends BeanModel { */ static async sendStats(io, monitorID, userID) { const hasClients = getTotalClientInRoom(io, userID) > 0; + let uptimeCalculator = await UptimeCalculator.getUptimeCalculator(monitorID); if (hasClients) { - await Monitor.sendAvgPing(24, io, monitorID, userID); - await Monitor.sendUptime(24, io, monitorID, userID); - await Monitor.sendUptime(24 * 30, io, monitorID, userID); + // Send 24 hour average ping + let data24h = await uptimeCalculator.get24Hour(); + io.to(userID).emit("avgPing", monitorID, +data24h.avgPing.toFixed(2)); + + // Send 24 hour uptime + io.to(userID).emit("uptime", monitorID, 24, data24h.uptime); + + let data30d = await uptimeCalculator.get30Day(); + + // Send 30 day uptime + io.to(userID).emit("uptime", monitorID, 720, data30d.uptime); + + // Send Cert Info await Monitor.sendCertInfo(io, monitorID, userID); } else { log.debug("monitor", "No clients in the room, no need to send stats"); } } - /** - * Send the average ping to user - * @param {number} duration Hours - * @param {Server} io Socket instance to send data to - * @param {number} monitorID ID of monitor to read - * @param {number} userID ID of user to send data to - * @returns {void} - */ - static async sendAvgPing(duration, io, monitorID, userID) { - const timeLogger = new TimeLogger(); - const sqlHourOffset = Database.sqlHourOffset(); - - let avgPing = parseInt(await R.getCell(` - SELECT AVG(ping) - FROM heartbeat - WHERE time > ${sqlHourOffset} - AND ping IS NOT NULL - AND monitor_id = ? `, [ - -duration, - monitorID, - ])); - - timeLogger.print(`[Monitor: ${monitorID}] avgPing`); - - io.to(userID).emit("avgPing", monitorID, avgPing); - } - /** * Send certificate information to client * @param {Server} io Socket server instance @@ -1201,101 +1184,6 @@ class Monitor extends BeanModel { } } - /** - * Uptime with calculation - * Calculation based on: - * https://www.uptrends.com/support/kb/reporting/calculation-of-uptime-and-downtime - * @param {number} duration Hours - * @param {number} monitorID ID of monitor to calculate - * @param {boolean} forceNoCache Should the uptime be recalculated? - * @returns {number} Uptime of monitor - */ - static async calcUptime(duration, monitorID, forceNoCache = false) { - - if (!forceNoCache) { - let cachedUptime = UptimeCacheList.getUptime(monitorID, duration); - if (cachedUptime != null) { - return cachedUptime; - } - } - - const timeLogger = new TimeLogger(); - - const startTime = R.isoDateTime(dayjs.utc().subtract(duration, "hour")); - - // Handle if heartbeat duration longer than the target duration - // e.g. If the last beat's duration is bigger that the 24hrs window, it will use the duration between the (beat time - window margin) (THEN case in SQL) - let result = await R.getRow(` - SELECT - -- SUM all duration, also trim off the beat out of time window - SUM( - CASE - WHEN (JULIANDAY(\`time\`) - JULIANDAY(?)) * 86400 < duration - THEN (JULIANDAY(\`time\`) - JULIANDAY(?)) * 86400 - ELSE duration - END - ) AS total_duration, - - -- SUM all uptime duration, also trim off the beat out of time window - SUM( - CASE - WHEN (status = 1 OR status = 3) - THEN - CASE - WHEN (JULIANDAY(\`time\`) - JULIANDAY(?)) * 86400 < duration - THEN (JULIANDAY(\`time\`) - JULIANDAY(?)) * 86400 - ELSE duration - END - END - ) AS uptime_duration - FROM heartbeat - WHERE time > ? - AND monitor_id = ? - `, [ - startTime, startTime, startTime, startTime, startTime, - monitorID, - ]); - - timeLogger.print(`[Monitor: ${monitorID}][${duration}] sendUptime`); - - let totalDuration = result.total_duration; - let uptimeDuration = result.uptime_duration; - let uptime = 0; - - if (totalDuration > 0) { - uptime = uptimeDuration / totalDuration; - if (uptime < 0) { - uptime = 0; - } - - } else { - // Handle new monitor with only one beat, because the beat's duration = 0 - let status = parseInt(await R.getCell("SELECT `status` FROM heartbeat WHERE monitor_id = ?", [ monitorID ])); - - if (status === UP) { - uptime = 1; - } - } - - // Cache - UptimeCacheList.addUptime(monitorID, duration, uptime); - - return uptime; - } - - /** - * Send Uptime - * @param {number} duration Hours - * @param {Server} io Socket server instance - * @param {number} monitorID ID of monitor to send - * @param {number} userID ID of user to send to - * @returns {void} - */ - static async sendUptime(duration, io, monitorID, userID) { - const uptime = await this.calcUptime(duration, monitorID); - io.to(userID).emit("uptime", monitorID, duration, uptime); - } - /** * Has status of monitor changed since last beat? * @param {boolean} isFirstBeat Is this the first beat of this monitor? diff --git a/server/routers/api-router.js b/server/routers/api-router.js index 866ba8e1c1..bfb89da195 100644 --- a/server/routers/api-router.js +++ b/server/routers/api-router.js @@ -7,7 +7,6 @@ const dayjs = require("dayjs"); const { UP, MAINTENANCE, DOWN, PENDING, flipStatus, log } = require("../../src/util"); const StatusPage = require("../model/status_page"); const { UptimeKumaServer } = require("../uptime-kuma-server"); -const { UptimeCacheList } = require("../uptime-cache-list"); const { makeBadge } = require("badge-maker"); const { badgeConstants } = require("../config"); const { Prometheus } = require("../prometheus"); @@ -89,7 +88,7 @@ router.get("/api/push/:pushToken", async (request, response) => { await R.store(bean); io.to(monitor.user_id).emit("heartbeat", bean.toJSON()); - UptimeCacheList.clearCache(monitor.id); + Monitor.sendStats(io, monitor.id, monitor.user_id); new Prometheus(monitor).update(bean, undefined); diff --git a/server/uptime-cache-list.js b/server/uptime-cache-list.js deleted file mode 100644 index 3d2a684c94..0000000000 --- a/server/uptime-cache-list.js +++ /dev/null @@ -1,51 +0,0 @@ -const { log } = require("../src/util"); -class UptimeCacheList { - /** - * list[monitorID][duration] - */ - static list = {}; - - /** - * Get the uptime for a specific period - * @param {number} monitorID ID of monitor to query - * @param {number} duration Duration to query - * @returns {(number|null)} Uptime for provided duration, if it exists - */ - static getUptime(monitorID, duration) { - if (UptimeCacheList.list[monitorID] && UptimeCacheList.list[monitorID][duration]) { - log.debug("UptimeCacheList", "getUptime: " + monitorID + " " + duration); - return UptimeCacheList.list[monitorID][duration]; - } else { - return null; - } - } - - /** - * Add uptime for specified monitor - * @param {number} monitorID ID of monitor to insert for - * @param {number} duration Duration to insert for - * @param {number} uptime Uptime to add - * @returns {void} - */ - static addUptime(monitorID, duration, uptime) { - log.debug("UptimeCacheList", "addUptime: " + monitorID + " " + duration); - if (!UptimeCacheList.list[monitorID]) { - UptimeCacheList.list[monitorID] = {}; - } - UptimeCacheList.list[monitorID][duration] = uptime; - } - - /** - * Clear cache for specified monitor - * @param {number} monitorID ID of monitor to clear - * @returns {void} - */ - static clearCache(monitorID) { - log.debug("UptimeCacheList", "clearCache: " + monitorID); - delete UptimeCacheList.list[monitorID]; - } -} - -module.exports = { - UptimeCacheList, -}; diff --git a/server/uptime-calculator.js b/server/uptime-calculator.js index 7431bd1df3..35ca84d22a 100644 --- a/server/uptime-calculator.js +++ b/server/uptime-calculator.js @@ -97,19 +97,23 @@ class UptimeCalculator { if (flatStatus === UP) { this.minutelyUptimeDataList[divisionKey].up += 1; this.dailyUptimeDataList[dailyKey].up += 1; + + // Only UP status can update the ping + if (!isNaN(ping)) { + // Add avg ping + let count = this.minutelyUptimeDataList[divisionKey].up + this.minutelyUptimeDataList[divisionKey].down; + this.minutelyUptimeDataList[divisionKey].ping = (this.minutelyUptimeDataList[divisionKey].ping * (count - 1) + ping) / count; + + // Add avg ping (daily) + count = this.dailyUptimeDataList[dailyKey].up + this.dailyUptimeDataList[dailyKey].down; + this.dailyUptimeDataList[dailyKey].ping = (this.dailyUptimeDataList[dailyKey].ping * (count - 1) + ping) / count; + } + } else { this.minutelyUptimeDataList[divisionKey].down += 1; this.dailyUptimeDataList[dailyKey].down += 1; } - // Add avg ping - let count = this.minutelyUptimeDataList[divisionKey].up + this.minutelyUptimeDataList[divisionKey].down; - this.minutelyUptimeDataList[divisionKey].ping = (this.minutelyUptimeDataList[divisionKey].ping * (count - 1) + ping) / count; - - // Add avg ping (daily) - count = this.dailyUptimeDataList[dailyKey].up + this.dailyUptimeDataList[dailyKey].down; - this.dailyUptimeDataList[dailyKey].ping = (this.dailyUptimeDataList[dailyKey].ping * (count - 1) + ping) / count; - if (this.dailyUptimeDataList[dailyKey] !== this.lastDailyUptimeData) { this.lastDailyUptimeData = this.dailyUptimeDataList[dailyKey]; } From fdc4648ed6e219b3ba4467381af631daf92c1ddc Mon Sep 17 00:00:00 2001 From: Louis Lam Date: Sun, 20 Aug 2023 03:21:56 +0800 Subject: [PATCH 11/16] WIP --- server/database.js | 28 ++-- server/model/monitor.js | 12 +- server/routers/api-router.js | 33 ++-- server/routers/status-page-router.js | 5 +- server/server.js | 6 +- server/uptime-calculator.js | 169 +++++++++++++++----- server/util-server.js | 23 --- test/backend-test/test-uptime-calculator.js | 49 +++--- 8 files changed, 201 insertions(+), 124 deletions(-) diff --git a/server/database.js b/server/database.js index e098039634..eb3db1897b 100644 --- a/server/database.js +++ b/server/database.js @@ -358,6 +358,7 @@ class Database { } /** + * TODO * @returns {Promise} */ static async rollbackLatestPatch() { @@ -590,14 +591,6 @@ class Database { } } - /** - * Aquire a direct connection to database - * @returns {any} Database connection - */ - static getBetterSQLite3Database() { - return R.knex.client.acquireConnection(); - } - /** * Special handle, because tarn.js throw a promise reject that cannot be caught * @returns {Promise} @@ -626,20 +619,23 @@ class Database { log.info("db", "Waiting to close the database"); } } - log.info("db", "SQLite closed"); + log.info("db", "Database closed"); process.removeListener("unhandledRejection", listener); } /** - * Get the size of the database + * Get the size of the database (SQLite only) * @returns {number} Size of database */ static getSize() { - log.debug("db", "Database.getSize()"); - let stats = fs.statSync(Database.sqlitePath); - log.debug("db", stats); - return stats.size; + if (Database.dbConfig.type === "sqlite") { + log.debug("db", "Database.getSize()"); + let stats = fs.statSync(Database.sqlitePath); + log.debug("db", stats); + return stats.size; + } + return 0; } /** @@ -647,7 +643,9 @@ class Database { * @returns {Promise} */ static async shrink() { - await R.exec("VACUUM"); + if (Database.dbConfig.type === "sqlite") { + await R.exec("VACUUM"); + } } /** diff --git a/server/model/monitor.js b/server/model/monitor.js index ccd1f99727..cc2acd98b6 100644 --- a/server/model/monitor.js +++ b/server/model/monitor.js @@ -964,15 +964,17 @@ class Monitor extends BeanModel { log.warn("monitor", `Monitor #${this.id} '${this.name}': Failing: ${bean.msg} | Interval: ${beatInterval} seconds | Type: ${this.type} | Down Count: ${bean.downCount} | Resend Interval: ${this.resendInterval}`); } - log.debug("monitor", `[${this.name}] Send to socket`); - - io.to(this.user_id).emit("heartbeat", bean.toJSON()); - Monitor.sendStats(io, this.id, this.user_id); - + // Calculate uptime let uptimeCalculator = await UptimeCalculator.getUptimeCalculator(this.id); let endTimeDayjs = await uptimeCalculator.update(bean.status, parseFloat(bean.ping)); bean.end_time = R.isoDateTimeMillis(endTimeDayjs); + // Send to frontend + log.debug("monitor", `[${this.name}] Send to socket`); + io.to(this.user_id).emit("heartbeat", bean.toJSON()); + Monitor.sendStats(io, this.id, this.user_id); + + // Store to database log.debug("monitor", `[${this.name}] Store`); await R.store(bean); diff --git a/server/routers/api-router.js b/server/routers/api-router.js index bfb89da195..d480ed6ed6 100644 --- a/server/routers/api-router.js +++ b/server/routers/api-router.js @@ -11,6 +11,7 @@ const { makeBadge } = require("badge-maker"); const { badgeConstants } = require("../config"); const { Prometheus } = require("../prometheus"); const Database = require("../database"); +const { UptimeCalculator } = require("../uptime-calculator"); let router = express.Router(); @@ -205,9 +206,13 @@ router.get("/api/badge/:id/uptime/:duration?", cache("5 minutes"), async (reques try { const requestedMonitorId = parseInt(request.params.id, 10); // if no duration is given, set value to 24 (h) - const requestedDuration = request.params.duration !== undefined ? parseInt(request.params.duration, 10) : 24; + let requestedDuration = request.params.duration !== undefined ? request.params.duration : "24h"; const overrideValue = value && parseFloat(value); + if (requestedDuration === "24") { + requestedDuration = "24h"; + } + let publicMonitor = await R.getRow(` SELECT monitor_group.monitor_id FROM monitor_group, \`group\` WHERE monitor_group.group_id = \`group\`.id @@ -224,10 +229,8 @@ router.get("/api/badge/:id/uptime/:duration?", cache("5 minutes"), async (reques badgeValues.message = "N/A"; badgeValues.color = badgeConstants.naColor; } else { - const uptime = overrideValue ?? await Monitor.calcUptime( - requestedDuration, - requestedMonitorId - ); + const uptimeCalculator = await UptimeCalculator.getUptimeCalculator(requestedMonitorId); + const uptime = overrideValue ?? uptimeCalculator.getDataByDuration(requestedDuration).uptime; // limit the displayed uptime percentage to four (two, when displayed as percent) decimal digits const cleanUptime = (uptime * 100).toPrecision(4); @@ -273,21 +276,19 @@ router.get("/api/badge/:id/ping/:duration?", cache("5 minutes"), async (request, const requestedMonitorId = parseInt(request.params.id, 10); // Default duration is 24 (h) if not defined in queryParam, limited to 720h (30d) - const requestedDuration = Math.min(request.params.duration ? parseInt(request.params.duration, 10) : 24, 720); + let requestedDuration = request.params.duration !== undefined ? request.params.duration : "24h"; const overrideValue = value && parseFloat(value); + if (requestedDuration === "24") { + requestedDuration = "24h"; + } + const sqlHourOffset = Database.sqlHourOffset(); - const publicAvgPing = parseInt(await R.getCell(` - SELECT AVG(ping) FROM monitor_group, \`group\`, heartbeat - WHERE monitor_group.group_id = \`group\`.id - AND heartbeat.time > ${sqlHourOffset} - AND heartbeat.ping IS NOT NULL - AND public = 1 - AND heartbeat.monitor_id = ? - `, - [ -requestedDuration, requestedMonitorId ] - )); + // Check if monitor is public + + const uptimeCalculator = await UptimeCalculator.getUptimeCalculator(requestedMonitorId); + const publicAvgPing = uptimeCalculator.getDataByDuration(requestedDuration).avgPing; const badgeValues = { style }; diff --git a/server/routers/status-page-router.js b/server/routers/status-page-router.js index b60f286ee3..f8c347050d 100644 --- a/server/routers/status-page-router.js +++ b/server/routers/status-page-router.js @@ -7,6 +7,7 @@ const { R } = require("redbean-node"); const Monitor = require("../model/monitor"); const { badgeConstants } = require("../config"); const { makeBadge } = require("badge-maker"); +const { UptimeCalculator } = require("../uptime-calculator"); let router = express.Router(); @@ -92,8 +93,8 @@ router.get("/api/status-page/heartbeat/:slug", cache("1 minutes"), async (reques list = R.convertToBeans("heartbeat", list); heartbeatList[monitorID] = list.reverse().map(row => row.toPublicJSON()); - const type = 24; - uptimeList[`${monitorID}_${type}`] = await Monitor.calcUptime(type, monitorID); + const uptimeCalculator = await UptimeCalculator.getUptimeCalculator(monitorID); + uptimeList[`${monitorID}_24`] = uptimeCalculator.get24Hour().uptime; } response.json({ diff --git a/server/server.js b/server/server.js index cedef1d507..072f90486a 100644 --- a/server/server.js +++ b/server/server.js @@ -84,7 +84,7 @@ log.info("server", "Importing this project modules"); log.debug("server", "Importing Monitor"); const Monitor = require("./model/monitor"); log.debug("server", "Importing Settings"); -const { getSettings, setSettings, setting, initJWTSecret, checkLogin, startUnitTest, FBSD, doubleCheckPassword, startE2eTests, +const { getSettings, setSettings, setting, initJWTSecret, checkLogin, FBSD, doubleCheckPassword, startE2eTests, allowDevAllOrigin } = require("./util-server"); @@ -1659,10 +1659,6 @@ let needSetup = false; startMonitors(); checkVersion.startInterval(); - if (testMode) { - startUnitTest(); - } - if (e2eTestMode) { startE2eTests(); } diff --git a/server/uptime-calculator.js b/server/uptime-calculator.js index 35ca84d22a..3782a840c5 100644 --- a/server/uptime-calculator.js +++ b/server/uptime-calculator.js @@ -79,61 +79,100 @@ class UptimeCalculator { async init(monitorID) { this.monitorID = monitorID; - // Object.assign(new Foo, { a: 1 }) + let now = this.getCurrentDate(); + + // Load minutely data from database (recent 24 hours only) + let minutelyStatBeans = await R.find("stat_minutely", " monitor_id = ? AND timestamp > ? ORDER BY timestamp", [ + monitorID, + this.getMinutelyKey(now.subtract(24, "hour")), + ]); + + // TODO + + // Load daily data from database (recent 365 days only) + let dailyStatBeans = await R.find("stat_daily", " monitor_id = ? AND timestamp > ? ORDER BY timestamp", [ + monitorID, + this.getDailyKey(now.subtract(365, "day").unix()), + ]); + + // TODO + } /** * @param {number} status status - * @param {number} ping + * @param {number} ping Ping * @returns {dayjs.Dayjs} date * @throws {Error} Invalid status */ async update(status, ping = 0) { let date = this.getCurrentDate(); + + // Don't count MAINTENANCE into uptime + if (status === MAINTENANCE) { + return date; + } + let flatStatus = this.flatStatus(status); + + if (flatStatus === DOWN && ping > 0) { + log.warn("uptime-calc", "The ping is not effective when the status is DOWN"); + } + let divisionKey = this.getMinutelyKey(date); let dailyKey = this.getDailyKey(divisionKey); + let minutelyData = this.minutelyUptimeDataList[divisionKey]; + let dailyData = this.dailyUptimeDataList[dailyKey]; + if (flatStatus === UP) { - this.minutelyUptimeDataList[divisionKey].up += 1; - this.dailyUptimeDataList[dailyKey].up += 1; + minutelyData.up += 1; + dailyData.up += 1; // Only UP status can update the ping if (!isNaN(ping)) { // Add avg ping - let count = this.minutelyUptimeDataList[divisionKey].up + this.minutelyUptimeDataList[divisionKey].down; - this.minutelyUptimeDataList[divisionKey].ping = (this.minutelyUptimeDataList[divisionKey].ping * (count - 1) + ping) / count; + // The first beat of the minute, the ping is the current ping + if (minutelyData.up === 1) { + minutelyData.avgPing = ping; + } else { + minutelyData.avgPing = (minutelyData.avgPing * (minutelyData.up - 1) + ping) / minutelyData.up; + } // Add avg ping (daily) - count = this.dailyUptimeDataList[dailyKey].up + this.dailyUptimeDataList[dailyKey].down; - this.dailyUptimeDataList[dailyKey].ping = (this.dailyUptimeDataList[dailyKey].ping * (count - 1) + ping) / count; + // The first beat of the day, the ping is the current ping + if (minutelyData.up === 1) { + dailyData.avgPing = ping; + } else { + dailyData.avgPing = (dailyData.avgPing * (dailyData.up - 1) + ping) / dailyData.up; + } } } else { - this.minutelyUptimeDataList[divisionKey].down += 1; - this.dailyUptimeDataList[dailyKey].down += 1; + minutelyData.down += 1; + dailyData.down += 1; } - if (this.dailyUptimeDataList[dailyKey] !== this.lastDailyUptimeData) { - this.lastDailyUptimeData = this.dailyUptimeDataList[dailyKey]; + if (dailyData !== this.lastDailyUptimeData) { + this.lastDailyUptimeData = dailyData; } - if (this.minutelyUptimeDataList[divisionKey] !== this.lastUptimeData) { - this.lastUptimeData = this.minutelyUptimeDataList[divisionKey]; + if (minutelyData !== this.lastUptimeData) { + this.lastUptimeData = minutelyData; } // Update database if (!process.env.TEST_BACKEND) { let dailyStatBean = await this.getDailyStatBean(dailyKey); - dailyStatBean.up = this.dailyUptimeDataList[dailyKey].up; - dailyStatBean.down = this.dailyUptimeDataList[dailyKey].down; - dailyStatBean.ping = this.dailyUptimeDataList[dailyKey].ping; + dailyStatBean.up = dailyData.up; + dailyStatBean.down = dailyData.down; + dailyStatBean.ping = dailyData.ping; await R.store(dailyStatBean); let minutelyStatBean = await this.getMinutelyStatBean(divisionKey); - minutelyStatBean.up = this.minutelyUptimeDataList[divisionKey].up; - minutelyStatBean.down = this.minutelyUptimeDataList[divisionKey].down; - minutelyStatBean.ping = this.minutelyUptimeDataList[divisionKey].ping; + minutelyStatBean.up = minutelyData.up; + minutelyStatBean.down = minutelyData.down; + minutelyStatBean.ping = minutelyData.ping; await R.store(minutelyStatBean); } @@ -210,7 +249,7 @@ class UptimeCalculator { this.minutelyUptimeDataList.push(divisionKey, { up: 0, down: 0, - ping: 0, + avgPing: 0, }); } @@ -239,7 +278,7 @@ class UptimeCalculator { this.dailyUptimeDataList.push(dailyKey, { up: 0, down: 0, - ping: 0, + avgPing: 0, }); } @@ -255,7 +294,7 @@ class UptimeCalculator { flatStatus(status) { switch (status) { case UP: - case MAINTENANCE: + // case MAINTENANCE: return UP; case DOWN: case PENDING: @@ -286,8 +325,16 @@ class UptimeCalculator { }; let totalPing = 0; + let endTimestamp; - for (let i = 0; i < num; i++) { + if (type === "day") { + endTimestamp = key - 86400 * (num - 1); + } else { + endTimestamp = key - 60 * (num - 1); + } + + // Sum up all data in the specified time range + while (key >= endTimestamp) { let data; if (type === "day") { @@ -299,7 +346,7 @@ class UptimeCalculator { if (data) { total.up += data.up; total.down += data.down; - totalPing += data.ping; + totalPing += data.avgPing * data.up; } // Previous day @@ -310,55 +357,84 @@ class UptimeCalculator { } } + let uptimeData = new UptimeDataResult(); + if (total.up === 0 && total.down === 0) { if (type === "day" && this.lastDailyUptimeData) { total = this.lastDailyUptimeData; + totalPing = total.avgPing * total.up; } else if (type === "minute" && this.lastUptimeData) { total = this.lastUptimeData; + totalPing = total.avgPing * total.up; } else { - return { - uptime: 0, - avgPing: 0, - }; + uptimeData.uptime = 0; + uptimeData.avgPing = null; + return uptimeData; } } - return { - uptime: total.up / (total.up + total.down), - avgPing: totalPing / total.up, - }; + let avgPing; + + if (total.up === 0) { + avgPing = null; + } else { + avgPing = totalPing / total.up; + } + + uptimeData.uptime = total.up / (total.up + total.down); + uptimeData.avgPing = avgPing; + return uptimeData; } /** - * + * Get the uptime data by duration + * @param {'24h'|'30d'|'1y'} duration Only accept 24h, 30d, 1y + * @returns {UptimeDataResult} UptimeDataResult + * @throws {Error} Invalid duration + */ + getDataByDuration(duration) { + if (duration === "24h") { + return this.get24Hour(); + } else if (duration === "30d") { + return this.get30Day(); + } else if (duration === "1y") { + return this.get1Year(); + } else { + throw new Error("Invalid duration"); + } + } + + /** + * 1440 = 24 * 60mins + * @returns {UptimeDataResult} UptimeDataResult */ get24Hour() { - return this.getData(24, "minute"); + return this.getData(1440, "minute"); } /** - * + * @returns {UptimeDataResult} UptimeDataResult */ get7Day() { return this.getData(7); } /** - * + * @returns {UptimeDataResult} UptimeDataResult */ get30Day() { return this.getData(30); } /** - * + * @returns {UptimeDataResult} UptimeDataResult */ get1Year() { return this.getData(365); } /** - * + * @returns {UptimeDataResult} UptimeDataResult */ getCurrentDate() { return dayjs.utc(); @@ -366,6 +442,19 @@ class UptimeCalculator { } +class UptimeDataResult { + /** + * @type {number} Uptime + */ + uptime; + + /** + * @type {number} Average ping + */ + avgPing; +} + module.exports = { - UptimeCalculator + UptimeCalculator, + UptimeDataResult, }; diff --git a/server/util-server.js b/server/util-server.js index 64c4a0e1cb..1540f68953 100644 --- a/server/util-server.js +++ b/server/util-server.js @@ -847,29 +847,6 @@ exports.doubleCheckPassword = async (socket, currentPassword) => { return user; }; -/** - * Start Unit tests - * @returns {void} - */ -exports.startUnitTest = async () => { - console.log("Starting unit test..."); - const npm = /^win/.test(process.platform) ? "npm.cmd" : "npm"; - const child = childProcess.spawn(npm, [ "run", "test-backend" ]); - - child.stdout.on("data", (data) => { - console.log(data.toString()); - }); - - child.stderr.on("data", (data) => { - console.log(data.toString()); - }); - - child.on("close", function (code) { - console.log("Jest exit code: " + code); - process.exit(code); - }); -}; - /** * Start end-to-end tests * @returns {void} diff --git a/test/backend-test/test-uptime-calculator.js b/test/backend-test/test-uptime-calculator.js index b17d517b78..b56ac946da 100644 --- a/test/backend-test/test-uptime-calculator.js +++ b/test/backend-test/test-uptime-calculator.js @@ -55,7 +55,7 @@ test("Test update - PENDING", async (t) => { test("Test flatStatus", async (t) => { let c2 = new UptimeCalculator(); assert.strictEqual(c2.flatStatus(UP), UP); - assert.strictEqual(c2.flatStatus(MAINTENANCE), UP); + //assert.strictEqual(c2.flatStatus(MAINTENANCE), UP); assert.strictEqual(c2.flatStatus(DOWN), DOWN); assert.strictEqual(c2.flatStatus(PENDING), DOWN); }); @@ -102,53 +102,59 @@ test("Test lastDailyUptimeData", async (t) => { assert.strictEqual(c2.lastDailyUptimeData.up, 1); }); -test("Test get24HourUptime", async (t) => { +test("Test get24Hour Uptime and Avg Ping", async (t) => { UptimeCalculator.currentDate = dayjs.utc("2023-08-12 20:46:59"); // No data let c2 = new UptimeCalculator(); let data = c2.get24Hour(); assert.strictEqual(data.uptime, 0); - assert.strictEqual(data.avgPing, 0); + assert.strictEqual(data.avgPing, null); // 1 Up c2 = new UptimeCalculator(); - await c2.update(UP); + await c2.update(UP, 100); let uptime = c2.get24Hour().uptime; assert.strictEqual(uptime, 1); + assert.strictEqual(c2.get24Hour().avgPing, 100); // 2 Up c2 = new UptimeCalculator(); - await c2.update(UP); - await c2.update(UP); + await c2.update(UP, 100); + await c2.update(UP, 200); uptime = c2.get24Hour().uptime; assert.strictEqual(uptime, 1); + assert.strictEqual(c2.get24Hour().avgPing, 150); // 3 Up c2 = new UptimeCalculator(); - await c2.update(UP); - await c2.update(UP); - await c2.update(UP); + await c2.update(UP, 0); + await c2.update(UP, 100); + await c2.update(UP, 400); uptime = c2.get24Hour().uptime; assert.strictEqual(uptime, 1); + assert.strictEqual(c2.get24Hour().avgPing, 166.66666666666666); // 1 MAINTENANCE c2 = new UptimeCalculator(); await c2.update(MAINTENANCE); uptime = c2.get24Hour().uptime; - assert.strictEqual(uptime, 1); + assert.strictEqual(uptime, 0); + assert.strictEqual(c2.get24Hour().avgPing, null); // 1 PENDING c2 = new UptimeCalculator(); await c2.update(PENDING); uptime = c2.get24Hour().uptime; assert.strictEqual(uptime, 0); + assert.strictEqual(c2.get24Hour().avgPing, null); // 1 DOWN c2 = new UptimeCalculator(); await c2.update(DOWN); uptime = c2.get24Hour().uptime; assert.strictEqual(uptime, 0); + assert.strictEqual(c2.get24Hour().avgPing, null); // 2 DOWN c2 = new UptimeCalculator(); @@ -156,35 +162,41 @@ test("Test get24HourUptime", async (t) => { await c2.update(DOWN); uptime = c2.get24Hour().uptime; assert.strictEqual(uptime, 0); + assert.strictEqual(c2.get24Hour().avgPing, null); // 1 DOWN, 1 UP c2 = new UptimeCalculator(); await c2.update(DOWN); - await c2.update(UP); + await c2.update(UP, 0.5); uptime = c2.get24Hour().uptime; assert.strictEqual(uptime, 0.5); + assert.strictEqual(c2.get24Hour().avgPing, 0.5); // 1 UP, 1 DOWN c2 = new UptimeCalculator(); - await c2.update(UP); + await c2.update(UP, 123); await c2.update(DOWN); uptime = c2.get24Hour().uptime; assert.strictEqual(uptime, 0.5); + assert.strictEqual(c2.get24Hour().avgPing, 123); // Add 24 hours c2 = new UptimeCalculator(); - await c2.update(UP); - await c2.update(UP); - await c2.update(UP); - await c2.update(UP); + await c2.update(UP, 0); + await c2.update(UP, 0); + await c2.update(UP, 0); + await c2.update(UP, 1); await c2.update(DOWN); uptime = c2.get24Hour().uptime; assert.strictEqual(uptime, 0.8); + assert.strictEqual(c2.get24Hour().avgPing, 0.25); + UptimeCalculator.currentDate = UptimeCalculator.currentDate.add(24, "hour"); // After 24 hours, even if there is no data, the uptime should be still 80% uptime = c2.get24Hour().uptime; assert.strictEqual(uptime, 0.8); + assert.strictEqual(c2.get24Hour().avgPing, 0.25); // Add more 24 hours (48 hours) UptimeCalculator.currentDate = UptimeCalculator.currentDate.add(24, "hour"); @@ -192,6 +204,7 @@ test("Test get24HourUptime", async (t) => { // After 48 hours, even if there is no data, the uptime should be still 80% uptime = c2.get24Hour().uptime; assert.strictEqual(uptime, 0.8); + assert.strictEqual(c2.get24Hour().avgPing, 0.25); }); test("Test get7DayUptime", async (t) => { @@ -227,7 +240,7 @@ test("Test get7DayUptime", async (t) => { c2 = new UptimeCalculator(); await c2.update(MAINTENANCE); uptime = c2.get7Day().uptime; - assert.strictEqual(uptime, 1); + assert.strictEqual(uptime, 0); // 1 PENDING c2 = new UptimeCalculator(); @@ -387,7 +400,7 @@ test("Worst case", async (t) => { } else if (rand < 0.75) { c.update(MAINTENANCE); if (UptimeCalculator.currentDate.unix() > actualStartDate) { - up++; + //up++; } } else { c.update(PENDING); From fd5b0e657f7277eccc72b03f822dc6f50f8872c3 Mon Sep 17 00:00:00 2001 From: Louis Lam Date: Sun, 20 Aug 2023 03:24:02 +0800 Subject: [PATCH 12/16] WIP --- src/components/NotificationDialog.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/NotificationDialog.vue b/src/components/NotificationDialog.vue index e2024aaebd..170a1d4cf5 100644 --- a/src/components/NotificationDialog.vue +++ b/src/components/NotificationDialog.vue @@ -199,7 +199,7 @@ export default { }, }, - watch: { + watch: { "notification.type"(to, from) { let oldName; if (from) { From d67a82a8c1f68f7fb8ea3c50cfde8b327785612e Mon Sep 17 00:00:00 2001 From: Louis Lam Date: Wed, 30 Aug 2023 02:31:41 +0800 Subject: [PATCH 13/16] WIP --- package-lock.json | 2 +- server/model/monitor.js | 10 ++++++---- server/uptime-calculator.js | 35 ++++++++++++++++++++++------------- src/components/Uptime.vue | 4 +++- src/pages/Details.vue | 13 +++++++++++++ 5 files changed, 45 insertions(+), 19 deletions(-) diff --git a/package-lock.json b/package-lock.json index 21e2bc32eb..611011e231 100644 --- a/package-lock.json +++ b/package-lock.json @@ -119,7 +119,7 @@ "stylelint": "^15.10.1", "stylelint-config-standard": "~25.0.0", "terser": "~5.15.0", - "test": "^3.3.0", + "test": "~3.3.0", "timezones-list": "~3.0.1", "typescript": "~4.4.4", "v-pagination-3": "~0.1.7", diff --git a/server/model/monitor.js b/server/model/monitor.js index cc2acd98b6..cffd60b09f 100644 --- a/server/model/monitor.js +++ b/server/model/monitor.js @@ -21,7 +21,6 @@ const { DockerHost } = require("../docker"); const Gamedig = require("gamedig"); const jsonata = require("jsonata"); const jwt = require("jsonwebtoken"); -const Database = require("../database"); const { UptimeCalculator } = require("../uptime-calculator"); /** @@ -1153,16 +1152,19 @@ class Monitor extends BeanModel { if (hasClients) { // Send 24 hour average ping let data24h = await uptimeCalculator.get24Hour(); - io.to(userID).emit("avgPing", monitorID, +data24h.avgPing.toFixed(2)); + io.to(userID).emit("avgPing", monitorID, (data24h.avgPing) ? data24h.avgPing.toFixed(2) : null); // Send 24 hour uptime io.to(userID).emit("uptime", monitorID, 24, data24h.uptime); - let data30d = await uptimeCalculator.get30Day(); - // Send 30 day uptime + let data30d = await uptimeCalculator.get30Day(); io.to(userID).emit("uptime", monitorID, 720, data30d.uptime); + // Send 1-year uptime + let data1y = await uptimeCalculator.get1Year(); + io.to(userID).emit("uptime", monitorID, "1y", data1y.uptime); + // Send Cert Info await Monitor.sendCertInfo(io, monitorID, userID); } else { diff --git a/server/uptime-calculator.js b/server/uptime-calculator.js index 3782a840c5..2172c77bec 100644 --- a/server/uptime-calculator.js +++ b/server/uptime-calculator.js @@ -161,21 +161,30 @@ class UptimeCalculator { this.lastUptimeData = minutelyData; } - // Update database + // Don't store data in test mode if (!process.env.TEST_BACKEND) { - let dailyStatBean = await this.getDailyStatBean(dailyKey); - dailyStatBean.up = dailyData.up; - dailyStatBean.down = dailyData.down; - dailyStatBean.ping = dailyData.ping; - await R.store(dailyStatBean); - - let minutelyStatBean = await this.getMinutelyStatBean(divisionKey); - minutelyStatBean.up = minutelyData.up; - minutelyStatBean.down = minutelyData.down; - minutelyStatBean.ping = minutelyData.ping; - await R.store(minutelyStatBean); + return date; } + let dailyStatBean = await this.getDailyStatBean(dailyKey); + dailyStatBean.up = dailyData.up; + dailyStatBean.down = dailyData.down; + dailyStatBean.ping = dailyData.ping; + await R.store(dailyStatBean); + + let minutelyStatBean = await this.getMinutelyStatBean(divisionKey); + minutelyStatBean.up = minutelyData.up; + minutelyStatBean.down = minutelyData.down; + minutelyStatBean.ping = minutelyData.ping; + await R.store(minutelyStatBean); + + // Remove the old data + log.debug("uptime-calc", "Remove old data"); + await R.exec("DELETE FROM stat_minutely WHERE monitor_id = ? AND timestamp < ?", [ + this.monitorID, + this.getMinutelyKey(date.subtract(24, "hour")), + ]); + return date; } @@ -434,7 +443,7 @@ class UptimeCalculator { } /** - * @returns {UptimeDataResult} UptimeDataResult + * @returns {dayjs.Dayjs} Current date */ getCurrentDate() { return dayjs.utc(); diff --git a/src/components/Uptime.vue b/src/components/Uptime.vue index afb82fa5ee..64bbd4e51d 100644 --- a/src/components/Uptime.vue +++ b/src/components/Uptime.vue @@ -84,10 +84,12 @@ export default { }, title() { + if (this.type === "1y") { + return `1${this.$t("-year")}`; + } if (this.type === "720") { return `30${this.$t("-day")}`; } - return `24${this.$t("-hour")}`; } }, diff --git a/src/pages/Details.vue b/src/pages/Details.vue index 77a1009ee1..847fcb5748 100644 --- a/src/pages/Details.vue +++ b/src/pages/Details.vue @@ -95,6 +95,8 @@ + +

{{ $t("Uptime") }}

(24{{ $t("-hour") }})

@@ -102,6 +104,8 @@
+ +

{{ $t("Uptime") }}

(30{{ $t("-day") }})

@@ -110,6 +114,15 @@
+ +
+

{{ $t("Uptime") }}

+

(1{{ $t("-year") }})

+ + + +
+

{{ $t("Cert Exp.") }}

()

From d0c2a5e292a856bef11c1cdb4bdb52874ac481d2 Mon Sep 17 00:00:00 2001 From: Louis Lam Date: Wed, 30 Aug 2023 02:42:04 +0800 Subject: [PATCH 14/16] WIP --- server/uptime-calculator.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/uptime-calculator.js b/server/uptime-calculator.js index 2172c77bec..05854ff446 100644 --- a/server/uptime-calculator.js +++ b/server/uptime-calculator.js @@ -162,7 +162,8 @@ class UptimeCalculator { } // Don't store data in test mode - if (!process.env.TEST_BACKEND) { + if (process.env.TEST_BACKEND) { + log.debug("uptime-calc", "Skip storing data in test mode"); return date; } From ad0d4116f94adcf60df5e6a9bd1347e25137f982 Mon Sep 17 00:00:00 2001 From: Louis Lam Date: Fri, 1 Sep 2023 05:03:44 +0800 Subject: [PATCH 15/16] WIP --- .../2023-08-18-0301-heartbeat.js | 5 +---- server/client.js | 6 ++---- server/database.js | 3 ++- server/uptime-calculator.js | 19 ++++++++++++++++--- 4 files changed, 21 insertions(+), 12 deletions(-) diff --git a/db/knex_migrations/2023-08-18-0301-heartbeat.js b/db/knex_migrations/2023-08-18-0301-heartbeat.js index 6b391e7fff..fe4152b488 100644 --- a/db/knex_migrations/2023-08-18-0301-heartbeat.js +++ b/db/knex_migrations/2023-08-18-0301-heartbeat.js @@ -2,10 +2,7 @@ exports.up = function (knex) { // Add new column heartbeat.end_time return knex.schema .alterTable("heartbeat", function (table) { - table.timestamp("end_time").nullable().defaultTo(null); - - // Change time's datatype to timestamp - table.timestamp("time").alter(); + table.datetime("end_time").nullable().defaultTo(null); }); }; diff --git a/server/client.js b/server/client.js index e00fdb1ee5..167c3f59a9 100644 --- a/server/client.js +++ b/server/client.js @@ -45,8 +45,6 @@ async function sendNotificationList(socket) { * @returns {Promise} */ async function sendHeartbeatList(socket, monitorID, toUser = false, overwrite = false) { - const timeLogger = new TimeLogger(); - let list = await R.getAll(` SELECT * FROM heartbeat WHERE monitor_id = ? @@ -58,13 +56,13 @@ async function sendHeartbeatList(socket, monitorID, toUser = false, overwrite = let result = list.reverse(); + console.log(result); + if (toUser) { io.to(socket.userID).emit("heartbeatList", monitorID, result, overwrite); } else { socket.emit("heartbeatList", monitorID, result, overwrite); } - - timeLogger.print(`[Monitor: ${monitorID}] sendHeartbeatList`); } /** diff --git a/server/database.js b/server/database.js index eb3db1897b..736d7043ce 100644 --- a/server/database.js +++ b/server/database.js @@ -185,7 +185,7 @@ class Database { let mariadbPoolConfig = { afterCreate: function (conn, done) { - conn.query("SET time_zone = \"+00:00\";", (_) => { }); + } }; @@ -239,6 +239,7 @@ class Database { user: dbConfig.username, password: dbConfig.password, database: dbConfig.dbName, + timezone: "UTC", }, pool: mariadbPoolConfig, }; diff --git a/server/uptime-calculator.js b/server/uptime-calculator.js index 05854ff446..2b7f8b2e95 100644 --- a/server/uptime-calculator.js +++ b/server/uptime-calculator.js @@ -87,7 +87,14 @@ class UptimeCalculator { this.getMinutelyKey(now.subtract(24, "hour")), ]); - // TODO + for (let bean of minutelyStatBeans) { + let key = bean.timestamp; + this.minutelyUptimeDataList.push(key, { + up: bean.up, + down: bean.down, + avgPing: bean.ping, + }); + } // Load daily data from database (recent 365 days only) let dailyStatBeans = await R.find("stat_daily", " monitor_id = ? AND timestamp > ? ORDER BY timestamp", [ @@ -95,8 +102,14 @@ class UptimeCalculator { this.getDailyKey(now.subtract(365, "day").unix()), ]); - // TODO - + for (let bean of dailyStatBeans) { + let key = bean.timestamp; + this.dailyUptimeDataList.push(key, { + up: bean.up, + down: bean.down, + avgPing: bean.ping, + }); + } } /** From 7e2ba4ef86fe27c5f5c9c8071ae1dbbcd68c970e Mon Sep 17 00:00:00 2001 From: Louis Lam Date: Fri, 1 Sep 2023 05:17:58 +0800 Subject: [PATCH 16/16] WIP --- server/client.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/client.js b/server/client.js index 167c3f59a9..b25aa3d6a0 100644 --- a/server/client.js +++ b/server/client.js @@ -56,8 +56,6 @@ async function sendHeartbeatList(socket, monitorID, toUser = false, overwrite = let result = list.reverse(); - console.log(result); - if (toUser) { io.to(socket.userID).emit("heartbeatList", monitorID, result, overwrite); } else {