Skip to content

Commit

Permalink
Revert "Restart running monitors if no heartbeat (#3952)" (#4088)
Browse files Browse the repository at this point in the history 
This reverts commit c43223a.
  • Loading branch information
@louislam
louislam authored Nov 23, 2023
1 parent 8e61158 commit 121d1a1
Show file tree
Hide file tree
Showing 2 changed files with 1 addition and 106 deletions.
18 changes: 1 addition & 17 deletions server/model/monitor.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ const dayjs = require("dayjs");
const axios = require("axios");
const { Prometheus } = require("../prometheus");
const { log, UP, DOWN, PENDING, MAINTENANCE, flipStatus, TimeLogger, MAX_INTERVAL_SECOND, MIN_INTERVAL_SECOND,
SQL_DATETIME_FORMAT, isDev, sleep, getRandomInt
SQL_DATETIME_FORMAT
} = require("../../src/util");
const { tcping, ping, dnsResolve, checkCertificate, checkStatusCode, getTotalClientInRoom, setting, mssqlQuery, postgresQuery, mysqlQuery, mqttAsync, setSetting, httpNtlm, radius, grpcQuery,
redisPingAsync, mongodbPing, kafkaProducerAsync, getOidcTokenClientCredentials, rootCertificatesFingerprints, axiosAbortSignal
Expand Down Expand Up @@ -329,16 +329,6 @@ class Monitor extends BeanModel {
}
}

// Evil
if (isDev) {
if (process.env.EVIL_RANDOM_MONITOR_SLEEP === "SURE") {
if (getRandomInt(0, 100) === 0) {
log.debug("evil", `[${this.name}] Evil mode: Random sleep: ` + beatInterval * 10000);
await sleep(beatInterval * 10000);
}
}
}

// Expose here for prometheus update
// undefined if not https
let tlsInfo = undefined;
Expand Down Expand Up @@ -1024,7 +1014,6 @@ class Monitor extends BeanModel {
if (! this.isStop) {
log.debug("monitor", `[${this.name}] SetTimeout for next check.`);
this.heartbeatInterval = setTimeout(safeBeat, beatInterval * 1000);
this.lastScheduleBeatTime = dayjs();
} else {
log.info("monitor", `[${this.name}] isStop = true, no next check.`);
}
Expand All @@ -1034,9 +1023,7 @@ class Monitor extends BeanModel {
/** Get a heartbeat and handle errors */
const safeBeat = async () => {
try {
this.lastStartBeatTime = dayjs();
await beat();
this.lastEndBeatTime = dayjs();
} catch (e) {
console.trace(e);
UptimeKumaServer.errorLog(e, false);
Expand All @@ -1045,9 +1032,6 @@ class Monitor extends BeanModel {
if (! this.isStop) {
log.info("monitor", "Try to restart the monitor");
this.heartbeatInterval = setTimeout(safeBeat, this.interval * 1000);
this.lastScheduleBeatTime = dayjs();
} else {
log.info("monitor", "isStop = true, no next check.");
}
}
};
Expand Down
89 changes: 0 additions & 89 deletions server/uptime-kuma-server.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ const { Settings } = require("./settings");
const dayjs = require("dayjs");
const childProcess = require("child_process");
const path = require("path");
const axios = require("axios");
// DO NOT IMPORT HERE IF THE MODULES USED `UptimeKumaServer.getInstance()`, put at the bottom of this file instead.

/**
Expand Down Expand Up @@ -63,8 +62,6 @@ class UptimeKumaServer {
*/
jwtSecret = null;

checkMonitorsInterval = null;

static getInstance(args) {
if (UptimeKumaServer.instance == null) {
UptimeKumaServer.instance = new UptimeKumaServer(args);
Expand All @@ -78,9 +75,6 @@ class UptimeKumaServer {
const sslCert = args["ssl-cert"] || process.env.UPTIME_KUMA_SSL_CERT || process.env.SSL_CERT || undefined;
const sslKeyPassphrase = args["ssl-key-passphrase"] || process.env.UPTIME_KUMA_SSL_KEY_PASSPHRASE || process.env.SSL_KEY_PASSPHRASE || undefined;

// Set default axios timeout to 5 minutes instead of infinity
axios.defaults.timeout = 300 * 1000;

log.info("server", "Creating express and socket.io instance");
this.app = express();
if (sslKey && sslCert) {
Expand Down Expand Up @@ -352,10 +346,6 @@ class UptimeKumaServer {
if (enable || enable === null) {
this.startNSCDServices();
}

this.checkMonitorsInterval = setInterval(() => {
this.checkMonitors();
}, 60 * 1000);
}

/**
Expand All @@ -368,8 +358,6 @@ class UptimeKumaServer {
if (enable || enable === null) {
this.stopNSCDServices();
}

clearInterval(this.checkMonitorsInterval);
}

/**
Expand Down Expand Up @@ -400,83 +388,6 @@ class UptimeKumaServer {
}
}
}

/**
* Start the specified monitor
* @param {number} monitorID ID of monitor to start
* @returns {Promise<void>}
*/
async startMonitor(monitorID) {
log.info("manage", `Resume Monitor: ${monitorID} by server`);

await R.exec("UPDATE monitor SET active = 1 WHERE id = ?", [
monitorID,
]);

let monitor = await R.findOne("monitor", " id = ? ", [
monitorID,
]);

if (monitor.id in this.monitorList) {
this.monitorList[monitor.id].stop();
}

this.monitorList[monitor.id] = monitor;
monitor.start(this.io);
}

/**
* Restart a given monitor
* @param {number} monitorID ID of monitor to start
* @returns {Promise<void>}
*/
async restartMonitor(monitorID) {
return await this.startMonitor(monitorID);
}

/**
* Check if monitors are running properly
*/
async checkMonitors() {
log.debug("monitor_checker", "Checking monitors");

for (let monitorID in this.monitorList) {
let monitor = this.monitorList[monitorID];

// Not for push monitor
if (monitor.type === "push") {
continue;
}

if (!monitor.active) {
continue;
}

// Check the lastStartBeatTime, if it is too long, then restart
if (monitor.lastScheduleBeatTime ) {
let diff = dayjs().diff(monitor.lastStartBeatTime, "second");

if (diff > monitor.interval * 1.5) {
log.error("monitor_checker", `Monitor Interval: ${monitor.interval} Monitor ` + monitorID + " lastStartBeatTime diff: " + diff);
log.error("monitor_checker", "Unexpected error: Monitor " + monitorID + " is struck for unknown reason");
log.error("monitor_checker", "Last start beat time: " + R.isoDateTime(monitor.lastStartBeatTime));
log.error("monitor_checker", "Last end beat time: " + R.isoDateTime(monitor.lastEndBeatTime));
log.error("monitor_checker", "Last ScheduleBeatTime: " + R.isoDateTime(monitor.lastScheduleBeatTime));

// Restart
log.error("monitor_checker", `Restarting monitor ${monitorID} automatically now`);
this.restartMonitor(monitorID);
} else {
//log.debug("monitor_checker", "Monitor " + monitorID + " is running normally");
}
} else {
//log.debug("monitor_checker", "Monitor " + monitorID + " is not started yet, skipp");
}

}

log.debug("monitor_checker", "Checking monitors end");
}
}

module.exports = {
Expand Down

0 comments on commit 121d1a1

Please sign in to comment.