-
Notifications
You must be signed in to change notification settings - Fork 51
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Adding healtcheck and reconnection mechanism to the postgres archive …
…driver (#1997) It starts an asynchronous infinite task that checks the connectivity with the database. In case of error, the postgres_healthcheck task tries to reconnect for a while, and if it determines that the connection cannot be resumed, then it invokes a callback indicating that situation. For the case of the `wakunode2` app, this callback quits the application itself and adds a log trace indicating the connectivity issue with the database.
- Loading branch information
1 parent
5638bd0
commit 1fb13b0
Showing
6 changed files
with
87 additions
and
6 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
47 changes: 47 additions & 0 deletions
47
waku/waku_archive/driver/postgres_driver/postgres_healthcheck.nim
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,47 @@ | ||
when (NimMajor, NimMinor) < (1, 4): | ||
{.push raises: [Defect].} | ||
else: | ||
{.push raises: [].} | ||
|
||
import | ||
chronos, | ||
stew/results | ||
import | ||
../../driver, | ||
../../../common/databases/db_postgres | ||
|
||
## Simple query to validate that the postgres is working and attending requests | ||
const HealthCheckQuery = "SELECT version();" | ||
const CheckConnectivityInterval = 30.seconds | ||
const MaxNumTrials = 20 | ||
const TrialInterval = 1.seconds | ||
|
||
proc checkConnectivity*(connPool: PgAsyncPool, | ||
onErrAction: OnErrHandler) {.async.} = | ||
|
||
while true: | ||
|
||
(await connPool.exec(HealthCheckQuery)).isOkOr: | ||
|
||
## The connection failed once. Let's try reconnecting for a while. | ||
## Notice that the 'exec' proc tries to establish a new connection. | ||
|
||
block errorBlock: | ||
## Force close all the opened connections. No need to close gracefully. | ||
(await connPool.resetConnPool()).isOkOr: | ||
onErrAction("checkConnectivity resetConnPool error: " & error) | ||
|
||
var numTrial = 0 | ||
while numTrial < MaxNumTrials: | ||
let res = await connPool.exec(HealthCheckQuery) | ||
if res.isOk(): | ||
## Connection resumed. Let's go back to the normal healthcheck. | ||
break errorBlock | ||
|
||
await sleepAsync(TrialInterval) | ||
numTrial.inc() | ||
|
||
## The connection couldn't be resumed. Let's inform the upper layers. | ||
onErrAction("postgres health check error: " & error) | ||
|
||
await sleepAsync(CheckConnectivityInterval) |