diff --git a/tests/waku_archive/test_retention_policy.nim b/tests/waku_archive/test_retention_policy.nim index fbbcca4218..ff039adef9 100644 --- a/tests/waku_archive/test_retention_policy.nim +++ b/tests/waku_archive/test_retention_policy.nim @@ -62,8 +62,8 @@ suite "Waku Archive - Retention policy": test "size retention policy - windowed message deletion": ## Given let - # in megabytes - sizeLimit:float = 0.05 + # in bytes + sizeLimit:int64 = 52428 excess = 325 let driver = newTestArchiveDriver() @@ -92,9 +92,7 @@ suite "Waku Archive - Retention policy": ## Then # calculate the current database size - let pageSize = (waitFor driver.getPagesSize()).tryGet() - let pageCount = (waitFor driver.getPagesCount()).tryGet() - let sizeDB = float(pageCount * pageSize) / (1024.0 * 1024.0) + let sizeDB = int64((waitFor driver.getDatabaseSize()).tryGet()) # NOTE: since vacuumin is done manually, this needs to be revisited if vacuuming done automatically @@ -105,7 +103,7 @@ suite "Waku Archive - Retention policy": require (sizeDB >= sizeLimit) require (waitFor retentionPolicy.execute(driver)).isOk() - # get the number or rows from DB + # get the number or rows from database let rowCountAfterDeletion = (waitFor driver.getMessagesCount()).tryGet() check: diff --git a/waku/common/databases/db_sqlite.nim b/waku/common/databases/db_sqlite.nim index 837cf3ab86..f30fbdbd61 100644 --- a/waku/common/databases/db_sqlite.nim +++ b/waku/common/databases/db_sqlite.nim @@ -295,6 +295,20 @@ proc getPageCount*(db: SqliteDatabase): DatabaseResult[int64] = return ok(count) +proc getDatabaseSize*(db: SqliteDatabase): DatabaseResult[int64] = + # get the database page size in bytes + var pageSize: int64 = ?db.getPageSize() + + if pageSize == 0: + return err("failed to get page size ") + + # get the database page count + let pageCount = ?db.getPageCount() + + let databaseSize = (pageSize * pageCount) + + return ok(databaseSize) + proc gatherSqlitePageStats*(db: SqliteDatabase): DatabaseResult[(int64, int64, int64)] = let diff --git a/waku/waku_archive/driver.nim b/waku/waku_archive/driver.nim index 38cffb372f..d6cccd6721 100644 --- a/waku/waku_archive/driver.nim +++ b/waku/waku_archive/driver.nim @@ -52,6 +52,9 @@ method getPagesCount*(driver: ArchiveDriver): method getPagesSize*(driver: ArchiveDriver): Future[ArchiveDriverResult[int64]] {.base, async.} = discard +method getDatabaseSize*(driver: ArchiveDriver): + Future[ArchiveDriverResult[int64]] {.base, async.} = discard + method performVacuum*(driver: ArchiveDriver): Future[ArchiveDriverResult[void]] {.base, async.} = discard diff --git a/waku/waku_archive/driver/postgres_driver/postgres_driver.nim b/waku/waku_archive/driver/postgres_driver/postgres_driver.nim index e41f85b77c..11f7793816 100644 --- a/waku/waku_archive/driver/postgres_driver/postgres_driver.nim +++ b/waku/waku_archive/driver/postgres_driver/postgres_driver.nim @@ -434,6 +434,15 @@ proc getInt(s: PostgresDriver, return ok(retInt) +method getDatabaseSize*(s: PostgresDriver): + Future[ArchiveDriverResult[int64]] {.async.} = + + let intRes = (await s.getInt("SELECT pg_database_size(current_database())")).valueOr: + return err("error in getDatabaseSize: " & error) + + let databaseSize: int64 = int64(intRes) + return ok(databaseSize) + method getMessagesCount*(s: PostgresDriver): Future[ArchiveDriverResult[int64]] {.async.} = diff --git a/waku/waku_archive/driver/queue_driver/queue_driver.nim b/waku/waku_archive/driver/queue_driver/queue_driver.nim index 948521ccbb..9fd266e6d8 100644 --- a/waku/waku_archive/driver/queue_driver/queue_driver.nim +++ b/waku/waku_archive/driver/queue_driver/queue_driver.nim @@ -289,6 +289,10 @@ method getPagesSize*(driver: QueueDriver): Future[ArchiveDriverResult[int64]] {.async} = return ok(int64(driver.len())) +method getDatabasesSize*(driver: QueueDriver): + Future[ArchiveDriverResult[int64]] {.async} = + return ok(int64(driver.len())) + method performVacuum*(driver: QueueDriver): Future[ArchiveDriverResult[void]] {.async.} = return err("interface method not implemented") diff --git a/waku/waku_archive/driver/sqlite_driver/sqlite_driver.nim b/waku/waku_archive/driver/sqlite_driver/sqlite_driver.nim index a6039d4ece..1d0cbc5bc5 100644 --- a/waku/waku_archive/driver/sqlite_driver/sqlite_driver.nim +++ b/waku/waku_archive/driver/sqlite_driver/sqlite_driver.nim @@ -120,6 +120,10 @@ method getPagesSize*(s: SqliteDriver): Future[ArchiveDriverResult[int64]] {.async.} = return s.db.getPageSize() +method getDatabaseSize*(s: SqliteDriver): + Future[ArchiveDriverResult[int64]] {.async.} = + return s.db.getDatabaseSize() + method performVacuum*(s: SqliteDriver): Future[ArchiveDriverResult[void]] {.async.} = return s.db.performSqliteVacuum() diff --git a/waku/waku_archive/retention_policy/builder.nim b/waku/waku_archive/retention_policy/builder.nim index 2c4b3cf12a..0588eb4bbc 100644 --- a/waku/waku_archive/retention_policy/builder.nim +++ b/waku/waku_archive/retention_policy/builder.nim @@ -56,24 +56,27 @@ proc new*(T: type RetentionPolicy, var retentionSize: string retentionSize = policyArgs - # captures the size unit such as Gb or Mb + # captures the size unit such as GB or MB let sizeUnit = retentionSize.substr(retentionSize.len-2) # captures the string type number data of the size provided let sizeQuantityStr = retentionSize.substr(0,retentionSize.len-3) # to hold the numeric value data of size - var sizeQuantity: float + var inptSizeQuantity: float + var sizeQuantity: int64 if sizeUnit in ["gb", "Gb", "GB", "gB"]: # parse the actual value into integer type var try: - sizeQuantity = parseFloat(sizeQuantityStr) + inptSizeQuantity = parseFloat(sizeQuantityStr) except ValueError: return err("invalid size retention policy argument: " & getCurrentExceptionMsg()) - # Gb data is converted into Mb for uniform processing - sizeQuantity = sizeQuantity * 1024 + # GB data is converted into bytes for uniform processing + sizeQuantity = int64(inptSizeQuantity * 1024.0 * 1024.0 * 1024.0) elif sizeUnit in ["mb", "Mb", "MB", "mB"]: try: - sizeQuantity = parseFloat(sizeQuantityStr) + inptSizeQuantity = parseFloat(sizeQuantityStr) + # MB data is converted into bytes for uniform processing + sizeQuantity = int64(inptSizeQuantity * 1024.0 * 1024.0) except ValueError: return err("invalid size retention policy argument") else: diff --git a/waku/waku_archive/retention_policy/retention_policy_size.nim b/waku/waku_archive/retention_policy/retention_policy_size.nim index 8d645083bd..8b512230b2 100644 --- a/waku/waku_archive/retention_policy/retention_policy_size.nim +++ b/waku/waku_archive/retention_policy/retention_policy_size.nim @@ -16,22 +16,22 @@ import logScope: topics = "waku archive retention_policy" -# default size is 30 Gb -const DefaultRetentionSize*: float = 30_720 +# default size is 30 GiB or 32212254720.0 in bytes +const DefaultRetentionSize*: int64 = 32212254720 # to remove 20% of the outdated data from database const DeleteLimit = 0.80 type # SizeRetentionPolicy implements auto delete as follows: - # - sizeLimit is the size in megabytes (Mbs) the database can grow upto + # - sizeLimit is the size in bytes the database can grow upto # to reduce the size of the databases, remove the rows/number-of-messages # DeleteLimit is the total number of messages to delete beyond this limit # when the database size crosses the sizeLimit, then only a fraction of messages are kept, # rest of the outdated message are deleted using deleteOldestMessagesNotWithinLimit(), # upon deletion process the fragmented space is retrieve back using Vacuum process. SizeRetentionPolicy* = ref object of RetentionPolicy - sizeLimit: float + sizeLimit: int64 proc init*(T: type SizeRetentionPolicy, size=DefaultRetentionSize): T = SizeRetentionPolicy( @@ -42,21 +42,12 @@ method execute*(p: SizeRetentionPolicy, driver: ArchiveDriver): Future[RetentionPolicyResult[void]] {.async.} = ## when db size overshoots the database limit, shread 20% of outdated messages + # get size of database + let dbSize = (await driver.getDatabaseSize()).valueOr: + return err("failed to get database size: " & $error) - # get page size of database - let pageSizeRes = await driver.getPagesSize() - let pageSize: int64 = int64(pageSizeRes.valueOr(0) div 1024) - - if pageSize == 0: - return err("failed to get Page size: " & pageSizeRes.error) - - # to get the size of the database, pageCount and PageSize is required - # get page count in "messages" database - let pageCount = (await driver.getPagesCount()).valueOr: - return err("failed to get Pages count: " & $error) - - # database size in megabytes (Mb) - let totalSizeOfDB: float = float(pageSize * pageCount)/1024.0 + # database size in bytes + let totalSizeOfDB: int64 = int64(dbSize) if totalSizeOfDB < p.sizeLimit: return ok()