From a99480fcd1f6592424687ea3b66dc1d109de0739 Mon Sep 17 00:00:00 2001 From: Pablo Castellano Date: Sat, 1 Jun 2024 20:04:11 +0200 Subject: [PATCH 1/8] Swarm batch information updater Introduce new periodic job to retrieve Swarm batch information such as label, TTL and others. A new environment variable SWARM_BATCH_ID is used to define the batch id used to store blobs. We are querying the bee endpoint directly because bee-js doesn't support the /stamps endpoint yet. --- .nvmrc | 2 +- apps/docs/src/app/docs/environment/page.md | 11 +- apps/rest-api-server/package.json | 1 + apps/rest-api-server/src/env.ts | 6 + apps/rest-api-server/src/index.ts | 10 + packages/db/prisma/schema.prisma | 14 +- packages/swarm-syncer/package.json | 30 +++ packages/swarm-syncer/src/PeriodicUpdater.ts | 120 ++++++++++ packages/swarm-syncer/src/SwarmStampSyncer.ts | 67 ++++++ packages/swarm-syncer/src/env.ts | 17 ++ packages/swarm-syncer/src/errors.ts | 21 ++ packages/swarm-syncer/src/index.ts | 2 + packages/swarm-syncer/src/logger.ts | 3 + .../swarm-syncer/src/updaters/SwarmUpdater.ts | 46 ++++ packages/swarm-syncer/src/utils.ts | 13 + .../test/OverallStatsUpdater.test.ts | 222 ++++++++++++++++++ .../swarm-syncer/test/PeriodicUpdater.test.ts | 125 ++++++++++ .../test/SwarmStampSyncer.test.ts | 120 ++++++++++ .../PeriodicUpdater.test.ts.snap | 9 + .../__snapshots__/StatsSyncer.test.ts.snap | 9 + packages/swarm-syncer/tsconfig.json | 4 + packages/swarm-syncer/vitest.config.ts | 5 + pnpm-lock.yaml | 121 ++++------ turbo.json | 37 ++- 24 files changed, 923 insertions(+), 92 deletions(-) create mode 100644 packages/swarm-syncer/package.json create mode 100644 packages/swarm-syncer/src/PeriodicUpdater.ts create mode 100644 packages/swarm-syncer/src/SwarmStampSyncer.ts create mode 100644 packages/swarm-syncer/src/env.ts create mode 100644 packages/swarm-syncer/src/errors.ts create mode 100644 packages/swarm-syncer/src/index.ts create mode 100644 packages/swarm-syncer/src/logger.ts create mode 100644 packages/swarm-syncer/src/updaters/SwarmUpdater.ts create mode 100644 packages/swarm-syncer/src/utils.ts create mode 100644 packages/swarm-syncer/test/OverallStatsUpdater.test.ts create mode 100644 packages/swarm-syncer/test/PeriodicUpdater.test.ts create mode 100644 packages/swarm-syncer/test/SwarmStampSyncer.test.ts create mode 100644 packages/swarm-syncer/test/__snapshots__/PeriodicUpdater.test.ts.snap create mode 100644 packages/swarm-syncer/test/__snapshots__/StatsSyncer.test.ts.snap create mode 100644 packages/swarm-syncer/tsconfig.json create mode 100644 packages/swarm-syncer/vitest.config.ts diff --git a/.nvmrc b/.nvmrc index 1e24c0215..87834047a 100644 --- a/.nvmrc +++ b/.nvmrc @@ -1 +1 @@ -18.15.0 \ No newline at end of file +20.12.2 diff --git a/apps/docs/src/app/docs/environment/page.md b/apps/docs/src/app/docs/environment/page.md index 0802535fd..fb09b75b1 100644 --- a/apps/docs/src/app/docs/environment/page.md +++ b/apps/docs/src/app/docs/environment/page.md @@ -65,11 +65,12 @@ At the moment Postgres is the default storage and Blobscan won't be able to run **Ethereum Swarm** -| Variable | Description | Required | Default value | -| ----------------------- | -------------------- | -------- | ------------- | -| `SWARM_STORAGE_ENABLED` | Store blobs in Swarm | No | `false` | -| `BEE_ENDPOINT` | Bee endpoint | No | (empty) | -| `BEE_DEBUG_ENDPOINT` | Bee debug endpoint | No | (empty) | +| Variable | Description | Required | Default value | +| ----------------------- | -------------------------- | -------- | ------------- | +| `SWARM_BATCH_ID` | Swarm address of the stamp | No | (empty) | +| `SWARM_STORAGE_ENABLED` | Store blobs in Swarm | No | `false` | +| `BEE_ENDPOINT` | Bee endpoint | No | (empty) | +| `BEE_DEBUG_ENDPOINT` | Bee debug endpoint | No | (empty) | ## Blob propagator diff --git a/apps/rest-api-server/package.json b/apps/rest-api-server/package.json index cacc1b9a3..68af875f1 100644 --- a/apps/rest-api-server/package.json +++ b/apps/rest-api-server/package.json @@ -19,6 +19,7 @@ "@blobscan/logger": "workspace:^0.1.0", "@blobscan/open-telemetry": "workspace:^0.0.7", "@blobscan/stats-syncer": "workspace:^0.1.8", + "@blobscan/swarm-syncer": "workspace:^0.0.1", "@blobscan/zod": "workspace:^0.1.0", "@opentelemetry/instrumentation-express": "^0.33.0", "@sentry/node": "^7.109.0", diff --git a/apps/rest-api-server/src/env.ts b/apps/rest-api-server/src/env.ts index 09b03292a..b910f4591 100644 --- a/apps/rest-api-server/src/env.ts +++ b/apps/rest-api-server/src/env.ts @@ -11,6 +11,9 @@ import { export const env = createEnv({ envOptions: { server: { + // FIXME + // BEE_ENDPOINT: requiredStorageConfigSchema("SWARM", z.string().url()), + BEE_ENDPOINT: z.string().optional(), BLOBSCAN_API_BASE_URL: z .string() .url() @@ -24,6 +27,9 @@ export const env = createEnv({ DENCUN_FORK_SLOT: z.coerce.number().optional(), STATS_SYNCER_DAILY_CRON_PATTERN: z.string().default("30 0 * * * *"), STATS_SYNCER_OVERALL_CRON_PATTERN: z.string().default("*/15 * * * *"), + SWARM_BATCH_ID: z.string().optional(), + SWARM_STORAGE_ENABLED: booleanSchema.default("false"), + SWARM_SYNCER_CRON: z.string().default("42 * * * *"), SENTRY_DSN_API: z.string().url().optional(), }, diff --git a/apps/rest-api-server/src/index.ts b/apps/rest-api-server/src/index.ts index e62bf6f75..b1f8b35ad 100644 --- a/apps/rest-api-server/src/index.ts +++ b/apps/rest-api-server/src/index.ts @@ -15,6 +15,7 @@ import { } from "@blobscan/api"; import { collectDefaultMetrics } from "@blobscan/open-telemetry"; import { StatsSyncer } from "@blobscan/stats-syncer"; +import { SwarmStampSyncer } from "@blobscan/swarm-syncer"; import { env } from "./env"; import { logger } from "./logger"; @@ -37,6 +38,15 @@ statsSyncer.start({ }, }); + +if (env.SWARM_STORAGE_ENABLED && env.SWARM_BATCH_ID) { + const swarmSyncer = new SwarmStampSyncer({ + redisUri: env.REDIS_URI, + env.BEE_ENDPOINT, //FIXME + ); + swarmSyncer.start(env.SWARM_SYNCER_CRON); +} + const app = express(); app.use(cors()); diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma index b6b3b93fd..49f354b55 100644 --- a/packages/db/prisma/schema.prisma +++ b/packages/db/prisma/schema.prisma @@ -53,10 +53,18 @@ model BlockchainSyncState { @@map("blockchain_sync_state") } +//TODO: Rename to SwarmBatchStorage model BlobStoragesState { - id Int @id @default(autoincrement()) - swarmDataId String? @map("swarm_data_id") - swarmDataTTL Int? @map("swarm_data_ttl") + id Int @id @default(autoincrement()) + swarmDataId String? @map("swarm_data_id") // TODO: rename to batchId + swarmDataTTL Int? @map("swarm_data_ttl") // TODO: rename to batchTtl + updatedAt DateTime @default(now()) @map("updated_at") + label String @map("label") + depth Int @map("depth") + utilization Int @map("utilization") + amount String @map("amount") + immutableFlag Boolean @map("immutable_flag") + exists Boolean @map("exists") @@map("blob_storages_state") } diff --git a/packages/swarm-syncer/package.json b/packages/swarm-syncer/package.json new file mode 100644 index 000000000..ef11757ed --- /dev/null +++ b/packages/swarm-syncer/package.json @@ -0,0 +1,30 @@ +{ + "name": "@blobscan/swarm-syncer", + "description": "Blobscan's swarm stamps data synchronizer", + "private": true, + "version": "0.0.1", + "main": "./src/index.ts", + "scripts": { + "clean": "rm -rf .turbo node_modules", + "lint": "eslint .", + "lint:fix": "pnpm lint --fix", + "type-check": "tsc --noEmit", + "test": "pnpm with-env:test vitest", + "test:ui": "pnpm with-env:test vitest --ui", + "with-env:test": ". ../../.env.test --" + }, + "dependencies": { + "@blobscan/dayjs": "workspace:^0.0.2", + "@blobscan/db": "workspace:^0.7.0", + "@blobscan/logger": "workspace:^0.1.0", + "axios": "^1.7.2", + "bullmq": "^4.13.2", + "ioredis": "^5.3.2" + }, + "eslintConfig": { + "root": true, + "extends": [ + "@blobscan/eslint-config/base" + ] + } +} diff --git a/packages/swarm-syncer/src/PeriodicUpdater.ts b/packages/swarm-syncer/src/PeriodicUpdater.ts new file mode 100644 index 000000000..03dd4bab1 --- /dev/null +++ b/packages/swarm-syncer/src/PeriodicUpdater.ts @@ -0,0 +1,120 @@ +/* eslint-disable @typescript-eslint/no-misused-promises */ +import { Queue, Worker } from "bullmq"; +import type { Redis } from "ioredis"; + +import { createModuleLogger } from "@blobscan/logger"; +import type { Logger } from "@blobscan/logger"; + +import { ErrorException, PeriodicUpdaterError } from "./errors"; +import { createRedisConnection } from "./utils"; + +// TODO: Refactor, this file is duplicate (packages: stats-syncer and swarm-syncer) +export type PeriodicUpdaterConfig = { + name: string; + redisUriOrConnection: string | Redis; + updaterFn: () => Promise; +}; + +export class PeriodicUpdater { + name: string; + protected worker: Worker; + protected queue: Queue; + protected updaterFn: () => Promise; + protected logger: Logger; + + constructor({ + name, + redisUriOrConnection, + updaterFn, + }: PeriodicUpdaterConfig) { + const isRedisUri = typeof redisUriOrConnection === "string"; + this.name = name; + this.logger = createModuleLogger("stats-syncer", this.name); + + let connection: Redis; + + if (isRedisUri) { + connection = createRedisConnection(redisUriOrConnection); + + connection.on("error", (err) => { + this.logger.error( + new ErrorException("A Redis connection error ocurred", err) + ); + }); + } else { + connection = redisUriOrConnection; + } + + this.queue = new Queue(this.name, { + connection, + }); + + this.worker = new Worker(this.queue.name, updaterFn, { + connection, + }); + + this.updaterFn = updaterFn; + + this.queue.on("error", (err) => { + this.logger.error(new ErrorException("A queue error occurred", err)); + }); + + this.worker.on("failed", (_, err) => { + this.logger.error(new ErrorException("A worker error ocurred", err)); + }); + } + + async start(cronPattern: string) { + try { + const jobName = `${this.name}-job`; + const repeatableJob = await this.queue.add(jobName, null, { + repeat: { + pattern: cronPattern, + }, + }); + + return repeatableJob; + } catch (err) { + throw new PeriodicUpdaterError( + this.name, + "An error ocurred when starting updater", + err + ); + } + } + + close() { + const teardownPromise: Promise = Promise.resolve(); + + return teardownPromise + .finally(async () => { + await this.#performClosingOperation(() => + this.worker.removeAllListeners().close(true) + ); + }) + .finally(async () => { + await this.#performClosingOperation(() => + this.queue.obliterate({ force: true }) + ); + }) + .finally(async () => { + await this.#performClosingOperation(() => + this.queue.removeAllListeners().close() + ); + }); + } + + async #performClosingOperation(operation: () => Promise) { + try { + await operation(); + } catch (err) { + const err_ = new PeriodicUpdaterError( + this.name, + "An error ocurred when performing closing operation", + err + ); + + throw err_; + } + } +} diff --git a/packages/swarm-syncer/src/SwarmStampSyncer.ts b/packages/swarm-syncer/src/SwarmStampSyncer.ts new file mode 100644 index 000000000..836bd08f2 --- /dev/null +++ b/packages/swarm-syncer/src/SwarmStampSyncer.ts @@ -0,0 +1,67 @@ +/* eslint-disable @typescript-eslint/no-misused-promises */ +import type { Redis } from "ioredis"; + +import { ErrorException, SwarmStampSyncerError } from "./errors"; +import { logger } from "./logger"; +import { SwarmStampUpdater } from "./updaters/SwarmUpdater"; +import { createRedisConnection } from "./utils"; + +export type SwarmStampSyncerOptions = { + redisUri: string; + beeEndpoint: string; +}; + +export class SwarmStampSyncer { + protected connection: Redis; + protected swarmUpdater: SwarmStampUpdater; + + constructor({ redisUri, beeEndpoint }: SwarmStampSyncerOptions) { + const connection = createRedisConnection(redisUri); + + connection.on("error", (err) => { + logger.error(new ErrorException("The Redis connection failed", err)); + }); + + this.connection = connection; + this.swarmUpdater = new SwarmStampUpdater(connection, beeEndpoint); + } + + async start(cronPattern: string) { + try { + await this.swarmUpdater.start(cronPattern); + logger.info("Swarm stamp syncer started successfully."); + } catch (err) { + const err_ = new SwarmStampSyncerError( + "An error occurred when starting swarm stamps syncer", + err + ); + + logger.error(err_); + + throw err_; + } + } + + async close() { + try { + await this.swarmUpdater + .close() + .finally(() => { + this.connection.removeAllListeners(); + + if (this.connection.status === "ready") this.connection.disconnect(); + }); + + logger.info("Stats syncer closed successfully."); + } catch (err) { + const err_ = new SwarmStampSyncerError( + "An error ocurred when closing syncer", + err + ); + + logger.error(err_); + + throw err_; + } + } +} diff --git a/packages/swarm-syncer/src/env.ts b/packages/swarm-syncer/src/env.ts new file mode 100644 index 000000000..9e4e5a43e --- /dev/null +++ b/packages/swarm-syncer/src/env.ts @@ -0,0 +1,17 @@ +import { + booleanSchema, + createEnv, + presetEnvOptions, +} from "@blobscan/zod"; + +export const env = createEnv({ + envOptions: { + server: { + SWARM_STORAGE_ENABLED: booleanSchema.default("false"), + }, + + ...presetEnvOptions, + }, +}); + +export type EnvVars = typeof env; diff --git a/packages/swarm-syncer/src/errors.ts b/packages/swarm-syncer/src/errors.ts new file mode 100644 index 000000000..0c67edd9f --- /dev/null +++ b/packages/swarm-syncer/src/errors.ts @@ -0,0 +1,21 @@ +export class ErrorException extends Error { + constructor(message: string, cause?: unknown) { + super(message, { + cause, + }); + + this.name = this.constructor.name; + } +} + +export class SwarmStampSyncerError extends ErrorException { + constructor(message: string) { + super(`Stats syncer failed: ${message}`); + } +} + +export class PeriodicUpdaterError extends ErrorException { + constructor(updaterName: string, message: string, cause: unknown) { + super(`Updater "${updaterName}" failed: ${message}`, cause); + } +} diff --git a/packages/swarm-syncer/src/index.ts b/packages/swarm-syncer/src/index.ts new file mode 100644 index 000000000..ab099d4a6 --- /dev/null +++ b/packages/swarm-syncer/src/index.ts @@ -0,0 +1,2 @@ +export { SwarmStampSyncer } from "./SwarmStampSyncer"; +export type { PeriodicUpdater, PeriodicUpdaterConfig } from "./PeriodicUpdater"; diff --git a/packages/swarm-syncer/src/logger.ts b/packages/swarm-syncer/src/logger.ts new file mode 100644 index 000000000..a7022ae8a --- /dev/null +++ b/packages/swarm-syncer/src/logger.ts @@ -0,0 +1,3 @@ +import { createModuleLogger } from "@blobscan/logger"; + +export const logger = createModuleLogger("swarm-stamp-syncer"); diff --git a/packages/swarm-syncer/src/updaters/SwarmUpdater.ts b/packages/swarm-syncer/src/updaters/SwarmUpdater.ts new file mode 100644 index 000000000..a2a28e842 --- /dev/null +++ b/packages/swarm-syncer/src/updaters/SwarmUpdater.ts @@ -0,0 +1,46 @@ +import type { Redis } from "ioredis"; +import { prisma } from "@blobscan/db"; +import { env } from "../env"; +const axios = require('axios'); +import { PeriodicUpdater } from "../PeriodicUpdater"; +import { SwarmStampSyncerError } from "../errors"; + +export class SwarmStampUpdater extends PeriodicUpdater { + constructor( + redisUriOrConnection: string | Redis, + beeEndpoint: string, + ) { + const name = "swarm-stamp"; + super({ + name, + redisUriOrConnection, + updaterFn: async () => { + const batchId = env.SWARM_BATCH_ID; + const url = `${beeEndpoint}/stamps/${batchId}`; + const response = await axios.get(url); + const data = response.data; + + if (response.status != 200) { + throw new SwarmStampSyncerError(`Stamps endpoint returned status ${response.status} for batch id ${batchId}`); + } + + await prisma.blobStoragesState.update({ + data: { + swarmDataTTL: data.batchTTL, + label: data.label, + depth: data.depth, + utilization: data.utilization, + amount: data.amount, + immutableFlag: data.immutableFlag, + exists: data.exists, + }, + where: { + swarmDataId: batchId, + }, + }); + + this.logger.info(`Updated swarm stamp ${batchId}`); + }, + }); + } +} diff --git a/packages/swarm-syncer/src/utils.ts b/packages/swarm-syncer/src/utils.ts new file mode 100644 index 000000000..24f5cbbf7 --- /dev/null +++ b/packages/swarm-syncer/src/utils.ts @@ -0,0 +1,13 @@ +import { Redis } from "ioredis"; + +import dayjs from "@blobscan/dayjs"; + +export function createRedisConnection(uri: string) { + return new Redis(uri, { + maxRetriesPerRequest: null, + }); +} + +export function formatDate(date: Date | string | dayjs.Dayjs) { + return dayjs(date).format("YYYY-MM-DD"); +} diff --git a/packages/swarm-syncer/test/OverallStatsUpdater.test.ts b/packages/swarm-syncer/test/OverallStatsUpdater.test.ts new file mode 100644 index 000000000..bed3404d7 --- /dev/null +++ b/packages/swarm-syncer/test/OverallStatsUpdater.test.ts @@ -0,0 +1,222 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { prisma } from "@blobscan/db"; +import { fixtures, omitDBTimestampFields } from "@blobscan/test"; + +import { OverallStatsUpdater } from "../src/updaters/OverallStatsUpdater"; + +class OverallStatsUpdaterMock extends OverallStatsUpdater { + constructor( + redisUri = process.env.REDIS_URI ?? "", + config: ConstructorParameters[1] = {} + ) { + const lowestSlot = + config.lowestSlot ?? fixtures.blockchainSyncState[0]?.lastLowerSyncedSlot; + super(redisUri, { + ...config, + lowestSlot, + }); + } + + getWorker() { + return this.worker; + } + + getWorkerProcessor() { + return this.updaterFn; + } + + getQueue() { + return this.queue; + } +} + +function getAllOverallStats() { + const uniqueArgs = { + where: { + id: 1, + }, + }; + + return Promise.all([ + prisma.blobOverallStats.findUnique(uniqueArgs), + prisma.blockOverallStats.findUnique(uniqueArgs), + prisma.transactionOverallStats.findUnique(uniqueArgs), + ]).then((allOverallStats) => + allOverallStats.map((stats) => + stats ? omitDBTimestampFields(stats) : undefined + ) + ); +} + +describe("OverallStatsUpdater", () => { + let overallStatsUpdater: OverallStatsUpdaterMock; + + beforeEach(() => { + overallStatsUpdater = new OverallStatsUpdaterMock(); + + return async () => { + await overallStatsUpdater.close(); + }; + }); + + it("should aggregate all overall stats correctly", async () => { + const workerProcessor = overallStatsUpdater.getWorkerProcessor(); + + const incrementTransactionSpy = vi.spyOn(prisma, "$transaction"); + + await workerProcessor(); + + const [blobOverallStats, blockOverallStats, transactionOverallStats] = + await getAllOverallStats(); + + expect( + incrementTransactionSpy, + "Expect to aggregate overall stats within a transaction" + ).toHaveBeenCalledOnce(); + expect(blobOverallStats, "Incorrect blob overall stats aggregation") + .toMatchInlineSnapshot(` + { + "avgBlobSize": 1175, + "id": 1, + "totalBlobSize": 9400n, + "totalBlobs": 8, + "totalUniqueBlobs": 1, + } + `); + expect(blockOverallStats, "Incorrect block overall stats aggregation") + .toMatchInlineSnapshot(` + { + "avgBlobAsCalldataFee": 5406666.666666667, + "avgBlobFee": 114000000, + "avgBlobGasPrice": 21.33333333333333, + "id": 1, + "totalBlobAsCalldataFee": "16220000", + "totalBlobAsCalldataGasUsed": "760000", + "totalBlobFee": "342000000", + "totalBlobGasUsed": "16000000", + "totalBlocks": 3, + } + `); + expect( + transactionOverallStats, + "Incorrect transaction overall stats aggregation" + ).toMatchInlineSnapshot(` + { + "avgMaxBlobGasFee": 100, + "id": 1, + "totalTransactions": 4, + "totalUniqueReceivers": 0, + "totalUniqueSenders": 0, + } + `); + }); + + it("should aggregate overall stats in batches correctly when there are too many blocks", async () => { + const batchSize = 2; + const workerProcessor = new OverallStatsUpdaterMock(undefined, { + batchSize, + }).getWorkerProcessor(); + const incrementTransactionSpy = vi.spyOn(prisma, "$transaction"); + const blockchainSyncState = fixtures.blockchainSyncState[0]; + const lastAggregatedBlock = blockchainSyncState + ? blockchainSyncState.lastAggregatedBlock + 1 + : 0; + const lastFinalizedBlock = + fixtures.blockchainSyncState[0]?.lastFinalizedBlock ?? 0; + const batches = Math.ceil( + (lastFinalizedBlock - lastAggregatedBlock + 1) / batchSize + ); + + await workerProcessor(); + + expect( + incrementTransactionSpy, + "Incorrect number of stats aggregation calls" + ).toHaveBeenCalledTimes(batches); + }); + + it("should update last aggregated block to last finalized block after aggregation", async () => { + const workerProcessor = overallStatsUpdater.getWorkerProcessor(); + const expectedLastAggregatedBlock = + fixtures.blockchainSyncState[0]?.lastFinalizedBlock; + + await workerProcessor(); + + const lastAggregatedBlock = await prisma.blockchainSyncState + .findUnique({ + select: { + lastAggregatedBlock: true, + }, + where: { + id: 1, + }, + }) + .then((state) => state?.lastAggregatedBlock); + + expect(lastAggregatedBlock).toBe(expectedLastAggregatedBlock); + }); + + it("should skip aggregation when no finalized block has been set", async () => { + const workerProcessor = overallStatsUpdater.getWorkerProcessor(); + + await prisma.blockchainSyncState.update({ + data: { + lastFinalizedBlock: null, + }, + where: { + id: 1, + }, + }); + + await workerProcessor(); + + const allOverallStats = await getAllOverallStats().then((allOverallStats) => + allOverallStats.filter((stats) => !!stats) + ); + + expect(allOverallStats).toEqual([]); + }); + + it("should skip aggregation when no blocks have been indexed yet", async () => { + const workerProcessor = overallStatsUpdater.getWorkerProcessor(); + + vi.spyOn(prisma.block, "findLatest").mockResolvedValueOnce(null); + + await workerProcessor(); + + const allOverallStats = await getAllOverallStats().then((allOverallStats) => + allOverallStats.filter((stats) => !!stats) + ); + + expect(allOverallStats).toEqual([]); + }); + + it("should skip aggregation when the lowest slot hasn't been reached yet", async () => { + const workerProcessor = new OverallStatsUpdaterMock(undefined, { + lowestSlot: 1, + }).getWorkerProcessor(); + + await workerProcessor(); + + const allOverallStats = await getAllOverallStats().then((allOverallStats) => + allOverallStats.filter((stats) => !!stats) + ); + + expect(allOverallStats).toEqual([]); + }); + + it("should skip aggregation when there is no new finalized blocks", async () => { + const workerProcessor = overallStatsUpdater.getWorkerProcessor(); + + await workerProcessor(); + + const allOverallStats = await getAllOverallStats(); + + await workerProcessor(); + + const allOverallStatsAfter = await getAllOverallStats(); + + expect(allOverallStats).toEqual(allOverallStatsAfter); + }); +}); diff --git a/packages/swarm-syncer/test/PeriodicUpdater.test.ts b/packages/swarm-syncer/test/PeriodicUpdater.test.ts new file mode 100644 index 000000000..9354c905e --- /dev/null +++ b/packages/swarm-syncer/test/PeriodicUpdater.test.ts @@ -0,0 +1,125 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { testValidError } from "@blobscan/test"; + +import { PeriodicUpdater } from "../src/PeriodicUpdater"; +import type { PeriodicUpdaterConfig } from "../src/PeriodicUpdater"; +import { PeriodicUpdaterError } from "../src/errors"; + +class PeriodicUpdaterMock extends PeriodicUpdater { + constructor({ + name, + redisUriOrConnection, + updaterFn, + }: Partial = {}) { + super({ + name: name ?? "test-updater", + redisUriOrConnection: redisUriOrConnection ?? "redis://localhost:6379/1", + updaterFn: updaterFn ?? (() => Promise.resolve()), + }); + } + + getWorker() { + return this.worker; + } + + getQueue() { + return this.queue; + } +} + +describe("PeriodicUpdater", () => { + let periodicUpdater: PeriodicUpdaterMock; + + beforeEach(() => { + periodicUpdater = new PeriodicUpdaterMock(); + + return async () => { + await periodicUpdater.close(); + }; + }); + + it("should create an updater correctly", async () => { + const queue = periodicUpdater.getQueue(); + const worker = periodicUpdater.getWorker(); + const isPaused = await queue.isPaused(); + + expect(worker.isRunning(), "Expected worker to be running").toBeTruthy(); + expect(isPaused, "Expected queue to be running").toBeFalsy(); + }); + + describe("when running an updater", () => { + it("should set up a repeatable job correctly", async () => { + const queue = periodicUpdater.getQueue(); + const cronPattern = "* * * * *"; + + await periodicUpdater.start(cronPattern); + + const jobs = await queue.getRepeatableJobs(); + + expect(jobs.length, "Expected one repeatable job").toBe(1); + expect(jobs[0]?.pattern, "Repetable job cron pattern mismatch").toEqual( + cronPattern + ); + }); + + testValidError( + "should throw a valid error when failing to run", + async () => { + const queue = periodicUpdater.getQueue(); + + vi.spyOn(queue, "add").mockRejectedValueOnce(new Error("Queue error")); + + await periodicUpdater.start("* * * * *"); + }, + PeriodicUpdaterError, + { checkCause: true } + ); + }); + + describe("when closing an updater", () => { + it("should close correctly", async () => { + const queue = periodicUpdater.getQueue(); + const worker = periodicUpdater.getWorker(); + + const queueCloseSpy = vi.spyOn(queue, "close").mockResolvedValueOnce(); + const queueRemoveAllListenersSpy = vi + .spyOn(queue, "removeAllListeners") + .mockReturnValueOnce(queue); + + const workerCloseSpy = vi.spyOn(worker, "close").mockResolvedValueOnce(); + const workerRemoveAllListenersSpy = vi + .spyOn(worker, "removeAllListeners") + .mockReturnValueOnce(worker); + + await periodicUpdater.close(); + + expect(queueCloseSpy).toHaveBeenCalledOnce(); + expect(workerCloseSpy).toHaveBeenCalledOnce(); + + expect(queueRemoveAllListenersSpy).toHaveBeenCalledOnce(); + expect(workerRemoveAllListenersSpy).toHaveBeenCalledOnce(); + }); + }); + + testValidError( + "should throw a valid error when failing to close it", + async () => { + const queue = periodicUpdater.getQueue(); + const worker = periodicUpdater.getWorker(); + + vi.spyOn(queue, "close").mockRejectedValueOnce( + new Error("Queue closing error") + ); + vi.spyOn(worker, "close").mockRejectedValueOnce( + new Error("Worker closing error") + ); + + await periodicUpdater.close(); + }, + PeriodicUpdaterError, + { + checkCause: true, + } + ); +}); diff --git a/packages/swarm-syncer/test/SwarmStampSyncer.test.ts b/packages/swarm-syncer/test/SwarmStampSyncer.test.ts new file mode 100644 index 000000000..586199941 --- /dev/null +++ b/packages/swarm-syncer/test/SwarmStampSyncer.test.ts @@ -0,0 +1,120 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { testValidError } from "@blobscan/test"; + +import { StatsSyncer } from "../src/StatsSyncer"; +import { StatsSyncerError } from "../src/errors"; + +class StatsSyncerMock extends StatsSyncer { + constructor(redisUri = "redis://localhost:6379/1") { + super({ redisUri }); + } + + getConnection() { + return this.connection; + } + + getDailyStatsUpdater() { + return this.dailyStatsUpdater; + } + + getOverallStatsUpdater() { + return this.overallStatsUpdater; + } +} + +describe("StatsSyncer", () => { + let statsSyncer: StatsSyncerMock; + + beforeEach(() => { + statsSyncer = new StatsSyncerMock(); + + return async () => { + await statsSyncer.close(); + }; + }); + + describe("when running the stats syncer", () => { + it("should start updaters correctly", async () => { + const dailyStatsUpdaterStartSpy = vi.spyOn( + statsSyncer.getDailyStatsUpdater(), + "start" + ); + const overallStatsUpdaterStartSpy = vi.spyOn( + statsSyncer.getOverallStatsUpdater(), + "start" + ); + + await statsSyncer.start({ + cronPatterns: { daily: "* * * * *", overall: "* * * * *" }, + }); + + expect(dailyStatsUpdaterStartSpy).toHaveBeenCalledWith("* * * * *"); + expect(overallStatsUpdaterStartSpy).toHaveBeenCalledWith("* * * * *"); + }); + + testValidError( + "should throw a valid error when failing to start it", + async () => { + const cronPatterns = { + daily: "* * * * *", + overall: "* * * * *", + }; + + vi.spyOn( + statsSyncer.getDailyStatsUpdater(), + "start" + ).mockRejectedValueOnce( + new Error( + "Something happened when trying to start daily stats updater" + ) + ); + + await statsSyncer.start({ + cronPatterns, + }); + }, + StatsSyncerError, + { + checkCause: true, + } + ); + }); + + describe("when closing the stats syncer", () => { + it("should close the updaters", async () => { + const closeStatsSyncer = new StatsSyncerMock(); + const dailyStatsUpdater = closeStatsSyncer.getDailyStatsUpdater(); + const overallStatsUpdater = closeStatsSyncer.getOverallStatsUpdater(); + const connection = closeStatsSyncer.getConnection(); + + const dailyStatsUpdaterCloseSpy = vi.spyOn(dailyStatsUpdater, "close"); + const overallStatsUpdaterCloseSpy = vi.spyOn( + overallStatsUpdater, + "close" + ); + const connectionCloseSpy = vi.spyOn(connection, "disconnect"); + + await closeStatsSyncer.close(); + + expect(dailyStatsUpdaterCloseSpy).toHaveBeenCalledOnce(); + expect(overallStatsUpdaterCloseSpy).toHaveBeenCalledOnce(); + expect(connectionCloseSpy).toHaveBeenCalledOnce(); + }); + + testValidError( + "should throw a valid error when failing to close it", + async () => { + const closeStatsSyncer = new StatsSyncerMock(); + const dailyStatsUpdater = closeStatsSyncer.getDailyStatsUpdater(); + vi.spyOn(dailyStatsUpdater, "close").mockRejectedValueOnce( + new Error("Some daily stats updater closing error") + ); + + await closeStatsSyncer.close(); + }, + StatsSyncerError, + { checkCause: true } + ); + }); +}); diff --git a/packages/swarm-syncer/test/__snapshots__/PeriodicUpdater.test.ts.snap b/packages/swarm-syncer/test/__snapshots__/PeriodicUpdater.test.ts.snap new file mode 100644 index 000000000..ad2709e36 --- /dev/null +++ b/packages/swarm-syncer/test/__snapshots__/PeriodicUpdater.test.ts.snap @@ -0,0 +1,9 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`PeriodicUpdater > should throw a valid error when failing to close it 1`] = `"Updater \\"test-updater\\" failed: An error ocurred when performing closing operation"`; + +exports[`PeriodicUpdater > should throw a valid error when failing to close it 2`] = `[Error: Queue closing error]`; + +exports[`PeriodicUpdater > when running an updater > should throw a valid error when failing to run 1`] = `"Updater \\"test-updater\\" failed: An error ocurred when starting updater"`; + +exports[`PeriodicUpdater > when running an updater > should throw a valid error when failing to run 2`] = `[Error: Queue error]`; diff --git a/packages/swarm-syncer/test/__snapshots__/StatsSyncer.test.ts.snap b/packages/swarm-syncer/test/__snapshots__/StatsSyncer.test.ts.snap new file mode 100644 index 000000000..1ab5d442b --- /dev/null +++ b/packages/swarm-syncer/test/__snapshots__/StatsSyncer.test.ts.snap @@ -0,0 +1,9 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`StatsSyncer > when closing the stats syncer > should throw a valid error when failing to close it 1`] = `"Stats syncer failed: An error ocurred when closing syncer"`; + +exports[`StatsSyncer > when closing the stats syncer > should throw a valid error when failing to close it 2`] = `[Error: Some daily stats updater closing error]`; + +exports[`StatsSyncer > when running the stats syncer > should throw a valid error when failing to start it 1`] = `"Stats syncer failed: An error occurred when starting syncer"`; + +exports[`StatsSyncer > when running the stats syncer > should throw a valid error when failing to start it 2`] = `[Error: Something happened when trying to start daily stats updater]`; diff --git a/packages/swarm-syncer/tsconfig.json b/packages/swarm-syncer/tsconfig.json new file mode 100644 index 000000000..aff0b2dcd --- /dev/null +++ b/packages/swarm-syncer/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "@blobscan/tsconfig/base.production.json", + "include": ["src/**/*.ts", "test/**/*.ts", "vitest.config.ts"] +} diff --git a/packages/swarm-syncer/vitest.config.ts b/packages/swarm-syncer/vitest.config.ts new file mode 100644 index 000000000..8fdeaf1f5 --- /dev/null +++ b/packages/swarm-syncer/vitest.config.ts @@ -0,0 +1,5 @@ +import { defineProject } from "vitest/config"; + +import { sharedProjectConfig } from "../../vitest.shared"; + +export default defineProject(sharedProjectConfig); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e9cdf6713..7bd0ff606 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -103,7 +103,7 @@ importers: version: 2.2.1 '@tailwindcss/typography': specifier: ^0.5.7 - version: 0.5.13(tailwindcss@3.4.3(ts-node@10.9.2(@types/node@20.12.7)(typescript@5.4.5))) + version: 0.5.13(tailwindcss@3.4.3(ts-node@10.9.2(@types/node@18.19.31)(typescript@5.4.5))) autoprefixer: specifier: ^10.4.14 version: 10.4.19(postcss@8.4.38) @@ -142,7 +142,7 @@ importers: version: 1.2.1 tailwindcss: specifier: ^3.3.1 - version: 3.4.3(ts-node@10.9.2(@types/node@20.12.7)(typescript@5.4.5)) + version: 3.4.3(ts-node@10.9.2(@types/node@18.19.31)(typescript@5.4.5)) devDependencies: eslint: specifier: ^8.45.0 @@ -174,6 +174,9 @@ importers: '@blobscan/stats-syncer': specifier: workspace:^0.1.8 version: link:../../packages/stats-syncer + '@blobscan/swarm-syncer': + specifier: workspace:^0.0.1 + version: link:../../packages/swarm-syncer '@blobscan/zod': specifier: workspace:^0.1.0 version: link:../../packages/zod @@ -605,6 +608,27 @@ importers: specifier: ^5.3.2 version: 5.4.1 + packages/swarm-syncer: + dependencies: + '@blobscan/dayjs': + specifier: workspace:^0.0.2 + version: link:../dayjs + '@blobscan/db': + specifier: workspace:^0.7.0 + version: link:../db + '@blobscan/logger': + specifier: workspace:^0.1.0 + version: link:../logger + axios: + specifier: ^1.7.2 + version: 1.7.2 + bullmq: + specifier: ^4.13.2 + version: 4.17.0 + ioredis: + specifier: ^5.3.2 + version: 5.4.1 + packages/test: devDependencies: '@prisma/client': @@ -675,7 +699,7 @@ importers: version: 8.4.38 tailwindcss: specifier: ^3.3.1 - version: 3.4.3(ts-node@10.9.2(@types/node@20.12.7)(typescript@5.4.5)) + version: 3.4.3(ts-node@10.9.2(@types/node@18.19.31)(typescript@5.4.5)) tooling/typescript: {} @@ -2831,9 +2855,6 @@ packages: '@types/node@18.19.31': resolution: {integrity: sha512-ArgCD39YpyyrtFKIqMDvjz79jto5fcI/SVUs2HwB+f0dAzq68yqOdyaSivLiLugSziTpNXLQrVb7RZFmdZzbhA==} - '@types/node@20.12.7': - resolution: {integrity: sha512-wq0cICSkRLVaf3UGLMGItu/PtdY7oaXaI/RVU+xliKVOtRna3PRY57ZDfztpDL0n11vfymMUnXv8QwYCO7L1wg==} - '@types/normalize-package-data@2.4.4': resolution: {integrity: sha512-37i+OaWTh9qeK4LSHPsyRC7NahnGotNuZvjLSgcPzblpHB3rrCJxAOgI5gCdKm7coonsaX1Of0ILiTcnZjbfxA==} @@ -3185,6 +3206,9 @@ packages: axios@1.6.8: resolution: {integrity: sha512-v/ZHtJDU39mDpyBoFVkETcd/uNdxrWRrg3bKpOKzXFA6Bvqopts6ALSMU3y6ijYxbw2B+wPrIv46egTzJXCLGQ==} + axios@1.7.2: + resolution: {integrity: sha512-2A8QhOMrbomlDuiLeK9XibIBzuHeRcqqNOHp0Cyp5EoJ1IFDh+XZH3A6BkXtv0K4gFGCI0Y4BM7B1wOEi0Rmgw==} + axobject-query@3.2.1: resolution: {integrity: sha512-jsyHu61e6N4Vbz/v18DHwWYKK0bSWLqn47eeDSKPB7m8tqMHF9YJ+mhIk2lVteyZrY8tnSj/jHOv4YiTCuCJgg==} @@ -9180,13 +9204,13 @@ snapshots: mini-svg-data-uri: 1.4.4 tailwindcss: 3.4.3(ts-node@10.9.2(@types/node@18.19.31)(typescript@5.4.5)) - '@tailwindcss/typography@0.5.13(tailwindcss@3.4.3(ts-node@10.9.2(@types/node@20.12.7)(typescript@5.4.5)))': + '@tailwindcss/typography@0.5.13(tailwindcss@3.4.3(ts-node@10.9.2(@types/node@18.19.31)(typescript@5.4.5)))': dependencies: lodash.castarray: 4.4.0 lodash.isplainobject: 4.0.6 lodash.merge: 4.6.2 postcss-selector-parser: 6.0.10 - tailwindcss: 3.4.3(ts-node@10.9.2(@types/node@20.12.7)(typescript@5.4.5)) + tailwindcss: 3.4.3(ts-node@10.9.2(@types/node@18.19.31)(typescript@5.4.5)) '@tanstack/query-core@4.36.1': {} @@ -9339,11 +9363,6 @@ snapshots: dependencies: undici-types: 5.26.5 - '@types/node@20.12.7': - dependencies: - undici-types: 5.26.5 - optional: true - '@types/normalize-package-data@2.4.4': {} '@types/prettier@2.7.3': {} @@ -9789,6 +9808,14 @@ snapshots: transitivePeerDependencies: - debug + axios@1.7.2: + dependencies: + follow-redirects: 1.15.6 + form-data: 4.0.0 + proxy-from-env: 1.1.0 + transitivePeerDependencies: + - debug + axobject-query@3.2.1: dependencies: dequal: 2.0.3 @@ -10622,7 +10649,7 @@ snapshots: '@typescript-eslint/parser': 5.62.0(eslint@8.57.0)(typescript@5.4.5) eslint: 8.57.0 eslint-import-resolver-node: 0.3.9 - eslint-import-resolver-typescript: 3.6.1(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0) + eslint-import-resolver-typescript: 3.6.1(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(eslint@8.57.0))(eslint@8.57.0) eslint-plugin-import: 2.29.1(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0) eslint-plugin-jsx-a11y: 6.8.0(eslint@8.57.0) eslint-plugin-react: 7.32.2(eslint@8.57.0) @@ -10650,12 +10677,12 @@ snapshots: transitivePeerDependencies: - supports-color - eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0): + eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(eslint@8.57.0))(eslint@8.57.0): dependencies: debug: 4.3.4 enhanced-resolve: 5.15.0 eslint: 8.57.0 - eslint-module-utils: 2.8.0(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0))(eslint@8.57.0) + eslint-module-utils: 2.8.0(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(eslint@8.57.0))(eslint@8.57.0))(eslint@8.57.0) eslint-plugin-import: 2.29.1(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0) fast-glob: 3.3.2 get-tsconfig: 4.7.2 @@ -10667,14 +10694,14 @@ snapshots: - eslint-import-resolver-webpack - supports-color - eslint-module-utils@2.8.0(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0))(eslint@8.57.0): + eslint-module-utils@2.8.0(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(eslint@8.57.0))(eslint@8.57.0))(eslint@8.57.0): dependencies: debug: 3.2.7 optionalDependencies: '@typescript-eslint/parser': 5.62.0(eslint@8.57.0)(typescript@5.4.5) eslint: 8.57.0 eslint-import-resolver-node: 0.3.9 - eslint-import-resolver-typescript: 3.6.1(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0) + eslint-import-resolver-typescript: 3.6.1(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(eslint@8.57.0))(eslint@8.57.0) transitivePeerDependencies: - supports-color @@ -10688,7 +10715,7 @@ snapshots: doctrine: 2.1.0 eslint: 8.57.0 eslint-import-resolver-node: 0.3.9 - eslint-module-utils: 2.8.0(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0))(eslint@8.57.0) + eslint-module-utils: 2.8.0(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(eslint@8.57.0))(eslint@8.57.0))(eslint@8.57.0) hasown: 2.0.2 is-core-module: 2.13.1 is-glob: 4.0.3 @@ -10715,7 +10742,7 @@ snapshots: doctrine: 2.1.0 eslint: 8.57.0 eslint-import-resolver-node: 0.3.9 - eslint-module-utils: 2.8.0(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0))(eslint@8.57.0) + eslint-module-utils: 2.8.0(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(eslint@8.57.0))(eslint@8.57.0))(eslint@8.57.0) hasown: 2.0.2 is-core-module: 2.13.1 is-glob: 4.0.3 @@ -12308,14 +12335,6 @@ snapshots: postcss: 8.4.38 ts-node: 10.9.2(@types/node@18.19.31)(typescript@5.4.5) - postcss-load-config@4.0.2(postcss@8.4.38)(ts-node@10.9.2(@types/node@20.12.7)(typescript@5.4.5)): - dependencies: - lilconfig: 3.1.1 - yaml: 2.4.2 - optionalDependencies: - postcss: 8.4.38 - ts-node: 10.9.2(@types/node@20.12.7)(typescript@5.4.5) - postcss-nested@6.0.1(postcss@8.4.38): dependencies: postcss: 8.4.38 @@ -13090,33 +13109,6 @@ snapshots: transitivePeerDependencies: - ts-node - tailwindcss@3.4.3(ts-node@10.9.2(@types/node@20.12.7)(typescript@5.4.5)): - dependencies: - '@alloc/quick-lru': 5.2.0 - arg: 5.0.2 - chokidar: 3.5.3 - didyoumean: 1.2.2 - dlv: 1.1.3 - fast-glob: 3.3.2 - glob-parent: 6.0.2 - is-glob: 4.0.3 - jiti: 1.21.0 - lilconfig: 2.1.0 - micromatch: 4.0.5 - normalize-path: 3.0.0 - object-hash: 3.0.0 - picocolors: 1.0.0 - postcss: 8.4.38 - postcss-import: 15.1.0(postcss@8.4.38) - postcss-js: 4.0.1(postcss@8.4.38) - postcss-load-config: 4.0.2(postcss@8.4.38)(ts-node@10.9.2(@types/node@20.12.7)(typescript@5.4.5)) - postcss-nested: 6.0.1(postcss@8.4.38) - postcss-selector-parser: 6.0.13 - resolve: 1.22.8 - sucrase: 3.34.0 - transitivePeerDependencies: - - ts-node - tapable@2.2.1: {} tar-fs@2.1.1: @@ -13272,25 +13264,6 @@ snapshots: v8-compile-cache-lib: 3.0.1 yn: 3.1.1 - ts-node@10.9.2(@types/node@20.12.7)(typescript@5.4.5): - dependencies: - '@cspotcode/source-map-support': 0.8.1 - '@tsconfig/node10': 1.0.9 - '@tsconfig/node12': 1.0.11 - '@tsconfig/node14': 1.0.3 - '@tsconfig/node16': 1.0.4 - '@types/node': 20.12.7 - acorn: 8.11.3 - acorn-walk: 8.3.2 - arg: 4.1.3 - create-require: 1.1.1 - diff: 4.0.2 - make-error: 1.3.6 - typescript: 5.4.5 - v8-compile-cache-lib: 3.0.1 - yn: 3.1.1 - optional: true - tsconfig-paths@3.15.0: dependencies: '@types/json5': 0.0.29 diff --git a/turbo.json b/turbo.json index fefa3b0f9..af18096ec 100644 --- a/turbo.json +++ b/turbo.json @@ -1,13 +1,19 @@ { "$schema": "https://turborepo.org/schema.json", - "globalDependencies": ["**/.env"], + "globalDependencies": [ + "**/.env" + ], "pipeline": { "db:generate": { - "inputs": ["prisma/schema.prisma"], + "inputs": [ + "prisma/schema.prisma" + ], "cache": false }, "push": { - "inputs": ["prisma/schema.prisma"], + "inputs": [ + "prisma/schema.prisma" + ], "cache": false }, "dev": { @@ -15,20 +21,31 @@ "cache": false }, "build": { - "dependsOn": ["^build", "^db:generate"], - "outputs": [".next/**"] + "dependsOn": [ + "^build", + "^db:generate" + ], + "outputs": [ + ".next/**" + ] }, "lint": {}, "lint:fix": {}, "svg:format": {}, "test": { - "inputs": ["test/**/*.test.ts"] + "inputs": [ + "test/**/*.test.ts" + ] }, "test:ui": { - "inputs": ["test/**/*.test.ts"] + "inputs": [ + "test/**/*.test.ts" + ] }, "test:dev": { - "inputs": ["test/**/*.test.ts"], + "inputs": [ + "test/**/*.test.ts" + ], "cache": false }, "test:setup": { @@ -38,7 +55,9 @@ "cache": false }, "type-check": { - "dependsOn": ["@blobscan/db#db:generate"], + "dependsOn": [ + "@blobscan/db#db:generate" + ], "cache": false } }, From 0d8f9b8b62087f5e35f7ee3357d5ab9a1dcefb51 Mon Sep 17 00:00:00 2001 From: Pablo Castellano Date: Wed, 5 Jun 2024 09:31:39 +0200 Subject: [PATCH 2/8] Removed unneeded fields + fix lint --- packages/db/prisma/schema.prisma | 8 +------- packages/swarm-syncer/src/SwarmStampSyncer.ts | 2 +- packages/swarm-syncer/src/updaters/SwarmUpdater.ts | 8 +------- 3 files changed, 3 insertions(+), 15 deletions(-) diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma index 49f354b55..4f2d4a97a 100644 --- a/packages/db/prisma/schema.prisma +++ b/packages/db/prisma/schema.prisma @@ -59,12 +59,6 @@ model BlobStoragesState { swarmDataId String? @map("swarm_data_id") // TODO: rename to batchId swarmDataTTL Int? @map("swarm_data_ttl") // TODO: rename to batchTtl updatedAt DateTime @default(now()) @map("updated_at") - label String @map("label") - depth Int @map("depth") - utilization Int @map("utilization") - amount String @map("amount") - immutableFlag Boolean @map("immutable_flag") - exists Boolean @map("exists") @@map("blob_storages_state") } @@ -267,7 +261,7 @@ model BlobDailyStats { } // NextAuth.js Models -// NOTE: When using postgresql, mysql or sqlserver, +// NOTE: When using postgresql, mysql or sqlserver, // uncomment the @db.Text annotations below // @see https://next-auth.js.org/schemas/models // model Account { diff --git a/packages/swarm-syncer/src/SwarmStampSyncer.ts b/packages/swarm-syncer/src/SwarmStampSyncer.ts index 836bd08f2..b02a204ea 100644 --- a/packages/swarm-syncer/src/SwarmStampSyncer.ts +++ b/packages/swarm-syncer/src/SwarmStampSyncer.ts @@ -1,4 +1,4 @@ -/* eslint-disable @typescript-eslint/no-misused-promises */ + import type { Redis } from "ioredis"; import { ErrorException, SwarmStampSyncerError } from "./errors"; diff --git a/packages/swarm-syncer/src/updaters/SwarmUpdater.ts b/packages/swarm-syncer/src/updaters/SwarmUpdater.ts index a2a28e842..b0db7de74 100644 --- a/packages/swarm-syncer/src/updaters/SwarmUpdater.ts +++ b/packages/swarm-syncer/src/updaters/SwarmUpdater.ts @@ -27,18 +27,12 @@ export class SwarmStampUpdater extends PeriodicUpdater { await prisma.blobStoragesState.update({ data: { swarmDataTTL: data.batchTTL, - label: data.label, - depth: data.depth, - utilization: data.utilization, - amount: data.amount, - immutableFlag: data.immutableFlag, - exists: data.exists, }, where: { swarmDataId: batchId, }, }); - + this.logger.info(`Updated swarm stamp ${batchId}`); }, }); From 33939533cf153b8caefff1b70c6dca5a6fe5c53b Mon Sep 17 00:00:00 2001 From: PJColombo Date: Wed, 12 Jun 2024 17:49:50 +0200 Subject: [PATCH 3/8] refactor: abstract the stats syncer to support general-purpose synchronization workers --- .changeset/long-turkeys-impress.md | 11 + apps/rest-api-server/package.json | 3 +- apps/rest-api-server/src/index.ts | 28 +-- apps/rest-api-server/src/syncers.ts | 32 +++ packages/stats-syncer/src/PeriodicUpdater.ts | 119 ---------- packages/stats-syncer/src/StatsSyncer.ts | 84 ------- packages/stats-syncer/src/errors.ts | 21 -- packages/stats-syncer/src/index.ts | 2 - packages/stats-syncer/src/logger.ts | 3 - .../test/OverallStatsUpdater.test.ts | 222 ------------------ .../stats-syncer/test/PeriodicUpdater.test.ts | 125 ---------- .../stats-syncer/test/StatsSyncer.test.ts | 120 ---------- packages/swarm-syncer/package.json | 30 --- packages/swarm-syncer/src/SwarmStampSyncer.ts | 67 ------ packages/swarm-syncer/src/env.ts | 17 -- packages/swarm-syncer/src/errors.ts | 21 -- packages/swarm-syncer/src/index.ts | 2 - packages/swarm-syncer/src/logger.ts | 3 - .../swarm-syncer/src/updaters/SwarmUpdater.ts | 40 ---- packages/swarm-syncer/src/utils.ts | 13 - .../test/SwarmStampSyncer.test.ts | 120 ---------- .../__snapshots__/StatsSyncer.test.ts.snap | 9 - packages/swarm-syncer/tsconfig.json | 4 - packages/swarm-syncer/vitest.config.ts | 5 - .../{stats-syncer => syncers}/CHANGELOG.md | 2 +- .../{stats-syncer => syncers}/package.json | 2 +- .../src/BaseSyncer.ts} | 77 +++--- packages/syncers/src/errors.ts | 15 ++ packages/syncers/src/index.ts | 2 + .../src/syncers/DailyStatsSyncer.ts} | 17 +- .../src/syncers/OverallStatsSyncer.ts} | 31 ++- packages/syncers/src/syncers/index.ts | 4 + .../{stats-syncer => syncers}/src/utils.ts | 0 .../test/BaseSyncer.test.ts} | 46 ++-- .../test/DailyStatsSyncer.test.fixtures.ts} | 0 .../test/DailyStatsSyncer.test.ts} | 28 +-- .../test/DailyStatsSyncer.test.utils.ts} | 0 .../test/OverallStatsSyncer.test.ts} | 20 +- packages/syncers/test/SwarmSyncer.test.ts | 0 .../__snapshots__/BaseSyncer.test.ts.snap} | 4 +- .../PeriodicUpdater.test.ts.snap | 4 +- .../__snapshots__/StatsSyncer.test.ts.snap | 0 .../__snapshots__/SyncerManager.test.ts.snap | 9 + .../{stats-syncer => syncers}/tsconfig.json | 0 .../vitest.config.ts | 0 packages/test/src/fixtures/postgres/data.json | 3 +- pnpm-lock.yaml | 57 +---- 47 files changed, 211 insertions(+), 1211 deletions(-) create mode 100644 .changeset/long-turkeys-impress.md create mode 100644 apps/rest-api-server/src/syncers.ts delete mode 100644 packages/stats-syncer/src/PeriodicUpdater.ts delete mode 100644 packages/stats-syncer/src/StatsSyncer.ts delete mode 100644 packages/stats-syncer/src/errors.ts delete mode 100644 packages/stats-syncer/src/index.ts delete mode 100644 packages/stats-syncer/src/logger.ts delete mode 100644 packages/stats-syncer/test/OverallStatsUpdater.test.ts delete mode 100644 packages/stats-syncer/test/PeriodicUpdater.test.ts delete mode 100644 packages/stats-syncer/test/StatsSyncer.test.ts delete mode 100644 packages/swarm-syncer/package.json delete mode 100644 packages/swarm-syncer/src/SwarmStampSyncer.ts delete mode 100644 packages/swarm-syncer/src/env.ts delete mode 100644 packages/swarm-syncer/src/errors.ts delete mode 100644 packages/swarm-syncer/src/index.ts delete mode 100644 packages/swarm-syncer/src/logger.ts delete mode 100644 packages/swarm-syncer/src/updaters/SwarmUpdater.ts delete mode 100644 packages/swarm-syncer/src/utils.ts delete mode 100644 packages/swarm-syncer/test/SwarmStampSyncer.test.ts delete mode 100644 packages/swarm-syncer/test/__snapshots__/StatsSyncer.test.ts.snap delete mode 100644 packages/swarm-syncer/tsconfig.json delete mode 100644 packages/swarm-syncer/vitest.config.ts rename packages/{stats-syncer => syncers}/CHANGELOG.md (99%) rename packages/{stats-syncer => syncers}/package.json (95%) rename packages/{swarm-syncer/src/PeriodicUpdater.ts => syncers/src/BaseSyncer.ts} (55%) create mode 100644 packages/syncers/src/errors.ts create mode 100644 packages/syncers/src/index.ts rename packages/{stats-syncer/src/updaters/DailyStatsUpdater.ts => syncers/src/syncers/DailyStatsSyncer.ts} (88%) rename packages/{stats-syncer/src/updaters/OverallStatsUpdater.ts => syncers/src/syncers/OverallStatsSyncer.ts} (87%) create mode 100644 packages/syncers/src/syncers/index.ts rename packages/{stats-syncer => syncers}/src/utils.ts (100%) rename packages/{swarm-syncer/test/PeriodicUpdater.test.ts => syncers/test/BaseSyncer.test.ts} (74%) rename packages/{stats-syncer/test/DailyStatsUpdater.test.fixtures.ts => syncers/test/DailyStatsSyncer.test.fixtures.ts} (100%) rename packages/{stats-syncer/test/DailyStatsUpdater.test.ts => syncers/test/DailyStatsSyncer.test.ts} (78%) rename packages/{stats-syncer/test/DailyStatsUpdater.test.utils.ts => syncers/test/DailyStatsSyncer.test.utils.ts} (100%) rename packages/{swarm-syncer/test/OverallStatsUpdater.test.ts => syncers/test/OverallStatsSyncer.test.ts} (91%) create mode 100644 packages/syncers/test/SwarmSyncer.test.ts rename packages/{swarm-syncer/test/__snapshots__/PeriodicUpdater.test.ts.snap => syncers/test/__snapshots__/BaseSyncer.test.ts.snap} (64%) rename packages/{stats-syncer => syncers}/test/__snapshots__/PeriodicUpdater.test.ts.snap (73%) rename packages/{stats-syncer => syncers}/test/__snapshots__/StatsSyncer.test.ts.snap (100%) create mode 100644 packages/syncers/test/__snapshots__/SyncerManager.test.ts.snap rename packages/{stats-syncer => syncers}/tsconfig.json (100%) rename packages/{stats-syncer => syncers}/vitest.config.ts (100%) diff --git a/.changeset/long-turkeys-impress.md b/.changeset/long-turkeys-impress.md new file mode 100644 index 000000000..c8e68856f --- /dev/null +++ b/.changeset/long-turkeys-impress.md @@ -0,0 +1,11 @@ +--- +"@blobscan/rest-api-server": minor +"@blobscan/syncers": minor +--- + +Refactored the stats syncer package to support general-purpose synchronization workers/queues. + +Key changes include: + + • Renamed the package to syncers. + • Exported each syncer directly, removing the StatsSyncer managing entity. diff --git a/apps/rest-api-server/package.json b/apps/rest-api-server/package.json index 68af875f1..ba869f474 100644 --- a/apps/rest-api-server/package.json +++ b/apps/rest-api-server/package.json @@ -18,8 +18,7 @@ "@blobscan/api": "workspace:^0.9.0", "@blobscan/logger": "workspace:^0.1.0", "@blobscan/open-telemetry": "workspace:^0.0.7", - "@blobscan/stats-syncer": "workspace:^0.1.8", - "@blobscan/swarm-syncer": "workspace:^0.0.1", + "@blobscan/syncers": "workspace:^0.1.9", "@blobscan/zod": "workspace:^0.1.0", "@opentelemetry/instrumentation-express": "^0.33.0", "@sentry/node": "^7.109.0", diff --git a/apps/rest-api-server/src/index.ts b/apps/rest-api-server/src/index.ts index b1f8b35ad..e2120081d 100644 --- a/apps/rest-api-server/src/index.ts +++ b/apps/rest-api-server/src/index.ts @@ -14,38 +14,16 @@ import { gracefulShutdown as apiGracefulShutdown, } from "@blobscan/api"; import { collectDefaultMetrics } from "@blobscan/open-telemetry"; -import { StatsSyncer } from "@blobscan/stats-syncer"; -import { SwarmStampSyncer } from "@blobscan/swarm-syncer"; import { env } from "./env"; import { logger } from "./logger"; import { morganMiddleware } from "./morgan"; import { openApiDocument } from "./openapi"; -import { getNetworkDencunForkSlot } from "./utils"; +import { setUpSyncers } from "./syncers"; collectDefaultMetrics(); -const statsSyncer = new StatsSyncer({ - redisUri: env.REDIS_URI, - lowestSlot: - env.DENCUN_FORK_SLOT ?? getNetworkDencunForkSlot(env.NETWORK_NAME), -}); - -statsSyncer.start({ - cronPatterns: { - daily: env.STATS_SYNCER_DAILY_CRON_PATTERN, - overall: env.STATS_SYNCER_OVERALL_CRON_PATTERN, - }, -}); - - -if (env.SWARM_STORAGE_ENABLED && env.SWARM_BATCH_ID) { - const swarmSyncer = new SwarmStampSyncer({ - redisUri: env.REDIS_URI, - env.BEE_ENDPOINT, //FIXME - ); - swarmSyncer.start(env.SWARM_SYNCER_CRON); -} +const closeSyncers = setUpSyncers(); const app = express(); @@ -83,7 +61,7 @@ async function gracefulShutdown(signal: string) { logger.debug(`Received ${signal}. Shutting down...`); await apiGracefulShutdown() - .finally(() => statsSyncer.close()) + .finally(() => closeSyncers()) .finally(() => { server.close(() => { logger.debug("Server shut down successfully"); diff --git a/apps/rest-api-server/src/syncers.ts b/apps/rest-api-server/src/syncers.ts new file mode 100644 index 000000000..a6ec847c8 --- /dev/null +++ b/apps/rest-api-server/src/syncers.ts @@ -0,0 +1,32 @@ +/* eslint-disable @typescript-eslint/no-misused-promises */ + +import { + DailyStatsSyncer, + OverallStatsSyncer, + createRedisConnection, +} from "@blobscan/syncers"; + +import { env } from "./env"; +import { getNetworkDencunForkSlot } from "./utils"; + +export function setUpSyncers() { + const connection = createRedisConnection(env.REDIS_URI); + + const dailyStatsSyncer = new DailyStatsSyncer({ + cronPattern: env.STATS_SYNCER_DAILY_CRON_PATTERN, + redisUriOrConnection: connection, + }); + + const overallStatsSyncer = new OverallStatsSyncer({ + cronPattern: env.STATS_SYNCER_OVERALL_CRON_PATTERN, + redisUriOrConnection: connection, + lowestSlot: + env.DENCUN_FORK_SLOT ?? getNetworkDencunForkSlot(env.NETWORK_NAME), + }); + + Promise.all([dailyStatsSyncer.start(), overallStatsSyncer.start()]); + + return () => { + return dailyStatsSyncer.close().finally(() => overallStatsSyncer.close()); + }; +} diff --git a/packages/stats-syncer/src/PeriodicUpdater.ts b/packages/stats-syncer/src/PeriodicUpdater.ts deleted file mode 100644 index 6ef76c0d9..000000000 --- a/packages/stats-syncer/src/PeriodicUpdater.ts +++ /dev/null @@ -1,119 +0,0 @@ -/* eslint-disable @typescript-eslint/no-misused-promises */ -import { Queue, Worker } from "bullmq"; -import type { Redis } from "ioredis"; - -import { createModuleLogger } from "@blobscan/logger"; -import type { Logger } from "@blobscan/logger"; - -import { ErrorException, PeriodicUpdaterError } from "./errors"; -import { createRedisConnection } from "./utils"; - -export type PeriodicUpdaterConfig = { - name: string; - redisUriOrConnection: string | Redis; - updaterFn: () => Promise; -}; - -export class PeriodicUpdater { - name: string; - protected worker: Worker; - protected queue: Queue; - protected updaterFn: () => Promise; - protected logger: Logger; - - constructor({ - name, - redisUriOrConnection, - updaterFn, - }: PeriodicUpdaterConfig) { - const isRedisUri = typeof redisUriOrConnection === "string"; - this.name = name; - this.logger = createModuleLogger("stats-syncer", this.name); - - let connection: Redis; - - if (isRedisUri) { - connection = createRedisConnection(redisUriOrConnection); - - connection.on("error", (err) => { - this.logger.error( - new ErrorException("A Redis connection error ocurred", err) - ); - }); - } else { - connection = redisUriOrConnection; - } - - this.queue = new Queue(this.name, { - connection, - }); - - this.worker = new Worker(this.queue.name, updaterFn, { - connection, - }); - - this.updaterFn = updaterFn; - - this.queue.on("error", (err) => { - this.logger.error(new ErrorException("A queue error occurred", err)); - }); - - this.worker.on("failed", (_, err) => { - this.logger.error(new ErrorException("A worker error ocurred", err)); - }); - } - - async start(cronPattern: string) { - try { - const jobName = `${this.name}-job`; - const repeatableJob = await this.queue.add(jobName, null, { - repeat: { - pattern: cronPattern, - }, - }); - - return repeatableJob; - } catch (err) { - throw new PeriodicUpdaterError( - this.name, - "An error ocurred when starting updater", - err - ); - } - } - - close() { - const teardownPromise: Promise = Promise.resolve(); - - return teardownPromise - .finally(async () => { - await this.#performClosingOperation(() => - this.worker.removeAllListeners().close(true) - ); - }) - .finally(async () => { - await this.#performClosingOperation(() => - this.queue.obliterate({ force: true }) - ); - }) - .finally(async () => { - await this.#performClosingOperation(() => - this.queue.removeAllListeners().close() - ); - }); - } - - async #performClosingOperation(operation: () => Promise) { - try { - await operation(); - } catch (err) { - const err_ = new PeriodicUpdaterError( - this.name, - "An error ocurred when performing closing operation", - err - ); - - throw err_; - } - } -} diff --git a/packages/stats-syncer/src/StatsSyncer.ts b/packages/stats-syncer/src/StatsSyncer.ts deleted file mode 100644 index 54f19dcbf..000000000 --- a/packages/stats-syncer/src/StatsSyncer.ts +++ /dev/null @@ -1,84 +0,0 @@ -/* eslint-disable @typescript-eslint/no-misused-promises */ -import type { Redis } from "ioredis"; - -import { ErrorException, StatsSyncerError } from "./errors"; -import { logger } from "./logger"; -import { DailyStatsUpdater } from "./updaters/DailyStatsUpdater"; -import { OverallStatsUpdater } from "./updaters/OverallStatsUpdater"; -import { createRedisConnection } from "./utils"; - -export type StatsSyncerOptions = { - redisUri: string; - lowestSlot?: number; -}; - -export class StatsSyncer { - protected connection: Redis; - protected dailyStatsUpdater: DailyStatsUpdater; - protected overallStatsUpdater: OverallStatsUpdater; - - constructor({ redisUri, lowestSlot }: StatsSyncerOptions) { - const connection = createRedisConnection(redisUri); - - connection.on("error", (err) => { - logger.error(new ErrorException("The Redis connection failed", err)); - }); - - this.connection = connection; - this.dailyStatsUpdater = new DailyStatsUpdater(connection); - this.overallStatsUpdater = new OverallStatsUpdater(connection, { - lowestSlot, - }); - } - - async start(config: { - cronPatterns: { - daily: string; - overall: string; - }; - }) { - try { - const cronPatterns = config.cronPatterns; - - await Promise.all([ - this.dailyStatsUpdater.start(cronPatterns.daily), - this.overallStatsUpdater.start(cronPatterns.overall), - ]); - - logger.info("Stats syncer started successfully."); - } catch (err) { - const err_ = new StatsSyncerError( - "An error occurred when starting syncer", - err - ); - - logger.error(err_); - - throw err_; - } - } - - async close() { - try { - await this.dailyStatsUpdater - .close() - .finally(() => this.overallStatsUpdater.close()) - .finally(() => { - this.connection.removeAllListeners(); - - if (this.connection.status === "ready") this.connection.disconnect(); - }); - - logger.info("Stats syncer closed successfully."); - } catch (err) { - const err_ = new StatsSyncerError( - "An error ocurred when closing syncer", - err - ); - - logger.error(err_); - - throw err_; - } - } -} diff --git a/packages/stats-syncer/src/errors.ts b/packages/stats-syncer/src/errors.ts deleted file mode 100644 index ab95c6e9e..000000000 --- a/packages/stats-syncer/src/errors.ts +++ /dev/null @@ -1,21 +0,0 @@ -export class ErrorException extends Error { - constructor(message: string, cause?: unknown) { - super(message, { - cause, - }); - - this.name = this.constructor.name; - } -} - -export class StatsSyncerError extends ErrorException { - constructor(message: string, cause: unknown) { - super(`Stats syncer failed: ${message}`, cause); - } -} - -export class PeriodicUpdaterError extends ErrorException { - constructor(updaterName: string, message: string, cause: unknown) { - super(`Updater "${updaterName}" failed: ${message}`, cause); - } -} diff --git a/packages/stats-syncer/src/index.ts b/packages/stats-syncer/src/index.ts deleted file mode 100644 index fcfe8511b..000000000 --- a/packages/stats-syncer/src/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export { StatsSyncer } from "./StatsSyncer"; -export type { PeriodicUpdater, PeriodicUpdaterConfig } from "./PeriodicUpdater"; diff --git a/packages/stats-syncer/src/logger.ts b/packages/stats-syncer/src/logger.ts deleted file mode 100644 index 96c6dcf6a..000000000 --- a/packages/stats-syncer/src/logger.ts +++ /dev/null @@ -1,3 +0,0 @@ -import { createModuleLogger } from "@blobscan/logger"; - -export const logger = createModuleLogger("stats-syncer"); diff --git a/packages/stats-syncer/test/OverallStatsUpdater.test.ts b/packages/stats-syncer/test/OverallStatsUpdater.test.ts deleted file mode 100644 index bed3404d7..000000000 --- a/packages/stats-syncer/test/OverallStatsUpdater.test.ts +++ /dev/null @@ -1,222 +0,0 @@ -import { beforeEach, describe, expect, it, vi } from "vitest"; - -import { prisma } from "@blobscan/db"; -import { fixtures, omitDBTimestampFields } from "@blobscan/test"; - -import { OverallStatsUpdater } from "../src/updaters/OverallStatsUpdater"; - -class OverallStatsUpdaterMock extends OverallStatsUpdater { - constructor( - redisUri = process.env.REDIS_URI ?? "", - config: ConstructorParameters[1] = {} - ) { - const lowestSlot = - config.lowestSlot ?? fixtures.blockchainSyncState[0]?.lastLowerSyncedSlot; - super(redisUri, { - ...config, - lowestSlot, - }); - } - - getWorker() { - return this.worker; - } - - getWorkerProcessor() { - return this.updaterFn; - } - - getQueue() { - return this.queue; - } -} - -function getAllOverallStats() { - const uniqueArgs = { - where: { - id: 1, - }, - }; - - return Promise.all([ - prisma.blobOverallStats.findUnique(uniqueArgs), - prisma.blockOverallStats.findUnique(uniqueArgs), - prisma.transactionOverallStats.findUnique(uniqueArgs), - ]).then((allOverallStats) => - allOverallStats.map((stats) => - stats ? omitDBTimestampFields(stats) : undefined - ) - ); -} - -describe("OverallStatsUpdater", () => { - let overallStatsUpdater: OverallStatsUpdaterMock; - - beforeEach(() => { - overallStatsUpdater = new OverallStatsUpdaterMock(); - - return async () => { - await overallStatsUpdater.close(); - }; - }); - - it("should aggregate all overall stats correctly", async () => { - const workerProcessor = overallStatsUpdater.getWorkerProcessor(); - - const incrementTransactionSpy = vi.spyOn(prisma, "$transaction"); - - await workerProcessor(); - - const [blobOverallStats, blockOverallStats, transactionOverallStats] = - await getAllOverallStats(); - - expect( - incrementTransactionSpy, - "Expect to aggregate overall stats within a transaction" - ).toHaveBeenCalledOnce(); - expect(blobOverallStats, "Incorrect blob overall stats aggregation") - .toMatchInlineSnapshot(` - { - "avgBlobSize": 1175, - "id": 1, - "totalBlobSize": 9400n, - "totalBlobs": 8, - "totalUniqueBlobs": 1, - } - `); - expect(blockOverallStats, "Incorrect block overall stats aggregation") - .toMatchInlineSnapshot(` - { - "avgBlobAsCalldataFee": 5406666.666666667, - "avgBlobFee": 114000000, - "avgBlobGasPrice": 21.33333333333333, - "id": 1, - "totalBlobAsCalldataFee": "16220000", - "totalBlobAsCalldataGasUsed": "760000", - "totalBlobFee": "342000000", - "totalBlobGasUsed": "16000000", - "totalBlocks": 3, - } - `); - expect( - transactionOverallStats, - "Incorrect transaction overall stats aggregation" - ).toMatchInlineSnapshot(` - { - "avgMaxBlobGasFee": 100, - "id": 1, - "totalTransactions": 4, - "totalUniqueReceivers": 0, - "totalUniqueSenders": 0, - } - `); - }); - - it("should aggregate overall stats in batches correctly when there are too many blocks", async () => { - const batchSize = 2; - const workerProcessor = new OverallStatsUpdaterMock(undefined, { - batchSize, - }).getWorkerProcessor(); - const incrementTransactionSpy = vi.spyOn(prisma, "$transaction"); - const blockchainSyncState = fixtures.blockchainSyncState[0]; - const lastAggregatedBlock = blockchainSyncState - ? blockchainSyncState.lastAggregatedBlock + 1 - : 0; - const lastFinalizedBlock = - fixtures.blockchainSyncState[0]?.lastFinalizedBlock ?? 0; - const batches = Math.ceil( - (lastFinalizedBlock - lastAggregatedBlock + 1) / batchSize - ); - - await workerProcessor(); - - expect( - incrementTransactionSpy, - "Incorrect number of stats aggregation calls" - ).toHaveBeenCalledTimes(batches); - }); - - it("should update last aggregated block to last finalized block after aggregation", async () => { - const workerProcessor = overallStatsUpdater.getWorkerProcessor(); - const expectedLastAggregatedBlock = - fixtures.blockchainSyncState[0]?.lastFinalizedBlock; - - await workerProcessor(); - - const lastAggregatedBlock = await prisma.blockchainSyncState - .findUnique({ - select: { - lastAggregatedBlock: true, - }, - where: { - id: 1, - }, - }) - .then((state) => state?.lastAggregatedBlock); - - expect(lastAggregatedBlock).toBe(expectedLastAggregatedBlock); - }); - - it("should skip aggregation when no finalized block has been set", async () => { - const workerProcessor = overallStatsUpdater.getWorkerProcessor(); - - await prisma.blockchainSyncState.update({ - data: { - lastFinalizedBlock: null, - }, - where: { - id: 1, - }, - }); - - await workerProcessor(); - - const allOverallStats = await getAllOverallStats().then((allOverallStats) => - allOverallStats.filter((stats) => !!stats) - ); - - expect(allOverallStats).toEqual([]); - }); - - it("should skip aggregation when no blocks have been indexed yet", async () => { - const workerProcessor = overallStatsUpdater.getWorkerProcessor(); - - vi.spyOn(prisma.block, "findLatest").mockResolvedValueOnce(null); - - await workerProcessor(); - - const allOverallStats = await getAllOverallStats().then((allOverallStats) => - allOverallStats.filter((stats) => !!stats) - ); - - expect(allOverallStats).toEqual([]); - }); - - it("should skip aggregation when the lowest slot hasn't been reached yet", async () => { - const workerProcessor = new OverallStatsUpdaterMock(undefined, { - lowestSlot: 1, - }).getWorkerProcessor(); - - await workerProcessor(); - - const allOverallStats = await getAllOverallStats().then((allOverallStats) => - allOverallStats.filter((stats) => !!stats) - ); - - expect(allOverallStats).toEqual([]); - }); - - it("should skip aggregation when there is no new finalized blocks", async () => { - const workerProcessor = overallStatsUpdater.getWorkerProcessor(); - - await workerProcessor(); - - const allOverallStats = await getAllOverallStats(); - - await workerProcessor(); - - const allOverallStatsAfter = await getAllOverallStats(); - - expect(allOverallStats).toEqual(allOverallStatsAfter); - }); -}); diff --git a/packages/stats-syncer/test/PeriodicUpdater.test.ts b/packages/stats-syncer/test/PeriodicUpdater.test.ts deleted file mode 100644 index 9354c905e..000000000 --- a/packages/stats-syncer/test/PeriodicUpdater.test.ts +++ /dev/null @@ -1,125 +0,0 @@ -import { beforeEach, describe, expect, it, vi } from "vitest"; - -import { testValidError } from "@blobscan/test"; - -import { PeriodicUpdater } from "../src/PeriodicUpdater"; -import type { PeriodicUpdaterConfig } from "../src/PeriodicUpdater"; -import { PeriodicUpdaterError } from "../src/errors"; - -class PeriodicUpdaterMock extends PeriodicUpdater { - constructor({ - name, - redisUriOrConnection, - updaterFn, - }: Partial = {}) { - super({ - name: name ?? "test-updater", - redisUriOrConnection: redisUriOrConnection ?? "redis://localhost:6379/1", - updaterFn: updaterFn ?? (() => Promise.resolve()), - }); - } - - getWorker() { - return this.worker; - } - - getQueue() { - return this.queue; - } -} - -describe("PeriodicUpdater", () => { - let periodicUpdater: PeriodicUpdaterMock; - - beforeEach(() => { - periodicUpdater = new PeriodicUpdaterMock(); - - return async () => { - await periodicUpdater.close(); - }; - }); - - it("should create an updater correctly", async () => { - const queue = periodicUpdater.getQueue(); - const worker = periodicUpdater.getWorker(); - const isPaused = await queue.isPaused(); - - expect(worker.isRunning(), "Expected worker to be running").toBeTruthy(); - expect(isPaused, "Expected queue to be running").toBeFalsy(); - }); - - describe("when running an updater", () => { - it("should set up a repeatable job correctly", async () => { - const queue = periodicUpdater.getQueue(); - const cronPattern = "* * * * *"; - - await periodicUpdater.start(cronPattern); - - const jobs = await queue.getRepeatableJobs(); - - expect(jobs.length, "Expected one repeatable job").toBe(1); - expect(jobs[0]?.pattern, "Repetable job cron pattern mismatch").toEqual( - cronPattern - ); - }); - - testValidError( - "should throw a valid error when failing to run", - async () => { - const queue = periodicUpdater.getQueue(); - - vi.spyOn(queue, "add").mockRejectedValueOnce(new Error("Queue error")); - - await periodicUpdater.start("* * * * *"); - }, - PeriodicUpdaterError, - { checkCause: true } - ); - }); - - describe("when closing an updater", () => { - it("should close correctly", async () => { - const queue = periodicUpdater.getQueue(); - const worker = periodicUpdater.getWorker(); - - const queueCloseSpy = vi.spyOn(queue, "close").mockResolvedValueOnce(); - const queueRemoveAllListenersSpy = vi - .spyOn(queue, "removeAllListeners") - .mockReturnValueOnce(queue); - - const workerCloseSpy = vi.spyOn(worker, "close").mockResolvedValueOnce(); - const workerRemoveAllListenersSpy = vi - .spyOn(worker, "removeAllListeners") - .mockReturnValueOnce(worker); - - await periodicUpdater.close(); - - expect(queueCloseSpy).toHaveBeenCalledOnce(); - expect(workerCloseSpy).toHaveBeenCalledOnce(); - - expect(queueRemoveAllListenersSpy).toHaveBeenCalledOnce(); - expect(workerRemoveAllListenersSpy).toHaveBeenCalledOnce(); - }); - }); - - testValidError( - "should throw a valid error when failing to close it", - async () => { - const queue = periodicUpdater.getQueue(); - const worker = periodicUpdater.getWorker(); - - vi.spyOn(queue, "close").mockRejectedValueOnce( - new Error("Queue closing error") - ); - vi.spyOn(worker, "close").mockRejectedValueOnce( - new Error("Worker closing error") - ); - - await periodicUpdater.close(); - }, - PeriodicUpdaterError, - { - checkCause: true, - } - ); -}); diff --git a/packages/stats-syncer/test/StatsSyncer.test.ts b/packages/stats-syncer/test/StatsSyncer.test.ts deleted file mode 100644 index 586199941..000000000 --- a/packages/stats-syncer/test/StatsSyncer.test.ts +++ /dev/null @@ -1,120 +0,0 @@ -import { beforeEach, describe, expect, it, vi } from "vitest"; - -import { testValidError } from "@blobscan/test"; - -import { StatsSyncer } from "../src/StatsSyncer"; -import { StatsSyncerError } from "../src/errors"; - -class StatsSyncerMock extends StatsSyncer { - constructor(redisUri = "redis://localhost:6379/1") { - super({ redisUri }); - } - - getConnection() { - return this.connection; - } - - getDailyStatsUpdater() { - return this.dailyStatsUpdater; - } - - getOverallStatsUpdater() { - return this.overallStatsUpdater; - } -} - -describe("StatsSyncer", () => { - let statsSyncer: StatsSyncerMock; - - beforeEach(() => { - statsSyncer = new StatsSyncerMock(); - - return async () => { - await statsSyncer.close(); - }; - }); - - describe("when running the stats syncer", () => { - it("should start updaters correctly", async () => { - const dailyStatsUpdaterStartSpy = vi.spyOn( - statsSyncer.getDailyStatsUpdater(), - "start" - ); - const overallStatsUpdaterStartSpy = vi.spyOn( - statsSyncer.getOverallStatsUpdater(), - "start" - ); - - await statsSyncer.start({ - cronPatterns: { daily: "* * * * *", overall: "* * * * *" }, - }); - - expect(dailyStatsUpdaterStartSpy).toHaveBeenCalledWith("* * * * *"); - expect(overallStatsUpdaterStartSpy).toHaveBeenCalledWith("* * * * *"); - }); - - testValidError( - "should throw a valid error when failing to start it", - async () => { - const cronPatterns = { - daily: "* * * * *", - overall: "* * * * *", - }; - - vi.spyOn( - statsSyncer.getDailyStatsUpdater(), - "start" - ).mockRejectedValueOnce( - new Error( - "Something happened when trying to start daily stats updater" - ) - ); - - await statsSyncer.start({ - cronPatterns, - }); - }, - StatsSyncerError, - { - checkCause: true, - } - ); - }); - - describe("when closing the stats syncer", () => { - it("should close the updaters", async () => { - const closeStatsSyncer = new StatsSyncerMock(); - const dailyStatsUpdater = closeStatsSyncer.getDailyStatsUpdater(); - const overallStatsUpdater = closeStatsSyncer.getOverallStatsUpdater(); - const connection = closeStatsSyncer.getConnection(); - - const dailyStatsUpdaterCloseSpy = vi.spyOn(dailyStatsUpdater, "close"); - const overallStatsUpdaterCloseSpy = vi.spyOn( - overallStatsUpdater, - "close" - ); - const connectionCloseSpy = vi.spyOn(connection, "disconnect"); - - await closeStatsSyncer.close(); - - expect(dailyStatsUpdaterCloseSpy).toHaveBeenCalledOnce(); - expect(overallStatsUpdaterCloseSpy).toHaveBeenCalledOnce(); - expect(connectionCloseSpy).toHaveBeenCalledOnce(); - }); - - testValidError( - "should throw a valid error when failing to close it", - async () => { - const closeStatsSyncer = new StatsSyncerMock(); - const dailyStatsUpdater = closeStatsSyncer.getDailyStatsUpdater(); - vi.spyOn(dailyStatsUpdater, "close").mockRejectedValueOnce( - new Error("Some daily stats updater closing error") - ); - - await closeStatsSyncer.close(); - }, - StatsSyncerError, - { checkCause: true } - ); - }); -}); diff --git a/packages/swarm-syncer/package.json b/packages/swarm-syncer/package.json deleted file mode 100644 index ef11757ed..000000000 --- a/packages/swarm-syncer/package.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "name": "@blobscan/swarm-syncer", - "description": "Blobscan's swarm stamps data synchronizer", - "private": true, - "version": "0.0.1", - "main": "./src/index.ts", - "scripts": { - "clean": "rm -rf .turbo node_modules", - "lint": "eslint .", - "lint:fix": "pnpm lint --fix", - "type-check": "tsc --noEmit", - "test": "pnpm with-env:test vitest", - "test:ui": "pnpm with-env:test vitest --ui", - "with-env:test": ". ../../.env.test --" - }, - "dependencies": { - "@blobscan/dayjs": "workspace:^0.0.2", - "@blobscan/db": "workspace:^0.7.0", - "@blobscan/logger": "workspace:^0.1.0", - "axios": "^1.7.2", - "bullmq": "^4.13.2", - "ioredis": "^5.3.2" - }, - "eslintConfig": { - "root": true, - "extends": [ - "@blobscan/eslint-config/base" - ] - } -} diff --git a/packages/swarm-syncer/src/SwarmStampSyncer.ts b/packages/swarm-syncer/src/SwarmStampSyncer.ts deleted file mode 100644 index b02a204ea..000000000 --- a/packages/swarm-syncer/src/SwarmStampSyncer.ts +++ /dev/null @@ -1,67 +0,0 @@ - -import type { Redis } from "ioredis"; - -import { ErrorException, SwarmStampSyncerError } from "./errors"; -import { logger } from "./logger"; -import { SwarmStampUpdater } from "./updaters/SwarmUpdater"; -import { createRedisConnection } from "./utils"; - -export type SwarmStampSyncerOptions = { - redisUri: string; - beeEndpoint: string; -}; - -export class SwarmStampSyncer { - protected connection: Redis; - protected swarmUpdater: SwarmStampUpdater; - - constructor({ redisUri, beeEndpoint }: SwarmStampSyncerOptions) { - const connection = createRedisConnection(redisUri); - - connection.on("error", (err) => { - logger.error(new ErrorException("The Redis connection failed", err)); - }); - - this.connection = connection; - this.swarmUpdater = new SwarmStampUpdater(connection, beeEndpoint); - } - - async start(cronPattern: string) { - try { - await this.swarmUpdater.start(cronPattern); - logger.info("Swarm stamp syncer started successfully."); - } catch (err) { - const err_ = new SwarmStampSyncerError( - "An error occurred when starting swarm stamps syncer", - err - ); - - logger.error(err_); - - throw err_; - } - } - - async close() { - try { - await this.swarmUpdater - .close() - .finally(() => { - this.connection.removeAllListeners(); - - if (this.connection.status === "ready") this.connection.disconnect(); - }); - - logger.info("Stats syncer closed successfully."); - } catch (err) { - const err_ = new SwarmStampSyncerError( - "An error ocurred when closing syncer", - err - ); - - logger.error(err_); - - throw err_; - } - } -} diff --git a/packages/swarm-syncer/src/env.ts b/packages/swarm-syncer/src/env.ts deleted file mode 100644 index 9e4e5a43e..000000000 --- a/packages/swarm-syncer/src/env.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { - booleanSchema, - createEnv, - presetEnvOptions, -} from "@blobscan/zod"; - -export const env = createEnv({ - envOptions: { - server: { - SWARM_STORAGE_ENABLED: booleanSchema.default("false"), - }, - - ...presetEnvOptions, - }, -}); - -export type EnvVars = typeof env; diff --git a/packages/swarm-syncer/src/errors.ts b/packages/swarm-syncer/src/errors.ts deleted file mode 100644 index 0c67edd9f..000000000 --- a/packages/swarm-syncer/src/errors.ts +++ /dev/null @@ -1,21 +0,0 @@ -export class ErrorException extends Error { - constructor(message: string, cause?: unknown) { - super(message, { - cause, - }); - - this.name = this.constructor.name; - } -} - -export class SwarmStampSyncerError extends ErrorException { - constructor(message: string) { - super(`Stats syncer failed: ${message}`); - } -} - -export class PeriodicUpdaterError extends ErrorException { - constructor(updaterName: string, message: string, cause: unknown) { - super(`Updater "${updaterName}" failed: ${message}`, cause); - } -} diff --git a/packages/swarm-syncer/src/index.ts b/packages/swarm-syncer/src/index.ts deleted file mode 100644 index ab099d4a6..000000000 --- a/packages/swarm-syncer/src/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export { SwarmStampSyncer } from "./SwarmStampSyncer"; -export type { PeriodicUpdater, PeriodicUpdaterConfig } from "./PeriodicUpdater"; diff --git a/packages/swarm-syncer/src/logger.ts b/packages/swarm-syncer/src/logger.ts deleted file mode 100644 index a7022ae8a..000000000 --- a/packages/swarm-syncer/src/logger.ts +++ /dev/null @@ -1,3 +0,0 @@ -import { createModuleLogger } from "@blobscan/logger"; - -export const logger = createModuleLogger("swarm-stamp-syncer"); diff --git a/packages/swarm-syncer/src/updaters/SwarmUpdater.ts b/packages/swarm-syncer/src/updaters/SwarmUpdater.ts deleted file mode 100644 index b0db7de74..000000000 --- a/packages/swarm-syncer/src/updaters/SwarmUpdater.ts +++ /dev/null @@ -1,40 +0,0 @@ -import type { Redis } from "ioredis"; -import { prisma } from "@blobscan/db"; -import { env } from "../env"; -const axios = require('axios'); -import { PeriodicUpdater } from "../PeriodicUpdater"; -import { SwarmStampSyncerError } from "../errors"; - -export class SwarmStampUpdater extends PeriodicUpdater { - constructor( - redisUriOrConnection: string | Redis, - beeEndpoint: string, - ) { - const name = "swarm-stamp"; - super({ - name, - redisUriOrConnection, - updaterFn: async () => { - const batchId = env.SWARM_BATCH_ID; - const url = `${beeEndpoint}/stamps/${batchId}`; - const response = await axios.get(url); - const data = response.data; - - if (response.status != 200) { - throw new SwarmStampSyncerError(`Stamps endpoint returned status ${response.status} for batch id ${batchId}`); - } - - await prisma.blobStoragesState.update({ - data: { - swarmDataTTL: data.batchTTL, - }, - where: { - swarmDataId: batchId, - }, - }); - - this.logger.info(`Updated swarm stamp ${batchId}`); - }, - }); - } -} diff --git a/packages/swarm-syncer/src/utils.ts b/packages/swarm-syncer/src/utils.ts deleted file mode 100644 index 24f5cbbf7..000000000 --- a/packages/swarm-syncer/src/utils.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { Redis } from "ioredis"; - -import dayjs from "@blobscan/dayjs"; - -export function createRedisConnection(uri: string) { - return new Redis(uri, { - maxRetriesPerRequest: null, - }); -} - -export function formatDate(date: Date | string | dayjs.Dayjs) { - return dayjs(date).format("YYYY-MM-DD"); -} diff --git a/packages/swarm-syncer/test/SwarmStampSyncer.test.ts b/packages/swarm-syncer/test/SwarmStampSyncer.test.ts deleted file mode 100644 index 586199941..000000000 --- a/packages/swarm-syncer/test/SwarmStampSyncer.test.ts +++ /dev/null @@ -1,120 +0,0 @@ -import { beforeEach, describe, expect, it, vi } from "vitest"; - -import { testValidError } from "@blobscan/test"; - -import { StatsSyncer } from "../src/StatsSyncer"; -import { StatsSyncerError } from "../src/errors"; - -class StatsSyncerMock extends StatsSyncer { - constructor(redisUri = "redis://localhost:6379/1") { - super({ redisUri }); - } - - getConnection() { - return this.connection; - } - - getDailyStatsUpdater() { - return this.dailyStatsUpdater; - } - - getOverallStatsUpdater() { - return this.overallStatsUpdater; - } -} - -describe("StatsSyncer", () => { - let statsSyncer: StatsSyncerMock; - - beforeEach(() => { - statsSyncer = new StatsSyncerMock(); - - return async () => { - await statsSyncer.close(); - }; - }); - - describe("when running the stats syncer", () => { - it("should start updaters correctly", async () => { - const dailyStatsUpdaterStartSpy = vi.spyOn( - statsSyncer.getDailyStatsUpdater(), - "start" - ); - const overallStatsUpdaterStartSpy = vi.spyOn( - statsSyncer.getOverallStatsUpdater(), - "start" - ); - - await statsSyncer.start({ - cronPatterns: { daily: "* * * * *", overall: "* * * * *" }, - }); - - expect(dailyStatsUpdaterStartSpy).toHaveBeenCalledWith("* * * * *"); - expect(overallStatsUpdaterStartSpy).toHaveBeenCalledWith("* * * * *"); - }); - - testValidError( - "should throw a valid error when failing to start it", - async () => { - const cronPatterns = { - daily: "* * * * *", - overall: "* * * * *", - }; - - vi.spyOn( - statsSyncer.getDailyStatsUpdater(), - "start" - ).mockRejectedValueOnce( - new Error( - "Something happened when trying to start daily stats updater" - ) - ); - - await statsSyncer.start({ - cronPatterns, - }); - }, - StatsSyncerError, - { - checkCause: true, - } - ); - }); - - describe("when closing the stats syncer", () => { - it("should close the updaters", async () => { - const closeStatsSyncer = new StatsSyncerMock(); - const dailyStatsUpdater = closeStatsSyncer.getDailyStatsUpdater(); - const overallStatsUpdater = closeStatsSyncer.getOverallStatsUpdater(); - const connection = closeStatsSyncer.getConnection(); - - const dailyStatsUpdaterCloseSpy = vi.spyOn(dailyStatsUpdater, "close"); - const overallStatsUpdaterCloseSpy = vi.spyOn( - overallStatsUpdater, - "close" - ); - const connectionCloseSpy = vi.spyOn(connection, "disconnect"); - - await closeStatsSyncer.close(); - - expect(dailyStatsUpdaterCloseSpy).toHaveBeenCalledOnce(); - expect(overallStatsUpdaterCloseSpy).toHaveBeenCalledOnce(); - expect(connectionCloseSpy).toHaveBeenCalledOnce(); - }); - - testValidError( - "should throw a valid error when failing to close it", - async () => { - const closeStatsSyncer = new StatsSyncerMock(); - const dailyStatsUpdater = closeStatsSyncer.getDailyStatsUpdater(); - vi.spyOn(dailyStatsUpdater, "close").mockRejectedValueOnce( - new Error("Some daily stats updater closing error") - ); - - await closeStatsSyncer.close(); - }, - StatsSyncerError, - { checkCause: true } - ); - }); -}); diff --git a/packages/swarm-syncer/test/__snapshots__/StatsSyncer.test.ts.snap b/packages/swarm-syncer/test/__snapshots__/StatsSyncer.test.ts.snap deleted file mode 100644 index 1ab5d442b..000000000 --- a/packages/swarm-syncer/test/__snapshots__/StatsSyncer.test.ts.snap +++ /dev/null @@ -1,9 +0,0 @@ -// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html - -exports[`StatsSyncer > when closing the stats syncer > should throw a valid error when failing to close it 1`] = `"Stats syncer failed: An error ocurred when closing syncer"`; - -exports[`StatsSyncer > when closing the stats syncer > should throw a valid error when failing to close it 2`] = `[Error: Some daily stats updater closing error]`; - -exports[`StatsSyncer > when running the stats syncer > should throw a valid error when failing to start it 1`] = `"Stats syncer failed: An error occurred when starting syncer"`; - -exports[`StatsSyncer > when running the stats syncer > should throw a valid error when failing to start it 2`] = `[Error: Something happened when trying to start daily stats updater]`; diff --git a/packages/swarm-syncer/tsconfig.json b/packages/swarm-syncer/tsconfig.json deleted file mode 100644 index aff0b2dcd..000000000 --- a/packages/swarm-syncer/tsconfig.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "extends": "@blobscan/tsconfig/base.production.json", - "include": ["src/**/*.ts", "test/**/*.ts", "vitest.config.ts"] -} diff --git a/packages/swarm-syncer/vitest.config.ts b/packages/swarm-syncer/vitest.config.ts deleted file mode 100644 index 8fdeaf1f5..000000000 --- a/packages/swarm-syncer/vitest.config.ts +++ /dev/null @@ -1,5 +0,0 @@ -import { defineProject } from "vitest/config"; - -import { sharedProjectConfig } from "../../vitest.shared"; - -export default defineProject(sharedProjectConfig); diff --git a/packages/stats-syncer/CHANGELOG.md b/packages/syncers/CHANGELOG.md similarity index 99% rename from packages/stats-syncer/CHANGELOG.md rename to packages/syncers/CHANGELOG.md index 35e2a92b2..f1d175f2d 100644 --- a/packages/stats-syncer/CHANGELOG.md +++ b/packages/syncers/CHANGELOG.md @@ -1,4 +1,4 @@ -# @blobscan/stats-syncer +# @blobscan/syncers ## 0.1.9 diff --git a/packages/stats-syncer/package.json b/packages/syncers/package.json similarity index 95% rename from packages/stats-syncer/package.json rename to packages/syncers/package.json index 7bd1d58d5..01eafe66e 100644 --- a/packages/stats-syncer/package.json +++ b/packages/syncers/package.json @@ -1,5 +1,5 @@ { - "name": "@blobscan/stats-syncer", + "name": "@blobscan/syncers", "description": "Blobscan's stats syncer", "private": true, "version": "0.1.9", diff --git a/packages/swarm-syncer/src/PeriodicUpdater.ts b/packages/syncers/src/BaseSyncer.ts similarity index 55% rename from packages/swarm-syncer/src/PeriodicUpdater.ts rename to packages/syncers/src/BaseSyncer.ts index 03dd4bab1..48949a46e 100644 --- a/packages/swarm-syncer/src/PeriodicUpdater.ts +++ b/packages/syncers/src/BaseSyncer.ts @@ -5,42 +5,44 @@ import type { Redis } from "ioredis"; import { createModuleLogger } from "@blobscan/logger"; import type { Logger } from "@blobscan/logger"; -import { ErrorException, PeriodicUpdaterError } from "./errors"; +import { ErrorException, SyncerError } from "./errors"; import { createRedisConnection } from "./utils"; -// TODO: Refactor, this file is duplicate (packages: stats-syncer and swarm-syncer) -export type PeriodicUpdaterConfig = { +export interface CommonSyncerConfig { + redisUriOrConnection: Redis | string; + cronPattern: string; +} + +export interface BaseSyncerConfig extends CommonSyncerConfig { name: string; - redisUriOrConnection: string | Redis; - updaterFn: () => Promise; -}; + syncerFn: () => Promise; +} -export class PeriodicUpdater { +export class BaseSyncer { name: string; - protected worker: Worker; - protected queue: Queue; - protected updaterFn: () => Promise; + cronPattern: string; + + protected syncerFn: () => Promise; protected logger: Logger; + protected connection: Redis; + protected worker: Worker | undefined; + protected queue: Queue | undefined; + constructor({ name, + cronPattern, redisUriOrConnection, - updaterFn, - }: PeriodicUpdaterConfig) { - const isRedisUri = typeof redisUriOrConnection === "string"; - this.name = name; - this.logger = createModuleLogger("stats-syncer", this.name); + syncerFn, + }: BaseSyncerConfig) { + this.name = `${name}-syncer`; + this.cronPattern = cronPattern; + this.logger = createModuleLogger(this.name); let connection: Redis; - if (isRedisUri) { + if (typeof redisUriOrConnection === "string") { connection = createRedisConnection(redisUriOrConnection); - - connection.on("error", (err) => { - this.logger.error( - new ErrorException("A Redis connection error ocurred", err) - ); - }); } else { connection = redisUriOrConnection; } @@ -49,12 +51,10 @@ export class PeriodicUpdater { connection, }); - this.worker = new Worker(this.queue.name, updaterFn, { + this.worker = new Worker(this.queue.name, syncerFn, { connection, }); - this.updaterFn = updaterFn; - this.queue.on("error", (err) => { this.logger.error(new ErrorException("A queue error occurred", err)); }); @@ -62,22 +62,27 @@ export class PeriodicUpdater { this.worker.on("failed", (_, err) => { this.logger.error(new ErrorException("A worker error ocurred", err)); }); + + this.connection = connection; + this.syncerFn = syncerFn; } - async start(cronPattern: string) { + async start() { try { const jobName = `${this.name}-job`; - const repeatableJob = await this.queue.add(jobName, null, { + const repeatableJob = await this.queue?.add(jobName, null, { repeat: { - pattern: cronPattern, + pattern: this.cronPattern, }, }); + this.logger.info("Syncer started successfully"); + return repeatableJob; } catch (err) { - throw new PeriodicUpdaterError( + throw new SyncerError( this.name, - "An error ocurred when starting updater", + "An error ocurred when starting syncer", err ); } @@ -89,26 +94,28 @@ export class PeriodicUpdater { return teardownPromise .finally(async () => { await this.#performClosingOperation(() => - this.worker.removeAllListeners().close(true) + this.worker?.removeAllListeners().close(true) ); }) .finally(async () => { await this.#performClosingOperation(() => - this.queue.obliterate({ force: true }) + this.queue?.obliterate({ force: true }) ); }) .finally(async () => { await this.#performClosingOperation(() => - this.queue.removeAllListeners().close() + this.queue?.removeAllListeners().close() ); + + this.logger.info("Syncer closed successfully"); }); } - async #performClosingOperation(operation: () => Promise) { + async #performClosingOperation(operation: () => Promise | undefined) { try { await operation(); } catch (err) { - const err_ = new PeriodicUpdaterError( + const err_ = new SyncerError( this.name, "An error ocurred when performing closing operation", err diff --git a/packages/syncers/src/errors.ts b/packages/syncers/src/errors.ts new file mode 100644 index 000000000..6afc5bf78 --- /dev/null +++ b/packages/syncers/src/errors.ts @@ -0,0 +1,15 @@ +export class ErrorException extends Error { + constructor(message: string, cause?: unknown) { + super(message, { + cause, + }); + + this.name = this.constructor.name; + } +} + +export class SyncerError extends ErrorException { + constructor(syncerName: string, message: string, cause: unknown) { + super(`Syncer "${syncerName}" failed: ${message}`, cause); + } +} diff --git a/packages/syncers/src/index.ts b/packages/syncers/src/index.ts new file mode 100644 index 000000000..1cf748880 --- /dev/null +++ b/packages/syncers/src/index.ts @@ -0,0 +1,2 @@ +export * from "./syncers"; +export { createRedisConnection } from "./utils"; diff --git a/packages/stats-syncer/src/updaters/DailyStatsUpdater.ts b/packages/syncers/src/syncers/DailyStatsSyncer.ts similarity index 88% rename from packages/stats-syncer/src/updaters/DailyStatsUpdater.ts rename to packages/syncers/src/syncers/DailyStatsSyncer.ts index ace23fba1..f07d2df86 100644 --- a/packages/stats-syncer/src/updaters/DailyStatsUpdater.ts +++ b/packages/syncers/src/syncers/DailyStatsSyncer.ts @@ -1,9 +1,8 @@ -import type { Redis } from "ioredis"; - import { normalizeDailyDate, normalizeDate, prisma } from "@blobscan/db"; import type { PrismaPromise, RawDatePeriod } from "@blobscan/db"; -import { PeriodicUpdater } from "../PeriodicUpdater"; +import { BaseSyncer } from "../BaseSyncer"; +import type { CommonSyncerConfig } from "../BaseSyncer"; import { formatDate } from "../utils"; interface DailyStatsModel { @@ -20,13 +19,17 @@ const dailyStatsModels: Record = { transaction: prisma.transactionDailyStats, }; -export class DailyStatsUpdater extends PeriodicUpdater { - constructor(redisUriOrConnection: string | Redis) { - const name = "daily"; +export type DailyStatsSyncerConfig = CommonSyncerConfig; + +export class DailyStatsSyncer extends BaseSyncer { + constructor({ redisUriOrConnection, cronPattern }: DailyStatsSyncerConfig) { + const name = "daily-stats"; + super({ name, redisUriOrConnection, - updaterFn: async () => { + cronPattern, + syncerFn: async () => { const findLatestArgs: { select: { day: boolean; diff --git a/packages/stats-syncer/src/updaters/OverallStatsUpdater.ts b/packages/syncers/src/syncers/OverallStatsSyncer.ts similarity index 87% rename from packages/stats-syncer/src/updaters/OverallStatsUpdater.ts rename to packages/syncers/src/syncers/OverallStatsSyncer.ts index ae3935c3f..3720b1bb3 100644 --- a/packages/stats-syncer/src/updaters/OverallStatsUpdater.ts +++ b/packages/syncers/src/syncers/OverallStatsSyncer.ts @@ -1,14 +1,13 @@ -import type { Redis } from "ioredis"; - import { prisma } from "@blobscan/db"; import type { BlockNumberRange, Prisma } from "@blobscan/db"; -import { PeriodicUpdater } from "../PeriodicUpdater"; +import { BaseSyncer } from "../BaseSyncer"; +import type { CommonSyncerConfig } from "../BaseSyncer"; -export type OverallStatsUpdaterOptions = { +export interface OverallStatsSyncerConfig extends CommonSyncerConfig { batchSize?: number; lowestSlot?: number; -}; +} const DEFAULT_BATCH_SIZE = 2_000_000; const DEFAULT_INITIAL_SLOT = 0; @@ -17,21 +16,19 @@ function isUnset(value: T | undefined | null): value is null | undefined { return value === undefined || value === null; } -export class OverallStatsUpdater extends PeriodicUpdater { - constructor( - redisUriOrConnection: string | Redis, - options: OverallStatsUpdaterOptions = {} - ) { - const name = "overall"; +export class OverallStatsSyncer extends BaseSyncer { + constructor({ + cronPattern, + redisUriOrConnection, + batchSize = DEFAULT_BATCH_SIZE, + lowestSlot = DEFAULT_INITIAL_SLOT, + }: OverallStatsSyncerConfig) { + const name = "overall-stats"; super({ name, + cronPattern, redisUriOrConnection, - updaterFn: async () => { - const { - batchSize = DEFAULT_BATCH_SIZE, - lowestSlot = DEFAULT_INITIAL_SLOT, - } = options ?? {}; - + syncerFn: async () => { const [blockchainSyncState, latestBlock] = await Promise.all([ prisma.blockchainSyncState.findUnique({ select: { diff --git a/packages/syncers/src/syncers/index.ts b/packages/syncers/src/syncers/index.ts new file mode 100644 index 000000000..4fbba13aa --- /dev/null +++ b/packages/syncers/src/syncers/index.ts @@ -0,0 +1,4 @@ +export { DailyStatsSyncer } from "./DailyStatsSyncer"; +export type { DailyStatsSyncerConfig } from "./DailyStatsSyncer"; +export { OverallStatsSyncer } from "./OverallStatsSyncer"; +export type { OverallStatsSyncerConfig } from "./OverallStatsSyncer"; diff --git a/packages/stats-syncer/src/utils.ts b/packages/syncers/src/utils.ts similarity index 100% rename from packages/stats-syncer/src/utils.ts rename to packages/syncers/src/utils.ts diff --git a/packages/swarm-syncer/test/PeriodicUpdater.test.ts b/packages/syncers/test/BaseSyncer.test.ts similarity index 74% rename from packages/swarm-syncer/test/PeriodicUpdater.test.ts rename to packages/syncers/test/BaseSyncer.test.ts index 9354c905e..2d480ddf7 100644 --- a/packages/swarm-syncer/test/PeriodicUpdater.test.ts +++ b/packages/syncers/test/BaseSyncer.test.ts @@ -2,28 +2,29 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import { testValidError } from "@blobscan/test"; -import { PeriodicUpdater } from "../src/PeriodicUpdater"; -import type { PeriodicUpdaterConfig } from "../src/PeriodicUpdater"; -import { PeriodicUpdaterError } from "../src/errors"; - -class PeriodicUpdaterMock extends PeriodicUpdater { - constructor({ - name, - redisUriOrConnection, - updaterFn, - }: Partial = {}) { +import { BaseSyncer } from "../src/BaseSyncer"; +import type { BaseSyncerConfig } from "../src/BaseSyncer"; +import { SyncerError } from "../src/errors"; + +class PeriodicUpdaterMock extends BaseSyncer { + constructor({ name, syncerFn: updaterFn }: Partial = {}) { super({ name: name ?? "test-updater", - redisUriOrConnection: redisUriOrConnection ?? "redis://localhost:6379/1", - updaterFn: updaterFn ?? (() => Promise.resolve()), + redisUriOrConnection: "redis://localhost:6379/1", + cronPattern: "* * * * *", + syncerFn: updaterFn ?? (() => Promise.resolve()), }); } getWorker() { + if (!this.worker) throw new Error("Worker not initialized"); + return this.worker; } getQueue() { + if (!this.queue) throw new Error("Queue not initialized"); + return this.queue; } } @@ -51,15 +52,14 @@ describe("PeriodicUpdater", () => { describe("when running an updater", () => { it("should set up a repeatable job correctly", async () => { const queue = periodicUpdater.getQueue(); - const cronPattern = "* * * * *"; - await periodicUpdater.start(cronPattern); + await periodicUpdater.start(); const jobs = await queue.getRepeatableJobs(); expect(jobs.length, "Expected one repeatable job").toBe(1); expect(jobs[0]?.pattern, "Repetable job cron pattern mismatch").toEqual( - cronPattern + "* * * * *" ); }); @@ -70,17 +70,21 @@ describe("PeriodicUpdater", () => { vi.spyOn(queue, "add").mockRejectedValueOnce(new Error("Queue error")); - await periodicUpdater.start("* * * * *"); + await periodicUpdater.start(); }, - PeriodicUpdaterError, + SyncerError, { checkCause: true } ); }); describe("when closing an updater", () => { it("should close correctly", async () => { - const queue = periodicUpdater.getQueue(); - const worker = periodicUpdater.getWorker(); + const closingPeriodicUpdater = new PeriodicUpdaterMock(); + + await closingPeriodicUpdater.start(); + + const queue = closingPeriodicUpdater.getQueue(); + const worker = closingPeriodicUpdater.getWorker(); const queueCloseSpy = vi.spyOn(queue, "close").mockResolvedValueOnce(); const queueRemoveAllListenersSpy = vi @@ -92,7 +96,7 @@ describe("PeriodicUpdater", () => { .spyOn(worker, "removeAllListeners") .mockReturnValueOnce(worker); - await periodicUpdater.close(); + await closingPeriodicUpdater.close(); expect(queueCloseSpy).toHaveBeenCalledOnce(); expect(workerCloseSpy).toHaveBeenCalledOnce(); @@ -117,7 +121,7 @@ describe("PeriodicUpdater", () => { await periodicUpdater.close(); }, - PeriodicUpdaterError, + SyncerError, { checkCause: true, } diff --git a/packages/stats-syncer/test/DailyStatsUpdater.test.fixtures.ts b/packages/syncers/test/DailyStatsSyncer.test.fixtures.ts similarity index 100% rename from packages/stats-syncer/test/DailyStatsUpdater.test.fixtures.ts rename to packages/syncers/test/DailyStatsSyncer.test.fixtures.ts diff --git a/packages/stats-syncer/test/DailyStatsUpdater.test.ts b/packages/syncers/test/DailyStatsSyncer.test.ts similarity index 78% rename from packages/stats-syncer/test/DailyStatsUpdater.test.ts rename to packages/syncers/test/DailyStatsSyncer.test.ts index de7c2daed..7557a39c9 100644 --- a/packages/stats-syncer/test/DailyStatsUpdater.test.ts +++ b/packages/syncers/test/DailyStatsSyncer.test.ts @@ -3,18 +3,18 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import { prisma } from "@blobscan/db"; import { fixtures } from "@blobscan/test"; -import { DailyStatsUpdater } from "../src/updaters/DailyStatsUpdater"; +import { DailyStatsSyncer } from "../src/syncers/"; import { formatDate } from "../src/utils"; -import { CURRENT_DAY_DATA } from "./DailyStatsUpdater.test.fixtures"; +import { CURRENT_DAY_DATA } from "./DailyStatsSyncer.test.fixtures"; import { expectDailyStatsDatesToBeEqual, getAllDailyStatsDates, indexNewBlock, -} from "./DailyStatsUpdater.test.utils"; +} from "./DailyStatsSyncer.test.utils"; -class DailyStatsUpdaterMock extends DailyStatsUpdater { +class DailyStatsSyncerMock extends DailyStatsSyncer { constructor(redisUri = process.env.REDIS_URI ?? "") { - super(redisUri); + super({ redisUriOrConnection: redisUri, cronPattern: "* * * * *" }); } getWorker() { @@ -22,7 +22,7 @@ class DailyStatsUpdaterMock extends DailyStatsUpdater { } getWorkerProcessor() { - return this.updaterFn; + return this.syncerFn; } getQueue() { @@ -30,23 +30,23 @@ class DailyStatsUpdaterMock extends DailyStatsUpdater { } } -describe("DailyStatsUpdater", () => { +describe("DailyStatsSyncer", () => { const allExpectedDates = Array.from( new Set(fixtures.blocks.map((block) => formatDate(block.timestamp))) ).sort((a, b) => (a < b ? -1 : 1)); - let dailyStatsUpdater: DailyStatsUpdaterMock; + let dailyStatsSyncer: DailyStatsSyncerMock; beforeEach(() => { - dailyStatsUpdater = new DailyStatsUpdaterMock(); + dailyStatsSyncer = new DailyStatsSyncerMock(); return async () => { - await dailyStatsUpdater.close(); + await dailyStatsSyncer.close(); }; }); it("should aggregate data for all available days", async () => { - const workerProcessor = dailyStatsUpdater.getWorkerProcessor(); + const workerProcessor = dailyStatsSyncer.getWorkerProcessor(); await workerProcessor(); @@ -61,7 +61,7 @@ describe("DailyStatsUpdater", () => { }); it("should skip aggregation if not all blocks have been indexed for the last day", async () => { - const workerProcessor = dailyStatsUpdater.getWorkerProcessor(); + const workerProcessor = dailyStatsSyncer.getWorkerProcessor(); await indexNewBlock(CURRENT_DAY_DATA); @@ -75,7 +75,7 @@ describe("DailyStatsUpdater", () => { }); it("should skip aggregation if no blocks have been indexed yet", async () => { - const workerProcessor = dailyStatsUpdater.getWorkerProcessor(); + const workerProcessor = dailyStatsSyncer.getWorkerProcessor(); const findLatestSpy = vi .spyOn(prisma.block, "findLatest") @@ -92,7 +92,7 @@ describe("DailyStatsUpdater", () => { it("should skip aggregation if already up to date", async () => { await indexNewBlock(CURRENT_DAY_DATA); - const workerProcessor = dailyStatsUpdater.getWorkerProcessor(); + const workerProcessor = dailyStatsSyncer.getWorkerProcessor(); await workerProcessor(); diff --git a/packages/stats-syncer/test/DailyStatsUpdater.test.utils.ts b/packages/syncers/test/DailyStatsSyncer.test.utils.ts similarity index 100% rename from packages/stats-syncer/test/DailyStatsUpdater.test.utils.ts rename to packages/syncers/test/DailyStatsSyncer.test.utils.ts diff --git a/packages/swarm-syncer/test/OverallStatsUpdater.test.ts b/packages/syncers/test/OverallStatsSyncer.test.ts similarity index 91% rename from packages/swarm-syncer/test/OverallStatsUpdater.test.ts rename to packages/syncers/test/OverallStatsSyncer.test.ts index bed3404d7..fbd13987e 100644 --- a/packages/swarm-syncer/test/OverallStatsUpdater.test.ts +++ b/packages/syncers/test/OverallStatsSyncer.test.ts @@ -3,16 +3,16 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import { prisma } from "@blobscan/db"; import { fixtures, omitDBTimestampFields } from "@blobscan/test"; -import { OverallStatsUpdater } from "../src/updaters/OverallStatsUpdater"; +import { OverallStatsSyncer } from "../src/syncers/OverallStatsSyncer"; +import type { OverallStatsSyncerConfig } from "../src/syncers/OverallStatsSyncer"; -class OverallStatsUpdaterMock extends OverallStatsUpdater { - constructor( - redisUri = process.env.REDIS_URI ?? "", - config: ConstructorParameters[1] = {} - ) { +class OverallStatsUpdaterMock extends OverallStatsSyncer { + constructor(config: Partial = {}) { const lowestSlot = config.lowestSlot ?? fixtures.blockchainSyncState[0]?.lastLowerSyncedSlot; - super(redisUri, { + super({ + cronPattern: "* * * * *", + redisUriOrConnection: "redis://localhost:6379/1", ...config, lowestSlot, }); @@ -23,7 +23,7 @@ class OverallStatsUpdaterMock extends OverallStatsUpdater { } getWorkerProcessor() { - return this.updaterFn; + return this.syncerFn; } getQueue() { @@ -114,7 +114,7 @@ describe("OverallStatsUpdater", () => { it("should aggregate overall stats in batches correctly when there are too many blocks", async () => { const batchSize = 2; - const workerProcessor = new OverallStatsUpdaterMock(undefined, { + const workerProcessor = new OverallStatsUpdaterMock({ batchSize, }).getWorkerProcessor(); const incrementTransactionSpy = vi.spyOn(prisma, "$transaction"); @@ -193,7 +193,7 @@ describe("OverallStatsUpdater", () => { }); it("should skip aggregation when the lowest slot hasn't been reached yet", async () => { - const workerProcessor = new OverallStatsUpdaterMock(undefined, { + const workerProcessor = new OverallStatsUpdaterMock({ lowestSlot: 1, }).getWorkerProcessor(); diff --git a/packages/syncers/test/SwarmSyncer.test.ts b/packages/syncers/test/SwarmSyncer.test.ts new file mode 100644 index 000000000..e69de29bb diff --git a/packages/swarm-syncer/test/__snapshots__/PeriodicUpdater.test.ts.snap b/packages/syncers/test/__snapshots__/BaseSyncer.test.ts.snap similarity index 64% rename from packages/swarm-syncer/test/__snapshots__/PeriodicUpdater.test.ts.snap rename to packages/syncers/test/__snapshots__/BaseSyncer.test.ts.snap index ad2709e36..dba557d81 100644 --- a/packages/swarm-syncer/test/__snapshots__/PeriodicUpdater.test.ts.snap +++ b/packages/syncers/test/__snapshots__/BaseSyncer.test.ts.snap @@ -1,9 +1,9 @@ // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html -exports[`PeriodicUpdater > should throw a valid error when failing to close it 1`] = `"Updater \\"test-updater\\" failed: An error ocurred when performing closing operation"`; +exports[`PeriodicUpdater > should throw a valid error when failing to close it 1`] = `"Syncer \\"test-updater-syncer\\" failed: An error ocurred when performing closing operation"`; exports[`PeriodicUpdater > should throw a valid error when failing to close it 2`] = `[Error: Queue closing error]`; -exports[`PeriodicUpdater > when running an updater > should throw a valid error when failing to run 1`] = `"Updater \\"test-updater\\" failed: An error ocurred when starting updater"`; +exports[`PeriodicUpdater > when running an updater > should throw a valid error when failing to run 1`] = `"Syncer \\"test-updater-syncer\\" failed: An error ocurred when starting syncer"`; exports[`PeriodicUpdater > when running an updater > should throw a valid error when failing to run 2`] = `[Error: Queue error]`; diff --git a/packages/stats-syncer/test/__snapshots__/PeriodicUpdater.test.ts.snap b/packages/syncers/test/__snapshots__/PeriodicUpdater.test.ts.snap similarity index 73% rename from packages/stats-syncer/test/__snapshots__/PeriodicUpdater.test.ts.snap rename to packages/syncers/test/__snapshots__/PeriodicUpdater.test.ts.snap index ad2709e36..5c860cbe0 100644 --- a/packages/stats-syncer/test/__snapshots__/PeriodicUpdater.test.ts.snap +++ b/packages/syncers/test/__snapshots__/PeriodicUpdater.test.ts.snap @@ -1,9 +1,9 @@ // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html -exports[`PeriodicUpdater > should throw a valid error when failing to close it 1`] = `"Updater \\"test-updater\\" failed: An error ocurred when performing closing operation"`; +exports[`PeriodicUpdater > should throw a valid error when failing to close it 1`] = `"Updater \\"test-updater-updater\\" failed: An error ocurred when performing closing operation"`; exports[`PeriodicUpdater > should throw a valid error when failing to close it 2`] = `[Error: Queue closing error]`; -exports[`PeriodicUpdater > when running an updater > should throw a valid error when failing to run 1`] = `"Updater \\"test-updater\\" failed: An error ocurred when starting updater"`; +exports[`PeriodicUpdater > when running an updater > should throw a valid error when failing to run 1`] = `"Updater \\"test-updater-updater\\" failed: An error ocurred when starting updater"`; exports[`PeriodicUpdater > when running an updater > should throw a valid error when failing to run 2`] = `[Error: Queue error]`; diff --git a/packages/stats-syncer/test/__snapshots__/StatsSyncer.test.ts.snap b/packages/syncers/test/__snapshots__/StatsSyncer.test.ts.snap similarity index 100% rename from packages/stats-syncer/test/__snapshots__/StatsSyncer.test.ts.snap rename to packages/syncers/test/__snapshots__/StatsSyncer.test.ts.snap diff --git a/packages/syncers/test/__snapshots__/SyncerManager.test.ts.snap b/packages/syncers/test/__snapshots__/SyncerManager.test.ts.snap new file mode 100644 index 000000000..606d6a8bc --- /dev/null +++ b/packages/syncers/test/__snapshots__/SyncerManager.test.ts.snap @@ -0,0 +1,9 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`SyncerManager > when closing the syncer manager > should throw a valid error when failing to close it 1`] = `"Periodic updater manager failed: An error ocurred when closing syncers"`; + +exports[`SyncerManager > when closing the syncer manager > should throw a valid error when failing to close it 2`] = `[Error: Some daily stats updater closing error]`; + +exports[`SyncerManager > when running the syncer manager > should throw a valid error when failing to start it 1`] = `"Periodic updater manager failed: An error occurred when starting syncers"`; + +exports[`SyncerManager > when running the syncer manager > should throw a valid error when failing to start it 2`] = `[Error: Something happened when trying to start daily stats updater]`; diff --git a/packages/stats-syncer/tsconfig.json b/packages/syncers/tsconfig.json similarity index 100% rename from packages/stats-syncer/tsconfig.json rename to packages/syncers/tsconfig.json diff --git a/packages/stats-syncer/vitest.config.ts b/packages/syncers/vitest.config.ts similarity index 100% rename from packages/stats-syncer/vitest.config.ts rename to packages/syncers/vitest.config.ts diff --git a/packages/test/src/fixtures/postgres/data.json b/packages/test/src/fixtures/postgres/data.json index 2158e14f9..f62b1561b 100644 --- a/packages/test/src/fixtures/postgres/data.json +++ b/packages/test/src/fixtures/postgres/data.json @@ -13,7 +13,8 @@ { "id": 1, "swarmDataId": "batch-1", - "swarmDataTTL": 1000 + "swarmDataTTL": 1000, + "updatedAt": "2023-10-31T12:10:00Z" } ], "blocks": [ diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7bd0ff606..68830c843 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -171,12 +171,9 @@ importers: '@blobscan/open-telemetry': specifier: workspace:^0.0.7 version: link:../../packages/open-telemetry - '@blobscan/stats-syncer': - specifier: workspace:^0.1.8 - version: link:../../packages/stats-syncer - '@blobscan/swarm-syncer': - specifier: workspace:^0.0.1 - version: link:../../packages/swarm-syncer + '@blobscan/syncers': + specifier: workspace:^0.1.9 + version: link:../../packages/syncers '@blobscan/zod': specifier: workspace:^0.1.0 version: link:../../packages/zod @@ -590,25 +587,7 @@ importers: specifier: ^14.2.0 version: 14.2.0 - packages/stats-syncer: - dependencies: - '@blobscan/dayjs': - specifier: workspace:^0.0.2 - version: link:../dayjs - '@blobscan/db': - specifier: workspace:^0.7.0 - version: link:../db - '@blobscan/logger': - specifier: workspace:^0.1.0 - version: link:../logger - bullmq: - specifier: ^4.13.2 - version: 4.17.0 - ioredis: - specifier: ^5.3.2 - version: 5.4.1 - - packages/swarm-syncer: + packages/syncers: dependencies: '@blobscan/dayjs': specifier: workspace:^0.0.2 @@ -619,9 +598,6 @@ importers: '@blobscan/logger': specifier: workspace:^0.1.0 version: link:../logger - axios: - specifier: ^1.7.2 - version: 1.7.2 bullmq: specifier: ^4.13.2 version: 4.17.0 @@ -3206,9 +3182,6 @@ packages: axios@1.6.8: resolution: {integrity: sha512-v/ZHtJDU39mDpyBoFVkETcd/uNdxrWRrg3bKpOKzXFA6Bvqopts6ALSMU3y6ijYxbw2B+wPrIv46egTzJXCLGQ==} - axios@1.7.2: - resolution: {integrity: sha512-2A8QhOMrbomlDuiLeK9XibIBzuHeRcqqNOHp0Cyp5EoJ1IFDh+XZH3A6BkXtv0K4gFGCI0Y4BM7B1wOEi0Rmgw==} - axobject-query@3.2.1: resolution: {integrity: sha512-jsyHu61e6N4Vbz/v18DHwWYKK0bSWLqn47eeDSKPB7m8tqMHF9YJ+mhIk2lVteyZrY8tnSj/jHOv4YiTCuCJgg==} @@ -9808,14 +9781,6 @@ snapshots: transitivePeerDependencies: - debug - axios@1.7.2: - dependencies: - follow-redirects: 1.15.6 - form-data: 4.0.0 - proxy-from-env: 1.1.0 - transitivePeerDependencies: - - debug - axobject-query@3.2.1: dependencies: dequal: 2.0.3 @@ -10649,7 +10614,7 @@ snapshots: '@typescript-eslint/parser': 5.62.0(eslint@8.57.0)(typescript@5.4.5) eslint: 8.57.0 eslint-import-resolver-node: 0.3.9 - eslint-import-resolver-typescript: 3.6.1(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(eslint@8.57.0))(eslint@8.57.0) + eslint-import-resolver-typescript: 3.6.1(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0) eslint-plugin-import: 2.29.1(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0) eslint-plugin-jsx-a11y: 6.8.0(eslint@8.57.0) eslint-plugin-react: 7.32.2(eslint@8.57.0) @@ -10677,12 +10642,12 @@ snapshots: transitivePeerDependencies: - supports-color - eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(eslint@8.57.0))(eslint@8.57.0): + eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0): dependencies: debug: 4.3.4 enhanced-resolve: 5.15.0 eslint: 8.57.0 - eslint-module-utils: 2.8.0(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(eslint@8.57.0))(eslint@8.57.0))(eslint@8.57.0) + eslint-module-utils: 2.8.0(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0))(eslint@8.57.0) eslint-plugin-import: 2.29.1(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-typescript@3.6.1)(eslint@8.57.0) fast-glob: 3.3.2 get-tsconfig: 4.7.2 @@ -10694,14 +10659,14 @@ snapshots: - eslint-import-resolver-webpack - supports-color - eslint-module-utils@2.8.0(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(eslint@8.57.0))(eslint@8.57.0))(eslint@8.57.0): + eslint-module-utils@2.8.0(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0))(eslint@8.57.0): dependencies: debug: 3.2.7 optionalDependencies: '@typescript-eslint/parser': 5.62.0(eslint@8.57.0)(typescript@5.4.5) eslint: 8.57.0 eslint-import-resolver-node: 0.3.9 - eslint-import-resolver-typescript: 3.6.1(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(eslint@8.57.0))(eslint@8.57.0) + eslint-import-resolver-typescript: 3.6.1(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0) transitivePeerDependencies: - supports-color @@ -10715,7 +10680,7 @@ snapshots: doctrine: 2.1.0 eslint: 8.57.0 eslint-import-resolver-node: 0.3.9 - eslint-module-utils: 2.8.0(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(eslint@8.57.0))(eslint@8.57.0))(eslint@8.57.0) + eslint-module-utils: 2.8.0(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0))(eslint@8.57.0) hasown: 2.0.2 is-core-module: 2.13.1 is-glob: 4.0.3 @@ -10742,7 +10707,7 @@ snapshots: doctrine: 2.1.0 eslint: 8.57.0 eslint-import-resolver-node: 0.3.9 - eslint-module-utils: 2.8.0(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1(eslint@8.57.0))(eslint@8.57.0))(eslint@8.57.0) + eslint-module-utils: 2.8.0(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.6.1(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-plugin-import@2.29.1)(eslint@8.57.0))(eslint@8.57.0) hasown: 2.0.2 is-core-module: 2.13.1 is-glob: 4.0.3 From 363a5aae45e087b8938325a472e2c1c1dcfde42d Mon Sep 17 00:00:00 2001 From: PJColombo Date: Thu, 13 Jun 2024 13:21:16 +0200 Subject: [PATCH 4/8] feat(db): add prisma migration field that adds a new `updated_at` field --- .changeset/moody-fireants-itch.md | 5 +++++ .../20240613112056_add_updated_at_field/migration.sql | 2 ++ 2 files changed, 7 insertions(+) create mode 100644 .changeset/moody-fireants-itch.md create mode 100644 packages/db/prisma/migrations/20240613112056_add_updated_at_field/migration.sql diff --git a/.changeset/moody-fireants-itch.md b/.changeset/moody-fireants-itch.md new file mode 100644 index 000000000..f2d56790f --- /dev/null +++ b/.changeset/moody-fireants-itch.md @@ -0,0 +1,5 @@ +--- +"@blobscan/db": minor +--- + +Added an updated at field to blob storages state model diff --git a/packages/db/prisma/migrations/20240613112056_add_updated_at_field/migration.sql b/packages/db/prisma/migrations/20240613112056_add_updated_at_field/migration.sql new file mode 100644 index 000000000..e9b6d9116 --- /dev/null +++ b/packages/db/prisma/migrations/20240613112056_add_updated_at_field/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "blob_storages_state" ADD COLUMN "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP; From fb9c08409ff43f2be14197bf0db9f5a2d2965ee9 Mon Sep 17 00:00:00 2001 From: PJColombo Date: Thu, 13 Jun 2024 18:19:35 +0200 Subject: [PATCH 5/8] feat(syncers): add swarm stamp syncer --- .changeset/fair-rabbits-cross.md | 5 + package.json | 1 + packages/syncers/package.json | 2 + packages/syncers/src/errors.ts | 35 ++ packages/syncers/src/index.ts | 1 + .../syncers/src/syncers/SwarmStampSyncer.ts | 73 ++++ packages/syncers/src/syncers/index.ts | 2 + .../syncers/test/SwarmStampSyncer.test.ts | 179 ++++++++++ packages/syncers/test/SwarmSyncer.test.ts | 0 .../SwarmStampSyncer.test.ts.snap | 9 + packages/syncers/test/helpers.ts | 11 + packages/test/src/fixtures/index.ts | 8 +- packages/test/src/fixtures/postgres/data.json | 2 +- pnpm-lock.yaml | 312 +++++++++++++++++- 14 files changed, 628 insertions(+), 12 deletions(-) create mode 100644 .changeset/fair-rabbits-cross.md create mode 100644 packages/syncers/src/syncers/SwarmStampSyncer.ts create mode 100644 packages/syncers/test/SwarmStampSyncer.test.ts delete mode 100644 packages/syncers/test/SwarmSyncer.test.ts create mode 100644 packages/syncers/test/__snapshots__/SwarmStampSyncer.test.ts.snap create mode 100644 packages/syncers/test/helpers.ts diff --git a/.changeset/fair-rabbits-cross.md b/.changeset/fair-rabbits-cross.md new file mode 100644 index 000000000..e887917db --- /dev/null +++ b/.changeset/fair-rabbits-cross.md @@ -0,0 +1,5 @@ +--- +"@blobscan/syncers": minor +--- + +Added swarm stamp syncer diff --git a/package.json b/package.json index 53021d87f..87753bbd3 100644 --- a/package.json +++ b/package.json @@ -50,6 +50,7 @@ "@vitest/coverage-v8": "^0.34.3", "@vitest/ui": "^0.34.1", "dotenv-cli": "^7.2.1", + "msw": "^2.3.1", "prettier": "^2.8.8", "prettier-plugin-tailwindcss": "^0.2.8", "ts-node": "^10.9.1", diff --git a/packages/syncers/package.json b/packages/syncers/package.json index 01eafe66e..039fa09a7 100644 --- a/packages/syncers/package.json +++ b/packages/syncers/package.json @@ -17,6 +17,8 @@ "@blobscan/dayjs": "workspace:^0.0.2", "@blobscan/db": "workspace:^0.7.0", "@blobscan/logger": "workspace:^0.1.0", + "@blobscan/zod": "workspace:^0.1.0", + "axios": "^1.7.2", "bullmq": "^4.13.2", "ioredis": "^5.3.2" }, diff --git a/packages/syncers/src/errors.ts b/packages/syncers/src/errors.ts index 6afc5bf78..8967ff33d 100644 --- a/packages/syncers/src/errors.ts +++ b/packages/syncers/src/errors.ts @@ -1,3 +1,7 @@ +import type { AxiosError } from "axios"; + +import { z } from "@blobscan/zod"; + export class ErrorException extends Error { constructor(message: string, cause?: unknown) { super(message, { @@ -13,3 +17,34 @@ export class SyncerError extends ErrorException { super(`Syncer "${syncerName}" failed: ${message}`, cause); } } + +const swarmApiResponseErrorSchema = z.object({ + code: z.number(), + message: z.string(), + reasons: z.array(z.unknown()).optional(), +}); + +export class SwarmNodeError extends ErrorException { + code: number | undefined; + reasons?: unknown[]; + + constructor(error: AxiosError) { + let message: string; + let code: number | undefined; + const result = swarmApiResponseErrorSchema.safeParse(error.response?.data); + let reasons: unknown[] | undefined; + + if (result.success) { + code = result.data.code; + message = result.data.message; + reasons = result.data.reasons; + } else { + message = error.message; + } + + super(message, error.cause); + + this.code = code; + this.reasons = reasons; + } +} diff --git a/packages/syncers/src/index.ts b/packages/syncers/src/index.ts index 1cf748880..2197f0b3b 100644 --- a/packages/syncers/src/index.ts +++ b/packages/syncers/src/index.ts @@ -1,2 +1,3 @@ +export { BaseSyncer } from "./BaseSyncer"; export * from "./syncers"; export { createRedisConnection } from "./utils"; diff --git a/packages/syncers/src/syncers/SwarmStampSyncer.ts b/packages/syncers/src/syncers/SwarmStampSyncer.ts new file mode 100644 index 000000000..25da38b49 --- /dev/null +++ b/packages/syncers/src/syncers/SwarmStampSyncer.ts @@ -0,0 +1,73 @@ +import type { AxiosResponse } from "axios"; +import { AxiosError } from "axios"; +import axios from "axios"; + +import { prisma } from "@blobscan/db"; + +import { BaseSyncer } from "../BaseSyncer"; +import type { CommonSyncerConfig } from "../BaseSyncer"; +import { SwarmNodeError } from "../errors"; + +type BatchData = { + batchID: string; + batchTTL: number; +}; + +export interface SwarmStampSyncerConfig extends CommonSyncerConfig { + beeEndpoint: string; + batchId: string; +} + +export class SwarmStampSyncer extends BaseSyncer { + constructor({ + cronPattern, + redisUriOrConnection, + batchId, + beeEndpoint, + }: SwarmStampSyncerConfig) { + const name = "swarm-stamp"; + super({ + name, + cronPattern, + redisUriOrConnection, + syncerFn: async () => { + let response: AxiosResponse; + + try { + const url = `${beeEndpoint}/stamps/${batchId}`; + + response = await axios.get(url); + } catch (err) { + let cause = err; + + if (err instanceof AxiosError) { + cause = new SwarmNodeError(err); + } + + throw new Error(`Failed to fetch stamp batch "${batchId}"`, { + cause, + }); + } + + const { batchTTL } = response.data; + + await prisma.blobStoragesState.upsert({ + create: { + swarmDataId: batchId, + swarmDataTTL: batchTTL, + }, + update: { + swarmDataTTL: batchTTL, + updatedAt: new Date(), + }, + where: { + id: 1, + swarmDataId: batchId, + }, + }); + + this.logger.info(`Swarm stamp data with batch ID "${batchId}" updated`); + }, + }); + } +} diff --git a/packages/syncers/src/syncers/index.ts b/packages/syncers/src/syncers/index.ts index 4fbba13aa..660eaf482 100644 --- a/packages/syncers/src/syncers/index.ts +++ b/packages/syncers/src/syncers/index.ts @@ -2,3 +2,5 @@ export { DailyStatsSyncer } from "./DailyStatsSyncer"; export type { DailyStatsSyncerConfig } from "./DailyStatsSyncer"; export { OverallStatsSyncer } from "./OverallStatsSyncer"; export type { OverallStatsSyncerConfig } from "./OverallStatsSyncer"; +export { SwarmStampSyncer } from "./SwarmStampSyncer"; +export type { SwarmStampSyncerConfig } from "./SwarmStampSyncer"; diff --git a/packages/syncers/test/SwarmStampSyncer.test.ts b/packages/syncers/test/SwarmStampSyncer.test.ts new file mode 100644 index 000000000..84ed0d478 --- /dev/null +++ b/packages/syncers/test/SwarmStampSyncer.test.ts @@ -0,0 +1,179 @@ +/* eslint-disable @typescript-eslint/no-misused-promises */ +import { http, HttpResponse } from "msw"; +import { setupServer } from "msw/node"; +import { beforeAll, beforeEach, describe, expect, it } from "vitest"; + +import type { BlobStoragesState } from "@blobscan/db"; +import { prisma } from "@blobscan/db"; +import { fixtures, testValidError } from "@blobscan/test"; + +import type { SwarmStampSyncerConfig } from "../src/syncers/SwarmStampSyncer"; +import { SwarmStampSyncer } from "../src/syncers/SwarmStampSyncer"; + +const BEE_ENDPOINT = process.env.BEE_ENDPOINT ?? "http://localhost:1633"; + +class SwarmStampSyncerMock extends SwarmStampSyncer { + constructor({ batchId, cronPattern }: Partial = {}) { + super({ + redisUriOrConnection: process.env.REDIS_URI ?? "", + cronPattern: cronPattern ?? "* * * * *", + batchId: batchId ?? process.env.SWARM_BATCH_ID ?? "", + beeEndpoint: BEE_ENDPOINT, + }); + } + + getQueue() { + return this.queue; + } + + getWorkerProcessor() { + return this.syncerFn; + } +} + +describe("SwarmStampSyncer", () => { + const expectedBatchId = fixtures.blobStoragesState[0]?.swarmDataId as string; + const expectedBatchTTL = 1000; + + let swarmStampSyncer: SwarmStampSyncerMock; + + beforeAll(() => { + const baseUrl = `${BEE_ENDPOINT}/stamps`; + const server = setupServer( + ...[ + http.get(`${baseUrl}/:batchId`, ({ request }) => { + const batchId = request.url.split("/").pop(); + + if (!batchId || batchId.length !== 64) { + return HttpResponse.json( + { + code: 400, + message: "invalid path params", + reasons: [ + { + field: "batch_id", + error: "odd length hex string", + }, + ], + }, + { status: 400 } + ); + } + + if (batchId !== expectedBatchId) { + return HttpResponse.json( + { + code: 404, + message: "issuer does not exist", + }, + { status: 404 } + ); + } + + return HttpResponse.json( + { + batchID: expectedBatchId, + batchTTL: expectedBatchTTL, + }, + { + status: 200, + } + ); + }), + ] + ); + + server.listen(); + + return () => { + server.close(); + }; + }); + + beforeEach(() => { + swarmStampSyncer = new SwarmStampSyncerMock(); + + return async () => { + await swarmStampSyncer.close(); + }; + }); + + describe("when creating a new swarm batch data row in the db", async () => { + let blobStorageState: BlobStoragesState | null = null; + + beforeEach(async () => { + await prisma.blobStoragesState.deleteMany(); + + const workerProcessor = swarmStampSyncer.getWorkerProcessor(); + + await workerProcessor().catch((err) => console.log(err)); + + blobStorageState = await prisma.blobStoragesState.findFirst(); + }); + + it("should create it with the correct swarm stamp batch ID", async () => { + expect(blobStorageState?.swarmDataId).toBe(process.env.SWARM_BATCH_ID); + }); + + it("should create it with the correct batch TTL", async () => { + expect(blobStorageState?.swarmDataTTL).toBe(expectedBatchTTL); + }); + }); + + it("should update the batch TTl", async () => { + await prisma.blobStoragesState.update({ + data: { + swarmDataTTL: 99999, + }, + where: { + id: 1, + }, + }); + + const workerProcessor = swarmStampSyncer.getWorkerProcessor(); + await workerProcessor(); + + const blobStorageState = await prisma.blobStoragesState.findFirst(); + + expect(blobStorageState?.swarmDataTTL).toBe(expectedBatchTTL); + }); + + testValidError( + "should fail when trying to fetch a non-existing batch", + async () => { + const failingSwarmStampSyncer = new SwarmStampSyncerMock({ + batchId: + "6b538866048cfb6e9e1d06805374c51572c11219d2d550c03e6e277366cb0371", + }); + const failingWorkerProcessor = + failingSwarmStampSyncer.getWorkerProcessor(); + + await failingWorkerProcessor().finally(async () => { + await failingSwarmStampSyncer.close(); + }); + }, + Error, + { + checkCause: true, + } + ); + + testValidError( + "should fail when trying to fetch an invalid batch", + async () => { + const failingSwarmStampSyncer = new SwarmStampSyncerMock({ + batchId: "invalid-batch", + }); + const failingWorkerProcessor = + failingSwarmStampSyncer.getWorkerProcessor(); + + await failingWorkerProcessor().finally(async () => { + await failingSwarmStampSyncer.close(); + }); + }, + Error, + { + checkCause: true, + } + ); +}); diff --git a/packages/syncers/test/SwarmSyncer.test.ts b/packages/syncers/test/SwarmSyncer.test.ts deleted file mode 100644 index e69de29bb..000000000 diff --git a/packages/syncers/test/__snapshots__/SwarmStampSyncer.test.ts.snap b/packages/syncers/test/__snapshots__/SwarmStampSyncer.test.ts.snap new file mode 100644 index 000000000..8385c3d78 --- /dev/null +++ b/packages/syncers/test/__snapshots__/SwarmStampSyncer.test.ts.snap @@ -0,0 +1,9 @@ +// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html + +exports[`SwarmStampSyncer > should fail when trying to fetch a non-existing batch 1`] = `"Failed to fetch stamp batch \\"6b538866048cfb6e9e1d06805374c51572c11219d2d550c03e6e277366cb0371\\""`; + +exports[`SwarmStampSyncer > should fail when trying to fetch a non-existing batch 2`] = `[SwarmNodeError: issuer does not exist]`; + +exports[`SwarmStampSyncer > should fail when trying to fetch an invalid batch 1`] = `"Failed to fetch stamp batch \\"invalid-batch\\""`; + +exports[`SwarmStampSyncer > should fail when trying to fetch an invalid batch 2`] = `[SwarmNodeError: invalid path params]`; diff --git a/packages/syncers/test/helpers.ts b/packages/syncers/test/helpers.ts new file mode 100644 index 000000000..0c79d5a59 --- /dev/null +++ b/packages/syncers/test/helpers.ts @@ -0,0 +1,11 @@ +import { setupServer } from "msw/node"; + +export function createServer(handlers: Parameters[0][]) { + const server = setupServer(...handlers); + + server.listen(); + + return () => { + server.close(); + }; +} diff --git a/packages/test/src/fixtures/index.ts b/packages/test/src/fixtures/index.ts index b96e6fed0..5a31ad036 100644 --- a/packages/test/src/fixtures/index.ts +++ b/packages/test/src/fixtures/index.ts @@ -1,9 +1,5 @@ -import { - BlobData, - BlobDataStorageReference, - type PrismaClient, - type Rollup, -} from "@prisma/client"; +import type { BlobData, BlobDataStorageReference } from "@prisma/client"; +import type { PrismaClient, Rollup } from "@prisma/client"; import POSTGRES_DATA from "./postgres/data.json"; diff --git a/packages/test/src/fixtures/postgres/data.json b/packages/test/src/fixtures/postgres/data.json index f62b1561b..a227867c2 100644 --- a/packages/test/src/fixtures/postgres/data.json +++ b/packages/test/src/fixtures/postgres/data.json @@ -12,7 +12,7 @@ "blobStoragesState": [ { "id": 1, - "swarmDataId": "batch-1", + "swarmDataId": "f89e63edf757f06e89933761d6d46592d03026efb9871f9d244f34da86b6c242", "swarmDataTTL": 1000, "updatedAt": "2023-10-31T12:10:00Z" } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 68830c843..940d5e137 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -59,6 +59,9 @@ importers: dotenv-cli: specifier: ^7.2.1 version: 7.4.1 + msw: + specifier: ^2.3.1 + version: 2.3.1(typescript@5.4.5) prettier: specifier: ^2.8.8 version: 2.8.8 @@ -103,7 +106,7 @@ importers: version: 2.2.1 '@tailwindcss/typography': specifier: ^0.5.7 - version: 0.5.13(tailwindcss@3.4.3(ts-node@10.9.2(@types/node@18.19.31)(typescript@5.4.5))) + version: 0.5.13(tailwindcss@3.4.3(ts-node@10.9.2(@types/node@20.14.2)(typescript@5.4.5))) autoprefixer: specifier: ^10.4.14 version: 10.4.19(postcss@8.4.38) @@ -142,7 +145,7 @@ importers: version: 1.2.1 tailwindcss: specifier: ^3.3.1 - version: 3.4.3(ts-node@10.9.2(@types/node@18.19.31)(typescript@5.4.5)) + version: 3.4.3(ts-node@10.9.2(@types/node@20.14.2)(typescript@5.4.5)) devDependencies: eslint: specifier: ^8.45.0 @@ -598,6 +601,12 @@ importers: '@blobscan/logger': specifier: workspace:^0.1.0 version: link:../logger + '@blobscan/zod': + specifier: workspace:^0.1.0 + version: link:../zod + axios: + specifier: ^1.7.2 + version: 1.7.2 bullmq: specifier: ^4.13.2 version: 4.17.0 @@ -675,7 +684,7 @@ importers: version: 8.4.38 tailwindcss: specifier: ^3.3.1 - version: 3.4.3(ts-node@10.9.2(@types/node@18.19.31)(typescript@5.4.5)) + version: 3.4.3(ts-node@10.9.2(@types/node@20.14.2)(typescript@5.4.5)) tooling/typescript: {} @@ -1391,6 +1400,12 @@ packages: '@bcoe/v8-coverage@0.2.3': resolution: {integrity: sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==} + '@bundled-es-modules/cookie@2.0.0': + resolution: {integrity: sha512-Or6YHg/kamKHpxULAdSqhGqnWFneIXu1NKvvfBBzKGwpVsYuFIQ5aBPHDnnoR3ghW1nvSkALd+EF9iMtY7Vjxw==} + + '@bundled-es-modules/statuses@1.0.1': + resolution: {integrity: sha512-yn7BklA5acgcBr+7w064fGV+SGIFySjCKpqjcWgBAIfrAkY+4GQTJJHQMeT3V/sgz23VTEVV8TtOmkvJAhFVfg==} + '@changesets/apply-release-plan@7.0.0': resolution: {integrity: sha512-vfi69JR416qC9hWmFGSxj7N6wA5J222XNBmezSVATPWDVPIF7gkd4d8CpbEbXmRWbVrkoli3oerGS6dcL/BGsQ==} @@ -1791,6 +1806,22 @@ packages: '@vue/compiler-sfc': optional: true + '@inquirer/confirm@3.1.9': + resolution: {integrity: sha512-UF09aejxCi4Xqm6N/jJAiFXArXfi9al52AFaSD+2uIHnhZGtd1d6lIGTRMPouVSJxbGEi+HkOWSYaiEY/+szUw==} + engines: {node: '>=18'} + + '@inquirer/core@8.2.2': + resolution: {integrity: sha512-K8SuNX45jEFlX3EBJpu9B+S2TISzMPGXZIuJ9ME924SqbdW6Pt6fIkKvXg7mOEOKJ4WxpQsxj0UTfcL/A434Ww==} + engines: {node: '>=18'} + + '@inquirer/figures@1.0.3': + resolution: {integrity: sha512-ErXXzENMH5pJt5/ssXV0DfWUZqly8nGzf0UcBV9xTnP+KyffE2mqyxIMBrZ8ijQck2nU0TQm40EQB53YreyWHw==} + engines: {node: '>=18'} + + '@inquirer/type@1.3.3': + resolution: {integrity: sha512-xTUt0NulylX27/zMx04ZYar/kr1raaiFTVvQ5feljQsiAgdm0WPj4S73/ye0fbslh+15QrIuDvfCXTek7pMY5A==} + engines: {node: '>=18'} + '@ioredis/commands@1.2.0': resolution: {integrity: sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==} @@ -1898,6 +1929,14 @@ packages: cpu: [x64] os: [win32] + '@mswjs/cookies@1.1.0': + resolution: {integrity: sha512-0ZcCVQxifZmhwNBoQIrystCb+2sWBY2Zw8lpfJBPCHGCA/HWqehITeCRVIv4VMy8MPlaHo2w2pTHFV2pFfqKPw==} + engines: {node: '>=18'} + + '@mswjs/interceptors@0.29.1': + resolution: {integrity: sha512-3rDakgJZ77+RiQUuSK69t1F0m8BQKA8Vh5DCS5V0DWvNY67zob2JhhQrhCO0AKLGINTRSFd1tBaHcJTkhefoSw==} + engines: {node: '>=18'} + '@next-auth/prisma-adapter@1.0.7': resolution: {integrity: sha512-Cdko4KfcmKjsyHFrWwZ//lfLUbcLqlyFqjd/nYE2m3aZ7tjMNUjpks47iw7NTCnXf+5UWz5Ypyt1dSs1EP5QJw==} peerDependencies: @@ -2063,6 +2102,15 @@ packages: resolution: {integrity: sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==} engines: {node: '>= 8'} + '@open-draft/deferred-promise@2.2.0': + resolution: {integrity: sha512-CecwLWx3rhxVQF6V4bAgPS5t+So2sTbPgAzafKkVizyi7tlwpcFpdFqq+wqF2OwNBmqFuu6tOyouTuxgpMfzmA==} + + '@open-draft/logger@0.3.0': + resolution: {integrity: sha512-X2g45fzhxH238HKO4xbSr7+wBS8Fvw6ixhTDuvLd5mqh6bJJCFAPwU9mPDxbcrRtfxv4u5IHCEH77BmxvXmmxQ==} + + '@open-draft/until@2.1.0': + resolution: {integrity: sha512-U69T3ItWHvLwGg5eJ0n3I62nWuE6ilHlmz7zM0npLBRvPRd7e6NYmg54vvRtP5mZG7kZqZCFVdsTWo7BPtBujg==} + '@opentelemetry/api-logs@0.41.2': resolution: {integrity: sha512-JEV2RAqijAFdWeT6HddYymfnkiRu2ASxoTBr4WsnGJhOjWZkEy6vp+Sx9ozr1NaIODOa2HUyckExIqQjn6qywQ==} engines: {node: '>=14'} @@ -2765,6 +2813,9 @@ packages: '@types/connect@3.4.38': resolution: {integrity: sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==} + '@types/cookie@0.6.0': + resolution: {integrity: sha512-4Kh9a6B2bQciAhf7FSuMRRkUWecJgJu9nPnx3yzpsfXX/c50REIqpHY4C82bXP90qrLtXtkDxTZosYO3UpOwlA==} + '@types/cors@2.8.17': resolution: {integrity: sha512-8CGDvrBj1zgo2qE+oS3pOCyYNqCPryMWY2bGfwA0dcfopWGgxs+78df0Rs3rc9THP4JkOhLsAa+15VdpAqkcUA==} @@ -2825,12 +2876,18 @@ packages: '@types/morgan@1.9.9': resolution: {integrity: sha512-iRYSDKVaC6FkGSpEVVIvrRGw0DfJMiQzIn3qr2G5B3C//AWkulhXgaBd7tS9/J79GWSYMTHGs7PfI5b3Y8m+RQ==} + '@types/mute-stream@0.0.4': + resolution: {integrity: sha512-CPM9nzrCPPJHQNA9keH9CVkVI+WR5kMa+7XEs5jcGQ0VoAGnLv242w8lIVgwAEfmE4oufJRaTc9PNLQl0ioAow==} + '@types/node@12.20.55': resolution: {integrity: sha512-J8xLz7q2OFulZ2cyGTLE1TbbZcjpno7FaN6zdJNrgAdrJ+DZzh/uFR6YrTb4C+nXakvud8Q4+rbhoIWlYQbUFQ==} '@types/node@18.19.31': resolution: {integrity: sha512-ArgCD39YpyyrtFKIqMDvjz79jto5fcI/SVUs2HwB+f0dAzq68yqOdyaSivLiLugSziTpNXLQrVb7RZFmdZzbhA==} + '@types/node@20.14.2': + resolution: {integrity: sha512-xyu6WAMVwv6AKFLB+e/7ySZVr/0zLCzOa7rSpq6jNwpqOrUbcACDWC+53d4n2QHOnDou0fbIsg8wZu/sxrnI4Q==} + '@types/normalize-package-data@2.4.4': resolution: {integrity: sha512-37i+OaWTh9qeK4LSHPsyRC7NahnGotNuZvjLSgcPzblpHB3rrCJxAOgI5gCdKm7coonsaX1Of0ILiTcnZjbfxA==} @@ -2876,6 +2933,9 @@ packages: '@types/shimmer@1.0.5': resolution: {integrity: sha512-9Hp0ObzwwO57DpLFF0InUjUm/II8GmKAvzbefxQTihCb7KI6yc9yzf0nLc4mVdby5N4DRCgQM2wCup9KTieeww==} + '@types/statuses@2.0.5': + resolution: {integrity: sha512-jmIUGWrAiwu3dZpxntxieC+1n/5c3mjrImkmOSQ2NC5uP6cYO4aAZDdSmRcI5C1oiTmqlZGHC+/NmJrKogbP5A==} + '@types/strip-bom@3.0.0': resolution: {integrity: sha512-xevGOReSYGM7g/kUBZzPqCrR/KYAo+F0yiPc85WFTJa0MSLtyFTVTU6cJu/aV4mid7IffDIWqo69THF2o4JiEQ==} @@ -2888,6 +2948,9 @@ packages: '@types/triple-beam@1.3.5': resolution: {integrity: sha512-6WaYesThRMCl19iryMYP7/x2OVgCtbIVflDGFpWnb9irXI3UjYE4AzmYuiUKY1AJstGijoY+MgUszMgRxIYTYw==} + '@types/wrap-ansi@3.0.0': + resolution: {integrity: sha512-ltIpx+kM7g/MLRZfkbL7EsCEjfzCcScLpkg37eXEtx5kmrAKBkTJwd1GIAjDSL8wTpM6Hzn5YO4pSb91BEwu1g==} + '@types/zrender@4.0.6': resolution: {integrity: sha512-1jZ9bJn2BsfmYFPBHtl5o3uV+ILejAtGrDcYSpT4qaVKEI/0YY+arw3XHU04Ebd8Nca3SQ7uNcLaqiL+tTFVMg==} @@ -3058,6 +3121,10 @@ packages: resolution: {integrity: sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw==} engines: {node: '>=6'} + ansi-escapes@4.3.2: + resolution: {integrity: sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ==} + engines: {node: '>=8'} + ansi-regex@5.0.1: resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==} engines: {node: '>=8'} @@ -3182,6 +3249,9 @@ packages: axios@1.6.8: resolution: {integrity: sha512-v/ZHtJDU39mDpyBoFVkETcd/uNdxrWRrg3bKpOKzXFA6Bvqopts6ALSMU3y6ijYxbw2B+wPrIv46egTzJXCLGQ==} + axios@1.7.2: + resolution: {integrity: sha512-2A8QhOMrbomlDuiLeK9XibIBzuHeRcqqNOHp0Cyp5EoJ1IFDh+XZH3A6BkXtv0K4gFGCI0Y4BM7B1wOEi0Rmgw==} + axobject-query@3.2.1: resolution: {integrity: sha512-jsyHu61e6N4Vbz/v18DHwWYKK0bSWLqn47eeDSKPB7m8tqMHF9YJ+mhIk2lVteyZrY8tnSj/jHOv4YiTCuCJgg==} @@ -3397,6 +3467,14 @@ packages: classnames@2.5.1: resolution: {integrity: sha512-saHYOzhIQs6wy2sVxTM6bUDsQO4F50V9RQ22qBpEdCW+I+/Wmke2HOl6lS6dTpdxVhb88/I6+Hs+438c3lfUow==} + cli-spinners@2.9.2: + resolution: {integrity: sha512-ywqV+5MmyL4E7ybXgKys4DugZbX0FC6LnwrhjuykIjnK9k8OQacQ7axGKnjDXWNhns0xot3bZI5h55H8yo9cJg==} + engines: {node: '>=6'} + + cli-width@4.1.0: + resolution: {integrity: sha512-ouuZd4/dm2Sw5Gmqy6bGyNNNe1qt9RpmxveLSO7KcgsTnU7RXfsw+/bukWGo1abgBiMAic068rclZsO4IWmmxQ==} + engines: {node: '>= 12'} + client-only@0.0.1: resolution: {integrity: sha512-IV3Ou0jSMzZrd3pZ48nLkT9DA7Ag1pnPzaiQhpW7c3RbcqqzvzzVu+L8gfqMp/8IM2MQtSiqaCxrrcfu8I8rMA==} @@ -4328,6 +4406,10 @@ packages: graphemer@1.4.0: resolution: {integrity: sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==} + graphql@16.8.2: + resolution: {integrity: sha512-cvVIBILwuoSyD54U4cF/UXDh5yAobhNV/tPygI4lZhgOIJQE/WLWC4waBRb4I6bDVYb3OVx3lfHbaQOEoUD5sg==} + engines: {node: ^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0} + gtoken@6.1.2: resolution: {integrity: sha512-4ccGpzz7YAr7lxrT2neugmXQ3hP9ho2gcaityLVkiUecAiwiy60Ii8gRbZeOsXV19fYaRjgBSshs8kXw+NKCPQ==} engines: {node: '>=12.0.0'} @@ -4376,6 +4458,9 @@ packages: resolution: {integrity: sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==} engines: {node: '>= 0.4'} + headers-polyfill@4.0.3: + resolution: {integrity: sha512-IScLbePpkvO846sIwOtOTDjutRMWdXdJmXdMvk6gCBHxFO8d+QKOQedyZSxFTTFYRSmlgSTDtXqqq4pcenBXLQ==} + hexer@1.5.0: resolution: {integrity: sha512-dyrPC8KzBzUJ19QTIo1gXNqIISRXQ0NwteW6OeQHRN4ZuZeHkdODfj0zHBdOlHbRY8GqbqK57C9oWSvQZizFsg==} engines: {node: '>= 0.10.x'} @@ -4545,6 +4630,9 @@ packages: resolution: {integrity: sha512-5KoIu2Ngpyek75jXodFvnafB6DJgr3u8uuK0LEZJjrU19DrMD3EVERaR8sjz8CCGgpZvxPl9SuE1GMVPFHx1mw==} engines: {node: '>= 0.4'} + is-node-process@1.2.0: + resolution: {integrity: sha512-Vg4o6/fqPxIjtxgUH5QLJhwZ7gW5diGCVlXpuUfELC62CuxM1iHcRe51f2W1FDy04Ai4KJkagKjx3XaqyfRKXw==} + is-number-object@1.0.7: resolution: {integrity: sha512-k1U0IRzLMo7ZlYIfzRu23Oh6MiIFasgpb9X76eqfFZAqwH44UI4KTBvBYIZ1dSL9ZzChTB9ShHfLkR4pdW5krQ==} engines: {node: '>= 0.4'} @@ -5049,9 +5137,23 @@ packages: msgpackr@1.10.1: resolution: {integrity: sha512-r5VRLv9qouXuLiIBrLpl2d5ZvPt8svdQTl5/vMvE4nzDMyEX4sgW5yWhuBBj5UmgwOTWj8CIdSXn5sAfsHAWIQ==} + msw@2.3.1: + resolution: {integrity: sha512-ocgvBCLn/5l3jpl1lssIb3cniuACJLoOfZu01e3n5dbJrpA5PeeWn28jCLgQDNt6d7QT8tF2fYRzm9JoEHtiig==} + engines: {node: '>=18'} + hasBin: true + peerDependencies: + typescript: '>= 4.7.x' + peerDependenciesMeta: + typescript: + optional: true + multiformats@9.9.0: resolution: {integrity: sha512-HoMUjhH9T8DDBNT+6xzkrd9ga/XiBI4xLr58LJACwK6G3HTOPeMz4nB4KJs33L2BelrIJa7P0VuNaVF3hMYfjg==} + mute-stream@1.0.0: + resolution: {integrity: sha512-avsJQhyd+680gKXyG/sQc0nXaC6rBkPOfyHYcFb9+hdkqQkR9bdnkJ0AMZhke0oesPqIO+mFFJ+IdBc7mst4IA==} + engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0} + mz@2.7.0: resolution: {integrity: sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q==} @@ -5276,6 +5378,9 @@ packages: outdent@0.5.0: resolution: {integrity: sha512-/jHxFIzoMXdqPzTaCpFzAAWhpkSjZPF4Vsn6jAfNpmbH/ymsmd7Qc6VE9BGn0L6YMj6uwpQLxCECpus4ukKS9Q==} + outvariant@1.4.2: + resolution: {integrity: sha512-Ou3dJ6bA/UJ5GVHxah4LnqDwZRwAmWxrG3wtrHrbGnP4RnLCtA64A4F+ae7Y8ww660JaddSoArUR5HjipWSHAQ==} + p-cancelable@3.0.0: resolution: {integrity: sha512-mlVgR3PGuzlo0MmTdk4cXqXWlwQDLnONTAg6sm62XkMJEiRxN3GL3SffkYvqwonbkJBcrI7Uvv5Zh9yjvn2iUw==} engines: {node: '>=12.20'} @@ -5351,6 +5456,9 @@ packages: path-to-regexp@0.1.7: resolution: {integrity: sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ==} + path-to-regexp@6.2.2: + resolution: {integrity: sha512-GQX3SSMokngb36+whdpRXE+3f9V8UzyAorlYvOGx87ufGHehNTn5lCxrKtLyZ4Yl/wEKnNnr98ZzOwwDZV5ogw==} + path-type@4.0.0: resolution: {integrity: sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==} engines: {node: '>=8'} @@ -5894,6 +6002,10 @@ packages: signal-exit@3.0.7: resolution: {integrity: sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==} + signal-exit@4.1.0: + resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==} + engines: {node: '>=14'} + simple-concat@1.0.1: resolution: {integrity: sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==} @@ -5994,6 +6106,9 @@ packages: streamx@2.16.1: resolution: {integrity: sha512-m9QYj6WygWyWa3H1YY69amr4nVgy61xfjys7xO7kviL5rfIEc2naf+ewFiOA+aEJD7y0JO3h2GoiUv4TDwEGzQ==} + strict-event-emitter@0.5.1: + resolution: {integrity: sha512-vMgjE/GGEPEFnhFub6pa4FmJBRBVOLpIII2hvCZ8Kzb7K0hlHo7mQv6xYrBvCL2LtAIBwFUK8wvuJgTVSQ5MFQ==} + string-template@0.2.1: resolution: {integrity: sha512-Yptehjogou2xm4UJbxJ4CxgZx12HBfeystp0y3x7s4Dj32ltVVG1Gg8YhKjHZkHicuKpZX/ffilA8505VbUbpw==} @@ -6341,6 +6456,10 @@ packages: resolution: {integrity: sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==} engines: {node: '>=10'} + type-fest@0.21.3: + resolution: {integrity: sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==} + engines: {node: '>=10'} + type-fest@0.6.0: resolution: {integrity: sha512-q+MB8nYR1KDLrgr4G5yemftpMC7/QLqVndBmEEdqzmNj5dcFOO4Oo8qlwZE3ULT3+Zim1F8Kq4cBnikNhlCMlg==} engines: {node: '>=8'} @@ -6353,6 +6472,10 @@ packages: resolution: {integrity: sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA==} engines: {node: '>=8'} + type-fest@4.20.0: + resolution: {integrity: sha512-MBh+PHUHHisjXf4tlx0CFWoMdjx8zCMLJHOjnV1prABYZFHqtFOyauCIK2/7w4oIfwkF8iNhLtnJEfVY2vn3iw==} + engines: {node: '>=16'} + type-is@1.6.18: resolution: {integrity: sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==} engines: {node: '>= 0.6'} @@ -7615,6 +7738,14 @@ snapshots: '@bcoe/v8-coverage@0.2.3': {} + '@bundled-es-modules/cookie@2.0.0': + dependencies: + cookie: 0.5.0 + + '@bundled-es-modules/statuses@1.0.1': + dependencies: + statuses: 2.0.1 + '@changesets/apply-release-plan@7.0.0': dependencies: '@babel/runtime': 7.23.2 @@ -8255,6 +8386,31 @@ snapshots: transitivePeerDependencies: - supports-color + '@inquirer/confirm@3.1.9': + dependencies: + '@inquirer/core': 8.2.2 + '@inquirer/type': 1.3.3 + + '@inquirer/core@8.2.2': + dependencies: + '@inquirer/figures': 1.0.3 + '@inquirer/type': 1.3.3 + '@types/mute-stream': 0.0.4 + '@types/node': 20.14.2 + '@types/wrap-ansi': 3.0.0 + ansi-escapes: 4.3.2 + chalk: 4.1.2 + cli-spinners: 2.9.2 + cli-width: 4.1.0 + mute-stream: 1.0.0 + signal-exit: 4.1.0 + strip-ansi: 6.0.1 + wrap-ansi: 6.2.0 + + '@inquirer/figures@1.0.3': {} + + '@inquirer/type@1.3.3': {} + '@ioredis/commands@1.2.0': {} '@istanbuljs/schema@0.1.3': {} @@ -8368,6 +8524,17 @@ snapshots: '@msgpackr-extract/msgpackr-extract-win32-x64@3.0.2': optional: true + '@mswjs/cookies@1.1.0': {} + + '@mswjs/interceptors@0.29.1': + dependencies: + '@open-draft/deferred-promise': 2.2.0 + '@open-draft/logger': 0.3.0 + '@open-draft/until': 2.1.0 + is-node-process: 1.2.0 + outvariant: 1.4.2 + strict-event-emitter: 0.5.1 + '@next-auth/prisma-adapter@1.0.7(@prisma/client@5.13.0(prisma@5.13.0))(next-auth@4.24.7(next@13.5.6(@babel/core@7.24.5)(@opentelemetry/api@1.8.0)(react-dom@18.2.0(react@18.2.0))(react@18.2.0))(react-dom@18.2.0(react@18.2.0))(react@18.2.0))': dependencies: '@prisma/client': 5.13.0(prisma@5.13.0) @@ -8470,6 +8637,15 @@ snapshots: '@nodelib/fs.scandir': 2.1.5 fastq: 1.15.0 + '@open-draft/deferred-promise@2.2.0': {} + + '@open-draft/logger@0.3.0': + dependencies: + is-node-process: 1.2.0 + outvariant: 1.4.2 + + '@open-draft/until@2.1.0': {} + '@opentelemetry/api-logs@0.41.2': dependencies: '@opentelemetry/api': 1.8.0 @@ -9177,13 +9353,13 @@ snapshots: mini-svg-data-uri: 1.4.4 tailwindcss: 3.4.3(ts-node@10.9.2(@types/node@18.19.31)(typescript@5.4.5)) - '@tailwindcss/typography@0.5.13(tailwindcss@3.4.3(ts-node@10.9.2(@types/node@18.19.31)(typescript@5.4.5)))': + '@tailwindcss/typography@0.5.13(tailwindcss@3.4.3(ts-node@10.9.2(@types/node@20.14.2)(typescript@5.4.5)))': dependencies: lodash.castarray: 4.4.0 lodash.isplainobject: 4.0.6 lodash.merge: 4.6.2 postcss-selector-parser: 6.0.10 - tailwindcss: 3.4.3(ts-node@10.9.2(@types/node@18.19.31)(typescript@5.4.5)) + tailwindcss: 3.4.3(ts-node@10.9.2(@types/node@20.14.2)(typescript@5.4.5)) '@tanstack/query-core@4.36.1': {} @@ -9258,6 +9434,8 @@ snapshots: dependencies: '@types/node': 18.19.31 + '@types/cookie@0.6.0': {} + '@types/cors@2.8.17': dependencies: '@types/node': 18.19.31 @@ -9330,12 +9508,20 @@ snapshots: dependencies: '@types/node': 18.19.31 + '@types/mute-stream@0.0.4': + dependencies: + '@types/node': 18.19.31 + '@types/node@12.20.55': {} '@types/node@18.19.31': dependencies: undici-types: 5.26.5 + '@types/node@20.14.2': + dependencies: + undici-types: 5.26.5 + '@types/normalize-package-data@2.4.4': {} '@types/prettier@2.7.3': {} @@ -9383,6 +9569,8 @@ snapshots: '@types/shimmer@1.0.5': {} + '@types/statuses@2.0.5': {} + '@types/strip-bom@3.0.0': {} '@types/strip-json-comments@0.0.30': {} @@ -9394,6 +9582,8 @@ snapshots: '@types/triple-beam@1.3.5': {} + '@types/wrap-ansi@3.0.0': {} + '@types/zrender@4.0.6': {} '@typescript-eslint/eslint-plugin@5.62.0(@typescript-eslint/parser@5.62.0(eslint@8.57.0)(typescript@5.4.5))(eslint@8.57.0)(typescript@5.4.5)': @@ -9642,6 +9832,10 @@ snapshots: ansi-colors@4.1.3: {} + ansi-escapes@4.3.2: + dependencies: + type-fest: 0.21.3 + ansi-regex@5.0.1: {} ansi-styles@3.2.1: @@ -9781,6 +9975,14 @@ snapshots: transitivePeerDependencies: - debug + axios@1.7.2: + dependencies: + follow-redirects: 1.15.6 + form-data: 4.0.0 + proxy-from-env: 1.1.0 + transitivePeerDependencies: + - debug + axobject-query@3.2.1: dependencies: dequal: 2.0.3 @@ -10045,6 +10247,10 @@ snapshots: classnames@2.5.1: {} + cli-spinners@2.9.2: {} + + cli-width@4.1.0: {} + client-only@0.0.1: {} cliui@6.0.0: @@ -11232,6 +11438,8 @@ snapshots: graphemer@1.4.0: {} + graphql@16.8.2: {} + gtoken@6.1.2: dependencies: gaxios: 5.1.3 @@ -11285,6 +11493,8 @@ snapshots: dependencies: function-bind: 1.1.2 + headers-polyfill@4.0.3: {} + hexer@1.5.0: dependencies: ansi-color: 0.2.1 @@ -11468,6 +11678,8 @@ snapshots: is-negative-zero@2.0.3: {} + is-node-process@1.2.0: {} + is-number-object@1.0.7: dependencies: has-tostringtag: 1.0.2 @@ -11944,8 +12156,32 @@ snapshots: optionalDependencies: msgpackr-extract: 3.0.2 + msw@2.3.1(typescript@5.4.5): + dependencies: + '@bundled-es-modules/cookie': 2.0.0 + '@bundled-es-modules/statuses': 1.0.1 + '@inquirer/confirm': 3.1.9 + '@mswjs/cookies': 1.1.0 + '@mswjs/interceptors': 0.29.1 + '@open-draft/until': 2.1.0 + '@types/cookie': 0.6.0 + '@types/statuses': 2.0.5 + chalk: 4.1.2 + graphql: 16.8.2 + headers-polyfill: 4.0.3 + is-node-process: 1.2.0 + outvariant: 1.4.2 + path-to-regexp: 6.2.2 + strict-event-emitter: 0.5.1 + type-fest: 4.20.0 + yargs: 17.7.2 + optionalDependencies: + typescript: 5.4.5 + multiformats@9.9.0: {} + mute-stream@1.0.0: {} + mz@2.7.0: dependencies: any-promise: 1.3.0 @@ -12190,6 +12426,8 @@ snapshots: outdent@0.5.0: {} + outvariant@1.4.2: {} + p-cancelable@3.0.0: {} p-filter@2.1.0: @@ -12252,6 +12490,8 @@ snapshots: path-to-regexp@0.1.7: {} + path-to-regexp@6.2.2: {} + path-type@4.0.0: {} pathe@1.1.2: {} @@ -12300,6 +12540,14 @@ snapshots: postcss: 8.4.38 ts-node: 10.9.2(@types/node@18.19.31)(typescript@5.4.5) + postcss-load-config@4.0.2(postcss@8.4.38)(ts-node@10.9.2(@types/node@20.14.2)(typescript@5.4.5)): + dependencies: + lilconfig: 3.1.1 + yaml: 2.4.2 + optionalDependencies: + postcss: 8.4.38 + ts-node: 10.9.2(@types/node@20.14.2)(typescript@5.4.5) + postcss-nested@6.0.1(postcss@8.4.38): dependencies: postcss: 8.4.38 @@ -12811,6 +13059,8 @@ snapshots: signal-exit@3.0.7: {} + signal-exit@4.1.0: {} + simple-concat@1.0.1: {} simple-functional-loader@1.2.1: @@ -12916,6 +13166,8 @@ snapshots: optionalDependencies: bare-events: 2.2.2 + strict-event-emitter@0.5.1: {} + string-template@0.2.1: {} string-width@4.2.3: @@ -13074,6 +13326,33 @@ snapshots: transitivePeerDependencies: - ts-node + tailwindcss@3.4.3(ts-node@10.9.2(@types/node@20.14.2)(typescript@5.4.5)): + dependencies: + '@alloc/quick-lru': 5.2.0 + arg: 5.0.2 + chokidar: 3.5.3 + didyoumean: 1.2.2 + dlv: 1.1.3 + fast-glob: 3.3.2 + glob-parent: 6.0.2 + is-glob: 4.0.3 + jiti: 1.21.0 + lilconfig: 2.1.0 + micromatch: 4.0.5 + normalize-path: 3.0.0 + object-hash: 3.0.0 + picocolors: 1.0.0 + postcss: 8.4.38 + postcss-import: 15.1.0(postcss@8.4.38) + postcss-js: 4.0.1(postcss@8.4.38) + postcss-load-config: 4.0.2(postcss@8.4.38)(ts-node@10.9.2(@types/node@20.14.2)(typescript@5.4.5)) + postcss-nested: 6.0.1(postcss@8.4.38) + postcss-selector-parser: 6.0.13 + resolve: 1.22.8 + sucrase: 3.34.0 + transitivePeerDependencies: + - ts-node + tapable@2.2.1: {} tar-fs@2.1.1: @@ -13229,6 +13508,25 @@ snapshots: v8-compile-cache-lib: 3.0.1 yn: 3.1.1 + ts-node@10.9.2(@types/node@20.14.2)(typescript@5.4.5): + dependencies: + '@cspotcode/source-map-support': 0.8.1 + '@tsconfig/node10': 1.0.9 + '@tsconfig/node12': 1.0.11 + '@tsconfig/node14': 1.0.3 + '@tsconfig/node16': 1.0.4 + '@types/node': 20.14.2 + acorn: 8.11.3 + acorn-walk: 8.3.2 + arg: 4.1.3 + create-require: 1.1.1 + diff: 4.0.2 + make-error: 1.3.6 + typescript: 5.4.5 + v8-compile-cache-lib: 3.0.1 + yn: 3.1.1 + optional: true + tsconfig-paths@3.15.0: dependencies: '@types/json5': 0.0.29 @@ -13305,12 +13603,16 @@ snapshots: type-fest@0.20.2: {} + type-fest@0.21.3: {} + type-fest@0.6.0: {} type-fest@0.7.1: {} type-fest@0.8.1: {} + type-fest@4.20.0: {} + type-is@1.6.18: dependencies: media-typer: 0.3.0 From e74971f8cf198d33d62b41451d3dc096e66070ae Mon Sep 17 00:00:00 2001 From: PJColombo Date: Fri, 14 Jun 2024 12:48:11 +0200 Subject: [PATCH 6/8] feat(rest-api-server): add swarm stamp syncer --- .changeset/cold-terms-accept.md | 5 +++ .env.test | 2 +- apps/rest-api-server/src/env.ts | 2 +- apps/rest-api-server/src/syncers.ts | 56 ++++++++++++++++++++++------- turbo.json | 39 ++++++-------------- 5 files changed, 61 insertions(+), 43 deletions(-) create mode 100644 .changeset/cold-terms-accept.md diff --git a/.changeset/cold-terms-accept.md b/.changeset/cold-terms-accept.md new file mode 100644 index 000000000..665d9c179 --- /dev/null +++ b/.changeset/cold-terms-accept.md @@ -0,0 +1,5 @@ +--- +"@blobscan/rest-api-server": minor +--- + +Added Swarm stamp syncer diff --git a/.env.test b/.env.test index 574ea0991..161f11d1c 100644 --- a/.env.test +++ b/.env.test @@ -21,7 +21,7 @@ GOOGLE_STORAGE_ENABLED=true BEE_DEBUG_ENDPOINT=http://localhost:1635 BEE_ENDPOINT=http://localhost:1633 - +SWARM_BATCH_ID=f89e63edf757f06e89933761d6d46592d03026efb9871f9d244f34da86b6c242 FILE_SYSTEM_STORAGE_PATH=test-blobscan-blobs diff --git a/apps/rest-api-server/src/env.ts b/apps/rest-api-server/src/env.ts index b910f4591..edf8e4c3a 100644 --- a/apps/rest-api-server/src/env.ts +++ b/apps/rest-api-server/src/env.ts @@ -25,11 +25,11 @@ export const env = createEnv({ METRICS_ENABLED: booleanSchema.default("false"), REDIS_URI: z.string().default("redis://localhost:6379"), DENCUN_FORK_SLOT: z.coerce.number().optional(), + SYNCER_SWARM_STAMP_CRON_PATTERN: z.string().default("*/15 * * * *"), STATS_SYNCER_DAILY_CRON_PATTERN: z.string().default("30 0 * * * *"), STATS_SYNCER_OVERALL_CRON_PATTERN: z.string().default("*/15 * * * *"), SWARM_BATCH_ID: z.string().optional(), SWARM_STORAGE_ENABLED: booleanSchema.default("false"), - SWARM_SYNCER_CRON: z.string().default("42 * * * *"), SENTRY_DSN_API: z.string().url().optional(), }, diff --git a/apps/rest-api-server/src/syncers.ts b/apps/rest-api-server/src/syncers.ts index a6ec847c8..7ae4fda8d 100644 --- a/apps/rest-api-server/src/syncers.ts +++ b/apps/rest-api-server/src/syncers.ts @@ -1,32 +1,62 @@ /* eslint-disable @typescript-eslint/no-misused-promises */ - +import type { BaseSyncer } from "@blobscan/syncers"; import { DailyStatsSyncer, OverallStatsSyncer, + SwarmStampSyncer, createRedisConnection, } from "@blobscan/syncers"; import { env } from "./env"; +import { logger } from "./logger"; import { getNetworkDencunForkSlot } from "./utils"; export function setUpSyncers() { const connection = createRedisConnection(env.REDIS_URI); + const syncers: BaseSyncer[] = []; + + if (env.SWARM_STORAGE_ENABLED) { + if (!env.SWARM_BATCH_ID) { + logger.warn(`Swarm stamp syncer not created: no batch ID provided`); + } else if (!env.BEE_ENDPOINT) { + logger.warn("Swarm stamp syncer not created: no Bee endpoint provided"); + } else { + syncers.push( + new SwarmStampSyncer({ + cronPattern: env.SYNCER_SWARM_STAMP_CRON_PATTERN, + redisUriOrConnection: connection, + batchId: env.SWARM_BATCH_ID, + beeEndpoint: env.BEE_ENDPOINT, + }) + ); + } + } - const dailyStatsSyncer = new DailyStatsSyncer({ - cronPattern: env.STATS_SYNCER_DAILY_CRON_PATTERN, - redisUriOrConnection: connection, - }); + syncers.push( + new DailyStatsSyncer({ + cronPattern: env.STATS_SYNCER_DAILY_CRON_PATTERN, + redisUriOrConnection: connection, + }) + ); - const overallStatsSyncer = new OverallStatsSyncer({ - cronPattern: env.STATS_SYNCER_OVERALL_CRON_PATTERN, - redisUriOrConnection: connection, - lowestSlot: - env.DENCUN_FORK_SLOT ?? getNetworkDencunForkSlot(env.NETWORK_NAME), - }); + syncers.push( + new OverallStatsSyncer({ + cronPattern: env.STATS_SYNCER_OVERALL_CRON_PATTERN, + redisUriOrConnection: connection, + lowestSlot: + env.DENCUN_FORK_SLOT ?? getNetworkDencunForkSlot(env.NETWORK_NAME), + }) + ); - Promise.all([dailyStatsSyncer.start(), overallStatsSyncer.start()]); + Promise.all(syncers.map((syncer) => syncer.start())); return () => { - return dailyStatsSyncer.close().finally(() => overallStatsSyncer.close()); + let teardownPromise = Promise.resolve(); + + for (const syncer of syncers) { + teardownPromise = teardownPromise.finally(() => syncer.close()); + } + + return teardownPromise; }; } diff --git a/turbo.json b/turbo.json index af18096ec..de7ae1370 100644 --- a/turbo.json +++ b/turbo.json @@ -1,19 +1,13 @@ { "$schema": "https://turborepo.org/schema.json", - "globalDependencies": [ - "**/.env" - ], + "globalDependencies": ["**/.env"], "pipeline": { "db:generate": { - "inputs": [ - "prisma/schema.prisma" - ], + "inputs": ["prisma/schema.prisma"], "cache": false }, "push": { - "inputs": [ - "prisma/schema.prisma" - ], + "inputs": ["prisma/schema.prisma"], "cache": false }, "dev": { @@ -21,31 +15,20 @@ "cache": false }, "build": { - "dependsOn": [ - "^build", - "^db:generate" - ], - "outputs": [ - ".next/**" - ] + "dependsOn": ["^build", "^db:generate"], + "outputs": [".next/**"] }, "lint": {}, "lint:fix": {}, "svg:format": {}, "test": { - "inputs": [ - "test/**/*.test.ts" - ] + "inputs": ["test/**/*.test.ts"] }, "test:ui": { - "inputs": [ - "test/**/*.test.ts" - ] + "inputs": ["test/**/*.test.ts"] }, "test:dev": { - "inputs": [ - "test/**/*.test.ts" - ], + "inputs": ["test/**/*.test.ts"], "cache": false }, "test:setup": { @@ -55,9 +38,7 @@ "cache": false }, "type-check": { - "dependsOn": [ - "@blobscan/db#db:generate" - ], + "dependsOn": ["@blobscan/db#db:generate"], "cache": false } }, @@ -105,8 +86,10 @@ "REDIS_URI", "SECRET_KEY", "SENTRY_DSN_API", + "SYNCER_SWARM_STAMP_CRON_PATTERN", "SKIP_ENV_VALIDATION", "SWARM_STORAGE_ENABLED", + "SWARM_BATCH_ID", "VERCEL_URL", "TS_NODE" ] From 3d3299fd5d1181050d259fdcc9ce71d0a5dd6381 Mon Sep 17 00:00:00 2001 From: PJColombo Date: Fri, 14 Jun 2024 13:25:42 +0200 Subject: [PATCH 7/8] fix(rest-api-server): wait for teardown operations to finish --- apps/rest-api-server/src/index.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/apps/rest-api-server/src/index.ts b/apps/rest-api-server/src/index.ts index e2120081d..92b844e25 100644 --- a/apps/rest-api-server/src/index.ts +++ b/apps/rest-api-server/src/index.ts @@ -61,7 +61,9 @@ async function gracefulShutdown(signal: string) { logger.debug(`Received ${signal}. Shutting down...`); await apiGracefulShutdown() - .finally(() => closeSyncers()) + .finally(async () => { + await closeSyncers(); + }) .finally(() => { server.close(() => { logger.debug("Server shut down successfully"); @@ -70,7 +72,11 @@ async function gracefulShutdown(signal: string) { } // Listen for TERM signal .e.g. kill -process.on("SIGTERM", () => void gracefulShutdown("SIGTERM")); +process.on("SIGTERM", async () => { + await gracefulShutdown("SIGTERM"); +}); // Listen for INT signal e.g. Ctrl-C -process.on("SIGINT", () => void gracefulShutdown("SIGINT")); +process.on("SIGINT", async () => { + await gracefulShutdown("SIGINT"); +}); From f567c52a15df132379860dd53d5c7265718a8b51 Mon Sep 17 00:00:00 2001 From: Pablo Castellano Date: Tue, 18 Jun 2024 18:54:23 +0200 Subject: [PATCH 8/8] Some changes Rename variable SYNCER_SWARM_STAMP_CRON_PATTERN to SWARM_STAMP_CRON_PATTERN Changed logger.warn to logger.error + improve message --- apps/docs/src/app/docs/environment/page.md | 13 +++---- apps/rest-api-server/src/env.ts | 2 +- apps/rest-api-server/src/syncers.ts | 7 ++-- turbo.json | 41 ++++++++++++++++------ 4 files changed, 41 insertions(+), 22 deletions(-) diff --git a/apps/docs/src/app/docs/environment/page.md b/apps/docs/src/app/docs/environment/page.md index fb09b75b1..249f4f194 100644 --- a/apps/docs/src/app/docs/environment/page.md +++ b/apps/docs/src/app/docs/environment/page.md @@ -65,12 +65,13 @@ At the moment Postgres is the default storage and Blobscan won't be able to run **Ethereum Swarm** -| Variable | Description | Required | Default value | -| ----------------------- | -------------------------- | -------- | ------------- | -| `SWARM_BATCH_ID` | Swarm address of the stamp | No | (empty) | -| `SWARM_STORAGE_ENABLED` | Store blobs in Swarm | No | `false` | -| `BEE_ENDPOINT` | Bee endpoint | No | (empty) | -| `BEE_DEBUG_ENDPOINT` | Bee debug endpoint | No | (empty) | +| Variable | Description | Required | Default value | +| -------------------------- | -------------------------- | ------------------------------- | -------------- | +| `SWARM_STORAGE_ENABLED` | Store blobs in Swarm | No | `false` | +| `SWARM_BATCH_ID` | Swarm address of the stamp | If `SWARM_STORAGE_ENABLED=true` | (empty) | +| `SWARM_STAMP_CRON_PATTERN` | Cron pattern for swarm job | No | `*/15 * * * *` | +| `BEE_ENDPOINT` | Bee endpoint | No | (empty) | +| `BEE_DEBUG_ENDPOINT` | Bee debug endpoint | No | (empty) | ## Blob propagator diff --git a/apps/rest-api-server/src/env.ts b/apps/rest-api-server/src/env.ts index edf8e4c3a..e7a153aea 100644 --- a/apps/rest-api-server/src/env.ts +++ b/apps/rest-api-server/src/env.ts @@ -25,7 +25,7 @@ export const env = createEnv({ METRICS_ENABLED: booleanSchema.default("false"), REDIS_URI: z.string().default("redis://localhost:6379"), DENCUN_FORK_SLOT: z.coerce.number().optional(), - SYNCER_SWARM_STAMP_CRON_PATTERN: z.string().default("*/15 * * * *"), + SWARM_STAMP_CRON_PATTERN: z.string().default("*/15 * * * *"), STATS_SYNCER_DAILY_CRON_PATTERN: z.string().default("30 0 * * * *"), STATS_SYNCER_OVERALL_CRON_PATTERN: z.string().default("*/15 * * * *"), SWARM_BATCH_ID: z.string().optional(), diff --git a/apps/rest-api-server/src/syncers.ts b/apps/rest-api-server/src/syncers.ts index 7ae4fda8d..084a0e6c7 100644 --- a/apps/rest-api-server/src/syncers.ts +++ b/apps/rest-api-server/src/syncers.ts @@ -1,4 +1,3 @@ -/* eslint-disable @typescript-eslint/no-misused-promises */ import type { BaseSyncer } from "@blobscan/syncers"; import { DailyStatsSyncer, @@ -17,13 +16,13 @@ export function setUpSyncers() { if (env.SWARM_STORAGE_ENABLED) { if (!env.SWARM_BATCH_ID) { - logger.warn(`Swarm stamp syncer not created: no batch ID provided`); + logger.error(`Can't initialize Swarm stamp job: no batch ID provided`); } else if (!env.BEE_ENDPOINT) { - logger.warn("Swarm stamp syncer not created: no Bee endpoint provided"); + logger.error("Can't initialize Swarm stamp job: no Bee endpoint provided"); } else { syncers.push( new SwarmStampSyncer({ - cronPattern: env.SYNCER_SWARM_STAMP_CRON_PATTERN, + cronPattern: env.SWARM_STAMP_CRON_PATTERN, redisUriOrConnection: connection, batchId: env.SWARM_BATCH_ID, beeEndpoint: env.BEE_ENDPOINT, diff --git a/turbo.json b/turbo.json index de7ae1370..09713622a 100644 --- a/turbo.json +++ b/turbo.json @@ -1,13 +1,19 @@ { "$schema": "https://turborepo.org/schema.json", - "globalDependencies": ["**/.env"], + "globalDependencies": [ + "**/.env" + ], "pipeline": { "db:generate": { - "inputs": ["prisma/schema.prisma"], + "inputs": [ + "prisma/schema.prisma" + ], "cache": false }, "push": { - "inputs": ["prisma/schema.prisma"], + "inputs": [ + "prisma/schema.prisma" + ], "cache": false }, "dev": { @@ -15,20 +21,31 @@ "cache": false }, "build": { - "dependsOn": ["^build", "^db:generate"], - "outputs": [".next/**"] + "dependsOn": [ + "^build", + "^db:generate" + ], + "outputs": [ + ".next/**" + ] }, "lint": {}, "lint:fix": {}, "svg:format": {}, "test": { - "inputs": ["test/**/*.test.ts"] + "inputs": [ + "test/**/*.test.ts" + ] }, "test:ui": { - "inputs": ["test/**/*.test.ts"] + "inputs": [ + "test/**/*.test.ts" + ] }, "test:dev": { - "inputs": ["test/**/*.test.ts"], + "inputs": [ + "test/**/*.test.ts" + ], "cache": false }, "test:setup": { @@ -38,7 +55,9 @@ "cache": false }, "type-check": { - "dependsOn": ["@blobscan/db#db:generate"], + "dependsOn": [ + "@blobscan/db#db:generate" + ], "cache": false } }, @@ -86,11 +105,11 @@ "REDIS_URI", "SECRET_KEY", "SENTRY_DSN_API", - "SYNCER_SWARM_STAMP_CRON_PATTERN", + "SWARM_STAMP_CRON_PATTERN", "SKIP_ENV_VALIDATION", "SWARM_STORAGE_ENABLED", "SWARM_BATCH_ID", "VERCEL_URL", "TS_NODE" ] -} +} \ No newline at end of file