From 2c7143d51d1de8538c6bb56964ab77f06c45c79c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20T=C3=B3rz?= Date: Thu, 13 Jul 2023 08:49:48 +0200 Subject: [PATCH 01/19] implement clock --- packages/backend/src/core/sync/Clock.ts | 29 +++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 packages/backend/src/core/sync/Clock.ts diff --git a/packages/backend/src/core/sync/Clock.ts b/packages/backend/src/core/sync/Clock.ts new file mode 100644 index 000000000..81f0747c7 --- /dev/null +++ b/packages/backend/src/core/sync/Clock.ts @@ -0,0 +1,29 @@ +type IntervalUnit = 's' | 'm' | 'h' | 'd' +type Interval = `${number}${IntervalUnit}` + +export class Clock { + constructor() {} + + onEvery(interval: Interval, callback: () => void) { + callback() + setInterval(() => { + callback() + }, this.toMiliseconds(interval)) + } + + toMiliseconds(interval: Interval) { + const unit = interval.slice(-1) + const number = Number(interval.slice(0, -1)) + + switch (unit) { + case 's': + return number * 1000 + case 'm': + return number * 60 * 1000 + case 'h': + return number * 60 * 60 * 1000 + case 'd': + return number * 24 * 60 * 60 * 1000 + } + } +} From 3266bba7d23f8b0943b9dd9e6a253ccafedfa86d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20T=C3=B3rz?= Date: Thu, 13 Jul 2023 08:50:13 +0200 Subject: [PATCH 02/19] implement L2TransactionDownloader --- .../src/config/starkex/StarkexConfig.ts | 3 +- .../backend/src/config/starkex/apex-goerli.ts | 5 +- .../src/core/sync/L2TransactionDownloader.ts | 97 +++++++++++++++++++ .../database/L2TransactionRepository.ts | 45 ++++++--- .../src/peripherals/database/shared/types.ts | 4 +- .../starkware/L2TransactionClient.ts | 16 ++- .../starkware/toPerpetualTransactions.ts | 2 +- 7 files changed, 151 insertions(+), 21 deletions(-) create mode 100644 packages/backend/src/core/sync/L2TransactionDownloader.ts diff --git a/packages/backend/src/config/starkex/StarkexConfig.ts b/packages/backend/src/config/starkex/StarkexConfig.ts index 254d21a98..490c9e7fc 100644 --- a/packages/backend/src/config/starkex/StarkexConfig.ts +++ b/packages/backend/src/config/starkex/StarkexConfig.ts @@ -83,6 +83,7 @@ export interface GatewayConfig { } export interface L2TransactionApiConfig { - getUrl: (startApexId: number, expectCount: number) => string + getTransactionsUrl: (startApexId: number, expectCount: number) => string + getThirdPartyIdByTransactionIdUrl: (transactionId: number) => string auth: ClientAuth } diff --git a/packages/backend/src/config/starkex/apex-goerli.ts b/packages/backend/src/config/starkex/apex-goerli.ts index 27b62fa7d..31e96bd23 100644 --- a/packages/backend/src/config/starkex/apex-goerli.ts +++ b/packages/backend/src/config/starkex/apex-goerli.ts @@ -37,11 +37,14 @@ export function getApexGoerliConfig(): StarkexConfig { auth: clientAuth, }, l2TransactionApi: { - getUrl: (startId, expectCount) => { + getTransactionsUrl: (startId, expectCount) => { return `${getEnv( 'APEX_TRANSACTION_API_URL' )}?startApexId=${startId}&expectCount=${expectCount}` }, + getThirdPartyIdByTransactionIdUrl: (transactionId: number) => { + return `${getEnv('APEX_THIRD_PARTY_ID_API_URL')}?txId=${transactionId}` + }, auth: clientAuth, }, collateralAsset: { diff --git a/packages/backend/src/core/sync/L2TransactionDownloader.ts b/packages/backend/src/core/sync/L2TransactionDownloader.ts new file mode 100644 index 000000000..24d4f854a --- /dev/null +++ b/packages/backend/src/core/sync/L2TransactionDownloader.ts @@ -0,0 +1,97 @@ +import { KeyValueStore } from '../../peripherals/database/KeyValueStore' +import { L2TransactionRepository } from '../../peripherals/database/L2TransactionRepository' +import { L2TransactionClient } from '../../peripherals/starkware/L2TransactionClient' +import { Logger } from '../../tools/Logger' +import { Clock } from './Clock' + +export class L2TransactionDownloader { + private clock = new Clock() + private PAGE_SIZE = 100 + private isRunning = false + private lastSyncedThirdPartyId: number | undefined + + constructor( + private readonly l2TransactionClient: L2TransactionClient, + private readonly l2TransactionRepository: L2TransactionRepository, + private readonly keyValueStore: KeyValueStore, + private readonly logger: Logger + ) { + this.logger = this.logger.for(this) + } + + start() { + this.clock.onEvery('15s', () => this.downloadNewTransactions()) + } + + async downloadNewTransactions() { + if (this.isRunning) { + return + } + this.isRunning = true + + this.logger.info('Starting L2 transaction downloader') + const lastIncluded = await this.l2TransactionRepository.findLatestIncluded() + if (!lastIncluded) { + this.isRunning = false + return + } + + this.lastSyncedThirdPartyId = Number( + await this.keyValueStore.findByKey('lastSyncedThirdPartyId') + ) + + let thirdPartyIdToSync = this.lastSyncedThirdPartyId + ? this.lastSyncedThirdPartyId + : await this.l2TransactionClient.getThirdPartyIdByTransactionId( + lastIncluded.transactionId + ) + + if (!thirdPartyIdToSync) { + this.isRunning = false + return + } + + while (true) { + this.logger.info(thirdPartyIdToSync.toString()) + const transactions = await this.addTransactions(thirdPartyIdToSync) + + if (!transactions) { + break + } + this.lastSyncedThirdPartyId = + transactions[transactions.length - 1]?.thirdPartyId + + thirdPartyIdToSync += this.PAGE_SIZE + } + + if (this.lastSyncedThirdPartyId) { + await this.keyValueStore.addOrUpdate({ + key: 'lastSyncedThirdPartyId', + value: this.lastSyncedThirdPartyId.toString(), + }) + } + this.isRunning = false + } + + private async addTransactions(thirdPartyId: number) { + this.logger.info(`Downloading transactions from ${thirdPartyId}`) + const transactions = + await this.l2TransactionClient.getPerpetualTransactions( + thirdPartyId, + this.PAGE_SIZE + ) + + if (!transactions) { + this.logger.info('No transactions found') + return + } + + for (const transaction of transactions) { + await this.l2TransactionRepository.add({ + transactionId: transaction.transactionId, + data: transaction.transaction, + }) + } + return transactions + } +} diff --git a/packages/backend/src/peripherals/database/L2TransactionRepository.ts b/packages/backend/src/peripherals/database/L2TransactionRepository.ts index cccb27a7f..6b1954b62 100644 --- a/packages/backend/src/peripherals/database/L2TransactionRepository.ts +++ b/packages/backend/src/peripherals/database/L2TransactionRepository.ts @@ -20,8 +20,8 @@ interface Record< > { id: number transactionId: number - stateUpdateId: number - blockNumber: number + stateUpdateId: number | undefined + blockNumber: number | undefined parentId: number | undefined state: 'alternative' | 'replaced' | undefined starkKeyA: StarkKey | undefined @@ -32,8 +32,8 @@ interface Record< interface AggregatedRecord { id: number transactionId: number - stateUpdateId: number - blockNumber: number + stateUpdateId: number | undefined + blockNumber: number | undefined originalTransaction: PerpetualL2TransactionData alternativeTransactions: PerpetualL2TransactionData[] } @@ -65,9 +65,9 @@ export class L2TransactionRepository extends BaseRepository { async add(record: { transactionId: number - stateUpdateId: number - blockNumber: number data: PerpetualL2TransactionData + stateUpdateId?: number + blockNumber?: number }): Promise { const knex = await this.knex() @@ -96,11 +96,11 @@ export class L2TransactionRepository extends BaseRepository { private async addSingleTransaction( record: { transactionId: number - stateUpdateId: number - blockNumber: number isAlternative: boolean data: Exclude parentId?: number + stateUpdateId?: number + blockNumber?: number }, knex: Knex ) { @@ -126,10 +126,10 @@ export class L2TransactionRepository extends BaseRepository { private async addMultiTransaction( record: { transactionId: number - stateUpdateId: number - blockNumber: number isAlternative: boolean data: PerpetualL2MultiTransactionData + stateUpdateId?: number + blockNumber?: number }, knex: Knex ) { @@ -291,7 +291,18 @@ export class L2TransactionRepository extends BaseRepository { .orderBy('state_update_id', 'desc') .limit(1) .first() - return results?.state_update_id + return results?.state_update_id ? results.state_update_id : undefined + } + + async findLatestIncluded(): Promise { + const knex = await this.knex() + const row = await knex('l2_transactions') + .whereNotNull('state_update_id') + .orderBy('transaction_id', 'desc') + .limit(1) + .first() + + return row ? toRecord(row) : undefined } async deleteAfterBlock(blockNumber: number) { @@ -311,8 +322,8 @@ function toRecord(row: L2TransactionRow): Record { return { id: row.id, transactionId: row.transaction_id, - stateUpdateId: row.state_update_id, - blockNumber: row.block_number, + stateUpdateId: row.state_update_id ? row.state_update_id : undefined, + blockNumber: row.block_number ? row.block_number : undefined, parentId: row.parent_id ? row.parent_id : undefined, state: row.state ? row.state : undefined, starkKeyA: row.stark_key_a ? StarkKey(row.stark_key_a) : undefined, @@ -328,8 +339,12 @@ function toAggregatedRecord( return { id: transaction.id, transactionId: transaction.transaction_id, - stateUpdateId: transaction.state_update_id, - blockNumber: transaction.block_number, + stateUpdateId: transaction.state_update_id + ? transaction.state_update_id + : undefined, + blockNumber: transaction.block_number + ? transaction.block_number + : undefined, originalTransaction: decodeTransactionData(transaction.data), alternativeTransactions: alternatives.map((alternative) => decodeTransactionData(alternative.data) diff --git a/packages/backend/src/peripherals/database/shared/types.ts b/packages/backend/src/peripherals/database/shared/types.ts index 470590b4d..429c3df51 100644 --- a/packages/backend/src/peripherals/database/shared/types.ts +++ b/packages/backend/src/peripherals/database/shared/types.ts @@ -275,8 +275,8 @@ declare module 'knex/types/tables' { interface L2TransactionRow { id: number transaction_id: number - state_update_id: number - block_number: number + state_update_id: number | null + block_number: number | null parent_id: number | null state: 'alternative' | 'replaced' | null stark_key_a: string | null diff --git a/packages/backend/src/peripherals/starkware/L2TransactionClient.ts b/packages/backend/src/peripherals/starkware/L2TransactionClient.ts index 8c85ca5f4..7750fbbff 100644 --- a/packages/backend/src/peripherals/starkware/L2TransactionClient.ts +++ b/packages/backend/src/peripherals/starkware/L2TransactionClient.ts @@ -12,8 +12,22 @@ export class L2TransactionClient extends BaseClient { super(options.auth) } + async getThirdPartyIdByTransactionId(transactionId: number) { + const url = this.options.getThirdPartyIdByTransactionIdUrl(transactionId) + + const res = await this.fetchClient.fetch(url, this.requestInit) + const text = await res.text() + const thirdPartyId = Number(text) + // thirdPartyId is equal 0 if the transaction is not found + return thirdPartyId !== 0 ? thirdPartyId : undefined + } + async getPerpetualTransactions(startId: number, pageSize: number) { const data = await this.getTransactions(startId, pageSize) + //TODO: remove this + if (JSON.stringify(data) === '{}') { + return undefined + } const parsed = PerpetualL2TransactionResponse.parse(data) return toPerpetualL2Transactions(parsed) } @@ -22,7 +36,7 @@ export class L2TransactionClient extends BaseClient { startId: number, pageSize: number ): Promise { - const url = this.options.getUrl(startId, pageSize) + const url = this.options.getTransactionsUrl(startId, pageSize) const res = await this.fetchClient.fetchRetry(url, this.requestInit) return res.json() diff --git a/packages/backend/src/peripherals/starkware/toPerpetualTransactions.ts b/packages/backend/src/peripherals/starkware/toPerpetualTransactions.ts index 5963c777b..8521fd3f1 100644 --- a/packages/backend/src/peripherals/starkware/toPerpetualTransactions.ts +++ b/packages/backend/src/peripherals/starkware/toPerpetualTransactions.ts @@ -15,9 +15,9 @@ import { import { AssetOraclePrice, OrderTypeResponse, - PerpetualL2Transaction as TransactionSchema, SignatureResponse, SignedOraclePrice, + PerpetualL2Transaction as TransactionSchema, } from './schema/PerpetualBatchInfoResponse' import { PerpetualL2TransactionResponse } from './schema/PerpetualL2TransactionResponse' export interface PerpetualL2Transaction { From 73afa8d8b3003bac24bbfb339453c633bd414a85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20T=C3=B3rz?= Date: Thu, 13 Jul 2023 08:51:00 +0200 Subject: [PATCH 03/19] run adding l2 transaction in transaction with locked table --- .../core/collectors/FeederGatewayCollector.ts | 48 ++++++++++++------- .../src/core/sync/L2TransactionDownloader.ts | 16 ++++--- .../database/L2TransactionRepository.ts | 41 +++++++++++++--- 3 files changed, 74 insertions(+), 31 deletions(-) diff --git a/packages/backend/src/core/collectors/FeederGatewayCollector.ts b/packages/backend/src/core/collectors/FeederGatewayCollector.ts index 9f4019d0e..dea7c8752 100644 --- a/packages/backend/src/core/collectors/FeederGatewayCollector.ts +++ b/packages/backend/src/core/collectors/FeederGatewayCollector.ts @@ -51,25 +51,37 @@ export class FeederGatewayCollector { return } - for (const transactionInfo of data.transactionsInfo) { - await this.l2TransactionRepository.add({ - stateUpdateId: stateUpdate.id, - blockNumber: stateUpdate.blockNumber, - transactionId: transactionInfo.originalTransactionId, - data: transactionInfo.originalTransaction, - }) - if (!transactionInfo.alternativeTransactions) { - continue - } - for (const alternativeTransaction of transactionInfo.alternativeTransactions) { - await this.l2TransactionRepository.add({ - stateUpdateId: stateUpdate.id, - blockNumber: stateUpdate.blockNumber, - transactionId: transactionInfo.originalTransactionId, - data: alternativeTransaction, - }) + const transactionIds = data.transactionsInfo.map( + (tx) => tx.originalTransactionId + ) + + await this.l2TransactionRepository.runInTransactionWithLockedTable( + async (trx) => { + await this.l2TransactionRepository.deleteByTransactionId( + trx, + ...transactionIds + ) + for (const transactionInfo of data.transactionsInfo) { + await this.l2TransactionRepository.add(trx, { + stateUpdateId: stateUpdate.id, + blockNumber: stateUpdate.blockNumber, + transactionId: transactionInfo.originalTransactionId, + data: transactionInfo.originalTransaction, + }) + if (!transactionInfo.alternativeTransactions) { + continue + } + for (const alternativeTransaction of transactionInfo.alternativeTransactions) { + await this.l2TransactionRepository.add(trx, { + stateUpdateId: stateUpdate.id, + blockNumber: stateUpdate.blockNumber, + transactionId: transactionInfo.originalTransactionId, + data: alternativeTransaction, + }) + } + } } - } + ) } } diff --git a/packages/backend/src/core/sync/L2TransactionDownloader.ts b/packages/backend/src/core/sync/L2TransactionDownloader.ts index 24d4f854a..f061bf20f 100644 --- a/packages/backend/src/core/sync/L2TransactionDownloader.ts +++ b/packages/backend/src/core/sync/L2TransactionDownloader.ts @@ -86,12 +86,16 @@ export class L2TransactionDownloader { return } - for (const transaction of transactions) { - await this.l2TransactionRepository.add({ - transactionId: transaction.transactionId, - data: transaction.transaction, - }) - } + await this.l2TransactionRepository.runInTransactionWithLockedTable( + async (trx) => { + for (const transaction of transactions) { + await this.l2TransactionRepository.add(trx, { + transactionId: transaction.transactionId, + data: transaction.transaction, + }) + } + } + ) return transactions } } diff --git a/packages/backend/src/peripherals/database/L2TransactionRepository.ts b/packages/backend/src/peripherals/database/L2TransactionRepository.ts index 6b1954b62..bb5726623 100644 --- a/packages/backend/src/peripherals/database/L2TransactionRepository.ts +++ b/packages/backend/src/peripherals/database/L2TransactionRepository.ts @@ -63,13 +63,16 @@ export class L2TransactionRepository extends BaseRepository { /* eslint-enable @typescript-eslint/unbound-method */ } - async add(record: { - transactionId: number - data: PerpetualL2TransactionData - stateUpdateId?: number - blockNumber?: number - }): Promise { - const knex = await this.knex() + async add( + trx: Knex.Transaction, + record: { + transactionId: number + data: PerpetualL2TransactionData + stateUpdateId?: number + blockNumber?: number + } + ): Promise { + const knex = await this.knex(trx) const count = await this.countByTransactionId(record.transactionId) const isAlternative = count > 0 @@ -312,10 +315,34 @@ export class L2TransactionRepository extends BaseRepository { .delete() } + async deleteByTransactionId( + trx?: Knex.Transaction, + ...transactionIds: number[] + ) { + const knex = await this.knex(trx) + return knex('l2_transactions') + .whereIn('transaction_id', transactionIds) + .delete() + } + async deleteAll() { const knex = await this.knex() return knex('l2_transactions').delete() } + + async runInTransactionWithLockedTable( + fun: Parameters[0] + ) { + await this.runInTransaction(async (trx) => { + await this.lockTable(trx) + await fun(trx) + }) + } + + private async lockTable(trx: Knex.Transaction) { + const knex = await this.knex(trx) + await knex.raw('LOCK TABLE l2_transactions IN ROW EXCLUSIVE MODE;') + } } function toRecord(row: L2TransactionRow): Record { From 35108024b6c54a01b090ea3b2fa840388ebac64c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20T=C3=B3rz?= Date: Thu, 13 Jul 2023 08:51:09 +0200 Subject: [PATCH 04/19] add tests --- .../collectors/FeederGatewayCollector.test.ts | 67 ++++++++++-- .../core/collectors/FeederGatewayCollector.ts | 43 ++++---- .../src/core/sync/L2TransactionDownloader.ts | 11 +- .../database/L2TransactionRepository.test.ts | 75 ++++++++++++- .../database/L2TransactionRepository.ts | 20 ++-- .../starkware/L2TransactionClient.test.ts | 102 +++++++++++++++--- .../starkware/L2TransactionClient.ts | 10 +- 7 files changed, 265 insertions(+), 63 deletions(-) diff --git a/packages/backend/src/core/collectors/FeederGatewayCollector.test.ts b/packages/backend/src/core/collectors/FeederGatewayCollector.test.ts index f202d4e40..6a32d2514 100644 --- a/packages/backend/src/core/collectors/FeederGatewayCollector.test.ts +++ b/packages/backend/src/core/collectors/FeederGatewayCollector.test.ts @@ -1,5 +1,6 @@ import { EthereumAddress, Hash256, StarkKey, Timestamp } from '@explorer/types' import { expect, mockFn, mockObject } from 'earl' +import { Knex } from 'knex' import { L2TransactionRepository } from '../../peripherals/database/L2TransactionRepository' import { @@ -21,9 +22,16 @@ describe(FeederGatewayCollector.name, () => { } as PerpetualBatchInfo }), }) + const knexTransaction = mockObject({}) const mockL2TransactionRepository = mockObject({ findLatestStateUpdateId: mockFn().resolvesTo(undefined), add: mockFn().resolvesTo(1), + runInTransactionWithLockedTable: mockFn( + async (fun: (trx: Knex.Transaction) => Promise) => { + await fun(knexTransaction) + } + ), + deleteByTransactionIds: mockFn().resolvesTo(undefined), }) const mockStateUpdateRepository = mockObject({ findById: mockFn(async (id: number) => fakeStateUpdateRecord(id)), @@ -38,13 +46,23 @@ describe(FeederGatewayCollector.name, () => { await feederGatewayCollector.collect() + expect(mockStateUpdateRepository.findLast).toHaveBeenCalledTimes(1) expect( mockL2TransactionRepository.findLatestStateUpdateId ).toHaveBeenCalledTimes(1) - + expect( + mockL2TransactionRepository.runInTransactionWithLockedTable + ).toHaveBeenCalledTimes(5) for (const i of [1, 2, 3, 4, 5]) { expect(mockStateUpdateRepository.findById).toHaveBeenNthCalledWith(i, i) const stateUpdate = fakeStateUpdateRecord(i) + expect( + mockL2TransactionRepository.deleteByTransactionIds + ).toHaveBeenNthCalledWith( + i, + [(i - 1) * 2 + 1, (i - 1) * 2 + 2], + knexTransaction + ) expect( mockFeederGatewayClient.getPerpetualBatchInfo ).toHaveBeenNthCalledWith(i, stateUpdate.batchId) @@ -56,7 +74,8 @@ describe(FeederGatewayCollector.name, () => { blockNumber: stateUpdate.blockNumber, transactionId: transactionsInfo[0]!.originalTransactionId, data: transactionsInfo[0]!.originalTransaction, - } + }, + knexTransaction ) expect(mockL2TransactionRepository.add).toHaveBeenNthCalledWith( 4 * (i - 1) + 2, @@ -65,7 +84,8 @@ describe(FeederGatewayCollector.name, () => { blockNumber: stateUpdate.blockNumber, transactionId: transactionsInfo[1]!.originalTransactionId, data: transactionsInfo[1]!.originalTransaction, - } + }, + knexTransaction ) expect(mockL2TransactionRepository.add).toHaveBeenNthCalledWith( 4 * (i - 1) + 3, @@ -74,7 +94,8 @@ describe(FeederGatewayCollector.name, () => { blockNumber: stateUpdate.blockNumber, transactionId: transactionsInfo[1]!.originalTransactionId, data: transactionsInfo[1]!.alternativeTransactions![0]!, - } + }, + knexTransaction ) expect(mockL2TransactionRepository.add).toHaveBeenNthCalledWith( 4 * (i - 1) + 4, @@ -83,7 +104,8 @@ describe(FeederGatewayCollector.name, () => { blockNumber: stateUpdate.blockNumber, transactionId: transactionsInfo[1]!.originalTransactionId, data: transactionsInfo[1]!.alternativeTransactions![1]!, - } + }, + knexTransaction ) } expect( @@ -101,9 +123,16 @@ describe(FeederGatewayCollector.name, () => { } as PerpetualBatchInfo }), }) + const knexTransaction = mockObject({}) const mockL2TransactionRepository = mockObject({ findLatestStateUpdateId: mockFn().resolvesTo(6), add: mockFn().resolvesTo(1), + runInTransactionWithLockedTable: mockFn( + async (fun: (trx: Knex.Transaction) => Promise) => { + await fun(knexTransaction) + } + ), + deleteByTransactionIds: mockFn().resolvesTo(0), }) const mockStateUpdateRepository = mockObject({ findById: mockFn(async (id: number) => fakeStateUpdateRecord(id)), @@ -118,16 +147,28 @@ describe(FeederGatewayCollector.name, () => { await feederGatewayCollector.collect() + expect(mockStateUpdateRepository.findLast).toHaveBeenCalledTimes(1) expect( mockL2TransactionRepository.findLatestStateUpdateId ).toHaveBeenCalledTimes(1) + expect( + mockL2TransactionRepository.runInTransactionWithLockedTable + ).toHaveBeenCalledTimes(4) + for (const i of [7, 8, 9, 10]) { expect(mockStateUpdateRepository.findById).toHaveBeenNthCalledWith( i - 6, i ) const stateUpdate = fakeStateUpdateRecord(i) + expect( + mockL2TransactionRepository.deleteByTransactionIds + ).toHaveBeenNthCalledWith( + i - 6, + [(i - 1) * 2 + 1, (i - 1) * 2 + 2], + knexTransaction + ) expect( mockFeederGatewayClient.getPerpetualBatchInfo ).toHaveBeenNthCalledWith(i - 6, stateUpdate.batchId) @@ -139,7 +180,8 @@ describe(FeederGatewayCollector.name, () => { blockNumber: stateUpdate.blockNumber, transactionId: transactionsInfo[0]!.originalTransactionId, data: transactionsInfo[0]!.originalTransaction, - } + }, + knexTransaction ) expect(mockL2TransactionRepository.add).toHaveBeenNthCalledWith( 4 * (i - 7) + 2, @@ -148,7 +190,8 @@ describe(FeederGatewayCollector.name, () => { blockNumber: stateUpdate.blockNumber, transactionId: transactionsInfo[1]!.originalTransactionId, data: transactionsInfo[1]!.originalTransaction, - } + }, + knexTransaction ) expect(mockL2TransactionRepository.add).toHaveBeenNthCalledWith( 4 * (i - 7) + 3, @@ -157,7 +200,8 @@ describe(FeederGatewayCollector.name, () => { blockNumber: stateUpdate.blockNumber, transactionId: transactionsInfo[1]!.originalTransactionId, data: transactionsInfo[1]!.alternativeTransactions![0]!, - } + }, + knexTransaction ) expect(mockL2TransactionRepository.add).toHaveBeenNthCalledWith( 4 * (i - 7) + 4, @@ -166,7 +210,8 @@ describe(FeederGatewayCollector.name, () => { blockNumber: stateUpdate.blockNumber, transactionId: transactionsInfo[1]!.originalTransactionId, data: transactionsInfo[1]!.alternativeTransactions![1]!, - } + }, + knexTransaction ) } expect( @@ -282,7 +327,7 @@ const fakeTransactionsInfo = ( return [ { wasReplaced: false, - originalTransactionId: batchId + 1, + originalTransactionId: 2 * batchId + 1, originalTransaction: { type: 'Deposit', starkKey: StarkKey.fake(`1${batchId}`), @@ -292,7 +337,7 @@ const fakeTransactionsInfo = ( }, { wasReplaced: false, - originalTransactionId: 2, + originalTransactionId: 2 * batchId + 2, originalTransaction: { positionId: 1234n, starkKey: StarkKey.fake(`2${batchId}`), diff --git a/packages/backend/src/core/collectors/FeederGatewayCollector.ts b/packages/backend/src/core/collectors/FeederGatewayCollector.ts index dea7c8752..cb111bda5 100644 --- a/packages/backend/src/core/collectors/FeederGatewayCollector.ts +++ b/packages/backend/src/core/collectors/FeederGatewayCollector.ts @@ -51,33 +51,38 @@ export class FeederGatewayCollector { return } - const transactionIds = data.transactionsInfo.map( - (tx) => tx.originalTransactionId - ) - await this.l2TransactionRepository.runInTransactionWithLockedTable( async (trx) => { - await this.l2TransactionRepository.deleteByTransactionId( - trx, - ...transactionIds + const transactionIds = data.transactionsInfo.map( + (tx) => tx.originalTransactionId + ) + await this.l2TransactionRepository.deleteByTransactionIds( + transactionIds, + trx ) for (const transactionInfo of data.transactionsInfo) { - await this.l2TransactionRepository.add(trx, { - stateUpdateId: stateUpdate.id, - blockNumber: stateUpdate.blockNumber, - transactionId: transactionInfo.originalTransactionId, - data: transactionInfo.originalTransaction, - }) + await this.l2TransactionRepository.add( + { + stateUpdateId: stateUpdate.id, + blockNumber: stateUpdate.blockNumber, + transactionId: transactionInfo.originalTransactionId, + data: transactionInfo.originalTransaction, + }, + trx + ) if (!transactionInfo.alternativeTransactions) { continue } for (const alternativeTransaction of transactionInfo.alternativeTransactions) { - await this.l2TransactionRepository.add(trx, { - stateUpdateId: stateUpdate.id, - blockNumber: stateUpdate.blockNumber, - transactionId: transactionInfo.originalTransactionId, - data: alternativeTransaction, - }) + await this.l2TransactionRepository.add( + { + stateUpdateId: stateUpdate.id, + blockNumber: stateUpdate.blockNumber, + transactionId: transactionInfo.originalTransactionId, + data: alternativeTransaction, + }, + trx + ) } } } diff --git a/packages/backend/src/core/sync/L2TransactionDownloader.ts b/packages/backend/src/core/sync/L2TransactionDownloader.ts index f061bf20f..e797bdb3e 100644 --- a/packages/backend/src/core/sync/L2TransactionDownloader.ts +++ b/packages/backend/src/core/sync/L2TransactionDownloader.ts @@ -89,10 +89,13 @@ export class L2TransactionDownloader { await this.l2TransactionRepository.runInTransactionWithLockedTable( async (trx) => { for (const transaction of transactions) { - await this.l2TransactionRepository.add(trx, { - transactionId: transaction.transactionId, - data: transaction.transaction, - }) + await this.l2TransactionRepository.add( + { + transactionId: transaction.transactionId, + data: transaction.transaction, + }, + trx + ) } } ) diff --git a/packages/backend/src/peripherals/database/L2TransactionRepository.test.ts b/packages/backend/src/peripherals/database/L2TransactionRepository.test.ts index d835db50f..c1ed7ae25 100644 --- a/packages/backend/src/peripherals/database/L2TransactionRepository.test.ts +++ b/packages/backend/src/peripherals/database/L2TransactionRepository.test.ts @@ -7,7 +7,8 @@ import { StarkKey, Timestamp, } from '@explorer/types' -import { expect } from 'earl' +import { expect, mockFn } from 'earl' +import { Knex } from 'knex' import { beforeEach, it } from 'mocha' import { setupDatabaseTestSuite } from '../../test/database' @@ -1330,4 +1331,76 @@ describe(L2TransactionRepository.name, () => { expect(deletedTransaction).toBeNullish() }) }) + + describe( + L2TransactionRepository.prototype.deleteByTransactionIds.name, + () => { + const record = { + stateUpdateId: 1, + transactionId: 1234, + blockNumber: 12345, + data: { + type: 'Deposit', + starkKey: StarkKey.fake(), + positionId: 1234n, + amount: 5000n, + }, + } as const + const recordToBeDeleted = { + stateUpdateId: 2, + transactionId: 12345, + blockNumber: 123456, + data: { + type: 'Deposit', + starkKey: StarkKey.fake(), + positionId: 1234n, + amount: 5000n, + }, + } as const + const recordToBeDeleted2 = { + stateUpdateId: 3, + transactionId: 123453, + blockNumber: 1234565, + data: { + type: 'Deposit', + starkKey: StarkKey.fake(), + positionId: 1234n, + amount: 5000n, + }, + } as const + + it('deletes a transaction by transaction id', async () => { + const id = await repository.add(record) + const deletedId = await repository.add(recordToBeDeleted) + const deletedId2 = await repository.add(recordToBeDeleted2) + await repository.deleteByTransactionIds([ + recordToBeDeleted.transactionId, + recordToBeDeleted2.transactionId, + ]) + + const transaction = await repository.findById(id) + const deletedTransaction = await repository.findById(deletedId) + const deletedTransaction2 = await repository.findById(deletedId2) + + expect(transaction).not.toBeNullish() + expect(deletedTransaction).toBeNullish() + expect(deletedTransaction2).toBeNullish() + }) + } + ) + + describe( + L2TransactionRepository.prototype.runInTransactionWithLockedTable.name, + () => { + it('runs a function in a transaction with a locked table', async () => { + const mockedLockTableFn = mockFn(async () => {}) + repository.lockTable = mockedLockTableFn + const fn = mockFn(async (trx: Knex.Transaction) => {}) + await repository.runInTransactionWithLockedTable(fn) + + expect(mockedLockTableFn).toHaveBeenCalledTimes(1) + expect(fn).toHaveBeenCalledTimes(1) + }) + } + ) }) diff --git a/packages/backend/src/peripherals/database/L2TransactionRepository.ts b/packages/backend/src/peripherals/database/L2TransactionRepository.ts index bb5726623..99afe1362 100644 --- a/packages/backend/src/peripherals/database/L2TransactionRepository.ts +++ b/packages/backend/src/peripherals/database/L2TransactionRepository.ts @@ -59,18 +59,22 @@ export class L2TransactionRepository extends BaseRepository { this.findByTransactionId = this.wrapFind(this.findByTransactionId) this.findLatestStateUpdateId = this.wrapFind(this.findLatestStateUpdateId) this.deleteAfterBlock = this.wrapDelete(this.deleteAfterBlock) + this.deleteByTransactionIds = this.wrapDelete(this.deleteByTransactionIds) this.deleteAll = this.wrapDelete(this.deleteAll) + this.runInTransactionWithLockedTable = this.wrapAny( + this.runInTransactionWithLockedTable + ) /* eslint-enable @typescript-eslint/unbound-method */ } async add( - trx: Knex.Transaction, record: { transactionId: number data: PerpetualL2TransactionData stateUpdateId?: number blockNumber?: number - } + }, + trx?: Knex.Transaction ): Promise { const knex = await this.knex(trx) @@ -315,9 +319,9 @@ export class L2TransactionRepository extends BaseRepository { .delete() } - async deleteByTransactionId( - trx?: Knex.Transaction, - ...transactionIds: number[] + async deleteByTransactionIds( + transactionIds: number[], + trx?: Knex.Transaction ) { const knex = await this.knex(trx) return knex('l2_transactions') @@ -331,15 +335,15 @@ export class L2TransactionRepository extends BaseRepository { } async runInTransactionWithLockedTable( - fun: Parameters[0] + fn: (trx: Knex.Transaction) => Promise ) { await this.runInTransaction(async (trx) => { await this.lockTable(trx) - await fun(trx) + await fn(trx) }) } - private async lockTable(trx: Knex.Transaction) { + async lockTable(trx: Knex.Transaction) { const knex = await this.knex(trx) await knex.raw('LOCK TABLE l2_transactions IN ROW EXCLUSIVE MODE;') } diff --git a/packages/backend/src/peripherals/starkware/L2TransactionClient.test.ts b/packages/backend/src/peripherals/starkware/L2TransactionClient.test.ts index 585cf0902..df7cc02c3 100644 --- a/packages/backend/src/peripherals/starkware/L2TransactionClient.test.ts +++ b/packages/backend/src/peripherals/starkware/L2TransactionClient.test.ts @@ -1,6 +1,6 @@ import { expect, mockFn, mockObject } from 'earl' -import { GatewayConfig } from '../../config/starkex/StarkexConfig' +import { L2TransactionApiConfig } from '../../config/starkex/StarkexConfig' import { EXAMPLE_PERPETUAL_TRANSACTIONS } from '../../test/starkwareData' import { FetchClient } from './FetchClient' import { L2TransactionClient } from './L2TransactionClient' @@ -8,38 +8,108 @@ import { PerpetualL2TransactionResponse } from './schema/PerpetualL2TransactionR import { toPerpetualL2Transactions } from './toPerpetualTransactions' describe(L2TransactionClient.name, () => { - const getUrl = mockFn().returns('gateway-url') - const options: GatewayConfig = mockObject({ - getUrl, + const getTransactionsUrl = mockFn().returns('get-transactions-url') + const getThirdPartyIdByTransactionIdUrl = mockFn().returns( + 'get-third-party-id-by-transaction-id-url' + ) + const options: L2TransactionApiConfig = { + getTransactionsUrl, + getThirdPartyIdByTransactionIdUrl, auth: { type: 'bearerToken', bearerToken: 'random-token', }, - }) + } describe(L2TransactionClient.prototype.getPerpetualTransactions.name, () => { - const fetchClient = mockObject({ - fetchRetry: mockFn().resolvesTo({ - json: mockFn().resolvesTo(EXAMPLE_PERPETUAL_TRANSACTIONS), - }), - }) - const transactionClient = new L2TransactionClient(options, fetchClient) - it('should fetch transactions and parse them', async () => { + const fetchClient = mockObject({ + fetchRetry: mockFn().resolvesTo({ + json: mockFn().resolvesTo(EXAMPLE_PERPETUAL_TRANSACTIONS), + }), + }) + const transactionClient = new L2TransactionClient(options, fetchClient) + const response = await transactionClient.getPerpetualTransactions(0, 0) - expect(getUrl).toHaveBeenCalledWith(0, 0) + expect(getTransactionsUrl).toHaveBeenCalledWith(0, 0) expect(fetchClient.fetchRetry).toHaveBeenCalledWith( - 'gateway-url', + 'get-transactions-url', expect.anything() ) - expect(fetchClient.fetchRetry).toHaveBeenExhausted() - expect(getUrl).toHaveBeenExhausted() expect(response).toEqual( toPerpetualL2Transactions( PerpetualL2TransactionResponse.parse(EXAMPLE_PERPETUAL_TRANSACTIONS) ) ) }) + + it('should return undefined if no transactions are returned', async () => { + const fetchClient = mockObject({ + fetchRetry: mockFn().resolvesTo({ + json: mockFn().resolvesTo({}), + }), + }) + const transactionClient = new L2TransactionClient(options, fetchClient) + + const response = await transactionClient.getPerpetualTransactions(0, 0) + + expect(getTransactionsUrl).toHaveBeenCalledWith(0, 0) + expect(fetchClient.fetchRetry).toHaveBeenCalledWith( + 'get-transactions-url', + expect.anything() + ) + expect(response).toEqual(undefined) + }) }) + + describe( + L2TransactionClient.prototype.getThirdPartyIdByTransactionId.name, + () => { + it('should fetch third party id', async () => { + const textResponse = '2' + const fetchClient = mockObject({ + fetchRetry: mockFn().resolvesTo({ + text: mockFn().resolvesTo(textResponse), + }), + }) + const transactionClient = new L2TransactionClient(options, fetchClient) + + const response = await transactionClient.getThirdPartyIdByTransactionId( + 1 + ) + + expect(getThirdPartyIdByTransactionIdUrl).toHaveBeenCalledWith(1) + expect(fetchClient.fetchRetry).toHaveBeenCalledWith( + 'get-third-party-id-by-transaction-id-url', + expect.anything() + ) + expect(response).toEqual(Number(textResponse)) + }) + + it('should return undefined if 0 is returned', async () => { + it('should fetch third party id', async () => { + const fetchClient = mockObject({ + fetchRetry: mockFn().resolvesTo({ + text: mockFn().resolvesTo('0'), + }), + }) + const transactionClient = new L2TransactionClient( + options, + fetchClient + ) + + const response = + await transactionClient.getThirdPartyIdByTransactionId(1) + + expect(getThirdPartyIdByTransactionIdUrl).toHaveBeenCalledWith(1) + expect(fetchClient.fetchRetry).toHaveBeenCalledWith( + 'get-third-party-id-by-transaction-id-url', + expect.anything() + ) + expect(response).toEqual(undefined) + }) + }) + } + ) }) diff --git a/packages/backend/src/peripherals/starkware/L2TransactionClient.ts b/packages/backend/src/peripherals/starkware/L2TransactionClient.ts index 7750fbbff..d11111783 100644 --- a/packages/backend/src/peripherals/starkware/L2TransactionClient.ts +++ b/packages/backend/src/peripherals/starkware/L2TransactionClient.ts @@ -1,3 +1,5 @@ +import { isEmpty } from 'lodash' + import { L2TransactionApiConfig } from '../../config/starkex/StarkexConfig' import { BaseClient } from './BaseClient' import { FetchClient } from './FetchClient' @@ -15,7 +17,7 @@ export class L2TransactionClient extends BaseClient { async getThirdPartyIdByTransactionId(transactionId: number) { const url = this.options.getThirdPartyIdByTransactionIdUrl(transactionId) - const res = await this.fetchClient.fetch(url, this.requestInit) + const res = await this.fetchClient.fetchRetry(url, this.requestInit) const text = await res.text() const thirdPartyId = Number(text) // thirdPartyId is equal 0 if the transaction is not found @@ -24,10 +26,11 @@ export class L2TransactionClient extends BaseClient { async getPerpetualTransactions(startId: number, pageSize: number) { const data = await this.getTransactions(startId, pageSize) - //TODO: remove this - if (JSON.stringify(data) === '{}') { + + if (isEmpty(data)) { return undefined } + const parsed = PerpetualL2TransactionResponse.parse(data) return toPerpetualL2Transactions(parsed) } @@ -37,7 +40,6 @@ export class L2TransactionClient extends BaseClient { pageSize: number ): Promise { const url = this.options.getTransactionsUrl(startId, pageSize) - const res = await this.fetchClient.fetchRetry(url, this.requestInit) return res.json() } From 3c0b66b92c8772682d4a2f0bd26a22806aaad75d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20T=C3=B3rz?= Date: Mon, 17 Jul 2023 15:49:45 +0200 Subject: [PATCH 05/19] improve LiveL2TransactionDownloader --- packages/backend/src/Application.ts | 26 +++ .../src/config/starkex/apex-mainnet.ts | 5 +- packages/backend/src/core/IDataSyncService.ts | 5 +- .../src/core/PerpetualValidiumSyncService.ts | 10 +- .../src/core/sync/L2TransactionDownloader.ts | 104 ------------ .../core/sync/LiveL2TransactionDownloader.ts | 154 ++++++++++++++++++ .../src/core/sync/SyncScheduler.test.ts | 120 +++++++++++--- .../backend/src/core/sync/SyncScheduler.ts | 8 +- .../src/peripherals/database/KeyValueStore.ts | 1 + .../database/L2TransactionRepository.ts | 1 + .../peripherals/starkware/schema/regexes.ts | 2 +- 11 files changed, 306 insertions(+), 130 deletions(-) delete mode 100644 packages/backend/src/core/sync/L2TransactionDownloader.ts create mode 100644 packages/backend/src/core/sync/LiveL2TransactionDownloader.ts diff --git a/packages/backend/src/Application.ts b/packages/backend/src/Application.ts index 5d7bee43b..619c61daa 100644 --- a/packages/backend/src/Application.ts +++ b/packages/backend/src/Application.ts @@ -55,6 +55,8 @@ import { SpotValidiumSyncService } from './core/SpotValidiumSyncService' import { SpotValidiumUpdater } from './core/SpotValidiumUpdater' import { StatusService } from './core/StatusService' import { BlockDownloader } from './core/sync/BlockDownloader' +import { Clock } from './core/sync/Clock' +import { LiveL2TransactionDownloader } from './core/sync/LiveL2TransactionDownloader' import { SyncScheduler } from './core/sync/SyncScheduler' import { TransactionStatusService } from './core/TransactionStatusService' import { UserService } from './core/UserService' @@ -85,6 +87,7 @@ import { TokenInspector } from './peripherals/ethereum/TokenInspector' import { AvailabilityGatewayClient } from './peripherals/starkware/AvailabilityGatewayClient' import { FeederGatewayClient } from './peripherals/starkware/FeederGatewayClient' import { FetchClient } from './peripherals/starkware/FetchClient' +import { L2TransactionClient } from './peripherals/starkware/L2TransactionClient' import { handleServerError, reportError } from './tools/ErrorReporter' import { Logger } from './tools/Logger' import { shouldShowL2Transactions } from './utils/shouldShowL2Transactions' @@ -100,6 +103,8 @@ export class Application { reportError, }) + const clock = new Clock() + // #endregion tools // #region peripherals @@ -219,6 +224,7 @@ export class Application { let stateTransitionCollector: IStateTransitionCollector let feederGatewayCollector: FeederGatewayCollector | undefined + let l2TransactionDownloader: LiveL2TransactionDownloader | undefined if (config.starkex.dataAvailabilityMode === 'validium') { const availabilityGatewayClient = new AvailabilityGatewayClient( @@ -235,6 +241,24 @@ export class Application { ) stateTransitionCollector = perpetualValidiumStateTransitionCollector + const l2TransactionClient = config.starkex.l2TransactionApi + ? new L2TransactionClient( + config.starkex.l2TransactionApi, + fetchClient + ) + : undefined + + l2TransactionDownloader = l2TransactionClient + ? new LiveL2TransactionDownloader( + l2TransactionClient, + l2TransactionRepository, + stateUpdateRepository, + kvStore, + clock, + logger + ) + : undefined + const feederGatewayClient = config.starkex.feederGateway ? new FeederGatewayClient( config.starkex.feederGateway, @@ -278,6 +302,7 @@ export class Application { perpetualValidiumUpdater, withdrawalAllowedCollector, feederGatewayCollector, + l2TransactionDownloader, logger ) } else { @@ -644,6 +669,7 @@ export class Application { if (config.enableSync) { transactionStatusService.start() await syncScheduler.start() + await l2TransactionDownloader?.start() await blockDownloader.start() } diff --git a/packages/backend/src/config/starkex/apex-mainnet.ts b/packages/backend/src/config/starkex/apex-mainnet.ts index 881ddffa4..573f1bb7a 100644 --- a/packages/backend/src/config/starkex/apex-mainnet.ts +++ b/packages/backend/src/config/starkex/apex-mainnet.ts @@ -37,11 +37,14 @@ export function getApexMainnetConfig(): StarkexConfig { auth: clientAuth, }, l2TransactionApi: { - getUrl: (startId, expectCount) => { + getTransactionsUrl: (startId, expectCount) => { return `${getEnv( 'APEX_TRANSACTION_API_URL' )}?startApexId=${startId}&expectCount=${expectCount}` }, + getThirdPartyIdByTransactionIdUrl: (transactionId) => { + return `${getEnv('APEX_THIRD_PARTY_ID_API_URL')}?txId=${transactionId}` + }, auth: clientAuth, }, collateralAsset: { diff --git a/packages/backend/src/core/IDataSyncService.ts b/packages/backend/src/core/IDataSyncService.ts index 0fccc2f94..6ce7f7d59 100644 --- a/packages/backend/src/core/IDataSyncService.ts +++ b/packages/backend/src/core/IDataSyncService.ts @@ -4,11 +4,12 @@ import { PerpetualRollupStateTransition } from './PerpetualRollupUpdater' import { ValidiumStateTransition } from './PerpetualValidiumUpdater' export interface IDataSyncService { - sync(blockRange: BlockRange): Promise + sync(blockRange: BlockRange, isTip?: boolean): Promise + // I made isTip optional but as soon as we will support other types like PerpetualRollup etc. we will need to make it required. processStateTransitions( stateTransitions: | ValidiumStateTransition[] | PerpetualRollupStateTransition[] - ): Promise + ): Promise discardAfter(blockNumber: BlockNumber): Promise } diff --git a/packages/backend/src/core/PerpetualValidiumSyncService.ts b/packages/backend/src/core/PerpetualValidiumSyncService.ts index 82381e364..49cec13b3 100644 --- a/packages/backend/src/core/PerpetualValidiumSyncService.ts +++ b/packages/backend/src/core/PerpetualValidiumSyncService.ts @@ -13,6 +13,7 @@ import { PerpetualValidiumUpdater, ValidiumStateTransition, } from './PerpetualValidiumUpdater' +import { LiveL2TransactionDownloader } from './sync/LiveL2TransactionDownloader' export class PerpetualValidiumSyncService implements IDataSyncService { constructor( @@ -24,12 +25,15 @@ export class PerpetualValidiumSyncService implements IDataSyncService { private readonly perpetualValidiumUpdater: PerpetualValidiumUpdater, private readonly withdrawalAllowedCollector: WithdrawalAllowedCollector, private readonly feederGatewayCollector: FeederGatewayCollector | undefined, + private readonly L2TransactionDownloader: + | LiveL2TransactionDownloader + | undefined, private readonly logger: Logger ) { this.logger = logger.for(this) } - async sync(blockRange: BlockRange) { + async sync(blockRange: BlockRange, isTip: boolean) { const userRegistrations = await this.userRegistrationCollector.collect( blockRange ) @@ -49,6 +53,10 @@ export class PerpetualValidiumSyncService implements IDataSyncService { await this.processStateTransitions(stateTransitions) await this.feederGatewayCollector?.collect() + + if (isTip) { + await this.L2TransactionDownloader?.enableSync() + } } async processStateTransitions(stateTransitions: ValidiumStateTransition[]) { diff --git a/packages/backend/src/core/sync/L2TransactionDownloader.ts b/packages/backend/src/core/sync/L2TransactionDownloader.ts deleted file mode 100644 index e797bdb3e..000000000 --- a/packages/backend/src/core/sync/L2TransactionDownloader.ts +++ /dev/null @@ -1,104 +0,0 @@ -import { KeyValueStore } from '../../peripherals/database/KeyValueStore' -import { L2TransactionRepository } from '../../peripherals/database/L2TransactionRepository' -import { L2TransactionClient } from '../../peripherals/starkware/L2TransactionClient' -import { Logger } from '../../tools/Logger' -import { Clock } from './Clock' - -export class L2TransactionDownloader { - private clock = new Clock() - private PAGE_SIZE = 100 - private isRunning = false - private lastSyncedThirdPartyId: number | undefined - - constructor( - private readonly l2TransactionClient: L2TransactionClient, - private readonly l2TransactionRepository: L2TransactionRepository, - private readonly keyValueStore: KeyValueStore, - private readonly logger: Logger - ) { - this.logger = this.logger.for(this) - } - - start() { - this.clock.onEvery('15s', () => this.downloadNewTransactions()) - } - - async downloadNewTransactions() { - if (this.isRunning) { - return - } - this.isRunning = true - - this.logger.info('Starting L2 transaction downloader') - const lastIncluded = await this.l2TransactionRepository.findLatestIncluded() - if (!lastIncluded) { - this.isRunning = false - return - } - - this.lastSyncedThirdPartyId = Number( - await this.keyValueStore.findByKey('lastSyncedThirdPartyId') - ) - - let thirdPartyIdToSync = this.lastSyncedThirdPartyId - ? this.lastSyncedThirdPartyId - : await this.l2TransactionClient.getThirdPartyIdByTransactionId( - lastIncluded.transactionId - ) - - if (!thirdPartyIdToSync) { - this.isRunning = false - return - } - - while (true) { - this.logger.info(thirdPartyIdToSync.toString()) - const transactions = await this.addTransactions(thirdPartyIdToSync) - - if (!transactions) { - break - } - this.lastSyncedThirdPartyId = - transactions[transactions.length - 1]?.thirdPartyId - - thirdPartyIdToSync += this.PAGE_SIZE - } - - if (this.lastSyncedThirdPartyId) { - await this.keyValueStore.addOrUpdate({ - key: 'lastSyncedThirdPartyId', - value: this.lastSyncedThirdPartyId.toString(), - }) - } - this.isRunning = false - } - - private async addTransactions(thirdPartyId: number) { - this.logger.info(`Downloading transactions from ${thirdPartyId}`) - const transactions = - await this.l2TransactionClient.getPerpetualTransactions( - thirdPartyId, - this.PAGE_SIZE - ) - - if (!transactions) { - this.logger.info('No transactions found') - return - } - - await this.l2TransactionRepository.runInTransactionWithLockedTable( - async (trx) => { - for (const transaction of transactions) { - await this.l2TransactionRepository.add( - { - transactionId: transaction.transactionId, - data: transaction.transaction, - }, - trx - ) - } - } - ) - return transactions - } -} diff --git a/packages/backend/src/core/sync/LiveL2TransactionDownloader.ts b/packages/backend/src/core/sync/LiveL2TransactionDownloader.ts new file mode 100644 index 000000000..6fd1270b0 --- /dev/null +++ b/packages/backend/src/core/sync/LiveL2TransactionDownloader.ts @@ -0,0 +1,154 @@ +import { Knex } from 'knex' + +import { KeyValueStore } from '../../peripherals/database/KeyValueStore' +import { L2TransactionRepository } from '../../peripherals/database/L2TransactionRepository' +import { StateUpdateRepository } from '../../peripherals/database/StateUpdateRepository' +import { L2TransactionClient } from '../../peripherals/starkware/L2TransactionClient' +import { PerpetualL2Transaction } from '../../peripherals/starkware/toPerpetualTransactions' +import { Logger } from '../../tools/Logger' +import { Clock } from './Clock' + +export class LiveL2TransactionDownloader { + private PAGE_SIZE = 100 + private isRunning = false + private lastSyncedThirdPartyId: number | undefined + private enabled = false + + constructor( + private readonly l2TransactionClient: L2TransactionClient, + private readonly l2TransactionRepository: L2TransactionRepository, + private readonly stateUpdateRepository: StateUpdateRepository, + private readonly keyValueStore: KeyValueStore, + private readonly clock: Clock, + private readonly logger: Logger + ) { + this.logger = this.logger.for(this) + } + + async start() { + this.logger.info('Starting L2 transaction downloader') + + await this.initialize() + this.clock.onEvery('5s', () => this.sync()) + } + + private async initialize() { + const lastSyncedThirdPartyId = await this.keyValueStore.findByKey( + 'lastSyncedThirdPartyId' + ) + if (!lastSyncedThirdPartyId) { + return + } + + this.enabled = true + this.lastSyncedThirdPartyId = lastSyncedThirdPartyId + } + + async enableSync() { + if (this.enabled) { + return + } + + const lastStateUpdate = await this.stateUpdateRepository.findLast() + if (!lastStateUpdate) { + return + } + const lastIncluded = await this.l2TransactionRepository.findLatestIncluded() + if (!lastIncluded) { + return + } + + if (lastStateUpdate.id !== lastIncluded.stateUpdateId) { + return + } + + const lastSyncedThirdPartyId = + await this.l2TransactionClient.getThirdPartyIdByTransactionId( + lastIncluded.transactionId + ) + if (!lastSyncedThirdPartyId) { + return + } + this.logger.info('Enabling L2 transaction downloader') + await this.updateLastSyncedThirdPartyId(lastSyncedThirdPartyId) + this.enabled = true + } + + private async sync() { + if (this.isRunning) { + return + } + this.isRunning = true + + const lastSyncedThirdPartyId = this.lastSyncedThirdPartyId + if (!lastSyncedThirdPartyId) { + this.isRunning = false + return + } + + await this.l2TransactionRepository.runInTransactionWithLockedTable( + async (trx) => { + await this.downloadAndAddTransactions(lastSyncedThirdPartyId + 1, trx) + } + ) + } + + private async downloadAndAddTransactions( + thirdPartyId: number, + trx: Knex.Transaction + ) { + this.logger.info(`Downloading live transactions from ${thirdPartyId}`) + + const transactions = + await this.l2TransactionClient.getPerpetualTransactions( + thirdPartyId, + this.PAGE_SIZE + ) + + if (!transactions) { + this.logger.info('No transactions found') + this.isRunning = false + return + } + + await this.addTransactions(transactions, trx) + + const lastSyncedThirdPartyId = thirdPartyId + this.PAGE_SIZE + await this.updateLastSyncedThirdPartyId(lastSyncedThirdPartyId, trx) + + if (transactions.length === this.PAGE_SIZE) { + await this.downloadAndAddTransactions(lastSyncedThirdPartyId, trx) + } else { + this.isRunning = false + } + } + + private async addTransactions( + transactions: PerpetualL2Transaction[], + trx: Knex.Transaction + ) { + for (const transaction of transactions) { + await this.l2TransactionRepository.add( + { + transactionId: transaction.transactionId, + data: transaction.transaction, + }, + trx + ) + } + } + + private async updateLastSyncedThirdPartyId( + lastSyncedThirdPartyId: number, + trx?: Knex.Transaction + ) { + await this.keyValueStore.addOrUpdate( + { + key: 'lastSyncedThirdPartyId', + value: lastSyncedThirdPartyId, + }, + trx + ) + this.lastSyncedThirdPartyId = lastSyncedThirdPartyId + } +} diff --git a/packages/backend/src/core/sync/SyncScheduler.test.ts b/packages/backend/src/core/sync/SyncScheduler.test.ts index 336c39165..773b6cb8a 100644 --- a/packages/backend/src/core/sync/SyncScheduler.test.ts +++ b/packages/backend/src/core/sync/SyncScheduler.test.ts @@ -6,12 +6,13 @@ import { BlockRange } from '../../model' import { KeyValueStore } from '../../peripherals/database/KeyValueStore' import { Logger } from '../../tools/Logger' import { PerpetualRollupSyncService } from '../PerpetualRollupSyncService' +import { PerpetualValidiumSyncService } from '../PerpetualValidiumSyncService' import { Preprocessor } from '../preprocessing/Preprocessor' import { BlockDownloader } from './BlockDownloader' import { SyncScheduler } from './SyncScheduler' import { Block } from './syncSchedulerReducer' -describe(SyncScheduler.name, () => { +describe.only(SyncScheduler.name, () => { const block = (number: number): Block => ({ number, hash: Hash256.fake(number.toString()), @@ -94,11 +95,12 @@ describe(SyncScheduler.name, () => { describe(SyncScheduler.prototype.dispatch.name, () => { it('handles a successful sync', async () => { + const isTip = true const mockKeyValueStore = mockObject({ addOrUpdate: mockFn().resolvesTo('lastBlockNumberSynced'), }) const blockDownloader = mockObject() - const dataSyncService = mockObject({ + const dataSyncService = mockObject({ sync: async () => {}, discardAfter: async () => {}, }) @@ -113,6 +115,8 @@ describe(SyncScheduler.name, () => { Logger.SILENT, { earliestBlock: 1_000_000 } ) + const mockIsTipFn = mockFn().returns(isTip) + syncScheduler.isTip = mockIsTipFn syncScheduler.dispatch({ type: 'initialized', @@ -121,9 +125,11 @@ describe(SyncScheduler.name, () => { }) await waitForExpect(() => { + expect(mockIsTipFn).toHaveBeenOnlyCalledWith(1_000_003) expect(dataSyncService.discardAfter).toHaveBeenOnlyCalledWith(1_000_000) expect(dataSyncService.sync).toHaveBeenOnlyCalledWith( - new BlockRange([block(1_000_001), block(1_000_002)]) + new BlockRange([block(1_000_001), block(1_000_002)]), + isTip ) expect(mockKeyValueStore.addOrUpdate).toHaveBeenOnlyCalledWith({ key: 'lastBlockNumberSynced', @@ -134,11 +140,12 @@ describe(SyncScheduler.name, () => { }) it('handles a failing sync', async () => { + const isTip = false const mockKeyValueStore = mockObject({ addOrUpdate: mockFn().resolvesTo('lastBlockNumberSynced'), }) const blockDownloader = mockObject() - const dataSyncService = mockObject({ + const dataSyncService = mockObject({ sync: mockFn().rejectsWith(new Error('oops')), discardAfter: async () => {}, }) @@ -153,6 +160,8 @@ describe(SyncScheduler.name, () => { Logger.SILENT, { earliestBlock: 1_000_000 } ) + const mockIsTipFn = mockFn().returns(isTip) + syncScheduler.isTip = mockIsTipFn syncScheduler.dispatch({ type: 'initialized', @@ -161,8 +170,10 @@ describe(SyncScheduler.name, () => { }) await waitForExpect(() => { + expect(mockIsTipFn).toHaveBeenOnlyCalledWith(1_000_003) expect(dataSyncService.sync).toHaveBeenOnlyCalledWith( - new BlockRange([block(1_000_001), block(1_000_002)]) + new BlockRange([block(1_000_001), block(1_000_002)]), + isTip ) expect(mockKeyValueStore.addOrUpdate).not.toHaveBeenCalled() }) @@ -172,11 +183,12 @@ describe(SyncScheduler.name, () => { }) it('handles a successful discardAfter', async () => { + const isTip = true const mockKeyValueStore = mockObject({ addOrUpdate: mockFn().resolvesTo('lastBlockNumberSynced'), }) const blockDownloader = mockObject() - const dataSyncService = mockObject({ + const dataSyncService = mockObject({ sync: async () => {}, discardAfter: async () => {}, }) @@ -191,6 +203,8 @@ describe(SyncScheduler.name, () => { Logger.SILENT, { earliestBlock: 1_000_000 } ) + const mockIsTipFn = mockFn().returns(isTip) + syncScheduler.isTip = mockIsTipFn syncScheduler.dispatch({ type: 'initialized', @@ -207,8 +221,10 @@ describe(SyncScheduler.name, () => { expect(dataSyncService.discardAfter).toHaveBeenNthCalledWith(1, 999_999) expect(dataSyncService.discardAfter).toHaveBeenNthCalledWith(2, 999_999) + expect(mockIsTipFn).toHaveBeenOnlyCalledWith(1_000_002) expect(dataSyncService.sync).toHaveBeenOnlyCalledWith( - new BlockRange([block(1_000_000), block(1_000_001)]) + new BlockRange([block(1_000_000), block(1_000_001)]), + isTip ) expect(mockKeyValueStore.addOrUpdate).toHaveBeenCalledTimes(2) @@ -271,9 +287,11 @@ describe(SyncScheduler.name, () => { }) describe(SyncScheduler.prototype.handleSync.name, () => { + const maxBlockNumber = 10 + it('triggers data sync only if block range is inside the limit', async () => { - const maxBlockNumber = 10 - const dataSyncService = mockObject({ + const isTip = true + const dataSyncService = mockObject({ discardAfter: async () => {}, sync: async () => {}, }) @@ -291,28 +309,92 @@ describe(SyncScheduler.name, () => { Logger.SILENT, { earliestBlock: 1, maxBlockNumber } ) + const mockIsTipFn = mockFn().returns(isTip) + syncScheduler.isTip = mockIsTipFn - await syncScheduler.handleSync( - new BlockRange([block(maxBlockNumber - 2), block(maxBlockNumber - 1)]) - ) + const blocks = new BlockRange([ + block(maxBlockNumber - 2), + block(maxBlockNumber - 1), + ]) + + await syncScheduler.handleSync(blocks) await waitForExpect(() => { + expect(mockIsTipFn).toHaveBeenOnlyCalledWith(blocks.end) expect(dataSyncService.discardAfter).toHaveBeenCalledTimes(1) - expect(dataSyncService.sync).toHaveBeenCalledTimes(1) + expect(dataSyncService.sync).toHaveBeenOnlyCalledWith( + expect.a(BlockRange), + isTip + ) expect(mockKeyValueStore.addOrUpdate).toHaveBeenCalledTimes(1) expect(preprocessor.sync).toHaveBeenCalled() }) + }) - await syncScheduler.handleSync( - new BlockRange([block(maxBlockNumber), block(maxBlockNumber + 1)]) + it('skips data sync if block range is outside the limit', async () => { + const dataSyncService = mockObject({ + discardAfter: mockFn(), + sync: mockFn(), + }) + const mockKeyValueStore = mockObject({ + addOrUpdate: mockFn(), + }) + const preprocessor = mockObject>({ + sync: mockFn(), + }) + + const syncScheduler = new SyncScheduler( + mockKeyValueStore, + mockObject(), + dataSyncService, + preprocessor, + Logger.SILENT, + { earliestBlock: 1, maxBlockNumber } ) + const mockIsTipFn = mockFn().returns(false) + syncScheduler.isTip = mockIsTipFn + + const blocks = new BlockRange([ + block(maxBlockNumber), + block(maxBlockNumber + 1), + ]) + + await syncScheduler.handleSync(blocks) await waitForExpect(() => { - expect(dataSyncService.discardAfter).toHaveBeenCalledTimes(1) - expect(dataSyncService.sync).toHaveBeenCalledTimes(1) - expect(mockKeyValueStore.addOrUpdate).toHaveBeenCalledTimes(1) - expect(preprocessor.sync).toHaveBeenCalled() + expect(mockIsTipFn).not.toHaveBeenCalled() + expect(dataSyncService.discardAfter).not.toHaveBeenCalled() + expect(dataSyncService.sync).not.toHaveBeenCalled() + expect(mockKeyValueStore.addOrUpdate).not.toHaveBeenCalled() + expect(preprocessor.sync).not.toHaveBeenCalled() }) }) }) + + describe(SyncScheduler.prototype.isTip.name, () => { + const syncScheduler = new SyncScheduler( + mockObject(), + mockObject(), + mockObject(), + mockObject>(), + Logger.SILENT, + { earliestBlock: 1, maxBlockNumber: 10 } + ) + + beforeEach(() => { + syncScheduler.dispatch({ + type: 'initialized', + lastSynced: 1_000_000, + knownBlocks: [block(1_000_001), block(1_000_002)], + }) + }) + + it('returns false if the block range is not the tip', () => { + expect(syncScheduler.isTip(1_000_001)).toEqual(false) + }) + + it('returns true if the block range is the tip', () => { + expect(syncScheduler.isTip(1_000_003)).toEqual(true) + }) + }) }) diff --git a/packages/backend/src/core/sync/SyncScheduler.ts b/packages/backend/src/core/sync/SyncScheduler.ts index 6dec08320..18d194d8d 100644 --- a/packages/backend/src/core/sync/SyncScheduler.ts +++ b/packages/backend/src/core/sync/SyncScheduler.ts @@ -75,7 +75,6 @@ export class SyncScheduler { name: 'action', execute: async () => { this.logger.debug({ method: 'effect', effect: effect.type }) - if (effect.type === 'sync') { await this.handleSync(effect.blocks) } else { @@ -86,6 +85,10 @@ export class SyncScheduler { } } + isTip(syncedBlockNumber: number) { + return this.state.remaining.end === syncedBlockNumber + } + async handleSync(blocks: BlockRange) { if (blocks.end > this.maxBlockNumber) { this.logger.info( @@ -102,8 +105,9 @@ export class SyncScheduler { return } try { + const isTip = this.isTip(blocks.end) await this.dataSyncService.discardAfter(blocks.start - 1) - await this.dataSyncService.sync(blocks) + await this.dataSyncService.sync(blocks, isTip) await this.kvStore.addOrUpdate({ key: 'lastBlockNumberSynced', value: blocks.end - 1, diff --git a/packages/backend/src/peripherals/database/KeyValueStore.ts b/packages/backend/src/peripherals/database/KeyValueStore.ts index 9d81b7566..db4db8183 100644 --- a/packages/backend/src/peripherals/database/KeyValueStore.ts +++ b/packages/backend/src/peripherals/database/KeyValueStore.ts @@ -11,6 +11,7 @@ export type KeyValueRecord = z.infer export const KeyValueRecord = z.union([ z.object({ key: z.literal('softwareMigrationNumber'), value: stringAsInt() }), z.object({ key: z.literal('lastBlockNumberSynced'), value: stringAsInt() }), + z.object({ key: z.literal('lastSyncedThirdPartyId'), value: stringAsInt() }), z.object({ key: z.literal('userStatisticsPreprocessorCaughtUp'), value: stringAsBoolean(), diff --git a/packages/backend/src/peripherals/database/L2TransactionRepository.ts b/packages/backend/src/peripherals/database/L2TransactionRepository.ts index 99afe1362..74b14ae2e 100644 --- a/packages/backend/src/peripherals/database/L2TransactionRepository.ts +++ b/packages/backend/src/peripherals/database/L2TransactionRepository.ts @@ -295,6 +295,7 @@ export class L2TransactionRepository extends BaseRepository { const knex = await this.knex() const results = await knex('l2_transactions') .select('state_update_id') + .whereNotNull('state_update_id') .orderBy('state_update_id', 'desc') .limit(1) .first() diff --git a/packages/backend/src/peripherals/starkware/schema/regexes.ts b/packages/backend/src/peripherals/starkware/schema/regexes.ts index c1fc64167..cdb47bdb4 100644 --- a/packages/backend/src/peripherals/starkware/schema/regexes.ts +++ b/packages/backend/src/peripherals/starkware/schema/regexes.ts @@ -6,6 +6,6 @@ export const PedersenHash = z.string().regex(/^0[a-f\d]{63}$/) export const Hash256_0x = z.string().regex(/^0x[a-f\d]{1,64}$/) export const Hash256 = z.string().regex(/^[a-f\d]{1,64}$/) export const StarkKey0x = z.string().regex(/^0x[a-f\d]{1,64}$/) -export const AssetHash0x = z.string().regex(/^0x[a-f\d]{0,63}$/) +export const AssetHash0x = z.string().regex(/^0x[a-f\d]{1,64}$/) export const AssetId = z.string().regex(/^0x[a-f\d]{30}$/) export const EthereumAddress = z.string().regex(/^0x[a-fA-F0-9]{40}$/) From 6b9903f6b3200cdf2f4737fa954292458063b226 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20T=C3=B3rz?= Date: Mon, 17 Jul 2023 16:03:42 +0200 Subject: [PATCH 06/19] improve live l2 transaction log --- .../backend/src/core/sync/LiveL2TransactionDownloader.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/core/sync/LiveL2TransactionDownloader.ts b/packages/backend/src/core/sync/LiveL2TransactionDownloader.ts index 6fd1270b0..3e4b0929f 100644 --- a/packages/backend/src/core/sync/LiveL2TransactionDownloader.ts +++ b/packages/backend/src/core/sync/LiveL2TransactionDownloader.ts @@ -26,9 +26,10 @@ export class LiveL2TransactionDownloader { } async start() { - this.logger.info('Starting L2 transaction downloader') - await this.initialize() + this.logger.info('Starting L2 transaction downloader', { + enabled: this.enabled, + }) this.clock.onEvery('5s', () => this.sync()) } From 593ea6b0caaca08507d484a47efd67e624f492a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20T=C3=B3rz?= Date: Tue, 18 Jul 2023 12:47:28 +0200 Subject: [PATCH 07/19] add live l2 transaction downloader tests --- .../sync/LiveL2TransactionDownloader.test.ts | 401 ++++++++++++++++++ .../core/sync/LiveL2TransactionDownloader.ts | 20 +- .../src/core/sync/SyncScheduler.test.ts | 2 +- 3 files changed, 412 insertions(+), 11 deletions(-) create mode 100644 packages/backend/src/core/sync/LiveL2TransactionDownloader.test.ts diff --git a/packages/backend/src/core/sync/LiveL2TransactionDownloader.test.ts b/packages/backend/src/core/sync/LiveL2TransactionDownloader.test.ts new file mode 100644 index 000000000..bd0e3cac1 --- /dev/null +++ b/packages/backend/src/core/sync/LiveL2TransactionDownloader.test.ts @@ -0,0 +1,401 @@ +import { StarkKey } from '@explorer/types' +import { randomInt } from 'crypto' +import { expect, mockFn, mockObject } from 'earl' +import { Knex } from 'knex' +import { range } from 'lodash' +import waitForExpect from 'wait-for-expect' + +import { KeyValueStore } from '../../peripherals/database/KeyValueStore' +import { L2TransactionRepository } from '../../peripherals/database/L2TransactionRepository' +import { StateUpdateRepository } from '../../peripherals/database/StateUpdateRepository' +import { L2TransactionClient } from '../../peripherals/starkware/L2TransactionClient' +import { PerpetualL2Transaction } from '../../peripherals/starkware/toPerpetualTransactions' +import { Logger } from '../../tools/Logger' +import { Clock } from './Clock' +import { LiveL2TransactionDownloader } from './LiveL2TransactionDownloader' + +const fakeL2Transaction = ( + transaction?: Partial +): PerpetualL2Transaction => ({ + thirdPartyId: randomInt(100000), + transactionId: randomInt(100000), + transaction: { + type: 'Deposit', + positionId: BigInt(randomInt(100000)), + starkKey: StarkKey.fake(), + amount: BigInt(randomInt(100000)), + }, + ...transaction, +}) + +describe.only(LiveL2TransactionDownloader.name, () => { + describe(LiveL2TransactionDownloader.prototype.start.name, () => { + it('should initialize, start and sync', async () => { + const thirdPartyId = 1200005 + const firstTxs = range(100).map(() => fakeL2Transaction()) + const secondTxs = range(99).map(() => fakeL2Transaction()) + + const mockKnexTransaction = mockObject({}) + const mockClock = mockObject({ + onEvery: mockFn((_, cb) => cb()), + }) + const mockKeyValueStore = mockObject({ + findByKey: mockFn().resolvesTo(thirdPartyId), + addOrUpdate: mockFn().resolvesTo('lastSyncedThirdPartyId'), + }) + const mockLiveL2TransactionClient = mockObject({ + getPerpetualTransactions: mockFn() + .resolvesToOnce(firstTxs) + .resolvesToOnce(secondTxs), + }) + const mockL2TransactionRepository = mockObject({ + add: mockFn().resolvesTo(1), + runInTransactionWithLockedTable: mockFn( + async (fun: (trx: Knex.Transaction) => Promise) => { + await fun(mockKnexTransaction) + } + ), + }) + + const liveL2TransactionDownloader = new LiveL2TransactionDownloader( + mockLiveL2TransactionClient, + mockL2TransactionRepository, + mockObject(), + mockKeyValueStore, + mockClock, + Logger.SILENT + ) + + await liveL2TransactionDownloader.start() + + expect(mockKeyValueStore.findByKey).toHaveBeenOnlyCalledWith( + 'lastSyncedThirdPartyId' + ) + expect(liveL2TransactionDownloader.isEnabled).toEqual(true) + expect(mockClock.onEvery).toHaveBeenOnlyCalledWith( + '5s', + expect.a(Function) + ) + expect( + mockLiveL2TransactionClient.getPerpetualTransactions + ).toHaveBeenNthCalledWith(1, thirdPartyId + 1, 100) + + await waitForExpect(() => { + firstTxs.forEach((tx, i) => { + expect(mockL2TransactionRepository.add).toHaveBeenNthCalledWith( + i + 1, + { + transactionId: tx.transactionId, + data: tx.transaction, + }, + mockKnexTransaction + ) + }) + + expect(mockKeyValueStore.addOrUpdate).toHaveBeenNthCalledWith( + 1, + { + key: 'lastSyncedThirdPartyId', + value: thirdPartyId + firstTxs.length + 1, + }, + mockKnexTransaction + ) + }) + + expect( + mockLiveL2TransactionClient.getPerpetualTransactions + ).toHaveBeenNthCalledWith(2, thirdPartyId + firstTxs.length + 1, 100) + await waitForExpect(() => { + secondTxs.forEach((tx, i) => { + expect(mockL2TransactionRepository.add).toHaveBeenNthCalledWith( + firstTxs.length + i + 1, + { + data: tx.transaction, + transactionId: tx.transactionId, + }, + mockKnexTransaction + ) + + expect(mockKeyValueStore.addOrUpdate).toHaveBeenNthCalledWith( + 2, + { + key: 'lastSyncedThirdPartyId', + value: thirdPartyId + firstTxs.length + secondTxs.length + 1, + }, + mockKnexTransaction + ) + }) + }) + + expect( + mockLiveL2TransactionClient.getPerpetualTransactions + ).toHaveBeenExhausted() + }) + + it('should not sync if no lastSyncedThirdPartyId in db', async () => { + const mockL2TransactionRepository = mockObject({ + runInTransactionWithLockedTable: mockFn(), + }) + const mockKeyValueStore = mockObject({ + findByKey: mockFn().resolvesTo(undefined), + }) + const mockClock = mockObject({ + onEvery: mockFn((_, cb) => cb()), + }) + + const liveL2TransactionDownloader = new LiveL2TransactionDownloader( + mockObject(), + mockL2TransactionRepository, + mockObject(), + mockKeyValueStore, + mockClock, + Logger.SILENT + ) + + await liveL2TransactionDownloader.start() + + expect(mockKeyValueStore.findByKey).toHaveBeenOnlyCalledWith( + 'lastSyncedThirdPartyId' + ) + expect(liveL2TransactionDownloader.isEnabled).toEqual(false) + expect(mockClock.onEvery).toHaveBeenOnlyCalledWith( + '5s', + expect.a(Function) + ) + expect( + mockL2TransactionRepository.runInTransactionWithLockedTable + ).not.toHaveBeenCalled() + }) + + it('should not sync again if already running', async () => { + const mockL2TransactionRepository = mockObject({ + runInTransactionWithLockedTable: mockFn(), + }) + const mockKeyValueStore = mockObject({ + findByKey: mockFn().resolvesTo(1), + }) + const mockClock = mockObject({ + onEvery: mockFn((_, cb) => cb()), + }) + + const liveL2TransactionDownloader = new LiveL2TransactionDownloader( + mockObject(), + mockL2TransactionRepository, + mockObject(), + mockKeyValueStore, + mockClock, + Logger.SILENT + ) + liveL2TransactionDownloader.isRunning = true + + await liveL2TransactionDownloader.start() + + expect(mockKeyValueStore.findByKey).toHaveBeenOnlyCalledWith( + 'lastSyncedThirdPartyId' + ) + expect(liveL2TransactionDownloader.isEnabled).toEqual(true) + expect(mockClock.onEvery).toHaveBeenOnlyCalledWith( + '5s', + expect.a(Function) + ) + expect( + mockL2TransactionRepository.runInTransactionWithLockedTable + ).not.toHaveBeenCalled() + }) + }) + + describe(LiveL2TransactionDownloader.prototype.enableSync.name, () => { + it('should not do anything if already enabled', async () => { + const mockStateUpdateRepository = mockObject({ + findLast: mockFn(), + }) + + const liveL2TransactionDownloader = new LiveL2TransactionDownloader( + mockObject(), + mockObject(), + mockStateUpdateRepository, + mockObject(), + mockObject(), + Logger.SILENT + ) + + liveL2TransactionDownloader.isEnabled = true + + await liveL2TransactionDownloader.enableSync() + + expect(mockStateUpdateRepository.findLast).not.toHaveBeenCalled() + }) + + it('should not do anything if no last state update', async () => { + const mockStateUpdateRepository = mockObject({ + findLast: mockFn().resolvesTo(undefined), + }) + const mockL2TransactionRepository = mockObject({ + findLatestIncluded: mockFn(), + }) + + const liveL2TransactionDownloader = new LiveL2TransactionDownloader( + mockObject(), + mockObject(), + mockStateUpdateRepository, + mockObject(), + mockObject(), + Logger.SILENT + ) + + await liveL2TransactionDownloader.enableSync() + + expect(mockStateUpdateRepository.findLast).toHaveBeenCalled() + expect( + mockL2TransactionRepository.findLatestIncluded + ).not.toHaveBeenCalled() + expect(liveL2TransactionDownloader.isEnabled).toEqual(false) + }) + + it('should not do anything if no last included transaction', async () => { + const mockStateUpdateRepository = mockObject({ + findLast: mockFn().resolvesTo({}), + }) + const mockL2TransactionRepository = mockObject({ + findLatestIncluded: mockFn().resolvesTo(undefined), + }) + const mockL2TransactionClient = mockObject({ + getThirdPartyIdByTransactionId: mockFn(), + }) + + const liveL2TransactionDownloader = new LiveL2TransactionDownloader( + mockObject(), + mockL2TransactionRepository, + mockStateUpdateRepository, + mockObject(), + mockObject(), + Logger.SILENT + ) + + await liveL2TransactionDownloader.enableSync() + + expect(mockStateUpdateRepository.findLast).toHaveBeenCalled() + expect(mockL2TransactionRepository.findLatestIncluded).toHaveBeenCalled() + expect( + mockL2TransactionClient.getThirdPartyIdByTransactionId + ).not.toHaveBeenCalled() + expect(liveL2TransactionDownloader.isEnabled).toEqual(false) + }) + + it('should not do anything if last state update does not match last included transaction', async () => { + const mockStateUpdateRepository = mockObject({ + findLast: mockFn().resolvesTo({ id: 5 }), + }) + const mockL2TransactionRepository = mockObject({ + findLatestIncluded: mockFn().resolvesTo({ stateUpdateId: 10 }), + }) + const mockL2TransactionClient = mockObject({ + getThirdPartyIdByTransactionId: mockFn(), + }) + + const liveL2TransactionDownloader = new LiveL2TransactionDownloader( + mockObject(), + mockL2TransactionRepository, + mockStateUpdateRepository, + mockObject(), + mockObject(), + Logger.SILENT + ) + + await liveL2TransactionDownloader.enableSync() + + expect(mockStateUpdateRepository.findLast).toHaveBeenCalled() + expect(mockL2TransactionRepository.findLatestIncluded).toHaveBeenCalled() + expect( + mockL2TransactionClient.getThirdPartyIdByTransactionId + ).not.toHaveBeenCalled() + expect(liveL2TransactionDownloader.isEnabled).toEqual(false) + }) + + it('should not do anything if no last synced third party id returned from api', async () => { + const stateUpdateId = 5 + const transactionId = 20 + const mockStateUpdateRepository = mockObject({ + findLast: mockFn().resolvesTo({ id: stateUpdateId }), + }) + const mockL2TransactionRepository = mockObject({ + findLatestIncluded: mockFn().resolvesTo({ + stateUpdateId, + transactionId, + }), + }) + const mockL2TransactionClient = mockObject({ + getThirdPartyIdByTransactionId: mockFn().resolvesTo(undefined), + }) + const mockKeyValueStore = mockObject({ + addOrUpdate: mockFn(), + }) + + const liveL2TransactionDownloader = new LiveL2TransactionDownloader( + mockL2TransactionClient, + mockL2TransactionRepository, + mockStateUpdateRepository, + mockKeyValueStore, + mockObject(), + Logger.SILENT + ) + + await liveL2TransactionDownloader.enableSync() + + expect(mockStateUpdateRepository.findLast).toHaveBeenCalled() + expect(mockL2TransactionRepository.findLatestIncluded).toHaveBeenCalled() + expect( + mockL2TransactionClient.getThirdPartyIdByTransactionId + ).toHaveBeenCalledWith(transactionId) + expect(mockKeyValueStore.addOrUpdate).not.toHaveBeenCalled() + expect(liveL2TransactionDownloader.isEnabled).toEqual(false) + }) + + it('should enable sync', async () => { + const stateUpdateId = 5 + const transactionId = 20 + const thirdPartyId = 200 + + const mockStateUpdateRepository = mockObject({ + findLast: mockFn().resolvesTo({ id: stateUpdateId }), + }) + const mockL2TransactionRepository = mockObject({ + findLatestIncluded: mockFn().resolvesTo({ + stateUpdateId, + transactionId, + }), + }) + const mockL2TransactionClient = mockObject({ + getThirdPartyIdByTransactionId: mockFn().resolvesTo(thirdPartyId), + }) + const mockKeyValueStore = mockObject({ + addOrUpdate: mockFn().resolvesTo('lastSyncedThirdPartyId'), + }) + + const liveL2TransactionDownloader = new LiveL2TransactionDownloader( + mockL2TransactionClient, + mockL2TransactionRepository, + mockStateUpdateRepository, + mockKeyValueStore, + mockObject(), + Logger.SILENT + ) + + await liveL2TransactionDownloader.enableSync() + + expect(mockStateUpdateRepository.findLast).toHaveBeenCalled() + expect(mockL2TransactionRepository.findLatestIncluded).toHaveBeenCalled() + expect( + mockL2TransactionClient.getThirdPartyIdByTransactionId + ).toHaveBeenCalledWith(transactionId) + expect(mockKeyValueStore.addOrUpdate).toHaveBeenCalledWith( + { + key: 'lastSyncedThirdPartyId', + value: thirdPartyId, + }, + undefined + ) + expect(liveL2TransactionDownloader.isEnabled).toEqual(true) + }) + }) +}) diff --git a/packages/backend/src/core/sync/LiveL2TransactionDownloader.ts b/packages/backend/src/core/sync/LiveL2TransactionDownloader.ts index 3e4b0929f..426a17346 100644 --- a/packages/backend/src/core/sync/LiveL2TransactionDownloader.ts +++ b/packages/backend/src/core/sync/LiveL2TransactionDownloader.ts @@ -10,9 +10,9 @@ import { Clock } from './Clock' export class LiveL2TransactionDownloader { private PAGE_SIZE = 100 - private isRunning = false private lastSyncedThirdPartyId: number | undefined - private enabled = false + isRunning = false + isEnabled = false constructor( private readonly l2TransactionClient: L2TransactionClient, @@ -28,7 +28,7 @@ export class LiveL2TransactionDownloader { async start() { await this.initialize() this.logger.info('Starting L2 transaction downloader', { - enabled: this.enabled, + enabled: this.isEnabled, }) this.clock.onEvery('5s', () => this.sync()) } @@ -41,12 +41,12 @@ export class LiveL2TransactionDownloader { return } - this.enabled = true + this.isEnabled = true this.lastSyncedThirdPartyId = lastSyncedThirdPartyId } async enableSync() { - if (this.enabled) { + if (this.isEnabled) { return } @@ -72,13 +72,14 @@ export class LiveL2TransactionDownloader { } this.logger.info('Enabling L2 transaction downloader') await this.updateLastSyncedThirdPartyId(lastSyncedThirdPartyId) - this.enabled = true + this.isEnabled = true } private async sync() { - if (this.isRunning) { + if (this.isRunning || !this.isEnabled) { return } + this.isRunning = true const lastSyncedThirdPartyId = this.lastSyncedThirdPartyId @@ -86,7 +87,6 @@ export class LiveL2TransactionDownloader { this.isRunning = false return } - await this.l2TransactionRepository.runInTransactionWithLockedTable( async (trx) => { await this.downloadAndAddTransactions(lastSyncedThirdPartyId + 1, trx) @@ -107,14 +107,13 @@ export class LiveL2TransactionDownloader { ) if (!transactions) { - this.logger.info('No transactions found') this.isRunning = false return } await this.addTransactions(transactions, trx) - const lastSyncedThirdPartyId = thirdPartyId + this.PAGE_SIZE + const lastSyncedThirdPartyId = thirdPartyId + transactions.length await this.updateLastSyncedThirdPartyId(lastSyncedThirdPartyId, trx) if (transactions.length === this.PAGE_SIZE) { @@ -150,6 +149,7 @@ export class LiveL2TransactionDownloader { }, trx ) + this.lastSyncedThirdPartyId = lastSyncedThirdPartyId } } diff --git a/packages/backend/src/core/sync/SyncScheduler.test.ts b/packages/backend/src/core/sync/SyncScheduler.test.ts index 773b6cb8a..2d1919ba2 100644 --- a/packages/backend/src/core/sync/SyncScheduler.test.ts +++ b/packages/backend/src/core/sync/SyncScheduler.test.ts @@ -12,7 +12,7 @@ import { BlockDownloader } from './BlockDownloader' import { SyncScheduler } from './SyncScheduler' import { Block } from './syncSchedulerReducer' -describe.only(SyncScheduler.name, () => { +describe(SyncScheduler.name, () => { const block = (number: number): Block => ({ number, hash: Hash256.fake(number.toString()), From d95e2e907d623351d386eb90ec2e730d439a8b56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20T=C3=B3rz?= Date: Tue, 18 Jul 2023 13:00:33 +0200 Subject: [PATCH 08/19] replace count with find for adding transactions --- .../database/L2TransactionRepository.test.ts | 9 ++++++ .../database/L2TransactionRepository.ts | 32 +++++++++++++++++-- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/packages/backend/src/peripherals/database/L2TransactionRepository.test.ts b/packages/backend/src/peripherals/database/L2TransactionRepository.test.ts index c1ed7ae25..463dc3fa3 100644 --- a/packages/backend/src/peripherals/database/L2TransactionRepository.test.ts +++ b/packages/backend/src/peripherals/database/L2TransactionRepository.test.ts @@ -913,6 +913,15 @@ describe(L2TransactionRepository.name, () => { }) }) + describe( + L2TransactionRepository.prototype.findOldestByTransactionId.name, + () => { + it('returns oldest transaction', async () => { + throw Error('Add implementation after merge with preprocessing') + }) + } + ) + describe( L2TransactionRepository.prototype.getPaginatedWithoutMulti.name, () => { diff --git a/packages/backend/src/peripherals/database/L2TransactionRepository.ts b/packages/backend/src/peripherals/database/L2TransactionRepository.ts index 74b14ae2e..f6011510f 100644 --- a/packages/backend/src/peripherals/database/L2TransactionRepository.ts +++ b/packages/backend/src/peripherals/database/L2TransactionRepository.ts @@ -78,10 +78,26 @@ export class L2TransactionRepository extends BaseRepository { ): Promise { const knex = await this.knex(trx) - const count = await this.countByTransactionId(record.transactionId) - const isAlternative = count > 0 + const existingRecord = await this.findOldestByTransactionId( + record.transactionId + ) + const isLive = record.stateUpdateId === undefined + + const isAlternative = !!existingRecord + + /* + If live transactions are somehow behind the transactions from feeder gateway, we should not add them + Although we should add them if transaction in database is not included as it is alternative to the one in database + */ + if ( + isLive && + existingRecord?.transactionId === record.transactionId && + existingRecord.stateUpdateId !== undefined + ) { + return 0 + } - if (count === 1) { + if (existingRecord && existingRecord.state === undefined) { await knex('l2_transactions') .update({ state: 'replaced' }) .where({ transaction_id: record.transactionId }) @@ -291,6 +307,16 @@ export class L2TransactionRepository extends BaseRepository { return toAggregatedRecord(originalTransaction, alternativeTransactions) } + async findOldestByTransactionId(id: number): Promise { + const knex = await this.knex() + const row = await knex('l2_transactions') + .where({ transaction_id: id }) + .orderBy('id', 'asc') + .first() + + return row ? toRecord(row) : undefined + } + async findLatestStateUpdateId(): Promise { const knex = await this.knex() const results = await knex('l2_transactions') From 256d89c1e7921237c780c92af831989f78e87c26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20T=C3=B3rz?= Date: Tue, 18 Jul 2023 13:01:04 +0200 Subject: [PATCH 09/19] remove .only --- .../backend/src/core/sync/LiveL2TransactionDownloader.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/core/sync/LiveL2TransactionDownloader.test.ts b/packages/backend/src/core/sync/LiveL2TransactionDownloader.test.ts index bd0e3cac1..b076aa756 100644 --- a/packages/backend/src/core/sync/LiveL2TransactionDownloader.test.ts +++ b/packages/backend/src/core/sync/LiveL2TransactionDownloader.test.ts @@ -28,7 +28,7 @@ const fakeL2Transaction = ( ...transaction, }) -describe.only(LiveL2TransactionDownloader.name, () => { +describe(LiveL2TransactionDownloader.name, () => { describe(LiveL2TransactionDownloader.prototype.start.name, () => { it('should initialize, start and sync', async () => { const thirdPartyId = 1200005 From 6fbfd211a666253bca3908a56ac49257e2d5bb06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20T=C3=B3rz?= Date: Tue, 18 Jul 2023 14:05:16 +0200 Subject: [PATCH 10/19] fix linter errors --- packages/backend/src/core/IDataSyncService.ts | 2 +- packages/backend/src/core/sync/Clock.ts | 2 -- .../backend/src/core/sync/LiveL2TransactionDownloader.test.ts | 2 +- packages/backend/src/core/sync/LiveL2TransactionDownloader.ts | 2 ++ .../src/peripherals/database/L2TransactionRepository.test.ts | 2 +- .../backend/src/peripherals/starkware/L2TransactionClient.ts | 2 +- .../src/peripherals/starkware/toPerpetualTransactions.ts | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/backend/src/core/IDataSyncService.ts b/packages/backend/src/core/IDataSyncService.ts index 6ce7f7d59..b2a1fbdae 100644 --- a/packages/backend/src/core/IDataSyncService.ts +++ b/packages/backend/src/core/IDataSyncService.ts @@ -10,6 +10,6 @@ export interface IDataSyncService { stateTransitions: | ValidiumStateTransition[] | PerpetualRollupStateTransition[] - ): Promise + ): Promise discardAfter(blockNumber: BlockNumber): Promise } diff --git a/packages/backend/src/core/sync/Clock.ts b/packages/backend/src/core/sync/Clock.ts index 81f0747c7..44fcb7494 100644 --- a/packages/backend/src/core/sync/Clock.ts +++ b/packages/backend/src/core/sync/Clock.ts @@ -2,8 +2,6 @@ type IntervalUnit = 's' | 'm' | 'h' | 'd' type Interval = `${number}${IntervalUnit}` export class Clock { - constructor() {} - onEvery(interval: Interval, callback: () => void) { callback() setInterval(() => { diff --git a/packages/backend/src/core/sync/LiveL2TransactionDownloader.test.ts b/packages/backend/src/core/sync/LiveL2TransactionDownloader.test.ts index b076aa756..11acace32 100644 --- a/packages/backend/src/core/sync/LiveL2TransactionDownloader.test.ts +++ b/packages/backend/src/core/sync/LiveL2TransactionDownloader.test.ts @@ -2,7 +2,7 @@ import { StarkKey } from '@explorer/types' import { randomInt } from 'crypto' import { expect, mockFn, mockObject } from 'earl' import { Knex } from 'knex' -import { range } from 'lodash' +import range from 'lodash/range' import waitForExpect from 'wait-for-expect' import { KeyValueStore } from '../../peripherals/database/KeyValueStore' diff --git a/packages/backend/src/core/sync/LiveL2TransactionDownloader.ts b/packages/backend/src/core/sync/LiveL2TransactionDownloader.ts index 426a17346..4e15de04a 100644 --- a/packages/backend/src/core/sync/LiveL2TransactionDownloader.ts +++ b/packages/backend/src/core/sync/LiveL2TransactionDownloader.ts @@ -30,6 +30,8 @@ export class LiveL2TransactionDownloader { this.logger.info('Starting L2 transaction downloader', { enabled: this.isEnabled, }) + + // eslint-disable-next-line @typescript-eslint/no-misused-promises this.clock.onEvery('5s', () => this.sync()) } diff --git a/packages/backend/src/peripherals/database/L2TransactionRepository.test.ts b/packages/backend/src/peripherals/database/L2TransactionRepository.test.ts index 463dc3fa3..2941feafa 100644 --- a/packages/backend/src/peripherals/database/L2TransactionRepository.test.ts +++ b/packages/backend/src/peripherals/database/L2TransactionRepository.test.ts @@ -1404,7 +1404,7 @@ describe(L2TransactionRepository.name, () => { it('runs a function in a transaction with a locked table', async () => { const mockedLockTableFn = mockFn(async () => {}) repository.lockTable = mockedLockTableFn - const fn = mockFn(async (trx: Knex.Transaction) => {}) + const fn = mockFn(async (_: Knex.Transaction) => {}) await repository.runInTransactionWithLockedTable(fn) expect(mockedLockTableFn).toHaveBeenCalledTimes(1) diff --git a/packages/backend/src/peripherals/starkware/L2TransactionClient.ts b/packages/backend/src/peripherals/starkware/L2TransactionClient.ts index d11111783..f6ff99e61 100644 --- a/packages/backend/src/peripherals/starkware/L2TransactionClient.ts +++ b/packages/backend/src/peripherals/starkware/L2TransactionClient.ts @@ -1,4 +1,4 @@ -import { isEmpty } from 'lodash' +import isEmpty from 'lodash/isEmpty' import { L2TransactionApiConfig } from '../../config/starkex/StarkexConfig' import { BaseClient } from './BaseClient' diff --git a/packages/backend/src/peripherals/starkware/toPerpetualTransactions.ts b/packages/backend/src/peripherals/starkware/toPerpetualTransactions.ts index 8521fd3f1..5963c777b 100644 --- a/packages/backend/src/peripherals/starkware/toPerpetualTransactions.ts +++ b/packages/backend/src/peripherals/starkware/toPerpetualTransactions.ts @@ -15,9 +15,9 @@ import { import { AssetOraclePrice, OrderTypeResponse, + PerpetualL2Transaction as TransactionSchema, SignatureResponse, SignedOraclePrice, - PerpetualL2Transaction as TransactionSchema, } from './schema/PerpetualBatchInfoResponse' import { PerpetualL2TransactionResponse } from './schema/PerpetualL2TransactionResponse' export interface PerpetualL2Transaction { From 915e2236cec9d6b40434efb4727737a635ca9d59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20T=C3=B3rz?= Date: Thu, 20 Jul 2023 10:04:07 +0200 Subject: [PATCH 11/19] remove randomness from tests --- .../src/core/sync/LiveL2TransactionDownloader.test.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/packages/backend/src/core/sync/LiveL2TransactionDownloader.test.ts b/packages/backend/src/core/sync/LiveL2TransactionDownloader.test.ts index 11acace32..7364ada00 100644 --- a/packages/backend/src/core/sync/LiveL2TransactionDownloader.test.ts +++ b/packages/backend/src/core/sync/LiveL2TransactionDownloader.test.ts @@ -1,5 +1,4 @@ import { StarkKey } from '@explorer/types' -import { randomInt } from 'crypto' import { expect, mockFn, mockObject } from 'earl' import { Knex } from 'knex' import range from 'lodash/range' @@ -17,13 +16,13 @@ import { LiveL2TransactionDownloader } from './LiveL2TransactionDownloader' const fakeL2Transaction = ( transaction?: Partial ): PerpetualL2Transaction => ({ - thirdPartyId: randomInt(100000), - transactionId: randomInt(100000), + thirdPartyId: 1024, + transactionId: 2048, transaction: { type: 'Deposit', - positionId: BigInt(randomInt(100000)), + positionId: 4096n, starkKey: StarkKey.fake(), - amount: BigInt(randomInt(100000)), + amount: 8196n, }, ...transaction, }) From 2655d8bde64a724e1cd225f4477cec45290d2833 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20T=C3=B3rz?= Date: Thu, 20 Jul 2023 10:07:41 +0200 Subject: [PATCH 12/19] rename L2TransactionDownloader to LiveL2TransactionDownloader --- packages/backend/src/Application.ts | 4 +- .../src/config/starkex/StarkexConfig.ts | 4 +- .../sync/LiveL2TransactionDownloader.test.ts | 32 ++--- .../core/sync/LiveL2TransactionDownloader.ts | 6 +- .../starkware/L2TransactionClient.test.ts | 115 --------------- .../starkware/LiveL2TransactionClient.test.ts | 135 ++++++++++++++++++ ...onClient.ts => LiveL2TransactionClient.ts} | 12 +- .../schema/PerpetualL2TransactionResponse.ts | 24 ---- ...erpetualLiveL2TransactionResponse.test.ts} | 4 +- .../PerpetualLiveL2TransactionResponse.ts | 24 ++++ .../starkware/toPerpetualTransactions.ts | 6 +- 11 files changed, 193 insertions(+), 173 deletions(-) delete mode 100644 packages/backend/src/peripherals/starkware/L2TransactionClient.test.ts create mode 100644 packages/backend/src/peripherals/starkware/LiveL2TransactionClient.test.ts rename packages/backend/src/peripherals/starkware/{L2TransactionClient.ts => LiveL2TransactionClient.ts} (72%) delete mode 100644 packages/backend/src/peripherals/starkware/schema/PerpetualL2TransactionResponse.ts rename packages/backend/src/peripherals/starkware/schema/{PerpetualL2TransactionResponse.test.ts => PerpetualLiveL2TransactionResponse.test.ts} (60%) create mode 100644 packages/backend/src/peripherals/starkware/schema/PerpetualLiveL2TransactionResponse.ts diff --git a/packages/backend/src/Application.ts b/packages/backend/src/Application.ts index 619c61daa..585c8c72d 100644 --- a/packages/backend/src/Application.ts +++ b/packages/backend/src/Application.ts @@ -87,7 +87,7 @@ import { TokenInspector } from './peripherals/ethereum/TokenInspector' import { AvailabilityGatewayClient } from './peripherals/starkware/AvailabilityGatewayClient' import { FeederGatewayClient } from './peripherals/starkware/FeederGatewayClient' import { FetchClient } from './peripherals/starkware/FetchClient' -import { L2TransactionClient } from './peripherals/starkware/L2TransactionClient' +import { LiveL2TransactionClient } from './peripherals/starkware/LiveL2TransactionClient' import { handleServerError, reportError } from './tools/ErrorReporter' import { Logger } from './tools/Logger' import { shouldShowL2Transactions } from './utils/shouldShowL2Transactions' @@ -242,7 +242,7 @@ export class Application { stateTransitionCollector = perpetualValidiumStateTransitionCollector const l2TransactionClient = config.starkex.l2TransactionApi - ? new L2TransactionClient( + ? new LiveL2TransactionClient( config.starkex.l2TransactionApi, fetchClient ) diff --git a/packages/backend/src/config/starkex/StarkexConfig.ts b/packages/backend/src/config/starkex/StarkexConfig.ts index 490c9e7fc..5467d36e0 100644 --- a/packages/backend/src/config/starkex/StarkexConfig.ts +++ b/packages/backend/src/config/starkex/StarkexConfig.ts @@ -38,7 +38,7 @@ export interface PerpetualValidiumConfig { blockchain: BlockchainConfig availabilityGateway: GatewayConfig feederGateway: GatewayConfig | undefined - l2TransactionApi: L2TransactionApiConfig | undefined + l2TransactionApi: LiveL2TransactionApiConfig | undefined contracts: { perpetual: EthereumAddress } @@ -82,7 +82,7 @@ export interface GatewayConfig { auth: ClientAuth } -export interface L2TransactionApiConfig { +export interface LiveL2TransactionApiConfig { getTransactionsUrl: (startApexId: number, expectCount: number) => string getThirdPartyIdByTransactionIdUrl: (transactionId: number) => string auth: ClientAuth diff --git a/packages/backend/src/core/sync/LiveL2TransactionDownloader.test.ts b/packages/backend/src/core/sync/LiveL2TransactionDownloader.test.ts index 7364ada00..26d1be427 100644 --- a/packages/backend/src/core/sync/LiveL2TransactionDownloader.test.ts +++ b/packages/backend/src/core/sync/LiveL2TransactionDownloader.test.ts @@ -7,7 +7,7 @@ import waitForExpect from 'wait-for-expect' import { KeyValueStore } from '../../peripherals/database/KeyValueStore' import { L2TransactionRepository } from '../../peripherals/database/L2TransactionRepository' import { StateUpdateRepository } from '../../peripherals/database/StateUpdateRepository' -import { L2TransactionClient } from '../../peripherals/starkware/L2TransactionClient' +import { LiveL2TransactionClient } from '../../peripherals/starkware/LiveL2TransactionClient' import { PerpetualL2Transaction } from '../../peripherals/starkware/toPerpetualTransactions' import { Logger } from '../../tools/Logger' import { Clock } from './Clock' @@ -42,8 +42,8 @@ describe(LiveL2TransactionDownloader.name, () => { findByKey: mockFn().resolvesTo(thirdPartyId), addOrUpdate: mockFn().resolvesTo('lastSyncedThirdPartyId'), }) - const mockLiveL2TransactionClient = mockObject({ - getPerpetualTransactions: mockFn() + const mockLiveL2TransactionClient = mockObject({ + getPerpetualLiveTransactions: mockFn() .resolvesToOnce(firstTxs) .resolvesToOnce(secondTxs), }) @@ -76,7 +76,7 @@ describe(LiveL2TransactionDownloader.name, () => { expect.a(Function) ) expect( - mockLiveL2TransactionClient.getPerpetualTransactions + mockLiveL2TransactionClient.getPerpetualLiveTransactions ).toHaveBeenNthCalledWith(1, thirdPartyId + 1, 100) await waitForExpect(() => { @@ -102,7 +102,7 @@ describe(LiveL2TransactionDownloader.name, () => { }) expect( - mockLiveL2TransactionClient.getPerpetualTransactions + mockLiveL2TransactionClient.getPerpetualLiveTransactions ).toHaveBeenNthCalledWith(2, thirdPartyId + firstTxs.length + 1, 100) await waitForExpect(() => { secondTxs.forEach((tx, i) => { @@ -127,7 +127,7 @@ describe(LiveL2TransactionDownloader.name, () => { }) expect( - mockLiveL2TransactionClient.getPerpetualTransactions + mockLiveL2TransactionClient.getPerpetualLiveTransactions ).toHaveBeenExhausted() }) @@ -143,7 +143,7 @@ describe(LiveL2TransactionDownloader.name, () => { }) const liveL2TransactionDownloader = new LiveL2TransactionDownloader( - mockObject(), + mockObject(), mockL2TransactionRepository, mockObject(), mockKeyValueStore, @@ -178,7 +178,7 @@ describe(LiveL2TransactionDownloader.name, () => { }) const liveL2TransactionDownloader = new LiveL2TransactionDownloader( - mockObject(), + mockObject(), mockL2TransactionRepository, mockObject(), mockKeyValueStore, @@ -210,7 +210,7 @@ describe(LiveL2TransactionDownloader.name, () => { }) const liveL2TransactionDownloader = new LiveL2TransactionDownloader( - mockObject(), + mockObject(), mockObject(), mockStateUpdateRepository, mockObject(), @@ -234,7 +234,7 @@ describe(LiveL2TransactionDownloader.name, () => { }) const liveL2TransactionDownloader = new LiveL2TransactionDownloader( - mockObject(), + mockObject(), mockObject(), mockStateUpdateRepository, mockObject(), @@ -258,12 +258,12 @@ describe(LiveL2TransactionDownloader.name, () => { const mockL2TransactionRepository = mockObject({ findLatestIncluded: mockFn().resolvesTo(undefined), }) - const mockL2TransactionClient = mockObject({ + const mockL2TransactionClient = mockObject({ getThirdPartyIdByTransactionId: mockFn(), }) const liveL2TransactionDownloader = new LiveL2TransactionDownloader( - mockObject(), + mockObject(), mockL2TransactionRepository, mockStateUpdateRepository, mockObject(), @@ -288,12 +288,12 @@ describe(LiveL2TransactionDownloader.name, () => { const mockL2TransactionRepository = mockObject({ findLatestIncluded: mockFn().resolvesTo({ stateUpdateId: 10 }), }) - const mockL2TransactionClient = mockObject({ + const mockL2TransactionClient = mockObject({ getThirdPartyIdByTransactionId: mockFn(), }) const liveL2TransactionDownloader = new LiveL2TransactionDownloader( - mockObject(), + mockObject(), mockL2TransactionRepository, mockStateUpdateRepository, mockObject(), @@ -323,7 +323,7 @@ describe(LiveL2TransactionDownloader.name, () => { transactionId, }), }) - const mockL2TransactionClient = mockObject({ + const mockL2TransactionClient = mockObject({ getThirdPartyIdByTransactionId: mockFn().resolvesTo(undefined), }) const mockKeyValueStore = mockObject({ @@ -364,7 +364,7 @@ describe(LiveL2TransactionDownloader.name, () => { transactionId, }), }) - const mockL2TransactionClient = mockObject({ + const mockL2TransactionClient = mockObject({ getThirdPartyIdByTransactionId: mockFn().resolvesTo(thirdPartyId), }) const mockKeyValueStore = mockObject({ diff --git a/packages/backend/src/core/sync/LiveL2TransactionDownloader.ts b/packages/backend/src/core/sync/LiveL2TransactionDownloader.ts index 4e15de04a..1cf1efdd6 100644 --- a/packages/backend/src/core/sync/LiveL2TransactionDownloader.ts +++ b/packages/backend/src/core/sync/LiveL2TransactionDownloader.ts @@ -3,7 +3,7 @@ import { Knex } from 'knex' import { KeyValueStore } from '../../peripherals/database/KeyValueStore' import { L2TransactionRepository } from '../../peripherals/database/L2TransactionRepository' import { StateUpdateRepository } from '../../peripherals/database/StateUpdateRepository' -import { L2TransactionClient } from '../../peripherals/starkware/L2TransactionClient' +import { LiveL2TransactionClient } from '../../peripherals/starkware/LiveL2TransactionClient' import { PerpetualL2Transaction } from '../../peripherals/starkware/toPerpetualTransactions' import { Logger } from '../../tools/Logger' import { Clock } from './Clock' @@ -15,7 +15,7 @@ export class LiveL2TransactionDownloader { isEnabled = false constructor( - private readonly l2TransactionClient: L2TransactionClient, + private readonly l2TransactionClient: LiveL2TransactionClient, private readonly l2TransactionRepository: L2TransactionRepository, private readonly stateUpdateRepository: StateUpdateRepository, private readonly keyValueStore: KeyValueStore, @@ -103,7 +103,7 @@ export class LiveL2TransactionDownloader { this.logger.info(`Downloading live transactions from ${thirdPartyId}`) const transactions = - await this.l2TransactionClient.getPerpetualTransactions( + await this.l2TransactionClient.getPerpetualLiveTransactions( thirdPartyId, this.PAGE_SIZE ) diff --git a/packages/backend/src/peripherals/starkware/L2TransactionClient.test.ts b/packages/backend/src/peripherals/starkware/L2TransactionClient.test.ts deleted file mode 100644 index df7cc02c3..000000000 --- a/packages/backend/src/peripherals/starkware/L2TransactionClient.test.ts +++ /dev/null @@ -1,115 +0,0 @@ -import { expect, mockFn, mockObject } from 'earl' - -import { L2TransactionApiConfig } from '../../config/starkex/StarkexConfig' -import { EXAMPLE_PERPETUAL_TRANSACTIONS } from '../../test/starkwareData' -import { FetchClient } from './FetchClient' -import { L2TransactionClient } from './L2TransactionClient' -import { PerpetualL2TransactionResponse } from './schema/PerpetualL2TransactionResponse' -import { toPerpetualL2Transactions } from './toPerpetualTransactions' - -describe(L2TransactionClient.name, () => { - const getTransactionsUrl = mockFn().returns('get-transactions-url') - const getThirdPartyIdByTransactionIdUrl = mockFn().returns( - 'get-third-party-id-by-transaction-id-url' - ) - const options: L2TransactionApiConfig = { - getTransactionsUrl, - getThirdPartyIdByTransactionIdUrl, - auth: { - type: 'bearerToken', - bearerToken: 'random-token', - }, - } - - describe(L2TransactionClient.prototype.getPerpetualTransactions.name, () => { - it('should fetch transactions and parse them', async () => { - const fetchClient = mockObject({ - fetchRetry: mockFn().resolvesTo({ - json: mockFn().resolvesTo(EXAMPLE_PERPETUAL_TRANSACTIONS), - }), - }) - const transactionClient = new L2TransactionClient(options, fetchClient) - - const response = await transactionClient.getPerpetualTransactions(0, 0) - - expect(getTransactionsUrl).toHaveBeenCalledWith(0, 0) - expect(fetchClient.fetchRetry).toHaveBeenCalledWith( - 'get-transactions-url', - expect.anything() - ) - expect(response).toEqual( - toPerpetualL2Transactions( - PerpetualL2TransactionResponse.parse(EXAMPLE_PERPETUAL_TRANSACTIONS) - ) - ) - }) - - it('should return undefined if no transactions are returned', async () => { - const fetchClient = mockObject({ - fetchRetry: mockFn().resolvesTo({ - json: mockFn().resolvesTo({}), - }), - }) - const transactionClient = new L2TransactionClient(options, fetchClient) - - const response = await transactionClient.getPerpetualTransactions(0, 0) - - expect(getTransactionsUrl).toHaveBeenCalledWith(0, 0) - expect(fetchClient.fetchRetry).toHaveBeenCalledWith( - 'get-transactions-url', - expect.anything() - ) - expect(response).toEqual(undefined) - }) - }) - - describe( - L2TransactionClient.prototype.getThirdPartyIdByTransactionId.name, - () => { - it('should fetch third party id', async () => { - const textResponse = '2' - const fetchClient = mockObject({ - fetchRetry: mockFn().resolvesTo({ - text: mockFn().resolvesTo(textResponse), - }), - }) - const transactionClient = new L2TransactionClient(options, fetchClient) - - const response = await transactionClient.getThirdPartyIdByTransactionId( - 1 - ) - - expect(getThirdPartyIdByTransactionIdUrl).toHaveBeenCalledWith(1) - expect(fetchClient.fetchRetry).toHaveBeenCalledWith( - 'get-third-party-id-by-transaction-id-url', - expect.anything() - ) - expect(response).toEqual(Number(textResponse)) - }) - - it('should return undefined if 0 is returned', async () => { - it('should fetch third party id', async () => { - const fetchClient = mockObject({ - fetchRetry: mockFn().resolvesTo({ - text: mockFn().resolvesTo('0'), - }), - }) - const transactionClient = new L2TransactionClient( - options, - fetchClient - ) - - const response = - await transactionClient.getThirdPartyIdByTransactionId(1) - - expect(getThirdPartyIdByTransactionIdUrl).toHaveBeenCalledWith(1) - expect(fetchClient.fetchRetry).toHaveBeenCalledWith( - 'get-third-party-id-by-transaction-id-url', - expect.anything() - ) - expect(response).toEqual(undefined) - }) - }) - } - ) -}) diff --git a/packages/backend/src/peripherals/starkware/LiveL2TransactionClient.test.ts b/packages/backend/src/peripherals/starkware/LiveL2TransactionClient.test.ts new file mode 100644 index 000000000..bc32722fd --- /dev/null +++ b/packages/backend/src/peripherals/starkware/LiveL2TransactionClient.test.ts @@ -0,0 +1,135 @@ +import { expect, mockFn, mockObject } from 'earl' + +import { LiveL2TransactionApiConfig } from '../../config/starkex/StarkexConfig' +import { EXAMPLE_PERPETUAL_TRANSACTIONS } from '../../test/starkwareData' +import { FetchClient } from './FetchClient' +import { LiveL2TransactionClient } from './LiveL2TransactionClient' +import { PerpetualLiveL2TransactionResponse } from './schema/PerpetualLiveL2TransactionResponse' +import { toPerpetualL2Transactions } from './toPerpetualTransactions' + +describe(LiveL2TransactionClient.name, () => { + const getTransactionsUrl = mockFn().returns('get-transactions-url') + const getThirdPartyIdByTransactionIdUrl = mockFn().returns( + 'get-third-party-id-by-transaction-id-url' + ) + const options: LiveL2TransactionApiConfig = { + getTransactionsUrl, + getThirdPartyIdByTransactionIdUrl, + auth: { + type: 'bearerToken', + bearerToken: 'random-token', + }, + } + + describe( + LiveL2TransactionClient.prototype.getPerpetualLiveTransactions.name, + () => { + it('should fetch transactions and parse them', async () => { + const fetchClient = mockObject({ + fetchRetry: mockFn().resolvesTo({ + json: mockFn().resolvesTo(EXAMPLE_PERPETUAL_TRANSACTIONS), + }), + }) + const transactionClient = new LiveL2TransactionClient( + options, + fetchClient + ) + + const response = await transactionClient.getPerpetualLiveTransactions( + 0, + 0 + ) + + expect(getTransactionsUrl).toHaveBeenCalledWith(0, 0) + expect(fetchClient.fetchRetry).toHaveBeenCalledWith( + 'get-transactions-url', + expect.anything() + ) + expect(response).toEqual( + toPerpetualL2Transactions( + PerpetualLiveL2TransactionResponse.parse( + EXAMPLE_PERPETUAL_TRANSACTIONS + ) + ) + ) + }) + + it('should return undefined if no transactions are returned', async () => { + const fetchClient = mockObject({ + fetchRetry: mockFn().resolvesTo({ + json: mockFn().resolvesTo({}), + }), + }) + const transactionClient = new LiveL2TransactionClient( + options, + fetchClient + ) + + const response = await transactionClient.getPerpetualLiveTransactions( + 0, + 0 + ) + + expect(getTransactionsUrl).toHaveBeenCalledWith(0, 0) + expect(fetchClient.fetchRetry).toHaveBeenCalledWith( + 'get-transactions-url', + expect.anything() + ) + expect(response).toEqual(undefined) + }) + } + ) + + describe( + LiveL2TransactionClient.prototype.getThirdPartyIdByTransactionId.name, + () => { + it('should fetch third party id', async () => { + const textResponse = '2' + const fetchClient = mockObject({ + fetchRetry: mockFn().resolvesTo({ + text: mockFn().resolvesTo(textResponse), + }), + }) + const transactionClient = new LiveL2TransactionClient( + options, + fetchClient + ) + + const response = await transactionClient.getThirdPartyIdByTransactionId( + 1 + ) + + expect(getThirdPartyIdByTransactionIdUrl).toHaveBeenCalledWith(1) + expect(fetchClient.fetchRetry).toHaveBeenCalledWith( + 'get-third-party-id-by-transaction-id-url', + expect.anything() + ) + expect(response).toEqual(Number(textResponse)) + }) + + it('should return undefined if 0 is returned', async () => { + it('should fetch third party id', async () => { + const fetchClient = mockObject({ + fetchRetry: mockFn().resolvesTo({ + text: mockFn().resolvesTo('0'), + }), + }) + const transactionClient = new LiveL2TransactionClient( + options, + fetchClient + ) + + const response = + await transactionClient.getThirdPartyIdByTransactionId(1) + + expect(getThirdPartyIdByTransactionIdUrl).toHaveBeenCalledWith(1) + expect(fetchClient.fetchRetry).toHaveBeenCalledWith( + 'get-third-party-id-by-transaction-id-url', + expect.anything() + ) + expect(response).toEqual(undefined) + }) + }) + } + ) +}) diff --git a/packages/backend/src/peripherals/starkware/L2TransactionClient.ts b/packages/backend/src/peripherals/starkware/LiveL2TransactionClient.ts similarity index 72% rename from packages/backend/src/peripherals/starkware/L2TransactionClient.ts rename to packages/backend/src/peripherals/starkware/LiveL2TransactionClient.ts index f6ff99e61..321e2cdde 100644 --- a/packages/backend/src/peripherals/starkware/L2TransactionClient.ts +++ b/packages/backend/src/peripherals/starkware/LiveL2TransactionClient.ts @@ -1,14 +1,14 @@ import isEmpty from 'lodash/isEmpty' -import { L2TransactionApiConfig } from '../../config/starkex/StarkexConfig' +import { LiveL2TransactionApiConfig } from '../../config/starkex/StarkexConfig' import { BaseClient } from './BaseClient' import { FetchClient } from './FetchClient' -import { PerpetualL2TransactionResponse } from './schema/PerpetualL2TransactionResponse' +import { PerpetualLiveL2TransactionResponse } from './schema/PerpetualLiveL2TransactionResponse' import { toPerpetualL2Transactions } from './toPerpetualTransactions' -export class L2TransactionClient extends BaseClient { +export class LiveL2TransactionClient extends BaseClient { constructor( - private readonly options: L2TransactionApiConfig, + private readonly options: LiveL2TransactionApiConfig, private readonly fetchClient: FetchClient ) { super(options.auth) @@ -24,14 +24,14 @@ export class L2TransactionClient extends BaseClient { return thirdPartyId !== 0 ? thirdPartyId : undefined } - async getPerpetualTransactions(startId: number, pageSize: number) { + async getPerpetualLiveTransactions(startId: number, pageSize: number) { const data = await this.getTransactions(startId, pageSize) if (isEmpty(data)) { return undefined } - const parsed = PerpetualL2TransactionResponse.parse(data) + const parsed = PerpetualLiveL2TransactionResponse.parse(data) return toPerpetualL2Transactions(parsed) } diff --git a/packages/backend/src/peripherals/starkware/schema/PerpetualL2TransactionResponse.ts b/packages/backend/src/peripherals/starkware/schema/PerpetualL2TransactionResponse.ts deleted file mode 100644 index c23d51130..000000000 --- a/packages/backend/src/peripherals/starkware/schema/PerpetualL2TransactionResponse.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { z } from 'zod' - -import { PerpetualL2Transaction } from './PerpetualBatchInfoResponse' - -const PerpetualL2TransactionResponseTransactionInfo = z.strictObject({ - tx: PerpetualL2Transaction, - tx_id: z.number(), -}) -const PerpetualL2TransactionResponseTransaction = z.strictObject({ - apex_id: z.number(), - tx_info: z - .string() - .transform((s) => - PerpetualL2TransactionResponseTransactionInfo.parse(JSON.parse(s)) - ), -}) - -export type PerpetualL2TransactionResponse = z.infer< - typeof PerpetualL2TransactionResponse -> -export const PerpetualL2TransactionResponse = z.strictObject({ - count: z.number(), - txs: z.array(PerpetualL2TransactionResponseTransaction), -}) diff --git a/packages/backend/src/peripherals/starkware/schema/PerpetualL2TransactionResponse.test.ts b/packages/backend/src/peripherals/starkware/schema/PerpetualLiveL2TransactionResponse.test.ts similarity index 60% rename from packages/backend/src/peripherals/starkware/schema/PerpetualL2TransactionResponse.test.ts rename to packages/backend/src/peripherals/starkware/schema/PerpetualLiveL2TransactionResponse.test.ts index 3b0149f8b..24e1f203d 100644 --- a/packages/backend/src/peripherals/starkware/schema/PerpetualL2TransactionResponse.test.ts +++ b/packages/backend/src/peripherals/starkware/schema/PerpetualLiveL2TransactionResponse.test.ts @@ -1,12 +1,12 @@ import { expect } from 'earl' import { EXAMPLE_PERPETUAL_TRANSACTIONS } from '../../../test/starkwareData' -import { PerpetualL2TransactionResponse } from './PerpetualL2TransactionResponse' +import { PerpetualLiveL2TransactionResponse } from './PerpetualLiveL2TransactionResponse' describe('PerpetualL2TransactionResponse', () => { it('can parse real data', () => { const fn = () => - PerpetualL2TransactionResponse.parse(EXAMPLE_PERPETUAL_TRANSACTIONS) + PerpetualLiveL2TransactionResponse.parse(EXAMPLE_PERPETUAL_TRANSACTIONS) expect(fn).not.toThrow() }) }) diff --git a/packages/backend/src/peripherals/starkware/schema/PerpetualLiveL2TransactionResponse.ts b/packages/backend/src/peripherals/starkware/schema/PerpetualLiveL2TransactionResponse.ts new file mode 100644 index 000000000..2166afedb --- /dev/null +++ b/packages/backend/src/peripherals/starkware/schema/PerpetualLiveL2TransactionResponse.ts @@ -0,0 +1,24 @@ +import { z } from 'zod' + +import { PerpetualL2Transaction } from './PerpetualBatchInfoResponse' + +const PerpetualLiveL2TransactionResponseTransactionInfo = z.strictObject({ + tx: PerpetualL2Transaction, + tx_id: z.number(), +}) +const PerpetualLiveL2TransactionResponseTransaction = z.strictObject({ + apex_id: z.number(), + tx_info: z + .string() + .transform((s) => + PerpetualLiveL2TransactionResponseTransactionInfo.parse(JSON.parse(s)) + ), +}) + +export type PerpetualLiveL2TransactionResponse = z.infer< + typeof PerpetualLiveL2TransactionResponse +> +export const PerpetualLiveL2TransactionResponse = z.strictObject({ + count: z.number(), + txs: z.array(PerpetualLiveL2TransactionResponseTransaction), +}) diff --git a/packages/backend/src/peripherals/starkware/toPerpetualTransactions.ts b/packages/backend/src/peripherals/starkware/toPerpetualTransactions.ts index 5963c777b..63964b0c9 100644 --- a/packages/backend/src/peripherals/starkware/toPerpetualTransactions.ts +++ b/packages/backend/src/peripherals/starkware/toPerpetualTransactions.ts @@ -15,11 +15,11 @@ import { import { AssetOraclePrice, OrderTypeResponse, - PerpetualL2Transaction as TransactionSchema, SignatureResponse, SignedOraclePrice, + PerpetualL2Transaction as TransactionSchema, } from './schema/PerpetualBatchInfoResponse' -import { PerpetualL2TransactionResponse } from './schema/PerpetualL2TransactionResponse' +import { PerpetualLiveL2TransactionResponse } from './schema/PerpetualLiveL2TransactionResponse' export interface PerpetualL2Transaction { thirdPartyId: number transactionId: number @@ -27,7 +27,7 @@ export interface PerpetualL2Transaction { } export function toPerpetualL2Transactions( - response: PerpetualL2TransactionResponse + response: PerpetualLiveL2TransactionResponse ): PerpetualL2Transaction[] { return response.txs.map((tx) => { return { From 60f9680ec82bdbba008444bd3f468336ca067d76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20T=C3=B3rz?= Date: Thu, 20 Jul 2023 11:14:23 +0200 Subject: [PATCH 13/19] add tests --- .../sync/LiveL2TransactionDownloader.test.ts | 2 +- .../database/L2TransactionRepository.test.ts | 50 +++++++++++++++++++ .../database/L2TransactionRepository.ts | 4 ++ 3 files changed, 55 insertions(+), 1 deletion(-) diff --git a/packages/backend/src/core/sync/LiveL2TransactionDownloader.test.ts b/packages/backend/src/core/sync/LiveL2TransactionDownloader.test.ts index 26d1be427..f9ec5e83d 100644 --- a/packages/backend/src/core/sync/LiveL2TransactionDownloader.test.ts +++ b/packages/backend/src/core/sync/LiveL2TransactionDownloader.test.ts @@ -4,12 +4,12 @@ import { Knex } from 'knex' import range from 'lodash/range' import waitForExpect from 'wait-for-expect' +import { Logger } from '@l2beat/backend-tools' import { KeyValueStore } from '../../peripherals/database/KeyValueStore' import { L2TransactionRepository } from '../../peripherals/database/L2TransactionRepository' import { StateUpdateRepository } from '../../peripherals/database/StateUpdateRepository' import { LiveL2TransactionClient } from '../../peripherals/starkware/LiveL2TransactionClient' import { PerpetualL2Transaction } from '../../peripherals/starkware/toPerpetualTransactions' -import { Logger } from '../../tools/Logger' import { Clock } from './Clock' import { LiveL2TransactionDownloader } from './LiveL2TransactionDownloader' diff --git a/packages/backend/src/peripherals/database/L2TransactionRepository.test.ts b/packages/backend/src/peripherals/database/L2TransactionRepository.test.ts index fc830f15b..c83babaa4 100644 --- a/packages/backend/src/peripherals/database/L2TransactionRepository.test.ts +++ b/packages/backend/src/peripherals/database/L2TransactionRepository.test.ts @@ -691,6 +691,56 @@ describe(L2TransactionRepository.name, () => { } ) + describe( + L2TransactionRepository.prototype.findOldestByTransactionId.name, + () => { + it('returns undefined if there are no transactions', async () => { + const transaction = await repository.findOldestByTransactionId(1234) + + expect(transaction).toBeNullish() + }) + + it('returns the oldest transaction', async () => { + const id = await repository.add(genericDepositTransaction) + await repository.add({ + ...genericDepositTransaction, + data: { ...genericDepositTransaction.data, amount: 1000n }, + }) + + const transaction = await repository.findOldestByTransactionId( + genericDepositTransaction.transactionId + ) + + expect(transaction?.id).toEqual(id) + }) + } + ) + + describe(L2TransactionRepository.prototype.findLatestIncluded.name, () => { + it('returns undefined if there are no included transactions', async () => { + await repository.add({ + ...genericDepositTransaction, + stateUpdateId: undefined, + }) + + const transaction = await repository.findLatestIncluded() + + expect(transaction).toBeNullish() + }) + + it('returns the latest included transaction', async () => { + const id = await repository.add(genericWithdrawalToAddressTransaction) + await repository.add({ + ...genericDepositTransaction, + stateUpdateId: undefined, + }) + + const transaction = await repository.findLatestIncluded() + + expect(transaction?.id).toEqual(id) + }) + }) + describe(L2TransactionRepository.prototype.deleteAfterBlock.name, () => { it('deletes transactions after a block', async () => { const record = { diff --git a/packages/backend/src/peripherals/database/L2TransactionRepository.ts b/packages/backend/src/peripherals/database/L2TransactionRepository.ts index 5cd91e73d..fe0a9bce4 100644 --- a/packages/backend/src/peripherals/database/L2TransactionRepository.ts +++ b/packages/backend/src/peripherals/database/L2TransactionRepository.ts @@ -73,6 +73,10 @@ export class L2TransactionRepository extends BaseRepository { this.findAggregatedByTransactionId ) this.findLatestStateUpdateId = this.wrapFind(this.findLatestStateUpdateId) + this.findOldestByTransactionId = this.wrapFind( + this.findOldestByTransactionId + ) + this.findLatestIncluded = this.wrapFind(this.findLatestIncluded) this.deleteAfterBlock = this.wrapDelete(this.deleteAfterBlock) this.deleteByTransactionIds = this.wrapDelete(this.deleteByTransactionIds) this.deleteAll = this.wrapDelete(this.deleteAll) From 94b5e76ba21dea9c3b4b5629004f6f6b7a0209e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20T=C3=B3rz?= Date: Thu, 20 Jul 2023 11:21:42 +0200 Subject: [PATCH 14/19] fix build issues --- .../backend/src/core/sync/LiveL2TransactionDownloader.test.ts | 2 +- packages/backend/src/core/sync/LiveL2TransactionDownloader.ts | 2 +- .../src/peripherals/starkware/toPerpetualTransactions.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/backend/src/core/sync/LiveL2TransactionDownloader.test.ts b/packages/backend/src/core/sync/LiveL2TransactionDownloader.test.ts index f9ec5e83d..6fb34c0a5 100644 --- a/packages/backend/src/core/sync/LiveL2TransactionDownloader.test.ts +++ b/packages/backend/src/core/sync/LiveL2TransactionDownloader.test.ts @@ -1,10 +1,10 @@ import { StarkKey } from '@explorer/types' +import { Logger } from '@l2beat/backend-tools' import { expect, mockFn, mockObject } from 'earl' import { Knex } from 'knex' import range from 'lodash/range' import waitForExpect from 'wait-for-expect' -import { Logger } from '@l2beat/backend-tools' import { KeyValueStore } from '../../peripherals/database/KeyValueStore' import { L2TransactionRepository } from '../../peripherals/database/L2TransactionRepository' import { StateUpdateRepository } from '../../peripherals/database/StateUpdateRepository' diff --git a/packages/backend/src/core/sync/LiveL2TransactionDownloader.ts b/packages/backend/src/core/sync/LiveL2TransactionDownloader.ts index 1cf1efdd6..09e2e0924 100644 --- a/packages/backend/src/core/sync/LiveL2TransactionDownloader.ts +++ b/packages/backend/src/core/sync/LiveL2TransactionDownloader.ts @@ -1,11 +1,11 @@ import { Knex } from 'knex' +import { Logger } from '@l2beat/backend-tools' import { KeyValueStore } from '../../peripherals/database/KeyValueStore' import { L2TransactionRepository } from '../../peripherals/database/L2TransactionRepository' import { StateUpdateRepository } from '../../peripherals/database/StateUpdateRepository' import { LiveL2TransactionClient } from '../../peripherals/starkware/LiveL2TransactionClient' import { PerpetualL2Transaction } from '../../peripherals/starkware/toPerpetualTransactions' -import { Logger } from '../../tools/Logger' import { Clock } from './Clock' export class LiveL2TransactionDownloader { diff --git a/packages/backend/src/peripherals/starkware/toPerpetualTransactions.ts b/packages/backend/src/peripherals/starkware/toPerpetualTransactions.ts index 63964b0c9..90164ef21 100644 --- a/packages/backend/src/peripherals/starkware/toPerpetualTransactions.ts +++ b/packages/backend/src/peripherals/starkware/toPerpetualTransactions.ts @@ -15,9 +15,9 @@ import { import { AssetOraclePrice, OrderTypeResponse, + PerpetualL2Transaction as TransactionSchema, SignatureResponse, SignedOraclePrice, - PerpetualL2Transaction as TransactionSchema, } from './schema/PerpetualBatchInfoResponse' import { PerpetualLiveL2TransactionResponse } from './schema/PerpetualLiveL2TransactionResponse' export interface PerpetualL2Transaction { From 492b885c6556185af76c9348a92135d81333c6a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20T=C3=B3rz?= Date: Thu, 20 Jul 2023 11:22:45 +0200 Subject: [PATCH 15/19] fix after merge bug --- .../src/peripherals/database/L2TransactionRepository.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/peripherals/database/L2TransactionRepository.ts b/packages/backend/src/peripherals/database/L2TransactionRepository.ts index fe0a9bce4..81a1a55ef 100644 --- a/packages/backend/src/peripherals/database/L2TransactionRepository.ts +++ b/packages/backend/src/peripherals/database/L2TransactionRepository.ts @@ -399,8 +399,10 @@ export class L2TransactionRepository extends BaseRepository { return row ? toRecord(row) : undefined } - async findLatestStateUpdateId(): Promise { - const knex = await this.knex() + async findLatestStateUpdateId( + trx?: Knex.Transaction + ): Promise { + const knex = await this.knex(trx) const results = await knex('l2_transactions') .select('state_update_id') .whereNotNull('state_update_id') From 38d7dcf9d5955c7b6227fcd8e9c19adce0b1178c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20T=C3=B3rz?= Date: Thu, 20 Jul 2023 14:59:53 +0200 Subject: [PATCH 16/19] add live txs count --- .../src/api/controllers/HomeController.ts | 34 ++- .../src/api/controllers/UserController.ts | 21 +- .../database/L2TransactionRepository.test.ts | 270 ++++++++++++++++++ .../database/L2TransactionRepository.ts | 58 ++++ 4 files changed, 365 insertions(+), 18 deletions(-) diff --git a/packages/backend/src/api/controllers/HomeController.ts b/packages/backend/src/api/controllers/HomeController.ts index 00e1788e4..920f69f8a 100644 --- a/packages/backend/src/api/controllers/HomeController.ts +++ b/packages/backend/src/api/controllers/HomeController.ts @@ -45,6 +45,7 @@ export class HomeController { const [ l2Transactions, lastStateDetailsWithL2TransactionsStatistics, + liveL2TransactionsStatistics, stateUpdates, stateUpdatesCount, forcedUserTransactions, @@ -54,6 +55,7 @@ export class HomeController { ] = await Promise.all([ this.l2TransactionRepository.getPaginatedWithoutMulti(paginationOpts), this.preprocessedStateDetailsRepository.findLastWithL2TransactionsStatistics(), + this.l2TransactionRepository.getLiveStatistics(), this.preprocessedStateDetailsRepository.getPaginated(paginationOpts), this.preprocessedStateDetailsRepository.countAll(), this.userTransactionRepository.getPaginated({ @@ -82,13 +84,16 @@ export class HomeController { forcedTransactionCount: update.forcedTransactionCount, })) + const totalL2Transactions = + sumUpTransactionCount( + lastStateDetailsWithL2TransactionsStatistics?.cumulativeL2TransactionsStatistics + ) + sumUpTransactionCount(liveL2TransactionsStatistics) + const content = renderHomePage({ context, tutorials: [], // explicitly no tutorials l2Transactions: l2Transactions.map(l2TransactionToEntry), - totalL2Transactions: sumUpTransactionCount( - lastStateDetailsWithL2TransactionsStatistics?.cumulativeL2TransactionsStatistics - ), + totalL2Transactions, stateUpdates: stateUpdateEntries, totalStateUpdates: stateUpdatesCount, forcedTransactions: forcedTransactionEntries, @@ -109,18 +114,25 @@ export class HomeController { ): Promise { const context = await this.pageContextService.getPageContext(givenUser) - const [l2Transactions, lastStateDetailsWithL2TransactionsStatistics] = - await Promise.all([ - this.l2TransactionRepository.getPaginatedWithoutMulti(pagination), - this.preprocessedStateDetailsRepository.findLastWithL2TransactionsStatistics(), - ]) + const [ + l2Transactions, + lastStateDetailsWithL2TransactionsStatistics, + liveL2TransactionsStatistics, + ] = await Promise.all([ + this.l2TransactionRepository.getPaginatedWithoutMulti(pagination), + this.preprocessedStateDetailsRepository.findLastWithL2TransactionsStatistics(), + this.l2TransactionRepository.getLiveStatistics(), + ]) + + const totalL2Transactions = + sumUpTransactionCount( + lastStateDetailsWithL2TransactionsStatistics?.cumulativeL2TransactionsStatistics + ) + sumUpTransactionCount(liveL2TransactionsStatistics) const content = renderHomeL2TransactionsPage({ context, l2Transactions: l2Transactions.map(l2TransactionToEntry), - total: sumUpTransactionCount( - lastStateDetailsWithL2TransactionsStatistics?.cumulativeL2TransactionsStatistics - ), + total: totalL2Transactions, ...pagination, }) return { type: 'success', content } diff --git a/packages/backend/src/api/controllers/UserController.ts b/packages/backend/src/api/controllers/UserController.ts index 430061e11..0400f7881 100644 --- a/packages/backend/src/api/controllers/UserController.ts +++ b/packages/backend/src/api/controllers/UserController.ts @@ -129,6 +129,7 @@ export class UserController { history, l2Transactions, preprocessedUserL2TransactionsStatistics, + liveL2TransactionStatistics, sentTransactions, userTransactions, userTransactionsCount, @@ -155,6 +156,7 @@ export class UserController { this.preprocessedUserL2TransactionsStatisticsRepository.findLatestByStarkKey( starkKey ), + this.l2TransactionRepository.getLiveStatisticsByStarkKey(starkKey), this.sentTransactionRepository.getByStarkKey(starkKey), this.userTransactionRepository.getByStarkKey( starkKey, @@ -214,14 +216,16 @@ export class UserController { starkKey ) + const totalL2Transactions = sumUpTransactionCount( + preprocessedUserL2TransactionsStatistics?.cumulativeL2TransactionsStatistics + ) + sumUpTransactionCount(liveL2TransactionStatistics) + const content = renderUserPage({ context, starkKey, ethereumAddress: registeredUser?.ethAddress, l2Transactions: l2Transactions.map(l2TransactionToEntry), - totalL2Transactions: sumUpTransactionCount( - preprocessedUserL2TransactionsStatistics?.cumulativeL2TransactionsStatistics - ), + totalL2Transactions, withdrawableAssets: withdrawableAssets.map((asset) => ({ asset: { hashOrId: @@ -318,7 +322,7 @@ export class UserController { pagination: PaginationOptions ): Promise { const context = await this.pageContextService.getPageContext(givenUser) - const [l2Transactions, preprocessedUserL2TransactionsStatistics] = + const [l2Transactions, preprocessedUserL2TransactionsStatistics, liveL2TransactionStatistics] = await Promise.all([ this.l2TransactionRepository.getUserSpecificPaginated( starkKey, @@ -327,15 +331,18 @@ export class UserController { this.preprocessedUserL2TransactionsStatisticsRepository.findLatestByStarkKey( starkKey ), + this.l2TransactionRepository.getLiveStatisticsByStarkKey(starkKey), ]) + const totalL2Transactions = sumUpTransactionCount( + preprocessedUserL2TransactionsStatistics?.cumulativeL2TransactionsStatistics + ) + sumUpTransactionCount(liveL2TransactionStatistics) + const content = renderUserL2TransactionsPage({ context, starkKey, l2Transactions: l2Transactions.map(l2TransactionToEntry), - total: sumUpTransactionCount( - preprocessedUserL2TransactionsStatistics?.cumulativeL2TransactionsStatistics - ), + total: totalL2Transactions, ...pagination, }) return { type: 'success', content } diff --git a/packages/backend/src/peripherals/database/L2TransactionRepository.test.ts b/packages/backend/src/peripherals/database/L2TransactionRepository.test.ts index c83babaa4..fc5b64d4c 100644 --- a/packages/backend/src/peripherals/database/L2TransactionRepository.test.ts +++ b/packages/backend/src/peripherals/database/L2TransactionRepository.test.ts @@ -741,6 +741,276 @@ describe(L2TransactionRepository.name, () => { }) }) + describe(L2TransactionRepository.prototype.getLiveStatistics.name, () => { + it('returns zeros if there are no transactions', async () => { + const statistics = await repository.getLiveStatistics() + + expect(statistics).toEqual({ + depositCount: 0, + withdrawalToAddressCount: 0, + forcedWithdrawalCount: 0, + tradeCount: 0, + forcedTradeCount: 0, + transferCount: 0, + conditionalTransferCount: 0, + liquidateCount: 0, + deleverageCount: 0, + fundingTickCount: 0, + oraclePricesTickCount: 0, + multiTransactionCount: 0, + replacedTransactionsCount: 0, + }) + }) + + it('returns correct statistics', async () => { + await repository.add(genericDepositTransaction) + await repository.add({ + ...genericDepositTransaction, + transactionId: 1222, + stateUpdateId: undefined, + }) + await repository.add({ + ...genericMultiTransaction([genericDepositTransaction.data]), + transactionId: 1223, + stateUpdateId: undefined, + }) + await repository.add({ + ...genericDepositTransaction, + transactionId: 1222, + stateUpdateId: undefined, + }) + + const statistics = await repository.getLiveStatistics() + + expect(statistics).toEqual({ + depositCount: 3, + withdrawalToAddressCount: 0, + forcedWithdrawalCount: 0, + tradeCount: 0, + forcedTradeCount: 0, + transferCount: 0, + conditionalTransferCount: 0, + liquidateCount: 0, + deleverageCount: 0, + fundingTickCount: 0, + oraclePricesTickCount: 0, + multiTransactionCount: 1, + replacedTransactionsCount: 1, + }) + }) + }) + + describe( + L2TransactionRepository.prototype.getLiveStatisticsByStarkKey.name, + () => { + it('returns zeros if there are no transactions', async () => { + await repository.add(genericDepositTransaction) + const statistics = await repository.getLiveStatisticsByStarkKey( + StarkKey.fake() + ) + + expect(statistics).toEqual({ + depositCount: 0, + withdrawalToAddressCount: 0, + forcedWithdrawalCount: 0, + tradeCount: 0, + forcedTradeCount: 0, + transferCount: 0, + conditionalTransferCount: 0, + liquidateCount: 0, + deleverageCount: 0, + fundingTickCount: 0, + oraclePricesTickCount: 0, + replacedTransactionsCount: 0, + }) + }) + + it('returns correct statistics', async () => { + await repository.add(genericDepositTransaction) + await repository.add({ + ...genericDepositTransaction, + transactionId: 1292, + stateUpdateId: undefined, + data: { + ...genericDepositTransaction.data, + starkKey: StarkKey.fake(), + }, + }) + await repository.add({ + ...genericDepositTransaction, + transactionId: 1222, + stateUpdateId: undefined, + }) + await repository.add({ + ...genericMultiTransaction([genericDepositTransaction.data]), + transactionId: 1223, + stateUpdateId: undefined, + }) + await repository.add({ + ...genericDepositTransaction, + transactionId: 1222, + stateUpdateId: undefined, + }) + + const statistics = await repository.getLiveStatisticsByStarkKey( + genericDepositTransaction.data.starkKey + ) + + expect(statistics).toEqual({ + depositCount: 3, + withdrawalToAddressCount: 0, + forcedWithdrawalCount: 0, + tradeCount: 0, + forcedTradeCount: 0, + transferCount: 0, + conditionalTransferCount: 0, + liquidateCount: 0, + deleverageCount: 0, + fundingTickCount: 0, + oraclePricesTickCount: 0, + replacedTransactionsCount: 1, + }) + }) + } + ) + + describe( + L2TransactionRepository.prototype.getStatisticsByStateUpdateId.name, + () => { + it('returns zeros if there are no transactions', async () => { + await repository.add({ ...genericDepositTransaction, stateUpdateId: 1 }) + const statistics = await repository.getStatisticsByStateUpdateId(2) + + expect(statistics).toEqual({ + depositCount: 0, + withdrawalToAddressCount: 0, + forcedWithdrawalCount: 0, + tradeCount: 0, + forcedTradeCount: 0, + transferCount: 0, + conditionalTransferCount: 0, + liquidateCount: 0, + deleverageCount: 0, + fundingTickCount: 0, + oraclePricesTickCount: 0, + multiTransactionCount: 0, + replacedTransactionsCount: 0, + }) + }) + + it('returns correct statistics', async () => { + await repository.add({ + ...genericDepositTransaction, + transactionId: 1199, + stateUpdateId: 1, + }) + await repository.add({ ...genericDepositTransaction, stateUpdateId: 2 }) + await repository.add({ ...genericDepositTransaction, stateUpdateId: 2 }) + await repository.add({ + ...genericMultiTransaction([genericDepositTransaction.data]), + stateUpdateId: 2, + }) + await repository.add({ + ...genericDepositTransaction, + transactionId: 2000, + stateUpdateId: 2, + }) + + const statistics = await repository.getStatisticsByStateUpdateId(2) + + expect(statistics).toEqual({ + depositCount: 4, + withdrawalToAddressCount: 0, + forcedWithdrawalCount: 0, + tradeCount: 0, + forcedTradeCount: 0, + transferCount: 0, + conditionalTransferCount: 0, + liquidateCount: 0, + deleverageCount: 0, + fundingTickCount: 0, + oraclePricesTickCount: 0, + multiTransactionCount: 1, + replacedTransactionsCount: 1, + }) + }) + } + ) + + describe( + L2TransactionRepository.prototype.getStatisticsByStateUpdateIdAndStarkKey + .name, + () => { + it('returns 0 if there are no transactions', async () => { + await repository.add({ ...genericDepositTransaction, stateUpdateId: 1 }) + const statistics = + await repository.getStatisticsByStateUpdateIdAndStarkKey( + 2, + StarkKey.fake() + ) + + expect(statistics).toEqual({ + depositCount: 0, + withdrawalToAddressCount: 0, + forcedWithdrawalCount: 0, + tradeCount: 0, + forcedTradeCount: 0, + transferCount: 0, + conditionalTransferCount: 0, + liquidateCount: 0, + deleverageCount: 0, + fundingTickCount: 0, + oraclePricesTickCount: 0, + replacedTransactionsCount: 0, + }) + }) + + it('returns correct statistics', async () => { + await repository.add({ + ...genericDepositTransaction, + transactionId: 1199, + stateUpdateId: 1, + data: { + ...genericDepositTransaction.data, + starkKey: StarkKey.fake(), + }, + }) + await repository.add({ ...genericDepositTransaction, stateUpdateId: 2 }) + await repository.add({ ...genericDepositTransaction, stateUpdateId: 2 }) + await repository.add({ + ...genericMultiTransaction([genericDepositTransaction.data]), + stateUpdateId: 2, + }) + await repository.add({ + ...genericDepositTransaction, + transactionId: 2000, + stateUpdateId: 2, + }) + + const statistics = + await repository.getStatisticsByStateUpdateIdAndStarkKey( + 2, + genericDepositTransaction.data.starkKey + ) + + expect(statistics).toEqual({ + depositCount: 4, + withdrawalToAddressCount: 0, + forcedWithdrawalCount: 0, + tradeCount: 0, + forcedTradeCount: 0, + transferCount: 0, + conditionalTransferCount: 0, + liquidateCount: 0, + deleverageCount: 0, + fundingTickCount: 0, + oraclePricesTickCount: 0, + replacedTransactionsCount: 1, + }) + }) + } + ) + describe(L2TransactionRepository.prototype.deleteAfterBlock.name, () => { it('deletes transactions after a block', async () => { const record = { diff --git a/packages/backend/src/peripherals/database/L2TransactionRepository.ts b/packages/backend/src/peripherals/database/L2TransactionRepository.ts index 81a1a55ef..2fc9b482f 100644 --- a/packages/backend/src/peripherals/database/L2TransactionRepository.ts +++ b/packages/backend/src/peripherals/database/L2TransactionRepository.ts @@ -245,6 +245,64 @@ export class L2TransactionRepository extends BaseRepository { return uniqStarkKeys.map((starkKey) => StarkKey(starkKey)) } + async getLiveStatistics(): Promise { + const knex = await this.knex() + + const countGroupedByType = (await knex('l2_transactions') + .select('type') + .whereNull('state_update_id') + .count() + .groupBy('type')) as { + type: PerpetualL2TransactionData['type'] + count: number + }[] + + const [replaced] = await knex('l2_transactions') + .whereNull('state_update_id') + .andWhere({ state: 'replaced' }) + .count() + + return toPreprocessedL2TransactionsStatistics( + countGroupedByType, + Number(replaced?.count ?? 0) + ) + } + + async getLiveStatisticsByStarkKey( + starkKey: StarkKey + ): Promise { + const knex = await this.knex() + + const countGroupedByType = (await knex('l2_transactions') + .select('type') + .whereNull('state_update_id') + .andWhere((qB) => + qB + .where({ stark_key_a: starkKey.toString() }) + .orWhere({ stark_key_b: starkKey.toString() }) + ) + .count() + .groupBy('type')) as { + type: Exclude + count: number + }[] + + const [replaced] = await knex('l2_transactions') + .whereNull('state_update_id') + .andWhere({ state: 'replaced' }) + .andWhere((qB) => + qB + .where({ stark_key_a: starkKey.toString() }) + .orWhere({ stark_key_b: starkKey.toString() }) + ) + .count() + + return toPreprocessedUserL2TransactionsStatistics( + countGroupedByType, + Number(replaced?.count ?? 0) + ) + } + async getStatisticsByStateUpdateId( stateUpdateId: number, trx?: Knex.Transaction From 03bf0afebd789dd36fd5656f616bc23f52028de1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20T=C3=B3rz?= Date: Fri, 21 Jul 2023 10:50:18 +0200 Subject: [PATCH 17/19] fix linter issues --- .../src/api/controllers/UserController.ts | 39 +++++++++++-------- .../core/sync/LiveL2TransactionDownloader.ts | 2 +- .../database/L2TransactionRepository.ts | 2 + 3 files changed, 25 insertions(+), 18 deletions(-) diff --git a/packages/backend/src/api/controllers/UserController.ts b/packages/backend/src/api/controllers/UserController.ts index 0400f7881..0dba08105 100644 --- a/packages/backend/src/api/controllers/UserController.ts +++ b/packages/backend/src/api/controllers/UserController.ts @@ -216,9 +216,10 @@ export class UserController { starkKey ) - const totalL2Transactions = sumUpTransactionCount( - preprocessedUserL2TransactionsStatistics?.cumulativeL2TransactionsStatistics - ) + sumUpTransactionCount(liveL2TransactionStatistics) + const totalL2Transactions = + sumUpTransactionCount( + preprocessedUserL2TransactionsStatistics?.cumulativeL2TransactionsStatistics + ) + sumUpTransactionCount(liveL2TransactionStatistics) const content = renderUserPage({ context, @@ -322,21 +323,25 @@ export class UserController { pagination: PaginationOptions ): Promise { const context = await this.pageContextService.getPageContext(givenUser) - const [l2Transactions, preprocessedUserL2TransactionsStatistics, liveL2TransactionStatistics] = - await Promise.all([ - this.l2TransactionRepository.getUserSpecificPaginated( - starkKey, - pagination - ), - this.preprocessedUserL2TransactionsStatisticsRepository.findLatestByStarkKey( - starkKey - ), - this.l2TransactionRepository.getLiveStatisticsByStarkKey(starkKey), - ]) + const [ + l2Transactions, + preprocessedUserL2TransactionsStatistics, + liveL2TransactionStatistics, + ] = await Promise.all([ + this.l2TransactionRepository.getUserSpecificPaginated( + starkKey, + pagination + ), + this.preprocessedUserL2TransactionsStatisticsRepository.findLatestByStarkKey( + starkKey + ), + this.l2TransactionRepository.getLiveStatisticsByStarkKey(starkKey), + ]) - const totalL2Transactions = sumUpTransactionCount( - preprocessedUserL2TransactionsStatistics?.cumulativeL2TransactionsStatistics - ) + sumUpTransactionCount(liveL2TransactionStatistics) + const totalL2Transactions = + sumUpTransactionCount( + preprocessedUserL2TransactionsStatistics?.cumulativeL2TransactionsStatistics + ) + sumUpTransactionCount(liveL2TransactionStatistics) const content = renderUserL2TransactionsPage({ context, diff --git a/packages/backend/src/core/sync/LiveL2TransactionDownloader.ts b/packages/backend/src/core/sync/LiveL2TransactionDownloader.ts index 09e2e0924..9fee3ee1a 100644 --- a/packages/backend/src/core/sync/LiveL2TransactionDownloader.ts +++ b/packages/backend/src/core/sync/LiveL2TransactionDownloader.ts @@ -1,6 +1,6 @@ +import { Logger } from '@l2beat/backend-tools' import { Knex } from 'knex' -import { Logger } from '@l2beat/backend-tools' import { KeyValueStore } from '../../peripherals/database/KeyValueStore' import { L2TransactionRepository } from '../../peripherals/database/L2TransactionRepository' import { StateUpdateRepository } from '../../peripherals/database/StateUpdateRepository' diff --git a/packages/backend/src/peripherals/database/L2TransactionRepository.ts b/packages/backend/src/peripherals/database/L2TransactionRepository.ts index 2fc9b482f..138f16e1e 100644 --- a/packages/backend/src/peripherals/database/L2TransactionRepository.ts +++ b/packages/backend/src/peripherals/database/L2TransactionRepository.ts @@ -276,6 +276,7 @@ export class L2TransactionRepository extends BaseRepository { const countGroupedByType = (await knex('l2_transactions') .select('type') .whereNull('state_update_id') + //eslint-disable-next-line @typescript-eslint/no-misused-promises .andWhere((qB) => qB .where({ stark_key_a: starkKey.toString() }) @@ -290,6 +291,7 @@ export class L2TransactionRepository extends BaseRepository { const [replaced] = await knex('l2_transactions') .whereNull('state_update_id') .andWhere({ state: 'replaced' }) + //eslint-disable-next-line @typescript-eslint/no-misused-promises .andWhere((qB) => qB .where({ stark_key_a: starkKey.toString() }) From 9ae419867742964a7b955756d618824267440da7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20T=C3=B3rz?= <93620601+torztomasz@users.noreply.github.com> Date: Fri, 21 Jul 2023 10:54:01 +0200 Subject: [PATCH 18/19] add l2TransactionsEnabled flag to config (#432) --- packages/backend/src/Application.ts | 76 ++--- .../backend/src/api/routers/FrontendRouter.ts | 3 +- .../api/routers/PerpetualFrontendRouter.ts | 3 +- .../src/config/starkex/StarkexConfig.ts | 54 ++- .../backend/src/config/starkex/apex-goerli.ts | 1 + .../src/config/starkex/apex-mainnet.ts | 1 + .../backend/src/config/starkex/dydx-local.ts | 1 + .../src/config/starkex/dydx-mainnet.ts | 1 + .../src/config/starkex/gammax-goerli.ts | 3 +- .../src/config/starkex/myria-goerli.ts | 1 + .../src/core/PageContextService.test.ts | 4 +- .../backend/src/core/PageContextService.ts | 5 +- .../collectors/FeederGatewayCollector.test.ts | 40 ++- .../core/collectors/FeederGatewayCollector.ts | 5 +- .../core/preprocessing/Preprocessor.test.ts | 323 ++++++++++-------- .../src/core/preprocessing/Preprocessor.ts | 53 +-- .../src/utils/shouldShowL2Transactions.ts | 9 - 17 files changed, 320 insertions(+), 263 deletions(-) delete mode 100644 packages/backend/src/utils/shouldShowL2Transactions.ts diff --git a/packages/backend/src/Application.ts b/packages/backend/src/Application.ts index 20ebbfa2f..a6be0663b 100644 --- a/packages/backend/src/Application.ts +++ b/packages/backend/src/Application.ts @@ -224,8 +224,42 @@ export class Application { | PerpetualRollupUpdater let stateTransitionCollector: IStateTransitionCollector - let feederGatewayCollector: FeederGatewayCollector | undefined - let l2TransactionDownloader: LiveL2TransactionDownloader | undefined + const feederGatewayClient = config.starkex.enableL2Transactions + ? new FeederGatewayClient( + config.starkex.feederGateway, + fetchClient, + logger + ) + : undefined + + const feederGatewayCollector = feederGatewayClient + ? new FeederGatewayCollector( + feederGatewayClient, + l2TransactionRepository, + stateUpdateRepository, + logger, + config.starkex.enableL2Transactions + ) + : undefined + + const liveL2TransactionClient = + config.starkex.enableL2Transactions && config.starkex.l2TransactionApi + ? new LiveL2TransactionClient( + config.starkex.l2TransactionApi, + fetchClient + ) + : undefined + + const liveL2TransactionDownloader = liveL2TransactionClient + ? new LiveL2TransactionDownloader( + liveL2TransactionClient, + l2TransactionRepository, + stateUpdateRepository, + kvStore, + clock, + logger + ) + : undefined if (config.starkex.dataAvailabilityMode === 'validium') { const availabilityGatewayClient = new AvailabilityGatewayClient( @@ -242,40 +276,6 @@ export class Application { ) stateTransitionCollector = perpetualValidiumStateTransitionCollector - const l2TransactionClient = config.starkex.l2TransactionApi - ? new LiveL2TransactionClient( - config.starkex.l2TransactionApi, - fetchClient - ) - : undefined - - l2TransactionDownloader = l2TransactionClient - ? new LiveL2TransactionDownloader( - l2TransactionClient, - l2TransactionRepository, - stateUpdateRepository, - kvStore, - clock, - logger - ) - : undefined - - const feederGatewayClient = config.starkex.feederGateway - ? new FeederGatewayClient( - config.starkex.feederGateway, - fetchClient, - logger - ) - : undefined - feederGatewayCollector = feederGatewayClient - ? new FeederGatewayCollector( - feederGatewayClient, - l2TransactionRepository, - stateUpdateRepository, - logger - ) - : undefined - const perpetualCairoOutputCollector = new PerpetualCairoOutputCollector( ethereumClient, config.starkex @@ -303,7 +303,7 @@ export class Application { perpetualValidiumUpdater, withdrawalAllowedCollector, feederGatewayCollector, - l2TransactionDownloader, + liveL2TransactionDownloader, logger ) } else { @@ -682,7 +682,7 @@ export class Application { if (config.enableSync) { transactionStatusService.start() await syncScheduler.start() - await l2TransactionDownloader?.start() + await liveL2TransactionDownloader?.start() await blockDownloader.start() } diff --git a/packages/backend/src/api/routers/FrontendRouter.ts b/packages/backend/src/api/routers/FrontendRouter.ts index f7fe70fde..ae5b4a2e9 100644 --- a/packages/backend/src/api/routers/FrontendRouter.ts +++ b/packages/backend/src/api/routers/FrontendRouter.ts @@ -4,7 +4,6 @@ import Router from '@koa/router' import * as z from 'zod' import { Config } from '../../config' -import { shouldShowL2Transactions } from '../../utils/shouldShowL2Transactions' import { ForcedActionController } from '../controllers/ForcedActionController' import { ForcedTradeOfferController } from '../controllers/ForcedTradeOfferController' import { HomeController } from '../controllers/HomeController' @@ -367,7 +366,7 @@ export function createFrontendRouter( ) ) - if (shouldShowL2Transactions(config)) { + if (config.starkex.enableL2Transactions) { router.get( '/l2-transactions', withTypedContext( diff --git a/packages/backend/src/api/routers/PerpetualFrontendRouter.ts b/packages/backend/src/api/routers/PerpetualFrontendRouter.ts index 76cdcf7ec..a19b1dde3 100644 --- a/packages/backend/src/api/routers/PerpetualFrontendRouter.ts +++ b/packages/backend/src/api/routers/PerpetualFrontendRouter.ts @@ -4,7 +4,6 @@ import Router from '@koa/router' import { z } from 'zod' import { Config } from '../../config' -import { shouldShowL2Transactions } from '../../utils/shouldShowL2Transactions' import { ForcedActionController } from '../controllers/ForcedActionController' import { ForcedTradeOfferController } from '../controllers/ForcedTradeOfferController' import { L2TransactionController } from '../controllers/L2TransactionController' @@ -68,7 +67,7 @@ export function addPerpetualTradingRoutes( ) ) - if (shouldShowL2Transactions(config)) { + if (config.starkex.enableL2Transactions) { router.get( '/l2-transactions/:transactionId{/:multiIndex}?', withTypedContext( diff --git a/packages/backend/src/config/starkex/StarkexConfig.ts b/packages/backend/src/config/starkex/StarkexConfig.ts index 5467d36e0..9e424c193 100644 --- a/packages/backend/src/config/starkex/StarkexConfig.ts +++ b/packages/backend/src/config/starkex/StarkexConfig.ts @@ -1,27 +1,21 @@ import { CollateralAsset, InstanceName, TradingMode } from '@explorer/shared' import { EthereumAddress } from '@explorer/types' -type CheckTradingMode = Exclude< - T['tradingMode'], - TradingMode -> extends never - ? T - : never - export type StarkexConfig = - CheckTradingMode< - T extends 'perpetual' - ? PerpetualRollupConfig | PerpetualValidiumConfig - : T extends 'spot' - ? SpotValidiumConfig - : PerpetualRollupConfig | PerpetualValidiumConfig | SpotValidiumConfig - > + T extends 'perpetual' + ? PerpetualRollupConfig | PerpetualValidiumConfig + : T extends 'spot' + ? SpotValidiumConfig + : PerpetualRollupConfig | PerpetualValidiumConfig | SpotValidiumConfig -export interface PerpetualRollupConfig { +type BaseConfig = { instanceName: InstanceName - dataAvailabilityMode: 'rollup' - tradingMode: 'perpetual' + dataAvailabilityMode: D + tradingMode: T blockchain: BlockchainConfig +} & L2TransactionsConfig + +export type PerpetualRollupConfig = BaseConfig<'perpetual', 'rollup'> & { contracts: { perpetual: EthereumAddress registry: EthereumAddress @@ -31,25 +25,15 @@ export interface PerpetualRollupConfig { collateralAsset: CollateralAsset } -export interface PerpetualValidiumConfig { - instanceName: InstanceName - dataAvailabilityMode: 'validium' - tradingMode: 'perpetual' - blockchain: BlockchainConfig +export type PerpetualValidiumConfig = BaseConfig<'perpetual', 'validium'> & { availabilityGateway: GatewayConfig - feederGateway: GatewayConfig | undefined - l2TransactionApi: LiveL2TransactionApiConfig | undefined contracts: { perpetual: EthereumAddress } collateralAsset: CollateralAsset } -export interface SpotValidiumConfig { - instanceName: InstanceName - dataAvailabilityMode: 'validium' - tradingMode: 'spot' - blockchain: BlockchainConfig +export type SpotValidiumConfig = BaseConfig<'spot', 'validium'> & { availabilityGateway: GatewayConfig contracts: { perpetual: EthereumAddress @@ -65,6 +49,18 @@ export interface BlockchainConfig { maxBlockNumber: number } +export type L2TransactionsConfig = + | { + enableL2Transactions: true + feederGateway: GatewayConfig + l2TransactionApi: LiveL2TransactionApiConfig | undefined + } + | { + enableL2Transactions: false + feederGateway?: GatewayConfig + l2TransactionApi?: LiveL2TransactionApiConfig + } + export type ClientAuth = | { type: 'bearerToken' diff --git a/packages/backend/src/config/starkex/apex-goerli.ts b/packages/backend/src/config/starkex/apex-goerli.ts index 3f7dc0644..a4c61825b 100644 --- a/packages/backend/src/config/starkex/apex-goerli.ts +++ b/packages/backend/src/config/starkex/apex-goerli.ts @@ -30,6 +30,7 @@ export function getApexGoerliConfig(env: Env): StarkexConfig { }, auth: clientAuth, }, + enableL2Transactions: true, feederGateway: { getUrl: (batchId: number) => { return `${env.string('APEX_FG_URL')}?batchId=${batchId}` diff --git a/packages/backend/src/config/starkex/apex-mainnet.ts b/packages/backend/src/config/starkex/apex-mainnet.ts index 2a24fcf9e..818fa2d47 100644 --- a/packages/backend/src/config/starkex/apex-mainnet.ts +++ b/packages/backend/src/config/starkex/apex-mainnet.ts @@ -30,6 +30,7 @@ export function getApexMainnetConfig(env: Env): StarkexConfig { }, auth: clientAuth, }, + enableL2Transactions: true, feederGateway: { getUrl: (batchId: number) => { return `${env.string('APEX_FG_URL')}?batchId=${batchId}` diff --git a/packages/backend/src/config/starkex/dydx-local.ts b/packages/backend/src/config/starkex/dydx-local.ts index e11b30c21..fe1ca09a3 100644 --- a/packages/backend/src/config/starkex/dydx-local.ts +++ b/packages/backend/src/config/starkex/dydx-local.ts @@ -16,6 +16,7 @@ export function getDydxLocalConfig(env: Env): StarkexConfig { minBlockNumber: 0, maxBlockNumber: env.integer('MAX_BLOCK_NUMBER', Infinity), }, + enableL2Transactions: false, contracts: { perpetual: EthereumAddress('0x27fac828D6E6862901ea8471fF22552D84e155D0'), registry: EthereumAddress('0xE068d37a67cAb19e0A6DFE88e720f076cfA7140E'), diff --git a/packages/backend/src/config/starkex/dydx-mainnet.ts b/packages/backend/src/config/starkex/dydx-mainnet.ts index 7189f65b4..f72423d18 100644 --- a/packages/backend/src/config/starkex/dydx-mainnet.ts +++ b/packages/backend/src/config/starkex/dydx-mainnet.ts @@ -16,6 +16,7 @@ export function getDydxMainnetConfig(env: Env): StarkexConfig { minBlockNumber: 11813207, maxBlockNumber: env.integer('MAX_BLOCK_NUMBER', Infinity), }, + enableL2Transactions: false, contracts: { perpetual: EthereumAddress('0xD54f502e184B6B739d7D27a6410a67dc462D69c8'), registry: EthereumAddress('0xEfbCcE4659db72eC6897F46783303708cf9ACef8'), diff --git a/packages/backend/src/config/starkex/gammax-goerli.ts b/packages/backend/src/config/starkex/gammax-goerli.ts index 28adf0374..077bc2bc0 100644 --- a/packages/backend/src/config/starkex/gammax-goerli.ts +++ b/packages/backend/src/config/starkex/gammax-goerli.ts @@ -19,6 +19,7 @@ export function getGammaxGoerliConfig(env: Env): StarkexConfig { contracts: { perpetual: EthereumAddress('0x6E5de338D71af33B57831C5552775f54394d181B'), }, + enableL2Transactions: false, availabilityGateway: { getUrl: (batchId: number) => { return `${env.string('GAMMAX_AG_URL')}?batch_id=${batchId}` @@ -30,8 +31,6 @@ export function getGammaxGoerliConfig(env: Env): StarkexConfig { userKey: env.string('GAMMAX_AG_USER_KEY'), }, }, - feederGateway: undefined, - l2TransactionApi: undefined, collateralAsset: { assetId: AssetId('COLLATERAL-1'), assetHash: AssetHash( diff --git a/packages/backend/src/config/starkex/myria-goerli.ts b/packages/backend/src/config/starkex/myria-goerli.ts index 271d668ea..cce5d2d19 100644 --- a/packages/backend/src/config/starkex/myria-goerli.ts +++ b/packages/backend/src/config/starkex/myria-goerli.ts @@ -27,6 +27,7 @@ export function getMyriaGoerliConfig(env: Env): StarkexConfig { userKey: env.string('MYRIA_AG_USER_KEY'), }, }, + enableL2Transactions: false, contracts: { perpetual: EthereumAddress('0xF82C423a30E317f34f9b0997627F2F9c5d239Ad9'), }, diff --git a/packages/backend/src/core/PageContextService.test.ts b/packages/backend/src/core/PageContextService.test.ts index d42411aef..11bca5ca1 100644 --- a/packages/backend/src/core/PageContextService.test.ts +++ b/packages/backend/src/core/PageContextService.test.ts @@ -45,7 +45,7 @@ describe(PageContextService.name, () => { expect(context).toEqual({ user: undefined, tradingMode: 'perpetual', - showL2Transactions: false, + showL2Transactions: perpetualConfig.starkex.enableL2Transactions, chainId: 1, instanceName: perpetualConfig.starkex.instanceName, collateralAsset: fakeCollateralAsset, @@ -70,7 +70,7 @@ describe(PageContextService.name, () => { expect(context).toEqual({ user: undefined, tradingMode: 'spot', - showL2Transactions: false, + showL2Transactions: spotConfig.starkex.enableL2Transactions, chainId: 5, instanceName: spotConfig.starkex.instanceName, }) diff --git a/packages/backend/src/core/PageContextService.ts b/packages/backend/src/core/PageContextService.ts index 0d8e95f8f..fc5ea3136 100644 --- a/packages/backend/src/core/PageContextService.ts +++ b/packages/backend/src/core/PageContextService.ts @@ -7,7 +7,6 @@ import { } from '@explorer/shared' import { Config } from '../config' -import { shouldShowL2Transactions } from '../utils/shouldShowL2Transactions' import { UserService } from './UserService' export class PageContextService { @@ -26,7 +25,7 @@ export class PageContextService { instanceName: this.config.starkex.instanceName, chainId: this.config.starkex.blockchain.chainId, collateralAsset: this.config.starkex.collateralAsset, - showL2Transactions: shouldShowL2Transactions(this.config), + showL2Transactions: this.config.starkex.enableL2Transactions, } } @@ -35,7 +34,7 @@ export class PageContextService { tradingMode: this.config.starkex.tradingMode, chainId: this.config.starkex.blockchain.chainId, instanceName: this.config.starkex.instanceName, - showL2Transactions: shouldShowL2Transactions(this.config), + showL2Transactions: this.config.starkex.enableL2Transactions, } } diff --git a/packages/backend/src/core/collectors/FeederGatewayCollector.test.ts b/packages/backend/src/core/collectors/FeederGatewayCollector.test.ts index 873492994..cfd6b5e11 100644 --- a/packages/backend/src/core/collectors/FeederGatewayCollector.test.ts +++ b/packages/backend/src/core/collectors/FeederGatewayCollector.test.ts @@ -2,6 +2,7 @@ import { EthereumAddress, Hash256, StarkKey, Timestamp } from '@explorer/types' import { Logger } from '@l2beat/backend-tools' import { expect, mockFn, mockObject } from 'earl' import { Knex } from 'knex' +import { it } from 'mocha' import { L2TransactionRepository } from '../../peripherals/database/L2TransactionRepository' import { @@ -41,7 +42,8 @@ describe(FeederGatewayCollector.name, () => { mockFeederGatewayClient, mockL2TransactionRepository, mockStateUpdateRepository, - Logger.SILENT + Logger.SILENT, + true ) await feederGatewayCollector.collect() @@ -142,7 +144,8 @@ describe(FeederGatewayCollector.name, () => { mockFeederGatewayClient, mockL2TransactionRepository, mockStateUpdateRepository, - Logger.SILENT + Logger.SILENT, + true ) await feederGatewayCollector.collect() @@ -237,7 +240,8 @@ describe(FeederGatewayCollector.name, () => { findById: mockFn().resolvesTo({} as StateUpdateRecord), findLast: mockFn().resolvesTo(fakeStateUpdateRecord(10)), }), - Logger.SILENT + Logger.SILENT, + true ) await feederGatewayCollector.collect() @@ -259,7 +263,8 @@ describe(FeederGatewayCollector.name, () => { mockObject(), mockedL2TransactionRepository, mockedStateUpdateRepository, - Logger.SILENT + Logger.SILENT, + true ) await feederGatewayCollector.collect() @@ -269,6 +274,27 @@ describe(FeederGatewayCollector.name, () => { ).not.toHaveBeenCalled() expect(mockedL2TransactionRepository.add).not.toHaveBeenCalled() }) + + it('should not do anything if l2 transactions are disabled', async () => { + const mockedL2TransactionRepository = mockObject( + { + findLatestStateUpdateId: mockFn(), + } + ) + const feederGatewayCollector = new FeederGatewayCollector( + mockObject(), + mockedL2TransactionRepository, + mockObject(), + Logger.SILENT, + false + ) + + await feederGatewayCollector.collect() + + expect( + mockedL2TransactionRepository.findLatestStateUpdateId + ).not.toHaveBeenCalled() + }) }) it('should throw error if there is no state update in db for given stateUpdateId', async () => { @@ -283,7 +309,8 @@ describe(FeederGatewayCollector.name, () => { findById: mockFn().resolvesTo(undefined), findLast: mockFn().resolvesTo(fakeStateUpdateRecord(10)), }), - Logger.SILENT + Logger.SILENT, + true ) await expect(feederGatewayCollector.collect()).toBeRejectedWith( @@ -300,7 +327,8 @@ describe(FeederGatewayCollector.name, () => { mockObject(), mockedL2TransactionRepository, mockObject(), - Logger.SILENT + Logger.SILENT, + mockObject() ) it('should discard transactions after given block number', async () => { diff --git a/packages/backend/src/core/collectors/FeederGatewayCollector.ts b/packages/backend/src/core/collectors/FeederGatewayCollector.ts index 3ca33f43c..700a54ed5 100644 --- a/packages/backend/src/core/collectors/FeederGatewayCollector.ts +++ b/packages/backend/src/core/collectors/FeederGatewayCollector.ts @@ -9,10 +9,13 @@ export class FeederGatewayCollector { private readonly feederGatewayClient: FeederGatewayClient, private readonly l2TransactionRepository: L2TransactionRepository, private readonly stateUpdateRepository: StateUpdateRepository, - private readonly logger: Logger + private readonly logger: Logger, + private readonly l2TransactionsEnabled: boolean ) {} async collect() { + if (!this.l2TransactionsEnabled) return + const latestStateUpdate = await this.stateUpdateRepository.findLast() if (!latestStateUpdate) return diff --git a/packages/backend/src/core/preprocessing/Preprocessor.test.ts b/packages/backend/src/core/preprocessing/Preprocessor.test.ts index d76b21762..33cfb9b35 100644 --- a/packages/backend/src/core/preprocessing/Preprocessor.test.ts +++ b/packages/backend/src/core/preprocessing/Preprocessor.test.ts @@ -38,9 +38,9 @@ describe(Preprocessor.name, () => { mockObject(), mockObject(), mockObject(), - mockObject(), - Logger.SILENT + Logger.SILENT, + mockObject() ) const directions: SyncDirection[] = [ @@ -104,7 +104,8 @@ describe(Preprocessor.name, () => { mockUserStatisticsPreprocessor, mockUserL2TransactionsPreprocessor, mockObject(), - Logger.SILENT + Logger.SILENT, + mockObject() ) await preprocessor.catchUp() @@ -160,7 +161,8 @@ describe(Preprocessor.name, () => { mockUserStatisticsPreprocessor, mockUserL2TransactionsPreprocessor, mockObject(), - Logger.SILENT + Logger.SILENT, + mockObject() ) await preprocessor.catchUp() @@ -194,7 +196,8 @@ describe(Preprocessor.name, () => { mockObject(), mockObject(), mockObject(), - Logger.SILENT + Logger.SILENT, + mockObject() ) expect(await preprocessor.getLastSyncedStateUpdate()).toEqual(undefined) @@ -220,7 +223,8 @@ describe(Preprocessor.name, () => { mockObject(), mockObject(), mockObject(), - Logger.SILENT + Logger.SILENT, + mockObject() ) const lastStateUpdate = await preprocessor.getLastSyncedStateUpdate() expect(lastStateUpdate).toEqual(fakeStateUpdate) @@ -247,7 +251,8 @@ describe(Preprocessor.name, () => { mockObject(), mockObject(), mockObject(), - Logger.SILENT + Logger.SILENT, + mockObject() ) const mockGetLastSyncedStateUpdate = mockFn().resolvesTo(undefined) preprocessor.getLastSyncedStateUpdate = mockGetLastSyncedStateUpdate @@ -273,7 +278,8 @@ describe(Preprocessor.name, () => { mockObject(), mockObject(), mockObject(), - Logger.SILENT + Logger.SILENT, + mockObject() ) const mockGetLastSyncedStateUpdate = mockFn().resolvesTo(undefined) preprocessor.getLastSyncedStateUpdate = mockGetLastSyncedStateUpdate @@ -302,7 +308,8 @@ describe(Preprocessor.name, () => { mockObject(), mockObject(), mockObject(), - Logger.SILENT + Logger.SILENT, + mockObject() ) const mockGetLastSyncedStateUpdate = mockFn().resolvesTo(fakeStateUpdate) preprocessor.getLastSyncedStateUpdate = mockGetLastSyncedStateUpdate @@ -333,7 +340,8 @@ describe(Preprocessor.name, () => { mockObject(), mockObject(), mockObject(), - Logger.SILENT + Logger.SILENT, + mockObject() ) const mockGetLastSyncedStateUpdate = mockFn().resolvesTo(fakeStateUpdate) preprocessor.getLastSyncedStateUpdate = mockGetLastSyncedStateUpdate @@ -365,7 +373,8 @@ describe(Preprocessor.name, () => { mockObject(), mockObject(), mockObject(), - Logger.SILENT + Logger.SILENT, + mockObject() ) const mockGetLastSyncedStateUpdate = mockFn().resolvesTo(fakeStateUpdate) preprocessor.getLastSyncedStateUpdate = mockGetLastSyncedStateUpdate @@ -399,7 +408,8 @@ describe(Preprocessor.name, () => { mockObject(), mockObject(), mockObject(), - Logger.SILENT + Logger.SILENT, + mockObject() ) const mockGetLastSyncedStateUpdate = mockFn().resolvesTo(fakeStateUpdate) preprocessor.getLastSyncedStateUpdate = mockGetLastSyncedStateUpdate @@ -432,7 +442,8 @@ describe(Preprocessor.name, () => { mockObject(), mockObject(), mockObject(), - Logger.SILENT + Logger.SILENT, + mockObject() ) const mockGetLastSyncedStateUpdate = mockFn().resolvesTo(fakeStateUpdate) preprocessor.getLastSyncedStateUpdate = mockGetLastSyncedStateUpdate @@ -466,7 +477,8 @@ describe(Preprocessor.name, () => { mockObject(), mockObject(), mockObject(), - Logger.SILENT + Logger.SILENT, + mockObject() ) const mockGetLastSyncedStateUpdate = mockFn().resolvesTo(fakeStateUpdate10) @@ -501,7 +513,8 @@ describe(Preprocessor.name, () => { mockObject(), mockObject(), mockObject(), - Logger.SILENT + Logger.SILENT, + mockObject() ) const mockGetLastSyncedStateUpdate = mockFn().resolvesTo(fakeStateUpdate10) @@ -515,83 +528,95 @@ describe(Preprocessor.name, () => { }) describe(Preprocessor.prototype.preprocessNextStateUpdate.name, () => { - it('correctly updates preprocessedStateUpdateRepository in SQL transaction', async () => { - const preprocessL2TransactionTo = 200 - const fakeStateUpdate1 = generateFakeStateUpdate(1) - const fakeStateUpdate2 = generateFakeStateUpdate(2) - const mockKnexTransaction = mockObject() - const mockPerpetualHistoryPreprocessor = - mockObject({ - preprocessNextStateUpdate: async () => undefined, + const l2TransactionsEnabledValues = [false, true] + l2TransactionsEnabledValues.forEach((l2TransactionsEnabled) => { + it(`correctly updates preprocessedStateUpdateRepository in SQL transaction for l2TransactionsEnabled=${l2TransactionsEnabled.toString()}`, async () => { + const preprocessL2TransactionTo = 200 + const fakeStateUpdate1 = generateFakeStateUpdate(1) + const fakeStateUpdate2 = generateFakeStateUpdate(2) + const mockKnexTransaction = mockObject() + const mockPerpetualHistoryPreprocessor = + mockObject({ + preprocessNextStateUpdate: async () => undefined, + }) + const mockStateDetailsPreprocessor = + mockObject({ + preprocessNextStateUpdate: async () => undefined, + catchUpL2Transactions: mockFn(async () => {}), + }) + const mockUserStatisticsPreprocessor = + mockObject({ + preprocessNextStateUpdate: async () => undefined, + }) + const mockUserL2TransactionsPreprocessor = + mockObject({ + catchUp: mockFn(async () => {}), + }) + const stateUpdateRepo = mockObject({ + findById: async (id: number) => ({ [2]: fakeStateUpdate2 }[id]), }) - const mockStateDetailsPreprocessor = mockObject( - { - preprocessNextStateUpdate: async () => undefined, - catchUpL2Transactions: mockFn(async () => {}), - } - ) - const mockUserStatisticsPreprocessor = - mockObject({ - preprocessNextStateUpdate: async () => undefined, - }) - const mockUserL2TransactionsPreprocessor = - mockObject({ - catchUp: mockFn(async () => {}), + const preprocessedRepo = mockObject({ + findLast: async () => ({ + stateUpdateId: fakeStateUpdate1.id, + stateTransitionHash: fakeStateUpdate1.stateTransitionHash, + }), + add: async () => 0, + runInTransaction: async (fn) => fn(mockKnexTransaction), }) - const stateUpdateRepo = mockObject({ - findById: async (id: number) => ({ [2]: fakeStateUpdate2 }[id]), - }) - const preprocessedRepo = mockObject({ - findLast: async () => ({ - stateUpdateId: fakeStateUpdate1.id, - stateTransitionHash: fakeStateUpdate1.stateTransitionHash, - }), - add: async () => 0, - runInTransaction: async (fn) => fn(mockKnexTransaction), + const preprocessor = new Preprocessor( + mockObject(), + preprocessedRepo, + stateUpdateRepo, + mockPerpetualHistoryPreprocessor, + mockStateDetailsPreprocessor, + mockUserStatisticsPreprocessor, + mockUserL2TransactionsPreprocessor, + mockObject(), + Logger.SILENT, + l2TransactionsEnabled + ) + + const mockedGetStateUpdateIdToCatchUpL2TransactionsTo = + mockFn().resolvesTo(preprocessL2TransactionTo) + preprocessor.getStateUpdateIdToCatchUpL2TransactionsTo = + mockedGetStateUpdateIdToCatchUpL2TransactionsTo + + await preprocessor.preprocessNextStateUpdate() + expect(preprocessedRepo.add).toHaveBeenOnlyCalledWith( + { + stateUpdateId: fakeStateUpdate2.id, + stateTransitionHash: fakeStateUpdate2.stateTransitionHash, + }, + mockKnexTransaction + ) + expect( + mockPerpetualHistoryPreprocessor.preprocessNextStateUpdate + ).toHaveBeenOnlyCalledWith(mockKnexTransaction, fakeStateUpdate2) + expect( + mockStateDetailsPreprocessor.preprocessNextStateUpdate + ).toHaveBeenOnlyCalledWith(mockKnexTransaction, fakeStateUpdate2) + expect( + mockUserStatisticsPreprocessor.preprocessNextStateUpdate + ).toHaveBeenOnlyCalledWith(mockKnexTransaction, fakeStateUpdate2) + + if (l2TransactionsEnabled) { + expect( + mockedGetStateUpdateIdToCatchUpL2TransactionsTo + ).toHaveBeenCalledWith(mockKnexTransaction, fakeStateUpdate2.id) + expect( + mockStateDetailsPreprocessor.catchUpL2Transactions + ).toHaveBeenOnlyCalledWith( + mockKnexTransaction, + preprocessL2TransactionTo + ) + expect( + mockUserL2TransactionsPreprocessor.catchUp + ).toHaveBeenOnlyCalledWith( + mockKnexTransaction, + preprocessL2TransactionTo + ) + } }) - const preprocessor = new Preprocessor( - mockObject(), - preprocessedRepo, - stateUpdateRepo, - mockPerpetualHistoryPreprocessor, - mockStateDetailsPreprocessor, - mockUserStatisticsPreprocessor, - mockUserL2TransactionsPreprocessor, - mockObject(), - Logger.SILENT - ) - - const mockedGetStateUpdateIdToCatchUpL2TransactionsTo = - mockFn().resolvesTo(preprocessL2TransactionTo) - preprocessor.getStateUpdateIdToCatchUpL2TransactionsTo = - mockedGetStateUpdateIdToCatchUpL2TransactionsTo - - await preprocessor.preprocessNextStateUpdate() - expect(preprocessedRepo.add).toHaveBeenOnlyCalledWith( - { - stateUpdateId: fakeStateUpdate2.id, - stateTransitionHash: fakeStateUpdate2.stateTransitionHash, - }, - mockKnexTransaction - ) - expect( - mockPerpetualHistoryPreprocessor.preprocessNextStateUpdate - ).toHaveBeenOnlyCalledWith(mockKnexTransaction, fakeStateUpdate2) - expect( - mockStateDetailsPreprocessor.preprocessNextStateUpdate - ).toHaveBeenOnlyCalledWith(mockKnexTransaction, fakeStateUpdate2) - expect( - mockUserStatisticsPreprocessor.preprocessNextStateUpdate - ).toHaveBeenOnlyCalledWith(mockKnexTransaction, fakeStateUpdate2) - expect( - mockedGetStateUpdateIdToCatchUpL2TransactionsTo - ).toHaveBeenCalledWith(mockKnexTransaction, fakeStateUpdate2.id) - expect( - mockStateDetailsPreprocessor.catchUpL2Transactions - ).toHaveBeenOnlyCalledWith(mockKnexTransaction, preprocessL2TransactionTo) - expect( - mockUserL2TransactionsPreprocessor.catchUp - ).toHaveBeenOnlyCalledWith(mockKnexTransaction, preprocessL2TransactionTo) }) it('throws when next state update is missing', async () => { @@ -617,7 +642,8 @@ describe(Preprocessor.name, () => { mockObject(), mockObject(), mockObject(), - Logger.SILENT + Logger.SILENT, + mockObject() ) await expect(preprocessor.preprocessNextStateUpdate()).toBeRejectedWith( 'Preprocessing was requested, but next state update (3) is missing' @@ -626,62 +652,67 @@ describe(Preprocessor.name, () => { }) describe(Preprocessor.prototype.rollbackOneStateUpdate.name, () => { - it('correctly updates preprocessedStateUpdateRepository in SQL transaction', async () => { - const fakeStateUpdate = generateFakeStateUpdate(2) - const mockKnexTransaction = mockObject() - const mockPerpetualHistoryPreprocessor = - mockObject({ - rollbackOneStateUpdate: async () => undefined, + const l2TransactionsEnabledValues = [false, true] + l2TransactionsEnabledValues.forEach((l2TransactionsEnabled) => { + it(`correctly updates preprocessedStateUpdateRepository in SQL transaction if l2TransactionsEnabled=${l2TransactionsEnabled.toString()}`, async () => { + const fakeStateUpdate = generateFakeStateUpdate(2) + const mockKnexTransaction = mockObject() + const mockPerpetualHistoryPreprocessor = + mockObject({ + rollbackOneStateUpdate: async () => undefined, + }) + const mockStateDetailsPreprocessor = + mockObject({ + rollbackOneStateUpdate: async () => undefined, + }) + const mockUserStatisticsPreprocessor = + mockObject({ + rollbackOneStateUpdate: async () => undefined, + }) + const preprocessedRepo = mockObject({ + findLast: async () => ({ + stateUpdateId: fakeStateUpdate.id, + stateTransitionHash: fakeStateUpdate.stateTransitionHash, + }), + runInTransaction: async (fn) => fn(mockKnexTransaction), + deleteByStateUpdateId: async () => 1, }) - const mockStateDetailsPreprocessor = mockObject( - { - rollbackOneStateUpdate: async () => undefined, + const mockUserL2TransactionsPreprocessor = + mockObject({ + rollbackOneStateUpdate: mockFn().resolvesTo(undefined), + }) + const preprocessor = new Preprocessor( + mockObject(), + preprocessedRepo, + mockObject(), + mockPerpetualHistoryPreprocessor, + mockStateDetailsPreprocessor, + mockUserStatisticsPreprocessor, + mockUserL2TransactionsPreprocessor, + mockObject(), + Logger.SILENT, + l2TransactionsEnabled + ) + await preprocessor.rollbackOneStateUpdate() + expect(preprocessedRepo.deleteByStateUpdateId).toHaveBeenOnlyCalledWith( + fakeStateUpdate.id, + mockKnexTransaction + ) + expect( + mockPerpetualHistoryPreprocessor.rollbackOneStateUpdate + ).toHaveBeenOnlyCalledWith(mockKnexTransaction, fakeStateUpdate.id) + expect( + mockStateDetailsPreprocessor.rollbackOneStateUpdate + ).toHaveBeenOnlyCalledWith(mockKnexTransaction, fakeStateUpdate.id) + expect( + mockUserStatisticsPreprocessor.rollbackOneStateUpdate + ).toHaveBeenOnlyCalledWith(mockKnexTransaction, fakeStateUpdate.id) + if (l2TransactionsEnabled) { + expect( + mockUserL2TransactionsPreprocessor.rollbackOneStateUpdate + ).toHaveBeenOnlyCalledWith(mockKnexTransaction, fakeStateUpdate.id) } - ) - const mockUserStatisticsPreprocessor = - mockObject({ - rollbackOneStateUpdate: async () => undefined, - }) - const preprocessedRepo = mockObject({ - findLast: async () => ({ - stateUpdateId: fakeStateUpdate.id, - stateTransitionHash: fakeStateUpdate.stateTransitionHash, - }), - runInTransaction: async (fn) => fn(mockKnexTransaction), - deleteByStateUpdateId: async () => 1, }) - const mockUserL2TransactionsPreprocessor = - mockObject({ - rollbackOneStateUpdate: mockFn().resolvesTo(undefined), - }) - const preprocessor = new Preprocessor( - mockObject(), - preprocessedRepo, - mockObject(), - mockPerpetualHistoryPreprocessor, - mockStateDetailsPreprocessor, - mockUserStatisticsPreprocessor, - mockUserL2TransactionsPreprocessor, - mockObject(), - Logger.SILENT - ) - await preprocessor.rollbackOneStateUpdate() - expect(preprocessedRepo.deleteByStateUpdateId).toHaveBeenOnlyCalledWith( - fakeStateUpdate.id, - mockKnexTransaction - ) - expect( - mockPerpetualHistoryPreprocessor.rollbackOneStateUpdate - ).toHaveBeenOnlyCalledWith(mockKnexTransaction, fakeStateUpdate.id) - expect( - mockStateDetailsPreprocessor.rollbackOneStateUpdate - ).toHaveBeenOnlyCalledWith(mockKnexTransaction, fakeStateUpdate.id) - expect( - mockUserStatisticsPreprocessor.rollbackOneStateUpdate - ).toHaveBeenOnlyCalledWith(mockKnexTransaction, fakeStateUpdate.id) - expect( - mockUserL2TransactionsPreprocessor.rollbackOneStateUpdate - ).toHaveBeenOnlyCalledWith(mockKnexTransaction, fakeStateUpdate.id) }) it('throws when there are no preprocessings to roll back', async () => { @@ -699,7 +730,8 @@ describe(Preprocessor.name, () => { mockObject(), mockObject(), mockObject(), - Logger.SILENT + Logger.SILENT, + mockObject() ) await expect(preprocessor.rollbackOneStateUpdate()).toBeRejectedWith( 'Preprocessing rollback was requested, but there is nothing to roll back' @@ -729,7 +761,8 @@ describe(Preprocessor.name, () => { mockObject(), mockObject(), mockedL2TransactionRepository, - Logger.SILENT + Logger.SILENT, + mockObject() ) it('returns the latest l2 transaction state update id if it is smaller than processed state update id', async () => { diff --git a/packages/backend/src/core/preprocessing/Preprocessor.ts b/packages/backend/src/core/preprocessing/Preprocessor.ts index 87338d954..d7001d469 100644 --- a/packages/backend/src/core/preprocessing/Preprocessor.ts +++ b/packages/backend/src/core/preprocessing/Preprocessor.ts @@ -27,6 +27,7 @@ export class Preprocessor { private userL2TransactionsPreprocessor: UserL2TransactionsStatisticsPreprocessor, private l2TransactionRepository: L2TransactionRepository, private logger: Logger, + private l2TransactionsEnabled: boolean, private isEnabled: boolean = true ) { this.logger = this.logger.for(this) @@ -130,14 +131,17 @@ export class Preprocessor { trx, lastProcessedStateUpdate.stateUpdateId ) - await this.stateDetailsPreprocessor.catchUpL2Transactions( - trx, - lastProcessedStateUpdate.stateUpdateId - ) - await this.userL2TransactionsPreprocessor.catchUp( - trx, - lastProcessedStateUpdate.stateUpdateId - ) + + if (this.l2TransactionsEnabled) { + await this.stateDetailsPreprocessor.catchUpL2Transactions( + trx, + lastProcessedStateUpdate.stateUpdateId + ) + await this.userL2TransactionsPreprocessor.catchUp( + trx, + lastProcessedStateUpdate.stateUpdateId + ) + } } ) } @@ -187,24 +191,25 @@ export class Preprocessor { nextStateUpdate ) - // We cannot assume that Feeder and Availability Gateway are in sync - // with the state updates. We need to catch up with L2 transactions - // after each state update to make sure it is preprocessed as far as possible. - const preprocessL2TransactionTo = - await this.getStateUpdateIdToCatchUpL2TransactionsTo( + if (this.l2TransactionsEnabled) { + // We cannot assume that Feeder and Availability Gateway are in sync + // with the state updates. We need to catch up with L2 transactions + // after each state update to make sure it is preprocessed as far as possible. + const preprocessL2TransactionTo = + await this.getStateUpdateIdToCatchUpL2TransactionsTo( + trx, + nextStateUpdate.id + ) + + await this.stateDetailsPreprocessor.catchUpL2Transactions( trx, - nextStateUpdate.id + preprocessL2TransactionTo ) - - await this.stateDetailsPreprocessor.catchUpL2Transactions( - trx, - preprocessL2TransactionTo - ) - await this.userL2TransactionsPreprocessor.catchUp( - trx, - preprocessL2TransactionTo - ) - + await this.userL2TransactionsPreprocessor.catchUp( + trx, + preprocessL2TransactionTo + ) + } // END TRANSACTION } ) diff --git a/packages/backend/src/utils/shouldShowL2Transactions.ts b/packages/backend/src/utils/shouldShowL2Transactions.ts deleted file mode 100644 index 26864d378..000000000 --- a/packages/backend/src/utils/shouldShowL2Transactions.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { Config } from '../config' - -export const shouldShowL2Transactions = (config: Config) => { - return ( - config.starkex.dataAvailabilityMode === 'validium' && - config.starkex.tradingMode === 'perpetual' && - !!config.starkex.feederGateway - ) -} From 7620ae067d06e7c9e1ba58d21dcbad61e8f28936 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20T=C3=B3rz?= Date: Fri, 21 Jul 2023 13:43:00 +0200 Subject: [PATCH 19/19] add possibility to exclude l2 transaction types in config --- packages/backend/src/Application.ts | 18 ++- .../src/api/controllers/HomeController.ts | 34 ++++- .../api/controllers/StateUpdateController.ts | 18 ++- .../src/api/controllers/UserController.ts | 28 +++- .../backend/src/api/routers/FrontendRouter.ts | 2 +- .../api/routers/PerpetualFrontendRouter.ts | 2 +- .../src/config/starkex/StarkexConfig.ts | 30 +++-- .../backend/src/config/starkex/apex-goerli.ts | 37 +++--- .../src/config/starkex/apex-mainnet.ts | 38 +++--- .../backend/src/config/starkex/dydx-local.ts | 2 +- .../src/config/starkex/dydx-mainnet.ts | 2 +- .../src/config/starkex/gammax-goerli.ts | 4 +- .../src/config/starkex/myria-goerli.ts | 2 +- .../src/core/PageContextService.test.ts | 4 +- .../backend/src/core/PageContextService.ts | 4 +- .../database/L2TransactionRepository.test.ts | 123 ++++++++++++++++-- .../database/L2TransactionRepository.ts | 35 +++-- ...eprocessedL2TransactionsStatistics.test.ts | 22 ++++ .../PreprocessedL2TransactionsStatistics.ts | 14 +- packages/backend/src/utils/uncapitalize.ts | 3 + 20 files changed, 323 insertions(+), 99 deletions(-) create mode 100644 packages/backend/src/utils/uncapitalize.ts diff --git a/packages/backend/src/Application.ts b/packages/backend/src/Application.ts index a6be0663b..ac57b781e 100644 --- a/packages/backend/src/Application.ts +++ b/packages/backend/src/Application.ts @@ -224,9 +224,9 @@ export class Application { | PerpetualRollupUpdater let stateTransitionCollector: IStateTransitionCollector - const feederGatewayClient = config.starkex.enableL2Transactions + const feederGatewayClient = config.starkex.l2Transactions.enabled ? new FeederGatewayClient( - config.starkex.feederGateway, + config.starkex.l2Transactions.feederGateway, fetchClient, logger ) @@ -238,14 +238,15 @@ export class Application { l2TransactionRepository, stateUpdateRepository, logger, - config.starkex.enableL2Transactions + config.starkex.l2Transactions.enabled ) : undefined const liveL2TransactionClient = - config.starkex.enableL2Transactions && config.starkex.l2TransactionApi + config.starkex.l2Transactions.enabled && + config.starkex.l2Transactions.liveApi ? new LiveL2TransactionClient( - config.starkex.l2TransactionApi, + config.starkex.l2Transactions.liveApi, fetchClient ) : undefined @@ -564,7 +565,8 @@ export class Application { userTransactionRepository, forcedTradeOfferRepository, l2TransactionRepository, - preprocessedStateDetailsRepository + preprocessedStateDetailsRepository, + config.starkex.l2Transactions.excludeTypes ) const userController = new UserController( @@ -580,6 +582,7 @@ export class Application { withdrawableAssetRepository, preprocessedUserStatisticsRepository, preprocessedUserL2TransactionsStatisticsRepository, + config.starkex.l2Transactions.excludeTypes, config.starkex.contracts.perpetual ) const stateUpdateController = new StateUpdateController( @@ -589,7 +592,8 @@ export class Application { userTransactionRepository, l2TransactionRepository, preprocessedAssetHistoryRepository, - preprocessedStateDetailsRepository + preprocessedStateDetailsRepository, + config.starkex.l2Transactions.excludeTypes ) const transactionController = new TransactionController( pageContextService, diff --git a/packages/backend/src/api/controllers/HomeController.ts b/packages/backend/src/api/controllers/HomeController.ts index 920f69f8a..77799b646 100644 --- a/packages/backend/src/api/controllers/HomeController.ts +++ b/packages/backend/src/api/controllers/HomeController.ts @@ -6,6 +6,7 @@ import { } from '@explorer/frontend' import { UserDetails } from '@explorer/shared' +import { L2TransactionTypesToExclude } from '../../config/starkex/StarkexConfig' import { AssetDetailsService } from '../../core/AssetDetailsService' import { ForcedTradeOfferViewService } from '../../core/ForcedTradeOfferViewService' import { PageContextService } from '../../core/PageContextService' @@ -34,7 +35,10 @@ export class HomeController { private readonly userTransactionRepository: UserTransactionRepository, private readonly forcedTradeOfferRepository: ForcedTradeOfferRepository, private readonly l2TransactionRepository: L2TransactionRepository, - private readonly preprocessedStateDetailsRepository: PreprocessedStateDetailsRepository + private readonly preprocessedStateDetailsRepository: PreprocessedStateDetailsRepository, + private readonly excludeL2TransactionTypes: + | L2TransactionTypesToExclude + | undefined ) {} async getHomePage( @@ -53,7 +57,10 @@ export class HomeController { availableOffers, availableOffersCount, ] = await Promise.all([ - this.l2TransactionRepository.getPaginatedWithoutMulti(paginationOpts), + this.l2TransactionRepository.getPaginatedWithoutMulti( + paginationOpts, + this.excludeL2TransactionTypes + ), this.preprocessedStateDetailsRepository.findLastWithL2TransactionsStatistics(), this.l2TransactionRepository.getLiveStatistics(), this.preprocessedStateDetailsRepository.getPaginated(paginationOpts), @@ -86,8 +93,13 @@ export class HomeController { const totalL2Transactions = sumUpTransactionCount( - lastStateDetailsWithL2TransactionsStatistics?.cumulativeL2TransactionsStatistics - ) + sumUpTransactionCount(liveL2TransactionsStatistics) + lastStateDetailsWithL2TransactionsStatistics?.cumulativeL2TransactionsStatistics, + this.excludeL2TransactionTypes + ) + + sumUpTransactionCount( + liveL2TransactionsStatistics, + this.excludeL2TransactionTypes + ) const content = renderHomePage({ context, @@ -119,15 +131,23 @@ export class HomeController { lastStateDetailsWithL2TransactionsStatistics, liveL2TransactionsStatistics, ] = await Promise.all([ - this.l2TransactionRepository.getPaginatedWithoutMulti(pagination), + this.l2TransactionRepository.getPaginatedWithoutMulti( + pagination, + this.excludeL2TransactionTypes + ), this.preprocessedStateDetailsRepository.findLastWithL2TransactionsStatistics(), this.l2TransactionRepository.getLiveStatistics(), ]) const totalL2Transactions = sumUpTransactionCount( - lastStateDetailsWithL2TransactionsStatistics?.cumulativeL2TransactionsStatistics - ) + sumUpTransactionCount(liveL2TransactionsStatistics) + lastStateDetailsWithL2TransactionsStatistics?.cumulativeL2TransactionsStatistics, + this.excludeL2TransactionTypes + ) + + sumUpTransactionCount( + liveL2TransactionsStatistics, + this.excludeL2TransactionTypes + ) const content = renderHomeL2TransactionsPage({ context, diff --git a/packages/backend/src/api/controllers/StateUpdateController.ts b/packages/backend/src/api/controllers/StateUpdateController.ts index 83ae085db..f03311b08 100644 --- a/packages/backend/src/api/controllers/StateUpdateController.ts +++ b/packages/backend/src/api/controllers/StateUpdateController.ts @@ -7,6 +7,7 @@ import { import { UserDetails } from '@explorer/shared' import { AssetHash } from '@explorer/types' +import { L2TransactionTypesToExclude } from '../../config/starkex/StarkexConfig' import { AssetDetailsMap } from '../../core/AssetDetailsMap' import { AssetDetailsService } from '../../core/AssetDetailsService' import { PageContextService } from '../../core/PageContextService' @@ -40,7 +41,10 @@ export class StateUpdateController { private readonly userTransactionRepository: UserTransactionRepository, private readonly l2TransactionRepository: L2TransactionRepository, private readonly preprocessedAssetHistoryRepository: PreprocessedAssetHistoryRepository, - private readonly preprocessedStateDetailsRepository: PreprocessedStateDetailsRepository + private readonly preprocessedStateDetailsRepository: PreprocessedStateDetailsRepository, + private readonly excludeL2TransactionTypes: + | L2TransactionTypesToExclude + | undefined ) {} async getStateUpdatePage( @@ -82,7 +86,8 @@ export class StateUpdateController { { offset: 0, limit: 6, - } + }, + this.excludeL2TransactionTypes ), this.preprocessedStateDetailsRepository.findByStateUpdateId( stateUpdateId @@ -132,7 +137,8 @@ export class StateUpdateController { l2Transactions: l2Transactions.map(l2TransactionToEntry), totalL2Transactions: preprocessedStateDetails?.l2TransactionsStatistics ? sumUpTransactionCount( - preprocessedStateDetails.l2TransactionsStatistics + preprocessedStateDetails.l2TransactionsStatistics, + this.excludeL2TransactionTypes ) : 'processing', transactions, @@ -152,7 +158,8 @@ export class StateUpdateController { const [l2Transactions, preprocessedStateDetails] = await Promise.all([ this.l2TransactionRepository.getPaginatedWithoutMultiByStateUpdateId( stateUpdateId, - pagination + pagination, + this.excludeL2TransactionTypes ), this.preprocessedStateDetailsRepository.findByStateUpdateId( stateUpdateId @@ -166,7 +173,8 @@ export class StateUpdateController { ...pagination, total: preprocessedStateDetails?.l2TransactionsStatistics ? sumUpTransactionCount( - preprocessedStateDetails.l2TransactionsStatistics + preprocessedStateDetails.l2TransactionsStatistics, + this.excludeL2TransactionTypes ) : 'processing', }) diff --git a/packages/backend/src/api/controllers/UserController.ts b/packages/backend/src/api/controllers/UserController.ts index 0dba08105..335339d58 100644 --- a/packages/backend/src/api/controllers/UserController.ts +++ b/packages/backend/src/api/controllers/UserController.ts @@ -19,6 +19,7 @@ import { } from '@explorer/shared' import { AssetHash, AssetId, EthereumAddress, StarkKey } from '@explorer/types' +import { L2TransactionTypesToExclude } from '../../config/starkex/StarkexConfig' import { AssetDetailsMap } from '../../core/AssetDetailsMap' import { AssetDetailsService } from '../../core/AssetDetailsService' import { ForcedTradeOfferViewService } from '../../core/ForcedTradeOfferViewService' @@ -63,6 +64,9 @@ export class UserController { private readonly withdrawableAssetRepository: WithdrawableAssetRepository, private readonly preprocessedUserStatisticsRepository: PreprocessedUserStatisticsRepository, private readonly preprocessedUserL2TransactionsStatisticsRepository: PreprocessedUserL2TransactionsStatisticsRepository, + private readonly excludeL2TransactionTypes: + | L2TransactionTypesToExclude + | undefined, private readonly exchangeAddress: EthereumAddress ) {} @@ -151,7 +155,8 @@ export class UserController { ), this.l2TransactionRepository.getUserSpecificPaginated( starkKey, - paginationOpts + paginationOpts, + this.excludeL2TransactionTypes ), this.preprocessedUserL2TransactionsStatisticsRepository.findLatestByStarkKey( starkKey @@ -218,8 +223,13 @@ export class UserController { const totalL2Transactions = sumUpTransactionCount( - preprocessedUserL2TransactionsStatistics?.cumulativeL2TransactionsStatistics - ) + sumUpTransactionCount(liveL2TransactionStatistics) + preprocessedUserL2TransactionsStatistics?.cumulativeL2TransactionsStatistics, + this.excludeL2TransactionTypes + ) + + sumUpTransactionCount( + liveL2TransactionStatistics, + this.excludeL2TransactionTypes + ) const content = renderUserPage({ context, @@ -330,7 +340,8 @@ export class UserController { ] = await Promise.all([ this.l2TransactionRepository.getUserSpecificPaginated( starkKey, - pagination + pagination, + this.excludeL2TransactionTypes ), this.preprocessedUserL2TransactionsStatisticsRepository.findLatestByStarkKey( starkKey @@ -340,8 +351,13 @@ export class UserController { const totalL2Transactions = sumUpTransactionCount( - preprocessedUserL2TransactionsStatistics?.cumulativeL2TransactionsStatistics - ) + sumUpTransactionCount(liveL2TransactionStatistics) + preprocessedUserL2TransactionsStatistics?.cumulativeL2TransactionsStatistics, + this.excludeL2TransactionTypes + ) + + sumUpTransactionCount( + liveL2TransactionStatistics, + this.excludeL2TransactionTypes + ) const content = renderUserL2TransactionsPage({ context, diff --git a/packages/backend/src/api/routers/FrontendRouter.ts b/packages/backend/src/api/routers/FrontendRouter.ts index ae5b4a2e9..814e2ac18 100644 --- a/packages/backend/src/api/routers/FrontendRouter.ts +++ b/packages/backend/src/api/routers/FrontendRouter.ts @@ -366,7 +366,7 @@ export function createFrontendRouter( ) ) - if (config.starkex.enableL2Transactions) { + if (config.starkex.l2Transactions.enabled) { router.get( '/l2-transactions', withTypedContext( diff --git a/packages/backend/src/api/routers/PerpetualFrontendRouter.ts b/packages/backend/src/api/routers/PerpetualFrontendRouter.ts index a19b1dde3..85e1b47d8 100644 --- a/packages/backend/src/api/routers/PerpetualFrontendRouter.ts +++ b/packages/backend/src/api/routers/PerpetualFrontendRouter.ts @@ -67,7 +67,7 @@ export function addPerpetualTradingRoutes( ) ) - if (config.starkex.enableL2Transactions) { + if (config.starkex.l2Transactions.enabled) { router.get( '/l2-transactions/:transactionId{/:multiIndex}?', withTypedContext( diff --git a/packages/backend/src/config/starkex/StarkexConfig.ts b/packages/backend/src/config/starkex/StarkexConfig.ts index 9e424c193..130168ac3 100644 --- a/packages/backend/src/config/starkex/StarkexConfig.ts +++ b/packages/backend/src/config/starkex/StarkexConfig.ts @@ -1,4 +1,9 @@ -import { CollateralAsset, InstanceName, TradingMode } from '@explorer/shared' +import { + CollateralAsset, + InstanceName, + PerpetualL2TransactionData, + TradingMode, +} from '@explorer/shared' import { EthereumAddress } from '@explorer/types' export type StarkexConfig = @@ -8,12 +13,13 @@ export type StarkexConfig = ? SpotValidiumConfig : PerpetualRollupConfig | PerpetualValidiumConfig | SpotValidiumConfig -type BaseConfig = { +interface BaseConfig { instanceName: InstanceName dataAvailabilityMode: D tradingMode: T blockchain: BlockchainConfig -} & L2TransactionsConfig + l2Transactions: L2TransactionsConfig +} export type PerpetualRollupConfig = BaseConfig<'perpetual', 'rollup'> & { contracts: { @@ -49,17 +55,25 @@ export interface BlockchainConfig { maxBlockNumber: number } -export type L2TransactionsConfig = +export type L2TransactionTypesToExclude = Exclude< + PerpetualL2TransactionData['type'], + 'MultiTransaction' +>[] + +export type L2TransactionsConfig = { + excludeTypes?: L2TransactionTypesToExclude +} & ( | { - enableL2Transactions: true + enabled: true feederGateway: GatewayConfig - l2TransactionApi: LiveL2TransactionApiConfig | undefined + liveApi: LiveL2TransactionApiConfig | undefined } | { - enableL2Transactions: false + enabled: false feederGateway?: GatewayConfig - l2TransactionApi?: LiveL2TransactionApiConfig + liveApi?: LiveL2TransactionApiConfig } +) export type ClientAuth = | { diff --git a/packages/backend/src/config/starkex/apex-goerli.ts b/packages/backend/src/config/starkex/apex-goerli.ts index a4c61825b..b7ebb6353 100644 --- a/packages/backend/src/config/starkex/apex-goerli.ts +++ b/packages/backend/src/config/starkex/apex-goerli.ts @@ -30,25 +30,28 @@ export function getApexGoerliConfig(env: Env): StarkexConfig { }, auth: clientAuth, }, - enableL2Transactions: true, - feederGateway: { - getUrl: (batchId: number) => { - return `${env.string('APEX_FG_URL')}?batchId=${batchId}` + l2Transactions: { + enabled: true, + excludeTypes: ['OraclePricesTick'], + feederGateway: { + getUrl: (batchId: number) => { + return `${env.string('APEX_FG_URL')}?batchId=${batchId}` + }, + auth: clientAuth, }, - auth: clientAuth, - }, - l2TransactionApi: { - getTransactionsUrl: (startId, expectCount) => { - return `${env.string( - 'APEX_TRANSACTION_API_URL' - )}?startApexId=${startId}&expectCount=${expectCount}` + liveApi: { + getTransactionsUrl: (startId, expectCount) => { + return `${env.string( + 'APEX_TRANSACTION_API_URL' + )}?startApexId=${startId}&expectCount=${expectCount}` + }, + getThirdPartyIdByTransactionIdUrl: (transactionId: number) => { + return `${env.string( + 'APEX_THIRD_PARTY_ID_API_URL' + )}?txId=${transactionId}` + }, + auth: clientAuth, }, - getThirdPartyIdByTransactionIdUrl: (transactionId: number) => { - return `${env.string( - 'APEX_THIRD_PARTY_ID_API_URL' - )}?txId=${transactionId}` - }, - auth: clientAuth, }, collateralAsset: { assetId: AssetId('SLF-6'), diff --git a/packages/backend/src/config/starkex/apex-mainnet.ts b/packages/backend/src/config/starkex/apex-mainnet.ts index 818fa2d47..766570285 100644 --- a/packages/backend/src/config/starkex/apex-mainnet.ts +++ b/packages/backend/src/config/starkex/apex-mainnet.ts @@ -30,26 +30,30 @@ export function getApexMainnetConfig(env: Env): StarkexConfig { }, auth: clientAuth, }, - enableL2Transactions: true, - feederGateway: { - getUrl: (batchId: number) => { - return `${env.string('APEX_FG_URL')}?batchId=${batchId}` - }, - auth: clientAuth, - }, - l2TransactionApi: { - getTransactionsUrl: (startId, expectCount) => { - return `${env.string( - 'APEX_TRANSACTION_API_URL' - )}?startApexId=${startId}&expectCount=${expectCount}` + l2Transactions: { + enabled: true, + excludeTypes: ['OraclePricesTick'], + feederGateway: { + getUrl: (batchId: number) => { + return `${env.string('APEX_FG_URL')}?batchId=${batchId}` + }, + auth: clientAuth, }, - getThirdPartyIdByTransactionIdUrl: (transactionId) => { - return `${env.string( - 'APEX_THIRD_PARTY_ID_API_URL' - )}?txId=${transactionId}` + liveApi: { + getTransactionsUrl: (startId, expectCount) => { + return `${env.string( + 'APEX_TRANSACTION_API_URL' + )}?startApexId=${startId}&expectCount=${expectCount}` + }, + getThirdPartyIdByTransactionIdUrl: (transactionId) => { + return `${env.string( + 'APEX_THIRD_PARTY_ID_API_URL' + )}?txId=${transactionId}` + }, + auth: clientAuth, }, - auth: clientAuth, }, + collateralAsset: { assetId: AssetId('USDC-6'), assetHash: AssetHash( diff --git a/packages/backend/src/config/starkex/dydx-local.ts b/packages/backend/src/config/starkex/dydx-local.ts index fe1ca09a3..a062a069e 100644 --- a/packages/backend/src/config/starkex/dydx-local.ts +++ b/packages/backend/src/config/starkex/dydx-local.ts @@ -16,7 +16,7 @@ export function getDydxLocalConfig(env: Env): StarkexConfig { minBlockNumber: 0, maxBlockNumber: env.integer('MAX_BLOCK_NUMBER', Infinity), }, - enableL2Transactions: false, + l2Transactions: { enabled: false }, contracts: { perpetual: EthereumAddress('0x27fac828D6E6862901ea8471fF22552D84e155D0'), registry: EthereumAddress('0xE068d37a67cAb19e0A6DFE88e720f076cfA7140E'), diff --git a/packages/backend/src/config/starkex/dydx-mainnet.ts b/packages/backend/src/config/starkex/dydx-mainnet.ts index f72423d18..94fce6984 100644 --- a/packages/backend/src/config/starkex/dydx-mainnet.ts +++ b/packages/backend/src/config/starkex/dydx-mainnet.ts @@ -16,7 +16,7 @@ export function getDydxMainnetConfig(env: Env): StarkexConfig { minBlockNumber: 11813207, maxBlockNumber: env.integer('MAX_BLOCK_NUMBER', Infinity), }, - enableL2Transactions: false, + l2Transactions: { enabled: false }, contracts: { perpetual: EthereumAddress('0xD54f502e184B6B739d7D27a6410a67dc462D69c8'), registry: EthereumAddress('0xEfbCcE4659db72eC6897F46783303708cf9ACef8'), diff --git a/packages/backend/src/config/starkex/gammax-goerli.ts b/packages/backend/src/config/starkex/gammax-goerli.ts index 077bc2bc0..de43f1978 100644 --- a/packages/backend/src/config/starkex/gammax-goerli.ts +++ b/packages/backend/src/config/starkex/gammax-goerli.ts @@ -19,7 +19,9 @@ export function getGammaxGoerliConfig(env: Env): StarkexConfig { contracts: { perpetual: EthereumAddress('0x6E5de338D71af33B57831C5552775f54394d181B'), }, - enableL2Transactions: false, + l2Transactions: { + enabled: false, + }, availabilityGateway: { getUrl: (batchId: number) => { return `${env.string('GAMMAX_AG_URL')}?batch_id=${batchId}` diff --git a/packages/backend/src/config/starkex/myria-goerli.ts b/packages/backend/src/config/starkex/myria-goerli.ts index cce5d2d19..b47ed6480 100644 --- a/packages/backend/src/config/starkex/myria-goerli.ts +++ b/packages/backend/src/config/starkex/myria-goerli.ts @@ -27,7 +27,7 @@ export function getMyriaGoerliConfig(env: Env): StarkexConfig { userKey: env.string('MYRIA_AG_USER_KEY'), }, }, - enableL2Transactions: false, + l2Transactions: { enabled: false }, contracts: { perpetual: EthereumAddress('0xF82C423a30E317f34f9b0997627F2F9c5d239Ad9'), }, diff --git a/packages/backend/src/core/PageContextService.test.ts b/packages/backend/src/core/PageContextService.test.ts index 11bca5ca1..7f17ba4b9 100644 --- a/packages/backend/src/core/PageContextService.test.ts +++ b/packages/backend/src/core/PageContextService.test.ts @@ -45,7 +45,7 @@ describe(PageContextService.name, () => { expect(context).toEqual({ user: undefined, tradingMode: 'perpetual', - showL2Transactions: perpetualConfig.starkex.enableL2Transactions, + showL2Transactions: perpetualConfig.starkex.l2Transactions.enabled, chainId: 1, instanceName: perpetualConfig.starkex.instanceName, collateralAsset: fakeCollateralAsset, @@ -70,7 +70,7 @@ describe(PageContextService.name, () => { expect(context).toEqual({ user: undefined, tradingMode: 'spot', - showL2Transactions: spotConfig.starkex.enableL2Transactions, + showL2Transactions: spotConfig.starkex.l2Transactions.enabled, chainId: 5, instanceName: spotConfig.starkex.instanceName, }) diff --git a/packages/backend/src/core/PageContextService.ts b/packages/backend/src/core/PageContextService.ts index fc5ea3136..4a2d67d07 100644 --- a/packages/backend/src/core/PageContextService.ts +++ b/packages/backend/src/core/PageContextService.ts @@ -25,7 +25,7 @@ export class PageContextService { instanceName: this.config.starkex.instanceName, chainId: this.config.starkex.blockchain.chainId, collateralAsset: this.config.starkex.collateralAsset, - showL2Transactions: this.config.starkex.enableL2Transactions, + showL2Transactions: this.config.starkex.l2Transactions.enabled, } } @@ -34,7 +34,7 @@ export class PageContextService { tradingMode: this.config.starkex.tradingMode, chainId: this.config.starkex.blockchain.chainId, instanceName: this.config.starkex.instanceName, - showL2Transactions: this.config.starkex.enableL2Transactions, + showL2Transactions: this.config.starkex.l2Transactions.enabled, } } diff --git a/packages/backend/src/peripherals/database/L2TransactionRepository.test.ts b/packages/backend/src/peripherals/database/L2TransactionRepository.test.ts index fc5b64d4c..a649bc757 100644 --- a/packages/backend/src/peripherals/database/L2TransactionRepository.test.ts +++ b/packages/backend/src/peripherals/database/L2TransactionRepository.test.ts @@ -12,8 +12,10 @@ import { } from '@explorer/types' import { Logger } from '@l2beat/backend-tools' import { expect } from 'earl' +import { uniq } from 'lodash' import { beforeEach, it } from 'mocha' +import { L2TransactionTypesToExclude } from '../../config/starkex/StarkexConfig' import { setupDatabaseTestSuite } from '../../test/database' import { L2TransactionRepository } from './L2TransactionRepository' @@ -478,6 +480,41 @@ describe(L2TransactionRepository.name, () => { expect(records.map((x) => x.id)).toEqual(ids.reverse().slice(0, 5)) }) + + it('filters out transactions with excluded types', async () => { + const ids = [] + const excludedTypes = ['Deposit'] as L2TransactionTypesToExclude + + for (let i = 0; i < 10; i++) { + ids.push( + await repository.add({ + ...genericDepositTransaction, + transactionId: 1234 + i, + }) + ) + } + const multiTransaction = genericMultiTransaction([ + genericDepositTransaction.data, + genericWithdrawalToAddressTransaction.data, + ]) + const multiId = await repository.add(multiTransaction) + multiTransaction.data.transactions.forEach((_, index) => + ids.push(multiId + index + 1) + ) + + const records = await repository.getPaginatedWithoutMulti( + { + limit: 100, + offset: 0, + }, + excludedTypes + ) + + expect(records.map((x) => x.id)).toEqual(ids.reverse().slice(0, 1)) + expect(uniq(records.map((t) => t.data.type))).not.toEqualUnsorted( + excludedTypes + ) + }) } ) @@ -561,22 +598,49 @@ describe(L2TransactionRepository.name, () => { }) it('respects the limit parameter', async () => { - const records = await repository.getUserSpecificPaginated(starkKey, { - limit: 6, - offset: 0, - }) + const records = await repository.getUserSpecificPaginated( + starkKey, + + { + limit: 6, + offset: 0, + } + ) expect(records.map((x) => x.id)).toEqual(ids.slice(4, 10).reverse()) }) it('respects the offset parameter', async () => { - const records = await repository.getUserSpecificPaginated(starkKey, { - limit: 6, - offset: 2, - }) + const records = await repository.getUserSpecificPaginated( + starkKey, + + { + limit: 6, + offset: 2, + } + ) expect(records.map((x) => x.id)).toEqual(ids.slice(2, 8).reverse()) }) + + it('filters out transactions with excluded types', async () => { + const excludedTypes = ['Transfer'] as L2TransactionTypesToExclude + + const records = await repository.getUserSpecificPaginated( + starkKey, + + { + limit: 100, + offset: 0, + }, + excludedTypes + ) + + expect(records.map((x) => x.id)).toEqual(ids.slice(0, 5).reverse()) + expect(uniq(records.map((t) => t.data.type))).not.toEqualUnsorted( + excludedTypes + ) + }) } ) @@ -659,6 +723,49 @@ describe(L2TransactionRepository.name, () => { ...ids.slice(12, 15), ]) }) + + it('filters out transactions with excluded types', async () => { + const ids = [] + const excludedTypes = ['Deposit'] as L2TransactionTypesToExclude + + for (let i = 0; i < 20; i++) { + ids.push( + await repository.add({ + ...genericDepositTransaction, + transactionId: 1234 + i, + stateUpdateId: i < 10 ? 1 : 2, + }) + ) + } + const multiTransaction = { + ...genericMultiTransaction([ + genericDepositTransaction.data, + genericWithdrawalToAddressTransaction.data, + ]), + transactionId: 1254, + } + const multiId = await repository.add(multiTransaction) + multiTransaction.data.transactions.forEach((_, index) => + ids.push(multiId + index + 1) + ) + + const records = + await repository.getPaginatedWithoutMultiByStateUpdateId( + 1, + + { + limit: 100, + offset: 0, + }, + excludedTypes + ) + + ids.reverse() + expect(records.map((x) => x.id)).toEqual(ids.slice(0, 1)) + expect(uniq(records.map((t) => t.data.type))).not.toEqualUnsorted( + excludedTypes + ) + }) } ) diff --git a/packages/backend/src/peripherals/database/L2TransactionRepository.ts b/packages/backend/src/peripherals/database/L2TransactionRepository.ts index 138f16e1e..6c8299e33 100644 --- a/packages/backend/src/peripherals/database/L2TransactionRepository.ts +++ b/packages/backend/src/peripherals/database/L2TransactionRepository.ts @@ -8,6 +8,7 @@ import { Knex } from 'knex' import { L2TransactionRow } from 'knex/types/tables' import uniq from 'lodash/uniq' +import { L2TransactionTypesToExclude } from '../../config/starkex/StarkexConfig' import { PaginationOptions } from '../../model/PaginationOptions' import { decodeL2TransactionData, @@ -368,11 +369,15 @@ export class L2TransactionRepository extends BaseRepository { ) } - async getPaginatedWithoutMulti({ offset, limit }: PaginationOptions) { + async getPaginatedWithoutMulti( + { offset, limit }: PaginationOptions, + excludeL2TransactionTypes: L2TransactionTypesToExclude = [] + ) { const knex = await this.knex() const rows = await knex('l2_transactions') + .whereNotIn('type', excludeL2TransactionTypes) // We filter out the multi transactions because we show the child transactions instead - .whereNot({ type: 'MultiTransaction' }) + .andWhereNot({ type: 'MultiTransaction' }) .orderBy('id', 'desc') .offset(offset) .limit(limit) @@ -382,17 +387,23 @@ export class L2TransactionRepository extends BaseRepository { async getUserSpecificPaginated( starkKey: StarkKey, - { offset, limit }: PaginationOptions + { offset, limit }: PaginationOptions, + excludeL2TransactionTypes: L2TransactionTypesToExclude = [] ) { const knex = await this.knex() // We do not need to filter multi transactions because they are not user specific const rows = await knex('l2_transactions') - .where({ - stark_key_a: starkKey.toString(), - }) - .orWhere({ - stark_key_b: starkKey.toString(), - }) + .whereNotIn('type', excludeL2TransactionTypes) + // eslint-disable-next-line @typescript-eslint/no-misused-promises + .andWhere((qB) => + qB + .where({ + stark_key_a: starkKey.toString(), + }) + .orWhere({ + stark_key_b: starkKey.toString(), + }) + ) .orderBy('id', 'desc') .limit(limit) .offset(offset) @@ -401,11 +412,13 @@ export class L2TransactionRepository extends BaseRepository { async getPaginatedWithoutMultiByStateUpdateId( stateUpdateId: number, - { offset, limit }: PaginationOptions + { offset, limit }: PaginationOptions, + excludeL2TransactionTypes: L2TransactionTypesToExclude = [] ) { const knex = await this.knex() const rows = await knex('l2_transactions') - .where({ state_update_id: stateUpdateId }) + .whereNotIn('type', excludeL2TransactionTypes) + .andWhere({ state_update_id: stateUpdateId }) // We filter out the multi transactions because we show the child transactions instead .andWhereNot({ type: 'MultiTransaction' }) .orderBy('id', 'desc') diff --git a/packages/backend/src/peripherals/database/PreprocessedL2TransactionsStatistics.test.ts b/packages/backend/src/peripherals/database/PreprocessedL2TransactionsStatistics.test.ts index 511ca1b59..0dc676684 100644 --- a/packages/backend/src/peripherals/database/PreprocessedL2TransactionsStatistics.test.ts +++ b/packages/backend/src/peripherals/database/PreprocessedL2TransactionsStatistics.test.ts @@ -48,4 +48,26 @@ describe(sumUpTransactionCount.name, () => { expect(result).toEqual(66) }) + + it('sums up all the values for PreprocessedUserL2TransactionsStatistics with excluded types', () => { + const result = sumUpTransactionCount( + { + depositCount: 1, + withdrawalToAddressCount: 2, + forcedWithdrawalCount: 3, + tradeCount: 4, + forcedTradeCount: 5, + transferCount: 6, + conditionalTransferCount: 7, + liquidateCount: 8, + deleverageCount: 9, + fundingTickCount: 10, + oraclePricesTickCount: 11, + replacedTransactionsCount: 13, + }, + ['Deposit', 'WithdrawalToAddress', 'ForcedWithdrawal'] + ) + + expect(result).toEqual(66 - 1 - 2 - 3) + }) }) diff --git a/packages/backend/src/peripherals/database/PreprocessedL2TransactionsStatistics.ts b/packages/backend/src/peripherals/database/PreprocessedL2TransactionsStatistics.ts index 9e4e11a06..4c7453f94 100644 --- a/packages/backend/src/peripherals/database/PreprocessedL2TransactionsStatistics.ts +++ b/packages/backend/src/peripherals/database/PreprocessedL2TransactionsStatistics.ts @@ -1,5 +1,8 @@ import { PerpetualL2TransactionData } from '@explorer/shared' +import { L2TransactionTypesToExclude } from '../../config/starkex/StarkexConfig' +import { uncapitalize } from '../../utils/uncapitalize' + type PreprocessedL2TransactionsStatisticsKeys = `${Uncapitalize< PerpetualL2TransactionData['type'] >}Count` @@ -20,7 +23,8 @@ export function sumUpTransactionCount( statistics: | PreprocessedL2TransactionsStatistics | PreprocessedUserL2TransactionsStatistics - | undefined + | undefined, + excludeL2TransactionTypes: L2TransactionTypesToExclude = [] ) { if (!statistics) return 0 @@ -30,12 +34,16 @@ export function sumUpTransactionCount( ? statistics.multiTransactionCount : 0 - const replacedAndMultiTransactionCount = + let initialValue = multiTransactionCount + statistics.replacedTransactionsCount + for (const type of excludeL2TransactionTypes) { + initialValue += statistics[`${uncapitalize(type)}Count`] + } + return Object.values(statistics).reduce( (sum, value) => sum + value, - -replacedAndMultiTransactionCount + -initialValue ) } diff --git a/packages/backend/src/utils/uncapitalize.ts b/packages/backend/src/utils/uncapitalize.ts new file mode 100644 index 000000000..d24834aa3 --- /dev/null +++ b/packages/backend/src/utils/uncapitalize.ts @@ -0,0 +1,3 @@ +export function uncapitalize(str: T): Uncapitalize { + return (str.charAt(0).toLowerCase() + str.slice(1)) as Uncapitalize +}